diff --git a/media/libjxl/moz.build b/media/libjxl/moz.build --- a/media/libjxl/moz.build +++ b/media/libjxl/moz.build @@ -17,11 +17,10 @@ "/third_party/jpeg-xl/lib/jxl/aux_out.cc", "/third_party/jpeg-xl/lib/jxl/base/cache_aligned.cc", "/third_party/jpeg-xl/lib/jxl/base/data_parallel.cc", "/third_party/jpeg-xl/lib/jxl/base/padded_bytes.cc", "/third_party/jpeg-xl/lib/jxl/base/random.cc", - "/third_party/jpeg-xl/lib/jxl/base/status.cc", "/third_party/jpeg-xl/lib/jxl/blending.cc", "/third_party/jpeg-xl/lib/jxl/box_content_decoder.cc", "/third_party/jpeg-xl/lib/jxl/chroma_from_luma.cc", "/third_party/jpeg-xl/lib/jxl/coeff_order.cc", "/third_party/jpeg-xl/lib/jxl/color_encoding_internal.cc", @@ -44,10 +43,11 @@ "/third_party/jpeg-xl/lib/jxl/decode.cc", "/third_party/jpeg-xl/lib/jxl/decode_to_jpeg.cc", "/third_party/jpeg-xl/lib/jxl/enc_bit_writer.cc", "/third_party/jpeg-xl/lib/jxl/entropy_coder.cc", "/third_party/jpeg-xl/lib/jxl/epf.cc", + "/third_party/jpeg-xl/lib/jxl/exif.cc", "/third_party/jpeg-xl/lib/jxl/fast_dct.cc", "/third_party/jpeg-xl/lib/jxl/fields.cc", "/third_party/jpeg-xl/lib/jxl/frame_header.cc", "/third_party/jpeg-xl/lib/jxl/gauss_blur.cc", "/third_party/jpeg-xl/lib/jxl/headers.cc", diff --git a/media/libjxl/moz.yaml b/media/libjxl/moz.yaml --- a/media/libjxl/moz.yaml +++ b/media/libjxl/moz.yaml @@ -8,13 +8,13 @@ name: libjxl description: JPEG XL image format reference implementation url: https://github.com/libjxl/libjxl - release: commit 318c592d98b97d103941b90d47107f06a10c71da (2022-03-21T20:44:46Z). + release: 4da1c812f47ea4ca3a09ca135099e494e85e9656 (2022-04-26T11:25:32Z). - revision: 318c592d98b97d103941b90d47107f06a10c71da + revision: 4da1c812f47ea4ca3a09ca135099e494e85e9656 license: Apache-2.0 license-file: LICENSE diff --git a/media/libvpx/config/generic/vp8_rtcd.h b/media/libvpx/config/generic/vp8_rtcd.h --- a/media/libvpx/config/generic/vp8_rtcd.h +++ b/media/libvpx/config/generic/vp8_rtcd.h @@ -79,13 +79,10 @@ #define vp8_diamond_search_sad vp8_diamond_search_sad_c void vp8_fast_quantize_b_c(struct block *, struct blockd *); #define vp8_fast_quantize_b vp8_fast_quantize_b_c -int vp8_full_search_sad_c(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -#define vp8_full_search_sad vp8_full_search_sad_c - void vp8_loop_filter_bh_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); #define vp8_loop_filter_bh vp8_loop_filter_bh_c void vp8_loop_filter_bv_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); #define vp8_loop_filter_bv vp8_loop_filter_bv_c diff --git a/media/libvpx/config/generic/vp9_rtcd.h b/media/libvpx/config/generic/vp9_rtcd.h --- a/media/libvpx/config/generic/vp9_rtcd.h +++ b/media/libvpx/config/generic/vp9_rtcd.h @@ -62,14 +62,14 @@ #define vp9_iht4x4_16_add vp9_iht4x4_16_add_c void vp9_iht8x8_64_add_c(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); #define vp9_iht8x8_64_add vp9_iht8x8_64_add_c -void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); #define vp9_quantize_fp vp9_quantize_fp_c -void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); #define vp9_quantize_fp_32x32 vp9_quantize_fp_32x32_c void vp9_scale_and_extend_frame_c(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); #define vp9_scale_and_extend_frame vp9_scale_and_extend_frame_c diff --git a/media/libvpx/config/generic/vpx_config.h b/media/libvpx/config/generic/vpx_config.h --- a/media/libvpx/config/generic/vpx_config.h +++ b/media/libvpx/config/generic/vpx_config.h @@ -13,10 +13,11 @@ #define VPX_ARCH_ARM 0 #define VPX_ARCH_MIPS 0 #define VPX_ARCH_X86 0 #define VPX_ARCH_X86_64 0 #define VPX_ARCH_PPC 0 +#define VPX_ARCH_LOONGARCH 0 #define HAVE_NEON 0 #define HAVE_NEON_ASM 0 #define HAVE_MIPS32 0 #define HAVE_DSPR2 0 #define HAVE_MSA 0 @@ -30,10 +31,12 @@ #define HAVE_AVX 0 #define HAVE_AVX2 0 #define HAVE_AVX512 0 #define HAVE_VSX 0 #define HAVE_MMI 0 +#define HAVE_LSX 0 +#define HAVE_LASX 0 #define HAVE_VPX_PORTS 1 #define HAVE_PTHREAD_H 1 #define CONFIG_DEPENDENCY_TRACKING 1 #define CONFIG_EXTERNAL_BUILD 1 #define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/generic/vpx_config.asm b/media/libvpx/config/generic/vpx_config.asm --- a/media/libvpx/config/generic/vpx_config.asm +++ b/media/libvpx/config/generic/vpx_config.asm @@ -1,13 +1,14 @@ @ This file was created from a .asm file @ using the ads2gas.pl script. - .syntax unified +.syntax unified .equ VPX_ARCH_ARM , 0 .equ VPX_ARCH_MIPS , 0 .equ VPX_ARCH_X86 , 0 .equ VPX_ARCH_X86_64 , 0 .equ VPX_ARCH_PPC , 0 +.equ VPX_ARCH_LOONGARCH , 0 .equ HAVE_NEON , 0 .equ HAVE_NEON_ASM , 0 .equ HAVE_MIPS32 , 0 .equ HAVE_DSPR2 , 0 .equ HAVE_MSA , 0 @@ -21,10 +22,12 @@ .equ HAVE_AVX , 0 .equ HAVE_AVX2 , 0 .equ HAVE_AVX512 , 0 .equ HAVE_VSX , 0 .equ HAVE_MMI , 0 +.equ HAVE_LSX , 0 +.equ HAVE_LASX , 0 .equ HAVE_VPX_PORTS , 1 .equ HAVE_PTHREAD_H , 1 .equ CONFIG_DEPENDENCY_TRACKING , 1 .equ CONFIG_EXTERNAL_BUILD , 1 .equ CONFIG_INSTALL_DOCS , 0 @@ -85,6 +88,6 @@ .equ CONFIG_MISMATCH_DEBUG , 0 .equ CONFIG_FP_MB_STATS , 0 .equ CONFIG_EMULATE_HARDWARE , 0 .equ CONFIG_NON_GREEDY_MV , 0 .equ CONFIG_RATE_CTRL , 0 - .section .note.GNU-stack,"",%progbits + .section .note.GNU-stack,"",%progbits diff --git a/media/libvpx/config/generic/vpx_dsp_rtcd.h b/media/libvpx/config/generic/vpx_dsp_rtcd.h --- a/media/libvpx/config/generic/vpx_dsp_rtcd.h +++ b/media/libvpx/config/generic/vpx_dsp_rtcd.h @@ -283,11 +283,11 @@ #define vpx_idct8x8_64_add vpx_idct8x8_64_add_c int16_t vpx_int_pro_col_c(const uint8_t *ref, const int width); #define vpx_int_pro_col vpx_int_pro_col_c -void vpx_int_pro_row_c(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_c(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); #define vpx_int_pro_row vpx_int_pro_row_c void vpx_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int stride); #define vpx_iwht4x4_16_add vpx_iwht4x4_16_add_c @@ -343,166 +343,133 @@ #define vpx_mse8x16 vpx_mse8x16_c unsigned int vpx_mse8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse); #define vpx_mse8x8 vpx_mse8x8_c -void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); #define vpx_quantize_b vpx_quantize_b_c -void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); #define vpx_quantize_b_32x32 vpx_quantize_b_32x32_c unsigned int vpx_sad16x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x16 vpx_sad16x16_c unsigned int vpx_sad16x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x16_avg vpx_sad16x16_avg_c -void vpx_sad16x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x16x3 vpx_sad16x16x3_c - -void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x16x4d vpx_sad16x16x4d_c -void vpx_sad16x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x16x8 vpx_sad16x16x8_c - unsigned int vpx_sad16x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x32 vpx_sad16x32_c unsigned int vpx_sad16x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x32_avg vpx_sad16x32_avg_c -void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x32x4d vpx_sad16x32x4d_c unsigned int vpx_sad16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x8 vpx_sad16x8_c unsigned int vpx_sad16x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x8_avg vpx_sad16x8_avg_c -void vpx_sad16x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x8x3 vpx_sad16x8x3_c - -void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x8x4d vpx_sad16x8x4d_c -void vpx_sad16x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x8x8 vpx_sad16x8x8_c - unsigned int vpx_sad32x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad32x16 vpx_sad32x16_c unsigned int vpx_sad32x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad32x16_avg vpx_sad32x16_avg_c -void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad32x16x4d vpx_sad32x16x4d_c unsigned int vpx_sad32x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad32x32 vpx_sad32x32_c unsigned int vpx_sad32x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad32x32_avg vpx_sad32x32_avg_c -void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad32x32x4d vpx_sad32x32x4d_c -void vpx_sad32x32x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad32x32x8 vpx_sad32x32x8_c - unsigned int vpx_sad32x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad32x64 vpx_sad32x64_c unsigned int vpx_sad32x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad32x64_avg vpx_sad32x64_avg_c -void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad32x64x4d vpx_sad32x64x4d_c unsigned int vpx_sad4x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad4x4 vpx_sad4x4_c unsigned int vpx_sad4x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad4x4_avg vpx_sad4x4_avg_c -void vpx_sad4x4x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad4x4x3 vpx_sad4x4x3_c - -void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad4x4x4d vpx_sad4x4x4d_c -void vpx_sad4x4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad4x4x8 vpx_sad4x4x8_c - unsigned int vpx_sad4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad4x8 vpx_sad4x8_c unsigned int vpx_sad4x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad4x8_avg vpx_sad4x8_avg_c -void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad4x8x4d vpx_sad4x8x4d_c unsigned int vpx_sad64x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad64x32 vpx_sad64x32_c unsigned int vpx_sad64x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad64x32_avg vpx_sad64x32_avg_c -void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad64x32x4d vpx_sad64x32x4d_c unsigned int vpx_sad64x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad64x64 vpx_sad64x64_c unsigned int vpx_sad64x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad64x64_avg vpx_sad64x64_avg_c -void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad64x64x4d vpx_sad64x64x4d_c unsigned int vpx_sad8x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x16 vpx_sad8x16_c unsigned int vpx_sad8x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x16_avg vpx_sad8x16_avg_c -void vpx_sad8x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x16x3 vpx_sad8x16x3_c - -void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x16x4d vpx_sad8x16x4d_c -void vpx_sad8x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x16x8 vpx_sad8x16x8_c - unsigned int vpx_sad8x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x4 vpx_sad8x4_c unsigned int vpx_sad8x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x4_avg vpx_sad8x4_avg_c -void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x4x4d vpx_sad8x4x4d_c unsigned int vpx_sad8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x8 vpx_sad8x8_c unsigned int vpx_sad8x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x8_avg vpx_sad8x8_avg_c -void vpx_sad8x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x8x3 vpx_sad8x8x3_c - -void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x8x4d vpx_sad8x8x4d_c -void vpx_sad8x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x8x8 vpx_sad8x8x8_c - int vpx_satd_c(const int16_t *coeff, int length); #define vpx_satd vpx_satd_c void vpx_scaled_2d_c(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h); #define vpx_scaled_2d vpx_scaled_2d_c diff --git a/media/libvpx/config/linux/arm/vp8_rtcd.h b/media/libvpx/config/linux/arm/vp8_rtcd.h --- a/media/libvpx/config/linux/arm/vp8_rtcd.h +++ b/media/libvpx/config/linux/arm/vp8_rtcd.h @@ -94,13 +94,10 @@ void vp8_fast_quantize_b_c(struct block *, struct blockd *); void vp8_fast_quantize_b_neon(struct block *, struct blockd *); RTCD_EXTERN void (*vp8_fast_quantize_b)(struct block *, struct blockd *); -int vp8_full_search_sad_c(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -#define vp8_full_search_sad vp8_full_search_sad_c - void vp8_loop_filter_bh_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); void vp8_loop_filter_bh_neon(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); RTCD_EXTERN void (*vp8_loop_filter_bh)(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); void vp8_loop_filter_bv_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); diff --git a/media/libvpx/config/linux/arm/vp9_rtcd.h b/media/libvpx/config/linux/arm/vp9_rtcd.h --- a/media/libvpx/config/linux/arm/vp9_rtcd.h +++ b/media/libvpx/config/linux/arm/vp9_rtcd.h @@ -40,17 +40,20 @@ int vp9_diamond_search_sad_c(const struct macroblock *x, const struct search_site_config *cfg, struct mv *ref_mv, struct mv *best_mv, int search_param, int sad_per_bit, int *num00, const struct vp9_variance_vtable *fn_ptr, const struct mv *center_mv); #define vp9_diamond_search_sad vp9_diamond_search_sad_c void vp9_fht16x16_c(const int16_t *input, tran_low_t *output, int stride, int tx_type); -#define vp9_fht16x16 vp9_fht16x16_c +void vp9_fht16x16_neon(const int16_t *input, tran_low_t *output, int stride, int tx_type); +RTCD_EXTERN void (*vp9_fht16x16)(const int16_t *input, tran_low_t *output, int stride, int tx_type); void vp9_fht4x4_c(const int16_t *input, tran_low_t *output, int stride, int tx_type); -#define vp9_fht4x4 vp9_fht4x4_c +void vp9_fht4x4_neon(const int16_t *input, tran_low_t *output, int stride, int tx_type); +RTCD_EXTERN void (*vp9_fht4x4)(const int16_t *input, tran_low_t *output, int stride, int tx_type); void vp9_fht8x8_c(const int16_t *input, tran_low_t *output, int stride, int tx_type); -#define vp9_fht8x8 vp9_fht8x8_c +void vp9_fht8x8_neon(const int16_t *input, tran_low_t *output, int stride, int tx_type); +RTCD_EXTERN void (*vp9_fht8x8)(const int16_t *input, tran_low_t *output, int stride, int tx_type); void vp9_fwht4x4_c(const int16_t *input, tran_low_t *output, int stride); #define vp9_fwht4x4 vp9_fwht4x4_c void vp9_iht16x16_256_add_c(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); @@ -63,17 +66,17 @@ void vp9_iht8x8_64_add_c(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); void vp9_iht8x8_64_add_neon(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); RTCD_EXTERN void (*vp9_iht8x8_64_add)(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); -void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vp9_quantize_fp)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vp9_quantize_fp)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vp9_quantize_fp_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vp9_quantize_fp_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); void vp9_scale_and_extend_frame_c(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); void vp9_scale_and_extend_frame_neon(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); RTCD_EXTERN void (*vp9_scale_and_extend_frame)(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); @@ -89,10 +92,16 @@ (void)flags; vp9_block_error_fp = vp9_block_error_fp_c; if (flags & HAS_NEON) vp9_block_error_fp = vp9_block_error_fp_neon; + vp9_fht16x16 = vp9_fht16x16_c; + if (flags & HAS_NEON) vp9_fht16x16 = vp9_fht16x16_neon; + vp9_fht4x4 = vp9_fht4x4_c; + if (flags & HAS_NEON) vp9_fht4x4 = vp9_fht4x4_neon; + vp9_fht8x8 = vp9_fht8x8_c; + if (flags & HAS_NEON) vp9_fht8x8 = vp9_fht8x8_neon; vp9_iht16x16_256_add = vp9_iht16x16_256_add_c; if (flags & HAS_NEON) vp9_iht16x16_256_add = vp9_iht16x16_256_add_neon; vp9_iht4x4_16_add = vp9_iht4x4_16_add_c; if (flags & HAS_NEON) vp9_iht4x4_16_add = vp9_iht4x4_16_add_neon; vp9_iht8x8_64_add = vp9_iht8x8_64_add_c; diff --git a/media/libvpx/config/linux/arm/vpx_config.h b/media/libvpx/config/linux/arm/vpx_config.h --- a/media/libvpx/config/linux/arm/vpx_config.h +++ b/media/libvpx/config/linux/arm/vpx_config.h @@ -13,10 +13,11 @@ #define VPX_ARCH_ARM 1 #define VPX_ARCH_MIPS 0 #define VPX_ARCH_X86 0 #define VPX_ARCH_X86_64 0 #define VPX_ARCH_PPC 0 +#define VPX_ARCH_LOONGARCH 0 #define HAVE_NEON 1 #define HAVE_NEON_ASM 1 #define HAVE_MIPS32 0 #define HAVE_DSPR2 0 #define HAVE_MSA 0 @@ -30,10 +31,12 @@ #define HAVE_AVX 0 #define HAVE_AVX2 0 #define HAVE_AVX512 0 #define HAVE_VSX 0 #define HAVE_MMI 0 +#define HAVE_LSX 0 +#define HAVE_LASX 0 #define HAVE_VPX_PORTS 1 #define HAVE_PTHREAD_H 1 #define CONFIG_DEPENDENCY_TRACKING 1 #define CONFIG_EXTERNAL_BUILD 1 #define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/linux/arm/vpx_config.asm b/media/libvpx/config/linux/arm/vpx_config.asm --- a/media/libvpx/config/linux/arm/vpx_config.asm +++ b/media/libvpx/config/linux/arm/vpx_config.asm @@ -1,13 +1,14 @@ @ This file was created from a .asm file @ using the ads2gas.pl script. - .syntax unified +.syntax unified .equ VPX_ARCH_ARM , 1 .equ VPX_ARCH_MIPS , 0 .equ VPX_ARCH_X86 , 0 .equ VPX_ARCH_X86_64 , 0 .equ VPX_ARCH_PPC , 0 +.equ VPX_ARCH_LOONGARCH , 0 .equ HAVE_NEON , 1 .equ HAVE_NEON_ASM , 1 .equ HAVE_MIPS32 , 0 .equ HAVE_DSPR2 , 0 .equ HAVE_MSA , 0 @@ -21,10 +22,12 @@ .equ HAVE_AVX , 0 .equ HAVE_AVX2 , 0 .equ HAVE_AVX512 , 0 .equ HAVE_VSX , 0 .equ HAVE_MMI , 0 +.equ HAVE_LSX , 0 +.equ HAVE_LASX , 0 .equ HAVE_VPX_PORTS , 1 .equ HAVE_PTHREAD_H , 1 .equ CONFIG_DEPENDENCY_TRACKING , 1 .equ CONFIG_EXTERNAL_BUILD , 1 .equ CONFIG_INSTALL_DOCS , 0 @@ -85,6 +88,6 @@ .equ CONFIG_MISMATCH_DEBUG , 0 .equ CONFIG_FP_MB_STATS , 0 .equ CONFIG_EMULATE_HARDWARE , 0 .equ CONFIG_NON_GREEDY_MV , 0 .equ CONFIG_RATE_CTRL , 0 - .section .note.GNU-stack,"",%progbits + .section .note.GNU-stack,"",%progbits diff --git a/media/libvpx/config/linux/arm/vpx_dsp_rtcd.h b/media/libvpx/config/linux/arm/vpx_dsp_rtcd.h --- a/media/libvpx/config/linux/arm/vpx_dsp_rtcd.h +++ b/media/libvpx/config/linux/arm/vpx_dsp_rtcd.h @@ -350,13 +350,13 @@ int16_t vpx_int_pro_col_c(const uint8_t *ref, const int width); int16_t vpx_int_pro_col_neon(const uint8_t *ref, const int width); RTCD_EXTERN int16_t (*vpx_int_pro_col)(const uint8_t *ref, const int width); -void vpx_int_pro_row_c(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); -void vpx_int_pro_row_neon(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); -RTCD_EXTERN void (*vpx_int_pro_row)(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_c(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_neon(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); +RTCD_EXTERN void (*vpx_int_pro_row)(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); void vpx_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int stride); #define vpx_iwht4x4_16_add vpx_iwht4x4_16_add_c void vpx_iwht4x4_1_add_c(const tran_low_t *input, uint8_t *dest, int stride); @@ -425,206 +425,173 @@ #define vpx_mse8x16 vpx_mse8x16_c unsigned int vpx_mse8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse); #define vpx_mse8x8 vpx_mse8x8_c -void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vpx_quantize_b)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vpx_quantize_b)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vpx_quantize_b_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vpx_quantize_b_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); unsigned int vpx_sad16x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x16_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad16x16)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x16_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad16x16_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad16x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x16x3 vpx_sad16x16x3_c - -void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad16x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x16x8 vpx_sad16x16x8_c +void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad16x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad16x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x32_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad16x32)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x32_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad16x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad16x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x8_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x8_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad16x8_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad16x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x8x3 vpx_sad16x8x3_c - -void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad16x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x8x8 vpx_sad16x8x8_c +void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad16x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad32x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x16)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x16_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x16_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad32x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad32x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x32)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x32_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad32x32x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad32x32x8 vpx_sad32x32x8_c +void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad32x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad32x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x64)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x64_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x64_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad32x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad4x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x4_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad4x4)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x4_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad4x4_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad4x4x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad4x4x3 vpx_sad4x4x3_c - -void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad4x4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad4x4x8 vpx_sad4x4x8_c +void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x4x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad4x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x8_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad4x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x8_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad4x8_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad4x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad64x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad64x32)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x32_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad64x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad64x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad64x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad64x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad64x64)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x64_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad64x64_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad64x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad64x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad8x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x16_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad8x16)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x16_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad8x16_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad8x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x16x3 vpx_sad8x16x3_c - -void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad8x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x16x8 vpx_sad8x16x8_c +void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad8x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad8x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x4_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad8x4)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x4_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad8x4_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x4x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x4x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad8x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x8_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x8_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad8x8_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad8x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x8x3 vpx_sad8x8x3_c - -void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad8x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x8x8 vpx_sad8x8x8_c +void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad8x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); int vpx_satd_c(const int16_t *coeff, int length); int vpx_satd_neon(const int16_t *coeff, int length); RTCD_EXTERN int (*vpx_satd)(const int16_t *coeff, int length); diff --git a/media/libvpx/config/linux/arm64/vp8_rtcd.h b/media/libvpx/config/linux/arm64/vp8_rtcd.h --- a/media/libvpx/config/linux/arm64/vp8_rtcd.h +++ b/media/libvpx/config/linux/arm64/vp8_rtcd.h @@ -94,13 +94,10 @@ void vp8_fast_quantize_b_c(struct block *, struct blockd *); void vp8_fast_quantize_b_neon(struct block *, struct blockd *); #define vp8_fast_quantize_b vp8_fast_quantize_b_neon -int vp8_full_search_sad_c(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -#define vp8_full_search_sad vp8_full_search_sad_c - void vp8_loop_filter_bh_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); void vp8_loop_filter_bh_neon(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); #define vp8_loop_filter_bh vp8_loop_filter_bh_neon void vp8_loop_filter_bv_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); diff --git a/media/libvpx/config/linux/arm64/vp9_rtcd.h b/media/libvpx/config/linux/arm64/vp9_rtcd.h --- a/media/libvpx/config/linux/arm64/vp9_rtcd.h +++ b/media/libvpx/config/linux/arm64/vp9_rtcd.h @@ -40,17 +40,20 @@ int vp9_diamond_search_sad_c(const struct macroblock *x, const struct search_site_config *cfg, struct mv *ref_mv, struct mv *best_mv, int search_param, int sad_per_bit, int *num00, const struct vp9_variance_vtable *fn_ptr, const struct mv *center_mv); #define vp9_diamond_search_sad vp9_diamond_search_sad_c void vp9_fht16x16_c(const int16_t *input, tran_low_t *output, int stride, int tx_type); -#define vp9_fht16x16 vp9_fht16x16_c +void vp9_fht16x16_neon(const int16_t *input, tran_low_t *output, int stride, int tx_type); +#define vp9_fht16x16 vp9_fht16x16_neon void vp9_fht4x4_c(const int16_t *input, tran_low_t *output, int stride, int tx_type); -#define vp9_fht4x4 vp9_fht4x4_c +void vp9_fht4x4_neon(const int16_t *input, tran_low_t *output, int stride, int tx_type); +#define vp9_fht4x4 vp9_fht4x4_neon void vp9_fht8x8_c(const int16_t *input, tran_low_t *output, int stride, int tx_type); -#define vp9_fht8x8 vp9_fht8x8_c +void vp9_fht8x8_neon(const int16_t *input, tran_low_t *output, int stride, int tx_type); +#define vp9_fht8x8 vp9_fht8x8_neon void vp9_fwht4x4_c(const int16_t *input, tran_low_t *output, int stride); #define vp9_fwht4x4 vp9_fwht4x4_c void vp9_iht16x16_256_add_c(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); @@ -63,16 +66,16 @@ void vp9_iht8x8_64_add_c(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); void vp9_iht8x8_64_add_neon(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); #define vp9_iht8x8_64_add vp9_iht8x8_64_add_neon -void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); #define vp9_quantize_fp vp9_quantize_fp_neon -void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); #define vp9_quantize_fp_32x32 vp9_quantize_fp_32x32_neon void vp9_scale_and_extend_frame_c(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); void vp9_scale_and_extend_frame_neon(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); #define vp9_scale_and_extend_frame vp9_scale_and_extend_frame_neon diff --git a/media/libvpx/config/linux/arm64/vpx_config.h b/media/libvpx/config/linux/arm64/vpx_config.h --- a/media/libvpx/config/linux/arm64/vpx_config.h +++ b/media/libvpx/config/linux/arm64/vpx_config.h @@ -13,10 +13,11 @@ #define VPX_ARCH_ARM 1 #define VPX_ARCH_MIPS 0 #define VPX_ARCH_X86 0 #define VPX_ARCH_X86_64 0 #define VPX_ARCH_PPC 0 +#define VPX_ARCH_LOONGARCH 0 #define HAVE_NEON 1 #define HAVE_NEON_ASM 0 #define HAVE_MIPS32 0 #define HAVE_DSPR2 0 #define HAVE_MSA 0 @@ -30,10 +31,12 @@ #define HAVE_AVX 0 #define HAVE_AVX2 0 #define HAVE_AVX512 0 #define HAVE_VSX 0 #define HAVE_MMI 0 +#define HAVE_LSX 0 +#define HAVE_LASX 0 #define HAVE_VPX_PORTS 1 #define HAVE_PTHREAD_H 1 #define CONFIG_DEPENDENCY_TRACKING 1 #define CONFIG_EXTERNAL_BUILD 1 #define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/linux/arm64/vpx_config.asm b/media/libvpx/config/linux/arm64/vpx_config.asm --- a/media/libvpx/config/linux/arm64/vpx_config.asm +++ b/media/libvpx/config/linux/arm64/vpx_config.asm @@ -1,13 +1,14 @@ @ This file was created from a .asm file @ using the ads2gas.pl script. - .syntax unified +.syntax unified .equ VPX_ARCH_ARM , 1 .equ VPX_ARCH_MIPS , 0 .equ VPX_ARCH_X86 , 0 .equ VPX_ARCH_X86_64 , 0 .equ VPX_ARCH_PPC , 0 +.equ VPX_ARCH_LOONGARCH , 0 .equ HAVE_NEON , 1 .equ HAVE_NEON_ASM , 0 .equ HAVE_MIPS32 , 0 .equ HAVE_DSPR2 , 0 .equ HAVE_MSA , 0 @@ -21,10 +22,12 @@ .equ HAVE_AVX , 0 .equ HAVE_AVX2 , 0 .equ HAVE_AVX512 , 0 .equ HAVE_VSX , 0 .equ HAVE_MMI , 0 +.equ HAVE_LSX , 0 +.equ HAVE_LASX , 0 .equ HAVE_VPX_PORTS , 1 .equ HAVE_PTHREAD_H , 1 .equ CONFIG_DEPENDENCY_TRACKING , 1 .equ CONFIG_EXTERNAL_BUILD , 1 .equ CONFIG_INSTALL_DOCS , 0 @@ -85,6 +88,6 @@ .equ CONFIG_MISMATCH_DEBUG , 0 .equ CONFIG_FP_MB_STATS , 0 .equ CONFIG_EMULATE_HARDWARE , 0 .equ CONFIG_NON_GREEDY_MV , 0 .equ CONFIG_RATE_CTRL , 0 - .section .note.GNU-stack,"",%progbits + .section .note.GNU-stack,"",%progbits diff --git a/media/libvpx/config/linux/arm64/vpx_dsp_rtcd.h b/media/libvpx/config/linux/arm64/vpx_dsp_rtcd.h --- a/media/libvpx/config/linux/arm64/vpx_dsp_rtcd.h +++ b/media/libvpx/config/linux/arm64/vpx_dsp_rtcd.h @@ -350,12 +350,12 @@ int16_t vpx_int_pro_col_c(const uint8_t *ref, const int width); int16_t vpx_int_pro_col_neon(const uint8_t *ref, const int width); #define vpx_int_pro_col vpx_int_pro_col_neon -void vpx_int_pro_row_c(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); -void vpx_int_pro_row_neon(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_c(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_neon(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); #define vpx_int_pro_row vpx_int_pro_row_neon void vpx_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int stride); #define vpx_iwht4x4_16_add vpx_iwht4x4_16_add_c @@ -425,207 +425,174 @@ #define vpx_mse8x16 vpx_mse8x16_c unsigned int vpx_mse8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse); #define vpx_mse8x8 vpx_mse8x8_c -void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); #define vpx_quantize_b vpx_quantize_b_neon -void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); #define vpx_quantize_b_32x32 vpx_quantize_b_32x32_neon unsigned int vpx_sad16x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x16_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x16 vpx_sad16x16_neon unsigned int vpx_sad16x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x16_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x16_avg vpx_sad16x16_avg_neon -void vpx_sad16x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x16x3 vpx_sad16x16x3_c - -void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x16x4d vpx_sad16x16x4d_neon -void vpx_sad16x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x16x8 vpx_sad16x16x8_c - unsigned int vpx_sad16x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x32_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x32 vpx_sad16x32_neon unsigned int vpx_sad16x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x32_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x32_avg vpx_sad16x32_avg_neon -void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x32x4d vpx_sad16x32x4d_neon unsigned int vpx_sad16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x8_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x8 vpx_sad16x8_neon unsigned int vpx_sad16x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x8_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x8_avg vpx_sad16x8_avg_neon -void vpx_sad16x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x8x3 vpx_sad16x8x3_c - -void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x8x4d vpx_sad16x8x4d_neon -void vpx_sad16x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x8x8 vpx_sad16x8x8_c - unsigned int vpx_sad32x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad32x16 vpx_sad32x16_neon unsigned int vpx_sad32x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x16_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad32x16_avg vpx_sad32x16_avg_neon -void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad32x16x4d vpx_sad32x16x4d_neon unsigned int vpx_sad32x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad32x32 vpx_sad32x32_neon unsigned int vpx_sad32x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x32_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad32x32_avg vpx_sad32x32_avg_neon -void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad32x32x4d vpx_sad32x32x4d_neon -void vpx_sad32x32x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad32x32x8 vpx_sad32x32x8_c - unsigned int vpx_sad32x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad32x64 vpx_sad32x64_neon unsigned int vpx_sad32x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x64_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad32x64_avg vpx_sad32x64_avg_neon -void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad32x64x4d vpx_sad32x64x4d_neon unsigned int vpx_sad4x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x4_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad4x4 vpx_sad4x4_neon unsigned int vpx_sad4x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x4_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad4x4_avg vpx_sad4x4_avg_neon -void vpx_sad4x4x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad4x4x3 vpx_sad4x4x3_c - -void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x4x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad4x4x4d vpx_sad4x4x4d_neon -void vpx_sad4x4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad4x4x8 vpx_sad4x4x8_c - unsigned int vpx_sad4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x8_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad4x8 vpx_sad4x8_neon unsigned int vpx_sad4x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x8_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad4x8_avg vpx_sad4x8_avg_neon -void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad4x8x4d vpx_sad4x8x4d_neon unsigned int vpx_sad64x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad64x32 vpx_sad64x32_neon unsigned int vpx_sad64x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x32_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad64x32_avg vpx_sad64x32_avg_neon -void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad64x32x4d vpx_sad64x32x4d_neon unsigned int vpx_sad64x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad64x64 vpx_sad64x64_neon unsigned int vpx_sad64x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x64_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad64x64_avg vpx_sad64x64_avg_neon -void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad64x64x4d vpx_sad64x64x4d_neon unsigned int vpx_sad8x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x16_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x16 vpx_sad8x16_neon unsigned int vpx_sad8x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x16_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x16_avg vpx_sad8x16_avg_neon -void vpx_sad8x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x16x3 vpx_sad8x16x3_c - -void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x16x4d vpx_sad8x16x4d_neon -void vpx_sad8x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x16x8 vpx_sad8x16x8_c - unsigned int vpx_sad8x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x4_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x4 vpx_sad8x4_neon unsigned int vpx_sad8x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x4_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x4_avg vpx_sad8x4_avg_neon -void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x4x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x4x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x4x4d vpx_sad8x4x4d_neon unsigned int vpx_sad8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x8_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x8 vpx_sad8x8_neon unsigned int vpx_sad8x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x8_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x8_avg vpx_sad8x8_avg_neon -void vpx_sad8x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x8x3 vpx_sad8x8x3_c - -void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x8x4d vpx_sad8x8x4d_neon -void vpx_sad8x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x8x8 vpx_sad8x8x8_c - int vpx_satd_c(const int16_t *coeff, int length); int vpx_satd_neon(const int16_t *coeff, int length); #define vpx_satd vpx_satd_neon void vpx_scaled_2d_c(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h); diff --git a/media/libvpx/config/linux/ia32/vp8_rtcd.h b/media/libvpx/config/linux/ia32/vp8_rtcd.h --- a/media/libvpx/config/linux/ia32/vp8_rtcd.h +++ b/media/libvpx/config/linux/ia32/vp8_rtcd.h @@ -121,15 +121,10 @@ void vp8_filter_by_weight8x8_c(unsigned char *src, int src_stride, unsigned char *dst, int dst_stride, int src_weight); void vp8_filter_by_weight8x8_sse2(unsigned char *src, int src_stride, unsigned char *dst, int dst_stride, int src_weight); RTCD_EXTERN void (*vp8_filter_by_weight8x8)(unsigned char *src, int src_stride, unsigned char *dst, int dst_stride, int src_weight); -int vp8_full_search_sad_c(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -int vp8_full_search_sadx3(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -int vp8_full_search_sadx8(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -RTCD_EXTERN int (*vp8_full_search_sad)(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); - void vp8_loop_filter_bh_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); void vp8_loop_filter_bh_sse2(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); RTCD_EXTERN void (*vp8_loop_filter_bh)(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); void vp8_loop_filter_bv_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); @@ -276,13 +271,10 @@ if (flags & HAS_SSSE3) vp8_fast_quantize_b = vp8_fast_quantize_b_ssse3; vp8_filter_by_weight16x16 = vp8_filter_by_weight16x16_c; if (flags & HAS_SSE2) vp8_filter_by_weight16x16 = vp8_filter_by_weight16x16_sse2; vp8_filter_by_weight8x8 = vp8_filter_by_weight8x8_c; if (flags & HAS_SSE2) vp8_filter_by_weight8x8 = vp8_filter_by_weight8x8_sse2; - vp8_full_search_sad = vp8_full_search_sad_c; - if (flags & HAS_SSE3) vp8_full_search_sad = vp8_full_search_sadx3; - if (flags & HAS_SSE4_1) vp8_full_search_sad = vp8_full_search_sadx8; vp8_loop_filter_bh = vp8_loop_filter_bh_c; if (flags & HAS_SSE2) vp8_loop_filter_bh = vp8_loop_filter_bh_sse2; vp8_loop_filter_bv = vp8_loop_filter_bv_c; if (flags & HAS_SSE2) vp8_loop_filter_bv = vp8_loop_filter_bv_sse2; vp8_loop_filter_mbh = vp8_loop_filter_mbh_c; diff --git a/media/libvpx/config/linux/ia32/vp9_rtcd.h b/media/libvpx/config/linux/ia32/vp9_rtcd.h --- a/media/libvpx/config/linux/ia32/vp9_rtcd.h +++ b/media/libvpx/config/linux/ia32/vp9_rtcd.h @@ -83,16 +83,16 @@ void vp9_iht8x8_64_add_c(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); void vp9_iht8x8_64_add_sse2(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); RTCD_EXTERN void (*vp9_iht8x8_64_add)(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); -void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vp9_quantize_fp)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vp9_quantize_fp)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); #define vp9_quantize_fp_32x32 vp9_quantize_fp_32x32_c void vp9_scale_and_extend_frame_c(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); void vp9_scale_and_extend_frame_ssse3(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); RTCD_EXTERN void (*vp9_scale_and_extend_frame)(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); diff --git a/media/libvpx/config/linux/ia32/vpx_config.h b/media/libvpx/config/linux/ia32/vpx_config.h --- a/media/libvpx/config/linux/ia32/vpx_config.h +++ b/media/libvpx/config/linux/ia32/vpx_config.h @@ -13,10 +13,11 @@ #define VPX_ARCH_ARM 0 #define VPX_ARCH_MIPS 0 #define VPX_ARCH_X86 1 #define VPX_ARCH_X86_64 0 #define VPX_ARCH_PPC 0 +#define VPX_ARCH_LOONGARCH 0 #define HAVE_NEON 0 #define HAVE_NEON_ASM 0 #define HAVE_MIPS32 0 #define HAVE_DSPR2 0 #define HAVE_MSA 0 @@ -30,10 +31,12 @@ #define HAVE_AVX 1 #define HAVE_AVX2 1 #define HAVE_AVX512 0 #define HAVE_VSX 0 #define HAVE_MMI 0 +#define HAVE_LSX 0 +#define HAVE_LASX 0 #define HAVE_VPX_PORTS 1 #define HAVE_PTHREAD_H 1 #define CONFIG_DEPENDENCY_TRACKING 1 #define CONFIG_EXTERNAL_BUILD 1 #define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/linux/ia32/vpx_config.asm b/media/libvpx/config/linux/ia32/vpx_config.asm --- a/media/libvpx/config/linux/ia32/vpx_config.asm +++ b/media/libvpx/config/linux/ia32/vpx_config.asm @@ -1,10 +1,11 @@ %define VPX_ARCH_ARM 0 %define VPX_ARCH_MIPS 0 %define VPX_ARCH_X86 1 %define VPX_ARCH_X86_64 0 %define VPX_ARCH_PPC 0 +%define VPX_ARCH_LOONGARCH 0 %define HAVE_NEON 0 %define HAVE_NEON_ASM 0 %define HAVE_MIPS32 0 %define HAVE_DSPR2 0 %define HAVE_MSA 0 @@ -18,10 +19,12 @@ %define HAVE_AVX 1 %define HAVE_AVX2 1 %define HAVE_AVX512 0 %define HAVE_VSX 0 %define HAVE_MMI 0 +%define HAVE_LSX 0 +%define HAVE_LASX 0 %define HAVE_VPX_PORTS 1 %define HAVE_PTHREAD_H 1 %define CONFIG_DEPENDENCY_TRACKING 1 %define CONFIG_EXTERNAL_BUILD 1 %define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/linux/ia32/vpx_dsp_rtcd.h b/media/libvpx/config/linux/ia32/vpx_dsp_rtcd.h --- a/media/libvpx/config/linux/ia32/vpx_dsp_rtcd.h +++ b/media/libvpx/config/linux/ia32/vpx_dsp_rtcd.h @@ -379,13 +379,13 @@ int16_t vpx_int_pro_col_c(const uint8_t *ref, const int width); int16_t vpx_int_pro_col_sse2(const uint8_t *ref, const int width); RTCD_EXTERN int16_t (*vpx_int_pro_col)(const uint8_t *ref, const int width); -void vpx_int_pro_row_c(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); -void vpx_int_pro_row_sse2(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); -RTCD_EXTERN void (*vpx_int_pro_row)(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_c(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_sse2(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); +RTCD_EXTERN void (*vpx_int_pro_row)(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); void vpx_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int stride); void vpx_iwht4x4_16_add_sse2(const tran_low_t *input, uint8_t *dest, int stride); RTCD_EXTERN void (*vpx_iwht4x4_16_add)(const tran_low_t *input, uint8_t *dest, int stride); @@ -478,74 +478,56 @@ void vpx_post_proc_down_and_across_mb_row_c(unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size); void vpx_post_proc_down_and_across_mb_row_sse2(unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size); RTCD_EXTERN void (*vpx_post_proc_down_and_across_mb_row)(unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size); -void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vpx_quantize_b)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vpx_quantize_b)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vpx_quantize_b_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vpx_quantize_b_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); unsigned int vpx_sad16x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad16x16)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad16x16_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad16x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x3_ssse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad16x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad16x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad16x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad16x32)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad16x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad16x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad16x8_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad16x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x3_ssse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad16x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad16x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad32x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x16)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -553,13 +535,13 @@ unsigned int vpx_sad32x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x16_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x16_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad32x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad32x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x32)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -567,18 +549,14 @@ unsigned int vpx_sad32x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x32_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad32x32x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x8_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x32x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x32x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad32x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad32x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x64)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -586,45 +564,37 @@ unsigned int vpx_sad32x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x64_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x64_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x64_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad32x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad4x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x4_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad4x4)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x4_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad4x4_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad4x4x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad4x4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad4x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad4x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad4x8_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad4x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad64x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad64x32)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -632,13 +602,13 @@ unsigned int vpx_sad64x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x32_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad64x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad64x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad64x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad64x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad64x64)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -646,66 +616,50 @@ unsigned int vpx_sad64x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x64_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x64_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad64x64_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x64x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad64x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x64x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad64x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad8x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad8x16)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad8x16_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad8x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad8x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad8x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad8x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x4_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad8x4)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x4_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad8x4_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad8x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad8x8_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad8x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad8x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad8x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); int vpx_satd_c(const int16_t *coeff, int length); int vpx_satd_sse2(const int16_t *coeff, int length); int vpx_satd_avx2(const int16_t *coeff, int length); RTCD_EXTERN int (*vpx_satd)(const int16_t *coeff, int length); @@ -1211,34 +1165,24 @@ if (flags & HAS_AVX) vpx_quantize_b_32x32 = vpx_quantize_b_32x32_avx; vpx_sad16x16 = vpx_sad16x16_c; if (flags & HAS_SSE2) vpx_sad16x16 = vpx_sad16x16_sse2; vpx_sad16x16_avg = vpx_sad16x16_avg_c; if (flags & HAS_SSE2) vpx_sad16x16_avg = vpx_sad16x16_avg_sse2; - vpx_sad16x16x3 = vpx_sad16x16x3_c; - if (flags & HAS_SSE3) vpx_sad16x16x3 = vpx_sad16x16x3_sse3; - if (flags & HAS_SSSE3) vpx_sad16x16x3 = vpx_sad16x16x3_ssse3; vpx_sad16x16x4d = vpx_sad16x16x4d_c; if (flags & HAS_SSE2) vpx_sad16x16x4d = vpx_sad16x16x4d_sse2; - vpx_sad16x16x8 = vpx_sad16x16x8_c; - if (flags & HAS_SSE4_1) vpx_sad16x16x8 = vpx_sad16x16x8_sse4_1; vpx_sad16x32 = vpx_sad16x32_c; if (flags & HAS_SSE2) vpx_sad16x32 = vpx_sad16x32_sse2; vpx_sad16x32_avg = vpx_sad16x32_avg_c; if (flags & HAS_SSE2) vpx_sad16x32_avg = vpx_sad16x32_avg_sse2; vpx_sad16x32x4d = vpx_sad16x32x4d_c; if (flags & HAS_SSE2) vpx_sad16x32x4d = vpx_sad16x32x4d_sse2; vpx_sad16x8 = vpx_sad16x8_c; if (flags & HAS_SSE2) vpx_sad16x8 = vpx_sad16x8_sse2; vpx_sad16x8_avg = vpx_sad16x8_avg_c; if (flags & HAS_SSE2) vpx_sad16x8_avg = vpx_sad16x8_avg_sse2; - vpx_sad16x8x3 = vpx_sad16x8x3_c; - if (flags & HAS_SSE3) vpx_sad16x8x3 = vpx_sad16x8x3_sse3; - if (flags & HAS_SSSE3) vpx_sad16x8x3 = vpx_sad16x8x3_ssse3; vpx_sad16x8x4d = vpx_sad16x8x4d_c; if (flags & HAS_SSE2) vpx_sad16x8x4d = vpx_sad16x8x4d_sse2; - vpx_sad16x8x8 = vpx_sad16x8x8_c; - if (flags & HAS_SSE4_1) vpx_sad16x8x8 = vpx_sad16x8x8_sse4_1; vpx_sad32x16 = vpx_sad32x16_c; if (flags & HAS_SSE2) vpx_sad32x16 = vpx_sad32x16_sse2; if (flags & HAS_AVX2) vpx_sad32x16 = vpx_sad32x16_avx2; vpx_sad32x16_avg = vpx_sad32x16_avg_c; if (flags & HAS_SSE2) vpx_sad32x16_avg = vpx_sad32x16_avg_sse2; @@ -1252,12 +1196,10 @@ if (flags & HAS_SSE2) vpx_sad32x32_avg = vpx_sad32x32_avg_sse2; if (flags & HAS_AVX2) vpx_sad32x32_avg = vpx_sad32x32_avg_avx2; vpx_sad32x32x4d = vpx_sad32x32x4d_c; if (flags & HAS_SSE2) vpx_sad32x32x4d = vpx_sad32x32x4d_sse2; if (flags & HAS_AVX2) vpx_sad32x32x4d = vpx_sad32x32x4d_avx2; - vpx_sad32x32x8 = vpx_sad32x32x8_c; - if (flags & HAS_AVX2) vpx_sad32x32x8 = vpx_sad32x32x8_avx2; vpx_sad32x64 = vpx_sad32x64_c; if (flags & HAS_SSE2) vpx_sad32x64 = vpx_sad32x64_sse2; if (flags & HAS_AVX2) vpx_sad32x64 = vpx_sad32x64_avx2; vpx_sad32x64_avg = vpx_sad32x64_avg_c; if (flags & HAS_SSE2) vpx_sad32x64_avg = vpx_sad32x64_avg_sse2; @@ -1266,16 +1208,12 @@ if (flags & HAS_SSE2) vpx_sad32x64x4d = vpx_sad32x64x4d_sse2; vpx_sad4x4 = vpx_sad4x4_c; if (flags & HAS_SSE2) vpx_sad4x4 = vpx_sad4x4_sse2; vpx_sad4x4_avg = vpx_sad4x4_avg_c; if (flags & HAS_SSE2) vpx_sad4x4_avg = vpx_sad4x4_avg_sse2; - vpx_sad4x4x3 = vpx_sad4x4x3_c; - if (flags & HAS_SSE3) vpx_sad4x4x3 = vpx_sad4x4x3_sse3; vpx_sad4x4x4d = vpx_sad4x4x4d_c; if (flags & HAS_SSE2) vpx_sad4x4x4d = vpx_sad4x4x4d_sse2; - vpx_sad4x4x8 = vpx_sad4x4x8_c; - if (flags & HAS_SSE4_1) vpx_sad4x4x8 = vpx_sad4x4x8_sse4_1; vpx_sad4x8 = vpx_sad4x8_c; if (flags & HAS_SSE2) vpx_sad4x8 = vpx_sad4x8_sse2; vpx_sad4x8_avg = vpx_sad4x8_avg_c; if (flags & HAS_SSE2) vpx_sad4x8_avg = vpx_sad4x8_avg_sse2; vpx_sad4x8x4d = vpx_sad4x8x4d_c; @@ -1299,32 +1237,24 @@ if (flags & HAS_AVX2) vpx_sad64x64x4d = vpx_sad64x64x4d_avx2; vpx_sad8x16 = vpx_sad8x16_c; if (flags & HAS_SSE2) vpx_sad8x16 = vpx_sad8x16_sse2; vpx_sad8x16_avg = vpx_sad8x16_avg_c; if (flags & HAS_SSE2) vpx_sad8x16_avg = vpx_sad8x16_avg_sse2; - vpx_sad8x16x3 = vpx_sad8x16x3_c; - if (flags & HAS_SSE3) vpx_sad8x16x3 = vpx_sad8x16x3_sse3; vpx_sad8x16x4d = vpx_sad8x16x4d_c; if (flags & HAS_SSE2) vpx_sad8x16x4d = vpx_sad8x16x4d_sse2; - vpx_sad8x16x8 = vpx_sad8x16x8_c; - if (flags & HAS_SSE4_1) vpx_sad8x16x8 = vpx_sad8x16x8_sse4_1; vpx_sad8x4 = vpx_sad8x4_c; if (flags & HAS_SSE2) vpx_sad8x4 = vpx_sad8x4_sse2; vpx_sad8x4_avg = vpx_sad8x4_avg_c; if (flags & HAS_SSE2) vpx_sad8x4_avg = vpx_sad8x4_avg_sse2; vpx_sad8x4x4d = vpx_sad8x4x4d_c; if (flags & HAS_SSE2) vpx_sad8x4x4d = vpx_sad8x4x4d_sse2; vpx_sad8x8 = vpx_sad8x8_c; if (flags & HAS_SSE2) vpx_sad8x8 = vpx_sad8x8_sse2; vpx_sad8x8_avg = vpx_sad8x8_avg_c; if (flags & HAS_SSE2) vpx_sad8x8_avg = vpx_sad8x8_avg_sse2; - vpx_sad8x8x3 = vpx_sad8x8x3_c; - if (flags & HAS_SSE3) vpx_sad8x8x3 = vpx_sad8x8x3_sse3; vpx_sad8x8x4d = vpx_sad8x8x4d_c; if (flags & HAS_SSE2) vpx_sad8x8x4d = vpx_sad8x8x4d_sse2; - vpx_sad8x8x8 = vpx_sad8x8x8_c; - if (flags & HAS_SSE4_1) vpx_sad8x8x8 = vpx_sad8x8x8_sse4_1; vpx_satd = vpx_satd_c; if (flags & HAS_SSE2) vpx_satd = vpx_satd_sse2; if (flags & HAS_AVX2) vpx_satd = vpx_satd_avx2; vpx_scaled_2d = vpx_scaled_2d_c; if (flags & HAS_SSSE3) vpx_scaled_2d = vpx_scaled_2d_ssse3; diff --git a/media/libvpx/config/linux/x64/vp8_rtcd.h b/media/libvpx/config/linux/x64/vp8_rtcd.h --- a/media/libvpx/config/linux/x64/vp8_rtcd.h +++ b/media/libvpx/config/linux/x64/vp8_rtcd.h @@ -121,15 +121,10 @@ void vp8_filter_by_weight8x8_c(unsigned char *src, int src_stride, unsigned char *dst, int dst_stride, int src_weight); void vp8_filter_by_weight8x8_sse2(unsigned char *src, int src_stride, unsigned char *dst, int dst_stride, int src_weight); #define vp8_filter_by_weight8x8 vp8_filter_by_weight8x8_sse2 -int vp8_full_search_sad_c(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -int vp8_full_search_sadx3(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -int vp8_full_search_sadx8(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -RTCD_EXTERN int (*vp8_full_search_sad)(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); - void vp8_loop_filter_bh_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); void vp8_loop_filter_bh_sse2(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); #define vp8_loop_filter_bh vp8_loop_filter_bh_sse2 void vp8_loop_filter_bv_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); @@ -240,13 +235,10 @@ if (flags & HAS_SSSE3) vp8_bilinear_predict8x8 = vp8_bilinear_predict8x8_ssse3; vp8_copy32xn = vp8_copy32xn_sse2; if (flags & HAS_SSE3) vp8_copy32xn = vp8_copy32xn_sse3; vp8_fast_quantize_b = vp8_fast_quantize_b_sse2; if (flags & HAS_SSSE3) vp8_fast_quantize_b = vp8_fast_quantize_b_ssse3; - vp8_full_search_sad = vp8_full_search_sad_c; - if (flags & HAS_SSE3) vp8_full_search_sad = vp8_full_search_sadx3; - if (flags & HAS_SSE4_1) vp8_full_search_sad = vp8_full_search_sadx8; vp8_regular_quantize_b = vp8_regular_quantize_b_sse2; if (flags & HAS_SSE4_1) vp8_regular_quantize_b = vp8_regular_quantize_b_sse4_1; vp8_sixtap_predict16x16 = vp8_sixtap_predict16x16_sse2; if (flags & HAS_SSSE3) vp8_sixtap_predict16x16 = vp8_sixtap_predict16x16_ssse3; vp8_sixtap_predict4x4 = vp8_sixtap_predict4x4_mmx; diff --git a/media/libvpx/config/linux/x64/vp9_rtcd.h b/media/libvpx/config/linux/x64/vp9_rtcd.h --- a/media/libvpx/config/linux/x64/vp9_rtcd.h +++ b/media/libvpx/config/linux/x64/vp9_rtcd.h @@ -83,19 +83,19 @@ void vp9_iht8x8_64_add_c(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); void vp9_iht8x8_64_add_sse2(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); #define vp9_iht8x8_64_add vp9_iht8x8_64_add_sse2 -void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vp9_quantize_fp)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vp9_quantize_fp)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vp9_quantize_fp_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vp9_quantize_fp_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); void vp9_scale_and_extend_frame_c(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); void vp9_scale_and_extend_frame_ssse3(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); RTCD_EXTERN void (*vp9_scale_and_extend_frame)(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); diff --git a/media/libvpx/config/linux/x64/vpx_config.h b/media/libvpx/config/linux/x64/vpx_config.h --- a/media/libvpx/config/linux/x64/vpx_config.h +++ b/media/libvpx/config/linux/x64/vpx_config.h @@ -13,10 +13,11 @@ #define VPX_ARCH_ARM 0 #define VPX_ARCH_MIPS 0 #define VPX_ARCH_X86 0 #define VPX_ARCH_X86_64 1 #define VPX_ARCH_PPC 0 +#define VPX_ARCH_LOONGARCH 0 #define HAVE_NEON 0 #define HAVE_NEON_ASM 0 #define HAVE_MIPS32 0 #define HAVE_DSPR2 0 #define HAVE_MSA 0 @@ -30,10 +31,12 @@ #define HAVE_AVX 1 #define HAVE_AVX2 1 #define HAVE_AVX512 0 #define HAVE_VSX 0 #define HAVE_MMI 0 +#define HAVE_LSX 0 +#define HAVE_LASX 0 #define HAVE_VPX_PORTS 1 #define HAVE_PTHREAD_H 1 #define CONFIG_DEPENDENCY_TRACKING 1 #define CONFIG_EXTERNAL_BUILD 1 #define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/linux/x64/vpx_config.asm b/media/libvpx/config/linux/x64/vpx_config.asm --- a/media/libvpx/config/linux/x64/vpx_config.asm +++ b/media/libvpx/config/linux/x64/vpx_config.asm @@ -1,10 +1,11 @@ %define VPX_ARCH_ARM 0 %define VPX_ARCH_MIPS 0 %define VPX_ARCH_X86 0 %define VPX_ARCH_X86_64 1 %define VPX_ARCH_PPC 0 +%define VPX_ARCH_LOONGARCH 0 %define HAVE_NEON 0 %define HAVE_NEON_ASM 0 %define HAVE_MIPS32 0 %define HAVE_DSPR2 0 %define HAVE_MSA 0 @@ -18,10 +19,12 @@ %define HAVE_AVX 1 %define HAVE_AVX2 1 %define HAVE_AVX512 0 %define HAVE_VSX 0 %define HAVE_MMI 0 +%define HAVE_LSX 0 +%define HAVE_LASX 0 %define HAVE_VPX_PORTS 1 %define HAVE_PTHREAD_H 1 %define CONFIG_DEPENDENCY_TRACKING 1 %define CONFIG_EXTERNAL_BUILD 1 %define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/linux/x64/vpx_dsp_rtcd.h b/media/libvpx/config/linux/x64/vpx_dsp_rtcd.h --- a/media/libvpx/config/linux/x64/vpx_dsp_rtcd.h +++ b/media/libvpx/config/linux/x64/vpx_dsp_rtcd.h @@ -381,12 +381,12 @@ int16_t vpx_int_pro_col_c(const uint8_t *ref, const int width); int16_t vpx_int_pro_col_sse2(const uint8_t *ref, const int width); #define vpx_int_pro_col vpx_int_pro_col_sse2 -void vpx_int_pro_row_c(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); -void vpx_int_pro_row_sse2(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_c(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_sse2(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); #define vpx_int_pro_row vpx_int_pro_row_sse2 void vpx_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int stride); void vpx_iwht4x4_16_add_sse2(const tran_low_t *input, uint8_t *dest, int stride); #define vpx_iwht4x4_16_add vpx_iwht4x4_16_add_sse2 @@ -480,87 +480,69 @@ void vpx_post_proc_down_and_across_mb_row_c(unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size); void vpx_post_proc_down_and_across_mb_row_sse2(unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size); #define vpx_post_proc_down_and_across_mb_row vpx_post_proc_down_and_across_mb_row_sse2 -void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vpx_quantize_b)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vpx_quantize_b)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vpx_quantize_b_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vpx_quantize_b_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); unsigned int vpx_sad16x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x16 vpx_sad16x16_sse2 unsigned int vpx_sad16x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x16_avg vpx_sad16x16_avg_sse2 -void vpx_sad16x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x3_ssse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x16x4d vpx_sad16x16x4d_sse2 -void vpx_sad16x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - unsigned int vpx_sad16x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x32 vpx_sad16x32_sse2 unsigned int vpx_sad16x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x32_avg vpx_sad16x32_avg_sse2 -void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x32x4d vpx_sad16x32x4d_sse2 unsigned int vpx_sad16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x8 vpx_sad16x8_sse2 unsigned int vpx_sad16x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x8_avg vpx_sad16x8_avg_sse2 -void vpx_sad16x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x3_ssse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x8x4d vpx_sad16x8x4d_sse2 -void vpx_sad16x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - unsigned int vpx_sad32x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x16)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x16_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x16_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad32x16x4d vpx_sad32x16x4d_sse2 unsigned int vpx_sad32x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -569,18 +551,14 @@ unsigned int vpx_sad32x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x32_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad32x32x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x8_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x32x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x32x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad32x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad32x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x64)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -588,44 +566,36 @@ unsigned int vpx_sad32x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x64_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x64_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x64_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad32x64x4d vpx_sad32x64x4d_sse2 unsigned int vpx_sad4x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x4_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad4x4 vpx_sad4x4_sse2 unsigned int vpx_sad4x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x4_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad4x4_avg vpx_sad4x4_avg_sse2 -void vpx_sad4x4x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad4x4x4d vpx_sad4x4x4d_sse2 -void vpx_sad4x4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - unsigned int vpx_sad4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad4x8 vpx_sad4x8_sse2 unsigned int vpx_sad4x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad4x8_avg vpx_sad4x8_avg_sse2 -void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad4x8x4d vpx_sad4x8x4d_sse2 unsigned int vpx_sad64x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -634,12 +604,12 @@ unsigned int vpx_sad64x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x32_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad64x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad64x32x4d vpx_sad64x32x4d_sse2 unsigned int vpx_sad64x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -648,67 +618,51 @@ unsigned int vpx_sad64x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x64_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x64_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad64x64_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x64x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad64x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x64x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad64x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad8x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x16 vpx_sad8x16_sse2 unsigned int vpx_sad8x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x16_avg vpx_sad8x16_avg_sse2 -void vpx_sad8x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x16x4d vpx_sad8x16x4d_sse2 -void vpx_sad8x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - unsigned int vpx_sad8x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x4_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x4 vpx_sad8x4_sse2 unsigned int vpx_sad8x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x4_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x4_avg vpx_sad8x4_avg_sse2 -void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x4x4d vpx_sad8x4x4d_sse2 unsigned int vpx_sad8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x8 vpx_sad8x8_sse2 unsigned int vpx_sad8x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x8_avg vpx_sad8x8_avg_sse2 -void vpx_sad8x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x8x4d vpx_sad8x8x4d_sse2 -void vpx_sad8x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - int vpx_satd_c(const int16_t *coeff, int length); int vpx_satd_sse2(const int16_t *coeff, int length); int vpx_satd_avx2(const int16_t *coeff, int length); RTCD_EXTERN int (*vpx_satd)(const int16_t *coeff, int length); @@ -1058,58 +1012,34 @@ if (flags & HAS_SSSE3) vpx_quantize_b = vpx_quantize_b_ssse3; if (flags & HAS_AVX) vpx_quantize_b = vpx_quantize_b_avx; vpx_quantize_b_32x32 = vpx_quantize_b_32x32_c; if (flags & HAS_SSSE3) vpx_quantize_b_32x32 = vpx_quantize_b_32x32_ssse3; if (flags & HAS_AVX) vpx_quantize_b_32x32 = vpx_quantize_b_32x32_avx; - vpx_sad16x16x3 = vpx_sad16x16x3_c; - if (flags & HAS_SSE3) vpx_sad16x16x3 = vpx_sad16x16x3_sse3; - if (flags & HAS_SSSE3) vpx_sad16x16x3 = vpx_sad16x16x3_ssse3; - vpx_sad16x16x8 = vpx_sad16x16x8_c; - if (flags & HAS_SSE4_1) vpx_sad16x16x8 = vpx_sad16x16x8_sse4_1; - vpx_sad16x8x3 = vpx_sad16x8x3_c; - if (flags & HAS_SSE3) vpx_sad16x8x3 = vpx_sad16x8x3_sse3; - if (flags & HAS_SSSE3) vpx_sad16x8x3 = vpx_sad16x8x3_ssse3; - vpx_sad16x8x8 = vpx_sad16x8x8_c; - if (flags & HAS_SSE4_1) vpx_sad16x8x8 = vpx_sad16x8x8_sse4_1; vpx_sad32x16 = vpx_sad32x16_sse2; if (flags & HAS_AVX2) vpx_sad32x16 = vpx_sad32x16_avx2; vpx_sad32x16_avg = vpx_sad32x16_avg_sse2; if (flags & HAS_AVX2) vpx_sad32x16_avg = vpx_sad32x16_avg_avx2; vpx_sad32x32 = vpx_sad32x32_sse2; if (flags & HAS_AVX2) vpx_sad32x32 = vpx_sad32x32_avx2; vpx_sad32x32_avg = vpx_sad32x32_avg_sse2; if (flags & HAS_AVX2) vpx_sad32x32_avg = vpx_sad32x32_avg_avx2; vpx_sad32x32x4d = vpx_sad32x32x4d_sse2; if (flags & HAS_AVX2) vpx_sad32x32x4d = vpx_sad32x32x4d_avx2; - vpx_sad32x32x8 = vpx_sad32x32x8_c; - if (flags & HAS_AVX2) vpx_sad32x32x8 = vpx_sad32x32x8_avx2; vpx_sad32x64 = vpx_sad32x64_sse2; if (flags & HAS_AVX2) vpx_sad32x64 = vpx_sad32x64_avx2; vpx_sad32x64_avg = vpx_sad32x64_avg_sse2; if (flags & HAS_AVX2) vpx_sad32x64_avg = vpx_sad32x64_avg_avx2; - vpx_sad4x4x3 = vpx_sad4x4x3_c; - if (flags & HAS_SSE3) vpx_sad4x4x3 = vpx_sad4x4x3_sse3; - vpx_sad4x4x8 = vpx_sad4x4x8_c; - if (flags & HAS_SSE4_1) vpx_sad4x4x8 = vpx_sad4x4x8_sse4_1; vpx_sad64x32 = vpx_sad64x32_sse2; if (flags & HAS_AVX2) vpx_sad64x32 = vpx_sad64x32_avx2; vpx_sad64x32_avg = vpx_sad64x32_avg_sse2; if (flags & HAS_AVX2) vpx_sad64x32_avg = vpx_sad64x32_avg_avx2; vpx_sad64x64 = vpx_sad64x64_sse2; if (flags & HAS_AVX2) vpx_sad64x64 = vpx_sad64x64_avx2; vpx_sad64x64_avg = vpx_sad64x64_avg_sse2; if (flags & HAS_AVX2) vpx_sad64x64_avg = vpx_sad64x64_avg_avx2; vpx_sad64x64x4d = vpx_sad64x64x4d_sse2; if (flags & HAS_AVX2) vpx_sad64x64x4d = vpx_sad64x64x4d_avx2; - vpx_sad8x16x3 = vpx_sad8x16x3_c; - if (flags & HAS_SSE3) vpx_sad8x16x3 = vpx_sad8x16x3_sse3; - vpx_sad8x16x8 = vpx_sad8x16x8_c; - if (flags & HAS_SSE4_1) vpx_sad8x16x8 = vpx_sad8x16x8_sse4_1; - vpx_sad8x8x3 = vpx_sad8x8x3_c; - if (flags & HAS_SSE3) vpx_sad8x8x3 = vpx_sad8x8x3_sse3; - vpx_sad8x8x8 = vpx_sad8x8x8_c; - if (flags & HAS_SSE4_1) vpx_sad8x8x8 = vpx_sad8x8x8_sse4_1; vpx_satd = vpx_satd_sse2; if (flags & HAS_AVX2) vpx_satd = vpx_satd_avx2; vpx_scaled_2d = vpx_scaled_2d_c; if (flags & HAS_SSSE3) vpx_scaled_2d = vpx_scaled_2d_ssse3; vpx_sub_pixel_avg_variance16x16 = vpx_sub_pixel_avg_variance16x16_sse2; diff --git a/media/libvpx/config/mac/ia32/vp8_rtcd.h b/media/libvpx/config/mac/ia32/vp8_rtcd.h --- a/media/libvpx/config/mac/ia32/vp8_rtcd.h +++ b/media/libvpx/config/mac/ia32/vp8_rtcd.h @@ -121,15 +121,10 @@ void vp8_filter_by_weight8x8_c(unsigned char *src, int src_stride, unsigned char *dst, int dst_stride, int src_weight); void vp8_filter_by_weight8x8_sse2(unsigned char *src, int src_stride, unsigned char *dst, int dst_stride, int src_weight); RTCD_EXTERN void (*vp8_filter_by_weight8x8)(unsigned char *src, int src_stride, unsigned char *dst, int dst_stride, int src_weight); -int vp8_full_search_sad_c(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -int vp8_full_search_sadx3(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -int vp8_full_search_sadx8(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -RTCD_EXTERN int (*vp8_full_search_sad)(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); - void vp8_loop_filter_bh_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); void vp8_loop_filter_bh_sse2(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); RTCD_EXTERN void (*vp8_loop_filter_bh)(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); void vp8_loop_filter_bv_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); @@ -276,13 +271,10 @@ if (flags & HAS_SSSE3) vp8_fast_quantize_b = vp8_fast_quantize_b_ssse3; vp8_filter_by_weight16x16 = vp8_filter_by_weight16x16_c; if (flags & HAS_SSE2) vp8_filter_by_weight16x16 = vp8_filter_by_weight16x16_sse2; vp8_filter_by_weight8x8 = vp8_filter_by_weight8x8_c; if (flags & HAS_SSE2) vp8_filter_by_weight8x8 = vp8_filter_by_weight8x8_sse2; - vp8_full_search_sad = vp8_full_search_sad_c; - if (flags & HAS_SSE3) vp8_full_search_sad = vp8_full_search_sadx3; - if (flags & HAS_SSE4_1) vp8_full_search_sad = vp8_full_search_sadx8; vp8_loop_filter_bh = vp8_loop_filter_bh_c; if (flags & HAS_SSE2) vp8_loop_filter_bh = vp8_loop_filter_bh_sse2; vp8_loop_filter_bv = vp8_loop_filter_bv_c; if (flags & HAS_SSE2) vp8_loop_filter_bv = vp8_loop_filter_bv_sse2; vp8_loop_filter_mbh = vp8_loop_filter_mbh_c; diff --git a/media/libvpx/config/mac/ia32/vp9_rtcd.h b/media/libvpx/config/mac/ia32/vp9_rtcd.h --- a/media/libvpx/config/mac/ia32/vp9_rtcd.h +++ b/media/libvpx/config/mac/ia32/vp9_rtcd.h @@ -83,16 +83,16 @@ void vp9_iht8x8_64_add_c(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); void vp9_iht8x8_64_add_sse2(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); RTCD_EXTERN void (*vp9_iht8x8_64_add)(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); -void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vp9_quantize_fp)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vp9_quantize_fp)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); #define vp9_quantize_fp_32x32 vp9_quantize_fp_32x32_c void vp9_scale_and_extend_frame_c(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); void vp9_scale_and_extend_frame_ssse3(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); RTCD_EXTERN void (*vp9_scale_and_extend_frame)(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); diff --git a/media/libvpx/config/mac/ia32/vpx_config.h b/media/libvpx/config/mac/ia32/vpx_config.h --- a/media/libvpx/config/mac/ia32/vpx_config.h +++ b/media/libvpx/config/mac/ia32/vpx_config.h @@ -13,10 +13,11 @@ #define VPX_ARCH_ARM 0 #define VPX_ARCH_MIPS 0 #define VPX_ARCH_X86 1 #define VPX_ARCH_X86_64 0 #define VPX_ARCH_PPC 0 +#define VPX_ARCH_LOONGARCH 0 #define HAVE_NEON 0 #define HAVE_NEON_ASM 0 #define HAVE_MIPS32 0 #define HAVE_DSPR2 0 #define HAVE_MSA 0 @@ -30,10 +31,12 @@ #define HAVE_AVX 1 #define HAVE_AVX2 1 #define HAVE_AVX512 0 #define HAVE_VSX 0 #define HAVE_MMI 0 +#define HAVE_LSX 0 +#define HAVE_LASX 0 #define HAVE_VPX_PORTS 1 #define HAVE_PTHREAD_H 1 #define CONFIG_DEPENDENCY_TRACKING 1 #define CONFIG_EXTERNAL_BUILD 1 #define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/mac/ia32/vpx_config.asm b/media/libvpx/config/mac/ia32/vpx_config.asm --- a/media/libvpx/config/mac/ia32/vpx_config.asm +++ b/media/libvpx/config/mac/ia32/vpx_config.asm @@ -1,10 +1,11 @@ %define VPX_ARCH_ARM 0 %define VPX_ARCH_MIPS 0 %define VPX_ARCH_X86 1 %define VPX_ARCH_X86_64 0 %define VPX_ARCH_PPC 0 +%define VPX_ARCH_LOONGARCH 0 %define HAVE_NEON 0 %define HAVE_NEON_ASM 0 %define HAVE_MIPS32 0 %define HAVE_DSPR2 0 %define HAVE_MSA 0 @@ -18,10 +19,12 @@ %define HAVE_AVX 1 %define HAVE_AVX2 1 %define HAVE_AVX512 0 %define HAVE_VSX 0 %define HAVE_MMI 0 +%define HAVE_LSX 0 +%define HAVE_LASX 0 %define HAVE_VPX_PORTS 1 %define HAVE_PTHREAD_H 1 %define CONFIG_DEPENDENCY_TRACKING 1 %define CONFIG_EXTERNAL_BUILD 1 %define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/mac/ia32/vpx_dsp_rtcd.h b/media/libvpx/config/mac/ia32/vpx_dsp_rtcd.h --- a/media/libvpx/config/mac/ia32/vpx_dsp_rtcd.h +++ b/media/libvpx/config/mac/ia32/vpx_dsp_rtcd.h @@ -379,13 +379,13 @@ int16_t vpx_int_pro_col_c(const uint8_t *ref, const int width); int16_t vpx_int_pro_col_sse2(const uint8_t *ref, const int width); RTCD_EXTERN int16_t (*vpx_int_pro_col)(const uint8_t *ref, const int width); -void vpx_int_pro_row_c(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); -void vpx_int_pro_row_sse2(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); -RTCD_EXTERN void (*vpx_int_pro_row)(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_c(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_sse2(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); +RTCD_EXTERN void (*vpx_int_pro_row)(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); void vpx_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int stride); void vpx_iwht4x4_16_add_sse2(const tran_low_t *input, uint8_t *dest, int stride); RTCD_EXTERN void (*vpx_iwht4x4_16_add)(const tran_low_t *input, uint8_t *dest, int stride); @@ -478,74 +478,56 @@ void vpx_post_proc_down_and_across_mb_row_c(unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size); void vpx_post_proc_down_and_across_mb_row_sse2(unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size); RTCD_EXTERN void (*vpx_post_proc_down_and_across_mb_row)(unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size); -void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vpx_quantize_b)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vpx_quantize_b)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vpx_quantize_b_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vpx_quantize_b_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); unsigned int vpx_sad16x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad16x16)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad16x16_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad16x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x3_ssse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad16x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad16x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad16x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad16x32)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad16x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad16x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad16x8_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad16x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x3_ssse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad16x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad16x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad32x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x16)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -553,13 +535,13 @@ unsigned int vpx_sad32x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x16_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x16_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad32x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad32x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x32)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -567,18 +549,14 @@ unsigned int vpx_sad32x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x32_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad32x32x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x8_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x32x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x32x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad32x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad32x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x64)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -586,45 +564,37 @@ unsigned int vpx_sad32x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x64_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x64_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x64_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad32x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad4x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x4_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad4x4)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x4_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad4x4_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad4x4x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad4x4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad4x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad4x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad4x8_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad4x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad64x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad64x32)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -632,13 +602,13 @@ unsigned int vpx_sad64x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x32_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad64x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad64x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad64x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad64x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad64x64)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -646,66 +616,50 @@ unsigned int vpx_sad64x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x64_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x64_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad64x64_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x64x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad64x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x64x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad64x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad8x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad8x16)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad8x16_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad8x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad8x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad8x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad8x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x4_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad8x4)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x4_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad8x4_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad8x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad8x8_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad8x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad8x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad8x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); int vpx_satd_c(const int16_t *coeff, int length); int vpx_satd_sse2(const int16_t *coeff, int length); int vpx_satd_avx2(const int16_t *coeff, int length); RTCD_EXTERN int (*vpx_satd)(const int16_t *coeff, int length); @@ -1211,34 +1165,24 @@ if (flags & HAS_AVX) vpx_quantize_b_32x32 = vpx_quantize_b_32x32_avx; vpx_sad16x16 = vpx_sad16x16_c; if (flags & HAS_SSE2) vpx_sad16x16 = vpx_sad16x16_sse2; vpx_sad16x16_avg = vpx_sad16x16_avg_c; if (flags & HAS_SSE2) vpx_sad16x16_avg = vpx_sad16x16_avg_sse2; - vpx_sad16x16x3 = vpx_sad16x16x3_c; - if (flags & HAS_SSE3) vpx_sad16x16x3 = vpx_sad16x16x3_sse3; - if (flags & HAS_SSSE3) vpx_sad16x16x3 = vpx_sad16x16x3_ssse3; vpx_sad16x16x4d = vpx_sad16x16x4d_c; if (flags & HAS_SSE2) vpx_sad16x16x4d = vpx_sad16x16x4d_sse2; - vpx_sad16x16x8 = vpx_sad16x16x8_c; - if (flags & HAS_SSE4_1) vpx_sad16x16x8 = vpx_sad16x16x8_sse4_1; vpx_sad16x32 = vpx_sad16x32_c; if (flags & HAS_SSE2) vpx_sad16x32 = vpx_sad16x32_sse2; vpx_sad16x32_avg = vpx_sad16x32_avg_c; if (flags & HAS_SSE2) vpx_sad16x32_avg = vpx_sad16x32_avg_sse2; vpx_sad16x32x4d = vpx_sad16x32x4d_c; if (flags & HAS_SSE2) vpx_sad16x32x4d = vpx_sad16x32x4d_sse2; vpx_sad16x8 = vpx_sad16x8_c; if (flags & HAS_SSE2) vpx_sad16x8 = vpx_sad16x8_sse2; vpx_sad16x8_avg = vpx_sad16x8_avg_c; if (flags & HAS_SSE2) vpx_sad16x8_avg = vpx_sad16x8_avg_sse2; - vpx_sad16x8x3 = vpx_sad16x8x3_c; - if (flags & HAS_SSE3) vpx_sad16x8x3 = vpx_sad16x8x3_sse3; - if (flags & HAS_SSSE3) vpx_sad16x8x3 = vpx_sad16x8x3_ssse3; vpx_sad16x8x4d = vpx_sad16x8x4d_c; if (flags & HAS_SSE2) vpx_sad16x8x4d = vpx_sad16x8x4d_sse2; - vpx_sad16x8x8 = vpx_sad16x8x8_c; - if (flags & HAS_SSE4_1) vpx_sad16x8x8 = vpx_sad16x8x8_sse4_1; vpx_sad32x16 = vpx_sad32x16_c; if (flags & HAS_SSE2) vpx_sad32x16 = vpx_sad32x16_sse2; if (flags & HAS_AVX2) vpx_sad32x16 = vpx_sad32x16_avx2; vpx_sad32x16_avg = vpx_sad32x16_avg_c; if (flags & HAS_SSE2) vpx_sad32x16_avg = vpx_sad32x16_avg_sse2; @@ -1252,12 +1196,10 @@ if (flags & HAS_SSE2) vpx_sad32x32_avg = vpx_sad32x32_avg_sse2; if (flags & HAS_AVX2) vpx_sad32x32_avg = vpx_sad32x32_avg_avx2; vpx_sad32x32x4d = vpx_sad32x32x4d_c; if (flags & HAS_SSE2) vpx_sad32x32x4d = vpx_sad32x32x4d_sse2; if (flags & HAS_AVX2) vpx_sad32x32x4d = vpx_sad32x32x4d_avx2; - vpx_sad32x32x8 = vpx_sad32x32x8_c; - if (flags & HAS_AVX2) vpx_sad32x32x8 = vpx_sad32x32x8_avx2; vpx_sad32x64 = vpx_sad32x64_c; if (flags & HAS_SSE2) vpx_sad32x64 = vpx_sad32x64_sse2; if (flags & HAS_AVX2) vpx_sad32x64 = vpx_sad32x64_avx2; vpx_sad32x64_avg = vpx_sad32x64_avg_c; if (flags & HAS_SSE2) vpx_sad32x64_avg = vpx_sad32x64_avg_sse2; @@ -1266,16 +1208,12 @@ if (flags & HAS_SSE2) vpx_sad32x64x4d = vpx_sad32x64x4d_sse2; vpx_sad4x4 = vpx_sad4x4_c; if (flags & HAS_SSE2) vpx_sad4x4 = vpx_sad4x4_sse2; vpx_sad4x4_avg = vpx_sad4x4_avg_c; if (flags & HAS_SSE2) vpx_sad4x4_avg = vpx_sad4x4_avg_sse2; - vpx_sad4x4x3 = vpx_sad4x4x3_c; - if (flags & HAS_SSE3) vpx_sad4x4x3 = vpx_sad4x4x3_sse3; vpx_sad4x4x4d = vpx_sad4x4x4d_c; if (flags & HAS_SSE2) vpx_sad4x4x4d = vpx_sad4x4x4d_sse2; - vpx_sad4x4x8 = vpx_sad4x4x8_c; - if (flags & HAS_SSE4_1) vpx_sad4x4x8 = vpx_sad4x4x8_sse4_1; vpx_sad4x8 = vpx_sad4x8_c; if (flags & HAS_SSE2) vpx_sad4x8 = vpx_sad4x8_sse2; vpx_sad4x8_avg = vpx_sad4x8_avg_c; if (flags & HAS_SSE2) vpx_sad4x8_avg = vpx_sad4x8_avg_sse2; vpx_sad4x8x4d = vpx_sad4x8x4d_c; @@ -1299,32 +1237,24 @@ if (flags & HAS_AVX2) vpx_sad64x64x4d = vpx_sad64x64x4d_avx2; vpx_sad8x16 = vpx_sad8x16_c; if (flags & HAS_SSE2) vpx_sad8x16 = vpx_sad8x16_sse2; vpx_sad8x16_avg = vpx_sad8x16_avg_c; if (flags & HAS_SSE2) vpx_sad8x16_avg = vpx_sad8x16_avg_sse2; - vpx_sad8x16x3 = vpx_sad8x16x3_c; - if (flags & HAS_SSE3) vpx_sad8x16x3 = vpx_sad8x16x3_sse3; vpx_sad8x16x4d = vpx_sad8x16x4d_c; if (flags & HAS_SSE2) vpx_sad8x16x4d = vpx_sad8x16x4d_sse2; - vpx_sad8x16x8 = vpx_sad8x16x8_c; - if (flags & HAS_SSE4_1) vpx_sad8x16x8 = vpx_sad8x16x8_sse4_1; vpx_sad8x4 = vpx_sad8x4_c; if (flags & HAS_SSE2) vpx_sad8x4 = vpx_sad8x4_sse2; vpx_sad8x4_avg = vpx_sad8x4_avg_c; if (flags & HAS_SSE2) vpx_sad8x4_avg = vpx_sad8x4_avg_sse2; vpx_sad8x4x4d = vpx_sad8x4x4d_c; if (flags & HAS_SSE2) vpx_sad8x4x4d = vpx_sad8x4x4d_sse2; vpx_sad8x8 = vpx_sad8x8_c; if (flags & HAS_SSE2) vpx_sad8x8 = vpx_sad8x8_sse2; vpx_sad8x8_avg = vpx_sad8x8_avg_c; if (flags & HAS_SSE2) vpx_sad8x8_avg = vpx_sad8x8_avg_sse2; - vpx_sad8x8x3 = vpx_sad8x8x3_c; - if (flags & HAS_SSE3) vpx_sad8x8x3 = vpx_sad8x8x3_sse3; vpx_sad8x8x4d = vpx_sad8x8x4d_c; if (flags & HAS_SSE2) vpx_sad8x8x4d = vpx_sad8x8x4d_sse2; - vpx_sad8x8x8 = vpx_sad8x8x8_c; - if (flags & HAS_SSE4_1) vpx_sad8x8x8 = vpx_sad8x8x8_sse4_1; vpx_satd = vpx_satd_c; if (flags & HAS_SSE2) vpx_satd = vpx_satd_sse2; if (flags & HAS_AVX2) vpx_satd = vpx_satd_avx2; vpx_scaled_2d = vpx_scaled_2d_c; if (flags & HAS_SSSE3) vpx_scaled_2d = vpx_scaled_2d_ssse3; diff --git a/media/libvpx/config/mac/x64/vp8_rtcd.h b/media/libvpx/config/mac/x64/vp8_rtcd.h --- a/media/libvpx/config/mac/x64/vp8_rtcd.h +++ b/media/libvpx/config/mac/x64/vp8_rtcd.h @@ -121,15 +121,10 @@ void vp8_filter_by_weight8x8_c(unsigned char *src, int src_stride, unsigned char *dst, int dst_stride, int src_weight); void vp8_filter_by_weight8x8_sse2(unsigned char *src, int src_stride, unsigned char *dst, int dst_stride, int src_weight); #define vp8_filter_by_weight8x8 vp8_filter_by_weight8x8_sse2 -int vp8_full_search_sad_c(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -int vp8_full_search_sadx3(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -int vp8_full_search_sadx8(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -RTCD_EXTERN int (*vp8_full_search_sad)(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); - void vp8_loop_filter_bh_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); void vp8_loop_filter_bh_sse2(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); #define vp8_loop_filter_bh vp8_loop_filter_bh_sse2 void vp8_loop_filter_bv_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); @@ -240,13 +235,10 @@ if (flags & HAS_SSSE3) vp8_bilinear_predict8x8 = vp8_bilinear_predict8x8_ssse3; vp8_copy32xn = vp8_copy32xn_sse2; if (flags & HAS_SSE3) vp8_copy32xn = vp8_copy32xn_sse3; vp8_fast_quantize_b = vp8_fast_quantize_b_sse2; if (flags & HAS_SSSE3) vp8_fast_quantize_b = vp8_fast_quantize_b_ssse3; - vp8_full_search_sad = vp8_full_search_sad_c; - if (flags & HAS_SSE3) vp8_full_search_sad = vp8_full_search_sadx3; - if (flags & HAS_SSE4_1) vp8_full_search_sad = vp8_full_search_sadx8; vp8_regular_quantize_b = vp8_regular_quantize_b_sse2; if (flags & HAS_SSE4_1) vp8_regular_quantize_b = vp8_regular_quantize_b_sse4_1; vp8_sixtap_predict16x16 = vp8_sixtap_predict16x16_sse2; if (flags & HAS_SSSE3) vp8_sixtap_predict16x16 = vp8_sixtap_predict16x16_ssse3; vp8_sixtap_predict4x4 = vp8_sixtap_predict4x4_mmx; diff --git a/media/libvpx/config/mac/x64/vp9_rtcd.h b/media/libvpx/config/mac/x64/vp9_rtcd.h --- a/media/libvpx/config/mac/x64/vp9_rtcd.h +++ b/media/libvpx/config/mac/x64/vp9_rtcd.h @@ -83,19 +83,19 @@ void vp9_iht8x8_64_add_c(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); void vp9_iht8x8_64_add_sse2(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); #define vp9_iht8x8_64_add vp9_iht8x8_64_add_sse2 -void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vp9_quantize_fp)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vp9_quantize_fp)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vp9_quantize_fp_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vp9_quantize_fp_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); void vp9_scale_and_extend_frame_c(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); void vp9_scale_and_extend_frame_ssse3(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); RTCD_EXTERN void (*vp9_scale_and_extend_frame)(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); diff --git a/media/libvpx/config/mac/x64/vpx_config.h b/media/libvpx/config/mac/x64/vpx_config.h --- a/media/libvpx/config/mac/x64/vpx_config.h +++ b/media/libvpx/config/mac/x64/vpx_config.h @@ -13,10 +13,11 @@ #define VPX_ARCH_ARM 0 #define VPX_ARCH_MIPS 0 #define VPX_ARCH_X86 0 #define VPX_ARCH_X86_64 1 #define VPX_ARCH_PPC 0 +#define VPX_ARCH_LOONGARCH 0 #define HAVE_NEON 0 #define HAVE_NEON_ASM 0 #define HAVE_MIPS32 0 #define HAVE_DSPR2 0 #define HAVE_MSA 0 @@ -30,10 +31,12 @@ #define HAVE_AVX 1 #define HAVE_AVX2 1 #define HAVE_AVX512 0 #define HAVE_VSX 0 #define HAVE_MMI 0 +#define HAVE_LSX 0 +#define HAVE_LASX 0 #define HAVE_VPX_PORTS 1 #define HAVE_PTHREAD_H 1 #define CONFIG_DEPENDENCY_TRACKING 1 #define CONFIG_EXTERNAL_BUILD 1 #define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/mac/x64/vpx_config.asm b/media/libvpx/config/mac/x64/vpx_config.asm --- a/media/libvpx/config/mac/x64/vpx_config.asm +++ b/media/libvpx/config/mac/x64/vpx_config.asm @@ -1,10 +1,11 @@ %define VPX_ARCH_ARM 0 %define VPX_ARCH_MIPS 0 %define VPX_ARCH_X86 0 %define VPX_ARCH_X86_64 1 %define VPX_ARCH_PPC 0 +%define VPX_ARCH_LOONGARCH 0 %define HAVE_NEON 0 %define HAVE_NEON_ASM 0 %define HAVE_MIPS32 0 %define HAVE_DSPR2 0 %define HAVE_MSA 0 @@ -18,10 +19,12 @@ %define HAVE_AVX 1 %define HAVE_AVX2 1 %define HAVE_AVX512 0 %define HAVE_VSX 0 %define HAVE_MMI 0 +%define HAVE_LSX 0 +%define HAVE_LASX 0 %define HAVE_VPX_PORTS 1 %define HAVE_PTHREAD_H 1 %define CONFIG_DEPENDENCY_TRACKING 1 %define CONFIG_EXTERNAL_BUILD 1 %define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/mac/x64/vpx_dsp_rtcd.h b/media/libvpx/config/mac/x64/vpx_dsp_rtcd.h --- a/media/libvpx/config/mac/x64/vpx_dsp_rtcd.h +++ b/media/libvpx/config/mac/x64/vpx_dsp_rtcd.h @@ -381,12 +381,12 @@ int16_t vpx_int_pro_col_c(const uint8_t *ref, const int width); int16_t vpx_int_pro_col_sse2(const uint8_t *ref, const int width); #define vpx_int_pro_col vpx_int_pro_col_sse2 -void vpx_int_pro_row_c(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); -void vpx_int_pro_row_sse2(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_c(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_sse2(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); #define vpx_int_pro_row vpx_int_pro_row_sse2 void vpx_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int stride); void vpx_iwht4x4_16_add_sse2(const tran_low_t *input, uint8_t *dest, int stride); #define vpx_iwht4x4_16_add vpx_iwht4x4_16_add_sse2 @@ -480,87 +480,69 @@ void vpx_post_proc_down_and_across_mb_row_c(unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size); void vpx_post_proc_down_and_across_mb_row_sse2(unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size); #define vpx_post_proc_down_and_across_mb_row vpx_post_proc_down_and_across_mb_row_sse2 -void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vpx_quantize_b)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vpx_quantize_b)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vpx_quantize_b_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vpx_quantize_b_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); unsigned int vpx_sad16x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x16 vpx_sad16x16_sse2 unsigned int vpx_sad16x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x16_avg vpx_sad16x16_avg_sse2 -void vpx_sad16x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x3_ssse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x16x4d vpx_sad16x16x4d_sse2 -void vpx_sad16x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - unsigned int vpx_sad16x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x32 vpx_sad16x32_sse2 unsigned int vpx_sad16x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x32_avg vpx_sad16x32_avg_sse2 -void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x32x4d vpx_sad16x32x4d_sse2 unsigned int vpx_sad16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x8 vpx_sad16x8_sse2 unsigned int vpx_sad16x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x8_avg vpx_sad16x8_avg_sse2 -void vpx_sad16x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x3_ssse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x8x4d vpx_sad16x8x4d_sse2 -void vpx_sad16x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - unsigned int vpx_sad32x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x16)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x16_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x16_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad32x16x4d vpx_sad32x16x4d_sse2 unsigned int vpx_sad32x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -569,18 +551,14 @@ unsigned int vpx_sad32x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x32_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad32x32x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x8_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x32x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x32x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad32x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad32x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x64)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -588,44 +566,36 @@ unsigned int vpx_sad32x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x64_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x64_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x64_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad32x64x4d vpx_sad32x64x4d_sse2 unsigned int vpx_sad4x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x4_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad4x4 vpx_sad4x4_sse2 unsigned int vpx_sad4x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x4_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad4x4_avg vpx_sad4x4_avg_sse2 -void vpx_sad4x4x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad4x4x4d vpx_sad4x4x4d_sse2 -void vpx_sad4x4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - unsigned int vpx_sad4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad4x8 vpx_sad4x8_sse2 unsigned int vpx_sad4x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad4x8_avg vpx_sad4x8_avg_sse2 -void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad4x8x4d vpx_sad4x8x4d_sse2 unsigned int vpx_sad64x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -634,12 +604,12 @@ unsigned int vpx_sad64x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x32_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad64x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad64x32x4d vpx_sad64x32x4d_sse2 unsigned int vpx_sad64x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -648,67 +618,51 @@ unsigned int vpx_sad64x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x64_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x64_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad64x64_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x64x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad64x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x64x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad64x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad8x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x16 vpx_sad8x16_sse2 unsigned int vpx_sad8x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x16_avg vpx_sad8x16_avg_sse2 -void vpx_sad8x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x16x4d vpx_sad8x16x4d_sse2 -void vpx_sad8x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - unsigned int vpx_sad8x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x4_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x4 vpx_sad8x4_sse2 unsigned int vpx_sad8x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x4_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x4_avg vpx_sad8x4_avg_sse2 -void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x4x4d vpx_sad8x4x4d_sse2 unsigned int vpx_sad8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x8 vpx_sad8x8_sse2 unsigned int vpx_sad8x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x8_avg vpx_sad8x8_avg_sse2 -void vpx_sad8x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x8x4d vpx_sad8x8x4d_sse2 -void vpx_sad8x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - int vpx_satd_c(const int16_t *coeff, int length); int vpx_satd_sse2(const int16_t *coeff, int length); int vpx_satd_avx2(const int16_t *coeff, int length); RTCD_EXTERN int (*vpx_satd)(const int16_t *coeff, int length); @@ -1058,58 +1012,34 @@ if (flags & HAS_SSSE3) vpx_quantize_b = vpx_quantize_b_ssse3; if (flags & HAS_AVX) vpx_quantize_b = vpx_quantize_b_avx; vpx_quantize_b_32x32 = vpx_quantize_b_32x32_c; if (flags & HAS_SSSE3) vpx_quantize_b_32x32 = vpx_quantize_b_32x32_ssse3; if (flags & HAS_AVX) vpx_quantize_b_32x32 = vpx_quantize_b_32x32_avx; - vpx_sad16x16x3 = vpx_sad16x16x3_c; - if (flags & HAS_SSE3) vpx_sad16x16x3 = vpx_sad16x16x3_sse3; - if (flags & HAS_SSSE3) vpx_sad16x16x3 = vpx_sad16x16x3_ssse3; - vpx_sad16x16x8 = vpx_sad16x16x8_c; - if (flags & HAS_SSE4_1) vpx_sad16x16x8 = vpx_sad16x16x8_sse4_1; - vpx_sad16x8x3 = vpx_sad16x8x3_c; - if (flags & HAS_SSE3) vpx_sad16x8x3 = vpx_sad16x8x3_sse3; - if (flags & HAS_SSSE3) vpx_sad16x8x3 = vpx_sad16x8x3_ssse3; - vpx_sad16x8x8 = vpx_sad16x8x8_c; - if (flags & HAS_SSE4_1) vpx_sad16x8x8 = vpx_sad16x8x8_sse4_1; vpx_sad32x16 = vpx_sad32x16_sse2; if (flags & HAS_AVX2) vpx_sad32x16 = vpx_sad32x16_avx2; vpx_sad32x16_avg = vpx_sad32x16_avg_sse2; if (flags & HAS_AVX2) vpx_sad32x16_avg = vpx_sad32x16_avg_avx2; vpx_sad32x32 = vpx_sad32x32_sse2; if (flags & HAS_AVX2) vpx_sad32x32 = vpx_sad32x32_avx2; vpx_sad32x32_avg = vpx_sad32x32_avg_sse2; if (flags & HAS_AVX2) vpx_sad32x32_avg = vpx_sad32x32_avg_avx2; vpx_sad32x32x4d = vpx_sad32x32x4d_sse2; if (flags & HAS_AVX2) vpx_sad32x32x4d = vpx_sad32x32x4d_avx2; - vpx_sad32x32x8 = vpx_sad32x32x8_c; - if (flags & HAS_AVX2) vpx_sad32x32x8 = vpx_sad32x32x8_avx2; vpx_sad32x64 = vpx_sad32x64_sse2; if (flags & HAS_AVX2) vpx_sad32x64 = vpx_sad32x64_avx2; vpx_sad32x64_avg = vpx_sad32x64_avg_sse2; if (flags & HAS_AVX2) vpx_sad32x64_avg = vpx_sad32x64_avg_avx2; - vpx_sad4x4x3 = vpx_sad4x4x3_c; - if (flags & HAS_SSE3) vpx_sad4x4x3 = vpx_sad4x4x3_sse3; - vpx_sad4x4x8 = vpx_sad4x4x8_c; - if (flags & HAS_SSE4_1) vpx_sad4x4x8 = vpx_sad4x4x8_sse4_1; vpx_sad64x32 = vpx_sad64x32_sse2; if (flags & HAS_AVX2) vpx_sad64x32 = vpx_sad64x32_avx2; vpx_sad64x32_avg = vpx_sad64x32_avg_sse2; if (flags & HAS_AVX2) vpx_sad64x32_avg = vpx_sad64x32_avg_avx2; vpx_sad64x64 = vpx_sad64x64_sse2; if (flags & HAS_AVX2) vpx_sad64x64 = vpx_sad64x64_avx2; vpx_sad64x64_avg = vpx_sad64x64_avg_sse2; if (flags & HAS_AVX2) vpx_sad64x64_avg = vpx_sad64x64_avg_avx2; vpx_sad64x64x4d = vpx_sad64x64x4d_sse2; if (flags & HAS_AVX2) vpx_sad64x64x4d = vpx_sad64x64x4d_avx2; - vpx_sad8x16x3 = vpx_sad8x16x3_c; - if (flags & HAS_SSE3) vpx_sad8x16x3 = vpx_sad8x16x3_sse3; - vpx_sad8x16x8 = vpx_sad8x16x8_c; - if (flags & HAS_SSE4_1) vpx_sad8x16x8 = vpx_sad8x16x8_sse4_1; - vpx_sad8x8x3 = vpx_sad8x8x3_c; - if (flags & HAS_SSE3) vpx_sad8x8x3 = vpx_sad8x8x3_sse3; - vpx_sad8x8x8 = vpx_sad8x8x8_c; - if (flags & HAS_SSE4_1) vpx_sad8x8x8 = vpx_sad8x8x8_sse4_1; vpx_satd = vpx_satd_sse2; if (flags & HAS_AVX2) vpx_satd = vpx_satd_avx2; vpx_scaled_2d = vpx_scaled_2d_c; if (flags & HAS_SSSE3) vpx_scaled_2d = vpx_scaled_2d_ssse3; vpx_sub_pixel_avg_variance16x16 = vpx_sub_pixel_avg_variance16x16_sse2; diff --git a/media/libvpx/config/win/aarch64/vp8_rtcd.h b/media/libvpx/config/win/aarch64/vp8_rtcd.h --- a/media/libvpx/config/win/aarch64/vp8_rtcd.h +++ b/media/libvpx/config/win/aarch64/vp8_rtcd.h @@ -94,13 +94,10 @@ void vp8_fast_quantize_b_c(struct block *, struct blockd *); void vp8_fast_quantize_b_neon(struct block *, struct blockd *); #define vp8_fast_quantize_b vp8_fast_quantize_b_neon -int vp8_full_search_sad_c(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -#define vp8_full_search_sad vp8_full_search_sad_c - void vp8_loop_filter_bh_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); void vp8_loop_filter_bh_neon(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); #define vp8_loop_filter_bh vp8_loop_filter_bh_neon void vp8_loop_filter_bv_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); diff --git a/media/libvpx/config/win/aarch64/vp9_rtcd.h b/media/libvpx/config/win/aarch64/vp9_rtcd.h --- a/media/libvpx/config/win/aarch64/vp9_rtcd.h +++ b/media/libvpx/config/win/aarch64/vp9_rtcd.h @@ -40,17 +40,20 @@ int vp9_diamond_search_sad_c(const struct macroblock *x, const struct search_site_config *cfg, struct mv *ref_mv, struct mv *best_mv, int search_param, int sad_per_bit, int *num00, const struct vp9_variance_vtable *fn_ptr, const struct mv *center_mv); #define vp9_diamond_search_sad vp9_diamond_search_sad_c void vp9_fht16x16_c(const int16_t *input, tran_low_t *output, int stride, int tx_type); -#define vp9_fht16x16 vp9_fht16x16_c +void vp9_fht16x16_neon(const int16_t *input, tran_low_t *output, int stride, int tx_type); +#define vp9_fht16x16 vp9_fht16x16_neon void vp9_fht4x4_c(const int16_t *input, tran_low_t *output, int stride, int tx_type); -#define vp9_fht4x4 vp9_fht4x4_c +void vp9_fht4x4_neon(const int16_t *input, tran_low_t *output, int stride, int tx_type); +#define vp9_fht4x4 vp9_fht4x4_neon void vp9_fht8x8_c(const int16_t *input, tran_low_t *output, int stride, int tx_type); -#define vp9_fht8x8 vp9_fht8x8_c +void vp9_fht8x8_neon(const int16_t *input, tran_low_t *output, int stride, int tx_type); +#define vp9_fht8x8 vp9_fht8x8_neon void vp9_fwht4x4_c(const int16_t *input, tran_low_t *output, int stride); #define vp9_fwht4x4 vp9_fwht4x4_c void vp9_iht16x16_256_add_c(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); @@ -63,16 +66,16 @@ void vp9_iht8x8_64_add_c(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); void vp9_iht8x8_64_add_neon(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); #define vp9_iht8x8_64_add vp9_iht8x8_64_add_neon -void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); #define vp9_quantize_fp vp9_quantize_fp_neon -void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); #define vp9_quantize_fp_32x32 vp9_quantize_fp_32x32_neon void vp9_scale_and_extend_frame_c(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); void vp9_scale_and_extend_frame_neon(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); #define vp9_scale_and_extend_frame vp9_scale_and_extend_frame_neon diff --git a/media/libvpx/config/win/aarch64/vpx_config.h b/media/libvpx/config/win/aarch64/vpx_config.h --- a/media/libvpx/config/win/aarch64/vpx_config.h +++ b/media/libvpx/config/win/aarch64/vpx_config.h @@ -13,10 +13,11 @@ #define VPX_ARCH_ARM 1 #define VPX_ARCH_MIPS 0 #define VPX_ARCH_X86 0 #define VPX_ARCH_X86_64 0 #define VPX_ARCH_PPC 0 +#define VPX_ARCH_LOONGARCH 0 #define HAVE_NEON 1 #define HAVE_NEON_ASM 0 #define HAVE_MIPS32 0 #define HAVE_DSPR2 0 #define HAVE_MSA 0 @@ -30,10 +31,12 @@ #define HAVE_AVX 0 #define HAVE_AVX2 0 #define HAVE_AVX512 0 #define HAVE_VSX 0 #define HAVE_MMI 0 +#define HAVE_LSX 0 +#define HAVE_LASX 0 #define HAVE_VPX_PORTS 1 #define HAVE_PTHREAD_H 0 #define CONFIG_DEPENDENCY_TRACKING 1 #define CONFIG_EXTERNAL_BUILD 1 #define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/win/aarch64/vpx_config.asm b/media/libvpx/config/win/aarch64/vpx_config.asm --- a/media/libvpx/config/win/aarch64/vpx_config.asm +++ b/media/libvpx/config/win/aarch64/vpx_config.asm @@ -1,13 +1,14 @@ @ This file was created from a .asm file @ using the ads2gas.pl script. - .syntax unified +.syntax unified .equ VPX_ARCH_ARM , 1 .equ VPX_ARCH_MIPS , 0 .equ VPX_ARCH_X86 , 0 .equ VPX_ARCH_X86_64 , 0 .equ VPX_ARCH_PPC , 0 +.equ VPX_ARCH_LOONGARCH , 0 .equ HAVE_NEON , 1 .equ HAVE_NEON_ASM , 0 .equ HAVE_MIPS32 , 0 .equ HAVE_DSPR2 , 0 .equ HAVE_MSA , 0 @@ -21,10 +22,12 @@ .equ HAVE_AVX , 0 .equ HAVE_AVX2 , 0 .equ HAVE_AVX512 , 0 .equ HAVE_VSX , 0 .equ HAVE_MMI , 0 +.equ HAVE_LSX , 0 +.equ HAVE_LASX , 0 .equ HAVE_VPX_PORTS , 1 .equ HAVE_PTHREAD_H , 0 .equ CONFIG_DEPENDENCY_TRACKING , 1 .equ CONFIG_EXTERNAL_BUILD , 1 .equ CONFIG_INSTALL_DOCS , 0 @@ -85,6 +88,6 @@ .equ CONFIG_MISMATCH_DEBUG , 0 .equ CONFIG_FP_MB_STATS , 0 .equ CONFIG_EMULATE_HARDWARE , 0 .equ CONFIG_NON_GREEDY_MV , 0 .equ CONFIG_RATE_CTRL , 0 - .section .note.GNU-stack,"",%progbits + .section .note.GNU-stack,"",%progbits diff --git a/media/libvpx/config/win/aarch64/vpx_dsp_rtcd.h b/media/libvpx/config/win/aarch64/vpx_dsp_rtcd.h --- a/media/libvpx/config/win/aarch64/vpx_dsp_rtcd.h +++ b/media/libvpx/config/win/aarch64/vpx_dsp_rtcd.h @@ -350,12 +350,12 @@ int16_t vpx_int_pro_col_c(const uint8_t *ref, const int width); int16_t vpx_int_pro_col_neon(const uint8_t *ref, const int width); #define vpx_int_pro_col vpx_int_pro_col_neon -void vpx_int_pro_row_c(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); -void vpx_int_pro_row_neon(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_c(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_neon(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); #define vpx_int_pro_row vpx_int_pro_row_neon void vpx_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int stride); #define vpx_iwht4x4_16_add vpx_iwht4x4_16_add_c @@ -425,207 +425,174 @@ #define vpx_mse8x16 vpx_mse8x16_c unsigned int vpx_mse8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse); #define vpx_mse8x8 vpx_mse8x8_c -void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); #define vpx_quantize_b vpx_quantize_b_neon -void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); #define vpx_quantize_b_32x32 vpx_quantize_b_32x32_neon unsigned int vpx_sad16x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x16_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x16 vpx_sad16x16_neon unsigned int vpx_sad16x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x16_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x16_avg vpx_sad16x16_avg_neon -void vpx_sad16x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x16x3 vpx_sad16x16x3_c - -void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x16x4d vpx_sad16x16x4d_neon -void vpx_sad16x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x16x8 vpx_sad16x16x8_c - unsigned int vpx_sad16x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x32_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x32 vpx_sad16x32_neon unsigned int vpx_sad16x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x32_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x32_avg vpx_sad16x32_avg_neon -void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x32x4d vpx_sad16x32x4d_neon unsigned int vpx_sad16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x8_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x8 vpx_sad16x8_neon unsigned int vpx_sad16x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x8_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x8_avg vpx_sad16x8_avg_neon -void vpx_sad16x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x8x3 vpx_sad16x8x3_c - -void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x8x4d vpx_sad16x8x4d_neon -void vpx_sad16x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x8x8 vpx_sad16x8x8_c - unsigned int vpx_sad32x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad32x16 vpx_sad32x16_neon unsigned int vpx_sad32x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x16_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad32x16_avg vpx_sad32x16_avg_neon -void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad32x16x4d vpx_sad32x16x4d_neon unsigned int vpx_sad32x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad32x32 vpx_sad32x32_neon unsigned int vpx_sad32x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x32_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad32x32_avg vpx_sad32x32_avg_neon -void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad32x32x4d vpx_sad32x32x4d_neon -void vpx_sad32x32x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad32x32x8 vpx_sad32x32x8_c - unsigned int vpx_sad32x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad32x64 vpx_sad32x64_neon unsigned int vpx_sad32x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x64_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad32x64_avg vpx_sad32x64_avg_neon -void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad32x64x4d vpx_sad32x64x4d_neon unsigned int vpx_sad4x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x4_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad4x4 vpx_sad4x4_neon unsigned int vpx_sad4x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x4_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad4x4_avg vpx_sad4x4_avg_neon -void vpx_sad4x4x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad4x4x3 vpx_sad4x4x3_c - -void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x4x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad4x4x4d vpx_sad4x4x4d_neon -void vpx_sad4x4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad4x4x8 vpx_sad4x4x8_c - unsigned int vpx_sad4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x8_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad4x8 vpx_sad4x8_neon unsigned int vpx_sad4x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x8_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad4x8_avg vpx_sad4x8_avg_neon -void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad4x8x4d vpx_sad4x8x4d_neon unsigned int vpx_sad64x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad64x32 vpx_sad64x32_neon unsigned int vpx_sad64x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x32_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad64x32_avg vpx_sad64x32_avg_neon -void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad64x32x4d vpx_sad64x32x4d_neon unsigned int vpx_sad64x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad64x64 vpx_sad64x64_neon unsigned int vpx_sad64x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x64_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad64x64_avg vpx_sad64x64_avg_neon -void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad64x64x4d vpx_sad64x64x4d_neon unsigned int vpx_sad8x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x16_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x16 vpx_sad8x16_neon unsigned int vpx_sad8x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x16_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x16_avg vpx_sad8x16_avg_neon -void vpx_sad8x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x16x3 vpx_sad8x16x3_c - -void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x16x4d vpx_sad8x16x4d_neon -void vpx_sad8x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x16x8 vpx_sad8x16x8_c - unsigned int vpx_sad8x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x4_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x4 vpx_sad8x4_neon unsigned int vpx_sad8x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x4_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x4_avg vpx_sad8x4_avg_neon -void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x4x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x4x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x4x4d vpx_sad8x4x4d_neon unsigned int vpx_sad8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x8_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x8 vpx_sad8x8_neon unsigned int vpx_sad8x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x8_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x8_avg vpx_sad8x8_avg_neon -void vpx_sad8x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x8x3 vpx_sad8x8x3_c - -void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x8x4d vpx_sad8x8x4d_neon -void vpx_sad8x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x8x8 vpx_sad8x8x8_c - int vpx_satd_c(const int16_t *coeff, int length); int vpx_satd_neon(const int16_t *coeff, int length); #define vpx_satd vpx_satd_neon void vpx_scaled_2d_c(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h); diff --git a/media/libvpx/config/win/ia32/vp8_rtcd.h b/media/libvpx/config/win/ia32/vp8_rtcd.h --- a/media/libvpx/config/win/ia32/vp8_rtcd.h +++ b/media/libvpx/config/win/ia32/vp8_rtcd.h @@ -121,15 +121,10 @@ void vp8_filter_by_weight8x8_c(unsigned char *src, int src_stride, unsigned char *dst, int dst_stride, int src_weight); void vp8_filter_by_weight8x8_sse2(unsigned char *src, int src_stride, unsigned char *dst, int dst_stride, int src_weight); RTCD_EXTERN void (*vp8_filter_by_weight8x8)(unsigned char *src, int src_stride, unsigned char *dst, int dst_stride, int src_weight); -int vp8_full_search_sad_c(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -int vp8_full_search_sadx3(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -int vp8_full_search_sadx8(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -RTCD_EXTERN int (*vp8_full_search_sad)(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); - void vp8_loop_filter_bh_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); void vp8_loop_filter_bh_sse2(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); RTCD_EXTERN void (*vp8_loop_filter_bh)(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); void vp8_loop_filter_bv_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); @@ -276,13 +271,10 @@ if (flags & HAS_SSSE3) vp8_fast_quantize_b = vp8_fast_quantize_b_ssse3; vp8_filter_by_weight16x16 = vp8_filter_by_weight16x16_c; if (flags & HAS_SSE2) vp8_filter_by_weight16x16 = vp8_filter_by_weight16x16_sse2; vp8_filter_by_weight8x8 = vp8_filter_by_weight8x8_c; if (flags & HAS_SSE2) vp8_filter_by_weight8x8 = vp8_filter_by_weight8x8_sse2; - vp8_full_search_sad = vp8_full_search_sad_c; - if (flags & HAS_SSE3) vp8_full_search_sad = vp8_full_search_sadx3; - if (flags & HAS_SSE4_1) vp8_full_search_sad = vp8_full_search_sadx8; vp8_loop_filter_bh = vp8_loop_filter_bh_c; if (flags & HAS_SSE2) vp8_loop_filter_bh = vp8_loop_filter_bh_sse2; vp8_loop_filter_bv = vp8_loop_filter_bv_c; if (flags & HAS_SSE2) vp8_loop_filter_bv = vp8_loop_filter_bv_sse2; vp8_loop_filter_mbh = vp8_loop_filter_mbh_c; diff --git a/media/libvpx/config/win/ia32/vp9_rtcd.h b/media/libvpx/config/win/ia32/vp9_rtcd.h --- a/media/libvpx/config/win/ia32/vp9_rtcd.h +++ b/media/libvpx/config/win/ia32/vp9_rtcd.h @@ -83,16 +83,16 @@ void vp9_iht8x8_64_add_c(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); void vp9_iht8x8_64_add_sse2(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); RTCD_EXTERN void (*vp9_iht8x8_64_add)(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); -void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vp9_quantize_fp)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vp9_quantize_fp)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); #define vp9_quantize_fp_32x32 vp9_quantize_fp_32x32_c void vp9_scale_and_extend_frame_c(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); void vp9_scale_and_extend_frame_ssse3(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); RTCD_EXTERN void (*vp9_scale_and_extend_frame)(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); diff --git a/media/libvpx/config/win/ia32/vpx_config.h b/media/libvpx/config/win/ia32/vpx_config.h --- a/media/libvpx/config/win/ia32/vpx_config.h +++ b/media/libvpx/config/win/ia32/vpx_config.h @@ -13,10 +13,11 @@ #define VPX_ARCH_ARM 0 #define VPX_ARCH_MIPS 0 #define VPX_ARCH_X86 1 #define VPX_ARCH_X86_64 0 #define VPX_ARCH_PPC 0 +#define VPX_ARCH_LOONGARCH 0 #define HAVE_NEON 0 #define HAVE_NEON_ASM 0 #define HAVE_MIPS32 0 #define HAVE_DSPR2 0 #define HAVE_MSA 0 @@ -30,10 +31,12 @@ #define HAVE_AVX 1 #define HAVE_AVX2 1 #define HAVE_AVX512 0 #define HAVE_VSX 0 #define HAVE_MMI 0 +#define HAVE_LSX 0 +#define HAVE_LASX 0 #define HAVE_VPX_PORTS 1 #define HAVE_PTHREAD_H 0 #define CONFIG_DEPENDENCY_TRACKING 1 #define CONFIG_EXTERNAL_BUILD 1 #define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/win/ia32/vpx_config.asm b/media/libvpx/config/win/ia32/vpx_config.asm --- a/media/libvpx/config/win/ia32/vpx_config.asm +++ b/media/libvpx/config/win/ia32/vpx_config.asm @@ -1,10 +1,11 @@ %define VPX_ARCH_ARM 0 %define VPX_ARCH_MIPS 0 %define VPX_ARCH_X86 1 %define VPX_ARCH_X86_64 0 %define VPX_ARCH_PPC 0 +%define VPX_ARCH_LOONGARCH 0 %define HAVE_NEON 0 %define HAVE_NEON_ASM 0 %define HAVE_MIPS32 0 %define HAVE_DSPR2 0 %define HAVE_MSA 0 @@ -18,10 +19,12 @@ %define HAVE_AVX 1 %define HAVE_AVX2 1 %define HAVE_AVX512 0 %define HAVE_VSX 0 %define HAVE_MMI 0 +%define HAVE_LSX 0 +%define HAVE_LASX 0 %define HAVE_VPX_PORTS 1 %define HAVE_PTHREAD_H 0 %define CONFIG_DEPENDENCY_TRACKING 1 %define CONFIG_EXTERNAL_BUILD 1 %define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/win/ia32/vpx_dsp_rtcd.h b/media/libvpx/config/win/ia32/vpx_dsp_rtcd.h --- a/media/libvpx/config/win/ia32/vpx_dsp_rtcd.h +++ b/media/libvpx/config/win/ia32/vpx_dsp_rtcd.h @@ -379,13 +379,13 @@ int16_t vpx_int_pro_col_c(const uint8_t *ref, const int width); int16_t vpx_int_pro_col_sse2(const uint8_t *ref, const int width); RTCD_EXTERN int16_t (*vpx_int_pro_col)(const uint8_t *ref, const int width); -void vpx_int_pro_row_c(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); -void vpx_int_pro_row_sse2(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); -RTCD_EXTERN void (*vpx_int_pro_row)(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_c(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_sse2(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); +RTCD_EXTERN void (*vpx_int_pro_row)(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); void vpx_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int stride); void vpx_iwht4x4_16_add_sse2(const tran_low_t *input, uint8_t *dest, int stride); RTCD_EXTERN void (*vpx_iwht4x4_16_add)(const tran_low_t *input, uint8_t *dest, int stride); @@ -478,74 +478,56 @@ void vpx_post_proc_down_and_across_mb_row_c(unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size); void vpx_post_proc_down_and_across_mb_row_sse2(unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size); RTCD_EXTERN void (*vpx_post_proc_down_and_across_mb_row)(unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size); -void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vpx_quantize_b)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vpx_quantize_b)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vpx_quantize_b_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vpx_quantize_b_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); unsigned int vpx_sad16x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad16x16)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad16x16_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad16x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x3_ssse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad16x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad16x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad16x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad16x32)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad16x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad16x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad16x8_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad16x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x3_ssse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad16x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad16x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad32x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x16)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -553,13 +535,13 @@ unsigned int vpx_sad32x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x16_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x16_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad32x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad32x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x32)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -567,18 +549,14 @@ unsigned int vpx_sad32x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x32_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad32x32x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x8_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x32x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x32x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad32x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad32x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x64)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -586,45 +564,37 @@ unsigned int vpx_sad32x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x64_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x64_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x64_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad32x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad4x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x4_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad4x4)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x4_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad4x4_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad4x4x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad4x4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad4x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad4x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad4x8_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad4x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad64x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad64x32)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -632,13 +602,13 @@ unsigned int vpx_sad64x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x32_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad64x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad64x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad64x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad64x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad64x64)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -646,66 +616,50 @@ unsigned int vpx_sad64x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x64_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x64_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad64x64_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x64x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad64x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x64x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad64x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad8x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad8x16)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad8x16_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad8x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad8x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad8x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad8x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x4_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad8x4)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x4_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad8x4_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad8x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad8x8_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad8x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad8x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad8x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); int vpx_satd_c(const int16_t *coeff, int length); int vpx_satd_sse2(const int16_t *coeff, int length); int vpx_satd_avx2(const int16_t *coeff, int length); RTCD_EXTERN int (*vpx_satd)(const int16_t *coeff, int length); @@ -1211,34 +1165,24 @@ if (flags & HAS_AVX) vpx_quantize_b_32x32 = vpx_quantize_b_32x32_avx; vpx_sad16x16 = vpx_sad16x16_c; if (flags & HAS_SSE2) vpx_sad16x16 = vpx_sad16x16_sse2; vpx_sad16x16_avg = vpx_sad16x16_avg_c; if (flags & HAS_SSE2) vpx_sad16x16_avg = vpx_sad16x16_avg_sse2; - vpx_sad16x16x3 = vpx_sad16x16x3_c; - if (flags & HAS_SSE3) vpx_sad16x16x3 = vpx_sad16x16x3_sse3; - if (flags & HAS_SSSE3) vpx_sad16x16x3 = vpx_sad16x16x3_ssse3; vpx_sad16x16x4d = vpx_sad16x16x4d_c; if (flags & HAS_SSE2) vpx_sad16x16x4d = vpx_sad16x16x4d_sse2; - vpx_sad16x16x8 = vpx_sad16x16x8_c; - if (flags & HAS_SSE4_1) vpx_sad16x16x8 = vpx_sad16x16x8_sse4_1; vpx_sad16x32 = vpx_sad16x32_c; if (flags & HAS_SSE2) vpx_sad16x32 = vpx_sad16x32_sse2; vpx_sad16x32_avg = vpx_sad16x32_avg_c; if (flags & HAS_SSE2) vpx_sad16x32_avg = vpx_sad16x32_avg_sse2; vpx_sad16x32x4d = vpx_sad16x32x4d_c; if (flags & HAS_SSE2) vpx_sad16x32x4d = vpx_sad16x32x4d_sse2; vpx_sad16x8 = vpx_sad16x8_c; if (flags & HAS_SSE2) vpx_sad16x8 = vpx_sad16x8_sse2; vpx_sad16x8_avg = vpx_sad16x8_avg_c; if (flags & HAS_SSE2) vpx_sad16x8_avg = vpx_sad16x8_avg_sse2; - vpx_sad16x8x3 = vpx_sad16x8x3_c; - if (flags & HAS_SSE3) vpx_sad16x8x3 = vpx_sad16x8x3_sse3; - if (flags & HAS_SSSE3) vpx_sad16x8x3 = vpx_sad16x8x3_ssse3; vpx_sad16x8x4d = vpx_sad16x8x4d_c; if (flags & HAS_SSE2) vpx_sad16x8x4d = vpx_sad16x8x4d_sse2; - vpx_sad16x8x8 = vpx_sad16x8x8_c; - if (flags & HAS_SSE4_1) vpx_sad16x8x8 = vpx_sad16x8x8_sse4_1; vpx_sad32x16 = vpx_sad32x16_c; if (flags & HAS_SSE2) vpx_sad32x16 = vpx_sad32x16_sse2; if (flags & HAS_AVX2) vpx_sad32x16 = vpx_sad32x16_avx2; vpx_sad32x16_avg = vpx_sad32x16_avg_c; if (flags & HAS_SSE2) vpx_sad32x16_avg = vpx_sad32x16_avg_sse2; @@ -1252,12 +1196,10 @@ if (flags & HAS_SSE2) vpx_sad32x32_avg = vpx_sad32x32_avg_sse2; if (flags & HAS_AVX2) vpx_sad32x32_avg = vpx_sad32x32_avg_avx2; vpx_sad32x32x4d = vpx_sad32x32x4d_c; if (flags & HAS_SSE2) vpx_sad32x32x4d = vpx_sad32x32x4d_sse2; if (flags & HAS_AVX2) vpx_sad32x32x4d = vpx_sad32x32x4d_avx2; - vpx_sad32x32x8 = vpx_sad32x32x8_c; - if (flags & HAS_AVX2) vpx_sad32x32x8 = vpx_sad32x32x8_avx2; vpx_sad32x64 = vpx_sad32x64_c; if (flags & HAS_SSE2) vpx_sad32x64 = vpx_sad32x64_sse2; if (flags & HAS_AVX2) vpx_sad32x64 = vpx_sad32x64_avx2; vpx_sad32x64_avg = vpx_sad32x64_avg_c; if (flags & HAS_SSE2) vpx_sad32x64_avg = vpx_sad32x64_avg_sse2; @@ -1266,16 +1208,12 @@ if (flags & HAS_SSE2) vpx_sad32x64x4d = vpx_sad32x64x4d_sse2; vpx_sad4x4 = vpx_sad4x4_c; if (flags & HAS_SSE2) vpx_sad4x4 = vpx_sad4x4_sse2; vpx_sad4x4_avg = vpx_sad4x4_avg_c; if (flags & HAS_SSE2) vpx_sad4x4_avg = vpx_sad4x4_avg_sse2; - vpx_sad4x4x3 = vpx_sad4x4x3_c; - if (flags & HAS_SSE3) vpx_sad4x4x3 = vpx_sad4x4x3_sse3; vpx_sad4x4x4d = vpx_sad4x4x4d_c; if (flags & HAS_SSE2) vpx_sad4x4x4d = vpx_sad4x4x4d_sse2; - vpx_sad4x4x8 = vpx_sad4x4x8_c; - if (flags & HAS_SSE4_1) vpx_sad4x4x8 = vpx_sad4x4x8_sse4_1; vpx_sad4x8 = vpx_sad4x8_c; if (flags & HAS_SSE2) vpx_sad4x8 = vpx_sad4x8_sse2; vpx_sad4x8_avg = vpx_sad4x8_avg_c; if (flags & HAS_SSE2) vpx_sad4x8_avg = vpx_sad4x8_avg_sse2; vpx_sad4x8x4d = vpx_sad4x8x4d_c; @@ -1299,32 +1237,24 @@ if (flags & HAS_AVX2) vpx_sad64x64x4d = vpx_sad64x64x4d_avx2; vpx_sad8x16 = vpx_sad8x16_c; if (flags & HAS_SSE2) vpx_sad8x16 = vpx_sad8x16_sse2; vpx_sad8x16_avg = vpx_sad8x16_avg_c; if (flags & HAS_SSE2) vpx_sad8x16_avg = vpx_sad8x16_avg_sse2; - vpx_sad8x16x3 = vpx_sad8x16x3_c; - if (flags & HAS_SSE3) vpx_sad8x16x3 = vpx_sad8x16x3_sse3; vpx_sad8x16x4d = vpx_sad8x16x4d_c; if (flags & HAS_SSE2) vpx_sad8x16x4d = vpx_sad8x16x4d_sse2; - vpx_sad8x16x8 = vpx_sad8x16x8_c; - if (flags & HAS_SSE4_1) vpx_sad8x16x8 = vpx_sad8x16x8_sse4_1; vpx_sad8x4 = vpx_sad8x4_c; if (flags & HAS_SSE2) vpx_sad8x4 = vpx_sad8x4_sse2; vpx_sad8x4_avg = vpx_sad8x4_avg_c; if (flags & HAS_SSE2) vpx_sad8x4_avg = vpx_sad8x4_avg_sse2; vpx_sad8x4x4d = vpx_sad8x4x4d_c; if (flags & HAS_SSE2) vpx_sad8x4x4d = vpx_sad8x4x4d_sse2; vpx_sad8x8 = vpx_sad8x8_c; if (flags & HAS_SSE2) vpx_sad8x8 = vpx_sad8x8_sse2; vpx_sad8x8_avg = vpx_sad8x8_avg_c; if (flags & HAS_SSE2) vpx_sad8x8_avg = vpx_sad8x8_avg_sse2; - vpx_sad8x8x3 = vpx_sad8x8x3_c; - if (flags & HAS_SSE3) vpx_sad8x8x3 = vpx_sad8x8x3_sse3; vpx_sad8x8x4d = vpx_sad8x8x4d_c; if (flags & HAS_SSE2) vpx_sad8x8x4d = vpx_sad8x8x4d_sse2; - vpx_sad8x8x8 = vpx_sad8x8x8_c; - if (flags & HAS_SSE4_1) vpx_sad8x8x8 = vpx_sad8x8x8_sse4_1; vpx_satd = vpx_satd_c; if (flags & HAS_SSE2) vpx_satd = vpx_satd_sse2; if (flags & HAS_AVX2) vpx_satd = vpx_satd_avx2; vpx_scaled_2d = vpx_scaled_2d_c; if (flags & HAS_SSSE3) vpx_scaled_2d = vpx_scaled_2d_ssse3; diff --git a/media/libvpx/config/win/x64/vp8_rtcd.h b/media/libvpx/config/win/x64/vp8_rtcd.h --- a/media/libvpx/config/win/x64/vp8_rtcd.h +++ b/media/libvpx/config/win/x64/vp8_rtcd.h @@ -121,15 +121,10 @@ void vp8_filter_by_weight8x8_c(unsigned char *src, int src_stride, unsigned char *dst, int dst_stride, int src_weight); void vp8_filter_by_weight8x8_sse2(unsigned char *src, int src_stride, unsigned char *dst, int dst_stride, int src_weight); #define vp8_filter_by_weight8x8 vp8_filter_by_weight8x8_sse2 -int vp8_full_search_sad_c(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -int vp8_full_search_sadx3(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -int vp8_full_search_sadx8(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -RTCD_EXTERN int (*vp8_full_search_sad)(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); - void vp8_loop_filter_bh_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); void vp8_loop_filter_bh_sse2(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); #define vp8_loop_filter_bh vp8_loop_filter_bh_sse2 void vp8_loop_filter_bv_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); @@ -240,13 +235,10 @@ if (flags & HAS_SSSE3) vp8_bilinear_predict8x8 = vp8_bilinear_predict8x8_ssse3; vp8_copy32xn = vp8_copy32xn_sse2; if (flags & HAS_SSE3) vp8_copy32xn = vp8_copy32xn_sse3; vp8_fast_quantize_b = vp8_fast_quantize_b_sse2; if (flags & HAS_SSSE3) vp8_fast_quantize_b = vp8_fast_quantize_b_ssse3; - vp8_full_search_sad = vp8_full_search_sad_c; - if (flags & HAS_SSE3) vp8_full_search_sad = vp8_full_search_sadx3; - if (flags & HAS_SSE4_1) vp8_full_search_sad = vp8_full_search_sadx8; vp8_regular_quantize_b = vp8_regular_quantize_b_sse2; if (flags & HAS_SSE4_1) vp8_regular_quantize_b = vp8_regular_quantize_b_sse4_1; vp8_sixtap_predict16x16 = vp8_sixtap_predict16x16_sse2; if (flags & HAS_SSSE3) vp8_sixtap_predict16x16 = vp8_sixtap_predict16x16_ssse3; vp8_sixtap_predict4x4 = vp8_sixtap_predict4x4_mmx; diff --git a/media/libvpx/config/win/x64/vp9_rtcd.h b/media/libvpx/config/win/x64/vp9_rtcd.h --- a/media/libvpx/config/win/x64/vp9_rtcd.h +++ b/media/libvpx/config/win/x64/vp9_rtcd.h @@ -83,19 +83,19 @@ void vp9_iht8x8_64_add_c(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); void vp9_iht8x8_64_add_sse2(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); #define vp9_iht8x8_64_add vp9_iht8x8_64_add_sse2 -void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vp9_quantize_fp)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vp9_quantize_fp)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vp9_quantize_fp_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vp9_quantize_fp_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); void vp9_scale_and_extend_frame_c(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); void vp9_scale_and_extend_frame_ssse3(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); RTCD_EXTERN void (*vp9_scale_and_extend_frame)(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); diff --git a/media/libvpx/config/win/x64/vpx_config.h b/media/libvpx/config/win/x64/vpx_config.h --- a/media/libvpx/config/win/x64/vpx_config.h +++ b/media/libvpx/config/win/x64/vpx_config.h @@ -13,10 +13,11 @@ #define VPX_ARCH_ARM 0 #define VPX_ARCH_MIPS 0 #define VPX_ARCH_X86 0 #define VPX_ARCH_X86_64 1 #define VPX_ARCH_PPC 0 +#define VPX_ARCH_LOONGARCH 0 #define HAVE_NEON 0 #define HAVE_NEON_ASM 0 #define HAVE_MIPS32 0 #define HAVE_DSPR2 0 #define HAVE_MSA 0 @@ -30,10 +31,12 @@ #define HAVE_AVX 1 #define HAVE_AVX2 1 #define HAVE_AVX512 0 #define HAVE_VSX 0 #define HAVE_MMI 0 +#define HAVE_LSX 0 +#define HAVE_LASX 0 #define HAVE_VPX_PORTS 1 #define HAVE_PTHREAD_H 0 #define CONFIG_DEPENDENCY_TRACKING 1 #define CONFIG_EXTERNAL_BUILD 1 #define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/win/x64/vpx_config.asm b/media/libvpx/config/win/x64/vpx_config.asm --- a/media/libvpx/config/win/x64/vpx_config.asm +++ b/media/libvpx/config/win/x64/vpx_config.asm @@ -1,10 +1,11 @@ %define VPX_ARCH_ARM 0 %define VPX_ARCH_MIPS 0 %define VPX_ARCH_X86 0 %define VPX_ARCH_X86_64 1 %define VPX_ARCH_PPC 0 +%define VPX_ARCH_LOONGARCH 0 %define HAVE_NEON 0 %define HAVE_NEON_ASM 0 %define HAVE_MIPS32 0 %define HAVE_DSPR2 0 %define HAVE_MSA 0 @@ -18,10 +19,12 @@ %define HAVE_AVX 1 %define HAVE_AVX2 1 %define HAVE_AVX512 0 %define HAVE_VSX 0 %define HAVE_MMI 0 +%define HAVE_LSX 0 +%define HAVE_LASX 0 %define HAVE_VPX_PORTS 1 %define HAVE_PTHREAD_H 0 %define CONFIG_DEPENDENCY_TRACKING 1 %define CONFIG_EXTERNAL_BUILD 1 %define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/win/x64/vpx_dsp_rtcd.h b/media/libvpx/config/win/x64/vpx_dsp_rtcd.h --- a/media/libvpx/config/win/x64/vpx_dsp_rtcd.h +++ b/media/libvpx/config/win/x64/vpx_dsp_rtcd.h @@ -381,12 +381,12 @@ int16_t vpx_int_pro_col_c(const uint8_t *ref, const int width); int16_t vpx_int_pro_col_sse2(const uint8_t *ref, const int width); #define vpx_int_pro_col vpx_int_pro_col_sse2 -void vpx_int_pro_row_c(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); -void vpx_int_pro_row_sse2(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_c(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_sse2(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); #define vpx_int_pro_row vpx_int_pro_row_sse2 void vpx_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int stride); void vpx_iwht4x4_16_add_sse2(const tran_low_t *input, uint8_t *dest, int stride); #define vpx_iwht4x4_16_add vpx_iwht4x4_16_add_sse2 @@ -480,87 +480,69 @@ void vpx_post_proc_down_and_across_mb_row_c(unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size); void vpx_post_proc_down_and_across_mb_row_sse2(unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size); #define vpx_post_proc_down_and_across_mb_row vpx_post_proc_down_and_across_mb_row_sse2 -void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vpx_quantize_b)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vpx_quantize_b)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vpx_quantize_b_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vpx_quantize_b_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); unsigned int vpx_sad16x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x16 vpx_sad16x16_sse2 unsigned int vpx_sad16x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x16_avg vpx_sad16x16_avg_sse2 -void vpx_sad16x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x3_ssse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x16x4d vpx_sad16x16x4d_sse2 -void vpx_sad16x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - unsigned int vpx_sad16x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x32 vpx_sad16x32_sse2 unsigned int vpx_sad16x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x32_avg vpx_sad16x32_avg_sse2 -void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x32x4d vpx_sad16x32x4d_sse2 unsigned int vpx_sad16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x8 vpx_sad16x8_sse2 unsigned int vpx_sad16x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x8_avg vpx_sad16x8_avg_sse2 -void vpx_sad16x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x3_ssse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x8x4d vpx_sad16x8x4d_sse2 -void vpx_sad16x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - unsigned int vpx_sad32x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x16)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x16_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x16_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad32x16x4d vpx_sad32x16x4d_sse2 unsigned int vpx_sad32x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -569,18 +551,14 @@ unsigned int vpx_sad32x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x32_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad32x32x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x8_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x32x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x32x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad32x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad32x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x64)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -588,44 +566,36 @@ unsigned int vpx_sad32x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x64_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x64_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x64_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad32x64x4d vpx_sad32x64x4d_sse2 unsigned int vpx_sad4x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x4_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad4x4 vpx_sad4x4_sse2 unsigned int vpx_sad4x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x4_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad4x4_avg vpx_sad4x4_avg_sse2 -void vpx_sad4x4x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad4x4x4d vpx_sad4x4x4d_sse2 -void vpx_sad4x4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - unsigned int vpx_sad4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad4x8 vpx_sad4x8_sse2 unsigned int vpx_sad4x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad4x8_avg vpx_sad4x8_avg_sse2 -void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad4x8x4d vpx_sad4x8x4d_sse2 unsigned int vpx_sad64x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -634,12 +604,12 @@ unsigned int vpx_sad64x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x32_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad64x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad64x32x4d vpx_sad64x32x4d_sse2 unsigned int vpx_sad64x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -648,67 +618,51 @@ unsigned int vpx_sad64x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x64_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x64_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad64x64_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x64x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad64x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x64x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad64x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad8x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x16 vpx_sad8x16_sse2 unsigned int vpx_sad8x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x16_avg vpx_sad8x16_avg_sse2 -void vpx_sad8x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x16x4d vpx_sad8x16x4d_sse2 -void vpx_sad8x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - unsigned int vpx_sad8x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x4_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x4 vpx_sad8x4_sse2 unsigned int vpx_sad8x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x4_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x4_avg vpx_sad8x4_avg_sse2 -void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x4x4d vpx_sad8x4x4d_sse2 unsigned int vpx_sad8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x8 vpx_sad8x8_sse2 unsigned int vpx_sad8x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x8_avg vpx_sad8x8_avg_sse2 -void vpx_sad8x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x8x4d vpx_sad8x8x4d_sse2 -void vpx_sad8x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - int vpx_satd_c(const int16_t *coeff, int length); int vpx_satd_sse2(const int16_t *coeff, int length); int vpx_satd_avx2(const int16_t *coeff, int length); RTCD_EXTERN int (*vpx_satd)(const int16_t *coeff, int length); @@ -1058,58 +1012,34 @@ if (flags & HAS_SSSE3) vpx_quantize_b = vpx_quantize_b_ssse3; if (flags & HAS_AVX) vpx_quantize_b = vpx_quantize_b_avx; vpx_quantize_b_32x32 = vpx_quantize_b_32x32_c; if (flags & HAS_SSSE3) vpx_quantize_b_32x32 = vpx_quantize_b_32x32_ssse3; if (flags & HAS_AVX) vpx_quantize_b_32x32 = vpx_quantize_b_32x32_avx; - vpx_sad16x16x3 = vpx_sad16x16x3_c; - if (flags & HAS_SSE3) vpx_sad16x16x3 = vpx_sad16x16x3_sse3; - if (flags & HAS_SSSE3) vpx_sad16x16x3 = vpx_sad16x16x3_ssse3; - vpx_sad16x16x8 = vpx_sad16x16x8_c; - if (flags & HAS_SSE4_1) vpx_sad16x16x8 = vpx_sad16x16x8_sse4_1; - vpx_sad16x8x3 = vpx_sad16x8x3_c; - if (flags & HAS_SSE3) vpx_sad16x8x3 = vpx_sad16x8x3_sse3; - if (flags & HAS_SSSE3) vpx_sad16x8x3 = vpx_sad16x8x3_ssse3; - vpx_sad16x8x8 = vpx_sad16x8x8_c; - if (flags & HAS_SSE4_1) vpx_sad16x8x8 = vpx_sad16x8x8_sse4_1; vpx_sad32x16 = vpx_sad32x16_sse2; if (flags & HAS_AVX2) vpx_sad32x16 = vpx_sad32x16_avx2; vpx_sad32x16_avg = vpx_sad32x16_avg_sse2; if (flags & HAS_AVX2) vpx_sad32x16_avg = vpx_sad32x16_avg_avx2; vpx_sad32x32 = vpx_sad32x32_sse2; if (flags & HAS_AVX2) vpx_sad32x32 = vpx_sad32x32_avx2; vpx_sad32x32_avg = vpx_sad32x32_avg_sse2; if (flags & HAS_AVX2) vpx_sad32x32_avg = vpx_sad32x32_avg_avx2; vpx_sad32x32x4d = vpx_sad32x32x4d_sse2; if (flags & HAS_AVX2) vpx_sad32x32x4d = vpx_sad32x32x4d_avx2; - vpx_sad32x32x8 = vpx_sad32x32x8_c; - if (flags & HAS_AVX2) vpx_sad32x32x8 = vpx_sad32x32x8_avx2; vpx_sad32x64 = vpx_sad32x64_sse2; if (flags & HAS_AVX2) vpx_sad32x64 = vpx_sad32x64_avx2; vpx_sad32x64_avg = vpx_sad32x64_avg_sse2; if (flags & HAS_AVX2) vpx_sad32x64_avg = vpx_sad32x64_avg_avx2; - vpx_sad4x4x3 = vpx_sad4x4x3_c; - if (flags & HAS_SSE3) vpx_sad4x4x3 = vpx_sad4x4x3_sse3; - vpx_sad4x4x8 = vpx_sad4x4x8_c; - if (flags & HAS_SSE4_1) vpx_sad4x4x8 = vpx_sad4x4x8_sse4_1; vpx_sad64x32 = vpx_sad64x32_sse2; if (flags & HAS_AVX2) vpx_sad64x32 = vpx_sad64x32_avx2; vpx_sad64x32_avg = vpx_sad64x32_avg_sse2; if (flags & HAS_AVX2) vpx_sad64x32_avg = vpx_sad64x32_avg_avx2; vpx_sad64x64 = vpx_sad64x64_sse2; if (flags & HAS_AVX2) vpx_sad64x64 = vpx_sad64x64_avx2; vpx_sad64x64_avg = vpx_sad64x64_avg_sse2; if (flags & HAS_AVX2) vpx_sad64x64_avg = vpx_sad64x64_avg_avx2; vpx_sad64x64x4d = vpx_sad64x64x4d_sse2; if (flags & HAS_AVX2) vpx_sad64x64x4d = vpx_sad64x64x4d_avx2; - vpx_sad8x16x3 = vpx_sad8x16x3_c; - if (flags & HAS_SSE3) vpx_sad8x16x3 = vpx_sad8x16x3_sse3; - vpx_sad8x16x8 = vpx_sad8x16x8_c; - if (flags & HAS_SSE4_1) vpx_sad8x16x8 = vpx_sad8x16x8_sse4_1; - vpx_sad8x8x3 = vpx_sad8x8x3_c; - if (flags & HAS_SSE3) vpx_sad8x8x3 = vpx_sad8x8x3_sse3; - vpx_sad8x8x8 = vpx_sad8x8x8_c; - if (flags & HAS_SSE4_1) vpx_sad8x8x8 = vpx_sad8x8x8_sse4_1; vpx_satd = vpx_satd_sse2; if (flags & HAS_AVX2) vpx_satd = vpx_satd_avx2; vpx_scaled_2d = vpx_scaled_2d_c; if (flags & HAS_SSSE3) vpx_scaled_2d = vpx_scaled_2d_ssse3; vpx_sub_pixel_avg_variance16x16 = vpx_sub_pixel_avg_variance16x16_sse2; diff --git a/media/libvpx/libvpx/README b/media/libvpx/libvpx/README --- a/media/libvpx/libvpx/README +++ b/media/libvpx/libvpx/README @@ -101,10 +101,11 @@ x86-solaris-gcc x86-win32-gcc x86-win32-vs14 x86-win32-vs15 x86-win32-vs16 + x86-win32-vs17 x86_64-android-gcc x86_64-darwin9-gcc x86_64-darwin10-gcc x86_64-darwin11-gcc x86_64-darwin12-gcc @@ -122,10 +123,11 @@ x86_64-solaris-gcc x86_64-win64-gcc x86_64-win64-vs14 x86_64-win64-vs15 x86_64-win64-vs16 + x86_64-win64-vs17 generic-gnu The generic-gnu target, in conjunction with the CROSS environment variable, can be used to cross compile architectures that aren't explicitly listed, if the toolchain is a cross GNU (gcc/binutils) toolchain. Other POSIX toolchains diff --git a/media/libvpx/libvpx/build/make/Android.mk b/media/libvpx/libvpx/build/make/Android.mk --- a/media/libvpx/libvpx/build/make/Android.mk +++ b/media/libvpx/libvpx/build/make/Android.mk @@ -164,10 +164,13 @@ -I$(LIBVPX_PATH) \ -I$(ASM_CNV_PATH) \ -I$(ASM_CNV_PATH)/libvpx LOCAL_MODULE := libvpx +LOCAL_LICENSE_KINDS := SPDX-license-identifier-BSD +LOCAL_LICENSE_CONDITIONS := notice +LOCAL_NOTICE_FILE := $(LOCAL_PATH)/../../LICENSE $(LOCAL_PATH)/../../PATENTS ifeq ($(CONFIG_RUNTIME_CPU_DETECT),yes) LOCAL_STATIC_LIBRARIES := cpufeatures endif diff --git a/media/libvpx/libvpx/build/make/Makefile b/media/libvpx/libvpx/build/make/Makefile --- a/media/libvpx/libvpx/build/make/Makefile +++ b/media/libvpx/libvpx/build/make/Makefile @@ -149,10 +149,16 @@ # MIPS $(BUILD_PFX)%_msa.c.d: CFLAGS += -mmsa $(BUILD_PFX)%_msa.c.o: CFLAGS += -mmsa +# LOONGARCH +$(BUILD_PFX)%_lsx.c.d: CFLAGS += -mlsx +$(BUILD_PFX)%_lsx.c.o: CFLAGS += -mlsx +$(BUILD_PFX)%_lasx.c.d: CFLAGS += -mlasx +$(BUILD_PFX)%_lasx.c.o: CFLAGS += -mlasx + $(BUILD_PFX)%.c.d: %.c $(if $(quiet),@echo " [DEP] $@") $(qexec)mkdir -p $(dir $@) $(qexec)$(CC) $(INTERNAL_CFLAGS) $(CFLAGS) -M $< | $(fmt_deps) > $@ diff --git a/media/libvpx/libvpx/build/make/ads2armasm_ms.pl b/media/libvpx/libvpx/build/make/ads2armasm_ms.pl --- a/media/libvpx/libvpx/build/make/ads2armasm_ms.pl +++ b/media/libvpx/libvpx/build/make/ads2armasm_ms.pl @@ -26,11 +26,11 @@ s/^\s*ARM\s*$//; s/AREA\s+\|\|(.*)\|\|/AREA |$1|/; s/qsubaddx/qsax/i; s/qaddsubx/qasx/i; - thumb::FixThumbInstructions($_, 1); + thumb::FixThumbInstructions($_); s/ldrneb/ldrbne/i; s/ldrneh/ldrhne/i; s/^(\s*)ENDP.*/$&\n$1ALIGN 4/; diff --git a/media/libvpx/libvpx/build/make/ads2gas.pl b/media/libvpx/libvpx/build/make/ads2gas.pl --- a/media/libvpx/libvpx/build/make/ads2gas.pl +++ b/media/libvpx/libvpx/build/make/ads2gas.pl @@ -30,209 +30,128 @@ $elf = 0 if ($arg eq "-noelf"); } print "@ This file was created from a .asm file\n"; print "@ using the ads2gas.pl script.\n"; -print "\t.syntax unified\n"; +print ".syntax unified\n"; if ($thumb) { print "\t.thumb\n"; } # Stack of procedure names. @proc_stack = (); while () { - undef $comment; - undef $line; - $comment_char = ";"; - $comment_sub = "@"; - - # Handle comments. - if (/$comment_char/) - { - $comment = ""; - ($line, $comment) = /(.*?)$comment_char(.*)/; - $_ = $line; - } - # Load and store alignment s/@/,:/g; - # Hexadecimal constants prefaced by 0x - s/#&/#0x/g; - - # Convert :OR: to | - s/:OR:/ | /g; - - # Convert :AND: to & - s/:AND:/ & /g; - - # Convert :NOT: to ~ - s/:NOT:/ ~ /g; - - # Convert :SHL: to << - s/:SHL:/ << /g; - - # Convert :SHR: to >> - s/:SHR:/ >> /g; + # Comment character + s/;/@/; # Convert ELSE to .else s/\bELSE\b/.else/g; # Convert ENDIF to .endif s/\bENDIF\b/.endif/g; - # Convert ELSEIF to .elseif - s/\bELSEIF\b/.elseif/g; - - # Convert LTORG to .ltorg - s/\bLTORG\b/.ltorg/g; - - # Convert endfunc to nothing. - s/\bendfunc\b//ig; - - # Convert FUNCTION to nothing. - s/\bFUNCTION\b//g; - s/\bfunction\b//g; - - s/\bENTRY\b//g; - s/\bMSARMASM\b/0/g; - s/^\s+end\s+$//g; - - # Convert IF :DEF:to .if - # gcc doesn't have the ability to do a conditional - # if defined variable that is set by IF :DEF: on - # armasm, so convert it to a normal .if and then - # make sure to define a value elesewhere - if (s/\bIF :DEF:\b/.if /g) - { - s/=/==/g; - } - # Convert IF to .if - if (s/\bIF\b/.if/g) - { + if (s/\bIF\b/.if/g) { s/=+/==/g; } # Convert INCLUDE to .INCLUDE "file" - s/INCLUDE(\s*)(.*)$/.include $1\"$2\"/; - - # Code directive (ARM vs Thumb) - s/CODE([0-9][0-9])/.code $1/; + s/INCLUDE\s?(.*)$/.include \"$1\"/; # No AREA required # But ALIGNs in AREA must be obeyed - s/^\s*AREA.*ALIGN=([0-9])$/.text\n.p2align $1/; + s/^(\s*)\bAREA\b.*ALIGN=([0-9])$/$1.text\n$1.p2align $2/; # If no ALIGN, strip the AREA and align to 4 bytes - s/^\s*AREA.*$/.text\n.p2align 2/; - - # DCD to .word - # This one is for incoming symbols - s/DCD\s+\|(\w*)\|/.long $1/; - - # DCW to .short - s/DCW\s+\|(\w*)\|/.short $1/; - s/DCW(.*)/.short $1/; - - # Constants defined in scope - s/DCD(.*)/.long $1/; - s/DCB(.*)/.byte $1/; + s/^(\s*)\bAREA\b.*$/$1.text\n$1.p2align 2/; - # Make function visible to linker, and make additional symbol with - # prepended underscore + # Make function visible to linker. if ($elf) { - s/EXPORT\s+\|([\$\w]*)\|/.global $1 \n\t.type $1, function/; + s/(\s*)EXPORT\s+\|([\$\w]*)\|/$1.global $2\n$1.type $2, function/; } else { - s/EXPORT\s+\|([\$\w]*)\|/.global $1/; + s/(\s*)EXPORT\s+\|([\$\w]*)\|/$1.global $2/; } - s/IMPORT\s+\|([\$\w]*)\|/.global $1/; - s/EXPORT\s+([\$\w]*)/.global $1/; - s/export\s+([\$\w]*)/.global $1/; - - # No vertical bars required; make additional symbol with prepended - # underscore - s/^\|(\$?\w+)\|/_$1\n\t$1:/g; + # No vertical bars on function names + s/^\|(\$?\w+)\|/$1/g; # Labels need trailing colon -# s/^(\w+)/$1:/ if !/EQU/; - # put the colon at the end of the line in the macro s/^([a-zA-Z_0-9\$]+)/$1:/ if !/EQU/; # ALIGN directive s/\bALIGN\b/.balign/g; if ($thumb) { - # ARM code - we force everything to thumb with the declaration in the header - s/\sARM//g; + # ARM code - we force everything to thumb with the declaration in the + # header + s/\bARM\b//g; } else { # ARM code - s/\sARM/.arm/g; + s/\bARM\b/.arm/g; } # push/pop s/(push\s+)(r\d+)/stmdb sp\!, \{$2\}/g; s/(pop\s+)(r\d+)/ldmia sp\!, \{$2\}/g; - # NEON code - s/(vld1.\d+\s+)(q\d+)/$1\{$2\}/g; - s/(vtbl.\d+\s+[^,]+),([^,]+)/$1,\{$2\}/g; - if ($thumb) { - thumb::FixThumbInstructions($_, 0); + thumb::FixThumbInstructions($_); } # eabi_attributes numerical equivalents can be found in the # "ARM IHI 0045C" document. if ($elf) { # REQUIRE8 Stack is required to be 8-byte aligned - s/\sREQUIRE8/.eabi_attribute 24, 1 \@Tag_ABI_align_needed/g; + s/\bREQUIRE8\b/.eabi_attribute 24, 1 \@Tag_ABI_align_needed/g; # PRESERVE8 Stack 8-byte align is preserved - s/\sPRESERVE8/.eabi_attribute 25, 1 \@Tag_ABI_align_preserved/g; + s/\bPRESERVE8\b/.eabi_attribute 25, 1 \@Tag_ABI_align_preserved/g; } else { - s/\sREQUIRE8//; - s/\sPRESERVE8//; + s/\bREQUIRE8\b//; + s/\bPRESERVE8\b//; } # Use PROC and ENDP to give the symbols a .size directive. # This makes them show up properly in debugging tools like gdb and valgrind. - if (/\bPROC\b/) - { + if (/\bPROC\b/) { my $proc; - /^_([\.0-9A-Z_a-z]\w+)\b/; + # Match the function name so it can be stored in $proc + /^([\.0-9A-Z_a-z]\w+)\b/; $proc = $1; push(@proc_stack, $proc) if ($proc); s/\bPROC\b/@ $&/; } - if (/\bENDP\b/) - { + + if (/\bENDP\b/) { my $proc; s/\bENDP\b/@ $&/; $proc = pop(@proc_stack); - $_ = "\t.size $proc, .-$proc".$_ if ($proc and $elf); + $_ = ".size $proc, .-$proc".$_ if ($proc and $elf); } # EQU directive s/(\S+\s+)EQU(\s+\S+)/.equ $1, $2/; # Begin macro definition if (/\bMACRO\b/) { + # Process next line down, which will be the macro definition $_ = ; s/^/.macro/; - s/\$//g; # remove formal param reference - s/;/@/g; # change comment characters + s/\$//g; # Remove $ from the variables in the declaration } - # For macros, use \ to reference formal params - s/\$/\\/g; # End macro definition - s/\bMEND\b/.endm/; # No need to tell it where to stop assembling + s/\$/\\/g; # Use \ to reference formal parameters + # End macro definition + + s/\bMEND\b/.endm/; # No need to tell it where to stop assembling next if /^\s*END\s*$/; + s/[ \t]+$//; print; - print "$comment_sub$comment\n" if defined $comment; } # Mark that this object doesn't need an executable stack. -printf ("\t.section\t.note.GNU-stack,\"\",\%\%progbits\n") if $elf; +printf (" .section .note.GNU-stack,\"\",\%\%progbits\n") if $elf; diff --git a/media/libvpx/libvpx/build/make/ads2gas_apple.pl b/media/libvpx/libvpx/build/make/ads2gas_apple.pl --- a/media/libvpx/libvpx/build/make/ads2gas_apple.pl +++ b/media/libvpx/libvpx/build/make/ads2gas_apple.pl @@ -18,21 +18,18 @@ # Usage: cat inputfile | perl ads2gas_apple.pl > outputfile # print "@ This file was created from a .asm file\n"; print "@ using the ads2gas_apple.pl script.\n\n"; -print "\t.syntax unified\n"; +print ".syntax unified\n"; -my %register_aliases; my %macro_aliases; my @mapping_list = ("\$0", "\$1", "\$2", "\$3", "\$4", "\$5", "\$6", "\$7", "\$8", "\$9"); my @incoming_array; -my @imported_functions; - # Perl trim function to remove whitespace from the start and end of the string sub trim($) { my $string = shift; $string =~ s/^\s+//; @@ -44,159 +41,74 @@ { # Load and store alignment s/@/,:/g; # Comment character - s/;/ @/g; - - # Hexadecimal constants prefaced by 0x - s/#&/#0x/g; - - # Convert :OR: to | - s/:OR:/ | /g; - - # Convert :AND: to & - s/:AND:/ & /g; - - # Convert :NOT: to ~ - s/:NOT:/ ~ /g; - - # Convert :SHL: to << - s/:SHL:/ << /g; - - # Convert :SHR: to >> - s/:SHR:/ >> /g; + s/;/@/; # Convert ELSE to .else s/\bELSE\b/.else/g; # Convert ENDIF to .endif s/\bENDIF\b/.endif/g; - # Convert ELSEIF to .elseif - s/\bELSEIF\b/.elseif/g; - - # Convert LTORG to .ltorg - s/\bLTORG\b/.ltorg/g; - - # Convert IF :DEF:to .if - # gcc doesn't have the ability to do a conditional - # if defined variable that is set by IF :DEF: on - # armasm, so convert it to a normal .if and then - # make sure to define a value elesewhere - if (s/\bIF :DEF:\b/.if /g) - { - s/=/==/g; - } - # Convert IF to .if - if (s/\bIF\b/.if/g) - { - s/=/==/g; + if (s/\bIF\b/.if/g) { + s/=+/==/g; } # Convert INCLUDE to .INCLUDE "file" - s/INCLUDE(\s*)(.*)$/.include $1\"$2\"/; - - # Code directive (ARM vs Thumb) - s/CODE([0-9][0-9])/.code $1/; + s/INCLUDE\s?(.*)$/.include \"$1\"/; # No AREA required # But ALIGNs in AREA must be obeyed - s/^\s*AREA.*ALIGN=([0-9])$/.text\n.p2align $1/; + s/^(\s*)\bAREA\b.*ALIGN=([0-9])$/$1.text\n$1.p2align $2/; # If no ALIGN, strip the AREA and align to 4 bytes - s/^\s*AREA.*$/.text\n.p2align 2/; - - # DCD to .word - # This one is for incoming symbols - s/DCD\s+\|(\w*)\|/.long $1/; + s/^(\s*)\bAREA\b.*$/$1.text\n$1.p2align 2/; - # DCW to .short - s/DCW\s+\|(\w*)\|/.short $1/; - s/DCW(.*)/.short $1/; - - # Constants defined in scope - s/DCD(.*)/.long $1/; - s/DCB(.*)/.byte $1/; - - # Make function visible to linker, and make additional symbol with - # prepended underscore - s/EXPORT\s+\|([\$\w]*)\|/.globl _$1\n\t.globl $1/; + # Make function visible to linker. + s/EXPORT\s+\|([\$\w]*)\|/.globl _$1/; - # Prepend imported functions with _ - if (s/IMPORT\s+\|([\$\w]*)\|/.globl $1/) - { - $function = trim($1); - push(@imported_functions, $function); - } + # No vertical bars on function names + s/^\|(\$?\w+)\|/$1/g; - foreach $function (@imported_functions) - { - s/$function/_$function/; - } + # Labels and functions need a leading underscore and trailing colon + s/^([a-zA-Z_0-9\$]+)/_$1:/ if !/EQU/; - # No vertical bars required; make additional symbol with prepended - # underscore - s/^\|(\$?\w+)\|/_$1\n\t$1:/g; - - # Labels need trailing colon -# s/^(\w+)/$1:/ if !/EQU/; - # put the colon at the end of the line in the macro - s/^([a-zA-Z_0-9\$]+)/$1:/ if !/EQU/; + # Branches need to call the correct, underscored, function + s/^(\s+b[egln]?[teq]?\s+)([a-zA-Z_0-9\$]+)/$1 _$2/ if !/EQU/; # ALIGN directive s/\bALIGN\b/.balign/g; # Strip ARM - s/\sARM/@ ARM/g; + s/\s+ARM//; # Strip REQUIRE8 - #s/\sREQUIRE8/@ REQUIRE8/g; - s/\sREQUIRE8/@ /g; + s/\s+REQUIRE8//; # Strip PRESERVE8 - s/\sPRESERVE8/@ PRESERVE8/g; + s/\s+PRESERVE8//; # Strip PROC and ENDPROC - s/\bPROC\b/@/g; - s/\bENDP\b/@/g; + s/\bPROC\b//g; + s/\bENDP\b//g; # EQU directive - s/(.*)EQU(.*)/.set $1, $2/; + s/(\S+\s+)EQU(\s+\S+)/.equ $1, $2/; # Begin macro definition - if (/\bMACRO\b/) - { + if (/\bMACRO\b/) { # Process next line down, which will be the macro definition $_ = ; - - $trimmed = trim($_); - - # remove commas that are separating list - $trimmed =~ s/,//g; - - # string to array - @incoming_array = split(/\s+/, $trimmed); - - print ".macro @incoming_array[0]\n"; - - # remove the first element, as that is the name of the macro - shift (@incoming_array); - - @macro_aliases{@incoming_array} = @mapping_list; - - next; + s/^/.macro/; + s/\$//g; # Remove $ from the variables in the declaration } - while (($key, $value) = each(%macro_aliases)) - { - $key =~ s/\$/\\\$/; - s/$key\b/$value/g; - } + s/\$/\\/g; # Use \ to reference formal parameters + # End macro definition - # For macros, use \ to reference formal params -# s/\$/\\/g; # End macro definition - s/\bMEND\b/.endm/; # No need to tell it where to stop assembling + s/\bMEND\b/.endm/; # No need to tell it where to stop assembling next if /^\s*END\s*$/; - + s/[ \t]+$//; print; } diff --git a/media/libvpx/libvpx/build/make/configure.sh b/media/libvpx/libvpx/build/make/configure.sh --- a/media/libvpx/libvpx/build/make/configure.sh +++ b/media/libvpx/libvpx/build/make/configure.sh @@ -447,10 +447,21 @@ disable_feature avx512 RTCD_OPTIONS="${RTCD_OPTIONS}--disable-avx512 " fi } +check_inline_asm() { + log check_inline_asm "$@" + name="$1" + code="$2" + shift 2 + disable_feature $name + check_cc "$@" <> config.mk echo "TOOLCHAIN := ${toolchain}" >> config.mk @@ -764,19 +775,25 @@ tgt_isa=mips64 ;; *mips32el*) tgt_isa=mips32 ;; + loongarch32*) + tgt_isa=loongarch32 + ;; + loongarch64*) + tgt_isa=loongarch64 + ;; esac # detect tgt_os case "$gcctarget" in *darwin1[0-9]*) tgt_isa=x86_64 tgt_os=`echo $gcctarget | sed 's/.*\(darwin1[0-9]\).*/\1/'` ;; - *darwin20*) + *darwin2[0-1]*) tgt_isa=`uname -m` tgt_os=`echo $gcctarget | sed 's/.*\(darwin2[0-9]\).*/\1/'` ;; x86_64*mingw32*) tgt_os=win64 @@ -832,10 +849,15 @@ enable_feature mips ;; ppc*) enable_feature ppc ;; + loongarch*) + soft_enable lsx + soft_enable lasx + enable_feature loongarch + ;; esac # PIC is probably what we want when building shared libs enabled shared && soft_enable pic @@ -916,13 +938,13 @@ ;; *-darwin19-*) add_cflags "-mmacosx-version-min=10.15" add_ldflags "-mmacosx-version-min=10.15" ;; - *-darwin20-*) - add_cflags "-mmacosx-version-min=10.16 -arch ${toolchain%%-*}" - add_ldflags "-mmacosx-version-min=10.16 -arch ${toolchain%%-*}" + *-darwin2[0-1]-*) + add_cflags "-arch ${toolchain%%-*}" + add_ldflags "-arch ${toolchain%%-*}" ;; *-iphonesimulator-*) add_cflags "-miphoneos-version-min=${IOS_VERSION_MIN}" add_ldflags "-miphoneos-version-min=${IOS_VERSION_MIN}" iossim_sdk_dir="$(show_darwin_sdk_path iphonesimulator)" @@ -1417,10 +1439,19 @@ *) log "Warning: Unknown os $tgt_os while setting up $AS flags" ;; esac ;; + loongarch*) + link_with_cc=gcc + setup_gnu_toolchain + + enabled lsx && check_inline_asm lsx '"vadd.b $vr0, $vr1, $vr1"' + enabled lsx && soft_enable runtime_cpu_detect + enabled lasx && check_inline_asm lasx '"xvadd.b $xr0, $xr1, $xr1"' + enabled lasx && soft_enable runtime_cpu_detect + ;; *-gcc|generic-gnu) link_with_cc=gcc enable_feature gcc setup_gnu_toolchain ;; @@ -1519,10 +1550,26 @@ fi fi ;; esac + # only for LOONGARCH platforms + case ${toolchain} in + loongarch*) + if enabled big_endian; then + if enabled lsx; then + echo "lsx optimizations are available only for little endian platforms" + disable_feature lsx + fi + if enabled lasx; then + echo "lasx optimizations are available only for little endian platforms" + disable_feature lasx + fi + fi + ;; + esac + # glibc needs these if enabled linux; then add_cflags -D_LARGEFILE_SOURCE add_cflags -D_FILE_OFFSET_BITS=64 fi diff --git a/media/libvpx/libvpx/build/make/gen_msvs_sln.sh b/media/libvpx/libvpx/build/make/gen_msvs_sln.sh --- a/media/libvpx/libvpx/build/make/gen_msvs_sln.sh +++ b/media/libvpx/libvpx/build/make/gen_msvs_sln.sh @@ -23,11 +23,11 @@ files. Options: --help Print this message --out=outfile Redirect output to a file - --ver=version Version (14-16) of visual studio to generate for + --ver=version Version (14-17) of visual studio to generate for --target=isa-os-cc Target specifier EOF exit 1 } @@ -217,10 +217,11 @@ vs_ver="$optval" case $optval in 14) vs_year=2015 ;; 15) vs_year=2017 ;; 16) vs_year=2019 ;; + 17) vs_year=2022 ;; *) die Unrecognized Visual Studio Version in $opt ;; esac ;; --target=*) target="${optval}" ;; @@ -230,11 +231,11 @@ esac done outfile=${outfile:-/dev/stdout} mkoutfile=${mkoutfile:-/dev/stdout} case "${vs_ver}" in - 1[4-6]) + 1[4-7]) # VS has used Format Version 12.00 continuously since vs11. sln_vers="12.00" sln_vers_str="Visual Studio ${vs_year}" ;; esac diff --git a/media/libvpx/libvpx/build/make/gen_msvs_vcxproj.sh b/media/libvpx/libvpx/build/make/gen_msvs_vcxproj.sh --- a/media/libvpx/libvpx/build/make/gen_msvs_vcxproj.sh +++ b/media/libvpx/libvpx/build/make/gen_msvs_vcxproj.sh @@ -168,11 +168,11 @@ --enable-werror) werror=true ;; --ver=*) vs_ver="$optval" case "$optval" in - 1[4-6]) + 1[4-7]) ;; *) die Unrecognized Visual Studio Version in $opt ;; esac ;; @@ -342,10 +342,13 @@ tag_content PlatformToolset v141 fi if [ "$vs_ver" = "16" ]; then tag_content PlatformToolset v142 fi + if [ "$vs_ver" = "17" ]; then + tag_content PlatformToolset v143 + fi tag_content CharacterSet Unicode if [ "$config" = "Release" ]; then tag_content WholeProgramOptimization true fi close_tag PropertyGroup diff --git a/media/libvpx/libvpx/build/make/rtcd.pl b/media/libvpx/libvpx/build/make/rtcd.pl --- a/media/libvpx/libvpx/build/make/rtcd.pl +++ b/media/libvpx/libvpx/build/make/rtcd.pl @@ -385,10 +385,41 @@ #endif EOF common_bottom; } +sub loongarch() { + determine_indirection("c", @ALL_ARCHS); + + # Assign the helper variable for each enabled extension + foreach my $opt (@ALL_ARCHS) { + my $opt_uc = uc $opt; + eval "\$have_${opt}=\"flags & HAS_${opt_uc}\""; + } + + common_top; + print <) { + if (/HAVE_LSX=yes/) { + @ALL_ARCHS = filter("$opts{arch}", qw/lsx/); + last; + } + if (/HAVE_LASX=yes/) { + @ALL_ARCHS = filter("$opts{arch}", qw/lasx/); + last; + } + } + close CONFIG_FILE; + loongarch; } else { unoptimized; } __END__ diff --git a/media/libvpx/libvpx/build/make/thumb.pm b/media/libvpx/libvpx/build/make/thumb.pm --- a/media/libvpx/libvpx/build/make/thumb.pm +++ b/media/libvpx/libvpx/build/make/thumb.pm @@ -9,15 +9,12 @@ ## be found in the AUTHORS file in the root of the source tree. ## package thumb; -sub FixThumbInstructions($$) +sub FixThumbInstructions($) { - my $short_branches = $_[1]; - my $branch_shift_offset = $short_branches ? 1 : 0; - # Write additions with shifts, such as "add r10, r11, lsl #8", # in three operand form, "add r10, r10, r11, lsl #8". s/(add\s+)(r\d+),\s*(r\d+),\s*(lsl #\d+)/$1$2, $2, $3, $4/g; # Convert additions with a non-constant shift into a sequence diff --git a/media/libvpx/libvpx/configure b/media/libvpx/libvpx/configure --- a/media/libvpx/libvpx/configure +++ b/media/libvpx/libvpx/configure @@ -98,10 +98,11 @@ # all_platforms is a list of all supported target platforms. Maintain # alphabetically by architecture, generic-gnu last. all_platforms="${all_platforms} arm64-android-gcc" all_platforms="${all_platforms} arm64-darwin-gcc" all_platforms="${all_platforms} arm64-darwin20-gcc" +all_platforms="${all_platforms} arm64-darwin21-gcc" all_platforms="${all_platforms} arm64-linux-gcc" all_platforms="${all_platforms} arm64-win64-gcc" all_platforms="${all_platforms} arm64-win64-vs15" all_platforms="${all_platforms} armv7-android-gcc" #neon Cortex-A8 all_platforms="${all_platforms} armv7-darwin-gcc" #neon Cortex-A8 @@ -111,10 +112,12 @@ all_platforms="${all_platforms} armv7-win32-gcc" all_platforms="${all_platforms} armv7-win32-vs14" all_platforms="${all_platforms} armv7-win32-vs15" all_platforms="${all_platforms} armv7s-darwin-gcc" all_platforms="${all_platforms} armv8-linux-gcc" +all_platforms="${all_platforms} loongarch32-linux-gcc" +all_platforms="${all_platforms} loongarch64-linux-gcc" all_platforms="${all_platforms} mips32-linux-gcc" all_platforms="${all_platforms} mips64-linux-gcc" all_platforms="${all_platforms} ppc64le-linux-gcc" all_platforms="${all_platforms} sparc-solaris-gcc" all_platforms="${all_platforms} x86-android-gcc" @@ -137,10 +140,11 @@ all_platforms="${all_platforms} x86-solaris-gcc" all_platforms="${all_platforms} x86-win32-gcc" all_platforms="${all_platforms} x86-win32-vs14" all_platforms="${all_platforms} x86-win32-vs15" all_platforms="${all_platforms} x86-win32-vs16" +all_platforms="${all_platforms} x86-win32-vs17" all_platforms="${all_platforms} x86_64-android-gcc" all_platforms="${all_platforms} x86_64-darwin9-gcc" all_platforms="${all_platforms} x86_64-darwin10-gcc" all_platforms="${all_platforms} x86_64-darwin11-gcc" all_platforms="${all_platforms} x86_64-darwin12-gcc" @@ -150,18 +154,20 @@ all_platforms="${all_platforms} x86_64-darwin16-gcc" all_platforms="${all_platforms} x86_64-darwin17-gcc" all_platforms="${all_platforms} x86_64-darwin18-gcc" all_platforms="${all_platforms} x86_64-darwin19-gcc" all_platforms="${all_platforms} x86_64-darwin20-gcc" +all_platforms="${all_platforms} x86_64-darwin21-gcc" all_platforms="${all_platforms} x86_64-iphonesimulator-gcc" all_platforms="${all_platforms} x86_64-linux-gcc" all_platforms="${all_platforms} x86_64-linux-icc" all_platforms="${all_platforms} x86_64-solaris-gcc" all_platforms="${all_platforms} x86_64-win64-gcc" all_platforms="${all_platforms} x86_64-win64-vs14" all_platforms="${all_platforms} x86_64-win64-vs15" all_platforms="${all_platforms} x86_64-win64-vs16" +all_platforms="${all_platforms} x86_64-win64-vs17" all_platforms="${all_platforms} generic-gnu" # all_targets is a list of all targets that can be configured # note that these should be in dependency order for now. all_targets="libs examples tools docs" @@ -233,10 +239,11 @@ arm mips x86 x86_64 ppc + loongarch " ARCH_EXT_LIST_X86=" mmx sse sse2 @@ -248,10 +255,12 @@ avx512 " ARCH_EXT_LIST_LOONGSON=" mmi + lsx + lasx " ARCH_EXT_LIST=" neon neon_asm @@ -618,11 +627,14 @@ if enabled gcc; then enabled werror && check_add_cflags -Werror check_add_cflags -Wall check_add_cflags -Wdeclaration-after-statement check_add_cflags -Wdisabled-optimization + check_add_cflags -Wextra-semi + check_add_cflags -Wextra-semi-stmt check_add_cflags -Wfloat-conversion + check_add_cflags -Wformat=2 check_add_cflags -Wparentheses-equality check_add_cflags -Wpointer-arith check_add_cflags -Wtype-limits check_add_cflags -Wcast-qual check_add_cflags -Wvla diff --git a/media/libvpx/libvpx/examples/postproc.c b/media/libvpx/libvpx/examples/postproc.c --- a/media/libvpx/libvpx/examples/postproc.c +++ b/media/libvpx/libvpx/examples/postproc.c @@ -107,11 +107,11 @@ } else if (frame_cnt % 30 == 16) { vp8_postproc_cfg_t pp = { VP8_DEBLOCK | VP8_DEMACROBLOCK | VP8_MFQE, 4, 0 }; if (vpx_codec_control(&codec, VP8_SET_POSTPROC, &pp)) die_codec(&codec, "Failed to turn on postproc."); - }; + } // Decode the frame with 15ms deadline if (vpx_codec_decode(&codec, frame, (unsigned int)frame_size, NULL, 15000)) die_codec(&codec, "Failed to decode frame"); diff --git a/media/libvpx/libvpx/examples/svc_encodeframe.c b/media/libvpx/libvpx/examples/svc_encodeframe.c --- a/media/libvpx/libvpx/examples/svc_encodeframe.c +++ b/media/libvpx/libvpx/examples/svc_encodeframe.c @@ -19,10 +19,11 @@ #include #include #include #include #define VPX_DISABLE_CTRL_TYPECHECKS 1 +#include "../tools_common.h" #include "./vpx_config.h" #include "./svc_context.h" #include "vpx/vp8cx.h" #include "vpx/vpx_encoder.h" #include "vpx_mem/vpx_mem.h" @@ -93,12 +94,13 @@ static const SvcInternal_t *get_const_svc_internal(const SvcContext *svc_ctx) { if (svc_ctx == NULL) return NULL; return (const SvcInternal_t *)svc_ctx->internal; } -static int svc_log(SvcContext *svc_ctx, SVC_LOG_LEVEL level, const char *fmt, - ...) { +static VPX_TOOLS_FORMAT_PRINTF(3, 4) int svc_log(SvcContext *svc_ctx, + SVC_LOG_LEVEL level, + const char *fmt, ...) { char buf[512]; int retval = 0; va_list ap; if (level > svc_ctx->log_level) { @@ -262,11 +264,11 @@ for (i = 0; i < svc_ctx->spatial_layers; ++i) alt_ref_enabled += si->enable_auto_alt_ref[i]; if (alt_ref_enabled > REF_FRAMES - svc_ctx->spatial_layers) { svc_log(svc_ctx, SVC_LOG_ERROR, "svc: auto alt ref: Maxinum %d(REF_FRAMES - layers) layers could" - "enabled auto alt reference frame, but % layers are enabled\n", + "enabled auto alt reference frame, but %d layers are enabled\n", REF_FRAMES - svc_ctx->spatial_layers, alt_ref_enabled); res = VPX_CODEC_INVALID_PARAM; } return res; @@ -454,14 +456,15 @@ if (svc_ctx->temporal_layers < 1) svc_ctx->temporal_layers = 1; if (svc_ctx->temporal_layers > VPX_TS_MAX_LAYERS) svc_ctx->temporal_layers = VPX_TS_MAX_LAYERS; if (svc_ctx->temporal_layers * svc_ctx->spatial_layers > VPX_MAX_LAYERS) { - svc_log(svc_ctx, SVC_LOG_ERROR, - "spatial layers * temporal layers exceeds the maximum number of " - "allowed layers of %d\n", - svc_ctx->spatial_layers * svc_ctx->temporal_layers, VPX_MAX_LAYERS); + svc_log( + svc_ctx, SVC_LOG_ERROR, + "spatial layers * temporal layers (%d) exceeds the maximum number of " + "allowed layers of %d\n", + svc_ctx->spatial_layers * svc_ctx->temporal_layers, VPX_MAX_LAYERS); return VPX_CODEC_INVALID_PARAM; } res = assign_layer_bitrates(svc_ctx, enc_cfg); if (res != VPX_CODEC_OK) { svc_log(svc_ctx, SVC_LOG_ERROR, diff --git a/media/libvpx/libvpx/examples/twopass_encoder.c b/media/libvpx/libvpx/examples/twopass_encoder.c --- a/media/libvpx/libvpx/examples/twopass_encoder.c +++ b/media/libvpx/libvpx/examples/twopass_encoder.c @@ -219,11 +219,11 @@ if (w <= 0 || h <= 0 || (w % 2) != 0 || (h % 2) != 0) die("Invalid frame size: %dx%d", w, h); if (!vpx_img_alloc(&raw, VPX_IMG_FMT_I420, w, h, 1)) - die("Failed to allocate image", w, h); + die("Failed to allocate image (%dx%d)", w, h); printf("Using %s\n", vpx_codec_iface_name(encoder->codec_interface())); // Configuration res = vpx_codec_enc_config_default(encoder->codec_interface(), &cfg, 0); diff --git a/media/libvpx/libvpx/examples/vp8_multi_resolution_encoder.c b/media/libvpx/libvpx/examples/vp8_multi_resolution_encoder.c --- a/media/libvpx/libvpx/examples/vp8_multi_resolution_encoder.c +++ b/media/libvpx/libvpx/examples/vp8_multi_resolution_encoder.c @@ -350,11 +350,11 @@ width = (int)strtol(argv[1], NULL, 0); height = (int)strtol(argv[2], NULL, 0); framerate = (int)strtol(argv[3], NULL, 0); if (width < 16 || width % 2 || height < 16 || height % 2) - die("Invalid resolution: %ldx%ld", width, height); + die("Invalid resolution: %dx%d", width, height); /* Open input video file for encoding */ if (!(infile = fopen(argv[4], "rb"))) die("Failed to open %s for reading", argv[4]); @@ -378,11 +378,11 @@ for (i = 0; i < NUM_ENCODERS; i++) { num_temporal_layers[i] = (int)strtol(argv[2 * NUM_ENCODERS + 5 + i], NULL, 0); if (num_temporal_layers[i] < 1 || num_temporal_layers[i] > 3) die("Invalid temporal layers: %d, Must be 1, 2, or 3. \n", - num_temporal_layers); + num_temporal_layers[i]); } /* Open file to write out each spatially downsampled input stream. */ for (i = 0; i < NUM_ENCODERS - 1; i++) { // Highest resoln is encoder 0. @@ -466,11 +466,11 @@ cfg[2].g_threads = 1; /* Allocate image for each encoder */ for (i = 0; i < NUM_ENCODERS; i++) if (!vpx_img_alloc(&raw[i], VPX_IMG_FMT_I420, cfg[i].g_w, cfg[i].g_h, 32)) - die("Failed to allocate image", cfg[i].g_w, cfg[i].g_h); + die("Failed to allocate image (%dx%d)", cfg[i].g_w, cfg[i].g_h); if (raw[0].stride[VPX_PLANE_Y] == (int)raw[0].d_w) read_frame_p = mulres_read_frame; else read_frame_p = mulres_read_frame_by_row; diff --git a/media/libvpx/libvpx/examples/vp9_spatial_svc_encoder.c b/media/libvpx/libvpx/examples/vp9_spatial_svc_encoder.c --- a/media/libvpx/libvpx/examples/vp9_spatial_svc_encoder.c +++ b/media/libvpx/libvpx/examples/vp9_spatial_svc_encoder.c @@ -577,11 +577,12 @@ ref_frame_config->gld_fb_idx[sl] = 0; } ref_frame_config->alt_fb_idx[sl] = 0; } else if (tl == 1) { ref_frame_config->lst_fb_idx[sl] = sl; - ref_frame_config->gld_fb_idx[sl] = num_spatial_layers + sl - 1; + ref_frame_config->gld_fb_idx[sl] = + (sl == 0) ? 0 : num_spatial_layers + sl - 1; ref_frame_config->alt_fb_idx[sl] = num_spatial_layers + sl; } // Set the reference and update flags. if (!tl) { if (!sl) { diff --git a/media/libvpx/libvpx/examples/vpx_temporal_svc_encoder.c b/media/libvpx/libvpx/examples/vpx_temporal_svc_encoder.c --- a/media/libvpx/libvpx/examples/vpx_temporal_svc_encoder.c +++ b/media/libvpx/libvpx/examples/vpx_temporal_svc_encoder.c @@ -28,11 +28,11 @@ #include "../tools_common.h" #include "../video_writer.h" #define ROI_MAP 0 -#define zero(Dest) memset(&(Dest), 0, sizeof(Dest)); +#define zero(Dest) memset(&(Dest), 0, sizeof(Dest)) static const char *exec_name; void usage_exit(void) { exit(EXIT_FAILURE); } @@ -238,10 +238,42 @@ roi->roi_map[i * roi->cols + j] = 1; } } } } + +static void set_roi_skip_map(vpx_codec_enc_cfg_t *cfg, vpx_roi_map_t *roi, + int *skip_map, int *prev_mask_map, int frame_num) { + const int block_size = 8; + unsigned int i, j; + roi->rows = (cfg->g_h + block_size - 1) / block_size; + roi->cols = (cfg->g_w + block_size - 1) / block_size; + zero(roi->skip); + zero(roi->delta_q); + zero(roi->delta_lf); + memset(roi->ref_frame, -1, sizeof(roi->ref_frame)); + roi->ref_frame[1] = 1; + // Use segment 3 for skip. + roi->skip[3] = 1; + roi->roi_map = + (uint8_t *)calloc(roi->rows * roi->cols, sizeof(*roi->roi_map)); + for (i = 0; i < roi->rows; ++i) { + for (j = 0; j < roi->cols; ++j) { + const int idx = i * roi->cols + j; + // Use segment 3 for skip. + // prev_mask_map keeps track of blocks that have been stably on segment 3 + // for the past 10 frames. Only skip when the block is on segment 3 in + // both current map and prev_mask_map. + if (skip_map[idx] == 1 && prev_mask_map[idx] == 1) roi->roi_map[idx] = 3; + // Reset it every 10 frames so it doesn't propagate for too many frames. + if (frame_num % 10 == 0) + prev_mask_map[idx] = skip_map[idx]; + else if (prev_mask_map[idx] == 1 && skip_map[idx] == 0) + prev_mask_map[idx] = 0; + } + } +} #endif // Temporal scaling parameters: // NOTE: The 3 prediction frames cannot be used interchangeably due to // differences in the way they are handled throughout the code. The @@ -572,10 +604,27 @@ break; } } } +#if ROI_MAP +static void read_mask(FILE *mask_file, int *seg_map) { + int mask_rows, mask_cols, i, j; + int *map_start = seg_map; + fscanf(mask_file, "%d %d\n", &mask_cols, &mask_rows); + for (i = 0; i < mask_rows; i++) { + for (j = 0; j < mask_cols; j++) { + fscanf(mask_file, "%d ", &seg_map[j]); + // reverse the bit + seg_map[j] = 1 - seg_map[j]; + } + seg_map += mask_cols; + } + seg_map = map_start; +} +#endif + int main(int argc, char **argv) { VpxVideoWriter *outfile[VPX_TS_MAX_LAYERS] = { NULL }; vpx_codec_ctx_t codec; vpx_codec_enc_cfg_t cfg; int frame_cnt = 0; @@ -611,11 +660,18 @@ const int min_args = min_args_base; #endif // CONFIG_VP9_HIGHBITDEPTH double sum_bitrate = 0.0; double sum_bitrate2 = 0.0; double framerate = 30.0; - +#if ROI_MAP + FILE *mask_file = NULL; + int block_size = 8; + int mask_rows = 0; + int mask_cols = 0; + int *mask_map; + int *prev_mask_map; +#endif zero(rc.layer_target_bitrate); memset(&layer_id, 0, sizeof(vpx_svc_layer_id_t)); memset(&input_ctx, 0, sizeof(input_ctx)); /* Setup default input stream settings */ input_ctx.framerate.numerator = 30; @@ -655,13 +711,19 @@ layering_mode = (int)strtol(argv[12], NULL, 0); if (layering_mode < 0 || layering_mode > 13) { die("Invalid layering mode (0..12) %s", argv[12]); } +#if ROI_MAP + if (argc != min_args + mode_to_num_layers[layering_mode] + 1) { + die("Invalid number of arguments"); + } +#else if (argc != min_args + mode_to_num_layers[layering_mode]) { die("Invalid number of arguments"); } +#endif input_ctx.filename = argv[1]; open_input_file(&input_ctx); #if CONFIG_VP9_HIGHBITDEPTH @@ -685,18 +747,18 @@ if (input_ctx.file_type != FILE_TYPE_Y4M) { if (!vpx_img_alloc( &raw, bit_depth == VPX_BITS_8 ? VPX_IMG_FMT_I420 : VPX_IMG_FMT_I42016, width, height, 32)) { - die("Failed to allocate image", width, height); + die("Failed to allocate image (%dx%d)", width, height); } } #else // Y4M reader has its own allocation. if (input_ctx.file_type != FILE_TYPE_Y4M) { if (!vpx_img_alloc(&raw, VPX_IMG_FMT_I420, width, height, 32)) { - die("Failed to allocate image", width, height); + die("Failed to allocate image (%dx%d)", width, height); } } #endif // CONFIG_VP9_HIGHBITDEPTH // Populate encoder configuration. @@ -815,21 +877,27 @@ #else if (vpx_codec_enc_init(&codec, encoder->codec_interface(), &cfg, 0)) #endif // CONFIG_VP9_HIGHBITDEPTH die("Failed to initialize encoder"); +#if ROI_MAP + mask_rows = (cfg.g_h + block_size - 1) / block_size; + mask_cols = (cfg.g_w + block_size - 1) / block_size; + mask_map = (int *)calloc(mask_rows * mask_cols, sizeof(*mask_map)); + prev_mask_map = (int *)calloc(mask_rows * mask_cols, sizeof(*mask_map)); +#endif + if (strncmp(encoder->name, "vp8", 3) == 0) { vpx_codec_control(&codec, VP8E_SET_CPUUSED, -speed); vpx_codec_control(&codec, VP8E_SET_NOISE_SENSITIVITY, kVp8DenoiserOff); vpx_codec_control(&codec, VP8E_SET_STATIC_THRESHOLD, 1); vpx_codec_control(&codec, VP8E_SET_GF_CBR_BOOST_PCT, 0); #if ROI_MAP set_roi_map(encoder->name, &cfg, &roi); if (vpx_codec_control(&codec, VP8E_SET_ROI_MAP, &roi)) die_codec(&codec, "Failed to set ROI map"); #endif - } else if (strncmp(encoder->name, "vp9", 3) == 0) { vpx_svc_extra_cfg_t svc_params; memset(&svc_params, 0, sizeof(svc_params)); vpx_codec_control(&codec, VP9E_SET_POSTENCODE_DROP, 0); vpx_codec_control(&codec, VP9E_SET_DISABLE_OVERSHOOT_MAXQ_CBR, 0); @@ -841,16 +909,11 @@ vpx_codec_control(&codec, VP9E_SET_NOISE_SENSITIVITY, kVp9DenoiserOff); vpx_codec_control(&codec, VP8E_SET_STATIC_THRESHOLD, 1); vpx_codec_control(&codec, VP9E_SET_TUNE_CONTENT, 0); vpx_codec_control(&codec, VP9E_SET_TILE_COLUMNS, get_msb(cfg.g_threads)); vpx_codec_control(&codec, VP9E_SET_DISABLE_LOOPFILTER, 0); -#if ROI_MAP - set_roi_map(encoder->name, &cfg, &roi); - if (vpx_codec_control(&codec, VP9E_SET_ROI_MAP, &roi)) - die_codec(&codec, "Failed to set ROI map"); - vpx_codec_control(&codec, VP9E_SET_AQ_MODE, 0); -#endif + if (cfg.g_threads > 1) vpx_codec_control(&codec, VP9E_SET_ROW_MT, 1); else vpx_codec_control(&codec, VP9E_SET_ROW_MT, 0); if (vpx_codec_control(&codec, VP9E_SET_SVC, layering_mode > 0 ? 1 : 0)) @@ -879,10 +942,13 @@ frame_avail = 1; while (frame_avail || got_data) { struct vpx_usec_timer timer; vpx_codec_iter_t iter = NULL; const vpx_codec_cx_pkt_t *pkt; +#if ROI_MAP + char mask_file_name[255]; +#endif // Update the temporal layer_id. No spatial layers in this test. layer_id.spatial_layer_id = 0; layer_id.temporal_layer_id = cfg.ts_layer_id[frame_cnt % cfg.ts_periodicity]; layer_id.temporal_layer_id_per_spatial[0] = layer_id.temporal_layer_id; @@ -892,10 +958,23 @@ vpx_codec_control(&codec, VP8E_SET_TEMPORAL_LAYER_ID, layer_id.temporal_layer_id); } flags = layer_flags[frame_cnt % flag_periodicity]; if (layering_mode == 0) flags = 0; +#if ROI_MAP + snprintf(mask_file_name, sizeof(mask_file_name), "%s%05d.txt", + argv[argc - 1], frame_cnt); + mask_file = fopen(mask_file_name, "r"); + if (mask_file != NULL) { + read_mask(mask_file, mask_map); + fclose(mask_file); + // set_roi_map(encoder->name, &cfg, &roi); + set_roi_skip_map(&cfg, &roi, mask_map, prev_mask_map, frame_cnt); + if (vpx_codec_control(&codec, VP9E_SET_ROI_MAP, &roi)) + die_codec(&codec, "Failed to set ROI map"); + } +#endif frame_avail = read_frame(&input_ctx, &raw); if (frame_avail) ++rc.layer_input_frames[layer_id.temporal_layer_id]; vpx_usec_timer_start(&timer); if (vpx_codec_encode(&codec, frame_avail ? &raw : NULL, pts, 1, flags, VPX_DL_REALTIME)) { @@ -961,10 +1040,14 @@ } } ++frame_cnt; pts += frame_duration; } +#if ROI_MAP + free(mask_map); + free(prev_mask_map); +#endif close_input_file(&input_ctx); printout_rate_control_summary(&rc, &cfg, frame_cnt); printf("\n"); printf("Frame cnt and encoding time/FPS stats for encoding: %d %f %f \n", frame_cnt, 1000 * (float)cx_time / (double)(frame_cnt * 1000000), diff --git a/media/libvpx/libvpx/libs.mk b/media/libvpx/libvpx/libs.mk --- a/media/libvpx/libvpx/libs.mk +++ b/media/libvpx/libvpx/libs.mk @@ -92,19 +92,32 @@ INSTALL-LIBS-yes += include/vpx/vp8.h include/vpx/vp8cx.h INSTALL-LIBS-yes += include/vpx/vpx_ext_ratectrl.h INSTALL_MAPS += include/vpx/% $(SRC_PATH_BARE)/$(VP9_PREFIX)/% CODEC_DOC_SRCS += vpx/vp8.h vpx/vp8cx.h vpx/vpx_ext_ratectrl.h CODEC_DOC_SECTIONS += vp9 vp9_encoder +endif - RC_RTC_SRCS := $(addprefix $(VP9_PREFIX),$(call enabled,VP9_CX_SRCS)) - RC_RTC_SRCS += $(VP9_PREFIX)vp9cx.mk vpx/vp8.h vpx/vp8cx.h - RC_RTC_SRCS += vpx/vpx_ext_ratectrl.h +RC_RTC_SRCS := vpx/vp8.h vpx/vp8cx.h +RC_RTC_SRCS += vpx/vpx_ext_ratectrl.h +RC_RTC_SRCS += vpx/internal/vpx_ratectrl_rtc.h +ifeq ($(CONFIG_VP9_ENCODER),yes) + VP9_PREFIX=vp9/ + RC_RTC_SRCS += $(addprefix $(VP9_PREFIX),$(call enabled,VP9_CX_SRCS)) + RC_RTC_SRCS += $(VP9_PREFIX)vp9cx.mk RC_RTC_SRCS += $(VP9_PREFIX)ratectrl_rtc.cc RC_RTC_SRCS += $(VP9_PREFIX)ratectrl_rtc.h INSTALL-SRCS-$(CONFIG_CODEC_SRCS) += $(VP9_PREFIX)ratectrl_rtc.cc INSTALL-SRCS-$(CONFIG_CODEC_SRCS) += $(VP9_PREFIX)ratectrl_rtc.h endif +ifeq ($(CONFIG_VP8_ENCODER),yes) + VP8_PREFIX=vp8/ + RC_RTC_SRCS += $(addprefix $(VP8_PREFIX),$(call enabled,VP8_CX_SRCS)) + RC_RTC_SRCS += $(VP8_PREFIX)vp8_ratectrl_rtc.cc + RC_RTC_SRCS += $(VP8_PREFIX)vp8_ratectrl_rtc.h + INSTALL-SRCS-$(CONFIG_CODEC_SRCS) += $(VP8_PREFIX)vp8_ratectrl_rtc.cc + INSTALL-SRCS-$(CONFIG_CODEC_SRCS) += $(VP8_PREFIX)vp8_ratectrl_rtc.h +endif ifeq ($(CONFIG_VP9_DECODER),yes) VP9_PREFIX=vp9/ include $(SRC_PATH_BARE)/$(VP9_PREFIX)vp9dx.mk CODEC_SRCS-yes += $(addprefix $(VP9_PREFIX),$(call enabled,VP9_DX_SRCS)) @@ -124,11 +137,11 @@ endif ifeq ($(CONFIG_MSVS),yes) CODEC_LIB=$(if $(CONFIG_STATIC_MSVCRT),vpxmt,vpxmd) GTEST_LIB=$(if $(CONFIG_STATIC_MSVCRT),gtestmt,gtestmd) -RC_RTC_LIB=$(if $(CONFIG_STATIC_MSVCRT),vp9rcmt,vp9rcmd) +RC_RTC_LIB=$(if $(CONFIG_STATIC_MSVCRT),vpxrcmt,vpxrcmd) # This variable uses deferred expansion intentionally, since the results of # $(wildcard) may change during the course of the Make. VS_PLATFORMS = $(foreach d,$(wildcard */Release/$(CODEC_LIB).lib),$(word 1,$(subst /, ,$(d)))) endif @@ -247,20 +260,20 @@ PROJECTS-yes += vpx.$(VCPROJ_SFX) vpx.$(VCPROJ_SFX): vpx_config.asm vpx.$(VCPROJ_SFX): $(RTCD) -vp9rc.$(VCPROJ_SFX): \ +vpxrc.$(VCPROJ_SFX): \ VCPROJ_SRCS=$(filter-out $(addprefix %, $(ASM_INCLUDES)), $^) -vp9rc.$(VCPROJ_SFX): $(RC_RTC_SRCS) +vpxrc.$(VCPROJ_SFX): $(RC_RTC_SRCS) @echo " [CREATE] $@" $(qexec)$(GEN_VCPROJ) \ $(if $(CONFIG_SHARED),--dll,--lib) \ --target=$(TOOLCHAIN) \ $(if $(CONFIG_STATIC_MSVCRT),--static-crt) \ - --name=vp9rc \ + --name=vpxrc \ --proj-guid=C26FF952-9494-4838-9A3F-7F3D4F613385 \ --ver=$(CONFIG_VS_VERSION) \ --src-path-bare="$(SRC_PATH_BARE)" \ --out=$@ $(CFLAGS) \ --as=$(AS) \ @@ -273,14 +286,14 @@ vp8/%.c vp8/%.h vp9/%.c vp9/%.cc vp9/%.h vpx/% \ vpx_dsp/%), \ $(VCPROJ_SRCS)) \ --src-path-bare="$(SRC_PATH_BARE)" \ -PROJECTS-yes += vp9rc.$(VCPROJ_SFX) +PROJECTS-yes += vpxrc.$(VCPROJ_SFX) -vp9rc.$(VCPROJ_SFX): vpx_config.asm -vp9rc.$(VCPROJ_SFX): $(RTCD) +vpxrc.$(VCPROJ_SFX): vpx_config.asm +vpxrc.$(VCPROJ_SFX): $(RTCD) endif # ifeq ($(CONFIG_MSVS),yes) else # ifeq ($(CONFIG_EXTERNAL_BUILD),yes) LIBVPX_OBJS=$(call objs, $(filter-out $(ASM_INCLUDES), $(CODEC_SRCS))) OBJS-yes += $(LIBVPX_OBJS) @@ -396,16 +409,15 @@ $(qexec)echo 'Cflags: -I$${includedir}' >> $@ INSTALL-LIBS-yes += $(LIBSUBDIR)/pkgconfig/vpx.pc INSTALL_MAPS += $(LIBSUBDIR)/pkgconfig/%.pc %.pc CLEAN-OBJS += vpx.pc -ifeq ($(CONFIG_VP9_ENCODER),yes) - RC_RTC_OBJS=$(call objs,$(RC_RTC_SRCS)) +ifeq ($(CONFIG_ENCODERS),yes) RC_RTC_OBJS=$(call objs,$(RC_RTC_SRCS)) OBJS-yes += $(RC_RTC_OBJS) - LIBS-yes += $(BUILD_PFX)libvp9rc.a $(BUILD_PFX)libvp9rc_g.a - $(BUILD_PFX)libvp9rc_g.a: $(RC_RTC_OBJS) + LIBS-yes += $(BUILD_PFX)libvpxrc.a $(BUILD_PFX)libvpxrc_g.a + $(BUILD_PFX)libvpxrc_g.a: $(RC_RTC_OBJS) endif ifeq ($(CONFIG_VP9_ENCODER)$(CONFIG_RATE_CTRL),yesyes) SIMPLE_ENCODE_OBJS=$(call objs,$(SIMPLE_ENCODE_SRCS)) OBJS-yes += $(SIMPLE_ENCODE_OBJS) @@ -491,11 +503,11 @@ TEST_INTRA_PRED_SPEED_BIN=./test_intra_pred_speed$(EXE_SFX) TEST_INTRA_PRED_SPEED_SRCS=$(call addprefix_clean,test/,\ $(call enabled,TEST_INTRA_PRED_SPEED_SRCS)) TEST_INTRA_PRED_SPEED_OBJS := $(sort $(call objs,$(TEST_INTRA_PRED_SPEED_SRCS))) -ifeq ($(CONFIG_VP9_ENCODER),yes) +ifeq ($(CONFIG_ENCODERS),yes) RC_INTERFACE_TEST_BIN=./test_rc_interface$(EXE_SFX) RC_INTERFACE_TEST_SRCS=$(call addprefix_clean,test/,\ $(call enabled,RC_INTERFACE_TEST_SRCS)) RC_INTERFACE_TEST_OBJS := $(sort $(call objs,$(RC_INTERFACE_TEST_SRCS))) endif @@ -597,15 +609,15 @@ --out=$@ $(INTERNAL_CFLAGS) $(CFLAGS) \ -I. -I"$(SRC_PATH_BARE)/third_party/googletest/src/include" \ -L. -l$(CODEC_LIB) -l$(GTEST_LIB) $^ endif # TEST_INTRA_PRED_SPEED -ifeq ($(CONFIG_VP9_ENCODER),yes) +ifeq ($(CONFIG_ENCODERS),yes) ifneq ($(strip $(RC_INTERFACE_TEST_OBJS)),) PROJECTS-$(CONFIG_MSVS) += test_rc_interface.$(VCPROJ_SFX) test_rc_interface.$(VCPROJ_SFX): $(RC_INTERFACE_TEST_SRCS) vpx.$(VCPROJ_SFX) \ - vp9rc.$(VCPROJ_SFX) gtest.$(VCPROJ_SFX) + vpxrc.$(VCPROJ_SFX) gtest.$(VCPROJ_SFX) @echo " [CREATE] $@" $(qexec)$(GEN_VCPROJ) \ --exe \ --target=$(TOOLCHAIN) \ --name=test_rc_interface \ @@ -659,23 +671,23 @@ $(eval $(call linkerxx_template,$(TEST_INTRA_PRED_SPEED_BIN), \ $(TEST_INTRA_PRED_SPEED_OBJS) \ -L. -lvpx -lgtest $(extralibs) -lm)) endif # TEST_INTRA_PRED_SPEED -ifeq ($(CONFIG_VP9_ENCODER),yes) +ifeq ($(CONFIG_ENCODERS),yes) ifneq ($(strip $(RC_INTERFACE_TEST_OBJS)),) $(RC_INTERFACE_TEST_OBJS) $(RC_INTERFACE_TEST_OBJS:.o=.d): \ CXXFLAGS += $(GTEST_INCLUDES) OBJS-yes += $(RC_INTERFACE_TEST_OBJS) BINS-yes += $(RC_INTERFACE_TEST_BIN) -$(RC_INTERFACE_TEST_BIN): $(TEST_LIBS) libvp9rc.a +$(RC_INTERFACE_TEST_BIN): $(TEST_LIBS) libvpxrc.a $(eval $(call linkerxx_template,$(RC_INTERFACE_TEST_BIN), \ $(RC_INTERFACE_TEST_OBJS) \ - -L. -lvpx -lgtest -lvp9rc $(extralibs) -lm)) + -L. -lvpx -lgtest -lvpxrc $(extralibs) -lm)) endif # RC_INTERFACE_TEST -endif # CONFIG_VP9_ENCODER +endif # CONFIG_ENCODERS ifneq ($(strip $(SIMPLE_ENCODE_TEST_OBJS)),) $(SIMPLE_ENCODE_TEST_OBJS) $(SIMPLE_ENCODE_TEST_OBJS:.o=.d): \ CXXFLAGS += $(GTEST_INCLUDES) OBJS-yes += $(SIMPLE_ENCODE_TEST_OBJS) diff --git a/media/libvpx/libvpx/rate_hist.c b/media/libvpx/libvpx/rate_hist.c --- a/media/libvpx/libvpx/rate_hist.c +++ b/media/libvpx/libvpx/rate_hist.c @@ -191,44 +191,46 @@ return bucket[big_bucket].count; } static void show_histogram(const struct hist_bucket *bucket, int buckets, int total, int scale) { - const char *pat1, *pat2; + int width1, width2; int i; + if (!buckets) return; assert(bucket != NULL); + assert(buckets > 0); switch ((int)(log(bucket[buckets - 1].high) / log(10)) + 1) { case 1: case 2: - pat1 = "%4d %2s: "; - pat2 = "%4d-%2d: "; + width1 = 4; + width2 = 2; break; case 3: - pat1 = "%5d %3s: "; - pat2 = "%5d-%3d: "; + width1 = 5; + width2 = 3; break; case 4: - pat1 = "%6d %4s: "; - pat2 = "%6d-%4d: "; + width1 = 6; + width2 = 4; break; case 5: - pat1 = "%7d %5s: "; - pat2 = "%7d-%5d: "; + width1 = 7; + width2 = 5; break; case 6: - pat1 = "%8d %6s: "; - pat2 = "%8d-%6d: "; + width1 = 8; + width2 = 6; break; case 7: - pat1 = "%9d %7s: "; - pat2 = "%9d-%7d: "; + width1 = 9; + width2 = 7; break; default: - pat1 = "%12d %10s: "; - pat2 = "%12d-%10d: "; + width1 = 12; + width2 = 10; break; } for (i = 0; i < buckets; i++) { int len; @@ -239,13 +241,14 @@ len = HIST_BAR_MAX * bucket[i].count / scale; if (len < 1) len = 1; assert(len <= HIST_BAR_MAX); if (bucket[i].low == bucket[i].high) - fprintf(stderr, pat1, bucket[i].low, ""); + fprintf(stderr, "%*d %*s: ", width1, bucket[i].low, width2, ""); else - fprintf(stderr, pat2, bucket[i].low, bucket[i].high); + fprintf(stderr, "%*d-%*d: ", width1, bucket[i].low, width2, + bucket[i].high); for (j = 0; j < HIST_BAR_MAX; j++) fprintf(stderr, j < len ? "=" : " "); fprintf(stderr, "\t%5d (%6.2f%%)\n", bucket[i].count, pct); } } diff --git a/media/libvpx/libvpx/test/add_noise_test.cc b/media/libvpx/libvpx/test/add_noise_test.cc --- a/media/libvpx/libvpx/test/add_noise_test.cc +++ b/media/libvpx/libvpx/test/add_noise_test.cc @@ -21,11 +21,10 @@ namespace { static const int kNoiseSize = 3072; -// TODO(jimbankoski): make width and height integers not unsigned. typedef void (*AddNoiseFunc)(uint8_t *start, const int8_t *noise, int blackclamp, int whiteclamp, int width, int height, int pitch); typedef std::tuple AddNoiseTestFPParam; diff --git a/media/libvpx/libvpx/test/android/Android.mk b/media/libvpx/libvpx/test/android/Android.mk --- a/media/libvpx/libvpx/test/android/Android.mk +++ b/media/libvpx/libvpx/test/android/Android.mk @@ -32,10 +32,13 @@ LOCAL_MODULE := gtest LOCAL_C_INCLUDES := $(LOCAL_PATH)/third_party/googletest/src/ LOCAL_C_INCLUDES += $(LOCAL_PATH)/third_party/googletest/src/include/ LOCAL_EXPORT_C_INCLUDES := $(LOCAL_PATH)/third_party/googletest/src/include/ LOCAL_SRC_FILES := ./third_party/googletest/src/src/gtest-all.cc +LOCAL_LICENSE_KINDS := SPDX-license-identifier-BSD +LOCAL_LICENSE_CONDITIONS := notice +LOCAL_NOTICE_FILE := $(LOCAL_PATH)/../../LICENSE $(LOCAL_PATH)/../../PATENTS include $(BUILD_STATIC_LIBRARY) #libvpx_test include $(CLEAR_VARS) LOCAL_ARM_MODE := arm @@ -46,10 +49,13 @@ LOCAL_SHARED_LIBRARIES := vpx else LOCAL_STATIC_LIBRARIES += vpx endif +LOCAL_LICENSE_KINDS := SPDX-license-identifier-BSD +LOCAL_LICENSE_CONDITIONS := notice +LOCAL_NOTICE_FILE := $(LOCAL_PATH)/../../LICENSE $(LOCAL_PATH)/../../PATENTS include $(LOCAL_PATH)/test/test.mk LOCAL_C_INCLUDES := $(BINDINGS_DIR) FILTERED_SRC := $(sort $(filter %.cc %.c, $(LIBVPX_TEST_SRCS-yes))) LOCAL_SRC_FILES := $(addprefix ./test/, $(FILTERED_SRC)) # some test files depend on *_rtcd.h, ensure they're generated first. diff --git a/media/libvpx/libvpx/test/comp_avg_pred_test.cc b/media/libvpx/libvpx/test/comp_avg_pred_test.cc --- a/media/libvpx/libvpx/test/comp_avg_pred_test.cc +++ b/media/libvpx/libvpx/test/comp_avg_pred_test.cc @@ -181,6 +181,11 @@ #if HAVE_VSX INSTANTIATE_TEST_SUITE_P(VSX, AvgPredTest, ::testing::Values(&vpx_comp_avg_pred_vsx)); #endif // HAVE_VSX + +#if HAVE_LSX +INSTANTIATE_TEST_SUITE_P(LSX, AvgPredTest, + ::testing::Values(&vpx_comp_avg_pred_lsx)); +#endif // HAVE_LSX } // namespace diff --git a/media/libvpx/libvpx/test/convolve_test.cc b/media/libvpx/libvpx/test/convolve_test.cc --- a/media/libvpx/libvpx/test/convolve_test.cc +++ b/media/libvpx/libvpx/test/convolve_test.cc @@ -1447,10 +1447,23 @@ const ConvolveParam kArrayConvolve8_msa[] = { ALL_SIZES(convolve8_msa) }; INSTANTIATE_TEST_SUITE_P(MSA, ConvolveTest, ::testing::ValuesIn(kArrayConvolve8_msa)); #endif // HAVE_MSA +#if HAVE_LSX +const ConvolveFunctions convolve8_lsx( + vpx_convolve_copy_lsx, vpx_convolve_avg_lsx, vpx_convolve8_horiz_lsx, + vpx_convolve8_avg_horiz_lsx, vpx_convolve8_vert_lsx, + vpx_convolve8_avg_vert_lsx, vpx_convolve8_lsx, vpx_convolve8_avg_lsx, + vpx_scaled_horiz_c, vpx_scaled_avg_horiz_c, vpx_scaled_vert_c, + vpx_scaled_avg_vert_c, vpx_scaled_2d_c, vpx_scaled_avg_2d_c, 0); + +const ConvolveParam kArrayConvolve8_lsx[] = { ALL_SIZES(convolve8_lsx) }; +INSTANTIATE_TEST_SUITE_P(LSX, ConvolveTest, + ::testing::ValuesIn(kArrayConvolve8_lsx)); +#endif // HAVE_LSX + #if HAVE_VSX const ConvolveFunctions convolve8_vsx( vpx_convolve_copy_vsx, vpx_convolve_avg_vsx, vpx_convolve8_horiz_vsx, vpx_convolve8_avg_horiz_vsx, vpx_convolve8_vert_vsx, vpx_convolve8_avg_vert_vsx, vpx_convolve8_vsx, vpx_convolve8_avg_vsx, diff --git a/media/libvpx/libvpx/test/dct16x16_test.cc b/media/libvpx/libvpx/test/dct16x16_test.cc --- a/media/libvpx/libvpx/test/dct16x16_test.cc +++ b/media/libvpx/libvpx/test/dct16x16_test.cc @@ -866,6 +866,13 @@ INSTANTIATE_TEST_SUITE_P( VSX, Trans16x16DCT, ::testing::Values(make_tuple(&vpx_fdct16x16_c, &vpx_idct16x16_256_add_vsx, 0, VPX_BITS_8))); #endif // HAVE_VSX && !CONFIG_VP9_HIGHBITDEPTH && !CONFIG_EMULATE_HARDWARE + +#if HAVE_LSX && !CONFIG_VP9_HIGHBITDEPTH && !CONFIG_EMULATE_HARDWARE +INSTANTIATE_TEST_SUITE_P(LSX, Trans16x16DCT, + ::testing::Values(make_tuple(&vpx_fdct16x16_lsx, + &vpx_idct16x16_256_add_c, + 0, VPX_BITS_8))); +#endif // HAVE_LSX && !CONFIG_VP9_HIGHBITDEPTH && !CONFIG_EMULATE_HARDWARE } // namespace diff --git a/media/libvpx/libvpx/test/dct32x32_test.cc b/media/libvpx/libvpx/test/dct32x32_test.cc --- a/media/libvpx/libvpx/test/dct32x32_test.cc +++ b/media/libvpx/libvpx/test/dct32x32_test.cc @@ -394,6 +394,15 @@ ::testing::Values(make_tuple(&vpx_fdct32x32_c, &vpx_idct32x32_1024_add_vsx, 0, VPX_BITS_8), make_tuple(&vpx_fdct32x32_rd_vsx, &vpx_idct32x32_1024_add_vsx, 1, VPX_BITS_8))); #endif // HAVE_VSX && !CONFIG_VP9_HIGHBITDEPTH && !CONFIG_EMULATE_HARDWARE + +#if HAVE_LSX && !CONFIG_VP9_HIGHBITDEPTH && !CONFIG_EMULATE_HARDWARE +INSTANTIATE_TEST_SUITE_P( + LSX, Trans32x32Test, + ::testing::Values(make_tuple(&vpx_fdct32x32_lsx, + &vpx_idct32x32_1024_add_lsx, 0, VPX_BITS_8), + make_tuple(&vpx_fdct32x32_rd_lsx, + &vpx_idct32x32_1024_add_lsx, 1, VPX_BITS_8))); +#endif // HAVE_LSX && !CONFIG_VP9_HIGHBITDEPTH && !CONFIG_EMULATE_HARDWARE } // namespace diff --git a/media/libvpx/libvpx/test/dct_test.cc b/media/libvpx/libvpx/test/dct_test.cc --- a/media/libvpx/libvpx/test/dct_test.cc +++ b/media/libvpx/libvpx/test/dct_test.cc @@ -584,10 +584,25 @@ INSTANTIATE_TEST_SUITE_P(VSX, TransDCT, ::testing::Values(make_tuple(0, &dct_vsx_func_info, 0, VPX_BITS_8))); #endif // HAVE_VSX && !CONFIG_VP9_HIGHBITDEPTH && +#if HAVE_LSX && !CONFIG_VP9_HIGHBITDEPTH +static const FuncInfo dct_lsx_func_info[2] = { + { &fdct_wrapper, &idct_wrapper, + 16, 1 }, + { &fdct_wrapper, &idct_wrapper, + 32, 1 } +}; + +INSTANTIATE_TEST_SUITE_P( + LSX, TransDCT, + ::testing::Combine(::testing::Range(0, 2), + ::testing::Values(dct_lsx_func_info), + ::testing::Values(0), ::testing::Values(VPX_BITS_8))); +#endif // HAVE_LSX && !CONFIG_VP9_HIGHBITDEPTH + #endif // !CONFIG_EMULATE_HARDWARE /* -------------------------------------------------------------------------- */ class TransHT : public TransTestBase { @@ -639,12 +654,15 @@ 2 }, { &vp9_highbd_fht16x16_c, &highbd_iht_wrapper, 16, 2 }, #endif { &vp9_fht4x4_c, &iht_wrapper, 4, 1 }, + { &vp9_fht4x4_neon, &iht_wrapper, 4, 1 }, { &vp9_fht8x8_c, &iht_wrapper, 8, 1 }, - { &vp9_fht16x16_c, &iht_wrapper, 16, 1 } + { &vp9_fht8x8_neon, &iht_wrapper, 8, 1 }, + { &vp9_fht16x16_c, &iht_wrapper, 16, 1 }, + { &vp9_fht16x16_neon, &iht_wrapper, 16, 1 } }; INSTANTIATE_TEST_SUITE_P( NEON, TransHT, ::testing::Combine( @@ -751,6 +769,7 @@ INSTANTIATE_TEST_SUITE_P(VSX, TransWHT, ::testing::Values(make_tuple(0, &wht_vsx_func_info, 0, VPX_BITS_8))); #endif // HAVE_VSX && !CONFIG_EMULATE_HARDWARE + } // namespace diff --git a/media/libvpx/libvpx/test/encode_api_test.cc b/media/libvpx/libvpx/test/encode_api_test.cc --- a/media/libvpx/libvpx/test/encode_api_test.cc +++ b/media/libvpx/libvpx/test/encode_api_test.cc @@ -8,36 +8,37 @@ * be found in the AUTHORS file in the root of the source tree. */ #include #include +#include #include "third_party/googletest/src/include/gtest/gtest.h" #include "./vpx_config.h" +#include "test/video_source.h" #include "vpx/vp8cx.h" #include "vpx/vpx_encoder.h" namespace { -#define NELEMENTS(x) static_cast(sizeof(x) / sizeof(x[0])) +const vpx_codec_iface_t *kCodecIfaces[] = { +#if CONFIG_VP8_ENCODER + &vpx_codec_vp8_cx_algo, +#endif +#if CONFIG_VP9_ENCODER + &vpx_codec_vp9_cx_algo, +#endif +}; bool IsVP9(const vpx_codec_iface_t *iface) { static const char kVP9Name[] = "WebM Project VP9"; return strncmp(kVP9Name, vpx_codec_iface_name(iface), sizeof(kVP9Name) - 1) == 0; } TEST(EncodeAPI, InvalidParams) { - static const vpx_codec_iface_t *kCodecs[] = { -#if CONFIG_VP8_ENCODER - &vpx_codec_vp8_cx_algo, -#endif -#if CONFIG_VP9_ENCODER - &vpx_codec_vp9_cx_algo, -#endif - }; uint8_t buf[1] = { 0 }; vpx_image_t img; vpx_codec_ctx_t enc; vpx_codec_enc_cfg_t cfg; @@ -56,21 +57,21 @@ vpx_codec_enc_config_default(nullptr, nullptr, 0)); EXPECT_EQ(VPX_CODEC_INVALID_PARAM, vpx_codec_enc_config_default(nullptr, &cfg, 0)); EXPECT_NE(vpx_codec_error(nullptr), nullptr); - for (int i = 0; i < NELEMENTS(kCodecs); ++i) { - SCOPED_TRACE(vpx_codec_iface_name(kCodecs[i])); + for (const auto *iface : kCodecIfaces) { + SCOPED_TRACE(vpx_codec_iface_name(iface)); EXPECT_EQ(VPX_CODEC_INVALID_PARAM, - vpx_codec_enc_init(nullptr, kCodecs[i], nullptr, 0)); + vpx_codec_enc_init(nullptr, iface, nullptr, 0)); EXPECT_EQ(VPX_CODEC_INVALID_PARAM, - vpx_codec_enc_init(&enc, kCodecs[i], nullptr, 0)); + vpx_codec_enc_init(&enc, iface, nullptr, 0)); EXPECT_EQ(VPX_CODEC_INVALID_PARAM, - vpx_codec_enc_config_default(kCodecs[i], &cfg, 1)); + vpx_codec_enc_config_default(iface, &cfg, 1)); - EXPECT_EQ(VPX_CODEC_OK, vpx_codec_enc_config_default(kCodecs[i], &cfg, 0)); - EXPECT_EQ(VPX_CODEC_OK, vpx_codec_enc_init(&enc, kCodecs[i], &cfg, 0)); + EXPECT_EQ(VPX_CODEC_OK, vpx_codec_enc_config_default(iface, &cfg, 0)); + EXPECT_EQ(VPX_CODEC_OK, vpx_codec_enc_init(&enc, iface, &cfg, 0)); EXPECT_EQ(VPX_CODEC_OK, vpx_codec_encode(&enc, nullptr, 0, 0, 0, 0)); EXPECT_EQ(VPX_CODEC_OK, vpx_codec_destroy(&enc)); } } @@ -122,27 +123,18 @@ // Set up 2 spatial streams with 2 temporal layers per stream, and generate // invalid configuration by setting the temporal layer rate allocation // (ts_target_bitrate[]) to 0 for both layers. This should fail independent of // CONFIG_MULTI_RES_ENCODING. TEST(EncodeAPI, MultiResEncode) { - static const vpx_codec_iface_t *kCodecs[] = { -#if CONFIG_VP8_ENCODER - &vpx_codec_vp8_cx_algo, -#endif -#if CONFIG_VP9_ENCODER - &vpx_codec_vp9_cx_algo, -#endif - }; const int width = 1280; const int height = 720; const int width_down = width / 2; const int height_down = height / 2; const int target_bitrate = 1000; const int framerate = 30; - for (int c = 0; c < NELEMENTS(kCodecs); ++c) { - const vpx_codec_iface_t *const iface = kCodecs[c]; + for (const auto *iface : kCodecIfaces) { vpx_codec_ctx_t enc[2]; vpx_codec_enc_cfg_t cfg[2]; vpx_rational_t dsf[2] = { { 2, 1 }, { 2, 1 } }; memset(enc, 0, sizeof(enc)); @@ -308,6 +300,64 @@ EXPECT_EQ(vpx_codec_destroy(&enc), VPX_CODEC_OK); } } +void InitCodec(const vpx_codec_iface_t &iface, int width, int height, + vpx_codec_ctx_t *enc, vpx_codec_enc_cfg_t *cfg) { + ASSERT_EQ(vpx_codec_enc_config_default(&iface, cfg, 0), VPX_CODEC_OK); + cfg->g_w = width; + cfg->g_h = height; + cfg->g_lag_in_frames = 0; + cfg->g_pass = VPX_RC_ONE_PASS; + ASSERT_EQ(vpx_codec_enc_init(enc, &iface, cfg, 0), VPX_CODEC_OK); + + ASSERT_EQ(vpx_codec_control_(enc, VP8E_SET_CPUUSED, 2), VPX_CODEC_OK); +} + +// Encodes 1 frame of size |cfg.g_w| x |cfg.g_h| setting |enc|'s configuration +// to |cfg|. +void EncodeWithConfig(const vpx_codec_enc_cfg_t &cfg, vpx_codec_ctx_t *enc) { + libvpx_test::DummyVideoSource video; + video.SetSize(cfg.g_w, cfg.g_h); + video.Begin(); + EXPECT_EQ(vpx_codec_enc_config_set(enc, &cfg), VPX_CODEC_OK) + << vpx_codec_error_detail(enc); + + EXPECT_EQ(vpx_codec_encode(enc, video.img(), video.pts(), video.duration(), + /*flags=*/0, VPX_DL_GOOD_QUALITY), + VPX_CODEC_OK) + << vpx_codec_error_detail(enc); +} + +TEST(EncodeAPI, ConfigChangeThreadCount) { + constexpr int kWidth = 1920; + constexpr int kHeight = 1080; + + for (const auto *iface : kCodecIfaces) { + SCOPED_TRACE(vpx_codec_iface_name(iface)); + for (int i = 0; i < (IsVP9(iface) ? 2 : 1); ++i) { + vpx_codec_enc_cfg_t cfg; + struct Encoder { + ~Encoder() { EXPECT_EQ(vpx_codec_destroy(&ctx), VPX_CODEC_OK); } + vpx_codec_ctx_t ctx = {}; + } enc; + + EXPECT_NO_FATAL_FAILURE( + InitCodec(*iface, kWidth, kHeight, &enc.ctx, &cfg)); + if (IsVP9(iface)) { + EXPECT_EQ(vpx_codec_control_(&enc.ctx, VP9E_SET_TILE_COLUMNS, 6), + VPX_CODEC_OK); + EXPECT_EQ(vpx_codec_control_(&enc.ctx, VP9E_SET_ROW_MT, i), + VPX_CODEC_OK); + } + + for (const auto threads : { 1, 4, 8, 6, 2, 1 }) { + cfg.g_threads = threads; + EXPECT_NO_FATAL_FAILURE(EncodeWithConfig(cfg, &enc.ctx)) + << "iteration: " << i << " threads: " << threads; + } + } + } +} + } // namespace diff --git a/media/libvpx/libvpx/test/lpf_test.cc b/media/libvpx/libvpx/test/lpf_test.cc --- a/media/libvpx/libvpx/test/lpf_test.cc +++ b/media/libvpx/libvpx/test/lpf_test.cc @@ -145,11 +145,11 @@ loop_op_t loopfilter_op_; loop_op_t ref_loopfilter_op_; }; GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(Loop8Test6Param); -#if HAVE_NEON || HAVE_SSE2 || \ +#if HAVE_NEON || HAVE_SSE2 || (HAVE_LSX && !CONFIG_VP9_HIGHBITDEPTH) || \ (HAVE_DSPR2 || HAVE_MSA && !CONFIG_VP9_HIGHBITDEPTH) class Loop8Test9Param : public ::testing::TestWithParam { public: virtual ~Loop8Test9Param() {} virtual void SetUp() { @@ -167,11 +167,11 @@ dual_loop_op_t loopfilter_op_; dual_loop_op_t ref_loopfilter_op_; }; GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(Loop8Test9Param); #endif // HAVE_NEON || HAVE_SSE2 || (HAVE_DSPR2 || HAVE_MSA && - // (!CONFIG_VP9_HIGHBITDEPTH)) + // (!CONFIG_VP9_HIGHBITDEPTH) || (HAVE_LSX && !CONFIG_VP9_HIGHBITDEPTH)) TEST_P(Loop8Test6Param, OperationCheck) { ACMRandom rnd(ACMRandom::DeterministicSeed()); const int count_test_block = number_of_iterations; const int32_t p = kNumCoeffs / 32; @@ -279,11 +279,11 @@ << "Error: Loop8Test6Param, C output doesn't match SSE2 " "loopfilter output. " << "First failed at test case " << first_failure; } -#if HAVE_NEON || HAVE_SSE2 || \ +#if HAVE_NEON || HAVE_SSE2 || (HAVE_LSX && (!CONFIG_VP9_HIGHBITDEPTH)) || \ (HAVE_DSPR2 || HAVE_MSA && (!CONFIG_VP9_HIGHBITDEPTH)) TEST_P(Loop8Test9Param, OperationCheck) { ACMRandom rnd(ACMRandom::DeterministicSeed()); const int count_test_block = number_of_iterations; DECLARE_ALIGNED(PIXEL_WIDTH, Pixel, s[kNumCoeffs]); @@ -409,10 +409,11 @@ << "Error: Loop8Test9Param, C output doesn't match SSE2" "loopfilter output. " << "First failed at test case " << first_failure; } #endif // HAVE_NEON || HAVE_SSE2 || (HAVE_DSPR2 || HAVE_MSA && + // (!CONFIG_VP9_HIGHBITDEPTH)) || (HAVE_LSX && // (!CONFIG_VP9_HIGHBITDEPTH)) using std::make_tuple; #if HAVE_SSE2 @@ -690,6 +691,31 @@ &vpx_lpf_vertical_4_dual_c, 8), make_tuple(&vpx_lpf_vertical_8_dual_msa, &vpx_lpf_vertical_8_dual_c, 8))); #endif // HAVE_MSA && (!CONFIG_VP9_HIGHBITDEPTH) +#if HAVE_LSX && (!CONFIG_VP9_HIGHBITDEPTH) +INSTANTIATE_TEST_SUITE_P( + LSX, Loop8Test6Param, + ::testing::Values( + make_tuple(&vpx_lpf_horizontal_4_lsx, &vpx_lpf_horizontal_4_c, 8), + make_tuple(&vpx_lpf_horizontal_8_lsx, &vpx_lpf_horizontal_8_c, 8), + make_tuple(&vpx_lpf_horizontal_16_dual_lsx, + &vpx_lpf_horizontal_16_dual_c, 8), + make_tuple(&vpx_lpf_vertical_4_lsx, &vpx_lpf_vertical_4_c, 8), + make_tuple(&vpx_lpf_vertical_8_lsx, &vpx_lpf_vertical_8_c, 8), + make_tuple(&vpx_lpf_vertical_16_dual_lsx, &vpx_lpf_vertical_16_dual_c, + 8))); + +INSTANTIATE_TEST_SUITE_P( + LSX, Loop8Test9Param, + ::testing::Values(make_tuple(&vpx_lpf_horizontal_4_dual_lsx, + &vpx_lpf_horizontal_4_dual_c, 8), + make_tuple(&vpx_lpf_horizontal_8_dual_lsx, + &vpx_lpf_horizontal_8_dual_c, 8), + make_tuple(&vpx_lpf_vertical_4_dual_lsx, + &vpx_lpf_vertical_4_dual_c, 8), + make_tuple(&vpx_lpf_vertical_8_dual_lsx, + &vpx_lpf_vertical_8_dual_c, 8))); +#endif // HAVE_LSX && (!CONFIG_VP9_HIGHBITDEPTH) + } // namespace diff --git a/media/libvpx/libvpx/test/partial_idct_test.cc b/media/libvpx/libvpx/test/partial_idct_test.cc --- a/media/libvpx/libvpx/test/partial_idct_test.cc +++ b/media/libvpx/libvpx/test/partial_idct_test.cc @@ -952,8 +952,22 @@ INSTANTIATE_TEST_SUITE_P(MSA, PartialIDctTest, ::testing::ValuesIn(msa_partial_idct_tests)); #endif // HAVE_MSA && !CONFIG_VP9_HIGHBITDEPTH +#if HAVE_LSX && !CONFIG_VP9_HIGHBITDEPTH +const PartialInvTxfmParam lsx_partial_idct_tests[] = { + make_tuple(&vpx_fdct32x32_c, &wrapper, + &wrapper, TX_32X32, 1024, 8, 1), + make_tuple(&vpx_fdct32x32_c, &wrapper, + &wrapper, TX_32X32, 34, 8, 1), + make_tuple(&vpx_fdct32x32_c, &wrapper, + &wrapper, TX_32X32, 1, 8, 1), +}; + +INSTANTIATE_TEST_SUITE_P(LSX, PartialIDctTest, + ::testing::ValuesIn(lsx_partial_idct_tests)); +#endif // HAVE_LSX && !CONFIG_VP9_HIGHBITDEPTH + #endif // !CONFIG_EMULATE_HARDWARE } // namespace diff --git a/media/libvpx/libvpx/test/pp_filter_test.cc b/media/libvpx/libvpx/test/pp_filter_test.cc --- a/media/libvpx/libvpx/test/pp_filter_test.cc +++ b/media/libvpx/libvpx/test/pp_filter_test.cc @@ -113,11 +113,11 @@ } pixel_ptr += dst_image.stride(); } vpx_free(flimits_); -}; +} TEST_P(VpxPostProcDownAndAcrossMbRowTest, CheckCvsAssembly) { // Size of the underlying data block that will be filtered. // Y blocks are always a multiple of 16 wide and exactly 16 high. U and V // blocks are always a multiple of 8 wide and exactly 8 high. @@ -212,11 +212,11 @@ RunNTimes(INT16_MAX); PrintMedian("16x16"); vpx_free(flimits_); -}; +} class VpxMbPostProcAcrossIpTest : public AbstractBench, public ::testing::TestWithParam { public: diff --git a/media/libvpx/libvpx/test/ratectrl_rtc_test.cc b/media/libvpx/libvpx/test/ratectrl_rtc_test.cc deleted file mode 100644 --- a/media/libvpx/libvpx/test/ratectrl_rtc_test.cc +++ /dev/null @@ -1,376 +0,0 @@ -/* - * Copyright (c) 2020 The WebM project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ -#include "vp9/ratectrl_rtc.h" - -#include // NOLINT -#include - -#include "./vpx_config.h" -#include "third_party/googletest/src/include/gtest/gtest.h" -#include "test/codec_factory.h" -#include "test/encode_test_driver.h" -#include "test/i420_video_source.h" -#include "test/util.h" -#include "test/video_source.h" -#include "vpx/vpx_codec.h" -#include "vpx_ports/bitops.h" - -namespace { - -const size_t kNumFrames = 300; - -const int kTemporalId[4] = { 0, 2, 1, 2 }; - -class RcInterfaceTest - : public ::libvpx_test::EncoderTest, - public ::libvpx_test::CodecTestWith2Params { - public: - RcInterfaceTest() - : EncoderTest(GET_PARAM(0)), aq_mode_(GET_PARAM(1)), key_interval_(3000), - encoder_exit_(false) {} - - virtual ~RcInterfaceTest() {} - - protected: - virtual void SetUp() { - InitializeConfig(); - SetMode(::libvpx_test::kRealTime); - } - - virtual void PreEncodeFrameHook(libvpx_test::VideoSource *video, - libvpx_test::Encoder *encoder) { - if (video->frame() == 0) { - encoder->Control(VP8E_SET_CPUUSED, 7); - encoder->Control(VP9E_SET_AQ_MODE, aq_mode_); - encoder->Control(VP9E_SET_TUNE_CONTENT, 0); - encoder->Control(VP8E_SET_MAX_INTRA_BITRATE_PCT, 1000); - encoder->Control(VP9E_SET_RTC_EXTERNAL_RATECTRL, 1); - } - frame_params_.frame_type = - video->frame() % key_interval_ == 0 ? KEY_FRAME : INTER_FRAME; - if (rc_cfg_.rc_mode == VPX_CBR && frame_params_.frame_type == INTER_FRAME) { - // Disable golden frame update. - frame_flags_ |= VP8_EFLAG_NO_UPD_GF; - frame_flags_ |= VP8_EFLAG_NO_UPD_ARF; - } - encoder_exit_ = video->frame() == kNumFrames; - } - - virtual void PostEncodeFrameHook(::libvpx_test::Encoder *encoder) { - if (encoder_exit_) { - return; - } - int loopfilter_level, qp; - encoder->Control(VP9E_GET_LOOPFILTER_LEVEL, &loopfilter_level); - encoder->Control(VP8E_GET_LAST_QUANTIZER, &qp); - rc_api_->ComputeQP(frame_params_); - ASSERT_EQ(rc_api_->GetQP(), qp); - ASSERT_EQ(rc_api_->GetLoopfilterLevel(), loopfilter_level); - } - - virtual void FramePktHook(const vpx_codec_cx_pkt_t *pkt) { - rc_api_->PostEncodeUpdate(pkt->data.frame.sz); - } - - void RunOneLayer() { - SetConfig(GET_PARAM(2)); - rc_api_ = libvpx::VP9RateControlRTC::Create(rc_cfg_); - frame_params_.spatial_layer_id = 0; - frame_params_.temporal_layer_id = 0; - - ::libvpx_test::I420VideoSource video("desktop_office1.1280_720-020.yuv", - 1280, 720, 30, 1, 0, kNumFrames); - - ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); - } - - void RunOneLayerVBRPeriodicKey() { - if (GET_PARAM(2) != VPX_VBR) return; - key_interval_ = 100; - SetConfig(VPX_VBR); - rc_api_ = libvpx::VP9RateControlRTC::Create(rc_cfg_); - frame_params_.spatial_layer_id = 0; - frame_params_.temporal_layer_id = 0; - - ::libvpx_test::I420VideoSource video("desktop_office1.1280_720-020.yuv", - 1280, 720, 30, 1, 0, kNumFrames); - - ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); - } - - private: - void SetConfig(vpx_rc_mode rc_mode) { - rc_cfg_.width = 1280; - rc_cfg_.height = 720; - rc_cfg_.max_quantizer = 52; - rc_cfg_.min_quantizer = 2; - rc_cfg_.target_bandwidth = 1000; - rc_cfg_.buf_initial_sz = 600; - rc_cfg_.buf_optimal_sz = 600; - rc_cfg_.buf_sz = 1000; - rc_cfg_.undershoot_pct = 50; - rc_cfg_.overshoot_pct = 50; - rc_cfg_.max_intra_bitrate_pct = 1000; - rc_cfg_.framerate = 30.0; - rc_cfg_.ss_number_layers = 1; - rc_cfg_.ts_number_layers = 1; - rc_cfg_.scaling_factor_num[0] = 1; - rc_cfg_.scaling_factor_den[0] = 1; - rc_cfg_.layer_target_bitrate[0] = 1000; - rc_cfg_.max_quantizers[0] = 52; - rc_cfg_.min_quantizers[0] = 2; - rc_cfg_.rc_mode = rc_mode; - rc_cfg_.aq_mode = aq_mode_; - - // Encoder settings for ground truth. - cfg_.g_w = 1280; - cfg_.g_h = 720; - cfg_.rc_undershoot_pct = 50; - cfg_.rc_overshoot_pct = 50; - cfg_.rc_buf_initial_sz = 600; - cfg_.rc_buf_optimal_sz = 600; - cfg_.rc_buf_sz = 1000; - cfg_.rc_dropframe_thresh = 0; - cfg_.rc_min_quantizer = 2; - cfg_.rc_max_quantizer = 52; - cfg_.rc_end_usage = rc_mode; - cfg_.g_lag_in_frames = 0; - cfg_.g_error_resilient = 0; - cfg_.rc_target_bitrate = 1000; - cfg_.kf_min_dist = key_interval_; - cfg_.kf_max_dist = key_interval_; - } - - std::unique_ptr rc_api_; - libvpx::VP9RateControlRtcConfig rc_cfg_; - int aq_mode_; - int key_interval_; - libvpx::VP9FrameParamsQpRTC frame_params_; - bool encoder_exit_; -}; - -class RcInterfaceSvcTest : public ::libvpx_test::EncoderTest, - public ::libvpx_test::CodecTestWithParam { - public: - RcInterfaceSvcTest() : EncoderTest(GET_PARAM(0)), aq_mode_(GET_PARAM(1)) {} - virtual ~RcInterfaceSvcTest() {} - - protected: - virtual void SetUp() { - InitializeConfig(); - SetMode(::libvpx_test::kRealTime); - } - - virtual void PreEncodeFrameHook(libvpx_test::VideoSource *video, - ::libvpx_test::Encoder *encoder) { - if (video->frame() == 0) { - encoder->Control(VP8E_SET_CPUUSED, 7); - encoder->Control(VP9E_SET_AQ_MODE, aq_mode_); - encoder->Control(VP9E_SET_TUNE_CONTENT, 0); - encoder->Control(VP8E_SET_MAX_INTRA_BITRATE_PCT, 900); - encoder->Control(VP9E_SET_RTC_EXTERNAL_RATECTRL, 1); - encoder->Control(VP9E_SET_SVC, 1); - encoder->Control(VP9E_SET_SVC_PARAMETERS, &svc_params_); - } - - frame_params_.frame_type = video->frame() == 0 ? KEY_FRAME : INTER_FRAME; - if (rc_cfg_.rc_mode == VPX_CBR && frame_params_.frame_type == INTER_FRAME) { - // Disable golden frame update. - frame_flags_ |= VP8_EFLAG_NO_UPD_GF; - frame_flags_ |= VP8_EFLAG_NO_UPD_ARF; - } - encoder_exit_ = video->frame() == kNumFrames; - current_superframe_ = video->frame(); - } - - virtual void PostEncodeFrameHook(::libvpx_test::Encoder *encoder) { - ::libvpx_test::CxDataIterator iter = encoder->GetCxData(); - while (const vpx_codec_cx_pkt_t *pkt = iter.Next()) { - ParseSuperframeSizes(static_cast(pkt->data.frame.buf), - pkt->data.frame.sz); - for (int sl = 0; sl < rc_cfg_.ss_number_layers; sl++) { - frame_params_.spatial_layer_id = sl; - frame_params_.temporal_layer_id = kTemporalId[current_superframe_ % 4]; - rc_api_->ComputeQP(frame_params_); - frame_params_.frame_type = INTER_FRAME; - rc_api_->PostEncodeUpdate(sizes_[sl]); - } - } - if (!encoder_exit_) { - int loopfilter_level, qp; - encoder->Control(VP9E_GET_LOOPFILTER_LEVEL, &loopfilter_level); - encoder->Control(VP8E_GET_LAST_QUANTIZER, &qp); - ASSERT_EQ(rc_api_->GetQP(), qp); - ASSERT_EQ(rc_api_->GetLoopfilterLevel(), loopfilter_level); - } - } - // This method needs to be overridden because non-reference frames are - // expected to be mismatched frames as the encoder will avoid loopfilter on - // these frames. - virtual void MismatchHook(const vpx_image_t * /*img1*/, - const vpx_image_t * /*img2*/) {} - - void RunSvc() { - SetConfigSvc(); - rc_api_ = libvpx::VP9RateControlRTC::Create(rc_cfg_); - SetEncoderSvc(); - - ::libvpx_test::I420VideoSource video("desktop_office1.1280_720-020.yuv", - 1280, 720, 30, 1, 0, kNumFrames); - - ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); - } - - private: - vpx_codec_err_t ParseSuperframeSizes(const uint8_t *data, size_t data_sz) { - uint8_t marker = *(data + data_sz - 1); - if ((marker & 0xe0) == 0xc0) { - const uint32_t frames = (marker & 0x7) + 1; - const uint32_t mag = ((marker >> 3) & 0x3) + 1; - const size_t index_sz = 2 + mag * frames; - // This chunk is marked as having a superframe index but doesn't have - // enough data for it, thus it's an invalid superframe index. - if (data_sz < index_sz) return VPX_CODEC_CORRUPT_FRAME; - { - const uint8_t marker2 = *(data + data_sz - index_sz); - // This chunk is marked as having a superframe index but doesn't have - // the matching marker byte at the front of the index therefore it's an - // invalid chunk. - if (marker != marker2) return VPX_CODEC_CORRUPT_FRAME; - } - const uint8_t *x = &data[data_sz - index_sz + 1]; - for (uint32_t i = 0; i < frames; ++i) { - uint32_t this_sz = 0; - - for (uint32_t j = 0; j < mag; ++j) this_sz |= (*x++) << (j * 8); - sizes_[i] = this_sz; - } - } - return VPX_CODEC_OK; - } - - void SetEncoderSvc() { - cfg_.ss_number_layers = 3; - cfg_.ts_number_layers = 3; - cfg_.g_timebase.num = 1; - cfg_.g_timebase.den = 30; - svc_params_.scaling_factor_num[0] = 72; - svc_params_.scaling_factor_den[0] = 288; - svc_params_.scaling_factor_num[1] = 144; - svc_params_.scaling_factor_den[1] = 288; - svc_params_.scaling_factor_num[2] = 288; - svc_params_.scaling_factor_den[2] = 288; - for (int i = 0; i < VPX_MAX_LAYERS; ++i) { - svc_params_.max_quantizers[i] = 56; - svc_params_.min_quantizers[i] = 2; - } - cfg_.rc_end_usage = VPX_CBR; - cfg_.g_lag_in_frames = 0; - cfg_.g_error_resilient = 0; - // 3 temporal layers - cfg_.ts_rate_decimator[0] = 4; - cfg_.ts_rate_decimator[1] = 2; - cfg_.ts_rate_decimator[2] = 1; - cfg_.temporal_layering_mode = 3; - - cfg_.rc_buf_initial_sz = 500; - cfg_.rc_buf_optimal_sz = 600; - cfg_.rc_buf_sz = 1000; - cfg_.rc_min_quantizer = 2; - cfg_.rc_max_quantizer = 56; - cfg_.g_threads = 1; - cfg_.kf_max_dist = 9999; - cfg_.rc_target_bitrate = 1600; - cfg_.rc_overshoot_pct = 50; - cfg_.rc_undershoot_pct = 50; - - cfg_.layer_target_bitrate[0] = 100; - cfg_.layer_target_bitrate[1] = 140; - cfg_.layer_target_bitrate[2] = 200; - cfg_.layer_target_bitrate[3] = 250; - cfg_.layer_target_bitrate[4] = 350; - cfg_.layer_target_bitrate[5] = 500; - cfg_.layer_target_bitrate[6] = 450; - cfg_.layer_target_bitrate[7] = 630; - cfg_.layer_target_bitrate[8] = 900; - } - - void SetConfigSvc() { - rc_cfg_.width = 1280; - rc_cfg_.height = 720; - rc_cfg_.max_quantizer = 56; - rc_cfg_.min_quantizer = 2; - rc_cfg_.target_bandwidth = 1600; - rc_cfg_.buf_initial_sz = 500; - rc_cfg_.buf_optimal_sz = 600; - rc_cfg_.buf_sz = 1000; - rc_cfg_.undershoot_pct = 50; - rc_cfg_.overshoot_pct = 50; - rc_cfg_.max_intra_bitrate_pct = 900; - rc_cfg_.framerate = 30.0; - rc_cfg_.ss_number_layers = 3; - rc_cfg_.ts_number_layers = 3; - rc_cfg_.rc_mode = VPX_CBR; - - rc_cfg_.scaling_factor_num[0] = 1; - rc_cfg_.scaling_factor_den[0] = 4; - rc_cfg_.scaling_factor_num[1] = 2; - rc_cfg_.scaling_factor_den[1] = 4; - rc_cfg_.scaling_factor_num[2] = 4; - rc_cfg_.scaling_factor_den[2] = 4; - - rc_cfg_.ts_rate_decimator[0] = 4; - rc_cfg_.ts_rate_decimator[1] = 2; - rc_cfg_.ts_rate_decimator[2] = 1; - - rc_cfg_.layer_target_bitrate[0] = 100; - rc_cfg_.layer_target_bitrate[1] = 140; - rc_cfg_.layer_target_bitrate[2] = 200; - rc_cfg_.layer_target_bitrate[3] = 250; - rc_cfg_.layer_target_bitrate[4] = 350; - rc_cfg_.layer_target_bitrate[5] = 500; - rc_cfg_.layer_target_bitrate[6] = 450; - rc_cfg_.layer_target_bitrate[7] = 630; - rc_cfg_.layer_target_bitrate[8] = 900; - - for (int sl = 0; sl < rc_cfg_.ss_number_layers; ++sl) { - for (int tl = 0; tl < rc_cfg_.ts_number_layers; ++tl) { - const int i = sl * rc_cfg_.ts_number_layers + tl; - rc_cfg_.max_quantizers[i] = 56; - rc_cfg_.min_quantizers[i] = 2; - } - } - } - - int aq_mode_; - std::unique_ptr rc_api_; - libvpx::VP9RateControlRtcConfig rc_cfg_; - vpx_svc_extra_cfg_t svc_params_; - libvpx::VP9FrameParamsQpRTC frame_params_; - bool encoder_exit_; - int current_superframe_; - uint32_t sizes_[8]; -}; - -TEST_P(RcInterfaceTest, OneLayer) { RunOneLayer(); } - -TEST_P(RcInterfaceTest, OneLayerVBRPeriodicKey) { RunOneLayerVBRPeriodicKey(); } - -TEST_P(RcInterfaceSvcTest, Svc) { RunSvc(); } - -VP9_INSTANTIATE_TEST_SUITE(RcInterfaceTest, ::testing::Values(0, 3), - ::testing::Values(VPX_CBR, VPX_VBR)); -VP9_INSTANTIATE_TEST_SUITE(RcInterfaceSvcTest, ::testing::Values(0)); -} // namespace - -int main(int argc, char **argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/media/libvpx/libvpx/test/realtime_test.cc b/media/libvpx/libvpx/test/realtime_test.cc --- a/media/libvpx/libvpx/test/realtime_test.cc +++ b/media/libvpx/libvpx/test/realtime_test.cc @@ -5,10 +5,12 @@ * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ +#include + #include "test/codec_factory.h" #include "test/encode_test_driver.h" #include "test/util.h" #include "test/video_source.h" #include "third_party/googletest/src/include/gtest/gtest.h" @@ -50,10 +52,26 @@ void FramePktHook(const vpx_codec_cx_pkt_t * /*pkt*/) override { frame_packets_++; } + bool IsVP9() const { +#if CONFIG_VP9_ENCODER + return codec_ == &libvpx_test::kVP9; +#else + return false; +#endif + } + + void TestIntegerOverflow(unsigned int width, unsigned int height) { + ::libvpx_test::RandomVideoSource video; + video.SetSize(width, height); + video.set_limit(20); + cfg_.rc_target_bitrate = UINT_MAX; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + } + int frame_packets_; }; TEST_P(RealtimeTest, RealtimeFirstPassProducesFrames) { ::libvpx_test::RandomVideoSource video; @@ -61,16 +79,26 @@ video.set_limit(kFramesToEncode); ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); EXPECT_EQ(kFramesToEncode, frame_packets_); } -TEST_P(RealtimeTest, IntegerOverflow) { - ::libvpx_test::RandomVideoSource video; - video.SetSize(800, 480); - video.set_limit(20); - cfg_.rc_target_bitrate = 140000000; - ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); +TEST_P(RealtimeTest, IntegerOverflow) { TestIntegerOverflow(2048, 2048); } + +TEST_P(RealtimeTest, IntegerOverflowLarge) { + if (IsVP9()) { +#if VPX_ARCH_X86_64 + TestIntegerOverflow(16384, 16384); +#else + TestIntegerOverflow(4096, 4096); +#endif + } else { + GTEST_SKIP() + << "TODO(https://crbug.com/webm/1748,https://crbug.com/webm/1751):" + << " Enable this test after bitstream errors & undefined sanitizer " + "warnings are fixed."; + // TestIntegerOverflow(16383, 16383); + } } VP8_INSTANTIATE_TEST_SUITE(RealtimeTest, ::testing::Values(::libvpx_test::kRealTime)); VP9_INSTANTIATE_TEST_SUITE(RealtimeTest, diff --git a/media/libvpx/libvpx/test/register_state_check.h b/media/libvpx/libvpx/test/register_state_check.h --- a/media/libvpx/libvpx/test/register_state_check.h +++ b/media/libvpx/libvpx/test/register_state_check.h @@ -33,10 +33,11 @@ #undef NOMINMAX #define NOMINMAX #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #endif +#include #include #include inline bool operator==(const M128A &lhs, const M128A &rhs) { return (lhs.Low == rhs.Low && lhs.High == rhs.High); @@ -79,14 +80,17 @@ bool initialized_; CONTEXT pre_context_; }; -#define ASM_REGISTER_STATE_CHECK(statement) \ - do { \ - libvpx_test::RegisterStateCheck reg_check; \ - statement; \ +#define ASM_REGISTER_STATE_CHECK(statement) \ + do { \ + { \ + libvpx_test::RegisterStateCheck reg_check; \ + statement; \ + } \ + _ReadWriteBarrier(); \ } while (false) } // namespace libvpx_test #elif defined(CONFIG_SHARED) && defined(HAVE_NEON_ASM) && \ @@ -119,15 +123,26 @@ } int64_t pre_store_[8]; }; +#if defined(__GNUC__) +#define ASM_REGISTER_STATE_CHECK(statement) \ + do { \ + { \ + libvpx_test::RegisterStateCheck reg_check; \ + statement; \ + } \ + __asm__ volatile("" ::: "memory"); \ + } while (false) +#else #define ASM_REGISTER_STATE_CHECK(statement) \ do { \ libvpx_test::RegisterStateCheck reg_check; \ statement; \ } while (false) +#endif } // namespace libvpx_test #else @@ -167,14 +182,17 @@ } uint16_t pre_fpu_env_[14]; }; -#define API_REGISTER_STATE_CHECK(statement) \ - do { \ - libvpx_test::RegisterStateCheckMMX reg_check; \ - ASM_REGISTER_STATE_CHECK(statement); \ +#define API_REGISTER_STATE_CHECK(statement) \ + do { \ + { \ + libvpx_test::RegisterStateCheckMMX reg_check; \ + ASM_REGISTER_STATE_CHECK(statement); \ + } \ + __asm__ volatile("" ::: "memory"); \ } while (false) } // namespace libvpx_test #endif // __GNUC__ diff --git a/media/libvpx/libvpx/test/sad_test.cc b/media/libvpx/libvpx/test/sad_test.cc --- a/media/libvpx/libvpx/test/sad_test.cc +++ b/media/libvpx/libvpx/test/sad_test.cc @@ -54,12 +54,10 @@ typedef void (*SadMxNx8Func)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sad_array); -typedef TestParams SadMxNx8Param; - using libvpx_test::ACMRandom; namespace { template class SADTestBase : public ::testing::TestWithParam { @@ -264,34 +262,10 @@ ACMRandom rnd_; ParamType params_; }; -class SADx8Test : public SADTestBase { - public: - SADx8Test() : SADTestBase(GetParam()) {} - - protected: - void SADs(unsigned int *results) const { - const uint8_t *reference = GetReferenceFromOffset(0); - - ASM_REGISTER_STATE_CHECK(params_.func( - source_data_, source_stride_, reference, reference_stride_, results)); - } - - void CheckSADs() const { - uint32_t reference_sad; - DECLARE_ALIGNED(kDataAlignment, uint32_t, exp_sad[8]); - - SADs(exp_sad); - for (int offset = 0; offset < 8; ++offset) { - reference_sad = ReferenceSAD(offset); - EXPECT_EQ(reference_sad, exp_sad[offset]) << "offset " << offset; - } - } -}; - class SADx4Test : public SADTestBase { public: SADx4Test() : SADTestBase(GetParam()) {} protected: @@ -562,17 +536,10 @@ bit_depth_, elapsed_time); reference_stride_ = tmp_stride; } -TEST_P(SADx8Test, Regular) { - FillRandomWH(source_data_, source_stride_, params_.width, params_.height); - FillRandomWH(GetReferenceFromOffset(0), reference_stride_, params_.width + 8, - params_.height); - CheckSADs(); -} - //------------------------------------------------------------------------------ // C functions const SadMxNParam c_tests[] = { SadMxNParam(64, 64, &vpx_sad64x64_c), SadMxNParam(64, 32, &vpx_sad64x32_c), @@ -745,28 +712,10 @@ SadMxNx4Param(4, 4, &vpx_highbd_sad4x4x4d_c, 12), #endif // CONFIG_VP9_HIGHBITDEPTH }; INSTANTIATE_TEST_SUITE_P(C, SADx4Test, ::testing::ValuesIn(x4d_c_tests)); -// TODO(angiebird): implement the marked-down sad functions -const SadMxNx8Param x8_c_tests[] = { - // SadMxNx8Param(64, 64, &vpx_sad64x64x8_c), - // SadMxNx8Param(64, 32, &vpx_sad64x32x8_c), - // SadMxNx8Param(32, 64, &vpx_sad32x64x8_c), - SadMxNx8Param(32, 32, &vpx_sad32x32x8_c), - // SadMxNx8Param(32, 16, &vpx_sad32x16x8_c), - // SadMxNx8Param(16, 32, &vpx_sad16x32x8_c), - SadMxNx8Param(16, 16, &vpx_sad16x16x8_c), - SadMxNx8Param(16, 8, &vpx_sad16x8x8_c), - SadMxNx8Param(8, 16, &vpx_sad8x16x8_c), - SadMxNx8Param(8, 8, &vpx_sad8x8x8_c), - // SadMxNx8Param(8, 4, &vpx_sad8x4x8_c), - // SadMxNx8Param(4, 8, &vpx_sad4x8x8_c), - SadMxNx8Param(4, 4, &vpx_sad4x4x8_c), -}; -INSTANTIATE_TEST_SUITE_P(C, SADx8Test, ::testing::ValuesIn(x8_c_tests)); - //------------------------------------------------------------------------------ // ARM functions #if HAVE_NEON const SadMxNParam neon_tests[] = { SadMxNParam(64, 64, &vpx_sad64x64_neon), @@ -990,22 +939,10 @@ #if HAVE_SSSE3 // Only functions are x3, which do not have tests. #endif // HAVE_SSSE3 -#if HAVE_SSE4_1 -const SadMxNx8Param x8_sse4_1_tests[] = { - SadMxNx8Param(16, 16, &vpx_sad16x16x8_sse4_1), - SadMxNx8Param(16, 8, &vpx_sad16x8x8_sse4_1), - SadMxNx8Param(8, 16, &vpx_sad8x16x8_sse4_1), - SadMxNx8Param(8, 8, &vpx_sad8x8x8_sse4_1), - SadMxNx8Param(4, 4, &vpx_sad4x4x8_sse4_1), -}; -INSTANTIATE_TEST_SUITE_P(SSE4_1, SADx8Test, - ::testing::ValuesIn(x8_sse4_1_tests)); -#endif // HAVE_SSE4_1 - #if HAVE_AVX2 const SadMxNParam avx2_tests[] = { SadMxNParam(64, 64, &vpx_sad64x64_avx2), SadMxNParam(64, 32, &vpx_sad64x32_avx2), SadMxNParam(32, 64, &vpx_sad32x64_avx2), @@ -1027,15 +964,10 @@ SadMxNx4Param(64, 64, &vpx_sad64x64x4d_avx2), SadMxNx4Param(32, 32, &vpx_sad32x32x4d_avx2), }; INSTANTIATE_TEST_SUITE_P(AVX2, SADx4Test, ::testing::ValuesIn(x4d_avx2_tests)); -const SadMxNx8Param x8_avx2_tests[] = { - // SadMxNx8Param(64, 64, &vpx_sad64x64x8_c), - SadMxNx8Param(32, 32, &vpx_sad32x32x8_avx2), -}; -INSTANTIATE_TEST_SUITE_P(AVX2, SADx8Test, ::testing::ValuesIn(x8_avx2_tests)); #endif // HAVE_AVX2 #if HAVE_AVX512 const SadMxNx4Param x4d_avx512_tests[] = { SadMxNx4Param(64, 64, &vpx_sad64x64x4d_avx512), @@ -1194,6 +1126,34 @@ SadMxNx4Param(4, 8, &vpx_sad4x8x4d_mmi), SadMxNx4Param(4, 4, &vpx_sad4x4x4d_mmi), }; INSTANTIATE_TEST_SUITE_P(MMI, SADx4Test, ::testing::ValuesIn(x4d_mmi_tests)); #endif // HAVE_MMI + +//------------------------------------------------------------------------------ +// loongarch functions +#if HAVE_LSX +const SadMxNParam lsx_tests[] = { + SadMxNParam(64, 64, &vpx_sad64x64_lsx), + SadMxNParam(32, 32, &vpx_sad32x32_lsx), + SadMxNParam(16, 16, &vpx_sad16x16_lsx), +}; +INSTANTIATE_TEST_SUITE_P(LSX, SADTest, ::testing::ValuesIn(lsx_tests)); + +const SadMxNAvgParam avg_lsx_tests[] = { + SadMxNAvgParam(64, 64, &vpx_sad64x64_avg_lsx), + SadMxNAvgParam(32, 32, &vpx_sad32x32_avg_lsx), +}; +INSTANTIATE_TEST_SUITE_P(LSX, SADavgTest, ::testing::ValuesIn(avg_lsx_tests)); + +const SadMxNx4Param x4d_lsx_tests[] = { + SadMxNx4Param(64, 64, &vpx_sad64x64x4d_lsx), + SadMxNx4Param(64, 32, &vpx_sad64x32x4d_lsx), + SadMxNx4Param(32, 64, &vpx_sad32x64x4d_lsx), + SadMxNx4Param(32, 32, &vpx_sad32x32x4d_lsx), + SadMxNx4Param(16, 16, &vpx_sad16x16x4d_lsx), + SadMxNx4Param(8, 8, &vpx_sad8x8x4d_lsx), +}; +INSTANTIATE_TEST_SUITE_P(LSX, SADx4Test, ::testing::ValuesIn(x4d_lsx_tests)); +#endif // HAVE_LSX + } // namespace diff --git a/media/libvpx/libvpx/test/set_roi.cc b/media/libvpx/libvpx/test/set_roi.cc --- a/media/libvpx/libvpx/test/set_roi.cc +++ b/media/libvpx/libvpx/test/set_roi.cc @@ -159,8 +159,8 @@ } // Free allocated memory if (cpi.segmentation_map) vpx_free(cpi.segmentation_map); if (roi_map) vpx_free(roi_map); -}; +} } // namespace diff --git a/media/libvpx/libvpx/test/simple_encode_test.cc b/media/libvpx/libvpx/test/simple_encode_test.cc --- a/media/libvpx/libvpx/test/simple_encode_test.cc +++ b/media/libvpx/libvpx/test/simple_encode_test.cc @@ -35,17 +35,18 @@ const int height_ = 288; const int frame_rate_num_ = 30; const int frame_rate_den_ = 1; const int target_bitrate_ = 1000; const int num_frames_ = 17; + const int target_level_ = LEVEL_UNKNOWN; const std::string in_file_path_str_ = libvpx_test::GetDataPath() + "/bus_352x288_420_f20_b8.yuv"; }; TEST_F(SimpleEncodeTest, ComputeFirstPassStats) { SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, - target_bitrate_, num_frames_, + target_bitrate_, num_frames_, target_level_, in_file_path_str_.c_str()); simple_encode.ComputeFirstPassStats(); std::vector> frame_stats = simple_encode.ObserveFirstPassStats(); EXPECT_EQ(frame_stats.size(), static_cast(num_frames_)); @@ -62,11 +63,11 @@ } } TEST_F(SimpleEncodeTest, ObserveFirstPassMotionVectors) { SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, - target_bitrate_, num_frames_, + target_bitrate_, num_frames_, target_level_, in_file_path_str_.c_str()); simple_encode.ComputeFirstPassStats(); std::vector> fps_motion_vectors = simple_encode.ObserveFirstPassMotionVectors(); EXPECT_EQ(fps_motion_vectors.size(), static_cast(num_frames_)); @@ -84,20 +85,20 @@ } } TEST_F(SimpleEncodeTest, GetCodingFrameNum) { SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, - target_bitrate_, num_frames_, + target_bitrate_, num_frames_, target_level_, in_file_path_str_.c_str()); simple_encode.ComputeFirstPassStats(); const int num_coding_frames = simple_encode.GetCodingFrameNum(); EXPECT_EQ(num_coding_frames, 19); } TEST_F(SimpleEncodeTest, EncodeFrame) { SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, - target_bitrate_, num_frames_, + target_bitrate_, num_frames_, target_level_, in_file_path_str_.c_str()); simple_encode.ComputeFirstPassStats(); int num_coding_frames = simple_encode.GetCodingFrameNum(); EXPECT_GE(num_coding_frames, num_frames_); simple_encode.StartEncode(); @@ -136,11 +137,11 @@ simple_encode.EndEncode(); } TEST_F(SimpleEncodeTest, ObserveKeyFrameMap) { SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, - target_bitrate_, num_frames_, + target_bitrate_, num_frames_, target_level_, in_file_path_str_.c_str()); simple_encode.ComputeFirstPassStats(); std::vector key_frame_map = simple_encode.ObserveKeyFrameMap(); EXPECT_EQ(key_frame_map.size(), static_cast(num_frames_)); simple_encode.StartEncode(); @@ -165,11 +166,11 @@ simple_encode.EndEncode(); } TEST_F(SimpleEncodeTest, EncodeFrameWithTargetFrameBits) { SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, - target_bitrate_, num_frames_, + target_bitrate_, num_frames_, target_level_, in_file_path_str_.c_str()); simple_encode.ComputeFirstPassStats(); const int num_coding_frames = simple_encode.GetCodingFrameNum(); simple_encode.StartEncode(); for (int i = 0; i < num_coding_frames; ++i) { @@ -203,11 +204,11 @@ simple_encode.EndEncode(); } TEST_F(SimpleEncodeTest, EncodeFrameWithQuantizeIndex) { SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, - target_bitrate_, num_frames_, + target_bitrate_, num_frames_, target_level_, in_file_path_str_.c_str()); simple_encode.ComputeFirstPassStats(); const int num_coding_frames = simple_encode.GetCodingFrameNum(); simple_encode.StartEncode(); for (int i = 0; i < num_coding_frames; ++i) { @@ -235,11 +236,11 @@ std::vector ref_show_idx_list; { // The first encode. SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, target_bitrate_, num_frames_, - in_file_path_str_.c_str()); + target_level_, in_file_path_str_.c_str()); simple_encode.ComputeFirstPassStats(); const int num_coding_frames = simple_encode.GetCodingFrameNum(); simple_encode.StartEncode(); for (int i = 0; i < num_coding_frames; ++i) { EncodeFrameResult encode_frame_result; @@ -255,11 +256,11 @@ } { // The second encode with quantize index got from the first encode. SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, target_bitrate_, num_frames_, - in_file_path_str_.c_str()); + target_level_, in_file_path_str_.c_str()); simple_encode.ComputeFirstPassStats(); const int num_coding_frames = simple_encode.GetCodingFrameNum(); EXPECT_EQ(static_cast(num_coding_frames), quantize_index_list.size()); simple_encode.StartEncode(); @@ -284,11 +285,11 @@ const int num_rows_4x4 = GetNumUnit4x4(width_); const int num_cols_4x4 = GetNumUnit4x4(height_); const int num_units_4x4 = num_rows_4x4 * num_cols_4x4; // The first encode. SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, - target_bitrate_, num_frames_, + target_bitrate_, num_frames_, target_level_, in_file_path_str_.c_str()); simple_encode.ComputeFirstPassStats(); const int num_coding_frames = simple_encode.GetCodingFrameNum(); std::vector partition_info_list(num_units_4x4 * num_coding_frames); @@ -307,11 +308,11 @@ } simple_encode.EndEncode(); // The second encode. SimpleEncode simple_encode_2(width_, height_, frame_rate_num_, frame_rate_den_, target_bitrate_, num_frames_, - in_file_path_str_.c_str()); + target_level_, in_file_path_str_.c_str()); simple_encode_2.ComputeFirstPassStats(); const int num_coding_frames_2 = simple_encode_2.GetCodingFrameNum(); simple_encode_2.StartEncode(); for (int i = 0; i < num_coding_frames_2; ++i) { EncodeFrameResult encode_frame_result; @@ -355,11 +356,11 @@ const int num_rows_4x4 = GetNumUnit4x4(width_); const int num_cols_4x4 = GetNumUnit4x4(height_); const int num_units_4x4 = num_rows_4x4 * num_cols_4x4; // The first encode. SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, - target_bitrate_, num_frames_, + target_bitrate_, num_frames_, target_level_, in_file_path_str_.c_str()); simple_encode.ComputeFirstPassStats(); const int num_coding_frames = simple_encode.GetCodingFrameNum(); std::vector partition_info_list(num_units_4x4 * num_coding_frames); @@ -375,11 +376,11 @@ } simple_encode.EndEncode(); // The second encode. SimpleEncode simple_encode_2(width_, height_, frame_rate_num_, frame_rate_den_, target_bitrate_, num_frames_, - in_file_path_str_.c_str()); + target_level_, in_file_path_str_.c_str()); simple_encode_2.ComputeFirstPassStats(); const int num_coding_frames_2 = simple_encode_2.GetCodingFrameNum(); simple_encode_2.StartEncode(); for (int i = 0; i < num_coding_frames_2; ++i) { EncodeFrameResult encode_frame_result; @@ -415,11 +416,11 @@ std::vector gop_map(num_frames_, 0); { // The first encode. SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, target_bitrate_, num_frames_, - in_file_path_str_.c_str()); + target_level_, in_file_path_str_.c_str()); simple_encode.ComputeFirstPassStats(); simple_encode.StartEncode(); int coded_show_frame_count = 0; while (coded_show_frame_count < num_frames_) { @@ -447,11 +448,11 @@ { // The second encode with quantize index got from the first encode. // The external arfs are the same as the first encode. SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, target_bitrate_, num_frames_, - in_file_path_str_.c_str()); + target_level_, in_file_path_str_.c_str()); simple_encode.ComputeFirstPassStats(); simple_encode.SetExternalGroupOfPicturesMap(gop_map.data(), gop_map.size()); const int num_coding_frames = simple_encode.GetCodingFrameNum(); EXPECT_EQ(static_cast(num_coding_frames), quantize_index_list.size()); @@ -469,11 +470,11 @@ } } TEST_F(SimpleEncodeTest, SetExternalGroupOfPicturesMap) { SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, - target_bitrate_, num_frames_, + target_bitrate_, num_frames_, target_level_, in_file_path_str_.c_str()); simple_encode.ComputeFirstPassStats(); std::vector gop_map(num_frames_, 0); @@ -539,11 +540,11 @@ TEST_F(SimpleEncodeTest, GetEncodeFrameInfo) { // Makes sure that the encode_frame_info obtained from GetEncodeFrameInfo() // matches the counterpart in encode_frame_result obtained from EncodeFrame() SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, - target_bitrate_, num_frames_, + target_bitrate_, num_frames_, target_level_, in_file_path_str_.c_str()); simple_encode.ComputeFirstPassStats(); const int num_coding_frames = simple_encode.GetCodingFrameNum(); simple_encode.StartEncode(); for (int i = 0; i < num_coding_frames; ++i) { @@ -556,11 +557,11 @@ simple_encode.EndEncode(); } TEST_F(SimpleEncodeTest, GetFramePixelCount) { SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, - target_bitrate_, num_frames_, + target_bitrate_, num_frames_, target_level_, in_file_path_str_.c_str()); EXPECT_EQ(simple_encode.GetFramePixelCount(), static_cast(width_ * height_ * 3 / 2)); } diff --git a/media/libvpx/libvpx/test/svc_datarate_test.cc b/media/libvpx/libvpx/test/svc_datarate_test.cc --- a/media/libvpx/libvpx/test/svc_datarate_test.cc +++ b/media/libvpx/libvpx/test/svc_datarate_test.cc @@ -1352,11 +1352,10 @@ cfg_.kf_max_dist = kf_dist; key_frame_spacing_ = kf_dist; ResetModel(); AssignLayerBitrates(); ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); - // TODO(jianj): webm:1554 CheckLayerRateTargeting(number_spatial_layers_, number_temporal_layers_, 0.70, 1.15); #if CONFIG_VP9_DECODER // The non-reference frames are expected to be mismatched frames as the // encoder will avoid loopfilter on these frames. diff --git a/media/libvpx/libvpx/test/svc_end_to_end_test.cc b/media/libvpx/libvpx/test/svc_end_to_end_test.cc --- a/media/libvpx/libvpx/test/svc_end_to_end_test.cc +++ b/media/libvpx/libvpx/test/svc_end_to_end_test.cc @@ -13,10 +13,11 @@ #include "test/encode_test_driver.h" #include "test/i420_video_source.h" #include "test/svc_test.h" #include "test/util.h" #include "test/y4m_video_source.h" +#include "vp9/common/vp9_onyxc_int.h" #include "vpx/vpx_codec.h" #include "vpx_ports/bitops.h" namespace svc_test { namespace { @@ -137,10 +138,95 @@ virtual bool DoDecode() const { return current_video_frame_ >= frame_to_start_decode_; } + // Example pattern for spatial layers and 2 temporal layers used in the + // bypass/flexible mode. The pattern corresponds to the pattern + // VP9E_TEMPORAL_LAYERING_MODE_0101 (temporal_layering_mode == 2) used in + // non-flexible mode. + void set_frame_flags_bypass_mode( + int tl, int num_spatial_layers, int is_key_frame, + vpx_svc_ref_frame_config_t *ref_frame_config) { + int sl; + for (sl = 0; sl < num_spatial_layers; ++sl) + ref_frame_config->update_buffer_slot[sl] = 0; + + for (sl = 0; sl < num_spatial_layers; ++sl) { + // Set the buffer idx. + if (tl == 0) { + ref_frame_config->lst_fb_idx[sl] = sl; + if (sl) { + if (is_key_frame) { + ref_frame_config->lst_fb_idx[sl] = sl - 1; + ref_frame_config->gld_fb_idx[sl] = sl; + } else { + ref_frame_config->gld_fb_idx[sl] = sl - 1; + } + } else { + ref_frame_config->gld_fb_idx[sl] = 0; + } + ref_frame_config->alt_fb_idx[sl] = 0; + } else if (tl == 1) { + ref_frame_config->lst_fb_idx[sl] = sl; + ref_frame_config->gld_fb_idx[sl] = + (sl == 0) ? 0 : num_spatial_layers + sl - 1; + ref_frame_config->alt_fb_idx[sl] = num_spatial_layers + sl; + } + // Set the reference and update flags. + if (!tl) { + if (!sl) { + // Base spatial and base temporal (sl = 0, tl = 0) + ref_frame_config->reference_last[sl] = 1; + ref_frame_config->reference_golden[sl] = 0; + ref_frame_config->reference_alt_ref[sl] = 0; + ref_frame_config->update_buffer_slot[sl] |= + 1 << ref_frame_config->lst_fb_idx[sl]; + } else { + if (is_key_frame) { + ref_frame_config->reference_last[sl] = 1; + ref_frame_config->reference_golden[sl] = 0; + ref_frame_config->reference_alt_ref[sl] = 0; + ref_frame_config->update_buffer_slot[sl] |= + 1 << ref_frame_config->gld_fb_idx[sl]; + } else { + // Non-zero spatiall layer. + ref_frame_config->reference_last[sl] = 1; + ref_frame_config->reference_golden[sl] = 1; + ref_frame_config->reference_alt_ref[sl] = 1; + ref_frame_config->update_buffer_slot[sl] |= + 1 << ref_frame_config->lst_fb_idx[sl]; + } + } + } else if (tl == 1) { + if (!sl) { + // Base spatial and top temporal (tl = 1) + ref_frame_config->reference_last[sl] = 1; + ref_frame_config->reference_golden[sl] = 0; + ref_frame_config->reference_alt_ref[sl] = 0; + ref_frame_config->update_buffer_slot[sl] |= + 1 << ref_frame_config->alt_fb_idx[sl]; + } else { + // Non-zero spatial. + if (sl < num_spatial_layers - 1) { + ref_frame_config->reference_last[sl] = 1; + ref_frame_config->reference_golden[sl] = 1; + ref_frame_config->reference_alt_ref[sl] = 0; + ref_frame_config->update_buffer_slot[sl] |= + 1 << ref_frame_config->alt_fb_idx[sl]; + } else if (sl == num_spatial_layers - 1) { + // Top spatial and top temporal (non-reference -- doesn't + // update any reference buffers). + ref_frame_config->reference_last[sl] = 1; + ref_frame_config->reference_golden[sl] = 1; + ref_frame_config->reference_alt_ref[sl] = 0; + } + } + } + } + } + virtual void PreEncodeFrameHook(::libvpx_test::VideoSource *video, ::libvpx_test::Encoder *encoder) { current_video_frame_ = video->frame(); PreEncodeFrameHookSetup(video, encoder); if (video->frame() == 0) { @@ -156,10 +242,25 @@ // mismatch check on color space setting. encoder->Control(VP9E_SET_COLOR_SPACE, VPX_CS_BT_601); encoder->Control(VP9E_SET_DISABLE_LOOPFILTER, loopfilter_off_); } + if (flexible_mode_) { + vpx_svc_layer_id_t layer_id; + layer_id.spatial_layer_id = 0; + layer_id.temporal_layer_id = (video->frame() % 2 != 0); + temporal_layer_id_ = layer_id.temporal_layer_id; + for (int i = 0; i < number_spatial_layers_; i++) { + layer_id.temporal_layer_id_per_spatial[i] = temporal_layer_id_; + ref_frame_config_.duration[i] = 1; + } + encoder->Control(VP9E_SET_SVC_LAYER_ID, &layer_id); + set_frame_flags_bypass_mode(layer_id.temporal_layer_id, + number_spatial_layers_, 0, + &ref_frame_config_); + encoder->Control(VP9E_SET_SVC_REF_FRAME_CONFIG, &ref_frame_config_); + } if (video->frame() == frame_to_sync_) { encoder->Control(VP9E_SET_SVC_SPATIAL_LAYER_SYNC, &svc_layer_sync_); } } @@ -169,13 +270,18 @@ if (video->frame() < frame_to_sync_) { if (decode_to_layer_before_sync_ >= 0) decoder->Control(VP9_DECODE_SVC_SPATIAL_LAYER, decode_to_layer_before_sync_); } else { - if (decode_to_layer_after_sync_ >= 0) - decoder->Control(VP9_DECODE_SVC_SPATIAL_LAYER, - decode_to_layer_after_sync_); + if (decode_to_layer_after_sync_ >= 0) { + int decode_to_layer = decode_to_layer_after_sync_; + // Overlay frame is additional layer for intra-only. + if (video->frame() == frame_to_sync_ && intra_only_test_ && + decode_to_layer_after_sync_ == 0 && number_spatial_layers_ > 1) + decode_to_layer += 1; + decoder->Control(VP9_DECODE_SVC_SPATIAL_LAYER, decode_to_layer); + } } } #endif virtual void FramePktHook(const vpx_codec_cx_pkt_t *pkt) { @@ -219,10 +325,12 @@ bool intra_only_test_; int loopfilter_off_; vpx_svc_spatial_layer_sync_t svc_layer_sync_; unsigned int mismatch_nframes_; unsigned int num_nonref_frames_; + bool flexible_mode_; + vpx_svc_ref_frame_config_t ref_frame_config_; private: virtual void SetConfig(const int num_temporal_layer) { cfg_.rc_buf_initial_sz = 500; cfg_.rc_buf_optimal_sz = 500; @@ -244,11 +352,11 @@ cfg_.ts_rate_decimator[0] = 2; cfg_.ts_rate_decimator[1] = 1; cfg_.temporal_layering_mode = 2; } else if (num_temporal_layer == 1) { cfg_.ts_rate_decimator[0] = 1; - cfg_.temporal_layering_mode = 1; + cfg_.temporal_layering_mode = 0; } } }; // Test for sync layer for 1 pass CBR SVC: 3 spatial layers and @@ -268,10 +376,11 @@ svc_layer_sync_.spatial_layer_sync[0] = 1; ::libvpx_test::Y4mVideoSource video("niklas_1280_720_30.y4m", 0, 60); cfg_.rc_target_bitrate = 600; + flexible_mode_ = false; AssignLayerBitrates(); ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); #if CONFIG_VP9_DECODER // The non-reference frames are expected to be mismatched frames as the // encoder will avoid loopfilter on these frames. @@ -295,10 +404,11 @@ svc_layer_sync_.spatial_layer_sync[1] = 1; ::libvpx_test::I420VideoSource video("niklas_640_480_30.yuv", 640, 480, 30, 1, 0, 400); cfg_.rc_target_bitrate = 400; + flexible_mode_ = false; AssignLayerBitrates(); ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); #if CONFIG_VP9_DECODER // The non-reference frames are expected to be mismatched frames as the // encoder will avoid loopfilter on these frames. @@ -322,10 +432,11 @@ svc_layer_sync_.spatial_layer_sync[1] = 0; svc_layer_sync_.spatial_layer_sync[2] = 1; ::libvpx_test::Y4mVideoSource video("niklas_1280_720_30.y4m", 0, 60); cfg_.rc_target_bitrate = 600; + flexible_mode_ = false; AssignLayerBitrates(); ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); #if CONFIG_VP9_DECODER // The non-reference frames are expected to be mismatched frames as the // encoder will avoid loopfilter on these frames. @@ -349,10 +460,11 @@ svc_layer_sync_.spatial_layer_sync[1] = 1; svc_layer_sync_.spatial_layer_sync[2] = 1; ::libvpx_test::Y4mVideoSource video("niklas_1280_720_30.y4m", 0, 60); cfg_.rc_target_bitrate = 600; + flexible_mode_ = false; AssignLayerBitrates(); ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); #if CONFIG_VP9_DECODER // The non-reference frames are expected to be mismatched frames as the // encoder will avoid loopfilter on these frames. @@ -378,29 +490,86 @@ svc_layer_sync_.spatial_layer_sync[1] = 1; ::libvpx_test::I420VideoSource video("niklas_640_480_30.yuv", 640, 480, 30, 1, 0, 400); cfg_.rc_target_bitrate = 400; + flexible_mode_ = false; AssignLayerBitrates(); ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); #if CONFIG_VP9_DECODER // The non-reference frames are expected to be mismatched frames as the // encoder will avoid loopfilter on these frames. EXPECT_EQ(GetNonRefFrames(), GetMismatchFrames()); #endif } #endif +// Encode 3 spatial, 2 temporal layer in flexible mode but don't +// start decoding. During the sequence insert intra-only on base/qvga +// layer at frame 20 and start decoding only QVGA layer from there. +TEST_P(SyncFrameOnePassCbrSvc, + OnePassCbrSvc3SL3TLSyncFrameStartDecodeOnIntraOnlyQVGAFlex) { + SetSvcConfig(3, 2); + frame_to_start_decode_ = 20; + frame_to_sync_ = 20; + decode_to_layer_before_sync_ = 2; + decode_to_layer_after_sync_ = 0; + intra_only_test_ = true; + + // Set up svc layer sync structure. + svc_layer_sync_.base_layer_intra_only = 1; + svc_layer_sync_.spatial_layer_sync[0] = 1; + svc_layer_sync_.spatial_layer_sync[1] = 0; + svc_layer_sync_.spatial_layer_sync[2] = 0; + + ::libvpx_test::Y4mVideoSource video("niklas_1280_720_30.y4m", 0, 60); + cfg_.rc_target_bitrate = 600; + flexible_mode_ = true; + AssignLayerBitrates(); + cfg_.temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_BYPASS; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + // Can't check mismatch here because only base is decoded at + // frame sync, whereas encoder continues encoding all layers. +} + +// Encode 3 spatial, 3 temporal layer but don't start decoding. +// During the sequence insert intra-only on base/qvga layer at frame 20 +// and start decoding only QVGA layer from there. +TEST_P(SyncFrameOnePassCbrSvc, + OnePassCbrSvc3SL3TLSyncFrameStartDecodeOnIntraOnlyQVGA) { + SetSvcConfig(3, 3); + frame_to_start_decode_ = 20; + frame_to_sync_ = 20; + decode_to_layer_before_sync_ = 2; + decode_to_layer_after_sync_ = 0; + intra_only_test_ = true; + + // Set up svc layer sync structure. + svc_layer_sync_.base_layer_intra_only = 1; + svc_layer_sync_.spatial_layer_sync[0] = 1; + svc_layer_sync_.spatial_layer_sync[1] = 0; + svc_layer_sync_.spatial_layer_sync[2] = 0; + + ::libvpx_test::Y4mVideoSource video("niklas_1280_720_30.y4m", 0, 60); + cfg_.rc_target_bitrate = 600; + flexible_mode_ = false; + AssignLayerBitrates(); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + // Can't check mismatch here because only base is decoded at + // frame sync, whereas encoder continues encoding all layers. +} + // Start decoding from beginning of sequence, during sequence insert intra-only // on base/qvga layer. Decode all layers. TEST_P(SyncFrameOnePassCbrSvc, OnePassCbrSvc3SL3TLSyncFrameIntraOnlyQVGA) { SetSvcConfig(3, 3); frame_to_start_decode_ = 0; frame_to_sync_ = 20; decode_to_layer_before_sync_ = 2; - // The superframe containing intra-only layer will have 4 frames. Thus set the - // layer to decode after sync frame to 3. + // The superframe containing intra-only layer will have +1 frames. Thus set + // the layer to decode after sync frame to +1 from + // decode_to_layer_before_sync. decode_to_layer_after_sync_ = 3; intra_only_test_ = true; // Set up svc layer sync structure. svc_layer_sync_.base_layer_intra_only = 1; @@ -408,10 +577,11 @@ svc_layer_sync_.spatial_layer_sync[1] = 0; svc_layer_sync_.spatial_layer_sync[2] = 0; ::libvpx_test::Y4mVideoSource video("niklas_1280_720_30.y4m", 0, 60); cfg_.rc_target_bitrate = 600; + flexible_mode_ = false; AssignLayerBitrates(); ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); #if CONFIG_VP9_DECODER // The non-reference frames are expected to be mismatched frames as the // encoder will avoid loopfilter on these frames. @@ -424,12 +594,13 @@ TEST_P(SyncFrameOnePassCbrSvc, OnePassCbrSvc3SL3TLSyncFrameIntraOnlyVGA) { SetSvcConfig(3, 3); frame_to_start_decode_ = 0; frame_to_sync_ = 20; decode_to_layer_before_sync_ = 2; - // The superframe containing intra-only layer will have 4 frames. Thus set the - // layer to decode after sync frame to 3. + // The superframe containing intra-only layer will have +1 frames. Thus set + // the layer to decode after sync frame to +1 from + // decode_to_layer_before_sync. decode_to_layer_after_sync_ = 3; intra_only_test_ = true; // Set up svc layer sync structure. svc_layer_sync_.base_layer_intra_only = 1; @@ -437,10 +608,11 @@ svc_layer_sync_.spatial_layer_sync[1] = 1; svc_layer_sync_.spatial_layer_sync[2] = 0; ::libvpx_test::Y4mVideoSource video("niklas_1280_720_30.y4m", 0, 60); cfg_.rc_target_bitrate = 600; + flexible_mode_ = false; AssignLayerBitrates(); ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); #if CONFIG_VP9_DECODER // The non-reference frames are expected to be mismatched frames as the // encoder will avoid loopfilter on these frames. @@ -462,10 +634,11 @@ svc_layer_sync_.base_layer_intra_only = 1; svc_layer_sync_.spatial_layer_sync[0] = 1; ::libvpx_test::Y4mVideoSource video("niklas_1280_720_30.y4m", 0, 60); cfg_.rc_target_bitrate = 600; + flexible_mode_ = false; AssignLayerBitrates(); ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); #if CONFIG_VP9_DECODER // The non-reference frames are expected to be mismatched frames as the // encoder will avoid loopfilter on these frames. diff --git a/media/libvpx/libvpx/test/test-data.mk b/media/libvpx/libvpx/test/test-data.mk --- a/media/libvpx/libvpx/test/test-data.mk +++ b/media/libvpx/libvpx/test/test-data.mk @@ -4,10 +4,11 @@ LIBVPX_TEST_DATA-$(CONFIG_ENCODERS) += hantro_collage_w352h288.yuv LIBVPX_TEST_DATA-$(CONFIG_ENCODERS) += hantro_collage_w352h288_nv12.yuv LIBVPX_TEST_DATA-$(CONFIG_ENCODERS) += hantro_odd.yuv LIBVPX_TEST_DATA-$(CONFIG_ENCODERS) += desktop_office1.1280_720-020.yuv LIBVPX_TEST_DATA-$(CONFIG_ENCODERS) += slides_code_term_web_plot.1920_1080.yuv +LIBVPX_TEST_DATA-$(CONFIG_ENCODERS) += desktopqvga.320_240.yuv LIBVPX_TEST_DATA-$(CONFIG_ENCODERS) += park_joy_90p_10_420_20f.y4m LIBVPX_TEST_DATA-$(CONFIG_ENCODERS) += park_joy_90p_10_422_20f.y4m LIBVPX_TEST_DATA-$(CONFIG_ENCODERS) += park_joy_90p_10_444_20f.y4m LIBVPX_TEST_DATA-$(CONFIG_ENCODERS) += park_joy_90p_10_440.yuv diff --git a/media/libvpx/libvpx/test/test-data.sha1 b/media/libvpx/libvpx/test/test-data.sha1 --- a/media/libvpx/libvpx/test/test-data.sha1 +++ b/media/libvpx/libvpx/test/test-data.sha1 @@ -867,5 +867,6 @@ bac455906360b45338a16dd626ac5f19bc36a307 *desktop_office1.1280_720-020.yuv 094be4b80fa30bd227149ea16ab6476d549ea092 *slides_code_term_web_plot.1920_1080.yuv 518a0be998afece76d3df76047d51e256c591ff2 *invalid-bug-148271109.ivf d3964f9dad9f60363c81b688324d95b4ec7c8038 *invalid-bug-148271109.ivf.res ad18ca16f0a249fb3b7c38de0d9b327fed273f96 *hantro_collage_w352h288_nv12.yuv +8a0b2c350539859463d3546a67876c83ff6ff0ac *desktopqvga.320_240.yuv diff --git a/media/libvpx/libvpx/test/test.mk b/media/libvpx/libvpx/test/test.mk --- a/media/libvpx/libvpx/test/test.mk +++ b/media/libvpx/libvpx/test/test.mk @@ -154,10 +154,11 @@ LIBVPX_TEST_SRCS-yes += partial_idct_test.cc LIBVPX_TEST_SRCS-yes += superframe_test.cc LIBVPX_TEST_SRCS-yes += tile_independence_test.cc LIBVPX_TEST_SRCS-yes += vp9_boolcoder_test.cc LIBVPX_TEST_SRCS-yes += vp9_encoder_parms_get_to_decoder.cc +LIBVPX_TEST_SRCS-yes += vp9_roi_test.cc endif LIBVPX_TEST_SRCS-yes += convolve_test.cc LIBVPX_TEST_SRCS-yes += lpf_test.cc LIBVPX_TEST_SRCS-yes += vp9_intrapred_test.cc @@ -211,13 +212,15 @@ endif TEST_INTRA_PRED_SPEED_SRCS-yes := test_intra_pred_speed.cc TEST_INTRA_PRED_SPEED_SRCS-yes += ../md5_utils.h ../md5_utils.c -RC_INTERFACE_TEST_SRCS-$(CONFIG_VP9_ENCODER) := ratectrl_rtc_test.cc -RC_INTERFACE_TEST_SRCS-$(CONFIG_VP9_ENCODER) += encode_test_driver.cc -RC_INTERFACE_TEST_SRCS-$(CONFIG_VP9_ENCODER) += encode_test_driver.h +RC_INTERFACE_TEST_SRCS-yes := test_rc_interface.cc +RC_INTERFACE_TEST_SRCS-$(CONFIG_VP9_ENCODER) += vp9_ratectrl_rtc_test.cc +RC_INTERFACE_TEST_SRCS-$(CONFIG_VP8_ENCODER) += vp8_ratectrl_rtc_test.cc +RC_INTERFACE_TEST_SRCS-$(CONFIG_ENCODERS) += encode_test_driver.cc +RC_INTERFACE_TEST_SRCS-$(CONFIG_ENCODERS) += encode_test_driver.h RC_INTERFACE_TEST_SRCS-yes += decode_test_driver.cc RC_INTERFACE_TEST_SRCS-yes += decode_test_driver.h RC_INTERFACE_TEST_SRCS-yes += codec_factory.h endif # CONFIG_SHARED diff --git a/media/libvpx/libvpx/test/test_intra_pred_speed.cc b/media/libvpx/libvpx/test/test_intra_pred_speed.cc --- a/media/libvpx/libvpx/test/test_intra_pred_speed.cc +++ b/media/libvpx/libvpx/test/test_intra_pred_speed.cc @@ -46,15 +46,13 @@ const int mask = (1 << bd) - 1; for (int i = 0; i < kTotalPixels; ++i) ref_src[i] = rnd.Rand16() & mask; for (int i = 0; i < kBPS; ++i) left[i] = rnd.Rand16() & mask; for (int i = -1; i < kBPS; ++i) above[i] = rnd.Rand16() & mask; - // some code assumes the top row has been extended: - // d45/d63 C-code, for instance, but not the assembly. - // TODO(jzern): this style of extension isn't strictly necessary. + // d45/d63 require the top row to be extended. ASSERT_LE(block_size, kBPS); - for (int i = block_size; i < 2 * kBPS; ++i) { + for (int i = block_size; i < 2 * block_size; ++i) { above[i] = above[block_size - 1]; } } DECLARE_ALIGNED(16, Pixel, src[kTotalPixels]); diff --git a/media/libvpx/libvpx/test/test_rc_interface.cc b/media/libvpx/libvpx/test/test_rc_interface.cc new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/test/test_rc_interface.cc @@ -0,0 +1,6 @@ +#include "third_party/googletest/src/include/gtest/gtest.h" + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/media/libvpx/libvpx/test/variance_test.cc b/media/libvpx/libvpx/test/variance_test.cc --- a/media/libvpx/libvpx/test/variance_test.cc +++ b/media/libvpx/libvpx/test/variance_test.cc @@ -594,10 +594,11 @@ } protected: void RefTest(); void ExtremeRefTest(); + void SpeedTest(); ACMRandom rnd_; uint8_t *src_; uint8_t *ref_; uint8_t *sec_; @@ -679,10 +680,41 @@ EXPECT_EQ(var1, var2) << "for xoffset " << x << " and yoffset " << y; } } } +template +void SubpelVarianceTest::SpeedTest() { + // The only interesting points are 0, 4, and anything else. To make the loops + // simple we will use 0, 2 and 4. + for (int x = 0; x <= 4; x += 2) { + for (int y = 0; y <= 4; y += 2) { + if (!use_high_bit_depth()) { + memset(src_, 25, block_size()); + memset(ref_, 50, block_size()); +#if CONFIG_VP9_HIGHBITDEPTH + } else { + vpx_memset16(CONVERT_TO_SHORTPTR(src_), 25, block_size()); + vpx_memset16(CONVERT_TO_SHORTPTR(ref_), 50, block_size()); +#endif // CONFIG_VP9_HIGHBITDEPTH + } + unsigned int sse; + vpx_usec_timer timer; + vpx_usec_timer_start(&timer); + for (int i = 0; i < 1000000000 / block_size(); ++i) { + const uint32_t variance = + params_.func(ref_, width() + 1, x, y, src_, width(), &sse); + (void)variance; + } + vpx_usec_timer_mark(&timer); + const int elapsed_time = static_cast(vpx_usec_timer_elapsed(&timer)); + printf("SubpelVariance %dx%d xoffset: %d yoffset: %d time: %5d ms\n", + width(), height(), x, y, elapsed_time / 1000); + } + } +} + template <> void SubpelVarianceTest::RefTest() { for (int x = 0; x < 8; ++x) { for (int y = 0; y < 8; ++y) { if (!use_high_bit_depth()) { @@ -734,10 +766,11 @@ TEST_P(VpxVarianceTest, DISABLED_Speed) { SpeedTest(); } TEST_P(SumOfSquaresTest, Const) { ConstTest(); } TEST_P(SumOfSquaresTest, Ref) { RefTest(); } TEST_P(VpxSubpelVarianceTest, Ref) { RefTest(); } TEST_P(VpxSubpelVarianceTest, ExtremeRef) { ExtremeRefTest(); } +TEST_P(VpxSubpelVarianceTest, DISABLED_Speed) { SpeedTest(); } TEST_P(VpxSubpelAvgVarianceTest, Ref) { RefTest(); } INSTANTIATE_TEST_SUITE_P(C, SumOfSquaresTest, ::testing::Values(vpx_get_mb_ss_c)); @@ -1614,6 +1647,18 @@ SubpelAvgVarianceParams(3, 3, &vpx_sub_pixel_avg_variance8x8_mmi, 0), SubpelAvgVarianceParams(3, 2, &vpx_sub_pixel_avg_variance8x4_mmi, 0), SubpelAvgVarianceParams(2, 3, &vpx_sub_pixel_avg_variance4x8_mmi, 0), SubpelAvgVarianceParams(2, 2, &vpx_sub_pixel_avg_variance4x4_mmi, 0))); #endif // HAVE_MMI + +#if HAVE_LSX +INSTANTIATE_TEST_SUITE_P( + LSX, VpxVarianceTest, + ::testing::Values(VarianceParams(6, 6, &vpx_variance64x64_lsx), + VarianceParams(5, 5, &vpx_variance32x32_lsx), + VarianceParams(4, 4, &vpx_variance16x16_lsx))); + +INSTANTIATE_TEST_SUITE_P(LSX, VpxSubpelVarianceTest, + ::testing::Values(SubpelVarianceParams( + 5, 5, &vpx_sub_pixel_variance32x32_lsx, 0))); +#endif } // namespace diff --git a/media/libvpx/libvpx/test/video_source.h b/media/libvpx/libvpx/test/video_source.h --- a/media/libvpx/libvpx/test/video_source.h +++ b/media/libvpx/libvpx/test/video_source.h @@ -189,11 +189,12 @@ } void ReallocImage() { vpx_img_free(img_); img_ = vpx_img_alloc(NULL, format_, width_, height_, 32); - raw_sz_ = ((img_->w + 31) & ~31) * img_->h * img_->bps / 8; + ASSERT_NE(img_, nullptr); + raw_sz_ = ((img_->w + 31) & ~31u) * img_->h * img_->bps / 8; } vpx_image_t *img_; size_t raw_sz_; unsigned int limit_; diff --git a/media/libvpx/libvpx/test/vp8_fdct4x4_test.cc b/media/libvpx/libvpx/test/vp8_fdct4x4_test.cc --- a/media/libvpx/libvpx/test/vp8_fdct4x4_test.cc +++ b/media/libvpx/libvpx/test/vp8_fdct4x4_test.cc @@ -146,11 +146,11 @@ (abs(count_sign_block[j][0] - count_sign_block[j][1]) < 100000); } EXPECT_EQ(true, bias_acceptable) << "Error: 4x4 FDCT has a sign bias > 10% for input range [-15, 15]"; -}; +} TEST_P(FdctTest, RoundTripErrorCheck) { int max_error = 0; double total_error = 0; const int count_test_block = 1000000; @@ -179,11 +179,11 @@ EXPECT_GE(1, max_error) << "Error: FDCT/IDCT has an individual roundtrip error > 1"; EXPECT_GE(count_test_block, total_error) << "Error: FDCT/IDCT has average roundtrip error > 1 per block"; -}; +} INSTANTIATE_TEST_SUITE_P(C, FdctTest, ::testing::Values(vp8_short_fdct4x4_c)); #if HAVE_NEON INSTANTIATE_TEST_SUITE_P(NEON, FdctTest, diff --git a/media/libvpx/libvpx/test/vp8_ratectrl_rtc_test.cc b/media/libvpx/libvpx/test/vp8_ratectrl_rtc_test.cc new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/test/vp8_ratectrl_rtc_test.cc @@ -0,0 +1,343 @@ +/* + * Copyright (c) 2021 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include // NOLINT +#include + +#include "./vpx_config.h" +#include "third_party/googletest/src/include/gtest/gtest.h" +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/i420_video_source.h" +#include "test/util.h" +#include "test/video_source.h" +#include "vp8/vp8_ratectrl_rtc.h" +#include "vpx/vpx_codec.h" +#include "vpx_ports/bitops.h" + +namespace { + +struct Vp8RCTestVideo { + Vp8RCTestVideo() {} + Vp8RCTestVideo(const char *name_, int width_, int height_, + unsigned int frames_) + : name(name_), width(width_), height(height_), frames(frames_) {} + + friend std::ostream &operator<<(std::ostream &os, + const Vp8RCTestVideo &video) { + os << video.name << " " << video.width << " " << video.height << " " + << video.frames; + return os; + } + const char *name; + int width; + int height; + unsigned int frames; +}; + +const Vp8RCTestVideo kVp8RCTestVectors[] = { + Vp8RCTestVideo("niklas_640_480_30.yuv", 640, 480, 470), + Vp8RCTestVideo("desktop_office1.1280_720-020.yuv", 1280, 720, 300), +}; + +class Vp8RcInterfaceTest + : public ::libvpx_test::EncoderTest, + public ::libvpx_test::CodecTestWith2Params { + public: + Vp8RcInterfaceTest() + : EncoderTest(GET_PARAM(0)), key_interval_(3000), encoder_exit_(false) {} + virtual ~Vp8RcInterfaceTest() {} + + protected: + virtual void SetUp() { + InitializeConfig(); + SetMode(::libvpx_test::kRealTime); + } + + // From error_resilience_test.cc + int SetFrameFlags(int frame_num, int num_temp_layers) { + int frame_flags = 0; + if (num_temp_layers == 2) { + if (frame_num % 2 == 0) { + // Layer 0: predict from L and ARF, update L. + frame_flags = + VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF; + } else { + // Layer 1: predict from L, G and ARF, and update G. + frame_flags = VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST | + VP8_EFLAG_NO_UPD_ENTROPY; + } + } else if (num_temp_layers == 3) { + if (frame_num % 4 == 0) { + // Layer 0: predict from L, update L. + frame_flags = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | + VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF; + } else if ((frame_num - 2) % 4 == 0) { + // Layer 1: predict from L, G, update G. + frame_flags = + VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_REF_ARF; + } else if ((frame_num - 1) % 2 == 0) { + // Layer 2: predict from L, G, ARF; update ARG. + frame_flags = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_LAST; + } + } + return frame_flags; + } + + int SetLayerId(int frame_num, int num_temp_layers) { + int layer_id = 0; + if (num_temp_layers == 2) { + if (frame_num % 2 == 0) { + layer_id = 0; + } else { + layer_id = 1; + } + } else if (num_temp_layers == 3) { + if (frame_num % 4 == 0) { + layer_id = 0; + } else if ((frame_num - 2) % 4 == 0) { + layer_id = 1; + } else if ((frame_num - 1) % 2 == 0) { + layer_id = 2; + } + } + return layer_id; + } + + virtual void PreEncodeFrameHook(::libvpx_test::VideoSource *video, + ::libvpx_test::Encoder *encoder) { + if (rc_cfg_.ts_number_layers > 1) { + const int layer_id = SetLayerId(video->frame(), cfg_.ts_number_layers); + const int frame_flags = + SetFrameFlags(video->frame(), cfg_.ts_number_layers); + frame_params_.temporal_layer_id = layer_id; + if (video->frame() > 0) { + encoder->Control(VP8E_SET_TEMPORAL_LAYER_ID, layer_id); + encoder->Control(VP8E_SET_FRAME_FLAGS, frame_flags); + } + } else { + if (video->frame() == 0) { + encoder->Control(VP8E_SET_CPUUSED, -6); + encoder->Control(VP8E_SET_RTC_EXTERNAL_RATECTRL, 1); + encoder->Control(VP8E_SET_MAX_INTRA_BITRATE_PCT, 1000); + } + if (frame_params_.frame_type == INTER_FRAME) { + // Disable golden frame update. + frame_flags_ |= VP8_EFLAG_NO_UPD_GF; + frame_flags_ |= VP8_EFLAG_NO_UPD_ARF; + } + } + frame_params_.frame_type = + video->frame() % key_interval_ == 0 ? KEY_FRAME : INTER_FRAME; + encoder_exit_ = video->frame() == test_video_.frames; + } + + virtual void PostEncodeFrameHook(::libvpx_test::Encoder *encoder) { + if (encoder_exit_) { + return; + } + int qp; + encoder->Control(VP8E_GET_LAST_QUANTIZER, &qp); + rc_api_->ComputeQP(frame_params_); + ASSERT_EQ(rc_api_->GetQP(), qp); + } + + virtual void FramePktHook(const vpx_codec_cx_pkt_t *pkt) { + rc_api_->PostEncodeUpdate(pkt->data.frame.sz); + } + + void RunOneLayer() { + test_video_ = GET_PARAM(2); + target_bitrate_ = GET_PARAM(1); + if (test_video_.width == 1280 && target_bitrate_ == 200) return; + if (test_video_.width == 640 && target_bitrate_ == 1000) return; + SetConfig(); + rc_api_ = libvpx::VP8RateControlRTC::Create(rc_cfg_); + rc_api_->UpdateRateControl(rc_cfg_); + + ::libvpx_test::I420VideoSource video(test_video_.name, test_video_.width, + test_video_.height, 30, 1, 0, + test_video_.frames); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + } + + void RunPeriodicKey() { + test_video_ = GET_PARAM(2); + target_bitrate_ = GET_PARAM(1); + if (test_video_.width == 1280 && target_bitrate_ == 200) return; + if (test_video_.width == 640 && target_bitrate_ == 1000) return; + key_interval_ = 100; + SetConfig(); + rc_api_ = libvpx::VP8RateControlRTC::Create(rc_cfg_); + rc_api_->UpdateRateControl(rc_cfg_); + + ::libvpx_test::I420VideoSource video(test_video_.name, test_video_.width, + test_video_.height, 30, 1, 0, + test_video_.frames); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + } + + void RunTemporalLayers2TL() { + test_video_ = GET_PARAM(2); + target_bitrate_ = GET_PARAM(1); + if (test_video_.width == 1280 && target_bitrate_ == 200) return; + if (test_video_.width == 640 && target_bitrate_ == 1000) return; + SetConfigTemporalLayers(2); + rc_api_ = libvpx::VP8RateControlRTC::Create(rc_cfg_); + rc_api_->UpdateRateControl(rc_cfg_); + + ::libvpx_test::I420VideoSource video(test_video_.name, test_video_.width, + test_video_.height, 30, 1, 0, + test_video_.frames); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + } + + void RunTemporalLayers3TL() { + test_video_ = GET_PARAM(2); + target_bitrate_ = GET_PARAM(1); + if (test_video_.width == 1280 && target_bitrate_ == 200) return; + if (test_video_.width == 640 && target_bitrate_ == 1000) return; + SetConfigTemporalLayers(3); + rc_api_ = libvpx::VP8RateControlRTC::Create(rc_cfg_); + rc_api_->UpdateRateControl(rc_cfg_); + + ::libvpx_test::I420VideoSource video(test_video_.name, test_video_.width, + test_video_.height, 30, 1, 0, + test_video_.frames); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + } + + private: + void SetConfig() { + rc_cfg_.width = test_video_.width; + rc_cfg_.height = test_video_.height; + rc_cfg_.max_quantizer = 60; + rc_cfg_.min_quantizer = 2; + rc_cfg_.target_bandwidth = target_bitrate_; + rc_cfg_.buf_initial_sz = 600; + rc_cfg_.buf_optimal_sz = 600; + rc_cfg_.buf_sz = target_bitrate_; + rc_cfg_.undershoot_pct = 50; + rc_cfg_.overshoot_pct = 50; + rc_cfg_.max_intra_bitrate_pct = 1000; + rc_cfg_.framerate = 30.0; + rc_cfg_.layer_target_bitrate[0] = target_bitrate_; + + // Encoder settings for ground truth. + cfg_.g_w = test_video_.width; + cfg_.g_h = test_video_.height; + cfg_.rc_undershoot_pct = 50; + cfg_.rc_overshoot_pct = 50; + cfg_.rc_buf_initial_sz = 600; + cfg_.rc_buf_optimal_sz = 600; + cfg_.rc_buf_sz = target_bitrate_; + cfg_.rc_dropframe_thresh = 0; + cfg_.rc_min_quantizer = 2; + cfg_.rc_max_quantizer = 60; + cfg_.rc_end_usage = VPX_CBR; + cfg_.g_lag_in_frames = 0; + cfg_.g_error_resilient = 1; + cfg_.rc_target_bitrate = target_bitrate_; + cfg_.kf_min_dist = key_interval_; + cfg_.kf_max_dist = key_interval_; + } + + void SetConfigTemporalLayers(int temporal_layers) { + rc_cfg_.width = test_video_.width; + rc_cfg_.height = test_video_.height; + rc_cfg_.max_quantizer = 60; + rc_cfg_.min_quantizer = 2; + rc_cfg_.target_bandwidth = target_bitrate_; + rc_cfg_.buf_initial_sz = 600; + rc_cfg_.buf_optimal_sz = 600; + rc_cfg_.buf_sz = target_bitrate_; + rc_cfg_.undershoot_pct = 50; + rc_cfg_.overshoot_pct = 50; + rc_cfg_.max_intra_bitrate_pct = 1000; + rc_cfg_.framerate = 30.0; + if (temporal_layers == 2) { + rc_cfg_.layer_target_bitrate[0] = 60 * target_bitrate_ / 100; + rc_cfg_.layer_target_bitrate[1] = target_bitrate_; + rc_cfg_.ts_rate_decimator[0] = 2; + rc_cfg_.ts_rate_decimator[1] = 1; + } else if (temporal_layers == 3) { + rc_cfg_.layer_target_bitrate[0] = 40 * target_bitrate_ / 100; + rc_cfg_.layer_target_bitrate[1] = 60 * target_bitrate_ / 100; + rc_cfg_.layer_target_bitrate[2] = target_bitrate_; + rc_cfg_.ts_rate_decimator[0] = 4; + rc_cfg_.ts_rate_decimator[1] = 2; + rc_cfg_.ts_rate_decimator[2] = 1; + } + + rc_cfg_.ts_number_layers = temporal_layers; + + // Encoder settings for ground truth. + cfg_.g_w = test_video_.width; + cfg_.g_h = test_video_.height; + cfg_.rc_undershoot_pct = 50; + cfg_.rc_overshoot_pct = 50; + cfg_.rc_buf_initial_sz = 600; + cfg_.rc_buf_optimal_sz = 600; + cfg_.rc_buf_sz = target_bitrate_; + cfg_.rc_dropframe_thresh = 0; + cfg_.rc_min_quantizer = 2; + cfg_.rc_max_quantizer = 60; + cfg_.rc_end_usage = VPX_CBR; + cfg_.g_lag_in_frames = 0; + cfg_.g_error_resilient = 1; + cfg_.rc_target_bitrate = target_bitrate_; + cfg_.kf_min_dist = key_interval_; + cfg_.kf_max_dist = key_interval_; + // 2 Temporal layers, no spatial layers, CBR mode. + cfg_.ss_number_layers = 1; + cfg_.ts_number_layers = temporal_layers; + if (temporal_layers == 2) { + cfg_.ts_rate_decimator[0] = 2; + cfg_.ts_rate_decimator[1] = 1; + cfg_.ts_periodicity = 2; + cfg_.ts_target_bitrate[0] = 60 * cfg_.rc_target_bitrate / 100; + cfg_.ts_target_bitrate[1] = cfg_.rc_target_bitrate; + } else if (temporal_layers == 3) { + cfg_.ts_rate_decimator[0] = 4; + cfg_.ts_rate_decimator[1] = 2; + cfg_.ts_rate_decimator[2] = 1; + cfg_.ts_periodicity = 4; + cfg_.ts_target_bitrate[0] = 40 * cfg_.rc_target_bitrate / 100; + cfg_.ts_target_bitrate[1] = 60 * cfg_.rc_target_bitrate / 100; + cfg_.ts_target_bitrate[2] = cfg_.rc_target_bitrate; + } + } + + std::unique_ptr rc_api_; + libvpx::VP8RateControlRtcConfig rc_cfg_; + int key_interval_; + int target_bitrate_; + Vp8RCTestVideo test_video_; + libvpx::VP8FrameParamsQpRTC frame_params_; + bool encoder_exit_; +}; + +TEST_P(Vp8RcInterfaceTest, OneLayer) { RunOneLayer(); } + +TEST_P(Vp8RcInterfaceTest, OneLayerPeriodicKey) { RunPeriodicKey(); } + +TEST_P(Vp8RcInterfaceTest, TemporalLayers2TL) { RunTemporalLayers2TL(); } + +TEST_P(Vp8RcInterfaceTest, TemporalLayers3TL) { RunTemporalLayers3TL(); } + +VP8_INSTANTIATE_TEST_SUITE(Vp8RcInterfaceTest, + ::testing::Values(200, 400, 1000), + ::testing::ValuesIn(kVp8RCTestVectors)); + +} // namespace diff --git a/media/libvpx/libvpx/test/vp9_quantize_test.cc b/media/libvpx/libvpx/test/vp9_quantize_test.cc --- a/media/libvpx/libvpx/test/vp9_quantize_test.cc +++ b/media/libvpx/libvpx/test/vp9_quantize_test.cc @@ -36,40 +36,37 @@ namespace { const int number_of_iterations = 100; typedef void (*QuantizeFunc)(const tran_low_t *coeff, intptr_t count, - int skip_block, const int16_t *zbin, - const int16_t *round, const int16_t *quant, - const int16_t *quant_shift, tran_low_t *qcoeff, - tran_low_t *dqcoeff, const int16_t *dequant, - uint16_t *eob, const int16_t *scan, - const int16_t *iscan); + const int16_t *zbin, const int16_t *round, + const int16_t *quant, const int16_t *quant_shift, + tran_low_t *qcoeff, tran_low_t *dqcoeff, + const int16_t *dequant, uint16_t *eob, + const int16_t *scan, const int16_t *iscan); typedef std::tuple QuantizeParam; // Wrapper for FP version which does not use zbin or quant_shift. typedef void (*QuantizeFPFunc)(const tran_low_t *coeff, intptr_t count, - int skip_block, const int16_t *round, - const int16_t *quant, tran_low_t *qcoeff, - tran_low_t *dqcoeff, const int16_t *dequant, - uint16_t *eob, const int16_t *scan, - const int16_t *iscan); + const int16_t *round, const int16_t *quant, + tran_low_t *qcoeff, tran_low_t *dqcoeff, + const int16_t *dequant, uint16_t *eob, + const int16_t *scan, const int16_t *iscan); template -void QuantFPWrapper(const tran_low_t *coeff, intptr_t count, int skip_block, +void QuantFPWrapper(const tran_low_t *coeff, intptr_t count, const int16_t *zbin, const int16_t *round, const int16_t *quant, const int16_t *quant_shift, tran_low_t *qcoeff, tran_low_t *dqcoeff, const int16_t *dequant, uint16_t *eob, const int16_t *scan, const int16_t *iscan) { (void)zbin; (void)quant_shift; - fn(coeff, count, skip_block, round, quant, qcoeff, dqcoeff, dequant, eob, - scan, iscan); + fn(coeff, count, round, quant, qcoeff, dqcoeff, dequant, eob, scan, iscan); } class VP9QuantizeBase : public AbstractBench { public: VP9QuantizeBase(vpx_bit_depth_t bit_depth, int max_size, bool is_fp) @@ -136,11 +133,10 @@ Buffer qcoeff_; Buffer dqcoeff_; int16_t *r_ptr_; int16_t *q_ptr_; int count_; - int skip_block_; const scan_order *scan_; uint16_t eob_; }; class VP9QuantizeTest : public VP9QuantizeBase, @@ -155,30 +151,28 @@ const QuantizeFunc quantize_op_; const QuantizeFunc ref_quantize_op_; }; void VP9QuantizeTest::Run() { - quantize_op_(coeff_.TopLeftPixel(), count_, skip_block_, zbin_ptr_, r_ptr_, - q_ptr_, quant_shift_ptr_, qcoeff_.TopLeftPixel(), + quantize_op_(coeff_.TopLeftPixel(), count_, zbin_ptr_, r_ptr_, q_ptr_, + quant_shift_ptr_, qcoeff_.TopLeftPixel(), dqcoeff_.TopLeftPixel(), dequant_ptr_, &eob_, scan_->scan, scan_->iscan); } // This quantizer compares the AC coefficients to the quantization step size to // determine if further multiplication operations are needed. // Based on vp9_quantize_fp_sse2(). inline void quant_fp_nz(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *round_ptr, - const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, - uint16_t *eob_ptr, const int16_t *scan, - const int16_t *iscan, int is_32x32) { + const int16_t *round_ptr, const int16_t *quant_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan, + int is_32x32) { int i, eob = -1; const int thr = dequant_ptr[1] >> (1 + is_32x32); (void)iscan; - (void)skip_block; - assert(!skip_block); // Quantization pass: All coefficients with index >= zero_flag are // skippable. Note: zero_flag can be zero. for (i = 0; i < n_coeffs; i += 16) { int y; @@ -241,26 +235,24 @@ } *eob_ptr = eob + 1; } void quantize_fp_nz_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *round_ptr, - const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, - uint16_t *eob_ptr, const int16_t *scan, - const int16_t *iscan) { - quant_fp_nz(coeff_ptr, n_coeffs, skip_block, round_ptr, quant_ptr, qcoeff_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { + quant_fp_nz(coeff_ptr, n_coeffs, round_ptr, quant_ptr, qcoeff_ptr, dqcoeff_ptr, dequant_ptr, eob_ptr, scan, iscan, 0); } void quantize_fp_32x32_nz_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *round_ptr, - const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, - uint16_t *eob_ptr, const int16_t *scan, - const int16_t *iscan) { - quant_fp_nz(coeff_ptr, n_coeffs, skip_block, round_ptr, quant_ptr, qcoeff_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { + quant_fp_nz(coeff_ptr, n_coeffs, round_ptr, quant_ptr, qcoeff_ptr, dqcoeff_ptr, dequant_ptr, eob_ptr, scan, iscan, 1); } void GenerateHelperArrays(ACMRandom *rnd, int16_t *zbin, int16_t *round, int16_t *quant, int16_t *quant_shift, @@ -314,13 +306,10 @@ ASSERT_TRUE(ref_dqcoeff.Init()); uint16_t ref_eob = 0; eob_ = 0; for (int i = 0; i < number_of_iterations; ++i) { - // Test skip block for the first three iterations to catch all the different - // sizes. - const int skip_block = 0; TX_SIZE sz; if (max_size_ == 16) { sz = static_cast(i % 3); // TX_4X4, TX_8X8 TX_16X16 } else { sz = TX_32X32; @@ -330,17 +319,17 @@ count_ = (4 << sz) * (4 << sz); coeff_.Set(&rnd, -max_value_, max_value_); GenerateHelperArrays(&rnd, zbin_ptr_, round_ptr_, quant_ptr_, quant_shift_ptr_, dequant_ptr_, round_fp_ptr_, quant_fp_ptr_); - ref_quantize_op_(coeff_.TopLeftPixel(), count_, skip_block, zbin_ptr_, - r_ptr_, q_ptr_, quant_shift_ptr_, - ref_qcoeff.TopLeftPixel(), ref_dqcoeff.TopLeftPixel(), - dequant_ptr_, &ref_eob, scan_->scan, scan_->iscan); + ref_quantize_op_(coeff_.TopLeftPixel(), count_, zbin_ptr_, r_ptr_, q_ptr_, + quant_shift_ptr_, ref_qcoeff.TopLeftPixel(), + ref_dqcoeff.TopLeftPixel(), dequant_ptr_, &ref_eob, + scan_->scan, scan_->iscan); ASM_REGISTER_STATE_CHECK(quantize_op_( - coeff_.TopLeftPixel(), count_, skip_block, zbin_ptr_, r_ptr_, q_ptr_, + coeff_.TopLeftPixel(), count_, zbin_ptr_, r_ptr_, q_ptr_, quant_shift_ptr_, qcoeff_.TopLeftPixel(), dqcoeff_.TopLeftPixel(), dequant_ptr_, &eob_, scan_->scan, scan_->iscan)); EXPECT_TRUE(qcoeff_.CheckValues(ref_qcoeff)); EXPECT_TRUE(dqcoeff_.CheckValues(ref_dqcoeff)); @@ -370,11 +359,10 @@ uint16_t ref_eob = 0; eob_ = 0; const uint32_t max_index = max_size_ * max_size_ - 1; for (int i = 0; i < number_of_iterations; ++i) { - skip_block_ = 0; TX_SIZE sz; if (max_size_ == 16) { sz = static_cast(i % 3); // TX_4X4, TX_8X8 TX_16X16 } else { sz = TX_32X32; @@ -389,17 +377,17 @@ coeff_.TopLeftPixel()[rnd.RandRange(count_) & max_index] = static_cast(rnd.RandRange(max_value_ * 2)) - max_value_; GenerateHelperArrays(&rnd, zbin_ptr_, round_ptr_, quant_ptr_, quant_shift_ptr_, dequant_ptr_, round_fp_ptr_, quant_fp_ptr_); - ref_quantize_op_(coeff_.TopLeftPixel(), count_, skip_block_, zbin_ptr_, - r_ptr_, q_ptr_, quant_shift_ptr_, - ref_qcoeff.TopLeftPixel(), ref_dqcoeff.TopLeftPixel(), - dequant_ptr_, &ref_eob, scan_->scan, scan_->iscan); + ref_quantize_op_(coeff_.TopLeftPixel(), count_, zbin_ptr_, r_ptr_, q_ptr_, + quant_shift_ptr_, ref_qcoeff.TopLeftPixel(), + ref_dqcoeff.TopLeftPixel(), dequant_ptr_, &ref_eob, + scan_->scan, scan_->iscan); ASM_REGISTER_STATE_CHECK(quantize_op_( - coeff_.TopLeftPixel(), count_, skip_block_, zbin_ptr_, r_ptr_, q_ptr_, + coeff_.TopLeftPixel(), count_, zbin_ptr_, r_ptr_, q_ptr_, quant_shift_ptr_, qcoeff_.TopLeftPixel(), dqcoeff_.TopLeftPixel(), dequant_ptr_, &eob_, scan_->scan, scan_->iscan)); EXPECT_TRUE(qcoeff_.CheckValues(ref_qcoeff)); EXPECT_TRUE(dqcoeff_.CheckValues(ref_dqcoeff)); @@ -431,11 +419,10 @@ } for (TX_SIZE sz = starting_sz; sz <= ending_sz; ++sz) { // zbin > coeff, zbin < coeff. for (int i = 0; i < 2; ++i) { - skip_block_ = 0; // TX_TYPE defines the scan order. That is not relevant to the speed test. // Pick the first one. const TX_TYPE tx_type = DCT_DCT; count_ = (4 << sz) * (4 << sz); scan_ = &vp9_scan_orders[sz][tx_type]; diff --git a/media/libvpx/libvpx/test/vp9_ratectrl_rtc_test.cc b/media/libvpx/libvpx/test/vp9_ratectrl_rtc_test.cc new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/test/vp9_ratectrl_rtc_test.cc @@ -0,0 +1,373 @@ +/* + * Copyright (c) 2020 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "vp9/ratectrl_rtc.h" + +#include // NOLINT +#include + +#include "./vpx_config.h" +#include "third_party/googletest/src/include/gtest/gtest.h" +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/i420_video_source.h" +#include "test/util.h" +#include "test/video_source.h" +#include "vpx/vpx_codec.h" +#include "vpx_ports/bitops.h" + +namespace { + +const size_t kNumFrames = 300; + +const int kTemporalId[4] = { 0, 2, 1, 2 }; + +class RcInterfaceTest + : public ::libvpx_test::EncoderTest, + public ::libvpx_test::CodecTestWith2Params { + public: + RcInterfaceTest() + : EncoderTest(GET_PARAM(0)), aq_mode_(GET_PARAM(1)), key_interval_(3000), + encoder_exit_(false) {} + + virtual ~RcInterfaceTest() {} + + protected: + virtual void SetUp() { + InitializeConfig(); + SetMode(::libvpx_test::kRealTime); + } + + virtual void PreEncodeFrameHook(libvpx_test::VideoSource *video, + libvpx_test::Encoder *encoder) { + if (video->frame() == 0) { + encoder->Control(VP8E_SET_CPUUSED, 7); + encoder->Control(VP9E_SET_AQ_MODE, aq_mode_); + encoder->Control(VP9E_SET_TUNE_CONTENT, 0); + encoder->Control(VP8E_SET_MAX_INTRA_BITRATE_PCT, 1000); + encoder->Control(VP9E_SET_RTC_EXTERNAL_RATECTRL, 1); + } + frame_params_.frame_type = + video->frame() % key_interval_ == 0 ? KEY_FRAME : INTER_FRAME; + if (rc_cfg_.rc_mode == VPX_CBR && frame_params_.frame_type == INTER_FRAME) { + // Disable golden frame update. + frame_flags_ |= VP8_EFLAG_NO_UPD_GF; + frame_flags_ |= VP8_EFLAG_NO_UPD_ARF; + } + encoder_exit_ = video->frame() == kNumFrames; + } + + virtual void PostEncodeFrameHook(::libvpx_test::Encoder *encoder) { + if (encoder_exit_) { + return; + } + int loopfilter_level, qp; + encoder->Control(VP9E_GET_LOOPFILTER_LEVEL, &loopfilter_level); + encoder->Control(VP8E_GET_LAST_QUANTIZER, &qp); + rc_api_->ComputeQP(frame_params_); + ASSERT_EQ(rc_api_->GetQP(), qp); + ASSERT_EQ(rc_api_->GetLoopfilterLevel(), loopfilter_level); + } + + virtual void FramePktHook(const vpx_codec_cx_pkt_t *pkt) { + rc_api_->PostEncodeUpdate(pkt->data.frame.sz); + } + + void RunOneLayer() { + SetConfig(GET_PARAM(2)); + rc_api_ = libvpx::VP9RateControlRTC::Create(rc_cfg_); + frame_params_.spatial_layer_id = 0; + frame_params_.temporal_layer_id = 0; + + ::libvpx_test::I420VideoSource video("desktop_office1.1280_720-020.yuv", + 1280, 720, 30, 1, 0, kNumFrames); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + } + + void RunOneLayerVBRPeriodicKey() { + if (GET_PARAM(2) != VPX_VBR) return; + key_interval_ = 100; + SetConfig(VPX_VBR); + rc_api_ = libvpx::VP9RateControlRTC::Create(rc_cfg_); + frame_params_.spatial_layer_id = 0; + frame_params_.temporal_layer_id = 0; + + ::libvpx_test::I420VideoSource video("desktop_office1.1280_720-020.yuv", + 1280, 720, 30, 1, 0, kNumFrames); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + } + + private: + void SetConfig(vpx_rc_mode rc_mode) { + rc_cfg_.width = 1280; + rc_cfg_.height = 720; + rc_cfg_.max_quantizer = 52; + rc_cfg_.min_quantizer = 2; + rc_cfg_.target_bandwidth = 1000; + rc_cfg_.buf_initial_sz = 600; + rc_cfg_.buf_optimal_sz = 600; + rc_cfg_.buf_sz = 1000; + rc_cfg_.undershoot_pct = 50; + rc_cfg_.overshoot_pct = 50; + rc_cfg_.max_intra_bitrate_pct = 1000; + rc_cfg_.framerate = 30.0; + rc_cfg_.ss_number_layers = 1; + rc_cfg_.ts_number_layers = 1; + rc_cfg_.scaling_factor_num[0] = 1; + rc_cfg_.scaling_factor_den[0] = 1; + rc_cfg_.layer_target_bitrate[0] = 1000; + rc_cfg_.max_quantizers[0] = 52; + rc_cfg_.min_quantizers[0] = 2; + rc_cfg_.rc_mode = rc_mode; + rc_cfg_.aq_mode = aq_mode_; + + // Encoder settings for ground truth. + cfg_.g_w = 1280; + cfg_.g_h = 720; + cfg_.rc_undershoot_pct = 50; + cfg_.rc_overshoot_pct = 50; + cfg_.rc_buf_initial_sz = 600; + cfg_.rc_buf_optimal_sz = 600; + cfg_.rc_buf_sz = 1000; + cfg_.rc_dropframe_thresh = 0; + cfg_.rc_min_quantizer = 2; + cfg_.rc_max_quantizer = 52; + cfg_.rc_end_usage = rc_mode; + cfg_.g_lag_in_frames = 0; + cfg_.g_error_resilient = 0; + cfg_.rc_target_bitrate = 1000; + cfg_.kf_min_dist = key_interval_; + cfg_.kf_max_dist = key_interval_; + } + + std::unique_ptr rc_api_; + libvpx::VP9RateControlRtcConfig rc_cfg_; + int aq_mode_; + int key_interval_; + libvpx::VP9FrameParamsQpRTC frame_params_; + bool encoder_exit_; +}; + +class RcInterfaceSvcTest : public ::libvpx_test::EncoderTest, + public ::libvpx_test::CodecTestWithParam { + public: + RcInterfaceSvcTest() : EncoderTest(GET_PARAM(0)), aq_mode_(GET_PARAM(1)) {} + virtual ~RcInterfaceSvcTest() {} + + protected: + virtual void SetUp() { + InitializeConfig(); + SetMode(::libvpx_test::kRealTime); + } + + virtual void PreEncodeFrameHook(libvpx_test::VideoSource *video, + ::libvpx_test::Encoder *encoder) { + if (video->frame() == 0) { + encoder->Control(VP8E_SET_CPUUSED, 7); + encoder->Control(VP9E_SET_AQ_MODE, aq_mode_); + encoder->Control(VP9E_SET_TUNE_CONTENT, 0); + encoder->Control(VP8E_SET_MAX_INTRA_BITRATE_PCT, 900); + encoder->Control(VP9E_SET_RTC_EXTERNAL_RATECTRL, 1); + encoder->Control(VP9E_SET_SVC, 1); + encoder->Control(VP9E_SET_SVC_PARAMETERS, &svc_params_); + } + + frame_params_.frame_type = video->frame() == 0 ? KEY_FRAME : INTER_FRAME; + if (rc_cfg_.rc_mode == VPX_CBR && frame_params_.frame_type == INTER_FRAME) { + // Disable golden frame update. + frame_flags_ |= VP8_EFLAG_NO_UPD_GF; + frame_flags_ |= VP8_EFLAG_NO_UPD_ARF; + } + encoder_exit_ = video->frame() == kNumFrames; + current_superframe_ = video->frame(); + } + + virtual void PostEncodeFrameHook(::libvpx_test::Encoder *encoder) { + ::libvpx_test::CxDataIterator iter = encoder->GetCxData(); + while (const vpx_codec_cx_pkt_t *pkt = iter.Next()) { + ParseSuperframeSizes(static_cast(pkt->data.frame.buf), + pkt->data.frame.sz); + for (int sl = 0; sl < rc_cfg_.ss_number_layers; sl++) { + frame_params_.spatial_layer_id = sl; + frame_params_.temporal_layer_id = kTemporalId[current_superframe_ % 4]; + rc_api_->ComputeQP(frame_params_); + frame_params_.frame_type = INTER_FRAME; + rc_api_->PostEncodeUpdate(sizes_[sl]); + } + } + if (!encoder_exit_) { + int loopfilter_level, qp; + encoder->Control(VP9E_GET_LOOPFILTER_LEVEL, &loopfilter_level); + encoder->Control(VP8E_GET_LAST_QUANTIZER, &qp); + ASSERT_EQ(rc_api_->GetQP(), qp); + ASSERT_EQ(rc_api_->GetLoopfilterLevel(), loopfilter_level); + } + } + // This method needs to be overridden because non-reference frames are + // expected to be mismatched frames as the encoder will avoid loopfilter on + // these frames. + virtual void MismatchHook(const vpx_image_t * /*img1*/, + const vpx_image_t * /*img2*/) {} + + void RunSvc() { + SetConfigSvc(); + rc_api_ = libvpx::VP9RateControlRTC::Create(rc_cfg_); + SetEncoderSvc(); + + ::libvpx_test::I420VideoSource video("desktop_office1.1280_720-020.yuv", + 1280, 720, 30, 1, 0, kNumFrames); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + } + + private: + vpx_codec_err_t ParseSuperframeSizes(const uint8_t *data, size_t data_sz) { + uint8_t marker = *(data + data_sz - 1); + if ((marker & 0xe0) == 0xc0) { + const uint32_t frames = (marker & 0x7) + 1; + const uint32_t mag = ((marker >> 3) & 0x3) + 1; + const size_t index_sz = 2 + mag * frames; + // This chunk is marked as having a superframe index but doesn't have + // enough data for it, thus it's an invalid superframe index. + if (data_sz < index_sz) return VPX_CODEC_CORRUPT_FRAME; + { + const uint8_t marker2 = *(data + data_sz - index_sz); + // This chunk is marked as having a superframe index but doesn't have + // the matching marker byte at the front of the index therefore it's an + // invalid chunk. + if (marker != marker2) return VPX_CODEC_CORRUPT_FRAME; + } + const uint8_t *x = &data[data_sz - index_sz + 1]; + for (uint32_t i = 0; i < frames; ++i) { + uint32_t this_sz = 0; + + for (uint32_t j = 0; j < mag; ++j) this_sz |= (*x++) << (j * 8); + sizes_[i] = this_sz; + } + } + return VPX_CODEC_OK; + } + + void SetEncoderSvc() { + cfg_.ss_number_layers = 3; + cfg_.ts_number_layers = 3; + cfg_.g_timebase.num = 1; + cfg_.g_timebase.den = 30; + svc_params_.scaling_factor_num[0] = 72; + svc_params_.scaling_factor_den[0] = 288; + svc_params_.scaling_factor_num[1] = 144; + svc_params_.scaling_factor_den[1] = 288; + svc_params_.scaling_factor_num[2] = 288; + svc_params_.scaling_factor_den[2] = 288; + for (int i = 0; i < VPX_MAX_LAYERS; ++i) { + svc_params_.max_quantizers[i] = 56; + svc_params_.min_quantizers[i] = 2; + svc_params_.speed_per_layer[i] = 7; + } + cfg_.rc_end_usage = VPX_CBR; + cfg_.g_lag_in_frames = 0; + cfg_.g_error_resilient = 0; + // 3 temporal layers + cfg_.ts_rate_decimator[0] = 4; + cfg_.ts_rate_decimator[1] = 2; + cfg_.ts_rate_decimator[2] = 1; + cfg_.temporal_layering_mode = 3; + + cfg_.rc_buf_initial_sz = 500; + cfg_.rc_buf_optimal_sz = 600; + cfg_.rc_buf_sz = 1000; + cfg_.rc_min_quantizer = 2; + cfg_.rc_max_quantizer = 56; + cfg_.g_threads = 1; + cfg_.kf_max_dist = 9999; + cfg_.rc_target_bitrate = 1600; + cfg_.rc_overshoot_pct = 50; + cfg_.rc_undershoot_pct = 50; + + cfg_.layer_target_bitrate[0] = 100; + cfg_.layer_target_bitrate[1] = 140; + cfg_.layer_target_bitrate[2] = 200; + cfg_.layer_target_bitrate[3] = 250; + cfg_.layer_target_bitrate[4] = 350; + cfg_.layer_target_bitrate[5] = 500; + cfg_.layer_target_bitrate[6] = 450; + cfg_.layer_target_bitrate[7] = 630; + cfg_.layer_target_bitrate[8] = 900; + } + + void SetConfigSvc() { + rc_cfg_.width = 1280; + rc_cfg_.height = 720; + rc_cfg_.max_quantizer = 56; + rc_cfg_.min_quantizer = 2; + rc_cfg_.target_bandwidth = 1600; + rc_cfg_.buf_initial_sz = 500; + rc_cfg_.buf_optimal_sz = 600; + rc_cfg_.buf_sz = 1000; + rc_cfg_.undershoot_pct = 50; + rc_cfg_.overshoot_pct = 50; + rc_cfg_.max_intra_bitrate_pct = 900; + rc_cfg_.framerate = 30.0; + rc_cfg_.ss_number_layers = 3; + rc_cfg_.ts_number_layers = 3; + rc_cfg_.rc_mode = VPX_CBR; + rc_cfg_.aq_mode = aq_mode_; + + rc_cfg_.scaling_factor_num[0] = 1; + rc_cfg_.scaling_factor_den[0] = 4; + rc_cfg_.scaling_factor_num[1] = 2; + rc_cfg_.scaling_factor_den[1] = 4; + rc_cfg_.scaling_factor_num[2] = 4; + rc_cfg_.scaling_factor_den[2] = 4; + + rc_cfg_.ts_rate_decimator[0] = 4; + rc_cfg_.ts_rate_decimator[1] = 2; + rc_cfg_.ts_rate_decimator[2] = 1; + + rc_cfg_.layer_target_bitrate[0] = 100; + rc_cfg_.layer_target_bitrate[1] = 140; + rc_cfg_.layer_target_bitrate[2] = 200; + rc_cfg_.layer_target_bitrate[3] = 250; + rc_cfg_.layer_target_bitrate[4] = 350; + rc_cfg_.layer_target_bitrate[5] = 500; + rc_cfg_.layer_target_bitrate[6] = 450; + rc_cfg_.layer_target_bitrate[7] = 630; + rc_cfg_.layer_target_bitrate[8] = 900; + + for (int sl = 0; sl < rc_cfg_.ss_number_layers; ++sl) { + for (int tl = 0; tl < rc_cfg_.ts_number_layers; ++tl) { + const int i = sl * rc_cfg_.ts_number_layers + tl; + rc_cfg_.max_quantizers[i] = 56; + rc_cfg_.min_quantizers[i] = 2; + } + } + } + + int aq_mode_; + std::unique_ptr rc_api_; + libvpx::VP9RateControlRtcConfig rc_cfg_; + vpx_svc_extra_cfg_t svc_params_; + libvpx::VP9FrameParamsQpRTC frame_params_; + bool encoder_exit_; + int current_superframe_; + uint32_t sizes_[8]; +}; + +TEST_P(RcInterfaceTest, OneLayer) { RunOneLayer(); } + +TEST_P(RcInterfaceTest, OneLayerVBRPeriodicKey) { RunOneLayerVBRPeriodicKey(); } + +TEST_P(RcInterfaceSvcTest, Svc) { RunSvc(); } + +VP9_INSTANTIATE_TEST_SUITE(RcInterfaceTest, ::testing::Values(0, 3), + ::testing::Values(VPX_CBR, VPX_VBR)); +VP9_INSTANTIATE_TEST_SUITE(RcInterfaceSvcTest, ::testing::Values(0, 3)); +} // namespace diff --git a/media/libvpx/libvpx/test/vp9_roi_test.cc b/media/libvpx/libvpx/test/vp9_roi_test.cc new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/test/vp9_roi_test.cc @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2014 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#include + +#include "third_party/googletest/src/include/gtest/gtest.h" + +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/i420_video_source.h" +#include "test/util.h" +#include "test/video_source.h" +#include "test/y4m_video_source.h" +#include "test/yuv_video_source.h" +#include "vpx/vp8cx.h" +#include "vpx/vpx_encoder.h" + +#define MASK_WIDTH 40 +#define MASK_HEIGHT 30 +#define MASK_SIZE MASK_WIDTH *MASK_HEIGHT + +namespace { + +const int mask[MASK_SIZE] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0 +}; + +class RoiMaskBackgroundSkip : public ::libvpx_test::EncoderTest, + public ::testing::Test { + protected: + RoiMaskBackgroundSkip() : EncoderTest(&::libvpx_test::kVP9) {} + virtual ~RoiMaskBackgroundSkip() { free(roi_.roi_map); } + + virtual void SetUp() { + InitializeConfig(); + SetMode(::libvpx_test::kRealTime); + SetRoi(); + } + + void SetRoi() { + const int block_size = 8; + unsigned int i, j; + roi_.rows = (cfg_.g_h + block_size - 1) / block_size; + roi_.cols = (cfg_.g_w + block_size - 1) / block_size; + memset(&roi_.skip, 0, sizeof(roi_.skip)); + memset(&roi_.delta_q, 0, sizeof(roi_.delta_q)); + memset(&roi_.delta_lf, 0, sizeof(roi_.delta_lf)); + memset(roi_.ref_frame, -1, sizeof(roi_.ref_frame)); + roi_.ref_frame[1] = 1; + // Use segment 3 for skip. + roi_.skip[3] = 1; + roi_.roi_map = + (uint8_t *)calloc(roi_.rows * roi_.cols, sizeof(*roi_.roi_map)); + for (i = 0; i < roi_.rows; ++i) { + for (j = 0; j < roi_.cols; ++j) { + const int idx = i * roi_.cols + j; + if (mask[idx] == 1) roi_.roi_map[idx] = 3; + } + } + } + + virtual void PreEncodeFrameHook(::libvpx_test::VideoSource *video, + ::libvpx_test::Encoder *encoder) { + if (video->frame() == 0) { + encoder->Control(VP8E_SET_CPUUSED, 7); + encoder->Control(VP9E_SET_AQ_MODE, 3); + } + encoder->Control(VP9E_SET_ROI_MAP, &roi_); + } + + private: + vpx_roi_map_t roi_; +}; + +TEST_F(RoiMaskBackgroundSkip, RoiMaskNoMismatch) { + cfg_.rc_buf_initial_sz = 500; + cfg_.rc_buf_optimal_sz = 500; + cfg_.rc_buf_sz = 1000; + cfg_.rc_undershoot_pct = 20; + cfg_.rc_undershoot_pct = 20; + cfg_.rc_dropframe_thresh = 10; + cfg_.rc_min_quantizer = 0; + cfg_.rc_max_quantizer = 50; + cfg_.rc_end_usage = VPX_CBR; + cfg_.rc_target_bitrate = 200; + cfg_.g_lag_in_frames = 0; + cfg_.kf_max_dist = 9999; + + ::libvpx_test::I420VideoSource video("desktopqvga.320_240.yuv", 320, 240, 30, + 1, 0, 150); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); +} +} // namespace diff --git a/media/libvpx/libvpx/test/vp9_thread_test.cc b/media/libvpx/libvpx/test/vp9_thread_test.cc --- a/media/libvpx/libvpx/test/vp9_thread_test.cc +++ b/media/libvpx/libvpx/test/vp9_thread_test.cc @@ -146,15 +146,10 @@ } // ----------------------------------------------------------------------------- // Multi-threaded decode tests #if CONFIG_WEBM_IO -struct FileList { - const char *name; - const char *expected_md5; -}; - // Decodes |filename| with |num_threads|. Returns the md5 of the decoded frames. string DecodeFile(const string &filename, int num_threads) { libvpx_test::WebMVideoSource video(filename); video.Init(); @@ -180,20 +175,10 @@ } } return string(md5.Get()); } -void DecodeFiles(const FileList files[]) { - for (const FileList *iter = files; iter->name != nullptr; ++iter) { - SCOPED_TRACE(iter->name); - for (int t = 1; t <= 8; ++t) { - EXPECT_EQ(iter->expected_md5, DecodeFile(iter->name, t)) - << "threads = " << t; - } - } -} - // Trivial serialized thread worker interface implementation. // Note any worker that requires synchronization between other workers will // hang. namespace impl { namespace { @@ -214,14 +199,10 @@ TEST(VPxWorkerThreadTest, TestSerialInterface) { static const VPxWorkerInterface serial_interface = { impl::Init, impl::Reset, impl::Sync, impl::Launch, impl::Execute, impl::End }; - // TODO(jzern): Avoid using a file that will use the row-based thread - // loopfilter, with the simple serialized implementation it will hang. This is - // due to its expectation that rows will be run in parallel as they wait on - // progress in the row above before proceeding. static const char expected_md5[] = "b35a1b707b28e82be025d960aba039bc"; static const char filename[] = "vp90-2-03-size-226x226.webm"; VPxWorkerInterface default_interface = *vpx_get_worker_interface(); EXPECT_NE(vpx_set_worker_interface(&serial_interface), 0); @@ -230,92 +211,85 @@ // Reset the interface. EXPECT_NE(vpx_set_worker_interface(&default_interface), 0); EXPECT_EQ(expected_md5, DecodeFile(filename, 2)); } -TEST(VP9DecodeMultiThreadedTest, NoTilesNonFrameParallel) { - // no tiles or frame parallel; this exercises loop filter threading. - EXPECT_EQ("b35a1b707b28e82be025d960aba039bc", - DecodeFile("vp90-2-03-size-226x226.webm", 2)); -} +struct FileParam { + const char *name; + const char *expected_md5; + friend std::ostream &operator<<(std::ostream &os, const FileParam ¶m) { + return os << "file name: " << param.name + << " digest: " << param.expected_md5; + } +}; -TEST(VP9DecodeMultiThreadedTest, FrameParallel) { - static const FileList files[] = { { "vp90-2-08-tile_1x2_frame_parallel.webm", - "68ede6abd66bae0a2edf2eb9232241b6" }, - { "vp90-2-08-tile_1x4_frame_parallel.webm", - "368ebc6ebf3a5e478d85b2c3149b2848" }, - { "vp90-2-08-tile_1x8_frame_parallel.webm", - "17e439da2388aff3a0f69cb22579c6c1" }, - { nullptr, nullptr } }; +class VP9DecodeMultiThreadedTest : public ::testing::TestWithParam { +}; - DecodeFiles(files); +TEST_P(VP9DecodeMultiThreadedTest, Decode) { + for (int t = 1; t <= 8; ++t) { + EXPECT_EQ(GetParam().expected_md5, DecodeFile(GetParam().name, t)) + << "threads = " << t; + } } -TEST(VP9DecodeMultiThreadedTest, FrameParallelResize) { - static const FileList files[] = { - { "vp90-2-14-resize-fp-tiles-1-16.webm", - "0cd5e632c326297e975f38949c31ea94" }, - { "vp90-2-14-resize-fp-tiles-1-2-4-8-16.webm", - "5c78a96a42e7f4a4f6b2edcdb791e44c" }, - { "vp90-2-14-resize-fp-tiles-1-2.webm", - "e030450ae85c3277be2a418769df98e2" }, - { "vp90-2-14-resize-fp-tiles-1-4.webm", - "312eed4e2b64eb7a4e7f18916606a430" }, - { "vp90-2-14-resize-fp-tiles-16-1.webm", - "1755c16d8af16a9cb3fe7338d90abe52" }, - { "vp90-2-14-resize-fp-tiles-16-2.webm", - "500300592d3fcb6f12fab25e48aaf4df" }, - { "vp90-2-14-resize-fp-tiles-16-4.webm", - "47c48379fa6331215d91c67648e1af6e" }, - { "vp90-2-14-resize-fp-tiles-16-8-4-2-1.webm", - "eecf17290739bc708506fa4827665989" }, - { "vp90-2-14-resize-fp-tiles-16-8.webm", - "29b6bb54e4c26b5ca85d5de5fed94e76" }, - { "vp90-2-14-resize-fp-tiles-1-8.webm", - "1b6f175e08cd82cf84bb800ac6d1caa3" }, - { "vp90-2-14-resize-fp-tiles-2-16.webm", - "ca3b03e4197995d8d5444ede7a6c0804" }, - { "vp90-2-14-resize-fp-tiles-2-1.webm", - "99aec065369d70bbb78ccdff65afed3f" }, - { "vp90-2-14-resize-fp-tiles-2-4.webm", - "22d0ebdb49b87d2920a85aea32e1afd5" }, - { "vp90-2-14-resize-fp-tiles-2-8.webm", - "c2115cf051c62e0f7db1d4a783831541" }, - { "vp90-2-14-resize-fp-tiles-4-16.webm", - "c690d7e1719b31367564cac0af0939cb" }, - { "vp90-2-14-resize-fp-tiles-4-1.webm", - "a926020b2cc3e15ad4cc271853a0ff26" }, - { "vp90-2-14-resize-fp-tiles-4-2.webm", - "42699063d9e581f1993d0cf890c2be78" }, - { "vp90-2-14-resize-fp-tiles-4-8.webm", - "7f76d96036382f45121e3d5aa6f8ec52" }, - { "vp90-2-14-resize-fp-tiles-8-16.webm", - "76a43fcdd7e658542913ea43216ec55d" }, - { "vp90-2-14-resize-fp-tiles-8-1.webm", - "8e3fbe89486ca60a59299dea9da91378" }, - { "vp90-2-14-resize-fp-tiles-8-2.webm", - "ae96f21f21b6370cc0125621b441fc52" }, - { "vp90-2-14-resize-fp-tiles-8-4.webm", - "3eb4f24f10640d42218f7fd7b9fd30d4" }, - { nullptr, nullptr } - }; +const FileParam kNoTilesNonFrameParallelFiles[] = { + { "vp90-2-03-size-226x226.webm", "b35a1b707b28e82be025d960aba039bc" } +}; + +const FileParam kFrameParallelFiles[] = { + { "vp90-2-08-tile_1x2_frame_parallel.webm", + "68ede6abd66bae0a2edf2eb9232241b6" }, + { "vp90-2-08-tile_1x4_frame_parallel.webm", + "368ebc6ebf3a5e478d85b2c3149b2848" }, + { "vp90-2-08-tile_1x8_frame_parallel.webm", + "17e439da2388aff3a0f69cb22579c6c1" }, +}; - DecodeFiles(files); -} +const FileParam kFrameParallelResizeFiles[] = { + { "vp90-2-14-resize-fp-tiles-1-16.webm", "0cd5e632c326297e975f38949c31ea94" }, + { "vp90-2-14-resize-fp-tiles-1-2-4-8-16.webm", + "5c78a96a42e7f4a4f6b2edcdb791e44c" }, + { "vp90-2-14-resize-fp-tiles-1-2.webm", "e030450ae85c3277be2a418769df98e2" }, + { "vp90-2-14-resize-fp-tiles-1-4.webm", "312eed4e2b64eb7a4e7f18916606a430" }, + { "vp90-2-14-resize-fp-tiles-16-1.webm", "1755c16d8af16a9cb3fe7338d90abe52" }, + { "vp90-2-14-resize-fp-tiles-16-2.webm", "500300592d3fcb6f12fab25e48aaf4df" }, + { "vp90-2-14-resize-fp-tiles-16-4.webm", "47c48379fa6331215d91c67648e1af6e" }, + { "vp90-2-14-resize-fp-tiles-16-8-4-2-1.webm", + "eecf17290739bc708506fa4827665989" }, + { "vp90-2-14-resize-fp-tiles-16-8.webm", "29b6bb54e4c26b5ca85d5de5fed94e76" }, + { "vp90-2-14-resize-fp-tiles-1-8.webm", "1b6f175e08cd82cf84bb800ac6d1caa3" }, + { "vp90-2-14-resize-fp-tiles-2-16.webm", "ca3b03e4197995d8d5444ede7a6c0804" }, + { "vp90-2-14-resize-fp-tiles-2-1.webm", "99aec065369d70bbb78ccdff65afed3f" }, + { "vp90-2-14-resize-fp-tiles-2-4.webm", "22d0ebdb49b87d2920a85aea32e1afd5" }, + { "vp90-2-14-resize-fp-tiles-2-8.webm", "c2115cf051c62e0f7db1d4a783831541" }, + { "vp90-2-14-resize-fp-tiles-4-16.webm", "c690d7e1719b31367564cac0af0939cb" }, + { "vp90-2-14-resize-fp-tiles-4-1.webm", "a926020b2cc3e15ad4cc271853a0ff26" }, + { "vp90-2-14-resize-fp-tiles-4-2.webm", "42699063d9e581f1993d0cf890c2be78" }, + { "vp90-2-14-resize-fp-tiles-4-8.webm", "7f76d96036382f45121e3d5aa6f8ec52" }, + { "vp90-2-14-resize-fp-tiles-8-16.webm", "76a43fcdd7e658542913ea43216ec55d" }, + { "vp90-2-14-resize-fp-tiles-8-1.webm", "8e3fbe89486ca60a59299dea9da91378" }, + { "vp90-2-14-resize-fp-tiles-8-2.webm", "ae96f21f21b6370cc0125621b441fc52" }, + { "vp90-2-14-resize-fp-tiles-8-4.webm", "3eb4f24f10640d42218f7fd7b9fd30d4" }, +}; -TEST(VP9DecodeMultiThreadedTest, NonFrameParallel) { - static const FileList files[] = { - { "vp90-2-08-tile_1x2.webm", "570b4a5d5a70d58b5359671668328a16" }, - { "vp90-2-08-tile_1x4.webm", "988d86049e884c66909d2d163a09841a" }, - { "vp90-2-08-tile_1x8.webm", "0941902a52e9092cb010905eab16364c" }, - { "vp90-2-08-tile-4x1.webm", "06505aade6647c583c8e00a2f582266f" }, - { "vp90-2-08-tile-4x4.webm", "85c2299892460d76e2c600502d52bfe2" }, - { nullptr, nullptr } - }; +const FileParam kNonFrameParallelFiles[] = { + { "vp90-2-08-tile_1x2.webm", "570b4a5d5a70d58b5359671668328a16" }, + { "vp90-2-08-tile_1x4.webm", "988d86049e884c66909d2d163a09841a" }, + { "vp90-2-08-tile_1x8.webm", "0941902a52e9092cb010905eab16364c" }, + { "vp90-2-08-tile-4x1.webm", "06505aade6647c583c8e00a2f582266f" }, + { "vp90-2-08-tile-4x4.webm", "85c2299892460d76e2c600502d52bfe2" }, +}; - DecodeFiles(files); -} +INSTANTIATE_TEST_SUITE_P(NoTilesNonFrameParallel, VP9DecodeMultiThreadedTest, + ::testing::ValuesIn(kNoTilesNonFrameParallelFiles)); +INSTANTIATE_TEST_SUITE_P(FrameParallel, VP9DecodeMultiThreadedTest, + ::testing::ValuesIn(kFrameParallelFiles)); +INSTANTIATE_TEST_SUITE_P(FrameParallelResize, VP9DecodeMultiThreadedTest, + ::testing::ValuesIn(kFrameParallelResizeFiles)); +INSTANTIATE_TEST_SUITE_P(NonFrameParallel, VP9DecodeMultiThreadedTest, + ::testing::ValuesIn(kNonFrameParallelFiles)); #endif // CONFIG_WEBM_IO INSTANTIATE_TEST_SUITE_P(Synchronous, VPxWorkerThreadTest, ::testing::Bool()); } // namespace diff --git a/media/libvpx/libvpx/test/yuv_temporal_filter_test.cc b/media/libvpx/libvpx/test/yuv_temporal_filter_test.cc --- a/media/libvpx/libvpx/test/yuv_temporal_filter_test.cc +++ b/media/libvpx/libvpx/test/yuv_temporal_filter_test.cc @@ -672,21 +672,21 @@ block_width, block_height, ss_x, ss_y, strength, blk_fw, use_32x32, \ y_accumulator, y_count, u_accumulator, u_count, v_accumulator, \ v_count); \ } -WRAP_HIGHBD_FUNC(vp9_highbd_apply_temporal_filter_c, 10); -WRAP_HIGHBD_FUNC(vp9_highbd_apply_temporal_filter_c, 12); +WRAP_HIGHBD_FUNC(vp9_highbd_apply_temporal_filter_c, 10) +WRAP_HIGHBD_FUNC(vp9_highbd_apply_temporal_filter_c, 12) INSTANTIATE_TEST_SUITE_P( C, YUVTemporalFilterTest, ::testing::Values( TemporalFilterWithBd(&wrap_vp9_highbd_apply_temporal_filter_c_10, 10), TemporalFilterWithBd(&wrap_vp9_highbd_apply_temporal_filter_c_12, 12))); #if HAVE_SSE4_1 -WRAP_HIGHBD_FUNC(vp9_highbd_apply_temporal_filter_sse4_1, 10); -WRAP_HIGHBD_FUNC(vp9_highbd_apply_temporal_filter_sse4_1, 12); +WRAP_HIGHBD_FUNC(vp9_highbd_apply_temporal_filter_sse4_1, 10) +WRAP_HIGHBD_FUNC(vp9_highbd_apply_temporal_filter_sse4_1, 12) INSTANTIATE_TEST_SUITE_P( SSE4_1, YUVTemporalFilterTest, ::testing::Values( TemporalFilterWithBd(&wrap_vp9_highbd_apply_temporal_filter_sse4_1_10, diff --git a/media/libvpx/libvpx/third_party/googletest/README.libvpx b/media/libvpx/libvpx/third_party/googletest/README.libvpx --- a/media/libvpx/libvpx/third_party/googletest/README.libvpx +++ b/media/libvpx/libvpx/third_party/googletest/README.libvpx @@ -1,7 +1,7 @@ URL: https://github.com/google/googletest.git -Version: release-1.10.0-224-g23b2a3b1 +Version: release-1.11.0 License: BSD License File: LICENSE Description: Google's framework for writing C++ tests on a variety of platforms @@ -11,13 +11,11 @@ failures, various options for running the tests, and XML test report generation. Local Modifications: - Remove everything but: + CONTRIBUTORS googletest/ - CONTRIBUTORS include - LICENSE README.md src -- Enable kErrorOnUninstantiatedParameterizedTest and - kErrorOnUninstantiatedTypeParameterizedTest in gtest.cc + LICENSE diff --git a/media/libvpx/libvpx/third_party/googletest/src/CONTRIBUTORS b/media/libvpx/libvpx/third_party/googletest/src/CONTRIBUTORS --- a/media/libvpx/libvpx/third_party/googletest/src/CONTRIBUTORS +++ b/media/libvpx/libvpx/third_party/googletest/src/CONTRIBUTORS @@ -3,36 +3,61 @@ # who commit code to the project are encouraged to add their names # here. Please keep the list sorted by first names. Ajay Joshi Balázs Dán +Benoit Sigoure Bharat Mediratta +Bogdan Piloca Chandler Carruth Chris Prince Chris Taylor Dan Egnor +Dave MacLachlan +David Anderson +Dean Sturtevant Eric Roman +Gene Volovich Hady Zalek +Hal Burch Jeffrey Yasskin +Jim Keller +Joe Walnes +Jon Wray Jói Sigurðsson Keir Mierle Keith Ray Kenton Varda +Kostya Serebryany Krystian Kuzniarek +Lev Makhlis Manuel Klimek +Mario Tanev +Mark Paskin Markus Heule +Matthew Simmons Mika Raento +Mike Bland Miklós Fazekas +Neal Norwitz +Nermin Ozkiranartli +Owen Carlsen +Paneendra Ba Pasi Valminen Patrick Hanna Patrick Riley +Paul Menage Peter Kaminski +Piotr Kaminski Preston Jackson Rainer Klaffenboeck Russ Cox Russ Rufer Sean Mcafee Sigurður Ásgeirsson +Sverre Sundsdal +Takeshi Yoshino Tracy Bialik Vadim Berman Vlad Losev +Wolfgang Klier Zhanyong Wan diff --git a/media/libvpx/libvpx/third_party/googletest/src/README.md b/media/libvpx/libvpx/third_party/googletest/src/README.md --- a/media/libvpx/libvpx/third_party/googletest/src/README.md +++ b/media/libvpx/libvpx/third_party/googletest/src/README.md @@ -1,58 +1,76 @@ ### Generic Build Instructions #### Setup -To build Google Test and your tests that use it, you need to tell your build +To build GoogleTest and your tests that use it, you need to tell your build system where to find its headers and source files. The exact way to do it depends on which build system you use, and is usually straightforward. ### Build with CMake -Google Test comes with a CMake build script +GoogleTest comes with a CMake build script ([CMakeLists.txt](https://github.com/google/googletest/blob/master/CMakeLists.txt)) that can be used on a wide range of platforms ("C" stands for cross-platform.). If you don't have CMake installed already, you can download it for free from . CMake works by generating native makefiles or build projects that can be used in -the compiler environment of your choice. You can either build Google Test as a +the compiler environment of your choice. You can either build GoogleTest as a standalone project or it can be incorporated into an existing CMake build for another project. #### Standalone CMake Project -When building Google Test as a standalone project, the typical workflow starts -with: - - mkdir mybuild # Create a directory to hold the build output. - cd mybuild - cmake ${GTEST_DIR} # Generate native build scripts. - -If you want to build Google Test's samples, you should replace the last command +When building GoogleTest as a standalone project, the typical workflow starts with - cmake -Dgtest_build_samples=ON ${GTEST_DIR} +``` +git clone https://github.com/google/googletest.git -b release-1.10.0 +cd googletest # Main directory of the cloned repository. +mkdir build # Create a directory to hold the build output. +cd build +cmake .. # Generate native build scripts for GoogleTest. +``` + +The above command also includes GoogleMock by default. And so, if you want to +build only GoogleTest, you should replace the last command with + +``` +cmake .. -DBUILD_GMOCK=OFF +``` If you are on a \*nix system, you should now see a Makefile in the current -directory. Just type 'make' to build gtest. +directory. Just type `make` to build GoogleTest. And then you can simply install +GoogleTest if you are a system administrator. + +``` +make +sudo make install # Install in /usr/local/ by default +``` If you use Windows and have Visual Studio installed, a `gtest.sln` file and several `.vcproj` files will be created. You can then build them using Visual Studio. On Mac OS X with Xcode installed, a `.xcodeproj` file will be generated. #### Incorporating Into An Existing CMake Project -If you want to use gtest in a project which already uses CMake, then a more -robust and flexible approach is to build gtest as part of that project directly. -This is done by making the GoogleTest source code available to the main build -and adding it using CMake's `add_subdirectory()` command. This has the -significant advantage that the same compiler and linker settings are used -between gtest and the rest of your project, so issues associated with using -incompatible libraries (eg debug/release), etc. are avoided. This is +If you want to use GoogleTest in a project which already uses CMake, the easiest +way is to get installed libraries and headers. + +* Import GoogleTest by using `find_package` (or `pkg_check_modules`). For + example, if `find_package(GTest CONFIG REQUIRED)` succeeds, you can use the + libraries as `GTest::gtest`, `GTest::gmock`. + +And a more robust and flexible approach is to build GoogleTest as part of that +project directly. This is done by making the GoogleTest source code available to +the main build and adding it using CMake's `add_subdirectory()` command. This +has the significant advantage that the same compiler and linker settings are +used between GoogleTest and the rest of your project, so issues associated with +using incompatible libraries (eg debug/release), etc. are avoided. This is particularly useful on Windows. Making GoogleTest's source code available to the main build can be done a few different ways: * Download the GoogleTest source code manually and place it at a known location. This is the least flexible approach and can make it more difficult @@ -62,143 +80,96 @@ keep up to date. Some organizations may not permit this method. * Add GoogleTest as a git submodule or equivalent. This may not always be possible or appropriate. Git submodules, for example, have their own set of advantages and drawbacks. * Use CMake to download GoogleTest as part of the build's configure step. This - is just a little more complex, but doesn't have the limitations of the other - methods. - -The last of the above methods is implemented with a small piece of CMake code in -a separate file (e.g. `CMakeLists.txt.in`) which is copied to the build area and -then invoked as a sub-build _during the CMake stage_. That directory is then -pulled into the main build with `add_subdirectory()`. For example: - -New file `CMakeLists.txt.in`: - -```cmake -cmake_minimum_required(VERSION 2.8.2) - -project(googletest-download NONE) + approach doesn't have the limitations of the other methods. -include(ExternalProject) -ExternalProject_Add(googletest - GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG master - SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/googletest-src" - BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/googletest-build" - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" - TEST_COMMAND "" -) -``` +The last of the above methods is implemented with a small piece of CMake code +that downloads and pulls the GoogleTest code into the main build. -Existing build's `CMakeLists.txt`: +Just add to your `CMakeLists.txt`: ```cmake -# Download and unpack googletest at configure time -configure_file(CMakeLists.txt.in googletest-download/CMakeLists.txt) -execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" . - RESULT_VARIABLE result - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/googletest-download ) -if(result) - message(FATAL_ERROR "CMake step for googletest failed: ${result}") -endif() -execute_process(COMMAND ${CMAKE_COMMAND} --build . - RESULT_VARIABLE result - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/googletest-download ) -if(result) - message(FATAL_ERROR "Build step for googletest failed: ${result}") -endif() - -# Prevent overriding the parent project's compiler/linker -# settings on Windows +include(FetchContent) +FetchContent_Declare( + googletest + # Specify the commit you depend on and update it regularly. + URL https://github.com/google/googletest/archive/609281088cfefc76f9d0ce82e1ff6c30cc3591e5.zip +) +# For Windows: Prevent overriding the parent project's compiler/linker settings set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) - -# Add googletest directly to our build. This defines -# the gtest and gtest_main targets. -add_subdirectory(${CMAKE_CURRENT_BINARY_DIR}/googletest-src - ${CMAKE_CURRENT_BINARY_DIR}/googletest-build - EXCLUDE_FROM_ALL) - -# The gtest/gtest_main targets carry header search path -# dependencies automatically when using CMake 2.8.11 or -# later. Otherwise we have to add them here ourselves. -if (CMAKE_VERSION VERSION_LESS 2.8.11) - include_directories("${gtest_SOURCE_DIR}/include") -endif() +FetchContent_MakeAvailable(googletest) # Now simply link against gtest or gtest_main as needed. Eg add_executable(example example.cpp) target_link_libraries(example gtest_main) add_test(NAME example_test COMMAND example) ``` -Note that this approach requires CMake 2.8.2 or later due to its use of the -`ExternalProject_Add()` command. The above technique is discussed in more detail -in [this separate article](http://crascit.com/2015/07/25/cmake-gtest/) which -also contains a link to a fully generalized implementation of the technique. +Note that this approach requires CMake 3.14 or later due to its use of the +`FetchContent_MakeAvailable()` command. ##### Visual Studio Dynamic vs Static Runtimes By default, new Visual Studio projects link the C runtimes dynamically but -Google Test links them statically. This will generate an error that looks +GoogleTest links them statically. This will generate an error that looks something like the following: gtest.lib(gtest-all.obj) : error LNK2038: mismatch detected for 'RuntimeLibrary': value 'MTd_StaticDebug' doesn't match value 'MDd_DynamicDebug' in main.obj -Google Test already has a CMake option for this: `gtest_force_shared_crt` +GoogleTest already has a CMake option for this: `gtest_force_shared_crt` Enabling this option will make gtest link the runtimes dynamically too, and match the project in which it is included. #### C++ Standard Version An environment that supports C++11 is required in order to successfully build -Google Test. One way to ensure this is to specify the standard in the top-level +GoogleTest. One way to ensure this is to specify the standard in the top-level project, for example by using the `set(CMAKE_CXX_STANDARD 11)` command. If this -is not feasible, for example in a C project using Google Test for validation, +is not feasible, for example in a C project using GoogleTest for validation, then it can be specified by adding it to the options for cmake via the `DCMAKE_CXX_FLAGS` option. -### Tweaking Google Test +### Tweaking GoogleTest -Google Test can be used in diverse environments. The default configuration may +GoogleTest can be used in diverse environments. The default configuration may not work (or may not work well) out of the box in some environments. However, -you can easily tweak Google Test by defining control macros on the compiler +you can easily tweak GoogleTest by defining control macros on the compiler command line. Generally, these macros are named like `GTEST_XYZ` and you define them to either 1 or 0 to enable or disable a certain feature. We list the most frequently used macros below. For a complete list, see file [include/gtest/internal/gtest-port.h](https://github.com/google/googletest/blob/master/googletest/include/gtest/internal/gtest-port.h). ### Multi-threaded Tests -Google Test is thread-safe where the pthread library is available. After +GoogleTest is thread-safe where the pthread library is available. After `#include "gtest/gtest.h"`, you can check the `GTEST_IS_THREADSAFE` macro to see whether this is the case (yes if the macro is `#defined` to 1, no if it's undefined.). -If Google Test doesn't correctly detect whether pthread is available in your +If GoogleTest doesn't correctly detect whether pthread is available in your environment, you can force it with -DGTEST_HAS_PTHREAD=1 or -DGTEST_HAS_PTHREAD=0 -When Google Test uses pthread, you may need to add flags to your compiler and/or +When GoogleTest uses pthread, you may need to add flags to your compiler and/or linker to select the pthread library, or you'll get link errors. If you use the -CMake script or the deprecated Autotools script, this is taken care of for you. -If you use your own build script, you'll need to read your compiler and linker's -manual to figure out what flags to add. +CMake script, this is taken care of for you. If you use your own build script, +you'll need to read your compiler and linker's manual to figure out what flags +to add. ### As a Shared Library (DLL) -Google Test is compact, so most users can build and link it as a static library -for the simplicity. You can choose to use Google Test as a shared library (known +GoogleTest is compact, so most users can build and link it as a static library +for the simplicity. You can choose to use GoogleTest as a shared library (known as a DLL on Windows) if you prefer. To compile *gtest* as a shared library, add -DGTEST_CREATE_SHARED_LIBRARY=1 @@ -214,26 +185,26 @@ Note: while the above steps aren't technically necessary today when using some compilers (e.g. GCC), they may become necessary in the future, if we decide to improve the speed of loading the library (see for details). Therefore you are recommended -to always add the above flags when using Google Test as a shared library. -Otherwise a future release of Google Test may break your build script. +to always add the above flags when using GoogleTest as a shared library. +Otherwise a future release of GoogleTest may break your build script. ### Avoiding Macro Name Clashes In C++, macros don't obey namespaces. Therefore two libraries that both define a macro of the same name will clash if you `#include` both definitions. In case a -Google Test macro clashes with another library, you can force Google Test to +GoogleTest macro clashes with another library, you can force GoogleTest to rename its macro to avoid the conflict. -Specifically, if both Google Test and some other code define macro FOO, you can +Specifically, if both GoogleTest and some other code define macro FOO, you can add -DGTEST_DONT_DEFINE_FOO=1 -to the compiler flags to tell Google Test to change the macro's name from `FOO` +to the compiler flags to tell GoogleTest to change the macro's name from `FOO` to `GTEST_FOO`. Currently `FOO` can be `FAIL`, `SUCCEED`, or `TEST`. For example, with `-DGTEST_DONT_DEFINE_TEST=1`, you'll need to write GTEST_TEST(SomeTest, DoesThis) { ... } diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-death-test.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-death-test.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-death-test.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-death-test.h @@ -33,12 +33,12 @@ // This header file defines the public API for death tests. It is // #included by gtest.h so a user doesn't need to include this // directly. // GOOGLETEST_CM0001 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_GTEST_DEATH_TEST_H_ -#define GTEST_INCLUDE_GTEST_GTEST_DEATH_TEST_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_GTEST_DEATH_TEST_H_ +#define GOOGLETEST_INCLUDE_GTEST_GTEST_DEATH_TEST_H_ #include "gtest/internal/gtest-death-test-internal.h" namespace testing { @@ -95,10 +95,14 @@ // return WIFSIGNALED(exit_code) && WTERMSIG(exit_code) == SIGHUP; // } // // ASSERT_EXIT(client.HangUpServer(), KilledBySIGHUP, "Hanging up!"); // +// The final parameter to each of these macros is a matcher applied to any data +// the sub-process wrote to stderr. For compatibility with existing tests, a +// bare string is interpreted as a regular expression matcher. +// // On the regular expressions used in death tests: // // GOOGLETEST_CM0005 DO NOT DELETE // On POSIX-compliant systems (*nix), we use the library, // which uses the POSIX extended regex syntax. @@ -160,43 +164,42 @@ // /absolute/path/to/bar_test are fine, but foo_test is not). This // is rarely a problem as people usually don't put the test binary // directory in PATH. // -// Asserts that a given statement causes the program to exit, with an -// integer exit status that satisfies predicate, and emitting error output -// that matches regex. -# define ASSERT_EXIT(statement, predicate, regex) \ - GTEST_DEATH_TEST_(statement, predicate, regex, GTEST_FATAL_FAILURE_) +// Asserts that a given `statement` causes the program to exit, with an +// integer exit status that satisfies `predicate`, and emitting error output +// that matches `matcher`. +# define ASSERT_EXIT(statement, predicate, matcher) \ + GTEST_DEATH_TEST_(statement, predicate, matcher, GTEST_FATAL_FAILURE_) -// Like ASSERT_EXIT, but continues on to successive tests in the +// Like `ASSERT_EXIT`, but continues on to successive tests in the // test suite, if any: -# define EXPECT_EXIT(statement, predicate, regex) \ - GTEST_DEATH_TEST_(statement, predicate, regex, GTEST_NONFATAL_FAILURE_) +# define EXPECT_EXIT(statement, predicate, matcher) \ + GTEST_DEATH_TEST_(statement, predicate, matcher, GTEST_NONFATAL_FAILURE_) -// Asserts that a given statement causes the program to exit, either by +// Asserts that a given `statement` causes the program to exit, either by // explicitly exiting with a nonzero exit code or being killed by a -// signal, and emitting error output that matches regex. -# define ASSERT_DEATH(statement, regex) \ - ASSERT_EXIT(statement, ::testing::internal::ExitedUnsuccessfully, regex) +// signal, and emitting error output that matches `matcher`. +# define ASSERT_DEATH(statement, matcher) \ + ASSERT_EXIT(statement, ::testing::internal::ExitedUnsuccessfully, matcher) -// Like ASSERT_DEATH, but continues on to successive tests in the +// Like `ASSERT_DEATH`, but continues on to successive tests in the // test suite, if any: -# define EXPECT_DEATH(statement, regex) \ - EXPECT_EXIT(statement, ::testing::internal::ExitedUnsuccessfully, regex) +# define EXPECT_DEATH(statement, matcher) \ + EXPECT_EXIT(statement, ::testing::internal::ExitedUnsuccessfully, matcher) // Two predicate classes that can be used in {ASSERT,EXPECT}_EXIT*: // Tests that an exit code describes a normal exit with a given exit code. class GTEST_API_ ExitedWithCode { public: explicit ExitedWithCode(int exit_code); + ExitedWithCode(const ExitedWithCode&) = default; + void operator=(const ExitedWithCode& other) = delete; bool operator()(int exit_status) const; private: - // No implementation - assignment is unsupported. - void operator=(const ExitedWithCode& other); - const int exit_code_; }; # if !GTEST_OS_WINDOWS && !GTEST_OS_FUCHSIA // Tests that an exit code describes an exit due to termination by a @@ -338,6 +341,6 @@ GTEST_UNSUPPORTED_DEATH_TEST(statement, regex, return) #endif } // namespace testing -#endif // GTEST_INCLUDE_GTEST_GTEST_DEATH_TEST_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_GTEST_DEATH_TEST_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-matchers.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-matchers.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-matchers.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-matchers.h @@ -30,17 +30,14 @@ // The Google C++ Testing and Mocking Framework (Google Test) // // This file implements just enough of the matcher interface to allow // EXPECT_DEATH and friends to accept a matcher argument. -// IWYU pragma: private, include "testing/base/public/gunit.h" -// IWYU pragma: friend third_party/googletest/googlemock/.* -// IWYU pragma: friend third_party/googletest/googletest/.* +#ifndef GOOGLETEST_INCLUDE_GTEST_GTEST_MATCHERS_H_ +#define GOOGLETEST_INCLUDE_GTEST_GTEST_MATCHERS_H_ -#ifndef GTEST_INCLUDE_GTEST_GTEST_MATCHERS_H_ -#define GTEST_INCLUDE_GTEST_GTEST_MATCHERS_H_ - +#include #include #include #include #include @@ -61,24 +58,20 @@ /* Symbol involving type with internal linkage not defined */) namespace testing { // To implement a matcher Foo for type T, define: -// 1. a class FooMatcherImpl that implements the -// MatcherInterface interface, and -// 2. a factory function that creates a Matcher object from a -// FooMatcherImpl*. +// 1. a class FooMatcherMatcher that implements the matcher interface: +// using is_gtest_matcher = void; +// bool MatchAndExplain(const T&, std::ostream*); +// (MatchResultListener* can also be used instead of std::ostream*) +// void DescribeTo(std::ostream*); +// void DescribeNegationTo(std::ostream*); // -// The two-level delegation design makes it possible to allow a user -// to write "v" instead of "Eq(v)" where a Matcher is expected, which -// is impossible if we pass matchers by pointers. It also eases -// ownership management as Matcher objects can now be copied like -// plain values. +// 2. a factory function that creates a Matcher object from a +// FooMatcherMatcher. -// MatchResultListener is an abstract class. Its << operator can be -// used by a matcher to explain why a value matches or doesn't match. -// class MatchResultListener { public: // Creates a listener object with the given underlying ostream. The // listener does not own the ostream, and does not dereference it // in the constructor or destructor. @@ -111,11 +104,11 @@ inline MatchResultListener::~MatchResultListener() { } // An instance of a subclass of this knows how to describe itself as a // matcher. -class MatcherDescriberInterface { +class GTEST_API_ MatcherDescriberInterface { public: virtual ~MatcherDescriberInterface() {} // Describes this matcher to an ostream. The function should print // a verb phrase that describes the property a value matching this @@ -179,35 +172,10 @@ // virtual void DescribeNegationTo(::std::ostream* os) const; }; namespace internal { -// Converts a MatcherInterface to a MatcherInterface. -template -class MatcherInterfaceAdapter : public MatcherInterface { - public: - explicit MatcherInterfaceAdapter(const MatcherInterface* impl) - : impl_(impl) {} - ~MatcherInterfaceAdapter() override { delete impl_; } - - void DescribeTo(::std::ostream* os) const override { impl_->DescribeTo(os); } - - void DescribeNegationTo(::std::ostream* os) const override { - impl_->DescribeNegationTo(os); - } - - bool MatchAndExplain(const T& x, - MatchResultListener* listener) const override { - return impl_->MatchAndExplain(x, listener); - } - - private: - const MatcherInterface* const impl_; - - GTEST_DISALLOW_COPY_AND_ASSIGN_(MatcherInterfaceAdapter); -}; - struct AnyEq { template bool operator()(const A& a, const B& b) const { return a == b; } }; struct AnyNe { @@ -250,34 +218,57 @@ private: GTEST_DISALLOW_COPY_AND_ASSIGN_(StreamMatchResultListener); }; +struct SharedPayloadBase { + std::atomic ref{1}; + void Ref() { ref.fetch_add(1, std::memory_order_relaxed); } + bool Unref() { return ref.fetch_sub(1, std::memory_order_acq_rel) == 1; } +}; + +template +struct SharedPayload : SharedPayloadBase { + explicit SharedPayload(const T& v) : value(v) {} + explicit SharedPayload(T&& v) : value(std::move(v)) {} + + static void Destroy(SharedPayloadBase* shared) { + delete static_cast(shared); + } + + T value; +}; + // An internal class for implementing Matcher, which will derive // from it. We put functionalities common to all Matcher // specializations here to avoid code duplication. template -class MatcherBase { +class MatcherBase : private MatcherDescriberInterface { public: // Returns true if and only if the matcher matches x; also explains the // match result to 'listener'. bool MatchAndExplain(const T& x, MatchResultListener* listener) const { - return impl_->MatchAndExplain(x, listener); + GTEST_CHECK_(vtable_ != nullptr); + return vtable_->match_and_explain(*this, x, listener); } // Returns true if and only if this matcher matches x. bool Matches(const T& x) const { DummyMatchResultListener dummy; return MatchAndExplain(x, &dummy); } // Describes this matcher to an ostream. - void DescribeTo(::std::ostream* os) const { impl_->DescribeTo(os); } + void DescribeTo(::std::ostream* os) const final { + GTEST_CHECK_(vtable_ != nullptr); + vtable_->describe(*this, os, false); + } // Describes the negation of this matcher to an ostream. - void DescribeNegationTo(::std::ostream* os) const { - impl_->DescribeNegationTo(os); + void DescribeNegationTo(::std::ostream* os) const final { + GTEST_CHECK_(vtable_ != nullptr); + vtable_->describe(*this, os, true); } // Explains why x matches, or doesn't match, the matcher. void ExplainMatchResultTo(const T& x, ::std::ostream* os) const { StreamMatchResultListener listener(os); @@ -286,35 +277,198 @@ // Returns the describer for this matcher object; retains ownership // of the describer, which is only guaranteed to be alive when // this matcher object is alive. const MatcherDescriberInterface* GetDescriber() const { - return impl_.get(); + if (vtable_ == nullptr) return nullptr; + return vtable_->get_describer(*this); } protected: - MatcherBase() {} + MatcherBase() : vtable_(nullptr) {} // Constructs a matcher from its implementation. - explicit MatcherBase(const MatcherInterface* impl) : impl_(impl) {} + template + explicit MatcherBase(const MatcherInterface* impl) { + Init(impl); + } + + template ::type::is_gtest_matcher> + MatcherBase(M&& m) { // NOLINT + Init(std::forward(m)); + } + + MatcherBase(const MatcherBase& other) + : vtable_(other.vtable_), buffer_(other.buffer_) { + if (IsShared()) buffer_.shared->Ref(); + } - template - explicit MatcherBase( - const MatcherInterface* impl, - typename std::enable_if::value>::type* = - nullptr) - : impl_(new internal::MatcherInterfaceAdapter(impl)) {} + MatcherBase& operator=(const MatcherBase& other) { + if (this == &other) return *this; + Destroy(); + vtable_ = other.vtable_; + buffer_ = other.buffer_; + if (IsShared()) buffer_.shared->Ref(); + return *this; + } - MatcherBase(const MatcherBase&) = default; - MatcherBase& operator=(const MatcherBase&) = default; - MatcherBase(MatcherBase&&) = default; - MatcherBase& operator=(MatcherBase&&) = default; + MatcherBase(MatcherBase&& other) + : vtable_(other.vtable_), buffer_(other.buffer_) { + other.vtable_ = nullptr; + } - virtual ~MatcherBase() {} + MatcherBase& operator=(MatcherBase&& other) { + if (this == &other) return *this; + Destroy(); + vtable_ = other.vtable_; + buffer_ = other.buffer_; + other.vtable_ = nullptr; + return *this; + } + + ~MatcherBase() override { Destroy(); } private: - std::shared_ptr> impl_; + struct VTable { + bool (*match_and_explain)(const MatcherBase&, const T&, + MatchResultListener*); + void (*describe)(const MatcherBase&, std::ostream*, bool negation); + // Returns the captured object if it implements the interface, otherwise + // returns the MatcherBase itself. + const MatcherDescriberInterface* (*get_describer)(const MatcherBase&); + // Called on shared instances when the reference count reaches 0. + void (*shared_destroy)(SharedPayloadBase*); + }; + + bool IsShared() const { + return vtable_ != nullptr && vtable_->shared_destroy != nullptr; + } + + // If the implementation uses a listener, call that. + template + static auto MatchAndExplainImpl(const MatcherBase& m, const T& value, + MatchResultListener* listener) + -> decltype(P::Get(m).MatchAndExplain(value, listener->stream())) { + return P::Get(m).MatchAndExplain(value, listener->stream()); + } + + template + static auto MatchAndExplainImpl(const MatcherBase& m, const T& value, + MatchResultListener* listener) + -> decltype(P::Get(m).MatchAndExplain(value, listener)) { + return P::Get(m).MatchAndExplain(value, listener); + } + + template + static void DescribeImpl(const MatcherBase& m, std::ostream* os, + bool negation) { + if (negation) { + P::Get(m).DescribeNegationTo(os); + } else { + P::Get(m).DescribeTo(os); + } + } + + template + static const MatcherDescriberInterface* GetDescriberImpl( + const MatcherBase& m) { + // If the impl is a MatcherDescriberInterface, then return it. + // Otherwise use MatcherBase itself. + // This allows us to implement the GetDescriber() function without support + // from the impl, but some users really want to get their impl back when + // they call GetDescriber(). + // We use std::get on a tuple as a workaround of not having `if constexpr`. + return std::get<( + std::is_convertible::value + ? 1 + : 0)>(std::make_tuple(&m, &P::Get(m))); + } + + template + const VTable* GetVTable() { + static constexpr VTable kVTable = {&MatchAndExplainImpl

, + &DescribeImpl

, &GetDescriberImpl

, + P::shared_destroy}; + return &kVTable; + } + + union Buffer { + // Add some types to give Buffer some common alignment/size use cases. + void* ptr; + double d; + int64_t i; + // And add one for the out-of-line cases. + SharedPayloadBase* shared; + }; + + void Destroy() { + if (IsShared() && buffer_.shared->Unref()) { + vtable_->shared_destroy(buffer_.shared); + } + } + + template + static constexpr bool IsInlined() { + return sizeof(M) <= sizeof(Buffer) && alignof(M) <= alignof(Buffer) && + std::is_trivially_copy_constructible::value && + std::is_trivially_destructible::value; + } + + template ()> + struct ValuePolicy { + static const M& Get(const MatcherBase& m) { + // When inlined along with Init, need to be explicit to avoid violating + // strict aliasing rules. + const M *ptr = static_cast( + static_cast(&m.buffer_)); + return *ptr; + } + static void Init(MatcherBase& m, M impl) { + ::new (static_cast(&m.buffer_)) M(impl); + } + static constexpr auto shared_destroy = nullptr; + }; + + template + struct ValuePolicy { + using Shared = SharedPayload; + static const M& Get(const MatcherBase& m) { + return static_cast(m.buffer_.shared)->value; + } + template + static void Init(MatcherBase& m, Arg&& arg) { + m.buffer_.shared = new Shared(std::forward(arg)); + } + static constexpr auto shared_destroy = &Shared::Destroy; + }; + + template + struct ValuePolicy*, B> { + using M = const MatcherInterface; + using Shared = SharedPayload>; + static const M& Get(const MatcherBase& m) { + return *static_cast(m.buffer_.shared)->value; + } + static void Init(MatcherBase& m, M* impl) { + m.buffer_.shared = new Shared(std::unique_ptr(impl)); + } + + static constexpr auto shared_destroy = &Shared::Destroy; + }; + + template + void Init(M&& m) { + using MM = typename std::decay::type; + using Policy = ValuePolicy; + vtable_ = GetVTable(); + Policy::Init(*this, std::forward(m)); + } + + const VTable* vtable_; + Buffer buffer_; }; } // namespace internal // A Matcher is a copyable and IMMUTABLE (except by assignment) @@ -338,10 +492,14 @@ const MatcherInterface* impl, typename std::enable_if::value>::type* = nullptr) : internal::MatcherBase(impl) {} + template ::type::is_gtest_matcher> + Matcher(M&& m) : internal::MatcherBase(std::forward(m)) {} // NOLINT + // Implicit constructor here allows people to write // EXPECT_CALL(foo, Bar(5)) instead of EXPECT_CALL(foo, Bar(Eq(5))) sometimes Matcher(T value); // NOLINT }; @@ -355,10 +513,15 @@ Matcher() {} explicit Matcher(const MatcherInterface* impl) : internal::MatcherBase(impl) {} + template ::type::is_gtest_matcher> + Matcher(M&& m) // NOLINT + : internal::MatcherBase(std::forward(m)) {} + // Allows the user to write str instead of Eq(str) sometimes, where // str is a std::string object. Matcher(const std::string& s); // NOLINT // Allows the user to write "foo" instead of Eq("foo") sometimes. @@ -374,10 +537,15 @@ explicit Matcher(const MatcherInterface* impl) : internal::MatcherBase(impl) {} explicit Matcher(const MatcherInterface* impl) : internal::MatcherBase(impl) {} + template ::type::is_gtest_matcher> + Matcher(M&& m) // NOLINT + : internal::MatcherBase(std::forward(m)) {} + // Allows the user to write str instead of Eq(str) sometimes, where // str is a string object. Matcher(const std::string& s); // NOLINT // Allows the user to write "foo" instead of Eq("foo") sometimes. @@ -395,10 +563,16 @@ Matcher() {} explicit Matcher(const MatcherInterface* impl) : internal::MatcherBase(impl) {} + template ::type::is_gtest_matcher> + Matcher(M&& m) // NOLINT + : internal::MatcherBase(std::forward(m)) { + } + // Allows the user to write str instead of Eq(str) sometimes, where // str is a std::string object. Matcher(const std::string& s); // NOLINT // Allows the user to write "foo" instead of Eq("foo") sometimes. @@ -417,10 +591,15 @@ explicit Matcher(const MatcherInterface* impl) : internal::MatcherBase(impl) {} explicit Matcher(const MatcherInterface* impl) : internal::MatcherBase(impl) {} + template ::type::is_gtest_matcher> + Matcher(M&& m) // NOLINT + : internal::MatcherBase(std::forward(m)) {} + // Allows the user to write str instead of Eq(str) sometimes, where // str is a std::string object. Matcher(const std::string& s); // NOLINT // Allows the user to write "foo" instead of Eq("foo") sometimes. @@ -527,41 +706,36 @@ // a "bare" type (i.e. neither 'const T' nor 'T&'). template class ComparisonBase { public: explicit ComparisonBase(const Rhs& rhs) : rhs_(rhs) {} + + using is_gtest_matcher = void; + template - operator Matcher() const { - return Matcher(new Impl(rhs_)); + bool MatchAndExplain(const Lhs& lhs, std::ostream*) const { + return Op()(lhs, Unwrap(rhs_)); + } + void DescribeTo(std::ostream* os) const { + *os << D::Desc() << " "; + UniversalPrint(Unwrap(rhs_), os); + } + void DescribeNegationTo(std::ostream* os) const { + *os << D::NegatedDesc() << " "; + UniversalPrint(Unwrap(rhs_), os); } private: template - static const T& Unwrap(const T& v) { return v; } + static const T& Unwrap(const T& v) { + return v; + } template - static const T& Unwrap(std::reference_wrapper v) { return v; } + static const T& Unwrap(std::reference_wrapper v) { + return v; + } - template - class Impl : public MatcherInterface { - public: - explicit Impl(const Rhs& rhs) : rhs_(rhs) {} - bool MatchAndExplain(Lhs lhs, - MatchResultListener* /* listener */) const override { - return Op()(lhs, Unwrap(rhs_)); - } - void DescribeTo(::std::ostream* os) const override { - *os << D::Desc() << " "; - UniversalPrint(Unwrap(rhs_), os); - } - void DescribeNegationTo(::std::ostream* os) const override { - *os << D::NegatedDesc() << " "; - UniversalPrint(Unwrap(rhs_), os); - } - - private: - Rhs rhs_; - }; Rhs rhs_; }; template class EqMatcher : public ComparisonBase, Rhs, AnyEq> { @@ -610,10 +784,14 @@ : ComparisonBase, Rhs, AnyGe>(rhs) { } static const char* Desc() { return "is >="; } static const char* NegatedDesc() { return "isn't >="; } }; +template ::value>::type> +using StringLike = T; + // Implements polymorphic matchers MatchesRegex(regex) and // ContainsRegex(regex), which can be used as a Matcher as long as // T can be converted to a string. class MatchesRegexMatcher { public: @@ -670,24 +848,26 @@ // The matcher takes ownership of 'regex'. inline PolymorphicMatcher MatchesRegex( const internal::RE* regex) { return MakePolymorphicMatcher(internal::MatchesRegexMatcher(regex, true)); } -inline PolymorphicMatcher MatchesRegex( - const std::string& regex) { - return MatchesRegex(new internal::RE(regex)); +template +PolymorphicMatcher MatchesRegex( + const internal::StringLike& regex) { + return MatchesRegex(new internal::RE(std::string(regex))); } // Matches a string that contains regular expression 'regex'. // The matcher takes ownership of 'regex'. inline PolymorphicMatcher ContainsRegex( const internal::RE* regex) { return MakePolymorphicMatcher(internal::MatchesRegexMatcher(regex, false)); } -inline PolymorphicMatcher ContainsRegex( - const std::string& regex) { - return ContainsRegex(new internal::RE(regex)); +template +PolymorphicMatcher ContainsRegex( + const internal::StringLike& regex) { + return ContainsRegex(new internal::RE(std::string(regex))); } // Creates a polymorphic matcher that matches anything equal to x. // Note: if the parameter of Eq() were declared as const T&, Eq("foo") // wouldn't compile. @@ -745,6 +925,6 @@ } } // namespace testing GTEST_DISABLE_MSC_WARNINGS_POP_() // 4251 5046 -#endif // GTEST_INCLUDE_GTEST_GTEST_MATCHERS_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_GTEST_MATCHERS_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-message.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-message.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-message.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-message.h @@ -42,12 +42,12 @@ // to CHANGE WITHOUT NOTICE. Therefore DO NOT DEPEND ON IT in a user // program! // GOOGLETEST_CM0001 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_GTEST_MESSAGE_H_ -#define GTEST_INCLUDE_GTEST_GTEST_MESSAGE_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_GTEST_MESSAGE_H_ +#define GOOGLETEST_INCLUDE_GTEST_GTEST_MESSAGE_H_ #include #include #include @@ -214,6 +214,6 @@ } // namespace internal } // namespace testing GTEST_DISABLE_MSC_WARNINGS_POP_() // 4251 -#endif // GTEST_INCLUDE_GTEST_GTEST_MESSAGE_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_GTEST_MESSAGE_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-param-test.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-param-test.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-param-test.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-param-test.h @@ -28,16 +28,13 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // Macros and functions for implementing parameterized tests // in Google C++ Testing and Mocking Framework (Google Test) // -// This file is generated by a SCRIPT. DO NOT EDIT BY HAND! -// // GOOGLETEST_CM0001 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ -#define GTEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ - +#ifndef GOOGLETEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ +#define GOOGLETEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ // Value-parameterized tests allow you to test your code with different // parameters without writing multiple copies of the same test. // // Here is how you use value-parameterized tests: @@ -369,12 +366,10 @@ // the Cartesian product of elements from the sequences generated by // gen1, gen2, ..., genN. The sequence elements will have a type of // std::tuple where T1, T2, ..., TN are the types // of elements from sequences produces by gen1, gen2, ..., genN. // -// Combine can have up to 10 arguments. -// // Example: // // This will instantiate tests in test suite AnimalTest each one with // the parameter values tuple("cat", BLACK), tuple("cat", WHITE), // tuple("dog", BLACK), and tuple("dog", WHITE): @@ -426,11 +421,12 @@ GTEST_STRINGIFY_(test_suite_name), \ ::testing::internal::CodeLocation(__FILE__, __LINE__)) \ ->AddTestPattern( \ GTEST_STRINGIFY_(test_suite_name), GTEST_STRINGIFY_(test_name), \ new ::testing::internal::TestMetaFactory()); \ + test_suite_name, test_name)>(), \ + ::testing::internal::CodeLocation(__FILE__, __LINE__)); \ return 0; \ } \ static int gtest_registering_dummy_ GTEST_ATTRIBUTE_UNUSED_; \ GTEST_DISALLOW_COPY_AND_ASSIGN_(GTEST_TEST_CLASS_NAME_(test_suite_name, \ test_name)); \ @@ -506,6 +502,6 @@ INSTANTIATE_TEST_SUITE_P #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ } // namespace testing -#endif // GTEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-printers.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-printers.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-printers.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-printers.h @@ -95,204 +95,225 @@ // being defined as many user-defined container types don't have // value_type. // GOOGLETEST_CM0001 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_GTEST_PRINTERS_H_ -#define GTEST_INCLUDE_GTEST_GTEST_PRINTERS_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_GTEST_PRINTERS_H_ +#define GOOGLETEST_INCLUDE_GTEST_GTEST_PRINTERS_H_ #include +#include #include // NOLINT #include #include #include #include #include #include + #include "gtest/internal/gtest-internal.h" #include "gtest/internal/gtest-port.h" -#if GTEST_HAS_ABSL -#include "absl/strings/string_view.h" -#include "absl/types/optional.h" -#include "absl/types/variant.h" -#endif // GTEST_HAS_ABSL - namespace testing { -// Definitions in the 'internal' and 'internal2' name spaces are -// subject to change without notice. DO NOT USE THEM IN USER CODE! -namespace internal2 { +// Definitions in the internal* namespaces are subject to change without notice. +// DO NOT USE THEM IN USER CODE! +namespace internal { -// Prints the given number of bytes in the given object to the given -// ostream. -GTEST_API_ void PrintBytesInObjectTo(const unsigned char* obj_bytes, - size_t count, - ::std::ostream* os); +template +void UniversalPrint(const T& value, ::std::ostream* os); -// For selecting which printer to use when a given type has neither << -// nor PrintTo(). -enum TypeKind { - kProtobuf, // a protobuf type - kConvertibleToInteger, // a type implicitly convertible to BiggestInt - // (e.g. a named or unnamed enum type) -#if GTEST_INTERNAL_HAS_STRING_VIEW - kConvertibleToStringView, // a type implicitly convertible to - // absl::string_view or std::string_view -#endif - kOtherType // anything else -}; +// Used to print an STL-style container when the user doesn't define +// a PrintTo() for it. +struct ContainerPrinter { + template (0)) == sizeof(IsContainer)) && + !IsRecursiveContainer::value>::type> + static void PrintValue(const T& container, std::ostream* os) { + const size_t kMaxCount = 32; // The maximum number of elements to print. + *os << '{'; + size_t count = 0; + for (auto&& elem : container) { + if (count > 0) { + *os << ','; + if (count == kMaxCount) { // Enough has been printed. + *os << " ..."; + break; + } + } + *os << ' '; + // We cannot call PrintTo(elem, os) here as PrintTo() doesn't + // handle `elem` being a native array. + internal::UniversalPrint(elem, os); + ++count; + } -// TypeWithoutFormatter::PrintValue(value, os) is called -// by the universal printer to print a value of type T when neither -// operator<< nor PrintTo() is defined for T, where kTypeKind is the -// "kind" of T as defined by enum TypeKind. -template -class TypeWithoutFormatter { - public: - // This default version is called when kTypeKind is kOtherType. - static void PrintValue(const T& value, ::std::ostream* os) { - PrintBytesInObjectTo( - static_cast( - reinterpret_cast(std::addressof(value))), - sizeof(value), os); + if (count > 0) { + *os << ' '; + } + *os << '}'; } }; -// We print a protobuf using its ShortDebugString() when the string -// doesn't exceed this many characters; otherwise we print it using -// DebugString() for better readability. -const size_t kProtobufOneLinerMaxLength = 50; +// Used to print a pointer that is neither a char pointer nor a member +// pointer, when the user doesn't define PrintTo() for it. (A member +// variable pointer or member function pointer doesn't really point to +// a location in the address space. Their representation is +// implementation-defined. Therefore they will be printed as raw +// bytes.) +struct FunctionPointerPrinter { + template ::value>::type> + static void PrintValue(T* p, ::std::ostream* os) { + if (p == nullptr) { + *os << "NULL"; + } else { + // T is a function type, so '*os << p' doesn't do what we want + // (it just prints p as bool). We want to print p as a const + // void*. + *os << reinterpret_cast(p); + } + } +}; + +struct PointerPrinter { + template + static void PrintValue(T* p, ::std::ostream* os) { + if (p == nullptr) { + *os << "NULL"; + } else { + // T is not a function type. We just call << to print p, + // relying on ADL to pick up user-defined << for their pointer + // types, if any. + *os << p; + } + } +}; -template -class TypeWithoutFormatter { - public: +namespace internal_stream_operator_without_lexical_name_lookup { + +// The presence of an operator<< here will terminate lexical scope lookup +// straight away (even though it cannot be a match because of its argument +// types). Thus, the two operator<< calls in StreamPrinter will find only ADL +// candidates. +struct LookupBlocker {}; +void operator<<(LookupBlocker, LookupBlocker); + +struct StreamPrinter { + template ::value>::type, + // Only accept types for which we can find a streaming operator via + // ADL (possibly involving implicit conversions). + typename = decltype(std::declval() + << std::declval())> + static void PrintValue(const T& value, ::std::ostream* os) { + // Call streaming operator found by ADL, possibly with implicit conversions + // of the arguments. + *os << value; + } +}; + +} // namespace internal_stream_operator_without_lexical_name_lookup + +struct ProtobufPrinter { + // We print a protobuf using its ShortDebugString() when the string + // doesn't exceed this many characters; otherwise we print it using + // DebugString() for better readability. + static const size_t kProtobufOneLinerMaxLength = 50; + + template ::value>::type> static void PrintValue(const T& value, ::std::ostream* os) { std::string pretty_str = value.ShortDebugString(); if (pretty_str.length() > kProtobufOneLinerMaxLength) { pretty_str = "\n" + value.DebugString(); } *os << ("<" + pretty_str + ">"); } }; -template -class TypeWithoutFormatter { - public: +struct ConvertibleToIntegerPrinter { // Since T has no << operator or PrintTo() but can be implicitly // converted to BiggestInt, we print it as a BiggestInt. // // Most likely T is an enum type (either named or unnamed), in which // case printing it as an integer is the desired behavior. In case // T is not an enum, printing it as an integer is the best we can do // given that it has no user-defined printer. + static void PrintValue(internal::BiggestInt value, ::std::ostream* os) { + *os << value; + } +}; + +struct ConvertibleToStringViewPrinter { +#if GTEST_INTERNAL_HAS_STRING_VIEW + static void PrintValue(internal::StringView value, ::std::ostream* os) { + internal::UniversalPrint(value, os); + } +#endif +}; + + +// Prints the given number of bytes in the given object to the given +// ostream. +GTEST_API_ void PrintBytesInObjectTo(const unsigned char* obj_bytes, + size_t count, + ::std::ostream* os); +struct RawBytesPrinter { + // SFINAE on `sizeof` to make sure we have a complete type. + template static void PrintValue(const T& value, ::std::ostream* os) { - const internal::BiggestInt kBigInt = value; - *os << kBigInt; + PrintBytesInObjectTo( + static_cast( + // Load bearing cast to void* to support iOS + reinterpret_cast(std::addressof(value))), + sizeof(value), os); } }; -#if GTEST_INTERNAL_HAS_STRING_VIEW -template -class TypeWithoutFormatter { - public: - // Since T has neither operator<< nor PrintTo() but can be implicitly - // converted to absl::string_view, we print it as a absl::string_view - // (or std::string_view). - // - // Note: the implementation is further below, as it depends on - // internal::PrintTo symbol which is defined later in the file. - static void PrintValue(const T& value, ::std::ostream* os); +struct FallbackPrinter { + template + static void PrintValue(const T&, ::std::ostream* os) { + *os << "(incomplete type)"; + } }; -#endif + +// Try every printer in order and return the first one that works. +template +struct FindFirstPrinter : FindFirstPrinter {}; + +template +struct FindFirstPrinter< + T, decltype(Printer::PrintValue(std::declval(), nullptr)), + Printer, Printers...> { + using type = Printer; +}; -// Prints the given value to the given ostream. If the value is a -// protocol message, its debug string is printed; if it's an enum or -// of a type implicitly convertible to BiggestInt, it's printed as an -// integer; otherwise the bytes in the value are printed. This is -// what UniversalPrinter::Print() does when it knows nothing about -// type T and T has neither << operator nor PrintTo(). -// -// A user can override this behavior for a class type Foo by defining -// a << operator in the namespace where Foo is defined. -// -// We put this operator in namespace 'internal2' instead of 'internal' -// to simplify the implementation, as much code in 'internal' needs to -// use << in STL, which would conflict with our own << were it defined -// in 'internal'. -// -// Note that this operator<< takes a generic std::basic_ostream type instead of the more restricted std::ostream. If -// we define it to take an std::ostream instead, we'll get an -// "ambiguous overloads" compiler error when trying to print a type -// Foo that supports streaming to std::basic_ostream, as the compiler cannot tell whether -// operator<<(std::ostream&, const T&) or -// operator<<(std::basic_stream, const Foo&) is more -// specific. -template -::std::basic_ostream& operator<<( - ::std::basic_ostream& os, const T& x) { - TypeWithoutFormatter::value - ? kProtobuf - : std::is_convertible< - const T&, internal::BiggestInt>::value - ? kConvertibleToInteger - : -#if GTEST_INTERNAL_HAS_STRING_VIEW - std::is_convertible< - const T&, internal::StringView>::value - ? kConvertibleToStringView - : -#endif - kOtherType)>::PrintValue(x, &os); - return os; +// Select the best printer in the following order: +// - Print containers (they have begin/end/etc). +// - Print function pointers. +// - Print object pointers. +// - Use the stream operator, if available. +// - Print protocol buffers. +// - Print types convertible to BiggestInt. +// - Print types convertible to StringView, if available. +// - Fallback to printing the raw bytes of the object. +template +void PrintWithFallback(const T& value, ::std::ostream* os) { + using Printer = typename FindFirstPrinter< + T, void, ContainerPrinter, FunctionPointerPrinter, PointerPrinter, + internal_stream_operator_without_lexical_name_lookup::StreamPrinter, + ProtobufPrinter, ConvertibleToIntegerPrinter, + ConvertibleToStringViewPrinter, RawBytesPrinter, FallbackPrinter>::type; + Printer::PrintValue(value, os); } -} // namespace internal2 -} // namespace testing - -// This namespace MUST NOT BE NESTED IN ::testing, or the name look-up -// magic needed for implementing UniversalPrinter won't work. -namespace testing_internal { - -// Used to print a value that is not an STL-style container when the -// user doesn't define PrintTo() for it. -template -void DefaultPrintNonContainerTo(const T& value, ::std::ostream* os) { - // With the following statement, during unqualified name lookup, - // testing::internal2::operator<< appears as if it was declared in - // the nearest enclosing namespace that contains both - // ::testing_internal and ::testing::internal2, i.e. the global - // namespace. For more details, refer to the C++ Standard section - // 7.3.4-1 [namespace.udir]. This allows us to fall back onto - // testing::internal2::operator<< in case T doesn't come with a << - // operator. - - using ::testing::internal2::operator<<; - - // Assuming T is defined in namespace foo, in the next statement, - // the compiler will consider all of: - // - // 1. foo::operator<< (thanks to Koenig look-up), - // 2. ::operator<< (as the current namespace is enclosed in ::), - // 3. testing::internal2::operator<< (thanks to the using statement above). - // - // The operator<< whose type matches T best will be picked. - // - // We deliberately allow #2 to be a candidate, as sometimes it's - // impossible to define #1 (e.g. when foo is ::std, defining - // anything in it is undefined behavior unless you are a compiler - // vendor.). - *os << value; -} - -} // namespace testing_internal - -namespace testing { -namespace internal { - // FormatForComparison::Format(value) formats a // value of type ToPrint that is an operand of a comparison assertion // (e.g. ASSERT_EQ). OtherOperand is the type of the other operand in // the comparison, and is used to help determine the best way to // format the value. In particular, when the value is a C string @@ -337,10 +358,18 @@ GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(char); GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(const char); GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(wchar_t); GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(const wchar_t); +#ifdef __cpp_char8_t +GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(char8_t); +GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(const char8_t); +#endif +GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(char16_t); +GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(const char16_t); +GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(char32_t); +GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(const char32_t); #undef GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_ // If a C string is compared with an STL string object, we know it's meant // to point to a NUL-terminated string, and thus can print it as a string. @@ -354,10 +383,18 @@ } \ } GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(char, ::std::string); GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const char, ::std::string); +#ifdef __cpp_char8_t +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(char8_t, ::std::u8string); +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const char8_t, ::std::u8string); +#endif +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(char16_t, ::std::u16string); +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const char16_t, ::std::u16string); +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(char32_t, ::std::u32string); +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const char32_t, ::std::u32string); #if GTEST_HAS_STD_WSTRING GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(wchar_t, ::std::wstring); GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const wchar_t, ::std::wstring); #endif @@ -386,89 +423,10 @@ // function template), as we need to partially specialize it for // reference types, which cannot be done with function templates. template class UniversalPrinter; -template -void UniversalPrint(const T& value, ::std::ostream* os); - -enum DefaultPrinterType { - kPrintContainer, - kPrintPointer, - kPrintFunctionPointer, - kPrintOther, -}; -template struct WrapPrinterType {}; - -// Used to print an STL-style container when the user doesn't define -// a PrintTo() for it. -template -void DefaultPrintTo(WrapPrinterType /* dummy */, - const C& container, ::std::ostream* os) { - const size_t kMaxCount = 32; // The maximum number of elements to print. - *os << '{'; - size_t count = 0; - for (typename C::const_iterator it = container.begin(); - it != container.end(); ++it, ++count) { - if (count > 0) { - *os << ','; - if (count == kMaxCount) { // Enough has been printed. - *os << " ..."; - break; - } - } - *os << ' '; - // We cannot call PrintTo(*it, os) here as PrintTo() doesn't - // handle *it being a native array. - internal::UniversalPrint(*it, os); - } - - if (count > 0) { - *os << ' '; - } - *os << '}'; -} - -// Used to print a pointer that is neither a char pointer nor a member -// pointer, when the user doesn't define PrintTo() for it. (A member -// variable pointer or member function pointer doesn't really point to -// a location in the address space. Their representation is -// implementation-defined. Therefore they will be printed as raw -// bytes.) -template -void DefaultPrintTo(WrapPrinterType /* dummy */, - T* p, ::std::ostream* os) { - if (p == nullptr) { - *os << "NULL"; - } else { - // T is not a function type. We just call << to print p, - // relying on ADL to pick up user-defined << for their pointer - // types, if any. - *os << p; - } -} -template -void DefaultPrintTo(WrapPrinterType /* dummy */, - T* p, ::std::ostream* os) { - if (p == nullptr) { - *os << "NULL"; - } else { - // T is a function type, so '*os << p' doesn't do what we want - // (it just prints p as bool). We want to print p as a const - // void*. - *os << reinterpret_cast(p); - } -} - -// Used to print a non-container, non-pointer value when the user -// doesn't define PrintTo() for it. -template -void DefaultPrintTo(WrapPrinterType /* dummy */, - const T& value, ::std::ostream* os) { - ::testing_internal::DefaultPrintNonContainerTo(value, os); -} - // Prints the given value using the << operator if it has one; // otherwise prints the bytes in it. This is what // UniversalPrinter::Print() does when PrintTo() is not specialized // or overloaded for type T. // @@ -478,40 +436,11 @@ // Foo is not desirable (e.g. the coding style may prevent doing it, // or there is already a << operator but it doesn't do what the user // wants). template void PrintTo(const T& value, ::std::ostream* os) { - // DefaultPrintTo() is overloaded. The type of its first argument - // determines which version will be picked. - // - // Note that we check for container types here, prior to we check - // for protocol message types in our operator<<. The rationale is: - // - // For protocol messages, we want to give people a chance to - // override Google Mock's format by defining a PrintTo() or - // operator<<. For STL containers, other formats can be - // incompatible with Google Mock's format for the container - // elements; therefore we check for container types here to ensure - // that our format is used. - // - // Note that MSVC and clang-cl do allow an implicit conversion from - // pointer-to-function to pointer-to-object, but clang-cl warns on it. - // So don't use ImplicitlyConvertible if it can be helped since it will - // cause this warning, and use a separate overload of DefaultPrintTo for - // function pointers so that the `*os << p` in the object pointer overload - // doesn't cause that warning either. - DefaultPrintTo( - WrapPrinterType < - (sizeof(IsContainerTest(0)) == sizeof(IsContainer)) && - !IsRecursiveContainer::value - ? kPrintContainer - : !std::is_pointer::value - ? kPrintOther - : std::is_function::type>::value - ? kPrintFunctionPointer - : kPrintPointer > (), - value, os); + internal::PrintWithFallback(value, os); } // The following list of PrintTo() overloads tells // UniversalPrinter::Print() how to print standard types (built-in // types, strings, plain arrays, and pointers). @@ -538,10 +467,20 @@ // as signed integer when wchar_t is implemented by the compiler // as a signed type and is printed as an unsigned integer when wchar_t // is implemented as an unsigned type. GTEST_API_ void PrintTo(wchar_t wc, ::std::ostream* os); +GTEST_API_ void PrintTo(char32_t c, ::std::ostream* os); +inline void PrintTo(char16_t c, ::std::ostream* os) { + PrintTo(ImplicitCast_(c), os); +} +#ifdef __cpp_char8_t +inline void PrintTo(char8_t c, ::std::ostream* os) { + PrintTo(ImplicitCast_(c), os); +} +#endif + // Overloads for C strings. GTEST_API_ void PrintTo(const char* s, ::std::ostream* os); inline void PrintTo(char* s, ::std::ostream* os) { PrintTo(ImplicitCast_(s), os); } @@ -558,10 +497,27 @@ PrintTo(ImplicitCast_(s), os); } inline void PrintTo(unsigned char* s, ::std::ostream* os) { PrintTo(ImplicitCast_(s), os); } +#ifdef __cpp_char8_t +// Overloads for u8 strings. +GTEST_API_ void PrintTo(const char8_t* s, ::std::ostream* os); +inline void PrintTo(char8_t* s, ::std::ostream* os) { + PrintTo(ImplicitCast_(s), os); +} +#endif +// Overloads for u16 strings. +GTEST_API_ void PrintTo(const char16_t* s, ::std::ostream* os); +inline void PrintTo(char16_t* s, ::std::ostream* os) { + PrintTo(ImplicitCast_(s), os); +} +// Overloads for u32 strings. +GTEST_API_ void PrintTo(const char32_t* s, ::std::ostream* os); +inline void PrintTo(char32_t* s, ::std::ostream* os) { + PrintTo(ImplicitCast_(s), os); +} // MSVC can be configured to define wchar_t as a typedef of unsigned // short. It defines _NATIVE_WCHAR_T_DEFINED when wchar_t is a native // type. When wchar_t is a typedef, defining an overload for const // wchar_t* would cause unsigned short* be printed as a wide string, @@ -592,10 +548,30 @@ GTEST_API_ void PrintStringTo(const ::std::string&s, ::std::ostream* os); inline void PrintTo(const ::std::string& s, ::std::ostream* os) { PrintStringTo(s, os); } +// Overloads for ::std::u8string +#ifdef __cpp_char8_t +GTEST_API_ void PrintU8StringTo(const ::std::u8string& s, ::std::ostream* os); +inline void PrintTo(const ::std::u8string& s, ::std::ostream* os) { + PrintU8StringTo(s, os); +} +#endif + +// Overloads for ::std::u16string +GTEST_API_ void PrintU16StringTo(const ::std::u16string& s, ::std::ostream* os); +inline void PrintTo(const ::std::u16string& s, ::std::ostream* os) { + PrintU16StringTo(s, os); +} + +// Overloads for ::std::u32string +GTEST_API_ void PrintU32StringTo(const ::std::u32string& s, ::std::ostream* os); +inline void PrintTo(const ::std::u32string& s, ::std::ostream* os) { + PrintU32StringTo(s, os); +} + // Overloads for ::std::wstring. #if GTEST_HAS_STD_WSTRING GTEST_API_ void PrintWideStringTo(const ::std::wstring&s, ::std::ostream* os); inline void PrintTo(const ::std::wstring& s, ::std::ostream* os) { PrintWideStringTo(s, os); @@ -614,10 +590,47 @@ template void PrintTo(std::reference_wrapper ref, ::std::ostream* os) { UniversalPrinter::Print(ref.get(), os); } +inline const void* VoidifyPointer(const void* p) { return p; } +inline const void* VoidifyPointer(volatile const void* p) { + return const_cast(p); +} + +template +void PrintSmartPointer(const Ptr& ptr, std::ostream* os, char) { + if (ptr == nullptr) { + *os << "(nullptr)"; + } else { + // We can't print the value. Just print the pointer.. + *os << "(" << (VoidifyPointer)(ptr.get()) << ")"; + } +} +template ::value && + !std::is_array::value>::type> +void PrintSmartPointer(const Ptr& ptr, std::ostream* os, int) { + if (ptr == nullptr) { + *os << "(nullptr)"; + } else { + *os << "(ptr = " << (VoidifyPointer)(ptr.get()) << ", value = "; + UniversalPrinter::Print(*ptr, os); + *os << ")"; + } +} + +template +void PrintTo(const std::unique_ptr& ptr, std::ostream* os) { + (PrintSmartPointer)(ptr, os, 0); +} + +template +void PrintTo(const std::shared_ptr& ptr, std::ostream* os) { + (PrintSmartPointer)(ptr, os, 0); +} + // Helper function for printing a tuple. T must be instantiated with // a tuple type. template void PrintTupleTo(const T&, std::integral_constant, ::std::ostream*) {} @@ -679,51 +692,93 @@ } GTEST_DISABLE_MSC_WARNINGS_POP_() }; -#if GTEST_HAS_ABSL +// Remove any const-qualifiers before passing a type to UniversalPrinter. +template +class UniversalPrinter : public UniversalPrinter {}; + +#if GTEST_INTERNAL_HAS_ANY + +// Printer for std::any / absl::any -// Printer for absl::optional +template <> +class UniversalPrinter { + public: + static void Print(const Any& value, ::std::ostream* os) { + if (value.has_value()) { + *os << "value of type " << GetTypeName(value); + } else { + *os << "no value"; + } + } + + private: + static std::string GetTypeName(const Any& value) { +#if GTEST_HAS_RTTI + return internal::GetTypeName(value.type()); +#else + static_cast(value); // possibly unused + return ""; +#endif // GTEST_HAS_RTTI + } +}; + +#endif // GTEST_INTERNAL_HAS_ANY + +#if GTEST_INTERNAL_HAS_OPTIONAL + +// Printer for std::optional / absl::optional template -class UniversalPrinter<::absl::optional> { +class UniversalPrinter> { public: - static void Print(const ::absl::optional& value, ::std::ostream* os) { + static void Print(const Optional& value, ::std::ostream* os) { *os << '('; if (!value) { *os << "nullopt"; } else { UniversalPrint(*value, os); } *os << ')'; } }; -// Printer for absl::variant +#endif // GTEST_INTERNAL_HAS_OPTIONAL + +#if GTEST_INTERNAL_HAS_VARIANT + +// Printer for std::variant / absl::variant template -class UniversalPrinter<::absl::variant> { +class UniversalPrinter> { public: - static void Print(const ::absl::variant& value, ::std::ostream* os) { + static void Print(const Variant& value, ::std::ostream* os) { *os << '('; - absl::visit(Visitor{os}, value); +#if GTEST_HAS_ABSL + absl::visit(Visitor{os, value.index()}, value); +#else + std::visit(Visitor{os, value.index()}, value); +#endif // GTEST_HAS_ABSL *os << ')'; } private: struct Visitor { template void operator()(const U& u) const { - *os << "'" << GetTypeName() << "' with value "; + *os << "'" << GetTypeName() << "(index = " << index + << ")' with value "; UniversalPrint(u, os); } ::std::ostream* os; + std::size_t index; }; }; -#endif // GTEST_HAS_ABSL +#endif // GTEST_INTERNAL_HAS_VARIANT // UniversalPrintArray(begin, len, os) prints an array of 'len' // elements, starting at address 'begin'. template void UniversalPrintArray(const T* begin, size_t len, ::std::ostream* os) { @@ -748,10 +803,24 @@ } // This overload prints a (const) char array compactly. GTEST_API_ void UniversalPrintArray( const char* begin, size_t len, ::std::ostream* os); +#ifdef __cpp_char8_t +// This overload prints a (const) char8_t array compactly. +GTEST_API_ void UniversalPrintArray(const char8_t* begin, size_t len, + ::std::ostream* os); +#endif + +// This overload prints a (const) char16_t array compactly. +GTEST_API_ void UniversalPrintArray(const char16_t* begin, size_t len, + ::std::ostream* os); + +// This overload prints a (const) char32_t array compactly. +GTEST_API_ void UniversalPrintArray(const char32_t* begin, size_t len, + ::std::ostream* os); + // This overload prints a (const) wchar_t array compactly. GTEST_API_ void UniversalPrintArray( const wchar_t* begin, size_t len, ::std::ostream* os); // Implements printing an array type T[N]. @@ -820,16 +889,59 @@ UniversalPrint(std::string(str), os); } } }; template <> -class UniversalTersePrinter { +class UniversalTersePrinter : public UniversalTersePrinter { +}; + +#ifdef __cpp_char8_t +template <> +class UniversalTersePrinter { public: - static void Print(char* str, ::std::ostream* os) { - UniversalTersePrinter::Print(str, os); + static void Print(const char8_t* str, ::std::ostream* os) { + if (str == nullptr) { + *os << "NULL"; + } else { + UniversalPrint(::std::u8string(str), os); + } } }; +template <> +class UniversalTersePrinter + : public UniversalTersePrinter {}; +#endif + +template <> +class UniversalTersePrinter { + public: + static void Print(const char16_t* str, ::std::ostream* os) { + if (str == nullptr) { + *os << "NULL"; + } else { + UniversalPrint(::std::u16string(str), os); + } + } +}; +template <> +class UniversalTersePrinter + : public UniversalTersePrinter {}; + +template <> +class UniversalTersePrinter { + public: + static void Print(const char32_t* str, ::std::ostream* os) { + if (str == nullptr) { + *os << "NULL"; + } else { + UniversalPrint(::std::u32string(str), os); + } + } +}; +template <> +class UniversalTersePrinter + : public UniversalTersePrinter {}; #if GTEST_HAS_STD_WSTRING template <> class UniversalTersePrinter { public: @@ -898,20 +1010,10 @@ return result; } } // namespace internal -#if GTEST_INTERNAL_HAS_STRING_VIEW -namespace internal2 { -template -void TypeWithoutFormatter::PrintValue( - const T& value, ::std::ostream* os) { - internal::PrintTo(internal::StringView(value), os); -} -} // namespace internal2 -#endif - template ::std::string PrintToString(const T& value) { ::std::stringstream ss; internal::UniversalTersePrinter::Print(value, &ss); return ss.str(); @@ -922,6 +1024,6 @@ // Include any custom printer added by the local installation. // We must include this header at the end to make sure it can use the // declarations from this file. #include "gtest/internal/custom/gtest-printers.h" -#endif // GTEST_INCLUDE_GTEST_GTEST_PRINTERS_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_GTEST_PRINTERS_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-spi.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-spi.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-spi.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-spi.h @@ -31,12 +31,12 @@ // Utilities for testing Google Test itself and code that uses Google Test // (e.g. frameworks built on top of Google Test). // GOOGLETEST_CM0004 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_GTEST_SPI_H_ -#define GTEST_INCLUDE_GTEST_GTEST_SPI_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_GTEST_SPI_H_ +#define GOOGLETEST_INCLUDE_GTEST_GTEST_SPI_H_ #include "gtest/gtest.h" GTEST_DISABLE_MSC_WARNINGS_PUSH_(4251 \ /* class A needs to have dll-interface to be used by clients of class B */) @@ -233,6 +233,6 @@ >est_failures);\ if (::testing::internal::AlwaysTrue()) { statement; }\ }\ } while (::testing::internal::AlwaysFalse()) -#endif // GTEST_INCLUDE_GTEST_GTEST_SPI_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_GTEST_SPI_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-test-part.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-test-part.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-test-part.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-test-part.h @@ -27,12 +27,12 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // GOOGLETEST_CM0001 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_GTEST_TEST_PART_H_ -#define GTEST_INCLUDE_GTEST_GTEST_TEST_PART_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_GTEST_TEST_PART_H_ +#define GOOGLETEST_INCLUDE_GTEST_GTEST_TEST_PART_H_ #include #include #include "gtest/internal/gtest-internal.h" #include "gtest/internal/gtest-string.h" @@ -179,6 +179,6 @@ } // namespace testing GTEST_DISABLE_MSC_WARNINGS_POP_() // 4251 -#endif // GTEST_INCLUDE_GTEST_GTEST_TEST_PART_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_GTEST_TEST_PART_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-typed-test.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-typed-test.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-typed-test.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-typed-test.h @@ -27,12 +27,12 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // GOOGLETEST_CM0001 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_GTEST_TYPED_TEST_H_ -#define GTEST_INCLUDE_GTEST_GTEST_TYPED_TEST_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_GTEST_TYPED_TEST_H_ +#define GOOGLETEST_INCLUDE_GTEST_GTEST_TYPED_TEST_H_ // This header implements typed tests and type-parameterized tests. // Typed (aka type-driven) tests repeat the same test for types in a // list. You must know which types you want to test with when writing @@ -173,12 +173,10 @@ #include "gtest/internal/gtest-port.h" #include "gtest/internal/gtest-type-util.h" // Implements typed tests. -#if GTEST_HAS_TYPED_TEST - // INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. // // Expands to the name of the typedef for the type parameters of the // given test suite. #define GTEST_TYPE_PARAMS_(TestSuiteName) gtest_type_params_##TestSuiteName##_ @@ -228,16 +226,12 @@ #define TYPED_TEST_CASE \ static_assert(::testing::internal::TypedTestCaseIsDeprecated(), ""); \ TYPED_TEST_SUITE #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ -#endif // GTEST_HAS_TYPED_TEST - // Implements type-parameterized tests. -#if GTEST_HAS_TYPED_TEST_P - // INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. // // Expands to the namespace name that the type-parameterized tests for // the given type-parameterized test suite are defined in. The exact // name of the namespace is subject to change without notice. @@ -330,8 +324,6 @@ static_assert( \ ::testing::internal::InstantiateTypedTestCase_P_IsDeprecated(), ""); \ INSTANTIATE_TYPED_TEST_SUITE_P #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ -#endif // GTEST_HAS_TYPED_TEST_P - -#endif // GTEST_INCLUDE_GTEST_GTEST_TYPED_TEST_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_GTEST_TYPED_TEST_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest.h @@ -47,12 +47,12 @@ // registration from Barthelemy Dagenais' (barthelemy@prologique.com) // easyUnit framework. // GOOGLETEST_CM0001 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_GTEST_H_ -#define GTEST_INCLUDE_GTEST_GTEST_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_GTEST_H_ +#define GOOGLETEST_INCLUDE_GTEST_GTEST_H_ #include #include #include #include @@ -99,10 +99,14 @@ // This flag enables using colors in terminal output. Available values are // "yes" to enable colors, "no" (disable colors), or "auto" (the default) // to let Google Test decide. GTEST_DECLARE_string_(color); +// This flag controls whether the test runner should continue execution past +// first failure. +GTEST_DECLARE_bool_(fail_fast); + // This flag sets up the filter to select by name using a glob pattern // the tests to run. If the filter is not given all tests are executed. GTEST_DECLARE_string_(filter); // This flag controls whether Google Test installs a signal handler that dumps @@ -115,10 +119,13 @@ // This flag controls whether Google Test emits a detailed XML report to a file // in addition to its normal textual output. GTEST_DECLARE_string_(output); +// This flags control whether Google Test prints only test failures. +GTEST_DECLARE_bool_(brief); + // This flags control whether Google Test prints the elapsed time for each // test. GTEST_DECLARE_bool_(print_time); // This flags control whether Google Test prints UTF8 characters as text. @@ -409,27 +416,28 @@ friend class TestInfo; // The d'tor is virtual as we intend to inherit from Test. virtual ~Test(); - // Sets up the stuff shared by all tests in this test case. + // Sets up the stuff shared by all tests in this test suite. // // Google Test will call Foo::SetUpTestSuite() before running the first - // test in test case Foo. Hence a sub-class can define its own + // test in test suite Foo. Hence a sub-class can define its own // SetUpTestSuite() method to shadow the one defined in the super // class. static void SetUpTestSuite() {} // Tears down the stuff shared by all tests in this test suite. // // Google Test will call Foo::TearDownTestSuite() after running the last - // test in test case Foo. Hence a sub-class can define its own + // test in test suite Foo. Hence a sub-class can define its own // TearDownTestSuite() method to shadow the one defined in the super // class. static void TearDownTestSuite() {} - // Legacy API is deprecated but still available + // Legacy API is deprecated but still available. Use SetUpTestSuite and + // TearDownTestSuite instead. #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ static void TearDownTestCase() {} static void SetUpTestCase() {} #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ @@ -663,11 +671,11 @@ // Clears the object. void Clear(); // Protects mutable state of the property vector and of owned // properties, whose values may be updated. - internal::Mutex test_properites_mutex_; + internal::Mutex test_properties_mutex_; // The vector of TestPartResults std::vector test_part_results_; // The vector of TestProperties std::vector test_properties_; @@ -793,10 +801,13 @@ // Creates the test object, runs it, records its result, and then // deletes it. void Run(); + // Skip and records the test result for this object. + void Skip(); + static void ClearTestResult(TestInfo* test_info) { test_info->result_.Clear(); } // These fields are immutable properties of the test. @@ -941,10 +952,13 @@ } // Runs every test in this TestSuite. void Run(); + // Skips the execution of tests under this TestSuite + void Skip(); + // Runs SetUpTestSuite() for this TestSuite. This wrapper is needed // for catching exceptions thrown from SetUpTestSuite(). void RunSetUpTestSuite() { if (set_up_tc_ != nullptr) { (*set_up_tc_)(); @@ -1533,18 +1547,10 @@ } return CmpHelperEQFailure(lhs_expression, rhs_expression, lhs, rhs); } -// With this overloaded version, we allow anonymous enums to be used -// in {ASSERT|EXPECT}_EQ when compiled with gcc 4, as anonymous enums -// can be implicitly cast to BiggestInt. -GTEST_API_ AssertionResult CmpHelperEQ(const char* lhs_expression, - const char* rhs_expression, - BiggestInt lhs, - BiggestInt rhs); - class EqHelper { public: // This templatized version is for the general case. template < typename T1, typename T2, @@ -1597,15 +1603,10 @@ // A macro for implementing the helper functions needed to implement // ASSERT_?? and EXPECT_??. It is here just to avoid copy-and-paste // of similar code. // -// For each templatized helper function, we also define an overloaded -// version for BiggestInt in order to reduce code bloat and allow -// anonymous enums to be used with {ASSERT|EXPECT}_?? when compiled -// with gcc 4. -// // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. #define GTEST_IMPL_CMP_HELPER_(op_name, op)\ template \ AssertionResult CmpHelper##op_name(const char* expr1, const char* expr2, \ @@ -1613,26 +1614,24 @@ if (val1 op val2) {\ return AssertionSuccess();\ } else {\ return CmpHelperOpFailure(expr1, expr2, val1, val2, #op);\ }\ -}\ -GTEST_API_ AssertionResult CmpHelper##op_name(\ - const char* expr1, const char* expr2, BiggestInt val1, BiggestInt val2) +} // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. // Implements the helper function for {ASSERT|EXPECT}_NE -GTEST_IMPL_CMP_HELPER_(NE, !=); +GTEST_IMPL_CMP_HELPER_(NE, !=) // Implements the helper function for {ASSERT|EXPECT}_LE -GTEST_IMPL_CMP_HELPER_(LE, <=); +GTEST_IMPL_CMP_HELPER_(LE, <=) // Implements the helper function for {ASSERT|EXPECT}_LT -GTEST_IMPL_CMP_HELPER_(LT, <); +GTEST_IMPL_CMP_HELPER_(LT, <) // Implements the helper function for {ASSERT|EXPECT}_GE -GTEST_IMPL_CMP_HELPER_(GE, >=); +GTEST_IMPL_CMP_HELPER_(GE, >=) // Implements the helper function for {ASSERT|EXPECT}_GT -GTEST_IMPL_CMP_HELPER_(GT, >); +GTEST_IMPL_CMP_HELPER_(GT, >) #undef GTEST_IMPL_CMP_HELPER_ // The helper function for {ASSERT|EXPECT}_STREQ. // @@ -1805,16 +1804,10 @@ AssertHelperData* const data_; GTEST_DISALLOW_COPY_AND_ASSIGN_(AssertHelper); }; -enum GTestColor { COLOR_DEFAULT, COLOR_RED, COLOR_GREEN, COLOR_YELLOW }; - -GTEST_API_ GTEST_ATTRIBUTE_PRINTF_(2, 3) void ColoredPrintf(GTestColor color, - const char* fmt, - ...); - } // namespace internal // The pure interface class that all value-parameterized tests inherit from. // A value-parameterized class must inherit from both ::testing::Test and // ::testing::WithParamInterface. In most cases that just means inheriting @@ -1967,23 +1960,42 @@ GTEST_TEST_ANY_THROW_(statement, GTEST_FATAL_FAILURE_) // Boolean assertions. Condition can be either a Boolean expression or an // AssertionResult. For more information on how to use AssertionResult with // these macros see comments on that class. -#define EXPECT_TRUE(condition) \ +#define GTEST_EXPECT_TRUE(condition) \ GTEST_TEST_BOOLEAN_(condition, #condition, false, true, \ GTEST_NONFATAL_FAILURE_) -#define EXPECT_FALSE(condition) \ +#define GTEST_EXPECT_FALSE(condition) \ GTEST_TEST_BOOLEAN_(!(condition), #condition, true, false, \ GTEST_NONFATAL_FAILURE_) -#define ASSERT_TRUE(condition) \ +#define GTEST_ASSERT_TRUE(condition) \ GTEST_TEST_BOOLEAN_(condition, #condition, false, true, \ GTEST_FATAL_FAILURE_) -#define ASSERT_FALSE(condition) \ +#define GTEST_ASSERT_FALSE(condition) \ GTEST_TEST_BOOLEAN_(!(condition), #condition, true, false, \ GTEST_FATAL_FAILURE_) +// Define these macros to 1 to omit the definition of the corresponding +// EXPECT or ASSERT, which clashes with some users' own code. + +#if !GTEST_DONT_DEFINE_EXPECT_TRUE +#define EXPECT_TRUE(condition) GTEST_EXPECT_TRUE(condition) +#endif + +#if !GTEST_DONT_DEFINE_EXPECT_FALSE +#define EXPECT_FALSE(condition) GTEST_EXPECT_FALSE(condition) +#endif + +#if !GTEST_DONT_DEFINE_ASSERT_TRUE +#define ASSERT_TRUE(condition) GTEST_ASSERT_TRUE(condition) +#endif + +#if !GTEST_DONT_DEFINE_ASSERT_FALSE +#define ASSERT_FALSE(condition) GTEST_ASSERT_FALSE(condition) +#endif + // Macros for testing equalities and inequalities. // // * {ASSERT|EXPECT}_EQ(v1, v2): Tests that v1 == v2 // * {ASSERT|EXPECT}_NE(v1, v2): Tests that v1 != v2 // * {ASSERT|EXPECT}_LT(v1, v2): Tests that v1 < v2 @@ -2478,6 +2490,6 @@ return ::testing::UnitTest::GetInstance()->Run(); } GTEST_DISABLE_MSC_WARNINGS_POP_() // 4251 -#endif // GTEST_INCLUDE_GTEST_GTEST_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_GTEST_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest_pred_impl.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest_pred_impl.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest_pred_impl.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest_pred_impl.h @@ -31,12 +31,12 @@ // 'gen_gtest_pred_impl.py 5'. DO NOT EDIT BY HAND! // // Implements a family of generic predicate assertion macros. // GOOGLETEST_CM0001 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_GTEST_PRED_IMPL_H_ -#define GTEST_INCLUDE_GTEST_GTEST_PRED_IMPL_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_GTEST_PRED_IMPL_H_ +#define GOOGLETEST_INCLUDE_GTEST_GTEST_PRED_IMPL_H_ #include "gtest/gtest.h" namespace testing { @@ -354,6 +354,6 @@ } // namespace testing -#endif // GTEST_INCLUDE_GTEST_GTEST_PRED_IMPL_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_GTEST_PRED_IMPL_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest_prod.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest_prod.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest_prod.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest_prod.h @@ -29,12 +29,12 @@ // // Google C++ Testing and Mocking Framework definitions useful in production code. // GOOGLETEST_CM0003 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_GTEST_PROD_H_ -#define GTEST_INCLUDE_GTEST_GTEST_PROD_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_GTEST_PROD_H_ +#define GOOGLETEST_INCLUDE_GTEST_GTEST_PROD_H_ // When you need to test the private or protected members of a class, // use the FRIEND_TEST macro to declare your tests as friends of the // class. For example: // @@ -56,6 +56,6 @@ // For example, putting MyClassTest in an anonymous namespace will not work. #define FRIEND_TEST(test_case_name, test_name)\ friend class test_case_name##_##test_name##_Test -#endif // GTEST_INCLUDE_GTEST_GTEST_PROD_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_GTEST_PROD_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/custom/gtest-port.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/custom/gtest-port.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/custom/gtest-port.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/custom/gtest-port.h @@ -29,9 +29,9 @@ // // Injection point for custom user configurations. See README for details // // ** Custom implementation starts here ** -#ifndef GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PORT_H_ -#define GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PORT_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PORT_H_ +#define GOOGLETEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PORT_H_ -#endif // GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PORT_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PORT_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/custom/gtest-printers.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/custom/gtest-printers.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/custom/gtest-printers.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/custom/gtest-printers.h @@ -34,9 +34,9 @@ // // Injection point for custom user configurations. See README for details // // ** Custom implementation starts here ** -#ifndef GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PRINTERS_H_ -#define GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PRINTERS_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PRINTERS_H_ +#define GOOGLETEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PRINTERS_H_ -#endif // GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PRINTERS_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PRINTERS_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/custom/gtest.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/custom/gtest.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/custom/gtest.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/custom/gtest.h @@ -29,9 +29,9 @@ // // Injection point for custom user configurations. See README for details // // ** Custom implementation starts here ** -#ifndef GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_H_ -#define GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_H_ +#define GOOGLETEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_H_ -#endif // GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-death-test-internal.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-death-test-internal.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-death-test-internal.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-death-test-internal.h @@ -31,12 +31,12 @@ // // This header file defines internal utilities needed for implementing // death tests. They are subject to change without notice. // GOOGLETEST_CM0001 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_DEATH_TEST_INTERNAL_H_ -#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_DEATH_TEST_INTERNAL_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_DEATH_TEST_INTERNAL_H_ +#define GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_DEATH_TEST_INTERNAL_H_ #include "gtest/gtest-matchers.h" #include "gtest/internal/gtest-internal.h" #include @@ -299,6 +299,6 @@ #endif // GTEST_HAS_DEATH_TEST } // namespace internal } // namespace testing -#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_DEATH_TEST_INTERNAL_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_DEATH_TEST_INTERNAL_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-filepath.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-filepath.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-filepath.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-filepath.h @@ -35,12 +35,12 @@ // This file is #included in gtest/internal/gtest-internal.h. // Do not include this header file separately! // GOOGLETEST_CM0001 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_FILEPATH_H_ -#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_FILEPATH_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_FILEPATH_H_ +#define GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_FILEPATH_H_ #include "gtest/internal/gtest-string.h" GTEST_DISABLE_MSC_WARNINGS_PUSH_(4251 \ /* class A needs to have dll-interface to be used by clients of class B */) @@ -193,11 +193,11 @@ // the primary path separator '\\', so that for example "bar\\/\\foo" becomes // "bar\\foo". void Normalize(); - // Returns a pointer to the last occurence of a valid path separator in + // Returns a pointer to the last occurrence of a valid path separator in // the FilePath. On Windows, for example, both '/' and '\' are valid path // separators. Returns NULL if no path separator was found. const char* FindLastPathSeparator() const; std::string pathname_; @@ -206,6 +206,6 @@ } // namespace internal } // namespace testing GTEST_DISABLE_MSC_WARNINGS_POP_() // 4251 -#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_FILEPATH_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_FILEPATH_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-internal.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-internal.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-internal.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-internal.h @@ -32,12 +32,12 @@ // This header file declares functions and macros used internally by // Google Test. They are subject to change without notice. // GOOGLETEST_CM0001 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_ -#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_ +#define GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_ #include "gtest/internal/gtest-port.h" #if GTEST_OS_LINUX # include @@ -88,11 +88,13 @@ // Complaining about the argument to GTEST_STRINGIFY_ being empty. // This is allowed by the spec. #define GTEST_STRINGIFY_HELPER_(name, ...) #name #define GTEST_STRINGIFY_(...) GTEST_STRINGIFY_HELPER_(__VA_ARGS__, ) -namespace proto2 { class Message; } +namespace proto2 { +class MessageLite; +} namespace testing { // Forward declarations. @@ -283,11 +285,11 @@ // calculations are done with 80-bit precision, while double has 64 // bits. Therefore, 4 should be enough for ordinary use. // // See the following article for more details on ULP: // http://randomascii.wordpress.com/2012/02/25/comparing-floating-point-numbers-2012-edition/ - static const size_t kMaxUlps = 4; + static const uint32_t kMaxUlps = 4; // Constructs a FloatingPoint from a raw floating-point number. // // On an Intel CPU, passing a non-normalized NAN (Not a Number) // around may change its bits, although the new value is guaranteed @@ -516,10 +518,11 @@ using Test = typename std::conditional::type; static SetUpTearDownSuiteFuncType GetSetUpCaseOrSuite(const char* filename, int line_num) { +#ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ SetUpTearDownSuiteFuncType test_case_fp = GetNotDefaultOrNull(&T::SetUpTestCase, &Test::SetUpTestCase); SetUpTearDownSuiteFuncType test_suite_fp = GetNotDefaultOrNull(&T::SetUpTestSuite, &Test::SetUpTestSuite); @@ -527,14 +530,20 @@ << "Test can not provide both SetUpTestSuite and SetUpTestCase, please " "make sure there is only one present at " << filename << ":" << line_num; return test_case_fp != nullptr ? test_case_fp : test_suite_fp; +#else + (void)(filename); + (void)(line_num); + return &T::SetUpTestSuite; +#endif } static SetUpTearDownSuiteFuncType GetTearDownCaseOrSuite(const char* filename, int line_num) { +#ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ SetUpTearDownSuiteFuncType test_case_fp = GetNotDefaultOrNull(&T::TearDownTestCase, &Test::TearDownTestCase); SetUpTearDownSuiteFuncType test_suite_fp = GetNotDefaultOrNull(&T::TearDownTestSuite, &Test::TearDownTestSuite); @@ -542,23 +551,28 @@ << "Test can not provide both TearDownTestSuite and TearDownTestCase," " please make sure there is only one present at" << filename << ":" << line_num; return test_case_fp != nullptr ? test_case_fp : test_suite_fp; +#else + (void)(filename); + (void)(line_num); + return &T::TearDownTestSuite; +#endif } }; // Creates a new TestInfo object and registers it with Google Test; // returns the created object. // // Arguments: // -// test_suite_name: name of the test suite +// test_suite_name: name of the test suite // name: name of the test -// type_param the name of the test's type parameter, or NULL if +// type_param: the name of the test's type parameter, or NULL if // this is not a typed or a type-parameterized test. -// value_param text representation of the test's value parameter, +// value_param: text representation of the test's value parameter, // or NULL if this is not a type-parameterized test. // code_location: code location where the test is defined // fixture_class_id: ID of the test fixture class // set_up_tc: pointer to the function that sets up the test suite // tear_down_tc: pointer to the function that tears down the test suite @@ -574,12 +588,10 @@ // If *pstr starts with the given prefix, modifies *pstr to be right // past the prefix and returns true; otherwise leaves *pstr unchanged // and returns false. None of pstr, *pstr, and prefix can be NULL. GTEST_API_ bool SkipPrefix(const char* prefix, const char** pstr); -#if GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P - GTEST_DISABLE_MSC_WARNINGS_PUSH_(4251 \ /* class A needs to have dll-interface to be used by clients of class B */) // State of the definition of a type-parameterized test suite. class GTEST_API_ TypedTestSuitePState { @@ -807,12 +819,10 @@ std::vector() /*type_names*/) { return true; } }; -#endif // GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P - // Returns the current OS stack trace as an std::string. // // The maximum number of stack frames to be included is specified by // the gtest_stack_trace_depth flag. The skip_count parameter // specifies the number of top frames to be skipped, which doesn't @@ -876,15 +886,38 @@ // Turns const U&, U&, const U, and U all into U. #define GTEST_REMOVE_REFERENCE_AND_CONST_(T) \ typename std::remove_const::type>::type -// IsAProtocolMessage::value is a compile-time bool constant that's -// true if and only if T is type proto2::Message or a subclass of it. +// HasDebugStringAndShortDebugString::value is a compile-time bool constant +// that's true if and only if T has methods DebugString() and ShortDebugString() +// that return std::string. template -struct IsAProtocolMessage - : public std::is_convertible {}; +class HasDebugStringAndShortDebugString { + private: + template + static auto CheckDebugString(C*) -> typename std::is_same< + std::string, decltype(std::declval().DebugString())>::type; + template + static std::false_type CheckDebugString(...); + + template + static auto CheckShortDebugString(C*) -> typename std::is_same< + std::string, decltype(std::declval().ShortDebugString())>::type; + template + static std::false_type CheckShortDebugString(...); + + using HasDebugStringType = decltype(CheckDebugString(nullptr)); + using HasShortDebugStringType = decltype(CheckShortDebugString(nullptr)); + + public: + static constexpr bool value = + HasDebugStringType::value && HasShortDebugStringType::value; +}; + +template +constexpr bool HasDebugStringAndShortDebugString::value; // When the compiler sees expression IsContainerTest(0), if C is an // STL-style container class, the first overload of IsContainerTest // will be viable (since both C::iterator* and C::const_iterator* are // valid types and NULL can be implicitly converted to them). It will @@ -1116,12 +1149,10 @@ } const Element* array_; size_t size_; void (NativeArray::*clone_)(const Element*, size_t); - - GTEST_DISALLOW_ASSIGN_(NativeArray); }; // Backport of std::index_sequence. template struct IndexSequence { @@ -1141,16 +1172,22 @@ }; // Backport of std::make_index_sequence. // It uses O(ln(N)) instantiation depth. template -struct MakeIndexSequence - : DoubleSequence::type, +struct MakeIndexSequenceImpl + : DoubleSequence::type, N / 2>::type {}; template <> -struct MakeIndexSequence<0> : IndexSequence<> {}; +struct MakeIndexSequenceImpl<0> : IndexSequence<> {}; + +template +using MakeIndexSequence = typename MakeIndexSequenceImpl::type; + +template +using IndexSequenceFor = typename MakeIndexSequence::type; template struct Ignore { Ignore(...); // NOLINT }; @@ -1172,21 +1209,25 @@ using type = decltype(ElemFromListImpl::type>::Apply( static_cast(nullptr)...)); }; +struct FlatTupleConstructTag {}; + template class FlatTuple; template struct FlatTupleElemBase; template struct FlatTupleElemBase, I> { using value_type = typename ElemFromList::type; FlatTupleElemBase() = default; - explicit FlatTupleElemBase(value_type t) : value(std::move(t)) {} + template + explicit FlatTupleElemBase(FlatTupleConstructTag, Arg&& t) + : value(std::forward(t)) {} value_type value; }; template struct FlatTupleBase; @@ -1194,12 +1235,34 @@ template struct FlatTupleBase, IndexSequence> : FlatTupleElemBase, Idx>... { using Indices = IndexSequence; FlatTupleBase() = default; - explicit FlatTupleBase(T... t) - : FlatTupleElemBase, Idx>(std::move(t))... {} + template + explicit FlatTupleBase(FlatTupleConstructTag, Args&&... args) + : FlatTupleElemBase, Idx>(FlatTupleConstructTag{}, + std::forward(args))... {} + + template + const typename ElemFromList::type& Get() const { + return FlatTupleElemBase, I>::value; + } + + template + typename ElemFromList::type& Get() { + return FlatTupleElemBase, I>::value; + } + + template + auto Apply(F&& f) -> decltype(std::forward(f)(this->Get()...)) { + return std::forward(f)(Get()...); + } + + template + auto Apply(F&& f) const -> decltype(std::forward(f)(this->Get()...)) { + return std::forward(f)(Get()...); + } }; // Analog to std::tuple but with different tradeoffs. // This class minimizes the template instantiation depth, thus allowing more // elements than std::tuple would. std::tuple has been seen to require an @@ -1216,21 +1279,16 @@ using Indices = typename FlatTupleBase< FlatTuple, typename MakeIndexSequence::type>::Indices; public: FlatTuple() = default; - explicit FlatTuple(T... t) : FlatTuple::FlatTupleBase(std::move(t)...) {} + template + explicit FlatTuple(FlatTupleConstructTag tag, Args&&... args) + : FlatTuple::FlatTupleBase(tag, std::forward(args)...) {} - template - const typename ElemFromList::type& Get() const { - return static_cast*>(this)->value; - } - - template - typename ElemFromList::type& Get() { - return static_cast*>(this)->value; - } + using FlatTuple::FlatTupleBase::Apply; + using FlatTuple::FlatTupleBase::Get; }; // Utility functions to be called with static_assert to induce deprecation // warnings. GTEST_INTERNAL_DEPRECATED( @@ -1259,10 +1317,26 @@ constexpr bool InstantiateTypedTestCase_P_IsDeprecated() { return true; } } // namespace internal } // namespace testing +namespace std { +// Some standard library implementations use `struct tuple_size` and some use +// `class tuple_size`. Clang warns about the mismatch. +// https://reviews.llvm.org/D55466 +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wmismatched-tags" +#endif +template +struct tuple_size> + : std::integral_constant {}; +#ifdef __clang__ +#pragma clang diagnostic pop +#endif +} // namespace std + #define GTEST_MESSAGE_AT_(file, line, message, result_type) \ ::testing::internal::AssertHelper(result_type, file, line, message) \ = ::testing::Message() #define GTEST_MESSAGE_(message, result_type) \ @@ -1281,48 +1355,102 @@ return GTEST_MESSAGE_(message, ::testing::TestPartResult::kSkip) // Suppress MSVC warning 4072 (unreachable code) for the code following // statement if it returns or throws (or doesn't return or throw in some // situations). +// NOTE: The "else" is important to keep this expansion to prevent a top-level +// "else" from attaching to our "if". #define GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement) \ - if (::testing::internal::AlwaysTrue()) { statement; } - -#define GTEST_TEST_THROW_(statement, expected_exception, fail) \ - GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ - if (::testing::internal::ConstCharPtr gtest_msg = "") { \ - bool gtest_caught_expected = false; \ - try { \ - GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ - } \ - catch (expected_exception const&) { \ - gtest_caught_expected = true; \ - } \ - catch (...) { \ - gtest_msg.value = \ - "Expected: " #statement " throws an exception of type " \ - #expected_exception ".\n Actual: it throws a different type."; \ - goto GTEST_CONCAT_TOKEN_(gtest_label_testthrow_, __LINE__); \ - } \ - if (!gtest_caught_expected) { \ - gtest_msg.value = \ - "Expected: " #statement " throws an exception of type " \ - #expected_exception ".\n Actual: it throws nothing."; \ - goto GTEST_CONCAT_TOKEN_(gtest_label_testthrow_, __LINE__); \ - } \ - } else \ - GTEST_CONCAT_TOKEN_(gtest_label_testthrow_, __LINE__): \ - fail(gtest_msg.value) + if (::testing::internal::AlwaysTrue()) { \ + statement; \ + } else /* NOLINT */ \ + static_assert(true, "") // User must have a semicolon after expansion. #if GTEST_HAS_EXCEPTIONS -#define GTEST_TEST_NO_THROW_CATCH_STD_EXCEPTION_() \ - catch (std::exception const& e) { \ - gtest_msg.value = ( \ - "it throws std::exception-derived exception with description: \"" \ - ); \ - gtest_msg.value += e.what(); \ - gtest_msg.value += "\"."; \ +namespace testing { +namespace internal { + +class NeverThrown { + public: + const char* what() const noexcept { + return "this exception should never be thrown"; + } +}; + +} // namespace internal +} // namespace testing + +#if GTEST_HAS_RTTI + +#define GTEST_EXCEPTION_TYPE_(e) ::testing::internal::GetTypeName(typeid(e)) + +#else // GTEST_HAS_RTTI + +#define GTEST_EXCEPTION_TYPE_(e) \ + std::string { "an std::exception-derived error" } + +#endif // GTEST_HAS_RTTI + +#define GTEST_TEST_THROW_CATCH_STD_EXCEPTION_(statement, expected_exception) \ + catch (typename std::conditional< \ + std::is_same::type>::type, \ + std::exception>::value, \ + const ::testing::internal::NeverThrown&, const std::exception&>::type \ + e) { \ + gtest_msg.value = "Expected: " #statement \ + " throws an exception of type " #expected_exception \ + ".\n Actual: it throws "; \ + gtest_msg.value += GTEST_EXCEPTION_TYPE_(e); \ + gtest_msg.value += " with description \""; \ + gtest_msg.value += e.what(); \ + gtest_msg.value += "\"."; \ + goto GTEST_CONCAT_TOKEN_(gtest_label_testthrow_, __LINE__); \ + } + +#else // GTEST_HAS_EXCEPTIONS + +#define GTEST_TEST_THROW_CATCH_STD_EXCEPTION_(statement, expected_exception) + +#endif // GTEST_HAS_EXCEPTIONS + +#define GTEST_TEST_THROW_(statement, expected_exception, fail) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::TrueWithString gtest_msg{}) { \ + bool gtest_caught_expected = false; \ + try { \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + } catch (expected_exception const&) { \ + gtest_caught_expected = true; \ + } \ + GTEST_TEST_THROW_CATCH_STD_EXCEPTION_(statement, expected_exception) \ + catch (...) { \ + gtest_msg.value = "Expected: " #statement \ + " throws an exception of type " #expected_exception \ + ".\n Actual: it throws a different type."; \ + goto GTEST_CONCAT_TOKEN_(gtest_label_testthrow_, __LINE__); \ + } \ + if (!gtest_caught_expected) { \ + gtest_msg.value = "Expected: " #statement \ + " throws an exception of type " #expected_exception \ + ".\n Actual: it throws nothing."; \ + goto GTEST_CONCAT_TOKEN_(gtest_label_testthrow_, __LINE__); \ + } \ + } else /*NOLINT*/ \ + GTEST_CONCAT_TOKEN_(gtest_label_testthrow_, __LINE__) \ + : fail(gtest_msg.value.c_str()) + +#if GTEST_HAS_EXCEPTIONS + +#define GTEST_TEST_NO_THROW_CATCH_STD_EXCEPTION_() \ + catch (std::exception const& e) { \ + gtest_msg.value = "it throws "; \ + gtest_msg.value += GTEST_EXCEPTION_TYPE_(e); \ + gtest_msg.value += " with description \""; \ + gtest_msg.value += e.what(); \ + gtest_msg.value += "\"."; \ goto GTEST_CONCAT_TOKEN_(gtest_label_testnothrow_, __LINE__); \ } #else // GTEST_HAS_EXCEPTIONS @@ -1365,11 +1493,11 @@ " Actual: it doesn't.") // Implements Boolean test assertions such as EXPECT_TRUE. expression can be // either a boolean expression or an AssertionResult. text is a textual -// represenation of expression as it was passed into the EXPECT_TRUE. +// representation of expression as it was passed into the EXPECT_TRUE. #define GTEST_TEST_BOOLEAN_(expression, text, actual, expected, fail) \ GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ if (const ::testing::AssertionResult gtest_ar_ = \ ::testing::AssertionResult(expression)) \ ; \ @@ -1402,11 +1530,11 @@ static_assert(sizeof(GTEST_STRINGIFY_(test_name)) > 1, \ "test_name must not be empty"); \ class GTEST_TEST_CLASS_NAME_(test_suite_name, test_name) \ : public parent_class { \ public: \ - GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)() {} \ + GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)() = default; \ ~GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)() override = default; \ GTEST_DISALLOW_COPY_AND_ASSIGN_(GTEST_TEST_CLASS_NAME_(test_suite_name, \ test_name)); \ GTEST_DISALLOW_MOVE_AND_ASSIGN_(GTEST_TEST_CLASS_NAME_(test_suite_name, \ test_name)); \ @@ -1427,6 +1555,6 @@ parent_class>::GetTearDownCaseOrSuite(__FILE__, __LINE__), \ new ::testing::internal::TestFactoryImpl); \ void GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)::TestBody() -#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-param-util.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-param-util.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-param-util.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-param-util.h @@ -30,12 +30,12 @@ // Type and function utilities for implementing parameterized tests. // GOOGLETEST_CM0001 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_H_ -#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_H_ +#define GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_H_ #include #include #include @@ -457,11 +457,11 @@ public: virtual ~ParameterizedTestSuiteInfoBase() {} // Base part of test suite name for display purposes. virtual const std::string& GetTestSuiteName() const = 0; - // Test case id to verify identity. + // Test suite id to verify identity. virtual TypeId GetTestSuiteTypeId() const = 0; // UnitTest class invokes this method to register tests in this // test suite right before running them in RUN_ALL_TESTS macro. // This method should not be called more than once on any single // instance of a ParameterizedTestSuiteInfoBase derived class. @@ -476,11 +476,11 @@ // INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. // // Report a the name of a test_suit as safe to ignore // as the side effect of construction of this type. -struct MarkAsIgnored { +struct GTEST_API_ MarkAsIgnored { explicit MarkAsIgnored(const char* test_suite); }; GTEST_API_ void InsertSyntheticTestCase(const std::string& name, CodeLocation location, bool has_test_p); @@ -505,26 +505,27 @@ explicit ParameterizedTestSuiteInfo(const char* name, CodeLocation code_location) : test_suite_name_(name), code_location_(code_location) {} - // Test case base name for display purposes. + // Test suite base name for display purposes. const std::string& GetTestSuiteName() const override { return test_suite_name_; } - // Test case id to verify identity. + // Test suite id to verify identity. TypeId GetTestSuiteTypeId() const override { return GetTypeId(); } // TEST_P macro uses AddTestPattern() to record information // about a single test in a LocalTestInfo structure. // test_suite_name is the base name of the test suite (without invocation // prefix). test_base_name is the name of an individual test without // parameter index. For the test SequenceA/FooTest.DoBar/1 FooTest is // test suite base name and DoBar is test base name. void AddTestPattern(const char* test_suite_name, const char* test_base_name, - TestMetaFactoryBase* meta_factory) { - tests_.push_back(std::shared_ptr( - new TestInfo(test_suite_name, test_base_name, meta_factory))); + TestMetaFactoryBase* meta_factory, + CodeLocation code_location) { + tests_.push_back(std::shared_ptr(new TestInfo( + test_suite_name, test_base_name, meta_factory, code_location))); } // INSTANTIATE_TEST_SUITE_P macro uses AddGenerator() to record information // about a generator. int AddTestSuiteInstantiation(const std::string& instantiation_name, GeneratorCreationFunc* func, @@ -587,11 +588,11 @@ } test_name_stream << param_name; MakeAndRegisterTestInfo( test_suite_name.c_str(), test_name_stream.GetString().c_str(), nullptr, // No type parameter. - PrintToString(*param_it).c_str(), code_location_, + PrintToString(*param_it).c_str(), test_info->code_location, GetTestSuiteTypeId(), SuiteApiResolver::GetSetUpCaseOrSuite(file, line), SuiteApiResolver::GetTearDownCaseOrSuite(file, line), test_info->test_meta_factory->CreateTestFactory(*param_it)); } // for param_it @@ -608,18 +609,21 @@ private: // LocalTestInfo structure keeps information about a single test registered // with TEST_P macro. struct TestInfo { TestInfo(const char* a_test_suite_base_name, const char* a_test_base_name, - TestMetaFactoryBase* a_test_meta_factory) + TestMetaFactoryBase* a_test_meta_factory, + CodeLocation a_code_location) : test_suite_base_name(a_test_suite_base_name), test_base_name(a_test_base_name), - test_meta_factory(a_test_meta_factory) {} + test_meta_factory(a_test_meta_factory), + code_location(a_code_location) {} const std::string test_suite_base_name; const std::string test_base_name; const std::unique_ptr > test_meta_factory; + const CodeLocation code_location; }; using TestInfoContainer = ::std::vector >; // Records data received from INSTANTIATE_TEST_SUITE_P macros: // @@ -648,11 +652,11 @@ if (name.empty()) return false; // Check for invalid characters for (std::string::size_type index = 0; index < name.size(); ++index) { - if (!isalnum(name[index]) && name[index] != '_') + if (!IsAlNum(name[index]) && name[index] != '_') return false; } return true; } @@ -777,14 +781,19 @@ const Container& container); namespace internal { // Used in the Values() function to provide polymorphic capabilities. +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4100) +#endif + template class ValueArray { public: - ValueArray(Ts... v) : v_{std::move(v)...} {} + explicit ValueArray(Ts... v) : v_(FlatTupleConstructTag{}, std::move(v)...) {} template operator ParamGenerator() const { // NOLINT return ValuesIn(MakeVector(MakeIndexSequence())); } @@ -796,10 +805,14 @@ } FlatTuple v_; }; +#ifdef _MSC_VER +#pragma warning(pop) +#endif + template class CartesianProductGenerator : public ParamGeneratorInterface<::std::tuple> { public: typedef ::std::tuple ParamType; @@ -929,6 +942,6 @@ }; } // namespace internal } // namespace testing -#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-port-arch.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-port-arch.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-port-arch.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-port-arch.h @@ -30,12 +30,12 @@ // The Google C++ Testing and Mocking Framework (Google Test) // // This header file defines the GTEST_OS_* macro. // It is separate from gtest-port.h so that custom/gtest-port.h can include it. -#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_ARCH_H_ -#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_ARCH_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_ARCH_H_ +#define GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_ARCH_H_ // Determines the platform on which Google Test is compiled. #ifdef __CYGWIN__ # define GTEST_OS_CYGWIN 1 # elif defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__) @@ -66,10 +66,11 @@ # endif // _WIN32_WCE #elif defined __OS2__ # define GTEST_OS_OS2 1 #elif defined __APPLE__ # define GTEST_OS_MAC 1 +# include # if TARGET_OS_IPHONE # define GTEST_OS_IOS 1 # endif #elif defined __DragonFly__ # define GTEST_OS_DRAGONFLY 1 @@ -104,8 +105,10 @@ #define GTEST_OS_HAIKU 1 #elif defined ESP8266 #define GTEST_OS_ESP8266 1 #elif defined ESP32 #define GTEST_OS_ESP32 1 +#elif defined(__XTENSA__) +#define GTEST_OS_XTENSA 1 #endif // __CYGWIN__ -#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_ARCH_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_ARCH_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-port.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-port.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-port.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-port.h @@ -38,12 +38,12 @@ // files are expected to #include this. Therefore, it cannot #include // any other Google Test header. // GOOGLETEST_CM0001 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_ -#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_ +#define GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_ // Environment-describing macros // ----------------------------- // // Google Test can be used in many different environments. Macros in @@ -197,13 +197,22 @@ // GTEST_MUST_USE_RESULT_ - declares that a function's result must be used. // GTEST_INTENTIONAL_CONST_COND_PUSH_ - start code section where MSVC C4127 is // suppressed (constant conditional). // GTEST_INTENTIONAL_CONST_COND_POP_ - finish code section where MSVC C4127 // is suppressed. +// GTEST_INTERNAL_HAS_ANY - for enabling UniversalPrinter or +// UniversalPrinter specializations. +// GTEST_INTERNAL_HAS_OPTIONAL - for enabling UniversalPrinter +// or +// UniversalPrinter +// specializations. // GTEST_INTERNAL_HAS_STRING_VIEW - for enabling Matcher or // Matcher // specializations. +// GTEST_INTERNAL_HAS_VARIANT - for enabling UniversalPrinter or +// UniversalPrinter +// specializations. // // Synchronization: // Mutex, MutexLock, ThreadLocal, GetThreadCount() // - synchronization primitives. // @@ -250,10 +259,12 @@ #include // for isspace, etc #include // for ptrdiff_t #include #include #include + +#include #include #include #include #ifndef _WIN32_WCE @@ -265,10 +276,11 @@ # include # include #endif #include // NOLINT +#include #include #include // NOLINT #include #include // NOLINT @@ -345,10 +357,14 @@ // Assume CRITICAL_SECTION is a typedef of _RTL_CRITICAL_SECTION. // This assumption is verified by // WindowsTypesTest.CRITICAL_SECTIONIs_RTL_CRITICAL_SECTION. typedef struct _RTL_CRITICAL_SECTION GTEST_CRITICAL_SECTION; #endif +#elif GTEST_OS_XTENSA +#include +// Xtensa toolchains define strcasecmp in the string.h header instead of +// strings.h. string.h is already included. #else // This assumes that non-Windows OSes provide unistd.h. For OSes where this // is not the case, we need to include headers that provide the functions // mentioned above. # include @@ -365,11 +381,11 @@ #ifndef GTEST_HAS_POSIX_RE # if GTEST_OS_LINUX_ANDROID // On Android, is only available starting with Gingerbread. # define GTEST_HAS_POSIX_RE (__ANDROID_API__ >= 9) # else -# define GTEST_HAS_POSIX_RE (!GTEST_OS_WINDOWS) +#define GTEST_HAS_POSIX_RE (!GTEST_OS_WINDOWS && !GTEST_OS_XTENSA) # endif #endif #if GTEST_USES_PCRE // The appropriate headers have already been included. @@ -450,11 +466,11 @@ // Cygwin 1.7 and below doesn't support ::std::wstring. // Solaris' libc++ doesn't support it either. Android has // no support for it at least as recent as Froyo (2.2). #define GTEST_HAS_STD_WSTRING \ (!(GTEST_OS_LINUX_ANDROID || GTEST_OS_CYGWIN || GTEST_OS_SOLARIS || \ - GTEST_OS_HAIKU || GTEST_OS_ESP32 || GTEST_OS_ESP8266)) + GTEST_OS_HAIKU || GTEST_OS_ESP32 || GTEST_OS_ESP8266 || GTEST_OS_XTENSA)) #endif // GTEST_HAS_STD_WSTRING // Determines whether RTTI is available. #ifndef GTEST_HAS_RTTI @@ -575,11 +591,11 @@ // output correctness and to implement death tests. #ifndef GTEST_HAS_STREAM_REDIRECTION // By default, we assume that stream redirection is supported on all // platforms except known mobile ones. #if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_WINDOWS_PHONE || \ - GTEST_OS_WINDOWS_RT || GTEST_OS_ESP8266 + GTEST_OS_WINDOWS_RT || GTEST_OS_ESP8266 || GTEST_OS_XTENSA # define GTEST_HAS_STREAM_REDIRECTION 0 # else # define GTEST_HAS_STREAM_REDIRECTION 1 # endif // !GTEST_OS_WINDOWS_MOBILE #endif // GTEST_HAS_STREAM_REDIRECTION @@ -677,23 +693,23 @@ type& operator=(type const &) = delete // A macro to disallow copy constructor and operator= // This should be used in the private: declarations for a class. #define GTEST_DISALLOW_COPY_AND_ASSIGN_(type) \ - type(type const &) = delete; \ - GTEST_DISALLOW_ASSIGN_(type) + type(type const&) = delete; \ + type& operator=(type const&) = delete // A macro to disallow move operator= // This should be used in the private: declarations for a class. #define GTEST_DISALLOW_MOVE_ASSIGN_(type) \ type& operator=(type &&) noexcept = delete // A macro to disallow move constructor and operator= // This should be used in the private: declarations for a class. #define GTEST_DISALLOW_MOVE_AND_ASSIGN_(type) \ - type(type &&) noexcept = delete; \ - GTEST_DISALLOW_MOVE_ASSIGN_(type) + type(type&&) noexcept = delete; \ + type& operator=(type&&) noexcept = delete // Tell the compiler to warn about unused return values for functions declared // with this macro. The macro should be used on function declarations // following the argument list: // @@ -916,12 +932,10 @@ # else // GTEST_USES_SIMPLE_RE const char* full_pattern_; // For FullMatch(); # endif - - GTEST_DISALLOW_ASSIGN_(RE); }; #endif // GTEST_USES_PCRE // Formats a source file path and a line number as they would appear @@ -1924,10 +1938,23 @@ return isupper(static_cast(ch)) != 0; } inline bool IsXDigit(char ch) { return isxdigit(static_cast(ch)) != 0; } +#ifdef __cpp_char8_t +inline bool IsXDigit(char8_t ch) { + return isxdigit(static_cast(ch)) != 0; +} +#endif +inline bool IsXDigit(char16_t ch) { + const unsigned char low_byte = static_cast(ch); + return ch == low_byte && isxdigit(low_byte) != 0; +} +inline bool IsXDigit(char32_t ch) { + const unsigned char low_byte = static_cast(ch); + return ch == low_byte && isxdigit(low_byte) != 0; +} inline bool IsXDigit(wchar_t ch) { const unsigned char low_byte = static_cast(ch); return ch == low_byte && isxdigit(low_byte) != 0; } @@ -1958,20 +1985,20 @@ #if GTEST_OS_WINDOWS typedef struct _stat StatStruct; # ifdef __BORLANDC__ -inline int IsATTY(int fd) { return isatty(fd); } +inline int DoIsATTY(int fd) { return isatty(fd); } inline int StrCaseCmp(const char* s1, const char* s2) { return stricmp(s1, s2); } inline char* StrDup(const char* src) { return strdup(src); } # else // !__BORLANDC__ # if GTEST_OS_WINDOWS_MOBILE -inline int IsATTY(int /* fd */) { return 0; } +inline int DoIsATTY(int /* fd */) { return 0; } # else -inline int IsATTY(int fd) { return _isatty(fd); } +inline int DoIsATTY(int fd) { return _isatty(fd); } # endif // GTEST_OS_WINDOWS_MOBILE inline int StrCaseCmp(const char* s1, const char* s2) { return _stricmp(s1, s2); } inline char* StrDup(const char* src) { return _strdup(src); } @@ -1992,11 +2019,11 @@ #elif GTEST_OS_ESP8266 typedef struct stat StatStruct; inline int FileNo(FILE* file) { return fileno(file); } -inline int IsATTY(int fd) { return isatty(fd); } +inline int DoIsATTY(int fd) { return isatty(fd); } inline int Stat(const char* path, StatStruct* buf) { // stat function not implemented on ESP8266 return 0; } inline int StrCaseCmp(const char* s1, const char* s2) { @@ -2009,34 +2036,54 @@ #else typedef struct stat StatStruct; inline int FileNo(FILE* file) { return fileno(file); } -inline int IsATTY(int fd) { return isatty(fd); } +inline int DoIsATTY(int fd) { return isatty(fd); } inline int Stat(const char* path, StatStruct* buf) { return stat(path, buf); } inline int StrCaseCmp(const char* s1, const char* s2) { return strcasecmp(s1, s2); } inline char* StrDup(const char* src) { return strdup(src); } inline int RmDir(const char* dir) { return rmdir(dir); } inline bool IsDir(const StatStruct& st) { return S_ISDIR(st.st_mode); } #endif // GTEST_OS_WINDOWS +inline int IsATTY(int fd) { + // DoIsATTY might change errno (for example ENOTTY in case you redirect stdout + // to a file on Linux), which is unexpected, so save the previous value, and + // restore it after the call. + int savedErrno = errno; + int isAttyValue = DoIsATTY(fd); + errno = savedErrno; + + return isAttyValue; +} + // Functions deprecated by MSVC 8.0. GTEST_DISABLE_MSC_DEPRECATED_PUSH_() // ChDir(), FReopen(), FDOpen(), Read(), Write(), Close(), and // StrError() aren't needed on Windows CE at this time and thus not // defined there. -#if !GTEST_OS_WINDOWS_MOBILE && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT +#if !GTEST_OS_WINDOWS_MOBILE && !GTEST_OS_WINDOWS_PHONE && \ + !GTEST_OS_WINDOWS_RT && !GTEST_OS_ESP8266 && !GTEST_OS_XTENSA inline int ChDir(const char* dir) { return chdir(dir); } #endif inline FILE* FOpen(const char* path, const char* mode) { +#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MINGW + struct wchar_codecvt : public std::codecvt {}; + std::wstring_convert converter; + std::wstring wide_path = converter.from_bytes(path); + std::wstring wide_mode = converter.from_bytes(mode); + return _wfopen(wide_path.c_str(), wide_mode.c_str()); +#else // GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MINGW return fopen(path, mode); +#endif // GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MINGW } #if !GTEST_OS_WINDOWS_MOBILE inline FILE *FReopen(const char* path, const char* mode, FILE* stream) { return freopen(path, mode, stream); } @@ -2053,11 +2100,11 @@ inline int Close(int fd) { return close(fd); } inline const char* StrError(int errnum) { return strerror(errnum); } #endif inline const char* GetEnv(const char* name) { #if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_WINDOWS_PHONE || \ - GTEST_OS_WINDOWS_RT || GTEST_OS_ESP8266 + GTEST_OS_WINDOWS_RT || GTEST_OS_ESP8266 || GTEST_OS_XTENSA // We are on an embedded platform, which has no environment variables. static_cast(name); // To prevent 'unused argument' warning. return nullptr; #elif defined(__BORLANDC__) || defined(__SunOS_5_8) || defined(__SunOS_5_9) // Environment variables which we programmatically clear will be set to the @@ -2189,11 +2236,12 @@ #endif // !defined(GTEST_EXCLUSIVE_LOCK_REQUIRED_) // Parses 'str' for a 32-bit signed integer. If successful, writes the result // to *value and returns true; otherwise leaves *value unchanged and returns // false. -bool ParseInt32(const Message& src_text, const char* str, int32_t* value); +GTEST_API_ bool ParseInt32(const Message& src_text, const char* str, + int32_t* value); // Parses a bool/int32_t/string from the environment variable // corresponding to the given Google Test flag. bool BoolFromGTestEnv(const char* flag, bool default_val); GTEST_API_ int32_t Int32FromGTestEnv(const char* flag, int32_t default_val); @@ -2222,10 +2270,68 @@ #endif #endif // !defined(GTEST_INTERNAL_DEPRECATED) #if GTEST_HAS_ABSL +// Always use absl::any for UniversalPrinter<> specializations if googletest +// is built with absl support. +#define GTEST_INTERNAL_HAS_ANY 1 +#include "absl/types/any.h" +namespace testing { +namespace internal { +using Any = ::absl::any; +} // namespace internal +} // namespace testing +#else +#ifdef __has_include +#if __has_include() && __cplusplus >= 201703L +// Otherwise for C++17 and higher use std::any for UniversalPrinter<> +// specializations. +#define GTEST_INTERNAL_HAS_ANY 1 +#include +namespace testing { +namespace internal { +using Any = ::std::any; +} // namespace internal +} // namespace testing +// The case where absl is configured NOT to alias std::any is not +// supported. +#endif // __has_include() && __cplusplus >= 201703L +#endif // __has_include +#endif // GTEST_HAS_ABSL + +#if GTEST_HAS_ABSL +// Always use absl::optional for UniversalPrinter<> specializations if +// googletest is built with absl support. +#define GTEST_INTERNAL_HAS_OPTIONAL 1 +#include "absl/types/optional.h" +namespace testing { +namespace internal { +template +using Optional = ::absl::optional; +} // namespace internal +} // namespace testing +#else +#ifdef __has_include +#if __has_include() && __cplusplus >= 201703L +// Otherwise for C++17 and higher use std::optional for UniversalPrinter<> +// specializations. +#define GTEST_INTERNAL_HAS_OPTIONAL 1 +#include +namespace testing { +namespace internal { +template +using Optional = ::std::optional; +} // namespace internal +} // namespace testing +// The case where absl is configured NOT to alias std::optional is not +// supported. +#endif // __has_include() && __cplusplus >= 201703L +#endif // __has_include +#endif // GTEST_HAS_ABSL + +#if GTEST_HAS_ABSL // Always use absl::string_view for Matcher<> specializations if googletest // is built with absl support. # define GTEST_INTERNAL_HAS_STRING_VIEW 1 #include "absl/strings/string_view.h" namespace testing { @@ -2249,6 +2355,35 @@ // supported. # endif // __has_include() && __cplusplus >= 201703L # endif // __has_include #endif // GTEST_HAS_ABSL -#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_ +#if GTEST_HAS_ABSL +// Always use absl::variant for UniversalPrinter<> specializations if googletest +// is built with absl support. +#define GTEST_INTERNAL_HAS_VARIANT 1 +#include "absl/types/variant.h" +namespace testing { +namespace internal { +template +using Variant = ::absl::variant; +} // namespace internal +} // namespace testing +#else +#ifdef __has_include +#if __has_include() && __cplusplus >= 201703L +// Otherwise for C++17 and higher use std::variant for UniversalPrinter<> +// specializations. +#define GTEST_INTERNAL_HAS_VARIANT 1 +#include +namespace testing { +namespace internal { +template +using Variant = ::std::variant; +} // namespace internal +} // namespace testing +// The case where absl is configured NOT to alias std::variant is not supported. +#endif // __has_include() && __cplusplus >= 201703L +#endif // __has_include +#endif // GTEST_HAS_ABSL + +#endif // GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-string.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-string.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-string.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-string.h @@ -36,12 +36,12 @@ // This header file is #included by gtest-internal.h. // It should not be #included by other files. // GOOGLETEST_CM0001 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_STRING_H_ -#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_STRING_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_STRING_H_ +#define GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_STRING_H_ #ifdef __BORLANDC__ // string.h is not guaranteed to provide strcpy on C++ Builder. # include #endif @@ -147,10 +147,13 @@ const std::string& str, const std::string& suffix); // Formats an int value as "%02d". static std::string FormatIntWidth2(int value); // "%02d" for width == 2 + // Formats an int value to given width with leading zeros. + static std::string FormatIntWidthN(int value, int width); + // Formats an int value as "%X". static std::string FormatHexInt(int value); // Formats an int value as "%X". static std::string FormatHexUInt32(uint32_t value); @@ -167,6 +170,6 @@ GTEST_API_ std::string StringStreamToString(::std::stringstream* stream); } // namespace internal } // namespace testing -#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_STRING_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_STRING_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-type-util.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-type-util.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-type-util.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-type-util.h @@ -30,12 +30,12 @@ // Type utilities needed for implementing typed and type-parameterized // tests. // GOOGLETEST_CM0001 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ -#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ +#define GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ #include "gtest/internal/gtest-port.h" // #ifdef __GNUC__ is too general here. It is possible to use gcc without using // libstdc++ (which is where cxxabi.h comes from). @@ -62,42 +62,44 @@ } } return s; } -// GetTypeName() returns a human-readable name of type T. +#if GTEST_HAS_RTTI +// GetTypeName(const std::type_info&) returns a human-readable name of type T. +inline std::string GetTypeName(const std::type_info& type) { + const char* const name = type.name(); +#if GTEST_HAS_CXXABI_H_ || defined(__HP_aCC) + int status = 0; + // gcc's implementation of typeid(T).name() mangles the type name, + // so we have to demangle it. +#if GTEST_HAS_CXXABI_H_ + using abi::__cxa_demangle; +#endif // GTEST_HAS_CXXABI_H_ + char* const readable_name = __cxa_demangle(name, nullptr, nullptr, &status); + const std::string name_str(status == 0 ? readable_name : name); + free(readable_name); + return CanonicalizeForStdLibVersioning(name_str); +#else + return name; +#endif // GTEST_HAS_CXXABI_H_ || __HP_aCC +} +#endif // GTEST_HAS_RTTI + +// GetTypeName() returns a human-readable name of type T if and only if +// RTTI is enabled, otherwise it returns a dummy type name. // NB: This function is also used in Google Mock, so don't move it inside of // the typed-test-only section below. template std::string GetTypeName() { -# if GTEST_HAS_RTTI - - const char* const name = typeid(T).name(); -# if GTEST_HAS_CXXABI_H_ || defined(__HP_aCC) - int status = 0; - // gcc's implementation of typeid(T).name() mangles the type name, - // so we have to demangle it. -# if GTEST_HAS_CXXABI_H_ - using abi::__cxa_demangle; -# endif // GTEST_HAS_CXXABI_H_ - char* const readable_name = __cxa_demangle(name, nullptr, nullptr, &status); - const std::string name_str(status == 0 ? readable_name : name); - free(readable_name); - return CanonicalizeForStdLibVersioning(name_str); -# else - return name; -# endif // GTEST_HAS_CXXABI_H_ || __HP_aCC - -# else - +#if GTEST_HAS_RTTI + return GetTypeName(typeid(T)); +#else return ""; - -# endif // GTEST_HAS_RTTI +#endif // GTEST_HAS_RTTI } -#if GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P - // A unique type indicating an empty node struct None {}; # define GTEST_TEMPLATE_ template class @@ -169,15 +171,13 @@ public: using type = typename proxy::type; }; -#endif // GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P - } // namespace internal template using Types = internal::ProxyTypeList; } // namespace testing -#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/src/gtest-death-test.cc b/media/libvpx/libvpx/third_party/googletest/src/src/gtest-death-test.cc --- a/media/libvpx/libvpx/third_party/googletest/src/src/gtest-death-test.cc +++ b/media/libvpx/libvpx/third_party/googletest/src/src/gtest-death-test.cc @@ -30,10 +30,11 @@ // // This file implements death tests. #include "gtest/gtest-death-test.h" +#include #include #include "gtest/internal/gtest-port.h" #include "gtest/internal/custom/gtest.h" @@ -245,11 +246,11 @@ msg << "couldn't detect the number of threads."; } else { msg << "detected " << thread_count << " threads."; } msg << " See " - "https://github.com/google/googletest/blob/master/googletest/docs/" + "https://github.com/google/googletest/blob/master/docs/" "advanced.md#death-tests-and-threads" << " for more explanation and suggested solutions, especially if" << " this is the last message you see before your test times out."; return msg.GetString(); } @@ -862,11 +863,11 @@ char* const* Argv() { return &args_[0]; } int size() { - return args_.size() - 1; + return static_cast(args_.size()) - 1; } private: std::vector args_; }; @@ -888,22 +889,21 @@ status_zx = zx::port::create(0, &port); GTEST_DEATH_TEST_CHECK_(status_zx == ZX_OK); // Register to wait for the child process to terminate. status_zx = child_process_.wait_async( - port, kProcessKey, ZX_PROCESS_TERMINATED, ZX_WAIT_ASYNC_ONCE); + port, kProcessKey, ZX_PROCESS_TERMINATED, 0); GTEST_DEATH_TEST_CHECK_(status_zx == ZX_OK); // Register to wait for the socket to be readable or closed. status_zx = stderr_socket_.wait_async( - port, kSocketKey, ZX_SOCKET_READABLE | ZX_SOCKET_PEER_CLOSED, - ZX_WAIT_ASYNC_ONCE); + port, kSocketKey, ZX_SOCKET_READABLE | ZX_SOCKET_PEER_CLOSED, 0); GTEST_DEATH_TEST_CHECK_(status_zx == ZX_OK); // Register to wait for an exception. status_zx = exception_channel_.wait_async( - port, kExceptionKey, ZX_CHANNEL_READABLE, ZX_WAIT_ASYNC_ONCE); + port, kExceptionKey, ZX_CHANNEL_READABLE, 0); GTEST_DEATH_TEST_CHECK_(status_zx == ZX_OK); bool process_terminated = false; bool socket_closed = false; do { @@ -939,12 +939,11 @@ if (status_zx == ZX_ERR_PEER_CLOSED) { socket_closed = true; } else { GTEST_DEATH_TEST_CHECK_(status_zx == ZX_ERR_SHOULD_WAIT); status_zx = stderr_socket_.wait_async( - port, kSocketKey, ZX_SOCKET_READABLE | ZX_SOCKET_PEER_CLOSED, - ZX_WAIT_ASYNC_ONCE); + port, kSocketKey, ZX_SOCKET_READABLE | ZX_SOCKET_PEER_CLOSED, 0); GTEST_DEATH_TEST_CHECK_(status_zx == ZX_OK); } } else { GTEST_DEATH_TEST_CHECK_(packet.signal.observed & ZX_SOCKET_PEER_CLOSED); socket_closed = true; @@ -953,16 +952,16 @@ } while (!process_terminated && !socket_closed); ReadAndInterpretStatusByte(); zx_info_process_t buffer; - status_zx = child_process_.get_info( - ZX_INFO_PROCESS, &buffer, sizeof(buffer), nullptr, nullptr); + status_zx = child_process_.get_info(ZX_INFO_PROCESS, &buffer, sizeof(buffer), + nullptr, nullptr); GTEST_DEATH_TEST_CHECK_(status_zx == ZX_OK); - GTEST_DEATH_TEST_CHECK_(buffer.exited); - set_status(buffer.return_code); + GTEST_DEATH_TEST_CHECK_(buffer.flags & ZX_INFO_PROCESS_FLAG_EXITED); + set_status(static_cast(buffer.return_code)); return status(); } // The AssumeRole process for a Fuchsia death test. It creates a child // process with the same executable as the current process to run the @@ -1223,25 +1222,13 @@ struct ExecDeathTestArgs { char* const* argv; // Command-line arguments for the child's call to exec int close_fd; // File descriptor to close; the read end of a pipe }; -# if GTEST_OS_MAC -inline char** GetEnviron() { - // When Google Test is built as a framework on MacOS X, the environ variable - // is unavailable. Apple's documentation (man environ) recommends using - // _NSGetEnviron() instead. - return *_NSGetEnviron(); -} -# else -// Some POSIX platforms expect you to declare environ. extern "C" makes -// it reside in the global namespace. +# if GTEST_OS_QNX extern "C" char** environ; -inline char** GetEnviron() { return environ; } -# endif // GTEST_OS_MAC - -# if !GTEST_OS_QNX +# else // GTEST_OS_QNX // The main function for a threadsafe-style death test child process. // This function is called in a clone()-ed process and thus must avoid // any potentially unsafe operations like malloc or libc functions. static int ExecDeathTestChildMain(void* child_arg) { ExecDeathTestArgs* const args = static_cast(child_arg); @@ -1257,22 +1244,22 @@ DeathTestAbort(std::string("chdir(\"") + original_dir + "\") failed: " + GetLastErrnoDescription()); return EXIT_FAILURE; } - // We can safely call execve() as it's a direct system call. We + // We can safely call execv() as it's almost a direct system call. We // cannot use execvp() as it's a libc function and thus potentially - // unsafe. Since execve() doesn't search the PATH, the user must + // unsafe. Since execv() doesn't search the PATH, the user must // invoke the test program via a valid path that contains at least // one path separator. - execve(args->argv[0], args->argv, GetEnviron()); - DeathTestAbort(std::string("execve(") + args->argv[0] + ", ...) in " + + execv(args->argv[0], args->argv); + DeathTestAbort(std::string("execv(") + args->argv[0] + ", ...) in " + original_dir + " failed: " + GetLastErrnoDescription()); return EXIT_FAILURE; } -# endif // !GTEST_OS_QNX +# endif // GTEST_OS_QNX # if GTEST_HAS_CLONE // Two utility routines that together determine the direction the stack // grows. // This could be accomplished more elegantly by a single recursive @@ -1282,23 +1269,28 @@ // GTEST_NO_INLINE_ is required to prevent GCC 4.6 from inlining // StackLowerThanAddress into StackGrowsDown, which then doesn't give // correct answer. static void StackLowerThanAddress(const void* ptr, bool* result) GTEST_NO_INLINE_; +// Make sure sanitizers do not tamper with the stack here. +// Ideally, we want to use `__builtin_frame_address` instead of a local variable +// address with sanitizer disabled, but it does not work when the +// compiler optimizes the stack frame out, which happens on PowerPC targets. // HWAddressSanitizer add a random tag to the MSB of the local variable address, // making comparison result unpredictable. +GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ GTEST_ATTRIBUTE_NO_SANITIZE_HWADDRESS_ static void StackLowerThanAddress(const void* ptr, bool* result) { - int dummy; - *result = (&dummy < ptr); + int dummy = 0; + *result = std::less()(&dummy, ptr); } // Make sure AddressSanitizer does not tamper with the stack here. GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ GTEST_ATTRIBUTE_NO_SANITIZE_HWADDRESS_ static bool StackGrowsDown() { - int dummy; + int dummy = 0; bool result; StackLowerThanAddress(&dummy, &result); return result; } # endif // GTEST_HAS_CLONE @@ -1337,12 +1329,11 @@ GTEST_DEATH_TEST_CHECK_SYSCALL_(fd_flags = fcntl(close_fd, F_GETFD)); GTEST_DEATH_TEST_CHECK_SYSCALL_(fcntl(close_fd, F_SETFD, fd_flags | FD_CLOEXEC)); struct inheritance inherit = {0}; // spawn is a system call. - child_pid = - spawn(args.argv[0], 0, nullptr, &inherit, args.argv, GetEnviron()); + child_pid = spawn(args.argv[0], 0, nullptr, &inherit, args.argv, environ); // Restores the current working directory. GTEST_DEATH_TEST_CHECK_(fchdir(cwd_fd) != -1); GTEST_DEATH_TEST_CHECK_SYSCALL_(close(cwd_fd)); # else // GTEST_OS_QNX diff --git a/media/libvpx/libvpx/third_party/googletest/src/src/gtest-filepath.cc b/media/libvpx/libvpx/third_party/googletest/src/src/gtest-filepath.cc --- a/media/libvpx/libvpx/third_party/googletest/src/src/gtest-filepath.cc +++ b/media/libvpx/libvpx/third_party/googletest/src/src/gtest-filepath.cc @@ -90,12 +90,13 @@ #endif } // Returns the current working directory, or "" if unsuccessful. FilePath FilePath::GetCurrentDir() { -#if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_WINDOWS_PHONE || \ - GTEST_OS_WINDOWS_RT || GTEST_OS_ESP8266 || GTEST_OS_ESP32 +#if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_WINDOWS_PHONE || \ + GTEST_OS_WINDOWS_RT || GTEST_OS_ESP8266 || GTEST_OS_ESP32 || \ + GTEST_OS_XTENSA // These platforms do not have a current directory, so we just return // something reasonable. return FilePath(kCurrentDirectoryString); #elif GTEST_OS_WINDOWS char cwd[GTEST_PATH_MAX_ + 1] = { '\0' }; @@ -207,11 +208,11 @@ LPCWSTR unicode = String::AnsiToUtf16(pathname_.c_str()); const DWORD attributes = GetFileAttributes(unicode); delete [] unicode; return attributes != kInvalidFileAttributes; #else - posix::StatStruct file_stat; + posix::StatStruct file_stat{}; return posix::Stat(pathname_.c_str(), &file_stat) == 0; #endif // GTEST_OS_WINDOWS_MOBILE } // Returns true if pathname describes a directory in the file-system @@ -234,11 +235,11 @@ if ((attributes != kInvalidFileAttributes) && (attributes & FILE_ATTRIBUTE_DIRECTORY)) { result = true; } #else - posix::StatStruct file_stat; + posix::StatStruct file_stat{}; result = posix::Stat(path.c_str(), &file_stat) == 0 && posix::IsDir(file_stat); #endif // GTEST_OS_WINDOWS_MOBILE return result; @@ -321,11 +322,11 @@ LPCWSTR unicode = String::AnsiToUtf16(removed_sep.c_str()); int result = CreateDirectory(unicode, nullptr) ? 0 : -1; delete [] unicode; #elif GTEST_OS_WINDOWS int result = _mkdir(pathname_.c_str()); -#elif GTEST_OS_ESP8266 +#elif GTEST_OS_ESP8266 || GTEST_OS_XTENSA // do nothing int result = 0; #else int result = mkdir(pathname_.c_str(), 0777); #endif // GTEST_OS_WINDOWS_MOBILE @@ -347,36 +348,22 @@ // Removes any redundant separators that might be in the pathname. // For example, "bar///foo" becomes "bar/foo". Does not eliminate other // redundancies that might be in a pathname involving "." or "..". void FilePath::Normalize() { - if (pathname_.c_str() == nullptr) { - pathname_ = ""; - return; - } - const char* src = pathname_.c_str(); - char* const dest = new char[pathname_.length() + 1]; - char* dest_ptr = dest; - memset(dest_ptr, 0, pathname_.length() + 1); + auto out = pathname_.begin(); - while (*src != '\0') { - *dest_ptr = *src; - if (!IsPathSeparator(*src)) { - src++; + for (const char character : pathname_) { + if (!IsPathSeparator(character)) { + *(out++) = character; + } else if (out == pathname_.begin() || *std::prev(out) != kPathSeparator) { + *(out++) = kPathSeparator; } else { -#if GTEST_HAS_ALT_PATH_SEP_ - if (*dest_ptr == kAlternatePathSeparator) { - *dest_ptr = kPathSeparator; - } -#endif - while (IsPathSeparator(*src)) - src++; + continue; } - dest_ptr++; } - *dest_ptr = '\0'; - pathname_ = dest; - delete[] dest; + + pathname_.erase(out, pathname_.end()); } } // namespace internal } // namespace testing diff --git a/media/libvpx/libvpx/third_party/googletest/src/src/gtest-internal-inl.h b/media/libvpx/libvpx/third_party/googletest/src/src/gtest-internal-inl.h --- a/media/libvpx/libvpx/third_party/googletest/src/src/gtest-internal-inl.h +++ b/media/libvpx/libvpx/third_party/googletest/src/src/gtest-internal-inl.h @@ -29,12 +29,12 @@ // Utility functions and classes used by the Google C++ testing framework.// // This file contains purely Google Test's internal implementation. Please // DO NOT #INCLUDE IT IN A USER PROGRAM. -#ifndef GTEST_SRC_GTEST_INTERNAL_INL_H_ -#define GTEST_SRC_GTEST_INTERNAL_INL_H_ +#ifndef GOOGLETEST_SRC_GTEST_INTERNAL_INL_H_ +#define GOOGLETEST_SRC_GTEST_INTERNAL_INL_H_ #ifndef _WIN32_WCE # include #endif // !_WIN32_WCE #include @@ -82,13 +82,15 @@ // Names of the flags (needed for parsing Google Test flags). const char kAlsoRunDisabledTestsFlag[] = "also_run_disabled_tests"; const char kBreakOnFailureFlag[] = "break_on_failure"; const char kCatchExceptionsFlag[] = "catch_exceptions"; const char kColorFlag[] = "color"; +const char kFailFast[] = "fail_fast"; const char kFilterFlag[] = "filter"; const char kListTestsFlag[] = "list_tests"; const char kOutputFlag[] = "output"; +const char kBriefFlag[] = "brief"; const char kPrintTimeFlag[] = "print_time"; const char kPrintUTF8Flag[] = "print_utf8"; const char kRandomSeedFlag[] = "random_seed"; const char kRepeatFlag[] = "repeat"; const char kShuffleFlag[] = "shuffle"; @@ -162,14 +164,16 @@ break_on_failure_ = GTEST_FLAG(break_on_failure); catch_exceptions_ = GTEST_FLAG(catch_exceptions); color_ = GTEST_FLAG(color); death_test_style_ = GTEST_FLAG(death_test_style); death_test_use_fork_ = GTEST_FLAG(death_test_use_fork); + fail_fast_ = GTEST_FLAG(fail_fast); filter_ = GTEST_FLAG(filter); internal_run_death_test_ = GTEST_FLAG(internal_run_death_test); list_tests_ = GTEST_FLAG(list_tests); output_ = GTEST_FLAG(output); + brief_ = GTEST_FLAG(brief); print_time_ = GTEST_FLAG(print_time); print_utf8_ = GTEST_FLAG(print_utf8); random_seed_ = GTEST_FLAG(random_seed); repeat_ = GTEST_FLAG(repeat); shuffle_ = GTEST_FLAG(shuffle); @@ -185,13 +189,15 @@ GTEST_FLAG(catch_exceptions) = catch_exceptions_; GTEST_FLAG(color) = color_; GTEST_FLAG(death_test_style) = death_test_style_; GTEST_FLAG(death_test_use_fork) = death_test_use_fork_; GTEST_FLAG(filter) = filter_; + GTEST_FLAG(fail_fast) = fail_fast_; GTEST_FLAG(internal_run_death_test) = internal_run_death_test_; GTEST_FLAG(list_tests) = list_tests_; GTEST_FLAG(output) = output_; + GTEST_FLAG(brief) = brief_; GTEST_FLAG(print_time) = print_time_; GTEST_FLAG(print_utf8) = print_utf8_; GTEST_FLAG(random_seed) = random_seed_; GTEST_FLAG(repeat) = repeat_; GTEST_FLAG(shuffle) = shuffle_; @@ -206,14 +212,16 @@ bool break_on_failure_; bool catch_exceptions_; std::string color_; std::string death_test_style_; bool death_test_use_fork_; + bool fail_fast_; std::string filter_; std::string internal_run_death_test_; bool list_tests_; std::string output_; + bool brief_; bool print_time_; bool print_utf8_; int32_t random_seed_; int32_t repeat_; bool shuffle_; @@ -384,17 +392,10 @@ // none was explicitly specified. static std::string GetAbsolutePathToOutputFile(); // Functions for processing the gtest_filter flag. - // Returns true if and only if the wildcard pattern matches the string. - // The first ':' or '\0' character in pattern marks the end of it. - // - // This recursive algorithm isn't very efficient, but is clear and - // works well enough for matching test names, which are short. - static bool PatternMatchesString(const char *pattern, const char *str); - // Returns true if and only if the user-specified filter matches the test // suite name and the test name. static bool FilterMatchesTest(const std::string& test_suite_name, const std::string& test_name); @@ -645,14 +646,14 @@ // exist, creates one and returns it. // // Arguments: // // test_suite_name: name of the test suite - // type_param: the name of the test's type parameter, or NULL if - // this is not a typed or a type-parameterized test. - // set_up_tc: pointer to the function that sets up the test suite - // tear_down_tc: pointer to the function that tears down the test suite + // type_param: the name of the test's type parameter, or NULL if + // this is not a typed or a type-parameterized test. + // set_up_tc: pointer to the function that sets up the test suite + // tear_down_tc: pointer to the function that tears down the test suite TestSuite* GetTestSuite(const char* test_suite_name, const char* type_param, internal::SetUpTestSuiteFunc set_up_tc, internal::TearDownTestSuiteFunc tear_down_tc); // Legacy API is deprecated but still available @@ -672,10 +673,11 @@ // tear_down_tc: pointer to the function that tears down the test suite // test_info: the TestInfo object void AddTestInfo(internal::SetUpTestSuiteFunc set_up_tc, internal::TearDownTestSuiteFunc tear_down_tc, TestInfo* test_info) { +#if GTEST_HAS_DEATH_TEST // In order to support thread-safe death tests, we need to // remember the original working directory when the test program // was first invoked. We cannot do this in RUN_ALL_TESTS(), as // the user may have changed the current directory before calling // RUN_ALL_TESTS(). Therefore we capture the current directory in @@ -684,10 +686,11 @@ if (original_working_dir_.IsEmpty()) { original_working_dir_.Set(FilePath::GetCurrentDir()); GTEST_CHECK_(!original_working_dir_.IsEmpty()) << "Failed to get the current working directory."; } +#endif // GTEST_HAS_DEATH_TEST GetTestSuite(test_info->test_suite_name(), test_info->type_param(), set_up_tc, tear_down_tc) ->AddTestInfo(test_info); } @@ -1159,17 +1162,17 @@ FormatBool(unit_test.Passed()) + "&elapsed_time=" + StreamableToString(unit_test.elapsed_time()) + "ms"); } // Note that "event=TestCaseStart" is a wire format and has to remain - // "case" for compatibilty + // "case" for compatibility void OnTestCaseStart(const TestCase& test_case) override { SendLn(std::string("event=TestCaseStart&name=") + test_case.name()); } // Note that "event=TestCaseEnd" is a wire format and has to remain - // "case" for compatibilty + // "case" for compatibility void OnTestCaseEnd(const TestCase& test_case) override { SendLn("event=TestCaseEnd&passed=" + FormatBool(test_case.Passed()) + "&elapsed_time=" + StreamableToString(test_case.elapsed_time()) + "ms"); } @@ -1213,6 +1216,6 @@ } // namespace internal } // namespace testing GTEST_DISABLE_MSC_WARNINGS_POP_() // 4251 -#endif // GTEST_SRC_GTEST_INTERNAL_INL_H_ +#endif // GOOGLETEST_SRC_GTEST_INTERNAL_INL_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/src/gtest-port.cc b/media/libvpx/libvpx/third_party/googletest/src/src/gtest-port.cc --- a/media/libvpx/libvpx/third_party/googletest/src/src/gtest-port.cc +++ b/media/libvpx/libvpx/third_party/googletest/src/src/gtest-port.cc @@ -196,21 +196,22 @@ // get number of structs size_t size; if (sysctl(mib, miblen, NULL, &size, NULL, 0)) { return 0; } - mib[5] = size / mib[4]; + + mib[5] = static_cast(size / static_cast(mib[4])); // populate array of structs struct kinfo_proc info[mib[5]]; if (sysctl(mib, miblen, &info, &size, NULL, 0)) { return 0; } // exclude empty members - int nthreads = 0; - for (int i = 0; i < size / mib[4]; i++) { + size_t nthreads = 0; + for (size_t i = 0; i < size / static_cast(mib[4]); i++) { if (info[i].p_tid != -1) nthreads++; } return nthreads; } @@ -685,12 +686,12 @@ static Mutex mutex_; // Protects access to GetThreadMapLocked() and its return value. static Mutex thread_map_mutex_; }; -Mutex ThreadLocalRegistryImpl::mutex_(Mutex::kStaticMutex); -Mutex ThreadLocalRegistryImpl::thread_map_mutex_(Mutex::kStaticMutex); +Mutex ThreadLocalRegistryImpl::mutex_(Mutex::kStaticMutex); // NOLINT +Mutex ThreadLocalRegistryImpl::thread_map_mutex_(Mutex::kStaticMutex); // NOLINT ThreadLocalValueHolderBase* ThreadLocalRegistry::GetValueOnCurrentThread( const ThreadLocalBase* thread_local_instance) { return ThreadLocalRegistryImpl::GetValueOnCurrentThread( thread_local_instance); @@ -1092,13 +1093,13 @@ GTEST_CHECK_(captured_fd != -1) << "Unable to open temporary file " << temp_file_path; filename_ = temp_file_path; # else // There's no guarantee that a test has write access to the current - // directory, so we create the temporary file in the /tmp directory - // instead. We use /tmp on most systems, and /sdcard on Android. - // That's because Android doesn't have /tmp. + // directory, so we create the temporary file in a temporary directory. + std::string name_template; + # if GTEST_OS_LINUX_ANDROID // Note: Android applications are expected to call the framework's // Context.getExternalStorageDirectory() method through JNI to get // the location of the world-writable SD Card directory. However, // this requires a Context handle, which cannot be retrieved @@ -1107,21 +1108,50 @@ // run in a Dalvik process (e.g. when running it through 'adb shell'). // // The location /data/local/tmp is directly accessible from native code. // '/sdcard' and other variants cannot be relied on, as they are not // guaranteed to be mounted, or may have a delay in mounting. - char name_template[] = "/data/local/tmp/gtest_captured_stream.XXXXXX"; + name_template = "/data/local/tmp/"; +# elif GTEST_OS_IOS + char user_temp_dir[PATH_MAX + 1]; + + // Documented alternative to NSTemporaryDirectory() (for obtaining creating + // a temporary directory) at + // https://developer.apple.com/library/archive/documentation/Security/Conceptual/SecureCodingGuide/Articles/RaceConditions.html#//apple_ref/doc/uid/TP40002585-SW10 + // + // _CS_DARWIN_USER_TEMP_DIR (as well as _CS_DARWIN_USER_CACHE_DIR) is not + // documented in the confstr() man page at + // https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/confstr.3.html#//apple_ref/doc/man/3/confstr + // but are still available, according to the WebKit patches at + // https://trac.webkit.org/changeset/262004/webkit + // https://trac.webkit.org/changeset/263705/webkit + // + // The confstr() implementation falls back to getenv("TMPDIR"). See + // https://opensource.apple.com/source/Libc/Libc-1439.100.3/gen/confstr.c.auto.html + ::confstr(_CS_DARWIN_USER_TEMP_DIR, user_temp_dir, sizeof(user_temp_dir)); + + name_template = user_temp_dir; + if (name_template.back() != GTEST_PATH_SEP_[0]) + name_template.push_back(GTEST_PATH_SEP_[0]); # else - char name_template[] = "/tmp/captured_stream.XXXXXX"; -# endif // GTEST_OS_LINUX_ANDROID - const int captured_fd = mkstemp(name_template); + name_template = "/tmp/"; +# endif + name_template.append("gtest_captured_stream.XXXXXX"); + + // mkstemp() modifies the string bytes in place, and does not go beyond the + // string's length. This results in well-defined behavior in C++17. + // + // The const_cast is needed below C++17. The constraints on std::string + // implementations in C++11 and above make assumption behind the const_cast + // fairly safe. + const int captured_fd = ::mkstemp(const_cast(name_template.data())); if (captured_fd == -1) { GTEST_LOG_(WARNING) << "Failed to create tmp file " << name_template << " for test; does the test have access to the /tmp directory?"; } - filename_ = name_template; + filename_ = std::move(name_template); # endif // GTEST_OS_WINDOWS fflush(nullptr); dup2(captured_fd, fd_); close(captured_fd); } diff --git a/media/libvpx/libvpx/third_party/googletest/src/src/gtest-printers.cc b/media/libvpx/libvpx/third_party/googletest/src/src/gtest-printers.cc --- a/media/libvpx/libvpx/third_party/googletest/src/src/gtest-printers.cc +++ b/media/libvpx/libvpx/third_party/googletest/src/src/gtest-printers.cc @@ -40,15 +40,20 @@ // type Foo by defining either operator<<(::std::ostream&, const Foo&) // or void PrintTo(const Foo&, ::std::ostream*) in the namespace that // defines Foo. #include "gtest/gtest-printers.h" + #include + #include +#include #include #include // NOLINT #include +#include + #include "gtest/internal/gtest-port.h" #include "src/gtest-internal-inl.h" namespace testing { @@ -100,13 +105,23 @@ PrintByteSegmentInObjectTo(obj_bytes, resume_pos, count - resume_pos, os); } *os << ">"; } +// Helpers for widening a character to char32_t. Since the standard does not +// specify if char / wchar_t is signed or unsigned, it is important to first +// convert it to the unsigned type of the same width before widening it to +// char32_t. +template +char32_t ToChar32(CharType in) { + return static_cast( + static_cast::type>(in)); +} + } // namespace -namespace internal2 { +namespace internal { // Delegates to PrintBytesInObjectToImpl() to print the bytes in the // given object. The delegation simplifies the implementation, which // uses the << operator and thus is easier done outside of the // ::testing::internal namespace, which contains a << operator that @@ -114,14 +129,10 @@ void PrintBytesInObjectTo(const unsigned char* obj_bytes, size_t count, ostream* os) { PrintBytesInObjectToImpl(obj_bytes, count, os); } -} // namespace internal2 - -namespace internal { - // Depending on the value of a char (or wchar_t), we print it in one // of three formats: // - as is if it's a printable ASCII (e.g. 'a', '2', ' '), // - as a hexadecimal escape sequence (e.g. '\x7F'), or // - as a special escape sequence (e.g. '\r', '\n'). @@ -132,22 +143,19 @@ }; // Returns true if c is a printable ASCII character. We test the // value of c directly instead of calling isprint(), which is buggy on // Windows Mobile. -inline bool IsPrintableAscii(wchar_t c) { - return 0x20 <= c && c <= 0x7E; -} +inline bool IsPrintableAscii(char32_t c) { return 0x20 <= c && c <= 0x7E; } -// Prints a wide or narrow char c as a character literal without the -// quotes, escaping it when necessary; returns how c was formatted. -// The template argument UnsignedChar is the unsigned version of Char, -// which is the type of c. -template +// Prints c (of type char, char8_t, char16_t, char32_t, or wchar_t) as a +// character literal without the quotes, escaping it when necessary; returns how +// c was formatted. +template static CharFormat PrintAsCharLiteralTo(Char c, ostream* os) { - wchar_t w_c = static_cast(c); - switch (w_c) { + const char32_t u_c = ToChar32(c); + switch (u_c) { case L'\0': *os << "\\0"; break; case L'\'': *os << "\\'"; @@ -175,55 +183,96 @@ break; case L'\v': *os << "\\v"; break; default: - if (IsPrintableAscii(w_c)) { + if (IsPrintableAscii(u_c)) { *os << static_cast(c); return kAsIs; } else { ostream::fmtflags flags = os->flags(); - *os << "\\x" << std::hex << std::uppercase - << static_cast(static_cast(c)); + *os << "\\x" << std::hex << std::uppercase << static_cast(u_c); os->flags(flags); return kHexEscape; } } return kSpecialEscape; } -// Prints a wchar_t c as if it's part of a string literal, escaping it when +// Prints a char32_t c as if it's part of a string literal, escaping it when // necessary; returns how c was formatted. -static CharFormat PrintAsStringLiteralTo(wchar_t c, ostream* os) { +static CharFormat PrintAsStringLiteralTo(char32_t c, ostream* os) { switch (c) { case L'\'': *os << "'"; return kAsIs; case L'"': *os << "\\\""; return kSpecialEscape; default: - return PrintAsCharLiteralTo(c, os); + return PrintAsCharLiteralTo(c, os); } } +static const char* GetCharWidthPrefix(char) { + return ""; +} + +static const char* GetCharWidthPrefix(signed char) { + return ""; +} + +static const char* GetCharWidthPrefix(unsigned char) { + return ""; +} + +#ifdef __cpp_char8_t +static const char* GetCharWidthPrefix(char8_t) { + return "u8"; +} +#endif + +static const char* GetCharWidthPrefix(char16_t) { + return "u"; +} + +static const char* GetCharWidthPrefix(char32_t) { + return "U"; +} + +static const char* GetCharWidthPrefix(wchar_t) { + return "L"; +} + // Prints a char c as if it's part of a string literal, escaping it when // necessary; returns how c was formatted. static CharFormat PrintAsStringLiteralTo(char c, ostream* os) { - return PrintAsStringLiteralTo( - static_cast(static_cast(c)), os); + return PrintAsStringLiteralTo(ToChar32(c), os); +} + +#ifdef __cpp_char8_t +static CharFormat PrintAsStringLiteralTo(char8_t c, ostream* os) { + return PrintAsStringLiteralTo(ToChar32(c), os); +} +#endif + +static CharFormat PrintAsStringLiteralTo(char16_t c, ostream* os) { + return PrintAsStringLiteralTo(ToChar32(c), os); } -// Prints a wide or narrow character c and its code. '\0' is printed -// as "'\\0'", other unprintable characters are also properly escaped -// using the standard C++ escape sequence. The template argument -// UnsignedChar is the unsigned version of Char, which is the type of c. -template +static CharFormat PrintAsStringLiteralTo(wchar_t c, ostream* os) { + return PrintAsStringLiteralTo(ToChar32(c), os); +} + +// Prints a character c (of type char, char8_t, char16_t, char32_t, or wchar_t) +// and its code. '\0' is printed as "'\\0'", other unprintable characters are +// also properly escaped using the standard C++ escape sequence. +template void PrintCharAndCodeTo(Char c, ostream* os) { // First, print c as a literal in the most readable form we can find. - *os << ((sizeof(c) > 1) ? "L'" : "'"); - const CharFormat format = PrintAsCharLiteralTo(c, os); + *os << GetCharWidthPrefix(c) << "'"; + const CharFormat format = PrintAsCharLiteralTo(c, os); *os << "'"; // To aid user debugging, we also print c's code in decimal, unless // it's 0 (in which case c was printed as '\\0', making the code // obvious). @@ -240,45 +289,45 @@ *os << ", 0x" << String::FormatHexInt(static_cast(c)); } *os << ")"; } -void PrintTo(unsigned char c, ::std::ostream* os) { - PrintCharAndCodeTo(c, os); -} -void PrintTo(signed char c, ::std::ostream* os) { - PrintCharAndCodeTo(c, os); -} +void PrintTo(unsigned char c, ::std::ostream* os) { PrintCharAndCodeTo(c, os); } +void PrintTo(signed char c, ::std::ostream* os) { PrintCharAndCodeTo(c, os); } // Prints a wchar_t as a symbol if it is printable or as its internal // code otherwise and also as its code. L'\0' is printed as "L'\\0'". -void PrintTo(wchar_t wc, ostream* os) { - PrintCharAndCodeTo(wc, os); +void PrintTo(wchar_t wc, ostream* os) { PrintCharAndCodeTo(wc, os); } + +// TODO(dcheng): Consider making this delegate to PrintCharAndCodeTo() as well. +void PrintTo(char32_t c, ::std::ostream* os) { + *os << std::hex << "U+" << std::uppercase << std::setfill('0') << std::setw(4) + << static_cast(c); } // Prints the given array of characters to the ostream. CharType must be either -// char or wchar_t. +// char, char8_t, char16_t, char32_t, or wchar_t. // The array starts at begin, the length is len, it may include '\0' characters // and may not be NUL-terminated. template GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ GTEST_ATTRIBUTE_NO_SANITIZE_HWADDRESS_ GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ static CharFormat PrintCharsAsStringTo( const CharType* begin, size_t len, ostream* os) { - const char* const kQuoteBegin = sizeof(CharType) == 1 ? "\"" : "L\""; - *os << kQuoteBegin; + const char* const quote_prefix = GetCharWidthPrefix(*begin); + *os << quote_prefix << "\""; bool is_previous_hex = false; CharFormat print_format = kAsIs; for (size_t index = 0; index < len; ++index) { const CharType cur = begin[index]; if (is_previous_hex && IsXDigit(cur)) { // Previous character is of '\x..' form and this character can be // interpreted as another hexadecimal digit in its number. Break string to // disambiguate. - *os << "\" " << kQuoteBegin; + *os << "\" " << quote_prefix << "\""; } is_previous_hex = PrintAsStringLiteralTo(cur, os) == kHexEscape; // Remember if any characters required hex escaping. if (is_previous_hex) { print_format = kHexEscape; @@ -320,42 +369,70 @@ // Prints a (const) char array of 'len' elements, starting at address 'begin'. void UniversalPrintArray(const char* begin, size_t len, ostream* os) { UniversalPrintCharArray(begin, len, os); } +#ifdef __cpp_char8_t +// Prints a (const) char8_t array of 'len' elements, starting at address +// 'begin'. +void UniversalPrintArray(const char8_t* begin, size_t len, ostream* os) { + UniversalPrintCharArray(begin, len, os); +} +#endif + +// Prints a (const) char16_t array of 'len' elements, starting at address +// 'begin'. +void UniversalPrintArray(const char16_t* begin, size_t len, ostream* os) { + UniversalPrintCharArray(begin, len, os); +} + +// Prints a (const) char32_t array of 'len' elements, starting at address +// 'begin'. +void UniversalPrintArray(const char32_t* begin, size_t len, ostream* os) { + UniversalPrintCharArray(begin, len, os); +} + // Prints a (const) wchar_t array of 'len' elements, starting at address // 'begin'. void UniversalPrintArray(const wchar_t* begin, size_t len, ostream* os) { UniversalPrintCharArray(begin, len, os); } -// Prints the given C string to the ostream. -void PrintTo(const char* s, ostream* os) { +namespace { + +// Prints a null-terminated C-style string to the ostream. +template +void PrintCStringTo(const Char* s, ostream* os) { if (s == nullptr) { *os << "NULL"; } else { *os << ImplicitCast_(s) << " pointing to "; - PrintCharsAsStringTo(s, strlen(s), os); + PrintCharsAsStringTo(s, std::char_traits::length(s), os); } } +} // anonymous namespace + +void PrintTo(const char* s, ostream* os) { PrintCStringTo(s, os); } + +#ifdef __cpp_char8_t +void PrintTo(const char8_t* s, ostream* os) { PrintCStringTo(s, os); } +#endif + +void PrintTo(const char16_t* s, ostream* os) { PrintCStringTo(s, os); } + +void PrintTo(const char32_t* s, ostream* os) { PrintCStringTo(s, os); } + // MSVC compiler can be configured to define whar_t as a typedef // of unsigned short. Defining an overload for const wchar_t* in that case // would cause pointers to unsigned shorts be printed as wide strings, // possibly accessing more memory than intended and causing invalid // memory accesses. MSVC defines _NATIVE_WCHAR_T_DEFINED symbol when // wchar_t is implemented as a native type. #if !defined(_MSC_VER) || defined(_NATIVE_WCHAR_T_DEFINED) // Prints the given wide C string to the ostream. -void PrintTo(const wchar_t* s, ostream* os) { - if (s == nullptr) { - *os << "NULL"; - } else { - *os << ImplicitCast_(s) << " pointing to "; - PrintCharsAsStringTo(s, wcslen(s), os); - } -} +void PrintTo(const wchar_t* s, ostream* os) { PrintCStringTo(s, os); } #endif // wchar_t is native namespace { bool ContainsUnprintableControlCodes(const char* str, size_t length) { @@ -429,10 +506,24 @@ ConditionalPrintAsText(s.data(), s.size(), os); } } } +#ifdef __cpp_char8_t +void PrintU8StringTo(const ::std::u8string& s, ostream* os) { + PrintCharsAsStringTo(s.data(), s.size(), os); +} +#endif + +void PrintU16StringTo(const ::std::u16string& s, ostream* os) { + PrintCharsAsStringTo(s.data(), s.size(), os); +} + +void PrintU32StringTo(const ::std::u32string& s, ostream* os) { + PrintCharsAsStringTo(s.data(), s.size(), os); +} + #if GTEST_HAS_STD_WSTRING void PrintWideStringTo(const ::std::wstring& s, ostream* os) { PrintCharsAsStringTo(s.data(), s.size(), os); } #endif // GTEST_HAS_STD_WSTRING diff --git a/media/libvpx/libvpx/third_party/googletest/src/src/gtest-typed-test.cc b/media/libvpx/libvpx/third_party/googletest/src/src/gtest-typed-test.cc --- a/media/libvpx/libvpx/third_party/googletest/src/src/gtest-typed-test.cc +++ b/media/libvpx/libvpx/third_party/googletest/src/src/gtest-typed-test.cc @@ -33,12 +33,10 @@ #include "gtest/gtest.h" namespace testing { namespace internal { -#if GTEST_HAS_TYPED_TEST_P - // Skips to the first non-space char in str. Returns an empty string if str // contains only whitespace characters. static const char* SkipSpaces(const char* str) { while (IsSpace(*str)) str++; @@ -76,21 +74,11 @@ if (tests.count(name) != 0) { errors << "Test " << name << " is listed more than once.\n"; continue; } - bool found = false; - for (RegisteredTestIter it = registered_tests_.begin(); - it != registered_tests_.end(); - ++it) { - if (name == it->first) { - found = true; - break; - } - } - - if (found) { + if (registered_tests_.count(name) != 0) { tests.insert(name); } else { errors << "No test named " << name << " can be found in this test suite.\n"; } @@ -113,9 +101,7 @@ } return registered_tests; } -#endif // GTEST_HAS_TYPED_TEST_P - } // namespace internal } // namespace testing diff --git a/media/libvpx/libvpx/third_party/googletest/src/src/gtest.cc b/media/libvpx/libvpx/third_party/googletest/src/src/gtest.cc --- a/media/libvpx/libvpx/third_party/googletest/src/src/gtest.cc +++ b/media/libvpx/libvpx/third_party/googletest/src/src/gtest.cc @@ -33,19 +33,20 @@ #include "gtest/gtest.h" #include "gtest/internal/custom/gtest.h" #include "gtest/gtest-spi.h" #include -#include #include #include #include #include #include #include #include +#include // NOLINT +#include #include #include #include #include #include @@ -53,12 +54,10 @@ #include #include #if GTEST_OS_LINUX -# define GTEST_HAS_GETTIMEOFDAY_ 1 - # include // NOLINT # include // NOLINT # include // NOLINT // Declares vsnprintf(). This header is not available on Windows. # include // NOLINT @@ -66,11 +65,10 @@ # include // NOLINT # include // NOLINT # include #elif GTEST_OS_ZOS -# define GTEST_HAS_GETTIMEOFDAY_ 1 # include // NOLINT // On z/OS we additionally need strings.h for strcasecmp. # include // NOLINT @@ -84,29 +82,23 @@ # include // NOLINT # undef min #ifdef _MSC_VER # include // NOLINT -# include // NOLINT #endif # include // NOLINT # include // NOLINT # include // NOLINT # include // NOLINT # if GTEST_OS_WINDOWS_MINGW -// MinGW has gettimeofday() but not _ftime64(). -# define GTEST_HAS_GETTIMEOFDAY_ 1 # include // NOLINT # endif // GTEST_OS_WINDOWS_MINGW #else -// Assume other platforms have gettimeofday(). -# define GTEST_HAS_GETTIMEOFDAY_ 1 - // cpplint thinks that the header is already included, so we want to // silence it. # include // NOLINT # include // NOLINT @@ -211,10 +203,25 @@ return testbridge_test_only; } return kUniversalFilter; } +// Bazel passes in the argument to '--test_runner_fail_fast' via the +// TESTBRIDGE_TEST_RUNNER_FAIL_FAST environment variable. +static bool GetDefaultFailFast() { + const char* const testbridge_test_runner_fail_fast = + internal::posix::GetEnv("TESTBRIDGE_TEST_RUNNER_FAIL_FAST"); + if (testbridge_test_runner_fail_fast != nullptr) { + return strcmp(testbridge_test_runner_fail_fast, "1") == 0; + } + return false; +} + +GTEST_DEFINE_bool_( + fail_fast, internal::BoolFromGTestEnv("fail_fast", GetDefaultFailFast()), + "True if and only if a test failure should stop further test execution."); + GTEST_DEFINE_bool_( also_run_disabled_tests, internal::BoolFromGTestEnv("also_run_disabled_tests", false), "Run disabled tests too, in addition to the tests normally being run."); @@ -271,10 +278,14 @@ "If a directory is specified, output files will be created " "within that directory, with file-names based on the test " "executable's name and, if necessary, made unique by adding " "digits."); +GTEST_DEFINE_bool_( + brief, internal::BoolFromGTestEnv("brief", false), + "True if only test failures should be displayed in text output."); + GTEST_DEFINE_bool_(print_time, internal::BoolFromGTestEnv("print_time", true), "True if and only if " GTEST_NAME_ " should display elapsed time in text output."); GTEST_DEFINE_bool_(print_utf8, internal::BoolFromGTestEnv("print_utf8", true), @@ -477,19 +488,19 @@ "code that always depend on code that provides TEST_P. Failing to do " "so is often an indication of dead code, e.g. the last TEST_P was " "removed but the rest got left behind."; std::string message = - "Paramaterized test suite " + name + + "Parameterized test suite " + name + (has_test_p ? kMissingInstantiation : kMissingTestCase) + "\n\n" "To suppress this error for this test suite, insert the following line " "(in a non-header) in the namespace it is defined in:" "\n\n" "GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(" + name + ");"; - std::string full_name = "UninstantiatedParamaterizedTestSuite<" + name + ">"; + std::string full_name = "UninstantiatedParameterizedTestSuite<" + name + ">"; RegisterTest( // "GoogleTestVerification", full_name.c_str(), nullptr, // No type parameter. nullptr, // No value parameter. location.file.c_str(), location.line, [message, location] { @@ -532,27 +543,27 @@ for (const auto& testcase : suites_) { if (testcase.second.instantiated) continue; if (ignored.find(testcase.first) != ignored.end()) continue; std::string message = - "Type paramaterized test suite " + testcase.first + + "Type parameterized test suite " + testcase.first + " is defined via REGISTER_TYPED_TEST_SUITE_P, but never instantiated " "via INSTANTIATE_TYPED_TEST_SUITE_P. None of the test cases will run." "\n\n" "Ideally, TYPED_TEST_P definitions should only ever be included as " "part of binaries that intend to use them. (As opposed to, for " "example, being placed in a library that may be linked in to get other " "utilities.)" "\n\n" "To suppress this error for this test suite, insert the following line " - "(in a non-header) in the namespace it is definedin in:" + "(in a non-header) in the namespace it is defined in:" "\n\n" "GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(" + testcase.first + ");"; std::string full_name = - "UninstantiatedTypeParamaterizedTestSuite<" + testcase.first + ">"; + "UninstantiatedTypeParameterizedTestSuite<" + testcase.first + ">"; RegisterTest( // "GoogleTestVerification", full_name.c_str(), nullptr, // No type parameter. nullptr, // No value parameter. testcase.second.code_location.file.c_str(), @@ -633,51 +644,86 @@ output_name, internal::GetCurrentExecutableName(), GetOutputFormat().c_str())); return result.string(); } -// Returns true if and only if the wildcard pattern matches the string. -// The first ':' or '\0' character in pattern marks the end of it. +// Returns true if and only if the wildcard pattern matches the string. Each +// pattern consists of regular characters, single-character wildcards (?), and +// multi-character wildcards (*). // -// This recursive algorithm isn't very efficient, but is clear and -// works well enough for matching test names, which are short. -bool UnitTestOptions::PatternMatchesString(const char *pattern, - const char *str) { - switch (*pattern) { - case '\0': - case ':': // Either ':' or '\0' marks the end of the pattern. - return *str == '\0'; - case '?': // Matches any single character. - return *str != '\0' && PatternMatchesString(pattern + 1, str + 1); - case '*': // Matches any string (possibly empty) of characters. - return (*str != '\0' && PatternMatchesString(pattern, str + 1)) || - PatternMatchesString(pattern + 1, str); - default: // Non-special character. Matches itself. - return *pattern == *str && - PatternMatchesString(pattern + 1, str + 1); - } -} - -bool UnitTestOptions::MatchesFilter( - const std::string& name, const char* filter) { - const char *cur_pattern = filter; - for (;;) { - if (PatternMatchesString(cur_pattern, name.c_str())) { +// This function implements a linear-time string globbing algorithm based on +// https://research.swtch.com/glob. +static bool PatternMatchesString(const std::string& name_str, + const char* pattern, const char* pattern_end) { + const char* name = name_str.c_str(); + const char* const name_begin = name; + const char* const name_end = name + name_str.size(); + + const char* pattern_next = pattern; + const char* name_next = name; + + while (pattern < pattern_end || name < name_end) { + if (pattern < pattern_end) { + switch (*pattern) { + default: // Match an ordinary character. + if (name < name_end && *name == *pattern) { + ++pattern; + ++name; + continue; + } + break; + case '?': // Match any single character. + if (name < name_end) { + ++pattern; + ++name; + continue; + } + break; + case '*': + // Match zero or more characters. Start by skipping over the wildcard + // and matching zero characters from name. If that fails, restart and + // match one more character than the last attempt. + pattern_next = pattern; + name_next = name + 1; + ++pattern; + continue; + } + } + // Failed to match a character. Restart if possible. + if (name_begin < name_next && name_next <= name_end) { + pattern = pattern_next; + name = name_next; + continue; + } + return false; + } + return true; +} + +bool UnitTestOptions::MatchesFilter(const std::string& name_str, + const char* filter) { + // The filter is a list of patterns separated by colons (:). + const char* pattern = filter; + while (true) { + // Find the bounds of this pattern. + const char* const next_sep = strchr(pattern, ':'); + const char* const pattern_end = + next_sep != nullptr ? next_sep : pattern + strlen(pattern); + + // Check if this pattern matches name_str. + if (PatternMatchesString(name_str, pattern, pattern_end)) { return true; } - // Finds the next pattern in the filter. - cur_pattern = strchr(cur_pattern, ':'); - - // Returns if no more pattern can be found. - if (cur_pattern == nullptr) { + // Give up on this pattern. However, if we found a pattern separator (:), + // advance to the next pattern (skipping over the separator) and restart. + if (next_sep == nullptr) { return false; } - - // Skips the pattern separater (the ':' character). - cur_pattern++; - } + pattern = next_sep + 1; + } + return true; } // Returns true if and only if the user-specified filter matches the test // suite name and the test name. bool UnitTestOptions::FilterMatchesTest(const std::string& test_suite_name, @@ -983,48 +1029,34 @@ // Skips the user-specified number of frames plus this function // itself. ); // NOLINT } -// Returns the current time in milliseconds. +// A helper class for measuring elapsed times. +class Timer { + public: + Timer() : start_(std::chrono::steady_clock::now()) {} + + // Return time elapsed in milliseconds since the timer was created. + TimeInMillis Elapsed() { + return std::chrono::duration_cast( + std::chrono::steady_clock::now() - start_) + .count(); + } + + private: + std::chrono::steady_clock::time_point start_; +}; + +// Returns a timestamp as milliseconds since the epoch. Note this time may jump +// around subject to adjustments by the system, to measure elapsed time use +// Timer instead. TimeInMillis GetTimeInMillis() { -#if GTEST_OS_WINDOWS_MOBILE || defined(__BORLANDC__) - // Difference between 1970-01-01 and 1601-01-01 in milliseconds. - // http://analogous.blogspot.com/2005/04/epoch.html - const TimeInMillis kJavaEpochToWinFileTimeDelta = - static_cast(116444736UL) * 100000UL; - const DWORD kTenthMicrosInMilliSecond = 10000; - - SYSTEMTIME now_systime; - FILETIME now_filetime; - ULARGE_INTEGER now_int64; - GetSystemTime(&now_systime); - if (SystemTimeToFileTime(&now_systime, &now_filetime)) { - now_int64.LowPart = now_filetime.dwLowDateTime; - now_int64.HighPart = now_filetime.dwHighDateTime; - now_int64.QuadPart = (now_int64.QuadPart / kTenthMicrosInMilliSecond) - - kJavaEpochToWinFileTimeDelta; - return now_int64.QuadPart; - } - return 0; -#elif GTEST_OS_WINDOWS && !GTEST_HAS_GETTIMEOFDAY_ - __timeb64 now; - - // MSVC 8 deprecates _ftime64(), so we want to suppress warning 4996 - // (deprecated function) there. - GTEST_DISABLE_MSC_DEPRECATED_PUSH_() - _ftime64(&now); - GTEST_DISABLE_MSC_DEPRECATED_POP_() - - return static_cast(now.time) * 1000 + now.millitm; -#elif GTEST_HAS_GETTIMEOFDAY_ - struct timeval now; - gettimeofday(&now, nullptr); - return static_cast(now.tv_sec) * 1000 + now.tv_usec / 1000; -#else -# error "Don't know how to get the current time on your system." -#endif + return std::chrono::duration_cast( + std::chrono::system_clock::now() - + std::chrono::system_clock::from_time_t(0)) + .count(); } // Utilities // class String. @@ -1535,10 +1567,35 @@ double val2, double abs_error) { const double diff = fabs(val1 - val2); if (diff <= abs_error) return AssertionSuccess(); + // Find the value which is closest to zero. + const double min_abs = std::min(fabs(val1), fabs(val2)); + // Find the distance to the next double from that value. + const double epsilon = + nextafter(min_abs, std::numeric_limits::infinity()) - min_abs; + // Detect the case where abs_error is so small that EXPECT_NEAR is + // effectively the same as EXPECT_EQUAL, and give an informative error + // message so that the situation can be more easily understood without + // requiring exotic floating-point knowledge. + // Don't do an epsilon check if abs_error is zero because that implies + // that an equality check was actually intended. + if (!(std::isnan)(val1) && !(std::isnan)(val2) && abs_error > 0 && + abs_error < epsilon) { + return AssertionFailure() + << "The difference between " << expr1 << " and " << expr2 << " is " + << diff << ", where\n" + << expr1 << " evaluates to " << val1 << ",\n" + << expr2 << " evaluates to " << val2 << ".\nThe abs_error parameter " + << abs_error_expr << " evaluates to " << abs_error + << " which is smaller than the minimum distance between doubles for " + "numbers of this magnitude which is " + << epsilon + << ", thus making this EXPECT_NEAR check equivalent to " + "EXPECT_EQUAL. Consider using EXPECT_DOUBLE_EQ instead."; + } return AssertionFailure() << "The difference between " << expr1 << " and " << expr2 << " is " << diff << ", which exceeds " << abs_error_expr << ", where\n" << expr1 << " evaluates to " << val1 << ",\n" << expr2 << " evaluates to " << val2 << ", and\n" @@ -1597,61 +1654,10 @@ return internal::FloatingPointLE(expr1, expr2, val1, val2); } namespace internal { -// The helper function for {ASSERT|EXPECT}_EQ with int or enum -// arguments. -AssertionResult CmpHelperEQ(const char* lhs_expression, - const char* rhs_expression, - BiggestInt lhs, - BiggestInt rhs) { - if (lhs == rhs) { - return AssertionSuccess(); - } - - return EqFailure(lhs_expression, - rhs_expression, - FormatForComparisonFailureMessage(lhs, rhs), - FormatForComparisonFailureMessage(rhs, lhs), - false); -} - -// A macro for implementing the helper functions needed to implement -// ASSERT_?? and EXPECT_?? with integer or enum arguments. It is here -// just to avoid copy-and-paste of similar code. -#define GTEST_IMPL_CMP_HELPER_(op_name, op)\ -AssertionResult CmpHelper##op_name(const char* expr1, const char* expr2, \ - BiggestInt val1, BiggestInt val2) {\ - if (val1 op val2) {\ - return AssertionSuccess();\ - } else {\ - return AssertionFailure() \ - << "Expected: (" << expr1 << ") " #op " (" << expr2\ - << "), actual: " << FormatForComparisonFailureMessage(val1, val2)\ - << " vs " << FormatForComparisonFailureMessage(val2, val1);\ - }\ -} - -// Implements the helper function for {ASSERT|EXPECT}_NE with int or -// enum arguments. -GTEST_IMPL_CMP_HELPER_(NE, !=) -// Implements the helper function for {ASSERT|EXPECT}_LE with int or -// enum arguments. -GTEST_IMPL_CMP_HELPER_(LE, <=) -// Implements the helper function for {ASSERT|EXPECT}_LT with int or -// enum arguments. -GTEST_IMPL_CMP_HELPER_(LT, < ) -// Implements the helper function for {ASSERT|EXPECT}_GE with int or -// enum arguments. -GTEST_IMPL_CMP_HELPER_(GE, >=) -// Implements the helper function for {ASSERT|EXPECT}_GT with int or -// enum arguments. -GTEST_IMPL_CMP_HELPER_(GT, > ) - -#undef GTEST_IMPL_CMP_HELPER_ - // The helper function for {ASSERT|EXPECT}_STREQ. AssertionResult CmpHelperSTREQ(const char* lhs_expression, const char* rhs_expression, const char* lhs, const char* rhs) { @@ -2121,12 +2127,17 @@ suffix.c_str()); } // Formats an int value as "%02d". std::string String::FormatIntWidth2(int value) { + return FormatIntWidthN(value, 2); +} + +// Formats an int value to given width with leading zeros. +std::string String::FormatIntWidthN(int value, int width) { std::stringstream ss; - ss << std::setfill('0') << std::setw(2) << value; + ss << std::setfill('0') << std::setw(width) << value; return ss.str(); } // Formats an int value as "%X". std::string String::FormatHexUInt32(uint32_t value) { @@ -2174,11 +2185,13 @@ // Appends the user message if it's non-empty. const std::string user_msg_string = user_msg.GetString(); if (user_msg_string.empty()) { return gtest_msg; } - + if (gtest_msg.empty()) { + return user_msg_string; + } return gtest_msg + "\n" + user_msg_string; } } // namespace internal @@ -2226,11 +2239,11 @@ void TestResult::RecordProperty(const std::string& xml_element, const TestProperty& test_property) { if (!ValidateTestProperty(xml_element, test_property)) { return; } - internal::MutexLock lock(&test_properites_mutex_); + internal::MutexLock lock(&test_properties_mutex_); const std::vector::iterator property_with_matching_key = std::find_if(test_properties_.begin(), test_properties_.end(), internal::TestPropertyKeyIs(test_property.key())); if (property_with_matching_key == test_properties_.end()) { test_properties_.push_back(test_property); @@ -2253,11 +2266,12 @@ }; // The list of reserved attributes used in the element of XML // output. static const char* const kReservedTestSuiteAttributes[] = { - "disabled", "errors", "failures", "name", "tests", "time", "timestamp"}; + "disabled", "errors", "failures", "name", + "tests", "time", "timestamp", "skipped"}; // The list of reserved attributes used in the element of XML output. static const char* const kReservedTestCaseAttributes[] = { "classname", "name", "status", "time", "type_param", "value_param", "file", "line"}; @@ -2266,11 +2280,11 @@ // still RecordProperty("result") or "RecordProperty(timestamp") static const char* const kReservedOutputTestCaseAttributes[] = { "classname", "name", "status", "time", "type_param", "value_param", "file", "line", "result", "timestamp"}; -template +template std::vector ArrayAsVector(const char* const (&array)[kSize]) { return std::vector(array, array + kSize); } static std::vector GetReservedAttributesForElement( @@ -2710,10 +2724,11 @@ location_(a_code_location), fixture_class_id_(fixture_class_id), should_run_(false), is_disabled_(false), matches_filter_(false), + is_in_another_shard_(false), factory_(factory), result_() {} // Destructs a TestInfo object. TestInfo::~TestInfo() { delete factory_; } @@ -2723,11 +2738,11 @@ // Creates a new TestInfo object and registers it with Google Test; // returns the created object. // // Arguments: // -// test_suite_name: name of the test suite +// test_suite_name: name of the test suite // name: name of the test // type_param: the name of the test's type parameter, or NULL if // this is not a typed or a type-parameterized test. // value_param: text representation of the test's value parameter, // or NULL if this is not a value-parameterized test. @@ -2825,11 +2840,12 @@ TestEventListener* repeater = UnitTest::GetInstance()->listeners().repeater(); // Notifies the unit test event listeners that a test is about to start. repeater->OnTestStart(*this); - const TimeInMillis start = internal::GetTimeInMillis(); + result_.set_start_timestamp(internal::GetTimeInMillis()); + internal::Timer timer; impl->os_stack_trace_getter()->UponLeavingGTest(); // Creates the test object. Test* const test = internal::HandleExceptionsInMethodIfSupported( @@ -2850,21 +2866,42 @@ impl->os_stack_trace_getter()->UponLeavingGTest(); internal::HandleExceptionsInMethodIfSupported( test, &Test::DeleteSelf_, "the test fixture's destructor"); } - result_.set_start_timestamp(start); - result_.set_elapsed_time(internal::GetTimeInMillis() - start); + result_.set_elapsed_time(timer.Elapsed()); // Notifies the unit test event listener that a test has just finished. repeater->OnTestEnd(*this); // Tells UnitTest to stop associating assertion results to this // test. impl->set_current_test_info(nullptr); } +// Skip and records a skipped test result for this object. +void TestInfo::Skip() { + if (!should_run_) return; + + internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); + impl->set_current_test_info(this); + + TestEventListener* repeater = UnitTest::GetInstance()->listeners().repeater(); + + // Notifies the unit test event listeners that a test is about to start. + repeater->OnTestStart(*this); + + const TestPartResult test_part_result = + TestPartResult(TestPartResult::kSkip, this->file(), this->line(), ""); + impl->GetTestPartResultReporterForCurrentThread()->ReportTestPartResult( + test_part_result); + + // Notifies the unit test event listener that a test has just finished. + repeater->OnTestEnd(*this); + impl->set_current_test_info(nullptr); +} + // class TestSuite // Gets the number of successful tests in this test suite. int TestSuite::successful_test_count() const { return CountIf(test_info_list_, TestPassed); @@ -2907,11 +2944,11 @@ // Creates a TestSuite with the given name. // // Arguments: // -// name: name of the test suite +// a_name: name of the test suite // a_type_param: the name of the test suite's type parameter, or NULL if // this is not a typed or a type-parameterized test suite. // set_up_tc: pointer to the function that sets up the test suite // tear_down_tc: pointer to the function that tears down the test suite TestSuite::TestSuite(const char* a_name, const char* a_type_param, @@ -2962,34 +2999,71 @@ TestEventListener* repeater = UnitTest::GetInstance()->listeners().repeater(); // Call both legacy and the new API repeater->OnTestSuiteStart(*this); // Legacy API is deprecated but still available -#ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI +#ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ repeater->OnTestCaseStart(*this); -#endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI +#endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ impl->os_stack_trace_getter()->UponLeavingGTest(); internal::HandleExceptionsInMethodIfSupported( this, &TestSuite::RunSetUpTestSuite, "SetUpTestSuite()"); start_timestamp_ = internal::GetTimeInMillis(); + internal::Timer timer; for (int i = 0; i < total_test_count(); i++) { GetMutableTestInfo(i)->Run(); - } - elapsed_time_ = internal::GetTimeInMillis() - start_timestamp_; + if (GTEST_FLAG(fail_fast) && GetMutableTestInfo(i)->result()->Failed()) { + for (int j = i + 1; j < total_test_count(); j++) { + GetMutableTestInfo(j)->Skip(); + } + break; + } + } + elapsed_time_ = timer.Elapsed(); impl->os_stack_trace_getter()->UponLeavingGTest(); internal::HandleExceptionsInMethodIfSupported( this, &TestSuite::RunTearDownTestSuite, "TearDownTestSuite()"); // Call both legacy and the new API repeater->OnTestSuiteEnd(*this); // Legacy API is deprecated but still available -#ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI +#ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ repeater->OnTestCaseEnd(*this); -#endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI +#endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ + + impl->set_current_test_suite(nullptr); +} + +// Skips all tests under this TestSuite. +void TestSuite::Skip() { + if (!should_run_) return; + + internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); + impl->set_current_test_suite(this); + + TestEventListener* repeater = UnitTest::GetInstance()->listeners().repeater(); + + // Call both legacy and the new API + repeater->OnTestSuiteStart(*this); +// Legacy API is deprecated but still available +#ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ + repeater->OnTestCaseStart(*this); +#endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ + + for (int i = 0; i < total_test_count(); i++) { + GetMutableTestInfo(i)->Skip(); + } + + // Call both legacy and the new API + repeater->OnTestSuiteEnd(*this); + // Legacy API is deprecated but still available +#ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ + repeater->OnTestCaseEnd(*this); +#endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ impl->set_current_test_suite(nullptr); } // Clears the results of all tests in this test suite. @@ -3037,11 +3111,11 @@ // to "Failure", as the user usually doesn't care about the difference // between the two when viewing the test result. static const char * TestPartResultTypeToString(TestPartResult::Type type) { switch (type) { case TestPartResult::kSkip: - return "Skipped"; + return "Skipped\n"; case TestPartResult::kSuccess: return "Success"; case TestPartResult::kNonFatalFailure: case TestPartResult::kFatalFailure: @@ -3054,10 +3128,13 @@ return "Unknown result type"; } } namespace internal { +namespace { +enum class GTestColor { kDefault, kRed, kGreen, kYellow }; +} // namespace // Prints a TestPartResult to an std::string. static std::string PrintTestPartResultToString( const TestPartResult& test_part_result) { return (Message() @@ -3091,13 +3168,16 @@ !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT && !GTEST_OS_WINDOWS_MINGW // Returns the character attribute for the given color. static WORD GetColorAttribute(GTestColor color) { switch (color) { - case COLOR_RED: return FOREGROUND_RED; - case COLOR_GREEN: return FOREGROUND_GREEN; - case COLOR_YELLOW: return FOREGROUND_RED | FOREGROUND_GREEN; + case GTestColor::kRed: + return FOREGROUND_RED; + case GTestColor::kGreen: + return FOREGROUND_GREEN; + case GTestColor::kYellow: + return FOREGROUND_RED | FOREGROUND_GREEN; default: return 0; } } static int GetBitOffset(WORD color_mask) { @@ -3131,17 +3211,20 @@ return new_color; } #else -// Returns the ANSI color code for the given color. COLOR_DEFAULT is +// Returns the ANSI color code for the given color. GTestColor::kDefault is // an invalid input. static const char* GetAnsiColorCode(GTestColor color) { switch (color) { - case COLOR_RED: return "1"; - case COLOR_GREEN: return "2"; - case COLOR_YELLOW: return "3"; + case GTestColor::kRed: + return "1"; + case GTestColor::kGreen: + return "2"; + case GTestColor::kYellow: + return "3"; default: return nullptr; } } @@ -3186,21 +3269,23 @@ // Helpers for printing colored strings to stdout. Note that on Windows, we // cannot simply emit special characters and have the terminal change colors. // This routine must actually emit the characters rather than return a string // that would be colored when printed, as can be done on Linux. -void ColoredPrintf(GTestColor color, const char* fmt, ...) { + +GTEST_ATTRIBUTE_PRINTF_(2, 3) +static void ColoredPrintf(GTestColor color, const char *fmt, ...) { va_list args; va_start(args, fmt); #if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_ZOS || GTEST_OS_IOS || \ GTEST_OS_WINDOWS_PHONE || GTEST_OS_WINDOWS_RT || defined(ESP_PLATFORM) const bool use_color = AlwaysFalse(); #else static const bool in_color_mode = ShouldUseColor(posix::IsATTY(posix::FileNo(stdout)) != 0); - const bool use_color = in_color_mode && (color != COLOR_DEFAULT); + const bool use_color = in_color_mode && (color != GTestColor::kDefault); #endif // GTEST_OS_WINDOWS_MOBILE || GTEST_OS_ZOS if (!use_color) { vprintf(fmt, args); va_end(args); @@ -3308,47 +3393,46 @@ const char* const filter = GTEST_FLAG(filter).c_str(); // Prints the filter if it's not *. This reminds the user that some // tests may be skipped. if (!String::CStringEquals(filter, kUniversalFilter)) { - ColoredPrintf(COLOR_YELLOW, - "Note: %s filter = %s\n", GTEST_NAME_, filter); + ColoredPrintf(GTestColor::kYellow, "Note: %s filter = %s\n", GTEST_NAME_, + filter); } if (internal::ShouldShard(kTestTotalShards, kTestShardIndex, false)) { const int32_t shard_index = Int32FromEnvOrDie(kTestShardIndex, -1); - ColoredPrintf(COLOR_YELLOW, - "Note: This is test shard %d of %s.\n", + ColoredPrintf(GTestColor::kYellow, "Note: This is test shard %d of %s.\n", static_cast(shard_index) + 1, internal::posix::GetEnv(kTestTotalShards)); } if (GTEST_FLAG(shuffle)) { - ColoredPrintf(COLOR_YELLOW, + ColoredPrintf(GTestColor::kYellow, "Note: Randomizing tests' orders with a seed of %d .\n", unit_test.random_seed()); } - ColoredPrintf(COLOR_GREEN, "[==========] "); + ColoredPrintf(GTestColor::kGreen, "[==========] "); printf("Running %s from %s.\n", FormatTestCount(unit_test.test_to_run_count()).c_str(), FormatTestSuiteCount(unit_test.test_suite_to_run_count()).c_str()); fflush(stdout); } void PrettyUnitTestResultPrinter::OnEnvironmentsSetUpStart( const UnitTest& /*unit_test*/) { - ColoredPrintf(COLOR_GREEN, "[----------] "); + ColoredPrintf(GTestColor::kGreen, "[----------] "); printf("Global test environment set-up.\n"); fflush(stdout); } #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ void PrettyUnitTestResultPrinter::OnTestCaseStart(const TestCase& test_case) { const std::string counts = FormatCountableNoun(test_case.test_to_run_count(), "test", "tests"); - ColoredPrintf(COLOR_GREEN, "[----------] "); + ColoredPrintf(GTestColor::kGreen, "[----------] "); printf("%s from %s", counts.c_str(), test_case.name()); if (test_case.type_param() == nullptr) { printf("\n"); } else { printf(", where %s = %s\n", kTypeParamLabel, test_case.type_param()); @@ -3358,11 +3442,11 @@ #else void PrettyUnitTestResultPrinter::OnTestSuiteStart( const TestSuite& test_suite) { const std::string counts = FormatCountableNoun(test_suite.test_to_run_count(), "test", "tests"); - ColoredPrintf(COLOR_GREEN, "[----------] "); + ColoredPrintf(GTestColor::kGreen, "[----------] "); printf("%s from %s", counts.c_str(), test_suite.name()); if (test_suite.type_param() == nullptr) { printf("\n"); } else { printf(", where %s = %s\n", kTypeParamLabel, test_suite.type_param()); @@ -3370,11 +3454,11 @@ fflush(stdout); } #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ void PrettyUnitTestResultPrinter::OnTestStart(const TestInfo& test_info) { - ColoredPrintf(COLOR_GREEN, "[ RUN ] "); + ColoredPrintf(GTestColor::kGreen, "[ RUN ] "); PrintTestName(test_info.test_suite_name(), test_info.name()); printf("\n"); fflush(stdout); } @@ -3393,15 +3477,15 @@ } } void PrettyUnitTestResultPrinter::OnTestEnd(const TestInfo& test_info) { if (test_info.result()->Passed()) { - ColoredPrintf(COLOR_GREEN, "[ OK ] "); + ColoredPrintf(GTestColor::kGreen, "[ OK ] "); } else if (test_info.result()->Skipped()) { - ColoredPrintf(COLOR_GREEN, "[ SKIPPED ] "); + ColoredPrintf(GTestColor::kGreen, "[ SKIPPED ] "); } else { - ColoredPrintf(COLOR_RED, "[ FAILED ] "); + ColoredPrintf(GTestColor::kRed, "[ FAILED ] "); } PrintTestName(test_info.test_suite_name(), test_info.name()); if (test_info.result()->Failed()) PrintFullTestCommentIfPresent(test_info); @@ -3418,39 +3502,39 @@ void PrettyUnitTestResultPrinter::OnTestCaseEnd(const TestCase& test_case) { if (!GTEST_FLAG(print_time)) return; const std::string counts = FormatCountableNoun(test_case.test_to_run_count(), "test", "tests"); - ColoredPrintf(COLOR_GREEN, "[----------] "); + ColoredPrintf(GTestColor::kGreen, "[----------] "); printf("%s from %s (%s ms total)\n\n", counts.c_str(), test_case.name(), internal::StreamableToString(test_case.elapsed_time()).c_str()); fflush(stdout); } #else void PrettyUnitTestResultPrinter::OnTestSuiteEnd(const TestSuite& test_suite) { if (!GTEST_FLAG(print_time)) return; const std::string counts = FormatCountableNoun(test_suite.test_to_run_count(), "test", "tests"); - ColoredPrintf(COLOR_GREEN, "[----------] "); + ColoredPrintf(GTestColor::kGreen, "[----------] "); printf("%s from %s (%s ms total)\n\n", counts.c_str(), test_suite.name(), internal::StreamableToString(test_suite.elapsed_time()).c_str()); fflush(stdout); } #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ void PrettyUnitTestResultPrinter::OnEnvironmentsTearDownStart( const UnitTest& /*unit_test*/) { - ColoredPrintf(COLOR_GREEN, "[----------] "); + ColoredPrintf(GTestColor::kGreen, "[----------] "); printf("Global test environment tear-down\n"); fflush(stdout); } // Internal helper for printing the list of failed tests. void PrettyUnitTestResultPrinter::PrintFailedTests(const UnitTest& unit_test) { const int failed_test_count = unit_test.failed_test_count(); - ColoredPrintf(COLOR_RED, "[ FAILED ] "); + ColoredPrintf(GTestColor::kRed, "[ FAILED ] "); printf("%s, listed below:\n", FormatTestCount(failed_test_count).c_str()); for (int i = 0; i < unit_test.total_test_suite_count(); ++i) { const TestSuite& test_suite = *unit_test.GetTestSuite(i); if (!test_suite.should_run() || (test_suite.failed_test_count() == 0)) { @@ -3459,11 +3543,11 @@ for (int j = 0; j < test_suite.total_test_count(); ++j) { const TestInfo& test_info = *test_suite.GetTestInfo(j); if (!test_info.should_run() || !test_info.result()->Failed()) { continue; } - ColoredPrintf(COLOR_RED, "[ FAILED ] "); + ColoredPrintf(GTestColor::kRed, "[ FAILED ] "); printf("%s.%s", test_suite.name(), test_info.name()); PrintFullTestCommentIfPresent(test_info); printf("\n"); } } @@ -3480,11 +3564,11 @@ const TestSuite& test_suite = *unit_test.GetTestSuite(i); if (!test_suite.should_run()) { continue; } if (test_suite.ad_hoc_test_result().Failed()) { - ColoredPrintf(COLOR_RED, "[ FAILED ] "); + ColoredPrintf(GTestColor::kRed, "[ FAILED ] "); printf("%s: SetUpTestSuite or TearDownTestSuite\n", test_suite.name()); ++suite_failure_count; } } if (suite_failure_count > 0) { @@ -3508,34 +3592,34 @@ for (int j = 0; j < test_suite.total_test_count(); ++j) { const TestInfo& test_info = *test_suite.GetTestInfo(j); if (!test_info.should_run() || !test_info.result()->Skipped()) { continue; } - ColoredPrintf(COLOR_GREEN, "[ SKIPPED ] "); + ColoredPrintf(GTestColor::kGreen, "[ SKIPPED ] "); printf("%s.%s", test_suite.name(), test_info.name()); printf("\n"); } } } void PrettyUnitTestResultPrinter::OnTestIterationEnd(const UnitTest& unit_test, int /*iteration*/) { - ColoredPrintf(COLOR_GREEN, "[==========] "); + ColoredPrintf(GTestColor::kGreen, "[==========] "); printf("%s from %s ran.", FormatTestCount(unit_test.test_to_run_count()).c_str(), FormatTestSuiteCount(unit_test.test_suite_to_run_count()).c_str()); if (GTEST_FLAG(print_time)) { printf(" (%s ms total)", internal::StreamableToString(unit_test.elapsed_time()).c_str()); } printf("\n"); - ColoredPrintf(COLOR_GREEN, "[ PASSED ] "); + ColoredPrintf(GTestColor::kGreen, "[ PASSED ] "); printf("%s.\n", FormatTestCount(unit_test.successful_test_count()).c_str()); const int skipped_test_count = unit_test.skipped_test_count(); if (skipped_test_count > 0) { - ColoredPrintf(COLOR_GREEN, "[ SKIPPED ] "); + ColoredPrintf(GTestColor::kGreen, "[ SKIPPED ] "); printf("%s, listed below:\n", FormatTestCount(skipped_test_count).c_str()); PrintSkippedTests(unit_test); } if (!unit_test.Passed()) { @@ -3546,21 +3630,123 @@ int num_disabled = unit_test.reportable_disabled_test_count(); if (num_disabled && !GTEST_FLAG(also_run_disabled_tests)) { if (unit_test.Passed()) { printf("\n"); // Add a spacer if no FAILURE banner is displayed. } - ColoredPrintf(COLOR_YELLOW, - " YOU HAVE %d DISABLED %s\n\n", - num_disabled, - num_disabled == 1 ? "TEST" : "TESTS"); + ColoredPrintf(GTestColor::kYellow, " YOU HAVE %d DISABLED %s\n\n", + num_disabled, num_disabled == 1 ? "TEST" : "TESTS"); } // Ensure that Google Test output is printed before, e.g., heapchecker output. fflush(stdout); } // End PrettyUnitTestResultPrinter +// This class implements the TestEventListener interface. +// +// Class BriefUnitTestResultPrinter is copyable. +class BriefUnitTestResultPrinter : public TestEventListener { + public: + BriefUnitTestResultPrinter() {} + static void PrintTestName(const char* test_suite, const char* test) { + printf("%s.%s", test_suite, test); + } + + // The following methods override what's in the TestEventListener class. + void OnTestProgramStart(const UnitTest& /*unit_test*/) override {} + void OnTestIterationStart(const UnitTest& /*unit_test*/, + int /*iteration*/) override {} + void OnEnvironmentsSetUpStart(const UnitTest& /*unit_test*/) override {} + void OnEnvironmentsSetUpEnd(const UnitTest& /*unit_test*/) override {} +#ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ + void OnTestCaseStart(const TestCase& /*test_case*/) override {} +#else + void OnTestSuiteStart(const TestSuite& /*test_suite*/) override {} +#endif // OnTestCaseStart + + void OnTestStart(const TestInfo& /*test_info*/) override {} + + void OnTestPartResult(const TestPartResult& result) override; + void OnTestEnd(const TestInfo& test_info) override; +#ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ + void OnTestCaseEnd(const TestCase& /*test_case*/) override {} +#else + void OnTestSuiteEnd(const TestSuite& /*test_suite*/) override {} +#endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ + + void OnEnvironmentsTearDownStart(const UnitTest& /*unit_test*/) override {} + void OnEnvironmentsTearDownEnd(const UnitTest& /*unit_test*/) override {} + void OnTestIterationEnd(const UnitTest& unit_test, int iteration) override; + void OnTestProgramEnd(const UnitTest& /*unit_test*/) override {} +}; + +// Called after an assertion failure. +void BriefUnitTestResultPrinter::OnTestPartResult( + const TestPartResult& result) { + switch (result.type()) { + // If the test part succeeded, we don't need to do anything. + case TestPartResult::kSuccess: + return; + default: + // Print failure message from the assertion + // (e.g. expected this and got that). + PrintTestPartResult(result); + fflush(stdout); + } +} + +void BriefUnitTestResultPrinter::OnTestEnd(const TestInfo& test_info) { + if (test_info.result()->Failed()) { + ColoredPrintf(GTestColor::kRed, "[ FAILED ] "); + PrintTestName(test_info.test_suite_name(), test_info.name()); + PrintFullTestCommentIfPresent(test_info); + + if (GTEST_FLAG(print_time)) { + printf(" (%s ms)\n", + internal::StreamableToString(test_info.result()->elapsed_time()) + .c_str()); + } else { + printf("\n"); + } + fflush(stdout); + } +} + +void BriefUnitTestResultPrinter::OnTestIterationEnd(const UnitTest& unit_test, + int /*iteration*/) { + ColoredPrintf(GTestColor::kGreen, "[==========] "); + printf("%s from %s ran.", + FormatTestCount(unit_test.test_to_run_count()).c_str(), + FormatTestSuiteCount(unit_test.test_suite_to_run_count()).c_str()); + if (GTEST_FLAG(print_time)) { + printf(" (%s ms total)", + internal::StreamableToString(unit_test.elapsed_time()).c_str()); + } + printf("\n"); + ColoredPrintf(GTestColor::kGreen, "[ PASSED ] "); + printf("%s.\n", FormatTestCount(unit_test.successful_test_count()).c_str()); + + const int skipped_test_count = unit_test.skipped_test_count(); + if (skipped_test_count > 0) { + ColoredPrintf(GTestColor::kGreen, "[ SKIPPED ] "); + printf("%s.\n", FormatTestCount(skipped_test_count).c_str()); + } + + int num_disabled = unit_test.reportable_disabled_test_count(); + if (num_disabled && !GTEST_FLAG(also_run_disabled_tests)) { + if (unit_test.Passed()) { + printf("\n"); // Add a spacer if no FAILURE banner is displayed. + } + ColoredPrintf(GTestColor::kYellow, " YOU HAVE %d DISABLED %s\n\n", + num_disabled, num_disabled == 1 ? "TEST" : "TESTS"); + } + // Ensure that Google Test output is printed before, e.g., heapchecker output. + fflush(stdout); +} + +// End BriefUnitTestResultPrinter + // class TestEventRepeater // // This class forwards events to other event listeners. class TestEventRepeater : public TestEventListener { public: @@ -3740,10 +3926,20 @@ const std::string& value); // Streams an XML CDATA section, escaping invalid CDATA sequences as needed. static void OutputXmlCDataSection(::std::ostream* stream, const char* data); + // Streams a test suite XML stanza containing the given test result. + // + // Requires: result.Failed() + static void OutputXmlTestSuiteForTestResult(::std::ostream* stream, + const TestResult& result); + + // Streams an XML representation of a TestResult object. + static void OutputXmlTestResult(::std::ostream* stream, + const TestResult& result); + // Streams an XML representation of a TestInfo object. static void OutputXmlTestInfo(::std::ostream* stream, const char* test_suite_name, const TestInfo& test_info); @@ -3898,10 +4094,14 @@ // Windows' localtime(), which has a thread-local tm buffer. struct tm* tm_ptr = localtime(&seconds); // NOLINT if (tm_ptr == nullptr) return false; *out = *tm_ptr; return true; +#elif defined(__STDC_LIB_EXT1__) + // Uses localtime_s when available as localtime_r is only available from + // C23 standard. + return localtime_s(&seconds, out) != nullptr; #else return localtime_r(&seconds, out) != nullptr; #endif } @@ -3909,17 +4109,18 @@ // 8601 format, without the timezone information. std::string FormatEpochTimeInMillisAsIso8601(TimeInMillis ms) { struct tm time_struct; if (!PortableLocaltime(static_cast(ms / 1000), &time_struct)) return ""; - // YYYY-MM-DDThh:mm:ss + // YYYY-MM-DDThh:mm:ss.sss return StreamableToString(time_struct.tm_year + 1900) + "-" + String::FormatIntWidth2(time_struct.tm_mon + 1) + "-" + String::FormatIntWidth2(time_struct.tm_mday) + "T" + String::FormatIntWidth2(time_struct.tm_hour) + ":" + String::FormatIntWidth2(time_struct.tm_min) + ":" + - String::FormatIntWidth2(time_struct.tm_sec); + String::FormatIntWidth2(time_struct.tm_sec) + "." + + String::FormatIntWidthN(static_cast(ms % 1000), 3); } // Streams an XML CDATA section, escaping invalid CDATA sequences as needed. void XmlUnitTestResultPrinter::OutputXmlCDataSection(::std::ostream* stream, const char* data) { @@ -3954,10 +4155,47 @@ << ">."; *stream << " " << name << "=\"" << EscapeXmlAttribute(value) << "\""; } +// Streams a test suite XML stanza containing the given test result. +void XmlUnitTestResultPrinter::OutputXmlTestSuiteForTestResult( + ::std::ostream* stream, const TestResult& result) { + // Output the boilerplate for a minimal test suite with one test. + *stream << " "; + + // Output the boilerplate for a minimal test case with a single test. + *stream << " \n"; +} + // Prints an XML representation of a TestInfo object. void XmlUnitTestResultPrinter::OutputXmlTestInfo(::std::ostream* stream, const char* test_suite_name, const TestInfo& test_info) { const TestResult& result = *test_info.result(); @@ -3997,34 +4235,53 @@ OutputXmlAttribute( stream, kTestsuite, "timestamp", FormatEpochTimeInMillisAsIso8601(result.start_timestamp())); OutputXmlAttribute(stream, kTestsuite, "classname", test_suite_name); + OutputXmlTestResult(stream, result); +} + +void XmlUnitTestResultPrinter::OutputXmlTestResult(::std::ostream* stream, + const TestResult& result) { int failures = 0; + int skips = 0; for (int i = 0; i < result.total_part_count(); ++i) { const TestPartResult& part = result.GetTestPartResult(i); if (part.failed()) { - if (++failures == 1) { + if (++failures == 1 && skips == 0) { *stream << ">\n"; } const std::string location = internal::FormatCompilerIndependentFileLocation(part.file_name(), part.line_number()); const std::string summary = location + "\n" + part.summary(); *stream << " "; const std::string detail = location + "\n" + part.message(); OutputXmlCDataSection(stream, RemoveInvalidXmlCharacters(detail).c_str()); *stream << "\n"; + } else if (part.skipped()) { + if (++skips == 1 && failures == 0) { + *stream << ">\n"; + } + const std::string location = + internal::FormatCompilerIndependentFileLocation(part.file_name(), + part.line_number()); + const std::string summary = location + "\n" + part.summary(); + *stream << " "; + const std::string detail = location + "\n" + part.message(); + OutputXmlCDataSection(stream, RemoveInvalidXmlCharacters(detail).c_str()); + *stream << "\n"; } } - if (failures == 0 && result.test_property_count() == 0) { + if (failures == 0 && skips == 0 && result.test_property_count() == 0) { *stream << " />\n"; } else { - if (failures == 0) { + if (failures == 0 && skips == 0) { *stream << ">\n"; } OutputXmlTestProperties(stream, result); *stream << " \n"; } @@ -4042,11 +4299,15 @@ OutputXmlAttribute(stream, kTestsuite, "failures", StreamableToString(test_suite.failed_test_count())); OutputXmlAttribute( stream, kTestsuite, "disabled", StreamableToString(test_suite.reportable_disabled_test_count())); + OutputXmlAttribute(stream, kTestsuite, "skipped", + StreamableToString(test_suite.skipped_test_count())); + OutputXmlAttribute(stream, kTestsuite, "errors", "0"); + OutputXmlAttribute(stream, kTestsuite, "time", FormatTimeInMillisAsSeconds(test_suite.elapsed_time())); OutputXmlAttribute( stream, kTestsuite, "timestamp", FormatEpochTimeInMillisAsIso8601(test_suite.start_timestamp())); @@ -4093,10 +4354,17 @@ for (int i = 0; i < unit_test.total_test_suite_count(); ++i) { if (unit_test.GetTestSuite(i)->reportable_test_count() > 0) PrintXmlTestSuite(stream, *unit_test.GetTestSuite(i)); } + + // If there was a test failure outside of one of the test suites (like in a + // test environment) include that in the output. + if (unit_test.ad_hoc_test_result().Failed()) { + OutputXmlTestSuiteForTestResult(stream, unit_test.ad_hoc_test_result()); + } + *stream << "\n"; } void XmlUnitTestResultPrinter::PrintXmlTestsList( std::ostream* stream, const std::vector& test_suites) { @@ -4183,10 +4451,20 @@ const std::string& name, int value, const std::string& indent, bool comma = true); + // Streams a test suite JSON stanza containing the given test result. + // + // Requires: result.Failed() + static void OutputJsonTestSuiteForTestResult(::std::ostream* stream, + const TestResult& result); + + // Streams a JSON representation of a TestResult object. + static void OutputJsonTestResult(::std::ostream* stream, + const TestResult& result); + // Streams a JSON representation of a TestInfo object. static void OutputJsonTestInfo(::std::ostream* stream, const char* test_suite_name, const TestInfo& test_info); @@ -4333,10 +4611,52 @@ *stream << indent << "\"" << name << "\": " << StreamableToString(value); if (comma) *stream << ",\n"; } +// Streams a test suite JSON stanza containing the given test result. +void JsonUnitTestResultPrinter::OutputJsonTestSuiteForTestResult( + ::std::ostream* stream, const TestResult& result) { + // Output the boilerplate for a new test suite. + *stream << Indent(4) << "{\n"; + OutputJsonKey(stream, "testsuite", "name", "NonTestSuiteFailure", Indent(6)); + OutputJsonKey(stream, "testsuite", "tests", 1, Indent(6)); + if (!GTEST_FLAG(list_tests)) { + OutputJsonKey(stream, "testsuite", "failures", 1, Indent(6)); + OutputJsonKey(stream, "testsuite", "disabled", 0, Indent(6)); + OutputJsonKey(stream, "testsuite", "skipped", 0, Indent(6)); + OutputJsonKey(stream, "testsuite", "errors", 0, Indent(6)); + OutputJsonKey(stream, "testsuite", "time", + FormatTimeInMillisAsDuration(result.elapsed_time()), + Indent(6)); + OutputJsonKey(stream, "testsuite", "timestamp", + FormatEpochTimeInMillisAsRFC3339(result.start_timestamp()), + Indent(6)); + } + *stream << Indent(6) << "\"testsuite\": [\n"; + + // Output the boilerplate for a new test case. + *stream << Indent(8) << "{\n"; + OutputJsonKey(stream, "testcase", "name", "", Indent(10)); + OutputJsonKey(stream, "testcase", "status", "RUN", Indent(10)); + OutputJsonKey(stream, "testcase", "result", "COMPLETED", Indent(10)); + OutputJsonKey(stream, "testcase", "timestamp", + FormatEpochTimeInMillisAsRFC3339(result.start_timestamp()), + Indent(10)); + OutputJsonKey(stream, "testcase", "time", + FormatTimeInMillisAsDuration(result.elapsed_time()), + Indent(10)); + OutputJsonKey(stream, "testcase", "classname", "", Indent(10), false); + *stream << TestPropertiesAsJson(result, Indent(10)); + + // Output the actual test result. + OutputJsonTestResult(stream, result); + + // Finish the test suite. + *stream << "\n" << Indent(6) << "]\n" << Indent(4) << "}"; +} + // Prints a JSON representation of a TestInfo object. void JsonUnitTestResultPrinter::OutputJsonTestInfo(::std::ostream* stream, const char* test_suite_name, const TestInfo& test_info) { const TestResult& result = *test_info.result(); @@ -4375,10 +4695,17 @@ FormatTimeInMillisAsDuration(result.elapsed_time()), kIndent); OutputJsonKey(stream, kTestsuite, "classname", test_suite_name, kIndent, false); *stream << TestPropertiesAsJson(result, kIndent); + OutputJsonTestResult(stream, result); +} + +void JsonUnitTestResultPrinter::OutputJsonTestResult(::std::ostream* stream, + const TestResult& result) { + const std::string kIndent = Indent(10); + int failures = 0; for (int i = 0; i < result.total_part_count(); ++i) { const TestPartResult& part = result.GetTestPartResult(i); if (part.failed()) { *stream << ",\n"; @@ -4485,10 +4812,16 @@ } PrintJsonTestSuite(stream, *unit_test.GetTestSuite(i)); } } + // If there was a test failure outside of one of the test suites (like in a + // test environment) include that in the output. + if (unit_test.ad_hoc_test_result().Failed()) { + OutputJsonTestSuiteForTestResult(stream, unit_test.ad_hoc_test_result()); + } + *stream << "\n" << kIndent << "]\n" << "}\n"; } void JsonUnitTestResultPrinter::PrintJsonTestList( std::ostream* stream, const std::vector& test_suites) { @@ -5307,10 +5640,14 @@ // Configures listeners for XML output. This makes it possible for users // to shut down the default XML output before invoking RUN_ALL_TESTS. ConfigureXmlOutput(); + if (GTEST_FLAG(brief)) { + listeners()->SetDefaultResultPrinter(new BriefUnitTestResultPrinter); + } + #if GTEST_CAN_STREAM_RESULTS_ // Configures listeners for streaming test results to the specified server. ConfigureStreamingOutput(); #endif // GTEST_CAN_STREAM_RESULTS_ @@ -5352,14 +5689,14 @@ // TESTS ARE NOT SHUFFLED. // // Arguments: // // test_suite_name: name of the test suite -// type_param: the name of the test suite's type parameter, or NULL if -// this is not a typed or a type-parameterized test suite. -// set_up_tc: pointer to the function that sets up the test suite -// tear_down_tc: pointer to the function that tears down the test suite +// type_param: the name of the test suite's type parameter, or NULL if +// this is not a typed or a type-parameterized test suite. +// set_up_tc: pointer to the function that sets up the test suite +// tear_down_tc: pointer to the function that tears down the test suite TestSuite* UnitTestImpl::GetTestSuite( const char* test_suite_name, const char* type_param, internal::SetUpTestSuiteFunc set_up_tc, internal::TearDownTestSuiteFunc tear_down_tc) { // Can we find a TestSuite with the given name? @@ -5473,11 +5810,11 @@ for (int i = 0; gtest_repeat_forever || i != repeat; i++) { // We want to preserve failures generated by ad-hoc test // assertions executed before RUN_ALL_TESTS(). ClearNonAdHocTestResult(); - const TimeInMillis start = GetTimeInMillis(); + Timer timer; // Shuffles test suites and tests if requested. if (has_tests_to_run && GTEST_FLAG(shuffle)) { random()->Reseed(static_cast(random_seed_)); // This should be done before calling OnTestIterationStart(), @@ -5514,21 +5851,36 @@ fflush(stdout); } else if (!Test::HasFatalFailure()) { for (int test_index = 0; test_index < total_test_suite_count(); test_index++) { GetMutableSuiteCase(test_index)->Run(); + if (GTEST_FLAG(fail_fast) && + GetMutableSuiteCase(test_index)->Failed()) { + for (int j = test_index + 1; j < total_test_suite_count(); j++) { + GetMutableSuiteCase(j)->Skip(); + } + break; + } + } + } else if (Test::HasFatalFailure()) { + // If there was a fatal failure during the global setup then we know we + // aren't going to run any tests. Explicitly mark all of the tests as + // skipped to make this obvious in the output. + for (int test_index = 0; test_index < total_test_suite_count(); + test_index++) { + GetMutableSuiteCase(test_index)->Skip(); } } // Tears down all environments in reverse order afterwards. repeater->OnEnvironmentsTearDownStart(*parent_); std::for_each(environments_.rbegin(), environments_.rend(), TearDownEnvironment); repeater->OnEnvironmentsTearDownEnd(*parent_); } - elapsed_time_ = GetTimeInMillis() - start; + elapsed_time_ = timer.Elapsed(); // Tells the unit test event listener that the tests have just finished. repeater->OnTestIterationEnd(*parent_, i); // Gets the result and clears it. @@ -5552,18 +5904,18 @@ repeater->OnTestProgramEnd(*parent_); if (!gtest_is_initialized_before_run_all_tests) { ColoredPrintf( - COLOR_RED, + GTestColor::kRed, "\nIMPORTANT NOTICE - DO NOT IGNORE:\n" "This test program did NOT call " GTEST_INIT_GOOGLE_TEST_NAME_ "() before calling RUN_ALL_TESTS(). This is INVALID. Soon " GTEST_NAME_ " will start to enforce the valid usage. " "Please fix it ASAP, or IT WILL START TO FAIL.\n"); // NOLINT #if GTEST_FOR_GOOGLE_ - ColoredPrintf(COLOR_RED, + ColoredPrintf(GTestColor::kRed, "For more details, see http://wiki/Main/ValidGUnitMain.\n"); #endif // GTEST_FOR_GOOGLE_ } return !failed; @@ -5576,11 +5928,11 @@ void WriteToShardStatusFileIfNeeded() { const char* const test_shard_file = posix::GetEnv(kTestShardStatusFile); if (test_shard_file != nullptr) { FILE* const file = posix::FOpen(test_shard_file, "w"); if (file == nullptr) { - ColoredPrintf(COLOR_RED, + ColoredPrintf(GTestColor::kRed, "Could not write to the test shard status file \"%s\" " "specified by the %s environment variable.\n", test_shard_file, kTestShardStatusFile); fflush(stdout); exit(EXIT_FAILURE); @@ -5610,28 +5962,28 @@ } else if (total_shards == -1 && shard_index != -1) { const Message msg = Message() << "Invalid environment variables: you have " << kTestShardIndex << " = " << shard_index << ", but have left " << kTestTotalShards << " unset.\n"; - ColoredPrintf(COLOR_RED, "%s", msg.GetString().c_str()); + ColoredPrintf(GTestColor::kRed, "%s", msg.GetString().c_str()); fflush(stdout); exit(EXIT_FAILURE); } else if (total_shards != -1 && shard_index == -1) { const Message msg = Message() << "Invalid environment variables: you have " << kTestTotalShards << " = " << total_shards << ", but have left " << kTestShardIndex << " unset.\n"; - ColoredPrintf(COLOR_RED, "%s", msg.GetString().c_str()); + ColoredPrintf(GTestColor::kRed, "%s", msg.GetString().c_str()); fflush(stdout); exit(EXIT_FAILURE); } else if (shard_index < 0 || shard_index >= total_shards) { const Message msg = Message() << "Invalid environment variables: we require 0 <= " << kTestShardIndex << " < " << kTestTotalShards << ", but you have " << kTestShardIndex << "=" << shard_index << ", " << kTestTotalShards << "=" << total_shards << ".\n"; - ColoredPrintf(COLOR_RED, "%s", msg.GetString().c_str()); + ColoredPrintf(GTestColor::kRed, "%s", msg.GetString().c_str()); fflush(stdout); exit(EXIT_FAILURE); } return total_shards > 1; @@ -6017,11 +6369,11 @@ // @G changes the color to green. // @Y changes the color to yellow. // @D changes to the default terminal text color. // static void PrintColorEncoded(const char* str) { - GTestColor color = COLOR_DEFAULT; // The current color. + GTestColor color = GTestColor::kDefault; // The current color. // Conceptually, we split the string into segments divided by escape // sequences. Then we print one segment at a time. At the end of // each iteration, the str pointer advances to the beginning of the // next segment. @@ -6037,116 +6389,144 @@ const char ch = p[1]; str = p + 2; if (ch == '@') { ColoredPrintf(color, "@"); } else if (ch == 'D') { - color = COLOR_DEFAULT; + color = GTestColor::kDefault; } else if (ch == 'R') { - color = COLOR_RED; + color = GTestColor::kRed; } else if (ch == 'G') { - color = COLOR_GREEN; + color = GTestColor::kGreen; } else if (ch == 'Y') { - color = COLOR_YELLOW; + color = GTestColor::kYellow; } else { --str; } } } static const char kColorEncodedHelpMessage[] = -"This program contains tests written using " GTEST_NAME_ ". You can use the\n" -"following command line flags to control its behavior:\n" -"\n" -"Test Selection:\n" -" @G--" GTEST_FLAG_PREFIX_ "list_tests@D\n" -" List the names of all tests instead of running them. The name of\n" -" TEST(Foo, Bar) is \"Foo.Bar\".\n" -" @G--" GTEST_FLAG_PREFIX_ "filter=@YPOSTIVE_PATTERNS" + "This program contains tests written using " GTEST_NAME_ + ". You can use the\n" + "following command line flags to control its behavior:\n" + "\n" + "Test Selection:\n" + " @G--" GTEST_FLAG_PREFIX_ + "list_tests@D\n" + " List the names of all tests instead of running them. The name of\n" + " TEST(Foo, Bar) is \"Foo.Bar\".\n" + " @G--" GTEST_FLAG_PREFIX_ + "filter=@YPOSITIVE_PATTERNS" "[@G-@YNEGATIVE_PATTERNS]@D\n" -" Run only the tests whose name matches one of the positive patterns but\n" -" none of the negative patterns. '?' matches any single character; '*'\n" -" matches any substring; ':' separates two patterns.\n" -" @G--" GTEST_FLAG_PREFIX_ "also_run_disabled_tests@D\n" -" Run all disabled tests too.\n" -"\n" -"Test Execution:\n" -" @G--" GTEST_FLAG_PREFIX_ "repeat=@Y[COUNT]@D\n" -" Run the tests repeatedly; use a negative count to repeat forever.\n" -" @G--" GTEST_FLAG_PREFIX_ "shuffle@D\n" -" Randomize tests' orders on every iteration.\n" -" @G--" GTEST_FLAG_PREFIX_ "random_seed=@Y[NUMBER]@D\n" -" Random number seed to use for shuffling test orders (between 1 and\n" -" 99999, or 0 to use a seed based on the current time).\n" -"\n" -"Test Output:\n" -" @G--" GTEST_FLAG_PREFIX_ "color=@Y(@Gyes@Y|@Gno@Y|@Gauto@Y)@D\n" -" Enable/disable colored output. The default is @Gauto@D.\n" -" -@G-" GTEST_FLAG_PREFIX_ "print_time=0@D\n" -" Don't print the elapsed time of each test.\n" -" @G--" GTEST_FLAG_PREFIX_ "output=@Y(@Gjson@Y|@Gxml@Y)[@G:@YDIRECTORY_PATH@G" - GTEST_PATH_SEP_ "@Y|@G:@YFILE_PATH]@D\n" -" Generate a JSON or XML report in the given directory or with the given\n" -" file name. @YFILE_PATH@D defaults to @Gtest_detail.xml@D.\n" + " Run only the tests whose name matches one of the positive patterns " + "but\n" + " none of the negative patterns. '?' matches any single character; " + "'*'\n" + " matches any substring; ':' separates two patterns.\n" + " @G--" GTEST_FLAG_PREFIX_ + "also_run_disabled_tests@D\n" + " Run all disabled tests too.\n" + "\n" + "Test Execution:\n" + " @G--" GTEST_FLAG_PREFIX_ + "repeat=@Y[COUNT]@D\n" + " Run the tests repeatedly; use a negative count to repeat forever.\n" + " @G--" GTEST_FLAG_PREFIX_ + "shuffle@D\n" + " Randomize tests' orders on every iteration.\n" + " @G--" GTEST_FLAG_PREFIX_ + "random_seed=@Y[NUMBER]@D\n" + " Random number seed to use for shuffling test orders (between 1 and\n" + " 99999, or 0 to use a seed based on the current time).\n" + "\n" + "Test Output:\n" + " @G--" GTEST_FLAG_PREFIX_ + "color=@Y(@Gyes@Y|@Gno@Y|@Gauto@Y)@D\n" + " Enable/disable colored output. The default is @Gauto@D.\n" + " @G--" GTEST_FLAG_PREFIX_ + "brief=1@D\n" + " Only print test failures.\n" + " @G--" GTEST_FLAG_PREFIX_ + "print_time=0@D\n" + " Don't print the elapsed time of each test.\n" + " @G--" GTEST_FLAG_PREFIX_ + "output=@Y(@Gjson@Y|@Gxml@Y)[@G:@YDIRECTORY_PATH@G" GTEST_PATH_SEP_ + "@Y|@G:@YFILE_PATH]@D\n" + " Generate a JSON or XML report in the given directory or with the " + "given\n" + " file name. @YFILE_PATH@D defaults to @Gtest_detail.xml@D.\n" # if GTEST_CAN_STREAM_RESULTS_ -" @G--" GTEST_FLAG_PREFIX_ "stream_result_to=@YHOST@G:@YPORT@D\n" -" Stream test results to the given server.\n" + " @G--" GTEST_FLAG_PREFIX_ + "stream_result_to=@YHOST@G:@YPORT@D\n" + " Stream test results to the given server.\n" # endif // GTEST_CAN_STREAM_RESULTS_ -"\n" -"Assertion Behavior:\n" + "\n" + "Assertion Behavior:\n" # if GTEST_HAS_DEATH_TEST && !GTEST_OS_WINDOWS -" @G--" GTEST_FLAG_PREFIX_ "death_test_style=@Y(@Gfast@Y|@Gthreadsafe@Y)@D\n" -" Set the default death test style.\n" + " @G--" GTEST_FLAG_PREFIX_ + "death_test_style=@Y(@Gfast@Y|@Gthreadsafe@Y)@D\n" + " Set the default death test style.\n" # endif // GTEST_HAS_DEATH_TEST && !GTEST_OS_WINDOWS -" @G--" GTEST_FLAG_PREFIX_ "break_on_failure@D\n" -" Turn assertion failures into debugger break-points.\n" -" @G--" GTEST_FLAG_PREFIX_ "throw_on_failure@D\n" -" Turn assertion failures into C++ exceptions for use by an external\n" -" test framework.\n" -" @G--" GTEST_FLAG_PREFIX_ "catch_exceptions=0@D\n" -" Do not report exceptions as test failures. Instead, allow them\n" -" to crash the program or throw a pop-up (on Windows).\n" -"\n" -"Except for @G--" GTEST_FLAG_PREFIX_ "list_tests@D, you can alternatively set " + " @G--" GTEST_FLAG_PREFIX_ + "break_on_failure@D\n" + " Turn assertion failures into debugger break-points.\n" + " @G--" GTEST_FLAG_PREFIX_ + "throw_on_failure@D\n" + " Turn assertion failures into C++ exceptions for use by an external\n" + " test framework.\n" + " @G--" GTEST_FLAG_PREFIX_ + "catch_exceptions=0@D\n" + " Do not report exceptions as test failures. Instead, allow them\n" + " to crash the program or throw a pop-up (on Windows).\n" + "\n" + "Except for @G--" GTEST_FLAG_PREFIX_ + "list_tests@D, you can alternatively set " "the corresponding\n" -"environment variable of a flag (all letters in upper-case). For example, to\n" -"disable colored text output, you can either specify @G--" GTEST_FLAG_PREFIX_ + "environment variable of a flag (all letters in upper-case). For example, " + "to\n" + "disable colored text output, you can either specify " + "@G--" GTEST_FLAG_PREFIX_ "color=no@D or set\n" -"the @G" GTEST_FLAG_PREFIX_UPPER_ "COLOR@D environment variable to @Gno@D.\n" -"\n" -"For more information, please read the " GTEST_NAME_ " documentation at\n" -"@G" GTEST_PROJECT_URL_ "@D. If you find a bug in " GTEST_NAME_ "\n" -"(not one in your own code or tests), please report it to\n" -"@G<" GTEST_DEV_EMAIL_ ">@D.\n"; + "the @G" GTEST_FLAG_PREFIX_UPPER_ + "COLOR@D environment variable to @Gno@D.\n" + "\n" + "For more information, please read the " GTEST_NAME_ + " documentation at\n" + "@G" GTEST_PROJECT_URL_ "@D. If you find a bug in " GTEST_NAME_ + "\n" + "(not one in your own code or tests), please report it to\n" + "@G<" GTEST_DEV_EMAIL_ ">@D.\n"; static bool ParseGoogleTestFlag(const char* const arg) { return ParseBoolFlag(arg, kAlsoRunDisabledTestsFlag, >EST_FLAG(also_run_disabled_tests)) || - ParseBoolFlag(arg, kBreakOnFailureFlag, - >EST_FLAG(break_on_failure)) || - ParseBoolFlag(arg, kCatchExceptionsFlag, - >EST_FLAG(catch_exceptions)) || - ParseStringFlag(arg, kColorFlag, >EST_FLAG(color)) || - ParseStringFlag(arg, kDeathTestStyleFlag, - >EST_FLAG(death_test_style)) || - ParseBoolFlag(arg, kDeathTestUseFork, - >EST_FLAG(death_test_use_fork)) || - ParseStringFlag(arg, kFilterFlag, >EST_FLAG(filter)) || - ParseStringFlag(arg, kInternalRunDeathTestFlag, - >EST_FLAG(internal_run_death_test)) || - ParseBoolFlag(arg, kListTestsFlag, >EST_FLAG(list_tests)) || - ParseStringFlag(arg, kOutputFlag, >EST_FLAG(output)) || - ParseBoolFlag(arg, kPrintTimeFlag, >EST_FLAG(print_time)) || - ParseBoolFlag(arg, kPrintUTF8Flag, >EST_FLAG(print_utf8)) || - ParseInt32Flag(arg, kRandomSeedFlag, >EST_FLAG(random_seed)) || - ParseInt32Flag(arg, kRepeatFlag, >EST_FLAG(repeat)) || - ParseBoolFlag(arg, kShuffleFlag, >EST_FLAG(shuffle)) || - ParseInt32Flag(arg, kStackTraceDepthFlag, - >EST_FLAG(stack_trace_depth)) || - ParseStringFlag(arg, kStreamResultToFlag, - >EST_FLAG(stream_result_to)) || - ParseBoolFlag(arg, kThrowOnFailureFlag, - >EST_FLAG(throw_on_failure)); + ParseBoolFlag(arg, kBreakOnFailureFlag, + >EST_FLAG(break_on_failure)) || + ParseBoolFlag(arg, kCatchExceptionsFlag, + >EST_FLAG(catch_exceptions)) || + ParseStringFlag(arg, kColorFlag, >EST_FLAG(color)) || + ParseStringFlag(arg, kDeathTestStyleFlag, + >EST_FLAG(death_test_style)) || + ParseBoolFlag(arg, kDeathTestUseFork, + >EST_FLAG(death_test_use_fork)) || + ParseBoolFlag(arg, kFailFast, >EST_FLAG(fail_fast)) || + ParseStringFlag(arg, kFilterFlag, >EST_FLAG(filter)) || + ParseStringFlag(arg, kInternalRunDeathTestFlag, + >EST_FLAG(internal_run_death_test)) || + ParseBoolFlag(arg, kListTestsFlag, >EST_FLAG(list_tests)) || + ParseStringFlag(arg, kOutputFlag, >EST_FLAG(output)) || + ParseBoolFlag(arg, kBriefFlag, >EST_FLAG(brief)) || + ParseBoolFlag(arg, kPrintTimeFlag, >EST_FLAG(print_time)) || + ParseBoolFlag(arg, kPrintUTF8Flag, >EST_FLAG(print_utf8)) || + ParseInt32Flag(arg, kRandomSeedFlag, >EST_FLAG(random_seed)) || + ParseInt32Flag(arg, kRepeatFlag, >EST_FLAG(repeat)) || + ParseBoolFlag(arg, kShuffleFlag, >EST_FLAG(shuffle)) || + ParseInt32Flag(arg, kStackTraceDepthFlag, + >EST_FLAG(stack_trace_depth)) || + ParseStringFlag(arg, kStreamResultToFlag, + >EST_FLAG(stream_result_to)) || + ParseBoolFlag(arg, kThrowOnFailureFlag, >EST_FLAG(throw_on_failure)); } #if GTEST_USE_OWN_FLAGFILE_FLAG_ static void LoadFlagsFromFile(const std::string& path) { FILE* flagfile = posix::FOpen(path.c_str(), "r"); @@ -6312,28 +6692,35 @@ } std::string TempDir() { #if defined(GTEST_CUSTOM_TEMPDIR_FUNCTION_) return GTEST_CUSTOM_TEMPDIR_FUNCTION_(); -#endif - -#if GTEST_OS_WINDOWS_MOBILE +#elif GTEST_OS_WINDOWS_MOBILE return "\\temp\\"; #elif GTEST_OS_WINDOWS const char* temp_dir = internal::posix::GetEnv("TEMP"); - if (temp_dir == nullptr || temp_dir[0] == '\0') + if (temp_dir == nullptr || temp_dir[0] == '\0') { return "\\temp\\"; - else if (temp_dir[strlen(temp_dir) - 1] == '\\') + } else if (temp_dir[strlen(temp_dir) - 1] == '\\') { return temp_dir; - else + } else { return std::string(temp_dir) + "\\"; + } #elif GTEST_OS_LINUX_ANDROID const char* temp_dir = internal::posix::GetEnv("TEST_TMPDIR"); - if (temp_dir == nullptr || temp_dir[0] == '\0') + if (temp_dir == nullptr || temp_dir[0] == '\0') { return "/data/local/tmp/"; - else + } else { return temp_dir; + } +#elif GTEST_OS_LINUX + const char* temp_dir = internal::posix::GetEnv("TEST_TMPDIR"); + if (temp_dir == nullptr || temp_dir[0] == '\0') { + return "/tmp/"; + } else { + return temp_dir; + } #else return "/tmp/"; #endif // GTEST_OS_WINDOWS_MOBILE } diff --git a/media/libvpx/libvpx/tools_common.h b/media/libvpx/libvpx/tools_common.h --- a/media/libvpx/libvpx/tools_common.h +++ b/media/libvpx/libvpx/tools_common.h @@ -114,16 +114,28 @@ #define VPX_NO_RETURN __declspec(noreturn) #else #define VPX_NO_RETURN #endif +// Tells the compiler to perform `printf` format string checking if the +// compiler supports it; see the 'format' attribute in +// . +#define VPX_TOOLS_FORMAT_PRINTF(string_index, first_to_check) +#if defined(__has_attribute) +#if __has_attribute(format) +#undef VPX_TOOLS_FORMAT_PRINTF +#define VPX_TOOLS_FORMAT_PRINTF(string_index, first_to_check) \ + __attribute__((__format__(__printf__, string_index, first_to_check))) +#endif +#endif + /* Sets a stdio stream into binary mode */ FILE *set_binary_mode(FILE *stream); -VPX_NO_RETURN void die(const char *fmt, ...); -VPX_NO_RETURN void fatal(const char *fmt, ...); -void warn(const char *fmt, ...); +VPX_NO_RETURN void die(const char *fmt, ...) VPX_TOOLS_FORMAT_PRINTF(1, 2); +VPX_NO_RETURN void fatal(const char *fmt, ...) VPX_TOOLS_FORMAT_PRINTF(1, 2); +void warn(const char *fmt, ...) VPX_TOOLS_FORMAT_PRINTF(1, 2); VPX_NO_RETURN void die_codec(vpx_codec_ctx_t *ctx, const char *s); /* The tool including this file must define usage_exit() */ VPX_NO_RETURN void usage_exit(void); diff --git a/media/libvpx/libvpx/vp8/common/blockd.h b/media/libvpx/libvpx/vp8/common/blockd.h --- a/media/libvpx/libvpx/vp8/common/blockd.h +++ b/media/libvpx/libvpx/vp8/common/blockd.h @@ -56,11 +56,11 @@ } ENTROPY_CONTEXT_PLANES; extern const unsigned char vp8_block2left[25]; extern const unsigned char vp8_block2above[25]; -#define VP8_COMBINEENTROPYCONTEXTS(Dest, A, B) Dest = (A) + (B); +#define VP8_COMBINEENTROPYCONTEXTS(Dest, A, B) Dest = (A) + (B) typedef enum { KEY_FRAME = 0, INTER_FRAME = 1 } FRAME_TYPE; typedef enum { DC_PRED, /* average of above and left pixels */ diff --git a/media/libvpx/libvpx/vp8/common/common.h b/media/libvpx/libvpx/vp8/common/common.h --- a/media/libvpx/libvpx/vp8/common/common.h +++ b/media/libvpx/libvpx/vp8/common/common.h @@ -22,26 +22,26 @@ #endif /* Only need this for fixed-size arrays, for structs just assign. */ #define vp8_copy(Dest, Src) \ - { \ + do { \ assert(sizeof(Dest) == sizeof(Src)); \ memcpy(Dest, Src, sizeof(Src)); \ - } + } while (0) /* Use this for variably-sized arrays. */ #define vp8_copy_array(Dest, Src, N) \ - { \ + do { \ assert(sizeof(*(Dest)) == sizeof(*(Src))); \ memcpy(Dest, Src, (N) * sizeof(*(Src))); \ - } + } while (0) -#define vp8_zero(Dest) memset(&(Dest), 0, sizeof(Dest)); +#define vp8_zero(Dest) memset(&(Dest), 0, sizeof(Dest)) -#define vp8_zero_array(Dest, N) memset(Dest, 0, (N) * sizeof(*(Dest))); +#define vp8_zero_array(Dest, N) memset(Dest, 0, (N) * sizeof(*(Dest))) #ifdef __cplusplus } // extern "C" #endif diff --git a/media/libvpx/libvpx/vp8/common/generic/systemdependent.c b/media/libvpx/libvpx/vp8/common/generic/systemdependent.c --- a/media/libvpx/libvpx/vp8/common/generic/systemdependent.c +++ b/media/libvpx/libvpx/vp8/common/generic/systemdependent.c @@ -16,10 +16,12 @@ #include "vpx_ports/x86.h" #elif VPX_ARCH_PPC #include "vpx_ports/ppc.h" #elif VPX_ARCH_MIPS #include "vpx_ports/mips.h" +#elif VPX_ARCH_LOONGARCH +#include "vpx_ports/loongarch.h" #endif #include "vp8/common/onyxc_int.h" #include "vp8/common/systemdependent.h" #if CONFIG_MULTITHREAD @@ -98,10 +100,12 @@ ctx->cpu_caps = x86_simd_caps(); #elif VPX_ARCH_PPC ctx->cpu_caps = ppc_simd_caps(); #elif VPX_ARCH_MIPS ctx->cpu_caps = mips_cpu_caps(); +#elif VPX_ARCH_LOONGARCH + ctx->cpu_caps = loongarch_cpu_caps(); #else // generic-gnu targets. ctx->cpu_caps = 0; #endif } diff --git a/media/libvpx/libvpx/vp8/common/loongarch/idct_lsx.c b/media/libvpx/libvpx/vp8/common/loongarch/idct_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vp8/common/loongarch/idct_lsx.c @@ -0,0 +1,325 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "./vp8_rtcd.h" +#include "vp8/common/blockd.h" +#include "vpx_util/loongson_intrinsics.h" + +static const int32_t cospi8sqrt2minus1 = 20091; +static const int32_t sinpi8sqrt2 = 35468; + +#define TRANSPOSE8X4_SH_SH(in0, in1, in2, in3, out0, out1, out2, out3) \ + { \ + __m128i tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ + \ + DUP2_ARG2(__lsx_vilvl_h, in1, in0, in3, in2, tmp0_m, tmp1_m); \ + DUP2_ARG2(__lsx_vilvh_h, in1, in0, in3, in2, tmp2_m, tmp3_m); \ + DUP2_ARG2(__lsx_vilvl_w, tmp1_m, tmp0_m, tmp3_m, tmp2_m, out0, out2); \ + DUP2_ARG2(__lsx_vilvh_w, tmp1_m, tmp0_m, tmp3_m, tmp2_m, out1, out3); \ + } + +#define TRANSPOSE_TWO_4x4_H(in0, in1, in2, in3, out0, out1, out2, out3) \ + { \ + __m128i s4_m, s5_m, s6_m, s7_m; \ + \ + TRANSPOSE8X4_SH_SH(in0, in1, in2, in3, s4_m, s5_m, s6_m, s7_m); \ + DUP2_ARG2(__lsx_vilvl_d, s6_m, s4_m, s7_m, s5_m, out0, out2); \ + out1 = __lsx_vilvh_d(s6_m, s4_m); \ + out3 = __lsx_vilvh_d(s7_m, s5_m); \ + } + +#define EXPAND_TO_H_MULTIPLY_SINPI8SQRT2_PCK_TO_W(in) \ + ({ \ + __m128i out_m; \ + __m128i zero_m = __lsx_vldi(0); \ + __m128i tmp1_m, tmp2_m; \ + __m128i sinpi8_sqrt2_m = __lsx_vreplgr2vr_w(sinpi8sqrt2); \ + \ + tmp1_m = __lsx_vilvl_h(in, zero_m); \ + tmp2_m = __lsx_vilvh_h(in, zero_m); \ + tmp1_m = __lsx_vsrai_w(tmp1_m, 16); \ + tmp2_m = __lsx_vsrai_w(tmp2_m, 16); \ + tmp1_m = __lsx_vmul_w(tmp1_m, sinpi8_sqrt2_m); \ + tmp1_m = __lsx_vsrai_w(tmp1_m, 16); \ + tmp2_m = __lsx_vmul_w(tmp2_m, sinpi8_sqrt2_m); \ + tmp2_m = __lsx_vsrai_w(tmp2_m, 16); \ + out_m = __lsx_vpickev_h(tmp2_m, tmp1_m); \ + \ + out_m; \ + }) + +#define VP8_IDCT_1D_H(in0, in1, in2, in3, out0, out1, out2, out3) \ + { \ + __m128i a1_m, b1_m, c1_m, d1_m; \ + __m128i c_tmp1_m, c_tmp2_m; \ + __m128i d_tmp1_m, d_tmp2_m; \ + __m128i const_cospi8sqrt2minus1_m; \ + \ + const_cospi8sqrt2minus1_m = __lsx_vreplgr2vr_h(cospi8sqrt2minus1); \ + a1_m = __lsx_vadd_h(in0, in2); \ + b1_m = __lsx_vsub_h(in0, in2); \ + c_tmp1_m = EXPAND_TO_H_MULTIPLY_SINPI8SQRT2_PCK_TO_W(in1); \ + \ + c_tmp2_m = __lsx_vmuh_h(in3, const_cospi8sqrt2minus1_m); \ + c_tmp2_m = __lsx_vslli_h(c_tmp2_m, 1); \ + c_tmp2_m = __lsx_vsrai_h(c_tmp2_m, 1); \ + c_tmp2_m = __lsx_vadd_h(in3, c_tmp2_m); \ + c1_m = __lsx_vsub_h(c_tmp1_m, c_tmp2_m); \ + \ + d_tmp1_m = __lsx_vmuh_h(in1, const_cospi8sqrt2minus1_m); \ + d_tmp1_m = __lsx_vslli_h(d_tmp1_m, 1); \ + d_tmp1_m = __lsx_vsrai_h(d_tmp1_m, 1); \ + d_tmp1_m = __lsx_vadd_h(in1, d_tmp1_m); \ + d_tmp2_m = EXPAND_TO_H_MULTIPLY_SINPI8SQRT2_PCK_TO_W(in3); \ + d1_m = __lsx_vadd_h(d_tmp1_m, d_tmp2_m); \ + LSX_BUTTERFLY_4_H(a1_m, b1_m, c1_m, d1_m, out0, out1, out2, out3); \ + } + +#define VP8_IDCT_1D_W(in0, in1, in2, in3, out0, out1, out2, out3) \ + { \ + __m128i a1_m, b1_m, c1_m, d1_m; \ + __m128i c_tmp1_m, c_tmp2_m, d_tmp1_m, d_tmp2_m; \ + __m128i const_cospi8sqrt2minus1_m, sinpi8_sqrt2_m; \ + \ + const_cospi8sqrt2minus1_m = __lsx_vreplgr2vr_w(cospi8sqrt2minus1); \ + sinpi8_sqrt2_m = __lsx_vreplgr2vr_w(sinpi8sqrt2); \ + a1_m = __lsx_vadd_w(in0, in2); \ + b1_m = __lsx_vsub_w(in0, in2); \ + c_tmp1_m = __lsx_vmul_w(in1, sinpi8_sqrt2_m); \ + c_tmp1_m = __lsx_vsrai_w(c_tmp1_m, 16); \ + c_tmp2_m = __lsx_vmul_w(in3, const_cospi8sqrt2minus1_m); \ + c_tmp2_m = __lsx_vsrai_w(c_tmp2_m, 16); \ + c_tmp2_m = __lsx_vadd_w(in3, c_tmp2_m); \ + c1_m = __lsx_vsub_w(c_tmp1_m, c_tmp2_m); \ + d_tmp1_m = __lsx_vmul_w(in1, const_cospi8sqrt2minus1_m); \ + d_tmp1_m = __lsx_vsrai_w(d_tmp1_m, 16); \ + d_tmp1_m = __lsx_vadd_w(in1, d_tmp1_m); \ + d_tmp2_m = __lsx_vmul_w(in3, sinpi8_sqrt2_m); \ + d_tmp2_m = __lsx_vsrai_w(d_tmp2_m, 16); \ + d1_m = __lsx_vadd_w(d_tmp1_m, d_tmp2_m); \ + LSX_BUTTERFLY_4_W(a1_m, b1_m, c1_m, d1_m, out0, out1, out2, out3); \ + } + +#define UNPCK_SH_SW(in, out0, out1) \ + { \ + out0 = __lsx_vsllwil_w_h(in, 0); \ + out1 = __lsx_vexth_w_h(in); \ + } + +static void idct4x4_addconst_lsx(int16_t in_dc, uint8_t *pred, + int32_t pred_stride, uint8_t *dest, + int32_t dest_stride) { + __m128i vec, res0, res1, res2, res3, dst0, dst1; + __m128i pred0, pred1, pred2, pred3; + __m128i zero = __lsx_vldi(0); + + int32_t pred_stride2 = pred_stride << 1; + int32_t pred_stride3 = pred_stride2 + pred_stride; + + vec = __lsx_vreplgr2vr_h(in_dc); + vec = __lsx_vsrari_h(vec, 3); + pred0 = __lsx_vld(pred, 0); + DUP2_ARG2(__lsx_vldx, pred, pred_stride, pred, pred_stride2, pred1, pred2); + pred3 = __lsx_vldx(pred, pred_stride3); + DUP4_ARG2(__lsx_vilvl_b, zero, pred0, zero, pred1, zero, pred2, zero, pred3, + res0, res1, res2, res3); + DUP4_ARG2(__lsx_vadd_h, res0, vec, res1, vec, res2, vec, res3, vec, res0, + res1, res2, res3); + res0 = __lsx_vclip255_h(res0); + res1 = __lsx_vclip255_h(res1); + res2 = __lsx_vclip255_h(res2); + res3 = __lsx_vclip255_h(res3); + + DUP2_ARG2(__lsx_vpickev_b, res1, res0, res3, res2, dst0, dst1); + dst0 = __lsx_vpickev_w(dst1, dst0); + __lsx_vstelm_w(dst0, dest, 0, 0); + dest += dest_stride; + __lsx_vstelm_w(dst0, dest, 0, 1); + dest += dest_stride; + __lsx_vstelm_w(dst0, dest, 0, 2); + dest += dest_stride; + __lsx_vstelm_w(dst0, dest, 0, 3); +} + +void vp8_dc_only_idct_add_lsx(int16_t input_dc, uint8_t *pred_ptr, + int32_t pred_stride, uint8_t *dst_ptr, + int32_t dst_stride) { + idct4x4_addconst_lsx(input_dc, pred_ptr, pred_stride, dst_ptr, dst_stride); +} + +static void dequant_idct4x4_addblk_2x_lsx(int16_t *input, + int16_t *dequant_input, uint8_t *dest, + int32_t dest_stride) { + __m128i dest0, dest1, dest2, dest3; + __m128i in0, in1, in2, in3, mul0, mul1, mul2, mul3, dequant_in0, dequant_in1; + __m128i hz0, hz1, hz2, hz3, vt0, vt1, vt2, vt3, res0, res1, res2, res3; + __m128i hz0l, hz1l, hz2l, hz3l, hz0r, hz1r, hz2r, hz3r; + __m128i vt0l, vt1l, vt2l, vt3l, vt0r, vt1r, vt2r, vt3r; + __m128i zero = __lsx_vldi(0); + + int32_t dest_stride2 = dest_stride << 1; + int32_t dest_stride3 = dest_stride2 + dest_stride; + + DUP4_ARG2(__lsx_vld, input, 0, input, 16, input, 32, input, 48, in0, in1, in2, + in3); + DUP2_ARG2(__lsx_vld, dequant_input, 0, dequant_input, 16, dequant_in0, + dequant_in1); + + DUP4_ARG2(__lsx_vmul_h, in0, dequant_in0, in1, dequant_in1, in2, dequant_in0, + in3, dequant_in1, mul0, mul1, mul2, mul3); + DUP2_ARG2(__lsx_vpickev_d, mul2, mul0, mul3, mul1, in0, in2); + DUP2_ARG2(__lsx_vpickod_d, mul2, mul0, mul3, mul1, in1, in3); + + VP8_IDCT_1D_H(in0, in1, in2, in3, hz0, hz1, hz2, hz3); + TRANSPOSE_TWO_4x4_H(hz0, hz1, hz2, hz3, hz0, hz1, hz2, hz3); + UNPCK_SH_SW(hz0, hz0r, hz0l); + UNPCK_SH_SW(hz1, hz1r, hz1l); + UNPCK_SH_SW(hz2, hz2r, hz2l); + UNPCK_SH_SW(hz3, hz3r, hz3l); + VP8_IDCT_1D_W(hz0l, hz1l, hz2l, hz3l, vt0l, vt1l, vt2l, vt3l); + DUP4_ARG2(__lsx_vsrari_w, vt0l, 3, vt1l, 3, vt2l, 3, vt3l, 3, vt0l, vt1l, + vt2l, vt3l); + VP8_IDCT_1D_W(hz0r, hz1r, hz2r, hz3r, vt0r, vt1r, vt2r, vt3r); + DUP4_ARG2(__lsx_vsrari_w, vt0r, 3, vt1r, 3, vt2r, 3, vt3r, 3, vt0r, vt1r, + vt2r, vt3r); + DUP4_ARG2(__lsx_vpickev_h, vt0l, vt0r, vt1l, vt1r, vt2l, vt2r, vt3l, vt3r, + vt0, vt1, vt2, vt3); + TRANSPOSE_TWO_4x4_H(vt0, vt1, vt2, vt3, vt0, vt1, vt2, vt3); + dest0 = __lsx_vld(dest, 0); + DUP2_ARG2(__lsx_vldx, dest, dest_stride, dest, dest_stride2, dest1, dest2); + dest3 = __lsx_vldx(dest, dest_stride3); + DUP4_ARG2(__lsx_vilvl_b, zero, dest0, zero, dest1, zero, dest2, zero, dest3, + res0, res1, res2, res3); + DUP4_ARG2(__lsx_vadd_h, res0, vt0, res1, vt1, res2, vt2, res3, vt3, res0, + res1, res2, res3); + + res0 = __lsx_vclip255_h(res0); + res1 = __lsx_vclip255_h(res1); + res2 = __lsx_vclip255_h(res2); + res3 = __lsx_vclip255_h(res3); + DUP2_ARG2(__lsx_vpickev_b, res1, res0, res3, res2, vt0l, vt1l); + + __lsx_vstelm_d(vt0l, dest, 0, 0); + __lsx_vstelm_d(vt0l, dest + dest_stride, 0, 1); + __lsx_vstelm_d(vt1l, dest + dest_stride2, 0, 0); + __lsx_vstelm_d(vt1l, dest + dest_stride3, 0, 1); + + __lsx_vst(zero, input, 0); + __lsx_vst(zero, input, 16); + __lsx_vst(zero, input, 32); + __lsx_vst(zero, input, 48); +} + +static void dequant_idct_addconst_2x_lsx(int16_t *input, int16_t *dequant_input, + uint8_t *dest, int32_t dest_stride) { + __m128i input_dc0, input_dc1, vec, res0, res1, res2, res3; + __m128i dest0, dest1, dest2, dest3; + __m128i zero = __lsx_vldi(0); + int32_t dest_stride2 = dest_stride << 1; + int32_t dest_stride3 = dest_stride2 + dest_stride; + + input_dc0 = __lsx_vreplgr2vr_h(input[0] * dequant_input[0]); + input_dc1 = __lsx_vreplgr2vr_h(input[16] * dequant_input[0]); + DUP2_ARG2(__lsx_vsrari_h, input_dc0, 3, input_dc1, 3, input_dc0, input_dc1); + vec = __lsx_vpickev_d(input_dc1, input_dc0); + input[0] = 0; + input[16] = 0; + dest0 = __lsx_vld(dest, 0); + DUP2_ARG2(__lsx_vldx, dest, dest_stride, dest, dest_stride2, dest1, dest2); + dest3 = __lsx_vldx(dest, dest_stride3); + DUP4_ARG2(__lsx_vilvl_b, zero, dest0, zero, dest1, zero, dest2, zero, dest3, + res0, res1, res2, res3); + DUP4_ARG2(__lsx_vadd_h, res0, vec, res1, vec, res2, vec, res3, vec, res0, + res1, res2, res3); + res0 = __lsx_vclip255_h(res0); + res1 = __lsx_vclip255_h(res1); + res2 = __lsx_vclip255_h(res2); + res3 = __lsx_vclip255_h(res3); + + DUP2_ARG2(__lsx_vpickev_b, res1, res0, res3, res2, res0, res1); + __lsx_vstelm_d(res0, dest, 0, 0); + __lsx_vstelm_d(res0, dest + dest_stride, 0, 1); + __lsx_vstelm_d(res1, dest + dest_stride2, 0, 0); + __lsx_vstelm_d(res1, dest + dest_stride3, 0, 1); +} + +void vp8_dequant_idct_add_y_block_lsx(int16_t *q, int16_t *dq, uint8_t *dst, + int32_t stride, char *eobs) { + int16_t *eobs_h = (int16_t *)eobs; + uint8_t i; + + for (i = 4; i--;) { + if (eobs_h[0]) { + if (eobs_h[0] & 0xfefe) { + dequant_idct4x4_addblk_2x_lsx(q, dq, dst, stride); + } else { + dequant_idct_addconst_2x_lsx(q, dq, dst, stride); + } + } + + q += 32; + + if (eobs_h[1]) { + if (eobs_h[1] & 0xfefe) { + dequant_idct4x4_addblk_2x_lsx(q, dq, dst + 8, stride); + } else { + dequant_idct_addconst_2x_lsx(q, dq, dst + 8, stride); + } + } + + q += 32; + dst += (4 * stride); + eobs_h += 2; + } +} + +void vp8_dequant_idct_add_uv_block_lsx(int16_t *q, int16_t *dq, uint8_t *dst_u, + uint8_t *dst_v, int32_t stride, + char *eobs) { + int16_t *eobs_h = (int16_t *)eobs; + if (eobs_h[0]) { + if (eobs_h[0] & 0xfefe) { + dequant_idct4x4_addblk_2x_lsx(q, dq, dst_u, stride); + } else { + dequant_idct_addconst_2x_lsx(q, dq, dst_u, stride); + } + } + + q += 32; + dst_u += (stride * 4); + + if (eobs_h[1]) { + if (eobs_h[1] & 0xfefe) { + dequant_idct4x4_addblk_2x_lsx(q, dq, dst_u, stride); + } else { + dequant_idct_addconst_2x_lsx(q, dq, dst_u, stride); + } + } + + q += 32; + + if (eobs_h[2]) { + if (eobs_h[2] & 0xfefe) { + dequant_idct4x4_addblk_2x_lsx(q, dq, dst_v, stride); + } else { + dequant_idct_addconst_2x_lsx(q, dq, dst_v, stride); + } + } + q += 32; + dst_v += (stride * 4); + + if (eobs_h[3]) { + if (eobs_h[3] & 0xfefe) { + dequant_idct4x4_addblk_2x_lsx(q, dq, dst_v, stride); + } else { + dequant_idct_addconst_2x_lsx(q, dq, dst_v, stride); + } + } +} diff --git a/media/libvpx/libvpx/vp8/common/loongarch/loopfilter_filters_lsx.c b/media/libvpx/libvpx/vp8/common/loongarch/loopfilter_filters_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vp8/common/loongarch/loopfilter_filters_lsx.c @@ -0,0 +1,743 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Lu Wang + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "./vp8_rtcd.h" +#include "vp8/common/loopfilter.h" +#include "vpx_util/loongson_intrinsics.h" + +#define VP8_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev) \ + { \ + __m128i p1_m, p0_m, q0_m, q1_m, filt, q0_sub_p0, t1, t2; \ + const __m128i cnst4b = __lsx_vldi(4); \ + const __m128i cnst3b = __lsx_vldi(3); \ + \ + p1_m = __lsx_vxori_b(p1, 0x80); \ + p0_m = __lsx_vxori_b(p0, 0x80); \ + q0_m = __lsx_vxori_b(q0, 0x80); \ + q1_m = __lsx_vxori_b(q1, 0x80); \ + \ + filt = __lsx_vssub_b(p1_m, q1_m); \ + filt = __lsx_vand_v(filt, hev); \ + q0_sub_p0 = __lsx_vssub_b(q0_m, p0_m); \ + filt = __lsx_vsadd_b(filt, q0_sub_p0); \ + filt = __lsx_vsadd_b(filt, q0_sub_p0); \ + filt = __lsx_vsadd_b(filt, q0_sub_p0); \ + filt = __lsx_vand_v(filt, mask); \ + t1 = __lsx_vsadd_b(filt, cnst4b); \ + t1 = __lsx_vsra_b(t1, cnst3b); \ + t2 = __lsx_vsadd_b(filt, cnst3b); \ + t2 = __lsx_vsra_b(t2, cnst3b); \ + q0_m = __lsx_vssub_b(q0_m, t1); \ + q0 = __lsx_vxori_b(q0_m, 0x80); \ + p0_m = __lsx_vsadd_b(p0_m, t2); \ + p0 = __lsx_vxori_b(p0_m, 0x80); \ + filt = __lsx_vsrari_b(t1, 1); \ + hev = __lsx_vxori_b(hev, 0xff); \ + filt = __lsx_vand_v(filt, hev); \ + q1_m = __lsx_vssub_b(q1_m, filt); \ + q1 = __lsx_vxori_b(q1_m, 0x80); \ + p1_m = __lsx_vsadd_b(p1_m, filt); \ + p1 = __lsx_vxori_b(p1_m, 0x80); \ + } + +#define VP8_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev) \ + { \ + __m128i p2_m, p1_m, p0_m, q2_m, q1_m, q0_m; \ + __m128i u, filt, t1, t2, filt_sign, q0_sub_p0; \ + __m128i filt_r, filt_l; \ + __m128i temp0, temp1, temp2, temp3; \ + const __m128i cnst4b = __lsx_vldi(4); \ + const __m128i cnst3b = __lsx_vldi(3); \ + const __m128i cnst9h = __lsx_vldi(1033); \ + const __m128i cnst63h = __lsx_vldi(1087); \ + \ + p2_m = __lsx_vxori_b(p2, 0x80); \ + p1_m = __lsx_vxori_b(p1, 0x80); \ + p0_m = __lsx_vxori_b(p0, 0x80); \ + q0_m = __lsx_vxori_b(q0, 0x80); \ + q1_m = __lsx_vxori_b(q1, 0x80); \ + q2_m = __lsx_vxori_b(q2, 0x80); \ + \ + filt = __lsx_vssub_b(p1_m, q1_m); \ + q0_sub_p0 = __lsx_vssub_b(q0_m, p0_m); \ + filt = __lsx_vsadd_b(filt, q0_sub_p0); \ + filt = __lsx_vsadd_b(filt, q0_sub_p0); \ + filt = __lsx_vsadd_b(filt, q0_sub_p0); \ + filt = __lsx_vand_v(filt, mask); \ + \ + t2 = __lsx_vand_v(filt, hev); \ + hev = __lsx_vxori_b(hev, 0xff); \ + filt = __lsx_vand_v(hev, filt); \ + t1 = __lsx_vsadd_b(t2, cnst4b); \ + t1 = __lsx_vsra_b(t1, cnst3b); \ + t2 = __lsx_vsadd_b(t2, cnst3b); \ + t2 = __lsx_vsra_b(t2, cnst3b); \ + q0_m = __lsx_vssub_b(q0_m, t1); \ + p0_m = __lsx_vsadd_b(p0_m, t2); \ + filt_sign = __lsx_vslti_b(filt, 0); \ + filt_r = __lsx_vilvl_b(filt_sign, filt); \ + filt_l = __lsx_vilvh_b(filt_sign, filt); \ + temp0 = __lsx_vmul_h(filt_r, cnst9h); \ + temp1 = __lsx_vadd_h(temp0, cnst63h); \ + temp2 = __lsx_vmul_h(filt_l, cnst9h); \ + temp3 = __lsx_vadd_h(temp2, cnst63h); \ + \ + u = __lsx_vssrani_b_h(temp3, temp1, 7); \ + q2_m = __lsx_vssub_b(q2_m, u); \ + p2_m = __lsx_vsadd_b(p2_m, u); \ + q2 = __lsx_vxori_b(q2_m, 0x80); \ + p2 = __lsx_vxori_b(p2_m, 0x80); \ + \ + temp1 = __lsx_vadd_h(temp1, temp0); \ + temp3 = __lsx_vadd_h(temp3, temp2); \ + \ + u = __lsx_vssrani_b_h(temp3, temp1, 7); \ + q1_m = __lsx_vssub_b(q1_m, u); \ + p1_m = __lsx_vsadd_b(p1_m, u); \ + q1 = __lsx_vxori_b(q1_m, 0x80); \ + p1 = __lsx_vxori_b(p1_m, 0x80); \ + \ + temp1 = __lsx_vadd_h(temp1, temp0); \ + temp3 = __lsx_vadd_h(temp3, temp2); \ + \ + u = __lsx_vssrani_b_h(temp3, temp1, 7); \ + q0_m = __lsx_vssub_b(q0_m, u); \ + p0_m = __lsx_vsadd_b(p0_m, u); \ + q0 = __lsx_vxori_b(q0_m, 0x80); \ + p0 = __lsx_vxori_b(p0_m, 0x80); \ + } + +#define LPF_MASK_HEV(p3_in, p2_in, p1_in, p0_in, q0_in, q1_in, q2_in, q3_in, \ + limit_in, b_limit_in, thresh_in, hev_out, mask_out, \ + flat_out) \ + { \ + __m128i p3_asub_p2_m, p2_asub_p1_m, p1_asub_p0_m, q1_asub_q0_m; \ + __m128i p1_asub_q1_m, p0_asub_q0_m, q3_asub_q2_m, q2_asub_q1_m; \ + \ + p3_asub_p2_m = __lsx_vabsd_bu(p3_in, p2_in); \ + p2_asub_p1_m = __lsx_vabsd_bu(p2_in, p1_in); \ + p1_asub_p0_m = __lsx_vabsd_bu(p1_in, p0_in); \ + q1_asub_q0_m = __lsx_vabsd_bu(q1_in, q0_in); \ + q2_asub_q1_m = __lsx_vabsd_bu(q2_in, q1_in); \ + q3_asub_q2_m = __lsx_vabsd_bu(q3_in, q2_in); \ + p0_asub_q0_m = __lsx_vabsd_bu(p0_in, q0_in); \ + p1_asub_q1_m = __lsx_vabsd_bu(p1_in, q1_in); \ + flat_out = __lsx_vmax_bu(p1_asub_p0_m, q1_asub_q0_m); \ + hev_out = __lsx_vslt_bu(thresh_in, flat_out); \ + p0_asub_q0_m = __lsx_vsadd_bu(p0_asub_q0_m, p0_asub_q0_m); \ + p1_asub_q1_m = __lsx_vsrli_b(p1_asub_q1_m, 1); \ + p0_asub_q0_m = __lsx_vsadd_bu(p0_asub_q0_m, p1_asub_q1_m); \ + mask_out = __lsx_vslt_bu(b_limit_in, p0_asub_q0_m); \ + mask_out = __lsx_vmax_bu(flat_out, mask_out); \ + p3_asub_p2_m = __lsx_vmax_bu(p3_asub_p2_m, p2_asub_p1_m); \ + mask_out = __lsx_vmax_bu(p3_asub_p2_m, mask_out); \ + q2_asub_q1_m = __lsx_vmax_bu(q2_asub_q1_m, q3_asub_q2_m); \ + mask_out = __lsx_vmax_bu(q2_asub_q1_m, mask_out); \ + mask_out = __lsx_vslt_bu(limit_in, mask_out); \ + mask_out = __lsx_vxori_b(mask_out, 0xff); \ + } + +#define VP8_ST6x1_B(in0, in0_idx, in1, in1_idx, pdst, stride) \ + { \ + __lsx_vstelm_w(in0, pdst, 0, in0_idx); \ + __lsx_vstelm_h(in1, pdst + stride, 0, in1_idx); \ + } + +static void loop_filter_horizontal_4_dual_lsx(uint8_t *src, int32_t pitch, + const uint8_t *b_limit0_ptr, + const uint8_t *limit0_ptr, + const uint8_t *thresh0_ptr, + const uint8_t *b_limit1_ptr, + const uint8_t *limit1_ptr, + const uint8_t *thresh1_ptr) { + int32_t pitch_x2 = pitch << 1; + int32_t pitch_x3 = pitch_x2 + pitch; + int32_t pitch_x4 = pitch << 2; + + __m128i mask, hev, flat; + __m128i thresh0, b_limit0, limit0, thresh1, b_limit1, limit1; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + + DUP4_ARG2(__lsx_vldx, src, -pitch_x4, src, -pitch_x3, src, -pitch_x2, src, + -pitch, p3, p2, p1, p0); + q0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, pitch, src, pitch_x2, q1, q2); + q3 = __lsx_vldx(src, pitch_x3); + + thresh0 = __lsx_vreplgr2vr_b(*thresh0_ptr); + thresh1 = __lsx_vreplgr2vr_b(*thresh1_ptr); + thresh0 = __lsx_vilvl_d(thresh1, thresh0); + + b_limit0 = __lsx_vreplgr2vr_b(*b_limit0_ptr); + b_limit1 = __lsx_vreplgr2vr_b(*b_limit1_ptr); + b_limit0 = __lsx_vilvl_d(b_limit1, b_limit0); + + limit0 = __lsx_vreplgr2vr_b(*limit0_ptr); + limit1 = __lsx_vreplgr2vr_b(*limit1_ptr); + limit0 = __lsx_vilvl_d(limit1, limit0); + + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit0, b_limit0, thresh0, hev, + mask, flat); + VP8_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev); + + __lsx_vstx(p1, src, -pitch_x2); + __lsx_vstx(p0, src, -pitch); + __lsx_vst(q0, src, 0); + __lsx_vstx(q1, src, pitch); +} + +static void loop_filter_vertical_4_dual_lsx(uint8_t *src, int32_t pitch, + const uint8_t *b_limit0_ptr, + const uint8_t *limit0_ptr, + const uint8_t *thresh0_ptr, + const uint8_t *b_limit1_ptr, + const uint8_t *limit1_ptr, + const uint8_t *thresh1_ptr) { + uint8_t *src_tmp0 = src - 4; + int32_t pitch_x2 = pitch << 1; + int32_t pitch_x3 = pitch_x2 + pitch; + int32_t pitch_x4 = pitch << 2; + __m128i mask, hev, flat; + __m128i thresh0, b_limit0, limit0, thresh1, b_limit1, limit1; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i row0, row1, row2, row3, row4, row5, row6, row7; + __m128i row8, row9, row10, row11, row12, row13, row14, row15; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + + row0 = __lsx_vld(src_tmp0, 0); + DUP2_ARG2(__lsx_vldx, src_tmp0, pitch, src_tmp0, pitch_x2, row1, row2); + row3 = __lsx_vldx(src_tmp0, pitch_x3); + src_tmp0 += pitch_x4; + row4 = __lsx_vld(src_tmp0, 0); + DUP2_ARG2(__lsx_vldx, src_tmp0, pitch, src_tmp0, pitch_x2, row5, row6); + row7 = __lsx_vldx(src_tmp0, pitch_x3); + src_tmp0 += pitch_x4; + + row8 = __lsx_vld(src_tmp0, 0); + DUP2_ARG2(__lsx_vldx, src_tmp0, pitch, src_tmp0, pitch_x2, row9, row10); + row11 = __lsx_vldx(src_tmp0, pitch_x3); + src_tmp0 += pitch_x4; + row12 = __lsx_vld(src_tmp0, 0); + DUP2_ARG2(__lsx_vldx, src_tmp0, pitch, src_tmp0, pitch_x2, row13, row14); + row15 = __lsx_vldx(src_tmp0, pitch_x3); + + LSX_TRANSPOSE16x8_B(row0, row1, row2, row3, row4, row5, row6, row7, row8, + row9, row10, row11, row12, row13, row14, row15, p3, p2, + p1, p0, q0, q1, q2, q3); + + thresh0 = __lsx_vldrepl_b(thresh0_ptr, 0); + thresh1 = __lsx_vldrepl_b(thresh1_ptr, 0); + thresh0 = __lsx_vilvl_d(thresh1, thresh0); + + b_limit0 = __lsx_vldrepl_b(b_limit0_ptr, 0); + b_limit1 = __lsx_vldrepl_b(b_limit1_ptr, 0); + b_limit0 = __lsx_vilvl_d(b_limit1, b_limit0); + + limit0 = __lsx_vldrepl_b(limit0_ptr, 0); + limit1 = __lsx_vldrepl_b(limit1_ptr, 0); + limit0 = __lsx_vilvl_d(limit1, limit0); + + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit0, b_limit0, thresh0, hev, + mask, flat); + VP8_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev); + + DUP2_ARG2(__lsx_vilvl_b, p0, p1, q1, q0, tmp0, tmp1); + tmp2 = __lsx_vilvl_h(tmp1, tmp0); + tmp3 = __lsx_vilvh_h(tmp1, tmp0); + DUP2_ARG2(__lsx_vilvh_b, p0, p1, q1, q0, tmp0, tmp1); + tmp4 = __lsx_vilvl_h(tmp1, tmp0); + tmp5 = __lsx_vilvh_h(tmp1, tmp0); + + src -= 2; + __lsx_vstelm_w(tmp2, src, 0, 0); + src += pitch; + __lsx_vstelm_w(tmp2, src, 0, 1); + src += pitch; + __lsx_vstelm_w(tmp2, src, 0, 2); + src += pitch; + __lsx_vstelm_w(tmp2, src, 0, 3); + src += pitch; + + __lsx_vstelm_w(tmp3, src, 0, 0); + src += pitch; + __lsx_vstelm_w(tmp3, src, 0, 1); + src += pitch; + __lsx_vstelm_w(tmp3, src, 0, 2); + src += pitch; + __lsx_vstelm_w(tmp3, src, 0, 3); + src += pitch; + + __lsx_vstelm_w(tmp4, src, 0, 0); + src += pitch; + __lsx_vstelm_w(tmp4, src, 0, 1); + src += pitch; + __lsx_vstelm_w(tmp4, src, 0, 2); + src += pitch; + __lsx_vstelm_w(tmp4, src, 0, 3); + src += pitch; + + __lsx_vstelm_w(tmp5, src, 0, 0); + src += pitch; + __lsx_vstelm_w(tmp5, src, 0, 1); + src += pitch; + __lsx_vstelm_w(tmp5, src, 0, 2); + src += pitch; + __lsx_vstelm_w(tmp5, src, 0, 3); +} + +static void loop_filter_horizontal_edge_uv_lsx(uint8_t *src_u, uint8_t *src_v, + int32_t pitch, + const uint8_t b_limit_in, + const uint8_t limit_in, + const uint8_t thresh_in) { + int32_t pitch_x2 = pitch << 1; + int32_t pitch_x3 = pitch_x2 + pitch; + int32_t pitch_x4 = pitch << 2; + + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i mask, hev, flat, thresh, limit, b_limit; + __m128i p3_u, p2_u, p1_u, p0_u, q3_u, q2_u, q1_u, q0_u; + __m128i p3_v, p2_v, p1_v, p0_v, q3_v, q2_v, q1_v, q0_v; + + thresh = __lsx_vreplgr2vr_b(thresh_in); + limit = __lsx_vreplgr2vr_b(limit_in); + b_limit = __lsx_vreplgr2vr_b(b_limit_in); + + DUP4_ARG2(__lsx_vldx, src_u, -pitch_x4, src_u, -pitch_x3, src_u, -pitch_x2, + src_u, -pitch, p3_u, p2_u, p1_u, p0_u); + q0_u = __lsx_vld(src_u, 0); + DUP2_ARG2(__lsx_vldx, src_u, pitch, src_u, pitch_x2, q1_u, q2_u); + q3_u = __lsx_vldx(src_u, pitch_x3); + + DUP4_ARG2(__lsx_vldx, src_v, -pitch_x4, src_v, -pitch_x3, src_v, -pitch_x2, + src_v, -pitch, p3_v, p2_v, p1_v, p0_v); + q0_v = __lsx_vld(src_v, 0); + DUP2_ARG2(__lsx_vldx, src_v, pitch, src_v, pitch_x2, q1_v, q2_v); + q3_v = __lsx_vldx(src_v, pitch_x3); + + /* right 8 element of p3 are u pixel and + left 8 element of p3 are v pixel */ + DUP4_ARG2(__lsx_vilvl_d, p3_v, p3_u, p2_v, p2_u, p1_v, p1_u, p0_v, p0_u, p3, + p2, p1, p0); + DUP4_ARG2(__lsx_vilvl_d, q0_v, q0_u, q1_v, q1_u, q2_v, q2_u, q3_v, q3_u, q0, + q1, q2, q3); + + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + VP8_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev); + + __lsx_vstelm_d(q1, src_u + pitch, 0, 0); + __lsx_vstelm_d(q0, src_u, 0, 0); + __lsx_vstelm_d(p0, src_u - pitch, 0, 0); + __lsx_vstelm_d(p1, src_u - pitch_x2, 0, 0); + + __lsx_vstelm_d(q1, src_v + pitch, 0, 1); + __lsx_vstelm_d(q0, src_v, 0, 1); + __lsx_vstelm_d(p0, src_v - pitch, 0, 1); + __lsx_vstelm_d(p1, src_v - pitch_x2, 0, 1); +} + +static void loop_filter_vertical_edge_uv_lsx(uint8_t *src_u, uint8_t *src_v, + int32_t pitch, + const uint8_t b_limit_in, + const uint8_t limit_in, + const uint8_t thresh_in) { + uint8_t *src_u_tmp, *src_v_tmp; + int32_t pitch_x2 = pitch << 1; + int32_t pitch_x3 = pitch_x2 + pitch; + int32_t pitch_x4 = pitch << 2; + + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i mask, hev, flat, thresh, limit, b_limit; + __m128i row0, row1, row2, row3, row4, row5, row6, row7, row8; + __m128i row9, row10, row11, row12, row13, row14, row15; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + + thresh = __lsx_vreplgr2vr_b(thresh_in); + limit = __lsx_vreplgr2vr_b(limit_in); + b_limit = __lsx_vreplgr2vr_b(b_limit_in); + + src_u_tmp = src_u - 4; + row0 = __lsx_vld(src_u_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_u_tmp, pitch, src_u_tmp, pitch_x2, row1, row2); + row3 = __lsx_vldx(src_u_tmp, pitch_x3); + src_u_tmp += pitch_x4; + row4 = __lsx_vld(src_u_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_u_tmp, pitch, src_u_tmp, pitch_x2, row5, row6); + row7 = __lsx_vldx(src_u_tmp, pitch_x3); + + src_v_tmp = src_v - 4; + row8 = __lsx_vld(src_v_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_v_tmp, pitch, src_v_tmp, pitch_x2, row9, row10); + row11 = __lsx_vldx(src_v_tmp, pitch_x3); + src_v_tmp += pitch_x4; + row12 = __lsx_vld(src_v_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_v_tmp, pitch, src_v_tmp, pitch_x2, row13, row14); + row15 = __lsx_vldx(src_v_tmp, pitch_x3); + + LSX_TRANSPOSE16x8_B(row0, row1, row2, row3, row4, row5, row6, row7, row8, + row9, row10, row11, row12, row13, row14, row15, p3, p2, + p1, p0, q0, q1, q2, q3); + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + VP8_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev); + + DUP2_ARG2(__lsx_vilvl_b, p0, p1, q1, q0, tmp0, tmp1); + tmp2 = __lsx_vilvl_h(tmp1, tmp0); + tmp3 = __lsx_vilvh_h(tmp1, tmp0); + + tmp0 = __lsx_vilvh_b(p0, p1); + tmp1 = __lsx_vilvh_b(q1, q0); + tmp4 = __lsx_vilvl_h(tmp1, tmp0); + tmp5 = __lsx_vilvh_h(tmp1, tmp0); + + src_u_tmp += 2; + __lsx_vstelm_w(tmp2, src_u_tmp - pitch_x4, 0, 0); + __lsx_vstelm_w(tmp2, src_u_tmp - pitch_x3, 0, 1); + __lsx_vstelm_w(tmp2, src_u_tmp - pitch_x2, 0, 2); + __lsx_vstelm_w(tmp2, src_u_tmp - pitch, 0, 3); + + __lsx_vstelm_w(tmp3, src_u_tmp, 0, 0); + __lsx_vstelm_w(tmp3, src_u_tmp + pitch, 0, 1); + __lsx_vstelm_w(tmp3, src_u_tmp + pitch_x2, 0, 2); + __lsx_vstelm_w(tmp3, src_u_tmp + pitch_x3, 0, 3); + + src_v_tmp += 2; + __lsx_vstelm_w(tmp4, src_v_tmp - pitch_x4, 0, 0); + __lsx_vstelm_w(tmp4, src_v_tmp - pitch_x3, 0, 1); + __lsx_vstelm_w(tmp4, src_v_tmp - pitch_x2, 0, 2); + __lsx_vstelm_w(tmp4, src_v_tmp - pitch, 0, 3); + + __lsx_vstelm_w(tmp5, src_v_tmp, 0, 0); + __lsx_vstelm_w(tmp5, src_v_tmp + pitch, 0, 1); + __lsx_vstelm_w(tmp5, src_v_tmp + pitch_x2, 0, 2); + __lsx_vstelm_w(tmp5, src_v_tmp + pitch_x3, 0, 3); +} + +static inline void mbloop_filter_horizontal_edge_y_lsx( + uint8_t *src, int32_t pitch, const uint8_t b_limit_in, + const uint8_t limit_in, const uint8_t thresh_in) { + uint8_t *temp_src; + int32_t pitch_x2 = pitch << 1; + int32_t pitch_x3 = pitch_x2 + pitch; + int32_t pitch_x4 = pitch << 2; + + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i mask, hev, flat, thresh, limit, b_limit; + + DUP2_ARG2(__lsx_vldrepl_b, &b_limit_in, 0, &limit_in, 0, b_limit, limit); + thresh = __lsx_vldrepl_b(&thresh_in, 0); + + temp_src = src - pitch_x4; + DUP4_ARG2(__lsx_vldx, temp_src, 0, temp_src, pitch, temp_src, pitch_x2, + temp_src, pitch_x3, p3, p2, p1, p0); + temp_src += pitch_x4; + DUP4_ARG2(__lsx_vldx, temp_src, 0, temp_src, pitch, temp_src, pitch_x2, + temp_src, pitch_x3, q0, q1, q2, q3); + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + VP8_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev); + + temp_src = src - pitch_x3; + __lsx_vstx(p2, temp_src, 0); + __lsx_vstx(p1, temp_src, pitch); + __lsx_vstx(p0, temp_src, pitch_x2); + __lsx_vstx(q0, temp_src, pitch_x3); + temp_src += pitch_x4; + __lsx_vstx(q1, temp_src, 0); + __lsx_vstx(q2, temp_src, pitch); +} + +static inline void mbloop_filter_horizontal_edge_uv_lsx( + uint8_t *src_u, uint8_t *src_v, int32_t pitch, const uint8_t b_limit_in, + const uint8_t limit_in, const uint8_t thresh_in) { + uint8_t *temp_src; + int32_t pitch_x2 = pitch << 1; + int32_t pitch_x3 = pitch_x2 + pitch; + int32_t pitch_x4 = pitch << 2; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i mask, hev, flat, thresh, limit, b_limit; + __m128i p3_u, p2_u, p1_u, p0_u, q3_u, q2_u, q1_u, q0_u; + __m128i p3_v, p2_v, p1_v, p0_v, q3_v, q2_v, q1_v, q0_v; + + DUP2_ARG2(__lsx_vldrepl_b, &b_limit_in, 0, &limit_in, 0, b_limit, limit); + thresh = __lsx_vldrepl_b(&thresh_in, 0); + + temp_src = src_u - pitch_x4; + DUP4_ARG2(__lsx_vldx, temp_src, 0, temp_src, pitch, temp_src, pitch_x2, + temp_src, pitch_x3, p3_u, p2_u, p1_u, p0_u); + temp_src += pitch_x4; + DUP4_ARG2(__lsx_vldx, temp_src, 0, temp_src, pitch, temp_src, pitch_x2, + temp_src, pitch_x3, q0_u, q1_u, q2_u, q3_u); + temp_src = src_v - pitch_x4; + DUP4_ARG2(__lsx_vldx, temp_src, 0, temp_src, pitch, temp_src, pitch_x2, + temp_src, pitch_x3, p3_v, p2_v, p1_v, p0_v); + temp_src += pitch_x4; + DUP4_ARG2(__lsx_vldx, temp_src, 0, temp_src, pitch, temp_src, pitch_x2, + temp_src, pitch_x3, q0_v, q1_v, q2_v, q3_v); + + DUP4_ARG2(__lsx_vilvl_d, p3_v, p3_u, p2_v, p2_u, p1_v, p1_u, p0_v, p0_u, p3, + p2, p1, p0); + DUP4_ARG2(__lsx_vilvl_d, q0_v, q0_u, q1_v, q1_u, q2_v, q2_u, q3_v, q3_u, q0, + q1, q2, q3); + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + VP8_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev); + + src_u -= pitch_x3; + __lsx_vstelm_d(p2, src_u, 0, 0); + __lsx_vstelm_d(p1, src_u + pitch, 0, 0); + __lsx_vstelm_d(p0, src_u + pitch_x2, 0, 0); + __lsx_vstelm_d(q0, src_u + pitch_x3, 0, 0); + src_u += pitch_x4; + __lsx_vstelm_d(q1, src_u, 0, 0); + src_u += pitch; + __lsx_vstelm_d(q2, src_u, 0, 0); + + src_v -= pitch_x3; + __lsx_vstelm_d(p2, src_v, 0, 1); + __lsx_vstelm_d(p1, src_v + pitch, 0, 1); + __lsx_vstelm_d(p0, src_v + pitch_x2, 0, 1); + __lsx_vstelm_d(q0, src_v + pitch_x3, 0, 1); + src_v += pitch_x4; + __lsx_vstelm_d(q1, src_v, 0, 1); + src_v += pitch; + __lsx_vstelm_d(q2, src_v, 0, 1); +} + +static inline void mbloop_filter_vertical_edge_y_lsx(uint8_t *src, + int32_t pitch, + const uint8_t b_limit_in, + const uint8_t limit_in, + const uint8_t thresh_in) { + uint8_t *temp_src; + int32_t pitch_x2 = pitch << 1; + int32_t pitch_x3 = pitch_x2 + pitch; + int32_t pitch_x4 = pitch << 2; + + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i mask, hev, flat, thresh, limit, b_limit; + __m128i row0, row1, row2, row3, row4, row5, row6, row7, row8; + __m128i row9, row10, row11, row12, row13, row14, row15; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + + DUP2_ARG2(__lsx_vldrepl_b, &b_limit_in, 0, &limit_in, 0, b_limit, limit); + thresh = __lsx_vldrepl_b(&thresh_in, 0); + temp_src = src - 4; + DUP4_ARG2(__lsx_vldx, temp_src, 0, temp_src, pitch, temp_src, pitch_x2, + temp_src, pitch_x3, row0, row1, row2, row3); + temp_src += pitch_x4; + DUP4_ARG2(__lsx_vldx, temp_src, 0, temp_src, pitch, temp_src, pitch_x2, + temp_src, pitch_x3, row4, row5, row6, row7); + temp_src += pitch_x4; + DUP4_ARG2(__lsx_vldx, temp_src, 0, temp_src, pitch, temp_src, pitch_x2, + temp_src, pitch_x3, row8, row9, row10, row11); + temp_src += pitch_x4; + DUP4_ARG2(__lsx_vldx, temp_src, 0, temp_src, pitch, temp_src, pitch_x2, + temp_src, pitch_x3, row12, row13, row14, row15); + temp_src -= pitch_x4; + LSX_TRANSPOSE16x8_B(row0, row1, row2, row3, row4, row5, row6, row7, row8, + row9, row10, row11, row12, row13, row14, row15, p3, p2, + p1, p0, q0, q1, q2, q3); + + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + VP8_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev); + DUP2_ARG2(__lsx_vilvl_b, p1, p2, q0, p0, tmp0, tmp1); + tmp3 = __lsx_vilvl_h(tmp1, tmp0); + tmp4 = __lsx_vilvh_h(tmp1, tmp0); + DUP2_ARG2(__lsx_vilvh_b, p1, p2, q0, p0, tmp0, tmp1); + tmp6 = __lsx_vilvl_h(tmp1, tmp0); + tmp7 = __lsx_vilvh_h(tmp1, tmp0); + tmp2 = __lsx_vilvl_b(q2, q1); + tmp5 = __lsx_vilvh_b(q2, q1); + + temp_src = src - 3; + VP8_ST6x1_B(tmp3, 0, tmp2, 0, temp_src, 4); + temp_src += pitch; + VP8_ST6x1_B(tmp3, 1, tmp2, 1, temp_src, 4); + temp_src += pitch; + VP8_ST6x1_B(tmp3, 2, tmp2, 2, temp_src, 4); + temp_src += pitch; + VP8_ST6x1_B(tmp3, 3, tmp2, 3, temp_src, 4); + temp_src += pitch; + VP8_ST6x1_B(tmp4, 0, tmp2, 4, temp_src, 4); + temp_src += pitch; + VP8_ST6x1_B(tmp4, 1, tmp2, 5, temp_src, 4); + temp_src += pitch; + VP8_ST6x1_B(tmp4, 2, tmp2, 6, temp_src, 4); + temp_src += pitch; + VP8_ST6x1_B(tmp4, 3, tmp2, 7, temp_src, 4); + temp_src += pitch; + VP8_ST6x1_B(tmp6, 0, tmp5, 0, temp_src, 4); + temp_src += pitch; + VP8_ST6x1_B(tmp6, 1, tmp5, 1, temp_src, 4); + temp_src += pitch; + VP8_ST6x1_B(tmp6, 2, tmp5, 2, temp_src, 4); + temp_src += pitch; + VP8_ST6x1_B(tmp6, 3, tmp5, 3, temp_src, 4); + temp_src += pitch; + VP8_ST6x1_B(tmp7, 0, tmp5, 4, temp_src, 4); + temp_src += pitch; + VP8_ST6x1_B(tmp7, 1, tmp5, 5, temp_src, 4); + temp_src += pitch; + VP8_ST6x1_B(tmp7, 2, tmp5, 6, temp_src, 4); + temp_src += pitch; + VP8_ST6x1_B(tmp7, 3, tmp5, 7, temp_src, 4); +} + +static inline void mbloop_filter_vertical_edge_uv_lsx( + uint8_t *src_u, uint8_t *src_v, int32_t pitch, const uint8_t b_limit_in, + const uint8_t limit_in, const uint8_t thresh_in) { + int32_t pitch_x2 = pitch << 1; + int32_t pitch_x3 = pitch_x2 + pitch; + int32_t pitch_x4 = pitch << 2; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i mask, hev, flat, thresh, limit, b_limit; + __m128i row0, row1, row2, row3, row4, row5, row6, row7, row8; + __m128i row9, row10, row11, row12, row13, row14, row15; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + + DUP2_ARG2(__lsx_vldrepl_b, &b_limit_in, 0, &limit_in, 0, b_limit, limit); + thresh = __lsx_vldrepl_b(&thresh_in, 0); + + src_u -= 4; + DUP4_ARG2(__lsx_vldx, src_u, 0, src_u, pitch, src_u, pitch_x2, src_u, + pitch_x3, row0, row1, row2, row3); + src_u += pitch_x4; + DUP4_ARG2(__lsx_vldx, src_u, 0, src_u, pitch, src_u, pitch_x2, src_u, + pitch_x3, row4, row5, row6, row7); + src_v -= 4; + DUP4_ARG2(__lsx_vldx, src_v, 0, src_v, pitch, src_v, pitch_x2, src_v, + pitch_x3, row8, row9, row10, row11); + src_v += pitch_x4; + DUP4_ARG2(__lsx_vldx, src_v, 0, src_v, pitch, src_v, pitch_x2, src_v, + pitch_x3, row12, row13, row14, row15); + LSX_TRANSPOSE16x8_B(row0, row1, row2, row3, row4, row5, row6, row7, row8, + row9, row10, row11, row12, row13, row14, row15, p3, p2, + p1, p0, q0, q1, q2, q3); + + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + VP8_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev); + + DUP2_ARG2(__lsx_vilvl_b, p1, p2, q0, p0, tmp0, tmp1); + tmp3 = __lsx_vilvl_h(tmp1, tmp0); + tmp4 = __lsx_vilvh_h(tmp1, tmp0); + DUP2_ARG2(__lsx_vilvh_b, p1, p2, q0, p0, tmp0, tmp1); + tmp6 = __lsx_vilvl_h(tmp1, tmp0); + tmp7 = __lsx_vilvh_h(tmp1, tmp0); + tmp2 = __lsx_vilvl_b(q2, q1); + tmp5 = __lsx_vilvh_b(q2, q1); + + src_u += 1 - pitch_x4; + VP8_ST6x1_B(tmp3, 0, tmp2, 0, src_u, 4); + src_u += pitch; + VP8_ST6x1_B(tmp3, 1, tmp2, 1, src_u, 4); + src_u += pitch; + VP8_ST6x1_B(tmp3, 2, tmp2, 2, src_u, 4); + src_u += pitch; + VP8_ST6x1_B(tmp3, 3, tmp2, 3, src_u, 4); + src_u += pitch; + VP8_ST6x1_B(tmp4, 0, tmp2, 4, src_u, 4); + src_u += pitch; + VP8_ST6x1_B(tmp4, 1, tmp2, 5, src_u, 4); + src_u += pitch; + VP8_ST6x1_B(tmp4, 2, tmp2, 6, src_u, 4); + src_u += pitch; + VP8_ST6x1_B(tmp4, 3, tmp2, 7, src_u, 4); + + src_v += 1 - pitch_x4; + VP8_ST6x1_B(tmp6, 0, tmp5, 0, src_v, 4); + src_v += pitch; + VP8_ST6x1_B(tmp6, 1, tmp5, 1, src_v, 4); + src_v += pitch; + VP8_ST6x1_B(tmp6, 2, tmp5, 2, src_v, 4); + src_v += pitch; + VP8_ST6x1_B(tmp6, 3, tmp5, 3, src_v, 4); + src_v += pitch; + VP8_ST6x1_B(tmp7, 0, tmp5, 4, src_v, 4); + src_v += pitch; + VP8_ST6x1_B(tmp7, 1, tmp5, 5, src_v, 4); + src_v += pitch; + VP8_ST6x1_B(tmp7, 2, tmp5, 6, src_v, 4); + src_v += pitch; + VP8_ST6x1_B(tmp7, 3, tmp5, 7, src_v, 4); +} + +void vp8_loop_filter_mbh_lsx(uint8_t *src_y, uint8_t *src_u, uint8_t *src_v, + int32_t pitch_y, int32_t pitch_u_v, + loop_filter_info *lpf_info_ptr) { + mbloop_filter_horizontal_edge_y_lsx(src_y, pitch_y, *lpf_info_ptr->mblim, + *lpf_info_ptr->lim, + *lpf_info_ptr->hev_thr); + if (src_u) { + mbloop_filter_horizontal_edge_uv_lsx( + src_u, src_v, pitch_u_v, *lpf_info_ptr->mblim, *lpf_info_ptr->lim, + *lpf_info_ptr->hev_thr); + } +} + +void vp8_loop_filter_mbv_lsx(uint8_t *src_y, uint8_t *src_u, uint8_t *src_v, + int32_t pitch_y, int32_t pitch_u_v, + loop_filter_info *lpf_info_ptr) { + mbloop_filter_vertical_edge_y_lsx(src_y, pitch_y, *lpf_info_ptr->mblim, + *lpf_info_ptr->lim, *lpf_info_ptr->hev_thr); + if (src_u) { + mbloop_filter_vertical_edge_uv_lsx(src_u, src_v, pitch_u_v, + *lpf_info_ptr->mblim, *lpf_info_ptr->lim, + *lpf_info_ptr->hev_thr); + } +} + +void vp8_loop_filter_bh_lsx(uint8_t *src_y, uint8_t *src_u, uint8_t *src_v, + int32_t pitch_y, int32_t pitch_u_v, + loop_filter_info *lpf_info_ptr) { + loop_filter_horizontal_4_dual_lsx(src_y + 4 * pitch_y, pitch_y, + lpf_info_ptr->blim, lpf_info_ptr->lim, + lpf_info_ptr->hev_thr, lpf_info_ptr->blim, + lpf_info_ptr->lim, lpf_info_ptr->hev_thr); + loop_filter_horizontal_4_dual_lsx(src_y + 8 * pitch_y, pitch_y, + lpf_info_ptr->blim, lpf_info_ptr->lim, + lpf_info_ptr->hev_thr, lpf_info_ptr->blim, + lpf_info_ptr->lim, lpf_info_ptr->hev_thr); + loop_filter_horizontal_4_dual_lsx(src_y + 12 * pitch_y, pitch_y, + lpf_info_ptr->blim, lpf_info_ptr->lim, + lpf_info_ptr->hev_thr, lpf_info_ptr->blim, + lpf_info_ptr->lim, lpf_info_ptr->hev_thr); + if (src_u) { + loop_filter_horizontal_edge_uv_lsx( + src_u + (4 * pitch_u_v), src_v + (4 * pitch_u_v), pitch_u_v, + *lpf_info_ptr->blim, *lpf_info_ptr->lim, *lpf_info_ptr->hev_thr); + } +} + +void vp8_loop_filter_bv_lsx(uint8_t *src_y, uint8_t *src_u, uint8_t *src_v, + int32_t pitch_y, int32_t pitch_u_v, + loop_filter_info *lpf_info_ptr) { + loop_filter_vertical_4_dual_lsx(src_y + 4, pitch_y, lpf_info_ptr->blim, + lpf_info_ptr->lim, lpf_info_ptr->hev_thr, + lpf_info_ptr->blim, lpf_info_ptr->lim, + lpf_info_ptr->hev_thr); + loop_filter_vertical_4_dual_lsx(src_y + 8, pitch_y, lpf_info_ptr->blim, + lpf_info_ptr->lim, lpf_info_ptr->hev_thr, + lpf_info_ptr->blim, lpf_info_ptr->lim, + lpf_info_ptr->hev_thr); + loop_filter_vertical_4_dual_lsx(src_y + 12, pitch_y, lpf_info_ptr->blim, + lpf_info_ptr->lim, lpf_info_ptr->hev_thr, + lpf_info_ptr->blim, lpf_info_ptr->lim, + lpf_info_ptr->hev_thr); + if (src_u) { + loop_filter_vertical_edge_uv_lsx(src_u + 4, src_v + 4, pitch_u_v, + *lpf_info_ptr->blim, *lpf_info_ptr->lim, + *lpf_info_ptr->hev_thr); + } +} diff --git a/media/libvpx/libvpx/vp8/common/loongarch/sixtap_filter_lsx.c b/media/libvpx/libvpx/vp8/common/loongarch/sixtap_filter_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vp8/common/loongarch/sixtap_filter_lsx.c @@ -0,0 +1,1164 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Lu Wang + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "./vp8_rtcd.h" +#include "vp8/common/filter.h" +#include "vpx_ports/mem.h" +#include "vpx_util/loongson_intrinsics.h" + +DECLARE_ALIGNED(16, static const int8_t, vp8_subpel_filters_lsx[7][8]) = { + { 0, -6, 123, 12, -1, 0, 0, 0 }, + { 2, -11, 108, 36, -8, 1, 0, 0 }, /* New 1/4 pel 6 tap filter */ + { 0, -9, 93, 50, -6, 0, 0, 0 }, + { 3, -16, 77, 77, -16, 3, 0, 0 }, /* New 1/2 pel 6 tap filter */ + { 0, -6, 50, 93, -9, 0, 0, 0 }, + { 1, -8, 36, 108, -11, 2, 0, 0 }, /* New 1/4 pel 6 tap filter */ + { 0, -1, 12, 123, -6, 0, 0, 0 }, +}; + +static const uint8_t vp8_mc_filt_mask_arr[16 * 3] = { + /* 8 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, + /* 4 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20, + /* 4 width cases */ + 8, 9, 9, 10, 10, 11, 11, 12, 24, 25, 25, 26, 26, 27, 27, 28 +}; + +#define DPADD_H3(in0, in1, in2, coeff0, coeff1, coeff2) \ + ({ \ + __m128i out0_m; \ + \ + out0_m = __lsx_vdp2_h_b(in0, coeff0); \ + out0_m = __lsx_vdp2add_h_b(out0_m, in1, coeff1); \ + out0_m = __lsx_vdp2add_h_b(out0_m, in2, coeff2); \ + \ + out0_m; \ + }) + +#define HORIZ_6TAP_FILT(src0, src1, mask0, mask1, mask2, filt_h0, filt_h1, \ + filt_h2) \ + ({ \ + __m128i vec0_m, vec1_m, vec2_m; \ + __m128i hz_out_m; \ + \ + DUP2_ARG3(__lsx_vshuf_b, src0, src1, mask0, src0, src1, mask1, vec0_m, \ + vec1_m); \ + vec2_m = __lsx_vshuf_b(src0, src1, mask2); \ + hz_out_m = DPADD_H3(vec0_m, vec1_m, vec2_m, filt_h0, filt_h1, filt_h2); \ + \ + hz_out_m = __lsx_vsrari_h(hz_out_m, VP8_FILTER_SHIFT); \ + hz_out_m = __lsx_vsat_h(hz_out_m, 7); \ + \ + hz_out_m; \ + }) + +#define HORIZ_6TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, \ + mask2, filt0, filt1, filt2, out0, out1, \ + out2, out3) \ + ({ \ + __m128i vec0_m, vec1_m, vec2_m, vec3_m, vec4_m, vec5_m, vec6_m, vec7_m; \ + \ + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src1, mask0, vec0_m, \ + vec1_m); \ + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask0, src3, src3, mask0, vec2_m, \ + vec3_m); \ + DUP4_ARG2(__lsx_vdp2_h_b, vec0_m, filt0, vec1_m, filt0, vec2_m, filt0, \ + vec3_m, filt0, out0, out1, out2, out3); \ + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask1, src1, src1, mask1, vec0_m, \ + vec1_m); \ + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask1, src3, src3, mask1, vec2_m, \ + vec3_m); \ + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask2, src1, src1, mask2, vec4_m, \ + vec5_m); \ + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask2, src3, src3, mask2, vec6_m, \ + vec7_m); \ + DUP4_ARG3(__lsx_vdp2add_h_b, out0, vec0_m, filt1, out1, vec1_m, filt1, \ + out2, vec2_m, filt1, out3, vec3_m, filt1, out0, out1, out2, \ + out3); \ + DUP4_ARG3(__lsx_vdp2add_h_b, out0, vec4_m, filt2, out1, vec5_m, filt2, \ + out2, vec6_m, filt2, out3, vec7_m, filt2, out0, out1, out2, \ + out3); \ + }) + +#define FILT_4TAP_DPADD_H(vec0, vec1, filt0, filt1) \ + ({ \ + __m128i tmp0; \ + \ + tmp0 = __lsx_vdp2_h_b(vec0, filt0); \ + tmp0 = __lsx_vdp2add_h_b(tmp0, vec1, filt1); \ + \ + tmp0; \ + }) + +#define HORIZ_4TAP_FILT(src0, src1, mask0, mask1, filt_h0, filt_h1) \ + ({ \ + __m128i vec0_m, vec1_m; \ + __m128i hz_out_m; \ + \ + DUP2_ARG3(__lsx_vshuf_b, src0, src1, mask0, src0, src1, mask1, vec0_m, \ + vec1_m); \ + hz_out_m = FILT_4TAP_DPADD_H(vec0_m, vec1_m, filt_h0, filt_h1); \ + hz_out_m = __lsx_vsrari_h(hz_out_m, VP8_FILTER_SHIFT); \ + hz_out_m = __lsx_vsat_h(hz_out_m, 7); \ + \ + hz_out_m; \ + }) + +#define HORIZ_4TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, \ + filt0, filt1, out0, out1, out2, out3) \ + ({ \ + __m128i vec0_m, vec1_m, vec2_m, vec3_m; \ + \ + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src1, mask0, vec0_m, \ + vec1_m); \ + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask0, src3, src3, mask0, vec2_m, \ + vec3_m); \ + DUP4_ARG2(__lsx_vdp2_h_b, vec0_m, filt0, vec1_m, filt0, vec2_m, filt0, \ + vec3_m, filt0, out0, out1, out2, out3); \ + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask1, src1, src1, mask1, vec0_m, \ + vec1_m); \ + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask1, src3, src3, mask1, vec2_m, \ + vec3_m); \ + DUP4_ARG3(__lsx_vdp2add_h_b, out0, vec0_m, filt1, out1, vec1_m, filt1, \ + out2, vec2_m, filt1, out3, vec3_m, filt1, out0, out1, out2, \ + out3); \ + }) + +static void common_hz_6t_8w_lsx(uint8_t *RESTRICT src, int32_t src_stride, + uint8_t *RESTRICT dst, int32_t dst_stride, + const int8_t *filter, int32_t height) { + uint32_t loop_cnt; + int32_t src_stride_x2 = src_stride << 1; + int32_t src_stride_x3 = src_stride_x2 + src_stride; + int32_t src_stride_x4 = src_stride << 2; + int32_t dst_stride_x2 = dst_stride << 1; + int32_t dst_stride_x3 = dst_stride_x2 + dst_stride; + int32_t dst_stride_x4 = dst_stride << 2; + __m128i src0, src1, src2, src3, filt0, filt1, filt2; + __m128i mask0, mask1, mask2, tmp0, tmp1; + __m128i filt, out0, out1, out2, out3; + + mask0 = __lsx_vld(vp8_mc_filt_mask_arr, 0); + src -= 2; + + filt = __lsx_vld(filter, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt0, filt1); + filt2 = __lsx_vreplvei_h(filt, 2); + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src0, src1, src2, src3); + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + src += src_stride_x4; + HORIZ_6TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, filt0, + filt1, filt2, out0, out1, out2, out3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, VP8_FILTER_SHIFT, out3, out2, + VP8_FILTER_SHIFT, tmp0, tmp1); + DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + __lsx_vstelm_d(tmp0, dst, 0, 0); + __lsx_vstelm_d(tmp0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(tmp1, dst + dst_stride_x2, 0, 0); + __lsx_vstelm_d(tmp1, dst + dst_stride_x3, 0, 1); + dst += dst_stride_x4; + + for (loop_cnt = (height >> 2) - 1; loop_cnt--;) { + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src0, src1, src2, src3); + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + src += src_stride_x4; + HORIZ_6TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, + filt0, filt1, filt2, out0, out1, out2, out3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, VP8_FILTER_SHIFT, out3, out2, + VP8_FILTER_SHIFT, tmp0, tmp1); + DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + __lsx_vstelm_d(tmp0, dst, 0, 0); + __lsx_vstelm_d(tmp0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(tmp1, dst + dst_stride_x2, 0, 0); + __lsx_vstelm_d(tmp1, dst + dst_stride_x3, 0, 1); + dst += dst_stride_x4; + } +} + +static void common_hz_6t_16w_lsx(uint8_t *RESTRICT src, int32_t src_stride, + uint8_t *RESTRICT dst, int32_t dst_stride, + const int8_t *filter, int32_t height) { + uint32_t loop_cnt; + int32_t src_stride_x2 = src_stride << 1; + int32_t src_stride_x3 = src_stride_x2 + src_stride; + int32_t src_stride_x4 = src_stride << 2; + int32_t dst_stride_x2 = dst_stride << 1; + int32_t dst_stride_x3 = dst_stride_x2 + dst_stride; + int32_t dst_stride_x4 = dst_stride << 2; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, filt0, filt1, filt2; + __m128i mask0, mask1, mask2, out; + __m128i filt, out0, out1, out2, out3, out4, out5, out6, out7; + + mask0 = __lsx_vld(vp8_mc_filt_mask_arr, 0); + src -= 2; + + filt = __lsx_vld(filter, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt0, filt1); + filt2 = __lsx_vreplvei_h(filt, 2); + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src0, src2, src4, src6); + src += 8; + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src1, src3, src5, src7); + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + DUP4_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src6, 128, src7, 128, src4, + src5, src6, src7); + src += src_stride_x4 - 8; + + HORIZ_6TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, + filt0, filt1, filt2, out0, out1, out2, out3); + HORIZ_6TAP_8WID_4VECS_FILT(src4, src5, src6, src7, mask0, mask1, mask2, + filt0, filt1, filt2, out4, out5, out6, out7); + DUP4_ARG2(__lsx_vsrari_h, out0, VP8_FILTER_SHIFT, out1, VP8_FILTER_SHIFT, + out2, VP8_FILTER_SHIFT, out3, VP8_FILTER_SHIFT, out0, out1, out2, + out3); + DUP4_ARG2(__lsx_vsrari_h, out4, VP8_FILTER_SHIFT, out5, VP8_FILTER_SHIFT, + out6, VP8_FILTER_SHIFT, out7, VP8_FILTER_SHIFT, out4, out5, out6, + out7); + DUP4_ARG2(__lsx_vsat_h, out0, 7, out1, 7, out2, 7, out3, 7, out0, out1, + out2, out3); + DUP4_ARG2(__lsx_vsat_h, out4, 7, out5, 7, out6, 7, out7, 7, out4, out5, + out6, out7); + out = __lsx_vpickev_b(out1, out0); + out = __lsx_vxori_b(out, 128); + __lsx_vst(out, dst, 0); + out = __lsx_vpickev_b(out3, out2); + out = __lsx_vxori_b(out, 128); + __lsx_vstx(out, dst, dst_stride); + out = __lsx_vpickev_b(out5, out4); + out = __lsx_vxori_b(out, 128); + __lsx_vstx(out, dst, dst_stride_x2); + out = __lsx_vpickev_b(out7, out6); + out = __lsx_vxori_b(out, 128); + __lsx_vstx(out, dst, dst_stride_x3); + dst += dst_stride_x4; + } +} + +static void common_vt_6t_8w_lsx(uint8_t *RESTRICT src, int32_t src_stride, + uint8_t *RESTRICT dst, int32_t dst_stride, + const int8_t *filter, int32_t height) { + uint32_t loop_cnt; + int32_t src_stride_x2 = src_stride << 1; + int32_t src_stride_x3 = src_stride_x2 + src_stride; + int32_t src_stride_x4 = src_stride << 2; + int32_t dst_stride_x2 = dst_stride << 1; + int32_t dst_stride_x3 = dst_stride_x2 + dst_stride; + int32_t dst_stride_x4 = dst_stride << 2; + __m128i src0, src1, src2, src3, src4, src7, src8, src9, src10; + __m128i src10_r, src32_r, src76_r, src98_r, src21_r, src43_r, src87_r; + __m128i src109_r, filt0, filt1, filt2; + __m128i tmp0, tmp1; + __m128i filt, out0_r, out1_r, out2_r, out3_r; + + src -= src_stride_x2; + filt = __lsx_vld(filter, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt0, filt1); + filt2 = __lsx_vreplvei_h(filt, 2); + + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src0, src1, src2, src3); + src += src_stride_x4; + src4 = __lsx_vld(src, 0); + src += src_stride; + + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + src4 = __lsx_vxori_b(src4, 128); + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src2, src1, src4, src3, + src10_r, src32_r, src21_r, src43_r); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src7, src8, src9, src10); + DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7, + src8, src9, src10); + src += src_stride_x4; + + DUP4_ARG2(__lsx_vilvl_b, src7, src4, src8, src7, src9, src8, src10, src9, + src76_r, src87_r, src98_r, src109_r); + out0_r = DPADD_H3(src10_r, src32_r, src76_r, filt0, filt1, filt2); + out1_r = DPADD_H3(src21_r, src43_r, src87_r, filt0, filt1, filt2); + out2_r = DPADD_H3(src32_r, src76_r, src98_r, filt0, filt1, filt2); + out3_r = DPADD_H3(src43_r, src87_r, src109_r, filt0, filt1, filt2); + DUP2_ARG3(__lsx_vssrarni_b_h, out1_r, out0_r, VP8_FILTER_SHIFT, out3_r, + out2_r, VP8_FILTER_SHIFT, tmp0, tmp1); + DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + __lsx_vstelm_d(tmp0, dst, 0, 0); + __lsx_vstelm_d(tmp0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(tmp1, dst + dst_stride_x2, 0, 0); + __lsx_vstelm_d(tmp1, dst + dst_stride_x3, 0, 1); + dst += dst_stride_x4; + + src10_r = src76_r; + src32_r = src98_r; + src21_r = src87_r; + src43_r = src109_r; + src4 = src10; + } +} + +static void common_vt_6t_16w_lsx(uint8_t *RESTRICT src, int32_t src_stride, + uint8_t *RESTRICT dst, int32_t dst_stride, + const int8_t *filter, int32_t height) { + uint32_t loop_cnt; + int32_t src_stride_x2 = src_stride << 1; + int32_t src_stride_x3 = src_stride_x2 + src_stride; + int32_t src_stride_x4 = src_stride << 2; + int32_t dst_stride_x2 = dst_stride << 1; + int32_t dst_stride_x3 = dst_stride_x2 + dst_stride; + int32_t dst_stride_x4 = dst_stride << 2; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8; + __m128i src10_r, src32_r, src54_r, src76_r, src21_r, src43_r, src65_r; + __m128i src87_r, src10_l, src32_l, src54_l, src76_l, src21_l, src43_l; + __m128i src65_l, src87_l, filt0, filt1, filt2; + __m128i tmp0, tmp1, tmp2, tmp3; + __m128i filt, out0_r, out1_r, out2_r, out3_r, out0_l, out1_l, out2_l, out3_l; + + src -= src_stride_x2; + filt = __lsx_vld(filter, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt0, filt1); + filt2 = __lsx_vreplvei_h(filt, 2); + + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src0, src1, src2, src3); + src += src_stride_x4; + src4 = __lsx_vldx(src, 0); + src += src_stride; + + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + src4 = __lsx_vxori_b(src4, 128); + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src4, src3, src2, src1, + src10_r, src32_r, src43_r, src21_r); + DUP4_ARG2(__lsx_vilvh_b, src1, src0, src3, src2, src4, src3, src2, src1, + src10_l, src32_l, src43_l, src21_l); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src5, src6, src7, src8); + src += src_stride_x4; + + DUP4_ARG2(__lsx_vxori_b, src5, 128, src6, 128, src7, 128, src8, 128, src5, + src6, src7, src8); + DUP4_ARG2(__lsx_vilvl_b, src5, src4, src6, src5, src7, src6, src8, src7, + src54_r, src65_r, src76_r, src87_r); + DUP4_ARG2(__lsx_vilvh_b, src5, src4, src6, src5, src7, src6, src8, src7, + src54_l, src65_l, src76_l, src87_l); + out0_r = DPADD_H3(src10_r, src32_r, src54_r, filt0, filt1, filt2); + out1_r = DPADD_H3(src21_r, src43_r, src65_r, filt0, filt1, filt2); + out2_r = DPADD_H3(src32_r, src54_r, src76_r, filt0, filt1, filt2); + out3_r = DPADD_H3(src43_r, src65_r, src87_r, filt0, filt1, filt2); + out0_l = DPADD_H3(src10_l, src32_l, src54_l, filt0, filt1, filt2); + out1_l = DPADD_H3(src21_l, src43_l, src65_l, filt0, filt1, filt2); + out2_l = DPADD_H3(src32_l, src54_l, src76_l, filt0, filt1, filt2); + out3_l = DPADD_H3(src43_l, src65_l, src87_l, filt0, filt1, filt2); + DUP4_ARG3(__lsx_vssrarni_b_h, out0_l, out0_r, VP8_FILTER_SHIFT, out1_l, + out1_r, VP8_FILTER_SHIFT, out2_l, out2_r, VP8_FILTER_SHIFT, + out3_l, out3_r, VP8_FILTER_SHIFT, tmp0, tmp1, tmp2, tmp3); + DUP4_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp2, 128, tmp3, 128, tmp0, + tmp1, tmp2, tmp3); + __lsx_vstx(tmp0, dst, 0); + __lsx_vstx(tmp1, dst, dst_stride); + __lsx_vstx(tmp2, dst, dst_stride_x2); + __lsx_vstx(tmp3, dst, dst_stride_x3); + dst += dst_stride_x4; + + src10_r = src54_r; + src32_r = src76_r; + src21_r = src65_r; + src43_r = src87_r; + src10_l = src54_l; + src32_l = src76_l; + src21_l = src65_l; + src43_l = src87_l; + src4 = src8; + } +} + +static void common_hv_6ht_6vt_8w_lsx(uint8_t *RESTRICT src, int32_t src_stride, + uint8_t *RESTRICT dst, int32_t dst_stride, + const int8_t *filter_horiz, + const int8_t *filter_vert, + int32_t height) { + uint32_t loop_cnt; + int32_t src_stride_x2 = src_stride << 1; + int32_t src_stride_x3 = src_stride_x2 + src_stride; + int32_t src_stride_x4 = src_stride << 2; + int32_t dst_stride_x2 = dst_stride << 1; + int32_t dst_stride_x3 = dst_stride_x2 + dst_stride; + int32_t dst_stride_x4 = dst_stride << 2; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8; + __m128i filt_hz0, filt_hz1, filt_hz2; + __m128i mask0, mask1, mask2, vec0, vec1; + __m128i filt, filt_vt0, filt_vt1, filt_vt2; + __m128i hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6; + __m128i hz_out7, hz_out8, out0, out1, out2, out3, out4, out5, out6, out7; + __m128i tmp0, tmp1, tmp2, tmp3; + + mask0 = __lsx_vld(vp8_mc_filt_mask_arr, 0); + src -= (2 + src_stride_x2); + + filt = __lsx_vld(filter_horiz, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt_hz0, filt_hz1); + filt_hz2 = __lsx_vreplvei_h(filt, 2); + + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src0, src1, src2, src3); + src += src_stride_x4; + src4 = __lsx_vldx(src, 0); + src += src_stride; + + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + src4 = __lsx_vxori_b(src4, 128); + + hz_out0 = HORIZ_6TAP_FILT(src0, src0, mask0, mask1, mask2, filt_hz0, filt_hz1, + filt_hz2); + hz_out1 = HORIZ_6TAP_FILT(src1, src1, mask0, mask1, mask2, filt_hz0, filt_hz1, + filt_hz2); + hz_out2 = HORIZ_6TAP_FILT(src2, src2, mask0, mask1, mask2, filt_hz0, filt_hz1, + filt_hz2); + hz_out3 = HORIZ_6TAP_FILT(src3, src3, mask0, mask1, mask2, filt_hz0, filt_hz1, + filt_hz2); + hz_out4 = HORIZ_6TAP_FILT(src4, src4, mask0, mask1, mask2, filt_hz0, filt_hz1, + filt_hz2); + filt = __lsx_vld(filter_vert, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt_vt0, filt_vt1); + filt_vt2 = __lsx_vreplvei_h(filt, 2); + + DUP4_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, hz_out2, + hz_out1, hz_out4, hz_out3, out0, out1, out3, out4); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src5, src6, src7, src8); + src += src_stride_x4; + + DUP4_ARG2(__lsx_vxori_b, src5, 128, src6, 128, src7, 128, src8, 128, src5, + src6, src7, src8); + hz_out5 = HORIZ_6TAP_FILT(src5, src5, mask0, mask1, mask2, filt_hz0, + filt_hz1, filt_hz2); + out2 = __lsx_vpackev_b(hz_out5, hz_out4); + tmp0 = DPADD_H3(out0, out1, out2, filt_vt0, filt_vt1, filt_vt2); + + hz_out6 = HORIZ_6TAP_FILT(src6, src6, mask0, mask1, mask2, filt_hz0, + filt_hz1, filt_hz2); + out5 = __lsx_vpackev_b(hz_out6, hz_out5); + tmp1 = DPADD_H3(out3, out4, out5, filt_vt0, filt_vt1, filt_vt2); + + hz_out7 = HORIZ_6TAP_FILT(src7, src7, mask0, mask1, mask2, filt_hz0, + filt_hz1, filt_hz2); + out7 = __lsx_vpackev_b(hz_out7, hz_out6); + tmp2 = DPADD_H3(out1, out2, out7, filt_vt0, filt_vt1, filt_vt2); + + hz_out8 = HORIZ_6TAP_FILT(src8, src8, mask0, mask1, mask2, filt_hz0, + filt_hz1, filt_hz2); + out6 = __lsx_vpackev_b(hz_out8, hz_out7); + tmp3 = DPADD_H3(out4, out5, out6, filt_vt0, filt_vt1, filt_vt2); + + DUP2_ARG3(__lsx_vssrarni_b_h, tmp1, tmp0, VP8_FILTER_SHIFT, tmp3, tmp2, + VP8_FILTER_SHIFT, vec0, vec1); + DUP2_ARG2(__lsx_vxori_b, vec0, 128, vec1, 128, vec0, vec1); + + __lsx_vstelm_d(vec0, dst, 0, 0); + __lsx_vstelm_d(vec0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(vec1, dst + dst_stride_x2, 0, 0); + __lsx_vstelm_d(vec1, dst + dst_stride_x3, 0, 1); + dst += dst_stride_x4; + + hz_out4 = hz_out8; + out0 = out2; + out1 = out7; + out3 = out5; + out4 = out6; + } +} + +static void common_hv_6ht_6vt_16w_lsx(uint8_t *RESTRICT src, int32_t src_stride, + uint8_t *RESTRICT dst, int32_t dst_stride, + const int8_t *filter_horiz, + const int8_t *filter_vert, + int32_t height) { + common_hv_6ht_6vt_8w_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert, height); + common_hv_6ht_6vt_8w_lsx(src + 8, src_stride, dst + 8, dst_stride, + filter_horiz, filter_vert, height); +} + +static void common_hz_4t_8w_lsx(uint8_t *RESTRICT src, int32_t src_stride, + uint8_t *RESTRICT dst, int32_t dst_stride, + const int8_t *filter, int32_t height) { + uint32_t loop_cnt; + int32_t src_stride_x2 = src_stride << 1; + int32_t src_stride_x3 = src_stride_x2 + src_stride; + int32_t src_stride_x4 = src_stride << 2; + int32_t dst_stride_x2 = dst_stride << 1; + int32_t dst_stride_x3 = dst_stride_x2 + dst_stride; + int32_t dst_stride_x4 = dst_stride << 2; + __m128i src0, src1, src2, src3, filt0, filt1, mask0, mask1; + __m128i tmp0, tmp1; + __m128i filt, out0, out1, out2, out3; + + mask0 = __lsx_vld(vp8_mc_filt_mask_arr, 0); + src -= 1; + + filt = __lsx_vld(filter, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt0, filt1); + mask1 = __lsx_vaddi_bu(mask0, 2); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src0, src1, src2, src3); + src += src_stride_x4; + + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_4TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, filt0, + filt1, out0, out1, out2, out3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, VP8_FILTER_SHIFT, out3, out2, + VP8_FILTER_SHIFT, tmp0, tmp1); + DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + __lsx_vstelm_d(tmp0, dst, 0, 0); + __lsx_vstelm_d(tmp0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(tmp1, dst + dst_stride_x2, 0, 0); + __lsx_vstelm_d(tmp1, dst + dst_stride_x3, 0, 1); + dst += dst_stride_x4; + } +} + +static void common_hz_4t_16w_lsx(uint8_t *RESTRICT src, int32_t src_stride, + uint8_t *RESTRICT dst, int32_t dst_stride, + const int8_t *filter, int32_t height) { + uint32_t loop_cnt; + int32_t src_stride_x2 = src_stride << 1; + int32_t src_stride_x3 = src_stride_x2 + src_stride; + int32_t src_stride_x4 = src_stride << 2; + int32_t dst_stride_x2 = dst_stride << 1; + int32_t dst_stride_x3 = dst_stride_x2 + dst_stride; + int32_t dst_stride_x4 = dst_stride << 2; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i filt0, filt1, mask0, mask1; + __m128i filt, out0, out1, out2, out3, out4, out5, out6, out7; + + mask0 = __lsx_vld(vp8_mc_filt_mask_arr, 0); + src -= 1; + + filt = __lsx_vld(filter, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt0, filt1); + mask1 = __lsx_vaddi_bu(mask0, 2); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src0, src2, src4, src6); + src += 8; + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src1, src3, src5, src7); + src += src_stride_x4 - 8; + + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + DUP4_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src6, 128, src7, 128, src4, + src5, src6, src7); + HORIZ_4TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, filt0, + filt1, out0, out1, out2, out3); + HORIZ_4TAP_8WID_4VECS_FILT(src4, src5, src6, src7, mask0, mask1, filt0, + filt1, out4, out5, out6, out7); + DUP4_ARG3(__lsx_vssrarni_b_h, out1, out0, VP8_FILTER_SHIFT, out3, out2, + VP8_FILTER_SHIFT, out5, out4, VP8_FILTER_SHIFT, out7, out6, + VP8_FILTER_SHIFT, out0, out1, out2, out3); + DUP4_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out2, 128, out3, 128, out0, + out1, out2, out3); + __lsx_vstx(out0, dst, 0); + __lsx_vstx(out1, dst, dst_stride); + __lsx_vstx(out2, dst, dst_stride_x2); + __lsx_vstx(out3, dst, dst_stride_x3); + dst += dst_stride_x4; + } +} + +static void common_vt_4t_8w_lsx(uint8_t *RESTRICT src, int32_t src_stride, + uint8_t *RESTRICT dst, int32_t dst_stride, + const int8_t *filter, int32_t height) { + uint32_t loop_cnt; + int32_t src_stride_x2 = src_stride << 1; + int32_t src_stride_x3 = src_stride_x2 + src_stride; + int32_t src_stride_x4 = src_stride << 2; + int32_t dst_stride_x2 = dst_stride << 1; + int32_t dst_stride_x3 = dst_stride_x2 + dst_stride; + int32_t dst_stride_x4 = dst_stride << 2; + __m128i src0, src1, src2, src7, src8, src9, src10; + __m128i src10_r, src72_r, src98_r, src21_r, src87_r, src109_r, filt0, filt1; + __m128i tmp0, tmp1; + __m128i filt, out0_r, out1_r, out2_r, out3_r; + + src -= src_stride; + filt = __lsx_vld(filter, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt0, filt1); + + DUP2_ARG2(__lsx_vldx, src, 0, src, src_stride, src0, src1); + src2 = __lsx_vldx(src, src_stride_x2); + src += src_stride_x3; + + DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + src2 = __lsx_vxori_b(src2, 128); + DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src10_r, src21_r); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src7, src8, src9, src10); + src += src_stride_x4; + + DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7, + src8, src9, src10); + DUP4_ARG2(__lsx_vilvl_b, src7, src2, src8, src7, src9, src8, src10, src9, + src72_r, src87_r, src98_r, src109_r); + out0_r = FILT_4TAP_DPADD_H(src10_r, src72_r, filt0, filt1); + out1_r = FILT_4TAP_DPADD_H(src21_r, src87_r, filt0, filt1); + out2_r = FILT_4TAP_DPADD_H(src72_r, src98_r, filt0, filt1); + out3_r = FILT_4TAP_DPADD_H(src87_r, src109_r, filt0, filt1); + DUP2_ARG3(__lsx_vssrarni_b_h, out1_r, out0_r, VP8_FILTER_SHIFT, out3_r, + out2_r, VP8_FILTER_SHIFT, tmp0, tmp1); + DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + __lsx_vstelm_d(tmp0, dst, 0, 0); + __lsx_vstelm_d(tmp0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(tmp1, dst + dst_stride_x2, 0, 0); + __lsx_vstelm_d(tmp1, dst + dst_stride_x3, 0, 1); + dst += dst_stride_x4; + + src10_r = src98_r; + src21_r = src109_r; + src2 = src10; + } +} + +static void common_vt_4t_16w_lsx(uint8_t *RESTRICT src, int32_t src_stride, + uint8_t *RESTRICT dst, int32_t dst_stride, + const int8_t *filter, int32_t height) { + uint32_t loop_cnt; + int32_t src_stride_x2 = src_stride << 1; + int32_t src_stride_x3 = src_stride_x2 + src_stride; + int32_t src_stride_x4 = src_stride << 2; + int32_t dst_stride_x2 = dst_stride << 1; + int32_t dst_stride_x3 = dst_stride_x2 + dst_stride; + int32_t dst_stride_x4 = dst_stride << 2; + __m128i src0, src1, src2, src3, src4, src5, src6; + __m128i src10_r, src32_r, src54_r, src21_r, src43_r, src65_r, src10_l; + __m128i src32_l, src54_l, src21_l, src43_l, src65_l, filt0, filt1; + __m128i tmp0, tmp1, tmp2, tmp3; + __m128i filt, out0_r, out1_r, out2_r, out3_r, out0_l, out1_l, out2_l, out3_l; + + src -= src_stride; + filt = __lsx_vld(filter, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt0, filt1); + + DUP2_ARG2(__lsx_vldx, src, 0, src, src_stride, src0, src1); + src2 = __lsx_vldx(src, src_stride_x2); + src += src_stride_x3; + + DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + src2 = __lsx_vxori_b(src2, 128); + DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src10_r, src21_r); + DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, src10_l, src21_l); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src3, src4, src5, src6); + src += src_stride_x4; + + DUP4_ARG2(__lsx_vxori_b, src3, 128, src4, 128, src5, 128, src6, 128, src3, + src4, src5, src6); + DUP4_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, src5, src4, src6, src5, + src32_r, src43_r, src54_r, src65_r); + DUP4_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, src5, src4, src6, src5, + src32_l, src43_l, src54_l, src65_l); + out0_r = FILT_4TAP_DPADD_H(src10_r, src32_r, filt0, filt1); + out1_r = FILT_4TAP_DPADD_H(src21_r, src43_r, filt0, filt1); + out2_r = FILT_4TAP_DPADD_H(src32_r, src54_r, filt0, filt1); + out3_r = FILT_4TAP_DPADD_H(src43_r, src65_r, filt0, filt1); + out0_l = FILT_4TAP_DPADD_H(src10_l, src32_l, filt0, filt1); + out1_l = FILT_4TAP_DPADD_H(src21_l, src43_l, filt0, filt1); + out2_l = FILT_4TAP_DPADD_H(src32_l, src54_l, filt0, filt1); + out3_l = FILT_4TAP_DPADD_H(src43_l, src65_l, filt0, filt1); + DUP4_ARG3(__lsx_vssrarni_b_h, out0_l, out0_r, VP8_FILTER_SHIFT, out1_l, + out1_r, VP8_FILTER_SHIFT, out2_l, out2_r, VP8_FILTER_SHIFT, + out3_l, out3_r, VP8_FILTER_SHIFT, tmp0, tmp1, tmp2, tmp3); + DUP4_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp2, 128, tmp3, 128, tmp0, + tmp1, tmp2, tmp3); + __lsx_vstx(tmp0, dst, 0); + __lsx_vstx(tmp1, dst, dst_stride); + __lsx_vstx(tmp2, dst, dst_stride_x2); + __lsx_vstx(tmp3, dst, dst_stride_x3); + dst += dst_stride_x4; + + src10_r = src54_r; + src21_r = src65_r; + src10_l = src54_l; + src21_l = src65_l; + src2 = src6; + } +} + +static inline void common_hv_4ht_4vt_8w_lsx( + uint8_t *RESTRICT src, int32_t src_stride, uint8_t *RESTRICT dst, + int32_t dst_stride, const int8_t *filter_horiz, const int8_t *filter_vert, + int32_t height) { + uint32_t loop_cnt; + int32_t src_stride_x2 = src_stride << 1; + int32_t src_stride_x3 = src_stride_x2 + src_stride; + int32_t src_stride_x4 = src_stride << 2; + int32_t dst_stride_x2 = dst_stride << 1; + int32_t dst_stride_x3 = dst_stride_x2 + dst_stride; + int32_t dst_stride_x4 = dst_stride << 2; + __m128i src0, src1, src2, src3, src4, src5, src6, filt_hz0, filt_hz1; + __m128i mask0, mask1, out0, out1; + __m128i filt, filt_vt0, filt_vt1, tmp0, tmp1, tmp2, tmp3; + __m128i hz_out0, hz_out1, hz_out2, hz_out3; + __m128i vec0, vec1, vec2, vec3, vec4; + + mask0 = __lsx_vld(vp8_mc_filt_mask_arr, 0); + src -= 1 + src_stride; + + filt = __lsx_vld(filter_horiz, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt_hz0, filt_hz1); + mask1 = __lsx_vaddi_bu(mask0, 2); + + DUP2_ARG2(__lsx_vldx, src, 0, src, src_stride, src0, src1); + src2 = __lsx_vldx(src, src_stride_x2); + src += src_stride_x3; + + DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + src2 = __lsx_vxori_b(src2, 128); + hz_out0 = HORIZ_4TAP_FILT(src0, src0, mask0, mask1, filt_hz0, filt_hz1); + hz_out1 = HORIZ_4TAP_FILT(src1, src1, mask0, mask1, filt_hz0, filt_hz1); + hz_out2 = HORIZ_4TAP_FILT(src2, src2, mask0, mask1, filt_hz0, filt_hz1); + DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out2, hz_out1, vec0, vec2); + + filt = __lsx_vld(filter_vert, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt_vt0, filt_vt1); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src3, src4, src5, src6); + src += src_stride_x4; + + DUP4_ARG2(__lsx_vxori_b, src3, 128, src4, 128, src5, 128, src6, 128, src3, + src4, src5, src6); + hz_out3 = HORIZ_4TAP_FILT(src3, src3, mask0, mask1, filt_hz0, filt_hz1); + vec1 = __lsx_vpackev_b(hz_out3, hz_out2); + tmp0 = FILT_4TAP_DPADD_H(vec0, vec1, filt_vt0, filt_vt1); + + hz_out0 = HORIZ_4TAP_FILT(src4, src4, mask0, mask1, filt_hz0, filt_hz1); + vec3 = __lsx_vpackev_b(hz_out0, hz_out3); + tmp1 = FILT_4TAP_DPADD_H(vec2, vec3, filt_vt0, filt_vt1); + + hz_out1 = HORIZ_4TAP_FILT(src5, src5, mask0, mask1, filt_hz0, filt_hz1); + vec4 = __lsx_vpackev_b(hz_out1, hz_out0); + tmp2 = FILT_4TAP_DPADD_H(vec1, vec4, filt_vt0, filt_vt1); + + hz_out2 = HORIZ_4TAP_FILT(src6, src6, mask0, mask1, filt_hz0, filt_hz1); + DUP2_ARG2(__lsx_vpackev_b, hz_out0, hz_out3, hz_out2, hz_out1, vec0, vec1); + tmp3 = FILT_4TAP_DPADD_H(vec0, vec1, filt_vt0, filt_vt1); + + DUP2_ARG3(__lsx_vssrarni_b_h, tmp1, tmp0, 7, tmp3, tmp2, 7, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + __lsx_vstelm_d(out0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(out1, dst + dst_stride_x2, 0, 0); + __lsx_vstelm_d(out1, dst + dst_stride_x3, 0, 1); + dst += dst_stride_x4; + + vec0 = vec4; + vec2 = vec1; + } +} + +static void common_hv_4ht_4vt_16w_lsx(uint8_t *RESTRICT src, int32_t src_stride, + uint8_t *RESTRICT dst, int32_t dst_stride, + const int8_t *filter_horiz, + const int8_t *filter_vert, + int32_t height) { + common_hv_4ht_4vt_8w_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert, height); + common_hv_4ht_4vt_8w_lsx(src + 8, src_stride, dst + 8, dst_stride, + filter_horiz, filter_vert, height); +} + +static inline void common_hv_6ht_4vt_8w_lsx( + uint8_t *RESTRICT src, int32_t src_stride, uint8_t *RESTRICT dst, + int32_t dst_stride, const int8_t *filter_horiz, const int8_t *filter_vert, + int32_t height) { + uint32_t loop_cnt; + int32_t src_stride_x2 = src_stride << 1; + int32_t src_stride_x3 = src_stride_x2 + src_stride; + int32_t src_stride_x4 = src_stride << 2; + int32_t dst_stride_x2 = dst_stride << 1; + int32_t dst_stride_x3 = dst_stride_x2 + dst_stride; + int32_t dst_stride_x4 = dst_stride << 2; + + __m128i src0, src1, src2, src3, src4, src5, src6; + __m128i filt_hz0, filt_hz1, filt_hz2, mask0, mask1, mask2; + __m128i filt, filt_vt0, filt_vt1, hz_out0, hz_out1, hz_out2, hz_out3; + __m128i tmp0, tmp1, tmp2, tmp3, vec0, vec1, vec2, vec3; + __m128i out0, out1; + + mask0 = __lsx_vld(vp8_mc_filt_mask_arr, 0); + src -= (2 + src_stride); + + filt = __lsx_vld(filter_horiz, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt_hz0, filt_hz1); + filt_hz2 = __lsx_vreplvei_h(filt, 2); + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + + DUP2_ARG2(__lsx_vldx, src, 0, src, src_stride, src0, src1); + src2 = __lsx_vldx(src, src_stride_x2); + src += src_stride_x3; + + DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + src2 = __lsx_vxori_b(src2, 128); + hz_out0 = HORIZ_6TAP_FILT(src0, src0, mask0, mask1, mask2, filt_hz0, filt_hz1, + filt_hz2); + hz_out1 = HORIZ_6TAP_FILT(src1, src1, mask0, mask1, mask2, filt_hz0, filt_hz1, + filt_hz2); + hz_out2 = HORIZ_6TAP_FILT(src2, src2, mask0, mask1, mask2, filt_hz0, filt_hz1, + filt_hz2); + DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out2, hz_out1, vec0, vec2); + + filt = __lsx_vld(filter_vert, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt_vt0, filt_vt1); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src3, src4, src5, src6); + src += src_stride_x4; + DUP4_ARG2(__lsx_vxori_b, src3, 128, src4, 128, src5, 128, src6, 128, src3, + src4, src5, src6); + + hz_out3 = HORIZ_6TAP_FILT(src3, src3, mask0, mask1, mask2, filt_hz0, + filt_hz1, filt_hz2); + vec1 = __lsx_vpackev_b(hz_out3, hz_out2); + tmp0 = FILT_4TAP_DPADD_H(vec0, vec1, filt_vt0, filt_vt1); + + hz_out0 = HORIZ_6TAP_FILT(src4, src4, mask0, mask1, mask2, filt_hz0, + filt_hz1, filt_hz2); + vec3 = __lsx_vpackev_b(hz_out0, hz_out3); + tmp1 = FILT_4TAP_DPADD_H(vec2, vec3, filt_vt0, filt_vt1); + + hz_out1 = HORIZ_6TAP_FILT(src5, src5, mask0, mask1, mask2, filt_hz0, + filt_hz1, filt_hz2); + vec0 = __lsx_vpackev_b(hz_out1, hz_out0); + tmp2 = FILT_4TAP_DPADD_H(vec1, vec0, filt_vt0, filt_vt1); + + hz_out2 = HORIZ_6TAP_FILT(src6, src6, mask0, mask1, mask2, filt_hz0, + filt_hz1, filt_hz2); + DUP2_ARG2(__lsx_vpackev_b, hz_out0, hz_out3, hz_out2, hz_out1, vec1, vec2); + tmp3 = FILT_4TAP_DPADD_H(vec1, vec2, filt_vt0, filt_vt1); + + DUP2_ARG3(__lsx_vssrarni_b_h, tmp1, tmp0, 7, tmp3, tmp2, 7, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + __lsx_vstelm_d(out0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(out1, dst + dst_stride_x2, 0, 0); + __lsx_vstelm_d(out1, dst + dst_stride_x3, 0, 1); + dst += dst_stride_x4; + } +} + +static void common_hv_6ht_4vt_16w_lsx(uint8_t *RESTRICT src, int32_t src_stride, + uint8_t *RESTRICT dst, int32_t dst_stride, + const int8_t *filter_horiz, + const int8_t *filter_vert, + int32_t height) { + common_hv_6ht_4vt_8w_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert, height); + common_hv_6ht_4vt_8w_lsx(src + 8, src_stride, dst + 8, dst_stride, + filter_horiz, filter_vert, height); +} + +static inline void common_hv_4ht_6vt_8w_lsx( + uint8_t *RESTRICT src, int32_t src_stride, uint8_t *RESTRICT dst, + int32_t dst_stride, const int8_t *filter_horiz, const int8_t *filter_vert, + int32_t height) { + uint32_t loop_cnt; + int32_t src_stride_x2 = src_stride << 1; + int32_t src_stride_x3 = src_stride_x2 + src_stride; + int32_t src_stride_x4 = src_stride << 2; + int32_t dst_stride_x2 = dst_stride << 1; + int32_t dst_stride_x3 = dst_stride_x2 + dst_stride; + int32_t dst_stride_x4 = dst_stride << 2; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8; + __m128i filt_hz0, filt_hz1, mask0, mask1; + __m128i filt, filt_vt0, filt_vt1, filt_vt2, tmp0, tmp1, tmp2, tmp3; + __m128i hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6; + __m128i hz_out7, hz_out8, out0, out1, out2, out3, out4, out5, out6, out7; + __m128i vec0, vec1; + + mask0 = __lsx_vld(vp8_mc_filt_mask_arr, 0); + src -= 1 + src_stride_x2; + + filt = __lsx_vld(filter_horiz, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt_hz0, filt_hz1); + mask1 = __lsx_vaddi_bu(mask0, 2); + + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src0, src1, src2, src3); + src += src_stride_x4; + src4 = __lsx_vld(src, 0); + src += src_stride; + + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + src4 = __lsx_vxori_b(src4, 128); + hz_out0 = HORIZ_4TAP_FILT(src0, src0, mask0, mask1, filt_hz0, filt_hz1); + hz_out1 = HORIZ_4TAP_FILT(src1, src1, mask0, mask1, filt_hz0, filt_hz1); + hz_out2 = HORIZ_4TAP_FILT(src2, src2, mask0, mask1, filt_hz0, filt_hz1); + hz_out3 = HORIZ_4TAP_FILT(src3, src3, mask0, mask1, filt_hz0, filt_hz1); + hz_out4 = HORIZ_4TAP_FILT(src4, src4, mask0, mask1, filt_hz0, filt_hz1); + DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, out0, out1); + DUP2_ARG2(__lsx_vpackev_b, hz_out2, hz_out1, hz_out4, hz_out3, out3, out4); + + filt = __lsx_vld(filter_vert, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt_vt0, filt_vt1); + filt_vt2 = __lsx_vreplvei_h(filt, 2); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src5, src6, src7, src8); + src += src_stride_x4; + + DUP4_ARG2(__lsx_vxori_b, src5, 128, src6, 128, src7, 128, src8, 128, src5, + src6, src7, src8); + hz_out5 = HORIZ_4TAP_FILT(src5, src5, mask0, mask1, filt_hz0, filt_hz1); + out2 = __lsx_vpackev_b(hz_out5, hz_out4); + tmp0 = DPADD_H3(out0, out1, out2, filt_vt0, filt_vt1, filt_vt2); + + hz_out6 = HORIZ_4TAP_FILT(src6, src6, mask0, mask1, filt_hz0, filt_hz1); + out5 = __lsx_vpackev_b(hz_out6, hz_out5); + tmp1 = DPADD_H3(out3, out4, out5, filt_vt0, filt_vt1, filt_vt2); + + hz_out7 = HORIZ_4TAP_FILT(src7, src7, mask0, mask1, filt_hz0, filt_hz1); + out6 = __lsx_vpackev_b(hz_out7, hz_out6); + tmp2 = DPADD_H3(out1, out2, out6, filt_vt0, filt_vt1, filt_vt2); + + hz_out8 = HORIZ_4TAP_FILT(src8, src8, mask0, mask1, filt_hz0, filt_hz1); + out7 = __lsx_vpackev_b(hz_out8, hz_out7); + tmp3 = DPADD_H3(out4, out5, out7, filt_vt0, filt_vt1, filt_vt2); + DUP2_ARG3(__lsx_vssrarni_b_h, tmp1, tmp0, 7, tmp3, tmp2, 7, vec0, vec1); + DUP2_ARG2(__lsx_vxori_b, vec0, 128, vec1, 128, vec0, vec1); + __lsx_vstelm_d(vec0, dst, 0, 0); + __lsx_vstelm_d(vec0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(vec1, dst + dst_stride_x2, 0, 0); + __lsx_vstelm_d(vec1, dst + dst_stride_x3, 0, 1); + dst += dst_stride_x4; + hz_out4 = hz_out8; + out0 = out2; + out1 = out6; + out3 = out5; + out4 = out7; + } +} + +static void common_hv_4ht_6vt_16w_lsx(uint8_t *RESTRICT src, int32_t src_stride, + uint8_t *RESTRICT dst, int32_t dst_stride, + const int8_t *filter_horiz, + const int8_t *filter_vert, + int32_t height) { + common_hv_4ht_6vt_8w_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert, height); + common_hv_4ht_6vt_8w_lsx(src + 8, src_stride, dst + 8, dst_stride, + filter_horiz, filter_vert, height); +} + +typedef void (*PVp8SixtapPredictFunc1)( + uint8_t *RESTRICT src, int32_t src_stride, uint8_t *RESTRICT dst, + int32_t dst_stride, const int8_t *filter_horiz, const int8_t *filter_vert, + int32_t height); + +typedef void (*PVp8SixtapPredictFunc2)(uint8_t *RESTRICT src, + int32_t src_stride, + uint8_t *RESTRICT dst, + int32_t dst_stride, const int8_t *filter, + int32_t height); + +void vp8_sixtap_predict8x8_lsx(uint8_t *RESTRICT src, int32_t src_stride, + int32_t xoffset, int32_t yoffset, + uint8_t *RESTRICT dst, int32_t dst_stride) { + const int8_t *h_filter = vp8_subpel_filters_lsx[xoffset - 1]; + const int8_t *v_filter = vp8_subpel_filters_lsx[yoffset - 1]; + + static PVp8SixtapPredictFunc1 Predict8x8Funcs1[4] = { + common_hv_6ht_6vt_8w_lsx, + common_hv_6ht_4vt_8w_lsx, + common_hv_4ht_6vt_8w_lsx, + common_hv_4ht_4vt_8w_lsx, + }; + + static PVp8SixtapPredictFunc2 Predict8x8Funcs2[4] = { common_vt_6t_8w_lsx, + common_vt_4t_8w_lsx, + common_hz_6t_8w_lsx, + common_hz_4t_8w_lsx }; + + if (yoffset < 8 && xoffset < 8) { + if (yoffset) { + if (xoffset) { + switch (xoffset & 1) { + case 0: + switch (yoffset & 1) { + case 0: + Predict8x8Funcs1[0](src, src_stride, dst, dst_stride, h_filter, + v_filter, 8); + break; + + case 1: + Predict8x8Funcs1[1](src, src_stride, dst, dst_stride, h_filter, + v_filter + 1, 8); + break; + } + break; + + case 1: + switch (yoffset & 1) { + case 0: + Predict8x8Funcs1[2](src, src_stride, dst, dst_stride, + h_filter + 1, v_filter, 8); + break; + + case 1: + Predict8x8Funcs1[3](src, src_stride, dst, dst_stride, + h_filter + 1, v_filter + 1, 8); + break; + } + break; + } + } else { + switch (yoffset & 1) { + case 0: + Predict8x8Funcs2[0](src, src_stride, dst, dst_stride, v_filter, 8); + break; + + case 1: + Predict8x8Funcs2[1](src, src_stride, dst, dst_stride, v_filter + 1, + 8); + break; + } + } + } else { + switch (xoffset & 1) { + case 1: + Predict8x8Funcs2[3](src, src_stride, dst, dst_stride, h_filter + 1, + 8); + break; + } + switch (xoffset) { + case 0: vp8_copy_mem8x8(src, src_stride, dst, dst_stride); break; + case 2: + case 4: + case 6: + Predict8x8Funcs2[2](src, src_stride, dst, dst_stride, h_filter, 8); + break; + } + } + } +} + +void vp8_sixtap_predict16x16_lsx(uint8_t *RESTRICT src, int32_t src_stride, + int32_t xoffset, int32_t yoffset, + uint8_t *RESTRICT dst, int32_t dst_stride) { + const int8_t *h_filter = vp8_subpel_filters_lsx[xoffset - 1]; + const int8_t *v_filter = vp8_subpel_filters_lsx[yoffset - 1]; + + static PVp8SixtapPredictFunc1 Predict16x16Funcs1[4] = { + common_hv_6ht_6vt_16w_lsx, + common_hv_6ht_4vt_16w_lsx, + common_hv_4ht_6vt_16w_lsx, + common_hv_4ht_4vt_16w_lsx, + }; + + static PVp8SixtapPredictFunc2 Predict16x16Funcs2[4] = { + common_vt_6t_16w_lsx, common_vt_4t_16w_lsx, common_hz_6t_16w_lsx, + common_hz_4t_16w_lsx + }; + + if (yoffset < 8 && xoffset < 8) { + if (yoffset) { + if (xoffset) { + switch (xoffset & 1) { + case 0: + switch (yoffset & 1) { + case 0: + Predict16x16Funcs1[0](src, src_stride, dst, dst_stride, + h_filter, v_filter, 16); + break; + + case 1: + Predict16x16Funcs1[1](src, src_stride, dst, dst_stride, + h_filter, v_filter + 1, 16); + break; + } + break; + + case 1: + switch (yoffset & 1) { + case 0: + Predict16x16Funcs1[2](src, src_stride, dst, dst_stride, + h_filter + 1, v_filter, 16); + break; + + case 1: + Predict16x16Funcs1[3](src, src_stride, dst, dst_stride, + h_filter, v_filter + 1, 16); + break; + } + break; + } + } else { + switch (yoffset & 1) { + case 0: + Predict16x16Funcs2[0](src, src_stride, dst, dst_stride, v_filter, + 16); + break; + + case 1: + Predict16x16Funcs2[1](src, src_stride, dst, dst_stride, + v_filter + 1, 16); + break; + } + } + } else { + switch (xoffset & 1) { + case 1: + Predict16x16Funcs2[3](src, src_stride, dst, dst_stride, h_filter + 1, + 16); + break; + } + switch (xoffset) { + case 0: vp8_copy_mem16x16(src, src_stride, dst, dst_stride); break; + case 2: + case 4: + case 6: + Predict16x16Funcs2[2](src, src_stride, dst, dst_stride, h_filter, 16); + break; + } + } + } +} diff --git a/media/libvpx/libvpx/vp8/common/rtcd_defs.pl b/media/libvpx/libvpx/vp8/common/rtcd_defs.pl --- a/media/libvpx/libvpx/vp8/common/rtcd_defs.pl +++ b/media/libvpx/libvpx/vp8/common/rtcd_defs.pl @@ -36,29 +36,29 @@ add_proto qw/void vp8_dequant_idct_add/, "short *input, short *dq, unsigned char *dest, int stride"; specialize qw/vp8_dequant_idct_add mmx neon dspr2 msa mmi/; add_proto qw/void vp8_dequant_idct_add_y_block/, "short *q, short *dq, unsigned char *dst, int stride, char *eobs"; -specialize qw/vp8_dequant_idct_add_y_block sse2 neon dspr2 msa mmi/; +specialize qw/vp8_dequant_idct_add_y_block sse2 neon dspr2 msa mmi lsx/; add_proto qw/void vp8_dequant_idct_add_uv_block/, "short *q, short *dq, unsigned char *dst_u, unsigned char *dst_v, int stride, char *eobs"; -specialize qw/vp8_dequant_idct_add_uv_block sse2 neon dspr2 msa mmi/; +specialize qw/vp8_dequant_idct_add_uv_block sse2 neon dspr2 msa mmi lsx/; # # Loopfilter # add_proto qw/void vp8_loop_filter_mbv/, "unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi"; -specialize qw/vp8_loop_filter_mbv sse2 neon dspr2 msa mmi/; +specialize qw/vp8_loop_filter_mbv sse2 neon dspr2 msa mmi lsx/; add_proto qw/void vp8_loop_filter_bv/, "unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi"; -specialize qw/vp8_loop_filter_bv sse2 neon dspr2 msa mmi/; +specialize qw/vp8_loop_filter_bv sse2 neon dspr2 msa mmi lsx/; add_proto qw/void vp8_loop_filter_mbh/, "unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi"; -specialize qw/vp8_loop_filter_mbh sse2 neon dspr2 msa mmi/; +specialize qw/vp8_loop_filter_mbh sse2 neon dspr2 msa mmi lsx/; add_proto qw/void vp8_loop_filter_bh/, "unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi"; -specialize qw/vp8_loop_filter_bh sse2 neon dspr2 msa mmi/; +specialize qw/vp8_loop_filter_bh sse2 neon dspr2 msa mmi lsx/; add_proto qw/void vp8_loop_filter_simple_mbv/, "unsigned char *y_ptr, int y_stride, const unsigned char *blimit"; specialize qw/vp8_loop_filter_simple_mbv sse2 neon msa mmi/; $vp8_loop_filter_simple_mbv_c=vp8_loop_filter_simple_vertical_edge_c; @@ -106,11 +106,11 @@ add_proto qw/void vp8_short_inv_walsh4x4/, "short *input, short *mb_dqcoeff"; specialize qw/vp8_short_inv_walsh4x4 sse2 neon dspr2 msa mmi/; #idct1_scalar_add add_proto qw/void vp8_dc_only_idct_add/, "short input_dc, unsigned char *pred_ptr, int pred_stride, unsigned char *dst_ptr, int dst_stride"; -specialize qw/vp8_dc_only_idct_add mmx neon dspr2 msa mmi/; +specialize qw/vp8_dc_only_idct_add mmx neon dspr2 msa mmi lsx/; # # RECON # add_proto qw/void vp8_copy_mem16x16/, "unsigned char *src, int src_stride, unsigned char *dst, int dst_stride"; @@ -144,14 +144,14 @@ # # Subpixel # add_proto qw/void vp8_sixtap_predict16x16/, "unsigned char *src_ptr, int src_pixels_per_line, int xoffset, int yoffset, unsigned char *dst_ptr, int dst_pitch"; -specialize qw/vp8_sixtap_predict16x16 sse2 ssse3 neon dspr2 msa mmi/; +specialize qw/vp8_sixtap_predict16x16 sse2 ssse3 neon dspr2 msa mmi lsx/; add_proto qw/void vp8_sixtap_predict8x8/, "unsigned char *src_ptr, int src_pixels_per_line, int xoffset, int yoffset, unsigned char *dst_ptr, int dst_pitch"; -specialize qw/vp8_sixtap_predict8x8 sse2 ssse3 neon dspr2 msa mmi/; +specialize qw/vp8_sixtap_predict8x8 sse2 ssse3 neon dspr2 msa mmi lsx/; add_proto qw/void vp8_sixtap_predict8x4/, "unsigned char *src_ptr, int src_pixels_per_line, int xoffset, int yoffset, unsigned char *dst_ptr, int dst_pitch"; specialize qw/vp8_sixtap_predict8x4 sse2 ssse3 neon dspr2 msa mmi/; add_proto qw/void vp8_sixtap_predict4x4/, "unsigned char *src_ptr, int src_pixels_per_line, int xoffset, int yoffset, unsigned char *dst_ptr, int dst_pitch"; @@ -214,15 +214,10 @@ specialize qw/vp8_mbuverror sse2 msa/; # # Motion search # -add_proto qw/int vp8_full_search_sad/, "struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv"; -specialize qw/vp8_full_search_sad sse3 sse4_1/; -$vp8_full_search_sad_sse3=vp8_full_search_sadx3; -$vp8_full_search_sad_sse4_1=vp8_full_search_sadx8; - add_proto qw/int vp8_refining_search_sad/, "struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int error_per_bit, int search_range, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv"; specialize qw/vp8_refining_search_sad sse2 msa/; $vp8_refining_search_sad_sse2=vp8_refining_search_sadx4; $vp8_refining_search_sad_msa=vp8_refining_search_sadx4; diff --git a/media/libvpx/libvpx/vp8/common/x86/bilinear_filter_sse2.c b/media/libvpx/libvpx/vp8/common/x86/bilinear_filter_sse2.c --- a/media/libvpx/libvpx/vp8/common/x86/bilinear_filter_sse2.c +++ b/media/libvpx/libvpx/vp8/common/x86/bilinear_filter_sse2.c @@ -311,14 +311,14 @@ const __m128i row_1_filtered = _mm_mullo_epi16(row_1, vfilter_1); const __m128i sum = _mm_add_epi16(row_0_filtered, row_1_filtered); const __m128i compensated = _mm_add_epi16(sum, round_factor); const __m128i shifted = _mm_srai_epi16(compensated, VP8_FILTER_SHIFT); __m128i packed = _mm_packus_epi16(shifted, shifted); - storeu_uint32(dst, _mm_cvtsi128_si32(packed)); + storeu_int32(dst, _mm_cvtsi128_si32(packed)); packed = _mm_srli_si128(packed, 4); dst += stride; - storeu_uint32(dst, _mm_cvtsi128_si32(packed)); + storeu_int32(dst, _mm_cvtsi128_si32(packed)); dst += stride; src += 8; } } } diff --git a/media/libvpx/libvpx/vp8/decoder/decodeframe.c b/media/libvpx/libvpx/vp8/decoder/decodeframe.c --- a/media/libvpx/libvpx/vp8/decoder/decodeframe.c +++ b/media/libvpx/libvpx/vp8/decoder/decodeframe.c @@ -870,12 +870,12 @@ xd->frame_type = pc->frame_type; xd->mode_info_context->mbmi.mode = DC_PRED; xd->mode_info_stride = pc->mode_info_stride; xd->corrupted = 0; /* init without corruption */ - xd->fullpixel_mask = 0xffffffff; - if (pc->full_pixel) xd->fullpixel_mask = 0xfffffff8; + xd->fullpixel_mask = ~0; + if (pc->full_pixel) xd->fullpixel_mask = ~7; } int vp8_decode_frame(VP8D_COMP *pbi) { vp8_reader *const bc = &pbi->mbc[8]; VP8_COMMON *const pc = &pbi->common; diff --git a/media/libvpx/libvpx/vp8/decoder/decodemv.c b/media/libvpx/libvpx/vp8/decoder/decodemv.c --- a/media/libvpx/libvpx/vp8/decoder/decodemv.c +++ b/media/libvpx/libvpx/vp8/decoder/decodemv.c @@ -171,11 +171,12 @@ { 223, 1, 34 }, /* SUBMVREF_LEFT_ABOVE_SAME */ { 179, 121, 1 }, /* SUBMVREF_ABOVE_ZED */ { 208, 1, 1 } /* SUBMVREF_LEFT_ABOVE_ZED */ }; -static const vp8_prob *get_sub_mv_ref_prob(const int left, const int above) { +static const vp8_prob *get_sub_mv_ref_prob(const uint32_t left, + const uint32_t above) { int lez = (left == 0); int aez = (above == 0); int lea = (left == above); const vp8_prob *prob; diff --git a/media/libvpx/libvpx/vp8/decoder/threading.c b/media/libvpx/libvpx/vp8/decoder/threading.c --- a/media/libvpx/libvpx/vp8/decoder/threading.c +++ b/media/libvpx/libvpx/vp8/decoder/threading.c @@ -72,13 +72,13 @@ memcpy(mbd->dequant_y1_dc, xd->dequant_y1_dc, sizeof(xd->dequant_y1_dc)); memcpy(mbd->dequant_y1, xd->dequant_y1, sizeof(xd->dequant_y1)); memcpy(mbd->dequant_y2, xd->dequant_y2, sizeof(xd->dequant_y2)); memcpy(mbd->dequant_uv, xd->dequant_uv, sizeof(xd->dequant_uv)); - mbd->fullpixel_mask = 0xffffffff; + mbd->fullpixel_mask = ~0; - if (pc->full_pixel) mbd->fullpixel_mask = 0xfffffff8; + if (pc->full_pixel) mbd->fullpixel_mask = ~7; } for (i = 0; i < pc->mb_rows; ++i) vpx_atomic_store_release(&pbi->mt_current_mb_col[i], -1); } diff --git a/media/libvpx/libvpx/vp8/encoder/bitstream.c b/media/libvpx/libvpx/vp8/encoder/bitstream.c --- a/media/libvpx/libvpx/vp8/encoder/bitstream.c +++ b/media/libvpx/libvpx/vp8/encoder/bitstream.c @@ -170,13 +170,12 @@ } validate_buffer(w->buffer + w->pos, 1, w->buffer_end, w->error); w->buffer[w->pos++] = (lowvalue >> (24 - offset)) & 0xff; - lowvalue <<= offset; shift = count; - lowvalue &= 0xffffff; + lowvalue = (int)(((uint64_t)lowvalue << offset) & 0xffffff); count -= 8; } lowvalue <<= shift; } while (n); @@ -221,13 +220,12 @@ } validate_buffer(w->buffer + w->pos, 1, w->buffer_end, w->error); w->buffer[w->pos++] = (lowvalue >> (24 - offset)) & 0xff; - lowvalue <<= offset; shift = count; - lowvalue &= 0xffffff; + lowvalue = (int)(((uint64_t)lowvalue << offset) & 0xffffff); count -= 8; } lowvalue <<= shift; } while (n2); diff --git a/media/libvpx/libvpx/vp8/encoder/block.h b/media/libvpx/libvpx/vp8/encoder/block.h --- a/media/libvpx/libvpx/vp8/encoder/block.h +++ b/media/libvpx/libvpx/vp8/encoder/block.h @@ -90,12 +90,11 @@ int *mb_norm_activity_ptr; signed int act_zbin_adj; signed int last_act_zbin_adj; int *mvcost[2]; - /* MSVC generates code that thinks this is 16-byte aligned */ - DECLARE_ALIGNED(16, int*, mvsadcost[2]); + int *mvsadcost[2]; int (*mbmode_cost)[MB_MODE_COUNT]; int (*intra_uv_mode_cost)[MB_MODE_COUNT]; int (*bmode_costs)[10][10]; int *inter_bmode_costs; int (*token_costs)[COEF_BANDS][PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS]; diff --git a/media/libvpx/libvpx/vp8/encoder/boolhuff.h b/media/libvpx/libvpx/vp8/encoder/boolhuff.h --- a/media/libvpx/libvpx/vp8/encoder/boolhuff.h +++ b/media/libvpx/libvpx/vp8/encoder/boolhuff.h @@ -92,13 +92,12 @@ } validate_buffer(bc->buffer + bc->pos, 1, bc->buffer_end, bc->error); bc->buffer[bc->pos++] = (lowvalue >> (24 - offset) & 0xff); - lowvalue <<= offset; shift = count; - lowvalue &= 0xffffff; + lowvalue = (int)(((uint64_t)lowvalue << offset) & 0xffffff); count -= 8; } lowvalue <<= shift; bc->count = count; diff --git a/media/libvpx/libvpx/vp8/encoder/encodeframe.c b/media/libvpx/libvpx/vp8/encoder/encodeframe.c --- a/media/libvpx/libvpx/vp8/encoder/encodeframe.c +++ b/media/libvpx/libvpx/vp8/encoder/encodeframe.c @@ -67,14 +67,13 @@ static const unsigned char VP8_VAR_OFFS[16] = { 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128 }; /* Original activity measure from Tim T's code. */ -static unsigned int tt_activity_measure(VP8_COMP *cpi, MACROBLOCK *x) { +static unsigned int tt_activity_measure(MACROBLOCK *x) { unsigned int act; unsigned int sse; - (void)cpi; /* TODO: This could also be done over smaller areas (8x8), but that would * require extensive changes elsewhere, as lambda is assumed to be fixed * over an entire MB in most of the code. * Another option is to compute four 8x8 variances, and pick a single * lambda using a non-linear combination (e.g., the smallest, or second @@ -88,32 +87,25 @@ if (act < 8 << 12) act = act < 5 << 12 ? act : 5 << 12; return act; } -/* Stub for alternative experimental activity measures. */ -static unsigned int alt_activity_measure(VP8_COMP *cpi, MACROBLOCK *x, - int use_dc_pred) { - return vp8_encode_intra(cpi, x, use_dc_pred); -} - /* Measure the activity of the current macroblock * What we measure here is TBD so abstracted to this function */ #define ALT_ACT_MEASURE 1 -static unsigned int mb_activity_measure(VP8_COMP *cpi, MACROBLOCK *x, - int mb_row, int mb_col) { +static unsigned int mb_activity_measure(MACROBLOCK *x, int mb_row, int mb_col) { unsigned int mb_activity; if (ALT_ACT_MEASURE) { int use_dc_pred = (mb_col || mb_row) && (!mb_col || !mb_row); - /* Or use and alternative. */ - mb_activity = alt_activity_measure(cpi, x, use_dc_pred); + /* Or use an alternative. */ + mb_activity = vp8_encode_intra(x, use_dc_pred); } else { /* Original activity measure from Tim T's code. */ - mb_activity = tt_activity_measure(cpi, x); + mb_activity = tt_activity_measure(x); } if (mb_activity < VP8_ACTIVITY_AVG_MIN) mb_activity = VP8_ACTIVITY_AVG_MIN; return mb_activity; @@ -262,11 +254,11 @@ #endif /* Copy current mb to a buffer */ vp8_copy_mem16x16(x->src.y_buffer, x->src.y_stride, x->thismb, 16); /* measure activity */ - mb_activity = mb_activity_measure(cpi, x, mb_row, mb_col); + mb_activity = mb_activity_measure(x, mb_row, mb_col); /* Keep frame sum */ activity_sum += mb_activity; /* Store MB level activity details. */ @@ -632,16 +624,17 @@ } else { vp8_calc_ref_frame_costs(x->ref_frame_cost, cpi->prob_intra_coded, cpi->prob_last_coded, cpi->prob_gf_coded); } - xd->fullpixel_mask = 0xffffffff; - if (cm->full_pixel) xd->fullpixel_mask = 0xfffffff8; + xd->fullpixel_mask = ~0; + if (cm->full_pixel) xd->fullpixel_mask = ~7; vp8_zero(x->coef_counts); vp8_zero(x->ymode_count); - vp8_zero(x->uv_mode_count) x->prediction_error = 0; + vp8_zero(x->uv_mode_count); + x->prediction_error = 0; x->intra_error = 0; vp8_zero(x->count_mb_ref_frame_usage); } #if CONFIG_MULTITHREAD @@ -764,16 +757,16 @@ sem_post(&cpi->h_event_start_encoding[i]); } for (mb_row = 0; mb_row < cm->mb_rows; mb_row += (cpi->encoding_thread_count + 1)) { - vp8_zero(cm->left_context) + vp8_zero(cm->left_context); #if CONFIG_REALTIME_ONLY & CONFIG_ONTHEFLY_BITPACKING - tp = cpi->tok; + tp = cpi->tok; #else - tp = cpi->tok + mb_row * (cm->mb_cols * 16 * 24); + tp = cpi->tok + mb_row * (cm->mb_cols * 16 * 24); #endif encode_mb_row(cpi, cm, mb_row, x, xd, &tp, segment_counts, &totalrate); /* adjust to the next row of mbs */ @@ -856,14 +849,14 @@ #endif // CONFIG_MULTITHREAD { /* for each macroblock row in image */ for (mb_row = 0; mb_row < cm->mb_rows; ++mb_row) { - vp8_zero(cm->left_context) + vp8_zero(cm->left_context); #if CONFIG_REALTIME_ONLY & CONFIG_ONTHEFLY_BITPACKING - tp = cpi->tok; + tp = cpi->tok; #endif encode_mb_row(cpi, cm, mb_row, x, xd, &tp, segment_counts, &totalrate); /* adjust to the next row of mbs */ diff --git a/media/libvpx/libvpx/vp8/encoder/encodeintra.h b/media/libvpx/libvpx/vp8/encoder/encodeintra.h --- a/media/libvpx/libvpx/vp8/encoder/encodeintra.h +++ b/media/libvpx/libvpx/vp8/encoder/encodeintra.h @@ -14,11 +14,11 @@ #ifdef __cplusplus extern "C" { #endif -int vp8_encode_intra(VP8_COMP *cpi, MACROBLOCK *x, int use_dc_pred); +int vp8_encode_intra(MACROBLOCK *x, int use_dc_pred); void vp8_encode_intra16x16mby(MACROBLOCK *x); void vp8_encode_intra16x16mbuv(MACROBLOCK *x); void vp8_encode_intra4x4mby(MACROBLOCK *mb); void vp8_encode_intra4x4block(MACROBLOCK *x, int ib); #ifdef __cplusplus diff --git a/media/libvpx/libvpx/vp8/encoder/encodeintra.c b/media/libvpx/libvpx/vp8/encoder/encodeintra.c --- a/media/libvpx/libvpx/vp8/encoder/encodeintra.c +++ b/media/libvpx/libvpx/vp8/encoder/encodeintra.c @@ -16,14 +16,13 @@ #include "vp8/common/reconintra4x4.h" #include "encodemb.h" #include "vp8/common/invtrans.h" #include "encodeintra.h" -int vp8_encode_intra(VP8_COMP *cpi, MACROBLOCK *x, int use_dc_pred) { +int vp8_encode_intra(MACROBLOCK *x, int use_dc_pred) { int i; int intra_pred_var = 0; - (void)cpi; if (use_dc_pred) { x->e_mbd.mode_info_context->mbmi.mode = DC_PRED; x->e_mbd.mode_info_context->mbmi.uv_mode = DC_PRED; x->e_mbd.mode_info_context->mbmi.ref_frame = INTRA_FRAME; diff --git a/media/libvpx/libvpx/vp8/encoder/encodemv.c b/media/libvpx/libvpx/vp8/encoder/encodemv.c --- a/media/libvpx/libvpx/vp8/encoder/encodemv.c +++ b/media/libvpx/libvpx/vp8/encoder/encodemv.c @@ -158,11 +158,11 @@ static void calc_prob(vp8_prob *p, const unsigned int ct[2]) { const unsigned int tot = ct[0] + ct[1]; if (tot) { - const vp8_prob x = ((ct[0] * 255) / tot) & -2; + const vp8_prob x = ((ct[0] * 255) / tot) & ~1u; *p = x ? x : 1; } } static void update(vp8_writer *const w, const unsigned int ct[2], @@ -203,12 +203,15 @@ vp8_prob Pnew[MVPcount]; (void)rc; vp8_copy_array(Pnew, default_mvc, MVPcount); - vp8_zero(is_short_ct) vp8_zero(sign_ct) vp8_zero(bit_ct) vp8_zero(short_ct) - vp8_zero(short_bct) + vp8_zero(is_short_ct); + vp8_zero(sign_ct); + vp8_zero(bit_ct); + vp8_zero(short_ct); + vp8_zero(short_bct); /* j=0 */ { const int c = events[mv_max]; diff --git a/media/libvpx/libvpx/vp8/encoder/ethreading.c b/media/libvpx/libvpx/vp8/encoder/ethreading.c --- a/media/libvpx/libvpx/vp8/encoder/ethreading.c +++ b/media/libvpx/libvpx/vp8/encoder/ethreading.c @@ -468,12 +468,12 @@ mbd->left_context = &cm->left_context; mb->mvc = cm->fc.mvc; setup_mbby_copy(&mbr_ei[i].mb, x); - mbd->fullpixel_mask = 0xffffffff; - if (cm->full_pixel) mbd->fullpixel_mask = 0xfffffff8; + mbd->fullpixel_mask = ~0; + if (cm->full_pixel) mbd->fullpixel_mask = ~7; vp8_zero(mb->coef_counts); vp8_zero(x->ymode_count); mb->skip_true_count = 0; vp8_zero(mb->MVcount); diff --git a/media/libvpx/libvpx/vp8/encoder/firstpass.c b/media/libvpx/libvpx/vp8/encoder/firstpass.c --- a/media/libvpx/libvpx/vp8/encoder/firstpass.c +++ b/media/libvpx/libvpx/vp8/encoder/firstpass.c @@ -565,11 +565,11 @@ /* Copy current mb to a buffer */ vp8_copy_mem16x16(x->src.y_buffer, x->src.y_stride, x->thismb, 16); /* do intra 16x16 prediction */ - this_error = vp8_encode_intra(cpi, x, use_dc_pred); + this_error = vp8_encode_intra(x, use_dc_pred); /* "intrapenalty" below deals with situations where the intra * and inter error scores are very low (eg a plain black frame) * We do not have special cases in first pass for 0,0 and * nearest etc so all inter modes carry an overhead cost @@ -1629,11 +1629,10 @@ double loop_decay_rate = 1.00; /* Starting decay rate */ double this_frame_mv_in_out = 0.0; double mv_in_out_accumulator = 0.0; double abs_mv_in_out_accumulator = 0.0; - double mod_err_per_mb_accumulator = 0.0; int max_bits = frame_max_bits(cpi); /* Max for a single frame */ unsigned int allow_alt_ref = cpi->oxcf.play_alternate && cpi->oxcf.lag_in_frames; @@ -1680,13 +1679,10 @@ /* Accumulate error score of frames in this gf group */ mod_frame_err = calculate_modified_err(cpi, this_frame); gf_group_err += mod_frame_err; - mod_err_per_mb_accumulator += - mod_frame_err / DOUBLE_DIVIDE_CHECK((double)cpi->common.MBs); - if (EOF == input_stats(cpi, &next_frame)) break; /* Test for the case where there is a brief flash but the prediction * quality back to an earlier frame is then restored. */ diff --git a/media/libvpx/libvpx/vp8/encoder/lookahead.c b/media/libvpx/libvpx/vp8/encoder/lookahead.c --- a/media/libvpx/libvpx/vp8/encoder/lookahead.c +++ b/media/libvpx/libvpx/vp8/encoder/lookahead.c @@ -64,12 +64,12 @@ /* Keep last frame in lookahead buffer by increasing depth by 1.*/ depth += 1; /* Align the buffer dimensions */ - width = (width + 15) & ~15; - height = (height + 15) & ~15; + width = (width + 15) & ~15u; + height = (height + 15) & ~15u; /* Allocate the lookahead structures */ ctx = calloc(1, sizeof(*ctx)); if (ctx) { ctx->max_sz = depth; diff --git a/media/libvpx/libvpx/vp8/encoder/mcomp.h b/media/libvpx/libvpx/vp8/encoder/mcomp.h --- a/media/libvpx/libvpx/vp8/encoder/mcomp.h +++ b/media/libvpx/libvpx/vp8/encoder/mcomp.h @@ -48,14 +48,14 @@ fractional_mv_step_fp vp8_find_best_sub_pixel_step_iteratively; fractional_mv_step_fp vp8_find_best_sub_pixel_step; fractional_mv_step_fp vp8_find_best_half_pixel_step; fractional_mv_step_fp vp8_skip_fractional_mv_step; -typedef int (*vp8_full_search_fn_t)(MACROBLOCK *x, BLOCK *b, BLOCKD *d, - int_mv *ref_mv, int sad_per_bit, - int distance, vp8_variance_fn_ptr_t *fn_ptr, - int *mvcost[2], int_mv *center_mv); +int vp8_full_search_sad(MACROBLOCK *x, BLOCK *b, BLOCKD *d, int_mv *ref_mv, + int sad_per_bit, int distance, + vp8_variance_fn_ptr_t *fn_ptr, int *mvcost[2], + int_mv *center_mv); typedef int (*vp8_refining_search_fn_t)(MACROBLOCK *x, BLOCK *b, BLOCKD *d, int_mv *ref_mv, int sad_per_bit, int distance, vp8_variance_fn_ptr_t *fn_ptr, diff --git a/media/libvpx/libvpx/vp8/encoder/mcomp.c b/media/libvpx/libvpx/vp8/encoder/mcomp.c --- a/media/libvpx/libvpx/vp8/encoder/mcomp.c +++ b/media/libvpx/libvpx/vp8/encoder/mcomp.c @@ -202,23 +202,25 @@ #define IFMVCV(r, c, s, e) \ if (c >= minc && c <= maxc && r >= minr && r <= maxr) s else e; /* returns distortion + motion vector cost */ #define ERR(r, c) (MVC(r, c) + DIST(r, c)) /* checks if (r,c) has better score than previous best */ -#define CHECK_BETTER(v, r, c) \ - IFMVCV(r, c, \ - { \ - thismse = DIST(r, c); \ - if ((v = (MVC(r, c) + thismse)) < besterr) { \ - besterr = v; \ - br = r; \ - bc = c; \ - *distortion = thismse; \ - *sse1 = sse; \ - } \ - }, \ - v = UINT_MAX;) +#define CHECK_BETTER(v, r, c) \ + do { \ + IFMVCV(r, c, \ + { \ + thismse = DIST(r, c); \ + if ((v = (MVC(r, c) + thismse)) < besterr) { \ + besterr = v; \ + br = r; \ + bc = c; \ + *distortion = thismse; \ + *sse1 = sse; \ + } \ + }, \ + v = UINT_MAX;) \ + } while (0) int vp8_find_best_sub_pixel_step_iteratively(MACROBLOCK *x, BLOCK *b, BLOCKD *d, int_mv *bestmv, int_mv *ref_mv, int error_per_bit, const vp8_variance_fn_ptr_t *vfp, @@ -798,37 +800,37 @@ return bestmse; } #define CHECK_BOUNDS(range) \ - { \ + do { \ all_in = 1; \ all_in &= ((br - range) >= x->mv_row_min); \ all_in &= ((br + range) <= x->mv_row_max); \ all_in &= ((bc - range) >= x->mv_col_min); \ all_in &= ((bc + range) <= x->mv_col_max); \ - } + } while (0) #define CHECK_POINT \ { \ if (this_mv.as_mv.col < x->mv_col_min) continue; \ if (this_mv.as_mv.col > x->mv_col_max) continue; \ if (this_mv.as_mv.row < x->mv_row_min) continue; \ if (this_mv.as_mv.row > x->mv_row_max) continue; \ } #define CHECK_BETTER \ - { \ + do { \ if (thissad < bestsad) { \ thissad += \ mvsad_err_cost(&this_mv, &fcenter_mv, mvsadcost, sad_per_bit); \ if (thissad < bestsad) { \ bestsad = thissad; \ best_site = i; \ } \ } \ - } + } while (0) static const MV next_chkpts[6][3] = { { { -2, 0 }, { -1, -2 }, { 1, -2 } }, { { -1, -2 }, { 1, -2 }, { 2, 0 } }, { { 1, -2 }, { 2, 0 }, { 1, 2 } }, { { 2, 0 }, { 1, 2 }, { -1, 2 } }, { { 1, 2 }, { -1, 2 }, { -2, 0 } }, { { -1, 2 }, { -2, 0 }, { -1, -2 } } @@ -899,30 +901,30 @@ #else (void)search_param; #endif /* hex search */ - CHECK_BOUNDS(2) + CHECK_BOUNDS(2); if (all_in) { for (i = 0; i < 6; ++i) { this_mv.as_mv.row = br + hex[i].row; this_mv.as_mv.col = bc + hex[i].col; this_offset = base_offset + (this_mv.as_mv.row * in_what_stride) + this_mv.as_mv.col; thissad = vfp->sdf(what, what_stride, this_offset, in_what_stride); - CHECK_BETTER + CHECK_BETTER; } } else { for (i = 0; i < 6; ++i) { this_mv.as_mv.row = br + hex[i].row; this_mv.as_mv.col = bc + hex[i].col; CHECK_POINT this_offset = base_offset + (this_mv.as_mv.row * in_what_stride) + this_mv.as_mv.col; thissad = vfp->sdf(what, what_stride, this_offset, in_what_stride); - CHECK_BETTER + CHECK_BETTER; } } if (best_site == -1) { goto cal_neighbors; @@ -932,30 +934,30 @@ k = best_site; } for (j = 1; j < hex_range; ++j) { best_site = -1; - CHECK_BOUNDS(2) + CHECK_BOUNDS(2); if (all_in) { for (i = 0; i < 3; ++i) { this_mv.as_mv.row = br + next_chkpts[k][i].row; this_mv.as_mv.col = bc + next_chkpts[k][i].col; this_offset = base_offset + (this_mv.as_mv.row * (in_what_stride)) + this_mv.as_mv.col; thissad = vfp->sdf(what, what_stride, this_offset, in_what_stride); - CHECK_BETTER + CHECK_BETTER; } } else { for (i = 0; i < 3; ++i) { this_mv.as_mv.row = br + next_chkpts[k][i].row; this_mv.as_mv.col = bc + next_chkpts[k][i].col; CHECK_POINT this_offset = base_offset + (this_mv.as_mv.row * (in_what_stride)) + this_mv.as_mv.col; thissad = vfp->sdf(what, what_stride, this_offset, in_what_stride); - CHECK_BETTER + CHECK_BETTER; } } if (best_site == -1) { break; @@ -973,30 +975,30 @@ /* check 4 1-away neighbors */ cal_neighbors: for (j = 0; j < dia_range; ++j) { best_site = -1; - CHECK_BOUNDS(1) + CHECK_BOUNDS(1); if (all_in) { for (i = 0; i < 4; ++i) { this_mv.as_mv.row = br + neighbors[i].row; this_mv.as_mv.col = bc + neighbors[i].col; this_offset = base_offset + (this_mv.as_mv.row * (in_what_stride)) + this_mv.as_mv.col; thissad = vfp->sdf(what, what_stride, this_offset, in_what_stride); - CHECK_BETTER + CHECK_BETTER; } } else { for (i = 0; i < 4; ++i) { this_mv.as_mv.row = br + neighbors[i].row; this_mv.as_mv.col = bc + neighbors[i].col; CHECK_POINT this_offset = base_offset + (this_mv.as_mv.row * (in_what_stride)) + this_mv.as_mv.col; thissad = vfp->sdf(what, what_stride, this_offset, in_what_stride); - CHECK_BETTER + CHECK_BETTER; } } if (best_site == -1) { break; @@ -1276,14 +1278,14 @@ return fn_ptr->vf(what, what_stride, best_address, in_what_stride, &thissad) + mv_err_cost(&this_mv, center_mv, mvcost, x->errorperbit); } #endif // HAVE_SSE2 || HAVE_MSA -int vp8_full_search_sad_c(MACROBLOCK *x, BLOCK *b, BLOCKD *d, int_mv *ref_mv, - int sad_per_bit, int distance, - vp8_variance_fn_ptr_t *fn_ptr, int *mvcost[2], - int_mv *center_mv) { +int vp8_full_search_sad(MACROBLOCK *x, BLOCK *b, BLOCKD *d, int_mv *ref_mv, + int sad_per_bit, int distance, + vp8_variance_fn_ptr_t *fn_ptr, int *mvcost[2], + int_mv *center_mv) { unsigned char *what = (*(b->base_src) + b->src); int what_stride = b->src_stride; unsigned char *in_what; int pre_stride = x->e_mbd.pre.y_stride; unsigned char *base_pre = x->e_mbd.pre.y_buffer; @@ -1323,12 +1325,12 @@ /* Baseline value at the centre */ bestsad = fn_ptr->sdf(what, what_stride, bestaddress, in_what_stride) + mvsad_err_cost(best_mv, &fcenter_mv, mvsadcost, sad_per_bit); - /* Apply further limits to prevent us looking using vectors that - * stretch beyiond the UMV border + /* Apply further limits to prevent us looking using vectors that stretch + * beyond the UMV border */ if (col_min < x->mv_col_min) col_min = x->mv_col_min; if (col_max > x->mv_col_max) col_max = x->mv_col_max; @@ -1341,126 +1343,10 @@ check_here = r * mv_stride + in_what + col_min; for (c = col_min; c < col_max; ++c) { thissad = fn_ptr->sdf(what, what_stride, check_here, in_what_stride); - this_mv.as_mv.col = c; - thissad += mvsad_err_cost(&this_mv, &fcenter_mv, mvsadcost, sad_per_bit); - - if (thissad < bestsad) { - bestsad = thissad; - best_mv->as_mv.row = r; - best_mv->as_mv.col = c; - bestaddress = check_here; - } - - check_here++; - } - } - - this_mv.as_mv.row = best_mv->as_mv.row << 3; - this_mv.as_mv.col = best_mv->as_mv.col << 3; - - return fn_ptr->vf(what, what_stride, bestaddress, in_what_stride, &thissad) + - mv_err_cost(&this_mv, center_mv, mvcost, x->errorperbit); -} - -#if HAVE_SSSE3 -int vp8_full_search_sadx3(MACROBLOCK *x, BLOCK *b, BLOCKD *d, int_mv *ref_mv, - int sad_per_bit, int distance, - vp8_variance_fn_ptr_t *fn_ptr, int *mvcost[2], - int_mv *center_mv) { - unsigned char *what = (*(b->base_src) + b->src); - int what_stride = b->src_stride; - unsigned char *in_what; - int pre_stride = x->e_mbd.pre.y_stride; - unsigned char *base_pre = x->e_mbd.pre.y_buffer; - int in_what_stride = pre_stride; - int mv_stride = pre_stride; - unsigned char *bestaddress; - int_mv *best_mv = &d->bmi.mv; - int_mv this_mv; - unsigned int bestsad; - unsigned int thissad; - int r, c; - - unsigned char *check_here; - - int ref_row = ref_mv->as_mv.row; - int ref_col = ref_mv->as_mv.col; - - int row_min = ref_row - distance; - int row_max = ref_row + distance; - int col_min = ref_col - distance; - int col_max = ref_col + distance; - - unsigned int sad_array[3]; - - int *mvsadcost[2]; - int_mv fcenter_mv; - - mvsadcost[0] = x->mvsadcost[0]; - mvsadcost[1] = x->mvsadcost[1]; - fcenter_mv.as_mv.row = center_mv->as_mv.row >> 3; - fcenter_mv.as_mv.col = center_mv->as_mv.col >> 3; - - /* Work out the mid point for the search */ - in_what = base_pre + d->offset; - bestaddress = in_what + (ref_row * pre_stride) + ref_col; - - best_mv->as_mv.row = ref_row; - best_mv->as_mv.col = ref_col; - - /* Baseline value at the centre */ - bestsad = fn_ptr->sdf(what, what_stride, bestaddress, in_what_stride) + - mvsad_err_cost(best_mv, &fcenter_mv, mvsadcost, sad_per_bit); - - /* Apply further limits to prevent us looking using vectors that stretch - * beyond the UMV border - */ - if (col_min < x->mv_col_min) col_min = x->mv_col_min; - - if (col_max > x->mv_col_max) col_max = x->mv_col_max; - - if (row_min < x->mv_row_min) row_min = x->mv_row_min; - - if (row_max > x->mv_row_max) row_max = x->mv_row_max; - - for (r = row_min; r < row_max; ++r) { - this_mv.as_mv.row = r; - check_here = r * mv_stride + in_what + col_min; - c = col_min; - - while ((c + 2) < col_max) { - int i; - - fn_ptr->sdx3f(what, what_stride, check_here, in_what_stride, sad_array); - - for (i = 0; i < 3; ++i) { - thissad = sad_array[i]; - - if (thissad < bestsad) { - this_mv.as_mv.col = c; - thissad += - mvsad_err_cost(&this_mv, &fcenter_mv, mvsadcost, sad_per_bit); - - if (thissad < bestsad) { - bestsad = thissad; - best_mv->as_mv.row = r; - best_mv->as_mv.col = c; - bestaddress = check_here; - } - } - - check_here++; - c++; - } - } - - while (c < col_max) { - thissad = fn_ptr->sdf(what, what_stride, check_here, in_what_stride); - if (thissad < bestsad) { this_mv.as_mv.col = c; thissad += mvsad_err_cost(&this_mv, &fcenter_mv, mvsadcost, sad_per_bit); @@ -1471,169 +1357,19 @@ bestaddress = check_here; } } check_here++; - c++; - } - } - - this_mv.as_mv.row = best_mv->as_mv.row << 3; - this_mv.as_mv.col = best_mv->as_mv.col << 3; - - return fn_ptr->vf(what, what_stride, bestaddress, in_what_stride, &thissad) + - mv_err_cost(&this_mv, center_mv, mvcost, x->errorperbit); -} -#endif // HAVE_SSSE3 - -#if HAVE_SSE4_1 -int vp8_full_search_sadx8(MACROBLOCK *x, BLOCK *b, BLOCKD *d, int_mv *ref_mv, - int sad_per_bit, int distance, - vp8_variance_fn_ptr_t *fn_ptr, int *mvcost[2], - int_mv *center_mv) { - unsigned char *what = (*(b->base_src) + b->src); - int what_stride = b->src_stride; - int pre_stride = x->e_mbd.pre.y_stride; - unsigned char *base_pre = x->e_mbd.pre.y_buffer; - unsigned char *in_what; - int in_what_stride = pre_stride; - int mv_stride = pre_stride; - unsigned char *bestaddress; - int_mv *best_mv = &d->bmi.mv; - int_mv this_mv; - unsigned int bestsad; - unsigned int thissad; - int r, c; - - unsigned char *check_here; - - int ref_row = ref_mv->as_mv.row; - int ref_col = ref_mv->as_mv.col; - - int row_min = ref_row - distance; - int row_max = ref_row + distance; - int col_min = ref_col - distance; - int col_max = ref_col + distance; - - DECLARE_ALIGNED(16, unsigned int, sad_array8[8]); - unsigned int sad_array[3]; - - int *mvsadcost[2]; - int_mv fcenter_mv; - - mvsadcost[0] = x->mvsadcost[0]; - mvsadcost[1] = x->mvsadcost[1]; - fcenter_mv.as_mv.row = center_mv->as_mv.row >> 3; - fcenter_mv.as_mv.col = center_mv->as_mv.col >> 3; - - /* Work out the mid point for the search */ - in_what = base_pre + d->offset; - bestaddress = in_what + (ref_row * pre_stride) + ref_col; - - best_mv->as_mv.row = ref_row; - best_mv->as_mv.col = ref_col; - - /* Baseline value at the centre */ - bestsad = fn_ptr->sdf(what, what_stride, bestaddress, in_what_stride) + - mvsad_err_cost(best_mv, &fcenter_mv, mvsadcost, sad_per_bit); - - /* Apply further limits to prevent us looking using vectors that stretch - * beyond the UMV border - */ - if (col_min < x->mv_col_min) col_min = x->mv_col_min; - - if (col_max > x->mv_col_max) col_max = x->mv_col_max; - - if (row_min < x->mv_row_min) row_min = x->mv_row_min; - - if (row_max > x->mv_row_max) row_max = x->mv_row_max; - - for (r = row_min; r < row_max; ++r) { - this_mv.as_mv.row = r; - check_here = r * mv_stride + in_what + col_min; - c = col_min; - - while ((c + 7) < col_max) { - int i; - - fn_ptr->sdx8f(what, what_stride, check_here, in_what_stride, sad_array8); - - for (i = 0; i < 8; ++i) { - thissad = sad_array8[i]; - - if (thissad < bestsad) { - this_mv.as_mv.col = c; - thissad += - mvsad_err_cost(&this_mv, &fcenter_mv, mvsadcost, sad_per_bit); - - if (thissad < bestsad) { - bestsad = thissad; - best_mv->as_mv.row = r; - best_mv->as_mv.col = c; - bestaddress = check_here; - } - } - - check_here++; - c++; - } - } - - while ((c + 2) < col_max) { - int i; - - fn_ptr->sdx3f(what, what_stride, check_here, in_what_stride, sad_array); - - for (i = 0; i < 3; ++i) { - thissad = sad_array[i]; - - if (thissad < bestsad) { - this_mv.as_mv.col = c; - thissad += - mvsad_err_cost(&this_mv, &fcenter_mv, mvsadcost, sad_per_bit); - - if (thissad < bestsad) { - bestsad = thissad; - best_mv->as_mv.row = r; - best_mv->as_mv.col = c; - bestaddress = check_here; - } - } - - check_here++; - c++; - } - } - - while (c < col_max) { - thissad = fn_ptr->sdf(what, what_stride, check_here, in_what_stride); - - if (thissad < bestsad) { - this_mv.as_mv.col = c; - thissad += - mvsad_err_cost(&this_mv, &fcenter_mv, mvsadcost, sad_per_bit); - - if (thissad < bestsad) { - bestsad = thissad; - best_mv->as_mv.row = r; - best_mv->as_mv.col = c; - bestaddress = check_here; - } - } - - check_here++; - c++; } } this_mv.as_mv.row = best_mv->as_mv.row * 8; this_mv.as_mv.col = best_mv->as_mv.col * 8; return fn_ptr->vf(what, what_stride, bestaddress, in_what_stride, &thissad) + mv_err_cost(&this_mv, center_mv, mvcost, x->errorperbit); } -#endif // HAVE_SSE4_1 int vp8_refining_search_sad_c(MACROBLOCK *x, BLOCK *b, BLOCKD *d, int_mv *ref_mv, int error_per_bit, int search_range, vp8_variance_fn_ptr_t *fn_ptr, int *mvcost[2], int_mv *center_mv) { diff --git a/media/libvpx/libvpx/vp8/encoder/onyx_if.c b/media/libvpx/libvpx/vp8/encoder/onyx_if.c --- a/media/libvpx/libvpx/vp8/encoder/onyx_if.c +++ b/media/libvpx/libvpx/vp8/encoder/onyx_if.c @@ -34,10 +34,11 @@ #include "vpx_mem/vpx_mem.h" #include "vp8/common/reconintra.h" #include "vp8/common/swapyv12buffer.h" #include "vp8/common/threading.h" #include "vpx_ports/system_state.h" +#include "vpx_ports/vpx_once.h" #include "vpx_ports/vpx_timer.h" #include "vpx_util/vpx_write_yuv_frame.h" #if VPX_ARCH_ARM #include "vpx_ports/arm.h" #endif @@ -181,11 +182,11 @@ #ifdef PACKET_TESTING extern FILE *vpxlogc; #endif -static void save_layer_context(VP8_COMP *cpi) { +void vp8_save_layer_context(VP8_COMP *cpi) { LAYER_CONTEXT *lc = &cpi->layer_context[cpi->current_layer]; /* Save layer dependent coding state */ lc->target_bandwidth = cpi->target_bandwidth; lc->starting_buffer_level = cpi->oxcf.starting_buffer_level; @@ -220,11 +221,11 @@ memcpy(lc->count_mb_ref_frame_usage, cpi->mb.count_mb_ref_frame_usage, sizeof(cpi->mb.count_mb_ref_frame_usage)); } -static void restore_layer_context(VP8_COMP *cpi, const int layer) { +void vp8_restore_layer_context(VP8_COMP *cpi, const int layer) { LAYER_CONTEXT *lc = &cpi->layer_context[layer]; /* Restore layer dependent coding state */ cpi->current_layer = layer; cpi->target_bandwidth = lc->target_bandwidth; @@ -267,13 +268,13 @@ int64_t llval = val; return (int)(llval * llnum / llden); } -static void init_temporal_layer_context(VP8_COMP *cpi, VP8_CONFIG *oxcf, - const int layer, - double prev_layer_framerate) { +void vp8_init_temporal_layer_context(VP8_COMP *cpi, VP8_CONFIG *oxcf, + const int layer, + double prev_layer_framerate) { LAYER_CONTEXT *lc = &cpi->layer_context[layer]; lc->framerate = cpi->output_framerate / cpi->oxcf.rate_decimator[layer]; lc->target_bandwidth = cpi->oxcf.target_bitrate[layer] * 1000; @@ -334,16 +335,16 @@ const int curr_num_layers = cpi->oxcf.number_of_layers; // If the previous state was 1 layer, get current layer context from cpi. // We need this to set the layer context for the new layers below. if (prev_num_layers == 1) { cpi->current_layer = 0; - save_layer_context(cpi); + vp8_save_layer_context(cpi); } for (i = 0; i < curr_num_layers; ++i) { LAYER_CONTEXT *lc = &cpi->layer_context[i]; if (i >= prev_num_layers) { - init_temporal_layer_context(cpi, oxcf, i, prev_layer_framerate); + vp8_init_temporal_layer_context(cpi, oxcf, i, prev_layer_framerate); } // The initial buffer levels are set based on their starting levels. // We could set the buffer levels based on the previous state (normalized // properly by the layer bandwidths) but we would need to keep track of // the previous set of layer bandwidths (i.e., target_bitrate[i]) @@ -354,19 +355,19 @@ // TDOD(marpan): Should we set the rate_correction_factor and // active_worst/best_quality to values derived from the previous layer // state (to smooth-out quality dips/rate fluctuation at transition)? // We need to treat the 1 layer case separately: oxcf.target_bitrate[i] - // is not set for 1 layer, and the restore_layer_context/save_context() + // is not set for 1 layer, and the vp8_restore_layer_context/save_context() // are not called in the encoding loop, so we need to call it here to // pass the layer context state to |cpi|. if (curr_num_layers == 1) { lc->target_bandwidth = cpi->oxcf.target_bandwidth; lc->buffer_level = cpi->oxcf.starting_buffer_level_in_ms * lc->target_bandwidth / 1000; lc->bits_off_target = lc->buffer_level; - restore_layer_context(cpi, 0); + vp8_restore_layer_context(cpi, 0); } prev_layer_framerate = cpi->output_framerate / cpi->oxcf.rate_decimator[i]; } } @@ -392,20 +393,17 @@ set_default_lf_deltas(cpi); } static void dealloc_raw_frame_buffers(VP8_COMP *cpi); -void vp8_initialize_enc(void) { - static volatile int init_done = 0; - - if (!init_done) { - vpx_dsp_rtcd(); - vp8_init_intra_predictors(); - init_done = 1; - } +static void initialize_enc(void) { + vpx_dsp_rtcd(); + vp8_init_intra_predictors(); } +void vp8_initialize_enc(void) { once(initialize_enc); } + static void dealloc_compressor_data(VP8_COMP *cpi) { vpx_free(cpi->tplist); cpi->tplist = NULL; /* Delete last frame MV storage buffers */ @@ -1021,11 +1019,11 @@ /* This has a big hit on quality. Last resort */ if (Speed >= 15) sf->half_pixel_search = 0; memset(cpi->mb.error_bins, 0, sizeof(cpi->mb.error_bins)); - }; /* switch */ + } /* switch */ /* Slow quant, dct and trellis not worthwhile for first pass * so make sure they are always turned off. */ if (cpi->pass == 1) { @@ -1272,11 +1270,11 @@ if (framerate < .1) framerate = 30; cpi->framerate = framerate; cpi->output_framerate = framerate; cpi->per_frame_bandwidth = - (int)(cpi->oxcf.target_bandwidth / cpi->output_framerate); + (int)round(cpi->oxcf.target_bandwidth / cpi->output_framerate); cpi->av_per_frame_bandwidth = cpi->per_frame_bandwidth; cpi->min_frame_bandwidth = (int)(cpi->av_per_frame_bandwidth * cpi->oxcf.two_pass_vbrmin_section / 100); /* Set Maximum gf/arf interval */ @@ -1363,11 +1361,11 @@ if (cpi->oxcf.number_of_layers > 1) { unsigned int i; double prev_layer_framerate = 0; for (i = 0; i < cpi->oxcf.number_of_layers; ++i) { - init_temporal_layer_context(cpi, oxcf, i, prev_layer_framerate); + vp8_init_temporal_layer_context(cpi, oxcf, i, prev_layer_framerate); prev_layer_framerate = cpi->output_framerate / cpi->oxcf.rate_decimator[i]; } } @@ -1380,11 +1378,11 @@ for (i = 1; i < 512; ++i) cpi->fixed_divide[i] = 0x80000 / i; } #endif } -static void update_layer_contexts(VP8_COMP *cpi) { +void vp8_update_layer_contexts(VP8_COMP *cpi) { VP8_CONFIG *oxcf = &cpi->oxcf; /* Update snapshots of the layer contexts to reflect new parameters */ if (oxcf->number_of_layers > 1) { unsigned int i; @@ -1415,12 +1413,12 @@ } /* Work out the average size of a frame within this layer */ if (i > 0) { lc->avg_frame_size_for_layer = - (int)((oxcf->target_bitrate[i] - oxcf->target_bitrate[i - 1]) * - 1000 / (lc->framerate - prev_layer_framerate)); + (int)round((oxcf->target_bitrate[i] - oxcf->target_bitrate[i - 1]) * + 1000 / (lc->framerate - prev_layer_framerate)); } prev_layer_framerate = lc->framerate; } } @@ -1908,10 +1906,11 @@ cpi->source_alt_ref_active = 0; cpi->common.refresh_alt_ref_frame = 0; cpi->force_maxqp = 0; cpi->frames_since_last_drop_overshoot = 0; + cpi->rt_always_update_correction_factor = 0; cpi->b_calculate_psnr = CONFIG_INTERNAL_STATS; #if CONFIG_INTERNAL_STATS cpi->b_calculate_ssimg = 0; @@ -2011,51 +2010,40 @@ #endif cpi->fn_ptr[BLOCK_16X16].sdf = vpx_sad16x16; cpi->fn_ptr[BLOCK_16X16].vf = vpx_variance16x16; cpi->fn_ptr[BLOCK_16X16].svf = vpx_sub_pixel_variance16x16; - cpi->fn_ptr[BLOCK_16X16].sdx3f = vpx_sad16x16x3; - cpi->fn_ptr[BLOCK_16X16].sdx8f = vpx_sad16x16x8; cpi->fn_ptr[BLOCK_16X16].sdx4df = vpx_sad16x16x4d; cpi->fn_ptr[BLOCK_16X8].sdf = vpx_sad16x8; cpi->fn_ptr[BLOCK_16X8].vf = vpx_variance16x8; cpi->fn_ptr[BLOCK_16X8].svf = vpx_sub_pixel_variance16x8; - cpi->fn_ptr[BLOCK_16X8].sdx3f = vpx_sad16x8x3; - cpi->fn_ptr[BLOCK_16X8].sdx8f = vpx_sad16x8x8; cpi->fn_ptr[BLOCK_16X8].sdx4df = vpx_sad16x8x4d; cpi->fn_ptr[BLOCK_8X16].sdf = vpx_sad8x16; cpi->fn_ptr[BLOCK_8X16].vf = vpx_variance8x16; cpi->fn_ptr[BLOCK_8X16].svf = vpx_sub_pixel_variance8x16; - cpi->fn_ptr[BLOCK_8X16].sdx3f = vpx_sad8x16x3; - cpi->fn_ptr[BLOCK_8X16].sdx8f = vpx_sad8x16x8; cpi->fn_ptr[BLOCK_8X16].sdx4df = vpx_sad8x16x4d; cpi->fn_ptr[BLOCK_8X8].sdf = vpx_sad8x8; cpi->fn_ptr[BLOCK_8X8].vf = vpx_variance8x8; cpi->fn_ptr[BLOCK_8X8].svf = vpx_sub_pixel_variance8x8; - cpi->fn_ptr[BLOCK_8X8].sdx3f = vpx_sad8x8x3; - cpi->fn_ptr[BLOCK_8X8].sdx8f = vpx_sad8x8x8; cpi->fn_ptr[BLOCK_8X8].sdx4df = vpx_sad8x8x4d; cpi->fn_ptr[BLOCK_4X4].sdf = vpx_sad4x4; cpi->fn_ptr[BLOCK_4X4].vf = vpx_variance4x4; cpi->fn_ptr[BLOCK_4X4].svf = vpx_sub_pixel_variance4x4; - cpi->fn_ptr[BLOCK_4X4].sdx3f = vpx_sad4x4x3; - cpi->fn_ptr[BLOCK_4X4].sdx8f = vpx_sad4x4x8; cpi->fn_ptr[BLOCK_4X4].sdx4df = vpx_sad4x4x4d; #if VPX_ARCH_X86 || VPX_ARCH_X86_64 cpi->fn_ptr[BLOCK_16X16].copymem = vp8_copy32xn; cpi->fn_ptr[BLOCK_16X8].copymem = vp8_copy32xn; cpi->fn_ptr[BLOCK_8X16].copymem = vp8_copy32xn; cpi->fn_ptr[BLOCK_8X8].copymem = vp8_copy32xn; cpi->fn_ptr[BLOCK_4X4].copymem = vp8_copy32xn; #endif - cpi->full_search_sad = vp8_full_search_sad; cpi->diamond_search_sad = vp8_diamond_search_sad; cpi->refining_search_sad = vp8_refining_search_sad; /* make sure frame 1 is okay */ cpi->mb.error_bins[0] = cpi->common.MBs; @@ -3258,11 +3246,11 @@ } break; #endif // !CONFIG_REALTIME_ONLY default: cpi->per_frame_bandwidth = - (int)(cpi->target_bandwidth / cpi->output_framerate); + (int)round(cpi->target_bandwidth / cpi->output_framerate); break; } /* Default turn off buffer to buffer copying */ cm->copy_buffer_to_gf = 0; @@ -3478,11 +3466,11 @@ * at each frame. * * Note that dropping a key frame can be problematic if spatial * resampling is also active */ - if (cpi->decimation_factor > 0) { + if (cpi->decimation_factor > 0 && cpi->drop_frames_allowed) { switch (cpi->decimation_factor) { case 1: cpi->per_frame_bandwidth = cpi->per_frame_bandwidth * 3 / 2; break; case 2: @@ -4014,11 +4002,12 @@ vpx_clear_system_state(); if (frame_over_shoot_limit == 0) frame_over_shoot_limit = 1; /* Are we are overshooting and up against the limit of active max Q. */ - if (((cpi->pass != 2) || + if (!cpi->rt_always_update_correction_factor && + ((cpi->pass != 2) || (cpi->oxcf.end_usage == USAGE_STREAM_FROM_SERVER)) && (Q == cpi->active_worst_quality) && (cpi->active_worst_quality < cpi->worst_quality) && (cpi->projected_frame_size > frame_over_shoot_limit)) { int over_size_percent = @@ -4512,14 +4501,14 @@ /* Clip the buffer level to the maximum specified buffer size */ if (cpi->bits_off_target > cpi->oxcf.maximum_buffer_size) { cpi->bits_off_target = cpi->oxcf.maximum_buffer_size; } - // If the frame dropper is not enabled, don't let the buffer level go below - // some threshold, given here by -|maximum_buffer_size|. For now we only do - // this for screen content input. - if (cpi->drop_frames_allowed == 0 && cpi->oxcf.screen_content_mode && + // Don't let the buffer level go below some threshold, given here + // by -|maximum_buffer_size|. For now we only do this for + // screen content input. + if (cpi->oxcf.screen_content_mode && cpi->bits_off_target < -cpi->oxcf.maximum_buffer_size) { cpi->bits_off_target = -cpi->oxcf.maximum_buffer_size; } /* Rolling monitors of whether we are over or underspending used to @@ -4550,12 +4539,12 @@ if (cpi->oxcf.number_of_layers > 1) { unsigned int i; for (i = cpi->current_layer + 1; i < cpi->oxcf.number_of_layers; ++i) { LAYER_CONTEXT *lc = &cpi->layer_context[i]; - int bits_off_for_this_layer = (int)(lc->target_bandwidth / lc->framerate - - cpi->projected_frame_size); + int bits_off_for_this_layer = (int)round( + lc->target_bandwidth / lc->framerate - cpi->projected_frame_size); lc->bits_off_target += bits_off_for_this_layer; /* Clip buffer level to maximum buffer size for the layer */ if (lc->bits_off_target > lc->maximum_buffer_size) { @@ -4988,21 +4977,21 @@ } if (cpi->oxcf.number_of_layers > 1) { int layer; - update_layer_contexts(cpi); + vp8_update_layer_contexts(cpi); /* Restore layer specific context & set frame rate */ if (cpi->temporal_layer_id >= 0) { layer = cpi->temporal_layer_id; } else { layer = cpi->oxcf .layer_id[cpi->temporal_pattern_counter % cpi->oxcf.periodicity]; } - restore_layer_context(cpi, layer); + vp8_restore_layer_context(cpi, layer); vp8_new_framerate(cpi, cpi->layer_context[layer].framerate); } if (cpi->compressor_speed == 2) { vpx_usec_timer_start(&tsctimer); @@ -5129,11 +5118,11 @@ cm->refresh_last_frame = 1; cm->frame_type = INTER_FRAME; } /* Save layer specific state */ - if (cpi->oxcf.number_of_layers > 1) save_layer_context(cpi); + if (cpi->oxcf.number_of_layers > 1) vp8_save_layer_context(cpi); vpx_usec_timer_mark(&cmptimer); cpi->time_compress_data += vpx_usec_timer_elapsed(&cmptimer); if (cpi->b_calculate_psnr && cpi->pass != 1 && cm->show_frame) { diff --git a/media/libvpx/libvpx/vp8/encoder/onyx_int.h b/media/libvpx/libvpx/vp8/encoder/onyx_int.h --- a/media/libvpx/libvpx/vp8/encoder/onyx_int.h +++ b/media/libvpx/libvpx/vp8/encoder/onyx_int.h @@ -547,11 +547,10 @@ unsigned int partition_sz[MAX_PARTITIONS]; unsigned char *partition_d[MAX_PARTITIONS]; unsigned char *partition_d_end[MAX_PARTITIONS]; fractional_mv_step_fp *find_fractional_mv_step; - vp8_full_search_fn_t full_search_sad; vp8_refining_search_fn_t refining_search_sad; vp8_diamond_search_fn_t diamond_search_sad; vp8_variance_fn_ptr_t fn_ptr[BLOCK_MAX_SEGMENTS]; uint64_t time_receive_data; uint64_t time_compress_data; @@ -700,16 +699,26 @@ // Use the static threshold from ROI settings. int use_roi_static_threshold; int ext_refresh_frame_flags_pending; + + // Always update correction factor used for rate control after each frame for + // realtime encoding. + int rt_always_update_correction_factor; } VP8_COMP; void vp8_initialize_enc(void); void vp8_alloc_compressor_data(VP8_COMP *cpi); int vp8_reverse_trans(int x); +void vp8_init_temporal_layer_context(VP8_COMP *cpi, VP8_CONFIG *oxcf, + const int layer, + double prev_layer_framerate); +void vp8_update_layer_contexts(VP8_COMP *cpi); +void vp8_save_layer_context(VP8_COMP *cpi); +void vp8_restore_layer_context(VP8_COMP *cpi, const int layer); void vp8_new_framerate(VP8_COMP *cpi, double framerate); void vp8_loopfilter_frame(VP8_COMP *cpi, VP8_COMMON *cm); void vp8_pack_bitstream(VP8_COMP *cpi, unsigned char *dest, unsigned char *dest_end, size_t *size); diff --git a/media/libvpx/libvpx/vp8/encoder/ratectrl.c b/media/libvpx/libvpx/vp8/encoder/ratectrl.c --- a/media/libvpx/libvpx/vp8/encoder/ratectrl.c +++ b/media/libvpx/libvpx/vp8/encoder/ratectrl.c @@ -312,11 +312,11 @@ else if (cpi->common.current_video_frame == 0) { /* 1 Pass there is no information on which to base size so use * bandwidth per second * fraction of the initial buffer * level */ - target = cpi->oxcf.starting_buffer_level / 2; + target = (uint64_t)cpi->oxcf.starting_buffer_level / 2; if (target > cpi->oxcf.target_bandwidth * 3 / 2) { target = cpi->oxcf.target_bandwidth * 3 / 2; } } else { @@ -325,11 +325,12 @@ : cpi->ni_av_qi; int initial_boost = 32; /* |3.0 * per_frame_bandwidth| */ /* Boost depends somewhat on frame rate: only used for 1 layer case. */ if (cpi->oxcf.number_of_layers == 1) { - kf_boost = VPXMAX(initial_boost, (int)(2 * cpi->output_framerate - 16)); + kf_boost = + VPXMAX(initial_boost, (int)round(2 * cpi->output_framerate - 16)); } else { /* Initial factor: set target size to: |3.0 * per_frame_bandwidth|. */ kf_boost = initial_boost; } @@ -778,10 +779,11 @@ } else { cpi->active_worst_quality = cpi->worst_quality; } } else { int percent_high = 0; + int64_t target = cpi->this_frame_target; if ((cpi->oxcf.end_usage == USAGE_STREAM_FROM_SERVER) && (cpi->buffer_level > cpi->oxcf.optimal_buffer_level)) { percent_high = (int)((cpi->buffer_level - cpi->oxcf.optimal_buffer_level) / @@ -795,11 +797,13 @@ percent_high = cpi->oxcf.over_shoot_pct; } else if (percent_high < 0) { percent_high = 0; } - cpi->this_frame_target += (cpi->this_frame_target * percent_high) / 200; + target += (target * percent_high) / 200; + target = VPXMIN(target, INT_MAX); + cpi->this_frame_target = (int)target; /* Are we allowing control of active_worst_allowed_q according * to buffer level. */ if (cpi->auto_worst_q && cpi->ni_frames > 150) { @@ -1076,12 +1080,12 @@ } } /* Work out a size correction factor. */ if (projected_size_based_on_q > 0) { - correction_factor = - (100 * cpi->projected_frame_size) / projected_size_based_on_q; + correction_factor = (int)((100 * (int64_t)cpi->projected_frame_size) / + projected_size_based_on_q); } /* More heavily damped adjustment used if we have been oscillating * either side of target */ diff --git a/media/libvpx/libvpx/vp8/encoder/rdopt.c b/media/libvpx/libvpx/vp8/encoder/rdopt.c --- a/media/libvpx/libvpx/vp8/encoder/rdopt.c +++ b/media/libvpx/libvpx/vp8/encoder/rdopt.c @@ -1095,12 +1095,12 @@ if ((cpi->compressor_speed == 0) && (bestsme >> sseshift) > 4000) { /* Check if mvp_full is within the range. */ vp8_clamp_mv(&mvp_full, x->mv_col_min, x->mv_col_max, x->mv_row_min, x->mv_row_max); - thissme = cpi->full_search_sad(x, c, e, &mvp_full, sadpb, 16, - v_fn_ptr, x->mvcost, bsi->ref_mv); + thissme = vp8_full_search_sad(x, c, e, &mvp_full, sadpb, 16, + v_fn_ptr, x->mvcost, bsi->ref_mv); if (thissme < bestsme) { bestsme = thissme; mode_mv[NEW4X4].as_int = e->bmi.mv.as_int; } else { diff --git a/media/libvpx/libvpx/vp8/encoder/x86/quantize_sse4.c b/media/libvpx/libvpx/vp8/encoder/x86/quantize_sse4.c --- a/media/libvpx/libvpx/vp8/encoder/x86/quantize_sse4.c +++ b/media/libvpx/libvpx/vp8/encoder/x86/quantize_sse4.c @@ -9,32 +9,18 @@ */ #include /* SSE4.1 */ #include "./vp8_rtcd.h" -#include "vp8/common/entropy.h" /* vp8_default_inv_zig_zag */ #include "vp8/encoder/block.h" - -#define SELECT_EOB(i, z, x, y, q) \ - do { \ - short boost = *zbin_boost_ptr; \ - /* Technically _mm_extract_epi16() returns an int: */ \ - /* https://bugs.llvm.org/show_bug.cgi?id=41657 */ \ - short x_z = (short)_mm_extract_epi16(x, z); \ - short y_z = (short)_mm_extract_epi16(y, z); \ - int cmp = (x_z < boost) | (y_z == 0); \ - zbin_boost_ptr++; \ - if (cmp) break; \ - q = _mm_insert_epi16(q, y_z, z); \ - eob = i; \ - zbin_boost_ptr = b->zrun_zbin_boost; \ - } while (0) +#include "vpx_ports/bitops.h" /* get_lsb */ void vp8_regular_quantize_b_sse4_1(BLOCK *b, BLOCKD *d) { - char eob = 0; + int eob = -1; short *zbin_boost_ptr = b->zrun_zbin_boost; - + __m128i zbin_boost0 = _mm_load_si128((__m128i *)(zbin_boost_ptr)); + __m128i zbin_boost1 = _mm_load_si128((__m128i *)(zbin_boost_ptr + 8)); __m128i x0, x1, y0, y1, x_minus_zbin0, x_minus_zbin1, dqcoeff0, dqcoeff1; __m128i quant_shift0 = _mm_load_si128((__m128i *)(b->quant_shift)); __m128i quant_shift1 = _mm_load_si128((__m128i *)(b->quant_shift + 8)); __m128i z0 = _mm_load_si128((__m128i *)(b->coeff)); __m128i z1 = _mm_load_si128((__m128i *)(b->coeff + 8)); @@ -45,12 +31,16 @@ __m128i round1 = _mm_load_si128((__m128i *)(b->round + 8)); __m128i quant0 = _mm_load_si128((__m128i *)(b->quant)); __m128i quant1 = _mm_load_si128((__m128i *)(b->quant + 8)); __m128i dequant0 = _mm_load_si128((__m128i *)(d->dequant)); __m128i dequant1 = _mm_load_si128((__m128i *)(d->dequant + 8)); - __m128i qcoeff0 = _mm_setzero_si128(); - __m128i qcoeff1 = _mm_setzero_si128(); + __m128i qcoeff0, qcoeff1, t0, t1, x_shuf0, x_shuf1; + uint32_t mask, ymask; + DECLARE_ALIGNED(16, static const uint8_t, + zig_zag_mask[16]) = { 0, 1, 4, 8, 5, 2, 3, 6, + 9, 12, 13, 10, 7, 11, 14, 15 }; + DECLARE_ALIGNED(16, uint16_t, qcoeff[16]) = { 0 }; /* Duplicate to all lanes. */ zbin_extra = _mm_shufflelo_epi16(zbin_extra, 0); zbin_extra = _mm_unpacklo_epi16(zbin_extra, zbin_extra); @@ -86,34 +76,63 @@ /* Restore the sign. */ y0 = _mm_sign_epi16(y0, z0); y1 = _mm_sign_epi16(y1, z1); - /* The loop gets unrolled anyway. Avoid the vp8_default_zig_zag1d lookup. */ - SELECT_EOB(1, 0, x_minus_zbin0, y0, qcoeff0); - SELECT_EOB(2, 1, x_minus_zbin0, y0, qcoeff0); - SELECT_EOB(3, 4, x_minus_zbin0, y0, qcoeff0); - SELECT_EOB(4, 0, x_minus_zbin1, y1, qcoeff1); - SELECT_EOB(5, 5, x_minus_zbin0, y0, qcoeff0); - SELECT_EOB(6, 2, x_minus_zbin0, y0, qcoeff0); - SELECT_EOB(7, 3, x_minus_zbin0, y0, qcoeff0); - SELECT_EOB(8, 6, x_minus_zbin0, y0, qcoeff0); - SELECT_EOB(9, 1, x_minus_zbin1, y1, qcoeff1); - SELECT_EOB(10, 4, x_minus_zbin1, y1, qcoeff1); - SELECT_EOB(11, 5, x_minus_zbin1, y1, qcoeff1); - SELECT_EOB(12, 2, x_minus_zbin1, y1, qcoeff1); - SELECT_EOB(13, 7, x_minus_zbin0, y0, qcoeff0); - SELECT_EOB(14, 3, x_minus_zbin1, y1, qcoeff1); - SELECT_EOB(15, 6, x_minus_zbin1, y1, qcoeff1); - SELECT_EOB(16, 7, x_minus_zbin1, y1, qcoeff1); + { + const __m128i zig_zag_i16_0 = + _mm_setr_epi8(0, 1, 2, 3, 8, 9, 14, 15, 10, 11, 4, 5, 6, 7, 12, 13); + const __m128i zig_zag_i16_1 = + _mm_setr_epi8(0, 1, 6, 7, 8, 9, 2, 3, 14, 15, 4, 5, 10, 11, 12, 13); + + /* The first part of the zig zag needs a value + * from x_minus_zbin1 and vice versa. */ + t1 = _mm_alignr_epi8(x_minus_zbin1, x_minus_zbin1, 2); + t0 = _mm_blend_epi16(x_minus_zbin0, t1, 0x80); + t1 = _mm_blend_epi16(t1, x_minus_zbin0, 0x80); + x_shuf0 = _mm_shuffle_epi8(t0, zig_zag_i16_0); + x_shuf1 = _mm_shuffle_epi8(t1, zig_zag_i16_1); + } + + /* Check if y is nonzero and put it in zig zag order. */ + t0 = _mm_packs_epi16(y0, y1); + t0 = _mm_cmpeq_epi8(t0, _mm_setzero_si128()); + t0 = _mm_shuffle_epi8(t0, _mm_load_si128((const __m128i *)zig_zag_mask)); + ymask = _mm_movemask_epi8(t0) ^ 0xffff; + + for (;;) { + t0 = _mm_cmpgt_epi16(zbin_boost0, x_shuf0); + t1 = _mm_cmpgt_epi16(zbin_boost1, x_shuf1); + t0 = _mm_packs_epi16(t0, t1); + mask = _mm_movemask_epi8(t0); + mask = ~mask & ymask; + if (!mask) break; + /* |eob| will contain the index of the next found element where: + * boost[i - old_eob - 1] <= x[zigzag[i]] && y[zigzag[i]] != 0 */ + eob = get_lsb(mask); + /* Need to clear the mask from processed elements so that + * they are no longer counted in the next iteration. */ + ymask &= ~1U << eob; + /* It's safe to read ahead of this buffer if struct VP8_COMP has at + * least 32 bytes before the zrun_zbin_boost_* fields (it has 384). + * Any data read outside of the buffer is masked by the updated |ymask|. */ + zbin_boost0 = _mm_loadu_si128((__m128i *)(zbin_boost_ptr - eob - 1)); + zbin_boost1 = _mm_loadu_si128((__m128i *)(zbin_boost_ptr - eob + 7)); + qcoeff[zig_zag_mask[eob]] = 0xffff; + } + + qcoeff0 = _mm_load_si128((__m128i *)(qcoeff)); + qcoeff1 = _mm_load_si128((__m128i *)(qcoeff + 8)); + qcoeff0 = _mm_and_si128(qcoeff0, y0); + qcoeff1 = _mm_and_si128(qcoeff1, y1); _mm_store_si128((__m128i *)(d->qcoeff), qcoeff0); _mm_store_si128((__m128i *)(d->qcoeff + 8), qcoeff1); dqcoeff0 = _mm_mullo_epi16(qcoeff0, dequant0); dqcoeff1 = _mm_mullo_epi16(qcoeff1, dequant1); _mm_store_si128((__m128i *)(d->dqcoeff), dqcoeff0); _mm_store_si128((__m128i *)(d->dqcoeff + 8), dqcoeff1); - *d->eob = eob; + *d->eob = eob + 1; } diff --git a/media/libvpx/libvpx/vp8/encoder/x86/vp8_quantize_ssse3.c b/media/libvpx/libvpx/vp8/encoder/x86/vp8_quantize_ssse3.c --- a/media/libvpx/libvpx/vp8/encoder/x86/vp8_quantize_ssse3.c +++ b/media/libvpx/libvpx/vp8/encoder/x86/vp8_quantize_ssse3.c @@ -10,35 +10,11 @@ #include /* SSSE3 */ #include "./vp8_rtcd.h" #include "vp8/encoder/block.h" - -/* bitscan reverse (bsr) */ -#if defined(_MSC_VER) -#include -#pragma intrinsic(_BitScanReverse) -static int bsr(int mask) { - unsigned long eob; - _BitScanReverse(&eob, mask); - eob++; - if (mask == 0) eob = 0; - return eob; -} -#else -static int bsr(int mask) { - int eob; -#if defined(__GNUC__) && __GNUC__ - __asm__ __volatile__("bsr %1, %0" : "=r"(eob) : "r"(mask) : "flags"); -#elif defined(__SUNPRO_C) || defined(__SUNPRO_CC) - asm volatile("bsr %1, %0" : "=r"(eob) : "r"(mask) : "flags"); -#endif - eob++; - if (mask == 0) eob = 0; - return eob; -} -#endif +#include "vpx_ports/bitops.h" /* get_msb */ void vp8_fast_quantize_b_ssse3(BLOCK *b, BLOCKD *d) { int eob, mask; __m128i z0 = _mm_load_si128((__m128i *)(b->coeff)); @@ -106,9 +82,12 @@ x = _mm_shuffle_epi8(x, zig_zag); mask = _mm_movemask_epi8(x); - eob = bsr(mask); + /* x2 is needed to increase the result from non-zero masks by 1, + * +1 is needed to mask undefined behavior for a null argument, + * the result of get_msb(1) is 0 */ + eob = get_msb(mask * 2 + 1); - *d->eob = 0xFF & eob; + *d->eob = eob; } diff --git a/media/libvpx/libvpx/vp8/vp8_common.mk b/media/libvpx/libvpx/vp8/vp8_common.mk --- a/media/libvpx/libvpx/vp8/vp8_common.mk +++ b/media/libvpx/libvpx/vp8/vp8_common.mk @@ -122,10 +122,15 @@ ifeq ($(CONFIG_POSTPROC),yes) VP8_COMMON_SRCS-$(HAVE_MSA) += common/mips/msa/mfqe_msa.c endif +# common (loongarch LSX intrinsics) +VP8_COMMON_SRCS-$(HAVE_LSX) += common/loongarch/loopfilter_filters_lsx.c +VP8_COMMON_SRCS-$(HAVE_LSX) += common/loongarch/sixtap_filter_lsx.c +VP8_COMMON_SRCS-$(HAVE_LSX) += common/loongarch/idct_lsx.c + # common (neon intrinsics) VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/loopfilter_arm.c VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/loopfilter_arm.h VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/bilinearpredict_neon.c VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/copymem_neon.c diff --git a/media/libvpx/libvpx/vp8/vp8_cx_iface.c b/media/libvpx/libvpx/vp8/vp8_cx_iface.c --- a/media/libvpx/libvpx/vp8/vp8_cx_iface.c +++ b/media/libvpx/libvpx/vp8/vp8_cx_iface.c @@ -16,11 +16,10 @@ #include "vpx/internal/vpx_codec_internal.h" #include "vpx_version.h" #include "vpx_mem/vpx_mem.h" #include "vpx_ports/static_assert.h" #include "vpx_ports/system_state.h" -#include "vpx_ports/vpx_once.h" #include "vpx_util/vpx_timestamp.h" #include "vp8/encoder/onyx_int.h" #include "vpx/vp8cx.h" #include "vp8/encoder/firstpass.h" #include "vp8/common/onyx.h" @@ -337,11 +336,13 @@ oxcf->end_usage = USAGE_CONSTRAINED_QUALITY; } else if (cfg.rc_end_usage == VPX_Q) { oxcf->end_usage = USAGE_CONSTANT_QUALITY; } - oxcf->target_bandwidth = cfg.rc_target_bitrate; + // Cap the target rate to 1000 Mbps to avoid some integer overflows in + // target bandwidth calculations. + oxcf->target_bandwidth = VPXMIN(cfg.rc_target_bitrate, 1000000); oxcf->rc_max_intra_bitrate_pct = vp8_cfg.rc_max_intra_bitrate_pct; oxcf->gf_cbr_boost_pct = vp8_cfg.gf_cbr_boost_pct; oxcf->best_allowed_q = cfg.rc_min_quantizer; oxcf->worst_allowed_q = cfg.rc_max_quantizer; @@ -603,10 +604,21 @@ struct vp8_extracfg extra_cfg = ctx->vp8_cfg; extra_cfg.screen_content_mode = CAST(VP8E_SET_SCREEN_CONTENT_MODE, args); return update_extracfg(ctx, &extra_cfg); } +static vpx_codec_err_t ctrl_set_rtc_external_ratectrl(vpx_codec_alg_priv_t *ctx, + va_list args) { + VP8_COMP *cpi = ctx->cpi; + const unsigned int data = CAST(VP8E_SET_GF_CBR_BOOST_PCT, args); + if (data) { + cpi->cyclic_refresh_mode_enabled = 0; + cpi->rt_always_update_correction_factor = 1; + } + return VPX_CODEC_OK; +} + static vpx_codec_err_t vp8e_mr_alloc_mem(const vpx_codec_enc_cfg_t *cfg, void **mem_loc) { vpx_codec_err_t res = VPX_CODEC_OK; #if CONFIG_MULTI_RES_ENCODING @@ -679,11 +691,11 @@ ctx->priv->enc.total_encoders = mr_cfg->mr_total_resolutions; } else { ctx->priv->enc.total_encoders = 1; } - once(vp8_initialize_enc); + vp8_initialize_enc(); res = validate_config(priv, &priv->cfg, &priv->vp8_cfg, 0); if (!res) { priv->pts_offset_initialized = 0; @@ -924,23 +936,14 @@ ctx->timestamp_ratio.num / ctx->timestamp_ratio.den; if (img != NULL) { res = image2yuvconfig(img, &sd); - if (sd.y_width != ctx->cfg.g_w || sd.y_height != ctx->cfg.g_h) { - /* from vpx_encoder.h for g_w/g_h: - "Note that the frames passed as input to the encoder must have this - resolution" - */ - ctx->base.err_detail = "Invalid input frame resolution"; - res = VPX_CODEC_INVALID_PARAM; - } else { - if (vp8_receive_raw_frame(ctx->cpi, ctx->next_frame_flag | lib_flags, - &sd, dst_time_stamp, dst_end_time_stamp)) { - VP8_COMP *cpi = (VP8_COMP *)ctx->cpi; - res = update_error_state(ctx, &cpi->common.error); - } + if (vp8_receive_raw_frame(ctx->cpi, ctx->next_frame_flag | lib_flags, &sd, + dst_time_stamp, dst_end_time_stamp)) { + VP8_COMP *cpi = (VP8_COMP *)ctx->cpi; + res = update_error_state(ctx, &cpi->common.error); } /* reset for next frame */ ctx->next_frame_flag = 0; } @@ -1250,10 +1253,11 @@ { VP8E_SET_TUNING, set_tuning }, { VP8E_SET_CQ_LEVEL, set_cq_level }, { VP8E_SET_MAX_INTRA_BITRATE_PCT, set_rc_max_intra_bitrate_pct }, { VP8E_SET_SCREEN_CONTENT_MODE, set_screen_content_mode }, { VP8E_SET_GF_CBR_BOOST_PCT, ctrl_set_rc_gf_cbr_boost_pct }, + { VP8E_SET_RTC_EXTERNAL_RATECTRL, ctrl_set_rtc_external_ratectrl }, { -1, NULL }, }; static vpx_codec_enc_cfg_map_t vp8e_usage_cfg_map[] = { { 0, diff --git a/media/libvpx/libvpx/vp8/vp8_dx_iface.c b/media/libvpx/libvpx/vp8/vp8_dx_iface.c --- a/media/libvpx/libvpx/vp8/vp8_dx_iface.c +++ b/media/libvpx/libvpx/vp8/vp8_dx_iface.c @@ -369,12 +369,10 @@ int i; #endif pc->Width = ctx->si.w; pc->Height = ctx->si.h; { - int prev_mb_rows = pc->mb_rows; - if (setjmp(pbi->common.error.jmp)) { pbi->common.error.setjmp = 0; /* on failure clear the cached resolution to ensure a full * reallocation is attempted on resync. */ ctx->si.w = 0; @@ -396,10 +394,16 @@ pc->Height = h; vpx_internal_error(&pc->error, VPX_CODEC_CORRUPT_FRAME, "Invalid frame height"); } +#if CONFIG_MULTITHREAD + if (vpx_atomic_load_acquire(&pbi->b_multithreaded_rd)) { + vp8mt_de_alloc_temp_buffers(pbi, pc->mb_rows); + } +#endif + if (vp8_alloc_frame_buffers(pc, pc->Width, pc->Height)) { vpx_internal_error(&pc->error, VPX_CODEC_MEM_ERROR, "Failed to allocate frame buffers"); } @@ -440,14 +444,12 @@ #endif #if CONFIG_MULTITHREAD if (vpx_atomic_load_acquire(&pbi->b_multithreaded_rd)) { - vp8mt_alloc_temp_buffers(pbi, pc->Width, prev_mb_rows); + vp8mt_alloc_temp_buffers(pbi, pc->Width, 0); } -#else - (void)prev_mb_rows; #endif } pbi->common.error.setjmp = 0; diff --git a/media/libvpx/libvpx/vp8/vp8_ratectrl_rtc.h b/media/libvpx/libvpx/vp8/vp8_ratectrl_rtc.h new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vp8/vp8_ratectrl_rtc.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2021 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VPX_VP8_RATECTRL_RTC_H_ +#define VPX_VP8_RATECTRL_RTC_H_ + +#include +#include + +#include "vp8/encoder/onyx_int.h" +#include "vp8/common/common.h" +#include "vpx/internal/vpx_ratectrl_rtc.h" + +namespace libvpx { +struct VP8RateControlRtcConfig : public VpxRateControlRtcConfig { + public: + VP8RateControlRtcConfig() { + vp8_zero(layer_target_bitrate); + vp8_zero(ts_rate_decimator); + } +}; + +struct VP8FrameParamsQpRTC { + FRAME_TYPE frame_type; + int temporal_layer_id; +}; + +class VP8RateControlRTC { + public: + static std::unique_ptr Create( + const VP8RateControlRtcConfig &cfg); + ~VP8RateControlRTC() { + if (cpi_) { + vpx_free(cpi_->gf_active_flags); + vpx_free(cpi_); + } + } + + void UpdateRateControl(const VP8RateControlRtcConfig &rc_cfg); + // GetQP() needs to be called after ComputeQP() to get the latest QP + int GetQP() const; + // int GetLoopfilterLevel() const; + void ComputeQP(const VP8FrameParamsQpRTC &frame_params); + // Feedback to rate control with the size of current encoded frame + void PostEncodeUpdate(uint64_t encoded_frame_size); + + private: + VP8RateControlRTC() {} + void InitRateControl(const VP8RateControlRtcConfig &cfg); + VP8_COMP *cpi_; + int q_; +}; + +} // namespace libvpx + +#endif // VPX_VP8_RATECTRL_RTC_H_ diff --git a/media/libvpx/libvpx/vp8/vp8_ratectrl_rtc.cc b/media/libvpx/libvpx/vp8/vp8_ratectrl_rtc.cc new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vp8/vp8_ratectrl_rtc.cc @@ -0,0 +1,347 @@ +/* + * Copyright (c) 2021 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#include "vp8/vp8_ratectrl_rtc.h" +#include "vp8/encoder/ratectrl.h" +#include "vpx_ports/system_state.h" + +namespace libvpx { +/* Quant MOD */ +static const int kQTrans[] = { + 0, 1, 2, 3, 4, 5, 7, 8, 9, 10, 12, 13, 15, 17, 18, 19, + 20, 21, 23, 24, 25, 26, 27, 28, 29, 30, 31, 33, 35, 37, 39, 41, + 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 64, 67, 70, 73, 76, 79, + 82, 85, 88, 91, 94, 97, 100, 103, 106, 109, 112, 115, 118, 121, 124, 127, +}; + +static const unsigned char kf_high_motion_minq[QINDEX_RANGE] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, + 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, + 5, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 8, 8, 9, 9, 10, 10, + 10, 10, 11, 11, 11, 11, 12, 12, 13, 13, 13, 13, 14, 14, 15, 15, 15, 15, 16, + 16, 16, 16, 17, 17, 18, 18, 18, 18, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, + 22, 22, 23, 23, 24, 25, 25, 26, 26, 27, 28, 28, 29, 30 +}; + +static const unsigned char inter_minq[QINDEX_RANGE] = { + 0, 0, 1, 1, 2, 3, 3, 4, 4, 5, 6, 6, 7, 8, 8, 9, 9, 10, 11, + 11, 12, 13, 13, 14, 15, 15, 16, 17, 17, 18, 19, 20, 20, 21, 22, 22, 23, 24, + 24, 25, 26, 27, 27, 28, 29, 30, 30, 31, 32, 33, 33, 34, 35, 36, 36, 37, 38, + 39, 39, 40, 41, 42, 42, 43, 44, 45, 46, 46, 47, 48, 49, 50, 50, 51, 52, 53, + 54, 55, 55, 56, 57, 58, 59, 60, 60, 61, 62, 63, 64, 65, 66, 67, 67, 68, 69, + 70, 71, 72, 73, 74, 75, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 86, + 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100 +}; + +static int rescale(int val, int num, int denom) { + int64_t llnum = num; + int64_t llden = denom; + int64_t llval = val; + + return (int)(llval * llnum / llden); +} + +std::unique_ptr VP8RateControlRTC::Create( + const VP8RateControlRtcConfig &cfg) { + std::unique_ptr rc_api(new (std::nothrow) + VP8RateControlRTC()); + if (!rc_api) return nullptr; + rc_api->cpi_ = static_cast(vpx_memalign(32, sizeof(*cpi_))); + if (!rc_api->cpi_) return nullptr; + vp8_zero(*rc_api->cpi_); + + rc_api->InitRateControl(cfg); + + return rc_api; +} + +void VP8RateControlRTC::InitRateControl(const VP8RateControlRtcConfig &rc_cfg) { + VP8_COMMON *cm = &cpi_->common; + VP8_CONFIG *oxcf = &cpi_->oxcf; + oxcf->end_usage = USAGE_STREAM_FROM_SERVER; + cpi_->pass = 0; + cm->show_frame = 1; + oxcf->drop_frames_water_mark = 0; + cm->current_video_frame = 0; + cpi_->auto_gold = 1; + cpi_->key_frame_count = 1; + cpi_->rate_correction_factor = 1.0; + cpi_->key_frame_rate_correction_factor = 1.0; + cpi_->cyclic_refresh_mode_enabled = 0; + cpi_->auto_worst_q = 1; + cpi_->kf_overspend_bits = 0; + cpi_->kf_bitrate_adjustment = 0; + cpi_->gf_overspend_bits = 0; + cpi_->non_gf_bitrate_adjustment = 0; + UpdateRateControl(rc_cfg); + cpi_->buffer_level = oxcf->starting_buffer_level; + cpi_->bits_off_target = oxcf->starting_buffer_level; +} + +void VP8RateControlRTC::UpdateRateControl( + const VP8RateControlRtcConfig &rc_cfg) { + VP8_COMMON *cm = &cpi_->common; + VP8_CONFIG *oxcf = &cpi_->oxcf; + vpx_clear_system_state(); + cm->Width = rc_cfg.width; + cm->Height = rc_cfg.height; + oxcf->Width = rc_cfg.width; + oxcf->Height = rc_cfg.height; + oxcf->worst_allowed_q = kQTrans[rc_cfg.max_quantizer]; + oxcf->best_allowed_q = kQTrans[rc_cfg.min_quantizer]; + cpi_->worst_quality = oxcf->worst_allowed_q; + cpi_->best_quality = oxcf->best_allowed_q; + cpi_->output_framerate = rc_cfg.framerate; + oxcf->target_bandwidth = + static_cast(1000 * rc_cfg.target_bandwidth); + cpi_->ref_framerate = cpi_->output_framerate; + oxcf->fixed_q = -1; + oxcf->error_resilient_mode = 1; + oxcf->starting_buffer_level_in_ms = rc_cfg.buf_initial_sz; + oxcf->optimal_buffer_level_in_ms = rc_cfg.buf_optimal_sz; + oxcf->maximum_buffer_size_in_ms = rc_cfg.buf_sz; + oxcf->starting_buffer_level = rc_cfg.buf_initial_sz; + oxcf->optimal_buffer_level = rc_cfg.buf_optimal_sz; + oxcf->maximum_buffer_size = rc_cfg.buf_sz; + oxcf->number_of_layers = rc_cfg.ts_number_layers; + cpi_->buffered_mode = oxcf->optimal_buffer_level > 0; + oxcf->under_shoot_pct = rc_cfg.undershoot_pct; + oxcf->over_shoot_pct = rc_cfg.overshoot_pct; + cpi_->oxcf.rc_max_intra_bitrate_pct = rc_cfg.max_intra_bitrate_pct; + cpi_->framerate = rc_cfg.framerate; + for (int i = 0; i < KEY_FRAME_CONTEXT; ++i) { + cpi_->prior_key_frame_distance[i] = + static_cast(cpi_->output_framerate); + } + + if (oxcf->number_of_layers > 1) { + memcpy(oxcf->target_bitrate, rc_cfg.layer_target_bitrate, + sizeof(rc_cfg.layer_target_bitrate)); + memcpy(oxcf->rate_decimator, rc_cfg.ts_rate_decimator, + sizeof(rc_cfg.ts_rate_decimator)); + oxcf->periodicity = 2; + + double prev_layer_framerate = 0; + for (unsigned int i = 0; i < oxcf->number_of_layers; ++i) { + vp8_init_temporal_layer_context(cpi_, oxcf, i, prev_layer_framerate); + prev_layer_framerate = cpi_->output_framerate / oxcf->rate_decimator[i]; + } + } + + cpi_->total_actual_bits = 0; + cpi_->total_target_vs_actual = 0; + + cm->mb_rows = cm->Height >> 4; + cm->mb_cols = cm->Width >> 4; + cm->MBs = cm->mb_rows * cm->mb_cols; + cm->mode_info_stride = cm->mb_cols + 1; + + oxcf->starting_buffer_level = + rescale((int)oxcf->starting_buffer_level, oxcf->target_bandwidth, 1000); + /* Set or reset optimal and maximum buffer levels. */ + if (oxcf->optimal_buffer_level == 0) { + oxcf->optimal_buffer_level = oxcf->target_bandwidth / 8; + } else { + oxcf->optimal_buffer_level = + rescale((int)oxcf->optimal_buffer_level, oxcf->target_bandwidth, 1000); + } + if (oxcf->maximum_buffer_size == 0) { + oxcf->maximum_buffer_size = oxcf->target_bandwidth / 8; + } else { + oxcf->maximum_buffer_size = + rescale((int)oxcf->maximum_buffer_size, oxcf->target_bandwidth, 1000); + } + + if (cpi_->bits_off_target > oxcf->maximum_buffer_size) { + cpi_->bits_off_target = oxcf->maximum_buffer_size; + cpi_->buffer_level = cpi_->bits_off_target; + } + + vp8_new_framerate(cpi_, cpi_->framerate); + vpx_clear_system_state(); +} + +void VP8RateControlRTC::ComputeQP(const VP8FrameParamsQpRTC &frame_params) { + VP8_COMMON *const cm = &cpi_->common; + vpx_clear_system_state(); + if (cpi_->oxcf.number_of_layers > 1) { + cpi_->temporal_layer_id = frame_params.temporal_layer_id; + const int layer = frame_params.temporal_layer_id; + vp8_update_layer_contexts(cpi_); + /* Restore layer specific context & set frame rate */ + vp8_restore_layer_context(cpi_, layer); + vp8_new_framerate(cpi_, cpi_->layer_context[layer].framerate); + } + cm->frame_type = frame_params.frame_type; + cm->refresh_golden_frame = (cm->frame_type == KEY_FRAME) ? 1 : 0; + cm->refresh_alt_ref_frame = (cm->frame_type == KEY_FRAME) ? 1 : 0; + if (cm->frame_type == KEY_FRAME && cpi_->common.current_video_frame > 0) { + cpi_->common.frame_flags |= FRAMEFLAGS_KEY; + } + + vp8_pick_frame_size(cpi_); + + if (cpi_->buffer_level >= cpi_->oxcf.optimal_buffer_level && + cpi_->buffered_mode) { + /* Max adjustment is 1/4 */ + int Adjustment = cpi_->active_worst_quality / 4; + if (Adjustment) { + int buff_lvl_step; + if (cpi_->buffer_level < cpi_->oxcf.maximum_buffer_size) { + buff_lvl_step = (int)((cpi_->oxcf.maximum_buffer_size - + cpi_->oxcf.optimal_buffer_level) / + Adjustment); + if (buff_lvl_step) { + Adjustment = + (int)((cpi_->buffer_level - cpi_->oxcf.optimal_buffer_level) / + buff_lvl_step); + } else { + Adjustment = 0; + } + } + cpi_->active_worst_quality -= Adjustment; + if (cpi_->active_worst_quality < cpi_->active_best_quality) { + cpi_->active_worst_quality = cpi_->active_best_quality; + } + } + } + + if (cpi_->ni_frames > 150) { + int q = cpi_->active_worst_quality; + if (cm->frame_type == KEY_FRAME) { + cpi_->active_best_quality = kf_high_motion_minq[q]; + } else { + cpi_->active_best_quality = inter_minq[q]; + } + + if (cpi_->buffer_level >= cpi_->oxcf.maximum_buffer_size) { + cpi_->active_best_quality = cpi_->best_quality; + + } else if (cpi_->buffer_level > cpi_->oxcf.optimal_buffer_level) { + int Fraction = + (int)(((cpi_->buffer_level - cpi_->oxcf.optimal_buffer_level) * 128) / + (cpi_->oxcf.maximum_buffer_size - + cpi_->oxcf.optimal_buffer_level)); + int min_qadjustment = + ((cpi_->active_best_quality - cpi_->best_quality) * Fraction) / 128; + + cpi_->active_best_quality -= min_qadjustment; + } + } + + /* Clip the active best and worst quality values to limits */ + if (cpi_->active_worst_quality > cpi_->worst_quality) { + cpi_->active_worst_quality = cpi_->worst_quality; + } + if (cpi_->active_best_quality < cpi_->best_quality) { + cpi_->active_best_quality = cpi_->best_quality; + } + if (cpi_->active_worst_quality < cpi_->active_best_quality) { + cpi_->active_worst_quality = cpi_->active_best_quality; + } + + q_ = vp8_regulate_q(cpi_, cpi_->this_frame_target); + vp8_set_quantizer(cpi_, q_); + vpx_clear_system_state(); +} + +int VP8RateControlRTC::GetQP() const { return q_; } + +void VP8RateControlRTC::PostEncodeUpdate(uint64_t encoded_frame_size) { + VP8_COMMON *const cm = &cpi_->common; + vpx_clear_system_state(); + cpi_->total_byte_count += encoded_frame_size; + cpi_->projected_frame_size = static_cast(encoded_frame_size << 3); + if (cpi_->oxcf.number_of_layers > 1) { + for (unsigned int i = cpi_->current_layer + 1; + i < cpi_->oxcf.number_of_layers; ++i) { + cpi_->layer_context[i].total_byte_count += encoded_frame_size; + } + } + + vp8_update_rate_correction_factors(cpi_, 2); + + cpi_->last_q[cm->frame_type] = cm->base_qindex; + + if (cm->frame_type == KEY_FRAME) { + vp8_adjust_key_frame_context(cpi_); + } + + /* Keep a record of ambient average Q. */ + if (cm->frame_type != KEY_FRAME) { + cpi_->avg_frame_qindex = + (2 + 3 * cpi_->avg_frame_qindex + cm->base_qindex) >> 2; + } + /* Keep a record from which we can calculate the average Q excluding + * key frames. + */ + if (cm->frame_type != KEY_FRAME) { + cpi_->ni_frames++; + /* Damp value for first few frames */ + if (cpi_->ni_frames > 150) { + cpi_->ni_tot_qi += q_; + cpi_->ni_av_qi = (cpi_->ni_tot_qi / cpi_->ni_frames); + } else { + cpi_->ni_tot_qi += q_; + cpi_->ni_av_qi = + ((cpi_->ni_tot_qi / cpi_->ni_frames) + cpi_->worst_quality + 1) / 2; + } + + /* If the average Q is higher than what was used in the last + * frame (after going through the recode loop to keep the frame + * size within range) then use the last frame value - 1. The -1 + * is designed to stop Q and hence the data rate, from + * progressively falling away during difficult sections, but at + * the same time reduce the number of itterations around the + * recode loop. + */ + if (q_ > cpi_->ni_av_qi) cpi_->ni_av_qi = q_ - 1; + } + + cpi_->bits_off_target += + cpi_->av_per_frame_bandwidth - cpi_->projected_frame_size; + if (cpi_->bits_off_target > cpi_->oxcf.maximum_buffer_size) { + cpi_->bits_off_target = cpi_->oxcf.maximum_buffer_size; + } + + cpi_->total_actual_bits += cpi_->projected_frame_size; + cpi_->buffer_level = cpi_->bits_off_target; + + /* Propagate values to higher temporal layers */ + if (cpi_->oxcf.number_of_layers > 1) { + for (unsigned int i = cpi_->current_layer + 1; + i < cpi_->oxcf.number_of_layers; ++i) { + LAYER_CONTEXT *lc = &cpi_->layer_context[i]; + int bits_off_for_this_layer = (int)round( + lc->target_bandwidth / lc->framerate - cpi_->projected_frame_size); + + lc->bits_off_target += bits_off_for_this_layer; + + /* Clip buffer level to maximum buffer size for the layer */ + if (lc->bits_off_target > lc->maximum_buffer_size) { + lc->bits_off_target = lc->maximum_buffer_size; + } + + lc->total_actual_bits += cpi_->projected_frame_size; + lc->total_target_vs_actual += bits_off_for_this_layer; + lc->buffer_level = lc->bits_off_target; + } + } + + cpi_->common.current_video_frame++; + cpi_->frames_since_key++; + + if (cpi_->oxcf.number_of_layers > 1) vp8_save_layer_context(cpi_); + vpx_clear_system_state(); +} +} // namespace libvpx diff --git a/media/libvpx/libvpx/vp9/common/vp9_alloccommon.c b/media/libvpx/libvpx/vp9/common/vp9_alloccommon.c --- a/media/libvpx/libvpx/vp9/common/vp9_alloccommon.c +++ b/media/libvpx/libvpx/vp9/common/vp9_alloccommon.c @@ -71,10 +71,12 @@ } void vp9_free_ref_frame_buffers(BufferPool *pool) { int i; + if (!pool) return; + for (i = 0; i < FRAME_BUFFERS; ++i) { if (!pool->frame_bufs[i].released && pool->frame_bufs[i].raw_frame_buffer.data != NULL) { pool->release_fb_cb(pool->cb_priv, &pool->frame_bufs[i].raw_frame_buffer); pool->frame_bufs[i].ref_count = 0; @@ -98,11 +100,11 @@ (void)cm; #endif } void vp9_free_context_buffers(VP9_COMMON *cm) { - cm->free_mi(cm); + if (cm->free_mi) cm->free_mi(cm); free_seg_map(cm); vpx_free(cm->above_context); cm->above_context = NULL; vpx_free(cm->above_seg_context); cm->above_seg_context = NULL; diff --git a/media/libvpx/libvpx/vp9/common/vp9_common.h b/media/libvpx/libvpx/vp9/common/vp9_common.h --- a/media/libvpx/libvpx/vp9/common/vp9_common.h +++ b/media/libvpx/libvpx/vp9/common/vp9_common.h @@ -25,14 +25,14 @@ extern "C" { #endif // Only need this for fixed-size arrays, for structs just assign. #define vp9_copy(dest, src) \ - { \ + do { \ assert(sizeof(dest) == sizeof(src)); \ memcpy(dest, src, sizeof(src)); \ - } + } while (0) // Use this for variably-sized arrays. #define vp9_copy_array(dest, src, n) \ { \ assert(sizeof(*(dest)) == sizeof(*(src))); \ diff --git a/media/libvpx/libvpx/vp9/common/vp9_frame_buffers.c b/media/libvpx/libvpx/vp9/common/vp9_frame_buffers.c --- a/media/libvpx/libvpx/vp9/common/vp9_frame_buffers.c +++ b/media/libvpx/libvpx/vp9/common/vp9_frame_buffers.c @@ -12,18 +12,21 @@ #include "vp9/common/vp9_frame_buffers.h" #include "vpx_mem/vpx_mem.h" int vp9_alloc_internal_frame_buffers(InternalFrameBufferList *list) { + const int num_buffers = VP9_MAXIMUM_REF_BUFFERS + VPX_MAXIMUM_WORK_BUFFERS; assert(list != NULL); vp9_free_internal_frame_buffers(list); - list->num_internal_frame_buffers = - VP9_MAXIMUM_REF_BUFFERS + VPX_MAXIMUM_WORK_BUFFERS; - list->int_fb = (InternalFrameBuffer *)vpx_calloc( - list->num_internal_frame_buffers, sizeof(*list->int_fb)); - return (list->int_fb == NULL); + list->int_fb = + (InternalFrameBuffer *)vpx_calloc(num_buffers, sizeof(*list->int_fb)); + if (list->int_fb) { + list->num_internal_frame_buffers = num_buffers; + return 0; + } + return -1; } void vp9_free_internal_frame_buffers(InternalFrameBufferList *list) { int i; @@ -33,10 +36,11 @@ vpx_free(list->int_fb[i].data); list->int_fb[i].data = NULL; } vpx_free(list->int_fb); list->int_fb = NULL; + list->num_internal_frame_buffers = 0; } int vp9_get_frame_buffer(void *cb_priv, size_t min_size, vpx_codec_frame_buffer_t *fb) { int i; diff --git a/media/libvpx/libvpx/vp9/common/vp9_postproc.c b/media/libvpx/libvpx/vp9/common/vp9_postproc.c --- a/media/libvpx/libvpx/vp9/common/vp9_postproc.c +++ b/media/libvpx/libvpx/vp9/common/vp9_postproc.c @@ -358,10 +358,11 @@ if (flags & (VP9D_DEMACROBLOCK | VP9D_DEBLOCK)) { if (!cm->postproc_state.limits) { cm->postproc_state.limits = vpx_calloc(unscaled_width, sizeof(*cm->postproc_state.limits)); + if (!cm->postproc_state.limits) return 1; } } if (flags & VP9D_ADDNOISE) { if (!cm->postproc_state.generated_noise) { diff --git a/media/libvpx/libvpx/vp9/common/vp9_rtcd.c b/media/libvpx/libvpx/vp9/common/vp9_rtcd.c --- a/media/libvpx/libvpx/vp9/common/vp9_rtcd.c +++ b/media/libvpx/libvpx/vp9/common/vp9_rtcd.c @@ -10,10 +10,6 @@ #include "./vpx_config.h" #define RTCD_C #include "./vp9_rtcd.h" #include "vpx_ports/vpx_once.h" -void vp9_rtcd() { - // TODO(JBB): Remove this once, by insuring that both the encoder and - // decoder setup functions are protected by once(); - once(setup_rtcd_internal); -} +void vp9_rtcd() { once(setup_rtcd_internal); } diff --git a/media/libvpx/libvpx/vp9/common/vp9_rtcd_defs.pl b/media/libvpx/libvpx/vp9/common/vp9_rtcd_defs.pl --- a/media/libvpx/libvpx/vp9/common/vp9_rtcd_defs.pl +++ b/media/libvpx/libvpx/vp9/common/vp9_rtcd_defs.pl @@ -126,14 +126,14 @@ add_proto qw/int64_t vp9_block_error/, "const tran_low_t *coeff, const tran_low_t *dqcoeff, intptr_t block_size, int64_t *ssz"; add_proto qw/int64_t vp9_block_error_fp/, "const tran_low_t *coeff, const tran_low_t *dqcoeff, int block_size"; -add_proto qw/void vp9_quantize_fp/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; +add_proto qw/void vp9_quantize_fp/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; specialize qw/vp9_quantize_fp neon sse2 avx2 vsx/, "$ssse3_x86_64"; -add_proto qw/void vp9_quantize_fp_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; +add_proto qw/void vp9_quantize_fp_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; specialize qw/vp9_quantize_fp_32x32 neon vsx/, "$ssse3_x86_64"; if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { specialize qw/vp9_block_error avx2 sse2/; @@ -157,13 +157,13 @@ add_proto qw/void vp9_fwht4x4/, "const int16_t *input, tran_low_t *output, int stride"; # Note that there are more specializations appended when CONFIG_VP9_HIGHBITDEPTH # is off. -specialize qw/vp9_fht4x4 sse2/; -specialize qw/vp9_fht8x8 sse2/; -specialize qw/vp9_fht16x16 sse2/; +specialize qw/vp9_fht4x4 sse2 neon/; +specialize qw/vp9_fht8x8 sse2 neon/; +specialize qw/vp9_fht16x16 sse2 neon/; specialize qw/vp9_fwht4x4 sse2/; if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") ne "yes") { # Note that these specializations are appended to the above ones. specialize qw/vp9_fht4x4 msa/; specialize qw/vp9_fht8x8 msa/; @@ -193,13 +193,13 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { # ENCODEMB INVOKE - add_proto qw/void vp9_highbd_quantize_fp/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; + add_proto qw/void vp9_highbd_quantize_fp/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; - add_proto qw/void vp9_highbd_quantize_fp_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan" ; + add_proto qw/void vp9_highbd_quantize_fp_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan" ; # fdct functions add_proto qw/void vp9_highbd_fht4x4/, "const int16_t *input, tran_low_t *output, int stride, int tx_type"; add_proto qw/void vp9_highbd_fht8x8/, "const int16_t *input, tran_low_t *output, int stride, int tx_type"; diff --git a/media/libvpx/libvpx/vp9/common/vp9_seg_common.h b/media/libvpx/libvpx/vp9/common/vp9_seg_common.h --- a/media/libvpx/libvpx/vp9/common/vp9_seg_common.h +++ b/media/libvpx/libvpx/vp9/common/vp9_seg_common.h @@ -23,10 +23,15 @@ #define MAX_SEGMENTS 8 #define SEG_TREE_PROBS (MAX_SEGMENTS - 1) #define PREDICTION_PROBS 3 +// Segment ID used to skip background encoding +#define BACKGROUND_SEG_SKIP_ID 3 +// Number of frames that don't skip after a key frame +#define FRAMES_NO_SKIPPING_AFTER_KEY 20 + // Segment level features. typedef enum { SEG_LVL_ALT_Q = 0, // Use alternate Quantizer .... SEG_LVL_ALT_LF = 1, // Use alternate loop filter value... SEG_LVL_REF_FRAME = 2, // Optional Segment reference frame diff --git a/media/libvpx/libvpx/vp9/encoder/arm/neon/vp9_dct_neon.c b/media/libvpx/libvpx/vp9/encoder/arm/neon/vp9_dct_neon.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vp9/encoder/arm/neon/vp9_dct_neon.c @@ -0,0 +1,1460 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "./vpx_config.h" +#include "./vp9_rtcd.h" +#include "./vpx_dsp_rtcd.h" + +#include "vpx_dsp/txfm_common.h" +#include "vpx_dsp/arm/mem_neon.h" +#include "vpx_dsp/arm/transpose_neon.h" +#include "vpx_dsp/arm/fdct_neon.h" + +static INLINE void load_buffer_4x4(const int16_t *input, int16x8_t *in, + int stride) { + // { 0, 1, 1, 1, 1, 1, 1, 1 }; + const int16x8_t nonzero_bias_a = vextq_s16(vdupq_n_s16(0), vdupq_n_s16(1), 7); + // { 1, 0, 0, 0, 0, 0, 0, 0 }; + const int16x8_t nonzero_bias_b = vextq_s16(vdupq_n_s16(1), vdupq_n_s16(0), 7); + int16x8_t mask; + + int16x4_t input_0 = vshl_n_s16(vld1_s16(input + 0 * stride), 4); + int16x4_t input_1 = vshl_n_s16(vld1_s16(input + 1 * stride), 4); + int16x4_t input_2 = vshl_n_s16(vld1_s16(input + 2 * stride), 4); + int16x4_t input_3 = vshl_n_s16(vld1_s16(input + 3 * stride), 4); + + in[0] = vcombine_s16(input_0, input_1); + in[1] = vcombine_s16(input_2, input_3); + + // Copy the SSE method, use a mask to avoid an 'if' branch here to increase by + // one non-zero first elements + mask = vreinterpretq_s16_u16(vceqq_s16(in[0], nonzero_bias_a)); + in[0] = vaddq_s16(in[0], mask); + in[0] = vaddq_s16(in[0], nonzero_bias_b); +} + +static INLINE void write_buffer_4x4(tran_low_t *output, int16x8_t *res) { + const int16x8_t one_s16 = vdupq_n_s16(1); + res[0] = vaddq_s16(res[0], one_s16); + res[1] = vaddq_s16(res[1], one_s16); + res[0] = vshrq_n_s16(res[0], 2); + res[1] = vshrq_n_s16(res[1], 2); + store_s16q_to_tran_low(output + 0 * 8, res[0]); + store_s16q_to_tran_low(output + 1 * 8, res[1]); +} + +static INLINE void fadst4x4_neon(int16x8_t *in) { + int32x4_t u0, u1, u2, u3; + int16x4_t out_0, out_1, out_2, out_3; + const int32x4_t k__DCT_CONST_ROUNDING = vdupq_n_s32(DCT_CONST_ROUNDING); + + const int16x4_t s0 = vget_low_s16(in[0]); // | x_00 | x_01 | x_02 | x_03 | + const int16x4_t s1 = vget_high_s16(in[0]); // | x_10 | x_11 | x_12 | x_13 | + const int16x4_t s2 = vget_low_s16(in[1]); // | x_20 | x_21 | x_22 | x_23 | + const int16x4_t s3 = vget_high_s16(in[1]); // | x_30 | x_31 | x_32 | x_33 | + + // s0 * sinpi_1_9, s0 * sinpi_4_9 + // Must expand all elements to s32. See 'needs32' comment in fwd_txfm.c. + const int32x4_t s0s1_9 = vmull_n_s16(s0, sinpi_1_9); + const int32x4_t s0s4_9 = vmull_n_s16(s0, sinpi_4_9); + // s1 * sinpi_1_9, s1 * sinpi_2_9 + const int32x4_t s1s1_9 = vmull_n_s16(s1, sinpi_1_9); + const int32x4_t s1s2_9 = vmull_n_s16(s1, sinpi_2_9); + // s2 * sinpi_3_9 + const int32x4_t s2s3_9 = vmull_n_s16(s2, sinpi_3_9); + // s3 * sinpi_2_9, s3 * sinpi_4_9 + const int32x4_t s3s2_9 = vmull_n_s16(s3, sinpi_2_9); + const int32x4_t s3s4_9 = vmull_n_s16(s3, sinpi_4_9); + + // (s0 + s1) * sinpi_3_9 + const int32x4_t s0_p_s1 = vaddl_s16(s0, s1); + const int32x4_t s0_p_s1_m_s3 = vsubw_s16(s0_p_s1, s3); + + // s_0 * sinpi_1_9 + s_1 * sinpi_2_9 + // s_0 * sinpi_4_9 - s_1 * sinpi_1_9 + const int32x4_t s0s1_9_p_s1s2_9 = vaddq_s32(s0s1_9, s1s2_9); + const int32x4_t s0s4_9_m_s1s1_9 = vsubq_s32(s0s4_9, s1s1_9); + /* + * t0 = s0s1_9 + s1s2_9 + s3s4_9 + * t1 = (s0 + s1) * sinpi_3_9 - s3 * sinpi_3_9 + * t2 = s0s4_9 - s1s1_9 + s3s2_9 + * t3 = s2s3_9 + */ + const int32x4_t t0 = vaddq_s32(s0s1_9_p_s1s2_9, s3s4_9); + const int32x4_t t1 = vmulq_n_s32(s0_p_s1_m_s3, sinpi_3_9); + const int32x4_t t2 = vaddq_s32(s0s4_9_m_s1s1_9, s3s2_9); + const int32x4_t t3 = s2s3_9; + /* + * u0 = t0 + t3 + * u1 = t1 + * u2 = t2 - t3 + * u3 = t2 - t0 + t3 + */ + u0 = vaddq_s32(t0, t3); + u1 = t1; + u2 = vsubq_s32(t2, t3); + u3 = vaddq_s32(vsubq_s32(t2, t0), t3); + + // fdct_round_shift + u0 = vaddq_s32(u0, k__DCT_CONST_ROUNDING); + u1 = vaddq_s32(u1, k__DCT_CONST_ROUNDING); + u2 = vaddq_s32(u2, k__DCT_CONST_ROUNDING); + u3 = vaddq_s32(u3, k__DCT_CONST_ROUNDING); + + out_0 = vshrn_n_s32(u0, DCT_CONST_BITS); + out_1 = vshrn_n_s32(u1, DCT_CONST_BITS); + out_2 = vshrn_n_s32(u2, DCT_CONST_BITS); + out_3 = vshrn_n_s32(u3, DCT_CONST_BITS); + + transpose_s16_4x4d(&out_0, &out_1, &out_2, &out_3); + + in[0] = vcombine_s16(out_0, out_1); + in[1] = vcombine_s16(out_2, out_3); +} + +void vp9_fht4x4_neon(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + int16x8_t in[2]; + + switch (tx_type) { + case DCT_DCT: vpx_fdct4x4_neon(input, output, stride); break; + case ADST_DCT: + load_buffer_4x4(input, in, stride); + fadst4x4_neon(in); + vpx_fdct4x4_pass1_neon((int16x4_t *)in); + write_buffer_4x4(output, in); + break; + case DCT_ADST: + load_buffer_4x4(input, in, stride); + vpx_fdct4x4_pass1_neon((int16x4_t *)in); + fadst4x4_neon(in); + write_buffer_4x4(output, in); + break; + default: + assert(tx_type == ADST_ADST); + load_buffer_4x4(input, in, stride); + fadst4x4_neon(in); + fadst4x4_neon(in); + write_buffer_4x4(output, in); + break; + } +} + +static INLINE void load_buffer_8x8(const int16_t *input, int16x8_t *in, + int stride) { + in[0] = vshlq_n_s16(vld1q_s16(input + 0 * stride), 2); + in[1] = vshlq_n_s16(vld1q_s16(input + 1 * stride), 2); + in[2] = vshlq_n_s16(vld1q_s16(input + 2 * stride), 2); + in[3] = vshlq_n_s16(vld1q_s16(input + 3 * stride), 2); + in[4] = vshlq_n_s16(vld1q_s16(input + 4 * stride), 2); + in[5] = vshlq_n_s16(vld1q_s16(input + 5 * stride), 2); + in[6] = vshlq_n_s16(vld1q_s16(input + 6 * stride), 2); + in[7] = vshlq_n_s16(vld1q_s16(input + 7 * stride), 2); +} + +/* right shift and rounding + * first get the sign bit (bit 15). + * If bit == 1, it's the simple case of shifting right by one bit. + * If bit == 2, it essentially computes the expression: + * + * out[j * 16 + i] = (temp_out[j] + 1 + (temp_out[j] < 0)) >> 2; + * + * for each row. + */ +static INLINE void right_shift_8x8(int16x8_t *res, const int bit) { + int16x8_t sign0 = vshrq_n_s16(res[0], 15); + int16x8_t sign1 = vshrq_n_s16(res[1], 15); + int16x8_t sign2 = vshrq_n_s16(res[2], 15); + int16x8_t sign3 = vshrq_n_s16(res[3], 15); + int16x8_t sign4 = vshrq_n_s16(res[4], 15); + int16x8_t sign5 = vshrq_n_s16(res[5], 15); + int16x8_t sign6 = vshrq_n_s16(res[6], 15); + int16x8_t sign7 = vshrq_n_s16(res[7], 15); + + if (bit == 2) { + const int16x8_t const_rounding = vdupq_n_s16(1); + res[0] = vaddq_s16(res[0], const_rounding); + res[1] = vaddq_s16(res[1], const_rounding); + res[2] = vaddq_s16(res[2], const_rounding); + res[3] = vaddq_s16(res[3], const_rounding); + res[4] = vaddq_s16(res[4], const_rounding); + res[5] = vaddq_s16(res[5], const_rounding); + res[6] = vaddq_s16(res[6], const_rounding); + res[7] = vaddq_s16(res[7], const_rounding); + } + + res[0] = vsubq_s16(res[0], sign0); + res[1] = vsubq_s16(res[1], sign1); + res[2] = vsubq_s16(res[2], sign2); + res[3] = vsubq_s16(res[3], sign3); + res[4] = vsubq_s16(res[4], sign4); + res[5] = vsubq_s16(res[5], sign5); + res[6] = vsubq_s16(res[6], sign6); + res[7] = vsubq_s16(res[7], sign7); + + if (bit == 1) { + res[0] = vshrq_n_s16(res[0], 1); + res[1] = vshrq_n_s16(res[1], 1); + res[2] = vshrq_n_s16(res[2], 1); + res[3] = vshrq_n_s16(res[3], 1); + res[4] = vshrq_n_s16(res[4], 1); + res[5] = vshrq_n_s16(res[5], 1); + res[6] = vshrq_n_s16(res[6], 1); + res[7] = vshrq_n_s16(res[7], 1); + } else { + res[0] = vshrq_n_s16(res[0], 2); + res[1] = vshrq_n_s16(res[1], 2); + res[2] = vshrq_n_s16(res[2], 2); + res[3] = vshrq_n_s16(res[3], 2); + res[4] = vshrq_n_s16(res[4], 2); + res[5] = vshrq_n_s16(res[5], 2); + res[6] = vshrq_n_s16(res[6], 2); + res[7] = vshrq_n_s16(res[7], 2); + } +} + +static INLINE void write_buffer_8x8(tran_low_t *output, int16x8_t *res, + int stride) { + store_s16q_to_tran_low(output + 0 * stride, res[0]); + store_s16q_to_tran_low(output + 1 * stride, res[1]); + store_s16q_to_tran_low(output + 2 * stride, res[2]); + store_s16q_to_tran_low(output + 3 * stride, res[3]); + store_s16q_to_tran_low(output + 4 * stride, res[4]); + store_s16q_to_tran_low(output + 5 * stride, res[5]); + store_s16q_to_tran_low(output + 6 * stride, res[6]); + store_s16q_to_tran_low(output + 7 * stride, res[7]); +} + +static INLINE void fadst8x8_neon(int16x8_t *in) { + int16x4_t x0_lo, x0_hi, x1_lo, x1_hi, x2_lo, x2_hi, x3_lo, x3_hi, x4_lo, + x4_hi, x5_lo, x5_hi, x6_lo, x6_hi, x7_lo, x7_hi; + int32x4_t s0_lo, s0_hi, s1_lo, s1_hi, s2_lo, s2_hi, s3_lo, s3_hi, s4_lo, + s4_hi, s5_lo, s5_hi, s6_lo, s6_hi, s7_lo, s7_hi; + int32x4_t t0_lo, t0_hi, t1_lo, t1_hi, t2_lo, t2_hi, t3_lo, t3_hi, t4_lo, + t4_hi, t5_lo, t5_hi, t6_lo, t6_hi, t7_lo, t7_hi; + const int32x4_t k__DCT_CONST_ROUNDING = vdupq_n_s32(DCT_CONST_ROUNDING); + + x0_lo = vget_low_s16(in[7]); + x0_hi = vget_high_s16(in[7]); + x1_lo = vget_low_s16(in[0]); + x1_hi = vget_high_s16(in[0]); + x2_lo = vget_low_s16(in[5]); + x2_hi = vget_high_s16(in[5]); + x3_lo = vget_low_s16(in[2]); + x3_hi = vget_high_s16(in[2]); + x4_lo = vget_low_s16(in[3]); + x4_hi = vget_high_s16(in[3]); + x5_lo = vget_low_s16(in[4]); + x5_hi = vget_high_s16(in[4]); + x6_lo = vget_low_s16(in[1]); + x6_hi = vget_high_s16(in[1]); + x7_lo = vget_low_s16(in[6]); + x7_hi = vget_high_s16(in[6]); + + // stage 1 + // s0 = cospi_2_64 * x0 + cospi_30_64 * x1; + s0_lo = vaddq_s32(vmull_n_s16(x0_lo, cospi_2_64), + vmull_n_s16(x1_lo, cospi_30_64)); + s0_hi = vaddq_s32(vmull_n_s16(x0_hi, cospi_2_64), + vmull_n_s16(x1_hi, cospi_30_64)); + // s1 = cospi_30_64 * x0 - cospi_2_64 * x1; + s1_lo = vsubq_s32(vmull_n_s16(x0_lo, cospi_30_64), + vmull_n_s16(x1_lo, cospi_2_64)); + s1_hi = vsubq_s32(vmull_n_s16(x0_hi, cospi_30_64), + vmull_n_s16(x1_hi, cospi_2_64)); + // s2 = cospi_10_64 * x2 + cospi_22_64 * x3; + s2_lo = vaddq_s32(vmull_n_s16(x2_lo, cospi_10_64), + vmull_n_s16(x3_lo, cospi_22_64)); + s2_hi = vaddq_s32(vmull_n_s16(x2_hi, cospi_10_64), + vmull_n_s16(x3_hi, cospi_22_64)); + // s3 = cospi_22_64 * x2 - cospi_10_64 * x3; + s3_lo = vsubq_s32(vmull_n_s16(x2_lo, cospi_22_64), + vmull_n_s16(x3_lo, cospi_10_64)); + s3_hi = vsubq_s32(vmull_n_s16(x2_hi, cospi_22_64), + vmull_n_s16(x3_hi, cospi_10_64)); + // s4 = cospi_18_64 * x4 + cospi_14_64 * x5; + s4_lo = vaddq_s32(vmull_n_s16(x4_lo, cospi_18_64), + vmull_n_s16(x5_lo, cospi_14_64)); + s4_hi = vaddq_s32(vmull_n_s16(x4_hi, cospi_18_64), + vmull_n_s16(x5_hi, cospi_14_64)); + // s5 = cospi_14_64 * x4 - cospi_18_64 * x5; + s5_lo = vsubq_s32(vmull_n_s16(x4_lo, cospi_14_64), + vmull_n_s16(x5_lo, cospi_18_64)); + s5_hi = vsubq_s32(vmull_n_s16(x4_hi, cospi_14_64), + vmull_n_s16(x5_hi, cospi_18_64)); + // s6 = cospi_26_64 * x6 + cospi_6_64 * x7; + s6_lo = vaddq_s32(vmull_n_s16(x6_lo, cospi_26_64), + vmull_n_s16(x7_lo, cospi_6_64)); + s6_hi = vaddq_s32(vmull_n_s16(x6_hi, cospi_26_64), + vmull_n_s16(x7_hi, cospi_6_64)); + // s7 = cospi_6_64 * x6 - cospi_26_64 * x7; + s7_lo = vsubq_s32(vmull_n_s16(x6_lo, cospi_6_64), + vmull_n_s16(x7_lo, cospi_26_64)); + s7_hi = vsubq_s32(vmull_n_s16(x6_hi, cospi_6_64), + vmull_n_s16(x7_hi, cospi_26_64)); + + // fdct_round_shift + t0_lo = vaddq_s32(s0_lo, s4_lo); + t0_hi = vaddq_s32(s0_hi, s4_hi); + t1_lo = vaddq_s32(s1_lo, s5_lo); + t1_hi = vaddq_s32(s1_hi, s5_hi); + t2_lo = vaddq_s32(s2_lo, s6_lo); + t2_hi = vaddq_s32(s2_hi, s6_hi); + t3_lo = vaddq_s32(s3_lo, s7_lo); + t3_hi = vaddq_s32(s3_hi, s7_hi); + t4_lo = vsubq_s32(s0_lo, s4_lo); + t4_hi = vsubq_s32(s0_hi, s4_hi); + t5_lo = vsubq_s32(s1_lo, s5_lo); + t5_hi = vsubq_s32(s1_hi, s5_hi); + t6_lo = vsubq_s32(s2_lo, s6_lo); + t6_hi = vsubq_s32(s2_hi, s6_hi); + t7_lo = vsubq_s32(s3_lo, s7_lo); + t7_hi = vsubq_s32(s3_hi, s7_hi); + + t0_lo = vaddq_s32(t0_lo, k__DCT_CONST_ROUNDING); + t0_hi = vaddq_s32(t0_hi, k__DCT_CONST_ROUNDING); + t1_lo = vaddq_s32(t1_lo, k__DCT_CONST_ROUNDING); + t1_hi = vaddq_s32(t1_hi, k__DCT_CONST_ROUNDING); + t2_lo = vaddq_s32(t2_lo, k__DCT_CONST_ROUNDING); + t2_hi = vaddq_s32(t2_hi, k__DCT_CONST_ROUNDING); + t3_lo = vaddq_s32(t3_lo, k__DCT_CONST_ROUNDING); + t3_hi = vaddq_s32(t3_hi, k__DCT_CONST_ROUNDING); + t4_lo = vaddq_s32(t4_lo, k__DCT_CONST_ROUNDING); + t4_hi = vaddq_s32(t4_hi, k__DCT_CONST_ROUNDING); + t5_lo = vaddq_s32(t5_lo, k__DCT_CONST_ROUNDING); + t5_hi = vaddq_s32(t5_hi, k__DCT_CONST_ROUNDING); + t6_lo = vaddq_s32(t6_lo, k__DCT_CONST_ROUNDING); + t6_hi = vaddq_s32(t6_hi, k__DCT_CONST_ROUNDING); + t7_lo = vaddq_s32(t7_lo, k__DCT_CONST_ROUNDING); + t7_hi = vaddq_s32(t7_hi, k__DCT_CONST_ROUNDING); + + t0_lo = vshrq_n_s32(t0_lo, DCT_CONST_BITS); + t0_hi = vshrq_n_s32(t0_hi, DCT_CONST_BITS); + t1_lo = vshrq_n_s32(t1_lo, DCT_CONST_BITS); + t1_hi = vshrq_n_s32(t1_hi, DCT_CONST_BITS); + t2_lo = vshrq_n_s32(t2_lo, DCT_CONST_BITS); + t2_hi = vshrq_n_s32(t2_hi, DCT_CONST_BITS); + t3_lo = vshrq_n_s32(t3_lo, DCT_CONST_BITS); + t3_hi = vshrq_n_s32(t3_hi, DCT_CONST_BITS); + t4_lo = vshrq_n_s32(t4_lo, DCT_CONST_BITS); + t4_hi = vshrq_n_s32(t4_hi, DCT_CONST_BITS); + t5_lo = vshrq_n_s32(t5_lo, DCT_CONST_BITS); + t5_hi = vshrq_n_s32(t5_hi, DCT_CONST_BITS); + t6_lo = vshrq_n_s32(t6_lo, DCT_CONST_BITS); + t6_hi = vshrq_n_s32(t6_hi, DCT_CONST_BITS); + t7_lo = vshrq_n_s32(t7_lo, DCT_CONST_BITS); + t7_hi = vshrq_n_s32(t7_hi, DCT_CONST_BITS); + + // stage 2 + s0_lo = t0_lo; + s0_hi = t0_hi; + s1_lo = t1_lo; + s1_hi = t1_hi; + s2_lo = t2_lo; + s2_hi = t2_hi; + s3_lo = t3_lo; + s3_hi = t3_hi; + s4_lo = vaddq_s32(vmulq_n_s32(t4_lo, cospi_8_64), + vmulq_n_s32(t5_lo, cospi_24_64)); + s4_hi = vaddq_s32(vmulq_n_s32(t4_hi, cospi_8_64), + vmulq_n_s32(t5_hi, cospi_24_64)); + s5_lo = vsubq_s32(vmulq_n_s32(t4_lo, cospi_24_64), + vmulq_n_s32(t5_lo, cospi_8_64)); + s5_hi = vsubq_s32(vmulq_n_s32(t4_hi, cospi_24_64), + vmulq_n_s32(t5_hi, cospi_8_64)); + s6_lo = vaddq_s32(vmulq_n_s32(t6_lo, -cospi_24_64), + vmulq_n_s32(t7_lo, cospi_8_64)); + s6_hi = vaddq_s32(vmulq_n_s32(t6_hi, -cospi_24_64), + vmulq_n_s32(t7_hi, cospi_8_64)); + s7_lo = vaddq_s32(vmulq_n_s32(t6_lo, cospi_8_64), + vmulq_n_s32(t7_lo, cospi_24_64)); + s7_hi = vaddq_s32(vmulq_n_s32(t6_hi, cospi_8_64), + vmulq_n_s32(t7_hi, cospi_24_64)); + + // s0 + s2 + t0_lo = vaddq_s32(s0_lo, s2_lo); + t0_hi = vaddq_s32(s0_hi, s2_hi); + // s1 + s3 + t1_lo = vaddq_s32(s1_lo, s3_lo); + t1_hi = vaddq_s32(s1_hi, s3_hi); + // s0 - s2 + t2_lo = vsubq_s32(s0_lo, s2_lo); + t2_hi = vsubq_s32(s0_hi, s2_hi); + // s1 - s3 + t3_lo = vsubq_s32(s1_lo, s3_lo); + t3_hi = vsubq_s32(s1_hi, s3_hi); + // s4 + s6 + t4_lo = vaddq_s32(s4_lo, s6_lo); + t4_hi = vaddq_s32(s4_hi, s6_hi); + // s5 + s7 + t5_lo = vaddq_s32(s5_lo, s7_lo); + t5_hi = vaddq_s32(s5_hi, s7_hi); + // s4 - s6 + t6_lo = vsubq_s32(s4_lo, s6_lo); + t6_hi = vsubq_s32(s4_hi, s6_hi); + // s5 - s7 + t7_lo = vsubq_s32(s5_lo, s7_lo); + t7_hi = vsubq_s32(s5_hi, s7_hi); + + // fdct_round_shift + t4_lo = vaddq_s32(t4_lo, k__DCT_CONST_ROUNDING); + t4_hi = vaddq_s32(t4_hi, k__DCT_CONST_ROUNDING); + t5_lo = vaddq_s32(t5_lo, k__DCT_CONST_ROUNDING); + t5_hi = vaddq_s32(t5_hi, k__DCT_CONST_ROUNDING); + t6_lo = vaddq_s32(t6_lo, k__DCT_CONST_ROUNDING); + t6_hi = vaddq_s32(t6_hi, k__DCT_CONST_ROUNDING); + t7_lo = vaddq_s32(t7_lo, k__DCT_CONST_ROUNDING); + t7_hi = vaddq_s32(t7_hi, k__DCT_CONST_ROUNDING); + t4_lo = vshrq_n_s32(t4_lo, DCT_CONST_BITS); + t4_hi = vshrq_n_s32(t4_hi, DCT_CONST_BITS); + t5_lo = vshrq_n_s32(t5_lo, DCT_CONST_BITS); + t5_hi = vshrq_n_s32(t5_hi, DCT_CONST_BITS); + t6_lo = vshrq_n_s32(t6_lo, DCT_CONST_BITS); + t6_hi = vshrq_n_s32(t6_hi, DCT_CONST_BITS); + t7_lo = vshrq_n_s32(t7_lo, DCT_CONST_BITS); + t7_hi = vshrq_n_s32(t7_hi, DCT_CONST_BITS); + + // stage 3 + // cospi_16_64 * (x2 + x3) + s2_lo = vmulq_n_s32(vaddq_s32(t2_lo, t3_lo), cospi_16_64); + s2_hi = vmulq_n_s32(vaddq_s32(t2_hi, t3_hi), cospi_16_64); + // cospi_16_64 * (x2 - x3) + s3_lo = vmulq_n_s32(vsubq_s32(t2_lo, t3_lo), cospi_16_64); + s3_hi = vmulq_n_s32(vsubq_s32(t2_hi, t3_hi), cospi_16_64); + // cospi_16_64 * (x6 + x7) + s6_lo = vmulq_n_s32(vaddq_s32(t6_lo, t7_lo), cospi_16_64); + s6_hi = vmulq_n_s32(vaddq_s32(t6_hi, t7_hi), cospi_16_64); + // cospi_16_64 * (x2 - x3) + s7_lo = vmulq_n_s32(vsubq_s32(t6_lo, t7_lo), cospi_16_64); + s7_hi = vmulq_n_s32(vsubq_s32(t6_hi, t7_hi), cospi_16_64); + + // final fdct_round_shift + t2_lo = vaddq_s32(s2_lo, k__DCT_CONST_ROUNDING); + t2_hi = vaddq_s32(s2_hi, k__DCT_CONST_ROUNDING); + t3_lo = vaddq_s32(s3_lo, k__DCT_CONST_ROUNDING); + t3_hi = vaddq_s32(s3_hi, k__DCT_CONST_ROUNDING); + t6_lo = vaddq_s32(s6_lo, k__DCT_CONST_ROUNDING); + t6_hi = vaddq_s32(s6_hi, k__DCT_CONST_ROUNDING); + t7_lo = vaddq_s32(s7_lo, k__DCT_CONST_ROUNDING); + t7_hi = vaddq_s32(s7_hi, k__DCT_CONST_ROUNDING); + + x2_lo = vshrn_n_s32(t2_lo, DCT_CONST_BITS); + x2_hi = vshrn_n_s32(t2_hi, DCT_CONST_BITS); + x3_lo = vshrn_n_s32(t3_lo, DCT_CONST_BITS); + x3_hi = vshrn_n_s32(t3_hi, DCT_CONST_BITS); + x6_lo = vshrn_n_s32(t6_lo, DCT_CONST_BITS); + x6_hi = vshrn_n_s32(t6_hi, DCT_CONST_BITS); + x7_lo = vshrn_n_s32(t7_lo, DCT_CONST_BITS); + x7_hi = vshrn_n_s32(t7_hi, DCT_CONST_BITS); + + // x0, x1, x4, x5 narrow down to 16-bits directly + x0_lo = vmovn_s32(t0_lo); + x0_hi = vmovn_s32(t0_hi); + x1_lo = vmovn_s32(t1_lo); + x1_hi = vmovn_s32(t1_hi); + x4_lo = vmovn_s32(t4_lo); + x4_hi = vmovn_s32(t4_hi); + x5_lo = vmovn_s32(t5_lo); + x5_hi = vmovn_s32(t5_hi); + + in[0] = vcombine_s16(x0_lo, x0_hi); + in[1] = vnegq_s16(vcombine_s16(x4_lo, x4_hi)); + in[2] = vcombine_s16(x6_lo, x6_hi); + in[3] = vnegq_s16(vcombine_s16(x2_lo, x2_hi)); + in[4] = vcombine_s16(x3_lo, x3_hi); + in[5] = vnegq_s16(vcombine_s16(x7_lo, x7_hi)); + in[6] = vcombine_s16(x5_lo, x5_hi); + in[7] = vnegq_s16(vcombine_s16(x1_lo, x1_hi)); + + transpose_s16_8x8(&in[0], &in[1], &in[2], &in[3], &in[4], &in[5], &in[6], + &in[7]); +} + +void vp9_fht8x8_neon(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + int16x8_t in[8]; + + switch (tx_type) { + case DCT_DCT: vpx_fdct8x8_neon(input, output, stride); break; + case ADST_DCT: + load_buffer_8x8(input, in, stride); + fadst8x8_neon(in); + vpx_fdct8x8_pass1_neon(in); + right_shift_8x8(in, 1); + write_buffer_8x8(output, in, 8); + break; + case DCT_ADST: + load_buffer_8x8(input, in, stride); + vpx_fdct8x8_pass1_neon(in); + fadst8x8_neon(in); + right_shift_8x8(in, 1); + write_buffer_8x8(output, in, 8); + break; + default: + assert(tx_type == ADST_ADST); + load_buffer_8x8(input, in, stride); + fadst8x8_neon(in); + fadst8x8_neon(in); + right_shift_8x8(in, 1); + write_buffer_8x8(output, in, 8); + break; + } +} + +static INLINE void load_buffer_16x16(const int16_t *input, int16x8_t *in0, + int16x8_t *in1, int stride) { + // load first 8 columns + load_buffer_8x8(input, in0, stride); + load_buffer_8x8(input + 8 * stride, in0 + 8, stride); + + input += 8; + // load second 8 columns + load_buffer_8x8(input, in1, stride); + load_buffer_8x8(input + 8 * stride, in1 + 8, stride); +} + +static INLINE void write_buffer_16x16(tran_low_t *output, int16x8_t *in0, + int16x8_t *in1, int stride) { + // write first 8 columns + write_buffer_8x8(output, in0, stride); + write_buffer_8x8(output + 8 * stride, in0 + 8, stride); + + // write second 8 columns + output += 8; + write_buffer_8x8(output, in1, stride); + write_buffer_8x8(output + 8 * stride, in1 + 8, stride); +} + +static INLINE void right_shift_16x16(int16x8_t *res0, int16x8_t *res1) { + // perform rounding operations + right_shift_8x8(res0, 2); + right_shift_8x8(res0 + 8, 2); + right_shift_8x8(res1, 2); + right_shift_8x8(res1 + 8, 2); +} + +static void fdct16_8col(int16x8_t *in) { + // perform 16x16 1-D DCT for 8 columns + int16x8_t i[8], s1[8], s2[8], s3[8], t[8]; + int16x4_t t_lo[8], t_hi[8]; + int32x4_t u_lo[8], u_hi[8]; + const int32x4_t k__DCT_CONST_ROUNDING = vdupq_n_s32(DCT_CONST_ROUNDING); + + // stage 1 + i[0] = vaddq_s16(in[0], in[15]); + i[1] = vaddq_s16(in[1], in[14]); + i[2] = vaddq_s16(in[2], in[13]); + i[3] = vaddq_s16(in[3], in[12]); + i[4] = vaddq_s16(in[4], in[11]); + i[5] = vaddq_s16(in[5], in[10]); + i[6] = vaddq_s16(in[6], in[9]); + i[7] = vaddq_s16(in[7], in[8]); + + vpx_fdct8x8_pass1_neon(i); + transpose_s16_8x8(&i[0], &i[1], &i[2], &i[3], &i[4], &i[5], &i[6], &i[7]); + + // step 2 + s1[0] = vsubq_s16(in[7], in[8]); + s1[1] = vsubq_s16(in[6], in[9]); + s1[2] = vsubq_s16(in[5], in[10]); + s1[3] = vsubq_s16(in[4], in[11]); + s1[4] = vsubq_s16(in[3], in[12]); + s1[5] = vsubq_s16(in[2], in[13]); + s1[6] = vsubq_s16(in[1], in[14]); + s1[7] = vsubq_s16(in[0], in[15]); + + t[2] = vsubq_s16(s1[5], s1[2]); + t[3] = vsubq_s16(s1[4], s1[3]); + t[4] = vaddq_s16(s1[4], s1[3]); + t[5] = vaddq_s16(s1[5], s1[2]); + + t_lo[2] = vget_low_s16(t[2]); + t_hi[2] = vget_high_s16(t[2]); + t_lo[3] = vget_low_s16(t[3]); + t_hi[3] = vget_high_s16(t[3]); + t_lo[4] = vget_low_s16(t[4]); + t_hi[4] = vget_high_s16(t[4]); + t_lo[5] = vget_low_s16(t[5]); + t_hi[5] = vget_high_s16(t[5]); + + u_lo[2] = vmull_n_s16(t_lo[2], cospi_16_64); + u_hi[2] = vmull_n_s16(t_hi[2], cospi_16_64); + u_lo[3] = vmull_n_s16(t_lo[3], cospi_16_64); + u_hi[3] = vmull_n_s16(t_hi[3], cospi_16_64); + u_lo[4] = vmull_n_s16(t_lo[4], cospi_16_64); + u_hi[4] = vmull_n_s16(t_hi[4], cospi_16_64); + u_lo[5] = vmull_n_s16(t_lo[5], cospi_16_64); + u_hi[5] = vmull_n_s16(t_hi[5], cospi_16_64); + + u_lo[2] = vaddq_s32(u_lo[2], k__DCT_CONST_ROUNDING); + u_hi[2] = vaddq_s32(u_hi[2], k__DCT_CONST_ROUNDING); + u_lo[3] = vaddq_s32(u_lo[3], k__DCT_CONST_ROUNDING); + u_hi[3] = vaddq_s32(u_hi[3], k__DCT_CONST_ROUNDING); + u_lo[4] = vaddq_s32(u_lo[4], k__DCT_CONST_ROUNDING); + u_hi[4] = vaddq_s32(u_hi[4], k__DCT_CONST_ROUNDING); + u_lo[5] = vaddq_s32(u_lo[5], k__DCT_CONST_ROUNDING); + u_hi[5] = vaddq_s32(u_hi[5], k__DCT_CONST_ROUNDING); + + t_lo[2] = vshrn_n_s32(u_lo[2], DCT_CONST_BITS); + t_hi[2] = vshrn_n_s32(u_hi[2], DCT_CONST_BITS); + t_lo[3] = vshrn_n_s32(u_lo[3], DCT_CONST_BITS); + t_hi[3] = vshrn_n_s32(u_hi[3], DCT_CONST_BITS); + t_lo[4] = vshrn_n_s32(u_lo[4], DCT_CONST_BITS); + t_hi[4] = vshrn_n_s32(u_hi[4], DCT_CONST_BITS); + t_lo[5] = vshrn_n_s32(u_lo[5], DCT_CONST_BITS); + t_hi[5] = vshrn_n_s32(u_hi[5], DCT_CONST_BITS); + + s2[2] = vcombine_s16(t_lo[2], t_hi[2]); + s2[3] = vcombine_s16(t_lo[3], t_hi[3]); + s2[4] = vcombine_s16(t_lo[4], t_hi[4]); + s2[5] = vcombine_s16(t_lo[5], t_hi[5]); + + // step 3 + s3[0] = vaddq_s16(s1[0], s2[3]); + s3[1] = vaddq_s16(s1[1], s2[2]); + s3[2] = vsubq_s16(s1[1], s2[2]); + s3[3] = vsubq_s16(s1[0], s2[3]); + s3[4] = vsubq_s16(s1[7], s2[4]); + s3[5] = vsubq_s16(s1[6], s2[5]); + s3[6] = vaddq_s16(s1[6], s2[5]); + s3[7] = vaddq_s16(s1[7], s2[4]); + + // step 4 + t_lo[0] = vget_low_s16(s3[0]); + t_hi[0] = vget_high_s16(s3[0]); + t_lo[1] = vget_low_s16(s3[1]); + t_hi[1] = vget_high_s16(s3[1]); + t_lo[2] = vget_low_s16(s3[2]); + t_hi[2] = vget_high_s16(s3[2]); + t_lo[3] = vget_low_s16(s3[3]); + t_hi[3] = vget_high_s16(s3[3]); + t_lo[4] = vget_low_s16(s3[4]); + t_hi[4] = vget_high_s16(s3[4]); + t_lo[5] = vget_low_s16(s3[5]); + t_hi[5] = vget_high_s16(s3[5]); + t_lo[6] = vget_low_s16(s3[6]); + t_hi[6] = vget_high_s16(s3[6]); + t_lo[7] = vget_low_s16(s3[7]); + t_hi[7] = vget_high_s16(s3[7]); + + u_lo[1] = vaddq_s32(vmull_n_s16(t_lo[1], -cospi_8_64), + vmull_n_s16(t_lo[6], cospi_24_64)); + u_hi[1] = vaddq_s32(vmull_n_s16(t_hi[1], -cospi_8_64), + vmull_n_s16(t_hi[6], cospi_24_64)); + u_lo[2] = vaddq_s32(vmull_n_s16(t_lo[2], cospi_24_64), + vmull_n_s16(t_lo[5], cospi_8_64)); + u_hi[2] = vaddq_s32(vmull_n_s16(t_hi[2], cospi_24_64), + vmull_n_s16(t_hi[5], cospi_8_64)); + u_lo[5] = vaddq_s32(vmull_n_s16(t_lo[2], cospi_8_64), + vmull_n_s16(t_lo[5], -cospi_24_64)); + u_hi[5] = vaddq_s32(vmull_n_s16(t_hi[2], cospi_8_64), + vmull_n_s16(t_hi[5], -cospi_24_64)); + u_lo[6] = vaddq_s32(vmull_n_s16(t_lo[1], cospi_24_64), + vmull_n_s16(t_lo[6], cospi_8_64)); + u_hi[6] = vaddq_s32(vmull_n_s16(t_hi[1], cospi_24_64), + vmull_n_s16(t_hi[6], cospi_8_64)); + + u_lo[1] = vaddq_s32(u_lo[1], k__DCT_CONST_ROUNDING); + u_hi[1] = vaddq_s32(u_hi[1], k__DCT_CONST_ROUNDING); + u_lo[2] = vaddq_s32(u_lo[2], k__DCT_CONST_ROUNDING); + u_hi[2] = vaddq_s32(u_hi[2], k__DCT_CONST_ROUNDING); + u_lo[5] = vaddq_s32(u_lo[5], k__DCT_CONST_ROUNDING); + u_hi[5] = vaddq_s32(u_hi[5], k__DCT_CONST_ROUNDING); + u_lo[6] = vaddq_s32(u_lo[6], k__DCT_CONST_ROUNDING); + u_hi[6] = vaddq_s32(u_hi[6], k__DCT_CONST_ROUNDING); + + t_lo[1] = vshrn_n_s32(u_lo[1], DCT_CONST_BITS); + t_hi[1] = vshrn_n_s32(u_hi[1], DCT_CONST_BITS); + t_lo[2] = vshrn_n_s32(u_lo[2], DCT_CONST_BITS); + t_hi[2] = vshrn_n_s32(u_hi[2], DCT_CONST_BITS); + t_lo[5] = vshrn_n_s32(u_lo[5], DCT_CONST_BITS); + t_hi[5] = vshrn_n_s32(u_hi[5], DCT_CONST_BITS); + t_lo[6] = vshrn_n_s32(u_lo[6], DCT_CONST_BITS); + t_hi[6] = vshrn_n_s32(u_hi[6], DCT_CONST_BITS); + + s2[1] = vcombine_s16(t_lo[1], t_hi[1]); + s2[2] = vcombine_s16(t_lo[2], t_hi[2]); + s2[5] = vcombine_s16(t_lo[5], t_hi[5]); + s2[6] = vcombine_s16(t_lo[6], t_hi[6]); + + // step 5 + s1[0] = vaddq_s16(s3[0], s2[1]); + s1[1] = vsubq_s16(s3[0], s2[1]); + s1[2] = vaddq_s16(s3[3], s2[2]); + s1[3] = vsubq_s16(s3[3], s2[2]); + s1[4] = vsubq_s16(s3[4], s2[5]); + s1[5] = vaddq_s16(s3[4], s2[5]); + s1[6] = vsubq_s16(s3[7], s2[6]); + s1[7] = vaddq_s16(s3[7], s2[6]); + + // step 6 + t_lo[0] = vget_low_s16(s1[0]); + t_hi[0] = vget_high_s16(s1[0]); + t_lo[1] = vget_low_s16(s1[1]); + t_hi[1] = vget_high_s16(s1[1]); + t_lo[2] = vget_low_s16(s1[2]); + t_hi[2] = vget_high_s16(s1[2]); + t_lo[3] = vget_low_s16(s1[3]); + t_hi[3] = vget_high_s16(s1[3]); + t_lo[4] = vget_low_s16(s1[4]); + t_hi[4] = vget_high_s16(s1[4]); + t_lo[5] = vget_low_s16(s1[5]); + t_hi[5] = vget_high_s16(s1[5]); + t_lo[6] = vget_low_s16(s1[6]); + t_hi[6] = vget_high_s16(s1[6]); + t_lo[7] = vget_low_s16(s1[7]); + t_hi[7] = vget_high_s16(s1[7]); + + // step1[0] * cospi_30_64 + step1[7] * cospi_2_64; + u_lo[0] = vaddq_s32(vmull_n_s16(t_lo[0], cospi_30_64), + vmull_n_s16(t_lo[7], cospi_2_64)); + u_hi[0] = vaddq_s32(vmull_n_s16(t_hi[0], cospi_30_64), + vmull_n_s16(t_hi[7], cospi_2_64)); + + // step1[1] * cospi_14_64 + step1[6] * cospi_18_64; + u_lo[1] = vaddq_s32(vmull_n_s16(t_lo[1], cospi_14_64), + vmull_n_s16(t_lo[6], cospi_18_64)); + u_hi[1] = vaddq_s32(vmull_n_s16(t_hi[1], cospi_14_64), + vmull_n_s16(t_hi[6], cospi_18_64)); + + // step1[2] * cospi_22_64 + step1[5] * cospi_10_64; + u_lo[2] = vaddq_s32(vmull_n_s16(t_lo[2], cospi_22_64), + vmull_n_s16(t_lo[5], cospi_10_64)); + u_hi[2] = vaddq_s32(vmull_n_s16(t_hi[2], cospi_22_64), + vmull_n_s16(t_hi[5], cospi_10_64)); + + // step1[3] * cospi_6_64 + step1[4] * cospi_26_64; + u_lo[3] = vaddq_s32(vmull_n_s16(t_lo[3], cospi_6_64), + vmull_n_s16(t_lo[4], cospi_26_64)); + u_hi[3] = vaddq_s32(vmull_n_s16(t_hi[3], cospi_6_64), + vmull_n_s16(t_hi[4], cospi_26_64)); + + // step1[3] * -cospi_26_64 + step1[4] * cospi_6_64; + u_lo[4] = vaddq_s32(vmull_n_s16(t_lo[3], -cospi_26_64), + vmull_n_s16(t_lo[4], cospi_6_64)); + u_hi[4] = vaddq_s32(vmull_n_s16(t_hi[3], -cospi_26_64), + vmull_n_s16(t_hi[4], cospi_6_64)); + + // step1[2] * -cospi_10_64 + step1[5] * cospi_22_64; + u_lo[5] = vaddq_s32(vmull_n_s16(t_lo[2], -cospi_10_64), + vmull_n_s16(t_lo[5], cospi_22_64)); + u_hi[5] = vaddq_s32(vmull_n_s16(t_hi[2], -cospi_10_64), + vmull_n_s16(t_hi[5], cospi_22_64)); + + // step1[1] * -cospi_18_64 + step1[6] * cospi_14_64; + u_lo[6] = vaddq_s32(vmull_n_s16(t_lo[1], -cospi_18_64), + vmull_n_s16(t_lo[6], cospi_14_64)); + u_hi[6] = vaddq_s32(vmull_n_s16(t_hi[1], -cospi_18_64), + vmull_n_s16(t_hi[6], cospi_14_64)); + + // step1[0] * -cospi_2_64 + step1[7] * cospi_30_64; + u_lo[7] = vaddq_s32(vmull_n_s16(t_lo[0], -cospi_2_64), + vmull_n_s16(t_lo[7], cospi_30_64)); + u_hi[7] = vaddq_s32(vmull_n_s16(t_hi[0], -cospi_2_64), + vmull_n_s16(t_hi[7], cospi_30_64)); + + // final fdct_round_shift + u_lo[0] = vaddq_s32(u_lo[0], k__DCT_CONST_ROUNDING); + u_hi[0] = vaddq_s32(u_hi[0], k__DCT_CONST_ROUNDING); + u_lo[1] = vaddq_s32(u_lo[1], k__DCT_CONST_ROUNDING); + u_hi[1] = vaddq_s32(u_hi[1], k__DCT_CONST_ROUNDING); + u_lo[2] = vaddq_s32(u_lo[2], k__DCT_CONST_ROUNDING); + u_hi[2] = vaddq_s32(u_hi[2], k__DCT_CONST_ROUNDING); + u_lo[3] = vaddq_s32(u_lo[3], k__DCT_CONST_ROUNDING); + u_hi[3] = vaddq_s32(u_hi[3], k__DCT_CONST_ROUNDING); + u_lo[4] = vaddq_s32(u_lo[4], k__DCT_CONST_ROUNDING); + u_hi[4] = vaddq_s32(u_hi[4], k__DCT_CONST_ROUNDING); + u_lo[5] = vaddq_s32(u_lo[5], k__DCT_CONST_ROUNDING); + u_hi[5] = vaddq_s32(u_hi[5], k__DCT_CONST_ROUNDING); + u_lo[6] = vaddq_s32(u_lo[6], k__DCT_CONST_ROUNDING); + u_hi[6] = vaddq_s32(u_hi[6], k__DCT_CONST_ROUNDING); + u_lo[7] = vaddq_s32(u_lo[7], k__DCT_CONST_ROUNDING); + u_hi[7] = vaddq_s32(u_hi[7], k__DCT_CONST_ROUNDING); + + t_lo[0] = vshrn_n_s32(u_lo[0], DCT_CONST_BITS); + t_hi[0] = vshrn_n_s32(u_hi[0], DCT_CONST_BITS); + t_lo[1] = vshrn_n_s32(u_lo[1], DCT_CONST_BITS); + t_hi[1] = vshrn_n_s32(u_hi[1], DCT_CONST_BITS); + t_lo[2] = vshrn_n_s32(u_lo[2], DCT_CONST_BITS); + t_hi[2] = vshrn_n_s32(u_hi[2], DCT_CONST_BITS); + t_lo[3] = vshrn_n_s32(u_lo[3], DCT_CONST_BITS); + t_hi[3] = vshrn_n_s32(u_hi[3], DCT_CONST_BITS); + t_lo[4] = vshrn_n_s32(u_lo[4], DCT_CONST_BITS); + t_hi[4] = vshrn_n_s32(u_hi[4], DCT_CONST_BITS); + t_lo[5] = vshrn_n_s32(u_lo[5], DCT_CONST_BITS); + t_hi[5] = vshrn_n_s32(u_hi[5], DCT_CONST_BITS); + t_lo[6] = vshrn_n_s32(u_lo[6], DCT_CONST_BITS); + t_hi[6] = vshrn_n_s32(u_hi[6], DCT_CONST_BITS); + t_lo[7] = vshrn_n_s32(u_lo[7], DCT_CONST_BITS); + t_hi[7] = vshrn_n_s32(u_hi[7], DCT_CONST_BITS); + + in[0] = i[0]; + in[2] = i[1]; + in[4] = i[2]; + in[6] = i[3]; + in[8] = i[4]; + in[10] = i[5]; + in[12] = i[6]; + in[14] = i[7]; + in[1] = vcombine_s16(t_lo[0], t_hi[0]); + in[3] = vcombine_s16(t_lo[4], t_hi[4]); + in[5] = vcombine_s16(t_lo[2], t_hi[2]); + in[7] = vcombine_s16(t_lo[6], t_hi[6]); + in[9] = vcombine_s16(t_lo[1], t_hi[1]); + in[11] = vcombine_s16(t_lo[5], t_hi[5]); + in[13] = vcombine_s16(t_lo[3], t_hi[3]); + in[15] = vcombine_s16(t_lo[7], t_hi[7]); +} + +static void fadst16_8col(int16x8_t *in) { + // perform 16x16 1-D ADST for 8 columns + int16x4_t x_lo[16], x_hi[16]; + int32x4_t s_lo[16], s_hi[16]; + int32x4_t t_lo[16], t_hi[16]; + const int32x4_t k__DCT_CONST_ROUNDING = vdupq_n_s32(DCT_CONST_ROUNDING); + + x_lo[0] = vget_low_s16(in[15]); + x_hi[0] = vget_high_s16(in[15]); + x_lo[1] = vget_low_s16(in[0]); + x_hi[1] = vget_high_s16(in[0]); + x_lo[2] = vget_low_s16(in[13]); + x_hi[2] = vget_high_s16(in[13]); + x_lo[3] = vget_low_s16(in[2]); + x_hi[3] = vget_high_s16(in[2]); + x_lo[4] = vget_low_s16(in[11]); + x_hi[4] = vget_high_s16(in[11]); + x_lo[5] = vget_low_s16(in[4]); + x_hi[5] = vget_high_s16(in[4]); + x_lo[6] = vget_low_s16(in[9]); + x_hi[6] = vget_high_s16(in[9]); + x_lo[7] = vget_low_s16(in[6]); + x_hi[7] = vget_high_s16(in[6]); + x_lo[8] = vget_low_s16(in[7]); + x_hi[8] = vget_high_s16(in[7]); + x_lo[9] = vget_low_s16(in[8]); + x_hi[9] = vget_high_s16(in[8]); + x_lo[10] = vget_low_s16(in[5]); + x_hi[10] = vget_high_s16(in[5]); + x_lo[11] = vget_low_s16(in[10]); + x_hi[11] = vget_high_s16(in[10]); + x_lo[12] = vget_low_s16(in[3]); + x_hi[12] = vget_high_s16(in[3]); + x_lo[13] = vget_low_s16(in[12]); + x_hi[13] = vget_high_s16(in[12]); + x_lo[14] = vget_low_s16(in[1]); + x_hi[14] = vget_high_s16(in[1]); + x_lo[15] = vget_low_s16(in[14]); + x_hi[15] = vget_high_s16(in[14]); + + // stage 1 + // s0 = cospi_1_64 * x0 + cospi_31_64 * x1; + s_lo[0] = vaddq_s32(vmull_n_s16(x_lo[0], cospi_1_64), + vmull_n_s16(x_lo[1], cospi_31_64)); + s_hi[0] = vaddq_s32(vmull_n_s16(x_hi[0], cospi_1_64), + vmull_n_s16(x_hi[1], cospi_31_64)); + // s1 = cospi_31_64 * x0 - cospi_1_64 * x1; + s_lo[1] = vsubq_s32(vmull_n_s16(x_lo[0], cospi_31_64), + vmull_n_s16(x_lo[1], cospi_1_64)); + s_hi[1] = vsubq_s32(vmull_n_s16(x_hi[0], cospi_31_64), + vmull_n_s16(x_hi[1], cospi_1_64)); + // s2 = cospi_5_64 * x2 + cospi_27_64 * x3; + s_lo[2] = vaddq_s32(vmull_n_s16(x_lo[2], cospi_5_64), + vmull_n_s16(x_lo[3], cospi_27_64)); + s_hi[2] = vaddq_s32(vmull_n_s16(x_hi[2], cospi_5_64), + vmull_n_s16(x_hi[3], cospi_27_64)); + // s3 = cospi_27_64 * x2 - cospi_5_64 * x3; + s_lo[3] = vsubq_s32(vmull_n_s16(x_lo[2], cospi_27_64), + vmull_n_s16(x_lo[3], cospi_5_64)); + s_hi[3] = vsubq_s32(vmull_n_s16(x_hi[2], cospi_27_64), + vmull_n_s16(x_hi[3], cospi_5_64)); + // s4 = cospi_9_64 * x4 + cospi_23_64 * x5; + s_lo[4] = vaddq_s32(vmull_n_s16(x_lo[4], cospi_9_64), + vmull_n_s16(x_lo[5], cospi_23_64)); + s_hi[4] = vaddq_s32(vmull_n_s16(x_hi[4], cospi_9_64), + vmull_n_s16(x_hi[5], cospi_23_64)); + // s5 = cospi_23_64 * x4 - cospi_9_64 * x5; + s_lo[5] = vsubq_s32(vmull_n_s16(x_lo[4], cospi_23_64), + vmull_n_s16(x_lo[5], cospi_9_64)); + s_hi[5] = vsubq_s32(vmull_n_s16(x_hi[4], cospi_23_64), + vmull_n_s16(x_hi[5], cospi_9_64)); + // s6 = cospi_13_64 * x6 + cospi_19_64 * x7; + s_lo[6] = vaddq_s32(vmull_n_s16(x_lo[6], cospi_13_64), + vmull_n_s16(x_lo[7], cospi_19_64)); + s_hi[6] = vaddq_s32(vmull_n_s16(x_hi[6], cospi_13_64), + vmull_n_s16(x_hi[7], cospi_19_64)); + // s7 = cospi_19_64 * x6 - cospi_13_64 * x7; + s_lo[7] = vsubq_s32(vmull_n_s16(x_lo[6], cospi_19_64), + vmull_n_s16(x_lo[7], cospi_13_64)); + s_hi[7] = vsubq_s32(vmull_n_s16(x_hi[6], cospi_19_64), + vmull_n_s16(x_hi[7], cospi_13_64)); + // s8 = cospi_17_64 * x8 + cospi_15_64 * x9; + s_lo[8] = vaddq_s32(vmull_n_s16(x_lo[8], cospi_17_64), + vmull_n_s16(x_lo[9], cospi_15_64)); + s_hi[8] = vaddq_s32(vmull_n_s16(x_hi[8], cospi_17_64), + vmull_n_s16(x_hi[9], cospi_15_64)); + // s9 = cospi_15_64 * x8 - cospi_17_64 * x9; + s_lo[9] = vsubq_s32(vmull_n_s16(x_lo[8], cospi_15_64), + vmull_n_s16(x_lo[9], cospi_17_64)); + s_hi[9] = vsubq_s32(vmull_n_s16(x_hi[8], cospi_15_64), + vmull_n_s16(x_hi[9], cospi_17_64)); + // s10 = cospi_21_64 * x10 + cospi_11_64 * x11; + s_lo[10] = vaddq_s32(vmull_n_s16(x_lo[10], cospi_21_64), + vmull_n_s16(x_lo[11], cospi_11_64)); + s_hi[10] = vaddq_s32(vmull_n_s16(x_hi[10], cospi_21_64), + vmull_n_s16(x_hi[11], cospi_11_64)); + // s11 = cospi_11_64 * x10 - cospi_21_64 * x11; + s_lo[11] = vsubq_s32(vmull_n_s16(x_lo[10], cospi_11_64), + vmull_n_s16(x_lo[11], cospi_21_64)); + s_hi[11] = vsubq_s32(vmull_n_s16(x_hi[10], cospi_11_64), + vmull_n_s16(x_hi[11], cospi_21_64)); + // s12 = cospi_25_64 * x12 + cospi_7_64 * x13; + s_lo[12] = vaddq_s32(vmull_n_s16(x_lo[12], cospi_25_64), + vmull_n_s16(x_lo[13], cospi_7_64)); + s_hi[12] = vaddq_s32(vmull_n_s16(x_hi[12], cospi_25_64), + vmull_n_s16(x_hi[13], cospi_7_64)); + // s13 = cospi_7_64 * x12 - cospi_25_64 * x13; + s_lo[13] = vsubq_s32(vmull_n_s16(x_lo[12], cospi_7_64), + vmull_n_s16(x_lo[13], cospi_25_64)); + s_hi[13] = vsubq_s32(vmull_n_s16(x_hi[12], cospi_7_64), + vmull_n_s16(x_hi[13], cospi_25_64)); + // s14 = cospi_29_64 * x14 + cospi_3_64 * x15; + s_lo[14] = vaddq_s32(vmull_n_s16(x_lo[14], cospi_29_64), + vmull_n_s16(x_lo[15], cospi_3_64)); + s_hi[14] = vaddq_s32(vmull_n_s16(x_hi[14], cospi_29_64), + vmull_n_s16(x_hi[15], cospi_3_64)); + // s15 = cospi_3_64 * x14 - cospi_29_64 * x15; + s_lo[15] = vsubq_s32(vmull_n_s16(x_lo[14], cospi_3_64), + vmull_n_s16(x_lo[15], cospi_29_64)); + s_hi[15] = vsubq_s32(vmull_n_s16(x_hi[14], cospi_3_64), + vmull_n_s16(x_hi[15], cospi_29_64)); + + // fdct_round_shift + t_lo[0] = vaddq_s32(s_lo[0], s_lo[8]); + t_hi[0] = vaddq_s32(s_hi[0], s_hi[8]); + t_lo[1] = vaddq_s32(s_lo[1], s_lo[9]); + t_hi[1] = vaddq_s32(s_hi[1], s_hi[9]); + t_lo[2] = vaddq_s32(s_lo[2], s_lo[10]); + t_hi[2] = vaddq_s32(s_hi[2], s_hi[10]); + t_lo[3] = vaddq_s32(s_lo[3], s_lo[11]); + t_hi[3] = vaddq_s32(s_hi[3], s_hi[11]); + t_lo[4] = vaddq_s32(s_lo[4], s_lo[12]); + t_hi[4] = vaddq_s32(s_hi[4], s_hi[12]); + t_lo[5] = vaddq_s32(s_lo[5], s_lo[13]); + t_hi[5] = vaddq_s32(s_hi[5], s_hi[13]); + t_lo[6] = vaddq_s32(s_lo[6], s_lo[14]); + t_hi[6] = vaddq_s32(s_hi[6], s_hi[14]); + t_lo[7] = vaddq_s32(s_lo[7], s_lo[15]); + t_hi[7] = vaddq_s32(s_hi[7], s_hi[15]); + t_lo[8] = vsubq_s32(s_lo[0], s_lo[8]); + t_hi[8] = vsubq_s32(s_hi[0], s_hi[8]); + t_lo[9] = vsubq_s32(s_lo[1], s_lo[9]); + t_hi[9] = vsubq_s32(s_hi[1], s_hi[9]); + t_lo[10] = vsubq_s32(s_lo[2], s_lo[10]); + t_hi[10] = vsubq_s32(s_hi[2], s_hi[10]); + t_lo[11] = vsubq_s32(s_lo[3], s_lo[11]); + t_hi[11] = vsubq_s32(s_hi[3], s_hi[11]); + t_lo[12] = vsubq_s32(s_lo[4], s_lo[12]); + t_hi[12] = vsubq_s32(s_hi[4], s_hi[12]); + t_lo[13] = vsubq_s32(s_lo[5], s_lo[13]); + t_hi[13] = vsubq_s32(s_hi[5], s_hi[13]); + t_lo[14] = vsubq_s32(s_lo[6], s_lo[14]); + t_hi[14] = vsubq_s32(s_hi[6], s_hi[14]); + t_lo[15] = vsubq_s32(s_lo[7], s_lo[15]); + t_hi[15] = vsubq_s32(s_hi[7], s_hi[15]); + + t_lo[0] = vaddq_s32(t_lo[0], k__DCT_CONST_ROUNDING); + t_hi[0] = vaddq_s32(t_hi[0], k__DCT_CONST_ROUNDING); + t_lo[1] = vaddq_s32(t_lo[1], k__DCT_CONST_ROUNDING); + t_hi[1] = vaddq_s32(t_hi[1], k__DCT_CONST_ROUNDING); + t_lo[2] = vaddq_s32(t_lo[2], k__DCT_CONST_ROUNDING); + t_hi[2] = vaddq_s32(t_hi[2], k__DCT_CONST_ROUNDING); + t_lo[3] = vaddq_s32(t_lo[3], k__DCT_CONST_ROUNDING); + t_hi[3] = vaddq_s32(t_hi[3], k__DCT_CONST_ROUNDING); + t_lo[4] = vaddq_s32(t_lo[4], k__DCT_CONST_ROUNDING); + t_hi[4] = vaddq_s32(t_hi[4], k__DCT_CONST_ROUNDING); + t_lo[5] = vaddq_s32(t_lo[5], k__DCT_CONST_ROUNDING); + t_hi[5] = vaddq_s32(t_hi[5], k__DCT_CONST_ROUNDING); + t_lo[6] = vaddq_s32(t_lo[6], k__DCT_CONST_ROUNDING); + t_hi[6] = vaddq_s32(t_hi[6], k__DCT_CONST_ROUNDING); + t_lo[7] = vaddq_s32(t_lo[7], k__DCT_CONST_ROUNDING); + t_hi[7] = vaddq_s32(t_hi[7], k__DCT_CONST_ROUNDING); + t_lo[8] = vaddq_s32(t_lo[8], k__DCT_CONST_ROUNDING); + t_hi[8] = vaddq_s32(t_hi[8], k__DCT_CONST_ROUNDING); + t_lo[9] = vaddq_s32(t_lo[9], k__DCT_CONST_ROUNDING); + t_hi[9] = vaddq_s32(t_hi[9], k__DCT_CONST_ROUNDING); + t_lo[10] = vaddq_s32(t_lo[10], k__DCT_CONST_ROUNDING); + t_hi[10] = vaddq_s32(t_hi[10], k__DCT_CONST_ROUNDING); + t_lo[11] = vaddq_s32(t_lo[11], k__DCT_CONST_ROUNDING); + t_hi[11] = vaddq_s32(t_hi[11], k__DCT_CONST_ROUNDING); + t_lo[12] = vaddq_s32(t_lo[12], k__DCT_CONST_ROUNDING); + t_hi[12] = vaddq_s32(t_hi[12], k__DCT_CONST_ROUNDING); + t_lo[13] = vaddq_s32(t_lo[13], k__DCT_CONST_ROUNDING); + t_hi[13] = vaddq_s32(t_hi[13], k__DCT_CONST_ROUNDING); + t_lo[14] = vaddq_s32(t_lo[14], k__DCT_CONST_ROUNDING); + t_hi[14] = vaddq_s32(t_hi[14], k__DCT_CONST_ROUNDING); + t_lo[15] = vaddq_s32(t_lo[15], k__DCT_CONST_ROUNDING); + t_hi[15] = vaddq_s32(t_hi[15], k__DCT_CONST_ROUNDING); + + t_lo[0] = vshrq_n_s32(t_lo[0], DCT_CONST_BITS); + t_hi[0] = vshrq_n_s32(t_hi[0], DCT_CONST_BITS); + t_lo[1] = vshrq_n_s32(t_lo[1], DCT_CONST_BITS); + t_hi[1] = vshrq_n_s32(t_hi[1], DCT_CONST_BITS); + t_lo[2] = vshrq_n_s32(t_lo[2], DCT_CONST_BITS); + t_hi[2] = vshrq_n_s32(t_hi[2], DCT_CONST_BITS); + t_lo[3] = vshrq_n_s32(t_lo[3], DCT_CONST_BITS); + t_hi[3] = vshrq_n_s32(t_hi[3], DCT_CONST_BITS); + t_lo[4] = vshrq_n_s32(t_lo[4], DCT_CONST_BITS); + t_hi[4] = vshrq_n_s32(t_hi[4], DCT_CONST_BITS); + t_lo[5] = vshrq_n_s32(t_lo[5], DCT_CONST_BITS); + t_hi[5] = vshrq_n_s32(t_hi[5], DCT_CONST_BITS); + t_lo[6] = vshrq_n_s32(t_lo[6], DCT_CONST_BITS); + t_hi[6] = vshrq_n_s32(t_hi[6], DCT_CONST_BITS); + t_lo[7] = vshrq_n_s32(t_lo[7], DCT_CONST_BITS); + t_hi[7] = vshrq_n_s32(t_hi[7], DCT_CONST_BITS); + t_lo[8] = vshrq_n_s32(t_lo[8], DCT_CONST_BITS); + t_hi[8] = vshrq_n_s32(t_hi[8], DCT_CONST_BITS); + t_lo[9] = vshrq_n_s32(t_lo[9], DCT_CONST_BITS); + t_hi[9] = vshrq_n_s32(t_hi[9], DCT_CONST_BITS); + t_lo[10] = vshrq_n_s32(t_lo[10], DCT_CONST_BITS); + t_hi[10] = vshrq_n_s32(t_hi[10], DCT_CONST_BITS); + t_lo[11] = vshrq_n_s32(t_lo[11], DCT_CONST_BITS); + t_hi[11] = vshrq_n_s32(t_hi[11], DCT_CONST_BITS); + t_lo[12] = vshrq_n_s32(t_lo[12], DCT_CONST_BITS); + t_hi[12] = vshrq_n_s32(t_hi[12], DCT_CONST_BITS); + t_lo[13] = vshrq_n_s32(t_lo[13], DCT_CONST_BITS); + t_hi[13] = vshrq_n_s32(t_hi[13], DCT_CONST_BITS); + t_lo[14] = vshrq_n_s32(t_lo[14], DCT_CONST_BITS); + t_hi[14] = vshrq_n_s32(t_hi[14], DCT_CONST_BITS); + t_lo[15] = vshrq_n_s32(t_lo[15], DCT_CONST_BITS); + t_hi[15] = vshrq_n_s32(t_hi[15], DCT_CONST_BITS); + + // stage 2 + s_lo[0] = t_lo[0]; + s_hi[0] = t_hi[0]; + s_lo[1] = t_lo[1]; + s_hi[1] = t_hi[1]; + s_lo[2] = t_lo[2]; + s_hi[2] = t_hi[2]; + s_lo[3] = t_lo[3]; + s_hi[3] = t_hi[3]; + s_lo[4] = t_lo[4]; + s_hi[4] = t_hi[4]; + s_lo[5] = t_lo[5]; + s_hi[5] = t_hi[5]; + s_lo[6] = t_lo[6]; + s_hi[6] = t_hi[6]; + s_lo[7] = t_lo[7]; + s_hi[7] = t_hi[7]; + // s8 = x8 * cospi_4_64 + x9 * cospi_28_64; + s_lo[8] = vaddq_s32(vmulq_n_s32(t_lo[8], cospi_4_64), + vmulq_n_s32(t_lo[9], cospi_28_64)); + s_hi[8] = vaddq_s32(vmulq_n_s32(t_hi[8], cospi_4_64), + vmulq_n_s32(t_hi[9], cospi_28_64)); + // s9 = x8 * cospi_28_64 - x9 * cospi_4_64; + s_lo[9] = vsubq_s32(vmulq_n_s32(t_lo[8], cospi_28_64), + vmulq_n_s32(t_lo[9], cospi_4_64)); + s_hi[9] = vsubq_s32(vmulq_n_s32(t_hi[8], cospi_28_64), + vmulq_n_s32(t_hi[9], cospi_4_64)); + // s10 = x10 * cospi_20_64 + x11 * cospi_12_64; + s_lo[10] = vaddq_s32(vmulq_n_s32(t_lo[10], cospi_20_64), + vmulq_n_s32(t_lo[11], cospi_12_64)); + s_hi[10] = vaddq_s32(vmulq_n_s32(t_hi[10], cospi_20_64), + vmulq_n_s32(t_hi[11], cospi_12_64)); + // s11 = x10 * cospi_12_64 - x11 * cospi_20_64; + s_lo[11] = vsubq_s32(vmulq_n_s32(t_lo[10], cospi_12_64), + vmulq_n_s32(t_lo[11], cospi_20_64)); + s_hi[11] = vsubq_s32(vmulq_n_s32(t_hi[10], cospi_12_64), + vmulq_n_s32(t_hi[11], cospi_20_64)); + // s12 = -x12 * cospi_28_64 + x13 * cospi_4_64; + s_lo[12] = vaddq_s32(vmulq_n_s32(t_lo[12], -cospi_28_64), + vmulq_n_s32(t_lo[13], cospi_4_64)); + s_hi[12] = vaddq_s32(vmulq_n_s32(t_hi[12], -cospi_28_64), + vmulq_n_s32(t_hi[13], cospi_4_64)); + // s13 = x12 * cospi_4_64 + x13 * cospi_28_64; + s_lo[13] = vaddq_s32(vmulq_n_s32(t_lo[12], cospi_4_64), + vmulq_n_s32(t_lo[13], cospi_28_64)); + s_hi[13] = vaddq_s32(vmulq_n_s32(t_hi[12], cospi_4_64), + vmulq_n_s32(t_hi[13], cospi_28_64)); + // s14 = -x14 * cospi_12_64 + x15 * cospi_20_64; + s_lo[14] = vaddq_s32(vmulq_n_s32(t_lo[14], -cospi_12_64), + vmulq_n_s32(t_lo[15], cospi_20_64)); + s_hi[14] = vaddq_s32(vmulq_n_s32(t_hi[14], -cospi_12_64), + vmulq_n_s32(t_hi[15], cospi_20_64)); + // s15 = x14 * cospi_20_64 + x15 * cospi_12_64; + s_lo[15] = vaddq_s32(vmulq_n_s32(t_lo[14], cospi_20_64), + vmulq_n_s32(t_lo[15], cospi_12_64)); + s_hi[15] = vaddq_s32(vmulq_n_s32(t_hi[14], cospi_20_64), + vmulq_n_s32(t_hi[15], cospi_12_64)); + + // s0 + s4 + t_lo[0] = vaddq_s32(s_lo[0], s_lo[4]); + t_hi[0] = vaddq_s32(s_hi[0], s_hi[4]); + // s1 + s5 + t_lo[1] = vaddq_s32(s_lo[1], s_lo[5]); + t_hi[1] = vaddq_s32(s_hi[1], s_hi[5]); + // s2 + s6 + t_lo[2] = vaddq_s32(s_lo[2], s_lo[6]); + t_hi[2] = vaddq_s32(s_hi[2], s_hi[6]); + // s3 + s7 + t_lo[3] = vaddq_s32(s_lo[3], s_lo[7]); + t_hi[3] = vaddq_s32(s_hi[3], s_hi[7]); + // s0 - s4 + t_lo[4] = vsubq_s32(s_lo[0], s_lo[4]); + t_hi[4] = vsubq_s32(s_hi[0], s_hi[4]); + // s1 - s7 + t_lo[5] = vsubq_s32(s_lo[1], s_lo[5]); + t_hi[5] = vsubq_s32(s_hi[1], s_hi[5]); + // s2 - s6 + t_lo[6] = vsubq_s32(s_lo[2], s_lo[6]); + t_hi[6] = vsubq_s32(s_hi[2], s_hi[6]); + // s3 - s7 + t_lo[7] = vsubq_s32(s_lo[3], s_lo[7]); + t_hi[7] = vsubq_s32(s_hi[3], s_hi[7]); + // s8 + s12 + t_lo[8] = vaddq_s32(s_lo[8], s_lo[12]); + t_hi[8] = vaddq_s32(s_hi[8], s_hi[12]); + // s9 + s13 + t_lo[9] = vaddq_s32(s_lo[9], s_lo[13]); + t_hi[9] = vaddq_s32(s_hi[9], s_hi[13]); + // s10 + s14 + t_lo[10] = vaddq_s32(s_lo[10], s_lo[14]); + t_hi[10] = vaddq_s32(s_hi[10], s_hi[14]); + // s11 + s15 + t_lo[11] = vaddq_s32(s_lo[11], s_lo[15]); + t_hi[11] = vaddq_s32(s_hi[11], s_hi[15]); + // s8 + s12 + t_lo[12] = vsubq_s32(s_lo[8], s_lo[12]); + t_hi[12] = vsubq_s32(s_hi[8], s_hi[12]); + // s9 + s13 + t_lo[13] = vsubq_s32(s_lo[9], s_lo[13]); + t_hi[13] = vsubq_s32(s_hi[9], s_hi[13]); + // s10 + s14 + t_lo[14] = vsubq_s32(s_lo[10], s_lo[14]); + t_hi[14] = vsubq_s32(s_hi[10], s_hi[14]); + // s11 + s15 + t_lo[15] = vsubq_s32(s_lo[11], s_lo[15]); + t_hi[15] = vsubq_s32(s_hi[11], s_hi[15]); + + t_lo[8] = vaddq_s32(t_lo[8], k__DCT_CONST_ROUNDING); + t_hi[8] = vaddq_s32(t_hi[8], k__DCT_CONST_ROUNDING); + t_lo[9] = vaddq_s32(t_lo[9], k__DCT_CONST_ROUNDING); + t_hi[9] = vaddq_s32(t_hi[9], k__DCT_CONST_ROUNDING); + t_lo[10] = vaddq_s32(t_lo[10], k__DCT_CONST_ROUNDING); + t_hi[10] = vaddq_s32(t_hi[10], k__DCT_CONST_ROUNDING); + t_lo[11] = vaddq_s32(t_lo[11], k__DCT_CONST_ROUNDING); + t_hi[11] = vaddq_s32(t_hi[11], k__DCT_CONST_ROUNDING); + t_lo[12] = vaddq_s32(t_lo[12], k__DCT_CONST_ROUNDING); + t_hi[12] = vaddq_s32(t_hi[12], k__DCT_CONST_ROUNDING); + t_lo[13] = vaddq_s32(t_lo[13], k__DCT_CONST_ROUNDING); + t_hi[13] = vaddq_s32(t_hi[13], k__DCT_CONST_ROUNDING); + t_lo[14] = vaddq_s32(t_lo[14], k__DCT_CONST_ROUNDING); + t_hi[14] = vaddq_s32(t_hi[14], k__DCT_CONST_ROUNDING); + t_lo[15] = vaddq_s32(t_lo[15], k__DCT_CONST_ROUNDING); + t_hi[15] = vaddq_s32(t_hi[15], k__DCT_CONST_ROUNDING); + t_lo[8] = vshrq_n_s32(t_lo[8], DCT_CONST_BITS); + t_hi[8] = vshrq_n_s32(t_hi[8], DCT_CONST_BITS); + t_lo[9] = vshrq_n_s32(t_lo[9], DCT_CONST_BITS); + t_hi[9] = vshrq_n_s32(t_hi[9], DCT_CONST_BITS); + t_lo[10] = vshrq_n_s32(t_lo[10], DCT_CONST_BITS); + t_hi[10] = vshrq_n_s32(t_hi[10], DCT_CONST_BITS); + t_lo[11] = vshrq_n_s32(t_lo[11], DCT_CONST_BITS); + t_hi[11] = vshrq_n_s32(t_hi[11], DCT_CONST_BITS); + t_lo[12] = vshrq_n_s32(t_lo[12], DCT_CONST_BITS); + t_hi[12] = vshrq_n_s32(t_hi[12], DCT_CONST_BITS); + t_lo[13] = vshrq_n_s32(t_lo[13], DCT_CONST_BITS); + t_hi[13] = vshrq_n_s32(t_hi[13], DCT_CONST_BITS); + t_lo[14] = vshrq_n_s32(t_lo[14], DCT_CONST_BITS); + t_hi[14] = vshrq_n_s32(t_hi[14], DCT_CONST_BITS); + t_lo[15] = vshrq_n_s32(t_lo[15], DCT_CONST_BITS); + t_hi[15] = vshrq_n_s32(t_hi[15], DCT_CONST_BITS); + + // stage 3 + s_lo[0] = t_lo[0]; + s_hi[0] = t_hi[0]; + s_lo[1] = t_lo[1]; + s_hi[1] = t_hi[1]; + s_lo[2] = t_lo[2]; + s_hi[2] = t_hi[2]; + s_lo[3] = t_lo[3]; + s_hi[3] = t_hi[3]; + // s4 = x4 * cospi_8_64 + x5 * cospi_24_64; + s_lo[4] = vaddq_s32(vmulq_n_s32(t_lo[4], cospi_8_64), + vmulq_n_s32(t_lo[5], cospi_24_64)); + s_hi[4] = vaddq_s32(vmulq_n_s32(t_hi[4], cospi_8_64), + vmulq_n_s32(t_hi[5], cospi_24_64)); + // s5 = x4 * cospi_24_64 - x5 * cospi_8_64; + s_lo[5] = vaddq_s32(vmulq_n_s32(t_lo[4], cospi_24_64), + vmulq_n_s32(t_lo[5], -cospi_8_64)); + s_hi[5] = vaddq_s32(vmulq_n_s32(t_hi[4], cospi_24_64), + vmulq_n_s32(t_hi[5], -cospi_8_64)); + // s6 = -x6 * cospi_24_64 + x7 * cospi_8_64; + s_lo[6] = vaddq_s32(vmulq_n_s32(t_lo[6], -cospi_24_64), + vmulq_n_s32(t_lo[7], cospi_8_64)); + s_hi[6] = vaddq_s32(vmulq_n_s32(t_hi[6], -cospi_24_64), + vmulq_n_s32(t_hi[7], cospi_8_64)); + // s7 = x6 * cospi_8_64 + x7 * cospi_24_64; + s_lo[7] = vaddq_s32(vmulq_n_s32(t_lo[6], cospi_8_64), + vmulq_n_s32(t_lo[7], cospi_24_64)); + s_hi[7] = vaddq_s32(vmulq_n_s32(t_hi[6], cospi_8_64), + vmulq_n_s32(t_hi[7], cospi_24_64)); + s_lo[8] = t_lo[8]; + s_hi[8] = t_hi[8]; + s_lo[9] = t_lo[9]; + s_hi[9] = t_hi[9]; + s_lo[10] = t_lo[10]; + s_hi[10] = t_hi[10]; + s_lo[11] = t_lo[11]; + s_hi[11] = t_hi[11]; + // s12 = x12 * cospi_8_64 + x13 * cospi_24_64; + s_lo[12] = vaddq_s32(vmulq_n_s32(t_lo[12], cospi_8_64), + vmulq_n_s32(t_lo[13], cospi_24_64)); + s_hi[12] = vaddq_s32(vmulq_n_s32(t_hi[12], cospi_8_64), + vmulq_n_s32(t_hi[13], cospi_24_64)); + // s13 = x12 * cospi_24_64 - x13 * cospi_8_64; + s_lo[13] = vaddq_s32(vmulq_n_s32(t_lo[12], cospi_24_64), + vmulq_n_s32(t_lo[13], -cospi_8_64)); + s_hi[13] = vaddq_s32(vmulq_n_s32(t_hi[12], cospi_24_64), + vmulq_n_s32(t_hi[13], -cospi_8_64)); + // s14 = -x14 * cospi_24_64 + x15 * cospi_8_64; + s_lo[14] = vaddq_s32(vmulq_n_s32(t_lo[14], -cospi_24_64), + vmulq_n_s32(t_lo[15], cospi_8_64)); + s_hi[14] = vaddq_s32(vmulq_n_s32(t_hi[14], -cospi_24_64), + vmulq_n_s32(t_hi[15], cospi_8_64)); + // s15 = x14 * cospi_8_64 + x15 * cospi_24_64; + s_lo[15] = vaddq_s32(vmulq_n_s32(t_lo[14], cospi_8_64), + vmulq_n_s32(t_lo[15], cospi_24_64)); + s_hi[15] = vaddq_s32(vmulq_n_s32(t_hi[14], cospi_8_64), + vmulq_n_s32(t_hi[15], cospi_24_64)); + + // s0 + s4 + t_lo[0] = vaddq_s32(s_lo[0], s_lo[2]); + t_hi[0] = vaddq_s32(s_hi[0], s_hi[2]); + // s1 + s3 + t_lo[1] = vaddq_s32(s_lo[1], s_lo[3]); + t_hi[1] = vaddq_s32(s_hi[1], s_hi[3]); + // s0 - s4 + t_lo[2] = vsubq_s32(s_lo[0], s_lo[2]); + t_hi[2] = vsubq_s32(s_hi[0], s_hi[2]); + // s1 - s3 + t_lo[3] = vsubq_s32(s_lo[1], s_lo[3]); + t_hi[3] = vsubq_s32(s_hi[1], s_hi[3]); + // s4 + s6 + t_lo[4] = vaddq_s32(s_lo[4], s_lo[6]); + t_hi[4] = vaddq_s32(s_hi[4], s_hi[6]); + // s5 + s7 + t_lo[5] = vaddq_s32(s_lo[5], s_lo[7]); + t_hi[5] = vaddq_s32(s_hi[5], s_hi[7]); + // s4 - s6 + t_lo[6] = vsubq_s32(s_lo[4], s_lo[6]); + t_hi[6] = vsubq_s32(s_hi[4], s_hi[6]); + // s5 - s7 + t_lo[7] = vsubq_s32(s_lo[5], s_lo[7]); + t_hi[7] = vsubq_s32(s_hi[5], s_hi[7]); + // s8 + s10 + t_lo[8] = vaddq_s32(s_lo[8], s_lo[10]); + t_hi[8] = vaddq_s32(s_hi[8], s_hi[10]); + // s9 + s11 + t_lo[9] = vaddq_s32(s_lo[9], s_lo[11]); + t_hi[9] = vaddq_s32(s_hi[9], s_hi[11]); + // s8 - s10 + t_lo[10] = vsubq_s32(s_lo[8], s_lo[10]); + t_hi[10] = vsubq_s32(s_hi[8], s_hi[10]); + // s9 - s11 + t_lo[11] = vsubq_s32(s_lo[9], s_lo[11]); + t_hi[11] = vsubq_s32(s_hi[9], s_hi[11]); + // s12 + s14 + t_lo[12] = vaddq_s32(s_lo[12], s_lo[14]); + t_hi[12] = vaddq_s32(s_hi[12], s_hi[14]); + // s13 + s15 + t_lo[13] = vaddq_s32(s_lo[13], s_lo[15]); + t_hi[13] = vaddq_s32(s_hi[13], s_hi[15]); + // s12 - s14 + t_lo[14] = vsubq_s32(s_lo[12], s_lo[14]); + t_hi[14] = vsubq_s32(s_hi[12], s_hi[14]); + // s13 - s15 + t_lo[15] = vsubq_s32(s_lo[13], s_lo[15]); + t_hi[15] = vsubq_s32(s_hi[13], s_hi[15]); + + t_lo[4] = vaddq_s32(t_lo[4], k__DCT_CONST_ROUNDING); + t_hi[4] = vaddq_s32(t_hi[4], k__DCT_CONST_ROUNDING); + t_lo[5] = vaddq_s32(t_lo[5], k__DCT_CONST_ROUNDING); + t_hi[5] = vaddq_s32(t_hi[5], k__DCT_CONST_ROUNDING); + t_lo[6] = vaddq_s32(t_lo[6], k__DCT_CONST_ROUNDING); + t_hi[6] = vaddq_s32(t_hi[6], k__DCT_CONST_ROUNDING); + t_lo[7] = vaddq_s32(t_lo[7], k__DCT_CONST_ROUNDING); + t_hi[7] = vaddq_s32(t_hi[7], k__DCT_CONST_ROUNDING); + t_lo[12] = vaddq_s32(t_lo[12], k__DCT_CONST_ROUNDING); + t_hi[12] = vaddq_s32(t_hi[12], k__DCT_CONST_ROUNDING); + t_lo[13] = vaddq_s32(t_lo[13], k__DCT_CONST_ROUNDING); + t_hi[13] = vaddq_s32(t_hi[13], k__DCT_CONST_ROUNDING); + t_lo[14] = vaddq_s32(t_lo[14], k__DCT_CONST_ROUNDING); + t_hi[14] = vaddq_s32(t_hi[14], k__DCT_CONST_ROUNDING); + t_lo[15] = vaddq_s32(t_lo[15], k__DCT_CONST_ROUNDING); + t_hi[15] = vaddq_s32(t_hi[15], k__DCT_CONST_ROUNDING); + t_lo[4] = vshrq_n_s32(t_lo[4], DCT_CONST_BITS); + t_hi[4] = vshrq_n_s32(t_hi[4], DCT_CONST_BITS); + t_lo[5] = vshrq_n_s32(t_lo[5], DCT_CONST_BITS); + t_hi[5] = vshrq_n_s32(t_hi[5], DCT_CONST_BITS); + t_lo[6] = vshrq_n_s32(t_lo[6], DCT_CONST_BITS); + t_hi[6] = vshrq_n_s32(t_hi[6], DCT_CONST_BITS); + t_lo[7] = vshrq_n_s32(t_lo[7], DCT_CONST_BITS); + t_hi[7] = vshrq_n_s32(t_hi[7], DCT_CONST_BITS); + t_lo[12] = vshrq_n_s32(t_lo[12], DCT_CONST_BITS); + t_hi[12] = vshrq_n_s32(t_hi[12], DCT_CONST_BITS); + t_lo[13] = vshrq_n_s32(t_lo[13], DCT_CONST_BITS); + t_hi[13] = vshrq_n_s32(t_hi[13], DCT_CONST_BITS); + t_lo[14] = vshrq_n_s32(t_lo[14], DCT_CONST_BITS); + t_hi[14] = vshrq_n_s32(t_hi[14], DCT_CONST_BITS); + t_lo[15] = vshrq_n_s32(t_lo[15], DCT_CONST_BITS); + t_hi[15] = vshrq_n_s32(t_hi[15], DCT_CONST_BITS); + + // stage 4 + // s2 = (-cospi_16_64) * (x2 + x3); + s_lo[2] = vmulq_n_s32(vaddq_s32(t_lo[2], t_lo[3]), -cospi_16_64); + s_hi[2] = vmulq_n_s32(vaddq_s32(t_hi[2], t_hi[3]), -cospi_16_64); + // s3 = cospi_16_64 * (x2 - x3); + s_lo[3] = vmulq_n_s32(vsubq_s32(t_lo[2], t_lo[3]), cospi_16_64); + s_hi[3] = vmulq_n_s32(vsubq_s32(t_hi[2], t_hi[3]), cospi_16_64); + // s6 = cospi_16_64 * (x6 + x7); + s_lo[6] = vmulq_n_s32(vaddq_s32(t_lo[6], t_lo[7]), cospi_16_64); + s_hi[6] = vmulq_n_s32(vaddq_s32(t_hi[6], t_hi[7]), cospi_16_64); + // s7 = cospi_16_64 * (-x6 + x7); + s_lo[7] = vmulq_n_s32(vsubq_s32(t_lo[7], t_lo[6]), cospi_16_64); + s_hi[7] = vmulq_n_s32(vsubq_s32(t_hi[7], t_hi[6]), cospi_16_64); + // s10 = cospi_16_64 * (x10 + x11); + s_lo[10] = vmulq_n_s32(vaddq_s32(t_lo[10], t_lo[11]), cospi_16_64); + s_hi[10] = vmulq_n_s32(vaddq_s32(t_hi[10], t_hi[11]), cospi_16_64); + // s11 = cospi_16_64 * (-x10 + x11); + s_lo[11] = vmulq_n_s32(vsubq_s32(t_lo[11], t_lo[10]), cospi_16_64); + s_hi[11] = vmulq_n_s32(vsubq_s32(t_hi[11], t_hi[10]), cospi_16_64); + // s14 = (-cospi_16_64) * (x14 + x15); + s_lo[14] = vmulq_n_s32(vaddq_s32(t_lo[14], t_lo[15]), -cospi_16_64); + s_hi[14] = vmulq_n_s32(vaddq_s32(t_hi[14], t_hi[15]), -cospi_16_64); + // s15 = cospi_16_64 * (x14 - x15); + s_lo[15] = vmulq_n_s32(vsubq_s32(t_lo[14], t_lo[15]), cospi_16_64); + s_hi[15] = vmulq_n_s32(vsubq_s32(t_hi[14], t_hi[15]), cospi_16_64); + + // final fdct_round_shift + t_lo[2] = vaddq_s32(s_lo[2], k__DCT_CONST_ROUNDING); + t_hi[2] = vaddq_s32(s_hi[2], k__DCT_CONST_ROUNDING); + t_lo[3] = vaddq_s32(s_lo[3], k__DCT_CONST_ROUNDING); + t_hi[3] = vaddq_s32(s_hi[3], k__DCT_CONST_ROUNDING); + t_lo[6] = vaddq_s32(s_lo[6], k__DCT_CONST_ROUNDING); + t_hi[6] = vaddq_s32(s_hi[6], k__DCT_CONST_ROUNDING); + t_lo[7] = vaddq_s32(s_lo[7], k__DCT_CONST_ROUNDING); + t_hi[7] = vaddq_s32(s_hi[7], k__DCT_CONST_ROUNDING); + t_lo[10] = vaddq_s32(s_lo[10], k__DCT_CONST_ROUNDING); + t_hi[10] = vaddq_s32(s_hi[10], k__DCT_CONST_ROUNDING); + t_lo[11] = vaddq_s32(s_lo[11], k__DCT_CONST_ROUNDING); + t_hi[11] = vaddq_s32(s_hi[11], k__DCT_CONST_ROUNDING); + t_lo[14] = vaddq_s32(s_lo[14], k__DCT_CONST_ROUNDING); + t_hi[14] = vaddq_s32(s_hi[14], k__DCT_CONST_ROUNDING); + t_lo[15] = vaddq_s32(s_lo[15], k__DCT_CONST_ROUNDING); + t_hi[15] = vaddq_s32(s_hi[15], k__DCT_CONST_ROUNDING); + + x_lo[2] = vshrn_n_s32(t_lo[2], DCT_CONST_BITS); + x_hi[2] = vshrn_n_s32(t_hi[2], DCT_CONST_BITS); + x_lo[3] = vshrn_n_s32(t_lo[3], DCT_CONST_BITS); + x_hi[3] = vshrn_n_s32(t_hi[3], DCT_CONST_BITS); + x_lo[6] = vshrn_n_s32(t_lo[6], DCT_CONST_BITS); + x_hi[6] = vshrn_n_s32(t_hi[6], DCT_CONST_BITS); + x_lo[7] = vshrn_n_s32(t_lo[7], DCT_CONST_BITS); + x_hi[7] = vshrn_n_s32(t_hi[7], DCT_CONST_BITS); + x_lo[10] = vshrn_n_s32(t_lo[10], DCT_CONST_BITS); + x_hi[10] = vshrn_n_s32(t_hi[10], DCT_CONST_BITS); + x_lo[11] = vshrn_n_s32(t_lo[11], DCT_CONST_BITS); + x_hi[11] = vshrn_n_s32(t_hi[11], DCT_CONST_BITS); + x_lo[14] = vshrn_n_s32(t_lo[14], DCT_CONST_BITS); + x_hi[14] = vshrn_n_s32(t_hi[14], DCT_CONST_BITS); + x_lo[15] = vshrn_n_s32(t_lo[15], DCT_CONST_BITS); + x_hi[15] = vshrn_n_s32(t_hi[15], DCT_CONST_BITS); + + // x0, x1, x4, x5, x8, x9, x12, x13 narrow down to 16-bits directly + x_lo[0] = vmovn_s32(t_lo[0]); + x_hi[0] = vmovn_s32(t_hi[0]); + x_lo[1] = vmovn_s32(t_lo[1]); + x_hi[1] = vmovn_s32(t_hi[1]); + x_lo[4] = vmovn_s32(t_lo[4]); + x_hi[4] = vmovn_s32(t_hi[4]); + x_lo[5] = vmovn_s32(t_lo[5]); + x_hi[5] = vmovn_s32(t_hi[5]); + x_lo[8] = vmovn_s32(t_lo[8]); + x_hi[8] = vmovn_s32(t_hi[8]); + x_lo[9] = vmovn_s32(t_lo[9]); + x_hi[9] = vmovn_s32(t_hi[9]); + x_lo[12] = vmovn_s32(t_lo[12]); + x_hi[12] = vmovn_s32(t_hi[12]); + x_lo[13] = vmovn_s32(t_lo[13]); + x_hi[13] = vmovn_s32(t_hi[13]); + + in[0] = vcombine_s16(x_lo[0], x_hi[0]); + in[1] = vnegq_s16(vcombine_s16(x_lo[8], x_hi[8])); + in[2] = vcombine_s16(x_lo[12], x_hi[12]); + in[3] = vnegq_s16(vcombine_s16(x_lo[4], x_hi[4])); + in[4] = vcombine_s16(x_lo[6], x_hi[6]); + in[5] = vcombine_s16(x_lo[14], x_hi[14]); + in[6] = vcombine_s16(x_lo[10], x_hi[10]); + in[7] = vcombine_s16(x_lo[2], x_hi[2]); + in[8] = vcombine_s16(x_lo[3], x_hi[3]); + in[9] = vcombine_s16(x_lo[11], x_hi[11]); + in[10] = vcombine_s16(x_lo[15], x_hi[15]); + in[11] = vcombine_s16(x_lo[7], x_hi[7]); + in[12] = vcombine_s16(x_lo[5], x_hi[5]); + in[13] = vnegq_s16(vcombine_s16(x_lo[13], x_hi[13])); + in[14] = vcombine_s16(x_lo[9], x_hi[9]); + in[15] = vnegq_s16(vcombine_s16(x_lo[1], x_hi[1])); +} + +static void fdct16x16_neon(int16x8_t *in0, int16x8_t *in1) { + // Left half. + fdct16_8col(in0); + // Right half. + fdct16_8col(in1); + transpose_s16_16x16(in0, in1); +} + +static void fadst16x16_neon(int16x8_t *in0, int16x8_t *in1) { + fadst16_8col(in0); + fadst16_8col(in1); + transpose_s16_16x16(in0, in1); +} + +void vp9_fht16x16_neon(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + int16x8_t in0[16], in1[16]; + + switch (tx_type) { + case DCT_DCT: vpx_fdct16x16_neon(input, output, stride); break; + case ADST_DCT: + load_buffer_16x16(input, in0, in1, stride); + fadst16x16_neon(in0, in1); + right_shift_16x16(in0, in1); + fdct16x16_neon(in0, in1); + write_buffer_16x16(output, in0, in1, 16); + break; + case DCT_ADST: + load_buffer_16x16(input, in0, in1, stride); + fdct16x16_neon(in0, in1); + right_shift_16x16(in0, in1); + fadst16x16_neon(in0, in1); + write_buffer_16x16(output, in0, in1, 16); + break; + default: + assert(tx_type == ADST_ADST); + load_buffer_16x16(input, in0, in1, stride); + fadst16x16_neon(in0, in1); + right_shift_16x16(in0, in1); + fadst16x16_neon(in0, in1); + write_buffer_16x16(output, in0, in1, 16); + break; + } +} diff --git a/media/libvpx/libvpx/vp9/encoder/arm/neon/vp9_quantize_neon.c b/media/libvpx/libvpx/vp9/encoder/arm/neon/vp9_quantize_neon.c --- a/media/libvpx/libvpx/vp9/encoder/arm/neon/vp9_quantize_neon.c +++ b/media/libvpx/libvpx/vp9/encoder/arm/neon/vp9_quantize_neon.c @@ -41,15 +41,14 @@ vst1q_s16(dqcoeff, vcombine_s16(vmovn_s32(dqcoeff_0), vmovn_s32(dqcoeff_1))); #endif // CONFIG_VP9_HIGHBITDEPTH } void vp9_quantize_fp_neon(const tran_low_t *coeff_ptr, intptr_t count, - int skip_block, const int16_t *round_ptr, - const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, - uint16_t *eob_ptr, const int16_t *scan, - const int16_t *iscan) { + const int16_t *round_ptr, const int16_t *quant_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { // Quantization pass: All coefficients with index >= zero_flag are // skippable. Note: zero_flag can be zero. int i; const int16x8_t v_zero = vdupq_n_s16(0); const int16x8_t v_one = vdupq_n_s16(1); @@ -57,12 +56,10 @@ int16x8_t v_round = vmovq_n_s16(round_ptr[1]); int16x8_t v_quant = vmovq_n_s16(quant_ptr[1]); int16x8_t v_dequant = vmovq_n_s16(dequant_ptr[1]); (void)scan; - (void)skip_block; - assert(!skip_block); // adjust for dc v_round = vsetq_lane_s16(round_ptr[0], v_round, 0); v_quant = vsetq_lane_s16(quant_ptr[0], v_quant, 0); v_dequant = vsetq_lane_s16(dequant_ptr[0], v_dequant, 0); @@ -136,11 +133,11 @@ static INLINE int32x4_t extract_sign_bit(int32x4_t a) { return vreinterpretq_s32_u32(vshrq_n_u32(vreinterpretq_u32_s32(a), 31)); } void vp9_quantize_fp_32x32_neon(const tran_low_t *coeff_ptr, intptr_t count, - int skip_block, const int16_t *round_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan) { const int16x8_t one = vdupq_n_s16(1); @@ -165,12 +162,10 @@ int16x8_t qcoeff = vqaddq_s16(coeff_abs, round); int32x4_t dqcoeff_0, dqcoeff_1; uint16x8_t eob_max; (void)scan; (void)count; - (void)skip_block; - assert(!skip_block); // coeff * quant_ptr[]) >> 15 qcoeff = vqdmulhq_s16(qcoeff, quant); // Restore sign. diff --git a/media/libvpx/libvpx/vp9/encoder/ppc/vp9_quantize_vsx.c b/media/libvpx/libvpx/vp9/encoder/ppc/vp9_quantize_vsx.c --- a/media/libvpx/libvpx/vp9/encoder/ppc/vp9_quantize_vsx.c +++ b/media/libvpx/libvpx/vp9/encoder/ppc/vp9_quantize_vsx.c @@ -37,15 +37,14 @@ a = vec_max(a, vec_perm(a, a, vec_perm32)); return vec_max(a, vec_perm(a, a, vec_perm16)); } void vp9_quantize_fp_vsx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *round_ptr, - const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, - uint16_t *eob_ptr, const int16_t *scan, - const int16_t *iscan) { + const int16_t *round_ptr, const int16_t *quant_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { int16x8_t qcoeff0, qcoeff1, dqcoeff0, dqcoeff1, eob; bool16x8_t zero_coeff0, zero_coeff1; int16x8_t round = vec_vsx_ld(0, round_ptr); int16x8_t quant = vec_vsx_ld(0, quant_ptr); @@ -54,12 +53,10 @@ int16x8_t coeff1 = vec_vsx_ld(16, coeff_ptr); int16x8_t scan0 = vec_vsx_ld(0, iscan); int16x8_t scan1 = vec_vsx_ld(16, iscan); (void)scan; - (void)skip_block; - assert(!skip_block); // First set of 8 coeff starts with DC + 7 AC qcoeff0 = vec_mulhi(vec_vaddshs(vec_abs(coeff0), round), quant); zero_coeff0 = vec_cmpeq(qcoeff0, vec_zeros_s16); qcoeff0 = vec_sign(qcoeff0, coeff0); @@ -163,11 +160,11 @@ dqcoeffo = vec_sra(dqcoeffo, vec_ones_u32); return (int16x8_t)vec_perm(dqcoeffe, dqcoeffo, vec_perm_odd_even_pack); } void vp9_quantize_fp_32x32_vsx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *round_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan) { // In stage 1, we quantize 16 coeffs (DC + 15 AC) @@ -192,13 +189,11 @@ int16x8_t thres = vec_sra(dequant, vec_splats((uint16_t)2)); int16x8_t abs_coeff0 = vec_abs(coeff0); int16x8_t abs_coeff1 = vec_abs(coeff1); (void)scan; - (void)skip_block; (void)n_coeffs; - assert(!skip_block); mask0 = vec_cmpge(abs_coeff0, thres); round = vec_sra(vec_add(round, vec_ones_s16), vec_ones_u16); // First set of 8 coeff starts with DC + 7 AC qcoeff0 = vec_madds(vec_vaddshs(abs_coeff0, round), quant, vec_zeros_s16); diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_aq_cyclicrefresh.c b/media/libvpx/libvpx/vp9/encoder/vp9_aq_cyclicrefresh.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_aq_cyclicrefresh.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_aq_cyclicrefresh.c @@ -495,11 +495,13 @@ cpi->svc.layer_context[cpi->svc.temporal_layer_id].is_key_frame) || (!cpi->use_svc && cr->content_mode && rc->avg_frame_low_motion < thresh_low_motion && rc->frames_since_key > 40) || (!cpi->use_svc && rc->avg_frame_qindex[INTER_FRAME] > qp_max_thresh && - rc->frames_since_key > 20)) { + rc->frames_since_key > 20) || + (cpi->roi.enabled && cpi->roi.skip[BACKGROUND_SEG_SKIP_ID] && + rc->frames_since_key > FRAMES_NO_SKIPPING_AFTER_KEY)) { cr->apply_cyclic_refresh = 0; return; } cr->percent_refresh = 10; if (cr->reduce_refresh) cr->percent_refresh = 5; diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_bitstream.c b/media/libvpx/libvpx/vp9/encoder/vp9_bitstream.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_bitstream.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_bitstream.c @@ -234,15 +234,15 @@ } } } } -static void pack_inter_mode_mvs( - VP9_COMP *cpi, const MACROBLOCKD *const xd, - const MB_MODE_INFO_EXT *const mbmi_ext, vpx_writer *w, - unsigned int *const max_mv_magnitude, - int interp_filter_selected[MAX_REF_FRAMES][SWITCHABLE]) { +static void pack_inter_mode_mvs(VP9_COMP *cpi, const MACROBLOCKD *const xd, + const MB_MODE_INFO_EXT *const mbmi_ext, + vpx_writer *w, + unsigned int *const max_mv_magnitude, + int interp_filter_selected[][SWITCHABLE]) { VP9_COMMON *const cm = &cpi->common; const nmv_context *nmvc = &cm->fc->nmvc; const struct segmentation *const seg = &cm->seg; const MODE_INFO *const mi = xd->mi[0]; const PREDICTION_MODE mode = mi->mode; @@ -371,15 +371,16 @@ } write_intra_mode(w, mi->uv_mode, vp9_kf_uv_mode_prob[mi->mode]); } -static void write_modes_b( - VP9_COMP *cpi, MACROBLOCKD *const xd, const TileInfo *const tile, - vpx_writer *w, TOKENEXTRA **tok, const TOKENEXTRA *const tok_end, - int mi_row, int mi_col, unsigned int *const max_mv_magnitude, - int interp_filter_selected[MAX_REF_FRAMES][SWITCHABLE]) { +static void write_modes_b(VP9_COMP *cpi, MACROBLOCKD *const xd, + const TileInfo *const tile, vpx_writer *w, + TOKENEXTRA **tok, const TOKENEXTRA *const tok_end, + int mi_row, int mi_col, + unsigned int *const max_mv_magnitude, + int interp_filter_selected[][SWITCHABLE]) { const VP9_COMMON *const cm = &cpi->common; const MB_MODE_INFO_EXT *const mbmi_ext = cpi->td.mb.mbmi_ext_base + (mi_row * cm->mi_cols + mi_col); MODE_INFO *m; @@ -420,16 +421,16 @@ } else { assert(p == PARTITION_SPLIT); } } -static void write_modes_sb( - VP9_COMP *cpi, MACROBLOCKD *const xd, const TileInfo *const tile, - vpx_writer *w, TOKENEXTRA **tok, const TOKENEXTRA *const tok_end, - int mi_row, int mi_col, BLOCK_SIZE bsize, - unsigned int *const max_mv_magnitude, - int interp_filter_selected[MAX_REF_FRAMES][SWITCHABLE]) { +static void write_modes_sb(VP9_COMP *cpi, MACROBLOCKD *const xd, + const TileInfo *const tile, vpx_writer *w, + TOKENEXTRA **tok, const TOKENEXTRA *const tok_end, + int mi_row, int mi_col, BLOCK_SIZE bsize, + unsigned int *const max_mv_magnitude, + int interp_filter_selected[][SWITCHABLE]) { const VP9_COMMON *const cm = &cpi->common; const int bsl = b_width_log2_lookup[bsize]; const int bs = (1 << bsl) / 4; PARTITION_TYPE partition; BLOCK_SIZE subsize; @@ -483,15 +484,14 @@ if (bsize >= BLOCK_8X8 && (bsize == BLOCK_8X8 || partition != PARTITION_SPLIT)) update_partition_context(xd, mi_row, mi_col, subsize, bsize); } -static void write_modes( - VP9_COMP *cpi, MACROBLOCKD *const xd, const TileInfo *const tile, - vpx_writer *w, int tile_row, int tile_col, - unsigned int *const max_mv_magnitude, - int interp_filter_selected[MAX_REF_FRAMES][SWITCHABLE]) { +static void write_modes(VP9_COMP *cpi, MACROBLOCKD *const xd, + const TileInfo *const tile, vpx_writer *w, int tile_row, + int tile_col, unsigned int *const max_mv_magnitude, + int interp_filter_selected[][SWITCHABLE]) { const VP9_COMMON *const cm = &cpi->common; int mi_row, mi_col, tile_sb_row; TOKENEXTRA *tok = NULL; TOKENEXTRA *tok_end = NULL; @@ -552,20 +552,20 @@ int stepsize = cpi->sf.coeff_prob_appx_step; switch (cpi->sf.use_fast_coef_updates) { case TWO_LOOP: { /* dry run to see if there is any update at all needed */ - int savings = 0; + int64_t savings = 0; int update[2] = { 0, 0 }; for (i = 0; i < PLANE_TYPES; ++i) { for (j = 0; j < REF_TYPES; ++j) { for (k = 0; k < COEF_BANDS; ++k) { for (l = 0; l < BAND_COEFF_CONTEXTS(k); ++l) { for (t = 0; t < entropy_nodes_update; ++t) { vpx_prob newp = new_coef_probs[i][j][k][l][t]; const vpx_prob oldp = old_coef_probs[i][j][k][l][t]; - int s; + int64_t s; int u = 0; if (t == PIVOT_NODE) s = vp9_prob_diff_update_savings_search_model( frame_branch_ct[i][j][k][l][0], oldp, &newp, upd, stepsize); @@ -598,11 +598,11 @@ // calc probs and branch cts for this frame only for (t = 0; t < entropy_nodes_update; ++t) { vpx_prob newp = new_coef_probs[i][j][k][l][t]; vpx_prob *oldp = old_coef_probs[i][j][k][l] + t; const vpx_prob upd = DIFF_UPDATE_PROB; - int s; + int64_t s; int u = 0; if (t == PIVOT_NODE) s = vp9_prob_diff_update_savings_search_model( frame_branch_ct[i][j][k][l][0], *oldp, &newp, upd, stepsize); @@ -634,11 +634,11 @@ for (l = 0; l < BAND_COEFF_CONTEXTS(k); ++l) { // calc probs and branch cts for this frame only for (t = 0; t < entropy_nodes_update; ++t) { vpx_prob newp = new_coef_probs[i][j][k][l][t]; vpx_prob *oldp = old_coef_probs[i][j][k][l] + t; - int s; + int64_t s; int u = 0; if (t == PIVOT_NODE) { s = vp9_prob_diff_update_savings_search_model( frame_branch_ct[i][j][k][l][0], *oldp, &newp, upd, @@ -961,25 +961,24 @@ vpx_free(cpi->vp9_bitstream_worker_data); cpi->vp9_bitstream_worker_data = NULL; } } -static int encode_tiles_buffer_alloc(VP9_COMP *const cpi) { +static void encode_tiles_buffer_alloc(VP9_COMP *const cpi) { + VP9_COMMON *const cm = &cpi->common; int i; const size_t worker_data_size = cpi->num_workers * sizeof(*cpi->vp9_bitstream_worker_data); - cpi->vp9_bitstream_worker_data = vpx_memalign(16, worker_data_size); + CHECK_MEM_ERROR(cm, cpi->vp9_bitstream_worker_data, + vpx_memalign(16, worker_data_size)); memset(cpi->vp9_bitstream_worker_data, 0, worker_data_size); - if (!cpi->vp9_bitstream_worker_data) return 1; for (i = 1; i < cpi->num_workers; ++i) { cpi->vp9_bitstream_worker_data[i].dest_size = cpi->oxcf.width * cpi->oxcf.height; - cpi->vp9_bitstream_worker_data[i].dest = - vpx_malloc(cpi->vp9_bitstream_worker_data[i].dest_size); - if (!cpi->vp9_bitstream_worker_data[i].dest) return 1; + CHECK_MEM_ERROR(cm, cpi->vp9_bitstream_worker_data[i].dest, + vpx_malloc(cpi->vp9_bitstream_worker_data[i].dest_size)); } - return 0; } static size_t encode_tiles_mt(VP9_COMP *cpi, uint8_t *data_ptr) { const VPxWorkerInterface *const winterface = vpx_get_worker_interface(); VP9_COMMON *const cm = &cpi->common; @@ -990,11 +989,11 @@ if (!cpi->vp9_bitstream_worker_data || cpi->vp9_bitstream_worker_data[1].dest_size > (cpi->oxcf.width * cpi->oxcf.height)) { vp9_bitstream_encode_tiles_buffer_dealloc(cpi); - if (encode_tiles_buffer_alloc(cpi)) return 0; + encode_tiles_buffer_alloc(cpi); } while (tile_col < tile_cols) { int i, j; for (i = 0; i < num_workers && tile_col < tile_cols; ++i) { diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_cost.h b/media/libvpx/libvpx/vp9/encoder/vp9_cost.h --- a/media/libvpx/libvpx/vp9/encoder/vp9_cost.h +++ b/media/libvpx/libvpx/vp9/encoder/vp9_cost.h @@ -27,13 +27,12 @@ #define vp9_cost_one(prob) vp9_cost_zero(256 - (prob)) #define vp9_cost_bit(prob, bit) vp9_cost_zero((bit) ? 256 - (prob) : (prob)) -static INLINE unsigned int cost_branch256(const unsigned int ct[2], - vpx_prob p) { - return ct[0] * vp9_cost_zero(p) + ct[1] * vp9_cost_one(p); +static INLINE uint64_t cost_branch256(const unsigned int ct[2], vpx_prob p) { + return (uint64_t)ct[0] * vp9_cost_zero(p) + (uint64_t)ct[1] * vp9_cost_one(p); } static INLINE int treed_cost(vpx_tree tree, const vpx_prob *probs, int bits, int len) { int cost = 0; diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_encodeframe.c b/media/libvpx/libvpx/vp9/encoder/vp9_encodeframe.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_encodeframe.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_encodeframe.c @@ -5511,20 +5511,10 @@ x->sb_mvrow_part = 0; x->sb_pickmode_part = 0; x->arf_frame_usage = 0; x->lastgolden_frame_usage = 0; - if (seg->enabled) { - const uint8_t *const map = - seg->update_map ? cpi->segmentation_map : cm->last_frame_seg_map; - int segment_id = get_segment_id(cm, map, BLOCK_64X64, mi_row, mi_col); - seg_skip = segfeature_active(seg, segment_id, SEG_LVL_SKIP); - if (seg_skip) { - partition_search_type = FIXED_PARTITION; - } - } - if (cpi->compute_source_sad_onepass && cpi->sf.use_source_sad) { int shift = cpi->Source->y_stride * (mi_row << 3) + (mi_col << 3); int sb_offset2 = ((cm->mi_cols + 7) >> 3) * (mi_row >> 3) + (mi_col >> 3); int64_t source_sad = avg_source_sad(cpi, x, shift, sb_offset2); if (sf->adapt_partition_source_sad && @@ -5532,10 +5522,42 @@ source_sad > sf->adapt_partition_thresh && (cpi->refresh_golden_frame || cpi->refresh_alt_ref_frame))) partition_search_type = REFERENCE_PARTITION; } + if (seg->enabled) { + const uint8_t *const map = + seg->update_map ? cpi->segmentation_map : cm->last_frame_seg_map; + int segment_id = get_segment_id(cm, map, BLOCK_64X64, mi_row, mi_col); + seg_skip = segfeature_active(seg, segment_id, SEG_LVL_SKIP); + + if (cpi->roi.enabled && cpi->roi.skip[BACKGROUND_SEG_SKIP_ID] && + cpi->rc.frames_since_key > FRAMES_NO_SKIPPING_AFTER_KEY && + x->content_state_sb > kLowSadLowSumdiff) { + // For ROI with skip, force segment = 0 (no skip) over whole + // superblock to avoid artifacts if temporal change in source_sad is + // not 0. + int xi, yi; + const int bw = num_8x8_blocks_wide_lookup[BLOCK_64X64]; + const int bh = num_8x8_blocks_high_lookup[BLOCK_64X64]; + const int xmis = VPXMIN(cm->mi_cols - mi_col, bw); + const int ymis = VPXMIN(cm->mi_rows - mi_row, bh); + const int block_index = mi_row * cm->mi_cols + mi_col; + set_mode_info_offsets(cm, x, xd, mi_row, mi_col); + for (yi = 0; yi < ymis; yi++) + for (xi = 0; xi < xmis; xi++) { + int map_offset = block_index + yi * cm->mi_cols + xi; + cpi->segmentation_map[map_offset] = 0; + } + set_segment_index(cpi, x, mi_row, mi_col, BLOCK_64X64, 0); + seg_skip = 0; + } + if (seg_skip) { + partition_search_type = FIXED_PARTITION; + } + } + // Set the partition type of the 64X64 block switch (partition_search_type) { case VAR_BASED_PARTITION: // TODO(jingning, marpan): The mode decision and encoding process // support both intra and inter sub8x8 block coding for RTC mode. diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_encodemb.c b/media/libvpx/libvpx/vp9/encoder/vp9_encodemb.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_encodemb.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_encodemb.c @@ -364,64 +364,61 @@ #if CONFIG_VP9_HIGHBITDEPTH if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { switch (tx_size) { case TX_32X32: highbd_fdct32x32(x->use_lp32x32fdct, src_diff, coeff, diff_stride); - vp9_highbd_quantize_fp_32x32(coeff, 1024, x->skip_block, p->round_fp, - p->quant_fp, qcoeff, dqcoeff, pd->dequant, - eob, scan_order->scan, scan_order->iscan); + vp9_highbd_quantize_fp_32x32(coeff, 1024, p->round_fp, p->quant_fp, + qcoeff, dqcoeff, pd->dequant, eob, + scan_order->scan, scan_order->iscan); break; case TX_16X16: vpx_highbd_fdct16x16(src_diff, coeff, diff_stride); - vp9_highbd_quantize_fp(coeff, 256, x->skip_block, p->round_fp, - p->quant_fp, qcoeff, dqcoeff, pd->dequant, eob, - scan_order->scan, scan_order->iscan); + vp9_highbd_quantize_fp(coeff, 256, p->round_fp, p->quant_fp, qcoeff, + dqcoeff, pd->dequant, eob, scan_order->scan, + scan_order->iscan); break; case TX_8X8: vpx_highbd_fdct8x8(src_diff, coeff, diff_stride); - vp9_highbd_quantize_fp(coeff, 64, x->skip_block, p->round_fp, - p->quant_fp, qcoeff, dqcoeff, pd->dequant, eob, - scan_order->scan, scan_order->iscan); + vp9_highbd_quantize_fp(coeff, 64, p->round_fp, p->quant_fp, qcoeff, + dqcoeff, pd->dequant, eob, scan_order->scan, + scan_order->iscan); break; default: assert(tx_size == TX_4X4); x->fwd_txfm4x4(src_diff, coeff, diff_stride); - vp9_highbd_quantize_fp(coeff, 16, x->skip_block, p->round_fp, - p->quant_fp, qcoeff, dqcoeff, pd->dequant, eob, - scan_order->scan, scan_order->iscan); + vp9_highbd_quantize_fp(coeff, 16, p->round_fp, p->quant_fp, qcoeff, + dqcoeff, pd->dequant, eob, scan_order->scan, + scan_order->iscan); break; } return; } #endif // CONFIG_VP9_HIGHBITDEPTH switch (tx_size) { case TX_32X32: fdct32x32(x->use_lp32x32fdct, src_diff, coeff, diff_stride); - vp9_quantize_fp_32x32(coeff, 1024, x->skip_block, p->round_fp, - p->quant_fp, qcoeff, dqcoeff, pd->dequant, eob, - scan_order->scan, scan_order->iscan); + vp9_quantize_fp_32x32(coeff, 1024, p->round_fp, p->quant_fp, qcoeff, + dqcoeff, pd->dequant, eob, scan_order->scan, + scan_order->iscan); break; case TX_16X16: vpx_fdct16x16(src_diff, coeff, diff_stride); - vp9_quantize_fp(coeff, 256, x->skip_block, p->round_fp, p->quant_fp, - qcoeff, dqcoeff, pd->dequant, eob, scan_order->scan, - scan_order->iscan); + vp9_quantize_fp(coeff, 256, p->round_fp, p->quant_fp, qcoeff, dqcoeff, + pd->dequant, eob, scan_order->scan, scan_order->iscan); break; case TX_8X8: vpx_fdct8x8(src_diff, coeff, diff_stride); - vp9_quantize_fp(coeff, 64, x->skip_block, p->round_fp, p->quant_fp, - qcoeff, dqcoeff, pd->dequant, eob, scan_order->scan, - scan_order->iscan); + vp9_quantize_fp(coeff, 64, p->round_fp, p->quant_fp, qcoeff, dqcoeff, + pd->dequant, eob, scan_order->scan, scan_order->iscan); break; default: assert(tx_size == TX_4X4); x->fwd_txfm4x4(src_diff, coeff, diff_stride); - vp9_quantize_fp(coeff, 16, x->skip_block, p->round_fp, p->quant_fp, - qcoeff, dqcoeff, pd->dequant, eob, scan_order->scan, - scan_order->iscan); + vp9_quantize_fp(coeff, 16, p->round_fp, p->quant_fp, qcoeff, dqcoeff, + pd->dequant, eob, scan_order->scan, scan_order->iscan); break; } } void vp9_xform_quant_dc(MACROBLOCK *x, int plane, int block, int row, int col, @@ -442,59 +439,55 @@ #if CONFIG_VP9_HIGHBITDEPTH if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { switch (tx_size) { case TX_32X32: vpx_highbd_fdct32x32_1(src_diff, coeff, diff_stride); - vpx_highbd_quantize_dc_32x32(coeff, x->skip_block, p->round, - p->quant_fp[0], qcoeff, dqcoeff, - pd->dequant[0], eob); + vpx_highbd_quantize_dc_32x32(coeff, p->round, p->quant_fp[0], qcoeff, + dqcoeff, pd->dequant[0], eob); break; case TX_16X16: vpx_highbd_fdct16x16_1(src_diff, coeff, diff_stride); - vpx_highbd_quantize_dc(coeff, 256, x->skip_block, p->round, - p->quant_fp[0], qcoeff, dqcoeff, pd->dequant[0], - eob); + vpx_highbd_quantize_dc(coeff, 256, p->round, p->quant_fp[0], qcoeff, + dqcoeff, pd->dequant[0], eob); break; case TX_8X8: vpx_highbd_fdct8x8_1(src_diff, coeff, diff_stride); - vpx_highbd_quantize_dc(coeff, 64, x->skip_block, p->round, - p->quant_fp[0], qcoeff, dqcoeff, pd->dequant[0], - eob); + vpx_highbd_quantize_dc(coeff, 64, p->round, p->quant_fp[0], qcoeff, + dqcoeff, pd->dequant[0], eob); break; default: assert(tx_size == TX_4X4); x->fwd_txfm4x4(src_diff, coeff, diff_stride); - vpx_highbd_quantize_dc(coeff, 16, x->skip_block, p->round, - p->quant_fp[0], qcoeff, dqcoeff, pd->dequant[0], - eob); + vpx_highbd_quantize_dc(coeff, 16, p->round, p->quant_fp[0], qcoeff, + dqcoeff, pd->dequant[0], eob); break; } return; } #endif // CONFIG_VP9_HIGHBITDEPTH switch (tx_size) { case TX_32X32: vpx_fdct32x32_1(src_diff, coeff, diff_stride); - vpx_quantize_dc_32x32(coeff, x->skip_block, p->round, p->quant_fp[0], - qcoeff, dqcoeff, pd->dequant[0], eob); + vpx_quantize_dc_32x32(coeff, p->round, p->quant_fp[0], qcoeff, dqcoeff, + pd->dequant[0], eob); break; case TX_16X16: vpx_fdct16x16_1(src_diff, coeff, diff_stride); - vpx_quantize_dc(coeff, 256, x->skip_block, p->round, p->quant_fp[0], - qcoeff, dqcoeff, pd->dequant[0], eob); + vpx_quantize_dc(coeff, 256, p->round, p->quant_fp[0], qcoeff, dqcoeff, + pd->dequant[0], eob); break; case TX_8X8: vpx_fdct8x8_1(src_diff, coeff, diff_stride); - vpx_quantize_dc(coeff, 64, x->skip_block, p->round, p->quant_fp[0], - qcoeff, dqcoeff, pd->dequant[0], eob); + vpx_quantize_dc(coeff, 64, p->round, p->quant_fp[0], qcoeff, dqcoeff, + pd->dequant[0], eob); break; default: assert(tx_size == TX_4X4); x->fwd_txfm4x4(src_diff, coeff, diff_stride); - vpx_quantize_dc(coeff, 16, x->skip_block, p->round, p->quant_fp[0], - qcoeff, dqcoeff, pd->dequant[0], eob); + vpx_quantize_dc(coeff, 16, p->round, p->quant_fp[0], qcoeff, dqcoeff, + pd->dequant[0], eob); break; } } void vp9_xform_quant(MACROBLOCK *x, int plane, int block, int row, int col, @@ -516,68 +509,63 @@ #if CONFIG_VP9_HIGHBITDEPTH if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { switch (tx_size) { case TX_32X32: highbd_fdct32x32(x->use_lp32x32fdct, src_diff, coeff, diff_stride); - vpx_highbd_quantize_b_32x32(coeff, 1024, x->skip_block, p->zbin, - p->round, p->quant, p->quant_shift, qcoeff, - dqcoeff, pd->dequant, eob, scan_order->scan, - scan_order->iscan); + vpx_highbd_quantize_b_32x32( + coeff, 1024, p->zbin, p->round, p->quant, p->quant_shift, qcoeff, + dqcoeff, pd->dequant, eob, scan_order->scan, scan_order->iscan); break; case TX_16X16: vpx_highbd_fdct16x16(src_diff, coeff, diff_stride); - vpx_highbd_quantize_b(coeff, 256, x->skip_block, p->zbin, p->round, - p->quant, p->quant_shift, qcoeff, dqcoeff, - pd->dequant, eob, scan_order->scan, - scan_order->iscan); + vpx_highbd_quantize_b(coeff, 256, p->zbin, p->round, p->quant, + p->quant_shift, qcoeff, dqcoeff, pd->dequant, eob, + scan_order->scan, scan_order->iscan); break; case TX_8X8: vpx_highbd_fdct8x8(src_diff, coeff, diff_stride); - vpx_highbd_quantize_b(coeff, 64, x->skip_block, p->zbin, p->round, - p->quant, p->quant_shift, qcoeff, dqcoeff, - pd->dequant, eob, scan_order->scan, - scan_order->iscan); + vpx_highbd_quantize_b(coeff, 64, p->zbin, p->round, p->quant, + p->quant_shift, qcoeff, dqcoeff, pd->dequant, eob, + scan_order->scan, scan_order->iscan); break; default: assert(tx_size == TX_4X4); x->fwd_txfm4x4(src_diff, coeff, diff_stride); - vpx_highbd_quantize_b(coeff, 16, x->skip_block, p->zbin, p->round, - p->quant, p->quant_shift, qcoeff, dqcoeff, - pd->dequant, eob, scan_order->scan, - scan_order->iscan); + vpx_highbd_quantize_b(coeff, 16, p->zbin, p->round, p->quant, + p->quant_shift, qcoeff, dqcoeff, pd->dequant, eob, + scan_order->scan, scan_order->iscan); break; } return; } #endif // CONFIG_VP9_HIGHBITDEPTH switch (tx_size) { case TX_32X32: fdct32x32(x->use_lp32x32fdct, src_diff, coeff, diff_stride); - vpx_quantize_b_32x32(coeff, 1024, x->skip_block, p->zbin, p->round, - p->quant, p->quant_shift, qcoeff, dqcoeff, - pd->dequant, eob, scan_order->scan, - scan_order->iscan); + vpx_quantize_b_32x32(coeff, 1024, p->zbin, p->round, p->quant, + p->quant_shift, qcoeff, dqcoeff, pd->dequant, eob, + scan_order->scan, scan_order->iscan); break; case TX_16X16: vpx_fdct16x16(src_diff, coeff, diff_stride); - vpx_quantize_b(coeff, 256, x->skip_block, p->zbin, p->round, p->quant, - p->quant_shift, qcoeff, dqcoeff, pd->dequant, eob, - scan_order->scan, scan_order->iscan); + vpx_quantize_b(coeff, 256, p->zbin, p->round, p->quant, p->quant_shift, + qcoeff, dqcoeff, pd->dequant, eob, scan_order->scan, + scan_order->iscan); break; case TX_8X8: vpx_fdct8x8(src_diff, coeff, diff_stride); - vpx_quantize_b(coeff, 64, x->skip_block, p->zbin, p->round, p->quant, - p->quant_shift, qcoeff, dqcoeff, pd->dequant, eob, - scan_order->scan, scan_order->iscan); + vpx_quantize_b(coeff, 64, p->zbin, p->round, p->quant, p->quant_shift, + qcoeff, dqcoeff, pd->dequant, eob, scan_order->scan, + scan_order->iscan); break; default: assert(tx_size == TX_4X4); x->fwd_txfm4x4(src_diff, coeff, diff_stride); - vpx_quantize_b(coeff, 16, x->skip_block, p->zbin, p->round, p->quant, - p->quant_shift, qcoeff, dqcoeff, pd->dequant, eob, - scan_order->scan, scan_order->iscan); + vpx_quantize_b(coeff, 16, p->zbin, p->round, p->quant, p->quant_shift, + qcoeff, dqcoeff, pd->dequant, eob, scan_order->scan, + scan_order->iscan); break; } } static void encode_block(int plane, int block, int row, int col, @@ -867,14 +855,13 @@ case TX_32X32: if (!x->skip_recode) { vpx_highbd_subtract_block(32, 32, src_diff, diff_stride, src, src_stride, dst, dst_stride, xd->bd); highbd_fdct32x32(x->use_lp32x32fdct, src_diff, coeff, diff_stride); - vpx_highbd_quantize_b_32x32(coeff, 1024, x->skip_block, p->zbin, - p->round, p->quant, p->quant_shift, - qcoeff, dqcoeff, pd->dequant, eob, - scan_order->scan, scan_order->iscan); + vpx_highbd_quantize_b_32x32( + coeff, 1024, p->zbin, p->round, p->quant, p->quant_shift, qcoeff, + dqcoeff, pd->dequant, eob, scan_order->scan, scan_order->iscan); } if (args->enable_coeff_opt && !x->skip_recode) { *a = *l = vp9_optimize_b(x, plane, block, tx_size, entropy_ctx) > 0; } if (!x->skip_encode && *eob) { @@ -887,14 +874,13 @@ src_stride, dst, dst_stride, xd->bd); if (tx_type == DCT_DCT) vpx_highbd_fdct16x16(src_diff, coeff, diff_stride); else vp9_highbd_fht16x16(src_diff, coeff, diff_stride, tx_type); - vpx_highbd_quantize_b(coeff, 256, x->skip_block, p->zbin, p->round, - p->quant, p->quant_shift, qcoeff, dqcoeff, - pd->dequant, eob, scan_order->scan, - scan_order->iscan); + vpx_highbd_quantize_b(coeff, 256, p->zbin, p->round, p->quant, + p->quant_shift, qcoeff, dqcoeff, pd->dequant, + eob, scan_order->scan, scan_order->iscan); } if (args->enable_coeff_opt && !x->skip_recode) { *a = *l = vp9_optimize_b(x, plane, block, tx_size, entropy_ctx) > 0; } if (!x->skip_encode && *eob) { @@ -908,14 +894,13 @@ src_stride, dst, dst_stride, xd->bd); if (tx_type == DCT_DCT) vpx_highbd_fdct8x8(src_diff, coeff, diff_stride); else vp9_highbd_fht8x8(src_diff, coeff, diff_stride, tx_type); - vpx_highbd_quantize_b(coeff, 64, x->skip_block, p->zbin, p->round, - p->quant, p->quant_shift, qcoeff, dqcoeff, - pd->dequant, eob, scan_order->scan, - scan_order->iscan); + vpx_highbd_quantize_b(coeff, 64, p->zbin, p->round, p->quant, + p->quant_shift, qcoeff, dqcoeff, pd->dequant, + eob, scan_order->scan, scan_order->iscan); } if (args->enable_coeff_opt && !x->skip_recode) { *a = *l = vp9_optimize_b(x, plane, block, tx_size, entropy_ctx) > 0; } if (!x->skip_encode && *eob) { @@ -930,14 +915,13 @@ src_stride, dst, dst_stride, xd->bd); if (tx_type != DCT_DCT) vp9_highbd_fht4x4(src_diff, coeff, diff_stride, tx_type); else x->fwd_txfm4x4(src_diff, coeff, diff_stride); - vpx_highbd_quantize_b(coeff, 16, x->skip_block, p->zbin, p->round, - p->quant, p->quant_shift, qcoeff, dqcoeff, - pd->dequant, eob, scan_order->scan, - scan_order->iscan); + vpx_highbd_quantize_b(coeff, 16, p->zbin, p->round, p->quant, + p->quant_shift, qcoeff, dqcoeff, pd->dequant, + eob, scan_order->scan, scan_order->iscan); } if (args->enable_coeff_opt && !x->skip_recode) { *a = *l = vp9_optimize_b(x, plane, block, tx_size, entropy_ctx) > 0; } if (!x->skip_encode && *eob) { @@ -962,14 +946,13 @@ case TX_32X32: if (!x->skip_recode) { vpx_subtract_block(32, 32, src_diff, diff_stride, src, src_stride, dst, dst_stride); fdct32x32(x->use_lp32x32fdct, src_diff, coeff, diff_stride); - vpx_quantize_b_32x32(coeff, 1024, x->skip_block, p->zbin, p->round, - p->quant, p->quant_shift, qcoeff, dqcoeff, - pd->dequant, eob, scan_order->scan, - scan_order->iscan); + vpx_quantize_b_32x32(coeff, 1024, p->zbin, p->round, p->quant, + p->quant_shift, qcoeff, dqcoeff, pd->dequant, eob, + scan_order->scan, scan_order->iscan); } if (args->enable_coeff_opt && !x->skip_recode) { *a = *l = vp9_optimize_b(x, plane, block, tx_size, entropy_ctx) > 0; } if (!x->skip_encode && *eob) @@ -978,13 +961,13 @@ case TX_16X16: if (!x->skip_recode) { vpx_subtract_block(16, 16, src_diff, diff_stride, src, src_stride, dst, dst_stride); vp9_fht16x16(src_diff, coeff, diff_stride, tx_type); - vpx_quantize_b(coeff, 256, x->skip_block, p->zbin, p->round, p->quant, - p->quant_shift, qcoeff, dqcoeff, pd->dequant, eob, - scan_order->scan, scan_order->iscan); + vpx_quantize_b(coeff, 256, p->zbin, p->round, p->quant, p->quant_shift, + qcoeff, dqcoeff, pd->dequant, eob, scan_order->scan, + scan_order->iscan); } if (args->enable_coeff_opt && !x->skip_recode) { *a = *l = vp9_optimize_b(x, plane, block, tx_size, entropy_ctx) > 0; } if (!x->skip_encode && *eob) @@ -993,13 +976,13 @@ case TX_8X8: if (!x->skip_recode) { vpx_subtract_block(8, 8, src_diff, diff_stride, src, src_stride, dst, dst_stride); vp9_fht8x8(src_diff, coeff, diff_stride, tx_type); - vpx_quantize_b(coeff, 64, x->skip_block, p->zbin, p->round, p->quant, - p->quant_shift, qcoeff, dqcoeff, pd->dequant, eob, - scan_order->scan, scan_order->iscan); + vpx_quantize_b(coeff, 64, p->zbin, p->round, p->quant, p->quant_shift, + qcoeff, dqcoeff, pd->dequant, eob, scan_order->scan, + scan_order->iscan); } if (args->enable_coeff_opt && !x->skip_recode) { *a = *l = vp9_optimize_b(x, plane, block, tx_size, entropy_ctx) > 0; } if (!x->skip_encode && *eob) @@ -1012,13 +995,13 @@ dst_stride); if (tx_type != DCT_DCT) vp9_fht4x4(src_diff, coeff, diff_stride, tx_type); else x->fwd_txfm4x4(src_diff, coeff, diff_stride); - vpx_quantize_b(coeff, 16, x->skip_block, p->zbin, p->round, p->quant, - p->quant_shift, qcoeff, dqcoeff, pd->dequant, eob, - scan_order->scan, scan_order->iscan); + vpx_quantize_b(coeff, 16, p->zbin, p->round, p->quant, p->quant_shift, + qcoeff, dqcoeff, pd->dequant, eob, scan_order->scan, + scan_order->iscan); } if (args->enable_coeff_opt && !x->skip_recode) { *a = *l = vp9_optimize_b(x, plane, block, tx_size, entropy_ctx) > 0; } if (!x->skip_encode && *eob) { diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_encoder.h b/media/libvpx/libvpx/vp9/encoder/vp9_encoder.h --- a/media/libvpx/libvpx/vp9/encoder/vp9_encoder.h +++ b/media/libvpx/libvpx/vp9/encoder/vp9_encoder.h @@ -1194,10 +1194,17 @@ static INLINE int frame_is_kf_gf_arf(const VP9_COMP *cpi) { return frame_is_intra_only(&cpi->common) || cpi->refresh_alt_ref_frame || (cpi->refresh_golden_frame && !cpi->rc.is_src_frame_alt_ref); } +static INLINE int ref_frame_to_flag(int8_t ref_frame) { + static const int kVp9RefFlagList[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG, + VP9_ALT_FLAG }; + assert(ref_frame >= LAST_FRAME && ref_frame <= ALTREF_FRAME); + return kVp9RefFlagList[ref_frame]; +} + static INLINE int get_ref_frame_map_idx(const VP9_COMP *cpi, MV_REFERENCE_FRAME ref_frame) { if (ref_frame == LAST_FRAME) { return cpi->lst_fb_idx; } else if (ref_frame == GOLDEN_FRAME) { diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_encoder.c b/media/libvpx/libvpx/vp9/encoder/vp9_encoder.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_encoder.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_encoder.c @@ -23,10 +23,11 @@ #if CONFIG_INTERNAL_STATS #include "vpx_dsp/ssim.h" #endif #include "vpx_ports/mem.h" #include "vpx_ports/system_state.h" +#include "vpx_ports/vpx_once.h" #include "vpx_ports/vpx_timer.h" #if CONFIG_BITSTREAM_DEBUG || CONFIG_MISMATCH_DEBUG #include "vpx_util/vpx_debug_util.h" #endif // CONFIG_BITSTREAM_DEBUG || CONFIG_MISMATCH_DEBUG @@ -583,12 +584,10 @@ const int *delta_lf = roi->delta_lf; const int *skip = roi->skip; int ref_frame[8]; int internal_delta_q[MAX_SEGMENTS]; int i; - static const int flag_list[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG, - VP9_ALT_FLAG }; // TODO(jianj): Investigate why ROI not working in speed < 5 or in non // realtime mode. if (cpi->oxcf.mode != REALTIME || cpi->oxcf.speed < 5) return; if (!roi->enabled) return; @@ -616,20 +615,20 @@ vp9_enable_segfeature(seg, i, SEG_LVL_ALT_LF); vp9_set_segdata(seg, i, SEG_LVL_ALT_LF, delta_lf[i]); } if (skip[i] != 0) { vp9_enable_segfeature(seg, i, SEG_LVL_SKIP); - vp9_set_segdata(seg, i, SEG_LVL_SKIP, skip[i]); + vp9_set_segdata(seg, i, SEG_LVL_SKIP, 0); } if (ref_frame[i] >= 0) { int valid_ref = 1; // ALTREF is not used as reference for nonrd_pickmode with 0 lag. if (ref_frame[i] == ALTREF_FRAME && cpi->sf.use_nonrd_pick_mode) valid_ref = 0; // If GOLDEN is selected, make sure it's set as reference. if (ref_frame[i] == GOLDEN_FRAME && - !(cpi->ref_frame_flags & flag_list[ref_frame[i]])) { + !(cpi->ref_frame_flags & ref_frame_to_flag(ref_frame[i]))) { valid_ref = 0; } // GOLDEN was updated in previous encoded frame, so GOLDEN and LAST are // same reference. if (ref_frame[i] == GOLDEN_FRAME && cpi->rc.frames_since_golden == 0) @@ -927,27 +926,24 @@ cm->mi_grid_base = temp_base; cm->mi_grid_visible = cm->mi_grid_base + cm->mi_stride + 1; cm->prev_mi_grid_visible = cm->prev_mi_grid_base + cm->mi_stride + 1; } -void vp9_initialize_enc(void) { - static volatile int init_done = 0; - - if (!init_done) { - vp9_rtcd(); - vpx_dsp_rtcd(); - vpx_scale_rtcd(); - vp9_init_intra_predictors(); - vp9_init_me_luts(); - vp9_rc_init_minq_luts(); - vp9_entropy_mv_init(); +static void initialize_enc(void) { + vp9_rtcd(); + vpx_dsp_rtcd(); + vpx_scale_rtcd(); + vp9_init_intra_predictors(); + vp9_init_me_luts(); + vp9_rc_init_minq_luts(); + vp9_entropy_mv_init(); #if !CONFIG_REALTIME_ONLY - vp9_temporal_filter_init(); -#endif - init_done = 1; - } -} + vp9_temporal_filter_init(); +#endif +} + +void vp9_initialize_enc(void) { once(initialize_enc); } static void dealloc_compressor_data(VP9_COMP *cpi) { VP9_COMMON *const cm = &cpi->common; int i; @@ -1381,25 +1377,26 @@ vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR, "Failed to allocate scaled raw source frame buffer"); #endif } -static int alloc_context_buffers_ext(VP9_COMP *cpi) { +static void alloc_context_buffers_ext(VP9_COMP *cpi) { VP9_COMMON *cm = &cpi->common; int mi_size = cm->mi_cols * cm->mi_rows; - cpi->mbmi_ext_base = vpx_calloc(mi_size, sizeof(*cpi->mbmi_ext_base)); - if (!cpi->mbmi_ext_base) return 1; - - return 0; + CHECK_MEM_ERROR(cm, cpi->mbmi_ext_base, + vpx_calloc(mi_size, sizeof(*cpi->mbmi_ext_base))); } static void alloc_compressor_data(VP9_COMP *cpi) { VP9_COMMON *cm = &cpi->common; int sb_rows; - vp9_alloc_context_buffers(cm, cm->width, cm->height); + if (vp9_alloc_context_buffers(cm, cm->width, cm->height)) { + vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR, + "Failed to allocate context buffers"); + } alloc_context_buffers_ext(cpi); vpx_free(cpi->tile_tok[0][0]); @@ -1571,19 +1568,17 @@ rc->bits_off_target = VPXMIN(rc->bits_off_target, rc->maximum_buffer_size); rc->buffer_level = VPXMIN(rc->buffer_level, rc->maximum_buffer_size); } #if CONFIG_VP9_HIGHBITDEPTH -// TODO(angiebird): make sdx8f available for highbitdepth if needed #define HIGHBD_BFP(BT, SDF, SDAF, VF, SVF, SVAF, SDX4DF) \ cpi->fn_ptr[BT].sdf = SDF; \ cpi->fn_ptr[BT].sdaf = SDAF; \ cpi->fn_ptr[BT].vf = VF; \ cpi->fn_ptr[BT].svf = SVF; \ cpi->fn_ptr[BT].svaf = SVAF; \ - cpi->fn_ptr[BT].sdx4df = SDX4DF; \ - cpi->fn_ptr[BT].sdx8f = NULL; + cpi->fn_ptr[BT].sdx4df = SDX4DF; #define MAKE_BFP_SAD_WRAPPER(fnname) \ static unsigned int fnname##_bits8(const uint8_t *src_ptr, \ int source_stride, \ const uint8_t *ref_ptr, int ref_stride) { \ @@ -2221,20 +2216,22 @@ unsigned int *subsampling_x, unsigned int *subsampling_y) { switch (fmt) { case VPX_IMG_FMT_I420: case VPX_IMG_FMT_YV12: + case VPX_IMG_FMT_NV12: case VPX_IMG_FMT_I422: case VPX_IMG_FMT_I42016: case VPX_IMG_FMT_I42216: *subsampling_x = 1; break; default: *subsampling_x = 0; break; } switch (fmt) { case VPX_IMG_FMT_I420: case VPX_IMG_FMT_I440: case VPX_IMG_FMT_YV12: + case VPX_IMG_FMT_NV12: case VPX_IMG_FMT_I42016: case VPX_IMG_FMT_I44016: *subsampling_y = 1; break; default: *subsampling_y = 0; break; } } @@ -2561,71 +2558,65 @@ // Allocate memory to store variances for a frame. CHECK_MEM_ERROR(cm, cpi->source_diff_var, vpx_calloc(cm->MBs, sizeof(diff))); cpi->source_var_thresh = 0; cpi->frames_till_next_var_check = 0; -#define BFP(BT, SDF, SDAF, VF, SVF, SVAF, SDX4DF, SDX8F) \ - cpi->fn_ptr[BT].sdf = SDF; \ - cpi->fn_ptr[BT].sdaf = SDAF; \ - cpi->fn_ptr[BT].vf = VF; \ - cpi->fn_ptr[BT].svf = SVF; \ - cpi->fn_ptr[BT].svaf = SVAF; \ - cpi->fn_ptr[BT].sdx4df = SDX4DF; \ - cpi->fn_ptr[BT].sdx8f = SDX8F; - - // TODO(angiebird): make sdx8f available for every block size +#define BFP(BT, SDF, SDAF, VF, SVF, SVAF, SDX4DF) \ + cpi->fn_ptr[BT].sdf = SDF; \ + cpi->fn_ptr[BT].sdaf = SDAF; \ + cpi->fn_ptr[BT].vf = VF; \ + cpi->fn_ptr[BT].svf = SVF; \ + cpi->fn_ptr[BT].svaf = SVAF; \ + cpi->fn_ptr[BT].sdx4df = SDX4DF; + BFP(BLOCK_32X16, vpx_sad32x16, vpx_sad32x16_avg, vpx_variance32x16, vpx_sub_pixel_variance32x16, vpx_sub_pixel_avg_variance32x16, - vpx_sad32x16x4d, NULL) + vpx_sad32x16x4d) BFP(BLOCK_16X32, vpx_sad16x32, vpx_sad16x32_avg, vpx_variance16x32, vpx_sub_pixel_variance16x32, vpx_sub_pixel_avg_variance16x32, - vpx_sad16x32x4d, NULL) + vpx_sad16x32x4d) BFP(BLOCK_64X32, vpx_sad64x32, vpx_sad64x32_avg, vpx_variance64x32, vpx_sub_pixel_variance64x32, vpx_sub_pixel_avg_variance64x32, - vpx_sad64x32x4d, NULL) + vpx_sad64x32x4d) BFP(BLOCK_32X64, vpx_sad32x64, vpx_sad32x64_avg, vpx_variance32x64, vpx_sub_pixel_variance32x64, vpx_sub_pixel_avg_variance32x64, - vpx_sad32x64x4d, NULL) + vpx_sad32x64x4d) BFP(BLOCK_32X32, vpx_sad32x32, vpx_sad32x32_avg, vpx_variance32x32, vpx_sub_pixel_variance32x32, vpx_sub_pixel_avg_variance32x32, - vpx_sad32x32x4d, vpx_sad32x32x8) + vpx_sad32x32x4d) BFP(BLOCK_64X64, vpx_sad64x64, vpx_sad64x64_avg, vpx_variance64x64, vpx_sub_pixel_variance64x64, vpx_sub_pixel_avg_variance64x64, - vpx_sad64x64x4d, NULL) + vpx_sad64x64x4d) BFP(BLOCK_16X16, vpx_sad16x16, vpx_sad16x16_avg, vpx_variance16x16, vpx_sub_pixel_variance16x16, vpx_sub_pixel_avg_variance16x16, - vpx_sad16x16x4d, vpx_sad16x16x8) + vpx_sad16x16x4d) BFP(BLOCK_16X8, vpx_sad16x8, vpx_sad16x8_avg, vpx_variance16x8, vpx_sub_pixel_variance16x8, vpx_sub_pixel_avg_variance16x8, - vpx_sad16x8x4d, vpx_sad16x8x8) + vpx_sad16x8x4d) BFP(BLOCK_8X16, vpx_sad8x16, vpx_sad8x16_avg, vpx_variance8x16, vpx_sub_pixel_variance8x16, vpx_sub_pixel_avg_variance8x16, - vpx_sad8x16x4d, vpx_sad8x16x8) + vpx_sad8x16x4d) BFP(BLOCK_8X8, vpx_sad8x8, vpx_sad8x8_avg, vpx_variance8x8, - vpx_sub_pixel_variance8x8, vpx_sub_pixel_avg_variance8x8, vpx_sad8x8x4d, - vpx_sad8x8x8) + vpx_sub_pixel_variance8x8, vpx_sub_pixel_avg_variance8x8, vpx_sad8x8x4d) BFP(BLOCK_8X4, vpx_sad8x4, vpx_sad8x4_avg, vpx_variance8x4, - vpx_sub_pixel_variance8x4, vpx_sub_pixel_avg_variance8x4, vpx_sad8x4x4d, - NULL) + vpx_sub_pixel_variance8x4, vpx_sub_pixel_avg_variance8x4, vpx_sad8x4x4d) BFP(BLOCK_4X8, vpx_sad4x8, vpx_sad4x8_avg, vpx_variance4x8, - vpx_sub_pixel_variance4x8, vpx_sub_pixel_avg_variance4x8, vpx_sad4x8x4d, - NULL) + vpx_sub_pixel_variance4x8, vpx_sub_pixel_avg_variance4x8, vpx_sad4x8x4d) BFP(BLOCK_4X4, vpx_sad4x4, vpx_sad4x4_avg, vpx_variance4x4, - vpx_sub_pixel_variance4x4, vpx_sub_pixel_avg_variance4x4, vpx_sad4x4x4d, - vpx_sad4x4x8) + vpx_sub_pixel_variance4x4, vpx_sub_pixel_avg_variance4x4, vpx_sad4x4x4d) #if CONFIG_VP9_HIGHBITDEPTH highbd_set_var_fns(cpi); #endif @@ -2674,11 +2665,10 @@ static void free_tpl_buffer(VP9_COMP *cpi); void vp9_remove_compressor(VP9_COMP *cpi) { VP9_COMMON *cm; unsigned int i; - int t; if (!cpi) return; #if CONFIG_INTERNAL_STATS vpx_free(cpi->ssim_vars); @@ -2787,32 +2777,14 @@ vpx_free(cpi->kmeans_data_arr); } free_tpl_buffer(cpi); - for (t = 0; t < cpi->num_workers; ++t) { - VPxWorker *const worker = &cpi->workers[t]; - EncWorkerData *const thread_data = &cpi->tile_thr_data[t]; - - // Deallocate allocated threads. - vpx_get_worker_interface()->end(worker); - - // Deallocate allocated thread data. - if (t < cpi->num_workers - 1) { - vpx_free(thread_data->td->counts); - vp9_free_pc_tree(thread_data->td); - vpx_free(thread_data->td); - } - } - vpx_free(cpi->tile_thr_data); - vpx_free(cpi->workers); + vp9_loop_filter_dealloc(&cpi->lf_row_sync); + vp9_bitstream_encode_tiles_buffer_dealloc(cpi); vp9_row_mt_mem_dealloc(cpi); - - if (cpi->num_workers > 1) { - vp9_loop_filter_dealloc(&cpi->lf_row_sync); - vp9_bitstream_encode_tiles_buffer_dealloc(cpi); - } + vp9_encode_free_mt_data(cpi); #if !CONFIG_REALTIME_ONLY vp9_alt_ref_aq_destroy(cpi->alt_ref_aq); #endif @@ -3710,13 +3682,13 @@ case 4: case 5: l = 100; break; case 6: l = 150; break; } if (!cpi->common.postproc_state.limits) { - cpi->common.postproc_state.limits = - vpx_calloc(cpi->un_scaled_source->y_width, - sizeof(*cpi->common.postproc_state.limits)); + CHECK_MEM_ERROR(cm, cpi->common.postproc_state.limits, + vpx_calloc(cpi->un_scaled_source->y_width, + sizeof(*cpi->common.postproc_state.limits))); } vp9_denoise(&cpi->common, cpi->Source, cpi->Source, l, cpi->common.postproc_state.limits); } #endif // CONFIG_VP9_POSTPROC @@ -4135,15 +4107,26 @@ // it may be pretty bad for rate-control, // and I should handle it somehow vp9_alt_ref_aq_setup_map(cpi->alt_ref_aq, cpi); } else { #endif - if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ) { + // If ROI is enabled and skip feature is used for segmentation, apply cyclic + // refresh but not apply ROI for skip for the first 20 frames (defined by + // FRAMES_NO_SKIPPING_AFTER_KEY) after key frame to improve quality. + if (cpi->roi.enabled && !frame_is_intra_only(cm)) { + if (cpi->roi.skip[BACKGROUND_SEG_SKIP_ID]) { + if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ) + vp9_cyclic_refresh_setup(cpi); + if (cpi->rc.frames_since_key > FRAMES_NO_SKIPPING_AFTER_KEY) + apply_roi_map(cpi); + } else { + apply_roi_map(cpi); + } + } else if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ) { vp9_cyclic_refresh_setup(cpi); - } else if (cpi->roi.enabled && !frame_is_intra_only(cm)) { - apply_roi_map(cpi); - } + } + #if !CONFIG_REALTIME_ONLY } #endif apply_active_map(cpi); @@ -6628,23 +6611,26 @@ const scan_order *const scan_order = &vp9_default_scan_orders[tx_size]; uint16_t eob; int pix_num = 1 << num_pels_log2_lookup[txsize_to_bsize[tx_size]]; const int shift = tx_size == TX_32X32 ? 0 : 2; + // skip block condition should be handled before this is called. + assert(!x->skip_block); + #if CONFIG_VP9_HIGHBITDEPTH if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { - vp9_highbd_quantize_fp_32x32(coeff, pix_num, x->skip_block, p->round_fp, - p->quant_fp, qcoeff, dqcoeff, pd->dequant, - &eob, scan_order->scan, scan_order->iscan); + vp9_highbd_quantize_fp_32x32(coeff, pix_num, p->round_fp, p->quant_fp, + qcoeff, dqcoeff, pd->dequant, &eob, + scan_order->scan, scan_order->iscan); } else { - vp9_quantize_fp_32x32(coeff, pix_num, x->skip_block, p->round_fp, - p->quant_fp, qcoeff, dqcoeff, pd->dequant, &eob, - scan_order->scan, scan_order->iscan); + vp9_quantize_fp_32x32(coeff, pix_num, p->round_fp, p->quant_fp, qcoeff, + dqcoeff, pd->dequant, &eob, scan_order->scan, + scan_order->iscan); } #else - vp9_quantize_fp_32x32(coeff, pix_num, x->skip_block, p->round_fp, p->quant_fp, - qcoeff, dqcoeff, pd->dequant, &eob, scan_order->scan, + vp9_quantize_fp_32x32(coeff, pix_num, p->round_fp, p->quant_fp, qcoeff, + dqcoeff, pd->dequant, &eob, scan_order->scan, scan_order->iscan); #endif // CONFIG_VP9_HIGHBITDEPTH *recon_error = vp9_block_error(coeff, dqcoeff, pix_num, sse) >> shift; *recon_error = VPXMAX(*recon_error, 1); diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_ethread.h b/media/libvpx/libvpx/vp9/encoder/vp9_ethread.h --- a/media/libvpx/libvpx/vp9/encoder/vp9_ethread.h +++ b/media/libvpx/libvpx/vp9/encoder/vp9_ethread.h @@ -40,10 +40,15 @@ int *cur_col; int sync_range; int rows; } VP9RowMTSync; +// Frees EncWorkerData related allocations made by vp9_encode_*_mt(). +// row_mt specific data is freed with vp9_row_mt_mem_dealloc() and is not +// called by this function. +void vp9_encode_free_mt_data(struct VP9_COMP *cpi); + void vp9_encode_tiles_mt(struct VP9_COMP *cpi); void vp9_encode_tiles_row_mt(struct VP9_COMP *cpi); void vp9_encode_fp_row_mt(struct VP9_COMP *cpi); diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_ethread.c b/media/libvpx/libvpx/vp9/encoder/vp9_ethread.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_ethread.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_ethread.c @@ -6,10 +6,12 @@ * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ +#include "vp9/common/vp9_thread_common.h" +#include "vp9/encoder/vp9_bitstream.h" #include "vp9/encoder/vp9_encodeframe.h" #include "vp9/encoder/vp9_encoder.h" #include "vp9/encoder/vp9_ethread.h" #include "vp9/encoder/vp9_firstpass.h" #include "vp9/encoder/vp9_multi_thread.h" @@ -77,64 +79,63 @@ static void create_enc_workers(VP9_COMP *cpi, int num_workers) { VP9_COMMON *const cm = &cpi->common; const VPxWorkerInterface *const winterface = vpx_get_worker_interface(); int i; - - // Only run once to create threads and allocate thread data. - if (cpi->num_workers == 0) { - int allocated_workers = num_workers; - - // While using SVC, we need to allocate threads according to the highest - // resolution. When row based multithreading is enabled, it is OK to - // allocate more threads than the number of max tile columns. - if (cpi->use_svc && !cpi->row_mt) { - int max_tile_cols = get_max_tile_cols(cpi); - allocated_workers = VPXMIN(cpi->oxcf.max_threads, max_tile_cols); - } + // While using SVC, we need to allocate threads according to the highest + // resolution. When row based multithreading is enabled, it is OK to + // allocate more threads than the number of max tile columns. + if (cpi->use_svc && !cpi->row_mt) { + int max_tile_cols = get_max_tile_cols(cpi); + num_workers = VPXMIN(cpi->oxcf.max_threads, max_tile_cols); + } + assert(num_workers > 0); + if (num_workers == cpi->num_workers) return; + vp9_loop_filter_dealloc(&cpi->lf_row_sync); + vp9_bitstream_encode_tiles_buffer_dealloc(cpi); + vp9_encode_free_mt_data(cpi); - CHECK_MEM_ERROR(cm, cpi->workers, - vpx_malloc(allocated_workers * sizeof(*cpi->workers))); + CHECK_MEM_ERROR(cm, cpi->workers, + vpx_malloc(num_workers * sizeof(*cpi->workers))); - CHECK_MEM_ERROR(cm, cpi->tile_thr_data, - vpx_calloc(allocated_workers, sizeof(*cpi->tile_thr_data))); + CHECK_MEM_ERROR(cm, cpi->tile_thr_data, + vpx_calloc(num_workers, sizeof(*cpi->tile_thr_data))); - for (i = 0; i < allocated_workers; i++) { - VPxWorker *const worker = &cpi->workers[i]; - EncWorkerData *thread_data = &cpi->tile_thr_data[i]; + for (i = 0; i < num_workers; i++) { + VPxWorker *const worker = &cpi->workers[i]; + EncWorkerData *thread_data = &cpi->tile_thr_data[i]; - ++cpi->num_workers; - winterface->init(worker); + ++cpi->num_workers; + winterface->init(worker); - if (i < allocated_workers - 1) { - thread_data->cpi = cpi; + if (i < num_workers - 1) { + thread_data->cpi = cpi; - // Allocate thread data. - CHECK_MEM_ERROR(cm, thread_data->td, - vpx_memalign(32, sizeof(*thread_data->td))); - vp9_zero(*thread_data->td); + // Allocate thread data. + CHECK_MEM_ERROR(cm, thread_data->td, + vpx_memalign(32, sizeof(*thread_data->td))); + vp9_zero(*thread_data->td); - // Set up pc_tree. - thread_data->td->leaf_tree = NULL; - thread_data->td->pc_tree = NULL; - vp9_setup_pc_tree(cm, thread_data->td); + // Set up pc_tree. + thread_data->td->leaf_tree = NULL; + thread_data->td->pc_tree = NULL; + vp9_setup_pc_tree(cm, thread_data->td); - // Allocate frame counters in thread data. - CHECK_MEM_ERROR(cm, thread_data->td->counts, - vpx_calloc(1, sizeof(*thread_data->td->counts))); + // Allocate frame counters in thread data. + CHECK_MEM_ERROR(cm, thread_data->td->counts, + vpx_calloc(1, sizeof(*thread_data->td->counts))); - // Create threads - if (!winterface->reset(worker)) - vpx_internal_error(&cm->error, VPX_CODEC_ERROR, - "Tile encoder thread creation failed"); - } else { - // Main thread acts as a worker and uses the thread data in cpi. - thread_data->cpi = cpi; - thread_data->td = &cpi->td; - } - winterface->sync(worker); + // Create threads + if (!winterface->reset(worker)) + vpx_internal_error(&cm->error, VPX_CODEC_ERROR, + "Tile encoder thread creation failed"); + } else { + // Main thread acts as a worker and uses the thread data in cpi. + thread_data->cpi = cpi; + thread_data->td = &cpi->td; } + winterface->sync(worker); } } static void launch_enc_workers(VP9_COMP *cpi, VPxWorkerHook hook, void *data2, int num_workers) { @@ -167,10 +168,31 @@ VPxWorker *const worker = &cpi->workers[i]; winterface->sync(worker); } } +void vp9_encode_free_mt_data(struct VP9_COMP *cpi) { + int t; + for (t = 0; t < cpi->num_workers; ++t) { + VPxWorker *const worker = &cpi->workers[t]; + EncWorkerData *const thread_data = &cpi->tile_thr_data[t]; + + // Deallocate allocated threads. + vpx_get_worker_interface()->end(worker); + + // Deallocate allocated thread data. + if (t < cpi->num_workers - 1) { + vpx_free(thread_data->td->counts); + vp9_free_pc_tree(thread_data->td); + vpx_free(thread_data->td); + } + } + vpx_free(cpi->tile_thr_data); + vpx_free(cpi->workers); + cpi->num_workers = 0; +} + void vp9_encode_tiles_mt(VP9_COMP *cpi) { VP9_COMMON *const cm = &cpi->common; const int tile_cols = 1 << cm->log2_tile_cols; const int num_workers = VPXMIN(cpi->oxcf.max_threads, tile_cols); int i; diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_mbgraph.c b/media/libvpx/libvpx/vp9/encoder/vp9_mbgraph.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_mbgraph.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_mbgraph.c @@ -217,11 +217,11 @@ MACROBLOCK *const x = &cpi->td.mb; MACROBLOCKD *const xd = &x->e_mbd; VP9_COMMON *const cm = &cpi->common; int mb_col, mb_row, offset = 0; - int mb_y_offset = 0, arf_y_offset = 0, gld_y_offset = 0; + int mb_y_offset = 0; MV gld_top_mv = { 0, 0 }; MODE_INFO mi_local; MODE_INFO mi_above, mi_left; vp9_zero(mi_local); @@ -241,12 +241,10 @@ mi_local.ref_frame[1] = NONE; for (mb_row = 0; mb_row < cm->mb_rows; mb_row++) { MV gld_left_mv = gld_top_mv; int mb_y_in_offset = mb_y_offset; - int arf_y_in_offset = arf_y_offset; - int gld_y_in_offset = gld_y_offset; // Set up limit values for motion vectors to prevent them extending outside // the UMV borders. x->mv_limits.col_min = -BORDER_MV_PIXELS_B16; x->mv_limits.col_max = (cm->mb_cols - 1) * 8 + BORDER_MV_PIXELS_B16; @@ -264,22 +262,18 @@ } // Signal to vp9_predict_intra_block() that left is available xd->left_mi = &mi_left; mb_y_in_offset += 16; - gld_y_in_offset += 16; - arf_y_in_offset += 16; x->mv_limits.col_min -= 16; x->mv_limits.col_max -= 16; } // Signal to vp9_predict_intra_block() that above is available xd->above_mi = &mi_above; mb_y_offset += buf->y_stride * 16; - gld_y_offset += golden_ref->y_stride * 16; - if (alt_ref) arf_y_offset += alt_ref->y_stride * 16; x->mv_limits.row_min -= 16; x->mv_limits.row_max -= 16; offset += cm->mb_cols; } } diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_mcomp.h b/media/libvpx/libvpx/vp9/encoder/vp9_mcomp.h --- a/media/libvpx/libvpx/vp9/encoder/vp9_mcomp.h +++ b/media/libvpx/libvpx/vp9/encoder/vp9_mcomp.h @@ -91,20 +91,10 @@ extern fractional_mv_step_fp vp9_find_best_sub_pixel_tree_pruned_evenmore; extern fractional_mv_step_fp vp9_skip_sub_pixel_tree; extern fractional_mv_step_fp vp9_return_max_sub_pixel_mv; extern fractional_mv_step_fp vp9_return_min_sub_pixel_mv; -typedef int (*vp9_full_search_fn_t)(const MACROBLOCK *x, const MV *ref_mv, - int sad_per_bit, int distance, - const vp9_variance_fn_ptr_t *fn_ptr, - const MV *center_mv, MV *best_mv); - -typedef int (*vp9_refining_search_fn_t)(const MACROBLOCK *x, MV *ref_mv, - int sad_per_bit, int distance, - const vp9_variance_fn_ptr_t *fn_ptr, - const MV *center_mv); - typedef int (*vp9_diamond_search_fn_t)( const MACROBLOCK *x, const search_site_config *cfg, MV *ref_mv, MV *best_mv, int search_param, int sad_per_bit, int *num00, const vp9_variance_fn_ptr_t *fn_ptr, const MV *center_mv); diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_mcomp.c b/media/libvpx/libvpx/vp9/encoder/vp9_mcomp.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_mcomp.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_mcomp.c @@ -157,63 +157,67 @@ return &buf[(r >> 3) * stride + (c >> 3)]; } #if CONFIG_VP9_HIGHBITDEPTH /* checks if (r, c) has better score than previous best */ -#define CHECK_BETTER(v, r, c) \ - if (c >= minc && c <= maxc && r >= minr && r <= maxr) { \ - int64_t tmpmse; \ - const MV mv = { r, c }; \ - const MV ref_mv = { rr, rc }; \ - if (second_pred == NULL) { \ - thismse = vfp->svf(pre(y, y_stride, r, c), y_stride, sp(c), sp(r), z, \ - src_stride, &sse); \ - } else { \ - thismse = vfp->svaf(pre(y, y_stride, r, c), y_stride, sp(c), sp(r), z, \ - src_stride, &sse, second_pred); \ - } \ - tmpmse = thismse; \ - tmpmse += mv_err_cost(&mv, &ref_mv, mvjcost, mvcost, error_per_bit); \ - if (tmpmse >= INT_MAX) { \ - v = INT_MAX; \ - } else if ((v = (uint32_t)tmpmse) < besterr) { \ - besterr = v; \ - br = r; \ - bc = c; \ - *distortion = thismse; \ - *sse1 = sse; \ - } \ - } else { \ - v = INT_MAX; \ - } +#define CHECK_BETTER(v, r, c) \ + do { \ + if (c >= minc && c <= maxc && r >= minr && r <= maxr) { \ + int64_t tmpmse; \ + const MV mv = { r, c }; \ + const MV ref_mv = { rr, rc }; \ + if (second_pred == NULL) { \ + thismse = vfp->svf(pre(y, y_stride, r, c), y_stride, sp(c), sp(r), z, \ + src_stride, &sse); \ + } else { \ + thismse = vfp->svaf(pre(y, y_stride, r, c), y_stride, sp(c), sp(r), z, \ + src_stride, &sse, second_pred); \ + } \ + tmpmse = thismse; \ + tmpmse += mv_err_cost(&mv, &ref_mv, mvjcost, mvcost, error_per_bit); \ + if (tmpmse >= INT_MAX) { \ + v = INT_MAX; \ + } else if ((v = (uint32_t)tmpmse) < besterr) { \ + besterr = v; \ + br = r; \ + bc = c; \ + *distortion = thismse; \ + *sse1 = sse; \ + } \ + } else { \ + v = INT_MAX; \ + } \ + } while (0) #else /* checks if (r, c) has better score than previous best */ -#define CHECK_BETTER(v, r, c) \ - if (c >= minc && c <= maxc && r >= minr && r <= maxr) { \ - const MV mv = { r, c }; \ - const MV ref_mv = { rr, rc }; \ - if (second_pred == NULL) \ - thismse = vfp->svf(pre(y, y_stride, r, c), y_stride, sp(c), sp(r), z, \ - src_stride, &sse); \ - else \ - thismse = vfp->svaf(pre(y, y_stride, r, c), y_stride, sp(c), sp(r), z, \ - src_stride, &sse, second_pred); \ - if ((v = mv_err_cost(&mv, &ref_mv, mvjcost, mvcost, error_per_bit) + \ - thismse) < besterr) { \ - besterr = v; \ - br = r; \ - bc = c; \ - *distortion = thismse; \ - *sse1 = sse; \ - } \ - } else { \ - v = INT_MAX; \ - } +#define CHECK_BETTER(v, r, c) \ + do { \ + if (c >= minc && c <= maxc && r >= minr && r <= maxr) { \ + const MV mv = { r, c }; \ + const MV ref_mv = { rr, rc }; \ + if (second_pred == NULL) \ + thismse = vfp->svf(pre(y, y_stride, r, c), y_stride, sp(c), sp(r), z, \ + src_stride, &sse); \ + else \ + thismse = vfp->svaf(pre(y, y_stride, r, c), y_stride, sp(c), sp(r), z, \ + src_stride, &sse, second_pred); \ + if ((v = mv_err_cost(&mv, &ref_mv, mvjcost, mvcost, error_per_bit) + \ + thismse) < besterr) { \ + besterr = v; \ + br = r; \ + bc = c; \ + *distortion = thismse; \ + *sse1 = sse; \ + } \ + } else { \ + v = INT_MAX; \ + } \ + } while (0) #endif #define FIRST_LEVEL_CHECKS \ - { \ + do { \ unsigned int left, right, up, down, diag; \ CHECK_BETTER(left, tr, tc - hstep); \ CHECK_BETTER(right, tr, tc + hstep); \ CHECK_BETTER(up, tr - hstep, tc); \ CHECK_BETTER(down, tr + hstep, tc); \ @@ -222,14 +226,14 @@ case 0: CHECK_BETTER(diag, tr - hstep, tc - hstep); break; \ case 1: CHECK_BETTER(diag, tr - hstep, tc + hstep); break; \ case 2: CHECK_BETTER(diag, tr + hstep, tc - hstep); break; \ case 3: CHECK_BETTER(diag, tr + hstep, tc + hstep); break; \ } \ - } + } while (0) #define SECOND_LEVEL_CHECKS \ - { \ + do { \ int kr, kc; \ unsigned int second; \ if (tr != br && tc != bc) { \ kr = br - tr; \ kc = bc - tc; \ @@ -254,11 +258,11 @@ case 2: CHECK_BETTER(second, tr + kr, tc + hstep); break; \ case 1: \ case 3: CHECK_BETTER(second, tr + kr, tc - hstep); break; \ } \ } \ - } + } while (0) #define SETUP_SUBPEL_SEARCH \ const uint8_t *const z = x->plane[0].src.buf; \ const int src_stride = x->plane[0].src.stride; \ const MACROBLOCKD *xd = &x->e_mbd; \ @@ -288,11 +292,11 @@ maxc = subpel_mv_limits.col_max; \ minr = subpel_mv_limits.row_min; \ maxr = subpel_mv_limits.row_max; \ \ bestmv->row *= 8; \ - bestmv->col *= 8; + bestmv->col *= 8 static unsigned int setup_center_error( const MACROBLOCKD *xd, const MV *bestmv, const MV *ref_mv, int error_per_bit, const vp9_variance_fn_ptr_t *vfp, const uint8_t *const src, const int src_stride, const uint8_t *const y, @@ -676,52 +680,56 @@ } // TODO(yunqing): this part can be further refactored. #if CONFIG_VP9_HIGHBITDEPTH /* checks if (r, c) has better score than previous best */ -#define CHECK_BETTER1(v, r, c) \ - if (c >= minc && c <= maxc && r >= minr && r <= maxr) { \ - int64_t tmpmse; \ - const MV mv = { r, c }; \ - const MV ref_mv = { rr, rc }; \ - thismse = \ - accurate_sub_pel_search(xd, &mv, x->me_sf, kernel, vfp, z, src_stride, \ - y, y_stride, second_pred, w, h, &sse); \ - tmpmse = thismse; \ - tmpmse += mv_err_cost(&mv, &ref_mv, mvjcost, mvcost, error_per_bit); \ - if (tmpmse >= INT_MAX) { \ - v = INT_MAX; \ - } else if ((v = (uint32_t)tmpmse) < besterr) { \ - besterr = v; \ - br = r; \ - bc = c; \ - *distortion = thismse; \ - *sse1 = sse; \ - } \ - } else { \ - v = INT_MAX; \ - } +#define CHECK_BETTER1(v, r, c) \ + do { \ + if (c >= minc && c <= maxc && r >= minr && r <= maxr) { \ + int64_t tmpmse; \ + const MV mv = { r, c }; \ + const MV ref_mv = { rr, rc }; \ + thismse = accurate_sub_pel_search(xd, &mv, x->me_sf, kernel, vfp, z, \ + src_stride, y, y_stride, second_pred, \ + w, h, &sse); \ + tmpmse = thismse; \ + tmpmse += mv_err_cost(&mv, &ref_mv, mvjcost, mvcost, error_per_bit); \ + if (tmpmse >= INT_MAX) { \ + v = INT_MAX; \ + } else if ((v = (uint32_t)tmpmse) < besterr) { \ + besterr = v; \ + br = r; \ + bc = c; \ + *distortion = thismse; \ + *sse1 = sse; \ + } \ + } else { \ + v = INT_MAX; \ + } \ + } while (0) #else /* checks if (r, c) has better score than previous best */ -#define CHECK_BETTER1(v, r, c) \ - if (c >= minc && c <= maxc && r >= minr && r <= maxr) { \ - const MV mv = { r, c }; \ - const MV ref_mv = { rr, rc }; \ - thismse = \ - accurate_sub_pel_search(xd, &mv, x->me_sf, kernel, vfp, z, src_stride, \ - y, y_stride, second_pred, w, h, &sse); \ - if ((v = mv_err_cost(&mv, &ref_mv, mvjcost, mvcost, error_per_bit) + \ - thismse) < besterr) { \ - besterr = v; \ - br = r; \ - bc = c; \ - *distortion = thismse; \ - *sse1 = sse; \ - } \ - } else { \ - v = INT_MAX; \ - } +#define CHECK_BETTER1(v, r, c) \ + do { \ + if (c >= minc && c <= maxc && r >= minr && r <= maxr) { \ + const MV mv = { r, c }; \ + const MV ref_mv = { rr, rc }; \ + thismse = accurate_sub_pel_search(xd, &mv, x->me_sf, kernel, vfp, z, \ + src_stride, y, y_stride, second_pred, \ + w, h, &sse); \ + if ((v = mv_err_cost(&mv, &ref_mv, mvjcost, mvcost, error_per_bit) + \ + thismse) < besterr) { \ + besterr = v; \ + br = r; \ + bc = c; \ + *distortion = thismse; \ + *sse1 = sse; \ + } \ + } else { \ + v = INT_MAX; \ + } \ + } while (0) #endif uint32_t vp9_find_best_sub_pixel_tree( const MACROBLOCK *x, MV *bestmv, const MV *ref_mv, int allow_hp, @@ -1786,33 +1794,10 @@ start_col = VPXMAX(center_mv->col - range, mv_limits->col_min); end_row = VPXMIN(center_mv->row + range, mv_limits->row_max); end_col = VPXMIN(center_mv->col + range, mv_limits->col_max); for (r = start_row; r <= end_row; r += 1) { c = start_col; - // sdx8f may not be available some block size - if (fn_ptr->sdx8f) { - while (c + 7 <= end_col) { - unsigned int sads[8]; - const MV mv = { r, c }; - const uint8_t *buf = get_buf_from_mv(pre, &mv); - fn_ptr->sdx8f(src->buf, src->stride, buf, pre->stride, sads); - - for (i = 0; i < 8; ++i) { - int64_t sad = (int64_t)sads[i] << LOG2_PRECISION; - if (sad < best_sad) { - const MV mv = { r, c + i }; - sad += lambda * - vp9_nb_mvs_inconsistency(&mv, nb_full_mvs, full_mv_num); - if (sad < best_sad) { - best_sad = sad; - *best_mv = mv; - } - } - } - c += 8; - } - } while (c + 3 <= end_col) { unsigned int sads[4]; const uint8_t *addrs[4]; for (i = 0; i < 4; ++i) { const MV mv = { r, c + i }; @@ -2960,11 +2945,11 @@ (void)tr; \ (void)tc; \ (void)sse; \ (void)thismse; \ (void)cost_list; \ - (void)use_accurate_subpel_search; + (void)use_accurate_subpel_search // Return the maximum MV. uint32_t vp9_return_max_sub_pixel_mv( const MACROBLOCK *x, MV *bestmv, const MV *ref_mv, int allow_hp, int error_per_bit, const vp9_variance_fn_ptr_t *vfp, int forced_stop, diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_multi_thread.c b/media/libvpx/libvpx/vp9/encoder/vp9_multi_thread.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_multi_thread.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_multi_thread.c @@ -34,11 +34,11 @@ // lock the mutex for queue access #if CONFIG_MULTITHREAD pthread_mutex_lock(mutex_handle); #endif next = job_queue_hdl->next; - if (NULL != next) { + if (next != NULL) { JobQueue *job_queue = (JobQueue *)next; job_info = &job_queue->job_info; // Update the next job in the queue job_queue_hdl->next = job_queue->next; job_queue_hdl->num_jobs_acquired++; @@ -56,13 +56,14 @@ VP9_COMMON *const cm = &cpi->common; const int sb_rows = (mi_cols_aligned_to_sb(cm->mi_rows) >> MI_BLOCK_SIZE_LOG2) + 1; int i; - this_tile->row_base_thresh_freq_fact = + CHECK_MEM_ERROR( + cm, this_tile->row_base_thresh_freq_fact, (int *)vpx_calloc(sb_rows * BLOCK_SIZES * MAX_MODES, - sizeof(*(this_tile->row_base_thresh_freq_fact))); + sizeof(*(this_tile->row_base_thresh_freq_fact)))); for (i = 0; i < sb_rows * BLOCK_SIZES * MAX_MODES; i++) this_tile->row_base_thresh_freq_fact[i] = RD_THRESH_INIT_FACT; } void vp9_row_mt_mem_alloc(VP9_COMP *cpi) { @@ -82,12 +83,12 @@ multi_thread_ctxt->allocated_tile_cols = tile_cols; multi_thread_ctxt->allocated_tile_rows = tile_rows; multi_thread_ctxt->allocated_vert_unit_rows = jobs_per_tile_col; - multi_thread_ctxt->job_queue = - (JobQueue *)vpx_memalign(32, total_jobs * sizeof(JobQueue)); + CHECK_MEM_ERROR(cm, multi_thread_ctxt->job_queue, + (JobQueue *)vpx_memalign(32, total_jobs * sizeof(JobQueue))); #if CONFIG_MULTITHREAD // Create mutex for each tile for (tile_col = 0; tile_col < tile_cols; tile_col++) { RowMTInfo *row_mt_info = &multi_thread_ctxt->row_mt_info[tile_col]; diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_non_greedy_mv.c b/media/libvpx/libvpx/vp9/encoder/vp9_non_greedy_mv.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_non_greedy_mv.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_non_greedy_mv.c @@ -176,10 +176,11 @@ vp9_free_motion_field_info(motion_field_info); } motion_field_info->frame_num = frame_num; motion_field_info->motion_field_array = vpx_calloc(frame_num, sizeof(*motion_field_info->motion_field_array)); + if (!motion_field_info->motion_field_array) return STATUS_FAILED; for (frame_idx = 0; frame_idx < frame_num; ++frame_idx) { for (rf_idx = 0; rf_idx < MAX_INTER_REF_FRAMES; ++rf_idx) { for (square_block_idx = 0; square_block_idx < SQUARE_BLOCK_SIZES; ++square_block_idx) { BLOCK_SIZE bsize = square_block_idx_to_bsize(square_block_idx); @@ -420,10 +421,11 @@ FloatMV *output = (FloatMV *)malloc(rows * cols * sizeof(FloatMV)); int idx; int row, col; int bw = 4 << b_width_log2_lookup[bsize]; int bh = 4 << b_height_log2_lookup[bsize]; + if (!(input && output)) goto fail; // copy search results to input buffer for (idx = 0; idx < rows * cols; ++idx) { input[idx].row = (float)search_mf[idx].row / bh; input[idx].col = (float)search_mf[idx].col / bw; } @@ -448,10 +450,11 @@ // copy smoothed results to output for (idx = 0; idx < rows * cols; ++idx) { smooth_mf[idx].row = (int)(input[idx].row * bh); smooth_mf[idx].col = (int)(input[idx].col * bw); } +fail: free(input); free(output); } void vp9_get_local_structure(const YV12_BUFFER_CONFIG *cur_frame, diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_pickmode.c b/media/libvpx/libvpx/vp9/encoder/vp9_pickmode.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_pickmode.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_pickmode.c @@ -769,28 +769,31 @@ uint16_t *const eob = &p->eobs[block]; const int diff_stride = bw; const int16_t *src_diff; src_diff = &p->src_diff[(r * diff_stride + c) << 2]; + // skip block condition should be handled before this is called. + assert(!x->skip_block); + switch (tx_size) { case TX_16X16: vpx_hadamard_16x16(src_diff, diff_stride, coeff); - vp9_quantize_fp(coeff, 256, x->skip_block, p->round_fp, p->quant_fp, - qcoeff, dqcoeff, pd->dequant, eob, scan_order->scan, + vp9_quantize_fp(coeff, 256, p->round_fp, p->quant_fp, qcoeff, + dqcoeff, pd->dequant, eob, scan_order->scan, scan_order->iscan); break; case TX_8X8: vpx_hadamard_8x8(src_diff, diff_stride, coeff); - vp9_quantize_fp(coeff, 64, x->skip_block, p->round_fp, p->quant_fp, - qcoeff, dqcoeff, pd->dequant, eob, scan_order->scan, + vp9_quantize_fp(coeff, 64, p->round_fp, p->quant_fp, qcoeff, + dqcoeff, pd->dequant, eob, scan_order->scan, scan_order->iscan); break; default: assert(tx_size == TX_4X4); x->fwd_txfm4x4(src_diff, coeff, diff_stride); - vp9_quantize_fp(coeff, 16, x->skip_block, p->round_fp, p->quant_fp, - qcoeff, dqcoeff, pd->dequant, eob, scan_order->scan, + vp9_quantize_fp(coeff, 16, p->round_fp, p->quant_fp, qcoeff, + dqcoeff, pd->dequant, eob, scan_order->scan, scan_order->iscan); break; } *skippable &= (*eob == 0); eob_cost += 1; @@ -1245,11 +1248,11 @@ static INLINE void find_predictors( VP9_COMP *cpi, MACROBLOCK *x, MV_REFERENCE_FRAME ref_frame, int_mv frame_mv[MB_MODE_COUNT][MAX_REF_FRAMES], int const_motion[MAX_REF_FRAMES], int *ref_frame_skip_mask, - const int flag_list[4], TileDataEnc *tile_data, int mi_row, int mi_col, + TileDataEnc *tile_data, int mi_row, int mi_col, struct buf_2d yv12_mb[4][MAX_MB_PLANE], BLOCK_SIZE bsize, int force_skip_low_temp_var, int comp_pred_allowed) { VP9_COMMON *const cm = &cpi->common; MACROBLOCKD *const xd = &x->e_mbd; const YV12_BUFFER_CONFIG *yv12 = get_ref_frame_buffer(cpi, ref_frame); @@ -1257,11 +1260,11 @@ // TODO(jingning) placeholder for inter-frame non-RD mode decision. x->pred_mv_sad[ref_frame] = INT_MAX; frame_mv[NEWMV][ref_frame].as_int = INVALID_MV; frame_mv[ZEROMV][ref_frame].as_int = 0; // this needs various further optimizations. to be continued.. - if ((cpi->ref_frame_flags & flag_list[ref_frame]) && (yv12 != NULL)) { + if ((cpi->ref_frame_flags & ref_frame_to_flag(ref_frame)) && (yv12 != NULL)) { int_mv *const candidates = x->mbmi_ext->ref_mvs[ref_frame]; const struct scale_factors *const sf = &cm->frame_refs[ref_frame - 1].sf; vp9_setup_pred_block(xd, yv12_mb[ref_frame], yv12, mi_row, mi_col, sf, sf); if (cm->use_prev_frame_mvs || comp_pred_allowed) { vp9_find_mv_refs(cm, xd, xd->mi[0], ref_frame, candidates, mi_row, mi_col, @@ -1688,12 +1691,10 @@ MV_REFERENCE_FRAME ref_frame; MV_REFERENCE_FRAME usable_ref_frame, second_ref_frame; int_mv frame_mv[MB_MODE_COUNT][MAX_REF_FRAMES]; uint8_t mode_checked[MB_MODE_COUNT][MAX_REF_FRAMES]; struct buf_2d yv12_mb[4][MAX_MB_PLANE]; - static const int flag_list[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG, - VP9_ALT_FLAG }; RD_COST this_rdc, best_rdc; // var_y and sse_y are saved to be used in skipping checking unsigned int var_y = UINT_MAX; unsigned int sse_y = UINT_MAX; const int intra_cost_penalty = @@ -1923,18 +1924,18 @@ // For svc mode, on spatial_layer_id > 0: if the reference has different scale // constrain the inter mode to only test zero motion. if (cpi->use_svc && svc->force_zero_mode_spatial_ref && svc->spatial_layer_id > 0 && !gf_temporal_ref) { - if (cpi->ref_frame_flags & flag_list[LAST_FRAME]) { + if (cpi->ref_frame_flags & VP9_LAST_FLAG) { struct scale_factors *const sf = &cm->frame_refs[LAST_FRAME - 1].sf; if (vp9_is_scaled(sf)) { svc_force_zero_mode[LAST_FRAME - 1] = 1; inter_layer_ref = LAST_FRAME; } } - if (cpi->ref_frame_flags & flag_list[GOLDEN_FRAME]) { + if (cpi->ref_frame_flags & VP9_GOLD_FLAG) { struct scale_factors *const sf = &cm->frame_refs[GOLDEN_FRAME - 1].sf; if (vp9_is_scaled(sf)) { svc_force_zero_mode[GOLDEN_FRAME - 1] = 1; inter_layer_ref = GOLDEN_FRAME; } @@ -1955,11 +1956,11 @@ if (sf->disable_golden_ref && (x->content_state_sb != kVeryHighSad || cpi->rc.avg_frame_low_motion < 60)) usable_ref_frame = LAST_FRAME; - if (!((cpi->ref_frame_flags & flag_list[GOLDEN_FRAME]) && + if (!((cpi->ref_frame_flags & VP9_GOLD_FLAG) && !svc_force_zero_mode[GOLDEN_FRAME - 1] && !force_skip_low_temp_var)) use_golden_nonzeromv = 0; if (cpi->oxcf.speed >= 8 && !cpi->use_svc && ((cpi->rc.frames_since_golden + 1) < x->last_sb_high_content || @@ -1983,16 +1984,15 @@ for (ref_frame = LAST_FRAME; ref_frame <= usable_ref_frame; ++ref_frame) { // Skip find_predictor if the reference frame is not in the // ref_frame_flags (i.e., not used as a reference for this frame). skip_ref_find_pred[ref_frame] = - !(cpi->ref_frame_flags & flag_list[ref_frame]); + !(cpi->ref_frame_flags & ref_frame_to_flag(ref_frame)); if (!skip_ref_find_pred[ref_frame]) { find_predictors(cpi, x, ref_frame, frame_mv, const_motion, - &ref_frame_skip_mask, flag_list, tile_data, mi_row, - mi_col, yv12_mb, bsize, force_skip_low_temp_var, - comp_modes > 0); + &ref_frame_skip_mask, tile_data, mi_row, mi_col, yv12_mb, + bsize, force_skip_low_temp_var, comp_modes > 0); } } if (cpi->use_svc || cpi->oxcf.speed <= 7 || bsize < BLOCK_32X32) x->sb_use_mv_part = 0; @@ -2012,11 +2012,11 @@ // For SVC with quality layers, when QP of lower layer is lower // than current layer: force check of GF-ZEROMV before early exit // due to skip flag. if (svc->spatial_layer_id > 0 && no_scaling && - (cpi->ref_frame_flags & flag_list[GOLDEN_FRAME]) && + (cpi->ref_frame_flags & VP9_GOLD_FLAG) && cm->base_qindex > svc->lower_layer_qindex + 10) force_test_gf_zeromv = 1; // For low motion content use x->sb_is_skin in addition to VeryHighSad // for setting large_block. @@ -2092,11 +2092,12 @@ } if (comp_pred) { if (!cpi->allow_comp_inter_inter) continue; // Skip compound inter modes if ARF is not available. - if (!(cpi->ref_frame_flags & flag_list[second_ref_frame])) continue; + if (!(cpi->ref_frame_flags & ref_frame_to_flag(second_ref_frame))) + continue; // Do not allow compound prediction if the segment level reference frame // feature is in use as in this case there can only be one reference. if (segfeature_active(seg, mi->segment_id, SEG_LVL_REF_FRAME)) continue; } @@ -2105,11 +2106,11 @@ if (ref_frame == GOLDEN_FRAME && cpi->oxcf.rc_mode == VPX_CBR && ((cpi->use_svc && sse_zeromv_normalized < thresh_svc_skip_golden) || (!cpi->use_svc && sse_zeromv_normalized < thresh_skip_golden))) continue; - if (!(cpi->ref_frame_flags & flag_list[ref_frame])) continue; + if (!(cpi->ref_frame_flags & ref_frame_to_flag(ref_frame))) continue; // For screen content. If zero_temp_sad source is computed: skip // non-zero motion check for stationary blocks. If the superblock is // non-stationary then for flat blocks skip the zero last check (keep golden // as it may be inter-layer reference). Otherwise (if zero_temp_sad_source @@ -2188,22 +2189,22 @@ !(frame_mv[this_mode][ref_frame].as_int == 0 && ref_frame == LAST_FRAME)) { if (usable_ref_frame < ALTREF_FRAME) { if (!force_skip_low_temp_var && usable_ref_frame > LAST_FRAME) { i = (ref_frame == LAST_FRAME) ? GOLDEN_FRAME : LAST_FRAME; - if ((cpi->ref_frame_flags & flag_list[i])) + if ((cpi->ref_frame_flags & ref_frame_to_flag(i))) if (x->pred_mv_sad[ref_frame] > (x->pred_mv_sad[i] << 1)) ref_frame_skip_mask |= (1 << ref_frame); } } else if (!cpi->rc.is_src_frame_alt_ref && !(frame_mv[this_mode][ref_frame].as_int == 0 && ref_frame == ALTREF_FRAME)) { int ref1 = (ref_frame == GOLDEN_FRAME) ? LAST_FRAME : GOLDEN_FRAME; int ref2 = (ref_frame == ALTREF_FRAME) ? LAST_FRAME : ALTREF_FRAME; - if (((cpi->ref_frame_flags & flag_list[ref1]) && + if (((cpi->ref_frame_flags & ref_frame_to_flag(ref1)) && (x->pred_mv_sad[ref_frame] > (x->pred_mv_sad[ref1] << 1))) || - ((cpi->ref_frame_flags & flag_list[ref2]) && + ((cpi->ref_frame_flags & ref_frame_to_flag(ref2)) && (x->pred_mv_sad[ref_frame] > (x->pred_mv_sad[ref2] << 1)))) ref_frame_skip_mask |= (1 << ref_frame); } } if (ref_frame_skip_mask & (1 << ref_frame)) continue; @@ -2486,11 +2487,11 @@ // temporal layer. if (svc->spatial_layer_id && !gf_temporal_ref) { perform_intra_pred = svc->temporal_layer_id == 0 || svc->layer_context[svc->temporal_layer_id].is_key_frame || - !(cpi->ref_frame_flags & flag_list[GOLDEN_FRAME]) || + !(cpi->ref_frame_flags & VP9_GOLD_FLAG) || (!svc->layer_context[svc->temporal_layer_id].is_key_frame && svc_force_zero_mode[best_pickmode.best_ref_frame - 1]); inter_mode_thresh = (inter_mode_thresh << 1) + inter_mode_thresh; } if ((cpi->oxcf.lag_in_frames > 0 && cpi->oxcf.rc_mode == VPX_VBR && @@ -2745,12 +2746,10 @@ const struct segmentation *const seg = &cm->seg; MV_REFERENCE_FRAME ref_frame, second_ref_frame = NONE; MV_REFERENCE_FRAME best_ref_frame = NONE; unsigned char segment_id = mi->segment_id; struct buf_2d yv12_mb[4][MAX_MB_PLANE]; - static const int flag_list[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG, - VP9_ALT_FLAG }; int64_t best_rd = INT64_MAX; b_mode_info bsi[MAX_REF_FRAMES][4]; int ref_frame_skip_mask = 0; const int num_4x4_blocks_wide = num_4x4_blocks_wide_lookup[bsize]; const int num_4x4_blocks_high = num_4x4_blocks_high_lookup[bsize]; @@ -2762,11 +2761,12 @@ for (ref_frame = LAST_FRAME; ref_frame <= GOLDEN_FRAME; ++ref_frame) { const YV12_BUFFER_CONFIG *yv12 = get_ref_frame_buffer(cpi, ref_frame); int_mv dummy_mv[2]; x->pred_mv_sad[ref_frame] = INT_MAX; - if ((cpi->ref_frame_flags & flag_list[ref_frame]) && (yv12 != NULL)) { + if ((cpi->ref_frame_flags & ref_frame_to_flag(ref_frame)) && + (yv12 != NULL)) { int_mv *const candidates = mbmi_ext->ref_mvs[ref_frame]; const struct scale_factors *const sf = &cm->frame_refs[ref_frame - 1].sf; vp9_setup_pred_block(xd, yv12_mb[ref_frame], yv12, mi_row, mi_col, sf, sf); vp9_find_mv_refs(cm, xd, xd->mi[0], ref_frame, candidates, mi_row, mi_col, diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_quantize.c b/media/libvpx/libvpx/vp9/encoder/vp9_quantize.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_quantize.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_quantize.c @@ -20,19 +20,16 @@ #include "vp9/encoder/vp9_encoder.h" #include "vp9/encoder/vp9_quantize.h" #include "vp9/encoder/vp9_rd.h" void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *round_ptr, - const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, - uint16_t *eob_ptr, const int16_t *scan, - const int16_t *iscan) { + const int16_t *round_ptr, const int16_t *quant_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { int i, eob = -1; (void)iscan; - (void)skip_block; - assert(!skip_block); memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); // Quantization pass: All coefficients with index >= zero_flag are @@ -54,21 +51,19 @@ *eob_ptr = eob + 1; } #if CONFIG_VP9_HIGHBITDEPTH void vp9_highbd_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *round_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan) { int i; int eob = -1; (void)iscan; - (void)skip_block; - assert(!skip_block); memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); // Quantization pass: All coefficients with index >= zero_flag are @@ -89,19 +84,16 @@ #endif // TODO(jingning) Refactor this file and combine functions with similar // operations. void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *round_ptr, - const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan) { int i, eob = -1; (void)iscan; - (void)skip_block; - assert(!skip_block); memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); for (i = 0; i < n_coeffs; i++) { @@ -124,19 +116,17 @@ *eob_ptr = eob + 1; } #if CONFIG_VP9_HIGHBITDEPTH void vp9_highbd_quantize_fp_32x32_c( - const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, - const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, - const int16_t *scan, const int16_t *iscan) { + const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, + const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, + const int16_t *iscan) { int i, eob = -1; (void)iscan; - (void)skip_block; - assert(!skip_block); memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); for (i = 0; i < n_coeffs; i++) { @@ -174,20 +164,19 @@ return; } #if CONFIG_VP9_HIGHBITDEPTH if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { - vpx_highbd_quantize_b(BLOCK_OFFSET(p->coeff, block), n_coeffs, - x->skip_block, p->zbin, p->round, p->quant, - p->quant_shift, qcoeff, dqcoeff, pd->dequant, - &p->eobs[block], scan, iscan); + vpx_highbd_quantize_b(BLOCK_OFFSET(p->coeff, block), n_coeffs, p->zbin, + p->round, p->quant, p->quant_shift, qcoeff, dqcoeff, + pd->dequant, &p->eobs[block], scan, iscan); return; } #endif - vpx_quantize_b(BLOCK_OFFSET(p->coeff, block), n_coeffs, x->skip_block, - p->zbin, p->round, p->quant, p->quant_shift, qcoeff, dqcoeff, - pd->dequant, &p->eobs[block], scan, iscan); + vpx_quantize_b(BLOCK_OFFSET(p->coeff, block), n_coeffs, p->zbin, p->round, + p->quant, p->quant_shift, qcoeff, dqcoeff, pd->dequant, + &p->eobs[block], scan, iscan); } static void invert_quant(int16_t *quant, int16_t *shift, int d) { unsigned t; int l, m; diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_ratectrl.c b/media/libvpx/libvpx/vp9/encoder/vp9_ratectrl.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_ratectrl.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_ratectrl.c @@ -221,24 +221,25 @@ // Clip the frame target to the maximum allowed value. if (target > rc->max_frame_bandwidth) target = rc->max_frame_bandwidth; if (oxcf->rc_max_inter_bitrate_pct) { - const int max_rate = - rc->avg_frame_bandwidth * oxcf->rc_max_inter_bitrate_pct / 100; - target = VPXMIN(target, max_rate); + const int64_t max_rate = + (int64_t)rc->avg_frame_bandwidth * oxcf->rc_max_inter_bitrate_pct / 100; + // target is of type int and VPXMIN cannot evaluate to larger than target + target = (int)VPXMIN(target, max_rate); } return target; } int vp9_rc_clamp_iframe_target_size(const VP9_COMP *const cpi, int target) { const RATE_CONTROL *rc = &cpi->rc; const VP9EncoderConfig *oxcf = &cpi->oxcf; if (oxcf->rc_max_intra_bitrate_pct) { - const int max_rate = - rc->avg_frame_bandwidth * oxcf->rc_max_intra_bitrate_pct / 100; - target = VPXMIN(target, max_rate); + const int64_t max_rate = + (int64_t)rc->avg_frame_bandwidth * oxcf->rc_max_intra_bitrate_pct / 100; + target = (int)VPXMIN(target, max_rate); } if (target > rc->max_frame_bandwidth) target = rc->max_frame_bandwidth; return target; } @@ -275,13 +276,13 @@ if (use_timestamp && cpi->svc.use_set_ref_frame_config && svc->number_temporal_layers == 1 && ts_delta > 0 && svc->current_superframe > 0) { // TODO(marpan): This may need to be modified for temporal layers. const double framerate_pts = 10000000.0 / ts_delta; - lrc->bits_off_target += (int)(lc->target_bandwidth / framerate_pts); + lrc->bits_off_target += (int)round(lc->target_bandwidth / framerate_pts); } else { - lrc->bits_off_target += (int)(lc->target_bandwidth / lc->framerate); + lrc->bits_off_target += (int)round(lc->target_bandwidth / lc->framerate); } // Clip buffer level to maximum buffer size for the layer. lrc->bits_off_target = VPXMIN(lrc->bits_off_target, lrc->maximum_buffer_size); lrc->buffer_level = lrc->bits_off_target; @@ -2211,11 +2212,10 @@ // Also if intra-only is inserted on very first frame, don't allow if // if number of temporal layers > 1. This is because on intra-only frame // only 3 reference buffers can be updated, but for temporal layers > 1 // we generally need to use buffer slots 4 and 5. if ((cm->current_video_frame == 0 && svc->number_temporal_layers > 1) || - svc->temporal_layering_mode == VP9E_TEMPORAL_LAYERING_MODE_BYPASS || svc->number_spatial_layers > 3 || svc->number_temporal_layers > 3 || svc->number_spatial_layers == 1) return; cm->show_frame = 0; cm->intra_only = 1; @@ -2232,23 +2232,33 @@ int i; int count = 0; cpi->lst_fb_idx = -1; cpi->gld_fb_idx = -1; cpi->alt_fb_idx = -1; + svc->update_buffer_slot[0] = 0; // For intra-only frame we need to refresh all slots that were // being used for the base layer (fb_idx_base[i] == 1). // Start with assigning last first, then golden and then alt. for (i = 0; i < REF_FRAMES; ++i) { - if (svc->fb_idx_base[i] == 1) count++; + if (svc->fb_idx_base[i] == 1) { + svc->update_buffer_slot[0] |= 1 << i; + count++; + } if (count == 1 && cpi->lst_fb_idx == -1) cpi->lst_fb_idx = i; if (count == 2 && cpi->gld_fb_idx == -1) cpi->gld_fb_idx = i; if (count == 3 && cpi->alt_fb_idx == -1) cpi->alt_fb_idx = i; } // If golden or alt is not being used for base layer, then set them // to the lst_fb_idx. if (cpi->gld_fb_idx == -1) cpi->gld_fb_idx = cpi->lst_fb_idx; if (cpi->alt_fb_idx == -1) cpi->alt_fb_idx = cpi->lst_fb_idx; + if (svc->temporal_layering_mode == VP9E_TEMPORAL_LAYERING_MODE_BYPASS) { + cpi->ext_refresh_last_frame = 0; + cpi->ext_refresh_golden_frame = 0; + cpi->ext_refresh_alt_ref_frame = 0; + cpi->ref_frame_flags = 0; + } } } void vp9_rc_get_svc_params(VP9_COMP *cpi) { VP9_COMMON *const cm = &cpi->common; @@ -2387,10 +2397,13 @@ } if (svc->set_intra_only_frame) { set_intra_only_frame(cpi); target = vp9_calc_iframe_target_size_one_pass_cbr(cpi); } + // Overlay frame predicts from LAST (intra-only) + if (svc->previous_frame_is_intra_only) cpi->ref_frame_flags |= VP9_LAST_FLAG; + // Any update/change of global cyclic refresh parameters (amount/delta-qp) // should be done here, before the frame qp is selected. if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ) vp9_cyclic_refresh_update_parameters(cpi); diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_rdopt.c b/media/libvpx/libvpx/vp9/encoder/vp9_rdopt.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_rdopt.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_rdopt.c @@ -3313,12 +3313,10 @@ int_mv frame_mv[MB_MODE_COUNT][MAX_REF_FRAMES]; struct buf_2d yv12_mb[4][MAX_MB_PLANE]; int_mv single_newmv[MAX_REF_FRAMES] = { { 0 } }; INTERP_FILTER single_inter_filter[MB_MODE_COUNT][MAX_REF_FRAMES]; int single_skippable[MB_MODE_COUNT][MAX_REF_FRAMES]; - static const int flag_list[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG, - VP9_ALT_FLAG }; int64_t best_rd = best_rd_so_far; int64_t best_pred_diff[REFERENCE_MODES]; int64_t best_pred_rd[REFERENCE_MODES]; int64_t best_filter_rd[SWITCHABLE_FILTER_CONTEXTS]; int64_t best_filter_diff[SWITCHABLE_FILTER_CONTEXTS]; @@ -3390,22 +3388,22 @@ rd_cost->rate = INT_MAX; for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { x->pred_mv_sad[ref_frame] = INT_MAX; - if ((cpi->ref_frame_flags & flag_list[ref_frame]) && + if ((cpi->ref_frame_flags & ref_frame_to_flag(ref_frame)) && !(is_rect_partition && (ctx->skip_ref_frame_mask & (1 << ref_frame)))) { assert(get_ref_frame_buffer(cpi, ref_frame) != NULL); setup_buffer_inter(cpi, x, ref_frame, bsize, mi_row, mi_col, frame_mv[NEARESTMV], frame_mv[NEARMV], yv12_mb); } frame_mv[NEWMV][ref_frame].as_int = INVALID_MV; frame_mv[ZEROMV][ref_frame].as_int = 0; } for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { - if (!(cpi->ref_frame_flags & flag_list[ref_frame])) { + if (!(cpi->ref_frame_flags & ref_frame_to_flag(ref_frame))) { // Skip checking missing references in both single and compound reference // modes. Note that a mode will be skipped if both reference frames // are masked out. ref_frame_skip_mask[0] |= (1 << ref_frame); ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK; @@ -3607,11 +3605,12 @@ if (cm->ref_frame_sign_bias[ref_frame] == cm->ref_frame_sign_bias[second_ref_frame]) continue; // Skip compound inter modes if ARF is not available. - if (!(cpi->ref_frame_flags & flag_list[second_ref_frame])) continue; + if (!(cpi->ref_frame_flags & ref_frame_to_flag(second_ref_frame))) + continue; // Do not allow compound prediction if the segment level reference frame // feature is in use as in this case there can only be one reference. if (segfeature_active(seg, segment_id, SEG_LVL_REF_FRAME)) continue; @@ -4138,12 +4137,10 @@ MV_REFERENCE_FRAME ref_frame, second_ref_frame; unsigned char segment_id = mi->segment_id; int comp_pred, i; int_mv frame_mv[MB_MODE_COUNT][MAX_REF_FRAMES]; struct buf_2d yv12_mb[4][MAX_MB_PLANE]; - static const int flag_list[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG, - VP9_ALT_FLAG }; int64_t best_rd = best_rd_so_far; int64_t best_yrd = best_rd_so_far; // FIXME(rbultje) more precise int64_t best_pred_diff[REFERENCE_MODES]; int64_t best_pred_rd[REFERENCE_MODES]; int64_t best_filter_rd[SWITCHABLE_FILTER_CONTEXTS]; @@ -4189,11 +4186,11 @@ rate_uv_intra = INT_MAX; rd_cost->rate = INT_MAX; for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ref_frame++) { - if (cpi->ref_frame_flags & flag_list[ref_frame]) { + if (cpi->ref_frame_flags & ref_frame_to_flag(ref_frame)) { setup_buffer_inter(cpi, x, ref_frame, bsize, mi_row, mi_col, frame_mv[NEARESTMV], frame_mv[NEARMV], yv12_mb); } else { ref_frame_skip_mask[0] |= (1 << ref_frame); ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK; @@ -4274,11 +4271,12 @@ if (cm->ref_frame_sign_bias[ref_frame] == cm->ref_frame_sign_bias[second_ref_frame]) continue; - if (!(cpi->ref_frame_flags & flag_list[second_ref_frame])) continue; + if (!(cpi->ref_frame_flags & ref_frame_to_flag(second_ref_frame))) + continue; // Do not allow compound prediction if the segment level reference frame // feature is in use as in this case there can only be one reference. if (segfeature_active(seg, segment_id, SEG_LVL_REF_FRAME)) continue; if ((sf->mode_search_skip_flags & FLAG_SKIP_COMP_BESTINTRA) && diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_speed_features.c b/media/libvpx/libvpx/vp9/encoder/vp9_speed_features.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_speed_features.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_speed_features.c @@ -493,15 +493,14 @@ sf->reference_masking = (svc->number_spatial_layers == 1); if (sf->reference_masking == 1 && (cpi->external_resize == 1 || cpi->oxcf.resize_mode == RESIZE_DYNAMIC)) { MV_REFERENCE_FRAME ref_frame; - static const int flag_list[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG, - VP9_ALT_FLAG }; for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { const YV12_BUFFER_CONFIG *yv12 = get_ref_frame_buffer(cpi, ref_frame); - if (yv12 != NULL && (cpi->ref_frame_flags & flag_list[ref_frame])) { + if (yv12 != NULL && + (cpi->ref_frame_flags & ref_frame_to_flag(ref_frame))) { const struct scale_factors *const scale_fac = &cm->frame_refs[ref_frame - 1].sf; if (vp9_is_scaled(scale_fac)) sf->reference_masking = 0; } } @@ -651,12 +650,14 @@ sf->adapt_partition_thresh = (cm->width * cm->height <= 640 * 360) ? 40000 : 60000; if (cpi->content_state_sb_fd == NULL && (!cpi->use_svc || svc->spatial_layer_id == svc->number_spatial_layers - 1)) { - cpi->content_state_sb_fd = (uint8_t *)vpx_calloc( - (cm->mi_stride >> 3) * ((cm->mi_rows >> 3) + 1), sizeof(uint8_t)); + CHECK_MEM_ERROR(cm, cpi->content_state_sb_fd, + (uint8_t *)vpx_calloc( + (cm->mi_stride >> 3) * ((cm->mi_rows >> 3) + 1), + sizeof(uint8_t))); } } if (cpi->oxcf.rc_mode == VPX_CBR && content != VP9E_CONTENT_SCREEN) { // Enable short circuit for low temporal variance. sf->short_circuit_low_temp_var = 1; @@ -803,18 +804,21 @@ if (sf->use_altref_onepass) { if (cpi->rc.is_src_frame_alt_ref && cm->frame_type != KEY_FRAME) { sf->partition_search_type = FIXED_PARTITION; sf->always_this_block_size = BLOCK_64X64; } - if (cpi->count_arf_frame_usage == NULL) - cpi->count_arf_frame_usage = + if (cpi->count_arf_frame_usage == NULL) { + CHECK_MEM_ERROR( + cm, cpi->count_arf_frame_usage, (uint8_t *)vpx_calloc((cm->mi_stride >> 3) * ((cm->mi_rows >> 3) + 1), - sizeof(*cpi->count_arf_frame_usage)); + sizeof(*cpi->count_arf_frame_usage))); + } if (cpi->count_lastgolden_frame_usage == NULL) - cpi->count_lastgolden_frame_usage = + CHECK_MEM_ERROR( + cm, cpi->count_lastgolden_frame_usage, (uint8_t *)vpx_calloc((cm->mi_stride >> 3) * ((cm->mi_rows >> 3) + 1), - sizeof(*cpi->count_lastgolden_frame_usage)); + sizeof(*cpi->count_lastgolden_frame_usage))); } if (svc->previous_frame_is_intra_only) { sf->partition_search_type = FIXED_PARTITION; sf->always_this_block_size = BLOCK_64X64; } diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_subexp.h b/media/libvpx/libvpx/vp9/encoder/vp9_subexp.h --- a/media/libvpx/libvpx/vp9/encoder/vp9_subexp.h +++ b/media/libvpx/libvpx/vp9/encoder/vp9_subexp.h @@ -23,17 +23,18 @@ vpx_prob oldp); void vp9_cond_prob_diff_update(struct vpx_writer *w, vpx_prob *oldp, const unsigned int ct[2]); -int vp9_prob_diff_update_savings_search(const unsigned int *ct, vpx_prob oldp, - vpx_prob *bestp, vpx_prob upd); +int64_t vp9_prob_diff_update_savings_search(const unsigned int *ct, + vpx_prob oldp, vpx_prob *bestp, + vpx_prob upd); -int vp9_prob_diff_update_savings_search_model(const unsigned int *ct, - const vpx_prob oldp, - vpx_prob *bestp, vpx_prob upd, - int stepsize); +int64_t vp9_prob_diff_update_savings_search_model(const unsigned int *ct, + const vpx_prob oldp, + vpx_prob *bestp, vpx_prob upd, + int stepsize); #ifdef __cplusplus } // extern "C" #endif diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_subexp.c b/media/libvpx/libvpx/vp9/encoder/vp9_subexp.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_subexp.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_subexp.c @@ -112,42 +112,43 @@ void vp9_write_prob_diff_update(vpx_writer *w, vpx_prob newp, vpx_prob oldp) { const int delp = remap_prob(newp, oldp); encode_term_subexp(w, delp); } -int vp9_prob_diff_update_savings_search(const unsigned int *ct, vpx_prob oldp, - vpx_prob *bestp, vpx_prob upd) { - const int old_b = cost_branch256(ct, oldp); - int bestsavings = 0; +int64_t vp9_prob_diff_update_savings_search(const unsigned int *ct, + vpx_prob oldp, vpx_prob *bestp, + vpx_prob upd) { + const int64_t old_b = cost_branch256(ct, oldp); + int64_t bestsavings = 0; vpx_prob newp, bestnewp = oldp; const int step = *bestp > oldp ? -1 : 1; const int upd_cost = vp9_cost_one(upd) - vp9_cost_zero(upd); if (old_b > upd_cost + (MIN_DELP_BITS << VP9_PROB_COST_SHIFT)) { for (newp = *bestp; newp != oldp; newp += step) { - const int new_b = cost_branch256(ct, newp); - const int update_b = prob_diff_update_cost(newp, oldp) + upd_cost; - const int savings = old_b - new_b - update_b; + const int64_t new_b = cost_branch256(ct, newp); + const int64_t update_b = prob_diff_update_cost(newp, oldp) + upd_cost; + const int64_t savings = old_b - new_b - update_b; if (savings > bestsavings) { bestsavings = savings; bestnewp = newp; } } } *bestp = bestnewp; return bestsavings; } -int vp9_prob_diff_update_savings_search_model(const unsigned int *ct, - const vpx_prob oldp, - vpx_prob *bestp, vpx_prob upd, - int stepsize) { - int i, old_b, new_b, update_b, savings, bestsavings; - int newp; - const int step_sign = *bestp > oldp ? -1 : 1; - const int step = stepsize * step_sign; - const int upd_cost = vp9_cost_one(upd) - vp9_cost_zero(upd); +int64_t vp9_prob_diff_update_savings_search_model(const unsigned int *ct, + const vpx_prob oldp, + vpx_prob *bestp, vpx_prob upd, + int stepsize) { + int64_t i, old_b, new_b, update_b, savings, bestsavings; + int64_t newp; + const int64_t step_sign = *bestp > oldp ? -1 : 1; + const int64_t step = stepsize * step_sign; + const int64_t upd_cost = vp9_cost_one(upd) - vp9_cost_zero(upd); const vpx_prob *newplist, *oldplist; vpx_prob bestnewp; oldplist = vp9_pareto8_full[oldp - 1]; old_b = cost_branch256(ct + 2 * PIVOT_NODE, oldp); for (i = UNCONSTRAINED_NODES; i < ENTROPY_NODES; ++i) @@ -160,18 +161,18 @@ if (old_b > upd_cost + (MIN_DELP_BITS << VP9_PROB_COST_SHIFT)) { for (newp = *bestp; (newp - oldp) * step_sign < 0; newp += step) { if (newp < 1 || newp > 255) continue; newplist = vp9_pareto8_full[newp - 1]; - new_b = cost_branch256(ct + 2 * PIVOT_NODE, newp); + new_b = cost_branch256(ct + 2 * PIVOT_NODE, (vpx_prob)newp); for (i = UNCONSTRAINED_NODES; i < ENTROPY_NODES; ++i) new_b += cost_branch256(ct + 2 * i, newplist[i - UNCONSTRAINED_NODES]); - update_b = prob_diff_update_cost(newp, oldp) + upd_cost; + update_b = prob_diff_update_cost((vpx_prob)newp, oldp) + upd_cost; savings = old_b - new_b - update_b; if (savings > bestsavings) { bestsavings = savings; - bestnewp = newp; + bestnewp = (vpx_prob)newp; } } } *bestp = bestnewp; @@ -180,11 +181,11 @@ void vp9_cond_prob_diff_update(vpx_writer *w, vpx_prob *oldp, const unsigned int ct[2]) { const vpx_prob upd = DIFF_UPDATE_PROB; vpx_prob newp = get_binary_prob(ct[0], ct[1]); - const int savings = + const int64_t savings = vp9_prob_diff_update_savings_search(ct, *oldp, &newp, upd); assert(newp >= 1); if (savings > 0) { vpx_write(w, 1, upd); vp9_write_prob_diff_update(w, newp, *oldp); diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_svc_layercontext.c b/media/libvpx/libvpx/vp9/encoder/vp9_svc_layercontext.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_svc_layercontext.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_svc_layercontext.c @@ -460,34 +460,31 @@ static void reset_fb_idx_unused(VP9_COMP *const cpi) { // If a reference frame is not referenced or refreshed, then set the // fb_idx for that reference to the first one used/referenced. // This is to avoid setting fb_idx for a reference to a slot that is not // used/needed (i.e., since that reference is not referenced or refreshed). - static const int flag_list[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG, - VP9_ALT_FLAG }; MV_REFERENCE_FRAME ref_frame; MV_REFERENCE_FRAME first_ref = 0; int first_fb_idx = 0; int fb_idx[3] = { cpi->lst_fb_idx, cpi->gld_fb_idx, cpi->alt_fb_idx }; for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ref_frame++) { - if (cpi->ref_frame_flags & flag_list[ref_frame]) { + if (cpi->ref_frame_flags & ref_frame_to_flag(ref_frame)) { first_ref = ref_frame; first_fb_idx = fb_idx[ref_frame - 1]; break; } } if (first_ref > 0) { - if (first_ref != LAST_FRAME && - !(cpi->ref_frame_flags & flag_list[LAST_FRAME]) && + if (first_ref != LAST_FRAME && !(cpi->ref_frame_flags & VP9_LAST_FLAG) && !cpi->ext_refresh_last_frame) cpi->lst_fb_idx = first_fb_idx; else if (first_ref != GOLDEN_FRAME && - !(cpi->ref_frame_flags & flag_list[GOLDEN_FRAME]) && + !(cpi->ref_frame_flags & VP9_GOLD_FLAG) && !cpi->ext_refresh_golden_frame) cpi->gld_fb_idx = first_fb_idx; else if (first_ref != ALTREF_FRAME && - !(cpi->ref_frame_flags & flag_list[ALTREF_FRAME]) && + !(cpi->ref_frame_flags & VP9_ALT_FLAG) && !cpi->ext_refresh_alt_ref_frame) cpi->alt_fb_idx = first_fb_idx; } } @@ -728,12 +725,10 @@ if (svc->reference_altref[sl]) cpi->ref_frame_flags |= VP9_ALT_FLAG; } void vp9_copy_flags_ref_update_idx(VP9_COMP *const cpi) { SVC *const svc = &cpi->svc; - static const int flag_list[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG, - VP9_ALT_FLAG }; int sl = svc->spatial_layer_id; svc->lst_fb_idx[sl] = cpi->lst_fb_idx; svc->gld_fb_idx[sl] = cpi->gld_fb_idx; svc->alt_fb_idx[sl] = cpi->alt_fb_idx; // For the fixed SVC mode: pass the refresh_lst/gld/alt_frame flags to the @@ -752,16 +747,13 @@ // TODO(jianj): Remove these 3, deprecated. svc->update_last[sl] = (uint8_t)cpi->refresh_last_frame; svc->update_golden[sl] = (uint8_t)cpi->refresh_golden_frame; svc->update_altref[sl] = (uint8_t)cpi->refresh_alt_ref_frame; - svc->reference_last[sl] = - (uint8_t)(cpi->ref_frame_flags & flag_list[LAST_FRAME]); - svc->reference_golden[sl] = - (uint8_t)(cpi->ref_frame_flags & flag_list[GOLDEN_FRAME]); - svc->reference_altref[sl] = - (uint8_t)(cpi->ref_frame_flags & flag_list[ALTREF_FRAME]); + svc->reference_last[sl] = (uint8_t)(cpi->ref_frame_flags & VP9_LAST_FLAG); + svc->reference_golden[sl] = (uint8_t)(cpi->ref_frame_flags & VP9_GOLD_FLAG); + svc->reference_altref[sl] = (uint8_t)(cpi->ref_frame_flags & VP9_ALT_FLAG); } int vp9_one_pass_cbr_svc_start_layer(VP9_COMP *const cpi) { int width = 0, height = 0; SVC *const svc = &cpi->svc; @@ -1078,19 +1070,18 @@ !svc->layer_context[svc->temporal_layer_id].is_key_frame && !svc->superframe_has_layer_sync) || svc->disable_inter_layer_pred == INTER_LAYER_PRED_OFF || svc->drop_spatial_layer[sl - 1]) { MV_REFERENCE_FRAME ref_frame; - static const int flag_list[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG, - VP9_ALT_FLAG }; for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { const YV12_BUFFER_CONFIG *yv12 = get_ref_frame_buffer(cpi, ref_frame); - if (yv12 != NULL && (cpi->ref_frame_flags & flag_list[ref_frame])) { + if (yv12 != NULL && + (cpi->ref_frame_flags & ref_frame_to_flag(ref_frame))) { const struct scale_factors *const scale_fac = &cm->frame_refs[ref_frame - 1].sf; if (vp9_is_scaled(scale_fac)) { - cpi->ref_frame_flags &= (~flag_list[ref_frame]); + cpi->ref_frame_flags &= (~ref_frame_to_flag(ref_frame)); // Point golden/altref frame buffer index to last. if (!svc->simulcast_mode) { if (ref_frame == GOLDEN_FRAME) cpi->gld_fb_idx = cpi->lst_fb_idx; else if (ref_frame == ALTREF_FRAME) @@ -1241,18 +1232,24 @@ } } void vp9_svc_update_ref_frame_buffer_idx(VP9_COMP *const cpi) { SVC *const svc = &cpi->svc; + int i = 0; // Update the usage of frame buffer index for base spatial layers. if (svc->spatial_layer_id == 0) { if ((cpi->ref_frame_flags & VP9_LAST_FLAG) || cpi->refresh_last_frame) svc->fb_idx_base[cpi->lst_fb_idx] = 1; if ((cpi->ref_frame_flags & VP9_GOLD_FLAG) || cpi->refresh_golden_frame) svc->fb_idx_base[cpi->gld_fb_idx] = 1; if ((cpi->ref_frame_flags & VP9_ALT_FLAG) || cpi->refresh_alt_ref_frame) svc->fb_idx_base[cpi->alt_fb_idx] = 1; + // For bypass/flexible mode: check for refresh slots. + if (svc->temporal_layering_mode == VP9E_TEMPORAL_LAYERING_MODE_BYPASS) { + for (i = 0; i < REF_FRAMES; ++i) + if (svc->update_buffer_slot[0] & (1 << i)) svc->fb_idx_base[i] = 1; + } } } static void vp9_svc_update_ref_frame_bypass_mode(VP9_COMP *const cpi) { // For non-flexible/bypass SVC mode: check for refreshing other buffer diff --git a/media/libvpx/libvpx/vp9/encoder/x86/temporal_filter_sse4.c b/media/libvpx/libvpx/vp9/encoder/x86/temporal_filter_sse4.c --- a/media/libvpx/libvpx/vp9/encoder/x86/temporal_filter_sse4.c +++ b/media/libvpx/libvpx/vp9/encoder/x86/temporal_filter_sse4.c @@ -268,17 +268,15 @@ // Apply temporal filter to the luma components. This performs temporal // filtering on a luma block of 16 X block_height. Use blk_fw as an array of // size 4 for the weights for each of the 4 subblocks if blk_fw is not NULL, // else use top_weight for top half, and bottom weight for bottom half. static void vp9_apply_temporal_filter_luma_16( - const uint8_t *y_src, int y_src_stride, const uint8_t *y_pre, - int y_pre_stride, const uint8_t *u_src, const uint8_t *v_src, - int uv_src_stride, const uint8_t *u_pre, const uint8_t *v_pre, - int uv_pre_stride, unsigned int block_width, unsigned int block_height, - int ss_x, int ss_y, int strength, int use_whole_blk, uint32_t *y_accum, - uint16_t *y_count, const uint16_t *y_dist, const uint16_t *u_dist, - const uint16_t *v_dist, const int16_t *const *neighbors_first, + const uint8_t *y_pre, int y_pre_stride, unsigned int block_width, + unsigned int block_height, int ss_x, int ss_y, int strength, + int use_whole_blk, uint32_t *y_accum, uint16_t *y_count, + const uint16_t *y_dist, const uint16_t *u_dist, const uint16_t *v_dist, + const int16_t *const *neighbors_first, const int16_t *const *neighbors_second, int top_weight, int bottom_weight, const int *blk_fw) { const int rounding = (1 << strength) >> 1; __m128i weight_first, weight_second; @@ -299,11 +297,10 @@ assert(strength >= 0); assert(strength <= 6); assert(block_width == 16); - (void)block_width; // Initialize the weights if (blk_fw) { weight_first = _mm_set1_epi16(blk_fw[0]); @@ -340,21 +337,16 @@ sum_row_second = average_8(sum_row_second, &mul_second, strength, rounding, &weight_second); accumulate_and_store_16(sum_row_first, sum_row_second, y_pre, y_count, y_accum); - y_src += y_src_stride; y_pre += y_pre_stride; y_count += y_pre_stride; y_accum += y_pre_stride; y_dist += DIST_STRIDE; - u_src += uv_src_stride; - u_pre += uv_pre_stride; u_dist += DIST_STRIDE; - v_src += uv_src_stride; - v_pre += uv_pre_stride; v_dist += DIST_STRIDE; // Then all the rows except the last one mul_first = _mm_load_si128((const __m128i *)neighbors_first[1]); mul_second = _mm_load_si128((const __m128i *)neighbors_second[1]); @@ -390,15 +382,11 @@ // Only calculate the new chroma distortion if we are at a pixel that // corresponds to a new chroma row read_chroma_dist_row_16(ss_x, u_dist, v_dist, &u_first, &u_second, &v_first, &v_second); - u_src += uv_src_stride; - u_pre += uv_pre_stride; u_dist += DIST_STRIDE; - v_src += uv_src_stride; - v_pre += uv_pre_stride; v_dist += DIST_STRIDE; } sum_row_first = _mm_adds_epu16(sum_row_first, u_first); sum_row_second = _mm_adds_epu16(sum_row_second, u_second); @@ -411,11 +399,10 @@ sum_row_second = average_8(sum_row_second, &mul_second, strength, rounding, &weight_second); accumulate_and_store_16(sum_row_first, sum_row_second, y_pre, y_count, y_accum); - y_src += y_src_stride; y_pre += y_pre_stride; y_count += y_pre_stride; y_accum += y_pre_stride; y_dist += DIST_STRIDE; } @@ -456,17 +443,14 @@ y_accum); } // Perform temporal filter for the luma component. static void vp9_apply_temporal_filter_luma( - const uint8_t *y_src, int y_src_stride, const uint8_t *y_pre, - int y_pre_stride, const uint8_t *u_src, const uint8_t *v_src, - int uv_src_stride, const uint8_t *u_pre, const uint8_t *v_pre, - int uv_pre_stride, unsigned int block_width, unsigned int block_height, - int ss_x, int ss_y, int strength, const int *blk_fw, int use_whole_blk, - uint32_t *y_accum, uint16_t *y_count, const uint16_t *y_dist, - const uint16_t *u_dist, const uint16_t *v_dist) { + const uint8_t *y_pre, int y_pre_stride, unsigned int block_width, + unsigned int block_height, int ss_x, int ss_y, int strength, + const int *blk_fw, int use_whole_blk, uint32_t *y_accum, uint16_t *y_count, + const uint16_t *y_dist, const uint16_t *u_dist, const uint16_t *v_dist) { unsigned int blk_col = 0, uv_blk_col = 0; const unsigned int blk_col_step = 16, uv_blk_col_step = 16 >> ss_x; const unsigned int mid_width = block_width >> 1, last_width = block_width - blk_col_step; int top_weight = blk_fw[0], @@ -474,43 +458,36 @@ const int16_t *const *neighbors_first; const int16_t *const *neighbors_second; if (block_width == 16) { // Special Case: The blockwidth is 16 and we are operating on a row of 16 - // chroma pixels. In this case, we can't use the usualy left-midle-right + // chroma pixels. In this case, we can't use the usual left-middle-right // pattern. We also don't support splitting now. neighbors_first = LUMA_LEFT_COLUMN_NEIGHBORS; neighbors_second = LUMA_RIGHT_COLUMN_NEIGHBORS; if (use_whole_blk) { vp9_apply_temporal_filter_luma_16( - y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride, - u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, - u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, 16, - block_height, ss_x, ss_y, strength, use_whole_blk, y_accum + blk_col, - y_count + blk_col, y_dist + blk_col, u_dist + uv_blk_col, - v_dist + uv_blk_col, neighbors_first, neighbors_second, top_weight, - bottom_weight, NULL); + y_pre + blk_col, y_pre_stride, 16, block_height, ss_x, ss_y, strength, + use_whole_blk, y_accum + blk_col, y_count + blk_col, y_dist + blk_col, + u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors_first, + neighbors_second, top_weight, bottom_weight, NULL); } else { vp9_apply_temporal_filter_luma_16( - y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride, - u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, - u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, 16, - block_height, ss_x, ss_y, strength, use_whole_blk, y_accum + blk_col, - y_count + blk_col, y_dist + blk_col, u_dist + uv_blk_col, - v_dist + uv_blk_col, neighbors_first, neighbors_second, 0, 0, blk_fw); + y_pre + blk_col, y_pre_stride, 16, block_height, ss_x, ss_y, strength, + use_whole_blk, y_accum + blk_col, y_count + blk_col, y_dist + blk_col, + u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors_first, + neighbors_second, 0, 0, blk_fw); } return; } // Left neighbors_first = LUMA_LEFT_COLUMN_NEIGHBORS; neighbors_second = LUMA_MIDDLE_COLUMN_NEIGHBORS; vp9_apply_temporal_filter_luma_16( - y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride, - u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, u_pre + uv_blk_col, - v_pre + uv_blk_col, uv_pre_stride, 16, block_height, ss_x, ss_y, strength, + y_pre + blk_col, y_pre_stride, 16, block_height, ss_x, ss_y, strength, use_whole_blk, y_accum + blk_col, y_count + blk_col, y_dist + blk_col, u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors_first, neighbors_second, top_weight, bottom_weight, NULL); blk_col += blk_col_step; @@ -519,17 +496,14 @@ // Middle First neighbors_first = LUMA_MIDDLE_COLUMN_NEIGHBORS; for (; blk_col < mid_width; blk_col += blk_col_step, uv_blk_col += uv_blk_col_step) { vp9_apply_temporal_filter_luma_16( - y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride, - u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, - u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, 16, block_height, - ss_x, ss_y, strength, use_whole_blk, y_accum + blk_col, - y_count + blk_col, y_dist + blk_col, u_dist + uv_blk_col, - v_dist + uv_blk_col, neighbors_first, neighbors_second, top_weight, - bottom_weight, NULL); + y_pre + blk_col, y_pre_stride, 16, block_height, ss_x, ss_y, strength, + use_whole_blk, y_accum + blk_col, y_count + blk_col, y_dist + blk_col, + u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors_first, + neighbors_second, top_weight, bottom_weight, NULL); } if (!use_whole_blk) { top_weight = blk_fw[1]; bottom_weight = blk_fw[3]; @@ -537,39 +511,31 @@ // Middle Second for (; blk_col < last_width; blk_col += blk_col_step, uv_blk_col += uv_blk_col_step) { vp9_apply_temporal_filter_luma_16( - y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride, - u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, - u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, 16, block_height, - ss_x, ss_y, strength, use_whole_blk, y_accum + blk_col, - y_count + blk_col, y_dist + blk_col, u_dist + uv_blk_col, - v_dist + uv_blk_col, neighbors_first, neighbors_second, top_weight, - bottom_weight, NULL); + y_pre + blk_col, y_pre_stride, 16, block_height, ss_x, ss_y, strength, + use_whole_blk, y_accum + blk_col, y_count + blk_col, y_dist + blk_col, + u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors_first, + neighbors_second, top_weight, bottom_weight, NULL); } // Right neighbors_second = LUMA_RIGHT_COLUMN_NEIGHBORS; vp9_apply_temporal_filter_luma_16( - y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride, - u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, u_pre + uv_blk_col, - v_pre + uv_blk_col, uv_pre_stride, 16, block_height, ss_x, ss_y, strength, + y_pre + blk_col, y_pre_stride, 16, block_height, ss_x, ss_y, strength, use_whole_blk, y_accum + blk_col, y_count + blk_col, y_dist + blk_col, u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors_first, neighbors_second, top_weight, bottom_weight, NULL); } // Apply temporal filter to the chroma components. This performs temporal // filtering on a chroma block of 8 X uv_height. If blk_fw is not NULL, use // blk_fw as an array of size 4 for the weights for each of the 4 subblocks, // else use top_weight for top half, and bottom weight for bottom half. static void vp9_apply_temporal_filter_chroma_8( - const uint8_t *y_src, int y_src_stride, const uint8_t *y_pre, - int y_pre_stride, const uint8_t *u_src, const uint8_t *v_src, - int uv_src_stride, const uint8_t *u_pre, const uint8_t *v_pre, - int uv_pre_stride, unsigned int uv_block_width, + const uint8_t *u_pre, const uint8_t *v_pre, int uv_pre_stride, unsigned int uv_block_height, int ss_x, int ss_y, int strength, uint32_t *u_accum, uint16_t *u_count, uint32_t *v_accum, uint16_t *v_count, const uint16_t *y_dist, const uint16_t *u_dist, const uint16_t *v_dist, const int16_t *const *neighbors, int top_weight, int bottom_weight, const int *blk_fw) { @@ -585,13 +551,11 @@ __m128i u_sum_row, v_sum_row; // Loop variable unsigned int h; - (void)uv_block_width; - - // Initilize weight + // Initialize weight if (blk_fw) { weight = _mm_setr_epi16(blk_fw[0], blk_fw[0], blk_fw[0], blk_fw[0], blk_fw[1], blk_fw[1], blk_fw[1], blk_fw[1]); } else { weight = _mm_set1_epi16(top_weight); @@ -619,23 +583,19 @@ v_sum_row = average_8(v_sum_row, &mul, strength, rounding, &weight); accumulate_and_store_8(u_sum_row, u_pre, u_count, u_accum); accumulate_and_store_8(v_sum_row, v_pre, v_count, v_accum); - u_src += uv_src_stride; u_pre += uv_pre_stride; u_dist += DIST_STRIDE; - v_src += uv_src_stride; v_pre += uv_pre_stride; v_dist += DIST_STRIDE; u_count += uv_pre_stride; u_accum += uv_pre_stride; v_count += uv_pre_stride; v_accum += uv_pre_stride; - y_src += y_src_stride * (1 + ss_y); - y_pre += y_pre_stride * (1 + ss_y); y_dist += DIST_STRIDE * (1 + ss_y); // Then all the rows except the last one mul = _mm_load_si128((const __m128i *)neighbors[1]); @@ -674,23 +634,19 @@ v_sum_row = average_8(v_sum_row, &mul, strength, rounding, &weight); accumulate_and_store_8(u_sum_row, u_pre, u_count, u_accum); accumulate_and_store_8(v_sum_row, v_pre, v_count, v_accum); - u_src += uv_src_stride; u_pre += uv_pre_stride; u_dist += DIST_STRIDE; - v_src += uv_src_stride; v_pre += uv_pre_stride; v_dist += DIST_STRIDE; u_count += uv_pre_stride; u_accum += uv_pre_stride; v_count += uv_pre_stride; v_accum += uv_pre_stride; - y_src += y_src_stride * (1 + ss_y); - y_pre += y_pre_stride * (1 + ss_y); y_dist += DIST_STRIDE * (1 + ss_y); } // The last row mul = _mm_load_si128((const __m128i *)neighbors[0]); @@ -717,16 +673,14 @@ accumulate_and_store_8(v_sum_row, v_pre, v_count, v_accum); } // Perform temporal filter for the chroma components. static void vp9_apply_temporal_filter_chroma( - const uint8_t *y_src, int y_src_stride, const uint8_t *y_pre, - int y_pre_stride, const uint8_t *u_src, const uint8_t *v_src, - int uv_src_stride, const uint8_t *u_pre, const uint8_t *v_pre, - int uv_pre_stride, unsigned int block_width, unsigned int block_height, - int ss_x, int ss_y, int strength, const int *blk_fw, int use_whole_blk, - uint32_t *u_accum, uint16_t *u_count, uint32_t *v_accum, uint16_t *v_count, + const uint8_t *u_pre, const uint8_t *v_pre, int uv_pre_stride, + unsigned int block_width, unsigned int block_height, int ss_x, int ss_y, + int strength, const int *blk_fw, int use_whole_blk, uint32_t *u_accum, + uint16_t *u_count, uint32_t *v_accum, uint16_t *v_count, const uint16_t *y_dist, const uint16_t *u_dist, const uint16_t *v_dist) { const unsigned int uv_width = block_width >> ss_x, uv_height = block_height >> ss_y; unsigned int blk_col = 0, uv_blk_col = 0; @@ -749,26 +703,21 @@ neighbors = CHROMA_SINGLE_SS_SINGLE_COLUMN_NEIGHBORS; } if (use_whole_blk) { vp9_apply_temporal_filter_chroma_8( - y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride, - u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, - u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, uv_width, - uv_height, ss_x, ss_y, strength, u_accum + uv_blk_col, - u_count + uv_blk_col, v_accum + uv_blk_col, v_count + uv_blk_col, - y_dist + blk_col, u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors, - top_weight, bottom_weight, NULL); + u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, uv_height, + ss_x, ss_y, strength, u_accum + uv_blk_col, u_count + uv_blk_col, + v_accum + uv_blk_col, v_count + uv_blk_col, y_dist + blk_col, + u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors, top_weight, + bottom_weight, NULL); } else { vp9_apply_temporal_filter_chroma_8( - y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride, - u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, - u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, uv_width, - uv_height, ss_x, ss_y, strength, u_accum + uv_blk_col, - u_count + uv_blk_col, v_accum + uv_blk_col, v_count + uv_blk_col, - y_dist + blk_col, u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors, - 0, 0, blk_fw); + u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, uv_height, + ss_x, ss_y, strength, u_accum + uv_blk_col, u_count + uv_blk_col, + v_accum + uv_blk_col, v_count + uv_blk_col, y_dist + blk_col, + u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors, 0, 0, blk_fw); } return; } @@ -780,14 +729,12 @@ } else { neighbors = CHROMA_NO_SS_LEFT_COLUMN_NEIGHBORS; } vp9_apply_temporal_filter_chroma_8( - y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride, - u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, u_pre + uv_blk_col, - v_pre + uv_blk_col, uv_pre_stride, uv_width, uv_height, ss_x, ss_y, - strength, u_accum + uv_blk_col, u_count + uv_blk_col, + u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, uv_height, ss_x, + ss_y, strength, u_accum + uv_blk_col, u_count + uv_blk_col, v_accum + uv_blk_col, v_count + uv_blk_col, y_dist + blk_col, u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors, top_weight, bottom_weight, NULL); blk_col += blk_col_step; @@ -803,17 +750,15 @@ } for (; uv_blk_col < uv_mid_width; blk_col += blk_col_step, uv_blk_col += uv_blk_col_step) { vp9_apply_temporal_filter_chroma_8( - y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride, - u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, - u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, uv_width, - uv_height, ss_x, ss_y, strength, u_accum + uv_blk_col, - u_count + uv_blk_col, v_accum + uv_blk_col, v_count + uv_blk_col, - y_dist + blk_col, u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors, - top_weight, bottom_weight, NULL); + u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, uv_height, ss_x, + ss_y, strength, u_accum + uv_blk_col, u_count + uv_blk_col, + v_accum + uv_blk_col, v_count + uv_blk_col, y_dist + blk_col, + u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors, top_weight, + bottom_weight, NULL); } if (!use_whole_blk) { top_weight = blk_fw[1]; bottom_weight = blk_fw[3]; @@ -821,17 +766,15 @@ // Middle Second for (; uv_blk_col < uv_last_width; blk_col += blk_col_step, uv_blk_col += uv_blk_col_step) { vp9_apply_temporal_filter_chroma_8( - y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride, - u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, - u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, uv_width, - uv_height, ss_x, ss_y, strength, u_accum + uv_blk_col, - u_count + uv_blk_col, v_accum + uv_blk_col, v_count + uv_blk_col, - y_dist + blk_col, u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors, - top_weight, bottom_weight, NULL); + u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, uv_height, ss_x, + ss_y, strength, u_accum + uv_blk_col, u_count + uv_blk_col, + v_accum + uv_blk_col, v_count + uv_blk_col, y_dist + blk_col, + u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors, top_weight, + bottom_weight, NULL); } // Right if (ss_x && ss_y) { neighbors = CHROMA_DOUBLE_SS_RIGHT_COLUMN_NEIGHBORS; @@ -840,14 +783,12 @@ } else { neighbors = CHROMA_NO_SS_RIGHT_COLUMN_NEIGHBORS; } vp9_apply_temporal_filter_chroma_8( - y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride, - u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, u_pre + uv_blk_col, - v_pre + uv_blk_col, uv_pre_stride, uv_width, uv_height, ss_x, ss_y, - strength, u_accum + uv_blk_col, u_count + uv_blk_col, + u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, uv_height, ss_x, + ss_y, strength, u_accum + uv_blk_col, u_count + uv_blk_col, v_accum + uv_blk_col, v_count + uv_blk_col, y_dist + blk_col, u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors, top_weight, bottom_weight, NULL); } @@ -884,16 +825,16 @@ assert(strength >= 0 && strength <= 6 && "invalid temporal filter strength"); assert(blk_fw[0] >= 0 && "filter weight must be positive"); assert( (use_whole_blk || (blk_fw[1] >= 0 && blk_fw[2] >= 0 && blk_fw[3] >= 0)) && "subblock filter weight must be positive"); - assert(blk_fw[0] <= 2 && "sublock filter weight must be less than 2"); + assert(blk_fw[0] <= 2 && "subblock filter weight must be less than 2"); assert( (use_whole_blk || (blk_fw[1] <= 2 && blk_fw[2] <= 2 && blk_fw[3] <= 2)) && "subblock filter weight must be less than 2"); - // Precompute the difference sqaured + // Precompute the difference squared for (row = 0; row < block_height; row++) { for (blk_col = 0; blk_col < block_width; blk_col += 16) { store_dist_16(y_src_ptr + blk_col, y_pre_ptr + blk_col, y_dist_ptr + blk_col); } @@ -920,17 +861,15 @@ y_dist_ptr = y_dist + 1; u_dist_ptr = u_dist + 1; v_dist_ptr = v_dist + 1; - vp9_apply_temporal_filter_luma( - y_src, y_src_stride, y_pre, y_pre_stride, u_src, v_src, uv_src_stride, - u_pre, v_pre, uv_pre_stride, block_width, block_height, ss_x, ss_y, - strength, blk_fw_ptr, use_whole_blk, y_accum, y_count, y_dist_ptr, - u_dist_ptr, v_dist_ptr); + vp9_apply_temporal_filter_luma(y_pre, y_pre_stride, block_width, block_height, + ss_x, ss_y, strength, blk_fw_ptr, + use_whole_blk, y_accum, y_count, y_dist_ptr, + u_dist_ptr, v_dist_ptr); vp9_apply_temporal_filter_chroma( - y_src, y_src_stride, y_pre, y_pre_stride, u_src, v_src, uv_src_stride, u_pre, v_pre, uv_pre_stride, block_width, block_height, ss_x, ss_y, strength, blk_fw_ptr, use_whole_blk, u_accum, u_count, v_accum, v_count, y_dist_ptr, u_dist_ptr, v_dist_ptr); } diff --git a/media/libvpx/libvpx/vp9/encoder/x86/vp9_diamond_search_sad_avx.c b/media/libvpx/libvpx/vp9/encoder/x86/vp9_diamond_search_sad_avx.c --- a/media/libvpx/libvpx/vp9/encoder/x86/vp9_diamond_search_sad_avx.c +++ b/media/libvpx/libvpx/vp9/encoder/x86/vp9_diamond_search_sad_avx.c @@ -280,11 +280,18 @@ local_best_idx = _mm_extract_epi32(v_loidx_d, 0); } // Update the global minimum if the local minimum is smaller if (LIKELY(local_best_sad < best_sad)) { +#if defined(__GNUC__) && __GNUC__ >= 4 && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif new_bmv = ((const int_mv *)&v_these_mv_w)[local_best_idx]; +#if defined(__GNUC__) && __GNUC__ >= 4 && !defined(__clang__) +#pragma GCC diagnostic pop +#endif new_best_address = ((const uint8_t **)v_blocka)[local_best_idx]; best_sad = local_best_sad; } } diff --git a/media/libvpx/libvpx/vp9/encoder/x86/vp9_quantize_avx2.c b/media/libvpx/libvpx/vp9/encoder/x86/vp9_quantize_avx2.c --- a/media/libvpx/libvpx/vp9/encoder/x86/vp9_quantize_avx2.c +++ b/media/libvpx/libvpx/vp9/encoder/x86/vp9_quantize_avx2.c @@ -45,22 +45,19 @@ const __m256i iscan_plus_one = _mm256_sub_epi16(iscan, nzero_coeff0); return _mm256_and_si256(iscan_plus_one, nzero_coeff0); } void vp9_quantize_fp_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *round_ptr, - const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, - uint16_t *eob_ptr, const int16_t *scan, - const int16_t *iscan) { + const int16_t *round_ptr, const int16_t *quant_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { __m128i eob; __m256i round256, quant256, dequant256; __m256i eob256, thr256; (void)scan; - (void)skip_block; - assert(!skip_block); coeff_ptr += n_coeffs; iscan += n_coeffs; qcoeff_ptr += n_coeffs; dqcoeff_ptr += n_coeffs; diff --git a/media/libvpx/libvpx/vp9/encoder/x86/vp9_quantize_sse2.c b/media/libvpx/libvpx/vp9/encoder/x86/vp9_quantize_sse2.c --- a/media/libvpx/libvpx/vp9/encoder/x86/vp9_quantize_sse2.c +++ b/media/libvpx/libvpx/vp9/encoder/x86/vp9_quantize_sse2.c @@ -16,24 +16,21 @@ #include "vpx/vpx_integer.h" #include "vpx_dsp/vpx_dsp_common.h" #include "vpx_dsp/x86/bitdepth_conversion_sse2.h" void vp9_quantize_fp_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *round_ptr, - const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, - uint16_t *eob_ptr, const int16_t *scan, - const int16_t *iscan) { + const int16_t *round_ptr, const int16_t *quant_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { __m128i zero; __m128i thr; int nzflag; __m128i eob; __m128i round, quant, dequant; (void)scan; - (void)skip_block; - assert(!skip_block); coeff_ptr += n_coeffs; iscan += n_coeffs; qcoeff_ptr += n_coeffs; dqcoeff_ptr += n_coeffs; diff --git a/media/libvpx/libvpx/vp9/encoder/x86/vp9_quantize_ssse3_x86_64.asm b/media/libvpx/libvpx/vp9/encoder/x86/vp9_quantize_ssse3_x86_64.asm --- a/media/libvpx/libvpx/vp9/encoder/x86/vp9_quantize_ssse3_x86_64.asm +++ b/media/libvpx/libvpx/vp9/encoder/x86/vp9_quantize_ssse3_x86_64.asm @@ -17,22 +17,22 @@ pw_1: times 8 dw 1 SECTION .text %macro QUANTIZE_FP 2 -cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, skip, round, quant, \ +cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, round, quant, \ qcoeff, dqcoeff, dequant, \ eob, scan, iscan ; actual quantize loop - setup pointers, rounders, etc. movifnidn coeffq, coeffmp movifnidn ncoeffq, ncoeffmp - mov r2, dequantmp movifnidn roundq, roundmp movifnidn quantq, quantmp mova m1, [roundq] ; m1 = round mova m2, [quantq] ; m2 = quant + mov r2, dequantmp %ifidn %1, fp_32x32 pcmpeqw m5, m5 psrlw m5, 15 paddw m1, m5 psrlw m1, 1 ; m1 = (m1 + 1) / 2 diff --git a/media/libvpx/libvpx/vp9/ratectrl_rtc.h b/media/libvpx/libvpx/vp9/ratectrl_rtc.h --- a/media/libvpx/libvpx/vp9/ratectrl_rtc.h +++ b/media/libvpx/libvpx/vp9/ratectrl_rtc.h @@ -20,83 +20,48 @@ #include "vp9/vp9_iface_common.h" #include "vp9/encoder/vp9_aq_cyclicrefresh.h" #include "vp9/encoder/vp9_encoder.h" #include "vp9/encoder/vp9_firstpass.h" #include "vp9/vp9_cx_iface.h" +#include "vpx/internal/vpx_ratectrl_rtc.h" #include "vpx_mem/vpx_mem.h" namespace libvpx { -struct VP9RateControlRtcConfig { +struct VP9RateControlRtcConfig : public VpxRateControlRtcConfig { public: VP9RateControlRtcConfig() { - width = 1280; - height = 720; - max_quantizer = 63; - min_quantizer = 2; - target_bandwidth = 1000; - buf_initial_sz = 600; - buf_optimal_sz = 600; - buf_sz = 1000; - undershoot_pct = overshoot_pct = 50; - max_intra_bitrate_pct = 50; - max_inter_bitrate_pct = 0; - framerate = 30.0; - ss_number_layers = ts_number_layers = 1; - rc_mode = VPX_CBR; - aq_mode = 0; vp9_zero(max_quantizers); vp9_zero(min_quantizers); vp9_zero(scaling_factor_den); vp9_zero(scaling_factor_num); vp9_zero(layer_target_bitrate); vp9_zero(ts_rate_decimator); scaling_factor_num[0] = 1; scaling_factor_den[0] = 1; - layer_target_bitrate[0] = static_cast(target_bandwidth); max_quantizers[0] = max_quantizer; min_quantizers[0] = min_quantizer; - ts_rate_decimator[0] = 1; } - int width; - int height; - // 0-63 - int max_quantizer; - int min_quantizer; - int64_t target_bandwidth; - int64_t buf_initial_sz; - int64_t buf_optimal_sz; - int64_t buf_sz; - int undershoot_pct; - int overshoot_pct; - int max_intra_bitrate_pct; - int max_inter_bitrate_pct; - double framerate; // Number of spatial layers int ss_number_layers; // Number of temporal layers int ts_number_layers; int max_quantizers[VPX_MAX_LAYERS]; int min_quantizers[VPX_MAX_LAYERS]; int scaling_factor_num[VPX_SS_MAX_LAYERS]; int scaling_factor_den[VPX_SS_MAX_LAYERS]; - int layer_target_bitrate[VPX_MAX_LAYERS]; - int ts_rate_decimator[VPX_TS_MAX_LAYERS]; - // vbr, cbr - enum vpx_rc_mode rc_mode; - int aq_mode; }; struct VP9FrameParamsQpRTC { FRAME_TYPE frame_type; int spatial_layer_id; int temporal_layer_id; }; // This interface allows using VP9 real-time rate control without initializing -// the encoder. To use this interface, you need to link with libvp9rc.a. +// the encoder. To use this interface, you need to link with libvpxrc.a. // // #include "vp9/ratectrl_rtc.h" // VP9RateControlRTC rc_api; // VP9RateControlRtcConfig cfg; // VP9FrameParamsQpRTC frame_params; diff --git a/media/libvpx/libvpx/vp9/ratectrl_rtc.cc b/media/libvpx/libvpx/vp9/ratectrl_rtc.cc --- a/media/libvpx/libvpx/vp9/ratectrl_rtc.cc +++ b/media/libvpx/libvpx/vp9/ratectrl_rtc.cc @@ -23,19 +23,26 @@ const VP9RateControlRtcConfig &cfg) { std::unique_ptr rc_api(new (std::nothrow) VP9RateControlRTC()); if (!rc_api) return nullptr; rc_api->cpi_ = static_cast(vpx_memalign(32, sizeof(*cpi_))); - if (!rc_api->cpi_) return nullptr; + if (!rc_api->cpi_) { + rc_api.reset(); + return nullptr; + } vp9_zero(*rc_api->cpi_); rc_api->InitRateControl(cfg); if (cfg.aq_mode) { VP9_COMP *const cpi = rc_api->cpi_; cpi->segmentation_map = static_cast( vpx_calloc(cpi->common.mi_rows * cpi->common.mi_cols, sizeof(*cpi->segmentation_map))); + if (!cpi->segmentation_map) { + rc_api.reset(); + return nullptr; + } cpi->cyclic_refresh = vp9_cyclic_refresh_alloc(cpi->common.mi_rows, cpi->common.mi_cols); cpi->cyclic_refresh->content_mode = 0; } return rc_api; @@ -103,11 +110,11 @@ cpi_->oxcf.rc_max_intra_bitrate_pct = rc_cfg.max_intra_bitrate_pct; cpi_->oxcf.rc_max_inter_bitrate_pct = rc_cfg.max_inter_bitrate_pct; cpi_->framerate = rc_cfg.framerate; cpi_->svc.number_spatial_layers = rc_cfg.ss_number_layers; cpi_->svc.number_temporal_layers = rc_cfg.ts_number_layers; - + vp9_set_mb_mi(cm, cm->width, cm->height); for (int sl = 0; sl < cpi_->svc.number_spatial_layers; ++sl) { for (int tl = 0; tl < cpi_->svc.number_temporal_layers; ++tl) { const int layer = LAYER_IDS_TO_IDX(sl, tl, cpi_->svc.number_temporal_layers); LAYER_CONTEXT *lc = &cpi_->svc.layer_context[layer]; diff --git a/media/libvpx/libvpx/vp9/simple_encode.h b/media/libvpx/libvpx/vp9/simple_encode.h --- a/media/libvpx/libvpx/vp9/simple_encode.h +++ b/media/libvpx/libvpx/vp9/simple_encode.h @@ -42,10 +42,30 @@ kRefFrameTypeFuture = 2, kRefFrameTypeMax = 3, kRefFrameTypeNone = -1, }; +enum VP9_LEVEL { + LEVEL_UNKNOWN = 0, + LEVEL_AUTO = 1, + LEVEL_1 = 10, + LEVEL_1_1 = 11, + LEVEL_2 = 20, + LEVEL_2_1 = 21, + LEVEL_3 = 30, + LEVEL_3_1 = 31, + LEVEL_4 = 40, + LEVEL_4_1 = 41, + LEVEL_5 = 50, + LEVEL_5_1 = 51, + LEVEL_5_2 = 52, + LEVEL_6 = 60, + LEVEL_6_1 = 61, + LEVEL_6_2 = 62, + LEVEL_MAX = 255 +}; + enum GopMapFlag { kGopMapFlagStart = 1 << 0, // Indicate this location is the start of a group of pictures. kGopMapFlagUseAltRef = 1 << 1, // Indicate this group of pictures will use an alt ref. Only set @@ -341,11 +361,12 @@ public: // When outfile_path is set, the encoder will output the bitstream in ivf // format. SimpleEncode(int frame_width, int frame_height, int frame_rate_num, int frame_rate_den, int target_bitrate, int num_frames, - const char *infile_path, const char *outfile_path = nullptr); + int target_level, const char *infile_path, + const char *outfile_path = nullptr); ~SimpleEncode(); SimpleEncode(SimpleEncode &) = delete; SimpleEncode &operator=(const SimpleEncode &) = delete; // Adjusts the encoder's coding speed. @@ -511,10 +532,11 @@ int frame_rate_num_; int frame_rate_den_; int target_bitrate_; int num_frames_; int encode_speed_; + int target_level_; std::FILE *in_file_; std::FILE *out_file_; std::unique_ptr impl_ptr_; diff --git a/media/libvpx/libvpx/vp9/simple_encode.cc b/media/libvpx/libvpx/vp9/simple_encode.cc --- a/media/libvpx/libvpx/vp9/simple_encode.cc +++ b/media/libvpx/libvpx/vp9/simple_encode.cc @@ -108,10 +108,11 @@ static VP9_COMP *init_encoder(const VP9EncoderConfig *oxcf, vpx_img_fmt_t img_fmt) { VP9_COMP *cpi; BufferPool *buffer_pool = (BufferPool *)vpx_calloc(1, sizeof(*buffer_pool)); + if (!buffer_pool) return NULL; vp9_initialize_enc(); cpi = vp9_create_compressor(oxcf, buffer_pool); vp9_update_compressor_with_img_fmt(cpi, img_fmt); return cpi; } @@ -780,15 +781,16 @@ } } static VP9EncoderConfig GetEncodeConfig( int frame_width, int frame_height, vpx_rational_t frame_rate, - int target_bitrate, int encode_speed, vpx_enc_pass enc_pass, + int target_bitrate, int encode_speed, int target_level, + vpx_enc_pass enc_pass, const std::vector &encode_config_list) { - VP9EncoderConfig oxcf = - vp9_get_encoder_config(frame_width, frame_height, frame_rate, - target_bitrate, encode_speed, enc_pass); + VP9EncoderConfig oxcf = vp9_get_encoder_config( + frame_width, frame_height, frame_rate, target_bitrate, encode_speed, + target_level, enc_pass); for (const auto &config : encode_config_list) { UpdateEncodeConfig(config, &oxcf); } if (enc_pass == VPX_RC_FIRST_PASS) { oxcf.lag_in_frames = 0; @@ -797,20 +799,21 @@ return oxcf; } SimpleEncode::SimpleEncode(int frame_width, int frame_height, int frame_rate_num, int frame_rate_den, - int target_bitrate, int num_frames, + int target_bitrate, int num_frames, int target_level, const char *infile_path, const char *outfile_path) { impl_ptr_ = std::unique_ptr(new EncodeImpl()); frame_width_ = frame_width; frame_height_ = frame_height; frame_rate_num_ = frame_rate_num; frame_rate_den_ = frame_rate_den; target_bitrate_ = target_bitrate; num_frames_ = num_frames; encode_speed_ = 0; + target_level_ = target_level; frame_coding_index_ = 0; show_frame_count_ = 0; key_frame_group_index_ = 0; @@ -858,23 +861,23 @@ } else { enc_pass = VPX_RC_LAST_PASS; } const vpx_rational_t frame_rate = make_vpx_rational(frame_rate_num_, frame_rate_den_); - const VP9EncoderConfig oxcf = - GetEncodeConfig(frame_width_, frame_height_, frame_rate, target_bitrate_, - encode_speed_, enc_pass, impl_ptr_->encode_config_list); + const VP9EncoderConfig oxcf = GetEncodeConfig( + frame_width_, frame_height_, frame_rate, target_bitrate_, encode_speed_, + target_level_, enc_pass, impl_ptr_->encode_config_list); vp9_dump_encoder_config(&oxcf, fp); return StatusOk; } void SimpleEncode::ComputeFirstPassStats() { vpx_rational_t frame_rate = make_vpx_rational(frame_rate_num_, frame_rate_den_); const VP9EncoderConfig oxcf = GetEncodeConfig( frame_width_, frame_height_, frame_rate, target_bitrate_, encode_speed_, - VPX_RC_FIRST_PASS, impl_ptr_->encode_config_list); + target_level_, VPX_RC_FIRST_PASS, impl_ptr_->encode_config_list); impl_ptr_->cpi = init_encoder(&oxcf, impl_ptr_->img_fmt); struct lookahead_ctx *lookahead = impl_ptr_->cpi->lookahead; int i; int use_highbitdepth = 0; const int num_rows_16x16 = get_num_unit_16x16(frame_height_); @@ -1036,11 +1039,11 @@ assert(impl_ptr_->first_pass_stats.size() > 0); vpx_rational_t frame_rate = make_vpx_rational(frame_rate_num_, frame_rate_den_); VP9EncoderConfig oxcf = GetEncodeConfig( frame_width_, frame_height_, frame_rate, target_bitrate_, encode_speed_, - VPX_RC_LAST_PASS, impl_ptr_->encode_config_list); + target_level_, VPX_RC_LAST_PASS, impl_ptr_->encode_config_list); vpx_fixed_buf_t stats; stats.buf = GetVectorData(impl_ptr_->first_pass_stats); stats.sz = sizeof(impl_ptr_->first_pass_stats[0]) * impl_ptr_->first_pass_stats.size(); @@ -1264,11 +1267,11 @@ const int allow_alt_ref = 1; vpx_rational_t frame_rate = make_vpx_rational(frame_rate_num_, frame_rate_den_); const VP9EncoderConfig oxcf = GetEncodeConfig( frame_width_, frame_height_, frame_rate, target_bitrate_, encode_speed_, - VPX_RC_LAST_PASS, impl_ptr_->encode_config_list); + target_level_, VPX_RC_LAST_PASS, impl_ptr_->encode_config_list); FRAME_INFO frame_info = vp9_get_frame_info(&oxcf); fps_init_first_pass_info(&twopass.first_pass_info, GetVectorData(impl_ptr_->first_pass_stats), num_frames_); unsigned int screen_area = frame_info.frame_width * frame_info.frame_height; @@ -1283,11 +1286,11 @@ static_cast(num_frames_) + 1); vpx_rational_t frame_rate = make_vpx_rational(frame_rate_num_, frame_rate_den_); const VP9EncoderConfig oxcf = GetEncodeConfig( frame_width_, frame_height_, frame_rate, target_bitrate_, encode_speed_, - VPX_RC_LAST_PASS, impl_ptr_->encode_config_list); + target_level_, VPX_RC_LAST_PASS, impl_ptr_->encode_config_list); TWO_PASS twopass; fps_init_first_pass_info(&twopass.first_pass_info, GetVectorData(impl_ptr_->first_pass_stats), num_frames_); std::vector key_frame_map(num_frames_, 0); diff --git a/media/libvpx/libvpx/vp9/vp9_cx_iface.h b/media/libvpx/libvpx/vp9/vp9_cx_iface.h --- a/media/libvpx/libvpx/vp9/vp9_cx_iface.h +++ b/media/libvpx/libvpx/vp9/vp9_cx_iface.h @@ -18,10 +18,11 @@ #endif VP9EncoderConfig vp9_get_encoder_config(int frame_width, int frame_height, vpx_rational_t frame_rate, int target_bitrate, int encode_speed, + int target_level, vpx_enc_pass enc_pass); void vp9_dump_encoder_config(const VP9EncoderConfig *oxcf, FILE *fp); FRAME_INFO vp9_get_frame_info(const VP9EncoderConfig *oxcf); diff --git a/media/libvpx/libvpx/vp9/vp9_cx_iface.c b/media/libvpx/libvpx/vp9/vp9_cx_iface.c --- a/media/libvpx/libvpx/vp9/vp9_cx_iface.c +++ b/media/libvpx/libvpx/vp9/vp9_cx_iface.c @@ -13,11 +13,10 @@ #include "./vpx_config.h" #include "vpx/vpx_encoder.h" #include "vpx/vpx_ext_ratectrl.h" #include "vpx_dsp/psnr.h" -#include "vpx_ports/vpx_once.h" #include "vpx_ports/static_assert.h" #include "vpx_ports/system_state.h" #include "vpx_util/vpx_timestamp.h" #include "vpx/internal/vpx_codec_internal.h" #include "./vpx_version.h" @@ -379,12 +378,12 @@ case VPX_IMG_FMT_I422: case VPX_IMG_FMT_I444: case VPX_IMG_FMT_I440: if (ctx->cfg.g_profile != (unsigned int)PROFILE_1) { ERROR( - "Invalid image format. I422, I444, I440, NV12 images are " - "not supported in profile."); + "Invalid image format. I422, I444, I440 images are not supported " + "in profile."); } break; case VPX_IMG_FMT_I42216: case VPX_IMG_FMT_I44416: case VPX_IMG_FMT_I44016: @@ -395,12 +394,12 @@ "not supported in profile."); } break; default: ERROR( - "Invalid image format. Only YV12, I420, I422, I444 images are " - "supported."); + "Invalid image format. Only YV12, I420, I422, I444, I440, NV12 " + "images are supported."); break; } if (img->d_w != ctx->cfg.g_w || img->d_h != ctx->cfg.g_h) ERROR("Image size must match encoder init configuration size"); @@ -521,12 +520,13 @@ oxcf->rc_mode = cfg->rc_end_usage; raw_target_rate = (unsigned int)((int64_t)oxcf->width * oxcf->height * oxcf->bit_depth * 3 * oxcf->init_framerate / 1000); - // Cap target bitrate to raw rate - cfg->rc_target_bitrate = VPXMIN(raw_target_rate, cfg->rc_target_bitrate); + // Cap target bitrate to raw rate or 1000Mbps, whichever is less + cfg->rc_target_bitrate = + VPXMIN(VPXMIN(raw_target_rate, cfg->rc_target_bitrate), 1000000); // Convert target bandwidth from Kbit/s to Bit/s oxcf->target_bandwidth = 1000 * (int64_t)cfg->rc_target_bitrate; oxcf->rc_max_intra_bitrate_pct = extra_cfg->rc_max_intra_bitrate_pct; oxcf->rc_max_inter_bitrate_pct = extra_cfg->rc_max_inter_bitrate_pct; @@ -1093,11 +1093,11 @@ priv->cfg = *ctx->config.enc; ctx->config.enc = &priv->cfg; } priv->extra_cfg = default_extra_cfg; - once(vp9_initialize_enc); + vp9_initialize_enc(); res = validate_config(priv, &priv->cfg, &priv->extra_cfg); if (res == VPX_CODEC_OK) { priv->pts_offset_initialized = 0; @@ -1356,24 +1356,15 @@ if (ctx->base.init_flags & VPX_CODEC_USE_PSNR) cpi->b_calculate_psnr = 1; if (img != NULL) { res = image2yuvconfig(img, &sd); - if (sd.y_width != ctx->cfg.g_w || sd.y_height != ctx->cfg.g_h) { - /* from vpx_encoder.h for g_w/g_h: - "Note that the frames passed as input to the encoder must have this - resolution" - */ - ctx->base.err_detail = "Invalid input frame resolution"; - res = VPX_CODEC_INVALID_PARAM; - } else { - // Store the original flags in to the frame buffer. Will extract the - // key frame flag when we actually encode this frame. - if (vp9_receive_raw_frame(cpi, flags | ctx->next_frame_flags, &sd, - dst_time_stamp, dst_end_time_stamp)) { - res = update_error_state(ctx, &cpi->common.error); - } + // Store the original flags in to the frame buffer. Will extract the + // key frame flag when we actually encode this frame. + if (vp9_receive_raw_frame(cpi, flags | ctx->next_frame_flags, &sd, + dst_time_stamp, dst_end_time_stamp)) { + res = update_error_state(ctx, &cpi->common.error); } ctx->next_frame_flags = 0; } cx_data = ctx->cx_data; @@ -2150,10 +2141,11 @@ } VP9EncoderConfig vp9_get_encoder_config(int frame_width, int frame_height, vpx_rational_t frame_rate, int target_bitrate, int encode_speed, + int target_level, vpx_enc_pass enc_pass) { /* This function will generate the same VP9EncoderConfig used by the * vpxenc command given below. * The configs in the vpxenc command corresponds to parameters of * vp9_get_encoder_config() as follows. @@ -2161,10 +2153,11 @@ * WIDTH: frame_width * HEIGHT: frame_height * FPS: frame_rate * BITRATE: target_bitrate * CPU_USED:encode_speed + * TARGET_LEVEL: target_level * * INPUT, OUTPUT, LIMIT will not affect VP9EncoderConfig * * vpxenc command: * INPUT=bus_cif.y4m @@ -2173,19 +2166,20 @@ * HEIGHT=288 * BITRATE=600 * FPS=30/1 * LIMIT=150 * CPU_USED=0 + * TARGET_LEVEL=0 * ./vpxenc --limit=$LIMIT --width=$WIDTH --height=$HEIGHT --fps=$FPS * --lag-in-frames=25 \ * --codec=vp9 --good --cpu-used=CPU_USED --threads=0 --profile=0 \ * --min-q=0 --max-q=63 --auto-alt-ref=1 --passes=2 --kf-max-dist=150 \ * --kf-min-dist=0 --drop-frame=0 --static-thresh=0 --bias-pct=50 \ * --minsection-pct=0 --maxsection-pct=150 --arnr-maxframes=7 --psnr \ * --arnr-strength=5 --sharpness=0 --undershoot-pct=100 --overshoot-pct=100 \ * --frame-parallel=0 --tile-columns=0 --cpu-used=0 --end-usage=vbr \ - * --target-bitrate=$BITRATE -o $OUTPUT $INPUT + * --target-bitrate=$BITRATE --target-level=0 -o $OUTPUT $INPUT */ VP9EncoderConfig oxcf; vp9_extracfg extra_cfg = get_extra_cfg(); vpx_codec_enc_cfg_t enc_cfg = get_enc_cfg( @@ -2199,10 +2193,11 @@ oxcf.max_threads = 0; oxcf.tile_columns = 0; oxcf.frame_parallel_decoding_mode = 0; oxcf.two_pass_vbrmax_section = 150; oxcf.speed = abs(encode_speed); + oxcf.target_level = target_level; return oxcf; } #define DUMP_STRUCT_VALUE(fp, structure, value) \ fprintf(fp, #value " %" PRId64 "\n", (int64_t)(structure)->value) diff --git a/media/libvpx/libvpx/vp9/vp9_dx_iface.c b/media/libvpx/libvpx/vp9/vp9_dx_iface.c --- a/media/libvpx/libvpx/vp9/vp9_dx_iface.c +++ b/media/libvpx/libvpx/vp9/vp9_dx_iface.c @@ -199,11 +199,11 @@ set_error_detail(ctx, error->has_detail ? error->detail : NULL); return error->error_code; } -static void init_buffer_callbacks(vpx_codec_alg_priv_t *ctx) { +static vpx_codec_err_t init_buffer_callbacks(vpx_codec_alg_priv_t *ctx) { VP9_COMMON *const cm = &ctx->pbi->common; BufferPool *const pool = cm->buffer_pool; cm->new_fb_idx = INVALID_IDX; cm->byte_alignment = ctx->byte_alignment; @@ -215,16 +215,20 @@ pool->cb_priv = ctx->ext_priv; } else { pool->get_fb_cb = vp9_get_frame_buffer; pool->release_fb_cb = vp9_release_frame_buffer; - if (vp9_alloc_internal_frame_buffers(&pool->int_frame_buffers)) + if (vp9_alloc_internal_frame_buffers(&pool->int_frame_buffers)) { vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR, "Failed to initialize internal frame buffers"); + return VPX_CODEC_MEM_ERROR; + } pool->cb_priv = &pool->int_frame_buffers; } + + return VPX_CODEC_OK; } static void set_default_ppflags(vp8_postproc_cfg_t *cfg) { cfg->post_proc_flag = VP8_DEBLOCK | VP8_DEMACROBLOCK; cfg->deblocking_level = 4; @@ -276,13 +280,11 @@ // If postprocessing was enabled by the application and a // configuration has not been provided, default it. if (!ctx->postproc_cfg_set && (ctx->base.init_flags & VPX_CODEC_USE_POSTPROC)) set_default_ppflags(&ctx->postproc_cfg); - init_buffer_callbacks(ctx); - - return VPX_CODEC_OK; + return init_buffer_callbacks(ctx); } static INLINE void check_resync(vpx_codec_alg_priv_t *const ctx, const VP9Decoder *const pbi) { // Clear resync flag if the decoder got a key frame or intra only frame. diff --git a/media/libvpx/libvpx/vp9/vp9cx.mk b/media/libvpx/libvpx/vp9/vp9cx.mk --- a/media/libvpx/libvpx/vp9/vp9cx.mk +++ b/media/libvpx/libvpx/vp9/vp9cx.mk @@ -125,10 +125,11 @@ VP9_CX_SRCS-$(HAVE_SSSE3) += encoder/x86/vp9_quantize_ssse3_x86_64.asm endif VP9_CX_SRCS-$(HAVE_SSE2) += encoder/x86/vp9_dct_intrin_sse2.c VP9_CX_SRCS-$(HAVE_SSSE3) += encoder/x86/vp9_frame_scale_ssse3.c +VP9_CX_SRCS-$(HAVE_NEON) += encoder/arm/neon/vp9_dct_neon.c ifeq ($(CONFIG_VP9_TEMPORAL_DENOISING),yes) VP9_CX_SRCS-$(HAVE_SSE2) += encoder/x86/vp9_denoiser_sse2.c VP9_CX_SRCS-$(HAVE_NEON) += encoder/arm/neon/vp9_denoiser_neon.c endif diff --git a/media/libvpx/libvpx/vpx/internal/vpx_codec_internal.h b/media/libvpx/libvpx/vpx/internal/vpx_codec_internal.h --- a/media/libvpx/libvpx/vpx/internal/vpx_codec_internal.h +++ b/media/libvpx/libvpx/vpx/internal/vpx_codec_internal.h @@ -433,13 +433,25 @@ #undef CLANG_ANALYZER_NORETURN #define CLANG_ANALYZER_NORETURN __attribute__((analyzer_noreturn)) #endif #endif +// Tells the compiler to perform `printf` format string checking if the +// compiler supports it; see the 'format' attribute in +// . +#define LIBVPX_FORMAT_PRINTF(string_index, first_to_check) +#if defined(__has_attribute) +#if __has_attribute(format) +#undef LIBVPX_FORMAT_PRINTF +#define LIBVPX_FORMAT_PRINTF(string_index, first_to_check) \ + __attribute__((__format__(__printf__, string_index, first_to_check))) +#endif +#endif + void vpx_internal_error(struct vpx_internal_error_info *info, - vpx_codec_err_t error, const char *fmt, - ...) CLANG_ANALYZER_NORETURN; + vpx_codec_err_t error, const char *fmt, ...) + LIBVPX_FORMAT_PRINTF(3, 4) CLANG_ANALYZER_NORETURN; #ifdef __cplusplus } // extern "C" #endif diff --git a/media/libvpx/libvpx/vpx/internal/vpx_ratectrl_rtc.h b/media/libvpx/libvpx/vpx/internal/vpx_ratectrl_rtc.h new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx/internal/vpx_ratectrl_rtc.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2021 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VPX_VPX_INTERNAL_VPX_RATECTRL_RTC_H_ +#define VPX_VPX_INTERNAL_VPX_RATECTRL_RTC_H_ + +#include "vpx/vpx_encoder.h" + +namespace libvpx { +struct VpxRateControlRtcConfig { + public: + VpxRateControlRtcConfig() { + width = 1280; + height = 720; + max_quantizer = 63; + min_quantizer = 2; + target_bandwidth = 1000; + buf_initial_sz = 600; + buf_optimal_sz = 600; + buf_sz = 1000; + undershoot_pct = overshoot_pct = 50; + max_intra_bitrate_pct = 50; + max_inter_bitrate_pct = 0; + framerate = 30.0; + ts_number_layers = 1; + rc_mode = VPX_CBR; + aq_mode = 0; + layer_target_bitrate[0] = static_cast(target_bandwidth); + ts_rate_decimator[0] = 1; + } + + int width; + int height; + // 0-63 + int max_quantizer; + int min_quantizer; + int64_t target_bandwidth; + int64_t buf_initial_sz; + int64_t buf_optimal_sz; + int64_t buf_sz; + int undershoot_pct; + int overshoot_pct; + int max_intra_bitrate_pct; + int max_inter_bitrate_pct; + double framerate; + // Number of temporal layers + int ts_number_layers; + int layer_target_bitrate[VPX_MAX_LAYERS]; + int ts_rate_decimator[VPX_TS_MAX_LAYERS]; + // vbr, cbr + enum vpx_rc_mode rc_mode; + int aq_mode; +}; +} // namespace libvpx +#endif // VPX_VPX_INTERNAL_VPX_RATECTRL_RTC_H_ diff --git a/media/libvpx/libvpx/vpx/src/vpx_encoder.c b/media/libvpx/libvpx/vpx/src/vpx_encoder.c --- a/media/libvpx/libvpx/vpx/src/vpx_encoder.c +++ b/media/libvpx/libvpx/vpx/src/vpx_encoder.c @@ -171,11 +171,11 @@ * consistency with the SSE unit's 64 bit precision. */ #include "vpx_ports/x86.h" #define FLOATING_POINT_INIT() \ do { \ - unsigned short x87_orig_mode = x87_set_double_precision(); + unsigned short x87_orig_mode = x87_set_double_precision() #define FLOATING_POINT_RESTORE() \ x87_set_control_word(x87_orig_mode); \ } \ while (0) diff --git a/media/libvpx/libvpx/vpx/vp8cx.h b/media/libvpx/libvpx/vpx/vp8cx.h --- a/media/libvpx/libvpx/vpx/vp8cx.h +++ b/media/libvpx/libvpx/vpx/vp8cx.h @@ -492,22 +492,10 @@ * * Supported in codecs: VP9 */ VP9E_SET_COLOR_SPACE, - /*!\brief Codec control function to set temporal layering mode. - * \note Valid ranges: 0..3, default is "0" - * (VP9E_TEMPORAL_LAYERING_MODE_NOLAYERING). - * 0 = VP9E_TEMPORAL_LAYERING_MODE_NOLAYERING - * 1 = VP9E_TEMPORAL_LAYERING_MODE_BYPASS - * 2 = VP9E_TEMPORAL_LAYERING_MODE_0101 - * 3 = VP9E_TEMPORAL_LAYERING_MODE_0212 - * - * Supported in codecs: VP9 - */ - VP9E_SET_TEMPORAL_LAYERING_MODE, - /*!\brief Codec control function to set minimum interval between GF/ARF frames * * By default the value is set as 4. * * Supported in codecs: VP9 @@ -740,10 +728,21 @@ * The size of the array passed in should be #VPX_SS_MAX_LAYERS. * * Supported in codecs: VP9 */ VP9E_GET_LAST_QUANTIZER_SVC_LAYERS, + + /*!\brief Codec control to disable internal features in rate control. + * + * This will turn off cyclic refresh for vp8. + * + * With this, the rate control is expected to work exactly the same as the + * interface provided in vp8_ratectrl_rtc.cc/h + * + * Supported in codecs: VP8 + */ + VP8E_SET_RTC_EXTERNAL_RATECTRL, }; /*!\brief vpx 1-D scaling mode * * This set of constants define 1-D vpx scaling modes @@ -795,12 +794,12 @@ * region within a frame. */ unsigned char *roi_map; unsigned int rows; /**< Number of rows. */ unsigned int cols; /**< Number of columns. */ /*! VP8 only uses the first 4 segments. VP9 uses 8 segments. */ - int delta_q[8]; /**< Quantizer deltas. */ - int delta_lf[8]; /**< Loop filter deltas. */ + int delta_q[8]; /**< Quantizer deltas. Valid range: [-63, 63].*/ + int delta_lf[8]; /**< Loop filter deltas. Valid range: [-63, 63].*/ /*! skip and ref frame segment is only used in VP9. */ int skip[8]; /**< Skip this block. */ int ref_frame[8]; /**< Reference frame for this block. */ /*! Static breakout threshold for each segment. Only used in VP8. */ unsigned int static_threshold[4]; @@ -939,32 +938,16 @@ * Defines the data types that VP8E control functions take. Note that * additional common controls are defined in vp8.h * */ -VPX_CTRL_USE_TYPE(VP8E_SET_FRAME_FLAGS, int) -#define VPX_CTRL_VP8E_SET_FRAME_FLAGS -VPX_CTRL_USE_TYPE(VP8E_SET_TEMPORAL_LAYER_ID, int) -#define VPX_CTRL_VP8E_SET_TEMPORAL_LAYER_ID VPX_CTRL_USE_TYPE(VP8E_SET_ROI_MAP, vpx_roi_map_t *) #define VPX_CTRL_VP8E_SET_ROI_MAP -VPX_CTRL_USE_TYPE(VP9E_SET_ROI_MAP, vpx_roi_map_t *) -#define VPX_CTRL_VP9E_SET_ROI_MAP VPX_CTRL_USE_TYPE(VP8E_SET_ACTIVEMAP, vpx_active_map_t *) #define VPX_CTRL_VP8E_SET_ACTIVEMAP VPX_CTRL_USE_TYPE(VP8E_SET_SCALEMODE, vpx_scaling_mode_t *) #define VPX_CTRL_VP8E_SET_SCALEMODE - -VPX_CTRL_USE_TYPE(VP9E_SET_SVC, int) -#define VPX_CTRL_VP9E_SET_SVC -VPX_CTRL_USE_TYPE(VP9E_SET_SVC_PARAMETERS, void *) -#define VPX_CTRL_VP9E_SET_SVC_PARAMETERS -VPX_CTRL_USE_TYPE(VP9E_REGISTER_CX_CALLBACK, void *) -#define VPX_CTRL_VP9E_REGISTER_CX_CALLBACK -VPX_CTRL_USE_TYPE(VP9E_SET_SVC_LAYER_ID, vpx_svc_layer_id_t *) -#define VPX_CTRL_VP9E_SET_SVC_LAYER_ID - VPX_CTRL_USE_TYPE(VP8E_SET_CPUUSED, int) #define VPX_CTRL_VP8E_SET_CPUUSED VPX_CTRL_USE_TYPE(VP8E_SET_ENABLEAUTOALTREF, unsigned int) #define VPX_CTRL_VP8E_SET_ENABLEAUTOALTREF VPX_CTRL_USE_TYPE(VP8E_SET_NOISE_SENSITIVITY, unsigned int) @@ -973,144 +956,121 @@ #define VPX_CTRL_VP8E_SET_SHARPNESS VPX_CTRL_USE_TYPE(VP8E_SET_STATIC_THRESHOLD, unsigned int) #define VPX_CTRL_VP8E_SET_STATIC_THRESHOLD VPX_CTRL_USE_TYPE(VP8E_SET_TOKEN_PARTITIONS, int) /* vp8e_token_partitions */ #define VPX_CTRL_VP8E_SET_TOKEN_PARTITIONS - +VPX_CTRL_USE_TYPE(VP8E_GET_LAST_QUANTIZER, int *) +#define VPX_CTRL_VP8E_GET_LAST_QUANTIZER +VPX_CTRL_USE_TYPE(VP8E_GET_LAST_QUANTIZER_64, int *) +#define VPX_CTRL_VP8E_GET_LAST_QUANTIZER_64 VPX_CTRL_USE_TYPE(VP8E_SET_ARNR_MAXFRAMES, unsigned int) #define VPX_CTRL_VP8E_SET_ARNR_MAXFRAMES VPX_CTRL_USE_TYPE(VP8E_SET_ARNR_STRENGTH, unsigned int) #define VPX_CTRL_VP8E_SET_ARNR_STRENGTH VPX_CTRL_USE_TYPE_DEPRECATED(VP8E_SET_ARNR_TYPE, unsigned int) #define VPX_CTRL_VP8E_SET_ARNR_TYPE VPX_CTRL_USE_TYPE(VP8E_SET_TUNING, int) /* vp8e_tuning */ #define VPX_CTRL_VP8E_SET_TUNING VPX_CTRL_USE_TYPE(VP8E_SET_CQ_LEVEL, unsigned int) #define VPX_CTRL_VP8E_SET_CQ_LEVEL - +VPX_CTRL_USE_TYPE(VP8E_SET_MAX_INTRA_BITRATE_PCT, unsigned int) +#define VPX_CTRL_VP8E_SET_MAX_INTRA_BITRATE_PCT +VPX_CTRL_USE_TYPE(VP8E_SET_FRAME_FLAGS, int) +#define VPX_CTRL_VP8E_SET_FRAME_FLAGS +VPX_CTRL_USE_TYPE(VP9E_SET_MAX_INTER_BITRATE_PCT, unsigned int) +#define VPX_CTRL_VP9E_SET_MAX_INTER_BITRATE_PCT +VPX_CTRL_USE_TYPE(VP9E_SET_GF_CBR_BOOST_PCT, unsigned int) +#define VPX_CTRL_VP9E_SET_GF_CBR_BOOST_PCT +VPX_CTRL_USE_TYPE(VP8E_SET_TEMPORAL_LAYER_ID, int) +#define VPX_CTRL_VP8E_SET_TEMPORAL_LAYER_ID +VPX_CTRL_USE_TYPE(VP8E_SET_SCREEN_CONTENT_MODE, unsigned int) +#define VPX_CTRL_VP8E_SET_SCREEN_CONTENT_MODE +VPX_CTRL_USE_TYPE(VP9E_SET_LOSSLESS, unsigned int) +#define VPX_CTRL_VP9E_SET_LOSSLESS VPX_CTRL_USE_TYPE(VP9E_SET_TILE_COLUMNS, int) #define VPX_CTRL_VP9E_SET_TILE_COLUMNS VPX_CTRL_USE_TYPE(VP9E_SET_TILE_ROWS, int) #define VPX_CTRL_VP9E_SET_TILE_ROWS - -VPX_CTRL_USE_TYPE(VP9E_SET_TPL, int) -#define VPX_CTRL_VP9E_SET_TPL - -VPX_CTRL_USE_TYPE(VP8E_GET_LAST_QUANTIZER, int *) -#define VPX_CTRL_VP8E_GET_LAST_QUANTIZER -VPX_CTRL_USE_TYPE(VP8E_GET_LAST_QUANTIZER_64, int *) -#define VPX_CTRL_VP8E_GET_LAST_QUANTIZER_64 -VPX_CTRL_USE_TYPE(VP9E_GET_LAST_QUANTIZER_SVC_LAYERS, int *) -#define VPX_CTRL_VP9E_GET_LAST_QUANTIZER_SVC_LAYERS - -VPX_CTRL_USE_TYPE(VP9E_GET_SVC_LAYER_ID, vpx_svc_layer_id_t *) -#define VPX_CTRL_VP9E_GET_SVC_LAYER_ID - -VPX_CTRL_USE_TYPE(VP8E_SET_MAX_INTRA_BITRATE_PCT, unsigned int) -#define VPX_CTRL_VP8E_SET_MAX_INTRA_BITRATE_PCT -VPX_CTRL_USE_TYPE(VP9E_SET_MAX_INTER_BITRATE_PCT, unsigned int) -#define VPX_CTRL_VP9E_SET_MAX_INTER_BITRATE_PCT - -VPX_CTRL_USE_TYPE(VP8E_SET_GF_CBR_BOOST_PCT, unsigned int) -#define VPX_CTRL_VP8E_SET_GF_CBR_BOOST_PCT - -VPX_CTRL_USE_TYPE(VP8E_SET_SCREEN_CONTENT_MODE, unsigned int) -#define VPX_CTRL_VP8E_SET_SCREEN_CONTENT_MODE - -VPX_CTRL_USE_TYPE(VP9E_SET_GF_CBR_BOOST_PCT, unsigned int) -#define VPX_CTRL_VP9E_SET_GF_CBR_BOOST_PCT - -VPX_CTRL_USE_TYPE(VP9E_SET_LOSSLESS, unsigned int) -#define VPX_CTRL_VP9E_SET_LOSSLESS - VPX_CTRL_USE_TYPE(VP9E_SET_FRAME_PARALLEL_DECODING, unsigned int) #define VPX_CTRL_VP9E_SET_FRAME_PARALLEL_DECODING - VPX_CTRL_USE_TYPE(VP9E_SET_AQ_MODE, unsigned int) #define VPX_CTRL_VP9E_SET_AQ_MODE - -VPX_CTRL_USE_TYPE(VP9E_SET_ALT_REF_AQ, int) -#define VPX_CTRL_VP9E_SET_ALT_REF_AQ - VPX_CTRL_USE_TYPE(VP9E_SET_FRAME_PERIODIC_BOOST, unsigned int) #define VPX_CTRL_VP9E_SET_FRAME_PERIODIC_BOOST - VPX_CTRL_USE_TYPE(VP9E_SET_NOISE_SENSITIVITY, unsigned int) #define VPX_CTRL_VP9E_SET_NOISE_SENSITIVITY - +VPX_CTRL_USE_TYPE(VP9E_SET_SVC, int) +#define VPX_CTRL_VP9E_SET_SVC +VPX_CTRL_USE_TYPE(VP9E_SET_ROI_MAP, vpx_roi_map_t *) +#define VPX_CTRL_VP9E_SET_ROI_MAP +VPX_CTRL_USE_TYPE(VP9E_SET_SVC_PARAMETERS, void *) +#define VPX_CTRL_VP9E_SET_SVC_PARAMETERS +VPX_CTRL_USE_TYPE(VP9E_SET_SVC_LAYER_ID, vpx_svc_layer_id_t *) +#define VPX_CTRL_VP9E_SET_SVC_LAYER_ID VPX_CTRL_USE_TYPE(VP9E_SET_TUNE_CONTENT, int) /* vp9e_tune_content */ #define VPX_CTRL_VP9E_SET_TUNE_CONTENT - +VPX_CTRL_USE_TYPE(VP9E_GET_SVC_LAYER_ID, vpx_svc_layer_id_t *) +#define VPX_CTRL_VP9E_GET_SVC_LAYER_ID +VPX_CTRL_USE_TYPE(VP9E_REGISTER_CX_CALLBACK, void *) +#define VPX_CTRL_VP9E_REGISTER_CX_CALLBACK VPX_CTRL_USE_TYPE(VP9E_SET_COLOR_SPACE, int) #define VPX_CTRL_VP9E_SET_COLOR_SPACE - VPX_CTRL_USE_TYPE(VP9E_SET_MIN_GF_INTERVAL, unsigned int) #define VPX_CTRL_VP9E_SET_MIN_GF_INTERVAL - VPX_CTRL_USE_TYPE(VP9E_SET_MAX_GF_INTERVAL, unsigned int) #define VPX_CTRL_VP9E_SET_MAX_GF_INTERVAL - VPX_CTRL_USE_TYPE(VP9E_GET_ACTIVEMAP, vpx_active_map_t *) #define VPX_CTRL_VP9E_GET_ACTIVEMAP - VPX_CTRL_USE_TYPE(VP9E_SET_COLOR_RANGE, int) #define VPX_CTRL_VP9E_SET_COLOR_RANGE - VPX_CTRL_USE_TYPE(VP9E_SET_SVC_REF_FRAME_CONFIG, vpx_svc_ref_frame_config_t *) #define VPX_CTRL_VP9E_SET_SVC_REF_FRAME_CONFIG - VPX_CTRL_USE_TYPE(VP9E_SET_RENDER_SIZE, int *) #define VPX_CTRL_VP9E_SET_RENDER_SIZE - VPX_CTRL_USE_TYPE(VP9E_SET_TARGET_LEVEL, unsigned int) #define VPX_CTRL_VP9E_SET_TARGET_LEVEL - VPX_CTRL_USE_TYPE(VP9E_SET_ROW_MT, unsigned int) #define VPX_CTRL_VP9E_SET_ROW_MT - VPX_CTRL_USE_TYPE(VP9E_GET_LEVEL, int *) #define VPX_CTRL_VP9E_GET_LEVEL - -VPX_CTRL_USE_TYPE(VP9E_GET_LOOPFILTER_LEVEL, int *) -#define VPX_CTRL_VP9E_GET_LOOPFILTER_LEVEL - +VPX_CTRL_USE_TYPE(VP9E_SET_ALT_REF_AQ, int) +#define VPX_CTRL_VP9E_SET_ALT_REF_AQ +VPX_CTRL_USE_TYPE(VP8E_SET_GF_CBR_BOOST_PCT, unsigned int) +#define VPX_CTRL_VP8E_SET_GF_CBR_BOOST_PCT VPX_CTRL_USE_TYPE(VP9E_ENABLE_MOTION_VECTOR_UNIT_TEST, unsigned int) #define VPX_CTRL_VP9E_ENABLE_MOTION_VECTOR_UNIT_TEST - VPX_CTRL_USE_TYPE(VP9E_SET_SVC_INTER_LAYER_PRED, unsigned int) #define VPX_CTRL_VP9E_SET_SVC_INTER_LAYER_PRED - VPX_CTRL_USE_TYPE(VP9E_SET_SVC_FRAME_DROP_LAYER, vpx_svc_frame_drop_t *) #define VPX_CTRL_VP9E_SET_SVC_FRAME_DROP_LAYER - VPX_CTRL_USE_TYPE(VP9E_GET_SVC_REF_FRAME_CONFIG, vpx_svc_ref_frame_config_t *) #define VPX_CTRL_VP9E_GET_SVC_REF_FRAME_CONFIG - VPX_CTRL_USE_TYPE(VP9E_SET_SVC_GF_TEMPORAL_REF, unsigned int) #define VPX_CTRL_VP9E_SET_SVC_GF_TEMPORAL_REF - VPX_CTRL_USE_TYPE(VP9E_SET_SVC_SPATIAL_LAYER_SYNC, vpx_svc_spatial_layer_sync_t *) #define VPX_CTRL_VP9E_SET_SVC_SPATIAL_LAYER_SYNC - +VPX_CTRL_USE_TYPE(VP9E_SET_TPL, int) +#define VPX_CTRL_VP9E_SET_TPL VPX_CTRL_USE_TYPE(VP9E_SET_POSTENCODE_DROP, unsigned int) #define VPX_CTRL_VP9E_SET_POSTENCODE_DROP - VPX_CTRL_USE_TYPE(VP9E_SET_DELTA_Q_UV, int) #define VPX_CTRL_VP9E_SET_DELTA_Q_UV - VPX_CTRL_USE_TYPE(VP9E_SET_DISABLE_OVERSHOOT_MAXQ_CBR, int) #define VPX_CTRL_VP9E_SET_DISABLE_OVERSHOOT_MAXQ_CBR - VPX_CTRL_USE_TYPE(VP9E_SET_DISABLE_LOOPFILTER, int) #define VPX_CTRL_VP9E_SET_DISABLE_LOOPFILTER - +VPX_CTRL_USE_TYPE(VP9E_SET_EXTERNAL_RATE_CONTROL, vpx_rc_funcs_t *) +#define VPX_CTRL_VP9E_SET_EXTERNAL_RATE_CONTROL VPX_CTRL_USE_TYPE(VP9E_SET_RTC_EXTERNAL_RATECTRL, int) #define VPX_CTRL_VP9E_SET_RTC_EXTERNAL_RATECTRL - -VPX_CTRL_USE_TYPE(VP9E_SET_EXTERNAL_RATE_CONTROL, vpx_rc_funcs_t *) -#define VPX_CTRL_VP9E_SET_EXTERNAL_RATE_CONTROL +VPX_CTRL_USE_TYPE(VP9E_GET_LOOPFILTER_LEVEL, int *) +#define VPX_CTRL_VP9E_GET_LOOPFILTER_LEVEL +VPX_CTRL_USE_TYPE(VP9E_GET_LAST_QUANTIZER_SVC_LAYERS, int *) +#define VPX_CTRL_VP9E_GET_LAST_QUANTIZER_SVC_LAYERS +VPX_CTRL_USE_TYPE(VP8E_SET_RTC_EXTERNAL_RATECTRL, int) +#define VPX_CTRL_VP8E_SET_RTC_EXTERNAL_RATECTRL /*!\endcond */ /*! @} - end defgroup vp8_encoder */ #ifdef __cplusplus } // extern "C" diff --git a/media/libvpx/libvpx/vpx/vp8dx.h b/media/libvpx/libvpx/vpx/vp8dx.h --- a/media/libvpx/libvpx/vpx/vp8dx.h +++ b/media/libvpx/libvpx/vpx/vp8dx.h @@ -175,32 +175,34 @@ #define VPX_CTRL_VP8D_GET_LAST_REF_UPDATES VPX_CTRL_USE_TYPE(VP8D_GET_FRAME_CORRUPTED, int *) #define VPX_CTRL_VP8D_GET_FRAME_CORRUPTED VPX_CTRL_USE_TYPE(VP8D_GET_LAST_REF_USED, int *) #define VPX_CTRL_VP8D_GET_LAST_REF_USED -VPX_CTRL_USE_TYPE(VPXD_GET_LAST_QUANTIZER, int *) -#define VPX_CTRL_VPXD_GET_LAST_QUANTIZER VPX_CTRL_USE_TYPE(VPXD_SET_DECRYPTOR, vpx_decrypt_init *) #define VPX_CTRL_VPXD_SET_DECRYPTOR VPX_CTRL_USE_TYPE(VP8D_SET_DECRYPTOR, vpx_decrypt_init *) #define VPX_CTRL_VP8D_SET_DECRYPTOR +VPX_CTRL_USE_TYPE(VP9D_GET_FRAME_SIZE, int *) +#define VPX_CTRL_VP9D_GET_FRAME_SIZE VPX_CTRL_USE_TYPE(VP9D_GET_DISPLAY_SIZE, int *) #define VPX_CTRL_VP9D_GET_DISPLAY_SIZE VPX_CTRL_USE_TYPE(VP9D_GET_BIT_DEPTH, unsigned int *) #define VPX_CTRL_VP9D_GET_BIT_DEPTH -VPX_CTRL_USE_TYPE(VP9D_GET_FRAME_SIZE, int *) -#define VPX_CTRL_VP9D_GET_FRAME_SIZE +VPX_CTRL_USE_TYPE(VP9_SET_BYTE_ALIGNMENT, int) +#define VPX_CTRL_VP9_SET_BYTE_ALIGNMENT VPX_CTRL_USE_TYPE(VP9_INVERT_TILE_DECODE_ORDER, int) #define VPX_CTRL_VP9_INVERT_TILE_DECODE_ORDER -#define VPX_CTRL_VP9_DECODE_SVC_SPATIAL_LAYER -VPX_CTRL_USE_TYPE(VP9_DECODE_SVC_SPATIAL_LAYER, int) +VPX_CTRL_USE_TYPE(VP9_SET_SKIP_LOOP_FILTER, int) #define VPX_CTRL_VP9_SET_SKIP_LOOP_FILTER -VPX_CTRL_USE_TYPE(VP9_SET_SKIP_LOOP_FILTER, int) -#define VPX_CTRL_VP9_DECODE_SET_ROW_MT +VPX_CTRL_USE_TYPE(VP9_DECODE_SVC_SPATIAL_LAYER, int) +#define VPX_CTRL_VP9_DECODE_SVC_SPATIAL_LAYER +VPX_CTRL_USE_TYPE(VPXD_GET_LAST_QUANTIZER, int *) +#define VPX_CTRL_VPXD_GET_LAST_QUANTIZER VPX_CTRL_USE_TYPE(VP9D_SET_ROW_MT, int) +#define VPX_CTRL_VP9_DECODE_SET_ROW_MT +VPX_CTRL_USE_TYPE(VP9D_SET_LOOP_FILTER_OPT, int) #define VPX_CTRL_VP9_SET_LOOP_FILTER_OPT -VPX_CTRL_USE_TYPE(VP9D_SET_LOOP_FILTER_OPT, int) /*!\endcond */ /*! @} - end defgroup vp8_decoder */ #ifdef __cplusplus diff --git a/media/libvpx/libvpx/vpx/vpx_codec.mk b/media/libvpx/libvpx/vpx/vpx_codec.mk --- a/media/libvpx/libvpx/vpx/vpx_codec.mk +++ b/media/libvpx/libvpx/vpx/vpx_codec.mk @@ -31,10 +31,11 @@ API_SRCS-yes += src/vpx_decoder.c API_SRCS-yes += vpx_decoder.h API_SRCS-yes += src/vpx_encoder.c API_SRCS-yes += vpx_encoder.h API_SRCS-yes += internal/vpx_codec_internal.h +API_SRCS-yes += internal/vpx_ratectrl_rtc.h API_SRCS-yes += src/vpx_codec.c API_SRCS-yes += src/vpx_image.c API_SRCS-yes += vpx_codec.h API_SRCS-yes += vpx_codec.mk API_SRCS-yes += vpx_frame_buffer.h diff --git a/media/libvpx/libvpx/vpx_dsp/arm/fdct16x16_neon.h b/media/libvpx/libvpx/vpx_dsp/arm/fdct16x16_neon.h new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/arm/fdct16x16_neon.h @@ -0,0 +1,327 @@ +/* + * Copyright (c) 2017 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VPX_VPX_DSP_ARM_FDCT16X16_NEON_H_ +#define VPX_VPX_DSP_ARM_FDCT16X16_NEON_H_ + +#include + +static INLINE void load(const int16_t *a, int stride, int16x8_t *b /*[16]*/) { + b[0] = vld1q_s16(a); + a += stride; + b[1] = vld1q_s16(a); + a += stride; + b[2] = vld1q_s16(a); + a += stride; + b[3] = vld1q_s16(a); + a += stride; + b[4] = vld1q_s16(a); + a += stride; + b[5] = vld1q_s16(a); + a += stride; + b[6] = vld1q_s16(a); + a += stride; + b[7] = vld1q_s16(a); + a += stride; + b[8] = vld1q_s16(a); + a += stride; + b[9] = vld1q_s16(a); + a += stride; + b[10] = vld1q_s16(a); + a += stride; + b[11] = vld1q_s16(a); + a += stride; + b[12] = vld1q_s16(a); + a += stride; + b[13] = vld1q_s16(a); + a += stride; + b[14] = vld1q_s16(a); + a += stride; + b[15] = vld1q_s16(a); +} + +// Store 8 16x8 values, assuming stride == 16. +static INLINE void store(tran_low_t *a, const int16x8_t *b /*[8]*/) { + store_s16q_to_tran_low(a, b[0]); + a += 16; + store_s16q_to_tran_low(a, b[1]); + a += 16; + store_s16q_to_tran_low(a, b[2]); + a += 16; + store_s16q_to_tran_low(a, b[3]); + a += 16; + store_s16q_to_tran_low(a, b[4]); + a += 16; + store_s16q_to_tran_low(a, b[5]); + a += 16; + store_s16q_to_tran_low(a, b[6]); + a += 16; + store_s16q_to_tran_low(a, b[7]); +} + +// Load step of each pass. Add and subtract clear across the input, requiring +// all 16 values to be loaded. For the first pass it also multiplies by 4. + +// To maybe reduce register usage this could be combined with the load() step to +// get the first 4 and last 4 values, cross those, then load the middle 8 values +// and cross them. +static INLINE void cross_input(const int16x8_t *a /*[16]*/, + int16x8_t *b /*[16]*/, const int pass) { + if (pass == 0) { + b[0] = vshlq_n_s16(vaddq_s16(a[0], a[15]), 2); + b[1] = vshlq_n_s16(vaddq_s16(a[1], a[14]), 2); + b[2] = vshlq_n_s16(vaddq_s16(a[2], a[13]), 2); + b[3] = vshlq_n_s16(vaddq_s16(a[3], a[12]), 2); + b[4] = vshlq_n_s16(vaddq_s16(a[4], a[11]), 2); + b[5] = vshlq_n_s16(vaddq_s16(a[5], a[10]), 2); + b[6] = vshlq_n_s16(vaddq_s16(a[6], a[9]), 2); + b[7] = vshlq_n_s16(vaddq_s16(a[7], a[8]), 2); + + b[8] = vshlq_n_s16(vsubq_s16(a[7], a[8]), 2); + b[9] = vshlq_n_s16(vsubq_s16(a[6], a[9]), 2); + b[10] = vshlq_n_s16(vsubq_s16(a[5], a[10]), 2); + b[11] = vshlq_n_s16(vsubq_s16(a[4], a[11]), 2); + b[12] = vshlq_n_s16(vsubq_s16(a[3], a[12]), 2); + b[13] = vshlq_n_s16(vsubq_s16(a[2], a[13]), 2); + b[14] = vshlq_n_s16(vsubq_s16(a[1], a[14]), 2); + b[15] = vshlq_n_s16(vsubq_s16(a[0], a[15]), 2); + } else { + b[0] = vaddq_s16(a[0], a[15]); + b[1] = vaddq_s16(a[1], a[14]); + b[2] = vaddq_s16(a[2], a[13]); + b[3] = vaddq_s16(a[3], a[12]); + b[4] = vaddq_s16(a[4], a[11]); + b[5] = vaddq_s16(a[5], a[10]); + b[6] = vaddq_s16(a[6], a[9]); + b[7] = vaddq_s16(a[7], a[8]); + + b[8] = vsubq_s16(a[7], a[8]); + b[9] = vsubq_s16(a[6], a[9]); + b[10] = vsubq_s16(a[5], a[10]); + b[11] = vsubq_s16(a[4], a[11]); + b[12] = vsubq_s16(a[3], a[12]); + b[13] = vsubq_s16(a[2], a[13]); + b[14] = vsubq_s16(a[1], a[14]); + b[15] = vsubq_s16(a[0], a[15]); + } +} + +// Quarter round at the beginning of the second pass. Can't use vrshr (rounding) +// because this only adds 1, not 1 << 2. +static INLINE void partial_round_shift(int16x8_t *a /*[16]*/) { + const int16x8_t one = vdupq_n_s16(1); + a[0] = vshrq_n_s16(vaddq_s16(a[0], one), 2); + a[1] = vshrq_n_s16(vaddq_s16(a[1], one), 2); + a[2] = vshrq_n_s16(vaddq_s16(a[2], one), 2); + a[3] = vshrq_n_s16(vaddq_s16(a[3], one), 2); + a[4] = vshrq_n_s16(vaddq_s16(a[4], one), 2); + a[5] = vshrq_n_s16(vaddq_s16(a[5], one), 2); + a[6] = vshrq_n_s16(vaddq_s16(a[6], one), 2); + a[7] = vshrq_n_s16(vaddq_s16(a[7], one), 2); + a[8] = vshrq_n_s16(vaddq_s16(a[8], one), 2); + a[9] = vshrq_n_s16(vaddq_s16(a[9], one), 2); + a[10] = vshrq_n_s16(vaddq_s16(a[10], one), 2); + a[11] = vshrq_n_s16(vaddq_s16(a[11], one), 2); + a[12] = vshrq_n_s16(vaddq_s16(a[12], one), 2); + a[13] = vshrq_n_s16(vaddq_s16(a[13], one), 2); + a[14] = vshrq_n_s16(vaddq_s16(a[14], one), 2); + a[15] = vshrq_n_s16(vaddq_s16(a[15], one), 2); +} + +// fdct_round_shift((a +/- b) * c) +static INLINE void butterfly_one_coeff(const int16x8_t a, const int16x8_t b, + const tran_high_t c, int16x8_t *add, + int16x8_t *sub) { + const int32x4_t a0 = vmull_n_s16(vget_low_s16(a), c); + const int32x4_t a1 = vmull_n_s16(vget_high_s16(a), c); + const int32x4_t sum0 = vmlal_n_s16(a0, vget_low_s16(b), c); + const int32x4_t sum1 = vmlal_n_s16(a1, vget_high_s16(b), c); + const int32x4_t diff0 = vmlsl_n_s16(a0, vget_low_s16(b), c); + const int32x4_t diff1 = vmlsl_n_s16(a1, vget_high_s16(b), c); + const int16x4_t rounded0 = vqrshrn_n_s32(sum0, 14); + const int16x4_t rounded1 = vqrshrn_n_s32(sum1, 14); + const int16x4_t rounded2 = vqrshrn_n_s32(diff0, 14); + const int16x4_t rounded3 = vqrshrn_n_s32(diff1, 14); + *add = vcombine_s16(rounded0, rounded1); + *sub = vcombine_s16(rounded2, rounded3); +} + +// fdct_round_shift(a * c0 +/- b * c1) +static INLINE void butterfly_two_coeff(const int16x8_t a, const int16x8_t b, + const tran_coef_t c0, + const tran_coef_t c1, int16x8_t *add, + int16x8_t *sub) { + const int32x4_t a0 = vmull_n_s16(vget_low_s16(a), c0); + const int32x4_t a1 = vmull_n_s16(vget_high_s16(a), c0); + const int32x4_t a2 = vmull_n_s16(vget_low_s16(a), c1); + const int32x4_t a3 = vmull_n_s16(vget_high_s16(a), c1); + const int32x4_t sum0 = vmlal_n_s16(a2, vget_low_s16(b), c0); + const int32x4_t sum1 = vmlal_n_s16(a3, vget_high_s16(b), c0); + const int32x4_t diff0 = vmlsl_n_s16(a0, vget_low_s16(b), c1); + const int32x4_t diff1 = vmlsl_n_s16(a1, vget_high_s16(b), c1); + const int16x4_t rounded0 = vqrshrn_n_s32(sum0, 14); + const int16x4_t rounded1 = vqrshrn_n_s32(sum1, 14); + const int16x4_t rounded2 = vqrshrn_n_s32(diff0, 14); + const int16x4_t rounded3 = vqrshrn_n_s32(diff1, 14); + *add = vcombine_s16(rounded0, rounded1); + *sub = vcombine_s16(rounded2, rounded3); +} + +// Transpose 8x8 to a new location. Don't use transpose_neon.h because those +// are all in-place. +static INLINE void transpose_8x8(const int16x8_t *a /*[8]*/, + int16x8_t *b /*[8]*/) { + // Swap 16 bit elements. + const int16x8x2_t c0 = vtrnq_s16(a[0], a[1]); + const int16x8x2_t c1 = vtrnq_s16(a[2], a[3]); + const int16x8x2_t c2 = vtrnq_s16(a[4], a[5]); + const int16x8x2_t c3 = vtrnq_s16(a[6], a[7]); + + // Swap 32 bit elements. + const int32x4x2_t d0 = vtrnq_s32(vreinterpretq_s32_s16(c0.val[0]), + vreinterpretq_s32_s16(c1.val[0])); + const int32x4x2_t d1 = vtrnq_s32(vreinterpretq_s32_s16(c0.val[1]), + vreinterpretq_s32_s16(c1.val[1])); + const int32x4x2_t d2 = vtrnq_s32(vreinterpretq_s32_s16(c2.val[0]), + vreinterpretq_s32_s16(c3.val[0])); + const int32x4x2_t d3 = vtrnq_s32(vreinterpretq_s32_s16(c2.val[1]), + vreinterpretq_s32_s16(c3.val[1])); + + // Swap 64 bit elements + const int16x8x2_t e0 = vpx_vtrnq_s64_to_s16(d0.val[0], d2.val[0]); + const int16x8x2_t e1 = vpx_vtrnq_s64_to_s16(d1.val[0], d3.val[0]); + const int16x8x2_t e2 = vpx_vtrnq_s64_to_s16(d0.val[1], d2.val[1]); + const int16x8x2_t e3 = vpx_vtrnq_s64_to_s16(d1.val[1], d3.val[1]); + + b[0] = e0.val[0]; + b[1] = e1.val[0]; + b[2] = e2.val[0]; + b[3] = e3.val[0]; + b[4] = e0.val[1]; + b[5] = e1.val[1]; + b[6] = e2.val[1]; + b[7] = e3.val[1]; +} + +// Main body of fdct16x16. +static void vpx_fdct16x16_body(const int16x8_t *in /*[16]*/, + int16x8_t *out /*[16]*/) { + int16x8_t s[8]; + int16x8_t x[4]; + int16x8_t step[8]; + + // stage 1 + // From fwd_txfm.c: Work on the first eight values; fdct8(input, + // even_results);" + s[0] = vaddq_s16(in[0], in[7]); + s[1] = vaddq_s16(in[1], in[6]); + s[2] = vaddq_s16(in[2], in[5]); + s[3] = vaddq_s16(in[3], in[4]); + s[4] = vsubq_s16(in[3], in[4]); + s[5] = vsubq_s16(in[2], in[5]); + s[6] = vsubq_s16(in[1], in[6]); + s[7] = vsubq_s16(in[0], in[7]); + + // fdct4(step, step); + x[0] = vaddq_s16(s[0], s[3]); + x[1] = vaddq_s16(s[1], s[2]); + x[2] = vsubq_s16(s[1], s[2]); + x[3] = vsubq_s16(s[0], s[3]); + + // out[0] = fdct_round_shift((x0 + x1) * cospi_16_64) + // out[8] = fdct_round_shift((x0 - x1) * cospi_16_64) + butterfly_one_coeff(x[0], x[1], cospi_16_64, &out[0], &out[8]); + // out[4] = fdct_round_shift(x3 * cospi_8_64 + x2 * cospi_24_64); + // out[12] = fdct_round_shift(x3 * cospi_24_64 - x2 * cospi_8_64); + butterfly_two_coeff(x[3], x[2], cospi_24_64, cospi_8_64, &out[4], &out[12]); + + // Stage 2 + // Re-using source s5/s6 + // s5 = fdct_round_shift((s6 - s5) * cospi_16_64) + // s6 = fdct_round_shift((s6 + s5) * cospi_16_64) + butterfly_one_coeff(s[6], s[5], cospi_16_64, &s[6], &s[5]); + + // Stage 3 + x[0] = vaddq_s16(s[4], s[5]); + x[1] = vsubq_s16(s[4], s[5]); + x[2] = vsubq_s16(s[7], s[6]); + x[3] = vaddq_s16(s[7], s[6]); + + // Stage 4 + // out[2] = fdct_round_shift(x0 * cospi_28_64 + x3 * cospi_4_64) + // out[14] = fdct_round_shift(x3 * cospi_28_64 + x0 * -cospi_4_64) + butterfly_two_coeff(x[3], x[0], cospi_28_64, cospi_4_64, &out[2], &out[14]); + // out[6] = fdct_round_shift(x1 * cospi_12_64 + x2 * cospi_20_64) + // out[10] = fdct_round_shift(x2 * cospi_12_64 + x1 * -cospi_20_64) + butterfly_two_coeff(x[2], x[1], cospi_12_64, cospi_20_64, &out[10], &out[6]); + + // step 2 + // From fwd_txfm.c: Work on the next eight values; step1 -> odd_results" + // That file distinguished between "in_high" and "step1" but the only + // difference is that "in_high" is the first 8 values and "step 1" is the + // second. Here, since they are all in one array, "step1" values are += 8. + + // step2[2] = fdct_round_shift((step1[5] - step1[2]) * cospi_16_64) + // step2[3] = fdct_round_shift((step1[4] - step1[3]) * cospi_16_64) + // step2[4] = fdct_round_shift((step1[4] + step1[3]) * cospi_16_64) + // step2[5] = fdct_round_shift((step1[5] + step1[2]) * cospi_16_64) + butterfly_one_coeff(in[13], in[10], cospi_16_64, &s[5], &s[2]); + butterfly_one_coeff(in[12], in[11], cospi_16_64, &s[4], &s[3]); + + // step 3 + s[0] = vaddq_s16(in[8], s[3]); + s[1] = vaddq_s16(in[9], s[2]); + x[0] = vsubq_s16(in[9], s[2]); + x[1] = vsubq_s16(in[8], s[3]); + x[2] = vsubq_s16(in[15], s[4]); + x[3] = vsubq_s16(in[14], s[5]); + s[6] = vaddq_s16(in[14], s[5]); + s[7] = vaddq_s16(in[15], s[4]); + + // step 4 + // step2[1] = fdct_round_shift(step3[1] *-cospi_8_64 + step3[6] * cospi_24_64) + // step2[6] = fdct_round_shift(step3[1] * cospi_24_64 + step3[6] * cospi_8_64) + butterfly_two_coeff(s[6], s[1], cospi_24_64, cospi_8_64, &s[6], &s[1]); + + // step2[2] = fdct_round_shift(step3[2] * cospi_24_64 + step3[5] * cospi_8_64) + // step2[5] = fdct_round_shift(step3[2] * cospi_8_64 - step3[5] * cospi_24_64) + butterfly_two_coeff(x[0], x[3], cospi_8_64, cospi_24_64, &s[2], &s[5]); + + // step 5 + step[0] = vaddq_s16(s[0], s[1]); + step[1] = vsubq_s16(s[0], s[1]); + step[2] = vaddq_s16(x[1], s[2]); + step[3] = vsubq_s16(x[1], s[2]); + step[4] = vsubq_s16(x[2], s[5]); + step[5] = vaddq_s16(x[2], s[5]); + step[6] = vsubq_s16(s[7], s[6]); + step[7] = vaddq_s16(s[7], s[6]); + + // step 6 + // out[1] = fdct_round_shift(step1[0] * cospi_30_64 + step1[7] * cospi_2_64) + // out[9] = fdct_round_shift(step1[1] * cospi_14_64 + step1[6] * cospi_18_64) + // out[5] = fdct_round_shift(step1[2] * cospi_22_64 + step1[5] * cospi_10_64) + // out[13] = fdct_round_shift(step1[3] * cospi_6_64 + step1[4] * cospi_26_64) + // out[3] = fdct_round_shift(step1[3] * -cospi_26_64 + step1[4] * cospi_6_64) + // out[11] = fdct_round_shift(step1[2] * -cospi_10_64 + step1[5] * + // cospi_22_64) + // out[7] = fdct_round_shift(step1[1] * -cospi_18_64 + step1[6] * cospi_14_64) + // out[15] = fdct_round_shift(step1[0] * -cospi_2_64 + step1[7] * cospi_30_64) + butterfly_two_coeff(step[6], step[1], cospi_14_64, cospi_18_64, &out[9], + &out[7]); + butterfly_two_coeff(step[7], step[0], cospi_30_64, cospi_2_64, &out[1], + &out[15]); + butterfly_two_coeff(step[4], step[3], cospi_6_64, cospi_26_64, &out[13], + &out[3]); + butterfly_two_coeff(step[5], step[2], cospi_22_64, cospi_10_64, &out[5], + &out[11]); +} + +#endif // VPX_VPX_DSP_ARM_FDCT16X16_NEON_H_ diff --git a/media/libvpx/libvpx/vpx_dsp/arm/fdct16x16_neon.c b/media/libvpx/libvpx/vpx_dsp/arm/fdct16x16_neon.c --- a/media/libvpx/libvpx/vpx_dsp/arm/fdct16x16_neon.c +++ b/media/libvpx/libvpx/vpx_dsp/arm/fdct16x16_neon.c @@ -13,10 +13,11 @@ #include "./vpx_config.h" #include "./vpx_dsp_rtcd.h" #include "vpx_dsp/txfm_common.h" #include "vpx_dsp/arm/mem_neon.h" #include "vpx_dsp/arm/transpose_neon.h" +#include "vpx_dsp/arm/fdct16x16_neon.h" // Some builds of gcc 4.9.2 and .3 have trouble with some of the inline // functions. #if !defined(__clang__) && !defined(__ANDROID__) && defined(__GNUC__) && \ __GNUC__ == 4 && __GNUC_MINOR__ == 9 && __GNUC_PATCHLEVEL__ < 4 @@ -25,343 +26,33 @@ vpx_fdct16x16_c(input, output, stride); } #else -static INLINE void load(const int16_t *a, int stride, int16x8_t *b /*[16]*/) { - b[0] = vld1q_s16(a); - a += stride; - b[1] = vld1q_s16(a); - a += stride; - b[2] = vld1q_s16(a); - a += stride; - b[3] = vld1q_s16(a); - a += stride; - b[4] = vld1q_s16(a); - a += stride; - b[5] = vld1q_s16(a); - a += stride; - b[6] = vld1q_s16(a); - a += stride; - b[7] = vld1q_s16(a); - a += stride; - b[8] = vld1q_s16(a); - a += stride; - b[9] = vld1q_s16(a); - a += stride; - b[10] = vld1q_s16(a); - a += stride; - b[11] = vld1q_s16(a); - a += stride; - b[12] = vld1q_s16(a); - a += stride; - b[13] = vld1q_s16(a); - a += stride; - b[14] = vld1q_s16(a); - a += stride; - b[15] = vld1q_s16(a); -} - -// Store 8 16x8 values, assuming stride == 16. -static INLINE void store(tran_low_t *a, const int16x8_t *b /*[8]*/) { - store_s16q_to_tran_low(a, b[0]); - a += 16; - store_s16q_to_tran_low(a, b[1]); - a += 16; - store_s16q_to_tran_low(a, b[2]); - a += 16; - store_s16q_to_tran_low(a, b[3]); - a += 16; - store_s16q_to_tran_low(a, b[4]); - a += 16; - store_s16q_to_tran_low(a, b[5]); - a += 16; - store_s16q_to_tran_low(a, b[6]); - a += 16; - store_s16q_to_tran_low(a, b[7]); -} - -// Load step of each pass. Add and subtract clear across the input, requiring -// all 16 values to be loaded. For the first pass it also multiplies by 4. - -// To maybe reduce register usage this could be combined with the load() step to -// get the first 4 and last 4 values, cross those, then load the middle 8 values -// and cross them. -static INLINE void cross_input(const int16x8_t *a /*[16]*/, - int16x8_t *b /*[16]*/, const int pass) { - if (pass == 0) { - b[0] = vshlq_n_s16(vaddq_s16(a[0], a[15]), 2); - b[1] = vshlq_n_s16(vaddq_s16(a[1], a[14]), 2); - b[2] = vshlq_n_s16(vaddq_s16(a[2], a[13]), 2); - b[3] = vshlq_n_s16(vaddq_s16(a[3], a[12]), 2); - b[4] = vshlq_n_s16(vaddq_s16(a[4], a[11]), 2); - b[5] = vshlq_n_s16(vaddq_s16(a[5], a[10]), 2); - b[6] = vshlq_n_s16(vaddq_s16(a[6], a[9]), 2); - b[7] = vshlq_n_s16(vaddq_s16(a[7], a[8]), 2); - - b[8] = vshlq_n_s16(vsubq_s16(a[7], a[8]), 2); - b[9] = vshlq_n_s16(vsubq_s16(a[6], a[9]), 2); - b[10] = vshlq_n_s16(vsubq_s16(a[5], a[10]), 2); - b[11] = vshlq_n_s16(vsubq_s16(a[4], a[11]), 2); - b[12] = vshlq_n_s16(vsubq_s16(a[3], a[12]), 2); - b[13] = vshlq_n_s16(vsubq_s16(a[2], a[13]), 2); - b[14] = vshlq_n_s16(vsubq_s16(a[1], a[14]), 2); - b[15] = vshlq_n_s16(vsubq_s16(a[0], a[15]), 2); - } else { - b[0] = vaddq_s16(a[0], a[15]); - b[1] = vaddq_s16(a[1], a[14]); - b[2] = vaddq_s16(a[2], a[13]); - b[3] = vaddq_s16(a[3], a[12]); - b[4] = vaddq_s16(a[4], a[11]); - b[5] = vaddq_s16(a[5], a[10]); - b[6] = vaddq_s16(a[6], a[9]); - b[7] = vaddq_s16(a[7], a[8]); - - b[8] = vsubq_s16(a[7], a[8]); - b[9] = vsubq_s16(a[6], a[9]); - b[10] = vsubq_s16(a[5], a[10]); - b[11] = vsubq_s16(a[4], a[11]); - b[12] = vsubq_s16(a[3], a[12]); - b[13] = vsubq_s16(a[2], a[13]); - b[14] = vsubq_s16(a[1], a[14]); - b[15] = vsubq_s16(a[0], a[15]); - } -} - -// Quarter round at the beginning of the second pass. Can't use vrshr (rounding) -// because this only adds 1, not 1 << 2. -static INLINE void partial_round_shift(int16x8_t *a /*[16]*/) { - const int16x8_t one = vdupq_n_s16(1); - a[0] = vshrq_n_s16(vaddq_s16(a[0], one), 2); - a[1] = vshrq_n_s16(vaddq_s16(a[1], one), 2); - a[2] = vshrq_n_s16(vaddq_s16(a[2], one), 2); - a[3] = vshrq_n_s16(vaddq_s16(a[3], one), 2); - a[4] = vshrq_n_s16(vaddq_s16(a[4], one), 2); - a[5] = vshrq_n_s16(vaddq_s16(a[5], one), 2); - a[6] = vshrq_n_s16(vaddq_s16(a[6], one), 2); - a[7] = vshrq_n_s16(vaddq_s16(a[7], one), 2); - a[8] = vshrq_n_s16(vaddq_s16(a[8], one), 2); - a[9] = vshrq_n_s16(vaddq_s16(a[9], one), 2); - a[10] = vshrq_n_s16(vaddq_s16(a[10], one), 2); - a[11] = vshrq_n_s16(vaddq_s16(a[11], one), 2); - a[12] = vshrq_n_s16(vaddq_s16(a[12], one), 2); - a[13] = vshrq_n_s16(vaddq_s16(a[13], one), 2); - a[14] = vshrq_n_s16(vaddq_s16(a[14], one), 2); - a[15] = vshrq_n_s16(vaddq_s16(a[15], one), 2); -} - -// fdct_round_shift((a +/- b) * c) -static INLINE void butterfly_one_coeff(const int16x8_t a, const int16x8_t b, - const tran_high_t c, int16x8_t *add, - int16x8_t *sub) { - const int32x4_t a0 = vmull_n_s16(vget_low_s16(a), c); - const int32x4_t a1 = vmull_n_s16(vget_high_s16(a), c); - const int32x4_t sum0 = vmlal_n_s16(a0, vget_low_s16(b), c); - const int32x4_t sum1 = vmlal_n_s16(a1, vget_high_s16(b), c); - const int32x4_t diff0 = vmlsl_n_s16(a0, vget_low_s16(b), c); - const int32x4_t diff1 = vmlsl_n_s16(a1, vget_high_s16(b), c); - const int16x4_t rounded0 = vqrshrn_n_s32(sum0, 14); - const int16x4_t rounded1 = vqrshrn_n_s32(sum1, 14); - const int16x4_t rounded2 = vqrshrn_n_s32(diff0, 14); - const int16x4_t rounded3 = vqrshrn_n_s32(diff1, 14); - *add = vcombine_s16(rounded0, rounded1); - *sub = vcombine_s16(rounded2, rounded3); -} - -// fdct_round_shift(a * c0 +/- b * c1) -static INLINE void butterfly_two_coeff(const int16x8_t a, const int16x8_t b, - const tran_coef_t c0, - const tran_coef_t c1, int16x8_t *add, - int16x8_t *sub) { - const int32x4_t a0 = vmull_n_s16(vget_low_s16(a), c0); - const int32x4_t a1 = vmull_n_s16(vget_high_s16(a), c0); - const int32x4_t a2 = vmull_n_s16(vget_low_s16(a), c1); - const int32x4_t a3 = vmull_n_s16(vget_high_s16(a), c1); - const int32x4_t sum0 = vmlal_n_s16(a2, vget_low_s16(b), c0); - const int32x4_t sum1 = vmlal_n_s16(a3, vget_high_s16(b), c0); - const int32x4_t diff0 = vmlsl_n_s16(a0, vget_low_s16(b), c1); - const int32x4_t diff1 = vmlsl_n_s16(a1, vget_high_s16(b), c1); - const int16x4_t rounded0 = vqrshrn_n_s32(sum0, 14); - const int16x4_t rounded1 = vqrshrn_n_s32(sum1, 14); - const int16x4_t rounded2 = vqrshrn_n_s32(diff0, 14); - const int16x4_t rounded3 = vqrshrn_n_s32(diff1, 14); - *add = vcombine_s16(rounded0, rounded1); - *sub = vcombine_s16(rounded2, rounded3); -} - -// Transpose 8x8 to a new location. Don't use transpose_neon.h because those -// are all in-place. -static INLINE void transpose_8x8(const int16x8_t *a /*[8]*/, - int16x8_t *b /*[8]*/) { - // Swap 16 bit elements. - const int16x8x2_t c0 = vtrnq_s16(a[0], a[1]); - const int16x8x2_t c1 = vtrnq_s16(a[2], a[3]); - const int16x8x2_t c2 = vtrnq_s16(a[4], a[5]); - const int16x8x2_t c3 = vtrnq_s16(a[6], a[7]); - - // Swap 32 bit elements. - const int32x4x2_t d0 = vtrnq_s32(vreinterpretq_s32_s16(c0.val[0]), - vreinterpretq_s32_s16(c1.val[0])); - const int32x4x2_t d1 = vtrnq_s32(vreinterpretq_s32_s16(c0.val[1]), - vreinterpretq_s32_s16(c1.val[1])); - const int32x4x2_t d2 = vtrnq_s32(vreinterpretq_s32_s16(c2.val[0]), - vreinterpretq_s32_s16(c3.val[0])); - const int32x4x2_t d3 = vtrnq_s32(vreinterpretq_s32_s16(c2.val[1]), - vreinterpretq_s32_s16(c3.val[1])); - - // Swap 64 bit elements - const int16x8x2_t e0 = vpx_vtrnq_s64_to_s16(d0.val[0], d2.val[0]); - const int16x8x2_t e1 = vpx_vtrnq_s64_to_s16(d1.val[0], d3.val[0]); - const int16x8x2_t e2 = vpx_vtrnq_s64_to_s16(d0.val[1], d2.val[1]); - const int16x8x2_t e3 = vpx_vtrnq_s64_to_s16(d1.val[1], d3.val[1]); - - b[0] = e0.val[0]; - b[1] = e1.val[0]; - b[2] = e2.val[0]; - b[3] = e3.val[0]; - b[4] = e0.val[1]; - b[5] = e1.val[1]; - b[6] = e2.val[1]; - b[7] = e3.val[1]; -} - -// Main body of fdct16x16. -static void dct_body(const int16x8_t *in /*[16]*/, int16x8_t *out /*[16]*/) { - int16x8_t s[8]; - int16x8_t x[4]; - int16x8_t step[8]; - - // stage 1 - // From fwd_txfm.c: Work on the first eight values; fdct8(input, - // even_results);" - s[0] = vaddq_s16(in[0], in[7]); - s[1] = vaddq_s16(in[1], in[6]); - s[2] = vaddq_s16(in[2], in[5]); - s[3] = vaddq_s16(in[3], in[4]); - s[4] = vsubq_s16(in[3], in[4]); - s[5] = vsubq_s16(in[2], in[5]); - s[6] = vsubq_s16(in[1], in[6]); - s[7] = vsubq_s16(in[0], in[7]); - - // fdct4(step, step); - x[0] = vaddq_s16(s[0], s[3]); - x[1] = vaddq_s16(s[1], s[2]); - x[2] = vsubq_s16(s[1], s[2]); - x[3] = vsubq_s16(s[0], s[3]); - - // out[0] = fdct_round_shift((x0 + x1) * cospi_16_64) - // out[8] = fdct_round_shift((x0 - x1) * cospi_16_64) - butterfly_one_coeff(x[0], x[1], cospi_16_64, &out[0], &out[8]); - // out[4] = fdct_round_shift(x3 * cospi_8_64 + x2 * cospi_24_64); - // out[12] = fdct_round_shift(x3 * cospi_24_64 - x2 * cospi_8_64); - butterfly_two_coeff(x[3], x[2], cospi_24_64, cospi_8_64, &out[4], &out[12]); - - // Stage 2 - // Re-using source s5/s6 - // s5 = fdct_round_shift((s6 - s5) * cospi_16_64) - // s6 = fdct_round_shift((s6 + s5) * cospi_16_64) - butterfly_one_coeff(s[6], s[5], cospi_16_64, &s[6], &s[5]); - - // Stage 3 - x[0] = vaddq_s16(s[4], s[5]); - x[1] = vsubq_s16(s[4], s[5]); - x[2] = vsubq_s16(s[7], s[6]); - x[3] = vaddq_s16(s[7], s[6]); - - // Stage 4 - // out[2] = fdct_round_shift(x0 * cospi_28_64 + x3 * cospi_4_64) - // out[14] = fdct_round_shift(x3 * cospi_28_64 + x0 * -cospi_4_64) - butterfly_two_coeff(x[3], x[0], cospi_28_64, cospi_4_64, &out[2], &out[14]); - // out[6] = fdct_round_shift(x1 * cospi_12_64 + x2 * cospi_20_64) - // out[10] = fdct_round_shift(x2 * cospi_12_64 + x1 * -cospi_20_64) - butterfly_two_coeff(x[2], x[1], cospi_12_64, cospi_20_64, &out[10], &out[6]); - - // step 2 - // From fwd_txfm.c: Work on the next eight values; step1 -> odd_results" - // That file distinguished between "in_high" and "step1" but the only - // difference is that "in_high" is the first 8 values and "step 1" is the - // second. Here, since they are all in one array, "step1" values are += 8. - - // step2[2] = fdct_round_shift((step1[5] - step1[2]) * cospi_16_64) - // step2[3] = fdct_round_shift((step1[4] - step1[3]) * cospi_16_64) - // step2[4] = fdct_round_shift((step1[4] + step1[3]) * cospi_16_64) - // step2[5] = fdct_round_shift((step1[5] + step1[2]) * cospi_16_64) - butterfly_one_coeff(in[13], in[10], cospi_16_64, &s[5], &s[2]); - butterfly_one_coeff(in[12], in[11], cospi_16_64, &s[4], &s[3]); - - // step 3 - s[0] = vaddq_s16(in[8], s[3]); - s[1] = vaddq_s16(in[9], s[2]); - x[0] = vsubq_s16(in[9], s[2]); - x[1] = vsubq_s16(in[8], s[3]); - x[2] = vsubq_s16(in[15], s[4]); - x[3] = vsubq_s16(in[14], s[5]); - s[6] = vaddq_s16(in[14], s[5]); - s[7] = vaddq_s16(in[15], s[4]); - - // step 4 - // step2[1] = fdct_round_shift(step3[1] *-cospi_8_64 + step3[6] * cospi_24_64) - // step2[6] = fdct_round_shift(step3[1] * cospi_24_64 + step3[6] * cospi_8_64) - butterfly_two_coeff(s[6], s[1], cospi_24_64, cospi_8_64, &s[6], &s[1]); - - // step2[2] = fdct_round_shift(step3[2] * cospi_24_64 + step3[5] * cospi_8_64) - // step2[5] = fdct_round_shift(step3[2] * cospi_8_64 - step3[5] * cospi_24_64) - butterfly_two_coeff(x[0], x[3], cospi_8_64, cospi_24_64, &s[2], &s[5]); - - // step 5 - step[0] = vaddq_s16(s[0], s[1]); - step[1] = vsubq_s16(s[0], s[1]); - step[2] = vaddq_s16(x[1], s[2]); - step[3] = vsubq_s16(x[1], s[2]); - step[4] = vsubq_s16(x[2], s[5]); - step[5] = vaddq_s16(x[2], s[5]); - step[6] = vsubq_s16(s[7], s[6]); - step[7] = vaddq_s16(s[7], s[6]); - - // step 6 - // out[1] = fdct_round_shift(step1[0] * cospi_30_64 + step1[7] * cospi_2_64) - // out[9] = fdct_round_shift(step1[1] * cospi_14_64 + step1[6] * cospi_18_64) - // out[5] = fdct_round_shift(step1[2] * cospi_22_64 + step1[5] * cospi_10_64) - // out[13] = fdct_round_shift(step1[3] * cospi_6_64 + step1[4] * cospi_26_64) - // out[3] = fdct_round_shift(step1[3] * -cospi_26_64 + step1[4] * cospi_6_64) - // out[11] = fdct_round_shift(step1[2] * -cospi_10_64 + step1[5] * - // cospi_22_64) - // out[7] = fdct_round_shift(step1[1] * -cospi_18_64 + step1[6] * cospi_14_64) - // out[15] = fdct_round_shift(step1[0] * -cospi_2_64 + step1[7] * cospi_30_64) - butterfly_two_coeff(step[6], step[1], cospi_14_64, cospi_18_64, &out[9], - &out[7]); - butterfly_two_coeff(step[7], step[0], cospi_30_64, cospi_2_64, &out[1], - &out[15]); - butterfly_two_coeff(step[4], step[3], cospi_6_64, cospi_26_64, &out[13], - &out[3]); - butterfly_two_coeff(step[5], step[2], cospi_22_64, cospi_10_64, &out[5], - &out[11]); -} - void vpx_fdct16x16_neon(const int16_t *input, tran_low_t *output, int stride) { int16x8_t temp0[16]; int16x8_t temp1[16]; int16x8_t temp2[16]; int16x8_t temp3[16]; // Left half. load(input, stride, temp0); cross_input(temp0, temp1, 0); - dct_body(temp1, temp0); + vpx_fdct16x16_body(temp1, temp0); // Right half. load(input + 8, stride, temp1); cross_input(temp1, temp2, 0); - dct_body(temp2, temp1); + vpx_fdct16x16_body(temp2, temp1); // Transpose top left and top right quarters into one contiguous location to // process to the top half. transpose_8x8(&temp0[0], &temp2[0]); transpose_8x8(&temp1[0], &temp2[8]); partial_round_shift(temp2); cross_input(temp2, temp3, 1); - dct_body(temp3, temp2); + vpx_fdct16x16_body(temp3, temp2); transpose_s16_8x8(&temp2[0], &temp2[1], &temp2[2], &temp2[3], &temp2[4], &temp2[5], &temp2[6], &temp2[7]); transpose_s16_8x8(&temp2[8], &temp2[9], &temp2[10], &temp2[11], &temp2[12], &temp2[13], &temp2[14], &temp2[15]); store(output, temp2); @@ -373,11 +64,11 @@ transpose_8x8(&temp0[8], &temp1[0]); transpose_s16_8x8(&temp1[8], &temp1[9], &temp1[10], &temp1[11], &temp1[12], &temp1[13], &temp1[14], &temp1[15]); partial_round_shift(temp1); cross_input(temp1, temp0, 1); - dct_body(temp0, temp1); + vpx_fdct16x16_body(temp0, temp1); transpose_s16_8x8(&temp1[0], &temp1[1], &temp1[2], &temp1[3], &temp1[4], &temp1[5], &temp1[6], &temp1[7]); transpose_s16_8x8(&temp1[8], &temp1[9], &temp1[10], &temp1[11], &temp1[12], &temp1[13], &temp1[14], &temp1[15]); store(output, temp1); diff --git a/media/libvpx/libvpx/vpx_dsp/arm/fdct32x32_neon.c b/media/libvpx/libvpx/vpx_dsp/arm/fdct32x32_neon.c --- a/media/libvpx/libvpx/vpx_dsp/arm/fdct32x32_neon.c +++ b/media/libvpx/libvpx/vpx_dsp/arm/fdct32x32_neon.c @@ -151,11 +151,11 @@ #define STORE_S16(src, index, dest) \ do { \ store_s16q_to_tran_low(dest, src[index]); \ dest += 8; \ - } while (0); + } while (0) // Store 32 16x8 values, assuming stride == 32. // Slight twist: store horizontally in blocks of 8. static INLINE void store(tran_low_t *a, const int16x8_t *b) { STORE_S16(b, 0, a); diff --git a/media/libvpx/libvpx/vpx_dsp/arm/fdct_neon.h b/media/libvpx/libvpx/vpx_dsp/arm/fdct_neon.h new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/arm/fdct_neon.h @@ -0,0 +1,213 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VPX_VPX_DSP_ARM_FDCT_NEON_H_ +#define VPX_VPX_DSP_ARM_FDCT_NEON_H_ + +#include + +static INLINE void vpx_fdct4x4_pass1_neon(int16x4_t *in) { + const int16x8_t input_01 = vcombine_s16(in[0], in[1]); + const int16x8_t input_32 = vcombine_s16(in[3], in[2]); + + // in_0 +/- in_3, in_1 +/- in_2 + const int16x8_t s_01 = vaddq_s16(input_01, input_32); + const int16x8_t s_32 = vsubq_s16(input_01, input_32); + + // step_0 +/- step_1, step_2 +/- step_3 + const int16x4_t s_0 = vget_low_s16(s_01); + const int16x4_t s_1 = vget_high_s16(s_01); + const int16x4_t s_2 = vget_high_s16(s_32); + const int16x4_t s_3 = vget_low_s16(s_32); + + // (s_0 +/- s_1) * cospi_16_64 + // Must expand all elements to s32. See 'needs32' comment in fwd_txfm.c. + const int32x4_t s_0_p_s_1 = vaddl_s16(s_0, s_1); + const int32x4_t s_0_m_s_1 = vsubl_s16(s_0, s_1); + const int32x4_t temp1 = vmulq_n_s32(s_0_p_s_1, cospi_16_64); + const int32x4_t temp2 = vmulq_n_s32(s_0_m_s_1, cospi_16_64); + + // fdct_round_shift + int16x4_t out_0 = vrshrn_n_s32(temp1, DCT_CONST_BITS); + int16x4_t out_2 = vrshrn_n_s32(temp2, DCT_CONST_BITS); + + // s_3 * cospi_8_64 + s_2 * cospi_24_64 + // s_3 * cospi_24_64 - s_2 * cospi_8_64 + const int32x4_t s_3_cospi_8_64 = vmull_n_s16(s_3, cospi_8_64); + const int32x4_t s_3_cospi_24_64 = vmull_n_s16(s_3, cospi_24_64); + + const int32x4_t temp3 = vmlal_n_s16(s_3_cospi_8_64, s_2, cospi_24_64); + const int32x4_t temp4 = vmlsl_n_s16(s_3_cospi_24_64, s_2, cospi_8_64); + + // fdct_round_shift + int16x4_t out_1 = vrshrn_n_s32(temp3, DCT_CONST_BITS); + int16x4_t out_3 = vrshrn_n_s32(temp4, DCT_CONST_BITS); + + transpose_s16_4x4d(&out_0, &out_1, &out_2, &out_3); + + in[0] = out_0; + in[1] = out_1; + in[2] = out_2; + in[3] = out_3; +} + +static INLINE void vpx_fdct8x8_pass1_notranspose_neon(int16x8_t *in, + int16x8_t *out) { + const int16x8_t v_s0 = vaddq_s16(in[0], in[7]); + const int16x8_t v_s1 = vaddq_s16(in[1], in[6]); + const int16x8_t v_s2 = vaddq_s16(in[2], in[5]); + const int16x8_t v_s3 = vaddq_s16(in[3], in[4]); + const int16x8_t v_s4 = vsubq_s16(in[3], in[4]); + const int16x8_t v_s5 = vsubq_s16(in[2], in[5]); + const int16x8_t v_s6 = vsubq_s16(in[1], in[6]); + const int16x8_t v_s7 = vsubq_s16(in[0], in[7]); + // fdct4(step, step); + int16x8_t v_x0 = vaddq_s16(v_s0, v_s3); + int16x8_t v_x1 = vaddq_s16(v_s1, v_s2); + int16x8_t v_x2 = vsubq_s16(v_s1, v_s2); + int16x8_t v_x3 = vsubq_s16(v_s0, v_s3); + // fdct4(step, step); + int32x4_t v_t0_lo = vaddl_s16(vget_low_s16(v_x0), vget_low_s16(v_x1)); + int32x4_t v_t0_hi = vaddl_s16(vget_high_s16(v_x0), vget_high_s16(v_x1)); + int32x4_t v_t1_lo = vsubl_s16(vget_low_s16(v_x0), vget_low_s16(v_x1)); + int32x4_t v_t1_hi = vsubl_s16(vget_high_s16(v_x0), vget_high_s16(v_x1)); + int32x4_t v_t2_lo = vmull_n_s16(vget_low_s16(v_x2), cospi_24_64); + int32x4_t v_t2_hi = vmull_n_s16(vget_high_s16(v_x2), cospi_24_64); + int32x4_t v_t3_lo = vmull_n_s16(vget_low_s16(v_x3), cospi_24_64); + int32x4_t v_t3_hi = vmull_n_s16(vget_high_s16(v_x3), cospi_24_64); + v_t2_lo = vmlal_n_s16(v_t2_lo, vget_low_s16(v_x3), cospi_8_64); + v_t2_hi = vmlal_n_s16(v_t2_hi, vget_high_s16(v_x3), cospi_8_64); + v_t3_lo = vmlsl_n_s16(v_t3_lo, vget_low_s16(v_x2), cospi_8_64); + v_t3_hi = vmlsl_n_s16(v_t3_hi, vget_high_s16(v_x2), cospi_8_64); + v_t0_lo = vmulq_n_s32(v_t0_lo, cospi_16_64); + v_t0_hi = vmulq_n_s32(v_t0_hi, cospi_16_64); + v_t1_lo = vmulq_n_s32(v_t1_lo, cospi_16_64); + v_t1_hi = vmulq_n_s32(v_t1_hi, cospi_16_64); + { + const int16x4_t a = vrshrn_n_s32(v_t0_lo, DCT_CONST_BITS); + const int16x4_t b = vrshrn_n_s32(v_t0_hi, DCT_CONST_BITS); + const int16x4_t c = vrshrn_n_s32(v_t1_lo, DCT_CONST_BITS); + const int16x4_t d = vrshrn_n_s32(v_t1_hi, DCT_CONST_BITS); + const int16x4_t e = vrshrn_n_s32(v_t2_lo, DCT_CONST_BITS); + const int16x4_t f = vrshrn_n_s32(v_t2_hi, DCT_CONST_BITS); + const int16x4_t g = vrshrn_n_s32(v_t3_lo, DCT_CONST_BITS); + const int16x4_t h = vrshrn_n_s32(v_t3_hi, DCT_CONST_BITS); + out[0] = vcombine_s16(a, c); // 00 01 02 03 40 41 42 43 + out[2] = vcombine_s16(e, g); // 20 21 22 23 60 61 62 63 + out[4] = vcombine_s16(b, d); // 04 05 06 07 44 45 46 47 + out[6] = vcombine_s16(f, h); // 24 25 26 27 64 65 66 67 + } + // Stage 2 + v_x0 = vsubq_s16(v_s6, v_s5); + v_x1 = vaddq_s16(v_s6, v_s5); + v_t0_lo = vmull_n_s16(vget_low_s16(v_x0), cospi_16_64); + v_t0_hi = vmull_n_s16(vget_high_s16(v_x0), cospi_16_64); + v_t1_lo = vmull_n_s16(vget_low_s16(v_x1), cospi_16_64); + v_t1_hi = vmull_n_s16(vget_high_s16(v_x1), cospi_16_64); + { + const int16x4_t a = vrshrn_n_s32(v_t0_lo, DCT_CONST_BITS); + const int16x4_t b = vrshrn_n_s32(v_t0_hi, DCT_CONST_BITS); + const int16x4_t c = vrshrn_n_s32(v_t1_lo, DCT_CONST_BITS); + const int16x4_t d = vrshrn_n_s32(v_t1_hi, DCT_CONST_BITS); + const int16x8_t ab = vcombine_s16(a, b); + const int16x8_t cd = vcombine_s16(c, d); + // Stage 3 + v_x0 = vaddq_s16(v_s4, ab); + v_x1 = vsubq_s16(v_s4, ab); + v_x2 = vsubq_s16(v_s7, cd); + v_x3 = vaddq_s16(v_s7, cd); + } + // Stage 4 + v_t0_lo = vmull_n_s16(vget_low_s16(v_x3), cospi_4_64); + v_t0_hi = vmull_n_s16(vget_high_s16(v_x3), cospi_4_64); + v_t0_lo = vmlal_n_s16(v_t0_lo, vget_low_s16(v_x0), cospi_28_64); + v_t0_hi = vmlal_n_s16(v_t0_hi, vget_high_s16(v_x0), cospi_28_64); + v_t1_lo = vmull_n_s16(vget_low_s16(v_x1), cospi_12_64); + v_t1_hi = vmull_n_s16(vget_high_s16(v_x1), cospi_12_64); + v_t1_lo = vmlal_n_s16(v_t1_lo, vget_low_s16(v_x2), cospi_20_64); + v_t1_hi = vmlal_n_s16(v_t1_hi, vget_high_s16(v_x2), cospi_20_64); + v_t2_lo = vmull_n_s16(vget_low_s16(v_x2), cospi_12_64); + v_t2_hi = vmull_n_s16(vget_high_s16(v_x2), cospi_12_64); + v_t2_lo = vmlsl_n_s16(v_t2_lo, vget_low_s16(v_x1), cospi_20_64); + v_t2_hi = vmlsl_n_s16(v_t2_hi, vget_high_s16(v_x1), cospi_20_64); + v_t3_lo = vmull_n_s16(vget_low_s16(v_x3), cospi_28_64); + v_t3_hi = vmull_n_s16(vget_high_s16(v_x3), cospi_28_64); + v_t3_lo = vmlsl_n_s16(v_t3_lo, vget_low_s16(v_x0), cospi_4_64); + v_t3_hi = vmlsl_n_s16(v_t3_hi, vget_high_s16(v_x0), cospi_4_64); + { + const int16x4_t a = vrshrn_n_s32(v_t0_lo, DCT_CONST_BITS); + const int16x4_t b = vrshrn_n_s32(v_t0_hi, DCT_CONST_BITS); + const int16x4_t c = vrshrn_n_s32(v_t1_lo, DCT_CONST_BITS); + const int16x4_t d = vrshrn_n_s32(v_t1_hi, DCT_CONST_BITS); + const int16x4_t e = vrshrn_n_s32(v_t2_lo, DCT_CONST_BITS); + const int16x4_t f = vrshrn_n_s32(v_t2_hi, DCT_CONST_BITS); + const int16x4_t g = vrshrn_n_s32(v_t3_lo, DCT_CONST_BITS); + const int16x4_t h = vrshrn_n_s32(v_t3_hi, DCT_CONST_BITS); + out[1] = vcombine_s16(a, c); // 10 11 12 13 50 51 52 53 + out[3] = vcombine_s16(e, g); // 30 31 32 33 70 71 72 73 + out[5] = vcombine_s16(b, d); // 14 15 16 17 54 55 56 57 + out[7] = vcombine_s16(f, h); // 34 35 36 37 74 75 76 77 + } +} + +static INLINE void vpx_fdct8x8_pass1_neon(int16x8_t *in) { + int16x8_t out[8]; + vpx_fdct8x8_pass1_notranspose_neon(in, out); + // transpose 8x8 + // Can't use transpose_s16_8x8() because the values are arranged in two 4x8 + // columns. + { + // 00 01 02 03 40 41 42 43 + // 10 11 12 13 50 51 52 53 + // 20 21 22 23 60 61 62 63 + // 30 31 32 33 70 71 72 73 + // 04 05 06 07 44 45 46 47 + // 14 15 16 17 54 55 56 57 + // 24 25 26 27 64 65 66 67 + // 34 35 36 37 74 75 76 77 + const int32x4x2_t r02_s32 = + vtrnq_s32(vreinterpretq_s32_s16(out[0]), vreinterpretq_s32_s16(out[2])); + const int32x4x2_t r13_s32 = + vtrnq_s32(vreinterpretq_s32_s16(out[1]), vreinterpretq_s32_s16(out[3])); + const int32x4x2_t r46_s32 = + vtrnq_s32(vreinterpretq_s32_s16(out[4]), vreinterpretq_s32_s16(out[6])); + const int32x4x2_t r57_s32 = + vtrnq_s32(vreinterpretq_s32_s16(out[5]), vreinterpretq_s32_s16(out[7])); + const int16x8x2_t r01_s16 = + vtrnq_s16(vreinterpretq_s16_s32(r02_s32.val[0]), + vreinterpretq_s16_s32(r13_s32.val[0])); + const int16x8x2_t r23_s16 = + vtrnq_s16(vreinterpretq_s16_s32(r02_s32.val[1]), + vreinterpretq_s16_s32(r13_s32.val[1])); + const int16x8x2_t r45_s16 = + vtrnq_s16(vreinterpretq_s16_s32(r46_s32.val[0]), + vreinterpretq_s16_s32(r57_s32.val[0])); + const int16x8x2_t r67_s16 = + vtrnq_s16(vreinterpretq_s16_s32(r46_s32.val[1]), + vreinterpretq_s16_s32(r57_s32.val[1])); + in[0] = r01_s16.val[0]; + in[1] = r01_s16.val[1]; + in[2] = r23_s16.val[0]; + in[3] = r23_s16.val[1]; + in[4] = r45_s16.val[0]; + in[5] = r45_s16.val[1]; + in[6] = r67_s16.val[0]; + in[7] = r67_s16.val[1]; + // 00 10 20 30 40 50 60 70 + // 01 11 21 31 41 51 61 71 + // 02 12 22 32 42 52 62 72 + // 03 13 23 33 43 53 63 73 + // 04 14 24 34 44 54 64 74 + // 05 15 25 35 45 55 65 75 + // 06 16 26 36 46 56 66 76 + // 07 17 27 37 47 57 67 77 + } +} +#endif // VPX_VPX_DSP_ARM_FDCT_NEON_H_ diff --git a/media/libvpx/libvpx/vpx_dsp/arm/fdct_neon.c b/media/libvpx/libvpx/vpx_dsp/arm/fdct_neon.c --- a/media/libvpx/libvpx/vpx_dsp/arm/fdct_neon.c +++ b/media/libvpx/libvpx/vpx_dsp/arm/fdct_neon.c @@ -13,78 +13,37 @@ #include "./vpx_config.h" #include "./vpx_dsp_rtcd.h" #include "vpx_dsp/txfm_common.h" #include "vpx_dsp/vpx_dsp_common.h" #include "vpx_dsp/arm/idct_neon.h" +#include "vpx_dsp/arm/fdct_neon.h" #include "vpx_dsp/arm/mem_neon.h" #include "vpx_dsp/arm/transpose_neon.h" void vpx_fdct4x4_neon(const int16_t *input, tran_low_t *final_output, int stride) { int i; // input[M * stride] * 16 - int16x4_t input_0 = vshl_n_s16(vld1_s16(input + 0 * stride), 4); - int16x4_t input_1 = vshl_n_s16(vld1_s16(input + 1 * stride), 4); - int16x4_t input_2 = vshl_n_s16(vld1_s16(input + 2 * stride), 4); - int16x4_t input_3 = vshl_n_s16(vld1_s16(input + 3 * stride), 4); + int16x4_t in[4]; + in[0] = vshl_n_s16(vld1_s16(input + 0 * stride), 4); + in[1] = vshl_n_s16(vld1_s16(input + 1 * stride), 4); + in[2] = vshl_n_s16(vld1_s16(input + 2 * stride), 4); + in[3] = vshl_n_s16(vld1_s16(input + 3 * stride), 4); // If the very first value != 0, then add 1. if (input[0] != 0) { const int16x4_t one = vreinterpret_s16_s64(vdup_n_s64(1)); - input_0 = vadd_s16(input_0, one); + in[0] = vadd_s16(in[0], one); } - for (i = 0; i < 2; ++i) { - const int16x8_t input_01 = vcombine_s16(input_0, input_1); - const int16x8_t input_32 = vcombine_s16(input_3, input_2); - - // in_0 +/- in_3, in_1 +/- in_2 - const int16x8_t s_01 = vaddq_s16(input_01, input_32); - const int16x8_t s_32 = vsubq_s16(input_01, input_32); - - // step_0 +/- step_1, step_2 +/- step_3 - const int16x4_t s_0 = vget_low_s16(s_01); - const int16x4_t s_1 = vget_high_s16(s_01); - const int16x4_t s_2 = vget_high_s16(s_32); - const int16x4_t s_3 = vget_low_s16(s_32); - - // (s_0 +/- s_1) * cospi_16_64 - // Must expand all elements to s32. See 'needs32' comment in fwd_txfm.c. - const int32x4_t s_0_p_s_1 = vaddl_s16(s_0, s_1); - const int32x4_t s_0_m_s_1 = vsubl_s16(s_0, s_1); - const int32x4_t temp1 = vmulq_n_s32(s_0_p_s_1, cospi_16_64); - const int32x4_t temp2 = vmulq_n_s32(s_0_m_s_1, cospi_16_64); - - // fdct_round_shift - int16x4_t out_0 = vrshrn_n_s32(temp1, DCT_CONST_BITS); - int16x4_t out_2 = vrshrn_n_s32(temp2, DCT_CONST_BITS); - - // s_3 * cospi_8_64 + s_2 * cospi_24_64 - // s_3 * cospi_24_64 - s_2 * cospi_8_64 - const int32x4_t s_3_cospi_8_64 = vmull_n_s16(s_3, cospi_8_64); - const int32x4_t s_3_cospi_24_64 = vmull_n_s16(s_3, cospi_24_64); - - const int32x4_t temp3 = vmlal_n_s16(s_3_cospi_8_64, s_2, cospi_24_64); - const int32x4_t temp4 = vmlsl_n_s16(s_3_cospi_24_64, s_2, cospi_8_64); - - // fdct_round_shift - int16x4_t out_1 = vrshrn_n_s32(temp3, DCT_CONST_BITS); - int16x4_t out_3 = vrshrn_n_s32(temp4, DCT_CONST_BITS); - - transpose_s16_4x4d(&out_0, &out_1, &out_2, &out_3); - - input_0 = out_0; - input_1 = out_1; - input_2 = out_2; - input_3 = out_3; + vpx_fdct4x4_pass1_neon(in); } - { // Not quite a rounding shift. Only add 1 despite shifting by 2. const int16x8_t one = vdupq_n_s16(1); - int16x8_t out_01 = vcombine_s16(input_0, input_1); - int16x8_t out_23 = vcombine_s16(input_2, input_3); + int16x8_t out_01 = vcombine_s16(in[0], in[1]); + int16x8_t out_23 = vcombine_s16(in[2], in[3]); out_01 = vshrq_n_s16(vaddq_s16(out_01, one), 2); out_23 = vshrq_n_s16(vaddq_s16(out_23, one), 2); store_s16q_to_tran_low(final_output + 0 * 8, out_01); store_s16q_to_tran_low(final_output + 1 * 8, out_23); } diff --git a/media/libvpx/libvpx/vpx_dsp/arm/fwd_txfm_neon.c b/media/libvpx/libvpx/vpx_dsp/arm/fwd_txfm_neon.c --- a/media/libvpx/libvpx/vpx_dsp/arm/fwd_txfm_neon.c +++ b/media/libvpx/libvpx/vpx_dsp/arm/fwd_txfm_neon.c @@ -13,198 +13,56 @@ #include "./vpx_config.h" #include "./vpx_dsp_rtcd.h" #include "vpx_dsp/txfm_common.h" #include "vpx_dsp/vpx_dsp_common.h" #include "vpx_dsp/arm/idct_neon.h" +#include "vpx_dsp/arm/fdct_neon.h" #include "vpx_dsp/arm/mem_neon.h" void vpx_fdct8x8_neon(const int16_t *input, tran_low_t *final_output, int stride) { int i; // stage 1 - int16x8_t input_0 = vshlq_n_s16(vld1q_s16(&input[0 * stride]), 2); - int16x8_t input_1 = vshlq_n_s16(vld1q_s16(&input[1 * stride]), 2); - int16x8_t input_2 = vshlq_n_s16(vld1q_s16(&input[2 * stride]), 2); - int16x8_t input_3 = vshlq_n_s16(vld1q_s16(&input[3 * stride]), 2); - int16x8_t input_4 = vshlq_n_s16(vld1q_s16(&input[4 * stride]), 2); - int16x8_t input_5 = vshlq_n_s16(vld1q_s16(&input[5 * stride]), 2); - int16x8_t input_6 = vshlq_n_s16(vld1q_s16(&input[6 * stride]), 2); - int16x8_t input_7 = vshlq_n_s16(vld1q_s16(&input[7 * stride]), 2); + int16x8_t in[8]; + in[0] = vshlq_n_s16(vld1q_s16(&input[0 * stride]), 2); + in[1] = vshlq_n_s16(vld1q_s16(&input[1 * stride]), 2); + in[2] = vshlq_n_s16(vld1q_s16(&input[2 * stride]), 2); + in[3] = vshlq_n_s16(vld1q_s16(&input[3 * stride]), 2); + in[4] = vshlq_n_s16(vld1q_s16(&input[4 * stride]), 2); + in[5] = vshlq_n_s16(vld1q_s16(&input[5 * stride]), 2); + in[6] = vshlq_n_s16(vld1q_s16(&input[6 * stride]), 2); + in[7] = vshlq_n_s16(vld1q_s16(&input[7 * stride]), 2); for (i = 0; i < 2; ++i) { - int16x8_t out_0, out_1, out_2, out_3, out_4, out_5, out_6, out_7; - const int16x8_t v_s0 = vaddq_s16(input_0, input_7); - const int16x8_t v_s1 = vaddq_s16(input_1, input_6); - const int16x8_t v_s2 = vaddq_s16(input_2, input_5); - const int16x8_t v_s3 = vaddq_s16(input_3, input_4); - const int16x8_t v_s4 = vsubq_s16(input_3, input_4); - const int16x8_t v_s5 = vsubq_s16(input_2, input_5); - const int16x8_t v_s6 = vsubq_s16(input_1, input_6); - const int16x8_t v_s7 = vsubq_s16(input_0, input_7); - // fdct4(step, step); - int16x8_t v_x0 = vaddq_s16(v_s0, v_s3); - int16x8_t v_x1 = vaddq_s16(v_s1, v_s2); - int16x8_t v_x2 = vsubq_s16(v_s1, v_s2); - int16x8_t v_x3 = vsubq_s16(v_s0, v_s3); - // fdct4(step, step); - int32x4_t v_t0_lo = vaddl_s16(vget_low_s16(v_x0), vget_low_s16(v_x1)); - int32x4_t v_t0_hi = vaddl_s16(vget_high_s16(v_x0), vget_high_s16(v_x1)); - int32x4_t v_t1_lo = vsubl_s16(vget_low_s16(v_x0), vget_low_s16(v_x1)); - int32x4_t v_t1_hi = vsubl_s16(vget_high_s16(v_x0), vget_high_s16(v_x1)); - int32x4_t v_t2_lo = vmull_n_s16(vget_low_s16(v_x2), cospi_24_64); - int32x4_t v_t2_hi = vmull_n_s16(vget_high_s16(v_x2), cospi_24_64); - int32x4_t v_t3_lo = vmull_n_s16(vget_low_s16(v_x3), cospi_24_64); - int32x4_t v_t3_hi = vmull_n_s16(vget_high_s16(v_x3), cospi_24_64); - v_t2_lo = vmlal_n_s16(v_t2_lo, vget_low_s16(v_x3), cospi_8_64); - v_t2_hi = vmlal_n_s16(v_t2_hi, vget_high_s16(v_x3), cospi_8_64); - v_t3_lo = vmlsl_n_s16(v_t3_lo, vget_low_s16(v_x2), cospi_8_64); - v_t3_hi = vmlsl_n_s16(v_t3_hi, vget_high_s16(v_x2), cospi_8_64); - v_t0_lo = vmulq_n_s32(v_t0_lo, cospi_16_64); - v_t0_hi = vmulq_n_s32(v_t0_hi, cospi_16_64); - v_t1_lo = vmulq_n_s32(v_t1_lo, cospi_16_64); - v_t1_hi = vmulq_n_s32(v_t1_hi, cospi_16_64); - { - const int16x4_t a = vrshrn_n_s32(v_t0_lo, DCT_CONST_BITS); - const int16x4_t b = vrshrn_n_s32(v_t0_hi, DCT_CONST_BITS); - const int16x4_t c = vrshrn_n_s32(v_t1_lo, DCT_CONST_BITS); - const int16x4_t d = vrshrn_n_s32(v_t1_hi, DCT_CONST_BITS); - const int16x4_t e = vrshrn_n_s32(v_t2_lo, DCT_CONST_BITS); - const int16x4_t f = vrshrn_n_s32(v_t2_hi, DCT_CONST_BITS); - const int16x4_t g = vrshrn_n_s32(v_t3_lo, DCT_CONST_BITS); - const int16x4_t h = vrshrn_n_s32(v_t3_hi, DCT_CONST_BITS); - out_0 = vcombine_s16(a, c); // 00 01 02 03 40 41 42 43 - out_2 = vcombine_s16(e, g); // 20 21 22 23 60 61 62 63 - out_4 = vcombine_s16(b, d); // 04 05 06 07 44 45 46 47 - out_6 = vcombine_s16(f, h); // 24 25 26 27 64 65 66 67 - } - // Stage 2 - v_x0 = vsubq_s16(v_s6, v_s5); - v_x1 = vaddq_s16(v_s6, v_s5); - v_t0_lo = vmull_n_s16(vget_low_s16(v_x0), cospi_16_64); - v_t0_hi = vmull_n_s16(vget_high_s16(v_x0), cospi_16_64); - v_t1_lo = vmull_n_s16(vget_low_s16(v_x1), cospi_16_64); - v_t1_hi = vmull_n_s16(vget_high_s16(v_x1), cospi_16_64); - { - const int16x4_t a = vrshrn_n_s32(v_t0_lo, DCT_CONST_BITS); - const int16x4_t b = vrshrn_n_s32(v_t0_hi, DCT_CONST_BITS); - const int16x4_t c = vrshrn_n_s32(v_t1_lo, DCT_CONST_BITS); - const int16x4_t d = vrshrn_n_s32(v_t1_hi, DCT_CONST_BITS); - const int16x8_t ab = vcombine_s16(a, b); - const int16x8_t cd = vcombine_s16(c, d); - // Stage 3 - v_x0 = vaddq_s16(v_s4, ab); - v_x1 = vsubq_s16(v_s4, ab); - v_x2 = vsubq_s16(v_s7, cd); - v_x3 = vaddq_s16(v_s7, cd); - } - // Stage 4 - v_t0_lo = vmull_n_s16(vget_low_s16(v_x3), cospi_4_64); - v_t0_hi = vmull_n_s16(vget_high_s16(v_x3), cospi_4_64); - v_t0_lo = vmlal_n_s16(v_t0_lo, vget_low_s16(v_x0), cospi_28_64); - v_t0_hi = vmlal_n_s16(v_t0_hi, vget_high_s16(v_x0), cospi_28_64); - v_t1_lo = vmull_n_s16(vget_low_s16(v_x1), cospi_12_64); - v_t1_hi = vmull_n_s16(vget_high_s16(v_x1), cospi_12_64); - v_t1_lo = vmlal_n_s16(v_t1_lo, vget_low_s16(v_x2), cospi_20_64); - v_t1_hi = vmlal_n_s16(v_t1_hi, vget_high_s16(v_x2), cospi_20_64); - v_t2_lo = vmull_n_s16(vget_low_s16(v_x2), cospi_12_64); - v_t2_hi = vmull_n_s16(vget_high_s16(v_x2), cospi_12_64); - v_t2_lo = vmlsl_n_s16(v_t2_lo, vget_low_s16(v_x1), cospi_20_64); - v_t2_hi = vmlsl_n_s16(v_t2_hi, vget_high_s16(v_x1), cospi_20_64); - v_t3_lo = vmull_n_s16(vget_low_s16(v_x3), cospi_28_64); - v_t3_hi = vmull_n_s16(vget_high_s16(v_x3), cospi_28_64); - v_t3_lo = vmlsl_n_s16(v_t3_lo, vget_low_s16(v_x0), cospi_4_64); - v_t3_hi = vmlsl_n_s16(v_t3_hi, vget_high_s16(v_x0), cospi_4_64); - { - const int16x4_t a = vrshrn_n_s32(v_t0_lo, DCT_CONST_BITS); - const int16x4_t b = vrshrn_n_s32(v_t0_hi, DCT_CONST_BITS); - const int16x4_t c = vrshrn_n_s32(v_t1_lo, DCT_CONST_BITS); - const int16x4_t d = vrshrn_n_s32(v_t1_hi, DCT_CONST_BITS); - const int16x4_t e = vrshrn_n_s32(v_t2_lo, DCT_CONST_BITS); - const int16x4_t f = vrshrn_n_s32(v_t2_hi, DCT_CONST_BITS); - const int16x4_t g = vrshrn_n_s32(v_t3_lo, DCT_CONST_BITS); - const int16x4_t h = vrshrn_n_s32(v_t3_hi, DCT_CONST_BITS); - out_1 = vcombine_s16(a, c); // 10 11 12 13 50 51 52 53 - out_3 = vcombine_s16(e, g); // 30 31 32 33 70 71 72 73 - out_5 = vcombine_s16(b, d); // 14 15 16 17 54 55 56 57 - out_7 = vcombine_s16(f, h); // 34 35 36 37 74 75 76 77 - } - // transpose 8x8 - // Can't use transpose_s16_8x8() because the values are arranged in two 4x8 - // columns. - { - // 00 01 02 03 40 41 42 43 - // 10 11 12 13 50 51 52 53 - // 20 21 22 23 60 61 62 63 - // 30 31 32 33 70 71 72 73 - // 04 05 06 07 44 45 46 47 - // 14 15 16 17 54 55 56 57 - // 24 25 26 27 64 65 66 67 - // 34 35 36 37 74 75 76 77 - const int32x4x2_t r02_s32 = - vtrnq_s32(vreinterpretq_s32_s16(out_0), vreinterpretq_s32_s16(out_2)); - const int32x4x2_t r13_s32 = - vtrnq_s32(vreinterpretq_s32_s16(out_1), vreinterpretq_s32_s16(out_3)); - const int32x4x2_t r46_s32 = - vtrnq_s32(vreinterpretq_s32_s16(out_4), vreinterpretq_s32_s16(out_6)); - const int32x4x2_t r57_s32 = - vtrnq_s32(vreinterpretq_s32_s16(out_5), vreinterpretq_s32_s16(out_7)); - const int16x8x2_t r01_s16 = - vtrnq_s16(vreinterpretq_s16_s32(r02_s32.val[0]), - vreinterpretq_s16_s32(r13_s32.val[0])); - const int16x8x2_t r23_s16 = - vtrnq_s16(vreinterpretq_s16_s32(r02_s32.val[1]), - vreinterpretq_s16_s32(r13_s32.val[1])); - const int16x8x2_t r45_s16 = - vtrnq_s16(vreinterpretq_s16_s32(r46_s32.val[0]), - vreinterpretq_s16_s32(r57_s32.val[0])); - const int16x8x2_t r67_s16 = - vtrnq_s16(vreinterpretq_s16_s32(r46_s32.val[1]), - vreinterpretq_s16_s32(r57_s32.val[1])); - input_0 = r01_s16.val[0]; - input_1 = r01_s16.val[1]; - input_2 = r23_s16.val[0]; - input_3 = r23_s16.val[1]; - input_4 = r45_s16.val[0]; - input_5 = r45_s16.val[1]; - input_6 = r67_s16.val[0]; - input_7 = r67_s16.val[1]; - // 00 10 20 30 40 50 60 70 - // 01 11 21 31 41 51 61 71 - // 02 12 22 32 42 52 62 72 - // 03 13 23 33 43 53 63 73 - // 04 14 24 34 44 54 64 74 - // 05 15 25 35 45 55 65 75 - // 06 16 26 36 46 56 66 76 - // 07 17 27 37 47 57 67 77 - } + vpx_fdct8x8_pass1_neon(in); } // for { // from vpx_dct_sse2.c // Post-condition (division by two) // division of two 16 bits signed numbers using shifts // n / 2 = (n - (n >> 15)) >> 1 - const int16x8_t sign_in0 = vshrq_n_s16(input_0, 15); - const int16x8_t sign_in1 = vshrq_n_s16(input_1, 15); - const int16x8_t sign_in2 = vshrq_n_s16(input_2, 15); - const int16x8_t sign_in3 = vshrq_n_s16(input_3, 15); - const int16x8_t sign_in4 = vshrq_n_s16(input_4, 15); - const int16x8_t sign_in5 = vshrq_n_s16(input_5, 15); - const int16x8_t sign_in6 = vshrq_n_s16(input_6, 15); - const int16x8_t sign_in7 = vshrq_n_s16(input_7, 15); - input_0 = vhsubq_s16(input_0, sign_in0); - input_1 = vhsubq_s16(input_1, sign_in1); - input_2 = vhsubq_s16(input_2, sign_in2); - input_3 = vhsubq_s16(input_3, sign_in3); - input_4 = vhsubq_s16(input_4, sign_in4); - input_5 = vhsubq_s16(input_5, sign_in5); - input_6 = vhsubq_s16(input_6, sign_in6); - input_7 = vhsubq_s16(input_7, sign_in7); + const int16x8_t sign_in0 = vshrq_n_s16(in[0], 15); + const int16x8_t sign_in1 = vshrq_n_s16(in[1], 15); + const int16x8_t sign_in2 = vshrq_n_s16(in[2], 15); + const int16x8_t sign_in3 = vshrq_n_s16(in[3], 15); + const int16x8_t sign_in4 = vshrq_n_s16(in[4], 15); + const int16x8_t sign_in5 = vshrq_n_s16(in[5], 15); + const int16x8_t sign_in6 = vshrq_n_s16(in[6], 15); + const int16x8_t sign_in7 = vshrq_n_s16(in[7], 15); + in[0] = vhsubq_s16(in[0], sign_in0); + in[1] = vhsubq_s16(in[1], sign_in1); + in[2] = vhsubq_s16(in[2], sign_in2); + in[3] = vhsubq_s16(in[3], sign_in3); + in[4] = vhsubq_s16(in[4], sign_in4); + in[5] = vhsubq_s16(in[5], sign_in5); + in[6] = vhsubq_s16(in[6], sign_in6); + in[7] = vhsubq_s16(in[7], sign_in7); // store results - store_s16q_to_tran_low(final_output + 0 * 8, input_0); - store_s16q_to_tran_low(final_output + 1 * 8, input_1); - store_s16q_to_tran_low(final_output + 2 * 8, input_2); - store_s16q_to_tran_low(final_output + 3 * 8, input_3); - store_s16q_to_tran_low(final_output + 4 * 8, input_4); - store_s16q_to_tran_low(final_output + 5 * 8, input_5); - store_s16q_to_tran_low(final_output + 6 * 8, input_6); - store_s16q_to_tran_low(final_output + 7 * 8, input_7); + store_s16q_to_tran_low(final_output + 0 * 8, in[0]); + store_s16q_to_tran_low(final_output + 1 * 8, in[1]); + store_s16q_to_tran_low(final_output + 2 * 8, in[2]); + store_s16q_to_tran_low(final_output + 3 * 8, in[3]); + store_s16q_to_tran_low(final_output + 4 * 8, in[4]); + store_s16q_to_tran_low(final_output + 5 * 8, in[5]); + store_s16q_to_tran_low(final_output + 6 * 8, in[6]); + store_s16q_to_tran_low(final_output + 7 * 8, in[7]); } } diff --git a/media/libvpx/libvpx/vpx_dsp/arm/idct4x4_add_neon.asm b/media/libvpx/libvpx/vpx_dsp/arm/idct4x4_add_neon.asm --- a/media/libvpx/libvpx/vpx_dsp/arm/idct4x4_add_neon.asm +++ b/media/libvpx/libvpx/vpx_dsp/arm/idct4x4_add_neon.asm @@ -15,11 +15,11 @@ AREA ||.text||, CODE, READONLY, ALIGN=2 INCLUDE vpx_dsp/arm/idct_neon.asm.S - AREA Block, CODE, READONLY ; name this block of code + AREA Block, CODE, READONLY ;void vpx_idct4x4_16_add_neon(int16_t *input, uint8_t *dest, int stride) ; ; r0 int16_t input ; r1 uint8_t *dest ; r2 int stride) diff --git a/media/libvpx/libvpx/vpx_dsp/arm/quantize_neon.c b/media/libvpx/libvpx/vpx_dsp/arm/quantize_neon.c --- a/media/libvpx/libvpx/vpx_dsp/arm/quantize_neon.c +++ b/media/libvpx/libvpx/vpx_dsp/arm/quantize_neon.c @@ -30,22 +30,20 @@ vst1q_s16(dqcoeff, vcombine_s16(vmovn_s32(dqcoeff_0), vmovn_s32(dqcoeff_1))); #endif // CONFIG_VP9_HIGHBITDEPTH } void vpx_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *zbin_ptr, - const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *zbin_ptr, const int16_t *round_ptr, + const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan) { const int16x8_t one = vdupq_n_s16(1); const int16x8_t neg_one = vdupq_n_s16(-1); uint16x8_t eob_max; (void)scan; - (void)skip_block; - assert(!skip_block); // Process first 8 values which include a dc component. { // Only the first element of each vector is DC. const int16x8_t zbin = vld1q_s16(zbin_ptr); @@ -187,11 +185,11 @@ } // Main difference is that zbin values are halved before comparison and dqcoeff // values are divided by 2. zbin is rounded but dqcoeff is not. void vpx_quantize_b_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *zbin_ptr, + const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, @@ -200,12 +198,10 @@ const int16x8_t neg_one = vdupq_n_s16(-1); uint16x8_t eob_max; int i; (void)scan; (void)n_coeffs; // Because we will always calculate 32*32. - (void)skip_block; - assert(!skip_block); // Process first 8 values which include a dc component. { // Only the first element of each vector is DC. const int16x8_t zbin = vrshrq_n_s16(vld1q_s16(zbin_ptr), 1); diff --git a/media/libvpx/libvpx/vpx_dsp/arm/sad4d_neon.c b/media/libvpx/libvpx/vpx_dsp/arm/sad4d_neon.c --- a/media/libvpx/libvpx/vpx_dsp/arm/sad4d_neon.c +++ b/media/libvpx/libvpx/vpx_dsp/arm/sad4d_neon.c @@ -29,11 +29,11 @@ } static INLINE void sad4x_4d(const uint8_t *const src_ptr, const int src_stride, const uint8_t *const ref_array[4], const int ref_stride, const int height, - uint32_t *const res) { + uint32_t sad_array[4]) { int i; uint16x8_t abs[2] = { vdupq_n_u16(0), vdupq_n_u16(0) }; #if !defined(__aarch64__) uint16x4_t a[2]; #endif @@ -59,30 +59,30 @@ #else a[0] = vpadd_u16(vget_low_u16(abs[0]), vget_high_u16(abs[0])); a[1] = vpadd_u16(vget_low_u16(abs[1]), vget_high_u16(abs[1])); r = vpaddlq_u16(vcombine_u16(a[0], a[1])); #endif - vst1q_u32(res, r); + vst1q_u32(sad_array, r); } void vpx_sad4x4x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { - sad4x_4d(src_ptr, src_stride, ref_array, ref_stride, 4, res); + uint32_t sad_array[4]) { + sad4x_4d(src_ptr, src_stride, ref_array, ref_stride, 4, sad_array); } void vpx_sad4x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { - sad4x_4d(src_ptr, src_stride, ref_array, ref_stride, 8, res); + uint32_t sad_array[4]) { + sad4x_4d(src_ptr, src_stride, ref_array, ref_stride, 8, sad_array); } //////////////////////////////////////////////////////////////////////////////// // Can handle 512 pixels' sad sum (such as 16x32 or 32x16) -static INLINE void sad_512_pel_final_neon(const uint16x8_t *sum /*[4]*/, - uint32_t *const res) { +static INLINE void sad_512_pel_final_neon(const uint16x8_t sum[4], + uint32_t sad_array[4]) { #if defined(__aarch64__) const uint16x8_t a0 = vpaddq_u16(sum[0], sum[1]); const uint16x8_t a1 = vpaddq_u16(sum[2], sum[3]); const uint16x8_t b0 = vpaddq_u16(a0, a1); const uint32x4_t r = vpaddlq_u16(b0); @@ -93,50 +93,50 @@ const uint16x4_t a3 = vadd_u16(vget_low_u16(sum[3]), vget_high_u16(sum[3])); const uint16x4_t b0 = vpadd_u16(a0, a1); const uint16x4_t b1 = vpadd_u16(a2, a3); const uint32x4_t r = vpaddlq_u16(vcombine_u16(b0, b1)); #endif - vst1q_u32(res, r); + vst1q_u32(sad_array, r); } #if defined(__arm__) || !defined(__ARM_FEATURE_DOTPROD) // Can handle 1024 pixels' sad sum (such as 32x32) -static INLINE void sad_1024_pel_final_neon(const uint16x8_t *sum /*[4]*/, - uint32_t *const res) { +static INLINE void sad_1024_pel_final_neon(const uint16x8_t sum[4], + uint32_t sad_array[4]) { #if defined(__aarch64__) const uint16x8_t a0 = vpaddq_u16(sum[0], sum[1]); const uint16x8_t a1 = vpaddq_u16(sum[2], sum[3]); const uint32x4_t b0 = vpaddlq_u16(a0); const uint32x4_t b1 = vpaddlq_u16(a1); const uint32x4_t r = vpaddq_u32(b0, b1); - vst1q_u32(res, r); + vst1q_u32(sad_array, r); #else const uint16x4_t a0 = vpadd_u16(vget_low_u16(sum[0]), vget_high_u16(sum[0])); const uint16x4_t a1 = vpadd_u16(vget_low_u16(sum[1]), vget_high_u16(sum[1])); const uint16x4_t a2 = vpadd_u16(vget_low_u16(sum[2]), vget_high_u16(sum[2])); const uint16x4_t a3 = vpadd_u16(vget_low_u16(sum[3]), vget_high_u16(sum[3])); const uint32x4_t b0 = vpaddlq_u16(vcombine_u16(a0, a1)); const uint32x4_t b1 = vpaddlq_u16(vcombine_u16(a2, a3)); const uint32x2_t c0 = vpadd_u32(vget_low_u32(b0), vget_high_u32(b0)); const uint32x2_t c1 = vpadd_u32(vget_low_u32(b1), vget_high_u32(b1)); - vst1q_u32(res, vcombine_u32(c0, c1)); + vst1q_u32(sad_array, vcombine_u32(c0, c1)); #endif } // Can handle 2048 pixels' sad sum (such as 32x64 or 64x32) -static INLINE void sad_2048_pel_final_neon(const uint16x8_t *sum /*[4]*/, - uint32_t *const res) { +static INLINE void sad_2048_pel_final_neon(const uint16x8_t sum[4], + uint32_t sad_array[4]) { #if defined(__aarch64__) const uint32x4_t a0 = vpaddlq_u16(sum[0]); const uint32x4_t a1 = vpaddlq_u16(sum[1]); const uint32x4_t a2 = vpaddlq_u16(sum[2]); const uint32x4_t a3 = vpaddlq_u16(sum[3]); const uint32x4_t b0 = vpaddq_u32(a0, a1); const uint32x4_t b1 = vpaddq_u32(a2, a3); const uint32x4_t r = vpaddq_u32(b0, b1); - vst1q_u32(res, r); + vst1q_u32(sad_array, r); #else const uint32x4_t a0 = vpaddlq_u16(sum[0]); const uint32x4_t a1 = vpaddlq_u16(sum[1]); const uint32x4_t a2 = vpaddlq_u16(sum[2]); const uint32x4_t a3 = vpaddlq_u16(sum[3]); @@ -144,17 +144,17 @@ const uint32x2_t b1 = vadd_u32(vget_low_u32(a1), vget_high_u32(a1)); const uint32x2_t b2 = vadd_u32(vget_low_u32(a2), vget_high_u32(a2)); const uint32x2_t b3 = vadd_u32(vget_low_u32(a3), vget_high_u32(a3)); const uint32x2_t c0 = vpadd_u32(b0, b1); const uint32x2_t c1 = vpadd_u32(b2, b3); - vst1q_u32(res, vcombine_u32(c0, c1)); + vst1q_u32(sad_array, vcombine_u32(c0, c1)); #endif } // Can handle 4096 pixels' sad sum (such as 64x64) -static INLINE void sad_4096_pel_final_neon(const uint16x8_t *sum /*[8]*/, - uint32_t *const res) { +static INLINE void sad_4096_pel_final_neon(const uint16x8_t sum[8], + uint32_t sad_array[4]) { #if defined(__aarch64__) const uint32x4_t a0 = vpaddlq_u16(sum[0]); const uint32x4_t a1 = vpaddlq_u16(sum[1]); const uint32x4_t a2 = vpaddlq_u16(sum[2]); const uint32x4_t a3 = vpaddlq_u16(sum[3]); @@ -167,11 +167,11 @@ const uint32x4_t b2 = vaddq_u32(a4, a5); const uint32x4_t b3 = vaddq_u32(a6, a7); const uint32x4_t c0 = vpaddq_u32(b0, b1); const uint32x4_t c1 = vpaddq_u32(b2, b3); const uint32x4_t r = vpaddq_u32(c0, c1); - vst1q_u32(res, r); + vst1q_u32(sad_array, r); #else const uint32x4_t a0 = vpaddlq_u16(sum[0]); const uint32x4_t a1 = vpaddlq_u16(sum[1]); const uint32x4_t a2 = vpaddlq_u16(sum[2]); const uint32x4_t a3 = vpaddlq_u16(sum[3]); @@ -187,19 +187,19 @@ const uint32x2_t c1 = vadd_u32(vget_low_u32(b1), vget_high_u32(b1)); const uint32x2_t c2 = vadd_u32(vget_low_u32(b2), vget_high_u32(b2)); const uint32x2_t c3 = vadd_u32(vget_low_u32(b3), vget_high_u32(b3)); const uint32x2_t d0 = vpadd_u32(c0, c1); const uint32x2_t d1 = vpadd_u32(c2, c3); - vst1q_u32(res, vcombine_u32(d0, d1)); + vst1q_u32(sad_array, vcombine_u32(d0, d1)); #endif } #endif static INLINE void sad8x_4d(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res, const int height) { + uint32_t sad_array[4], const int height) { int i, j; const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2], ref_array[3] }; uint16x8_t sum[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0) }; @@ -212,29 +212,29 @@ ref_loop[j] += ref_stride; sum[j] = vabal_u8(sum[j], s, b_u8); } } - sad_512_pel_final_neon(sum, res); + sad_512_pel_final_neon(sum, sad_array); } void vpx_sad8x4x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { - sad8x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 4); + uint32_t sad_array[4]) { + sad8x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 4); } void vpx_sad8x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { - sad8x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 8); + uint32_t sad_array[4]) { + sad8x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 8); } void vpx_sad8x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { - sad8x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 16); + uint32_t sad_array[4]) { + sad8x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 16); } //////////////////////////////////////////////////////////////////////////////// #if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \ @@ -247,11 +247,11 @@ *sum = vdotq_u32(*sum, diff, vdupq_n_u8(1)); } static INLINE void sad16x_4d(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res, const int height) { + uint32_t sad_array[4], const int height) { int i; uint32x4_t r0, r1; const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2], ref_array[3] }; uint32x4_t sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0), @@ -265,11 +265,11 @@ sad16_neon(ref_loop[3] + i * ref_stride, s, &sum[3]); } r0 = vpaddq_u32(sum[0], sum[1]); r1 = vpaddq_u32(sum[2], sum[3]); - vst1q_u32(res, vpaddq_u32(r0, r1)); + vst1q_u32(sad_array, vpaddq_u32(r0, r1)); } #else static INLINE void sad16_neon(const uint8_t *ref_ptr, const uint8x16_t src_ptr, @@ -279,11 +279,11 @@ *sum = vabal_u8(*sum, vget_high_u8(src_ptr), vget_high_u8(r)); } static INLINE void sad16x_4d(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res, const int height) { + uint32_t sad_array[4], const int height) { int i; const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2], ref_array[3] }; uint16x8_t sum[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0) }; @@ -300,41 +300,41 @@ ref_loop[2] += ref_stride; sad16_neon(ref_loop[3], s, &sum[3]); ref_loop[3] += ref_stride; } - sad_512_pel_final_neon(sum, res); + sad_512_pel_final_neon(sum, sad_array); } #endif void vpx_sad16x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { - sad16x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 8); + uint32_t sad_array[4]) { + sad16x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 8); } void vpx_sad16x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { - sad16x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 16); + uint32_t sad_array[4]) { + sad16x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 16); } void vpx_sad16x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { - sad16x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 32); + uint32_t sad_array[4]) { + sad16x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 32); } //////////////////////////////////////////////////////////////////////////////// #if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \ (__ARM_FEATURE_DOTPROD == 1) static INLINE void sad32x_4d(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res, const int height) { + uint32_t sad_array[4], const int height) { int i; uint32x4_t r0, r1; const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2], ref_array[3] }; @@ -363,29 +363,29 @@ ref_loop[3] += ref_stride; } r0 = vpaddq_u32(sum[0], sum[1]); r1 = vpaddq_u32(sum[2], sum[3]); - vst1q_u32(res, vpaddq_u32(r0, r1)); + vst1q_u32(sad_array, vpaddq_u32(r0, r1)); } void vpx_sad32x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { - sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 16); + uint32_t sad_array[4]) { + sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 16); } void vpx_sad32x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { - sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 32); + uint32_t sad_array[4]) { + sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 32); } void vpx_sad32x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { - sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 64); + uint32_t sad_array[4]) { + sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 64); } #else static INLINE void sad32x_4d(const uint8_t *src_ptr, int src_stride, @@ -420,30 +420,30 @@ } } void vpx_sad32x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { + uint32_t sad_array[4]) { uint16x8_t sum[4]; sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, 16, sum); - sad_512_pel_final_neon(sum, res); + sad_512_pel_final_neon(sum, sad_array); } void vpx_sad32x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { + uint32_t sad_array[4]) { uint16x8_t sum[4]; sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, 32, sum); - sad_1024_pel_final_neon(sum, res); + sad_1024_pel_final_neon(sum, sad_array); } void vpx_sad32x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { + uint32_t sad_array[4]) { uint16x8_t sum[4]; sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, 64, sum); - sad_2048_pel_final_neon(sum, res); + sad_2048_pel_final_neon(sum, sad_array); } #endif //////////////////////////////////////////////////////////////////////////////// @@ -451,11 +451,11 @@ #if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \ (__ARM_FEATURE_DOTPROD == 1) void vpx_sad64x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { + uint32_t sad_array[4]) { int i; uint32x4_t r0, r1; const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2], ref_array[3] }; uint32x4_t sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0), @@ -495,16 +495,16 @@ ref_loop[3] += ref_stride; } r0 = vpaddq_u32(sum[0], sum[1]); r1 = vpaddq_u32(sum[2], sum[3]); - vst1q_u32(res, vpaddq_u32(r0, r1)); + vst1q_u32(sad_array, vpaddq_u32(r0, r1)); } void vpx_sad64x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { + uint32_t sad_array[4]) { int i; uint32x4_t r0, r1, r2, r3; const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2], ref_array[3] }; uint32x4_t sum[8] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0), @@ -549,18 +549,18 @@ r1 = vpaddq_u32(sum[2], sum[3]); r2 = vpaddq_u32(sum[4], sum[5]); r3 = vpaddq_u32(sum[6], sum[7]); r0 = vpaddq_u32(r0, r1); r1 = vpaddq_u32(r2, r3); - vst1q_u32(res, vpaddq_u32(r0, r1)); + vst1q_u32(sad_array, vpaddq_u32(r0, r1)); } #else void vpx_sad64x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { + uint32_t sad_array[4]) { int i; const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2], ref_array[3] }; uint16x8_t sum[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0) }; @@ -597,16 +597,16 @@ ref_loop[1] += ref_stride; ref_loop[2] += ref_stride; ref_loop[3] += ref_stride; } - sad_2048_pel_final_neon(sum, res); + sad_2048_pel_final_neon(sum, sad_array); } void vpx_sad64x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { + uint32_t sad_array[4]) { int i; const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2], ref_array[3] }; uint16x8_t sum[8] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0), @@ -644,9 +644,9 @@ ref_loop[1] += ref_stride; ref_loop[2] += ref_stride; ref_loop[3] += ref_stride; } - sad_4096_pel_final_neon(sum, res); + sad_4096_pel_final_neon(sum, sad_array); } #endif diff --git a/media/libvpx/libvpx/vpx_dsp/arm/sad_neon.c b/media/libvpx/libvpx/vpx_dsp/arm/sad_neon.c --- a/media/libvpx/libvpx/vpx_dsp/arm/sad_neon.c +++ b/media/libvpx/libvpx/vpx_dsp/arm/sad_neon.c @@ -108,11 +108,11 @@ abs = vabal_u8(abs, a_u8, avg); } return abs; } -#define sad8xN(n) \ +#define SAD8XN(n) \ uint32_t vpx_sad8x##n##_neon(const uint8_t *src_ptr, int src_stride, \ const uint8_t *ref_ptr, int ref_stride) { \ const uint16x8_t abs = sad8x(src_ptr, src_stride, ref_ptr, ref_stride, n); \ return horizontal_add_uint16x8(abs); \ } \ @@ -123,13 +123,13 @@ const uint16x8_t abs = \ sad8x_avg(src_ptr, src_stride, ref_ptr, ref_stride, second_pred, n); \ return horizontal_add_uint16x8(abs); \ } -sad8xN(4); -sad8xN(8); -sad8xN(16); +SAD8XN(4) +SAD8XN(8) +SAD8XN(16) static INLINE uint16x8_t sad16x(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const int height) { int i; @@ -165,11 +165,11 @@ abs = vabal_u8(abs, vget_high_u8(a_u8), vget_high_u8(avg)); } return abs; } -#define sad16xN(n) \ +#define SAD16XN(n) \ uint32_t vpx_sad16x##n##_neon(const uint8_t *src_ptr, int src_stride, \ const uint8_t *ref_ptr, int ref_stride) { \ const uint16x8_t abs = \ sad16x(src_ptr, src_stride, ref_ptr, ref_stride, n); \ return horizontal_add_uint16x8(abs); \ @@ -181,13 +181,13 @@ const uint16x8_t abs = \ sad16x_avg(src_ptr, src_stride, ref_ptr, ref_stride, second_pred, n); \ return horizontal_add_uint16x8(abs); \ } -sad16xN(8); -sad16xN(16); -sad16xN(32); +SAD16XN(8) +SAD16XN(16) +SAD16XN(32) static INLINE uint16x8_t sad32x(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const int height) { int i; @@ -233,11 +233,11 @@ abs = vabal_u8(abs, vget_high_u8(a_hi), vget_high_u8(avg_hi)); } return abs; } -#define sad32xN(n) \ +#define SAD32XN(n) \ uint32_t vpx_sad32x##n##_neon(const uint8_t *src_ptr, int src_stride, \ const uint8_t *ref_ptr, int ref_stride) { \ const uint16x8_t abs = \ sad32x(src_ptr, src_stride, ref_ptr, ref_stride, n); \ return horizontal_add_uint16x8(abs); \ @@ -249,13 +249,13 @@ const uint16x8_t abs = \ sad32x_avg(src_ptr, src_stride, ref_ptr, ref_stride, second_pred, n); \ return horizontal_add_uint16x8(abs); \ } -sad32xN(16); -sad32xN(32); -sad32xN(64); +SAD32XN(16) +SAD32XN(32) +SAD32XN(64) static INLINE uint32x4_t sad64x(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const int height) { int i; @@ -331,11 +331,11 @@ const uint32x4_t sum = vpaddlq_u16(abs_0); return vpadalq_u16(sum, abs_1); } } -#define sad64xN(n) \ +#define SAD64XN(n) \ uint32_t vpx_sad64x##n##_neon(const uint8_t *src_ptr, int src_stride, \ const uint8_t *ref_ptr, int ref_stride) { \ const uint32x4_t abs = \ sad64x(src_ptr, src_stride, ref_ptr, ref_stride, n); \ return horizontal_add_uint32x4(abs); \ @@ -347,7 +347,7 @@ const uint32x4_t abs = \ sad64x_avg(src_ptr, src_stride, ref_ptr, ref_stride, second_pred, n); \ return horizontal_add_uint32x4(abs); \ } -sad64xN(32); -sad64xN(64); +SAD64XN(32) +SAD64XN(64) diff --git a/media/libvpx/libvpx/vpx_dsp/arm/subpel_variance_neon.c b/media/libvpx/libvpx/vpx_dsp/arm/subpel_variance_neon.c --- a/media/libvpx/libvpx/vpx_dsp/arm/subpel_variance_neon.c +++ b/media/libvpx/libvpx/vpx_dsp/arm/subpel_variance_neon.c @@ -95,11 +95,11 @@ } } // 4xM filter writes an extra row to fdata because it processes two rows at a // time. -#define sub_pixel_varianceNxM(n, m) \ +#define SUB_PIXEL_VARIANCENXM(n, m) \ uint32_t vpx_sub_pixel_variance##n##x##m##_neon( \ const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, \ const uint8_t *ref_ptr, int ref_stride, uint32_t *sse) { \ uint8_t temp0[n * (m + (n == 4 ? 2 : 1))]; \ uint8_t temp1[n * m]; \ @@ -121,27 +121,27 @@ bilinear_filters[y_offset]); \ } \ return vpx_variance##n##x##m(temp1, n, ref_ptr, ref_stride, sse); \ } -sub_pixel_varianceNxM(4, 4); -sub_pixel_varianceNxM(4, 8); -sub_pixel_varianceNxM(8, 4); -sub_pixel_varianceNxM(8, 8); -sub_pixel_varianceNxM(8, 16); -sub_pixel_varianceNxM(16, 8); -sub_pixel_varianceNxM(16, 16); -sub_pixel_varianceNxM(16, 32); -sub_pixel_varianceNxM(32, 16); -sub_pixel_varianceNxM(32, 32); -sub_pixel_varianceNxM(32, 64); -sub_pixel_varianceNxM(64, 32); -sub_pixel_varianceNxM(64, 64); +SUB_PIXEL_VARIANCENXM(4, 4) +SUB_PIXEL_VARIANCENXM(4, 8) +SUB_PIXEL_VARIANCENXM(8, 4) +SUB_PIXEL_VARIANCENXM(8, 8) +SUB_PIXEL_VARIANCENXM(8, 16) +SUB_PIXEL_VARIANCENXM(16, 8) +SUB_PIXEL_VARIANCENXM(16, 16) +SUB_PIXEL_VARIANCENXM(16, 32) +SUB_PIXEL_VARIANCENXM(32, 16) +SUB_PIXEL_VARIANCENXM(32, 32) +SUB_PIXEL_VARIANCENXM(32, 64) +SUB_PIXEL_VARIANCENXM(64, 32) +SUB_PIXEL_VARIANCENXM(64, 64) // 4xM filter writes an extra row to fdata because it processes two rows at a // time. -#define sub_pixel_avg_varianceNxM(n, m) \ +#define SUB_PIXEL_AVG_VARIANCENXM(n, m) \ uint32_t vpx_sub_pixel_avg_variance##n##x##m##_neon( \ const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, \ const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, \ const uint8_t *second_pred) { \ uint8_t temp0[n * (m + (n == 4 ? 2 : 1))]; \ @@ -167,18 +167,18 @@ vpx_comp_avg_pred(temp0, second_pred, n, m, temp1, n); \ \ return vpx_variance##n##x##m(temp0, n, ref_ptr, ref_stride, sse); \ } -sub_pixel_avg_varianceNxM(4, 4); -sub_pixel_avg_varianceNxM(4, 8); -sub_pixel_avg_varianceNxM(8, 4); -sub_pixel_avg_varianceNxM(8, 8); -sub_pixel_avg_varianceNxM(8, 16); -sub_pixel_avg_varianceNxM(16, 8); -sub_pixel_avg_varianceNxM(16, 16); -sub_pixel_avg_varianceNxM(16, 32); -sub_pixel_avg_varianceNxM(32, 16); -sub_pixel_avg_varianceNxM(32, 32); -sub_pixel_avg_varianceNxM(32, 64); -sub_pixel_avg_varianceNxM(64, 32); -sub_pixel_avg_varianceNxM(64, 64); +SUB_PIXEL_AVG_VARIANCENXM(4, 4) +SUB_PIXEL_AVG_VARIANCENXM(4, 8) +SUB_PIXEL_AVG_VARIANCENXM(8, 4) +SUB_PIXEL_AVG_VARIANCENXM(8, 8) +SUB_PIXEL_AVG_VARIANCENXM(8, 16) +SUB_PIXEL_AVG_VARIANCENXM(16, 8) +SUB_PIXEL_AVG_VARIANCENXM(16, 16) +SUB_PIXEL_AVG_VARIANCENXM(16, 32) +SUB_PIXEL_AVG_VARIANCENXM(32, 16) +SUB_PIXEL_AVG_VARIANCENXM(32, 32) +SUB_PIXEL_AVG_VARIANCENXM(32, 64) +SUB_PIXEL_AVG_VARIANCENXM(64, 32) +SUB_PIXEL_AVG_VARIANCENXM(64, 64) diff --git a/media/libvpx/libvpx/vpx_dsp/arm/transpose_neon.h b/media/libvpx/libvpx/vpx_dsp/arm/transpose_neon.h --- a/media/libvpx/libvpx/vpx_dsp/arm/transpose_neon.h +++ b/media/libvpx/libvpx/vpx_dsp/arm/transpose_neon.h @@ -1182,10 +1182,49 @@ *o13 = e5.val[1]; *o14 = e6.val[1]; *o15 = e7.val[1]; } +static INLINE void transpose_s16_16x16(int16x8_t *in0, int16x8_t *in1) { + int16x8_t t[8]; + + // transpose the 4 8x8 quadrants separately but first swap quadrants 2 and 3. + t[0] = in0[8]; + t[1] = in0[9]; + t[2] = in0[10]; + t[3] = in0[11]; + t[4] = in0[12]; + t[5] = in0[13]; + t[6] = in0[14]; + t[7] = in0[15]; + in0[8] = in1[0]; + in0[9] = in1[1]; + in0[10] = in1[2]; + in0[11] = in1[3]; + in0[12] = in1[4]; + in0[13] = in1[5]; + in0[14] = in1[6]; + in0[15] = in1[7]; + in1[0] = t[0]; + in1[1] = t[1]; + in1[2] = t[2]; + in1[3] = t[3]; + in1[4] = t[4]; + in1[5] = t[5]; + in1[6] = t[6]; + in1[7] = t[7]; + + transpose_s16_8x8(&in0[0], &in0[1], &in0[2], &in0[3], &in0[4], &in0[5], + &in0[6], &in0[7]); + transpose_s16_8x8(&in0[8], &in0[9], &in0[10], &in0[11], &in0[12], &in0[13], + &in0[14], &in0[15]); + transpose_s16_8x8(&in1[0], &in1[1], &in1[2], &in1[3], &in1[4], &in1[5], + &in1[6], &in1[7]); + transpose_s16_8x8(&in1[8], &in1[9], &in1[10], &in1[11], &in1[12], &in1[13], + &in1[14], &in1[15]); +} + static INLINE void load_and_transpose_u8_4x8(const uint8_t *a, const int a_stride, uint8x8_t *a0, uint8x8_t *a1, uint8x8_t *a2, uint8x8_t *a3) { uint8x8_t a4, a5, a6, a7; diff --git a/media/libvpx/libvpx/vpx_dsp/arm/variance_neon.c b/media/libvpx/libvpx/vpx_dsp/arm/variance_neon.c --- a/media/libvpx/libvpx/vpx_dsp/arm/variance_neon.c +++ b/media/libvpx/libvpx/vpx_dsp/arm/variance_neon.c @@ -266,11 +266,11 @@ const uint8_t *ref_ptr, int ref_stride, unsigned int *sse, int *sum) { variance_neon_w16(src_ptr, src_stride, ref_ptr, ref_stride, 16, 16, sse, sum); } -#define varianceNxM(n, m, shift) \ +#define VARIANCENXM(n, m, shift) \ unsigned int vpx_variance##n##x##m##_neon( \ const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \ int ref_stride, unsigned int *sse) { \ int sum; \ if (n == 4) \ @@ -286,20 +286,20 @@ return *sse - ((sum * sum) >> shift); \ else \ return *sse - (uint32_t)(((int64_t)sum * sum) >> shift); \ } -varianceNxM(4, 4, 4); -varianceNxM(4, 8, 5); -varianceNxM(8, 4, 5); -varianceNxM(8, 8, 6); -varianceNxM(8, 16, 7); -varianceNxM(16, 8, 7); -varianceNxM(16, 16, 8); -varianceNxM(16, 32, 9); -varianceNxM(32, 16, 9); -varianceNxM(32, 32, 10); +VARIANCENXM(4, 4, 4) +VARIANCENXM(4, 8, 5) +VARIANCENXM(8, 4, 5) +VARIANCENXM(8, 8, 6) +VARIANCENXM(8, 16, 7) +VARIANCENXM(16, 8, 7) +VARIANCENXM(16, 16, 8) +VARIANCENXM(16, 32, 9) +VARIANCENXM(32, 16, 9) +VARIANCENXM(32, 32, 10) unsigned int vpx_variance32x64_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse) { int sum1, sum2; diff --git a/media/libvpx/libvpx/vpx_dsp/arm/vpx_convolve8_neon_asm.h b/media/libvpx/libvpx/vpx_dsp/arm/vpx_convolve8_neon_asm.h --- a/media/libvpx/libvpx/vpx_dsp/arm/vpx_convolve8_neon_asm.h +++ b/media/libvpx/libvpx/vpx_dsp/arm/vpx_convolve8_neon_asm.h @@ -15,15 +15,15 @@ void vpx_convolve8_##dir##_filter_##type##_neon( \ const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, \ ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, \ int x_step_q4, int y0_q4, int y_step_q4, int w, int h); -DECLARE_FILTER(horiz, type1); -DECLARE_FILTER(avg_horiz, type1); -DECLARE_FILTER(horiz, type2); -DECLARE_FILTER(avg_horiz, type2); -DECLARE_FILTER(vert, type1); -DECLARE_FILTER(avg_vert, type1); -DECLARE_FILTER(vert, type2); -DECLARE_FILTER(avg_vert, type2); +DECLARE_FILTER(horiz, type1) +DECLARE_FILTER(avg_horiz, type1) +DECLARE_FILTER(horiz, type2) +DECLARE_FILTER(avg_horiz, type2) +DECLARE_FILTER(vert, type1) +DECLARE_FILTER(avg_vert, type1) +DECLARE_FILTER(vert, type2) +DECLARE_FILTER(avg_vert, type2) #endif // VPX_VPX_DSP_ARM_VPX_CONVOLVE8_NEON_ASM_H_ diff --git a/media/libvpx/libvpx/vpx_dsp/arm/vpx_convolve8_neon_asm.c b/media/libvpx/libvpx/vpx_dsp/arm/vpx_convolve8_neon_asm.c --- a/media/libvpx/libvpx/vpx_dsp/arm/vpx_convolve8_neon_asm.c +++ b/media/libvpx/libvpx/vpx_dsp/arm/vpx_convolve8_neon_asm.c @@ -33,9 +33,9 @@ src, src_stride, dst, dst_stride, filter, x0_q4, x_step_q4, y0_q4, \ y_step_q4, w, h); \ } \ } -DEFINE_FILTER(horiz); -DEFINE_FILTER(avg_horiz); -DEFINE_FILTER(vert); -DEFINE_FILTER(avg_vert); +DEFINE_FILTER(horiz) +DEFINE_FILTER(avg_horiz) +DEFINE_FILTER(vert) +DEFINE_FILTER(avg_vert) diff --git a/media/libvpx/libvpx/vpx_dsp/fastssim.c b/media/libvpx/libvpx/vpx_dsp/fastssim.c --- a/media/libvpx/libvpx/vpx_dsp/fastssim.c +++ b/media/libvpx/libvpx/vpx_dsp/fastssim.c @@ -45,11 +45,11 @@ fs_level *level; int nlevels; unsigned *col_buf; }; -static void fs_ctx_init(fs_ctx *_ctx, int _w, int _h, int _nlevels) { +static int fs_ctx_init(fs_ctx *_ctx, int _w, int _h, int _nlevels) { unsigned char *data; size_t data_size; int lw; int lh; int l; @@ -69,10 +69,11 @@ data_size += level_size; lw = (lw + 1) >> 1; lh = (lh + 1) >> 1; } data = (unsigned char *)malloc(data_size); + if (!data) return -1; _ctx->level = (fs_level *)data; _ctx->nlevels = _nlevels; data += _nlevels * sizeof(*_ctx->level); lw = (_w + 1) >> 1; lh = (_h + 1) >> 1; @@ -93,10 +94,11 @@ data += im_size * sizeof(*_ctx->level[l].ssim); lw = (lw + 1) >> 1; lh = (lh + 1) >> 1; } _ctx->col_buf = (unsigned *)data; + return 0; } static void fs_ctx_clear(fs_ctx *_ctx) { free(_ctx->level); } static void fs_downsample_level(fs_ctx *_ctx, int _l) { @@ -454,11 +456,11 @@ uint32_t _shift) { fs_ctx ctx; double ret; int l; ret = 1; - fs_ctx_init(&ctx, _w, _h, FS_NLEVELS); + if (fs_ctx_init(&ctx, _w, _h, FS_NLEVELS)) return 99.0; fs_downsample_level0(&ctx, _src, _systride, _dst, _dystride, _w, _h, _bd, _shift); for (l = 0; l < FS_NLEVELS - 1; l++) { fs_calc_structure(&ctx, l, _bd); ret *= fs_average(&ctx, l); diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/avg_pred_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/avg_pred_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/avg_pred_lsx.c @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include "./vpx_dsp_rtcd.h" +#include "vpx/vpx_integer.h" +#include "vpx_util/loongson_intrinsics.h" + +void vpx_comp_avg_pred_lsx(uint8_t *comp_pred, const uint8_t *pred, int width, + int height, const uint8_t *ref, int ref_stride) { + // width > 8 || width == 8 || width == 4 + if (width > 8) { + int i, j; + for (i = 0; i < height; ++i) { + for (j = 0; j < width; j += 16) { + __m128i p, r, avg; + + p = __lsx_vld(pred + j, 0); + r = __lsx_vld(ref + j, 0); + avg = __lsx_vavgr_bu(p, r); + __lsx_vst(avg, comp_pred + j, 0); + } + comp_pred += width; + pred += width; + ref += ref_stride; + } + } else if (width == 8) { + int i = height * width; + do { + __m128i p, r, r_0, r_1; + + p = __lsx_vld(pred, 0); + r_0 = __lsx_vld(ref, 0); + ref += ref_stride; + r_1 = __lsx_vld(ref, 0); + ref += ref_stride; + r = __lsx_vilvl_d(r_1, r_0); + r = __lsx_vavgr_bu(p, r); + + __lsx_vst(r, comp_pred, 0); + + pred += 16; + comp_pred += 16; + i -= 16; + } while (i); + } else { // width = 4 + int i = height * width; + assert(width == 4); + do { + __m128i p, r, r_0, r_1, r_2, r_3; + p = __lsx_vld(pred, 0); + + if (width == ref_stride) { + r = __lsx_vld(ref, 0); + ref += 16; + } else { + r_0 = __lsx_vld(ref, 0); + ref += ref_stride; + r_1 = __lsx_vld(ref, 0); + ref += ref_stride; + r_2 = __lsx_vld(ref, 0); + ref += ref_stride; + r_3 = __lsx_vld(ref, 0); + ref += ref_stride; + DUP2_ARG2(__lsx_vilvl_w, r_1, r_0, r_3, r_2, r_0, r_2); + r = __lsx_vilvl_d(r_2, r_0); + } + r = __lsx_vavgr_bu(p, r); + + __lsx_vst(r, comp_pred, 0); + comp_pred += 16; + pred += 16; + i -= 16; + } while (i); + } +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/fwd_dct32x32_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/fwd_dct32x32_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/fwd_dct32x32_lsx.c @@ -0,0 +1,1176 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "./vpx_dsp_rtcd.h" +#include "vpx_dsp/loongarch/fwd_txfm_lsx.h" +#include "vpx_dsp/fwd_txfm.h" + +#define UNPCK_SH_SW(in, out0, out1) \ + { \ + out0 = __lsx_vsllwil_w_h(in, 0); \ + out1 = __lsx_vexth_w_h(in); \ + } + +static void fdct8x32_1d_column_load_butterfly(const int16_t *input, + int32_t src_stride, + int16_t *temp_buff) { + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i step0, step1, step2, step3; + __m128i in0_1, in1_1, in2_1, in3_1, in4_1, in5_1, in6_1, in7_1; + __m128i step0_1, step1_1, step2_1, step3_1; + + int32_t stride = src_stride << 1; + int32_t stride2 = stride << 1; + int32_t stride3 = stride2 + stride; + const int16_t *input_tmp = (int16_t *)input; + + in0 = __lsx_vld(input_tmp, 0); + DUP2_ARG2(__lsx_vldx, input_tmp, stride, input_tmp, stride2, in1, in2); + in3 = __lsx_vldx(input_tmp, stride3); + + input_tmp += stride2; + in0_1 = __lsx_vld(input_tmp, 0); + DUP2_ARG2(__lsx_vldx, input_tmp, stride, input_tmp, stride2, in1_1, in2_1); + in3_1 = __lsx_vldx(input_tmp, stride3); + + input_tmp = input + (src_stride * 24); + in4_1 = __lsx_vld(input_tmp, 0); + DUP2_ARG2(__lsx_vldx, input_tmp, stride, input_tmp, stride2, in5_1, in6_1); + in7_1 = __lsx_vldx(input_tmp, stride3); + + input_tmp += stride2; + in4 = __lsx_vld(input_tmp, 0); + DUP2_ARG2(__lsx_vldx, input_tmp, stride, input_tmp, stride2, in5, in6); + in7 = __lsx_vldx(input_tmp, stride3); + + DUP4_ARG2(__lsx_vslli_h, in0, 2, in1, 2, in2, 2, in3, 2, in0, in1, in2, in3); + DUP4_ARG2(__lsx_vslli_h, in4, 2, in5, 2, in6, 2, in7, 2, in4, in5, in6, in7); + DUP4_ARG2(__lsx_vslli_h, in0_1, 2, in1_1, 2, in2_1, 2, in3_1, 2, in0_1, in1_1, + in2_1, in3_1); + DUP4_ARG2(__lsx_vslli_h, in4_1, 2, in5_1, 2, in6_1, 2, in7_1, 2, in4_1, in5_1, + in6_1, in7_1); + LSX_BUTTERFLY_8_H(in0, in1, in2, in3, in4, in5, in6, in7, step0, step1, step2, + step3, in4, in5, in6, in7); + LSX_BUTTERFLY_8_H(in0_1, in1_1, in2_1, in3_1, in4_1, in5_1, in6_1, in7_1, + step0_1, step1_1, step2_1, step3_1, in4_1, in5_1, in6_1, + in7_1); + + __lsx_vst(step0, temp_buff, 0); + __lsx_vst(step1, temp_buff, 16); + __lsx_vst(step2, temp_buff, 32); + __lsx_vst(step3, temp_buff, 48); + + __lsx_vst(in4, temp_buff, 448); + __lsx_vst(in5, temp_buff, 464); + __lsx_vst(in6, temp_buff, 480); + __lsx_vst(in7, temp_buff, 496); + + __lsx_vst(step0_1, temp_buff, 64); + __lsx_vst(step1_1, temp_buff, 80); + __lsx_vst(step2_1, temp_buff, 96); + __lsx_vst(step3_1, temp_buff, 112); + + __lsx_vst(in4_1, temp_buff, 384); + __lsx_vst(in5_1, temp_buff, 400); + __lsx_vst(in6_1, temp_buff, 416); + __lsx_vst(in7_1, temp_buff, 432); + + /* 3rd and 4th set */ + input_tmp = input + (src_stride * 8); + in0 = __lsx_vld(input_tmp, 0); + DUP2_ARG2(__lsx_vldx, input_tmp, stride, input_tmp, stride2, in1, in2); + in3 = __lsx_vldx(input_tmp, stride3); + + input_tmp += stride2; + in0_1 = __lsx_vld(input_tmp, 0); + DUP2_ARG2(__lsx_vldx, input_tmp, stride, input_tmp, stride2, in1_1, in2_1); + in3_1 = __lsx_vldx(input_tmp, stride3); + + input_tmp += stride2; + in4_1 = __lsx_vld(input_tmp, 0); + DUP2_ARG2(__lsx_vldx, input_tmp, stride, input_tmp, stride2, in5_1, in6_1); + in7_1 = __lsx_vldx(input_tmp, stride3); + + input_tmp += stride2; + in4 = __lsx_vld(input_tmp, 0); + DUP2_ARG2(__lsx_vldx, input_tmp, stride, input_tmp, stride2, in5, in6); + in7 = __lsx_vldx(input_tmp, stride3); + DUP4_ARG2(__lsx_vslli_h, in0, 2, in1, 2, in2, 2, in3, 2, in0, in1, in2, in3); + DUP4_ARG2(__lsx_vslli_h, in4, 2, in5, 2, in6, 2, in7, 2, in4, in5, in6, in7); + DUP4_ARG2(__lsx_vslli_h, in0_1, 2, in1_1, 2, in2_1, 2, in3_1, 2, in0_1, in1_1, + in2_1, in3_1); + DUP4_ARG2(__lsx_vslli_h, in4_1, 2, in5_1, 2, in6_1, 2, in7_1, 2, in4_1, in5_1, + in6_1, in7_1); + + LSX_BUTTERFLY_8_H(in0, in1, in2, in3, in4, in5, in6, in7, step0, step1, step2, + step3, in4, in5, in6, in7); + LSX_BUTTERFLY_8_H(in0_1, in1_1, in2_1, in3_1, in4_1, in5_1, in6_1, in7_1, + step0_1, step1_1, step2_1, step3_1, in4_1, in5_1, in6_1, + in7_1); + + __lsx_vst(step0, temp_buff, 128); + __lsx_vst(step1, temp_buff, 144); + __lsx_vst(step2, temp_buff, 160); + __lsx_vst(step3, temp_buff, 176); + + __lsx_vst(in4, temp_buff, 320); + __lsx_vst(in5, temp_buff, 336); + __lsx_vst(in6, temp_buff, 352); + __lsx_vst(in7, temp_buff, 368); + + __lsx_vst(step0_1, temp_buff, 192); + __lsx_vst(step1_1, temp_buff, 208); + __lsx_vst(step2_1, temp_buff, 224); + __lsx_vst(step3_1, temp_buff, 240); + + __lsx_vst(in4_1, temp_buff, 256); + __lsx_vst(in5_1, temp_buff, 272); + __lsx_vst(in6_1, temp_buff, 288); + __lsx_vst(in7_1, temp_buff, 304); +} + +static void fdct8x32_1d_column_even_store(int16_t *input, int16_t *temp) { + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i in8, in9, in10, in11, in12, in13, in14, in15; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i temp0, temp1; + + /* fdct even */ + DUP4_ARG2(__lsx_vld, input, 0, input, 16, input, 32, input, 48, in0, in1, in2, + in3); + DUP4_ARG2(__lsx_vld, input, 192, input, 208, input, 224, input, 240, in12, + in13, in14, in15); + LSX_BUTTERFLY_8_H(in0, in1, in2, in3, in12, in13, in14, in15, vec0, vec1, + vec2, vec3, in12, in13, in14, in15); + DUP4_ARG2(__lsx_vld, input, 64, input, 80, input, 96, input, 112, in4, in5, + in6, in7); + DUP4_ARG2(__lsx_vld, input, 128, input, 144, input, 160, input, 176, in8, in9, + in10, in11); + LSX_BUTTERFLY_8_H(in4, in5, in6, in7, in8, in9, in10, in11, vec4, vec5, vec6, + vec7, in8, in9, in10, in11); + + /* Stage 3 */ + DUP4_ARG2(__lsx_vadd_h, vec0, vec7, vec1, vec6, vec2, vec5, vec3, vec4, in0, + in1, in2, in3); + LSX_BUTTERFLY_4_H(in0, in1, in2, in3, temp0, in4, in1, in0); + DOTP_CONST_PAIR(temp0, in4, cospi_16_64, cospi_16_64, temp1, temp0); + FDCT32_POSTPROC_2V_POS_H(temp0, temp1); + __lsx_vst(temp0, temp, 0); + __lsx_vst(temp1, temp, 1024); + + DOTP_CONST_PAIR(in0, in1, cospi_24_64, cospi_8_64, temp1, temp0); + FDCT32_POSTPROC_2V_POS_H(temp0, temp1); + __lsx_vst(temp0, temp, 512); + __lsx_vst(temp1, temp, 1536); + + DUP4_ARG2(__lsx_vsub_h, vec0, vec7, vec1, vec6, vec2, vec5, vec3, vec4, vec7, + vec6, vec5, vec4); + DOTP_CONST_PAIR(vec6, vec5, cospi_16_64, cospi_16_64, vec5, vec6); + DUP2_ARG2(__lsx_vadd_h, vec4, vec5, vec7, vec6, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_28_64, cospi_4_64, temp1, temp0); + FDCT32_POSTPROC_2V_POS_H(temp0, temp1); + __lsx_vst(temp0, temp, 256); + __lsx_vst(temp1, temp, 1792); + + DUP2_ARG2(__lsx_vsub_h, vec4, vec5, vec7, vec6, vec4, vec7); + DOTP_CONST_PAIR(vec7, vec4, cospi_12_64, cospi_20_64, temp1, temp0); + FDCT32_POSTPROC_2V_POS_H(temp0, temp1); + __lsx_vst(temp0, temp, 1280); + __lsx_vst(temp1, temp, 768); + + DOTP_CONST_PAIR(in13, in10, cospi_16_64, cospi_16_64, vec2, vec5); + DOTP_CONST_PAIR(in12, in11, cospi_16_64, cospi_16_64, vec3, vec4); + DUP4_ARG2(__lsx_vadd_h, in8, vec3, in9, vec2, in14, vec5, in15, vec4, in0, + vec1, vec6, in2); + DOTP_CONST_PAIR(vec6, vec1, cospi_24_64, cospi_8_64, in1, in3); + DUP2_ARG2(__lsx_vadd_h, in0, in1, in2, in3, vec0, vec7); + DOTP_CONST_PAIR(vec7, vec0, cospi_30_64, cospi_2_64, temp1, temp0); + FDCT32_POSTPROC_2V_POS_H(temp0, temp1); + __lsx_vst(temp0, temp, 128); + __lsx_vst(temp1, temp, 1920); + + DUP2_ARG2(__lsx_vsub_h, in0, in1, in2, in3, in0, in2); + DOTP_CONST_PAIR(in2, in0, cospi_14_64, cospi_18_64, temp1, temp0); + FDCT32_POSTPROC_2V_POS_H(temp0, temp1); + __lsx_vst(temp0, temp, 1152); + __lsx_vst(temp1, temp, 896); + + DUP2_ARG2(__lsx_vsub_h, in9, vec2, in14, vec5, vec2, vec5); + temp0 = __lsx_vneg_h(vec2); + DOTP_CONST_PAIR(temp0, vec5, cospi_24_64, cospi_8_64, in2, in1); + DUP4_ARG2(__lsx_vsub_h, in8, vec3, in15, vec4, in3, in2, in0, in1, in3, in0, + vec2, vec5); + DOTP_CONST_PAIR(vec5, vec2, cospi_22_64, cospi_10_64, temp1, temp0); + FDCT32_POSTPROC_2V_POS_H(temp0, temp1); + __lsx_vst(temp0, temp, 640); + __lsx_vst(temp1, temp, 1408); + + DUP2_ARG2(__lsx_vadd_h, in3, in2, in0, in1, vec3, vec4); + DOTP_CONST_PAIR(vec4, vec3, cospi_6_64, cospi_26_64, temp0, temp1); + FDCT32_POSTPROC_2V_POS_H(temp0, temp1); + __lsx_vst(temp0, temp, 384); + __lsx_vst(temp1, temp, 1664); +} + +static void fdct8x32_1d_column_odd_store(int16_t *input, int16_t *temp_ptr) { + __m128i in16, in17, in18, in19, in20, in21, in22, in23; + __m128i in24, in25, in26, in27, in28, in29, in30, in31, vec4, vec5; + __m128i tmp0, tmp1; + + DUP4_ARG2(__lsx_vld, input, 64, input, 80, input, 160, input, 176, in20, in21, + in26, in27); + + DOTP_CONST_PAIR(in27, in20, cospi_16_64, cospi_16_64, in20, in27); + DOTP_CONST_PAIR(in26, in21, cospi_16_64, cospi_16_64, in21, in26); + + DUP4_ARG2(__lsx_vld, input, 32, input, 48, input, 192, input, 208, in18, in19, + in28, in29); + + vec4 = __lsx_vsub_h(in19, in20); + __lsx_vst(vec4, input, 64); + vec4 = __lsx_vsub_h(in18, in21); + __lsx_vst(vec4, input, 80); + vec4 = __lsx_vsub_h(in29, in26); + __lsx_vst(vec4, input, 160); + vec4 = __lsx_vsub_h(in28, in27); + __lsx_vst(vec4, input, 176); + + in21 = __lsx_vadd_h(in18, in21); + in20 = __lsx_vadd_h(in19, in20); + in27 = __lsx_vadd_h(in28, in27); + in26 = __lsx_vadd_h(in29, in26); + + DUP4_ARG2(__lsx_vld, input, 96, input, 112, input, 128, input, 144, in22, + in23, in24, in25); + DOTP_CONST_PAIR(in25, in22, cospi_16_64, cospi_16_64, in22, in25); + DOTP_CONST_PAIR(in24, in23, cospi_16_64, cospi_16_64, in23, in24); + + DUP4_ARG2(__lsx_vld, input, 0, input, 16, input, 224, input, 240, in16, in17, + in30, in31); + + vec4 = __lsx_vsub_h(in17, in22); + __lsx_vst(vec4, input, 32); + vec4 = __lsx_vsub_h(in16, in23); + __lsx_vst(vec4, input, 48); + vec4 = __lsx_vsub_h(in31, in24); + __lsx_vst(vec4, input, 192); + vec4 = __lsx_vsub_h(in30, in25); + __lsx_vst(vec4, input, 208); + + DUP4_ARG2(__lsx_vadd_h, in16, in23, in17, in22, in30, in25, in31, in24, in16, + in17, in30, in31); + DOTP_CONST_PAIR(in26, in21, cospi_24_64, cospi_8_64, in18, in29); + DOTP_CONST_PAIR(in27, in20, cospi_24_64, cospi_8_64, in19, in28); + DUP4_ARG2(__lsx_vadd_h, in16, in19, in17, in18, in30, in29, in31, in28, in27, + in22, in21, in25); + DOTP_CONST_PAIR(in21, in22, cospi_28_64, cospi_4_64, in26, in24); + DUP2_ARG2(__lsx_vadd_h, in27, in26, in25, in24, in23, in20); + DOTP_CONST_PAIR(in20, in23, cospi_31_64, cospi_1_64, vec4, vec5); + FDCT32_POSTPROC_2V_POS_H(vec5, vec4); + __lsx_vst(vec5, temp_ptr, 0); + __lsx_vst(vec4, temp_ptr, 1920); + + DUP2_ARG2(__lsx_vsub_h, in27, in26, in25, in24, in22, in21); + DOTP_CONST_PAIR(in21, in22, cospi_15_64, cospi_17_64, vec5, vec4); + FDCT32_POSTPROC_2V_POS_H(vec5, vec4); + __lsx_vst(vec5, temp_ptr, 896); + __lsx_vst(vec4, temp_ptr, 1024); + + DUP4_ARG2(__lsx_vsub_h, in17, in18, in16, in19, in31, in28, in30, in29, in23, + in26, in24, in20); + tmp0 = __lsx_vneg_h(in23); + DOTP_CONST_PAIR(tmp0, in20, cospi_28_64, cospi_4_64, in27, in25); + DUP2_ARG2(__lsx_vsub_h, in26, in27, in24, in25, in23, in20); + DOTP_CONST_PAIR(in20, in23, cospi_23_64, cospi_9_64, vec4, vec5); + FDCT32_POSTPROC_2V_POS_H(vec5, vec4); + __lsx_vst(vec4, temp_ptr, 1408); + __lsx_vst(vec5, temp_ptr, 512); + + DUP2_ARG2(__lsx_vadd_h, in26, in27, in24, in25, in22, in21); + DOTP_CONST_PAIR(in21, in22, cospi_7_64, cospi_25_64, vec4, vec5); + FDCT32_POSTPROC_2V_POS_H(vec5, vec4); + __lsx_vst(vec4, temp_ptr, 384); + __lsx_vst(vec5, temp_ptr, 1536); + + DUP4_ARG2(__lsx_vld, input, 32, input, 48, input, 64, input, 80, in22, in23, + in20, in21); + DUP4_ARG2(__lsx_vld, input, 160, input, 176, input, 192, input, 208, in26, + in27, in24, in25); + in16 = in20; + in17 = in21; + DUP2_ARG1(__lsx_vneg_h, in16, in17, tmp0, tmp1); + DOTP_CONST_PAIR(tmp0, in27, cospi_24_64, cospi_8_64, in20, in27); + DOTP_CONST_PAIR(tmp1, in26, cospi_24_64, cospi_8_64, in21, in26); + DUP4_ARG2(__lsx_vsub_h, in23, in20, in22, in21, in25, in26, in24, in27, in28, + in17, in18, in31); + DOTP_CONST_PAIR(in18, in17, cospi_12_64, cospi_20_64, in29, in30); + DUP2_ARG2(__lsx_vadd_h, in28, in29, in31, in30, in16, in19); + DOTP_CONST_PAIR(in19, in16, cospi_27_64, cospi_5_64, vec5, vec4); + FDCT32_POSTPROC_2V_POS_H(vec5, vec4); + __lsx_vst(vec5, temp_ptr, 1664); + __lsx_vst(vec4, temp_ptr, 256); + + DUP2_ARG2(__lsx_vsub_h, in28, in29, in31, in30, in17, in18); + DOTP_CONST_PAIR(in18, in17, cospi_11_64, cospi_21_64, vec5, vec4); + FDCT32_POSTPROC_2V_POS_H(vec5, vec4); + __lsx_vst(vec5, temp_ptr, 640); + __lsx_vst(vec4, temp_ptr, 1280); + + DUP4_ARG2(__lsx_vadd_h, in22, in21, in23, in20, in24, in27, in25, in26, in16, + in29, in30, in19); + tmp0 = __lsx_vneg_h(in16); + DOTP_CONST_PAIR(tmp0, in19, cospi_12_64, cospi_20_64, in28, in31); + DUP2_ARG2(__lsx_vsub_h, in29, in28, in30, in31, in16, in19); + DOTP_CONST_PAIR(in19, in16, cospi_19_64, cospi_13_64, vec5, vec4); + FDCT32_POSTPROC_2V_POS_H(vec5, vec4); + __lsx_vst(vec5, temp_ptr, 1152); + __lsx_vst(vec4, temp_ptr, 768); + + DUP2_ARG2(__lsx_vadd_h, in29, in28, in30, in31, in17, in18); + DOTP_CONST_PAIR(in18, in17, cospi_3_64, cospi_29_64, vec5, vec4); + FDCT32_POSTPROC_2V_POS_H(vec5, vec4); + __lsx_vst(vec5, temp_ptr, 128); + __lsx_vst(vec4, temp_ptr, 1792); +} + +static void fdct8x32_1d_column(const int16_t *input, int32_t src_stride, + int16_t *tmp_buf, int16_t *tmp_buf_big) { + fdct8x32_1d_column_load_butterfly(input, src_stride, tmp_buf); + fdct8x32_1d_column_even_store(tmp_buf, tmp_buf_big); + fdct8x32_1d_column_odd_store(tmp_buf + 128, (tmp_buf_big + 32)); +} + +static void fdct8x32_1d_row_load_butterfly(int16_t *temp_buff, + int16_t *output) { + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i in8, in9, in10, in11, in12, in13, in14, in15; + __m128i step0, step1, step2, step3, step4, step5, step6, step7; + + DUP4_ARG2(__lsx_vld, temp_buff, 0, temp_buff, 64, temp_buff, 128, temp_buff, + 192, in0, in1, in2, in3); + DUP4_ARG2(__lsx_vld, temp_buff, 256, temp_buff, 320, temp_buff, 384, + temp_buff, 448, in4, in5, in6, in7); + DUP4_ARG2(__lsx_vld, temp_buff, 48, temp_buff, 112, temp_buff, 176, temp_buff, + 240, in8, in9, in10, in11); + DUP4_ARG2(__lsx_vld, temp_buff, 304, temp_buff, 368, temp_buff, 432, + temp_buff, 496, in12, in13, in14, in15); + LSX_TRANSPOSE8x8_H(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + LSX_TRANSPOSE8x8_H(in8, in9, in10, in11, in12, in13, in14, in15, in8, in9, + in10, in11, in12, in13, in14, in15); + LSX_BUTTERFLY_16_H(in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, in10, + in11, in12, in13, in14, in15, step0, step1, step2, step3, + step4, step5, step6, step7, in8, in9, in10, in11, in12, + in13, in14, in15); + + __lsx_vst(step0, output, 0); + __lsx_vst(step1, output, 16); + __lsx_vst(step2, output, 32); + __lsx_vst(step3, output, 48); + __lsx_vst(step4, output, 64); + __lsx_vst(step5, output, 80); + __lsx_vst(step6, output, 96); + __lsx_vst(step7, output, 112); + + __lsx_vst(in8, output, 384); + __lsx_vst(in9, output, 400); + __lsx_vst(in10, output, 416); + __lsx_vst(in11, output, 432); + __lsx_vst(in12, output, 448); + __lsx_vst(in13, output, 464); + __lsx_vst(in14, output, 480); + __lsx_vst(in15, output, 496); + + /* 2nd set */ + DUP4_ARG2(__lsx_vld, temp_buff, 16, temp_buff, 80, temp_buff, 144, temp_buff, + 208, in0, in1, in2, in3); + DUP4_ARG2(__lsx_vld, temp_buff, 272, temp_buff, 336, temp_buff, 400, + temp_buff, 464, in4, in5, in6, in7); + DUP4_ARG2(__lsx_vld, temp_buff, 32, temp_buff, 96, temp_buff, 160, temp_buff, + 224, in8, in9, in10, in11); + DUP4_ARG2(__lsx_vld, temp_buff, 288, temp_buff, 352, temp_buff, 416, + temp_buff, 480, in12, in13, in14, in15); + LSX_TRANSPOSE8x8_H(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + LSX_TRANSPOSE8x8_H(in8, in9, in10, in11, in12, in13, in14, in15, in8, in9, + in10, in11, in12, in13, in14, in15); + LSX_BUTTERFLY_16_H(in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, in10, + in11, in12, in13, in14, in15, step0, step1, step2, step3, + step4, step5, step6, step7, in8, in9, in10, in11, in12, + in13, in14, in15); + + __lsx_vst(step0, output, 128); + __lsx_vst(step1, output, 144); + __lsx_vst(step2, output, 160); + __lsx_vst(step3, output, 176); + __lsx_vst(step4, output, 192); + __lsx_vst(step5, output, 208); + __lsx_vst(step6, output, 224); + __lsx_vst(step7, output, 240); + + __lsx_vst(in8, output, 256); + __lsx_vst(in9, output, 272); + __lsx_vst(in10, output, 288); + __lsx_vst(in11, output, 304); + __lsx_vst(in12, output, 320); + __lsx_vst(in13, output, 336); + __lsx_vst(in14, output, 352); + __lsx_vst(in15, output, 368); +} + +static void fdct8x32_1d_row_even_4x(int16_t *input, int16_t *interm_ptr, + int16_t *out) { + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i in8, in9, in10, in11, in12, in13, in14, in15; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i vec0_l, vec1_l, vec2_l, vec3_l, vec4_l, vec5_l, vec6_l, vec7_l; + __m128i vec0_r, vec1_r, vec2_r, vec3_r, vec4_r, vec5_r, vec6_r, vec7_r; + __m128i tmp0_w, tmp1_w, tmp2_w, tmp3_w; + + /* fdct32 even */ + /* stage 2 */ + DUP4_ARG2(__lsx_vld, input, 0, input, 16, input, 32, input, 48, in0, in1, in2, + in3); + DUP4_ARG2(__lsx_vld, input, 64, input, 80, input, 96, input, 112, in4, in5, + in6, in7); + DUP4_ARG2(__lsx_vld, input, 128, input, 144, input, 160, input, 176, in8, in9, + in10, in11); + DUP4_ARG2(__lsx_vld, input, 192, input, 208, input, 224, input, 240, in12, + in13, in14, in15); + + LSX_BUTTERFLY_16_H(in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, in10, + in11, in12, in13, in14, in15, vec0, vec1, vec2, vec3, vec4, + vec5, vec6, vec7, in8, in9, in10, in11, in12, in13, in14, + in15); + + __lsx_vst(vec0, interm_ptr, 0); + __lsx_vst(vec1, interm_ptr, 16); + __lsx_vst(vec2, interm_ptr, 32); + __lsx_vst(vec3, interm_ptr, 48); + __lsx_vst(vec4, interm_ptr, 64); + __lsx_vst(vec5, interm_ptr, 80); + __lsx_vst(vec6, interm_ptr, 96); + __lsx_vst(vec7, interm_ptr, 112); + + __lsx_vst(in8, interm_ptr, 128); + __lsx_vst(in9, interm_ptr, 144); + __lsx_vst(in10, interm_ptr, 160); + __lsx_vst(in11, interm_ptr, 176); + __lsx_vst(in12, interm_ptr, 192); + __lsx_vst(in13, interm_ptr, 208); + __lsx_vst(in14, interm_ptr, 224); + __lsx_vst(in15, interm_ptr, 240); + + /* Stage 3 */ + UNPCK_SH_SW(vec0, vec0_l, vec0_r); + UNPCK_SH_SW(vec1, vec1_l, vec1_r); + UNPCK_SH_SW(vec2, vec2_l, vec2_r); + UNPCK_SH_SW(vec3, vec3_l, vec3_r); + UNPCK_SH_SW(vec4, vec4_l, vec4_r); + UNPCK_SH_SW(vec5, vec5_l, vec5_r); + UNPCK_SH_SW(vec6, vec6_l, vec6_r); + UNPCK_SH_SW(vec7, vec7_l, vec7_r); + DUP4_ARG2(__lsx_vadd_w, vec0_r, vec7_r, vec1_r, vec6_r, vec2_r, vec5_r, + vec3_r, vec4_r, tmp0_w, tmp1_w, tmp2_w, tmp3_w); + LSX_BUTTERFLY_4_W(tmp0_w, tmp1_w, tmp2_w, tmp3_w, vec4_r, vec6_r, vec7_r, + vec5_r); + DUP4_ARG2(__lsx_vadd_w, vec0_l, vec7_l, vec1_l, vec6_l, vec2_l, vec5_l, + vec3_l, vec4_l, vec0_r, vec1_r, vec2_r, vec3_r); + + tmp3_w = __lsx_vadd_w(vec0_r, vec3_r); + vec0_r = __lsx_vsub_w(vec0_r, vec3_r); + vec3_r = __lsx_vadd_w(vec1_r, vec2_r); + vec1_r = __lsx_vsub_w(vec1_r, vec2_r); + + DOTP_CONST_PAIR_W(vec4_r, vec6_r, tmp3_w, vec3_r, cospi_16_64, cospi_16_64, + vec4_r, tmp3_w, vec6_r, vec3_r); + FDCT32_POSTPROC_NEG_W(vec4_r); + FDCT32_POSTPROC_NEG_W(tmp3_w); + FDCT32_POSTPROC_NEG_W(vec6_r); + FDCT32_POSTPROC_NEG_W(vec3_r); + DUP2_ARG2(__lsx_vpickev_h, vec4_r, tmp3_w, vec6_r, vec3_r, vec4, vec5); + __lsx_vst(vec5, out, 0); + __lsx_vst(vec4, out, 16); + + DOTP_CONST_PAIR_W(vec5_r, vec7_r, vec0_r, vec1_r, cospi_24_64, cospi_8_64, + vec4_r, tmp3_w, vec6_r, vec3_r); + FDCT32_POSTPROC_NEG_W(vec4_r); + FDCT32_POSTPROC_NEG_W(tmp3_w); + FDCT32_POSTPROC_NEG_W(vec6_r); + FDCT32_POSTPROC_NEG_W(vec3_r); + DUP2_ARG2(__lsx_vpickev_h, vec4_r, tmp3_w, vec6_r, vec3_r, vec4, vec5); + __lsx_vst(vec5, out, 32); + __lsx_vst(vec4, out, 48); + + DUP4_ARG2(__lsx_vld, interm_ptr, 0, interm_ptr, 16, interm_ptr, 32, + interm_ptr, 48, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vld, interm_ptr, 64, interm_ptr, 80, interm_ptr, 96, + interm_ptr, 112, vec4, vec5, vec6, vec7); + DUP4_ARG2(__lsx_vsub_h, vec3, vec4, vec2, vec5, vec1, vec6, vec0, vec7, vec4, + vec5, vec6, vec7); + DOTP_CONST_PAIR(vec6, vec5, cospi_16_64, cospi_16_64, vec5, vec6); + DUP2_ARG2(__lsx_vadd_h, vec4, vec5, vec7, vec6, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_28_64, cospi_4_64, in5, in4); + FDCT_POSTPROC_2V_NEG_H(in4, in5); + __lsx_vst(in4, out, 64); + __lsx_vst(in5, out, 112); + + DUP2_ARG2(__lsx_vsub_h, vec4, vec5, vec7, vec6, vec4, vec7); + DOTP_CONST_PAIR(vec7, vec4, cospi_12_64, cospi_20_64, in5, in4); + FDCT_POSTPROC_2V_NEG_H(in4, in5); + __lsx_vst(in4, out, 80); + __lsx_vst(in5, out, 96); + + DUP4_ARG2(__lsx_vld, interm_ptr, 128, interm_ptr, 144, interm_ptr, 160, + interm_ptr, 176, in8, in9, in10, in11); + DUP4_ARG2(__lsx_vld, interm_ptr, 192, interm_ptr, 208, interm_ptr, 224, + interm_ptr, 240, in12, in13, in14, in15); + DOTP_CONST_PAIR(in13, in10, cospi_16_64, cospi_16_64, vec2, vec5); + DOTP_CONST_PAIR(in12, in11, cospi_16_64, cospi_16_64, vec3, vec4); + DUP4_ARG2(__lsx_vadd_h, in8, vec3, in9, vec2, in14, vec5, in15, vec4, in0, + vec1, vec6, in2); + DOTP_CONST_PAIR(vec6, vec1, cospi_24_64, cospi_8_64, in1, in3); + DUP2_ARG2(__lsx_vadd_h, in0, in1, in2, in3, vec0, vec7); + DOTP_CONST_PAIR(vec7, vec0, cospi_30_64, cospi_2_64, in5, in4); + FDCT_POSTPROC_2V_NEG_H(in4, in5); + __lsx_vst(in4, out, 128); + __lsx_vst(in5, out, 240); + + DUP2_ARG2(__lsx_vsub_h, in0, in1, in2, in3, in0, in2); + DOTP_CONST_PAIR(in2, in0, cospi_14_64, cospi_18_64, in5, in4); + FDCT_POSTPROC_2V_NEG_H(in4, in5); + __lsx_vst(in4, out, 144); + __lsx_vst(in5, out, 224); + + DUP2_ARG2(__lsx_vsub_h, in9, vec2, in14, vec5, vec2, vec5); + tmp0_w = __lsx_vneg_h(vec2); + DOTP_CONST_PAIR(tmp0_w, vec5, cospi_24_64, cospi_8_64, in2, in1); + DUP4_ARG2(__lsx_vsub_h, in8, vec3, in15, vec4, in3, in2, in0, in1, in3, in0, + vec2, vec5); + DOTP_CONST_PAIR(vec5, vec2, cospi_22_64, cospi_10_64, in5, in4); + FDCT_POSTPROC_2V_NEG_H(in4, in5); + __lsx_vst(in4, out, 160); + __lsx_vst(in5, out, 208); + + DUP2_ARG2(__lsx_vadd_h, in3, in2, in0, in1, vec3, vec4); + DOTP_CONST_PAIR(vec4, vec3, cospi_6_64, cospi_26_64, in4, in5); + FDCT_POSTPROC_2V_NEG_H(in4, in5); + __lsx_vst(in4, out, 192); + __lsx_vst(in5, out, 176); +} + +static void fdct8x32_1d_row_even(int16_t *temp, int16_t *out) { + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i in8, in9, in10, in11, in12, in13, in14, in15; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, temp0, temp1; + + /* fdct32 even */ + /* stage 2 */ + DUP4_ARG2(__lsx_vld, temp, 0, temp, 16, temp, 32, temp, 48, in0, in1, in2, + in3); + DUP4_ARG2(__lsx_vld, temp, 64, temp, 80, temp, 96, temp, 112, in4, in5, in6, + in7); + DUP4_ARG2(__lsx_vld, temp, 128, temp, 144, temp, 160, temp, 176, in8, in9, + in10, in11); + DUP4_ARG2(__lsx_vld, temp, 192, temp, 208, temp, 224, temp, 240, in12, in13, + in14, in15); + + LSX_BUTTERFLY_16_H(in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, in10, + in11, in12, in13, in14, in15, vec0, vec1, vec2, vec3, vec4, + vec5, vec6, vec7, in8, in9, in10, in11, in12, in13, in14, + in15); + /* Stage 3 */ + DUP4_ARG2(__lsx_vadd_h, vec0, vec7, vec1, vec6, vec2, vec5, vec3, vec4, in0, + in1, in2, in3); + LSX_BUTTERFLY_4_H(in0, in1, in2, in3, temp0, in4, in1, in0); + DOTP_CONST_PAIR(temp0, in4, cospi_16_64, cospi_16_64, temp1, temp0); + FDCT_POSTPROC_2V_NEG_H(temp0, temp1); + __lsx_vst(temp0, out, 0); + __lsx_vst(temp1, out, 16); + + DOTP_CONST_PAIR(in0, in1, cospi_24_64, cospi_8_64, temp1, temp0); + FDCT_POSTPROC_2V_NEG_H(temp0, temp1); + __lsx_vst(temp0, out, 32); + __lsx_vst(temp1, out, 48); + + DUP4_ARG2(__lsx_vsub_h, vec3, vec4, vec2, vec5, vec1, vec6, vec0, vec7, vec4, + vec5, vec6, vec7); + DOTP_CONST_PAIR(vec6, vec5, cospi_16_64, cospi_16_64, vec5, vec6); + DUP2_ARG2(__lsx_vadd_h, vec4, vec5, vec7, vec6, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_28_64, cospi_4_64, temp1, temp0); + FDCT_POSTPROC_2V_NEG_H(temp0, temp1); + __lsx_vst(temp0, out, 64); + __lsx_vst(temp1, out, 112); + + DUP2_ARG2(__lsx_vsub_h, vec4, vec5, vec7, vec6, vec4, vec7); + DOTP_CONST_PAIR(vec7, vec4, cospi_12_64, cospi_20_64, temp1, temp0); + FDCT_POSTPROC_2V_NEG_H(temp0, temp1); + __lsx_vst(temp0, out, 80); + __lsx_vst(temp1, out, 96); + + DOTP_CONST_PAIR(in13, in10, cospi_16_64, cospi_16_64, vec2, vec5); + DOTP_CONST_PAIR(in12, in11, cospi_16_64, cospi_16_64, vec3, vec4); + DUP4_ARG2(__lsx_vadd_h, in8, vec3, in9, vec2, in14, vec5, in15, vec4, in0, + vec1, vec6, in2); + DOTP_CONST_PAIR(vec6, vec1, cospi_24_64, cospi_8_64, in1, in3); + DUP2_ARG2(__lsx_vadd_h, in0, in1, in2, in3, vec0, vec7); + DOTP_CONST_PAIR(vec7, vec0, cospi_30_64, cospi_2_64, temp1, temp0); + FDCT_POSTPROC_2V_NEG_H(temp0, temp1); + __lsx_vst(temp0, out, 128); + __lsx_vst(temp1, out, 240); + + DUP2_ARG2(__lsx_vsub_h, in0, in1, in2, in3, in0, in2); + DOTP_CONST_PAIR(in2, in0, cospi_14_64, cospi_18_64, temp1, temp0); + FDCT_POSTPROC_2V_NEG_H(temp0, temp1); + __lsx_vst(temp0, out, 144); + __lsx_vst(temp1, out, 224); + + DUP2_ARG2(__lsx_vsub_h, in9, vec2, in14, vec5, vec2, vec5); + temp0 = __lsx_vneg_h(vec2); + DOTP_CONST_PAIR(temp0, vec5, cospi_24_64, cospi_8_64, in2, in1); + DUP4_ARG2(__lsx_vsub_h, in8, vec3, in15, vec4, in3, in2, in0, in1, in3, in0, + vec2, vec5) + DOTP_CONST_PAIR(vec5, vec2, cospi_22_64, cospi_10_64, temp1, temp0); + FDCT_POSTPROC_2V_NEG_H(temp0, temp1); + __lsx_vst(temp0, out, 160); + __lsx_vst(temp1, out, 208); + + DUP2_ARG2(__lsx_vadd_h, in3, in2, in0, in1, vec3, vec4); + DOTP_CONST_PAIR(vec4, vec3, cospi_6_64, cospi_26_64, temp0, temp1); + FDCT_POSTPROC_2V_NEG_H(temp0, temp1); + __lsx_vst(temp0, out, 192); + __lsx_vst(temp1, out, 176); +} + +static void fdct8x32_1d_row_odd(int16_t *temp, int16_t *interm_ptr, + int16_t *out) { + __m128i in16, in17, in18, in19, in20, in21, in22, in23; + __m128i in24, in25, in26, in27, in28, in29, in30, in31, vec4, vec5; + __m128i tmp0, tmp1; + + in20 = __lsx_vld(temp, 64); + in21 = __lsx_vld(temp, 80); + in26 = __lsx_vld(temp, 160); + in27 = __lsx_vld(temp, 176); + + DOTP_CONST_PAIR(in27, in20, cospi_16_64, cospi_16_64, in20, in27); + DOTP_CONST_PAIR(in26, in21, cospi_16_64, cospi_16_64, in21, in26); + + in18 = __lsx_vld(temp, 32); + in19 = __lsx_vld(temp, 48); + in28 = __lsx_vld(temp, 192); + in29 = __lsx_vld(temp, 208); + + vec4 = __lsx_vsub_h(in19, in20); + __lsx_vst(vec4, interm_ptr, 64); + vec4 = __lsx_vsub_h(in18, in21); + __lsx_vst(vec4, interm_ptr, 176); + vec4 = __lsx_vsub_h(in28, in27); + __lsx_vst(vec4, interm_ptr, 112); + vec4 = __lsx_vsub_h(in29, in26); + __lsx_vst(vec4, interm_ptr, 128); + + DUP4_ARG2(__lsx_vadd_h, in18, in21, in19, in20, in28, in27, in29, in26, in21, + in20, in27, in26); + + in22 = __lsx_vld(temp, 96); + in23 = __lsx_vld(temp, 112); + in24 = __lsx_vld(temp, 128); + in25 = __lsx_vld(temp, 144); + + DOTP_CONST_PAIR(in25, in22, cospi_16_64, cospi_16_64, in22, in25); + DOTP_CONST_PAIR(in24, in23, cospi_16_64, cospi_16_64, in23, in24); + + in16 = __lsx_vld(temp, 0); + in17 = __lsx_vld(temp, 16); + in30 = __lsx_vld(temp, 224); + in31 = __lsx_vld(temp, 240); + + vec4 = __lsx_vsub_h(in17, in22); + __lsx_vst(vec4, interm_ptr, 80); + vec4 = __lsx_vsub_h(in30, in25); + __lsx_vst(vec4, interm_ptr, 96); + vec4 = __lsx_vsub_h(in31, in24); + __lsx_vst(vec4, interm_ptr, 144); + vec4 = __lsx_vsub_h(in16, in23); + __lsx_vst(vec4, interm_ptr, 160); + + DUP4_ARG2(__lsx_vadd_h, in16, in23, in17, in22, in30, in25, in31, in24, in16, + in17, in30, in31); + DOTP_CONST_PAIR(in26, in21, cospi_24_64, cospi_8_64, in18, in29); + DOTP_CONST_PAIR(in27, in20, cospi_24_64, cospi_8_64, in19, in28); + + DUP4_ARG2(__lsx_vadd_h, in16, in19, in17, in18, in30, in29, in31, in28, in27, + in22, in21, in25); + DOTP_CONST_PAIR(in21, in22, cospi_28_64, cospi_4_64, in26, in24); + DUP2_ARG2(__lsx_vadd_h, in27, in26, in25, in24, in23, in20); + + DOTP_CONST_PAIR(in20, in23, cospi_31_64, cospi_1_64, vec4, vec5); + FDCT_POSTPROC_2V_NEG_H(vec5, vec4); + __lsx_vst(vec5, out, 0); + __lsx_vst(vec4, out, 240); + + DUP2_ARG2(__lsx_vsub_h, in27, in26, in25, in24, in22, in21); + + DOTP_CONST_PAIR(in21, in22, cospi_15_64, cospi_17_64, vec5, vec4); + FDCT_POSTPROC_2V_NEG_H(vec5, vec4); + __lsx_vst(vec5, out, 224); + __lsx_vst(vec4, out, 16); + + DUP4_ARG2(__lsx_vsub_h, in17, in18, in16, in19, in31, in28, in30, in29, in23, + in26, in24, in20); + tmp0 = __lsx_vneg_h(in23); + DOTP_CONST_PAIR(tmp0, in20, cospi_28_64, cospi_4_64, in27, in25); + DUP2_ARG2(__lsx_vsub_h, in26, in27, in24, in25, in23, in20); + + DOTP_CONST_PAIR(in20, in23, cospi_23_64, cospi_9_64, vec4, vec5); + FDCT_POSTPROC_2V_NEG_H(vec5, vec4); + __lsx_vst(vec4, out, 32); + __lsx_vst(vec5, out, 208); + + DUP2_ARG2(__lsx_vadd_h, in26, in27, in24, in25, in22, in21); + DOTP_CONST_PAIR(in21, in22, cospi_7_64, cospi_25_64, vec4, vec5); + FDCT_POSTPROC_2V_NEG_H(vec5, vec4); + __lsx_vst(vec4, out, 48); + __lsx_vst(vec5, out, 192); + + in20 = __lsx_vld(interm_ptr, 64); + in21 = __lsx_vld(interm_ptr, 176); + in27 = __lsx_vld(interm_ptr, 112); + in26 = __lsx_vld(interm_ptr, 128); + + in16 = in20; + in17 = in21; + DUP2_ARG1(__lsx_vneg_h, in16, in17, tmp0, tmp1); + DOTP_CONST_PAIR(tmp0, in27, cospi_24_64, cospi_8_64, in20, in27); + DOTP_CONST_PAIR(tmp1, in26, cospi_24_64, cospi_8_64, in21, in26); + + in22 = __lsx_vld(interm_ptr, 80); + in25 = __lsx_vld(interm_ptr, 96); + in24 = __lsx_vld(interm_ptr, 144); + in23 = __lsx_vld(interm_ptr, 160); + + DUP4_ARG2(__lsx_vsub_h, in23, in20, in22, in21, in25, in26, in24, in27, in28, + in17, in18, in31); + DOTP_CONST_PAIR(in18, in17, cospi_12_64, cospi_20_64, in29, in30); + DUP2_ARG2(__lsx_vadd_h, in28, in29, in31, in30, in16, in19); + DOTP_CONST_PAIR(in19, in16, cospi_27_64, cospi_5_64, vec5, vec4); + FDCT_POSTPROC_2V_NEG_H(vec5, vec4); + __lsx_vst(vec5, out, 64); + __lsx_vst(vec4, out, 176); + + DUP2_ARG2(__lsx_vsub_h, in28, in29, in31, in30, in17, in18); + DOTP_CONST_PAIR(in18, in17, cospi_11_64, cospi_21_64, vec5, vec4); + FDCT_POSTPROC_2V_NEG_H(vec5, vec4); + __lsx_vst(vec5, out, 80); + __lsx_vst(vec4, out, 160); + + DUP4_ARG2(__lsx_vadd_h, in22, in21, in23, in20, in24, in27, in25, in26, in16, + in29, in30, in19); + tmp0 = __lsx_vneg_h(in16); + DOTP_CONST_PAIR(tmp0, in19, cospi_12_64, cospi_20_64, in28, in31); + DUP2_ARG2(__lsx_vsub_h, in29, in28, in30, in31, in16, in19); + + DOTP_CONST_PAIR(in19, in16, cospi_19_64, cospi_13_64, vec5, vec4); + FDCT_POSTPROC_2V_NEG_H(vec5, vec4); + __lsx_vst(vec5, out, 144); + __lsx_vst(vec4, out, 96); + + DUP2_ARG2(__lsx_vadd_h, in29, in28, in30, in31, in17, in18); + + DOTP_CONST_PAIR(in18, in17, cospi_3_64, cospi_29_64, vec5, vec4); + FDCT_POSTPROC_2V_NEG_H(vec5, vec4); + __lsx_vst(vec4, out, 112); + __lsx_vst(vec5, out, 128); +} + +static void fdct8x32_1d_row_transpose_store(int16_t *temp, int16_t *output) { + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i in0_1, in1_1, in2_1, in3_1, in4_1, in5_1, in6_1, in7_1; + + /* 1st set */ + in0 = __lsx_vld(temp, 0); + in4 = __lsx_vld(temp, 64); + in2 = __lsx_vld(temp, 128); + in6 = __lsx_vld(temp, 192); + in1 = __lsx_vld(temp, 256); + in7 = __lsx_vld(temp, 304); + in3 = __lsx_vld(temp, 384); + in5 = __lsx_vld(temp, 432); + + LSX_TRANSPOSE8x8_H(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + + /* 2nd set */ + in0_1 = __lsx_vld(temp, 32); + in1_1 = __lsx_vld(temp, 464); + in2_1 = __lsx_vld(temp, 160); + in3_1 = __lsx_vld(temp, 336); + in4_1 = __lsx_vld(temp, 96); + in5_1 = __lsx_vld(temp, 352); + in6_1 = __lsx_vld(temp, 224); + in7_1 = __lsx_vld(temp, 480); + + __lsx_vst(in0, output, 0); + __lsx_vst(in1, output, 64); + __lsx_vst(in2, output, 128); + __lsx_vst(in3, output, 192); + __lsx_vst(in4, output, 256); + __lsx_vst(in5, output, 320); + __lsx_vst(in6, output, 384); + __lsx_vst(in7, output, 448); + + LSX_TRANSPOSE8x8_H(in0_1, in1_1, in2_1, in3_1, in4_1, in5_1, in6_1, in7_1, + in0_1, in1_1, in2_1, in3_1, in4_1, in5_1, in6_1, in7_1); + + /* 3rd set */ + in0 = __lsx_vld(temp, 16); + in1 = __lsx_vld(temp, 272); + in2 = __lsx_vld(temp, 144); + in3 = __lsx_vld(temp, 400); + in4 = __lsx_vld(temp, 80); + in5 = __lsx_vld(temp, 416); + in6 = __lsx_vld(temp, 208); + in7 = __lsx_vld(temp, 288); + + __lsx_vst(in0_1, output, 16); + __lsx_vst(in1_1, output, 80); + __lsx_vst(in2_1, output, 144); + __lsx_vst(in3_1, output, 208); + __lsx_vst(in4_1, output, 272); + __lsx_vst(in5_1, output, 336); + __lsx_vst(in6_1, output, 400); + __lsx_vst(in7_1, output, 464); + + LSX_TRANSPOSE8x8_H(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + + __lsx_vst(in0, output, 32); + __lsx_vst(in1, output, 96); + __lsx_vst(in2, output, 160); + __lsx_vst(in3, output, 224); + __lsx_vst(in4, output, 288); + __lsx_vst(in5, output, 352); + __lsx_vst(in6, output, 416); + __lsx_vst(in7, output, 480); + + /* 4th set */ + in0_1 = __lsx_vld(temp, 48); + in1_1 = __lsx_vld(temp, 448); + in2_1 = __lsx_vld(temp, 176); + in3_1 = __lsx_vld(temp, 320); + in4_1 = __lsx_vld(temp, 112); + in5_1 = __lsx_vld(temp, 368); + in6_1 = __lsx_vld(temp, 240); + in7_1 = __lsx_vld(temp, 496); + + LSX_TRANSPOSE8x8_H(in0_1, in1_1, in2_1, in3_1, in4_1, in5_1, in6_1, in7_1, + in0_1, in1_1, in2_1, in3_1, in4_1, in5_1, in6_1, in7_1); + + __lsx_vst(in0_1, output, 48); + __lsx_vst(in1_1, output, 112); + __lsx_vst(in2_1, output, 176); + __lsx_vst(in3_1, output, 240); + __lsx_vst(in4_1, output, 304); + __lsx_vst(in5_1, output, 368); + __lsx_vst(in6_1, output, 432); + __lsx_vst(in7_1, output, 496); +} + +static void fdct32x8_1d_row(int16_t *temp, int16_t *temp_buf, int16_t *output) { + fdct8x32_1d_row_load_butterfly(temp, temp_buf); + fdct8x32_1d_row_even(temp_buf, temp_buf); + fdct8x32_1d_row_odd(temp_buf + 128, temp, temp_buf + 128); + fdct8x32_1d_row_transpose_store(temp_buf, output); +} + +static void fdct32x8_1d_row_4x(int16_t *tmp_buf_big, int16_t *tmp_buf, + int16_t *output) { + fdct8x32_1d_row_load_butterfly(tmp_buf_big, tmp_buf); + fdct8x32_1d_row_even_4x(tmp_buf, tmp_buf_big, tmp_buf); + fdct8x32_1d_row_odd(tmp_buf + 128, tmp_buf_big, tmp_buf + 128); + fdct8x32_1d_row_transpose_store(tmp_buf, output); +} + +void vpx_fdct32x32_lsx(const int16_t *input, int16_t *output, + int32_t src_stride) { + int i; + DECLARE_ALIGNED(32, int16_t, tmp_buf_big[1024]); + DECLARE_ALIGNED(32, int16_t, tmp_buf[256]); + + /* column transform */ + for (i = 0; i < 4; ++i) { + fdct8x32_1d_column(input + (8 * i), src_stride, tmp_buf, + tmp_buf_big + (8 * i)); + } + + /* row transform */ + fdct32x8_1d_row_4x(tmp_buf_big, tmp_buf, output); + + /* row transform */ + for (i = 1; i < 4; ++i) { + fdct32x8_1d_row(tmp_buf_big + (i * 256), tmp_buf, output + (i * 256)); + } +} + +static void fdct8x32_1d_row_even_rd(int16_t *temp, int16_t *out) { + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i in8, in9, in10, in11, in12, in13, in14, in15; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, temp0, temp1; + + /* fdct32 even */ + /* stage 2 */ + DUP4_ARG2(__lsx_vld, temp, 0, temp, 16, temp, 32, temp, 48, in0, in1, in2, + in3); + DUP4_ARG2(__lsx_vld, temp, 64, temp, 80, temp, 96, temp, 112, in4, in5, in6, + in7); + DUP4_ARG2(__lsx_vld, temp, 128, temp, 144, temp, 160, temp, 176, in8, in9, + in10, in11); + DUP4_ARG2(__lsx_vld, temp, 192, temp, 208, temp, 224, temp, 240, in12, in13, + in14, in15); + LSX_BUTTERFLY_16_H(in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, in10, + in11, in12, in13, in14, in15, vec0, vec1, vec2, vec3, vec4, + vec5, vec6, vec7, in8, in9, in10, in11, in12, in13, in14, + in15); + + FDCT_POSTPROC_2V_NEG_H(vec0, vec1); + FDCT_POSTPROC_2V_NEG_H(vec2, vec3); + FDCT_POSTPROC_2V_NEG_H(vec4, vec5); + FDCT_POSTPROC_2V_NEG_H(vec6, vec7); + FDCT_POSTPROC_2V_NEG_H(in8, in9); + FDCT_POSTPROC_2V_NEG_H(in10, in11); + FDCT_POSTPROC_2V_NEG_H(in12, in13); + FDCT_POSTPROC_2V_NEG_H(in14, in15); + + /* Stage 3 */ + DUP4_ARG2(__lsx_vadd_h, vec0, vec7, vec1, vec6, vec2, vec5, vec3, vec4, in0, + in1, in2, in3); + + temp0 = __lsx_vadd_h(in0, in3); + in0 = __lsx_vsub_h(in0, in3); + in3 = __lsx_vadd_h(in1, in2); + in1 = __lsx_vsub_h(in1, in2); + + DOTP_CONST_PAIR(temp0, in3, cospi_16_64, cospi_16_64, temp1, temp0); + __lsx_vst(temp0, out, 0); + __lsx_vst(temp1, out, 16); + + DOTP_CONST_PAIR(in0, in1, cospi_24_64, cospi_8_64, temp1, temp0); + __lsx_vst(temp0, out, 32); + __lsx_vst(temp1, out, 48); + + DUP4_ARG2(__lsx_vsub_h, vec3, vec4, vec2, vec5, vec1, vec6, vec0, vec7, vec4, + vec5, vec6, vec7); + DOTP_CONST_PAIR(vec6, vec5, cospi_16_64, cospi_16_64, vec5, vec6); + DUP2_ARG2(__lsx_vadd_h, vec4, vec5, vec7, vec6, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_28_64, cospi_4_64, temp1, temp0); + __lsx_vst(temp0, out, 64); + __lsx_vst(temp1, out, 112); + + DUP2_ARG2(__lsx_vsub_h, vec4, vec5, vec7, vec6, vec4, vec7); + DOTP_CONST_PAIR(vec7, vec4, cospi_12_64, cospi_20_64, temp1, temp0); + __lsx_vst(temp0, out, 80); + __lsx_vst(temp1, out, 96); + + DOTP_CONST_PAIR(in13, in10, cospi_16_64, cospi_16_64, vec2, vec5); + DOTP_CONST_PAIR(in12, in11, cospi_16_64, cospi_16_64, vec3, vec4); + DUP4_ARG2(__lsx_vadd_h, in8, vec3, in9, vec2, in14, vec5, in15, vec4, in0, + vec1, vec6, in2); + DOTP_CONST_PAIR(vec6, vec1, cospi_24_64, cospi_8_64, in1, in3); + DUP2_ARG2(__lsx_vadd_h, in0, in1, in2, in3, vec0, vec7); + DOTP_CONST_PAIR(vec7, vec0, cospi_30_64, cospi_2_64, temp1, temp0); + __lsx_vst(temp0, out, 128); + __lsx_vst(temp1, out, 240); + + DUP2_ARG2(__lsx_vsub_h, in0, in1, in2, in3, in0, in2); + DOTP_CONST_PAIR(in2, in0, cospi_14_64, cospi_18_64, temp1, temp0); + __lsx_vst(temp0, out, 144); + __lsx_vst(temp1, out, 224); + + DUP2_ARG2(__lsx_vsub_h, in9, vec2, in14, vec5, vec2, vec5); + temp0 = __lsx_vneg_h(vec2); + DOTP_CONST_PAIR(temp0, vec5, cospi_24_64, cospi_8_64, in2, in1); + DUP4_ARG2(__lsx_vsub_h, in8, vec3, in15, vec4, in3, in2, in0, in1, in3, in0, + vec2, vec5); + DOTP_CONST_PAIR(vec5, vec2, cospi_22_64, cospi_10_64, temp1, temp0); + __lsx_vst(temp0, out, 160); + __lsx_vst(temp1, out, 208); + + DUP2_ARG2(__lsx_vadd_h, in3, in2, in0, in1, vec3, vec4); + DOTP_CONST_PAIR(vec4, vec3, cospi_6_64, cospi_26_64, temp0, temp1); + __lsx_vst(temp0, out, 192); + __lsx_vst(temp1, out, 176); +} + +static void fdct8x32_1d_row_odd_rd(int16_t *temp, int16_t *interm_ptr, + int16_t *out) { + __m128i in16, in17, in18, in19, in20, in21, in22, in23; + __m128i in24, in25, in26, in27, in28, in29, in30, in31; + __m128i vec4, vec5, tmp0, tmp1; + + in20 = __lsx_vld(temp, 64); + in21 = __lsx_vld(temp, 80); + in26 = __lsx_vld(temp, 160); + in27 = __lsx_vld(temp, 176); + + DOTP_CONST_PAIR(in27, in20, cospi_16_64, cospi_16_64, in20, in27); + DOTP_CONST_PAIR(in26, in21, cospi_16_64, cospi_16_64, in21, in26); + + FDCT_POSTPROC_2V_NEG_H(in20, in21); + FDCT_POSTPROC_2V_NEG_H(in26, in27); + + in18 = __lsx_vld(temp, 32); + in19 = __lsx_vld(temp, 48); + in28 = __lsx_vld(temp, 192); + in29 = __lsx_vld(temp, 208); + + FDCT_POSTPROC_2V_NEG_H(in18, in19); + FDCT_POSTPROC_2V_NEG_H(in28, in29); + + vec4 = __lsx_vsub_h(in19, in20); + __lsx_vst(vec4, interm_ptr, 64); + vec4 = __lsx_vsub_h(in18, in21); + __lsx_vst(vec4, interm_ptr, 176); + vec4 = __lsx_vsub_h(in29, in26); + __lsx_vst(vec4, interm_ptr, 128); + vec4 = __lsx_vsub_h(in28, in27); + __lsx_vst(vec4, interm_ptr, 112); + + DUP4_ARG2(__lsx_vadd_h, in18, in21, in19, in20, in28, in27, in29, in26, in21, + in20, in27, in26); + + in22 = __lsx_vld(temp, 96); + in23 = __lsx_vld(temp, 112); + in24 = __lsx_vld(temp, 128); + in25 = __lsx_vld(temp, 144); + + DOTP_CONST_PAIR(in25, in22, cospi_16_64, cospi_16_64, in22, in25); + DOTP_CONST_PAIR(in24, in23, cospi_16_64, cospi_16_64, in23, in24); + FDCT_POSTPROC_2V_NEG_H(in22, in23); + FDCT_POSTPROC_2V_NEG_H(in24, in25); + + in16 = __lsx_vld(temp, 0); + in17 = __lsx_vld(temp, 16); + in30 = __lsx_vld(temp, 224); + in31 = __lsx_vld(temp, 240); + + FDCT_POSTPROC_2V_NEG_H(in16, in17); + FDCT_POSTPROC_2V_NEG_H(in30, in31); + + vec4 = __lsx_vsub_h(in17, in22); + __lsx_vst(vec4, interm_ptr, 80); + vec4 = __lsx_vsub_h(in30, in25); + __lsx_vst(vec4, interm_ptr, 96); + vec4 = __lsx_vsub_h(in31, in24); + __lsx_vst(vec4, interm_ptr, 144); + vec4 = __lsx_vsub_h(in16, in23); + __lsx_vst(vec4, interm_ptr, 160); + + DUP4_ARG2(__lsx_vadd_h, in16, in23, in17, in22, in30, in25, in31, in24, in16, + in17, in30, in31); + DOTP_CONST_PAIR(in26, in21, cospi_24_64, cospi_8_64, in18, in29); + DOTP_CONST_PAIR(in27, in20, cospi_24_64, cospi_8_64, in19, in28); + DUP4_ARG2(__lsx_vadd_h, in16, in19, in17, in18, in30, in29, in31, in28, in27, + in22, in21, in25); + DOTP_CONST_PAIR(in21, in22, cospi_28_64, cospi_4_64, in26, in24); + DUP2_ARG2(__lsx_vadd_h, in27, in26, in25, in24, in23, in20); + DOTP_CONST_PAIR(in20, in23, cospi_31_64, cospi_1_64, vec4, vec5); + __lsx_vst(vec5, out, 0); + __lsx_vst(vec4, out, 240); + + DUP2_ARG2(__lsx_vsub_h, in27, in26, in25, in24, in22, in21); + DOTP_CONST_PAIR(in21, in22, cospi_15_64, cospi_17_64, vec5, vec4); + __lsx_vst(vec5, out, 224); + __lsx_vst(vec4, out, 16); + + DUP4_ARG2(__lsx_vsub_h, in17, in18, in16, in19, in31, in28, in30, in29, in23, + in26, in24, in20); + tmp0 = __lsx_vneg_h(in23); + DOTP_CONST_PAIR(tmp0, in20, cospi_28_64, cospi_4_64, in27, in25); + DUP2_ARG2(__lsx_vsub_h, in26, in27, in24, in25, in23, in20); + DOTP_CONST_PAIR(in20, in23, cospi_23_64, cospi_9_64, vec4, vec5); + __lsx_vst(vec4, out, 32); + __lsx_vst(vec5, out, 208); + + DUP2_ARG2(__lsx_vadd_h, in26, in27, in24, in25, in22, in21); + DOTP_CONST_PAIR(in21, in22, cospi_7_64, cospi_25_64, vec4, vec5); + __lsx_vst(vec4, out, 48); + __lsx_vst(vec5, out, 192); + + in20 = __lsx_vld(interm_ptr, 64); + in21 = __lsx_vld(interm_ptr, 176); + in27 = __lsx_vld(interm_ptr, 112); + in26 = __lsx_vld(interm_ptr, 128); + + in16 = in20; + in17 = in21; + DUP2_ARG1(__lsx_vneg_h, in16, in17, tmp0, tmp1); + DOTP_CONST_PAIR(tmp0, in27, cospi_24_64, cospi_8_64, in20, in27); + DOTP_CONST_PAIR(tmp1, in26, cospi_24_64, cospi_8_64, in21, in26); + + in22 = __lsx_vld(interm_ptr, 80); + in25 = __lsx_vld(interm_ptr, 96); + in24 = __lsx_vld(interm_ptr, 144); + in23 = __lsx_vld(interm_ptr, 160); + + DUP4_ARG2(__lsx_vsub_h, in23, in20, in22, in21, in25, in26, in24, in27, in28, + in17, in18, in31); + DOTP_CONST_PAIR(in18, in17, cospi_12_64, cospi_20_64, in29, in30); + in16 = __lsx_vadd_h(in28, in29); + in19 = __lsx_vadd_h(in31, in30); + DOTP_CONST_PAIR(in19, in16, cospi_27_64, cospi_5_64, vec5, vec4); + __lsx_vst(vec5, out, 64); + __lsx_vst(vec4, out, 176); + + DUP2_ARG2(__lsx_vsub_h, in28, in29, in31, in30, in17, in18); + DOTP_CONST_PAIR(in18, in17, cospi_11_64, cospi_21_64, vec5, vec4); + __lsx_vst(vec5, out, 80); + __lsx_vst(vec4, out, 160); + + DUP4_ARG2(__lsx_vadd_h, in22, in21, in23, in20, in24, in27, in25, in26, in16, + in29, in30, in19); + tmp0 = __lsx_vneg_h(in16); + DOTP_CONST_PAIR(tmp0, in19, cospi_12_64, cospi_20_64, in28, in31); + DUP2_ARG2(__lsx_vsub_h, in29, in28, in30, in31, in16, in19); + DOTP_CONST_PAIR(in19, in16, cospi_19_64, cospi_13_64, vec5, vec4); + __lsx_vst(vec5, out, 144); + __lsx_vst(vec4, out, 96); + + DUP2_ARG2(__lsx_vadd_h, in29, in28, in30, in31, in17, in18); + DOTP_CONST_PAIR(in18, in17, cospi_3_64, cospi_29_64, vec5, vec4); + __lsx_vst(vec4, out, 112); + __lsx_vst(vec5, out, 128); +} + +static void fdct32x8_1d_row_rd(int16_t *tmp_buf_big, int16_t *tmp_buf, + int16_t *output) { + fdct8x32_1d_row_load_butterfly(tmp_buf_big, tmp_buf); + fdct8x32_1d_row_even_rd(tmp_buf, tmp_buf); + fdct8x32_1d_row_odd_rd((tmp_buf + 128), tmp_buf_big, (tmp_buf + 128)); + fdct8x32_1d_row_transpose_store(tmp_buf, output); +} + +void vpx_fdct32x32_rd_lsx(const int16_t *input, int16_t *out, + int32_t src_stride) { + int32_t i; + DECLARE_ALIGNED(32, int16_t, tmp_buf_big[1024]); + DECLARE_ALIGNED(32, int16_t, tmp_buf[256]); + + /* column transform */ + for (i = 0; i < 4; ++i) { + fdct8x32_1d_column(input + (8 * i), src_stride, &tmp_buf[0], + &tmp_buf_big[0] + (8 * i)); + } + /* row transform */ + for (i = 0; i < 4; ++i) { + fdct32x8_1d_row_rd(&tmp_buf_big[0] + (8 * i * 32), &tmp_buf[0], + out + (8 * i * 32)); + } +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/fwd_txfm_lsx.h b/media/libvpx/libvpx/vpx_dsp/loongarch/fwd_txfm_lsx.h new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/fwd_txfm_lsx.h @@ -0,0 +1,282 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VPX_VPX_DSP_LOONGARCH_FWD_TXFM_LSX_H_ +#define VPX_VPX_DSP_LOONGARCH_FWD_TXFM_LSX_H_ + +#include "vpx_dsp/loongarch/txfm_macros_lsx.h" +#include "vpx_dsp/txfm_common.h" + +#define FDCT32_POSTPROC_2V_POS_H(vec0, vec1) \ + { \ + __m128i tp0_m, tp1_m; \ + __m128i one = __lsx_vreplgr2vr_h(1); \ + \ + tp0_m = __lsx_vslei_h(vec0, 0); \ + tp1_m = __lsx_vslei_h(vec1, 0); \ + tp0_m = __lsx_vxori_b(tp0_m, 255); \ + tp1_m = __lsx_vxori_b(tp1_m, 255); \ + vec0 = __lsx_vadd_h(vec0, one); \ + vec1 = __lsx_vadd_h(vec1, one); \ + tp0_m = __lsx_vand_v(one, tp0_m); \ + tp1_m = __lsx_vand_v(one, tp1_m); \ + vec0 = __lsx_vadd_h(vec0, tp0_m); \ + vec1 = __lsx_vadd_h(vec1, tp1_m); \ + vec0 = __lsx_vsrai_h(vec0, 2); \ + vec1 = __lsx_vsrai_h(vec1, 2); \ + } + +#define FDCT_POSTPROC_2V_NEG_H(vec0, vec1) \ + { \ + __m128i tp0_m, tp1_m; \ + __m128i one_m = __lsx_vldi(0x401); \ + \ + tp0_m = __lsx_vslti_h(vec0, 0); \ + tp1_m = __lsx_vslti_h(vec1, 0); \ + vec0 = __lsx_vadd_h(vec0, one_m); \ + vec1 = __lsx_vadd_h(vec1, one_m); \ + tp0_m = __lsx_vand_v(one_m, tp0_m); \ + tp1_m = __lsx_vand_v(one_m, tp1_m); \ + vec0 = __lsx_vadd_h(vec0, tp0_m); \ + vec1 = __lsx_vadd_h(vec1, tp1_m); \ + vec0 = __lsx_vsrai_h(vec0, 2); \ + vec1 = __lsx_vsrai_h(vec1, 2); \ + } + +#define FDCT32_POSTPROC_NEG_W(vec) \ + { \ + __m128i temp_m; \ + __m128i one_m = __lsx_vreplgr2vr_w(1); \ + \ + temp_m = __lsx_vslti_w(vec, 0); \ + vec = __lsx_vadd_w(vec, one_m); \ + temp_m = __lsx_vand_v(one_m, temp_m); \ + vec = __lsx_vadd_w(vec, temp_m); \ + vec = __lsx_vsrai_w(vec, 2); \ + } + +#define DOTP_CONST_PAIR_W(reg0_left, reg1_left, reg0_right, reg1_right, \ + const0, const1, out0, out1, out2, out3) \ + { \ + __m128i s0_m, s1_m, s2_m, s3_m, s4_m, s5_m, s6_m, s7_m; \ + __m128i tp0_m, tp1_m, tp2_m, tp3_m, _tmp0, _tmp1; \ + __m128i k0_m = __lsx_vreplgr2vr_w((int32_t)const0); \ + \ + s0_m = __lsx_vreplgr2vr_w((int32_t)const1); \ + k0_m = __lsx_vpackev_w(s0_m, k0_m); \ + \ + DUP2_ARG1(__lsx_vneg_w, reg1_left, reg1_right, _tmp0, _tmp1); \ + s1_m = __lsx_vilvl_w(_tmp0, reg0_left); \ + s0_m = __lsx_vilvh_w(_tmp0, reg0_left); \ + s3_m = __lsx_vilvl_w(reg0_left, reg1_left); \ + s2_m = __lsx_vilvh_w(reg0_left, reg1_left); \ + s5_m = __lsx_vilvl_w(_tmp1, reg0_right); \ + s4_m = __lsx_vilvh_w(_tmp1, reg0_right); \ + s7_m = __lsx_vilvl_w(reg0_right, reg1_right); \ + s6_m = __lsx_vilvh_w(reg0_right, reg1_right); \ + DUP2_ARG2(__lsx_vdp2_d_w, s0_m, k0_m, s1_m, k0_m, tp0_m, tp1_m); \ + DUP2_ARG2(__lsx_vdp2_d_w, s4_m, k0_m, s5_m, k0_m, tp2_m, tp3_m); \ + DUP2_ARG3(__lsx_vssrarni_w_d, tp0_m, tp1_m, DCT_CONST_BITS, tp2_m, tp3_m, \ + DCT_CONST_BITS, out0, out1); \ + DUP2_ARG2(__lsx_vdp2_d_w, s2_m, k0_m, s3_m, k0_m, tp0_m, tp1_m); \ + DUP2_ARG2(__lsx_vdp2_d_w, s6_m, k0_m, s7_m, k0_m, tp2_m, tp3_m); \ + DUP2_ARG3(__lsx_vssrarni_w_d, tp0_m, tp1_m, DCT_CONST_BITS, tp2_m, tp3_m, \ + DCT_CONST_BITS, out2, out3); \ + } + +#define VP9_ADDBLK_ST8x4_UB(dst, _stride, _stride2, _stride3, in0, in1, in2, \ + in3) \ + { \ + __m128i dst0_m, dst1_m, dst2_m, dst3_m; \ + __m128i tmp0_m, tmp1_m; \ + __m128i res0_m, res1_m, res2_m, res3_m; \ + \ + dst0_m = __lsx_vld(dst, 0); \ + DUP2_ARG2(__lsx_vldx, dst, _stride, dst, _stride2, dst1_m, dst2_m); \ + dst3_m = __lsx_vldx(dst, _stride3); \ + DUP4_ARG2(__lsx_vsllwil_hu_bu, dst0_m, 0, dst1_m, 0, dst2_m, 0, dst3_m, 0, \ + res0_m, res1_m, res2_m, res3_m); \ + DUP4_ARG2(__lsx_vadd_h, res0_m, in0, res1_m, in1, res2_m, in2, res3_m, \ + in3, res0_m, res1_m, res2_m, res3_m); \ + DUP2_ARG3(__lsx_vssrarni_bu_h, res1_m, res0_m, 0, res3_m, res2_m, 0, \ + tmp0_m, tmp1_m); \ + __lsx_vstelm_d(tmp0_m, dst, 0, 0); \ + __lsx_vstelm_d(tmp0_m, dst + _stride, 0, 1); \ + __lsx_vstelm_d(tmp1_m, dst + _stride2, 0, 0); \ + __lsx_vstelm_d(tmp1_m, dst + _stride3, 0, 1); \ + } + +#define FDCT8x16_EVEN(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, \ + out2, out3, out4, out5, out6, out7) \ + { \ + __m128i s0_m, s1_m, s2_m, s3_m, s4_m, s5_m, s6_m, s7_m; \ + __m128i x0_m, x1_m, x2_m, x3_m; \ + __m128i coeff_m = { 0x187e3b21d2bf2d41, 0x238e35370c7c3ec5 }; \ + \ + /* FDCT stage1 */ \ + LSX_BUTTERFLY_8_H(in0, in1, in2, in3, in4, in5, in6, in7, s0_m, s1_m, \ + s2_m, s3_m, s4_m, s5_m, s6_m, s7_m); \ + LSX_BUTTERFLY_4_H(s0_m, s1_m, s2_m, s3_m, x0_m, x1_m, x2_m, x3_m); \ + DUP2_ARG2(__lsx_vilvh_h, x1_m, x0_m, x3_m, x2_m, s0_m, s2_m); \ + DUP2_ARG2(__lsx_vilvl_h, x1_m, x0_m, x3_m, x2_m, s1_m, s3_m); \ + DUP2_ARG2(__lsx_vreplvei_h, coeff_m, 0, coeff_m, 1, x0_m, x1_m); \ + x1_m = __lsx_vpackev_h(x1_m, x0_m); \ + DOT_SHIFT_RIGHT_PCK_H(s0_m, s1_m, x1_m, out4); \ + \ + DUP2_ARG2(__lsx_vreplvei_h, coeff_m, 2, coeff_m, 3, x2_m, x3_m); \ + x2_m = __lsx_vneg_h(x2_m); \ + x2_m = __lsx_vpackev_h(x3_m, x2_m); \ + DOT_SHIFT_RIGHT_PCK_H(s2_m, s3_m, x2_m, out6); \ + \ + DOT_SHIFT_RIGHT_PCK_H(s0_m, s1_m, x0_m, out0); \ + x2_m = __lsx_vreplvei_h(coeff_m, 2); \ + x2_m = __lsx_vpackev_h(x2_m, x3_m); \ + DOT_SHIFT_RIGHT_PCK_H(s2_m, s3_m, x2_m, out2); \ + \ + /* stage2 */ \ + s1_m = __lsx_vilvl_h(s5_m, s6_m); \ + s0_m = __lsx_vilvh_h(s5_m, s6_m); \ + \ + DOT_SHIFT_RIGHT_PCK_H(s0_m, s1_m, x0_m, s6_m); \ + DOT_SHIFT_RIGHT_PCK_H(s0_m, s1_m, x1_m, s5_m); \ + \ + /* stage3 */ \ + LSX_BUTTERFLY_4_H(s4_m, s7_m, s6_m, s5_m, x0_m, x3_m, x2_m, x1_m); \ + \ + /* stage4 */ \ + DUP2_ARG2(__lsx_vilvh_h, x3_m, x0_m, x2_m, x1_m, s4_m, s6_m); \ + DUP2_ARG2(__lsx_vilvl_h, x3_m, x0_m, x2_m, x1_m, s5_m, s7_m); \ + \ + DUP2_ARG2(__lsx_vreplvei_h, coeff_m, 4, coeff_m, 5, x0_m, x1_m); \ + x1_m = __lsx_vpackev_h(x0_m, x1_m); \ + DOT_SHIFT_RIGHT_PCK_H(s4_m, s5_m, x1_m, out1); \ + \ + DUP2_ARG2(__lsx_vreplvei_h, coeff_m, 6, coeff_m, 7, x2_m, x3_m); \ + x2_m = __lsx_vpackev_h(x3_m, x2_m); \ + DOT_SHIFT_RIGHT_PCK_H(s6_m, s7_m, x2_m, out5); \ + \ + x1_m = __lsx_vreplvei_h(coeff_m, 5); \ + x0_m = __lsx_vneg_h(x0_m); \ + x0_m = __lsx_vpackev_h(x1_m, x0_m); \ + DOT_SHIFT_RIGHT_PCK_H(s4_m, s5_m, x0_m, out7); \ + \ + x2_m = __lsx_vreplvei_h(coeff_m, 6); \ + x3_m = __lsx_vneg_h(x3_m); \ + x2_m = __lsx_vpackev_h(x2_m, x3_m); \ + DOT_SHIFT_RIGHT_PCK_H(s6_m, s7_m, x2_m, out3); \ + } + +#define FDCT8x16_ODD(input0, input1, input2, input3, input4, input5, input6, \ + input7, out1, out3, out5, out7, out9, out11, out13, \ + out15) \ + { \ + __m128i stp21_m, stp22_m, stp23_m, stp24_m, stp25_m, stp26_m; \ + __m128i stp30_m, stp31_m, stp32_m, stp33_m, stp34_m, stp35_m; \ + __m128i stp36_m, stp37_m, vec0_m, vec1_m; \ + __m128i vec2_m, vec3_m, vec4_m, vec5_m, vec6_m; \ + __m128i cnst0_m, cnst1_m, cnst4_m, cnst5_m; \ + __m128i coeff_m = { 0x187e3b21d2bf2d41, 0x238e3537e782c4df }; \ + __m128i coeff1_m = { 0x289a317906463fb1, 0x12943d3f1e2b3871 }; \ + __m128i coeff2_m = { 0xed6cd766c78fc04f, 0x0 }; \ + \ + /* stp 1 */ \ + DUP2_ARG2(__lsx_vilvh_h, input2, input5, input3, input4, vec2_m, vec4_m); \ + DUP2_ARG2(__lsx_vilvl_h, input2, input5, input3, input4, vec3_m, vec5_m); \ + \ + cnst4_m = __lsx_vreplvei_h(coeff_m, 0); \ + DOT_SHIFT_RIGHT_PCK_H(vec2_m, vec3_m, cnst4_m, stp25_m); \ + \ + cnst5_m = __lsx_vreplvei_h(coeff_m, 1); \ + cnst5_m = __lsx_vpackev_h(cnst5_m, cnst4_m); \ + DOT_SHIFT_RIGHT_PCK_H(vec2_m, vec3_m, cnst5_m, stp22_m); \ + DOT_SHIFT_RIGHT_PCK_H(vec4_m, vec5_m, cnst4_m, stp24_m); \ + DOT_SHIFT_RIGHT_PCK_H(vec4_m, vec5_m, cnst5_m, stp23_m); \ + \ + /* stp2 */ \ + LSX_BUTTERFLY_4_H(input0, input1, stp22_m, stp23_m, stp30_m, stp31_m, \ + stp32_m, stp33_m); \ + LSX_BUTTERFLY_4_H(input7, input6, stp25_m, stp24_m, stp37_m, stp36_m, \ + stp35_m, stp34_m); \ + \ + DUP2_ARG2(__lsx_vilvh_h, stp36_m, stp31_m, stp35_m, stp32_m, vec2_m, \ + vec4_m); \ + DUP2_ARG2(__lsx_vilvl_h, stp36_m, stp31_m, stp35_m, stp32_m, vec3_m, \ + vec5_m); \ + \ + DUP2_ARG2(__lsx_vreplvei_h, coeff_m, 2, coeff_m, 3, cnst0_m, cnst1_m); \ + cnst0_m = __lsx_vpackev_h(cnst0_m, cnst1_m); \ + DOT_SHIFT_RIGHT_PCK_H(vec2_m, vec3_m, cnst0_m, stp26_m); \ + \ + cnst0_m = __lsx_vreplvei_h(coeff_m, 4); \ + cnst1_m = __lsx_vpackev_h(cnst1_m, cnst0_m); \ + DOT_SHIFT_RIGHT_PCK_H(vec2_m, vec3_m, cnst1_m, stp21_m); \ + \ + DUP2_ARG2(__lsx_vreplvei_h, coeff_m, 5, coeff_m, 2, cnst0_m, cnst1_m); \ + cnst1_m = __lsx_vpackev_h(cnst0_m, cnst1_m); \ + DOT_SHIFT_RIGHT_PCK_H(vec4_m, vec5_m, cnst1_m, stp25_m); \ + \ + cnst0_m = __lsx_vreplvei_h(coeff_m, 3); \ + cnst1_m = __lsx_vpackev_h(cnst1_m, cnst0_m); \ + DOT_SHIFT_RIGHT_PCK_H(vec4_m, vec5_m, cnst1_m, stp22_m); \ + \ + /* stp4 */ \ + LSX_BUTTERFLY_4_H(stp30_m, stp37_m, stp26_m, stp21_m, vec6_m, vec2_m, \ + vec4_m, vec5_m); \ + LSX_BUTTERFLY_4_H(stp33_m, stp34_m, stp25_m, stp22_m, stp21_m, stp23_m, \ + stp24_m, stp31_m); \ + \ + vec1_m = __lsx_vilvl_h(vec2_m, vec6_m); \ + vec0_m = __lsx_vilvh_h(vec2_m, vec6_m); \ + DUP2_ARG2(__lsx_vreplvei_h, coeff1_m, 0, coeff1_m, 1, cnst0_m, cnst1_m); \ + cnst0_m = __lsx_vpackev_h(cnst0_m, cnst1_m); \ + \ + DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst0_m, out1); \ + \ + cnst0_m = __lsx_vreplvei_h(coeff2_m, 0); \ + cnst0_m = __lsx_vpackev_h(cnst1_m, cnst0_m); \ + DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst0_m, out15); \ + \ + vec1_m = __lsx_vilvl_h(vec4_m, vec5_m); \ + vec0_m = __lsx_vilvh_h(vec4_m, vec5_m); \ + DUP2_ARG2(__lsx_vreplvei_h, coeff1_m, 2, coeff1_m, 3, cnst0_m, cnst1_m); \ + cnst1_m = __lsx_vpackev_h(cnst1_m, cnst0_m); \ + \ + DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst1_m, out9); \ + \ + cnst1_m = __lsx_vreplvei_h(coeff2_m, 2); \ + cnst0_m = __lsx_vpackev_h(cnst0_m, cnst1_m); \ + DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst0_m, out7); \ + \ + vec1_m = __lsx_vilvl_h(stp23_m, stp21_m); \ + vec0_m = __lsx_vilvh_h(stp23_m, stp21_m); \ + DUP2_ARG2(__lsx_vreplvei_h, coeff1_m, 4, coeff1_m, 5, cnst0_m, cnst1_m); \ + cnst0_m = __lsx_vpackev_h(cnst0_m, cnst1_m); \ + DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst0_m, out5); \ + \ + cnst0_m = __lsx_vreplvei_h(coeff2_m, 1); \ + cnst0_m = __lsx_vpackev_h(cnst1_m, cnst0_m); \ + DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst0_m, out11); \ + \ + vec1_m = __lsx_vilvl_h(stp24_m, stp31_m); \ + vec0_m = __lsx_vilvh_h(stp24_m, stp31_m); \ + DUP2_ARG2(__lsx_vreplvei_h, coeff1_m, 6, coeff1_m, 7, cnst0_m, cnst1_m); \ + cnst1_m = __lsx_vpackev_h(cnst1_m, cnst0_m); \ + \ + DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst1_m, out13); \ + \ + cnst1_m = __lsx_vreplvei_h(coeff2_m, 3); \ + cnst0_m = __lsx_vpackev_h(cnst0_m, cnst1_m); \ + DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst0_m, out3); \ + } + +void fdct8x16_1d_column(const int16_t *input, int16_t *tmp_ptr, + int32_t src_stride); +void fdct16x8_1d_row(int16_t *input, int16_t *output); +#endif // VPX_VPX_DSP_LOONGARCH_FWD_TXFM_LSX_H_ diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/fwd_txfm_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/fwd_txfm_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/fwd_txfm_lsx.c @@ -0,0 +1,258 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "./vpx_dsp_rtcd.h" +#include "vpx_dsp/loongarch/fwd_txfm_lsx.h" + +#if !CONFIG_VP9_HIGHBITDEPTH +void fdct8x16_1d_column(const int16_t *input, int16_t *tmp_ptr, + int32_t src_stride) { + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i in8, in9, in10, in11, in12, in13, in14, in15; + __m128i stp21, stp22, stp23, stp24, stp25, stp26, stp30; + __m128i stp31, stp32, stp33, stp34, stp35, stp36, stp37; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, cnst0, cnst1, cnst4, cnst5; + __m128i coeff = { 0x187e3b21d2bf2d41, 0x238e3537e782c4df }; + __m128i coeff1 = { 0x289a317906463fb1, 0x12943d3f1e2b3871 }; + __m128i coeff2 = { 0xed6cd766c78fc04f, 0x0 }; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride4 = src_stride2 << 1; + int32_t src_stride6 = src_stride4 + src_stride2; + int32_t src_stride8 = src_stride4 << 1; + int16_t *input_tmp = (int16_t *)input; + in0 = __lsx_vld(input_tmp, 0); + DUP4_ARG2(__lsx_vldx, input_tmp, src_stride2, input_tmp, src_stride4, + input_tmp, src_stride6, input_tmp, src_stride8, in1, in2, in3, in4); + input_tmp += src_stride4; + DUP4_ARG2(__lsx_vldx, input_tmp, src_stride2, input_tmp, src_stride4, + input_tmp, src_stride6, input_tmp, src_stride8, in5, in6, in7, in8); + input_tmp += src_stride4; + DUP4_ARG2(__lsx_vldx, input_tmp, src_stride2, input_tmp, src_stride4, + input_tmp, src_stride6, input_tmp, src_stride8, in9, in10, in11, + in12); + input_tmp += src_stride4; + DUP2_ARG2(__lsx_vldx, input_tmp, src_stride2, input_tmp, src_stride4, in13, + in14); + input_tmp += src_stride2; + in15 = __lsx_vldx(input_tmp, src_stride2); + + DUP4_ARG2(__lsx_vslli_h, in0, 2, in1, 2, in2, 2, in3, 2, in0, in1, in2, in3); + DUP4_ARG2(__lsx_vslli_h, in4, 2, in5, 2, in6, 2, in7, 2, in4, in5, in6, in7); + DUP4_ARG2(__lsx_vslli_h, in8, 2, in9, 2, in10, 2, in11, 2, in8, in9, in10, + in11); + DUP4_ARG2(__lsx_vslli_h, in12, 2, in13, 2, in14, 2, in15, 2, in12, in13, in14, + in15); + DUP4_ARG2(__lsx_vadd_h, in0, in15, in1, in14, in2, in13, in3, in12, tmp0, + tmp1, tmp2, tmp3); + DUP4_ARG2(__lsx_vadd_h, in4, in11, in5, in10, in6, in9, in7, in8, tmp4, tmp5, + tmp6, tmp7); + FDCT8x16_EVEN(tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp0, tmp1, + tmp2, tmp3, tmp4, tmp5, tmp6, tmp7); + __lsx_vst(tmp0, tmp_ptr, 0); + __lsx_vst(tmp1, tmp_ptr, 64); + __lsx_vst(tmp2, tmp_ptr, 128); + __lsx_vst(tmp3, tmp_ptr, 192); + __lsx_vst(tmp4, tmp_ptr, 256); + __lsx_vst(tmp5, tmp_ptr, 320); + __lsx_vst(tmp6, tmp_ptr, 384); + __lsx_vst(tmp7, tmp_ptr, 448); + DUP4_ARG2(__lsx_vsub_h, in0, in15, in1, in14, in2, in13, in3, in12, in15, + in14, in13, in12); + DUP4_ARG2(__lsx_vsub_h, in4, in11, in5, in10, in6, in9, in7, in8, in11, in10, + in9, in8); + + tmp_ptr += 16; + + /* stp 1 */ + DUP2_ARG2(__lsx_vilvh_h, in10, in13, in11, in12, vec2, vec4); + DUP2_ARG2(__lsx_vilvl_h, in10, in13, in11, in12, vec3, vec5); + + cnst4 = __lsx_vreplvei_h(coeff, 0); + DOT_SHIFT_RIGHT_PCK_H(vec2, vec3, cnst4, stp25); + + cnst5 = __lsx_vreplvei_h(coeff, 1); + cnst5 = __lsx_vpackev_h(cnst5, cnst4); + DOT_SHIFT_RIGHT_PCK_H(vec2, vec3, cnst5, stp22); + DOT_SHIFT_RIGHT_PCK_H(vec4, vec5, cnst4, stp24); + DOT_SHIFT_RIGHT_PCK_H(vec4, vec5, cnst5, stp23); + + /* stp2 */ + LSX_BUTTERFLY_4_H(in8, in9, stp22, stp23, stp30, stp31, stp32, stp33); + LSX_BUTTERFLY_4_H(in15, in14, stp25, stp24, stp37, stp36, stp35, stp34); + DUP2_ARG2(__lsx_vilvh_h, stp36, stp31, stp35, stp32, vec2, vec4); + DUP2_ARG2(__lsx_vilvl_h, stp36, stp31, stp35, stp32, vec3, vec5); + DUP2_ARG2(__lsx_vreplvei_h, coeff, 2, coeff, 3, cnst0, cnst1); + cnst0 = __lsx_vpackev_h(cnst0, cnst1); + DOT_SHIFT_RIGHT_PCK_H(vec2, vec3, cnst0, stp26); + + cnst0 = __lsx_vreplvei_h(coeff, 4); + cnst1 = __lsx_vpackev_h(cnst1, cnst0); + DOT_SHIFT_RIGHT_PCK_H(vec2, vec3, cnst1, stp21); + + LSX_BUTTERFLY_4_H(stp30, stp37, stp26, stp21, in8, in15, in14, in9); + vec1 = __lsx_vilvl_h(in15, in8); + vec0 = __lsx_vilvh_h(in15, in8); + + DUP2_ARG2(__lsx_vreplvei_h, coeff1, 0, coeff1, 1, cnst0, cnst1); + cnst0 = __lsx_vpackev_h(cnst0, cnst1); + + DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst0, in8); + __lsx_vst(in8, tmp_ptr, 0); + + cnst0 = __lsx_vreplvei_h(coeff2, 0); + cnst0 = __lsx_vpackev_h(cnst1, cnst0); + DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst0, in8); + __lsx_vst(in8, tmp_ptr, 448); + + vec1 = __lsx_vilvl_h(in14, in9); + vec0 = __lsx_vilvh_h(in14, in9); + DUP2_ARG2(__lsx_vreplvei_h, coeff1, 2, coeff1, 3, cnst0, cnst1); + cnst1 = __lsx_vpackev_h(cnst1, cnst0); + + DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst1, in8); + __lsx_vst(in8, tmp_ptr, 256); + + cnst1 = __lsx_vreplvei_h(coeff2, 2); + cnst0 = __lsx_vpackev_h(cnst0, cnst1); + DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst0, in8); + __lsx_vst(in8, tmp_ptr, 192); + + DUP2_ARG2(__lsx_vreplvei_h, coeff, 2, coeff, 5, cnst0, cnst1); + cnst1 = __lsx_vpackev_h(cnst1, cnst0); + DOT_SHIFT_RIGHT_PCK_H(vec4, vec5, cnst1, stp25); + + cnst1 = __lsx_vreplvei_h(coeff, 3); + cnst1 = __lsx_vpackev_h(cnst0, cnst1); + DOT_SHIFT_RIGHT_PCK_H(vec4, vec5, cnst1, stp22); + + /* stp4 */ + DUP2_ARG2(__lsx_vadd_h, stp34, stp25, stp33, stp22, in13, in10); + + vec1 = __lsx_vilvl_h(in13, in10); + vec0 = __lsx_vilvh_h(in13, in10); + DUP2_ARG2(__lsx_vreplvei_h, coeff1, 4, coeff1, 5, cnst0, cnst1); + cnst0 = __lsx_vpackev_h(cnst0, cnst1); + DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst0, in8); + __lsx_vst(in8, tmp_ptr, 128); + + cnst0 = __lsx_vreplvei_h(coeff2, 1); + cnst0 = __lsx_vpackev_h(cnst1, cnst0); + DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst0, in8); + __lsx_vst(in8, tmp_ptr, 320); + + DUP2_ARG2(__lsx_vsub_h, stp34, stp25, stp33, stp22, in12, in11); + vec1 = __lsx_vilvl_h(in12, in11); + vec0 = __lsx_vilvh_h(in12, in11); + DUP2_ARG2(__lsx_vreplvei_h, coeff1, 6, coeff1, 7, cnst0, cnst1); + cnst1 = __lsx_vpackev_h(cnst1, cnst0); + + DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst1, in8); + __lsx_vst(in8, tmp_ptr, 384); + + cnst1 = __lsx_vreplvei_h(coeff2, 3); + cnst0 = __lsx_vpackev_h(cnst0, cnst1); + DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst0, in8); + __lsx_vst(in8, tmp_ptr, 64); +} + +void fdct16x8_1d_row(int16_t *input, int16_t *output) { + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i in8, in9, in10, in11, in12, in13, in14, in15; + int16_t *input_tmp = input; + + DUP4_ARG2(__lsx_vld, input, 0, input, 32, input, 64, input, 96, in0, in1, in2, + in3); + DUP4_ARG2(__lsx_vld, input, 128, input, 160, input, 192, input, 224, in4, in5, + in6, in7); + DUP4_ARG2(__lsx_vld, input_tmp, 16, input_tmp, 48, input_tmp, 80, input_tmp, + 112, in8, in9, in10, in11); + DUP4_ARG2(__lsx_vld, input_tmp, 144, input_tmp, 176, input_tmp, 208, + input_tmp, 240, in12, in13, in14, in15); + + LSX_TRANSPOSE8x8_H(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + LSX_TRANSPOSE8x8_H(in8, in9, in10, in11, in12, in13, in14, in15, in8, in9, + in10, in11, in12, in13, in14, in15); + DUP4_ARG2(__lsx_vaddi_hu, in0, 1, in1, 1, in2, 1, in3, 1, in0, in1, in2, in3); + DUP4_ARG2(__lsx_vaddi_hu, in4, 1, in5, 1, in6, 1, in7, 1, in4, in5, in6, in7); + DUP4_ARG2(__lsx_vaddi_hu, in8, 1, in9, 1, in10, 1, in11, 1, in8, in9, in10, + in11); + DUP4_ARG2(__lsx_vaddi_hu, in12, 1, in13, 1, in14, 1, in15, 1, in12, in13, + in14, in15); + + DUP4_ARG2(__lsx_vsrai_h, in0, 2, in1, 2, in2, 2, in3, 2, in0, in1, in2, in3); + DUP4_ARG2(__lsx_vsrai_h, in4, 2, in5, 2, in6, 2, in7, 2, in4, in5, in6, in7); + DUP4_ARG2(__lsx_vsrai_h, in8, 2, in9, 2, in10, 2, in11, 2, in8, in9, in10, + in11); + DUP4_ARG2(__lsx_vsrai_h, in12, 2, in13, 2, in14, 2, in15, 2, in12, in13, in14, + in15); + LSX_BUTTERFLY_16_H(in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, in10, + in11, in12, in13, in14, in15, tmp0, tmp1, tmp2, tmp3, tmp4, + tmp5, tmp6, tmp7, in8, in9, in10, in11, in12, in13, in14, + in15); + __lsx_vst(in8, input, 0); + __lsx_vst(in9, input, 32); + __lsx_vst(in10, input, 64); + __lsx_vst(in11, input, 96); + __lsx_vst(in12, input, 128); + __lsx_vst(in13, input, 160); + __lsx_vst(in14, input, 192); + __lsx_vst(in15, input, 224); + + FDCT8x16_EVEN(tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp0, tmp1, + tmp2, tmp3, tmp4, tmp5, tmp6, tmp7); + DUP4_ARG2(__lsx_vld, input, 0, input, 32, input, 64, input, 96, in8, in9, + in10, in11); + DUP4_ARG2(__lsx_vld, input, 128, input, 160, input, 192, input, 224, in12, + in13, in14, in15); + FDCT8x16_ODD(in8, in9, in10, in11, in12, in13, in14, in15, in0, in1, in2, in3, + in4, in5, in6, in7); + LSX_TRANSPOSE8x8_H(tmp0, in0, tmp1, in1, tmp2, in2, tmp3, in3, tmp0, in0, + tmp1, in1, tmp2, in2, tmp3, in3); + __lsx_vst(tmp0, output, 0); + __lsx_vst(in0, output, 32); + __lsx_vst(tmp1, output, 64); + __lsx_vst(in1, output, 96); + __lsx_vst(tmp2, output, 128); + __lsx_vst(in2, output, 160); + __lsx_vst(tmp3, output, 192); + __lsx_vst(in3, output, 224); + + LSX_TRANSPOSE8x8_H(tmp4, in4, tmp5, in5, tmp6, in6, tmp7, in7, tmp4, in4, + tmp5, in5, tmp6, in6, tmp7, in7); + __lsx_vst(tmp4, output, 16); + __lsx_vst(in4, output, 48); + __lsx_vst(tmp5, output, 80); + __lsx_vst(in5, output, 112); + __lsx_vst(tmp6, output, 144); + __lsx_vst(in6, output, 176); + __lsx_vst(tmp7, output, 208); + __lsx_vst(in7, output, 240); +} + +void vpx_fdct16x16_lsx(const int16_t *input, int16_t *output, + int32_t src_stride) { + int32_t i; + DECLARE_ALIGNED(32, int16_t, tmp_buf[16 * 16]); + + /* column transform */ + for (i = 0; i < 2; ++i) { + fdct8x16_1d_column((input + 8 * i), (&tmp_buf[0] + 8 * i), src_stride); + } + + /* row transform */ + for (i = 0; i < 2; ++i) { + fdct16x8_1d_row((&tmp_buf[0] + (128 * i)), (output + (128 * i))); + } +} +#endif // !CONFIG_VP9_HIGHBITDEPTH diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/idct32x32_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/idct32x32_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/idct32x32_lsx.c @@ -0,0 +1,834 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "./vpx_dsp_rtcd.h" +#include "vpx_dsp/loongarch/fwd_txfm_lsx.h" + +#define UNPCK_UB_SH(_in, _out0, _out1) \ + { \ + _out0 = __lsx_vsllwil_hu_bu(_in, 0); \ + _out1 = __lsx_vexth_hu_bu(_in); \ + } + +static void idct32x8_row_transpose_store(const int16_t *input, + int16_t *tmp_buf) { + __m128i m0, m1, m2, m3, m4, m5, m6, m7; + __m128i n0, n1, n2, n3, n4, n5, n6, n7; + + /* 1st & 2nd 8x8 */ + DUP4_ARG2(__lsx_vld, input, 0, input, 64, input, 128, input, 192, m0, n0, m1, + n1); + DUP4_ARG2(__lsx_vld, input, 256, input, 320, input, 384, input, 448, m2, n2, + m3, n3); + DUP4_ARG2(__lsx_vld, input, 16, input, 80, input, 144, input, 208, m4, n4, m5, + n5); + DUP4_ARG2(__lsx_vld, input, 272, input, 336, input, 400, input, 464, m6, n6, + m7, n7); + + LSX_TRANSPOSE8x8_H(m0, n0, m1, n1, m2, n2, m3, n3, m0, n0, m1, n1, m2, n2, m3, + n3); + LSX_TRANSPOSE8x8_H(m4, n4, m5, n5, m6, n6, m7, n7, m4, n4, m5, n5, m6, n6, m7, + n7); + + __lsx_vst(m0, tmp_buf, 0); + __lsx_vst(n0, tmp_buf, 16); + __lsx_vst(m1, tmp_buf, 32); + __lsx_vst(n1, tmp_buf, 48); + __lsx_vst(m2, tmp_buf, 64); + __lsx_vst(n2, tmp_buf, 80); + __lsx_vst(m3, tmp_buf, 96); + __lsx_vst(n3, tmp_buf, 112); + __lsx_vst(m4, tmp_buf, 128); + __lsx_vst(n4, tmp_buf, 144); + __lsx_vst(m5, tmp_buf, 160); + __lsx_vst(n5, tmp_buf, 176); + __lsx_vst(m6, tmp_buf, 192); + __lsx_vst(n6, tmp_buf, 208); + __lsx_vst(m7, tmp_buf, 224); + __lsx_vst(n7, tmp_buf, 240); + + /* 3rd & 4th 8x8 */ + DUP4_ARG2(__lsx_vld, input, 32, input, 96, input, 160, input, 224, m0, n0, m1, + n1); + DUP4_ARG2(__lsx_vld, input, 288, input, 352, input, 416, input, 480, m2, n2, + m3, n3); + DUP4_ARG2(__lsx_vld, input, 48, input, 112, input, 176, input, 240, m4, n4, + m5, n5); + DUP4_ARG2(__lsx_vld, input, 304, input, 368, input, 432, input, 496, m6, n6, + m7, n7); + + LSX_TRANSPOSE8x8_H(m0, n0, m1, n1, m2, n2, m3, n3, m0, n0, m1, n1, m2, n2, m3, + n3); + LSX_TRANSPOSE8x8_H(m4, n4, m5, n5, m6, n6, m7, n7, m4, n4, m5, n5, m6, n6, m7, + n7); + + __lsx_vst(m0, tmp_buf, 256); + __lsx_vst(n0, tmp_buf, 272); + __lsx_vst(m1, tmp_buf, 288); + __lsx_vst(n1, tmp_buf, 304); + __lsx_vst(m2, tmp_buf, 320); + __lsx_vst(n2, tmp_buf, 336); + __lsx_vst(m3, tmp_buf, 352); + __lsx_vst(n3, tmp_buf, 368); + __lsx_vst(m4, tmp_buf, 384); + __lsx_vst(n4, tmp_buf, 400); + __lsx_vst(m5, tmp_buf, 416); + __lsx_vst(n5, tmp_buf, 432); + __lsx_vst(m6, tmp_buf, 448); + __lsx_vst(n6, tmp_buf, 464); + __lsx_vst(m7, tmp_buf, 480); + __lsx_vst(n7, tmp_buf, 496); +} + +static void idct32x8_row_even_process_store(int16_t *tmp_buf, + int16_t *tmp_eve_buf) { + __m128i vec0, vec1, vec2, vec3, loc0, loc1, loc2, loc3; + __m128i reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7; + __m128i stp0, stp1, stp2, stp3, stp4, stp5, stp6, stp7; + __m128i tmp0; + + /* Even stage 1 */ + DUP4_ARG2(__lsx_vld, tmp_buf, 0, tmp_buf, 64, tmp_buf, 128, tmp_buf, 192, + reg0, reg1, reg2, reg3); + DUP4_ARG2(__lsx_vld, tmp_buf, 256, tmp_buf, 320, tmp_buf, 384, tmp_buf, 448, + reg4, reg5, reg6, reg7); + + DOTP_CONST_PAIR(reg1, reg7, cospi_28_64, cospi_4_64, reg1, reg7); + DOTP_CONST_PAIR(reg5, reg3, cospi_12_64, cospi_20_64, reg5, reg3); + LSX_BUTTERFLY_4_H(reg1, reg7, reg3, reg5, vec1, vec3, vec2, vec0); + DOTP_CONST_PAIR(vec2, vec0, cospi_16_64, cospi_16_64, loc2, loc3); + + loc1 = vec3; + loc0 = vec1; + + DOTP_CONST_PAIR(reg0, reg4, cospi_16_64, cospi_16_64, reg0, reg4); + DOTP_CONST_PAIR(reg2, reg6, cospi_24_64, cospi_8_64, reg2, reg6); + LSX_BUTTERFLY_4_H(reg4, reg0, reg2, reg6, vec1, vec3, vec2, vec0); + LSX_BUTTERFLY_4_H(vec0, vec1, loc1, loc0, stp3, stp0, stp7, stp4); + LSX_BUTTERFLY_4_H(vec2, vec3, loc3, loc2, stp2, stp1, stp6, stp5); + + /* Even stage 2 */ + DUP4_ARG2(__lsx_vld, tmp_buf, 32, tmp_buf, 96, tmp_buf, 160, tmp_buf, 224, + reg0, reg1, reg2, reg3); + DUP4_ARG2(__lsx_vld, tmp_buf, 288, tmp_buf, 352, tmp_buf, 416, tmp_buf, 480, + reg4, reg5, reg6, reg7); + DOTP_CONST_PAIR(reg0, reg7, cospi_30_64, cospi_2_64, reg0, reg7); + DOTP_CONST_PAIR(reg4, reg3, cospi_14_64, cospi_18_64, reg4, reg3); + DOTP_CONST_PAIR(reg2, reg5, cospi_22_64, cospi_10_64, reg2, reg5); + DOTP_CONST_PAIR(reg6, reg1, cospi_6_64, cospi_26_64, reg6, reg1); + + vec0 = __lsx_vadd_h(reg0, reg4); + reg0 = __lsx_vsub_h(reg0, reg4); + reg4 = __lsx_vadd_h(reg6, reg2); + reg6 = __lsx_vsub_h(reg6, reg2); + reg2 = __lsx_vadd_h(reg1, reg5); + reg1 = __lsx_vsub_h(reg1, reg5); + reg5 = __lsx_vadd_h(reg7, reg3); + reg7 = __lsx_vsub_h(reg7, reg3); + reg3 = vec0; + + vec1 = reg2; + reg2 = __lsx_vadd_h(reg3, reg4); + reg3 = __lsx_vsub_h(reg3, reg4); + reg4 = __lsx_vsub_h(reg5, vec1); + reg5 = __lsx_vadd_h(reg5, vec1); + + tmp0 = __lsx_vneg_h(reg6); + DOTP_CONST_PAIR(reg7, reg0, cospi_24_64, cospi_8_64, reg0, reg7); + DOTP_CONST_PAIR(tmp0, reg1, cospi_24_64, cospi_8_64, reg6, reg1); + + vec0 = __lsx_vsub_h(reg0, reg6); + reg0 = __lsx_vadd_h(reg0, reg6); + vec1 = __lsx_vsub_h(reg7, reg1); + reg7 = __lsx_vadd_h(reg7, reg1); + + DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, reg6, reg1); + DOTP_CONST_PAIR(reg4, reg3, cospi_16_64, cospi_16_64, reg3, reg4); + + /* Even stage 3 : Dependency on Even stage 1 & Even stage 2 */ + LSX_BUTTERFLY_4_H(stp0, stp1, reg7, reg5, loc1, loc3, loc2, loc0); + __lsx_vst(loc0, tmp_eve_buf, 240); + __lsx_vst(loc1, tmp_eve_buf, 0); + __lsx_vst(loc2, tmp_eve_buf, 224); + __lsx_vst(loc3, tmp_eve_buf, 16); + + LSX_BUTTERFLY_4_H(stp2, stp3, reg4, reg1, loc1, loc3, loc2, loc0); + __lsx_vst(loc0, tmp_eve_buf, 208); + __lsx_vst(loc1, tmp_eve_buf, 32); + __lsx_vst(loc2, tmp_eve_buf, 192); + __lsx_vst(loc3, tmp_eve_buf, 48); + + /* Store 8 */ + LSX_BUTTERFLY_4_H(stp4, stp5, reg6, reg3, loc1, loc3, loc2, loc0); + __lsx_vst(loc0, tmp_eve_buf, 176); + __lsx_vst(loc1, tmp_eve_buf, 64); + __lsx_vst(loc2, tmp_eve_buf, 160); + __lsx_vst(loc3, tmp_eve_buf, 80); + + LSX_BUTTERFLY_4_H(stp6, stp7, reg2, reg0, loc1, loc3, loc2, loc0); + __lsx_vst(loc0, tmp_eve_buf, 144); + __lsx_vst(loc1, tmp_eve_buf, 96); + __lsx_vst(loc2, tmp_eve_buf, 128); + __lsx_vst(loc3, tmp_eve_buf, 112); +} + +static void idct32x8_row_odd_process_store(int16_t *tmp_buf, + int16_t *tmp_odd_buf) { + __m128i vec0, vec1, vec2, vec3, loc0, loc1, loc2, loc3; + __m128i reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7; + + /* Odd stage 1 */ + DUP4_ARG2(__lsx_vld, tmp_buf, 16, tmp_buf, 112, tmp_buf, 144, tmp_buf, 240, + reg0, reg1, reg2, reg3); + DUP4_ARG2(__lsx_vld, tmp_buf, 272, tmp_buf, 368, tmp_buf, 400, tmp_buf, 496, + reg4, reg5, reg6, reg7); + + DOTP_CONST_PAIR(reg0, reg7, cospi_31_64, cospi_1_64, reg0, reg7); + DOTP_CONST_PAIR(reg4, reg3, cospi_15_64, cospi_17_64, reg3, reg4); + DOTP_CONST_PAIR(reg2, reg5, cospi_23_64, cospi_9_64, reg2, reg5); + DOTP_CONST_PAIR(reg6, reg1, cospi_7_64, cospi_25_64, reg1, reg6); + + vec0 = __lsx_vadd_h(reg0, reg3); + reg0 = __lsx_vsub_h(reg0, reg3); + reg3 = __lsx_vadd_h(reg7, reg4); + reg7 = __lsx_vsub_h(reg7, reg4); + reg4 = __lsx_vadd_h(reg1, reg2); + reg1 = __lsx_vsub_h(reg1, reg2); + reg2 = __lsx_vadd_h(reg6, reg5); + reg6 = __lsx_vsub_h(reg6, reg5); + reg5 = vec0; + + /* 4 Stores */ + DUP2_ARG2(__lsx_vadd_h, reg5, reg4, reg3, reg2, vec0, vec1); + __lsx_vst(vec0, tmp_odd_buf, 64); + __lsx_vst(vec1, tmp_odd_buf, 80); + + DUP2_ARG2(__lsx_vsub_h, reg5, reg4, reg3, reg2, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_24_64, cospi_8_64, vec0, vec1); + __lsx_vst(vec0, tmp_odd_buf, 0); + __lsx_vst(vec1, tmp_odd_buf, 16); + + /* 4 Stores */ + DOTP_CONST_PAIR(reg7, reg0, cospi_28_64, cospi_4_64, reg0, reg7); + DOTP_CONST_PAIR(reg6, reg1, -cospi_4_64, cospi_28_64, reg1, reg6); + LSX_BUTTERFLY_4_H(reg0, reg7, reg6, reg1, vec0, vec1, vec2, vec3); + __lsx_vst(vec0, tmp_odd_buf, 96); + __lsx_vst(vec1, tmp_odd_buf, 112); + + DOTP_CONST_PAIR(vec2, vec3, cospi_24_64, cospi_8_64, vec2, vec3); + __lsx_vst(vec2, tmp_odd_buf, 32); + __lsx_vst(vec3, tmp_odd_buf, 48); + + /* Odd stage 2 */ + /* 8 loads */ + DUP4_ARG2(__lsx_vld, tmp_buf, 48, tmp_buf, 80, tmp_buf, 176, tmp_buf, 208, + reg0, reg1, reg2, reg3); + DUP4_ARG2(__lsx_vld, tmp_buf, 304, tmp_buf, 336, tmp_buf, 432, tmp_buf, 464, + reg4, reg5, reg6, reg7); + + DOTP_CONST_PAIR(reg1, reg6, cospi_27_64, cospi_5_64, reg1, reg6); + DOTP_CONST_PAIR(reg5, reg2, cospi_11_64, cospi_21_64, reg2, reg5); + DOTP_CONST_PAIR(reg3, reg4, cospi_19_64, cospi_13_64, reg3, reg4); + DOTP_CONST_PAIR(reg7, reg0, cospi_3_64, cospi_29_64, reg0, reg7); + + /* 4 Stores */ + DUP4_ARG2(__lsx_vsub_h, reg1, reg2, reg6, reg5, reg0, reg3, reg7, reg4, vec0, + vec1, vec2, vec3); + DOTP_CONST_PAIR(vec1, vec0, cospi_12_64, cospi_20_64, loc0, loc1); + DOTP_CONST_PAIR(vec3, vec2, -cospi_20_64, cospi_12_64, loc2, loc3); + + LSX_BUTTERFLY_4_H(loc3, loc2, loc0, loc1, vec1, vec0, vec2, vec3); + __lsx_vst(vec0, tmp_odd_buf, 192); + __lsx_vst(vec1, tmp_odd_buf, 240); + + DOTP_CONST_PAIR(vec3, vec2, -cospi_8_64, cospi_24_64, vec0, vec1); + __lsx_vst(vec0, tmp_odd_buf, 160); + __lsx_vst(vec1, tmp_odd_buf, 176); + + /* 4 Stores */ + DUP4_ARG2(__lsx_vadd_h, reg1, reg2, reg6, reg5, reg0, reg3, reg7, reg4, vec1, + vec2, vec0, vec3); + LSX_BUTTERFLY_4_H(vec0, vec3, vec2, vec1, reg0, reg1, reg3, reg2); + __lsx_vst(reg0, tmp_odd_buf, 208); + __lsx_vst(reg1, tmp_odd_buf, 224); + + DOTP_CONST_PAIR(reg3, reg2, -cospi_8_64, cospi_24_64, reg0, reg1); + __lsx_vst(reg0, tmp_odd_buf, 128); + __lsx_vst(reg1, tmp_odd_buf, 144); + + /* Odd stage 3 : Dependency on Odd stage 1 & Odd stage 2 */ + + /* Load 8 & Store 8 */ + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 0, tmp_odd_buf, 16, tmp_odd_buf, 32, + tmp_odd_buf, 48, reg0, reg1, reg2, reg3); + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 128, tmp_odd_buf, 144, tmp_odd_buf, 160, + tmp_odd_buf, 176, reg4, reg5, reg6, reg7); + DUP4_ARG2(__lsx_vadd_h, reg0, reg4, reg1, reg5, reg2, reg6, reg3, reg7, loc0, + loc1, loc2, loc3); + __lsx_vst(loc0, tmp_odd_buf, 0); + __lsx_vst(loc1, tmp_odd_buf, 16); + __lsx_vst(loc2, tmp_odd_buf, 32); + __lsx_vst(loc3, tmp_odd_buf, 48); + + DUP2_ARG2(__lsx_vsub_h, reg0, reg4, reg1, reg5, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc0, loc1); + + DUP2_ARG2(__lsx_vsub_h, reg2, reg6, reg3, reg7, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc2, loc3); + __lsx_vst(loc0, tmp_odd_buf, 128); + __lsx_vst(loc1, tmp_odd_buf, 144); + __lsx_vst(loc2, tmp_odd_buf, 160); + __lsx_vst(loc3, tmp_odd_buf, 176); + + /* Load 8 & Store 8 */ + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 64, tmp_odd_buf, 80, tmp_odd_buf, 96, + tmp_odd_buf, 112, reg1, reg2, reg0, reg3); + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 192, tmp_odd_buf, 208, tmp_odd_buf, 224, + tmp_odd_buf, 240, reg4, reg5, reg6, reg7); + + DUP4_ARG2(__lsx_vadd_h, reg0, reg4, reg1, reg5, reg2, reg6, reg3, reg7, loc0, + loc1, loc2, loc3); + __lsx_vst(loc0, tmp_odd_buf, 64); + __lsx_vst(loc1, tmp_odd_buf, 80); + __lsx_vst(loc2, tmp_odd_buf, 96); + __lsx_vst(loc3, tmp_odd_buf, 112); + + DUP2_ARG2(__lsx_vsub_h, reg0, reg4, reg3, reg7, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc0, loc1); + DUP2_ARG2(__lsx_vsub_h, reg1, reg5, reg2, reg6, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc2, loc3); + __lsx_vst(loc0, tmp_odd_buf, 192); + __lsx_vst(loc1, tmp_odd_buf, 208); + __lsx_vst(loc2, tmp_odd_buf, 224); + __lsx_vst(loc3, tmp_odd_buf, 240); +} + +static void idct_butterfly_transpose_store(int16_t *tmp_buf, + int16_t *tmp_eve_buf, + int16_t *tmp_odd_buf, int16_t *dst) { + __m128i vec0, vec1, vec2, vec3, loc0, loc1, loc2, loc3; + __m128i m0, m1, m2, m3, m4, m5, m6, m7; + __m128i n0, n1, n2, n3, n4, n5, n6, n7; + __m128i reg0, reg1, reg2, reg3; + + /* FINAL BUTTERFLY : Dependency on Even & Odd */ + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 0, tmp_odd_buf, 144, tmp_odd_buf, 224, + tmp_odd_buf, 96, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vld, tmp_eve_buf, 0, tmp_eve_buf, 128, tmp_eve_buf, 64, + tmp_eve_buf, 192, loc0, loc1, loc2, loc3); + + DUP4_ARG2(__lsx_vadd_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, m0, + m4, m2, m6); + DUP4_ARG2(__lsx_vsub_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, reg0, + reg1, reg2, reg3); + __lsx_vst(reg0, tmp_buf, 496); + __lsx_vst(reg1, tmp_buf, 368); + __lsx_vst(reg2, tmp_buf, 432); + __lsx_vst(reg3, tmp_buf, 304); + + /* Load 8 & Store 8 */ + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 64, tmp_odd_buf, 208, tmp_odd_buf, 160, + tmp_odd_buf, 48, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vld, tmp_eve_buf, 32, tmp_eve_buf, 160, tmp_eve_buf, 96, + tmp_eve_buf, 224, loc0, loc1, loc2, loc3); + + DUP4_ARG2(__lsx_vadd_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, m1, + m5, m3, m7); + DUP4_ARG2(__lsx_vsub_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, reg0, + reg1, reg2, reg3); + __lsx_vst(reg0, tmp_buf, 464); + __lsx_vst(reg1, tmp_buf, 336); + __lsx_vst(reg2, tmp_buf, 400); + __lsx_vst(reg3, tmp_buf, 272); + + /* Load 8 & Store 8 */ + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 32, tmp_odd_buf, 176, tmp_odd_buf, 192, + tmp_odd_buf, 112, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vld, tmp_eve_buf, 16, tmp_eve_buf, 144, tmp_eve_buf, 80, + tmp_eve_buf, 208, loc0, loc1, loc2, loc3); + + DUP4_ARG2(__lsx_vadd_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, n0, + n4, n2, n6); + DUP4_ARG2(__lsx_vsub_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, reg0, + reg1, reg2, reg3); + __lsx_vst(reg0, tmp_buf, 480); + __lsx_vst(reg1, tmp_buf, 352); + __lsx_vst(reg2, tmp_buf, 416); + __lsx_vst(reg3, tmp_buf, 288); + + /* Load 8 & Store 8 */ + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 80, tmp_odd_buf, 240, tmp_odd_buf, 128, + tmp_odd_buf, 16, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vld, tmp_eve_buf, 48, tmp_eve_buf, 176, tmp_eve_buf, 112, + tmp_eve_buf, 240, loc0, loc1, loc2, loc3); + DUP4_ARG2(__lsx_vadd_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, n1, + n5, n3, n7); + DUP4_ARG2(__lsx_vsub_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, reg0, + reg1, reg2, reg3); + __lsx_vst(reg0, tmp_buf, 448); + __lsx_vst(reg1, tmp_buf, 320); + __lsx_vst(reg2, tmp_buf, 384); + __lsx_vst(reg3, tmp_buf, 256); + + /* Transpose : 16 vectors */ + /* 1st & 2nd 8x8 */ + LSX_TRANSPOSE8x8_H(m0, n0, m1, n1, m2, n2, m3, n3, m0, n0, m1, n1, m2, n2, m3, + n3); + __lsx_vst(m0, dst, 0); + __lsx_vst(n0, dst, 64); + __lsx_vst(m1, dst, 128); + __lsx_vst(n1, dst, 192); + __lsx_vst(m2, dst, 256); + __lsx_vst(n2, dst, 320); + __lsx_vst(m3, dst, 384); + __lsx_vst(n3, dst, 448); + + LSX_TRANSPOSE8x8_H(m4, n4, m5, n5, m6, n6, m7, n7, m4, n4, m5, n5, m6, n6, m7, + n7); + __lsx_vst(m4, dst, 16); + __lsx_vst(n4, dst, 80); + __lsx_vst(m5, dst, 144); + __lsx_vst(n5, dst, 208); + __lsx_vst(m6, dst, 272); + __lsx_vst(n6, dst, 336); + __lsx_vst(m7, dst, 400); + __lsx_vst(n7, dst, 464); + + /* 3rd & 4th 8x8 */ + DUP4_ARG2(__lsx_vld, tmp_buf, 256, tmp_buf, 272, tmp_buf, 288, tmp_buf, 304, + m0, n0, m1, n1); + DUP4_ARG2(__lsx_vld, tmp_buf, 320, tmp_buf, 336, tmp_buf, 352, tmp_buf, 368, + m2, n2, m3, n3); + DUP4_ARG2(__lsx_vld, tmp_buf, 384, tmp_buf, 400, tmp_buf, 416, tmp_buf, 432, + m4, n4, m5, n5); + DUP4_ARG2(__lsx_vld, tmp_buf, 448, tmp_buf, 464, tmp_buf, 480, tmp_buf, 496, + m6, n6, m7, n7); + LSX_TRANSPOSE8x8_H(m0, n0, m1, n1, m2, n2, m3, n3, m0, n0, m1, n1, m2, n2, m3, + n3); + LSX_TRANSPOSE8x8_H(m4, n4, m5, n5, m6, n6, m7, n7, m4, n4, m5, n5, m6, n6, m7, + n7); + __lsx_vst(m0, dst, 32); + __lsx_vst(n0, dst, 96); + __lsx_vst(m1, dst, 160); + __lsx_vst(n1, dst, 224); + __lsx_vst(m2, dst, 288); + __lsx_vst(n2, dst, 352); + __lsx_vst(m3, dst, 416); + __lsx_vst(n3, dst, 480); + __lsx_vst(m4, dst, 48); + __lsx_vst(n4, dst, 112); + __lsx_vst(m5, dst, 176); + __lsx_vst(n5, dst, 240); + __lsx_vst(m6, dst, 304); + __lsx_vst(n6, dst, 368); + __lsx_vst(m7, dst, 432); + __lsx_vst(n7, dst, 496); +} + +static void idct32x8_1d_rows_lsx(const int16_t *input, int16_t *output) { + DECLARE_ALIGNED(32, int16_t, tmp_buf[8 * 32]); + DECLARE_ALIGNED(32, int16_t, tmp_odd_buf[16 * 8]); + DECLARE_ALIGNED(32, int16_t, tmp_eve_buf[16 * 8]); + + idct32x8_row_transpose_store(input, &tmp_buf[0]); + idct32x8_row_even_process_store(&tmp_buf[0], &tmp_eve_buf[0]); + idct32x8_row_odd_process_store(&tmp_buf[0], &tmp_odd_buf[0]); + idct_butterfly_transpose_store(&tmp_buf[0], &tmp_eve_buf[0], &tmp_odd_buf[0], + output); +} + +static void idct8x32_column_even_process_store(int16_t *tmp_buf, + int16_t *tmp_eve_buf) { + __m128i vec0, vec1, vec2, vec3, loc0, loc1, loc2, loc3; + __m128i reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7; + __m128i stp0, stp1, stp2, stp3, stp4, stp5, stp6, stp7; + __m128i tmp0; + + /* Even stage 1 */ + DUP4_ARG2(__lsx_vld, tmp_buf, 0, tmp_buf, 256, tmp_buf, 512, tmp_buf, 768, + reg0, reg1, reg2, reg3); + DUP4_ARG2(__lsx_vld, tmp_buf, 1024, tmp_buf, 1280, tmp_buf, 1536, tmp_buf, + 1792, reg4, reg5, reg6, reg7); + tmp_buf += 64; + + DOTP_CONST_PAIR(reg1, reg7, cospi_28_64, cospi_4_64, reg1, reg7); + DOTP_CONST_PAIR(reg5, reg3, cospi_12_64, cospi_20_64, reg5, reg3); + LSX_BUTTERFLY_4_H(reg1, reg7, reg3, reg5, vec1, vec3, vec2, vec0); + DOTP_CONST_PAIR(vec2, vec0, cospi_16_64, cospi_16_64, loc2, loc3); + + loc1 = vec3; + loc0 = vec1; + + DOTP_CONST_PAIR(reg0, reg4, cospi_16_64, cospi_16_64, reg0, reg4); + DOTP_CONST_PAIR(reg2, reg6, cospi_24_64, cospi_8_64, reg2, reg6); + LSX_BUTTERFLY_4_H(reg4, reg0, reg2, reg6, vec1, vec3, vec2, vec0); + LSX_BUTTERFLY_4_H(vec0, vec1, loc1, loc0, stp3, stp0, stp7, stp4); + LSX_BUTTERFLY_4_H(vec2, vec3, loc3, loc2, stp2, stp1, stp6, stp5); + + /* Even stage 2 */ + /* Load 8 */ + DUP4_ARG2(__lsx_vld, tmp_buf, 0, tmp_buf, 256, tmp_buf, 512, tmp_buf, 768, + reg0, reg1, reg2, reg3); + DUP4_ARG2(__lsx_vld, tmp_buf, 1024, tmp_buf, 1280, tmp_buf, 1536, tmp_buf, + 1792, reg4, reg5, reg6, reg7); + DOTP_CONST_PAIR(reg0, reg7, cospi_30_64, cospi_2_64, reg0, reg7); + DOTP_CONST_PAIR(reg4, reg3, cospi_14_64, cospi_18_64, reg4, reg3); + DOTP_CONST_PAIR(reg2, reg5, cospi_22_64, cospi_10_64, reg2, reg5); + DOTP_CONST_PAIR(reg6, reg1, cospi_6_64, cospi_26_64, reg6, reg1); + + vec0 = __lsx_vadd_h(reg0, reg4); + reg0 = __lsx_vsub_h(reg0, reg4); + reg4 = __lsx_vadd_h(reg6, reg2); + reg6 = __lsx_vsub_h(reg6, reg2); + reg2 = __lsx_vadd_h(reg1, reg5); + reg1 = __lsx_vsub_h(reg1, reg5); + reg5 = __lsx_vadd_h(reg7, reg3); + reg7 = __lsx_vsub_h(reg7, reg3); + reg3 = vec0; + + vec1 = reg2; + reg2 = __lsx_vadd_h(reg3, reg4); + reg3 = __lsx_vsub_h(reg3, reg4); + reg4 = __lsx_vsub_h(reg5, vec1); + reg5 = __lsx_vadd_h(reg5, vec1); + + tmp0 = __lsx_vneg_h(reg6); + DOTP_CONST_PAIR(reg7, reg0, cospi_24_64, cospi_8_64, reg0, reg7); + DOTP_CONST_PAIR(tmp0, reg1, cospi_24_64, cospi_8_64, reg6, reg1); + + vec0 = __lsx_vsub_h(reg0, reg6); + reg0 = __lsx_vadd_h(reg0, reg6); + vec1 = __lsx_vsub_h(reg7, reg1); + reg7 = __lsx_vadd_h(reg7, reg1); + + DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, reg6, reg1); + DOTP_CONST_PAIR(reg4, reg3, cospi_16_64, cospi_16_64, reg3, reg4); + + /* Even stage 3 : Dependency on Even stage 1 & Even stage 2 */ + /* Store 8 */ + LSX_BUTTERFLY_4_H(stp0, stp1, reg7, reg5, loc1, loc3, loc2, loc0); + __lsx_vst(loc1, tmp_eve_buf, 0); + __lsx_vst(loc3, tmp_eve_buf, 16); + __lsx_vst(loc2, tmp_eve_buf, 224); + __lsx_vst(loc0, tmp_eve_buf, 240); + + LSX_BUTTERFLY_4_H(stp2, stp3, reg4, reg1, loc1, loc3, loc2, loc0); + __lsx_vst(loc1, tmp_eve_buf, 32); + __lsx_vst(loc3, tmp_eve_buf, 48); + __lsx_vst(loc2, tmp_eve_buf, 192); + __lsx_vst(loc0, tmp_eve_buf, 208); + + /* Store 8 */ + LSX_BUTTERFLY_4_H(stp4, stp5, reg6, reg3, loc1, loc3, loc2, loc0); + __lsx_vst(loc1, tmp_eve_buf, 64); + __lsx_vst(loc3, tmp_eve_buf, 80); + __lsx_vst(loc2, tmp_eve_buf, 160); + __lsx_vst(loc0, tmp_eve_buf, 176); + + LSX_BUTTERFLY_4_H(stp6, stp7, reg2, reg0, loc1, loc3, loc2, loc0); + __lsx_vst(loc1, tmp_eve_buf, 96); + __lsx_vst(loc3, tmp_eve_buf, 112); + __lsx_vst(loc2, tmp_eve_buf, 128); + __lsx_vst(loc0, tmp_eve_buf, 144); +} + +static void idct8x32_column_odd_process_store(int16_t *tmp_buf, + int16_t *tmp_odd_buf) { + __m128i vec0, vec1, vec2, vec3, loc0, loc1, loc2, loc3; + __m128i reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7; + + /* Odd stage 1 */ + DUP4_ARG2(__lsx_vld, tmp_buf, 64, tmp_buf, 448, tmp_buf, 576, tmp_buf, 960, + reg0, reg1, reg2, reg3); + DUP4_ARG2(__lsx_vld, tmp_buf, 1088, tmp_buf, 1472, tmp_buf, 1600, tmp_buf, + 1984, reg4, reg5, reg6, reg7); + + DOTP_CONST_PAIR(reg0, reg7, cospi_31_64, cospi_1_64, reg0, reg7); + DOTP_CONST_PAIR(reg4, reg3, cospi_15_64, cospi_17_64, reg3, reg4); + DOTP_CONST_PAIR(reg2, reg5, cospi_23_64, cospi_9_64, reg2, reg5); + DOTP_CONST_PAIR(reg6, reg1, cospi_7_64, cospi_25_64, reg1, reg6); + + vec0 = __lsx_vadd_h(reg0, reg3); + reg0 = __lsx_vsub_h(reg0, reg3); + reg3 = __lsx_vadd_h(reg7, reg4); + reg7 = __lsx_vsub_h(reg7, reg4); + reg4 = __lsx_vadd_h(reg1, reg2); + reg1 = __lsx_vsub_h(reg1, reg2); + reg2 = __lsx_vadd_h(reg6, reg5); + reg6 = __lsx_vsub_h(reg6, reg5); + reg5 = vec0; + + /* 4 Stores */ + DUP2_ARG2(__lsx_vadd_h, reg5, reg4, reg3, reg2, vec0, vec1); + __lsx_vst(vec0, tmp_odd_buf, 64); + __lsx_vst(vec1, tmp_odd_buf, 80); + DUP2_ARG2(__lsx_vsub_h, reg5, reg4, reg3, reg2, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_24_64, cospi_8_64, vec0, vec1); + __lsx_vst(vec0, tmp_odd_buf, 0); + __lsx_vst(vec1, tmp_odd_buf, 16); + + /* 4 Stores */ + DOTP_CONST_PAIR(reg7, reg0, cospi_28_64, cospi_4_64, reg0, reg7); + DOTP_CONST_PAIR(reg6, reg1, -cospi_4_64, cospi_28_64, reg1, reg6); + LSX_BUTTERFLY_4_H(reg0, reg7, reg6, reg1, vec0, vec1, vec2, vec3); + DOTP_CONST_PAIR(vec2, vec3, cospi_24_64, cospi_8_64, vec2, vec3); + __lsx_vst(vec0, tmp_odd_buf, 96); + __lsx_vst(vec1, tmp_odd_buf, 112); + __lsx_vst(vec2, tmp_odd_buf, 32); + __lsx_vst(vec3, tmp_odd_buf, 48); + + /* Odd stage 2 */ + /* 8 loads */ + DUP4_ARG2(__lsx_vld, tmp_buf, 192, tmp_buf, 320, tmp_buf, 704, tmp_buf, 832, + reg0, reg1, reg2, reg3); + DUP4_ARG2(__lsx_vld, tmp_buf, 1216, tmp_buf, 1344, tmp_buf, 1728, tmp_buf, + 1856, reg4, reg5, reg6, reg7); + DOTP_CONST_PAIR(reg1, reg6, cospi_27_64, cospi_5_64, reg1, reg6); + DOTP_CONST_PAIR(reg5, reg2, cospi_11_64, cospi_21_64, reg2, reg5); + DOTP_CONST_PAIR(reg3, reg4, cospi_19_64, cospi_13_64, reg3, reg4); + DOTP_CONST_PAIR(reg7, reg0, cospi_3_64, cospi_29_64, reg0, reg7); + + /* 4 Stores */ + DUP4_ARG2(__lsx_vsub_h, reg1, reg2, reg6, reg5, reg0, reg3, reg7, reg4, vec0, + vec1, vec2, vec3); + DOTP_CONST_PAIR(vec1, vec0, cospi_12_64, cospi_20_64, loc0, loc1); + DOTP_CONST_PAIR(vec3, vec2, -cospi_20_64, cospi_12_64, loc2, loc3); + LSX_BUTTERFLY_4_H(loc2, loc3, loc1, loc0, vec0, vec1, vec3, vec2); + __lsx_vst(vec0, tmp_odd_buf, 192); + __lsx_vst(vec1, tmp_odd_buf, 240); + DOTP_CONST_PAIR(vec3, vec2, -cospi_8_64, cospi_24_64, vec0, vec1); + __lsx_vst(vec0, tmp_odd_buf, 160); + __lsx_vst(vec1, tmp_odd_buf, 176); + + /* 4 Stores */ + DUP4_ARG2(__lsx_vadd_h, reg0, reg3, reg1, reg2, reg5, reg6, reg4, reg7, vec0, + vec1, vec2, vec3); + LSX_BUTTERFLY_4_H(vec0, vec3, vec2, vec1, reg0, reg1, reg3, reg2); + __lsx_vst(reg0, tmp_odd_buf, 208); + __lsx_vst(reg1, tmp_odd_buf, 224); + DOTP_CONST_PAIR(reg3, reg2, -cospi_8_64, cospi_24_64, reg0, reg1); + __lsx_vst(reg0, tmp_odd_buf, 128); + __lsx_vst(reg1, tmp_odd_buf, 144); + + /* Odd stage 3 : Dependency on Odd stage 1 & Odd stage 2 */ + /* Load 8 & Store 8 */ + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 0, tmp_odd_buf, 16, tmp_odd_buf, 32, + tmp_odd_buf, 48, reg0, reg1, reg2, reg3); + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 128, tmp_odd_buf, 144, tmp_odd_buf, 160, + tmp_odd_buf, 176, reg4, reg5, reg6, reg7); + DUP4_ARG2(__lsx_vadd_h, reg0, reg4, reg1, reg5, reg2, reg6, reg3, reg7, loc0, + loc1, loc2, loc3); + __lsx_vst(loc0, tmp_odd_buf, 0); + __lsx_vst(loc1, tmp_odd_buf, 16); + __lsx_vst(loc2, tmp_odd_buf, 32); + __lsx_vst(loc3, tmp_odd_buf, 48); + + DUP2_ARG2(__lsx_vsub_h, reg0, reg4, reg1, reg5, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc0, loc1); + DUP2_ARG2(__lsx_vsub_h, reg2, reg6, reg3, reg7, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc2, loc3); + __lsx_vst(loc0, tmp_odd_buf, 128); + __lsx_vst(loc1, tmp_odd_buf, 144); + __lsx_vst(loc2, tmp_odd_buf, 160); + __lsx_vst(loc3, tmp_odd_buf, 176); + + /* Load 8 & Store 8 */ + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 64, tmp_odd_buf, 80, tmp_odd_buf, 96, + tmp_odd_buf, 112, reg1, reg2, reg0, reg3); + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 192, tmp_odd_buf, 208, tmp_odd_buf, 224, + tmp_odd_buf, 240, reg4, reg5, reg6, reg7); + DUP4_ARG2(__lsx_vadd_h, reg0, reg4, reg1, reg5, reg2, reg6, reg3, reg7, loc0, + loc1, loc2, loc3); + __lsx_vst(loc0, tmp_odd_buf, 64); + __lsx_vst(loc1, tmp_odd_buf, 80); + __lsx_vst(loc2, tmp_odd_buf, 96); + __lsx_vst(loc3, tmp_odd_buf, 112); + + DUP2_ARG2(__lsx_vsub_h, reg0, reg4, reg3, reg7, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc0, loc1); + DUP2_ARG2(__lsx_vsub_h, reg1, reg5, reg2, reg6, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc2, loc3); + __lsx_vst(loc0, tmp_odd_buf, 192); + __lsx_vst(loc1, tmp_odd_buf, 208); + __lsx_vst(loc2, tmp_odd_buf, 224); + __lsx_vst(loc3, tmp_odd_buf, 240); +} + +static void idct8x32_column_butterfly_addblk(int16_t *tmp_eve_buf, + int16_t *tmp_odd_buf, uint8_t *dst, + int32_t dst_stride) { + __m128i vec0, vec1, vec2, vec3, loc0, loc1, loc2, loc3; + __m128i m0, m1, m2, m3, m4, m5, m6, m7; + __m128i n0, n1, n2, n3, n4, n5, n6, n7; + int32_t stride = dst_stride << 2; + int32_t stride2 = stride << 1; + int32_t stride3 = stride + stride2; + + /* FINAL BUTTERFLY : Dependency on Even & Odd */ + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 0, tmp_odd_buf, 144, tmp_odd_buf, 224, + tmp_odd_buf, 96, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vld, tmp_eve_buf, 0, tmp_eve_buf, 128, tmp_eve_buf, 64, + tmp_eve_buf, 192, loc0, loc1, loc2, loc3); + + DUP4_ARG2(__lsx_vadd_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, m0, + m4, m2, m6); + DUP4_ARG2(__lsx_vsrari_h, m0, 6, m2, 6, m4, 6, m6, 6, m0, m2, m4, m6); + VP9_ADDBLK_ST8x4_UB(dst, stride, stride2, stride3, m0, m2, m4, m6); + DUP4_ARG2(__lsx_vsub_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, m6, + m2, m4, m0); + DUP4_ARG2(__lsx_vsrari_h, m0, 6, m2, 6, m4, 6, m6, 6, m0, m2, m4, m6); + VP9_ADDBLK_ST8x4_UB((dst + 19 * dst_stride), stride, stride2, stride3, m0, m2, + m4, m6); + + /* Load 8 & Store 8 */ + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 64, tmp_odd_buf, 208, tmp_odd_buf, 160, + tmp_odd_buf, 48, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vld, tmp_eve_buf, 32, tmp_eve_buf, 160, tmp_eve_buf, 96, + tmp_eve_buf, 224, loc0, loc1, loc2, loc3); + + DUP4_ARG2(__lsx_vadd_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, m1, + m5, m3, m7); + DUP4_ARG2(__lsx_vsrari_h, m1, 6, m3, 6, m5, 6, m7, 6, m1, m3, m5, m7); + VP9_ADDBLK_ST8x4_UB((dst + 2 * dst_stride), stride, stride2, stride3, m1, m3, + m5, m7); + DUP4_ARG2(__lsx_vsub_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, m7, + m3, m5, m1); + DUP4_ARG2(__lsx_vsrari_h, m1, 6, m3, 6, m5, 6, m7, 6, m1, m3, m5, m7); + VP9_ADDBLK_ST8x4_UB((dst + 17 * dst_stride), stride, stride2, stride3, m1, m3, + m5, m7); + + /* Load 8 & Store 8 */ + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 32, tmp_odd_buf, 176, tmp_odd_buf, 192, + tmp_odd_buf, 112, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vld, tmp_eve_buf, 16, tmp_eve_buf, 144, tmp_eve_buf, 80, + tmp_eve_buf, 208, loc0, loc1, loc2, loc3); + DUP4_ARG2(__lsx_vadd_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, n0, + n4, n2, n6); + DUP4_ARG2(__lsx_vsrari_h, n0, 6, n2, 6, n4, 6, n6, 6, n0, n2, n4, n6); + VP9_ADDBLK_ST8x4_UB((dst + dst_stride), stride, stride2, stride3, n0, n2, n4, + n6); + DUP4_ARG2(__lsx_vsub_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, n6, + n2, n4, n0); + DUP4_ARG2(__lsx_vsrari_h, n0, 6, n2, 6, n4, 6, n6, 6, n0, n2, n4, n6); + VP9_ADDBLK_ST8x4_UB((dst + 18 * dst_stride), stride, stride2, stride3, n0, n2, + n4, n6); + + /* Load 8 & Store 8 */ + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 80, tmp_odd_buf, 240, tmp_odd_buf, 128, + tmp_odd_buf, 16, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vld, tmp_eve_buf, 48, tmp_eve_buf, 176, tmp_eve_buf, 112, + tmp_eve_buf, 240, loc0, loc1, loc2, loc3); + DUP4_ARG2(__lsx_vadd_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, n1, + n5, n3, n7); + DUP4_ARG2(__lsx_vsrari_h, n1, 6, n3, 6, n5, 6, n7, 6, n1, n3, n5, n7); + VP9_ADDBLK_ST8x4_UB((dst + 3 * dst_stride), stride, stride2, stride3, n1, n3, + n5, n7); + DUP4_ARG2(__lsx_vsub_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, n7, + n3, n5, n1); + DUP4_ARG2(__lsx_vsrari_h, n1, 6, n3, 6, n5, 6, n7, 6, n1, n3, n5, n7); + VP9_ADDBLK_ST8x4_UB((dst + 16 * dst_stride), stride, stride2, stride3, n1, n3, + n5, n7); +} + +static void idct8x32_1d_columns_addblk_lsx(int16_t *input, uint8_t *dst, + int32_t dst_stride) { + DECLARE_ALIGNED(32, int16_t, tmp_odd_buf[16 * 8]); + DECLARE_ALIGNED(32, int16_t, tmp_eve_buf[16 * 8]); + + idct8x32_column_even_process_store(input, &tmp_eve_buf[0]); + idct8x32_column_odd_process_store(input, &tmp_odd_buf[0]); + idct8x32_column_butterfly_addblk(&tmp_eve_buf[0], &tmp_odd_buf[0], dst, + dst_stride); +} + +void vpx_idct32x32_1024_add_lsx(const int16_t *input, uint8_t *dst, + int32_t dst_stride) { + int32_t i; + DECLARE_ALIGNED(32, int16_t, out_arr[32 * 32]); + int16_t *out_ptr = out_arr; + + /* transform rows */ + for (i = 0; i < 4; ++i) { + /* process 32 * 8 block */ + idct32x8_1d_rows_lsx((input + (i << 8)), (out_ptr + (i << 8))); + } + + for (i = 0; i < 4; ++i) { + /* process 8 * 32 block */ + idct8x32_1d_columns_addblk_lsx((out_ptr + (i << 3)), (dst + (i << 3)), + dst_stride); + } +} + +void vpx_idct32x32_34_add_lsx(const int16_t *input, uint8_t *dst, + int32_t dst_stride) { + int32_t i; + DECLARE_ALIGNED(32, int16_t, out_arr[32 * 32]); + int16_t *out_ptr = out_arr; + __m128i zero = __lsx_vldi(0); + + for (i = 32; i--;) { + __lsx_vst(zero, out_ptr, 0); + __lsx_vst(zero, out_ptr, 16); + __lsx_vst(zero, out_ptr, 32); + __lsx_vst(zero, out_ptr, 48); + out_ptr += 32; + } + + out_ptr = out_arr; + + /* rows: only upper-left 8x8 has non-zero coeff */ + idct32x8_1d_rows_lsx(input, out_ptr); + + /* transform columns */ + for (i = 0; i < 4; ++i) { + /* process 8 * 32 block */ + idct8x32_1d_columns_addblk_lsx((out_ptr + (i << 3)), (dst + (i << 3)), + dst_stride); + } +} + +void vpx_idct32x32_1_add_lsx(const int16_t *input, uint8_t *dst, + int32_t dst_stride) { + int32_t i; + int16_t out; + __m128i dst0, dst1, dst2, dst3, tmp0, tmp1, tmp2, tmp3; + __m128i res0, res1, res2, res3, res4, res5, res6, res7, vec; + + out = ROUND_POWER_OF_TWO((input[0] * cospi_16_64), DCT_CONST_BITS); + out = ROUND_POWER_OF_TWO((out * cospi_16_64), DCT_CONST_BITS); + out = ROUND_POWER_OF_TWO(out, 6); + + vec = __lsx_vreplgr2vr_h(out); + + for (i = 16; i--;) { + DUP2_ARG2(__lsx_vld, dst, 0, dst, 16, dst0, dst1); + dst2 = __lsx_vldx(dst, dst_stride); + dst3 = __lsx_vldx(dst + 16, dst_stride); + + UNPCK_UB_SH(dst0, res0, res4); + UNPCK_UB_SH(dst1, res1, res5); + UNPCK_UB_SH(dst2, res2, res6); + UNPCK_UB_SH(dst3, res3, res7); + + DUP4_ARG2(__lsx_vadd_h, res0, vec, res1, vec, res2, vec, res3, vec, res0, + res1, res2, res3); + DUP4_ARG2(__lsx_vadd_h, res4, vec, res5, vec, res6, vec, res7, vec, res4, + res5, res6, res7); + DUP4_ARG3(__lsx_vssrarni_bu_h, res4, res0, 0, res5, res1, 0, res6, res2, 0, + res7, res3, 0, tmp0, tmp1, tmp2, tmp3); + __lsx_vst(tmp0, dst, 0); + __lsx_vst(tmp1, dst, 16); + dst += dst_stride; + __lsx_vst(tmp2, dst, 0); + __lsx_vst(tmp3, dst, 16); + dst += dst_stride; + } +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/intrapred_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/intrapred_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/intrapred_lsx.c @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Lu Wang + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "./vpx_dsp_rtcd.h" +#include "vpx_util/loongson_intrinsics.h" + +static inline void intra_predict_dc_8x8_lsx(const uint8_t *src_top, + const uint8_t *src_left, + uint8_t *dst, int32_t dst_stride) { + uint64_t val0, val1; + int32_t dst_stride_x2 = dst_stride << 1; + int32_t dst_stride_x3 = dst_stride_x2 + dst_stride; + int32_t dst_stride_x4 = dst_stride << 2; + __m128i store, sum_h, sum_w, sum_d; + __m128i src = { 0 }; + + val0 = *(const uint64_t *)src_top; + val1 = *(const uint64_t *)src_left; + DUP2_ARG3(__lsx_vinsgr2vr_d, src, val0, 0, src, val1, 1, src, src); + sum_h = __lsx_vhaddw_hu_bu(src, src); + sum_w = __lsx_vhaddw_wu_hu(sum_h, sum_h); + sum_d = __lsx_vhaddw_du_wu(sum_w, sum_w); + sum_w = __lsx_vpickev_w(sum_d, sum_d); + sum_d = __lsx_vhaddw_du_wu(sum_w, sum_w); + sum_w = __lsx_vsrari_w(sum_d, 4); + store = __lsx_vreplvei_b(sum_w, 0); + + __lsx_vstelm_d(store, dst, 0, 0); + __lsx_vstelm_d(store, dst + dst_stride, 0, 0); + __lsx_vstelm_d(store, dst + dst_stride_x2, 0, 0); + __lsx_vstelm_d(store, dst + dst_stride_x3, 0, 0); + dst += dst_stride_x4; + __lsx_vstelm_d(store, dst, 0, 0); + __lsx_vstelm_d(store, dst + dst_stride, 0, 0); + __lsx_vstelm_d(store, dst + dst_stride_x2, 0, 0); + __lsx_vstelm_d(store, dst + dst_stride_x3, 0, 0); +} + +static inline void intra_predict_dc_16x16_lsx(const uint8_t *src_top, + const uint8_t *src_left, + uint8_t *dst, + int32_t dst_stride) { + int32_t dst_stride_x2 = dst_stride << 1; + int32_t dst_stride_x3 = dst_stride_x2 + dst_stride; + int32_t dst_stride_x4 = dst_stride << 2; + __m128i top, left, out; + __m128i sum_h, sum_top, sum_left; + __m128i sum_w; + __m128i sum_d; + + DUP2_ARG2(__lsx_vld, src_top, 0, src_left, 0, top, left); + DUP2_ARG2(__lsx_vhaddw_hu_bu, top, top, left, left, sum_top, sum_left); + sum_h = __lsx_vadd_h(sum_top, sum_left); + sum_w = __lsx_vhaddw_wu_hu(sum_h, sum_h); + sum_d = __lsx_vhaddw_du_wu(sum_w, sum_w); + sum_w = __lsx_vpickev_w(sum_d, sum_d); + sum_d = __lsx_vhaddw_du_wu(sum_w, sum_w); + sum_w = __lsx_vsrari_w(sum_d, 5); + out = __lsx_vreplvei_b(sum_w, 0); + + __lsx_vstx(out, dst, 0); + __lsx_vstx(out, dst, dst_stride); + __lsx_vstx(out, dst, dst_stride_x2); + __lsx_vstx(out, dst, dst_stride_x3); + dst += dst_stride_x4; + __lsx_vstx(out, dst, 0); + __lsx_vstx(out, dst, dst_stride); + __lsx_vstx(out, dst, dst_stride_x2); + __lsx_vstx(out, dst, dst_stride_x3); + dst += dst_stride_x4; + __lsx_vstx(out, dst, 0); + __lsx_vstx(out, dst, dst_stride); + __lsx_vstx(out, dst, dst_stride_x2); + __lsx_vstx(out, dst, dst_stride_x3); + dst += dst_stride_x4; + __lsx_vstx(out, dst, 0); + __lsx_vstx(out, dst, dst_stride); + __lsx_vstx(out, dst, dst_stride_x2); + __lsx_vstx(out, dst, dst_stride_x3); +} + +void vpx_dc_predictor_8x8_lsx(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left) { + intra_predict_dc_8x8_lsx(above, left, dst, y_stride); +} + +void vpx_dc_predictor_16x16_lsx(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left) { + intra_predict_dc_16x16_lsx(above, left, dst, y_stride); +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/loopfilter_16_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/loopfilter_16_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/loopfilter_16_lsx.c @@ -0,0 +1,1330 @@ +/* + * Copyright (c) 2022 Loongson Technology Corporation Limited + * Contributed by Hecai Yuan + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "./vpx_dsp_rtcd.h" +#include "vpx_dsp/loongarch/loopfilter_lsx.h" +#include "vpx_ports/mem.h" + +#define LSX_LD_8(_src, _stride, _stride2, _stride3, _stride4, _in0, _in1, \ + _in2, _in3, _in4, _in5, _in6, _in7) \ + { \ + _in0 = __lsx_vld(_src, 0); \ + _in1 = __lsx_vldx(_src, _stride); \ + _in2 = __lsx_vldx(_src, _stride2); \ + _in3 = __lsx_vldx(_src, _stride3); \ + _src += _stride4; \ + _in4 = __lsx_vld(_src, 0); \ + _in5 = __lsx_vldx(_src, _stride); \ + _in6 = __lsx_vldx(_src, _stride2); \ + _in7 = __lsx_vldx(_src, _stride3); \ + } + +#define LSX_ST_8(_dst0, _dst1, _dst2, _dst3, _dst4, _dst5, _dst6, _dst7, _dst, \ + _stride, _stride2, _stride3, _stride4) \ + { \ + __lsx_vst(_dst0, _dst, 0); \ + __lsx_vstx(_dst1, _dst, _stride); \ + __lsx_vstx(_dst2, _dst, _stride2); \ + __lsx_vstx(_dst3, _dst, _stride3); \ + _dst += _stride4; \ + __lsx_vst(_dst4, _dst, 0); \ + __lsx_vstx(_dst5, _dst, _stride); \ + __lsx_vstx(_dst6, _dst, _stride2); \ + __lsx_vstx(_dst7, _dst, _stride3); \ + } + +static int32_t hz_lpf_t4_and_t8_16w(uint8_t *dst, int32_t stride, + uint8_t *filter48, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr) { + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i p2_out, p1_out, p0_out, q0_out, q1_out, q2_out; + __m128i flat, mask, hev, thresh, b_limit, limit; + __m128i p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l; + __m128i p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h; + __m128i p2_filt8_l, p1_filt8_l, p0_filt8_l; + __m128i q0_filt8_l, q1_filt8_l, q2_filt8_l; + __m128i p2_filt8_h, p1_filt8_h, p0_filt8_h; + __m128i q0_filt8_h, q1_filt8_h, q2_filt8_h; + __m128i zero = __lsx_vldi(0); + + int32_t stride2 = stride << 1; + int32_t stride3 = stride2 + stride; + int32_t stride4 = stride2 << 1; + + /* load vector elements */ + DUP4_ARG2(__lsx_vldx, dst, -stride4, dst, -stride3, dst, -stride2, dst, + -stride, p3, p2, p1, p0); + + q0 = __lsx_vld(dst, 0); + DUP2_ARG2(__lsx_vldx, dst, stride, dst, stride2, q1, q2); + q3 = __lsx_vldx(dst, stride3); + + thresh = __lsx_vreplgr2vr_b(*thresh_ptr); + b_limit = __lsx_vreplgr2vr_b(*b_limit_ptr); + limit = __lsx_vreplgr2vr_b(*limit_ptr); + /* mask and hev */ + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + VP9_FLAT4(p3, p2, p0, q0, q2, q3, flat); + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, q1_out); + + if (__lsx_bz_v(flat)) { + __lsx_vstx(p1_out, dst, -stride2); + __lsx_vstx(p0_out, dst, -stride); + __lsx_vst(q0_out, dst, 0); + __lsx_vstx(q1_out, dst, stride); + + return 1; + } + + DUP4_ARG2(__lsx_vilvl_b, zero, p3, zero, p2, zero, p1, zero, p0, p3_l, p2_l, + p1_l, p0_l); + DUP4_ARG2(__lsx_vilvl_b, zero, q0, zero, q1, zero, q2, zero, q3, q0_l, q1_l, + q2_l, q3_l); + + VP9_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l, q1_filt8_l, q2_filt8_l); + + DUP4_ARG2(__lsx_vilvh_b, zero, p3, zero, p2, zero, p1, zero, p0, p3_h, p2_h, + p1_h, p0_h); + DUP4_ARG2(__lsx_vilvh_b, zero, q0, zero, q1, zero, q2, zero, q3, q0_h, q1_h, + q2_h, q3_h); + VP9_FILTER8(p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h, p2_filt8_h, + p1_filt8_h, p0_filt8_h, q0_filt8_h, q1_filt8_h, q2_filt8_h); + + /* convert 16 bit output data into 8 bit */ + DUP4_ARG2(__lsx_vpickev_b, p2_filt8_h, p2_filt8_l, p1_filt8_h, p1_filt8_l, + p0_filt8_h, p0_filt8_l, q0_filt8_h, q0_filt8_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l); + DUP2_ARG2(__lsx_vpickev_b, q1_filt8_h, q1_filt8_l, q2_filt8_h, q2_filt8_l, + q1_filt8_l, q2_filt8_l); + + /* store pixel values */ + DUP4_ARG3(__lsx_vbitsel_v, p2, p2_filt8_l, flat, p1_out, p1_filt8_l, flat, + p0_out, p0_filt8_l, flat, q0_out, q0_filt8_l, flat, p2_out, p1_out, + p0_out, q0_out); + DUP2_ARG3(__lsx_vbitsel_v, q1_out, q1_filt8_l, flat, q2, q2_filt8_l, flat, + q1_out, q2_out); + + __lsx_vst(p2_out, filter48, 0); + __lsx_vst(p1_out, filter48, 16); + __lsx_vst(p0_out, filter48, 32); + __lsx_vst(q0_out, filter48, 48); + __lsx_vst(q1_out, filter48, 64); + __lsx_vst(q2_out, filter48, 80); + __lsx_vst(flat, filter48, 96); + + return 0; +} + +static void hz_lpf_t16_16w(uint8_t *dst, int32_t stride, uint8_t *filter48) { + int32_t stride2 = stride << 1; + int32_t stride3 = stride2 + stride; + int32_t stride4 = stride2 << 1; + uint8_t *dst_tmp0 = dst - stride4; + uint8_t *dst_tmp1 = dst + stride4; + + __m128i flat, flat2, filter8; + __m128i zero = __lsx_vldi(0); + __m128i p7, p6, p5, p4, p3, p2, p1, p0, q0, q1, q2, q3, q4, q5, q6, q7; + __m128i out_h, out_l; + v8u16 p7_l_in, p6_l_in, p5_l_in, p4_l_in; + v8u16 p3_l_in, p2_l_in, p1_l_in, p0_l_in; + v8u16 q7_l_in, q6_l_in, q5_l_in, q4_l_in; + v8u16 q3_l_in, q2_l_in, q1_l_in, q0_l_in; + v8u16 p7_h_in, p6_h_in, p5_h_in, p4_h_in; + v8u16 p3_h_in, p2_h_in, p1_h_in, p0_h_in; + v8u16 q7_h_in, q6_h_in, q5_h_in, q4_h_in; + v8u16 q3_h_in, q2_h_in, q1_h_in, q0_h_in; + v8u16 tmp0_l, tmp1_l, tmp0_h, tmp1_h; + + flat = __lsx_vld(filter48, 96); + + DUP4_ARG2(__lsx_vldx, dst_tmp0, -stride4, dst_tmp0, -stride3, dst_tmp0, + -stride2, dst_tmp0, -stride, p7, p6, p5, p4); + + p3 = __lsx_vld(dst_tmp0, 0); + DUP2_ARG2(__lsx_vldx, dst_tmp0, stride, dst_tmp0, stride2, p2, p1); + p0 = __lsx_vldx(dst_tmp0, stride3); + + q0 = __lsx_vld(dst, 0); + DUP2_ARG2(__lsx_vldx, dst, stride, dst, stride2, q1, q2); + q3 = __lsx_vldx(dst, stride3); + + q4 = __lsx_vld(dst_tmp1, 0); + DUP2_ARG2(__lsx_vldx, dst_tmp1, stride, dst_tmp1, stride2, q5, q6); + q7 = __lsx_vldx(dst_tmp1, stride3); + + VP9_FLAT5(p7, p6, p5, p4, p0, q0, q4, q5, q6, q7, flat, flat2); + + if (__lsx_bz_v(flat2)) { + DUP4_ARG2(__lsx_vld, filter48, 0, filter48, 16, filter48, 32, filter48, 48, + p2, p1, p0, q0); + DUP2_ARG2(__lsx_vld, filter48, 64, filter48, 80, q1, q2); + __lsx_vstx(p2, dst, -stride3); + __lsx_vstx(p1, dst, -stride2); + __lsx_vstx(p0, dst, -stride); + __lsx_vst(q0, dst, 0); + __lsx_vstx(q1, dst, stride); + __lsx_vstx(q2, dst, stride2); + } else { + dst = dst_tmp0 - stride3; + + p7_l_in = (v8u16)__lsx_vilvl_b(zero, p7); + p6_l_in = (v8u16)__lsx_vilvl_b(zero, p6); + p5_l_in = (v8u16)__lsx_vilvl_b(zero, p5); + p4_l_in = (v8u16)__lsx_vilvl_b(zero, p4); + p3_l_in = (v8u16)__lsx_vilvl_b(zero, p3); + p2_l_in = (v8u16)__lsx_vilvl_b(zero, p2); + p1_l_in = (v8u16)__lsx_vilvl_b(zero, p1); + p0_l_in = (v8u16)__lsx_vilvl_b(zero, p0); + + q0_l_in = (v8u16)__lsx_vilvl_b(zero, q0); + + tmp0_l = p7_l_in << 3; + tmp0_l -= p7_l_in; + tmp0_l += p6_l_in; + tmp0_l += q0_l_in; + tmp1_l = p6_l_in + p5_l_in; + tmp1_l += p4_l_in; + tmp1_l += p3_l_in; + tmp1_l += p2_l_in; + tmp1_l += p1_l_in; + tmp1_l += p0_l_in; + tmp1_l += tmp0_l; + + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + p7_h_in = (v8u16)__lsx_vilvh_b(zero, p7); + p6_h_in = (v8u16)__lsx_vilvh_b(zero, p6); + p5_h_in = (v8u16)__lsx_vilvh_b(zero, p5); + p4_h_in = (v8u16)__lsx_vilvh_b(zero, p4); + + p3_h_in = (v8u16)__lsx_vilvh_b(zero, p3); + p2_h_in = (v8u16)__lsx_vilvh_b(zero, p2); + p1_h_in = (v8u16)__lsx_vilvh_b(zero, p1); + p0_h_in = (v8u16)__lsx_vilvh_b(zero, p0); + q0_h_in = (v8u16)__lsx_vilvh_b(zero, q0); + + tmp0_h = p7_h_in << 3; + tmp0_h -= p7_h_in; + tmp0_h += p6_h_in; + tmp0_h += q0_h_in; + tmp1_h = p6_h_in + p5_h_in; + tmp1_h += p4_h_in; + tmp1_h += p3_h_in; + tmp1_h += p2_h_in; + tmp1_h += p1_h_in; + tmp1_h += p0_h_in; + tmp1_h += tmp0_h; + + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + p6 = __lsx_vbitsel_v(p6, out_l, flat2); + __lsx_vst(p6, dst, 0); + dst += stride; + + /* p5 */ + q1_l_in = (v8u16)__lsx_vilvl_b(zero, q1); + tmp0_l = p5_l_in - p6_l_in; + tmp0_l += q1_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + q1_h_in = (v8u16)__lsx_vilvh_b(zero, q1); + tmp0_h = p5_h_in - p6_h_in; + tmp0_h += q1_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + p5 = __lsx_vbitsel_v(p5, out_l, flat2); + __lsx_vst(p5, dst, 0); + dst += stride; + + /* p4 */ + q2_l_in = (v8u16)__lsx_vilvl_b(zero, q2); + tmp0_l = p4_l_in - p5_l_in; + tmp0_l += q2_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + q2_h_in = (v8u16)__lsx_vilvh_b(zero, q2); + tmp0_h = p4_h_in - p5_h_in; + tmp0_h += q2_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + p4 = __lsx_vbitsel_v(p4, out_l, flat2); + __lsx_vst(p4, dst, 0); + dst += stride; + + /* p3 */ + q3_l_in = (v8u16)__lsx_vilvl_b(zero, q3); + tmp0_l = p3_l_in - p4_l_in; + tmp0_l += q3_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + q3_h_in = (v8u16)__lsx_vilvh_b(zero, q3); + tmp0_h = p3_h_in - p4_h_in; + tmp0_h += q3_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + p3 = __lsx_vbitsel_v(p3, out_l, flat2); + __lsx_vst(p3, dst, 0); + dst += stride; + + /* p2 */ + q4_l_in = (v8u16)__lsx_vilvl_b(zero, q4); + filter8 = __lsx_vld(filter48, 0); + tmp0_l = p2_l_in - p3_l_in; + tmp0_l += q4_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + q4_h_in = (v8u16)__lsx_vilvh_b(zero, q4); + tmp0_h = p2_h_in - p3_h_in; + tmp0_h += q4_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 0); + dst += stride; + + /* p1 */ + q5_l_in = (v8u16)__lsx_vilvl_b(zero, q5); + filter8 = __lsx_vld(filter48, 16); + tmp0_l = p1_l_in - p2_l_in; + tmp0_l += q5_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + q5_h_in = (v8u16)__lsx_vilvh_b(zero, q5); + tmp0_h = p1_h_in - p2_h_in; + tmp0_h += q5_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 0); + dst += stride; + + /* p0 */ + q6_l_in = (v8u16)__lsx_vilvl_b(zero, q6); + filter8 = __lsx_vld(filter48, 32); + tmp0_l = p0_l_in - p1_l_in; + tmp0_l += q6_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + q6_h_in = (v8u16)__lsx_vilvh_b(zero, q6); + tmp0_h = p0_h_in - p1_h_in; + tmp0_h += q6_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 0); + dst += stride; + + /* q0 */ + q7_l_in = (v8u16)__lsx_vilvl_b(zero, q7); + filter8 = __lsx_vld(filter48, 48); + tmp0_l = q7_l_in - p0_l_in; + tmp0_l += q0_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + q7_h_in = (v8u16)__lsx_vilvh_b(zero, q7); + tmp0_h = q7_h_in - p0_h_in; + tmp0_h += q0_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 0); + dst += stride; + + /* q1 */ + filter8 = __lsx_vld(filter48, 64); + tmp0_l = q7_l_in - q0_l_in; + tmp0_l += q1_l_in; + tmp0_l -= p6_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + tmp0_h = q7_h_in - q0_h_in; + tmp0_h += q1_h_in; + tmp0_h -= p6_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 0); + dst += stride; + + /* q2 */ + filter8 = __lsx_vld(filter48, 80); + tmp0_l = q7_l_in - q1_l_in; + tmp0_l += q2_l_in; + tmp0_l -= p5_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + tmp0_h = q7_h_in - q1_h_in; + tmp0_h += q2_h_in; + tmp0_h -= p5_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 0); + dst += stride; + + /* q3 */ + tmp0_l = q7_l_in - q2_l_in; + tmp0_l += q3_l_in; + tmp0_l -= p4_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + tmp0_h = q7_h_in - q2_h_in; + tmp0_h += q3_h_in; + tmp0_h -= p4_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + q3 = __lsx_vbitsel_v(q3, out_l, flat2); + __lsx_vst(q3, dst, 0); + dst += stride; + + /* q4 */ + tmp0_l = q7_l_in - q3_l_in; + tmp0_l += q4_l_in; + tmp0_l -= p3_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + tmp0_h = q7_h_in - q3_h_in; + tmp0_h += q4_h_in; + tmp0_h -= p3_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + q4 = __lsx_vbitsel_v(q4, out_l, flat2); + __lsx_vst(q4, dst, 0); + dst += stride; + + /* q5 */ + tmp0_l = q7_l_in - q4_l_in; + tmp0_l += q5_l_in; + tmp0_l -= p2_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + tmp0_h = q7_h_in - q4_h_in; + tmp0_h += q5_h_in; + tmp0_h -= p2_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + q5 = __lsx_vbitsel_v(q5, out_l, flat2); + __lsx_vst(q5, dst, 0); + dst += stride; + + /* q6 */ + tmp0_l = q7_l_in - q5_l_in; + tmp0_l += q6_l_in; + tmp0_l -= p1_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + tmp0_h = q7_h_in - q5_h_in; + tmp0_h += q6_h_in; + tmp0_h -= p1_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + q6 = __lsx_vbitsel_v(q6, out_l, flat2); + __lsx_vst(q6, dst, 0); + } +} + +static void mb_lpf_horizontal_edge_dual(uint8_t *dst, int32_t stride, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr) { + DECLARE_ALIGNED(16, uint8_t, filter48[16 * 8]); + uint8_t early_exit = 0; + + early_exit = hz_lpf_t4_and_t8_16w(dst, stride, &filter48[0], b_limit_ptr, + limit_ptr, thresh_ptr); + + if (early_exit == 0) { + hz_lpf_t16_16w(dst, stride, filter48); + } +} + +static void mb_lpf_horizontal_edge(uint8_t *dst, int32_t stride, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr, int32_t count) { + if (count == 1) { + __m128i flat2, mask, hev, flat, thresh, b_limit, limit; + __m128i p3, p2, p1, p0, q3, q2, q1, q0, p7, p6, p5, p4, q4, q5, q6, q7; + __m128i p2_out, p1_out, p0_out, q0_out, q1_out, q2_out; + __m128i p0_filter16, p1_filter16; + __m128i p2_filter8, p1_filter8, p0_filter8; + __m128i q0_filter8, q1_filter8, q2_filter8; + __m128i p7_l, p6_l, p5_l, p4_l, q7_l, q6_l, q5_l, q4_l; + __m128i p3_l, p2_l, p1_l, p0_l, q3_l, q2_l, q1_l, q0_l; + __m128i zero = __lsx_vldi(0); + __m128i tmp0, tmp1, tmp2; + + int32_t stride2 = stride << 1; + int32_t stride3 = 2 + stride; + int32_t stride4 = stride << 2; + uint8_t *dst_tmp0 = dst - stride4; + uint8_t *dst_tmp1 = dst + stride4; + + /* load vector elements */ + DUP4_ARG2(__lsx_vldx, dst, -stride4, dst, -stride3, dst, -stride2, dst, + -stride, p3, p2, p1, p0); + q0 = __lsx_vld(dst, 0); + DUP2_ARG2(__lsx_vldx, dst, stride, dst, stride2, q1, q2); + q3 = __lsx_vldx(dst, stride3); + + thresh = __lsx_vreplgr2vr_b(*thresh_ptr); + b_limit = __lsx_vreplgr2vr_b(*b_limit_ptr); + limit = __lsx_vreplgr2vr_b(*limit_ptr); + + /* filter_mask* */ + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + VP9_FLAT4(p3, p2, p0, q0, q2, q3, flat); + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, + q1_out); + flat = __lsx_vilvl_d(zero, flat); + if (__lsx_bz_v(flat)) { + __lsx_vstelm_d(p1_out, dst - stride2, 0, 0); + __lsx_vstelm_d(p0_out, dst - stride, 0, 0); + __lsx_vstelm_d(q0_out, dst, 0, 0); + __lsx_vstelm_d(q1_out, dst + stride, 0, 0); + } else { + /* convert 8 bit input data into 16 bit */ + DUP4_ARG2(__lsx_vilvl_b, zero, p3, zero, p2, zero, p1, zero, p0, p3_l, + p2_l, p1_l, p0_l); + DUP4_ARG2(__lsx_vilvl_b, zero, q0, zero, q1, zero, q2, zero, q3, q0_l, + q1_l, q2_l, q3_l); + VP9_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filter8, + p1_filter8, p0_filter8, q0_filter8, q1_filter8, q2_filter8); + + /* convert 16 bit output data into 8 bit */ + DUP4_ARG2(__lsx_vpickev_b, zero, p2_filter8, zero, p1_filter8, zero, + p0_filter8, zero, q0_filter8, p2_filter8, p1_filter8, + p0_filter8, q0_filter8); + DUP2_ARG2(__lsx_vpickev_b, zero, q1_filter8, zero, q2_filter8, q1_filter8, + q2_filter8); + + /* store pixel values */ + p2_out = __lsx_vbitsel_v(p2, p2_filter8, flat); + p1_out = __lsx_vbitsel_v(p1_out, p1_filter8, flat); + p0_out = __lsx_vbitsel_v(p0_out, p0_filter8, flat); + q0_out = __lsx_vbitsel_v(q0_out, q0_filter8, flat); + q1_out = __lsx_vbitsel_v(q1_out, q1_filter8, flat); + q2_out = __lsx_vbitsel_v(q2, q2_filter8, flat); + + /* load 16 vector elements */ + DUP4_ARG2(__lsx_vldx, dst_tmp0, -stride4, dst_tmp0, -stride3, dst_tmp0, + -stride2, dst_tmp0, -stride, p7, p6, p5, p4); + q4 = __lsx_vld(dst_tmp1, 0); + DUP2_ARG2(__lsx_vldx, dst_tmp1, stride, dst_tmp1, stride2, q5, q6); + q7 = __lsx_vldx(dst_tmp1, stride3); + + VP9_FLAT5(p7, p6, p5, p4, p0, q0, q4, q5, q6, q7, flat, flat2); + + if (__lsx_bz_v(flat2)) { + dst -= stride3; + __lsx_vstelm_d(p2_out, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(p1_out, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(p0_out, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(q0_out, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(q1_out, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(q2_out, dst, 0, 0); + } else { + /* LSB(right) 8 pixel operation */ + DUP4_ARG2(__lsx_vilvl_b, zero, p7, zero, p6, zero, p5, zero, p4, p7_l, + p6_l, p5_l, p4_l); + DUP4_ARG2(__lsx_vilvl_b, zero, q4, zero, q5, zero, q6, zero, q7, q4_l, + q5_l, q6_l, q7_l); + + tmp0 = __lsx_vslli_h(p7_l, 3); + tmp0 = __lsx_vsub_h(tmp0, p7_l); + tmp0 = __lsx_vadd_h(tmp0, p6_l); + tmp0 = __lsx_vadd_h(tmp0, q0_l); + + dst = dst_tmp0 - stride3; + + /* calculation of p6 and p5 */ + tmp1 = __lsx_vadd_h(p6_l, p5_l); + tmp1 = __lsx_vadd_h(tmp1, p4_l); + tmp1 = __lsx_vadd_h(tmp1, p3_l); + tmp1 = __lsx_vadd_h(tmp1, p2_l); + tmp1 = __lsx_vadd_h(tmp1, p1_l); + tmp1 = __lsx_vadd_h(tmp1, p0_l); + tmp1 = __lsx_vadd_h(tmp1, tmp0); + p0_filter16 = __lsx_vsrari_h(tmp1, 4); + tmp0 = __lsx_vsub_h(p5_l, p6_l); + tmp0 = __lsx_vadd_h(tmp0, q1_l); + tmp0 = __lsx_vsub_h(tmp0, p7_l); + tmp1 = __lsx_vadd_h(tmp1, tmp0); + p1_filter16 = __lsx_vsrari_h(tmp1, 4); + DUP2_ARG2(__lsx_vpickev_b, zero, p0_filter16, zero, p1_filter16, + p0_filter16, p1_filter16); + DUP2_ARG3(__lsx_vbitsel_v, p6, p0_filter16, flat2, p5, p1_filter16, + flat2, p0_filter16, p1_filter16); + __lsx_vstelm_d(p0_filter16, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(p1_filter16, dst, 0, 0); + dst += stride; + + /* calculation of p4 and p3 */ + tmp0 = __lsx_vsub_h(p4_l, p5_l); + tmp0 = __lsx_vadd_h(tmp0, q2_l); + tmp0 = __lsx_vsub_h(tmp0, p7_l); + tmp2 = __lsx_vsub_h(p3_l, p4_l); + tmp2 = __lsx_vadd_h(tmp2, q3_l); + tmp2 = __lsx_vsub_h(tmp2, p7_l); + tmp1 = __lsx_vadd_h(tmp1, tmp0); + p0_filter16 = __lsx_vsrari_h(tmp1, 4); + tmp1 = __lsx_vadd_h(tmp1, tmp2); + p1_filter16 = __lsx_vsrari_h(tmp1, 4); + DUP2_ARG2(__lsx_vpickev_b, zero, p0_filter16, zero, p1_filter16, + p0_filter16, p1_filter16); + DUP2_ARG3(__lsx_vbitsel_v, p4, p0_filter16, flat2, p3, p1_filter16, + flat2, p0_filter16, p1_filter16); + __lsx_vstelm_d(p0_filter16, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(p1_filter16, dst, 0, 0); + dst += stride; + + /* calculation of p2 and p1 */ + tmp0 = __lsx_vsub_h(p2_l, p3_l); + tmp0 = __lsx_vadd_h(tmp0, q4_l); + tmp0 = __lsx_vsub_h(tmp0, p7_l); + tmp2 = __lsx_vsub_h(p1_l, p2_l); + tmp2 = __lsx_vadd_h(tmp2, q5_l); + tmp2 = __lsx_vsub_h(tmp2, p7_l); + tmp1 = __lsx_vadd_h(tmp1, tmp0); + p0_filter16 = __lsx_vsrari_h(tmp1, 4); + tmp1 = __lsx_vadd_h(tmp1, tmp2); + p1_filter16 = __lsx_vsrari_h(tmp1, 4); + DUP2_ARG2(__lsx_vpickev_b, zero, p0_filter16, zero, p1_filter16, + p0_filter16, p1_filter16); + DUP2_ARG3(__lsx_vbitsel_v, p2_out, p0_filter16, flat2, p1_out, + p1_filter16, flat2, p0_filter16, p1_filter16); + __lsx_vstelm_d(p0_filter16, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(p1_filter16, dst, 0, 0); + dst += stride; + + /* calculation of p0 and q0 */ + tmp0 = __lsx_vsub_h(p0_l, p1_l); + tmp0 = __lsx_vadd_h(tmp0, q6_l); + tmp0 = __lsx_vsub_h(tmp0, p7_l); + tmp2 = __lsx_vsub_h(q7_l, p0_l); + tmp2 = __lsx_vadd_h(tmp2, q0_l); + tmp2 = __lsx_vsub_h(tmp2, p7_l); + tmp1 = __lsx_vadd_h(tmp1, tmp0); + p0_filter16 = __lsx_vsrari_h(tmp1, 4); + tmp1 = __lsx_vadd_h(tmp1, tmp2); + p1_filter16 = __lsx_vsrari_h(tmp1, 4); + DUP2_ARG2(__lsx_vpickev_b, zero, p0_filter16, zero, p1_filter16, + p0_filter16, p1_filter16); + DUP2_ARG3(__lsx_vbitsel_v, p0_out, p0_filter16, flat2, q0_out, + p1_filter16, flat2, p0_filter16, p1_filter16); + __lsx_vstelm_d(p0_filter16, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(p1_filter16, dst, 0, 0); + dst += stride; + + /* calculation of q1 and q2 */ + tmp0 = __lsx_vsub_h(q7_l, q0_l); + tmp0 = __lsx_vadd_h(tmp0, q1_l); + tmp0 = __lsx_vsub_h(tmp0, p6_l); + tmp2 = __lsx_vsub_h(q7_l, q1_l); + tmp2 = __lsx_vadd_h(tmp2, q2_l); + tmp2 = __lsx_vsub_h(tmp2, p5_l); + tmp1 = __lsx_vadd_h(tmp1, tmp0); + p0_filter16 = __lsx_vsrari_h(tmp1, 4); + tmp1 = __lsx_vadd_h(tmp1, tmp2); + p1_filter16 = __lsx_vsrari_h(tmp1, 4); + DUP2_ARG2(__lsx_vpickev_b, zero, p0_filter16, zero, p1_filter16, + p0_filter16, p1_filter16); + DUP2_ARG3(__lsx_vbitsel_v, q1_out, p0_filter16, flat2, q2_out, + p1_filter16, flat2, p0_filter16, p1_filter16); + __lsx_vstelm_d(p0_filter16, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(p1_filter16, dst, 0, 0); + dst += stride; + + /* calculation of q3 and q4 */ + tmp0 = __lsx_vsub_h(q7_l, q2_l); + tmp0 = __lsx_vadd_h(tmp0, q3_l); + tmp0 = __lsx_vsub_h(tmp0, p4_l); + tmp2 = __lsx_vsub_h(q7_l, q3_l); + tmp2 = __lsx_vadd_h(tmp2, q4_l); + tmp2 = __lsx_vsub_h(tmp2, p3_l); + tmp1 = __lsx_vadd_h(tmp1, tmp0); + p0_filter16 = __lsx_vsrari_h(tmp1, 4); + tmp1 = __lsx_vadd_h(tmp1, tmp2); + p1_filter16 = __lsx_vsrari_h(tmp1, 4); + DUP2_ARG2(__lsx_vpickev_b, zero, p0_filter16, zero, p1_filter16, + p0_filter16, p1_filter16); + DUP2_ARG3(__lsx_vbitsel_v, q3, p0_filter16, flat2, q4, p1_filter16, + flat2, p0_filter16, p1_filter16); + __lsx_vstelm_d(p0_filter16, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(p1_filter16, dst, 0, 0); + dst += stride; + + /* calculation of q5 and q6 */ + tmp0 = __lsx_vsub_h(q7_l, q4_l); + tmp0 = __lsx_vadd_h(tmp0, q5_l); + tmp0 = __lsx_vsub_h(tmp0, p2_l); + tmp2 = __lsx_vsub_h(q7_l, q5_l); + tmp2 = __lsx_vadd_h(tmp2, q6_l); + tmp2 = __lsx_vsub_h(tmp2, p1_l); + tmp1 = __lsx_vadd_h(tmp1, tmp0); + p0_filter16 = __lsx_vsrari_h(tmp1, 4); + tmp1 = __lsx_vadd_h(tmp1, tmp2); + p1_filter16 = __lsx_vsrari_h(tmp1, 4); + DUP2_ARG2(__lsx_vpickev_b, zero, p0_filter16, zero, p1_filter16, + p0_filter16, p1_filter16); + DUP2_ARG3(__lsx_vbitsel_v, q5, p0_filter16, flat2, q6, p1_filter16, + flat2, p0_filter16, p1_filter16); + __lsx_vstelm_d(p0_filter16, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(p1_filter16, dst, 0, 0); + } + } + } else { + mb_lpf_horizontal_edge_dual(dst, stride, b_limit_ptr, limit_ptr, + thresh_ptr); + } +} + +void vpx_lpf_horizontal_16_dual_lsx(uint8_t *dst, int32_t stride, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr) { + mb_lpf_horizontal_edge(dst, stride, b_limit_ptr, limit_ptr, thresh_ptr, 2); +} + +static void transpose_16x16(uint8_t *input, int32_t in_stride, uint8_t *output, + int32_t out_stride) { + __m128i row0, row1, row2, row3, row4, row5, row6, row7; + __m128i row8, row9, row10, row11, row12, row13, row14, row15; + __m128i tmp0, tmp1, tmp4, tmp5, tmp6, tmp7; + __m128i tmp2, tmp3; + __m128i p7, p6, p5, p4, p3, p2, p1, p0, q0, q1, q2, q3, q4, q5, q6, q7; + int32_t in_stride2 = in_stride << 1; + int32_t in_stride3 = in_stride2 + in_stride; + int32_t in_stride4 = in_stride2 << 1; + int32_t out_stride2 = out_stride << 1; + int32_t out_stride3 = out_stride2 + out_stride; + int32_t out_stride4 = out_stride2 << 1; + + LSX_LD_8(input, in_stride, in_stride2, in_stride3, in_stride4, row0, row1, + row2, row3, row4, row5, row6, row7); + input += in_stride4; + LSX_LD_8(input, in_stride, in_stride2, in_stride3, in_stride4, row8, row9, + row10, row11, row12, row13, row14, row15); + + LSX_TRANSPOSE16x8_B(row0, row1, row2, row3, row4, row5, row6, row7, row8, + row9, row10, row11, row12, row13, row14, row15, p7, p6, + p5, p4, p3, p2, p1, p0); + + /* transpose 16x8 matrix into 8x16 */ + /* total 8 intermediate register and 32 instructions */ + q7 = __lsx_vpackod_d(row8, row0); + q6 = __lsx_vpackod_d(row9, row1); + q5 = __lsx_vpackod_d(row10, row2); + q4 = __lsx_vpackod_d(row11, row3); + q3 = __lsx_vpackod_d(row12, row4); + q2 = __lsx_vpackod_d(row13, row5); + q1 = __lsx_vpackod_d(row14, row6); + q0 = __lsx_vpackod_d(row15, row7); + + DUP2_ARG2(__lsx_vpackev_b, q6, q7, q4, q5, tmp0, tmp1); + DUP2_ARG2(__lsx_vpackod_b, q6, q7, q4, q5, tmp4, tmp5); + + DUP2_ARG2(__lsx_vpackev_b, q2, q3, q0, q1, q5, q7); + DUP2_ARG2(__lsx_vpackod_b, q2, q3, q0, q1, tmp6, tmp7); + + DUP2_ARG2(__lsx_vpackev_h, tmp1, tmp0, q7, q5, tmp2, tmp3); + q0 = __lsx_vpackev_w(tmp3, tmp2); + q4 = __lsx_vpackod_w(tmp3, tmp2); + + tmp2 = __lsx_vpackod_h(tmp1, tmp0); + tmp3 = __lsx_vpackod_h(q7, q5); + q2 = __lsx_vpackev_w(tmp3, tmp2); + q6 = __lsx_vpackod_w(tmp3, tmp2); + + DUP2_ARG2(__lsx_vpackev_h, tmp5, tmp4, tmp7, tmp6, tmp2, tmp3); + q1 = __lsx_vpackev_w(tmp3, tmp2); + q5 = __lsx_vpackod_w(tmp3, tmp2); + + tmp2 = __lsx_vpackod_h(tmp5, tmp4); + tmp3 = __lsx_vpackod_h(tmp7, tmp6); + q3 = __lsx_vpackev_w(tmp3, tmp2); + q7 = __lsx_vpackod_w(tmp3, tmp2); + + LSX_ST_8(p7, p6, p5, p4, p3, p2, p1, p0, output, out_stride, out_stride2, + out_stride3, out_stride4); + output += out_stride4; + LSX_ST_8(q0, q1, q2, q3, q4, q5, q6, q7, output, out_stride, out_stride2, + out_stride3, out_stride4); +} + +static int32_t vt_lpf_t4_and_t8_16w(uint8_t *dst, uint8_t *filter48, + uint8_t *dst_org, int32_t stride, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr) { + int32_t stride2 = stride << 1; + int32_t stride3 = stride2 + stride; + int32_t stride4 = stride2 << 1; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i p2_out, p1_out, p0_out, q0_out, q1_out, q2_out; + __m128i flat, mask, hev, thresh, b_limit, limit; + __m128i p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l; + __m128i p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h; + __m128i p2_filt8_l, p1_filt8_l, p0_filt8_l; + __m128i q0_filt8_l, q1_filt8_l, q2_filt8_l; + __m128i p2_filt8_h, p1_filt8_h, p0_filt8_h; + __m128i q0_filt8_h, q1_filt8_h, q2_filt8_h; + __m128i vec0, vec1, vec2, vec3, vec4, vec5; + __m128i zero = __lsx_vldi(0); + + /* load vector elements */ + DUP4_ARG2(__lsx_vld, dst, -64, dst, -48, dst, -32, dst, -16, p3, p2, p1, p0); + DUP4_ARG2(__lsx_vld, dst, 0, dst, 16, dst, 32, dst, 48, q0, q1, q2, q3); + + thresh = __lsx_vreplgr2vr_b(*thresh_ptr); + b_limit = __lsx_vreplgr2vr_b(*b_limit_ptr); + limit = __lsx_vreplgr2vr_b(*limit_ptr); + + /* mask and hev */ + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + /* flat4 */ + VP9_FLAT4(p3, p2, p0, q0, q2, q3, flat); + /* filter4 */ + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, q1_out); + + /* if flat is zero for all pixels, then no need to calculate other filter */ + if (__lsx_bz_v(flat)) { + DUP2_ARG2(__lsx_vilvl_b, p0_out, p1_out, q1_out, q0_out, vec0, vec1); + vec2 = __lsx_vilvl_h(vec1, vec0); + vec3 = __lsx_vilvh_h(vec1, vec0); + DUP2_ARG2(__lsx_vilvh_b, p0_out, p1_out, q1_out, q0_out, vec0, vec1); + vec4 = __lsx_vilvl_h(vec1, vec0); + vec5 = __lsx_vilvh_h(vec1, vec0); + + dst_org -= 2; + __lsx_vstelm_w(vec2, dst_org, 0, 0); + __lsx_vstelm_w(vec2, dst_org + stride, 0, 1); + __lsx_vstelm_w(vec2, dst_org + stride2, 0, 2); + __lsx_vstelm_w(vec2, dst_org + stride3, 0, 3); + dst_org += stride4; + __lsx_vstelm_w(vec3, dst_org, 0, 0); + __lsx_vstelm_w(vec3, dst_org + stride, 0, 1); + __lsx_vstelm_w(vec3, dst_org + stride2, 0, 2); + __lsx_vstelm_w(vec3, dst_org + stride3, 0, 3); + dst_org += stride4; + __lsx_vstelm_w(vec4, dst_org, 0, 0); + __lsx_vstelm_w(vec4, dst_org + stride, 0, 1); + __lsx_vstelm_w(vec4, dst_org + stride2, 0, 2); + __lsx_vstelm_w(vec4, dst_org + stride3, 0, 3); + dst_org += stride4; + __lsx_vstelm_w(vec5, dst_org, 0, 0); + __lsx_vstelm_w(vec5, dst_org + stride, 0, 1); + __lsx_vstelm_w(vec5, dst_org + stride2, 0, 2); + __lsx_vstelm_w(vec5, dst_org + stride3, 0, 3); + + return 1; + } + + DUP4_ARG2(__lsx_vilvl_b, zero, p3, zero, p2, zero, p1, zero, p0, p3_l, p2_l, + p1_l, p0_l); + DUP4_ARG2(__lsx_vilvl_b, zero, q0, zero, q1, zero, q2, zero, q3, q0_l, q1_l, + q2_l, q3_l); + VP9_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l, q1_filt8_l, q2_filt8_l); + DUP4_ARG2(__lsx_vilvh_b, zero, p3, zero, p2, zero, p1, zero, p0, p3_h, p2_h, + p1_h, p0_h); + DUP4_ARG2(__lsx_vilvh_b, zero, q0, zero, q1, zero, q2, zero, q3, q0_h, q1_h, + q2_h, q3_h); + VP9_FILTER8(p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h, p2_filt8_h, + p1_filt8_h, p0_filt8_h, q0_filt8_h, q1_filt8_h, q2_filt8_h); + + /* convert 16 bit output data into 8 bit */ + DUP4_ARG2(__lsx_vpickev_b, p2_filt8_h, p2_filt8_l, p1_filt8_h, p1_filt8_l, + p0_filt8_h, p0_filt8_l, q0_filt8_h, q0_filt8_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l); + DUP2_ARG2(__lsx_vpickev_b, q1_filt8_h, q1_filt8_l, q2_filt8_h, q2_filt8_l, + q1_filt8_l, q2_filt8_l); + + /* store pixel values */ + p2_out = __lsx_vbitsel_v(p2, p2_filt8_l, flat); + p1_out = __lsx_vbitsel_v(p1_out, p1_filt8_l, flat); + p0_out = __lsx_vbitsel_v(p0_out, p0_filt8_l, flat); + q0_out = __lsx_vbitsel_v(q0_out, q0_filt8_l, flat); + q1_out = __lsx_vbitsel_v(q1_out, q1_filt8_l, flat); + q2_out = __lsx_vbitsel_v(q2, q2_filt8_l, flat); + + __lsx_vst(p2_out, filter48, 0); + __lsx_vst(p1_out, filter48, 16); + __lsx_vst(p0_out, filter48, 32); + __lsx_vst(q0_out, filter48, 48); + __lsx_vst(q1_out, filter48, 64); + __lsx_vst(q2_out, filter48, 80); + __lsx_vst(flat, filter48, 96); + + return 0; +} + +static int32_t vt_lpf_t16_16w(uint8_t *dst, uint8_t *dst_org, int32_t stride, + uint8_t *filter48) { + __m128i zero = __lsx_vldi(0); + __m128i flat, flat2, filter8; + __m128i p7, p6, p5, p4, p3, p2, p1, p0, q0, q1, q2, q3, q4, q5, q6, q7; + __m128i out_l, out_h; + v8u16 p7_l_in, p6_l_in, p5_l_in, p4_l_in; + v8u16 p3_l_in, p2_l_in, p1_l_in, p0_l_in; + v8u16 q7_l_in, q6_l_in, q5_l_in, q4_l_in; + v8u16 q3_l_in, q2_l_in, q1_l_in, q0_l_in; + v8u16 p7_h_in, p6_h_in, p5_h_in, p4_h_in; + v8u16 p3_h_in, p2_h_in, p1_h_in, p0_h_in; + v8u16 q7_h_in, q6_h_in, q5_h_in, q4_h_in; + v8u16 q3_h_in, q2_h_in, q1_h_in, q0_h_in; + v8u16 tmp0_l, tmp1_l, tmp0_h, tmp1_h; + uint8_t *dst_tmp = dst - 128; + + flat = __lsx_vld(filter48, 96); + + DUP4_ARG2(__lsx_vld, dst_tmp, 0, dst_tmp, 16, dst_tmp, 32, dst_tmp, 48, p7, + p6, p5, p4); + DUP4_ARG2(__lsx_vld, dst_tmp, 64, dst_tmp, 80, dst_tmp, 96, dst_tmp, 112, p3, + p2, p1, p0); + DUP4_ARG2(__lsx_vld, dst, 0, dst, 16, dst, 32, dst, 48, q0, q1, q2, q3); + DUP4_ARG2(__lsx_vld, dst, 64, dst, 80, dst, 96, dst, 112, q4, q5, q6, q7); + + VP9_FLAT5(p7, p6, p5, p4, p0, q0, q4, q5, q6, q7, flat, flat2); + /* if flat2 is zero for all pixels, then no need to calculate other filter */ + if (__lsx_bz_v(flat2)) { + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + + DUP4_ARG2(__lsx_vld, filter48, 0, filter48, 16, filter48, 32, filter48, 48, + p2, p1, p0, q0); + DUP2_ARG2(__lsx_vld, filter48, 64, filter48, 80, q1, q2); + + DUP2_ARG2(__lsx_vilvl_b, p1, p2, q0, p0, vec0, vec1); + vec3 = __lsx_vilvl_h(vec1, vec0); + vec4 = __lsx_vilvh_h(vec1, vec0); + DUP2_ARG2(__lsx_vilvh_b, p1, p2, q0, p0, vec0, vec1); + vec6 = __lsx_vilvl_h(vec1, vec0); + vec7 = __lsx_vilvh_h(vec1, vec0); + vec2 = __lsx_vilvl_b(q2, q1); + vec5 = __lsx_vilvh_b(q2, q1); + + dst_org -= 3; + __lsx_vstelm_w(vec3, dst_org, 0, 0); + __lsx_vstelm_h(vec2, dst_org, 4, 0); + dst_org += stride; + __lsx_vstelm_w(vec3, dst_org, 0, 1); + __lsx_vstelm_h(vec2, dst_org, 4, 1); + dst_org += stride; + __lsx_vstelm_w(vec3, dst_org, 0, 2); + __lsx_vstelm_h(vec2, dst_org, 4, 2); + dst_org += stride; + __lsx_vstelm_w(vec3, dst_org, 0, 3); + __lsx_vstelm_h(vec2, dst_org, 4, 3); + dst_org += stride; + __lsx_vstelm_w(vec4, dst_org, 0, 0); + __lsx_vstelm_h(vec2, dst_org, 4, 4); + dst_org += stride; + __lsx_vstelm_w(vec4, dst_org, 0, 1); + __lsx_vstelm_h(vec2, dst_org, 4, 5); + dst_org += stride; + __lsx_vstelm_w(vec4, dst_org, 0, 2); + __lsx_vstelm_h(vec2, dst_org, 4, 6); + dst_org += stride; + __lsx_vstelm_w(vec4, dst_org, 0, 3); + __lsx_vstelm_h(vec2, dst_org, 4, 7); + dst_org += stride; + __lsx_vstelm_w(vec6, dst_org, 0, 0); + __lsx_vstelm_h(vec5, dst_org, 4, 0); + dst_org += stride; + __lsx_vstelm_w(vec6, dst_org, 0, 1); + __lsx_vstelm_h(vec5, dst_org, 4, 1); + dst_org += stride; + __lsx_vstelm_w(vec6, dst_org, 0, 2); + __lsx_vstelm_h(vec5, dst_org, 4, 2); + dst_org += stride; + __lsx_vstelm_w(vec6, dst_org, 0, 3); + __lsx_vstelm_h(vec5, dst_org, 4, 3); + dst_org += stride; + __lsx_vstelm_w(vec7, dst_org, 0, 0); + __lsx_vstelm_h(vec5, dst_org, 4, 4); + dst_org += stride; + __lsx_vstelm_w(vec7, dst_org, 0, 1); + __lsx_vstelm_h(vec5, dst_org, 4, 5); + dst_org += stride; + __lsx_vstelm_w(vec7, dst_org, 0, 2); + __lsx_vstelm_h(vec5, dst_org, 4, 6); + dst_org += stride; + __lsx_vstelm_w(vec7, dst_org, 0, 3); + __lsx_vstelm_h(vec5, dst_org, 4, 7); + + return 1; + } + + dst -= 7 * 16; + + p7_l_in = (v8u16)__lsx_vilvl_b(zero, p7); + p6_l_in = (v8u16)__lsx_vilvl_b(zero, p6); + p5_l_in = (v8u16)__lsx_vilvl_b(zero, p5); + p4_l_in = (v8u16)__lsx_vilvl_b(zero, p4); + p3_l_in = (v8u16)__lsx_vilvl_b(zero, p3); + p2_l_in = (v8u16)__lsx_vilvl_b(zero, p2); + p1_l_in = (v8u16)__lsx_vilvl_b(zero, p1); + p0_l_in = (v8u16)__lsx_vilvl_b(zero, p0); + q0_l_in = (v8u16)__lsx_vilvl_b(zero, q0); + + tmp0_l = p7_l_in << 3; + tmp0_l -= p7_l_in; + tmp0_l += p6_l_in; + tmp0_l += q0_l_in; + tmp1_l = p6_l_in + p5_l_in; + tmp1_l += p4_l_in; + tmp1_l += p3_l_in; + tmp1_l += p2_l_in; + tmp1_l += p1_l_in; + tmp1_l += p0_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + p7_h_in = (v8u16)__lsx_vilvh_b(zero, p7); + p6_h_in = (v8u16)__lsx_vilvh_b(zero, p6); + p5_h_in = (v8u16)__lsx_vilvh_b(zero, p5); + p4_h_in = (v8u16)__lsx_vilvh_b(zero, p4); + p3_h_in = (v8u16)__lsx_vilvh_b(zero, p3); + p2_h_in = (v8u16)__lsx_vilvh_b(zero, p2); + p1_h_in = (v8u16)__lsx_vilvh_b(zero, p1); + p0_h_in = (v8u16)__lsx_vilvh_b(zero, p0); + q0_h_in = (v8u16)__lsx_vilvh_b(zero, q0); + + tmp0_h = p7_h_in << 3; + tmp0_h -= p7_h_in; + tmp0_h += p6_h_in; + tmp0_h += q0_h_in; + tmp1_h = p6_h_in + p5_h_in; + tmp1_h += p4_h_in; + tmp1_h += p3_h_in; + tmp1_h += p2_h_in; + tmp1_h += p1_h_in; + tmp1_h += p0_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + p6 = __lsx_vbitsel_v(p6, out_l, flat2); + __lsx_vst(p6, dst, 0); + + /* p5 */ + q1_l_in = (v8u16)__lsx_vilvl_b(zero, q1); + tmp0_l = p5_l_in - p6_l_in; + tmp0_l += q1_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + q1_h_in = (v8u16)__lsx_vilvh_b(zero, q1); + tmp0_h = p5_h_in - p6_h_in; + tmp0_h += q1_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + p5 = __lsx_vbitsel_v(p5, out_l, flat2); + __lsx_vst(p5, dst, 16); + + /* p4 */ + q2_l_in = (v8u16)__lsx_vilvl_b(zero, q2); + tmp0_l = p4_l_in - p5_l_in; + tmp0_l += q2_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + q2_h_in = (v8u16)__lsx_vilvh_b(zero, q2); + tmp0_h = p4_h_in - p5_h_in; + tmp0_h += q2_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + p4 = __lsx_vbitsel_v(p4, out_l, flat2); + __lsx_vst(p4, dst, 16 * 2); + + /* p3 */ + q3_l_in = (v8u16)__lsx_vilvl_b(zero, q3); + tmp0_l = p3_l_in - p4_l_in; + tmp0_l += q3_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + q3_h_in = (v8u16)__lsx_vilvh_b(zero, q3); + tmp0_h = p3_h_in - p4_h_in; + tmp0_h += q3_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + p3 = __lsx_vbitsel_v(p3, out_l, flat2); + __lsx_vst(p3, dst, 16 * 3); + + /* p2 */ + q4_l_in = (v8u16)__lsx_vilvl_b(zero, q4); + filter8 = __lsx_vld(filter48, 0); + tmp0_l = p2_l_in - p3_l_in; + tmp0_l += q4_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + q4_h_in = (v8u16)__lsx_vilvh_b(zero, q4); + tmp0_h = p2_h_in - p3_h_in; + tmp0_h += q4_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 16 * 4); + + /* p1 */ + q5_l_in = (v8u16)__lsx_vilvl_b(zero, q5); + filter8 = __lsx_vld(filter48, 16); + tmp0_l = p1_l_in - p2_l_in; + tmp0_l += q5_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + q5_h_in = (v8u16)__lsx_vilvh_b(zero, q5); + tmp0_h = p1_h_in - p2_h_in; + tmp0_h += q5_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)(tmp1_h), 4); + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 16 * 5); + + /* p0 */ + q6_l_in = (v8u16)__lsx_vilvl_b(zero, q6); + filter8 = __lsx_vld(filter48, 32); + tmp0_l = p0_l_in - p1_l_in; + tmp0_l += q6_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + q6_h_in = (v8u16)__lsx_vilvh_b(zero, q6); + tmp0_h = p0_h_in - p1_h_in; + tmp0_h += q6_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 16 * 6); + + /* q0 */ + q7_l_in = (v8u16)__lsx_vilvl_b(zero, q7); + filter8 = __lsx_vld(filter48, 48); + tmp0_l = q7_l_in - p0_l_in; + tmp0_l += q0_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + q7_h_in = (v8u16)__lsx_vilvh_b(zero, q7); + tmp0_h = q7_h_in - p0_h_in; + tmp0_h += q0_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 16 * 7); + + /* q1 */ + filter8 = __lsx_vld(filter48, 64); + tmp0_l = q7_l_in - q0_l_in; + tmp0_l += q1_l_in; + tmp0_l -= p6_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + tmp0_h = q7_h_in - q0_h_in; + tmp0_h += q1_h_in; + tmp0_h -= p6_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 16 * 8); + + /* q2 */ + filter8 = __lsx_vld(filter48, 80); + tmp0_l = q7_l_in - q1_l_in; + tmp0_l += q2_l_in; + tmp0_l -= p5_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + tmp0_h = q7_h_in - q1_h_in; + tmp0_h += q2_h_in; + tmp0_h -= p5_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 16 * 9); + + /* q3 */ + tmp0_l = q7_l_in - q2_l_in; + tmp0_l += q3_l_in; + tmp0_l -= p4_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + tmp0_h = q7_h_in - q2_h_in; + tmp0_h += q3_h_in; + tmp0_h -= p4_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + q3 = __lsx_vbitsel_v(q3, out_l, flat2); + __lsx_vst(q3, dst, 16 * 10); + + /* q4 */ + tmp0_l = q7_l_in - q3_l_in; + tmp0_l += q4_l_in; + tmp0_l -= p3_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + tmp0_h = q7_h_in - q3_h_in; + tmp0_h += q4_h_in; + tmp0_h -= p3_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + q4 = __lsx_vbitsel_v(q4, out_l, flat2); + __lsx_vst(q4, dst, 16 * 11); + + /* q5 */ + tmp0_l = q7_l_in - q4_l_in; + tmp0_l += q5_l_in; + tmp0_l -= p2_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + tmp0_h = q7_h_in - q4_h_in; + tmp0_h += q5_h_in; + tmp0_h -= p2_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + q5 = __lsx_vbitsel_v(q5, out_l, flat2); + __lsx_vst(q5, dst, 16 * 12); + + /* q6 */ + tmp0_l = q7_l_in - q5_l_in; + tmp0_l += q6_l_in; + tmp0_l -= p1_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + tmp0_h = q7_h_in - q5_h_in; + tmp0_h += q6_h_in; + tmp0_h -= p1_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + q6 = __lsx_vbitsel_v(q6, out_l, flat2); + __lsx_vst(q6, dst, 16 * 13); + + return 0; +} + +void vpx_lpf_vertical_16_dual_lsx(uint8_t *src, int32_t pitch, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr) { + uint8_t early_exit = 0; + DECLARE_ALIGNED(16, uint8_t, transposed_input[16 * 24]); + uint8_t *filter48 = &transposed_input[16 * 16]; + + transpose_16x16((src - 8), pitch, &transposed_input[0], 16); + + early_exit = + vt_lpf_t4_and_t8_16w((transposed_input + 16 * 8), &filter48[0], src, + pitch, b_limit_ptr, limit_ptr, thresh_ptr); + + if (early_exit == 0) { + early_exit = + vt_lpf_t16_16w((transposed_input + 16 * 8), src, pitch, &filter48[0]); + + if (early_exit == 0) { + transpose_16x16(transposed_input, 16, (src - 8), pitch); + } + } +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/loopfilter_4_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/loopfilter_4_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/loopfilter_4_lsx.c @@ -0,0 +1,214 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "./vpx_dsp_rtcd.h" +#include "vpx_dsp/loongarch/loopfilter_lsx.h" + +void vpx_lpf_horizontal_4_lsx(uint8_t *src, int32_t pitch, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr) { + __m128i mask, hev, flat, thresh, b_limit, limit; + __m128i p3, p2, p1, p0, q3, q2, q1, q0, p1_out, p0_out, q0_out, q1_out; + int32_t pitch2 = pitch << 1; + int32_t pitch3 = pitch2 + pitch; + int32_t pitch4 = pitch2 << 1; + + DUP4_ARG2(__lsx_vldx, src, -pitch4, src, -pitch3, src, -pitch2, src, -pitch, + p3, p2, p1, p0); + q0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, pitch, src, pitch2, q1, q2); + q3 = __lsx_vldx(src, pitch3); + + thresh = __lsx_vreplgr2vr_b(*thresh_ptr); + b_limit = __lsx_vreplgr2vr_b(*b_limit_ptr); + limit = __lsx_vreplgr2vr_b(*limit_ptr); + + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, q1_out); + + __lsx_vstelm_d(p1_out, src - pitch2, 0, 0); + __lsx_vstelm_d(p0_out, src - pitch, 0, 0); + __lsx_vstelm_d(q0_out, src, 0, 0); + __lsx_vstelm_d(q1_out, src + pitch, 0, 0); +} + +void vpx_lpf_horizontal_4_dual_lsx(uint8_t *src, int32_t pitch, + const uint8_t *b_limit0_ptr, + const uint8_t *limit0_ptr, + const uint8_t *thresh0_ptr, + const uint8_t *b_limit1_ptr, + const uint8_t *limit1_ptr, + const uint8_t *thresh1_ptr) { + __m128i mask, hev, flat, thresh0, b_limit0, limit0, thresh1, b_limit1, limit1; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + int32_t pitch2 = pitch << 1; + int32_t pitch3 = pitch2 + pitch; + int32_t pitch4 = pitch2 << 1; + + DUP4_ARG2(__lsx_vldx, src, -pitch4, src, -pitch3, src, -pitch2, src, -pitch, + p3, p2, p1, p0); + q0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, pitch, src, pitch2, q1, q2); + q3 = __lsx_vldx(src, pitch3); + + thresh0 = __lsx_vreplgr2vr_b(*thresh0_ptr); + thresh1 = __lsx_vreplgr2vr_b(*thresh1_ptr); + thresh0 = __lsx_vilvl_d(thresh1, thresh0); + + b_limit0 = __lsx_vreplgr2vr_b(*b_limit0_ptr); + b_limit1 = __lsx_vreplgr2vr_b(*b_limit1_ptr); + b_limit0 = __lsx_vilvl_d(b_limit1, b_limit0); + + limit0 = __lsx_vreplgr2vr_b(*limit0_ptr); + limit1 = __lsx_vreplgr2vr_b(*limit1_ptr); + limit0 = __lsx_vilvl_d(limit1, limit0); + + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit0, b_limit0, thresh0, hev, + mask, flat); + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1, p0, q0, q1); + + __lsx_vstx(p1, src, -pitch2); + __lsx_vstx(p0, src, -pitch); + __lsx_vst(q0, src, 0); + __lsx_vstx(q1, src, pitch); +} + +void vpx_lpf_vertical_4_lsx(uint8_t *src, int32_t pitch, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr) { + __m128i mask, hev, flat, limit, thresh, b_limit; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i vec0, vec1, vec2, vec3; + int32_t pitch2 = pitch << 1; + int32_t pitch3 = pitch2 + pitch; + int32_t pitch4 = pitch2 << 1; + uint8_t *src_tmp = src - 4; + + p3 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, pitch, src_tmp, pitch2, p2, p1); + p0 = __lsx_vldx(src_tmp, pitch3); + src_tmp += pitch4; + q0 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, pitch, src_tmp, pitch2, q1, q2); + q3 = __lsx_vldx(src_tmp, pitch3); + + thresh = __lsx_vreplgr2vr_b(*thresh_ptr); + b_limit = __lsx_vreplgr2vr_b(*b_limit_ptr); + limit = __lsx_vreplgr2vr_b(*limit_ptr); + + LSX_TRANSPOSE8x8_B(p3, p2, p1, p0, q0, q1, q2, q3, p3, p2, p1, p0, q0, q1, q2, + q3); + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1, p0, q0, q1); + DUP2_ARG2(__lsx_vilvl_b, p0, p1, q1, q0, vec0, vec1); + vec2 = __lsx_vilvl_h(vec1, vec0); + vec3 = __lsx_vilvh_h(vec1, vec0); + + src -= 2; + __lsx_vstelm_w(vec2, src, 0, 0); + src += pitch; + __lsx_vstelm_w(vec2, src, 0, 1); + src += pitch; + __lsx_vstelm_w(vec2, src, 0, 2); + src += pitch; + __lsx_vstelm_w(vec2, src, 0, 3); + src += pitch; + + __lsx_vstelm_w(vec3, src, 0, 0); + __lsx_vstelm_w(vec3, src + pitch, 0, 1); + __lsx_vstelm_w(vec3, src + pitch2, 0, 2); + __lsx_vstelm_w(vec3, src + pitch3, 0, 3); +} + +void vpx_lpf_vertical_4_dual_lsx(uint8_t *src, int32_t pitch, + const uint8_t *b_limit0_ptr, + const uint8_t *limit0_ptr, + const uint8_t *thresh0_ptr, + const uint8_t *b_limit1_ptr, + const uint8_t *limit1_ptr, + const uint8_t *thresh1_ptr) { + __m128i mask, hev, flat; + __m128i thresh0, b_limit0, limit0, thresh1, b_limit1, limit1; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i row0, row1, row2, row3, row4, row5, row6, row7; + __m128i row8, row9, row10, row11, row12, row13, row14, row15; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + int32_t pitch2 = pitch << 1; + int32_t pitch3 = pitch2 + pitch; + int32_t pitch4 = pitch2 << 1; + uint8_t *src_tmp = src - 4; + + row0 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, pitch, src_tmp, pitch2, row1, row2); + row3 = __lsx_vldx(src_tmp, pitch3); + src_tmp += pitch4; + row4 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, pitch, src_tmp, pitch2, row5, row6); + row7 = __lsx_vldx(src_tmp, pitch3); + src_tmp += pitch4; + row8 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, pitch, src_tmp, pitch2, row9, row10); + row11 = __lsx_vldx(src_tmp, pitch3); + src_tmp += pitch4; + row12 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, pitch, src_tmp, pitch2, row13, row14); + row15 = __lsx_vldx(src_tmp, pitch3); + + LSX_TRANSPOSE16x8_B(row0, row1, row2, row3, row4, row5, row6, row7, row8, + row9, row10, row11, row12, row13, row14, row15, p3, p2, + p1, p0, q0, q1, q2, q3); + + thresh0 = __lsx_vreplgr2vr_b(*thresh0_ptr); + thresh1 = __lsx_vreplgr2vr_b(*thresh1_ptr); + thresh0 = __lsx_vilvl_d(thresh1, thresh0); + + b_limit0 = __lsx_vreplgr2vr_b(*b_limit0_ptr); + b_limit1 = __lsx_vreplgr2vr_b(*b_limit1_ptr); + b_limit0 = __lsx_vilvl_d(b_limit1, b_limit0); + + limit0 = __lsx_vreplgr2vr_b(*limit0_ptr); + limit1 = __lsx_vreplgr2vr_b(*limit1_ptr); + limit0 = __lsx_vilvl_d(limit1, limit0); + + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit0, b_limit0, thresh0, hev, + mask, flat); + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1, p0, q0, q1); + DUP2_ARG2(__lsx_vilvl_b, p0, p1, q1, q0, tmp0, tmp1); + tmp2 = __lsx_vilvl_h(tmp1, tmp0); + tmp3 = __lsx_vilvh_h(tmp1, tmp0); + DUP2_ARG2(__lsx_vilvh_b, p0, p1, q1, q0, tmp0, tmp1); + tmp4 = __lsx_vilvl_h(tmp1, tmp0); + tmp5 = __lsx_vilvh_h(tmp1, tmp0); + + src -= 2; + __lsx_vstelm_w(tmp2, src, 0, 0); + __lsx_vstelm_w(tmp2, src + pitch, 0, 1); + __lsx_vstelm_w(tmp2, src + pitch2, 0, 2); + __lsx_vstelm_w(tmp2, src + pitch3, 0, 3); + src += pitch4; + __lsx_vstelm_w(tmp3, src, 0, 0); + __lsx_vstelm_w(tmp3, src + pitch, 0, 1); + __lsx_vstelm_w(tmp3, src + pitch2, 0, 2); + __lsx_vstelm_w(tmp3, src + pitch3, 0, 3); + src += pitch4; + __lsx_vstelm_w(tmp4, src, 0, 0); + __lsx_vstelm_w(tmp4, src + pitch, 0, 1); + __lsx_vstelm_w(tmp4, src + pitch2, 0, 2); + __lsx_vstelm_w(tmp4, src + pitch3, 0, 3); + src += pitch4; + __lsx_vstelm_w(tmp5, src, 0, 0); + __lsx_vstelm_w(tmp5, src + pitch, 0, 1); + __lsx_vstelm_w(tmp5, src + pitch2, 0, 2); + __lsx_vstelm_w(tmp5, src + pitch3, 0, 3); +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/loopfilter_8_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/loopfilter_8_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/loopfilter_8_lsx.c @@ -0,0 +1,469 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "./vpx_dsp_rtcd.h" +#include "vpx_dsp/loongarch/loopfilter_lsx.h" + +void vpx_lpf_horizontal_8_lsx(uint8_t *dst, int32_t stride, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr) { + __m128i mask, hev, flat, thresh, b_limit, limit; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i p2_out, p1_out, p0_out, q0_out, q1_out, q2_out; + __m128i p2_filter8, p1_filter8, p0_filter8; + __m128i q0_filter8, q1_filter8, q2_filter8; + __m128i p3_l, p2_l, p1_l, p0_l, q3_l, q2_l, q1_l, q0_l; + __m128i zero = __lsx_vldi(0); + + int32_t stride2 = stride << 1; + int32_t stride3 = stride2 + stride; + int32_t stride4 = stride2 << 1; + + /* load vector elements */ + DUP4_ARG2(__lsx_vldx, dst, -stride4, dst, -stride3, dst, -stride2, dst, + -stride, p3, p2, p1, p0); + q0 = __lsx_vld(dst, 0); + DUP2_ARG2(__lsx_vldx, dst, stride, dst, stride2, q1, q2); + q3 = __lsx_vldx(dst, stride3); + + thresh = __lsx_vreplgr2vr_b(*thresh_ptr); + b_limit = __lsx_vreplgr2vr_b(*b_limit_ptr); + limit = __lsx_vreplgr2vr_b(*limit_ptr); + + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + VP9_FLAT4(p3, p2, p0, q0, q2, q3, flat); + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, q1_out); + + flat = __lsx_vilvl_d(zero, flat); + + if (__lsx_bz_v(flat)) { + __lsx_vstelm_d(p1_out, dst - stride2, 0, 0); + __lsx_vstelm_d(p0_out, dst - stride, 0, 0); + __lsx_vstelm_d(q0_out, dst, 0, 0); + __lsx_vstelm_d(q1_out, dst + stride, 0, 0); + } else { + DUP4_ARG2(__lsx_vilvl_b, zero, p3, zero, p2, zero, p1, zero, p0, p3_l, p2_l, + p1_l, p0_l); + DUP4_ARG2(__lsx_vilvl_b, zero, q0, zero, q1, zero, q2, zero, q3, q0_l, q1_l, + q2_l, q3_l); + VP9_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filter8, + p1_filter8, p0_filter8, q0_filter8, q1_filter8, q2_filter8); + + /* convert 16 bit output data into 8 bit */ + DUP4_ARG2(__lsx_vpickev_b, zero, p2_filter8, zero, p1_filter8, zero, + p0_filter8, zero, q0_filter8, p2_filter8, p1_filter8, p0_filter8, + q0_filter8); + DUP2_ARG2(__lsx_vpickev_b, zero, q1_filter8, zero, q2_filter8, q1_filter8, + q2_filter8); + DUP4_ARG3(__lsx_vbitsel_v, p2, p2_filter8, flat, p1_out, p1_filter8, flat, + p0_out, p0_filter8, flat, q0_out, q0_filter8, flat, p2_out, + p1_out, p0_out, q0_out); + DUP2_ARG3(__lsx_vbitsel_v, q1_out, q1_filter8, flat, q2, q2_filter8, flat, + q1_out, q2_out); + dst -= stride3; + + __lsx_vstelm_d(p2_out, dst, 0, 0); + __lsx_vstelm_d(p1_out, dst + stride, 0, 0); + __lsx_vstelm_d(p0_out, dst + stride2, 0, 0); + __lsx_vstelm_d(q0_out, dst + stride3, 0, 0); + + dst += stride4; + __lsx_vstelm_d(q1_out, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(q2_out, dst, 0, 0); + } +} + +void vpx_lpf_horizontal_8_dual_lsx( + uint8_t *dst, int32_t stride, const uint8_t *b_limit0, + const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *b_limit1, + const uint8_t *limit1, const uint8_t *thresh1) { + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i p2_out, p1_out, p0_out, q0_out, q1_out, q2_out; + __m128i flat, mask, hev, tmp, thresh, b_limit, limit; + __m128i p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l; + __m128i p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h; + __m128i p2_filt8_l, p1_filt8_l, p0_filt8_l; + __m128i q0_filt8_l, q1_filt8_l, q2_filt8_l; + __m128i p2_filt8_h, p1_filt8_h, p0_filt8_h; + __m128i q0_filt8_h, q1_filt8_h, q2_filt8_h; + __m128i zero = __lsx_vldi(0); + + int32_t stride2 = stride << 1; + int32_t stride3 = stride2 + stride; + int32_t stride4 = stride2 << 1; + + DUP4_ARG2(__lsx_vldx, dst, -stride4, dst, -stride3, dst, -stride2, dst, + -stride, p3, p2, p1, p0); + q0 = __lsx_vld(dst, 0); + DUP2_ARG2(__lsx_vldx, dst, stride, dst, stride2, q1, q2); + q3 = __lsx_vldx(dst, stride3); + + thresh = __lsx_vreplgr2vr_b(*thresh0); + tmp = __lsx_vreplgr2vr_b(*thresh1); + thresh = __lsx_vilvl_d(tmp, thresh); + + b_limit = __lsx_vreplgr2vr_b(*b_limit0); + tmp = __lsx_vreplgr2vr_b(*b_limit1); + b_limit = __lsx_vilvl_d(tmp, b_limit); + + limit = __lsx_vreplgr2vr_b(*limit0); + tmp = __lsx_vreplgr2vr_b(*limit1); + limit = __lsx_vilvl_d(tmp, limit); + + /* mask and hev */ + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + VP9_FLAT4(p3, p2, p0, q0, q2, q3, flat); + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, q1_out); + + if (__lsx_bz_v(flat)) { + __lsx_vst(p1_out, dst - stride2, 0); + __lsx_vst(p0_out, dst - stride, 0); + __lsx_vst(q0_out, dst, 0); + __lsx_vst(q1_out, dst + stride, 0); + } else { + DUP4_ARG2(__lsx_vilvl_b, zero, p3, zero, p2, zero, p1, zero, p0, p3_l, p2_l, + p1_l, p0_l); + DUP4_ARG2(__lsx_vilvl_b, zero, q0, zero, q1, zero, q2, zero, q3, q0_l, q1_l, + q2_l, q3_l); + VP9_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l, q1_filt8_l, q2_filt8_l); + + DUP4_ARG2(__lsx_vilvh_b, zero, p3, zero, p2, zero, p1, zero, p0, p3_h, p2_h, + p1_h, p0_h); + DUP4_ARG2(__lsx_vilvh_b, zero, q0, zero, q1, zero, q2, zero, q3, q0_h, q1_h, + q2_h, q3_h); + VP9_FILTER8(p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h, p2_filt8_h, + p1_filt8_h, p0_filt8_h, q0_filt8_h, q1_filt8_h, q2_filt8_h); + + /* convert 16 bit output data into 8 bit */ + DUP4_ARG2(__lsx_vpickev_b, p2_filt8_h, p2_filt8_l, p1_filt8_h, p1_filt8_l, + p0_filt8_h, p0_filt8_l, q0_filt8_h, q0_filt8_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l); + DUP2_ARG2(__lsx_vpickev_b, q1_filt8_h, q1_filt8_l, q2_filt8_h, q2_filt8_l, + q1_filt8_l, q2_filt8_l); + + /* store pixel values */ + p2_out = __lsx_vbitsel_v(p2, p2_filt8_l, flat); + p1_out = __lsx_vbitsel_v(p1_out, p1_filt8_l, flat); + p0_out = __lsx_vbitsel_v(p0_out, p0_filt8_l, flat); + q0_out = __lsx_vbitsel_v(q0_out, q0_filt8_l, flat); + q1_out = __lsx_vbitsel_v(q1_out, q1_filt8_l, flat); + q2_out = __lsx_vbitsel_v(q2, q2_filt8_l, flat); + + __lsx_vst(p2_out, dst - stride3, 0); + __lsx_vst(p1_out, dst - stride2, 0); + __lsx_vst(p0_out, dst - stride, 0); + __lsx_vst(q0_out, dst, 0); + __lsx_vst(q1_out, dst + stride, 0); + __lsx_vst(q2_out, dst + stride2, 0); + } +} + +void vpx_lpf_vertical_8_lsx(uint8_t *dst, int32_t stride, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr) { + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i p1_out, p0_out, q0_out, q1_out; + __m128i flat, mask, hev, thresh, b_limit, limit; + __m128i p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l; + __m128i p2_filt8_l, p1_filt8_l, p0_filt8_l; + __m128i q0_filt8_l, q1_filt8_l, q2_filt8_l; + __m128i vec0, vec1, vec2, vec3, vec4; + __m128i zero = __lsx_vldi(0); + + int32_t stride2 = stride << 1; + int32_t stride3 = stride2 + stride; + int32_t stride4 = stride2 << 1; + uint8_t *dst_tmp = dst - 4; + + /* load vector elements */ + p3 = __lsx_vld(dst_tmp, 0); + DUP2_ARG2(__lsx_vldx, dst_tmp, stride, dst_tmp, stride2, p2, p1); + p0 = __lsx_vldx(dst_tmp, stride3); + dst_tmp += stride4; + q0 = __lsx_vld(dst_tmp, 0); + DUP2_ARG2(__lsx_vldx, dst_tmp, stride, dst_tmp, stride2, q1, q2); + q3 = __lsx_vldx(dst_tmp, stride3); + + LSX_TRANSPOSE8x8_B(p3, p2, p1, p0, q0, q1, q2, q3, p3, p2, p1, p0, q0, q1, q2, + q3); + + thresh = __lsx_vreplgr2vr_b(*thresh_ptr); + b_limit = __lsx_vreplgr2vr_b(*b_limit_ptr); + limit = __lsx_vreplgr2vr_b(*limit_ptr); + + /* mask and hev */ + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + /* flat4 */ + VP9_FLAT4(p3, p2, p0, q0, q2, q3, flat); + /* filter4 */ + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, q1_out); + + flat = __lsx_vilvl_d(zero, flat); + + /* if flat is zero for all pixels, then no need to calculate other filter */ + if (__lsx_bz_v(flat)) { + /* Store 4 pixels p1-_q1 */ + DUP2_ARG2(__lsx_vilvl_b, p0_out, p1_out, q1_out, q0_out, vec0, vec1); + vec2 = __lsx_vilvl_h(vec1, vec0); + vec3 = __lsx_vilvh_h(vec1, vec0); + + dst -= 2; + __lsx_vstelm_w(vec2, dst, 0, 0); + __lsx_vstelm_w(vec2, dst + stride, 0, 1); + __lsx_vstelm_w(vec2, dst + stride2, 0, 2); + __lsx_vstelm_w(vec2, dst + stride3, 0, 3); + dst += stride4; + __lsx_vstelm_w(vec3, dst, 0, 0); + __lsx_vstelm_w(vec3, dst + stride, 0, 1); + __lsx_vstelm_w(vec3, dst + stride2, 0, 2); + __lsx_vstelm_w(vec3, dst + stride3, 0, 3); + } else { + DUP4_ARG2(__lsx_vilvl_b, zero, p3, zero, p2, zero, p1, zero, p0, p3_l, p2_l, + p1_l, p0_l); + DUP4_ARG2(__lsx_vilvl_b, zero, q0, zero, q1, zero, q2, zero, q3, q0_l, q1_l, + q2_l, q3_l); + VP9_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l, q1_filt8_l, q2_filt8_l); + /* convert 16 bit output data into 8 bit */ + DUP4_ARG2(__lsx_vpickev_b, p2_filt8_l, p2_filt8_l, p1_filt8_l, p1_filt8_l, + p0_filt8_l, p0_filt8_l, q0_filt8_l, q0_filt8_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l); + DUP2_ARG2(__lsx_vpickev_b, q1_filt8_l, q1_filt8_l, q2_filt8_l, q2_filt8_l, + q1_filt8_l, q2_filt8_l); + /* store pixel values */ + p2 = __lsx_vbitsel_v(p2, p2_filt8_l, flat); + p1 = __lsx_vbitsel_v(p1_out, p1_filt8_l, flat); + p0 = __lsx_vbitsel_v(p0_out, p0_filt8_l, flat); + q0 = __lsx_vbitsel_v(q0_out, q0_filt8_l, flat); + q1 = __lsx_vbitsel_v(q1_out, q1_filt8_l, flat); + q2 = __lsx_vbitsel_v(q2, q2_filt8_l, flat); + + /* Store 6 pixels p2-_q2 */ + DUP2_ARG2(__lsx_vilvl_b, p1, p2, q0, p0, vec0, vec1); + vec2 = __lsx_vilvl_h(vec1, vec0); + vec3 = __lsx_vilvh_h(vec1, vec0); + vec4 = __lsx_vilvl_b(q2, q1); + + dst -= 3; + __lsx_vstelm_w(vec2, dst, 0, 0); + __lsx_vstelm_h(vec4, dst, 4, 0); + dst += stride; + __lsx_vstelm_w(vec2, dst, 0, 1); + __lsx_vstelm_h(vec4, dst, 4, 1); + dst += stride; + __lsx_vstelm_w(vec2, dst, 0, 2); + __lsx_vstelm_h(vec4, dst, 4, 2); + dst += stride; + __lsx_vstelm_w(vec2, dst, 0, 3); + __lsx_vstelm_h(vec4, dst, 4, 3); + dst += stride; + __lsx_vstelm_w(vec3, dst, 0, 0); + __lsx_vstelm_h(vec4, dst, 4, 4); + dst += stride; + __lsx_vstelm_w(vec3, dst, 0, 1); + __lsx_vstelm_h(vec4, dst, 4, 5); + dst += stride; + __lsx_vstelm_w(vec3, dst, 0, 2); + __lsx_vstelm_h(vec4, dst, 4, 6); + dst += stride; + __lsx_vstelm_w(vec3, dst, 0, 3); + __lsx_vstelm_h(vec4, dst, 4, 7); + } +} + +void vpx_lpf_vertical_8_dual_lsx(uint8_t *dst, int32_t stride, + const uint8_t *b_limit0, const uint8_t *limit0, + const uint8_t *thresh0, + const uint8_t *b_limit1, const uint8_t *limit1, + const uint8_t *thresh1) { + uint8_t *dst_tmp = dst - 4; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i p1_out, p0_out, q0_out, q1_out; + __m128i flat, mask, hev, thresh, b_limit, limit; + __m128i row4, row5, row6, row7, row12, row13, row14, row15; + __m128i p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l; + __m128i p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h; + __m128i p2_filt8_l, p1_filt8_l, p0_filt8_l; + __m128i q0_filt8_l, q1_filt8_l, q2_filt8_l; + __m128i p2_filt8_h, p1_filt8_h, p0_filt8_h; + __m128i q0_filt8_h, q1_filt8_h, q2_filt8_h; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i zero = __lsx_vldi(0); + int32_t stride2 = stride << 1; + int32_t stride3 = stride2 + stride; + int32_t stride4 = stride2 << 1; + + p0 = __lsx_vld(dst_tmp, 0); + DUP2_ARG2(__lsx_vldx, dst_tmp, stride, dst_tmp, stride2, p1, p2); + p3 = __lsx_vldx(dst_tmp, stride3); + dst_tmp += stride4; + row4 = __lsx_vld(dst_tmp, 0); + DUP2_ARG2(__lsx_vldx, dst_tmp, stride, dst_tmp, stride2, row5, row6); + row7 = __lsx_vldx(dst_tmp, stride3); + dst_tmp += stride4; + + q3 = __lsx_vld(dst_tmp, 0); + DUP2_ARG2(__lsx_vldx, dst_tmp, stride, dst_tmp, stride2, q2, q1); + q0 = __lsx_vldx(dst_tmp, stride3); + dst_tmp += stride4; + row12 = __lsx_vld(dst_tmp, 0); + DUP2_ARG2(__lsx_vldx, dst_tmp, stride, dst_tmp, stride2, row13, row14); + row15 = __lsx_vldx(dst_tmp, stride3); + + /* transpose 16x8 matrix into 8x16 */ + LSX_TRANSPOSE16x8_B(p0, p1, p2, p3, row4, row5, row6, row7, q3, q2, q1, q0, + row12, row13, row14, row15, p3, p2, p1, p0, q0, q1, q2, + q3); + + thresh = __lsx_vreplgr2vr_b(*thresh0); + vec0 = __lsx_vreplgr2vr_b(*thresh1); + thresh = __lsx_vilvl_d(vec0, thresh); + + b_limit = __lsx_vreplgr2vr_b(*b_limit0); + vec0 = __lsx_vreplgr2vr_b(*b_limit1); + b_limit = __lsx_vilvl_d(vec0, b_limit); + + limit = __lsx_vreplgr2vr_b(*limit0); + vec0 = __lsx_vreplgr2vr_b(*limit1); + limit = __lsx_vilvl_d(vec0, limit); + + /* mask and hev */ + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + /* flat4 */ + VP9_FLAT4(p3, p2, p0, q0, q2, q3, flat); + /* filter4 */ + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, q1_out); + /* if flat is zero for all pixels, then no need to calculate other filter */ + if (__lsx_bz_v(flat)) { + DUP2_ARG2(__lsx_vilvl_b, p0_out, p1_out, q1_out, q0_out, vec0, vec1); + vec2 = __lsx_vilvl_h(vec1, vec0); + vec3 = __lsx_vilvh_h(vec1, vec0); + DUP2_ARG2(__lsx_vilvh_b, p0_out, p1_out, q1_out, q0_out, vec0, vec1); + vec4 = __lsx_vilvl_h(vec1, vec0); + vec5 = __lsx_vilvh_h(vec1, vec0); + + dst -= 2; + __lsx_vstelm_w(vec2, dst, 0, 0); + __lsx_vstelm_w(vec2, dst + stride, 0, 1); + __lsx_vstelm_w(vec2, dst + stride2, 0, 2); + __lsx_vstelm_w(vec2, dst + stride3, 0, 3); + dst += stride4; + __lsx_vstelm_w(vec3, dst, 0, 0); + __lsx_vstelm_w(vec3, dst + stride, 0, 1); + __lsx_vstelm_w(vec3, dst + stride2, 0, 2); + __lsx_vstelm_w(vec3, dst + stride3, 0, 3); + dst += stride4; + __lsx_vstelm_w(vec4, dst, 0, 0); + __lsx_vstelm_w(vec4, dst + stride, 0, 1); + __lsx_vstelm_w(vec4, dst + stride2, 0, 2); + __lsx_vstelm_w(vec4, dst + stride3, 0, 3); + dst += stride4; + __lsx_vstelm_w(vec5, dst, 0, 0); + __lsx_vstelm_w(vec5, dst + stride, 0, 1); + __lsx_vstelm_w(vec5, dst + stride2, 0, 2); + __lsx_vstelm_w(vec5, dst + stride3, 0, 3); + } else { + DUP4_ARG2(__lsx_vilvl_b, zero, p3, zero, p2, zero, p1, zero, p0, p3_l, p2_l, + p1_l, p0_l); + DUP4_ARG2(__lsx_vilvl_b, zero, q0, zero, q1, zero, q2, zero, q3, q0_l, q1_l, + q2_l, q3_l); + VP9_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l, q1_filt8_l, q2_filt8_l); + DUP4_ARG2(__lsx_vilvh_b, zero, p3, zero, p2, zero, p1, zero, p0, p3_h, p2_h, + p1_h, p0_h); + DUP4_ARG2(__lsx_vilvh_b, zero, q0, zero, q1, zero, q2, zero, q3, q0_h, q1_h, + q2_h, q3_h); + + /* filter8 */ + VP9_FILTER8(p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h, p2_filt8_h, + p1_filt8_h, p0_filt8_h, q0_filt8_h, q1_filt8_h, q2_filt8_h); + + /* convert 16 bit output data into 8 bit */ + DUP4_ARG2(__lsx_vpickev_b, p2_filt8_h, p2_filt8_l, p1_filt8_h, p1_filt8_l, + p0_filt8_h, p0_filt8_l, q0_filt8_h, q0_filt8_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l); + DUP2_ARG2(__lsx_vpickev_b, q1_filt8_h, q1_filt8_l, q2_filt8_h, q2_filt8_l, + q1_filt8_l, q2_filt8_l); + + /* store pixel values */ + p2 = __lsx_vbitsel_v(p2, p2_filt8_l, flat); + p1 = __lsx_vbitsel_v(p1_out, p1_filt8_l, flat); + p0 = __lsx_vbitsel_v(p0_out, p0_filt8_l, flat); + q0 = __lsx_vbitsel_v(q0_out, q0_filt8_l, flat); + q1 = __lsx_vbitsel_v(q1_out, q1_filt8_l, flat); + q2 = __lsx_vbitsel_v(q2, q2_filt8_l, flat); + + DUP2_ARG2(__lsx_vilvl_b, p1, p2, q0, p0, vec0, vec1); + vec3 = __lsx_vilvl_h(vec1, vec0); + vec4 = __lsx_vilvh_h(vec1, vec0); + DUP2_ARG2(__lsx_vilvh_b, p1, p2, q0, p0, vec0, vec1); + vec6 = __lsx_vilvl_h(vec1, vec0); + vec7 = __lsx_vilvh_h(vec1, vec0); + vec2 = __lsx_vilvl_b(q2, q1); + vec5 = __lsx_vilvh_b(q2, q1); + + dst -= 3; + __lsx_vstelm_w(vec3, dst, 0, 0); + __lsx_vstelm_h(vec2, dst, 4, 0); + dst += stride; + __lsx_vstelm_w(vec3, dst, 0, 1); + __lsx_vstelm_h(vec2, dst, 4, 1); + dst += stride; + __lsx_vstelm_w(vec3, dst, 0, 2); + __lsx_vstelm_h(vec2, dst, 4, 2); + dst += stride; + __lsx_vstelm_w(vec3, dst, 0, 3); + __lsx_vstelm_h(vec2, dst, 4, 3); + dst += stride; + __lsx_vstelm_w(vec4, dst, 0, 0); + __lsx_vstelm_h(vec2, dst, 4, 4); + dst += stride; + __lsx_vstelm_w(vec4, dst, 0, 1); + __lsx_vstelm_h(vec2, dst, 4, 5); + dst += stride; + __lsx_vstelm_w(vec4, dst, 0, 2); + __lsx_vstelm_h(vec2, dst, 4, 6); + dst += stride; + __lsx_vstelm_w(vec4, dst, 0, 3); + __lsx_vstelm_h(vec2, dst, 4, 7); + dst += stride; + __lsx_vstelm_w(vec6, dst, 0, 0); + __lsx_vstelm_h(vec5, dst, 4, 0); + dst += stride; + __lsx_vstelm_w(vec6, dst, 0, 1); + __lsx_vstelm_h(vec5, dst, 4, 1); + dst += stride; + __lsx_vstelm_w(vec6, dst, 0, 2); + __lsx_vstelm_h(vec5, dst, 4, 2); + dst += stride; + __lsx_vstelm_w(vec6, dst, 0, 3); + __lsx_vstelm_h(vec5, dst, 4, 3); + dst += stride; + __lsx_vstelm_w(vec7, dst, 0, 0); + __lsx_vstelm_h(vec5, dst, 4, 4); + dst += stride; + __lsx_vstelm_w(vec7, dst, 0, 1); + __lsx_vstelm_h(vec5, dst, 4, 5); + dst += stride; + __lsx_vstelm_w(vec7, dst, 0, 2); + __lsx_vstelm_h(vec5, dst, 4, 6); + dst += stride; + __lsx_vstelm_w(vec7, dst, 0, 3); + __lsx_vstelm_h(vec5, dst, 4, 7); + } +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/loopfilter_lsx.h b/media/libvpx/libvpx/vpx_dsp/loongarch/loopfilter_lsx.h new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/loopfilter_lsx.h @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VPX_VPX_DSP_LOONGARCH_LOOPFILTER_LSX_H_ +#define VPX_VPX_DSP_LOONGARCH_LOOPFILTER_LSX_H_ + +#include "vpx_util/loongson_intrinsics.h" + +#define LPF_MASK_HEV(p3_in, p2_in, p1_in, p0_in, q0_in, q1_in, q2_in, q3_in, \ + limit_in, b_limit_in, thresh_in, hev_out, mask_out, \ + flat_out) \ + { \ + __m128i p3_asub_p2_m, p2_asub_p1_m, p1_asub_p0_m, q1_asub_q0_m; \ + __m128i p1_asub_q1_m, p0_asub_q0_m, q3_asub_q2_m, q2_asub_q1_m; \ + \ + /* absolute subtraction of pixel values */ \ + p3_asub_p2_m = __lsx_vabsd_bu(p3_in, p2_in); \ + p2_asub_p1_m = __lsx_vabsd_bu(p2_in, p1_in); \ + p1_asub_p0_m = __lsx_vabsd_bu(p1_in, p0_in); \ + q1_asub_q0_m = __lsx_vabsd_bu(q1_in, q0_in); \ + q2_asub_q1_m = __lsx_vabsd_bu(q2_in, q1_in); \ + q3_asub_q2_m = __lsx_vabsd_bu(q3_in, q2_in); \ + p0_asub_q0_m = __lsx_vabsd_bu(p0_in, q0_in); \ + p1_asub_q1_m = __lsx_vabsd_bu(p1_in, q1_in); \ + \ + /* calculation of hev */ \ + flat_out = __lsx_vmax_bu(p1_asub_p0_m, q1_asub_q0_m); \ + hev_out = __lsx_vslt_bu(thresh_in, flat_out); \ + \ + /* calculation of mask */ \ + p0_asub_q0_m = __lsx_vsadd_bu(p0_asub_q0_m, p0_asub_q0_m); \ + p1_asub_q1_m = __lsx_vsrli_b(p1_asub_q1_m, 1); \ + p0_asub_q0_m = __lsx_vsadd_bu(p0_asub_q0_m, p1_asub_q1_m); \ + mask_out = __lsx_vslt_bu(b_limit_in, p0_asub_q0_m); \ + mask_out = __lsx_vmax_bu(flat_out, mask_out); \ + p3_asub_p2_m = __lsx_vmax_bu(p3_asub_p2_m, p2_asub_p1_m); \ + mask_out = __lsx_vmax_bu(p3_asub_p2_m, mask_out); \ + q2_asub_q1_m = __lsx_vmax_bu(q2_asub_q1_m, q3_asub_q2_m); \ + mask_out = __lsx_vmax_bu(q2_asub_q1_m, mask_out); \ + \ + mask_out = __lsx_vslt_bu(limit_in, mask_out); \ + mask_out = __lsx_vxori_b(mask_out, 0xff); \ + } + +#define VP9_FLAT4(p3_in, p2_in, p0_in, q0_in, q2_in, q3_in, flat_out) \ + { \ + __m128i p2_asub_p0, q2_asub_q0, p3_asub_p0, q3_asub_q0; \ + __m128i flat4_tmp = __lsx_vldi(1); \ + \ + DUP4_ARG2(__lsx_vabsd_bu, p2_in, p0_in, q2_in, q0_in, p3_in, p0_in, q3_in, \ + q0_in, p2_asub_p0, q2_asub_q0, p3_asub_p0, q3_asub_q0); \ + p2_asub_p0 = __lsx_vmax_bu(p2_asub_p0, q2_asub_q0); \ + flat_out = __lsx_vmax_bu(p2_asub_p0, flat_out); \ + p3_asub_p0 = __lsx_vmax_bu(p3_asub_p0, q3_asub_q0); \ + flat_out = __lsx_vmax_bu(p3_asub_p0, flat_out); \ + \ + flat_out = __lsx_vslt_bu(flat4_tmp, flat_out); \ + flat_out = __lsx_vxori_b(flat_out, 0xff); \ + flat_out = flat_out & (mask); \ + } + +#define VP9_FLAT5(p7_in, p6_in, p5_in, p4_in, p0_in, q0_in, q4_in, q5_in, \ + q6_in, q7_in, flat_in, flat2_out) \ + { \ + __m128i flat5_tmp = __lsx_vldi(1); \ + __m128i p4_asub_p0, q4_asub_q0, p5_asub_p0, q5_asub_q0; \ + __m128i p6_asub_p0, q6_asub_q0, p7_asub_p0, q7_asub_q0; \ + DUP4_ARG2(__lsx_vabsd_bu, p4_in, p0_in, q4_in, q0_in, p5_in, p0_in, q5_in, \ + q0_in, p4_asub_p0, q4_asub_q0, p5_asub_p0, q5_asub_q0); \ + DUP4_ARG2(__lsx_vabsd_bu, p6_in, p0_in, q6_in, q0_in, p7_in, p0_in, q7_in, \ + q0_in, p6_asub_p0, q6_asub_q0, p7_asub_p0, q7_asub_q0); \ + \ + DUP2_ARG2(__lsx_vmax_bu, p4_asub_p0, q4_asub_q0, p5_asub_p0, q5_asub_q0, \ + p4_asub_p0, flat2_out); \ + flat2_out = __lsx_vmax_bu(p4_asub_p0, flat2_out); \ + p6_asub_p0 = __lsx_vmax_bu(p6_asub_p0, q6_asub_q0); \ + flat2_out = __lsx_vmax_bu(p6_asub_p0, flat2_out); \ + p7_asub_p0 = __lsx_vmax_bu(p7_asub_p0, q7_asub_q0); \ + flat2_out = __lsx_vmax_bu(p7_asub_p0, flat2_out); \ + flat2_out = __lsx_vslt_bu(flat5_tmp, flat2_out); \ + flat2_out = __lsx_vxori_b(flat2_out, 0xff); \ + flat2_out = flat2_out & flat_in; \ + } + +#define VP9_LPF_FILTER4_4W(p1_in, p0_in, q0_in, q1_in, mask, hev, p1_out, \ + p0_out, q0_out, q1_out) \ + { \ + __m128i p1_m, p0_m, q0_m, q1_m, filt, q0_sub_p0, t1, t2; \ + const __m128i cnst4b = __lsx_vldi(4); \ + const __m128i cnst3b = __lsx_vldi(3); \ + DUP4_ARG2(__lsx_vxori_b, p1_in, 0x80, p0_in, 0x80, q0_in, 0x80, q1_in, \ + 0x80, p1_m, p0_m, q0_m, q1_m); \ + filt = __lsx_vssub_b(p1_m, q1_m); \ + filt &= hev; \ + \ + q0_sub_p0 = __lsx_vssub_b(q0_m, p0_m); \ + filt = __lsx_vsadd_b(filt, q0_sub_p0); \ + filt = __lsx_vsadd_b(filt, q0_sub_p0); \ + filt = __lsx_vsadd_b(filt, q0_sub_p0); \ + filt &= mask; \ + DUP2_ARG2(__lsx_vsadd_b, filt, cnst4b, filt, cnst3b, t1, t2); \ + DUP2_ARG2(__lsx_vsrai_b, t1, 3, t2, 3, t1, t2); \ + \ + q0_m = __lsx_vssub_b(q0_m, t1); \ + p0_m = __lsx_vsadd_b(p0_m, t2); \ + DUP2_ARG2(__lsx_vxori_b, q0_m, 0x80, p0_m, 0x80, q0_out, p0_out); \ + \ + filt = __lsx_vsrari_b(t1, 1); \ + hev = __lsx_vxori_b(hev, 0xff); \ + filt &= hev; \ + q1_m = __lsx_vssub_b(q1_m, filt); \ + p1_m = __lsx_vsadd_b(p1_m, filt); \ + DUP2_ARG2(__lsx_vxori_b, q1_m, 0x80, p1_m, 0x80, q1_out, p1_out); \ + } + +#define VP9_FILTER8(p3_in, p2_in, p1_in, p0_in, q0_in, q1_in, q2_in, q3_in, \ + p2_filt8_out, p1_filt8_out, p0_filt8_out, q0_filt8_out, \ + q1_filt8_out, q2_filt8_out) \ + { \ + __m128i tmp_filt8_0, tmp_filt8_1, tmp_filt8_2; \ + \ + tmp_filt8_2 = __lsx_vadd_h(p2_in, p1_in); \ + tmp_filt8_2 = __lsx_vadd_h(tmp_filt8_2, p0_in); \ + tmp_filt8_0 = __lsx_vslli_h(p3_in, 1); \ + \ + tmp_filt8_0 = __lsx_vadd_h(tmp_filt8_0, tmp_filt8_2); \ + tmp_filt8_0 = __lsx_vadd_h(tmp_filt8_0, q0_in); \ + tmp_filt8_1 = __lsx_vadd_h(tmp_filt8_0, p3_in); \ + tmp_filt8_1 = __lsx_vadd_h(tmp_filt8_1, p2_in); \ + p2_filt8_out = __lsx_vsrari_h(tmp_filt8_1, 3); \ + \ + tmp_filt8_1 = __lsx_vadd_h(tmp_filt8_0, p1_in); \ + tmp_filt8_1 = __lsx_vadd_h(tmp_filt8_1, q1_in); \ + p1_filt8_out = __lsx_vsrari_h(tmp_filt8_1, 3); \ + \ + tmp_filt8_1 = __lsx_vadd_h(q2_in, q1_in); \ + tmp_filt8_1 = __lsx_vadd_h(tmp_filt8_1, q0_in); \ + tmp_filt8_2 = __lsx_vadd_h(tmp_filt8_2, tmp_filt8_1); \ + tmp_filt8_0 = __lsx_vadd_h(tmp_filt8_2, p0_in); \ + tmp_filt8_0 = __lsx_vadd_h(tmp_filt8_0, p3_in); \ + p0_filt8_out = __lsx_vsrari_h(tmp_filt8_0, 3); \ + \ + tmp_filt8_0 = __lsx_vadd_h(q2_in, q3_in); \ + tmp_filt8_0 = __lsx_vadd_h(p0_in, tmp_filt8_0); \ + tmp_filt8_0 = __lsx_vadd_h(tmp_filt8_0, tmp_filt8_1); \ + tmp_filt8_1 = __lsx_vadd_h(q3_in, q3_in); \ + tmp_filt8_1 = __lsx_vadd_h(tmp_filt8_1, tmp_filt8_0); \ + q2_filt8_out = __lsx_vsrari_h(tmp_filt8_1, 3); \ + \ + tmp_filt8_0 = __lsx_vadd_h(tmp_filt8_2, q3_in); \ + tmp_filt8_1 = __lsx_vadd_h(tmp_filt8_0, q0_in); \ + q0_filt8_out = __lsx_vsrari_h(tmp_filt8_1, 3); \ + \ + tmp_filt8_1 = __lsx_vsub_h(tmp_filt8_0, p2_in); \ + tmp_filt8_0 = __lsx_vadd_h(q1_in, q3_in); \ + tmp_filt8_1 = __lsx_vadd_h(tmp_filt8_0, tmp_filt8_1); \ + q1_filt8_out = __lsx_vsrari_h(tmp_filt8_1, 3); \ + } + +#endif // VPX_VPX_DSP_LOONGARCH_LOOPFILTER_LSX_H_ diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/sad_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/sad_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/sad_lsx.c @@ -0,0 +1,672 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "./vpx_dsp_rtcd.h" +#include "vpx_util/loongson_intrinsics.h" + +#define SAD_UB2_UH(in0, in1, ref0, ref1) \ + ({ \ + __m128i diff0_m, diff1_m, sad_m0; \ + __m128i sad_m = __lsx_vldi(0); \ + \ + diff0_m = __lsx_vabsd_bu(in0, ref0); \ + diff1_m = __lsx_vabsd_bu(in1, ref1); \ + \ + sad_m0 = __lsx_vhaddw_hu_bu(diff0_m, diff0_m); \ + sad_m = __lsx_vadd_h(sad_m, sad_m0); \ + sad_m0 = __lsx_vhaddw_hu_bu(diff1_m, diff1_m); \ + sad_m = __lsx_vadd_h(sad_m, sad_m0); \ + \ + sad_m; \ + }) + +#define HADD_UW_U32(in) \ + ({ \ + __m128i res0_m; \ + uint32_t sum_m; \ + res0_m = __lsx_vhaddw_du_wu(in, in); \ + res0_m = __lsx_vhaddw_qu_du(res0_m, res0_m); \ + sum_m = __lsx_vpickve2gr_w(res0_m, 0); \ + sum_m; \ + }) + +#define HADD_UH_U32(in) \ + ({ \ + __m128i res_m; \ + uint32_t sum_m; \ + res_m = __lsx_vhaddw_wu_hu(in, in); \ + sum_m = HADD_UW_U32(res_m); \ + sum_m; \ + }) + +#define HADD_SW_S32(in) \ + ({ \ + __m128i res0_m; \ + int32_t sum_m; \ + \ + res0_m = __lsx_vhaddw_d_w(in, in); \ + res0_m = __lsx_vhaddw_q_d(res0_m, res0_m); \ + sum_m = __lsx_vpickve2gr_w(res0_m, 0); \ + sum_m; \ + }) + +static uint32_t sad_16width_lsx(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + int32_t height) { + int32_t ht_cnt = (height >> 2); + __m128i src0, src1, ref0, ref1, sad_tmp; + __m128i sad = __lsx_vldi(0); + int32_t src_stride2 = src_stride << 1; + int32_t ref_stride2 = ref_stride << 1; + + for (; ht_cnt--;) { + DUP2_ARG2(__lsx_vld, src, 0, ref, 0, src0, ref0); + DUP2_ARG2(__lsx_vldx, src, src_stride, ref, ref_stride, src1, ref1); + src += src_stride2; + ref += ref_stride2; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad = __lsx_vadd_h(sad, sad_tmp); + + DUP2_ARG2(__lsx_vld, src, 0, ref, 0, src0, ref0); + DUP2_ARG2(__lsx_vldx, src, src_stride, ref, ref_stride, src1, ref1); + src += src_stride2; + ref += ref_stride2; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad = __lsx_vadd_h(sad, sad_tmp); + } + return HADD_UH_U32(sad); +} + +static uint32_t sad_32width_lsx(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + int32_t height) { + int32_t ht_cnt = (height >> 2); + __m128i src0, src1, ref0, ref1; + __m128i sad_tmp; + __m128i sad = __lsx_vldi(0); + + for (; ht_cnt--;) { + DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1); + src += src_stride; + DUP2_ARG2(__lsx_vld, ref, 0, ref, 16, ref0, ref1); + ref += ref_stride; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad = __lsx_vadd_h(sad, sad_tmp); + + DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1); + src += src_stride; + DUP2_ARG2(__lsx_vld, ref, 0, ref, 16, ref0, ref1); + ref += ref_stride; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad = __lsx_vadd_h(sad, sad_tmp); + + DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1); + src += src_stride; + DUP2_ARG2(__lsx_vld, ref, 0, ref, 16, ref0, ref1); + ref += ref_stride; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad = __lsx_vadd_h(sad, sad_tmp); + + DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1); + src += src_stride; + DUP2_ARG2(__lsx_vld, ref, 0, ref, 16, ref0, ref1); + ref += ref_stride; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad = __lsx_vadd_h(sad, sad_tmp); + } + return HADD_UH_U32(sad); +} + +static uint32_t sad_64width_lsx(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + int32_t height) { + int32_t ht_cnt = (height >> 1); + uint32_t sad = 0; + __m128i src0, src1, src2, src3; + __m128i ref0, ref1, ref2, ref3; + __m128i sad_tmp; + __m128i sad0 = __lsx_vldi(0); + __m128i sad1 = sad0; + + for (; ht_cnt--;) { + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2, + src3); + src += src_stride; + DUP4_ARG2(__lsx_vld, ref, 0, ref, 16, ref, 32, ref, 48, ref0, ref1, ref2, + ref3); + ref += ref_stride; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad0 = __lsx_vadd_h(sad0, sad_tmp); + sad_tmp = SAD_UB2_UH(src2, src3, ref2, ref3); + sad1 = __lsx_vadd_h(sad1, sad_tmp); + + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2, + src3); + src += src_stride; + DUP4_ARG2(__lsx_vld, ref, 0, ref, 16, ref, 32, ref, 48, ref0, ref1, ref2, + ref3); + ref += ref_stride; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad0 = __lsx_vadd_h(sad0, sad_tmp); + sad_tmp = SAD_UB2_UH(src2, src3, ref2, ref3); + sad1 = __lsx_vadd_h(sad1, sad_tmp); + } + + sad = HADD_UH_U32(sad0); + sad += HADD_UH_U32(sad1); + + return sad; +} + +static void sad_8width_x4d_lsx(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *const aref_ptr[], + int32_t ref_stride, int32_t height, + uint32_t *sad_array) { + int32_t ht_cnt = (height >> 2); + uint8_t *ref0_ptr, *ref1_ptr, *ref2_ptr, *ref3_ptr; + __m128i src0, src1, src2, src3, sad_tmp; + __m128i ref0, ref1, ref2, ref3, ref4, ref5, ref6, ref7; + __m128i ref8, ref9, ref10, ref11, ref12, ref13, ref14, ref15; + __m128i sad0 = __lsx_vldi(0); + __m128i sad1 = sad0; + __m128i sad2 = sad0; + __m128i sad3 = sad0; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + int32_t ref_stride2 = ref_stride << 1; + int32_t ref_stride3 = ref_stride2 + ref_stride; + int32_t ref_stride4 = ref_stride2 << 1; + + ref0_ptr = aref_ptr[0]; + ref1_ptr = aref_ptr[1]; + ref2_ptr = aref_ptr[2]; + ref3_ptr = aref_ptr[3]; + + for (; ht_cnt--;) { + src0 = __lsx_vld(src_ptr, 0); + DUP2_ARG2(__lsx_vldx, src_ptr, src_stride, src_ptr, src_stride2, src1, + src2); + src3 = __lsx_vldx(src_ptr, src_stride3); + src_ptr += src_stride4; + ref0 = __lsx_vld(ref0_ptr, 0); + DUP2_ARG2(__lsx_vldx, ref0_ptr, ref_stride, ref0_ptr, ref_stride2, ref1, + ref2); + ref3 = __lsx_vldx(ref0_ptr, ref_stride3); + ref0_ptr += ref_stride4; + ref4 = __lsx_vld(ref1_ptr, 0); + DUP2_ARG2(__lsx_vldx, ref1_ptr, ref_stride, ref1_ptr, ref_stride2, ref5, + ref6); + ref7 = __lsx_vldx(ref1_ptr, ref_stride3); + ref1_ptr += ref_stride4; + ref8 = __lsx_vld(ref2_ptr, 0); + DUP2_ARG2(__lsx_vldx, ref2_ptr, ref_stride, ref2_ptr, ref_stride2, ref9, + ref10); + ref11 = __lsx_vldx(ref2_ptr, ref_stride3); + ref2_ptr += ref_stride4; + ref12 = __lsx_vld(ref3_ptr, 0); + DUP2_ARG2(__lsx_vldx, ref3_ptr, ref_stride, ref3_ptr, ref_stride2, ref13, + ref14); + ref15 = __lsx_vldx(ref3_ptr, ref_stride3); + ref3_ptr += ref_stride4; + + DUP2_ARG2(__lsx_vpickev_d, src1, src0, src3, src2, src0, src1); + DUP2_ARG2(__lsx_vpickev_d, ref1, ref0, ref3, ref2, ref0, ref1); + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad0 = __lsx_vadd_h(sad0, sad_tmp); + + DUP2_ARG2(__lsx_vpickev_d, ref5, ref4, ref7, ref6, ref0, ref1); + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad1 = __lsx_vadd_h(sad1, sad_tmp); + + DUP2_ARG2(__lsx_vpickev_d, ref9, ref8, ref11, ref10, ref0, ref1); + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad2 = __lsx_vadd_h(sad2, sad_tmp); + + DUP2_ARG2(__lsx_vpickev_d, ref13, ref12, ref15, ref14, ref0, ref1); + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad3 = __lsx_vadd_h(sad3, sad_tmp); + } + sad_array[0] = HADD_UH_U32(sad0); + sad_array[1] = HADD_UH_U32(sad1); + sad_array[2] = HADD_UH_U32(sad2); + sad_array[3] = HADD_UH_U32(sad3); +} + +static void sad_16width_x4d_lsx(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *const aref_ptr[], + int32_t ref_stride, int32_t height, + uint32_t *sad_array) { + int32_t ht_cnt = (height >> 1); + const uint8_t *ref0_ptr, *ref1_ptr, *ref2_ptr, *ref3_ptr; + __m128i src, ref0, ref1, ref2, ref3, diff, sad_tmp; + __m128i sad0 = __lsx_vldi(0); + __m128i sad1 = sad0; + __m128i sad2 = sad0; + __m128i sad3 = sad0; + + ref0_ptr = aref_ptr[0]; + ref1_ptr = aref_ptr[1]; + ref2_ptr = aref_ptr[2]; + ref3_ptr = aref_ptr[3]; + + for (; ht_cnt--;) { + src = __lsx_vld(src_ptr, 0); + src_ptr += src_stride; + ref0 = __lsx_vld(ref0_ptr, 0); + ref0_ptr += ref_stride; + ref1 = __lsx_vld(ref1_ptr, 0); + ref1_ptr += ref_stride; + ref2 = __lsx_vld(ref2_ptr, 0); + ref2_ptr += ref_stride; + ref3 = __lsx_vld(ref3_ptr, 0); + ref3_ptr += ref_stride; + + diff = __lsx_vabsd_bu(src, ref0); + sad_tmp = __lsx_vhaddw_hu_bu(diff, diff); + sad0 = __lsx_vadd_h(sad0, sad_tmp); + diff = __lsx_vabsd_bu(src, ref1); + sad_tmp = __lsx_vhaddw_hu_bu(diff, diff); + sad1 = __lsx_vadd_h(sad1, sad_tmp); + diff = __lsx_vabsd_bu(src, ref2); + sad_tmp = __lsx_vhaddw_hu_bu(diff, diff); + sad2 = __lsx_vadd_h(sad2, sad_tmp); + diff = __lsx_vabsd_bu(src, ref3); + sad_tmp = __lsx_vhaddw_hu_bu(diff, diff); + sad3 = __lsx_vadd_h(sad3, sad_tmp); + + src = __lsx_vld(src_ptr, 0); + src_ptr += src_stride; + ref0 = __lsx_vld(ref0_ptr, 0); + ref0_ptr += ref_stride; + ref1 = __lsx_vld(ref1_ptr, 0); + ref1_ptr += ref_stride; + ref2 = __lsx_vld(ref2_ptr, 0); + ref2_ptr += ref_stride; + ref3 = __lsx_vld(ref3_ptr, 0); + ref3_ptr += ref_stride; + + diff = __lsx_vabsd_bu(src, ref0); + sad_tmp = __lsx_vhaddw_hu_bu(diff, diff); + sad0 = __lsx_vadd_h(sad0, sad_tmp); + diff = __lsx_vabsd_bu(src, ref1); + sad_tmp = __lsx_vhaddw_hu_bu(diff, diff); + sad1 = __lsx_vadd_h(sad1, sad_tmp); + diff = __lsx_vabsd_bu(src, ref2); + sad_tmp = __lsx_vhaddw_hu_bu(diff, diff); + sad2 = __lsx_vadd_h(sad2, sad_tmp); + diff = __lsx_vabsd_bu(src, ref3); + sad_tmp = __lsx_vhaddw_hu_bu(diff, diff); + sad3 = __lsx_vadd_h(sad3, sad_tmp); + } + sad_array[0] = HADD_UH_U32(sad0); + sad_array[1] = HADD_UH_U32(sad1); + sad_array[2] = HADD_UH_U32(sad2); + sad_array[3] = HADD_UH_U32(sad3); +} + +static void sad_32width_x4d_lsx(const uint8_t *src, int32_t src_stride, + const uint8_t *const aref_ptr[], + int32_t ref_stride, int32_t height, + uint32_t *sad_array) { + const uint8_t *ref0_ptr, *ref1_ptr, *ref2_ptr, *ref3_ptr; + int32_t ht_cnt = height; + __m128i src0, src1, ref0, ref1, sad_tmp; + __m128i sad0 = __lsx_vldi(0); + __m128i sad1 = sad0; + __m128i sad2 = sad0; + __m128i sad3 = sad0; + + ref0_ptr = aref_ptr[0]; + ref1_ptr = aref_ptr[1]; + ref2_ptr = aref_ptr[2]; + ref3_ptr = aref_ptr[3]; + + for (; ht_cnt--;) { + DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1); + src += src_stride; + + DUP2_ARG2(__lsx_vld, ref0_ptr, 0, ref0_ptr, 16, ref0, ref1); + ref0_ptr += ref_stride; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad0 = __lsx_vadd_h(sad0, sad_tmp); + + DUP2_ARG2(__lsx_vld, ref1_ptr, 0, ref1_ptr, 16, ref0, ref1); + ref1_ptr += ref_stride; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad1 = __lsx_vadd_h(sad1, sad_tmp); + + DUP2_ARG2(__lsx_vld, ref2_ptr, 0, ref2_ptr, 16, ref0, ref1); + ref2_ptr += ref_stride; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad2 = __lsx_vadd_h(sad2, sad_tmp); + + DUP2_ARG2(__lsx_vld, ref3_ptr, 0, ref3_ptr, 16, ref0, ref1); + ref3_ptr += ref_stride; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad3 = __lsx_vadd_h(sad3, sad_tmp); + } + sad_array[0] = HADD_UH_U32(sad0); + sad_array[1] = HADD_UH_U32(sad1); + sad_array[2] = HADD_UH_U32(sad2); + sad_array[3] = HADD_UH_U32(sad3); +} + +static void sad_64width_x4d_lsx(const uint8_t *src, int32_t src_stride, + const uint8_t *const aref_ptr[], + int32_t ref_stride, int32_t height, + uint32_t *sad_array) { + const uint8_t *ref0_ptr, *ref1_ptr, *ref2_ptr, *ref3_ptr; + int32_t ht_cnt = height; + __m128i src0, src1, src2, src3; + __m128i ref0, ref1, ref2, ref3; + __m128i sad, sad_tmp; + + __m128i sad0_0 = __lsx_vldi(0); + __m128i sad0_1 = sad0_0; + __m128i sad1_0 = sad0_0; + __m128i sad1_1 = sad0_0; + __m128i sad2_0 = sad0_0; + __m128i sad2_1 = sad0_0; + __m128i sad3_0 = sad0_0; + __m128i sad3_1 = sad0_0; + + ref0_ptr = aref_ptr[0]; + ref1_ptr = aref_ptr[1]; + ref2_ptr = aref_ptr[2]; + ref3_ptr = aref_ptr[3]; + + for (; ht_cnt--;) { + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2, + src3); + src += src_stride; + + DUP4_ARG2(__lsx_vld, ref0_ptr, 0, ref0_ptr, 16, ref0_ptr, 32, ref0_ptr, 48, + ref0, ref1, ref2, ref3); + ref0_ptr += ref_stride; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad0_0 = __lsx_vadd_h(sad0_0, sad_tmp); + sad_tmp = SAD_UB2_UH(src2, src3, ref2, ref3); + sad0_1 = __lsx_vadd_h(sad0_1, sad_tmp); + + DUP4_ARG2(__lsx_vld, ref1_ptr, 0, ref1_ptr, 16, ref1_ptr, 32, ref1_ptr, 48, + ref0, ref1, ref2, ref3); + ref1_ptr += ref_stride; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad1_0 = __lsx_vadd_h(sad1_0, sad_tmp); + sad_tmp = SAD_UB2_UH(src2, src3, ref2, ref3); + sad1_1 = __lsx_vadd_h(sad1_1, sad_tmp); + + DUP4_ARG2(__lsx_vld, ref2_ptr, 0, ref2_ptr, 16, ref2_ptr, 32, ref2_ptr, 48, + ref0, ref1, ref2, ref3); + ref2_ptr += ref_stride; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad2_0 = __lsx_vadd_h(sad2_0, sad_tmp); + sad_tmp = SAD_UB2_UH(src2, src3, ref2, ref3); + sad2_1 = __lsx_vadd_h(sad2_1, sad_tmp); + + DUP4_ARG2(__lsx_vld, ref3_ptr, 0, ref3_ptr, 16, ref3_ptr, 32, ref3_ptr, 48, + ref0, ref1, ref2, ref3); + ref3_ptr += ref_stride; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad3_0 = __lsx_vadd_h(sad3_0, sad_tmp); + sad_tmp = SAD_UB2_UH(src2, src3, ref2, ref3); + sad3_1 = __lsx_vadd_h(sad3_1, sad_tmp); + } + sad = __lsx_vhaddw_wu_hu(sad0_0, sad0_0); + sad_tmp = __lsx_vhaddw_wu_hu(sad0_1, sad0_1); + sad = __lsx_vadd_w(sad, sad_tmp); + sad_array[0] = HADD_UW_U32(sad); + + sad = __lsx_vhaddw_wu_hu(sad1_0, sad1_0); + sad_tmp = __lsx_vhaddw_wu_hu(sad1_1, sad1_1); + sad = __lsx_vadd_w(sad, sad_tmp); + sad_array[1] = HADD_UW_U32(sad); + + sad = __lsx_vhaddw_wu_hu(sad2_0, sad2_0); + sad_tmp = __lsx_vhaddw_wu_hu(sad2_1, sad2_1); + sad = __lsx_vadd_w(sad, sad_tmp); + sad_array[2] = HADD_UW_U32(sad); + + sad = __lsx_vhaddw_wu_hu(sad3_0, sad3_0); + sad_tmp = __lsx_vhaddw_wu_hu(sad3_1, sad3_1); + sad = __lsx_vadd_w(sad, sad_tmp); + sad_array[3] = HADD_UW_U32(sad); +} + +static uint32_t avgsad_32width_lsx(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + int32_t height, const uint8_t *sec_pred) { + int32_t ht_cnt = (height >> 2); + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i ref0, ref1, ref2, ref3, ref4, ref5, ref6, ref7; + __m128i pred0, pred1, pred2, pred3, pred4, pred5, pred6, pred7; + __m128i comp0, comp1, sad_tmp; + __m128i sad = __lsx_vldi(0); + uint8_t *src_tmp, *ref_tmp; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + int32_t ref_stride2 = ref_stride << 1; + int32_t ref_stride3 = ref_stride2 + ref_stride; + int32_t ref_stride4 = ref_stride2 << 1; + + for (; ht_cnt--;) { + src_tmp = (uint8_t *)src + 16; + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src4); + src6 = __lsx_vldx(src, src_stride3); + src1 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src3, + src5); + src7 = __lsx_vldx(src_tmp, src_stride3); + src += src_stride4; + + ref_tmp = (uint8_t *)ref + 16; + ref0 = __lsx_vld(ref, 0); + DUP2_ARG2(__lsx_vldx, ref, ref_stride, ref, ref_stride2, ref2, ref4); + ref6 = __lsx_vldx(ref, ref_stride3); + ref1 = __lsx_vld(ref_tmp, 0); + DUP2_ARG2(__lsx_vldx, ref_tmp, ref_stride, ref_tmp, ref_stride2, ref3, + ref5); + ref7 = __lsx_vldx(ref_tmp, ref_stride3); + ref += ref_stride4; + + DUP4_ARG2(__lsx_vld, sec_pred, 0, sec_pred, 32, sec_pred, 64, sec_pred, 96, + pred0, pred2, pred4, pred6); + DUP4_ARG2(__lsx_vld, sec_pred, 16, sec_pred, 48, sec_pred, 80, sec_pred, + 112, pred1, pred3, pred5, pred7); + sec_pred += 128; + + DUP2_ARG2(__lsx_vavgr_bu, pred0, ref0, pred1, ref1, comp0, comp1); + sad_tmp = SAD_UB2_UH(src0, src1, comp0, comp1); + sad = __lsx_vadd_h(sad, sad_tmp); + DUP2_ARG2(__lsx_vavgr_bu, pred2, ref2, pred3, ref3, comp0, comp1); + sad_tmp = SAD_UB2_UH(src2, src3, comp0, comp1); + sad = __lsx_vadd_h(sad, sad_tmp); + DUP2_ARG2(__lsx_vavgr_bu, pred4, ref4, pred5, ref5, comp0, comp1); + sad_tmp = SAD_UB2_UH(src4, src5, comp0, comp1); + sad = __lsx_vadd_h(sad, sad_tmp); + DUP2_ARG2(__lsx_vavgr_bu, pred6, ref6, pred7, ref7, comp0, comp1); + sad_tmp = SAD_UB2_UH(src6, src7, comp0, comp1); + sad = __lsx_vadd_h(sad, sad_tmp); + } + + return HADD_UH_U32(sad); +} + +static uint32_t avgsad_64width_lsx(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + int32_t height, const uint8_t *sec_pred) { + int32_t ht_cnt = (height >> 2); + __m128i src0, src1, src2, src3, ref0, ref1, ref2, ref3; + __m128i comp0, comp1, comp2, comp3, pred0, pred1, pred2, pred3; + __m128i sad, sad_tmp; + __m128i sad0 = __lsx_vldi(0); + __m128i sad1 = sad0; + + for (; ht_cnt--;) { + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2, + src3); + src += src_stride; + DUP4_ARG2(__lsx_vld, ref, 0, ref, 16, ref, 32, ref, 48, ref0, ref1, ref2, + ref3); + ref += ref_stride; + DUP4_ARG2(__lsx_vld, sec_pred, 0, sec_pred, 16, sec_pred, 32, sec_pred, 48, + pred0, pred1, pred2, pred3); + sec_pred += 64; + DUP4_ARG2(__lsx_vavgr_bu, pred0, ref0, pred1, ref1, pred2, ref2, pred3, + ref3, comp0, comp1, comp2, comp3); + sad_tmp = SAD_UB2_UH(src0, src1, comp0, comp1); + sad0 = __lsx_vadd_h(sad0, sad_tmp); + sad_tmp = SAD_UB2_UH(src2, src3, comp2, comp3); + sad1 = __lsx_vadd_h(sad1, sad_tmp); + + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2, + src3); + src += src_stride; + DUP4_ARG2(__lsx_vld, ref, 0, ref, 16, ref, 32, ref, 48, ref0, ref1, ref2, + ref3); + ref += ref_stride; + DUP4_ARG2(__lsx_vld, sec_pred, 0, sec_pred, 16, sec_pred, 32, sec_pred, 48, + pred0, pred1, pred2, pred3); + sec_pred += 64; + DUP4_ARG2(__lsx_vavgr_bu, pred0, ref0, pred1, ref1, pred2, ref2, pred3, + ref3, comp0, comp1, comp2, comp3); + sad_tmp = SAD_UB2_UH(src0, src1, comp0, comp1); + sad0 = __lsx_vadd_h(sad0, sad_tmp); + sad_tmp = SAD_UB2_UH(src2, src3, comp2, comp3); + sad1 = __lsx_vadd_h(sad1, sad_tmp); + + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2, + src3); + src += src_stride; + DUP4_ARG2(__lsx_vld, ref, 0, ref, 16, ref, 32, ref, 48, ref0, ref1, ref2, + ref3); + ref += ref_stride; + DUP4_ARG2(__lsx_vld, sec_pred, 0, sec_pred, 16, sec_pred, 32, sec_pred, 48, + pred0, pred1, pred2, pred3); + sec_pred += 64; + DUP4_ARG2(__lsx_vavgr_bu, pred0, ref0, pred1, ref1, pred2, ref2, pred3, + ref3, comp0, comp1, comp2, comp3); + sad_tmp = SAD_UB2_UH(src0, src1, comp0, comp1); + sad0 = __lsx_vadd_h(sad0, sad_tmp); + sad_tmp = SAD_UB2_UH(src2, src3, comp2, comp3); + sad1 = __lsx_vadd_h(sad1, sad_tmp); + + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2, + src3); + src += src_stride; + DUP4_ARG2(__lsx_vld, ref, 0, ref, 16, ref, 32, ref, 48, ref0, ref1, ref2, + ref3); + ref += ref_stride; + DUP4_ARG2(__lsx_vld, sec_pred, 0, sec_pred, 16, sec_pred, 32, sec_pred, 48, + pred0, pred1, pred2, pred3); + sec_pred += 64; + DUP4_ARG2(__lsx_vavgr_bu, pred0, ref0, pred1, ref1, pred2, ref2, pred3, + ref3, comp0, comp1, comp2, comp3); + sad_tmp = SAD_UB2_UH(src0, src1, comp0, comp1); + sad0 = __lsx_vadd_h(sad0, sad_tmp); + sad_tmp = SAD_UB2_UH(src2, src3, comp2, comp3); + sad1 = __lsx_vadd_h(sad1, sad_tmp); + } + sad = __lsx_vhaddw_wu_hu(sad0, sad0); + sad_tmp = __lsx_vhaddw_wu_hu(sad1, sad1); + sad = __lsx_vadd_w(sad, sad_tmp); + + return HADD_SW_S32(sad); +} + +#define VPX_SAD_16xHT_LSX(height) \ + uint32_t vpx_sad16x##height##_lsx(const uint8_t *src, int32_t src_stride, \ + const uint8_t *ref, int32_t ref_stride) { \ + return sad_16width_lsx(src, src_stride, ref, ref_stride, height); \ + } + +#define VPX_SAD_32xHT_LSX(height) \ + uint32_t vpx_sad32x##height##_lsx(const uint8_t *src, int32_t src_stride, \ + const uint8_t *ref, int32_t ref_stride) { \ + return sad_32width_lsx(src, src_stride, ref, ref_stride, height); \ + } + +#define VPX_SAD_64xHT_LSX(height) \ + uint32_t vpx_sad64x##height##_lsx(const uint8_t *src, int32_t src_stride, \ + const uint8_t *ref, int32_t ref_stride) { \ + return sad_64width_lsx(src, src_stride, ref, ref_stride, height); \ + } + +#define VPX_SAD_8xHTx4D_LSX(height) \ + void vpx_sad8x##height##x4d_lsx(const uint8_t *src, int32_t src_stride, \ + const uint8_t *const refs[4], \ + int32_t ref_stride, uint32_t sads[4]) { \ + sad_8width_x4d_lsx(src, src_stride, refs, ref_stride, height, sads); \ + } + +#define VPX_SAD_16xHTx4D_LSX(height) \ + void vpx_sad16x##height##x4d_lsx(const uint8_t *src, int32_t src_stride, \ + const uint8_t *const refs[], \ + int32_t ref_stride, uint32_t *sads) { \ + sad_16width_x4d_lsx(src, src_stride, refs, ref_stride, height, sads); \ + } + +#define VPX_SAD_32xHTx4D_LSX(height) \ + void vpx_sad32x##height##x4d_lsx(const uint8_t *src, int32_t src_stride, \ + const uint8_t *const refs[], \ + int32_t ref_stride, uint32_t *sads) { \ + sad_32width_x4d_lsx(src, src_stride, refs, ref_stride, height, sads); \ + } + +#define VPX_SAD_64xHTx4D_LSX(height) \ + void vpx_sad64x##height##x4d_lsx(const uint8_t *src, int32_t src_stride, \ + const uint8_t *const refs[], \ + int32_t ref_stride, uint32_t *sads) { \ + sad_64width_x4d_lsx(src, src_stride, refs, ref_stride, height, sads); \ + } + +#define VPX_AVGSAD_32xHT_LSX(height) \ + uint32_t vpx_sad32x##height##_avg_lsx( \ + const uint8_t *src, int32_t src_stride, const uint8_t *ref, \ + int32_t ref_stride, const uint8_t *second_pred) { \ + return avgsad_32width_lsx(src, src_stride, ref, ref_stride, height, \ + second_pred); \ + } + +#define VPX_AVGSAD_64xHT_LSX(height) \ + uint32_t vpx_sad64x##height##_avg_lsx( \ + const uint8_t *src, int32_t src_stride, const uint8_t *ref, \ + int32_t ref_stride, const uint8_t *second_pred) { \ + return avgsad_64width_lsx(src, src_stride, ref, ref_stride, height, \ + second_pred); \ + } + +#define SAD64 \ + VPX_SAD_64xHT_LSX(64) VPX_SAD_64xHTx4D_LSX(64) VPX_SAD_64xHTx4D_LSX(32) \ + VPX_AVGSAD_64xHT_LSX(64) + +SAD64 + +#define SAD32 \ + VPX_SAD_32xHT_LSX(32) VPX_SAD_32xHTx4D_LSX(32) VPX_SAD_32xHTx4D_LSX(64) \ + VPX_AVGSAD_32xHT_LSX(32) + +SAD32 + +#define SAD16 VPX_SAD_16xHT_LSX(16) VPX_SAD_16xHTx4D_LSX(16) + +SAD16 + +#define SAD8 VPX_SAD_8xHTx4D_LSX(8) + +SAD8 + +#undef SAD64 +#undef SAD32 +#undef SAD16 +#undef SAD8 diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/sub_pixel_variance_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/sub_pixel_variance_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/sub_pixel_variance_lsx.c @@ -0,0 +1,348 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "./vpx_dsp_rtcd.h" +#include "vpx_ports/mem.h" +#include "vpx_util/loongson_intrinsics.h" +#include "vpx_dsp/variance.h" + +#define HADD_SW_S32(in0, in1) \ + do { \ + __m128i res0_m; \ + \ + res0_m = __lsx_vhaddw_d_w(in0, in0); \ + res0_m = __lsx_vhaddw_q_d(res0_m, res0_m); \ + in1 = __lsx_vpickve2gr_w(res0_m, 0); \ + } while (0) + +#define HORIZ_2TAP_FILT_UH(in0, in1, mask, coeff, shift, in2) \ + do { \ + __m128i tmp0_m, tmp1_m; \ + \ + tmp0_m = __lsx_vshuf_b(in1, in0, mask); \ + tmp1_m = __lsx_vdp2_h_bu(tmp0_m, coeff); \ + in2 = __lsx_vsrari_h(tmp1_m, shift); \ + } while (0) + +#define CALC_MSE_AVG_B(src, ref, var, sub) \ + { \ + __m128i src_l0_m, src_l1_m; \ + __m128i res_l0_m, res_l1_m; \ + \ + src_l0_m = __lsx_vilvl_b(src, ref); \ + src_l1_m = __lsx_vilvh_b(src, ref); \ + DUP2_ARG2(__lsx_vhsubw_hu_bu, src_l0_m, src_l0_m, src_l1_m, src_l1_m, \ + res_l0_m, res_l1_m); \ + var = __lsx_vdp2add_w_h(var, res_l0_m, res_l0_m); \ + var = __lsx_vdp2add_w_h(var, res_l1_m, res_l1_m); \ + sub = __lsx_vadd_h(sub, res_l0_m); \ + sub = __lsx_vadd_h(sub, res_l1_m); \ + } + +static const uint8_t bilinear_filters_lsx[8][2] = { + { 128, 0 }, { 112, 16 }, { 96, 32 }, { 80, 48 }, + { 64, 64 }, { 48, 80 }, { 32, 96 }, { 16, 112 }, +}; + +#define VARIANCE_LARGE_WxH(sse, diff, shift) \ + (sse) - (((int64_t)(diff) * (diff)) >> (shift)) + +static uint32_t sub_pixel_sse_diff_16width_h_lsx( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *filter, int32_t height, int32_t *diff) { + uint32_t loop_cnt = (height >> 2); + int32_t res; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i dst0, dst1, dst2, dst3, filt0; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i out0, out1, out2, out3, out4, out5, out6, out7; + __m128i vec, var = __lsx_vldi(0); + __m128i avg = var; + __m128i mask = { 0x0403030202010100, 0x0807070606050504 }; + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + int32_t dst_stride4 = dst_stride2 << 1; + + filt0 = __lsx_vldrepl_h(filter, 0); + + for (; loop_cnt--;) { + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src0, src1); + src += src_stride; + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src2, src3); + src += src_stride; + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src4, src5); + src += src_stride; + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src6, src7); + src += src_stride; + + dst0 = __lsx_vld(dst, 0); + DUP2_ARG2(__lsx_vldx, dst, dst_stride, dst, dst_stride2, dst1, dst2); + dst3 = __lsx_vldx(dst, dst_stride3); + dst += dst_stride4; + + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1); + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3); + DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask, src5, src5, mask, vec4, vec5); + DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask, src7, src7, mask, vec6, vec7); + + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, + filt0, out0, out1, out2, out3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7, + filt0, out4, out5, out6, out7); + DUP4_ARG3(__lsx_vssrarni_bu_h, out1, out0, FILTER_BITS, out3, out2, + FILTER_BITS, out5, out4, FILTER_BITS, out7, out6, FILTER_BITS, + src0, src1, src2, src3); + CALC_MSE_AVG_B(src0, dst0, var, avg); + CALC_MSE_AVG_B(src1, dst1, var, avg); + CALC_MSE_AVG_B(src2, dst2, var, avg); + CALC_MSE_AVG_B(src3, dst3, var, avg); + } + vec = __lsx_vhaddw_w_h(avg, avg); + HADD_SW_S32(vec, *diff); + HADD_SW_S32(var, res); + + return res; +} + +static uint32_t sub_pixel_sse_diff_32width_h_lsx( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *filter, int32_t height, int32_t *diff) { + uint32_t sse = 0; + int32_t diff0[2]; + + sse += sub_pixel_sse_diff_16width_h_lsx(src, src_stride, dst, dst_stride, + filter, height, &diff0[0]); + src += 16; + dst += 16; + + sse += sub_pixel_sse_diff_16width_h_lsx(src, src_stride, dst, dst_stride, + filter, height, &diff0[1]); + + *diff = diff0[0] + diff0[1]; + + return sse; +} + +static uint32_t sub_pixel_sse_diff_16width_v_lsx( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *filter, int32_t height, int32_t *diff) { + uint32_t loop_cnt = (height >> 2); + int32_t res; + __m128i ref0, ref1, ref2, ref3, src0, src1, src2, src3, src4; + __m128i out0, out1, out2, out3, tmp0, tmp1, filt0, vec; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i var = __lsx_vldi(0); + __m128i avg = var; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + int32_t dst_stride4 = dst_stride2 << 1; + + filt0 = __lsx_vldrepl_h(filter, 0); + + src0 = __lsx_vld(src, 0); + src += src_stride; + + for (; loop_cnt--;) { + src1 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src3); + src4 = __lsx_vldx(src, src_stride3); + src += src_stride4; + ref0 = __lsx_vld(dst, 0); + DUP2_ARG2(__lsx_vldx, dst, dst_stride, dst, dst_stride2, ref1, ref2); + ref3 = __lsx_vldx(dst, dst_stride3); + dst += dst_stride4; + + DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, vec0, vec2); + DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, vec1, vec3); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1); + out0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + + DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, vec4, vec6); + DUP2_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, vec5, vec7); + DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp0, tmp1); + out1 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + + DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1); + out2 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp0, tmp1); + out3 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + + src0 = src4; + + CALC_MSE_AVG_B(out0, ref0, var, avg); + CALC_MSE_AVG_B(out1, ref1, var, avg); + CALC_MSE_AVG_B(out2, ref2, var, avg); + CALC_MSE_AVG_B(out3, ref3, var, avg); + } + vec = __lsx_vhaddw_w_h(avg, avg); + HADD_SW_S32(vec, *diff); + HADD_SW_S32(var, res); + + return res; +} + +static uint32_t sub_pixel_sse_diff_32width_v_lsx( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *filter, int32_t height, int32_t *diff) { + uint32_t sse = 0; + int32_t diff0[2]; + + sse += sub_pixel_sse_diff_16width_v_lsx(src, src_stride, dst, dst_stride, + filter, height, &diff0[0]); + src += 16; + dst += 16; + + sse += sub_pixel_sse_diff_16width_v_lsx(src, src_stride, dst, dst_stride, + filter, height, &diff0[1]); + + *diff = diff0[0] + diff0[1]; + + return sse; +} + +static uint32_t sub_pixel_sse_diff_16width_hv_lsx( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *filter_horiz, const uint8_t *filter_vert, + int32_t height, int32_t *diff) { + uint32_t loop_cnt = (height >> 2); + int32_t res; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i ref0, ref1, ref2, ref3, filt_hz, filt_vt, vec0, vec1; + __m128i hz_out0, hz_out1, hz_out2, hz_out3, tmp0, tmp1, vec; + __m128i var = __lsx_vldi(0); + __m128i avg = var; + __m128i mask = { 0x0403030202010100, 0x0807070606050504 }; + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + int32_t dst_stride4 = dst_stride2 << 1; + + filt_hz = __lsx_vldrepl_h(filter_horiz, 0); + filt_vt = __lsx_vldrepl_h(filter_vert, 0); + + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src0, src1); + src += src_stride; + + HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS, hz_out0); + HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS, hz_out2); + + for (; loop_cnt--;) { + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src0, src1); + src += src_stride; + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src2, src3); + src += src_stride; + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src4, src5); + src += src_stride; + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src6, src7); + src += src_stride; + + ref0 = __lsx_vld(dst, 0); + DUP2_ARG2(__lsx_vldx, dst, dst_stride, dst, dst_stride2, ref1, ref2); + ref3 = __lsx_vldx(dst, dst_stride3); + dst += dst_stride4; + + HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS, hz_out1); + HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS, hz_out3); + DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1); + src0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + + HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS, hz_out0); + HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS, hz_out2); + DUP2_ARG2(__lsx_vpackev_b, hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1); + src1 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + + HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS, hz_out1); + HORIZ_2TAP_FILT_UH(src5, src5, mask, filt_hz, FILTER_BITS, hz_out3); + DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1); + src2 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + + HORIZ_2TAP_FILT_UH(src6, src6, mask, filt_hz, FILTER_BITS, hz_out0); + HORIZ_2TAP_FILT_UH(src7, src7, mask, filt_hz, FILTER_BITS, hz_out2); + DUP2_ARG2(__lsx_vpackev_b, hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1); + src3 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + + CALC_MSE_AVG_B(src0, ref0, var, avg); + CALC_MSE_AVG_B(src1, ref1, var, avg); + CALC_MSE_AVG_B(src2, ref2, var, avg); + CALC_MSE_AVG_B(src3, ref3, var, avg); + } + vec = __lsx_vhaddw_w_h(avg, avg); + HADD_SW_S32(vec, *diff); + HADD_SW_S32(var, res); + + return res; +} + +static uint32_t sub_pixel_sse_diff_32width_hv_lsx( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *filter_horiz, const uint8_t *filter_vert, + int32_t height, int32_t *diff) { + uint32_t sse = 0; + int32_t diff0[2]; + + sse += sub_pixel_sse_diff_16width_hv_lsx(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, height, + &diff0[0]); + src += 16; + dst += 16; + + sse += sub_pixel_sse_diff_16width_hv_lsx(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, height, + &diff0[1]); + + *diff = diff0[0] + diff0[1]; + + return sse; +} + +#define VARIANCE_32Wx32H(sse, diff) VARIANCE_LARGE_WxH(sse, diff, 10) + +#define VPX_SUB_PIXEL_VARIANCE_WDXHT_LSX(wd, ht) \ + uint32_t vpx_sub_pixel_variance##wd##x##ht##_lsx( \ + const uint8_t *src, int32_t src_stride, int32_t x_offset, \ + int32_t y_offset, const uint8_t *ref, int32_t ref_stride, \ + uint32_t *sse) { \ + int32_t diff; \ + uint32_t var; \ + const uint8_t *h_filter = bilinear_filters_lsx[x_offset]; \ + const uint8_t *v_filter = bilinear_filters_lsx[y_offset]; \ + \ + if (y_offset) { \ + if (x_offset) { \ + *sse = sub_pixel_sse_diff_##wd##width_hv_lsx( \ + src, src_stride, ref, ref_stride, h_filter, v_filter, ht, &diff); \ + } else { \ + *sse = sub_pixel_sse_diff_##wd##width_v_lsx( \ + src, src_stride, ref, ref_stride, v_filter, ht, &diff); \ + } \ + \ + var = VARIANCE_##wd##Wx##ht##H(*sse, diff); \ + } else { \ + if (x_offset) { \ + *sse = sub_pixel_sse_diff_##wd##width_h_lsx( \ + src, src_stride, ref, ref_stride, h_filter, ht, &diff); \ + \ + var = VARIANCE_##wd##Wx##ht##H(*sse, diff); \ + } else { \ + var = vpx_variance##wd##x##ht##_lsx(src, src_stride, ref, ref_stride, \ + sse); \ + } \ + } \ + \ + return var; \ + } + +VPX_SUB_PIXEL_VARIANCE_WDXHT_LSX(32, 32) diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/txfm_macros_lsx.h b/media/libvpx/libvpx/vpx_dsp/loongarch/txfm_macros_lsx.h new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/txfm_macros_lsx.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VPX_VPX_DSP_LOONGARCH_TXFM_MACROS_LSX_H_ +#define VPX_VPX_DSP_LOONGARCH_TXFM_MACROS_LSX_H_ + +#include "vpx_util/loongson_intrinsics.h" + +#define DOTP_CONST_PAIR(reg0, reg1, cnst0, cnst1, out0, out1) \ + { \ + __m128i s0_m, s1_m, s2_m, s3_m, s4_m, s5_m; \ + __m128i k0_m, k1_m, k2_m, k3_m; \ + __m128i zero = __lsx_vldi(0); \ + \ + k0_m = __lsx_vreplgr2vr_h(cnst0); \ + k1_m = __lsx_vreplgr2vr_h(cnst1); \ + k2_m = __lsx_vpackev_h(k1_m, k0_m); \ + k0_m = __lsx_vpackev_h(zero, k0_m); \ + k1_m = __lsx_vpackev_h(k1_m, zero); \ + \ + s5_m = __lsx_vilvl_h(reg1, reg0); \ + s4_m = __lsx_vilvh_h(reg1, reg0); \ + s3_m = __lsx_vilvl_h(reg0, reg1); \ + s2_m = __lsx_vilvh_h(reg0, reg1); \ + \ + s1_m = __lsx_vdp2_w_h(s5_m, k0_m); \ + s0_m = __lsx_vdp2_w_h(s4_m, k0_m); \ + k3_m = __lsx_vdp2_w_h(s5_m, k1_m); \ + s1_m = __lsx_vsub_w(s1_m, k3_m); \ + k3_m = __lsx_vdp2_w_h(s4_m, k1_m); \ + s0_m = __lsx_vsub_w(s0_m, k3_m); \ + \ + out0 = __lsx_vssrarni_h_w(s0_m, s1_m, DCT_CONST_BITS); \ + \ + s1_m = __lsx_vdp2_w_h(s3_m, k2_m); \ + s0_m = __lsx_vdp2_w_h(s2_m, k2_m); \ + out1 = __lsx_vssrarni_h_w(s0_m, s1_m, DCT_CONST_BITS); \ + } + +#define DOT_SHIFT_RIGHT_PCK_H(in0, in1, in2, in3) \ + do { \ + __m128i tp0_m, tp1_m; \ + \ + DUP2_ARG2(__lsx_vdp2_w_h, in0, in2, in1, in2, tp1_m, tp0_m); \ + in3 = __lsx_vssrarni_h_w(tp1_m, tp0_m, DCT_CONST_BITS); \ + } while (0) + +#endif // VPX_VPX_DSP_LOONGARCH_TXFM_MACROS_LSX_H_ diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/variance_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/variance_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/variance_lsx.c @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "./vpx_dsp_rtcd.h" +#include "vpx_util/loongson_intrinsics.h" + +#define HADD_SW_S32(in) \ + ({ \ + __m128i res0_m; \ + int32_t sum_m; \ + \ + res0_m = __lsx_vhaddw_d_w(in, in); \ + res0_m = __lsx_vhaddw_q_d(res0_m, res0_m); \ + sum_m = __lsx_vpickve2gr_w(res0_m, 0); \ + sum_m; \ + }) + +#define CALC_MSE_AVG_B(src, ref, var, sub) \ + { \ + __m128i src_l0_m, src_l1_m; \ + __m128i res_l0_m, res_l1_m; \ + \ + src_l0_m = __lsx_vilvl_b(src, ref); \ + src_l1_m = __lsx_vilvh_b(src, ref); \ + DUP2_ARG2(__lsx_vhsubw_hu_bu, src_l0_m, src_l0_m, src_l1_m, src_l1_m, \ + res_l0_m, res_l1_m); \ + var = __lsx_vdp2add_w_h(var, res_l0_m, res_l0_m); \ + var = __lsx_vdp2add_w_h(var, res_l1_m, res_l1_m); \ + sub = __lsx_vadd_h(sub, res_l0_m); \ + sub = __lsx_vadd_h(sub, res_l1_m); \ + } + +#define VARIANCE_WxH(sse, diff, shift) \ + (sse) - (((uint32_t)(diff) * (diff)) >> (shift)) + +#define VARIANCE_LARGE_WxH(sse, diff, shift) \ + (sse) - (((int64_t)(diff) * (diff)) >> (shift)) + +static uint32_t sse_diff_16width_lsx(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *ref_ptr, int32_t ref_stride, + int32_t height, int32_t *diff) { + int32_t ht_cnt = (height >> 2); + __m128i src, ref, vec; + __m128i avg = __lsx_vldi(0); + __m128i var = avg; + + for (; ht_cnt--;) { + src = __lsx_vld(src_ptr, 0); + src_ptr += src_stride; + ref = __lsx_vld(ref_ptr, 0); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src, ref, var, avg); + + src = __lsx_vld(src_ptr, 0); + src_ptr += src_stride; + ref = __lsx_vld(ref_ptr, 0); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src, ref, var, avg); + src = __lsx_vld(src_ptr, 0); + src_ptr += src_stride; + ref = __lsx_vld(ref_ptr, 0); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src, ref, var, avg); + + src = __lsx_vld(src_ptr, 0); + src_ptr += src_stride; + ref = __lsx_vld(ref_ptr, 0); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src, ref, var, avg); + } + vec = __lsx_vhaddw_w_h(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t sse_diff_32width_lsx(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *ref_ptr, int32_t ref_stride, + int32_t height, int32_t *diff) { + int32_t ht_cnt = (height >> 2); + __m128i avg = __lsx_vldi(0); + __m128i src0, src1, ref0, ref1; + __m128i vec; + __m128i var = avg; + + for (; ht_cnt--;) { + DUP2_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src0, src1); + src_ptr += src_stride; + DUP2_ARG2(__lsx_vld, ref_ptr, 0, ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src0, ref0, var, avg); + CALC_MSE_AVG_B(src1, ref1, var, avg); + + DUP2_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src0, src1); + src_ptr += src_stride; + DUP2_ARG2(__lsx_vld, ref_ptr, 0, ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src0, ref0, var, avg); + CALC_MSE_AVG_B(src1, ref1, var, avg); + + DUP2_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src0, src1); + src_ptr += src_stride; + DUP2_ARG2(__lsx_vld, ref_ptr, 0, ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src0, ref0, var, avg); + CALC_MSE_AVG_B(src1, ref1, var, avg); + + DUP2_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src0, src1); + src_ptr += src_stride; + DUP2_ARG2(__lsx_vld, ref_ptr, 0, ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src0, ref0, var, avg); + CALC_MSE_AVG_B(src1, ref1, var, avg); + } + + vec = __lsx_vhaddw_w_h(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t sse_diff_64x64_lsx(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *ref_ptr, int32_t ref_stride, + int32_t *diff) { + int32_t ht_cnt = 32; + __m128i avg0 = __lsx_vldi(0); + __m128i src0, src1, src2, src3; + __m128i ref0, ref1, ref2, ref3; + __m128i vec0, vec1; + __m128i avg1 = avg0; + __m128i avg2 = avg0; + __m128i avg3 = avg0; + __m128i var = avg0; + + for (; ht_cnt--;) { + DUP4_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src_ptr, 32, src_ptr, 48, + src0, src1, src2, src3); + src_ptr += src_stride; + DUP4_ARG2(__lsx_vld, ref_ptr, 0, ref_ptr, 16, ref_ptr, 32, ref_ptr, 48, + ref0, ref1, ref2, ref3); + ref_ptr += ref_stride; + + CALC_MSE_AVG_B(src0, ref0, var, avg0); + CALC_MSE_AVG_B(src1, ref1, var, avg1); + CALC_MSE_AVG_B(src2, ref2, var, avg2); + CALC_MSE_AVG_B(src3, ref3, var, avg3); + DUP4_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src_ptr, 32, src_ptr, 48, + src0, src1, src2, src3); + src_ptr += src_stride; + DUP4_ARG2(__lsx_vld, ref_ptr, 0, ref_ptr, 16, ref_ptr, 32, ref_ptr, 48, + ref0, ref1, ref2, ref3); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src0, ref0, var, avg0); + CALC_MSE_AVG_B(src1, ref1, var, avg1); + CALC_MSE_AVG_B(src2, ref2, var, avg2); + CALC_MSE_AVG_B(src3, ref3, var, avg3); + } + vec0 = __lsx_vhaddw_w_h(avg0, avg0); + vec1 = __lsx_vhaddw_w_h(avg1, avg1); + vec0 = __lsx_vadd_w(vec0, vec1); + vec1 = __lsx_vhaddw_w_h(avg2, avg2); + vec0 = __lsx_vadd_w(vec0, vec1); + vec1 = __lsx_vhaddw_w_h(avg3, avg3); + vec0 = __lsx_vadd_w(vec0, vec1); + *diff = HADD_SW_S32(vec0); + + return HADD_SW_S32(var); +} + +#define VARIANCE_16Wx16H(sse, diff) VARIANCE_WxH(sse, diff, 8) + +#define VARIANCE_32Wx32H(sse, diff) VARIANCE_LARGE_WxH(sse, diff, 10) +#define VARIANCE_64Wx64H(sse, diff) VARIANCE_LARGE_WxH(sse, diff, 12) + +#define VPX_VARIANCE_WDXHT_LSX(wd, ht) \ + uint32_t vpx_variance##wd##x##ht##_lsx( \ + const uint8_t *src, int32_t src_stride, const uint8_t *ref, \ + int32_t ref_stride, uint32_t *sse) { \ + int32_t diff; \ + \ + *sse = \ + sse_diff_##wd##width_lsx(src, src_stride, ref, ref_stride, ht, &diff); \ + \ + return VARIANCE_##wd##Wx##ht##H(*sse, diff); \ + } + +VPX_VARIANCE_WDXHT_LSX(16, 16) +VPX_VARIANCE_WDXHT_LSX(32, 32) + +uint32_t vpx_variance64x64_lsx(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + uint32_t *sse) { + int32_t diff; + + *sse = sse_diff_64x64_lsx(src, src_stride, ref, ref_stride, &diff); + + return VARIANCE_64Wx64H(*sse, diff); +} + +void vpx_get16x16var_lsx(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, uint32_t *sse, + int32_t *sum) { + *sse = sse_diff_16width_lsx(src, src_stride, ref, ref_stride, 16, sum); +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_avg_horiz_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_avg_horiz_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_avg_horiz_lsx.c @@ -0,0 +1,972 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include "./vpx_dsp_rtcd.h" +#include "vpx_dsp/loongarch/vpx_convolve_lsx.h" + +static const uint8_t mc_filt_mask_arr[16 * 3] = { + /* 8 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, + /* 4 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20, + /* 4 width cases */ + 8, 9, 9, 10, 10, 11, 11, 12, 24, 25, 25, 26, 26, 27, 27, 28 +}; + +static void common_hz_8t_and_aver_dst_4x4_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter) { + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3; + __m128i filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3; + __m128i tmp0, tmp1; + __m128i dst0, dst1, dst2, dst3; + + mask0 = __lsx_vld(mc_filt_mask_arr, 16); + src -= 3; + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filter0, filter1, filter2, filter3, tmp0, tmp1); + dst0 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_w(dst_tmp, 0); + dst0 = __lsx_vilvl_w(dst1, dst0); + dst1 = __lsx_vilvl_w(dst3, dst2); + dst0 = __lsx_vilvl_d(dst1, dst0); + tmp0 = __lsx_vssrarni_b_h(tmp1, tmp0, 7); + tmp0 = __lsx_vxori_b(tmp0, 128); + dst0 = __lsx_vavgr_bu(tmp0, dst0); + __lsx_vstelm_w(dst0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(dst0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(dst0, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(dst0, dst, 0, 3); +} + +static void common_hz_8t_and_aver_dst_4x8_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter) { + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3, filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3, tmp0, tmp1, tmp2, tmp3; + __m128i dst0, dst1; + + mask0 = __lsx_vld(mc_filt_mask_arr, 16); + src -= 3; + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + src += src_stride; + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + tmp0 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + tmp1 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + tmp2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + tmp3 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + tmp0 = __lsx_vilvl_w(tmp1, tmp0); + tmp1 = __lsx_vilvl_w(tmp3, tmp2); + dst0 = __lsx_vilvl_d(tmp1, tmp0); + + tmp0 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + tmp1 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + tmp2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + tmp3 = __lsx_vldrepl_w(dst_tmp, 0); + tmp0 = __lsx_vilvl_w(tmp1, tmp0); + tmp1 = __lsx_vilvl_w(tmp3, tmp2); + dst1 = __lsx_vilvl_d(tmp1, tmp0); + + HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filter0, filter1, filter2, filter3, tmp0, tmp1); + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filter0, filter1, filter2, filter3, tmp2, tmp3); + DUP4_ARG3(__lsx_vssrarni_b_h, tmp0, tmp0, 7, tmp1, tmp1, 7, tmp2, tmp2, 7, + tmp3, tmp3, 7, tmp0, tmp1, tmp2, tmp3); + DUP2_ARG2(__lsx_vilvl_d, tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + DUP2_ARG2(__lsx_vavgr_bu, tmp0, dst0, tmp1, dst1, dst0, dst1); + __lsx_vstelm_w(dst0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(dst0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(dst0, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(dst0, dst, 0, 3); + dst += dst_stride; + __lsx_vstelm_w(dst1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(dst1, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(dst1, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(dst1, dst, 0, 3); +} + +static void common_hz_8t_and_aver_dst_4w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, int8_t *filter, + int32_t height) { + if (height == 4) { + common_hz_8t_and_aver_dst_4x4_lsx(src, src_stride, dst, dst_stride, filter); + } else if (height == 8) { + common_hz_8t_and_aver_dst_4x8_lsx(src, src_stride, dst, dst_stride, filter); + } +} + +static void common_hz_8t_and_aver_dst_8w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, int8_t *filter, + int32_t height) { + int32_t loop_cnt = height >> 2; + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3, filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3; + __m128i tmp0, tmp1, tmp2, tmp3; + __m128i dst0, dst1, dst2, dst3; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + uint8_t *_src = (uint8_t *)src - 3; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + for (; loop_cnt--;) { + src0 = __lsx_vld(_src, 0); + DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src1, src2); + src3 = __lsx_vldx(_src, src_stride3); + _src += src_stride4; + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, + mask3, filter0, filter1, filter2, filter3, tmp0, + tmp1, tmp2, tmp3); + dst0 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1); + DUP2_ARG3(__lsx_vssrarni_b_h, tmp1, tmp0, 7, tmp3, tmp2, 7, tmp0, tmp1); + DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + DUP2_ARG2(__lsx_vavgr_bu, tmp0, dst0, tmp1, dst1, dst0, dst1); + __lsx_vstelm_d(dst0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(dst0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(dst1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(dst1, dst, 0, 1); + dst += dst_stride; + } +} + +static void common_hz_8t_and_aver_dst_16w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter, int32_t height) { + int32_t loop_cnt = height >> 1; + int32_t dst_stride2 = dst_stride << 1; + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3, filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3, dst0, dst1, dst2, dst3; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + __m128i tmp8, tmp9, tmp10, tmp11, tmp12, tmp13, tmp14, tmp15; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + src -= 3; + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + for (; loop_cnt--;) { + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src0, src1); + src += src_stride; + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src2, src3); + src += src_stride; + dst0 = __lsx_vld(dst_tmp, 0); + dst1 = __lsx_vldx(dst_tmp, dst_stride); + dst_tmp += dst_stride2; + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src1, mask0, src2, src2, + mask0, src3, src3, mask0, tmp0, tmp1, tmp2, tmp3); + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask1, src1, src1, mask1, src2, src2, + mask1, src3, src3, mask1, tmp4, tmp5, tmp6, tmp7); + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask2, src1, src1, mask2, src2, src2, + mask2, src3, src3, mask2, tmp8, tmp9, tmp10, tmp11); + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask3, src1, src1, mask3, src2, src2, + mask3, src3, src3, mask3, tmp12, tmp13, tmp14, tmp15); + DUP4_ARG2(__lsx_vdp2_h_b, tmp0, filter0, tmp1, filter0, tmp2, filter0, tmp3, + filter0, tmp0, tmp1, tmp2, tmp3); + DUP4_ARG2(__lsx_vdp2_h_b, tmp8, filter2, tmp9, filter2, tmp10, filter2, + tmp11, filter2, tmp8, tmp9, tmp10, tmp11); + DUP4_ARG3(__lsx_vdp2add_h_b, tmp0, tmp4, filter1, tmp1, tmp5, filter1, tmp2, + tmp6, filter1, tmp3, tmp7, filter1, tmp0, tmp1, tmp2, tmp3); + DUP4_ARG3(__lsx_vdp2add_h_b, tmp8, tmp12, filter3, tmp9, tmp13, filter3, + tmp10, tmp14, filter3, tmp11, tmp15, filter3, tmp4, tmp5, tmp6, + tmp7); + DUP4_ARG2(__lsx_vsadd_h, tmp0, tmp4, tmp1, tmp5, tmp2, tmp6, tmp3, tmp7, + tmp0, tmp1, tmp2, tmp3); + DUP2_ARG3(__lsx_vssrarni_b_h, tmp1, tmp0, 7, tmp3, tmp2, 7, dst2, dst3); + DUP2_ARG2(__lsx_vxori_b, dst2, 128, dst3, 128, dst2, dst3); + DUP2_ARG2(__lsx_vavgr_bu, dst0, dst2, dst1, dst3, dst0, dst1); + __lsx_vst(dst0, dst, 0); + __lsx_vstx(dst1, dst, dst_stride); + dst += dst_stride2; + } +} + +static void common_hz_8t_and_aver_dst_32w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = height; + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3, filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3, dst0, dst1; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + __m128i tmp8, tmp9, tmp10, tmp11, tmp12, tmp13, tmp14, tmp15; + __m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 }; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + src -= 3; + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + for (; loop_cnt--;) { + DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src2); + src3 = __lsx_vld(src, 24); + src1 = __lsx_vshuf_b(src2, src0, shuff); + src += src_stride; + DUP2_ARG2(__lsx_vld, dst_tmp, 0, dst, 16, dst0, dst1); + dst_tmp += dst_stride; + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src1, mask0, src2, src2, + mask0, src3, src3, mask0, tmp0, tmp1, tmp2, tmp3); + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask1, src1, src1, mask1, src2, src2, + mask1, src3, src3, mask1, tmp4, tmp5, tmp6, tmp7); + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask2, src1, src1, mask2, src2, src2, + mask2, src3, src3, mask2, tmp8, tmp9, tmp10, tmp11); + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask3, src1, src1, mask3, src2, src2, + mask3, src3, src3, mask3, tmp12, tmp13, tmp14, tmp15); + DUP4_ARG2(__lsx_vdp2_h_b, tmp0, filter0, tmp1, filter0, tmp2, filter0, tmp3, + filter0, tmp0, tmp1, tmp2, tmp3); + DUP4_ARG2(__lsx_vdp2_h_b, tmp8, filter2, tmp9, filter2, tmp10, filter2, + tmp11, filter2, tmp8, tmp9, tmp10, tmp11); + DUP4_ARG3(__lsx_vdp2add_h_b, tmp0, tmp4, filter1, tmp1, tmp5, filter1, tmp2, + tmp6, filter1, tmp3, tmp7, filter1, tmp0, tmp1, tmp2, tmp3); + DUP4_ARG3(__lsx_vdp2add_h_b, tmp8, tmp12, filter3, tmp9, tmp13, filter3, + tmp10, tmp14, filter3, tmp11, tmp15, filter3, tmp4, tmp5, tmp6, + tmp7); + DUP4_ARG2(__lsx_vsadd_h, tmp0, tmp4, tmp1, tmp5, tmp2, tmp6, tmp3, tmp7, + tmp0, tmp1, tmp2, tmp3); + DUP2_ARG3(__lsx_vssrarni_b_h, tmp1, tmp0, 7, tmp3, tmp2, 7, tmp0, tmp1); + DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + DUP2_ARG2(__lsx_vavgr_bu, dst0, tmp0, dst1, tmp1, dst0, dst1); + __lsx_vst(dst0, dst, 0); + __lsx_vst(dst1, dst, 16); + dst += dst_stride; + } +} + +static void common_hz_8t_and_aver_dst_64w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter, int32_t height) { + int32_t loop_cnt = height; + __m128i src0, src1, src2, src3; + __m128i filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3; + __m128i out0, out1, out2, out3, dst0, dst1; + __m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 }; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + src -= 3; + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + for (; loop_cnt--;) { + DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src2); + src3 = __lsx_vld(src, 24); + src1 = __lsx_vshuf_b(src2, src0, shuff); + DUP2_ARG2(__lsx_vld, dst, 0, dst, 16, dst0, dst1); + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, + mask3, filter0, filter1, filter2, filter3, out0, + out1, out2, out3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + DUP2_ARG2(__lsx_vavgr_bu, out0, dst0, out1, dst1, out0, out1); + __lsx_vst(out0, dst, 0); + __lsx_vst(out1, dst, 16); + + DUP2_ARG2(__lsx_vld, src, 32, src, 48, src0, src2); + src3 = __lsx_vld(src, 56); + src1 = __lsx_vshuf_b(src2, src0, shuff); + DUP2_ARG2(__lsx_vld, dst, 32, dst, 48, dst0, dst1); + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, + mask3, filter0, filter1, filter2, filter3, out0, + out1, out2, out3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + DUP2_ARG2(__lsx_vavgr_bu, out0, dst0, out1, dst1, out0, out1); + __lsx_vst(out0, dst, 32); + __lsx_vst(out1, dst, 48); + src += src_stride; + dst += dst_stride; + } +} + +static void common_hz_2t_and_aver_dst_4x4_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter) { + __m128i src0, src1, src2, src3, mask; + __m128i dst0, dst1, dst2, dst3, vec0, vec1, filt0; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + uint8_t *dst_tmp = dst; + + mask = __lsx_vld(mc_filt_mask_arr, 16); + + /* rearranging filter */ + filt0 = __lsx_vldrepl_h(filter, 0); + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + dst0 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_w, dst1, dst0, dst3, dst2, dst0, dst1); + dst0 = __lsx_vilvl_d(dst1, dst0); + DUP2_ARG3(__lsx_vshuf_b, src1, src0, mask, src3, src2, mask, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec0, vec1); + vec0 = __lsx_vssrarni_bu_h(vec1, vec0, FILTER_BITS); + vec0 = __lsx_vavgr_bu(vec0, dst0); + __lsx_vstelm_w(vec0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(vec0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(vec0, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(vec0, dst, 0, 3); +} + +static void common_hz_2t_and_aver_dst_4x8_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter) { + __m128i src0, src1, src2, src3, src4, src5, src6, src7, mask; + __m128i filt0, vec0, vec1, vec2, vec3, res0, res1, res2, res3; + __m128i dst0, dst1, dst2, dst3, dst4; + __m128i vec4, vec5, vec6, vec7; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + uint8_t *src_tmp1 = (uint8_t *)src + src_stride4; + uint8_t *dst_tmp = dst; + + mask = __lsx_vld(mc_filt_mask_arr, 16); + + /* rearranging filter */ + filt0 = __lsx_vldrepl_h(filter, 0); + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + + src4 = __lsx_vld(src_tmp1, 0); + DUP2_ARG2(__lsx_vldx, src_tmp1, src_stride, src_tmp1, src_stride2, src5, + src6); + src7 = __lsx_vldx(src_tmp1, src_stride3); + + dst0 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_w, dst1, dst0, dst3, dst2, dst0, dst1); + dst0 = __lsx_vilvl_d(dst1, dst0); + + dst1 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst4 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_w, dst2, dst1, dst4, dst3, dst1, dst2); + dst1 = __lsx_vilvl_d(dst2, dst1); + + DUP2_ARG3(__lsx_vshuf_b, src1, src0, mask, src3, src2, mask, vec0, vec1); + DUP2_ARG3(__lsx_vshuf_b, src5, src4, mask, src7, src6, mask, vec2, vec3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0, + vec4, vec5, vec6, vec7); + DUP4_ARG3(__lsx_vssrarni_bu_h, vec4, vec4, FILTER_BITS, vec5, vec5, + FILTER_BITS, vec6, vec6, FILTER_BITS, vec7, vec7, FILTER_BITS, res0, + res1, res2, res3); + DUP2_ARG2(__lsx_vilvl_d, res1, res0, res3, res2, res0, res2); + DUP2_ARG2(__lsx_vavgr_bu, res0, dst0, res2, dst1, res0, res2); + + __lsx_vstelm_w(res0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(res0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(res0, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(res0, dst, 0, 3); + dst += dst_stride; + + __lsx_vstelm_w(res2, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(res2, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(res2, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(res2, dst, 0, 3); + dst += dst_stride; +} + +static void common_hz_2t_and_aver_dst_4w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, int8_t *filter, + int32_t height) { + if (height == 4) { + common_hz_2t_and_aver_dst_4x4_lsx(src, src_stride, dst, dst_stride, filter); + } else if (height == 8) { + common_hz_2t_and_aver_dst_4x8_lsx(src, src_stride, dst, dst_stride, filter); + } +} + +static void common_hz_2t_and_aver_dst_8x4_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter) { + __m128i src0, src1, src2, src3, mask; + __m128i filt0, dst0, dst1, dst2, dst3; + __m128i vec0, vec1, vec2, vec3; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + uint8_t *dst_tmp = dst; + + mask = __lsx_vld(mc_filt_mask_arr, 0); + + /* rearranging filter */ + filt0 = __lsx_vldrepl_h(filter, 0); + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1); + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0, + vec0, vec1, vec2, vec3); + DUP2_ARG3(__lsx_vssrarni_bu_h, vec1, vec0, FILTER_BITS, vec3, vec2, + FILTER_BITS, vec0, vec1); + dst0 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + + DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1); + DUP2_ARG2(__lsx_vavgr_bu, vec0, dst0, vec1, dst1, vec0, vec1); + __lsx_vstelm_d(vec0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(vec0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(vec1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(vec1, dst, 0, 1); +} + +static void common_hz_2t_and_aver_dst_8x8mult_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + __m128i src0, src1, src2, src3, mask; + __m128i filt0, dst0, dst1, dst2, dst3; + __m128i vec0, vec1, vec2, vec3; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + uint8_t *dst_tmp = dst; + + mask = __lsx_vld(mc_filt_mask_arr, 0); + + /* rearranging filter */ + filt0 = __lsx_vldrepl_h(filter, 0); + + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + src += src_stride; + + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1); + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0, + vec0, vec1, vec2, vec3); + DUP2_ARG3(__lsx_vssrarni_bu_h, vec1, vec0, FILTER_BITS, vec3, vec2, + FILTER_BITS, vec0, vec2); + dst0 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1); + + DUP2_ARG2(__lsx_vavgr_bu, vec0, dst0, vec2, dst1, vec0, vec2); + __lsx_vstelm_d(vec0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(vec0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(vec2, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(vec2, dst, 0, 1); + dst += dst_stride; + + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + src += src_stride; + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1); + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3); + + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0, + vec0, vec1, vec2, vec3); + DUP2_ARG3(__lsx_vssrarni_bu_h, vec1, vec0, FILTER_BITS, vec3, vec2, + FILTER_BITS, vec0, vec2); + dst0 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1); + DUP2_ARG2(__lsx_vavgr_bu, vec0, dst0, vec2, dst1, vec0, vec2); + __lsx_vstelm_d(vec0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(vec0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(vec2, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(vec2, dst, 0, 1); + dst += dst_stride; + + if (height == 16) { + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + src += src_stride; + + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1); + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, + filt0, vec0, vec1, vec2, vec3); + DUP2_ARG3(__lsx_vssrarni_bu_h, vec1, vec0, FILTER_BITS, vec3, vec2, + FILTER_BITS, vec0, vec2); + dst0 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1); + DUP2_ARG2(__lsx_vavgr_bu, vec0, dst0, vec2, dst1, vec0, vec2); + __lsx_vstelm_d(vec0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(vec0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(vec2, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(vec2, dst, 0, 1); + dst += dst_stride; + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1); + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, + filt0, vec0, vec1, vec2, vec3); + DUP2_ARG3(__lsx_vssrarni_bu_h, vec1, vec0, FILTER_BITS, vec3, vec2, + FILTER_BITS, vec0, vec2); + dst0 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1); + DUP2_ARG2(__lsx_vavgr_bu, vec0, dst0, vec2, dst1, vec0, vec2); + __lsx_vstelm_d(vec0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(vec0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(vec2, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(vec2, dst, 0, 1); + dst += dst_stride; + } +} + +static void common_hz_2t_and_aver_dst_8w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, int8_t *filter, + int32_t height) { + if (height == 4) { + common_hz_2t_and_aver_dst_8x4_lsx(src, src_stride, dst, dst_stride, filter); + } else { + common_hz_2t_and_aver_dst_8x8mult_lsx(src, src_stride, dst, dst_stride, + filter, height); + } +} + +static void common_hz_2t_and_aver_dst_16w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = (height >> 2) - 1; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, mask; + __m128i filt0, dst0; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i res0, res1, res2, res3, res4, res5, res6, res7; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + uint8_t *src_tmp1 = (uint8_t *)src + 8; + + mask = __lsx_vld(mc_filt_mask_arr, 0); + + /* rearranging filter */ + filt0 = __lsx_vldrepl_h(filter, 0); + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src4); + src6 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + src1 = __lsx_vld(src_tmp1, 0); + DUP2_ARG2(__lsx_vldx, src_tmp1, src_stride, src_tmp1, src_stride2, src3, + src5); + src7 = __lsx_vldx(src_tmp1, src_stride3); + src_tmp1 += src_stride4; + + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1); + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3); + DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask, src5, src5, mask, vec4, vec5); + DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask, src7, src7, mask, vec6, vec7); + + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0, + res0, res1, res2, res3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7, filt0, + res4, res5, res6, res7); + DUP4_ARG3(__lsx_vssrarni_bu_h, res1, res0, FILTER_BITS, res3, res2, + FILTER_BITS, res5, res4, FILTER_BITS, res7, res6, FILTER_BITS, res0, + res2, res4, res6); + dst0 = __lsx_vld(dst, 0); + res0 = __lsx_vavgr_bu(res0, dst0); + __lsx_vst(res0, dst, 0); + dst += dst_stride; + + dst0 = __lsx_vld(dst, 0); + res2 = __lsx_vavgr_bu(res2, dst0); + __lsx_vst(res2, dst, 0); + dst += dst_stride; + + dst0 = __lsx_vld(dst, 0); + res4 = __lsx_vavgr_bu(res4, dst0); + __lsx_vst(res4, dst, 0); + dst += dst_stride; + + dst0 = __lsx_vld(dst, 0); + res6 = __lsx_vavgr_bu(res6, dst0); + __lsx_vst(res6, dst, 0); + dst += dst_stride; + + for (; loop_cnt--;) { + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src4); + src6 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + src1 = __lsx_vld(src_tmp1, 0); + DUP2_ARG2(__lsx_vldx, src_tmp1, src_stride, src_tmp1, src_stride2, src3, + src5); + src7 = __lsx_vldx(src_tmp1, src_stride3); + src_tmp1 += src_stride4; + + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1); + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3); + DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask, src5, src5, mask, vec4, vec5); + DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask, src7, src7, mask, vec6, vec7); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, + filt0, res0, res1, res2, res3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7, + filt0, res4, res5, res6, res7); + + DUP4_ARG3(__lsx_vssrarni_bu_h, res1, res0, FILTER_BITS, res3, res2, + FILTER_BITS, res5, res4, FILTER_BITS, res7, res6, FILTER_BITS, + res0, res2, res4, res6); + dst0 = __lsx_vld(dst, 0); + res0 = __lsx_vavgr_bu(res0, dst0); + __lsx_vst(res0, dst, 0); + dst += dst_stride; + + dst0 = __lsx_vld(dst, 0); + res2 = __lsx_vavgr_bu(res2, dst0); + __lsx_vst(res2, dst, 0); + dst += dst_stride; + + dst0 = __lsx_vld(dst, 0); + res4 = __lsx_vavgr_bu(res4, dst0); + __lsx_vst(res4, dst, 0); + dst += dst_stride; + + dst0 = __lsx_vld(dst, 0); + res6 = __lsx_vavgr_bu(res6, dst0); + __lsx_vst(res6, dst, 0); + dst += dst_stride; + } +} + +static void common_hz_2t_and_aver_dst_32w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = (height >> 1); + __m128i src0, src1, src2, src3, src4, src5, src6, src7, mask; + __m128i filt0, dst0, dst1; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i res0, res1, res2, res3, res4, res5, res6, res7; + __m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 }; + + mask = __lsx_vld(mc_filt_mask_arr, 0); + + /* rearranging filter */ + filt0 = __lsx_vldrepl_h(filter, 0); + + for (; loop_cnt--;) { + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vld, src, 16, src, 24, src2, src3); + src1 = __lsx_vshuf_b(src2, src0, shuff); + src += src_stride; + src4 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vld, src, 16, src, 24, src6, src7); + src5 = __lsx_vshuf_b(src6, src4, shuff); + src += src_stride; + + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1); + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3); + DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask, src5, src5, mask, vec4, vec5); + DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask, src7, src7, mask, vec6, vec7); + + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, + filt0, res0, res1, res2, res3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7, + filt0, res4, res5, res6, res7); + DUP4_ARG3(__lsx_vssrarni_bu_h, res1, res0, FILTER_BITS, res3, res2, + FILTER_BITS, res5, res4, FILTER_BITS, res7, res6, FILTER_BITS, + res0, res2, res4, res6); + + DUP2_ARG2(__lsx_vld, dst, 0, dst, 16, dst0, dst1); + res0 = __lsx_vavgr_bu(res0, dst0); + __lsx_vst(res0, dst, 0); + res2 = __lsx_vavgr_bu(res2, dst1); + __lsx_vst(res2, dst, 16); + dst += dst_stride; + + DUP2_ARG2(__lsx_vld, dst, 0, dst, 16, dst0, dst1); + res4 = __lsx_vavgr_bu(res4, dst0); + __lsx_vst(res4, dst, 0); + res6 = __lsx_vavgr_bu(res6, dst1); + __lsx_vst(res6, dst, 16); + dst += dst_stride; + } +} + +static void common_hz_2t_and_aver_dst_64w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = height; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, mask; + __m128i filt0, dst0, dst1, dst2, dst3; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i out0, out1, out2, out3, out4, out5, out6, out7; + __m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 }; + + mask = __lsx_vld(mc_filt_mask_arr, 0); + + /* rearranging filter */ + filt0 = __lsx_vldrepl_h(filter, 0); + + for (; loop_cnt--;) { + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src2, src4, + src6); + src7 = __lsx_vld(src, 56); + DUP2_ARG3(__lsx_vshuf_b, src2, src0, shuff, src4, src2, shuff, src1, src3); + src5 = __lsx_vshuf_b(src6, src4, shuff); + src += src_stride; + + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1); + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3); + DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask, src5, src5, mask, vec4, vec5); + DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask, src7, src7, mask, vec6, vec7); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, + filt0, out0, out1, out2, out3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7, + filt0, out4, out5, out6, out7); + + DUP4_ARG3(__lsx_vssrarni_bu_h, out1, out0, FILTER_BITS, out3, out2, + FILTER_BITS, out5, out4, FILTER_BITS, out7, out6, FILTER_BITS, + out0, out2, out4, out6); + + DUP4_ARG2(__lsx_vld, dst, 0, dst, 16, dst, 32, dst, 48, dst0, dst1, dst2, + dst3); + out0 = __lsx_vavgr_bu(out0, dst0); + __lsx_vst(out0, dst, 0); + out2 = __lsx_vavgr_bu(out2, dst1); + __lsx_vst(out2, dst, 16); + out4 = __lsx_vavgr_bu(out4, dst2); + __lsx_vst(out4, dst, 32); + out6 = __lsx_vavgr_bu(out6, dst3); + __lsx_vst(out6, dst, 48); + dst += dst_stride; + } +} + +void vpx_convolve8_avg_horiz_lsx(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *filter, int x0_q4, + int x_step_q4, int y0_q4, int y_step_q4, int w, + int h) { + const int16_t *const filter_x = filter[x0_q4]; + int8_t cnt, filt_hor[8]; + + assert(x_step_q4 == 16); + assert(((const int32_t *)filter_x)[1] != 0x800000); + + for (cnt = 0; cnt < 8; ++cnt) { + filt_hor[cnt] = filter_x[cnt]; + } + + if (vpx_get_filter_taps(filter_x) == 2) { + switch (w) { + case 4: + common_hz_2t_and_aver_dst_4w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], h); + break; + case 8: + common_hz_2t_and_aver_dst_8w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], h); + break; + case 16: + common_hz_2t_and_aver_dst_16w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], h); + break; + + case 32: + common_hz_2t_and_aver_dst_32w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], h); + break; + case 64: + common_hz_2t_and_aver_dst_64w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], h); + break; + default: + vpx_convolve8_avg_horiz_c(src, src_stride, dst, dst_stride, filter, + x0_q4, x_step_q4, y0_q4, y_step_q4, w, h); + break; + } + } else { + switch (w) { + case 4: + common_hz_8t_and_aver_dst_4w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, h); + break; + case 8: + common_hz_8t_and_aver_dst_8w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, h); + break; + case 16: + common_hz_8t_and_aver_dst_16w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, h); + break; + case 32: + common_hz_8t_and_aver_dst_32w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, h); + break; + case 64: + common_hz_8t_and_aver_dst_64w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, h); + break; + default: + vpx_convolve8_avg_horiz_c(src, src_stride, dst, dst_stride, filter, + x0_q4, x_step_q4, y0_q4, y_step_q4, w, h); + break; + } + } +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_avg_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_avg_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_avg_lsx.c @@ -0,0 +1,748 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include "./vpx_dsp_rtcd.h" +#include "vpx_dsp/loongarch/vpx_convolve_lsx.h" + +static const uint8_t mc_filt_mask_arr[16 * 3] = { + /* 8 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, + /* 4 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20, + /* 4 width cases */ + 8, 9, 9, 10, 10, 11, 11, 12, 24, 25, 25, 26, 26, 27, 27, 28 +}; + +static void common_hv_8ht_8vt_and_aver_dst_4w_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + const int8_t *filter_horiz, const int8_t *filter_vert, int32_t height) { + uint32_t loop_cnt = height >> 2; + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i filt_hz0, filt_hz1, filt_hz2, filt_hz3; + __m128i filt_vt0, filt_vt1, filt_vt2, filt_vt3; + __m128i mask0, mask1, mask2, mask3; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + __m128i out0, out1; + __m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 }; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride + src_stride2; + int32_t src_stride4 = src_stride2 << 1; + uint8_t *_src = (uint8_t *)src - 3 - src_stride3; + + mask0 = __lsx_vld(mc_filt_mask_arr, 16); + DUP4_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_horiz, 2, filter_horiz, 4, + filter_horiz, 6, filt_hz0, filt_hz1, filt_hz2, filt_hz3); + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + + src0 = __lsx_vld(_src, 0); + DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src1, src2); + src3 = __lsx_vldx(_src, src_stride3); + _src += src_stride4; + src4 = __lsx_vld(_src, 0); + DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src5, src6); + _src += src_stride3; + + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + + tmp0 = HORIZ_8TAP_FILT(src0, src1, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + tmp2 = HORIZ_8TAP_FILT(src2, src3, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + tmp4 = HORIZ_8TAP_FILT(src4, src5, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + tmp5 = HORIZ_8TAP_FILT(src5, src6, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + DUP2_ARG3(__lsx_vshuf_b, tmp2, tmp0, shuff, tmp4, tmp2, shuff, tmp1, tmp3); + DUP4_ARG2(__lsx_vldrepl_h, filter_vert, 0, filter_vert, 2, filter_vert, 4, + filter_vert, 6, filt_vt0, filt_vt1, filt_vt2, filt_vt3); + DUP2_ARG2(__lsx_vpackev_b, tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + tmp2 = __lsx_vpackev_b(tmp5, tmp4); + for (; loop_cnt--;) { + src7 = __lsx_vld(_src, 0); + DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src8, src9); + src10 = __lsx_vldx(_src, src_stride3); + _src += src_stride4; + src2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + src3 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + src4 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + src5 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_w, src3, src2, src5, src4, src2, src3); + src2 = __lsx_vilvl_d(src3, src2); + DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7, + src8, src9, src10); + tmp3 = HORIZ_8TAP_FILT(src7, src8, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + tmp4 = __lsx_vshuf_b(tmp3, tmp5, shuff); + tmp4 = __lsx_vpackev_b(tmp3, tmp4); + out0 = FILT_8TAP_DPADD_S_H(tmp0, tmp1, tmp2, tmp4, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + src1 = HORIZ_8TAP_FILT(src9, src10, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src0 = __lsx_vshuf_b(src1, tmp3, shuff); + src0 = __lsx_vpackev_b(src1, src0); + out1 = FILT_8TAP_DPADD_S_H(tmp1, tmp2, tmp4, src0, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + out0 = __lsx_vssrarni_b_h(out1, out0, FILTER_BITS); + out0 = __lsx_vxori_b(out0, 128); + out0 = __lsx_vavgr_bu(out0, src2); + __lsx_vstelm_w(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 3); + dst += dst_stride; + + tmp5 = src1; + tmp0 = tmp2; + tmp1 = tmp4; + tmp2 = src0; + } +} + +static void common_hv_8ht_8vt_and_aver_dst_8w_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + const int8_t *filter_horiz, const int8_t *filter_vert, int32_t height) { + uint32_t loop_cnt = height >> 2; + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i filt_hz0, filt_hz1, filt_hz2, filt_hz3; + __m128i filt_vt0, filt_vt1, filt_vt2, filt_vt3; + __m128i mask0, mask1, mask2, mask3; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6; + __m128i out0, out1; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride + src_stride2; + int32_t src_stride4 = src_stride2 << 1; + uint8_t *_src = (uint8_t *)src - 3 - src_stride3; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + DUP4_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_horiz, 2, filter_horiz, 4, + filter_horiz, 6, filt_hz0, filt_hz1, filt_hz2, filt_hz3); + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + + src0 = __lsx_vld(_src, 0); + DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src1, src2); + src3 = __lsx_vldx(_src, src_stride3); + _src += src_stride4; + src4 = __lsx_vld(_src, 0); + DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src5, src6); + _src += src_stride3; + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + + src0 = HORIZ_8TAP_FILT(src0, src0, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src1 = HORIZ_8TAP_FILT(src1, src1, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src2 = HORIZ_8TAP_FILT(src2, src2, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src3 = HORIZ_8TAP_FILT(src3, src3, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src4 = HORIZ_8TAP_FILT(src4, src4, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src5 = HORIZ_8TAP_FILT(src5, src5, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src6 = HORIZ_8TAP_FILT(src6, src6, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + + DUP4_ARG2(__lsx_vldrepl_h, filter_vert, 0, filter_vert, 2, filter_vert, 4, + filter_vert, 6, filt_vt0, filt_vt1, filt_vt2, filt_vt3); + DUP4_ARG2(__lsx_vpackev_b, src1, src0, src3, src2, src5, src4, src2, src1, + tmp0, tmp1, tmp2, tmp4); + DUP2_ARG2(__lsx_vpackev_b, src4, src3, src6, src5, tmp5, tmp6); + + for (; loop_cnt--;) { + src7 = __lsx_vld(_src, 0); + DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src8, src9); + src10 = __lsx_vldx(_src, src_stride3); + _src += src_stride4; + + DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7, + src8, src9, src10); + src7 = HORIZ_8TAP_FILT(src7, src7, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + tmp3 = __lsx_vpackev_b(src7, src6); + out0 = FILT_8TAP_DPADD_S_H(tmp0, tmp1, tmp2, tmp3, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + src8 = HORIZ_8TAP_FILT(src8, src8, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src0 = __lsx_vpackev_b(src8, src7); + out1 = FILT_8TAP_DPADD_S_H(tmp4, tmp5, tmp6, src0, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + src9 = HORIZ_8TAP_FILT(src9, src9, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src1 = __lsx_vpackev_b(src9, src8); + src3 = FILT_8TAP_DPADD_S_H(tmp1, tmp2, tmp3, src1, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + src10 = HORIZ_8TAP_FILT(src10, src10, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src2 = __lsx_vpackev_b(src10, src9); + src4 = FILT_8TAP_DPADD_S_H(tmp5, tmp6, src0, src2, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, FILTER_BITS, src4, src3, + FILTER_BITS, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + src5 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + src7 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + src8 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + src9 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_d, src7, src5, src9, src8, src5, src7); + DUP2_ARG2(__lsx_vavgr_bu, out0, src5, out1, src7, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 1); + dst += dst_stride; + + src6 = src10; + tmp0 = tmp2; + tmp1 = tmp3; + tmp2 = src1; + tmp4 = tmp6; + tmp5 = src0; + tmp6 = src2; + } +} + +static void common_hv_8ht_8vt_and_aver_dst_16w_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + const int8_t *filter_horiz, const int8_t *filter_vert, int32_t height) { + common_hv_8ht_8vt_and_aver_dst_8w_lsx(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, height); + src += 8; + dst += 8; + + common_hv_8ht_8vt_and_aver_dst_8w_lsx(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, height); +} + +static void common_hv_8ht_8vt_and_aver_dst_32w_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + const int8_t *filter_horiz, const int8_t *filter_vert, int32_t height) { + int32_t multiple8_cnt; + + for (multiple8_cnt = 4; multiple8_cnt--;) { + common_hv_8ht_8vt_and_aver_dst_8w_lsx(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, height); + src += 8; + dst += 8; + } +} + +static void common_hv_8ht_8vt_and_aver_dst_64w_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + const int8_t *filter_horiz, const int8_t *filter_vert, int32_t height) { + int32_t multiple8_cnt; + + for (multiple8_cnt = 8; multiple8_cnt--;) { + common_hv_8ht_8vt_and_aver_dst_8w_lsx(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, height); + + src += 8; + dst += 8; + } +} + +static void common_hv_2ht_2vt_and_aver_dst_4x4_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert) { + __m128i src0, src1, src2, src3, src4, mask; + __m128i filt_hz, filt_vt, vec0, vec1; + __m128i dst0, dst1, dst2, dst3; + __m128i hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, tmp0, tmp1, out; + __m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 }; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + + mask = __lsx_vld(mc_filt_mask_arr, 16); + /* rearranging filter */ + DUP2_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_vert, 0, filt_hz, filt_vt); + + src0 = __lsx_vld(src, 0); + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src1, src2, src3, src4); + + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src1, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src2, src3, mask, filt_hz, FILTER_BITS); + hz_out4 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + hz_out1 = __lsx_vshuf_b(hz_out2, hz_out0, shuff); + hz_out3 = __lsx_vpickod_d(hz_out4, hz_out2); + DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1); + + dst0 = __lsx_vldrepl_w(dst, 0); + dst1 = __lsx_vldrepl_w(dst + dst_stride, 0); + dst2 = __lsx_vldrepl_w(dst + dst_stride2, 0); + dst3 = __lsx_vldrepl_w(dst + dst_stride3, 0); + dst0 = __lsx_vilvl_w(dst1, dst0); + dst1 = __lsx_vilvl_w(dst3, dst2); + dst0 = __lsx_vilvl_d(dst1, dst0); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1); + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + out = __lsx_vpickev_b(tmp1, tmp0); + out = __lsx_vavgr_bu(out, dst0); + __lsx_vstelm_w(out, dst, 0, 0); + __lsx_vstelm_w(out, dst + dst_stride, 0, 1); + __lsx_vstelm_w(out, dst + dst_stride2, 0, 2); + __lsx_vstelm_w(out, dst + dst_stride3, 0, 3); +} + +static void common_hv_2ht_2vt_and_aver_dst_4x8_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert) { + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, mask; + __m128i filt_hz, filt_vt, vec0, vec1, vec2, vec3, res0, res1; + __m128i hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6; + __m128i hz_out7, hz_out8, tmp0, tmp1, tmp2, tmp3; + __m128i dst0, dst1, dst2, dst3, dst4; + __m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 }; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + mask = __lsx_vld(mc_filt_mask_arr, 16); + + /* rearranging filter */ + filt_hz = __lsx_vldrepl_h(filter_horiz, 0); + filt_vt = __lsx_vldrepl_h(filter_vert, 0); + + src0 = __lsx_vld(src, 0); + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src1, src2, src3, src4); + src += src_stride4; + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src5, src6, src7, src8); + src += src_stride4; + + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src1, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src2, src3, mask, filt_hz, FILTER_BITS); + hz_out4 = HORIZ_2TAP_FILT_UH(src4, src5, mask, filt_hz, FILTER_BITS); + hz_out6 = HORIZ_2TAP_FILT_UH(src6, src7, mask, filt_hz, FILTER_BITS); + hz_out8 = HORIZ_2TAP_FILT_UH(src8, src8, mask, filt_hz, FILTER_BITS); + DUP2_ARG3(__lsx_vshuf_b, hz_out2, hz_out0, shuff, hz_out4, hz_out2, shuff, + hz_out1, hz_out3); + hz_out5 = __lsx_vshuf_b(hz_out6, hz_out4, shuff); + hz_out7 = __lsx_vpickod_d(hz_out8, hz_out6); + + dst0 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst0 = __lsx_vilvl_w(dst1, dst0); + dst1 = __lsx_vilvl_w(dst3, dst2); + dst0 = __lsx_vilvl_d(dst1, dst0); + + dst1 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst4 = __lsx_vldrepl_w(dst_tmp, 0); + dst1 = __lsx_vilvl_w(dst2, dst1); + dst2 = __lsx_vilvl_w(dst4, dst3); + dst1 = __lsx_vilvl_d(dst2, dst1); + + DUP4_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, hz_out5, + hz_out4, hz_out7, hz_out6, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, vec2, filt_vt, vec3, + filt_vt, tmp0, tmp1, tmp2, tmp3); + DUP4_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp2, + FILTER_BITS, tmp3, FILTER_BITS, tmp0, tmp1, tmp2, tmp3); + DUP2_ARG2(__lsx_vpickev_b, tmp1, tmp0, tmp3, tmp2, res0, res1); + DUP2_ARG2(__lsx_vavgr_bu, res0, dst0, res1, dst1, res0, res1); + + __lsx_vstelm_w(res0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(res0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(res0, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(res0, dst, 0, 3); + dst += dst_stride; + + __lsx_vstelm_w(res1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(res1, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(res1, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(res1, dst, 0, 3); +} + +static void common_hv_2ht_2vt_and_aver_dst_4w_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, int32_t height) { + if (height == 4) { + common_hv_2ht_2vt_and_aver_dst_4x4_lsx(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert); + } else if (height == 8) { + common_hv_2ht_2vt_and_aver_dst_4x8_lsx(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert); + } +} + +static void common_hv_2ht_2vt_and_aver_dst_8x4_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert) { + __m128i src0, src1, src2, src3, src4, mask; + __m128i filt_hz, filt_vt, vec0, vec1, vec2, vec3; + __m128i hz_out0, hz_out1, tmp0, tmp1, tmp2, tmp3; + __m128i dst0, dst1, dst2, dst3; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + uint8_t *dst_tmp = dst; + mask = __lsx_vld(mc_filt_mask_arr, 0); + /* rearranging filter */ + filt_hz = __lsx_vldrepl_h(filter_horiz, 0); + filt_vt = __lsx_vldrepl_h(filter_vert, 0); + + src0 = __lsx_vld(src, 0); + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src1, src2, src3, src4); + + dst0 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_d(dst_tmp, 0); + DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1); + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS); + hz_out1 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + vec0 = __lsx_vpackev_b(hz_out1, hz_out0); + tmp0 = __lsx_vdp2_h_bu(vec0, filt_vt); + + hz_out0 = HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS); + vec1 = __lsx_vpackev_b(hz_out0, hz_out1); + tmp1 = __lsx_vdp2_h_bu(vec1, filt_vt); + + hz_out1 = HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS); + vec2 = __lsx_vpackev_b(hz_out1, hz_out0); + tmp2 = __lsx_vdp2_h_bu(vec2, filt_vt); + + hz_out0 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + vec3 = __lsx_vpackev_b(hz_out0, hz_out1); + tmp3 = __lsx_vdp2_h_bu(vec3, filt_vt); + + DUP4_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp2, + FILTER_BITS, tmp3, FILTER_BITS, tmp0, tmp1, tmp2, tmp3); + PCKEV_AVG_ST4_D(tmp0, tmp1, tmp2, tmp3, dst0, dst1, dst, dst_stride); +} + +static void common_hv_2ht_2vt_and_aver_dst_8x8mult_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, int32_t height) { + uint32_t loop_cnt = (height >> 2); + __m128i src0, src1, src2, src3, src4, mask; + __m128i filt_hz, filt_vt, vec0; + __m128i hz_out0, hz_out1, tmp0, tmp1, tmp2, tmp3; + __m128i dst0, dst1, dst2, dst3; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + uint8_t *dst_tmp = dst; + + /* rearranging filter */ + mask = __lsx_vld(mc_filt_mask_arr, 0); + + filt_hz = __lsx_vldrepl_h(filter_horiz, 0); + filt_vt = __lsx_vldrepl_h(filter_vert, 0); + + src0 = __lsx_vld(src, 0); + src += src_stride; + + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS); + + for (; loop_cnt--;) { + src1 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src3); + src4 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + hz_out1 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + vec0 = __lsx_vpackev_b(hz_out1, hz_out0); + tmp0 = __lsx_vdp2_h_bu(vec0, filt_vt); + + hz_out0 = HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS); + vec0 = __lsx_vpackev_b(hz_out0, hz_out1); + tmp1 = __lsx_vdp2_h_bu(vec0, filt_vt); + + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + + hz_out1 = HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS); + vec0 = __lsx_vpackev_b(hz_out1, hz_out0); + tmp2 = __lsx_vdp2_h_bu(vec0, filt_vt); + + hz_out0 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + vec0 = __lsx_vpackev_b(hz_out0, hz_out1); + tmp3 = __lsx_vdp2_h_bu(vec0, filt_vt); + + DUP2_ARG2(__lsx_vsrari_h, tmp2, FILTER_BITS, tmp3, FILTER_BITS, tmp2, tmp3); + + dst0 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1); + PCKEV_AVG_ST4_D(tmp0, tmp1, tmp2, tmp3, dst0, dst1, dst, dst_stride); + dst += dst_stride; + } +} + +static void common_hv_2ht_2vt_and_aver_dst_8w_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, int32_t height) { + if (height == 4) { + common_hv_2ht_2vt_and_aver_dst_8x4_lsx(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert); + } else { + common_hv_2ht_2vt_and_aver_dst_8x8mult_lsx( + src, src_stride, dst, dst_stride, filter_horiz, filter_vert, height); + } +} + +static void common_hv_2ht_2vt_and_aver_dst_16w_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, int32_t height) { + uint8_t *src_tmp1; + uint32_t loop_cnt = (height >> 2); + __m128i src0, src1, src2, src3, src4, src5, src6, src7, mask; + __m128i filt_hz, filt_vt, vec0, vec1, dst0, dst1, dst2, dst3; + __m128i hz_out0, hz_out1, hz_out2, hz_out3, tmp0, tmp1, tmp3; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + int32_t dst_stride4 = dst_stride << 2; + + mask = __lsx_vld(mc_filt_mask_arr, 0); + /* rearranging filter */ + filt_hz = __lsx_vldrepl_h(filter_horiz, 0); + filt_vt = __lsx_vldrepl_h(filter_vert, 0); + + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src0, src1); + src += src_stride; + + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + + for (; loop_cnt--;) { + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src4); + src6 = __lsx_vldx(src, src_stride3); + src_tmp1 = (uint8_t *)(src + 8); + src1 = __lsx_vld(src_tmp1, 0); + DUP2_ARG2(__lsx_vldx, src_tmp1, src_stride, src_tmp1, src_stride2, src3, + src5); + src7 = __lsx_vldx(src_tmp1, src_stride3); + src += src_stride4; + dst0 = __lsx_vld(dst, 0); + DUP2_ARG2(__lsx_vldx, dst, dst_stride, dst, dst_stride2, dst1, dst2); + dst3 = __lsx_vldx(dst, dst_stride3); + + hz_out1 = HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS); + hz_out3 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1); + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + tmp3 = __lsx_vpickev_b(tmp1, tmp0); + tmp3 = __lsx_vavgr_bu(tmp3, dst0); + __lsx_vst(tmp3, dst, 0); + + hz_out0 = HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS); + DUP2_ARG2(__lsx_vpackev_b, hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1); + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + tmp3 = __lsx_vpickev_b(tmp1, tmp0); + tmp3 = __lsx_vavgr_bu(tmp3, dst1); + __lsx_vstx(tmp3, dst, dst_stride); + + hz_out1 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + hz_out3 = HORIZ_2TAP_FILT_UH(src5, src5, mask, filt_hz, FILTER_BITS); + DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1); + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + tmp3 = __lsx_vpickev_b(tmp1, tmp0); + tmp3 = __lsx_vavgr_bu(tmp3, dst2); + __lsx_vstx(tmp3, dst, dst_stride2); + + hz_out0 = HORIZ_2TAP_FILT_UH(src6, src6, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src7, src7, mask, filt_hz, FILTER_BITS); + DUP2_ARG2(__lsx_vpackev_b, hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1); + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + tmp3 = __lsx_vpickev_b(tmp1, tmp0); + tmp3 = __lsx_vavgr_bu(tmp3, dst3); + __lsx_vstx(tmp3, dst, dst_stride3); + dst += dst_stride4; + } +} + +static void common_hv_2ht_2vt_and_aver_dst_32w_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, int32_t height) { + common_hv_2ht_2vt_and_aver_dst_16w_lsx(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, height); + src += 16; + dst += 16; + + common_hv_2ht_2vt_and_aver_dst_16w_lsx(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, height); + src += 16; + dst += 16; +} + +static void common_hv_2ht_2vt_and_aver_dst_64w_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, int32_t height) { + int32_t multiple8_cnt; + for (multiple8_cnt = 4; multiple8_cnt--;) { + common_hv_2ht_2vt_and_aver_dst_16w_lsx(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, height); + src += 16; + dst += 16; + } +} + +void vpx_convolve8_avg_lsx(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *filter, int x0_q4, int x_step_q4, + int y0_q4, int y_step_q4, int w, int h) { + const int16_t *const filter_x = filter[x0_q4]; + const int16_t *const filter_y = filter[y0_q4]; + int8_t cnt, filt_hor[8], filt_ver[8]; + + assert(x_step_q4 == 16); + assert(y_step_q4 == 16); + assert(((const int32_t *)filter_x)[1] != 0x800000); + assert(((const int32_t *)filter_y)[1] != 0x800000); + + for (cnt = 0; cnt < 8; ++cnt) { + filt_hor[cnt] = filter_x[cnt]; + filt_ver[cnt] = filter_y[cnt]; + } + if (vpx_get_filter_taps(filter_x) == 2 && + vpx_get_filter_taps(filter_y) == 2) { + switch (w) { + case 4: + common_hv_2ht_2vt_and_aver_dst_4w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], + &filt_ver[3], h); + break; + case 8: + common_hv_2ht_2vt_and_aver_dst_8w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], + &filt_ver[3], h); + break; + case 16: + common_hv_2ht_2vt_and_aver_dst_16w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, + &filt_hor[3], &filt_ver[3], h); + break; + case 32: + common_hv_2ht_2vt_and_aver_dst_32w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, + &filt_hor[3], &filt_ver[3], h); + break; + case 64: + common_hv_2ht_2vt_and_aver_dst_64w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, + &filt_hor[3], &filt_ver[3], h); + break; + default: + vpx_convolve8_avg_c(src, src_stride, dst, dst_stride, filter, x0_q4, + x_step_q4, y0_q4, y_step_q4, w, h); + break; + } + } else if (vpx_get_filter_taps(filter_x) == 2 || + vpx_get_filter_taps(filter_y) == 2) { + vpx_convolve8_avg_c(src, src_stride, dst, dst_stride, filter, x0_q4, + x_step_q4, y0_q4, y_step_q4, w, h); + } else { + switch (w) { + case 4: + common_hv_8ht_8vt_and_aver_dst_4w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, + filt_ver, h); + break; + case 8: + common_hv_8ht_8vt_and_aver_dst_8w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, + filt_ver, h); + break; + case 16: + common_hv_8ht_8vt_and_aver_dst_16w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, + filt_ver, h); + break; + case 32: + common_hv_8ht_8vt_and_aver_dst_32w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, + filt_ver, h); + break; + case 64: + common_hv_8ht_8vt_and_aver_dst_64w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, + filt_ver, h); + break; + default: + vpx_convolve8_avg_c(src, src_stride, dst, dst_stride, filter, x0_q4, + x_step_q4, y0_q4, y_step_q4, w, h); + break; + } + } +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_avg_vert_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_avg_vert_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_avg_vert_lsx.c @@ -0,0 +1,918 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include "./vpx_dsp_rtcd.h" +#include "vpx_dsp/loongarch/vpx_convolve_lsx.h" + +static void common_vt_8t_and_aver_dst_4w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) { + uint32_t loop_cnt = (height >> 2); + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + __m128i reg0, reg1, reg2, reg3, reg4; + __m128i filter0, filter1, filter2, filter3; + __m128i out0, out1; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride + src_stride2; + int32_t src_stride4 = src_stride2 << 1; + uint8_t *src_tmp0 = (uint8_t *)src - src_stride3; + + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + src0 = __lsx_vld(src_tmp0, 0); + DUP2_ARG2(__lsx_vldx, src_tmp0, src_stride, src_tmp0, src_stride2, src1, + src2); + src3 = __lsx_vldx(src_tmp0, src_stride3); + src_tmp0 += src_stride4; + src4 = __lsx_vld(src_tmp0, 0); + DUP2_ARG2(__lsx_vldx, src_tmp0, src_stride, src_tmp0, src_stride2, src5, + src6); + src_tmp0 += src_stride3; + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, tmp0, + tmp1, tmp2, tmp3); + DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, tmp4, tmp5); + DUP2_ARG2(__lsx_vilvl_d, tmp3, tmp0, tmp4, tmp1, reg0, reg1); + reg2 = __lsx_vilvl_d(tmp5, tmp2); + DUP2_ARG2(__lsx_vxori_b, reg0, 128, reg1, 128, reg0, reg1); + reg2 = __lsx_vxori_b(reg2, 128); + + for (; loop_cnt--;) { + src7 = __lsx_vld(src_tmp0, 0); + DUP2_ARG2(__lsx_vldx, src_tmp0, src_stride, src_tmp0, src_stride2, src8, + src9); + src10 = __lsx_vldx(src_tmp0, src_stride3); + src_tmp0 += src_stride4; + src0 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + src1 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + src2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + src3 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_w, src1, src0, src3, src2, src0, src1); + src0 = __lsx_vilvl_d(src1, src0); + DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9, + tmp0, tmp1, tmp2, tmp3); + DUP2_ARG2(__lsx_vilvl_d, tmp1, tmp0, tmp3, tmp2, reg3, reg4); + DUP2_ARG2(__lsx_vxori_b, reg3, 128, reg4, 128, reg3, reg4); + out0 = FILT_8TAP_DPADD_S_H(reg0, reg1, reg2, reg3, filter0, filter1, + filter2, filter3); + out1 = FILT_8TAP_DPADD_S_H(reg1, reg2, reg3, reg4, filter0, filter1, + filter2, filter3); + out0 = __lsx_vssrarni_b_h(out1, out0, 7); + out0 = __lsx_vxori_b(out0, 128); + out0 = __lsx_vavgr_bu(out0, src0); + __lsx_vstelm_w(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 3); + dst += dst_stride; + reg0 = reg2; + reg1 = reg3; + reg2 = reg4; + src6 = src10; + } +} + +static void common_vt_8t_and_aver_dst_8w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) { + uint32_t loop_cnt = height >> 2; + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i tmp0, tmp1, tmp2, tmp3; + __m128i reg0, reg1, reg2, reg3, reg4, reg5; + __m128i filter0, filter1, filter2, filter3; + __m128i out0, out1, out2, out3; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride + src_stride2; + int32_t src_stride4 = src_stride2 << 1; + uint8_t *src_tmp0 = (uint8_t *)src - src_stride3; + + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + src0 = __lsx_vld(src_tmp0, 0); + DUP2_ARG2(__lsx_vldx, src_tmp0, src_stride, src_tmp0, src_stride2, src1, + src2); + src3 = __lsx_vldx(src_tmp0, src_stride3); + src_tmp0 += src_stride4; + src4 = __lsx_vld(src_tmp0, 0); + DUP2_ARG2(__lsx_vldx, src_tmp0, src_stride, src_tmp0, src_stride2, src5, + src6); + src_tmp0 += src_stride3; + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, reg0, + reg1, reg2, reg3); + DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, reg4, reg5); + + for (; loop_cnt--;) { + src7 = __lsx_vld(src_tmp0, 0); + DUP2_ARG2(__lsx_vldx, src_tmp0, src_stride, src_tmp0, src_stride2, src8, + src9); + src10 = __lsx_vldx(src_tmp0, src_stride3); + src_tmp0 += src_stride4; + src0 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + src1 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + src2 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + src3 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_d, src1, src0, src3, src2, src0, src1); + DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7, + src8, src9, src10); + DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9, + tmp0, tmp1, tmp2, tmp3); + out0 = FILT_8TAP_DPADD_S_H(reg0, reg1, reg2, tmp0, filter0, filter1, + filter2, filter3); + out1 = FILT_8TAP_DPADD_S_H(reg3, reg4, reg5, tmp1, filter0, filter1, + filter2, filter3); + out2 = FILT_8TAP_DPADD_S_H(reg1, reg2, tmp0, tmp2, filter0, filter1, + filter2, filter3); + out3 = FILT_8TAP_DPADD_S_H(reg4, reg5, tmp1, tmp3, filter0, filter1, + filter2, filter3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + DUP2_ARG2(__lsx_vavgr_bu, out0, src0, out1, src1, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 1); + dst += dst_stride; + reg0 = reg2; + reg1 = tmp0; + reg2 = tmp2; + reg3 = reg5; + reg4 = tmp1; + reg5 = tmp3; + src6 = src10; + } +} + +static void common_vt_8t_and_aver_dst_16w_mult_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height, int32_t width) { + uint8_t *src_tmp; + uint32_t cnt = width >> 4; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i filter0, filter1, filter2, filter3; + __m128i reg0, reg1, reg2, reg3, reg4, reg5; + __m128i reg6, reg7, reg8, reg9, reg10, reg11; + __m128i tmp0, tmp1, tmp2, tmp3; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride + src_stride2; + int32_t src_stride4 = src_stride2 << 1; + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + int32_t dst_stride4 = dst_stride2 << 1; + uint8_t *src_tmp0 = (uint8_t *)src - src_stride3; + + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + for (; cnt--;) { + uint32_t loop_cnt = height >> 2; + uint8_t *dst_reg = dst; + + src_tmp = src_tmp0; + src0 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src1, + src2); + src3 = __lsx_vldx(src_tmp, src_stride3); + src_tmp += src_stride4; + src4 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src5, + src6); + src_tmp += src_stride3; + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, + reg0, reg1, reg2, reg3); + DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, reg4, reg5); + DUP4_ARG2(__lsx_vilvh_b, src1, src0, src3, src2, src5, src4, src2, src1, + reg6, reg7, reg8, reg9); + DUP2_ARG2(__lsx_vilvh_b, src4, src3, src6, src5, reg10, reg11); + for (; loop_cnt--;) { + src7 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src8, + src9); + src10 = __lsx_vldx(src_tmp, src_stride3); + src_tmp += src_stride4; + DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, + src7, src8, src9, src10); + DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9, + src0, src1, src2, src3); + DUP4_ARG2(__lsx_vilvh_b, src7, src6, src8, src7, src9, src8, src10, src9, + src4, src5, src7, src8); + tmp0 = FILT_8TAP_DPADD_S_H(reg0, reg1, reg2, src0, filter0, filter1, + filter2, filter3); + tmp1 = FILT_8TAP_DPADD_S_H(reg3, reg4, reg5, src1, filter0, filter1, + filter2, filter3); + tmp2 = FILT_8TAP_DPADD_S_H(reg6, reg7, reg8, src4, filter0, filter1, + filter2, filter3); + tmp3 = FILT_8TAP_DPADD_S_H(reg9, reg10, reg11, src5, filter0, filter1, + filter2, filter3); + DUP2_ARG3(__lsx_vssrarni_b_h, tmp2, tmp0, 7, tmp3, tmp1, 7, tmp0, tmp1); + DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + tmp2 = __lsx_vld(dst_reg, 0); + tmp3 = __lsx_vldx(dst_reg, dst_stride); + DUP2_ARG2(__lsx_vavgr_bu, tmp0, tmp2, tmp1, tmp3, tmp0, tmp1); + __lsx_vst(tmp0, dst_reg, 0); + __lsx_vstx(tmp1, dst_reg, dst_stride); + tmp0 = FILT_8TAP_DPADD_S_H(reg1, reg2, src0, src2, filter0, filter1, + filter2, filter3); + tmp1 = FILT_8TAP_DPADD_S_H(reg4, reg5, src1, src3, filter0, filter1, + filter2, filter3); + tmp2 = FILT_8TAP_DPADD_S_H(reg7, reg8, src4, src7, filter0, filter1, + filter2, filter3); + tmp3 = FILT_8TAP_DPADD_S_H(reg10, reg11, src5, src8, filter0, filter1, + filter2, filter3); + DUP2_ARG3(__lsx_vssrarni_b_h, tmp2, tmp0, 7, tmp3, tmp1, 7, tmp0, tmp1); + DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + tmp2 = __lsx_vldx(dst_reg, dst_stride2); + tmp3 = __lsx_vldx(dst_reg, dst_stride3); + DUP2_ARG2(__lsx_vavgr_bu, tmp0, tmp2, tmp1, tmp3, tmp0, tmp1); + __lsx_vstx(tmp0, dst_reg, dst_stride2); + __lsx_vstx(tmp1, dst_reg, dst_stride3); + dst_reg += dst_stride4; + + reg0 = reg2; + reg1 = src0; + reg2 = src2; + reg3 = reg5; + reg4 = src1; + reg5 = src3; + reg6 = reg8; + reg7 = src4; + reg8 = src7; + reg9 = reg11; + reg10 = src5; + reg11 = src8; + src6 = src10; + } + src_tmp0 += 16; + dst += 16; + } +} + +static void common_vt_8t_and_aver_dst_16w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) { + common_vt_8t_and_aver_dst_16w_mult_lsx(src, src_stride, dst, dst_stride, + filter, height, 16); +} + +static void common_vt_8t_and_aver_dst_32w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) { + common_vt_8t_and_aver_dst_16w_mult_lsx(src, src_stride, dst, dst_stride, + filter, height, 32); +} + +static void common_vt_8t_and_aver_dst_64w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) { + common_vt_8t_and_aver_dst_16w_mult_lsx(src, src_stride, dst, dst_stride, + filter, height, 64); +} + +static void common_vt_2t_and_aver_dst_4x4_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter) { + __m128i src0, src1, src2, src3, src4; + __m128i dst0, dst1, dst2, dst3, out, filt0, src2110, src4332; + __m128i src10_r, src32_r, src21_r, src43_r; + __m128i tmp0, tmp1; + uint8_t *dst_tmp = dst; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + filt0 = __lsx_vldrepl_h(filter, 0); + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + src += src_stride4; + src4 = __lsx_vld(src, 0); + src += src_stride; + + dst0 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_w(dst_tmp, 0); + dst0 = __lsx_vilvl_w(dst1, dst0); + dst1 = __lsx_vilvl_w(dst3, dst2); + dst0 = __lsx_vilvl_d(dst1, dst0); + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src3, src2, src4, src3, + src10_r, src21_r, src32_r, src43_r); + DUP2_ARG2(__lsx_vilvl_d, src21_r, src10_r, src43_r, src32_r, src2110, + src4332); + DUP2_ARG2(__lsx_vdp2_h_bu, src2110, filt0, src4332, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + out = __lsx_vavgr_bu(tmp0, dst0); + __lsx_vstelm_w(out, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(out, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(out, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(out, dst, 0, 3); + dst += dst_stride; +} + +static void common_vt_2t_and_aver_dst_4x8_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter) { + __m128i dst0, dst1, dst2, dst3, dst4; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src87_r; + __m128i src10_r, src32_r, src54_r, src76_r, src21_r, src43_r, src65_r; + __m128i src2110, src4332, src6554, src8776, filt0; + __m128i tmp0, tmp1, tmp2, tmp3; + uint8_t *dst_tmp = dst; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + filt0 = __lsx_vldrepl_h(filter, 0); + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + src += src_stride4; + src4 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src5, src6); + src7 = __lsx_vldx(src, src_stride3); + src += src_stride4; + src8 = __lsx_vld(src, 0); + + dst0 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst0 = __lsx_vilvl_w(dst1, dst0); + dst1 = __lsx_vilvl_w(dst3, dst2); + dst0 = __lsx_vilvl_d(dst1, dst0); + + dst1 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst4 = __lsx_vldrepl_w(dst_tmp, 0); + dst1 = __lsx_vilvl_w(dst2, dst1); + dst2 = __lsx_vilvl_w(dst4, dst3); + dst1 = __lsx_vilvl_d(dst2, dst1); + + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src3, src2, src4, src3, + src10_r, src21_r, src32_r, src43_r); + DUP4_ARG2(__lsx_vilvl_b, src5, src4, src6, src5, src7, src6, src8, src7, + src54_r, src65_r, src76_r, src87_r); + DUP4_ARG2(__lsx_vilvl_d, src21_r, src10_r, src43_r, src32_r, src65_r, src54_r, + src87_r, src76_r, src2110, src4332, src6554, src8776); + DUP4_ARG2(__lsx_vdp2_h_bu, src2110, filt0, src4332, filt0, src6554, filt0, + src8776, filt0, tmp0, tmp1, tmp2, tmp3); + DUP2_ARG3(__lsx_vssrarni_bu_h, tmp1, tmp0, FILTER_BITS, tmp3, tmp2, + FILTER_BITS, tmp0, tmp2); + DUP2_ARG2(__lsx_vavgr_bu, tmp0, dst0, tmp2, dst1, tmp0, tmp2); + __lsx_vstelm_w(tmp0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(tmp0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(tmp0, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(tmp0, dst, 0, 3); + dst += dst_stride; + + __lsx_vstelm_w(tmp2, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(tmp2, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(tmp2, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(tmp2, dst, 0, 3); +} + +static void common_vt_2t_and_aver_dst_4w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, int8_t *filter, + int32_t height) { + if (height == 4) { + common_vt_2t_and_aver_dst_4x4_lsx(src, src_stride, dst, dst_stride, filter); + } else if (height == 8) { + common_vt_2t_and_aver_dst_4x8_lsx(src, src_stride, dst, dst_stride, filter); + } +} + +static void common_vt_2t_and_aver_dst_8x4_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter) { + __m128i src0, src1, src2, src3, src4; + __m128i dst0, dst1, dst2, dst3, vec0, vec1, vec2, vec3, filt0; + __m128i tmp0, tmp1, tmp2, tmp3; + uint8_t *dst_tmp = dst; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + filt0 = __lsx_vldrepl_h(filter, 0); + src0 = __lsx_vld(src, 0); + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src1, src2, src3, src4); + dst0 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1); + DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, vec0, vec1); + DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, vec2, vec3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0, + tmp0, tmp1, tmp2, tmp3); + DUP2_ARG3(__lsx_vssrarni_bu_h, tmp1, tmp0, FILTER_BITS, tmp3, tmp2, + FILTER_BITS, tmp0, tmp2); + DUP2_ARG2(__lsx_vavgr_bu, tmp0, dst0, tmp2, dst1, tmp0, tmp2); + __lsx_vstelm_d(tmp0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(tmp0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(tmp2, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(tmp2, dst, 0, 1); +} + +static void common_vt_2t_and_aver_dst_8x8mult_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = (height >> 3); + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8; + __m128i dst0, dst1, dst2, dst3, dst4, dst5; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, filt0; + __m128i tmp0, tmp1, tmp2, tmp3; + uint8_t *dst_tmp = dst; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + filt0 = __lsx_vldrepl_h(filter, 0); + src0 = __lsx_vld(src, 0); + src += src_stride; + + for (; loop_cnt--;) { + src1 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src3); + src4 = __lsx_vldx(src, src_stride3); + src += src_stride4; + src5 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src6, src7); + src8 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + dst0 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1); + + dst2 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst4 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst5 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_d, dst3, dst2, dst5, dst4, dst2, dst3); + + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src3, src2, src4, src3, + vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vilvl_b, src5, src4, src6, src5, src7, src6, src8, src7, + vec4, vec5, vec6, vec7); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, + filt0, tmp0, tmp1, tmp2, tmp3); + + DUP2_ARG3(__lsx_vssrarni_bu_h, tmp1, tmp0, FILTER_BITS, tmp3, tmp2, + FILTER_BITS, tmp0, tmp2); + DUP2_ARG2(__lsx_vavgr_bu, tmp0, dst0, tmp2, dst1, tmp0, tmp2); + __lsx_vstelm_d(tmp0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(tmp0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(tmp2, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(tmp2, dst, 0, 1); + dst += dst_stride; + + DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7, + filt0, tmp0, tmp1, tmp2, tmp3); + DUP2_ARG3(__lsx_vssrarni_bu_h, tmp1, tmp0, FILTER_BITS, tmp3, tmp2, + FILTER_BITS, tmp0, tmp2); + DUP2_ARG2(__lsx_vavgr_bu, tmp0, dst2, tmp2, dst3, tmp0, tmp2); + __lsx_vstelm_d(tmp0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(tmp0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(tmp2, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(tmp2, dst, 0, 1); + dst += dst_stride; + + src0 = src8; + } +} + +static void common_vt_2t_and_aver_dst_8w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, int8_t *filter, + int32_t height) { + if (height == 4) { + common_vt_2t_and_aver_dst_8x4_lsx(src, src_stride, dst, dst_stride, filter); + } else { + common_vt_2t_and_aver_dst_8x8mult_lsx(src, src_stride, dst, dst_stride, + filter, height); + } +} + +static void common_vt_2t_and_aver_dst_16w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = (height >> 2); + __m128i src0, src1, src2, src3, src4, dst0, dst1, dst2, dst3, filt0; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i tmp0, tmp1; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + + filt0 = __lsx_vldrepl_h(filter, 0); + src0 = __lsx_vld(src, 0); + src += src_stride; + + for (; loop_cnt--;) { + src1 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src3); + src4 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + dst0 = __lsx_vld(dst, 0); + DUP2_ARG2(__lsx_vldx, dst, dst_stride, dst, dst_stride2, dst1, dst2); + dst3 = __lsx_vldx(dst, dst_stride3); + + DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, vec0, vec2); + DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, vec1, vec3); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst0); + __lsx_vst(tmp0, dst, 0); + dst += dst_stride; + + DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, vec4, vec6); + DUP2_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, vec5, vec7); + DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst1); + __lsx_vst(tmp0, dst, 0); + dst += dst_stride; + + DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst2); + __lsx_vst(tmp0, dst, 0); + dst += dst_stride; + + DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst3); + __lsx_vst(tmp0, dst, 0); + dst += dst_stride; + + src0 = src4; + } +} + +static void common_vt_2t_and_aver_dst_32w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = (height >> 2); + uint8_t *src_tmp1; + uint8_t *dst_tmp1; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, filt0; + __m128i tmp0, tmp1; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + + filt0 = __lsx_vldrepl_h(filter, 0); + DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src5); + src += src_stride; + + for (; loop_cnt--;) { + src1 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src3); + src4 = __lsx_vldx(src, src_stride3); + + dst0 = __lsx_vld(dst, 0); + DUP2_ARG2(__lsx_vldx, dst, dst_stride, dst, dst_stride2, dst1, dst2); + dst3 = __lsx_vldx(dst, dst_stride3); + + DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, vec0, vec2); + DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, vec1, vec3); + + src_tmp1 = src + 16; + src6 = __lsx_vld(src_tmp1, 0); + DUP2_ARG2(__lsx_vldx, src_tmp1, src_stride, src_tmp1, src_stride2, src7, + src8); + src9 = __lsx_vldx(src_tmp1, src_stride3); + + dst_tmp1 = dst + 16; + dst4 = __lsx_vld(dst_tmp1, 0); + DUP2_ARG2(__lsx_vldx, dst_tmp1, dst_stride, dst_tmp1, dst_stride2, dst5, + dst6); + dst7 = __lsx_vldx(dst_tmp1, dst_stride3); + src += src_stride4; + + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst0); + __lsx_vst(tmp0, dst, 0); + + DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst1); + __lsx_vstx(tmp0, dst, dst_stride); + + DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, vec4, vec6); + DUP2_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, vec5, vec7); + DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst2); + __lsx_vstx(tmp0, dst, dst_stride2); + + DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst3); + __lsx_vstx(tmp0, dst, dst_stride3); + + DUP2_ARG2(__lsx_vilvl_b, src6, src5, src7, src6, vec0, vec2); + DUP2_ARG2(__lsx_vilvh_b, src6, src5, src7, src6, vec1, vec3); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst4); + __lsx_vst(tmp0, dst, 16); + + DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst5); + dst += dst_stride; + __lsx_vst(tmp0, dst, 16); + + DUP2_ARG2(__lsx_vilvl_b, src8, src7, src9, src8, vec4, vec6); + DUP2_ARG2(__lsx_vilvh_b, src8, src7, src9, src8, vec5, vec7); + DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst6); + dst += dst_stride; + __lsx_vst(tmp0, dst, 16); + + DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst7); + dst += dst_stride; + __lsx_vst(tmp0, dst, 16); + dst += dst_stride; + + src0 = src4; + src5 = src9; + } +} + +static void common_vt_2t_and_aver_dst_64w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = (height >> 1); + int32_t src_stride2 = src_stride << 1; + int32_t dst_stride2 = dst_stride << 1; + uint8_t *src_tmp1; + uint8_t *dst_tmp1; + __m128i src0, src1, src2, src3, src4, src5; + __m128i src6, src7, src8, src9, src10, src11, filt0; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i tmp0, tmp1; + + filt0 = __lsx_vldrepl_h(filter, 0); + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src3, src6, + src9); + src += src_stride; + + for (; loop_cnt--;) { + src2 = __lsx_vldx(src, src_stride); + dst1 = __lsx_vldx(dst, dst_stride); + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src1, src4, src7, + src10); + DUP4_ARG2(__lsx_vld, dst, 0, dst, 16, dst, 32, dst, 48, dst0, dst2, dst4, + dst6); + src_tmp1 = (uint8_t *)src + 16; + src5 = __lsx_vldx(src_tmp1, src_stride); + src_tmp1 = src_tmp1 + 16; + src8 = __lsx_vldx(src_tmp1, src_stride); + src_tmp1 = src_tmp1 + 16; + src11 = __lsx_vldx(src_tmp1, src_stride); + + dst_tmp1 = dst + 16; + dst3 = __lsx_vldx(dst_tmp1, dst_stride); + dst_tmp1 = dst + 32; + dst5 = __lsx_vldx(dst_tmp1, dst_stride); + dst_tmp1 = dst + 48; + dst7 = __lsx_vldx(dst_tmp1, dst_stride); + src += src_stride2; + + DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, vec0, vec2); + DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, vec1, vec3); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst0); + __lsx_vst(tmp0, dst, 0); + + DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst1); + __lsx_vstx(tmp0, dst, dst_stride); + + DUP2_ARG2(__lsx_vilvl_b, src4, src3, src5, src4, vec4, vec6); + DUP2_ARG2(__lsx_vilvh_b, src4, src3, src5, src4, vec5, vec7); + DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst2); + __lsx_vst(tmp0, dst, 16); + + dst_tmp1 = dst + 16; + DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst3); + __lsx_vstx(tmp0, dst_tmp1, dst_stride); + + DUP2_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, vec0, vec2); + DUP2_ARG2(__lsx_vilvh_b, src7, src6, src8, src7, vec1, vec3); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst4); + __lsx_vst(tmp0, dst, 32); + + dst_tmp1 = dst_tmp1 + 16; + DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst5); + __lsx_vstx(tmp0, dst_tmp1, dst_stride); + + DUP2_ARG2(__lsx_vilvl_b, src10, src9, src11, src10, vec4, vec6); + DUP2_ARG2(__lsx_vilvh_b, src10, src9, src11, src10, vec5, vec7); + DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst6); + __lsx_vst(tmp0, dst, 48); + + dst_tmp1 = dst_tmp1 + 16; + DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst7); + __lsx_vstx(tmp0, dst_tmp1, dst_stride); + dst += dst_stride2; + + src0 = src2; + src3 = src5; + src6 = src8; + src9 = src11; + } +} + +void vpx_convolve8_avg_vert_lsx(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *filter, int x0_q4, + int x_step_q4, int y0_q4, int y_step_q4, int w, + int h) { + const int16_t *const filter_y = filter[y0_q4]; + int8_t cnt, filt_ver[8]; + + assert(y_step_q4 == 16); + assert(((const int32_t *)filter_y)[1] != 0x800000); + + for (cnt = 0; cnt < 8; ++cnt) { + filt_ver[cnt] = filter_y[cnt]; + } + + if (vpx_get_filter_taps(filter_y) == 2) { + switch (w) { + case 4: + common_vt_2t_and_aver_dst_4w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_ver[3], h); + break; + case 8: + common_vt_2t_and_aver_dst_8w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_ver[3], h); + break; + case 16: + common_vt_2t_and_aver_dst_16w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_ver[3], h); + break; + case 32: + common_vt_2t_and_aver_dst_32w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_ver[3], h); + break; + case 64: + common_vt_2t_and_aver_dst_64w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_ver[3], h); + break; + default: + vpx_convolve8_avg_vert_c(src, src_stride, dst, dst_stride, filter, + x0_q4, x_step_q4, y0_q4, y_step_q4, w, h); + break; + } + } else { + switch (w) { + case 4: + common_vt_8t_and_aver_dst_4w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_ver, h); + break; + case 8: + common_vt_8t_and_aver_dst_8w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_ver, h); + break; + case 16: + common_vt_8t_and_aver_dst_16w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_ver, h); + + break; + case 32: + common_vt_8t_and_aver_dst_32w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_ver, h); + break; + case 64: + common_vt_8t_and_aver_dst_64w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_ver, h); + break; + default: + vpx_convolve8_avg_vert_c(src, src_stride, dst, dst_stride, filter, + x0_q4, x_step_q4, y0_q4, y_step_q4, w, h); + break; + } + } +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_horiz_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_horiz_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_horiz_lsx.c @@ -0,0 +1,841 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include "./vpx_dsp_rtcd.h" +#include "vpx_dsp/loongarch/vpx_convolve_lsx.h" + +static const uint8_t mc_filt_mask_arr[16 * 3] = { + /* 8 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, + /* 4 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20, + /* 4 width cases */ + 8, 9, 9, 10, 10, 11, 11, 12, 24, 25, 25, 26, 26, 27, 27, 28 +}; + +static void common_hz_8t_4x4_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter) { + __m128i src0, src1, src2, src3; + __m128i filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3; + __m128i out, out0, out1; + + mask0 = __lsx_vld(mc_filt_mask_arr, 16); + src -= 3; + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filter0, filter1, filter2, filter3, out0, out1); + out = __lsx_vssrarni_b_h(out1, out0, 7); + out = __lsx_vxori_b(out, 128); + __lsx_vstelm_w(out, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(out, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(out, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(out, dst, 0, 3); +} + +static void common_hz_8t_4x8_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter) { + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride + src_stride2; + int32_t src_stride4 = src_stride2 << 1; + __m128i src0, src1, src2, src3; + __m128i filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3; + __m128i out0, out1, out2, out3; + uint8_t *_src = (uint8_t *)src - 3; + + mask0 = __lsx_vld(mc_filt_mask_arr, 16); + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + src0 = __lsx_vld(_src, 0); + DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src1, src2); + src3 = __lsx_vldx(_src, src_stride3); + _src += src_stride4; + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filter0, filter1, filter2, filter3, out0, out1); + src0 = __lsx_vld(_src, 0); + DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src1, src2); + src3 = __lsx_vldx(_src, src_stride3); + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filter0, filter1, filter2, filter3, out2, out3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + + __lsx_vstelm_w(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 3); + dst += dst_stride; + __lsx_vstelm_w(out1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(out1, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(out1, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(out1, dst, 0, 3); +} + +static void common_hz_8t_4w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) { + if (height == 4) { + common_hz_8t_4x4_lsx(src, src_stride, dst, dst_stride, filter); + } else if (height == 8) { + common_hz_8t_4x8_lsx(src, src_stride, dst, dst_stride, filter); + } +} + +static void common_hz_8t_8x4_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter) { + __m128i src0, src1, src2, src3; + __m128i filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3; + __m128i out0, out1, out2, out3; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + src -= 3; + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filter0, filter1, filter2, filter3, out0, out1, + out2, out3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 1); +} + +static void common_hz_8t_8x8mult_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) { + uint32_t loop_cnt = height >> 2; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride + src_stride2; + int32_t src_stride4 = src_stride2 << 1; + __m128i src0, src1, src2, src3; + __m128i filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3; + __m128i out0, out1, out2, out3; + uint8_t *_src = (uint8_t *)src - 3; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + for (; loop_cnt--;) { + src0 = __lsx_vld(_src, 0); + DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src1, src2); + src3 = __lsx_vldx(_src, src_stride3); + _src += src_stride4; + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, + mask3, filter0, filter1, filter2, filter3, out0, + out1, out2, out3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 1); + dst += dst_stride; + } +} + +static void common_hz_8t_8w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) { + if (height == 4) { + common_hz_8t_8x4_lsx(src, src_stride, dst, dst_stride, filter); + } else { + common_hz_8t_8x8mult_lsx(src, src_stride, dst, dst_stride, filter, height); + } +} + +static void common_hz_8t_16w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) { + uint32_t loop_cnt = height >> 1; + int32_t stride = src_stride << 1; + __m128i src0, src1, src2, src3; + __m128i filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3; + __m128i out0, out1, out2, out3; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + src -= 3; + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + for (; loop_cnt--;) { + const uint8_t *_src = src + src_stride; + DUP2_ARG2(__lsx_vld, src, 0, _src, 0, src0, src2); + DUP2_ARG2(__lsx_vld, src, 8, _src, 8, src1, src3); + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, + mask3, filter0, filter1, filter2, filter3, out0, + out1, out2, out3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vst(out0, dst, 0); + dst += dst_stride; + __lsx_vst(out1, dst, 0); + dst += dst_stride; + src += stride; + } +} + +static void common_hz_8t_32w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) { + uint32_t loop_cnt = height >> 1; + __m128i src0, src1, src2, src3; + __m128i filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3; + __m128i out0, out1, out2, out3; + __m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 }; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + src -= 3; + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + for (; loop_cnt--;) { + DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src2); + src3 = __lsx_vld(src, 24); + src1 = __lsx_vshuf_b(src2, src0, shuff); + src += src_stride; + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, + mask3, filter0, filter1, filter2, filter3, out0, + out1, out2, out3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vst(out0, dst, 0); + __lsx_vst(out1, dst, 16); + + DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src2); + src3 = __lsx_vld(src, 24); + src1 = __lsx_vshuf_b(src2, src0, shuff); + src += src_stride; + + dst += dst_stride; + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, + mask3, filter0, filter1, filter2, filter3, out0, + out1, out2, out3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vst(out0, dst, 0); + __lsx_vst(out1, dst, 16); + dst += dst_stride; + } +} + +static void common_hz_8t_64w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) { + int32_t loop_cnt = height; + __m128i src0, src1, src2, src3; + __m128i filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3; + __m128i out0, out1, out2, out3; + __m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 }; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + src -= 3; + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + for (; loop_cnt--;) { + DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src2); + src3 = __lsx_vld(src, 24); + src1 = __lsx_vshuf_b(src2, src0, shuff); + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, + mask3, filter0, filter1, filter2, filter3, out0, + out1, out2, out3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vst(out0, dst, 0); + __lsx_vst(out1, dst, 16); + + DUP2_ARG2(__lsx_vld, src, 32, src, 48, src0, src2); + src3 = __lsx_vld(src, 56); + src1 = __lsx_vshuf_b(src2, src0, shuff); + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, + mask3, filter0, filter1, filter2, filter3, out0, + out1, out2, out3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vst(out0, dst, 32); + __lsx_vst(out1, dst, 48); + src += src_stride; + dst += dst_stride; + } +} + +static void common_hz_2t_4x4_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter) { + __m128i src0, src1, src2, src3, mask; + __m128i filt0, vec0, vec1, res0, res1; + __m128i vec2, vec3; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride + src_stride2; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride + dst_stride2; + + mask = __lsx_vld(mc_filt_mask_arr, 16); + /* rearranging filter */ + filt0 = __lsx_vldrepl_h(filter, 0); + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + DUP2_ARG3(__lsx_vshuf_b, src1, src0, mask, src3, src2, mask, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, vec3); + DUP2_ARG2(__lsx_vsrari_h, vec2, FILTER_BITS, vec3, FILTER_BITS, vec2, vec3); + DUP2_ARG2(__lsx_vpickev_b, vec2, vec2, vec3, vec3, res0, res1); + + __lsx_vstelm_w(res0, dst, 0, 0); + __lsx_vstelm_w(res0, dst + dst_stride, 0, 1); + __lsx_vstelm_w(res1, dst + dst_stride2, 0, 0); + __lsx_vstelm_w(res1, dst + dst_stride3, 0, 1); +} + +static void common_hz_2t_4x8_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter) { + __m128i vec0, vec1, vec2, vec3, filt0; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, mask; + __m128i res0, res1, res2, res3; + __m128i vec4, vec5, vec6, vec7; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride + src_stride2; + int32_t src_stride4 = src_stride2 << 1; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride + dst_stride2; + + uint8_t *src_tmp1 = src + src_stride4; + + mask = __lsx_vld(mc_filt_mask_arr, 16); + + /* rearranging filter */ + filt0 = __lsx_vldrepl_h(filter, 0); + + src0 = __lsx_vld(src, 0); + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src1, src2, src3, src4); + DUP2_ARG2(__lsx_vldx, src_tmp1, src_stride, src_tmp1, src_stride2, src5, + src6); + src7 = __lsx_vldx(src_tmp1, src_stride3); + + DUP4_ARG3(__lsx_vshuf_b, src1, src0, mask, src3, src2, mask, src5, src4, mask, + src7, src6, mask, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0, + vec4, vec5, vec6, vec7); + DUP4_ARG2(__lsx_vsrari_h, vec4, FILTER_BITS, vec5, FILTER_BITS, vec6, + FILTER_BITS, vec7, FILTER_BITS, vec4, vec5, vec6, vec7); + DUP4_ARG2(__lsx_vpickev_b, vec4, vec4, vec5, vec5, vec6, vec6, vec7, vec7, + res0, res1, res2, res3); + __lsx_vstelm_w(res0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(res0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(res1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(res1, dst, 0, 1); + dst += dst_stride; + + __lsx_vstelm_w(res2, dst, 0, 0); + __lsx_vstelm_w(res2, dst + dst_stride, 0, 1); + __lsx_vstelm_w(res3, dst + dst_stride2, 0, 0); + __lsx_vstelm_w(res3, dst + dst_stride3, 0, 1); +} + +static void common_hz_2t_4w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + if (height == 4) { + common_hz_2t_4x4_lsx(src, src_stride, dst, dst_stride, filter); + } else if (height == 8) { + common_hz_2t_4x8_lsx(src, src_stride, dst, dst_stride, filter); + } +} + +static void common_hz_2t_8x4_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter) { + __m128i filt0, mask; + __m128i src0, src1, src2, src3; + __m128i vec0, vec1, vec2, vec3; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + + mask = __lsx_vld(mc_filt_mask_arr, 0); + + /* rearranging filter */ + filt0 = __lsx_vldrepl_h(filter, 0); + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, src2, src2, mask, + src3, src3, mask, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0, + vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vsrari_h, vec0, FILTER_BITS, vec1, FILTER_BITS, vec2, + FILTER_BITS, vec3, FILTER_BITS, vec0, vec1, vec2, vec3); + DUP2_ARG2(__lsx_vpickev_b, vec1, vec0, vec3, vec2, src0, src1); + + __lsx_vstelm_d(src0, dst, 0, 0); + __lsx_vstelm_d(src0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(src1, dst + dst_stride2, 0, 0); + __lsx_vstelm_d(src1, dst + dst_stride3, 0, 1); +} + +static void common_hz_2t_8x8mult_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + __m128i filt0, mask; + __m128i src0, src1, src2, src3, out0, out1; + __m128i vec0, vec1, vec2, vec3; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + int32_t dst_stride4 = dst_stride2 << 1; + + mask = __lsx_vld(mc_filt_mask_arr, 0); + + /* rearranging filter */ + filt0 = __lsx_vldrepl_h(filter, 0); + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, src2, src2, mask, + src3, src3, mask, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0, + vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vsrari_h, vec0, FILTER_BITS, vec1, FILTER_BITS, vec2, + FILTER_BITS, vec3, FILTER_BITS, vec0, vec1, vec2, vec3); + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + DUP2_ARG2(__lsx_vpickev_b, vec1, vec0, vec3, vec2, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 1); + dst += dst_stride; + + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, src2, src2, mask, + src3, src3, mask, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0, + vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vsrari_h, vec0, FILTER_BITS, vec1, FILTER_BITS, vec2, + FILTER_BITS, vec3, FILTER_BITS, vec0, vec1, vec2, vec3); + DUP2_ARG2(__lsx_vpickev_b, vec1, vec0, vec3, vec2, out0, out1); + + __lsx_vstelm_d(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 1); + dst += dst_stride; + + if (height == 16) { + uint8_t *dst_tmp1 = dst + dst_stride4; + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, src2, src2, + mask, src3, src3, mask, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, + filt0, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vsrari_h, vec0, FILTER_BITS, vec1, FILTER_BITS, vec2, + FILTER_BITS, vec3, FILTER_BITS, vec0, vec1, vec2, vec3); + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + DUP2_ARG2(__lsx_vpickev_b, vec1, vec0, vec3, vec2, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + __lsx_vstelm_d(out0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(out1, dst + dst_stride2, 0, 0); + __lsx_vstelm_d(out1, dst + dst_stride3, 0, 1); + + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, src2, src2, + mask, src3, src3, mask, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, + filt0, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vsrari_h, vec0, FILTER_BITS, vec1, FILTER_BITS, vec2, + FILTER_BITS, vec3, FILTER_BITS, vec0, vec1, vec2, vec3); + DUP2_ARG2(__lsx_vpickev_b, vec1, vec0, vec3, vec2, out0, out1); + + __lsx_vstelm_d(out0, dst_tmp1, 0, 0); + __lsx_vstelm_d(out0, dst_tmp1 + dst_stride, 0, 1); + __lsx_vstelm_d(out1, dst_tmp1 + dst_stride2, 0, 0); + __lsx_vstelm_d(out1, dst_tmp1 + dst_stride3, 0, 1); + } +} + +static void common_hz_2t_8w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + if (height == 4) { + common_hz_2t_8x4_lsx(src, src_stride, dst, dst_stride, filter); + } else { + common_hz_2t_8x8mult_lsx(src, src_stride, dst, dst_stride, filter, height); + } +} + +static void common_hz_2t_16w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = (height >> 2) - 1; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, mask; + __m128i filt0, vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i out0, out1, out2, out3, out4, out5, out6, out7, tmp; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + uint8_t *src_tmp1 = src + 8; + mask = __lsx_vld(mc_filt_mask_arr, 0); + filt0 = __lsx_vldrepl_h(filter, 0); + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src4); + src6 = __lsx_vldx(src, src_stride3); + src1 = __lsx_vld(src_tmp1, 0); + DUP2_ARG2(__lsx_vldx, src_tmp1, src_stride, src_tmp1, src_stride2, src3, + src5); + src7 = __lsx_vldx(src_tmp1, src_stride3); + src += src_stride4; + + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, src2, src2, mask, + src3, src3, mask, vec0, vec1, vec2, vec3); + DUP4_ARG3(__lsx_vshuf_b, src4, src4, mask, src5, src5, mask, src6, src6, mask, + src7, src7, mask, vec4, vec5, vec6, vec7); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0, + out0, out1, out2, out3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7, filt0, + out4, out5, out6, out7); + DUP4_ARG2(__lsx_vsrari_h, out0, FILTER_BITS, out1, FILTER_BITS, out2, + FILTER_BITS, out3, FILTER_BITS, out0, out1, out2, out3); + DUP4_ARG2(__lsx_vsrari_h, out4, FILTER_BITS, out5, FILTER_BITS, out6, + FILTER_BITS, out7, FILTER_BITS, out4, out5, out6, out7); + + tmp = __lsx_vpickev_b(out1, out0); + __lsx_vst(tmp, dst, 0); + dst += dst_stride; + tmp = __lsx_vpickev_b(out3, out2); + __lsx_vst(tmp, dst, 0); + dst += dst_stride; + tmp = __lsx_vpickev_b(out5, out4); + __lsx_vst(tmp, dst, 0); + dst += dst_stride; + tmp = __lsx_vpickev_b(out7, out6); + __lsx_vst(tmp, dst, 0); + dst += dst_stride; + + for (; loop_cnt--;) { + src_tmp1 += src_stride4; + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src4); + src6 = __lsx_vldx(src, src_stride3); + + src1 = __lsx_vld(src_tmp1, 0); + DUP2_ARG2(__lsx_vldx, src_tmp1, src_stride, src_tmp1, src_stride2, src3, + src5); + src7 = __lsx_vldx(src_tmp1, src_stride3); + src += src_stride4; + + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, src2, src2, + mask, src3, src3, mask, vec0, vec1, vec2, vec3); + DUP4_ARG3(__lsx_vshuf_b, src4, src4, mask, src5, src5, mask, src6, src6, + mask, src7, src7, mask, vec4, vec5, vec6, vec7); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, + filt0, out0, out1, out2, out3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7, + filt0, out4, out5, out6, out7); + DUP4_ARG2(__lsx_vsrari_h, out0, FILTER_BITS, out1, FILTER_BITS, out2, + FILTER_BITS, out3, FILTER_BITS, out0, out1, out2, out3); + DUP4_ARG2(__lsx_vsrari_h, out4, FILTER_BITS, out5, FILTER_BITS, out6, + FILTER_BITS, out7, FILTER_BITS, out4, out5, out6, out7); + + tmp = __lsx_vpickev_b(out1, out0); + __lsx_vst(tmp, dst, 0); + dst += dst_stride; + tmp = __lsx_vpickev_b(out3, out2); + __lsx_vst(tmp, dst, 0); + dst += dst_stride; + tmp = __lsx_vpickev_b(out5, out4); + __lsx_vst(tmp, dst, 0); + dst += dst_stride; + tmp = __lsx_vpickev_b(out7, out6); + __lsx_vst(tmp, dst, 0); + dst += dst_stride; + } +} + +static void common_hz_2t_32w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = (height >> 1); + __m128i src0, src1, src2, src3, src4, src5, src6, src7, mask; + __m128i filt0, vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i out0, out1, out2, out3, out4, out5, out6, out7, tmp; + __m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 }; + + mask = __lsx_vld(mc_filt_mask_arr, 0); + /* rearranging filter */ + filt0 = __lsx_vldrepl_h(filter, 0); + + for (; loop_cnt--;) { + DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src2); + src3 = __lsx_vld(src, 24); + src1 = __lsx_vshuf_b(src2, src0, shuff); + src += src_stride; + DUP2_ARG2(__lsx_vld, src, 0, src, 16, src4, src6); + src7 = __lsx_vld(src, 24); + src5 = __lsx_vshuf_b(src6, src4, shuff); + src += src_stride; + + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, src2, src2, + mask, src3, src3, mask, vec0, vec1, vec2, vec3); + DUP4_ARG3(__lsx_vshuf_b, src4, src4, mask, src5, src5, mask, src6, src6, + mask, src7, src7, mask, vec4, vec5, vec6, vec7); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, + filt0, out0, out1, out2, out3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7, + filt0, out4, out5, out6, out7); + DUP4_ARG2(__lsx_vsrari_h, out0, FILTER_BITS, out1, FILTER_BITS, out2, + FILTER_BITS, out3, FILTER_BITS, out0, out1, out2, out3); + DUP4_ARG2(__lsx_vsrari_h, out4, FILTER_BITS, out5, FILTER_BITS, out6, + FILTER_BITS, out7, FILTER_BITS, out4, out5, out6, out7); + + tmp = __lsx_vpickev_b(out1, out0); + __lsx_vst(tmp, dst, 0); + tmp = __lsx_vpickev_b(out3, out2); + __lsx_vst(tmp, dst, 16); + dst += dst_stride; + + tmp = __lsx_vpickev_b(out5, out4); + __lsx_vst(tmp, dst, 0); + tmp = __lsx_vpickev_b(out7, out6); + __lsx_vst(tmp, dst, 16); + dst += dst_stride; + } +} + +static void common_hz_2t_64w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = height; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, mask; + __m128i filt0, vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i out0, out1, out2, out3, out4, out5, out6, out7, tmp; + __m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 }; + + mask = __lsx_vld(mc_filt_mask_arr, 0); + + /* rearranging filter */ + filt0 = __lsx_vldrepl_h(filter, 0); + + for (; loop_cnt--;) { + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src2, src4, + src6); + src7 = __lsx_vld(src, 56); + DUP2_ARG3(__lsx_vshuf_b, src2, src0, shuff, src4, src2, shuff, src1, src3); + src5 = __lsx_vshuf_b(src6, src4, shuff); + src += src_stride; + + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, src2, src2, + mask, src3, src3, mask, vec0, vec1, vec2, vec3); + DUP4_ARG3(__lsx_vshuf_b, src4, src4, mask, src5, src5, mask, src6, src6, + mask, src7, src7, mask, vec4, vec5, vec6, vec7); + + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, + filt0, out0, out1, out2, out3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7, + filt0, out4, out5, out6, out7); + DUP4_ARG2(__lsx_vsrari_h, out0, FILTER_BITS, out1, FILTER_BITS, out2, + FILTER_BITS, out3, FILTER_BITS, out0, out1, out2, out3); + DUP4_ARG2(__lsx_vsrari_h, out4, FILTER_BITS, out5, FILTER_BITS, out6, + FILTER_BITS, out7, FILTER_BITS, out4, out5, out6, out7); + + tmp = __lsx_vpickev_b(out1, out0); + __lsx_vst(tmp, dst, 0); + tmp = __lsx_vpickev_b(out3, out2); + __lsx_vst(tmp, dst, 16); + tmp = __lsx_vpickev_b(out5, out4); + __lsx_vst(tmp, dst, 32); + tmp = __lsx_vpickev_b(out7, out6); + __lsx_vst(tmp, dst, 48); + dst += dst_stride; + } +} + +void vpx_convolve8_horiz_lsx(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *filter, int x0_q4, + int x_step_q4, int y0_q4, int y_step_q4, int w, + int h) { + const int16_t *const filter_x = filter[x0_q4]; + int8_t cnt, filt_hor[8]; + + assert(x_step_q4 == 16); + assert(((const int32_t *)filter_x)[1] != 0x800000); + + for (cnt = 0; cnt < 8; ++cnt) { + filt_hor[cnt] = filter_x[cnt]; + } + if (vpx_get_filter_taps(filter_x) == 2) { + switch (w) { + case 4: + common_hz_2t_4w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + &filt_hor[3], h); + break; + case 8: + common_hz_2t_8w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + &filt_hor[3], h); + break; + case 16: + common_hz_2t_16w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + &filt_hor[3], h); + break; + case 32: + common_hz_2t_32w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + &filt_hor[3], h); + break; + case 64: + common_hz_2t_64w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + &filt_hor[3], h); + break; + default: + vpx_convolve8_horiz_c(src, src_stride, dst, dst_stride, filter, x0_q4, + x_step_q4, y0_q4, y_step_q4, w, h); + break; + } + } else { + switch (w) { + case 4: + common_hz_8t_4w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + filt_hor, h); + break; + case 8: + common_hz_8t_8w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + filt_hor, h); + break; + + case 16: + common_hz_8t_16w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + filt_hor, h); + break; + + case 32: + common_hz_8t_32w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + filt_hor, h); + break; + + case 64: + common_hz_8t_64w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + filt_hor, h); + break; + default: + vpx_convolve8_horiz_c(src, src_stride, dst, dst_stride, filter, x0_q4, + x_step_q4, y0_q4, y_step_q4, w, h); + break; + } + } +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_lsx.c @@ -0,0 +1,707 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include "./vpx_dsp_rtcd.h" +#include "vpx_dsp/loongarch/vpx_convolve_lsx.h" + +static const uint8_t mc_filt_mask_arr[16 * 3] = { + /* 8 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, + /* 4 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20, + /* 4 width cases */ + 8, 9, 9, 10, 10, 11, 11, 12, 24, 25, 25, 26, 26, 27, 27, 28 +}; + +static void common_hv_8ht_8vt_4w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, + int32_t height) { + uint32_t loop_cnt = (height >> 2); + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i filt_hz0, filt_hz1, filt_hz2, filt_hz3; + __m128i filt_vt0, filt_vt1, filt_vt2, filt_vt3; + __m128i mask0, mask1, mask2, mask3; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + __m128i out0, out1; + __m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 }; + + mask0 = __lsx_vld(mc_filt_mask_arr, 16); + src -= (3 + 3 * src_stride); + DUP4_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_horiz, 2, filter_horiz, 4, + filter_horiz, 6, filt_hz0, filt_hz1, filt_hz2, filt_hz3); + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + src += src_stride; + src4 = __lsx_vld(src, 0); + src += src_stride; + src5 = __lsx_vld(src, 0); + src += src_stride; + src6 = __lsx_vld(src, 0); + src += src_stride; + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + + tmp0 = HORIZ_8TAP_FILT(src0, src1, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + tmp2 = HORIZ_8TAP_FILT(src2, src3, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + tmp4 = HORIZ_8TAP_FILT(src4, src5, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + tmp5 = HORIZ_8TAP_FILT(src5, src6, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + DUP2_ARG3(__lsx_vshuf_b, tmp2, tmp0, shuff, tmp4, tmp2, shuff, tmp1, tmp3); + DUP4_ARG2(__lsx_vldrepl_h, filter_vert, 0, filter_vert, 2, filter_vert, 4, + filter_vert, 6, filt_vt0, filt_vt1, filt_vt2, filt_vt3); + DUP2_ARG2(__lsx_vpackev_b, tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + tmp2 = __lsx_vpackev_b(tmp5, tmp4); + + for (; loop_cnt--;) { + LSX_LD_4(src, src_stride, src7, src8, src9, src10); + src += src_stride; + DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7, + src8, src9, src10); + tmp3 = HORIZ_8TAP_FILT(src7, src8, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + tmp4 = __lsx_vshuf_b(tmp3, tmp5, shuff); + tmp4 = __lsx_vpackev_b(tmp3, tmp4); + out0 = FILT_8TAP_DPADD_S_H(tmp0, tmp1, tmp2, tmp4, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + src1 = HORIZ_8TAP_FILT(src9, src10, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src0 = __lsx_vshuf_b(src1, tmp3, shuff); + src0 = __lsx_vpackev_b(src1, src0); + out1 = FILT_8TAP_DPADD_S_H(tmp1, tmp2, tmp4, src0, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + out0 = __lsx_vssrarni_b_h(out1, out0, 7); + out0 = __lsx_vxori_b(out0, 128); + __lsx_vstelm_w(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 3); + dst += dst_stride; + + tmp5 = src1; + tmp0 = tmp2; + tmp1 = tmp4; + tmp2 = src0; + } +} + +static void common_hv_8ht_8vt_8w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, + int32_t height) { + uint32_t loop_cnt = (height >> 2); + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i filt_hz0, filt_hz1, filt_hz2, filt_hz3; + __m128i filt_vt0, filt_vt1, filt_vt2, filt_vt3; + __m128i mask0, mask1, mask2, mask3; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6; + __m128i out0, out1; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + src -= (3 + 3 * src_stride); + DUP4_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_horiz, 2, filter_horiz, 4, + filter_horiz, 6, filt_hz0, filt_hz1, filt_hz2, filt_hz3); + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + src += src_stride; + src4 = __lsx_vld(src, 0); + src += src_stride; + src5 = __lsx_vld(src, 0); + src += src_stride; + src6 = __lsx_vld(src, 0); + src += src_stride; + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + + src0 = HORIZ_8TAP_FILT(src0, src0, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src1 = HORIZ_8TAP_FILT(src1, src1, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src2 = HORIZ_8TAP_FILT(src2, src2, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src3 = HORIZ_8TAP_FILT(src3, src3, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src4 = HORIZ_8TAP_FILT(src4, src4, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src5 = HORIZ_8TAP_FILT(src5, src5, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src6 = HORIZ_8TAP_FILT(src6, src6, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + + DUP4_ARG2(__lsx_vldrepl_h, filter_vert, 0, filter_vert, 2, filter_vert, 4, + filter_vert, 6, filt_vt0, filt_vt1, filt_vt2, filt_vt3); + DUP4_ARG2(__lsx_vpackev_b, src1, src0, src3, src2, src5, src4, src2, src1, + tmp0, tmp1, tmp2, tmp4); + DUP2_ARG2(__lsx_vpackev_b, src4, src3, src6, src5, tmp5, tmp6); + + for (; loop_cnt--;) { + LSX_LD_4(src, src_stride, src7, src8, src9, src10); + src += src_stride; + DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7, + src8, src9, src10); + src7 = HORIZ_8TAP_FILT(src7, src7, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + tmp3 = __lsx_vpackev_b(src7, src6); + out0 = FILT_8TAP_DPADD_S_H(tmp0, tmp1, tmp2, tmp3, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + src8 = HORIZ_8TAP_FILT(src8, src8, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src0 = __lsx_vpackev_b(src8, src7); + out1 = FILT_8TAP_DPADD_S_H(tmp4, tmp5, tmp6, src0, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + src9 = HORIZ_8TAP_FILT(src9, src9, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src1 = __lsx_vpackev_b(src9, src8); + src3 = FILT_8TAP_DPADD_S_H(tmp1, tmp2, tmp3, src1, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + src10 = HORIZ_8TAP_FILT(src10, src10, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src2 = __lsx_vpackev_b(src10, src9); + src4 = FILT_8TAP_DPADD_S_H(tmp5, tmp6, src0, src2, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, src4, src3, 7, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 1); + dst += dst_stride; + + src6 = src10; + tmp0 = tmp2; + tmp1 = tmp3; + tmp2 = src1; + tmp4 = tmp6; + tmp5 = src0; + tmp6 = src2; + } +} + +static void common_hv_8ht_8vt_16w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, + int32_t height) { + common_hv_8ht_8vt_8w_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert, height); + src += 8; + dst += 8; + + common_hv_8ht_8vt_8w_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert, height); + src += 8; + dst += 8; +} + +static void common_hv_8ht_8vt_32w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, + int32_t height) { + int32_t multiple8_cnt; + for (multiple8_cnt = 4; multiple8_cnt--;) { + common_hv_8ht_8vt_8w_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert, height); + src += 8; + dst += 8; + } +} + +static void common_hv_8ht_8vt_64w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, + int32_t height) { + int32_t multiple8_cnt; + for (multiple8_cnt = 8; multiple8_cnt--;) { + common_hv_8ht_8vt_8w_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert, height); + src += 8; + dst += 8; + } +} + +static void common_hv_2ht_2vt_4x4_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, + int8_t *filter_vert) { + __m128i src0, src1, src2, src3, src4, mask; + __m128i filt_vt, filt_hz, vec0, vec1, res0, res1; + __m128i hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, tmp0, tmp1; + __m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 }; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride + src_stride2; + int32_t src_stride4 = src_stride2 << 1; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + mask = __lsx_vld(mc_filt_mask_arr, 16); + + /* rearranging filter */ + filt_hz = __lsx_vldrepl_h(filter_horiz, 0); + filt_vt = __lsx_vldrepl_h(filter_vert, 0); + + src0 = __lsx_vld(src, 0); + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src1, src2, src3, src4); + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src1, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src2, src3, mask, filt_hz, FILTER_BITS); + hz_out4 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + + hz_out1 = __lsx_vshuf_b(hz_out2, hz_out0, shuff); + hz_out3 = __lsx_vpickod_d(hz_out4, hz_out2); + + DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1); + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + DUP2_ARG2(__lsx_vpickev_b, tmp0, tmp0, tmp1, tmp1, res0, res1); + + __lsx_vstelm_w(res0, dst, 0, 0); + __lsx_vstelm_w(res0, dst + dst_stride, 0, 1); + __lsx_vstelm_w(res1, dst + dst_stride2, 0, 0); + __lsx_vstelm_w(res1, dst + dst_stride3, 0, 1); +} + +static void common_hv_2ht_2vt_4x8_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, + int8_t *filter_vert) { + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, mask; + __m128i res0, res1, res2, res3; + __m128i filt_hz, filt_vt, vec0, vec1, vec2, vec3; + __m128i hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6; + __m128i hz_out7, hz_out8, vec4, vec5, vec6, vec7; + __m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 }; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + int32_t dst_stride4 = dst_stride2 << 1; + + mask = __lsx_vld(mc_filt_mask_arr, 16); + + /* rearranging filter */ + DUP2_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_vert, 0, filt_hz, filt_vt); + + src0 = __lsx_vld(src, 0); + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src1, src2, src3, src4); + src += src_stride4; + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src5, src6, src7, src8); + src += src_stride4; + + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src1, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src2, src3, mask, filt_hz, FILTER_BITS); + hz_out4 = HORIZ_2TAP_FILT_UH(src4, src5, mask, filt_hz, FILTER_BITS); + hz_out6 = HORIZ_2TAP_FILT_UH(src6, src7, mask, filt_hz, FILTER_BITS); + hz_out8 = HORIZ_2TAP_FILT_UH(src8, src8, mask, filt_hz, FILTER_BITS); + + DUP2_ARG3(__lsx_vshuf_b, hz_out2, hz_out0, shuff, hz_out4, hz_out2, shuff, + hz_out1, hz_out3); + hz_out5 = __lsx_vshuf_b(hz_out6, hz_out4, shuff); + hz_out7 = __lsx_vpickod_d(hz_out8, hz_out6); + DUP4_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, hz_out5, + hz_out4, hz_out7, hz_out6, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, vec2, filt_vt, vec3, + filt_vt, vec4, vec5, vec6, vec7); + DUP4_ARG2(__lsx_vsrari_h, vec4, FILTER_BITS, vec5, FILTER_BITS, vec6, + FILTER_BITS, vec7, FILTER_BITS, vec4, vec5, vec6, vec7); + DUP4_ARG2(__lsx_vpickev_b, vec4, vec4, vec5, vec5, vec6, vec6, vec7, vec7, + res0, res1, res2, res3); + + __lsx_vstelm_w(res0, dst, 0, 0); + __lsx_vstelm_w(res0, dst + dst_stride, 0, 1); + __lsx_vstelm_w(res1, dst + dst_stride2, 0, 0); + __lsx_vstelm_w(res1, dst + dst_stride3, 0, 1); + dst += dst_stride4; + __lsx_vstelm_w(res2, dst, 0, 0); + __lsx_vstelm_w(res2, dst + dst_stride, 0, 1); + __lsx_vstelm_w(res3, dst + dst_stride2, 0, 0); + __lsx_vstelm_w(res3, dst + dst_stride3, 0, 1); +} + +static void common_hv_2ht_2vt_4w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, + int32_t height) { + if (height == 4) { + common_hv_2ht_2vt_4x4_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert); + } else if (height == 8) { + common_hv_2ht_2vt_4x8_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert); + } +} + +static void common_hv_2ht_2vt_8x4_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, + int8_t *filter_vert) { + __m128i src0, src1, src2, src3, src4, mask, out0, out1; + __m128i filt_hz, filt_vt, vec0, vec1, vec2, vec3; + __m128i hz_out0, hz_out1, tmp0, tmp1, tmp2, tmp3; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + + mask = __lsx_vld(mc_filt_mask_arr, 0); + + /* rearranging filter */ + DUP2_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_vert, 0, filt_hz, filt_vt); + + src0 = __lsx_vld(src, 0); + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src1, src2, src3, src4); + + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS); + hz_out1 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + vec0 = __lsx_vpackev_b(hz_out1, hz_out0); + tmp0 = __lsx_vdp2_h_bu(vec0, filt_vt); + + hz_out0 = HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS); + vec1 = __lsx_vpackev_b(hz_out0, hz_out1); + tmp1 = __lsx_vdp2_h_bu(vec1, filt_vt); + + hz_out1 = HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS); + vec2 = __lsx_vpackev_b(hz_out1, hz_out0); + tmp2 = __lsx_vdp2_h_bu(vec2, filt_vt); + + hz_out0 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + vec3 = __lsx_vpackev_b(hz_out0, hz_out1); + tmp3 = __lsx_vdp2_h_bu(vec3, filt_vt); + + DUP4_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp2, + FILTER_BITS, tmp3, FILTER_BITS, tmp0, tmp1, tmp2, tmp3); + DUP2_ARG2(__lsx_vpickev_b, tmp1, tmp0, tmp3, tmp2, out0, out1); + + __lsx_vstelm_d(out0, dst, 0, 0); + __lsx_vstelm_d(out0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(out1, dst + dst_stride2, 0, 0); + __lsx_vstelm_d(out1, dst + dst_stride3, 0, 1); +} + +static void common_hv_2ht_2vt_8x8mult_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter_horiz, + int8_t *filter_vert, int32_t height) { + uint32_t loop_cnt = (height >> 3); + __m128i src0, src1, src2, src3, src4, mask, out0, out1; + __m128i filt_hz, filt_vt, vec0; + __m128i hz_out0, hz_out1, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp8; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + mask = __lsx_vld(mc_filt_mask_arr, 0); + + /* rearranging filter */ + DUP2_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_vert, 0, filt_hz, filt_vt); + + src0 = __lsx_vld(src, 0); + src += src_stride; + + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS); + + for (; loop_cnt--;) { + src1 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src3); + src4 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + hz_out1 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + vec0 = __lsx_vpackev_b(hz_out1, hz_out0); + tmp1 = __lsx_vdp2_h_bu(vec0, filt_vt); + + hz_out0 = HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS); + vec0 = __lsx_vpackev_b(hz_out0, hz_out1); + tmp2 = __lsx_vdp2_h_bu(vec0, filt_vt); + + DUP2_ARG2(__lsx_vsrari_h, tmp1, FILTER_BITS, tmp2, FILTER_BITS, tmp1, tmp2); + + hz_out1 = HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS); + vec0 = __lsx_vpackev_b(hz_out1, hz_out0); + tmp3 = __lsx_vdp2_h_bu(vec0, filt_vt); + + hz_out0 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + src1 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src3); + src4 = __lsx_vldx(src, src_stride3); + src += src_stride4; + vec0 = __lsx_vpackev_b(hz_out0, hz_out1); + tmp4 = __lsx_vdp2_h_bu(vec0, filt_vt); + + DUP2_ARG2(__lsx_vsrari_h, tmp3, FILTER_BITS, tmp4, FILTER_BITS, tmp3, tmp4); + DUP2_ARG2(__lsx_vpickev_b, tmp2, tmp1, tmp4, tmp3, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 1); + dst += dst_stride; + + hz_out1 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + vec0 = __lsx_vpackev_b(hz_out1, hz_out0); + tmp5 = __lsx_vdp2_h_bu(vec0, filt_vt); + + hz_out0 = HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS); + vec0 = __lsx_vpackev_b(hz_out0, hz_out1); + tmp6 = __lsx_vdp2_h_bu(vec0, filt_vt); + + hz_out1 = HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS); + vec0 = __lsx_vpackev_b(hz_out1, hz_out0); + tmp7 = __lsx_vdp2_h_bu(vec0, filt_vt); + + hz_out0 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + vec0 = __lsx_vpackev_b(hz_out0, hz_out1); + tmp8 = __lsx_vdp2_h_bu(vec0, filt_vt); + + DUP4_ARG2(__lsx_vsrari_h, tmp5, FILTER_BITS, tmp6, FILTER_BITS, tmp7, + FILTER_BITS, tmp8, FILTER_BITS, tmp5, tmp6, tmp7, tmp8); + DUP2_ARG2(__lsx_vpickev_b, tmp6, tmp5, tmp8, tmp7, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 1); + dst += dst_stride; + } +} + +static void common_hv_2ht_2vt_8w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, + int32_t height) { + if (height == 4) { + common_hv_2ht_2vt_8x4_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert); + } else { + common_hv_2ht_2vt_8x8mult_lsx(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, height); + } +} + +static void common_hv_2ht_2vt_16w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, + int32_t height) { + uint32_t loop_cnt = (height >> 2); + __m128i src0, src1, src2, src3, src4, src5, src6, src7, mask; + __m128i filt_hz, filt_vt, vec0, vec1; + __m128i tmp, tmp1, tmp2, hz_out0, hz_out1, hz_out2, hz_out3; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + mask = __lsx_vld(mc_filt_mask_arr, 0); + + /* rearranging filter */ + DUP2_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_vert, 0, filt_hz, filt_vt); + + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src0, src1); + src += src_stride; + + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + + for (; loop_cnt--;) { + uint8_t *src_tmp0 = src + 8; + + DUP2_ARG2(__lsx_vld, src, 0, src_tmp0, 0, src0, src1); + DUP4_ARG2(__lsx_vldx, src, src_stride, src_tmp0, src_stride, src, + src_stride2, src_tmp0, src_stride2, src2, src3, src4, src5); + DUP2_ARG2(__lsx_vldx, src, src_stride3, src_tmp0, src_stride3, src6, src7); + src += src_stride4; + + hz_out1 = HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS); + hz_out3 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp1, tmp2); + DUP2_ARG2(__lsx_vsrari_h, tmp1, FILTER_BITS, tmp2, FILTER_BITS, tmp1, tmp2); + tmp = __lsx_vpickev_b(tmp2, tmp1); + __lsx_vst(tmp, dst, 0); + dst += dst_stride; + + hz_out0 = HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS); + DUP2_ARG2(__lsx_vpackev_b, hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp1, tmp2); + DUP2_ARG2(__lsx_vsrari_h, tmp1, FILTER_BITS, tmp2, FILTER_BITS, tmp1, tmp2); + tmp = __lsx_vpickev_b(tmp2, tmp1); + __lsx_vst(tmp, dst, 0); + dst += dst_stride; + + hz_out1 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + hz_out3 = HORIZ_2TAP_FILT_UH(src5, src5, mask, filt_hz, FILTER_BITS); + DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp1, tmp2); + DUP2_ARG2(__lsx_vsrari_h, tmp1, FILTER_BITS, tmp2, FILTER_BITS, tmp1, tmp2); + tmp = __lsx_vpickev_b(tmp2, tmp1); + __lsx_vst(tmp, dst, 0); + dst += dst_stride; + + hz_out0 = HORIZ_2TAP_FILT_UH(src6, src6, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src7, src7, mask, filt_hz, FILTER_BITS); + DUP2_ARG2(__lsx_vpackev_b, hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp1, tmp2); + DUP2_ARG2(__lsx_vsrari_h, tmp1, FILTER_BITS, tmp2, FILTER_BITS, tmp1, tmp2); + tmp = __lsx_vpickev_b(tmp2, tmp1); + __lsx_vst(tmp, dst, 0); + dst += dst_stride; + } +} + +static void common_hv_2ht_2vt_32w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, + int32_t height) { + common_hv_2ht_2vt_16w_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert, height); + src += 16; + dst += 16; + + common_hv_2ht_2vt_16w_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert, height); + src += 16; + dst += 16; +} + +static void common_hv_2ht_2vt_64w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, + int32_t height) { + int32_t multiple8_cnt; + for (multiple8_cnt = 4; multiple8_cnt--;) { + common_hv_2ht_2vt_16w_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert, height); + src += 16; + dst += 16; + } +} + +void vpx_convolve8_lsx(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, + ptrdiff_t dst_stride, const InterpKernel *filter, + int x0_q4, int32_t x_step_q4, int y0_q4, + int32_t y_step_q4, int32_t w, int32_t h) { + const int16_t *const filter_x = filter[x0_q4]; + const int16_t *const filter_y = filter[y0_q4]; + int8_t cnt, filt_hor[8], filt_ver[8]; + + assert(x_step_q4 == 16); + assert(y_step_q4 == 16); + assert(((const int32_t *)filter_x)[1] != 0x800000); + assert(((const int32_t *)filter_y)[1] != 0x800000); + + for (cnt = 0; cnt < 8; ++cnt) { + filt_hor[cnt] = filter_x[cnt]; + filt_ver[cnt] = filter_y[cnt]; + } + + if (vpx_get_filter_taps(filter_x) == 2 && + vpx_get_filter_taps(filter_y) == 2) { + switch (w) { + case 4: + common_hv_2ht_2vt_4w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], + &filt_ver[3], (int32_t)h); + break; + case 8: + common_hv_2ht_2vt_8w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], + &filt_ver[3], (int32_t)h); + break; + case 16: + common_hv_2ht_2vt_16w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], + &filt_ver[3], (int32_t)h); + break; + case 32: + common_hv_2ht_2vt_32w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], + &filt_ver[3], (int32_t)h); + break; + case 64: + common_hv_2ht_2vt_64w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], + &filt_ver[3], (int32_t)h); + break; + default: + vpx_convolve8_c(src, src_stride, dst, dst_stride, filter, x0_q4, + x_step_q4, y0_q4, y_step_q4, w, h); + break; + } + } else if (vpx_get_filter_taps(filter_x) == 2 || + vpx_get_filter_taps(filter_y) == 2) { + vpx_convolve8_c(src, src_stride, dst, dst_stride, filter, x0_q4, x_step_q4, + y0_q4, y_step_q4, w, h); + } else { + switch (w) { + case 4: + common_hv_8ht_8vt_4w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, filt_ver, + (int32_t)h); + break; + case 8: + common_hv_8ht_8vt_8w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, filt_ver, + (int32_t)h); + break; + case 16: + common_hv_8ht_8vt_16w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, filt_ver, + (int32_t)h); + break; + case 32: + common_hv_8ht_8vt_32w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, filt_ver, + (int32_t)h); + break; + case 64: + common_hv_8ht_8vt_64w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, filt_ver, + (int32_t)h); + break; + default: + vpx_convolve8_c(src, src_stride, dst, dst_stride, filter, x0_q4, + x_step_q4, y0_q4, y_step_q4, w, h); + break; + } + } +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_vert_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_vert_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_vert_lsx.c @@ -0,0 +1,870 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include "./vpx_dsp_rtcd.h" +#include "vpx_dsp/loongarch/vpx_convolve_lsx.h" + +static void common_vt_8t_4w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = height >> 2; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride + src_stride2; + int32_t src_stride4 = src_stride2 << 1; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + __m128i reg0, reg1, reg2, reg3, reg4; + __m128i filter0, filter1, filter2, filter3; + __m128i out0, out1; + uint8_t *_src = (uint8_t *)src - src_stride3; + + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + src0 = __lsx_vld(_src, 0); + DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src1, src2); + src3 = __lsx_vldx(_src, src_stride3); + _src += src_stride4; + src4 = __lsx_vld(_src, 0); + DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src5, src6); + _src += src_stride3; + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, tmp0, + tmp1, tmp2, tmp3); + DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, tmp4, tmp5); + DUP2_ARG2(__lsx_vilvl_d, tmp3, tmp0, tmp4, tmp1, reg0, reg1); + reg2 = __lsx_vilvl_d(tmp5, tmp2); + DUP2_ARG2(__lsx_vxori_b, reg0, 128, reg1, 128, reg0, reg1); + reg2 = __lsx_vxori_b(reg2, 128); + + for (; loop_cnt--;) { + src7 = __lsx_vld(_src, 0); + DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src8, src9); + src10 = __lsx_vldx(_src, src_stride3); + _src += src_stride4; + DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9, + tmp0, tmp1, tmp2, tmp3); + DUP2_ARG2(__lsx_vilvl_d, tmp1, tmp0, tmp3, tmp2, reg3, reg4); + DUP2_ARG2(__lsx_vxori_b, reg3, 128, reg4, 128, reg3, reg4); + out0 = FILT_8TAP_DPADD_S_H(reg0, reg1, reg2, reg3, filter0, filter1, + filter2, filter3); + out1 = FILT_8TAP_DPADD_S_H(reg1, reg2, reg3, reg4, filter0, filter1, + filter2, filter3); + out0 = __lsx_vssrarni_b_h(out1, out0, 7); + out0 = __lsx_vxori_b(out0, 128); + __lsx_vstelm_w(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 3); + dst += dst_stride; + + reg0 = reg2; + reg1 = reg3; + reg2 = reg4; + src6 = src10; + } +} + +static void common_vt_8t_8w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = height >> 2; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i tmp0, tmp1, tmp2, tmp3; + __m128i reg0, reg1, reg2, reg3, reg4, reg5; + __m128i filter0, filter1, filter2, filter3; + __m128i out0, out1, out2, out3; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride + src_stride2; + int32_t src_stride4 = src_stride2 << 1; + src = src - src_stride3; + + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + src += src_stride4; + src4 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src5, src6); + src += src_stride3; + + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, reg0, + reg1, reg2, reg3); + DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, reg4, reg5); + + for (; loop_cnt--;) { + src7 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src8, src9); + src10 = __lsx_vldx(src, src_stride3); + src += src_stride4; + DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7, + src8, src9, src10); + DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9, + tmp0, tmp1, tmp2, tmp3); + out0 = FILT_8TAP_DPADD_S_H(reg0, reg1, reg2, tmp0, filter0, filter1, + filter2, filter3); + out1 = FILT_8TAP_DPADD_S_H(reg3, reg4, reg5, tmp1, filter0, filter1, + filter2, filter3); + out2 = FILT_8TAP_DPADD_S_H(reg1, reg2, tmp0, tmp2, filter0, filter1, + filter2, filter3); + out3 = FILT_8TAP_DPADD_S_H(reg4, reg5, tmp1, tmp3, filter0, filter1, + filter2, filter3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 1); + dst += dst_stride; + + reg0 = reg2; + reg1 = tmp0; + reg2 = tmp2; + reg3 = reg5; + reg4 = tmp1; + reg5 = tmp3; + src6 = src10; + } +} + +static void common_vt_8t_16w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = height >> 2; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i filter0, filter1, filter2, filter3; + __m128i reg0, reg1, reg2, reg3, reg4, reg5; + __m128i reg6, reg7, reg8, reg9, reg10, reg11; + __m128i tmp0, tmp1, tmp2, tmp3; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride + src_stride2; + int32_t src_stride4 = src_stride2 << 1; + // uint8_t *_src = (uint8_t *)src - src_stride3; + src -= src_stride3; + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + src0 = __lsx_vld(src, 0); + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src1, src2, src3, src4); + src += src_stride4; + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src5, src6); + src += src_stride3; + + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, reg0, + reg1, reg2, reg3); + DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, reg4, reg5); + DUP4_ARG2(__lsx_vilvh_b, src1, src0, src3, src2, src5, src4, src2, src1, reg6, + reg7, reg8, reg9); + DUP2_ARG2(__lsx_vilvh_b, src4, src3, src6, src5, reg10, reg11); + + for (; loop_cnt--;) { + src7 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src8, src9); + src10 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7, + src8, src9, src10); + DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9, + src0, src1, src2, src3); + DUP4_ARG2(__lsx_vilvh_b, src7, src6, src8, src7, src9, src8, src10, src9, + src4, src5, src7, src8); + tmp0 = FILT_8TAP_DPADD_S_H(reg0, reg1, reg2, src0, filter0, filter1, + filter2, filter3); + tmp1 = FILT_8TAP_DPADD_S_H(reg3, reg4, reg5, src1, filter0, filter1, + filter2, filter3); + tmp2 = FILT_8TAP_DPADD_S_H(reg6, reg7, reg8, src4, filter0, filter1, + filter2, filter3); + tmp3 = FILT_8TAP_DPADD_S_H(reg9, reg10, reg11, src5, filter0, filter1, + filter2, filter3); + DUP2_ARG3(__lsx_vssrarni_b_h, tmp2, tmp0, 7, tmp3, tmp1, 7, tmp0, tmp1); + DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + __lsx_vst(tmp0, dst, 0); + dst += dst_stride; + __lsx_vst(tmp1, dst, 0); + dst += dst_stride; + tmp0 = FILT_8TAP_DPADD_S_H(reg1, reg2, src0, src2, filter0, filter1, + filter2, filter3); + tmp1 = FILT_8TAP_DPADD_S_H(reg4, reg5, src1, src3, filter0, filter1, + filter2, filter3); + tmp2 = FILT_8TAP_DPADD_S_H(reg7, reg8, src4, src7, filter0, filter1, + filter2, filter3); + tmp3 = FILT_8TAP_DPADD_S_H(reg10, reg11, src5, src8, filter0, filter1, + filter2, filter3); + DUP2_ARG3(__lsx_vssrarni_b_h, tmp2, tmp0, 7, tmp3, tmp1, 7, tmp0, tmp1); + DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + __lsx_vst(tmp0, dst, 0); + dst += dst_stride; + __lsx_vst(tmp1, dst, 0); + dst += dst_stride; + + reg0 = reg2; + reg1 = src0; + reg2 = src2; + reg3 = reg5; + reg4 = src1; + reg5 = src3; + reg6 = reg8; + reg7 = src4; + reg8 = src7; + reg9 = reg11; + reg10 = src5; + reg11 = src8; + src6 = src10; + } +} + +static void common_vt_8t_16w_mult_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height, + int32_t width) { + uint8_t *src_tmp; + uint8_t *dst_tmp; + uint32_t cnt = width >> 4; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i filter0, filter1, filter2, filter3; + __m128i reg0, reg1, reg2, reg3, reg4, reg5; + __m128i reg6, reg7, reg8, reg9, reg10, reg11; + __m128i tmp0, tmp1, tmp2, tmp3; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride + src_stride2; + int32_t src_stride4 = src_stride2 << 1; + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + int32_t dst_stride4 = dst_stride2 << 1; + src -= src_stride3; + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + for (; cnt--;) { + uint32_t loop_cnt = height >> 2; + + src_tmp = src; + dst_tmp = dst; + + src0 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src1, + src2); + src3 = __lsx_vldx(src_tmp, src_stride3); + src_tmp += src_stride4; + src4 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src5, + src6); + src_tmp += src_stride3; + + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, + reg0, reg1, reg2, reg3); + DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, reg4, reg5); + DUP4_ARG2(__lsx_vilvh_b, src1, src0, src3, src2, src5, src4, src2, src1, + reg6, reg7, reg8, reg9); + DUP2_ARG2(__lsx_vilvh_b, src4, src3, src6, src5, reg10, reg11); + + for (; loop_cnt--;) { + src7 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src8, + src9); + src10 = __lsx_vldx(src_tmp, src_stride3); + src_tmp += src_stride4; + DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, + src7, src8, src9, src10); + DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9, + src0, src1, src2, src3); + DUP4_ARG2(__lsx_vilvh_b, src7, src6, src8, src7, src9, src8, src10, src9, + src4, src5, src7, src8); + tmp0 = FILT_8TAP_DPADD_S_H(reg0, reg1, reg2, src0, filter0, filter1, + filter2, filter3); + tmp1 = FILT_8TAP_DPADD_S_H(reg3, reg4, reg5, src1, filter0, filter1, + filter2, filter3); + tmp2 = FILT_8TAP_DPADD_S_H(reg6, reg7, reg8, src4, filter0, filter1, + filter2, filter3); + tmp3 = FILT_8TAP_DPADD_S_H(reg9, reg10, reg11, src5, filter0, filter1, + filter2, filter3); + DUP2_ARG3(__lsx_vssrarni_b_h, tmp2, tmp0, 7, tmp3, tmp1, 7, tmp0, tmp1); + DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + __lsx_vst(tmp0, dst_tmp, 0); + __lsx_vstx(tmp1, dst_tmp, dst_stride); + tmp0 = FILT_8TAP_DPADD_S_H(reg1, reg2, src0, src2, filter0, filter1, + filter2, filter3); + tmp1 = FILT_8TAP_DPADD_S_H(reg4, reg5, src1, src3, filter0, filter1, + filter2, filter3); + tmp2 = FILT_8TAP_DPADD_S_H(reg7, reg8, src4, src7, filter0, filter1, + filter2, filter3); + tmp3 = FILT_8TAP_DPADD_S_H(reg10, reg11, src5, src8, filter0, filter1, + filter2, filter3); + DUP2_ARG3(__lsx_vssrarni_b_h, tmp2, tmp0, 7, tmp3, tmp1, 7, tmp0, tmp1); + DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + __lsx_vstx(tmp0, dst_tmp, dst_stride2); + __lsx_vstx(tmp1, dst_tmp, dst_stride3); + dst_tmp += dst_stride4; + + reg0 = reg2; + reg1 = src0; + reg2 = src2; + reg3 = reg5; + reg4 = src1; + reg5 = src3; + reg6 = reg8; + reg7 = src4; + reg8 = src7; + reg9 = reg11; + reg10 = src5; + reg11 = src8; + src6 = src10; + } + src += 16; + dst += 16; + } +} + +static void common_vt_8t_32w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + common_vt_8t_16w_mult_lsx(src, src_stride, dst, dst_stride, filter, height, + 32); +} + +static void common_vt_8t_64w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + common_vt_8t_16w_mult_lsx(src, src_stride, dst, dst_stride, filter, height, + 64); +} + +static void common_vt_2t_4x4_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter) { + __m128i src0, src1, src2, src3, src4; + __m128i src10_l, src32_l, src21_l, src43_l, src2110, src4332; + __m128i filt0, tmp0, tmp1; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + + filt0 = __lsx_vldrepl_h(filter, 0); + + src0 = __lsx_vld(src, 0); + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src1, src2, src3, src4); + src += (src_stride4 + src_stride); + + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src3, src2, src4, src3, + src10_l, src21_l, src32_l, src43_l); + DUP2_ARG2(__lsx_vilvl_d, src21_l, src10_l, src43_l, src32_l, src2110, + src4332); + DUP2_ARG2(__lsx_vdp2_h_bu, src2110, filt0, src4332, filt0, tmp0, tmp1); + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + src2110 = __lsx_vpickev_b(tmp1, tmp0); + + __lsx_vstelm_w(src2110, dst, 0, 0); + __lsx_vstelm_w(src2110, dst + dst_stride, 0, 1); + __lsx_vstelm_w(src2110, dst + dst_stride2, 0, 2); + __lsx_vstelm_w(src2110, dst + dst_stride3, 0, 3); +} + +static void common_vt_2t_4x8_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter) { + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8; + __m128i src10_l, src32_l, src54_l, src76_l, src21_l, src43_l; + __m128i src65_l, src87_l, src2110, src4332, src6554, src8776; + __m128i tmp0, tmp1, tmp2, tmp3; + __m128i filt0; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + int32_t dst_stride4 = dst_stride2 << 1; + + uint8_t *dst_tmp1 = dst + dst_stride4; + + filt0 = __lsx_vldrepl_h(filter, 0); + src0 = __lsx_vld(src, 0); + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src1, src2, src3, src4); + src += src_stride4; + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src5, src6, src7, src8); + src += (src_stride4 + src_stride); + + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src3, src2, src4, src3, + src10_l, src21_l, src32_l, src43_l); + DUP4_ARG2(__lsx_vilvl_b, src5, src4, src6, src5, src7, src6, src8, src7, + src54_l, src65_l, src76_l, src87_l); + DUP4_ARG2(__lsx_vilvl_d, src21_l, src10_l, src43_l, src32_l, src65_l, src54_l, + src87_l, src76_l, src2110, src4332, src6554, src8776); + DUP4_ARG2(__lsx_vdp2_h_bu, src2110, filt0, src4332, filt0, src6554, filt0, + src8776, filt0, tmp0, tmp1, tmp2, tmp3); + DUP4_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp2, + FILTER_BITS, tmp3, FILTER_BITS, tmp0, tmp1, tmp2, tmp3); + DUP2_ARG2(__lsx_vpickev_b, tmp1, tmp0, tmp3, tmp2, src2110, src4332); + + __lsx_vstelm_w(src2110, dst, 0, 0); + __lsx_vstelm_w(src2110, dst + dst_stride, 0, 1); + __lsx_vstelm_w(src2110, dst + dst_stride2, 0, 2); + __lsx_vstelm_w(src2110, dst + dst_stride3, 0, 3); + + __lsx_vstelm_w(src4332, dst_tmp1, 0, 0); + __lsx_vstelm_w(src4332, dst_tmp1 + dst_stride, 0, 1); + __lsx_vstelm_w(src4332, dst_tmp1 + dst_stride2, 0, 2); + __lsx_vstelm_w(src4332, dst_tmp1 + dst_stride3, 0, 3); +} + +static void common_vt_2t_4w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + if (height == 4) { + common_vt_2t_4x4_lsx(src, src_stride, dst, dst_stride, filter); + } else if (height == 8) { + common_vt_2t_4x8_lsx(src, src_stride, dst, dst_stride, filter); + } +} + +static void common_vt_2t_8x4_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter) { + __m128i src0, src1, src2, src3, src4, vec0, vec1, vec2, vec3, filt0; + __m128i out0, out1; + __m128i tmp0, tmp1, tmp2, tmp3; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + + /* rearranging filter_y */ + filt0 = __lsx_vldrepl_h(filter, 0); + + src0 = __lsx_vld(src, 0); + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src1, src2, src3, src4); + + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src3, src2, src4, src3, vec0, + vec1, vec2, vec3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0, + tmp0, tmp1, tmp2, tmp3); + DUP4_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp2, + FILTER_BITS, tmp3, FILTER_BITS, tmp0, tmp1, tmp2, tmp3); + DUP2_ARG2(__lsx_vpickev_b, tmp1, tmp0, tmp3, tmp2, out0, out1); + + __lsx_vstelm_d(out0, dst, 0, 0); + __lsx_vstelm_d(out0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(out1, dst + dst_stride2, 0, 0); + __lsx_vstelm_d(out1, dst + dst_stride3, 0, 1); +} + +static void common_vt_2t_8x8mult_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = (height >> 3); + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, filt0; + __m128i out0, out1; + __m128i tmp0, tmp1, tmp2, tmp3; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + int32_t dst_stride4 = dst_stride2 << 1; + + filt0 = __lsx_vldrepl_h(filter, 0); + src0 = __lsx_vld(src, 0); + src += src_stride; + + for (; loop_cnt--;) { + src1 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src3); + src4 = __lsx_vldx(src, src_stride3); + src += src_stride4; + src5 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src6, src7) + src8 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src3, src2, src4, src3, + vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vilvl_b, src5, src4, src6, src5, src7, src6, src8, src7, + vec4, vec5, vec6, vec7); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, + filt0, tmp0, tmp1, tmp2, tmp3); + DUP4_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp2, + FILTER_BITS, tmp3, FILTER_BITS, tmp0, tmp1, tmp2, tmp3); + DUP2_ARG2(__lsx_vpickev_b, tmp1, tmp0, tmp3, tmp2, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + __lsx_vstelm_d(out0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(out1, dst + dst_stride2, 0, 0); + __lsx_vstelm_d(out1, dst + dst_stride3, 0, 1); + dst += dst_stride4; + + DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7, + filt0, tmp0, tmp1, tmp2, tmp3); + DUP4_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp2, + FILTER_BITS, tmp3, FILTER_BITS, tmp0, tmp1, tmp2, tmp3); + DUP2_ARG2(__lsx_vpickev_b, tmp1, tmp0, tmp3, tmp2, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + __lsx_vstelm_d(out0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(out1, dst + dst_stride2, 0, 0); + __lsx_vstelm_d(out1, dst + dst_stride3, 0, 1); + dst += dst_stride4; + + src0 = src8; + } +} + +static void common_vt_2t_8w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + if (height == 4) { + common_vt_2t_8x4_lsx(src, src_stride, dst, dst_stride, filter); + } else { + common_vt_2t_8x8mult_lsx(src, src_stride, dst, dst_stride, filter, height); + } +} + +static void common_vt_2t_16w_lsx(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *filter, int y0_q4, + int y_step_q4, int w, int height) { + uint32_t loop_cnt = (height >> 2); + __m128i src0, src1, src2, src3, src4; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, filt0; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + const int16_t *const filter_y = filter[y0_q4]; + int8_t cnt, filt_ver[8]; + + assert(y_step_q4 == 16); + assert(((const int32_t *)filter_y)[1] != 0x800000); + + for (cnt = 8; cnt--;) { + filt_ver[cnt] = filter_y[cnt]; + } + + filt0 = __lsx_vldrepl_h(&filt_ver[3], 0); + + src0 = __lsx_vld(src, 0); + src += src_stride; + + for (; loop_cnt--;) { + src1 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src3); + src4 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, vec0, vec2); + DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, vec1, vec3); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1); + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + tmp4 = __lsx_vpickev_b(tmp1, tmp0); + __lsx_vst(tmp4, dst, 0); + dst += dst_stride; + + DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, vec4, vec6); + DUP2_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, vec5, vec7); + DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp2, tmp3); + DUP2_ARG2(__lsx_vsrari_h, tmp2, FILTER_BITS, tmp3, FILTER_BITS, tmp2, tmp3); + tmp4 = __lsx_vpickev_b(tmp3, tmp2); + __lsx_vst(tmp4, dst, 0); + dst += dst_stride; + + DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1); + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + tmp4 = __lsx_vpickev_b(tmp1, tmp0); + __lsx_vst(tmp4, dst, 0); + dst += dst_stride; + + DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp2, tmp3); + DUP2_ARG2(__lsx_vsrari_h, tmp2, FILTER_BITS, tmp3, FILTER_BITS, tmp2, tmp3); + tmp4 = __lsx_vpickev_b(tmp3, tmp2); + __lsx_vst(tmp4, dst, 0); + dst += dst_stride; + + src0 = src4; + } +} + +static void common_vt_2t_32w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = (height >> 2); + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, filt0; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + + uint8_t *src_tmp; + filt0 = __lsx_vldrepl_h(filter, 0); + + src0 = __lsx_vld(src, 0); + src5 = __lsx_vld(src, 16); + src += src_stride; + src_tmp = src + 16; + + for (; loop_cnt--;) { + DUP2_ARG2(__lsx_vld, src, 0, src_tmp, 0, src1, src6); + DUP4_ARG2(__lsx_vldx, src, src_stride, src_tmp, src_stride, src, + src_stride2, src_tmp, src_stride2, src2, src7, src3, src8); + DUP2_ARG2(__lsx_vldx, src, src_stride3, src_tmp, src_stride3, src4, src9); + DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, vec0, vec2); + DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, vec1, vec3); + src += src_stride4; + src_tmp += src_stride4; + + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1); + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + tmp4 = __lsx_vpickev_b(tmp1, tmp0); + __lsx_vst(tmp4, dst, 0); + + DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp2, tmp3); + DUP2_ARG2(__lsx_vsrari_h, tmp2, FILTER_BITS, tmp3, FILTER_BITS, tmp2, tmp3); + tmp4 = __lsx_vpickev_b(tmp3, tmp2); + __lsx_vstx(tmp4, dst, dst_stride); + + DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, vec4, vec6); + DUP2_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, vec5, vec7); + DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1); + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + tmp4 = __lsx_vpickev_b(tmp1, tmp0); + __lsx_vstx(tmp4, dst, dst_stride2); + + DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp2, tmp3); + DUP2_ARG2(__lsx_vsrari_h, tmp2, FILTER_BITS, tmp3, FILTER_BITS, tmp2, tmp3); + tmp4 = __lsx_vpickev_b(tmp3, tmp2); + __lsx_vstx(tmp4, dst, dst_stride3); + + DUP2_ARG2(__lsx_vilvl_b, src6, src5, src7, src6, vec0, vec2); + DUP2_ARG2(__lsx_vilvh_b, src6, src5, src7, src6, vec1, vec3); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1); + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + tmp4 = __lsx_vpickev_b(tmp1, tmp0); + __lsx_vst(tmp4, dst, 16); + + DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp2, tmp3); + DUP2_ARG2(__lsx_vsrari_h, tmp2, FILTER_BITS, tmp3, FILTER_BITS, tmp2, tmp3); + dst += dst_stride; + tmp4 = __lsx_vpickev_b(tmp3, tmp2); + __lsx_vst(tmp4, dst, 16); + + DUP2_ARG2(__lsx_vilvl_b, src8, src7, src9, src8, vec4, vec6); + DUP2_ARG2(__lsx_vilvh_b, src8, src7, src9, src8, vec5, vec7); + DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1); + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + dst += dst_stride; + tmp4 = __lsx_vpickev_b(tmp1, tmp0); + __lsx_vst(tmp4, dst, 16); + + DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp2, tmp3); + DUP2_ARG2(__lsx_vsrari_h, tmp2, FILTER_BITS, tmp3, FILTER_BITS, tmp2, tmp3); + dst += dst_stride; + tmp4 = __lsx_vpickev_b(tmp3, tmp2); + __lsx_vst(tmp4, dst, 16); + + dst += dst_stride; + + src0 = src4; + src5 = src9; + } +} + +static void common_vt_2t_64w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = (height >> 1); + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i src11, vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, filt0; + __m128i tmp, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + + int32_t src_stride2 = src_stride << 1; + int32_t dst_stride2 = dst_stride << 1; + uint8_t *dst_tmp1 = dst + dst_stride; + + filt0 = __lsx_vldrepl_h(filter, 0); + + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src3, src6, + src9); + src += src_stride; + + for (; loop_cnt--;) { + uint8_t *src_tmp0 = src + src_stride; + + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src1, src4, src7, + src10); + DUP4_ARG2(__lsx_vld, src_tmp0, 0, src_tmp0, 16, src_tmp0, 32, src_tmp0, 48, + src2, src5, src8, src11); + src += src_stride2; + + DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, vec0, vec2); + DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, vec1, vec3); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1); + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + tmp = __lsx_vpickev_b(tmp1, tmp0); + __lsx_vst(tmp, dst, 0); + + DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp2, tmp3); + DUP2_ARG2(__lsx_vsrari_h, tmp2, FILTER_BITS, tmp3, FILTER_BITS, tmp2, tmp3); + tmp = __lsx_vpickev_b(tmp3, tmp2); + __lsx_vst(tmp, dst_tmp1, 0); + + DUP2_ARG2(__lsx_vilvl_b, src4, src3, src5, src4, vec4, vec6); + DUP2_ARG2(__lsx_vilvh_b, src4, src3, src5, src4, vec5, vec7); + DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp4, tmp5); + DUP2_ARG2(__lsx_vsrari_h, tmp4, FILTER_BITS, tmp5, FILTER_BITS, tmp4, tmp5); + tmp = __lsx_vpickev_b(tmp5, tmp4); + __lsx_vst(tmp, dst, 16); + + DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp6, tmp7); + DUP2_ARG2(__lsx_vsrari_h, tmp6, FILTER_BITS, tmp7, FILTER_BITS, tmp6, tmp7); + tmp = __lsx_vpickev_b(tmp7, tmp6); + __lsx_vst(tmp, dst_tmp1, 16); + + DUP2_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, vec0, vec2); + DUP2_ARG2(__lsx_vilvh_b, src7, src6, src8, src7, vec1, vec3); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1); + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + tmp = __lsx_vpickev_b(tmp1, tmp0); + __lsx_vst(tmp, dst, 32); + + DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp2, tmp3); + DUP2_ARG2(__lsx_vsrari_h, tmp2, FILTER_BITS, tmp3, FILTER_BITS, tmp2, tmp3); + tmp = __lsx_vpickev_b(tmp3, tmp2); + __lsx_vst(tmp, dst_tmp1, 32); + + DUP2_ARG2(__lsx_vilvl_b, src10, src9, src11, src10, vec4, vec6); + DUP2_ARG2(__lsx_vilvh_b, src10, src9, src11, src10, vec5, vec7); + DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp4, tmp5); + DUP2_ARG2(__lsx_vsrari_h, tmp4, FILTER_BITS, tmp5, FILTER_BITS, tmp4, tmp5); + tmp = __lsx_vpickev_b(tmp5, tmp4); + __lsx_vst(tmp, dst, 48); + + DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp6, tmp7); + DUP2_ARG2(__lsx_vsrari_h, tmp6, FILTER_BITS, tmp7, FILTER_BITS, tmp6, tmp7); + tmp = __lsx_vpickev_b(tmp7, tmp6); + __lsx_vst(tmp, dst_tmp1, 48); + dst += dst_stride2; + dst_tmp1 += dst_stride2; + + src0 = src2; + src3 = src5; + src6 = src8; + src9 = src11; + } +} + +void vpx_convolve8_vert_lsx(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *filter, int x0_q4, + int32_t x_step_q4, int y0_q4, int y_step_q4, int w, + int h) { + const int16_t *const filter_y = filter[y0_q4]; + int8_t cnt, filt_ver[8]; + + assert(y_step_q4 == 16); + assert(((const int32_t *)filter_y)[1] != 0x800000); + + for (cnt = 8; cnt--;) { + filt_ver[cnt] = filter_y[cnt]; + } + + if (vpx_get_filter_taps(filter_y) == 2) { + switch (w) { + case 4: + common_vt_2t_4w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + &filt_ver[3], h); + break; + case 8: + common_vt_2t_8w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + &filt_ver[3], h); + break; + case 16: + common_vt_2t_16w_lsx(src, src_stride, dst, dst_stride, filter, y0_q4, + y_step_q4, w, h); + break; + case 32: + common_vt_2t_32w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + &filt_ver[3], h); + break; + case 64: + common_vt_2t_64w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + &filt_ver[3], h); + break; + default: + vpx_convolve8_vert_c(src, src_stride, dst, dst_stride, filter, x0_q4, + x_step_q4, y0_q4, y_step_q4, w, h); + break; + } + } else { + switch (w) { + case 4: + common_vt_8t_4w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + filt_ver, h); + break; + case 8: + common_vt_8t_8w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + filt_ver, h); + break; + case 16: + common_vt_8t_16w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + filt_ver, h); + break; + case 32: + common_vt_8t_32w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + filt_ver, h); + break; + case 64: + common_vt_8t_64w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + filt_ver, h); + break; + default: + vpx_convolve8_vert_c(src, src_stride, dst, dst_stride, filter, x0_q4, + x_step_q4, y0_q4, y_step_q4, w, h); + break; + } + } +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve_avg_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve_avg_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve_avg_lsx.c @@ -0,0 +1,321 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "./vpx_dsp_rtcd.h" +#include "vpx_util/loongson_intrinsics.h" + +static void avg_width4_lsx(const uint8_t *src, int32_t src_stride, uint8_t *dst, + int32_t dst_stride, int32_t height) { + int32_t cnt; + __m128i src0, src1; + __m128i dst0, dst1; + + int32_t src_stride2 = src_stride << 1; + + if ((height % 2) == 0) { + for (cnt = (height / 2); cnt--;) { + src0 = __lsx_vld(src, 0); + src1 = __lsx_vldx(src, src_stride); + src += src_stride2; + + dst0 = __lsx_vld(dst, 0); + dst1 = __lsx_vldx(dst, dst_stride); + DUP2_ARG2(__lsx_vavgr_bu, src0, dst0, src1, dst1, dst0, dst1); + + __lsx_vstelm_w(dst0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(dst1, dst, 0, 0); + dst += dst_stride; + } + } +} + +static void avg_width8_lsx(const uint8_t *src, int32_t src_stride, uint8_t *dst, + int32_t dst_stride, int32_t height) { + int32_t cnt = (height / 4); + __m128i src0, src1, src2, src3; + __m128i dst0, dst1, dst2, dst3; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + + for (; cnt--;) { + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + dst0 = __lsx_vld(dst, 0); + DUP2_ARG2(__lsx_vldx, dst, dst_stride, dst, dst_stride2, dst1, dst2); + dst3 = __lsx_vldx(dst, dst_stride3); + + DUP4_ARG2(__lsx_vavgr_bu, src0, dst0, src1, dst1, src2, dst2, src3, dst3, + dst0, dst1, dst2, dst3); + + __lsx_vstelm_d(dst0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(dst1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(dst2, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(dst3, dst, 0, 0); + dst += dst_stride; + } +} + +static void avg_width16_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, int32_t height) { + int32_t cnt = (height / 8); + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + int32_t dst_stride4 = dst_stride2 << 1; + + for (; cnt--;) { + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + src += src_stride4; + src4 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src5, src6); + src7 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + dst0 = __lsx_vld(dst, 0); + DUP2_ARG2(__lsx_vldx, dst, dst_stride, dst, dst_stride2, dst1, dst2); + dst3 = __lsx_vldx(dst, dst_stride3); + dst += dst_stride4; + dst4 = __lsx_vld(dst, 0); + DUP2_ARG2(__lsx_vldx, dst, dst_stride, dst, dst_stride2, dst5, dst6); + dst7 = __lsx_vldx(dst, dst_stride3); + dst -= dst_stride4; + + DUP4_ARG2(__lsx_vavgr_bu, src0, dst0, src1, dst1, src2, dst2, src3, dst3, + dst0, dst1, dst2, dst3); + DUP4_ARG2(__lsx_vavgr_bu, src4, dst4, src5, dst5, src6, dst6, src7, dst7, + dst4, dst5, dst6, dst7); + + __lsx_vst(dst0, dst, 0); + __lsx_vstx(dst1, dst, dst_stride); + __lsx_vstx(dst2, dst, dst_stride2); + __lsx_vstx(dst3, dst, dst_stride3); + dst += dst_stride4; + __lsx_vst(dst4, dst, 0); + __lsx_vstx(dst5, dst, dst_stride); + __lsx_vstx(dst6, dst, dst_stride2); + __lsx_vstx(dst7, dst, dst_stride3); + dst += dst_stride4; + } +} + +static void avg_width32_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, int32_t height) { + int32_t cnt = (height / 8); + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i src8, src9, src10, src11, src12, src13, src14, src15; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + __m128i dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + int32_t dst_stride4 = dst_stride2 << 1; + + for (; cnt--;) { + uint8_t *dst_tmp = dst; + uint8_t *dst_tmp1 = dst_tmp + 16; + uint8_t *src_tmp = src + 16; + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vld, src, 0, src_tmp, 0, src0, src1); + DUP4_ARG2(__lsx_vldx, src, src_stride, src_tmp, src_stride, src, + src_stride2, src_tmp, src_stride2, src2, src3, src4, src5); + DUP2_ARG2(__lsx_vldx, src, src_stride3, src_tmp, src_stride3, src6, src7); + src += src_stride4; + + DUP2_ARG2(__lsx_vld, dst_tmp, 0, dst_tmp1, 0, dst0, dst1); + DUP4_ARG2(__lsx_vldx, dst_tmp, dst_stride, dst_tmp1, dst_stride, dst_tmp, + dst_stride2, dst_tmp1, dst_stride2, dst2, dst3, dst4, dst5); + DUP2_ARG2(__lsx_vldx, dst_tmp, dst_stride3, dst_tmp1, dst_stride3, dst6, + dst7); + dst_tmp += dst_stride4; + dst_tmp1 += dst_stride4; + + src_tmp = src + 16; + DUP2_ARG2(__lsx_vld, src, 0, src_tmp, 0, src8, src9); + DUP4_ARG2(__lsx_vldx, src, src_stride, src_tmp, src_stride, src, + src_stride2, src_tmp, src_stride2, src10, src11, src12, src13); + DUP2_ARG2(__lsx_vldx, src, src_stride3, src_tmp, src_stride3, src14, src15); + src += src_stride4; + + DUP2_ARG2(__lsx_vld, dst_tmp, 0, dst_tmp1, 0, dst8, dst9); + DUP4_ARG2(__lsx_vldx, dst_tmp, dst_stride, dst_tmp1, dst_stride, dst_tmp, + dst_stride2, dst_tmp1, dst_stride2, dst10, dst11, dst12, dst13); + DUP2_ARG2(__lsx_vldx, dst_tmp, dst_stride3, dst_tmp1, dst_stride3, dst14, + dst15); + DUP4_ARG2(__lsx_vavgr_bu, src0, dst0, src1, dst1, src2, dst2, src3, dst3, + dst0, dst1, dst2, dst3); + DUP4_ARG2(__lsx_vavgr_bu, src4, dst4, src5, dst5, src6, dst6, src7, dst7, + dst4, dst5, dst6, dst7); + DUP4_ARG2(__lsx_vavgr_bu, src8, dst8, src9, dst9, src10, dst10, src11, + dst11, dst8, dst9, dst10, dst11); + DUP4_ARG2(__lsx_vavgr_bu, src12, dst12, src13, dst13, src14, dst14, src15, + dst15, dst12, dst13, dst14, dst15); + + dst_tmp = dst + 16; + __lsx_vst(dst0, dst, 0); + __lsx_vstx(dst2, dst, dst_stride); + __lsx_vstx(dst4, dst, dst_stride2); + __lsx_vstx(dst6, dst, dst_stride3); + __lsx_vst(dst1, dst_tmp, 0); + __lsx_vstx(dst3, dst_tmp, dst_stride); + __lsx_vstx(dst5, dst_tmp, dst_stride2); + __lsx_vstx(dst7, dst_tmp, dst_stride3); + dst += dst_stride4; + + __lsx_vst(dst8, dst, 0); + __lsx_vstx(dst10, dst, dst_stride); + __lsx_vstx(dst12, dst, dst_stride2); + __lsx_vstx(dst14, dst, dst_stride3); + __lsx_vst(dst9, dst_tmp1, 0); + __lsx_vstx(dst11, dst_tmp1, dst_stride); + __lsx_vstx(dst13, dst_tmp1, dst_stride2); + __lsx_vstx(dst15, dst_tmp1, dst_stride3); + dst += dst_stride4; + } +} + +static void avg_width64_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, int32_t height) { + int32_t cnt = (height / 4); + uint8_t *dst_tmp = dst; + + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i src8, src9, src10, src11, src12, src13, src14, src15; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + __m128i dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15; + + for (; cnt--;) { + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2, + src3); + src += src_stride; + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src4, src5, src6, + src7); + src += src_stride; + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src8, src9, src10, + src11); + src += src_stride; + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src12, src13, src14, + src15); + src += src_stride; + + DUP4_ARG2(__lsx_vld, dst_tmp, 0, dst_tmp, 16, dst_tmp, 32, dst_tmp, 48, + dst0, dst1, dst2, dst3); + dst_tmp += dst_stride; + DUP4_ARG2(__lsx_vld, dst_tmp, 0, dst_tmp, 16, dst_tmp, 32, dst_tmp, 48, + dst4, dst5, dst6, dst7); + dst_tmp += dst_stride; + DUP4_ARG2(__lsx_vld, dst_tmp, 0, dst_tmp, 16, dst_tmp, 32, dst_tmp, 48, + dst8, dst9, dst10, dst11); + dst_tmp += dst_stride; + DUP4_ARG2(__lsx_vld, dst_tmp, 0, dst_tmp, 16, dst_tmp, 32, dst_tmp, 48, + dst12, dst13, dst14, dst15); + dst_tmp += dst_stride; + + DUP4_ARG2(__lsx_vavgr_bu, src0, dst0, src1, dst1, src2, dst2, src3, dst3, + dst0, dst1, dst2, dst3); + DUP4_ARG2(__lsx_vavgr_bu, src4, dst4, src5, dst5, src6, dst6, src7, dst7, + dst4, dst5, dst6, dst7); + DUP4_ARG2(__lsx_vavgr_bu, src8, dst8, src9, dst9, src10, dst10, src11, + dst11, dst8, dst9, dst10, dst11); + DUP4_ARG2(__lsx_vavgr_bu, src12, dst12, src13, dst13, src14, dst14, src15, + dst15, dst12, dst13, dst14, dst15); + + __lsx_vst(dst0, dst, 0); + __lsx_vst(dst1, dst, 16); + __lsx_vst(dst2, dst, 32); + __lsx_vst(dst3, dst, 48); + dst += dst_stride; + __lsx_vst(dst4, dst, 0); + __lsx_vst(dst5, dst, 16); + __lsx_vst(dst6, dst, 32); + __lsx_vst(dst7, dst, 48); + dst += dst_stride; + __lsx_vst(dst8, dst, 0); + __lsx_vst(dst9, dst, 16); + __lsx_vst(dst10, dst, 32); + __lsx_vst(dst11, dst, 48); + dst += dst_stride; + __lsx_vst(dst12, dst, 0); + __lsx_vst(dst13, dst, 16); + __lsx_vst(dst14, dst, 32); + __lsx_vst(dst15, dst, 48); + dst += dst_stride; + } +} + +void vpx_convolve_avg_lsx(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *filter, int x0_q4, + int32_t x_step_q4, int y0_q4, int32_t y_step_q4, + int32_t w, int32_t h) { + (void)filter; + (void)x0_q4; + (void)x_step_q4; + (void)y0_q4; + (void)y_step_q4; + switch (w) { + case 4: { + avg_width4_lsx(src, src_stride, dst, dst_stride, h); + break; + } + + case 8: { + avg_width8_lsx(src, src_stride, dst, dst_stride, h); + break; + } + case 16: { + avg_width16_lsx(src, src_stride, dst, dst_stride, h); + break; + } + case 32: { + avg_width32_lsx(src, src_stride, dst, dst_stride, h); + break; + } + case 64: { + avg_width64_lsx(src, src_stride, dst, dst_stride, h); + break; + } + default: { + int32_t lp, cnt; + for (cnt = h; cnt--;) { + for (lp = 0; lp < w; ++lp) { + dst[lp] = (((dst[lp] + src[lp]) + 1) >> 1); + } + src += src_stride; + dst += dst_stride; + } + break; + } + } +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve_copy_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve_copy_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve_copy_lsx.c @@ -0,0 +1,438 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include "./vpx_dsp_rtcd.h" +#include "vpx_util/loongson_intrinsics.h" + +static void copy_width8_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, int32_t height) { + int32_t cnt; + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + if ((height % 12) == 0) { + for (cnt = (height / 12); cnt--;) { + src0 = __lsx_vld(src, 0); + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src1, src2, src3, src4); + src += src_stride4; + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src5, src6); + src += src_stride2; + src7 = __lsx_vldx(src, src_stride); + src += src_stride2; + + __lsx_vstelm_d(src0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src2, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src3, dst, 0, 0); + dst += dst_stride; + + __lsx_vstelm_d(src4, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src5, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src6, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src7, dst, 0, 0); + dst += dst_stride; + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + __lsx_vstelm_d(src0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src2, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src3, dst, 0, 0); + dst += dst_stride; + } + } else if ((height % 8) == 0) { + for (cnt = height >> 3; cnt--;) { + src0 = __lsx_vld(src, 0); + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src1, src2, src3, src4); + src += src_stride4; + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src5, src6); + src += src_stride2; + src7 = __lsx_vldx(src, src_stride); + src += src_stride2; + + __lsx_vstelm_d(src0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src2, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src3, dst, 0, 0); + dst += dst_stride; + + __lsx_vstelm_d(src4, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src5, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src6, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src7, dst, 0, 0); + dst += dst_stride; + } + } else if ((height % 4) == 0) { + for (cnt = (height / 4); cnt--;) { + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + __lsx_vstelm_d(src0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src2, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src3, dst, 0, 0); + dst += dst_stride; + } + } else if ((height % 2) == 0) { + for (cnt = (height / 2); cnt--;) { + src0 = __lsx_vld(src, 0); + src1 = __lsx_vldx(src, src_stride); + src += src_stride2; + + __lsx_vstelm_d(src0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src1, dst, 0, 0); + dst += dst_stride; + } + } +} + +static void copy_16multx8mult_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int32_t height, int32_t width) { + int32_t cnt, loop_cnt; + uint8_t *src_tmp; + uint8_t *dst_tmp; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + for (cnt = (width >> 4); cnt--;) { + src_tmp = (uint8_t *)src; + dst_tmp = dst; + + for (loop_cnt = (height >> 3); loop_cnt--;) { + src0 = __lsx_vld(src_tmp, 0); + DUP4_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src_tmp, + src_stride3, src_tmp, src_stride4, src1, src2, src3, src4); + src_tmp += src_stride4; + DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src5, + src6); + src_tmp += src_stride2; + src7 = __lsx_vldx(src_tmp, src_stride); + src_tmp += src_stride2; + + __lsx_vst(src0, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src1, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src2, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src3, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src4, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src5, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src6, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src7, dst_tmp, 0); + dst_tmp += dst_stride; + } + src += 16; + dst += 16; + } +} + +static void copy_width16_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, int32_t height) { + int32_t cnt; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + if ((height % 12) == 0) { + for (cnt = (height / 12); cnt--;) { + src0 = __lsx_vld(src, 0); + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src1, src2, src3, src4); + src += src_stride4; + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src5, src6); + src += src_stride2; + src7 = __lsx_vldx(src, src_stride); + src += src_stride2; + + __lsx_vst(src0, dst, 0); + dst += dst_stride; + __lsx_vst(src1, dst, 0); + dst += dst_stride; + __lsx_vst(src2, dst, 0); + dst += dst_stride; + __lsx_vst(src3, dst, 0); + dst += dst_stride; + __lsx_vst(src4, dst, 0); + dst += dst_stride; + __lsx_vst(src5, dst, 0); + dst += dst_stride; + __lsx_vst(src6, dst, 0); + dst += dst_stride; + __lsx_vst(src7, dst, 0); + dst += dst_stride; + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + __lsx_vst(src0, dst, 0); + dst += dst_stride; + __lsx_vst(src1, dst, 0); + dst += dst_stride; + __lsx_vst(src2, dst, 0); + dst += dst_stride; + __lsx_vst(src3, dst, 0); + dst += dst_stride; + } + } else if ((height % 8) == 0) { + copy_16multx8mult_lsx(src, src_stride, dst, dst_stride, height, 16); + } else if ((height % 4) == 0) { + for (cnt = (height >> 2); cnt--;) { + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + __lsx_vst(src0, dst, 0); + dst += dst_stride; + __lsx_vst(src1, dst, 0); + dst += dst_stride; + __lsx_vst(src2, dst, 0); + dst += dst_stride; + __lsx_vst(src3, dst, 0); + dst += dst_stride; + } + } +} + +static void copy_width32_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, int32_t height) { + int32_t cnt; + uint8_t *src_tmp; + uint8_t *dst_tmp; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + if ((height % 12) == 0) { + for (cnt = (height / 12); cnt--;) { + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + + src_tmp = (uint8_t *)src + 16; + src4 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src5, + src6); + src7 = __lsx_vldx(src_tmp, src_stride3); + src += src_stride4; + + __lsx_vst(src0, dst, 0); + dst += dst_stride; + __lsx_vst(src1, dst, 0); + dst += dst_stride; + __lsx_vst(src2, dst, 0); + dst += dst_stride; + __lsx_vst(src3, dst, 0); + dst += dst_stride; + + dst_tmp = dst + 16; + __lsx_vst(src4, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src5, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src6, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src7, dst_tmp, 0); + dst_tmp += dst_stride; + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + + src_tmp = (uint8_t *)src + 16; + src4 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src5, + src6); + src7 = __lsx_vldx(src_tmp, src_stride3); + src += src_stride4; + + __lsx_vst(src0, dst, 0); + dst += dst_stride; + __lsx_vst(src1, dst, 0); + dst += dst_stride; + __lsx_vst(src2, dst, 0); + dst += dst_stride; + __lsx_vst(src3, dst, 0); + dst += dst_stride; + + dst_tmp = dst + 16; + __lsx_vst(src4, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src5, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src6, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src7, dst_tmp, 0); + dst_tmp += dst_stride; + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + + src_tmp = (uint8_t *)src + 16; + src4 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src5, + src6); + src7 = __lsx_vldx(src_tmp, src_stride3); + src += src_stride4; + + __lsx_vst(src0, dst, 0); + dst += dst_stride; + __lsx_vst(src1, dst, 0); + dst += dst_stride; + __lsx_vst(src2, dst, 0); + dst += dst_stride; + __lsx_vst(src3, dst, 0); + dst += dst_stride; + + dst_tmp = dst + 16; + __lsx_vst(src4, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src5, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src6, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src7, dst_tmp, 0); + dst_tmp += dst_stride; + } + } else if ((height % 8) == 0) { + copy_16multx8mult_lsx(src, src_stride, dst, dst_stride, height, 32); + } else if ((height % 4) == 0) { + for (cnt = (height >> 2); cnt--;) { + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + + src_tmp = (uint8_t *)src + 16; + src4 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src5, + src6); + src7 = __lsx_vldx(src_tmp, src_stride3); + src += src_stride4; + + __lsx_vst(src0, dst, 0); + dst += dst_stride; + __lsx_vst(src1, dst, 0); + dst += dst_stride; + __lsx_vst(src2, dst, 0); + dst += dst_stride; + __lsx_vst(src3, dst, 0); + dst += dst_stride; + + dst_tmp = dst + 16; + __lsx_vst(src4, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src5, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src6, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src7, dst_tmp, 0); + dst_tmp += dst_stride; + } + } +} + +static void copy_width64_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, int32_t height) { + copy_16multx8mult_lsx(src, src_stride, dst, dst_stride, height, 64); +} + +void vpx_convolve_copy_lsx(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *filter, int x0_q4, + int32_t x_step_q4, int y0_q4, int32_t y_step_q4, + int32_t w, int32_t h) { + (void)filter; + (void)x0_q4; + (void)x_step_q4; + (void)y0_q4; + (void)y_step_q4; + + switch (w) { + case 4: { + uint32_t cnt; + __m128i tmp; + for (cnt = h; cnt--;) { + tmp = __lsx_vldrepl_w(src, 0); + __lsx_vstelm_w(tmp, dst, 0, 0); + src += src_stride; + dst += dst_stride; + } + break; + } + case 8: { + copy_width8_lsx(src, src_stride, dst, dst_stride, h); + break; + } + case 16: { + copy_width16_lsx(src, src_stride, dst, dst_stride, h); + break; + } + case 32: { + copy_width32_lsx(src, src_stride, dst, dst_stride, h); + break; + } + case 64: { + copy_width64_lsx(src, src_stride, dst, dst_stride, h); + break; + } + default: { + uint32_t cnt; + for (cnt = h; cnt--;) { + memcpy(dst, src, w); + src += src_stride; + dst += dst_stride; + } + break; + } + } +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve_lsx.h b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve_lsx.h new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve_lsx.h @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VPX_VPX_DSP_LOONGARCH_VPX_CONVOLVE_LSX_H_ +#define VPX_VPX_DSP_LOONGARCH_VPX_CONVOLVE_LSX_H_ + +#include "vpx_util/loongson_intrinsics.h" +#include "vpx_dsp/vpx_filter.h" + +#define LSX_LD_4(_src, _stride, _src0, _src1, _src2, _src3) \ + { \ + _src0 = __lsx_vld(_src, 0); \ + _src += _stride; \ + _src1 = __lsx_vld(_src, 0); \ + _src += _stride; \ + _src2 = __lsx_vld(_src, 0); \ + _src += _stride; \ + _src3 = __lsx_vld(_src, 0); \ + } + +#define FILT_8TAP_DPADD_S_H(_reg0, _reg1, _reg2, _reg3, _filter0, _filter1, \ + _filter2, _filter3) \ + ({ \ + __m128i _vec0, _vec1; \ + \ + _vec0 = __lsx_vdp2_h_b(_reg0, _filter0); \ + _vec0 = __lsx_vdp2add_h_b(_vec0, _reg1, _filter1); \ + _vec1 = __lsx_vdp2_h_b(_reg2, _filter2); \ + _vec1 = __lsx_vdp2add_h_b(_vec1, _reg3, _filter3); \ + _vec0 = __lsx_vsadd_h(_vec0, _vec1); \ + \ + _vec0; \ + }) + +#define HORIZ_8TAP_FILT(_src0, _src1, _mask0, _mask1, _mask2, _mask3, \ + _filt_h0, _filt_h1, _filt_h2, _filt_h3) \ + ({ \ + __m128i _tmp0, _tmp1, _tmp2, _tmp3; \ + __m128i _out; \ + \ + DUP4_ARG3(__lsx_vshuf_b, _src1, _src0, _mask0, _src1, _src0, _mask1, \ + _src1, _src0, _mask2, _src1, _src0, _mask3, _tmp0, _tmp1, _tmp2, \ + _tmp3); \ + _out = FILT_8TAP_DPADD_S_H(_tmp0, _tmp1, _tmp2, _tmp3, _filt_h0, _filt_h1, \ + _filt_h2, _filt_h3); \ + _out = __lsx_vsrari_h(_out, FILTER_BITS); \ + _out = __lsx_vsat_h(_out, 7); \ + \ + _out; \ + }) + +#define HORIZ_8TAP_4WID_4VECS_FILT(_src0, _src1, _src2, _src3, _mask0, _mask1, \ + _mask2, _mask3, _filter0, _filter1, \ + _filter2, _filter3, _out0, _out1) \ + { \ + __m128i _tmp0, _tmp1, _tmp2, _tmp3, _tmp4, _tmp5, _tmp6, _tmp7; \ + __m128i _reg0, _reg1, _reg2, _reg3; \ + \ + DUP2_ARG3(__lsx_vshuf_b, _src1, _src0, _mask0, _src3, _src2, _mask0, \ + _tmp0, _tmp1); \ + DUP2_ARG2(__lsx_vdp2_h_b, _tmp0, _filter0, _tmp1, _filter0, _reg0, _reg1); \ + DUP2_ARG3(__lsx_vshuf_b, _src1, _src0, _mask1, _src3, _src2, _mask1, \ + _tmp2, _tmp3); \ + DUP2_ARG3(__lsx_vdp2add_h_b, _reg0, _tmp2, _filter1, _reg1, _tmp3, \ + _filter1, _reg0, _reg1); \ + DUP2_ARG3(__lsx_vshuf_b, _src1, _src0, _mask2, _src3, _src2, _mask2, \ + _tmp4, _tmp5); \ + DUP2_ARG2(__lsx_vdp2_h_b, _tmp4, _filter2, _tmp5, _filter2, _reg2, _reg3); \ + DUP2_ARG3(__lsx_vshuf_b, _src1, _src0, _mask3, _src3, _src2, _mask3, \ + _tmp6, _tmp7); \ + DUP2_ARG3(__lsx_vdp2add_h_b, _reg2, _tmp6, _filter3, _reg3, _tmp7, \ + _filter3, _reg2, _reg3); \ + DUP2_ARG2(__lsx_vsadd_h, _reg0, _reg2, _reg1, _reg3, _out0, _out1); \ + } + +#define HORIZ_8TAP_8WID_4VECS_FILT( \ + _src0, _src1, _src2, _src3, _mask0, _mask1, _mask2, _mask3, _filter0, \ + _filter1, _filter2, _filter3, _out0, _out1, _out2, _out3) \ + { \ + __m128i _tmp0, _tmp1, _tmp2, _tmp3, _tmp4, _tmp5, _tmp6, _tmp7; \ + __m128i _reg0, _reg1, _reg2, _reg3, _reg4, _reg5, _reg6, _reg7; \ + \ + DUP4_ARG3(__lsx_vshuf_b, _src0, _src0, _mask0, _src1, _src1, _mask0, \ + _src2, _src2, _mask0, _src3, _src3, _mask0, _tmp0, _tmp1, _tmp2, \ + _tmp3); \ + DUP4_ARG2(__lsx_vdp2_h_b, _tmp0, _filter0, _tmp1, _filter0, _tmp2, \ + _filter0, _tmp3, _filter0, _reg0, _reg1, _reg2, _reg3); \ + DUP4_ARG3(__lsx_vshuf_b, _src0, _src0, _mask2, _src1, _src1, _mask2, \ + _src2, _src2, _mask2, _src3, _src3, _mask2, _tmp0, _tmp1, _tmp2, \ + _tmp3); \ + DUP4_ARG2(__lsx_vdp2_h_b, _tmp0, _filter2, _tmp1, _filter2, _tmp2, \ + _filter2, _tmp3, _filter2, _reg4, _reg5, _reg6, _reg7); \ + DUP4_ARG3(__lsx_vshuf_b, _src0, _src0, _mask1, _src1, _src1, _mask1, \ + _src2, _src2, _mask1, _src3, _src3, _mask1, _tmp4, _tmp5, _tmp6, \ + _tmp7); \ + DUP4_ARG3(__lsx_vdp2add_h_b, _reg0, _tmp4, _filter1, _reg1, _tmp5, \ + _filter1, _reg2, _tmp6, _filter1, _reg3, _tmp7, _filter1, _reg0, \ + _reg1, _reg2, _reg3); \ + DUP4_ARG3(__lsx_vshuf_b, _src0, _src0, _mask3, _src1, _src1, _mask3, \ + _src2, _src2, _mask3, _src3, _src3, _mask3, _tmp4, _tmp5, _tmp6, \ + _tmp7); \ + DUP4_ARG3(__lsx_vdp2add_h_b, _reg4, _tmp4, _filter3, _reg5, _tmp5, \ + _filter3, _reg6, _tmp6, _filter3, _reg7, _tmp7, _filter3, _reg4, \ + _reg5, _reg6, _reg7); \ + DUP4_ARG2(__lsx_vsadd_h, _reg0, _reg4, _reg1, _reg5, _reg2, _reg6, _reg3, \ + _reg7, _out0, _out1, _out2, _out3); \ + } + +#define HORIZ_2TAP_FILT_UH(in0, in1, mask, coeff, shift) \ + ({ \ + __m128i tmp0_m; \ + __m128i tmp1_m; \ + \ + tmp0_m = __lsx_vshuf_b(in1, in0, mask); \ + tmp1_m = __lsx_vdp2_h_bu(tmp0_m, coeff); \ + tmp1_m = __lsx_vsrari_h(tmp1_m, shift); \ + \ + tmp1_m; \ + }) + +#define PCKEV_AVG_ST4_D(in0, in1, in2, in3, dst0, dst1, pdst, stride) \ + { \ + __m128i tmp0_m, tmp1_m; \ + \ + DUP2_ARG2(__lsx_vpickev_b, in1, in0, in3, in2, tmp0_m, tmp1_m); \ + DUP2_ARG2(__lsx_vavgr_bu, tmp0_m, dst0, tmp1_m, dst1, tmp0_m, tmp1_m); \ + __lsx_vstelm_d(tmp0_m, pdst, 0, 0); \ + pdst += stride; \ + __lsx_vstelm_d(tmp0_m, pdst, 0, 1); \ + pdst += stride; \ + __lsx_vstelm_d(tmp1_m, pdst, 0, 0); \ + pdst += stride; \ + __lsx_vstelm_d(tmp1_m, pdst, 0, 1); \ + } + +#endif // VPX_VPX_DSP_LOONGARCH_VPX_CONVOLVE_LSX_H_ diff --git a/media/libvpx/libvpx/vpx_dsp/mips/sad_mmi.c b/media/libvpx/libvpx/vpx_dsp/mips/sad_mmi.c --- a/media/libvpx/libvpx/vpx_dsp/mips/sad_mmi.c +++ b/media/libvpx/libvpx/vpx_dsp/mips/sad_mmi.c @@ -332,23 +332,10 @@ "mthc1 $0, %[ftmp1] \n\t" \ "biadd %[ftmp1], %[ftmp1] \n\t" \ "paddw %[ftmp3], %[ftmp3], %[ftmp1] \n\t" #endif /* _MIPS_SIM == _ABIO32 */ -// depending on call sites, pass **ref_array to avoid & in subsequent call and -// de-dup with 4D below. -#define sadMxNxK_mmi(m, n, k) \ - void vpx_sad##m##x##n##x##k##_mmi(const uint8_t *src, int src_stride, \ - const uint8_t *ref_array, int ref_stride, \ - uint32_t *sad_array) { \ - int i; \ - for (i = 0; i < (k); ++i) \ - sad_array[i] = \ - vpx_sad##m##x##n##_mmi(src, src_stride, &ref_array[i], ref_stride); \ - } - -// This appears to be equivalent to the above when k == 4 and refs is const #define sadMxNx4D_mmi(m, n) \ void vpx_sad##m##x##n##x4d_mmi(const uint8_t *src, int src_stride, \ const uint8_t *const ref_array[], \ int ref_stride, uint32_t *sad_array) { \ int i; \ @@ -581,14 +568,10 @@ } vpx_sad16xN(32); vpx_sad16xN(16); vpx_sad16xN(8); -sadMxNxK_mmi(16, 16, 3); -sadMxNxK_mmi(16, 16, 8); -sadMxNxK_mmi(16, 8, 3); -sadMxNxK_mmi(16, 8, 8); sadMxNx4D_mmi(16, 32); sadMxNx4D_mmi(16, 16); sadMxNx4D_mmi(16, 8); static inline unsigned int vpx_sad_avg16x(const uint8_t *src, int src_stride, @@ -679,14 +662,10 @@ } vpx_sad8xN(16); vpx_sad8xN(8); vpx_sad8xN(4); -sadMxNxK_mmi(8, 16, 3); -sadMxNxK_mmi(8, 16, 8); -sadMxNxK_mmi(8, 8, 3); -sadMxNxK_mmi(8, 8, 8); sadMxNx4D_mmi(8, 16); sadMxNx4D_mmi(8, 8); sadMxNx4D_mmi(8, 4); static inline unsigned int vpx_sad_avg8x(const uint8_t *src, int src_stride, @@ -775,12 +754,10 @@ return vpx_sad4x(src, src_stride, ref, ref_stride, H); \ } vpx_sad4xN(8); vpx_sad4xN(4); -sadMxNxK_mmi(4, 4, 3); -sadMxNxK_mmi(4, 4, 8); sadMxNx4D_mmi(4, 8); sadMxNx4D_mmi(4, 4); static inline unsigned int vpx_sad_avg4x(const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, diff --git a/media/libvpx/libvpx/vpx_dsp/mips/sad_msa.c b/media/libvpx/libvpx/vpx_dsp/mips/sad_msa.c --- a/media/libvpx/libvpx/vpx_dsp/mips/sad_msa.c +++ b/media/libvpx/libvpx/vpx_dsp/mips/sad_msa.c @@ -157,384 +157,10 @@ sad += HADD_UH_U32(sad1); return sad; } -static void sad_4width_x3_msa(const uint8_t *src_ptr, int32_t src_stride, - const uint8_t *ref_ptr, int32_t ref_stride, - int32_t height, uint32_t *sad_array) { - int32_t ht_cnt; - uint32_t src0, src1, src2, src3; - v16u8 src = { 0 }; - v16u8 ref = { 0 }; - v16u8 ref0, ref1, ref2, ref3, diff; - v8u16 sad0 = { 0 }; - v8u16 sad1 = { 0 }; - v8u16 sad2 = { 0 }; - - for (ht_cnt = (height >> 2); ht_cnt--;) { - LW4(src_ptr, src_stride, src0, src1, src2, src3); - src_ptr += (4 * src_stride); - INSERT_W4_UB(src0, src1, src2, src3, src); - - LD_UB4(ref_ptr, ref_stride, ref0, ref1, ref2, ref3); - ref_ptr += (4 * ref_stride); - SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); - diff = __msa_asub_u_b(src, ref); - sad0 += __msa_hadd_u_h(diff, diff); - - SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1); - SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1); - SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); - diff = __msa_asub_u_b(src, ref); - sad1 += __msa_hadd_u_h(diff, diff); - - SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1); - SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1); - SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); - diff = __msa_asub_u_b(src, ref); - sad2 += __msa_hadd_u_h(diff, diff); - } - - sad_array[0] = HADD_UH_U32(sad0); - sad_array[1] = HADD_UH_U32(sad1); - sad_array[2] = HADD_UH_U32(sad2); -} - -static void sad_8width_x3_msa(const uint8_t *src, int32_t src_stride, - const uint8_t *ref, int32_t ref_stride, - int32_t height, uint32_t *sad_array) { - int32_t ht_cnt; - v16u8 src0, src1, src2, src3; - v16u8 ref0, ref1, ref00, ref11, ref22, ref33; - v8u16 sad0 = { 0 }; - v8u16 sad1 = { 0 }; - v8u16 sad2 = { 0 }; - - for (ht_cnt = (height >> 2); ht_cnt--;) { - LD_UB4(src, src_stride, src0, src1, src2, src3); - src += (4 * src_stride); - LD_UB4(ref, ref_stride, ref00, ref11, ref22, ref33); - ref += (4 * ref_stride); - PCKEV_D4_UB(src1, src0, src3, src2, ref11, ref00, ref33, ref22, src0, src1, - ref0, ref1); - sad0 += SAD_UB2_UH(src0, src1, ref0, ref1); - - SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1); - SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1); - PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1); - sad1 += SAD_UB2_UH(src0, src1, ref0, ref1); - - SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1); - SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1); - PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1); - sad2 += SAD_UB2_UH(src0, src1, ref0, ref1); - } - - sad_array[0] = HADD_UH_U32(sad0); - sad_array[1] = HADD_UH_U32(sad1); - sad_array[2] = HADD_UH_U32(sad2); -} - -static void sad_16width_x3_msa(const uint8_t *src_ptr, int32_t src_stride, - const uint8_t *ref_ptr, int32_t ref_stride, - int32_t height, uint32_t *sad_array) { - int32_t ht_cnt; - v16u8 src, ref, ref0, ref1, diff; - v8u16 sad0 = { 0 }; - v8u16 sad1 = { 0 }; - v8u16 sad2 = { 0 }; - - for (ht_cnt = (height >> 1); ht_cnt--;) { - src = LD_UB(src_ptr); - src_ptr += src_stride; - LD_UB2(ref_ptr, 16, ref0, ref1); - ref_ptr += ref_stride; - - diff = __msa_asub_u_b(src, ref0); - sad0 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 1); - diff = __msa_asub_u_b(src, ref); - sad1 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 2); - diff = __msa_asub_u_b(src, ref); - sad2 += __msa_hadd_u_h(diff, diff); - - src = LD_UB(src_ptr); - src_ptr += src_stride; - LD_UB2(ref_ptr, 16, ref0, ref1); - ref_ptr += ref_stride; - - diff = __msa_asub_u_b(src, ref0); - sad0 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 1); - diff = __msa_asub_u_b(src, ref); - sad1 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 2); - diff = __msa_asub_u_b(src, ref); - sad2 += __msa_hadd_u_h(diff, diff); - } - - sad_array[0] = HADD_UH_U32(sad0); - sad_array[1] = HADD_UH_U32(sad1); - sad_array[2] = HADD_UH_U32(sad2); -} - -static void sad_4width_x8_msa(const uint8_t *src_ptr, int32_t src_stride, - const uint8_t *ref_ptr, int32_t ref_stride, - int32_t height, uint32_t *sad_array) { - int32_t ht_cnt; - uint32_t src0, src1, src2, src3; - v16u8 ref0, ref1, ref2, ref3, diff; - v16u8 src = { 0 }; - v16u8 ref = { 0 }; - v8u16 sad0 = { 0 }; - v8u16 sad1 = { 0 }; - v8u16 sad2 = { 0 }; - v8u16 sad3 = { 0 }; - v8u16 sad4 = { 0 }; - v8u16 sad5 = { 0 }; - v8u16 sad6 = { 0 }; - v8u16 sad7 = { 0 }; - - for (ht_cnt = (height >> 2); ht_cnt--;) { - LW4(src_ptr, src_stride, src0, src1, src2, src3); - INSERT_W4_UB(src0, src1, src2, src3, src); - src_ptr += (4 * src_stride); - LD_UB4(ref_ptr, ref_stride, ref0, ref1, ref2, ref3); - ref_ptr += (4 * ref_stride); - - SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); - diff = __msa_asub_u_b(src, ref); - sad0 += __msa_hadd_u_h(diff, diff); - - SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1); - SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1); - SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); - diff = __msa_asub_u_b(src, ref); - sad1 += __msa_hadd_u_h(diff, diff); - - SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1); - SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1); - SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); - diff = __msa_asub_u_b(src, ref); - sad2 += __msa_hadd_u_h(diff, diff); - - SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1); - SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1); - SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); - diff = __msa_asub_u_b(src, ref); - sad3 += __msa_hadd_u_h(diff, diff); - - SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1); - SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1); - SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); - diff = __msa_asub_u_b(src, ref); - sad4 += __msa_hadd_u_h(diff, diff); - - SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1); - SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1); - SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); - diff = __msa_asub_u_b(src, ref); - sad5 += __msa_hadd_u_h(diff, diff); - - SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1); - SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1); - SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); - diff = __msa_asub_u_b(src, ref); - sad6 += __msa_hadd_u_h(diff, diff); - - SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1); - SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1); - SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); - diff = __msa_asub_u_b(src, ref); - sad7 += __msa_hadd_u_h(diff, diff); - } - - sad_array[0] = HADD_UH_U32(sad0); - sad_array[1] = HADD_UH_U32(sad1); - sad_array[2] = HADD_UH_U32(sad2); - sad_array[3] = HADD_UH_U32(sad3); - sad_array[4] = HADD_UH_U32(sad4); - sad_array[5] = HADD_UH_U32(sad5); - sad_array[6] = HADD_UH_U32(sad6); - sad_array[7] = HADD_UH_U32(sad7); -} - -static void sad_8width_x8_msa(const uint8_t *src, int32_t src_stride, - const uint8_t *ref, int32_t ref_stride, - int32_t height, uint32_t *sad_array) { - int32_t ht_cnt; - v16u8 src0, src1, src2, src3; - v16u8 ref0, ref1, ref00, ref11, ref22, ref33; - v8u16 sad0 = { 0 }; - v8u16 sad1 = { 0 }; - v8u16 sad2 = { 0 }; - v8u16 sad3 = { 0 }; - v8u16 sad4 = { 0 }; - v8u16 sad5 = { 0 }; - v8u16 sad6 = { 0 }; - v8u16 sad7 = { 0 }; - - for (ht_cnt = (height >> 2); ht_cnt--;) { - LD_UB4(src, src_stride, src0, src1, src2, src3); - src += (4 * src_stride); - LD_UB4(ref, ref_stride, ref00, ref11, ref22, ref33); - ref += (4 * ref_stride); - PCKEV_D4_UB(src1, src0, src3, src2, ref11, ref00, ref33, ref22, src0, src1, - ref0, ref1); - sad0 += SAD_UB2_UH(src0, src1, ref0, ref1); - - SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1); - SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1); - PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1); - sad1 += SAD_UB2_UH(src0, src1, ref0, ref1); - - SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1); - SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1); - PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1); - sad2 += SAD_UB2_UH(src0, src1, ref0, ref1); - - SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1); - SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1); - PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1); - sad3 += SAD_UB2_UH(src0, src1, ref0, ref1); - - SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1); - SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1); - PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1); - sad4 += SAD_UB2_UH(src0, src1, ref0, ref1); - - SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1); - SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1); - PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1); - sad5 += SAD_UB2_UH(src0, src1, ref0, ref1); - - SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1); - SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1); - PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1); - sad6 += SAD_UB2_UH(src0, src1, ref0, ref1); - - SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1); - SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1); - PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1); - sad7 += SAD_UB2_UH(src0, src1, ref0, ref1); - } - - sad_array[0] = HADD_UH_U32(sad0); - sad_array[1] = HADD_UH_U32(sad1); - sad_array[2] = HADD_UH_U32(sad2); - sad_array[3] = HADD_UH_U32(sad3); - sad_array[4] = HADD_UH_U32(sad4); - sad_array[5] = HADD_UH_U32(sad5); - sad_array[6] = HADD_UH_U32(sad6); - sad_array[7] = HADD_UH_U32(sad7); -} - -static void sad_16width_x8_msa(const uint8_t *src_ptr, int32_t src_stride, - const uint8_t *ref_ptr, int32_t ref_stride, - int32_t height, uint32_t *sad_array) { - int32_t ht_cnt; - v16u8 src, ref0, ref1, ref; - v16u8 diff; - v8u16 sad0 = { 0 }; - v8u16 sad1 = { 0 }; - v8u16 sad2 = { 0 }; - v8u16 sad3 = { 0 }; - v8u16 sad4 = { 0 }; - v8u16 sad5 = { 0 }; - v8u16 sad6 = { 0 }; - v8u16 sad7 = { 0 }; - - for (ht_cnt = (height >> 1); ht_cnt--;) { - src = LD_UB(src_ptr); - src_ptr += src_stride; - LD_UB2(ref_ptr, 16, ref0, ref1); - ref_ptr += ref_stride; - - diff = __msa_asub_u_b(src, ref0); - sad0 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 1); - diff = __msa_asub_u_b(src, ref); - sad1 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 2); - diff = __msa_asub_u_b(src, ref); - sad2 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 3); - diff = __msa_asub_u_b(src, ref); - sad3 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 4); - diff = __msa_asub_u_b(src, ref); - sad4 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 5); - diff = __msa_asub_u_b(src, ref); - sad5 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 6); - diff = __msa_asub_u_b(src, ref); - sad6 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 7); - diff = __msa_asub_u_b(src, ref); - sad7 += __msa_hadd_u_h(diff, diff); - - src = LD_UB(src_ptr); - src_ptr += src_stride; - LD_UB2(ref_ptr, 16, ref0, ref1); - ref_ptr += ref_stride; - - diff = __msa_asub_u_b(src, ref0); - sad0 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 1); - diff = __msa_asub_u_b(src, ref); - sad1 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 2); - diff = __msa_asub_u_b(src, ref); - sad2 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 3); - diff = __msa_asub_u_b(src, ref); - sad3 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 4); - diff = __msa_asub_u_b(src, ref); - sad4 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 5); - diff = __msa_asub_u_b(src, ref); - sad5 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 6); - diff = __msa_asub_u_b(src, ref); - sad6 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 7); - diff = __msa_asub_u_b(src, ref); - sad7 += __msa_hadd_u_h(diff, diff); - } - - sad_array[0] = HADD_UH_U32(sad0); - sad_array[1] = HADD_UH_U32(sad1); - sad_array[2] = HADD_UH_U32(sad2); - sad_array[3] = HADD_UH_U32(sad3); - sad_array[4] = HADD_UH_U32(sad4); - sad_array[5] = HADD_UH_U32(sad5); - sad_array[6] = HADD_UH_U32(sad6); - sad_array[7] = HADD_UH_U32(sad7); -} - static void sad_4width_x4d_msa(const uint8_t *src_ptr, int32_t src_stride, const uint8_t *const aref_ptr[], int32_t ref_stride, int32_t height, uint32_t *sad_array) { const uint8_t *ref0_ptr, *ref1_ptr, *ref2_ptr, *ref3_ptr; @@ -1035,84 +661,42 @@ uint32_t vpx_sad64x##height##_msa(const uint8_t *src, int32_t src_stride, \ const uint8_t *ref, int32_t ref_stride) { \ return sad_64width_msa(src, src_stride, ref, ref_stride, height); \ } -#define VPX_SAD_4xHEIGHTx3_MSA(height) \ - void vpx_sad4x##height##x3_msa(const uint8_t *src, int32_t src_stride, \ - const uint8_t *ref, int32_t ref_stride, \ - uint32_t *sads) { \ - sad_4width_x3_msa(src, src_stride, ref, ref_stride, height, sads); \ - } - -#define VPX_SAD_8xHEIGHTx3_MSA(height) \ - void vpx_sad8x##height##x3_msa(const uint8_t *src, int32_t src_stride, \ - const uint8_t *ref, int32_t ref_stride, \ - uint32_t *sads) { \ - sad_8width_x3_msa(src, src_stride, ref, ref_stride, height, sads); \ - } - -#define VPX_SAD_16xHEIGHTx3_MSA(height) \ - void vpx_sad16x##height##x3_msa(const uint8_t *src, int32_t src_stride, \ - const uint8_t *ref, int32_t ref_stride, \ - uint32_t *sads) { \ - sad_16width_x3_msa(src, src_stride, ref, ref_stride, height, sads); \ - } - -#define VPX_SAD_4xHEIGHTx8_MSA(height) \ - void vpx_sad4x##height##x8_msa(const uint8_t *src, int32_t src_stride, \ - const uint8_t *ref, int32_t ref_stride, \ - uint32_t *sads) { \ - sad_4width_x8_msa(src, src_stride, ref, ref_stride, height, sads); \ - } - -#define VPX_SAD_8xHEIGHTx8_MSA(height) \ - void vpx_sad8x##height##x8_msa(const uint8_t *src, int32_t src_stride, \ - const uint8_t *ref, int32_t ref_stride, \ - uint32_t *sads) { \ - sad_8width_x8_msa(src, src_stride, ref, ref_stride, height, sads); \ - } - -#define VPX_SAD_16xHEIGHTx8_MSA(height) \ - void vpx_sad16x##height##x8_msa(const uint8_t *src, int32_t src_stride, \ - const uint8_t *ref, int32_t ref_stride, \ - uint32_t *sads) { \ - sad_16width_x8_msa(src, src_stride, ref, ref_stride, height, sads); \ - } - #define VPX_SAD_4xHEIGHTx4D_MSA(height) \ void vpx_sad4x##height##x4d_msa(const uint8_t *src, int32_t src_stride, \ - const uint8_t *const refs[], \ - int32_t ref_stride, uint32_t *sads) { \ + const uint8_t *const refs[4], \ + int32_t ref_stride, uint32_t sads[4]) { \ sad_4width_x4d_msa(src, src_stride, refs, ref_stride, height, sads); \ } #define VPX_SAD_8xHEIGHTx4D_MSA(height) \ void vpx_sad8x##height##x4d_msa(const uint8_t *src, int32_t src_stride, \ - const uint8_t *const refs[], \ - int32_t ref_stride, uint32_t *sads) { \ + const uint8_t *const refs[4], \ + int32_t ref_stride, uint32_t sads[4]) { \ sad_8width_x4d_msa(src, src_stride, refs, ref_stride, height, sads); \ } #define VPX_SAD_16xHEIGHTx4D_MSA(height) \ void vpx_sad16x##height##x4d_msa(const uint8_t *src, int32_t src_stride, \ - const uint8_t *const refs[], \ - int32_t ref_stride, uint32_t *sads) { \ + const uint8_t *const refs[4], \ + int32_t ref_stride, uint32_t sads[4]) { \ sad_16width_x4d_msa(src, src_stride, refs, ref_stride, height, sads); \ } #define VPX_SAD_32xHEIGHTx4D_MSA(height) \ void vpx_sad32x##height##x4d_msa(const uint8_t *src, int32_t src_stride, \ - const uint8_t *const refs[], \ - int32_t ref_stride, uint32_t *sads) { \ + const uint8_t *const refs[4], \ + int32_t ref_stride, uint32_t sads[4]) { \ sad_32width_x4d_msa(src, src_stride, refs, ref_stride, height, sads); \ } #define VPX_SAD_64xHEIGHTx4D_MSA(height) \ void vpx_sad64x##height##x4d_msa(const uint8_t *src, int32_t src_stride, \ - const uint8_t *const refs[], \ - int32_t ref_stride, uint32_t *sads) { \ + const uint8_t *const refs[4], \ + int32_t ref_stride, uint32_t sads[4]) { \ sad_64width_x4d_msa(src, src_stride, refs, ref_stride, height, sads); \ } #define VPX_AVGSAD_4xHEIGHT_MSA(height) \ uint32_t vpx_sad4x##height##_avg_msa(const uint8_t *src, int32_t src_stride, \ @@ -1184,33 +768,25 @@ VPX_SAD_16xHEIGHTx4D_MSA(32); VPX_AVGSAD_16xHEIGHT_MSA(32); // 16x16 VPX_SAD_16xHEIGHT_MSA(16); -VPX_SAD_16xHEIGHTx3_MSA(16); -VPX_SAD_16xHEIGHTx8_MSA(16); VPX_SAD_16xHEIGHTx4D_MSA(16); VPX_AVGSAD_16xHEIGHT_MSA(16); // 16x8 VPX_SAD_16xHEIGHT_MSA(8); -VPX_SAD_16xHEIGHTx3_MSA(8); -VPX_SAD_16xHEIGHTx8_MSA(8); VPX_SAD_16xHEIGHTx4D_MSA(8); VPX_AVGSAD_16xHEIGHT_MSA(8); // 8x16 VPX_SAD_8xHEIGHT_MSA(16); -VPX_SAD_8xHEIGHTx3_MSA(16); -VPX_SAD_8xHEIGHTx8_MSA(16); VPX_SAD_8xHEIGHTx4D_MSA(16); VPX_AVGSAD_8xHEIGHT_MSA(16); // 8x8 VPX_SAD_8xHEIGHT_MSA(8); -VPX_SAD_8xHEIGHTx3_MSA(8); -VPX_SAD_8xHEIGHTx8_MSA(8); VPX_SAD_8xHEIGHTx4D_MSA(8); VPX_AVGSAD_8xHEIGHT_MSA(8); // 8x4 VPX_SAD_8xHEIGHT_MSA(4); @@ -1222,9 +798,7 @@ VPX_SAD_4xHEIGHTx4D_MSA(8); VPX_AVGSAD_4xHEIGHT_MSA(8); // 4x4 VPX_SAD_4xHEIGHT_MSA(4); -VPX_SAD_4xHEIGHTx3_MSA(4); -VPX_SAD_4xHEIGHTx8_MSA(4); VPX_SAD_4xHEIGHTx4D_MSA(4); VPX_AVGSAD_4xHEIGHT_MSA(4); diff --git a/media/libvpx/libvpx/vpx_dsp/ppc/quantize_vsx.c b/media/libvpx/libvpx/vpx_dsp/ppc/quantize_vsx.c --- a/media/libvpx/libvpx/vpx_dsp/ppc/quantize_vsx.c +++ b/media/libvpx/libvpx/vpx_dsp/ppc/quantize_vsx.c @@ -93,12 +93,12 @@ a = vec_max(a, vec_perm(a, a, vec_perm32)); return vec_max(a, vec_perm(a, a, vec_perm16)); } void vpx_quantize_b_vsx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *zbin_ptr, - const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *zbin_ptr, const int16_t *round_ptr, + const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan_ptr, const int16_t *iscan_ptr) { int16x8_t qcoeff0, qcoeff1, dqcoeff0, dqcoeff1, eob; @@ -120,12 +120,10 @@ zero_mask0 = vec_cmpge(coeff0_abs, zbin); zbin = vec_splat(zbin, 1); zero_mask1 = vec_cmpge(coeff1_abs, zbin); (void)scan_ptr; - (void)skip_block; - assert(!skip_block); qcoeff0 = quantize_coeff(coeff0, coeff0_abs, round, quant, quant_shift, zero_mask0); vec_vsx_st(qcoeff0, 0, qcoeff_ptr); round = vec_splat(round, 1); @@ -194,16 +192,18 @@ eob = vec_max_across(eob); *eob_ptr = eob[0]; } -void vpx_quantize_b_32x32_vsx( - const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, - const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, - const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, - const int16_t *scan_ptr, const int16_t *iscan_ptr) { +void vpx_quantize_b_32x32_vsx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + const int16_t *zbin_ptr, const int16_t *round_ptr, + const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan_ptr, + const int16_t *iscan_ptr) { // In stage 1, we quantize 16 coeffs (DC + 15 AC) // In stage 2, we loop 42 times and quantize 24 coeffs per iteration // (32 * 32 - 16) / 24 = 42 int num_itr = 42; // Offsets are in bytes, 16 coeffs = 32 bytes @@ -225,13 +225,11 @@ int16x8_t coeff0_abs = vec_abs(coeff0); int16x8_t coeff1_abs = vec_abs(coeff1); (void)scan_ptr; - (void)skip_block; (void)n_coeffs; - assert(!skip_block); // 32x32 quantization requires that zbin and round be divided by 2 zbin = vec_sra(vec_add(zbin, vec_ones_s16), vec_ones_u16); round = vec_sra(vec_add(round, vec_ones_s16), vec_ones_u16); diff --git a/media/libvpx/libvpx/vpx_dsp/quantize.h b/media/libvpx/libvpx/vpx_dsp/quantize.h --- a/media/libvpx/libvpx/vpx_dsp/quantize.h +++ b/media/libvpx/libvpx/vpx_dsp/quantize.h @@ -16,26 +16,25 @@ #ifdef __cplusplus extern "C" { #endif -void vpx_quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs, int skip_block, +void vpx_quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs, const int16_t *round_ptr, const int16_t quant, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t dequant, uint16_t *eob_ptr); -void vpx_quantize_dc_32x32(const tran_low_t *coeff_ptr, int skip_block, +void vpx_quantize_dc_32x32(const tran_low_t *coeff_ptr, const int16_t *round_ptr, const int16_t quant, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t dequant, uint16_t *eob_ptr); #if CONFIG_VP9_HIGHBITDEPTH void vpx_highbd_quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs, - int skip_block, const int16_t *round_ptr, - const int16_t quant, tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, const int16_t dequant, - uint16_t *eob_ptr); -void vpx_highbd_quantize_dc_32x32(const tran_low_t *coeff_ptr, int skip_block, + const int16_t *round_ptr, const int16_t quant, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t dequant, uint16_t *eob_ptr); +void vpx_highbd_quantize_dc_32x32(const tran_low_t *coeff_ptr, const int16_t *round_ptr, const int16_t quant, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t dequant, uint16_t *eob_ptr); #endif diff --git a/media/libvpx/libvpx/vpx_dsp/quantize.c b/media/libvpx/libvpx/vpx_dsp/quantize.c --- a/media/libvpx/libvpx/vpx_dsp/quantize.c +++ b/media/libvpx/libvpx/vpx_dsp/quantize.c @@ -13,11 +13,11 @@ #include "./vpx_dsp_rtcd.h" #include "vpx_dsp/quantize.h" #include "vpx_dsp/vpx_dsp_common.h" #include "vpx_mem/vpx_mem.h" -void vpx_quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs, int skip_block, +void vpx_quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs, const int16_t *round_ptr, const int16_t quant, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t dequant, uint16_t *eob_ptr) { const int rc = 0; const int coeff = coeff_ptr[rc]; @@ -26,46 +26,45 @@ int tmp, eob = -1; memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); - if (!skip_block) { - tmp = clamp(abs_coeff + round_ptr[rc != 0], INT16_MIN, INT16_MAX); - tmp = (tmp * quant) >> 16; - qcoeff_ptr[rc] = (tmp ^ coeff_sign) - coeff_sign; - dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant; - if (tmp) eob = 0; - } + tmp = clamp(abs_coeff + round_ptr[rc != 0], INT16_MIN, INT16_MAX); + tmp = (tmp * quant) >> 16; + qcoeff_ptr[rc] = (tmp ^ coeff_sign) - coeff_sign; + dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant; + if (tmp) eob = 0; + *eob_ptr = eob + 1; } #if CONFIG_VP9_HIGHBITDEPTH void vpx_highbd_quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs, - int skip_block, const int16_t *round_ptr, - const int16_t quant, tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, const int16_t dequant, - uint16_t *eob_ptr) { + const int16_t *round_ptr, const int16_t quant, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t dequant, uint16_t *eob_ptr) { int eob = -1; memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); - if (!skip_block) { + { const int coeff = coeff_ptr[0]; const int coeff_sign = (coeff >> 31); const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; const int64_t tmp = abs_coeff + round_ptr[0]; const int abs_qcoeff = (int)((tmp * quant) >> 16); qcoeff_ptr[0] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); dqcoeff_ptr[0] = qcoeff_ptr[0] * dequant; if (abs_qcoeff) eob = 0; } + *eob_ptr = eob + 1; } #endif -void vpx_quantize_dc_32x32(const tran_low_t *coeff_ptr, int skip_block, +void vpx_quantize_dc_32x32(const tran_low_t *coeff_ptr, const int16_t *round_ptr, const int16_t quant, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t dequant, uint16_t *eob_ptr) { const int n_coeffs = 1024; const int rc = 0; @@ -75,60 +74,57 @@ int tmp, eob = -1; memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); - if (!skip_block) { - tmp = clamp(abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1), - INT16_MIN, INT16_MAX); - tmp = (tmp * quant) >> 15; - qcoeff_ptr[rc] = (tmp ^ coeff_sign) - coeff_sign; - dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant / 2; - if (tmp) eob = 0; - } + tmp = clamp(abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1), INT16_MIN, + INT16_MAX); + tmp = (tmp * quant) >> 15; + qcoeff_ptr[rc] = (tmp ^ coeff_sign) - coeff_sign; + dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant / 2; + if (tmp) eob = 0; + *eob_ptr = eob + 1; } #if CONFIG_VP9_HIGHBITDEPTH -void vpx_highbd_quantize_dc_32x32(const tran_low_t *coeff_ptr, int skip_block, +void vpx_highbd_quantize_dc_32x32(const tran_low_t *coeff_ptr, const int16_t *round_ptr, const int16_t quant, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t dequant, uint16_t *eob_ptr) { const int n_coeffs = 1024; int eob = -1; memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); - if (!skip_block) { + { const int coeff = coeff_ptr[0]; const int coeff_sign = (coeff >> 31); const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; const int64_t tmp = abs_coeff + ROUND_POWER_OF_TWO(round_ptr[0], 1); const int abs_qcoeff = (int)((tmp * quant) >> 15); qcoeff_ptr[0] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); dqcoeff_ptr[0] = qcoeff_ptr[0] * dequant / 2; if (abs_qcoeff) eob = 0; } + *eob_ptr = eob + 1; } #endif void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *zbin_ptr, - const int16_t *round_ptr, const int16_t *quant_ptr, - const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, - uint16_t *eob_ptr, const int16_t *scan, - const int16_t *iscan) { + const int16_t *zbin_ptr, const int16_t *round_ptr, + const int16_t *quant_ptr, const int16_t *quant_shift_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { int i, non_zero_count = (int)n_coeffs, eob = -1; const int zbins[2] = { zbin_ptr[0], zbin_ptr[1] }; const int nzbins[2] = { zbins[0] * -1, zbins[1] * -1 }; (void)iscan; - (void)skip_block; - assert(!skip_block); memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); // Pre-scan pass @@ -164,22 +160,20 @@ *eob_ptr = eob + 1; } #if CONFIG_VP9_HIGHBITDEPTH void vpx_highbd_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *zbin_ptr, - const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *zbin_ptr, const int16_t *round_ptr, + const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan) { int i, non_zero_count = (int)n_coeffs, eob = -1; const int zbins[2] = { zbin_ptr[0], zbin_ptr[1] }; const int nzbins[2] = { zbins[0] * -1, zbins[1] * -1 }; (void)iscan; - (void)skip_block; - assert(!skip_block); memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); // Pre-scan pass @@ -213,12 +207,12 @@ *eob_ptr = eob + 1; } #endif void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *zbin_ptr, - const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *zbin_ptr, const int16_t *round_ptr, + const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan) { const int zbins[2] = { ROUND_POWER_OF_TWO(zbin_ptr[0], 1), @@ -227,12 +221,10 @@ int idx = 0; int idx_arr[1024]; int i, eob = -1; (void)iscan; - (void)skip_block; - assert(!skip_block); memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); // Pre-scan pass @@ -275,12 +267,12 @@ *eob_ptr = eob + 1; } #if CONFIG_VP9_HIGHBITDEPTH void vpx_highbd_quantize_b_32x32_c( - const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, - const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, + const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan) { const int zbins[2] = { ROUND_POWER_OF_TWO(zbin_ptr[0], 1), ROUND_POWER_OF_TWO(zbin_ptr[1], 1) }; @@ -288,12 +280,10 @@ int idx = 0; int idx_arr[1024]; int i, eob = -1; (void)iscan; - (void)skip_block; - assert(!skip_block); memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); // Pre-scan pass diff --git a/media/libvpx/libvpx/vpx_dsp/sad.c b/media/libvpx/libvpx/vpx_dsp/sad.c --- a/media/libvpx/libvpx/vpx_dsp/sad.c +++ b/media/libvpx/libvpx/vpx_dsp/sad.c @@ -43,27 +43,15 @@ DECLARE_ALIGNED(16, uint8_t, comp_pred[m * n]); \ vpx_comp_avg_pred_c(comp_pred, second_pred, m, n, ref_ptr, ref_stride); \ return sad(src_ptr, src_stride, comp_pred, m, m, n); \ } -// depending on call sites, pass **ref_array to avoid & in subsequent call and -// de-dup with 4D below. -#define sadMxNxK(m, n, k) \ - void vpx_sad##m##x##n##x##k##_c(const uint8_t *src_ptr, int src_stride, \ - const uint8_t *ref_ptr, int ref_stride, \ - uint32_t *sad_array) { \ - int i; \ - for (i = 0; i < k; ++i) \ - sad_array[i] = \ - vpx_sad##m##x##n##_c(src_ptr, src_stride, &ref_ptr[i], ref_stride); \ - } - -// This appears to be equivalent to the above when k == 4 and refs is const +// Compare |src_ptr| to 4 distinct references in |ref_array[4]| #define sadMxNx4D(m, n) \ void vpx_sad##m##x##n##x4d_c(const uint8_t *src_ptr, int src_stride, \ - const uint8_t *const ref_array[], \ - int ref_stride, uint32_t *sad_array) { \ + const uint8_t *const ref_array[4], \ + int ref_stride, uint32_t sad_array[4]) { \ int i; \ for (i = 0; i < 4; ++i) \ sad_array[i] = \ vpx_sad##m##x##n##_c(src_ptr, src_stride, ref_array[i], ref_stride); \ } @@ -81,11 +69,10 @@ sadMxN(32, 64) sadMxNx4D(32, 64) // 32x32 sadMxN(32, 32) -sadMxNxK(32, 32, 8) sadMxNx4D(32, 32) // 32x16 sadMxN(32, 16) sadMxNx4D(32, 16) @@ -94,30 +81,22 @@ sadMxN(16, 32) sadMxNx4D(16, 32) // 16x16 sadMxN(16, 16) -sadMxNxK(16, 16, 3) -sadMxNxK(16, 16, 8) sadMxNx4D(16, 16) // 16x8 sadMxN(16, 8) -sadMxNxK(16, 8, 3) -sadMxNxK(16, 8, 8) sadMxNx4D(16, 8) // 8x16 sadMxN(8, 16) -sadMxNxK(8, 16, 3) -sadMxNxK(8, 16, 8) sadMxNx4D(8, 16) // 8x8 sadMxN(8, 8) -sadMxNxK(8, 8, 3) -sadMxNxK(8, 8, 8) sadMxNx4D(8, 8) // 8x4 sadMxN(8, 4) sadMxNx4D(8, 4) @@ -126,12 +105,10 @@ sadMxN(4, 8) sadMxNx4D(4, 8) // 4x4 sadMxN(4, 4) -sadMxNxK(4, 4, 3) -sadMxNxK(4, 4, 8) sadMxNx4D(4, 4) /* clang-format on */ #if CONFIG_VP9_HIGHBITDEPTH static INLINE @@ -179,19 +156,19 @@ vpx_highbd_comp_avg_pred_c(comp_pred, CONVERT_TO_SHORTPTR(second_pred), m, \ n, CONVERT_TO_SHORTPTR(ref_ptr), ref_stride); \ return highbd_sadb(src_ptr, src_stride, comp_pred, m, m, n); \ } -#define highbd_sadMxNx4D(m, n) \ - void vpx_highbd_sad##m##x##n##x4d_c(const uint8_t *src_ptr, int src_stride, \ - const uint8_t *const ref_array[], \ - int ref_stride, uint32_t *sad_array) { \ - int i; \ - for (i = 0; i < 4; ++i) { \ - sad_array[i] = vpx_highbd_sad##m##x##n##_c(src_ptr, src_stride, \ - ref_array[i], ref_stride); \ - } \ +#define highbd_sadMxNx4D(m, n) \ + void vpx_highbd_sad##m##x##n##x4d_c(const uint8_t *src_ptr, int src_stride, \ + const uint8_t *const ref_array[4], \ + int ref_stride, uint32_t sad_array[4]) { \ + int i; \ + for (i = 0; i < 4; ++i) { \ + sad_array[i] = vpx_highbd_sad##m##x##n##_c(src_ptr, src_stride, \ + ref_array[i], ref_stride); \ + } \ } /* clang-format off */ // 64x64 highbd_sadMxN(64, 64) diff --git a/media/libvpx/libvpx/vpx_dsp/variance.h b/media/libvpx/libvpx/vpx_dsp/variance.h --- a/media/libvpx/libvpx/vpx_dsp/variance.h +++ b/media/libvpx/libvpx/vpx_dsp/variance.h @@ -57,12 +57,10 @@ #if CONFIG_VP8 typedef struct variance_vtable { vpx_sad_fn_t sdf; vpx_variance_fn_t vf; vpx_subpixvariance_fn_t svf; - vpx_sad_multi_fn_t sdx3f; - vpx_sad_multi_fn_t sdx8f; vpx_sad_multi_d_fn_t sdx4df; #if VPX_ARCH_X86 || VPX_ARCH_X86_64 vp8_copy32xn_fn_t copymem; #endif } vp8_variance_fn_ptr_t; @@ -74,11 +72,10 @@ vpx_sad_avg_fn_t sdaf; vpx_variance_fn_t vf; vpx_subpixvariance_fn_t svf; vpx_subp_avg_variance_fn_t svaf; vpx_sad_multi_d_fn_t sdx4df; - vpx_sad_multi_fn_t sdx8f; } vp9_variance_fn_ptr_t; #endif // CONFIG_VP9 #ifdef __cplusplus } // extern "C" diff --git a/media/libvpx/libvpx/vpx_dsp/vpx_dsp.mk b/media/libvpx/libvpx/vpx_dsp/vpx_dsp.mk --- a/media/libvpx/libvpx/vpx_dsp/vpx_dsp.mk +++ b/media/libvpx/libvpx/vpx_dsp/vpx_dsp.mk @@ -72,10 +72,11 @@ endif # CONFIG_POSTPROC DSP_SRCS-$(HAVE_NEON_ASM) += arm/intrapred_neon_asm$(ASM) DSP_SRCS-$(HAVE_NEON) += arm/intrapred_neon.c DSP_SRCS-$(HAVE_MSA) += mips/intrapred_msa.c +DSP_SRCS-$(HAVE_LSX) += loongarch/intrapred_lsx.c DSP_SRCS-$(HAVE_DSPR2) += mips/intrapred4_dspr2.c DSP_SRCS-$(HAVE_DSPR2) += mips/intrapred8_dspr2.c DSP_SRCS-$(HAVE_DSPR2) += mips/intrapred16_dspr2.c DSP_SRCS-$(HAVE_DSPR2) += mips/common_dspr2.h @@ -160,14 +161,25 @@ DSP_SRCS-$(HAVE_DSPR2) += mips/convolve8_horiz_dspr2.c DSP_SRCS-$(HAVE_DSPR2) += mips/convolve8_vert_dspr2.c DSP_SRCS-$(HAVE_VSX) += ppc/vpx_convolve_vsx.c +# common (lsx) +DSP_SRCS-$(HAVE_LSX) += loongarch/vpx_convolve8_avg_horiz_lsx.c +DSP_SRCS-$(HAVE_LSX) += loongarch/vpx_convolve8_avg_vert_lsx.c +DSP_SRCS-$(HAVE_LSX) += loongarch/vpx_convolve8_horiz_lsx.c +DSP_SRCS-$(HAVE_LSX) += loongarch/vpx_convolve8_vert_lsx.c +DSP_SRCS-$(HAVE_LSX) += loongarch/vpx_convolve8_lsx.c +DSP_SRCS-$(HAVE_LSX) += loongarch/vpx_convolve8_avg_lsx.c +DSP_SRCS-$(HAVE_LSX) += loongarch/vpx_convolve_avg_lsx.c +DSP_SRCS-$(HAVE_LSX) += loongarch/vpx_convolve_copy_lsx.c +DSP_SRCS-$(HAVE_LSX) += loongarch/vpx_convolve_lsx.h + # loop filters DSP_SRCS-yes += loopfilter.c -DSP_SRCS-$(HAVE_SSE2) += x86/loopfilter_intrin_sse2.c +DSP_SRCS-$(HAVE_SSE2) += x86/loopfilter_sse2.c DSP_SRCS-$(HAVE_AVX2) += x86/loopfilter_avx2.c ifeq ($(HAVE_NEON_ASM),yes) DSP_SRCS-yes += arm/loopfilter_16_neon$(ASM) DSP_SRCS-yes += arm/loopfilter_8_neon$(ASM) @@ -186,19 +198,25 @@ DSP_SRCS-$(HAVE_DSPR2) += mips/loopfilter_masks_dspr2.h DSP_SRCS-$(HAVE_DSPR2) += mips/loopfilter_mb_dspr2.c DSP_SRCS-$(HAVE_DSPR2) += mips/loopfilter_mb_horiz_dspr2.c DSP_SRCS-$(HAVE_DSPR2) += mips/loopfilter_mb_vert_dspr2.c +DSP_SRCS-$(HAVE_LSX) += loongarch/loopfilter_lsx.h +DSP_SRCS-$(HAVE_LSX) += loongarch/loopfilter_16_lsx.c +DSP_SRCS-$(HAVE_LSX) += loongarch/loopfilter_8_lsx.c +DSP_SRCS-$(HAVE_LSX) += loongarch/loopfilter_4_lsx.c + ifeq ($(CONFIG_VP9_HIGHBITDEPTH),yes) DSP_SRCS-$(HAVE_NEON) += arm/highbd_loopfilter_neon.c DSP_SRCS-$(HAVE_SSE2) += x86/highbd_loopfilter_sse2.c endif # CONFIG_VP9_HIGHBITDEPTH endif # CONFIG_VP9 DSP_SRCS-yes += txfm_common.h DSP_SRCS-$(HAVE_SSE2) += x86/txfm_common_sse2.h DSP_SRCS-$(HAVE_MSA) += mips/txfm_macros_msa.h +DSP_SRCS-$(HAVE_LSX) += loongarch/txfm_macros_lsx.h # forward transform ifeq ($(CONFIG_VP9_ENCODER),yes) DSP_SRCS-yes += fwd_txfm.c DSP_SRCS-yes += fwd_txfm.h DSP_SRCS-$(HAVE_SSE2) += x86/fwd_txfm_sse2.h @@ -215,13 +233,16 @@ DSP_SRCS-$(HAVE_NEON) += arm/fdct32x32_neon.c DSP_SRCS-$(HAVE_NEON) += arm/fdct_partial_neon.c DSP_SRCS-$(HAVE_NEON) += arm/fwd_txfm_neon.c DSP_SRCS-$(HAVE_MSA) += mips/fwd_txfm_msa.h DSP_SRCS-$(HAVE_MSA) += mips/fwd_txfm_msa.c +DSP_SRCS-$(HAVE_LSX) += loongarch/fwd_txfm_lsx.h +DSP_SRCS-$(HAVE_LSX) += loongarch/fwd_txfm_lsx.c ifneq ($(CONFIG_VP9_HIGHBITDEPTH),yes) DSP_SRCS-$(HAVE_MSA) += mips/fwd_dct32x32_msa.c +DSP_SRCS-$(HAVE_LSX) += loongarch/fwd_dct32x32_lsx.c endif # !CONFIG_VP9_HIGHBITDEPTH DSP_SRCS-$(HAVE_VSX) += ppc/fdct32x32_vsx.c endif # CONFIG_VP9_ENCODER @@ -250,10 +271,12 @@ DSP_SRCS-$(HAVE_DSPR2) += mips/itrans4_dspr2.c DSP_SRCS-$(HAVE_DSPR2) += mips/itrans8_dspr2.c DSP_SRCS-$(HAVE_DSPR2) += mips/itrans16_dspr2.c DSP_SRCS-$(HAVE_DSPR2) += mips/itrans32_dspr2.c DSP_SRCS-$(HAVE_DSPR2) += mips/itrans32_cols_dspr2.c + +DSP_SRCS-$(HAVE_LSX) += loongarch/idct32x32_lsx.c else # CONFIG_VP9_HIGHBITDEPTH DSP_SRCS-$(HAVE_NEON) += arm/highbd_idct4x4_add_neon.c DSP_SRCS-$(HAVE_NEON) += arm/highbd_idct8x8_add_neon.c DSP_SRCS-$(HAVE_NEON) += arm/highbd_idct16x16_add_neon.c DSP_SRCS-$(HAVE_NEON) += arm/highbd_idct32x32_add_neon.c @@ -340,16 +363,15 @@ DSP_SRCS-$(HAVE_NEON) += arm/subtract_neon.c DSP_SRCS-$(HAVE_MSA) += mips/sad_msa.c DSP_SRCS-$(HAVE_MSA) += mips/subtract_msa.c +DSP_SRCS-$(HAVE_LSX) += loongarch/sad_lsx.c + DSP_SRCS-$(HAVE_MMI) += mips/sad_mmi.c DSP_SRCS-$(HAVE_MMI) += mips/subtract_mmi.c -DSP_SRCS-$(HAVE_SSE3) += x86/sad_sse3.asm -DSP_SRCS-$(HAVE_SSSE3) += x86/sad_ssse3.asm -DSP_SRCS-$(HAVE_SSE4_1) += x86/sad_sse4.asm DSP_SRCS-$(HAVE_AVX2) += x86/sad4d_avx2.c DSP_SRCS-$(HAVE_AVX2) += x86/sad_avx2.c DSP_SRCS-$(HAVE_AVX512) += x86/sad4d_avx512.c DSP_SRCS-$(HAVE_SSE2) += x86/sad4d_sse2.asm @@ -375,10 +397,14 @@ DSP_SRCS-$(HAVE_NEON) += arm/variance_neon.c DSP_SRCS-$(HAVE_MSA) += mips/variance_msa.c DSP_SRCS-$(HAVE_MSA) += mips/sub_pixel_variance_msa.c +DSP_SRCS-$(HAVE_LSX) += loongarch/variance_lsx.c +DSP_SRCS-$(HAVE_LSX) += loongarch/sub_pixel_variance_lsx.c +DSP_SRCS-$(HAVE_LSX) += loongarch/avg_pred_lsx.c + DSP_SRCS-$(HAVE_MMI) += mips/variance_mmi.c DSP_SRCS-$(HAVE_SSE2) += x86/avg_pred_sse2.c DSP_SRCS-$(HAVE_SSE2) += x86/variance_sse2.c # Contains SSE2 and SSSE3 DSP_SRCS-$(HAVE_AVX2) += x86/variance_avx2.c diff --git a/media/libvpx/libvpx/vpx_dsp/vpx_dsp_rtcd_defs.pl b/media/libvpx/libvpx/vpx_dsp/vpx_dsp_rtcd_defs.pl --- a/media/libvpx/libvpx/vpx_dsp/vpx_dsp_rtcd_defs.pl +++ b/media/libvpx/libvpx/vpx_dsp/vpx_dsp_rtcd_defs.pl @@ -115,11 +115,11 @@ # TODO(crbug.com/webm/1522): Re-enable vsx implementation. specialize qw/vpx_tm_predictor_8x8 neon dspr2 msa sse2/; add_proto qw/void vpx_dc_predictor_8x8/, "uint8_t *dst, ptrdiff_t stride, const uint8_t *above, const uint8_t *left"; # TODO(crbug.com/webm/1522): Re-enable vsx implementation. -specialize qw/vpx_dc_predictor_8x8 dspr2 neon msa sse2/; +specialize qw/vpx_dc_predictor_8x8 dspr2 neon msa sse2 lsx/; add_proto qw/void vpx_dc_top_predictor_8x8/, "uint8_t *dst, ptrdiff_t stride, const uint8_t *above, const uint8_t *left"; specialize qw/vpx_dc_top_predictor_8x8 neon msa sse2/; add_proto qw/void vpx_dc_left_predictor_8x8/, "uint8_t *dst, ptrdiff_t stride, const uint8_t *above, const uint8_t *left"; @@ -153,11 +153,11 @@ add_proto qw/void vpx_tm_predictor_16x16/, "uint8_t *dst, ptrdiff_t stride, const uint8_t *above, const uint8_t *left"; specialize qw/vpx_tm_predictor_16x16 neon msa sse2 vsx/; add_proto qw/void vpx_dc_predictor_16x16/, "uint8_t *dst, ptrdiff_t stride, const uint8_t *above, const uint8_t *left"; -specialize qw/vpx_dc_predictor_16x16 dspr2 neon msa sse2 vsx/; +specialize qw/vpx_dc_predictor_16x16 dspr2 neon msa sse2 vsx lsx/; add_proto qw/void vpx_dc_top_predictor_16x16/, "uint8_t *dst, ptrdiff_t stride, const uint8_t *above, const uint8_t *left"; specialize qw/vpx_dc_top_predictor_16x16 neon msa sse2 vsx/; add_proto qw/void vpx_dc_left_predictor_16x16/, "uint8_t *dst, ptrdiff_t stride, const uint8_t *above, const uint8_t *left"; @@ -366,32 +366,32 @@ if (vpx_config("CONFIG_VP9") eq "yes") { # # Sub Pixel Filters # add_proto qw/void vpx_convolve_copy/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h"; -specialize qw/vpx_convolve_copy neon dspr2 msa sse2 vsx/; +specialize qw/vpx_convolve_copy neon dspr2 msa sse2 vsx lsx/; add_proto qw/void vpx_convolve_avg/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h"; -specialize qw/vpx_convolve_avg neon dspr2 msa sse2 vsx mmi/; +specialize qw/vpx_convolve_avg neon dspr2 msa sse2 vsx mmi lsx/; add_proto qw/void vpx_convolve8/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h"; -specialize qw/vpx_convolve8 sse2 ssse3 avx2 neon dspr2 msa vsx mmi/; +specialize qw/vpx_convolve8 sse2 ssse3 avx2 neon dspr2 msa vsx mmi lsx/; add_proto qw/void vpx_convolve8_horiz/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h"; -specialize qw/vpx_convolve8_horiz sse2 ssse3 avx2 neon dspr2 msa vsx mmi/; +specialize qw/vpx_convolve8_horiz sse2 ssse3 avx2 neon dspr2 msa vsx mmi lsx/; add_proto qw/void vpx_convolve8_vert/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h"; -specialize qw/vpx_convolve8_vert sse2 ssse3 avx2 neon dspr2 msa vsx mmi/; +specialize qw/vpx_convolve8_vert sse2 ssse3 avx2 neon dspr2 msa vsx mmi lsx/; add_proto qw/void vpx_convolve8_avg/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h"; -specialize qw/vpx_convolve8_avg sse2 ssse3 avx2 neon dspr2 msa vsx mmi/; +specialize qw/vpx_convolve8_avg sse2 ssse3 avx2 neon dspr2 msa vsx mmi lsx/; add_proto qw/void vpx_convolve8_avg_horiz/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h"; -specialize qw/vpx_convolve8_avg_horiz sse2 ssse3 avx2 neon dspr2 msa vsx mmi/; +specialize qw/vpx_convolve8_avg_horiz sse2 ssse3 avx2 neon dspr2 msa vsx mmi lsx/; add_proto qw/void vpx_convolve8_avg_vert/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h"; -specialize qw/vpx_convolve8_avg_vert sse2 ssse3 avx2 neon dspr2 msa vsx mmi/; +specialize qw/vpx_convolve8_avg_vert sse2 ssse3 avx2 neon dspr2 msa vsx mmi lsx/; add_proto qw/void vpx_scaled_2d/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h"; specialize qw/vpx_scaled_2d ssse3 neon msa/; add_proto qw/void vpx_scaled_horiz/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h"; @@ -440,41 +440,41 @@ # add_proto qw/void vpx_lpf_vertical_16/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh"; specialize qw/vpx_lpf_vertical_16 sse2 neon dspr2 msa/; add_proto qw/void vpx_lpf_vertical_16_dual/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh"; -specialize qw/vpx_lpf_vertical_16_dual sse2 neon dspr2 msa/; +specialize qw/vpx_lpf_vertical_16_dual sse2 neon dspr2 msa lsx/; add_proto qw/void vpx_lpf_vertical_8/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh"; -specialize qw/vpx_lpf_vertical_8 sse2 neon dspr2 msa/; +specialize qw/vpx_lpf_vertical_8 sse2 neon dspr2 msa lsx/; add_proto qw/void vpx_lpf_vertical_8_dual/, "uint8_t *s, int pitch, const uint8_t *blimit0, const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, const uint8_t *thresh1"; -specialize qw/vpx_lpf_vertical_8_dual sse2 neon dspr2 msa/; +specialize qw/vpx_lpf_vertical_8_dual sse2 neon dspr2 msa lsx/; add_proto qw/void vpx_lpf_vertical_4/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh"; -specialize qw/vpx_lpf_vertical_4 sse2 neon dspr2 msa/; +specialize qw/vpx_lpf_vertical_4 sse2 neon dspr2 msa lsx/; add_proto qw/void vpx_lpf_vertical_4_dual/, "uint8_t *s, int pitch, const uint8_t *blimit0, const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, const uint8_t *thresh1"; -specialize qw/vpx_lpf_vertical_4_dual sse2 neon dspr2 msa/; +specialize qw/vpx_lpf_vertical_4_dual sse2 neon dspr2 msa lsx/; add_proto qw/void vpx_lpf_horizontal_16/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh"; specialize qw/vpx_lpf_horizontal_16 sse2 avx2 neon dspr2 msa/; add_proto qw/void vpx_lpf_horizontal_16_dual/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh"; -specialize qw/vpx_lpf_horizontal_16_dual sse2 avx2 neon dspr2 msa/; +specialize qw/vpx_lpf_horizontal_16_dual sse2 avx2 neon dspr2 msa lsx/; add_proto qw/void vpx_lpf_horizontal_8/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh"; -specialize qw/vpx_lpf_horizontal_8 sse2 neon dspr2 msa/; +specialize qw/vpx_lpf_horizontal_8 sse2 neon dspr2 msa lsx/; add_proto qw/void vpx_lpf_horizontal_8_dual/, "uint8_t *s, int pitch, const uint8_t *blimit0, const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, const uint8_t *thresh1"; -specialize qw/vpx_lpf_horizontal_8_dual sse2 neon dspr2 msa/; +specialize qw/vpx_lpf_horizontal_8_dual sse2 neon dspr2 msa lsx/; add_proto qw/void vpx_lpf_horizontal_4/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh"; -specialize qw/vpx_lpf_horizontal_4 sse2 neon dspr2 msa/; +specialize qw/vpx_lpf_horizontal_4 sse2 neon dspr2 msa lsx/; add_proto qw/void vpx_lpf_horizontal_4_dual/, "uint8_t *s, int pitch, const uint8_t *blimit0, const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, const uint8_t *thresh1"; -specialize qw/vpx_lpf_horizontal_4_dual sse2 neon dspr2 msa/; +specialize qw/vpx_lpf_horizontal_4_dual sse2 neon dspr2 msa lsx/; } #CONFIG_VP9 if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { add_proto qw/void vpx_highbd_lpf_vertical_16/, "uint16_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh, int bd"; specialize qw/vpx_highbd_lpf_vertical_16 sse2 neon/; @@ -583,20 +583,20 @@ add_proto qw/void vpx_fdct8x8_1/, "const int16_t *input, tran_low_t *output, int stride"; specialize qw/vpx_fdct8x8_1 sse2 neon msa/; add_proto qw/void vpx_fdct16x16/, "const int16_t *input, tran_low_t *output, int stride"; - specialize qw/vpx_fdct16x16 neon sse2 msa/; + specialize qw/vpx_fdct16x16 neon sse2 msa lsx/; add_proto qw/void vpx_fdct16x16_1/, "const int16_t *input, tran_low_t *output, int stride"; specialize qw/vpx_fdct16x16_1 sse2 neon msa/; add_proto qw/void vpx_fdct32x32/, "const int16_t *input, tran_low_t *output, int stride"; - specialize qw/vpx_fdct32x32 neon sse2 avx2 msa/; + specialize qw/vpx_fdct32x32 neon sse2 avx2 msa lsx/; add_proto qw/void vpx_fdct32x32_rd/, "const int16_t *input, tran_low_t *output, int stride"; - specialize qw/vpx_fdct32x32_rd sse2 avx2 neon msa vsx/; + specialize qw/vpx_fdct32x32_rd sse2 avx2 neon msa vsx lsx/; add_proto qw/void vpx_fdct32x32_1/, "const int16_t *input, tran_low_t *output, int stride"; specialize qw/vpx_fdct32x32_1 sse2 neon msa/; } # CONFIG_VP9_HIGHBITDEPTH } # CONFIG_VP9_ENCODER @@ -650,16 +650,17 @@ specialize qw/vpx_idct16x16_38_add dspr2 msa/; $vpx_idct16x16_38_add_dspr2=vpx_idct16x16_256_add_dspr2; $vpx_idct16x16_38_add_msa=vpx_idct16x16_256_add_msa; specialize qw/vpx_idct16x16_10_add dspr2 msa/; specialize qw/vpx_idct16x16_1_add dspr2 msa/; - specialize qw/vpx_idct32x32_1024_add dspr2 msa/; + specialize qw/vpx_idct32x32_1024_add dspr2 msa lsx/; specialize qw/vpx_idct32x32_135_add dspr2 msa/; $vpx_idct32x32_135_add_dspr2=vpx_idct32x32_1024_add_dspr2; $vpx_idct32x32_135_add_msa=vpx_idct32x32_1024_add_msa; - specialize qw/vpx_idct32x32_34_add dspr2 msa/; - specialize qw/vpx_idct32x32_1_add dspr2 msa/; + $vpx_idct32x32_135_add_lsx=vpx_idct32x32_1024_add_lsx; + specialize qw/vpx_idct32x32_34_add dspr2 msa lsx/; + specialize qw/vpx_idct32x32_1_add dspr2 msa lsx/; specialize qw/vpx_iwht4x4_16_add msa/; specialize qw/vpx_iwht4x4_1_add msa/; } # !CONFIG_VP9_HIGHBITDEPTH } # !CONFIG_EMULATE_HARDWARE @@ -707,21 +708,21 @@ # # Quantization # if (vpx_config("CONFIG_VP9_ENCODER") eq "yes") { - add_proto qw/void vpx_quantize_b/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; + add_proto qw/void vpx_quantize_b/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; specialize qw/vpx_quantize_b neon sse2 ssse3 avx vsx/; - add_proto qw/void vpx_quantize_b_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; + add_proto qw/void vpx_quantize_b_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; specialize qw/vpx_quantize_b_32x32 neon ssse3 avx vsx/; if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { - add_proto qw/void vpx_highbd_quantize_b/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; + add_proto qw/void vpx_highbd_quantize_b/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; specialize qw/vpx_highbd_quantize_b sse2/; - add_proto qw/void vpx_highbd_quantize_b_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; + add_proto qw/void vpx_highbd_quantize_b_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; specialize qw/vpx_highbd_quantize_b_32x32 sse2/; } # CONFIG_VP9_HIGHBITDEPTH } # CONFIG_VP9_ENCODER if (vpx_config("CONFIG_ENCODERS") eq "yes") { @@ -733,29 +734,29 @@ # # Single block SAD # add_proto qw/unsigned int vpx_sad64x64/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride"; -specialize qw/vpx_sad64x64 neon avx2 msa sse2 vsx mmi/; +specialize qw/vpx_sad64x64 neon avx2 msa sse2 vsx mmi lsx/; add_proto qw/unsigned int vpx_sad64x32/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride"; specialize qw/vpx_sad64x32 neon avx2 msa sse2 vsx mmi/; add_proto qw/unsigned int vpx_sad32x64/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride"; specialize qw/vpx_sad32x64 neon avx2 msa sse2 vsx mmi/; add_proto qw/unsigned int vpx_sad32x32/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride"; -specialize qw/vpx_sad32x32 neon avx2 msa sse2 vsx mmi/; +specialize qw/vpx_sad32x32 neon avx2 msa sse2 vsx mmi lsx/; add_proto qw/unsigned int vpx_sad32x16/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride"; specialize qw/vpx_sad32x16 neon avx2 msa sse2 vsx mmi/; add_proto qw/unsigned int vpx_sad16x32/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride"; specialize qw/vpx_sad16x32 neon msa sse2 vsx mmi/; add_proto qw/unsigned int vpx_sad16x16/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride"; -specialize qw/vpx_sad16x16 neon msa sse2 vsx mmi/; +specialize qw/vpx_sad16x16 neon msa sse2 vsx mmi lsx/; add_proto qw/unsigned int vpx_sad16x8/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride"; specialize qw/vpx_sad16x8 neon msa sse2 vsx mmi/; add_proto qw/unsigned int vpx_sad8x16/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride"; @@ -822,31 +823,31 @@ add_proto qw/int vpx_satd/, "const int16_t *coeff, int length"; specialize qw/vpx_satd avx2 sse2 neon msa/; } - add_proto qw/void vpx_int_pro_row/, "int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height"; + add_proto qw/void vpx_int_pro_row/, "int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height"; specialize qw/vpx_int_pro_row sse2 neon msa/; add_proto qw/int16_t vpx_int_pro_col/, "const uint8_t *ref, const int width"; specialize qw/vpx_int_pro_col sse2 neon msa/; add_proto qw/int vpx_vector_var/, "const int16_t *ref, const int16_t *src, const int bwl"; specialize qw/vpx_vector_var neon sse2 msa/; } # CONFIG_VP9_ENCODER add_proto qw/unsigned int vpx_sad64x64_avg/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred"; -specialize qw/vpx_sad64x64_avg neon avx2 msa sse2 vsx mmi/; +specialize qw/vpx_sad64x64_avg neon avx2 msa sse2 vsx mmi lsx/; add_proto qw/unsigned int vpx_sad64x32_avg/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred"; specialize qw/vpx_sad64x32_avg neon avx2 msa sse2 vsx mmi/; add_proto qw/unsigned int vpx_sad32x64_avg/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred"; specialize qw/vpx_sad32x64_avg neon avx2 msa sse2 vsx mmi/; add_proto qw/unsigned int vpx_sad32x32_avg/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred"; -specialize qw/vpx_sad32x32_avg neon avx2 msa sse2 vsx mmi/; +specialize qw/vpx_sad32x32_avg neon avx2 msa sse2 vsx mmi lsx/; add_proto qw/unsigned int vpx_sad32x16_avg/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred"; specialize qw/vpx_sad32x16_avg neon avx2 msa sse2 vsx mmi/; add_proto qw/unsigned int vpx_sad16x32_avg/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred"; @@ -872,87 +873,49 @@ add_proto qw/unsigned int vpx_sad4x4_avg/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred"; specialize qw/vpx_sad4x4_avg neon msa sse2 mmi/; # -# Multi-block SAD, comparing a reference to N blocks 1 pixel apart horizontally -# -# Blocks of 3 -add_proto qw/void vpx_sad16x16x3/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad16x16x3 sse3 ssse3 msa mmi/; - -add_proto qw/void vpx_sad16x8x3/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad16x8x3 sse3 ssse3 msa mmi/; - -add_proto qw/void vpx_sad8x16x3/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad8x16x3 sse3 msa mmi/; - -add_proto qw/void vpx_sad8x8x3/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad8x8x3 sse3 msa mmi/; - -add_proto qw/void vpx_sad4x4x3/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad4x4x3 sse3 msa mmi/; - -# Blocks of 8 -add_proto qw/void vpx_sad32x32x8/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad32x32x8 avx2/; - -add_proto qw/void vpx_sad16x16x8/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad16x16x8 sse4_1 msa mmi/; - -add_proto qw/void vpx_sad16x8x8/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad16x8x8 sse4_1 msa mmi/; - -add_proto qw/void vpx_sad8x16x8/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad8x16x8 sse4_1 msa mmi/; - -add_proto qw/void vpx_sad8x8x8/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad8x8x8 sse4_1 msa mmi/; - -add_proto qw/void vpx_sad4x4x8/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad4x4x8 sse4_1 msa mmi/; - -# # Multi-block SAD, comparing a reference to N independent blocks # -add_proto qw/void vpx_sad64x64x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad64x64x4d avx512 avx2 neon msa sse2 vsx mmi/; +add_proto qw/void vpx_sad64x64x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]"; +specialize qw/vpx_sad64x64x4d avx512 avx2 neon msa sse2 vsx mmi lsx/; -add_proto qw/void vpx_sad64x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad64x32x4d neon msa sse2 vsx mmi/; +add_proto qw/void vpx_sad64x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]"; +specialize qw/vpx_sad64x32x4d neon msa sse2 vsx mmi lsx/; -add_proto qw/void vpx_sad32x64x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad32x64x4d neon msa sse2 vsx mmi/; +add_proto qw/void vpx_sad32x64x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]"; +specialize qw/vpx_sad32x64x4d neon msa sse2 vsx mmi lsx/; -add_proto qw/void vpx_sad32x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad32x32x4d avx2 neon msa sse2 vsx mmi/; +add_proto qw/void vpx_sad32x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]"; +specialize qw/vpx_sad32x32x4d avx2 neon msa sse2 vsx mmi lsx/; -add_proto qw/void vpx_sad32x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array"; +add_proto qw/void vpx_sad32x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_sad32x16x4d neon msa sse2 vsx mmi/; -add_proto qw/void vpx_sad16x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array"; +add_proto qw/void vpx_sad16x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_sad16x32x4d neon msa sse2 vsx mmi/; -add_proto qw/void vpx_sad16x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad16x16x4d neon msa sse2 vsx mmi/; +add_proto qw/void vpx_sad16x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]"; +specialize qw/vpx_sad16x16x4d neon msa sse2 vsx mmi lsx/; -add_proto qw/void vpx_sad16x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array"; +add_proto qw/void vpx_sad16x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_sad16x8x4d neon msa sse2 vsx mmi/; -add_proto qw/void vpx_sad8x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array"; +add_proto qw/void vpx_sad8x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_sad8x16x4d neon msa sse2 mmi/; -add_proto qw/void vpx_sad8x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad8x8x4d neon msa sse2 mmi/; +add_proto qw/void vpx_sad8x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]"; +specialize qw/vpx_sad8x8x4d neon msa sse2 mmi lsx/; -add_proto qw/void vpx_sad8x4x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array"; +add_proto qw/void vpx_sad8x4x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_sad8x4x4d neon msa sse2 mmi/; -add_proto qw/void vpx_sad4x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array"; +add_proto qw/void vpx_sad4x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_sad4x8x4d neon msa sse2 mmi/; -add_proto qw/void vpx_sad4x4x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array"; +add_proto qw/void vpx_sad4x4x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_sad4x4x4d neon msa sse2 mmi/; add_proto qw/uint64_t vpx_sum_squares_2d_i16/, "const int16_t *src, int stride, int size"; specialize qw/vpx_sum_squares_2d_i16 neon sse2 msa/; @@ -1062,47 +1025,47 @@ add_proto qw/unsigned int vpx_highbd_sad4x4_avg/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred"; # # Multi-block SAD, comparing a reference to N independent blocks # - add_proto qw/void vpx_highbd_sad64x64x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array"; + add_proto qw/void vpx_highbd_sad64x64x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_highbd_sad64x64x4d sse2/; - add_proto qw/void vpx_highbd_sad64x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array"; + add_proto qw/void vpx_highbd_sad64x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_highbd_sad64x32x4d sse2/; - add_proto qw/void vpx_highbd_sad32x64x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array"; + add_proto qw/void vpx_highbd_sad32x64x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_highbd_sad32x64x4d sse2/; - add_proto qw/void vpx_highbd_sad32x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array"; + add_proto qw/void vpx_highbd_sad32x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_highbd_sad32x32x4d sse2/; - add_proto qw/void vpx_highbd_sad32x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array"; + add_proto qw/void vpx_highbd_sad32x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_highbd_sad32x16x4d sse2/; - add_proto qw/void vpx_highbd_sad16x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array"; + add_proto qw/void vpx_highbd_sad16x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_highbd_sad16x32x4d sse2/; - add_proto qw/void vpx_highbd_sad16x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array"; + add_proto qw/void vpx_highbd_sad16x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_highbd_sad16x16x4d sse2/; - add_proto qw/void vpx_highbd_sad16x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array"; + add_proto qw/void vpx_highbd_sad16x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_highbd_sad16x8x4d sse2/; - add_proto qw/void vpx_highbd_sad8x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array"; + add_proto qw/void vpx_highbd_sad8x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_highbd_sad8x16x4d sse2/; - add_proto qw/void vpx_highbd_sad8x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array"; + add_proto qw/void vpx_highbd_sad8x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_highbd_sad8x8x4d sse2/; - add_proto qw/void vpx_highbd_sad8x4x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array"; + add_proto qw/void vpx_highbd_sad8x4x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_highbd_sad8x4x4d sse2/; - add_proto qw/void vpx_highbd_sad4x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array"; + add_proto qw/void vpx_highbd_sad4x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_highbd_sad4x8x4d sse2/; - add_proto qw/void vpx_highbd_sad4x4x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array"; + add_proto qw/void vpx_highbd_sad4x4x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_highbd_sad4x4x4d sse2/; # # Structured Similarity (SSIM) # @@ -1116,29 +1079,29 @@ # # Variance # add_proto qw/unsigned int vpx_variance64x64/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse"; - specialize qw/vpx_variance64x64 sse2 avx2 neon msa mmi vsx/; + specialize qw/vpx_variance64x64 sse2 avx2 neon msa mmi vsx lsx/; add_proto qw/unsigned int vpx_variance64x32/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse"; specialize qw/vpx_variance64x32 sse2 avx2 neon msa mmi vsx/; add_proto qw/unsigned int vpx_variance32x64/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse"; specialize qw/vpx_variance32x64 sse2 avx2 neon msa mmi vsx/; add_proto qw/unsigned int vpx_variance32x32/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse"; - specialize qw/vpx_variance32x32 sse2 avx2 neon msa mmi vsx/; + specialize qw/vpx_variance32x32 sse2 avx2 neon msa mmi vsx lsx/; add_proto qw/unsigned int vpx_variance32x16/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse"; specialize qw/vpx_variance32x16 sse2 avx2 neon msa mmi vsx/; add_proto qw/unsigned int vpx_variance16x32/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse"; specialize qw/vpx_variance16x32 sse2 avx2 neon msa mmi vsx/; add_proto qw/unsigned int vpx_variance16x16/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse"; - specialize qw/vpx_variance16x16 sse2 avx2 neon msa mmi vsx/; + specialize qw/vpx_variance16x16 sse2 avx2 neon msa mmi vsx lsx/; add_proto qw/unsigned int vpx_variance16x8/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse"; specialize qw/vpx_variance16x8 sse2 avx2 neon msa mmi vsx/; add_proto qw/unsigned int vpx_variance8x16/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse"; @@ -1158,11 +1121,11 @@ # # Specialty Variance # add_proto qw/void vpx_get16x16var/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse, int *sum"; - specialize qw/vpx_get16x16var sse2 avx2 neon msa vsx/; + specialize qw/vpx_get16x16var sse2 avx2 neon msa vsx lsx/; add_proto qw/void vpx_get8x8var/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse, int *sum"; specialize qw/vpx_get8x8var sse2 neon msa vsx/; add_proto qw/unsigned int vpx_mse16x16/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse"; @@ -1182,11 +1145,11 @@ add_proto qw/unsigned int vpx_get4x4sse_cs/, "const unsigned char *src_ptr, int src_stride, const unsigned char *ref_ptr, int ref_stride"; specialize qw/vpx_get4x4sse_cs neon msa vsx/; add_proto qw/void vpx_comp_avg_pred/, "uint8_t *comp_pred, const uint8_t *pred, int width, int height, const uint8_t *ref, int ref_stride"; - specialize qw/vpx_comp_avg_pred neon sse2 vsx/; + specialize qw/vpx_comp_avg_pred neon sse2 vsx lsx/; # # Subpixel Variance # add_proto qw/uint32_t vpx_sub_pixel_variance64x64/, "const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse"; @@ -1197,11 +1160,11 @@ add_proto qw/uint32_t vpx_sub_pixel_variance32x64/, "const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse"; specialize qw/vpx_sub_pixel_variance32x64 neon msa mmi sse2 ssse3/; add_proto qw/uint32_t vpx_sub_pixel_variance32x32/, "const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse"; - specialize qw/vpx_sub_pixel_variance32x32 avx2 neon msa mmi sse2 ssse3/; + specialize qw/vpx_sub_pixel_variance32x32 avx2 neon msa mmi sse2 ssse3 lsx/; add_proto qw/uint32_t vpx_sub_pixel_variance32x16/, "const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse"; specialize qw/vpx_sub_pixel_variance32x16 neon msa mmi sse2 ssse3/; add_proto qw/uint32_t vpx_sub_pixel_variance16x32/, "const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse"; diff --git a/media/libvpx/libvpx/vpx_dsp/x86/avg_intrin_sse2.c b/media/libvpx/libvpx/vpx_dsp/x86/avg_intrin_sse2.c --- a/media/libvpx/libvpx/vpx_dsp/x86/avg_intrin_sse2.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/avg_intrin_sse2.c @@ -462,11 +462,11 @@ } return _mm_cvtsi128_si32(accum); } -void vpx_int_pro_row_sse2(int16_t *hbuf, const uint8_t *ref, +void vpx_int_pro_row_sse2(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height) { int idx; __m128i zero = _mm_setzero_si128(); __m128i src_line = _mm_loadu_si128((const __m128i *)ref); __m128i s0 = _mm_unpacklo_epi8(src_line, zero); diff --git a/media/libvpx/libvpx/vpx_dsp/x86/avg_pred_sse2.c b/media/libvpx/libvpx/vpx_dsp/x86/avg_pred_sse2.c --- a/media/libvpx/libvpx/vpx_dsp/x86/avg_pred_sse2.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/avg_pred_sse2.c @@ -44,13 +44,13 @@ __m128i avg; if (width == ref_stride) { r = _mm_loadu_si128((const __m128i *)ref); ref += 16; } else if (width == 4) { - r = _mm_set_epi32(loadu_uint32(ref + 3 * ref_stride), - loadu_uint32(ref + 2 * ref_stride), - loadu_uint32(ref + ref_stride), loadu_uint32(ref)); + r = _mm_set_epi32(loadu_int32(ref + 3 * ref_stride), + loadu_int32(ref + 2 * ref_stride), + loadu_int32(ref + ref_stride), loadu_int32(ref)); ref += 4 * ref_stride; } else { const __m128i r_0 = _mm_loadl_epi64((const __m128i *)ref); assert(width == 8); diff --git a/media/libvpx/libvpx/vpx_dsp/x86/highbd_convolve_avx2.c b/media/libvpx/libvpx/vpx_dsp/x86/highbd_convolve_avx2.c --- a/media/libvpx/libvpx/vpx_dsp/x86/highbd_convolve_avx2.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/highbd_convolve_avx2.c @@ -1463,14 +1463,14 @@ #define vpx_highbd_filter_block1d4_v4_avg_avx2 \ vpx_highbd_filter_block1d4_v8_avg_avx2 #define vpx_highbd_filter_block1d4_h4_avg_avx2 \ vpx_highbd_filter_block1d4_h8_avg_avx2 -HIGH_FUN_CONV_1D(horiz, x0_q4, x_step_q4, h, src, , avx2, 0); +HIGH_FUN_CONV_1D(horiz, x0_q4, x_step_q4, h, src, , avx2, 0) HIGH_FUN_CONV_1D(vert, y0_q4, y_step_q4, v, - src - src_stride * (num_taps / 2 - 1), , avx2, 0); -HIGH_FUN_CONV_2D(, avx2, 0); + src - src_stride * (num_taps / 2 - 1), , avx2, 0) +HIGH_FUN_CONV_2D(, avx2, 0) // From vpx_dsp/x86/vpx_high_subpixel_8t_sse2.asm. highbd_filter8_1dfunction vpx_highbd_filter_block1d4_h8_avg_sse2; highbd_filter8_1dfunction vpx_highbd_filter_block1d4_v8_avg_sse2; @@ -1485,11 +1485,11 @@ #define vpx_highbd_filter_block1d4_v8_avg_avx2 \ vpx_highbd_filter_block1d4_v8_avg_sse2 #define vpx_highbd_filter_block1d4_v2_avg_avx2 \ vpx_highbd_filter_block1d4_v2_avg_sse2 -HIGH_FUN_CONV_1D(avg_horiz, x0_q4, x_step_q4, h, src, avg_, avx2, 1); +HIGH_FUN_CONV_1D(avg_horiz, x0_q4, x_step_q4, h, src, avg_, avx2, 1) HIGH_FUN_CONV_1D(avg_vert, y0_q4, y_step_q4, v, - src - src_stride * (num_taps / 2 - 1), avg_, avx2, 1); -HIGH_FUN_CONV_2D(avg_, avx2, 1); + src - src_stride * (num_taps / 2 - 1), avg_, avx2, 1) +HIGH_FUN_CONV_2D(avg_, avx2, 1) #undef HIGHBD_FUNC diff --git a/media/libvpx/libvpx/vpx_dsp/x86/highbd_quantize_intrin_sse2.c b/media/libvpx/libvpx/vpx_dsp/x86/highbd_quantize_intrin_sse2.c --- a/media/libvpx/libvpx/vpx_dsp/x86/highbd_quantize_intrin_sse2.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/highbd_quantize_intrin_sse2.c @@ -16,11 +16,11 @@ #include "vpx_mem/vpx_mem.h" #include "vpx_ports/mem.h" #if CONFIG_VP9_HIGHBITDEPTH void vpx_highbd_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t count, - int skip_block, const int16_t *zbin_ptr, + const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, @@ -37,12 +37,10 @@ nzbins[1] = _mm_setzero_si128(); nzbins[0] = _mm_sub_epi32(nzbins[0], zbins[0]); nzbins[1] = _mm_sub_epi32(nzbins[1], zbins[1]); (void)scan; - (void)skip_block; - assert(!skip_block); memset(qcoeff_ptr, 0, count * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, count * sizeof(*dqcoeff_ptr)); // Pre-scan pass @@ -92,12 +90,12 @@ } *eob_ptr = eob_i + 1; } void vpx_highbd_quantize_b_32x32_sse2( - const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, - const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, + const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan) { __m128i zbins[2]; __m128i nzbins[2]; @@ -105,12 +103,10 @@ int idx_arr[1024]; int i, eob = -1; const int zbin0_tmp = ROUND_POWER_OF_TWO(zbin_ptr[0], 1); const int zbin1_tmp = ROUND_POWER_OF_TWO(zbin_ptr[1], 1); (void)scan; - (void)skip_block; - assert(!skip_block); zbins[0] = _mm_set_epi32(zbin1_tmp, zbin1_tmp, zbin1_tmp, zbin0_tmp); zbins[1] = _mm_set1_epi32(zbin1_tmp); nzbins[0] = _mm_setzero_si128(); diff --git a/media/libvpx/libvpx/vpx_dsp/x86/highbd_variance_sse2.c b/media/libvpx/libvpx/vpx_dsp/x86/highbd_variance_sse2.c --- a/media/libvpx/libvpx/vpx_dsp/x86/highbd_variance_sse2.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/highbd_variance_sse2.c @@ -119,12 +119,12 @@ sum); \ *sum = ROUND_POWER_OF_TWO(*sum, 4); \ *sse = ROUND_POWER_OF_TWO(*sse, 8); \ } -HIGH_GET_VAR(16); -HIGH_GET_VAR(8); +HIGH_GET_VAR(16) +HIGH_GET_VAR(8) #undef HIGH_GET_VAR #define VAR_FN(w, h, block_size, shift) \ uint32_t vpx_highbd_8_variance##w##x##h##_sse2( \ @@ -165,20 +165,20 @@ vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ var = (int64_t)(*sse) - (((int64_t)sum * sum) >> (shift)); \ return (var >= 0) ? (uint32_t)var : 0; \ } -VAR_FN(64, 64, 16, 12); -VAR_FN(64, 32, 16, 11); -VAR_FN(32, 64, 16, 11); -VAR_FN(32, 32, 16, 10); -VAR_FN(32, 16, 16, 9); -VAR_FN(16, 32, 16, 9); -VAR_FN(16, 16, 16, 8); -VAR_FN(16, 8, 8, 7); -VAR_FN(8, 16, 8, 7); -VAR_FN(8, 8, 8, 6); +VAR_FN(64, 64, 16, 12) +VAR_FN(64, 32, 16, 11) +VAR_FN(32, 64, 16, 11) +VAR_FN(32, 32, 16, 10) +VAR_FN(32, 16, 16, 9) +VAR_FN(16, 32, 16, 9) +VAR_FN(16, 16, 16, 8) +VAR_FN(16, 8, 8, 7) +VAR_FN(8, 16, 8, 7) +VAR_FN(8, 8, 8, 6) #undef VAR_FN unsigned int vpx_highbd_8_mse16x16_sse2(const uint8_t *src8, int src_stride, const uint8_t *ref8, int ref_stride, @@ -253,14 +253,14 @@ int vpx_highbd_sub_pixel_variance##w##xh_##opt( \ const uint16_t *src, ptrdiff_t src_stride, int x_offset, int y_offset, \ const uint16_t *ref, ptrdiff_t ref_stride, int height, \ unsigned int *sse, void *unused0, void *unused); #define DECLS(opt) \ - DECL(8, opt); \ + DECL(8, opt) \ DECL(16, opt) -DECLS(sse2); +DECLS(sse2) #undef DECLS #undef DECL #define FN(w, h, wf, wlog2, hlog2, opt, cast) \ @@ -381,24 +381,24 @@ *sse_ptr = sse; \ var = (int64_t)(sse) - ((cast se * se) >> (wlog2 + hlog2)); \ return (var >= 0) ? (uint32_t)var : 0; \ } -#define FNS(opt) \ - FN(64, 64, 16, 6, 6, opt, (int64_t)); \ - FN(64, 32, 16, 6, 5, opt, (int64_t)); \ - FN(32, 64, 16, 5, 6, opt, (int64_t)); \ - FN(32, 32, 16, 5, 5, opt, (int64_t)); \ - FN(32, 16, 16, 5, 4, opt, (int64_t)); \ - FN(16, 32, 16, 4, 5, opt, (int64_t)); \ - FN(16, 16, 16, 4, 4, opt, (int64_t)); \ - FN(16, 8, 16, 4, 3, opt, (int64_t)); \ - FN(8, 16, 8, 3, 4, opt, (int64_t)); \ - FN(8, 8, 8, 3, 3, opt, (int64_t)); \ - FN(8, 4, 8, 3, 2, opt, (int64_t)); +#define FNS(opt) \ + FN(64, 64, 16, 6, 6, opt, (int64_t)) \ + FN(64, 32, 16, 6, 5, opt, (int64_t)) \ + FN(32, 64, 16, 5, 6, opt, (int64_t)) \ + FN(32, 32, 16, 5, 5, opt, (int64_t)) \ + FN(32, 16, 16, 5, 4, opt, (int64_t)) \ + FN(16, 32, 16, 4, 5, opt, (int64_t)) \ + FN(16, 16, 16, 4, 4, opt, (int64_t)) \ + FN(16, 8, 16, 4, 3, opt, (int64_t)) \ + FN(8, 16, 8, 3, 4, opt, (int64_t)) \ + FN(8, 8, 8, 3, 3, opt, (int64_t)) \ + FN(8, 4, 8, 3, 2, opt, (int64_t)) -FNS(sse2); +FNS(sse2) #undef FNS #undef FN // The 2 unused parameters are place holders for PIC enabled build. @@ -410,11 +410,11 @@ void *unused); #define DECLS(opt1) \ DECL(16, opt1) \ DECL(8, opt1) -DECLS(sse2); +DECLS(sse2) #undef DECL #undef DECLS #define FN(w, h, wf, wlog2, hlog2, opt, cast) \ uint32_t vpx_highbd_8_sub_pixel_avg_variance##w##x##h##_##opt( \ @@ -540,22 +540,22 @@ *sse_ptr = sse; \ var = (int64_t)(sse) - ((cast se * se) >> (wlog2 + hlog2)); \ return (var >= 0) ? (uint32_t)var : 0; \ } -#define FNS(opt1) \ - FN(64, 64, 16, 6, 6, opt1, (int64_t)); \ - FN(64, 32, 16, 6, 5, opt1, (int64_t)); \ - FN(32, 64, 16, 5, 6, opt1, (int64_t)); \ - FN(32, 32, 16, 5, 5, opt1, (int64_t)); \ - FN(32, 16, 16, 5, 4, opt1, (int64_t)); \ - FN(16, 32, 16, 4, 5, opt1, (int64_t)); \ - FN(16, 16, 16, 4, 4, opt1, (int64_t)); \ - FN(16, 8, 16, 4, 3, opt1, (int64_t)); \ - FN(8, 16, 8, 4, 3, opt1, (int64_t)); \ - FN(8, 8, 8, 3, 3, opt1, (int64_t)); \ - FN(8, 4, 8, 3, 2, opt1, (int64_t)); +#define FNS(opt1) \ + FN(64, 64, 16, 6, 6, opt1, (int64_t)) \ + FN(64, 32, 16, 6, 5, opt1, (int64_t)) \ + FN(32, 64, 16, 5, 6, opt1, (int64_t)) \ + FN(32, 32, 16, 5, 5, opt1, (int64_t)) \ + FN(32, 16, 16, 5, 4, opt1, (int64_t)) \ + FN(16, 32, 16, 4, 5, opt1, (int64_t)) \ + FN(16, 16, 16, 4, 4, opt1, (int64_t)) \ + FN(16, 8, 16, 4, 3, opt1, (int64_t)) \ + FN(8, 16, 8, 4, 3, opt1, (int64_t)) \ + FN(8, 8, 8, 3, 3, opt1, (int64_t)) \ + FN(8, 4, 8, 3, 2, opt1, (int64_t)) -FNS(sse2); +FNS(sse2) #undef FNS #undef FN diff --git a/media/libvpx/libvpx/vpx_dsp/x86/loopfilter_intrin_sse2.c b/media/libvpx/libvpx/vpx_dsp/x86/loopfilter_intrin_sse2.c --- a/media/libvpx/libvpx/vpx_dsp/x86/loopfilter_intrin_sse2.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/loopfilter_intrin_sse2.c @@ -209,25 +209,25 @@ // 04 14 24 34 05 15 25 35 06 16 26 36 07 17 27 37 qs1qs0 = _mm_unpackhi_epi8(ps1ps0, x0); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33 ps1ps0 = _mm_unpacklo_epi8(ps1ps0, x0); - storeu_uint32(s + 0 * pitch - 2, _mm_cvtsi128_si32(ps1ps0)); + storeu_int32(s + 0 * pitch - 2, _mm_cvtsi128_si32(ps1ps0)); ps1ps0 = _mm_srli_si128(ps1ps0, 4); - storeu_uint32(s + 1 * pitch - 2, _mm_cvtsi128_si32(ps1ps0)); + storeu_int32(s + 1 * pitch - 2, _mm_cvtsi128_si32(ps1ps0)); ps1ps0 = _mm_srli_si128(ps1ps0, 4); - storeu_uint32(s + 2 * pitch - 2, _mm_cvtsi128_si32(ps1ps0)); + storeu_int32(s + 2 * pitch - 2, _mm_cvtsi128_si32(ps1ps0)); ps1ps0 = _mm_srli_si128(ps1ps0, 4); - storeu_uint32(s + 3 * pitch - 2, _mm_cvtsi128_si32(ps1ps0)); + storeu_int32(s + 3 * pitch - 2, _mm_cvtsi128_si32(ps1ps0)); - storeu_uint32(s + 4 * pitch - 2, _mm_cvtsi128_si32(qs1qs0)); + storeu_int32(s + 4 * pitch - 2, _mm_cvtsi128_si32(qs1qs0)); qs1qs0 = _mm_srli_si128(qs1qs0, 4); - storeu_uint32(s + 5 * pitch - 2, _mm_cvtsi128_si32(qs1qs0)); + storeu_int32(s + 5 * pitch - 2, _mm_cvtsi128_si32(qs1qs0)); qs1qs0 = _mm_srli_si128(qs1qs0, 4); - storeu_uint32(s + 6 * pitch - 2, _mm_cvtsi128_si32(qs1qs0)); + storeu_int32(s + 6 * pitch - 2, _mm_cvtsi128_si32(qs1qs0)); qs1qs0 = _mm_srli_si128(qs1qs0, 4); - storeu_uint32(s + 7 * pitch - 2, _mm_cvtsi128_si32(qs1qs0)); + storeu_int32(s + 7 * pitch - 2, _mm_cvtsi128_si32(qs1qs0)); } void vpx_lpf_horizontal_16_sse2(unsigned char *s, int pitch, const unsigned char *blimit, const unsigned char *limit, diff --git a/media/libvpx/libvpx/vpx_dsp/x86/mem_sse2.h b/media/libvpx/libvpx/vpx_dsp/x86/mem_sse2.h --- a/media/libvpx/libvpx/vpx_dsp/x86/mem_sse2.h +++ b/media/libvpx/libvpx/vpx_dsp/x86/mem_sse2.h @@ -14,16 +14,16 @@ #include // SSE2 #include #include "./vpx_config.h" -static INLINE void storeu_uint32(void *dst, uint32_t v) { +static INLINE void storeu_int32(void *dst, int32_t v) { memcpy(dst, &v, sizeof(v)); } -static INLINE uint32_t loadu_uint32(const void *src) { - uint32_t v; +static INLINE int32_t loadu_int32(const void *src) { + int32_t v; memcpy(&v, src, sizeof(v)); return v; } static INLINE __m128i load_unaligned_u32(const void *a) { diff --git a/media/libvpx/libvpx/vpx_dsp/x86/quantize_avx.c b/media/libvpx/libvpx/vpx_dsp/x86/quantize_avx.c --- a/media/libvpx/libvpx/vpx_dsp/x86/quantize_avx.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/quantize_avx.c @@ -19,12 +19,12 @@ #include "vpx_dsp/x86/bitdepth_conversion_sse2.h" #include "vpx_dsp/x86/quantize_sse2.h" #include "vpx_dsp/x86/quantize_ssse3.h" void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *zbin_ptr, - const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *zbin_ptr, const int16_t *round_ptr, + const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan) { const __m128i zero = _mm_setzero_si128(); @@ -37,12 +37,10 @@ __m128i cmp_mask0, cmp_mask1; __m128i all_zero; __m128i eob = zero, eob0; (void)scan; - (void)skip_block; - assert(!skip_block); *eob_ptr = 0; load_b_values(zbin_ptr, &zbin, round_ptr, &round, quant_ptr, &quant, dequant_ptr, &dequant, quant_shift_ptr, &shift); @@ -143,12 +141,11 @@ *eob_ptr = accumulate_eob(eob); } void vpx_quantize_b_32x32_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *zbin_ptr, - const int16_t *round_ptr, + const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan) { @@ -164,12 +161,10 @@ __m128i all_zero; __m128i eob = zero, eob0; (void)scan; (void)n_coeffs; - (void)skip_block; - assert(!skip_block); // Setup global values. // The 32x32 halves zbin and round. zbin = _mm_load_si128((const __m128i *)zbin_ptr); // Shift with rounding. diff --git a/media/libvpx/libvpx/vpx_dsp/x86/quantize_sse2.c b/media/libvpx/libvpx/vpx_dsp/x86/quantize_sse2.c --- a/media/libvpx/libvpx/vpx_dsp/x86/quantize_sse2.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/quantize_sse2.c @@ -16,12 +16,12 @@ #include "vpx/vpx_integer.h" #include "vpx_dsp/x86/bitdepth_conversion_sse2.h" #include "vpx_dsp/x86/quantize_sse2.h" void vpx_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *zbin_ptr, - const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *zbin_ptr, const int16_t *round_ptr, + const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan) { const __m128i zero = _mm_setzero_si128(); @@ -32,12 +32,10 @@ __m128i qcoeff0, qcoeff1; __m128i cmp_mask0, cmp_mask1; __m128i eob, eob0; (void)scan; - (void)skip_block; - assert(!skip_block); // Setup global values. load_b_values(zbin_ptr, &zbin, round_ptr, &round, quant_ptr, &quant, dequant_ptr, &dequant, quant_shift_ptr, &shift); diff --git a/media/libvpx/libvpx/vpx_dsp/x86/quantize_ssse3.c b/media/libvpx/libvpx/vpx_dsp/x86/quantize_ssse3.c --- a/media/libvpx/libvpx/vpx_dsp/x86/quantize_ssse3.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/quantize_ssse3.c @@ -16,12 +16,12 @@ #include "vpx_dsp/x86/bitdepth_conversion_sse2.h" #include "vpx_dsp/x86/quantize_sse2.h" #include "vpx_dsp/x86/quantize_ssse3.h" void vpx_quantize_b_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *zbin_ptr, - const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *zbin_ptr, const int16_t *round_ptr, + const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan) { const __m128i zero = _mm_setzero_si128(); @@ -32,12 +32,10 @@ __m128i qcoeff0, qcoeff1; __m128i cmp_mask0, cmp_mask1; __m128i eob, eob0; (void)scan; - (void)skip_block; - assert(!skip_block); load_b_values(zbin_ptr, &zbin, round_ptr, &round, quant_ptr, &quant, dequant_ptr, &dequant, quant_shift_ptr, &shift); // Do DC and first 15 AC. @@ -109,11 +107,11 @@ *eob_ptr = accumulate_eob(eob); } void vpx_quantize_b_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *zbin_ptr, + const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, @@ -129,12 +127,10 @@ __m128i all_zero; __m128i eob = zero, eob0; (void)scan; (void)n_coeffs; - (void)skip_block; - assert(!skip_block); // Setup global values. // The 32x32 halves zbin and round. zbin = _mm_load_si128((const __m128i *)zbin_ptr); // Shift with rounding. diff --git a/media/libvpx/libvpx/vpx_dsp/x86/sad4d_avx2.c b/media/libvpx/libvpx/vpx_dsp/x86/sad4d_avx2.c --- a/media/libvpx/libvpx/vpx_dsp/x86/sad4d_avx2.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/sad4d_avx2.c @@ -9,12 +9,16 @@ */ #include // AVX2 #include "./vpx_dsp_rtcd.h" #include "vpx/vpx_integer.h" +// Note with sums[4] some versions of Visual Studio may fail due to parameter +// alignment, though the functions should be equivalent: +// error C2719: 'sums': formal parameter with requested alignment of 32 won't be +// aligned static INLINE void calc_final_4(const __m256i *const sums /*[4]*/, - uint32_t *sad_array) { + uint32_t sad_array[4]) { const __m256i t0 = _mm256_hadd_epi32(sums[0], sums[1]); const __m256i t1 = _mm256_hadd_epi32(sums[2], sums[3]); const __m256i t2 = _mm256_hadd_epi32(t0, t1); const __m128i sum = _mm_add_epi32(_mm256_castsi256_si128(t2), _mm256_extractf128_si256(t2, 1)); @@ -67,67 +71,10 @@ } calc_final_4(sums, sad_array); } -void vpx_sad32x32x8_avx2(const uint8_t *src_ptr, int src_stride, - const uint8_t *ref_ptr, int ref_stride, - uint32_t *sad_array) { - int i; - __m256i sums[8]; - - sums[0] = _mm256_setzero_si256(); - sums[1] = _mm256_setzero_si256(); - sums[2] = _mm256_setzero_si256(); - sums[3] = _mm256_setzero_si256(); - sums[4] = _mm256_setzero_si256(); - sums[5] = _mm256_setzero_si256(); - sums[6] = _mm256_setzero_si256(); - sums[7] = _mm256_setzero_si256(); - - for (i = 0; i < 32; i++) { - __m256i r[8]; - - // load src and all ref[] - const __m256i s = _mm256_load_si256((const __m256i *)src_ptr); - r[0] = _mm256_loadu_si256((const __m256i *)&ref_ptr[0]); - r[1] = _mm256_loadu_si256((const __m256i *)&ref_ptr[1]); - r[2] = _mm256_loadu_si256((const __m256i *)&ref_ptr[2]); - r[3] = _mm256_loadu_si256((const __m256i *)&ref_ptr[3]); - r[4] = _mm256_loadu_si256((const __m256i *)&ref_ptr[4]); - r[5] = _mm256_loadu_si256((const __m256i *)&ref_ptr[5]); - r[6] = _mm256_loadu_si256((const __m256i *)&ref_ptr[6]); - r[7] = _mm256_loadu_si256((const __m256i *)&ref_ptr[7]); - - // sum of the absolute differences between every ref[] to src - r[0] = _mm256_sad_epu8(r[0], s); - r[1] = _mm256_sad_epu8(r[1], s); - r[2] = _mm256_sad_epu8(r[2], s); - r[3] = _mm256_sad_epu8(r[3], s); - r[4] = _mm256_sad_epu8(r[4], s); - r[5] = _mm256_sad_epu8(r[5], s); - r[6] = _mm256_sad_epu8(r[6], s); - r[7] = _mm256_sad_epu8(r[7], s); - - // sum every ref[] - sums[0] = _mm256_add_epi32(sums[0], r[0]); - sums[1] = _mm256_add_epi32(sums[1], r[1]); - sums[2] = _mm256_add_epi32(sums[2], r[2]); - sums[3] = _mm256_add_epi32(sums[3], r[3]); - sums[4] = _mm256_add_epi32(sums[4], r[4]); - sums[5] = _mm256_add_epi32(sums[5], r[5]); - sums[6] = _mm256_add_epi32(sums[6], r[6]); - sums[7] = _mm256_add_epi32(sums[7], r[7]); - - src_ptr += src_stride; - ref_ptr += ref_stride; - } - - calc_final_4(sums, sad_array); - calc_final_4(sums + 4, sad_array + 4); -} - void vpx_sad64x64x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, uint32_t sad_array[4]) { __m256i sums[4]; int i; diff --git a/media/libvpx/libvpx/vpx_dsp/x86/sad4d_avx512.c b/media/libvpx/libvpx/vpx_dsp/x86/sad4d_avx512.c --- a/media/libvpx/libvpx/vpx_dsp/x86/sad4d_avx512.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/sad4d_avx512.c @@ -11,11 +11,11 @@ #include "./vpx_dsp_rtcd.h" #include "vpx/vpx_integer.h" void vpx_sad64x64x4d_avx512(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t res[4]) { + uint32_t sad_array[4]) { __m512i src_reg, ref0_reg, ref1_reg, ref2_reg, ref3_reg; __m512i sum_ref0, sum_ref1, sum_ref2, sum_ref3; __m512i sum_mlow, sum_mhigh; int i; const uint8_t *ref0, *ref1, *ref2, *ref3; @@ -76,8 +76,8 @@ sum256 = _mm256_add_epi32(_mm512_castsi512_si256(sum_mlow), _mm512_extracti32x8_epi32(sum_mlow, 1)); sum128 = _mm_add_epi32(_mm256_castsi256_si128(sum256), _mm256_extractf128_si256(sum256, 1)); - _mm_storeu_si128((__m128i *)(res), sum128); + _mm_storeu_si128((__m128i *)(sad_array), sum128); } } diff --git a/media/libvpx/libvpx/vpx_dsp/x86/sad_avx2.c b/media/libvpx/libvpx/vpx_dsp/x86/sad_avx2.c --- a/media/libvpx/libvpx/vpx_dsp/x86/sad_avx2.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/sad_avx2.c @@ -69,21 +69,21 @@ sum_sad128 = _mm_add_epi32(_mm256_castsi256_si128(sum_sad), sum_sad128); \ res = _mm_cvtsi128_si32(sum_sad128); \ return res; \ } -#define FSAD64 \ - FSAD64_H(64); \ - FSAD64_H(32); +#define FSAD64 \ + FSAD64_H(64) \ + FSAD64_H(32) -#define FSAD32 \ - FSAD32_H(64); \ - FSAD32_H(32); \ - FSAD32_H(16); +#define FSAD32 \ + FSAD32_H(64) \ + FSAD32_H(32) \ + FSAD32_H(16) -FSAD64; -FSAD32; +FSAD64 +FSAD32 #undef FSAD64 #undef FSAD32 #undef FSAD64_H #undef FSAD32_H @@ -158,21 +158,21 @@ sum_sad128 = _mm_add_epi32(_mm256_castsi256_si128(sum_sad), sum_sad128); \ res = _mm_cvtsi128_si32(sum_sad128); \ return res; \ } -#define FSADAVG64 \ - FSADAVG64_H(64); \ - FSADAVG64_H(32); +#define FSADAVG64 \ + FSADAVG64_H(64) \ + FSADAVG64_H(32) -#define FSADAVG32 \ - FSADAVG32_H(64); \ - FSADAVG32_H(32); \ - FSADAVG32_H(16); +#define FSADAVG32 \ + FSADAVG32_H(64) \ + FSADAVG32_H(32) \ + FSADAVG32_H(16) -FSADAVG64; -FSADAVG32; +FSADAVG64 +FSADAVG32 #undef FSADAVG64 #undef FSADAVG32 #undef FSADAVG64_H #undef FSADAVG32_H diff --git a/media/libvpx/libvpx/vpx_dsp/x86/sad_sse3.asm b/media/libvpx/libvpx/vpx_dsp/x86/sad_sse3.asm deleted file mode 100644 --- a/media/libvpx/libvpx/vpx_dsp/x86/sad_sse3.asm +++ /dev/null @@ -1,376 +0,0 @@ -; -; Copyright (c) 2010 The WebM project authors. All Rights Reserved. -; -; Use of this source code is governed by a BSD-style license -; that can be found in the LICENSE file in the root of the source -; tree. An additional intellectual property rights grant can be found -; in the file PATENTS. All contributing project authors may -; be found in the AUTHORS file in the root of the source tree. -; - -%include "vpx_ports/x86_abi_support.asm" - -%macro STACK_FRAME_CREATE_X3 0 -%if ABI_IS_32BIT - %define src_ptr rsi - %define src_stride rax - %define ref_ptr rdi - %define ref_stride rdx - %define end_ptr rcx - %define ret_var rbx - %define result_ptr arg(4) - %define height dword ptr arg(4) - push rbp - mov rbp, rsp - push rsi - push rdi - push rbx - - mov rsi, arg(0) ; src_ptr - mov rdi, arg(2) ; ref_ptr - - movsxd rax, dword ptr arg(1) ; src_stride - movsxd rdx, dword ptr arg(3) ; ref_stride -%else - %if LIBVPX_YASM_WIN64 - SAVE_XMM 7, u - %define src_ptr rcx - %define src_stride rdx - %define ref_ptr r8 - %define ref_stride r9 - %define end_ptr r10 - %define ret_var r11 - %define result_ptr [rsp+xmm_stack_space+8+4*8] - %define height dword ptr [rsp+xmm_stack_space+8+4*8] - %else - %define src_ptr rdi - %define src_stride rsi - %define ref_ptr rdx - %define ref_stride rcx - %define end_ptr r9 - %define ret_var r10 - %define result_ptr r8 - %define height r8 - %endif -%endif - -%endmacro - -%macro STACK_FRAME_DESTROY_X3 0 - %define src_ptr - %define src_stride - %define ref_ptr - %define ref_stride - %define end_ptr - %define ret_var - %define result_ptr - %define height - -%if ABI_IS_32BIT - pop rbx - pop rdi - pop rsi - pop rbp -%else - %if LIBVPX_YASM_WIN64 - RESTORE_XMM - %endif -%endif - ret -%endmacro - -%macro PROCESS_16X2X3 5 -%if %1==0 - movdqa xmm0, XMMWORD PTR [%2] - lddqu xmm5, XMMWORD PTR [%3] - lddqu xmm6, XMMWORD PTR [%3+1] - lddqu xmm7, XMMWORD PTR [%3+2] - - psadbw xmm5, xmm0 - psadbw xmm6, xmm0 - psadbw xmm7, xmm0 -%else - movdqa xmm0, XMMWORD PTR [%2] - lddqu xmm1, XMMWORD PTR [%3] - lddqu xmm2, XMMWORD PTR [%3+1] - lddqu xmm3, XMMWORD PTR [%3+2] - - psadbw xmm1, xmm0 - psadbw xmm2, xmm0 - psadbw xmm3, xmm0 - - paddw xmm5, xmm1 - paddw xmm6, xmm2 - paddw xmm7, xmm3 -%endif - movdqa xmm0, XMMWORD PTR [%2+%4] - lddqu xmm1, XMMWORD PTR [%3+%5] - lddqu xmm2, XMMWORD PTR [%3+%5+1] - lddqu xmm3, XMMWORD PTR [%3+%5+2] - -%if %1==0 || %1==1 - lea %2, [%2+%4*2] - lea %3, [%3+%5*2] -%endif - - psadbw xmm1, xmm0 - psadbw xmm2, xmm0 - psadbw xmm3, xmm0 - - paddw xmm5, xmm1 - paddw xmm6, xmm2 - paddw xmm7, xmm3 -%endmacro - -%macro PROCESS_8X2X3 5 -%if %1==0 - movq mm0, QWORD PTR [%2] - movq mm5, QWORD PTR [%3] - movq mm6, QWORD PTR [%3+1] - movq mm7, QWORD PTR [%3+2] - - psadbw mm5, mm0 - psadbw mm6, mm0 - psadbw mm7, mm0 -%else - movq mm0, QWORD PTR [%2] - movq mm1, QWORD PTR [%3] - movq mm2, QWORD PTR [%3+1] - movq mm3, QWORD PTR [%3+2] - - psadbw mm1, mm0 - psadbw mm2, mm0 - psadbw mm3, mm0 - - paddw mm5, mm1 - paddw mm6, mm2 - paddw mm7, mm3 -%endif - movq mm0, QWORD PTR [%2+%4] - movq mm1, QWORD PTR [%3+%5] - movq mm2, QWORD PTR [%3+%5+1] - movq mm3, QWORD PTR [%3+%5+2] - -%if %1==0 || %1==1 - lea %2, [%2+%4*2] - lea %3, [%3+%5*2] -%endif - - psadbw mm1, mm0 - psadbw mm2, mm0 - psadbw mm3, mm0 - - paddw mm5, mm1 - paddw mm6, mm2 - paddw mm7, mm3 -%endmacro - -SECTION .text - -;void int vpx_sad16x16x3_sse3( -; unsigned char *src_ptr, -; int src_stride, -; unsigned char *ref_ptr, -; int ref_stride, -; int *results) -globalsym(vpx_sad16x16x3_sse3) -sym(vpx_sad16x16x3_sse3): - - STACK_FRAME_CREATE_X3 - - PROCESS_16X2X3 0, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_16X2X3 2, src_ptr, ref_ptr, src_stride, ref_stride - - mov rcx, result_ptr - - movq xmm0, xmm5 - psrldq xmm5, 8 - - paddw xmm0, xmm5 - movd [rcx], xmm0 -;- - movq xmm0, xmm6 - psrldq xmm6, 8 - - paddw xmm0, xmm6 - movd [rcx+4], xmm0 -;- - movq xmm0, xmm7 - psrldq xmm7, 8 - - paddw xmm0, xmm7 - movd [rcx+8], xmm0 - - STACK_FRAME_DESTROY_X3 - -;void int vpx_sad16x8x3_sse3( -; unsigned char *src_ptr, -; int src_stride, -; unsigned char *ref_ptr, -; int ref_stride, -; int *results) -globalsym(vpx_sad16x8x3_sse3) -sym(vpx_sad16x8x3_sse3): - - STACK_FRAME_CREATE_X3 - - PROCESS_16X2X3 0, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_16X2X3 2, src_ptr, ref_ptr, src_stride, ref_stride - - mov rcx, result_ptr - - movq xmm0, xmm5 - psrldq xmm5, 8 - - paddw xmm0, xmm5 - movd [rcx], xmm0 -;- - movq xmm0, xmm6 - psrldq xmm6, 8 - - paddw xmm0, xmm6 - movd [rcx+4], xmm0 -;- - movq xmm0, xmm7 - psrldq xmm7, 8 - - paddw xmm0, xmm7 - movd [rcx+8], xmm0 - - STACK_FRAME_DESTROY_X3 - -;void int vpx_sad8x16x3_sse3( -; unsigned char *src_ptr, -; int src_stride, -; unsigned char *ref_ptr, -; int ref_stride, -; int *results) -globalsym(vpx_sad8x16x3_sse3) -sym(vpx_sad8x16x3_sse3): - - STACK_FRAME_CREATE_X3 - - PROCESS_8X2X3 0, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_8X2X3 2, src_ptr, ref_ptr, src_stride, ref_stride - - mov rcx, result_ptr - - punpckldq mm5, mm6 - - movq [rcx], mm5 - movd [rcx+8], mm7 - - STACK_FRAME_DESTROY_X3 - -;void int vpx_sad8x8x3_sse3( -; unsigned char *src_ptr, -; int src_stride, -; unsigned char *ref_ptr, -; int ref_stride, -; int *results) -globalsym(vpx_sad8x8x3_sse3) -sym(vpx_sad8x8x3_sse3): - - STACK_FRAME_CREATE_X3 - - PROCESS_8X2X3 0, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_8X2X3 2, src_ptr, ref_ptr, src_stride, ref_stride - - mov rcx, result_ptr - - punpckldq mm5, mm6 - - movq [rcx], mm5 - movd [rcx+8], mm7 - - STACK_FRAME_DESTROY_X3 - -;void int vpx_sad4x4x3_sse3( -; unsigned char *src_ptr, -; int src_stride, -; unsigned char *ref_ptr, -; int ref_stride, -; int *results) -globalsym(vpx_sad4x4x3_sse3) -sym(vpx_sad4x4x3_sse3): - - STACK_FRAME_CREATE_X3 - - movd mm0, DWORD PTR [src_ptr] - movd mm1, DWORD PTR [ref_ptr] - - movd mm2, DWORD PTR [src_ptr+src_stride] - movd mm3, DWORD PTR [ref_ptr+ref_stride] - - punpcklbw mm0, mm2 - punpcklbw mm1, mm3 - - movd mm4, DWORD PTR [ref_ptr+1] - movd mm5, DWORD PTR [ref_ptr+2] - - movd mm2, DWORD PTR [ref_ptr+ref_stride+1] - movd mm3, DWORD PTR [ref_ptr+ref_stride+2] - - psadbw mm1, mm0 - - punpcklbw mm4, mm2 - punpcklbw mm5, mm3 - - psadbw mm4, mm0 - psadbw mm5, mm0 - - lea src_ptr, [src_ptr+src_stride*2] - lea ref_ptr, [ref_ptr+ref_stride*2] - - movd mm0, DWORD PTR [src_ptr] - movd mm2, DWORD PTR [ref_ptr] - - movd mm3, DWORD PTR [src_ptr+src_stride] - movd mm6, DWORD PTR [ref_ptr+ref_stride] - - punpcklbw mm0, mm3 - punpcklbw mm2, mm6 - - movd mm3, DWORD PTR [ref_ptr+1] - movd mm7, DWORD PTR [ref_ptr+2] - - psadbw mm2, mm0 - - paddw mm1, mm2 - - movd mm2, DWORD PTR [ref_ptr+ref_stride+1] - movd mm6, DWORD PTR [ref_ptr+ref_stride+2] - - punpcklbw mm3, mm2 - punpcklbw mm7, mm6 - - psadbw mm3, mm0 - psadbw mm7, mm0 - - paddw mm3, mm4 - paddw mm7, mm5 - - mov rcx, result_ptr - - punpckldq mm1, mm3 - - movq [rcx], mm1 - movd [rcx+8], mm7 - - STACK_FRAME_DESTROY_X3 diff --git a/media/libvpx/libvpx/vpx_dsp/x86/sad_sse4.asm b/media/libvpx/libvpx/vpx_dsp/x86/sad_sse4.asm deleted file mode 100644 --- a/media/libvpx/libvpx/vpx_dsp/x86/sad_sse4.asm +++ /dev/null @@ -1,361 +0,0 @@ -; -; Copyright (c) 2010 The WebM project authors. All Rights Reserved. -; -; Use of this source code is governed by a BSD-style license -; that can be found in the LICENSE file in the root of the source -; tree. An additional intellectual property rights grant can be found -; in the file PATENTS. All contributing project authors may -; be found in the AUTHORS file in the root of the source tree. -; - - -%include "vpx_ports/x86_abi_support.asm" - -%macro PROCESS_16X2X8 1 -%if %1 - movdqa xmm0, XMMWORD PTR [rsi] - movq xmm1, MMWORD PTR [rdi] - movq xmm3, MMWORD PTR [rdi+8] - movq xmm2, MMWORD PTR [rdi+16] - punpcklqdq xmm1, xmm3 - punpcklqdq xmm3, xmm2 - - movdqa xmm2, xmm1 - mpsadbw xmm1, xmm0, 0x0 - mpsadbw xmm2, xmm0, 0x5 - - psrldq xmm0, 8 - - movdqa xmm4, xmm3 - mpsadbw xmm3, xmm0, 0x0 - mpsadbw xmm4, xmm0, 0x5 - - paddw xmm1, xmm2 - paddw xmm1, xmm3 - paddw xmm1, xmm4 -%else - movdqa xmm0, XMMWORD PTR [rsi] - movq xmm5, MMWORD PTR [rdi] - movq xmm3, MMWORD PTR [rdi+8] - movq xmm2, MMWORD PTR [rdi+16] - punpcklqdq xmm5, xmm3 - punpcklqdq xmm3, xmm2 - - movdqa xmm2, xmm5 - mpsadbw xmm5, xmm0, 0x0 - mpsadbw xmm2, xmm0, 0x5 - - psrldq xmm0, 8 - - movdqa xmm4, xmm3 - mpsadbw xmm3, xmm0, 0x0 - mpsadbw xmm4, xmm0, 0x5 - - paddw xmm5, xmm2 - paddw xmm5, xmm3 - paddw xmm5, xmm4 - - paddw xmm1, xmm5 -%endif - movdqa xmm0, XMMWORD PTR [rsi + rax] - movq xmm5, MMWORD PTR [rdi+ rdx] - movq xmm3, MMWORD PTR [rdi+ rdx+8] - movq xmm2, MMWORD PTR [rdi+ rdx+16] - punpcklqdq xmm5, xmm3 - punpcklqdq xmm3, xmm2 - - lea rsi, [rsi+rax*2] - lea rdi, [rdi+rdx*2] - - movdqa xmm2, xmm5 - mpsadbw xmm5, xmm0, 0x0 - mpsadbw xmm2, xmm0, 0x5 - - psrldq xmm0, 8 - movdqa xmm4, xmm3 - mpsadbw xmm3, xmm0, 0x0 - mpsadbw xmm4, xmm0, 0x5 - - paddw xmm5, xmm2 - paddw xmm5, xmm3 - paddw xmm5, xmm4 - - paddw xmm1, xmm5 -%endmacro - -%macro PROCESS_8X2X8 1 -%if %1 - movq xmm0, MMWORD PTR [rsi] - movq xmm1, MMWORD PTR [rdi] - movq xmm3, MMWORD PTR [rdi+8] - punpcklqdq xmm1, xmm3 - - movdqa xmm2, xmm1 - mpsadbw xmm1, xmm0, 0x0 - mpsadbw xmm2, xmm0, 0x5 - paddw xmm1, xmm2 -%else - movq xmm0, MMWORD PTR [rsi] - movq xmm5, MMWORD PTR [rdi] - movq xmm3, MMWORD PTR [rdi+8] - punpcklqdq xmm5, xmm3 - - movdqa xmm2, xmm5 - mpsadbw xmm5, xmm0, 0x0 - mpsadbw xmm2, xmm0, 0x5 - paddw xmm5, xmm2 - - paddw xmm1, xmm5 -%endif - movq xmm0, MMWORD PTR [rsi + rax] - movq xmm5, MMWORD PTR [rdi+ rdx] - movq xmm3, MMWORD PTR [rdi+ rdx+8] - punpcklqdq xmm5, xmm3 - - lea rsi, [rsi+rax*2] - lea rdi, [rdi+rdx*2] - - movdqa xmm2, xmm5 - mpsadbw xmm5, xmm0, 0x0 - mpsadbw xmm2, xmm0, 0x5 - paddw xmm5, xmm2 - - paddw xmm1, xmm5 -%endmacro - -%macro PROCESS_4X2X8 1 -%if %1 - movd xmm0, [rsi] - movq xmm1, MMWORD PTR [rdi] - movq xmm3, MMWORD PTR [rdi+8] - punpcklqdq xmm1, xmm3 - - mpsadbw xmm1, xmm0, 0x0 -%else - movd xmm0, [rsi] - movq xmm5, MMWORD PTR [rdi] - movq xmm3, MMWORD PTR [rdi+8] - punpcklqdq xmm5, xmm3 - - mpsadbw xmm5, xmm0, 0x0 - - paddw xmm1, xmm5 -%endif - movd xmm0, [rsi + rax] - movq xmm5, MMWORD PTR [rdi+ rdx] - movq xmm3, MMWORD PTR [rdi+ rdx+8] - punpcklqdq xmm5, xmm3 - - lea rsi, [rsi+rax*2] - lea rdi, [rdi+rdx*2] - - mpsadbw xmm5, xmm0, 0x0 - - paddw xmm1, xmm5 -%endmacro - -%macro WRITE_AS_INTS 0 - mov rdi, arg(4) ;Results - pxor xmm0, xmm0 - movdqa xmm2, xmm1 - punpcklwd xmm1, xmm0 - punpckhwd xmm2, xmm0 - - movdqa [rdi], xmm1 - movdqa [rdi + 16], xmm2 -%endmacro - -SECTION .text - -;void vpx_sad16x16x8_sse4_1( -; const unsigned char *src_ptr, -; int src_stride, -; const unsigned char *ref_ptr, -; int ref_stride, -; unsigned short *sad_array); -globalsym(vpx_sad16x16x8_sse4_1) -sym(vpx_sad16x16x8_sse4_1): - push rbp - mov rbp, rsp - SHADOW_ARGS_TO_STACK 5 - push rsi - push rdi - ; end prolog - - mov rsi, arg(0) ;src_ptr - mov rdi, arg(2) ;ref_ptr - - movsxd rax, dword ptr arg(1) ;src_stride - movsxd rdx, dword ptr arg(3) ;ref_stride - - PROCESS_16X2X8 1 - PROCESS_16X2X8 0 - PROCESS_16X2X8 0 - PROCESS_16X2X8 0 - PROCESS_16X2X8 0 - PROCESS_16X2X8 0 - PROCESS_16X2X8 0 - PROCESS_16X2X8 0 - - WRITE_AS_INTS - - ; begin epilog - pop rdi - pop rsi - UNSHADOW_ARGS - pop rbp - ret - - -;void vpx_sad16x8x8_sse4_1( -; const unsigned char *src_ptr, -; int src_stride, -; const unsigned char *ref_ptr, -; int ref_stride, -; unsigned short *sad_array -;); -globalsym(vpx_sad16x8x8_sse4_1) -sym(vpx_sad16x8x8_sse4_1): - push rbp - mov rbp, rsp - SHADOW_ARGS_TO_STACK 5 - push rsi - push rdi - ; end prolog - - mov rsi, arg(0) ;src_ptr - mov rdi, arg(2) ;ref_ptr - - movsxd rax, dword ptr arg(1) ;src_stride - movsxd rdx, dword ptr arg(3) ;ref_stride - - PROCESS_16X2X8 1 - PROCESS_16X2X8 0 - PROCESS_16X2X8 0 - PROCESS_16X2X8 0 - - WRITE_AS_INTS - - ; begin epilog - pop rdi - pop rsi - UNSHADOW_ARGS - pop rbp - ret - - -;void vpx_sad8x8x8_sse4_1( -; const unsigned char *src_ptr, -; int src_stride, -; const unsigned char *ref_ptr, -; int ref_stride, -; unsigned short *sad_array -;); -globalsym(vpx_sad8x8x8_sse4_1) -sym(vpx_sad8x8x8_sse4_1): - push rbp - mov rbp, rsp - SHADOW_ARGS_TO_STACK 5 - push rsi - push rdi - ; end prolog - - mov rsi, arg(0) ;src_ptr - mov rdi, arg(2) ;ref_ptr - - movsxd rax, dword ptr arg(1) ;src_stride - movsxd rdx, dword ptr arg(3) ;ref_stride - - PROCESS_8X2X8 1 - PROCESS_8X2X8 0 - PROCESS_8X2X8 0 - PROCESS_8X2X8 0 - - WRITE_AS_INTS - - ; begin epilog - pop rdi - pop rsi - UNSHADOW_ARGS - pop rbp - ret - - -;void vpx_sad8x16x8_sse4_1( -; const unsigned char *src_ptr, -; int src_stride, -; const unsigned char *ref_ptr, -; int ref_stride, -; unsigned short *sad_array -;); -globalsym(vpx_sad8x16x8_sse4_1) -sym(vpx_sad8x16x8_sse4_1): - push rbp - mov rbp, rsp - SHADOW_ARGS_TO_STACK 5 - push rsi - push rdi - ; end prolog - - mov rsi, arg(0) ;src_ptr - mov rdi, arg(2) ;ref_ptr - - movsxd rax, dword ptr arg(1) ;src_stride - movsxd rdx, dword ptr arg(3) ;ref_stride - - PROCESS_8X2X8 1 - PROCESS_8X2X8 0 - PROCESS_8X2X8 0 - PROCESS_8X2X8 0 - PROCESS_8X2X8 0 - PROCESS_8X2X8 0 - PROCESS_8X2X8 0 - PROCESS_8X2X8 0 - - WRITE_AS_INTS - - ; begin epilog - pop rdi - pop rsi - UNSHADOW_ARGS - pop rbp - ret - - -;void vpx_sad4x4x8_sse4_1( -; const unsigned char *src_ptr, -; int src_stride, -; const unsigned char *ref_ptr, -; int ref_stride, -; unsigned short *sad_array -;); -globalsym(vpx_sad4x4x8_sse4_1) -sym(vpx_sad4x4x8_sse4_1): - push rbp - mov rbp, rsp - SHADOW_ARGS_TO_STACK 5 - push rsi - push rdi - ; end prolog - - mov rsi, arg(0) ;src_ptr - mov rdi, arg(2) ;ref_ptr - - movsxd rax, dword ptr arg(1) ;src_stride - movsxd rdx, dword ptr arg(3) ;ref_stride - - PROCESS_4X2X8 1 - PROCESS_4X2X8 0 - - WRITE_AS_INTS - - ; begin epilog - pop rdi - pop rsi - UNSHADOW_ARGS - pop rbp - ret - - - - diff --git a/media/libvpx/libvpx/vpx_dsp/x86/sad_ssse3.asm b/media/libvpx/libvpx/vpx_dsp/x86/sad_ssse3.asm deleted file mode 100644 --- a/media/libvpx/libvpx/vpx_dsp/x86/sad_ssse3.asm +++ /dev/null @@ -1,372 +0,0 @@ -; -; Copyright (c) 2010 The WebM project authors. All Rights Reserved. -; -; Use of this source code is governed by a BSD-style license -; that can be found in the LICENSE file in the root of the source -; tree. An additional intellectual property rights grant can be found -; in the file PATENTS. All contributing project authors may -; be found in the AUTHORS file in the root of the source tree. -; - - -%include "vpx_ports/x86_abi_support.asm" - -%macro PROCESS_16X2X3 1 -%if %1 - movdqa xmm0, XMMWORD PTR [rsi] - lddqu xmm5, XMMWORD PTR [rdi] - lddqu xmm6, XMMWORD PTR [rdi+1] - lddqu xmm7, XMMWORD PTR [rdi+2] - - psadbw xmm5, xmm0 - psadbw xmm6, xmm0 - psadbw xmm7, xmm0 -%else - movdqa xmm0, XMMWORD PTR [rsi] - lddqu xmm1, XMMWORD PTR [rdi] - lddqu xmm2, XMMWORD PTR [rdi+1] - lddqu xmm3, XMMWORD PTR [rdi+2] - - psadbw xmm1, xmm0 - psadbw xmm2, xmm0 - psadbw xmm3, xmm0 - - paddw xmm5, xmm1 - paddw xmm6, xmm2 - paddw xmm7, xmm3 -%endif - movdqa xmm0, XMMWORD PTR [rsi+rax] - lddqu xmm1, XMMWORD PTR [rdi+rdx] - lddqu xmm2, XMMWORD PTR [rdi+rdx+1] - lddqu xmm3, XMMWORD PTR [rdi+rdx+2] - - lea rsi, [rsi+rax*2] - lea rdi, [rdi+rdx*2] - - psadbw xmm1, xmm0 - psadbw xmm2, xmm0 - psadbw xmm3, xmm0 - - paddw xmm5, xmm1 - paddw xmm6, xmm2 - paddw xmm7, xmm3 -%endmacro - -%macro PROCESS_16X2X3_OFFSET 2 -%if %1 - movdqa xmm0, XMMWORD PTR [rsi] - movdqa xmm4, XMMWORD PTR [rdi] - movdqa xmm7, XMMWORD PTR [rdi+16] - - movdqa xmm5, xmm7 - palignr xmm5, xmm4, %2 - - movdqa xmm6, xmm7 - palignr xmm6, xmm4, (%2+1) - - palignr xmm7, xmm4, (%2+2) - - psadbw xmm5, xmm0 - psadbw xmm6, xmm0 - psadbw xmm7, xmm0 -%else - movdqa xmm0, XMMWORD PTR [rsi] - movdqa xmm4, XMMWORD PTR [rdi] - movdqa xmm3, XMMWORD PTR [rdi+16] - - movdqa xmm1, xmm3 - palignr xmm1, xmm4, %2 - - movdqa xmm2, xmm3 - palignr xmm2, xmm4, (%2+1) - - palignr xmm3, xmm4, (%2+2) - - psadbw xmm1, xmm0 - psadbw xmm2, xmm0 - psadbw xmm3, xmm0 - - paddw xmm5, xmm1 - paddw xmm6, xmm2 - paddw xmm7, xmm3 -%endif - movdqa xmm0, XMMWORD PTR [rsi+rax] - movdqa xmm4, XMMWORD PTR [rdi+rdx] - movdqa xmm3, XMMWORD PTR [rdi+rdx+16] - - movdqa xmm1, xmm3 - palignr xmm1, xmm4, %2 - - movdqa xmm2, xmm3 - palignr xmm2, xmm4, (%2+1) - - palignr xmm3, xmm4, (%2+2) - - lea rsi, [rsi+rax*2] - lea rdi, [rdi+rdx*2] - - psadbw xmm1, xmm0 - psadbw xmm2, xmm0 - psadbw xmm3, xmm0 - - paddw xmm5, xmm1 - paddw xmm6, xmm2 - paddw xmm7, xmm3 -%endmacro - -%macro PROCESS_16X16X3_OFFSET 2 -%2_aligned_by_%1: - - sub rdi, %1 - - PROCESS_16X2X3_OFFSET 1, %1 - PROCESS_16X2X3_OFFSET 0, %1 - PROCESS_16X2X3_OFFSET 0, %1 - PROCESS_16X2X3_OFFSET 0, %1 - PROCESS_16X2X3_OFFSET 0, %1 - PROCESS_16X2X3_OFFSET 0, %1 - PROCESS_16X2X3_OFFSET 0, %1 - PROCESS_16X2X3_OFFSET 0, %1 - - jmp %2_store_off - -%endmacro - -%macro PROCESS_16X8X3_OFFSET 2 -%2_aligned_by_%1: - - sub rdi, %1 - - PROCESS_16X2X3_OFFSET 1, %1 - PROCESS_16X2X3_OFFSET 0, %1 - PROCESS_16X2X3_OFFSET 0, %1 - PROCESS_16X2X3_OFFSET 0, %1 - - jmp %2_store_off - -%endmacro - -SECTION .text - -;void int vpx_sad16x16x3_ssse3( -; unsigned char *src_ptr, -; int src_stride, -; unsigned char *ref_ptr, -; int ref_stride, -; int *results) -globalsym(vpx_sad16x16x3_ssse3) -sym(vpx_sad16x16x3_ssse3): - push rbp - mov rbp, rsp - SHADOW_ARGS_TO_STACK 5 - SAVE_XMM 7 - push rsi - push rdi - push rcx - ; end prolog - - mov rsi, arg(0) ;src_ptr - mov rdi, arg(2) ;ref_ptr - - mov rdx, 0xf - and rdx, rdi - - jmp .vpx_sad16x16x3_ssse3_skiptable -.vpx_sad16x16x3_ssse3_jumptable: - dd .vpx_sad16x16x3_ssse3_aligned_by_0 - .vpx_sad16x16x3_ssse3_do_jump - dd .vpx_sad16x16x3_ssse3_aligned_by_1 - .vpx_sad16x16x3_ssse3_do_jump - dd .vpx_sad16x16x3_ssse3_aligned_by_2 - .vpx_sad16x16x3_ssse3_do_jump - dd .vpx_sad16x16x3_ssse3_aligned_by_3 - .vpx_sad16x16x3_ssse3_do_jump - dd .vpx_sad16x16x3_ssse3_aligned_by_4 - .vpx_sad16x16x3_ssse3_do_jump - dd .vpx_sad16x16x3_ssse3_aligned_by_5 - .vpx_sad16x16x3_ssse3_do_jump - dd .vpx_sad16x16x3_ssse3_aligned_by_6 - .vpx_sad16x16x3_ssse3_do_jump - dd .vpx_sad16x16x3_ssse3_aligned_by_7 - .vpx_sad16x16x3_ssse3_do_jump - dd .vpx_sad16x16x3_ssse3_aligned_by_8 - .vpx_sad16x16x3_ssse3_do_jump - dd .vpx_sad16x16x3_ssse3_aligned_by_9 - .vpx_sad16x16x3_ssse3_do_jump - dd .vpx_sad16x16x3_ssse3_aligned_by_10 - .vpx_sad16x16x3_ssse3_do_jump - dd .vpx_sad16x16x3_ssse3_aligned_by_11 - .vpx_sad16x16x3_ssse3_do_jump - dd .vpx_sad16x16x3_ssse3_aligned_by_12 - .vpx_sad16x16x3_ssse3_do_jump - dd .vpx_sad16x16x3_ssse3_aligned_by_13 - .vpx_sad16x16x3_ssse3_do_jump - dd .vpx_sad16x16x3_ssse3_aligned_by_14 - .vpx_sad16x16x3_ssse3_do_jump - dd .vpx_sad16x16x3_ssse3_aligned_by_15 - .vpx_sad16x16x3_ssse3_do_jump -.vpx_sad16x16x3_ssse3_skiptable: - - call .vpx_sad16x16x3_ssse3_do_jump -.vpx_sad16x16x3_ssse3_do_jump: - pop rcx ; get the address of do_jump - mov rax, .vpx_sad16x16x3_ssse3_jumptable - .vpx_sad16x16x3_ssse3_do_jump - add rax, rcx ; get the absolute address of vpx_sad16x16x3_ssse3_jumptable - - movsxd rax, dword [rax + 4*rdx] ; get the 32 bit offset from the jumptable - add rcx, rax - - movsxd rax, dword ptr arg(1) ;src_stride - movsxd rdx, dword ptr arg(3) ;ref_stride - - jmp rcx - - PROCESS_16X16X3_OFFSET 0, .vpx_sad16x16x3_ssse3 - PROCESS_16X16X3_OFFSET 1, .vpx_sad16x16x3_ssse3 - PROCESS_16X16X3_OFFSET 2, .vpx_sad16x16x3_ssse3 - PROCESS_16X16X3_OFFSET 3, .vpx_sad16x16x3_ssse3 - PROCESS_16X16X3_OFFSET 4, .vpx_sad16x16x3_ssse3 - PROCESS_16X16X3_OFFSET 5, .vpx_sad16x16x3_ssse3 - PROCESS_16X16X3_OFFSET 6, .vpx_sad16x16x3_ssse3 - PROCESS_16X16X3_OFFSET 7, .vpx_sad16x16x3_ssse3 - PROCESS_16X16X3_OFFSET 8, .vpx_sad16x16x3_ssse3 - PROCESS_16X16X3_OFFSET 9, .vpx_sad16x16x3_ssse3 - PROCESS_16X16X3_OFFSET 10, .vpx_sad16x16x3_ssse3 - PROCESS_16X16X3_OFFSET 11, .vpx_sad16x16x3_ssse3 - PROCESS_16X16X3_OFFSET 12, .vpx_sad16x16x3_ssse3 - PROCESS_16X16X3_OFFSET 13, .vpx_sad16x16x3_ssse3 - PROCESS_16X16X3_OFFSET 14, .vpx_sad16x16x3_ssse3 - -.vpx_sad16x16x3_ssse3_aligned_by_15: - PROCESS_16X2X3 1 - PROCESS_16X2X3 0 - PROCESS_16X2X3 0 - PROCESS_16X2X3 0 - PROCESS_16X2X3 0 - PROCESS_16X2X3 0 - PROCESS_16X2X3 0 - PROCESS_16X2X3 0 - -.vpx_sad16x16x3_ssse3_store_off: - mov rdi, arg(4) ;Results - - movq xmm0, xmm5 - psrldq xmm5, 8 - - paddw xmm0, xmm5 - movd [rdi], xmm0 -;- - movq xmm0, xmm6 - psrldq xmm6, 8 - - paddw xmm0, xmm6 - movd [rdi+4], xmm0 -;- - movq xmm0, xmm7 - psrldq xmm7, 8 - - paddw xmm0, xmm7 - movd [rdi+8], xmm0 - - ; begin epilog - pop rcx - pop rdi - pop rsi - RESTORE_XMM - UNSHADOW_ARGS - pop rbp - ret - -;void int vpx_sad16x8x3_ssse3( -; unsigned char *src_ptr, -; int src_stride, -; unsigned char *ref_ptr, -; int ref_stride, -; int *results) -globalsym(vpx_sad16x8x3_ssse3) -sym(vpx_sad16x8x3_ssse3): - push rbp - mov rbp, rsp - SHADOW_ARGS_TO_STACK 5 - SAVE_XMM 7 - push rsi - push rdi - push rcx - ; end prolog - - mov rsi, arg(0) ;src_ptr - mov rdi, arg(2) ;ref_ptr - - mov rdx, 0xf - and rdx, rdi - - jmp .vpx_sad16x8x3_ssse3_skiptable -.vpx_sad16x8x3_ssse3_jumptable: - dd .vpx_sad16x8x3_ssse3_aligned_by_0 - .vpx_sad16x8x3_ssse3_do_jump - dd .vpx_sad16x8x3_ssse3_aligned_by_1 - .vpx_sad16x8x3_ssse3_do_jump - dd .vpx_sad16x8x3_ssse3_aligned_by_2 - .vpx_sad16x8x3_ssse3_do_jump - dd .vpx_sad16x8x3_ssse3_aligned_by_3 - .vpx_sad16x8x3_ssse3_do_jump - dd .vpx_sad16x8x3_ssse3_aligned_by_4 - .vpx_sad16x8x3_ssse3_do_jump - dd .vpx_sad16x8x3_ssse3_aligned_by_5 - .vpx_sad16x8x3_ssse3_do_jump - dd .vpx_sad16x8x3_ssse3_aligned_by_6 - .vpx_sad16x8x3_ssse3_do_jump - dd .vpx_sad16x8x3_ssse3_aligned_by_7 - .vpx_sad16x8x3_ssse3_do_jump - dd .vpx_sad16x8x3_ssse3_aligned_by_8 - .vpx_sad16x8x3_ssse3_do_jump - dd .vpx_sad16x8x3_ssse3_aligned_by_9 - .vpx_sad16x8x3_ssse3_do_jump - dd .vpx_sad16x8x3_ssse3_aligned_by_10 - .vpx_sad16x8x3_ssse3_do_jump - dd .vpx_sad16x8x3_ssse3_aligned_by_11 - .vpx_sad16x8x3_ssse3_do_jump - dd .vpx_sad16x8x3_ssse3_aligned_by_12 - .vpx_sad16x8x3_ssse3_do_jump - dd .vpx_sad16x8x3_ssse3_aligned_by_13 - .vpx_sad16x8x3_ssse3_do_jump - dd .vpx_sad16x8x3_ssse3_aligned_by_14 - .vpx_sad16x8x3_ssse3_do_jump - dd .vpx_sad16x8x3_ssse3_aligned_by_15 - .vpx_sad16x8x3_ssse3_do_jump -.vpx_sad16x8x3_ssse3_skiptable: - - call .vpx_sad16x8x3_ssse3_do_jump -.vpx_sad16x8x3_ssse3_do_jump: - pop rcx ; get the address of do_jump - mov rax, .vpx_sad16x8x3_ssse3_jumptable - .vpx_sad16x8x3_ssse3_do_jump - add rax, rcx ; get the absolute address of vpx_sad16x8x3_ssse3_jumptable - - movsxd rax, dword [rax + 4*rdx] ; get the 32 bit offset from the jumptable - add rcx, rax - - movsxd rax, dword ptr arg(1) ;src_stride - movsxd rdx, dword ptr arg(3) ;ref_stride - - jmp rcx - - PROCESS_16X8X3_OFFSET 0, .vpx_sad16x8x3_ssse3 - PROCESS_16X8X3_OFFSET 1, .vpx_sad16x8x3_ssse3 - PROCESS_16X8X3_OFFSET 2, .vpx_sad16x8x3_ssse3 - PROCESS_16X8X3_OFFSET 3, .vpx_sad16x8x3_ssse3 - PROCESS_16X8X3_OFFSET 4, .vpx_sad16x8x3_ssse3 - PROCESS_16X8X3_OFFSET 5, .vpx_sad16x8x3_ssse3 - PROCESS_16X8X3_OFFSET 6, .vpx_sad16x8x3_ssse3 - PROCESS_16X8X3_OFFSET 7, .vpx_sad16x8x3_ssse3 - PROCESS_16X8X3_OFFSET 8, .vpx_sad16x8x3_ssse3 - PROCESS_16X8X3_OFFSET 9, .vpx_sad16x8x3_ssse3 - PROCESS_16X8X3_OFFSET 10, .vpx_sad16x8x3_ssse3 - PROCESS_16X8X3_OFFSET 11, .vpx_sad16x8x3_ssse3 - PROCESS_16X8X3_OFFSET 12, .vpx_sad16x8x3_ssse3 - PROCESS_16X8X3_OFFSET 13, .vpx_sad16x8x3_ssse3 - PROCESS_16X8X3_OFFSET 14, .vpx_sad16x8x3_ssse3 - -.vpx_sad16x8x3_ssse3_aligned_by_15: - - PROCESS_16X2X3 1 - PROCESS_16X2X3 0 - PROCESS_16X2X3 0 - PROCESS_16X2X3 0 - -.vpx_sad16x8x3_ssse3_store_off: - mov rdi, arg(4) ;Results - - movq xmm0, xmm5 - psrldq xmm5, 8 - - paddw xmm0, xmm5 - movd [rdi], xmm0 -;- - movq xmm0, xmm6 - psrldq xmm6, 8 - - paddw xmm0, xmm6 - movd [rdi+4], xmm0 -;- - movq xmm0, xmm7 - psrldq xmm7, 8 - - paddw xmm0, xmm7 - movd [rdi+8], xmm0 - - ; begin epilog - pop rcx - pop rdi - pop rsi - RESTORE_XMM - UNSHADOW_ARGS - pop rbp - ret diff --git a/media/libvpx/libvpx/vpx_dsp/x86/variance_sse2.c b/media/libvpx/libvpx/vpx_dsp/x86/variance_sse2.c --- a/media/libvpx/libvpx/vpx_dsp/x86/variance_sse2.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/variance_sse2.c @@ -34,12 +34,12 @@ return add32x4_sse2(vsum); } static INLINE __m128i load4x2_sse2(const uint8_t *const p, const int stride) { - const __m128i p0 = _mm_cvtsi32_si128(loadu_uint32(p + 0 * stride)); - const __m128i p1 = _mm_cvtsi32_si128(loadu_uint32(p + 1 * stride)); + const __m128i p0 = _mm_cvtsi32_si128(loadu_int32(p + 0 * stride)); + const __m128i p1 = _mm_cvtsi32_si128(loadu_int32(p + 1 * stride)); const __m128i p01 = _mm_unpacklo_epi32(p0, p1); return _mm_unpacklo_epi8(p01, _mm_setzero_si128()); } static INLINE void variance_kernel_sse2(const __m128i src_ptr, @@ -469,27 +469,27 @@ *sse = sse_tmp; \ return sse_tmp - \ (unsigned int)(cast_prod(cast se * se) >> (wlog2 + hlog2)); \ } -#define FNS(opt1, opt2) \ - FN(64, 64, 16, 6, 6, opt1, (int64_t), (int64_t)); \ - FN(64, 32, 16, 6, 5, opt1, (int64_t), (int64_t)); \ - FN(32, 64, 16, 5, 6, opt1, (int64_t), (int64_t)); \ - FN(32, 32, 16, 5, 5, opt1, (int64_t), (int64_t)); \ - FN(32, 16, 16, 5, 4, opt1, (int64_t), (int64_t)); \ - FN(16, 32, 16, 4, 5, opt1, (int64_t), (int64_t)); \ - FN(16, 16, 16, 4, 4, opt1, (uint32_t), (int64_t)); \ - FN(16, 8, 16, 4, 3, opt1, (int32_t), (int32_t)); \ - FN(8, 16, 8, 3, 4, opt1, (int32_t), (int32_t)); \ - FN(8, 8, 8, 3, 3, opt1, (int32_t), (int32_t)); \ - FN(8, 4, 8, 3, 2, opt1, (int32_t), (int32_t)); \ - FN(4, 8, 4, 2, 3, opt1, (int32_t), (int32_t)); \ +#define FNS(opt1, opt2) \ + FN(64, 64, 16, 6, 6, opt1, (int64_t), (int64_t)) \ + FN(64, 32, 16, 6, 5, opt1, (int64_t), (int64_t)) \ + FN(32, 64, 16, 5, 6, opt1, (int64_t), (int64_t)) \ + FN(32, 32, 16, 5, 5, opt1, (int64_t), (int64_t)) \ + FN(32, 16, 16, 5, 4, opt1, (int64_t), (int64_t)) \ + FN(16, 32, 16, 4, 5, opt1, (int64_t), (int64_t)) \ + FN(16, 16, 16, 4, 4, opt1, (uint32_t), (int64_t)) \ + FN(16, 8, 16, 4, 3, opt1, (int32_t), (int32_t)) \ + FN(8, 16, 8, 3, 4, opt1, (int32_t), (int32_t)) \ + FN(8, 8, 8, 3, 3, opt1, (int32_t), (int32_t)) \ + FN(8, 4, 8, 3, 2, opt1, (int32_t), (int32_t)) \ + FN(4, 8, 4, 2, 3, opt1, (int32_t), (int32_t)) \ FN(4, 4, 4, 2, 2, opt1, (int32_t), (int32_t)) -FNS(sse2, sse2); -FNS(ssse3, ssse3); +FNS(sse2, sse2) +FNS(ssse3, ssse3) #undef FNS #undef FN // The 2 unused parameters are place holders for PIC enabled build. @@ -541,25 +541,25 @@ *sse = sse_tmp; \ return sse_tmp - \ (unsigned int)(cast_prod(cast se * se) >> (wlog2 + hlog2)); \ } -#define FNS(opt1, opt2) \ - FN(64, 64, 16, 6, 6, opt1, (int64_t), (int64_t)); \ - FN(64, 32, 16, 6, 5, opt1, (int64_t), (int64_t)); \ - FN(32, 64, 16, 5, 6, opt1, (int64_t), (int64_t)); \ - FN(32, 32, 16, 5, 5, opt1, (int64_t), (int64_t)); \ - FN(32, 16, 16, 5, 4, opt1, (int64_t), (int64_t)); \ - FN(16, 32, 16, 4, 5, opt1, (int64_t), (int64_t)); \ - FN(16, 16, 16, 4, 4, opt1, (uint32_t), (int64_t)); \ - FN(16, 8, 16, 4, 3, opt1, (uint32_t), (int32_t)); \ - FN(8, 16, 8, 3, 4, opt1, (uint32_t), (int32_t)); \ - FN(8, 8, 8, 3, 3, opt1, (uint32_t), (int32_t)); \ - FN(8, 4, 8, 3, 2, opt1, (uint32_t), (int32_t)); \ - FN(4, 8, 4, 2, 3, opt1, (uint32_t), (int32_t)); \ +#define FNS(opt1, opt2) \ + FN(64, 64, 16, 6, 6, opt1, (int64_t), (int64_t)) \ + FN(64, 32, 16, 6, 5, opt1, (int64_t), (int64_t)) \ + FN(32, 64, 16, 5, 6, opt1, (int64_t), (int64_t)) \ + FN(32, 32, 16, 5, 5, opt1, (int64_t), (int64_t)) \ + FN(32, 16, 16, 5, 4, opt1, (int64_t), (int64_t)) \ + FN(16, 32, 16, 4, 5, opt1, (int64_t), (int64_t)) \ + FN(16, 16, 16, 4, 4, opt1, (uint32_t), (int64_t)) \ + FN(16, 8, 16, 4, 3, opt1, (uint32_t), (int32_t)) \ + FN(8, 16, 8, 3, 4, opt1, (uint32_t), (int32_t)) \ + FN(8, 8, 8, 3, 3, opt1, (uint32_t), (int32_t)) \ + FN(8, 4, 8, 3, 2, opt1, (uint32_t), (int32_t)) \ + FN(4, 8, 4, 2, 3, opt1, (uint32_t), (int32_t)) \ FN(4, 4, 4, 2, 2, opt1, (uint32_t), (int32_t)) -FNS(sse2, sse); -FNS(ssse3, ssse3); +FNS(sse2, sse) +FNS(ssse3, ssse3) #undef FNS #undef FN diff --git a/media/libvpx/libvpx/vpx_dsp/x86/vpx_subpixel_4t_intrin_sse2.c b/media/libvpx/libvpx/vpx_dsp/x86/vpx_subpixel_4t_intrin_sse2.c --- a/media/libvpx/libvpx/vpx_dsp/x86/vpx_subpixel_4t_intrin_sse2.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/vpx_subpixel_4t_intrin_sse2.c @@ -1038,16 +1038,16 @@ // void vpx_convolve8_avg_vert_sse2(const uint8_t *src, ptrdiff_t src_stride, // uint8_t *dst, ptrdiff_t dst_stride, // const InterpKernel *filter, int x0_q4, // int32_t x_step_q4, int y0_q4, int y_step_q4, // int w, int h); -FUN_CONV_1D(horiz, x0_q4, x_step_q4, h, src, , sse2, 0); +FUN_CONV_1D(horiz, x0_q4, x_step_q4, h, src, , sse2, 0) FUN_CONV_1D(vert, y0_q4, y_step_q4, v, src - (num_taps / 2 - 1) * src_stride, , - sse2, 0); -FUN_CONV_1D(avg_horiz, x0_q4, x_step_q4, h, src, avg_, sse2, 1); + sse2, 0) +FUN_CONV_1D(avg_horiz, x0_q4, x_step_q4, h, src, avg_, sse2, 1) FUN_CONV_1D(avg_vert, y0_q4, y_step_q4, v, - src - (num_taps / 2 - 1) * src_stride, avg_, sse2, 1); + src - (num_taps / 2 - 1) * src_stride, avg_, sse2, 1) // void vpx_convolve8_sse2(const uint8_t *src, ptrdiff_t src_stride, // uint8_t *dst, ptrdiff_t dst_stride, // const InterpKernel *filter, int x0_q4, // int32_t x_step_q4, int y0_q4, int y_step_q4, @@ -1055,12 +1055,12 @@ // void vpx_convolve8_avg_sse2(const uint8_t *src, ptrdiff_t src_stride, // uint8_t *dst, ptrdiff_t dst_stride, // const InterpKernel *filter, int x0_q4, // int32_t x_step_q4, int y0_q4, int y_step_q4, // int w, int h); -FUN_CONV_2D(, sse2, 0); -FUN_CONV_2D(avg_, sse2, 1); +FUN_CONV_2D(, sse2, 0) +FUN_CONV_2D(avg_, sse2, 1) #if CONFIG_VP9_HIGHBITDEPTH && VPX_ARCH_X86_64 // From vpx_dsp/x86/vpx_high_subpixel_8t_sse2.asm. highbd_filter8_1dfunction vpx_highbd_filter_block1d16_v8_sse2; highbd_filter8_1dfunction vpx_highbd_filter_block1d16_h8_sse2; @@ -1137,16 +1137,16 @@ // const int16_t *filter_x, // int x_step_q4, // const int16_t *filter_y, // int y_step_q4, // int w, int h, int bd); -HIGH_FUN_CONV_1D(horiz, x0_q4, x_step_q4, h, src, , sse2, 0); +HIGH_FUN_CONV_1D(horiz, x0_q4, x_step_q4, h, src, , sse2, 0) HIGH_FUN_CONV_1D(vert, y0_q4, y_step_q4, v, - src - src_stride * (num_taps / 2 - 1), , sse2, 0); -HIGH_FUN_CONV_1D(avg_horiz, x0_q4, x_step_q4, h, src, avg_, sse2, 1); + src - src_stride * (num_taps / 2 - 1), , sse2, 0) +HIGH_FUN_CONV_1D(avg_horiz, x0_q4, x_step_q4, h, src, avg_, sse2, 1) HIGH_FUN_CONV_1D(avg_vert, y0_q4, y_step_q4, v, - src - src_stride * (num_taps / 2 - 1), avg_, sse2, 1); + src - src_stride * (num_taps / 2 - 1), avg_, sse2, 1) // void vpx_highbd_convolve8_sse2(const uint8_t *src, ptrdiff_t src_stride, // uint8_t *dst, ptrdiff_t dst_stride, // const InterpKernel *filter, int x0_q4, // int32_t x_step_q4, int y0_q4, int y_step_q4, @@ -1154,8 +1154,8 @@ // void vpx_highbd_convolve8_avg_sse2(const uint8_t *src, ptrdiff_t src_stride, // uint8_t *dst, ptrdiff_t dst_stride, // const InterpKernel *filter, int x0_q4, // int32_t x_step_q4, int y0_q4, // int y_step_q4, int w, int h, int bd); -HIGH_FUN_CONV_2D(, sse2, 0); -HIGH_FUN_CONV_2D(avg_, sse2, 1); +HIGH_FUN_CONV_2D(, sse2, 0) +HIGH_FUN_CONV_2D(avg_, sse2, 1) #endif // CONFIG_VP9_HIGHBITDEPTH && VPX_ARCH_X86_64 diff --git a/media/libvpx/libvpx/vpx_dsp/x86/vpx_subpixel_8t_intrin_avx2.c b/media/libvpx/libvpx/vpx_dsp/x86/vpx_subpixel_8t_intrin_avx2.c --- a/media/libvpx/libvpx/vpx_dsp/x86/vpx_subpixel_8t_intrin_avx2.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/vpx_subpixel_8t_intrin_avx2.c @@ -967,16 +967,16 @@ // void vpx_convolve8_avg_vert_avx2(const uint8_t *src, ptrdiff_t src_stride, // uint8_t *dst, ptrdiff_t dst_stride, // const InterpKernel *filter, int x0_q4, // int32_t x_step_q4, int y0_q4, // int y_step_q4, int w, int h); -FUN_CONV_1D(horiz, x0_q4, x_step_q4, h, src, , avx2, 0); +FUN_CONV_1D(horiz, x0_q4, x_step_q4, h, src, , avx2, 0) FUN_CONV_1D(vert, y0_q4, y_step_q4, v, src - src_stride * (num_taps / 2 - 1), , - avx2, 0); -FUN_CONV_1D(avg_horiz, x0_q4, x_step_q4, h, src, avg_, avx2, 1); + avx2, 0) +FUN_CONV_1D(avg_horiz, x0_q4, x_step_q4, h, src, avg_, avx2, 1) FUN_CONV_1D(avg_vert, y0_q4, y_step_q4, v, - src - src_stride * (num_taps / 2 - 1), avg_, avx2, 1); + src - src_stride * (num_taps / 2 - 1), avg_, avx2, 1) // void vpx_convolve8_avx2(const uint8_t *src, ptrdiff_t src_stride, // uint8_t *dst, ptrdiff_t dst_stride, // const InterpKernel *filter, int x0_q4, // int32_t x_step_q4, int y0_q4, int y_step_q4, @@ -984,8 +984,8 @@ // void vpx_convolve8_avg_avx2(const uint8_t *src, ptrdiff_t src_stride, // uint8_t *dst, ptrdiff_t dst_stride, // const InterpKernel *filter, int x0_q4, // int32_t x_step_q4, int y0_q4, int y_step_q4, // int w, int h); -FUN_CONV_2D(, avx2, 0); -FUN_CONV_2D(avg_, avx2, 1); +FUN_CONV_2D(, avx2, 0) +FUN_CONV_2D(avg_, avx2, 1) #endif // HAVE_AX2 && HAVE_SSSE3 diff --git a/media/libvpx/libvpx/vpx_dsp/x86/vpx_subpixel_8t_intrin_ssse3.c b/media/libvpx/libvpx/vpx_dsp/x86/vpx_subpixel_8t_intrin_ssse3.c --- a/media/libvpx/libvpx/vpx_dsp/x86/vpx_subpixel_8t_intrin_ssse3.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/vpx_subpixel_8t_intrin_ssse3.c @@ -729,16 +729,16 @@ // void vpx_convolve8_avg_vert_ssse3(const uint8_t *src, ptrdiff_t src_stride, // uint8_t *dst, ptrdiff_t dst_stride, // const InterpKernel *filter, int x0_q4, // int32_t x_step_q4, int y0_q4, // int y_step_q4, int w, int h); -FUN_CONV_1D(horiz, x0_q4, x_step_q4, h, src, , ssse3, 0); +FUN_CONV_1D(horiz, x0_q4, x_step_q4, h, src, , ssse3, 0) FUN_CONV_1D(vert, y0_q4, y_step_q4, v, src - src_stride * (num_taps / 2 - 1), , - ssse3, 0); -FUN_CONV_1D(avg_horiz, x0_q4, x_step_q4, h, src, avg_, ssse3, 1); + ssse3, 0) +FUN_CONV_1D(avg_horiz, x0_q4, x_step_q4, h, src, avg_, ssse3, 1) FUN_CONV_1D(avg_vert, y0_q4, y_step_q4, v, - src - src_stride * (num_taps / 2 - 1), avg_, ssse3, 1); + src - src_stride * (num_taps / 2 - 1), avg_, ssse3, 1) static void filter_horiz_w8_ssse3(const uint8_t *const src, const ptrdiff_t src_stride, uint8_t *const dst, const int16_t *const x_filter) { @@ -1081,7 +1081,7 @@ // void vpx_convolve8_avg_ssse3(const uint8_t *src, ptrdiff_t src_stride, // uint8_t *dst, ptrdiff_t dst_stride, // const InterpKernel *filter, int x0_q4, // int32_t x_step_q4, int y0_q4, int y_step_q4, // int w, int h); -FUN_CONV_2D(, ssse3, 0); -FUN_CONV_2D(avg_, ssse3, 1); +FUN_CONV_2D(, ssse3, 0) +FUN_CONV_2D(avg_, ssse3, 1) diff --git a/media/libvpx/libvpx/vpx_ports/bitops.h b/media/libvpx/libvpx/vpx_ports/bitops.h --- a/media/libvpx/libvpx/vpx_ports/bitops.h +++ b/media/libvpx/libvpx/vpx_ports/bitops.h @@ -24,32 +24,51 @@ #ifdef __cplusplus extern "C" { #endif -// These versions of get_msb() are only valid when n != 0 because all -// of the optimized versions are undefined when n == 0: +// These versions of get_lsb() and get_msb() are only valid when n != 0 +// because all of the optimized versions are undefined when n == 0: // https://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html // use GNU builtins where available. #if defined(__GNUC__) && \ ((__GNUC__ == 3 && __GNUC_MINOR__ >= 4) || __GNUC__ >= 4) +static INLINE int get_lsb(unsigned int n) { + assert(n != 0); + return __builtin_ctz(n); +} + static INLINE int get_msb(unsigned int n) { assert(n != 0); return 31 ^ __builtin_clz(n); } #elif defined(USE_MSC_INTRINSICS) +#pragma intrinsic(_BitScanForward) #pragma intrinsic(_BitScanReverse) +static INLINE int get_lsb(unsigned int n) { + unsigned long first_set_bit; // NOLINT(runtime/int) + _BitScanForward(&first_set_bit, n); + return first_set_bit; +} + static INLINE int get_msb(unsigned int n) { unsigned long first_set_bit; assert(n != 0); _BitScanReverse(&first_set_bit, n); return first_set_bit; } #undef USE_MSC_INTRINSICS #else +static INLINE int get_lsb(unsigned int n) { + int i; + assert(n != 0); + for (i = 0; i < 32 && !(n & 1); ++i) n >>= 1; + return i; +} + // Returns (int)floor(log2(n)). n must be > 0. static INLINE int get_msb(unsigned int n) { int log = 0; unsigned int value = n; int i; diff --git a/media/libvpx/libvpx/vpx_ports/loongarch.h b/media/libvpx/libvpx/vpx_ports/loongarch.h new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_ports/loongarch.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Jin Bo + * Contributed by Lu Wang + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VPX_VPX_PORTS_LOONGARCH_H_ +#define VPX_VPX_PORTS_LOONGARCH_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#define HAS_LSX 0x01 +#define HAS_LASX 0x02 + +int loongarch_cpu_caps(void); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // VPX_VPX_PORTS_LOONGARCH_H_ diff --git a/media/libvpx/libvpx/vpx_ports/loongarch_cpudetect.c b/media/libvpx/libvpx/vpx_ports/loongarch_cpudetect.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_ports/loongarch_cpudetect.c @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Jin Bo + * Contributed by Lu Wang + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "./vpx_config.h" +#include "vpx_ports/loongarch.h" + +#define LOONGARCH_CFG2 0x02 +#define LOONGARCH_CFG2_LSX (1 << 6) +#define LOONGARCH_CFG2_LASX (1 << 7) + +#if CONFIG_RUNTIME_CPU_DETECT +#if defined(__loongarch__) && defined(__linux__) +int loongarch_cpu_caps(void) { + int reg = 0; + int flag = 0; + + __asm__ volatile("cpucfg %0, %1 \n\t" : "+&r"(reg) : "r"(LOONGARCH_CFG2)); + if (reg & LOONGARCH_CFG2_LSX) flag |= HAS_LSX; + + if (reg & LOONGARCH_CFG2_LASX) flag |= HAS_LASX; + + return flag; +} +#else /* end __loongarch__ && __linux__ */ +#error \ + "--enable-runtime-cpu-detect selected, but no CPU detection method " \ +"available for your platform. Reconfigure with --disable-runtime-cpu-detect." +#endif +#else /* end CONFIG_RUNTIME_CPU_DETECT */ +int loongarch_cpu_caps(void) { return 0; } +#endif diff --git a/media/libvpx/libvpx/vpx_ports/mips.h b/media/libvpx/libvpx/vpx_ports/mips.h --- a/media/libvpx/libvpx/vpx_ports/mips.h +++ b/media/libvpx/libvpx/vpx_ports/mips.h @@ -6,12 +6,12 @@ * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ -#ifndef VPX_PORTS_MIPS_H_ -#define VPX_PORTS_MIPS_H_ +#ifndef VPX_VPX_PORTS_MIPS_H_ +#define VPX_VPX_PORTS_MIPS_H_ #ifdef __cplusplus extern "C" { #endif @@ -22,6 +22,6 @@ #ifdef __cplusplus } // extern "C" #endif -#endif // VPX_PORTS_MIPS_H_ +#endif // VPX_VPX_PORTS_MIPS_H_ diff --git a/media/libvpx/libvpx/vpx_ports/vpx_once.h b/media/libvpx/libvpx/vpx_ports/vpx_once.h --- a/media/libvpx/libvpx/vpx_ports/vpx_once.h +++ b/media/libvpx/libvpx/vpx_ports/vpx_once.h @@ -93,11 +93,11 @@ #elif CONFIG_MULTITHREAD && defined(__OS2__) #define INCL_DOS #include static void once(void (*func)(void)) { - static int done; + static volatile int done; /* If the initialization is complete, return early. */ if (done) return; /* Causes all other threads in the process to block themselves @@ -126,11 +126,11 @@ * so as long as your platform provides atomic loads/stores of pointers * no synchronization is strictly necessary. */ static void once(void (*func)(void)) { - static int done; + static volatile int done; if (!done) { func(); done = 1; } diff --git a/media/libvpx/libvpx/vpx_ports/vpx_ports.mk b/media/libvpx/libvpx/vpx_ports/vpx_ports.mk --- a/media/libvpx/libvpx/vpx_ports/vpx_ports.mk +++ b/media/libvpx/libvpx/vpx_ports/vpx_ports.mk @@ -26,11 +26,11 @@ # Visual Studio x64 does not support the _mm_empty() intrinsic. PORTS_SRCS-$(HAVE_MMX) += emms_mmx.asm endif ifeq ($(VPX_ARCH_X86_64),yes) -PORTS_SRCS-yes += float_control_word.asm +PORTS_SRCS-$(CONFIG_MSVS) += float_control_word.asm endif ifeq ($(VPX_ARCH_X86)$(VPX_ARCH_X86_64),yes) PORTS_SRCS-yes += x86.h PORTS_SRCS-yes += x86_abi_support.asm @@ -43,8 +43,11 @@ PORTS_SRCS-$(VPX_ARCH_PPC) += ppc.h PORTS_SRCS-$(VPX_ARCH_MIPS) += mips_cpudetect.c PORTS_SRCS-$(VPX_ARCH_MIPS) += mips.h +PORTS_SRCS-$(VPX_ARCH_LOONGARCH) += loongarch_cpudetect.c +PORTS_SRCS-$(VPX_ARCH_LOONGARCH) += loongarch.h + ifeq ($(VPX_ARCH_MIPS), yes) PORTS_SRCS-yes += asmdefs_mmi.h endif diff --git a/media/libvpx/libvpx/vpx_ports/x86.h b/media/libvpx/libvpx/vpx_ports/x86.h --- a/media/libvpx/libvpx/vpx_ports/x86.h +++ b/media/libvpx/libvpx/vpx_ports/x86.h @@ -45,19 +45,19 @@ #if defined(__GNUC__) && __GNUC__ || defined(__ANDROID__) #if VPX_ARCH_X86_64 #define cpuid(func, func2, ax, bx, cx, dx) \ __asm__ __volatile__("cpuid \n\t" \ : "=a"(ax), "=b"(bx), "=c"(cx), "=d"(dx) \ - : "a"(func), "c"(func2)); + : "a"(func), "c"(func2)) #else #define cpuid(func, func2, ax, bx, cx, dx) \ __asm__ __volatile__( \ "mov %%ebx, %%edi \n\t" \ "cpuid \n\t" \ "xchg %%edi, %%ebx \n\t" \ : "=a"(ax), "=D"(bx), "=c"(cx), "=d"(dx) \ - : "a"(func), "c"(func2)); + : "a"(func), "c"(func2)) #endif #elif defined(__SUNPRO_C) || \ defined(__SUNPRO_CC) /* end __GNUC__ or __ANDROID__*/ #if VPX_ARCH_X86_64 #define cpuid(func, func2, ax, bx, cx, dx) \ @@ -65,20 +65,20 @@ "xchg %rsi, %rbx \n\t" \ "cpuid \n\t" \ "movl %ebx, %edi \n\t" \ "xchg %rsi, %rbx \n\t" \ : "=a"(ax), "=D"(bx), "=c"(cx), "=d"(dx) \ - : "a"(func), "c"(func2)); + : "a"(func), "c"(func2)) #else #define cpuid(func, func2, ax, bx, cx, dx) \ asm volatile( \ "pushl %ebx \n\t" \ "cpuid \n\t" \ "movl %ebx, %edi \n\t" \ "popl %ebx \n\t" \ : "=a"(ax), "=D"(bx), "=c"(cx), "=d"(dx) \ - : "a"(func), "c"(func2)); + : "a"(func), "c"(func2)) #endif #else /* end __SUNPRO__ */ #if VPX_ARCH_X86_64 #if defined(_MSC_VER) && _MSC_VER > 1500 #define cpuid(func, func2, a, b, c, d) \ @@ -389,11 +389,11 @@ // Precision PC Field // Single Precision (24-Bits) 00B // Reserved 01B // Double Precision (53-Bits) 10B // Extended Precision (64-Bits) 11B - x87_set_control_word((mode & ~0x300) | 0x200); + x87_set_control_word((mode & ~0x300u) | 0x200u); return mode; } #ifdef __cplusplus } // extern "C" diff --git a/media/libvpx/libvpx/vpx_scale/generic/yv12config.c b/media/libvpx/libvpx/vpx_scale/generic/yv12config.c --- a/media/libvpx/libvpx/vpx_scale/generic/yv12config.c +++ b/media/libvpx/libvpx/vpx_scale/generic/yv12config.c @@ -62,10 +62,14 @@ int uvplane_size = (uv_height + border) * uv_stride; const size_t frame_size = yplane_size + 2 * uvplane_size; if (!ybf->buffer_alloc) { ybf->buffer_alloc = (uint8_t *)vpx_memalign(32, frame_size); + if (!ybf->buffer_alloc) { + ybf->buffer_alloc_sz = 0; + return -1; + } #if defined(__has_feature) #if __has_feature(memory_sanitizer) // This memset is needed for fixing the issue of using uninitialized // value in msan test. It will cause a perf loss, so only do this for // msan test. @@ -73,11 +77,11 @@ #endif #endif ybf->buffer_alloc_sz = frame_size; } - if (!ybf->buffer_alloc || ybf->buffer_alloc_sz < frame_size) return -1; + if (ybf->buffer_alloc_sz < frame_size) return -1; /* Only support allocating buffers that have a border that's a multiple * of 32. The border restriction is required to get 16-byte alignment of * the start of the chroma rows without introducing an arbitrary gap * between planes, which would break the semantics of things like diff --git a/media/libvpx/libvpx/vpx_util/loongson_intrinsics.h b/media/libvpx/libvpx/vpx_util/loongson_intrinsics.h new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_util/loongson_intrinsics.h @@ -0,0 +1,2090 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#ifndef VPX_VPX_UTIL_LOONGSON_INTRINSICS_H_ +#define VPX_VPX_UTIL_LOONGSON_INTRINSICS_H_ + +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + * Contributed by Shiyou Yin + * Xiwei Gu + * Lu Wang + * + * This file is a header file for loongarch builtin extension. + * + */ + +#ifndef LOONGSON_INTRINSICS_H +#define LOONGSON_INTRINSICS_H + +/** + * MAJOR version: Macro usage changes. + * MINOR version: Add new functions, or bug fixes. + * MICRO version: Comment changes or implementation changes. + */ +#define LSOM_VERSION_MAJOR 1 +#define LSOM_VERSION_MINOR 2 +#define LSOM_VERSION_MICRO 1 + +#define DUP2_ARG1(_INS, _IN0, _IN1, _OUT0, _OUT1) \ + { \ + _OUT0 = _INS(_IN0); \ + _OUT1 = _INS(_IN1); \ + } + +#define DUP2_ARG2(_INS, _IN0, _IN1, _IN2, _IN3, _OUT0, _OUT1) \ + { \ + _OUT0 = _INS(_IN0, _IN1); \ + _OUT1 = _INS(_IN2, _IN3); \ + } + +#define DUP2_ARG3(_INS, _IN0, _IN1, _IN2, _IN3, _IN4, _IN5, _OUT0, _OUT1) \ + { \ + _OUT0 = _INS(_IN0, _IN1, _IN2); \ + _OUT1 = _INS(_IN3, _IN4, _IN5); \ + } + +#define DUP4_ARG1(_INS, _IN0, _IN1, _IN2, _IN3, _OUT0, _OUT1, _OUT2, _OUT3) \ + { \ + DUP2_ARG1(_INS, _IN0, _IN1, _OUT0, _OUT1); \ + DUP2_ARG1(_INS, _IN2, _IN3, _OUT2, _OUT3); \ + } + +#define DUP4_ARG2(_INS, _IN0, _IN1, _IN2, _IN3, _IN4, _IN5, _IN6, _IN7, _OUT0, \ + _OUT1, _OUT2, _OUT3) \ + { \ + DUP2_ARG2(_INS, _IN0, _IN1, _IN2, _IN3, _OUT0, _OUT1); \ + DUP2_ARG2(_INS, _IN4, _IN5, _IN6, _IN7, _OUT2, _OUT3); \ + } + +#define DUP4_ARG3(_INS, _IN0, _IN1, _IN2, _IN3, _IN4, _IN5, _IN6, _IN7, _IN8, \ + _IN9, _IN10, _IN11, _OUT0, _OUT1, _OUT2, _OUT3) \ + { \ + DUP2_ARG3(_INS, _IN0, _IN1, _IN2, _IN3, _IN4, _IN5, _OUT0, _OUT1); \ + DUP2_ARG3(_INS, _IN6, _IN7, _IN8, _IN9, _IN10, _IN11, _OUT2, _OUT3); \ + } + +#ifdef __loongarch_sx +#include +/* + * ============================================================================= + * Description : Dot product & addition of byte vector elements + * Arguments : Inputs - in_c, in_h, in_l + * Outputs - out + * Return Type - halfword + * Details : Signed byte elements from in_h are multiplied by + * signed byte elements from in_l, and then added adjacent to + * each other to get a result twice the size of input. Then + * the results are added to signed half-word elements from in_c. + * Example : out = __lsx_vdp2add_h_b(in_c, in_h, in_l) + * in_c : 1,2,3,4, 1,2,3,4 + * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * in_l : 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,1 + * out : 23,40,41,26, 23,40,41,26 + * ============================================================================= + */ +static inline __m128i __lsx_vdp2add_h_b(__m128i in_c, __m128i in_h, + __m128i in_l) { + __m128i out; + + out = __lsx_vmaddwev_h_b(in_c, in_h, in_l); + out = __lsx_vmaddwod_h_b(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product & addition of byte vector elements + * Arguments : Inputs - in_c, in_h, in_l + * Outputs - out + * Return Type - halfword + * Details : Unsigned byte elements from in_h are multiplied by + * unsigned byte elements from in_l, and then added adjacent to + * each other to get a result twice the size of input. + * The results are added to signed half-word elements from in_c. + * Example : out = __lsx_vdp2add_h_bu(in_c, in_h, in_l) + * in_c : 1,2,3,4, 1,2,3,4 + * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * in_l : 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,1 + * out : 23,40,41,26, 23,40,41,26 + * ============================================================================= + */ +static inline __m128i __lsx_vdp2add_h_bu(__m128i in_c, __m128i in_h, + __m128i in_l) { + __m128i out; + + out = __lsx_vmaddwev_h_bu(in_c, in_h, in_l); + out = __lsx_vmaddwod_h_bu(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product & addition of byte vector elements + * Arguments : Inputs - in_c, in_h, in_l + * Outputs - out + * Return Type - halfword + * Details : Unsigned byte elements from in_h are multiplied by + * signed byte elements from in_l, and then added adjacent to + * each other to get a result twice the size of input. + * The results are added to signed half-word elements from in_c. + * Example : out = __lsx_vdp2add_h_bu_b(in_c, in_h, in_l) + * in_c : 1,1,1,1, 1,1,1,1 + * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * in_l : -1,-2,-3,-4, -5,-6,-7,-8, 1,2,3,4, 5,6,7,8 + * out : -4,-24,-60,-112, 6,26,62,114 + * ============================================================================= + */ +static inline __m128i __lsx_vdp2add_h_bu_b(__m128i in_c, __m128i in_h, + __m128i in_l) { + __m128i out; + + out = __lsx_vmaddwev_h_bu_b(in_c, in_h, in_l); + out = __lsx_vmaddwod_h_bu_b(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product & addition of half-word vector elements + * Arguments : Inputs - in_c, in_h, in_l + * Outputs - out + * Return Type - __m128i + * Details : Signed half-word elements from in_h are multiplied by + * signed half-word elements from in_l, and then added adjacent to + * each other to get a result twice the size of input. + * Then the results are added to signed word elements from in_c. + * Example : out = __lsx_vdp2add_h_b(in_c, in_h, in_l) + * in_c : 1,2,3,4 + * in_h : 1,2,3,4, 5,6,7,8 + * in_l : 8,7,6,5, 4,3,2,1 + * out : 23,40,41,26 + * ============================================================================= + */ +static inline __m128i __lsx_vdp2add_w_h(__m128i in_c, __m128i in_h, + __m128i in_l) { + __m128i out; + + out = __lsx_vmaddwev_w_h(in_c, in_h, in_l); + out = __lsx_vmaddwod_w_h(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of byte vector elements + * Arguments : Inputs - in_h, in_l + * Outputs - out + * Return Type - halfword + * Details : Signed byte elements from in_h are multiplied by + * signed byte elements from in_l, and then added adjacent to + * each other to get a result twice the size of input. + * Example : out = __lsx_vdp2_h_b(in_h, in_l) + * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * in_l : 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,1 + * out : 22,38,38,22, 22,38,38,22 + * ============================================================================= + */ +static inline __m128i __lsx_vdp2_h_b(__m128i in_h, __m128i in_l) { + __m128i out; + + out = __lsx_vmulwev_h_b(in_h, in_l); + out = __lsx_vmaddwod_h_b(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of byte vector elements + * Arguments : Inputs - in_h, in_l + * Outputs - out + * Return Type - halfword + * Details : Unsigned byte elements from in_h are multiplied by + * unsigned byte elements from in_l, and then added adjacent to + * each other to get a result twice the size of input. + * Example : out = __lsx_vdp2_h_bu(in_h, in_l) + * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * in_l : 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,1 + * out : 22,38,38,22, 22,38,38,22 + * ============================================================================= + */ +static inline __m128i __lsx_vdp2_h_bu(__m128i in_h, __m128i in_l) { + __m128i out; + + out = __lsx_vmulwev_h_bu(in_h, in_l); + out = __lsx_vmaddwod_h_bu(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of byte vector elements + * Arguments : Inputs - in_h, in_l + * Outputs - out + * Return Type - halfword + * Details : Unsigned byte elements from in_h are multiplied by + * signed byte elements from in_l, and then added adjacent to + * each other to get a result twice the size of input. + * Example : out = __lsx_vdp2_h_bu_b(in_h, in_l) + * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * in_l : 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,-1 + * out : 22,38,38,22, 22,38,38,6 + * ============================================================================= + */ +static inline __m128i __lsx_vdp2_h_bu_b(__m128i in_h, __m128i in_l) { + __m128i out; + + out = __lsx_vmulwev_h_bu_b(in_h, in_l); + out = __lsx_vmaddwod_h_bu_b(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of byte vector elements + * Arguments : Inputs - in_h, in_l + * Outputs - out + * Return Type - halfword + * Details : Signed byte elements from in_h are multiplied by + * signed byte elements from in_l, and then added adjacent to + * each other to get a result twice the size of input. + * Example : out = __lsx_vdp2_w_h(in_h, in_l) + * in_h : 1,2,3,4, 5,6,7,8 + * in_l : 8,7,6,5, 4,3,2,1 + * out : 22,38,38,22 + * ============================================================================= + */ +static inline __m128i __lsx_vdp2_w_h(__m128i in_h, __m128i in_l) { + __m128i out; + + out = __lsx_vmulwev_w_h(in_h, in_l); + out = __lsx_vmaddwod_w_h(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of byte vector elements + * Arguments : Inputs - in_h, in_l + * Outputs - out + * Return Type - double + * Details : Signed byte elements from in_h are multiplied by + * signed byte elements from in_l, and then added adjacent to + * each other to get a result twice the size of input. + * Example : out = __lsx_vdp2_d_w(in_h, in_l) + * in_h : 1,2,3,4 + * in_l : 8,7,6,5 + * out : 22,38 + * ============================================================================= + */ +static inline __m128i __lsx_vdp2_d_w(__m128i in_h, __m128i in_l) { + __m128i out; + + out = __lsx_vmulwev_d_w(in_h, in_l); + out = __lsx_vmaddwod_d_w(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Clip all halfword elements of input vector between min & max + * out = ((_in) < (min)) ? (min) : (((_in) > (max)) ? (max) : + * (_in)) + * Arguments : Inputs - _in (input vector) + * - min (min threshold) + * - max (max threshold) + * Outputs - out (output vector with clipped elements) + * Return Type - signed halfword + * Example : out = __lsx_vclip_h(_in) + * _in : -8,2,280,249, -8,255,280,249 + * min : 1,1,1,1, 1,1,1,1 + * max : 9,9,9,9, 9,9,9,9 + * out : 1,2,9,9, 1,9,9,9 + * ============================================================================= + */ +static inline __m128i __lsx_vclip_h(__m128i _in, __m128i min, __m128i max) { + __m128i out; + + out = __lsx_vmax_h(min, _in); + out = __lsx_vmin_h(max, out); + return out; +} + +/* + * ============================================================================= + * Description : Set each element of vector between 0 and 255 + * Arguments : Inputs - _in + * Outputs - out + * Return Type - halfword + * Details : Signed byte elements from _in are clamped between 0 and 255. + * Example : out = __lsx_vclip255_h(_in) + * _in : -8,255,280,249, -8,255,280,249 + * out : 0,255,255,249, 0,255,255,249 + * ============================================================================= + */ +static inline __m128i __lsx_vclip255_h(__m128i _in) { + __m128i out; + + out = __lsx_vmaxi_h(_in, 0); + out = __lsx_vsat_hu(out, 7); + return out; +} + +/* + * ============================================================================= + * Description : Set each element of vector between 0 and 255 + * Arguments : Inputs - _in + * Outputs - out + * Return Type - word + * Details : Signed byte elements from _in are clamped between 0 and 255. + * Example : out = __lsx_vclip255_w(_in) + * _in : -8,255,280,249 + * out : 0,255,255,249 + * ============================================================================= + */ +static inline __m128i __lsx_vclip255_w(__m128i _in) { + __m128i out; + + out = __lsx_vmaxi_w(_in, 0); + out = __lsx_vsat_wu(out, 7); + return out; +} + +/* + * ============================================================================= + * Description : Swap two variables + * Arguments : Inputs - _in0, _in1 + * Outputs - _in0, _in1 (in-place) + * Details : Swapping of two input variables using xor + * Example : LSX_SWAP(_in0, _in1) + * _in0 : 1,2,3,4 + * _in1 : 5,6,7,8 + * _in0(out) : 5,6,7,8 + * _in1(out) : 1,2,3,4 + * ============================================================================= + */ +#define LSX_SWAP(_in0, _in1) \ + { \ + _in0 = __lsx_vxor_v(_in0, _in1); \ + _in1 = __lsx_vxor_v(_in0, _in1); \ + _in0 = __lsx_vxor_v(_in0, _in1); \ + } + +/* + * ============================================================================= + * Description : Transpose 4x4 block with word elements in vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1, out2, out3 + * Details : + * Example : + * 1, 2, 3, 4 1, 5, 9,13 + * 5, 6, 7, 8 to 2, 6,10,14 + * 9,10,11,12 =====> 3, 7,11,15 + * 13,14,15,16 4, 8,12,16 + * ============================================================================= + */ +#define LSX_TRANSPOSE4x4_W(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ + { \ + __m128i _t0, _t1, _t2, _t3; \ + \ + _t0 = __lsx_vilvl_w(_in1, _in0); \ + _t1 = __lsx_vilvh_w(_in1, _in0); \ + _t2 = __lsx_vilvl_w(_in3, _in2); \ + _t3 = __lsx_vilvh_w(_in3, _in2); \ + _out0 = __lsx_vilvl_d(_t2, _t0); \ + _out1 = __lsx_vilvh_d(_t2, _t0); \ + _out2 = __lsx_vilvl_d(_t3, _t1); \ + _out3 = __lsx_vilvh_d(_t3, _t1); \ + } + +/* + * ============================================================================= + * Description : Transpose 8x8 block with byte elements in vectors + * Arguments : Inputs - _in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7 + * Outputs - _out0, _out1, _out2, _out3, _out4, _out5, _out6, + * _out7 + * Details : The rows of the matrix become columns, and the columns + * become rows. + * Example : LSX_TRANSPOSE8x8_B + * _in0 : 00,01,02,03,04,05,06,07, 00,00,00,00,00,00,00,00 + * _in1 : 10,11,12,13,14,15,16,17, 00,00,00,00,00,00,00,00 + * _in2 : 20,21,22,23,24,25,26,27, 00,00,00,00,00,00,00,00 + * _in3 : 30,31,32,33,34,35,36,37, 00,00,00,00,00,00,00,00 + * _in4 : 40,41,42,43,44,45,46,47, 00,00,00,00,00,00,00,00 + * _in5 : 50,51,52,53,54,55,56,57, 00,00,00,00,00,00,00,00 + * _in6 : 60,61,62,63,64,65,66,67, 00,00,00,00,00,00,00,00 + * _in7 : 70,71,72,73,74,75,76,77, 00,00,00,00,00,00,00,00 + * + * _ out0 : 00,10,20,30,40,50,60,70, 00,00,00,00,00,00,00,00 + * _ out1 : 01,11,21,31,41,51,61,71, 00,00,00,00,00,00,00,00 + * _ out2 : 02,12,22,32,42,52,62,72, 00,00,00,00,00,00,00,00 + * _ out3 : 03,13,23,33,43,53,63,73, 00,00,00,00,00,00,00,00 + * _ out4 : 04,14,24,34,44,54,64,74, 00,00,00,00,00,00,00,00 + * _ out5 : 05,15,25,35,45,55,65,75, 00,00,00,00,00,00,00,00 + * _ out6 : 06,16,26,36,46,56,66,76, 00,00,00,00,00,00,00,00 + * _ out7 : 07,17,27,37,47,57,67,77, 00,00,00,00,00,00,00,00 + * ============================================================================= + */ +#define LSX_TRANSPOSE8x8_B(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ + _out7) \ + { \ + __m128i zero = { 0 }; \ + __m128i shuf8 = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 }; \ + __m128i _t0, _t1, _t2, _t3, _t4, _t5, _t6, _t7; \ + \ + _t0 = __lsx_vilvl_b(_in2, _in0); \ + _t1 = __lsx_vilvl_b(_in3, _in1); \ + _t2 = __lsx_vilvl_b(_in6, _in4); \ + _t3 = __lsx_vilvl_b(_in7, _in5); \ + _t4 = __lsx_vilvl_b(_t1, _t0); \ + _t5 = __lsx_vilvh_b(_t1, _t0); \ + _t6 = __lsx_vilvl_b(_t3, _t2); \ + _t7 = __lsx_vilvh_b(_t3, _t2); \ + _out0 = __lsx_vilvl_w(_t6, _t4); \ + _out2 = __lsx_vilvh_w(_t6, _t4); \ + _out4 = __lsx_vilvl_w(_t7, _t5); \ + _out6 = __lsx_vilvh_w(_t7, _t5); \ + _out1 = __lsx_vshuf_b(zero, _out0, shuf8); \ + _out3 = __lsx_vshuf_b(zero, _out2, shuf8); \ + _out5 = __lsx_vshuf_b(zero, _out4, shuf8); \ + _out7 = __lsx_vshuf_b(zero, _out6, shuf8); \ + } + +/* + * ============================================================================= + * Description : Transpose 8x8 block with half-word elements in vectors + * Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7 + * Outputs - out0, out1, out2, out3, out4, out5, out6, out7 + * Details : + * Example : + * 00,01,02,03,04,05,06,07 00,10,20,30,40,50,60,70 + * 10,11,12,13,14,15,16,17 01,11,21,31,41,51,61,71 + * 20,21,22,23,24,25,26,27 02,12,22,32,42,52,62,72 + * 30,31,32,33,34,35,36,37 to 03,13,23,33,43,53,63,73 + * 40,41,42,43,44,45,46,47 ======> 04,14,24,34,44,54,64,74 + * 50,51,52,53,54,55,56,57 05,15,25,35,45,55,65,75 + * 60,61,62,63,64,65,66,67 06,16,26,36,46,56,66,76 + * 70,71,72,73,74,75,76,77 07,17,27,37,47,57,67,77 + * ============================================================================= + */ +#define LSX_TRANSPOSE8x8_H(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ + _out7) \ + { \ + __m128i _s0, _s1, _t0, _t1, _t2, _t3, _t4, _t5, _t6, _t7; \ + \ + _s0 = __lsx_vilvl_h(_in6, _in4); \ + _s1 = __lsx_vilvl_h(_in7, _in5); \ + _t0 = __lsx_vilvl_h(_s1, _s0); \ + _t1 = __lsx_vilvh_h(_s1, _s0); \ + _s0 = __lsx_vilvh_h(_in6, _in4); \ + _s1 = __lsx_vilvh_h(_in7, _in5); \ + _t2 = __lsx_vilvl_h(_s1, _s0); \ + _t3 = __lsx_vilvh_h(_s1, _s0); \ + _s0 = __lsx_vilvl_h(_in2, _in0); \ + _s1 = __lsx_vilvl_h(_in3, _in1); \ + _t4 = __lsx_vilvl_h(_s1, _s0); \ + _t5 = __lsx_vilvh_h(_s1, _s0); \ + _s0 = __lsx_vilvh_h(_in2, _in0); \ + _s1 = __lsx_vilvh_h(_in3, _in1); \ + _t6 = __lsx_vilvl_h(_s1, _s0); \ + _t7 = __lsx_vilvh_h(_s1, _s0); \ + \ + _out0 = __lsx_vpickev_d(_t0, _t4); \ + _out2 = __lsx_vpickev_d(_t1, _t5); \ + _out4 = __lsx_vpickev_d(_t2, _t6); \ + _out6 = __lsx_vpickev_d(_t3, _t7); \ + _out1 = __lsx_vpickod_d(_t0, _t4); \ + _out3 = __lsx_vpickod_d(_t1, _t5); \ + _out5 = __lsx_vpickod_d(_t2, _t6); \ + _out7 = __lsx_vpickod_d(_t3, _t7); \ + } + +/* + * ============================================================================= + * Description : Transpose input 8x4 byte block into 4x8 + * Arguments : Inputs - _in0, _in1, _in2, _in3 (input 8x4 byte block) + * Outputs - _out0, _out1, _out2, _out3 (output 4x8 byte block) + * Return Type - as per RTYPE + * Details : The rows of the matrix become columns, and the columns become + * rows. + * Example : LSX_TRANSPOSE8x4_B + * _in0 : 00,01,02,03,00,00,00,00, 00,00,00,00,00,00,00,00 + * _in1 : 10,11,12,13,00,00,00,00, 00,00,00,00,00,00,00,00 + * _in2 : 20,21,22,23,00,00,00,00, 00,00,00,00,00,00,00,00 + * _in3 : 30,31,32,33,00,00,00,00, 00,00,00,00,00,00,00,00 + * _in4 : 40,41,42,43,00,00,00,00, 00,00,00,00,00,00,00,00 + * _in5 : 50,51,52,53,00,00,00,00, 00,00,00,00,00,00,00,00 + * _in6 : 60,61,62,63,00,00,00,00, 00,00,00,00,00,00,00,00 + * _in7 : 70,71,72,73,00,00,00,00, 00,00,00,00,00,00,00,00 + * + * _out0 : 00,10,20,30,40,50,60,70, 00,00,00,00,00,00,00,00 + * _out1 : 01,11,21,31,41,51,61,71, 00,00,00,00,00,00,00,00 + * _out2 : 02,12,22,32,42,52,62,72, 00,00,00,00,00,00,00,00 + * _out3 : 03,13,23,33,43,53,63,73, 00,00,00,00,00,00,00,00 + * ============================================================================= + */ +#define LSX_TRANSPOSE8x4_B(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3) \ + { \ + __m128i _tmp0_m, _tmp1_m, _tmp2_m, _tmp3_m; \ + \ + _tmp0_m = __lsx_vpackev_w(_in4, _in0); \ + _tmp1_m = __lsx_vpackev_w(_in5, _in1); \ + _tmp2_m = __lsx_vilvl_b(_tmp1_m, _tmp0_m); \ + _tmp0_m = __lsx_vpackev_w(_in6, _in2); \ + _tmp1_m = __lsx_vpackev_w(_in7, _in3); \ + \ + _tmp3_m = __lsx_vilvl_b(_tmp1_m, _tmp0_m); \ + _tmp0_m = __lsx_vilvl_h(_tmp3_m, _tmp2_m); \ + _tmp1_m = __lsx_vilvh_h(_tmp3_m, _tmp2_m); \ + \ + _out0 = __lsx_vilvl_w(_tmp1_m, _tmp0_m); \ + _out2 = __lsx_vilvh_w(_tmp1_m, _tmp0_m); \ + _out1 = __lsx_vilvh_d(_out2, _out0); \ + _out3 = __lsx_vilvh_d(_out0, _out2); \ + } + +/* + * ============================================================================= + * Description : Transpose 16x8 block with byte elements in vectors + * Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7, in8 + * in9, in10, in11, in12, in13, in14, in15 + * Outputs - out0, out1, out2, out3, out4, out5, out6, out7 + * Details : + * Example : + * 000,001,002,003,004,005,006,007 + * 008,009,010,011,012,013,014,015 + * 016,017,018,019,020,021,022,023 + * 024,025,026,027,028,029,030,031 + * 032,033,034,035,036,037,038,039 + * 040,041,042,043,044,045,046,047 000,008,...,112,120 + * 048,049,050,051,052,053,054,055 001,009,...,113,121 + * 056,057,058,059,060,061,062,063 to 002,010,...,114,122 + * 064,068,066,067,068,069,070,071 =====> 003,011,...,115,123 + * 072,073,074,075,076,077,078,079 004,012,...,116,124 + * 080,081,082,083,084,085,086,087 005,013,...,117,125 + * 088,089,090,091,092,093,094,095 006,014,...,118,126 + * 096,097,098,099,100,101,102,103 007,015,...,119,127 + * 104,105,106,107,108,109,110,111 + * 112,113,114,115,116,117,118,119 + * 120,121,122,123,124,125,126,127 + * ============================================================================= + */ +#define LSX_TRANSPOSE16x8_B(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _in8, _in9, _in10, _in11, _in12, _in13, _in14, \ + _in15, _out0, _out1, _out2, _out3, _out4, _out5, \ + _out6, _out7) \ + { \ + __m128i _tmp0, _tmp1, _tmp2, _tmp3, _tmp4, _tmp5, _tmp6, _tmp7; \ + __m128i _t0, _t1, _t2, _t3, _t4, _t5, _t6, _t7; \ + DUP4_ARG2(__lsx_vilvl_b, _in2, _in0, _in3, _in1, _in6, _in4, _in7, _in5, \ + _tmp0, _tmp1, _tmp2, _tmp3); \ + DUP4_ARG2(__lsx_vilvl_b, _in10, _in8, _in11, _in9, _in14, _in12, _in15, \ + _in13, _tmp4, _tmp5, _tmp6, _tmp7); \ + DUP2_ARG2(__lsx_vilvl_b, _tmp1, _tmp0, _tmp3, _tmp2, _t0, _t2); \ + DUP2_ARG2(__lsx_vilvh_b, _tmp1, _tmp0, _tmp3, _tmp2, _t1, _t3); \ + DUP2_ARG2(__lsx_vilvl_b, _tmp5, _tmp4, _tmp7, _tmp6, _t4, _t6); \ + DUP2_ARG2(__lsx_vilvh_b, _tmp5, _tmp4, _tmp7, _tmp6, _t5, _t7); \ + DUP2_ARG2(__lsx_vilvl_w, _t2, _t0, _t3, _t1, _tmp0, _tmp4); \ + DUP2_ARG2(__lsx_vilvh_w, _t2, _t0, _t3, _t1, _tmp2, _tmp6); \ + DUP2_ARG2(__lsx_vilvl_w, _t6, _t4, _t7, _t5, _tmp1, _tmp5); \ + DUP2_ARG2(__lsx_vilvh_w, _t6, _t4, _t7, _t5, _tmp3, _tmp7); \ + DUP2_ARG2(__lsx_vilvl_d, _tmp1, _tmp0, _tmp3, _tmp2, _out0, _out2); \ + DUP2_ARG2(__lsx_vilvh_d, _tmp1, _tmp0, _tmp3, _tmp2, _out1, _out3); \ + DUP2_ARG2(__lsx_vilvl_d, _tmp5, _tmp4, _tmp7, _tmp6, _out4, _out6); \ + DUP2_ARG2(__lsx_vilvh_d, _tmp5, _tmp4, _tmp7, _tmp6, _out5, _out7); \ + } + +/* + * ============================================================================= + * Description : Butterfly of 4 input vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1, out2, out3 + * Details : Butterfly operation + * Example : + * out0 = in0 + in3; + * out1 = in1 + in2; + * out2 = in1 - in2; + * out3 = in0 - in3; + * ============================================================================= + */ +#define LSX_BUTTERFLY_4_B(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ + { \ + _out0 = __lsx_vadd_b(_in0, _in3); \ + _out1 = __lsx_vadd_b(_in1, _in2); \ + _out2 = __lsx_vsub_b(_in1, _in2); \ + _out3 = __lsx_vsub_b(_in0, _in3); \ + } +#define LSX_BUTTERFLY_4_H(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ + { \ + _out0 = __lsx_vadd_h(_in0, _in3); \ + _out1 = __lsx_vadd_h(_in1, _in2); \ + _out2 = __lsx_vsub_h(_in1, _in2); \ + _out3 = __lsx_vsub_h(_in0, _in3); \ + } +#define LSX_BUTTERFLY_4_W(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ + { \ + _out0 = __lsx_vadd_w(_in0, _in3); \ + _out1 = __lsx_vadd_w(_in1, _in2); \ + _out2 = __lsx_vsub_w(_in1, _in2); \ + _out3 = __lsx_vsub_w(_in0, _in3); \ + } +#define LSX_BUTTERFLY_4_D(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ + { \ + _out0 = __lsx_vadd_d(_in0, _in3); \ + _out1 = __lsx_vadd_d(_in1, _in2); \ + _out2 = __lsx_vsub_d(_in1, _in2); \ + _out3 = __lsx_vsub_d(_in0, _in3); \ + } + +/* + * ============================================================================= + * Description : Butterfly of 8 input vectors + * Arguments : Inputs - _in0, _in1, _in2, _in3, ~ + * Outputs - _out0, _out1, _out2, _out3, ~ + * Details : Butterfly operation + * Example : + * _out0 = _in0 + _in7; + * _out1 = _in1 + _in6; + * _out2 = _in2 + _in5; + * _out3 = _in3 + _in4; + * _out4 = _in3 - _in4; + * _out5 = _in2 - _in5; + * _out6 = _in1 - _in6; + * _out7 = _in0 - _in7; + * ============================================================================= + */ +#define LSX_BUTTERFLY_8_B(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ + _out7) \ + { \ + _out0 = __lsx_vadd_b(_in0, _in7); \ + _out1 = __lsx_vadd_b(_in1, _in6); \ + _out2 = __lsx_vadd_b(_in2, _in5); \ + _out3 = __lsx_vadd_b(_in3, _in4); \ + _out4 = __lsx_vsub_b(_in3, _in4); \ + _out5 = __lsx_vsub_b(_in2, _in5); \ + _out6 = __lsx_vsub_b(_in1, _in6); \ + _out7 = __lsx_vsub_b(_in0, _in7); \ + } + +#define LSX_BUTTERFLY_8_H(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ + _out7) \ + { \ + _out0 = __lsx_vadd_h(_in0, _in7); \ + _out1 = __lsx_vadd_h(_in1, _in6); \ + _out2 = __lsx_vadd_h(_in2, _in5); \ + _out3 = __lsx_vadd_h(_in3, _in4); \ + _out4 = __lsx_vsub_h(_in3, _in4); \ + _out5 = __lsx_vsub_h(_in2, _in5); \ + _out6 = __lsx_vsub_h(_in1, _in6); \ + _out7 = __lsx_vsub_h(_in0, _in7); \ + } + +#define LSX_BUTTERFLY_8_W(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ + _out7) \ + { \ + _out0 = __lsx_vadd_w(_in0, _in7); \ + _out1 = __lsx_vadd_w(_in1, _in6); \ + _out2 = __lsx_vadd_w(_in2, _in5); \ + _out3 = __lsx_vadd_w(_in3, _in4); \ + _out4 = __lsx_vsub_w(_in3, _in4); \ + _out5 = __lsx_vsub_w(_in2, _in5); \ + _out6 = __lsx_vsub_w(_in1, _in6); \ + _out7 = __lsx_vsub_w(_in0, _in7); \ + } + +#define LSX_BUTTERFLY_8_D(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ + _out7) \ + { \ + _out0 = __lsx_vadd_d(_in0, _in7); \ + _out1 = __lsx_vadd_d(_in1, _in6); \ + _out2 = __lsx_vadd_d(_in2, _in5); \ + _out3 = __lsx_vadd_d(_in3, _in4); \ + _out4 = __lsx_vsub_d(_in3, _in4); \ + _out5 = __lsx_vsub_d(_in2, _in5); \ + _out6 = __lsx_vsub_d(_in1, _in6); \ + _out7 = __lsx_vsub_d(_in0, _in7); \ + } + +/* + * ============================================================================= + * Description : Butterfly of 16 input vectors + * Arguments : Inputs - _in0, _in1, _in2, _in3, ~ + * Outputs - _out0, _out1, _out2, _out3, ~ + * Details : Butterfly operation + * Example : + * _out0 = _in0 + _in15; + * _out1 = _in1 + _in14; + * _out2 = _in2 + _in13; + * _out3 = _in3 + _in12; + * _out4 = _in4 + _in11; + * _out5 = _in5 + _in10; + * _out6 = _in6 + _in9; + * _out7 = _in7 + _in8; + * _out8 = _in7 - _in8; + * _out9 = _in6 - _in9; + * _out10 = _in5 - _in10; + * _out11 = _in4 - _in11; + * _out12 = _in3 - _in12; + * _out13 = _in2 - _in13; + * _out14 = _in1 - _in14; + * _out15 = _in0 - _in15; + * ============================================================================= + */ + +#define LSX_BUTTERFLY_16_B(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _in8, _in9, _in10, _in11, _in12, _in13, _in14, \ + _in15, _out0, _out1, _out2, _out3, _out4, _out5, \ + _out6, _out7, _out8, _out9, _out10, _out11, _out12, \ + _out13, _out14, _out15) \ + { \ + _out0 = __lsx_vadd_b(_in0, _in15); \ + _out1 = __lsx_vadd_b(_in1, _in14); \ + _out2 = __lsx_vadd_b(_in2, _in13); \ + _out3 = __lsx_vadd_b(_in3, _in12); \ + _out4 = __lsx_vadd_b(_in4, _in11); \ + _out5 = __lsx_vadd_b(_in5, _in10); \ + _out6 = __lsx_vadd_b(_in6, _in9); \ + _out7 = __lsx_vadd_b(_in7, _in8); \ + \ + _out8 = __lsx_vsub_b(_in7, _in8); \ + _out9 = __lsx_vsub_b(_in6, _in9); \ + _out10 = __lsx_vsub_b(_in5, _in10); \ + _out11 = __lsx_vsub_b(_in4, _in11); \ + _out12 = __lsx_vsub_b(_in3, _in12); \ + _out13 = __lsx_vsub_b(_in2, _in13); \ + _out14 = __lsx_vsub_b(_in1, _in14); \ + _out15 = __lsx_vsub_b(_in0, _in15); \ + } + +#define LSX_BUTTERFLY_16_H(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _in8, _in9, _in10, _in11, _in12, _in13, _in14, \ + _in15, _out0, _out1, _out2, _out3, _out4, _out5, \ + _out6, _out7, _out8, _out9, _out10, _out11, _out12, \ + _out13, _out14, _out15) \ + { \ + _out0 = __lsx_vadd_h(_in0, _in15); \ + _out1 = __lsx_vadd_h(_in1, _in14); \ + _out2 = __lsx_vadd_h(_in2, _in13); \ + _out3 = __lsx_vadd_h(_in3, _in12); \ + _out4 = __lsx_vadd_h(_in4, _in11); \ + _out5 = __lsx_vadd_h(_in5, _in10); \ + _out6 = __lsx_vadd_h(_in6, _in9); \ + _out7 = __lsx_vadd_h(_in7, _in8); \ + \ + _out8 = __lsx_vsub_h(_in7, _in8); \ + _out9 = __lsx_vsub_h(_in6, _in9); \ + _out10 = __lsx_vsub_h(_in5, _in10); \ + _out11 = __lsx_vsub_h(_in4, _in11); \ + _out12 = __lsx_vsub_h(_in3, _in12); \ + _out13 = __lsx_vsub_h(_in2, _in13); \ + _out14 = __lsx_vsub_h(_in1, _in14); \ + _out15 = __lsx_vsub_h(_in0, _in15); \ + } + +#define LSX_BUTTERFLY_16_W(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _in8, _in9, _in10, _in11, _in12, _in13, _in14, \ + _in15, _out0, _out1, _out2, _out3, _out4, _out5, \ + _out6, _out7, _out8, _out9, _out10, _out11, _out12, \ + _out13, _out14, _out15) \ + { \ + _out0 = __lsx_vadd_w(_in0, _in15); \ + _out1 = __lsx_vadd_w(_in1, _in14); \ + _out2 = __lsx_vadd_w(_in2, _in13); \ + _out3 = __lsx_vadd_w(_in3, _in12); \ + _out4 = __lsx_vadd_w(_in4, _in11); \ + _out5 = __lsx_vadd_w(_in5, _in10); \ + _out6 = __lsx_vadd_w(_in6, _in9); \ + _out7 = __lsx_vadd_w(_in7, _in8); \ + \ + _out8 = __lsx_vsub_w(_in7, _in8); \ + _out9 = __lsx_vsub_w(_in6, _in9); \ + _out10 = __lsx_vsub_w(_in5, _in10); \ + _out11 = __lsx_vsub_w(_in4, _in11); \ + _out12 = __lsx_vsub_w(_in3, _in12); \ + _out13 = __lsx_vsub_w(_in2, _in13); \ + _out14 = __lsx_vsub_w(_in1, _in14); \ + _out15 = __lsx_vsub_w(_in0, _in15); \ + } + +#define LSX_BUTTERFLY_16_D(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _in8, _in9, _in10, _in11, _in12, _in13, _in14, \ + _in15, _out0, _out1, _out2, _out3, _out4, _out5, \ + _out6, _out7, _out8, _out9, _out10, _out11, _out12, \ + _out13, _out14, _out15) \ + { \ + _out0 = __lsx_vadd_d(_in0, _in15); \ + _out1 = __lsx_vadd_d(_in1, _in14); \ + _out2 = __lsx_vadd_d(_in2, _in13); \ + _out3 = __lsx_vadd_d(_in3, _in12); \ + _out4 = __lsx_vadd_d(_in4, _in11); \ + _out5 = __lsx_vadd_d(_in5, _in10); \ + _out6 = __lsx_vadd_d(_in6, _in9); \ + _out7 = __lsx_vadd_d(_in7, _in8); \ + \ + _out8 = __lsx_vsub_d(_in7, _in8); \ + _out9 = __lsx_vsub_d(_in6, _in9); \ + _out10 = __lsx_vsub_d(_in5, _in10); \ + _out11 = __lsx_vsub_d(_in4, _in11); \ + _out12 = __lsx_vsub_d(_in3, _in12); \ + _out13 = __lsx_vsub_d(_in2, _in13); \ + _out14 = __lsx_vsub_d(_in1, _in14); \ + _out15 = __lsx_vsub_d(_in0, _in15); \ + } + +#endif // LSX + +#ifdef __loongarch_asx +#include +/* + * ============================================================================= + * Description : Dot product of byte vector elements + * Arguments : Inputs - in_h, in_l + * Output - out + * Return Type - signed halfword + * Details : Unsigned byte elements from in_h are multiplied with + * unsigned byte elements from in_l producing a result + * twice the size of input i.e. signed halfword. + * Then these multiplied results of adjacent odd-even elements + * are added to the out vector + * Example : See out = __lasx_xvdp2_w_h(in_h, in_l) + * ============================================================================= + */ +static inline __m256i __lasx_xvdp2_h_bu(__m256i in_h, __m256i in_l) { + __m256i out; + + out = __lasx_xvmulwev_h_bu(in_h, in_l); + out = __lasx_xvmaddwod_h_bu(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of byte vector elements + * Arguments : Inputs - in_h, in_l + * Output - out + * Return Type - signed halfword + * Details : Signed byte elements from in_h are multiplied with + * signed byte elements from in_l producing a result + * twice the size of input i.e. signed halfword. + * Then these multiplication results of adjacent odd-even elements + * are added to the out vector + * Example : See out = __lasx_xvdp2_w_h(in_h, in_l) + * ============================================================================= + */ +static inline __m256i __lasx_xvdp2_h_b(__m256i in_h, __m256i in_l) { + __m256i out; + + out = __lasx_xvmulwev_h_b(in_h, in_l); + out = __lasx_xvmaddwod_h_b(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of halfword vector elements + * Arguments : Inputs - in_h, in_l + * Output - out + * Return Type - signed word + * Details : Signed halfword elements from in_h are multiplied with + * signed halfword elements from in_l producing a result + * twice the size of input i.e. signed word. + * Then these multiplied results of adjacent odd-even elements + * are added to the out vector. + * Example : out = __lasx_xvdp2_w_h(in_h, in_l) + * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * in_l : 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,1 + * out : 22,38,38,22, 22,38,38,22 + * ============================================================================= + */ +static inline __m256i __lasx_xvdp2_w_h(__m256i in_h, __m256i in_l) { + __m256i out; + + out = __lasx_xvmulwev_w_h(in_h, in_l); + out = __lasx_xvmaddwod_w_h(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of word vector elements + * Arguments : Inputs - in_h, in_l + * Output - out + * Return Type - signed double + * Details : Signed word elements from in_h are multiplied with + * signed word elements from in_l producing a result + * twice the size of input i.e. signed double-word. + * Then these multiplied results of adjacent odd-even elements + * are added to the out vector. + * Example : See out = __lasx_xvdp2_w_h(in_h, in_l) + * ============================================================================= + */ +static inline __m256i __lasx_xvdp2_d_w(__m256i in_h, __m256i in_l) { + __m256i out; + + out = __lasx_xvmulwev_d_w(in_h, in_l); + out = __lasx_xvmaddwod_d_w(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of halfword vector elements + * Arguments : Inputs - in_h, in_l + * Output - out + * Return Type - signed word + * Details : Unsigned halfword elements from in_h are multiplied with + * signed halfword elements from in_l producing a result + * twice the size of input i.e. unsigned word. + * Multiplication result of adjacent odd-even elements + * are added to the out vector + * Example : See out = __lasx_xvdp2_w_h(in_h, in_l) + * ============================================================================= + */ +static inline __m256i __lasx_xvdp2_w_hu_h(__m256i in_h, __m256i in_l) { + __m256i out; + + out = __lasx_xvmulwev_w_hu_h(in_h, in_l); + out = __lasx_xvmaddwod_w_hu_h(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product & addition of byte vector elements + * Arguments : Inputs - in_h, in_l + * Output - out + * Return Type - halfword + * Details : Signed byte elements from in_h are multiplied with + * signed byte elements from in_l producing a result + * twice the size of input i.e. signed halfword. + * Then these multiplied results of adjacent odd-even elements + * are added to the in_c vector. + * Example : See out = __lasx_xvdp2add_w_h(in_c, in_h, in_l) + * ============================================================================= + */ +static inline __m256i __lasx_xvdp2add_h_b(__m256i in_c, __m256i in_h, + __m256i in_l) { + __m256i out; + + out = __lasx_xvmaddwev_h_b(in_c, in_h, in_l); + out = __lasx_xvmaddwod_h_b(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product & addition of byte vector elements + * Arguments : Inputs - in_h, in_l + * Output - out + * Return Type - halfword + * Details : Unsigned byte elements from in_h are multiplied with + * unsigned byte elements from in_l producing a result + * twice the size of input i.e. signed halfword. + * Then these multiplied results of adjacent odd-even elements + * are added to the in_c vector. + * Example : See out = __lasx_xvdp2add_w_h(in_c, in_h, in_l) + * ============================================================================= + */ +static inline __m256i __lasx_xvdp2add_h_bu(__m256i in_c, __m256i in_h, + __m256i in_l) { + __m256i out; + + out = __lasx_xvmaddwev_h_bu(in_c, in_h, in_l); + out = __lasx_xvmaddwod_h_bu(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product & addition of byte vector elements + * Arguments : Inputs - in_h, in_l + * Output - out + * Return Type - halfword + * Details : Unsigned byte elements from in_h are multiplied with + * signed byte elements from in_l producing a result + * twice the size of input i.e. signed halfword. + * Then these multiplied results of adjacent odd-even elements + * are added to the in_c vector. + * Example : See out = __lasx_xvdp2add_w_h(in_c, in_h, in_l) + * ============================================================================= + */ +static inline __m256i __lasx_xvdp2add_h_bu_b(__m256i in_c, __m256i in_h, + __m256i in_l) { + __m256i out; + + out = __lasx_xvmaddwev_h_bu_b(in_c, in_h, in_l); + out = __lasx_xvmaddwod_h_bu_b(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of halfword vector elements + * Arguments : Inputs - in_c, in_h, in_l + * Output - out + * Return Type - per RTYPE + * Details : Signed halfword elements from in_h are multiplied with + * signed halfword elements from in_l producing a result + * twice the size of input i.e. signed word. + * Multiplication result of adjacent odd-even elements + * are added to the in_c vector. + * Example : out = __lasx_xvdp2add_w_h(in_c, in_h, in_l) + * in_c : 1,2,3,4, 1,2,3,4 + * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8, + * in_l : 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,1, + * out : 23,40,41,26, 23,40,41,26 + * ============================================================================= + */ +static inline __m256i __lasx_xvdp2add_w_h(__m256i in_c, __m256i in_h, + __m256i in_l) { + __m256i out; + + out = __lasx_xvmaddwev_w_h(in_c, in_h, in_l); + out = __lasx_xvmaddwod_w_h(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of halfword vector elements + * Arguments : Inputs - in_c, in_h, in_l + * Output - out + * Return Type - signed word + * Details : Unsigned halfword elements from in_h are multiplied with + * unsigned halfword elements from in_l producing a result + * twice the size of input i.e. signed word. + * Multiplication result of adjacent odd-even elements + * are added to the in_c vector. + * Example : See out = __lasx_xvdp2add_w_h(in_c, in_h, in_l) + * ============================================================================= + */ +static inline __m256i __lasx_xvdp2add_w_hu(__m256i in_c, __m256i in_h, + __m256i in_l) { + __m256i out; + + out = __lasx_xvmaddwev_w_hu(in_c, in_h, in_l); + out = __lasx_xvmaddwod_w_hu(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of halfword vector elements + * Arguments : Inputs - in_c, in_h, in_l + * Output - out + * Return Type - signed word + * Details : Unsigned halfword elements from in_h are multiplied with + * signed halfword elements from in_l producing a result + * twice the size of input i.e. signed word. + * Multiplication result of adjacent odd-even elements + * are added to the in_c vector + * Example : See out = __lasx_xvdp2add_w_h(in_c, in_h, in_l) + * ============================================================================= + */ +static inline __m256i __lasx_xvdp2add_w_hu_h(__m256i in_c, __m256i in_h, + __m256i in_l) { + __m256i out; + + out = __lasx_xvmaddwev_w_hu_h(in_c, in_h, in_l); + out = __lasx_xvmaddwod_w_hu_h(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Vector Unsigned Dot Product and Subtract + * Arguments : Inputs - in_c, in_h, in_l + * Output - out + * Return Type - signed halfword + * Details : Unsigned byte elements from in_h are multiplied with + * unsigned byte elements from in_l producing a result + * twice the size of input i.e. signed halfword. + * Multiplication result of adjacent odd-even elements + * are added together and subtracted from double width elements + * in_c vector. + * Example : See out = __lasx_xvdp2sub_w_h(in_c, in_h, in_l) + * ============================================================================= + */ +static inline __m256i __lasx_xvdp2sub_h_bu(__m256i in_c, __m256i in_h, + __m256i in_l) { + __m256i out; + + out = __lasx_xvmulwev_h_bu(in_h, in_l); + out = __lasx_xvmaddwod_h_bu(out, in_h, in_l); + out = __lasx_xvsub_h(in_c, out); + return out; +} + +/* + * ============================================================================= + * Description : Vector Signed Dot Product and Subtract + * Arguments : Inputs - in_c, in_h, in_l + * Output - out + * Return Type - signed word + * Details : Signed halfword elements from in_h are multiplied with + * Signed halfword elements from in_l producing a result + * twice the size of input i.e. signed word. + * Multiplication result of adjacent odd-even elements + * are added together and subtracted from double width elements + * in_c vector. + * Example : out = __lasx_xvdp2sub_w_h(in_c, in_h, in_l) + * in_c : 0,0,0,0, 0,0,0,0 + * in_h : 3,1,3,0, 0,0,0,1, 0,0,1,1, 0,0,0,1 + * in_l : 2,1,1,0, 1,0,0,0, 0,0,1,0, 1,0,0,1 + * out : -7,-3,0,0, 0,-1,0,-1 + * ============================================================================= + */ +static inline __m256i __lasx_xvdp2sub_w_h(__m256i in_c, __m256i in_h, + __m256i in_l) { + __m256i out; + + out = __lasx_xvmulwev_w_h(in_h, in_l); + out = __lasx_xvmaddwod_w_h(out, in_h, in_l); + out = __lasx_xvsub_w(in_c, out); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of halfword vector elements + * Arguments : Inputs - in_h, in_l + * Output - out + * Return Type - signed word + * Details : Signed halfword elements from in_h are multiplied with + * signed halfword elements from in_l producing a result + * four times the size of input i.e. signed doubleword. + * Then these multiplication results of four adjacent elements + * are added together and stored to the out vector. + * Example : out = __lasx_xvdp4_d_h(in_h, in_l) + * in_h : 3,1,3,0, 0,0,0,1, 0,0,1,-1, 0,0,0,1 + * in_l : -2,1,1,0, 1,0,0,0, 0,0,1, 0, 1,0,0,1 + * out : -2,0,1,1 + * ============================================================================= + */ +static inline __m256i __lasx_xvdp4_d_h(__m256i in_h, __m256i in_l) { + __m256i out; + + out = __lasx_xvmulwev_w_h(in_h, in_l); + out = __lasx_xvmaddwod_w_h(out, in_h, in_l); + out = __lasx_xvhaddw_d_w(out, out); + return out; +} + +/* + * ============================================================================= + * Description : The high half of the vector elements are expanded and + * added after being doubled. + * Arguments : Inputs - in_h, in_l + * Output - out + * Details : The in_h vector and the in_l vector are added after the + * higher half of the two-fold sign extension (signed byte + * to signed halfword) and stored to the out vector. + * Example : See out = __lasx_xvaddwh_w_h(in_h, in_l) + * ============================================================================= + */ +static inline __m256i __lasx_xvaddwh_h_b(__m256i in_h, __m256i in_l) { + __m256i out; + + out = __lasx_xvilvh_b(in_h, in_l); + out = __lasx_xvhaddw_h_b(out, out); + return out; +} + +/* + * ============================================================================= + * Description : The high half of the vector elements are expanded and + * added after being doubled. + * Arguments : Inputs - in_h, in_l + * Output - out + * Details : The in_h vector and the in_l vector are added after the + * higher half of the two-fold sign extension (signed halfword + * to signed word) and stored to the out vector. + * Example : out = __lasx_xvaddwh_w_h(in_h, in_l) + * in_h : 3, 0,3,0, 0,0,0,-1, 0,0,1,-1, 0,0,0,1 + * in_l : 2,-1,1,2, 1,0,0, 0, 1,0,1, 0, 1,0,0,1 + * out : 1,0,0,-1, 1,0,0, 2 + * ============================================================================= + */ +static inline __m256i __lasx_xvaddwh_w_h(__m256i in_h, __m256i in_l) { + __m256i out; + + out = __lasx_xvilvh_h(in_h, in_l); + out = __lasx_xvhaddw_w_h(out, out); + return out; +} + +/* + * ============================================================================= + * Description : The low half of the vector elements are expanded and + * added after being doubled. + * Arguments : Inputs - in_h, in_l + * Output - out + * Details : The in_h vector and the in_l vector are added after the + * lower half of the two-fold sign extension (signed byte + * to signed halfword) and stored to the out vector. + * Example : See out = __lasx_xvaddwl_w_h(in_h, in_l) + * ============================================================================= + */ +static inline __m256i __lasx_xvaddwl_h_b(__m256i in_h, __m256i in_l) { + __m256i out; + + out = __lasx_xvilvl_b(in_h, in_l); + out = __lasx_xvhaddw_h_b(out, out); + return out; +} + +/* + * ============================================================================= + * Description : The low half of the vector elements are expanded and + * added after being doubled. + * Arguments : Inputs - in_h, in_l + * Output - out + * Details : The in_h vector and the in_l vector are added after the + * lower half of the two-fold sign extension (signed halfword + * to signed word) and stored to the out vector. + * Example : out = __lasx_xvaddwl_w_h(in_h, in_l) + * in_h : 3, 0,3,0, 0,0,0,-1, 0,0,1,-1, 0,0,0,1 + * in_l : 2,-1,1,2, 1,0,0, 0, 1,0,1, 0, 1,0,0,1 + * out : 5,-1,4,2, 1,0,2,-1 + * ============================================================================= + */ +static inline __m256i __lasx_xvaddwl_w_h(__m256i in_h, __m256i in_l) { + __m256i out; + + out = __lasx_xvilvl_h(in_h, in_l); + out = __lasx_xvhaddw_w_h(out, out); + return out; +} + +/* + * ============================================================================= + * Description : The low half of the vector elements are expanded and + * added after being doubled. + * Arguments : Inputs - in_h, in_l + * Output - out + * Details : The out vector and the out vector are added after the + * lower half of the two-fold zero extension (unsigned byte + * to unsigned halfword) and stored to the out vector. + * Example : See out = __lasx_xvaddwl_w_h(in_h, in_l) + * ============================================================================= + */ +static inline __m256i __lasx_xvaddwl_h_bu(__m256i in_h, __m256i in_l) { + __m256i out; + + out = __lasx_xvilvl_b(in_h, in_l); + out = __lasx_xvhaddw_hu_bu(out, out); + return out; +} + +/* + * ============================================================================= + * Description : The low half of the vector elements are expanded and + * added after being doubled. + * Arguments : Inputs - in_h, in_l + * Output - out + * Details : The in_l vector after double zero extension (unsigned byte to + * signed halfword)，added to the in_h vector. + * Example : See out = __lasx_xvaddw_w_w_h(in_h, in_l) + * ============================================================================= + */ +static inline __m256i __lasx_xvaddw_h_h_bu(__m256i in_h, __m256i in_l) { + __m256i out; + + out = __lasx_xvsllwil_hu_bu(in_l, 0); + out = __lasx_xvadd_h(in_h, out); + return out; +} + +/* + * ============================================================================= + * Description : The low half of the vector elements are expanded and + * added after being doubled. + * Arguments : Inputs - in_h, in_l + * Output - out + * Details : The in_l vector after double sign extension (signed halfword to + * signed word), added to the in_h vector. + * Example : out = __lasx_xvaddw_w_w_h(in_h, in_l) + * in_h : 0, 1,0,0, -1,0,0,1, + * in_l : 2,-1,1,2, 1,0,0,0, 0,0,1,0, 1,0,0,1, + * out : 2, 0,1,2, -1,0,1,1, + * ============================================================================= + */ +static inline __m256i __lasx_xvaddw_w_w_h(__m256i in_h, __m256i in_l) { + __m256i out; + + out = __lasx_xvsllwil_w_h(in_l, 0); + out = __lasx_xvadd_w(in_h, out); + return out; +} + +/* + * ============================================================================= + * Description : Multiplication and addition calculation after expansion + * of the lower half of the vector. + * Arguments : Inputs - in_c, in_h, in_l + * Output - out + * Details : The in_h vector and the in_l vector are multiplied after + * the lower half of the two-fold sign extension (signed halfword + * to signed word), and the result is added to the vector in_c, + * then stored to the out vector. + * Example : out = __lasx_xvmaddwl_w_h(in_c, in_h, in_l) + * in_c : 1,2,3,4, 5,6,7,8 + * in_h : 1,2,3,4, 1,2,3,4, 5,6,7,8, 5,6,7,8 + * in_l : 200, 300, 400, 500, 2000, 3000, 4000, 5000, + * -200,-300,-400,-500, -2000,-3000,-4000,-5000 + * out : 201, 602,1203,2004, -995, -1794,-2793,-3992 + * ============================================================================= + */ +static inline __m256i __lasx_xvmaddwl_w_h(__m256i in_c, __m256i in_h, + __m256i in_l) { + __m256i tmp0, tmp1, out; + + tmp0 = __lasx_xvsllwil_w_h(in_h, 0); + tmp1 = __lasx_xvsllwil_w_h(in_l, 0); + tmp0 = __lasx_xvmul_w(tmp0, tmp1); + out = __lasx_xvadd_w(tmp0, in_c); + return out; +} + +/* + * ============================================================================= + * Description : Multiplication and addition calculation after expansion + * of the higher half of the vector. + * Arguments : Inputs - in_c, in_h, in_l + * Output - out + * Details : The in_h vector and the in_l vector are multiplied after + * the higher half of the two-fold sign extension (signed + * halfword to signed word), and the result is added to + * the vector in_c, then stored to the out vector. + * Example : See out = __lasx_xvmaddwl_w_h(in_c, in_h, in_l) + * ============================================================================= + */ +static inline __m256i __lasx_xvmaddwh_w_h(__m256i in_c, __m256i in_h, + __m256i in_l) { + __m256i tmp0, tmp1, out; + + tmp0 = __lasx_xvilvh_h(in_h, in_h); + tmp1 = __lasx_xvilvh_h(in_l, in_l); + tmp0 = __lasx_xvmulwev_w_h(tmp0, tmp1); + out = __lasx_xvadd_w(tmp0, in_c); + return out; +} + +/* + * ============================================================================= + * Description : Multiplication calculation after expansion of the lower + * half of the vector. + * Arguments : Inputs - in_h, in_l + * Output - out + * Details : The in_h vector and the in_l vector are multiplied after + * the lower half of the two-fold sign extension (signed + * halfword to signed word), then stored to the out vector. + * Example : out = __lasx_xvmulwl_w_h(in_h, in_l) + * in_h : 3,-1,3,0, 0,0,0,-1, 0,0,1,-1, 0,0,0,1 + * in_l : 2,-1,1,2, 1,0,0, 0, 0,0,1, 0, 1,0,0,1 + * out : 6,1,3,0, 0,0,1,0 + * ============================================================================= + */ +static inline __m256i __lasx_xvmulwl_w_h(__m256i in_h, __m256i in_l) { + __m256i tmp0, tmp1, out; + + tmp0 = __lasx_xvsllwil_w_h(in_h, 0); + tmp1 = __lasx_xvsllwil_w_h(in_l, 0); + out = __lasx_xvmul_w(tmp0, tmp1); + return out; +} + +/* + * ============================================================================= + * Description : Multiplication calculation after expansion of the lower + * half of the vector. + * Arguments : Inputs - in_h, in_l + * Output - out + * Details : The in_h vector and the in_l vector are multiplied after + * the lower half of the two-fold sign extension (signed + * halfword to signed word), then stored to the out vector. + * Example : out = __lasx_xvmulwh_w_h(in_h, in_l) + * in_h : 3,-1,3,0, 0,0,0,-1, 0,0,1,-1, 0,0,0,1 + * in_l : 2,-1,1,2, 1,0,0, 0, 0,0,1, 0, 1,0,0,1 + * out : 0,0,0,0, 0,0,0,1 + * ============================================================================= + */ +static inline __m256i __lasx_xvmulwh_w_h(__m256i in_h, __m256i in_l) { + __m256i tmp0, tmp1, out; + + tmp0 = __lasx_xvilvh_h(in_h, in_h); + tmp1 = __lasx_xvilvh_h(in_l, in_l); + out = __lasx_xvmulwev_w_h(tmp0, tmp1); + return out; +} + +/* + * ============================================================================= + * Description : The low half of the vector elements are added to the high half + * after being doubled, then saturated. + * Arguments : Inputs - in_h, in_l + * Output - out + * Details : The in_h vector adds the in_l vector after the lower half of + * the two-fold zero extension (unsigned byte to unsigned + * halfword) and then saturated. The results are stored to the out + * vector. + * Example : out = __lasx_xvsaddw_hu_hu_bu(in_h, in_l) + * in_h : 2,65532,1,2, 1,0,0,0, 0,0,1,0, 1,0,0,1 + * in_l : 3,6,3,0, 0,0,0,1, 0,0,1,1, 0,0,0,1, 3,18,3,0, 0,0,0,1, 0,0,1,1, + * 0,0,0,1 + * out : 5,65535,4,2, 1,0,0,1, 3,18,4,0, 1,0,0,2, + * ============================================================================= + */ +static inline __m256i __lasx_xvsaddw_hu_hu_bu(__m256i in_h, __m256i in_l) { + __m256i tmp1, out; + __m256i zero = { 0 }; + + tmp1 = __lasx_xvilvl_b(zero, in_l); + out = __lasx_xvsadd_hu(in_h, tmp1); + return out; +} + +/* + * ============================================================================= + * Description : Clip all halfword elements of input vector between min & max + * out = ((in) < (min)) ? (min) : (((in) > (max)) ? (max) : (in)) + * Arguments : Inputs - in (input vector) + * - min (min threshold) + * - max (max threshold) + * Outputs - in (output vector with clipped elements) + * Return Type - signed halfword + * Example : out = __lasx_xvclip_h(in, min, max) + * in : -8,2,280,249, -8,255,280,249, 4,4,4,4, 5,5,5,5 + * min : 1,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1 + * max : 9,9,9,9, 9,9,9,9, 9,9,9,9, 9,9,9,9 + * out : 1,2,9,9, 1,9,9,9, 4,4,4,4, 5,5,5,5 + * ============================================================================= + */ +static inline __m256i __lasx_xvclip_h(__m256i in, __m256i min, __m256i max) { + __m256i out; + + out = __lasx_xvmax_h(min, in); + out = __lasx_xvmin_h(max, out); + return out; +} + +/* + * ============================================================================= + * Description : Clip all signed halfword elements of input vector + * between 0 & 255 + * Arguments : Inputs - in (input vector) + * Outputs - out (output vector with clipped elements) + * Return Type - signed halfword + * Example : See out = __lasx_xvclip255_w(in) + * ============================================================================= + */ +static inline __m256i __lasx_xvclip255_h(__m256i in) { + __m256i out; + + out = __lasx_xvmaxi_h(in, 0); + out = __lasx_xvsat_hu(out, 7); + return out; +} + +/* + * ============================================================================= + * Description : Clip all signed word elements of input vector + * between 0 & 255 + * Arguments : Inputs - in (input vector) + * Output - out (output vector with clipped elements) + * Return Type - signed word + * Example : out = __lasx_xvclip255_w(in) + * in : -8,255,280,249, -8,255,280,249 + * out : 0,255,255,249, 0,255,255,249 + * ============================================================================= + */ +static inline __m256i __lasx_xvclip255_w(__m256i in) { + __m256i out; + + out = __lasx_xvmaxi_w(in, 0); + out = __lasx_xvsat_wu(out, 7); + return out; +} + +/* + * ============================================================================= + * Description : Indexed halfword element values are replicated to all + * elements in output vector. If 'idx < 8' use xvsplati_l_*, + * if 'idx >= 8' use xvsplati_h_*. + * Arguments : Inputs - in, idx + * Output - out + * Details : Idx element value from in vector is replicated to all + * elements in out vector. + * Valid index range for halfword operation is 0-7 + * Example : out = __lasx_xvsplati_l_h(in, idx) + * in : 20,10,11,12, 13,14,15,16, 0,0,2,0, 0,0,0,0 + * idx : 0x02 + * out : 11,11,11,11, 11,11,11,11, 11,11,11,11, 11,11,11,11 + * ============================================================================= + */ +static inline __m256i __lasx_xvsplati_l_h(__m256i in, int idx) { + __m256i out; + + out = __lasx_xvpermi_q(in, in, 0x02); + out = __lasx_xvreplve_h(out, idx); + return out; +} + +/* + * ============================================================================= + * Description : Indexed halfword element values are replicated to all + * elements in output vector. If 'idx < 8' use xvsplati_l_*, + * if 'idx >= 8' use xvsplati_h_*. + * Arguments : Inputs - in, idx + * Output - out + * Details : Idx element value from in vector is replicated to all + * elements in out vector. + * Valid index range for halfword operation is 0-7 + * Example : out = __lasx_xvsplati_h_h(in, idx) + * in : 20,10,11,12, 13,14,15,16, 0,2,0,0, 0,0,0,0 + * idx : 0x09 + * out : 2,2,2,2, 2,2,2,2, 2,2,2,2, 2,2,2,2 + * ============================================================================= + */ +static inline __m256i __lasx_xvsplati_h_h(__m256i in, int idx) { + __m256i out; + + out = __lasx_xvpermi_q(in, in, 0x13); + out = __lasx_xvreplve_h(out, idx); + return out; +} + +/* + * ============================================================================= + * Description : Transpose 4x4 block with double-word elements in vectors + * Arguments : Inputs - _in0, _in1, _in2, _in3 + * Outputs - _out0, _out1, _out2, _out3 + * Example : LASX_TRANSPOSE4x4_D + * _in0 : 1,2,3,4 + * _in1 : 1,2,3,4 + * _in2 : 1,2,3,4 + * _in3 : 1,2,3,4 + * + * _out0 : 1,1,1,1 + * _out1 : 2,2,2,2 + * _out2 : 3,3,3,3 + * _out3 : 4,4,4,4 + * ============================================================================= + */ +#define LASX_TRANSPOSE4x4_D(_in0, _in1, _in2, _in3, _out0, _out1, _out2, \ + _out3) \ + { \ + __m256i _tmp0, _tmp1, _tmp2, _tmp3; \ + _tmp0 = __lasx_xvilvl_d(_in1, _in0); \ + _tmp1 = __lasx_xvilvh_d(_in1, _in0); \ + _tmp2 = __lasx_xvilvl_d(_in3, _in2); \ + _tmp3 = __lasx_xvilvh_d(_in3, _in2); \ + _out0 = __lasx_xvpermi_q(_tmp2, _tmp0, 0x20); \ + _out2 = __lasx_xvpermi_q(_tmp2, _tmp0, 0x31); \ + _out1 = __lasx_xvpermi_q(_tmp3, _tmp1, 0x20); \ + _out3 = __lasx_xvpermi_q(_tmp3, _tmp1, 0x31); \ + } + +/* + * ============================================================================= + * Description : Transpose 8x8 block with word elements in vectors + * Arguments : Inputs - _in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7 + * Outputs - _out0, _out1, _out2, _out3, _out4, _out5, _out6, + * _out7 + * Example : LASX_TRANSPOSE8x8_W + * _in0 : 1,2,3,4,5,6,7,8 + * _in1 : 2,2,3,4,5,6,7,8 + * _in2 : 3,2,3,4,5,6,7,8 + * _in3 : 4,2,3,4,5,6,7,8 + * _in4 : 5,2,3,4,5,6,7,8 + * _in5 : 6,2,3,4,5,6,7,8 + * _in6 : 7,2,3,4,5,6,7,8 + * _in7 : 8,2,3,4,5,6,7,8 + * + * _out0 : 1,2,3,4,5,6,7,8 + * _out1 : 2,2,2,2,2,2,2,2 + * _out2 : 3,3,3,3,3,3,3,3 + * _out3 : 4,4,4,4,4,4,4,4 + * _out4 : 5,5,5,5,5,5,5,5 + * _out5 : 6,6,6,6,6,6,6,6 + * _out6 : 7,7,7,7,7,7,7,7 + * _out7 : 8,8,8,8,8,8,8,8 + * ============================================================================= + */ +#define LASX_TRANSPOSE8x8_W(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ + _out7) \ + { \ + __m256i _s0_m, _s1_m; \ + __m256i _tmp0_m, _tmp1_m, _tmp2_m, _tmp3_m; \ + __m256i _tmp4_m, _tmp5_m, _tmp6_m, _tmp7_m; \ + \ + _s0_m = __lasx_xvilvl_w(_in2, _in0); \ + _s1_m = __lasx_xvilvl_w(_in3, _in1); \ + _tmp0_m = __lasx_xvilvl_w(_s1_m, _s0_m); \ + _tmp1_m = __lasx_xvilvh_w(_s1_m, _s0_m); \ + _s0_m = __lasx_xvilvh_w(_in2, _in0); \ + _s1_m = __lasx_xvilvh_w(_in3, _in1); \ + _tmp2_m = __lasx_xvilvl_w(_s1_m, _s0_m); \ + _tmp3_m = __lasx_xvilvh_w(_s1_m, _s0_m); \ + _s0_m = __lasx_xvilvl_w(_in6, _in4); \ + _s1_m = __lasx_xvilvl_w(_in7, _in5); \ + _tmp4_m = __lasx_xvilvl_w(_s1_m, _s0_m); \ + _tmp5_m = __lasx_xvilvh_w(_s1_m, _s0_m); \ + _s0_m = __lasx_xvilvh_w(_in6, _in4); \ + _s1_m = __lasx_xvilvh_w(_in7, _in5); \ + _tmp6_m = __lasx_xvilvl_w(_s1_m, _s0_m); \ + _tmp7_m = __lasx_xvilvh_w(_s1_m, _s0_m); \ + _out0 = __lasx_xvpermi_q(_tmp4_m, _tmp0_m, 0x20); \ + _out1 = __lasx_xvpermi_q(_tmp5_m, _tmp1_m, 0x20); \ + _out2 = __lasx_xvpermi_q(_tmp6_m, _tmp2_m, 0x20); \ + _out3 = __lasx_xvpermi_q(_tmp7_m, _tmp3_m, 0x20); \ + _out4 = __lasx_xvpermi_q(_tmp4_m, _tmp0_m, 0x31); \ + _out5 = __lasx_xvpermi_q(_tmp5_m, _tmp1_m, 0x31); \ + _out6 = __lasx_xvpermi_q(_tmp6_m, _tmp2_m, 0x31); \ + _out7 = __lasx_xvpermi_q(_tmp7_m, _tmp3_m, 0x31); \ + } + +/* + * ============================================================================= + * Description : Transpose input 16x8 byte block + * Arguments : Inputs - _in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, + * _in8, _in9, _in10, _in11, _in12, _in13, _in14, _in15 + * (input 16x8 byte block) + * Outputs - _out0, _out1, _out2, _out3, _out4, _out5, _out6, + * _out7 (output 8x16 byte block) + * Details : The rows of the matrix become columns, and the columns become + * rows. + * Example : See LASX_TRANSPOSE16x8_H + * ============================================================================= + */ +#define LASX_TRANSPOSE16x8_B(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _in8, _in9, _in10, _in11, _in12, _in13, _in14, \ + _in15, _out0, _out1, _out2, _out3, _out4, _out5, \ + _out6, _out7) \ + { \ + __m256i _tmp0_m, _tmp1_m, _tmp2_m, _tmp3_m; \ + __m256i _tmp4_m, _tmp5_m, _tmp6_m, _tmp7_m; \ + \ + _tmp0_m = __lasx_xvilvl_b(_in2, _in0); \ + _tmp1_m = __lasx_xvilvl_b(_in3, _in1); \ + _tmp2_m = __lasx_xvilvl_b(_in6, _in4); \ + _tmp3_m = __lasx_xvilvl_b(_in7, _in5); \ + _tmp4_m = __lasx_xvilvl_b(_in10, _in8); \ + _tmp5_m = __lasx_xvilvl_b(_in11, _in9); \ + _tmp6_m = __lasx_xvilvl_b(_in14, _in12); \ + _tmp7_m = __lasx_xvilvl_b(_in15, _in13); \ + _out0 = __lasx_xvilvl_b(_tmp1_m, _tmp0_m); \ + _out1 = __lasx_xvilvh_b(_tmp1_m, _tmp0_m); \ + _out2 = __lasx_xvilvl_b(_tmp3_m, _tmp2_m); \ + _out3 = __lasx_xvilvh_b(_tmp3_m, _tmp2_m); \ + _out4 = __lasx_xvilvl_b(_tmp5_m, _tmp4_m); \ + _out5 = __lasx_xvilvh_b(_tmp5_m, _tmp4_m); \ + _out6 = __lasx_xvilvl_b(_tmp7_m, _tmp6_m); \ + _out7 = __lasx_xvilvh_b(_tmp7_m, _tmp6_m); \ + _tmp0_m = __lasx_xvilvl_w(_out2, _out0); \ + _tmp2_m = __lasx_xvilvh_w(_out2, _out0); \ + _tmp4_m = __lasx_xvilvl_w(_out3, _out1); \ + _tmp6_m = __lasx_xvilvh_w(_out3, _out1); \ + _tmp1_m = __lasx_xvilvl_w(_out6, _out4); \ + _tmp3_m = __lasx_xvilvh_w(_out6, _out4); \ + _tmp5_m = __lasx_xvilvl_w(_out7, _out5); \ + _tmp7_m = __lasx_xvilvh_w(_out7, _out5); \ + _out0 = __lasx_xvilvl_d(_tmp1_m, _tmp0_m); \ + _out1 = __lasx_xvilvh_d(_tmp1_m, _tmp0_m); \ + _out2 = __lasx_xvilvl_d(_tmp3_m, _tmp2_m); \ + _out3 = __lasx_xvilvh_d(_tmp3_m, _tmp2_m); \ + _out4 = __lasx_xvilvl_d(_tmp5_m, _tmp4_m); \ + _out5 = __lasx_xvilvh_d(_tmp5_m, _tmp4_m); \ + _out6 = __lasx_xvilvl_d(_tmp7_m, _tmp6_m); \ + _out7 = __lasx_xvilvh_d(_tmp7_m, _tmp6_m); \ + } + +/* + * ============================================================================= + * Description : Transpose input 16x8 byte block + * Arguments : Inputs - _in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, + * _in8, _in9, _in10, _in11, _in12, _in13, _in14, _in15 + * (input 16x8 byte block) + * Outputs - _out0, _out1, _out2, _out3, _out4, _out5, _out6, + * _out7 (output 8x16 byte block) + * Details : The rows of the matrix become columns, and the columns become + * rows. + * Example : LASX_TRANSPOSE16x8_H + * _in0 : 1,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * _in1 : 2,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * _in2 : 3,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * _in3 : 4,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * _in4 : 5,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * _in5 : 6,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * _in6 : 7,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * _in7 : 8,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * _in8 : 9,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * _in9 : 1,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * _in10 : 0,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * _in11 : 2,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * _in12 : 3,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * _in13 : 7,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * _in14 : 5,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * _in15 : 6,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * + * _out0 : 1,2,3,4,5,6,7,8,9,1,0,2,3,7,5,6 + * _out1 : 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2 + * _out2 : 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3 + * _out3 : 4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4 + * _out4 : 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 + * _out5 : 6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6 + * _out6 : 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7 + * _out7 : 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8 + * ============================================================================= + */ +#define LASX_TRANSPOSE16x8_H(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _in8, _in9, _in10, _in11, _in12, _in13, _in14, \ + _in15, _out0, _out1, _out2, _out3, _out4, _out5, \ + _out6, _out7) \ + { \ + __m256i _tmp0_m, _tmp1_m, _tmp2_m, _tmp3_m; \ + __m256i _tmp4_m, _tmp5_m, _tmp6_m, _tmp7_m; \ + __m256i _t0, _t1, _t2, _t3, _t4, _t5, _t6, _t7; \ + \ + _tmp0_m = __lasx_xvilvl_h(_in2, _in0); \ + _tmp1_m = __lasx_xvilvl_h(_in3, _in1); \ + _tmp2_m = __lasx_xvilvl_h(_in6, _in4); \ + _tmp3_m = __lasx_xvilvl_h(_in7, _in5); \ + _tmp4_m = __lasx_xvilvl_h(_in10, _in8); \ + _tmp5_m = __lasx_xvilvl_h(_in11, _in9); \ + _tmp6_m = __lasx_xvilvl_h(_in14, _in12); \ + _tmp7_m = __lasx_xvilvl_h(_in15, _in13); \ + _t0 = __lasx_xvilvl_h(_tmp1_m, _tmp0_m); \ + _t1 = __lasx_xvilvh_h(_tmp1_m, _tmp0_m); \ + _t2 = __lasx_xvilvl_h(_tmp3_m, _tmp2_m); \ + _t3 = __lasx_xvilvh_h(_tmp3_m, _tmp2_m); \ + _t4 = __lasx_xvilvl_h(_tmp5_m, _tmp4_m); \ + _t5 = __lasx_xvilvh_h(_tmp5_m, _tmp4_m); \ + _t6 = __lasx_xvilvl_h(_tmp7_m, _tmp6_m); \ + _t7 = __lasx_xvilvh_h(_tmp7_m, _tmp6_m); \ + _tmp0_m = __lasx_xvilvl_d(_t2, _t0); \ + _tmp2_m = __lasx_xvilvh_d(_t2, _t0); \ + _tmp4_m = __lasx_xvilvl_d(_t3, _t1); \ + _tmp6_m = __lasx_xvilvh_d(_t3, _t1); \ + _tmp1_m = __lasx_xvilvl_d(_t6, _t4); \ + _tmp3_m = __lasx_xvilvh_d(_t6, _t4); \ + _tmp5_m = __lasx_xvilvl_d(_t7, _t5); \ + _tmp7_m = __lasx_xvilvh_d(_t7, _t5); \ + _out0 = __lasx_xvpermi_q(_tmp1_m, _tmp0_m, 0x20); \ + _out1 = __lasx_xvpermi_q(_tmp3_m, _tmp2_m, 0x20); \ + _out2 = __lasx_xvpermi_q(_tmp5_m, _tmp4_m, 0x20); \ + _out3 = __lasx_xvpermi_q(_tmp7_m, _tmp6_m, 0x20); \ + \ + _tmp0_m = __lasx_xvilvh_h(_in2, _in0); \ + _tmp1_m = __lasx_xvilvh_h(_in3, _in1); \ + _tmp2_m = __lasx_xvilvh_h(_in6, _in4); \ + _tmp3_m = __lasx_xvilvh_h(_in7, _in5); \ + _tmp4_m = __lasx_xvilvh_h(_in10, _in8); \ + _tmp5_m = __lasx_xvilvh_h(_in11, _in9); \ + _tmp6_m = __lasx_xvilvh_h(_in14, _in12); \ + _tmp7_m = __lasx_xvilvh_h(_in15, _in13); \ + _t0 = __lasx_xvilvl_h(_tmp1_m, _tmp0_m); \ + _t1 = __lasx_xvilvh_h(_tmp1_m, _tmp0_m); \ + _t2 = __lasx_xvilvl_h(_tmp3_m, _tmp2_m); \ + _t3 = __lasx_xvilvh_h(_tmp3_m, _tmp2_m); \ + _t4 = __lasx_xvilvl_h(_tmp5_m, _tmp4_m); \ + _t5 = __lasx_xvilvh_h(_tmp5_m, _tmp4_m); \ + _t6 = __lasx_xvilvl_h(_tmp7_m, _tmp6_m); \ + _t7 = __lasx_xvilvh_h(_tmp7_m, _tmp6_m); \ + _tmp0_m = __lasx_xvilvl_d(_t2, _t0); \ + _tmp2_m = __lasx_xvilvh_d(_t2, _t0); \ + _tmp4_m = __lasx_xvilvl_d(_t3, _t1); \ + _tmp6_m = __lasx_xvilvh_d(_t3, _t1); \ + _tmp1_m = __lasx_xvilvl_d(_t6, _t4); \ + _tmp3_m = __lasx_xvilvh_d(_t6, _t4); \ + _tmp5_m = __lasx_xvilvl_d(_t7, _t5); \ + _tmp7_m = __lasx_xvilvh_d(_t7, _t5); \ + _out4 = __lasx_xvpermi_q(_tmp1_m, _tmp0_m, 0x20); \ + _out5 = __lasx_xvpermi_q(_tmp3_m, _tmp2_m, 0x20); \ + _out6 = __lasx_xvpermi_q(_tmp5_m, _tmp4_m, 0x20); \ + _out7 = __lasx_xvpermi_q(_tmp7_m, _tmp6_m, 0x20); \ + } + +/* + * ============================================================================= + * Description : Transpose 4x4 block with halfword elements in vectors + * Arguments : Inputs - _in0, _in1, _in2, _in3 + * Outputs - _out0, _out1, _out2, _out3 + * Return Type - signed halfword + * Details : The rows of the matrix become columns, and the columns become + * rows. + * Example : See LASX_TRANSPOSE8x8_H + * ============================================================================= + */ +#define LASX_TRANSPOSE4x4_H(_in0, _in1, _in2, _in3, _out0, _out1, _out2, \ + _out3) \ + { \ + __m256i _s0_m, _s1_m; \ + \ + _s0_m = __lasx_xvilvl_h(_in1, _in0); \ + _s1_m = __lasx_xvilvl_h(_in3, _in2); \ + _out0 = __lasx_xvilvl_w(_s1_m, _s0_m); \ + _out2 = __lasx_xvilvh_w(_s1_m, _s0_m); \ + _out1 = __lasx_xvilvh_d(_out0, _out0); \ + _out3 = __lasx_xvilvh_d(_out2, _out2); \ + } + +/* + * ============================================================================= + * Description : Transpose input 8x8 byte block + * Arguments : Inputs - _in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7 + * (input 8x8 byte block) + * Outputs - _out0, _out1, _out2, _out3, _out4, _out5, _out6, + * _out7 (output 8x8 byte block) + * Example : See LASX_TRANSPOSE8x8_H + * ============================================================================= + */ +#define LASX_TRANSPOSE8x8_B(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ + _out7) \ + { \ + __m256i _tmp0_m, _tmp1_m, _tmp2_m, _tmp3_m; \ + __m256i _tmp4_m, _tmp5_m, _tmp6_m, _tmp7_m; \ + _tmp0_m = __lasx_xvilvl_b(_in2, _in0); \ + _tmp1_m = __lasx_xvilvl_b(_in3, _in1); \ + _tmp2_m = __lasx_xvilvl_b(_in6, _in4); \ + _tmp3_m = __lasx_xvilvl_b(_in7, _in5); \ + _tmp4_m = __lasx_xvilvl_b(_tmp1_m, _tmp0_m); \ + _tmp5_m = __lasx_xvilvh_b(_tmp1_m, _tmp0_m); \ + _tmp6_m = __lasx_xvilvl_b(_tmp3_m, _tmp2_m); \ + _tmp7_m = __lasx_xvilvh_b(_tmp3_m, _tmp2_m); \ + _out0 = __lasx_xvilvl_w(_tmp6_m, _tmp4_m); \ + _out2 = __lasx_xvilvh_w(_tmp6_m, _tmp4_m); \ + _out4 = __lasx_xvilvl_w(_tmp7_m, _tmp5_m); \ + _out6 = __lasx_xvilvh_w(_tmp7_m, _tmp5_m); \ + _out1 = __lasx_xvbsrl_v(_out0, 8); \ + _out3 = __lasx_xvbsrl_v(_out2, 8); \ + _out5 = __lasx_xvbsrl_v(_out4, 8); \ + _out7 = __lasx_xvbsrl_v(_out6, 8); \ + } + +/* + * ============================================================================= + * Description : Transpose 8x8 block with halfword elements in vectors. + * Arguments : Inputs - _in0, _in1, ~ + * Outputs - _out0, _out1, ~ + * Details : The rows of the matrix become columns, and the columns become + * rows. + * Example : LASX_TRANSPOSE8x8_H + * _in0 : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * _in1 : 8,2,3,4, 5,6,7,8, 8,2,3,4, 5,6,7,8 + * _in2 : 8,2,3,4, 5,6,7,8, 8,2,3,4, 5,6,7,8 + * _in3 : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * _in4 : 9,2,3,4, 5,6,7,8, 9,2,3,4, 5,6,7,8 + * _in5 : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * _in6 : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * _in7 : 9,2,3,4, 5,6,7,8, 9,2,3,4, 5,6,7,8 + * + * _out0 : 1,8,8,1, 9,1,1,9, 1,8,8,1, 9,1,1,9 + * _out1 : 2,2,2,2, 2,2,2,2, 2,2,2,2, 2,2,2,2 + * _out2 : 3,3,3,3, 3,3,3,3, 3,3,3,3, 3,3,3,3 + * _out3 : 4,4,4,4, 4,4,4,4, 4,4,4,4, 4,4,4,4 + * _out4 : 5,5,5,5, 5,5,5,5, 5,5,5,5, 5,5,5,5 + * _out5 : 6,6,6,6, 6,6,6,6, 6,6,6,6, 6,6,6,6 + * _out6 : 7,7,7,7, 7,7,7,7, 7,7,7,7, 7,7,7,7 + * _out7 : 8,8,8,8, 8,8,8,8, 8,8,8,8, 8,8,8,8 + * ============================================================================= + */ +#define LASX_TRANSPOSE8x8_H(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ + _out7) \ + { \ + __m256i _s0_m, _s1_m; \ + __m256i _tmp0_m, _tmp1_m, _tmp2_m, _tmp3_m; \ + __m256i _tmp4_m, _tmp5_m, _tmp6_m, _tmp7_m; \ + \ + _s0_m = __lasx_xvilvl_h(_in6, _in4); \ + _s1_m = __lasx_xvilvl_h(_in7, _in5); \ + _tmp0_m = __lasx_xvilvl_h(_s1_m, _s0_m); \ + _tmp1_m = __lasx_xvilvh_h(_s1_m, _s0_m); \ + _s0_m = __lasx_xvilvh_h(_in6, _in4); \ + _s1_m = __lasx_xvilvh_h(_in7, _in5); \ + _tmp2_m = __lasx_xvilvl_h(_s1_m, _s0_m); \ + _tmp3_m = __lasx_xvilvh_h(_s1_m, _s0_m); \ + \ + _s0_m = __lasx_xvilvl_h(_in2, _in0); \ + _s1_m = __lasx_xvilvl_h(_in3, _in1); \ + _tmp4_m = __lasx_xvilvl_h(_s1_m, _s0_m); \ + _tmp5_m = __lasx_xvilvh_h(_s1_m, _s0_m); \ + _s0_m = __lasx_xvilvh_h(_in2, _in0); \ + _s1_m = __lasx_xvilvh_h(_in3, _in1); \ + _tmp6_m = __lasx_xvilvl_h(_s1_m, _s0_m); \ + _tmp7_m = __lasx_xvilvh_h(_s1_m, _s0_m); \ + \ + _out0 = __lasx_xvpickev_d(_tmp0_m, _tmp4_m); \ + _out2 = __lasx_xvpickev_d(_tmp1_m, _tmp5_m); \ + _out4 = __lasx_xvpickev_d(_tmp2_m, _tmp6_m); \ + _out6 = __lasx_xvpickev_d(_tmp3_m, _tmp7_m); \ + _out1 = __lasx_xvpickod_d(_tmp0_m, _tmp4_m); \ + _out3 = __lasx_xvpickod_d(_tmp1_m, _tmp5_m); \ + _out5 = __lasx_xvpickod_d(_tmp2_m, _tmp6_m); \ + _out7 = __lasx_xvpickod_d(_tmp3_m, _tmp7_m); \ + } + +/* + * ============================================================================= + * Description : Butterfly of 4 input vectors + * Arguments : Inputs - _in0, _in1, _in2, _in3 + * Outputs - _out0, _out1, _out2, _out3 + * Details : Butterfly operation + * Example : LASX_BUTTERFLY_4 + * _out0 = _in0 + _in3; + * _out1 = _in1 + _in2; + * _out2 = _in1 - _in2; + * _out3 = _in0 - _in3; + * ============================================================================= + */ +#define LASX_BUTTERFLY_4_B(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ + { \ + _out0 = __lasx_xvadd_b(_in0, _in3); \ + _out1 = __lasx_xvadd_b(_in1, _in2); \ + _out2 = __lasx_xvsub_b(_in1, _in2); \ + _out3 = __lasx_xvsub_b(_in0, _in3); \ + } +#define LASX_BUTTERFLY_4_H(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ + { \ + _out0 = __lasx_xvadd_h(_in0, _in3); \ + _out1 = __lasx_xvadd_h(_in1, _in2); \ + _out2 = __lasx_xvsub_h(_in1, _in2); \ + _out3 = __lasx_xvsub_h(_in0, _in3); \ + } +#define LASX_BUTTERFLY_4_W(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ + { \ + _out0 = __lasx_xvadd_w(_in0, _in3); \ + _out1 = __lasx_xvadd_w(_in1, _in2); \ + _out2 = __lasx_xvsub_w(_in1, _in2); \ + _out3 = __lasx_xvsub_w(_in0, _in3); \ + } +#define LASX_BUTTERFLY_4_D(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ + { \ + _out0 = __lasx_xvadd_d(_in0, _in3); \ + _out1 = __lasx_xvadd_d(_in1, _in2); \ + _out2 = __lasx_xvsub_d(_in1, _in2); \ + _out3 = __lasx_xvsub_d(_in0, _in3); \ + } + +/* + * ============================================================================= + * Description : Butterfly of 8 input vectors + * Arguments : Inputs - _in0, _in1, _in2, _in3, ~ + * Outputs - _out0, _out1, _out2, _out3, ~ + * Details : Butterfly operation + * Example : LASX_BUTTERFLY_8 + * _out0 = _in0 + _in7; + * _out1 = _in1 + _in6; + * _out2 = _in2 + _in5; + * _out3 = _in3 + _in4; + * _out4 = _in3 - _in4; + * _out5 = _in2 - _in5; + * _out6 = _in1 - _in6; + * _out7 = _in0 - _in7; + * ============================================================================= + */ +#define LASX_BUTTERFLY_8_B(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ + _out7) \ + { \ + _out0 = __lasx_xvadd_b(_in0, _in7); \ + _out1 = __lasx_xvadd_b(_in1, _in6); \ + _out2 = __lasx_xvadd_b(_in2, _in5); \ + _out3 = __lasx_xvadd_b(_in3, _in4); \ + _out4 = __lasx_xvsub_b(_in3, _in4); \ + _out5 = __lasx_xvsub_b(_in2, _in5); \ + _out6 = __lasx_xvsub_b(_in1, _in6); \ + _out7 = __lasx_xvsub_b(_in0, _in7); \ + } + +#define LASX_BUTTERFLY_8_H(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ + _out7) \ + { \ + _out0 = __lasx_xvadd_h(_in0, _in7); \ + _out1 = __lasx_xvadd_h(_in1, _in6); \ + _out2 = __lasx_xvadd_h(_in2, _in5); \ + _out3 = __lasx_xvadd_h(_in3, _in4); \ + _out4 = __lasx_xvsub_h(_in3, _in4); \ + _out5 = __lasx_xvsub_h(_in2, _in5); \ + _out6 = __lasx_xvsub_h(_in1, _in6); \ + _out7 = __lasx_xvsub_h(_in0, _in7); \ + } + +#define LASX_BUTTERFLY_8_W(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ + _out7) \ + { \ + _out0 = __lasx_xvadd_w(_in0, _in7); \ + _out1 = __lasx_xvadd_w(_in1, _in6); \ + _out2 = __lasx_xvadd_w(_in2, _in5); \ + _out3 = __lasx_xvadd_w(_in3, _in4); \ + _out4 = __lasx_xvsub_w(_in3, _in4); \ + _out5 = __lasx_xvsub_w(_in2, _in5); \ + _out6 = __lasx_xvsub_w(_in1, _in6); \ + _out7 = __lasx_xvsub_w(_in0, _in7); \ + } + +#define LASX_BUTTERFLY_8_D(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ + _out7) \ + { \ + _out0 = __lasx_xvadd_d(_in0, _in7); \ + _out1 = __lasx_xvadd_d(_in1, _in6); \ + _out2 = __lasx_xvadd_d(_in2, _in5); \ + _out3 = __lasx_xvadd_d(_in3, _in4); \ + _out4 = __lasx_xvsub_d(_in3, _in4); \ + _out5 = __lasx_xvsub_d(_in2, _in5); \ + _out6 = __lasx_xvsub_d(_in1, _in6); \ + _out7 = __lasx_xvsub_d(_in0, _in7); \ + } + +#endif // LASX + +/* + * ============================================================================= + * Description : Print out elements in vector. + * Arguments : Inputs - RTYPE, _element_num, _in0, _enter + * Outputs - + * Details : Print out '_element_num' elements in 'RTYPE' vector '_in0', if + * '_enter' is TRUE, prefix "\nVP:" will be added first. + * Example : VECT_PRINT(v4i32,4,in0,1); // in0: 1,2,3,4 + * VP:1,2,3,4, + * ============================================================================= + */ +#define VECT_PRINT(RTYPE, element_num, in0, enter) \ + { \ + RTYPE _tmp0 = (RTYPE)in0; \ + int _i = 0; \ + if (enter) printf("\nVP:"); \ + for (_i = 0; _i < element_num; _i++) printf("%d,", _tmp0[_i]); \ + } + +#endif /* LOONGSON_INTRINSICS_H */ +#endif /* VPX_VPX_UTIL_LOONGSON_INTRINSICS_H_ */ diff --git a/media/libvpx/libvpx/vpxdec.c b/media/libvpx/libvpx/vpxdec.c --- a/media/libvpx/libvpx/vpxdec.c +++ b/media/libvpx/libvpx/vpxdec.c @@ -813,10 +813,14 @@ if (num_external_frame_buffers > 0) { ext_fb_list.num_external_frame_buffers = num_external_frame_buffers; ext_fb_list.ext_fb = (struct ExternalFrameBuffer *)calloc( num_external_frame_buffers, sizeof(*ext_fb_list.ext_fb)); + if (!ext_fb_list.ext_fb) { + fprintf(stderr, "Failed to allocate ExternalFrameBuffer\n"); + goto fail; + } if (vpx_codec_set_frame_buffer_functions(&decoder, get_vp9_frame_buffer, release_vp9_frame_buffer, &ext_fb_list)) { fprintf(stderr, "Failed to configure external frame buffers: %s\n", vpx_codec_error(&decoder)); @@ -928,10 +932,15 @@ render_height = render_size[1]; } } scaled_img = vpx_img_alloc(NULL, img->fmt, render_width, render_height, 16); + if (!scaled_img) { + fprintf(stderr, "Failed to allocate scaled image (%d x %d)\n", + render_width, render_height); + goto fail; + } scaled_img->bit_depth = img->bit_depth; } if (img->d_w != scaled_img->d_w || img->d_h != scaled_img->d_h) { #if CONFIG_LIBYUV @@ -964,10 +973,14 @@ img_shifted = NULL; } if (!img_shifted) { img_shifted = vpx_img_alloc(NULL, shifted_fmt, img->d_w, img->d_h, 16); + if (!img_shifted) { + fprintf(stderr, "Failed to allocate image\n"); + goto fail; + } img_shifted->bit_depth = output_bit_depth; } if (output_bit_depth > img->bit_depth) { vpx_img_upshift(img_shifted, img, output_bit_depth - img->bit_depth); } else { diff --git a/media/libvpx/libvpx/vpxenc.c b/media/libvpx/libvpx/vpxenc.c --- a/media/libvpx/libvpx/vpxenc.c +++ b/media/libvpx/libvpx/vpxenc.c @@ -56,12 +56,12 @@ } #define fwrite wrap_fwrite static const char *exec_name; -static void warn_or_exit_on_errorv(vpx_codec_ctx_t *ctx, int fatal, - const char *s, va_list ap) { +static VPX_TOOLS_FORMAT_PRINTF(3, 0) void warn_or_exit_on_errorv( + vpx_codec_ctx_t *ctx, int fatal, const char *s, va_list ap) { if (ctx->err) { const char *detail = vpx_codec_error_detail(ctx); vfprintf(stderr, s, ap); fprintf(stderr, ": %s\n", vpx_codec_error(ctx)); @@ -70,20 +70,22 @@ if (fatal) exit(EXIT_FAILURE); } } -static void ctx_exit_on_error(vpx_codec_ctx_t *ctx, const char *s, ...) { +static VPX_TOOLS_FORMAT_PRINTF(2, + 3) void ctx_exit_on_error(vpx_codec_ctx_t *ctx, + const char *s, ...) { va_list ap; va_start(ap, s); warn_or_exit_on_errorv(ctx, 1, s, ap); va_end(ap); } -static void warn_or_exit_on_error(vpx_codec_ctx_t *ctx, int fatal, - const char *s, ...) { +static VPX_TOOLS_FORMAT_PRINTF(3, 4) void warn_or_exit_on_error( + vpx_codec_ctx_t *ctx, int fatal, const char *s, ...) { va_list ap; va_start(ap, s); warn_or_exit_on_errorv(ctx, fatal, s, ap); va_end(ap); diff --git a/media/libvpx/libvpx/vpxstats.c b/media/libvpx/libvpx/vpxstats.c --- a/media/libvpx/libvpx/vpxstats.c +++ b/media/libvpx/libvpx/vpxstats.c @@ -39,11 +39,11 @@ rewind(stats->file); stats->buf.buf = malloc(stats->buf_alloc_sz); if (!stats->buf.buf) - fatal("Failed to allocate first-pass stats buffer (%lu bytes)", + fatal("Failed to allocate first-pass stats buffer (%u bytes)", (unsigned int)stats->buf_alloc_sz); nbytes = fread(stats->buf.buf, 1, stats->buf.sz, stats->file); res = (nbytes == stats->buf.sz); } diff --git a/media/libvpx/libvpx/warnings.c b/media/libvpx/libvpx/warnings.c --- a/media/libvpx/libvpx/warnings.c +++ b/media/libvpx/libvpx/warnings.c @@ -96,11 +96,11 @@ stream_config->g_lag_in_frames, global_config->deadline, stream_config->rc_end_usage, &warning_list); /* Count and print warnings. */ for (warning = warning_list.warning_node; warning != NULL; warning = warning->next_warning, ++num_warnings) { - warn(warning->warning_string); + warn("%s", warning->warning_string); } free_warning_list(&warning_list); if (num_warnings) { diff --git a/media/libvpx/libvpx/webmdec.cc b/media/libvpx/libvpx/webmdec.cc --- a/media/libvpx/libvpx/webmdec.cc +++ b/media/libvpx/libvpx/webmdec.cc @@ -208,10 +208,12 @@ } vpx_ctx->framerate.numerator = (i - 1) * 1000000; vpx_ctx->framerate.denominator = static_cast(webm_ctx->timestamp_ns / 1000); delete[] buffer; + // webm_ctx->buffer is assigned to the buffer pointer in webm_read_frame(). + webm_ctx->buffer = nullptr; get_first_cluster(webm_ctx); webm_ctx->block = nullptr; webm_ctx->block_entry = nullptr; webm_ctx->block_frame_index = 0; diff --git a/media/libvpx/libvpx/y4minput.c b/media/libvpx/libvpx/y4minput.c --- a/media/libvpx/libvpx/y4minput.c +++ b/media/libvpx/libvpx/y4minput.c @@ -1085,13 +1085,19 @@ ((y4m_ctx->pic_h + y4m_ctx->dst_c_dec_v - 1) / y4m_ctx->dst_c_dec_v); if (y4m_ctx->bit_depth == 8) y4m_ctx->dst_buf = (unsigned char *)malloc(y4m_ctx->dst_buf_sz); else y4m_ctx->dst_buf = (unsigned char *)malloc(2 * y4m_ctx->dst_buf_sz); + if (!y4m_ctx->dst_buf) return -1; - if (y4m_ctx->aux_buf_sz > 0) + if (y4m_ctx->aux_buf_sz > 0) { y4m_ctx->aux_buf = (unsigned char *)malloc(y4m_ctx->aux_buf_sz); + if (!y4m_ctx->aux_buf) { + free(y4m_ctx->dst_buf); + return -1; + } + } return 0; } void y4m_input_close(y4m_input *_y4m) { free(_y4m->dst_buf); diff --git a/media/libvpx/moz.yaml b/media/libvpx/moz.yaml --- a/media/libvpx/moz.yaml +++ b/media/libvpx/moz.yaml @@ -18,15 +18,15 @@ # Usually different from repository url url: https://chromium.googlesource.com/webm/libvpx # Human-readable identifier for this version/release # Generally "version NNN", "tag SSS", "bookmark SSS" - release: 626ff35955c2c35b806b3e0ecf551a1a8611cdbf (Wed Oct 06 17:41:19 2021). + release: 2858ef9ec6fe99d92074216c54a1ff7c09b1b667 (Thu Apr 28 17:03:32 2022). # Revision to pull in # Must be a long or short commit SHA (long preferred) - revision: 626ff35955c2c35b806b3e0ecf551a1a8611cdbf + revision: 2858ef9ec6fe99d92074216c54a1ff7c09b1b667 # The package's license, where possible using the mnemonic from # https://spdx.org/licenses/ # Multiple licenses can be specified (as a YAML list) # A "LICENSE" file must exist containing the full license text diff --git a/media/libvpx/sources.mozbuild b/media/libvpx/sources.mozbuild --- a/media/libvpx/sources.mozbuild +++ b/media/libvpx/sources.mozbuild @@ -233,22 +233,19 @@ 'libvpx/vpx_dsp/x86/intrapred_ssse3.asm', 'libvpx/vpx_dsp/x86/inv_txfm_sse2.c', 'libvpx/vpx_dsp/x86/inv_txfm_ssse3.c', 'libvpx/vpx_dsp/x86/inv_wht_sse2.asm', 'libvpx/vpx_dsp/x86/loopfilter_avx2.c', - 'libvpx/vpx_dsp/x86/loopfilter_intrin_sse2.c', + 'libvpx/vpx_dsp/x86/loopfilter_sse2.c', 'libvpx/vpx_dsp/x86/post_proc_sse2.c', 'libvpx/vpx_dsp/x86/quantize_avx.c', 'libvpx/vpx_dsp/x86/quantize_sse2.c', 'libvpx/vpx_dsp/x86/quantize_ssse3.c', 'libvpx/vpx_dsp/x86/sad4d_avx2.c', 'libvpx/vpx_dsp/x86/sad4d_sse2.asm', 'libvpx/vpx_dsp/x86/sad_avx2.c', 'libvpx/vpx_dsp/x86/sad_sse2.asm', - 'libvpx/vpx_dsp/x86/sad_sse3.asm', - 'libvpx/vpx_dsp/x86/sad_sse4.asm', - 'libvpx/vpx_dsp/x86/sad_ssse3.asm', 'libvpx/vpx_dsp/x86/ssim_opt_x86_64.asm', 'libvpx/vpx_dsp/x86/subpel_variance_sse2.asm', 'libvpx/vpx_dsp/x86/subtract_sse2.asm', 'libvpx/vpx_dsp/x86/sum_squares_sse2.c', 'libvpx/vpx_dsp/x86/variance_avx2.c', @@ -261,11 +258,10 @@ 'libvpx/vpx_dsp/x86/vpx_subpixel_8t_ssse3.asm', 'libvpx/vpx_dsp/x86/vpx_subpixel_bilinear_sse2.asm', 'libvpx/vpx_dsp/x86/vpx_subpixel_bilinear_ssse3.asm', 'libvpx/vpx_mem/vpx_mem.c', 'libvpx/vpx_ports/emms_mmx.asm', - 'libvpx/vpx_ports/float_control_word.asm', 'libvpx/vpx_scale/generic/gen_scalers.c', 'libvpx/vpx_scale/generic/vpx_scale.c', 'libvpx/vpx_scale/generic/yv12config.c', 'libvpx/vpx_scale/generic/yv12extend.c', 'libvpx/vpx_scale/vpx_scale_rtcd.c', @@ -500,22 +496,19 @@ 'libvpx/vpx_dsp/x86/intrapred_ssse3.asm', 'libvpx/vpx_dsp/x86/inv_txfm_sse2.c', 'libvpx/vpx_dsp/x86/inv_txfm_ssse3.c', 'libvpx/vpx_dsp/x86/inv_wht_sse2.asm', 'libvpx/vpx_dsp/x86/loopfilter_avx2.c', - 'libvpx/vpx_dsp/x86/loopfilter_intrin_sse2.c', + 'libvpx/vpx_dsp/x86/loopfilter_sse2.c', 'libvpx/vpx_dsp/x86/post_proc_sse2.c', 'libvpx/vpx_dsp/x86/quantize_avx.c', 'libvpx/vpx_dsp/x86/quantize_sse2.c', 'libvpx/vpx_dsp/x86/quantize_ssse3.c', 'libvpx/vpx_dsp/x86/sad4d_avx2.c', 'libvpx/vpx_dsp/x86/sad4d_sse2.asm', 'libvpx/vpx_dsp/x86/sad_avx2.c', 'libvpx/vpx_dsp/x86/sad_sse2.asm', - 'libvpx/vpx_dsp/x86/sad_sse3.asm', - 'libvpx/vpx_dsp/x86/sad_sse4.asm', - 'libvpx/vpx_dsp/x86/sad_ssse3.asm', 'libvpx/vpx_dsp/x86/subpel_variance_sse2.asm', 'libvpx/vpx_dsp/x86/subtract_sse2.asm', 'libvpx/vpx_dsp/x86/sum_squares_sse2.c', 'libvpx/vpx_dsp/x86/variance_avx2.c', 'libvpx/vpx_dsp/x86/variance_sse2.c', @@ -663,10 +656,11 @@ 'libvpx/vp9/decoder/vp9_decodemv.c', 'libvpx/vp9/decoder/vp9_decoder.c', 'libvpx/vp9/decoder/vp9_detokenize.c', 'libvpx/vp9/decoder/vp9_dsubexp.c', 'libvpx/vp9/decoder/vp9_job_queue.c', + 'libvpx/vp9/encoder/arm/neon/vp9_dct_neon.c', 'libvpx/vp9/encoder/arm/neon/vp9_error_neon.c', 'libvpx/vp9/encoder/arm/neon/vp9_frame_scale_neon.c', 'libvpx/vp9/encoder/arm/neon/vp9_quantize_neon.c', 'libvpx/vp9/encoder/vp9_aq_cyclicrefresh.c', 'libvpx/vp9/encoder/vp9_bitstream.c', @@ -906,10 +900,11 @@ 'libvpx/vp9/decoder/vp9_decodemv.c', 'libvpx/vp9/decoder/vp9_decoder.c', 'libvpx/vp9/decoder/vp9_detokenize.c', 'libvpx/vp9/decoder/vp9_dsubexp.c', 'libvpx/vp9/decoder/vp9_job_queue.c', + 'libvpx/vp9/encoder/arm/neon/vp9_dct_neon.c', 'libvpx/vp9/encoder/arm/neon/vp9_error_neon.c', 'libvpx/vp9/encoder/arm/neon/vp9_frame_scale_neon.c', 'libvpx/vp9/encoder/arm/neon/vp9_quantize_neon.c', 'libvpx/vp9/encoder/vp9_aq_cyclicrefresh.c', 'libvpx/vp9/encoder/vp9_bitstream.c', diff --git a/third_party/jpeg-xl/.github/workflows/build_test.yml b/third_party/jpeg-xl/.github/workflows/build_test.yml --- a/third_party/jpeg-xl/.github/workflows/build_test.yml +++ b/third_party/jpeg-xl/.github/workflows/build_test.yml @@ -44,11 +44,11 @@ # test on AVX3-enabled CPUs when changing this value. env_test_stack_size: 4000 # Build scalar-only hwy instructions. - name: scalar mode: release - cxxflags: -DHWY_DISABLED_TARGETS=~HWY_SCALAR + cxxflags: -DHWY_COMPILE_ONLY_SCALAR # Disabling optional features to speed up msan build a little bit. - name: msan skip_install: true cmake_args: >- -DJPEGXL_ENABLE_DEVTOOLS=OFF -DJPEGXL_ENABLE_PLUGINS=OFF @@ -57,12 +57,10 @@ skip_install: true - name: coverage apt_pkgs: gcovr # Coverage builds require a bit more RAM. env_test_stack_size: 2048 - # Exclude roundtrip tests from the unittest coverage. - ctest_args: -E '^JxlTest' # Build with support for decoding to JPEG bytes disabled. Produces a # smaller build if only decoding to pixels is needed. - name: release-nojpeg mode: release cxxflags: -DJXL_DEBUG_ON_ABORT=0 @@ -147,10 +145,11 @@ # hash as the starting point from the restore-keys entry. key: ${{ runner.os }}-${{ github.sha }}-${{ matrix.name }} restore-keys: | ${{ runner.os }}-${{ steps.git-env.outputs.parent }}-${{ matrix.name }} - name: Build + if: matrix.name != 'coverage' || env.WILL_RUN_TESTS == 'true' run: | mkdir -p ${CCACHE_DIR} echo "max_size = 200M" > ${CCACHE_DIR}/ccache.conf mode="${{ matrix.mode }}" build_tests=$([ "$WILL_RUN_TESTS" == "true" ] && echo "ON" || echo "OFF") @@ -214,14 +213,14 @@ with: flags: unittests files: build/coverage.xml - name: Fast benchmark ${{ matrix.mode }} if: | - github.event_name == 'push' || + matrix.name != 'coverage' && (github.event_name == 'push' || (github.event_name == 'pull_request' && ( matrix.test_in_pr || - contains(github.event.pull_request.labels.*.names, 'CI:full'))) + contains(github.event.pull_request.labels.*.names, 'CI:full')))) run: | STORE_IMAGES=0 ./ci.sh fast_benchmark # Run gbench once, just to make sure it runs, not for actual benchmarking. # This doesn't work on msan because we use gbench library from the system # which is not instrumented by msan. @@ -308,11 +307,10 @@ sudo apt update pkgs=( # Build dependencies cmake doxygen - libgtest-dev:${{ matrix.arch }} ninja-build pkg-config qemu-user-static xvfb diff --git a/third_party/jpeg-xl/.github/workflows/conformance.yml b/third_party/jpeg-xl/.github/workflows/conformance.yml --- a/third_party/jpeg-xl/.github/workflows/conformance.yml +++ b/third_party/jpeg-xl/.github/workflows/conformance.yml @@ -96,12 +96,11 @@ run: needs: build runs-on: ubuntu-latest strategy: matrix: - # 'main_level10' currently fails - name: [main_level5] + name: [main_level5, main_level10] steps: - name: Install deps run: | pip install numpy - name: Checkout the conformance source diff --git a/third_party/jpeg-xl/.github/workflows/release.yaml b/third_party/jpeg-xl/.github/workflows/release.yaml --- a/third_party/jpeg-xl/.github/workflows/release.yaml +++ b/third_party/jpeg-xl/.github/workflows/release.yaml @@ -144,10 +144,15 @@ "https://github.com/git/git/archive/refs/tags/v${git_version}.tar.gz" tar -zxf "v${git_version}.tar.gz" cd "git-${git_version}" make prefix=/usr -j4 install + - name: Set git safe dir + run: | + export GIT_CEILING_DIRECTORIES=/__w # only work before git v2.35.2 + git config --global --add safe.directory /__w/libjxl/libjxl + - name: Checkout the source uses: actions/checkout@v2 with: submodules: true fetch-depth: 1 diff --git a/third_party/jpeg-xl/AUTHORS b/third_party/jpeg-xl/AUTHORS --- a/third_party/jpeg-xl/AUTHORS +++ b/third_party/jpeg-xl/AUTHORS @@ -23,19 +23,23 @@ CanadianBaconBoi Daniel Novomeský David Burnett Dirk Lemstra Don Olmstead +Heiko Becker Jon Sneyers Kleis Auke Wolthuizen +L. E. Segovia Leo Izen Lovell Fuller Marcin Konicki +Martin Strunz Mathieu Malaterre Misaki Kasumi Petr Diblík Pieter Wuille Samuel Leong +Stephan T. Lavavej Vincent Torri xiota Yonatan Nebenzhal Ziemowit Zabawa diff --git a/third_party/jpeg-xl/CHANGELOG.md b/third_party/jpeg-xl/CHANGELOG.md --- a/third_party/jpeg-xl/CHANGELOG.md +++ b/third_party/jpeg-xl/CHANGELOG.md @@ -21,10 +21,13 @@ - decoder API: field added to `JxlFrameHeader`: a `JxlLayerInfo` struct that contains crop dimensions and offsets and blending information for the non-coalesced case. - decoder API: new function `JxlDecoderGetExtraChannelBlendInfo` to get the blending information for extra channels in the non-coalesced case. + - decoder API: new function `JxlDecoderSetMultithreadedImageOutCallback`, + allowing output callbacks to receive more information about the number of + threads on which they are running. - encoder API: added ability to set several encoder options to frames using `JxlEncoderFrameSettingsSetOption` - encoder API: new functions `JxlEncoderSetFrameHeader` and `JxlEncoderSetExtraChannelBlendInfo` to set animation and blending parameters of the frame, and `JxlEncoderInitFrameHeader` and diff --git a/third_party/jpeg-xl/CMakeLists.txt b/third_party/jpeg-xl/CMakeLists.txt --- a/third_party/jpeg-xl/CMakeLists.txt +++ b/third_party/jpeg-xl/CMakeLists.txt @@ -189,10 +189,19 @@ # Threads set(THREADS_PREFER_PTHREAD_FLAG YES) find_package(Threads REQUIRED) +# These settings are important to drive check_cxx_source_compiles +# See CMP0067 (min cmake version is 3.10 anyway) +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_EXTENSIONS OFF) +set(CMAKE_CXX_STANDARD_REQUIRED YES) + +# Atomics +find_package(Atomics REQUIRED) + if(JPEGXL_STATIC) if (MINGW) # In MINGW libstdc++ uses pthreads directly. When building statically a # program (regardless of whether the source code uses pthread or not) the # toolchain will add stdc++ and pthread to the linking step but stdc++ will @@ -296,23 +305,23 @@ endif () endif () # !MSVC include(GNUInstallDirs) -set(CMAKE_CXX_STANDARD 11) -set(CMAKE_CXX_EXTENSIONS OFF) -set(CMAKE_CXX_STANDARD_REQUIRED YES) - add_subdirectory(third_party) # Copy the JXL license file to the output build directory. configure_file("${CMAKE_CURRENT_SOURCE_DIR}/LICENSE" ${PROJECT_BINARY_DIR}/LICENSE.jpeg-xl COPYONLY) # Enable tests regardless of where they are defined. enable_testing() include(CTest) +# Specify default location of `testdata`: +if(NOT DEFINED JPEGXL_TEST_DATA_PATH) + set(JPEGXL_TEST_DATA_PATH "${PROJECT_SOURCE_DIR}/third_party/testdata") +endif() # Libraries. add_subdirectory(lib) if(BUILD_TESTING) @@ -417,11 +426,11 @@ MAIN_DEPENDENCY "${CMAKE_CURRENT_SOURCE_DIR}/doc/man/${PAGE}.txt") list(APPEND MANPAGE_FILES "${CMAKE_CURRENT_BINARY_DIR}/${PAGE}.1") list(APPEND MANPAGES "${PAGE}.1") endforeach() add_custom_target(manpages ALL DEPENDS ${MANPAGES}) - install(FILES ${MANPAGE_FILES} DESTINATION share/man/man1) + install(FILES ${MANPAGE_FILES} DESTINATION ${CMAKE_INSTALL_MANDIR}/man1) endif() # ASCIIDOC_PY_FOUND else() message(WARNING "asciidoc was not found, the man pages will not be installed.") endif() # ASCIIDOC != "ASCIIDOC-NOTFOUND" endif() # JPEGXL_ENABLE_MANPAGES diff --git a/third_party/jpeg-xl/ci.sh b/third_party/jpeg-xl/ci.sh --- a/third_party/jpeg-xl/ci.sh +++ b/third_party/jpeg-xl/ci.sh @@ -903,11 +903,11 @@ num_threads=1 fi local benchmark_args=( --input "${src_img_dir}/*.png" - --codec=jpeg:yuv420:q85,webp:q80,jxl:fast:d1,jxl:fast:d1:downsampling=8,jxl:fast:d4,jxl:fast:d4:downsampling=8,jxl:cheetah:m,jxl:m:cheetah:P6,jxl:m:falcon:q80 + --codec=jpeg:yuv420:q85,webp:q80,jxl:d1:6,jxl:d1:6:downsampling=8,jxl:d5:6,jxl:d5:6:downsampling=8,jxl:m:d0:2,jxl:m:d0:3,jxl:m:d2:2 --output_dir "${output_dir}" --noprofiler --show_progress --num_threads="${num_threads}" ) if [[ "${STORE_IMAGES}" == "1" ]]; then @@ -1018,15 +1018,15 @@ local jxl_jpeg_benchmarks=( "--num_reps=3" ) local images=( - "third_party/testdata/imagecompression.info/flower_foveon.png" + "third_party/testdata/third_party/imagecompression.info/flower_foveon.png" ) local jpg_images=( - "third_party/testdata/imagecompression.info/flower_foveon.png.im_q85_420.jpg" + "third_party/testdata/third_party/imagecompression.info/flower_foveon.png.im_q85_420.jpg" ) if [[ "${SKIP_CPUSET:-}" == "1" ]]; then # Use a single cpu config in this case. local cpu_confs=("?") diff --git a/third_party/jpeg-xl/cmake/FindAtomics.cmake b/third_party/jpeg-xl/cmake/FindAtomics.cmake new file mode 100644 --- /dev/null +++ b/third_party/jpeg-xl/cmake/FindAtomics.cmake @@ -0,0 +1,53 @@ +# Original issue: +# * https://gitlab.kitware.com/cmake/cmake/-/issues/23021#note_1098733 +# +# For reference: +# * https://gcc.gnu.org/wiki/Atomic/GCCMM +# +# riscv64 specific: +# * https://lists.debian.org/debian-riscv/2022/01/msg00009.html +# +# ATOMICS_FOUND - system has c++ atomics +# ATOMICS_LIBRARIES - libraries needed to use c++ atomics + +include(CheckCXXSourceCompiles) + +# RISC-V only has 32-bit and 64-bit atomic instructions. GCC is supposed +# to convert smaller atomics to those larger ones via masking and +# shifting like LLVM, but it’s a known bug that it does not. This means +# anything that wants to use atomics on 1-byte or 2-byte types needs +# -latomic, but not 4-byte or 8-byte (though it does no harm). +set(atomic_code + " + #include + #include + std::atomic n8 (0); // riscv64 + std::atomic n64 (0); // armel, mipsel, powerpc + int main() { + ++n8; + ++n64; + return 0; + }") + +check_cxx_source_compiles("${atomic_code}" ATOMICS_LOCK_FREE_INSTRUCTIONS) + +if(ATOMICS_LOCK_FREE_INSTRUCTIONS) + set(ATOMICS_FOUND TRUE) + set(ATOMICS_LIBRARIES) +else() + set(CMAKE_REQUIRED_LIBRARIES "-latomic") + check_cxx_source_compiles("${atomic_code}" ATOMICS_IN_LIBRARY) + set(CMAKE_REQUIRED_LIBRARIES) + if(ATOMICS_IN_LIBRARY) + set(ATOMICS_LIBRARY atomic) + include(FindPackageHandleStandardArgs) + find_package_handle_standard_args(Atomics DEFAULT_MSG ATOMICS_LIBRARY) + set(ATOMICS_LIBRARIES ${ATOMICS_LIBRARY}) + unset(ATOMICS_LIBRARY) + else() + if(Atomics_FIND_REQUIRED) + message(FATAL_ERROR "Neither lock free instructions nor -latomic found.") + endif() + endif() +endif() +unset(atomic_code) diff --git a/third_party/jpeg-xl/debian/copyright b/third_party/jpeg-xl/debian/copyright --- a/third_party/jpeg-xl/debian/copyright +++ b/third_party/jpeg-xl/debian/copyright @@ -36,11 +36,11 @@ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -Files: third_party/testdata/imagecompression.info/* +Files: third_party/testdata/third_party/imagecompression.info/* Copyright: their respective owners. License: License without any prohibitive copyright restrictions. See https://imagecompression.info/test_images/ for details. . These Images are available without any prohibitive copyright restrictions. @@ -56,11 +56,11 @@ 3. No payment is required for distribution of this material, it must be available freely under the conditions stated here. That is, it is prohibited to sell the material. 4. This notice may not be removed or altered from any distribution. -Files: third_party/testdata/pngsuite/* +Files: third_party/testdata/third_party/pngsuite/* Copyright: Willem van Schaik, 1996, 2011 License: PngSuite License See http://www.schaik.com/pngsuite/ for details. . Permission to use, copy, modify and distribute these images for any @@ -72,11 +72,11 @@ Files: third_party/testdata/raw.pixls/* Copyright: their respective owners listed in https://www.wesaturate.com/ License: CC0-1.0 -Files: third_party/testdata/wide-gamut-tests/ +Files: third_party/testdata/third_party/wide-gamut-tests/ Copyright: github.com/codelogic/wide-gamut-tests authors. License: Apache-2.0 License: Apache-2.0 Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/third_party/jpeg-xl/deps.sh b/third_party/jpeg-xl/deps.sh --- a/third_party/jpeg-xl/deps.sh +++ b/third_party/jpeg-xl/deps.sh @@ -11,10 +11,11 @@ MYDIR=$(dirname $(realpath "$0")) # Git revisions we use for the given submodules. Update these whenever you # update a git submodule. +THIRD_PARTY_BROTLI="35ef5c554d888bef217d449346067de05e269b30" THIRD_PARTY_GFLAGS="827c769e5fc98e0f2a34c47cef953cc6328abced" THIRD_PARTY_HIGHWAY="f13e3b956eb226561ac79427893ec0afd66f91a8" THIRD_PARTY_SKCMS="64374756e03700d649f897dbd98c95e78c30c7da" THIRD_PARTY_SJPEG="868ab558fad70fcbe8863ba4e85179eeb81cc840" THIRD_PARTY_ZLIB="cacf7f1d4e3d44d871b605da3b647f07d718623f" @@ -69,10 +70,11 @@ git -C "${MYDIR}" submodule update --init --recursive --depth 1 --recommend-shallow return 0 fi # Sources downloaded from a tarball. + download_github third_party/brotli google/brotli download_github third_party/gflags gflags/gflags download_github third_party/highway google/highway download_github third_party/sjpeg webmproject/sjpeg download_github third_party/skcms \ "https://skia.googlesource.com/skcms/+archive/" diff --git a/third_party/jpeg-xl/examples/CMakeLists.txt b/third_party/jpeg-xl/examples/CMakeLists.txt --- a/third_party/jpeg-xl/examples/CMakeLists.txt +++ b/third_party/jpeg-xl/examples/CMakeLists.txt @@ -23,13 +23,10 @@ target_link_libraries(decode_progressive PkgConfig::Jxl PkgConfig::JxlThreads) add_executable(encode_oneshot encode_oneshot.cc) target_link_libraries(encode_oneshot PkgConfig::Jxl PkgConfig::JxlThreads) -add_executable(jxlinfo jxlinfo.c) -target_link_libraries(jxlinfo PkgConfig::Jxl) - # Building a static binary with the static libjxl dependencies. How to load # static library configs from pkg-config and how to build static binaries # depends on the platform, and building static binaries in general has problems. # If you don't need static binaries you can remove this section. diff --git a/third_party/jpeg-xl/examples/examples.cmake b/third_party/jpeg-xl/examples/examples.cmake --- a/third_party/jpeg-xl/examples/examples.cmake +++ b/third_party/jpeg-xl/examples/examples.cmake @@ -7,15 +7,5 @@ target_link_libraries(decode_oneshot jxl_dec jxl_threads) add_executable(decode_progressive ${CMAKE_CURRENT_LIST_DIR}/decode_progressive.cc) target_link_libraries(decode_progressive jxl_dec jxl_threads) add_executable(encode_oneshot ${CMAKE_CURRENT_LIST_DIR}/encode_oneshot.cc) target_link_libraries(encode_oneshot jxl jxl_threads) - -add_executable(jxlinfo ${CMAKE_CURRENT_LIST_DIR}/jxlinfo.c) -target_link_libraries(jxlinfo jxl) - -if(NOT ${SANITIZER} STREQUAL "none") - # Linking a C test binary with the C++ JPEG XL implementation when using - # address sanitizer is not well supported by clang 9, so force using clang++ - # for linking this test if a sanitizer is used. - set_target_properties(jxlinfo PROPERTIES LINKER_LANGUAGE CXX) -endif() # SANITIZER != "none" diff --git a/third_party/jpeg-xl/examples/jxlinfo.c b/third_party/jpeg-xl/examples/jxlinfo.c deleted file mode 100644 --- a/third_party/jpeg-xl/examples/jxlinfo.c +++ /dev/null @@ -1,332 +0,0 @@ -// Copyright (c) the JPEG XL Project Authors. All rights reserved. -// -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// This example prints information from the main codestream header. - -#include -#include -#include -#include -#include - -#include "jxl/decode.h" - -int PrintBasicInfo(FILE* file) { - uint8_t* data = NULL; - size_t data_size = 0; - // In how large chunks to read from the file and try decoding the basic info. - const size_t chunk_size = 64; - - JxlDecoder* dec = JxlDecoderCreate(NULL); - if (!dec) { - fprintf(stderr, "JxlDecoderCreate failed\n"); - return 0; - } - - JxlDecoderSetKeepOrientation(dec, 1); - - if (JXL_DEC_SUCCESS != JxlDecoderSubscribeEvents( - dec, JXL_DEC_BASIC_INFO | JXL_DEC_COLOR_ENCODING | - JXL_DEC_FRAME | JXL_DEC_BOX)) { - fprintf(stderr, "JxlDecoderSubscribeEvents failed\n"); - JxlDecoderDestroy(dec); - return 0; - } - - JxlBasicInfo info; - int seen_basic_info = 0; - JxlFrameHeader frame_header; - - for (;;) { - // The first time, this will output JXL_DEC_NEED_MORE_INPUT because no - // input is set yet, this is ok since the input is set when handling this - // event. - JxlDecoderStatus status = JxlDecoderProcessInput(dec); - - if (status == JXL_DEC_ERROR) { - fprintf(stderr, "Decoder error\n"); - break; - } else if (status == JXL_DEC_NEED_MORE_INPUT) { - // The first time there is nothing to release and it returns 0, but that - // is ok. - size_t remaining = JxlDecoderReleaseInput(dec); - // move any remaining bytes to the front if necessary - if (remaining != 0) { - memmove(data, data + data_size - remaining, remaining); - } - // resize the buffer to append one more chunk of data - // TODO(lode): avoid unnecessary reallocations - data = (uint8_t*)realloc(data, remaining + chunk_size); - // append bytes read from the file behind the remaining bytes - size_t read_size = fread(data + remaining, 1, chunk_size, file); - if (read_size == 0 && feof(file)) { - fprintf(stderr, "Unexpected EOF\n"); - break; - } - data_size = remaining + read_size; - JxlDecoderSetInput(dec, data, data_size); - if (feof(file)) JxlDecoderCloseInput(dec); - } else if (status == JXL_DEC_SUCCESS) { - // Finished all processing. - break; - } else if (status == JXL_DEC_BASIC_INFO) { - if (JXL_DEC_SUCCESS != JxlDecoderGetBasicInfo(dec, &info)) { - fprintf(stderr, "JxlDecoderGetBasicInfo failed\n"); - break; - } - - seen_basic_info = 1; - - printf("dimensions: %ux%u\n", info.xsize, info.ysize); - printf("have_container: %d\n", info.have_container); - printf("uses_original_profile: %d\n", info.uses_original_profile); - printf("bits_per_sample: %d\n", info.bits_per_sample); - if (info.exponent_bits_per_sample) - printf("float, with exponent_bits_per_sample: %d\n", - info.exponent_bits_per_sample); - if (info.intensity_target != 255.f || info.min_nits != 0.f || - info.relative_to_max_display != 0 || - info.relative_to_max_display != 0.f) { - printf("intensity_target: %f\n", info.intensity_target); - printf("min_nits: %f\n", info.min_nits); - printf("relative_to_max_display: %d\n", info.relative_to_max_display); - printf("linear_below: %f\n", info.linear_below); - } - printf("have_preview: %d\n", info.have_preview); - if (info.have_preview) { - printf("preview xsize: %u\n", info.preview.xsize); - printf("preview ysize: %u\n", info.preview.ysize); - } - printf("have_animation: %d\n", info.have_animation); - if (info.have_animation) { - printf("ticks per second (numerator / denominator): %u / %u\n", - info.animation.tps_numerator, info.animation.tps_denominator); - printf("num_loops: %u\n", info.animation.num_loops); - printf("have_timecodes: %d\n", info.animation.have_timecodes); - } - printf("intrinsic xsize: %u\n", info.intrinsic_xsize); - printf("intrinsic ysize: %u\n", info.intrinsic_ysize); - const char* const orientation_string[8] = { - "Normal", "Flipped horizontally", - "Upside down", "Flipped vertically", - "Transposed", "90 degrees clockwise", - "Anti-Transposed", "90 degrees counter-clockwise"}; - if (info.orientation > 0 && info.orientation < 9) { - printf("orientation: %d (%s)\n", info.orientation, - orientation_string[info.orientation - 1]); - } else { - fprintf(stderr, "Invalid orientation\n"); - } - printf("num_color_channels: %d\n", info.num_color_channels); - printf("num_extra_channels: %d\n", info.num_extra_channels); - - const char* const ec_type_names[7] = {"Alpha", "Depth", - "Spot color", "Selection mask", - "K (of CMYK)", "CFA (Bayer data)", - "Thermal"}; - for (uint32_t i = 0; i < info.num_extra_channels; i++) { - JxlExtraChannelInfo extra; - if (JXL_DEC_SUCCESS != JxlDecoderGetExtraChannelInfo(dec, i, &extra)) { - fprintf(stderr, "JxlDecoderGetExtraChannelInfo failed\n"); - break; - } - printf("extra channel %u:\n", i); - printf(" type: %s\n", - (extra.type < 7 ? ec_type_names[extra.type] - : (extra.type == JXL_CHANNEL_OPTIONAL - ? "Unknown but can be ignored" - : "Unknown, please update your libjxl"))); - printf(" bits_per_sample: %u\n", extra.bits_per_sample); - if (extra.exponent_bits_per_sample > 0) { - printf(" float, with exponent_bits_per_sample: %u\n", - extra.exponent_bits_per_sample); - } - if (extra.dim_shift > 0) { - printf(" dim_shift: %u (upsampled %ux)\n", extra.dim_shift, - 1 << extra.dim_shift); - } - if (extra.name_length) { - char* name = malloc(extra.name_length + 1); - if (JXL_DEC_SUCCESS != JxlDecoderGetExtraChannelName( - dec, i, name, extra.name_length + 1)) { - fprintf(stderr, "JxlDecoderGetExtraChannelName failed\n"); - free(name); - break; - } - printf(" name: %s\n", name); - free(name); - } - if (extra.type == JXL_CHANNEL_ALPHA) - printf(" alpha_premultiplied: %d (%s)\n", extra.alpha_premultiplied, - extra.alpha_premultiplied ? "Premultiplied" - : "Non-premultiplied"); - if (extra.type == JXL_CHANNEL_SPOT_COLOR) { - printf(" spot_color: (%f, %f, %f) with opacity %f\n", - extra.spot_color[0], extra.spot_color[1], extra.spot_color[2], - extra.spot_color[3]); - } - if (extra.type == JXL_CHANNEL_CFA) - printf(" cfa_channel: %u\n", extra.cfa_channel); - } - } else if (status == JXL_DEC_COLOR_ENCODING) { - JxlPixelFormat format = {4, JXL_TYPE_FLOAT, JXL_LITTLE_ENDIAN, 0}; - printf("color profile:\n"); - - JxlColorEncoding color_encoding; - if (JXL_DEC_SUCCESS == - JxlDecoderGetColorAsEncodedProfile(dec, &format, - JXL_COLOR_PROFILE_TARGET_ORIGINAL, - &color_encoding)) { - printf(" format: JPEG XL encoded color profile\n"); - const char* const cs_string[4] = {"RGB color", "Grayscale", "XYB", - "Unknown"}; - const char* const wp_string[12] = {"", "D65", "Custom", "", "", "", - "", "", "", "", "E", "P3"}; - const char* const pr_string[12] = { - "", "sRGB", "Custom", "", "", "", "", "", "", "Rec.2100", "", "P3"}; - const char* const tf_string[19] = { - "", "709", "Unknown", "", "", "", "", "", "Linear", "", - "", "", "", "sRGB", "", "", "PQ", "DCI", "HLG"}; - const char* const ri_string[4] = {"Perceptual", "Relative", - "Saturation", "Absolute"}; - printf(" color_space: %d (%s)\n", color_encoding.color_space, - cs_string[color_encoding.color_space]); - printf(" white_point: %d (%s)\n", color_encoding.white_point, - wp_string[color_encoding.white_point]); - if (color_encoding.white_point == JXL_WHITE_POINT_CUSTOM) { - printf(" white_point XY: %f %f\n", color_encoding.white_point_xy[0], - color_encoding.white_point_xy[1]); - } - if (color_encoding.color_space == JXL_COLOR_SPACE_RGB || - color_encoding.color_space == JXL_COLOR_SPACE_UNKNOWN) { - printf(" primaries: %d (%s)\n", color_encoding.primaries, - pr_string[color_encoding.primaries]); - if (color_encoding.primaries == JXL_PRIMARIES_CUSTOM) { - printf(" red primaries XY: %f %f\n", - color_encoding.primaries_red_xy[0], - color_encoding.primaries_red_xy[1]); - printf(" green primaries XY: %f %f\n", - color_encoding.primaries_green_xy[0], - color_encoding.primaries_green_xy[1]); - printf(" blue primaries XY: %f %f\n", - color_encoding.primaries_blue_xy[0], - color_encoding.primaries_blue_xy[1]); - } - } - if (color_encoding.transfer_function == JXL_TRANSFER_FUNCTION_GAMMA) { - printf(" transfer_function: gamma: %f\n", color_encoding.gamma); - } else { - printf(" transfer_function: %d (%s)\n", - color_encoding.transfer_function, - tf_string[color_encoding.transfer_function]); - } - printf(" rendering_intent: %d (%s)\n", color_encoding.rendering_intent, - ri_string[color_encoding.rendering_intent]); - - } else { - // The profile is not in JPEG XL encoded form, get as ICC profile - // instead. - printf(" format: ICC profile\n"); - size_t profile_size; - if (JXL_DEC_SUCCESS != - JxlDecoderGetICCProfileSize(dec, &format, - JXL_COLOR_PROFILE_TARGET_ORIGINAL, - &profile_size)) { - fprintf(stderr, "JxlDecoderGetICCProfileSize failed\n"); - continue; - } - printf(" ICC profile size: %" PRIu64 "\n", (uint64_t)profile_size); - if (profile_size < 132) { - fprintf(stderr, "ICC profile too small\n"); - continue; - } - uint8_t* profile = (uint8_t*)malloc(profile_size); - if (JXL_DEC_SUCCESS != - JxlDecoderGetColorAsICCProfile(dec, &format, - JXL_COLOR_PROFILE_TARGET_ORIGINAL, - profile, profile_size)) { - fprintf(stderr, "JxlDecoderGetColorAsICCProfile failed\n"); - free(profile); - continue; - } - printf(" CMM type: \"%.4s\"\n", profile + 4); - printf(" color space: \"%.4s\"\n", profile + 16); - printf(" rendering intent: %d\n", (int)profile[67]); - free(profile); - } - } else if (status == JXL_DEC_FRAME) { - if (JXL_DEC_SUCCESS != JxlDecoderGetFrameHeader(dec, &frame_header)) { - fprintf(stderr, "JxlDecoderGetFrameHeader failed\n"); - break; - } - printf("frame:\n"); - if (frame_header.name_length) { - char* name = malloc(frame_header.name_length + 1); - if (JXL_DEC_SUCCESS != - JxlDecoderGetFrameName(dec, name, frame_header.name_length + 1)) { - fprintf(stderr, "JxlDecoderGetFrameName failed\n"); - free(name); - break; - } - printf(" name: %s\n", name); - free(name); - } - float ms = frame_header.duration * 1000.f * - info.animation.tps_denominator / info.animation.tps_numerator; - if (info.have_animation) { - printf(" duration: %u ticks (%f ms)\n", frame_header.duration, ms); - if (info.animation.have_timecodes) { - printf(" time code: %X\n", frame_header.timecode); - } - } - if (!frame_header.name_length && !info.have_animation) { - printf(" still frame, unnamed\n"); - } - } else if (status == JXL_DEC_BOX) { - JxlBoxType type; - uint64_t size; - JxlDecoderGetBoxType(dec, type, JXL_FALSE); - JxlDecoderGetBoxSizeRaw(dec, &size); - printf("box: type: \"%c%c%c%c\" size: %" PRIu64 "\n", type[0], type[1], - type[2], type[3], (uint64_t)size); - } else { - fprintf(stderr, "Unexpected decoder status\n"); - break; - } - } - - JxlDecoderDestroy(dec); - free(data); - - return seen_basic_info; -} - -int main(int argc, char* argv[]) { - if (argc != 2) { - fprintf(stderr, - "Usage: %s \n" - "Where:\n" - " jxl = input JPEG XL image filename\n", - argv[0]); - return 1; - } - - const char* jxl_filename = argv[1]; - - FILE* file = fopen(jxl_filename, "rb"); - if (!file) { - fprintf(stderr, "Failed to read file %s\n", jxl_filename); - return 1; - } - - if (!PrintBasicInfo(file)) { - fclose(file); - fprintf(stderr, "Couldn't print basic info\n"); - return 1; - } - - fclose(file); - return 0; -} diff --git a/third_party/jpeg-xl/lib/CMakeLists.txt b/third_party/jpeg-xl/lib/CMakeLists.txt --- a/third_party/jpeg-xl/lib/CMakeLists.txt +++ b/third_party/jpeg-xl/lib/CMakeLists.txt @@ -124,11 +124,11 @@ endif() # WIN32 # Internal flags for coverage builds: if(JPEGXL_ENABLE_COVERAGE) set(JPEGXL_COVERAGE_FLAGS - -g -O0 -fprofile-arcs -ftest-coverage -DJXL_DISABLE_SLOW_TESTS + -g -O0 -fprofile-arcs -ftest-coverage -DJXL_ENABLE_ASSERT=0 -DJXL_ENABLE_CHECK=0 ) endif() # JPEGXL_ENABLE_COVERAGE endif() #!MSVC diff --git a/third_party/jpeg-xl/lib/extras/codec.cc b/third_party/jpeg-xl/lib/extras/codec.cc --- a/third_party/jpeg-xl/lib/extras/codec.cc +++ b/third_party/jpeg-xl/lib/extras/codec.cc @@ -19,11 +19,10 @@ #endif #if JPEGXL_ENABLE_EXR #include "lib/extras/enc/exr.h" #endif -#include "lib/extras/codec_psd.h" #include "lib/extras/dec/decode.h" #include "lib/extras/enc/pgx.h" #include "lib/extras/enc/pnm.h" #include "lib/extras/packed_image_convert.h" #include "lib/jxl/base/file_io.h" @@ -41,17 +40,13 @@ const extras::ColorHints& color_hints, CodecInOut* io, ThreadPool* pool, extras::Codec* orig_codec) { if (bytes.size() < kMinBytes) return JXL_FAILURE("Too few bytes"); extras::PackedPixelFile ppf; - if (extras::DecodeBytes(bytes, color_hints, io->constraints, &ppf, pool, + if (extras::DecodeBytes(bytes, color_hints, io->constraints, &ppf, orig_codec)) { return ConvertPackedPixelFileToCodecInOut(ppf, pool, io); - } else if (extras::DecodeImagePSD(bytes, color_hints, pool, io)) { - // TODO(deymo): Migrate PSD codec too. - if (orig_codec) *orig_codec = extras::Codec::kPSD; - return true; } return JXL_FAILURE("Codecs failed to decode"); } Status SetFromFile(const std::string& pathname, @@ -122,13 +117,10 @@ case extras::Codec::kPGX: return extras::EncodeImagePGX(&io, c_desired, bits_per_sample, pool, bytes); case extras::Codec::kGIF: return JXL_FAILURE("Encoding to GIF is not implemented"); - case extras::Codec::kPSD: - return extras::EncodeImagePSD(&io, c_desired, bits_per_sample, pool, - bytes); case extras::Codec::kEXR: #if JPEGXL_ENABLE_EXR return extras::EncodeImageEXR(&io, c_desired, pool, bytes); #else return JXL_FAILURE("JPEG XL was built without OpenEXR support"); @@ -145,14 +137,17 @@ ThreadPool* pool) { const std::string extension = Extension(pathname); const extras::Codec codec = extras::CodecFromExtension(extension, &bits_per_sample); - // Warn about incorrect usage of PBM/PGM/PGX/PPM - only the latter supports + // Warn about incorrect usage of PGM/PGX/PPM - only the latter supports // color, but CodecFromExtension lumps them all together. if (codec == extras::Codec::kPNM && extension != ".pfm") { - if (!io.Main().IsGray() && extension != ".ppm") { + if (io.Main().HasAlpha() && extension != ".pam") { + JXL_WARNING( + "For images with alpha, the filename should end with .pam.\n"); + } else if (!io.Main().IsGray() && extension == ".pgm") { JXL_WARNING("For color images, the filename should end with .ppm.\n"); } else if (io.Main().IsGray() && extension == ".ppm") { JXL_WARNING( "For grayscale images, the filename should not end with .ppm.\n"); } diff --git a/third_party/jpeg-xl/lib/extras/codec_psd.h b/third_party/jpeg-xl/lib/extras/codec_psd.h deleted file mode 100644 --- a/third_party/jpeg-xl/lib/extras/codec_psd.h +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) the JPEG XL Project Authors. All rights reserved. -// -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#ifndef LIB_EXTRAS_CODEC_PSD_H_ -#define LIB_EXTRAS_CODEC_PSD_H_ - -// Decodes Photoshop PSD/PSB, preserving the layers - -#include -#include - -#include "lib/extras/dec/color_hints.h" -#include "lib/jxl/base/padded_bytes.h" -#include "lib/jxl/base/span.h" -#include "lib/jxl/base/status.h" -#include "lib/jxl/codec_in_out.h" -#include "lib/jxl/color_encoding_internal.h" - -namespace jxl { -namespace extras { - -// Decodes `bytes` into `io`. -Status DecodeImagePSD(const Span bytes, - const extras::ColorHints& color_hints, ThreadPool* pool, - CodecInOut* io); - -// Not implemented yet -Status EncodeImagePSD(const CodecInOut* io, const ColorEncoding& c_desired, - size_t bits_per_sample, ThreadPool* pool, - PaddedBytes* bytes); - -} // namespace extras -} // namespace jxl - -#endif // LIB_EXTRAS_CODEC_PSD_H_ diff --git a/third_party/jpeg-xl/lib/extras/codec_psd.cc b/third_party/jpeg-xl/lib/extras/codec_psd.cc deleted file mode 100644 --- a/third_party/jpeg-xl/lib/extras/codec_psd.cc +++ /dev/null @@ -1,621 +0,0 @@ -// Copyright (c) the JPEG XL Project Authors. All rights reserved. -// -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "lib/extras/codec_psd.h" - -#include -#include -#include - -#include -#include -#include -#include -#include - -#include "lib/jxl/base/bits.h" -#include "lib/jxl/base/byte_order.h" -#include "lib/jxl/base/compiler_specific.h" -#include "lib/jxl/base/file_io.h" -#include "lib/jxl/base/printf_macros.h" -#include "lib/jxl/color_management.h" -#include "lib/jxl/common.h" -#include "lib/jxl/fields.h" // AllDefault -#include "lib/jxl/image.h" -#include "lib/jxl/image_bundle.h" -#include "lib/jxl/image_ops.h" -#include "lib/jxl/luminance.h" - -namespace jxl { -namespace extras { -namespace { - -uint64_t get_be_int(int bytes, const uint8_t*& pos, const uint8_t* maxpos) { - uint64_t r = 0; - if (pos + bytes <= maxpos) { - if (bytes == 1) { - r = *pos; - } else if (bytes == 2) { - r = LoadBE16(pos); - } else if (bytes == 4) { - r = LoadBE32(pos); - } else if (bytes == 8) { - r = LoadBE64(pos); - } - } - pos += bytes; - return r; -} - -// Copies up to n bytes, without reading from maxpos (the STL-style end). -void safe_copy(const uint8_t* JXL_RESTRICT pos, - const uint8_t* JXL_RESTRICT maxpos, char* JXL_RESTRICT out, - size_t n) { - for (size_t i = 0; i < n; ++i) { - if (pos + i >= maxpos) return; - out[i] = pos[i]; - } -} - -// maxpos is the STL-style end! The valid range is up to [pos, maxpos). -int safe_strncmp(const uint8_t* pos, const uint8_t* maxpos, const char* s2, - size_t n) { - if (pos + n > maxpos) return 1; - return strncmp((const char*)pos, s2, n); -} -constexpr int PSD_VERBOSITY = 1; - -Status decode_layer(const uint8_t*& pos, const uint8_t* maxpos, - ImageBundle& layer, std::vector chans, - std::vector invert, int w, int h, int version, - int colormodel, bool is_layer, int depth) { - int compression_method = 2; - int nb_channels = chans.size(); - JXL_DEBUG_V(PSD_VERBOSITY, - "Trying to decode layer with dimensions %ix%i and %i channels", w, - h, nb_channels); - if (w <= 0 || h <= 0) return JXL_FAILURE("PSD: empty layer"); - for (int c = 0; c < nb_channels; c++) { - // skip nop byte padding - while (pos < maxpos && *pos == 128) pos++; - JXL_DEBUG_V(PSD_VERBOSITY, "Channel %i (pos %" PRIuS ")", c, (size_t)pos); - // Merged image stores all channels together (same compression method) - // Layers store channel per channel - if (is_layer || c == 0) { - compression_method = get_be_int(2, pos, maxpos); - JXL_DEBUG_V(PSD_VERBOSITY, "compression method: %i", compression_method); - if (compression_method > 1 || compression_method < 0) { - return JXL_FAILURE("PSD: can't handle compression method %i", - compression_method); - } - } - - if (!is_layer && c < colormodel) { - // skip to the extra channels - if (compression_method == 0) { - pos += w * h * (depth >> 3) * colormodel; - c = colormodel - 1; - continue; - } - size_t skip_amount = 0; - for (int i = 0; i < nb_channels; i++) { - if (i < colormodel) { - for (int y = 0; y < h; y++) { - skip_amount += get_be_int(2 * version, pos, maxpos); - } - } else { - pos += h * 2 * version; - } - } - pos += skip_amount; - c = colormodel - 1; - continue; - } - if (is_layer || c == 0) { - // skip the line-counts, we don't need them - if (compression_method == 1) { - pos += h * (is_layer ? 1 : nb_channels) * 2 * - version; // PSB uses 4 bytes per rowsize instead of 2 - } - } - int c_id = chans[c]; - if (c_id < 0) continue; // skip - if (static_cast(c_id) >= 3 + layer.extra_channels().size()) - return JXL_FAILURE("PSD: can't handle channel id %i", c_id); - ImageF& ch = (c_id < 3 ? layer.color()->Plane(c_id) - : layer.extra_channels()[c_id - 3]); - - for (int y = 0; y < h; y++) { - if (pos > maxpos) return JXL_FAILURE("PSD: premature end of input"); - float* const JXL_RESTRICT row = ch.Row(y); - if (compression_method == 0) { - // uncompressed is easy - if (depth == 8) { - for (int x = 0; x < w; x++) { - row[x] = get_be_int(1, pos, maxpos) * (1.f / 255.f); - } - } else if (depth == 16) { - for (int x = 0; x < w; x++) { - row[x] = get_be_int(2, pos, maxpos) * (1.f / 65535.f); - } - } else if (depth == 32) { - for (int x = 0; x < w; x++) { - uint32_t f = get_be_int(4, pos, maxpos); - memcpy(&row[x], &f, 4); - } - } - } else { - // RLE is not that hard - if (depth != 8) - return JXL_FAILURE("PSD: did not expect RLE with depth>1"); - for (int x = 0; x < w;) { - if (pos >= maxpos) return JXL_FAILURE("PSD: out of bounds"); - int8_t rle = *pos++; - if (rle <= 0) { - if (rle == -128) continue; // nop - int count = 1 - rle; - float v = get_be_int(1, pos, maxpos) * (1.f / 255.f); - while (count && x < w) { - row[x] = v; - count--; - x++; - } - if (count) return JXL_FAILURE("PSD: row overflow"); - } else { - int count = 1 + rle; - while (count && x < w) { - row[x] = get_be_int(1, pos, maxpos) * (1.f / 255.f); - count--; - x++; - } - if (count) return JXL_FAILURE("PSD: row overflow"); - } - } - } - if (invert[c]) { - // sometimes 0 means full ink - for (int x = 0; x < w; x++) { - row[x] = 1.f - row[x]; - } - } - } - JXL_DEBUG_V(PSD_VERBOSITY, "Channel %i read.", c); - } - - return true; -} - -} // namespace - -Status DecodeImagePSD(const Span bytes, - const ColorHints& /*color_hints*/, ThreadPool* pool, - CodecInOut* io) { - const uint8_t* pos = bytes.data(); - const uint8_t* maxpos = bytes.data() + bytes.size(); - if (safe_strncmp(pos, maxpos, "8BPS", 4)) return false; // not a PSD file - JXL_DEBUG_V(PSD_VERBOSITY, "trying psd decode"); - pos += 4; - int version = get_be_int(2, pos, maxpos); - JXL_DEBUG_V(PSD_VERBOSITY, "Version=%i", version); - if (version < 1 || version > 2) - return JXL_FAILURE("PSD: unknown format version"); - // PSD = version 1, PSB = version 2 - pos += 6; - int nb_channels = get_be_int(2, pos, maxpos); - size_t ysize = get_be_int(4, pos, maxpos); - size_t xsize = get_be_int(4, pos, maxpos); - const SizeConstraints* constraints = &io->constraints; - JXL_RETURN_IF_ERROR(VerifyDimensions(constraints, xsize, ysize)); - uint64_t total_pixel_count = static_cast(xsize) * ysize; - int bitdepth = get_be_int(2, pos, maxpos); - if (bitdepth != 8 && bitdepth != 16 && bitdepth != 32) { - return JXL_FAILURE("PSD: bit depth %i invalid or not supported", bitdepth); - } - if (bitdepth == 32) { - io->metadata.m.SetFloat32Samples(); - } else { - io->metadata.m.SetUintSamples(bitdepth); - } - int colormodel = get_be_int(2, pos, maxpos); - // 1 = Grayscale, 3 = RGB, 4 = CMYK - if (colormodel != 1 && colormodel != 3 && colormodel != 4) - return JXL_FAILURE("PSD: unsupported color model"); - - int real_nb_channels = colormodel; - std::vector> spotcolor; - - if (get_be_int(4, pos, maxpos)) - return JXL_FAILURE("PSD: Unsupported color mode section"); - - bool hasmergeddata = true; - bool have_alpha = false; - bool merged_has_alpha = false; - bool color_already_set = false; - size_t metalength = get_be_int(4, pos, maxpos); - const uint8_t* metaoffset = pos; - while (pos < metaoffset + metalength) { - char header[5] = "????"; - safe_copy(pos, maxpos, header, 4); - if (memcmp(header, "8BIM", 4) != 0) { - return JXL_FAILURE("PSD: Unexpected image resource header: %s", header); - } - pos += 4; - int id = get_be_int(2, pos, maxpos); - int namelength = get_be_int(1, pos, maxpos); - pos += namelength; - if (!(namelength & 1)) pos++; // padding to even length - size_t blocklength = get_be_int(4, pos, maxpos); - // JXL_DEBUG_V(PSD_VERBOSITY, "block id: %i | block length: %" PRIuS,id, - // blocklength); - if (pos > maxpos) return JXL_FAILURE("PSD: Unexpected end of file"); - if (id == 1039) { // ICC profile - size_t delta = maxpos - pos; - if (delta < blocklength) { - return JXL_FAILURE("PSD: Invalid block length"); - } - PaddedBytes icc; - icc.resize(blocklength); - memcpy(icc.data(), pos, blocklength); - if (!io->metadata.m.color_encoding.SetICC(std::move(icc))) { - return JXL_FAILURE("PSD: Invalid color profile"); - } - color_already_set = true; - } else if (id == 1057) { // compatibility mode or not? - if (get_be_int(4, pos, maxpos) != 1) { - return JXL_FAILURE("PSD: expected version=1 in id=1057 resource block"); - } - hasmergeddata = get_be_int(1, pos, maxpos); - pos++; - blocklength -= 6; // already skipped these bytes - } else if (id == 1077) { // spot colors - int version = get_be_int(4, pos, maxpos); - if (version != 1) { - return JXL_FAILURE( - "PSD: expected DisplayInfo version 1, got version %i", version); - } - int spotcolorcount = nb_channels - colormodel; - JXL_DEBUG_V(PSD_VERBOSITY, "Reading %i spot colors. %" PRIuS, - spotcolorcount, blocklength); - for (int k = 0; k < spotcolorcount; k++) { - int colorspace = get_be_int(2, pos, maxpos); - if ((colormodel == 3 && colorspace != 0) || - (colormodel == 4 && colorspace != 2)) { - return JXL_FAILURE( - "PSD: cannot handle spot colors in different color spaces than " - "image itself"); - } - if (colorspace == 2) JXL_WARNING("PSD: K ignored in CMYK spot color"); - std::vector color; - color.push_back(get_be_int(2, pos, maxpos) / 65535.f); // R or C - color.push_back(get_be_int(2, pos, maxpos) / 65535.f); // G or M - color.push_back(get_be_int(2, pos, maxpos) / 65535.f); // B or Y - color.push_back(get_be_int(2, pos, maxpos) / 65535.f); // ignored or K - color.push_back(get_be_int(2, pos, maxpos) / - 100.f); // solidity (alpha, basically) - int kind = get_be_int(1, pos, maxpos); - JXL_DEBUG_V(PSD_VERBOSITY, "Kind=%i", kind); - color.push_back(kind); - spotcolor.push_back(color); - if (kind == 2) { - JXL_DEBUG_V(PSD_VERBOSITY, "Actual spot color"); - } else if (kind == 1) { - JXL_DEBUG_V(PSD_VERBOSITY, "Mask (alpha) channel"); - } else if (kind == 0) { - JXL_DEBUG_V(PSD_VERBOSITY, "Selection (alpha) channel"); - } else { - return JXL_FAILURE("PSD: Unknown extra channel type"); - } - } - if (blocklength & 1) pos++; - blocklength = 0; - } - pos += blocklength; - if (blocklength & 1) pos++; // padding again - } - - // TODO(deymo): Apply color hints when PSD is converted to PackedPixelFile. - (void)color_already_set; - // JXL_RETURN_IF_ERROR(ApplyColorHints(color_hints, color_already_set, - // /*is_gray=*/false, io)); - - size_t layerlength = get_be_int(4 * version, pos, maxpos); - const uint8_t* after_layers_pos = pos + layerlength; - if (after_layers_pos < pos) return JXL_FAILURE("PSD: invalid layer length"); - if (layerlength) { - pos += 4 * version; // don't care about layerinfolength - JXL_DEBUG_V(PSD_VERBOSITY, "Layer section length: %" PRIuS, layerlength); - int layercount = static_cast(get_be_int(2, pos, maxpos)); - JXL_DEBUG_V(PSD_VERBOSITY, "Layer count: %i", layercount); - io->frames.clear(); - - if (layercount == 0) { - if (get_be_int(2, pos, maxpos) != 0) { - return JXL_FAILURE( - "PSD: Expected zero padding before additional layer info"); - } - while (pos < after_layers_pos) { - if (safe_strncmp(pos, maxpos, "8BIM", 4) && - safe_strncmp(pos, maxpos, "8B64", 4)) - return JXL_FAILURE("PSD: Unexpected layer info signature"); - pos += 4; - const uint8_t* tpos = pos; - pos += 4; - size_t blocklength = get_be_int(4 * version, pos, maxpos); - JXL_DEBUG_V(PSD_VERBOSITY, "Length=%" PRIuS, blocklength); - if (blocklength > 0) { - if (pos >= maxpos) return JXL_FAILURE("PSD: Unexpected end of file"); - size_t delta = maxpos - pos; - if (delta < blocklength) { - return JXL_FAILURE("PSD: Invalid block length"); - } - } - if (!safe_strncmp(tpos, maxpos, "Layr", 4) || - !safe_strncmp(tpos, maxpos, "Lr16", 4) || - !safe_strncmp(tpos, maxpos, "Lr32", 4)) { - layercount = static_cast(get_be_int(2, pos, maxpos)); - if (layercount < 0) { - return JXL_FAILURE("PSD: Invalid layer count"); - } - JXL_DEBUG_V(PSD_VERBOSITY, "Real layer count: %i", layercount); - if (layercount > 1) have_alpha = true; - break; - } - if (!safe_strncmp(tpos, maxpos, "Mtrn", 4) || - !safe_strncmp(tpos, maxpos, "Mt16", 4) || - !safe_strncmp(tpos, maxpos, "Mt32", 4)) { - JXL_DEBUG_V(PSD_VERBOSITY, "Merged layer has transparency channel"); - if (nb_channels > real_nb_channels) { - have_alpha = true; - merged_has_alpha = true; - } - } - pos += blocklength; - } - } else if (layercount < 0) { - // negative layer count indicates merged has alpha and it is to be shown - if (nb_channels > real_nb_channels) { - have_alpha = true; - merged_has_alpha = true; - } - layercount = -layercount; - } else { - // multiple layers implies there is alpha - have_alpha = true; - } - - ExtraChannelInfo info; - info.bit_depth.bits_per_sample = bitdepth; - info.dim_shift = 0; - - if (colormodel == 4) { // cmyk - info.type = ExtraChannel::kBlack; - io->metadata.m.extra_channel_info.push_back(info); - } - if (have_alpha) { - JXL_DEBUG_V(PSD_VERBOSITY, "Have alpha"); - real_nb_channels++; - info.type = ExtraChannel::kAlpha; - info.alpha_associated = - false; // true? PSD is not consistent with this, need to check - io->metadata.m.extra_channel_info.push_back(info); - } - if (merged_has_alpha && !spotcolor.empty() && spotcolor[0][5] == 1) { - // first alpha channel - spotcolor.erase(spotcolor.begin()); - } - for (size_t i = 0; i < spotcolor.size(); i++) { - real_nb_channels++; - if (spotcolor[i][5] == 2) { - info.type = ExtraChannel::kSpotColor; - info.spot_color[0] = spotcolor[i][0]; - info.spot_color[1] = spotcolor[i][1]; - info.spot_color[2] = spotcolor[i][2]; - info.spot_color[3] = spotcolor[i][4]; - } else if (spotcolor[i][5] == 1) { - info.type = ExtraChannel::kAlpha; - } else if (spotcolor[i][5] == 0) { - info.type = ExtraChannel::kSelectionMask; - } else - return JXL_FAILURE("PSD: unhandled extra channel"); - io->metadata.m.extra_channel_info.push_back(info); - } - std::vector> layer_chan_id; - std::vector layer_offsets(layercount + 1, 0); - std::vector is_real_layer(layercount, false); - for (int l = 0; l < layercount; l++) { - ImageBundle layer(&io->metadata.m); - layer.duration = 0; - layer.blend = (l > 0); - - layer.use_for_next_frame = (l + 1 < layercount); - layer.origin.y0 = get_be_int(4, pos, maxpos); - layer.origin.x0 = get_be_int(4, pos, maxpos); - size_t height = get_be_int(4, pos, maxpos) - layer.origin.y0; - size_t width = get_be_int(4, pos, maxpos) - layer.origin.x0; - JXL_DEBUG_V(PSD_VERBOSITY, - "Layer %i: %" PRIuS " x %" PRIuS " at origin (%i, %i)", l, - width, height, layer.origin.x0, layer.origin.y0); - int nb_chs = get_be_int(2, pos, maxpos); - JXL_DEBUG_V(PSD_VERBOSITY, " channels: %i", nb_chs); - std::vector chan_ids; - layer_offsets[l + 1] = layer_offsets[l]; - for (int lc = 0; lc < nb_chs; lc++) { - int id = get_be_int(2, pos, maxpos); - JXL_DEBUG_V(PSD_VERBOSITY, " id=%i", id); - if (id == 65535) { - chan_ids.push_back(colormodel); // alpha - } else if (id == 65534) { - chan_ids.push_back(-1); // layer mask, ignored - } else { - chan_ids.push_back(id); // color channel - } - layer_offsets[l + 1] += get_be_int(4 * version, pos, maxpos); - } - layer_chan_id.push_back(chan_ids); - if (safe_strncmp(pos, maxpos, "8BIM", 4)) - return JXL_FAILURE("PSD: Layer %i: Unexpected signature (not 8BIM)", l); - pos += 4; - if (safe_strncmp(pos, maxpos, "norm", 4)) { - return JXL_FAILURE( - "PSD: Layer %i: Cannot handle non-default blend mode", l); - } - pos += 4; - int opacity = get_be_int(1, pos, maxpos); - if (opacity < 100) { - JXL_WARNING( - "PSD: ignoring opacity of semi-transparent layer %i (opacity=%i)", - l, opacity); - } - pos++; // clipping - int flags = get_be_int(1, pos, maxpos); - pos++; - bool invisible = (flags & 2); - if (invisible) { - if (l + 1 < layercount) { - layer.blend = false; - layer.use_for_next_frame = false; - } else { - // TODO: instead add dummy last frame? - JXL_WARNING("PSD: invisible top layer was made visible"); - } - } - size_t extradata = get_be_int(4, pos, maxpos); - JXL_DEBUG_V(PSD_VERBOSITY, " extradata: %" PRIuS " bytes", extradata); - const uint8_t* after_extra = pos + extradata; - // TODO: deal with non-empty layer masks - pos += get_be_int(4, pos, maxpos); // skip layer mask data - pos += get_be_int(4, pos, maxpos); // skip layer blend range data - size_t namelength = get_be_int(1, pos, maxpos); - size_t delta = maxpos - pos; - if (delta < namelength) return JXL_FAILURE("PSD: Invalid block length"); - char lname[256] = {}; - memcpy(lname, pos, namelength); - lname[namelength] = 0; - JXL_DEBUG_V(PSD_VERBOSITY, " name: %s", lname); - pos = after_extra; - if (width == 0 || height == 0) { - JXL_DEBUG_V(PSD_VERBOSITY, - " NOT A REAL LAYER"); // probably layer group - continue; - } - is_real_layer[l] = true; - JXL_RETURN_IF_ERROR(VerifyDimensions(constraints, width, height)); - uint64_t pixel_count = static_cast(width) * height; - if (!SafeAdd(total_pixel_count, pixel_count, total_pixel_count)) { - return JXL_FAILURE("Image too big"); - } - if (total_pixel_count > constraints->dec_max_pixels) { - return JXL_FAILURE("Image too big"); - } - Image3F rgb(width, height); - layer.SetFromImage(std::move(rgb), io->metadata.m.color_encoding); - std::vector ec; - for (const auto& ec_meta : layer.metadata()->extra_channel_info) { - ImageF extra(width, height); - if (ec_meta.type == ExtraChannel::kAlpha) { - FillPlane(1.0f, &extra, Rect(extra)); // opaque - } else { - ZeroFillPlane(&extra, Rect(extra)); // zeroes - } - ec.push_back(std::move(extra)); - } - if (!ec.empty()) layer.SetExtraChannels(std::move(ec)); - layer.name = lname; - io->dec_pixels += layer.xsize() * layer.ysize(); - io->frames.push_back(std::move(layer)); - } - - std::vector invert(real_nb_channels, false); - int il = 0; - const uint8_t* bpos = pos; - for (int l = 0; l < layercount; l++) { - if (!is_real_layer[l]) continue; - pos = bpos + layer_offsets[l]; - if (pos < bpos) return JXL_FAILURE("PSD: invalid layer offset"); - JXL_DEBUG_V(PSD_VERBOSITY, "At position %i (%" PRIuS ")", - (int)(pos - bytes.data()), (size_t)pos); - ImageBundle& layer = io->frames[il++]; - std::vector& chan_id = layer_chan_id[l]; - if (chan_id.size() > invert.size()) invert.resize(chan_id.size(), false); - JXL_RETURN_IF_ERROR(decode_layer(pos, maxpos, layer, chan_id, invert, - layer.xsize(), layer.ysize(), version, - colormodel, true, bitdepth)); - } - } else - return JXL_FAILURE("PSD: no layer data found"); - - if (!hasmergeddata && !spotcolor.empty()) { - return JXL_FAILURE("PSD: extra channel data declared but not found"); - } - - if (!spotcolor.empty() || (hasmergeddata && io->frames.empty())) { - // PSD only has spot colors / extra alpha/mask data in the merged image - // We don't redundantly store the merged image, so we put it in the first - // layer (the next layers will kAdd zeroes to it) - pos = after_layers_pos; - bool have_only_merged = false; - if (io->frames.empty()) { - // There is only the merged image, no layers - ImageBundle nlayer(&io->metadata.m); - Image3F rgb(xsize, ysize); - nlayer.SetFromImage(std::move(rgb), io->metadata.m.color_encoding); - std::vector ec; - for (const auto& ec_meta : nlayer.metadata()->extra_channel_info) { - ImageF extra(xsize, ysize); - if (ec_meta.type == ExtraChannel::kAlpha) { - FillPlane(1.0f, &extra, Rect(extra)); // opaque - } else { - ZeroFillPlane(&extra, Rect(extra)); // zeroes - } - ec.push_back(std::move(extra)); - } - if (!ec.empty()) nlayer.SetExtraChannels(std::move(ec)); - io->dec_pixels += nlayer.xsize() * nlayer.ysize(); - io->frames.push_back(std::move(nlayer)); - have_only_merged = true; - } - ImageBundle& layer = io->frames[0]; - std::vector chan_id(real_nb_channels); - std::iota(chan_id.begin(), chan_id.end(), 0); - std::vector invert(real_nb_channels, false); - if (static_cast(spotcolor.size()) + colormodel + 1 < - real_nb_channels) { - return JXL_FAILURE("Inconsistent layer configuration"); - } - if (!merged_has_alpha) { - if (colormodel >= real_nb_channels) { - return JXL_FAILURE("Inconsistent layer configuration"); - } - chan_id.erase(chan_id.begin() + colormodel); - invert.erase(invert.begin() + colormodel); - } else { - colormodel++; - } - for (size_t i = colormodel; i < invert.size(); i++) { - if (spotcolor[i - colormodel][5] == 2) invert[i] = true; - if (spotcolor[i - colormodel][5] == 0) invert[i] = true; - } - JXL_RETURN_IF_ERROR(decode_layer( - pos, maxpos, layer, chan_id, invert, layer.xsize(), layer.ysize(), - version, (have_only_merged ? 0 : colormodel), false, bitdepth)); - } - - if (io->frames.empty()) return JXL_FAILURE("PSD: no layers"); - - io->SetSize(xsize, ysize); - - SetIntensityTarget(io); - - return true; -} - -Status EncodeImagePSD(const CodecInOut* io, const ColorEncoding& c_desired, - size_t bits_per_sample, ThreadPool* pool, - PaddedBytes* bytes) { - return JXL_FAILURE("PSD encoding not yet implemented"); -} - -} // namespace extras -} // namespace jxl diff --git a/third_party/jpeg-xl/lib/extras/codec_test.cc b/third_party/jpeg-xl/lib/extras/codec_test.cc --- a/third_party/jpeg-xl/lib/extras/codec_test.cc +++ b/third_party/jpeg-xl/lib/extras/codec_test.cc @@ -27,10 +27,35 @@ namespace jxl { namespace extras { namespace { +std::string ExtensionFromCodec(Codec codec, const bool is_gray, + const bool has_alpha, + const size_t bits_per_sample) { + switch (codec) { + case Codec::kJPG: + return ".jpg"; + case Codec::kPGX: + return ".pgx"; + case Codec::kPNG: + return ".png"; + case Codec::kPNM: + if (has_alpha) return ".pam"; + if (is_gray) return ".pgm"; + return (bits_per_sample == 32) ? ".pfm" : ".ppm"; + case Codec::kGIF: + return ".gif"; + case Codec::kEXR: + return ".exr"; + case Codec::kUnknown: + return std::string(); + } + JXL_UNREACHABLE; + return std::string(); +} + CodecInOut CreateTestImage(const size_t xsize, const size_t ysize, const bool is_gray, const bool add_alpha, const size_t bits_per_sample, const ColorEncoding& c_native) { Image3F image(xsize, ysize); @@ -74,11 +99,11 @@ if (codec == Codec::kPNM && add_alpha) return; // Our EXR codec always uses 16-bit premultiplied alpha, does not support // grayscale, and somehow does not have sufficient precision for this test. if (codec == Codec::kEXR) return; printf("Codec %s bps:%" PRIuS " gr:%d al:%d\n", - ExtensionFromCodec(codec, is_gray, bits_per_sample).c_str(), + ExtensionFromCodec(codec, is_gray, add_alpha, bits_per_sample).c_str(), bits_per_sample, is_gray, add_alpha); ColorEncoding c_native; c_native.SetColorSpace(is_gray ? ColorSpace::kGray : ColorSpace::kRGB); // Note: this must not be wider than c_external, otherwise gamut clipping @@ -229,15 +254,15 @@ #if 0 TEST(CodecTest, TestMetadataSRGB) { ThreadPoolInternal pool(12); - const char* paths[] = {"raw.pixls/DJI-FC6310-16bit_srgb8_v4_krita.png", - "raw.pixls/Google-Pixel2XL-16bit_srgb8_v4_krita.png", - "raw.pixls/HUAWEI-EVA-L09-16bit_srgb8_dt.png", - "raw.pixls/Nikon-D300-12bit_srgb8_dt.png", - "raw.pixls/Sony-DSC-RX1RM2-14bit_srgb8_v4_krita.png"}; + const char* paths[] = {"third_party/raw.pixls/DJI-FC6310-16bit_srgb8_v4_krita.png", + "third_party/raw.pixls/Google-Pixel2XL-16bit_srgb8_v4_krita.png", + "third_party/raw.pixls/HUAWEI-EVA-L09-16bit_srgb8_dt.png", + "third_party/raw.pixls/Nikon-D300-12bit_srgb8_dt.png", + "third_party/raw.pixls/Sony-DSC-RX1RM2-14bit_srgb8_v4_krita.png"}; for (const char* relative_pathname : paths) { const CodecInOut io = DecodeRoundtrip(relative_pathname, Codec::kPNG, &pool); EXPECT_EQ(8, io.metadata.m.bit_depth.bits_per_sample); EXPECT_FALSE(io.metadata.m.bit_depth.floating_point_sample); @@ -258,13 +283,13 @@ TEST(CodecTest, TestMetadataLinear) { ThreadPoolInternal pool(12); const char* paths[3] = { - "raw.pixls/Google-Pixel2XL-16bit_acescg_g1_v4_krita.png", - "raw.pixls/HUAWEI-EVA-L09-16bit_709_g1_dt.png", - "raw.pixls/Nikon-D300-12bit_2020_g1_dt.png", + "third_party/raw.pixls/Google-Pixel2XL-16bit_acescg_g1_v4_krita.png", + "third_party/raw.pixls/HUAWEI-EVA-L09-16bit_709_g1_dt.png", + "third_party/raw.pixls/Nikon-D300-12bit_2020_g1_dt.png", }; const WhitePoint white_points[3] = {WhitePoint::kCustom, WhitePoint::kD65, WhitePoint::kD65}; const Primaries primaries[3] = {Primaries::kCustom, Primaries::kSRGB, Primaries::k2100}; @@ -290,12 +315,12 @@ TEST(CodecTest, TestMetadataICC) { ThreadPoolInternal pool(12); const char* paths[] = { - "raw.pixls/DJI-FC6310-16bit_709_v4_krita.png", - "raw.pixls/Sony-DSC-RX1RM2-14bit_709_v4_krita.png", + "third_party/raw.pixls/DJI-FC6310-16bit_709_v4_krita.png", + "third_party/raw.pixls/Sony-DSC-RX1RM2-14bit_709_v4_krita.png", }; for (const char* relative_pathname : paths) { const CodecInOut io = DecodeRoundtrip(relative_pathname, Codec::kPNG, &pool); EXPECT_GE(16, io.metadata.m.bit_depth.bits_per_sample); @@ -313,32 +338,32 @@ EXPECT_EQ(Primaries::kSRGB, c_original.primaries); EXPECT_EQ(TransferFunction::k709, c_original.tf.GetTransferFunction()); } } -TEST(CodecTest, TestPNGSuite) { +TEST(CodecTest, Testthird_party/pngsuite) { ThreadPoolInternal pool(12); // Ensure we can load PNG with text, japanese UTF-8, compressed text. - (void)DecodeRoundtrip("pngsuite/ct1n0g04.png", Codec::kPNG, &pool); - (void)DecodeRoundtrip("pngsuite/ctjn0g04.png", Codec::kPNG, &pool); - (void)DecodeRoundtrip("pngsuite/ctzn0g04.png", Codec::kPNG, &pool); + (void)DecodeRoundtrip("third_party/pngsuite/ct1n0g04.png", Codec::kPNG, &pool); + (void)DecodeRoundtrip("third_party/pngsuite/ctjn0g04.png", Codec::kPNG, &pool); + (void)DecodeRoundtrip("third_party/pngsuite/ctzn0g04.png", Codec::kPNG, &pool); // Extract gAMA const CodecInOut b1 = - DecodeRoundtrip("pngsuite/g10n3p04.png", Codec::kPNG, &pool); + DecodeRoundtrip("third_party/pngsuite/g10n3p04.png", Codec::kPNG, &pool); EXPECT_TRUE(b1.metadata.color_encoding.tf.IsLinear()); // Extract cHRM const CodecInOut b_p = - DecodeRoundtrip("pngsuite/ccwn2c08.png", Codec::kPNG, &pool); + DecodeRoundtrip("third_party/pngsuite/ccwn2c08.png", Codec::kPNG, &pool); EXPECT_EQ(Primaries::kSRGB, b_p.metadata.color_encoding.primaries); EXPECT_EQ(WhitePoint::kD65, b_p.metadata.color_encoding.white_point); // Extract EXIF from (new-style) dedicated chunk const CodecInOut b_exif = - DecodeRoundtrip("pngsuite/exif2c08.png", Codec::kPNG, &pool); + DecodeRoundtrip("third_party/pngsuite/exif2c08.png", Codec::kPNG, &pool); EXPECT_EQ(978, b_exif.blobs.exif.size()); } #endif void VerifyWideGamutMetadata(const std::string& relative_pathname, @@ -357,17 +382,17 @@ EXPECT_EQ(primaries, c_original.primaries); } TEST(CodecTest, TestWideGamut) { ThreadPoolInternal pool(12); - // VerifyWideGamutMetadata("wide-gamut-tests/P3-sRGB-color-bars.png", + // VerifyWideGamutMetadata("third_party/wide-gamut-tests/P3-sRGB-color-bars.png", // Primaries::kP3, &pool); - VerifyWideGamutMetadata("wide-gamut-tests/P3-sRGB-color-ring.png", + VerifyWideGamutMetadata("third_party/wide-gamut-tests/P3-sRGB-color-ring.png", Primaries::kP3, &pool); - // VerifyWideGamutMetadata("wide-gamut-tests/R2020-sRGB-color-bars.png", + // VerifyWideGamutMetadata("third_party/wide-gamut-tests/R2020-sRGB-color-bars.png", // Primaries::k2100, &pool); - // VerifyWideGamutMetadata("wide-gamut-tests/R2020-sRGB-color-ring.png", + // VerifyWideGamutMetadata("third_party/wide-gamut-tests/R2020-sRGB-color-ring.png", // Primaries::k2100, &pool); } TEST(CodecTest, TestPNM) { TestCodecPNM(); } diff --git a/third_party/jpeg-xl/lib/extras/dec/apng.cc b/third_party/jpeg-xl/lib/extras/dec/apng.cc --- a/third_party/jpeg-xl/lib/extras/dec/apng.cc +++ b/third_party/jpeg-xl/lib/extras/dec/apng.cc @@ -41,10 +41,11 @@ #include #include #include +#include "jxl/codestream_header.h" #include "jxl/encode.h" #include "lib/jxl/base/compiler_specific.h" #include "lib/jxl/base/printf_macros.h" #include "lib/jxl/base/scope_guard.h" #include "lib/jxl/common.h" @@ -54,10 +55,16 @@ namespace jxl { namespace extras { namespace { +/* hIST chunk tail is not proccesed properly; skip this chunk completely; + see https://github.com/glennrp/libpng/pull/413 */ +const png_byte kIgnoredPngChunks[] = { + 104, 73, 83, 84, '\0' /* hIST */ +}; + // Returns floating-point value from the PNG encoding (times 10^5). static double F64FromU32(const uint32_t x) { return static_cast(x) * 1E-5; } @@ -161,10 +168,35 @@ } JXL_ASSERT(*nibble < 16); return true; } + // Returns false if invalid. + static JXL_INLINE Status DecodeDecimal(const char** pos, const char* end, + uint32_t* JXL_RESTRICT value) { + size_t len = 0; + *value = 0; + while (*pos < end) { + char next = **pos; + if (next >= '0' && next <= '9') { + *value = (*value * 10) + static_cast(next - '0'); + len++; + if (len > 8) { + break; + } + } else { + // Do not consume terminator (non-decimal digit). + break; + } + (*pos)++; + } + if (len == 0 || len > 8) { + return JXL_FAILURE("Failed to parse decimal"); + } + return true; + } + // Parses a PNG text chunk with key of the form "Raw profile type ####", with // #### a type. // Returns whether it could successfully parse the content. // We trust key and encoded are null-terminated because they come from // libpng. @@ -194,21 +226,19 @@ } if (pos == encoded_end) return false; // We parsed so far a \n, some number of non \n characters and are now // pointing at a \n. if (*(pos++) != '\n') return false; - unsigned long bytes_to_decode; - const int fields = sscanf(pos, "%8lu", &bytes_to_decode); - if (fields != 1) return false; // Failed to decode metadata header - JXL_ASSERT(pos + 8 <= encoded_end); - pos += 8; // read %8lu + uint32_t bytes_to_decode = 0; + JXL_RETURN_IF_ERROR(DecodeDecimal(&pos, encoded_end, &bytes_to_decode)); - // We need 2*bytes for the hex values plus 1 byte every 36 values. + // We need 2*bytes for the hex values plus 1 byte every 36 values, + // plus terminal \n for length. const unsigned long needed_bytes = bytes_to_decode * 2 + 1 + DivCeil(bytes_to_decode, 36); if (needed_bytes != static_cast(encoded_end - pos)) { - return JXL_FAILURE("Not enough bytes to parse %lu bytes in hex", + return JXL_FAILURE("Not enough bytes to parse %d bytes in hex", bytes_to_decode); } JXL_ASSERT(bytes->empty()); bytes->reserve(bytes_to_decode); @@ -249,11 +279,11 @@ constexpr uint32_t kId_gAMA = 0x414D4167; constexpr uint32_t kId_cHRM = 0x4D524863; constexpr uint32_t kId_eXIf = 0x66495865; struct APNGFrame { - PaddedBytes pixels; + std::vector pixels; std::vector rows; unsigned int w, h, delay_num, delay_den; }; struct Reader { @@ -268,11 +298,11 @@ } bool Eof() { return next == last; } }; const unsigned long cMaxPNGSize = 1000000UL; -const size_t kMaxPNGChunkSize = 100000000; // 100 MB +const size_t kMaxPNGChunkSize = 1lu << 30; // 1 GB void info_fn(png_structp png_ptr, png_infop info_ptr) { png_set_expand(png_ptr); png_set_palette_to_rgb(png_ptr); png_set_tRNS_to_alpha(png_ptr); @@ -282,15 +312,16 @@ void row_fn(png_structp png_ptr, png_bytep new_row, png_uint_32 row_num, int pass) { APNGFrame* frame = (APNGFrame*)png_get_progressive_ptr(png_ptr); JXL_CHECK(frame); + JXL_CHECK(row_num < frame->rows.size()); JXL_CHECK(frame->rows[row_num] < frame->pixels.data() + frame->pixels.size()); png_progressive_combine_row(png_ptr, frame->rows[row_num], new_row); } -inline unsigned int read_chunk(Reader* r, PaddedBytes* pChunk) { +inline unsigned int read_chunk(Reader* r, std::vector* pChunk) { unsigned char len[4]; if (r->Read(&len, 4)) { const auto size = png_get_uint_32(len); // Check first, to avoid overflow. if (size > kMaxPNGChunkSize) { @@ -305,22 +336,25 @@ } return 0; } int processing_start(png_structp& png_ptr, png_infop& info_ptr, void* frame_ptr, - bool hasInfo, PaddedBytes& chunkIHDR, - std::vector& chunksInfo) { + bool hasInfo, std::vector& chunkIHDR, + std::vector>& chunksInfo) { unsigned char header[8] = {137, 80, 78, 71, 13, 10, 26, 10}; png_ptr = png_create_read_struct(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL); info_ptr = png_create_info_struct(png_ptr); if (!png_ptr || !info_ptr) return 1; if (setjmp(png_jmpbuf(png_ptr))) { return 1; } + png_set_keep_unknown_chunks(png_ptr, 1, kIgnoredPngChunks, + (int)sizeof(kIgnoredPngChunks) / 5); + png_set_crc_action(png_ptr, PNG_CRC_QUIET_USE, PNG_CRC_QUIET_USE); png_set_progressive_read_fn(png_ptr, frame_ptr, info_fn, row_fn, NULL); png_process_data(png_ptr, info_ptr, header, 8); png_process_data(png_ptr, info_ptr, chunkIHDR.data(), chunkIHDR.size()); @@ -378,13 +412,13 @@ unsigned int id, j, w, h, w0, h0, x0, y0; unsigned int delay_num, delay_den, dop, bop, rowbytes, imagesize; unsigned char sig[8]; png_structp png_ptr = nullptr; png_infop info_ptr = nullptr; - PaddedBytes chunk; - PaddedBytes chunkIHDR; - std::vector chunksInfo; + std::vector chunk; + std::vector chunkIHDR; + std::vector> chunksInfo; bool isAnimated = false; bool hasInfo = false; APNGFrame frameRaw = {}; uint32_t num_channels; JxlPixelFormat format; @@ -592,12 +626,11 @@ png_charp name; png_uint_32 proflen = 0; auto ok = png_get_iCCP(png_ptr, info_ptr, &name, &compression_type, &profile, &proflen); if (ok && proflen) { - ppf->icc.resize(proflen); - memcpy(ppf->icc.data(), profile, proflen); + ppf->icc.assign(profile, profile + proflen); have_color = true; } else { // TODO(eustas): JXL_WARNING? } } else if (id == kId_sRGB) { @@ -668,11 +701,12 @@ bool should_blend = frame.blend_op == BLEND_OP_OVER; bool use_for_next_frame = has_nontrivial_background && frame.dispose_op != DISPOSE_OP_PREVIOUS; size_t x0 = frame.x0; size_t y0 = frame.y0; - + size_t xsize = frame.data.xsize; + size_t ysize = frame.data.ysize; if (previous_frame_should_be_cleared) { size_t xs = frame.data.xsize; size_t ys = frame.data.ysize; size_t px0 = frames[i - 1].x0; size_t py0 = frames[i - 1].y0; @@ -708,102 +742,56 @@ xs * bytes_per_pixel); } x0 = px0; y0 = py0; + xsize = pxs; + ysize = pys; should_blend = false; ppf->frames.emplace_back(std::move(new_data)); } else { // If all else fails, insert a dummy blank frame with kReplace. PackedImage blank(pxs, pys, frame.data.format); memset(blank.pixels(), 0, blank.pixels_size); ppf->frames.emplace_back(std::move(blank)); auto& pframe = ppf->frames.back(); - pframe.x0 = px0; - pframe.y0 = py0; + pframe.frame_info.layer_info.crop_x0 = px0; + pframe.frame_info.layer_info.crop_y0 = py0; + pframe.frame_info.layer_info.xsize = frame.xsize; + pframe.frame_info.layer_info.ysize = frame.ysize; pframe.frame_info.duration = 0; - pframe.blend = false; - pframe.use_for_next_frame = true; - + pframe.frame_info.layer_info.have_crop = 0; + pframe.frame_info.layer_info.blend_info.blendmode = JXL_BLEND_REPLACE; + pframe.frame_info.layer_info.blend_info.source = 0; + pframe.frame_info.layer_info.save_as_reference = 1; ppf->frames.emplace_back(std::move(frame.data)); } } else { ppf->frames.emplace_back(std::move(frame.data)); } auto& pframe = ppf->frames.back(); - pframe.x0 = x0; - pframe.y0 = y0; + pframe.frame_info.layer_info.crop_x0 = x0; + pframe.frame_info.layer_info.crop_y0 = y0; + pframe.frame_info.layer_info.xsize = xsize; + pframe.frame_info.layer_info.ysize = ysize; pframe.frame_info.duration = frame.duration; - pframe.blend = should_blend; - pframe.use_for_next_frame = use_for_next_frame; + pframe.frame_info.layer_info.blend_info.blendmode = + should_blend ? JXL_BLEND_BLEND : JXL_BLEND_REPLACE; + bool is_full_size = x0 == 0 && y0 == 0 && xsize == ppf->info.xsize && + ysize == ppf->info.ysize; + pframe.frame_info.layer_info.have_crop = is_full_size ? 0 : 1; + pframe.frame_info.layer_info.blend_info.source = should_blend ? 1 : 0; + pframe.frame_info.layer_info.blend_info.alpha = 0; + pframe.frame_info.layer_info.save_as_reference = use_for_next_frame ? 1 : 0; - if (has_nontrivial_background && - frame.dispose_op == DISPOSE_OP_BACKGROUND) { - previous_frame_should_be_cleared = true; - } else { - previous_frame_should_be_cleared = false; - } + previous_frame_should_be_cleared = + has_nontrivial_background && frame.dispose_op == DISPOSE_OP_BACKGROUND; } if (ppf->frames.empty()) return JXL_FAILURE("No frames decoded"); ppf->frames.back().frame_info.is_last = true; return true; } -static void PngWrite(png_structp png_ptr, png_bytep data, png_size_t length) { - PaddedBytes* bytes = static_cast(png_get_io_ptr(png_ptr)); - bytes->append(data, data + length); -} - -// Stores XMP and EXIF/IPTC into key/value strings for PNG -class BlobsWriterPNG { - public: - static Status Encode(const Blobs& blobs, std::vector* strings) { - if (!blobs.exif.empty()) { - JXL_RETURN_IF_ERROR(EncodeBase16("exif", blobs.exif, strings)); - } - if (!blobs.iptc.empty()) { - JXL_RETURN_IF_ERROR(EncodeBase16("iptc", blobs.iptc, strings)); - } - if (!blobs.xmp.empty()) { - JXL_RETURN_IF_ERROR(EncodeBase16("xmp", blobs.xmp, strings)); - } - return true; - } - - private: - static JXL_INLINE char EncodeNibble(const uint8_t nibble) { - JXL_ASSERT(nibble < 16); - return (nibble < 10) ? '0' + nibble : 'a' + nibble - 10; - } - - static Status EncodeBase16(const std::string& type, const PaddedBytes& bytes, - std::vector* strings) { - // Encoding: base16 with newline after 72 chars. - const size_t base16_size = - 2 * bytes.size() + DivCeil(bytes.size(), size_t(36)) + 1; - std::string base16; - base16.reserve(base16_size); - for (size_t i = 0; i < bytes.size(); ++i) { - if (i % 36 == 0) base16.push_back('\n'); - base16.push_back(EncodeNibble(bytes[i] >> 4)); - base16.push_back(EncodeNibble(bytes[i] & 0x0F)); - } - base16.push_back('\n'); - JXL_ASSERT(base16.length() == base16_size); - - char key[30]; - snprintf(key, sizeof(key), "Raw profile type %s", type.c_str()); - - char header[30]; - snprintf(header, sizeof(header), "\n%s\n%8" PRIuS, type.c_str(), - bytes.size()); - - strings->push_back(std::string(key)); - strings->push_back(std::string(header) + base16); - return true; - } -}; - } // namespace extras } // namespace jxl diff --git a/third_party/jpeg-xl/lib/extras/dec/color_hints.cc b/third_party/jpeg-xl/lib/extras/dec/color_hints.cc --- a/third_party/jpeg-xl/lib/extras/dec/color_hints.cc +++ b/third_party/jpeg-xl/lib/extras/dec/color_hints.cc @@ -40,13 +40,11 @@ return JXL_FAILURE("mismatch between file and color_space hint"); } got_color_space = true; } else if (key == "icc_pathname") { - PaddedBytes icc; - JXL_RETURN_IF_ERROR(ReadFile(value, &icc)); - ppf->icc = std::vector{icc.data(), icc.data() + icc.size()}; + JXL_RETURN_IF_ERROR(ReadFile(value, &ppf->icc)); got_color_space = true; } else { JXL_WARNING("Ignoring %s hint", key.c_str()); } return true; diff --git a/third_party/jpeg-xl/lib/extras/dec/decode.h b/third_party/jpeg-xl/lib/extras/dec/decode.h --- a/third_party/jpeg-xl/lib/extras/dec/decode.h +++ b/third_party/jpeg-xl/lib/extras/dec/decode.h @@ -31,12 +31,11 @@ kPNG, kPNM, kPGX, kJPG, kGIF, - kEXR, - kPSD + kEXR }; static inline constexpr uint64_t EnumBits(Codec /*unused*/) { // Return only fully-supported codecs (kGIF is decode-only). return MakeBit(Codec::kPNM) @@ -47,28 +46,23 @@ | MakeBit(Codec::kJPG) #endif #if JPEGXL_ENABLE_EXR | MakeBit(Codec::kEXR) #endif - | MakeBit(Codec::kPSD); + ; } -// Lower case ASCII including dot, e.g. ".png". -std::string ExtensionFromCodec(Codec codec, bool is_gray, - size_t bits_per_sample); - // If and only if extension is ".pfm", *bits_per_sample is updated to 32 so // that Encode() would encode to PFM instead of PPM. Codec CodecFromExtension(std::string extension, size_t* JXL_RESTRICT bits_per_sample = nullptr); // Decodes "bytes" and sets io->metadata.m. // color_space_hint may specify the color space, otherwise, defaults to sRGB. Status DecodeBytes(Span bytes, const ColorHints& color_hints, const SizeConstraints& constraints, - extras::PackedPixelFile* ppf, ThreadPool* pool = nullptr, - Codec* orig_codec = nullptr); + extras::PackedPixelFile* ppf, Codec* orig_codec = nullptr); } // namespace extras } // namespace jxl #endif // LIB_EXTRAS_DEC_DECODE_H_ diff --git a/third_party/jpeg-xl/lib/extras/dec/decode.cc b/third_party/jpeg-xl/lib/extras/dec/decode.cc --- a/third_party/jpeg-xl/lib/extras/dec/decode.cc +++ b/third_party/jpeg-xl/lib/extras/dec/decode.cc @@ -29,35 +29,10 @@ // Any valid encoding is larger (ensures codecs can read the first few bytes) constexpr size_t kMinBytes = 9; } // namespace -std::string ExtensionFromCodec(Codec codec, const bool is_gray, - const size_t bits_per_sample) { - switch (codec) { - case Codec::kJPG: - return ".jpg"; - case Codec::kPGX: - return ".pgx"; - case Codec::kPNG: - return ".png"; - case Codec::kPNM: - if (is_gray) return ".pgm"; - return (bits_per_sample == 32) ? ".pfm" : ".ppm"; - case Codec::kGIF: - return ".gif"; - case Codec::kEXR: - return ".exr"; - case Codec::kPSD: - return ".psd"; - case Codec::kUnknown: - return std::string(); - } - JXL_UNREACHABLE; - return std::string(); -} - Codec CodecFromExtension(std::string extension, size_t* JXL_RESTRICT bits_per_sample) { std::transform( extension.begin(), extension.end(), extension.begin(), [](char c) { return std::tolower(c, std::locale::classic()); }); @@ -66,14 +41,12 @@ if (extension == ".jpg") return Codec::kJPG; if (extension == ".jpeg") return Codec::kJPG; if (extension == ".pgx") return Codec::kPGX; - if (extension == ".pbm") { - if (bits_per_sample != nullptr) *bits_per_sample = 1; - return Codec::kPNM; - } + if (extension == ".pam") return Codec::kPNM; + if (extension == ".pnm") return Codec::kPNM; if (extension == ".pgm") return Codec::kPNM; if (extension == ".ppm") return Codec::kPNM; if (extension == ".pfm") { if (bits_per_sample != nullptr) *bits_per_sample = 32; return Codec::kPNM; @@ -81,20 +54,17 @@ if (extension == ".gif") return Codec::kGIF; if (extension == ".exr") return Codec::kEXR; - if (extension == ".psd") return Codec::kPSD; - return Codec::kUnknown; } Status DecodeBytes(const Span bytes, const ColorHints& color_hints, const SizeConstraints& constraints, - extras::PackedPixelFile* ppf, ThreadPool* pool, - Codec* orig_codec) { + extras::PackedPixelFile* ppf, Codec* orig_codec) { if (bytes.size() < kMinBytes) return JXL_FAILURE("Too few bytes"); *ppf = extras::PackedPixelFile(); // Default values when not set by decoders. @@ -121,11 +91,11 @@ else if (DecodeImageJPG(bytes, color_hints, constraints, ppf)) { codec = Codec::kJPG; } #endif #if JPEGXL_ENABLE_EXR - else if (DecodeImageEXR(bytes, color_hints, constraints, pool, ppf)) { + else if (DecodeImageEXR(bytes, color_hints, constraints, ppf)) { codec = Codec::kEXR; } #endif else { return JXL_FAILURE("Codecs failed to decode"); diff --git a/third_party/jpeg-xl/lib/extras/dec/exr.h b/third_party/jpeg-xl/lib/extras/dec/exr.h --- a/third_party/jpeg-xl/lib/extras/dec/exr.h +++ b/third_party/jpeg-xl/lib/extras/dec/exr.h @@ -19,12 +19,11 @@ namespace jxl { namespace extras { // Decodes `bytes` into `ppf`. color_hints are ignored. Status DecodeImageEXR(Span bytes, const ColorHints& color_hints, - const SizeConstraints& constraints, ThreadPool* pool, - PackedPixelFile* ppf); + const SizeConstraints& constraints, PackedPixelFile* ppf); } // namespace extras } // namespace jxl #endif // LIB_EXTRAS_DEC_EXR_H_ diff --git a/third_party/jpeg-xl/lib/extras/dec/exr.cc b/third_party/jpeg-xl/lib/extras/dec/exr.cc --- a/third_party/jpeg-xl/lib/extras/dec/exr.cc +++ b/third_party/jpeg-xl/lib/extras/dec/exr.cc @@ -27,22 +27,10 @@ using ExrInt64 = decltype(std::declval().tellg()); constexpr int kExrBitsPerSample = 16; constexpr int kExrAlphaBits = 16; -size_t GetNumThreads(ThreadPool* pool) { - size_t exr_num_threads = 1; - JXL_CHECK(RunOnPool( - pool, 0, 1, - [&](size_t num_threads) { - exr_num_threads = num_threads; - return true; - }, - [&](uint32_t /* task */, size_t /*thread*/) {}, "DecodeImageEXRThreads")); - return exr_num_threads; -} - class InMemoryIStream : public OpenEXR::IStream { public: // The data pointed to by `bytes` must outlive the InMemoryIStream. explicit InMemoryIStream(const Span bytes) : IStream(/*fileName=*/""), bytes_(bytes) {} @@ -72,19 +60,12 @@ }; } // namespace Status DecodeImageEXR(Span bytes, const ColorHints& color_hints, - const SizeConstraints& constraints, ThreadPool* pool, + const SizeConstraints& constraints, PackedPixelFile* ppf) { - // Get the number of threads we should be using for OpenEXR. - // OpenEXR creates its own set of threads, independent from ours. `pool` is - // only used for converting from a buffer of OpenEXR::Rgba to Image3F. - // TODO(sboukortt): look into changing that with OpenEXR 2.3 which allows - // custom thread pools according to its changelog. - OpenEXR::setGlobalThreadCount(GetNumThreads(pool)); - InMemoryIStream is(bytes); #ifdef __EXCEPTIONS std::unique_ptr input_ptr; try { @@ -147,31 +128,28 @@ std::min(input.dataWindow().max.y, input.displayWindow().max.y)); input.setFrameBuffer( input_rows.data() - input.dataWindow().min.x - start_y * row_size, /*xStride=*/1, /*yStride=*/row_size); input.readPixels(start_y, end_y); - JXL_RETURN_IF_ERROR(RunOnPool( - pool, start_y, end_y + 1, ThreadPool::NoInit, - [&](const uint32_t exr_y, size_t /* thread */) { - const int image_y = exr_y - input.displayWindow().min.y; - const OpenEXR::Rgba* const JXL_RESTRICT input_row = - &input_rows[(exr_y - start_y) * row_size]; - uint8_t* row = static_cast(frame.color.pixels()) + - frame.color.stride * image_y; - const uint32_t pixel_size = - (3 + (has_alpha ? 1 : 0)) * kExrBitsPerSample / 8; - for (int exr_x = std::max(input.dataWindow().min.x, - input.displayWindow().min.x); - exr_x <= - std::min(input.dataWindow().max.x, input.displayWindow().max.x); - ++exr_x) { - const int image_x = exr_x - input.displayWindow().min.x; - memcpy(row + image_x * pixel_size, - input_row + (exr_x - input.dataWindow().min.x), pixel_size); - } - }, - "DecodeImageEXR")); + for (int exr_y = start_y; exr_y <= end_y; ++exr_y) { + const int image_y = exr_y - input.displayWindow().min.y; + const OpenEXR::Rgba* const JXL_RESTRICT input_row = + &input_rows[(exr_y - start_y) * row_size]; + uint8_t* row = static_cast(frame.color.pixels()) + + frame.color.stride * image_y; + const uint32_t pixel_size = + (3 + (has_alpha ? 1 : 0)) * kExrBitsPerSample / 8; + for (int exr_x = + std::max(input.dataWindow().min.x, input.displayWindow().min.x); + exr_x <= + std::min(input.dataWindow().max.x, input.displayWindow().max.x); + ++exr_x) { + const int image_x = exr_x - input.displayWindow().min.x; + memcpy(row + image_x * pixel_size, + input_row + (exr_x - input.dataWindow().min.x), pixel_size); + } + } } ppf->color_encoding.transfer_function = JXL_TRANSFER_FUNCTION_LINEAR; ppf->color_encoding.color_space = JXL_COLOR_SPACE_RGB; ppf->color_encoding.primaries = JXL_PRIMARIES_SRGB; diff --git a/third_party/jpeg-xl/lib/extras/dec/gif.cc b/third_party/jpeg-xl/lib/extras/dec/gif.cc --- a/third_party/jpeg-xl/lib/extras/dec/gif.cc +++ b/third_party/jpeg-xl/lib/extras/dec/gif.cc @@ -10,10 +10,11 @@ #include #include #include +#include "jxl/codestream_header.h" #include "lib/jxl/base/compiler_specific.h" #include "lib/jxl/sanitizers.h" namespace jxl { namespace extras { @@ -255,19 +256,30 @@ msan::UnpoisonMemory(color_map->Colors, sizeof(*color_map->Colors) * color_map->ColorCount); GraphicsControlBlock gcb; DGifSavedExtensionToGCB(gif.get(), i, &gcb); msan::UnpoisonMemory(&gcb, sizeof(gcb)); - + bool is_full_size = total_rect.x0() == 0 && total_rect.y0() == 0 && + total_rect.xsize() == canvas.color.xsize && + total_rect.ysize() == canvas.color.ysize; if (ppf->info.have_animation) { frame->frame_info.duration = gcb.DelayTime; - frame->x0 = total_rect.x0(); - frame->y0 = total_rect.y0(); + frame->frame_info.layer_info.have_crop = static_cast(!is_full_size); + frame->frame_info.layer_info.crop_x0 = total_rect.x0(); + frame->frame_info.layer_info.crop_y0 = total_rect.y0(); + frame->frame_info.layer_info.xsize = frame->color.xsize; + frame->frame_info.layer_info.ysize = frame->color.ysize; if (last_base_was_none) { replace = true; } - frame->blend = !replace; + frame->frame_info.layer_info.blend_info.blendmode = + replace ? JXL_BLEND_REPLACE : JXL_BLEND_BLEND; + // We always only reference at most the last frame + frame->frame_info.layer_info.blend_info.source = + last_base_was_none ? 0u : 1u; + frame->frame_info.layer_info.blend_info.clamp = 1; + frame->frame_info.layer_info.blend_info.alpha = 0; // TODO(veluca): this could in principle be implemented. if (last_base_was_none && (total_rect.x0() != 0 || total_rect.y0() != 0 || total_rect.xsize() != canvas.color.xsize || total_rect.ysize() != canvas.color.ysize || !replace)) { @@ -276,18 +288,18 @@ "blended frames"); } switch (gcb.DisposalMode) { case DISPOSE_DO_NOT: case DISPOSE_BACKGROUND: - frame->use_for_next_frame = true; + frame->frame_info.layer_info.save_as_reference = 1u; last_base_was_none = false; break; case DISPOSE_PREVIOUS: - frame->use_for_next_frame = false; + frame->frame_info.layer_info.save_as_reference = 0u; break; default: - frame->use_for_next_frame = false; + frame->frame_info.layer_info.save_as_reference = 0u; last_base_was_none = true; } } // Update the canvas by creating a copy first. diff --git a/third_party/jpeg-xl/lib/extras/dec/pgx.cc b/third_party/jpeg-xl/lib/extras/dec/pgx.cc --- a/third_party/jpeg-xl/lib/extras/dec/pgx.cc +++ b/third_party/jpeg-xl/lib/extras/dec/pgx.cc @@ -115,23 +115,23 @@ JXL_RETURN_IF_ERROR(SkipSingleWhitespace()); JXL_RETURN_IF_ERROR(ParseUnsigned(&header->ysize)); // 0xa, or 0xd 0xa. JXL_RETURN_IF_ERROR(SkipLineBreak()); + // TODO(jon): could do up to 24-bit by converting the values to + // JXL_TYPE_FLOAT. if (header->bits_per_sample > 16) { return JXL_FAILURE("PGX: >16 bits not yet supported"); } // TODO(lode): support signed integers. This may require changing the way // external_image works. if (header->is_signed) { return JXL_FAILURE("PGX: signed not yet supported"); } size_t numpixels = header->xsize * header->ysize; - size_t bytes_per_pixel = header->bits_per_sample <= 8 ? 1 - : header->bits_per_sample <= 16 ? 2 - : 4; + size_t bytes_per_pixel = header->bits_per_sample <= 8 ? 1 : 2; if (pos_ + numpixels * bytes_per_pixel > end_) { return JXL_FAILURE("PGX: data too small"); } *pos = pos_; @@ -172,13 +172,11 @@ ppf->info.alpha_exponent_bits = 0; ppf->info.num_color_channels = 1; // Always grayscale ppf->info.orientation = JXL_ORIENT_IDENTITY; JxlDataType data_type; - if (header.bits_per_sample > 16) { - data_type = JXL_TYPE_UINT32; - } else if (header.bits_per_sample > 8) { + if (header.bits_per_sample > 8) { data_type = JXL_TYPE_UINT16; } else { data_type = JXL_TYPE_UINT8; } diff --git a/third_party/jpeg-xl/lib/extras/dec/pnm.cc b/third_party/jpeg-xl/lib/extras/dec/pnm.cc --- a/third_party/jpeg-xl/lib/extras/dec/pnm.cc +++ b/third_party/jpeg-xl/lib/extras/dec/pnm.cc @@ -17,11 +17,10 @@ namespace { struct HeaderPNM { size_t xsize; size_t ysize; - bool is_bit; // PBM bool is_gray; // PGM bool has_alpha; // PAM size_t bits_per_sample; bool floating_point; bool big_endian; @@ -37,18 +36,13 @@ // codec.cc ensures we have at least two bytes => no range check here. if (pos_[0] != 'P') return false; const uint8_t type = pos_[1]; pos_ += 2; - header->is_bit = false; - switch (type) { case '4': - header->is_bit = true; - header->is_gray = true; - header->bits_per_sample = 1; - return ParseHeaderPNM(header, pos); + return JXL_FAILURE("pbm not supported"); case '5': header->is_gray = true; return ParseHeaderPNM(header, pos); @@ -167,27 +161,31 @@ ++pos_; } return true; } - Status MatchString(const char* keyword) { + Status MatchString(const char* keyword, bool skipws = true) { const uint8_t* ppos = pos_; while (*keyword) { if (ppos >= end_) return JXL_FAILURE("PAM: unexpected end of input"); if (*keyword != *ppos) return false; ppos++; keyword++; } pos_ = ppos; - JXL_RETURN_IF_ERROR(SkipWhitespace()); + if (skipws) { + JXL_RETURN_IF_ERROR(SkipWhitespace()); + } else { + JXL_RETURN_IF_ERROR(SkipSingleWhitespace()); + } return true; } Status ParseHeaderPAM(HeaderPNM* header, const uint8_t** pos) { size_t depth = 3; size_t max_val = 255; - while (!MatchString("ENDHDR")) { + while (!MatchString("ENDHDR", /*skipws=*/false)) { JXL_RETURN_IF_ERROR(SkipWhitespace()); if (MatchString("WIDTH")) { JXL_RETURN_IF_ERROR(ParseUnsigned(&header->xsize)); } else if (MatchString("HEIGHT")) { JXL_RETURN_IF_ERROR(ParseUnsigned(&header->ysize)); @@ -229,11 +227,15 @@ return JXL_FAILURE("PAM: bad DEPTH"); } if (max_val == 0 || max_val >= 65536) { return JXL_FAILURE("PAM: bad MAXVAL"); } - header->bits_per_sample = CeilLog2Nonzero(max_val); + // e.g When `max_val` is 1 , we want 1 bit: + header->bits_per_sample = FloorLog2Nonzero(max_val) + 1; + if ((1u << header->bits_per_sample) - 1 != max_val) + return JXL_FAILURE("PNM: unsupported MaxVal (expected 2^n - 1)"); + // PAM does not pack bits as in PBM. header->floating_point = false; header->big_endian = true; *pos = pos_; return true; @@ -244,19 +246,19 @@ JXL_RETURN_IF_ERROR(ParseUnsigned(&header->xsize)); JXL_RETURN_IF_ERROR(SkipWhitespace()); JXL_RETURN_IF_ERROR(ParseUnsigned(&header->ysize)); - if (!header->is_bit) { - JXL_RETURN_IF_ERROR(SkipWhitespace()); - size_t max_val; - JXL_RETURN_IF_ERROR(ParseUnsigned(&max_val)); - if (max_val == 0 || max_val >= 65536) { - return JXL_FAILURE("PNM: bad MaxVal"); - } - header->bits_per_sample = CeilLog2Nonzero(max_val); + JXL_RETURN_IF_ERROR(SkipWhitespace()); + size_t max_val; + JXL_RETURN_IF_ERROR(ParseUnsigned(&max_val)); + if (max_val == 0 || max_val >= 65536) { + return JXL_FAILURE("PNM: bad MaxVal"); } + header->bits_per_sample = FloorLog2Nonzero(max_val) + 1; + if ((1u << header->bits_per_sample) - 1 != max_val) + return JXL_FAILURE("PNM: unsupported MaxVal (expected 2^n - 1)"); header->floating_point = false; header->big_endian = true; JXL_RETURN_IF_ERROR(SkipSingleWhitespace()); @@ -274,11 +276,16 @@ JXL_RETURN_IF_ERROR(SkipSingleWhitespace()); // The scale has no meaning as multiplier, only its sign is used to // indicate endianness. All software expects nominal range 0..1. double scale; JXL_RETURN_IF_ERROR(ParseSigned(&scale)); - header->big_endian = scale >= 0.0; + if (scale == 0.0) { + return JXL_FAILURE("PFM: bad scale factor value."); + } else if (std::abs(scale) != 1.0) { + JXL_WARNING("PFM: Discarding non-unit scale factor"); + } + header->big_endian = scale > 0.0; header->bits_per_sample = 32; header->floating_point = true; JXL_RETURN_IF_ERROR(SkipSingleWhitespace()); @@ -339,16 +346,12 @@ JxlDataType data_type; if (header.floating_point) { // There's no float16 pnm version. data_type = JXL_TYPE_FLOAT; } else { - if (header.bits_per_sample > 16) { - data_type = JXL_TYPE_UINT32; - } else if (header.bits_per_sample > 8) { + if (header.bits_per_sample > 8) { data_type = JXL_TYPE_UINT16; - } else if (header.is_bit) { - data_type = JXL_TYPE_BOOLEAN; } else { data_type = JXL_TYPE_UINT8; } } @@ -361,10 +364,11 @@ }; ppf->frames.clear(); ppf->frames.emplace_back(header.xsize, header.ysize, format); auto* frame = &ppf->frames.back(); + frame->color.bitdepth_from_format = false; frame->color.flipped_y = header.bits_per_sample == 32; // PFMs are flipped size_t pnm_remaining_size = bytes.data() + bytes.size() - pos; if (pnm_remaining_size < frame->color.pixels_size) { return JXL_FAILURE("PNM file too small"); } diff --git a/third_party/jpeg-xl/lib/extras/enc/apng.cc b/third_party/jpeg-xl/lib/extras/enc/apng.cc --- a/third_party/jpeg-xl/lib/extras/enc/apng.cc +++ b/third_party/jpeg-xl/lib/extras/enc/apng.cc @@ -47,10 +47,11 @@ #include "lib/jxl/base/printf_macros.h" #include "lib/jxl/color_encoding_internal.h" #include "lib/jxl/dec_external_image.h" #include "lib/jxl/enc_color_management.h" #include "lib/jxl/enc_image_bundle.h" +#include "lib/jxl/exif.h" #include "lib/jxl/frame_header.h" #include "lib/jxl/headers.h" #include "lib/jxl/image.h" #include "lib/jxl/image_bundle.h" #include "png.h" /* original (unpatched) libpng is ok */ @@ -68,11 +69,16 @@ // Stores XMP and EXIF/IPTC into key/value strings for PNG class BlobsWriterPNG { public: static Status Encode(const Blobs& blobs, std::vector* strings) { if (!blobs.exif.empty()) { - JXL_RETURN_IF_ERROR(EncodeBase16("exif", blobs.exif, strings)); + // PNG viewers typically ignore Exif orientation but not all of them do + // (and e.g. cjxl doesn't), so we overwrite the Exif orientation to the + // identity to avoid repeated orientation. + std::vector exif = blobs.exif; + ResetExifOrientation(exif); + JXL_RETURN_IF_ERROR(EncodeBase16("exif", exif, strings)); } if (!blobs.iptc.empty()) { JXL_RETURN_IF_ERROR(EncodeBase16("iptc", blobs.iptc, strings)); } if (!blobs.xmp.empty()) { @@ -85,11 +91,12 @@ static JXL_INLINE char EncodeNibble(const uint8_t nibble) { JXL_ASSERT(nibble < 16); return (nibble < 10) ? '0' + nibble : 'a' + nibble - 10; } - static Status EncodeBase16(const std::string& type, const PaddedBytes& bytes, + static Status EncodeBase16(const std::string& type, + const std::vector& bytes, std::vector* strings) { // Encoding: base16 with newline after 72 chars. const size_t base16_size = 2 * bytes.size() + DivCeil(bytes.size(), size_t(36)) + 1; std::string base16; @@ -156,16 +163,16 @@ JXL_RETURN_IF_ERROR(TransformIfNeeded(ib, c_desired, GetJxlCms(), pool, &store, &transformed)); size_t stride = ib.oriented_xsize() * DivCeil(c_desired.Channels() * bits_per_sample + alpha_bits, kBitsPerByte); - PaddedBytes raw_bytes(stride * ib.oriented_ysize()); + std::vector raw_bytes(stride * ib.oriented_ysize()); JXL_RETURN_IF_ERROR(ConvertToExternal( *transformed, bits_per_sample, /*float_out=*/false, c_desired.Channels() + (ib.HasAlpha() ? 1 : 0), JXL_BIG_ENDIAN, stride, - pool, raw_bytes.data(), raw_bytes.size(), /*out_callback=*/nullptr, - /*out_opaque=*/nullptr, metadata.GetOrientation())); + pool, raw_bytes.data(), raw_bytes.size(), + /*out_callback=*/{}, metadata.GetOrientation())); int width = ib.oriented_xsize(); int height = ib.oriented_ysize(); png_byte color_type = @@ -186,14 +193,14 @@ png_set_iCCP(png_ptr, info_ptr, "1", 0, c_desired.ICC().data(), c_desired.ICC().size()); std::vector textstrings; JXL_RETURN_IF_ERROR(BlobsWriterPNG::Encode(io->blobs, &textstrings)); - for (size_t i = 0; i + 1 < textstrings.size(); i += 2) { + for (size_t kk = 0; kk + 1 < textstrings.size(); kk += 2) { png_text text; - text.key = const_cast(textstrings[i].c_str()); - text.text = const_cast(textstrings[i + 1].c_str()); + text.key = const_cast(textstrings[kk].c_str()); + text.text = const_cast(textstrings[kk + 1].c_str()); text.compression = PNG_TEXT_COMPRESSION_zTXt; png_set_text(png_ptr, info_ptr, &text, 1); } png_write_info(png_ptr, info_ptr); @@ -239,20 +246,21 @@ png_write_flush(png_ptr); const size_t pos = bytes->size(); png_write_image(png_ptr, &rows[0]); png_write_flush(png_ptr); if (count > 0) { - PaddedBytes fdata(4); + std::vector fdata(4); png_save_uint_32(fdata.data(), anim_chunks++); size_t p = pos; while (p + 8 < bytes->size()) { size_t len = png_get_uint_32(bytes->data() + p); JXL_ASSERT(bytes->operator[](p + 4) == 'I'); JXL_ASSERT(bytes->operator[](p + 5) == 'D'); JXL_ASSERT(bytes->operator[](p + 6) == 'A'); JXL_ASSERT(bytes->operator[](p + 7) == 'T'); - fdata.append(bytes->data() + p + 8, bytes->data() + p + 8 + len); + fdata.insert(fdata.end(), bytes->data() + p + 8, + bytes->data() + p + 8 + len); p += len + 12; } bytes->resize(pos); png_byte fdat[5] = "fdAT"; diff --git a/third_party/jpeg-xl/lib/extras/enc/jpg.cc b/third_party/jpeg-xl/lib/extras/enc/jpg.cc --- a/third_party/jpeg-xl/lib/extras/enc/jpg.cc +++ b/third_party/jpeg-xl/lib/extras/enc/jpg.cc @@ -17,12 +17,14 @@ #include "lib/jxl/base/compiler_specific.h" #include "lib/jxl/base/status.h" #include "lib/jxl/color_encoding_internal.h" #include "lib/jxl/common.h" +#include "lib/jxl/dec_external_image.h" #include "lib/jxl/enc_color_management.h" #include "lib/jxl/enc_image_bundle.h" +#include "lib/jxl/exif.h" #include "lib/jxl/image.h" #include "lib/jxl/image_bundle.h" #include "lib/jxl/sanitizers.h" #if JPEGXL_ENABLE_SJPEG #include "sjpeg.h" @@ -31,27 +33,19 @@ namespace jxl { namespace extras { namespace { -constexpr float kJPEGSampleMultiplier = MAXJSAMPLE; constexpr unsigned char kICCSignature[12] = { 0x49, 0x43, 0x43, 0x5F, 0x50, 0x52, 0x4F, 0x46, 0x49, 0x4C, 0x45, 0x00}; constexpr int kICCMarker = JPEG_APP0 + 2; constexpr size_t kMaxBytesInMarker = 65533; constexpr unsigned char kExifSignature[6] = {0x45, 0x78, 0x69, 0x66, 0x00, 0x00}; constexpr int kExifMarker = JPEG_APP0 + 1; -constexpr float kJPEGSampleMin = 0; -constexpr float kJPEGSampleMax = MAXJSAMPLE; - -// TODO (jon): take orientation into account when writing jpeg output -// TODO (jon): write Exif blob also in sjpeg encoding -// TODO (jon): overwrite orientation in Exif blob to avoid double orientation - void WriteICCProfile(jpeg_compress_struct* const cinfo, const PaddedBytes& icc) { constexpr size_t kMaxIccBytesInMarker = kMaxBytesInMarker - sizeof kICCSignature - 2; const int num_markers = @@ -71,19 +65,19 @@ jpeg_write_m_byte(cinfo, icc[begin]); ++begin; } } } -void WriteExif(jpeg_compress_struct* const cinfo, const PaddedBytes& exif) { - if (exif.size() < 4) return; +void WriteExif(jpeg_compress_struct* const cinfo, + const std::vector& exif) { jpeg_write_m_header( cinfo, kExifMarker, - static_cast(exif.size() - 4 + sizeof kExifSignature)); + static_cast(exif.size() + sizeof kExifSignature)); for (const unsigned char c : kExifSignature) { jpeg_write_m_byte(cinfo, c); } - for (size_t i = 4; i < exif.size(); ++i) { + for (size_t i = 0; i < exif.size(); ++i) { jpeg_write_m_byte(cinfo, exif[i]); } } Status SetChromaSubsampling(const YCbCrChromaSubsampling& chroma_subsampling, @@ -113,12 +107,12 @@ cinfo.err = jpeg_std_error(&jerr); jpeg_create_compress(&cinfo); unsigned char* buffer = nullptr; unsigned long size = 0; jpeg_mem_dest(&cinfo, &buffer, &size); - cinfo.image_width = ib->xsize(); - cinfo.image_height = ib->ysize(); + cinfo.image_width = ib->oriented_xsize(); + cinfo.image_height = ib->oriented_ysize(); if (ib->IsGray()) { cinfo.input_components = 1; cinfo.in_color_space = JCS_GRAYSCALE; } else { cinfo.input_components = 3; @@ -132,31 +126,30 @@ jpeg_set_quality(&cinfo, quality, TRUE); jpeg_start_compress(&cinfo, TRUE); if (!ib->IsSRGB()) { WriteICCProfile(&cinfo, ib->c_current().ICC()); } - WriteExif(&cinfo, io->blobs.exif); + if (!io->blobs.exif.empty()) { + std::vector exif = io->blobs.exif; + ResetExifOrientation(exif); + WriteExif(&cinfo, exif); + } if (cinfo.input_components > 3 || cinfo.input_components < 0) return JXL_FAILURE("invalid numbers of components"); - std::unique_ptr row( - new JSAMPLE[cinfo.input_components * cinfo.image_width]); - for (size_t y = 0; y < ib->ysize(); ++y) { - const float* const JXL_RESTRICT input_row[3] = { - ib->color().ConstPlaneRow(0, y), ib->color().ConstPlaneRow(1, y), - ib->color().ConstPlaneRow(2, y)}; - for (size_t x = 0; x < ib->xsize(); ++x) { - for (size_t c = 0; c < static_cast(cinfo.input_components); ++c) { - JXL_RETURN_IF_ERROR(c < 3); - row[cinfo.input_components * x + c] = static_cast( - std::max(std::min(kJPEGSampleMultiplier * input_row[c][x] + .5f, - kJPEGSampleMax), - kJPEGSampleMin)); - } - } - JSAMPROW rows[] = {row.get()}; - jpeg_write_scanlines(&cinfo, rows, 1); + size_t stride = + ib->oriented_xsize() * cinfo.input_components * sizeof(JSAMPLE); + PaddedBytes raw_bytes(stride * ib->oriented_ysize()); + JXL_RETURN_IF_ERROR(ConvertToExternal( + *ib, BITS_IN_JSAMPLE, /*float_out=*/false, cinfo.input_components, + JXL_BIG_ENDIAN, stride, nullptr, raw_bytes.data(), raw_bytes.size(), + /*out_callback=*/{}, ib->metadata()->GetOrientation())); + + for (size_t y = 0; y < ib->oriented_ysize(); ++y) { + JSAMPROW row[] = {raw_bytes.data() + y * stride}; + + jpeg_write_scanlines(&cinfo, row, 1); } jpeg_finish_compress(&cinfo); jpeg_destroy_compress(&cinfo); bytes->resize(size); // Compressed image data is initialized by libjpeg, which we are not @@ -165,46 +158,45 @@ std::copy_n(buffer, size, bytes->data()); std::free(buffer); return true; } -Status EncodeWithSJpeg(const ImageBundle* ib, size_t quality, +Status EncodeWithSJpeg(const ImageBundle* ib, const CodecInOut* io, + size_t quality, const YCbCrChromaSubsampling& chroma_subsampling, PaddedBytes* bytes) { #if !JPEGXL_ENABLE_SJPEG return JXL_FAILURE("JPEG XL was built without sjpeg support"); #else sjpeg::EncoderParam param(quality); if (!ib->IsSRGB()) { param.iccp.assign(ib->metadata()->color_encoding.ICC().begin(), ib->metadata()->color_encoding.ICC().end()); } + std::vector exif = io->blobs.exif; + if (!exif.empty()) { + ResetExifOrientation(exif); + param.exif.assign(exif.begin(), exif.end()); + } if (chroma_subsampling.Is444()) { param.yuv_mode = SJPEG_YUV_444; } else if (chroma_subsampling.Is420()) { param.yuv_mode = SJPEG_YUV_SHARP; } else { return JXL_FAILURE("sjpeg does not support this chroma subsampling mode"); } - std::vector rgb; - rgb.reserve(ib->xsize() * ib->ysize() * 3); - for (size_t y = 0; y < ib->ysize(); ++y) { - const float* const rows[] = { - ib->color().ConstPlaneRow(0, y), - ib->color().ConstPlaneRow(1, y), - ib->color().ConstPlaneRow(2, y), - }; - for (size_t x = 0; x < ib->xsize(); ++x) { - for (const float* const row : rows) { - rgb.push_back(static_cast( - std::max(0.f, std::min(255.f, roundf(255.f * row[x]))))); - } - } - } + size_t stride = ib->oriented_xsize() * 3; + PaddedBytes rgb(ib->xsize() * ib->ysize() * 3); + JXL_RETURN_IF_ERROR( + ConvertToExternal(*ib, 8, /*float_out=*/false, 3, JXL_BIG_ENDIAN, stride, + nullptr, rgb.data(), rgb.size(), + /*out_callback=*/{}, ib->metadata()->GetOrientation())); + std::string output; - JXL_RETURN_IF_ERROR(sjpeg::Encode(rgb.data(), ib->xsize(), ib->ysize(), - ib->xsize() * 3, param, &output)); + JXL_RETURN_IF_ERROR(sjpeg::Encode(rgb.data(), ib->oriented_xsize(), + ib->oriented_ysize(), stride, param, + &output)); bytes->assign( reinterpret_cast(output.data()), reinterpret_cast(output.data() + output.size())); return true; #endif @@ -232,11 +224,11 @@ JXL_RETURN_IF_ERROR( EncodeWithLibJpeg(ib, io, quality, chroma_subsampling, bytes)); break; case JpegEncoder::kSJpeg: JXL_RETURN_IF_ERROR( - EncodeWithSJpeg(ib, quality, chroma_subsampling, bytes)); + EncodeWithSJpeg(ib, io, quality, chroma_subsampling, bytes)); break; default: return JXL_FAILURE("tried to use an unknown JPEG encoder"); } diff --git a/third_party/jpeg-xl/lib/extras/enc/pgx.cc b/third_party/jpeg-xl/lib/extras/enc/pgx.cc --- a/third_party/jpeg-xl/lib/extras/enc/pgx.cc +++ b/third_party/jpeg-xl/lib/extras/enc/pgx.cc @@ -73,12 +73,12 @@ size_t stride = ib.xsize() * (bits_per_sample / kBitsPerByte); JXL_RETURN_IF_ERROR( ConvertToExternal(*transformed, bits_per_sample, /*float_out=*/false, /*num_channels=*/1, JXL_BIG_ENDIAN, stride, pool, - pixels.data(), pixels.size(), /*out_callback=*/nullptr, - /*out_opaque=*/nullptr, metadata.GetOrientation())); + pixels.data(), pixels.size(), + /*out_callback=*/{}, metadata.GetOrientation())); char header[kMaxHeaderSize]; int header_size = 0; JXL_RETURN_IF_ERROR(EncodeHeader(ib, bits_per_sample, header, &header_size)); diff --git a/third_party/jpeg-xl/lib/extras/enc/pnm.cc b/third_party/jpeg-xl/lib/extras/enc/pnm.cc --- a/third_party/jpeg-xl/lib/extras/enc/pnm.cc +++ b/third_party/jpeg-xl/lib/extras/enc/pnm.cc @@ -33,34 +33,34 @@ constexpr size_t kMaxHeaderSize = 200; Status EncodeHeader(const PackedPixelFile& ppf, const size_t bits_per_sample, const bool little_endian, char* header, int* JXL_RESTRICT chars_written) { - if (ppf.info.alpha_bits > 0) return JXL_FAILURE("PNM: can't store alpha"); bool is_gray = ppf.info.num_color_channels <= 2; size_t oriented_xsize = ppf.info.orientation <= 4 ? ppf.info.xsize : ppf.info.ysize; size_t oriented_ysize = ppf.info.orientation <= 4 ? ppf.info.ysize : ppf.info.xsize; - - if (bits_per_sample == 32) { // PFM + if (ppf.info.alpha_bits > 0) { // PAM + if (bits_per_sample > 16) return JXL_FAILURE("PNM cannot have > 16 bits"); + const uint32_t max_val = (1U << bits_per_sample) - 1; + *chars_written = + snprintf(header, kMaxHeaderSize, + "P7\nWIDTH %" PRIuS "\nHEIGHT %" PRIuS + "\nDEPTH %u\nMAXVAL %u\nTUPLTYPE %s\nENDHDR\n", + oriented_xsize, oriented_ysize, is_gray ? 2 : 4, max_val, + is_gray ? "GRAYSCALE_ALPHA" : "RGB_ALPHA"); + JXL_RETURN_IF_ERROR(static_cast(*chars_written) < + kMaxHeaderSize); + } else if (bits_per_sample == 32) { // PFM const char type = is_gray ? 'f' : 'F'; const double scale = little_endian ? -1.0 : 1.0; *chars_written = snprintf(header, kMaxHeaderSize, "P%c\n%" PRIuS " %" PRIuS "\n%.1f\n", type, oriented_xsize, oriented_ysize, scale); JXL_RETURN_IF_ERROR(static_cast(*chars_written) < kMaxHeaderSize); - } else if (bits_per_sample == 1) { // PBM - if (is_gray) { - return JXL_FAILURE("Cannot encode color as PBM"); - } - *chars_written = - snprintf(header, kMaxHeaderSize, "P4\n%" PRIuS " %" PRIuS "\n", - oriented_xsize, oriented_ysize); - JXL_RETURN_IF_ERROR(static_cast(*chars_written) < - kMaxHeaderSize); } else { // PGM/PPM const uint32_t max_val = (1U << bits_per_sample) - 1; if (max_val >= 65536) return JXL_FAILURE("PNM cannot have > 16 bits"); const char type = is_gray ? '5' : '6'; *chars_written = @@ -98,11 +98,11 @@ Status EncodeImagePNM(const PackedPixelFile& ppf, size_t bits_per_sample, ThreadPool* pool, size_t frame_index, std::vector* bytes) { const bool floating_point = bits_per_sample > 16; - // Choose native for PFM; PGM/PPM require big-endian (N/A for PBM) + // Choose native for PFM; PGM/PPM require big-endian const JxlEndianness endianness = floating_point ? JXL_NATIVE_ENDIAN : JXL_BIG_ENDIAN; if (!ppf.metadata.exif.empty() || !ppf.metadata.iptc.empty() || !ppf.metadata.jumbf.empty() || !ppf.metadata.xmp.empty()) { JXL_WARNING("PNM encoder ignoring metadata - use a different codec"); diff --git a/third_party/jpeg-xl/lib/extras/packed_image.h b/third_party/jpeg-xl/lib/extras/packed_image.h --- a/third_party/jpeg-xl/lib/extras/packed_image.h +++ b/third_party/jpeg-xl/lib/extras/packed_image.h @@ -62,34 +62,36 @@ size_t ysize; // Whether the y coordinate is flipped (y=0 is the last row). bool flipped_y = false; + // Whether the range is determined by format or by JxlBasicInfo + // e.g. if format is UINT16 and JxlBasicInfo bits_per_sample is 10, + // then if bitdepth_from_format == true, the range is 0..65535 + // while if bitdepth_from_format == false, the range is 0..1023. + bool bitdepth_from_format = true; + // The number of bytes per row. size_t stride; // Pixel storage format and buffer size of the pixels_ pointer. JxlPixelFormat format; size_t pixels_size; static size_t BitsPerChannel(JxlDataType data_type) { switch (data_type) { - case JXL_TYPE_BOOLEAN: - return 1; case JXL_TYPE_UINT8: return 8; case JXL_TYPE_UINT16: return 16; - case JXL_TYPE_UINT32: - return 32; case JXL_TYPE_FLOAT: return 32; case JXL_TYPE_FLOAT16: return 16; - // No default, give compiler error if new type not handled. + default: + JXL_ABORT("Unhandled JxlDataType"); } - return 0; // Indicate invalid data type. } private: static size_t CalcStride(const JxlPixelFormat& format, size_t xsize) { size_t stride = xsize * (BitsPerChannel(format.data_type) * @@ -114,20 +116,10 @@ // The Frame metadata. JxlFrameHeader frame_info = {}; std::string name; - // Offset of the frame in the image. - // TODO(deymo): Add support in the API for this. - size_t x0 = 0; - size_t y0 = 0; - - // Whether this frame should be blended with the previous one. - // TODO(deymo): Maybe add support for this in the API. - bool blend = false; - bool use_for_next_frame = false; - // The pixel data for the color (or grayscale) channels. PackedImage color; // Extra channel image data. std::vector extra_channels; }; diff --git a/third_party/jpeg-xl/lib/extras/packed_image_convert.cc b/third_party/jpeg-xl/lib/extras/packed_image_convert.cc --- a/third_party/jpeg-xl/lib/extras/packed_image_convert.cc +++ b/third_party/jpeg-xl/lib/extras/packed_image_convert.cc @@ -63,25 +63,29 @@ PaddedBytes icc; icc.append(ppf.icc); if (!io->metadata.m.color_encoding.SetICC(std::move(icc))) { fprintf(stderr, "Warning: error setting ICC profile, assuming SRGB"); io->metadata.m.color_encoding = ColorEncoding::SRGB(is_gray); + } else { + if (io->metadata.m.color_encoding.IsGray() != is_gray) { + // E.g. JPG image has 3 channels, but gray ICC. + return JXL_FAILURE("Embedded ICC does not match image color type"); + } } } else { JXL_RETURN_IF_ERROR(ConvertExternalToInternalColorEncoding( ppf.color_encoding, &io->metadata.m.color_encoding)); + if (io->metadata.m.color_encoding.ICC().empty()) { + return JXL_FAILURE("Failed to serialize ICC"); + } } // Convert the extra blobs - io->blobs.exif.clear(); - io->blobs.exif.append(ppf.metadata.exif); - io->blobs.iptc.clear(); - io->blobs.iptc.append(ppf.metadata.iptc); - io->blobs.jumbf.clear(); - io->blobs.jumbf.append(ppf.metadata.jumbf); - io->blobs.xmp.clear(); - io->blobs.xmp.append(ppf.metadata.xmp); + io->blobs.exif = ppf.metadata.exif; + io->blobs.iptc = ppf.metadata.iptc; + io->blobs.jumbf = ppf.metadata.jumbf; + io->blobs.xmp = ppf.metadata.xmp; // Append all other extra channels. for (const PackedPixelFile::PackedExtraChannel& info : ppf.extra_channels_info) { ExtraChannelInfo out; @@ -105,34 +109,37 @@ io->dec_pixels = 0; io->frames.clear(); for (const auto& frame : ppf.frames) { JXL_ASSERT(frame.color.pixels() != nullptr); size_t frame_bits_per_sample = - frame.color.BitsPerChannel(frame.color.format.data_type); + (frame.color.bitdepth_from_format + ? frame.color.BitsPerChannel(frame.color.format.data_type) + : ppf.info.bits_per_sample); JXL_ASSERT(frame_bits_per_sample != 0); // It is ok for the frame.color.format.num_channels to not match the // number of channels on the image. JXL_ASSERT(1 <= frame.color.format.num_channels && frame.color.format.num_channels <= 4); const Span span( static_cast(frame.color.pixels()), frame.color.pixels_size); - Rect frame_rect = - Rect(frame.x0, frame.y0, frame.color.xsize, frame.color.ysize); + Rect frame_rect = Rect(frame.frame_info.layer_info.crop_x0, + frame.frame_info.layer_info.crop_y0, + frame.frame_info.layer_info.xsize, + frame.frame_info.layer_info.ysize); JXL_ASSERT(frame_rect.IsInside(Rect(0, 0, ppf.info.xsize, ppf.info.ysize))); - ImageBundle bundle(&io->metadata.m); if (ppf.info.have_animation) { bundle.duration = frame.frame_info.duration; - bundle.blend = frame.blend; - bundle.use_for_next_frame = frame.use_for_next_frame; + bundle.blend = frame.frame_info.layer_info.blend_info.blendmode > 0; + bundle.use_for_next_frame = + frame.frame_info.layer_info.save_as_reference > 0; + bundle.origin.x0 = frame.frame_info.layer_info.crop_x0; + bundle.origin.y0 = frame.frame_info.layer_info.crop_y0; } bundle.name = frame.name; // frame.frame_info.name_length is ignored here. - bundle.origin.x0 = frame.x0; - bundle.origin.y0 = frame.y0; - JXL_ASSERT(io->metadata.m.color_encoding.IsGray() == (frame.color.format.num_channels <= 2)); const bool float_in = frame.color.format.data_type == JXL_TYPE_FLOAT16 || frame.color.format.data_type == JXL_TYPE_FLOAT; @@ -142,15 +149,17 @@ /*alpha_is_premultiplied=*/ppf.info.alpha_premultiplied, frame_bits_per_sample, frame.color.format.endianness, /*flipped_y=*/frame.color.flipped_y, pool, &bundle, /*float_in=*/float_in, /*align=*/0)); - for (const auto& ppf_ec : frame.extra_channels) { - bundle.extra_channels().emplace_back(ppf_ec.xsize, ppf_ec.ysize); + bundle.extra_channels().resize(io->metadata.m.extra_channel_info.size()); + for (size_t i = 0; i < frame.extra_channels.size(); i++) { + const auto& ppf_ec = frame.extra_channels[i]; + bundle.extra_channels()[i] = ImageF(ppf_ec.xsize, ppf_ec.ysize); JXL_CHECK(BufferToImageF(ppf_ec.format, ppf_ec.xsize, ppf_ec.ysize, ppf_ec.pixels(), ppf_ec.pixels_size, pool, - &bundle.extra_channels().back())); + &bundle.extra_channels()[i])); } io->frames.push_back(std::move(bundle)); io->dec_pixels += frame.color.xsize * frame.color.ysize; } @@ -216,14 +225,14 @@ if (ppf->icc.empty()) { ConvertInternalToExternalColorEncoding(c_desired, &ppf->color_encoding); } // Convert the extra blobs - ppf->metadata.exif.assign(io.blobs.exif.begin(), io.blobs.exif.end()); - ppf->metadata.iptc.assign(io.blobs.iptc.begin(), io.blobs.iptc.end()); - ppf->metadata.jumbf.assign(io.blobs.jumbf.begin(), io.blobs.jumbf.end()); - ppf->metadata.xmp.assign(io.blobs.xmp.begin(), io.blobs.xmp.end()); + ppf->metadata.exif = io.blobs.exif; + ppf->metadata.iptc = io.blobs.iptc; + ppf->metadata.jumbf = io.blobs.jumbf; + ppf->metadata.xmp = io.blobs.xmp; const bool float_out = pixel_format.data_type == JXL_TYPE_FLOAT || pixel_format.data_type == JXL_TYPE_FLOAT16; // Convert the pixels ppf->frames.clear(); for (const auto& frame : io.frames) { @@ -238,12 +247,15 @@ /*endianness=*/pixel_format.endianness, /*align=*/pixel_format.align}; PackedFrame packed_frame(frame.oriented_xsize(), frame.oriented_ysize(), format); + packed_frame.color.bitdepth_from_format = float_out; const size_t bits_per_sample = - packed_frame.color.BitsPerChannel(pixel_format.data_type); + packed_frame.color.bitdepth_from_format + ? packed_frame.color.BitsPerChannel(pixel_format.data_type) + : ppf->info.bits_per_sample; packed_frame.name = frame.name; packed_frame.frame_info.name_length = frame.name.size(); // Color transform ImageBundle ib = frame.Copy(); const ImageBundle* to_color_transform = &ib; @@ -262,12 +274,11 @@ JXL_RETURN_IF_ERROR(ConvertToExternal( *transformed, bits_per_sample, float_out, format.num_channels, format.endianness, /* stride_out=*/packed_frame.color.stride, pool, packed_frame.color.pixels(), packed_frame.color.pixels_size, - /*out_callback=*/nullptr, /*out_opaque=*/nullptr, - frame.metadata()->GetOrientation())); + /*out_callback=*/{}, frame.metadata()->GetOrientation())); // TODO(firsching): Convert the extra channels, beside one potential alpha // channel. FIXME! JXL_CHECK(frame.extra_channels().size() <= has_alpha); ppf->frames.push_back(std::move(packed_frame)); diff --git a/third_party/jpeg-xl/lib/extras/tone_mapping_gbench.cc b/third_party/jpeg-xl/lib/extras/tone_mapping_gbench.cc --- a/third_party/jpeg-xl/lib/extras/tone_mapping_gbench.cc +++ b/third_party/jpeg-xl/lib/extras/tone_mapping_gbench.cc @@ -12,11 +12,11 @@ namespace jxl { static void BM_ToneMapping(benchmark::State& state) { CodecInOut image; const PaddedBytes image_bytes = - ReadTestData("imagecompression.info/flower_foveon.png"); + ReadTestData("third_party/imagecompression.info/flower_foveon.png"); JXL_CHECK(SetFromBytes(Span(image_bytes), &image)); // Convert to linear Rec. 2020 so that `ToneMapTo` doesn't have to and we // mainly measure the tone mapping itself. ColorEncoding linear_rec2020; diff --git a/third_party/jpeg-xl/lib/include/jxl/decode.h b/third_party/jpeg-xl/lib/include/jxl/decode.h --- a/third_party/jpeg-xl/lib/include/jxl/decode.h +++ b/third_party/jpeg-xl/lib/include/jxl/decode.h @@ -916,10 +916,60 @@ */ typedef void (*JxlImageOutCallback)(void* opaque, size_t x, size_t y, size_t num_pixels, const void* pixels); /** + * Initialization callback for @c JxlDecoderSetMultithreadedImageOutCallback. + * + * @see JxlDecoderSetMultithreadedImageOutCallback + * + * @param init_opaque optional user data, as given to + * @c JxlDecoderSetMultithreadedImageOutCallback. + * @param num_threads maximum number of threads that will call the @c run + * callback concurrently. + * @param num_pixels_per_thread maximum number of pixels that will be passed in + * one call to @c run. + * @return a pointer to data that will be passed to the @c run callback, or + * @c NULL if initialization failed. + */ +typedef void* (*JxlImageOutInitCallback)(void* init_opaque, size_t num_threads, + size_t num_pixels_per_thread); + +/** + * Worker callback for @c JxlDecoderSetMultithreadedImageOutCallback. + * + * @see JxlDecoderSetMultithreadedImageOutCallback + * + * @param run_opaque user data returned by the @c init callback. + * @param thread_id number in `[0, num_threads)` identifying the thread of the + * current invocation of the callback. + * @param x horizontal position of the first (leftmost) pixel of the pixel data. + * @param y vertical position of the pixel data. + * @param num_pixels number of pixels in the pixel data. May be less than the + * full @c xsize of the image, and will be at most equal to the @c + * num_pixels_per_thread that was passed to @c init. + * @param pixels pixel data as a horizontal stripe, in the format passed to @c + * JxlDecoderSetMultithreadedImageOutCallback. The data pointed to remains owned + * by the caller and is only guaranteed to outlive the current callback + * invocation. + */ +typedef void (*JxlImageOutRunCallback)(void* run_opaque, size_t thread_id, + size_t x, size_t y, size_t num_pixels, + const void* pixels); + +/** + * Destruction callback for @c JxlDecoderSetMultithreadedImageOutCallback, + * called after all invocations of the @c run callback to perform any + * appropriate clean-up of the @c run_opaque data returned by @c init. + * + * @see JxlDecoderSetMultithreadedImageOutCallback + * + * @param run_opaque user data returned by the @c init callback. + */ +typedef void (*JxlImageOutDestroyCallback)(void* run_opaque); + +/** * Sets pixel output callback. This is an alternative to * JxlDecoderSetImageOutBuffer. This can be set when the JXL_DEC_FRAME event * occurs, must be set when the JXL_DEC_NEED_IMAGE_OUT_BUFFER event occurs, and * applies only for the current frame. Only one of JxlDecoderSetImageOutBuffer * or JxlDecoderSetImageOutCallback may be used for the same frame, not both at @@ -948,12 +998,12 @@ * this pixel. A pixel may be visited zero times if it cannot be decoded yet * or if it was already decoded to full precision (this behavior is not * guaranteed). * * @param dec decoder object - * @param format format of the pixels. Object owned by user and its contents - * are copied internally. + * @param format format of the pixels. Object owned by user; its contents are + * copied internally. * @param callback the callback function receiving partial scanlines of pixel * data. * @param opaque optional user data, which will be passed on to the callback, * may be NULL. * @return JXL_DEC_SUCCESS on success, JXL_DEC_ERROR on error, such as @@ -961,10 +1011,34 @@ */ JXL_EXPORT JxlDecoderStatus JxlDecoderSetImageOutCallback(JxlDecoder* dec, const JxlPixelFormat* format, JxlImageOutCallback callback, void* opaque); +/** Similar to @c JxlDecoderSetImageOutCallback except that the callback is + * allowed an initialization phase during which it is informed of how many + * threads will call it concurrently, and those calls are further informed of + * which thread they are occurring in. + * + * @param dec decoder object + * @param format format of the pixels. Object owned by user; its contents are + * copied internally. + * @param init_callback initialization callback. + * @param run_callback the callback function receiving partial scanlines of + * pixel data. + * @param destroy_callback clean-up callback invoked after all calls to @c + * run_callback. May be NULL if no clean-up is necessary. + * @param init_opaque optional user data passed to @c init_callback, may be NULL + * (unlike the return value from @c init_callback which may only be NULL if + * initialization failed). + * @return JXL_DEC_SUCCESS on success, JXL_DEC_ERROR on error, such as @c + * JxlDecoderSetImageOutBuffer having already been called. + */ +JXL_EXPORT JxlDecoderStatus JxlDecoderSetMultithreadedImageOutCallback( + JxlDecoder* dec, const JxlPixelFormat* format, + JxlImageOutInitCallback init_callback, JxlImageOutRunCallback run_callback, + JxlImageOutDestroyCallback destroy_callback, void* init_opaque); + /** * Returns the minimum size in bytes of an extra channel pixel buffer for the * given format. This is the buffer for JxlDecoderSetExtraChannelBuffer. * Requires the basic image information is available in the decoder. * @@ -1123,12 +1197,12 @@ * box, this will return "brob" if the decompressed argument is JXL_FALSE, or * the underlying box type if the decompressed argument is JXL_TRUE. * * @param dec decoder object * @param type buffer to copy the type into - * @param decompressed which box type to get: JXL_TRUE to get the raw box type, - * which can be "brob", JXL_FALSE, get the underlying box type. + * @param decompressed which box type to get: JXL_FALSE to get the raw box type, + * which can be "brob", JXL_TRUE, get the underlying box type. * @return JXL_DEC_SUCCESS if the value is available, JXL_DEC_ERROR if not, for * example the JXL file does not use the container format. */ JXL_EXPORT JxlDecoderStatus JxlDecoderGetBoxType(JxlDecoder* dec, JxlBoxType type, diff --git a/third_party/jpeg-xl/lib/include/jxl/encode.h b/third_party/jpeg-xl/lib/include/jxl/encode.h --- a/third_party/jpeg-xl/lib/include/jxl/encode.h +++ b/third_party/jpeg-xl/lib/include/jxl/encode.h @@ -281,10 +281,17 @@ * If non-keyframes, i.e., frames with cropping, blending or patches are * attempted to be indexed, JXL_ENC_ERROR will occur. */ JXL_ENC_FRAME_INDEX_BOX = 31, + /** Sets brotli encode effort for use in JPEG recompression and compressed + * metadata boxes (brob). Can be -1 (default) or 0 (fastest) to 11 (slowest). + * Default is based on the general encode effort in case of JPEG + * recompression, and 4 for brob boxes. + */ + JXL_ENC_FRAME_SETTING_BROTLI_EFFORT = 32, + /** Enum value not to be used as an option. This value is added to force the * C compiler to have the enum to take a known size. */ JXL_ENC_FRAME_SETTING_FILL_ENUM = 65535, @@ -507,10 +514,12 @@ * - trichromatic, e.g. RGB * - trichromatic + alpha * * Extra channels not handled here need to be set by @ref * JxlEncoderSetExtraChannelBuffer. + * If the image has alpha, and alpha is not passed here, it will implicitly be + * set to all-opaque (an alpha value of 1.0 everywhere). * * The color profile of the pixels depends on the value of uses_original_profile * in the JxlBasicInfo. If true, the pixels are assumed to be encoded in the * original profile that is set with JxlEncoderSetColorEncoding or * JxlEncoderSetICCProfile. If false, the pixels are assumed to be nonlinear @@ -1047,11 +1056,11 @@ JXL_EXPORT JxlEncoderStatus JxlEncoderSetFrameDistance( JxlEncoderFrameSettings* frame_settings, float distance); /** DEPRECATED: use JxlEncoderSetFrameDistance instead. */ -JXL_EXPORT JxlEncoderStatus +JXL_EXPORT JXL_DEPRECATED JxlEncoderStatus JxlEncoderOptionsSetDistance(JxlEncoderFrameSettings*, float); /** * Create a new set of encoder options, with all values initially copied from * the @p source options, or set to default if @p source is NULL. @@ -1070,11 +1079,11 @@ JXL_EXPORT JxlEncoderFrameSettings* JxlEncoderFrameSettingsCreate( JxlEncoder* enc, const JxlEncoderFrameSettings* source); /** DEPRECATED: use JxlEncoderFrameSettingsCreate instead. */ -JXL_EXPORT JxlEncoderFrameSettings* JxlEncoderOptionsCreate( +JXL_EXPORT JXL_DEPRECATED JxlEncoderFrameSettings* JxlEncoderOptionsCreate( JxlEncoder*, const JxlEncoderFrameSettings*); /** * Sets a color encoding to be sRGB. * diff --git a/third_party/jpeg-xl/lib/include/jxl/types.h b/third_party/jpeg-xl/lib/include/jxl/types.h --- a/third_party/jpeg-xl/lib/include/jxl/types.h +++ b/third_party/jpeg-xl/lib/include/jxl/types.h @@ -14,10 +14,12 @@ #define JXL_TYPES_H_ #include #include +#include "jxl/jxl_export.h" + #if defined(__cplusplus) || defined(c_plusplus) extern "C" { #endif /** @@ -39,32 +41,30 @@ * (within gamut, may go outside this range for wide color gamut). Floating * point output, either JXL_TYPE_FLOAT or JXL_TYPE_FLOAT16, is recommended * for HDR and wide gamut images when color profile conversion is required. */ JXL_TYPE_FLOAT = 0, - /** Use 1-bit packed in uint8_t, first pixel in LSB, padded to uint8_t per - * row. - * TODO(lode): support first in MSB, other padding. - */ - JXL_TYPE_BOOLEAN, - /** Use type uint8_t. May clip wide color gamut data. */ - JXL_TYPE_UINT8, + JXL_TYPE_UINT8 = 2, /** Use type uint16_t. May clip wide color gamut data. */ - JXL_TYPE_UINT16, - - /** Use type uint32_t. May clip wide color gamut data. - */ - JXL_TYPE_UINT32, + JXL_TYPE_UINT16 = 3, /** Use 16-bit IEEE 754 half-precision floating point values */ - JXL_TYPE_FLOAT16, + JXL_TYPE_FLOAT16 = 5, } JxlDataType; +/* DEPRECATED: bit-packed 1-bit data type. Use JXL_TYPE_UINT8 instead. + */ +static const int JXL_DEPRECATED JXL_TYPE_BOOLEAN = 1; + +/* DEPRECATED: uint32_t data type. Use JXL_TYPE_FLOAT instead. + */ +static const int JXL_DEPRECATED JXL_TYPE_UINT32 = 4; + /** Ordering of multi-byte data. */ typedef enum { /** Use the endianness of the system, either little endian or big endian, * without forcing either specific endianness. Do not use if pixel data diff --git a/third_party/jpeg-xl/lib/jxl.cmake b/third_party/jpeg-xl/lib/jxl.cmake --- a/third_party/jpeg-xl/lib/jxl.cmake +++ b/third_party/jpeg-xl/lib/jxl.cmake @@ -37,13 +37,13 @@ jxl/base/padded_bytes.h jxl/base/printf_macros.h jxl/base/profiler.h jxl/base/random.cc jxl/base/random.h + jxl/base/sanitizer_definitions.h jxl/base/scope_guard.h jxl/base/span.h - jxl/base/status.cc jxl/base/status.h jxl/base/thread_pool_internal.h jxl/blending.cc jxl/blending.h jxl/box_content_decoder.cc @@ -104,10 +104,12 @@ jxl/enc_bit_writer.h jxl/entropy_coder.cc jxl/entropy_coder.h jxl/epf.cc jxl/epf.h + jxl/exif.cc + jxl/exif.h jxl/fast_dct-inl.h jxl/fast_dct.cc jxl/fast_dct.h jxl/fast_dct128-inl.h jxl/fast_dct16-inl.h @@ -342,20 +344,21 @@ set(JPEGXL_DEC_INTERNAL_LIBS brotlidec-static brotlicommon-static hwy + Threads::Threads + ${ATOMICS_LIBRARIES} ) if(JPEGXL_ENABLE_PROFILER) list(APPEND JPEGXL_DEC_INTERNAL_LIBS jxl_profiler) endif() set(JPEGXL_INTERNAL_LIBS ${JPEGXL_DEC_INTERNAL_LIBS} brotlienc-static - Threads::Threads ) # strips the -static suffix from all the elements in LIST function(strip_static OUTPUT_VAR LIB_LIST) foreach(lib IN LISTS ${LIB_LIST}) @@ -463,11 +466,11 @@ # for tests. add_library(jxl_dec-static STATIC $ ) target_link_libraries(jxl_dec-static - PUBLIC ${JPEGXL_COVERAGE_FLAGS} ${JPEGXL_DEC_INTERNAL_LIBS} hwy) + PUBLIC ${JPEGXL_COVERAGE_FLAGS} ${JPEGXL_DEC_INTERNAL_LIBS}) target_include_directories(jxl_dec-static PUBLIC "${PROJECT_SOURCE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include") @@ -484,11 +487,11 @@ # for tests. # TODO(lode): once the source files are correctly split so that it is possible # to do, remove $ here and depend on jxl_dec-static add_library(jxl-static STATIC ${JPEGXL_INTERNAL_OBJECTS}) target_link_libraries(jxl-static - PUBLIC ${JPEGXL_COVERAGE_FLAGS} ${JPEGXL_INTERNAL_LIBS} hwy) + PUBLIC ${JPEGXL_COVERAGE_FLAGS} ${JPEGXL_INTERNAL_LIBS}) target_include_directories(jxl-static PUBLIC "${PROJECT_SOURCE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include") diff --git a/third_party/jpeg-xl/lib/jxl/aux_out.h b/third_party/jpeg-xl/lib/jxl/aux_out.h --- a/third_party/jpeg-xl/lib/jxl/aux_out.h +++ b/third_party/jpeg-xl/lib/jxl/aux_out.h @@ -218,11 +218,11 @@ std::array min; std::array max; Image3MinMax(image, &min, &max); Image3B normalized(image.xsize(), image.ysize()); for (size_t c = 0; c < 3; ++c) { - float mul = min[c] == max[c] ? 0 : (1.0f / (max[c] - min[c])); + float mul = min[c] == max[c] ? 0 : (255.0f / (max[c] - min[c])); for (size_t y = 0; y < image.ysize(); ++y) { const T* JXL_RESTRICT row_in = image.ConstPlaneRow(c, y); uint8_t* JXL_RESTRICT row_out = normalized.PlaneRow(c, y); for (size_t x = 0; x < image.xsize(); ++x) { row_out[x] = static_cast((row_in[x] - min[c]) * mul); diff --git a/third_party/jpeg-xl/lib/jxl/base/sanitizer_definitions.h b/third_party/jpeg-xl/lib/jxl/base/sanitizer_definitions.h new file mode 100644 --- /dev/null +++ b/third_party/jpeg-xl/lib/jxl/base/sanitizer_definitions.h @@ -0,0 +1,44 @@ +// Copyright (c) the JPEG XL Project Authors. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef LIB_JXL_BASE_SANITIZER_DEFINITIONS_H_ +#define LIB_JXL_BASE_SANITIZER_DEFINITIONS_H_ + +#ifdef MEMORY_SANITIZER +#define JXL_MEMORY_SANITIZER 1 +#elif defined(__has_feature) +#if __has_feature(memory_sanitizer) +#define JXL_MEMORY_SANITIZER 1 +#else +#define JXL_MEMORY_SANITIZER 0 +#endif +#else +#define JXL_MEMORY_SANITIZER 0 +#endif + +#ifdef ADDRESS_SANITIZER +#define JXL_ADDRESS_SANITIZER 1 +#elif defined(__has_feature) +#if __has_feature(address_sanitizer) +#define JXL_ADDRESS_SANITIZER 1 +#else +#define JXL_ADDRESS_SANITIZER 0 +#endif +#else +#define JXL_ADDRESS_SANITIZER 0 +#endif + +#ifdef THREAD_SANITIZER +#define JXL_THREAD_SANITIZER 1 +#elif defined(__has_feature) +#if __has_feature(thread_sanitizer) +#define JXL_THREAD_SANITIZER 1 +#else +#define JXL_THREAD_SANITIZER 0 +#endif +#else +#define JXL_THREAD_SANITIZER 0 +#endif +#endif // LIB_JXL_BASE_SANITIZER_DEFINITIONS_H diff --git a/third_party/jpeg-xl/lib/jxl/base/status.h b/third_party/jpeg-xl/lib/jxl/base/status.h --- a/third_party/jpeg-xl/lib/jxl/base/status.h +++ b/third_party/jpeg-xl/lib/jxl/base/status.h @@ -12,10 +12,15 @@ #include #include #include #include "lib/jxl/base/compiler_specific.h" +#include "lib/jxl/base/sanitizer_definitions.h" + +#if JXL_ADDRESS_SANITIZER || JXL_MEMORY_SANITIZER || JXL_THREAD_SANITIZER +#include "sanitizer/common_interface_defs.h" // __sanitizer_print_stack_trace +#endif // defined(*_SANITIZER) namespace jxl { // Uncomment to abort when JXL_FAILURE or JXL_STATUS with a fatal error is // reached: @@ -68,11 +73,17 @@ // Print a debug message on standard error. You should use the JXL_DEBUG macro // instead of calling Debug directly. This function returns false, so it can be // used as a return value in JXL_FAILURE. JXL_FORMAT(1, 2) -bool Debug(const char* format, ...); +inline JXL_NOINLINE bool Debug(const char* format, ...) { + va_list args; + va_start(args, format); + vfprintf(stderr, format, args); + va_end(args); + return false; +} // Print a debug message on standard error if "enabled" is true. "enabled" is // normally a macro that evaluates to 0 or 1 at compile time, so the Debug // function is never called and optimized out in release builds. Note that the // arguments are compiled but not evaluated when enabled is false. The format @@ -111,11 +122,25 @@ #endif // JXL_DEBUG_WARNING #define JXL_WARNING(format, ...) \ JXL_DEBUG(JXL_DEBUG_WARNING, format, ##__VA_ARGS__) // Exits the program after printing a stack trace when possible. -JXL_NORETURN bool Abort(); +JXL_NORETURN inline JXL_NOINLINE bool Abort() { +#if JXL_ADDRESS_SANITIZER || JXL_MEMORY_SANITIZER || JXL_THREAD_SANITIZER + // If compiled with any sanitizer print a stack trace. This call doesn't crash + // the program, instead the trap below will crash it also allowing gdb to + // break there. + __sanitizer_print_stack_trace(); +#endif // *_SANITIZER) + +#if JXL_COMPILER_MSVC + __debugbreak(); + abort(); +#else + __builtin_trap(); +#endif +} // Exits the program after printing file/line plus a formatted string. #define JXL_ABORT(format, ...) \ ((JXL_DEBUG_ON_ABORT) && ::jxl::Debug(("%s:%d: JXL_ABORT: " format "\n"), \ __FILE__, __LINE__, ##__VA_ARGS__), \ diff --git a/third_party/jpeg-xl/lib/jxl/base/status.cc b/third_party/jpeg-xl/lib/jxl/base/status.cc deleted file mode 100644 --- a/third_party/jpeg-xl/lib/jxl/base/status.cc +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) the JPEG XL Project Authors. All rights reserved. -// -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -#include "lib/jxl/base/status.h" - -#include -#include -#include - -#include - -#include "lib/jxl/sanitizers.h" - -#if JXL_ADDRESS_SANITIZER || JXL_MEMORY_SANITIZER || JXL_THREAD_SANITIZER -#include "sanitizer/common_interface_defs.h" // __sanitizer_print_stack_trace -#endif // defined(*_SANITIZER) - -namespace jxl { - -bool Debug(const char* format, ...) { - va_list args; - va_start(args, format); - vfprintf(stderr, format, args); - va_end(args); - return false; -} - -bool Abort() { -#if JXL_ADDRESS_SANITIZER || JXL_MEMORY_SANITIZER || JXL_THREAD_SANITIZER - // If compiled with any sanitizer print a stack trace. This call doesn't crash - // the program, instead the trap below will crash it also allowing gdb to - // break there. - __sanitizer_print_stack_trace(); -#endif // *_SANITIZER) - -#if JXL_COMPILER_MSVC - __debugbreak(); - abort(); -#else - __builtin_trap(); -#endif -} - -} // namespace jxl diff --git a/third_party/jpeg-xl/lib/jxl/bits_test.cc b/third_party/jpeg-xl/lib/jxl/bits_test.cc --- a/third_party/jpeg-xl/lib/jxl/bits_test.cc +++ b/third_party/jpeg-xl/lib/jxl/bits_test.cc @@ -39,10 +39,14 @@ for (uint32_t i = 1; i <= 7; ++i) { EXPECT_EQ(expected[i - 1], FloorLog2Nonzero(i)) << " " << i; EXPECT_EQ(expected[i - 1], FloorLog2Nonzero(uint64_t(i))) << " " << i; } + EXPECT_EQ(11u, FloorLog2Nonzero(0x00000fffu)); // 4095 + EXPECT_EQ(12u, FloorLog2Nonzero(0x00001000u)); // 4096 + EXPECT_EQ(12u, FloorLog2Nonzero(0x00001001u)); // 4097 + EXPECT_EQ(31u, FloorLog2Nonzero(0x80000000u)); EXPECT_EQ(31u, FloorLog2Nonzero(0x80000001u)); EXPECT_EQ(31u, FloorLog2Nonzero(0xFFFFFFFFu)); EXPECT_EQ(31u, FloorLog2Nonzero(0x80000000ull)); @@ -60,10 +64,14 @@ for (uint32_t i = 1; i <= 7; ++i) { EXPECT_EQ(expected[i - 1], CeilLog2Nonzero(i)) << " " << i; EXPECT_EQ(expected[i - 1], CeilLog2Nonzero(uint64_t(i))) << " " << i; } + EXPECT_EQ(12u, CeilLog2Nonzero(0x00000fffu)); // 4095 + EXPECT_EQ(12u, CeilLog2Nonzero(0x00001000u)); // 4096 + EXPECT_EQ(13u, CeilLog2Nonzero(0x00001001u)); // 4097 + EXPECT_EQ(31u, CeilLog2Nonzero(0x80000000u)); EXPECT_EQ(32u, CeilLog2Nonzero(0x80000001u)); EXPECT_EQ(32u, CeilLog2Nonzero(0xFFFFFFFFu)); EXPECT_EQ(31u, CeilLog2Nonzero(0x80000000ull)); diff --git a/third_party/jpeg-xl/lib/jxl/blending.cc b/third_party/jpeg-xl/lib/jxl/blending.cc --- a/third_party/jpeg-xl/lib/jxl/blending.cc +++ b/third_party/jpeg-xl/lib/jxl/blending.cc @@ -76,11 +76,11 @@ PerformMulBlending(bg[3 + i] + x0, fg[3 + i] + x0, tmp.Row(3 + i), xsize, ec_blending[i].clamp); } else if (ec_blending[i].mode == PatchBlendMode::kReplace) { memcpy(tmp.Row(3 + i), fg[3 + i] + x0, xsize * sizeof(**fg)); } else if (ec_blending[i].mode == PatchBlendMode::kNone) { - memcpy(tmp.Row(3 + i), bg[3 + i] + x0, xsize * sizeof(**fg)); + if (xsize) memcpy(tmp.Row(3 + i), bg[3 + i] + x0, xsize * sizeof(**fg)); } else { JXL_ABORT("Unreachable"); } } size_t alpha = color_blending.alpha_channel; diff --git a/third_party/jpeg-xl/lib/jxl/butteraugli_wrapper.cc b/third_party/jpeg-xl/lib/jxl/butteraugli_wrapper.cc --- a/third_party/jpeg-xl/lib/jxl/butteraugli_wrapper.cc +++ b/third_party/jpeg-xl/lib/jxl/butteraugli_wrapper.cc @@ -34,26 +34,20 @@ break; case JXL_TYPE_FLOAT16: metadata->SetFloat16Samples(); potential_alpha_bits = 16; break; - case JXL_TYPE_UINT32: - metadata->SetUintSamples(32); - potential_alpha_bits = 16; - break; case JXL_TYPE_UINT16: metadata->SetUintSamples(16); potential_alpha_bits = 16; break; case JXL_TYPE_UINT8: metadata->SetUintSamples(8); potential_alpha_bits = 8; break; - case JXL_TYPE_BOOLEAN: - metadata->SetUintSamples(2); - potential_alpha_bits = 2; - break; + default: + JXL_ABORT("Unhandled JxlDataType"); } if (pixel_format->num_channels == 2 || pixel_format->num_channels == 4) { metadata->SetAlphaBits(potential_alpha_bits); } } @@ -114,13 +108,14 @@ api->intensity_target = v; } void JxlButteraugliApiDestroy(JxlButteraugliApi* api) { if (api) { + JxlMemoryManager local_memory_manager = api->memory_manager; // Call destructor directly since custom free function is used. api->~JxlButteraugliApi(); - jxl::MemoryManagerFree(&api->memory_manager, api); + jxl::MemoryManagerFree(&local_memory_manager, api); } } JxlButteraugliResult* JxlButteraugliCompute( const JxlButteraugliApi* api, uint32_t xsize, uint32_t ysize, @@ -198,10 +193,11 @@ return max_distance; } void JxlButteraugliResultDestroy(JxlButteraugliResult* result) { if (result) { + JxlMemoryManager local_memory_manager = result->memory_manager; // Call destructor directly since custom free function is used. result->~JxlButteraugliResult(); - jxl::MemoryManagerFree(&result->memory_manager, result); + jxl::MemoryManagerFree(&local_memory_manager, result); } } diff --git a/third_party/jpeg-xl/lib/jxl/codec_in_out.h b/third_party/jpeg-xl/lib/jxl/codec_in_out.h --- a/third_party/jpeg-xl/lib/jxl/codec_in_out.h +++ b/third_party/jpeg-xl/lib/jxl/codec_in_out.h @@ -54,14 +54,14 @@ using CodecIntervals = std::array; // RGB[A] or Y[A] // Optional text/EXIF metadata. struct Blobs { - PaddedBytes exif; - PaddedBytes iptc; - PaddedBytes jumbf; - PaddedBytes xmp; + std::vector exif; + std::vector iptc; + std::vector jumbf; + std::vector xmp; }; // Holds a preview, a main image or one or more frames, plus the inputs/outputs // to/from decoding/encoding. class CodecInOut { diff --git a/third_party/jpeg-xl/lib/jxl/common.h b/third_party/jpeg-xl/lib/jxl/common.h --- a/third_party/jpeg-xl/lib/jxl/common.h +++ b/third_party/jpeg-xl/lib/jxl/common.h @@ -15,10 +15,11 @@ #include // numeric_limits #include // unique_ptr #include #include "lib/jxl/base/compiler_specific.h" +#include "lib/jxl/base/padded_bytes.h" #ifndef JXL_HIGH_PRECISION #define JXL_HIGH_PRECISION 1 #endif @@ -190,8 +191,48 @@ // signed snprintf(data, sizeof(data), "%lld", static_cast(n)); } return data; } + +namespace { +static inline uint64_t DecodeVarInt(const uint8_t* input, size_t inputSize, + size_t* pos) { + size_t i; + uint64_t ret = 0; + for (i = 0; *pos + i < inputSize && i < 10; ++i) { + ret |= uint64_t(input[*pos + i] & 127) << uint64_t(7 * i); + // If the next-byte flag is not set, stop + if ((input[*pos + i] & 128) == 0) break; + } + // TODO: Return a decoding error if i == 10. + *pos += i + 1; + return ret; +} + +static inline bool EncodeVarInt(uint64_t value, size_t output_size, + size_t* output_pos, uint8_t* output) { + // While more than 7 bits of data are left, + // store 7 bits and set the next byte flag + while (value > 127) { + if (*output_pos > output_size) return false; + // |128: Set the next byte flag + output[(*output_pos)++] = ((uint8_t)(value & 127)) | 128; + // Remove the seven bits we just wrote + value >>= 7; + } + if (*output_pos > output_size) return false; + output[(*output_pos)++] = ((uint8_t)value) & 127; + return true; +} + +static inline void EncodeVarInt(uint64_t value, PaddedBytes* data) { + size_t pos = data->size(); + data->resize(data->size() + 9); + JXL_CHECK(EncodeVarInt(value, data->size(), &pos, data->data())); + data->resize(pos); +} +} // namespace + } // namespace jxl #endif // LIB_JXL_COMMON_H_ diff --git a/third_party/jpeg-xl/lib/jxl/convolve-inl.h b/third_party/jpeg-xl/lib/jxl/convolve-inl.h --- a/third_party/jpeg-xl/lib/jxl/convolve-inl.h +++ b/third_party/jpeg-xl/lib/jxl/convolve-inl.h @@ -42,11 +42,11 @@ return TableLookupLanes(c, indices); // ONML'KJII #elif HWY_TARGET == HWY_SCALAR return c; // Same (the first mirrored value is the last valid one) #else // 128 bit // c = LKJI -#if HWY_ARCH_X86 +#if HWY_TARGET <= (1 << HWY_HIGHEST_TARGET_BIT_X86) return V{_mm_shuffle_ps(c.raw, c.raw, _MM_SHUFFLE(2, 1, 0, 0))}; // KJII #else const D d; // TODO(deymo): Figure out if this can be optimized using a single vsri // instruction to convert LKJI to KJII. @@ -70,11 +70,11 @@ const D d; JXL_ASSERT(false); // unsupported, avoid calling this. return Zero(d); #else // 128 bit // c = LKJI -#if HWY_ARCH_X86 +#if HWY_TARGET <= (1 << HWY_HIGHEST_TARGET_BIT_X86) return V{_mm_shuffle_ps(c.raw, c.raw, _MM_SHUFFLE(1, 0, 0, 1))}; // JIIJ #else const D d; HWY_ALIGN constexpr int lanes[4] = {1, 0, 0, 1}; // JIIJ const auto indices = SetTableIndices(d, lanes); @@ -96,11 +96,11 @@ const D d; JXL_ASSERT(false); // unsupported, avoid calling this. return Zero(d); #else // 128 bit // c = LKJI -#if HWY_ARCH_X86 +#if HWY_TARGET <= (1 << HWY_HIGHEST_TARGET_BIT_X86) return V{_mm_shuffle_ps(c.raw, c.raw, _MM_SHUFFLE(0, 0, 1, 2))}; // IIJK #else const D d; HWY_ALIGN constexpr int lanes[4] = {2, 1, 0, 0}; // IIJK const auto indices = SetTableIndices(d, lanes); diff --git a/third_party/jpeg-xl/lib/jxl/dec_ans.cc b/third_party/jpeg-xl/lib/jxl/dec_ans.cc --- a/third_party/jpeg-xl/lib/jxl/dec_ans.cc +++ b/third_party/jpeg-xl/lib/jxl/dec_ans.cc @@ -72,13 +72,10 @@ } } else { int is_flat = input->ReadBits(1); if (is_flat == 1) { int alphabet_size = DecodeVarLenUint8(input) + 1; - if (alphabet_size == 0) { - return JXL_FAILURE("Invalid alphabet size for flat histogram."); - } *counts = CreateFlatHistogram(alphabet_size, 1 << precision_bits); return true; } uint32_t shift; diff --git a/third_party/jpeg-xl/lib/jxl/dec_cache.h b/third_party/jpeg-xl/lib/jxl/dec_cache.h --- a/third_party/jpeg-xl/lib/jxl/dec_cache.h +++ b/third_party/jpeg-xl/lib/jxl/dec_cache.h @@ -8,10 +8,11 @@ #include #include // HWY_ALIGN_MAX +#include "jxl/decode.h" #include "lib/jxl/ac_strategy.h" #include "lib/jxl/base/profiler.h" #include "lib/jxl/coeff_order.h" #include "lib/jxl/common.h" #include "lib/jxl/convolve.h" @@ -27,10 +28,35 @@ namespace jxl { constexpr size_t kSigmaBorder = 1; constexpr size_t kSigmaPadding = 2; +struct PixelCallback { + PixelCallback() = default; + PixelCallback(JxlImageOutInitCallback init, JxlImageOutRunCallback run, + JxlImageOutDestroyCallback destroy, void* init_opaque) + : init(init), run(run), destroy(destroy), init_opaque(init_opaque) { +#if JXL_ENABLE_ASSERT + const bool has_init = init != nullptr; + const bool has_run = run != nullptr; + const bool has_destroy = destroy != nullptr; + JXL_ASSERT(has_init == has_run && has_run == has_destroy); +#endif + } + + bool IsPresent() const { return run != nullptr; } + + void* Init(size_t num_threads, size_t num_pixels) const { + return init(init_opaque, num_threads, num_pixels); + } + + JxlImageOutInitCallback init = nullptr; + JxlImageOutRunCallback run = nullptr; + JxlImageOutDestroyCallback destroy = nullptr; + void* init_opaque = nullptr; +}; + // Per-frame decoder state. All the images here should be accessed through a // group rect (either with block units or pixel units). struct PassesDecoderState { PassesSharedState shared_storage; // Allows avoiding copies for encoder loop. @@ -63,11 +89,12 @@ // If true, rgb_output or callback output is RGBA using 4 instead of 3 bytes // per pixel. bool rgb_output_is_rgba; // Callback for line-by-line output. - std::function pixel_callback; + PixelCallback pixel_callback; + // Buffer of upsampling * kApplyImageFeaturesTileDim ones. std::vector opaque_alpha; // One row per thread std::vector> pixel_callback_rows; @@ -104,11 +131,10 @@ std::pow(1 / (1.25f), shared->frame_header.x_qm_scale - 2.0f); b_dm_multiplier = std::pow(1 / (1.25f), shared->frame_header.b_qm_scale - 2.0f); rgb_output = nullptr; - pixel_callback = nullptr; rgb_output_is_rgba = false; fast_xyb_srgb8_conversion = false; used_acs = 0; upsampler8x = GetUpsamplingStage(shared->metadata->transform_data, 0, 3); diff --git a/third_party/jpeg-xl/lib/jxl/dec_cache.cc b/third_party/jpeg-xl/lib/jxl/dec_cache.cc --- a/third_party/jpeg-xl/lib/jxl/dec_cache.cc +++ b/third_party/jpeg-xl/lib/jxl/dec_cache.cc @@ -178,16 +178,16 @@ builder.AddStage(GetSpotColorStage(3 + i, eci.spot_color)); } } } - if (pixel_callback) { + if (pixel_callback.IsPresent()) { builder.AddStage(GetWriteToPixelCallbackStage(pixel_callback, width, height, rgb_output_is_rgba, has_alpha, alpha_c)); } else if (rgb_output) { - builder.AddStage(GetWriteToU8Stage(rgb_output, rgb_stride, width, height, + builder.AddStage(GetWriteToU8Stage(rgb_output, rgb_stride, height, rgb_output_is_rgba, has_alpha, alpha_c)); } else { builder.AddStage(GetWriteToImageBundleStage( decoded, output_encoding_info.color_encoding)); diff --git a/third_party/jpeg-xl/lib/jxl/dec_external_image.h b/third_party/jpeg-xl/lib/jxl/dec_external_image.h --- a/third_party/jpeg-xl/lib/jxl/dec_external_image.h +++ b/third_party/jpeg-xl/lib/jxl/dec_external_image.h @@ -13,49 +13,43 @@ #include "jxl/decode.h" #include "jxl/types.h" #include "lib/jxl/base/status.h" #include "lib/jxl/color_encoding_internal.h" +#include "lib/jxl/dec_cache.h" #include "lib/jxl/image.h" #include "lib/jxl/image_bundle.h" namespace jxl { // Converts ib to interleaved void* pixel buffer with the given format. -// bits_per_sample: must be 8, 16 or 32, and must be 32 if float_out -// is true. 1 and 32 int are not yet implemented. +// bits_per_sample: must be 16 or 32 if float_out is true, and at most 16 +// if it is false. No bit packing is done. // num_channels: must be 1, 2, 3 or 4 for gray, gray+alpha, RGB, RGB+alpha. // This supports the features needed for the C API and does not perform // color space conversion. -// TODO(lode): support 1-bit output (bits_per_sample == 1) // TODO(lode): support rectangle crop. // stride_out is output scanline size in bytes, must be >= // output_xsize * output_bytes_per_pixel. // undo_orientation is an EXIF orientation to undo. Depending on the // orientation, the output xsize and ysize are swapped compared to input // xsize and ysize. Status ConvertToExternal(const jxl::ImageBundle& ib, size_t bits_per_sample, bool float_out, size_t num_channels, JxlEndianness endianness, size_t stride_out, jxl::ThreadPool* thread_pool, void* out_image, - size_t out_size, JxlImageOutCallback out_callback, - void* out_opaque, jxl::Orientation undo_orientation); + size_t out_size, const PixelCallback& out_callback, + jxl::Orientation undo_orientation); // Converts single-channel image to interleaved void* pixel buffer with the // given format, with a single channel. -// bits_per_sample: must be 8, 16 or 32, and must be 32 if float_out -// is true. 1 and 32 int are not yet implemented. // This supports the features needed for the C API to get extra channels. -// stride_out is output scanline size in bytes, must be >= -// output_xsize * output_bytes_per_pixel. -// undo_orientation is an EXIF orientation to undo. Depending on the -// orientation, the output xsize and ysize are swapped compared to input -// xsize and ysize. +// Arguments are similar to the multi-channel function above. Status ConvertToExternal(const jxl::ImageF& channel, size_t bits_per_sample, bool float_out, JxlEndianness endianness, size_t stride_out, jxl::ThreadPool* thread_pool, void* out_image, size_t out_size, - JxlImageOutCallback out_callback, void* out_opaque, + const PixelCallback& out_callback, jxl::Orientation undo_orientation); } // namespace jxl #endif // LIB_JXL_DEC_EXTERNAL_IMAGE_H_ diff --git a/third_party/jpeg-xl/lib/jxl/dec_external_image.cc b/third_party/jpeg-xl/lib/jxl/dec_external_image.cc --- a/third_party/jpeg-xl/lib/jxl/dec_external_image.cc +++ b/third_party/jpeg-xl/lib/jxl/dec_external_image.cc @@ -30,32 +30,13 @@ HWY_BEFORE_NAMESPACE(); namespace jxl { namespace HWY_NAMESPACE { +// TODO(jon): check if this can be replaced by a FloatToU16 function void FloatToU32(const float* in, uint32_t* out, size_t num, float mul, size_t bits_per_sample) { - // TODO(eustas): investigate 24..31 bpp cases. - if (bits_per_sample == 32) { - // Conversion to real 32-bit *unsigned* integers requires more intermediate - // precision that what is given by the usual f32 -> i32 conversion - // instructions, so we run the non-SIMD path for those. - const uint32_t cap = (1ull << bits_per_sample) - 1; - for (size_t x = 0; x < num; x++) { - float v = in[x]; - if (v >= 1.0f) { - out[x] = cap; - } else if (v >= 0.0f) { // Inverted condition => NaN -> 0. - out[x] = static_cast(v * mul + 0.5f); - } else { - out[x] = 0; - } - } - return; - } - - // General SIMD case for less than 32 bits output. const HWY_FULL(float) d; const hwy::HWY_NAMESPACE::Rebind du; // Unpoison accessing partially-uninitialized vectors with memory sanitizer. // This is because we run NearestInt() on the vector, which triggers msan even @@ -265,43 +246,40 @@ Status ConvertChannelsToExternal(const ImageF* channels[], size_t num_channels, size_t bits_per_sample, bool float_out, JxlEndianness endianness, size_t stride, jxl::ThreadPool* pool, void* out_image, size_t out_size, - JxlImageOutCallback out_callback, - void* out_opaque, + const PixelCallback& out_callback, jxl::Orientation undo_orientation) { JXL_DASSERT(num_channels != 0 && num_channels <= kConvertMaxChannels); JXL_DASSERT(channels[0] != nullptr); - - if (bits_per_sample < 1 || bits_per_sample > 32) { - return JXL_FAILURE("Invalid bits_per_sample value."); - } - if (!!out_image == !!out_callback) { + JXL_CHECK(float_out ? bits_per_sample == 16 || bits_per_sample == 32 + : bits_per_sample > 0 && bits_per_sample <= 16); + if (!!out_image == out_callback.IsPresent()) { return JXL_FAILURE( "Must provide either an out_image or an out_callback, but not both."); } - // TODO(deymo): Implement 1-bit per pixel packed in 8 samples per byte. - if (bits_per_sample == 1) { - return JXL_FAILURE("packed 1-bit per sample is not yet supported"); - } - if (bits_per_sample > 16 && bits_per_sample < 32) { - return JXL_FAILURE("not supported, try bits_per_sample=32"); - } - // bytes_per_channel and is only valid for bits_per_sample > 1. const size_t bytes_per_channel = DivCeil(bits_per_sample, jxl::kBitsPerByte); const size_t bytes_per_pixel = num_channels * bytes_per_channel; std::vector> row_out_callback; - auto InitOutCallback = [&](size_t num_threads) { - if (out_callback) { + const auto FreeCallbackOpaque = [&out_callback](void* p) { + out_callback.destroy(p); + }; + std::unique_ptr out_run_opaque( + nullptr, FreeCallbackOpaque); + auto InitOutCallback = [&](size_t num_threads) -> Status { + if (out_callback.IsPresent()) { + out_run_opaque.reset(out_callback.Init(num_threads, stride)); + JXL_RETURN_IF_ERROR(out_run_opaque != nullptr); row_out_callback.resize(num_threads); for (size_t i = 0; i < num_threads; ++i) { row_out_callback[i].resize(stride); } } + return true; }; // Channels used to store the transformed original channels if needed. ImageF temp_channels[kConvertMaxChannels]; if (undo_orientation != Orientation::kIdentity) { @@ -320,11 +298,11 @@ if (stride < bytes_per_pixel * xsize) { return JXL_FAILURE("stride is smaller than scanline width in bytes: %" PRIuS " vs %" PRIuS, stride, bytes_per_pixel * xsize); } - if (!out_callback && + if (!out_callback.IsPresent() && out_size < (ysize - 1) * stride + bytes_per_pixel * xsize) { return JXL_FAILURE("out_size is too small to store image"); } const bool little_endian = @@ -349,12 +327,11 @@ JXL_RETURN_IF_ERROR(RunOnPool( pool, 0, static_cast(ysize), [&](size_t num_threads) { f16_cache = Plane(xsize, num_channels * num_threads); - InitOutCallback(num_threads); - return true; + return InitOutCallback(num_threads); }, [&](const uint32_t task, const size_t thread) { const int64_t y = task; const float* JXL_RESTRICT row_in[kConvertMaxChannels]; for (size_t c = 0; c < num_channels; c++) { @@ -365,11 +342,11 @@ row_f16[c] = f16_cache.Row(c + thread * num_channels); HWY_DYNAMIC_DISPATCH(FloatToF16) (row_in[c], row_f16[c], xsize); } uint8_t* row_out = - out_callback + out_callback.IsPresent() ? row_out_callback[thread].data() : &(reinterpret_cast(out_image))[stride * y]; // interleave the one scanline hwy::float16_t* row_f16_out = reinterpret_cast(row_out); @@ -382,26 +359,24 @@ size_t size = xsize * num_channels * 2; for (size_t i = 0; i < size; i += 2) { std::swap(row_out[i + 0], row_out[i + 1]); } } - if (out_callback) { - (*out_callback)(out_opaque, 0, y, xsize, row_out); + if (out_callback.IsPresent()) { + out_callback.run(out_run_opaque.get(), thread, 0, y, xsize, + row_out); } }, "ConvertF16")); } else if (bits_per_sample == 32) { JXL_RETURN_IF_ERROR(RunOnPool( pool, 0, static_cast(ysize), - [&](size_t num_threads) { - InitOutCallback(num_threads); - return true; - }, + [&](size_t num_threads) { return InitOutCallback(num_threads); }, [&](const uint32_t task, const size_t thread) { const int64_t y = task; uint8_t* row_out = - out_callback + out_callback.IsPresent() ? row_out_callback[thread].data() : &(reinterpret_cast(out_image))[stride * y]; const float* JXL_RESTRICT row_in[kConvertMaxChannels]; for (size_t c = 0; c < num_channels; c++) { row_in[c] = channels[c] ? channels[c]->Row(y) : ones.Row(0); @@ -409,12 +384,13 @@ if (little_endian) { StoreFloatRow(row_in, num_channels, xsize, row_out); } else { StoreFloatRow(row_in, num_channels, xsize, row_out); } - if (out_callback) { - (*out_callback)(out_opaque, 0, y, xsize, row_out); + if (out_callback.IsPresent()) { + out_callback.run(out_run_opaque.get(), thread, 0, y, xsize, + row_out); } }, "ConvertFloat")); } else { return JXL_FAILURE("float other than 16-bit and 32-bit not supported"); @@ -426,17 +402,16 @@ Plane u32_cache; JXL_RETURN_IF_ERROR(RunOnPool( pool, 0, static_cast(ysize), [&](size_t num_threads) { u32_cache = Plane(xsize, num_channels * num_threads); - InitOutCallback(num_threads); - return true; + return InitOutCallback(num_threads); }, [&](const uint32_t task, const size_t thread) { const int64_t y = task; uint8_t* row_out = - out_callback + out_callback.IsPresent() ? row_out_callback[thread].data() : &(reinterpret_cast(out_image))[stride * y]; const float* JXL_RESTRICT row_in[kConvertMaxChannels]; for (size_t c = 0; c < num_channels; c++) { row_in[c] = channels[c] ? channels[c]->Row(y) : ones.Row(0); @@ -448,28 +423,22 @@ // intended to be initialized on a previous run. msan::PoisonMemory(row_u32[c], xsize * sizeof(row_u32[c][0])); HWY_DYNAMIC_DISPATCH(FloatToU32) (row_in[c], row_u32[c], xsize, mul, bits_per_sample); } - // TODO(deymo): add bits_per_sample == 1 case here. if (bits_per_sample <= 8) { StoreUintRow(row_u32, num_channels, xsize, 1, row_out); - } else if (bits_per_sample <= 16) { + } else { if (little_endian) { StoreUintRow(row_u32, num_channels, xsize, 2, row_out); } else { StoreUintRow(row_u32, num_channels, xsize, 2, row_out); } - } else { - if (little_endian) { - StoreUintRow(row_u32, num_channels, xsize, 4, row_out); - } else { - StoreUintRow(row_u32, num_channels, xsize, 4, row_out); - } } - if (out_callback) { - (*out_callback)(out_opaque, 0, y, xsize, row_out); + if (out_callback.IsPresent()) { + out_callback.run(out_run_opaque.get(), thread, 0, y, xsize, + row_out); } }, "ConvertUint")); } return true; @@ -479,12 +448,12 @@ Status ConvertToExternal(const jxl::ImageBundle& ib, size_t bits_per_sample, bool float_out, size_t num_channels, JxlEndianness endianness, size_t stride, jxl::ThreadPool* pool, void* out_image, - size_t out_size, JxlImageOutCallback out_callback, - void* out_opaque, jxl::Orientation undo_orientation) { + size_t out_size, const PixelCallback& out_callback, + jxl::Orientation undo_orientation) { bool want_alpha = num_channels == 2 || num_channels == 4; size_t color_channels = num_channels <= 2 ? 1 : 3; const Image3F* color = &ib.color(); // Undo premultiplied alpha. @@ -510,22 +479,22 @@ } JXL_ASSERT(num_channels == c); return ConvertChannelsToExternal( channels, num_channels, bits_per_sample, float_out, endianness, stride, - pool, out_image, out_size, out_callback, out_opaque, undo_orientation); + pool, out_image, out_size, out_callback, undo_orientation); } Status ConvertToExternal(const jxl::ImageF& channel, size_t bits_per_sample, bool float_out, JxlEndianness endianness, size_t stride, jxl::ThreadPool* pool, void* out_image, - size_t out_size, JxlImageOutCallback out_callback, - void* out_opaque, jxl::Orientation undo_orientation) { + size_t out_size, const PixelCallback& out_callback, + jxl::Orientation undo_orientation) { const ImageF* channels[1]; channels[0] = &channel; - return ConvertChannelsToExternal( - channels, 1, bits_per_sample, float_out, endianness, stride, pool, - out_image, out_size, out_callback, out_opaque, undo_orientation); + return ConvertChannelsToExternal(channels, 1, bits_per_sample, float_out, + endianness, stride, pool, out_image, + out_size, out_callback, undo_orientation); } } // namespace jxl #endif // HWY_ONCE diff --git a/third_party/jpeg-xl/lib/jxl/dec_external_image_gbench.cc b/third_party/jpeg-xl/lib/jxl/dec_external_image_gbench.cc --- a/third_party/jpeg-xl/lib/jxl/dec_external_image_gbench.cc +++ b/third_party/jpeg-xl/lib/jxl/dec_external_image_gbench.cc @@ -36,11 +36,11 @@ ib, /*bits_per_sample=*/8, /*float_out=*/false, num_channels, JXL_NATIVE_ENDIAN, /*stride*/ bytes_per_row, /*thread_pool=*/nullptr, interleaved.data(), interleaved.size(), - /*out_callback=*/nullptr, /*out_opaque=*/nullptr, + /*out_callback=*/{}, /*undo_orientation=*/jxl::Orientation::kIdentity)); } } // Pixels per second. diff --git a/third_party/jpeg-xl/lib/jxl/dec_file.cc b/third_party/jpeg-xl/lib/jxl/dec_file.cc --- a/third_party/jpeg-xl/lib/jxl/dec_file.cc +++ b/third_party/jpeg-xl/lib/jxl/dec_file.cc @@ -64,11 +64,27 @@ Status ret = true; { BitReader reader(file); BitReaderScopedCloser reader_closer(&reader, &ret); - (void)reader.ReadFixedBits<16>(); // skip marker + if (reader.ReadFixedBits<16>() != 0x0AFF) { + // We don't have a naked codestream. Make a quick & dirty attempt to find + // the codestream. + // TODO(jon): get rid of this whole function + const unsigned char* begin = file.data(); + const unsigned char* end = file.data() + file.size() - 4; + while (begin < end) { + if (!memcmp(begin, "jxlc", 4)) break; + begin++; + } + if (begin >= end) return JXL_FAILURE("Couldn't find jxl codestream"); + reader.SkipBits(8 * (begin - file.data() + 2)); + unsigned int firstbytes = reader.ReadFixedBits<16>(); + if (firstbytes != 0x0AFF) + return JXL_FAILURE("Codestream didn't start with FF0A but with %X", + firstbytes); + } { JXL_RETURN_IF_ERROR(DecodeHeaders(&reader, io)); size_t xsize = io->metadata.xsize(); size_t ysize = io->metadata.ysize(); diff --git a/third_party/jpeg-xl/lib/jxl/dec_frame.h b/third_party/jpeg-xl/lib/jxl/dec_frame.h --- a/third_party/jpeg-xl/lib/jxl/dec_frame.h +++ b/third_party/jpeg-xl/lib/jxl/dec_frame.h @@ -6,10 +6,11 @@ #ifndef LIB_JXL_DEC_FRAME_H_ #define LIB_JXL_DEC_FRAME_H_ #include +#include "jxl/decode.h" #include "lib/jxl/base/compiler_specific.h" #include "lib/jxl/base/data_parallel.h" #include "lib/jxl/base/span.h" #include "lib/jxl/base/status.h" #include "lib/jxl/blending.h" @@ -157,11 +158,11 @@ bool is_rgba, bool undo_orientation) const { if (!CanDoLowMemoryPath(undo_orientation)) return; dec_state_->rgb_output = rgb_output; dec_state_->rgb_output_is_rgba = is_rgba; dec_state_->rgb_stride = stride; - JXL_ASSERT(dec_state_->pixel_callback == nullptr); + JXL_ASSERT(!dec_state_->pixel_callback.IsPresent()); #if !JXL_HIGH_PRECISION if (decoded_->metadata()->xyb_encoded && dec_state_->output_encoding_info.color_encoding.IsSRGB() && dec_state_->output_encoding_info.all_default_opsin && HasFastXYBTosRGB8() && frame_header_.needs_color_transform()) { @@ -179,26 +180,24 @@ // @param undo_orientation: if true, indicates the frame decoder should apply // the exif orientation to bring the image to the intended display // orientation. Performing this operation is not yet supported, so this // results in not setting the buffer if the image has a non-identity EXIF // orientation. When outputting to the ImageBundle, no orientation is undone. - void MaybeSetFloatCallback( - const std::function& cb, - bool is_rgba, bool undo_orientation) const { + void MaybeSetFloatCallback(const PixelCallback& pixel_callback, bool is_rgba, + bool undo_orientation) const { if (!CanDoLowMemoryPath(undo_orientation)) return; - dec_state_->pixel_callback = cb; + dec_state_->pixel_callback = pixel_callback; dec_state_->rgb_output_is_rgba = is_rgba; JXL_ASSERT(dec_state_->rgb_output == nullptr); } // Returns true if the rgb output buffer passed by MaybeSetRGB8OutputBuffer // has been/will be populated by Flush() / FinalizeFrame(), or if a pixel // callback has been used. bool HasRGBBuffer() const { return dec_state_->rgb_output != nullptr || - dec_state_->pixel_callback != nullptr; + dec_state_->pixel_callback.IsPresent(); } private: Status ProcessDCGlobal(BitReader* br); Status ProcessDCGroup(size_t dc_group_id, BitReader* br); @@ -214,22 +213,23 @@ // Allocates storage for parallel decoding using up to `num_threads` threads // of up to `num_tasks` tasks. The value of `thread` passed to // `GetStorageLocation` must be smaller than the `num_threads` value passed // here. The value of `task` passed to `GetStorageLocation` must be smaller // than the value of `num_tasks` passed here. - void PrepareStorage(size_t num_threads, size_t num_tasks) { + Status PrepareStorage(size_t num_threads, size_t num_tasks) { size_t storage_size = std::min(num_threads, num_tasks); if (storage_size > group_dec_caches_.size()) { group_dec_caches_.resize(storage_size); } use_task_id_ = num_threads > num_tasks; if (dec_state_->render_pipeline) { - dec_state_->render_pipeline->PrepareForThreads( + JXL_RETURN_IF_ERROR(dec_state_->render_pipeline->PrepareForThreads( storage_size, /*use_group_ids=*/modular_frame_decoder_.UsesFullImage() && - frame_header_.encoding == FrameEncoding::kVarDCT); + frame_header_.encoding == FrameEncoding::kVarDCT)); } + return true; } size_t GetStorageLocation(size_t thread, size_t task) { if (use_task_id_) return task; return thread; @@ -240,22 +240,12 @@ // future frames, there are no spot colors to be rendered, and alpha is not // premultiplied, then low memory options can be used // (uint8 output buffer or float pixel callback). // TODO(veluca): reduce this set of restrictions. bool CanDoLowMemoryPath(bool undo_orientation) const { - if (undo_orientation && - decoded_->metadata()->GetOrientation() != Orientation::kIdentity) { - return false; - } - if (NeedsBlending(dec_state_)) return false; - if (frame_header_.CanBeReferenced()) return false; - if (render_spotcolors_ && - decoded_->metadata()->Find(ExtraChannel::kSpotColor)) { - return false; - } - if (decoded_->AlphaIsPremultiplied()) return false; - return true; + return !(undo_orientation && + decoded_->metadata()->GetOrientation() != Orientation::kIdentity); } PassesDecoderState* dec_state_; ThreadPool* pool_; std::vector section_offsets_; diff --git a/third_party/jpeg-xl/lib/jxl/dec_frame.cc b/third_party/jpeg-xl/lib/jxl/dec_frame.cc --- a/third_party/jpeg-xl/lib/jxl/dec_frame.cc +++ b/third_party/jpeg-xl/lib/jxl/dec_frame.cc @@ -263,10 +263,11 @@ decoded->ClearExtraChannels(); decoded->duration = frame_header_.animation_frame.duration; if (!frame_header_.nonserialized_is_preview && + (frame_header_.is_last || frame_header_.animation_frame.duration > 0) && (frame_header_.frame_type == kRegularFrame || frame_header_.frame_type == kSkipProgressive)) { ++dec_state_->visible_frame_index; dec_state_->nonvisible_frame_index = 0; } else { @@ -554,14 +555,15 @@ Status FrameDecoder::ProcessACGroup(size_t ac_group_id, BitReader* JXL_RESTRICT* br, size_t num_passes, size_t thread, bool force_draw, bool dc_only) { PROFILER_ZONE("process_group"); + size_t group_dim = frame_dim_.group_dim; const size_t gx = ac_group_id % frame_dim_.xsize_groups; const size_t gy = ac_group_id / frame_dim_.xsize_groups; - const size_t x = gx * frame_dim_.group_dim; - const size_t y = gy * frame_dim_.group_dim; + const size_t x = gx * group_dim; + const size_t y = gy * group_dim; RenderPipelineInput render_pipeline_input = dec_state_->render_pipeline->GetInputBuffers(ac_group_id, thread); bool should_run_pipeline = true; @@ -575,11 +577,11 @@ decoded_passes_per_ac_group_[ac_group_id], force_draw, dc_only, &should_run_pipeline)); } // don't limit to image dimensions here (is done in DecodeGroup) - const Rect mrect(x, y, frame_dim_.group_dim, frame_dim_.group_dim); + const Rect mrect(x, y, group_dim, group_dim); for (size_t i = 0; i < frame_header_.passes.num_passes; i++) { int minShift, maxShift; frame_header_.passes.GetDownsamplingBracket(i, minShift, maxShift); if (i >= decoded_passes_per_ac_group_[ac_group_id] && i < decoded_passes_per_ac_group_[ac_group_id] + num_passes) { @@ -610,18 +612,18 @@ for (size_t c = 0; c < 3; c++) { auto r = render_pipeline_input.GetBuffer(noise_c_start + c); rects[c].first = r.first; size_t x1 = r.second.x0() + r.second.xsize(); size_t y1 = r.second.y0() + r.second.ysize(); - rects[c].second = Rect(r.second.x0() + ix * kGroupDim, - r.second.y0() + iy * kGroupDim, kGroupDim, - kGroupDim, x1, y1); + rects[c].second = Rect(r.second.x0() + ix * group_dim, + r.second.y0() + iy * group_dim, group_dim, + group_dim, x1, y1); } Random3Planes(dec_state_->visible_frame_index, dec_state_->nonvisible_frame_index, - (gx * frame_header_.upsampling + ix) * kGroupDim, - (gy * frame_header_.upsampling + iy) * kGroupDim, + (gx * frame_header_.upsampling + ix) * group_dim, + (gy * frame_header_.upsampling + iy) * group_dim, rects[0], rects[1], rects[2]); } } } @@ -790,12 +792,12 @@ } JXL_RETURN_IF_ERROR(RunOnPool( pool_, 0, ac_group_sec.size(), [this](size_t num_threads) { - PrepareStorage(num_threads, decoded_passes_per_ac_group_.size()); - return true; + return PrepareStorage(num_threads, + decoded_passes_per_ac_group_.size()); }, [this, &ac_group_sec, &num_ac_passes, &num, §ions, §ion_status, &has_error](size_t g, size_t thread) { if (num_ac_passes[g] == 0) { // no new AC pass, nothing to do. return; @@ -861,12 +863,12 @@ } std::atomic has_error{false}; JXL_RETURN_IF_ERROR(RunOnPool( pool_, 0, decoded_passes_per_ac_group_.size(), [this](const size_t num_threads) { - PrepareStorage(num_threads, decoded_passes_per_ac_group_.size()); - return true; + return PrepareStorage(num_threads, + decoded_passes_per_ac_group_.size()); }, [this, &has_error](const uint32_t g, size_t thread) { if (decoded_passes_per_ac_group_[g] == frame_header_.passes.num_passes) { // This group was drawn already, nothing to do. diff --git a/third_party/jpeg-xl/lib/jxl/dec_group.cc b/third_party/jpeg-xl/lib/jxl/dec_group.cc --- a/third_party/jpeg-xl/lib/jxl/dec_group.cc +++ b/third_party/jpeg-xl/lib/jxl/dec_group.cc @@ -738,11 +738,11 @@ kRenderPipelineXOffset; } // Arguments set to 0/nullptr are not used. dec_state->upsampler8x->ProcessRow(input_rows, output_rows, /*xextra=*/0, src_rect.xsize(), 0, 0, - nullptr); + thread); } } return true; } diff --git a/third_party/jpeg-xl/lib/jxl/dec_modular.cc b/third_party/jpeg-xl/lib/jxl/dec_modular.cc --- a/third_party/jpeg-xl/lib/jxl/dec_modular.cc +++ b/third_party/jpeg-xl/lib/jxl/dec_modular.cc @@ -87,10 +87,20 @@ namespace jxl { HWY_EXPORT(MultiplySum); // Local function HWY_EXPORT(RgbFromSingle); // Local function HWY_EXPORT(SingleFromSingle); // Local function +// Slow conversion using double precision multiplication, only +// needed when the bit depth is too high for single precision +void SingleFromSingleAccurate(const size_t xsize, + const pixel_type* const JXL_RESTRICT row_in, + const double factor, float* row_out) { + for (size_t x = 0; x < xsize; x++) { + row_out[x] = row_in[x] * factor; + } +} + // convert custom [bits]-bit float (with [exp_bits] exponent bits) stored as int // back to binary32 float void int_to_float(const pixel_type* const JXL_RESTRICT row_in, float* const JXL_RESTRICT row_out, const size_t xsize, const int bits, const int exp_bits) { @@ -455,26 +465,26 @@ RenderPipelineInput& render_pipeline_input, Rect modular_rect) { const auto& frame_header = dec_state->shared->frame_header; const auto* metadata = frame_header.nonserialized_metadata; JXL_CHECK(gi.transform.empty()); - auto get_row = [&](Rect r, size_t c, size_t y) { - return render_pipeline_input.GetBuffer(c).second.Row( - render_pipeline_input.GetBuffer(c).first, y); + auto get_row = [&](size_t c, size_t y) { + const auto& buffer = render_pipeline_input.GetBuffer(c); + return buffer.second.Row(buffer.first, y); }; size_t c = 0; if (do_color) { const bool rgb_from_gray = metadata->m.color_encoding.IsGray() && frame_header.color_transform == ColorTransform::kNone; const bool fp = metadata->m.bit_depth.floating_point_sample && frame_header.color_transform != ColorTransform::kXYB; for (; c < 3; c++) { - float factor = full_image.bitdepth < 32 - ? 1.f / ((1u << full_image.bitdepth) - 1) - : 0; + double factor = full_image.bitdepth < 32 + ? 1.0 / ((1u << full_image.bitdepth) - 1) + : 0; size_t c_in = c; if (frame_header.color_transform == ColorTransform::kXYB) { factor = dec_state->shared->matrices.DCQuants()[c]; // XYB is encoded as YX(B-Y) if (c < 2) c_in = 1 - c; @@ -511,11 +521,11 @@ const size_t y = task; const pixel_type* const JXL_RESTRICT row_in = mr.Row(&ch_in.plane, y); const pixel_type* const JXL_RESTRICT row_in_Y = mr.Row(&gi.channel[0].plane, y); - float* const JXL_RESTRICT row_out = get_row(r, c, y); + float* const JXL_RESTRICT row_out = get_row(c, y); HWY_DYNAMIC_DISPATCH(MultiplySum) (xsize_shifted, row_in, row_in_Y, factor, row_out); }, "ModularIntToFloat")); } else if (fp) { @@ -527,15 +537,15 @@ const size_t y = task; const pixel_type* const JXL_RESTRICT row_in = mr.Row(&ch_in.plane, y); if (rgb_from_gray) { for (size_t cc = 0; cc < 3; cc++) { - float* const JXL_RESTRICT row_out = get_row(r, cc, y); + float* const JXL_RESTRICT row_out = get_row(cc, y); int_to_float(row_in, row_out, xsize_shifted, bits, exp_bits); } } else { - float* const JXL_RESTRICT row_out = get_row(r, c, y); + float* const JXL_RESTRICT row_out = get_row(c, y); int_to_float(row_in, row_out, xsize_shifted, bits, exp_bits); } }, "ModularIntToFloat_losslessfloat")); } else { @@ -544,17 +554,31 @@ [&](const uint32_t task, size_t /* thread */) { const size_t y = task; const pixel_type* const JXL_RESTRICT row_in = mr.Row(&ch_in.plane, y); if (rgb_from_gray) { - HWY_DYNAMIC_DISPATCH(RgbFromSingle) - (xsize_shifted, row_in, factor, get_row(r, 0, y), - get_row(r, 1, y), get_row(r, 2, y)); + if (full_image.bitdepth < 23) { + HWY_DYNAMIC_DISPATCH(RgbFromSingle) + (xsize_shifted, row_in, factor, get_row(0, y), get_row(1, y), + get_row(2, y)); + } else { + SingleFromSingleAccurate(xsize_shifted, row_in, factor, + get_row(0, y)); + SingleFromSingleAccurate(xsize_shifted, row_in, factor, + get_row(1, y)); + SingleFromSingleAccurate(xsize_shifted, row_in, factor, + get_row(2, y)); + } } else { - float* const JXL_RESTRICT row_out = get_row(r, c, y); - HWY_DYNAMIC_DISPATCH(SingleFromSingle) - (xsize_shifted, row_in, factor, row_out); + float* const JXL_RESTRICT row_out = get_row(c, y); + if (full_image.bitdepth < 23) { + HWY_DYNAMIC_DISPATCH(SingleFromSingle) + (xsize_shifted, row_in, factor, row_out); + } else { + SingleFromSingleAccurate(xsize_shifted, row_in, factor, + row_out); + } } }, "ModularIntToFloat")); } if (rgb_from_gray) { @@ -570,11 +594,11 @@ const ExtraChannelInfo& eci = metadata->m.extra_channel_info[ec]; int bits = eci.bit_depth.bits_per_sample; int exp_bits = eci.bit_depth.exponent_bits_per_sample; bool fp = eci.bit_depth.floating_point_sample; JXL_ASSERT(fp || bits < 32); - const float mul = fp ? 0 : (1.0f / ((1u << bits) - 1)); + const double factor = fp ? 0 : (1.0 / ((1u << bits) - 1)); JXL_ASSERT(c < gi.channel.size()); Channel& ch_in = gi.channel[c]; Rect r = render_pipeline_input.GetBuffer(3 + ec).second; Rect mr(modular_rect.x0() >> ch_in.hshift, modular_rect.y0() >> ch_in.vshift, @@ -587,12 +611,15 @@ r.Row(render_pipeline_input.GetBuffer(3 + ec).first, y); const pixel_type* const JXL_RESTRICT row_in = mr.Row(&ch_in.plane, y); if (fp) { int_to_float(row_in, row_out, r.xsize(), bits, exp_bits); } else { - for (size_t x = 0; x < r.xsize(); ++x) { - row_out[x] = row_in[x] * mul; + if (full_image.bitdepth < 23) { + HWY_DYNAMIC_DISPATCH(SingleFromSingle) + (r.xsize(), row_in, factor, row_out); + } else { + SingleFromSingleAccurate(r.xsize(), row_in, factor, row_out); } } } } return true; @@ -619,15 +646,14 @@ std::atomic has_error{false}; JXL_RETURN_IF_ERROR(RunOnPool( pool, 0, dec_state->shared->frame_dim.num_groups, [&](size_t num_threads) { - dec_state->render_pipeline->PrepareForThreads( + return dec_state->render_pipeline->PrepareForThreads( num_threads, /*use_group_ids=*/dec_state->shared->frame_header.encoding == FrameEncoding::kVarDCT); - return true; }, [&](const uint32_t group, size_t thread_id) { RenderPipelineInput input = dec_state->render_pipeline->GetInputBuffers(group, thread_id); if (!ModularImageToDecodedRect(gi, dec_state, nullptr, input, diff --git a/third_party/jpeg-xl/lib/jxl/dec_noise.h b/third_party/jpeg-xl/lib/jxl/dec_noise.h --- a/third_party/jpeg-xl/lib/jxl/dec_noise.h +++ b/third_party/jpeg-xl/lib/jxl/dec_noise.h @@ -18,20 +18,10 @@ #include "lib/jxl/image.h" #include "lib/jxl/noise.h" namespace jxl { -// Add a noise to Opsin image, loading generated random noise from `noise_rect` -// in `noise`. -void AddNoise(const NoiseParams& noise_params, const Rect& noise_rect, - const Image3F& noise, const Rect& opsin_rect, - const ColorCorrelationMap& cmap, Image3F* opsin); - -void RandomImage3(size_t visible_frame_index, size_t nonvisible_frame_index, - size_t x0, size_t y0, const Rect& rect, - Image3F* JXL_RESTRICT noise); - void Random3Planes(size_t visible_frame_index, size_t nonvisible_frame_index, size_t x0, size_t y0, const std::pair& plane0, const std::pair& plane1, const std::pair& plane2); diff --git a/third_party/jpeg-xl/lib/jxl/dec_noise.cc b/third_party/jpeg-xl/lib/jxl/dec_noise.cc --- a/third_party/jpeg-xl/lib/jxl/dec_noise.cc +++ b/third_party/jpeg-xl/lib/jxl/dec_noise.cc @@ -83,186 +83,10 @@ row + x); batch_pos += N; } } } - -// [0, max_value] -template -static HWY_INLINE V Clamp0ToMax(D d, const V x, const V max_value) { - const auto clamped = Min(x, max_value); - return ZeroIfNegative(clamped); -} - -// x is in [0+delta, 1+delta], delta ~= 0.06 -template -typename StrengthEval::V NoiseStrength(const StrengthEval& eval, - const typename StrengthEval::V x) { - return Clamp0ToMax(D(), eval(x), Set(D(), 1.0f)); -} - -// TODO(veluca): SIMD-fy. -class StrengthEvalLut { - public: - using V = Vec; - - explicit StrengthEvalLut(const NoiseParams& noise_params) -#if HWY_TARGET == HWY_SCALAR - : noise_params_(noise_params) -#endif - { -#if HWY_TARGET != HWY_SCALAR - uint32_t lut[8]; - memcpy(lut, noise_params.lut, sizeof(lut)); - for (size_t i = 0; i < 8; i++) { - low16_lut[2 * i] = (lut[i] >> 0) & 0xFF; - low16_lut[2 * i + 1] = (lut[i] >> 8) & 0xFF; - high16_lut[2 * i] = (lut[i] >> 16) & 0xFF; - high16_lut[2 * i + 1] = (lut[i] >> 24) & 0xFF; - } -#endif - } - - V operator()(const V vx) const { - constexpr size_t kScale = NoiseParams::kNumNoisePoints - 2; - auto scaled_vx = Max(Zero(D()), vx * Set(D(), kScale)); - auto floor_x = Floor(scaled_vx); - auto frac_x = scaled_vx - floor_x; - floor_x = IfThenElse(scaled_vx >= Set(D(), kScale + 1), Set(D(), kScale), - floor_x); - frac_x = IfThenElse(scaled_vx >= Set(D(), kScale + 1), Set(D(), 1), frac_x); - auto floor_x_int = ConvertTo(DI(), floor_x); -#if HWY_TARGET == HWY_SCALAR - auto low = Set(D(), noise_params_.lut[floor_x_int.raw]); - auto hi = Set(D(), noise_params_.lut[floor_x_int.raw + 1]); -#else - // Set each lane's bytes to {0, 0, 2x+1, 2x}. - auto floorx_indices_low = - floor_x_int * Set(DI(), 0x0202) + Set(DI(), 0x0100); - // Set each lane's bytes to {2x+1, 2x, 0, 0}. - auto floorx_indices_hi = - floor_x_int * Set(DI(), 0x02020000) + Set(DI(), 0x01000000); - // load LUT - auto low16 = BitCast(DI(), LoadDup128(DI8(), low16_lut)); - auto lowm = Set(DI(), 0xFFFF); - auto hi16 = BitCast(DI(), LoadDup128(DI8(), high16_lut)); - auto him = Set(DI(), 0xFFFF0000); - // low = noise_params.lut[floor_x] - auto low = - BitCast(D(), (TableLookupBytes(low16, floorx_indices_low) & lowm) | - (TableLookupBytes(hi16, floorx_indices_hi) & him)); - // hi = noise_params.lut[floor_x+1] - floorx_indices_low += Set(DI(), 0x0202); - floorx_indices_hi += Set(DI(), 0x02020000); - auto hi = - BitCast(D(), (TableLookupBytes(low16, floorx_indices_low) & lowm) | - (TableLookupBytes(hi16, floorx_indices_hi) & him)); -#endif - return MulAdd(hi - low, frac_x, low); - } - - private: -#if HWY_TARGET != HWY_SCALAR - // noise_params.lut transformed into two 16-bit lookup tables. - HWY_ALIGN uint8_t high16_lut[16]; - HWY_ALIGN uint8_t low16_lut[16]; -#else - const NoiseParams& noise_params_; -#endif -}; - -template -void AddNoiseToRGB(const D d, const Vec rnd_noise_r, - const Vec rnd_noise_g, const Vec rnd_noise_cor, - const Vec noise_strength_g, const Vec noise_strength_r, - float ytox, float ytob, float* JXL_RESTRICT out_x, - float* JXL_RESTRICT out_y, float* JXL_RESTRICT out_b) { - const auto kRGCorr = Set(d, 0.9921875f); // 127/128 - const auto kRGNCorr = Set(d, 0.0078125f); // 1/128 - - const auto red_noise = kRGNCorr * rnd_noise_r * noise_strength_r + - kRGCorr * rnd_noise_cor * noise_strength_r; - const auto green_noise = kRGNCorr * rnd_noise_g * noise_strength_g + - kRGCorr * rnd_noise_cor * noise_strength_g; - - auto vx = Load(d, out_x); - auto vy = Load(d, out_y); - auto vb = Load(d, out_b); - - vx += red_noise - green_noise + Set(d, ytox) * (red_noise + green_noise); - vy += red_noise + green_noise; - vb += Set(d, ytob) * (red_noise + green_noise); - - Store(vx, d, out_x); - Store(vy, d, out_y); - Store(vb, d, out_b); -} - -void AddNoise(const NoiseParams& noise_params, const Rect& noise_rect, - const Image3F& noise, const Rect& opsin_rect, - const ColorCorrelationMap& cmap, Image3F* opsin) { - if (!noise_params.HasAny()) return; - const StrengthEvalLut noise_model(noise_params); - D d; - const auto half = Set(d, 0.5f); - - const size_t xsize = opsin_rect.xsize(); - const size_t ysize = opsin_rect.ysize(); - - // With the prior subtract-random Laplacian approximation, rnd_* ranges were - // about [-1.5, 1.6]; Laplacian3 about doubles this to [-3.6, 3.6], so the - // normalizer is half of what it was before (0.5). - const auto norm_const = Set(d, 0.22f); - - float ytox = cmap.YtoXRatio(0); - float ytob = cmap.YtoBRatio(0); - - const size_t xsize_v = RoundUpTo(xsize, Lanes(d)); - - for (size_t y = 0; y < ysize; ++y) { - float* JXL_RESTRICT row_x = opsin_rect.PlaneRow(opsin, 0, y); - float* JXL_RESTRICT row_y = opsin_rect.PlaneRow(opsin, 1, y); - float* JXL_RESTRICT row_b = opsin_rect.PlaneRow(opsin, 2, y); - const float* JXL_RESTRICT row_rnd_r = noise_rect.ConstPlaneRow(noise, 0, y); - const float* JXL_RESTRICT row_rnd_g = noise_rect.ConstPlaneRow(noise, 1, y); - const float* JXL_RESTRICT row_rnd_c = noise_rect.ConstPlaneRow(noise, 2, y); - // Needed by the calls to Floor() in StrengthEvalLut. Only arithmetic and - // shuffles are otherwise done on the data, so this is safe. - msan::UnpoisonMemory(row_x + xsize, (xsize_v - xsize) * sizeof(float)); - msan::UnpoisonMemory(row_y + xsize, (xsize_v - xsize) * sizeof(float)); - for (size_t x = 0; x < xsize; x += Lanes(d)) { - const auto vx = Load(d, row_x + x); - const auto vy = Load(d, row_y + x); - const auto in_g = vy - vx; - const auto in_r = vy + vx; - const auto noise_strength_g = NoiseStrength(noise_model, in_g * half); - const auto noise_strength_r = NoiseStrength(noise_model, in_r * half); - const auto addit_rnd_noise_red = Load(d, row_rnd_r + x) * norm_const; - const auto addit_rnd_noise_green = Load(d, row_rnd_g + x) * norm_const; - const auto addit_rnd_noise_correlated = - Load(d, row_rnd_c + x) * norm_const; - AddNoiseToRGB(D(), addit_rnd_noise_red, addit_rnd_noise_green, - addit_rnd_noise_correlated, noise_strength_g, - noise_strength_r, ytox, ytob, row_x + x, row_y + x, - row_b + x); - } - msan::PoisonMemory(row_x + xsize, (xsize_v - xsize) * sizeof(float)); - msan::PoisonMemory(row_y + xsize, (xsize_v - xsize) * sizeof(float)); - msan::PoisonMemory(row_b + xsize, (xsize_v - xsize) * sizeof(float)); - } -} - -void RandomImage3(size_t visible_frame_index, size_t nonvisible_frame_index, - size_t x0, size_t y0, const Rect& rect, - Image3F* JXL_RESTRICT noise) { - HWY_ALIGN Xorshift128Plus rng(visible_frame_index, nonvisible_frame_index, x0, - y0); - RandomImage(&rng, rect, &noise->Plane(0)); - RandomImage(&rng, rect, &noise->Plane(1)); - RandomImage(&rng, rect, &noise->Plane(2)); -} - void Random3Planes(size_t visible_frame_index, size_t nonvisible_frame_index, size_t x0, size_t y0, const std::pair& plane0, const std::pair& plane1, const std::pair& plane2) { HWY_ALIGN Xorshift128Plus rng(visible_frame_index, nonvisible_frame_index, x0, @@ -278,26 +102,10 @@ HWY_AFTER_NAMESPACE(); #if HWY_ONCE namespace jxl { -HWY_EXPORT(AddNoise); -void AddNoise(const NoiseParams& noise_params, const Rect& noise_rect, - const Image3F& noise, const Rect& opsin_rect, - const ColorCorrelationMap& cmap, Image3F* opsin) { - return HWY_DYNAMIC_DISPATCH(AddNoise)(noise_params, noise_rect, noise, - opsin_rect, cmap, opsin); -} - -HWY_EXPORT(RandomImage3); -void RandomImage3(size_t visible_frame_index, size_t nonvisible_frame_index, - size_t x0, size_t y0, const Rect& rect, - Image3F* JXL_RESTRICT noise) { - return HWY_DYNAMIC_DISPATCH(RandomImage3)( - visible_frame_index, nonvisible_frame_index, x0, y0, rect, noise); -} - HWY_EXPORT(Random3Planes); void Random3Planes(size_t visible_frame_index, size_t nonvisible_frame_index, size_t x0, size_t y0, const std::pair& plane0, const std::pair& plane1, const std::pair& plane2) { diff --git a/third_party/jpeg-xl/lib/jxl/dec_patch_dictionary.h b/third_party/jpeg-xl/lib/jxl/dec_patch_dictionary.h --- a/third_party/jpeg-xl/lib/jxl/dec_patch_dictionary.h +++ b/third_party/jpeg-xl/lib/jxl/dec_patch_dictionary.h @@ -124,15 +124,10 @@ // Adds patches to a segment of `xsize` pixels, starting at `inout`, assumed // to be located at position (x0, y) in the frame. void AddOneRow(float* const* inout, size_t y, size_t x0, size_t xsize) const; - // Only adds patches that belong to the `image_rect` area of the decoded - // image, writing them to the `opsin_rect` area of `opsin`. - void AddTo(Image3F* opsin, const Rect& opsin_rect, - float* const* extra_channels, const Rect& image_rect) const; - // Returns dependencies of this patch dictionary on reference frame ids as a // bit mask: bits 0-3 indicate reference frame 0-3. int GetReferences() const; private: diff --git a/third_party/jpeg-xl/lib/jxl/dec_patch_dictionary.cc b/third_party/jpeg-xl/lib/jxl/dec_patch_dictionary.cc --- a/third_party/jpeg-xl/lib/jxl/dec_patch_dictionary.cc +++ b/third_party/jpeg-xl/lib/jxl/dec_patch_dictionary.cc @@ -154,13 +154,10 @@ positions_.shrink_to_fit(); if (!decoder.CheckANSFinalState()) { return JXL_FAILURE("ANS checksum failure."); } - if (!HasAny()) { - return JXL_FAILURE("Decoded patch dictionary but got none"); - } ComputePatchCache(); return true; } @@ -235,53 +232,6 @@ patch_x1 - patch_x0, pos.blending[0], pos.blending.data() + 1, shared_->metadata->m.extra_channel_info); } } - -void PatchDictionary::AddTo(Image3F* opsin, const Rect& opsin_rect, - float* const* extra_channels, - const Rect& image_rect) const { - JXL_CHECK(SameSize(opsin_rect, image_rect)); - if (patch_starts_.empty()) return; - size_t num_ec = shared_->metadata->m.num_extra_channels; - std::vector fg_ptrs(3 + num_ec); - std::vector bg_ptrs(3 + num_ec); - for (size_t y = image_rect.y0(); y < image_rect.y0() + image_rect.ysize(); - y++) { - if (y + 1 >= patch_starts_.size()) continue; - for (size_t id = patch_starts_[y]; id < patch_starts_[y + 1]; id++) { - const PatchPosition& pos = positions_[sorted_patches_[id]]; - size_t by = pos.y; - size_t bx = pos.x; - size_t xsize = pos.ref_pos.xsize; - JXL_DASSERT(y >= by); - JXL_DASSERT(y < by + pos.ref_pos.ysize); - size_t iy = y - by; - size_t ref = pos.ref_pos.ref; - if (bx >= image_rect.x0() + image_rect.xsize()) continue; - if (bx + xsize < image_rect.x0()) continue; - size_t x0 = std::max(bx, image_rect.x0()); - size_t x1 = std::min(bx + xsize, image_rect.x0() + image_rect.xsize()); - for (size_t c = 0; c < 3; c++) { - fg_ptrs[c] = - shared_->reference_frames[ref].frame->color()->ConstPlaneRow( - c, pos.ref_pos.y0 + iy) + - pos.ref_pos.x0 + x0 - bx; - bg_ptrs[c] = opsin_rect.PlaneRow(opsin, c, y - image_rect.y0()) + x0 - - image_rect.x0(); - } - for (size_t i = 0; i < num_ec; i++) { - fg_ptrs[3 + i] = - shared_->reference_frames[ref].frame->extra_channels()[i].ConstRow( - pos.ref_pos.y0 + iy) + - pos.ref_pos.x0 + x0 - bx; - bg_ptrs[3 + i] = extra_channels[i] + x0 - image_rect.x0(); - } - PerformBlending(bg_ptrs.data(), fg_ptrs.data(), bg_ptrs.data(), 0, - x1 - x0, pos.blending[0], pos.blending.data() + 1, - shared_->metadata->m.extra_channel_info); - } - } -} - } // namespace jxl diff --git a/third_party/jpeg-xl/lib/jxl/decode.cc b/third_party/jpeg-xl/lib/jxl/decode.cc --- a/third_party/jpeg-xl/lib/jxl/decode.cc +++ b/third_party/jpeg-xl/lib/jxl/decode.cc @@ -137,25 +137,21 @@ namespace { size_t BitsPerChannel(JxlDataType data_type) { switch (data_type) { - case JXL_TYPE_BOOLEAN: - return 1; case JXL_TYPE_UINT8: return 8; case JXL_TYPE_UINT16: return 16; - case JXL_TYPE_UINT32: - return 32; case JXL_TYPE_FLOAT: return 32; case JXL_TYPE_FLOAT16: return 16; - // No default, give compiler error if new type not handled. + default: + return 0; // signals unhandled JxlDataType } - return 0; // Indicate invalid data type. } enum class DecoderStage : uint32_t { kInited, // Decoder created, no JxlDecoderProcessInput called yet kStarted, // Running JxlDecoderProcessInput calls @@ -463,12 +459,19 @@ bool image_out_buffer_set; // Owned by the caller, buffers for DC image and full resolution images void* preview_out_buffer; void* image_out_buffer; - JxlImageOutCallback image_out_callback; - void* image_out_opaque; + JxlImageOutInitCallback image_out_init_callback; + JxlImageOutRunCallback image_out_run_callback; + JxlImageOutDestroyCallback image_out_destroy_callback; + void* image_out_init_opaque; + struct SimpleImageOutCallback { + JxlImageOutCallback callback; + void* opaque; + }; + SimpleImageOutCallback simple_image_out_callback; size_t preview_out_size; size_t image_out_size; JxlPixelFormat preview_out_format; @@ -607,10 +610,15 @@ namespace { bool CheckSizeLimit(JxlDecoder* dec, size_t xsize, size_t ysize) { if (!dec->memory_limit_base) return true; if (xsize == 0 || ysize == 0) return true; + if (xsize >= dec->memory_limit_base || ysize >= dec->memory_limit_base) { + return false; + } + // Rough estimate of real row length. + xsize = jxl::DivCeil(xsize, 32) * 32; size_t num_pixels = xsize * ysize; if (num_pixels / xsize != ysize) return false; // overflow if (num_pixels > dec->memory_limit_base) return false; return true; } @@ -668,12 +676,14 @@ dec->box_count = 0; dec->preview_out_buffer_set = false; dec->image_out_buffer_set = false; dec->preview_out_buffer = nullptr; dec->image_out_buffer = nullptr; - dec->image_out_callback = nullptr; - dec->image_out_opaque = nullptr; + dec->image_out_init_callback = nullptr; + dec->image_out_run_callback = nullptr; + dec->image_out_destroy_callback = nullptr; + dec->image_out_init_opaque = nullptr; dec->preview_out_size = 0; dec->image_out_size = 0; dec->extra_channel_output.clear(); dec->dec_pixels = 0; dec->next_in = 0; @@ -729,11 +739,11 @@ JxlDecoder* dec = new (alloc) JxlDecoder(); dec->memory_manager = local_memory_manager; #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION if (!memory_manager) { - dec->memory_limit_base = 1 << 21; + dec->memory_limit_base = 53 << 16; // Allow 5 x max_image_size processing units; every frame is accounted // as W x H CPU processing units, so there could be numerous small frames // or few larger ones. dec->cpu_limit_base = 5 * dec->memory_limit_base; } @@ -744,13 +754,14 @@ return dec; } void JxlDecoderDestroy(JxlDecoder* dec) { if (dec) { + JxlMemoryManager local_memory_manager = dec->memory_manager; // Call destructor directly since custom free function is used. dec->~JxlDecoder(); - jxl::MemoryManagerFree(&dec->memory_manager, dec); + jxl::MemoryManagerFree(&local_memory_manager, dec); } } void JxlDecoderRewind(JxlDecoder* dec) { JxlDecoderRewindDecodingState(dec); } @@ -1049,11 +1060,11 @@ // and treated as if it is 1. static JxlDecoderStatus ConvertImageInternal( const JxlDecoder* dec, const jxl::ImageBundle& frame, const JxlPixelFormat& format, bool want_extra_channel, size_t extra_channel_index, void* out_image, size_t out_size, - JxlImageOutCallback out_callback, void* out_opaque) { + const PixelCallback& out_callback) { // TODO(lode): handle mismatch of RGB/grayscale color profiles and pixel data // color/grayscale format const size_t stride = GetStride(dec, format); bool float_format = format.data_type == JXL_TYPE_FLOAT || @@ -1063,23 +1074,21 @@ ? jxl::Orientation::kIdentity : dec->metadata.m.GetOrientation(); jxl::Status status(true); if (want_extra_channel) { - status = jxl::ConvertToExternal( - frame.extra_channels()[extra_channel_index], - BitsPerChannel(format.data_type), float_format, format.endianness, - stride, dec->thread_pool.get(), out_image, out_size, - /*out_callback=*/out_callback, - /*out_opaque=*/out_opaque, undo_orientation); + JXL_ASSERT(extra_channel_index < frame.extra_channels().size()); + status = jxl::ConvertToExternal(frame.extra_channels()[extra_channel_index], + BitsPerChannel(format.data_type), + float_format, format.endianness, stride, + dec->thread_pool.get(), out_image, out_size, + out_callback, undo_orientation); } else { status = jxl::ConvertToExternal( frame, BitsPerChannel(format.data_type), float_format, format.num_channels, format.endianness, stride, dec->thread_pool.get(), - out_image, out_size, - /*out_callback=*/out_callback, - /*out_opaque=*/out_opaque, undo_orientation); + out_image, out_size, out_callback, undo_orientation); } return status ? JXL_DEC_SUCCESS : JXL_DEC_ERROR; } @@ -1250,12 +1259,12 @@ if (want_preview) { if (dec->preview_out_buffer) { JxlDecoderStatus status = ConvertImageInternal( dec, ib, dec->preview_out_format, /*want_extra_channel=*/false, /*extra_channel_index=*/0, dec->preview_out_buffer, - dec->preview_out_size, /*out_callback=*/nullptr, - /*out_opaque=*/nullptr); + dec->preview_out_size, + /*out_callback=*/{}); if (status != JXL_DEC_SUCCESS) return status; } return JXL_DEC_PREVIEW_IMAGE; } } @@ -1451,21 +1460,20 @@ IsLittleEndian()); bool swap_endianness = little_endian != IsLittleEndian(); // TODO(lode): Support more formats than just native endian float32 for // the low-memory callback path - if (dec->image_out_buffer_set && !!dec->image_out_callback && + if (dec->image_out_buffer_set && !!dec->image_out_init_callback && + !!dec->image_out_run_callback && dec->image_out_format.data_type == JXL_TYPE_FLOAT && dec->image_out_format.num_channels >= 3 && !swap_endianness && dec->frame_dec_in_progress) { bool is_rgba = dec->image_out_format.num_channels == 4; dec->frame_dec->MaybeSetFloatCallback( - [dec](const float* pixels, size_t x, size_t y, size_t num_pixels) { - JXL_DASSERT(num_pixels > 0); - dec->image_out_callback(dec->image_out_opaque, x, y, num_pixels, - pixels); - }, + PixelCallback{ + dec->image_out_init_callback, dec->image_out_run_callback, + dec->image_out_destroy_callback, dec->image_out_init_opaque}, is_rgba, !dec->keep_orientation); } size_t pos = dec->frame_start - dec->codestream_pos; if (pos >= size) { @@ -1555,25 +1563,34 @@ // Copy pixels if desired. JxlDecoderStatus status = ConvertImageInternal( dec, *dec->ib, dec->image_out_format, /*want_extra_channel=*/false, /*extra_channel_index=*/0, dec->image_out_buffer, - dec->image_out_size, dec->image_out_callback, - dec->image_out_opaque); + dec->image_out_size, + PixelCallback{dec->image_out_init_callback, + dec->image_out_run_callback, + dec->image_out_destroy_callback, + dec->image_out_init_opaque}); if (status != JXL_DEC_SUCCESS) return status; } dec->image_out_buffer_set = false; + bool has_ec = !dec->ib->extra_channels().empty(); for (size_t i = 0; i < dec->extra_channel_output.size(); ++i) { void* buffer = dec->extra_channel_output[i].buffer; // buffer nullptr indicates this extra channel is not requested if (!buffer) continue; + if (!has_ec) { + JXL_WARNING( + "Extra channels are not supported when callback is used"); + return JXL_DEC_ERROR; + } const JxlPixelFormat* format = &dec->extra_channel_output[i].format; JxlDecoderStatus status = ConvertImageInternal( dec, *dec->ib, *format, - /*want_extra_channel=*/true, i, buffer, - dec->extra_channel_output[i].buffer_size, nullptr, nullptr); + /*want_extra_channel=*/true, /*extra_channel_index=*/i, buffer, + dec->extra_channel_output[i].buffer_size, /*out_callback=*/{}); if (status != JXL_DEC_SUCCESS) return status; } dec->extra_channel_output.clear(); } @@ -2383,21 +2400,15 @@ return JXL_API_ERROR("Don't know frame dimensions yet"); } if (format->num_channels > 4) { return JXL_API_ERROR("More than 4 channels not supported"); } - if (format->data_type == JXL_TYPE_BOOLEAN) { - return JXL_API_ERROR("Boolean data type not yet supported"); - } - if (format->data_type == JXL_TYPE_UINT32) { - return JXL_API_ERROR("uint32 data type not yet supported"); - } *bits = BitsPerChannel(format->data_type); if (*bits == 0) { - return JXL_API_ERROR("Invalid data type"); + return JXL_API_ERROR("Invalid/unsupported data type"); } return JXL_DEC_SUCCESS; } @@ -2438,11 +2449,11 @@ dec->ib->ShrinkTo(xsize_nopadding, ysize_nopadding); JxlDecoderStatus status = jxl::ConvertImageInternal( dec, *dec->ib, dec->image_out_format, /*want_extra_channel=*/false, /*extra_channel_index=*/0, dec->image_out_buffer, dec->image_out_size, - /*out_callback=*/nullptr, /*out_opaque=*/nullptr); + /*out_callback=*/{}); dec->ib->ShrinkTo(xsize, ysize); if (status != JXL_DEC_SUCCESS) return status; return JXL_DEC_SUCCESS; } @@ -2546,11 +2557,11 @@ const JxlPixelFormat* format, void* buffer, size_t size) { if (!dec->got_basic_info || !(dec->orig_events_wanted & JXL_DEC_FULL_IMAGE)) { return JXL_API_ERROR("No image out buffer needed at this time"); } - if (dec->image_out_buffer_set && !!dec->image_out_callback) { + if (dec->image_out_buffer_set && !!dec->image_out_run_callback) { return JXL_API_ERROR( "Cannot change from image out callback to image out buffer"); } if (format->num_channels < 3 && !dec->metadata.m.color_encoding.IsGray()) { return JXL_API_ERROR("Grayscale output not possible for color image"); @@ -2632,23 +2643,55 @@ JxlDecoderStatus JxlDecoderSetImageOutCallback(JxlDecoder* dec, const JxlPixelFormat* format, JxlImageOutCallback callback, void* opaque) { + dec->simple_image_out_callback.callback = callback; + dec->simple_image_out_callback.opaque = opaque; + const auto init_callback = + +[](void* init_opaque, size_t num_threads, size_t num_pixels_per_thread) { + // No initialization to do, just reuse init_opaque as run_opaque. + return init_opaque; + }; + const auto run_callback = + +[](void* run_opaque, size_t thread_id, size_t x, size_t y, + size_t num_pixels, const void* pixels) { + const auto* const simple_callback = + static_cast(run_opaque); + simple_callback->callback(simple_callback->opaque, x, y, num_pixels, + pixels); + }; + const auto destroy_callback = +[](void* run_opaque) {}; + return JxlDecoderSetMultithreadedImageOutCallback( + dec, format, init_callback, run_callback, + /*destroy_callback=*/destroy_callback, &dec->simple_image_out_callback); +} + +JxlDecoderStatus JxlDecoderSetMultithreadedImageOutCallback( + JxlDecoder* dec, const JxlPixelFormat* format, + JxlImageOutInitCallback init_callback, JxlImageOutRunCallback run_callback, + JxlImageOutDestroyCallback destroy_callback, void* init_opaque) { if (dec->image_out_buffer_set && !!dec->image_out_buffer) { return JXL_API_ERROR( "Cannot change from image out buffer to image out callback"); } + if (init_callback == nullptr || run_callback == nullptr || + destroy_callback == nullptr) { + return JXL_API_ERROR("All callbacks are required"); + } + // Perform error checking for invalid format. size_t bits_dummy; JxlDecoderStatus status = PrepareSizeCheck(dec, format, &bits_dummy); if (status != JXL_DEC_SUCCESS) return status; dec->image_out_buffer_set = true; - dec->image_out_callback = callback; - dec->image_out_opaque = opaque; + dec->image_out_init_callback = init_callback; + dec->image_out_run_callback = run_callback; + dec->image_out_destroy_callback = destroy_callback; + dec->image_out_init_opaque = init_opaque; dec->image_out_format = *format; return JXL_DEC_SUCCESS; } diff --git a/third_party/jpeg-xl/lib/jxl/decode_test.cc b/third_party/jpeg-xl/lib/jxl/decode_test.cc --- a/third_party/jpeg-xl/lib/jxl/decode_test.cc +++ b/third_party/jpeg-xl/lib/jxl/decode_test.cc @@ -209,11 +209,11 @@ &pool, &jpeg_bytes)); jpeg_codestream->append(jpeg_bytes.data(), jpeg_bytes.data() + jpeg_bytes.size()); EXPECT_TRUE(jxl::jpeg::DecodeImageJPG( jxl::Span(jpeg_bytes.data(), jpeg_bytes.size()), &io)); - EXPECT_TRUE(EncodeJPEGData(*io.Main().jpeg_data, &jpeg_data)); + EXPECT_TRUE(EncodeJPEGData(*io.Main().jpeg_data, &jpeg_data, cparams)); io.metadata.m.xyb_encoded = false; #else // JPEGXL_ENABLE_JPEG JXL_ABORT( "unable to create reconstructible JPEG without JPEG support enabled"); #endif // JPEGXL_ENABLE_JPEG @@ -1232,11 +1232,12 @@ for (size_t i = 0; i < pixels.size(); i++) pixels[i] = 0; EXPECT_TRUE(ConvertToExternal( io.Main(), 16, /*float_out=*/false, orig_channels, JXL_BIG_ENDIAN, xsize * 2 * orig_channels, nullptr, pixels.data(), pixels.size(), - nullptr, nullptr, static_cast(config.orientation))); + /*out_callback=*/{}, + static_cast(config.orientation))); } if (config.upsampling == 1) { EXPECT_EQ(0u, jxl::test::ComparePixels(pixels.data(), pixels2.data(), xsize, ysize, format_orig, format)); } else { @@ -1404,16 +1405,12 @@ os << "f32"; break; case JXL_TYPE_FLOAT16: os << "f16"; break; - case JXL_TYPE_UINT32: - os << "u32"; - break; - case JXL_TYPE_BOOLEAN: - os << "b"; - break; + default: + JXL_ASSERT(false); }; if (jxl::test::GetDataBits(c.data_type) > jxl::kBitsPerByte) { if (c.endianness == JXL_NATIVE_ENDIAN) { // add nothing } else if (c.endianness == JXL_BIG_ENDIAN) { @@ -3566,11 +3563,11 @@ } #endif // JPEGXL_ENABLE_JPEG TEST(DecodeTest, JXL_TRANSCODE_JPEG_TEST(JPEGReconstructionTest)) { const std::string jpeg_path = - "imagecompression.info/flower_foveon.png.im_q85_420.jpg"; + "third_party/imagecompression.info/flower_foveon.png.im_q85_420.jpg"; const jxl::PaddedBytes orig = jxl::ReadTestData(jpeg_path); jxl::CodecInOut orig_io; ASSERT_TRUE( jxl::jpeg::DecodeImageJPG(jxl::Span(orig), &orig_io)); orig_io.metadata.m.xyb_encoded = false; @@ -3584,11 +3581,12 @@ orig_io.Main(), &enc_state, jxl::GetJxlCms(), /*pool=*/nullptr, &writer, /*aux_out=*/nullptr)); jxl::PaddedBytes jpeg_data; - ASSERT_TRUE(EncodeJPEGData(*orig_io.Main().jpeg_data.get(), &jpeg_data)); + ASSERT_TRUE( + EncodeJPEGData(*orig_io.Main().jpeg_data.get(), &jpeg_data, cparams)); jxl::PaddedBytes container; container.append(jxl::kContainerHeader, jxl::kContainerHeader + sizeof(jxl::kContainerHeader)); jxl::AppendBoxHeader(jxl::MakeBoxType("jbrd"), jpeg_data.size(), false, &container); @@ -4098,11 +4096,11 @@ jxl::CompressParams cparams; cparams.speed_tier = jxl::SpeedTier::kLightning; cparams.modular_mode = true; cparams.color_transform = jxl::ColorTransform::kNone; - cparams.quality_pair = {100, 100}; + cparams.butteraugli_distance = 0.f; jxl::PaddedBytes compressed; std::unique_ptr enc_state = jxl::make_unique(); EXPECT_TRUE(jxl::EncodeFile(cparams, &io, enc_state.get(), &compressed, diff --git a/third_party/jpeg-xl/lib/jxl/enc_adaptive_quantization.cc b/third_party/jpeg-xl/lib/jxl/enc_adaptive_quantization.cc --- a/third_party/jpeg-xl/lib/jxl/enc_adaptive_quantization.cc +++ b/third_party/jpeg-xl/lib/jxl/enc_adaptive_quantization.cc @@ -1100,13 +1100,14 @@ const ImageMetadata& metadata = *decoded.metadata(); JXL_CHECK(dec_state->PreparePipeline(&decoded, options)); hwy::AlignedUniquePtr group_dec_caches; - const auto allocate_storage = [&](const size_t num_threads) { - dec_state->render_pipeline->PrepareForThreads(num_threads, - /*use_group_ids=*/false); + const auto allocate_storage = [&](const size_t num_threads) -> Status { + JXL_RETURN_IF_ERROR( + dec_state->render_pipeline->PrepareForThreads(num_threads, + /*use_group_ids=*/false)); group_dec_caches = hwy::MakeUniqueAlignedArray(num_threads); return true; }; const auto process_group = [&](const uint32_t group_index, const size_t thread) { diff --git a/third_party/jpeg-xl/lib/jxl/enc_cache.cc b/third_party/jpeg-xl/lib/jxl/enc_cache.cc --- a/third_party/jpeg-xl/lib/jxl/enc_cache.cc +++ b/third_party/jpeg-xl/lib/jxl/enc_cache.cc @@ -96,12 +96,14 @@ cparams.gaborish = Override::kOff; // Use kVarDCT in max_error_mode for intermediate progressive DC, // and kModular for the smallest DC (first in the bitstream) if (cparams.progressive_dc == 0) { cparams.modular_mode = true; - cparams.quality_pair.first = cparams.quality_pair.second = - 99.f - enc_state->cparams.butteraugli_distance * 0.2f; + // TODO(jon): tweak mapping from image dist to dist for modular DC + cparams.butteraugli_distance = + std::max(kMinButteraugliDistance, + enc_state->cparams.butteraugli_distance * 0.03f); } ImageBundle ib(&shared.metadata->m); // This is a lie - dc is in XYB // (but EncodeFrame will skip RGB->XYB conversion anyway) ib.SetFromImage( diff --git a/third_party/jpeg-xl/lib/jxl/enc_external_image.cc b/third_party/jpeg-xl/lib/jxl/enc_external_image.cc --- a/third_party/jpeg-xl/lib/jxl/enc_external_image.cc +++ b/third_party/jpeg-xl/lib/jxl/enc_external_image.cc @@ -84,11 +84,10 @@ uint32_t JXL_INLINE Load8(const uint8_t* p) { return *p; } Status PixelFormatToExternal(const JxlPixelFormat& pixel_format, size_t* bitdepth, bool* float_in) { - // TODO(zond): Make this accept uint32. if (pixel_format.data_type == JXL_TYPE_FLOAT) { *bitdepth = 32; *float_in = true; } else if (pixel_format.data_type == JXL_TYPE_FLOAT16) { *bitdepth = 16; @@ -98,35 +97,24 @@ *float_in = false; } else if (pixel_format.data_type == JXL_TYPE_UINT16) { *bitdepth = 16; *float_in = false; } else { - return JXL_FAILURE("unsupported bitdepth"); + return JXL_FAILURE("unsupported pixel format data type"); } return true; } } // namespace Status ConvertFromExternal(Span bytes, size_t xsize, size_t ysize, size_t bits_per_sample, JxlEndianness endianness, ThreadPool* pool, ImageF* channel, bool float_in, size_t align) { // TODO(firsching): Avoid code duplication with the function below. - if (bits_per_sample < 1 || bits_per_sample > 32) { - return JXL_FAILURE("Invalid bits_per_sample value."); - } - // TODO(deymo): Implement 1-bit per sample as 8 samples per byte. In - // any other case we use DivCeil(bits_per_sample, 8) bytes per pixel per - // channel. - if (bits_per_sample == 1) { - return JXL_FAILURE("packed 1-bit per sample is not yet supported"); - } - - // bytes_per_pixel are only valid for - // bits_per_sample > 1. + JXL_CHECK(float_in ? bits_per_sample == 16 || bits_per_sample == 32 + : bits_per_sample > 0 && bits_per_sample <= 16); const size_t bytes_per_pixel = DivCeil(bits_per_sample, jxl::kBitsPerByte); - const size_t last_row_size = xsize * bytes_per_pixel; const size_t row_size = (align > 1 ? jxl::DivCeil(last_row_size, align) * align : last_row_size); const size_t bytes_to_read = row_size * (ysize - 1) + last_row_size; if (xsize == 0 || ysize == 0) return JXL_FAILURE("Empty image"); @@ -151,11 +139,11 @@ pool, 0, static_cast(ysize), ThreadPool::NoInit, [&](const uint32_t task, size_t /*thread*/) { const size_t y = task; size_t i = row_size * task; float* JXL_RESTRICT row_out = channel->Row(y); - if (bits_per_sample <= 16) { + if (bits_per_sample == 16) { if (little_endian) { for (size_t x = 0; x < xsize; ++x) { row_out[x] = LoadLEFloat16(in + i); i += bytes_per_pixel; } @@ -186,30 +174,20 @@ pool, 0, static_cast(ysize), ThreadPool::NoInit, [&](const uint32_t task, size_t /*thread*/) { const size_t y = task; size_t i = row_size * task; float* JXL_RESTRICT row_out = channel->Row(y); - // TODO(deymo): add bits_per_sample == 1 case here. Also maybe - // implement masking if bits_per_sample is not a multiple of 8. if (bits_per_sample <= 8) { LoadFloatRow(row_out, in + i, mul, xsize, bytes_per_pixel); - } else if (bits_per_sample <= 16) { + } else { if (little_endian) { LoadFloatRow(row_out, in + i, mul, xsize, bytes_per_pixel); } else { LoadFloatRow(row_out, in + i, mul, xsize, bytes_per_pixel); } - } else { - if (little_endian) { - LoadFloatRow(row_out, in + i, mul, xsize, - bytes_per_pixel); - } else { - LoadFloatRow(row_out, in + i, mul, xsize, - bytes_per_pixel); - } } }, "ConvertExtraChannelUint")); } @@ -219,30 +197,21 @@ size_t ysize, const ColorEncoding& c_current, size_t channels, bool alpha_is_premultiplied, size_t bits_per_sample, JxlEndianness endianness, bool flipped_y, ThreadPool* pool, ImageBundle* ib, bool float_in, size_t align) { - if (bits_per_sample < 1 || bits_per_sample > 32) { - return JXL_FAILURE("Invalid bits_per_sample value."); - } - // TODO(deymo): Implement 1-bit per sample as 8 samples per byte. In - // any other case we use DivCeil(bits_per_sample, 8) bytes per pixel per - // channel. - if (bits_per_sample == 1) { - return JXL_FAILURE("packed 1-bit per sample is not yet supported"); - } + JXL_CHECK(float_in ? bits_per_sample == 16 || bits_per_sample == 32 + : bits_per_sample > 0 && bits_per_sample <= 16); const size_t color_channels = c_current.Channels(); bool has_alpha = channels == 2 || channels == 4; if (channels < color_channels) { return JXL_FAILURE("Expected %" PRIuS " color channels, received only %" PRIuS " channels", color_channels, channels); } - // bytes_per_channel and bytes_per_pixel are only valid for - // bits_per_sample > 1. const size_t bytes_per_channel = DivCeil(bits_per_sample, jxl::kBitsPerByte); const size_t bytes_per_pixel = channels * bytes_per_channel; if (bits_per_sample > 16 && bits_per_sample < 32) { return JXL_FAILURE("not supported, try bits_per_sample=32"); } @@ -285,11 +254,11 @@ [&](const uint32_t task, size_t /*thread*/) { const size_t y = get_y(task); size_t i = row_size * task + (c * bits_per_sample / jxl::kBitsPerByte); float* JXL_RESTRICT row_out = color.PlaneRow(c, y); - if (bits_per_sample <= 16) { + if (bits_per_sample == 16) { if (little_endian) { for (size_t x = 0; x < xsize; ++x) { row_out[x] = LoadLEFloat16(in + i); i += bytes_per_pixel; } @@ -323,30 +292,20 @@ pool, 0, static_cast(ysize), ThreadPool::NoInit, [&](const uint32_t task, size_t /*thread*/) { const size_t y = get_y(task); size_t i = row_size * task + c * bytes_per_channel; float* JXL_RESTRICT row_out = color.PlaneRow(c, y); - // TODO(deymo): add bits_per_sample == 1 case here. Also maybe - // implement masking if bits_per_sample is not a multiple of 8. if (bits_per_sample <= 8) { LoadFloatRow(row_out, in + i, mul, xsize, bytes_per_pixel); - } else if (bits_per_sample <= 16) { + } else { if (little_endian) { LoadFloatRow(row_out, in + i, mul, xsize, bytes_per_pixel); } else { LoadFloatRow(row_out, in + i, mul, xsize, bytes_per_pixel); } - } else { - if (little_endian) { - LoadFloatRow(row_out, in + i, mul, xsize, - bytes_per_pixel); - } else { - LoadFloatRow(row_out, in + i, mul, xsize, - bytes_per_pixel); - } } }, "ConvertRGBUint")); } } @@ -369,11 +328,11 @@ [&](const uint32_t task, size_t /*thread*/) { const size_t y = get_y(task); size_t i = row_size * task + ((channels - 1) * bits_per_sample / jxl::kBitsPerByte); float* JXL_RESTRICT row_out = alpha.Row(y); - if (bits_per_sample <= 16) { + if (bits_per_sample == 16) { if (little_endian) { for (size_t x = 0; x < xsize; ++x) { row_out[x] = LoadLEFloat16(in + i); i += bytes_per_pixel; } @@ -404,36 +363,32 @@ pool, 0, static_cast(ysize), ThreadPool::NoInit, [&](const uint32_t task, size_t /*thread*/) { const size_t y = get_y(task); size_t i = row_size * task + (channels - 1) * bytes_per_channel; float* JXL_RESTRICT row_out = alpha.Row(y); - // TODO(deymo): add bits_per_sample == 1 case here. Also maybe - // implement masking if bits_per_sample is not a multiple of 8. if (bits_per_sample <= 8) { LoadFloatRow(row_out, in + i, mul, xsize, bytes_per_pixel); - } else if (bits_per_sample <= 16) { + } else { if (little_endian) { LoadFloatRow(row_out, in + i, mul, xsize, bytes_per_pixel); } else { LoadFloatRow(row_out, in + i, mul, xsize, bytes_per_pixel); } - } else { - if (little_endian) { - LoadFloatRow(row_out, in + i, mul, xsize, - bytes_per_pixel); - } else { - LoadFloatRow(row_out, in + i, mul, xsize, - bytes_per_pixel); - } } }, "ConvertAlphaUint")); } ib->SetAlpha(std::move(alpha), alpha_is_premultiplied); + } else if (!has_alpha && ib->HasAlpha()) { + // if alpha is not passed, but it is expected, then assume + // it is all-opaque + ImageF alpha(xsize, ysize); + FillImage(1.0f, &alpha); + ib->SetAlpha(std::move(alpha), alpha_is_premultiplied); } return true; } diff --git a/third_party/jpeg-xl/lib/jxl/enc_file.cc b/third_party/jpeg-xl/lib/jxl/enc_file.cc --- a/third_party/jpeg-xl/lib/jxl/enc_file.cc +++ b/third_party/jpeg-xl/lib/jxl/enc_file.cc @@ -18,10 +18,11 @@ #include "lib/jxl/color_encoding_internal.h" #include "lib/jxl/enc_bit_writer.h" #include "lib/jxl/enc_cache.h" #include "lib/jxl/enc_frame.h" #include "lib/jxl/enc_icc_codec.h" +#include "lib/jxl/exif.h" #include "lib/jxl/frame_header.h" #include "lib/jxl/headers.h" #include "lib/jxl/image_bundle.h" namespace jxl { @@ -62,55 +63,10 @@ /*suitable_for_downsampling_of_at_least=*/2}, {/*num_coefficients=*/8, /*shift=*/0, /*salient_only=*/false, /*suitable_for_downsampling_of_at_least=*/0}, }; -constexpr uint16_t kExifOrientationTag = 274; - -// Parses the Exif data just enough to extract any render-impacting info. -// If the Exif data is invalid or could not be parsed, then it is treated -// as a no-op. -// TODO (jon): tag 1 can be used to represent Adobe RGB 1998 if it has value -// "R03" -// TODO (jon): set intrinsic dimensions according to -// https://discourse.wicg.io/t/proposal-exif-image-resolution-auto-and-from-image/4326/24 -void InterpretExif(const PaddedBytes& exif, CodecMetadata* metadata) { - if (exif.size() < 12) return; // not enough bytes for a valid exif blob - const uint8_t* t = exif.data(); - bool bigendian = false; - if (LoadLE32(t) == 0x2A004D4D) { - bigendian = true; - } else if (LoadLE32(t) != 0x002A4949) { - return; // not a valid tiff header - } - t += 4; - uint32_t offset = (bigendian ? LoadBE32(t) : LoadLE32(t)); - if (exif.size() < 12 + offset + 2 || offset < 8) return; - t += offset - 4; - uint16_t nb_tags = (bigendian ? LoadBE16(t) : LoadLE16(t)); - t += 2; - while (nb_tags > 0) { - if (t + 12 >= exif.data() + exif.size()) return; - uint16_t tag = (bigendian ? LoadBE16(t) : LoadLE16(t)); - t += 2; - uint16_t type = (bigendian ? LoadBE16(t) : LoadLE16(t)); - t += 2; - uint32_t count = (bigendian ? LoadBE32(t) : LoadLE32(t)); - t += 4; - uint16_t value = (bigendian ? LoadBE16(t) : LoadLE16(t)); - t += 4; - if (tag == kExifOrientationTag) { - if (type == 3 && count == 1) { - if (value >= 1 && value <= 8) { - metadata->m.orientation = value; - } - } - } - nb_tags--; - } -} - Status PrepareCodecMetadataFromIO(const CompressParams& cparams, const CodecInOut* io, CodecMetadata* metadata) { *metadata = io->metadata; size_t ups = 1; @@ -119,13 +75,11 @@ JXL_RETURN_IF_ERROR(metadata->size.Set(io->xsize() * ups, io->ysize() * ups)); // Keep ICC profile in lossless modes because a reconstructed profile may be // slightly different (quantization). // Also keep ICC in JPEG reconstruction mode as we need byte-exact profiles. - const bool lossless_modular = - cparams.modular_mode && cparams.quality_pair.first == 100.0f; - if (!lossless_modular && !io->Main().IsJPEG()) { + if (!cparams.IsLossless() && !io->Main().IsJPEG()) { metadata->m.color_encoding.DecideIfWantICC(); } metadata->m.xyb_encoded = cparams.color_transform == ColorTransform::kXYB ? true : false; diff --git a/third_party/jpeg-xl/lib/jxl/enc_frame.cc b/third_party/jpeg-xl/lib/jxl/enc_frame.cc --- a/third_party/jpeg-xl/lib/jxl/enc_frame.cc +++ b/third_party/jpeg-xl/lib/jxl/enc_frame.cc @@ -252,14 +252,13 @@ } } } // Strength of EPF in modular mode. if (frame_header->encoding == FrameEncoding::kModular && - cparams.quality_pair.first < 100) { + !cparams.IsLossless()) { // TODO(veluca): this formula is nonsense. - loop_filter->epf_sigma_for_modular = - 20.0f * (1.0f - cparams.quality_pair.first / 100); + loop_filter->epf_sigma_for_modular = cparams.butteraugli_distance; } if (frame_header->encoding == FrameEncoding::kModular && cparams.lossy_palette) { loop_filter->epf_sigma_for_modular = 1.0f; } @@ -1103,16 +1102,10 @@ if (cparams.butteraugli_distance != 0 && cparams.butteraugli_distance < kMinButteraugliDistance) { return JXL_FAILURE("Butteraugli distance is too low (%f)", cparams.butteraugli_distance); } - if (cparams.butteraugli_distance > 0.9f && cparams.modular_mode == false && - cparams.quality_pair.first == 100) { - // in case the color image is lossy, make the alpha slightly lossy too - cparams.quality_pair.first = - std::max(90.f, 99.f - 0.3f * cparams.butteraugli_distance); - } if (ib.IsJPEG()) { cparams.gaborish = Override::kOff; cparams.epf = 0; cparams.modular_mode = false; @@ -1226,12 +1219,11 @@ // YCbCr) // If encoding a special DC or reference frame, don't do anything: // input is already in XYB. CopyImageTo(ib.color(), &opsin); } - bool lossless = (frame_header->encoding == FrameEncoding::kModular && - cparams.quality_pair.first == 100); + bool lossless = cparams.IsLossless(); if (ib.HasAlpha() && !ib.AlphaIsPremultiplied() && frame_header->frame_type == FrameType::kRegularFrame && !ApplyOverride(cparams.keep_invisible, lossless) && cparams.ec_resampling == cparams.resampling) { // simplify invisible pixels diff --git a/third_party/jpeg-xl/lib/jxl/enc_icc_codec.cc b/third_party/jpeg-xl/lib/jxl/enc_icc_codec.cc --- a/third_party/jpeg-xl/lib/jxl/enc_icc_codec.cc +++ b/third_party/jpeg-xl/lib/jxl/enc_icc_codec.cc @@ -20,33 +20,10 @@ #include "lib/jxl/icc_codec_common.h" namespace jxl { namespace { -bool EncodeVarInt(uint64_t value, size_t output_size, size_t* output_pos, - uint8_t* output) { - // While more than 7 bits of data are left, - // store 7 bits and set the next byte flag - while (value > 127) { - if (*output_pos > output_size) return false; - // |128: Set the next byte flag - output[(*output_pos)++] = ((uint8_t)(value & 127)) | 128; - // Remove the seven bits we just wrote - value >>= 7; - } - if (*output_pos > output_size) return false; - output[(*output_pos)++] = ((uint8_t)value) & 127; - return true; -} - -void EncodeVarInt(uint64_t value, PaddedBytes* data) { - size_t pos = data->size(); - data->resize(data->size() + 9); - JXL_CHECK(EncodeVarInt(value, data->size(), &pos, data->data())); - data->resize(pos); -} - // Unshuffles or de-interleaves bytes, for example with width 2, turns // "AaBbCcDc" into "ABCDabcd", this for example de-interleaves UTF-16 bytes into // first all the high order bytes, then all the low order bytes. // Transposes a matrix of width columns and ceil(size / width) rows. There are // size elements, size may be < width * height, if so the @@ -157,13 +134,13 @@ if (tag == kRtrcTag && pos + 24 < size) { bool ok = true; ok &= DecodeKeyword(icc, size, pos + 0) == kGtrcTag; ok &= DecodeKeyword(icc, size, pos + 12) == kBtrcTag; if (ok) { - for (size_t i = 0; i < 8; i++) { - if (icc[pos - 8 + i] != icc[pos + 4 + i]) ok = false; - if (icc[pos - 8 + i] != icc[pos + 16 + i]) ok = false; + for (size_t kk = 0; kk < 8; kk++) { + if (icc[pos - 8 + kk] != icc[pos + 4 + kk]) ok = false; + if (icc[pos - 8 + kk] != icc[pos + 16 + kk]) ok = false; } } if (ok) { tagcode = kCommandTagTRC; pos += 24; diff --git a/third_party/jpeg-xl/lib/jxl/enc_modular.h b/third_party/jpeg-xl/lib/jxl/enc_modular.h --- a/third_party/jpeg-xl/lib/jxl/enc_modular.h +++ b/third_party/jpeg-xl/lib/jxl/enc_modular.h @@ -78,12 +78,11 @@ std::vector> tokens; EntropyEncodingData code; std::vector context_map; FrameDimensions frame_dim; CompressParams cparams; - float quality = cparams.quality_pair.first; - float cquality = cparams.quality_pair.second; + float quality = 100.f; std::vector tree_splits; std::vector multiplier_info; std::vector> gi_channel; std::vector image_widths; Predictor delta_pred = Predictor::Average4; diff --git a/third_party/jpeg-xl/lib/jxl/enc_modular.cc b/third_party/jpeg-xl/lib/jxl/enc_modular.cc --- a/third_party/jpeg-xl/lib/jxl/enc_modular.cc +++ b/third_party/jpeg-xl/lib/jxl/enc_modular.cc @@ -226,15 +226,22 @@ // convert binary32 float that corresponds to custom [bits]-bit float (with // [exp_bits] exponent bits) to a [bits]-bit integer representation that should // fit in pixel_type Status float_to_int(const float* const row_in, pixel_type* const row_out, size_t xsize, unsigned int bits, unsigned int exp_bits, - bool fp, float factor) { + bool fp, double dfactor) { JXL_ASSERT(sizeof(pixel_type) * 8 >= bits); if (!fp) { - for (size_t x = 0; x < xsize; ++x) { - row_out[x] = row_in[x] * factor + (row_in[x] < 0 ? -0.5f : 0.5f); + if (bits > 22) { + for (size_t x = 0; x < xsize; ++x) { + row_out[x] = row_in[x] * dfactor + (row_in[x] < 0 ? -0.5 : 0.5); + } + } else { + float factor = dfactor; + for (size_t x = 0; x < xsize; ++x) { + row_out[x] = row_in[x] * factor + (row_in[x] < 0 ? -0.5f : 0.5f); + } } return true; } if (bits == 32 && fp) { JXL_ASSERT(exp_bits == 8); @@ -294,12 +301,11 @@ ModularFrameEncoder::ModularFrameEncoder(const FrameHeader& frame_header, const CompressParams& cparams_orig) : frame_dim(frame_header.ToFrameDimensions()), cparams(cparams_orig) { size_t num_streams = ModularStreamId::Num(frame_dim, frame_header.passes.num_passes); - if (cparams.modular_mode && - cparams.quality_pair == std::pair{100.0, 100.0}) { + if (cparams.IsLossless()) { switch (cparams.decoding_speed_tier) { case 0: break; case 1: cparams.options.wp_tree_mode = ModularOptions::TreeMode::kWPOnly; @@ -320,22 +326,21 @@ break; } } } if (cparams.decoding_speed_tier >= 1 && cparams.responsive && - cparams.quality_pair == std::make_pair(100.f, 100.f)) { + cparams.IsLossless()) { cparams.options.tree_kind = ModularOptions::TreeKind::kTrivialTreeNoPredictor; cparams.options.nb_repeats = 0; } stream_images.resize(num_streams); - if (cquality > 100) cquality = quality; // use a sensible default if nothing explicit is specified: // Squeeze for lossy, no squeeze for lossless if (cparams.responsive < 0) { - if (quality == 100) { + if (cparams.IsLossless()) { cparams.responsive = 0; } else { cparams.responsive = 1; } } @@ -394,18 +399,18 @@ if (cparams.options.predictor == static_cast(-1)) { // no explicit predictor(s) given, set a good default if ((cparams.speed_tier <= SpeedTier::kTortoise || cparams.modular_mode == false) && - quality == 100 && cparams.responsive == false) { + cparams.IsLossless() && cparams.responsive == false) { // TODO(veluca): allow all predictors that don't break residual // multipliers in lossy mode. cparams.options.predictor = Predictor::Variable; } else if (cparams.responsive || cparams.lossy_palette) { // zero predictor for Squeeze residues and lossy palette cparams.options.predictor = Predictor::Zero; - } else if (quality < 100) { + } else if (!cparams.IsLossless()) { // If not responsive and lossy. TODO(veluca): use near_lossless instead? cparams.options.predictor = Predictor::Gradient; } else if (cparams.speed_tier < SpeedTier::kFalcon) { // try median and weighted predictor for anything else cparams.options.predictor = Predictor::Best; @@ -418,10 +423,16 @@ } } else { delta_pred = cparams.options.predictor; if (cparams.lossy_palette) cparams.options.predictor = Predictor::Zero; } + if (!cparams.IsLossless()) { + if (cparams.options.predictor == Predictor::Weighted || + cparams.options.predictor == Predictor::Variable || + cparams.options.predictor == Predictor::Best) + cparams.options.predictor = Predictor::Zero; + } tree_splits.push_back(0); if (cparams.modular_mode == false) { cparams.options.fast_decode_multiplier = 1.0f; tree_splits.push_back(ModularStreamId::VarDCTDC(0).ID(frame_dim)); tree_splits.push_back(ModularStreamId::ModularDC(0).ID(frame_dim)); @@ -497,21 +508,33 @@ } else if (metadata.bit_depth.bits_per_sample > 32) { return JXL_FAILURE("bits_per_sample > 32 not supported"); } } + // in the non-float case, there is an implicit 0 sign bit + int max_bitdepth = + do_color ? metadata.bit_depth.bits_per_sample + (fp ? 0 : 1) : 0; Image& gi = stream_images[0]; gi = Image(xsize, ysize, metadata.bit_depth.bits_per_sample, nb_chans); int c = 0; if (cparams.color_transform == ColorTransform::kXYB && cparams.modular_mode == true) { - static const float enc_factors[3] = {32768.0f, 2048.0f, 2048.0f}; + float enc_factors[3] = {32768.0f, 2048.0f, 2048.0f}; + if (cparams.butteraugli_distance > 0 && !cparams.responsive) { + // quantize XYB here and then treat it as a lossless image + enc_factors[0] *= 1.f / (1.f + 23.f * cparams.butteraugli_distance); + enc_factors[1] *= 1.f / (1.f + 14.f * cparams.butteraugli_distance); + enc_factors[2] *= 1.f / (1.f + 14.f * cparams.butteraugli_distance); + cparams.butteraugli_distance = 0; + } if (cparams.manual_xyb_factors.size() == 3) { DequantMatricesSetCustomDC(&enc_state->shared.matrices, cparams.manual_xyb_factors.data()); + // TODO(jon): update max_bitdepth in this case } else { DequantMatricesSetCustomDC(&enc_state->shared.matrices, enc_factors); + max_bitdepth = 12; } } pixel_type maxval = gi.bitdepth < 32 ? (1u << gi.bitdepth) - 1 : 0; if (do_color) { for (; c < 3; c++) { @@ -521,11 +544,11 @@ continue; int c_out = c; // XYB is encoded as YX(B-Y) if (cparams.color_transform == ColorTransform::kXYB && c < 2) c_out = 1 - c_out; - float factor = maxval; + double factor = maxval; if (cparams.color_transform == ColorTransform::kXYB) factor = enc_state->shared.matrices.InvDCQuant(c); if (c == 2 && cparams.color_transform == ColorTransform::kXYB) { JXL_ASSERT(!fp); for (size_t y = 0; y < ysize; ++y) { @@ -533,10 +556,12 @@ pixel_type* const JXL_RESTRICT row_out = gi.channel[c_out].Row(y); pixel_type* const JXL_RESTRICT row_Y = gi.channel[0].Row(y); for (size_t x = 0; x < xsize; ++x) { row_out[x] = row_in[x] * factor + 0.5f; row_out[x] -= row_Y[x]; + // zero the lsb of B + row_out[x] = row_out[x] / 2 * 2; } } } else { int bits = metadata.bit_depth.bits_per_sample; int exp_bits = metadata.bit_depth.exponent_bits_per_sample; @@ -579,11 +604,12 @@ CeilLog2Nonzero(ecups) - CeilLog2Nonzero(frame_header.upsampling); int bits = eci.bit_depth.bits_per_sample; int exp_bits = eci.bit_depth.exponent_bits_per_sample; bool fp = eci.bit_depth.floating_point_sample; - float factor = (fp ? 1 : ((1u << eci.bit_depth.bits_per_sample) - 1)); + double factor = (fp ? 1 : ((1u << eci.bit_depth.bits_per_sample) - 1)); + if (bits + (fp ? 0 : 1) > max_bitdepth) max_bitdepth = bits + (fp ? 0 : 1); std::atomic has_error{false}; JXL_RETURN_IF_ERROR(RunOnPool( pool, 0, gi.channel[c].plane.ysize(), ThreadPool::NoInit, [&](const int task, const int thread) { const size_t y = task; @@ -597,13 +623,20 @@ "float2int")); if (has_error) return JXL_FAILURE("Error in float to integer conversion"); } JXL_ASSERT(c == nb_chans); + int level_max_bitdepth = (cparams.level == 5 ? 16 : 32); + if (max_bitdepth > level_max_bitdepth) + return JXL_FAILURE( + "Bitdepth too high for level %i (need %i bits, have only %i in this " + "level)", + cparams.level, max_bitdepth, level_max_bitdepth); + // Set options and apply transformations - if (quality < 100) { + if (cparams.butteraugli_distance > 0) { if (cparams.palette_colors != 0) { JXL_DEBUG_V(3, "Lossy encode, not doing palette transforms"); } if (cparams.color_transform == ColorTransform::kXYB) { cparams.channel_colors_pre_transform_percent = 0; @@ -702,90 +735,85 @@ do_transform(gi, maybe_palette_3, weighted::Header(), pool, cparams.options.zero_tokens); } } + // don't do an RCT if we're short on bits if (cparams.color_transform == ColorTransform::kNone && do_color && !fp && - gi.channel.size() - gi.nb_meta_channels >= 3) { + gi.channel.size() - gi.nb_meta_channels >= 3 && + max_bitdepth + 1 < level_max_bitdepth) { if (cparams.colorspace == 1 || (cparams.colorspace < 0 && - (quality < 100 || cparams.speed_tier > SpeedTier::kHare))) { + (!cparams.IsLossless() || cparams.speed_tier > SpeedTier::kHare))) { Transform ycocg{TransformId::kRCT}; ycocg.rct_type = 6; ycocg.begin_c = gi.nb_meta_channels; do_transform(gi, ycocg, weighted::Header(), pool); + max_bitdepth++; } else if (cparams.colorspace >= 2) { Transform sg(TransformId::kRCT); sg.begin_c = gi.nb_meta_channels; sg.rct_type = cparams.colorspace - 2; do_transform(gi, sg, weighted::Header(), pool); + max_bitdepth++; } } - if (cparams.responsive && !gi.channel.empty()) { + // don't do squeeze if we don't have some spare bits + if (cparams.responsive && !gi.channel.empty() && + max_bitdepth + 2 < level_max_bitdepth) { Transform t(TransformId::kSqueeze); t.squeezes = cparams.squeezes; do_transform(gi, t, weighted::Header(), pool); + max_bitdepth += 2; } + if (max_bitdepth + 1 > level_max_bitdepth) { + // force no group RCTs if we don't have a spare bit + cparams.colorspace = 0; + } + JXL_ASSERT(max_bitdepth <= level_max_bitdepth); + std::vector quants; - if (quality < 100 || cquality < 100) { + if (cparams.butteraugli_distance > 0) { quants.resize(gi.channel.size(), 1); - JXL_DEBUG_V( - 2, - "Adding quantization constants corresponding to luma quality %.2f " - "and chroma quality %.2f", - quality, cquality); + float quality = 0.25f * cparams.butteraugli_distance; + JXL_DEBUG_V(2, + "Adding quantization constants corresponding to distance %.3f ", + quality); if (!cparams.responsive) { JXL_DEBUG_V(1, "Warning: lossy compression without Squeeze " "transform is just color quantization."); - quality = (400 + quality) / 5; - cquality = (400 + cquality) / 5; - } - - // convert 'quality' to quantization scaling factor - if (quality > 50) { - quality = 200.0 - quality * 2.0; - } else { - quality = 900.0 - quality * 16.0; - } - if (cquality > 50) { - cquality = 200.0 - cquality * 2.0; - } else { - cquality = 900.0 - cquality * 16.0; + quality *= 0.1f; } if (cparams.color_transform != ColorTransform::kXYB) { - quality *= 0.01f * maxval / 255.f; - cquality *= 0.01f * maxval / 255.f; - } else { - quality *= 0.01f; - cquality *= 0.01f; + quality *= maxval / 255.f; } - if (cparams.options.nb_repeats == 0) { return JXL_FAILURE("nb_repeats = 0 not supported with modular lossy!"); } for (uint32_t i = gi.nb_meta_channels; i < gi.channel.size(); i++) { Channel& ch = gi.channel[i]; int shift = ch.hshift + ch.vshift; // number of pixel halvings if (shift > 16) shift = 16; if (shift > 0) shift--; int q; // assuming default Squeeze here - int component = ((i - gi.nb_meta_channels) % nb_chans); + int component = + (do_color ? 0 : 3) + ((i - gi.nb_meta_channels) % nb_chans); // last 4 channels are final chroma residuals if (nb_chans > 2 && i >= gi.channel.size() - 4 && cparams.responsive) { component = 1; } if (cparams.color_transform == ColorTransform::kXYB && component < 3) { - q = (component == 0 ? quality : cquality) * squeeze_quality_factor_xyb * + q = quality * squeeze_quality_factor_xyb * squeeze_xyb_qtable[component][shift]; } else { if (cparams.colorspace != 0 && component > 0 && component < 3) { - q = cquality * squeeze_quality_factor * squeeze_chroma_qtable[shift]; + q = quality * squeeze_quality_factor * squeeze_chroma_qtable[shift]; } else { q = quality * squeeze_quality_factor * squeeze_luma_factor * squeeze_luma_qtable[shift]; } } @@ -1286,16 +1314,14 @@ } if (gi.channel.empty()) return true; // Do some per-group transforms - float quality = cparams.quality_pair.first; - // Local palette // TODO(veluca): make this work with quantize-after-prediction in lossy // mode. - if (quality == 100 && cparams.palette_colors != 0 && + if (cparams.butteraugli_distance == 0.f && cparams.palette_colors != 0 && cparams.speed_tier < SpeedTier::kCheetah) { // all-channel palette (e.g. RGBA) if (gi.channel.size() - gi.nb_meta_channels > 1) { Transform maybe_palette(TransformId::kPalette); maybe_palette.begin_c = gi.nb_meta_channels; @@ -1319,12 +1345,13 @@ do_transform(gi, maybe_palette_3, weighted::Header()); } } // Local channel palette - if (cparams.channel_colors_percent > 0 && quality == 100 && - !cparams.lossy_palette && cparams.speed_tier < SpeedTier::kCheetah && + if (cparams.channel_colors_percent > 0 && + cparams.butteraugli_distance == 0.f && !cparams.lossy_palette && + cparams.speed_tier < SpeedTier::kCheetah && !(cparams.responsive && cparams.decoding_speed_tier >= 1)) { // single channel palette (like FLIF's ChannelCompact) size_t nb_channels = gi.channel.size() - gi.nb_meta_channels; for (size_t i = 0; i < nb_channels; i++) { int min, max; @@ -1346,12 +1373,13 @@ } } // lossless and no specific color transform specified: try Nothing, YCoCg, // and 17 RCTs - if (cparams.color_transform == ColorTransform::kNone && quality == 100 && - cparams.colorspace < 0 && gi.channel.size() - gi.nb_meta_channels >= 3 && + if (cparams.color_transform == ColorTransform::kNone && + cparams.IsLossless() && cparams.colorspace < 0 && + gi.channel.size() - gi.nb_meta_channels >= 3 && cparams.responsive == false && do_color && cparams.speed_tier <= SpeedTier::kHare) { Transform sg(TransformId::kRCT); sg.begin_c = gi.nb_meta_channels; size_t nb_rcts_to_try = 0; diff --git a/third_party/jpeg-xl/lib/jxl/enc_params.h b/third_party/jpeg-xl/lib/jxl/enc_params.h --- a/third_party/jpeg-xl/lib/jxl/enc_params.h +++ b/third_party/jpeg-xl/lib/jxl/enc_params.h @@ -125,10 +125,11 @@ // Try to achieve a maximum pixel-by-pixel error on each channel. bool max_error_mode = false; float max_error[3] = {0.0, 0.0, 0.0}; SpeedTier speed_tier = SpeedTier::kSquirrel; + int brotli_effort = -1; // 0 = default. // 1 = slightly worse quality. // 4 = fastest speed, lowest quality // TODO(veluca): hook this up to the C API. @@ -216,32 +217,30 @@ // modular mode options below ModularOptions options; int responsive = -1; // empty for default squeeze std::vector squeezes; - // A pair of . - std::pair quality_pair{100.f, 100.f}; int colorspace = -1; // Use Global channel palette if #colors < this percentage of range float channel_colors_pre_transform_percent = 95.f; // Use Local channel palette if #colors < this percentage of range float channel_colors_percent = 80.f; int palette_colors = 1 << 10; // up to 10-bit palette is probably worthwhile bool lossy_palette = false; // Returns whether these params are lossless as defined by SetLossless(); bool IsLossless() const { - return modular_mode && quality_pair.first == 100 && - quality_pair.second == 100 && - color_transform == jxl::ColorTransform::kNone; + // YCbCr is also considered lossless here since it's intended for + // source material that is already YCbCr (we don't do the fwd transform) + return modular_mode && butteraugli_distance == 0.0f && + color_transform != jxl::ColorTransform::kXYB; } // Sets the parameters required to make the codec lossless. void SetLossless() { modular_mode = true; - quality_pair.first = 100; - quality_pair.second = 100; + butteraugli_distance = 0.0f; color_transform = jxl::ColorTransform::kNone; } bool use_new_heuristics = false; @@ -253,10 +252,14 @@ int resampling = -1; int ec_resampling = -1; // Skip the downsampling before encoding if this is true. bool already_downsampled = false; + // Codestream level to conform to. + // -1: don't care + int level = -1; + std::vector manual_noise; std::vector manual_xyb_factors; }; static constexpr float kMinButteraugliForDynamicAR = 0.5f; diff --git a/third_party/jpeg-xl/lib/jxl/enc_patch_dictionary.cc b/third_party/jpeg-xl/lib/jxl/enc_patch_dictionary.cc --- a/third_party/jpeg-xl/lib/jxl/enc_patch_dictionary.cc +++ b/third_party/jpeg-xl/lib/jxl/enc_patch_dictionary.cc @@ -708,15 +708,10 @@ } CompressParams cparams = state->cparams; // Recursive application of patches could create very weird issues. cparams.patches = Override::kOff; - // TODO(veluca): possibly change heuristics here. - if (!cparams.modular_mode) { - cparams.quality_pair.first = cparams.quality_pair.second = - 90.f - cparams.butteraugli_distance * 5.f; - } RoundtripPatchFrame(&reference_frame, state, 0, cparams, cms, pool, true); // TODO(veluca): this assumes that applying patches is commutative, which is // not true for all blending modes. This code only produces kAdd patches, so diff --git a/third_party/jpeg-xl/lib/jxl/encode.cc b/third_party/jpeg-xl/lib/jxl/encode.cc --- a/third_party/jpeg-xl/lib/jxl/encode.cc +++ b/third_party/jpeg-xl/lib/jxl/encode.cc @@ -144,10 +144,14 @@ return -1; } // Level 5 checks + if (!m.modular_16_bit_buffer_sufficient) { + if (debug_string) *debug_string = "Too high modular bit depth"; + return 10; + } if (xsize > (1ull << 18ull) || ysize > (1ull << 18ull) || xsize * ysize > (1ull << 28ull)) { if (debug_string) *debug_string = "Too large image dimensions"; return 10; } @@ -246,10 +250,16 @@ } } // TODO(lode): preview should be added here if a preview image is added writer.ZeroPadToByte(); + + // Not actually the end of frame, but the end of metadata/ICC, but helps + // the next frame to start here for indexing purposes. + codestream_bytes_written_end_of_frame += + jxl::DivCeil(writer.BitsWritten(), 8); + bytes = std::move(writer).TakeBytes(); if (MustUseContainer()) { // Add "JXL " and ftyp box. output_byte_queue.insert( @@ -361,39 +371,45 @@ input_frame->option_values.header.layer_info.blend_info.clamp; frame_info.alpha_channel = input_frame->option_values.header.layer_info.blend_info.alpha; frame_info.extra_channel_blending_info.resize( metadata.m.num_extra_channels); - // If extra channel blend info has not been set, use the default values. - JxlBlendInfo default_blend_info; - JxlEncoderInitBlendInfo(&default_blend_info); + // If extra channel blend info has not been set, use the blend mode from the + // layer_info. + JxlBlendInfo default_blend_info = + input_frame->option_values.header.layer_info.blend_info; for (size_t i = 0; i < metadata.m.num_extra_channels; ++i) { auto& to = frame_info.extra_channel_blending_info[i]; const auto& from = i < input_frame->option_values.extra_channel_blend_info.size() ? input_frame->option_values.extra_channel_blend_info[i] : default_blend_info; to.mode = static_cast(from.blendmode); to.source = from.source; to.alpha_channel = from.alpha; + to.clamp = (from.clamp != 0); } if (input_frame->option_values.header.layer_info.have_crop) { ib.origin.x0 = input_frame->option_values.header.layer_info.crop_x0; ib.origin.y0 = input_frame->option_values.header.layer_info.crop_y0; } + JXL_ASSERT(writer.BitsWritten() == 0); if (!jxl::EncodeFrame(input_frame->option_values.cparams, frame_info, &metadata, input_frame->frame, &enc_state, cms, thread_pool.get(), &writer, /*aux_out=*/nullptr)) { return JXL_API_ERROR("Failed to encode frame"); } + codestream_bytes_written_beginning_of_frame = + codestream_bytes_written_end_of_frame; + codestream_bytes_written_end_of_frame += + jxl::DivCeil(writer.BitsWritten(), 8); // Possibly bytes already contains the codestream header: in case this is // the first frame, and the codestream header was not encoded as jxlp above. bytes.append(std::move(writer).TakeBytes()); - if (MustUseContainer()) { if (last_frame && jxlp_counter == 0) { // If this is the last frame and no jxlp boxes were used yet, it's // slighly more efficient to write a jxlc box since it has 4 bytes less // overhead. @@ -421,13 +437,14 @@ jxl::PaddedBytes compressed(4); // Prepend the original box type in the brob box contents for (size_t i = 0; i < 4; i++) { compressed[i] = static_cast(box->type[i]); } - if (JXL_ENC_SUCCESS != BrotliCompress(9, box->contents.data(), - box->contents.size(), - &compressed)) { + if (JXL_ENC_SUCCESS != + BrotliCompress((brotli_effort >= 0 ? brotli_effort : 4), + box->contents.data(), box->contents.size(), + &compressed)) { return JXL_API_ERROR("Brotli compression for brob box failed"); } jxl::AppendBoxHeader(jxl::MakeBoxType("brob"), compressed.size(), false, &output_byte_queue); output_byte_queue.insert(output_byte_queue.end(), compressed.data(), @@ -559,11 +576,12 @@ enc->metadata.m.bit_depth.exponent_bits_per_sample = info->exponent_bits_per_sample; enc->metadata.m.bit_depth.floating_point_sample = (info->exponent_bits_per_sample != 0u); enc->metadata.m.modular_16_bit_buffer_sufficient = - (info->exponent_bits_per_sample == 0u) && info->bits_per_sample <= 12; + (!info->uses_original_profile || info->bits_per_sample <= 12) && + info->alpha_bits <= 12; // The number of extra channels includes the alpha channel, so for example and // RGBA with no other extra channels, has exactly num_extra_channels == 1 enc->metadata.m.num_extra_channels = info->num_extra_channels; enc->metadata.m.extra_channel_info.resize(enc->metadata.m.num_extra_channels); @@ -618,11 +636,19 @@ enc->metadata.m.animation.tps_numerator = info->animation.tps_numerator; enc->metadata.m.animation.tps_denominator = info->animation.tps_denominator; enc->metadata.m.animation.num_loops = info->animation.num_loops; enc->metadata.m.animation.have_timecodes = info->animation.have_timecodes; } - + std::string level_message; + int required_level = VerifyLevelSettings(enc, &level_message); + if (required_level == -1 || + static_cast(enc->codestream_level) < required_level) { + return JXL_API_ERROR("%s", ("Codestream level verification for level " + + std::to_string(enc->codestream_level) + + " failed: " + level_message) + .c_str()); + } return JXL_ENC_SUCCESS; } void JxlEncoderInitExtraChannelInfo(JxlExtraChannelType type, JxlExtraChannelInfo* info) { @@ -650,20 +676,31 @@ } jxl::ExtraChannelInfo& channel = enc->metadata.m.extra_channel_info[index]; channel.type = static_cast(info->type); channel.bit_depth.bits_per_sample = info->bits_per_sample; + enc->metadata.m.modular_16_bit_buffer_sufficient &= + info->bits_per_sample <= 12; channel.bit_depth.exponent_bits_per_sample = info->exponent_bits_per_sample; channel.bit_depth.floating_point_sample = info->exponent_bits_per_sample != 0; channel.dim_shift = info->dim_shift; channel.name = ""; channel.alpha_associated = (info->alpha_premultiplied != 0); channel.cfa_channel = info->cfa_channel; channel.spot_color[0] = info->spot_color[0]; channel.spot_color[1] = info->spot_color[1]; channel.spot_color[2] = info->spot_color[2]; channel.spot_color[3] = info->spot_color[3]; + std::string level_message; + int required_level = VerifyLevelSettings(enc, &level_message); + if (required_level == -1 || + static_cast(enc->codestream_level) < required_level) { + return JXL_API_ERROR("%s", ("Codestream level verification for level " + + std::to_string(enc->codestream_level) + + " failed: " + level_message) + .c_str()); + } return JXL_ENC_SUCCESS; } JXL_EXPORT JxlEncoderStatus JxlEncoderSetExtraChannelName(JxlEncoder* enc, size_t index, @@ -686,10 +723,11 @@ if (source != nullptr) { opts->values = source->values; } else { opts->values.lossless = false; } + opts->values.cparams.level = enc->codestream_level; JxlEncoderFrameSettings* ret = opts.get(); enc->encoder_options.emplace_back(std::move(opts)); return ret; } @@ -699,10 +737,14 @@ return JxlEncoderFrameSettingsCreate(enc, source); } JxlEncoderStatus JxlEncoderSetFrameLossless( JxlEncoderFrameSettings* frame_settings, const JXL_BOOL lossless) { + if (lossless && frame_settings->enc->basic_info_set && + frame_settings->enc->metadata.m.xyb_encoded) { + return JXL_API_ERROR("Set use_original_profile=true for lossless encoding"); + } frame_settings->values.lossless = lossless; return JXL_ENC_SUCCESS; } JxlEncoderStatus JxlEncoderOptionsSetLossless( @@ -724,33 +766,10 @@ } if (distance > 0.f && distance < 0.01f) { distance = 0.01f; } frame_settings->values.cparams.butteraugli_distance = distance; - float jpeg_quality; - // Formula to translate butteraugli distance roughly into JPEG 0-100 quality. - // This is the inverse of the formula in cjxl.cc to translate JPEG quality - // into butteraugli distance. - if (distance > 6.56f) { - jpeg_quality = -5.456783f * std::log(0.0256f * distance - 0.16384f); - } else { - jpeg_quality = -11.11111f * distance + 101.11111f; - } - // Translate JPEG quality into the quality_pair setting for modular encoding. - // This is the formula also used in cjxl.cc to convert the command line JPEG - // quality parameter to the quality_pair setting. - // TODO(lode): combine the distance -> quality_pair conversion into a single - // formula, possibly altering it to a more suitable heuristic. - float quality; - if (jpeg_quality < 7.f) { - quality = std::min(35.f + (jpeg_quality - 7.f) * 3.0f, 100.0f); - } else { - quality = - std::min(35.f + (jpeg_quality - 7.f) * 65.f / 93.f, 100.0f); - } - frame_settings->values.cparams.quality_pair.first = - frame_settings->values.cparams.quality_pair.second = quality; return JXL_ENC_SUCCESS; } JxlEncoderStatus JxlEncoderOptionsSetDistance( JxlEncoderFrameSettings* frame_settings, float distance) { @@ -773,10 +792,19 @@ return JXL_ENC_ERROR; } frame_settings->values.cparams.speed_tier = static_cast(10 - value); return JXL_ENC_SUCCESS; + case JXL_ENC_FRAME_SETTING_BROTLI_EFFORT: + if (value < -1 || value > 11) { + return JXL_ENC_ERROR; + } + // set cparams for brotli use in JPEG frames + frame_settings->values.cparams.brotli_effort = value; + // set enc option for brotli use in brob boxes + frame_settings->enc->brotli_effort = value; + return JXL_ENC_SUCCESS; case JXL_ENC_FRAME_SETTING_DECODING_SPEED: if (value < 0 || value > 4) { return JXL_ENC_ERROR; } frame_settings->values.cparams.decoding_speed_tier = value; @@ -1006,11 +1034,12 @@ enc->input_queue.clear(); enc->num_queued_frames = 0; enc->num_queued_boxes = 0; enc->encoder_options.clear(); enc->output_byte_queue.clear(); - enc->output_bytes_flushed = 0; + enc->codestream_bytes_written_beginning_of_frame = 0; + enc->codestream_bytes_written_end_of_frame = 0; enc->wrote_bytes = false; enc->jxlp_counter = 0; enc->metadata = jxl::CodecMetadata(); enc->last_used_cparams = jxl::CompressParams(); enc->frames_closed = false; @@ -1023,13 +1052,14 @@ enc->codestream_level = 5; } void JxlEncoderDestroy(JxlEncoder* enc) { if (enc) { + JxlMemoryManager local_memory_manager = enc->memory_manager; // Call destructor directly since custom free function is used. enc->~JxlEncoder(); - jxl::MemoryManagerFree(&enc->memory_manager, enc); + jxl::MemoryManagerFree(&local_memory_manager, enc); } } JxlEncoderStatus JxlEncoderUseContainer(JxlEncoder* enc, JXL_BOOL use_container) { @@ -1060,11 +1090,14 @@ int JxlEncoderGetRequiredCodestreamLevel(const JxlEncoder* enc) { return VerifyLevelSettings(enc, nullptr); } -void JxlEncoderSetCms(JxlEncoder* enc, JxlCmsInterface cms) { enc->cms = cms; } +void JxlEncoderSetCms(JxlEncoder* enc, JxlCmsInterface cms) { + jxl::msan::MemoryIsInitialized(&cms, sizeof(cms)); + enc->cms = cms; +} JxlEncoderStatus JxlEncoderSetParallelRunner(JxlEncoder* enc, JxlParallelRunner parallel_runner, void* parallel_runner_opaque) { if (enc->thread_pool) return JXL_API_ERROR("parallel runner already set"); @@ -1136,11 +1169,12 @@ } if (frame_settings->enc->store_jpeg_metadata) { jxl::jpeg::JPEGData data_in = *io.Main().jpeg_data; jxl::PaddedBytes jpeg_data; - if (!jxl::jpeg::EncodeJPEGData(data_in, &jpeg_data)) { + if (!jxl::jpeg::EncodeJPEGData(data_in, &jpeg_data, + frame_settings->values.cparams)) { return JXL_ENC_ERROR; } frame_settings->enc->jpeg_metadata = std::vector( jpeg_data.data(), jpeg_data.data() + jpeg_data.size()); } @@ -1245,19 +1279,35 @@ has_interleaved_alpha = 0; // only first Alpha is initialized } else { queued_frame->ec_initialized.push_back(0); } } + queued_frame->frame.origin.x0 = + frame_settings->values.header.layer_info.crop_x0; + queued_frame->frame.origin.y0 = + frame_settings->values.header.layer_info.crop_y0; + queued_frame->frame.use_for_next_frame = + (frame_settings->values.header.layer_info.save_as_reference != 0u); + queued_frame->frame.blendmode = + frame_settings->values.header.layer_info.blend_info.blendmode == + JXL_BLEND_REPLACE + ? jxl::BlendMode::kReplace + : jxl::BlendMode::kBlend; + queued_frame->frame.blend = + frame_settings->values.header.layer_info.blend_info.source > 0; if (!jxl::BufferToImageBundle(*pixel_format, xsize, ysize, buffer, size, frame_settings->enc->thread_pool.get(), c_current, &(queued_frame->frame))) { return JXL_ENC_ERROR; } - if (frame_settings->values.lossless) { - queued_frame->option_values.cparams.SetLossless(); + if (frame_settings->values.lossless && + frame_settings->enc->metadata.m.xyb_encoded) { + return JXL_API_ERROR("Set use_original_profile=true for lossless encoding"); } + queued_frame->option_values.cparams.level = + frame_settings->enc->codestream_level; QueueFrame(frame_settings, queued_frame); return JXL_ENC_SUCCESS; } @@ -1348,11 +1398,10 @@ if (!enc->output_byte_queue.empty()) { size_t to_copy = std::min(*avail_out, enc->output_byte_queue.size()); std::copy_n(enc->output_byte_queue.begin(), to_copy, *next_out); *next_out += to_copy; *avail_out -= to_copy; - enc->output_bytes_flushed += to_copy; enc->output_byte_queue.erase(enc->output_byte_queue.begin(), enc->output_byte_queue.begin() + to_copy); } else if (!enc->input_queue.empty()) { if (enc->RefillOutputByteQueue() != JXL_ENC_SUCCESS) { return JXL_ENC_ERROR; @@ -1404,11 +1453,11 @@ return JXL_ENC_SUCCESS; } JxlEncoderStatus JxlEncoderSetFrameName(JxlEncoderFrameSettings* frame_settings, const char* frame_name) { - std::string str = frame_name; + std::string str = frame_name ? frame_name : ""; if (str.size() > 1071) { return JXL_API_ERROR("frame name can be max 1071 bytes long"); } frame_settings->values.frame_name = str; frame_settings->values.header.name_length = str.size(); diff --git a/third_party/jpeg-xl/lib/jxl/encode_internal.h b/third_party/jpeg-xl/lib/jxl/encode_internal.h --- a/third_party/jpeg-xl/lib/jxl/encode_internal.h +++ b/third_party/jpeg-xl/lib/jxl/encode_internal.h @@ -115,16 +115,19 @@ size_t num_queued_frames; size_t num_queued_boxes; std::vector input_queue; std::deque output_byte_queue; - size_t output_bytes_flushed; - // Get the current write position in the stream (for indexing use). - size_t BytePosition() const { - return output_bytes_flushed + output_byte_queue.size(); - } + // How many codestream bytes have been written, i.e., + // content of jxlc and jxlp boxes. Frame index box jxli + // requires position indices to point to codestream bytes, + // so we need to keep track of the total of flushed or queue + // codestream bytes. These bytes may be in a single jxlc box + // or accross multiple jxlp boxes. + size_t codestream_bytes_written_beginning_of_frame; + size_t codestream_bytes_written_end_of_frame; // Force using the container even if not needed bool use_container; // User declared they will add metadata boxes bool use_boxes; @@ -151,10 +154,11 @@ bool frames_closed; bool boxes_closed; bool basic_info_set; bool color_encoding_set; bool intensity_target_set; + int brotli_effort = -1; // Takes the first frame in the input_queue, encodes it, and appends // the bytes to the output_byte_queue. JxlEncoderStatus RefillOutputByteQueue(); diff --git a/third_party/jpeg-xl/lib/jxl/encode_test.cc b/third_party/jpeg-xl/lib/jxl/encode_test.cc --- a/third_party/jpeg-xl/lib/jxl/encode_test.cc +++ b/third_party/jpeg-xl/lib/jxl/encode_test.cc @@ -36,10 +36,11 @@ JxlBasicInfo basic_info; jxl::test::JxlBasicInfoSetFromPixelFormat(&basic_info, &pixel_format); basic_info.xsize = xsize; basic_info.ysize = ysize; basic_info.uses_original_profile = false; + EXPECT_EQ(JXL_ENC_SUCCESS, JxlEncoderSetCodestreamLevel(enc.get(), 10)); EXPECT_EQ(JXL_ENC_SUCCESS, JxlEncoderSetBasicInfo(enc.get(), &basic_info)); JxlColorEncoding color_encoding; JxlColorEncodingSetToSRGB(&color_encoding, /*is_gray=*/pixel_format.num_channels < 3); EXPECT_EQ(JXL_ENC_SUCCESS, @@ -56,11 +57,11 @@ EXPECT_NE(nullptr, enc.get()); JxlEncoderCloseInput(enc.get()); const std::string jpeg_path = - "imagecompression.info/flower_foveon.png.im_q85_420.jpg"; + "third_party/imagecompression.info/flower_foveon.png.im_q85_420.jpg"; const jxl::PaddedBytes orig = jxl::ReadTestData(jpeg_path); JxlEncoderFrameSettings* frame_settings = JxlEncoderFrameSettingsCreate(enc.get(), NULL); @@ -83,10 +84,11 @@ JxlBasicInfo basic_info; jxl::test::JxlBasicInfoSetFromPixelFormat(&basic_info, &pixel_format); basic_info.xsize = xsize; basic_info.ysize = ysize; basic_info.uses_original_profile = true; + EXPECT_EQ(JXL_ENC_SUCCESS, JxlEncoderSetCodestreamLevel(enc.get(), 10)); EXPECT_EQ(JXL_ENC_SUCCESS, JxlEncoderSetBasicInfo(enc.get(), &basic_info)); JxlEncoderFrameSettings* frame_settings = JxlEncoderFrameSettingsCreate(enc.get(), NULL); EXPECT_EQ(JXL_ENC_ERROR, JxlEncoderAddImageFrame(frame_settings, &pixel_format, @@ -171,10 +173,12 @@ if (frame_settings->values.lossless) { basic_info.uses_original_profile = true; } else { basic_info.uses_original_profile = false; } + // 16-bit alpha means this requires level 10 + EXPECT_EQ(JXL_ENC_SUCCESS, JxlEncoderSetCodestreamLevel(enc, 10)); EXPECT_EQ(JXL_ENC_SUCCESS, JxlEncoderSetBasicInfo(enc, &basic_info)); JxlColorEncoding color_encoding; JxlColorEncodingSetToSRGB(&color_encoding, /*is_gray=*/pixel_format.num_channels < 3); EXPECT_EQ(JXL_ENC_SUCCESS, JxlEncoderSetColorEncoding(enc, &color_encoding)); @@ -196,11 +200,10 @@ avail_out = compressed.size() - offset; } } compressed.resize(next_out - compressed.data()); EXPECT_EQ(JXL_ENC_SUCCESS, process_result); - jxl::DecompressParams dparams; jxl::CodecInOut decoded_io; EXPECT_TRUE(jxl::DecodeFile( dparams, jxl::Span(compressed.data(), compressed.size()), &decoded_io, /*pool=*/nullptr)); @@ -628,10 +631,11 @@ JxlBasicInfo basic_info; jxl::test::JxlBasicInfoSetFromPixelFormat(&basic_info, &pixel_format); basic_info.xsize = xsize; basic_info.ysize = ysize; basic_info.uses_original_profile = false; + EXPECT_EQ(JXL_ENC_SUCCESS, JxlEncoderSetCodestreamLevel(enc.get(), 10)); EXPECT_EQ(JXL_ENC_SUCCESS, JxlEncoderSetBasicInfo(enc.get(), &basic_info)); JxlColorEncoding color_encoding; JxlColorEncodingSetToSRGB(&color_encoding, /*is_gray=*/false); EXPECT_EQ(JXL_ENC_SUCCESS, @@ -737,11 +741,11 @@ EXPECT_EQ(0u, encoded_span.size()); EXPECT_EQ(0, memcmp("jxll", container.boxes[0].type, 4)); } TEST(EncodeTest, CodestreamLevelVerificationTest) { - JxlPixelFormat pixel_format = {4, JXL_TYPE_UINT16, JXL_BIG_ENDIAN, 0}; + JxlPixelFormat pixel_format = {4, JXL_TYPE_UINT8, JXL_BIG_ENDIAN, 0}; JxlBasicInfo basic_info; jxl::test::JxlBasicInfoSetFromPixelFormat(&basic_info, &pixel_format); basic_info.xsize = 64; basic_info.ysize = 64; @@ -753,25 +757,24 @@ EXPECT_EQ(5, JxlEncoderGetRequiredCodestreamLevel(enc.get())); // Set an image dimension that is too large for level 5, but fits in level 10 basic_info.xsize = 1ull << 30ull; + EXPECT_EQ(JXL_ENC_ERROR, JxlEncoderSetBasicInfo(enc.get(), &basic_info)); + EXPECT_EQ(JXL_ENC_SUCCESS, JxlEncoderSetCodestreamLevel(enc.get(), 10)); EXPECT_EQ(JXL_ENC_SUCCESS, JxlEncoderSetBasicInfo(enc.get(), &basic_info)); - EXPECT_EQ(10, JxlEncoderGetRequiredCodestreamLevel(enc.get())); // Set an image dimension that is too large even for level 10 basic_info.xsize = 1ull << 31ull; - EXPECT_EQ(JXL_ENC_SUCCESS, JxlEncoderSetBasicInfo(enc.get(), &basic_info)); - - EXPECT_EQ(-1, JxlEncoderGetRequiredCodestreamLevel(enc.get())); + EXPECT_EQ(JXL_ENC_ERROR, JxlEncoderSetBasicInfo(enc.get(), &basic_info)); } TEST(EncodeTest, JXL_TRANSCODE_JPEG_TEST(JPEGReconstructionTest)) { const std::string jpeg_path = - "imagecompression.info/flower_foveon.png.im_q85_420.jpg"; + "third_party/imagecompression.info/flower_foveon.png.im_q85_420.jpg"; const jxl::PaddedBytes orig = jxl::ReadTestData(jpeg_path); JxlEncoderPtr enc = JxlEncoderMake(nullptr); JxlEncoderFrameSettings* frame_settings = JxlEncoderFrameSettingsCreate(enc.get(), NULL); @@ -898,10 +901,11 @@ basic_info.orientation = JXL_ORIENT_ROTATE_90_CW; basic_info.animation.tps_numerator = 55; basic_info.animation.tps_denominator = 77; basic_info.animation.num_loops = 10; basic_info.animation.have_timecodes = JXL_TRUE; + EXPECT_EQ(JXL_ENC_SUCCESS, JxlEncoderSetCodestreamLevel(enc.get(), 10)); EXPECT_EQ(JXL_ENC_SUCCESS, JxlEncoderSetBasicInfo(enc.get(), &basic_info)); JxlColorEncoding color_encoding; JxlColorEncodingSetToSRGB(&color_encoding, /*is_gray=*/false); EXPECT_EQ(JXL_ENC_SUCCESS, JxlEncoderSetColorEncoding(enc.get(), &color_encoding)); @@ -999,10 +1003,11 @@ basic_info.ysize = ysize; basic_info.have_animation = true; basic_info.animation.tps_numerator = 1000; basic_info.animation.tps_denominator = 1; basic_info.animation.have_timecodes = JXL_TRUE; + EXPECT_EQ(JXL_ENC_SUCCESS, JxlEncoderSetCodestreamLevel(enc.get(), 10)); EXPECT_EQ(JXL_ENC_SUCCESS, JxlEncoderSetBasicInfo(enc.get(), &basic_info)); JxlColorEncoding color_encoding; JxlColorEncodingSetToSRGB(&color_encoding, /*is_gray=*/false); EXPECT_EQ(JXL_ENC_SUCCESS, JxlEncoderSetColorEncoding(enc.get(), &color_encoding)); @@ -1100,10 +1105,11 @@ jxl::test::JxlBasicInfoSetFromPixelFormat(&basic_info, &pixel_format); // Encoding a 300x300 frame in an image that is only 100x100 basic_info.xsize = 100; basic_info.ysize = 100; basic_info.uses_original_profile = JXL_TRUE; + EXPECT_EQ(JXL_ENC_SUCCESS, JxlEncoderSetCodestreamLevel(enc.get(), 10)); EXPECT_EQ(JXL_ENC_SUCCESS, JxlEncoderSetBasicInfo(enc.get(), &basic_info)); JxlColorEncoding color_encoding; JxlColorEncodingSetToSRGB(&color_encoding, /*is_gray=*/false); EXPECT_EQ(JXL_ENC_SUCCESS, JxlEncoderSetColorEncoding(enc.get(), &color_encoding)); @@ -1192,10 +1198,11 @@ JxlBasicInfo basic_info; jxl::test::JxlBasicInfoSetFromPixelFormat(&basic_info, &pixel_format); basic_info.xsize = xsize; basic_info.ysize = ysize; basic_info.uses_original_profile = false; + EXPECT_EQ(JXL_ENC_SUCCESS, JxlEncoderSetCodestreamLevel(enc.get(), 10)); EXPECT_EQ(JXL_ENC_SUCCESS, JxlEncoderSetBasicInfo(enc.get(), &basic_info)); JxlColorEncoding color_encoding; JxlColorEncodingSetToSRGB(&color_encoding, /*is_gray=*/false); EXPECT_EQ(JXL_ENC_SUCCESS, @@ -1298,11 +1305,11 @@ TEST(EncodeTest, JXL_TRANSCODE_JPEG_TEST(JPEGFrameTest)) { for (int skip_basic_info = 0; skip_basic_info < 2; skip_basic_info++) { for (int skip_color_encoding = 0; skip_color_encoding < 2; skip_color_encoding++) { const std::string jpeg_path = - "imagecompression.info/flower_foveon_cropped.jpg"; + "third_party/imagecompression.info/flower_foveon_cropped.jpg"; const jxl::PaddedBytes orig = jxl::ReadTestData(jpeg_path); jxl::CodecInOut orig_io; ASSERT_TRUE(SetFromBytes(jxl::Span(orig), &orig_io, /*pool=*/nullptr)); diff --git a/third_party/jpeg-xl/lib/jxl/exif.h b/third_party/jpeg-xl/lib/jxl/exif.h new file mode 100644 --- /dev/null +++ b/third_party/jpeg-xl/lib/jxl/exif.h @@ -0,0 +1,27 @@ +// Copyright (c) the JPEG XL Project Authors. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef LIB_JXL_EXIF_H_ +#define LIB_JXL_EXIF_H_ + +// Basic parsing of Exif (just enough for the render-impacting things +// like orientation) + +#include "lib/jxl/base/padded_bytes.h" +#include "lib/jxl/image_metadata.h" + +namespace jxl { + +// Parses the Exif data just enough to extract any render-impacting info. +// If the Exif data is invalid or could not be parsed, then it is treated +// as a no-op. +void InterpretExif(const std::vector& exif, CodecMetadata* metadata); + +// Sets the Exif orientation to the identity, to avoid repeated orientation +void ResetExifOrientation(std::vector& exif); + +} // namespace jxl + +#endif // LIB_JXL_EXIF_H_ diff --git a/third_party/jpeg-xl/lib/jxl/exif.cc b/third_party/jpeg-xl/lib/jxl/exif.cc new file mode 100644 --- /dev/null +++ b/third_party/jpeg-xl/lib/jxl/exif.cc @@ -0,0 +1,89 @@ +// Copyright (c) the JPEG XL Project Authors. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "lib/jxl/exif.h" + +namespace jxl { + +constexpr uint16_t kExifOrientationTag = 274; + +// Checks if a blob looks like Exif, and if so, sets bigendian +// according to the tiff endianness +bool IsExif(const std::vector& exif, bool* bigendian) { + if (exif.size() < 12) return false; // not enough bytes for a valid exif blob + const uint8_t* t = exif.data(); + if (LoadLE32(t) == 0x2A004D4D) { + *bigendian = true; + return true; + } else if (LoadLE32(t) == 0x002A4949) { + *bigendian = false; + return true; + } + return false; // not a valid tiff header +} + +// Finds the position of an Exif tag, or 0 if it is not found +size_t FindExifTagPosition(const std::vector& exif, uint16_t tagname) { + bool bigendian; + if (!IsExif(exif, &bigendian)) return 0; + const uint8_t* t = exif.data() + 4; + uint32_t offset = (bigendian ? LoadBE32(t) : LoadLE32(t)); + if (exif.size() < 12 + offset + 2 || offset < 8) return 0; + t += offset - 4; + uint16_t nb_tags = (bigendian ? LoadBE16(t) : LoadLE16(t)); + t += 2; + while (nb_tags > 0) { + if (t + 12 >= exif.data() + exif.size()) return 0; + uint16_t tag = (bigendian ? LoadBE16(t) : LoadLE16(t)); + t += 2; + if (tag == tagname) return static_cast(t - exif.data()); + t += 10; + nb_tags--; + } + return 0; +} + +// TODO (jon): tag 1 can be used to represent Adobe RGB 1998 if it has value +// "R03" +// TODO (jon): set intrinsic dimensions according to +// https://discourse.wicg.io/t/proposal-exif-image-resolution-auto-and-from-image/4326/24 +void InterpretExif(const std::vector& exif, CodecMetadata* metadata) { + bool bigendian; + if (!IsExif(exif, &bigendian)) return; + size_t o_pos = FindExifTagPosition(exif, kExifOrientationTag); + if (o_pos) { + const uint8_t* t = exif.data() + o_pos; + uint16_t type = (bigendian ? LoadBE16(t) : LoadLE16(t)); + t += 2; + uint32_t count = (bigendian ? LoadBE32(t) : LoadLE32(t)); + t += 4; + uint16_t value = (bigendian ? LoadBE16(t) : LoadLE16(t)); + t += 4; + if (type == 3 && count == 1 && value >= 1 && value <= 8) { + metadata->m.orientation = value; + } + } +} + +void ResetExifOrientation(std::vector& exif) { + bool bigendian; + if (!IsExif(exif, &bigendian)) return; + size_t o_pos = FindExifTagPosition(exif, kExifOrientationTag); + if (o_pos) { + uint8_t* t = exif.data() + o_pos; + uint16_t type = (bigendian ? LoadBE16(t) : LoadLE16(t)); + t += 2; + uint32_t count = (bigendian ? LoadBE32(t) : LoadLE32(t)); + t += 4; + if (type == 3 && count == 1) { + if (bigendian) + StoreBE16(1, t); + else + StoreLE16(1, t); + } + } +} + +} // namespace jxl diff --git a/third_party/jpeg-xl/lib/jxl/frame_header.h b/third_party/jpeg-xl/lib/jxl/frame_header.h --- a/third_party/jpeg-xl/lib/jxl/frame_header.h +++ b/third_party/jpeg-xl/lib/jxl/frame_header.h @@ -38,11 +38,11 @@ BitsOffset(10, 48), 0, &name_length)); if (visitor->IsReading()) { name->resize(name_length); } for (size_t i = 0; i < name_length; i++) { - uint32_t c = (*name)[i]; + uint32_t c = static_cast((*name)[i]); JXL_QUIET_RETURN_IF_ERROR(visitor->Bits(8, 0, &c)); (*name)[i] = static_cast(c); } return true; } diff --git a/third_party/jpeg-xl/lib/jxl/icc_codec.cc b/third_party/jpeg-xl/lib/jxl/icc_codec.cc --- a/third_party/jpeg-xl/lib/jxl/icc_codec.cc +++ b/third_party/jpeg-xl/lib/jxl/icc_codec.cc @@ -20,23 +20,10 @@ #include "lib/jxl/icc_codec_common.h" namespace jxl { namespace { -uint64_t DecodeVarInt(const uint8_t* input, size_t inputSize, size_t* pos) { - size_t i; - uint64_t ret = 0; - for (i = 0; *pos + i < inputSize && i < 10; ++i) { - ret |= uint64_t(input[*pos + i] & 127) << uint64_t(7 * i); - // If the next-byte flag is not set, stop - if ((input[*pos + i] & 128) == 0) break; - } - // TODO: Return a decoding error if i == 10. - *pos += i + 1; - return ret; -} - // Shuffles or interleaves bytes, for example with width 2, turns "ABCDabcd" // into "AaBbCcDc". Transposes a matrix of ceil(size / width) columns and // width rows. There are size elements, size may be < width * height, if so the // last elements of the rightmost column are missing, the missing spots are // transposed along with the filled spots, and the result has the missing diff --git a/third_party/jpeg-xl/lib/jxl/image.h b/third_party/jpeg-xl/lib/jxl/image.h --- a/third_party/jpeg-xl/lib/jxl/image.h +++ b/third_party/jpeg-xl/lib/jxl/image.h @@ -192,122 +192,133 @@ // Rectangular region in image(s). Factoring this out of Image instead of // shifting the pointer by x0/y0 allows this to apply to multiple images with // different resolutions (e.g. color transform and quantization field). // Can compare using SameSize(rect1, rect2). -class Rect { +template +class RectT { public: // Most windows are xsize_max * ysize_max, except those on the borders where // begin + size_max > end. - constexpr Rect(size_t xbegin, size_t ybegin, size_t xsize_max, - size_t ysize_max, size_t xend, size_t yend) + constexpr RectT(T xbegin, T ybegin, size_t xsize_max, size_t ysize_max, + T xend, T yend) : x0_(xbegin), y0_(ybegin), xsize_(ClampedSize(xbegin, xsize_max, xend)), ysize_(ClampedSize(ybegin, ysize_max, yend)) {} // Construct with origin and known size (typically from another Rect). - constexpr Rect(size_t xbegin, size_t ybegin, size_t xsize, size_t ysize) + constexpr RectT(T xbegin, T ybegin, size_t xsize, size_t ysize) : x0_(xbegin), y0_(ybegin), xsize_(xsize), ysize_(ysize) {} // Construct a rect that covers a whole image/plane/ImageBundle etc. - template - explicit Rect(const Image& image) - : Rect(0, 0, image.xsize(), image.ysize()) {} + template + explicit RectT(const ImageT& image) + : RectT(0, 0, image.xsize(), image.ysize()) {} - Rect() : Rect(0, 0, 0, 0) {} + RectT() : RectT(0, 0, 0, 0) {} - Rect(const Rect&) = default; - Rect& operator=(const Rect&) = default; + RectT(const RectT&) = default; + RectT& operator=(const RectT&) = default; // Construct a subrect that resides in an image/plane/ImageBundle etc. - template - Rect Crop(const Image& image) const { - return Rect(x0_, y0_, xsize_, ysize_, image.xsize(), image.ysize()); + template + RectT Crop(const ImageT& image) const { + return Intersection(RectT(image)); } // Construct a subrect that resides in the [0, ysize) x [0, xsize) region of // the current rect. - Rect Crop(size_t area_xsize, size_t area_ysize) const { - return Rect(x0_, y0_, xsize_, ysize_, area_xsize, area_ysize); + RectT Crop(size_t area_xsize, size_t area_ysize) const { + return Intersection(RectT(0, 0, area_xsize, area_ysize)); } // Returns a rect that only contains `num` lines with offset `y` from `y0()`. - Rect Lines(size_t y, size_t num) const { + RectT Lines(size_t y, size_t num) const { JXL_DASSERT(y + num <= ysize_); - return Rect(x0_, y0_ + y, xsize_, num); + return RectT(x0_, y0_ + y, xsize_, num); } - Rect Line(size_t y) const { return Lines(y, 1); } + RectT Line(size_t y) const { return Lines(y, 1); } - JXL_MUST_USE_RESULT Rect Intersection(const Rect& other) const { - return Rect(std::max(x0_, other.x0_), std::max(y0_, other.y0_), xsize_, - ysize_, std::min(x0_ + xsize_, other.x0_ + other.xsize_), - std::min(y0_ + ysize_, other.y0_ + other.ysize_)); + JXL_MUST_USE_RESULT RectT Intersection(const RectT& other) const { + return RectT(std::max(x0_, other.x0_), std::max(y0_, other.y0_), xsize_, + ysize_, std::min(x0_ + xsize_, other.x0_ + other.xsize_), + std::min(y0_ + ysize_, other.y0_ + other.ysize_)); } - JXL_MUST_USE_RESULT Rect Translate(int64_t x_offset, int64_t y_offset) const { - return Rect(x0_ + x_offset, y0_ + y_offset, xsize_, ysize_); + JXL_MUST_USE_RESULT RectT Translate(int64_t x_offset, + int64_t y_offset) const { + return RectT(x0_ + x_offset, y0_ + y_offset, xsize_, ysize_); } - template - T* Row(Plane* image, size_t y) const { + template + V* Row(Plane* image, size_t y) const { + JXL_DASSERT(y + y0_ >= 0); return image->Row(y + y0_) + x0_; } - template - const T* Row(const Plane* image, size_t y) const { + template + const V* Row(const Plane* image, size_t y) const { + JXL_DASSERT(y + y0_ >= 0); return image->Row(y + y0_) + x0_; } - template - T* PlaneRow(Image3* image, const size_t c, size_t y) const { + template + V* PlaneRow(Image3* image, const size_t c, size_t y) const { + JXL_DASSERT(y + y0_ >= 0); return image->PlaneRow(c, y + y0_) + x0_; } - template - const T* ConstRow(const Plane& image, size_t y) const { + template + const V* ConstRow(const Plane& image, size_t y) const { + JXL_DASSERT(y + y0_ >= 0); return image.ConstRow(y + y0_) + x0_; } - template - const T* ConstPlaneRow(const Image3& image, size_t c, size_t y) const { + template + const V* ConstPlaneRow(const Image3& image, size_t c, size_t y) const { + JXL_DASSERT(y + y0_ >= 0); return image.ConstPlaneRow(c, y + y0_) + x0_; } - bool IsInside(const Rect& other) const { + bool IsInside(const RectT& other) const { return x0_ >= other.x0() && x0_ + xsize_ <= other.x0() + other.xsize_ && y0_ >= other.y0() && y0_ + ysize_ <= other.y0() + other.ysize(); } // Returns true if this Rect fully resides in the given image. ImageT could be // Plane or Image3; however if ImageT is Rect, results are nonsensical. template bool IsInside(const ImageT& image) const { - return (x0_ + xsize_ <= image.xsize()) && (y0_ + ysize_ <= image.ysize()); + return IsInside(RectT(image)); } - size_t x0() const { return x0_; } - size_t y0() const { return y0_; } + T x0() const { return x0_; } + T y0() const { return y0_; } size_t xsize() const { return xsize_; } size_t ysize() const { return ysize_; } + T x1() const { return x0_ + xsize_; } + T y1() const { return y0_ + ysize_; } private: // Returns size_max, or whatever is left in [begin, end). - static constexpr size_t ClampedSize(size_t begin, size_t size_max, - size_t end) { - return (begin + size_max <= end) ? size_max - : (end > begin ? end - begin : 0); + static constexpr size_t ClampedSize(T begin, size_t size_max, T end) { + return (static_cast(begin + size_max) <= end) + ? size_max + : (end > begin ? end - begin : 0); } - size_t x0_; - size_t y0_; + T x0_; + T y0_; size_t xsize_; size_t ysize_; }; +using Rect = RectT; + // Currently, we abuse Image to either refer to an image that owns its storage // or one that doesn't. In similar vein, we abuse Image* function parameters to // either mean "assign to me" or "fill the provided image with data". // Hopefully, the "assign to me" meaning will go away and most images in the // codebase will not be backed by own storage. When this happens we can redesign diff --git a/third_party/jpeg-xl/lib/jxl/image.cc b/third_party/jpeg-xl/lib/jxl/image.cc --- a/third_party/jpeg-xl/lib/jxl/image.cc +++ b/third_party/jpeg-xl/lib/jxl/image.cc @@ -131,51 +131,10 @@ std::swap(orig_ysize_, other.orig_ysize_); std::swap(bytes_per_row_, other.bytes_per_row_); std::swap(bytes_, other.bytes_); } -ImageB ImageFromPacked(const uint8_t* packed, const size_t xsize, - const size_t ysize, const size_t bytes_per_row) { - JXL_ASSERT(bytes_per_row >= xsize); - ImageB image(xsize, ysize); - PROFILER_FUNC; - for (size_t y = 0; y < ysize; ++y) { - uint8_t* const JXL_RESTRICT row = image.Row(y); - const uint8_t* const JXL_RESTRICT packed_row = packed + y * bytes_per_row; - memcpy(row, packed_row, xsize); - } - return image; -} - -// Note that using mirroring here gives slightly worse results. -ImageF PadImage(const ImageF& in, const size_t xsize, const size_t ysize) { - JXL_ASSERT(xsize >= in.xsize()); - JXL_ASSERT(ysize >= in.ysize()); - ImageF out(xsize, ysize); - size_t y = 0; - for (; y < in.ysize(); ++y) { - const float* JXL_RESTRICT row_in = in.ConstRow(y); - float* JXL_RESTRICT row_out = out.Row(y); - memcpy(row_out, row_in, in.xsize() * sizeof(row_in[0])); - const int lastcol = in.xsize() - 1; - const float lastval = row_out[lastcol]; - for (size_t x = in.xsize(); x < xsize; ++x) { - row_out[x] = lastval; - } - } - - // TODO(janwas): no need to copy if we can 'extend' image: if rows are - // pointers to any memory? Or allocate larger image before IO? - const int lastrow = in.ysize() - 1; - for (; y < ysize; ++y) { - const float* JXL_RESTRICT row_in = out.ConstRow(lastrow); - float* JXL_RESTRICT row_out = out.Row(y); - memcpy(row_out, row_in, xsize * sizeof(row_out[0])); - } - return out; -} - Image3F PadImageMirror(const Image3F& in, const size_t xborder, const size_t yborder) { size_t xsize = in.xsize(); size_t ysize = in.ysize(); Image3F out(xsize + 2 * xborder, ysize + 2 * yborder); @@ -214,23 +173,10 @@ } } return out; } -Image3F PadImageToMultiple(const Image3F& in, const size_t N) { - PROFILER_FUNC; - const size_t xsize_blocks = DivCeil(in.xsize(), N); - const size_t ysize_blocks = DivCeil(in.ysize(), N); - const size_t xsize = N * xsize_blocks; - const size_t ysize = N * ysize_blocks; - ImageF out[3]; - for (size_t c = 0; c < 3; ++c) { - out[c] = PadImage(in.Plane(c), xsize, ysize); - } - return Image3F(std::move(out[0]), std::move(out[1]), std::move(out[2])); -} - void PadImageToBlockMultipleInPlace(Image3F* JXL_RESTRICT in) { PROFILER_FUNC; const size_t xsize_orig = in->xsize(); const size_t ysize_orig = in->ysize(); const size_t xsize = RoundUpToBlockDim(xsize_orig); diff --git a/third_party/jpeg-xl/lib/jxl/image_ops.h b/third_party/jpeg-xl/lib/jxl/image_ops.h --- a/third_party/jpeg-xl/lib/jxl/image_ops.h +++ b/third_party/jpeg-xl/lib/jxl/image_ops.h @@ -787,23 +787,14 @@ T* JXL_RESTRICT row = rect.Row(image, y); memset(row, 0, rect.xsize() * sizeof(T)); } } -// First, image is padded horizontally, with the rightmost value. -// Next, image is padded vertically, by repeating the last line. -ImageF PadImage(const ImageF& in, size_t xsize, size_t ysize); - // Pad an image with xborder columns on each vertical side and yboder rows // above and below, mirroring the image. Image3F PadImageMirror(const Image3F& in, size_t xborder, size_t yborder); -// First, image is padded horizontally, with the rightmost value. -// Next, image is padded vertically, by repeating the last line. -// Prefer PadImageToBlockMultipleInPlace if padding to kBlockDim. -Image3F PadImageToMultiple(const Image3F& in, size_t N); - // Same as above, but operates in-place. Assumes that the `in` image was // allocated large enough. void PadImageToBlockMultipleInPlace(Image3F* JXL_RESTRICT in); // Downsamples an image by a given factor. diff --git a/third_party/jpeg-xl/lib/jxl/jpeg/enc_jpeg_data.h b/third_party/jpeg-xl/lib/jxl/jpeg/enc_jpeg_data.h --- a/third_party/jpeg-xl/lib/jxl/jpeg/enc_jpeg_data.h +++ b/third_party/jpeg-xl/lib/jxl/jpeg/enc_jpeg_data.h @@ -6,15 +6,17 @@ #ifndef LIB_JXL_JPEG_ENC_JPEG_DATA_H_ #define LIB_JXL_JPEG_ENC_JPEG_DATA_H_ #include "lib/jxl/base/padded_bytes.h" #include "lib/jxl/codec_in_out.h" +#include "lib/jxl/enc_params.h" #include "lib/jxl/jpeg/jpeg_data.h" namespace jxl { namespace jpeg { -Status EncodeJPEGData(JPEGData& jpeg_data, PaddedBytes* bytes); +Status EncodeJPEGData(JPEGData& jpeg_data, PaddedBytes* bytes, + const CompressParams& cparams); Status SetColorEncodingFromJpegData(const jpeg::JPEGData& jpg, ColorEncoding* color_encoding); /** diff --git a/third_party/jpeg-xl/lib/jxl/jpeg/enc_jpeg_data.cc b/third_party/jpeg-xl/lib/jxl/jpeg/enc_jpeg_data.cc --- a/third_party/jpeg-xl/lib/jxl/jpeg/enc_jpeg_data.cc +++ b/third_party/jpeg-xl/lib/jxl/jpeg/enc_jpeg_data.cc @@ -228,22 +228,25 @@ } return color_encoding->SetICC(std::move(icc_profile)); } -Status EncodeJPEGData(JPEGData& jpeg_data, PaddedBytes* bytes) { +Status EncodeJPEGData(JPEGData& jpeg_data, PaddedBytes* bytes, + const CompressParams& cparams) { jpeg_data.app_marker_type.resize(jpeg_data.app_data.size(), AppMarkerType::kUnknown); JXL_RETURN_IF_ERROR(DetectIccProfile(jpeg_data)); JXL_RETURN_IF_ERROR(DetectBlobs(jpeg_data)); BitWriter writer; JXL_RETURN_IF_ERROR(Bundle::Write(jpeg_data, &writer, 0, nullptr)); writer.ZeroPadToByte(); *bytes = std::move(writer).TakeBytes(); BrotliEncoderState* brotli_enc = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr); - BrotliEncoderSetParameter(brotli_enc, BROTLI_PARAM_QUALITY, 11); + int effort = cparams.brotli_effort; + if (effort < 0) effort = 11 - static_cast(cparams.speed_tier); + BrotliEncoderSetParameter(brotli_enc, BROTLI_PARAM_QUALITY, effort); size_t total_data = 0; for (size_t i = 0; i < jpeg_data.app_data.size(); i++) { if (jpeg_data.app_marker_type[i] != AppMarkerType::kUnknown) { continue; } diff --git a/third_party/jpeg-xl/lib/jxl/jpeg/enc_jpeg_data_reader.cc b/third_party/jpeg-xl/lib/jxl/jpeg/enc_jpeg_data_reader.cc --- a/third_party/jpeg-xl/lib/jxl/jpeg/enc_jpeg_data_reader.cc +++ b/third_party/jpeg-xl/lib/jxl/jpeg/enc_jpeg_data_reader.cc @@ -93,10 +93,11 @@ size_t marker_len = ReadUint16(data, pos); int precision = ReadUint8(data, pos); int height = ReadUint16(data, pos); int width = ReadUint16(data, pos); int num_components = ReadUint8(data, pos); + // 'jbrd' is hardcoded for 8bits: JXL_JPEG_VERIFY_INPUT(precision, 8, 8, PRECISION); JXL_JPEG_VERIFY_INPUT(height, 1, kMaxDimPixels, HEIGHT); JXL_JPEG_VERIFY_INPUT(width, 1, kMaxDimPixels, WIDTH); JXL_JPEG_VERIFY_INPUT(num_components, 1, kMaxComponents, NUMCOMP); JXL_JPEG_VERIFY_LEN(3 * num_components); diff --git a/third_party/jpeg-xl/lib/jxl/jxl_test.cc b/third_party/jpeg-xl/lib/jxl/jxl_test.cc --- a/third_party/jpeg-xl/lib/jxl/jxl_test.cc +++ b/third_party/jpeg-xl/lib/jxl/jxl_test.cc @@ -122,11 +122,11 @@ #endif TEST(JxlTest, RoundtripTinyFast) { ThreadPool* pool = nullptr; const PaddedBytes orig = - ReadTestData("wesaturate/500px/u76c0g_bliznaca_srgb8.png"); + ReadTestData("third_party/wesaturate/500px/u76c0g_bliznaca_srgb8.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, pool)); io.ShrinkTo(32, 32); CompressParams cparams; @@ -140,11 +140,11 @@ } TEST(JxlTest, RoundtripSmallD1) { ThreadPool* pool = nullptr; const PaddedBytes orig = - ReadTestData("wesaturate/500px/u76c0g_bliznaca_srgb8.png"); + ReadTestData("third_party/wesaturate/500px/u76c0g_bliznaca_srgb8.png"); CompressParams cparams; cparams.butteraugli_distance = 1.0; DecompressParams dparams; CodecInOut io_out; @@ -181,11 +181,11 @@ } TEST(JxlTest, RoundtripOtherTransforms) { ThreadPool* pool = nullptr; const PaddedBytes orig = - ReadTestData("wesaturate/64px/a2d1un_nkitzmiller_srgb8.png"); + ReadTestData("third_party/wesaturate/64px/a2d1un_nkitzmiller_srgb8.png"); std::unique_ptr io = jxl::make_unique(); ASSERT_TRUE(SetFromBytes(Span(orig), io.get(), pool)); CompressParams cparams; // Slow modes access linear image for adaptive quant search @@ -213,11 +213,11 @@ } TEST(JxlTest, RoundtripResample2) { ThreadPool* pool = nullptr; const PaddedBytes orig = - ReadTestData("wesaturate/500px/u76c0g_bliznaca_srgb8.png"); + ReadTestData("third_party/wesaturate/500px/u76c0g_bliznaca_srgb8.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, pool)); io.ShrinkTo(io.xsize(), io.ysize()); CompressParams cparams; cparams.resampling = 2; @@ -230,11 +230,11 @@ } TEST(JxlTest, RoundtripResample2MT) { ThreadPoolInternal pool(4); const PaddedBytes orig = - ReadTestData("imagecompression.info/flower_foveon.png"); + ReadTestData("third_party/imagecompression.info/flower_foveon.png"); // image has to be large enough to have multiple groups after downsampling CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); CompressParams cparams; cparams.resampling = 2; @@ -252,11 +252,11 @@ // in random order. TEST(JxlTest, RoundtripOutOfOrderProcessing) { FakeParallelRunner fake_pool(/*order_seed=*/123, /*num_threads=*/8); ThreadPool pool(&JxlFakeParallelRunner, &fake_pool); const PaddedBytes orig = - ReadTestData("imagecompression.info/flower_foveon.png"); + ReadTestData("third_party/imagecompression.info/flower_foveon.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); // Image size is selected so that the block border needed is larger than the // amount of pixels available on the next block. io.ShrinkTo(513, 515); @@ -275,11 +275,11 @@ TEST(JxlTest, RoundtripOutOfOrderProcessingBorder) { FakeParallelRunner fake_pool(/*order_seed=*/47, /*num_threads=*/8); ThreadPool pool(&JxlFakeParallelRunner, &fake_pool); const PaddedBytes orig = - ReadTestData("imagecompression.info/flower_foveon.png"); + ReadTestData("third_party/imagecompression.info/flower_foveon.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); // Image size is selected so that the block border needed is larger than the // amount of pixels available on the next block. io.ShrinkTo(513, 515); @@ -298,11 +298,11 @@ } TEST(JxlTest, RoundtripResample4) { ThreadPool* pool = nullptr; const PaddedBytes orig = - ReadTestData("wesaturate/500px/u76c0g_bliznaca_srgb8.png"); + ReadTestData("third_party/wesaturate/500px/u76c0g_bliznaca_srgb8.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, pool)); io.ShrinkTo(io.xsize(), io.ysize()); CompressParams cparams; cparams.resampling = 4; @@ -315,11 +315,11 @@ } TEST(JxlTest, RoundtripResample8) { ThreadPool* pool = nullptr; const PaddedBytes orig = - ReadTestData("wesaturate/500px/u76c0g_bliznaca_srgb8.png"); + ReadTestData("third_party/wesaturate/500px/u76c0g_bliznaca_srgb8.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, pool)); io.ShrinkTo(io.xsize(), io.ysize()); CompressParams cparams; cparams.resampling = 8; @@ -332,11 +332,11 @@ } TEST(JxlTest, RoundtripUnalignedD2) { ThreadPool* pool = nullptr; const PaddedBytes orig = - ReadTestData("wesaturate/500px/u76c0g_bliznaca_srgb8.png"); + ReadTestData("third_party/wesaturate/500px/u76c0g_bliznaca_srgb8.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, pool)); io.ShrinkTo(io.xsize() / 12, io.ysize() / 7); CompressParams cparams; @@ -353,11 +353,11 @@ #if JXL_TEST_NL TEST(JxlTest, RoundtripMultiGroupNL) { ThreadPoolInternal pool(4); const PaddedBytes orig = - ReadTestData("imagecompression.info/flower_foveon.png"); + ReadTestData("third_party/imagecompression.info/flower_foveon.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); io.ShrinkTo(600, 1024); // partial X, full Y group CompressParams cparams; @@ -382,11 +382,11 @@ #endif TEST(JxlTest, RoundtripMultiGroup) { ThreadPoolInternal pool(4); const PaddedBytes orig = - ReadTestData("imagecompression.info/flower_foveon.png"); + ReadTestData("third_party/imagecompression.info/flower_foveon.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); io.ShrinkTo(600, 1024); CompressParams cparams; @@ -408,11 +408,11 @@ } TEST(JxlTest, RoundtripLargeFast) { ThreadPoolInternal pool(8); const PaddedBytes orig = - ReadTestData("imagecompression.info/flower_foveon.png"); + ReadTestData("third_party/imagecompression.info/flower_foveon.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); CompressParams cparams; cparams.speed_tier = SpeedTier::kSquirrel; @@ -423,11 +423,11 @@ } TEST(JxlTest, RoundtripDotsForceEpf) { ThreadPoolInternal pool(8); const PaddedBytes orig = - ReadTestData("wesaturate/500px/cvo9xd_keong_macan_srgb8.png"); + ReadTestData("third_party/wesaturate/500px/cvo9xd_keong_macan_srgb8.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); CompressParams cparams; cparams.epf = 2; @@ -443,11 +443,11 @@ // which involves additional processing including adaptive reconstruction. // Failing this may be a sign of race conditions or invalid memory accesses. TEST(JxlTest, RoundtripD2Consistent) { ThreadPoolInternal pool(8); const PaddedBytes orig = - ReadTestData("imagecompression.info/flower_foveon.png"); + ReadTestData("third_party/imagecompression.info/flower_foveon.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); CompressParams cparams; cparams.speed_tier = SpeedTier::kSquirrel; @@ -476,11 +476,11 @@ // Same as above, but for full image, testing multiple groups. TEST(JxlTest, RoundtripLargeConsistent) { ThreadPoolInternal pool(8); const PaddedBytes orig = - ReadTestData("imagecompression.info/flower_foveon.png"); + ReadTestData("third_party/imagecompression.info/flower_foveon.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); CompressParams cparams; cparams.speed_tier = SpeedTier::kSquirrel; @@ -506,11 +506,11 @@ #if JXL_TEST_NL TEST(JxlTest, RoundtripSmallNL) { ThreadPool* pool = nullptr; const PaddedBytes orig = - ReadTestData("wesaturate/500px/u76c0g_bliznaca_srgb8.png"); + ReadTestData("third_party/wesaturate/500px/u76c0g_bliznaca_srgb8.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, pool)); io.ShrinkTo(io.xsize() / 8, io.ysize() / 8); CompressParams cparams; @@ -527,11 +527,11 @@ #endif TEST(JxlTest, RoundtripNoGaborishNoAR) { ThreadPool* pool = nullptr; const PaddedBytes orig = - ReadTestData("wesaturate/500px/u76c0g_bliznaca_srgb8.png"); + ReadTestData("third_party/wesaturate/500px/u76c0g_bliznaca_srgb8.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, pool)); CompressParams cparams; cparams.gaborish = Override::kOff; @@ -547,11 +547,11 @@ } TEST(JxlTest, RoundtripSmallNoGaborish) { ThreadPool* pool = nullptr; const PaddedBytes orig = - ReadTestData("wesaturate/500px/u76c0g_bliznaca_srgb8.png"); + ReadTestData("third_party/wesaturate/500px/u76c0g_bliznaca_srgb8.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, pool)); io.ShrinkTo(io.xsize() / 8, io.ysize() / 8); CompressParams cparams; @@ -593,11 +593,11 @@ CodecInOut io2; EXPECT_LE(Roundtrip(&io, cparams, dparams, pool, &io2), 2000u); EXPECT_THAT(ButteraugliDistance(io, io2, cparams.ba_params, GetJxlCms(), /*distmap=*/nullptr, pool), - IsSlightlyBelow(0.2f)); + IsSlightlyBelow(0.04f)); } TEST(JxlTest, RoundtripSmallPatches) { ThreadPool* pool = nullptr; CodecInOut io; @@ -621,11 +621,11 @@ CodecInOut io2; EXPECT_LE(Roundtrip(&io, cparams, dparams, pool, &io2), 2000u); EXPECT_THAT(ButteraugliDistance(io, io2, cparams.ba_params, GetJxlCms(), /*distmap=*/nullptr, pool), - IsSlightlyBelow(0.2f)); + IsSlightlyBelow(0.04f)); } // Test header encoding of original bits per sample TEST(JxlTest, RoundtripImageBundleOriginalBits) { ThreadPool* pool = nullptr; @@ -698,12 +698,12 @@ } } TEST(JxlTest, RoundtripGrayscale) { ThreadPool* pool = nullptr; - const PaddedBytes orig = - ReadTestData("wesaturate/500px/cvo9xd_keong_macan_grayscale.png"); + const PaddedBytes orig = ReadTestData( + "third_party/wesaturate/500px/cvo9xd_keong_macan_grayscale.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, pool)); ASSERT_NE(io.xsize(), 0u); io.ShrinkTo(128, 128); EXPECT_TRUE(io.Main().IsGray()); @@ -754,12 +754,12 @@ } } TEST(JxlTest, RoundtripAlpha) { ThreadPool* pool = nullptr; - const PaddedBytes orig = - ReadTestData("wesaturate/500px/tmshre_riaphotographs_alpha.png"); + const PaddedBytes orig = ReadTestData( + "third_party/wesaturate/500px/tmshre_riaphotographs_alpha.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, pool)); ASSERT_NE(io.xsize(), 0u); ASSERT_TRUE(io.metadata.m.HasAlpha()); @@ -784,17 +784,17 @@ EXPECT_LE(compressed.size(), 10077u); EXPECT_THAT(ButteraugliDistance(io, io2, cparams.ba_params, GetJxlCms(), /*distmap=*/nullptr, pool), - IsSlightlyBelow(1.4)); + IsSlightlyBelow(1.2)); } TEST(JxlTest, RoundtripAlphaPremultiplied) { ThreadPool* pool = nullptr; - const PaddedBytes orig = - ReadTestData("wesaturate/500px/tmshre_riaphotographs_alpha.png"); + const PaddedBytes orig = ReadTestData( + "third_party/wesaturate/500px/tmshre_riaphotographs_alpha.png"); CodecInOut io, io_nopremul; ASSERT_TRUE(SetFromBytes(Span(orig), &io, pool)); ASSERT_TRUE(SetFromBytes(Span(orig), &io_nopremul, pool)); ASSERT_NE(io.xsize(), 0u); @@ -819,22 +819,22 @@ EXPECT_LE(compressed.size(), 10000u); EXPECT_THAT(ButteraugliDistance(io, io2, cparams.ba_params, GetJxlCms(), /*distmap=*/nullptr, pool), - IsSlightlyBelow(1.4)); + IsSlightlyBelow(1.2)); io2.Main().UnpremultiplyAlpha(); EXPECT_THAT( ButteraugliDistance(io_nopremul, io2, cparams.ba_params, GetJxlCms(), /*distmap=*/nullptr, pool), - IsSlightlyBelow(1.8)); + IsSlightlyBelow(1.35)); } TEST(JxlTest, RoundtripAlphaResampling) { ThreadPool* pool = nullptr; - const PaddedBytes orig = - ReadTestData("wesaturate/500px/tmshre_riaphotographs_alpha.png"); + const PaddedBytes orig = ReadTestData( + "third_party/wesaturate/500px/tmshre_riaphotographs_alpha.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, pool)); ASSERT_NE(io.xsize(), 0u); ASSERT_TRUE(io.metadata.m.HasAlpha()); @@ -862,12 +862,12 @@ IsSlightlyBelow(4.7)); } TEST(JxlTest, RoundtripAlphaResamplingOnlyAlpha) { ThreadPool* pool = nullptr; - const PaddedBytes orig = - ReadTestData("wesaturate/500px/tmshre_riaphotographs_alpha.png"); + const PaddedBytes orig = ReadTestData( + "third_party/wesaturate/500px/tmshre_riaphotographs_alpha.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, pool)); ASSERT_NE(io.xsize(), 0u); ASSERT_TRUE(io.metadata.m.HasAlpha()); @@ -894,12 +894,12 @@ IsSlightlyBelow(1.85)); } TEST(JxlTest, RoundtripAlphaNonMultipleOf8) { ThreadPool* pool = nullptr; - const PaddedBytes orig = - ReadTestData("wesaturate/500px/tmshre_riaphotographs_alpha.png"); + const PaddedBytes orig = ReadTestData( + "third_party/wesaturate/500px/tmshre_riaphotographs_alpha.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, pool)); ASSERT_NE(io.xsize(), 0u); ASSERT_TRUE(io.metadata.m.HasAlpha()); @@ -920,19 +920,19 @@ EXPECT_TRUE(EncodeFile(cparams, &io, &enc_state, &compressed, GetJxlCms(), aux_out, pool)); CodecInOut io2; EXPECT_TRUE(DecodeFile(dparams, compressed, &io2, pool)); - EXPECT_LE(compressed.size(), 200u); + EXPECT_LE(compressed.size(), 180u); // TODO(robryk): Fix the following line in presence of different alpha_bits in // the two contexts. // EXPECT_TRUE(SamePixels(io.Main().alpha(), io2.Main().alpha())); // TODO(robryk): Fix the distance estimate used in the encoder. EXPECT_THAT(ButteraugliDistance(io, io2, cparams.ba_params, GetJxlCms(), /*distmap=*/nullptr, pool), - IsSlightlyBelow(0.8)); + IsSlightlyBelow(0.9)); } TEST(JxlTest, RoundtripAlpha16) { ThreadPoolInternal pool(4); @@ -963,12 +963,11 @@ ASSERT_TRUE(io.metadata.m.HasAlpha()); ASSERT_TRUE(io.Main().HasAlpha()); CompressParams cparams; cparams.butteraugli_distance = 0.5; - // Prevent the test to be too slow, does not affect alpha - cparams.speed_tier = SpeedTier::kSquirrel; + cparams.speed_tier = SpeedTier::kWombat; DecompressParams dparams; io.metadata.m.SetUintSamples(16); EXPECT_TRUE(io.metadata.m.color_encoding.tf.IsSRGB()); PassesEncoderState enc_state; @@ -976,29 +975,30 @@ PaddedBytes compressed; EXPECT_TRUE(EncodeFile(cparams, &io, &enc_state, &compressed, GetJxlCms(), aux_out, &pool)); CodecInOut io2; EXPECT_TRUE(DecodeFile(dparams, compressed, &io2, &pool)); - - EXPECT_TRUE(SamePixels(*io.Main().alpha(), *io2.Main().alpha())); + EXPECT_THAT(ButteraugliDistance(io, io2, cparams.ba_params, GetJxlCms(), + /*distmap=*/nullptr, &pool), + IsSlightlyBelow(0.8)); } namespace { CompressParams CParamsForLossless() { CompressParams cparams; cparams.modular_mode = true; cparams.color_transform = jxl::ColorTransform::kNone; - cparams.quality_pair = {100, 100}; + cparams.butteraugli_distance = 0.f; cparams.options.predictor = {Predictor::Weighted}; return cparams; } } // namespace TEST(JxlTest, JXL_SLOW_TEST(RoundtripLossless8)) { ThreadPoolInternal pool(8); - const PaddedBytes orig = - ReadTestData("wesaturate/500px/tmshre_riaphotographs_srgb8.png"); + const PaddedBytes orig = ReadTestData( + "third_party/wesaturate/500px/tmshre_riaphotographs_srgb8.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); CompressParams cparams = CParamsForLossless(); DecompressParams dparams; @@ -1021,12 +1021,12 @@ EXPECT_EQ(ComputeDistance2(io.Main(), io2.Main(), GetJxlCms()), 0.0); } TEST(JxlTest, JXL_SLOW_TEST(RoundtripLosslessNoEncoderFastPathWP)) { ThreadPoolInternal pool(8); - const PaddedBytes orig = - ReadTestData("wesaturate/500px/tmshre_riaphotographs_srgb8.png"); + const PaddedBytes orig = ReadTestData( + "third_party/wesaturate/500px/tmshre_riaphotographs_srgb8.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); CompressParams cparams = CParamsForLossless(); cparams.speed_tier = SpeedTier::kFalcon; @@ -1038,12 +1038,12 @@ EXPECT_EQ(ComputeDistance2(io.Main(), io2.Main(), GetJxlCms()), 0.0); } TEST(JxlTest, JXL_SLOW_TEST(RoundtripLosslessNoEncoderFastPathGradient)) { ThreadPoolInternal pool(8); - const PaddedBytes orig = - ReadTestData("wesaturate/500px/tmshre_riaphotographs_srgb8.png"); + const PaddedBytes orig = ReadTestData( + "third_party/wesaturate/500px/tmshre_riaphotographs_srgb8.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); CompressParams cparams = CParamsForLossless(); cparams.speed_tier = SpeedTier::kThunder; @@ -1056,12 +1056,12 @@ EXPECT_EQ(ComputeDistance2(io.Main(), io2.Main(), GetJxlCms()), 0.0); } TEST(JxlTest, JXL_SLOW_TEST(RoundtripLosslessNoEncoderVeryFastPathGradient)) { ThreadPoolInternal pool(8); - const PaddedBytes orig = - ReadTestData("wesaturate/500px/tmshre_riaphotographs_srgb8.png"); + const PaddedBytes orig = ReadTestData( + "third_party/wesaturate/500px/tmshre_riaphotographs_srgb8.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); CompressParams cparams = CParamsForLossless(); cparams.speed_tier = SpeedTier::kLightning; @@ -1077,12 +1077,12 @@ EXPECT_EQ(ComputeDistance2(io.Main(), io3.Main(), GetJxlCms()), 0.0); } TEST(JxlTest, JXL_SLOW_TEST(RoundtripLossless8Falcon)) { ThreadPoolInternal pool(8); - const PaddedBytes orig = - ReadTestData("wesaturate/500px/tmshre_riaphotographs_srgb8.png"); + const PaddedBytes orig = ReadTestData( + "third_party/wesaturate/500px/tmshre_riaphotographs_srgb8.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); CompressParams cparams = CParamsForLossless(); cparams.speed_tier = SpeedTier::kFalcon; @@ -1094,12 +1094,12 @@ /*distmap=*/nullptr, &pool)); } TEST(JxlTest, RoundtripLossless8Alpha) { ThreadPool* pool = nullptr; - const PaddedBytes orig = - ReadTestData("wesaturate/500px/tmshre_riaphotographs_alpha.png"); + const PaddedBytes orig = ReadTestData( + "third_party/wesaturate/500px/tmshre_riaphotographs_alpha.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, pool)); EXPECT_EQ(8u, io.metadata.m.GetAlphaBits()); EXPECT_EQ(8u, io.metadata.m.bit_depth.bits_per_sample); EXPECT_FALSE(io.metadata.m.bit_depth.floating_point_sample); @@ -1219,15 +1219,15 @@ } TEST(JxlTest, RoundtripYCbCr420) { ThreadPool* pool = nullptr; const PaddedBytes orig = - ReadTestData("imagecompression.info/flower_foveon.png"); + ReadTestData("third_party/imagecompression.info/flower_foveon.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, pool)); - const PaddedBytes yuv420 = - ReadTestData("imagecompression.info/flower_foveon.png.ffmpeg.y4m"); + const PaddedBytes yuv420 = ReadTestData( + "third_party/imagecompression.info/flower_foveon.png.ffmpeg.y4m"); CodecInOut io2; ASSERT_TRUE(test::DecodeImageY4M(Span(yuv420), &io2)); CompressParams cparams = CParamsForLossless(); cparams.speed_tier = SpeedTier::kThunder; @@ -1249,11 +1249,11 @@ } TEST(JxlTest, RoundtripDots) { ThreadPool* pool = nullptr; const PaddedBytes orig = - ReadTestData("wesaturate/500px/cvo9xd_keong_macan_srgb8.png"); + ReadTestData("third_party/wesaturate/500px/cvo9xd_keong_macan_srgb8.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, pool)); ASSERT_NE(io.xsize(), 0u); @@ -1282,11 +1282,11 @@ } TEST(JxlTest, RoundtripNoise) { ThreadPool* pool = nullptr; const PaddedBytes orig = - ReadTestData("wesaturate/500px/cvo9xd_keong_macan_srgb8.png"); + ReadTestData("third_party/wesaturate/500px/cvo9xd_keong_macan_srgb8.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, pool)); ASSERT_NE(io.xsize(), 0u); @@ -1313,12 +1313,12 @@ IsSlightlyBelow(1.6)); } TEST(JxlTest, RoundtripLossless8Gray) { ThreadPool* pool = nullptr; - const PaddedBytes orig = - ReadTestData("wesaturate/500px/cvo9xd_keong_macan_grayscale.png"); + const PaddedBytes orig = ReadTestData( + "third_party/wesaturate/500px/cvo9xd_keong_macan_grayscale.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, pool)); CompressParams cparams = CParamsForLossless(); DecompressParams dparams; @@ -1446,11 +1446,11 @@ /*aux_out=*/nullptr, pool)); jpegxl::tools::JpegXlContainer enc_container; enc_container.codestream = std::move(codestream); jpeg::JPEGData data_in = *io.Main().jpeg_data; jxl::PaddedBytes jpeg_data; - EXPECT_TRUE(EncodeJPEGData(data_in, &jpeg_data)); + EXPECT_TRUE(EncodeJPEGData(data_in, &jpeg_data, cparams)); enc_container.jpeg_reconstruction = jpeg_data.data(); enc_container.jpeg_reconstruction_size = jpeg_data.size(); EXPECT_TRUE(EncodeJpegXlContainerOneShot(enc_container, &compressed)); jpegxl::tools::JpegXlContainer container; @@ -1472,22 +1472,22 @@ return compressed.size(); } TEST(JxlTest, JXL_TRANSCODE_JPEG_TEST(RoundtripJpegRecompression444)) { ThreadPoolInternal pool(8); - const PaddedBytes orig = - ReadTestData("imagecompression.info/flower_foveon.png.im_q85_444.jpg"); + const PaddedBytes orig = ReadTestData( + "third_party/imagecompression.info/flower_foveon.png.im_q85_444.jpg"); // JPEG size is 326'916 bytes. EXPECT_LE(RoundtripJpeg(orig, &pool), 256000u); } #if JPEGXL_ENABLE_JPEG TEST(JxlTest, JXL_TRANSCODE_JPEG_TEST(RoundtripJpegRecompressionToPixels)) { ThreadPoolInternal pool(8); - const PaddedBytes orig = - ReadTestData("imagecompression.info/flower_foveon.png.im_q85_444.jpg"); + const PaddedBytes orig = ReadTestData( + "third_party/imagecompression.info/flower_foveon.png.im_q85_444.jpg"); CodecInOut io; ASSERT_TRUE(jpeg::DecodeImageJPG(Span(orig), &io)); CodecInOut io2; ASSERT_TRUE(SetFromBytes(Span(orig), &io2, &pool)); @@ -1506,12 +1506,12 @@ IsSlightlyBelow(12)); } TEST(JxlTest, JXL_TRANSCODE_JPEG_TEST(RoundtripJpegRecompressionToPixels420)) { ThreadPoolInternal pool(8); - const PaddedBytes orig = - ReadTestData("imagecompression.info/flower_foveon.png.im_q85_420.jpg"); + const PaddedBytes orig = ReadTestData( + "third_party/imagecompression.info/flower_foveon.png.im_q85_420.jpg"); CodecInOut io; ASSERT_TRUE(jpeg::DecodeImageJPG(Span(orig), &io)); CodecInOut io2; ASSERT_TRUE(SetFromBytes(Span(orig), &io2, &pool)); @@ -1529,12 +1529,12 @@ } TEST(JxlTest, JXL_TRANSCODE_JPEG_TEST(RoundtripJpegRecompressionToPixels420EarlyFlush)) { ThreadPoolInternal pool(8); - const PaddedBytes orig = - ReadTestData("imagecompression.info/flower_foveon.png.im_q85_420.jpg"); + const PaddedBytes orig = ReadTestData( + "third_party/imagecompression.info/flower_foveon.png.im_q85_420.jpg"); CodecInOut io; ASSERT_TRUE(jpeg::DecodeImageJPG(Span(orig), &io)); CodecInOut io2; ASSERT_TRUE(SetFromBytes(Span(orig), &io2, &pool)); @@ -1553,12 +1553,12 @@ } TEST(JxlTest, JXL_TRANSCODE_JPEG_TEST(RoundtripJpegRecompressionToPixels420Mul16)) { ThreadPoolInternal pool(8); - const PaddedBytes orig = - ReadTestData("imagecompression.info/flower_foveon_cropped.jpg"); + const PaddedBytes orig = ReadTestData( + "third_party/imagecompression.info/flower_foveon_cropped.jpg"); CodecInOut io; ASSERT_TRUE(jpeg::DecodeImageJPG(Span(orig), &io)); CodecInOut io2; ASSERT_TRUE(SetFromBytes(Span(orig), &io2, &pool)); @@ -1577,11 +1577,12 @@ TEST(JxlTest, JXL_TRANSCODE_JPEG_TEST(RoundtripJpegRecompressionToPixels_asymmetric)) { ThreadPoolInternal pool(8); const PaddedBytes orig = ReadTestData( - "imagecompression.info/flower_foveon.png.im_q85_asymmetric.jpg"); + "third_party/imagecompression.info/" + "flower_foveon.png.im_q85_asymmetric.jpg"); CodecInOut io; ASSERT_TRUE(jpeg::DecodeImageJPG(Span(orig), &io)); CodecInOut io2; ASSERT_TRUE(SetFromBytes(Span(orig), &io2, &pool)); @@ -1600,79 +1601,82 @@ #endif TEST(JxlTest, JXL_TRANSCODE_JPEG_TEST(RoundtripJpegRecompressionGray)) { ThreadPoolInternal pool(8); - const PaddedBytes orig = - ReadTestData("imagecompression.info/flower_foveon.png.im_q85_gray.jpg"); + const PaddedBytes orig = ReadTestData( + "third_party/imagecompression.info/flower_foveon.png.im_q85_gray.jpg"); // JPEG size is 167'025 bytes. EXPECT_LE(RoundtripJpeg(orig, &pool), 140000u); } TEST(JxlTest, JXL_TRANSCODE_JPEG_TEST(RoundtripJpegRecompression420)) { ThreadPoolInternal pool(8); - const PaddedBytes orig = - ReadTestData("imagecompression.info/flower_foveon.png.im_q85_420.jpg"); + const PaddedBytes orig = ReadTestData( + "third_party/imagecompression.info/flower_foveon.png.im_q85_420.jpg"); // JPEG size is 226'018 bytes. EXPECT_LE(RoundtripJpeg(orig, &pool), 181050u); } TEST(JxlTest, JXL_TRANSCODE_JPEG_TEST(RoundtripJpegRecompression_luma_subsample)) { ThreadPoolInternal pool(8); const PaddedBytes orig = ReadTestData( - "imagecompression.info/flower_foveon.png.im_q85_luma_subsample.jpg"); + "third_party/imagecompression.info/" + "flower_foveon.png.im_q85_luma_subsample.jpg"); // JPEG size is 216'069 bytes. EXPECT_LE(RoundtripJpeg(orig, &pool), 181000u); } TEST(JxlTest, JXL_TRANSCODE_JPEG_TEST(RoundtripJpegRecompression444_12)) { // 444 JPEG that has an interesting sampling-factor (1x2, 1x2, 1x2). ThreadPoolInternal pool(8); const PaddedBytes orig = ReadTestData( - "imagecompression.info/flower_foveon.png.im_q85_444_1x2.jpg"); + "third_party/imagecompression.info/flower_foveon.png.im_q85_444_1x2.jpg"); // JPEG size is 329'942 bytes. EXPECT_LE(RoundtripJpeg(orig, &pool), 256000u); } TEST(JxlTest, JXL_TRANSCODE_JPEG_TEST(RoundtripJpegRecompression422)) { ThreadPoolInternal pool(8); - const PaddedBytes orig = - ReadTestData("imagecompression.info/flower_foveon.png.im_q85_422.jpg"); + const PaddedBytes orig = ReadTestData( + "third_party/imagecompression.info/flower_foveon.png.im_q85_422.jpg"); // JPEG size is 265'590 bytes. EXPECT_LE(RoundtripJpeg(orig, &pool), 209000u); } TEST(JxlTest, JXL_TRANSCODE_JPEG_TEST(RoundtripJpegRecompression440)) { ThreadPoolInternal pool(8); - const PaddedBytes orig = - ReadTestData("imagecompression.info/flower_foveon.png.im_q85_440.jpg"); + const PaddedBytes orig = ReadTestData( + "third_party/imagecompression.info/flower_foveon.png.im_q85_440.jpg"); // JPEG size is 262'249 bytes. EXPECT_LE(RoundtripJpeg(orig, &pool), 209000u); } TEST(JxlTest, JXL_TRANSCODE_JPEG_TEST(RoundtripJpegRecompression_asymmetric)) { // 2x vertical downsample of one chroma channel, 2x horizontal downsample of // the other. ThreadPoolInternal pool(8); const PaddedBytes orig = ReadTestData( - "imagecompression.info/flower_foveon.png.im_q85_asymmetric.jpg"); + "third_party/imagecompression.info/" + "flower_foveon.png.im_q85_asymmetric.jpg"); // JPEG size is 262'249 bytes. EXPECT_LE(RoundtripJpeg(orig, &pool), 209000u); } TEST(JxlTest, JXL_TRANSCODE_JPEG_TEST(RoundtripJpegRecompression420Progr)) { ThreadPoolInternal pool(8); const PaddedBytes orig = ReadTestData( - "imagecompression.info/flower_foveon.png.im_q85_420_progr.jpg"); + "third_party/imagecompression.info/" + "flower_foveon.png.im_q85_420_progr.jpg"); EXPECT_LE(RoundtripJpeg(orig, &pool), 181000u); } TEST(JxlTest, RoundtripProgressive) { ThreadPoolInternal pool(4); const PaddedBytes orig = - ReadTestData("imagecompression.info/flower_foveon.png"); + ReadTestData("third_party/imagecompression.info/flower_foveon.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); io.ShrinkTo(600, 1024); CompressParams cparams; @@ -1684,10 +1688,10 @@ cparams.progressive_mode = true; CodecInOut io2; EXPECT_LE(Roundtrip(&io, cparams, dparams, &pool, &io2), 40000u); EXPECT_THAT(ButteraugliDistance(io, io2, cparams.ba_params, GetJxlCms(), /*distmap=*/nullptr, &pool), - IsSlightlyBelow(2.5f)); + IsSlightlyBelow(1.1f)); } } // namespace } // namespace jxl diff --git a/third_party/jpeg-xl/lib/jxl/modular/encoding/context_predict.h b/third_party/jpeg-xl/lib/jxl/modular/encoding/context_predict.h --- a/third_party/jpeg-xl/lib/jxl/modular/encoding/context_predict.h +++ b/third_party/jpeg-xl/lib/jxl/modular/encoding/context_predict.h @@ -302,11 +302,11 @@ uint32_t context; Predictor predictor; int64_t offset; int32_t multiplier; }; - LookupResult Lookup(const Properties &properties) const { + JXL_INLINE LookupResult Lookup(const Properties &properties) const { uint32_t pos = 0; while (true) { const FlatDecisionNode &node = nodes_[pos]; if (node.property0 < 0) { return {node.childID, node.predictor, node.predictor_offset, @@ -414,10 +414,11 @@ enum PredictorMode { kUseTree = 1, kUseWP = 2, kForceComputeProperties = 4, kAllPredictions = 8, + kNoEdgeCases = 16 }; JXL_INLINE pixel_type_w PredictOne(Predictor p, pixel_type_w left, pixel_type_w top, pixel_type_w toptop, pixel_type_w topleft, pixel_type_w topright, @@ -459,26 +460,28 @@ return pixel_type_w{0}; } } template -inline PredictionResult Predict( +JXL_INLINE PredictionResult Predict( Properties *p, size_t w, const pixel_type *JXL_RESTRICT pp, const intptr_t onerow, const size_t x, const size_t y, Predictor predictor, const MATreeLookup *lookup, const Channel *references, weighted::State *wp_state, pixel_type_w *predictions) { // We start in position 3 because of 2 static properties + y. size_t offset = 3; constexpr bool compute_properties = mode & kUseTree || mode & kForceComputeProperties; - pixel_type_w left = (x ? pp[-1] : (y ? pp[-onerow] : 0)); - pixel_type_w top = (y ? pp[-onerow] : left); - pixel_type_w topleft = (x && y ? pp[-1 - onerow] : left); - pixel_type_w topright = (x + 1 < w && y ? pp[1 - onerow] : top); - pixel_type_w leftleft = (x > 1 ? pp[-2] : left); - pixel_type_w toptop = (y > 1 ? pp[-onerow - onerow] : top); - pixel_type_w toprightright = (x + 2 < w && y ? pp[2 - onerow] : topright); + constexpr bool nec = mode & kNoEdgeCases; + pixel_type_w left = (nec || x ? pp[-1] : (y ? pp[-onerow] : 0)); + pixel_type_w top = (nec || y ? pp[-onerow] : left); + pixel_type_w topleft = (nec || (x && y) ? pp[-1 - onerow] : left); + pixel_type_w topright = (nec || (x + 1 < w && y) ? pp[1 - onerow] : top); + pixel_type_w leftleft = (nec || x > 1 ? pp[-2] : left); + pixel_type_w toptop = (nec || y > 1 ? pp[-onerow - onerow] : top); + pixel_type_w toprightright = + (nec || (x + 2 < w && y) ? pp[2 - onerow] : topright); if (compute_properties) { // location (*p)[offset++] = x; // neighbors @@ -504,11 +507,11 @@ pixel_type_w wp_pred = 0; if (mode & kUseWP) { wp_pred = wp_state->Predict( x, y, w, top, left, topright, topleft, toptop, p, offset); } - if (compute_properties) { + if (!nec && compute_properties) { offset += weighted::kNumProperties; // Extra properties. const pixel_type *JXL_RESTRICT rp = references->Row(x); for (size_t i = 0; i < references->w; i++) { (*p)[offset++] = rp[i]; @@ -563,10 +566,19 @@ const Channel &references) { return detail::Predict( p, w, pp, onerow, x, y, Predictor::Zero, &tree_lookup, &references, /*wp_state=*/nullptr, /*predictions=*/nullptr); } +// Only use for y > 1, x > 1, x < w-2, and empty references +JXL_INLINE PredictionResult +PredictTreeNoWPNEC(Properties *p, size_t w, const pixel_type *JXL_RESTRICT pp, + const intptr_t onerow, const int x, const int y, + const MATreeLookup &tree_lookup, const Channel &references) { + return detail::Predict( + p, w, pp, onerow, x, y, Predictor::Zero, &tree_lookup, &references, + /*wp_state=*/nullptr, /*predictions=*/nullptr); +} inline PredictionResult PredictTreeWP(Properties *p, size_t w, const pixel_type *JXL_RESTRICT pp, const intptr_t onerow, const int x, const int y, diff --git a/third_party/jpeg-xl/lib/jxl/modular/encoding/encoding.cc b/third_party/jpeg-xl/lib/jxl/modular/encoding/encoding.cc --- a/third_party/jpeg-xl/lib/jxl/modular/encoding/encoding.cc +++ b/third_party/jpeg-xl/lib/jxl/modular/encoding/encoding.cc @@ -184,18 +184,27 @@ pixel_type v = make_pixel(value, multiplier, offset); for (size_t y = 0; y < channel.h; y++) { pixel_type *JXL_RESTRICT r = channel.Row(y); std::fill(r, r + channel.w, v); } - } else { JXL_DEBUG_V(8, "Fast track."); - for (size_t y = 0; y < channel.h; y++) { - pixel_type *JXL_RESTRICT r = channel.Row(y); - for (size_t x = 0; x < channel.w; x++) { - uint32_t v = reader->ReadHybridUintClustered(ctx_id, br); - r[x] = make_pixel(v, multiplier, offset); + if (multiplier == 1 && offset == 0) { + for (size_t y = 0; y < channel.h; y++) { + pixel_type *JXL_RESTRICT r = channel.Row(y); + for (size_t x = 0; x < channel.w; x++) { + uint32_t v = reader->ReadHybridUintClustered(ctx_id, br); + r[x] = UnpackSigned(v); + } + } + } else { + for (size_t y = 0; y < channel.h; y++) { + pixel_type *JXL_RESTRICT r = channel.Row(y); + for (size_t x = 0; x < channel.w; x++) { + uint32_t v = reader->ReadHybridUintClustered(ctx_id, br); + r[x] = make_pixel(v, multiplier, offset); + } } } } } else if (predictor == Predictor::Gradient && offset == 0 && multiplier == 1 && reader->HuffRleOnly()) { @@ -353,16 +362,40 @@ Channel references(properties.size() - kNumNonrefProperties, channel.w); for (size_t y = 0; y < channel.h; y++) { pixel_type *JXL_RESTRICT p = channel.Row(y); PrecomputeReferences(channel, y, *image, chan, &references); InitPropsRow(&properties, static_props, y); - for (size_t x = 0; x < channel.w; x++) { - PredictionResult res = - PredictTreeNoWP(&properties, channel.w, p + x, onerow, x, y, - tree_lookup, references); - uint64_t v = reader->ReadHybridUintClustered(res.context, br); - p[x] = make_pixel(v, res.multiplier, res.guess); + if (y > 1 && channel.w > 8 && references.w == 0) { + for (size_t x = 0; x < 2; x++) { + PredictionResult res = + PredictTreeNoWP(&properties, channel.w, p + x, onerow, x, y, + tree_lookup, references); + uint64_t v = reader->ReadHybridUintClustered(res.context, br); + p[x] = make_pixel(v, res.multiplier, res.guess); + } + for (size_t x = 2; x < channel.w - 2; x++) { + PredictionResult res = + PredictTreeNoWPNEC(&properties, channel.w, p + x, onerow, x, y, + tree_lookup, references); + uint64_t v = reader->ReadHybridUintClustered(res.context, br); + p[x] = make_pixel(v, res.multiplier, res.guess); + } + for (size_t x = channel.w - 2; x < channel.w; x++) { + PredictionResult res = + PredictTreeNoWP(&properties, channel.w, p + x, onerow, x, y, + tree_lookup, references); + uint64_t v = reader->ReadHybridUintClustered(res.context, br); + p[x] = make_pixel(v, res.multiplier, res.guess); + } + } else { + for (size_t x = 0; x < channel.w; x++) { + PredictionResult res = + PredictTreeNoWP(&properties, channel.w, p + x, onerow, x, y, + tree_lookup, references); + uint64_t v = reader->ReadHybridUintClustered(res.context, br); + p[x] = make_pixel(v, res.multiplier, res.guess); + } } } } else { JXL_DEBUG_V(8, "Slowest track."); MATreeLookup tree_lookup(tree); diff --git a/third_party/jpeg-xl/lib/jxl/modular/transform/enc_palette.cc b/third_party/jpeg-xl/lib/jxl/modular/transform/enc_palette.cc --- a/third_party/jpeg-xl/lib/jxl/modular/transform/enc_palette.cc +++ b/third_party/jpeg-xl/lib/jxl/modular/transform/enc_palette.cc @@ -548,22 +548,19 @@ Status FwdPalette(Image &input, uint32_t begin_c, uint32_t end_c, uint32_t &nb_colors, uint32_t &nb_deltas, bool ordered, bool lossy, Predictor &predictor, const weighted::Header &wp_header) { PaletteIterationData palette_iteration_data; - uint32_t nb = end_c - begin_c + 1; uint32_t nb_colors_orig = nb_colors; uint32_t nb_deltas_orig = nb_deltas; - // if no channel palette special case - if ((lossy || nb != 1) && input.bitdepth >= 8) { + // preprocessing pass in case of lossy palette + if (lossy && input.bitdepth >= 8) { JXL_RETURN_IF_ERROR(FwdPaletteIteration( - input, begin_c, end_c, nb_colors, nb_deltas, ordered, lossy, predictor, - wp_header, palette_iteration_data)); + input, begin_c, end_c, nb_colors_orig, nb_deltas_orig, ordered, lossy, + predictor, wp_header, palette_iteration_data)); } palette_iteration_data.final_run = true; - nb_colors = nb_colors_orig; - nb_deltas = nb_deltas_orig; return FwdPaletteIteration(input, begin_c, end_c, nb_colors, nb_deltas, ordered, lossy, predictor, wp_header, palette_iteration_data); } diff --git a/third_party/jpeg-xl/lib/jxl/modular/transform/squeeze.h b/third_party/jpeg-xl/lib/jxl/modular/transform/squeeze.h --- a/third_party/jpeg-xl/lib/jxl/modular/transform/squeeze.h +++ b/third_party/jpeg-xl/lib/jxl/modular/transform/squeeze.h @@ -73,14 +73,10 @@ if (diff - (diff & 1) < 2 * (a - n)) diff = 2 * (a - n); } return diff; } -void InvHSqueeze(Image &input, int c, int rc, ThreadPool *pool); - -void InvVSqueeze(Image &input, int c, int rc, ThreadPool *pool); - void DefaultSqueezeParameters(std::vector *parameters, const Image &image); Status CheckMetaSqueezeParams(const SqueezeParams ¶meter, int num_channels); diff --git a/third_party/jpeg-xl/lib/jxl/modular/transform/squeeze.cc b/third_party/jpeg-xl/lib/jxl/modular/transform/squeeze.cc --- a/third_party/jpeg-xl/lib/jxl/modular/transform/squeeze.cc +++ b/third_party/jpeg-xl/lib/jxl/modular/transform/squeeze.cc @@ -10,17 +10,84 @@ #include "lib/jxl/base/data_parallel.h" #include "lib/jxl/base/printf_macros.h" #include "lib/jxl/common.h" #include "lib/jxl/modular/modular_image.h" #include "lib/jxl/modular/transform/transform.h" +#undef HWY_TARGET_INCLUDE +#define HWY_TARGET_INCLUDE "lib/jxl/modular/transform/squeeze.cc" +#include +#include +#include "lib/jxl/simd_util-inl.h" + +HWY_BEFORE_NAMESPACE(); namespace jxl { +namespace HWY_NAMESPACE { + +using hwy::HWY_NAMESPACE::RebindToUnsigned; +using hwy::HWY_NAMESPACE::ShiftLeft; +using hwy::HWY_NAMESPACE::ShiftRight; + +#if HWY_TARGET != HWY_SCALAR + +JXL_INLINE void FastUnsqueeze(const pixel_type *JXL_RESTRICT p_residual, + const pixel_type *JXL_RESTRICT p_avg, + const pixel_type *JXL_RESTRICT p_navg, + const pixel_type *p_pout, + pixel_type *JXL_RESTRICT p_out, + pixel_type *p_nout) { + const HWY_CAPPED(pixel_type, 8) d; + const RebindToUnsigned du; + const size_t N = Lanes(d); + auto onethird = Set(d, 0x55555556); + for (size_t x = 0; x < 8; x += N) { + auto avg = Load(d, p_avg + x); + auto next_avg = Load(d, p_navg + x); + auto top = Load(d, p_pout + x); + // Equivalent to SmoothTendency(top,avg,next_avg), but without branches + auto Ba = top - avg; + auto an = avg - next_avg; + auto nonmono = Ba ^ an; + auto absBa = Abs(Ba); + auto absan = Abs(an); + auto absBn = Abs(top - next_avg); + // Compute a3 = absBa / 3 + auto a3e = BitCast(d, ShiftRight<32>(MulEven(absBa, onethird))); + auto a3oi = MulEven(Reverse(d, absBa), onethird); + auto a3o = BitCast( + d, Reverse(hwy::HWY_NAMESPACE::Repartition(), + a3oi)); + auto a3 = OddEven(a3o, a3e); + a3 += absBn + Set(d, 2); + auto absdiff = ShiftRight<2>(a3); + auto skipdiff = Ba != Zero(d); + skipdiff = And(skipdiff, an != Zero(d)); + skipdiff = And(skipdiff, nonmono < Zero(d)); + auto absBa2 = ShiftLeft<1>(absBa) + (absdiff & Set(d, 1)); + absdiff = + IfThenElse(absdiff > absBa2, ShiftLeft<1>(absBa) + Set(d, 1), absdiff); + auto absan2 = ShiftLeft<1>(absan); + absdiff = + IfThenElse(absdiff + (absdiff & Set(d, 1)) > absan2, absan2, absdiff); + auto diff1 = IfThenElse(top < next_avg, Neg(absdiff), absdiff); + auto tendency = IfThenZeroElse(skipdiff, diff1); + + auto diff_minus_tendency = Load(d, p_residual + x); + auto diff = diff_minus_tendency + tendency; + auto out = avg + ShiftRight<1>( + diff + BitCast(d, ShiftRight<31>(BitCast(du, diff)))); + Store(out, d, p_out + x); + Store(out - diff, d, p_nout + x); + } +} + +#endif Status InvHSqueeze(Image &input, uint32_t c, uint32_t rc, ThreadPool *pool) { JXL_ASSERT(c < input.channel.size()); JXL_ASSERT(rc < input.channel.size()); - const Channel &chin = input.channel[c]; + Channel &chin = input.channel[c]; const Channel &chin_residual = input.channel[rc]; // These must be valid since we ran MetaApply already. JXL_ASSERT(chin.w == DivCeil(chin.w + chin_residual.w, 2)); JXL_ASSERT(chin.h == chin_residual.h); @@ -40,46 +107,88 @@ if (chin_residual.h == 0) { // Short-circuit: channel with no pixels. input.channel[c] = std::move(chout); return true; } + auto unsqueeze_row = [&](size_t y, size_t x0) { + const pixel_type *JXL_RESTRICT p_residual = chin_residual.Row(y); + const pixel_type *JXL_RESTRICT p_avg = chin.Row(y); + pixel_type *JXL_RESTRICT p_out = chout.Row(y); + for (size_t x = x0; x < chin_residual.w; x++) { + pixel_type diff_minus_tendency = p_residual[x]; + pixel_type avg = p_avg[x]; + pixel_type next_avg = (x + 1 < chin.w ? p_avg[x + 1] : avg); + pixel_type left = (x ? p_out[(x << 1) - 1] : avg); + pixel_type tendency = SmoothTendency(left, avg, next_avg); + pixel_type diff = diff_minus_tendency + tendency; + pixel_type A = avg + (diff / 2); + p_out[(x << 1)] = A; + pixel_type B = A - diff; + p_out[(x << 1) + 1] = B; + } + if (chout.w & 1) p_out[chout.w - 1] = p_avg[chin.w - 1]; + }; - JXL_RETURN_IF_ERROR(RunOnPool( - pool, 0, chin.h, ThreadPool::NoInit, - [&](const uint32_t task, size_t /* thread */) { - const size_t y = task; - const pixel_type *JXL_RESTRICT p_residual = chin_residual.Row(y); - const pixel_type *JXL_RESTRICT p_avg = chin.Row(y); - pixel_type *JXL_RESTRICT p_out = chout.Row(y); + // somewhat complicated trickery just to be able to SIMD this. + // Horizontal unsqueeze has horizontal data dependencies, so we do + // 8 rows at a time and treat it as a vertical unsqueeze of a + // transposed 8x8 block (or 9x8 for one input). + static constexpr const size_t kRowsPerThread = 8; + const auto unsqueeze_span = [&](const uint32_t task, size_t /* thread */) { + const size_t y0 = task * kRowsPerThread; + const size_t rows = std::min(kRowsPerThread, chin.h - y0); + size_t x = 0; - // special case for x=0 so we don't have to check x>0 - pixel_type_w avg = p_avg[0]; - pixel_type_w next_avg = (1 < chin.w ? p_avg[1] : avg); - pixel_type_w tendency = SmoothTendency(avg, avg, next_avg); - pixel_type_w diff = p_residual[0] + tendency; - pixel_type_w A = - ((avg * 2) + diff + (diff > 0 ? -(diff & 1) : (diff & 1))) >> 1; - pixel_type_w B = A - diff; - p_out[0] = A; - p_out[1] = B; +#if HWY_TARGET != HWY_SCALAR + intptr_t onerow_in = chin.plane.PixelsPerRow(); + intptr_t onerow_inr = chin_residual.plane.PixelsPerRow(); + intptr_t onerow_out = chout.plane.PixelsPerRow(); + const pixel_type *JXL_RESTRICT p_residual = chin_residual.Row(y0); + const pixel_type *JXL_RESTRICT p_avg = chin.Row(y0); + pixel_type *JXL_RESTRICT p_out = chout.Row(y0); + HWY_ALIGN pixel_type b_p_avg[9 * kRowsPerThread]; + HWY_ALIGN pixel_type b_p_residual[8 * kRowsPerThread]; + HWY_ALIGN pixel_type b_p_out_even[8 * kRowsPerThread]; + HWY_ALIGN pixel_type b_p_out_odd[8 * kRowsPerThread]; + HWY_ALIGN pixel_type b_p_out_evenT[8 * kRowsPerThread]; + HWY_ALIGN pixel_type b_p_out_oddT[8 * kRowsPerThread]; + const HWY_CAPPED(pixel_type, 8) d; + const size_t N = Lanes(d); + if (chin_residual.w > 16 && rows == kRowsPerThread) { + for (; x < chin_residual.w - 9; x += 8) { + Transpose8x8Block(p_residual + x, b_p_residual, onerow_inr); + Transpose8x8Block(p_avg + x, b_p_avg, onerow_in); + for (size_t y = 0; y < kRowsPerThread; y++) { + b_p_avg[8 * 8 + y] = p_avg[x + 8 + onerow_in * y]; + } + for (size_t i = 0; i < 8; i++) { + FastUnsqueeze( + b_p_residual + 8 * i, b_p_avg + 8 * i, b_p_avg + 8 * (i + 1), + (x + i ? b_p_out_odd + 8 * ((x + i - 1) & 7) : b_p_avg + 8 * i), + b_p_out_even + 8 * i, b_p_out_odd + 8 * i); + } - for (size_t x = 1; x < chin_residual.w; x++) { - pixel_type_w diff_minus_tendency = p_residual[x]; - pixel_type_w avg = p_avg[x]; - pixel_type_w next_avg = (x + 1 < chin.w ? p_avg[x + 1] : avg); - pixel_type_w left = p_out[(x << 1) - 1]; - pixel_type_w tendency = SmoothTendency(left, avg, next_avg); - pixel_type_w diff = diff_minus_tendency + tendency; - pixel_type_w A = - ((avg * 2) + diff + (diff > 0 ? -(diff & 1) : (diff & 1))) >> 1; - p_out[x << 1] = A; - pixel_type_w B = A - diff; - p_out[(x << 1) + 1] = B; + Transpose8x8Block(b_p_out_even, b_p_out_evenT, 8); + Transpose8x8Block(b_p_out_odd, b_p_out_oddT, 8); + for (size_t y = 0; y < kRowsPerThread; y++) { + for (size_t i = 0; i < kRowsPerThread; i += N) { + auto even = Load(d, b_p_out_evenT + 8 * y + i); + auto odd = Load(d, b_p_out_oddT + 8 * y + i); + StoreInterleaved(d, even, odd, + p_out + ((x + i) << 1) + onerow_out * y); + } } - if (chout.w & 1) p_out[chout.w - 1] = p_avg[chin.w - 1]; - }, - "InvHorizontalSqueeze")); + } + } +#endif + for (size_t y = 0; y < rows; y++) { + unsqueeze_row(y0 + y, x); + } + }; + JXL_RETURN_IF_ERROR(RunOnPool(pool, 0, DivCeil(chin.h, kRowsPerThread), + ThreadPool::NoInit, unsqueeze_span, + "InvHorizontalSqueeze")); input.channel[c] = std::move(chout); return true; } Status InvVSqueeze(Image &input, uint32_t c, uint32_t rc, ThreadPool *pool) { @@ -109,46 +218,51 @@ // Short-circuit: channel with no pixels. input.channel[c] = std::move(chout); return true; } - intptr_t onerow_in = chin.plane.PixelsPerRow(); - intptr_t onerow_out = chout.plane.PixelsPerRow(); - constexpr int kColsPerThread = 64; - JXL_RETURN_IF_ERROR(RunOnPool( - pool, 0, DivCeil(chin.w, kColsPerThread), ThreadPool::NoInit, - [&](const uint32_t task, size_t /* thread */) { - const size_t x0 = task * kColsPerThread; - const size_t x1 = std::min((size_t)(task + 1) * kColsPerThread, chin.w); - // We only iterate up to std::min(chin_residual.h, chin.h) which is - // always chin_residual.h. - for (size_t y = 0; y < chin_residual.h; y++) { - const pixel_type *JXL_RESTRICT p_residual = chin_residual.Row(y); - const pixel_type *JXL_RESTRICT p_avg = chin.Row(y); - pixel_type *JXL_RESTRICT p_out = chout.Row(y << 1); - for (size_t x = x0; x < x1; x++) { - pixel_type_w diff_minus_tendency = p_residual[x]; - pixel_type_w avg = p_avg[x]; - - pixel_type_w next_avg = avg; - if (y + 1 < chin.h) next_avg = p_avg[x + onerow_in]; - pixel_type_w top = - (y > 0 ? p_out[static_cast(x) - onerow_out] : avg); - pixel_type_w tendency = SmoothTendency(top, avg, next_avg); - pixel_type_w diff = diff_minus_tendency + tendency; - pixel_type_w out = - ((avg * 2) + diff + (diff > 0 ? -(diff & 1) : (diff & 1))) >> 1; - - p_out[x] = out; - // If the chin_residual.h == chin.h, the output has an even number - // of rows so the next line is fine. Otherwise, this loop won't - // write to the last output row which is handled separately. - p_out[x + onerow_out] = p_out[x] - diff; - } - } - }, - "InvVertSqueeze")); + static constexpr const int kColsPerThread = 64; + const auto unsqueeze_slice = [&](const uint32_t task, size_t /* thread */) { + const size_t x0 = task * kColsPerThread; + const size_t x1 = std::min((size_t)(task + 1) * kColsPerThread, chin.w); + const size_t w = x1 - x0; + // We only iterate up to std::min(chin_residual.h, chin.h) which is + // always chin_residual.h. + for (size_t y = 0; y < chin_residual.h; y++) { + const pixel_type *JXL_RESTRICT p_residual = chin_residual.Row(y) + x0; + const pixel_type *JXL_RESTRICT p_avg = chin.Row(y) + x0; + const pixel_type *JXL_RESTRICT p_navg = + chin.Row(y + 1 < chin.h ? y + 1 : y) + x0; + pixel_type *JXL_RESTRICT p_out = chout.Row(y << 1) + x0; + pixel_type *JXL_RESTRICT p_nout = chout.Row((y << 1) + 1) + x0; + const pixel_type *p_pout = y > 0 ? chout.Row((y << 1) - 1) + x0 : p_avg; + size_t x = 0; +#if HWY_TARGET != HWY_SCALAR + for (; x + 7 < w; x += 8) { + FastUnsqueeze(p_residual + x, p_avg + x, p_navg + x, p_pout + x, + p_out + x, p_nout + x); + } +#endif + for (; x < w; x++) { + pixel_type avg = p_avg[x]; + pixel_type next_avg = p_navg[x]; + pixel_type top = p_pout[x]; + pixel_type tendency = SmoothTendency(top, avg, next_avg); + pixel_type diff_minus_tendency = p_residual[x]; + pixel_type diff = diff_minus_tendency + tendency; + pixel_type out = avg + (diff / 2); + p_out[x] = out; + // If the chin_residual.h == chin.h, the output has an even number + // of rows so the next line is fine. Otherwise, this loop won't + // write to the last output row which is handled separately. + p_nout[x] = out - diff; + } + } + }; + JXL_RETURN_IF_ERROR(RunOnPool(pool, 0, DivCeil(chin.w, kColsPerThread), + ThreadPool::NoInit, unsqueeze_slice, + "InvVertSqueeze")); if (chout.h & 1) { size_t y = chin.h - 1; const pixel_type *p_avg = chin.Row(y); pixel_type *p_out = chout.Row(y << 1); @@ -158,10 +272,66 @@ } input.channel[c] = std::move(chout); return true; } +Status InvSqueeze(Image &input, std::vector parameters, + ThreadPool *pool) { + for (int i = parameters.size() - 1; i >= 0; i--) { + JXL_RETURN_IF_ERROR( + CheckMetaSqueezeParams(parameters[i], input.channel.size())); + bool horizontal = parameters[i].horizontal; + bool in_place = parameters[i].in_place; + uint32_t beginc = parameters[i].begin_c; + uint32_t endc = parameters[i].begin_c + parameters[i].num_c - 1; + uint32_t offset; + if (in_place) { + offset = endc + 1; + } else { + offset = input.channel.size() + beginc - endc - 1; + } + if (beginc < input.nb_meta_channels) { + // This is checked in MetaSqueeze. + JXL_ASSERT(input.nb_meta_channels > parameters[i].num_c); + input.nb_meta_channels -= parameters[i].num_c; + } + + for (uint32_t c = beginc; c <= endc; c++) { + uint32_t rc = offset + c - beginc; + // MetaApply should imply that `rc` is within range, otherwise there's a + // programming bug. + JXL_ASSERT(rc < input.channel.size()); + if ((input.channel[c].w < input.channel[rc].w) || + (input.channel[c].h < input.channel[rc].h)) { + return JXL_FAILURE("Corrupted squeeze transform"); + } + if (horizontal) { + JXL_RETURN_IF_ERROR(InvHSqueeze(input, c, rc, pool)); + } else { + JXL_RETURN_IF_ERROR(InvVSqueeze(input, c, rc, pool)); + } + } + input.channel.erase(input.channel.begin() + offset, + input.channel.begin() + offset + (endc - beginc + 1)); + } + return true; +} + +} // namespace HWY_NAMESPACE +} // namespace jxl +HWY_AFTER_NAMESPACE(); + +#if HWY_ONCE + +namespace jxl { + +HWY_EXPORT(InvSqueeze); +Status InvSqueeze(Image &input, std::vector parameters, + ThreadPool *pool) { + return HWY_DYNAMIC_DISPATCH(InvSqueeze)(input, parameters, pool); +} + void DefaultSqueezeParameters(std::vector *parameters, const Image &image) { int nb_channels = image.channel.size() - image.nb_meta_channels; parameters->clear(); @@ -283,48 +453,8 @@ } } return true; } -Status InvSqueeze(Image &input, std::vector parameters, - ThreadPool *pool) { - for (int i = parameters.size() - 1; i >= 0; i--) { - JXL_RETURN_IF_ERROR( - CheckMetaSqueezeParams(parameters[i], input.channel.size())); - bool horizontal = parameters[i].horizontal; - bool in_place = parameters[i].in_place; - uint32_t beginc = parameters[i].begin_c; - uint32_t endc = parameters[i].begin_c + parameters[i].num_c - 1; - uint32_t offset; - if (in_place) { - offset = endc + 1; - } else { - offset = input.channel.size() + beginc - endc - 1; - } - if (beginc < input.nb_meta_channels) { - // This is checked in MetaSqueeze. - JXL_ASSERT(input.nb_meta_channels > parameters[i].num_c); - input.nb_meta_channels -= parameters[i].num_c; - } +} // namespace jxl - for (uint32_t c = beginc; c <= endc; c++) { - uint32_t rc = offset + c - beginc; - // MetaApply should imply that `rc` is within range, otherwise there's a - // programming bug. - JXL_ASSERT(rc < input.channel.size()); - if ((input.channel[c].w < input.channel[rc].w) || - (input.channel[c].h < input.channel[rc].h)) { - return JXL_FAILURE("Corrupted squeeze transform"); - } - if (horizontal) { - JXL_RETURN_IF_ERROR(InvHSqueeze(input, c, rc, pool)); - } else { - JXL_RETURN_IF_ERROR(InvVSqueeze(input, c, rc, pool)); - } - } - input.channel.erase(input.channel.begin() + offset, - input.channel.begin() + offset + (endc - beginc + 1)); - } - return true; -} - -} // namespace jxl +#endif \ No newline at end of file diff --git a/third_party/jpeg-xl/lib/jxl/modular_test.cc b/third_party/jpeg-xl/lib/jxl/modular_test.cc --- a/third_party/jpeg-xl/lib/jxl/modular_test.cc +++ b/third_party/jpeg-xl/lib/jxl/modular_test.cc @@ -23,10 +23,11 @@ #include "lib/jxl/color_encoding_internal.h" #include "lib/jxl/color_management.h" #include "lib/jxl/dec_file.h" #include "lib/jxl/dec_params.h" #include "lib/jxl/enc_butteraugli_comparator.h" +#include "lib/jxl/enc_butteraugli_pnorm.h" #include "lib/jxl/enc_cache.h" #include "lib/jxl/enc_color_management.h" #include "lib/jxl/enc_file.h" #include "lib/jxl/enc_params.h" #include "lib/jxl/image.h" @@ -43,15 +44,14 @@ using test::Roundtrip; void TestLosslessGroups(size_t group_size_shift) { ThreadPool* pool = nullptr; const PaddedBytes orig = - ReadTestData("imagecompression.info/flower_foveon.png"); + ReadTestData("third_party/imagecompression.info/flower_foveon.png"); CompressParams cparams; - cparams.modular_mode = true; + cparams.SetLossless(); cparams.modular_group_size_shift = group_size_shift; - cparams.color_transform = jxl::ColorTransform::kNone; DecompressParams dparams; CodecInOut io_out; size_t compressed_size; @@ -77,19 +77,18 @@ } TEST(ModularTest, RoundtripLosslessCustomWP_PermuteRCT) { ThreadPool* pool = nullptr; const PaddedBytes orig = - ReadTestData("wesaturate/500px/u76c0g_bliznaca_srgb8.png"); + ReadTestData("third_party/wesaturate/500px/u76c0g_bliznaca_srgb8.png"); CompressParams cparams; - cparams.modular_mode = true; + cparams.SetLossless(); // 9 = permute to GBR, to test the special case of permutation-only cparams.colorspace = 9; // slowest speed so different WP modes are tried cparams.speed_tier = SpeedTier::kTortoise; cparams.options.predictor = {Predictor::Weighted}; - cparams.color_transform = jxl::ColorTransform::kNone; DecompressParams dparams; CodecInOut io_out; size_t compressed_size; @@ -105,11 +104,11 @@ } TEST(ModularTest, RoundtripLossyDeltaPalette) { ThreadPool* pool = nullptr; const PaddedBytes orig = - ReadTestData("wesaturate/500px/u76c0g_bliznaca_srgb8.png"); + ReadTestData("third_party/wesaturate/500px/u76c0g_bliznaca_srgb8.png"); CompressParams cparams; cparams.modular_mode = true; cparams.color_transform = jxl::ColorTransform::kNone; cparams.lossy_palette = true; cparams.palette_colors = 0; @@ -131,14 +130,13 @@ IsSlightlyBelow(1.5)); } TEST(ModularTest, RoundtripLossyDeltaPaletteWP) { ThreadPool* pool = nullptr; const PaddedBytes orig = - ReadTestData("wesaturate/500px/u76c0g_bliznaca_srgb8.png"); + ReadTestData("third_party/wesaturate/500px/u76c0g_bliznaca_srgb8.png"); CompressParams cparams; - cparams.modular_mode = true; - cparams.color_transform = jxl::ColorTransform::kNone; + cparams.SetLossless(); cparams.lossy_palette = true; cparams.palette_colors = 0; cparams.options.predictor = jxl::Predictor::Weighted; DecompressParams dparams; @@ -159,37 +157,37 @@ } TEST(ModularTest, RoundtripLossy) { ThreadPool* pool = nullptr; const PaddedBytes orig = - ReadTestData("wesaturate/500px/u76c0g_bliznaca_srgb8.png"); + ReadTestData("third_party/wesaturate/500px/u76c0g_bliznaca_srgb8.png"); CompressParams cparams; cparams.modular_mode = true; - cparams.quality_pair = {80.0f, 80.0f}; + cparams.butteraugli_distance = 2.f; DecompressParams dparams; CodecInOut io_out; size_t compressed_size; CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, pool)); compressed_size = Roundtrip(&io, cparams, dparams, pool, &io_out); - EXPECT_LE(compressed_size, 40000u); + EXPECT_LE(compressed_size, 30000u); cparams.ba_params.intensity_target = 80.0f; EXPECT_THAT(ButteraugliDistance(io, io_out, cparams.ba_params, GetJxlCms(), /*distmap=*/nullptr, pool), - IsSlightlyBelow(2.0)); + IsSlightlyBelow(2.3)); } TEST(ModularTest, RoundtripLossy16) { ThreadPool* pool = nullptr; const PaddedBytes orig = - ReadTestData("raw.pixls/DJI-FC6310-16bit_709_v4_krita.png"); + ReadTestData("third_party/raw.pixls/DJI-FC6310-16bit_709_v4_krita.png"); CompressParams cparams; cparams.modular_mode = true; - cparams.quality_pair = {80.0f, 80.0f}; + cparams.butteraugli_distance = 2.f; DecompressParams dparams; CodecInOut io_out; size_t compressed_size; @@ -197,15 +195,15 @@ ASSERT_TRUE(SetFromBytes(Span(orig), &io, pool)); JXL_CHECK(io.TransformTo(ColorEncoding::SRGB(), GetJxlCms(), pool)); io.metadata.m.color_encoding = ColorEncoding::SRGB(); compressed_size = Roundtrip(&io, cparams, dparams, pool, &io_out); - EXPECT_LE(compressed_size, 400u); + EXPECT_LE(compressed_size, 300u); cparams.ba_params.intensity_target = 80.0f; EXPECT_THAT(ButteraugliDistance(io, io_out, cparams.ba_params, GetJxlCms(), /*distmap=*/nullptr, pool), - IsSlightlyBelow(1.5)); + IsSlightlyBelow(1.6)); } TEST(ModularTest, RoundtripExtraProperties) { constexpr size_t kSize = 250; Image image(kSize, kSize, /*bitdepth=*/8, 3); @@ -248,19 +246,19 @@ } } TEST(ModularTest, RoundtripLosslessCustomSqueeze) { ThreadPool* pool = nullptr; - const PaddedBytes orig = - ReadTestData("wesaturate/500px/tmshre_riaphotographs_srgb8.png"); + const PaddedBytes orig = ReadTestData( + "third_party/wesaturate/500px/tmshre_riaphotographs_srgb8.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, pool)); CompressParams cparams; cparams.modular_mode = true; cparams.color_transform = jxl::ColorTransform::kNone; - cparams.quality_pair = {100, 100}; + cparams.butteraugli_distance = 0.f; cparams.options.predictor = {Predictor::Zero}; cparams.speed_tier = SpeedTier::kThunder; cparams.responsive = 1; // Custom squeeze params, atm just for testing SqueezeParams p; @@ -279,10 +277,109 @@ EXPECT_LE(Roundtrip(&io, cparams, dparams, pool, &io2), 265000u); EXPECT_EQ(0.0, ButteraugliDistance(io, io2, cparams.ba_params, GetJxlCms(), /*distmap=*/nullptr, pool)); } +struct RoundtripLosslessConfig { + int bitdepth; + int responsive; +}; +class ModularTestParam + : public ::testing::TestWithParam {}; + +std::vector GenerateLosslessTests() { + std::vector all; + for (int responsive = 0; responsive <= 1; responsive++) { + for (int bitdepth = 1; bitdepth < 32; bitdepth++) { + if (responsive && bitdepth > 30) continue; + all.push_back({bitdepth, responsive}); + } + } + return all; +} +std::string LosslessTestDescription( + const testing::TestParamInfo& info) { + std::stringstream name; + name << info.param.bitdepth << "bit"; + if (info.param.responsive) name << "Squeeze"; + return name.str(); +} + +JXL_GTEST_INSTANTIATE_TEST_SUITE_P(RoundtripLossless, ModularTestParam, + testing::ValuesIn(GenerateLosslessTests()), + LosslessTestDescription); + +TEST_P(ModularTestParam, RoundtripLossless) { + RoundtripLosslessConfig config = GetParam(); + int bitdepth = config.bitdepth; + int responsive = config.responsive; + + ThreadPool* pool = nullptr; + Rng generator(123); + const PaddedBytes orig = + ReadTestData("third_party/wesaturate/500px/u76c0g_bliznaca_srgb8.png"); + CodecInOut io1; + ASSERT_TRUE(SetFromBytes(Span(orig), &io1, pool)); + + // vary the dimensions a bit, in case of bugs related to + // even vs odd width or height. + size_t xsize = 423 + bitdepth; + size_t ysize = 467 + bitdepth; + + CodecInOut io; + io.SetSize(xsize, ysize); + io.metadata.m.color_encoding = jxl::ColorEncoding::SRGB(false); + io.metadata.m.SetUintSamples(bitdepth); + + double factor = ((1lu << bitdepth) - 1lu); + double ifactor = 1.0 / factor; + Image3F noise_added(xsize, ysize); + + for (size_t c = 0; c < 3; c++) { + for (size_t y = 0; y < ysize; y++) { + const float* in = io1.Main().color()->PlaneRow(c, y); + float* out = noise_added.PlaneRow(c, y); + for (size_t x = 0; x < xsize; x++) { + // make the least significant bits random + float f = in[x] + generator.UniformF(0.0f, 1.f / 255.f); + if (f > 1.f) f = 1.f; + // quantize to the bitdepth we're testing + unsigned int u = f * factor + 0.5; + out[x] = u * ifactor; + } + } + } + io.SetFromImage(std::move(noise_added), jxl::ColorEncoding::SRGB(false)); + + CompressParams cparams; + cparams.modular_mode = true; + cparams.color_transform = jxl::ColorTransform::kNone; + cparams.butteraugli_distance = 0.f; + cparams.options.predictor = {Predictor::Zero}; + cparams.speed_tier = SpeedTier::kThunder; + cparams.responsive = responsive; + DecompressParams dparams; + CodecInOut io2; + EXPECT_LE(Roundtrip(&io, cparams, dparams, pool, &io2), + bitdepth * xsize * ysize / 3); + EXPECT_LE(0, ComputeDistance2(io.Main(), io2.Main(), GetJxlCms())); + size_t different = 0; + for (size_t c = 0; c < 3; c++) { + for (size_t y = 0; y < ysize; y++) { + const float* in = io.Main().color()->PlaneRow(c, y); + const float* out = io2.Main().color()->PlaneRow(c, y); + for (size_t x = 0; x < xsize; x++) { + uint32_t uin = in[x] * factor + 0.5; + uint32_t uout = out[x] * factor + 0.5; + // check that the integer values are identical + if (uin != uout) different++; + } + } + } + EXPECT_EQ(different, 0); +} + TEST(ModularTest, RoundtripLosslessCustomFloat) { ThreadPool* pool = nullptr; CodecInOut io; size_t xsize = 100, ysize = 300; io.SetSize(xsize, ysize); @@ -308,11 +405,11 @@ io.metadata.m.SetIntensityTarget(255); CompressParams cparams; cparams.modular_mode = true; cparams.color_transform = jxl::ColorTransform::kNone; - cparams.quality_pair = {100, 100}; + cparams.butteraugli_distance = 0.f; cparams.options.predictor = {Predictor::Zero}; cparams.speed_tier = SpeedTier::kThunder; cparams.decoding_speed_tier = 2; DecompressParams dparams; diff --git a/third_party/jpeg-xl/lib/jxl/passes_test.cc b/third_party/jpeg-xl/lib/jxl/passes_test.cc --- a/third_party/jpeg-xl/lib/jxl/passes_test.cc +++ b/third_party/jpeg-xl/lib/jxl/passes_test.cc @@ -35,11 +35,11 @@ using test::Roundtrip; TEST(PassesTest, RoundtripSmallPasses) { ThreadPool* pool = nullptr; const PaddedBytes orig = - ReadTestData("wesaturate/500px/u76c0g_bliznaca_srgb8.png"); + ReadTestData("third_party/wesaturate/500px/u76c0g_bliznaca_srgb8.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, pool)); io.ShrinkTo(io.xsize() / 8, io.ysize() / 8); CompressParams cparams; @@ -55,11 +55,11 @@ } TEST(PassesTest, RoundtripUnalignedPasses) { ThreadPool* pool = nullptr; const PaddedBytes orig = - ReadTestData("wesaturate/500px/u76c0g_bliznaca_srgb8.png"); + ReadTestData("third_party/wesaturate/500px/u76c0g_bliznaca_srgb8.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, pool)); io.ShrinkTo(io.xsize() / 12, io.ysize() / 7); CompressParams cparams; @@ -75,11 +75,11 @@ } TEST(PassesTest, RoundtripMultiGroupPasses) { ThreadPoolInternal pool(4); const PaddedBytes orig = - ReadTestData("imagecompression.info/flower_foveon.png"); + ReadTestData("third_party/imagecompression.info/flower_foveon.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); io.ShrinkTo(600, 1024); // partial X, full Y group CompressParams cparams; @@ -102,11 +102,11 @@ } TEST(PassesTest, RoundtripLargeFastPasses) { ThreadPoolInternal pool(8); const PaddedBytes orig = - ReadTestData("imagecompression.info/flower_foveon.png"); + ReadTestData("third_party/imagecompression.info/flower_foveon.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); CompressParams cparams; cparams.speed_tier = SpeedTier::kSquirrel; @@ -121,11 +121,11 @@ // which involves additional processing including adaptive reconstruction. // Failing this may be a sign of race conditions or invalid memory accesses. TEST(PassesTest, RoundtripProgressiveConsistent) { ThreadPoolInternal pool(8); const PaddedBytes orig = - ReadTestData("imagecompression.info/flower_foveon.png"); + ReadTestData("third_party/imagecompression.info/flower_foveon.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); CompressParams cparams; cparams.speed_tier = SpeedTier::kSquirrel; @@ -158,11 +158,11 @@ } TEST(PassesTest, AllDownsampleFeasible) { ThreadPoolInternal pool(8); const PaddedBytes orig = - ReadTestData("wesaturate/500px/u76c0g_bliznaca_srgb8.png"); + ReadTestData("third_party/wesaturate/500px/u76c0g_bliznaca_srgb8.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); PaddedBytes compressed; AuxOut aux; @@ -205,11 +205,11 @@ } TEST(PassesTest, AllDownsampleFeasibleQProgressive) { ThreadPoolInternal pool(8); const PaddedBytes orig = - ReadTestData("wesaturate/500px/u76c0g_bliznaca_srgb8.png"); + ReadTestData("third_party/wesaturate/500px/u76c0g_bliznaca_srgb8.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); PaddedBytes compressed; AuxOut aux; @@ -251,12 +251,12 @@ check, "TestQProgressive")); } TEST(PassesTest, ProgressiveDownsample2DegradesCorrectlyGrayscale) { ThreadPoolInternal pool(8); - const PaddedBytes orig = - ReadTestData("wesaturate/500px/cvo9xd_keong_macan_grayscale.png"); + const PaddedBytes orig = ReadTestData( + "third_party/wesaturate/500px/cvo9xd_keong_macan_grayscale.png"); CodecInOut io_orig; ASSERT_TRUE(SetFromBytes(Span(orig), &io_orig, &pool)); Rect rect(0, 0, io_orig.xsize(), 128); // need 2 DC groups for the DC frame to actually be progressive. Image3F large(4242, rect.ysize()); @@ -300,11 +300,11 @@ } TEST(PassesTest, ProgressiveDownsample2DegradesCorrectly) { ThreadPoolInternal pool(8); const PaddedBytes orig = - ReadTestData("imagecompression.info/flower_foveon.png"); + ReadTestData("third_party/imagecompression.info/flower_foveon.png"); CodecInOut io_orig; ASSERT_TRUE(SetFromBytes(Span(orig), &io_orig, &pool)); Rect rect(0, 0, io_orig.xsize(), 128); // need 2 DC groups for the DC frame to actually be progressive. Image3F large(4242, rect.ysize()); @@ -347,11 +347,11 @@ } TEST(PassesTest, NonProgressiveDCImage) { ThreadPoolInternal pool(8); const PaddedBytes orig = - ReadTestData("imagecompression.info/flower_foveon.png"); + ReadTestData("third_party/imagecompression.info/flower_foveon.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); PaddedBytes compressed; AuxOut aux; @@ -375,11 +375,11 @@ } TEST(PassesTest, RoundtripSmallNoGaborishPasses) { ThreadPool* pool = nullptr; const PaddedBytes orig = - ReadTestData("wesaturate/500px/u76c0g_bliznaca_srgb8.png"); + ReadTestData("third_party/wesaturate/500px/u76c0g_bliznaca_srgb8.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, pool)); io.ShrinkTo(io.xsize() / 8, io.ysize() / 8); CompressParams cparams; diff --git a/third_party/jpeg-xl/lib/jxl/patch_dictionary_test.cc b/third_party/jpeg-xl/lib/jxl/patch_dictionary_test.cc --- a/third_party/jpeg-xl/lib/jxl/patch_dictionary_test.cc +++ b/third_party/jpeg-xl/lib/jxl/patch_dictionary_test.cc @@ -22,12 +22,11 @@ const PaddedBytes orig = ReadTestData("jxl/grayscale_patches.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, pool)); CompressParams cparams; - cparams.color_transform = jxl::ColorTransform::kNone; - cparams.modular_mode = true; + cparams.SetLossless(); cparams.patches = jxl::Override::kOn; DecompressParams dparams; CodecInOut io2; // Without patches: ~25k diff --git a/third_party/jpeg-xl/lib/jxl/preview_test.cc b/third_party/jpeg-xl/lib/jxl/preview_test.cc --- a/third_party/jpeg-xl/lib/jxl/preview_test.cc +++ b/third_party/jpeg-xl/lib/jxl/preview_test.cc @@ -32,11 +32,11 @@ using test::Roundtrip; TEST(PreviewTest, RoundtripGivenPreview) { ThreadPool* pool = nullptr; const PaddedBytes orig = - ReadTestData("wesaturate/500px/u76c0g_bliznaca_srgb8.png"); + ReadTestData("third_party/wesaturate/500px/u76c0g_bliznaca_srgb8.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, pool)); io.ShrinkTo(io.xsize() / 8, io.ysize() / 8); // Same as main image io.preview_frame = io.Main().Copy(); diff --git a/third_party/jpeg-xl/lib/jxl/quantizer-inl.h b/third_party/jpeg-xl/lib/jxl/quantizer-inl.h --- a/third_party/jpeg-xl/lib/jxl/quantizer-inl.h +++ b/third_party/jpeg-xl/lib/jxl/quantizer-inl.h @@ -26,11 +26,10 @@ HWY_INLINE HWY_MAYBE_UNUSED Vec> AdjustQuantBias( DI di, const size_t c, const Vec quant_i, const float* HWY_RESTRICT biases) { const Rebind df; -#if JXL_HIGH_PRECISION const auto quant = ConvertTo(df, quant_i); // Compare |quant|, keep sign bit for negating result. const auto kSign = BitCast(df, Set(di, INT32_MIN)); const auto sign = And(quant, kSign); // TODO(janwas): = abs ^ orig @@ -54,16 +53,10 @@ // About 2E-5 worse than ReciprocalNR or division. const auto bias = NegMulAdd(Set(df, biases[3]), ApproximateReciprocal(quant), quant); return IfThenElse(is_01, one_bias, bias); -#else - auto sign = IfThenElseZero(quant_i < Zero(di), Set(di, INT32_MIN)); - return BitCast(df, IfThenElse(Abs(quant_i) == Set(di, 1), - sign | BitCast(di, Set(df, biases[c])), - BitCast(di, ConvertTo(df, quant_i)))); -#endif } } // namespace // NOLINTNEXTLINE(google-readability-namespace-comments) } // namespace HWY_NAMESPACE diff --git a/third_party/jpeg-xl/lib/jxl/render_pipeline/low_memory_render_pipeline.h b/third_party/jpeg-xl/lib/jxl/render_pipeline/low_memory_render_pipeline.h --- a/third_party/jpeg-xl/lib/jxl/render_pipeline/low_memory_render_pipeline.h +++ b/third_party/jpeg-xl/lib/jxl/render_pipeline/low_memory_render_pipeline.h @@ -44,15 +44,13 @@ std::pair BorderToStore(size_t c) const; bool use_group_ids_; - // Storage for borders between groups. - // We also store the extremal borders for simplicity. Horizontal borders are - // stored in an image as wide as the main frame, in top-to-bottom order (top - // border of a group first, followed by the bottom border, followed by top - // border of the next group). Vertical borders are similarly stored. + // Storage for borders between groups. Borders of adjacent groups are stacked + // together, e.g. bottom border of current group is followed by top border + // of next group. std::vector borders_horizontal_; std::vector borders_vertical_; // Manages the status of borders. GroupBorderAssigner group_border_assigner_; diff --git a/third_party/jpeg-xl/lib/jxl/render_pipeline/low_memory_render_pipeline.cc b/third_party/jpeg-xl/lib/jxl/render_pipeline/low_memory_render_pipeline.cc --- a/third_party/jpeg-xl/lib/jxl/render_pipeline/low_memory_render_pipeline.cc +++ b/third_party/jpeg-xl/lib/jxl/render_pipeline/low_memory_render_pipeline.cc @@ -48,28 +48,35 @@ auto borders = BorderToStore(c); size_t borderx_write = borders.first; size_t bordery_write = borders.second; - CopyImageTo( - Rect(group_data_x_border_, group_data_y_border_, x1 - x0, bordery_write), - in, Rect(x0, (gy * 2) * bordery_write, x1 - x0, bordery_write), - &borders_horizontal_[c]); - CopyImageTo( - Rect(group_data_x_border_, group_data_y_border_ + y1 - y0 - bordery_write, - x1 - x0, bordery_write), - in, Rect(x0, (gy * 2 + 1) * bordery_write, x1 - x0, bordery_write), - &borders_horizontal_[c]); - CopyImageTo( - Rect(group_data_x_border_, group_data_y_border_, borderx_write, y1 - y0), - in, Rect((gx * 2) * borderx_write, y0, borderx_write, y1 - y0), - &borders_vertical_[c]); - CopyImageTo(Rect(group_data_x_border_ + x1 - x0 - borderx_write, - group_data_y_border_, borderx_write, y1 - y0), - in, - Rect((gx * 2 + 1) * borderx_write, y0, borderx_write, y1 - y0), - &borders_vertical_[c]); + if (gy > 0) { + Rect from(group_data_x_border_, group_data_y_border_, x1 - x0, + bordery_write); + Rect to(x0, (gy * 2 - 1) * bordery_write, x1 - x0, bordery_write); + CopyImageTo(from, in, to, &borders_horizontal_[c]); + } + if (gy + 1 < frame_dimensions_.ysize_groups) { + Rect from(group_data_x_border_, + group_data_y_border_ + y1 - y0 - bordery_write, x1 - x0, + bordery_write); + Rect to(x0, (gy * 2) * bordery_write, x1 - x0, bordery_write); + CopyImageTo(from, in, to, &borders_horizontal_[c]); + } + if (gx > 0) { + Rect from(group_data_x_border_, group_data_y_border_, borderx_write, + y1 - y0); + Rect to((gx * 2 - 1) * borderx_write, y0, borderx_write, y1 - y0); + CopyImageTo(from, in, to, &borders_vertical_[c]); + } + if (gx + 1 < frame_dimensions_.xsize_groups) { + Rect from(group_data_x_border_ + x1 - x0 - borderx_write, + group_data_y_border_, borderx_write, y1 - y0); + Rect to((gx * 2) * borderx_write, y0, borderx_write, y1 - y0); + CopyImageTo(from, in, to, &borders_vertical_[c]); + } } void LowMemoryRenderPipeline::LoadBorders(size_t group_id, size_t c, const Rect& r, ImageF* out) { size_t gy = group_id / frame_dimensions_.xsize_groups; @@ -117,41 +124,41 @@ // Copy other groups' borders from the border storage. if (y0src < y0) { JXL_DASSERT(gy > 0); CopyImageTo( - Rect(x0src, (gy * 2 - 1) * bordery_write, x1src - x0src, bordery_write), + Rect(x0src, (gy * 2 - 2) * bordery_write, x1src - x0src, bordery_write), borders_horizontal_[c], Rect(group_data_x_border_ + x0src - x0, group_data_y_border_ - bordery_write, x1src - x0src, bordery_write), out); } if (y1src > y1) { // When copying the bottom border we must not be on the bottom groups. JXL_DASSERT(gy + 1 < frame_dimensions_.ysize_groups); CopyImageTo( - Rect(x0src, (gy * 2 + 2) * bordery_write, x1src - x0src, bordery_write), + Rect(x0src, (gy * 2 + 1) * bordery_write, x1src - x0src, bordery_write), borders_horizontal_[c], Rect(group_data_x_border_ + x0src - x0, group_data_y_border_ + y1 - y0, x1src - x0src, bordery_write), out); } if (x0src < x0) { JXL_DASSERT(gx > 0); CopyImageTo( - Rect((gx * 2 - 1) * borderx_write, y0src, borderx_write, y1src - y0src), + Rect((gx * 2 - 2) * borderx_write, y0src, borderx_write, y1src - y0src), borders_vertical_[c], Rect(group_data_x_border_ - borderx_write, group_data_y_border_ + y0src - y0, borderx_write, y1src - y0src), out); } if (x1src > x1) { // When copying the right border we must not be on the rightmost groups. JXL_DASSERT(gx + 1 < frame_dimensions_.xsize_groups); CopyImageTo( - Rect((gx * 2 + 2) * borderx_write, y0src, borderx_write, y1src - y0src), + Rect((gx * 2 + 1) * borderx_write, y0src, borderx_write, y1src - y0src), borders_vertical_[c], Rect(group_data_x_border_ + x1 - x0, group_data_y_border_ + y0src - y0, borderx_write, y1src - y0src), out); } @@ -175,20 +182,23 @@ } for (size_t c = 0; c < shifts.size(); c++) { auto borders = BorderToStore(c); size_t borderx = borders.first; size_t bordery = borders.second; - Rect horizontal = Rect( - 0, 0, - DivCeil(frame_dimensions_.xsize_upsampled_padded, 1 << shifts[c].first), - bordery * frame_dimensions_.ysize_groups * 2); + JXL_DASSERT(frame_dimensions_.xsize_groups > 0); + size_t num_xborders = (frame_dimensions_.xsize_groups - 1) * 2; + JXL_DASSERT(frame_dimensions_.ysize_groups > 0); + size_t num_yborders = (frame_dimensions_.ysize_groups - 1) * 2; + size_t downsampled_xsize = + DivCeil(frame_dimensions_.xsize_upsampled_padded, 1 << shifts[c].first); + size_t downsampled_ysize = DivCeil(frame_dimensions_.ysize_upsampled_padded, + 1 << shifts[c].second); + Rect horizontal = Rect(0, 0, downsampled_xsize, bordery * num_yborders); if (!SameSize(horizontal, borders_horizontal_[c])) { borders_horizontal_[c] = ImageF(horizontal.xsize(), horizontal.ysize()); } - Rect vertical = Rect(0, 0, borderx * frame_dimensions_.xsize_groups * 2, - DivCeil(frame_dimensions_.ysize_upsampled_padded, - 1 << shifts[c].second)); + Rect vertical = Rect(0, 0, borderx * num_xborders, downsampled_ysize); if (!SameSize(vertical, borders_vertical_[c])) { borders_vertical_[c] = ImageF(vertical.xsize(), vertical.ysize()); } } } @@ -361,14 +371,16 @@ GroupInputYSize(c) + group_data_y_border_ * 2); } } // TODO(veluca): avoid reallocating buffers if not needed. stage_data_.resize(num); - size_t stage_buffer_xsize = - (frame_dimensions_.group_dim << base_color_shift_) + - 2 * group_data_x_border_ + // maximum size of a rect - 2 * kRenderPipelineXOffset; // extra padding for processing + size_t upsampling = 1u << base_color_shift_; + size_t group_dim = frame_dimensions_.group_dim * upsampling; + size_t padding = + 2 * group_data_x_border_ * upsampling + // maximum size of a rect + 2 * kRenderPipelineXOffset; // extra padding for processing + size_t stage_buffer_xsize = group_dim + padding; for (size_t t = 0; t < num; t++) { stage_data_[t].resize(shifts.size()); for (size_t c = 0; c < shifts.size(); c++) { stage_data_[t][c].resize(stages_.size()); size_t next_y_border = 0; @@ -384,19 +396,20 @@ } } } if (first_image_dim_stage_ != stages_.size()) { out_of_frame_data_.resize(num); + size_t left_padding = std::max(0, frame_origin_.x0); + size_t middle_padding = group_dim; + ssize_t last_x = + frame_origin_.x0 + std::min(frame_dimensions_.xsize_groups * group_dim, + frame_dimensions_.xsize_upsampled); + last_x = Clamp1(last_x, 0, full_image_xsize_); + size_t right_padding = full_image_xsize_ - last_x; size_t out_of_frame_xsize = - 2 * kRenderPipelineXOffset + - std::max( - 0, - std::max( - frame_origin_.x0, - static_cast(full_image_xsize_) - frame_origin_.x0 - - static_cast(frame_dimensions_.xsize_upsampled))); - out_of_frame_xsize = std::max(out_of_frame_xsize, stage_buffer_xsize); + padding + + std::max(left_padding, std::max(middle_padding, right_padding)); for (size_t t = 0; t < num; t++) { out_of_frame_data_[t] = ImageF(out_of_frame_xsize, shifts.size()); } } } @@ -612,14 +625,14 @@ output_rows[c][iy] = get_row_buffer(i, y * (1 << shifty) + iy, c); } } } // Produce output rows. - stages_[i]->ProcessRow( - input_rows[i], output_rows, xpadding_for_output_[i], - group_rect[i].xsize(), group_rect[i].x0(), group_rect[i].y0() + y, - reinterpret_cast(temp_buffers_[thread_id].get())); + stages_[i]->ProcessRow(input_rows[i], output_rows, + xpadding_for_output_[i], group_rect[i].xsize(), + group_rect[i].x0(), group_rect[i].y0() + y, + thread_id); } // Process trailing stages, i.e. the final set of non-kInOut stages; they // all have the same input buffer and no need to use any mirroring. @@ -630,15 +643,14 @@ input_rows[first_trailing_stage_][c][0] = get_row_buffer( stage_input_for_channel_[first_trailing_stage_][c], y, c); } for (size_t i = first_trailing_stage_; i < first_image_dim_stage_; i++) { - stages_[i]->ProcessRow( - input_rows[first_trailing_stage_], output_rows, - /*xextra=*/0, group_rect[i].xsize(), group_rect[i].x0(), - group_rect[i].y0() + y, - reinterpret_cast(temp_buffers_[thread_id].get())); + stages_[i]->ProcessRow(input_rows[first_trailing_stage_], output_rows, + /*xextra=*/0, group_rect[i].xsize(), + group_rect[i].x0(), group_rect[i].y0() + y, + thread_id); } if (first_image_dim_stage_ == stages_.size()) continue; ssize_t full_image_y = @@ -663,15 +675,13 @@ full_image_x1 = std::min(full_image_x1, full_image_xsize_); if (full_image_x1 <= full_image_x0) continue; for (size_t i = first_image_dim_stage_; i < stages_.size(); i++) { - stages_[i]->ProcessRow( - input_rows[first_trailing_stage_], output_rows, - /*xextra=*/0, full_image_x1 - full_image_x0, full_image_x0, - full_image_y, - reinterpret_cast(temp_buffers_[thread_id].get())); + stages_[i]->ProcessRow(input_rows[first_trailing_stage_], output_rows, + /*xextra=*/0, full_image_x1 - full_image_x0, + full_image_x0, full_image_y, thread_id); } } } void LowMemoryRenderPipeline::RenderPadding(size_t thread_id, Rect rect) { @@ -686,14 +696,13 @@ for (size_t y = 0; y < rect.ysize(); y++) { stages_[first_image_dim_stage_ - 1]->ProcessPaddingRow( input_rows, rect.xsize(), rect.x0(), rect.y0() + y); for (size_t i = first_image_dim_stage_; i < stages_.size(); i++) { - stages_[i]->ProcessRow( - input_rows, output_rows, - /*xextra=*/0, rect.xsize(), rect.x0(), rect.y0() + y, - reinterpret_cast(temp_buffers_[thread_id].get())); + stages_[i]->ProcessRow(input_rows, output_rows, + /*xextra=*/0, rect.xsize(), rect.x0(), + rect.y0() + y, thread_id); } } } void LowMemoryRenderPipeline::ProcessBuffers(size_t group_id, @@ -708,54 +717,54 @@ size_t gy = group_id / frame_dimensions_.xsize_groups; size_t gx = group_id % frame_dimensions_.xsize_groups; if (first_image_dim_stage_ != stages_.size()) { - size_t x0 = std::max( - 0, (gx * frame_dimensions_.group_dim << base_color_shift_) + - frame_origin_.x0); - size_t y0 = std::max( - 0, (gy * frame_dimensions_.group_dim << base_color_shift_) + - frame_origin_.y0); - size_t x1 = std::min( - frame_origin_.x0 + std::min((gx + 1) * frame_dimensions_.group_dim - << base_color_shift_, - frame_dimensions_.xsize_upsampled), - full_image_xsize_); - size_t y1 = std::min( - frame_origin_.y0 + std::min((gy + 1) * frame_dimensions_.group_dim - << base_color_shift_, - frame_dimensions_.ysize_upsampled), - full_image_ysize_); - x0 = std::min(x1, x0); - y0 = std::min(y1, y0); + size_t group_dim = frame_dimensions_.group_dim << base_color_shift_; + RectT group_rect(gx * group_dim, gy * group_dim, group_dim, + group_dim); + RectT image_rect(0, 0, frame_dimensions_.xsize_upsampled, + frame_dimensions_.ysize_upsampled); + RectT full_image_rect(0, 0, full_image_xsize_, full_image_ysize_); + group_rect = group_rect.Translate(frame_origin_.x0, frame_origin_.y0); + image_rect = image_rect.Translate(frame_origin_.x0, frame_origin_.y0); + group_rect = + group_rect.Intersection(image_rect).Intersection(full_image_rect); + size_t x0 = group_rect.x0(); + size_t y0 = group_rect.y0(); + size_t x1 = group_rect.x1(); + size_t y1 = group_rect.y1(); - if (gx == 0 && gy == 0) { - RenderPadding(thread_id, Rect(0, 0, x0, y0)); - } - if (gy == 0) { - RenderPadding(thread_id, Rect(x0, 0, x1 - x0, y0)); - } - if (gx == 0) { - RenderPadding(thread_id, Rect(0, y0, x0, y1 - y0)); - } - if (gx == 0 && gy + 1 == frame_dimensions_.ysize_groups) { - RenderPadding(thread_id, Rect(0, y1, x0, full_image_ysize_ - y1)); - } - if (gy + 1 == frame_dimensions_.ysize_groups) { - RenderPadding(thread_id, Rect(x0, y1, x1 - x0, full_image_ysize_ - y1)); - } - if (gy == 0 && gx + 1 == frame_dimensions_.xsize_groups) { - RenderPadding(thread_id, Rect(x1, 0, full_image_xsize_ - x1, y0)); - } - if (gx + 1 == frame_dimensions_.xsize_groups) { - RenderPadding(thread_id, Rect(x1, y0, full_image_xsize_ - x1, y1 - y0)); - } - if (gy + 1 == frame_dimensions_.ysize_groups && - gx + 1 == frame_dimensions_.xsize_groups) { - RenderPadding(thread_id, Rect(x1, y1, full_image_xsize_ - x1, - full_image_ysize_ - y1)); + // Do not render padding if group is empty; if group is empty x0, y0 might + // have arbitrary values (from frame_origin). + if (group_rect.xsize() > 0 && group_rect.ysize() > 0) { + if (gx == 0 && gy == 0) { + RenderPadding(thread_id, Rect(0, 0, x0, y0)); + } + if (gy == 0) { + RenderPadding(thread_id, Rect(x0, 0, x1 - x0, y0)); + } + if (gx == 0) { + RenderPadding(thread_id, Rect(0, y0, x0, y1 - y0)); + } + if (gx == 0 && gy + 1 == frame_dimensions_.ysize_groups) { + RenderPadding(thread_id, Rect(0, y1, x0, full_image_ysize_ - y1)); + } + if (gy + 1 == frame_dimensions_.ysize_groups) { + RenderPadding(thread_id, Rect(x0, y1, x1 - x0, full_image_ysize_ - y1)); + } + if (gy == 0 && gx + 1 == frame_dimensions_.xsize_groups) { + RenderPadding(thread_id, Rect(x1, 0, full_image_xsize_ - x1, y0)); + } + if (gx + 1 == frame_dimensions_.xsize_groups) { + RenderPadding(thread_id, Rect(x1, y0, full_image_xsize_ - x1, y1 - y0)); + } + if (gy + 1 == frame_dimensions_.ysize_groups && + gx + 1 == frame_dimensions_.xsize_groups) { + RenderPadding(thread_id, Rect(x1, y1, full_image_xsize_ - x1, + full_image_ysize_ - y1)); + } } } Rect ready_rects[GroupBorderAssigner::kMaxToFinalize]; size_t num_ready_rects = 0; diff --git a/third_party/jpeg-xl/lib/jxl/render_pipeline/render_pipeline.h b/third_party/jpeg-xl/lib/jxl/render_pipeline/render_pipeline.h --- a/third_party/jpeg-xl/lib/jxl/render_pipeline/render_pipeline.h +++ b/third_party/jpeg-xl/lib/jxl/render_pipeline/render_pipeline.h @@ -85,11 +85,11 @@ // Allocates storage to run with `num` threads. If `use_group_ids` is true, // storage is allocated for each group, not each thread. The behaviour is // undefined if calling this function multiple times with a different value // for `use_group_ids`. - void PrepareForThreads(size_t num, bool use_group_ids); + Status PrepareForThreads(size_t num, bool use_group_ids); // Retrieves a buffer where input data should be stored by the callee. When // input has been provided for all buffers, the pipeline will complete its // processing. This method may be called multiple times concurrently from // different threads, provided that a different `thread_id` is given. @@ -113,13 +113,10 @@ FrameDimensions frame_dimensions_; std::vector group_completed_passes_; - // Indexed by thread_id - std::vector temp_buffers_; - friend class RenderPipelineInput; private: void InputReady(size_t group_id, size_t thread_id, const std::vector>& buffers); diff --git a/third_party/jpeg-xl/lib/jxl/render_pipeline/render_pipeline.cc b/third_party/jpeg-xl/lib/jxl/render_pipeline/render_pipeline.cc --- a/third_party/jpeg-xl/lib/jxl/render_pipeline/render_pipeline.cc +++ b/third_party/jpeg-xl/lib/jxl/render_pipeline/render_pipeline.cc @@ -114,22 +114,16 @@ } ProcessBuffers(group_id, thread_id); } -void RenderPipeline::PrepareForThreads(size_t num, bool use_group_ids) { - temp_buffers_.resize(num); - for (auto& thread_buffer : temp_buffers_) { - size_t size = 0; - for (const auto& stage : stages_) { - size = std::max(stage->settings_.temp_buffer_size, size); - } - if (size) { - thread_buffer = AllocateArray(sizeof(float) * size); - } +Status RenderPipeline::PrepareForThreads(size_t num, bool use_group_ids) { + for (const auto& stage : stages_) { + JXL_RETURN_IF_ERROR(stage->PrepareForThreads(num)); } PrepareForThreadsInternal(num, use_group_ids); + return true; } void RenderPipelineInput::Done() { JXL_ASSERT(pipeline_); pipeline_->InputReady(group_id_, thread_id_, buffers_); diff --git a/third_party/jpeg-xl/lib/jxl/render_pipeline/render_pipeline_stage.h b/third_party/jpeg-xl/lib/jxl/render_pipeline/render_pipeline_stage.h --- a/third_party/jpeg-xl/lib/jxl/render_pipeline/render_pipeline_stage.h +++ b/third_party/jpeg-xl/lib/jxl/render_pipeline/render_pipeline_stage.h @@ -53,14 +53,10 @@ // Log2 of the number of columns/rows of output that this stage will produce // for every input row for kInOut channels. size_t shift_x = 0; size_t shift_y = 0; - // Size (in floats) of the (aligned) per-thread temporary buffer to pass to - // ProcessRow. - size_t temp_buffer_size = 0; - static Settings ShiftX(size_t shift, size_t border) { Settings settings; settings.border_x = border; settings.shift_x = shift; return settings; @@ -71,12 +67,11 @@ settings.border_y = border; settings.shift_y = shift; return settings; } - static Settings Symmetric(size_t shift, size_t border, - size_t temp_buffer_size = 0) { + static Settings Symmetric(size_t shift, size_t border) { Settings settings; settings.border_x = settings.border_y = border; settings.shift_x = settings.shift_y = shift; return settings; } @@ -101,11 +96,11 @@ // `GroupBorderAssigner::kPaddingXRound`. If `settings_.temp_buffer_size` is // nonzero, `temp` will point to an HWY-aligned buffer of at least that number // of floats; concurrent calls will have different buffers. virtual void ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows, size_t xextra, size_t xsize, size_t xpos, size_t ypos, - float* JXL_RESTRICT temp) const = 0; + size_t thread_id) const = 0; protected: explicit RenderPipelineStage(Settings settings) : settings_(settings) {} virtual Status IsInitialized() const { return true; } @@ -113,10 +108,12 @@ // Informs the stage about the total size of each channel. Few stages will // actually need to use this information. virtual void SetInputSizes( const std::vector>& input_sizes) {} + virtual Status PrepareForThreads(size_t num_threads) { return true; } + // How each channel will be processed. Channels are numbered starting from // color channels (always 3) and followed by all other channels. virtual RenderPipelineChannelMode GetChannelMode(size_t c) const = 0; // Returns a pointer to the input row of channel `c` with offset `y`. diff --git a/third_party/jpeg-xl/lib/jxl/render_pipeline/render_pipeline_test.cc b/third_party/jpeg-xl/lib/jxl/render_pipeline/render_pipeline_test.cc --- a/third_party/jpeg-xl/lib/jxl/render_pipeline/render_pipeline_test.cc +++ b/third_party/jpeg-xl/lib/jxl/render_pipeline/render_pipeline_test.cc @@ -47,20 +47,20 @@ FrameDimensions frame_dimensions; frame_dimensions.Set(/*xsize=*/1024, /*ysize=*/1024, /*group_size_shift=*/0, /*max_hshift=*/0, /*max_vshift=*/0, /*modular_mode=*/false, /*upsampling=*/1); auto pipeline = std::move(builder).Finalize(frame_dimensions); - pipeline->PrepareForThreads(1, /*use_group_ids=*/false); + ASSERT_TRUE(pipeline->PrepareForThreads(1, /*use_group_ids=*/false)); for (size_t i = 0; i < frame_dimensions.num_groups; i++) { auto input_buffers = pipeline->GetInputBuffers(i, 0); FillPlane(0.0f, input_buffers.GetBuffer(0).first, input_buffers.GetBuffer(0).second); input_buffers.Done(); } - EXPECT_TRUE(pipeline->PassesWithAllInput() == 1); + EXPECT_EQ(pipeline->PassesWithAllInput(), 1); } TEST(RenderPipelineTest, BuildFast) { RenderPipeline::Builder builder(/*num_c=*/1); builder.AddStage(jxl::make_unique()); @@ -82,20 +82,20 @@ FrameDimensions frame_dimensions; frame_dimensions.Set(/*xsize=*/1024, /*ysize=*/1024, /*group_size_shift=*/0, /*max_hshift=*/0, /*max_vshift=*/0, /*modular_mode=*/false, /*upsampling=*/1); auto pipeline = std::move(builder).Finalize(frame_dimensions); - pipeline->PrepareForThreads(1, /*use_group_ids=*/false); + ASSERT_TRUE(pipeline->PrepareForThreads(1, /*use_group_ids=*/false)); for (size_t i = 0; i < frame_dimensions.num_groups; i++) { auto input_buffers = pipeline->GetInputBuffers(i, 0); FillPlane(0.0f, input_buffers.GetBuffer(0).first, input_buffers.GetBuffer(0).second); input_buffers.Done(); } - EXPECT_TRUE(pipeline->PassesWithAllInput() == 1); + EXPECT_EQ(pipeline->PassesWithAllInput(), 1); } struct RenderPipelineTestInputSettings { // Input image. std::string input_path; @@ -220,11 +220,11 @@ {3, 8}, {128, 128}, {256, 256}, {258, 258}, {533, 401}, {777, 777}, }; for (auto size : sizes) { RenderPipelineTestInputSettings settings; - settings.input_path = "imagecompression.info/flower_foveon.png"; + settings.input_path = "third_party/imagecompression.info/flower_foveon.png"; settings.xsize = size.first; settings.ysize = size.second; // Base settings. settings.cparams.butteraugli_distance = 1.0; @@ -346,42 +346,46 @@ { auto s = settings; s.cparams_descr = "ModularLossy"; s.cparams.modular_mode = true; - s.cparams.quality_pair = {90, 90}; + s.cparams.butteraugli_distance = 1.f; all_tests.push_back(s); } { auto s = settings; - s.input_path = "imagecompression.info/flower_foveon_alpha.png"; + s.input_path = + "third_party/imagecompression.info/flower_foveon_alpha.png"; s.cparams_descr = "AlphaVarDCT"; all_tests.push_back(s); } { auto s = settings; - s.input_path = "imagecompression.info/flower_foveon_alpha.png"; + s.input_path = + "third_party/imagecompression.info/flower_foveon_alpha.png"; s.cparams_descr = "AlphaVarDCTUpsamplingEPF"; s.cparams.epf = 1; s.cparams.ec_resampling = 2; all_tests.push_back(s); } { auto s = settings; s.cparams.modular_mode = true; s.cparams.butteraugli_distance = 0; - s.input_path = "imagecompression.info/flower_foveon_alpha.png"; + s.input_path = + "third_party/imagecompression.info/flower_foveon_alpha.png"; s.cparams_descr = "AlphaLossless"; all_tests.push_back(s); } { auto s = settings; - s.input_path = "imagecompression.info/flower_foveon_alpha.png"; + s.input_path = + "third_party/imagecompression.info/flower_foveon_alpha.png"; s.cparams_descr = "AlphaDownsample"; s.cparams.ec_resampling = 2; all_tests.push_back(s); } @@ -393,14 +397,14 @@ } } #if JPEGXL_ENABLE_TRANSCODE_JPEG for (const char* input : - {"imagecompression.info/flower_foveon.png.im_q85_444.jpg", - "imagecompression.info/flower_foveon.png.im_q85_420.jpg", - "imagecompression.info/flower_foveon.png.im_q85_422.jpg", - "imagecompression.info/flower_foveon.png.im_q85_440.jpg"}) { + {"third_party/imagecompression.info/flower_foveon.png.im_q85_444.jpg", + "third_party/imagecompression.info/flower_foveon.png.im_q85_420.jpg", + "third_party/imagecompression.info/flower_foveon.png.im_q85_422.jpg", + "third_party/imagecompression.info/flower_foveon.png.im_q85_440.jpg"}) { RenderPipelineTestInputSettings settings; settings.input_path = input; settings.jpeg_transcode = true; settings.xsize = 2268; settings.ysize = 1512; diff --git a/third_party/jpeg-xl/lib/jxl/render_pipeline/simple_render_pipeline.cc b/third_party/jpeg-xl/lib/jxl/render_pipeline/simple_render_pipeline.cc --- a/third_party/jpeg-xl/lib/jxl/render_pipeline/simple_render_pipeline.cc +++ b/third_party/jpeg-xl/lib/jxl/render_pipeline/simple_render_pipeline.cc @@ -180,14 +180,12 @@ for (size_t iy = 0; iy < output_rows[c].size(); iy++) { output_rows[c][iy] = output_channels[c]->Row( (y << stage->settings_.shift_y) + iy + kRenderPipelineXOffset); } } - stage->ProcessRow( - input_rows, output_rows, /*xextra=*/0, xsize, - /*xpos=*/0, y, - reinterpret_cast(temp_buffers_[thread_id].get())); + stage->ProcessRow(input_rows, output_rows, /*xextra=*/0, xsize, + /*xpos=*/0, y, thread_id); } } // Move new channels to current channels. for (size_t c = 0; c < channel_data_.size(); c++) { diff --git a/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_blending.cc b/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_blending.cc --- a/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_blending.cc +++ b/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_blending.cc @@ -101,11 +101,11 @@ Status IsInitialized() const override { return initialized_; } void ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows, size_t xextra, size_t xsize, size_t xpos, size_t ypos, - float* JXL_RESTRICT temp) const final { + size_t thread_id) const final { PROFILER_ZONE("Blend"); JXL_ASSERT(initialized_); const FrameOrigin& frame_origin = state_.frame_header.frame_origin; ssize_t bg_xpos = frame_origin.x0 + static_cast(xpos); ssize_t bg_ypos = frame_origin.y0 + static_cast(ypos); diff --git a/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_chroma_upsampling.cc b/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_chroma_upsampling.cc --- a/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_chroma_upsampling.cc +++ b/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_chroma_upsampling.cc @@ -23,11 +23,11 @@ /*shift=*/1, /*border=*/1)), c_(channel) {} void ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows, size_t xextra, size_t xsize, size_t xpos, size_t ypos, - float* JXL_RESTRICT temp) const final { + size_t thread_id) const final { PROFILER_ZONE("HorizontalChromaUpsampling"); HWY_FULL(float) df; xextra = RoundUpTo(xextra, Lanes(df)); auto threefour = Set(df, 0.75f); auto onefour = Set(df, 0.25f); @@ -62,11 +62,11 @@ /*shift=*/1, /*border=*/1)), c_(channel) {} void ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows, size_t xextra, size_t xsize, size_t xpos, size_t ypos, - float* JXL_RESTRICT temp) const final { + size_t thread_id) const final { PROFILER_ZONE("VerticalChromaUpsampling"); HWY_FULL(float) df; xextra = RoundUpTo(xextra, Lanes(df)); auto threefour = Set(df, 0.75f); auto onefour = Set(df, 0.25f); diff --git a/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_epf.cc b/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_epf.cc --- a/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_epf.cc +++ b/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_epf.cc @@ -58,11 +58,11 @@ *B = MulAdd(weight, cb, *B); } void ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows, size_t xextra, size_t xsize, size_t xpos, size_t ypos, - float* JXL_RESTRICT temp) const final { + size_t thread_id) const final { DF df; xextra = RoundUpTo(xextra, Lanes(df)); const float* JXL_RESTRICT row_sigma = sigma_->Row(ypos / kBlockDim + kSigmaPadding); @@ -194,11 +194,11 @@ *B = MulAdd(weight, cb, *B); } void ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows, size_t xextra, size_t xsize, size_t xpos, size_t ypos, - float* JXL_RESTRICT temp) const final { + size_t thread_id) const final { DF df; xextra = RoundUpTo(xextra, Lanes(df)); const float* JXL_RESTRICT row_sigma = sigma_->Row(ypos / kBlockDim + kSigmaPadding); @@ -379,11 +379,11 @@ *B = MulAdd(weight, cb, *B); } void ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows, size_t xextra, size_t xsize, size_t xpos, size_t ypos, - float* JXL_RESTRICT temp) const final { + size_t thread_id) const final { DF df; xextra = RoundUpTo(xextra, Lanes(df)); const float* JXL_RESTRICT row_sigma = sigma_->Row(ypos / kBlockDim + kSigmaPadding); diff --git a/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_gaborish.cc b/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_gaborish.cc --- a/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_gaborish.cc +++ b/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_gaborish.cc @@ -39,11 +39,11 @@ } } void ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows, size_t xextra, size_t xsize, size_t xpos, size_t ypos, - float* JXL_RESTRICT temp) const final { + size_t thread_id) const final { PROFILER_ZONE("Gaborish"); const HWY_FULL(float) d; for (size_t c = 0; c < 3; c++) { float* JXL_RESTRICT row_t = GetInputRow(input_rows, c, -1); diff --git a/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_noise.cc b/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_noise.cc --- a/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_noise.cc +++ b/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_noise.cc @@ -147,11 +147,11 @@ cmap_(cmap), first_c_(first_c) {} void ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows, size_t xextra, size_t xsize, size_t xpos, size_t ypos, - float* JXL_RESTRICT temp) const final { + size_t thread_id) const final { PROFILER_ZONE("Noise apply"); if (!noise_params_.HasAny()) return; const StrengthEvalLut noise_model(noise_params_); D d; @@ -228,11 +228,11 @@ /*shift=*/0, /*border=*/2)), first_c_(first_c) {} void ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows, size_t xextra, size_t xsize, size_t xpos, size_t ypos, - float* JXL_RESTRICT temp) const final { + size_t thread_id) const final { PROFILER_ZONE("Noise convolve"); const HWY_FULL(float) d; for (size_t c = first_c_; c < first_c_ + 3; c++) { float* JXL_RESTRICT rows[5]; diff --git a/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_patches.cc b/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_patches.cc --- a/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_patches.cc +++ b/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_patches.cc @@ -13,11 +13,11 @@ : RenderPipelineStage(RenderPipelineStage::Settings()), patches_(*patches) {} void ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows, size_t xextra, size_t xsize, size_t xpos, size_t ypos, - float* JXL_RESTRICT temp) const final { + size_t thread_id) const final { PROFILER_ZONE("RenderPatches"); std::vector row_ptrs(input_rows.size()); for (size_t i = 0; i < row_ptrs.size(); i++) { row_ptrs[i] = GetInputRow(input_rows, i, 0) - xextra; } diff --git a/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_splines.cc b/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_splines.cc --- a/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_splines.cc +++ b/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_splines.cc @@ -20,11 +20,11 @@ : RenderPipelineStage(RenderPipelineStage::Settings()), splines_(*splines) {} void ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows, size_t xextra, size_t xsize, size_t xpos, size_t ypos, - float* JXL_RESTRICT temp) const final { + size_t thread_id) const final { PROFILER_ZONE("RenderSplines"); float* row_x = GetInputRow(input_rows, 0, 0); float* row_y = GetInputRow(input_rows, 1, 0); float* row_b = GetInputRow(input_rows, 2, 0); splines_.AddToRow(row_x, row_y, row_b, Rect(xpos, ypos, xsize, 1)); diff --git a/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_spot.cc b/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_spot.cc --- a/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_spot.cc +++ b/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_spot.cc @@ -15,11 +15,11 @@ JXL_ASSERT(spot_c_ >= 3); } void ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows, size_t xextra, size_t xsize, size_t xpos, size_t ypos, - float* JXL_RESTRICT temp) const final { + size_t thread_id) const final { // TODO(veluca): add SIMD. PROFILER_ZONE("RenderSpotColors"); float scale = spot_color_[3]; for (size_t c = 0; c < 3; c++) { float* JXL_RESTRICT p = GetInputRow(input_rows, c, 0); diff --git a/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_upsampling.cc b/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_upsampling.cc --- a/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_upsampling.cc +++ b/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_upsampling.cc @@ -38,11 +38,11 @@ } } void ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows, size_t xextra, size_t xsize, size_t xpos, size_t ypos, - float* JXL_RESTRICT temp) const final { + size_t thread_id) const final { PROFILER_ZONE("Upsampling"); static HWY_FULL(float) df; size_t shift = settings_.shift_x; size_t N = 1 << shift; const size_t xsize_v = RoundUpTo(xsize, Lanes(df)); diff --git a/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_write.h b/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_write.h --- a/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_write.h +++ b/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_write.h @@ -6,10 +6,11 @@ #ifndef LIB_JXL_RENDER_PIPELINE_STAGE_WRITE_H_ #define LIB_JXL_RENDER_PIPELINE_STAGE_WRITE_H_ #include +#include "lib/jxl/dec_cache.h" #include "lib/jxl/image_bundle.h" #include "lib/jxl/render_pipeline/render_pipeline_stage.h" namespace jxl { @@ -18,18 +19,19 @@ // Gets a stage to write color channels to an Image3F. std::unique_ptr GetWriteToImage3FStage(Image3F* image); // Gets a stage to write to a uint8 buffer. -std::unique_ptr GetWriteToU8Stage( - uint8_t* rgb, size_t stride, size_t width, size_t height, bool rgba, - bool has_alpha, size_t alpha_c); +std::unique_ptr GetWriteToU8Stage(uint8_t* rgb, + size_t stride, + size_t height, bool rgba, + bool has_alpha, + size_t alpha_c); // Gets a stage to write to a pixel callback. std::unique_ptr GetWriteToPixelCallbackStage( - const std::function& - pixel_callback, - size_t width, size_t height, bool rgba, bool has_alpha, size_t alpha_c); + const PixelCallback& pixel_callback, size_t width, size_t height, bool rgba, + bool has_alpha, size_t alpha_c); } // namespace jxl #endif // LIB_JXL_RENDER_PIPELINE_STAGE_WRITE_H_ diff --git a/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_write.cc b/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_write.cc --- a/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_write.cc +++ b/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_write.cc @@ -4,10 +4,11 @@ // license that can be found in the LICENSE file. #include "lib/jxl/render_pipeline/stage_write.h" #include "lib/jxl/common.h" +#include "lib/jxl/dec_cache.h" #include "lib/jxl/image_bundle.h" #include "lib/jxl/sanitizers.h" #undef HWY_TARGET_INCLUDE #define HWY_TARGET_INCLUDE "lib/jxl/render_pipeline/stage_write.cc" @@ -73,24 +74,23 @@ #endif } class WriteToU8Stage : public RenderPipelineStage { public: - WriteToU8Stage(uint8_t* rgb, size_t stride, size_t width, size_t height, - bool rgba, bool has_alpha, size_t alpha_c) + WriteToU8Stage(uint8_t* rgb, size_t stride, size_t height, bool rgba, + bool has_alpha, size_t alpha_c) : RenderPipelineStage(RenderPipelineStage::Settings()), rgb_(rgb), stride_(stride), - width_(width), height_(height), rgba_(rgba), has_alpha_(has_alpha), alpha_c_(alpha_c) {} void ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows, size_t xextra, size_t xsize, size_t xpos, size_t ypos, - float* JXL_RESTRICT temp) const final { + size_t thread_id) const final { if (ypos >= height_) return; JXL_DASSERT(xextra == 0); size_t bytes = rgba_ ? 4 : 3; const float* JXL_RESTRICT row_in_r = GetInputRow(input_rows, 0, 0); const float* JXL_RESTRICT row_in_g = GetInputRow(input_rows, 1, 0); @@ -122,11 +122,11 @@ : Set(d, 255.0f); auto r8 = U8FromU32(BitCast(du, NearestInt(rf))); auto g8 = U8FromU32(BitCast(du, NearestInt(gf))); auto b8 = U8FromU32(BitCast(du, NearestInt(bf))); auto a8 = U8FromU32(BitCast(du, NearestInt(af))); - size_t n = width_ - xpos - x; + size_t n = xsize - x; if (JXL_LIKELY(n >= Lanes(d))) { StoreRGBA(D::Rebind(), r8, g8, b8, a8, rgba_, Lanes(d), n, rgb_ + base_ptr + bytes * x); } else { StoreRGBA(D::Rebind(), r8, g8, b8, a8, rgba_, n, n, @@ -144,23 +144,24 @@ const char* GetName() const override { return "WriteToU8"; } private: uint8_t* rgb_; size_t stride_; - size_t width_; size_t height_; bool rgba_; bool has_alpha_; size_t alpha_c_; std::vector opaque_alpha_; }; -std::unique_ptr GetWriteToU8Stage( - uint8_t* rgb, size_t stride, size_t width, size_t height, bool rgba, - bool has_alpha, size_t alpha_c) { - return jxl::make_unique(rgb, stride, width, height, rgba, - has_alpha, alpha_c); +std::unique_ptr GetWriteToU8Stage(uint8_t* rgb, + size_t stride, + size_t height, bool rgba, + bool has_alpha, + size_t alpha_c) { + return jxl::make_unique(rgb, stride, height, rgba, has_alpha, + alpha_c); } // NOLINTNEXTLINE(google-readability-namespace-comments) } // namespace HWY_NAMESPACE } // namespace jxl @@ -188,10 +189,11 @@ for (size_t c = 1; c < input_sizes.size(); c++) { JXL_ASSERT(input_sizes[c].first == input_sizes[0].first); JXL_ASSERT(input_sizes[c].second == input_sizes[0].second); } #endif + // TODO(eustas): what should we do in the case of "want only ECs"? image_bundle_->SetFromImage( Image3F(input_sizes[0].first, input_sizes[0].second), color_encoding_); // TODO(veluca): consider not reallocating ECs if not needed. image_bundle_->extra_channels().clear(); for (size_t c = 3; c < input_sizes.size(); c++) { @@ -200,11 +202,11 @@ } } void ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows, size_t xextra, size_t xsize, size_t xpos, size_t ypos, - float* JXL_RESTRICT temp) const final { + size_t thread_id) const final { for (size_t c = 0; c < 3; c++) { memcpy(image_bundle_->color()->PlaneRow(c, ypos) + xpos - xextra, GetInputRow(input_rows, c, 0) - xextra, sizeof(float) * (xsize + 2 * xextra)); } @@ -245,11 +247,11 @@ *image_ = Image3F(input_sizes[0].first, input_sizes[0].second); } void ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows, size_t xextra, size_t xsize, size_t xpos, size_t ypos, - float* JXL_RESTRICT temp) const final { + size_t thread_id) const final { for (size_t c = 0; c < 3; c++) { memcpy(image_->PlaneRow(c, ypos) + xpos - xextra, GetInputRow(input_rows, c, 0) - xextra, sizeof(float) * (xsize + 2 * xextra)); } @@ -266,28 +268,38 @@ Image3F* image_; }; class WriteToPixelCallbackStage : public RenderPipelineStage { public: - WriteToPixelCallbackStage( - const std::function& - pixel_callback, - size_t width, size_t height, bool rgba, bool has_alpha, size_t alpha_c) + WriteToPixelCallbackStage(const PixelCallback& pixel_callback, size_t width, + size_t height, bool rgba, bool has_alpha, + size_t alpha_c) : RenderPipelineStage(RenderPipelineStage::Settings()), pixel_callback_(pixel_callback), width_(width), height_(height), rgba_(rgba), has_alpha_(has_alpha), alpha_c_(alpha_c), - opaque_alpha_(kMaxPixelsPerCall, 1.0f) { - settings_.temp_buffer_size = kMaxPixelsPerCall * (rgba_ ? 4 : 3); + opaque_alpha_(kMaxPixelsPerCall, 1.0f) {} + + WriteToPixelCallbackStage(const WriteToPixelCallbackStage&) = delete; + WriteToPixelCallbackStage& operator=(const WriteToPixelCallbackStage&) = + delete; + WriteToPixelCallbackStage(WriteToPixelCallbackStage&&) = delete; + WriteToPixelCallbackStage& operator=(WriteToPixelCallbackStage&&) = delete; + + ~WriteToPixelCallbackStage() override { + if (run_opaque_) { + pixel_callback_.destroy(run_opaque_); + } } void ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows, size_t xextra, size_t xsize, size_t xpos, size_t ypos, - float* JXL_RESTRICT temp) const final { + size_t thread_id) const final { + JXL_DASSERT(run_opaque_); if (ypos >= height_) return; const float* line_buffers[4]; for (size_t c = 0; c < 3; c++) { line_buffers[c] = GetInputRow(input_rows, c, 0) - xextra; } @@ -300,19 +312,21 @@ // TODO(veluca): SIMD. ssize_t limit = std::min(xextra + xsize, width_ - xpos); for (ssize_t x0 = -xextra; x0 < limit; x0 += kMaxPixelsPerCall) { size_t j = 0; size_t ix = 0; + float* JXL_RESTRICT temp = + reinterpret_cast(temp_[thread_id].get()); for (; ix < kMaxPixelsPerCall && ssize_t(ix) + x0 < limit; ix++) { temp[j++] = line_buffers[0][ix]; temp[j++] = line_buffers[1][ix]; temp[j++] = line_buffers[2][ix]; if (rgba_) { temp[j++] = line_buffers[3][ix]; } } - pixel_callback_(temp, xpos + x0, ypos, ix); + pixel_callback_.run(run_opaque_, thread_id, xpos + x0, ypos, ix, temp); for (size_t c = 0; c < 3; c++) line_buffers[c] += kMaxPixelsPerCall; if (has_alpha_) line_buffers[3] += kMaxPixelsPerCall; } } @@ -323,19 +337,31 @@ } const char* GetName() const override { return "WritePixelCB"; } private: + Status PrepareForThreads(size_t num_threads) override { + run_opaque_ = + pixel_callback_.Init(num_threads, /*num_pixels=*/kMaxPixelsPerCall); + JXL_RETURN_IF_ERROR(run_opaque_ != nullptr); + temp_.resize(num_threads); + for (CacheAlignedUniquePtr& temp : temp_) { + temp = AllocateArray(sizeof(float) * kMaxPixelsPerCall * (rgba_ ? 4 : 3)); + } + return true; + } + static constexpr size_t kMaxPixelsPerCall = 1024; - const std::function& - pixel_callback_; + PixelCallback pixel_callback_; + void* run_opaque_ = nullptr; size_t width_; size_t height_; bool rgba_; bool has_alpha_; size_t alpha_c_; std::vector opaque_alpha_; + std::vector temp_; }; } // namespace std::unique_ptr GetWriteToImageBundleStage( @@ -346,21 +372,22 @@ std::unique_ptr GetWriteToImage3FStage(Image3F* image) { return jxl::make_unique(image); } -std::unique_ptr GetWriteToU8Stage( - uint8_t* rgb, size_t stride, size_t width, size_t height, bool rgba, - bool has_alpha, size_t alpha_c) { - return HWY_DYNAMIC_DISPATCH(GetWriteToU8Stage)(rgb, stride, width, height, - rgba, has_alpha, alpha_c); +std::unique_ptr GetWriteToU8Stage(uint8_t* rgb, + size_t stride, + size_t height, bool rgba, + bool has_alpha, + size_t alpha_c) { + return HWY_DYNAMIC_DISPATCH(GetWriteToU8Stage)(rgb, stride, height, rgba, + has_alpha, alpha_c); } std::unique_ptr GetWriteToPixelCallbackStage( - const std::function& - pixel_callback, - size_t width, size_t height, bool rgba, bool has_alpha, size_t alpha_c) { + const PixelCallback& pixel_callback, size_t width, size_t height, bool rgba, + bool has_alpha, size_t alpha_c) { return jxl::make_unique( pixel_callback, width, height, rgba, has_alpha, alpha_c); } } // namespace jxl diff --git a/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_xyb.cc b/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_xyb.cc --- a/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_xyb.cc +++ b/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_xyb.cc @@ -116,11 +116,11 @@ opsin_params_(opsin_params), op_(op) {} void ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows, size_t xextra, size_t xsize, size_t xpos, size_t ypos, - float* JXL_RESTRICT temp) const final { + size_t thread_id) const final { PROFILER_ZONE("UndoXYB"); const HWY_FULL(float) d; const size_t xsize_v = RoundUpTo(xsize, Lanes(d)); float* JXL_RESTRICT row0 = GetInputRow(input_rows, 0, 0); @@ -227,11 +227,11 @@ has_alpha_(has_alpha), alpha_c_(alpha_c) {} void ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows, size_t xextra, size_t xsize, size_t xpos, size_t ypos, - float* JXL_RESTRICT temp) const final { + size_t thread_id) const final { if (ypos >= height_) return; JXL_ASSERT(xextra == 0); const float* xyba[4] = { GetInputRow(input_rows, 0, 0), GetInputRow(input_rows, 1, 0), GetInputRow(input_rows, 2, 0), diff --git a/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_ycbcr.cc b/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_ycbcr.cc --- a/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_ycbcr.cc +++ b/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_ycbcr.cc @@ -18,11 +18,11 @@ public: kYCbCrStage() : RenderPipelineStage(RenderPipelineStage::Settings()) {} void ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows, size_t xextra, size_t xsize, size_t xpos, size_t ypos, - float* JXL_RESTRICT temp) const final { + size_t thread_id) const final { PROFILER_ZONE("UndoYCbCr"); const HWY_FULL(float) df; // Full-range BT.601 as defined by JFIF Clause 7: diff --git a/third_party/jpeg-xl/lib/jxl/render_pipeline/test_render_pipeline_stages.h b/third_party/jpeg-xl/lib/jxl/render_pipeline/test_render_pipeline_stages.h --- a/third_party/jpeg-xl/lib/jxl/render_pipeline/test_render_pipeline_stages.h +++ b/third_party/jpeg-xl/lib/jxl/render_pipeline/test_render_pipeline_stages.h @@ -20,11 +20,11 @@ UpsampleXSlowStage() : RenderPipelineStage(RenderPipelineStage::Settings::ShiftX(1, 1)) {} void ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows, size_t xextra, size_t xsize, size_t xpos, size_t ypos, - float* JXL_RESTRICT temp) const final { + size_t thread_id) const final { for (size_t c = 0; c < input_rows.size(); c++) { const float* row = GetInputRow(input_rows, c, 0); float* row_out = GetOutputRow(output_rows, c, 0); for (int64_t x = -xextra; x < (int64_t)(xsize + xextra); x++) { float xp = *(row + x - 1); @@ -50,11 +50,11 @@ UpsampleYSlowStage() : RenderPipelineStage(RenderPipelineStage::Settings::ShiftY(1, 1)) {} void ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows, size_t xextra, size_t xsize, size_t xpos, size_t ypos, - float* JXL_RESTRICT temp) const final { + size_t thread_id) const final { for (size_t c = 0; c < input_rows.size(); c++) { const float* rowp = GetInputRow(input_rows, c, -1); const float* rowc = GetInputRow(input_rows, c, 0); const float* rown = GetInputRow(input_rows, c, 1); float* row_out0 = GetOutputRow(output_rows, c, 0); @@ -82,11 +82,11 @@ public: Check0FinalStage() : RenderPipelineStage(RenderPipelineStage::Settings()) {} void ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows, size_t xextra, size_t xsize, size_t xpos, size_t ypos, - float* JXL_RESTRICT temp) const final { + size_t thread_id) const final { for (size_t c = 0; c < input_rows.size(); c++) { for (size_t x = 0; x < xsize; x++) { JXL_CHECK(fabsf(GetInputRow(input_rows, c, 0)[x]) < 1e-8); } } diff --git a/third_party/jpeg-xl/lib/jxl/roundtrip_test.cc b/third_party/jpeg-xl/lib/jxl/roundtrip_test.cc --- a/third_party/jpeg-xl/lib/jxl/roundtrip_test.cc +++ b/third_party/jpeg-xl/lib/jxl/roundtrip_test.cc @@ -49,11 +49,10 @@ switch (pixel_format.data_type) { case JXL_TYPE_UINT8: io.metadata.m.SetAlphaBits(8); break; case JXL_TYPE_UINT16: - case JXL_TYPE_UINT32: case JXL_TYPE_FLOAT: case JXL_TYPE_FLOAT16: io.metadata.m.SetAlphaBits(16); break; default: @@ -265,12 +264,13 @@ std::vector channel_infos; for (const auto& extra_channel : extra_channels) { auto channel_type = extra_channel.first; JxlExtraChannelInfo channel_info; JxlEncoderInitExtraChannelInfo(channel_type, &channel_info); - channel_info.bits_per_sample = basic_info.bits_per_sample; - channel_info.exponent_bits_per_sample = basic_info.exponent_bits_per_sample; + channel_info.bits_per_sample = (lossless ? basic_info.bits_per_sample : 8); + channel_info.exponent_bits_per_sample = + (lossless ? basic_info.exponent_bits_per_sample : 0); channel_infos.push_back(channel_info); } for (size_t index = 0; index < channel_infos.size(); index++) { EXPECT_EQ(JXL_ENC_SUCCESS, JxlEncoderSetExtraChannelInfo(enc, index + has_interleaved_alpha, @@ -444,16 +444,10 @@ extra_channel_bytes.data(), xsize, ysize, extra_channel_pixel_format, extra_channel_output_pixel_format), 0u); EXPECT_EQ(extra_channel, extra_channel_bytes); - } else { - EXPECT_EQ(jxl::test::ComparePixels( - extra_channel.data(), extra_channel_bytes.data(), xsize, - ysize, extra_channel_pixel_format, - extra_channel_output_pixel_format, 16.0), - 0u); } } } } // namespace @@ -586,10 +580,11 @@ JxlBasicInfo basic_info; jxl::test::JxlBasicInfoSetFromPixelFormat(&basic_info, &pixel_format); basic_info.xsize = xsize; basic_info.ysize = ysize; basic_info.uses_original_profile = false; + EXPECT_EQ(JXL_ENC_SUCCESS, JxlEncoderSetCodestreamLevel(enc, 10)); EXPECT_EQ(JXL_ENC_SUCCESS, JxlEncoderSetBasicInfo(enc, &basic_info)); JxlColorEncoding color_encoding; if (pixel_format.data_type == JXL_TYPE_FLOAT) { JxlColorEncodingSetToLinearSRGB(&color_encoding, /*is_gray=*/pixel_format.num_channels < 3); @@ -804,11 +799,11 @@ } #if JPEGXL_ENABLE_JPEG // Loading .jpg files requires libjpeg support. TEST(RoundtripTest, JXL_TRANSCODE_JPEG_TEST(TestJPEGReconstruction)) { const std::string jpeg_path = - "imagecompression.info/flower_foveon.png.im_q85_420.jpg"; + "third_party/imagecompression.info/flower_foveon.png.im_q85_420.jpg"; const jxl::PaddedBytes orig = jxl::ReadTestData(jpeg_path); jxl::CodecInOut orig_io; ASSERT_TRUE( SetFromBytes(jxl::Span(orig), &orig_io, /*pool=*/nullptr)); diff --git a/third_party/jpeg-xl/lib/jxl/sanitizers.h b/third_party/jpeg-xl/lib/jxl/sanitizers.h --- a/third_party/jpeg-xl/lib/jxl/sanitizers.h +++ b/third_party/jpeg-xl/lib/jxl/sanitizers.h @@ -8,48 +8,13 @@ #include #include #include "lib/jxl/base/compiler_specific.h" +#include "lib/jxl/base/sanitizer_definitions.h" #include "lib/jxl/image.h" -#ifdef MEMORY_SANITIZER -#define JXL_MEMORY_SANITIZER 1 -#elif defined(__has_feature) -#if __has_feature(memory_sanitizer) -#define JXL_MEMORY_SANITIZER 1 -#else -#define JXL_MEMORY_SANITIZER 0 -#endif -#else -#define JXL_MEMORY_SANITIZER 0 -#endif - -#ifdef ADDRESS_SANITIZER -#define JXL_ADDRESS_SANITIZER 1 -#elif defined(__has_feature) -#if __has_feature(address_sanitizer) -#define JXL_ADDRESS_SANITIZER 1 -#else -#define JXL_ADDRESS_SANITIZER 0 -#endif -#else -#define JXL_ADDRESS_SANITIZER 0 -#endif - -#ifdef THREAD_SANITIZER -#define JXL_THREAD_SANITIZER 1 -#elif defined(__has_feature) -#if __has_feature(thread_sanitizer) -#define JXL_THREAD_SANITIZER 1 -#else -#define JXL_THREAD_SANITIZER 0 -#endif -#else -#define JXL_THREAD_SANITIZER 0 -#endif - #if JXL_MEMORY_SANITIZER #include #include #include diff --git a/third_party/jpeg-xl/lib/jxl/simd_util-inl.h b/third_party/jpeg-xl/lib/jxl/simd_util-inl.h --- a/third_party/jpeg-xl/lib/jxl/simd_util-inl.h +++ b/third_party/jpeg-xl/lib/jxl/simd_util-inl.h @@ -12,10 +12,12 @@ #define LIB_JXL_SIMD_UTIL_INL_H_ #endif #include +#include "lib/jxl/base/compiler_specific.h" + HWY_BEFORE_NAMESPACE(); namespace jxl { namespace HWY_NAMESPACE { #if HWY_CAP_GE512 @@ -37,39 +39,45 @@ #endif // Stores v0[0], v1[0], v0[1], v1[1], ... to mem, in this order. Mem must be // aligned. -template -void StoreInterleaved(const DF df, V v0, V v1, float* mem) { +template +void StoreInterleaved(const DF df, V v0, V v1, T* mem) { + static_assert(sizeof(T) == 4, "only use StoreInterleaved for 4-byte types"); #if HWY_TARGET == HWY_SCALAR Store(v0, df, mem); Store(v1, df, mem + 1); #elif !HWY_CAP_GE256 Store(InterleaveLower(df, v0, v1), df, mem); Store(InterleaveUpper(df, v0, v1), df, mem + Lanes(df)); -#elif !HWY_CAP_GE512 - auto t0 = InterleaveLower(df, v0, v1); - auto t1 = InterleaveUpper(df, v0, v1); - Store(ConcatLowerLower(df, t1, t0), df, mem); - Store(ConcatUpperUpper(df, t1, t0), df, mem + Lanes(df)); #else - auto t0 = InterleaveLower(df, v0, v1); - auto t1 = InterleaveUpper(df, v0, v1); - Store(Concat4(df, Quarter<0>(df, t0), Quarter<0>(df, t1), Quarter<1>(df, t0), - Quarter<1>(df, t1)), - df, mem); - Store(Concat4(df, Quarter<2>(df, t0), Quarter<2>(df, t1), Quarter<3>(df, t0), - Quarter<3>(df, t1)), - df, mem + Lanes(df)); + if (!HWY_CAP_GE512 || Lanes(df) == 8) { + auto t0 = InterleaveLower(df, v0, v1); + auto t1 = InterleaveUpper(df, v0, v1); + Store(ConcatLowerLower(df, t1, t0), df, mem); + Store(ConcatUpperUpper(df, t1, t0), df, mem + Lanes(df)); + } else { +#if HWY_CAP_GE512 + auto t0 = InterleaveLower(df, v0, v1); + auto t1 = InterleaveUpper(df, v0, v1); + Store(Concat4(df, Quarter<0>(df, t0), Quarter<0>(df, t1), + Quarter<1>(df, t0), Quarter<1>(df, t1)), + df, mem); + Store(Concat4(df, Quarter<2>(df, t0), Quarter<2>(df, t1), + Quarter<3>(df, t0), Quarter<3>(df, t1)), + df, mem + Lanes(df)); +#endif + } #endif } // Stores v0[0], v1[0], v2[0], v3[0], v0[1] ... to mem, in this order. Mem must // be aligned. -template -void StoreInterleaved(const DF df, V v0, V v1, V v2, V v3, float* mem) { +template +void StoreInterleaved(const DF df, V v0, V v1, V v2, V v3, T* mem) { + static_assert(sizeof(T) == 4, "only use StoreInterleaved for 4-byte types"); #if HWY_TARGET == HWY_SCALAR Store(v0, df, mem); Store(v1, df, mem + 1); Store(v2, df, mem + 2); Store(v3, df, mem + 3); @@ -253,10 +261,88 @@ Quarter<3>(df, m7)), df, mem + 7 * Lanes(df)); #endif } +#if HWY_CAP_GE256 +JXL_INLINE void Transpose8x8Block(const int32_t* JXL_RESTRICT from, + int32_t* JXL_RESTRICT to, size_t fromstride) { + const HWY_CAPPED(int32_t, 8) d; + auto i0 = Load(d, from); + auto i1 = Load(d, from + 1 * fromstride); + auto i2 = Load(d, from + 2 * fromstride); + auto i3 = Load(d, from + 3 * fromstride); + auto i4 = Load(d, from + 4 * fromstride); + auto i5 = Load(d, from + 5 * fromstride); + auto i6 = Load(d, from + 6 * fromstride); + auto i7 = Load(d, from + 7 * fromstride); + + const auto q0 = InterleaveLower(d, i0, i2); + const auto q1 = InterleaveLower(d, i1, i3); + const auto q2 = InterleaveUpper(d, i0, i2); + const auto q3 = InterleaveUpper(d, i1, i3); + const auto q4 = InterleaveLower(d, i4, i6); + const auto q5 = InterleaveLower(d, i5, i7); + const auto q6 = InterleaveUpper(d, i4, i6); + const auto q7 = InterleaveUpper(d, i5, i7); + + const auto r0 = InterleaveLower(d, q0, q1); + const auto r1 = InterleaveUpper(d, q0, q1); + const auto r2 = InterleaveLower(d, q2, q3); + const auto r3 = InterleaveUpper(d, q2, q3); + const auto r4 = InterleaveLower(d, q4, q5); + const auto r5 = InterleaveUpper(d, q4, q5); + const auto r6 = InterleaveLower(d, q6, q7); + const auto r7 = InterleaveUpper(d, q6, q7); + + i0 = ConcatLowerLower(d, r4, r0); + i1 = ConcatLowerLower(d, r5, r1); + i2 = ConcatLowerLower(d, r6, r2); + i3 = ConcatLowerLower(d, r7, r3); + i4 = ConcatUpperUpper(d, r4, r0); + i5 = ConcatUpperUpper(d, r5, r1); + i6 = ConcatUpperUpper(d, r6, r2); + i7 = ConcatUpperUpper(d, r7, r3); + + Store(i0, d, to); + Store(i1, d, to + 1 * 8); + Store(i2, d, to + 2 * 8); + Store(i3, d, to + 3 * 8); + Store(i4, d, to + 4 * 8); + Store(i5, d, to + 5 * 8); + Store(i6, d, to + 6 * 8); + Store(i7, d, to + 7 * 8); +} +#elif HWY_TARGET != HWY_SCALAR +JXL_INLINE void Transpose8x8Block(const int32_t* JXL_RESTRICT from, + int32_t* JXL_RESTRICT to, size_t fromstride) { + const HWY_CAPPED(int32_t, 4) d; + for (size_t n = 0; n < 8; n += 4) { + for (size_t m = 0; m < 8; m += 4) { + auto p0 = Load(d, from + n * fromstride + m); + auto p1 = Load(d, from + (n + 1) * fromstride + m); + auto p2 = Load(d, from + (n + 2) * fromstride + m); + auto p3 = Load(d, from + (n + 3) * fromstride + m); + const auto q0 = InterleaveLower(d, p0, p2); + const auto q1 = InterleaveLower(d, p1, p3); + const auto q2 = InterleaveUpper(d, p0, p2); + const auto q3 = InterleaveUpper(d, p1, p3); + + const auto r0 = InterleaveLower(d, q0, q1); + const auto r1 = InterleaveUpper(d, q0, q1); + const auto r2 = InterleaveLower(d, q2, q3); + const auto r3 = InterleaveUpper(d, q2, q3); + Store(r0, d, to + m * 8 + n); + Store(r1, d, to + (1 + m) * 8 + n); + Store(r2, d, to + (2 + m) * 8 + n); + Store(r3, d, to + (3 + m) * 8 + n); + } + } +} + +#endif + // NOLINTNEXTLINE(google-readability-namespace-comments) } // namespace HWY_NAMESPACE } // namespace jxl HWY_AFTER_NAMESPACE(); diff --git a/third_party/jpeg-xl/lib/jxl/speed_tier_test.cc b/third_party/jpeg-xl/lib/jxl/speed_tier_test.cc --- a/third_party/jpeg-xl/lib/jxl/speed_tier_test.cc +++ b/third_party/jpeg-xl/lib/jxl/speed_tier_test.cc @@ -83,11 +83,11 @@ SpeedTierTestParams{SpeedTier::kTortoise, /*shrink8=*/true})); TEST_P(SpeedTierTest, Roundtrip) { const PaddedBytes orig = - ReadTestData("wesaturate/500px/u76c0g_bliznaca_srgb8.png"); + ReadTestData("third_party/wesaturate/500px/u76c0g_bliznaca_srgb8.png"); CodecInOut io; ThreadPoolInternal pool(8); ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); const SpeedTierTestParams& params = GetParam(); diff --git a/third_party/jpeg-xl/lib/jxl/test_utils.h b/third_party/jpeg-xl/lib/jxl/test_utils.h --- a/third_party/jpeg-xl/lib/jxl/test_utils.h +++ b/third_party/jpeg-xl/lib/jxl/test_utils.h @@ -76,28 +76,17 @@ break; case JXL_TYPE_UINT16: basic_info->bits_per_sample = 16; basic_info->exponent_bits_per_sample = 0; break; - case JXL_TYPE_UINT32: - basic_info->bits_per_sample = 32; - basic_info->exponent_bits_per_sample = 0; - break; - case JXL_TYPE_BOOLEAN: - basic_info->bits_per_sample = 1; - basic_info->exponent_bits_per_sample = 0; - break; + default: + JXL_ABORT("Unhandled JxlDataType"); } if (pixel_format->num_channels == 2 || pixel_format->num_channels == 4) { - basic_info->alpha_exponent_bits = 0; - if (basic_info->bits_per_sample == 32) { - basic_info->alpha_bits = 16; - basic_info->num_extra_channels = 1; - } else { - basic_info->alpha_bits = basic_info->bits_per_sample; - basic_info->num_extra_channels = 1; - } + basic_info->alpha_exponent_bits = basic_info->exponent_bits_per_sample; + basic_info->alpha_bits = basic_info->bits_per_sample; + basic_info->num_extra_channels = 1; } else { basic_info->alpha_exponent_bits = 0; basic_info->alpha_bits = 0; } } @@ -429,43 +418,37 @@ return LoadFloat16(bits16); } size_t GetPrecision(JxlDataType data_type) { switch (data_type) { - case JXL_TYPE_BOOLEAN: - return 1; case JXL_TYPE_UINT8: return 8; case JXL_TYPE_UINT16: return 16; - case JXL_TYPE_UINT32: - return 32; case JXL_TYPE_FLOAT: // Floating point mantissa precision return 24; case JXL_TYPE_FLOAT16: return 11; + default: + JXL_ABORT("Unhandled JxlDataType"); } - JXL_ASSERT(false); // unknown type } size_t GetDataBits(JxlDataType data_type) { switch (data_type) { - case JXL_TYPE_BOOLEAN: - return 1; case JXL_TYPE_UINT8: return 8; case JXL_TYPE_UINT16: return 16; - case JXL_TYPE_UINT32: - return 32; case JXL_TYPE_FLOAT: return 32; case JXL_TYPE_FLOAT16: return 16; + default: + JXL_ABORT("Unhandled JxlDataType"); } - JXL_ASSERT(false); // unknown type } // Procedure to convert pixels to double precision, not efficient, but // well-controlled for testing. It uses double, to be able to represent all // precisions needed for the maximum data types the API supports: uint32_t @@ -481,27 +464,11 @@ size_t stride = xsize * jxl::DivCeil(GetDataBits(format.data_type) * num_channels, jxl::kBitsPerByte); if (format.align > 1) stride = jxl::RoundUpTo(stride, format.align); - if (format.data_type == JXL_TYPE_BOOLEAN) { - for (size_t y = 0; y < ysize; ++y) { - jxl::BitReader br(jxl::Span(pixels + stride * y, stride)); - for (size_t x = 0; x < xsize; ++x) { - size_t j = (y * xsize + x) * 4; - double r = br.ReadBits(1); - double g = gray ? r : br.ReadBits(1); - double b = gray ? r : br.ReadBits(1); - double a = alpha ? br.ReadBits(1) : 1; - result[j + 0] = r; - result[j + 1] = g; - result[j + 2] = b; - result[j + 3] = a; - } - JXL_CHECK(br.Close()); - } - } else if (format.data_type == JXL_TYPE_UINT8) { + if (format.data_type == JXL_TYPE_UINT8) { double mul = 1.0 / 255.0; // Multiplier to bring to 0-1.0 range for (size_t y = 0; y < ysize; ++y) { for (size_t x = 0; x < xsize; ++x) { size_t j = (y * xsize + x) * 4; size_t i = y * stride + x * num_channels; @@ -541,35 +508,10 @@ result[j + 1] = g * mul; result[j + 2] = b * mul; result[j + 3] = a * mul; } } - } else if (format.data_type == JXL_TYPE_UINT32) { - double mul = 1.0 / 4294967295.0; // Multiplier to bring to 0-1.0 range - for (size_t y = 0; y < ysize; ++y) { - for (size_t x = 0; x < xsize; ++x) { - size_t j = (y * xsize + x) * 4; - size_t i = y * stride + x * num_channels * 4; - double r, g, b, a; - if (format.endianness == JXL_BIG_ENDIAN) { - r = LoadBE32(pixels + i); - g = gray ? r : LoadBE32(pixels + i + 4); - b = gray ? r : LoadBE32(pixels + i + 8); - a = alpha ? LoadBE32(pixels + i + num_channels * 2 - 4) : 4294967295; - - } else { - r = LoadLE32(pixels + i); - g = gray ? r : LoadLE32(pixels + i + 4); - b = gray ? r : LoadLE32(pixels + i + 8); - a = alpha ? LoadLE32(pixels + i + num_channels * 2 - 4) : 4294967295; - } - result[j + 0] = r * mul; - result[j + 1] = g * mul; - result[j + 2] = b * mul; - result[j + 3] = a * mul; - } - } } else if (format.data_type == JXL_TYPE_FLOAT) { for (size_t y = 0; y < ysize; ++y) { for (size_t x = 0; x < xsize; ++x) { size_t j = (y * xsize + x) * 4; size_t i = y * stride + x * num_channels * 4; diff --git a/third_party/jpeg-xl/lib/jxl_benchmark.cmake b/third_party/jpeg-xl/lib/jxl_benchmark.cmake --- a/third_party/jpeg-xl/lib/jxl_benchmark.cmake +++ b/third_party/jpeg-xl/lib/jxl_benchmark.cmake @@ -32,11 +32,11 @@ # Compiles all the benchmark files into a single binary. Individual benchmarks # can be run with --benchmark_filter. add_executable(jxl_gbench "${JPEGXL_INTERNAL_SOURCES_GBENCH}" gbench_main.cc) target_compile_definitions(jxl_gbench PRIVATE - -DTEST_DATA_PATH="${PROJECT_SOURCE_DIR}/third_party/testdata") + -DTEST_DATA_PATH="${JPEGXL_TEST_DATA_PATH}") target_link_libraries(jxl_gbench jxl_extras-static jxl-static benchmark::benchmark ) diff --git a/third_party/jpeg-xl/lib/jxl_extras.cmake b/third_party/jpeg-xl/lib/jxl_extras.cmake --- a/third_party/jpeg-xl/lib/jxl_extras.cmake +++ b/third_party/jpeg-xl/lib/jxl_extras.cmake @@ -4,12 +4,10 @@ # license that can be found in the LICENSE file. set(JPEGXL_EXTRAS_SOURCES extras/codec.cc extras/codec.h - extras/codec_psd.cc - extras/codec_psd.h extras/dec/color_description.cc extras/dec/color_description.h extras/dec/color_hints.cc extras/dec/color_hints.h extras/dec/decode.cc @@ -42,24 +40,25 @@ extras/dec/decode.h extras/dec/pgx.cc extras/dec/pgx.h extras/dec/pnm.cc extras/dec/pnm.h + extras/packed_image.h ) -# TODO(sboukortt): Make the code in dec/ sufficiently independent from the rest that it links on its own as a shared -# library. -add_library(jxl_extras_dec-static STATIC EXCLUDE_FROM_ALL - "${JPEGXL_EXTRAS_DEC_SOURCES}") -target_compile_options(jxl_extras_dec-static PRIVATE "${JPEGXL_INTERNAL_FLAGS}") -set_property(TARGET jxl_extras_dec-static PROPERTY POSITION_INDEPENDENT_CODE ON) -target_include_directories(jxl_extras_dec-static PUBLIC +add_library(jxl_extras_dec-obj OBJECT "${JPEGXL_EXTRAS_DEC_SOURCES}") +target_compile_options(jxl_extras_dec-obj PRIVATE "${JPEGXL_INTERNAL_FLAGS}") +target_compile_definitions(jxl_extras_dec-obj PRIVATE -DJXL_EXPORT=) +set_property(TARGET jxl_extras_dec-obj PROPERTY POSITION_INDEPENDENT_CODE ON) +target_include_directories(jxl_extras_dec-obj PUBLIC ${PROJECT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_BINARY_DIR}/include $ ) +set(JXL_EXTRAS_DEC_INTERNAL_LIBRARIES) +set(JXL_EXTRAS_DEC_PUBLIC_COMPILE_DEFINITIONS) # We only define a static library for jxl_extras since it uses internal parts # of jxl library which are not accessible from outside the library in the # shared library case. add_library(jxl_extras-static STATIC EXCLUDE_FROM_ALL @@ -72,32 +71,32 @@ jxl_extras_dec-static ) find_package(GIF 5.1) if(GIF_FOUND) - target_sources(jxl_extras_dec-static PRIVATE + target_sources(jxl_extras_dec-obj PRIVATE extras/dec/gif.cc extras/dec/gif.h ) - target_include_directories(jxl_extras_dec-static PRIVATE "${GIF_INCLUDE_DIRS}") - target_link_libraries(jxl_extras_dec-static PRIVATE ${GIF_LIBRARIES}) - target_compile_definitions(jxl_extras_dec-static PUBLIC -DJPEGXL_ENABLE_GIF=1) + target_include_directories(jxl_extras_dec-obj PRIVATE "${GIF_INCLUDE_DIRS}") + list(APPEND JXL_EXTRAS_DEC_INTERNAL_LIBRARIES ${GIF_LIBRARIES}) + list(APPEND JXL_EXTRAS_DEC_PUBLIC_DEFINITIONS -DJPEGXL_ENABLE_GIF=1) if(JPEGXL_DEP_LICENSE_DIR) configure_file("${JPEGXL_DEP_LICENSE_DIR}/libgif-dev/copyright" ${PROJECT_BINARY_DIR}/LICENSE.libgif COPYONLY) endif() # JPEGXL_DEP_LICENSE_DIR endif() find_package(JPEG) if(JPEG_FOUND) - target_sources(jxl_extras_dec-static PRIVATE + target_sources(jxl_extras_dec-obj PRIVATE extras/dec/jpg.cc extras/dec/jpg.h ) - target_include_directories(jxl_extras_dec-static PRIVATE "${JPEG_INCLUDE_DIRS}") - target_link_libraries(jxl_extras_dec-static PRIVATE ${JPEG_LIBRARIES}) - target_compile_definitions(jxl_extras_dec-static PUBLIC -DJPEGXL_ENABLE_JPEG=1) + target_include_directories(jxl_extras_dec-obj PRIVATE "${JPEG_INCLUDE_DIRS}") + list(APPEND JXL_EXTRAS_DEC_INTERNAL_LIBRARIES ${JPEG_LIBRARIES}) + list(APPEND JXL_EXTRAS_DEC_PUBLIC_DEFINITIONS -DJPEGXL_ENABLE_JPEG=1) target_sources(jxl_extras-static PRIVATE extras/enc/jpg.cc extras/enc/jpg.h ) target_include_directories(jxl_extras-static PRIVATE "${JPEG_INCLUDE_DIRS}") @@ -111,17 +110,17 @@ if(NOT JPEGXL_BUNDLE_LIBPNG) find_package(PNG) endif() if(PNG_FOUND) - target_sources(jxl_extras_dec-static PRIVATE + target_sources(jxl_extras_dec-obj PRIVATE extras/dec/apng.cc extras/dec/apng.h ) - target_include_directories(jxl_extras_dec-static PRIVATE "${PNG_INCLUDE_DIRS}") - target_link_libraries(jxl_extras_dec-static PRIVATE ${PNG_LIBRARIES}) - target_compile_definitions(jxl_extras_dec-static PUBLIC -DJPEGXL_ENABLE_APNG=1) + target_include_directories(jxl_extras_dec-obj PRIVATE "${PNG_INCLUDE_DIRS}") + list(APPEND JXL_EXTRAS_DEC_INTERNAL_LIBRARIES ${PNG_LIBRARIES}) + list(APPEND JXL_EXTRAS_DEC_PUBLIC_DEFINITIONS -DJPEGXL_ENABLE_APNG=1) target_sources(jxl_extras-static PRIVATE extras/enc/apng.cc extras/enc/apng.h ) target_include_directories(jxl_extras-static PUBLIC "${PNG_INCLUDE_DIRS}") @@ -137,16 +136,17 @@ endif () if (JPEGXL_ENABLE_OPENEXR) pkg_check_modules(OpenEXR IMPORTED_TARGET OpenEXR) if (OpenEXR_FOUND) - target_sources(jxl_extras_dec-static PRIVATE + target_sources(jxl_extras_dec-obj PRIVATE extras/dec/exr.cc extras/dec/exr.h ) - target_compile_definitions(jxl_extras_dec-static PUBLIC -DJPEGXL_ENABLE_EXR=1) - target_link_libraries(jxl_extras_dec-static PRIVATE PkgConfig::OpenEXR) + list(APPEND JXL_EXTRAS_DEC_PUBLIC_DEFINITIONS -DJPEGXL_ENABLE_EXR=1) + target_include_directories(jxl_extras_dec-obj PRIVATE "${OpenEXR_INCLUDE_DIRS}") + list(APPEND JXL_EXTRAS_DEC_INTERNAL_LIBRARIES PkgConfig::OpenEXR) target_sources(jxl_extras-static PRIVATE extras/enc/exr.cc extras/enc/exr.h ) target_compile_definitions(jxl_extras-static PUBLIC -DJPEGXL_ENABLE_EXR=1) @@ -163,5 +163,23 @@ set_source_files_properties(extras/dec/exr.cc extras/enc/exr.cc PROPERTIES COMPILE_FLAGS -fcxx-exceptions) endif() endif() endif() # OpenEXR_FOUND endif() # JPEGXL_ENABLE_OPENEXR + +target_compile_definitions(jxl_extras_dec-obj PRIVATE ${JXL_EXTRAS_DEC_PUBLIC_DEFINITIONS}) + +### Static library. +add_library(jxl_extras_dec-static STATIC $) +target_compile_definitions(jxl_extras_dec-static PUBLIC ${JXL_EXTRAS_DEC_PUBLIC_DEFINITIONS}) +target_link_libraries(jxl_extras_dec-static PRIVATE ${JXL_EXTRAS_DEC_INTERNAL_LIBRARIES}) + +### Shared library. +if (((NOT DEFINED "${TARGET_SUPPORTS_SHARED_LIBS}") OR + TARGET_SUPPORTS_SHARED_LIBS) AND NOT JPEGXL_STATIC AND BUILD_SHARED_LIBS) +add_library(jxl_extras_dec SHARED $) +target_compile_definitions(jxl_extras_dec PUBLIC ${JXL_EXTRAS_DEC_PUBLIC_DEFINITIONS}) +target_link_libraries(jxl_extras_dec PRIVATE ${JXL_EXTRAS_DEC_INTERNAL_LIBRARIES} jxl) +else() +add_library(jxl_extras_dec ALIAS jxl_extras_dec-static) +endif() # TARGET_SUPPORTS_SHARED_LIBS AND NOT JPEGXL_STATIC AND + # BUILD_SHARED_LIBS diff --git a/third_party/jpeg-xl/lib/jxl_tests.cmake b/third_party/jpeg-xl/lib/jxl_tests.cmake --- a/third_party/jpeg-xl/lib/jxl_tests.cmake +++ b/third_party/jpeg-xl/lib/jxl_tests.cmake @@ -83,11 +83,11 @@ target_compile_options(jxl_testlib-static PRIVATE ${JPEGXL_INTERNAL_FLAGS} ${JPEGXL_COVERAGE_FLAGS} ) target_compile_definitions(jxl_testlib-static PUBLIC - -DTEST_DATA_PATH="${PROJECT_SOURCE_DIR}/third_party/testdata") + -DTEST_DATA_PATH="${JPEGXL_TEST_DATA_PATH}") target_include_directories(jxl_testlib-static PUBLIC "${PROJECT_SOURCE_DIR}" ) target_link_libraries(jxl_testlib-static hwy jxl-static) diff --git a/third_party/jpeg-xl/lib/lib.gni b/third_party/jpeg-xl/lib/lib.gni --- a/third_party/jpeg-xl/lib/lib.gni +++ b/third_party/jpeg-xl/lib/lib.gni @@ -55,13 +55,13 @@ "jxl/base/padded_bytes.h", "jxl/base/printf_macros.h", "jxl/base/profiler.h", "jxl/base/random.cc", "jxl/base/random.h", + "jxl/base/sanitizer_definitions.h", "jxl/base/scope_guard.h", "jxl/base/span.h", - "jxl/base/status.cc", "jxl/base/status.h", "jxl/base/thread_pool_internal.h", "jxl/blending.cc", "jxl/blending.h", "jxl/box_content_decoder.cc", @@ -122,10 +122,12 @@ "jxl/enc_bit_writer.h", "jxl/entropy_coder.cc", "jxl/entropy_coder.h", "jxl/epf.cc", "jxl/epf.h", + "jxl/exif.cc", + "jxl/exif.h", "jxl/fast_dct-inl.h", "jxl/fast_dct.cc", "jxl/fast_dct.h", "jxl/fast_dct128-inl.h", "jxl/fast_dct16-inl.h", @@ -434,12 +436,10 @@ ] libjxl_extras_sources = [ "extras/codec.cc", "extras/codec.h", - "extras/codec_psd.cc", - "extras/codec_psd.h", "extras/dec/color_description.cc", "extras/dec/color_description.h", "extras/dec/color_hints.cc", "extras/dec/color_hints.h", "extras/dec/decode.cc", diff --git a/third_party/jpeg-xl/lib/threads/thread_parallel_runner.cc b/third_party/jpeg-xl/lib/threads/thread_parallel_runner.cc --- a/third_party/jpeg-xl/lib/threads/thread_parallel_runner.cc +++ b/third_party/jpeg-xl/lib/threads/thread_parallel_runner.cc @@ -86,13 +86,14 @@ void JxlThreadParallelRunnerDestroy(void* runner_opaque) { jpegxl::ThreadParallelRunner* runner = reinterpret_cast(runner_opaque); if (runner) { + JxlMemoryManager local_memory_manager = runner->memory_manager; // Call destructor directly since custom free function is used. runner->~ThreadParallelRunner(); - ThreadMemoryManagerFree(&runner->memory_manager, runner); + ThreadMemoryManagerFree(&local_memory_manager, runner); } } // Get default value for num_worker_threads parameter of // InitJxlThreadParallelRunner. diff --git a/third_party/jpeg-xl/plugins/gdk-pixbuf/CMakeLists.txt b/third_party/jpeg-xl/plugins/gdk-pixbuf/CMakeLists.txt --- a/third_party/jpeg-xl/plugins/gdk-pixbuf/CMakeLists.txt +++ b/third_party/jpeg-xl/plugins/gdk-pixbuf/CMakeLists.txt @@ -61,16 +61,20 @@ set(XVFB_PROGRAM_PREFIX "${XVFB_PROGRAM};-a") else() set(XVFB_PROGRAM_PREFIX "") endif() - add_test( - NAME pixbufloader_test_jxl - COMMAND - ${XVFB_PROGRAM_PREFIX} $ - "${CMAKE_CURRENT_SOURCE_DIR}/loaders_test.cache" - "${CMAKE_SOURCE_DIR}/third_party/testdata/jxl/blending/cropped_traffic_light.jxl" - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - ) - set_tests_properties(pixbufloader_test_jxl PROPERTIES SKIP_RETURN_CODE 254) + # libX11.so and libgdk-x11-2.0.so are not compiled with MSAN -> report + # use-of-uninitialized-value for string some internal string value. + if (NOT (SANITIZER STREQUAL "msan")) + add_test( + NAME pixbufloader_test_jxl + COMMAND + ${XVFB_PROGRAM_PREFIX} $ + "${CMAKE_CURRENT_SOURCE_DIR}/loaders_test.cache" + "${CMAKE_SOURCE_DIR}/third_party/testdata/jxl/blending/cropped_traffic_light.jxl" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + set_tests_properties(pixbufloader_test_jxl PROPERTIES SKIP_RETURN_CODE 254) + endif() endif() # Gdk_FOUND endif() # BUILD_TESTING diff --git a/third_party/jpeg-xl/third_party/CMakeLists.txt b/third_party/jpeg-xl/third_party/CMakeLists.txt --- a/third_party/jpeg-xl/third_party/CMakeLists.txt +++ b/third_party/jpeg-xl/third_party/CMakeLists.txt @@ -161,25 +161,32 @@ if(JPEGXL_ENABLE_TOOLS) if (JPEGXL_BUNDLE_GFLAGS) # Compile gflags from sources. set(GFLAGS_BUILD_TESTING OFF) add_subdirectory(gflags) + add_library(jxl_gflags ALIAS gflags_nothreads_static) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/gflags/COPYING.txt" ${PROJECT_BINARY_DIR}/LICENSE.gflags COPYONLY) else() # Use sytem libgflags-dev - find_package(gflags REQUIRED) + find_package(gflags) if (NOT gflags_FOUND) message(FATAL_ERROR "Gflags not found, install libgflags-dev or download gflags source to" "third_party/gflags from https://github.com/gflags/gflags. You can use" " ${PROJECT_SOURCE_DIR}/deps.sh to download the dependency.") endif() if(JPEGXL_DEP_LICENSE_DIR) configure_file("${JPEGXL_DEP_LICENSE_DIR}/libgflags-dev/copyright" ${PROJECT_BINARY_DIR}/LICENSE.gflags COPYONLY) endif() # JPEGXL_DEP_LICENSE_DIR + add_library(jxl_gflags INTERFACE) + if(JPEGXL_STATIC) + target_link_libraries(jxl_gflags INTERFACE gflags_static) + else() + target_link_libraries(jxl_gflags INTERFACE gflags) + endif() endif() endif() # libpng if (JPEGXL_EMSCRIPTEN) @@ -209,10 +216,12 @@ set(ZLIB_LIBRARY zlibstatic) add_subdirectory(libpng EXCLUDE_FROM_ALL) set(PNG_FOUND YES PARENT_SCOPE) set(PNG_INCLUDE_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/libpng/" PARENT_SCOPE) set(PNG_LIBRARIES png_static PARENT_SCOPE) + set_property(TARGET png_static PROPERTY POSITION_INDEPENDENT_CODE ON) + set_property(TARGET zlibstatic PROPERTY POSITION_INDEPENDENT_CODE ON) if(JPEGXL_DEP_LICENSE_DIR) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/libpng/LICENSE" ${PROJECT_BINARY_DIR}/LICENSE.libpng COPYONLY) endif() else()