diff --git a/media/libjxl/moz.yaml b/media/libjxl/moz.yaml --- a/media/libjxl/moz.yaml +++ b/media/libjxl/moz.yaml @@ -8,13 +8,13 @@ name: libjxl description: JPEG XL image format reference implementation url: https://github.com/libjxl/libjxl - release: commit 318c592d98b97d103941b90d47107f06a10c71da (2022-03-21T20:44:46Z). + release: c0249504f1d343a1e0415c2d7af4578cd9ef3320 (2022-05-13T16:23:26Z). - revision: 318c592d98b97d103941b90d47107f06a10c71da + revision: c0249504f1d343a1e0415c2d7af4578cd9ef3320 license: Apache-2.0 license-file: LICENSE diff --git a/media/libvpx/config/generic/vp8_rtcd.h b/media/libvpx/config/generic/vp8_rtcd.h --- a/media/libvpx/config/generic/vp8_rtcd.h +++ b/media/libvpx/config/generic/vp8_rtcd.h @@ -79,13 +79,10 @@ #define vp8_diamond_search_sad vp8_diamond_search_sad_c void vp8_fast_quantize_b_c(struct block *, struct blockd *); #define vp8_fast_quantize_b vp8_fast_quantize_b_c -int vp8_full_search_sad_c(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -#define vp8_full_search_sad vp8_full_search_sad_c - void vp8_loop_filter_bh_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); #define vp8_loop_filter_bh vp8_loop_filter_bh_c void vp8_loop_filter_bv_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); #define vp8_loop_filter_bv vp8_loop_filter_bv_c diff --git a/media/libvpx/config/generic/vp9_rtcd.h b/media/libvpx/config/generic/vp9_rtcd.h --- a/media/libvpx/config/generic/vp9_rtcd.h +++ b/media/libvpx/config/generic/vp9_rtcd.h @@ -62,14 +62,14 @@ #define vp9_iht4x4_16_add vp9_iht4x4_16_add_c void vp9_iht8x8_64_add_c(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); #define vp9_iht8x8_64_add vp9_iht8x8_64_add_c -void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); #define vp9_quantize_fp vp9_quantize_fp_c -void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); #define vp9_quantize_fp_32x32 vp9_quantize_fp_32x32_c void vp9_scale_and_extend_frame_c(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); #define vp9_scale_and_extend_frame vp9_scale_and_extend_frame_c diff --git a/media/libvpx/config/generic/vpx_config.h b/media/libvpx/config/generic/vpx_config.h --- a/media/libvpx/config/generic/vpx_config.h +++ b/media/libvpx/config/generic/vpx_config.h @@ -13,10 +13,11 @@ #define VPX_ARCH_ARM 0 #define VPX_ARCH_MIPS 0 #define VPX_ARCH_X86 0 #define VPX_ARCH_X86_64 0 #define VPX_ARCH_PPC 0 +#define VPX_ARCH_LOONGARCH 0 #define HAVE_NEON 0 #define HAVE_NEON_ASM 0 #define HAVE_MIPS32 0 #define HAVE_DSPR2 0 #define HAVE_MSA 0 @@ -30,10 +31,12 @@ #define HAVE_AVX 0 #define HAVE_AVX2 0 #define HAVE_AVX512 0 #define HAVE_VSX 0 #define HAVE_MMI 0 +#define HAVE_LSX 0 +#define HAVE_LASX 0 #define HAVE_VPX_PORTS 1 #define HAVE_PTHREAD_H 1 #define CONFIG_DEPENDENCY_TRACKING 1 #define CONFIG_EXTERNAL_BUILD 1 #define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/generic/vpx_config.asm b/media/libvpx/config/generic/vpx_config.asm --- a/media/libvpx/config/generic/vpx_config.asm +++ b/media/libvpx/config/generic/vpx_config.asm @@ -1,13 +1,14 @@ @ This file was created from a .asm file @ using the ads2gas.pl script. - .syntax unified +.syntax unified .equ VPX_ARCH_ARM , 0 .equ VPX_ARCH_MIPS , 0 .equ VPX_ARCH_X86 , 0 .equ VPX_ARCH_X86_64 , 0 .equ VPX_ARCH_PPC , 0 +.equ VPX_ARCH_LOONGARCH , 0 .equ HAVE_NEON , 0 .equ HAVE_NEON_ASM , 0 .equ HAVE_MIPS32 , 0 .equ HAVE_DSPR2 , 0 .equ HAVE_MSA , 0 @@ -21,10 +22,12 @@ .equ HAVE_AVX , 0 .equ HAVE_AVX2 , 0 .equ HAVE_AVX512 , 0 .equ HAVE_VSX , 0 .equ HAVE_MMI , 0 +.equ HAVE_LSX , 0 +.equ HAVE_LASX , 0 .equ HAVE_VPX_PORTS , 1 .equ HAVE_PTHREAD_H , 1 .equ CONFIG_DEPENDENCY_TRACKING , 1 .equ CONFIG_EXTERNAL_BUILD , 1 .equ CONFIG_INSTALL_DOCS , 0 @@ -85,6 +88,6 @@ .equ CONFIG_MISMATCH_DEBUG , 0 .equ CONFIG_FP_MB_STATS , 0 .equ CONFIG_EMULATE_HARDWARE , 0 .equ CONFIG_NON_GREEDY_MV , 0 .equ CONFIG_RATE_CTRL , 0 - .section .note.GNU-stack,"",%progbits + .section .note.GNU-stack,"",%progbits diff --git a/media/libvpx/config/generic/vpx_dsp_rtcd.h b/media/libvpx/config/generic/vpx_dsp_rtcd.h --- a/media/libvpx/config/generic/vpx_dsp_rtcd.h +++ b/media/libvpx/config/generic/vpx_dsp_rtcd.h @@ -283,11 +283,11 @@ #define vpx_idct8x8_64_add vpx_idct8x8_64_add_c int16_t vpx_int_pro_col_c(const uint8_t *ref, const int width); #define vpx_int_pro_col vpx_int_pro_col_c -void vpx_int_pro_row_c(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_c(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); #define vpx_int_pro_row vpx_int_pro_row_c void vpx_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int stride); #define vpx_iwht4x4_16_add vpx_iwht4x4_16_add_c @@ -343,166 +343,133 @@ #define vpx_mse8x16 vpx_mse8x16_c unsigned int vpx_mse8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse); #define vpx_mse8x8 vpx_mse8x8_c -void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); #define vpx_quantize_b vpx_quantize_b_c -void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); #define vpx_quantize_b_32x32 vpx_quantize_b_32x32_c unsigned int vpx_sad16x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x16 vpx_sad16x16_c unsigned int vpx_sad16x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x16_avg vpx_sad16x16_avg_c -void vpx_sad16x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x16x3 vpx_sad16x16x3_c - -void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x16x4d vpx_sad16x16x4d_c -void vpx_sad16x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x16x8 vpx_sad16x16x8_c - unsigned int vpx_sad16x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x32 vpx_sad16x32_c unsigned int vpx_sad16x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x32_avg vpx_sad16x32_avg_c -void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x32x4d vpx_sad16x32x4d_c unsigned int vpx_sad16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x8 vpx_sad16x8_c unsigned int vpx_sad16x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x8_avg vpx_sad16x8_avg_c -void vpx_sad16x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x8x3 vpx_sad16x8x3_c - -void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x8x4d vpx_sad16x8x4d_c -void vpx_sad16x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x8x8 vpx_sad16x8x8_c - unsigned int vpx_sad32x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad32x16 vpx_sad32x16_c unsigned int vpx_sad32x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad32x16_avg vpx_sad32x16_avg_c -void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad32x16x4d vpx_sad32x16x4d_c unsigned int vpx_sad32x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad32x32 vpx_sad32x32_c unsigned int vpx_sad32x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad32x32_avg vpx_sad32x32_avg_c -void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad32x32x4d vpx_sad32x32x4d_c -void vpx_sad32x32x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad32x32x8 vpx_sad32x32x8_c - unsigned int vpx_sad32x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad32x64 vpx_sad32x64_c unsigned int vpx_sad32x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad32x64_avg vpx_sad32x64_avg_c -void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad32x64x4d vpx_sad32x64x4d_c unsigned int vpx_sad4x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad4x4 vpx_sad4x4_c unsigned int vpx_sad4x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad4x4_avg vpx_sad4x4_avg_c -void vpx_sad4x4x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad4x4x3 vpx_sad4x4x3_c - -void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad4x4x4d vpx_sad4x4x4d_c -void vpx_sad4x4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad4x4x8 vpx_sad4x4x8_c - unsigned int vpx_sad4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad4x8 vpx_sad4x8_c unsigned int vpx_sad4x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad4x8_avg vpx_sad4x8_avg_c -void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad4x8x4d vpx_sad4x8x4d_c unsigned int vpx_sad64x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad64x32 vpx_sad64x32_c unsigned int vpx_sad64x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad64x32_avg vpx_sad64x32_avg_c -void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad64x32x4d vpx_sad64x32x4d_c unsigned int vpx_sad64x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad64x64 vpx_sad64x64_c unsigned int vpx_sad64x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad64x64_avg vpx_sad64x64_avg_c -void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad64x64x4d vpx_sad64x64x4d_c unsigned int vpx_sad8x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x16 vpx_sad8x16_c unsigned int vpx_sad8x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x16_avg vpx_sad8x16_avg_c -void vpx_sad8x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x16x3 vpx_sad8x16x3_c - -void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x16x4d vpx_sad8x16x4d_c -void vpx_sad8x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x16x8 vpx_sad8x16x8_c - unsigned int vpx_sad8x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x4 vpx_sad8x4_c unsigned int vpx_sad8x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x4_avg vpx_sad8x4_avg_c -void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x4x4d vpx_sad8x4x4d_c unsigned int vpx_sad8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x8 vpx_sad8x8_c unsigned int vpx_sad8x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x8_avg vpx_sad8x8_avg_c -void vpx_sad8x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x8x3 vpx_sad8x8x3_c - -void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x8x4d vpx_sad8x8x4d_c -void vpx_sad8x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x8x8 vpx_sad8x8x8_c - int vpx_satd_c(const int16_t *coeff, int length); #define vpx_satd vpx_satd_c void vpx_scaled_2d_c(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h); #define vpx_scaled_2d vpx_scaled_2d_c diff --git a/media/libvpx/config/linux/arm/vp8_rtcd.h b/media/libvpx/config/linux/arm/vp8_rtcd.h --- a/media/libvpx/config/linux/arm/vp8_rtcd.h +++ b/media/libvpx/config/linux/arm/vp8_rtcd.h @@ -94,13 +94,10 @@ void vp8_fast_quantize_b_c(struct block *, struct blockd *); void vp8_fast_quantize_b_neon(struct block *, struct blockd *); RTCD_EXTERN void (*vp8_fast_quantize_b)(struct block *, struct blockd *); -int vp8_full_search_sad_c(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -#define vp8_full_search_sad vp8_full_search_sad_c - void vp8_loop_filter_bh_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); void vp8_loop_filter_bh_neon(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); RTCD_EXTERN void (*vp8_loop_filter_bh)(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); void vp8_loop_filter_bv_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); diff --git a/media/libvpx/config/linux/arm/vp9_rtcd.h b/media/libvpx/config/linux/arm/vp9_rtcd.h --- a/media/libvpx/config/linux/arm/vp9_rtcd.h +++ b/media/libvpx/config/linux/arm/vp9_rtcd.h @@ -37,20 +37,24 @@ int64_t vp9_block_error_fp_c(const tran_low_t *coeff, const tran_low_t *dqcoeff, int block_size); int64_t vp9_block_error_fp_neon(const tran_low_t *coeff, const tran_low_t *dqcoeff, int block_size); RTCD_EXTERN int64_t (*vp9_block_error_fp)(const tran_low_t *coeff, const tran_low_t *dqcoeff, int block_size); int vp9_diamond_search_sad_c(const struct macroblock *x, const struct search_site_config *cfg, struct mv *ref_mv, struct mv *best_mv, int search_param, int sad_per_bit, int *num00, const struct vp9_variance_vtable *fn_ptr, const struct mv *center_mv); -#define vp9_diamond_search_sad vp9_diamond_search_sad_c +int vp9_diamond_search_sad_neon(const struct macroblock *x, const struct search_site_config *cfg, struct mv *ref_mv, struct mv *best_mv, int search_param, int sad_per_bit, int *num00, const struct vp9_variance_vtable *fn_ptr, const struct mv *center_mv); +RTCD_EXTERN int (*vp9_diamond_search_sad)(const struct macroblock *x, const struct search_site_config *cfg, struct mv *ref_mv, struct mv *best_mv, int search_param, int sad_per_bit, int *num00, const struct vp9_variance_vtable *fn_ptr, const struct mv *center_mv); void vp9_fht16x16_c(const int16_t *input, tran_low_t *output, int stride, int tx_type); -#define vp9_fht16x16 vp9_fht16x16_c +void vp9_fht16x16_neon(const int16_t *input, tran_low_t *output, int stride, int tx_type); +RTCD_EXTERN void (*vp9_fht16x16)(const int16_t *input, tran_low_t *output, int stride, int tx_type); void vp9_fht4x4_c(const int16_t *input, tran_low_t *output, int stride, int tx_type); -#define vp9_fht4x4 vp9_fht4x4_c +void vp9_fht4x4_neon(const int16_t *input, tran_low_t *output, int stride, int tx_type); +RTCD_EXTERN void (*vp9_fht4x4)(const int16_t *input, tran_low_t *output, int stride, int tx_type); void vp9_fht8x8_c(const int16_t *input, tran_low_t *output, int stride, int tx_type); -#define vp9_fht8x8 vp9_fht8x8_c +void vp9_fht8x8_neon(const int16_t *input, tran_low_t *output, int stride, int tx_type); +RTCD_EXTERN void (*vp9_fht8x8)(const int16_t *input, tran_low_t *output, int stride, int tx_type); void vp9_fwht4x4_c(const int16_t *input, tran_low_t *output, int stride); #define vp9_fwht4x4 vp9_fwht4x4_c void vp9_iht16x16_256_add_c(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); @@ -63,17 +67,17 @@ void vp9_iht8x8_64_add_c(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); void vp9_iht8x8_64_add_neon(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); RTCD_EXTERN void (*vp9_iht8x8_64_add)(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); -void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vp9_quantize_fp)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vp9_quantize_fp)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vp9_quantize_fp_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vp9_quantize_fp_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); void vp9_scale_and_extend_frame_c(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); void vp9_scale_and_extend_frame_neon(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); RTCD_EXTERN void (*vp9_scale_and_extend_frame)(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); @@ -89,10 +93,18 @@ (void)flags; vp9_block_error_fp = vp9_block_error_fp_c; if (flags & HAS_NEON) vp9_block_error_fp = vp9_block_error_fp_neon; + vp9_diamond_search_sad = vp9_diamond_search_sad_c; + if (flags & HAS_NEON) vp9_diamond_search_sad = vp9_diamond_search_sad_neon; + vp9_fht16x16 = vp9_fht16x16_c; + if (flags & HAS_NEON) vp9_fht16x16 = vp9_fht16x16_neon; + vp9_fht4x4 = vp9_fht4x4_c; + if (flags & HAS_NEON) vp9_fht4x4 = vp9_fht4x4_neon; + vp9_fht8x8 = vp9_fht8x8_c; + if (flags & HAS_NEON) vp9_fht8x8 = vp9_fht8x8_neon; vp9_iht16x16_256_add = vp9_iht16x16_256_add_c; if (flags & HAS_NEON) vp9_iht16x16_256_add = vp9_iht16x16_256_add_neon; vp9_iht4x4_16_add = vp9_iht4x4_16_add_c; if (flags & HAS_NEON) vp9_iht4x4_16_add = vp9_iht4x4_16_add_neon; vp9_iht8x8_64_add = vp9_iht8x8_64_add_c; diff --git a/media/libvpx/config/linux/arm/vpx_config.h b/media/libvpx/config/linux/arm/vpx_config.h --- a/media/libvpx/config/linux/arm/vpx_config.h +++ b/media/libvpx/config/linux/arm/vpx_config.h @@ -13,10 +13,11 @@ #define VPX_ARCH_ARM 1 #define VPX_ARCH_MIPS 0 #define VPX_ARCH_X86 0 #define VPX_ARCH_X86_64 0 #define VPX_ARCH_PPC 0 +#define VPX_ARCH_LOONGARCH 0 #define HAVE_NEON 1 #define HAVE_NEON_ASM 1 #define HAVE_MIPS32 0 #define HAVE_DSPR2 0 #define HAVE_MSA 0 @@ -30,10 +31,12 @@ #define HAVE_AVX 0 #define HAVE_AVX2 0 #define HAVE_AVX512 0 #define HAVE_VSX 0 #define HAVE_MMI 0 +#define HAVE_LSX 0 +#define HAVE_LASX 0 #define HAVE_VPX_PORTS 1 #define HAVE_PTHREAD_H 1 #define CONFIG_DEPENDENCY_TRACKING 1 #define CONFIG_EXTERNAL_BUILD 1 #define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/linux/arm/vpx_config.asm b/media/libvpx/config/linux/arm/vpx_config.asm --- a/media/libvpx/config/linux/arm/vpx_config.asm +++ b/media/libvpx/config/linux/arm/vpx_config.asm @@ -1,13 +1,14 @@ @ This file was created from a .asm file @ using the ads2gas.pl script. - .syntax unified +.syntax unified .equ VPX_ARCH_ARM , 1 .equ VPX_ARCH_MIPS , 0 .equ VPX_ARCH_X86 , 0 .equ VPX_ARCH_X86_64 , 0 .equ VPX_ARCH_PPC , 0 +.equ VPX_ARCH_LOONGARCH , 0 .equ HAVE_NEON , 1 .equ HAVE_NEON_ASM , 1 .equ HAVE_MIPS32 , 0 .equ HAVE_DSPR2 , 0 .equ HAVE_MSA , 0 @@ -21,10 +22,12 @@ .equ HAVE_AVX , 0 .equ HAVE_AVX2 , 0 .equ HAVE_AVX512 , 0 .equ HAVE_VSX , 0 .equ HAVE_MMI , 0 +.equ HAVE_LSX , 0 +.equ HAVE_LASX , 0 .equ HAVE_VPX_PORTS , 1 .equ HAVE_PTHREAD_H , 1 .equ CONFIG_DEPENDENCY_TRACKING , 1 .equ CONFIG_EXTERNAL_BUILD , 1 .equ CONFIG_INSTALL_DOCS , 0 @@ -85,6 +88,6 @@ .equ CONFIG_MISMATCH_DEBUG , 0 .equ CONFIG_FP_MB_STATS , 0 .equ CONFIG_EMULATE_HARDWARE , 0 .equ CONFIG_NON_GREEDY_MV , 0 .equ CONFIG_RATE_CTRL , 0 - .section .note.GNU-stack,"",%progbits + .section .note.GNU-stack,"",%progbits diff --git a/media/libvpx/config/linux/arm/vpx_dsp_rtcd.h b/media/libvpx/config/linux/arm/vpx_dsp_rtcd.h --- a/media/libvpx/config/linux/arm/vpx_dsp_rtcd.h +++ b/media/libvpx/config/linux/arm/vpx_dsp_rtcd.h @@ -350,13 +350,13 @@ int16_t vpx_int_pro_col_c(const uint8_t *ref, const int width); int16_t vpx_int_pro_col_neon(const uint8_t *ref, const int width); RTCD_EXTERN int16_t (*vpx_int_pro_col)(const uint8_t *ref, const int width); -void vpx_int_pro_row_c(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); -void vpx_int_pro_row_neon(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); -RTCD_EXTERN void (*vpx_int_pro_row)(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_c(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_neon(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); +RTCD_EXTERN void (*vpx_int_pro_row)(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); void vpx_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int stride); #define vpx_iwht4x4_16_add vpx_iwht4x4_16_add_c void vpx_iwht4x4_1_add_c(const tran_low_t *input, uint8_t *dest, int stride); @@ -425,206 +425,173 @@ #define vpx_mse8x16 vpx_mse8x16_c unsigned int vpx_mse8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse); #define vpx_mse8x8 vpx_mse8x8_c -void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vpx_quantize_b)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vpx_quantize_b)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vpx_quantize_b_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vpx_quantize_b_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); unsigned int vpx_sad16x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x16_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad16x16)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x16_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad16x16_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad16x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x16x3 vpx_sad16x16x3_c - -void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad16x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x16x8 vpx_sad16x16x8_c +void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad16x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad16x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x32_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad16x32)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x32_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad16x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad16x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x8_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x8_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad16x8_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad16x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x8x3 vpx_sad16x8x3_c - -void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad16x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x8x8 vpx_sad16x8x8_c +void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad16x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad32x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x16)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x16_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x16_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad32x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad32x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x32)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x32_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad32x32x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad32x32x8 vpx_sad32x32x8_c +void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad32x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad32x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x64)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x64_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x64_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad32x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad4x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x4_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad4x4)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x4_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad4x4_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad4x4x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad4x4x3 vpx_sad4x4x3_c - -void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad4x4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad4x4x8 vpx_sad4x4x8_c +void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x4x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad4x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x8_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad4x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x8_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad4x8_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad4x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad64x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad64x32)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x32_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad64x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad64x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad64x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad64x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad64x64)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x64_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad64x64_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad64x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad64x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad8x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x16_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad8x16)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x16_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad8x16_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad8x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x16x3 vpx_sad8x16x3_c - -void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad8x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x16x8 vpx_sad8x16x8_c +void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad8x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad8x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x4_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad8x4)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x4_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad8x4_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x4x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x4x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad8x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x8_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x8_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad8x8_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad8x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x8x3 vpx_sad8x8x3_c - -void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad8x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x8x8 vpx_sad8x8x8_c +void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad8x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); int vpx_satd_c(const int16_t *coeff, int length); int vpx_satd_neon(const int16_t *coeff, int length); RTCD_EXTERN int (*vpx_satd)(const int16_t *coeff, int length); diff --git a/media/libvpx/config/linux/arm64/vp8_rtcd.h b/media/libvpx/config/linux/arm64/vp8_rtcd.h --- a/media/libvpx/config/linux/arm64/vp8_rtcd.h +++ b/media/libvpx/config/linux/arm64/vp8_rtcd.h @@ -94,13 +94,10 @@ void vp8_fast_quantize_b_c(struct block *, struct blockd *); void vp8_fast_quantize_b_neon(struct block *, struct blockd *); #define vp8_fast_quantize_b vp8_fast_quantize_b_neon -int vp8_full_search_sad_c(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -#define vp8_full_search_sad vp8_full_search_sad_c - void vp8_loop_filter_bh_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); void vp8_loop_filter_bh_neon(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); #define vp8_loop_filter_bh vp8_loop_filter_bh_neon void vp8_loop_filter_bv_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); diff --git a/media/libvpx/config/linux/arm64/vp9_rtcd.h b/media/libvpx/config/linux/arm64/vp9_rtcd.h --- a/media/libvpx/config/linux/arm64/vp9_rtcd.h +++ b/media/libvpx/config/linux/arm64/vp9_rtcd.h @@ -37,20 +37,24 @@ int64_t vp9_block_error_fp_c(const tran_low_t *coeff, const tran_low_t *dqcoeff, int block_size); int64_t vp9_block_error_fp_neon(const tran_low_t *coeff, const tran_low_t *dqcoeff, int block_size); #define vp9_block_error_fp vp9_block_error_fp_neon int vp9_diamond_search_sad_c(const struct macroblock *x, const struct search_site_config *cfg, struct mv *ref_mv, struct mv *best_mv, int search_param, int sad_per_bit, int *num00, const struct vp9_variance_vtable *fn_ptr, const struct mv *center_mv); -#define vp9_diamond_search_sad vp9_diamond_search_sad_c +int vp9_diamond_search_sad_neon(const struct macroblock *x, const struct search_site_config *cfg, struct mv *ref_mv, struct mv *best_mv, int search_param, int sad_per_bit, int *num00, const struct vp9_variance_vtable *fn_ptr, const struct mv *center_mv); +#define vp9_diamond_search_sad vp9_diamond_search_sad_neon void vp9_fht16x16_c(const int16_t *input, tran_low_t *output, int stride, int tx_type); -#define vp9_fht16x16 vp9_fht16x16_c +void vp9_fht16x16_neon(const int16_t *input, tran_low_t *output, int stride, int tx_type); +#define vp9_fht16x16 vp9_fht16x16_neon void vp9_fht4x4_c(const int16_t *input, tran_low_t *output, int stride, int tx_type); -#define vp9_fht4x4 vp9_fht4x4_c +void vp9_fht4x4_neon(const int16_t *input, tran_low_t *output, int stride, int tx_type); +#define vp9_fht4x4 vp9_fht4x4_neon void vp9_fht8x8_c(const int16_t *input, tran_low_t *output, int stride, int tx_type); -#define vp9_fht8x8 vp9_fht8x8_c +void vp9_fht8x8_neon(const int16_t *input, tran_low_t *output, int stride, int tx_type); +#define vp9_fht8x8 vp9_fht8x8_neon void vp9_fwht4x4_c(const int16_t *input, tran_low_t *output, int stride); #define vp9_fwht4x4 vp9_fwht4x4_c void vp9_iht16x16_256_add_c(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); @@ -63,16 +67,16 @@ void vp9_iht8x8_64_add_c(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); void vp9_iht8x8_64_add_neon(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); #define vp9_iht8x8_64_add vp9_iht8x8_64_add_neon -void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); #define vp9_quantize_fp vp9_quantize_fp_neon -void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); #define vp9_quantize_fp_32x32 vp9_quantize_fp_32x32_neon void vp9_scale_and_extend_frame_c(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); void vp9_scale_and_extend_frame_neon(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); #define vp9_scale_and_extend_frame vp9_scale_and_extend_frame_neon diff --git a/media/libvpx/config/linux/arm64/vpx_config.h b/media/libvpx/config/linux/arm64/vpx_config.h --- a/media/libvpx/config/linux/arm64/vpx_config.h +++ b/media/libvpx/config/linux/arm64/vpx_config.h @@ -13,10 +13,11 @@ #define VPX_ARCH_ARM 1 #define VPX_ARCH_MIPS 0 #define VPX_ARCH_X86 0 #define VPX_ARCH_X86_64 0 #define VPX_ARCH_PPC 0 +#define VPX_ARCH_LOONGARCH 0 #define HAVE_NEON 1 #define HAVE_NEON_ASM 0 #define HAVE_MIPS32 0 #define HAVE_DSPR2 0 #define HAVE_MSA 0 @@ -30,10 +31,12 @@ #define HAVE_AVX 0 #define HAVE_AVX2 0 #define HAVE_AVX512 0 #define HAVE_VSX 0 #define HAVE_MMI 0 +#define HAVE_LSX 0 +#define HAVE_LASX 0 #define HAVE_VPX_PORTS 1 #define HAVE_PTHREAD_H 1 #define CONFIG_DEPENDENCY_TRACKING 1 #define CONFIG_EXTERNAL_BUILD 1 #define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/linux/arm64/vpx_config.asm b/media/libvpx/config/linux/arm64/vpx_config.asm --- a/media/libvpx/config/linux/arm64/vpx_config.asm +++ b/media/libvpx/config/linux/arm64/vpx_config.asm @@ -1,13 +1,14 @@ @ This file was created from a .asm file @ using the ads2gas.pl script. - .syntax unified +.syntax unified .equ VPX_ARCH_ARM , 1 .equ VPX_ARCH_MIPS , 0 .equ VPX_ARCH_X86 , 0 .equ VPX_ARCH_X86_64 , 0 .equ VPX_ARCH_PPC , 0 +.equ VPX_ARCH_LOONGARCH , 0 .equ HAVE_NEON , 1 .equ HAVE_NEON_ASM , 0 .equ HAVE_MIPS32 , 0 .equ HAVE_DSPR2 , 0 .equ HAVE_MSA , 0 @@ -21,10 +22,12 @@ .equ HAVE_AVX , 0 .equ HAVE_AVX2 , 0 .equ HAVE_AVX512 , 0 .equ HAVE_VSX , 0 .equ HAVE_MMI , 0 +.equ HAVE_LSX , 0 +.equ HAVE_LASX , 0 .equ HAVE_VPX_PORTS , 1 .equ HAVE_PTHREAD_H , 1 .equ CONFIG_DEPENDENCY_TRACKING , 1 .equ CONFIG_EXTERNAL_BUILD , 1 .equ CONFIG_INSTALL_DOCS , 0 @@ -85,6 +88,6 @@ .equ CONFIG_MISMATCH_DEBUG , 0 .equ CONFIG_FP_MB_STATS , 0 .equ CONFIG_EMULATE_HARDWARE , 0 .equ CONFIG_NON_GREEDY_MV , 0 .equ CONFIG_RATE_CTRL , 0 - .section .note.GNU-stack,"",%progbits + .section .note.GNU-stack,"",%progbits diff --git a/media/libvpx/config/linux/arm64/vpx_dsp_rtcd.h b/media/libvpx/config/linux/arm64/vpx_dsp_rtcd.h --- a/media/libvpx/config/linux/arm64/vpx_dsp_rtcd.h +++ b/media/libvpx/config/linux/arm64/vpx_dsp_rtcd.h @@ -350,12 +350,12 @@ int16_t vpx_int_pro_col_c(const uint8_t *ref, const int width); int16_t vpx_int_pro_col_neon(const uint8_t *ref, const int width); #define vpx_int_pro_col vpx_int_pro_col_neon -void vpx_int_pro_row_c(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); -void vpx_int_pro_row_neon(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_c(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_neon(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); #define vpx_int_pro_row vpx_int_pro_row_neon void vpx_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int stride); #define vpx_iwht4x4_16_add vpx_iwht4x4_16_add_c @@ -425,207 +425,174 @@ #define vpx_mse8x16 vpx_mse8x16_c unsigned int vpx_mse8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse); #define vpx_mse8x8 vpx_mse8x8_c -void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); #define vpx_quantize_b vpx_quantize_b_neon -void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); #define vpx_quantize_b_32x32 vpx_quantize_b_32x32_neon unsigned int vpx_sad16x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x16_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x16 vpx_sad16x16_neon unsigned int vpx_sad16x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x16_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x16_avg vpx_sad16x16_avg_neon -void vpx_sad16x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x16x3 vpx_sad16x16x3_c - -void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x16x4d vpx_sad16x16x4d_neon -void vpx_sad16x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x16x8 vpx_sad16x16x8_c - unsigned int vpx_sad16x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x32_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x32 vpx_sad16x32_neon unsigned int vpx_sad16x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x32_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x32_avg vpx_sad16x32_avg_neon -void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x32x4d vpx_sad16x32x4d_neon unsigned int vpx_sad16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x8_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x8 vpx_sad16x8_neon unsigned int vpx_sad16x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x8_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x8_avg vpx_sad16x8_avg_neon -void vpx_sad16x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x8x3 vpx_sad16x8x3_c - -void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x8x4d vpx_sad16x8x4d_neon -void vpx_sad16x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x8x8 vpx_sad16x8x8_c - unsigned int vpx_sad32x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad32x16 vpx_sad32x16_neon unsigned int vpx_sad32x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x16_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad32x16_avg vpx_sad32x16_avg_neon -void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad32x16x4d vpx_sad32x16x4d_neon unsigned int vpx_sad32x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad32x32 vpx_sad32x32_neon unsigned int vpx_sad32x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x32_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad32x32_avg vpx_sad32x32_avg_neon -void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad32x32x4d vpx_sad32x32x4d_neon -void vpx_sad32x32x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad32x32x8 vpx_sad32x32x8_c - unsigned int vpx_sad32x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad32x64 vpx_sad32x64_neon unsigned int vpx_sad32x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x64_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad32x64_avg vpx_sad32x64_avg_neon -void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad32x64x4d vpx_sad32x64x4d_neon unsigned int vpx_sad4x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x4_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad4x4 vpx_sad4x4_neon unsigned int vpx_sad4x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x4_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad4x4_avg vpx_sad4x4_avg_neon -void vpx_sad4x4x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad4x4x3 vpx_sad4x4x3_c - -void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x4x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad4x4x4d vpx_sad4x4x4d_neon -void vpx_sad4x4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad4x4x8 vpx_sad4x4x8_c - unsigned int vpx_sad4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x8_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad4x8 vpx_sad4x8_neon unsigned int vpx_sad4x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x8_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad4x8_avg vpx_sad4x8_avg_neon -void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad4x8x4d vpx_sad4x8x4d_neon unsigned int vpx_sad64x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad64x32 vpx_sad64x32_neon unsigned int vpx_sad64x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x32_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad64x32_avg vpx_sad64x32_avg_neon -void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad64x32x4d vpx_sad64x32x4d_neon unsigned int vpx_sad64x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad64x64 vpx_sad64x64_neon unsigned int vpx_sad64x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x64_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad64x64_avg vpx_sad64x64_avg_neon -void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad64x64x4d vpx_sad64x64x4d_neon unsigned int vpx_sad8x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x16_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x16 vpx_sad8x16_neon unsigned int vpx_sad8x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x16_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x16_avg vpx_sad8x16_avg_neon -void vpx_sad8x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x16x3 vpx_sad8x16x3_c - -void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x16x4d vpx_sad8x16x4d_neon -void vpx_sad8x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x16x8 vpx_sad8x16x8_c - unsigned int vpx_sad8x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x4_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x4 vpx_sad8x4_neon unsigned int vpx_sad8x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x4_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x4_avg vpx_sad8x4_avg_neon -void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x4x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x4x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x4x4d vpx_sad8x4x4d_neon unsigned int vpx_sad8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x8_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x8 vpx_sad8x8_neon unsigned int vpx_sad8x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x8_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x8_avg vpx_sad8x8_avg_neon -void vpx_sad8x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x8x3 vpx_sad8x8x3_c - -void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x8x4d vpx_sad8x8x4d_neon -void vpx_sad8x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x8x8 vpx_sad8x8x8_c - int vpx_satd_c(const int16_t *coeff, int length); int vpx_satd_neon(const int16_t *coeff, int length); #define vpx_satd vpx_satd_neon void vpx_scaled_2d_c(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h); diff --git a/media/libvpx/config/linux/ia32/vp8_rtcd.h b/media/libvpx/config/linux/ia32/vp8_rtcd.h --- a/media/libvpx/config/linux/ia32/vp8_rtcd.h +++ b/media/libvpx/config/linux/ia32/vp8_rtcd.h @@ -121,15 +121,10 @@ void vp8_filter_by_weight8x8_c(unsigned char *src, int src_stride, unsigned char *dst, int dst_stride, int src_weight); void vp8_filter_by_weight8x8_sse2(unsigned char *src, int src_stride, unsigned char *dst, int dst_stride, int src_weight); RTCD_EXTERN void (*vp8_filter_by_weight8x8)(unsigned char *src, int src_stride, unsigned char *dst, int dst_stride, int src_weight); -int vp8_full_search_sad_c(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -int vp8_full_search_sadx3(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -int vp8_full_search_sadx8(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -RTCD_EXTERN int (*vp8_full_search_sad)(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); - void vp8_loop_filter_bh_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); void vp8_loop_filter_bh_sse2(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); RTCD_EXTERN void (*vp8_loop_filter_bh)(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); void vp8_loop_filter_bv_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); @@ -276,13 +271,10 @@ if (flags & HAS_SSSE3) vp8_fast_quantize_b = vp8_fast_quantize_b_ssse3; vp8_filter_by_weight16x16 = vp8_filter_by_weight16x16_c; if (flags & HAS_SSE2) vp8_filter_by_weight16x16 = vp8_filter_by_weight16x16_sse2; vp8_filter_by_weight8x8 = vp8_filter_by_weight8x8_c; if (flags & HAS_SSE2) vp8_filter_by_weight8x8 = vp8_filter_by_weight8x8_sse2; - vp8_full_search_sad = vp8_full_search_sad_c; - if (flags & HAS_SSE3) vp8_full_search_sad = vp8_full_search_sadx3; - if (flags & HAS_SSE4_1) vp8_full_search_sad = vp8_full_search_sadx8; vp8_loop_filter_bh = vp8_loop_filter_bh_c; if (flags & HAS_SSE2) vp8_loop_filter_bh = vp8_loop_filter_bh_sse2; vp8_loop_filter_bv = vp8_loop_filter_bv_c; if (flags & HAS_SSE2) vp8_loop_filter_bv = vp8_loop_filter_bv_sse2; vp8_loop_filter_mbh = vp8_loop_filter_mbh_c; diff --git a/media/libvpx/config/linux/ia32/vp9_rtcd.h b/media/libvpx/config/linux/ia32/vp9_rtcd.h --- a/media/libvpx/config/linux/ia32/vp9_rtcd.h +++ b/media/libvpx/config/linux/ia32/vp9_rtcd.h @@ -83,16 +83,16 @@ void vp9_iht8x8_64_add_c(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); void vp9_iht8x8_64_add_sse2(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); RTCD_EXTERN void (*vp9_iht8x8_64_add)(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); -void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vp9_quantize_fp)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vp9_quantize_fp)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); #define vp9_quantize_fp_32x32 vp9_quantize_fp_32x32_c void vp9_scale_and_extend_frame_c(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); void vp9_scale_and_extend_frame_ssse3(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); RTCD_EXTERN void (*vp9_scale_and_extend_frame)(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); diff --git a/media/libvpx/config/linux/ia32/vpx_config.h b/media/libvpx/config/linux/ia32/vpx_config.h --- a/media/libvpx/config/linux/ia32/vpx_config.h +++ b/media/libvpx/config/linux/ia32/vpx_config.h @@ -13,10 +13,11 @@ #define VPX_ARCH_ARM 0 #define VPX_ARCH_MIPS 0 #define VPX_ARCH_X86 1 #define VPX_ARCH_X86_64 0 #define VPX_ARCH_PPC 0 +#define VPX_ARCH_LOONGARCH 0 #define HAVE_NEON 0 #define HAVE_NEON_ASM 0 #define HAVE_MIPS32 0 #define HAVE_DSPR2 0 #define HAVE_MSA 0 @@ -30,10 +31,12 @@ #define HAVE_AVX 1 #define HAVE_AVX2 1 #define HAVE_AVX512 0 #define HAVE_VSX 0 #define HAVE_MMI 0 +#define HAVE_LSX 0 +#define HAVE_LASX 0 #define HAVE_VPX_PORTS 1 #define HAVE_PTHREAD_H 1 #define CONFIG_DEPENDENCY_TRACKING 1 #define CONFIG_EXTERNAL_BUILD 1 #define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/linux/ia32/vpx_config.asm b/media/libvpx/config/linux/ia32/vpx_config.asm --- a/media/libvpx/config/linux/ia32/vpx_config.asm +++ b/media/libvpx/config/linux/ia32/vpx_config.asm @@ -1,10 +1,11 @@ %define VPX_ARCH_ARM 0 %define VPX_ARCH_MIPS 0 %define VPX_ARCH_X86 1 %define VPX_ARCH_X86_64 0 %define VPX_ARCH_PPC 0 +%define VPX_ARCH_LOONGARCH 0 %define HAVE_NEON 0 %define HAVE_NEON_ASM 0 %define HAVE_MIPS32 0 %define HAVE_DSPR2 0 %define HAVE_MSA 0 @@ -18,10 +19,12 @@ %define HAVE_AVX 1 %define HAVE_AVX2 1 %define HAVE_AVX512 0 %define HAVE_VSX 0 %define HAVE_MMI 0 +%define HAVE_LSX 0 +%define HAVE_LASX 0 %define HAVE_VPX_PORTS 1 %define HAVE_PTHREAD_H 1 %define CONFIG_DEPENDENCY_TRACKING 1 %define CONFIG_EXTERNAL_BUILD 1 %define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/linux/ia32/vpx_dsp_rtcd.h b/media/libvpx/config/linux/ia32/vpx_dsp_rtcd.h --- a/media/libvpx/config/linux/ia32/vpx_dsp_rtcd.h +++ b/media/libvpx/config/linux/ia32/vpx_dsp_rtcd.h @@ -379,13 +379,13 @@ int16_t vpx_int_pro_col_c(const uint8_t *ref, const int width); int16_t vpx_int_pro_col_sse2(const uint8_t *ref, const int width); RTCD_EXTERN int16_t (*vpx_int_pro_col)(const uint8_t *ref, const int width); -void vpx_int_pro_row_c(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); -void vpx_int_pro_row_sse2(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); -RTCD_EXTERN void (*vpx_int_pro_row)(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_c(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_sse2(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); +RTCD_EXTERN void (*vpx_int_pro_row)(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); void vpx_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int stride); void vpx_iwht4x4_16_add_sse2(const tran_low_t *input, uint8_t *dest, int stride); RTCD_EXTERN void (*vpx_iwht4x4_16_add)(const tran_low_t *input, uint8_t *dest, int stride); @@ -478,74 +478,56 @@ void vpx_post_proc_down_and_across_mb_row_c(unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size); void vpx_post_proc_down_and_across_mb_row_sse2(unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size); RTCD_EXTERN void (*vpx_post_proc_down_and_across_mb_row)(unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size); -void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vpx_quantize_b)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vpx_quantize_b)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vpx_quantize_b_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vpx_quantize_b_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); unsigned int vpx_sad16x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad16x16)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad16x16_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad16x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x3_ssse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad16x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad16x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad16x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad16x32)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad16x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad16x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad16x8_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad16x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x3_ssse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad16x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad16x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad32x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x16)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -553,13 +535,13 @@ unsigned int vpx_sad32x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x16_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x16_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad32x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad32x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x32)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -567,18 +549,14 @@ unsigned int vpx_sad32x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x32_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad32x32x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x8_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x32x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x32x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad32x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad32x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x64)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -586,45 +564,37 @@ unsigned int vpx_sad32x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x64_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x64_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x64_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad32x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad4x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x4_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad4x4)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x4_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad4x4_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad4x4x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad4x4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad4x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad4x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad4x8_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad4x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad64x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad64x32)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -632,13 +602,13 @@ unsigned int vpx_sad64x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x32_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad64x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad64x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad64x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad64x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad64x64)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -646,66 +616,50 @@ unsigned int vpx_sad64x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x64_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x64_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad64x64_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x64x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad64x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x64x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad64x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad8x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad8x16)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad8x16_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad8x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad8x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad8x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad8x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x4_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad8x4)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x4_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad8x4_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad8x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad8x8_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad8x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad8x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad8x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); int vpx_satd_c(const int16_t *coeff, int length); int vpx_satd_sse2(const int16_t *coeff, int length); int vpx_satd_avx2(const int16_t *coeff, int length); RTCD_EXTERN int (*vpx_satd)(const int16_t *coeff, int length); @@ -1211,34 +1165,24 @@ if (flags & HAS_AVX) vpx_quantize_b_32x32 = vpx_quantize_b_32x32_avx; vpx_sad16x16 = vpx_sad16x16_c; if (flags & HAS_SSE2) vpx_sad16x16 = vpx_sad16x16_sse2; vpx_sad16x16_avg = vpx_sad16x16_avg_c; if (flags & HAS_SSE2) vpx_sad16x16_avg = vpx_sad16x16_avg_sse2; - vpx_sad16x16x3 = vpx_sad16x16x3_c; - if (flags & HAS_SSE3) vpx_sad16x16x3 = vpx_sad16x16x3_sse3; - if (flags & HAS_SSSE3) vpx_sad16x16x3 = vpx_sad16x16x3_ssse3; vpx_sad16x16x4d = vpx_sad16x16x4d_c; if (flags & HAS_SSE2) vpx_sad16x16x4d = vpx_sad16x16x4d_sse2; - vpx_sad16x16x8 = vpx_sad16x16x8_c; - if (flags & HAS_SSE4_1) vpx_sad16x16x8 = vpx_sad16x16x8_sse4_1; vpx_sad16x32 = vpx_sad16x32_c; if (flags & HAS_SSE2) vpx_sad16x32 = vpx_sad16x32_sse2; vpx_sad16x32_avg = vpx_sad16x32_avg_c; if (flags & HAS_SSE2) vpx_sad16x32_avg = vpx_sad16x32_avg_sse2; vpx_sad16x32x4d = vpx_sad16x32x4d_c; if (flags & HAS_SSE2) vpx_sad16x32x4d = vpx_sad16x32x4d_sse2; vpx_sad16x8 = vpx_sad16x8_c; if (flags & HAS_SSE2) vpx_sad16x8 = vpx_sad16x8_sse2; vpx_sad16x8_avg = vpx_sad16x8_avg_c; if (flags & HAS_SSE2) vpx_sad16x8_avg = vpx_sad16x8_avg_sse2; - vpx_sad16x8x3 = vpx_sad16x8x3_c; - if (flags & HAS_SSE3) vpx_sad16x8x3 = vpx_sad16x8x3_sse3; - if (flags & HAS_SSSE3) vpx_sad16x8x3 = vpx_sad16x8x3_ssse3; vpx_sad16x8x4d = vpx_sad16x8x4d_c; if (flags & HAS_SSE2) vpx_sad16x8x4d = vpx_sad16x8x4d_sse2; - vpx_sad16x8x8 = vpx_sad16x8x8_c; - if (flags & HAS_SSE4_1) vpx_sad16x8x8 = vpx_sad16x8x8_sse4_1; vpx_sad32x16 = vpx_sad32x16_c; if (flags & HAS_SSE2) vpx_sad32x16 = vpx_sad32x16_sse2; if (flags & HAS_AVX2) vpx_sad32x16 = vpx_sad32x16_avx2; vpx_sad32x16_avg = vpx_sad32x16_avg_c; if (flags & HAS_SSE2) vpx_sad32x16_avg = vpx_sad32x16_avg_sse2; @@ -1252,12 +1196,10 @@ if (flags & HAS_SSE2) vpx_sad32x32_avg = vpx_sad32x32_avg_sse2; if (flags & HAS_AVX2) vpx_sad32x32_avg = vpx_sad32x32_avg_avx2; vpx_sad32x32x4d = vpx_sad32x32x4d_c; if (flags & HAS_SSE2) vpx_sad32x32x4d = vpx_sad32x32x4d_sse2; if (flags & HAS_AVX2) vpx_sad32x32x4d = vpx_sad32x32x4d_avx2; - vpx_sad32x32x8 = vpx_sad32x32x8_c; - if (flags & HAS_AVX2) vpx_sad32x32x8 = vpx_sad32x32x8_avx2; vpx_sad32x64 = vpx_sad32x64_c; if (flags & HAS_SSE2) vpx_sad32x64 = vpx_sad32x64_sse2; if (flags & HAS_AVX2) vpx_sad32x64 = vpx_sad32x64_avx2; vpx_sad32x64_avg = vpx_sad32x64_avg_c; if (flags & HAS_SSE2) vpx_sad32x64_avg = vpx_sad32x64_avg_sse2; @@ -1266,16 +1208,12 @@ if (flags & HAS_SSE2) vpx_sad32x64x4d = vpx_sad32x64x4d_sse2; vpx_sad4x4 = vpx_sad4x4_c; if (flags & HAS_SSE2) vpx_sad4x4 = vpx_sad4x4_sse2; vpx_sad4x4_avg = vpx_sad4x4_avg_c; if (flags & HAS_SSE2) vpx_sad4x4_avg = vpx_sad4x4_avg_sse2; - vpx_sad4x4x3 = vpx_sad4x4x3_c; - if (flags & HAS_SSE3) vpx_sad4x4x3 = vpx_sad4x4x3_sse3; vpx_sad4x4x4d = vpx_sad4x4x4d_c; if (flags & HAS_SSE2) vpx_sad4x4x4d = vpx_sad4x4x4d_sse2; - vpx_sad4x4x8 = vpx_sad4x4x8_c; - if (flags & HAS_SSE4_1) vpx_sad4x4x8 = vpx_sad4x4x8_sse4_1; vpx_sad4x8 = vpx_sad4x8_c; if (flags & HAS_SSE2) vpx_sad4x8 = vpx_sad4x8_sse2; vpx_sad4x8_avg = vpx_sad4x8_avg_c; if (flags & HAS_SSE2) vpx_sad4x8_avg = vpx_sad4x8_avg_sse2; vpx_sad4x8x4d = vpx_sad4x8x4d_c; @@ -1299,32 +1237,24 @@ if (flags & HAS_AVX2) vpx_sad64x64x4d = vpx_sad64x64x4d_avx2; vpx_sad8x16 = vpx_sad8x16_c; if (flags & HAS_SSE2) vpx_sad8x16 = vpx_sad8x16_sse2; vpx_sad8x16_avg = vpx_sad8x16_avg_c; if (flags & HAS_SSE2) vpx_sad8x16_avg = vpx_sad8x16_avg_sse2; - vpx_sad8x16x3 = vpx_sad8x16x3_c; - if (flags & HAS_SSE3) vpx_sad8x16x3 = vpx_sad8x16x3_sse3; vpx_sad8x16x4d = vpx_sad8x16x4d_c; if (flags & HAS_SSE2) vpx_sad8x16x4d = vpx_sad8x16x4d_sse2; - vpx_sad8x16x8 = vpx_sad8x16x8_c; - if (flags & HAS_SSE4_1) vpx_sad8x16x8 = vpx_sad8x16x8_sse4_1; vpx_sad8x4 = vpx_sad8x4_c; if (flags & HAS_SSE2) vpx_sad8x4 = vpx_sad8x4_sse2; vpx_sad8x4_avg = vpx_sad8x4_avg_c; if (flags & HAS_SSE2) vpx_sad8x4_avg = vpx_sad8x4_avg_sse2; vpx_sad8x4x4d = vpx_sad8x4x4d_c; if (flags & HAS_SSE2) vpx_sad8x4x4d = vpx_sad8x4x4d_sse2; vpx_sad8x8 = vpx_sad8x8_c; if (flags & HAS_SSE2) vpx_sad8x8 = vpx_sad8x8_sse2; vpx_sad8x8_avg = vpx_sad8x8_avg_c; if (flags & HAS_SSE2) vpx_sad8x8_avg = vpx_sad8x8_avg_sse2; - vpx_sad8x8x3 = vpx_sad8x8x3_c; - if (flags & HAS_SSE3) vpx_sad8x8x3 = vpx_sad8x8x3_sse3; vpx_sad8x8x4d = vpx_sad8x8x4d_c; if (flags & HAS_SSE2) vpx_sad8x8x4d = vpx_sad8x8x4d_sse2; - vpx_sad8x8x8 = vpx_sad8x8x8_c; - if (flags & HAS_SSE4_1) vpx_sad8x8x8 = vpx_sad8x8x8_sse4_1; vpx_satd = vpx_satd_c; if (flags & HAS_SSE2) vpx_satd = vpx_satd_sse2; if (flags & HAS_AVX2) vpx_satd = vpx_satd_avx2; vpx_scaled_2d = vpx_scaled_2d_c; if (flags & HAS_SSSE3) vpx_scaled_2d = vpx_scaled_2d_ssse3; diff --git a/media/libvpx/config/linux/x64/vp8_rtcd.h b/media/libvpx/config/linux/x64/vp8_rtcd.h --- a/media/libvpx/config/linux/x64/vp8_rtcd.h +++ b/media/libvpx/config/linux/x64/vp8_rtcd.h @@ -121,15 +121,10 @@ void vp8_filter_by_weight8x8_c(unsigned char *src, int src_stride, unsigned char *dst, int dst_stride, int src_weight); void vp8_filter_by_weight8x8_sse2(unsigned char *src, int src_stride, unsigned char *dst, int dst_stride, int src_weight); #define vp8_filter_by_weight8x8 vp8_filter_by_weight8x8_sse2 -int vp8_full_search_sad_c(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -int vp8_full_search_sadx3(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -int vp8_full_search_sadx8(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -RTCD_EXTERN int (*vp8_full_search_sad)(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); - void vp8_loop_filter_bh_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); void vp8_loop_filter_bh_sse2(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); #define vp8_loop_filter_bh vp8_loop_filter_bh_sse2 void vp8_loop_filter_bv_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); @@ -240,13 +235,10 @@ if (flags & HAS_SSSE3) vp8_bilinear_predict8x8 = vp8_bilinear_predict8x8_ssse3; vp8_copy32xn = vp8_copy32xn_sse2; if (flags & HAS_SSE3) vp8_copy32xn = vp8_copy32xn_sse3; vp8_fast_quantize_b = vp8_fast_quantize_b_sse2; if (flags & HAS_SSSE3) vp8_fast_quantize_b = vp8_fast_quantize_b_ssse3; - vp8_full_search_sad = vp8_full_search_sad_c; - if (flags & HAS_SSE3) vp8_full_search_sad = vp8_full_search_sadx3; - if (flags & HAS_SSE4_1) vp8_full_search_sad = vp8_full_search_sadx8; vp8_regular_quantize_b = vp8_regular_quantize_b_sse2; if (flags & HAS_SSE4_1) vp8_regular_quantize_b = vp8_regular_quantize_b_sse4_1; vp8_sixtap_predict16x16 = vp8_sixtap_predict16x16_sse2; if (flags & HAS_SSSE3) vp8_sixtap_predict16x16 = vp8_sixtap_predict16x16_ssse3; vp8_sixtap_predict4x4 = vp8_sixtap_predict4x4_mmx; diff --git a/media/libvpx/config/linux/x64/vp9_rtcd.h b/media/libvpx/config/linux/x64/vp9_rtcd.h --- a/media/libvpx/config/linux/x64/vp9_rtcd.h +++ b/media/libvpx/config/linux/x64/vp9_rtcd.h @@ -83,19 +83,19 @@ void vp9_iht8x8_64_add_c(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); void vp9_iht8x8_64_add_sse2(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); #define vp9_iht8x8_64_add vp9_iht8x8_64_add_sse2 -void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vp9_quantize_fp)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vp9_quantize_fp)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vp9_quantize_fp_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vp9_quantize_fp_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); void vp9_scale_and_extend_frame_c(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); void vp9_scale_and_extend_frame_ssse3(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); RTCD_EXTERN void (*vp9_scale_and_extend_frame)(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); diff --git a/media/libvpx/config/linux/x64/vpx_config.h b/media/libvpx/config/linux/x64/vpx_config.h --- a/media/libvpx/config/linux/x64/vpx_config.h +++ b/media/libvpx/config/linux/x64/vpx_config.h @@ -13,10 +13,11 @@ #define VPX_ARCH_ARM 0 #define VPX_ARCH_MIPS 0 #define VPX_ARCH_X86 0 #define VPX_ARCH_X86_64 1 #define VPX_ARCH_PPC 0 +#define VPX_ARCH_LOONGARCH 0 #define HAVE_NEON 0 #define HAVE_NEON_ASM 0 #define HAVE_MIPS32 0 #define HAVE_DSPR2 0 #define HAVE_MSA 0 @@ -30,10 +31,12 @@ #define HAVE_AVX 1 #define HAVE_AVX2 1 #define HAVE_AVX512 0 #define HAVE_VSX 0 #define HAVE_MMI 0 +#define HAVE_LSX 0 +#define HAVE_LASX 0 #define HAVE_VPX_PORTS 1 #define HAVE_PTHREAD_H 1 #define CONFIG_DEPENDENCY_TRACKING 1 #define CONFIG_EXTERNAL_BUILD 1 #define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/linux/x64/vpx_config.asm b/media/libvpx/config/linux/x64/vpx_config.asm --- a/media/libvpx/config/linux/x64/vpx_config.asm +++ b/media/libvpx/config/linux/x64/vpx_config.asm @@ -1,10 +1,11 @@ %define VPX_ARCH_ARM 0 %define VPX_ARCH_MIPS 0 %define VPX_ARCH_X86 0 %define VPX_ARCH_X86_64 1 %define VPX_ARCH_PPC 0 +%define VPX_ARCH_LOONGARCH 0 %define HAVE_NEON 0 %define HAVE_NEON_ASM 0 %define HAVE_MIPS32 0 %define HAVE_DSPR2 0 %define HAVE_MSA 0 @@ -18,10 +19,12 @@ %define HAVE_AVX 1 %define HAVE_AVX2 1 %define HAVE_AVX512 0 %define HAVE_VSX 0 %define HAVE_MMI 0 +%define HAVE_LSX 0 +%define HAVE_LASX 0 %define HAVE_VPX_PORTS 1 %define HAVE_PTHREAD_H 1 %define CONFIG_DEPENDENCY_TRACKING 1 %define CONFIG_EXTERNAL_BUILD 1 %define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/linux/x64/vpx_dsp_rtcd.h b/media/libvpx/config/linux/x64/vpx_dsp_rtcd.h --- a/media/libvpx/config/linux/x64/vpx_dsp_rtcd.h +++ b/media/libvpx/config/linux/x64/vpx_dsp_rtcd.h @@ -381,12 +381,12 @@ int16_t vpx_int_pro_col_c(const uint8_t *ref, const int width); int16_t vpx_int_pro_col_sse2(const uint8_t *ref, const int width); #define vpx_int_pro_col vpx_int_pro_col_sse2 -void vpx_int_pro_row_c(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); -void vpx_int_pro_row_sse2(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_c(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_sse2(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); #define vpx_int_pro_row vpx_int_pro_row_sse2 void vpx_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int stride); void vpx_iwht4x4_16_add_sse2(const tran_low_t *input, uint8_t *dest, int stride); #define vpx_iwht4x4_16_add vpx_iwht4x4_16_add_sse2 @@ -480,87 +480,69 @@ void vpx_post_proc_down_and_across_mb_row_c(unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size); void vpx_post_proc_down_and_across_mb_row_sse2(unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size); #define vpx_post_proc_down_and_across_mb_row vpx_post_proc_down_and_across_mb_row_sse2 -void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vpx_quantize_b)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vpx_quantize_b)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vpx_quantize_b_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vpx_quantize_b_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); unsigned int vpx_sad16x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x16 vpx_sad16x16_sse2 unsigned int vpx_sad16x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x16_avg vpx_sad16x16_avg_sse2 -void vpx_sad16x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x3_ssse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x16x4d vpx_sad16x16x4d_sse2 -void vpx_sad16x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - unsigned int vpx_sad16x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x32 vpx_sad16x32_sse2 unsigned int vpx_sad16x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x32_avg vpx_sad16x32_avg_sse2 -void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x32x4d vpx_sad16x32x4d_sse2 unsigned int vpx_sad16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x8 vpx_sad16x8_sse2 unsigned int vpx_sad16x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x8_avg vpx_sad16x8_avg_sse2 -void vpx_sad16x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x3_ssse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x8x4d vpx_sad16x8x4d_sse2 -void vpx_sad16x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - unsigned int vpx_sad32x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x16)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x16_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x16_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad32x16x4d vpx_sad32x16x4d_sse2 unsigned int vpx_sad32x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -569,18 +551,14 @@ unsigned int vpx_sad32x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x32_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad32x32x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x8_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x32x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x32x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad32x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad32x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x64)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -588,44 +566,36 @@ unsigned int vpx_sad32x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x64_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x64_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x64_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad32x64x4d vpx_sad32x64x4d_sse2 unsigned int vpx_sad4x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x4_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad4x4 vpx_sad4x4_sse2 unsigned int vpx_sad4x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x4_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad4x4_avg vpx_sad4x4_avg_sse2 -void vpx_sad4x4x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad4x4x4d vpx_sad4x4x4d_sse2 -void vpx_sad4x4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - unsigned int vpx_sad4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad4x8 vpx_sad4x8_sse2 unsigned int vpx_sad4x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad4x8_avg vpx_sad4x8_avg_sse2 -void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad4x8x4d vpx_sad4x8x4d_sse2 unsigned int vpx_sad64x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -634,12 +604,12 @@ unsigned int vpx_sad64x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x32_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad64x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad64x32x4d vpx_sad64x32x4d_sse2 unsigned int vpx_sad64x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -648,67 +618,51 @@ unsigned int vpx_sad64x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x64_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x64_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad64x64_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x64x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad64x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x64x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad64x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad8x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x16 vpx_sad8x16_sse2 unsigned int vpx_sad8x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x16_avg vpx_sad8x16_avg_sse2 -void vpx_sad8x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x16x4d vpx_sad8x16x4d_sse2 -void vpx_sad8x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - unsigned int vpx_sad8x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x4_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x4 vpx_sad8x4_sse2 unsigned int vpx_sad8x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x4_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x4_avg vpx_sad8x4_avg_sse2 -void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x4x4d vpx_sad8x4x4d_sse2 unsigned int vpx_sad8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x8 vpx_sad8x8_sse2 unsigned int vpx_sad8x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x8_avg vpx_sad8x8_avg_sse2 -void vpx_sad8x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x8x4d vpx_sad8x8x4d_sse2 -void vpx_sad8x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - int vpx_satd_c(const int16_t *coeff, int length); int vpx_satd_sse2(const int16_t *coeff, int length); int vpx_satd_avx2(const int16_t *coeff, int length); RTCD_EXTERN int (*vpx_satd)(const int16_t *coeff, int length); @@ -1058,58 +1012,34 @@ if (flags & HAS_SSSE3) vpx_quantize_b = vpx_quantize_b_ssse3; if (flags & HAS_AVX) vpx_quantize_b = vpx_quantize_b_avx; vpx_quantize_b_32x32 = vpx_quantize_b_32x32_c; if (flags & HAS_SSSE3) vpx_quantize_b_32x32 = vpx_quantize_b_32x32_ssse3; if (flags & HAS_AVX) vpx_quantize_b_32x32 = vpx_quantize_b_32x32_avx; - vpx_sad16x16x3 = vpx_sad16x16x3_c; - if (flags & HAS_SSE3) vpx_sad16x16x3 = vpx_sad16x16x3_sse3; - if (flags & HAS_SSSE3) vpx_sad16x16x3 = vpx_sad16x16x3_ssse3; - vpx_sad16x16x8 = vpx_sad16x16x8_c; - if (flags & HAS_SSE4_1) vpx_sad16x16x8 = vpx_sad16x16x8_sse4_1; - vpx_sad16x8x3 = vpx_sad16x8x3_c; - if (flags & HAS_SSE3) vpx_sad16x8x3 = vpx_sad16x8x3_sse3; - if (flags & HAS_SSSE3) vpx_sad16x8x3 = vpx_sad16x8x3_ssse3; - vpx_sad16x8x8 = vpx_sad16x8x8_c; - if (flags & HAS_SSE4_1) vpx_sad16x8x8 = vpx_sad16x8x8_sse4_1; vpx_sad32x16 = vpx_sad32x16_sse2; if (flags & HAS_AVX2) vpx_sad32x16 = vpx_sad32x16_avx2; vpx_sad32x16_avg = vpx_sad32x16_avg_sse2; if (flags & HAS_AVX2) vpx_sad32x16_avg = vpx_sad32x16_avg_avx2; vpx_sad32x32 = vpx_sad32x32_sse2; if (flags & HAS_AVX2) vpx_sad32x32 = vpx_sad32x32_avx2; vpx_sad32x32_avg = vpx_sad32x32_avg_sse2; if (flags & HAS_AVX2) vpx_sad32x32_avg = vpx_sad32x32_avg_avx2; vpx_sad32x32x4d = vpx_sad32x32x4d_sse2; if (flags & HAS_AVX2) vpx_sad32x32x4d = vpx_sad32x32x4d_avx2; - vpx_sad32x32x8 = vpx_sad32x32x8_c; - if (flags & HAS_AVX2) vpx_sad32x32x8 = vpx_sad32x32x8_avx2; vpx_sad32x64 = vpx_sad32x64_sse2; if (flags & HAS_AVX2) vpx_sad32x64 = vpx_sad32x64_avx2; vpx_sad32x64_avg = vpx_sad32x64_avg_sse2; if (flags & HAS_AVX2) vpx_sad32x64_avg = vpx_sad32x64_avg_avx2; - vpx_sad4x4x3 = vpx_sad4x4x3_c; - if (flags & HAS_SSE3) vpx_sad4x4x3 = vpx_sad4x4x3_sse3; - vpx_sad4x4x8 = vpx_sad4x4x8_c; - if (flags & HAS_SSE4_1) vpx_sad4x4x8 = vpx_sad4x4x8_sse4_1; vpx_sad64x32 = vpx_sad64x32_sse2; if (flags & HAS_AVX2) vpx_sad64x32 = vpx_sad64x32_avx2; vpx_sad64x32_avg = vpx_sad64x32_avg_sse2; if (flags & HAS_AVX2) vpx_sad64x32_avg = vpx_sad64x32_avg_avx2; vpx_sad64x64 = vpx_sad64x64_sse2; if (flags & HAS_AVX2) vpx_sad64x64 = vpx_sad64x64_avx2; vpx_sad64x64_avg = vpx_sad64x64_avg_sse2; if (flags & HAS_AVX2) vpx_sad64x64_avg = vpx_sad64x64_avg_avx2; vpx_sad64x64x4d = vpx_sad64x64x4d_sse2; if (flags & HAS_AVX2) vpx_sad64x64x4d = vpx_sad64x64x4d_avx2; - vpx_sad8x16x3 = vpx_sad8x16x3_c; - if (flags & HAS_SSE3) vpx_sad8x16x3 = vpx_sad8x16x3_sse3; - vpx_sad8x16x8 = vpx_sad8x16x8_c; - if (flags & HAS_SSE4_1) vpx_sad8x16x8 = vpx_sad8x16x8_sse4_1; - vpx_sad8x8x3 = vpx_sad8x8x3_c; - if (flags & HAS_SSE3) vpx_sad8x8x3 = vpx_sad8x8x3_sse3; - vpx_sad8x8x8 = vpx_sad8x8x8_c; - if (flags & HAS_SSE4_1) vpx_sad8x8x8 = vpx_sad8x8x8_sse4_1; vpx_satd = vpx_satd_sse2; if (flags & HAS_AVX2) vpx_satd = vpx_satd_avx2; vpx_scaled_2d = vpx_scaled_2d_c; if (flags & HAS_SSSE3) vpx_scaled_2d = vpx_scaled_2d_ssse3; vpx_sub_pixel_avg_variance16x16 = vpx_sub_pixel_avg_variance16x16_sse2; diff --git a/media/libvpx/config/mac/ia32/vp8_rtcd.h b/media/libvpx/config/mac/ia32/vp8_rtcd.h --- a/media/libvpx/config/mac/ia32/vp8_rtcd.h +++ b/media/libvpx/config/mac/ia32/vp8_rtcd.h @@ -121,15 +121,10 @@ void vp8_filter_by_weight8x8_c(unsigned char *src, int src_stride, unsigned char *dst, int dst_stride, int src_weight); void vp8_filter_by_weight8x8_sse2(unsigned char *src, int src_stride, unsigned char *dst, int dst_stride, int src_weight); RTCD_EXTERN void (*vp8_filter_by_weight8x8)(unsigned char *src, int src_stride, unsigned char *dst, int dst_stride, int src_weight); -int vp8_full_search_sad_c(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -int vp8_full_search_sadx3(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -int vp8_full_search_sadx8(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -RTCD_EXTERN int (*vp8_full_search_sad)(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); - void vp8_loop_filter_bh_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); void vp8_loop_filter_bh_sse2(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); RTCD_EXTERN void (*vp8_loop_filter_bh)(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); void vp8_loop_filter_bv_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); @@ -276,13 +271,10 @@ if (flags & HAS_SSSE3) vp8_fast_quantize_b = vp8_fast_quantize_b_ssse3; vp8_filter_by_weight16x16 = vp8_filter_by_weight16x16_c; if (flags & HAS_SSE2) vp8_filter_by_weight16x16 = vp8_filter_by_weight16x16_sse2; vp8_filter_by_weight8x8 = vp8_filter_by_weight8x8_c; if (flags & HAS_SSE2) vp8_filter_by_weight8x8 = vp8_filter_by_weight8x8_sse2; - vp8_full_search_sad = vp8_full_search_sad_c; - if (flags & HAS_SSE3) vp8_full_search_sad = vp8_full_search_sadx3; - if (flags & HAS_SSE4_1) vp8_full_search_sad = vp8_full_search_sadx8; vp8_loop_filter_bh = vp8_loop_filter_bh_c; if (flags & HAS_SSE2) vp8_loop_filter_bh = vp8_loop_filter_bh_sse2; vp8_loop_filter_bv = vp8_loop_filter_bv_c; if (flags & HAS_SSE2) vp8_loop_filter_bv = vp8_loop_filter_bv_sse2; vp8_loop_filter_mbh = vp8_loop_filter_mbh_c; diff --git a/media/libvpx/config/mac/ia32/vp9_rtcd.h b/media/libvpx/config/mac/ia32/vp9_rtcd.h --- a/media/libvpx/config/mac/ia32/vp9_rtcd.h +++ b/media/libvpx/config/mac/ia32/vp9_rtcd.h @@ -83,16 +83,16 @@ void vp9_iht8x8_64_add_c(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); void vp9_iht8x8_64_add_sse2(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); RTCD_EXTERN void (*vp9_iht8x8_64_add)(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); -void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vp9_quantize_fp)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vp9_quantize_fp)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); #define vp9_quantize_fp_32x32 vp9_quantize_fp_32x32_c void vp9_scale_and_extend_frame_c(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); void vp9_scale_and_extend_frame_ssse3(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); RTCD_EXTERN void (*vp9_scale_and_extend_frame)(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); diff --git a/media/libvpx/config/mac/ia32/vpx_config.h b/media/libvpx/config/mac/ia32/vpx_config.h --- a/media/libvpx/config/mac/ia32/vpx_config.h +++ b/media/libvpx/config/mac/ia32/vpx_config.h @@ -13,10 +13,11 @@ #define VPX_ARCH_ARM 0 #define VPX_ARCH_MIPS 0 #define VPX_ARCH_X86 1 #define VPX_ARCH_X86_64 0 #define VPX_ARCH_PPC 0 +#define VPX_ARCH_LOONGARCH 0 #define HAVE_NEON 0 #define HAVE_NEON_ASM 0 #define HAVE_MIPS32 0 #define HAVE_DSPR2 0 #define HAVE_MSA 0 @@ -30,10 +31,12 @@ #define HAVE_AVX 1 #define HAVE_AVX2 1 #define HAVE_AVX512 0 #define HAVE_VSX 0 #define HAVE_MMI 0 +#define HAVE_LSX 0 +#define HAVE_LASX 0 #define HAVE_VPX_PORTS 1 #define HAVE_PTHREAD_H 1 #define CONFIG_DEPENDENCY_TRACKING 1 #define CONFIG_EXTERNAL_BUILD 1 #define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/mac/ia32/vpx_config.asm b/media/libvpx/config/mac/ia32/vpx_config.asm --- a/media/libvpx/config/mac/ia32/vpx_config.asm +++ b/media/libvpx/config/mac/ia32/vpx_config.asm @@ -1,10 +1,11 @@ %define VPX_ARCH_ARM 0 %define VPX_ARCH_MIPS 0 %define VPX_ARCH_X86 1 %define VPX_ARCH_X86_64 0 %define VPX_ARCH_PPC 0 +%define VPX_ARCH_LOONGARCH 0 %define HAVE_NEON 0 %define HAVE_NEON_ASM 0 %define HAVE_MIPS32 0 %define HAVE_DSPR2 0 %define HAVE_MSA 0 @@ -18,10 +19,12 @@ %define HAVE_AVX 1 %define HAVE_AVX2 1 %define HAVE_AVX512 0 %define HAVE_VSX 0 %define HAVE_MMI 0 +%define HAVE_LSX 0 +%define HAVE_LASX 0 %define HAVE_VPX_PORTS 1 %define HAVE_PTHREAD_H 1 %define CONFIG_DEPENDENCY_TRACKING 1 %define CONFIG_EXTERNAL_BUILD 1 %define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/mac/ia32/vpx_dsp_rtcd.h b/media/libvpx/config/mac/ia32/vpx_dsp_rtcd.h --- a/media/libvpx/config/mac/ia32/vpx_dsp_rtcd.h +++ b/media/libvpx/config/mac/ia32/vpx_dsp_rtcd.h @@ -379,13 +379,13 @@ int16_t vpx_int_pro_col_c(const uint8_t *ref, const int width); int16_t vpx_int_pro_col_sse2(const uint8_t *ref, const int width); RTCD_EXTERN int16_t (*vpx_int_pro_col)(const uint8_t *ref, const int width); -void vpx_int_pro_row_c(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); -void vpx_int_pro_row_sse2(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); -RTCD_EXTERN void (*vpx_int_pro_row)(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_c(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_sse2(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); +RTCD_EXTERN void (*vpx_int_pro_row)(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); void vpx_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int stride); void vpx_iwht4x4_16_add_sse2(const tran_low_t *input, uint8_t *dest, int stride); RTCD_EXTERN void (*vpx_iwht4x4_16_add)(const tran_low_t *input, uint8_t *dest, int stride); @@ -478,74 +478,56 @@ void vpx_post_proc_down_and_across_mb_row_c(unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size); void vpx_post_proc_down_and_across_mb_row_sse2(unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size); RTCD_EXTERN void (*vpx_post_proc_down_and_across_mb_row)(unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size); -void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vpx_quantize_b)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vpx_quantize_b)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vpx_quantize_b_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vpx_quantize_b_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); unsigned int vpx_sad16x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad16x16)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad16x16_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad16x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x3_ssse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad16x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad16x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad16x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad16x32)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad16x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad16x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad16x8_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad16x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x3_ssse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad16x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad16x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad32x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x16)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -553,13 +535,13 @@ unsigned int vpx_sad32x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x16_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x16_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad32x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad32x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x32)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -567,18 +549,14 @@ unsigned int vpx_sad32x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x32_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad32x32x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x8_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x32x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x32x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad32x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad32x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x64)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -586,45 +564,37 @@ unsigned int vpx_sad32x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x64_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x64_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x64_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad32x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad4x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x4_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad4x4)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x4_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad4x4_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad4x4x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad4x4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad4x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad4x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad4x8_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad4x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad64x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad64x32)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -632,13 +602,13 @@ unsigned int vpx_sad64x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x32_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad64x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad64x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad64x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad64x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad64x64)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -646,66 +616,50 @@ unsigned int vpx_sad64x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x64_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x64_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad64x64_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x64x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad64x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x64x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad64x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad8x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad8x16)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad8x16_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad8x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad8x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad8x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad8x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x4_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad8x4)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x4_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad8x4_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad8x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad8x8_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad8x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad8x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad8x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); int vpx_satd_c(const int16_t *coeff, int length); int vpx_satd_sse2(const int16_t *coeff, int length); int vpx_satd_avx2(const int16_t *coeff, int length); RTCD_EXTERN int (*vpx_satd)(const int16_t *coeff, int length); @@ -1211,34 +1165,24 @@ if (flags & HAS_AVX) vpx_quantize_b_32x32 = vpx_quantize_b_32x32_avx; vpx_sad16x16 = vpx_sad16x16_c; if (flags & HAS_SSE2) vpx_sad16x16 = vpx_sad16x16_sse2; vpx_sad16x16_avg = vpx_sad16x16_avg_c; if (flags & HAS_SSE2) vpx_sad16x16_avg = vpx_sad16x16_avg_sse2; - vpx_sad16x16x3 = vpx_sad16x16x3_c; - if (flags & HAS_SSE3) vpx_sad16x16x3 = vpx_sad16x16x3_sse3; - if (flags & HAS_SSSE3) vpx_sad16x16x3 = vpx_sad16x16x3_ssse3; vpx_sad16x16x4d = vpx_sad16x16x4d_c; if (flags & HAS_SSE2) vpx_sad16x16x4d = vpx_sad16x16x4d_sse2; - vpx_sad16x16x8 = vpx_sad16x16x8_c; - if (flags & HAS_SSE4_1) vpx_sad16x16x8 = vpx_sad16x16x8_sse4_1; vpx_sad16x32 = vpx_sad16x32_c; if (flags & HAS_SSE2) vpx_sad16x32 = vpx_sad16x32_sse2; vpx_sad16x32_avg = vpx_sad16x32_avg_c; if (flags & HAS_SSE2) vpx_sad16x32_avg = vpx_sad16x32_avg_sse2; vpx_sad16x32x4d = vpx_sad16x32x4d_c; if (flags & HAS_SSE2) vpx_sad16x32x4d = vpx_sad16x32x4d_sse2; vpx_sad16x8 = vpx_sad16x8_c; if (flags & HAS_SSE2) vpx_sad16x8 = vpx_sad16x8_sse2; vpx_sad16x8_avg = vpx_sad16x8_avg_c; if (flags & HAS_SSE2) vpx_sad16x8_avg = vpx_sad16x8_avg_sse2; - vpx_sad16x8x3 = vpx_sad16x8x3_c; - if (flags & HAS_SSE3) vpx_sad16x8x3 = vpx_sad16x8x3_sse3; - if (flags & HAS_SSSE3) vpx_sad16x8x3 = vpx_sad16x8x3_ssse3; vpx_sad16x8x4d = vpx_sad16x8x4d_c; if (flags & HAS_SSE2) vpx_sad16x8x4d = vpx_sad16x8x4d_sse2; - vpx_sad16x8x8 = vpx_sad16x8x8_c; - if (flags & HAS_SSE4_1) vpx_sad16x8x8 = vpx_sad16x8x8_sse4_1; vpx_sad32x16 = vpx_sad32x16_c; if (flags & HAS_SSE2) vpx_sad32x16 = vpx_sad32x16_sse2; if (flags & HAS_AVX2) vpx_sad32x16 = vpx_sad32x16_avx2; vpx_sad32x16_avg = vpx_sad32x16_avg_c; if (flags & HAS_SSE2) vpx_sad32x16_avg = vpx_sad32x16_avg_sse2; @@ -1252,12 +1196,10 @@ if (flags & HAS_SSE2) vpx_sad32x32_avg = vpx_sad32x32_avg_sse2; if (flags & HAS_AVX2) vpx_sad32x32_avg = vpx_sad32x32_avg_avx2; vpx_sad32x32x4d = vpx_sad32x32x4d_c; if (flags & HAS_SSE2) vpx_sad32x32x4d = vpx_sad32x32x4d_sse2; if (flags & HAS_AVX2) vpx_sad32x32x4d = vpx_sad32x32x4d_avx2; - vpx_sad32x32x8 = vpx_sad32x32x8_c; - if (flags & HAS_AVX2) vpx_sad32x32x8 = vpx_sad32x32x8_avx2; vpx_sad32x64 = vpx_sad32x64_c; if (flags & HAS_SSE2) vpx_sad32x64 = vpx_sad32x64_sse2; if (flags & HAS_AVX2) vpx_sad32x64 = vpx_sad32x64_avx2; vpx_sad32x64_avg = vpx_sad32x64_avg_c; if (flags & HAS_SSE2) vpx_sad32x64_avg = vpx_sad32x64_avg_sse2; @@ -1266,16 +1208,12 @@ if (flags & HAS_SSE2) vpx_sad32x64x4d = vpx_sad32x64x4d_sse2; vpx_sad4x4 = vpx_sad4x4_c; if (flags & HAS_SSE2) vpx_sad4x4 = vpx_sad4x4_sse2; vpx_sad4x4_avg = vpx_sad4x4_avg_c; if (flags & HAS_SSE2) vpx_sad4x4_avg = vpx_sad4x4_avg_sse2; - vpx_sad4x4x3 = vpx_sad4x4x3_c; - if (flags & HAS_SSE3) vpx_sad4x4x3 = vpx_sad4x4x3_sse3; vpx_sad4x4x4d = vpx_sad4x4x4d_c; if (flags & HAS_SSE2) vpx_sad4x4x4d = vpx_sad4x4x4d_sse2; - vpx_sad4x4x8 = vpx_sad4x4x8_c; - if (flags & HAS_SSE4_1) vpx_sad4x4x8 = vpx_sad4x4x8_sse4_1; vpx_sad4x8 = vpx_sad4x8_c; if (flags & HAS_SSE2) vpx_sad4x8 = vpx_sad4x8_sse2; vpx_sad4x8_avg = vpx_sad4x8_avg_c; if (flags & HAS_SSE2) vpx_sad4x8_avg = vpx_sad4x8_avg_sse2; vpx_sad4x8x4d = vpx_sad4x8x4d_c; @@ -1299,32 +1237,24 @@ if (flags & HAS_AVX2) vpx_sad64x64x4d = vpx_sad64x64x4d_avx2; vpx_sad8x16 = vpx_sad8x16_c; if (flags & HAS_SSE2) vpx_sad8x16 = vpx_sad8x16_sse2; vpx_sad8x16_avg = vpx_sad8x16_avg_c; if (flags & HAS_SSE2) vpx_sad8x16_avg = vpx_sad8x16_avg_sse2; - vpx_sad8x16x3 = vpx_sad8x16x3_c; - if (flags & HAS_SSE3) vpx_sad8x16x3 = vpx_sad8x16x3_sse3; vpx_sad8x16x4d = vpx_sad8x16x4d_c; if (flags & HAS_SSE2) vpx_sad8x16x4d = vpx_sad8x16x4d_sse2; - vpx_sad8x16x8 = vpx_sad8x16x8_c; - if (flags & HAS_SSE4_1) vpx_sad8x16x8 = vpx_sad8x16x8_sse4_1; vpx_sad8x4 = vpx_sad8x4_c; if (flags & HAS_SSE2) vpx_sad8x4 = vpx_sad8x4_sse2; vpx_sad8x4_avg = vpx_sad8x4_avg_c; if (flags & HAS_SSE2) vpx_sad8x4_avg = vpx_sad8x4_avg_sse2; vpx_sad8x4x4d = vpx_sad8x4x4d_c; if (flags & HAS_SSE2) vpx_sad8x4x4d = vpx_sad8x4x4d_sse2; vpx_sad8x8 = vpx_sad8x8_c; if (flags & HAS_SSE2) vpx_sad8x8 = vpx_sad8x8_sse2; vpx_sad8x8_avg = vpx_sad8x8_avg_c; if (flags & HAS_SSE2) vpx_sad8x8_avg = vpx_sad8x8_avg_sse2; - vpx_sad8x8x3 = vpx_sad8x8x3_c; - if (flags & HAS_SSE3) vpx_sad8x8x3 = vpx_sad8x8x3_sse3; vpx_sad8x8x4d = vpx_sad8x8x4d_c; if (flags & HAS_SSE2) vpx_sad8x8x4d = vpx_sad8x8x4d_sse2; - vpx_sad8x8x8 = vpx_sad8x8x8_c; - if (flags & HAS_SSE4_1) vpx_sad8x8x8 = vpx_sad8x8x8_sse4_1; vpx_satd = vpx_satd_c; if (flags & HAS_SSE2) vpx_satd = vpx_satd_sse2; if (flags & HAS_AVX2) vpx_satd = vpx_satd_avx2; vpx_scaled_2d = vpx_scaled_2d_c; if (flags & HAS_SSSE3) vpx_scaled_2d = vpx_scaled_2d_ssse3; diff --git a/media/libvpx/config/mac/x64/vp8_rtcd.h b/media/libvpx/config/mac/x64/vp8_rtcd.h --- a/media/libvpx/config/mac/x64/vp8_rtcd.h +++ b/media/libvpx/config/mac/x64/vp8_rtcd.h @@ -121,15 +121,10 @@ void vp8_filter_by_weight8x8_c(unsigned char *src, int src_stride, unsigned char *dst, int dst_stride, int src_weight); void vp8_filter_by_weight8x8_sse2(unsigned char *src, int src_stride, unsigned char *dst, int dst_stride, int src_weight); #define vp8_filter_by_weight8x8 vp8_filter_by_weight8x8_sse2 -int vp8_full_search_sad_c(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -int vp8_full_search_sadx3(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -int vp8_full_search_sadx8(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -RTCD_EXTERN int (*vp8_full_search_sad)(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); - void vp8_loop_filter_bh_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); void vp8_loop_filter_bh_sse2(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); #define vp8_loop_filter_bh vp8_loop_filter_bh_sse2 void vp8_loop_filter_bv_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); @@ -240,13 +235,10 @@ if (flags & HAS_SSSE3) vp8_bilinear_predict8x8 = vp8_bilinear_predict8x8_ssse3; vp8_copy32xn = vp8_copy32xn_sse2; if (flags & HAS_SSE3) vp8_copy32xn = vp8_copy32xn_sse3; vp8_fast_quantize_b = vp8_fast_quantize_b_sse2; if (flags & HAS_SSSE3) vp8_fast_quantize_b = vp8_fast_quantize_b_ssse3; - vp8_full_search_sad = vp8_full_search_sad_c; - if (flags & HAS_SSE3) vp8_full_search_sad = vp8_full_search_sadx3; - if (flags & HAS_SSE4_1) vp8_full_search_sad = vp8_full_search_sadx8; vp8_regular_quantize_b = vp8_regular_quantize_b_sse2; if (flags & HAS_SSE4_1) vp8_regular_quantize_b = vp8_regular_quantize_b_sse4_1; vp8_sixtap_predict16x16 = vp8_sixtap_predict16x16_sse2; if (flags & HAS_SSSE3) vp8_sixtap_predict16x16 = vp8_sixtap_predict16x16_ssse3; vp8_sixtap_predict4x4 = vp8_sixtap_predict4x4_mmx; diff --git a/media/libvpx/config/mac/x64/vp9_rtcd.h b/media/libvpx/config/mac/x64/vp9_rtcd.h --- a/media/libvpx/config/mac/x64/vp9_rtcd.h +++ b/media/libvpx/config/mac/x64/vp9_rtcd.h @@ -83,19 +83,19 @@ void vp9_iht8x8_64_add_c(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); void vp9_iht8x8_64_add_sse2(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); #define vp9_iht8x8_64_add vp9_iht8x8_64_add_sse2 -void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vp9_quantize_fp)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vp9_quantize_fp)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vp9_quantize_fp_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vp9_quantize_fp_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); void vp9_scale_and_extend_frame_c(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); void vp9_scale_and_extend_frame_ssse3(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); RTCD_EXTERN void (*vp9_scale_and_extend_frame)(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); diff --git a/media/libvpx/config/mac/x64/vpx_config.h b/media/libvpx/config/mac/x64/vpx_config.h --- a/media/libvpx/config/mac/x64/vpx_config.h +++ b/media/libvpx/config/mac/x64/vpx_config.h @@ -13,10 +13,11 @@ #define VPX_ARCH_ARM 0 #define VPX_ARCH_MIPS 0 #define VPX_ARCH_X86 0 #define VPX_ARCH_X86_64 1 #define VPX_ARCH_PPC 0 +#define VPX_ARCH_LOONGARCH 0 #define HAVE_NEON 0 #define HAVE_NEON_ASM 0 #define HAVE_MIPS32 0 #define HAVE_DSPR2 0 #define HAVE_MSA 0 @@ -30,10 +31,12 @@ #define HAVE_AVX 1 #define HAVE_AVX2 1 #define HAVE_AVX512 0 #define HAVE_VSX 0 #define HAVE_MMI 0 +#define HAVE_LSX 0 +#define HAVE_LASX 0 #define HAVE_VPX_PORTS 1 #define HAVE_PTHREAD_H 1 #define CONFIG_DEPENDENCY_TRACKING 1 #define CONFIG_EXTERNAL_BUILD 1 #define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/mac/x64/vpx_config.asm b/media/libvpx/config/mac/x64/vpx_config.asm --- a/media/libvpx/config/mac/x64/vpx_config.asm +++ b/media/libvpx/config/mac/x64/vpx_config.asm @@ -1,10 +1,11 @@ %define VPX_ARCH_ARM 0 %define VPX_ARCH_MIPS 0 %define VPX_ARCH_X86 0 %define VPX_ARCH_X86_64 1 %define VPX_ARCH_PPC 0 +%define VPX_ARCH_LOONGARCH 0 %define HAVE_NEON 0 %define HAVE_NEON_ASM 0 %define HAVE_MIPS32 0 %define HAVE_DSPR2 0 %define HAVE_MSA 0 @@ -18,10 +19,12 @@ %define HAVE_AVX 1 %define HAVE_AVX2 1 %define HAVE_AVX512 0 %define HAVE_VSX 0 %define HAVE_MMI 0 +%define HAVE_LSX 0 +%define HAVE_LASX 0 %define HAVE_VPX_PORTS 1 %define HAVE_PTHREAD_H 1 %define CONFIG_DEPENDENCY_TRACKING 1 %define CONFIG_EXTERNAL_BUILD 1 %define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/mac/x64/vpx_dsp_rtcd.h b/media/libvpx/config/mac/x64/vpx_dsp_rtcd.h --- a/media/libvpx/config/mac/x64/vpx_dsp_rtcd.h +++ b/media/libvpx/config/mac/x64/vpx_dsp_rtcd.h @@ -381,12 +381,12 @@ int16_t vpx_int_pro_col_c(const uint8_t *ref, const int width); int16_t vpx_int_pro_col_sse2(const uint8_t *ref, const int width); #define vpx_int_pro_col vpx_int_pro_col_sse2 -void vpx_int_pro_row_c(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); -void vpx_int_pro_row_sse2(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_c(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_sse2(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); #define vpx_int_pro_row vpx_int_pro_row_sse2 void vpx_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int stride); void vpx_iwht4x4_16_add_sse2(const tran_low_t *input, uint8_t *dest, int stride); #define vpx_iwht4x4_16_add vpx_iwht4x4_16_add_sse2 @@ -480,87 +480,69 @@ void vpx_post_proc_down_and_across_mb_row_c(unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size); void vpx_post_proc_down_and_across_mb_row_sse2(unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size); #define vpx_post_proc_down_and_across_mb_row vpx_post_proc_down_and_across_mb_row_sse2 -void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vpx_quantize_b)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vpx_quantize_b)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vpx_quantize_b_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vpx_quantize_b_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); unsigned int vpx_sad16x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x16 vpx_sad16x16_sse2 unsigned int vpx_sad16x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x16_avg vpx_sad16x16_avg_sse2 -void vpx_sad16x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x3_ssse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x16x4d vpx_sad16x16x4d_sse2 -void vpx_sad16x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - unsigned int vpx_sad16x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x32 vpx_sad16x32_sse2 unsigned int vpx_sad16x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x32_avg vpx_sad16x32_avg_sse2 -void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x32x4d vpx_sad16x32x4d_sse2 unsigned int vpx_sad16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x8 vpx_sad16x8_sse2 unsigned int vpx_sad16x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x8_avg vpx_sad16x8_avg_sse2 -void vpx_sad16x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x3_ssse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x8x4d vpx_sad16x8x4d_sse2 -void vpx_sad16x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - unsigned int vpx_sad32x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x16)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x16_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x16_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad32x16x4d vpx_sad32x16x4d_sse2 unsigned int vpx_sad32x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -569,18 +551,14 @@ unsigned int vpx_sad32x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x32_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad32x32x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x8_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x32x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x32x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad32x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad32x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x64)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -588,44 +566,36 @@ unsigned int vpx_sad32x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x64_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x64_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x64_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad32x64x4d vpx_sad32x64x4d_sse2 unsigned int vpx_sad4x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x4_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad4x4 vpx_sad4x4_sse2 unsigned int vpx_sad4x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x4_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad4x4_avg vpx_sad4x4_avg_sse2 -void vpx_sad4x4x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad4x4x4d vpx_sad4x4x4d_sse2 -void vpx_sad4x4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - unsigned int vpx_sad4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad4x8 vpx_sad4x8_sse2 unsigned int vpx_sad4x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad4x8_avg vpx_sad4x8_avg_sse2 -void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad4x8x4d vpx_sad4x8x4d_sse2 unsigned int vpx_sad64x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -634,12 +604,12 @@ unsigned int vpx_sad64x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x32_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad64x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad64x32x4d vpx_sad64x32x4d_sse2 unsigned int vpx_sad64x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -648,67 +618,51 @@ unsigned int vpx_sad64x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x64_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x64_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad64x64_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x64x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad64x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x64x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad64x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad8x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x16 vpx_sad8x16_sse2 unsigned int vpx_sad8x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x16_avg vpx_sad8x16_avg_sse2 -void vpx_sad8x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x16x4d vpx_sad8x16x4d_sse2 -void vpx_sad8x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - unsigned int vpx_sad8x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x4_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x4 vpx_sad8x4_sse2 unsigned int vpx_sad8x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x4_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x4_avg vpx_sad8x4_avg_sse2 -void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x4x4d vpx_sad8x4x4d_sse2 unsigned int vpx_sad8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x8 vpx_sad8x8_sse2 unsigned int vpx_sad8x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x8_avg vpx_sad8x8_avg_sse2 -void vpx_sad8x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x8x4d vpx_sad8x8x4d_sse2 -void vpx_sad8x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - int vpx_satd_c(const int16_t *coeff, int length); int vpx_satd_sse2(const int16_t *coeff, int length); int vpx_satd_avx2(const int16_t *coeff, int length); RTCD_EXTERN int (*vpx_satd)(const int16_t *coeff, int length); @@ -1058,58 +1012,34 @@ if (flags & HAS_SSSE3) vpx_quantize_b = vpx_quantize_b_ssse3; if (flags & HAS_AVX) vpx_quantize_b = vpx_quantize_b_avx; vpx_quantize_b_32x32 = vpx_quantize_b_32x32_c; if (flags & HAS_SSSE3) vpx_quantize_b_32x32 = vpx_quantize_b_32x32_ssse3; if (flags & HAS_AVX) vpx_quantize_b_32x32 = vpx_quantize_b_32x32_avx; - vpx_sad16x16x3 = vpx_sad16x16x3_c; - if (flags & HAS_SSE3) vpx_sad16x16x3 = vpx_sad16x16x3_sse3; - if (flags & HAS_SSSE3) vpx_sad16x16x3 = vpx_sad16x16x3_ssse3; - vpx_sad16x16x8 = vpx_sad16x16x8_c; - if (flags & HAS_SSE4_1) vpx_sad16x16x8 = vpx_sad16x16x8_sse4_1; - vpx_sad16x8x3 = vpx_sad16x8x3_c; - if (flags & HAS_SSE3) vpx_sad16x8x3 = vpx_sad16x8x3_sse3; - if (flags & HAS_SSSE3) vpx_sad16x8x3 = vpx_sad16x8x3_ssse3; - vpx_sad16x8x8 = vpx_sad16x8x8_c; - if (flags & HAS_SSE4_1) vpx_sad16x8x8 = vpx_sad16x8x8_sse4_1; vpx_sad32x16 = vpx_sad32x16_sse2; if (flags & HAS_AVX2) vpx_sad32x16 = vpx_sad32x16_avx2; vpx_sad32x16_avg = vpx_sad32x16_avg_sse2; if (flags & HAS_AVX2) vpx_sad32x16_avg = vpx_sad32x16_avg_avx2; vpx_sad32x32 = vpx_sad32x32_sse2; if (flags & HAS_AVX2) vpx_sad32x32 = vpx_sad32x32_avx2; vpx_sad32x32_avg = vpx_sad32x32_avg_sse2; if (flags & HAS_AVX2) vpx_sad32x32_avg = vpx_sad32x32_avg_avx2; vpx_sad32x32x4d = vpx_sad32x32x4d_sse2; if (flags & HAS_AVX2) vpx_sad32x32x4d = vpx_sad32x32x4d_avx2; - vpx_sad32x32x8 = vpx_sad32x32x8_c; - if (flags & HAS_AVX2) vpx_sad32x32x8 = vpx_sad32x32x8_avx2; vpx_sad32x64 = vpx_sad32x64_sse2; if (flags & HAS_AVX2) vpx_sad32x64 = vpx_sad32x64_avx2; vpx_sad32x64_avg = vpx_sad32x64_avg_sse2; if (flags & HAS_AVX2) vpx_sad32x64_avg = vpx_sad32x64_avg_avx2; - vpx_sad4x4x3 = vpx_sad4x4x3_c; - if (flags & HAS_SSE3) vpx_sad4x4x3 = vpx_sad4x4x3_sse3; - vpx_sad4x4x8 = vpx_sad4x4x8_c; - if (flags & HAS_SSE4_1) vpx_sad4x4x8 = vpx_sad4x4x8_sse4_1; vpx_sad64x32 = vpx_sad64x32_sse2; if (flags & HAS_AVX2) vpx_sad64x32 = vpx_sad64x32_avx2; vpx_sad64x32_avg = vpx_sad64x32_avg_sse2; if (flags & HAS_AVX2) vpx_sad64x32_avg = vpx_sad64x32_avg_avx2; vpx_sad64x64 = vpx_sad64x64_sse2; if (flags & HAS_AVX2) vpx_sad64x64 = vpx_sad64x64_avx2; vpx_sad64x64_avg = vpx_sad64x64_avg_sse2; if (flags & HAS_AVX2) vpx_sad64x64_avg = vpx_sad64x64_avg_avx2; vpx_sad64x64x4d = vpx_sad64x64x4d_sse2; if (flags & HAS_AVX2) vpx_sad64x64x4d = vpx_sad64x64x4d_avx2; - vpx_sad8x16x3 = vpx_sad8x16x3_c; - if (flags & HAS_SSE3) vpx_sad8x16x3 = vpx_sad8x16x3_sse3; - vpx_sad8x16x8 = vpx_sad8x16x8_c; - if (flags & HAS_SSE4_1) vpx_sad8x16x8 = vpx_sad8x16x8_sse4_1; - vpx_sad8x8x3 = vpx_sad8x8x3_c; - if (flags & HAS_SSE3) vpx_sad8x8x3 = vpx_sad8x8x3_sse3; - vpx_sad8x8x8 = vpx_sad8x8x8_c; - if (flags & HAS_SSE4_1) vpx_sad8x8x8 = vpx_sad8x8x8_sse4_1; vpx_satd = vpx_satd_sse2; if (flags & HAS_AVX2) vpx_satd = vpx_satd_avx2; vpx_scaled_2d = vpx_scaled_2d_c; if (flags & HAS_SSSE3) vpx_scaled_2d = vpx_scaled_2d_ssse3; vpx_sub_pixel_avg_variance16x16 = vpx_sub_pixel_avg_variance16x16_sse2; diff --git a/media/libvpx/config/win/aarch64/vp8_rtcd.h b/media/libvpx/config/win/aarch64/vp8_rtcd.h --- a/media/libvpx/config/win/aarch64/vp8_rtcd.h +++ b/media/libvpx/config/win/aarch64/vp8_rtcd.h @@ -94,13 +94,10 @@ void vp8_fast_quantize_b_c(struct block *, struct blockd *); void vp8_fast_quantize_b_neon(struct block *, struct blockd *); #define vp8_fast_quantize_b vp8_fast_quantize_b_neon -int vp8_full_search_sad_c(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -#define vp8_full_search_sad vp8_full_search_sad_c - void vp8_loop_filter_bh_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); void vp8_loop_filter_bh_neon(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); #define vp8_loop_filter_bh vp8_loop_filter_bh_neon void vp8_loop_filter_bv_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); diff --git a/media/libvpx/config/win/aarch64/vp9_rtcd.h b/media/libvpx/config/win/aarch64/vp9_rtcd.h --- a/media/libvpx/config/win/aarch64/vp9_rtcd.h +++ b/media/libvpx/config/win/aarch64/vp9_rtcd.h @@ -37,20 +37,24 @@ int64_t vp9_block_error_fp_c(const tran_low_t *coeff, const tran_low_t *dqcoeff, int block_size); int64_t vp9_block_error_fp_neon(const tran_low_t *coeff, const tran_low_t *dqcoeff, int block_size); #define vp9_block_error_fp vp9_block_error_fp_neon int vp9_diamond_search_sad_c(const struct macroblock *x, const struct search_site_config *cfg, struct mv *ref_mv, struct mv *best_mv, int search_param, int sad_per_bit, int *num00, const struct vp9_variance_vtable *fn_ptr, const struct mv *center_mv); -#define vp9_diamond_search_sad vp9_diamond_search_sad_c +int vp9_diamond_search_sad_neon(const struct macroblock *x, const struct search_site_config *cfg, struct mv *ref_mv, struct mv *best_mv, int search_param, int sad_per_bit, int *num00, const struct vp9_variance_vtable *fn_ptr, const struct mv *center_mv); +#define vp9_diamond_search_sad vp9_diamond_search_sad_neon void vp9_fht16x16_c(const int16_t *input, tran_low_t *output, int stride, int tx_type); -#define vp9_fht16x16 vp9_fht16x16_c +void vp9_fht16x16_neon(const int16_t *input, tran_low_t *output, int stride, int tx_type); +#define vp9_fht16x16 vp9_fht16x16_neon void vp9_fht4x4_c(const int16_t *input, tran_low_t *output, int stride, int tx_type); -#define vp9_fht4x4 vp9_fht4x4_c +void vp9_fht4x4_neon(const int16_t *input, tran_low_t *output, int stride, int tx_type); +#define vp9_fht4x4 vp9_fht4x4_neon void vp9_fht8x8_c(const int16_t *input, tran_low_t *output, int stride, int tx_type); -#define vp9_fht8x8 vp9_fht8x8_c +void vp9_fht8x8_neon(const int16_t *input, tran_low_t *output, int stride, int tx_type); +#define vp9_fht8x8 vp9_fht8x8_neon void vp9_fwht4x4_c(const int16_t *input, tran_low_t *output, int stride); #define vp9_fwht4x4 vp9_fwht4x4_c void vp9_iht16x16_256_add_c(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); @@ -63,16 +67,16 @@ void vp9_iht8x8_64_add_c(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); void vp9_iht8x8_64_add_neon(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); #define vp9_iht8x8_64_add vp9_iht8x8_64_add_neon -void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); #define vp9_quantize_fp vp9_quantize_fp_neon -void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); #define vp9_quantize_fp_32x32 vp9_quantize_fp_32x32_neon void vp9_scale_and_extend_frame_c(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); void vp9_scale_and_extend_frame_neon(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); #define vp9_scale_and_extend_frame vp9_scale_and_extend_frame_neon diff --git a/media/libvpx/config/win/aarch64/vpx_config.h b/media/libvpx/config/win/aarch64/vpx_config.h --- a/media/libvpx/config/win/aarch64/vpx_config.h +++ b/media/libvpx/config/win/aarch64/vpx_config.h @@ -13,10 +13,11 @@ #define VPX_ARCH_ARM 1 #define VPX_ARCH_MIPS 0 #define VPX_ARCH_X86 0 #define VPX_ARCH_X86_64 0 #define VPX_ARCH_PPC 0 +#define VPX_ARCH_LOONGARCH 0 #define HAVE_NEON 1 #define HAVE_NEON_ASM 0 #define HAVE_MIPS32 0 #define HAVE_DSPR2 0 #define HAVE_MSA 0 @@ -30,10 +31,12 @@ #define HAVE_AVX 0 #define HAVE_AVX2 0 #define HAVE_AVX512 0 #define HAVE_VSX 0 #define HAVE_MMI 0 +#define HAVE_LSX 0 +#define HAVE_LASX 0 #define HAVE_VPX_PORTS 1 #define HAVE_PTHREAD_H 0 #define CONFIG_DEPENDENCY_TRACKING 1 #define CONFIG_EXTERNAL_BUILD 1 #define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/win/aarch64/vpx_config.asm b/media/libvpx/config/win/aarch64/vpx_config.asm --- a/media/libvpx/config/win/aarch64/vpx_config.asm +++ b/media/libvpx/config/win/aarch64/vpx_config.asm @@ -1,13 +1,14 @@ @ This file was created from a .asm file @ using the ads2gas.pl script. - .syntax unified +.syntax unified .equ VPX_ARCH_ARM , 1 .equ VPX_ARCH_MIPS , 0 .equ VPX_ARCH_X86 , 0 .equ VPX_ARCH_X86_64 , 0 .equ VPX_ARCH_PPC , 0 +.equ VPX_ARCH_LOONGARCH , 0 .equ HAVE_NEON , 1 .equ HAVE_NEON_ASM , 0 .equ HAVE_MIPS32 , 0 .equ HAVE_DSPR2 , 0 .equ HAVE_MSA , 0 @@ -21,10 +22,12 @@ .equ HAVE_AVX , 0 .equ HAVE_AVX2 , 0 .equ HAVE_AVX512 , 0 .equ HAVE_VSX , 0 .equ HAVE_MMI , 0 +.equ HAVE_LSX , 0 +.equ HAVE_LASX , 0 .equ HAVE_VPX_PORTS , 1 .equ HAVE_PTHREAD_H , 0 .equ CONFIG_DEPENDENCY_TRACKING , 1 .equ CONFIG_EXTERNAL_BUILD , 1 .equ CONFIG_INSTALL_DOCS , 0 @@ -85,6 +88,6 @@ .equ CONFIG_MISMATCH_DEBUG , 0 .equ CONFIG_FP_MB_STATS , 0 .equ CONFIG_EMULATE_HARDWARE , 0 .equ CONFIG_NON_GREEDY_MV , 0 .equ CONFIG_RATE_CTRL , 0 - .section .note.GNU-stack,"",%progbits + .section .note.GNU-stack,"",%progbits diff --git a/media/libvpx/config/win/aarch64/vpx_dsp_rtcd.h b/media/libvpx/config/win/aarch64/vpx_dsp_rtcd.h --- a/media/libvpx/config/win/aarch64/vpx_dsp_rtcd.h +++ b/media/libvpx/config/win/aarch64/vpx_dsp_rtcd.h @@ -350,12 +350,12 @@ int16_t vpx_int_pro_col_c(const uint8_t *ref, const int width); int16_t vpx_int_pro_col_neon(const uint8_t *ref, const int width); #define vpx_int_pro_col vpx_int_pro_col_neon -void vpx_int_pro_row_c(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); -void vpx_int_pro_row_neon(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_c(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_neon(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); #define vpx_int_pro_row vpx_int_pro_row_neon void vpx_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int stride); #define vpx_iwht4x4_16_add vpx_iwht4x4_16_add_c @@ -425,207 +425,174 @@ #define vpx_mse8x16 vpx_mse8x16_c unsigned int vpx_mse8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse); #define vpx_mse8x8 vpx_mse8x8_c -void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); #define vpx_quantize_b vpx_quantize_b_neon -void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); #define vpx_quantize_b_32x32 vpx_quantize_b_32x32_neon unsigned int vpx_sad16x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x16_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x16 vpx_sad16x16_neon unsigned int vpx_sad16x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x16_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x16_avg vpx_sad16x16_avg_neon -void vpx_sad16x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x16x3 vpx_sad16x16x3_c - -void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x16x4d vpx_sad16x16x4d_neon -void vpx_sad16x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x16x8 vpx_sad16x16x8_c - unsigned int vpx_sad16x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x32_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x32 vpx_sad16x32_neon unsigned int vpx_sad16x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x32_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x32_avg vpx_sad16x32_avg_neon -void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x32x4d vpx_sad16x32x4d_neon unsigned int vpx_sad16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x8_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x8 vpx_sad16x8_neon unsigned int vpx_sad16x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x8_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x8_avg vpx_sad16x8_avg_neon -void vpx_sad16x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x8x3 vpx_sad16x8x3_c - -void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x8x4d vpx_sad16x8x4d_neon -void vpx_sad16x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad16x8x8 vpx_sad16x8x8_c - unsigned int vpx_sad32x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad32x16 vpx_sad32x16_neon unsigned int vpx_sad32x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x16_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad32x16_avg vpx_sad32x16_avg_neon -void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad32x16x4d vpx_sad32x16x4d_neon unsigned int vpx_sad32x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad32x32 vpx_sad32x32_neon unsigned int vpx_sad32x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x32_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad32x32_avg vpx_sad32x32_avg_neon -void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad32x32x4d vpx_sad32x32x4d_neon -void vpx_sad32x32x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad32x32x8 vpx_sad32x32x8_c - unsigned int vpx_sad32x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad32x64 vpx_sad32x64_neon unsigned int vpx_sad32x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x64_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad32x64_avg vpx_sad32x64_avg_neon -void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad32x64x4d vpx_sad32x64x4d_neon unsigned int vpx_sad4x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x4_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad4x4 vpx_sad4x4_neon unsigned int vpx_sad4x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x4_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad4x4_avg vpx_sad4x4_avg_neon -void vpx_sad4x4x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad4x4x3 vpx_sad4x4x3_c - -void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x4x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad4x4x4d vpx_sad4x4x4d_neon -void vpx_sad4x4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad4x4x8 vpx_sad4x4x8_c - unsigned int vpx_sad4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x8_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad4x8 vpx_sad4x8_neon unsigned int vpx_sad4x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x8_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad4x8_avg vpx_sad4x8_avg_neon -void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad4x8x4d vpx_sad4x8x4d_neon unsigned int vpx_sad64x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad64x32 vpx_sad64x32_neon unsigned int vpx_sad64x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x32_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad64x32_avg vpx_sad64x32_avg_neon -void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad64x32x4d vpx_sad64x32x4d_neon unsigned int vpx_sad64x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad64x64 vpx_sad64x64_neon unsigned int vpx_sad64x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x64_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad64x64_avg vpx_sad64x64_avg_neon -void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad64x64x4d vpx_sad64x64x4d_neon unsigned int vpx_sad8x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x16_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x16 vpx_sad8x16_neon unsigned int vpx_sad8x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x16_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x16_avg vpx_sad8x16_avg_neon -void vpx_sad8x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x16x3 vpx_sad8x16x3_c - -void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x16x4d vpx_sad8x16x4d_neon -void vpx_sad8x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x16x8 vpx_sad8x16x8_c - unsigned int vpx_sad8x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x4_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x4 vpx_sad8x4_neon unsigned int vpx_sad8x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x4_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x4_avg vpx_sad8x4_avg_neon -void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x4x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x4x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x4x4d vpx_sad8x4x4d_neon unsigned int vpx_sad8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x8_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x8 vpx_sad8x8_neon unsigned int vpx_sad8x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x8_avg_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x8_avg vpx_sad8x8_avg_neon -void vpx_sad8x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x8x3 vpx_sad8x8x3_c - -void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x8x4d vpx_sad8x8x4d_neon -void vpx_sad8x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -#define vpx_sad8x8x8 vpx_sad8x8x8_c - int vpx_satd_c(const int16_t *coeff, int length); int vpx_satd_neon(const int16_t *coeff, int length); #define vpx_satd vpx_satd_neon void vpx_scaled_2d_c(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h); diff --git a/media/libvpx/config/win/ia32/vp8_rtcd.h b/media/libvpx/config/win/ia32/vp8_rtcd.h --- a/media/libvpx/config/win/ia32/vp8_rtcd.h +++ b/media/libvpx/config/win/ia32/vp8_rtcd.h @@ -121,15 +121,10 @@ void vp8_filter_by_weight8x8_c(unsigned char *src, int src_stride, unsigned char *dst, int dst_stride, int src_weight); void vp8_filter_by_weight8x8_sse2(unsigned char *src, int src_stride, unsigned char *dst, int dst_stride, int src_weight); RTCD_EXTERN void (*vp8_filter_by_weight8x8)(unsigned char *src, int src_stride, unsigned char *dst, int dst_stride, int src_weight); -int vp8_full_search_sad_c(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -int vp8_full_search_sadx3(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -int vp8_full_search_sadx8(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -RTCD_EXTERN int (*vp8_full_search_sad)(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); - void vp8_loop_filter_bh_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); void vp8_loop_filter_bh_sse2(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); RTCD_EXTERN void (*vp8_loop_filter_bh)(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); void vp8_loop_filter_bv_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); @@ -276,13 +271,10 @@ if (flags & HAS_SSSE3) vp8_fast_quantize_b = vp8_fast_quantize_b_ssse3; vp8_filter_by_weight16x16 = vp8_filter_by_weight16x16_c; if (flags & HAS_SSE2) vp8_filter_by_weight16x16 = vp8_filter_by_weight16x16_sse2; vp8_filter_by_weight8x8 = vp8_filter_by_weight8x8_c; if (flags & HAS_SSE2) vp8_filter_by_weight8x8 = vp8_filter_by_weight8x8_sse2; - vp8_full_search_sad = vp8_full_search_sad_c; - if (flags & HAS_SSE3) vp8_full_search_sad = vp8_full_search_sadx3; - if (flags & HAS_SSE4_1) vp8_full_search_sad = vp8_full_search_sadx8; vp8_loop_filter_bh = vp8_loop_filter_bh_c; if (flags & HAS_SSE2) vp8_loop_filter_bh = vp8_loop_filter_bh_sse2; vp8_loop_filter_bv = vp8_loop_filter_bv_c; if (flags & HAS_SSE2) vp8_loop_filter_bv = vp8_loop_filter_bv_sse2; vp8_loop_filter_mbh = vp8_loop_filter_mbh_c; diff --git a/media/libvpx/config/win/ia32/vp9_rtcd.h b/media/libvpx/config/win/ia32/vp9_rtcd.h --- a/media/libvpx/config/win/ia32/vp9_rtcd.h +++ b/media/libvpx/config/win/ia32/vp9_rtcd.h @@ -83,16 +83,16 @@ void vp9_iht8x8_64_add_c(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); void vp9_iht8x8_64_add_sse2(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); RTCD_EXTERN void (*vp9_iht8x8_64_add)(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); -void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vp9_quantize_fp)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vp9_quantize_fp)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); #define vp9_quantize_fp_32x32 vp9_quantize_fp_32x32_c void vp9_scale_and_extend_frame_c(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); void vp9_scale_and_extend_frame_ssse3(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); RTCD_EXTERN void (*vp9_scale_and_extend_frame)(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); diff --git a/media/libvpx/config/win/ia32/vpx_config.h b/media/libvpx/config/win/ia32/vpx_config.h --- a/media/libvpx/config/win/ia32/vpx_config.h +++ b/media/libvpx/config/win/ia32/vpx_config.h @@ -13,10 +13,11 @@ #define VPX_ARCH_ARM 0 #define VPX_ARCH_MIPS 0 #define VPX_ARCH_X86 1 #define VPX_ARCH_X86_64 0 #define VPX_ARCH_PPC 0 +#define VPX_ARCH_LOONGARCH 0 #define HAVE_NEON 0 #define HAVE_NEON_ASM 0 #define HAVE_MIPS32 0 #define HAVE_DSPR2 0 #define HAVE_MSA 0 @@ -30,10 +31,12 @@ #define HAVE_AVX 1 #define HAVE_AVX2 1 #define HAVE_AVX512 0 #define HAVE_VSX 0 #define HAVE_MMI 0 +#define HAVE_LSX 0 +#define HAVE_LASX 0 #define HAVE_VPX_PORTS 1 #define HAVE_PTHREAD_H 0 #define CONFIG_DEPENDENCY_TRACKING 1 #define CONFIG_EXTERNAL_BUILD 1 #define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/win/ia32/vpx_config.asm b/media/libvpx/config/win/ia32/vpx_config.asm --- a/media/libvpx/config/win/ia32/vpx_config.asm +++ b/media/libvpx/config/win/ia32/vpx_config.asm @@ -1,10 +1,11 @@ %define VPX_ARCH_ARM 0 %define VPX_ARCH_MIPS 0 %define VPX_ARCH_X86 1 %define VPX_ARCH_X86_64 0 %define VPX_ARCH_PPC 0 +%define VPX_ARCH_LOONGARCH 0 %define HAVE_NEON 0 %define HAVE_NEON_ASM 0 %define HAVE_MIPS32 0 %define HAVE_DSPR2 0 %define HAVE_MSA 0 @@ -18,10 +19,12 @@ %define HAVE_AVX 1 %define HAVE_AVX2 1 %define HAVE_AVX512 0 %define HAVE_VSX 0 %define HAVE_MMI 0 +%define HAVE_LSX 0 +%define HAVE_LASX 0 %define HAVE_VPX_PORTS 1 %define HAVE_PTHREAD_H 0 %define CONFIG_DEPENDENCY_TRACKING 1 %define CONFIG_EXTERNAL_BUILD 1 %define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/win/ia32/vpx_dsp_rtcd.h b/media/libvpx/config/win/ia32/vpx_dsp_rtcd.h --- a/media/libvpx/config/win/ia32/vpx_dsp_rtcd.h +++ b/media/libvpx/config/win/ia32/vpx_dsp_rtcd.h @@ -379,13 +379,13 @@ int16_t vpx_int_pro_col_c(const uint8_t *ref, const int width); int16_t vpx_int_pro_col_sse2(const uint8_t *ref, const int width); RTCD_EXTERN int16_t (*vpx_int_pro_col)(const uint8_t *ref, const int width); -void vpx_int_pro_row_c(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); -void vpx_int_pro_row_sse2(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); -RTCD_EXTERN void (*vpx_int_pro_row)(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_c(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_sse2(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); +RTCD_EXTERN void (*vpx_int_pro_row)(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); void vpx_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int stride); void vpx_iwht4x4_16_add_sse2(const tran_low_t *input, uint8_t *dest, int stride); RTCD_EXTERN void (*vpx_iwht4x4_16_add)(const tran_low_t *input, uint8_t *dest, int stride); @@ -478,74 +478,56 @@ void vpx_post_proc_down_and_across_mb_row_c(unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size); void vpx_post_proc_down_and_across_mb_row_sse2(unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size); RTCD_EXTERN void (*vpx_post_proc_down_and_across_mb_row)(unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size); -void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vpx_quantize_b)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vpx_quantize_b)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vpx_quantize_b_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vpx_quantize_b_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); unsigned int vpx_sad16x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad16x16)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad16x16_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad16x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x3_ssse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad16x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad16x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad16x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad16x32)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad16x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad16x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad16x8_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad16x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x3_ssse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad16x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad16x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad32x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x16)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -553,13 +535,13 @@ unsigned int vpx_sad32x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x16_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x16_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad32x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad32x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x32)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -567,18 +549,14 @@ unsigned int vpx_sad32x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x32_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad32x32x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x8_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x32x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x32x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad32x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad32x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x64)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -586,45 +564,37 @@ unsigned int vpx_sad32x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x64_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x64_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x64_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad32x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad4x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x4_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad4x4)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x4_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad4x4_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad4x4x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad4x4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad4x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad4x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad4x8_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad4x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad64x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad64x32)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -632,13 +602,13 @@ unsigned int vpx_sad64x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x32_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad64x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad64x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad64x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad64x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad64x64)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -646,66 +616,50 @@ unsigned int vpx_sad64x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x64_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x64_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad64x64_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x64x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad64x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x64x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad64x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad8x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad8x16)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad8x16_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad8x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad8x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad8x16x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad8x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x4_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad8x4)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x4_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad8x4_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad8x4x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad8x8_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad8x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad8x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad8x8x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); int vpx_satd_c(const int16_t *coeff, int length); int vpx_satd_sse2(const int16_t *coeff, int length); int vpx_satd_avx2(const int16_t *coeff, int length); RTCD_EXTERN int (*vpx_satd)(const int16_t *coeff, int length); @@ -1211,34 +1165,24 @@ if (flags & HAS_AVX) vpx_quantize_b_32x32 = vpx_quantize_b_32x32_avx; vpx_sad16x16 = vpx_sad16x16_c; if (flags & HAS_SSE2) vpx_sad16x16 = vpx_sad16x16_sse2; vpx_sad16x16_avg = vpx_sad16x16_avg_c; if (flags & HAS_SSE2) vpx_sad16x16_avg = vpx_sad16x16_avg_sse2; - vpx_sad16x16x3 = vpx_sad16x16x3_c; - if (flags & HAS_SSE3) vpx_sad16x16x3 = vpx_sad16x16x3_sse3; - if (flags & HAS_SSSE3) vpx_sad16x16x3 = vpx_sad16x16x3_ssse3; vpx_sad16x16x4d = vpx_sad16x16x4d_c; if (flags & HAS_SSE2) vpx_sad16x16x4d = vpx_sad16x16x4d_sse2; - vpx_sad16x16x8 = vpx_sad16x16x8_c; - if (flags & HAS_SSE4_1) vpx_sad16x16x8 = vpx_sad16x16x8_sse4_1; vpx_sad16x32 = vpx_sad16x32_c; if (flags & HAS_SSE2) vpx_sad16x32 = vpx_sad16x32_sse2; vpx_sad16x32_avg = vpx_sad16x32_avg_c; if (flags & HAS_SSE2) vpx_sad16x32_avg = vpx_sad16x32_avg_sse2; vpx_sad16x32x4d = vpx_sad16x32x4d_c; if (flags & HAS_SSE2) vpx_sad16x32x4d = vpx_sad16x32x4d_sse2; vpx_sad16x8 = vpx_sad16x8_c; if (flags & HAS_SSE2) vpx_sad16x8 = vpx_sad16x8_sse2; vpx_sad16x8_avg = vpx_sad16x8_avg_c; if (flags & HAS_SSE2) vpx_sad16x8_avg = vpx_sad16x8_avg_sse2; - vpx_sad16x8x3 = vpx_sad16x8x3_c; - if (flags & HAS_SSE3) vpx_sad16x8x3 = vpx_sad16x8x3_sse3; - if (flags & HAS_SSSE3) vpx_sad16x8x3 = vpx_sad16x8x3_ssse3; vpx_sad16x8x4d = vpx_sad16x8x4d_c; if (flags & HAS_SSE2) vpx_sad16x8x4d = vpx_sad16x8x4d_sse2; - vpx_sad16x8x8 = vpx_sad16x8x8_c; - if (flags & HAS_SSE4_1) vpx_sad16x8x8 = vpx_sad16x8x8_sse4_1; vpx_sad32x16 = vpx_sad32x16_c; if (flags & HAS_SSE2) vpx_sad32x16 = vpx_sad32x16_sse2; if (flags & HAS_AVX2) vpx_sad32x16 = vpx_sad32x16_avx2; vpx_sad32x16_avg = vpx_sad32x16_avg_c; if (flags & HAS_SSE2) vpx_sad32x16_avg = vpx_sad32x16_avg_sse2; @@ -1252,12 +1196,10 @@ if (flags & HAS_SSE2) vpx_sad32x32_avg = vpx_sad32x32_avg_sse2; if (flags & HAS_AVX2) vpx_sad32x32_avg = vpx_sad32x32_avg_avx2; vpx_sad32x32x4d = vpx_sad32x32x4d_c; if (flags & HAS_SSE2) vpx_sad32x32x4d = vpx_sad32x32x4d_sse2; if (flags & HAS_AVX2) vpx_sad32x32x4d = vpx_sad32x32x4d_avx2; - vpx_sad32x32x8 = vpx_sad32x32x8_c; - if (flags & HAS_AVX2) vpx_sad32x32x8 = vpx_sad32x32x8_avx2; vpx_sad32x64 = vpx_sad32x64_c; if (flags & HAS_SSE2) vpx_sad32x64 = vpx_sad32x64_sse2; if (flags & HAS_AVX2) vpx_sad32x64 = vpx_sad32x64_avx2; vpx_sad32x64_avg = vpx_sad32x64_avg_c; if (flags & HAS_SSE2) vpx_sad32x64_avg = vpx_sad32x64_avg_sse2; @@ -1266,16 +1208,12 @@ if (flags & HAS_SSE2) vpx_sad32x64x4d = vpx_sad32x64x4d_sse2; vpx_sad4x4 = vpx_sad4x4_c; if (flags & HAS_SSE2) vpx_sad4x4 = vpx_sad4x4_sse2; vpx_sad4x4_avg = vpx_sad4x4_avg_c; if (flags & HAS_SSE2) vpx_sad4x4_avg = vpx_sad4x4_avg_sse2; - vpx_sad4x4x3 = vpx_sad4x4x3_c; - if (flags & HAS_SSE3) vpx_sad4x4x3 = vpx_sad4x4x3_sse3; vpx_sad4x4x4d = vpx_sad4x4x4d_c; if (flags & HAS_SSE2) vpx_sad4x4x4d = vpx_sad4x4x4d_sse2; - vpx_sad4x4x8 = vpx_sad4x4x8_c; - if (flags & HAS_SSE4_1) vpx_sad4x4x8 = vpx_sad4x4x8_sse4_1; vpx_sad4x8 = vpx_sad4x8_c; if (flags & HAS_SSE2) vpx_sad4x8 = vpx_sad4x8_sse2; vpx_sad4x8_avg = vpx_sad4x8_avg_c; if (flags & HAS_SSE2) vpx_sad4x8_avg = vpx_sad4x8_avg_sse2; vpx_sad4x8x4d = vpx_sad4x8x4d_c; @@ -1299,32 +1237,24 @@ if (flags & HAS_AVX2) vpx_sad64x64x4d = vpx_sad64x64x4d_avx2; vpx_sad8x16 = vpx_sad8x16_c; if (flags & HAS_SSE2) vpx_sad8x16 = vpx_sad8x16_sse2; vpx_sad8x16_avg = vpx_sad8x16_avg_c; if (flags & HAS_SSE2) vpx_sad8x16_avg = vpx_sad8x16_avg_sse2; - vpx_sad8x16x3 = vpx_sad8x16x3_c; - if (flags & HAS_SSE3) vpx_sad8x16x3 = vpx_sad8x16x3_sse3; vpx_sad8x16x4d = vpx_sad8x16x4d_c; if (flags & HAS_SSE2) vpx_sad8x16x4d = vpx_sad8x16x4d_sse2; - vpx_sad8x16x8 = vpx_sad8x16x8_c; - if (flags & HAS_SSE4_1) vpx_sad8x16x8 = vpx_sad8x16x8_sse4_1; vpx_sad8x4 = vpx_sad8x4_c; if (flags & HAS_SSE2) vpx_sad8x4 = vpx_sad8x4_sse2; vpx_sad8x4_avg = vpx_sad8x4_avg_c; if (flags & HAS_SSE2) vpx_sad8x4_avg = vpx_sad8x4_avg_sse2; vpx_sad8x4x4d = vpx_sad8x4x4d_c; if (flags & HAS_SSE2) vpx_sad8x4x4d = vpx_sad8x4x4d_sse2; vpx_sad8x8 = vpx_sad8x8_c; if (flags & HAS_SSE2) vpx_sad8x8 = vpx_sad8x8_sse2; vpx_sad8x8_avg = vpx_sad8x8_avg_c; if (flags & HAS_SSE2) vpx_sad8x8_avg = vpx_sad8x8_avg_sse2; - vpx_sad8x8x3 = vpx_sad8x8x3_c; - if (flags & HAS_SSE3) vpx_sad8x8x3 = vpx_sad8x8x3_sse3; vpx_sad8x8x4d = vpx_sad8x8x4d_c; if (flags & HAS_SSE2) vpx_sad8x8x4d = vpx_sad8x8x4d_sse2; - vpx_sad8x8x8 = vpx_sad8x8x8_c; - if (flags & HAS_SSE4_1) vpx_sad8x8x8 = vpx_sad8x8x8_sse4_1; vpx_satd = vpx_satd_c; if (flags & HAS_SSE2) vpx_satd = vpx_satd_sse2; if (flags & HAS_AVX2) vpx_satd = vpx_satd_avx2; vpx_scaled_2d = vpx_scaled_2d_c; if (flags & HAS_SSSE3) vpx_scaled_2d = vpx_scaled_2d_ssse3; diff --git a/media/libvpx/config/win/x64/vp8_rtcd.h b/media/libvpx/config/win/x64/vp8_rtcd.h --- a/media/libvpx/config/win/x64/vp8_rtcd.h +++ b/media/libvpx/config/win/x64/vp8_rtcd.h @@ -121,15 +121,10 @@ void vp8_filter_by_weight8x8_c(unsigned char *src, int src_stride, unsigned char *dst, int dst_stride, int src_weight); void vp8_filter_by_weight8x8_sse2(unsigned char *src, int src_stride, unsigned char *dst, int dst_stride, int src_weight); #define vp8_filter_by_weight8x8 vp8_filter_by_weight8x8_sse2 -int vp8_full_search_sad_c(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -int vp8_full_search_sadx3(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -int vp8_full_search_sadx8(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); -RTCD_EXTERN int (*vp8_full_search_sad)(struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv); - void vp8_loop_filter_bh_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); void vp8_loop_filter_bh_sse2(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); #define vp8_loop_filter_bh vp8_loop_filter_bh_sse2 void vp8_loop_filter_bv_c(unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi); @@ -240,13 +235,10 @@ if (flags & HAS_SSSE3) vp8_bilinear_predict8x8 = vp8_bilinear_predict8x8_ssse3; vp8_copy32xn = vp8_copy32xn_sse2; if (flags & HAS_SSE3) vp8_copy32xn = vp8_copy32xn_sse3; vp8_fast_quantize_b = vp8_fast_quantize_b_sse2; if (flags & HAS_SSSE3) vp8_fast_quantize_b = vp8_fast_quantize_b_ssse3; - vp8_full_search_sad = vp8_full_search_sad_c; - if (flags & HAS_SSE3) vp8_full_search_sad = vp8_full_search_sadx3; - if (flags & HAS_SSE4_1) vp8_full_search_sad = vp8_full_search_sadx8; vp8_regular_quantize_b = vp8_regular_quantize_b_sse2; if (flags & HAS_SSE4_1) vp8_regular_quantize_b = vp8_regular_quantize_b_sse4_1; vp8_sixtap_predict16x16 = vp8_sixtap_predict16x16_sse2; if (flags & HAS_SSSE3) vp8_sixtap_predict16x16 = vp8_sixtap_predict16x16_ssse3; vp8_sixtap_predict4x4 = vp8_sixtap_predict4x4_mmx; diff --git a/media/libvpx/config/win/x64/vp9_rtcd.h b/media/libvpx/config/win/x64/vp9_rtcd.h --- a/media/libvpx/config/win/x64/vp9_rtcd.h +++ b/media/libvpx/config/win/x64/vp9_rtcd.h @@ -83,19 +83,19 @@ void vp9_iht8x8_64_add_c(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); void vp9_iht8x8_64_add_sse2(const tran_low_t *input, uint8_t *dest, int stride, int tx_type); #define vp9_iht8x8_64_add vp9_iht8x8_64_add_sse2 -void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vp9_quantize_fp)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vp9_quantize_fp)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vp9_quantize_fp_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vp9_quantize_fp_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vp9_quantize_fp_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vp9_quantize_fp_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); void vp9_scale_and_extend_frame_c(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); void vp9_scale_and_extend_frame_ssse3(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); RTCD_EXTERN void (*vp9_scale_and_extend_frame)(const struct yv12_buffer_config *src, struct yv12_buffer_config *dst, INTERP_FILTER filter_type, int phase_scaler); diff --git a/media/libvpx/config/win/x64/vpx_config.h b/media/libvpx/config/win/x64/vpx_config.h --- a/media/libvpx/config/win/x64/vpx_config.h +++ b/media/libvpx/config/win/x64/vpx_config.h @@ -13,10 +13,11 @@ #define VPX_ARCH_ARM 0 #define VPX_ARCH_MIPS 0 #define VPX_ARCH_X86 0 #define VPX_ARCH_X86_64 1 #define VPX_ARCH_PPC 0 +#define VPX_ARCH_LOONGARCH 0 #define HAVE_NEON 0 #define HAVE_NEON_ASM 0 #define HAVE_MIPS32 0 #define HAVE_DSPR2 0 #define HAVE_MSA 0 @@ -30,10 +31,12 @@ #define HAVE_AVX 1 #define HAVE_AVX2 1 #define HAVE_AVX512 0 #define HAVE_VSX 0 #define HAVE_MMI 0 +#define HAVE_LSX 0 +#define HAVE_LASX 0 #define HAVE_VPX_PORTS 1 #define HAVE_PTHREAD_H 0 #define CONFIG_DEPENDENCY_TRACKING 1 #define CONFIG_EXTERNAL_BUILD 1 #define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/win/x64/vpx_config.asm b/media/libvpx/config/win/x64/vpx_config.asm --- a/media/libvpx/config/win/x64/vpx_config.asm +++ b/media/libvpx/config/win/x64/vpx_config.asm @@ -1,10 +1,11 @@ %define VPX_ARCH_ARM 0 %define VPX_ARCH_MIPS 0 %define VPX_ARCH_X86 0 %define VPX_ARCH_X86_64 1 %define VPX_ARCH_PPC 0 +%define VPX_ARCH_LOONGARCH 0 %define HAVE_NEON 0 %define HAVE_NEON_ASM 0 %define HAVE_MIPS32 0 %define HAVE_DSPR2 0 %define HAVE_MSA 0 @@ -18,10 +19,12 @@ %define HAVE_AVX 1 %define HAVE_AVX2 1 %define HAVE_AVX512 0 %define HAVE_VSX 0 %define HAVE_MMI 0 +%define HAVE_LSX 0 +%define HAVE_LASX 0 %define HAVE_VPX_PORTS 1 %define HAVE_PTHREAD_H 0 %define CONFIG_DEPENDENCY_TRACKING 1 %define CONFIG_EXTERNAL_BUILD 1 %define CONFIG_INSTALL_DOCS 0 diff --git a/media/libvpx/config/win/x64/vpx_dsp_rtcd.h b/media/libvpx/config/win/x64/vpx_dsp_rtcd.h --- a/media/libvpx/config/win/x64/vpx_dsp_rtcd.h +++ b/media/libvpx/config/win/x64/vpx_dsp_rtcd.h @@ -381,12 +381,12 @@ int16_t vpx_int_pro_col_c(const uint8_t *ref, const int width); int16_t vpx_int_pro_col_sse2(const uint8_t *ref, const int width); #define vpx_int_pro_col vpx_int_pro_col_sse2 -void vpx_int_pro_row_c(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); -void vpx_int_pro_row_sse2(int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_c(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); +void vpx_int_pro_row_sse2(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height); #define vpx_int_pro_row vpx_int_pro_row_sse2 void vpx_iwht4x4_16_add_c(const tran_low_t *input, uint8_t *dest, int stride); void vpx_iwht4x4_16_add_sse2(const tran_low_t *input, uint8_t *dest, int stride); #define vpx_iwht4x4_16_add vpx_iwht4x4_16_add_sse2 @@ -480,87 +480,69 @@ void vpx_post_proc_down_and_across_mb_row_c(unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size); void vpx_post_proc_down_and_across_mb_row_sse2(unsigned char *src, unsigned char *dst, int src_pitch, int dst_pitch, int cols, unsigned char *flimits, int size); #define vpx_post_proc_down_and_across_mb_row vpx_post_proc_down_and_across_mb_row_sse2 -void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vpx_quantize_b)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vpx_quantize_b)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -void vpx_quantize_b_32x32_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); -RTCD_EXTERN void (*vpx_quantize_b_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +void vpx_quantize_b_32x32_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); +RTCD_EXTERN void (*vpx_quantize_b_32x32)(const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan); unsigned int vpx_sad16x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x16 vpx_sad16x16_sse2 unsigned int vpx_sad16x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x16_avg vpx_sad16x16_avg_sse2 -void vpx_sad16x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x3_ssse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x16x4d vpx_sad16x16x4d_sse2 -void vpx_sad16x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x16x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - unsigned int vpx_sad16x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x32 vpx_sad16x32_sse2 unsigned int vpx_sad16x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x32_avg vpx_sad16x32_avg_sse2 -void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x32x4d vpx_sad16x32x4d_sse2 unsigned int vpx_sad16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad16x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad16x8 vpx_sad16x8_sse2 unsigned int vpx_sad16x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad16x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad16x8_avg vpx_sad16x8_avg_sse2 -void vpx_sad16x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x3_ssse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad16x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad16x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad16x8x4d vpx_sad16x8x4d_sse2 -void vpx_sad16x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad16x8x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad16x8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - unsigned int vpx_sad32x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x16)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x16_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x16_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad32x16x4d vpx_sad32x16x4d_sse2 unsigned int vpx_sad32x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x32_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -569,18 +551,14 @@ unsigned int vpx_sad32x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x32_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); - -void vpx_sad32x32x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad32x32x8_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad32x32x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); +void vpx_sad32x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x32x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad32x32x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad32x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad32x64_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); RTCD_EXTERN unsigned int (*vpx_sad32x64)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -588,44 +566,36 @@ unsigned int vpx_sad32x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x64_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad32x64_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad32x64_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad32x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad32x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad32x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad32x64x4d vpx_sad32x64x4d_sse2 unsigned int vpx_sad4x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x4_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad4x4 vpx_sad4x4_sse2 unsigned int vpx_sad4x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x4_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad4x4_avg vpx_sad4x4_avg_sse2 -void vpx_sad4x4x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad4x4x4d vpx_sad4x4x4d_sse2 -void vpx_sad4x4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad4x4x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad4x4x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - unsigned int vpx_sad4x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad4x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad4x8 vpx_sad4x8_sse2 unsigned int vpx_sad4x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad4x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad4x8_avg vpx_sad4x8_avg_sse2 -void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad4x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad4x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad4x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad4x8x4d vpx_sad4x8x4d_sse2 unsigned int vpx_sad64x32_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x32_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -634,12 +604,12 @@ unsigned int vpx_sad64x32_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x32_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x32_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad64x32_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x32x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x32x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad64x32x4d vpx_sad64x32x4d_sse2 unsigned int vpx_sad64x64_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad64x64_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); @@ -648,67 +618,51 @@ unsigned int vpx_sad64x64_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x64_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad64x64_avg_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); RTCD_EXTERN unsigned int (*vpx_sad64x64_avg)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); -void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad64x64x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad64x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad64x64x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x64x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad64x64x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +RTCD_EXTERN void (*vpx_sad64x64x4d)(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); unsigned int vpx_sad8x16_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x16_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x16 vpx_sad8x16_sse2 unsigned int vpx_sad8x16_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x16_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x16_avg vpx_sad8x16_avg_sse2 -void vpx_sad8x16x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x16x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x16x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x16x4d vpx_sad8x16x4d_sse2 -void vpx_sad8x16x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x16x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x16x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - unsigned int vpx_sad8x4_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x4_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x4 vpx_sad8x4_sse2 unsigned int vpx_sad8x4_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x4_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x4_avg vpx_sad8x4_avg_sse2 -void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x4x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x4x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x4x4d vpx_sad8x4x4d_sse2 unsigned int vpx_sad8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); unsigned int vpx_sad8x8_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride); #define vpx_sad8x8 vpx_sad8x8_sse2 unsigned int vpx_sad8x8_avg_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); unsigned int vpx_sad8x8_avg_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred); #define vpx_sad8x8_avg vpx_sad8x8_avg_sse2 -void vpx_sad8x8x3_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x3_sse3(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x3)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - -void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array); +void vpx_sad8x8x4d_c(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); +void vpx_sad8x8x4d_sse2(const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]); #define vpx_sad8x8x4d vpx_sad8x8x4d_sse2 -void vpx_sad8x8x8_c(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -void vpx_sad8x8x8_sse4_1(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); -RTCD_EXTERN void (*vpx_sad8x8x8)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array); - int vpx_satd_c(const int16_t *coeff, int length); int vpx_satd_sse2(const int16_t *coeff, int length); int vpx_satd_avx2(const int16_t *coeff, int length); RTCD_EXTERN int (*vpx_satd)(const int16_t *coeff, int length); @@ -1058,58 +1012,34 @@ if (flags & HAS_SSSE3) vpx_quantize_b = vpx_quantize_b_ssse3; if (flags & HAS_AVX) vpx_quantize_b = vpx_quantize_b_avx; vpx_quantize_b_32x32 = vpx_quantize_b_32x32_c; if (flags & HAS_SSSE3) vpx_quantize_b_32x32 = vpx_quantize_b_32x32_ssse3; if (flags & HAS_AVX) vpx_quantize_b_32x32 = vpx_quantize_b_32x32_avx; - vpx_sad16x16x3 = vpx_sad16x16x3_c; - if (flags & HAS_SSE3) vpx_sad16x16x3 = vpx_sad16x16x3_sse3; - if (flags & HAS_SSSE3) vpx_sad16x16x3 = vpx_sad16x16x3_ssse3; - vpx_sad16x16x8 = vpx_sad16x16x8_c; - if (flags & HAS_SSE4_1) vpx_sad16x16x8 = vpx_sad16x16x8_sse4_1; - vpx_sad16x8x3 = vpx_sad16x8x3_c; - if (flags & HAS_SSE3) vpx_sad16x8x3 = vpx_sad16x8x3_sse3; - if (flags & HAS_SSSE3) vpx_sad16x8x3 = vpx_sad16x8x3_ssse3; - vpx_sad16x8x8 = vpx_sad16x8x8_c; - if (flags & HAS_SSE4_1) vpx_sad16x8x8 = vpx_sad16x8x8_sse4_1; vpx_sad32x16 = vpx_sad32x16_sse2; if (flags & HAS_AVX2) vpx_sad32x16 = vpx_sad32x16_avx2; vpx_sad32x16_avg = vpx_sad32x16_avg_sse2; if (flags & HAS_AVX2) vpx_sad32x16_avg = vpx_sad32x16_avg_avx2; vpx_sad32x32 = vpx_sad32x32_sse2; if (flags & HAS_AVX2) vpx_sad32x32 = vpx_sad32x32_avx2; vpx_sad32x32_avg = vpx_sad32x32_avg_sse2; if (flags & HAS_AVX2) vpx_sad32x32_avg = vpx_sad32x32_avg_avx2; vpx_sad32x32x4d = vpx_sad32x32x4d_sse2; if (flags & HAS_AVX2) vpx_sad32x32x4d = vpx_sad32x32x4d_avx2; - vpx_sad32x32x8 = vpx_sad32x32x8_c; - if (flags & HAS_AVX2) vpx_sad32x32x8 = vpx_sad32x32x8_avx2; vpx_sad32x64 = vpx_sad32x64_sse2; if (flags & HAS_AVX2) vpx_sad32x64 = vpx_sad32x64_avx2; vpx_sad32x64_avg = vpx_sad32x64_avg_sse2; if (flags & HAS_AVX2) vpx_sad32x64_avg = vpx_sad32x64_avg_avx2; - vpx_sad4x4x3 = vpx_sad4x4x3_c; - if (flags & HAS_SSE3) vpx_sad4x4x3 = vpx_sad4x4x3_sse3; - vpx_sad4x4x8 = vpx_sad4x4x8_c; - if (flags & HAS_SSE4_1) vpx_sad4x4x8 = vpx_sad4x4x8_sse4_1; vpx_sad64x32 = vpx_sad64x32_sse2; if (flags & HAS_AVX2) vpx_sad64x32 = vpx_sad64x32_avx2; vpx_sad64x32_avg = vpx_sad64x32_avg_sse2; if (flags & HAS_AVX2) vpx_sad64x32_avg = vpx_sad64x32_avg_avx2; vpx_sad64x64 = vpx_sad64x64_sse2; if (flags & HAS_AVX2) vpx_sad64x64 = vpx_sad64x64_avx2; vpx_sad64x64_avg = vpx_sad64x64_avg_sse2; if (flags & HAS_AVX2) vpx_sad64x64_avg = vpx_sad64x64_avg_avx2; vpx_sad64x64x4d = vpx_sad64x64x4d_sse2; if (flags & HAS_AVX2) vpx_sad64x64x4d = vpx_sad64x64x4d_avx2; - vpx_sad8x16x3 = vpx_sad8x16x3_c; - if (flags & HAS_SSE3) vpx_sad8x16x3 = vpx_sad8x16x3_sse3; - vpx_sad8x16x8 = vpx_sad8x16x8_c; - if (flags & HAS_SSE4_1) vpx_sad8x16x8 = vpx_sad8x16x8_sse4_1; - vpx_sad8x8x3 = vpx_sad8x8x3_c; - if (flags & HAS_SSE3) vpx_sad8x8x3 = vpx_sad8x8x3_sse3; - vpx_sad8x8x8 = vpx_sad8x8x8_c; - if (flags & HAS_SSE4_1) vpx_sad8x8x8 = vpx_sad8x8x8_sse4_1; vpx_satd = vpx_satd_sse2; if (flags & HAS_AVX2) vpx_satd = vpx_satd_avx2; vpx_scaled_2d = vpx_scaled_2d_c; if (flags & HAS_SSSE3) vpx_scaled_2d = vpx_scaled_2d_ssse3; vpx_sub_pixel_avg_variance16x16 = vpx_sub_pixel_avg_variance16x16_sse2; diff --git a/media/libvpx/libvpx/README b/media/libvpx/libvpx/README --- a/media/libvpx/libvpx/README +++ b/media/libvpx/libvpx/README @@ -101,10 +101,11 @@ x86-solaris-gcc x86-win32-gcc x86-win32-vs14 x86-win32-vs15 x86-win32-vs16 + x86-win32-vs17 x86_64-android-gcc x86_64-darwin9-gcc x86_64-darwin10-gcc x86_64-darwin11-gcc x86_64-darwin12-gcc @@ -122,10 +123,11 @@ x86_64-solaris-gcc x86_64-win64-gcc x86_64-win64-vs14 x86_64-win64-vs15 x86_64-win64-vs16 + x86_64-win64-vs17 generic-gnu The generic-gnu target, in conjunction with the CROSS environment variable, can be used to cross compile architectures that aren't explicitly listed, if the toolchain is a cross GNU (gcc/binutils) toolchain. Other POSIX toolchains diff --git a/media/libvpx/libvpx/args.c b/media/libvpx/libvpx/args.c --- a/media/libvpx/libvpx/args.c +++ b/media/libvpx/libvpx/args.c @@ -81,10 +81,11 @@ return *arg->argv; } char **argv_dup(int argc, const char **argv) { char **new_argv = malloc((argc + 1) * sizeof(*argv)); + if (!new_argv) return NULL; memcpy(new_argv, argv, argc * sizeof(*argv)); new_argv[argc] = NULL; return new_argv; } diff --git a/media/libvpx/libvpx/build/make/Android.mk b/media/libvpx/libvpx/build/make/Android.mk --- a/media/libvpx/libvpx/build/make/Android.mk +++ b/media/libvpx/libvpx/build/make/Android.mk @@ -164,10 +164,13 @@ -I$(LIBVPX_PATH) \ -I$(ASM_CNV_PATH) \ -I$(ASM_CNV_PATH)/libvpx LOCAL_MODULE := libvpx +LOCAL_LICENSE_KINDS := SPDX-license-identifier-BSD +LOCAL_LICENSE_CONDITIONS := notice +LOCAL_NOTICE_FILE := $(LOCAL_PATH)/../../LICENSE $(LOCAL_PATH)/../../PATENTS ifeq ($(CONFIG_RUNTIME_CPU_DETECT),yes) LOCAL_STATIC_LIBRARIES := cpufeatures endif diff --git a/media/libvpx/libvpx/build/make/Makefile b/media/libvpx/libvpx/build/make/Makefile --- a/media/libvpx/libvpx/build/make/Makefile +++ b/media/libvpx/libvpx/build/make/Makefile @@ -149,10 +149,16 @@ # MIPS $(BUILD_PFX)%_msa.c.d: CFLAGS += -mmsa $(BUILD_PFX)%_msa.c.o: CFLAGS += -mmsa +# LOONGARCH +$(BUILD_PFX)%_lsx.c.d: CFLAGS += -mlsx +$(BUILD_PFX)%_lsx.c.o: CFLAGS += -mlsx +$(BUILD_PFX)%_lasx.c.d: CFLAGS += -mlasx +$(BUILD_PFX)%_lasx.c.o: CFLAGS += -mlasx + $(BUILD_PFX)%.c.d: %.c $(if $(quiet),@echo " [DEP] $@") $(qexec)mkdir -p $(dir $@) $(qexec)$(CC) $(INTERNAL_CFLAGS) $(CFLAGS) -M $< | $(fmt_deps) > $@ diff --git a/media/libvpx/libvpx/build/make/ads2armasm_ms.pl b/media/libvpx/libvpx/build/make/ads2armasm_ms.pl --- a/media/libvpx/libvpx/build/make/ads2armasm_ms.pl +++ b/media/libvpx/libvpx/build/make/ads2armasm_ms.pl @@ -26,11 +26,11 @@ s/^\s*ARM\s*$//; s/AREA\s+\|\|(.*)\|\|/AREA |$1|/; s/qsubaddx/qsax/i; s/qaddsubx/qasx/i; - thumb::FixThumbInstructions($_, 1); + thumb::FixThumbInstructions($_); s/ldrneb/ldrbne/i; s/ldrneh/ldrhne/i; s/^(\s*)ENDP.*/$&\n$1ALIGN 4/; diff --git a/media/libvpx/libvpx/build/make/ads2gas.pl b/media/libvpx/libvpx/build/make/ads2gas.pl --- a/media/libvpx/libvpx/build/make/ads2gas.pl +++ b/media/libvpx/libvpx/build/make/ads2gas.pl @@ -30,209 +30,128 @@ $elf = 0 if ($arg eq "-noelf"); } print "@ This file was created from a .asm file\n"; print "@ using the ads2gas.pl script.\n"; -print "\t.syntax unified\n"; +print ".syntax unified\n"; if ($thumb) { print "\t.thumb\n"; } # Stack of procedure names. @proc_stack = (); while () { - undef $comment; - undef $line; - $comment_char = ";"; - $comment_sub = "@"; - - # Handle comments. - if (/$comment_char/) - { - $comment = ""; - ($line, $comment) = /(.*?)$comment_char(.*)/; - $_ = $line; - } - # Load and store alignment s/@/,:/g; - # Hexadecimal constants prefaced by 0x - s/#&/#0x/g; - - # Convert :OR: to | - s/:OR:/ | /g; - - # Convert :AND: to & - s/:AND:/ & /g; - - # Convert :NOT: to ~ - s/:NOT:/ ~ /g; - - # Convert :SHL: to << - s/:SHL:/ << /g; - - # Convert :SHR: to >> - s/:SHR:/ >> /g; + # Comment character + s/;/@/; # Convert ELSE to .else s/\bELSE\b/.else/g; # Convert ENDIF to .endif s/\bENDIF\b/.endif/g; - # Convert ELSEIF to .elseif - s/\bELSEIF\b/.elseif/g; - - # Convert LTORG to .ltorg - s/\bLTORG\b/.ltorg/g; - - # Convert endfunc to nothing. - s/\bendfunc\b//ig; - - # Convert FUNCTION to nothing. - s/\bFUNCTION\b//g; - s/\bfunction\b//g; - - s/\bENTRY\b//g; - s/\bMSARMASM\b/0/g; - s/^\s+end\s+$//g; - - # Convert IF :DEF:to .if - # gcc doesn't have the ability to do a conditional - # if defined variable that is set by IF :DEF: on - # armasm, so convert it to a normal .if and then - # make sure to define a value elesewhere - if (s/\bIF :DEF:\b/.if /g) - { - s/=/==/g; - } - # Convert IF to .if - if (s/\bIF\b/.if/g) - { + if (s/\bIF\b/.if/g) { s/=+/==/g; } # Convert INCLUDE to .INCLUDE "file" - s/INCLUDE(\s*)(.*)$/.include $1\"$2\"/; - - # Code directive (ARM vs Thumb) - s/CODE([0-9][0-9])/.code $1/; + s/INCLUDE\s?(.*)$/.include \"$1\"/; # No AREA required # But ALIGNs in AREA must be obeyed - s/^\s*AREA.*ALIGN=([0-9])$/.text\n.p2align $1/; + s/^(\s*)\bAREA\b.*ALIGN=([0-9])$/$1.text\n$1.p2align $2/; # If no ALIGN, strip the AREA and align to 4 bytes - s/^\s*AREA.*$/.text\n.p2align 2/; - - # DCD to .word - # This one is for incoming symbols - s/DCD\s+\|(\w*)\|/.long $1/; - - # DCW to .short - s/DCW\s+\|(\w*)\|/.short $1/; - s/DCW(.*)/.short $1/; - - # Constants defined in scope - s/DCD(.*)/.long $1/; - s/DCB(.*)/.byte $1/; + s/^(\s*)\bAREA\b.*$/$1.text\n$1.p2align 2/; - # Make function visible to linker, and make additional symbol with - # prepended underscore + # Make function visible to linker. if ($elf) { - s/EXPORT\s+\|([\$\w]*)\|/.global $1 \n\t.type $1, function/; + s/(\s*)EXPORT\s+\|([\$\w]*)\|/$1.global $2\n$1.type $2, function/; } else { - s/EXPORT\s+\|([\$\w]*)\|/.global $1/; + s/(\s*)EXPORT\s+\|([\$\w]*)\|/$1.global $2/; } - s/IMPORT\s+\|([\$\w]*)\|/.global $1/; - s/EXPORT\s+([\$\w]*)/.global $1/; - s/export\s+([\$\w]*)/.global $1/; - - # No vertical bars required; make additional symbol with prepended - # underscore - s/^\|(\$?\w+)\|/_$1\n\t$1:/g; + # No vertical bars on function names + s/^\|(\$?\w+)\|/$1/g; # Labels need trailing colon -# s/^(\w+)/$1:/ if !/EQU/; - # put the colon at the end of the line in the macro s/^([a-zA-Z_0-9\$]+)/$1:/ if !/EQU/; # ALIGN directive s/\bALIGN\b/.balign/g; if ($thumb) { - # ARM code - we force everything to thumb with the declaration in the header - s/\sARM//g; + # ARM code - we force everything to thumb with the declaration in the + # header + s/\bARM\b//g; } else { # ARM code - s/\sARM/.arm/g; + s/\bARM\b/.arm/g; } # push/pop s/(push\s+)(r\d+)/stmdb sp\!, \{$2\}/g; s/(pop\s+)(r\d+)/ldmia sp\!, \{$2\}/g; - # NEON code - s/(vld1.\d+\s+)(q\d+)/$1\{$2\}/g; - s/(vtbl.\d+\s+[^,]+),([^,]+)/$1,\{$2\}/g; - if ($thumb) { - thumb::FixThumbInstructions($_, 0); + thumb::FixThumbInstructions($_); } # eabi_attributes numerical equivalents can be found in the # "ARM IHI 0045C" document. if ($elf) { # REQUIRE8 Stack is required to be 8-byte aligned - s/\sREQUIRE8/.eabi_attribute 24, 1 \@Tag_ABI_align_needed/g; + s/\bREQUIRE8\b/.eabi_attribute 24, 1 \@Tag_ABI_align_needed/g; # PRESERVE8 Stack 8-byte align is preserved - s/\sPRESERVE8/.eabi_attribute 25, 1 \@Tag_ABI_align_preserved/g; + s/\bPRESERVE8\b/.eabi_attribute 25, 1 \@Tag_ABI_align_preserved/g; } else { - s/\sREQUIRE8//; - s/\sPRESERVE8//; + s/\bREQUIRE8\b//; + s/\bPRESERVE8\b//; } # Use PROC and ENDP to give the symbols a .size directive. # This makes them show up properly in debugging tools like gdb and valgrind. - if (/\bPROC\b/) - { + if (/\bPROC\b/) { my $proc; - /^_([\.0-9A-Z_a-z]\w+)\b/; + # Match the function name so it can be stored in $proc + /^([\.0-9A-Z_a-z]\w+)\b/; $proc = $1; push(@proc_stack, $proc) if ($proc); s/\bPROC\b/@ $&/; } - if (/\bENDP\b/) - { + + if (/\bENDP\b/) { my $proc; s/\bENDP\b/@ $&/; $proc = pop(@proc_stack); - $_ = "\t.size $proc, .-$proc".$_ if ($proc and $elf); + $_ = ".size $proc, .-$proc".$_ if ($proc and $elf); } # EQU directive s/(\S+\s+)EQU(\s+\S+)/.equ $1, $2/; # Begin macro definition if (/\bMACRO\b/) { + # Process next line down, which will be the macro definition $_ = ; s/^/.macro/; - s/\$//g; # remove formal param reference - s/;/@/g; # change comment characters + s/\$//g; # Remove $ from the variables in the declaration } - # For macros, use \ to reference formal params - s/\$/\\/g; # End macro definition - s/\bMEND\b/.endm/; # No need to tell it where to stop assembling + s/\$/\\/g; # Use \ to reference formal parameters + # End macro definition + + s/\bMEND\b/.endm/; # No need to tell it where to stop assembling next if /^\s*END\s*$/; + s/[ \t]+$//; print; - print "$comment_sub$comment\n" if defined $comment; } # Mark that this object doesn't need an executable stack. -printf ("\t.section\t.note.GNU-stack,\"\",\%\%progbits\n") if $elf; +printf (" .section .note.GNU-stack,\"\",\%\%progbits\n") if $elf; diff --git a/media/libvpx/libvpx/build/make/ads2gas_apple.pl b/media/libvpx/libvpx/build/make/ads2gas_apple.pl --- a/media/libvpx/libvpx/build/make/ads2gas_apple.pl +++ b/media/libvpx/libvpx/build/make/ads2gas_apple.pl @@ -18,21 +18,18 @@ # Usage: cat inputfile | perl ads2gas_apple.pl > outputfile # print "@ This file was created from a .asm file\n"; print "@ using the ads2gas_apple.pl script.\n\n"; -print "\t.syntax unified\n"; +print ".syntax unified\n"; -my %register_aliases; my %macro_aliases; my @mapping_list = ("\$0", "\$1", "\$2", "\$3", "\$4", "\$5", "\$6", "\$7", "\$8", "\$9"); my @incoming_array; -my @imported_functions; - # Perl trim function to remove whitespace from the start and end of the string sub trim($) { my $string = shift; $string =~ s/^\s+//; @@ -44,159 +41,74 @@ { # Load and store alignment s/@/,:/g; # Comment character - s/;/ @/g; - - # Hexadecimal constants prefaced by 0x - s/#&/#0x/g; - - # Convert :OR: to | - s/:OR:/ | /g; - - # Convert :AND: to & - s/:AND:/ & /g; - - # Convert :NOT: to ~ - s/:NOT:/ ~ /g; - - # Convert :SHL: to << - s/:SHL:/ << /g; - - # Convert :SHR: to >> - s/:SHR:/ >> /g; + s/;/@/; # Convert ELSE to .else s/\bELSE\b/.else/g; # Convert ENDIF to .endif s/\bENDIF\b/.endif/g; - # Convert ELSEIF to .elseif - s/\bELSEIF\b/.elseif/g; - - # Convert LTORG to .ltorg - s/\bLTORG\b/.ltorg/g; - - # Convert IF :DEF:to .if - # gcc doesn't have the ability to do a conditional - # if defined variable that is set by IF :DEF: on - # armasm, so convert it to a normal .if and then - # make sure to define a value elesewhere - if (s/\bIF :DEF:\b/.if /g) - { - s/=/==/g; - } - # Convert IF to .if - if (s/\bIF\b/.if/g) - { - s/=/==/g; + if (s/\bIF\b/.if/g) { + s/=+/==/g; } # Convert INCLUDE to .INCLUDE "file" - s/INCLUDE(\s*)(.*)$/.include $1\"$2\"/; - - # Code directive (ARM vs Thumb) - s/CODE([0-9][0-9])/.code $1/; + s/INCLUDE\s?(.*)$/.include \"$1\"/; # No AREA required # But ALIGNs in AREA must be obeyed - s/^\s*AREA.*ALIGN=([0-9])$/.text\n.p2align $1/; + s/^(\s*)\bAREA\b.*ALIGN=([0-9])$/$1.text\n$1.p2align $2/; # If no ALIGN, strip the AREA and align to 4 bytes - s/^\s*AREA.*$/.text\n.p2align 2/; - - # DCD to .word - # This one is for incoming symbols - s/DCD\s+\|(\w*)\|/.long $1/; + s/^(\s*)\bAREA\b.*$/$1.text\n$1.p2align 2/; - # DCW to .short - s/DCW\s+\|(\w*)\|/.short $1/; - s/DCW(.*)/.short $1/; - - # Constants defined in scope - s/DCD(.*)/.long $1/; - s/DCB(.*)/.byte $1/; - - # Make function visible to linker, and make additional symbol with - # prepended underscore - s/EXPORT\s+\|([\$\w]*)\|/.globl _$1\n\t.globl $1/; + # Make function visible to linker. + s/EXPORT\s+\|([\$\w]*)\|/.globl _$1/; - # Prepend imported functions with _ - if (s/IMPORT\s+\|([\$\w]*)\|/.globl $1/) - { - $function = trim($1); - push(@imported_functions, $function); - } + # No vertical bars on function names + s/^\|(\$?\w+)\|/$1/g; - foreach $function (@imported_functions) - { - s/$function/_$function/; - } + # Labels and functions need a leading underscore and trailing colon + s/^([a-zA-Z_0-9\$]+)/_$1:/ if !/EQU/; - # No vertical bars required; make additional symbol with prepended - # underscore - s/^\|(\$?\w+)\|/_$1\n\t$1:/g; - - # Labels need trailing colon -# s/^(\w+)/$1:/ if !/EQU/; - # put the colon at the end of the line in the macro - s/^([a-zA-Z_0-9\$]+)/$1:/ if !/EQU/; + # Branches need to call the correct, underscored, function + s/^(\s+b[egln]?[teq]?\s+)([a-zA-Z_0-9\$]+)/$1 _$2/ if !/EQU/; # ALIGN directive s/\bALIGN\b/.balign/g; # Strip ARM - s/\sARM/@ ARM/g; + s/\s+ARM//; # Strip REQUIRE8 - #s/\sREQUIRE8/@ REQUIRE8/g; - s/\sREQUIRE8/@ /g; + s/\s+REQUIRE8//; # Strip PRESERVE8 - s/\sPRESERVE8/@ PRESERVE8/g; + s/\s+PRESERVE8//; # Strip PROC and ENDPROC - s/\bPROC\b/@/g; - s/\bENDP\b/@/g; + s/\bPROC\b//g; + s/\bENDP\b//g; # EQU directive - s/(.*)EQU(.*)/.set $1, $2/; + s/(\S+\s+)EQU(\s+\S+)/.equ $1, $2/; # Begin macro definition - if (/\bMACRO\b/) - { + if (/\bMACRO\b/) { # Process next line down, which will be the macro definition $_ = ; - - $trimmed = trim($_); - - # remove commas that are separating list - $trimmed =~ s/,//g; - - # string to array - @incoming_array = split(/\s+/, $trimmed); - - print ".macro @incoming_array[0]\n"; - - # remove the first element, as that is the name of the macro - shift (@incoming_array); - - @macro_aliases{@incoming_array} = @mapping_list; - - next; + s/^/.macro/; + s/\$//g; # Remove $ from the variables in the declaration } - while (($key, $value) = each(%macro_aliases)) - { - $key =~ s/\$/\\\$/; - s/$key\b/$value/g; - } + s/\$/\\/g; # Use \ to reference formal parameters + # End macro definition - # For macros, use \ to reference formal params -# s/\$/\\/g; # End macro definition - s/\bMEND\b/.endm/; # No need to tell it where to stop assembling + s/\bMEND\b/.endm/; # No need to tell it where to stop assembling next if /^\s*END\s*$/; - + s/[ \t]+$//; print; } diff --git a/media/libvpx/libvpx/build/make/configure.sh b/media/libvpx/libvpx/build/make/configure.sh --- a/media/libvpx/libvpx/build/make/configure.sh +++ b/media/libvpx/libvpx/build/make/configure.sh @@ -447,10 +447,21 @@ disable_feature avx512 RTCD_OPTIONS="${RTCD_OPTIONS}--disable-avx512 " fi } +check_inline_asm() { + log check_inline_asm "$@" + name="$1" + code="$2" + shift 2 + disable_feature $name + check_cc "$@" <> config.mk echo "TOOLCHAIN := ${toolchain}" >> config.mk @@ -764,19 +775,25 @@ tgt_isa=mips64 ;; *mips32el*) tgt_isa=mips32 ;; + loongarch32*) + tgt_isa=loongarch32 + ;; + loongarch64*) + tgt_isa=loongarch64 + ;; esac # detect tgt_os case "$gcctarget" in *darwin1[0-9]*) tgt_isa=x86_64 tgt_os=`echo $gcctarget | sed 's/.*\(darwin1[0-9]\).*/\1/'` ;; - *darwin20*) + *darwin2[0-1]*) tgt_isa=`uname -m` tgt_os=`echo $gcctarget | sed 's/.*\(darwin2[0-9]\).*/\1/'` ;; x86_64*mingw32*) tgt_os=win64 @@ -832,10 +849,15 @@ enable_feature mips ;; ppc*) enable_feature ppc ;; + loongarch*) + soft_enable lsx + soft_enable lasx + enable_feature loongarch + ;; esac # PIC is probably what we want when building shared libs enabled shared && soft_enable pic @@ -916,13 +938,13 @@ ;; *-darwin19-*) add_cflags "-mmacosx-version-min=10.15" add_ldflags "-mmacosx-version-min=10.15" ;; - *-darwin20-*) - add_cflags "-mmacosx-version-min=10.16 -arch ${toolchain%%-*}" - add_ldflags "-mmacosx-version-min=10.16 -arch ${toolchain%%-*}" + *-darwin2[0-1]-*) + add_cflags "-arch ${toolchain%%-*}" + add_ldflags "-arch ${toolchain%%-*}" ;; *-iphonesimulator-*) add_cflags "-miphoneos-version-min=${IOS_VERSION_MIN}" add_ldflags "-miphoneos-version-min=${IOS_VERSION_MIN}" iossim_sdk_dir="$(show_darwin_sdk_path iphonesimulator)" @@ -1417,10 +1439,19 @@ *) log "Warning: Unknown os $tgt_os while setting up $AS flags" ;; esac ;; + loongarch*) + link_with_cc=gcc + setup_gnu_toolchain + + enabled lsx && check_inline_asm lsx '"vadd.b $vr0, $vr1, $vr1"' + enabled lsx && soft_enable runtime_cpu_detect + enabled lasx && check_inline_asm lasx '"xvadd.b $xr0, $xr1, $xr1"' + enabled lasx && soft_enable runtime_cpu_detect + ;; *-gcc|generic-gnu) link_with_cc=gcc enable_feature gcc setup_gnu_toolchain ;; @@ -1519,10 +1550,26 @@ fi fi ;; esac + # only for LOONGARCH platforms + case ${toolchain} in + loongarch*) + if enabled big_endian; then + if enabled lsx; then + echo "lsx optimizations are available only for little endian platforms" + disable_feature lsx + fi + if enabled lasx; then + echo "lasx optimizations are available only for little endian platforms" + disable_feature lasx + fi + fi + ;; + esac + # glibc needs these if enabled linux; then add_cflags -D_LARGEFILE_SOURCE add_cflags -D_FILE_OFFSET_BITS=64 fi diff --git a/media/libvpx/libvpx/build/make/gen_msvs_sln.sh b/media/libvpx/libvpx/build/make/gen_msvs_sln.sh --- a/media/libvpx/libvpx/build/make/gen_msvs_sln.sh +++ b/media/libvpx/libvpx/build/make/gen_msvs_sln.sh @@ -23,11 +23,11 @@ files. Options: --help Print this message --out=outfile Redirect output to a file - --ver=version Version (14-16) of visual studio to generate for + --ver=version Version (14-17) of visual studio to generate for --target=isa-os-cc Target specifier EOF exit 1 } @@ -217,10 +217,11 @@ vs_ver="$optval" case $optval in 14) vs_year=2015 ;; 15) vs_year=2017 ;; 16) vs_year=2019 ;; + 17) vs_year=2022 ;; *) die Unrecognized Visual Studio Version in $opt ;; esac ;; --target=*) target="${optval}" ;; @@ -230,11 +231,11 @@ esac done outfile=${outfile:-/dev/stdout} mkoutfile=${mkoutfile:-/dev/stdout} case "${vs_ver}" in - 1[4-6]) + 1[4-7]) # VS has used Format Version 12.00 continuously since vs11. sln_vers="12.00" sln_vers_str="Visual Studio ${vs_year}" ;; esac diff --git a/media/libvpx/libvpx/build/make/gen_msvs_vcxproj.sh b/media/libvpx/libvpx/build/make/gen_msvs_vcxproj.sh --- a/media/libvpx/libvpx/build/make/gen_msvs_vcxproj.sh +++ b/media/libvpx/libvpx/build/make/gen_msvs_vcxproj.sh @@ -168,11 +168,11 @@ --enable-werror) werror=true ;; --ver=*) vs_ver="$optval" case "$optval" in - 1[4-6]) + 1[4-7]) ;; *) die Unrecognized Visual Studio Version in $opt ;; esac ;; @@ -342,10 +342,13 @@ tag_content PlatformToolset v141 fi if [ "$vs_ver" = "16" ]; then tag_content PlatformToolset v142 fi + if [ "$vs_ver" = "17" ]; then + tag_content PlatformToolset v143 + fi tag_content CharacterSet Unicode if [ "$config" = "Release" ]; then tag_content WholeProgramOptimization true fi close_tag PropertyGroup diff --git a/media/libvpx/libvpx/build/make/rtcd.pl b/media/libvpx/libvpx/build/make/rtcd.pl --- a/media/libvpx/libvpx/build/make/rtcd.pl +++ b/media/libvpx/libvpx/build/make/rtcd.pl @@ -385,10 +385,41 @@ #endif EOF common_bottom; } +sub loongarch() { + determine_indirection("c", @ALL_ARCHS); + + # Assign the helper variable for each enabled extension + foreach my $opt (@ALL_ARCHS) { + my $opt_uc = uc $opt; + eval "\$have_${opt}=\"flags & HAS_${opt_uc}\""; + } + + common_top; + print <) { + if (/HAVE_LSX=yes/) { + @ALL_ARCHS = filter("$opts{arch}", qw/lsx/); + last; + } + if (/HAVE_LASX=yes/) { + @ALL_ARCHS = filter("$opts{arch}", qw/lasx/); + last; + } + } + close CONFIG_FILE; + loongarch; } else { unoptimized; } __END__ diff --git a/media/libvpx/libvpx/build/make/thumb.pm b/media/libvpx/libvpx/build/make/thumb.pm --- a/media/libvpx/libvpx/build/make/thumb.pm +++ b/media/libvpx/libvpx/build/make/thumb.pm @@ -9,15 +9,12 @@ ## be found in the AUTHORS file in the root of the source tree. ## package thumb; -sub FixThumbInstructions($$) +sub FixThumbInstructions($) { - my $short_branches = $_[1]; - my $branch_shift_offset = $short_branches ? 1 : 0; - # Write additions with shifts, such as "add r10, r11, lsl #8", # in three operand form, "add r10, r10, r11, lsl #8". s/(add\s+)(r\d+),\s*(r\d+),\s*(lsl #\d+)/$1$2, $2, $3, $4/g; # Convert additions with a non-constant shift into a sequence diff --git a/media/libvpx/libvpx/configure b/media/libvpx/libvpx/configure --- a/media/libvpx/libvpx/configure +++ b/media/libvpx/libvpx/configure @@ -98,10 +98,11 @@ # all_platforms is a list of all supported target platforms. Maintain # alphabetically by architecture, generic-gnu last. all_platforms="${all_platforms} arm64-android-gcc" all_platforms="${all_platforms} arm64-darwin-gcc" all_platforms="${all_platforms} arm64-darwin20-gcc" +all_platforms="${all_platforms} arm64-darwin21-gcc" all_platforms="${all_platforms} arm64-linux-gcc" all_platforms="${all_platforms} arm64-win64-gcc" all_platforms="${all_platforms} arm64-win64-vs15" all_platforms="${all_platforms} armv7-android-gcc" #neon Cortex-A8 all_platforms="${all_platforms} armv7-darwin-gcc" #neon Cortex-A8 @@ -111,10 +112,12 @@ all_platforms="${all_platforms} armv7-win32-gcc" all_platforms="${all_platforms} armv7-win32-vs14" all_platforms="${all_platforms} armv7-win32-vs15" all_platforms="${all_platforms} armv7s-darwin-gcc" all_platforms="${all_platforms} armv8-linux-gcc" +all_platforms="${all_platforms} loongarch32-linux-gcc" +all_platforms="${all_platforms} loongarch64-linux-gcc" all_platforms="${all_platforms} mips32-linux-gcc" all_platforms="${all_platforms} mips64-linux-gcc" all_platforms="${all_platforms} ppc64le-linux-gcc" all_platforms="${all_platforms} sparc-solaris-gcc" all_platforms="${all_platforms} x86-android-gcc" @@ -137,10 +140,11 @@ all_platforms="${all_platforms} x86-solaris-gcc" all_platforms="${all_platforms} x86-win32-gcc" all_platforms="${all_platforms} x86-win32-vs14" all_platforms="${all_platforms} x86-win32-vs15" all_platforms="${all_platforms} x86-win32-vs16" +all_platforms="${all_platforms} x86-win32-vs17" all_platforms="${all_platforms} x86_64-android-gcc" all_platforms="${all_platforms} x86_64-darwin9-gcc" all_platforms="${all_platforms} x86_64-darwin10-gcc" all_platforms="${all_platforms} x86_64-darwin11-gcc" all_platforms="${all_platforms} x86_64-darwin12-gcc" @@ -150,18 +154,20 @@ all_platforms="${all_platforms} x86_64-darwin16-gcc" all_platforms="${all_platforms} x86_64-darwin17-gcc" all_platforms="${all_platforms} x86_64-darwin18-gcc" all_platforms="${all_platforms} x86_64-darwin19-gcc" all_platforms="${all_platforms} x86_64-darwin20-gcc" +all_platforms="${all_platforms} x86_64-darwin21-gcc" all_platforms="${all_platforms} x86_64-iphonesimulator-gcc" all_platforms="${all_platforms} x86_64-linux-gcc" all_platforms="${all_platforms} x86_64-linux-icc" all_platforms="${all_platforms} x86_64-solaris-gcc" all_platforms="${all_platforms} x86_64-win64-gcc" all_platforms="${all_platforms} x86_64-win64-vs14" all_platforms="${all_platforms} x86_64-win64-vs15" all_platforms="${all_platforms} x86_64-win64-vs16" +all_platforms="${all_platforms} x86_64-win64-vs17" all_platforms="${all_platforms} generic-gnu" # all_targets is a list of all targets that can be configured # note that these should be in dependency order for now. all_targets="libs examples tools docs" @@ -233,10 +239,11 @@ arm mips x86 x86_64 ppc + loongarch " ARCH_EXT_LIST_X86=" mmx sse sse2 @@ -248,10 +255,12 @@ avx512 " ARCH_EXT_LIST_LOONGSON=" mmi + lsx + lasx " ARCH_EXT_LIST=" neon neon_asm @@ -618,11 +627,14 @@ if enabled gcc; then enabled werror && check_add_cflags -Werror check_add_cflags -Wall check_add_cflags -Wdeclaration-after-statement check_add_cflags -Wdisabled-optimization + check_add_cflags -Wextra-semi + check_add_cflags -Wextra-semi-stmt check_add_cflags -Wfloat-conversion + check_add_cflags -Wformat=2 check_add_cflags -Wparentheses-equality check_add_cflags -Wpointer-arith check_add_cflags -Wtype-limits check_add_cflags -Wcast-qual check_add_cflags -Wvla diff --git a/media/libvpx/libvpx/examples/postproc.c b/media/libvpx/libvpx/examples/postproc.c --- a/media/libvpx/libvpx/examples/postproc.c +++ b/media/libvpx/libvpx/examples/postproc.c @@ -107,11 +107,11 @@ } else if (frame_cnt % 30 == 16) { vp8_postproc_cfg_t pp = { VP8_DEBLOCK | VP8_DEMACROBLOCK | VP8_MFQE, 4, 0 }; if (vpx_codec_control(&codec, VP8_SET_POSTPROC, &pp)) die_codec(&codec, "Failed to turn on postproc."); - }; + } // Decode the frame with 15ms deadline if (vpx_codec_decode(&codec, frame, (unsigned int)frame_size, NULL, 15000)) die_codec(&codec, "Failed to decode frame"); diff --git a/media/libvpx/libvpx/examples/svc_encodeframe.c b/media/libvpx/libvpx/examples/svc_encodeframe.c --- a/media/libvpx/libvpx/examples/svc_encodeframe.c +++ b/media/libvpx/libvpx/examples/svc_encodeframe.c @@ -19,10 +19,11 @@ #include #include #include #include #define VPX_DISABLE_CTRL_TYPECHECKS 1 +#include "../tools_common.h" #include "./vpx_config.h" #include "./svc_context.h" #include "vpx/vp8cx.h" #include "vpx/vpx_encoder.h" #include "vpx_mem/vpx_mem.h" @@ -93,12 +94,13 @@ static const SvcInternal_t *get_const_svc_internal(const SvcContext *svc_ctx) { if (svc_ctx == NULL) return NULL; return (const SvcInternal_t *)svc_ctx->internal; } -static int svc_log(SvcContext *svc_ctx, SVC_LOG_LEVEL level, const char *fmt, - ...) { +static VPX_TOOLS_FORMAT_PRINTF(3, 4) int svc_log(SvcContext *svc_ctx, + SVC_LOG_LEVEL level, + const char *fmt, ...) { char buf[512]; int retval = 0; va_list ap; if (level > svc_ctx->log_level) { @@ -262,11 +264,11 @@ for (i = 0; i < svc_ctx->spatial_layers; ++i) alt_ref_enabled += si->enable_auto_alt_ref[i]; if (alt_ref_enabled > REF_FRAMES - svc_ctx->spatial_layers) { svc_log(svc_ctx, SVC_LOG_ERROR, "svc: auto alt ref: Maxinum %d(REF_FRAMES - layers) layers could" - "enabled auto alt reference frame, but % layers are enabled\n", + "enabled auto alt reference frame, but %d layers are enabled\n", REF_FRAMES - svc_ctx->spatial_layers, alt_ref_enabled); res = VPX_CODEC_INVALID_PARAM; } return res; @@ -454,14 +456,15 @@ if (svc_ctx->temporal_layers < 1) svc_ctx->temporal_layers = 1; if (svc_ctx->temporal_layers > VPX_TS_MAX_LAYERS) svc_ctx->temporal_layers = VPX_TS_MAX_LAYERS; if (svc_ctx->temporal_layers * svc_ctx->spatial_layers > VPX_MAX_LAYERS) { - svc_log(svc_ctx, SVC_LOG_ERROR, - "spatial layers * temporal layers exceeds the maximum number of " - "allowed layers of %d\n", - svc_ctx->spatial_layers * svc_ctx->temporal_layers, VPX_MAX_LAYERS); + svc_log( + svc_ctx, SVC_LOG_ERROR, + "spatial layers * temporal layers (%d) exceeds the maximum number of " + "allowed layers of %d\n", + svc_ctx->spatial_layers * svc_ctx->temporal_layers, VPX_MAX_LAYERS); return VPX_CODEC_INVALID_PARAM; } res = assign_layer_bitrates(svc_ctx, enc_cfg); if (res != VPX_CODEC_OK) { svc_log(svc_ctx, SVC_LOG_ERROR, diff --git a/media/libvpx/libvpx/examples/twopass_encoder.c b/media/libvpx/libvpx/examples/twopass_encoder.c --- a/media/libvpx/libvpx/examples/twopass_encoder.c +++ b/media/libvpx/libvpx/examples/twopass_encoder.c @@ -82,10 +82,11 @@ if (pkt->kind == VPX_CODEC_STATS_PKT) { const uint8_t *const pkt_buf = pkt->data.twopass_stats.buf; const size_t pkt_size = pkt->data.twopass_stats.sz; stats->buf = realloc(stats->buf, stats->sz + pkt_size); + if (!stats->buf) die("Failed to reallocate stats buffer."); memcpy((uint8_t *)stats->buf + stats->sz, pkt_buf, pkt_size); stats->sz += pkt_size; } } @@ -219,11 +220,11 @@ if (w <= 0 || h <= 0 || (w % 2) != 0 || (h % 2) != 0) die("Invalid frame size: %dx%d", w, h); if (!vpx_img_alloc(&raw, VPX_IMG_FMT_I420, w, h, 1)) - die("Failed to allocate image", w, h); + die("Failed to allocate image (%dx%d)", w, h); printf("Using %s\n", vpx_codec_iface_name(encoder->codec_interface())); // Configuration res = vpx_codec_enc_config_default(encoder->codec_interface(), &cfg, 0); diff --git a/media/libvpx/libvpx/examples/vp8_multi_resolution_encoder.c b/media/libvpx/libvpx/examples/vp8_multi_resolution_encoder.c --- a/media/libvpx/libvpx/examples/vp8_multi_resolution_encoder.c +++ b/media/libvpx/libvpx/examples/vp8_multi_resolution_encoder.c @@ -350,11 +350,11 @@ width = (int)strtol(argv[1], NULL, 0); height = (int)strtol(argv[2], NULL, 0); framerate = (int)strtol(argv[3], NULL, 0); if (width < 16 || width % 2 || height < 16 || height % 2) - die("Invalid resolution: %ldx%ld", width, height); + die("Invalid resolution: %dx%d", width, height); /* Open input video file for encoding */ if (!(infile = fopen(argv[4], "rb"))) die("Failed to open %s for reading", argv[4]); @@ -378,11 +378,11 @@ for (i = 0; i < NUM_ENCODERS; i++) { num_temporal_layers[i] = (int)strtol(argv[2 * NUM_ENCODERS + 5 + i], NULL, 0); if (num_temporal_layers[i] < 1 || num_temporal_layers[i] > 3) die("Invalid temporal layers: %d, Must be 1, 2, or 3. \n", - num_temporal_layers); + num_temporal_layers[i]); } /* Open file to write out each spatially downsampled input stream. */ for (i = 0; i < NUM_ENCODERS - 1; i++) { // Highest resoln is encoder 0. @@ -466,11 +466,11 @@ cfg[2].g_threads = 1; /* Allocate image for each encoder */ for (i = 0; i < NUM_ENCODERS; i++) if (!vpx_img_alloc(&raw[i], VPX_IMG_FMT_I420, cfg[i].g_w, cfg[i].g_h, 32)) - die("Failed to allocate image", cfg[i].g_w, cfg[i].g_h); + die("Failed to allocate image (%dx%d)", cfg[i].g_w, cfg[i].g_h); if (raw[0].stride[VPX_PLANE_Y] == (int)raw[0].d_w) read_frame_p = mulres_read_frame; else read_frame_p = mulres_read_frame_by_row; diff --git a/media/libvpx/libvpx/examples/vp9_spatial_svc_encoder.c b/media/libvpx/libvpx/examples/vp9_spatial_svc_encoder.c --- a/media/libvpx/libvpx/examples/vp9_spatial_svc_encoder.c +++ b/media/libvpx/libvpx/examples/vp9_spatial_svc_encoder.c @@ -220,10 +220,14 @@ app_input->frames_to_code = default_frames_to_code; app_input->frames_to_skip = default_frames_to_skip; // process command line options argv = argv_dup(argc - 1, argv_ + 1); + if (!argv) { + fprintf(stderr, "Error allocating argument list\n"); + exit(EXIT_FAILURE); + } for (argi = argj = argv; (*argj = *argi); argi += arg.argv_step) { arg.argv_step = 1; if (arg_match(&arg, &frames_arg, argi)) { app_input->frames_to_code = arg_parse_uint(&arg); @@ -355,10 +359,12 @@ open_input_file(&app_input->input_ctx); if (app_input->input_ctx.file_type == FILE_TYPE_Y4M) { enc_cfg->g_w = app_input->input_ctx.width; enc_cfg->g_h = app_input->input_ctx.height; + enc_cfg->g_timebase.den = app_input->input_ctx.framerate.numerator; + enc_cfg->g_timebase.num = app_input->input_ctx.framerate.denominator; } if (enc_cfg->g_w < 16 || enc_cfg->g_w % 2 || enc_cfg->g_h < 16 || enc_cfg->g_h % 2) die("Invalid resolution: %d x %d\n", enc_cfg->g_w, enc_cfg->g_h); @@ -577,11 +583,12 @@ ref_frame_config->gld_fb_idx[sl] = 0; } ref_frame_config->alt_fb_idx[sl] = 0; } else if (tl == 1) { ref_frame_config->lst_fb_idx[sl] = sl; - ref_frame_config->gld_fb_idx[sl] = num_spatial_layers + sl - 1; + ref_frame_config->gld_fb_idx[sl] = + (sl == 0) ? 0 : num_spatial_layers + sl - 1; ref_frame_config->alt_fb_idx[sl] = num_spatial_layers + sl; } // Set the reference and update flags. if (!tl) { if (!sl) { diff --git a/media/libvpx/libvpx/examples/vpx_temporal_svc_encoder.c b/media/libvpx/libvpx/examples/vpx_temporal_svc_encoder.c --- a/media/libvpx/libvpx/examples/vpx_temporal_svc_encoder.c +++ b/media/libvpx/libvpx/examples/vpx_temporal_svc_encoder.c @@ -28,11 +28,11 @@ #include "../tools_common.h" #include "../video_writer.h" #define ROI_MAP 0 -#define zero(Dest) memset(&(Dest), 0, sizeof(Dest)); +#define zero(Dest) memset(&(Dest), 0, sizeof(Dest)) static const char *exec_name; void usage_exit(void) { exit(EXIT_FAILURE); } @@ -238,10 +238,42 @@ roi->roi_map[i * roi->cols + j] = 1; } } } } + +static void set_roi_skip_map(vpx_codec_enc_cfg_t *cfg, vpx_roi_map_t *roi, + int *skip_map, int *prev_mask_map, int frame_num) { + const int block_size = 8; + unsigned int i, j; + roi->rows = (cfg->g_h + block_size - 1) / block_size; + roi->cols = (cfg->g_w + block_size - 1) / block_size; + zero(roi->skip); + zero(roi->delta_q); + zero(roi->delta_lf); + memset(roi->ref_frame, -1, sizeof(roi->ref_frame)); + roi->ref_frame[1] = 1; + // Use segment 3 for skip. + roi->skip[3] = 1; + roi->roi_map = + (uint8_t *)calloc(roi->rows * roi->cols, sizeof(*roi->roi_map)); + for (i = 0; i < roi->rows; ++i) { + for (j = 0; j < roi->cols; ++j) { + const int idx = i * roi->cols + j; + // Use segment 3 for skip. + // prev_mask_map keeps track of blocks that have been stably on segment 3 + // for the past 10 frames. Only skip when the block is on segment 3 in + // both current map and prev_mask_map. + if (skip_map[idx] == 1 && prev_mask_map[idx] == 1) roi->roi_map[idx] = 3; + // Reset it every 10 frames so it doesn't propagate for too many frames. + if (frame_num % 10 == 0) + prev_mask_map[idx] = skip_map[idx]; + else if (prev_mask_map[idx] == 1 && skip_map[idx] == 0) + prev_mask_map[idx] = 0; + } + } +} #endif // Temporal scaling parameters: // NOTE: The 3 prediction frames cannot be used interchangeably due to // differences in the way they are handled throughout the code. The @@ -572,10 +604,27 @@ break; } } } +#if ROI_MAP +static void read_mask(FILE *mask_file, int *seg_map) { + int mask_rows, mask_cols, i, j; + int *map_start = seg_map; + fscanf(mask_file, "%d %d\n", &mask_cols, &mask_rows); + for (i = 0; i < mask_rows; i++) { + for (j = 0; j < mask_cols; j++) { + fscanf(mask_file, "%d ", &seg_map[j]); + // reverse the bit + seg_map[j] = 1 - seg_map[j]; + } + seg_map += mask_cols; + } + seg_map = map_start; +} +#endif + int main(int argc, char **argv) { VpxVideoWriter *outfile[VPX_TS_MAX_LAYERS] = { NULL }; vpx_codec_ctx_t codec; vpx_codec_enc_cfg_t cfg; int frame_cnt = 0; @@ -611,11 +660,18 @@ const int min_args = min_args_base; #endif // CONFIG_VP9_HIGHBITDEPTH double sum_bitrate = 0.0; double sum_bitrate2 = 0.0; double framerate = 30.0; - +#if ROI_MAP + FILE *mask_file = NULL; + int block_size = 8; + int mask_rows = 0; + int mask_cols = 0; + int *mask_map; + int *prev_mask_map; +#endif zero(rc.layer_target_bitrate); memset(&layer_id, 0, sizeof(vpx_svc_layer_id_t)); memset(&input_ctx, 0, sizeof(input_ctx)); /* Setup default input stream settings */ input_ctx.framerate.numerator = 30; @@ -655,13 +711,19 @@ layering_mode = (int)strtol(argv[12], NULL, 0); if (layering_mode < 0 || layering_mode > 13) { die("Invalid layering mode (0..12) %s", argv[12]); } +#if ROI_MAP + if (argc != min_args + mode_to_num_layers[layering_mode] + 1) { + die("Invalid number of arguments"); + } +#else if (argc != min_args + mode_to_num_layers[layering_mode]) { die("Invalid number of arguments"); } +#endif input_ctx.filename = argv[1]; open_input_file(&input_ctx); #if CONFIG_VP9_HIGHBITDEPTH @@ -685,18 +747,18 @@ if (input_ctx.file_type != FILE_TYPE_Y4M) { if (!vpx_img_alloc( &raw, bit_depth == VPX_BITS_8 ? VPX_IMG_FMT_I420 : VPX_IMG_FMT_I42016, width, height, 32)) { - die("Failed to allocate image", width, height); + die("Failed to allocate image (%dx%d)", width, height); } } #else // Y4M reader has its own allocation. if (input_ctx.file_type != FILE_TYPE_Y4M) { if (!vpx_img_alloc(&raw, VPX_IMG_FMT_I420, width, height, 32)) { - die("Failed to allocate image", width, height); + die("Failed to allocate image (%dx%d)", width, height); } } #endif // CONFIG_VP9_HIGHBITDEPTH // Populate encoder configuration. @@ -815,21 +877,27 @@ #else if (vpx_codec_enc_init(&codec, encoder->codec_interface(), &cfg, 0)) #endif // CONFIG_VP9_HIGHBITDEPTH die("Failed to initialize encoder"); +#if ROI_MAP + mask_rows = (cfg.g_h + block_size - 1) / block_size; + mask_cols = (cfg.g_w + block_size - 1) / block_size; + mask_map = (int *)calloc(mask_rows * mask_cols, sizeof(*mask_map)); + prev_mask_map = (int *)calloc(mask_rows * mask_cols, sizeof(*mask_map)); +#endif + if (strncmp(encoder->name, "vp8", 3) == 0) { vpx_codec_control(&codec, VP8E_SET_CPUUSED, -speed); vpx_codec_control(&codec, VP8E_SET_NOISE_SENSITIVITY, kVp8DenoiserOff); vpx_codec_control(&codec, VP8E_SET_STATIC_THRESHOLD, 1); vpx_codec_control(&codec, VP8E_SET_GF_CBR_BOOST_PCT, 0); #if ROI_MAP set_roi_map(encoder->name, &cfg, &roi); if (vpx_codec_control(&codec, VP8E_SET_ROI_MAP, &roi)) die_codec(&codec, "Failed to set ROI map"); #endif - } else if (strncmp(encoder->name, "vp9", 3) == 0) { vpx_svc_extra_cfg_t svc_params; memset(&svc_params, 0, sizeof(svc_params)); vpx_codec_control(&codec, VP9E_SET_POSTENCODE_DROP, 0); vpx_codec_control(&codec, VP9E_SET_DISABLE_OVERSHOOT_MAXQ_CBR, 0); @@ -841,16 +909,11 @@ vpx_codec_control(&codec, VP9E_SET_NOISE_SENSITIVITY, kVp9DenoiserOff); vpx_codec_control(&codec, VP8E_SET_STATIC_THRESHOLD, 1); vpx_codec_control(&codec, VP9E_SET_TUNE_CONTENT, 0); vpx_codec_control(&codec, VP9E_SET_TILE_COLUMNS, get_msb(cfg.g_threads)); vpx_codec_control(&codec, VP9E_SET_DISABLE_LOOPFILTER, 0); -#if ROI_MAP - set_roi_map(encoder->name, &cfg, &roi); - if (vpx_codec_control(&codec, VP9E_SET_ROI_MAP, &roi)) - die_codec(&codec, "Failed to set ROI map"); - vpx_codec_control(&codec, VP9E_SET_AQ_MODE, 0); -#endif + if (cfg.g_threads > 1) vpx_codec_control(&codec, VP9E_SET_ROW_MT, 1); else vpx_codec_control(&codec, VP9E_SET_ROW_MT, 0); if (vpx_codec_control(&codec, VP9E_SET_SVC, layering_mode > 0 ? 1 : 0)) @@ -879,10 +942,13 @@ frame_avail = 1; while (frame_avail || got_data) { struct vpx_usec_timer timer; vpx_codec_iter_t iter = NULL; const vpx_codec_cx_pkt_t *pkt; +#if ROI_MAP + char mask_file_name[255]; +#endif // Update the temporal layer_id. No spatial layers in this test. layer_id.spatial_layer_id = 0; layer_id.temporal_layer_id = cfg.ts_layer_id[frame_cnt % cfg.ts_periodicity]; layer_id.temporal_layer_id_per_spatial[0] = layer_id.temporal_layer_id; @@ -892,10 +958,23 @@ vpx_codec_control(&codec, VP8E_SET_TEMPORAL_LAYER_ID, layer_id.temporal_layer_id); } flags = layer_flags[frame_cnt % flag_periodicity]; if (layering_mode == 0) flags = 0; +#if ROI_MAP + snprintf(mask_file_name, sizeof(mask_file_name), "%s%05d.txt", + argv[argc - 1], frame_cnt); + mask_file = fopen(mask_file_name, "r"); + if (mask_file != NULL) { + read_mask(mask_file, mask_map); + fclose(mask_file); + // set_roi_map(encoder->name, &cfg, &roi); + set_roi_skip_map(&cfg, &roi, mask_map, prev_mask_map, frame_cnt); + if (vpx_codec_control(&codec, VP9E_SET_ROI_MAP, &roi)) + die_codec(&codec, "Failed to set ROI map"); + } +#endif frame_avail = read_frame(&input_ctx, &raw); if (frame_avail) ++rc.layer_input_frames[layer_id.temporal_layer_id]; vpx_usec_timer_start(&timer); if (vpx_codec_encode(&codec, frame_avail ? &raw : NULL, pts, 1, flags, VPX_DL_REALTIME)) { @@ -961,10 +1040,14 @@ } } ++frame_cnt; pts += frame_duration; } +#if ROI_MAP + free(mask_map); + free(prev_mask_map); +#endif close_input_file(&input_ctx); printout_rate_control_summary(&rc, &cfg, frame_cnt); printf("\n"); printf("Frame cnt and encoding time/FPS stats for encoding: %d %f %f \n", frame_cnt, 1000 * (float)cx_time / (double)(frame_cnt * 1000000), diff --git a/media/libvpx/libvpx/libs.mk b/media/libvpx/libvpx/libs.mk --- a/media/libvpx/libvpx/libs.mk +++ b/media/libvpx/libvpx/libs.mk @@ -92,19 +92,32 @@ INSTALL-LIBS-yes += include/vpx/vp8.h include/vpx/vp8cx.h INSTALL-LIBS-yes += include/vpx/vpx_ext_ratectrl.h INSTALL_MAPS += include/vpx/% $(SRC_PATH_BARE)/$(VP9_PREFIX)/% CODEC_DOC_SRCS += vpx/vp8.h vpx/vp8cx.h vpx/vpx_ext_ratectrl.h CODEC_DOC_SECTIONS += vp9 vp9_encoder +endif - RC_RTC_SRCS := $(addprefix $(VP9_PREFIX),$(call enabled,VP9_CX_SRCS)) - RC_RTC_SRCS += $(VP9_PREFIX)vp9cx.mk vpx/vp8.h vpx/vp8cx.h - RC_RTC_SRCS += vpx/vpx_ext_ratectrl.h +RC_RTC_SRCS := vpx/vp8.h vpx/vp8cx.h +RC_RTC_SRCS += vpx/vpx_ext_ratectrl.h +RC_RTC_SRCS += vpx/internal/vpx_ratectrl_rtc.h +ifeq ($(CONFIG_VP9_ENCODER),yes) + VP9_PREFIX=vp9/ + RC_RTC_SRCS += $(addprefix $(VP9_PREFIX),$(call enabled,VP9_CX_SRCS)) + RC_RTC_SRCS += $(VP9_PREFIX)vp9cx.mk RC_RTC_SRCS += $(VP9_PREFIX)ratectrl_rtc.cc RC_RTC_SRCS += $(VP9_PREFIX)ratectrl_rtc.h INSTALL-SRCS-$(CONFIG_CODEC_SRCS) += $(VP9_PREFIX)ratectrl_rtc.cc INSTALL-SRCS-$(CONFIG_CODEC_SRCS) += $(VP9_PREFIX)ratectrl_rtc.h endif +ifeq ($(CONFIG_VP8_ENCODER),yes) + VP8_PREFIX=vp8/ + RC_RTC_SRCS += $(addprefix $(VP8_PREFIX),$(call enabled,VP8_CX_SRCS)) + RC_RTC_SRCS += $(VP8_PREFIX)vp8_ratectrl_rtc.cc + RC_RTC_SRCS += $(VP8_PREFIX)vp8_ratectrl_rtc.h + INSTALL-SRCS-$(CONFIG_CODEC_SRCS) += $(VP8_PREFIX)vp8_ratectrl_rtc.cc + INSTALL-SRCS-$(CONFIG_CODEC_SRCS) += $(VP8_PREFIX)vp8_ratectrl_rtc.h +endif ifeq ($(CONFIG_VP9_DECODER),yes) VP9_PREFIX=vp9/ include $(SRC_PATH_BARE)/$(VP9_PREFIX)vp9dx.mk CODEC_SRCS-yes += $(addprefix $(VP9_PREFIX),$(call enabled,VP9_DX_SRCS)) @@ -124,11 +137,11 @@ endif ifeq ($(CONFIG_MSVS),yes) CODEC_LIB=$(if $(CONFIG_STATIC_MSVCRT),vpxmt,vpxmd) GTEST_LIB=$(if $(CONFIG_STATIC_MSVCRT),gtestmt,gtestmd) -RC_RTC_LIB=$(if $(CONFIG_STATIC_MSVCRT),vp9rcmt,vp9rcmd) +RC_RTC_LIB=$(if $(CONFIG_STATIC_MSVCRT),vpxrcmt,vpxrcmd) # This variable uses deferred expansion intentionally, since the results of # $(wildcard) may change during the course of the Make. VS_PLATFORMS = $(foreach d,$(wildcard */Release/$(CODEC_LIB).lib),$(word 1,$(subst /, ,$(d)))) endif @@ -247,20 +260,20 @@ PROJECTS-yes += vpx.$(VCPROJ_SFX) vpx.$(VCPROJ_SFX): vpx_config.asm vpx.$(VCPROJ_SFX): $(RTCD) -vp9rc.$(VCPROJ_SFX): \ +vpxrc.$(VCPROJ_SFX): \ VCPROJ_SRCS=$(filter-out $(addprefix %, $(ASM_INCLUDES)), $^) -vp9rc.$(VCPROJ_SFX): $(RC_RTC_SRCS) +vpxrc.$(VCPROJ_SFX): $(RC_RTC_SRCS) @echo " [CREATE] $@" $(qexec)$(GEN_VCPROJ) \ $(if $(CONFIG_SHARED),--dll,--lib) \ --target=$(TOOLCHAIN) \ $(if $(CONFIG_STATIC_MSVCRT),--static-crt) \ - --name=vp9rc \ + --name=vpxrc \ --proj-guid=C26FF952-9494-4838-9A3F-7F3D4F613385 \ --ver=$(CONFIG_VS_VERSION) \ --src-path-bare="$(SRC_PATH_BARE)" \ --out=$@ $(CFLAGS) \ --as=$(AS) \ @@ -273,14 +286,14 @@ vp8/%.c vp8/%.h vp9/%.c vp9/%.cc vp9/%.h vpx/% \ vpx_dsp/%), \ $(VCPROJ_SRCS)) \ --src-path-bare="$(SRC_PATH_BARE)" \ -PROJECTS-yes += vp9rc.$(VCPROJ_SFX) +PROJECTS-yes += vpxrc.$(VCPROJ_SFX) -vp9rc.$(VCPROJ_SFX): vpx_config.asm -vp9rc.$(VCPROJ_SFX): $(RTCD) +vpxrc.$(VCPROJ_SFX): vpx_config.asm +vpxrc.$(VCPROJ_SFX): $(RTCD) endif # ifeq ($(CONFIG_MSVS),yes) else # ifeq ($(CONFIG_EXTERNAL_BUILD),yes) LIBVPX_OBJS=$(call objs, $(filter-out $(ASM_INCLUDES), $(CODEC_SRCS))) OBJS-yes += $(LIBVPX_OBJS) @@ -396,16 +409,15 @@ $(qexec)echo 'Cflags: -I$${includedir}' >> $@ INSTALL-LIBS-yes += $(LIBSUBDIR)/pkgconfig/vpx.pc INSTALL_MAPS += $(LIBSUBDIR)/pkgconfig/%.pc %.pc CLEAN-OBJS += vpx.pc -ifeq ($(CONFIG_VP9_ENCODER),yes) - RC_RTC_OBJS=$(call objs,$(RC_RTC_SRCS)) +ifeq ($(CONFIG_ENCODERS),yes) RC_RTC_OBJS=$(call objs,$(RC_RTC_SRCS)) OBJS-yes += $(RC_RTC_OBJS) - LIBS-yes += $(BUILD_PFX)libvp9rc.a $(BUILD_PFX)libvp9rc_g.a - $(BUILD_PFX)libvp9rc_g.a: $(RC_RTC_OBJS) + LIBS-yes += $(BUILD_PFX)libvpxrc.a $(BUILD_PFX)libvpxrc_g.a + $(BUILD_PFX)libvpxrc_g.a: $(RC_RTC_OBJS) endif ifeq ($(CONFIG_VP9_ENCODER)$(CONFIG_RATE_CTRL),yesyes) SIMPLE_ENCODE_OBJS=$(call objs,$(SIMPLE_ENCODE_SRCS)) OBJS-yes += $(SIMPLE_ENCODE_OBJS) @@ -491,11 +503,11 @@ TEST_INTRA_PRED_SPEED_BIN=./test_intra_pred_speed$(EXE_SFX) TEST_INTRA_PRED_SPEED_SRCS=$(call addprefix_clean,test/,\ $(call enabled,TEST_INTRA_PRED_SPEED_SRCS)) TEST_INTRA_PRED_SPEED_OBJS := $(sort $(call objs,$(TEST_INTRA_PRED_SPEED_SRCS))) -ifeq ($(CONFIG_VP9_ENCODER),yes) +ifeq ($(CONFIG_ENCODERS),yes) RC_INTERFACE_TEST_BIN=./test_rc_interface$(EXE_SFX) RC_INTERFACE_TEST_SRCS=$(call addprefix_clean,test/,\ $(call enabled,RC_INTERFACE_TEST_SRCS)) RC_INTERFACE_TEST_OBJS := $(sort $(call objs,$(RC_INTERFACE_TEST_SRCS))) endif @@ -597,15 +609,15 @@ --out=$@ $(INTERNAL_CFLAGS) $(CFLAGS) \ -I. -I"$(SRC_PATH_BARE)/third_party/googletest/src/include" \ -L. -l$(CODEC_LIB) -l$(GTEST_LIB) $^ endif # TEST_INTRA_PRED_SPEED -ifeq ($(CONFIG_VP9_ENCODER),yes) +ifeq ($(CONFIG_ENCODERS),yes) ifneq ($(strip $(RC_INTERFACE_TEST_OBJS)),) PROJECTS-$(CONFIG_MSVS) += test_rc_interface.$(VCPROJ_SFX) test_rc_interface.$(VCPROJ_SFX): $(RC_INTERFACE_TEST_SRCS) vpx.$(VCPROJ_SFX) \ - vp9rc.$(VCPROJ_SFX) gtest.$(VCPROJ_SFX) + vpxrc.$(VCPROJ_SFX) gtest.$(VCPROJ_SFX) @echo " [CREATE] $@" $(qexec)$(GEN_VCPROJ) \ --exe \ --target=$(TOOLCHAIN) \ --name=test_rc_interface \ @@ -659,23 +671,23 @@ $(eval $(call linkerxx_template,$(TEST_INTRA_PRED_SPEED_BIN), \ $(TEST_INTRA_PRED_SPEED_OBJS) \ -L. -lvpx -lgtest $(extralibs) -lm)) endif # TEST_INTRA_PRED_SPEED -ifeq ($(CONFIG_VP9_ENCODER),yes) +ifeq ($(CONFIG_ENCODERS),yes) ifneq ($(strip $(RC_INTERFACE_TEST_OBJS)),) $(RC_INTERFACE_TEST_OBJS) $(RC_INTERFACE_TEST_OBJS:.o=.d): \ CXXFLAGS += $(GTEST_INCLUDES) OBJS-yes += $(RC_INTERFACE_TEST_OBJS) BINS-yes += $(RC_INTERFACE_TEST_BIN) -$(RC_INTERFACE_TEST_BIN): $(TEST_LIBS) libvp9rc.a +$(RC_INTERFACE_TEST_BIN): $(TEST_LIBS) libvpxrc.a $(eval $(call linkerxx_template,$(RC_INTERFACE_TEST_BIN), \ $(RC_INTERFACE_TEST_OBJS) \ - -L. -lvpx -lgtest -lvp9rc $(extralibs) -lm)) + -L. -lvpx -lgtest -lvpxrc $(extralibs) -lm)) endif # RC_INTERFACE_TEST -endif # CONFIG_VP9_ENCODER +endif # CONFIG_ENCODERS ifneq ($(strip $(SIMPLE_ENCODE_TEST_OBJS)),) $(SIMPLE_ENCODE_TEST_OBJS) $(SIMPLE_ENCODE_TEST_OBJS:.o=.d): \ CXXFLAGS += $(GTEST_INCLUDES) OBJS-yes += $(SIMPLE_ENCODE_TEST_OBJS) diff --git a/media/libvpx/libvpx/rate_hist.c b/media/libvpx/libvpx/rate_hist.c --- a/media/libvpx/libvpx/rate_hist.c +++ b/media/libvpx/libvpx/rate_hist.c @@ -191,44 +191,46 @@ return bucket[big_bucket].count; } static void show_histogram(const struct hist_bucket *bucket, int buckets, int total, int scale) { - const char *pat1, *pat2; + int width1, width2; int i; + if (!buckets) return; assert(bucket != NULL); + assert(buckets > 0); switch ((int)(log(bucket[buckets - 1].high) / log(10)) + 1) { case 1: case 2: - pat1 = "%4d %2s: "; - pat2 = "%4d-%2d: "; + width1 = 4; + width2 = 2; break; case 3: - pat1 = "%5d %3s: "; - pat2 = "%5d-%3d: "; + width1 = 5; + width2 = 3; break; case 4: - pat1 = "%6d %4s: "; - pat2 = "%6d-%4d: "; + width1 = 6; + width2 = 4; break; case 5: - pat1 = "%7d %5s: "; - pat2 = "%7d-%5d: "; + width1 = 7; + width2 = 5; break; case 6: - pat1 = "%8d %6s: "; - pat2 = "%8d-%6d: "; + width1 = 8; + width2 = 6; break; case 7: - pat1 = "%9d %7s: "; - pat2 = "%9d-%7d: "; + width1 = 9; + width2 = 7; break; default: - pat1 = "%12d %10s: "; - pat2 = "%12d-%10d: "; + width1 = 12; + width2 = 10; break; } for (i = 0; i < buckets; i++) { int len; @@ -239,13 +241,14 @@ len = HIST_BAR_MAX * bucket[i].count / scale; if (len < 1) len = 1; assert(len <= HIST_BAR_MAX); if (bucket[i].low == bucket[i].high) - fprintf(stderr, pat1, bucket[i].low, ""); + fprintf(stderr, "%*d %*s: ", width1, bucket[i].low, width2, ""); else - fprintf(stderr, pat2, bucket[i].low, bucket[i].high); + fprintf(stderr, "%*d-%*d: ", width1, bucket[i].low, width2, + bucket[i].high); for (j = 0; j < HIST_BAR_MAX; j++) fprintf(stderr, j < len ? "=" : " "); fprintf(stderr, "\t%5d (%6.2f%%)\n", bucket[i].count, pct); } } diff --git a/media/libvpx/libvpx/test/active_map_test.cc b/media/libvpx/libvpx/test/active_map_test.cc --- a/media/libvpx/libvpx/test/active_map_test.cc +++ b/media/libvpx/libvpx/test/active_map_test.cc @@ -17,11 +17,12 @@ namespace { class ActiveMapTest : public ::libvpx_test::EncoderTest, - public ::libvpx_test::CodecTestWith2Params { + public ::libvpx_test::CodecTestWith3Params { protected: static const int kWidth = 208; static const int kHeight = 144; ActiveMapTest() : EncoderTest(GET_PARAM(0)) {} @@ -35,10 +36,11 @@ virtual void PreEncodeFrameHook(::libvpx_test::VideoSource *video, ::libvpx_test::Encoder *encoder) { if (video->frame() == 0) { encoder->Control(VP8E_SET_CPUUSED, cpu_used_); + encoder->Control(VP9E_SET_AQ_MODE, GET_PARAM(3)); } else if (video->frame() == 3) { vpx_active_map_t map = vpx_active_map_t(); /* clang-format off */ uint8_t active_map[9 * 13] = { 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, @@ -85,7 +87,7 @@ ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); } VP9_INSTANTIATE_TEST_SUITE(ActiveMapTest, ::testing::Values(::libvpx_test::kRealTime), - ::testing::Range(0, 9)); + ::testing::Range(5, 10), ::testing::Values(0, 3)); } // namespace diff --git a/media/libvpx/libvpx/test/add_noise_test.cc b/media/libvpx/libvpx/test/add_noise_test.cc --- a/media/libvpx/libvpx/test/add_noise_test.cc +++ b/media/libvpx/libvpx/test/add_noise_test.cc @@ -21,11 +21,10 @@ namespace { static const int kNoiseSize = 3072; -// TODO(jimbankoski): make width and height integers not unsigned. typedef void (*AddNoiseFunc)(uint8_t *start, const int8_t *noise, int blackclamp, int whiteclamp, int width, int height, int pitch); typedef std::tuple AddNoiseTestFPParam; diff --git a/media/libvpx/libvpx/test/android/Android.mk b/media/libvpx/libvpx/test/android/Android.mk --- a/media/libvpx/libvpx/test/android/Android.mk +++ b/media/libvpx/libvpx/test/android/Android.mk @@ -32,10 +32,13 @@ LOCAL_MODULE := gtest LOCAL_C_INCLUDES := $(LOCAL_PATH)/third_party/googletest/src/ LOCAL_C_INCLUDES += $(LOCAL_PATH)/third_party/googletest/src/include/ LOCAL_EXPORT_C_INCLUDES := $(LOCAL_PATH)/third_party/googletest/src/include/ LOCAL_SRC_FILES := ./third_party/googletest/src/src/gtest-all.cc +LOCAL_LICENSE_KINDS := SPDX-license-identifier-BSD +LOCAL_LICENSE_CONDITIONS := notice +LOCAL_NOTICE_FILE := $(LOCAL_PATH)/../../LICENSE $(LOCAL_PATH)/../../PATENTS include $(BUILD_STATIC_LIBRARY) #libvpx_test include $(CLEAR_VARS) LOCAL_ARM_MODE := arm @@ -46,10 +49,13 @@ LOCAL_SHARED_LIBRARIES := vpx else LOCAL_STATIC_LIBRARIES += vpx endif +LOCAL_LICENSE_KINDS := SPDX-license-identifier-BSD +LOCAL_LICENSE_CONDITIONS := notice +LOCAL_NOTICE_FILE := $(LOCAL_PATH)/../../LICENSE $(LOCAL_PATH)/../../PATENTS include $(LOCAL_PATH)/test/test.mk LOCAL_C_INCLUDES := $(BINDINGS_DIR) FILTERED_SRC := $(sort $(filter %.cc %.c, $(LIBVPX_TEST_SRCS-yes))) LOCAL_SRC_FILES := $(addprefix ./test/, $(FILTERED_SRC)) # some test files depend on *_rtcd.h, ensure they're generated first. diff --git a/media/libvpx/libvpx/test/comp_avg_pred_test.cc b/media/libvpx/libvpx/test/comp_avg_pred_test.cc --- a/media/libvpx/libvpx/test/comp_avg_pred_test.cc +++ b/media/libvpx/libvpx/test/comp_avg_pred_test.cc @@ -181,6 +181,11 @@ #if HAVE_VSX INSTANTIATE_TEST_SUITE_P(VSX, AvgPredTest, ::testing::Values(&vpx_comp_avg_pred_vsx)); #endif // HAVE_VSX + +#if HAVE_LSX +INSTANTIATE_TEST_SUITE_P(LSX, AvgPredTest, + ::testing::Values(&vpx_comp_avg_pred_lsx)); +#endif // HAVE_LSX } // namespace diff --git a/media/libvpx/libvpx/test/convolve_test.cc b/media/libvpx/libvpx/test/convolve_test.cc --- a/media/libvpx/libvpx/test/convolve_test.cc +++ b/media/libvpx/libvpx/test/convolve_test.cc @@ -1447,10 +1447,23 @@ const ConvolveParam kArrayConvolve8_msa[] = { ALL_SIZES(convolve8_msa) }; INSTANTIATE_TEST_SUITE_P(MSA, ConvolveTest, ::testing::ValuesIn(kArrayConvolve8_msa)); #endif // HAVE_MSA +#if HAVE_LSX +const ConvolveFunctions convolve8_lsx( + vpx_convolve_copy_lsx, vpx_convolve_avg_lsx, vpx_convolve8_horiz_lsx, + vpx_convolve8_avg_horiz_lsx, vpx_convolve8_vert_lsx, + vpx_convolve8_avg_vert_lsx, vpx_convolve8_lsx, vpx_convolve8_avg_lsx, + vpx_scaled_horiz_c, vpx_scaled_avg_horiz_c, vpx_scaled_vert_c, + vpx_scaled_avg_vert_c, vpx_scaled_2d_c, vpx_scaled_avg_2d_c, 0); + +const ConvolveParam kArrayConvolve8_lsx[] = { ALL_SIZES(convolve8_lsx) }; +INSTANTIATE_TEST_SUITE_P(LSX, ConvolveTest, + ::testing::ValuesIn(kArrayConvolve8_lsx)); +#endif // HAVE_LSX + #if HAVE_VSX const ConvolveFunctions convolve8_vsx( vpx_convolve_copy_vsx, vpx_convolve_avg_vsx, vpx_convolve8_horiz_vsx, vpx_convolve8_avg_horiz_vsx, vpx_convolve8_vert_vsx, vpx_convolve8_avg_vert_vsx, vpx_convolve8_vsx, vpx_convolve8_avg_vsx, diff --git a/media/libvpx/libvpx/test/dct16x16_test.cc b/media/libvpx/libvpx/test/dct16x16_test.cc --- a/media/libvpx/libvpx/test/dct16x16_test.cc +++ b/media/libvpx/libvpx/test/dct16x16_test.cc @@ -866,6 +866,13 @@ INSTANTIATE_TEST_SUITE_P( VSX, Trans16x16DCT, ::testing::Values(make_tuple(&vpx_fdct16x16_c, &vpx_idct16x16_256_add_vsx, 0, VPX_BITS_8))); #endif // HAVE_VSX && !CONFIG_VP9_HIGHBITDEPTH && !CONFIG_EMULATE_HARDWARE + +#if HAVE_LSX && !CONFIG_VP9_HIGHBITDEPTH && !CONFIG_EMULATE_HARDWARE +INSTANTIATE_TEST_SUITE_P(LSX, Trans16x16DCT, + ::testing::Values(make_tuple(&vpx_fdct16x16_lsx, + &vpx_idct16x16_256_add_c, + 0, VPX_BITS_8))); +#endif // HAVE_LSX && !CONFIG_VP9_HIGHBITDEPTH && !CONFIG_EMULATE_HARDWARE } // namespace diff --git a/media/libvpx/libvpx/test/dct32x32_test.cc b/media/libvpx/libvpx/test/dct32x32_test.cc --- a/media/libvpx/libvpx/test/dct32x32_test.cc +++ b/media/libvpx/libvpx/test/dct32x32_test.cc @@ -394,6 +394,15 @@ ::testing::Values(make_tuple(&vpx_fdct32x32_c, &vpx_idct32x32_1024_add_vsx, 0, VPX_BITS_8), make_tuple(&vpx_fdct32x32_rd_vsx, &vpx_idct32x32_1024_add_vsx, 1, VPX_BITS_8))); #endif // HAVE_VSX && !CONFIG_VP9_HIGHBITDEPTH && !CONFIG_EMULATE_HARDWARE + +#if HAVE_LSX && !CONFIG_VP9_HIGHBITDEPTH && !CONFIG_EMULATE_HARDWARE +INSTANTIATE_TEST_SUITE_P( + LSX, Trans32x32Test, + ::testing::Values(make_tuple(&vpx_fdct32x32_lsx, + &vpx_idct32x32_1024_add_lsx, 0, VPX_BITS_8), + make_tuple(&vpx_fdct32x32_rd_lsx, + &vpx_idct32x32_1024_add_lsx, 1, VPX_BITS_8))); +#endif // HAVE_LSX && !CONFIG_VP9_HIGHBITDEPTH && !CONFIG_EMULATE_HARDWARE } // namespace diff --git a/media/libvpx/libvpx/test/dct_test.cc b/media/libvpx/libvpx/test/dct_test.cc --- a/media/libvpx/libvpx/test/dct_test.cc +++ b/media/libvpx/libvpx/test/dct_test.cc @@ -584,10 +584,27 @@ INSTANTIATE_TEST_SUITE_P(VSX, TransDCT, ::testing::Values(make_tuple(0, &dct_vsx_func_info, 0, VPX_BITS_8))); #endif // HAVE_VSX && !CONFIG_VP9_HIGHBITDEPTH && +#if HAVE_LSX && !CONFIG_VP9_HIGHBITDEPTH +static const FuncInfo dct_lsx_func_info[4] = { + { &fdct_wrapper, &idct_wrapper, 4, 1 }, + { &fdct_wrapper, &idct_wrapper, 8, 1 }, + { &fdct_wrapper, &idct_wrapper, + 16, 1 }, + { &fdct_wrapper, &idct_wrapper, + 32, 1 } +}; + +INSTANTIATE_TEST_SUITE_P( + LSX, TransDCT, + ::testing::Combine(::testing::Range(0, 4), + ::testing::Values(dct_lsx_func_info), + ::testing::Values(0), ::testing::Values(VPX_BITS_8))); +#endif // HAVE_LSX && !CONFIG_VP9_HIGHBITDEPTH + #endif // !CONFIG_EMULATE_HARDWARE /* -------------------------------------------------------------------------- */ class TransHT : public TransTestBase { @@ -639,12 +656,15 @@ 2 }, { &vp9_highbd_fht16x16_c, &highbd_iht_wrapper, 16, 2 }, #endif { &vp9_fht4x4_c, &iht_wrapper, 4, 1 }, + { &vp9_fht4x4_neon, &iht_wrapper, 4, 1 }, { &vp9_fht8x8_c, &iht_wrapper, 8, 1 }, - { &vp9_fht16x16_c, &iht_wrapper, 16, 1 } + { &vp9_fht8x8_neon, &iht_wrapper, 8, 1 }, + { &vp9_fht16x16_c, &iht_wrapper, 16, 1 }, + { &vp9_fht16x16_neon, &iht_wrapper, 16, 1 } }; INSTANTIATE_TEST_SUITE_P( NEON, TransHT, ::testing::Combine( @@ -751,6 +771,7 @@ INSTANTIATE_TEST_SUITE_P(VSX, TransWHT, ::testing::Values(make_tuple(0, &wht_vsx_func_info, 0, VPX_BITS_8))); #endif // HAVE_VSX && !CONFIG_EMULATE_HARDWARE + } // namespace diff --git a/media/libvpx/libvpx/test/encode_api_test.cc b/media/libvpx/libvpx/test/encode_api_test.cc --- a/media/libvpx/libvpx/test/encode_api_test.cc +++ b/media/libvpx/libvpx/test/encode_api_test.cc @@ -8,36 +8,37 @@ * be found in the AUTHORS file in the root of the source tree. */ #include #include +#include #include "third_party/googletest/src/include/gtest/gtest.h" #include "./vpx_config.h" +#include "test/video_source.h" #include "vpx/vp8cx.h" #include "vpx/vpx_encoder.h" namespace { -#define NELEMENTS(x) static_cast(sizeof(x) / sizeof(x[0])) +const vpx_codec_iface_t *kCodecIfaces[] = { +#if CONFIG_VP8_ENCODER + &vpx_codec_vp8_cx_algo, +#endif +#if CONFIG_VP9_ENCODER + &vpx_codec_vp9_cx_algo, +#endif +}; bool IsVP9(const vpx_codec_iface_t *iface) { static const char kVP9Name[] = "WebM Project VP9"; return strncmp(kVP9Name, vpx_codec_iface_name(iface), sizeof(kVP9Name) - 1) == 0; } TEST(EncodeAPI, InvalidParams) { - static const vpx_codec_iface_t *kCodecs[] = { -#if CONFIG_VP8_ENCODER - &vpx_codec_vp8_cx_algo, -#endif -#if CONFIG_VP9_ENCODER - &vpx_codec_vp9_cx_algo, -#endif - }; uint8_t buf[1] = { 0 }; vpx_image_t img; vpx_codec_ctx_t enc; vpx_codec_enc_cfg_t cfg; @@ -56,21 +57,21 @@ vpx_codec_enc_config_default(nullptr, nullptr, 0)); EXPECT_EQ(VPX_CODEC_INVALID_PARAM, vpx_codec_enc_config_default(nullptr, &cfg, 0)); EXPECT_NE(vpx_codec_error(nullptr), nullptr); - for (int i = 0; i < NELEMENTS(kCodecs); ++i) { - SCOPED_TRACE(vpx_codec_iface_name(kCodecs[i])); + for (const auto *iface : kCodecIfaces) { + SCOPED_TRACE(vpx_codec_iface_name(iface)); EXPECT_EQ(VPX_CODEC_INVALID_PARAM, - vpx_codec_enc_init(nullptr, kCodecs[i], nullptr, 0)); + vpx_codec_enc_init(nullptr, iface, nullptr, 0)); EXPECT_EQ(VPX_CODEC_INVALID_PARAM, - vpx_codec_enc_init(&enc, kCodecs[i], nullptr, 0)); + vpx_codec_enc_init(&enc, iface, nullptr, 0)); EXPECT_EQ(VPX_CODEC_INVALID_PARAM, - vpx_codec_enc_config_default(kCodecs[i], &cfg, 1)); + vpx_codec_enc_config_default(iface, &cfg, 1)); - EXPECT_EQ(VPX_CODEC_OK, vpx_codec_enc_config_default(kCodecs[i], &cfg, 0)); - EXPECT_EQ(VPX_CODEC_OK, vpx_codec_enc_init(&enc, kCodecs[i], &cfg, 0)); + EXPECT_EQ(VPX_CODEC_OK, vpx_codec_enc_config_default(iface, &cfg, 0)); + EXPECT_EQ(VPX_CODEC_OK, vpx_codec_enc_init(&enc, iface, &cfg, 0)); EXPECT_EQ(VPX_CODEC_OK, vpx_codec_encode(&enc, nullptr, 0, 0, 0, 0)); EXPECT_EQ(VPX_CODEC_OK, vpx_codec_destroy(&enc)); } } @@ -122,27 +123,18 @@ // Set up 2 spatial streams with 2 temporal layers per stream, and generate // invalid configuration by setting the temporal layer rate allocation // (ts_target_bitrate[]) to 0 for both layers. This should fail independent of // CONFIG_MULTI_RES_ENCODING. TEST(EncodeAPI, MultiResEncode) { - static const vpx_codec_iface_t *kCodecs[] = { -#if CONFIG_VP8_ENCODER - &vpx_codec_vp8_cx_algo, -#endif -#if CONFIG_VP9_ENCODER - &vpx_codec_vp9_cx_algo, -#endif - }; const int width = 1280; const int height = 720; const int width_down = width / 2; const int height_down = height / 2; const int target_bitrate = 1000; const int framerate = 30; - for (int c = 0; c < NELEMENTS(kCodecs); ++c) { - const vpx_codec_iface_t *const iface = kCodecs[c]; + for (const auto *iface : kCodecIfaces) { vpx_codec_ctx_t enc[2]; vpx_codec_enc_cfg_t cfg[2]; vpx_rational_t dsf[2] = { { 2, 1 }, { 2, 1 } }; memset(enc, 0, sizeof(enc)); @@ -308,6 +300,64 @@ EXPECT_EQ(vpx_codec_destroy(&enc), VPX_CODEC_OK); } } +void InitCodec(const vpx_codec_iface_t &iface, int width, int height, + vpx_codec_ctx_t *enc, vpx_codec_enc_cfg_t *cfg) { + ASSERT_EQ(vpx_codec_enc_config_default(&iface, cfg, 0), VPX_CODEC_OK); + cfg->g_w = width; + cfg->g_h = height; + cfg->g_lag_in_frames = 0; + cfg->g_pass = VPX_RC_ONE_PASS; + ASSERT_EQ(vpx_codec_enc_init(enc, &iface, cfg, 0), VPX_CODEC_OK); + + ASSERT_EQ(vpx_codec_control_(enc, VP8E_SET_CPUUSED, 2), VPX_CODEC_OK); +} + +// Encodes 1 frame of size |cfg.g_w| x |cfg.g_h| setting |enc|'s configuration +// to |cfg|. +void EncodeWithConfig(const vpx_codec_enc_cfg_t &cfg, vpx_codec_ctx_t *enc) { + libvpx_test::DummyVideoSource video; + video.SetSize(cfg.g_w, cfg.g_h); + video.Begin(); + EXPECT_EQ(vpx_codec_enc_config_set(enc, &cfg), VPX_CODEC_OK) + << vpx_codec_error_detail(enc); + + EXPECT_EQ(vpx_codec_encode(enc, video.img(), video.pts(), video.duration(), + /*flags=*/0, VPX_DL_GOOD_QUALITY), + VPX_CODEC_OK) + << vpx_codec_error_detail(enc); +} + +TEST(EncodeAPI, ConfigChangeThreadCount) { + constexpr int kWidth = 1920; + constexpr int kHeight = 1080; + + for (const auto *iface : kCodecIfaces) { + SCOPED_TRACE(vpx_codec_iface_name(iface)); + for (int i = 0; i < (IsVP9(iface) ? 2 : 1); ++i) { + vpx_codec_enc_cfg_t cfg; + struct Encoder { + ~Encoder() { EXPECT_EQ(vpx_codec_destroy(&ctx), VPX_CODEC_OK); } + vpx_codec_ctx_t ctx = {}; + } enc; + + EXPECT_NO_FATAL_FAILURE( + InitCodec(*iface, kWidth, kHeight, &enc.ctx, &cfg)); + if (IsVP9(iface)) { + EXPECT_EQ(vpx_codec_control_(&enc.ctx, VP9E_SET_TILE_COLUMNS, 6), + VPX_CODEC_OK); + EXPECT_EQ(vpx_codec_control_(&enc.ctx, VP9E_SET_ROW_MT, i), + VPX_CODEC_OK); + } + + for (const auto threads : { 1, 4, 8, 6, 2, 1 }) { + cfg.g_threads = threads; + EXPECT_NO_FATAL_FAILURE(EncodeWithConfig(cfg, &enc.ctx)) + << "iteration: " << i << " threads: " << threads; + } + } + } +} + } // namespace diff --git a/media/libvpx/libvpx/test/fdct8x8_test.cc b/media/libvpx/libvpx/test/fdct8x8_test.cc --- a/media/libvpx/libvpx/test/fdct8x8_test.cc +++ b/media/libvpx/libvpx/test/fdct8x8_test.cc @@ -766,6 +766,13 @@ INSTANTIATE_TEST_SUITE_P(VSX, FwdTrans8x8DCT, ::testing::Values(make_tuple(&vpx_fdct8x8_c, &vpx_idct8x8_64_add_vsx, 0, VPX_BITS_8))); #endif // HAVE_VSX && !CONFIG_VP9_HIGHBITDEPTH && !CONFIG_EMULATE_HARDWARE + +#if HAVE_LSX && !CONFIG_VP9_HIGHBITDEPTH && !CONFIG_EMULATE_HARDWARE +INSTANTIATE_TEST_SUITE_P(LSX, FwdTrans8x8DCT, + ::testing::Values(make_tuple(&vpx_fdct8x8_lsx, + &vpx_idct8x8_64_add_c, 0, + VPX_BITS_8))); +#endif // HAVE_LSX && !CONFIG_VP9_HIGHBITDEPTH && !CONFIG_EMULATE_HARDWARE } // namespace diff --git a/media/libvpx/libvpx/test/hadamard_test.cc b/media/libvpx/libvpx/test/hadamard_test.cc --- a/media/libvpx/libvpx/test/hadamard_test.cc +++ b/media/libvpx/libvpx/test/hadamard_test.cc @@ -283,10 +283,17 @@ VSX, HadamardLowbdTest, ::testing::Values(HadamardFuncWithSize(&vpx_hadamard_8x8_vsx, 8), HadamardFuncWithSize(&vpx_hadamard_16x16_vsx, 16))); #endif // HAVE_VSX +#if HAVE_LSX +INSTANTIATE_TEST_SUITE_P( + LSX, HadamardLowbdTest, + ::testing::Values(HadamardFuncWithSize(&vpx_hadamard_8x8_lsx, 8), + HadamardFuncWithSize(&vpx_hadamard_16x16_lsx, 16))); +#endif // HAVE_LSX + #if CONFIG_VP9_HIGHBITDEPTH class HadamardHighbdTest : public HadamardTestBase { protected: virtual int16_t Rand() { return rnd_.Rand13Signed(); } }; diff --git a/media/libvpx/libvpx/test/lpf_test.cc b/media/libvpx/libvpx/test/lpf_test.cc --- a/media/libvpx/libvpx/test/lpf_test.cc +++ b/media/libvpx/libvpx/test/lpf_test.cc @@ -145,11 +145,11 @@ loop_op_t loopfilter_op_; loop_op_t ref_loopfilter_op_; }; GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(Loop8Test6Param); -#if HAVE_NEON || HAVE_SSE2 || \ +#if HAVE_NEON || HAVE_SSE2 || (HAVE_LSX && !CONFIG_VP9_HIGHBITDEPTH) || \ (HAVE_DSPR2 || HAVE_MSA && !CONFIG_VP9_HIGHBITDEPTH) class Loop8Test9Param : public ::testing::TestWithParam { public: virtual ~Loop8Test9Param() {} virtual void SetUp() { @@ -167,11 +167,11 @@ dual_loop_op_t loopfilter_op_; dual_loop_op_t ref_loopfilter_op_; }; GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(Loop8Test9Param); #endif // HAVE_NEON || HAVE_SSE2 || (HAVE_DSPR2 || HAVE_MSA && - // (!CONFIG_VP9_HIGHBITDEPTH)) + // (!CONFIG_VP9_HIGHBITDEPTH) || (HAVE_LSX && !CONFIG_VP9_HIGHBITDEPTH)) TEST_P(Loop8Test6Param, OperationCheck) { ACMRandom rnd(ACMRandom::DeterministicSeed()); const int count_test_block = number_of_iterations; const int32_t p = kNumCoeffs / 32; @@ -279,11 +279,11 @@ << "Error: Loop8Test6Param, C output doesn't match SSE2 " "loopfilter output. " << "First failed at test case " << first_failure; } -#if HAVE_NEON || HAVE_SSE2 || \ +#if HAVE_NEON || HAVE_SSE2 || (HAVE_LSX && (!CONFIG_VP9_HIGHBITDEPTH)) || \ (HAVE_DSPR2 || HAVE_MSA && (!CONFIG_VP9_HIGHBITDEPTH)) TEST_P(Loop8Test9Param, OperationCheck) { ACMRandom rnd(ACMRandom::DeterministicSeed()); const int count_test_block = number_of_iterations; DECLARE_ALIGNED(PIXEL_WIDTH, Pixel, s[kNumCoeffs]); @@ -409,10 +409,11 @@ << "Error: Loop8Test9Param, C output doesn't match SSE2" "loopfilter output. " << "First failed at test case " << first_failure; } #endif // HAVE_NEON || HAVE_SSE2 || (HAVE_DSPR2 || HAVE_MSA && + // (!CONFIG_VP9_HIGHBITDEPTH)) || (HAVE_LSX && // (!CONFIG_VP9_HIGHBITDEPTH)) using std::make_tuple; #if HAVE_SSE2 @@ -690,6 +691,31 @@ &vpx_lpf_vertical_4_dual_c, 8), make_tuple(&vpx_lpf_vertical_8_dual_msa, &vpx_lpf_vertical_8_dual_c, 8))); #endif // HAVE_MSA && (!CONFIG_VP9_HIGHBITDEPTH) +#if HAVE_LSX && (!CONFIG_VP9_HIGHBITDEPTH) +INSTANTIATE_TEST_SUITE_P( + LSX, Loop8Test6Param, + ::testing::Values( + make_tuple(&vpx_lpf_horizontal_4_lsx, &vpx_lpf_horizontal_4_c, 8), + make_tuple(&vpx_lpf_horizontal_8_lsx, &vpx_lpf_horizontal_8_c, 8), + make_tuple(&vpx_lpf_horizontal_16_dual_lsx, + &vpx_lpf_horizontal_16_dual_c, 8), + make_tuple(&vpx_lpf_vertical_4_lsx, &vpx_lpf_vertical_4_c, 8), + make_tuple(&vpx_lpf_vertical_8_lsx, &vpx_lpf_vertical_8_c, 8), + make_tuple(&vpx_lpf_vertical_16_dual_lsx, &vpx_lpf_vertical_16_dual_c, + 8))); + +INSTANTIATE_TEST_SUITE_P( + LSX, Loop8Test9Param, + ::testing::Values(make_tuple(&vpx_lpf_horizontal_4_dual_lsx, + &vpx_lpf_horizontal_4_dual_c, 8), + make_tuple(&vpx_lpf_horizontal_8_dual_lsx, + &vpx_lpf_horizontal_8_dual_c, 8), + make_tuple(&vpx_lpf_vertical_4_dual_lsx, + &vpx_lpf_vertical_4_dual_c, 8), + make_tuple(&vpx_lpf_vertical_8_dual_lsx, + &vpx_lpf_vertical_8_dual_c, 8))); +#endif // HAVE_LSX && (!CONFIG_VP9_HIGHBITDEPTH) + } // namespace diff --git a/media/libvpx/libvpx/test/partial_idct_test.cc b/media/libvpx/libvpx/test/partial_idct_test.cc --- a/media/libvpx/libvpx/test/partial_idct_test.cc +++ b/media/libvpx/libvpx/test/partial_idct_test.cc @@ -952,8 +952,22 @@ INSTANTIATE_TEST_SUITE_P(MSA, PartialIDctTest, ::testing::ValuesIn(msa_partial_idct_tests)); #endif // HAVE_MSA && !CONFIG_VP9_HIGHBITDEPTH +#if HAVE_LSX && !CONFIG_VP9_HIGHBITDEPTH +const PartialInvTxfmParam lsx_partial_idct_tests[] = { + make_tuple(&vpx_fdct32x32_c, &wrapper, + &wrapper, TX_32X32, 1024, 8, 1), + make_tuple(&vpx_fdct32x32_c, &wrapper, + &wrapper, TX_32X32, 34, 8, 1), + make_tuple(&vpx_fdct32x32_c, &wrapper, + &wrapper, TX_32X32, 1, 8, 1), +}; + +INSTANTIATE_TEST_SUITE_P(LSX, PartialIDctTest, + ::testing::ValuesIn(lsx_partial_idct_tests)); +#endif // HAVE_LSX && !CONFIG_VP9_HIGHBITDEPTH + #endif // !CONFIG_EMULATE_HARDWARE } // namespace diff --git a/media/libvpx/libvpx/test/pp_filter_test.cc b/media/libvpx/libvpx/test/pp_filter_test.cc --- a/media/libvpx/libvpx/test/pp_filter_test.cc +++ b/media/libvpx/libvpx/test/pp_filter_test.cc @@ -113,11 +113,11 @@ } pixel_ptr += dst_image.stride(); } vpx_free(flimits_); -}; +} TEST_P(VpxPostProcDownAndAcrossMbRowTest, CheckCvsAssembly) { // Size of the underlying data block that will be filtered. // Y blocks are always a multiple of 16 wide and exactly 16 high. U and V // blocks are always a multiple of 8 wide and exactly 8 high. @@ -212,11 +212,11 @@ RunNTimes(INT16_MAX); PrintMedian("16x16"); vpx_free(flimits_); -}; +} class VpxMbPostProcAcrossIpTest : public AbstractBench, public ::testing::TestWithParam { public: diff --git a/media/libvpx/libvpx/test/ratectrl_rtc_test.cc b/media/libvpx/libvpx/test/ratectrl_rtc_test.cc deleted file mode 100644 --- a/media/libvpx/libvpx/test/ratectrl_rtc_test.cc +++ /dev/null @@ -1,376 +0,0 @@ -/* - * Copyright (c) 2020 The WebM project authors. All Rights Reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ -#include "vp9/ratectrl_rtc.h" - -#include // NOLINT -#include - -#include "./vpx_config.h" -#include "third_party/googletest/src/include/gtest/gtest.h" -#include "test/codec_factory.h" -#include "test/encode_test_driver.h" -#include "test/i420_video_source.h" -#include "test/util.h" -#include "test/video_source.h" -#include "vpx/vpx_codec.h" -#include "vpx_ports/bitops.h" - -namespace { - -const size_t kNumFrames = 300; - -const int kTemporalId[4] = { 0, 2, 1, 2 }; - -class RcInterfaceTest - : public ::libvpx_test::EncoderTest, - public ::libvpx_test::CodecTestWith2Params { - public: - RcInterfaceTest() - : EncoderTest(GET_PARAM(0)), aq_mode_(GET_PARAM(1)), key_interval_(3000), - encoder_exit_(false) {} - - virtual ~RcInterfaceTest() {} - - protected: - virtual void SetUp() { - InitializeConfig(); - SetMode(::libvpx_test::kRealTime); - } - - virtual void PreEncodeFrameHook(libvpx_test::VideoSource *video, - libvpx_test::Encoder *encoder) { - if (video->frame() == 0) { - encoder->Control(VP8E_SET_CPUUSED, 7); - encoder->Control(VP9E_SET_AQ_MODE, aq_mode_); - encoder->Control(VP9E_SET_TUNE_CONTENT, 0); - encoder->Control(VP8E_SET_MAX_INTRA_BITRATE_PCT, 1000); - encoder->Control(VP9E_SET_RTC_EXTERNAL_RATECTRL, 1); - } - frame_params_.frame_type = - video->frame() % key_interval_ == 0 ? KEY_FRAME : INTER_FRAME; - if (rc_cfg_.rc_mode == VPX_CBR && frame_params_.frame_type == INTER_FRAME) { - // Disable golden frame update. - frame_flags_ |= VP8_EFLAG_NO_UPD_GF; - frame_flags_ |= VP8_EFLAG_NO_UPD_ARF; - } - encoder_exit_ = video->frame() == kNumFrames; - } - - virtual void PostEncodeFrameHook(::libvpx_test::Encoder *encoder) { - if (encoder_exit_) { - return; - } - int loopfilter_level, qp; - encoder->Control(VP9E_GET_LOOPFILTER_LEVEL, &loopfilter_level); - encoder->Control(VP8E_GET_LAST_QUANTIZER, &qp); - rc_api_->ComputeQP(frame_params_); - ASSERT_EQ(rc_api_->GetQP(), qp); - ASSERT_EQ(rc_api_->GetLoopfilterLevel(), loopfilter_level); - } - - virtual void FramePktHook(const vpx_codec_cx_pkt_t *pkt) { - rc_api_->PostEncodeUpdate(pkt->data.frame.sz); - } - - void RunOneLayer() { - SetConfig(GET_PARAM(2)); - rc_api_ = libvpx::VP9RateControlRTC::Create(rc_cfg_); - frame_params_.spatial_layer_id = 0; - frame_params_.temporal_layer_id = 0; - - ::libvpx_test::I420VideoSource video("desktop_office1.1280_720-020.yuv", - 1280, 720, 30, 1, 0, kNumFrames); - - ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); - } - - void RunOneLayerVBRPeriodicKey() { - if (GET_PARAM(2) != VPX_VBR) return; - key_interval_ = 100; - SetConfig(VPX_VBR); - rc_api_ = libvpx::VP9RateControlRTC::Create(rc_cfg_); - frame_params_.spatial_layer_id = 0; - frame_params_.temporal_layer_id = 0; - - ::libvpx_test::I420VideoSource video("desktop_office1.1280_720-020.yuv", - 1280, 720, 30, 1, 0, kNumFrames); - - ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); - } - - private: - void SetConfig(vpx_rc_mode rc_mode) { - rc_cfg_.width = 1280; - rc_cfg_.height = 720; - rc_cfg_.max_quantizer = 52; - rc_cfg_.min_quantizer = 2; - rc_cfg_.target_bandwidth = 1000; - rc_cfg_.buf_initial_sz = 600; - rc_cfg_.buf_optimal_sz = 600; - rc_cfg_.buf_sz = 1000; - rc_cfg_.undershoot_pct = 50; - rc_cfg_.overshoot_pct = 50; - rc_cfg_.max_intra_bitrate_pct = 1000; - rc_cfg_.framerate = 30.0; - rc_cfg_.ss_number_layers = 1; - rc_cfg_.ts_number_layers = 1; - rc_cfg_.scaling_factor_num[0] = 1; - rc_cfg_.scaling_factor_den[0] = 1; - rc_cfg_.layer_target_bitrate[0] = 1000; - rc_cfg_.max_quantizers[0] = 52; - rc_cfg_.min_quantizers[0] = 2; - rc_cfg_.rc_mode = rc_mode; - rc_cfg_.aq_mode = aq_mode_; - - // Encoder settings for ground truth. - cfg_.g_w = 1280; - cfg_.g_h = 720; - cfg_.rc_undershoot_pct = 50; - cfg_.rc_overshoot_pct = 50; - cfg_.rc_buf_initial_sz = 600; - cfg_.rc_buf_optimal_sz = 600; - cfg_.rc_buf_sz = 1000; - cfg_.rc_dropframe_thresh = 0; - cfg_.rc_min_quantizer = 2; - cfg_.rc_max_quantizer = 52; - cfg_.rc_end_usage = rc_mode; - cfg_.g_lag_in_frames = 0; - cfg_.g_error_resilient = 0; - cfg_.rc_target_bitrate = 1000; - cfg_.kf_min_dist = key_interval_; - cfg_.kf_max_dist = key_interval_; - } - - std::unique_ptr rc_api_; - libvpx::VP9RateControlRtcConfig rc_cfg_; - int aq_mode_; - int key_interval_; - libvpx::VP9FrameParamsQpRTC frame_params_; - bool encoder_exit_; -}; - -class RcInterfaceSvcTest : public ::libvpx_test::EncoderTest, - public ::libvpx_test::CodecTestWithParam { - public: - RcInterfaceSvcTest() : EncoderTest(GET_PARAM(0)), aq_mode_(GET_PARAM(1)) {} - virtual ~RcInterfaceSvcTest() {} - - protected: - virtual void SetUp() { - InitializeConfig(); - SetMode(::libvpx_test::kRealTime); - } - - virtual void PreEncodeFrameHook(libvpx_test::VideoSource *video, - ::libvpx_test::Encoder *encoder) { - if (video->frame() == 0) { - encoder->Control(VP8E_SET_CPUUSED, 7); - encoder->Control(VP9E_SET_AQ_MODE, aq_mode_); - encoder->Control(VP9E_SET_TUNE_CONTENT, 0); - encoder->Control(VP8E_SET_MAX_INTRA_BITRATE_PCT, 900); - encoder->Control(VP9E_SET_RTC_EXTERNAL_RATECTRL, 1); - encoder->Control(VP9E_SET_SVC, 1); - encoder->Control(VP9E_SET_SVC_PARAMETERS, &svc_params_); - } - - frame_params_.frame_type = video->frame() == 0 ? KEY_FRAME : INTER_FRAME; - if (rc_cfg_.rc_mode == VPX_CBR && frame_params_.frame_type == INTER_FRAME) { - // Disable golden frame update. - frame_flags_ |= VP8_EFLAG_NO_UPD_GF; - frame_flags_ |= VP8_EFLAG_NO_UPD_ARF; - } - encoder_exit_ = video->frame() == kNumFrames; - current_superframe_ = video->frame(); - } - - virtual void PostEncodeFrameHook(::libvpx_test::Encoder *encoder) { - ::libvpx_test::CxDataIterator iter = encoder->GetCxData(); - while (const vpx_codec_cx_pkt_t *pkt = iter.Next()) { - ParseSuperframeSizes(static_cast(pkt->data.frame.buf), - pkt->data.frame.sz); - for (int sl = 0; sl < rc_cfg_.ss_number_layers; sl++) { - frame_params_.spatial_layer_id = sl; - frame_params_.temporal_layer_id = kTemporalId[current_superframe_ % 4]; - rc_api_->ComputeQP(frame_params_); - frame_params_.frame_type = INTER_FRAME; - rc_api_->PostEncodeUpdate(sizes_[sl]); - } - } - if (!encoder_exit_) { - int loopfilter_level, qp; - encoder->Control(VP9E_GET_LOOPFILTER_LEVEL, &loopfilter_level); - encoder->Control(VP8E_GET_LAST_QUANTIZER, &qp); - ASSERT_EQ(rc_api_->GetQP(), qp); - ASSERT_EQ(rc_api_->GetLoopfilterLevel(), loopfilter_level); - } - } - // This method needs to be overridden because non-reference frames are - // expected to be mismatched frames as the encoder will avoid loopfilter on - // these frames. - virtual void MismatchHook(const vpx_image_t * /*img1*/, - const vpx_image_t * /*img2*/) {} - - void RunSvc() { - SetConfigSvc(); - rc_api_ = libvpx::VP9RateControlRTC::Create(rc_cfg_); - SetEncoderSvc(); - - ::libvpx_test::I420VideoSource video("desktop_office1.1280_720-020.yuv", - 1280, 720, 30, 1, 0, kNumFrames); - - ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); - } - - private: - vpx_codec_err_t ParseSuperframeSizes(const uint8_t *data, size_t data_sz) { - uint8_t marker = *(data + data_sz - 1); - if ((marker & 0xe0) == 0xc0) { - const uint32_t frames = (marker & 0x7) + 1; - const uint32_t mag = ((marker >> 3) & 0x3) + 1; - const size_t index_sz = 2 + mag * frames; - // This chunk is marked as having a superframe index but doesn't have - // enough data for it, thus it's an invalid superframe index. - if (data_sz < index_sz) return VPX_CODEC_CORRUPT_FRAME; - { - const uint8_t marker2 = *(data + data_sz - index_sz); - // This chunk is marked as having a superframe index but doesn't have - // the matching marker byte at the front of the index therefore it's an - // invalid chunk. - if (marker != marker2) return VPX_CODEC_CORRUPT_FRAME; - } - const uint8_t *x = &data[data_sz - index_sz + 1]; - for (uint32_t i = 0; i < frames; ++i) { - uint32_t this_sz = 0; - - for (uint32_t j = 0; j < mag; ++j) this_sz |= (*x++) << (j * 8); - sizes_[i] = this_sz; - } - } - return VPX_CODEC_OK; - } - - void SetEncoderSvc() { - cfg_.ss_number_layers = 3; - cfg_.ts_number_layers = 3; - cfg_.g_timebase.num = 1; - cfg_.g_timebase.den = 30; - svc_params_.scaling_factor_num[0] = 72; - svc_params_.scaling_factor_den[0] = 288; - svc_params_.scaling_factor_num[1] = 144; - svc_params_.scaling_factor_den[1] = 288; - svc_params_.scaling_factor_num[2] = 288; - svc_params_.scaling_factor_den[2] = 288; - for (int i = 0; i < VPX_MAX_LAYERS; ++i) { - svc_params_.max_quantizers[i] = 56; - svc_params_.min_quantizers[i] = 2; - } - cfg_.rc_end_usage = VPX_CBR; - cfg_.g_lag_in_frames = 0; - cfg_.g_error_resilient = 0; - // 3 temporal layers - cfg_.ts_rate_decimator[0] = 4; - cfg_.ts_rate_decimator[1] = 2; - cfg_.ts_rate_decimator[2] = 1; - cfg_.temporal_layering_mode = 3; - - cfg_.rc_buf_initial_sz = 500; - cfg_.rc_buf_optimal_sz = 600; - cfg_.rc_buf_sz = 1000; - cfg_.rc_min_quantizer = 2; - cfg_.rc_max_quantizer = 56; - cfg_.g_threads = 1; - cfg_.kf_max_dist = 9999; - cfg_.rc_target_bitrate = 1600; - cfg_.rc_overshoot_pct = 50; - cfg_.rc_undershoot_pct = 50; - - cfg_.layer_target_bitrate[0] = 100; - cfg_.layer_target_bitrate[1] = 140; - cfg_.layer_target_bitrate[2] = 200; - cfg_.layer_target_bitrate[3] = 250; - cfg_.layer_target_bitrate[4] = 350; - cfg_.layer_target_bitrate[5] = 500; - cfg_.layer_target_bitrate[6] = 450; - cfg_.layer_target_bitrate[7] = 630; - cfg_.layer_target_bitrate[8] = 900; - } - - void SetConfigSvc() { - rc_cfg_.width = 1280; - rc_cfg_.height = 720; - rc_cfg_.max_quantizer = 56; - rc_cfg_.min_quantizer = 2; - rc_cfg_.target_bandwidth = 1600; - rc_cfg_.buf_initial_sz = 500; - rc_cfg_.buf_optimal_sz = 600; - rc_cfg_.buf_sz = 1000; - rc_cfg_.undershoot_pct = 50; - rc_cfg_.overshoot_pct = 50; - rc_cfg_.max_intra_bitrate_pct = 900; - rc_cfg_.framerate = 30.0; - rc_cfg_.ss_number_layers = 3; - rc_cfg_.ts_number_layers = 3; - rc_cfg_.rc_mode = VPX_CBR; - - rc_cfg_.scaling_factor_num[0] = 1; - rc_cfg_.scaling_factor_den[0] = 4; - rc_cfg_.scaling_factor_num[1] = 2; - rc_cfg_.scaling_factor_den[1] = 4; - rc_cfg_.scaling_factor_num[2] = 4; - rc_cfg_.scaling_factor_den[2] = 4; - - rc_cfg_.ts_rate_decimator[0] = 4; - rc_cfg_.ts_rate_decimator[1] = 2; - rc_cfg_.ts_rate_decimator[2] = 1; - - rc_cfg_.layer_target_bitrate[0] = 100; - rc_cfg_.layer_target_bitrate[1] = 140; - rc_cfg_.layer_target_bitrate[2] = 200; - rc_cfg_.layer_target_bitrate[3] = 250; - rc_cfg_.layer_target_bitrate[4] = 350; - rc_cfg_.layer_target_bitrate[5] = 500; - rc_cfg_.layer_target_bitrate[6] = 450; - rc_cfg_.layer_target_bitrate[7] = 630; - rc_cfg_.layer_target_bitrate[8] = 900; - - for (int sl = 0; sl < rc_cfg_.ss_number_layers; ++sl) { - for (int tl = 0; tl < rc_cfg_.ts_number_layers; ++tl) { - const int i = sl * rc_cfg_.ts_number_layers + tl; - rc_cfg_.max_quantizers[i] = 56; - rc_cfg_.min_quantizers[i] = 2; - } - } - } - - int aq_mode_; - std::unique_ptr rc_api_; - libvpx::VP9RateControlRtcConfig rc_cfg_; - vpx_svc_extra_cfg_t svc_params_; - libvpx::VP9FrameParamsQpRTC frame_params_; - bool encoder_exit_; - int current_superframe_; - uint32_t sizes_[8]; -}; - -TEST_P(RcInterfaceTest, OneLayer) { RunOneLayer(); } - -TEST_P(RcInterfaceTest, OneLayerVBRPeriodicKey) { RunOneLayerVBRPeriodicKey(); } - -TEST_P(RcInterfaceSvcTest, Svc) { RunSvc(); } - -VP9_INSTANTIATE_TEST_SUITE(RcInterfaceTest, ::testing::Values(0, 3), - ::testing::Values(VPX_CBR, VPX_VBR)); -VP9_INSTANTIATE_TEST_SUITE(RcInterfaceSvcTest, ::testing::Values(0)); -} // namespace - -int main(int argc, char **argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/media/libvpx/libvpx/test/realtime_test.cc b/media/libvpx/libvpx/test/realtime_test.cc --- a/media/libvpx/libvpx/test/realtime_test.cc +++ b/media/libvpx/libvpx/test/realtime_test.cc @@ -5,10 +5,12 @@ * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ +#include + #include "test/codec_factory.h" #include "test/encode_test_driver.h" #include "test/util.h" #include "test/video_source.h" #include "third_party/googletest/src/include/gtest/gtest.h" @@ -50,10 +52,26 @@ void FramePktHook(const vpx_codec_cx_pkt_t * /*pkt*/) override { frame_packets_++; } + bool IsVP9() const { +#if CONFIG_VP9_ENCODER + return codec_ == &libvpx_test::kVP9; +#else + return false; +#endif + } + + void TestIntegerOverflow(unsigned int width, unsigned int height) { + ::libvpx_test::RandomVideoSource video; + video.SetSize(width, height); + video.set_limit(20); + cfg_.rc_target_bitrate = UINT_MAX; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + } + int frame_packets_; }; TEST_P(RealtimeTest, RealtimeFirstPassProducesFrames) { ::libvpx_test::RandomVideoSource video; @@ -61,16 +79,26 @@ video.set_limit(kFramesToEncode); ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); EXPECT_EQ(kFramesToEncode, frame_packets_); } -TEST_P(RealtimeTest, IntegerOverflow) { - ::libvpx_test::RandomVideoSource video; - video.SetSize(800, 480); - video.set_limit(20); - cfg_.rc_target_bitrate = 140000000; - ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); +TEST_P(RealtimeTest, IntegerOverflow) { TestIntegerOverflow(2048, 2048); } + +TEST_P(RealtimeTest, IntegerOverflowLarge) { + if (IsVP9()) { +#if VPX_ARCH_X86_64 + TestIntegerOverflow(16384, 16384); +#else + TestIntegerOverflow(4096, 4096); +#endif + } else { + GTEST_SKIP() + << "TODO(https://crbug.com/webm/1748,https://crbug.com/webm/1751):" + << " Enable this test after bitstream errors & undefined sanitizer " + "warnings are fixed."; + // TestIntegerOverflow(16383, 16383); + } } VP8_INSTANTIATE_TEST_SUITE(RealtimeTest, ::testing::Values(::libvpx_test::kRealTime)); VP9_INSTANTIATE_TEST_SUITE(RealtimeTest, diff --git a/media/libvpx/libvpx/test/register_state_check.h b/media/libvpx/libvpx/test/register_state_check.h --- a/media/libvpx/libvpx/test/register_state_check.h +++ b/media/libvpx/libvpx/test/register_state_check.h @@ -33,10 +33,11 @@ #undef NOMINMAX #define NOMINMAX #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #endif +#include #include #include inline bool operator==(const M128A &lhs, const M128A &rhs) { return (lhs.Low == rhs.Low && lhs.High == rhs.High); @@ -79,14 +80,17 @@ bool initialized_; CONTEXT pre_context_; }; -#define ASM_REGISTER_STATE_CHECK(statement) \ - do { \ - libvpx_test::RegisterStateCheck reg_check; \ - statement; \ +#define ASM_REGISTER_STATE_CHECK(statement) \ + do { \ + { \ + libvpx_test::RegisterStateCheck reg_check; \ + statement; \ + } \ + _ReadWriteBarrier(); \ } while (false) } // namespace libvpx_test #elif defined(CONFIG_SHARED) && defined(HAVE_NEON_ASM) && \ @@ -119,15 +123,26 @@ } int64_t pre_store_[8]; }; +#if defined(__GNUC__) +#define ASM_REGISTER_STATE_CHECK(statement) \ + do { \ + { \ + libvpx_test::RegisterStateCheck reg_check; \ + statement; \ + } \ + __asm__ volatile("" ::: "memory"); \ + } while (false) +#else #define ASM_REGISTER_STATE_CHECK(statement) \ do { \ libvpx_test::RegisterStateCheck reg_check; \ statement; \ } while (false) +#endif } // namespace libvpx_test #else @@ -167,14 +182,17 @@ } uint16_t pre_fpu_env_[14]; }; -#define API_REGISTER_STATE_CHECK(statement) \ - do { \ - libvpx_test::RegisterStateCheckMMX reg_check; \ - ASM_REGISTER_STATE_CHECK(statement); \ +#define API_REGISTER_STATE_CHECK(statement) \ + do { \ + { \ + libvpx_test::RegisterStateCheckMMX reg_check; \ + ASM_REGISTER_STATE_CHECK(statement); \ + } \ + __asm__ volatile("" ::: "memory"); \ } while (false) } // namespace libvpx_test #endif // __GNUC__ diff --git a/media/libvpx/libvpx/test/sad_test.cc b/media/libvpx/libvpx/test/sad_test.cc --- a/media/libvpx/libvpx/test/sad_test.cc +++ b/media/libvpx/libvpx/test/sad_test.cc @@ -54,12 +54,10 @@ typedef void (*SadMxNx8Func)(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sad_array); -typedef TestParams SadMxNx8Param; - using libvpx_test::ACMRandom; namespace { template class SADTestBase : public ::testing::TestWithParam { @@ -264,34 +262,10 @@ ACMRandom rnd_; ParamType params_; }; -class SADx8Test : public SADTestBase { - public: - SADx8Test() : SADTestBase(GetParam()) {} - - protected: - void SADs(unsigned int *results) const { - const uint8_t *reference = GetReferenceFromOffset(0); - - ASM_REGISTER_STATE_CHECK(params_.func( - source_data_, source_stride_, reference, reference_stride_, results)); - } - - void CheckSADs() const { - uint32_t reference_sad; - DECLARE_ALIGNED(kDataAlignment, uint32_t, exp_sad[8]); - - SADs(exp_sad); - for (int offset = 0; offset < 8; ++offset) { - reference_sad = ReferenceSAD(offset); - EXPECT_EQ(reference_sad, exp_sad[offset]) << "offset " << offset; - } - } -}; - class SADx4Test : public SADTestBase { public: SADx4Test() : SADTestBase(GetParam()) {} protected: @@ -562,17 +536,10 @@ bit_depth_, elapsed_time); reference_stride_ = tmp_stride; } -TEST_P(SADx8Test, Regular) { - FillRandomWH(source_data_, source_stride_, params_.width, params_.height); - FillRandomWH(GetReferenceFromOffset(0), reference_stride_, params_.width + 8, - params_.height); - CheckSADs(); -} - //------------------------------------------------------------------------------ // C functions const SadMxNParam c_tests[] = { SadMxNParam(64, 64, &vpx_sad64x64_c), SadMxNParam(64, 32, &vpx_sad64x32_c), @@ -745,28 +712,10 @@ SadMxNx4Param(4, 4, &vpx_highbd_sad4x4x4d_c, 12), #endif // CONFIG_VP9_HIGHBITDEPTH }; INSTANTIATE_TEST_SUITE_P(C, SADx4Test, ::testing::ValuesIn(x4d_c_tests)); -// TODO(angiebird): implement the marked-down sad functions -const SadMxNx8Param x8_c_tests[] = { - // SadMxNx8Param(64, 64, &vpx_sad64x64x8_c), - // SadMxNx8Param(64, 32, &vpx_sad64x32x8_c), - // SadMxNx8Param(32, 64, &vpx_sad32x64x8_c), - SadMxNx8Param(32, 32, &vpx_sad32x32x8_c), - // SadMxNx8Param(32, 16, &vpx_sad32x16x8_c), - // SadMxNx8Param(16, 32, &vpx_sad16x32x8_c), - SadMxNx8Param(16, 16, &vpx_sad16x16x8_c), - SadMxNx8Param(16, 8, &vpx_sad16x8x8_c), - SadMxNx8Param(8, 16, &vpx_sad8x16x8_c), - SadMxNx8Param(8, 8, &vpx_sad8x8x8_c), - // SadMxNx8Param(8, 4, &vpx_sad8x4x8_c), - // SadMxNx8Param(4, 8, &vpx_sad4x8x8_c), - SadMxNx8Param(4, 4, &vpx_sad4x4x8_c), -}; -INSTANTIATE_TEST_SUITE_P(C, SADx8Test, ::testing::ValuesIn(x8_c_tests)); - //------------------------------------------------------------------------------ // ARM functions #if HAVE_NEON const SadMxNParam neon_tests[] = { SadMxNParam(64, 64, &vpx_sad64x64_neon), @@ -990,22 +939,10 @@ #if HAVE_SSSE3 // Only functions are x3, which do not have tests. #endif // HAVE_SSSE3 -#if HAVE_SSE4_1 -const SadMxNx8Param x8_sse4_1_tests[] = { - SadMxNx8Param(16, 16, &vpx_sad16x16x8_sse4_1), - SadMxNx8Param(16, 8, &vpx_sad16x8x8_sse4_1), - SadMxNx8Param(8, 16, &vpx_sad8x16x8_sse4_1), - SadMxNx8Param(8, 8, &vpx_sad8x8x8_sse4_1), - SadMxNx8Param(4, 4, &vpx_sad4x4x8_sse4_1), -}; -INSTANTIATE_TEST_SUITE_P(SSE4_1, SADx8Test, - ::testing::ValuesIn(x8_sse4_1_tests)); -#endif // HAVE_SSE4_1 - #if HAVE_AVX2 const SadMxNParam avx2_tests[] = { SadMxNParam(64, 64, &vpx_sad64x64_avx2), SadMxNParam(64, 32, &vpx_sad64x32_avx2), SadMxNParam(32, 64, &vpx_sad32x64_avx2), @@ -1027,15 +964,10 @@ SadMxNx4Param(64, 64, &vpx_sad64x64x4d_avx2), SadMxNx4Param(32, 32, &vpx_sad32x32x4d_avx2), }; INSTANTIATE_TEST_SUITE_P(AVX2, SADx4Test, ::testing::ValuesIn(x4d_avx2_tests)); -const SadMxNx8Param x8_avx2_tests[] = { - // SadMxNx8Param(64, 64, &vpx_sad64x64x8_c), - SadMxNx8Param(32, 32, &vpx_sad32x32x8_avx2), -}; -INSTANTIATE_TEST_SUITE_P(AVX2, SADx8Test, ::testing::ValuesIn(x8_avx2_tests)); #endif // HAVE_AVX2 #if HAVE_AVX512 const SadMxNx4Param x4d_avx512_tests[] = { SadMxNx4Param(64, 64, &vpx_sad64x64x4d_avx512), @@ -1194,6 +1126,34 @@ SadMxNx4Param(4, 8, &vpx_sad4x8x4d_mmi), SadMxNx4Param(4, 4, &vpx_sad4x4x4d_mmi), }; INSTANTIATE_TEST_SUITE_P(MMI, SADx4Test, ::testing::ValuesIn(x4d_mmi_tests)); #endif // HAVE_MMI + +//------------------------------------------------------------------------------ +// loongarch functions +#if HAVE_LSX +const SadMxNParam lsx_tests[] = { + SadMxNParam(64, 64, &vpx_sad64x64_lsx), + SadMxNParam(32, 32, &vpx_sad32x32_lsx), + SadMxNParam(16, 16, &vpx_sad16x16_lsx), +}; +INSTANTIATE_TEST_SUITE_P(LSX, SADTest, ::testing::ValuesIn(lsx_tests)); + +const SadMxNAvgParam avg_lsx_tests[] = { + SadMxNAvgParam(64, 64, &vpx_sad64x64_avg_lsx), + SadMxNAvgParam(32, 32, &vpx_sad32x32_avg_lsx), +}; +INSTANTIATE_TEST_SUITE_P(LSX, SADavgTest, ::testing::ValuesIn(avg_lsx_tests)); + +const SadMxNx4Param x4d_lsx_tests[] = { + SadMxNx4Param(64, 64, &vpx_sad64x64x4d_lsx), + SadMxNx4Param(64, 32, &vpx_sad64x32x4d_lsx), + SadMxNx4Param(32, 64, &vpx_sad32x64x4d_lsx), + SadMxNx4Param(32, 32, &vpx_sad32x32x4d_lsx), + SadMxNx4Param(16, 16, &vpx_sad16x16x4d_lsx), + SadMxNx4Param(8, 8, &vpx_sad8x8x4d_lsx), +}; +INSTANTIATE_TEST_SUITE_P(LSX, SADx4Test, ::testing::ValuesIn(x4d_lsx_tests)); +#endif // HAVE_LSX + } // namespace diff --git a/media/libvpx/libvpx/test/set_roi.cc b/media/libvpx/libvpx/test/set_roi.cc --- a/media/libvpx/libvpx/test/set_roi.cc +++ b/media/libvpx/libvpx/test/set_roi.cc @@ -159,8 +159,8 @@ } // Free allocated memory if (cpi.segmentation_map) vpx_free(cpi.segmentation_map); if (roi_map) vpx_free(roi_map); -}; +} } // namespace diff --git a/media/libvpx/libvpx/test/simple_encode_test.cc b/media/libvpx/libvpx/test/simple_encode_test.cc --- a/media/libvpx/libvpx/test/simple_encode_test.cc +++ b/media/libvpx/libvpx/test/simple_encode_test.cc @@ -35,17 +35,18 @@ const int height_ = 288; const int frame_rate_num_ = 30; const int frame_rate_den_ = 1; const int target_bitrate_ = 1000; const int num_frames_ = 17; + const int target_level_ = LEVEL_UNKNOWN; const std::string in_file_path_str_ = libvpx_test::GetDataPath() + "/bus_352x288_420_f20_b8.yuv"; }; TEST_F(SimpleEncodeTest, ComputeFirstPassStats) { SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, - target_bitrate_, num_frames_, + target_bitrate_, num_frames_, target_level_, in_file_path_str_.c_str()); simple_encode.ComputeFirstPassStats(); std::vector> frame_stats = simple_encode.ObserveFirstPassStats(); EXPECT_EQ(frame_stats.size(), static_cast(num_frames_)); @@ -62,11 +63,11 @@ } } TEST_F(SimpleEncodeTest, ObserveFirstPassMotionVectors) { SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, - target_bitrate_, num_frames_, + target_bitrate_, num_frames_, target_level_, in_file_path_str_.c_str()); simple_encode.ComputeFirstPassStats(); std::vector> fps_motion_vectors = simple_encode.ObserveFirstPassMotionVectors(); EXPECT_EQ(fps_motion_vectors.size(), static_cast(num_frames_)); @@ -84,20 +85,20 @@ } } TEST_F(SimpleEncodeTest, GetCodingFrameNum) { SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, - target_bitrate_, num_frames_, + target_bitrate_, num_frames_, target_level_, in_file_path_str_.c_str()); simple_encode.ComputeFirstPassStats(); const int num_coding_frames = simple_encode.GetCodingFrameNum(); EXPECT_EQ(num_coding_frames, 19); } TEST_F(SimpleEncodeTest, EncodeFrame) { SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, - target_bitrate_, num_frames_, + target_bitrate_, num_frames_, target_level_, in_file_path_str_.c_str()); simple_encode.ComputeFirstPassStats(); int num_coding_frames = simple_encode.GetCodingFrameNum(); EXPECT_GE(num_coding_frames, num_frames_); simple_encode.StartEncode(); @@ -136,11 +137,11 @@ simple_encode.EndEncode(); } TEST_F(SimpleEncodeTest, ObserveKeyFrameMap) { SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, - target_bitrate_, num_frames_, + target_bitrate_, num_frames_, target_level_, in_file_path_str_.c_str()); simple_encode.ComputeFirstPassStats(); std::vector key_frame_map = simple_encode.ObserveKeyFrameMap(); EXPECT_EQ(key_frame_map.size(), static_cast(num_frames_)); simple_encode.StartEncode(); @@ -165,11 +166,11 @@ simple_encode.EndEncode(); } TEST_F(SimpleEncodeTest, EncodeFrameWithTargetFrameBits) { SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, - target_bitrate_, num_frames_, + target_bitrate_, num_frames_, target_level_, in_file_path_str_.c_str()); simple_encode.ComputeFirstPassStats(); const int num_coding_frames = simple_encode.GetCodingFrameNum(); simple_encode.StartEncode(); for (int i = 0; i < num_coding_frames; ++i) { @@ -203,11 +204,11 @@ simple_encode.EndEncode(); } TEST_F(SimpleEncodeTest, EncodeFrameWithQuantizeIndex) { SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, - target_bitrate_, num_frames_, + target_bitrate_, num_frames_, target_level_, in_file_path_str_.c_str()); simple_encode.ComputeFirstPassStats(); const int num_coding_frames = simple_encode.GetCodingFrameNum(); simple_encode.StartEncode(); for (int i = 0; i < num_coding_frames; ++i) { @@ -235,11 +236,11 @@ std::vector ref_show_idx_list; { // The first encode. SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, target_bitrate_, num_frames_, - in_file_path_str_.c_str()); + target_level_, in_file_path_str_.c_str()); simple_encode.ComputeFirstPassStats(); const int num_coding_frames = simple_encode.GetCodingFrameNum(); simple_encode.StartEncode(); for (int i = 0; i < num_coding_frames; ++i) { EncodeFrameResult encode_frame_result; @@ -255,11 +256,11 @@ } { // The second encode with quantize index got from the first encode. SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, target_bitrate_, num_frames_, - in_file_path_str_.c_str()); + target_level_, in_file_path_str_.c_str()); simple_encode.ComputeFirstPassStats(); const int num_coding_frames = simple_encode.GetCodingFrameNum(); EXPECT_EQ(static_cast(num_coding_frames), quantize_index_list.size()); simple_encode.StartEncode(); @@ -284,11 +285,11 @@ const int num_rows_4x4 = GetNumUnit4x4(width_); const int num_cols_4x4 = GetNumUnit4x4(height_); const int num_units_4x4 = num_rows_4x4 * num_cols_4x4; // The first encode. SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, - target_bitrate_, num_frames_, + target_bitrate_, num_frames_, target_level_, in_file_path_str_.c_str()); simple_encode.ComputeFirstPassStats(); const int num_coding_frames = simple_encode.GetCodingFrameNum(); std::vector partition_info_list(num_units_4x4 * num_coding_frames); @@ -307,11 +308,11 @@ } simple_encode.EndEncode(); // The second encode. SimpleEncode simple_encode_2(width_, height_, frame_rate_num_, frame_rate_den_, target_bitrate_, num_frames_, - in_file_path_str_.c_str()); + target_level_, in_file_path_str_.c_str()); simple_encode_2.ComputeFirstPassStats(); const int num_coding_frames_2 = simple_encode_2.GetCodingFrameNum(); simple_encode_2.StartEncode(); for (int i = 0; i < num_coding_frames_2; ++i) { EncodeFrameResult encode_frame_result; @@ -355,11 +356,11 @@ const int num_rows_4x4 = GetNumUnit4x4(width_); const int num_cols_4x4 = GetNumUnit4x4(height_); const int num_units_4x4 = num_rows_4x4 * num_cols_4x4; // The first encode. SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, - target_bitrate_, num_frames_, + target_bitrate_, num_frames_, target_level_, in_file_path_str_.c_str()); simple_encode.ComputeFirstPassStats(); const int num_coding_frames = simple_encode.GetCodingFrameNum(); std::vector partition_info_list(num_units_4x4 * num_coding_frames); @@ -375,11 +376,11 @@ } simple_encode.EndEncode(); // The second encode. SimpleEncode simple_encode_2(width_, height_, frame_rate_num_, frame_rate_den_, target_bitrate_, num_frames_, - in_file_path_str_.c_str()); + target_level_, in_file_path_str_.c_str()); simple_encode_2.ComputeFirstPassStats(); const int num_coding_frames_2 = simple_encode_2.GetCodingFrameNum(); simple_encode_2.StartEncode(); for (int i = 0; i < num_coding_frames_2; ++i) { EncodeFrameResult encode_frame_result; @@ -415,11 +416,11 @@ std::vector gop_map(num_frames_, 0); { // The first encode. SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, target_bitrate_, num_frames_, - in_file_path_str_.c_str()); + target_level_, in_file_path_str_.c_str()); simple_encode.ComputeFirstPassStats(); simple_encode.StartEncode(); int coded_show_frame_count = 0; while (coded_show_frame_count < num_frames_) { @@ -447,11 +448,11 @@ { // The second encode with quantize index got from the first encode. // The external arfs are the same as the first encode. SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, target_bitrate_, num_frames_, - in_file_path_str_.c_str()); + target_level_, in_file_path_str_.c_str()); simple_encode.ComputeFirstPassStats(); simple_encode.SetExternalGroupOfPicturesMap(gop_map.data(), gop_map.size()); const int num_coding_frames = simple_encode.GetCodingFrameNum(); EXPECT_EQ(static_cast(num_coding_frames), quantize_index_list.size()); @@ -469,11 +470,11 @@ } } TEST_F(SimpleEncodeTest, SetExternalGroupOfPicturesMap) { SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, - target_bitrate_, num_frames_, + target_bitrate_, num_frames_, target_level_, in_file_path_str_.c_str()); simple_encode.ComputeFirstPassStats(); std::vector gop_map(num_frames_, 0); @@ -539,11 +540,11 @@ TEST_F(SimpleEncodeTest, GetEncodeFrameInfo) { // Makes sure that the encode_frame_info obtained from GetEncodeFrameInfo() // matches the counterpart in encode_frame_result obtained from EncodeFrame() SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, - target_bitrate_, num_frames_, + target_bitrate_, num_frames_, target_level_, in_file_path_str_.c_str()); simple_encode.ComputeFirstPassStats(); const int num_coding_frames = simple_encode.GetCodingFrameNum(); simple_encode.StartEncode(); for (int i = 0; i < num_coding_frames; ++i) { @@ -556,11 +557,11 @@ simple_encode.EndEncode(); } TEST_F(SimpleEncodeTest, GetFramePixelCount) { SimpleEncode simple_encode(width_, height_, frame_rate_num_, frame_rate_den_, - target_bitrate_, num_frames_, + target_bitrate_, num_frames_, target_level_, in_file_path_str_.c_str()); EXPECT_EQ(simple_encode.GetFramePixelCount(), static_cast(width_ * height_ * 3 / 2)); } diff --git a/media/libvpx/libvpx/test/svc_datarate_test.cc b/media/libvpx/libvpx/test/svc_datarate_test.cc --- a/media/libvpx/libvpx/test/svc_datarate_test.cc +++ b/media/libvpx/libvpx/test/svc_datarate_test.cc @@ -1352,11 +1352,10 @@ cfg_.kf_max_dist = kf_dist; key_frame_spacing_ = kf_dist; ResetModel(); AssignLayerBitrates(); ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); - // TODO(jianj): webm:1554 CheckLayerRateTargeting(number_spatial_layers_, number_temporal_layers_, 0.70, 1.15); #if CONFIG_VP9_DECODER // The non-reference frames are expected to be mismatched frames as the // encoder will avoid loopfilter on these frames. diff --git a/media/libvpx/libvpx/test/svc_end_to_end_test.cc b/media/libvpx/libvpx/test/svc_end_to_end_test.cc --- a/media/libvpx/libvpx/test/svc_end_to_end_test.cc +++ b/media/libvpx/libvpx/test/svc_end_to_end_test.cc @@ -13,10 +13,11 @@ #include "test/encode_test_driver.h" #include "test/i420_video_source.h" #include "test/svc_test.h" #include "test/util.h" #include "test/y4m_video_source.h" +#include "vp9/common/vp9_onyxc_int.h" #include "vpx/vpx_codec.h" #include "vpx_ports/bitops.h" namespace svc_test { namespace { @@ -137,10 +138,95 @@ virtual bool DoDecode() const { return current_video_frame_ >= frame_to_start_decode_; } + // Example pattern for spatial layers and 2 temporal layers used in the + // bypass/flexible mode. The pattern corresponds to the pattern + // VP9E_TEMPORAL_LAYERING_MODE_0101 (temporal_layering_mode == 2) used in + // non-flexible mode. + void set_frame_flags_bypass_mode( + int tl, int num_spatial_layers, int is_key_frame, + vpx_svc_ref_frame_config_t *ref_frame_config) { + int sl; + for (sl = 0; sl < num_spatial_layers; ++sl) + ref_frame_config->update_buffer_slot[sl] = 0; + + for (sl = 0; sl < num_spatial_layers; ++sl) { + // Set the buffer idx. + if (tl == 0) { + ref_frame_config->lst_fb_idx[sl] = sl; + if (sl) { + if (is_key_frame) { + ref_frame_config->lst_fb_idx[sl] = sl - 1; + ref_frame_config->gld_fb_idx[sl] = sl; + } else { + ref_frame_config->gld_fb_idx[sl] = sl - 1; + } + } else { + ref_frame_config->gld_fb_idx[sl] = 0; + } + ref_frame_config->alt_fb_idx[sl] = 0; + } else if (tl == 1) { + ref_frame_config->lst_fb_idx[sl] = sl; + ref_frame_config->gld_fb_idx[sl] = + (sl == 0) ? 0 : num_spatial_layers + sl - 1; + ref_frame_config->alt_fb_idx[sl] = num_spatial_layers + sl; + } + // Set the reference and update flags. + if (!tl) { + if (!sl) { + // Base spatial and base temporal (sl = 0, tl = 0) + ref_frame_config->reference_last[sl] = 1; + ref_frame_config->reference_golden[sl] = 0; + ref_frame_config->reference_alt_ref[sl] = 0; + ref_frame_config->update_buffer_slot[sl] |= + 1 << ref_frame_config->lst_fb_idx[sl]; + } else { + if (is_key_frame) { + ref_frame_config->reference_last[sl] = 1; + ref_frame_config->reference_golden[sl] = 0; + ref_frame_config->reference_alt_ref[sl] = 0; + ref_frame_config->update_buffer_slot[sl] |= + 1 << ref_frame_config->gld_fb_idx[sl]; + } else { + // Non-zero spatiall layer. + ref_frame_config->reference_last[sl] = 1; + ref_frame_config->reference_golden[sl] = 1; + ref_frame_config->reference_alt_ref[sl] = 1; + ref_frame_config->update_buffer_slot[sl] |= + 1 << ref_frame_config->lst_fb_idx[sl]; + } + } + } else if (tl == 1) { + if (!sl) { + // Base spatial and top temporal (tl = 1) + ref_frame_config->reference_last[sl] = 1; + ref_frame_config->reference_golden[sl] = 0; + ref_frame_config->reference_alt_ref[sl] = 0; + ref_frame_config->update_buffer_slot[sl] |= + 1 << ref_frame_config->alt_fb_idx[sl]; + } else { + // Non-zero spatial. + if (sl < num_spatial_layers - 1) { + ref_frame_config->reference_last[sl] = 1; + ref_frame_config->reference_golden[sl] = 1; + ref_frame_config->reference_alt_ref[sl] = 0; + ref_frame_config->update_buffer_slot[sl] |= + 1 << ref_frame_config->alt_fb_idx[sl]; + } else if (sl == num_spatial_layers - 1) { + // Top spatial and top temporal (non-reference -- doesn't + // update any reference buffers). + ref_frame_config->reference_last[sl] = 1; + ref_frame_config->reference_golden[sl] = 1; + ref_frame_config->reference_alt_ref[sl] = 0; + } + } + } + } + } + virtual void PreEncodeFrameHook(::libvpx_test::VideoSource *video, ::libvpx_test::Encoder *encoder) { current_video_frame_ = video->frame(); PreEncodeFrameHookSetup(video, encoder); if (video->frame() == 0) { @@ -156,10 +242,25 @@ // mismatch check on color space setting. encoder->Control(VP9E_SET_COLOR_SPACE, VPX_CS_BT_601); encoder->Control(VP9E_SET_DISABLE_LOOPFILTER, loopfilter_off_); } + if (flexible_mode_) { + vpx_svc_layer_id_t layer_id; + layer_id.spatial_layer_id = 0; + layer_id.temporal_layer_id = (video->frame() % 2 != 0); + temporal_layer_id_ = layer_id.temporal_layer_id; + for (int i = 0; i < number_spatial_layers_; i++) { + layer_id.temporal_layer_id_per_spatial[i] = temporal_layer_id_; + ref_frame_config_.duration[i] = 1; + } + encoder->Control(VP9E_SET_SVC_LAYER_ID, &layer_id); + set_frame_flags_bypass_mode(layer_id.temporal_layer_id, + number_spatial_layers_, 0, + &ref_frame_config_); + encoder->Control(VP9E_SET_SVC_REF_FRAME_CONFIG, &ref_frame_config_); + } if (video->frame() == frame_to_sync_) { encoder->Control(VP9E_SET_SVC_SPATIAL_LAYER_SYNC, &svc_layer_sync_); } } @@ -169,13 +270,18 @@ if (video->frame() < frame_to_sync_) { if (decode_to_layer_before_sync_ >= 0) decoder->Control(VP9_DECODE_SVC_SPATIAL_LAYER, decode_to_layer_before_sync_); } else { - if (decode_to_layer_after_sync_ >= 0) - decoder->Control(VP9_DECODE_SVC_SPATIAL_LAYER, - decode_to_layer_after_sync_); + if (decode_to_layer_after_sync_ >= 0) { + int decode_to_layer = decode_to_layer_after_sync_; + // Overlay frame is additional layer for intra-only. + if (video->frame() == frame_to_sync_ && intra_only_test_ && + decode_to_layer_after_sync_ == 0 && number_spatial_layers_ > 1) + decode_to_layer += 1; + decoder->Control(VP9_DECODE_SVC_SPATIAL_LAYER, decode_to_layer); + } } } #endif virtual void FramePktHook(const vpx_codec_cx_pkt_t *pkt) { @@ -219,10 +325,12 @@ bool intra_only_test_; int loopfilter_off_; vpx_svc_spatial_layer_sync_t svc_layer_sync_; unsigned int mismatch_nframes_; unsigned int num_nonref_frames_; + bool flexible_mode_; + vpx_svc_ref_frame_config_t ref_frame_config_; private: virtual void SetConfig(const int num_temporal_layer) { cfg_.rc_buf_initial_sz = 500; cfg_.rc_buf_optimal_sz = 500; @@ -244,11 +352,11 @@ cfg_.ts_rate_decimator[0] = 2; cfg_.ts_rate_decimator[1] = 1; cfg_.temporal_layering_mode = 2; } else if (num_temporal_layer == 1) { cfg_.ts_rate_decimator[0] = 1; - cfg_.temporal_layering_mode = 1; + cfg_.temporal_layering_mode = 0; } } }; // Test for sync layer for 1 pass CBR SVC: 3 spatial layers and @@ -268,10 +376,11 @@ svc_layer_sync_.spatial_layer_sync[0] = 1; ::libvpx_test::Y4mVideoSource video("niklas_1280_720_30.y4m", 0, 60); cfg_.rc_target_bitrate = 600; + flexible_mode_ = false; AssignLayerBitrates(); ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); #if CONFIG_VP9_DECODER // The non-reference frames are expected to be mismatched frames as the // encoder will avoid loopfilter on these frames. @@ -295,10 +404,11 @@ svc_layer_sync_.spatial_layer_sync[1] = 1; ::libvpx_test::I420VideoSource video("niklas_640_480_30.yuv", 640, 480, 30, 1, 0, 400); cfg_.rc_target_bitrate = 400; + flexible_mode_ = false; AssignLayerBitrates(); ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); #if CONFIG_VP9_DECODER // The non-reference frames are expected to be mismatched frames as the // encoder will avoid loopfilter on these frames. @@ -322,10 +432,11 @@ svc_layer_sync_.spatial_layer_sync[1] = 0; svc_layer_sync_.spatial_layer_sync[2] = 1; ::libvpx_test::Y4mVideoSource video("niklas_1280_720_30.y4m", 0, 60); cfg_.rc_target_bitrate = 600; + flexible_mode_ = false; AssignLayerBitrates(); ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); #if CONFIG_VP9_DECODER // The non-reference frames are expected to be mismatched frames as the // encoder will avoid loopfilter on these frames. @@ -349,10 +460,11 @@ svc_layer_sync_.spatial_layer_sync[1] = 1; svc_layer_sync_.spatial_layer_sync[2] = 1; ::libvpx_test::Y4mVideoSource video("niklas_1280_720_30.y4m", 0, 60); cfg_.rc_target_bitrate = 600; + flexible_mode_ = false; AssignLayerBitrates(); ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); #if CONFIG_VP9_DECODER // The non-reference frames are expected to be mismatched frames as the // encoder will avoid loopfilter on these frames. @@ -378,29 +490,86 @@ svc_layer_sync_.spatial_layer_sync[1] = 1; ::libvpx_test::I420VideoSource video("niklas_640_480_30.yuv", 640, 480, 30, 1, 0, 400); cfg_.rc_target_bitrate = 400; + flexible_mode_ = false; AssignLayerBitrates(); ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); #if CONFIG_VP9_DECODER // The non-reference frames are expected to be mismatched frames as the // encoder will avoid loopfilter on these frames. EXPECT_EQ(GetNonRefFrames(), GetMismatchFrames()); #endif } #endif +// Encode 3 spatial, 2 temporal layer in flexible mode but don't +// start decoding. During the sequence insert intra-only on base/qvga +// layer at frame 20 and start decoding only QVGA layer from there. +TEST_P(SyncFrameOnePassCbrSvc, + OnePassCbrSvc3SL3TLSyncFrameStartDecodeOnIntraOnlyQVGAFlex) { + SetSvcConfig(3, 2); + frame_to_start_decode_ = 20; + frame_to_sync_ = 20; + decode_to_layer_before_sync_ = 2; + decode_to_layer_after_sync_ = 0; + intra_only_test_ = true; + + // Set up svc layer sync structure. + svc_layer_sync_.base_layer_intra_only = 1; + svc_layer_sync_.spatial_layer_sync[0] = 1; + svc_layer_sync_.spatial_layer_sync[1] = 0; + svc_layer_sync_.spatial_layer_sync[2] = 0; + + ::libvpx_test::Y4mVideoSource video("niklas_1280_720_30.y4m", 0, 60); + cfg_.rc_target_bitrate = 600; + flexible_mode_ = true; + AssignLayerBitrates(); + cfg_.temporal_layering_mode = VP9E_TEMPORAL_LAYERING_MODE_BYPASS; + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + // Can't check mismatch here because only base is decoded at + // frame sync, whereas encoder continues encoding all layers. +} + +// Encode 3 spatial, 3 temporal layer but don't start decoding. +// During the sequence insert intra-only on base/qvga layer at frame 20 +// and start decoding only QVGA layer from there. +TEST_P(SyncFrameOnePassCbrSvc, + OnePassCbrSvc3SL3TLSyncFrameStartDecodeOnIntraOnlyQVGA) { + SetSvcConfig(3, 3); + frame_to_start_decode_ = 20; + frame_to_sync_ = 20; + decode_to_layer_before_sync_ = 2; + decode_to_layer_after_sync_ = 0; + intra_only_test_ = true; + + // Set up svc layer sync structure. + svc_layer_sync_.base_layer_intra_only = 1; + svc_layer_sync_.spatial_layer_sync[0] = 1; + svc_layer_sync_.spatial_layer_sync[1] = 0; + svc_layer_sync_.spatial_layer_sync[2] = 0; + + ::libvpx_test::Y4mVideoSource video("niklas_1280_720_30.y4m", 0, 60); + cfg_.rc_target_bitrate = 600; + flexible_mode_ = false; + AssignLayerBitrates(); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + // Can't check mismatch here because only base is decoded at + // frame sync, whereas encoder continues encoding all layers. +} + // Start decoding from beginning of sequence, during sequence insert intra-only // on base/qvga layer. Decode all layers. TEST_P(SyncFrameOnePassCbrSvc, OnePassCbrSvc3SL3TLSyncFrameIntraOnlyQVGA) { SetSvcConfig(3, 3); frame_to_start_decode_ = 0; frame_to_sync_ = 20; decode_to_layer_before_sync_ = 2; - // The superframe containing intra-only layer will have 4 frames. Thus set the - // layer to decode after sync frame to 3. + // The superframe containing intra-only layer will have +1 frames. Thus set + // the layer to decode after sync frame to +1 from + // decode_to_layer_before_sync. decode_to_layer_after_sync_ = 3; intra_only_test_ = true; // Set up svc layer sync structure. svc_layer_sync_.base_layer_intra_only = 1; @@ -408,10 +577,11 @@ svc_layer_sync_.spatial_layer_sync[1] = 0; svc_layer_sync_.spatial_layer_sync[2] = 0; ::libvpx_test::Y4mVideoSource video("niklas_1280_720_30.y4m", 0, 60); cfg_.rc_target_bitrate = 600; + flexible_mode_ = false; AssignLayerBitrates(); ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); #if CONFIG_VP9_DECODER // The non-reference frames are expected to be mismatched frames as the // encoder will avoid loopfilter on these frames. @@ -424,12 +594,13 @@ TEST_P(SyncFrameOnePassCbrSvc, OnePassCbrSvc3SL3TLSyncFrameIntraOnlyVGA) { SetSvcConfig(3, 3); frame_to_start_decode_ = 0; frame_to_sync_ = 20; decode_to_layer_before_sync_ = 2; - // The superframe containing intra-only layer will have 4 frames. Thus set the - // layer to decode after sync frame to 3. + // The superframe containing intra-only layer will have +1 frames. Thus set + // the layer to decode after sync frame to +1 from + // decode_to_layer_before_sync. decode_to_layer_after_sync_ = 3; intra_only_test_ = true; // Set up svc layer sync structure. svc_layer_sync_.base_layer_intra_only = 1; @@ -437,10 +608,11 @@ svc_layer_sync_.spatial_layer_sync[1] = 1; svc_layer_sync_.spatial_layer_sync[2] = 0; ::libvpx_test::Y4mVideoSource video("niklas_1280_720_30.y4m", 0, 60); cfg_.rc_target_bitrate = 600; + flexible_mode_ = false; AssignLayerBitrates(); ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); #if CONFIG_VP9_DECODER // The non-reference frames are expected to be mismatched frames as the // encoder will avoid loopfilter on these frames. @@ -462,10 +634,11 @@ svc_layer_sync_.base_layer_intra_only = 1; svc_layer_sync_.spatial_layer_sync[0] = 1; ::libvpx_test::Y4mVideoSource video("niklas_1280_720_30.y4m", 0, 60); cfg_.rc_target_bitrate = 600; + flexible_mode_ = false; AssignLayerBitrates(); ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); #if CONFIG_VP9_DECODER // The non-reference frames are expected to be mismatched frames as the // encoder will avoid loopfilter on these frames. diff --git a/media/libvpx/libvpx/test/test-data.mk b/media/libvpx/libvpx/test/test-data.mk --- a/media/libvpx/libvpx/test/test-data.mk +++ b/media/libvpx/libvpx/test/test-data.mk @@ -4,10 +4,11 @@ LIBVPX_TEST_DATA-$(CONFIG_ENCODERS) += hantro_collage_w352h288.yuv LIBVPX_TEST_DATA-$(CONFIG_ENCODERS) += hantro_collage_w352h288_nv12.yuv LIBVPX_TEST_DATA-$(CONFIG_ENCODERS) += hantro_odd.yuv LIBVPX_TEST_DATA-$(CONFIG_ENCODERS) += desktop_office1.1280_720-020.yuv LIBVPX_TEST_DATA-$(CONFIG_ENCODERS) += slides_code_term_web_plot.1920_1080.yuv +LIBVPX_TEST_DATA-$(CONFIG_ENCODERS) += desktopqvga.320_240.yuv LIBVPX_TEST_DATA-$(CONFIG_ENCODERS) += park_joy_90p_10_420_20f.y4m LIBVPX_TEST_DATA-$(CONFIG_ENCODERS) += park_joy_90p_10_422_20f.y4m LIBVPX_TEST_DATA-$(CONFIG_ENCODERS) += park_joy_90p_10_444_20f.y4m LIBVPX_TEST_DATA-$(CONFIG_ENCODERS) += park_joy_90p_10_440.yuv diff --git a/media/libvpx/libvpx/test/test-data.sha1 b/media/libvpx/libvpx/test/test-data.sha1 --- a/media/libvpx/libvpx/test/test-data.sha1 +++ b/media/libvpx/libvpx/test/test-data.sha1 @@ -867,5 +867,6 @@ bac455906360b45338a16dd626ac5f19bc36a307 *desktop_office1.1280_720-020.yuv 094be4b80fa30bd227149ea16ab6476d549ea092 *slides_code_term_web_plot.1920_1080.yuv 518a0be998afece76d3df76047d51e256c591ff2 *invalid-bug-148271109.ivf d3964f9dad9f60363c81b688324d95b4ec7c8038 *invalid-bug-148271109.ivf.res ad18ca16f0a249fb3b7c38de0d9b327fed273f96 *hantro_collage_w352h288_nv12.yuv +8a0b2c350539859463d3546a67876c83ff6ff0ac *desktopqvga.320_240.yuv diff --git a/media/libvpx/libvpx/test/test.mk b/media/libvpx/libvpx/test/test.mk --- a/media/libvpx/libvpx/test/test.mk +++ b/media/libvpx/libvpx/test/test.mk @@ -154,10 +154,11 @@ LIBVPX_TEST_SRCS-yes += partial_idct_test.cc LIBVPX_TEST_SRCS-yes += superframe_test.cc LIBVPX_TEST_SRCS-yes += tile_independence_test.cc LIBVPX_TEST_SRCS-yes += vp9_boolcoder_test.cc LIBVPX_TEST_SRCS-yes += vp9_encoder_parms_get_to_decoder.cc +LIBVPX_TEST_SRCS-yes += vp9_roi_test.cc endif LIBVPX_TEST_SRCS-yes += convolve_test.cc LIBVPX_TEST_SRCS-yes += lpf_test.cc LIBVPX_TEST_SRCS-yes += vp9_intrapred_test.cc @@ -211,13 +212,15 @@ endif TEST_INTRA_PRED_SPEED_SRCS-yes := test_intra_pred_speed.cc TEST_INTRA_PRED_SPEED_SRCS-yes += ../md5_utils.h ../md5_utils.c -RC_INTERFACE_TEST_SRCS-$(CONFIG_VP9_ENCODER) := ratectrl_rtc_test.cc -RC_INTERFACE_TEST_SRCS-$(CONFIG_VP9_ENCODER) += encode_test_driver.cc -RC_INTERFACE_TEST_SRCS-$(CONFIG_VP9_ENCODER) += encode_test_driver.h +RC_INTERFACE_TEST_SRCS-yes := test_rc_interface.cc +RC_INTERFACE_TEST_SRCS-$(CONFIG_VP9_ENCODER) += vp9_ratectrl_rtc_test.cc +RC_INTERFACE_TEST_SRCS-$(CONFIG_VP8_ENCODER) += vp8_ratectrl_rtc_test.cc +RC_INTERFACE_TEST_SRCS-$(CONFIG_ENCODERS) += encode_test_driver.cc +RC_INTERFACE_TEST_SRCS-$(CONFIG_ENCODERS) += encode_test_driver.h RC_INTERFACE_TEST_SRCS-yes += decode_test_driver.cc RC_INTERFACE_TEST_SRCS-yes += decode_test_driver.h RC_INTERFACE_TEST_SRCS-yes += codec_factory.h endif # CONFIG_SHARED diff --git a/media/libvpx/libvpx/test/test_intra_pred_speed.cc b/media/libvpx/libvpx/test/test_intra_pred_speed.cc --- a/media/libvpx/libvpx/test/test_intra_pred_speed.cc +++ b/media/libvpx/libvpx/test/test_intra_pred_speed.cc @@ -46,15 +46,13 @@ const int mask = (1 << bd) - 1; for (int i = 0; i < kTotalPixels; ++i) ref_src[i] = rnd.Rand16() & mask; for (int i = 0; i < kBPS; ++i) left[i] = rnd.Rand16() & mask; for (int i = -1; i < kBPS; ++i) above[i] = rnd.Rand16() & mask; - // some code assumes the top row has been extended: - // d45/d63 C-code, for instance, but not the assembly. - // TODO(jzern): this style of extension isn't strictly necessary. + // d45/d63 require the top row to be extended. ASSERT_LE(block_size, kBPS); - for (int i = block_size; i < 2 * kBPS; ++i) { + for (int i = block_size; i < 2 * block_size; ++i) { above[i] = above[block_size - 1]; } } DECLARE_ALIGNED(16, Pixel, src[kTotalPixels]); diff --git a/media/libvpx/libvpx/test/test_rc_interface.cc b/media/libvpx/libvpx/test/test_rc_interface.cc new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/test/test_rc_interface.cc @@ -0,0 +1,6 @@ +#include "third_party/googletest/src/include/gtest/gtest.h" + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/media/libvpx/libvpx/test/variance_test.cc b/media/libvpx/libvpx/test/variance_test.cc --- a/media/libvpx/libvpx/test/variance_test.cc +++ b/media/libvpx/libvpx/test/variance_test.cc @@ -594,10 +594,11 @@ } protected: void RefTest(); void ExtremeRefTest(); + void SpeedTest(); ACMRandom rnd_; uint8_t *src_; uint8_t *ref_; uint8_t *sec_; @@ -679,10 +680,41 @@ EXPECT_EQ(var1, var2) << "for xoffset " << x << " and yoffset " << y; } } } +template +void SubpelVarianceTest::SpeedTest() { + // The only interesting points are 0, 4, and anything else. To make the loops + // simple we will use 0, 2 and 4. + for (int x = 0; x <= 4; x += 2) { + for (int y = 0; y <= 4; y += 2) { + if (!use_high_bit_depth()) { + memset(src_, 25, block_size()); + memset(ref_, 50, block_size()); +#if CONFIG_VP9_HIGHBITDEPTH + } else { + vpx_memset16(CONVERT_TO_SHORTPTR(src_), 25, block_size()); + vpx_memset16(CONVERT_TO_SHORTPTR(ref_), 50, block_size()); +#endif // CONFIG_VP9_HIGHBITDEPTH + } + unsigned int sse; + vpx_usec_timer timer; + vpx_usec_timer_start(&timer); + for (int i = 0; i < 1000000000 / block_size(); ++i) { + const uint32_t variance = + params_.func(ref_, width() + 1, x, y, src_, width(), &sse); + (void)variance; + } + vpx_usec_timer_mark(&timer); + const int elapsed_time = static_cast(vpx_usec_timer_elapsed(&timer)); + printf("SubpelVariance %dx%d xoffset: %d yoffset: %d time: %5d ms\n", + width(), height(), x, y, elapsed_time / 1000); + } + } +} + template <> void SubpelVarianceTest::RefTest() { for (int x = 0; x < 8; ++x) { for (int y = 0; y < 8; ++y) { if (!use_high_bit_depth()) { @@ -734,10 +766,11 @@ TEST_P(VpxVarianceTest, DISABLED_Speed) { SpeedTest(); } TEST_P(SumOfSquaresTest, Const) { ConstTest(); } TEST_P(SumOfSquaresTest, Ref) { RefTest(); } TEST_P(VpxSubpelVarianceTest, Ref) { RefTest(); } TEST_P(VpxSubpelVarianceTest, ExtremeRef) { ExtremeRefTest(); } +TEST_P(VpxSubpelVarianceTest, DISABLED_Speed) { SpeedTest(); } TEST_P(VpxSubpelAvgVarianceTest, Ref) { RefTest(); } INSTANTIATE_TEST_SUITE_P(C, SumOfSquaresTest, ::testing::Values(vpx_get_mb_ss_c)); @@ -1614,6 +1647,23 @@ SubpelAvgVarianceParams(3, 3, &vpx_sub_pixel_avg_variance8x8_mmi, 0), SubpelAvgVarianceParams(3, 2, &vpx_sub_pixel_avg_variance8x4_mmi, 0), SubpelAvgVarianceParams(2, 3, &vpx_sub_pixel_avg_variance4x8_mmi, 0), SubpelAvgVarianceParams(2, 2, &vpx_sub_pixel_avg_variance4x4_mmi, 0))); #endif // HAVE_MMI + +#if HAVE_LSX +INSTANTIATE_TEST_SUITE_P( + LSX, VpxVarianceTest, + ::testing::Values(VarianceParams(6, 6, &vpx_variance64x64_lsx), + VarianceParams(5, 5, &vpx_variance32x32_lsx), + VarianceParams(4, 4, &vpx_variance16x16_lsx), + VarianceParams(3, 3, &vpx_variance8x8_lsx))); + +INSTANTIATE_TEST_SUITE_P(LSX, VpxSubpelVarianceTest, + ::testing::Values(SubpelVarianceParams( + 5, 5, &vpx_sub_pixel_variance32x32_lsx, 0))); + +INSTANTIATE_TEST_SUITE_P(LSX, VpxSubpelAvgVarianceTest, + ::testing::Values(SubpelAvgVarianceParams( + 6, 6, &vpx_sub_pixel_avg_variance64x64_lsx, 0))); +#endif } // namespace diff --git a/media/libvpx/libvpx/test/video_source.h b/media/libvpx/libvpx/test/video_source.h --- a/media/libvpx/libvpx/test/video_source.h +++ b/media/libvpx/libvpx/test/video_source.h @@ -189,11 +189,12 @@ } void ReallocImage() { vpx_img_free(img_); img_ = vpx_img_alloc(NULL, format_, width_, height_, 32); - raw_sz_ = ((img_->w + 31) & ~31) * img_->h * img_->bps / 8; + ASSERT_NE(img_, nullptr); + raw_sz_ = ((img_->w + 31) & ~31u) * img_->h * img_->bps / 8; } vpx_image_t *img_; size_t raw_sz_; unsigned int limit_; diff --git a/media/libvpx/libvpx/test/vp8_fdct4x4_test.cc b/media/libvpx/libvpx/test/vp8_fdct4x4_test.cc --- a/media/libvpx/libvpx/test/vp8_fdct4x4_test.cc +++ b/media/libvpx/libvpx/test/vp8_fdct4x4_test.cc @@ -146,11 +146,11 @@ (abs(count_sign_block[j][0] - count_sign_block[j][1]) < 100000); } EXPECT_EQ(true, bias_acceptable) << "Error: 4x4 FDCT has a sign bias > 10% for input range [-15, 15]"; -}; +} TEST_P(FdctTest, RoundTripErrorCheck) { int max_error = 0; double total_error = 0; const int count_test_block = 1000000; @@ -179,11 +179,11 @@ EXPECT_GE(1, max_error) << "Error: FDCT/IDCT has an individual roundtrip error > 1"; EXPECT_GE(count_test_block, total_error) << "Error: FDCT/IDCT has average roundtrip error > 1 per block"; -}; +} INSTANTIATE_TEST_SUITE_P(C, FdctTest, ::testing::Values(vp8_short_fdct4x4_c)); #if HAVE_NEON INSTANTIATE_TEST_SUITE_P(NEON, FdctTest, diff --git a/media/libvpx/libvpx/test/vp8_ratectrl_rtc_test.cc b/media/libvpx/libvpx/test/vp8_ratectrl_rtc_test.cc new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/test/vp8_ratectrl_rtc_test.cc @@ -0,0 +1,343 @@ +/* + * Copyright (c) 2021 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include // NOLINT +#include + +#include "./vpx_config.h" +#include "third_party/googletest/src/include/gtest/gtest.h" +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/i420_video_source.h" +#include "test/util.h" +#include "test/video_source.h" +#include "vp8/vp8_ratectrl_rtc.h" +#include "vpx/vpx_codec.h" +#include "vpx_ports/bitops.h" + +namespace { + +struct Vp8RCTestVideo { + Vp8RCTestVideo() {} + Vp8RCTestVideo(const char *name_, int width_, int height_, + unsigned int frames_) + : name(name_), width(width_), height(height_), frames(frames_) {} + + friend std::ostream &operator<<(std::ostream &os, + const Vp8RCTestVideo &video) { + os << video.name << " " << video.width << " " << video.height << " " + << video.frames; + return os; + } + const char *name; + int width; + int height; + unsigned int frames; +}; + +const Vp8RCTestVideo kVp8RCTestVectors[] = { + Vp8RCTestVideo("niklas_640_480_30.yuv", 640, 480, 470), + Vp8RCTestVideo("desktop_office1.1280_720-020.yuv", 1280, 720, 300), +}; + +class Vp8RcInterfaceTest + : public ::libvpx_test::EncoderTest, + public ::libvpx_test::CodecTestWith2Params { + public: + Vp8RcInterfaceTest() + : EncoderTest(GET_PARAM(0)), key_interval_(3000), encoder_exit_(false) {} + virtual ~Vp8RcInterfaceTest() {} + + protected: + virtual void SetUp() { + InitializeConfig(); + SetMode(::libvpx_test::kRealTime); + } + + // From error_resilience_test.cc + int SetFrameFlags(int frame_num, int num_temp_layers) { + int frame_flags = 0; + if (num_temp_layers == 2) { + if (frame_num % 2 == 0) { + // Layer 0: predict from L and ARF, update L. + frame_flags = + VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF; + } else { + // Layer 1: predict from L, G and ARF, and update G. + frame_flags = VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST | + VP8_EFLAG_NO_UPD_ENTROPY; + } + } else if (num_temp_layers == 3) { + if (frame_num % 4 == 0) { + // Layer 0: predict from L, update L. + frame_flags = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_ARF | + VP8_EFLAG_NO_REF_GF | VP8_EFLAG_NO_REF_ARF; + } else if ((frame_num - 2) % 4 == 0) { + // Layer 1: predict from L, G, update G. + frame_flags = + VP8_EFLAG_NO_UPD_ARF | VP8_EFLAG_NO_UPD_LAST | VP8_EFLAG_NO_REF_ARF; + } else if ((frame_num - 1) % 2 == 0) { + // Layer 2: predict from L, G, ARF; update ARG. + frame_flags = VP8_EFLAG_NO_UPD_GF | VP8_EFLAG_NO_UPD_LAST; + } + } + return frame_flags; + } + + int SetLayerId(int frame_num, int num_temp_layers) { + int layer_id = 0; + if (num_temp_layers == 2) { + if (frame_num % 2 == 0) { + layer_id = 0; + } else { + layer_id = 1; + } + } else if (num_temp_layers == 3) { + if (frame_num % 4 == 0) { + layer_id = 0; + } else if ((frame_num - 2) % 4 == 0) { + layer_id = 1; + } else if ((frame_num - 1) % 2 == 0) { + layer_id = 2; + } + } + return layer_id; + } + + virtual void PreEncodeFrameHook(::libvpx_test::VideoSource *video, + ::libvpx_test::Encoder *encoder) { + if (rc_cfg_.ts_number_layers > 1) { + const int layer_id = SetLayerId(video->frame(), cfg_.ts_number_layers); + const int frame_flags = + SetFrameFlags(video->frame(), cfg_.ts_number_layers); + frame_params_.temporal_layer_id = layer_id; + if (video->frame() > 0) { + encoder->Control(VP8E_SET_TEMPORAL_LAYER_ID, layer_id); + encoder->Control(VP8E_SET_FRAME_FLAGS, frame_flags); + } + } else { + if (video->frame() == 0) { + encoder->Control(VP8E_SET_CPUUSED, -6); + encoder->Control(VP8E_SET_RTC_EXTERNAL_RATECTRL, 1); + encoder->Control(VP8E_SET_MAX_INTRA_BITRATE_PCT, 1000); + } + if (frame_params_.frame_type == INTER_FRAME) { + // Disable golden frame update. + frame_flags_ |= VP8_EFLAG_NO_UPD_GF; + frame_flags_ |= VP8_EFLAG_NO_UPD_ARF; + } + } + frame_params_.frame_type = + video->frame() % key_interval_ == 0 ? KEY_FRAME : INTER_FRAME; + encoder_exit_ = video->frame() == test_video_.frames; + } + + virtual void PostEncodeFrameHook(::libvpx_test::Encoder *encoder) { + if (encoder_exit_) { + return; + } + int qp; + encoder->Control(VP8E_GET_LAST_QUANTIZER, &qp); + rc_api_->ComputeQP(frame_params_); + ASSERT_EQ(rc_api_->GetQP(), qp); + } + + virtual void FramePktHook(const vpx_codec_cx_pkt_t *pkt) { + rc_api_->PostEncodeUpdate(pkt->data.frame.sz); + } + + void RunOneLayer() { + test_video_ = GET_PARAM(2); + target_bitrate_ = GET_PARAM(1); + if (test_video_.width == 1280 && target_bitrate_ == 200) return; + if (test_video_.width == 640 && target_bitrate_ == 1000) return; + SetConfig(); + rc_api_ = libvpx::VP8RateControlRTC::Create(rc_cfg_); + rc_api_->UpdateRateControl(rc_cfg_); + + ::libvpx_test::I420VideoSource video(test_video_.name, test_video_.width, + test_video_.height, 30, 1, 0, + test_video_.frames); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + } + + void RunPeriodicKey() { + test_video_ = GET_PARAM(2); + target_bitrate_ = GET_PARAM(1); + if (test_video_.width == 1280 && target_bitrate_ == 200) return; + if (test_video_.width == 640 && target_bitrate_ == 1000) return; + key_interval_ = 100; + SetConfig(); + rc_api_ = libvpx::VP8RateControlRTC::Create(rc_cfg_); + rc_api_->UpdateRateControl(rc_cfg_); + + ::libvpx_test::I420VideoSource video(test_video_.name, test_video_.width, + test_video_.height, 30, 1, 0, + test_video_.frames); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + } + + void RunTemporalLayers2TL() { + test_video_ = GET_PARAM(2); + target_bitrate_ = GET_PARAM(1); + if (test_video_.width == 1280 && target_bitrate_ == 200) return; + if (test_video_.width == 640 && target_bitrate_ == 1000) return; + SetConfigTemporalLayers(2); + rc_api_ = libvpx::VP8RateControlRTC::Create(rc_cfg_); + rc_api_->UpdateRateControl(rc_cfg_); + + ::libvpx_test::I420VideoSource video(test_video_.name, test_video_.width, + test_video_.height, 30, 1, 0, + test_video_.frames); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + } + + void RunTemporalLayers3TL() { + test_video_ = GET_PARAM(2); + target_bitrate_ = GET_PARAM(1); + if (test_video_.width == 1280 && target_bitrate_ == 200) return; + if (test_video_.width == 640 && target_bitrate_ == 1000) return; + SetConfigTemporalLayers(3); + rc_api_ = libvpx::VP8RateControlRTC::Create(rc_cfg_); + rc_api_->UpdateRateControl(rc_cfg_); + + ::libvpx_test::I420VideoSource video(test_video_.name, test_video_.width, + test_video_.height, 30, 1, 0, + test_video_.frames); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + } + + private: + void SetConfig() { + rc_cfg_.width = test_video_.width; + rc_cfg_.height = test_video_.height; + rc_cfg_.max_quantizer = 60; + rc_cfg_.min_quantizer = 2; + rc_cfg_.target_bandwidth = target_bitrate_; + rc_cfg_.buf_initial_sz = 600; + rc_cfg_.buf_optimal_sz = 600; + rc_cfg_.buf_sz = target_bitrate_; + rc_cfg_.undershoot_pct = 50; + rc_cfg_.overshoot_pct = 50; + rc_cfg_.max_intra_bitrate_pct = 1000; + rc_cfg_.framerate = 30.0; + rc_cfg_.layer_target_bitrate[0] = target_bitrate_; + + // Encoder settings for ground truth. + cfg_.g_w = test_video_.width; + cfg_.g_h = test_video_.height; + cfg_.rc_undershoot_pct = 50; + cfg_.rc_overshoot_pct = 50; + cfg_.rc_buf_initial_sz = 600; + cfg_.rc_buf_optimal_sz = 600; + cfg_.rc_buf_sz = target_bitrate_; + cfg_.rc_dropframe_thresh = 0; + cfg_.rc_min_quantizer = 2; + cfg_.rc_max_quantizer = 60; + cfg_.rc_end_usage = VPX_CBR; + cfg_.g_lag_in_frames = 0; + cfg_.g_error_resilient = 1; + cfg_.rc_target_bitrate = target_bitrate_; + cfg_.kf_min_dist = key_interval_; + cfg_.kf_max_dist = key_interval_; + } + + void SetConfigTemporalLayers(int temporal_layers) { + rc_cfg_.width = test_video_.width; + rc_cfg_.height = test_video_.height; + rc_cfg_.max_quantizer = 60; + rc_cfg_.min_quantizer = 2; + rc_cfg_.target_bandwidth = target_bitrate_; + rc_cfg_.buf_initial_sz = 600; + rc_cfg_.buf_optimal_sz = 600; + rc_cfg_.buf_sz = target_bitrate_; + rc_cfg_.undershoot_pct = 50; + rc_cfg_.overshoot_pct = 50; + rc_cfg_.max_intra_bitrate_pct = 1000; + rc_cfg_.framerate = 30.0; + if (temporal_layers == 2) { + rc_cfg_.layer_target_bitrate[0] = 60 * target_bitrate_ / 100; + rc_cfg_.layer_target_bitrate[1] = target_bitrate_; + rc_cfg_.ts_rate_decimator[0] = 2; + rc_cfg_.ts_rate_decimator[1] = 1; + } else if (temporal_layers == 3) { + rc_cfg_.layer_target_bitrate[0] = 40 * target_bitrate_ / 100; + rc_cfg_.layer_target_bitrate[1] = 60 * target_bitrate_ / 100; + rc_cfg_.layer_target_bitrate[2] = target_bitrate_; + rc_cfg_.ts_rate_decimator[0] = 4; + rc_cfg_.ts_rate_decimator[1] = 2; + rc_cfg_.ts_rate_decimator[2] = 1; + } + + rc_cfg_.ts_number_layers = temporal_layers; + + // Encoder settings for ground truth. + cfg_.g_w = test_video_.width; + cfg_.g_h = test_video_.height; + cfg_.rc_undershoot_pct = 50; + cfg_.rc_overshoot_pct = 50; + cfg_.rc_buf_initial_sz = 600; + cfg_.rc_buf_optimal_sz = 600; + cfg_.rc_buf_sz = target_bitrate_; + cfg_.rc_dropframe_thresh = 0; + cfg_.rc_min_quantizer = 2; + cfg_.rc_max_quantizer = 60; + cfg_.rc_end_usage = VPX_CBR; + cfg_.g_lag_in_frames = 0; + cfg_.g_error_resilient = 1; + cfg_.rc_target_bitrate = target_bitrate_; + cfg_.kf_min_dist = key_interval_; + cfg_.kf_max_dist = key_interval_; + // 2 Temporal layers, no spatial layers, CBR mode. + cfg_.ss_number_layers = 1; + cfg_.ts_number_layers = temporal_layers; + if (temporal_layers == 2) { + cfg_.ts_rate_decimator[0] = 2; + cfg_.ts_rate_decimator[1] = 1; + cfg_.ts_periodicity = 2; + cfg_.ts_target_bitrate[0] = 60 * cfg_.rc_target_bitrate / 100; + cfg_.ts_target_bitrate[1] = cfg_.rc_target_bitrate; + } else if (temporal_layers == 3) { + cfg_.ts_rate_decimator[0] = 4; + cfg_.ts_rate_decimator[1] = 2; + cfg_.ts_rate_decimator[2] = 1; + cfg_.ts_periodicity = 4; + cfg_.ts_target_bitrate[0] = 40 * cfg_.rc_target_bitrate / 100; + cfg_.ts_target_bitrate[1] = 60 * cfg_.rc_target_bitrate / 100; + cfg_.ts_target_bitrate[2] = cfg_.rc_target_bitrate; + } + } + + std::unique_ptr rc_api_; + libvpx::VP8RateControlRtcConfig rc_cfg_; + int key_interval_; + int target_bitrate_; + Vp8RCTestVideo test_video_; + libvpx::VP8FrameParamsQpRTC frame_params_; + bool encoder_exit_; +}; + +TEST_P(Vp8RcInterfaceTest, OneLayer) { RunOneLayer(); } + +TEST_P(Vp8RcInterfaceTest, OneLayerPeriodicKey) { RunPeriodicKey(); } + +TEST_P(Vp8RcInterfaceTest, TemporalLayers2TL) { RunTemporalLayers2TL(); } + +TEST_P(Vp8RcInterfaceTest, TemporalLayers3TL) { RunTemporalLayers3TL(); } + +VP8_INSTANTIATE_TEST_SUITE(Vp8RcInterfaceTest, + ::testing::Values(200, 400, 1000), + ::testing::ValuesIn(kVp8RCTestVectors)); + +} // namespace diff --git a/media/libvpx/libvpx/test/vp9_ethread_test.cc b/media/libvpx/libvpx/test/vp9_ethread_test.cc --- a/media/libvpx/libvpx/test/vp9_ethread_test.cc +++ b/media/libvpx/libvpx/test/vp9_ethread_test.cc @@ -96,10 +96,11 @@ EXPECT_EQ(pkt_size, kFirstPassStatsSz) << "Error: First pass stats size doesn't equal kFirstPassStatsSz"; firstpass_stats_.buf = realloc(firstpass_stats_.buf, firstpass_stats_.sz + pkt_size); + ASSERT_NE(firstpass_stats_.buf, nullptr); memcpy((uint8_t *)firstpass_stats_.buf + firstpass_stats_.sz, pkt_buf, pkt_size); firstpass_stats_.sz += pkt_size; } diff --git a/media/libvpx/libvpx/test/vp9_quantize_test.cc b/media/libvpx/libvpx/test/vp9_quantize_test.cc --- a/media/libvpx/libvpx/test/vp9_quantize_test.cc +++ b/media/libvpx/libvpx/test/vp9_quantize_test.cc @@ -36,40 +36,37 @@ namespace { const int number_of_iterations = 100; typedef void (*QuantizeFunc)(const tran_low_t *coeff, intptr_t count, - int skip_block, const int16_t *zbin, - const int16_t *round, const int16_t *quant, - const int16_t *quant_shift, tran_low_t *qcoeff, - tran_low_t *dqcoeff, const int16_t *dequant, - uint16_t *eob, const int16_t *scan, - const int16_t *iscan); + const int16_t *zbin, const int16_t *round, + const int16_t *quant, const int16_t *quant_shift, + tran_low_t *qcoeff, tran_low_t *dqcoeff, + const int16_t *dequant, uint16_t *eob, + const int16_t *scan, const int16_t *iscan); typedef std::tuple QuantizeParam; // Wrapper for FP version which does not use zbin or quant_shift. typedef void (*QuantizeFPFunc)(const tran_low_t *coeff, intptr_t count, - int skip_block, const int16_t *round, - const int16_t *quant, tran_low_t *qcoeff, - tran_low_t *dqcoeff, const int16_t *dequant, - uint16_t *eob, const int16_t *scan, - const int16_t *iscan); + const int16_t *round, const int16_t *quant, + tran_low_t *qcoeff, tran_low_t *dqcoeff, + const int16_t *dequant, uint16_t *eob, + const int16_t *scan, const int16_t *iscan); template -void QuantFPWrapper(const tran_low_t *coeff, intptr_t count, int skip_block, +void QuantFPWrapper(const tran_low_t *coeff, intptr_t count, const int16_t *zbin, const int16_t *round, const int16_t *quant, const int16_t *quant_shift, tran_low_t *qcoeff, tran_low_t *dqcoeff, const int16_t *dequant, uint16_t *eob, const int16_t *scan, const int16_t *iscan) { (void)zbin; (void)quant_shift; - fn(coeff, count, skip_block, round, quant, qcoeff, dqcoeff, dequant, eob, - scan, iscan); + fn(coeff, count, round, quant, qcoeff, dqcoeff, dequant, eob, scan, iscan); } class VP9QuantizeBase : public AbstractBench { public: VP9QuantizeBase(vpx_bit_depth_t bit_depth, int max_size, bool is_fp) @@ -136,11 +133,10 @@ Buffer qcoeff_; Buffer dqcoeff_; int16_t *r_ptr_; int16_t *q_ptr_; int count_; - int skip_block_; const scan_order *scan_; uint16_t eob_; }; class VP9QuantizeTest : public VP9QuantizeBase, @@ -155,30 +151,28 @@ const QuantizeFunc quantize_op_; const QuantizeFunc ref_quantize_op_; }; void VP9QuantizeTest::Run() { - quantize_op_(coeff_.TopLeftPixel(), count_, skip_block_, zbin_ptr_, r_ptr_, - q_ptr_, quant_shift_ptr_, qcoeff_.TopLeftPixel(), + quantize_op_(coeff_.TopLeftPixel(), count_, zbin_ptr_, r_ptr_, q_ptr_, + quant_shift_ptr_, qcoeff_.TopLeftPixel(), dqcoeff_.TopLeftPixel(), dequant_ptr_, &eob_, scan_->scan, scan_->iscan); } // This quantizer compares the AC coefficients to the quantization step size to // determine if further multiplication operations are needed. // Based on vp9_quantize_fp_sse2(). inline void quant_fp_nz(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *round_ptr, - const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, - uint16_t *eob_ptr, const int16_t *scan, - const int16_t *iscan, int is_32x32) { + const int16_t *round_ptr, const int16_t *quant_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan, + int is_32x32) { int i, eob = -1; const int thr = dequant_ptr[1] >> (1 + is_32x32); (void)iscan; - (void)skip_block; - assert(!skip_block); // Quantization pass: All coefficients with index >= zero_flag are // skippable. Note: zero_flag can be zero. for (i = 0; i < n_coeffs; i += 16) { int y; @@ -241,26 +235,24 @@ } *eob_ptr = eob + 1; } void quantize_fp_nz_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *round_ptr, - const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, - uint16_t *eob_ptr, const int16_t *scan, - const int16_t *iscan) { - quant_fp_nz(coeff_ptr, n_coeffs, skip_block, round_ptr, quant_ptr, qcoeff_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { + quant_fp_nz(coeff_ptr, n_coeffs, round_ptr, quant_ptr, qcoeff_ptr, dqcoeff_ptr, dequant_ptr, eob_ptr, scan, iscan, 0); } void quantize_fp_32x32_nz_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *round_ptr, - const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, - uint16_t *eob_ptr, const int16_t *scan, - const int16_t *iscan) { - quant_fp_nz(coeff_ptr, n_coeffs, skip_block, round_ptr, quant_ptr, qcoeff_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { + quant_fp_nz(coeff_ptr, n_coeffs, round_ptr, quant_ptr, qcoeff_ptr, dqcoeff_ptr, dequant_ptr, eob_ptr, scan, iscan, 1); } void GenerateHelperArrays(ACMRandom *rnd, int16_t *zbin, int16_t *round, int16_t *quant, int16_t *quant_shift, @@ -314,13 +306,10 @@ ASSERT_TRUE(ref_dqcoeff.Init()); uint16_t ref_eob = 0; eob_ = 0; for (int i = 0; i < number_of_iterations; ++i) { - // Test skip block for the first three iterations to catch all the different - // sizes. - const int skip_block = 0; TX_SIZE sz; if (max_size_ == 16) { sz = static_cast(i % 3); // TX_4X4, TX_8X8 TX_16X16 } else { sz = TX_32X32; @@ -330,17 +319,17 @@ count_ = (4 << sz) * (4 << sz); coeff_.Set(&rnd, -max_value_, max_value_); GenerateHelperArrays(&rnd, zbin_ptr_, round_ptr_, quant_ptr_, quant_shift_ptr_, dequant_ptr_, round_fp_ptr_, quant_fp_ptr_); - ref_quantize_op_(coeff_.TopLeftPixel(), count_, skip_block, zbin_ptr_, - r_ptr_, q_ptr_, quant_shift_ptr_, - ref_qcoeff.TopLeftPixel(), ref_dqcoeff.TopLeftPixel(), - dequant_ptr_, &ref_eob, scan_->scan, scan_->iscan); + ref_quantize_op_(coeff_.TopLeftPixel(), count_, zbin_ptr_, r_ptr_, q_ptr_, + quant_shift_ptr_, ref_qcoeff.TopLeftPixel(), + ref_dqcoeff.TopLeftPixel(), dequant_ptr_, &ref_eob, + scan_->scan, scan_->iscan); ASM_REGISTER_STATE_CHECK(quantize_op_( - coeff_.TopLeftPixel(), count_, skip_block, zbin_ptr_, r_ptr_, q_ptr_, + coeff_.TopLeftPixel(), count_, zbin_ptr_, r_ptr_, q_ptr_, quant_shift_ptr_, qcoeff_.TopLeftPixel(), dqcoeff_.TopLeftPixel(), dequant_ptr_, &eob_, scan_->scan, scan_->iscan)); EXPECT_TRUE(qcoeff_.CheckValues(ref_qcoeff)); EXPECT_TRUE(dqcoeff_.CheckValues(ref_dqcoeff)); @@ -370,11 +359,10 @@ uint16_t ref_eob = 0; eob_ = 0; const uint32_t max_index = max_size_ * max_size_ - 1; for (int i = 0; i < number_of_iterations; ++i) { - skip_block_ = 0; TX_SIZE sz; if (max_size_ == 16) { sz = static_cast(i % 3); // TX_4X4, TX_8X8 TX_16X16 } else { sz = TX_32X32; @@ -389,17 +377,17 @@ coeff_.TopLeftPixel()[rnd.RandRange(count_) & max_index] = static_cast(rnd.RandRange(max_value_ * 2)) - max_value_; GenerateHelperArrays(&rnd, zbin_ptr_, round_ptr_, quant_ptr_, quant_shift_ptr_, dequant_ptr_, round_fp_ptr_, quant_fp_ptr_); - ref_quantize_op_(coeff_.TopLeftPixel(), count_, skip_block_, zbin_ptr_, - r_ptr_, q_ptr_, quant_shift_ptr_, - ref_qcoeff.TopLeftPixel(), ref_dqcoeff.TopLeftPixel(), - dequant_ptr_, &ref_eob, scan_->scan, scan_->iscan); + ref_quantize_op_(coeff_.TopLeftPixel(), count_, zbin_ptr_, r_ptr_, q_ptr_, + quant_shift_ptr_, ref_qcoeff.TopLeftPixel(), + ref_dqcoeff.TopLeftPixel(), dequant_ptr_, &ref_eob, + scan_->scan, scan_->iscan); ASM_REGISTER_STATE_CHECK(quantize_op_( - coeff_.TopLeftPixel(), count_, skip_block_, zbin_ptr_, r_ptr_, q_ptr_, + coeff_.TopLeftPixel(), count_, zbin_ptr_, r_ptr_, q_ptr_, quant_shift_ptr_, qcoeff_.TopLeftPixel(), dqcoeff_.TopLeftPixel(), dequant_ptr_, &eob_, scan_->scan, scan_->iscan)); EXPECT_TRUE(qcoeff_.CheckValues(ref_qcoeff)); EXPECT_TRUE(dqcoeff_.CheckValues(ref_dqcoeff)); @@ -431,11 +419,10 @@ } for (TX_SIZE sz = starting_sz; sz <= ending_sz; ++sz) { // zbin > coeff, zbin < coeff. for (int i = 0; i < 2; ++i) { - skip_block_ = 0; // TX_TYPE defines the scan order. That is not relevant to the speed test. // Pick the first one. const TX_TYPE tx_type = DCT_DCT; count_ = (4 << sz) * (4 << sz); scan_ = &vp9_scan_orders[sz][tx_type]; diff --git a/media/libvpx/libvpx/test/vp9_ratectrl_rtc_test.cc b/media/libvpx/libvpx/test/vp9_ratectrl_rtc_test.cc new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/test/vp9_ratectrl_rtc_test.cc @@ -0,0 +1,373 @@ +/* + * Copyright (c) 2020 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#include "vp9/ratectrl_rtc.h" + +#include // NOLINT +#include + +#include "./vpx_config.h" +#include "third_party/googletest/src/include/gtest/gtest.h" +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/i420_video_source.h" +#include "test/util.h" +#include "test/video_source.h" +#include "vpx/vpx_codec.h" +#include "vpx_ports/bitops.h" + +namespace { + +const size_t kNumFrames = 300; + +const int kTemporalId[4] = { 0, 2, 1, 2 }; + +class RcInterfaceTest + : public ::libvpx_test::EncoderTest, + public ::libvpx_test::CodecTestWith2Params { + public: + RcInterfaceTest() + : EncoderTest(GET_PARAM(0)), aq_mode_(GET_PARAM(1)), key_interval_(3000), + encoder_exit_(false) {} + + virtual ~RcInterfaceTest() {} + + protected: + virtual void SetUp() { + InitializeConfig(); + SetMode(::libvpx_test::kRealTime); + } + + virtual void PreEncodeFrameHook(libvpx_test::VideoSource *video, + libvpx_test::Encoder *encoder) { + if (video->frame() == 0) { + encoder->Control(VP8E_SET_CPUUSED, 7); + encoder->Control(VP9E_SET_AQ_MODE, aq_mode_); + encoder->Control(VP9E_SET_TUNE_CONTENT, 0); + encoder->Control(VP8E_SET_MAX_INTRA_BITRATE_PCT, 1000); + encoder->Control(VP9E_SET_RTC_EXTERNAL_RATECTRL, 1); + } + frame_params_.frame_type = + video->frame() % key_interval_ == 0 ? KEY_FRAME : INTER_FRAME; + if (rc_cfg_.rc_mode == VPX_CBR && frame_params_.frame_type == INTER_FRAME) { + // Disable golden frame update. + frame_flags_ |= VP8_EFLAG_NO_UPD_GF; + frame_flags_ |= VP8_EFLAG_NO_UPD_ARF; + } + encoder_exit_ = video->frame() == kNumFrames; + } + + virtual void PostEncodeFrameHook(::libvpx_test::Encoder *encoder) { + if (encoder_exit_) { + return; + } + int loopfilter_level, qp; + encoder->Control(VP9E_GET_LOOPFILTER_LEVEL, &loopfilter_level); + encoder->Control(VP8E_GET_LAST_QUANTIZER, &qp); + rc_api_->ComputeQP(frame_params_); + ASSERT_EQ(rc_api_->GetQP(), qp); + ASSERT_EQ(rc_api_->GetLoopfilterLevel(), loopfilter_level); + } + + virtual void FramePktHook(const vpx_codec_cx_pkt_t *pkt) { + rc_api_->PostEncodeUpdate(pkt->data.frame.sz); + } + + void RunOneLayer() { + SetConfig(GET_PARAM(2)); + rc_api_ = libvpx::VP9RateControlRTC::Create(rc_cfg_); + frame_params_.spatial_layer_id = 0; + frame_params_.temporal_layer_id = 0; + + ::libvpx_test::I420VideoSource video("desktop_office1.1280_720-020.yuv", + 1280, 720, 30, 1, 0, kNumFrames); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + } + + void RunOneLayerVBRPeriodicKey() { + if (GET_PARAM(2) != VPX_VBR) return; + key_interval_ = 100; + SetConfig(VPX_VBR); + rc_api_ = libvpx::VP9RateControlRTC::Create(rc_cfg_); + frame_params_.spatial_layer_id = 0; + frame_params_.temporal_layer_id = 0; + + ::libvpx_test::I420VideoSource video("desktop_office1.1280_720-020.yuv", + 1280, 720, 30, 1, 0, kNumFrames); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + } + + private: + void SetConfig(vpx_rc_mode rc_mode) { + rc_cfg_.width = 1280; + rc_cfg_.height = 720; + rc_cfg_.max_quantizer = 52; + rc_cfg_.min_quantizer = 2; + rc_cfg_.target_bandwidth = 1000; + rc_cfg_.buf_initial_sz = 600; + rc_cfg_.buf_optimal_sz = 600; + rc_cfg_.buf_sz = 1000; + rc_cfg_.undershoot_pct = 50; + rc_cfg_.overshoot_pct = 50; + rc_cfg_.max_intra_bitrate_pct = 1000; + rc_cfg_.framerate = 30.0; + rc_cfg_.ss_number_layers = 1; + rc_cfg_.ts_number_layers = 1; + rc_cfg_.scaling_factor_num[0] = 1; + rc_cfg_.scaling_factor_den[0] = 1; + rc_cfg_.layer_target_bitrate[0] = 1000; + rc_cfg_.max_quantizers[0] = 52; + rc_cfg_.min_quantizers[0] = 2; + rc_cfg_.rc_mode = rc_mode; + rc_cfg_.aq_mode = aq_mode_; + + // Encoder settings for ground truth. + cfg_.g_w = 1280; + cfg_.g_h = 720; + cfg_.rc_undershoot_pct = 50; + cfg_.rc_overshoot_pct = 50; + cfg_.rc_buf_initial_sz = 600; + cfg_.rc_buf_optimal_sz = 600; + cfg_.rc_buf_sz = 1000; + cfg_.rc_dropframe_thresh = 0; + cfg_.rc_min_quantizer = 2; + cfg_.rc_max_quantizer = 52; + cfg_.rc_end_usage = rc_mode; + cfg_.g_lag_in_frames = 0; + cfg_.g_error_resilient = 0; + cfg_.rc_target_bitrate = 1000; + cfg_.kf_min_dist = key_interval_; + cfg_.kf_max_dist = key_interval_; + } + + std::unique_ptr rc_api_; + libvpx::VP9RateControlRtcConfig rc_cfg_; + int aq_mode_; + int key_interval_; + libvpx::VP9FrameParamsQpRTC frame_params_; + bool encoder_exit_; +}; + +class RcInterfaceSvcTest : public ::libvpx_test::EncoderTest, + public ::libvpx_test::CodecTestWithParam { + public: + RcInterfaceSvcTest() : EncoderTest(GET_PARAM(0)), aq_mode_(GET_PARAM(1)) {} + virtual ~RcInterfaceSvcTest() {} + + protected: + virtual void SetUp() { + InitializeConfig(); + SetMode(::libvpx_test::kRealTime); + } + + virtual void PreEncodeFrameHook(libvpx_test::VideoSource *video, + ::libvpx_test::Encoder *encoder) { + if (video->frame() == 0) { + encoder->Control(VP8E_SET_CPUUSED, 7); + encoder->Control(VP9E_SET_AQ_MODE, aq_mode_); + encoder->Control(VP9E_SET_TUNE_CONTENT, 0); + encoder->Control(VP8E_SET_MAX_INTRA_BITRATE_PCT, 900); + encoder->Control(VP9E_SET_RTC_EXTERNAL_RATECTRL, 1); + encoder->Control(VP9E_SET_SVC, 1); + encoder->Control(VP9E_SET_SVC_PARAMETERS, &svc_params_); + } + + frame_params_.frame_type = video->frame() == 0 ? KEY_FRAME : INTER_FRAME; + if (rc_cfg_.rc_mode == VPX_CBR && frame_params_.frame_type == INTER_FRAME) { + // Disable golden frame update. + frame_flags_ |= VP8_EFLAG_NO_UPD_GF; + frame_flags_ |= VP8_EFLAG_NO_UPD_ARF; + } + encoder_exit_ = video->frame() == kNumFrames; + current_superframe_ = video->frame(); + } + + virtual void PostEncodeFrameHook(::libvpx_test::Encoder *encoder) { + ::libvpx_test::CxDataIterator iter = encoder->GetCxData(); + while (const vpx_codec_cx_pkt_t *pkt = iter.Next()) { + ParseSuperframeSizes(static_cast(pkt->data.frame.buf), + pkt->data.frame.sz); + for (int sl = 0; sl < rc_cfg_.ss_number_layers; sl++) { + frame_params_.spatial_layer_id = sl; + frame_params_.temporal_layer_id = kTemporalId[current_superframe_ % 4]; + rc_api_->ComputeQP(frame_params_); + frame_params_.frame_type = INTER_FRAME; + rc_api_->PostEncodeUpdate(sizes_[sl]); + } + } + if (!encoder_exit_) { + int loopfilter_level, qp; + encoder->Control(VP9E_GET_LOOPFILTER_LEVEL, &loopfilter_level); + encoder->Control(VP8E_GET_LAST_QUANTIZER, &qp); + ASSERT_EQ(rc_api_->GetQP(), qp); + ASSERT_EQ(rc_api_->GetLoopfilterLevel(), loopfilter_level); + } + } + // This method needs to be overridden because non-reference frames are + // expected to be mismatched frames as the encoder will avoid loopfilter on + // these frames. + virtual void MismatchHook(const vpx_image_t * /*img1*/, + const vpx_image_t * /*img2*/) {} + + void RunSvc() { + SetConfigSvc(); + rc_api_ = libvpx::VP9RateControlRTC::Create(rc_cfg_); + SetEncoderSvc(); + + ::libvpx_test::I420VideoSource video("desktop_office1.1280_720-020.yuv", + 1280, 720, 30, 1, 0, kNumFrames); + + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); + } + + private: + vpx_codec_err_t ParseSuperframeSizes(const uint8_t *data, size_t data_sz) { + uint8_t marker = *(data + data_sz - 1); + if ((marker & 0xe0) == 0xc0) { + const uint32_t frames = (marker & 0x7) + 1; + const uint32_t mag = ((marker >> 3) & 0x3) + 1; + const size_t index_sz = 2 + mag * frames; + // This chunk is marked as having a superframe index but doesn't have + // enough data for it, thus it's an invalid superframe index. + if (data_sz < index_sz) return VPX_CODEC_CORRUPT_FRAME; + { + const uint8_t marker2 = *(data + data_sz - index_sz); + // This chunk is marked as having a superframe index but doesn't have + // the matching marker byte at the front of the index therefore it's an + // invalid chunk. + if (marker != marker2) return VPX_CODEC_CORRUPT_FRAME; + } + const uint8_t *x = &data[data_sz - index_sz + 1]; + for (uint32_t i = 0; i < frames; ++i) { + uint32_t this_sz = 0; + + for (uint32_t j = 0; j < mag; ++j) this_sz |= (*x++) << (j * 8); + sizes_[i] = this_sz; + } + } + return VPX_CODEC_OK; + } + + void SetEncoderSvc() { + cfg_.ss_number_layers = 3; + cfg_.ts_number_layers = 3; + cfg_.g_timebase.num = 1; + cfg_.g_timebase.den = 30; + svc_params_.scaling_factor_num[0] = 72; + svc_params_.scaling_factor_den[0] = 288; + svc_params_.scaling_factor_num[1] = 144; + svc_params_.scaling_factor_den[1] = 288; + svc_params_.scaling_factor_num[2] = 288; + svc_params_.scaling_factor_den[2] = 288; + for (int i = 0; i < VPX_MAX_LAYERS; ++i) { + svc_params_.max_quantizers[i] = 56; + svc_params_.min_quantizers[i] = 2; + svc_params_.speed_per_layer[i] = 7; + } + cfg_.rc_end_usage = VPX_CBR; + cfg_.g_lag_in_frames = 0; + cfg_.g_error_resilient = 0; + // 3 temporal layers + cfg_.ts_rate_decimator[0] = 4; + cfg_.ts_rate_decimator[1] = 2; + cfg_.ts_rate_decimator[2] = 1; + cfg_.temporal_layering_mode = 3; + + cfg_.rc_buf_initial_sz = 500; + cfg_.rc_buf_optimal_sz = 600; + cfg_.rc_buf_sz = 1000; + cfg_.rc_min_quantizer = 2; + cfg_.rc_max_quantizer = 56; + cfg_.g_threads = 1; + cfg_.kf_max_dist = 9999; + cfg_.rc_target_bitrate = 1600; + cfg_.rc_overshoot_pct = 50; + cfg_.rc_undershoot_pct = 50; + + cfg_.layer_target_bitrate[0] = 100; + cfg_.layer_target_bitrate[1] = 140; + cfg_.layer_target_bitrate[2] = 200; + cfg_.layer_target_bitrate[3] = 250; + cfg_.layer_target_bitrate[4] = 350; + cfg_.layer_target_bitrate[5] = 500; + cfg_.layer_target_bitrate[6] = 450; + cfg_.layer_target_bitrate[7] = 630; + cfg_.layer_target_bitrate[8] = 900; + } + + void SetConfigSvc() { + rc_cfg_.width = 1280; + rc_cfg_.height = 720; + rc_cfg_.max_quantizer = 56; + rc_cfg_.min_quantizer = 2; + rc_cfg_.target_bandwidth = 1600; + rc_cfg_.buf_initial_sz = 500; + rc_cfg_.buf_optimal_sz = 600; + rc_cfg_.buf_sz = 1000; + rc_cfg_.undershoot_pct = 50; + rc_cfg_.overshoot_pct = 50; + rc_cfg_.max_intra_bitrate_pct = 900; + rc_cfg_.framerate = 30.0; + rc_cfg_.ss_number_layers = 3; + rc_cfg_.ts_number_layers = 3; + rc_cfg_.rc_mode = VPX_CBR; + rc_cfg_.aq_mode = aq_mode_; + + rc_cfg_.scaling_factor_num[0] = 1; + rc_cfg_.scaling_factor_den[0] = 4; + rc_cfg_.scaling_factor_num[1] = 2; + rc_cfg_.scaling_factor_den[1] = 4; + rc_cfg_.scaling_factor_num[2] = 4; + rc_cfg_.scaling_factor_den[2] = 4; + + rc_cfg_.ts_rate_decimator[0] = 4; + rc_cfg_.ts_rate_decimator[1] = 2; + rc_cfg_.ts_rate_decimator[2] = 1; + + rc_cfg_.layer_target_bitrate[0] = 100; + rc_cfg_.layer_target_bitrate[1] = 140; + rc_cfg_.layer_target_bitrate[2] = 200; + rc_cfg_.layer_target_bitrate[3] = 250; + rc_cfg_.layer_target_bitrate[4] = 350; + rc_cfg_.layer_target_bitrate[5] = 500; + rc_cfg_.layer_target_bitrate[6] = 450; + rc_cfg_.layer_target_bitrate[7] = 630; + rc_cfg_.layer_target_bitrate[8] = 900; + + for (int sl = 0; sl < rc_cfg_.ss_number_layers; ++sl) { + for (int tl = 0; tl < rc_cfg_.ts_number_layers; ++tl) { + const int i = sl * rc_cfg_.ts_number_layers + tl; + rc_cfg_.max_quantizers[i] = 56; + rc_cfg_.min_quantizers[i] = 2; + } + } + } + + int aq_mode_; + std::unique_ptr rc_api_; + libvpx::VP9RateControlRtcConfig rc_cfg_; + vpx_svc_extra_cfg_t svc_params_; + libvpx::VP9FrameParamsQpRTC frame_params_; + bool encoder_exit_; + int current_superframe_; + uint32_t sizes_[8]; +}; + +TEST_P(RcInterfaceTest, OneLayer) { RunOneLayer(); } + +TEST_P(RcInterfaceTest, OneLayerVBRPeriodicKey) { RunOneLayerVBRPeriodicKey(); } + +TEST_P(RcInterfaceSvcTest, Svc) { RunSvc(); } + +VP9_INSTANTIATE_TEST_SUITE(RcInterfaceTest, ::testing::Values(0, 3), + ::testing::Values(VPX_CBR, VPX_VBR)); +VP9_INSTANTIATE_TEST_SUITE(RcInterfaceSvcTest, ::testing::Values(0, 3)); +} // namespace diff --git a/media/libvpx/libvpx/test/vp9_roi_test.cc b/media/libvpx/libvpx/test/vp9_roi_test.cc new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/test/vp9_roi_test.cc @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2014 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#include + +#include "third_party/googletest/src/include/gtest/gtest.h" + +#include "test/codec_factory.h" +#include "test/encode_test_driver.h" +#include "test/i420_video_source.h" +#include "test/util.h" +#include "test/video_source.h" +#include "test/y4m_video_source.h" +#include "test/yuv_video_source.h" +#include "vpx/vp8cx.h" +#include "vpx/vpx_encoder.h" + +#define MASK_WIDTH 40 +#define MASK_HEIGHT 30 +#define MASK_SIZE MASK_WIDTH *MASK_HEIGHT + +namespace { + +const int mask[MASK_SIZE] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, + 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0 +}; + +class RoiMaskBackgroundSkip : public ::libvpx_test::EncoderTest, + public ::testing::Test { + protected: + RoiMaskBackgroundSkip() : EncoderTest(&::libvpx_test::kVP9) {} + virtual ~RoiMaskBackgroundSkip() { free(roi_.roi_map); } + + virtual void SetUp() { + InitializeConfig(); + SetMode(::libvpx_test::kRealTime); + SetRoi(); + } + + void SetRoi() { + const int block_size = 8; + unsigned int i, j; + roi_.rows = (cfg_.g_h + block_size - 1) / block_size; + roi_.cols = (cfg_.g_w + block_size - 1) / block_size; + memset(&roi_.skip, 0, sizeof(roi_.skip)); + memset(&roi_.delta_q, 0, sizeof(roi_.delta_q)); + memset(&roi_.delta_lf, 0, sizeof(roi_.delta_lf)); + memset(roi_.ref_frame, -1, sizeof(roi_.ref_frame)); + roi_.ref_frame[1] = 1; + // Use segment 3 for skip. + roi_.skip[3] = 1; + roi_.roi_map = + (uint8_t *)calloc(roi_.rows * roi_.cols, sizeof(*roi_.roi_map)); + for (i = 0; i < roi_.rows; ++i) { + for (j = 0; j < roi_.cols; ++j) { + const int idx = i * roi_.cols + j; + if (mask[idx] == 1) roi_.roi_map[idx] = 3; + } + } + } + + virtual void PreEncodeFrameHook(::libvpx_test::VideoSource *video, + ::libvpx_test::Encoder *encoder) { + if (video->frame() == 0) { + encoder->Control(VP8E_SET_CPUUSED, 7); + encoder->Control(VP9E_SET_AQ_MODE, 3); + } + encoder->Control(VP9E_SET_ROI_MAP, &roi_); + } + + private: + vpx_roi_map_t roi_; +}; + +TEST_F(RoiMaskBackgroundSkip, RoiMaskNoMismatch) { + cfg_.rc_buf_initial_sz = 500; + cfg_.rc_buf_optimal_sz = 500; + cfg_.rc_buf_sz = 1000; + cfg_.rc_undershoot_pct = 20; + cfg_.rc_undershoot_pct = 20; + cfg_.rc_dropframe_thresh = 10; + cfg_.rc_min_quantizer = 0; + cfg_.rc_max_quantizer = 50; + cfg_.rc_end_usage = VPX_CBR; + cfg_.rc_target_bitrate = 200; + cfg_.g_lag_in_frames = 0; + cfg_.kf_max_dist = 9999; + + ::libvpx_test::I420VideoSource video("desktopqvga.320_240.yuv", 320, 240, 30, + 1, 0, 150); + ASSERT_NO_FATAL_FAILURE(RunLoop(&video)); +} +} // namespace diff --git a/media/libvpx/libvpx/test/vp9_thread_test.cc b/media/libvpx/libvpx/test/vp9_thread_test.cc --- a/media/libvpx/libvpx/test/vp9_thread_test.cc +++ b/media/libvpx/libvpx/test/vp9_thread_test.cc @@ -146,15 +146,10 @@ } // ----------------------------------------------------------------------------- // Multi-threaded decode tests #if CONFIG_WEBM_IO -struct FileList { - const char *name; - const char *expected_md5; -}; - // Decodes |filename| with |num_threads|. Returns the md5 of the decoded frames. string DecodeFile(const string &filename, int num_threads) { libvpx_test::WebMVideoSource video(filename); video.Init(); @@ -180,20 +175,10 @@ } } return string(md5.Get()); } -void DecodeFiles(const FileList files[]) { - for (const FileList *iter = files; iter->name != nullptr; ++iter) { - SCOPED_TRACE(iter->name); - for (int t = 1; t <= 8; ++t) { - EXPECT_EQ(iter->expected_md5, DecodeFile(iter->name, t)) - << "threads = " << t; - } - } -} - // Trivial serialized thread worker interface implementation. // Note any worker that requires synchronization between other workers will // hang. namespace impl { namespace { @@ -214,14 +199,10 @@ TEST(VPxWorkerThreadTest, TestSerialInterface) { static const VPxWorkerInterface serial_interface = { impl::Init, impl::Reset, impl::Sync, impl::Launch, impl::Execute, impl::End }; - // TODO(jzern): Avoid using a file that will use the row-based thread - // loopfilter, with the simple serialized implementation it will hang. This is - // due to its expectation that rows will be run in parallel as they wait on - // progress in the row above before proceeding. static const char expected_md5[] = "b35a1b707b28e82be025d960aba039bc"; static const char filename[] = "vp90-2-03-size-226x226.webm"; VPxWorkerInterface default_interface = *vpx_get_worker_interface(); EXPECT_NE(vpx_set_worker_interface(&serial_interface), 0); @@ -230,92 +211,85 @@ // Reset the interface. EXPECT_NE(vpx_set_worker_interface(&default_interface), 0); EXPECT_EQ(expected_md5, DecodeFile(filename, 2)); } -TEST(VP9DecodeMultiThreadedTest, NoTilesNonFrameParallel) { - // no tiles or frame parallel; this exercises loop filter threading. - EXPECT_EQ("b35a1b707b28e82be025d960aba039bc", - DecodeFile("vp90-2-03-size-226x226.webm", 2)); -} +struct FileParam { + const char *name; + const char *expected_md5; + friend std::ostream &operator<<(std::ostream &os, const FileParam ¶m) { + return os << "file name: " << param.name + << " digest: " << param.expected_md5; + } +}; -TEST(VP9DecodeMultiThreadedTest, FrameParallel) { - static const FileList files[] = { { "vp90-2-08-tile_1x2_frame_parallel.webm", - "68ede6abd66bae0a2edf2eb9232241b6" }, - { "vp90-2-08-tile_1x4_frame_parallel.webm", - "368ebc6ebf3a5e478d85b2c3149b2848" }, - { "vp90-2-08-tile_1x8_frame_parallel.webm", - "17e439da2388aff3a0f69cb22579c6c1" }, - { nullptr, nullptr } }; +class VP9DecodeMultiThreadedTest : public ::testing::TestWithParam { +}; - DecodeFiles(files); +TEST_P(VP9DecodeMultiThreadedTest, Decode) { + for (int t = 1; t <= 8; ++t) { + EXPECT_EQ(GetParam().expected_md5, DecodeFile(GetParam().name, t)) + << "threads = " << t; + } } -TEST(VP9DecodeMultiThreadedTest, FrameParallelResize) { - static const FileList files[] = { - { "vp90-2-14-resize-fp-tiles-1-16.webm", - "0cd5e632c326297e975f38949c31ea94" }, - { "vp90-2-14-resize-fp-tiles-1-2-4-8-16.webm", - "5c78a96a42e7f4a4f6b2edcdb791e44c" }, - { "vp90-2-14-resize-fp-tiles-1-2.webm", - "e030450ae85c3277be2a418769df98e2" }, - { "vp90-2-14-resize-fp-tiles-1-4.webm", - "312eed4e2b64eb7a4e7f18916606a430" }, - { "vp90-2-14-resize-fp-tiles-16-1.webm", - "1755c16d8af16a9cb3fe7338d90abe52" }, - { "vp90-2-14-resize-fp-tiles-16-2.webm", - "500300592d3fcb6f12fab25e48aaf4df" }, - { "vp90-2-14-resize-fp-tiles-16-4.webm", - "47c48379fa6331215d91c67648e1af6e" }, - { "vp90-2-14-resize-fp-tiles-16-8-4-2-1.webm", - "eecf17290739bc708506fa4827665989" }, - { "vp90-2-14-resize-fp-tiles-16-8.webm", - "29b6bb54e4c26b5ca85d5de5fed94e76" }, - { "vp90-2-14-resize-fp-tiles-1-8.webm", - "1b6f175e08cd82cf84bb800ac6d1caa3" }, - { "vp90-2-14-resize-fp-tiles-2-16.webm", - "ca3b03e4197995d8d5444ede7a6c0804" }, - { "vp90-2-14-resize-fp-tiles-2-1.webm", - "99aec065369d70bbb78ccdff65afed3f" }, - { "vp90-2-14-resize-fp-tiles-2-4.webm", - "22d0ebdb49b87d2920a85aea32e1afd5" }, - { "vp90-2-14-resize-fp-tiles-2-8.webm", - "c2115cf051c62e0f7db1d4a783831541" }, - { "vp90-2-14-resize-fp-tiles-4-16.webm", - "c690d7e1719b31367564cac0af0939cb" }, - { "vp90-2-14-resize-fp-tiles-4-1.webm", - "a926020b2cc3e15ad4cc271853a0ff26" }, - { "vp90-2-14-resize-fp-tiles-4-2.webm", - "42699063d9e581f1993d0cf890c2be78" }, - { "vp90-2-14-resize-fp-tiles-4-8.webm", - "7f76d96036382f45121e3d5aa6f8ec52" }, - { "vp90-2-14-resize-fp-tiles-8-16.webm", - "76a43fcdd7e658542913ea43216ec55d" }, - { "vp90-2-14-resize-fp-tiles-8-1.webm", - "8e3fbe89486ca60a59299dea9da91378" }, - { "vp90-2-14-resize-fp-tiles-8-2.webm", - "ae96f21f21b6370cc0125621b441fc52" }, - { "vp90-2-14-resize-fp-tiles-8-4.webm", - "3eb4f24f10640d42218f7fd7b9fd30d4" }, - { nullptr, nullptr } - }; +const FileParam kNoTilesNonFrameParallelFiles[] = { + { "vp90-2-03-size-226x226.webm", "b35a1b707b28e82be025d960aba039bc" } +}; + +const FileParam kFrameParallelFiles[] = { + { "vp90-2-08-tile_1x2_frame_parallel.webm", + "68ede6abd66bae0a2edf2eb9232241b6" }, + { "vp90-2-08-tile_1x4_frame_parallel.webm", + "368ebc6ebf3a5e478d85b2c3149b2848" }, + { "vp90-2-08-tile_1x8_frame_parallel.webm", + "17e439da2388aff3a0f69cb22579c6c1" }, +}; - DecodeFiles(files); -} +const FileParam kFrameParallelResizeFiles[] = { + { "vp90-2-14-resize-fp-tiles-1-16.webm", "0cd5e632c326297e975f38949c31ea94" }, + { "vp90-2-14-resize-fp-tiles-1-2-4-8-16.webm", + "5c78a96a42e7f4a4f6b2edcdb791e44c" }, + { "vp90-2-14-resize-fp-tiles-1-2.webm", "e030450ae85c3277be2a418769df98e2" }, + { "vp90-2-14-resize-fp-tiles-1-4.webm", "312eed4e2b64eb7a4e7f18916606a430" }, + { "vp90-2-14-resize-fp-tiles-16-1.webm", "1755c16d8af16a9cb3fe7338d90abe52" }, + { "vp90-2-14-resize-fp-tiles-16-2.webm", "500300592d3fcb6f12fab25e48aaf4df" }, + { "vp90-2-14-resize-fp-tiles-16-4.webm", "47c48379fa6331215d91c67648e1af6e" }, + { "vp90-2-14-resize-fp-tiles-16-8-4-2-1.webm", + "eecf17290739bc708506fa4827665989" }, + { "vp90-2-14-resize-fp-tiles-16-8.webm", "29b6bb54e4c26b5ca85d5de5fed94e76" }, + { "vp90-2-14-resize-fp-tiles-1-8.webm", "1b6f175e08cd82cf84bb800ac6d1caa3" }, + { "vp90-2-14-resize-fp-tiles-2-16.webm", "ca3b03e4197995d8d5444ede7a6c0804" }, + { "vp90-2-14-resize-fp-tiles-2-1.webm", "99aec065369d70bbb78ccdff65afed3f" }, + { "vp90-2-14-resize-fp-tiles-2-4.webm", "22d0ebdb49b87d2920a85aea32e1afd5" }, + { "vp90-2-14-resize-fp-tiles-2-8.webm", "c2115cf051c62e0f7db1d4a783831541" }, + { "vp90-2-14-resize-fp-tiles-4-16.webm", "c690d7e1719b31367564cac0af0939cb" }, + { "vp90-2-14-resize-fp-tiles-4-1.webm", "a926020b2cc3e15ad4cc271853a0ff26" }, + { "vp90-2-14-resize-fp-tiles-4-2.webm", "42699063d9e581f1993d0cf890c2be78" }, + { "vp90-2-14-resize-fp-tiles-4-8.webm", "7f76d96036382f45121e3d5aa6f8ec52" }, + { "vp90-2-14-resize-fp-tiles-8-16.webm", "76a43fcdd7e658542913ea43216ec55d" }, + { "vp90-2-14-resize-fp-tiles-8-1.webm", "8e3fbe89486ca60a59299dea9da91378" }, + { "vp90-2-14-resize-fp-tiles-8-2.webm", "ae96f21f21b6370cc0125621b441fc52" }, + { "vp90-2-14-resize-fp-tiles-8-4.webm", "3eb4f24f10640d42218f7fd7b9fd30d4" }, +}; -TEST(VP9DecodeMultiThreadedTest, NonFrameParallel) { - static const FileList files[] = { - { "vp90-2-08-tile_1x2.webm", "570b4a5d5a70d58b5359671668328a16" }, - { "vp90-2-08-tile_1x4.webm", "988d86049e884c66909d2d163a09841a" }, - { "vp90-2-08-tile_1x8.webm", "0941902a52e9092cb010905eab16364c" }, - { "vp90-2-08-tile-4x1.webm", "06505aade6647c583c8e00a2f582266f" }, - { "vp90-2-08-tile-4x4.webm", "85c2299892460d76e2c600502d52bfe2" }, - { nullptr, nullptr } - }; +const FileParam kNonFrameParallelFiles[] = { + { "vp90-2-08-tile_1x2.webm", "570b4a5d5a70d58b5359671668328a16" }, + { "vp90-2-08-tile_1x4.webm", "988d86049e884c66909d2d163a09841a" }, + { "vp90-2-08-tile_1x8.webm", "0941902a52e9092cb010905eab16364c" }, + { "vp90-2-08-tile-4x1.webm", "06505aade6647c583c8e00a2f582266f" }, + { "vp90-2-08-tile-4x4.webm", "85c2299892460d76e2c600502d52bfe2" }, +}; - DecodeFiles(files); -} +INSTANTIATE_TEST_SUITE_P(NoTilesNonFrameParallel, VP9DecodeMultiThreadedTest, + ::testing::ValuesIn(kNoTilesNonFrameParallelFiles)); +INSTANTIATE_TEST_SUITE_P(FrameParallel, VP9DecodeMultiThreadedTest, + ::testing::ValuesIn(kFrameParallelFiles)); +INSTANTIATE_TEST_SUITE_P(FrameParallelResize, VP9DecodeMultiThreadedTest, + ::testing::ValuesIn(kFrameParallelResizeFiles)); +INSTANTIATE_TEST_SUITE_P(NonFrameParallel, VP9DecodeMultiThreadedTest, + ::testing::ValuesIn(kNonFrameParallelFiles)); #endif // CONFIG_WEBM_IO INSTANTIATE_TEST_SUITE_P(Synchronous, VPxWorkerThreadTest, ::testing::Bool()); } // namespace diff --git a/media/libvpx/libvpx/test/yuv_temporal_filter_test.cc b/media/libvpx/libvpx/test/yuv_temporal_filter_test.cc --- a/media/libvpx/libvpx/test/yuv_temporal_filter_test.cc +++ b/media/libvpx/libvpx/test/yuv_temporal_filter_test.cc @@ -672,21 +672,21 @@ block_width, block_height, ss_x, ss_y, strength, blk_fw, use_32x32, \ y_accumulator, y_count, u_accumulator, u_count, v_accumulator, \ v_count); \ } -WRAP_HIGHBD_FUNC(vp9_highbd_apply_temporal_filter_c, 10); -WRAP_HIGHBD_FUNC(vp9_highbd_apply_temporal_filter_c, 12); +WRAP_HIGHBD_FUNC(vp9_highbd_apply_temporal_filter_c, 10) +WRAP_HIGHBD_FUNC(vp9_highbd_apply_temporal_filter_c, 12) INSTANTIATE_TEST_SUITE_P( C, YUVTemporalFilterTest, ::testing::Values( TemporalFilterWithBd(&wrap_vp9_highbd_apply_temporal_filter_c_10, 10), TemporalFilterWithBd(&wrap_vp9_highbd_apply_temporal_filter_c_12, 12))); #if HAVE_SSE4_1 -WRAP_HIGHBD_FUNC(vp9_highbd_apply_temporal_filter_sse4_1, 10); -WRAP_HIGHBD_FUNC(vp9_highbd_apply_temporal_filter_sse4_1, 12); +WRAP_HIGHBD_FUNC(vp9_highbd_apply_temporal_filter_sse4_1, 10) +WRAP_HIGHBD_FUNC(vp9_highbd_apply_temporal_filter_sse4_1, 12) INSTANTIATE_TEST_SUITE_P( SSE4_1, YUVTemporalFilterTest, ::testing::Values( TemporalFilterWithBd(&wrap_vp9_highbd_apply_temporal_filter_sse4_1_10, diff --git a/media/libvpx/libvpx/third_party/googletest/README.libvpx b/media/libvpx/libvpx/third_party/googletest/README.libvpx --- a/media/libvpx/libvpx/third_party/googletest/README.libvpx +++ b/media/libvpx/libvpx/third_party/googletest/README.libvpx @@ -1,7 +1,7 @@ URL: https://github.com/google/googletest.git -Version: release-1.10.0-224-g23b2a3b1 +Version: release-1.11.0 License: BSD License File: LICENSE Description: Google's framework for writing C++ tests on a variety of platforms @@ -11,13 +11,11 @@ failures, various options for running the tests, and XML test report generation. Local Modifications: - Remove everything but: + CONTRIBUTORS googletest/ - CONTRIBUTORS include - LICENSE README.md src -- Enable kErrorOnUninstantiatedParameterizedTest and - kErrorOnUninstantiatedTypeParameterizedTest in gtest.cc + LICENSE diff --git a/media/libvpx/libvpx/third_party/googletest/src/CONTRIBUTORS b/media/libvpx/libvpx/third_party/googletest/src/CONTRIBUTORS --- a/media/libvpx/libvpx/third_party/googletest/src/CONTRIBUTORS +++ b/media/libvpx/libvpx/third_party/googletest/src/CONTRIBUTORS @@ -3,36 +3,61 @@ # who commit code to the project are encouraged to add their names # here. Please keep the list sorted by first names. Ajay Joshi Balázs Dán +Benoit Sigoure Bharat Mediratta +Bogdan Piloca Chandler Carruth Chris Prince Chris Taylor Dan Egnor +Dave MacLachlan +David Anderson +Dean Sturtevant Eric Roman +Gene Volovich Hady Zalek +Hal Burch Jeffrey Yasskin +Jim Keller +Joe Walnes +Jon Wray Jói Sigurðsson Keir Mierle Keith Ray Kenton Varda +Kostya Serebryany Krystian Kuzniarek +Lev Makhlis Manuel Klimek +Mario Tanev +Mark Paskin Markus Heule +Matthew Simmons Mika Raento +Mike Bland Miklós Fazekas +Neal Norwitz +Nermin Ozkiranartli +Owen Carlsen +Paneendra Ba Pasi Valminen Patrick Hanna Patrick Riley +Paul Menage Peter Kaminski +Piotr Kaminski Preston Jackson Rainer Klaffenboeck Russ Cox Russ Rufer Sean Mcafee Sigurður Ásgeirsson +Sverre Sundsdal +Takeshi Yoshino Tracy Bialik Vadim Berman Vlad Losev +Wolfgang Klier Zhanyong Wan diff --git a/media/libvpx/libvpx/third_party/googletest/src/README.md b/media/libvpx/libvpx/third_party/googletest/src/README.md --- a/media/libvpx/libvpx/third_party/googletest/src/README.md +++ b/media/libvpx/libvpx/third_party/googletest/src/README.md @@ -1,58 +1,76 @@ ### Generic Build Instructions #### Setup -To build Google Test and your tests that use it, you need to tell your build +To build GoogleTest and your tests that use it, you need to tell your build system where to find its headers and source files. The exact way to do it depends on which build system you use, and is usually straightforward. ### Build with CMake -Google Test comes with a CMake build script +GoogleTest comes with a CMake build script ([CMakeLists.txt](https://github.com/google/googletest/blob/master/CMakeLists.txt)) that can be used on a wide range of platforms ("C" stands for cross-platform.). If you don't have CMake installed already, you can download it for free from . CMake works by generating native makefiles or build projects that can be used in -the compiler environment of your choice. You can either build Google Test as a +the compiler environment of your choice. You can either build GoogleTest as a standalone project or it can be incorporated into an existing CMake build for another project. #### Standalone CMake Project -When building Google Test as a standalone project, the typical workflow starts -with: - - mkdir mybuild # Create a directory to hold the build output. - cd mybuild - cmake ${GTEST_DIR} # Generate native build scripts. - -If you want to build Google Test's samples, you should replace the last command +When building GoogleTest as a standalone project, the typical workflow starts with - cmake -Dgtest_build_samples=ON ${GTEST_DIR} +``` +git clone https://github.com/google/googletest.git -b release-1.10.0 +cd googletest # Main directory of the cloned repository. +mkdir build # Create a directory to hold the build output. +cd build +cmake .. # Generate native build scripts for GoogleTest. +``` + +The above command also includes GoogleMock by default. And so, if you want to +build only GoogleTest, you should replace the last command with + +``` +cmake .. -DBUILD_GMOCK=OFF +``` If you are on a \*nix system, you should now see a Makefile in the current -directory. Just type 'make' to build gtest. +directory. Just type `make` to build GoogleTest. And then you can simply install +GoogleTest if you are a system administrator. + +``` +make +sudo make install # Install in /usr/local/ by default +``` If you use Windows and have Visual Studio installed, a `gtest.sln` file and several `.vcproj` files will be created. You can then build them using Visual Studio. On Mac OS X with Xcode installed, a `.xcodeproj` file will be generated. #### Incorporating Into An Existing CMake Project -If you want to use gtest in a project which already uses CMake, then a more -robust and flexible approach is to build gtest as part of that project directly. -This is done by making the GoogleTest source code available to the main build -and adding it using CMake's `add_subdirectory()` command. This has the -significant advantage that the same compiler and linker settings are used -between gtest and the rest of your project, so issues associated with using -incompatible libraries (eg debug/release), etc. are avoided. This is +If you want to use GoogleTest in a project which already uses CMake, the easiest +way is to get installed libraries and headers. + +* Import GoogleTest by using `find_package` (or `pkg_check_modules`). For + example, if `find_package(GTest CONFIG REQUIRED)` succeeds, you can use the + libraries as `GTest::gtest`, `GTest::gmock`. + +And a more robust and flexible approach is to build GoogleTest as part of that +project directly. This is done by making the GoogleTest source code available to +the main build and adding it using CMake's `add_subdirectory()` command. This +has the significant advantage that the same compiler and linker settings are +used between GoogleTest and the rest of your project, so issues associated with +using incompatible libraries (eg debug/release), etc. are avoided. This is particularly useful on Windows. Making GoogleTest's source code available to the main build can be done a few different ways: * Download the GoogleTest source code manually and place it at a known location. This is the least flexible approach and can make it more difficult @@ -62,143 +80,96 @@ keep up to date. Some organizations may not permit this method. * Add GoogleTest as a git submodule or equivalent. This may not always be possible or appropriate. Git submodules, for example, have their own set of advantages and drawbacks. * Use CMake to download GoogleTest as part of the build's configure step. This - is just a little more complex, but doesn't have the limitations of the other - methods. - -The last of the above methods is implemented with a small piece of CMake code in -a separate file (e.g. `CMakeLists.txt.in`) which is copied to the build area and -then invoked as a sub-build _during the CMake stage_. That directory is then -pulled into the main build with `add_subdirectory()`. For example: - -New file `CMakeLists.txt.in`: - -```cmake -cmake_minimum_required(VERSION 2.8.2) - -project(googletest-download NONE) + approach doesn't have the limitations of the other methods. -include(ExternalProject) -ExternalProject_Add(googletest - GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG master - SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/googletest-src" - BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/googletest-build" - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" - TEST_COMMAND "" -) -``` +The last of the above methods is implemented with a small piece of CMake code +that downloads and pulls the GoogleTest code into the main build. -Existing build's `CMakeLists.txt`: +Just add to your `CMakeLists.txt`: ```cmake -# Download and unpack googletest at configure time -configure_file(CMakeLists.txt.in googletest-download/CMakeLists.txt) -execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" . - RESULT_VARIABLE result - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/googletest-download ) -if(result) - message(FATAL_ERROR "CMake step for googletest failed: ${result}") -endif() -execute_process(COMMAND ${CMAKE_COMMAND} --build . - RESULT_VARIABLE result - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/googletest-download ) -if(result) - message(FATAL_ERROR "Build step for googletest failed: ${result}") -endif() - -# Prevent overriding the parent project's compiler/linker -# settings on Windows +include(FetchContent) +FetchContent_Declare( + googletest + # Specify the commit you depend on and update it regularly. + URL https://github.com/google/googletest/archive/609281088cfefc76f9d0ce82e1ff6c30cc3591e5.zip +) +# For Windows: Prevent overriding the parent project's compiler/linker settings set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) - -# Add googletest directly to our build. This defines -# the gtest and gtest_main targets. -add_subdirectory(${CMAKE_CURRENT_BINARY_DIR}/googletest-src - ${CMAKE_CURRENT_BINARY_DIR}/googletest-build - EXCLUDE_FROM_ALL) - -# The gtest/gtest_main targets carry header search path -# dependencies automatically when using CMake 2.8.11 or -# later. Otherwise we have to add them here ourselves. -if (CMAKE_VERSION VERSION_LESS 2.8.11) - include_directories("${gtest_SOURCE_DIR}/include") -endif() +FetchContent_MakeAvailable(googletest) # Now simply link against gtest or gtest_main as needed. Eg add_executable(example example.cpp) target_link_libraries(example gtest_main) add_test(NAME example_test COMMAND example) ``` -Note that this approach requires CMake 2.8.2 or later due to its use of the -`ExternalProject_Add()` command. The above technique is discussed in more detail -in [this separate article](http://crascit.com/2015/07/25/cmake-gtest/) which -also contains a link to a fully generalized implementation of the technique. +Note that this approach requires CMake 3.14 or later due to its use of the +`FetchContent_MakeAvailable()` command. ##### Visual Studio Dynamic vs Static Runtimes By default, new Visual Studio projects link the C runtimes dynamically but -Google Test links them statically. This will generate an error that looks +GoogleTest links them statically. This will generate an error that looks something like the following: gtest.lib(gtest-all.obj) : error LNK2038: mismatch detected for 'RuntimeLibrary': value 'MTd_StaticDebug' doesn't match value 'MDd_DynamicDebug' in main.obj -Google Test already has a CMake option for this: `gtest_force_shared_crt` +GoogleTest already has a CMake option for this: `gtest_force_shared_crt` Enabling this option will make gtest link the runtimes dynamically too, and match the project in which it is included. #### C++ Standard Version An environment that supports C++11 is required in order to successfully build -Google Test. One way to ensure this is to specify the standard in the top-level +GoogleTest. One way to ensure this is to specify the standard in the top-level project, for example by using the `set(CMAKE_CXX_STANDARD 11)` command. If this -is not feasible, for example in a C project using Google Test for validation, +is not feasible, for example in a C project using GoogleTest for validation, then it can be specified by adding it to the options for cmake via the `DCMAKE_CXX_FLAGS` option. -### Tweaking Google Test +### Tweaking GoogleTest -Google Test can be used in diverse environments. The default configuration may +GoogleTest can be used in diverse environments. The default configuration may not work (or may not work well) out of the box in some environments. However, -you can easily tweak Google Test by defining control macros on the compiler +you can easily tweak GoogleTest by defining control macros on the compiler command line. Generally, these macros are named like `GTEST_XYZ` and you define them to either 1 or 0 to enable or disable a certain feature. We list the most frequently used macros below. For a complete list, see file [include/gtest/internal/gtest-port.h](https://github.com/google/googletest/blob/master/googletest/include/gtest/internal/gtest-port.h). ### Multi-threaded Tests -Google Test is thread-safe where the pthread library is available. After +GoogleTest is thread-safe where the pthread library is available. After `#include "gtest/gtest.h"`, you can check the `GTEST_IS_THREADSAFE` macro to see whether this is the case (yes if the macro is `#defined` to 1, no if it's undefined.). -If Google Test doesn't correctly detect whether pthread is available in your +If GoogleTest doesn't correctly detect whether pthread is available in your environment, you can force it with -DGTEST_HAS_PTHREAD=1 or -DGTEST_HAS_PTHREAD=0 -When Google Test uses pthread, you may need to add flags to your compiler and/or +When GoogleTest uses pthread, you may need to add flags to your compiler and/or linker to select the pthread library, or you'll get link errors. If you use the -CMake script or the deprecated Autotools script, this is taken care of for you. -If you use your own build script, you'll need to read your compiler and linker's -manual to figure out what flags to add. +CMake script, this is taken care of for you. If you use your own build script, +you'll need to read your compiler and linker's manual to figure out what flags +to add. ### As a Shared Library (DLL) -Google Test is compact, so most users can build and link it as a static library -for the simplicity. You can choose to use Google Test as a shared library (known +GoogleTest is compact, so most users can build and link it as a static library +for the simplicity. You can choose to use GoogleTest as a shared library (known as a DLL on Windows) if you prefer. To compile *gtest* as a shared library, add -DGTEST_CREATE_SHARED_LIBRARY=1 @@ -214,26 +185,26 @@ Note: while the above steps aren't technically necessary today when using some compilers (e.g. GCC), they may become necessary in the future, if we decide to improve the speed of loading the library (see for details). Therefore you are recommended -to always add the above flags when using Google Test as a shared library. -Otherwise a future release of Google Test may break your build script. +to always add the above flags when using GoogleTest as a shared library. +Otherwise a future release of GoogleTest may break your build script. ### Avoiding Macro Name Clashes In C++, macros don't obey namespaces. Therefore two libraries that both define a macro of the same name will clash if you `#include` both definitions. In case a -Google Test macro clashes with another library, you can force Google Test to +GoogleTest macro clashes with another library, you can force GoogleTest to rename its macro to avoid the conflict. -Specifically, if both Google Test and some other code define macro FOO, you can +Specifically, if both GoogleTest and some other code define macro FOO, you can add -DGTEST_DONT_DEFINE_FOO=1 -to the compiler flags to tell Google Test to change the macro's name from `FOO` +to the compiler flags to tell GoogleTest to change the macro's name from `FOO` to `GTEST_FOO`. Currently `FOO` can be `FAIL`, `SUCCEED`, or `TEST`. For example, with `-DGTEST_DONT_DEFINE_TEST=1`, you'll need to write GTEST_TEST(SomeTest, DoesThis) { ... } diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-death-test.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-death-test.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-death-test.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-death-test.h @@ -33,12 +33,12 @@ // This header file defines the public API for death tests. It is // #included by gtest.h so a user doesn't need to include this // directly. // GOOGLETEST_CM0001 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_GTEST_DEATH_TEST_H_ -#define GTEST_INCLUDE_GTEST_GTEST_DEATH_TEST_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_GTEST_DEATH_TEST_H_ +#define GOOGLETEST_INCLUDE_GTEST_GTEST_DEATH_TEST_H_ #include "gtest/internal/gtest-death-test-internal.h" namespace testing { @@ -95,10 +95,14 @@ // return WIFSIGNALED(exit_code) && WTERMSIG(exit_code) == SIGHUP; // } // // ASSERT_EXIT(client.HangUpServer(), KilledBySIGHUP, "Hanging up!"); // +// The final parameter to each of these macros is a matcher applied to any data +// the sub-process wrote to stderr. For compatibility with existing tests, a +// bare string is interpreted as a regular expression matcher. +// // On the regular expressions used in death tests: // // GOOGLETEST_CM0005 DO NOT DELETE // On POSIX-compliant systems (*nix), we use the library, // which uses the POSIX extended regex syntax. @@ -160,43 +164,42 @@ // /absolute/path/to/bar_test are fine, but foo_test is not). This // is rarely a problem as people usually don't put the test binary // directory in PATH. // -// Asserts that a given statement causes the program to exit, with an -// integer exit status that satisfies predicate, and emitting error output -// that matches regex. -# define ASSERT_EXIT(statement, predicate, regex) \ - GTEST_DEATH_TEST_(statement, predicate, regex, GTEST_FATAL_FAILURE_) +// Asserts that a given `statement` causes the program to exit, with an +// integer exit status that satisfies `predicate`, and emitting error output +// that matches `matcher`. +# define ASSERT_EXIT(statement, predicate, matcher) \ + GTEST_DEATH_TEST_(statement, predicate, matcher, GTEST_FATAL_FAILURE_) -// Like ASSERT_EXIT, but continues on to successive tests in the +// Like `ASSERT_EXIT`, but continues on to successive tests in the // test suite, if any: -# define EXPECT_EXIT(statement, predicate, regex) \ - GTEST_DEATH_TEST_(statement, predicate, regex, GTEST_NONFATAL_FAILURE_) +# define EXPECT_EXIT(statement, predicate, matcher) \ + GTEST_DEATH_TEST_(statement, predicate, matcher, GTEST_NONFATAL_FAILURE_) -// Asserts that a given statement causes the program to exit, either by +// Asserts that a given `statement` causes the program to exit, either by // explicitly exiting with a nonzero exit code or being killed by a -// signal, and emitting error output that matches regex. -# define ASSERT_DEATH(statement, regex) \ - ASSERT_EXIT(statement, ::testing::internal::ExitedUnsuccessfully, regex) +// signal, and emitting error output that matches `matcher`. +# define ASSERT_DEATH(statement, matcher) \ + ASSERT_EXIT(statement, ::testing::internal::ExitedUnsuccessfully, matcher) -// Like ASSERT_DEATH, but continues on to successive tests in the +// Like `ASSERT_DEATH`, but continues on to successive tests in the // test suite, if any: -# define EXPECT_DEATH(statement, regex) \ - EXPECT_EXIT(statement, ::testing::internal::ExitedUnsuccessfully, regex) +# define EXPECT_DEATH(statement, matcher) \ + EXPECT_EXIT(statement, ::testing::internal::ExitedUnsuccessfully, matcher) // Two predicate classes that can be used in {ASSERT,EXPECT}_EXIT*: // Tests that an exit code describes a normal exit with a given exit code. class GTEST_API_ ExitedWithCode { public: explicit ExitedWithCode(int exit_code); + ExitedWithCode(const ExitedWithCode&) = default; + void operator=(const ExitedWithCode& other) = delete; bool operator()(int exit_status) const; private: - // No implementation - assignment is unsupported. - void operator=(const ExitedWithCode& other); - const int exit_code_; }; # if !GTEST_OS_WINDOWS && !GTEST_OS_FUCHSIA // Tests that an exit code describes an exit due to termination by a @@ -338,6 +341,6 @@ GTEST_UNSUPPORTED_DEATH_TEST(statement, regex, return) #endif } // namespace testing -#endif // GTEST_INCLUDE_GTEST_GTEST_DEATH_TEST_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_GTEST_DEATH_TEST_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-matchers.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-matchers.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-matchers.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-matchers.h @@ -30,17 +30,14 @@ // The Google C++ Testing and Mocking Framework (Google Test) // // This file implements just enough of the matcher interface to allow // EXPECT_DEATH and friends to accept a matcher argument. -// IWYU pragma: private, include "testing/base/public/gunit.h" -// IWYU pragma: friend third_party/googletest/googlemock/.* -// IWYU pragma: friend third_party/googletest/googletest/.* +#ifndef GOOGLETEST_INCLUDE_GTEST_GTEST_MATCHERS_H_ +#define GOOGLETEST_INCLUDE_GTEST_GTEST_MATCHERS_H_ -#ifndef GTEST_INCLUDE_GTEST_GTEST_MATCHERS_H_ -#define GTEST_INCLUDE_GTEST_GTEST_MATCHERS_H_ - +#include #include #include #include #include @@ -61,24 +58,20 @@ /* Symbol involving type with internal linkage not defined */) namespace testing { // To implement a matcher Foo for type T, define: -// 1. a class FooMatcherImpl that implements the -// MatcherInterface interface, and -// 2. a factory function that creates a Matcher object from a -// FooMatcherImpl*. +// 1. a class FooMatcherMatcher that implements the matcher interface: +// using is_gtest_matcher = void; +// bool MatchAndExplain(const T&, std::ostream*); +// (MatchResultListener* can also be used instead of std::ostream*) +// void DescribeTo(std::ostream*); +// void DescribeNegationTo(std::ostream*); // -// The two-level delegation design makes it possible to allow a user -// to write "v" instead of "Eq(v)" where a Matcher is expected, which -// is impossible if we pass matchers by pointers. It also eases -// ownership management as Matcher objects can now be copied like -// plain values. +// 2. a factory function that creates a Matcher object from a +// FooMatcherMatcher. -// MatchResultListener is an abstract class. Its << operator can be -// used by a matcher to explain why a value matches or doesn't match. -// class MatchResultListener { public: // Creates a listener object with the given underlying ostream. The // listener does not own the ostream, and does not dereference it // in the constructor or destructor. @@ -111,11 +104,11 @@ inline MatchResultListener::~MatchResultListener() { } // An instance of a subclass of this knows how to describe itself as a // matcher. -class MatcherDescriberInterface { +class GTEST_API_ MatcherDescriberInterface { public: virtual ~MatcherDescriberInterface() {} // Describes this matcher to an ostream. The function should print // a verb phrase that describes the property a value matching this @@ -179,35 +172,10 @@ // virtual void DescribeNegationTo(::std::ostream* os) const; }; namespace internal { -// Converts a MatcherInterface to a MatcherInterface. -template -class MatcherInterfaceAdapter : public MatcherInterface { - public: - explicit MatcherInterfaceAdapter(const MatcherInterface* impl) - : impl_(impl) {} - ~MatcherInterfaceAdapter() override { delete impl_; } - - void DescribeTo(::std::ostream* os) const override { impl_->DescribeTo(os); } - - void DescribeNegationTo(::std::ostream* os) const override { - impl_->DescribeNegationTo(os); - } - - bool MatchAndExplain(const T& x, - MatchResultListener* listener) const override { - return impl_->MatchAndExplain(x, listener); - } - - private: - const MatcherInterface* const impl_; - - GTEST_DISALLOW_COPY_AND_ASSIGN_(MatcherInterfaceAdapter); -}; - struct AnyEq { template bool operator()(const A& a, const B& b) const { return a == b; } }; struct AnyNe { @@ -250,34 +218,57 @@ private: GTEST_DISALLOW_COPY_AND_ASSIGN_(StreamMatchResultListener); }; +struct SharedPayloadBase { + std::atomic ref{1}; + void Ref() { ref.fetch_add(1, std::memory_order_relaxed); } + bool Unref() { return ref.fetch_sub(1, std::memory_order_acq_rel) == 1; } +}; + +template +struct SharedPayload : SharedPayloadBase { + explicit SharedPayload(const T& v) : value(v) {} + explicit SharedPayload(T&& v) : value(std::move(v)) {} + + static void Destroy(SharedPayloadBase* shared) { + delete static_cast(shared); + } + + T value; +}; + // An internal class for implementing Matcher, which will derive // from it. We put functionalities common to all Matcher // specializations here to avoid code duplication. template -class MatcherBase { +class MatcherBase : private MatcherDescriberInterface { public: // Returns true if and only if the matcher matches x; also explains the // match result to 'listener'. bool MatchAndExplain(const T& x, MatchResultListener* listener) const { - return impl_->MatchAndExplain(x, listener); + GTEST_CHECK_(vtable_ != nullptr); + return vtable_->match_and_explain(*this, x, listener); } // Returns true if and only if this matcher matches x. bool Matches(const T& x) const { DummyMatchResultListener dummy; return MatchAndExplain(x, &dummy); } // Describes this matcher to an ostream. - void DescribeTo(::std::ostream* os) const { impl_->DescribeTo(os); } + void DescribeTo(::std::ostream* os) const final { + GTEST_CHECK_(vtable_ != nullptr); + vtable_->describe(*this, os, false); + } // Describes the negation of this matcher to an ostream. - void DescribeNegationTo(::std::ostream* os) const { - impl_->DescribeNegationTo(os); + void DescribeNegationTo(::std::ostream* os) const final { + GTEST_CHECK_(vtable_ != nullptr); + vtable_->describe(*this, os, true); } // Explains why x matches, or doesn't match, the matcher. void ExplainMatchResultTo(const T& x, ::std::ostream* os) const { StreamMatchResultListener listener(os); @@ -286,35 +277,198 @@ // Returns the describer for this matcher object; retains ownership // of the describer, which is only guaranteed to be alive when // this matcher object is alive. const MatcherDescriberInterface* GetDescriber() const { - return impl_.get(); + if (vtable_ == nullptr) return nullptr; + return vtable_->get_describer(*this); } protected: - MatcherBase() {} + MatcherBase() : vtable_(nullptr) {} // Constructs a matcher from its implementation. - explicit MatcherBase(const MatcherInterface* impl) : impl_(impl) {} + template + explicit MatcherBase(const MatcherInterface* impl) { + Init(impl); + } + + template ::type::is_gtest_matcher> + MatcherBase(M&& m) { // NOLINT + Init(std::forward(m)); + } + + MatcherBase(const MatcherBase& other) + : vtable_(other.vtable_), buffer_(other.buffer_) { + if (IsShared()) buffer_.shared->Ref(); + } - template - explicit MatcherBase( - const MatcherInterface* impl, - typename std::enable_if::value>::type* = - nullptr) - : impl_(new internal::MatcherInterfaceAdapter(impl)) {} + MatcherBase& operator=(const MatcherBase& other) { + if (this == &other) return *this; + Destroy(); + vtable_ = other.vtable_; + buffer_ = other.buffer_; + if (IsShared()) buffer_.shared->Ref(); + return *this; + } - MatcherBase(const MatcherBase&) = default; - MatcherBase& operator=(const MatcherBase&) = default; - MatcherBase(MatcherBase&&) = default; - MatcherBase& operator=(MatcherBase&&) = default; + MatcherBase(MatcherBase&& other) + : vtable_(other.vtable_), buffer_(other.buffer_) { + other.vtable_ = nullptr; + } - virtual ~MatcherBase() {} + MatcherBase& operator=(MatcherBase&& other) { + if (this == &other) return *this; + Destroy(); + vtable_ = other.vtable_; + buffer_ = other.buffer_; + other.vtable_ = nullptr; + return *this; + } + + ~MatcherBase() override { Destroy(); } private: - std::shared_ptr> impl_; + struct VTable { + bool (*match_and_explain)(const MatcherBase&, const T&, + MatchResultListener*); + void (*describe)(const MatcherBase&, std::ostream*, bool negation); + // Returns the captured object if it implements the interface, otherwise + // returns the MatcherBase itself. + const MatcherDescriberInterface* (*get_describer)(const MatcherBase&); + // Called on shared instances when the reference count reaches 0. + void (*shared_destroy)(SharedPayloadBase*); + }; + + bool IsShared() const { + return vtable_ != nullptr && vtable_->shared_destroy != nullptr; + } + + // If the implementation uses a listener, call that. + template + static auto MatchAndExplainImpl(const MatcherBase& m, const T& value, + MatchResultListener* listener) + -> decltype(P::Get(m).MatchAndExplain(value, listener->stream())) { + return P::Get(m).MatchAndExplain(value, listener->stream()); + } + + template + static auto MatchAndExplainImpl(const MatcherBase& m, const T& value, + MatchResultListener* listener) + -> decltype(P::Get(m).MatchAndExplain(value, listener)) { + return P::Get(m).MatchAndExplain(value, listener); + } + + template + static void DescribeImpl(const MatcherBase& m, std::ostream* os, + bool negation) { + if (negation) { + P::Get(m).DescribeNegationTo(os); + } else { + P::Get(m).DescribeTo(os); + } + } + + template + static const MatcherDescriberInterface* GetDescriberImpl( + const MatcherBase& m) { + // If the impl is a MatcherDescriberInterface, then return it. + // Otherwise use MatcherBase itself. + // This allows us to implement the GetDescriber() function without support + // from the impl, but some users really want to get their impl back when + // they call GetDescriber(). + // We use std::get on a tuple as a workaround of not having `if constexpr`. + return std::get<( + std::is_convertible::value + ? 1 + : 0)>(std::make_tuple(&m, &P::Get(m))); + } + + template + const VTable* GetVTable() { + static constexpr VTable kVTable = {&MatchAndExplainImpl

, + &DescribeImpl

, &GetDescriberImpl

, + P::shared_destroy}; + return &kVTable; + } + + union Buffer { + // Add some types to give Buffer some common alignment/size use cases. + void* ptr; + double d; + int64_t i; + // And add one for the out-of-line cases. + SharedPayloadBase* shared; + }; + + void Destroy() { + if (IsShared() && buffer_.shared->Unref()) { + vtable_->shared_destroy(buffer_.shared); + } + } + + template + static constexpr bool IsInlined() { + return sizeof(M) <= sizeof(Buffer) && alignof(M) <= alignof(Buffer) && + std::is_trivially_copy_constructible::value && + std::is_trivially_destructible::value; + } + + template ()> + struct ValuePolicy { + static const M& Get(const MatcherBase& m) { + // When inlined along with Init, need to be explicit to avoid violating + // strict aliasing rules. + const M *ptr = static_cast( + static_cast(&m.buffer_)); + return *ptr; + } + static void Init(MatcherBase& m, M impl) { + ::new (static_cast(&m.buffer_)) M(impl); + } + static constexpr auto shared_destroy = nullptr; + }; + + template + struct ValuePolicy { + using Shared = SharedPayload; + static const M& Get(const MatcherBase& m) { + return static_cast(m.buffer_.shared)->value; + } + template + static void Init(MatcherBase& m, Arg&& arg) { + m.buffer_.shared = new Shared(std::forward(arg)); + } + static constexpr auto shared_destroy = &Shared::Destroy; + }; + + template + struct ValuePolicy*, B> { + using M = const MatcherInterface; + using Shared = SharedPayload>; + static const M& Get(const MatcherBase& m) { + return *static_cast(m.buffer_.shared)->value; + } + static void Init(MatcherBase& m, M* impl) { + m.buffer_.shared = new Shared(std::unique_ptr(impl)); + } + + static constexpr auto shared_destroy = &Shared::Destroy; + }; + + template + void Init(M&& m) { + using MM = typename std::decay::type; + using Policy = ValuePolicy; + vtable_ = GetVTable(); + Policy::Init(*this, std::forward(m)); + } + + const VTable* vtable_; + Buffer buffer_; }; } // namespace internal // A Matcher is a copyable and IMMUTABLE (except by assignment) @@ -338,10 +492,14 @@ const MatcherInterface* impl, typename std::enable_if::value>::type* = nullptr) : internal::MatcherBase(impl) {} + template ::type::is_gtest_matcher> + Matcher(M&& m) : internal::MatcherBase(std::forward(m)) {} // NOLINT + // Implicit constructor here allows people to write // EXPECT_CALL(foo, Bar(5)) instead of EXPECT_CALL(foo, Bar(Eq(5))) sometimes Matcher(T value); // NOLINT }; @@ -355,10 +513,15 @@ Matcher() {} explicit Matcher(const MatcherInterface* impl) : internal::MatcherBase(impl) {} + template ::type::is_gtest_matcher> + Matcher(M&& m) // NOLINT + : internal::MatcherBase(std::forward(m)) {} + // Allows the user to write str instead of Eq(str) sometimes, where // str is a std::string object. Matcher(const std::string& s); // NOLINT // Allows the user to write "foo" instead of Eq("foo") sometimes. @@ -374,10 +537,15 @@ explicit Matcher(const MatcherInterface* impl) : internal::MatcherBase(impl) {} explicit Matcher(const MatcherInterface* impl) : internal::MatcherBase(impl) {} + template ::type::is_gtest_matcher> + Matcher(M&& m) // NOLINT + : internal::MatcherBase(std::forward(m)) {} + // Allows the user to write str instead of Eq(str) sometimes, where // str is a string object. Matcher(const std::string& s); // NOLINT // Allows the user to write "foo" instead of Eq("foo") sometimes. @@ -395,10 +563,16 @@ Matcher() {} explicit Matcher(const MatcherInterface* impl) : internal::MatcherBase(impl) {} + template ::type::is_gtest_matcher> + Matcher(M&& m) // NOLINT + : internal::MatcherBase(std::forward(m)) { + } + // Allows the user to write str instead of Eq(str) sometimes, where // str is a std::string object. Matcher(const std::string& s); // NOLINT // Allows the user to write "foo" instead of Eq("foo") sometimes. @@ -417,10 +591,15 @@ explicit Matcher(const MatcherInterface* impl) : internal::MatcherBase(impl) {} explicit Matcher(const MatcherInterface* impl) : internal::MatcherBase(impl) {} + template ::type::is_gtest_matcher> + Matcher(M&& m) // NOLINT + : internal::MatcherBase(std::forward(m)) {} + // Allows the user to write str instead of Eq(str) sometimes, where // str is a std::string object. Matcher(const std::string& s); // NOLINT // Allows the user to write "foo" instead of Eq("foo") sometimes. @@ -527,41 +706,36 @@ // a "bare" type (i.e. neither 'const T' nor 'T&'). template class ComparisonBase { public: explicit ComparisonBase(const Rhs& rhs) : rhs_(rhs) {} + + using is_gtest_matcher = void; + template - operator Matcher() const { - return Matcher(new Impl(rhs_)); + bool MatchAndExplain(const Lhs& lhs, std::ostream*) const { + return Op()(lhs, Unwrap(rhs_)); + } + void DescribeTo(std::ostream* os) const { + *os << D::Desc() << " "; + UniversalPrint(Unwrap(rhs_), os); + } + void DescribeNegationTo(std::ostream* os) const { + *os << D::NegatedDesc() << " "; + UniversalPrint(Unwrap(rhs_), os); } private: template - static const T& Unwrap(const T& v) { return v; } + static const T& Unwrap(const T& v) { + return v; + } template - static const T& Unwrap(std::reference_wrapper v) { return v; } + static const T& Unwrap(std::reference_wrapper v) { + return v; + } - template - class Impl : public MatcherInterface { - public: - explicit Impl(const Rhs& rhs) : rhs_(rhs) {} - bool MatchAndExplain(Lhs lhs, - MatchResultListener* /* listener */) const override { - return Op()(lhs, Unwrap(rhs_)); - } - void DescribeTo(::std::ostream* os) const override { - *os << D::Desc() << " "; - UniversalPrint(Unwrap(rhs_), os); - } - void DescribeNegationTo(::std::ostream* os) const override { - *os << D::NegatedDesc() << " "; - UniversalPrint(Unwrap(rhs_), os); - } - - private: - Rhs rhs_; - }; Rhs rhs_; }; template class EqMatcher : public ComparisonBase, Rhs, AnyEq> { @@ -610,10 +784,14 @@ : ComparisonBase, Rhs, AnyGe>(rhs) { } static const char* Desc() { return "is >="; } static const char* NegatedDesc() { return "isn't >="; } }; +template ::value>::type> +using StringLike = T; + // Implements polymorphic matchers MatchesRegex(regex) and // ContainsRegex(regex), which can be used as a Matcher as long as // T can be converted to a string. class MatchesRegexMatcher { public: @@ -670,24 +848,26 @@ // The matcher takes ownership of 'regex'. inline PolymorphicMatcher MatchesRegex( const internal::RE* regex) { return MakePolymorphicMatcher(internal::MatchesRegexMatcher(regex, true)); } -inline PolymorphicMatcher MatchesRegex( - const std::string& regex) { - return MatchesRegex(new internal::RE(regex)); +template +PolymorphicMatcher MatchesRegex( + const internal::StringLike& regex) { + return MatchesRegex(new internal::RE(std::string(regex))); } // Matches a string that contains regular expression 'regex'. // The matcher takes ownership of 'regex'. inline PolymorphicMatcher ContainsRegex( const internal::RE* regex) { return MakePolymorphicMatcher(internal::MatchesRegexMatcher(regex, false)); } -inline PolymorphicMatcher ContainsRegex( - const std::string& regex) { - return ContainsRegex(new internal::RE(regex)); +template +PolymorphicMatcher ContainsRegex( + const internal::StringLike& regex) { + return ContainsRegex(new internal::RE(std::string(regex))); } // Creates a polymorphic matcher that matches anything equal to x. // Note: if the parameter of Eq() were declared as const T&, Eq("foo") // wouldn't compile. @@ -745,6 +925,6 @@ } } // namespace testing GTEST_DISABLE_MSC_WARNINGS_POP_() // 4251 5046 -#endif // GTEST_INCLUDE_GTEST_GTEST_MATCHERS_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_GTEST_MATCHERS_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-message.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-message.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-message.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-message.h @@ -42,12 +42,12 @@ // to CHANGE WITHOUT NOTICE. Therefore DO NOT DEPEND ON IT in a user // program! // GOOGLETEST_CM0001 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_GTEST_MESSAGE_H_ -#define GTEST_INCLUDE_GTEST_GTEST_MESSAGE_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_GTEST_MESSAGE_H_ +#define GOOGLETEST_INCLUDE_GTEST_GTEST_MESSAGE_H_ #include #include #include @@ -214,6 +214,6 @@ } // namespace internal } // namespace testing GTEST_DISABLE_MSC_WARNINGS_POP_() // 4251 -#endif // GTEST_INCLUDE_GTEST_GTEST_MESSAGE_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_GTEST_MESSAGE_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-param-test.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-param-test.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-param-test.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-param-test.h @@ -28,16 +28,13 @@ // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // Macros and functions for implementing parameterized tests // in Google C++ Testing and Mocking Framework (Google Test) // -// This file is generated by a SCRIPT. DO NOT EDIT BY HAND! -// // GOOGLETEST_CM0001 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ -#define GTEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ - +#ifndef GOOGLETEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ +#define GOOGLETEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ // Value-parameterized tests allow you to test your code with different // parameters without writing multiple copies of the same test. // // Here is how you use value-parameterized tests: @@ -369,12 +366,10 @@ // the Cartesian product of elements from the sequences generated by // gen1, gen2, ..., genN. The sequence elements will have a type of // std::tuple where T1, T2, ..., TN are the types // of elements from sequences produces by gen1, gen2, ..., genN. // -// Combine can have up to 10 arguments. -// // Example: // // This will instantiate tests in test suite AnimalTest each one with // the parameter values tuple("cat", BLACK), tuple("cat", WHITE), // tuple("dog", BLACK), and tuple("dog", WHITE): @@ -426,11 +421,12 @@ GTEST_STRINGIFY_(test_suite_name), \ ::testing::internal::CodeLocation(__FILE__, __LINE__)) \ ->AddTestPattern( \ GTEST_STRINGIFY_(test_suite_name), GTEST_STRINGIFY_(test_name), \ new ::testing::internal::TestMetaFactory()); \ + test_suite_name, test_name)>(), \ + ::testing::internal::CodeLocation(__FILE__, __LINE__)); \ return 0; \ } \ static int gtest_registering_dummy_ GTEST_ATTRIBUTE_UNUSED_; \ GTEST_DISALLOW_COPY_AND_ASSIGN_(GTEST_TEST_CLASS_NAME_(test_suite_name, \ test_name)); \ @@ -506,6 +502,6 @@ INSTANTIATE_TEST_SUITE_P #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ } // namespace testing -#endif // GTEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_GTEST_PARAM_TEST_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-printers.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-printers.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-printers.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-printers.h @@ -95,204 +95,225 @@ // being defined as many user-defined container types don't have // value_type. // GOOGLETEST_CM0001 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_GTEST_PRINTERS_H_ -#define GTEST_INCLUDE_GTEST_GTEST_PRINTERS_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_GTEST_PRINTERS_H_ +#define GOOGLETEST_INCLUDE_GTEST_GTEST_PRINTERS_H_ #include +#include #include // NOLINT #include #include #include #include #include #include + #include "gtest/internal/gtest-internal.h" #include "gtest/internal/gtest-port.h" -#if GTEST_HAS_ABSL -#include "absl/strings/string_view.h" -#include "absl/types/optional.h" -#include "absl/types/variant.h" -#endif // GTEST_HAS_ABSL - namespace testing { -// Definitions in the 'internal' and 'internal2' name spaces are -// subject to change without notice. DO NOT USE THEM IN USER CODE! -namespace internal2 { +// Definitions in the internal* namespaces are subject to change without notice. +// DO NOT USE THEM IN USER CODE! +namespace internal { -// Prints the given number of bytes in the given object to the given -// ostream. -GTEST_API_ void PrintBytesInObjectTo(const unsigned char* obj_bytes, - size_t count, - ::std::ostream* os); +template +void UniversalPrint(const T& value, ::std::ostream* os); -// For selecting which printer to use when a given type has neither << -// nor PrintTo(). -enum TypeKind { - kProtobuf, // a protobuf type - kConvertibleToInteger, // a type implicitly convertible to BiggestInt - // (e.g. a named or unnamed enum type) -#if GTEST_INTERNAL_HAS_STRING_VIEW - kConvertibleToStringView, // a type implicitly convertible to - // absl::string_view or std::string_view -#endif - kOtherType // anything else -}; +// Used to print an STL-style container when the user doesn't define +// a PrintTo() for it. +struct ContainerPrinter { + template (0)) == sizeof(IsContainer)) && + !IsRecursiveContainer::value>::type> + static void PrintValue(const T& container, std::ostream* os) { + const size_t kMaxCount = 32; // The maximum number of elements to print. + *os << '{'; + size_t count = 0; + for (auto&& elem : container) { + if (count > 0) { + *os << ','; + if (count == kMaxCount) { // Enough has been printed. + *os << " ..."; + break; + } + } + *os << ' '; + // We cannot call PrintTo(elem, os) here as PrintTo() doesn't + // handle `elem` being a native array. + internal::UniversalPrint(elem, os); + ++count; + } -// TypeWithoutFormatter::PrintValue(value, os) is called -// by the universal printer to print a value of type T when neither -// operator<< nor PrintTo() is defined for T, where kTypeKind is the -// "kind" of T as defined by enum TypeKind. -template -class TypeWithoutFormatter { - public: - // This default version is called when kTypeKind is kOtherType. - static void PrintValue(const T& value, ::std::ostream* os) { - PrintBytesInObjectTo( - static_cast( - reinterpret_cast(std::addressof(value))), - sizeof(value), os); + if (count > 0) { + *os << ' '; + } + *os << '}'; } }; -// We print a protobuf using its ShortDebugString() when the string -// doesn't exceed this many characters; otherwise we print it using -// DebugString() for better readability. -const size_t kProtobufOneLinerMaxLength = 50; +// Used to print a pointer that is neither a char pointer nor a member +// pointer, when the user doesn't define PrintTo() for it. (A member +// variable pointer or member function pointer doesn't really point to +// a location in the address space. Their representation is +// implementation-defined. Therefore they will be printed as raw +// bytes.) +struct FunctionPointerPrinter { + template ::value>::type> + static void PrintValue(T* p, ::std::ostream* os) { + if (p == nullptr) { + *os << "NULL"; + } else { + // T is a function type, so '*os << p' doesn't do what we want + // (it just prints p as bool). We want to print p as a const + // void*. + *os << reinterpret_cast(p); + } + } +}; + +struct PointerPrinter { + template + static void PrintValue(T* p, ::std::ostream* os) { + if (p == nullptr) { + *os << "NULL"; + } else { + // T is not a function type. We just call << to print p, + // relying on ADL to pick up user-defined << for their pointer + // types, if any. + *os << p; + } + } +}; -template -class TypeWithoutFormatter { - public: +namespace internal_stream_operator_without_lexical_name_lookup { + +// The presence of an operator<< here will terminate lexical scope lookup +// straight away (even though it cannot be a match because of its argument +// types). Thus, the two operator<< calls in StreamPrinter will find only ADL +// candidates. +struct LookupBlocker {}; +void operator<<(LookupBlocker, LookupBlocker); + +struct StreamPrinter { + template ::value>::type, + // Only accept types for which we can find a streaming operator via + // ADL (possibly involving implicit conversions). + typename = decltype(std::declval() + << std::declval())> + static void PrintValue(const T& value, ::std::ostream* os) { + // Call streaming operator found by ADL, possibly with implicit conversions + // of the arguments. + *os << value; + } +}; + +} // namespace internal_stream_operator_without_lexical_name_lookup + +struct ProtobufPrinter { + // We print a protobuf using its ShortDebugString() when the string + // doesn't exceed this many characters; otherwise we print it using + // DebugString() for better readability. + static const size_t kProtobufOneLinerMaxLength = 50; + + template ::value>::type> static void PrintValue(const T& value, ::std::ostream* os) { std::string pretty_str = value.ShortDebugString(); if (pretty_str.length() > kProtobufOneLinerMaxLength) { pretty_str = "\n" + value.DebugString(); } *os << ("<" + pretty_str + ">"); } }; -template -class TypeWithoutFormatter { - public: +struct ConvertibleToIntegerPrinter { // Since T has no << operator or PrintTo() but can be implicitly // converted to BiggestInt, we print it as a BiggestInt. // // Most likely T is an enum type (either named or unnamed), in which // case printing it as an integer is the desired behavior. In case // T is not an enum, printing it as an integer is the best we can do // given that it has no user-defined printer. + static void PrintValue(internal::BiggestInt value, ::std::ostream* os) { + *os << value; + } +}; + +struct ConvertibleToStringViewPrinter { +#if GTEST_INTERNAL_HAS_STRING_VIEW + static void PrintValue(internal::StringView value, ::std::ostream* os) { + internal::UniversalPrint(value, os); + } +#endif +}; + + +// Prints the given number of bytes in the given object to the given +// ostream. +GTEST_API_ void PrintBytesInObjectTo(const unsigned char* obj_bytes, + size_t count, + ::std::ostream* os); +struct RawBytesPrinter { + // SFINAE on `sizeof` to make sure we have a complete type. + template static void PrintValue(const T& value, ::std::ostream* os) { - const internal::BiggestInt kBigInt = value; - *os << kBigInt; + PrintBytesInObjectTo( + static_cast( + // Load bearing cast to void* to support iOS + reinterpret_cast(std::addressof(value))), + sizeof(value), os); } }; -#if GTEST_INTERNAL_HAS_STRING_VIEW -template -class TypeWithoutFormatter { - public: - // Since T has neither operator<< nor PrintTo() but can be implicitly - // converted to absl::string_view, we print it as a absl::string_view - // (or std::string_view). - // - // Note: the implementation is further below, as it depends on - // internal::PrintTo symbol which is defined later in the file. - static void PrintValue(const T& value, ::std::ostream* os); +struct FallbackPrinter { + template + static void PrintValue(const T&, ::std::ostream* os) { + *os << "(incomplete type)"; + } }; -#endif + +// Try every printer in order and return the first one that works. +template +struct FindFirstPrinter : FindFirstPrinter {}; + +template +struct FindFirstPrinter< + T, decltype(Printer::PrintValue(std::declval(), nullptr)), + Printer, Printers...> { + using type = Printer; +}; -// Prints the given value to the given ostream. If the value is a -// protocol message, its debug string is printed; if it's an enum or -// of a type implicitly convertible to BiggestInt, it's printed as an -// integer; otherwise the bytes in the value are printed. This is -// what UniversalPrinter::Print() does when it knows nothing about -// type T and T has neither << operator nor PrintTo(). -// -// A user can override this behavior for a class type Foo by defining -// a << operator in the namespace where Foo is defined. -// -// We put this operator in namespace 'internal2' instead of 'internal' -// to simplify the implementation, as much code in 'internal' needs to -// use << in STL, which would conflict with our own << were it defined -// in 'internal'. -// -// Note that this operator<< takes a generic std::basic_ostream type instead of the more restricted std::ostream. If -// we define it to take an std::ostream instead, we'll get an -// "ambiguous overloads" compiler error when trying to print a type -// Foo that supports streaming to std::basic_ostream, as the compiler cannot tell whether -// operator<<(std::ostream&, const T&) or -// operator<<(std::basic_stream, const Foo&) is more -// specific. -template -::std::basic_ostream& operator<<( - ::std::basic_ostream& os, const T& x) { - TypeWithoutFormatter::value - ? kProtobuf - : std::is_convertible< - const T&, internal::BiggestInt>::value - ? kConvertibleToInteger - : -#if GTEST_INTERNAL_HAS_STRING_VIEW - std::is_convertible< - const T&, internal::StringView>::value - ? kConvertibleToStringView - : -#endif - kOtherType)>::PrintValue(x, &os); - return os; +// Select the best printer in the following order: +// - Print containers (they have begin/end/etc). +// - Print function pointers. +// - Print object pointers. +// - Use the stream operator, if available. +// - Print protocol buffers. +// - Print types convertible to BiggestInt. +// - Print types convertible to StringView, if available. +// - Fallback to printing the raw bytes of the object. +template +void PrintWithFallback(const T& value, ::std::ostream* os) { + using Printer = typename FindFirstPrinter< + T, void, ContainerPrinter, FunctionPointerPrinter, PointerPrinter, + internal_stream_operator_without_lexical_name_lookup::StreamPrinter, + ProtobufPrinter, ConvertibleToIntegerPrinter, + ConvertibleToStringViewPrinter, RawBytesPrinter, FallbackPrinter>::type; + Printer::PrintValue(value, os); } -} // namespace internal2 -} // namespace testing - -// This namespace MUST NOT BE NESTED IN ::testing, or the name look-up -// magic needed for implementing UniversalPrinter won't work. -namespace testing_internal { - -// Used to print a value that is not an STL-style container when the -// user doesn't define PrintTo() for it. -template -void DefaultPrintNonContainerTo(const T& value, ::std::ostream* os) { - // With the following statement, during unqualified name lookup, - // testing::internal2::operator<< appears as if it was declared in - // the nearest enclosing namespace that contains both - // ::testing_internal and ::testing::internal2, i.e. the global - // namespace. For more details, refer to the C++ Standard section - // 7.3.4-1 [namespace.udir]. This allows us to fall back onto - // testing::internal2::operator<< in case T doesn't come with a << - // operator. - - using ::testing::internal2::operator<<; - - // Assuming T is defined in namespace foo, in the next statement, - // the compiler will consider all of: - // - // 1. foo::operator<< (thanks to Koenig look-up), - // 2. ::operator<< (as the current namespace is enclosed in ::), - // 3. testing::internal2::operator<< (thanks to the using statement above). - // - // The operator<< whose type matches T best will be picked. - // - // We deliberately allow #2 to be a candidate, as sometimes it's - // impossible to define #1 (e.g. when foo is ::std, defining - // anything in it is undefined behavior unless you are a compiler - // vendor.). - *os << value; -} - -} // namespace testing_internal - -namespace testing { -namespace internal { - // FormatForComparison::Format(value) formats a // value of type ToPrint that is an operand of a comparison assertion // (e.g. ASSERT_EQ). OtherOperand is the type of the other operand in // the comparison, and is used to help determine the best way to // format the value. In particular, when the value is a C string @@ -337,10 +358,18 @@ GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(char); GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(const char); GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(wchar_t); GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(const wchar_t); +#ifdef __cpp_char8_t +GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(char8_t); +GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(const char8_t); +#endif +GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(char16_t); +GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(const char16_t); +GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(char32_t); +GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_(const char32_t); #undef GTEST_IMPL_FORMAT_C_STRING_AS_POINTER_ // If a C string is compared with an STL string object, we know it's meant // to point to a NUL-terminated string, and thus can print it as a string. @@ -354,10 +383,18 @@ } \ } GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(char, ::std::string); GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const char, ::std::string); +#ifdef __cpp_char8_t +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(char8_t, ::std::u8string); +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const char8_t, ::std::u8string); +#endif +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(char16_t, ::std::u16string); +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const char16_t, ::std::u16string); +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(char32_t, ::std::u32string); +GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const char32_t, ::std::u32string); #if GTEST_HAS_STD_WSTRING GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(wchar_t, ::std::wstring); GTEST_IMPL_FORMAT_C_STRING_AS_STRING_(const wchar_t, ::std::wstring); #endif @@ -386,89 +423,10 @@ // function template), as we need to partially specialize it for // reference types, which cannot be done with function templates. template class UniversalPrinter; -template -void UniversalPrint(const T& value, ::std::ostream* os); - -enum DefaultPrinterType { - kPrintContainer, - kPrintPointer, - kPrintFunctionPointer, - kPrintOther, -}; -template struct WrapPrinterType {}; - -// Used to print an STL-style container when the user doesn't define -// a PrintTo() for it. -template -void DefaultPrintTo(WrapPrinterType /* dummy */, - const C& container, ::std::ostream* os) { - const size_t kMaxCount = 32; // The maximum number of elements to print. - *os << '{'; - size_t count = 0; - for (typename C::const_iterator it = container.begin(); - it != container.end(); ++it, ++count) { - if (count > 0) { - *os << ','; - if (count == kMaxCount) { // Enough has been printed. - *os << " ..."; - break; - } - } - *os << ' '; - // We cannot call PrintTo(*it, os) here as PrintTo() doesn't - // handle *it being a native array. - internal::UniversalPrint(*it, os); - } - - if (count > 0) { - *os << ' '; - } - *os << '}'; -} - -// Used to print a pointer that is neither a char pointer nor a member -// pointer, when the user doesn't define PrintTo() for it. (A member -// variable pointer or member function pointer doesn't really point to -// a location in the address space. Their representation is -// implementation-defined. Therefore they will be printed as raw -// bytes.) -template -void DefaultPrintTo(WrapPrinterType /* dummy */, - T* p, ::std::ostream* os) { - if (p == nullptr) { - *os << "NULL"; - } else { - // T is not a function type. We just call << to print p, - // relying on ADL to pick up user-defined << for their pointer - // types, if any. - *os << p; - } -} -template -void DefaultPrintTo(WrapPrinterType /* dummy */, - T* p, ::std::ostream* os) { - if (p == nullptr) { - *os << "NULL"; - } else { - // T is a function type, so '*os << p' doesn't do what we want - // (it just prints p as bool). We want to print p as a const - // void*. - *os << reinterpret_cast(p); - } -} - -// Used to print a non-container, non-pointer value when the user -// doesn't define PrintTo() for it. -template -void DefaultPrintTo(WrapPrinterType /* dummy */, - const T& value, ::std::ostream* os) { - ::testing_internal::DefaultPrintNonContainerTo(value, os); -} - // Prints the given value using the << operator if it has one; // otherwise prints the bytes in it. This is what // UniversalPrinter::Print() does when PrintTo() is not specialized // or overloaded for type T. // @@ -478,40 +436,11 @@ // Foo is not desirable (e.g. the coding style may prevent doing it, // or there is already a << operator but it doesn't do what the user // wants). template void PrintTo(const T& value, ::std::ostream* os) { - // DefaultPrintTo() is overloaded. The type of its first argument - // determines which version will be picked. - // - // Note that we check for container types here, prior to we check - // for protocol message types in our operator<<. The rationale is: - // - // For protocol messages, we want to give people a chance to - // override Google Mock's format by defining a PrintTo() or - // operator<<. For STL containers, other formats can be - // incompatible with Google Mock's format for the container - // elements; therefore we check for container types here to ensure - // that our format is used. - // - // Note that MSVC and clang-cl do allow an implicit conversion from - // pointer-to-function to pointer-to-object, but clang-cl warns on it. - // So don't use ImplicitlyConvertible if it can be helped since it will - // cause this warning, and use a separate overload of DefaultPrintTo for - // function pointers so that the `*os << p` in the object pointer overload - // doesn't cause that warning either. - DefaultPrintTo( - WrapPrinterType < - (sizeof(IsContainerTest(0)) == sizeof(IsContainer)) && - !IsRecursiveContainer::value - ? kPrintContainer - : !std::is_pointer::value - ? kPrintOther - : std::is_function::type>::value - ? kPrintFunctionPointer - : kPrintPointer > (), - value, os); + internal::PrintWithFallback(value, os); } // The following list of PrintTo() overloads tells // UniversalPrinter::Print() how to print standard types (built-in // types, strings, plain arrays, and pointers). @@ -538,10 +467,20 @@ // as signed integer when wchar_t is implemented by the compiler // as a signed type and is printed as an unsigned integer when wchar_t // is implemented as an unsigned type. GTEST_API_ void PrintTo(wchar_t wc, ::std::ostream* os); +GTEST_API_ void PrintTo(char32_t c, ::std::ostream* os); +inline void PrintTo(char16_t c, ::std::ostream* os) { + PrintTo(ImplicitCast_(c), os); +} +#ifdef __cpp_char8_t +inline void PrintTo(char8_t c, ::std::ostream* os) { + PrintTo(ImplicitCast_(c), os); +} +#endif + // Overloads for C strings. GTEST_API_ void PrintTo(const char* s, ::std::ostream* os); inline void PrintTo(char* s, ::std::ostream* os) { PrintTo(ImplicitCast_(s), os); } @@ -558,10 +497,27 @@ PrintTo(ImplicitCast_(s), os); } inline void PrintTo(unsigned char* s, ::std::ostream* os) { PrintTo(ImplicitCast_(s), os); } +#ifdef __cpp_char8_t +// Overloads for u8 strings. +GTEST_API_ void PrintTo(const char8_t* s, ::std::ostream* os); +inline void PrintTo(char8_t* s, ::std::ostream* os) { + PrintTo(ImplicitCast_(s), os); +} +#endif +// Overloads for u16 strings. +GTEST_API_ void PrintTo(const char16_t* s, ::std::ostream* os); +inline void PrintTo(char16_t* s, ::std::ostream* os) { + PrintTo(ImplicitCast_(s), os); +} +// Overloads for u32 strings. +GTEST_API_ void PrintTo(const char32_t* s, ::std::ostream* os); +inline void PrintTo(char32_t* s, ::std::ostream* os) { + PrintTo(ImplicitCast_(s), os); +} // MSVC can be configured to define wchar_t as a typedef of unsigned // short. It defines _NATIVE_WCHAR_T_DEFINED when wchar_t is a native // type. When wchar_t is a typedef, defining an overload for const // wchar_t* would cause unsigned short* be printed as a wide string, @@ -592,10 +548,30 @@ GTEST_API_ void PrintStringTo(const ::std::string&s, ::std::ostream* os); inline void PrintTo(const ::std::string& s, ::std::ostream* os) { PrintStringTo(s, os); } +// Overloads for ::std::u8string +#ifdef __cpp_char8_t +GTEST_API_ void PrintU8StringTo(const ::std::u8string& s, ::std::ostream* os); +inline void PrintTo(const ::std::u8string& s, ::std::ostream* os) { + PrintU8StringTo(s, os); +} +#endif + +// Overloads for ::std::u16string +GTEST_API_ void PrintU16StringTo(const ::std::u16string& s, ::std::ostream* os); +inline void PrintTo(const ::std::u16string& s, ::std::ostream* os) { + PrintU16StringTo(s, os); +} + +// Overloads for ::std::u32string +GTEST_API_ void PrintU32StringTo(const ::std::u32string& s, ::std::ostream* os); +inline void PrintTo(const ::std::u32string& s, ::std::ostream* os) { + PrintU32StringTo(s, os); +} + // Overloads for ::std::wstring. #if GTEST_HAS_STD_WSTRING GTEST_API_ void PrintWideStringTo(const ::std::wstring&s, ::std::ostream* os); inline void PrintTo(const ::std::wstring& s, ::std::ostream* os) { PrintWideStringTo(s, os); @@ -614,10 +590,47 @@ template void PrintTo(std::reference_wrapper ref, ::std::ostream* os) { UniversalPrinter::Print(ref.get(), os); } +inline const void* VoidifyPointer(const void* p) { return p; } +inline const void* VoidifyPointer(volatile const void* p) { + return const_cast(p); +} + +template +void PrintSmartPointer(const Ptr& ptr, std::ostream* os, char) { + if (ptr == nullptr) { + *os << "(nullptr)"; + } else { + // We can't print the value. Just print the pointer.. + *os << "(" << (VoidifyPointer)(ptr.get()) << ")"; + } +} +template ::value && + !std::is_array::value>::type> +void PrintSmartPointer(const Ptr& ptr, std::ostream* os, int) { + if (ptr == nullptr) { + *os << "(nullptr)"; + } else { + *os << "(ptr = " << (VoidifyPointer)(ptr.get()) << ", value = "; + UniversalPrinter::Print(*ptr, os); + *os << ")"; + } +} + +template +void PrintTo(const std::unique_ptr& ptr, std::ostream* os) { + (PrintSmartPointer)(ptr, os, 0); +} + +template +void PrintTo(const std::shared_ptr& ptr, std::ostream* os) { + (PrintSmartPointer)(ptr, os, 0); +} + // Helper function for printing a tuple. T must be instantiated with // a tuple type. template void PrintTupleTo(const T&, std::integral_constant, ::std::ostream*) {} @@ -679,51 +692,93 @@ } GTEST_DISABLE_MSC_WARNINGS_POP_() }; -#if GTEST_HAS_ABSL +// Remove any const-qualifiers before passing a type to UniversalPrinter. +template +class UniversalPrinter : public UniversalPrinter {}; + +#if GTEST_INTERNAL_HAS_ANY + +// Printer for std::any / absl::any -// Printer for absl::optional +template <> +class UniversalPrinter { + public: + static void Print(const Any& value, ::std::ostream* os) { + if (value.has_value()) { + *os << "value of type " << GetTypeName(value); + } else { + *os << "no value"; + } + } + + private: + static std::string GetTypeName(const Any& value) { +#if GTEST_HAS_RTTI + return internal::GetTypeName(value.type()); +#else + static_cast(value); // possibly unused + return ""; +#endif // GTEST_HAS_RTTI + } +}; + +#endif // GTEST_INTERNAL_HAS_ANY + +#if GTEST_INTERNAL_HAS_OPTIONAL + +// Printer for std::optional / absl::optional template -class UniversalPrinter<::absl::optional> { +class UniversalPrinter> { public: - static void Print(const ::absl::optional& value, ::std::ostream* os) { + static void Print(const Optional& value, ::std::ostream* os) { *os << '('; if (!value) { *os << "nullopt"; } else { UniversalPrint(*value, os); } *os << ')'; } }; -// Printer for absl::variant +#endif // GTEST_INTERNAL_HAS_OPTIONAL + +#if GTEST_INTERNAL_HAS_VARIANT + +// Printer for std::variant / absl::variant template -class UniversalPrinter<::absl::variant> { +class UniversalPrinter> { public: - static void Print(const ::absl::variant& value, ::std::ostream* os) { + static void Print(const Variant& value, ::std::ostream* os) { *os << '('; - absl::visit(Visitor{os}, value); +#if GTEST_HAS_ABSL + absl::visit(Visitor{os, value.index()}, value); +#else + std::visit(Visitor{os, value.index()}, value); +#endif // GTEST_HAS_ABSL *os << ')'; } private: struct Visitor { template void operator()(const U& u) const { - *os << "'" << GetTypeName() << "' with value "; + *os << "'" << GetTypeName() << "(index = " << index + << ")' with value "; UniversalPrint(u, os); } ::std::ostream* os; + std::size_t index; }; }; -#endif // GTEST_HAS_ABSL +#endif // GTEST_INTERNAL_HAS_VARIANT // UniversalPrintArray(begin, len, os) prints an array of 'len' // elements, starting at address 'begin'. template void UniversalPrintArray(const T* begin, size_t len, ::std::ostream* os) { @@ -748,10 +803,24 @@ } // This overload prints a (const) char array compactly. GTEST_API_ void UniversalPrintArray( const char* begin, size_t len, ::std::ostream* os); +#ifdef __cpp_char8_t +// This overload prints a (const) char8_t array compactly. +GTEST_API_ void UniversalPrintArray(const char8_t* begin, size_t len, + ::std::ostream* os); +#endif + +// This overload prints a (const) char16_t array compactly. +GTEST_API_ void UniversalPrintArray(const char16_t* begin, size_t len, + ::std::ostream* os); + +// This overload prints a (const) char32_t array compactly. +GTEST_API_ void UniversalPrintArray(const char32_t* begin, size_t len, + ::std::ostream* os); + // This overload prints a (const) wchar_t array compactly. GTEST_API_ void UniversalPrintArray( const wchar_t* begin, size_t len, ::std::ostream* os); // Implements printing an array type T[N]. @@ -820,16 +889,59 @@ UniversalPrint(std::string(str), os); } } }; template <> -class UniversalTersePrinter { +class UniversalTersePrinter : public UniversalTersePrinter { +}; + +#ifdef __cpp_char8_t +template <> +class UniversalTersePrinter { public: - static void Print(char* str, ::std::ostream* os) { - UniversalTersePrinter::Print(str, os); + static void Print(const char8_t* str, ::std::ostream* os) { + if (str == nullptr) { + *os << "NULL"; + } else { + UniversalPrint(::std::u8string(str), os); + } } }; +template <> +class UniversalTersePrinter + : public UniversalTersePrinter {}; +#endif + +template <> +class UniversalTersePrinter { + public: + static void Print(const char16_t* str, ::std::ostream* os) { + if (str == nullptr) { + *os << "NULL"; + } else { + UniversalPrint(::std::u16string(str), os); + } + } +}; +template <> +class UniversalTersePrinter + : public UniversalTersePrinter {}; + +template <> +class UniversalTersePrinter { + public: + static void Print(const char32_t* str, ::std::ostream* os) { + if (str == nullptr) { + *os << "NULL"; + } else { + UniversalPrint(::std::u32string(str), os); + } + } +}; +template <> +class UniversalTersePrinter + : public UniversalTersePrinter {}; #if GTEST_HAS_STD_WSTRING template <> class UniversalTersePrinter { public: @@ -898,20 +1010,10 @@ return result; } } // namespace internal -#if GTEST_INTERNAL_HAS_STRING_VIEW -namespace internal2 { -template -void TypeWithoutFormatter::PrintValue( - const T& value, ::std::ostream* os) { - internal::PrintTo(internal::StringView(value), os); -} -} // namespace internal2 -#endif - template ::std::string PrintToString(const T& value) { ::std::stringstream ss; internal::UniversalTersePrinter::Print(value, &ss); return ss.str(); @@ -922,6 +1024,6 @@ // Include any custom printer added by the local installation. // We must include this header at the end to make sure it can use the // declarations from this file. #include "gtest/internal/custom/gtest-printers.h" -#endif // GTEST_INCLUDE_GTEST_GTEST_PRINTERS_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_GTEST_PRINTERS_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-spi.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-spi.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-spi.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-spi.h @@ -31,12 +31,12 @@ // Utilities for testing Google Test itself and code that uses Google Test // (e.g. frameworks built on top of Google Test). // GOOGLETEST_CM0004 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_GTEST_SPI_H_ -#define GTEST_INCLUDE_GTEST_GTEST_SPI_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_GTEST_SPI_H_ +#define GOOGLETEST_INCLUDE_GTEST_GTEST_SPI_H_ #include "gtest/gtest.h" GTEST_DISABLE_MSC_WARNINGS_PUSH_(4251 \ /* class A needs to have dll-interface to be used by clients of class B */) @@ -233,6 +233,6 @@ >est_failures);\ if (::testing::internal::AlwaysTrue()) { statement; }\ }\ } while (::testing::internal::AlwaysFalse()) -#endif // GTEST_INCLUDE_GTEST_GTEST_SPI_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_GTEST_SPI_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-test-part.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-test-part.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-test-part.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-test-part.h @@ -27,12 +27,12 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // // GOOGLETEST_CM0001 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_GTEST_TEST_PART_H_ -#define GTEST_INCLUDE_GTEST_GTEST_TEST_PART_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_GTEST_TEST_PART_H_ +#define GOOGLETEST_INCLUDE_GTEST_GTEST_TEST_PART_H_ #include #include #include "gtest/internal/gtest-internal.h" #include "gtest/internal/gtest-string.h" @@ -179,6 +179,6 @@ } // namespace testing GTEST_DISABLE_MSC_WARNINGS_POP_() // 4251 -#endif // GTEST_INCLUDE_GTEST_GTEST_TEST_PART_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_GTEST_TEST_PART_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-typed-test.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-typed-test.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-typed-test.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest-typed-test.h @@ -27,12 +27,12 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // GOOGLETEST_CM0001 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_GTEST_TYPED_TEST_H_ -#define GTEST_INCLUDE_GTEST_GTEST_TYPED_TEST_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_GTEST_TYPED_TEST_H_ +#define GOOGLETEST_INCLUDE_GTEST_GTEST_TYPED_TEST_H_ // This header implements typed tests and type-parameterized tests. // Typed (aka type-driven) tests repeat the same test for types in a // list. You must know which types you want to test with when writing @@ -173,12 +173,10 @@ #include "gtest/internal/gtest-port.h" #include "gtest/internal/gtest-type-util.h" // Implements typed tests. -#if GTEST_HAS_TYPED_TEST - // INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. // // Expands to the name of the typedef for the type parameters of the // given test suite. #define GTEST_TYPE_PARAMS_(TestSuiteName) gtest_type_params_##TestSuiteName##_ @@ -228,16 +226,12 @@ #define TYPED_TEST_CASE \ static_assert(::testing::internal::TypedTestCaseIsDeprecated(), ""); \ TYPED_TEST_SUITE #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ -#endif // GTEST_HAS_TYPED_TEST - // Implements type-parameterized tests. -#if GTEST_HAS_TYPED_TEST_P - // INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. // // Expands to the namespace name that the type-parameterized tests for // the given type-parameterized test suite are defined in. The exact // name of the namespace is subject to change without notice. @@ -330,8 +324,6 @@ static_assert( \ ::testing::internal::InstantiateTypedTestCase_P_IsDeprecated(), ""); \ INSTANTIATE_TYPED_TEST_SUITE_P #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ -#endif // GTEST_HAS_TYPED_TEST_P - -#endif // GTEST_INCLUDE_GTEST_GTEST_TYPED_TEST_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_GTEST_TYPED_TEST_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest.h @@ -47,12 +47,12 @@ // registration from Barthelemy Dagenais' (barthelemy@prologique.com) // easyUnit framework. // GOOGLETEST_CM0001 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_GTEST_H_ -#define GTEST_INCLUDE_GTEST_GTEST_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_GTEST_H_ +#define GOOGLETEST_INCLUDE_GTEST_GTEST_H_ #include #include #include #include @@ -99,10 +99,14 @@ // This flag enables using colors in terminal output. Available values are // "yes" to enable colors, "no" (disable colors), or "auto" (the default) // to let Google Test decide. GTEST_DECLARE_string_(color); +// This flag controls whether the test runner should continue execution past +// first failure. +GTEST_DECLARE_bool_(fail_fast); + // This flag sets up the filter to select by name using a glob pattern // the tests to run. If the filter is not given all tests are executed. GTEST_DECLARE_string_(filter); // This flag controls whether Google Test installs a signal handler that dumps @@ -115,10 +119,13 @@ // This flag controls whether Google Test emits a detailed XML report to a file // in addition to its normal textual output. GTEST_DECLARE_string_(output); +// This flags control whether Google Test prints only test failures. +GTEST_DECLARE_bool_(brief); + // This flags control whether Google Test prints the elapsed time for each // test. GTEST_DECLARE_bool_(print_time); // This flags control whether Google Test prints UTF8 characters as text. @@ -409,27 +416,28 @@ friend class TestInfo; // The d'tor is virtual as we intend to inherit from Test. virtual ~Test(); - // Sets up the stuff shared by all tests in this test case. + // Sets up the stuff shared by all tests in this test suite. // // Google Test will call Foo::SetUpTestSuite() before running the first - // test in test case Foo. Hence a sub-class can define its own + // test in test suite Foo. Hence a sub-class can define its own // SetUpTestSuite() method to shadow the one defined in the super // class. static void SetUpTestSuite() {} // Tears down the stuff shared by all tests in this test suite. // // Google Test will call Foo::TearDownTestSuite() after running the last - // test in test case Foo. Hence a sub-class can define its own + // test in test suite Foo. Hence a sub-class can define its own // TearDownTestSuite() method to shadow the one defined in the super // class. static void TearDownTestSuite() {} - // Legacy API is deprecated but still available + // Legacy API is deprecated but still available. Use SetUpTestSuite and + // TearDownTestSuite instead. #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ static void TearDownTestCase() {} static void SetUpTestCase() {} #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ @@ -663,11 +671,11 @@ // Clears the object. void Clear(); // Protects mutable state of the property vector and of owned // properties, whose values may be updated. - internal::Mutex test_properites_mutex_; + internal::Mutex test_properties_mutex_; // The vector of TestPartResults std::vector test_part_results_; // The vector of TestProperties std::vector test_properties_; @@ -793,10 +801,13 @@ // Creates the test object, runs it, records its result, and then // deletes it. void Run(); + // Skip and records the test result for this object. + void Skip(); + static void ClearTestResult(TestInfo* test_info) { test_info->result_.Clear(); } // These fields are immutable properties of the test. @@ -941,10 +952,13 @@ } // Runs every test in this TestSuite. void Run(); + // Skips the execution of tests under this TestSuite + void Skip(); + // Runs SetUpTestSuite() for this TestSuite. This wrapper is needed // for catching exceptions thrown from SetUpTestSuite(). void RunSetUpTestSuite() { if (set_up_tc_ != nullptr) { (*set_up_tc_)(); @@ -1533,18 +1547,10 @@ } return CmpHelperEQFailure(lhs_expression, rhs_expression, lhs, rhs); } -// With this overloaded version, we allow anonymous enums to be used -// in {ASSERT|EXPECT}_EQ when compiled with gcc 4, as anonymous enums -// can be implicitly cast to BiggestInt. -GTEST_API_ AssertionResult CmpHelperEQ(const char* lhs_expression, - const char* rhs_expression, - BiggestInt lhs, - BiggestInt rhs); - class EqHelper { public: // This templatized version is for the general case. template < typename T1, typename T2, @@ -1597,15 +1603,10 @@ // A macro for implementing the helper functions needed to implement // ASSERT_?? and EXPECT_??. It is here just to avoid copy-and-paste // of similar code. // -// For each templatized helper function, we also define an overloaded -// version for BiggestInt in order to reduce code bloat and allow -// anonymous enums to be used with {ASSERT|EXPECT}_?? when compiled -// with gcc 4. -// // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. #define GTEST_IMPL_CMP_HELPER_(op_name, op)\ template \ AssertionResult CmpHelper##op_name(const char* expr1, const char* expr2, \ @@ -1613,26 +1614,24 @@ if (val1 op val2) {\ return AssertionSuccess();\ } else {\ return CmpHelperOpFailure(expr1, expr2, val1, val2, #op);\ }\ -}\ -GTEST_API_ AssertionResult CmpHelper##op_name(\ - const char* expr1, const char* expr2, BiggestInt val1, BiggestInt val2) +} // INTERNAL IMPLEMENTATION - DO NOT USE IN A USER PROGRAM. // Implements the helper function for {ASSERT|EXPECT}_NE -GTEST_IMPL_CMP_HELPER_(NE, !=); +GTEST_IMPL_CMP_HELPER_(NE, !=) // Implements the helper function for {ASSERT|EXPECT}_LE -GTEST_IMPL_CMP_HELPER_(LE, <=); +GTEST_IMPL_CMP_HELPER_(LE, <=) // Implements the helper function for {ASSERT|EXPECT}_LT -GTEST_IMPL_CMP_HELPER_(LT, <); +GTEST_IMPL_CMP_HELPER_(LT, <) // Implements the helper function for {ASSERT|EXPECT}_GE -GTEST_IMPL_CMP_HELPER_(GE, >=); +GTEST_IMPL_CMP_HELPER_(GE, >=) // Implements the helper function for {ASSERT|EXPECT}_GT -GTEST_IMPL_CMP_HELPER_(GT, >); +GTEST_IMPL_CMP_HELPER_(GT, >) #undef GTEST_IMPL_CMP_HELPER_ // The helper function for {ASSERT|EXPECT}_STREQ. // @@ -1805,16 +1804,10 @@ AssertHelperData* const data_; GTEST_DISALLOW_COPY_AND_ASSIGN_(AssertHelper); }; -enum GTestColor { COLOR_DEFAULT, COLOR_RED, COLOR_GREEN, COLOR_YELLOW }; - -GTEST_API_ GTEST_ATTRIBUTE_PRINTF_(2, 3) void ColoredPrintf(GTestColor color, - const char* fmt, - ...); - } // namespace internal // The pure interface class that all value-parameterized tests inherit from. // A value-parameterized class must inherit from both ::testing::Test and // ::testing::WithParamInterface. In most cases that just means inheriting @@ -1967,23 +1960,42 @@ GTEST_TEST_ANY_THROW_(statement, GTEST_FATAL_FAILURE_) // Boolean assertions. Condition can be either a Boolean expression or an // AssertionResult. For more information on how to use AssertionResult with // these macros see comments on that class. -#define EXPECT_TRUE(condition) \ +#define GTEST_EXPECT_TRUE(condition) \ GTEST_TEST_BOOLEAN_(condition, #condition, false, true, \ GTEST_NONFATAL_FAILURE_) -#define EXPECT_FALSE(condition) \ +#define GTEST_EXPECT_FALSE(condition) \ GTEST_TEST_BOOLEAN_(!(condition), #condition, true, false, \ GTEST_NONFATAL_FAILURE_) -#define ASSERT_TRUE(condition) \ +#define GTEST_ASSERT_TRUE(condition) \ GTEST_TEST_BOOLEAN_(condition, #condition, false, true, \ GTEST_FATAL_FAILURE_) -#define ASSERT_FALSE(condition) \ +#define GTEST_ASSERT_FALSE(condition) \ GTEST_TEST_BOOLEAN_(!(condition), #condition, true, false, \ GTEST_FATAL_FAILURE_) +// Define these macros to 1 to omit the definition of the corresponding +// EXPECT or ASSERT, which clashes with some users' own code. + +#if !GTEST_DONT_DEFINE_EXPECT_TRUE +#define EXPECT_TRUE(condition) GTEST_EXPECT_TRUE(condition) +#endif + +#if !GTEST_DONT_DEFINE_EXPECT_FALSE +#define EXPECT_FALSE(condition) GTEST_EXPECT_FALSE(condition) +#endif + +#if !GTEST_DONT_DEFINE_ASSERT_TRUE +#define ASSERT_TRUE(condition) GTEST_ASSERT_TRUE(condition) +#endif + +#if !GTEST_DONT_DEFINE_ASSERT_FALSE +#define ASSERT_FALSE(condition) GTEST_ASSERT_FALSE(condition) +#endif + // Macros for testing equalities and inequalities. // // * {ASSERT|EXPECT}_EQ(v1, v2): Tests that v1 == v2 // * {ASSERT|EXPECT}_NE(v1, v2): Tests that v1 != v2 // * {ASSERT|EXPECT}_LT(v1, v2): Tests that v1 < v2 @@ -2478,6 +2490,6 @@ return ::testing::UnitTest::GetInstance()->Run(); } GTEST_DISABLE_MSC_WARNINGS_POP_() // 4251 -#endif // GTEST_INCLUDE_GTEST_GTEST_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_GTEST_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest_pred_impl.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest_pred_impl.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest_pred_impl.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest_pred_impl.h @@ -31,12 +31,12 @@ // 'gen_gtest_pred_impl.py 5'. DO NOT EDIT BY HAND! // // Implements a family of generic predicate assertion macros. // GOOGLETEST_CM0001 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_GTEST_PRED_IMPL_H_ -#define GTEST_INCLUDE_GTEST_GTEST_PRED_IMPL_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_GTEST_PRED_IMPL_H_ +#define GOOGLETEST_INCLUDE_GTEST_GTEST_PRED_IMPL_H_ #include "gtest/gtest.h" namespace testing { @@ -354,6 +354,6 @@ } // namespace testing -#endif // GTEST_INCLUDE_GTEST_GTEST_PRED_IMPL_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_GTEST_PRED_IMPL_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest_prod.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest_prod.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest_prod.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/gtest_prod.h @@ -29,12 +29,12 @@ // // Google C++ Testing and Mocking Framework definitions useful in production code. // GOOGLETEST_CM0003 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_GTEST_PROD_H_ -#define GTEST_INCLUDE_GTEST_GTEST_PROD_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_GTEST_PROD_H_ +#define GOOGLETEST_INCLUDE_GTEST_GTEST_PROD_H_ // When you need to test the private or protected members of a class, // use the FRIEND_TEST macro to declare your tests as friends of the // class. For example: // @@ -56,6 +56,6 @@ // For example, putting MyClassTest in an anonymous namespace will not work. #define FRIEND_TEST(test_case_name, test_name)\ friend class test_case_name##_##test_name##_Test -#endif // GTEST_INCLUDE_GTEST_GTEST_PROD_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_GTEST_PROD_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/custom/gtest-port.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/custom/gtest-port.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/custom/gtest-port.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/custom/gtest-port.h @@ -29,9 +29,9 @@ // // Injection point for custom user configurations. See README for details // // ** Custom implementation starts here ** -#ifndef GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PORT_H_ -#define GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PORT_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PORT_H_ +#define GOOGLETEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PORT_H_ -#endif // GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PORT_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PORT_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/custom/gtest-printers.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/custom/gtest-printers.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/custom/gtest-printers.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/custom/gtest-printers.h @@ -34,9 +34,9 @@ // // Injection point for custom user configurations. See README for details // // ** Custom implementation starts here ** -#ifndef GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PRINTERS_H_ -#define GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PRINTERS_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PRINTERS_H_ +#define GOOGLETEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PRINTERS_H_ -#endif // GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PRINTERS_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_PRINTERS_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/custom/gtest.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/custom/gtest.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/custom/gtest.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/custom/gtest.h @@ -29,9 +29,9 @@ // // Injection point for custom user configurations. See README for details // // ** Custom implementation starts here ** -#ifndef GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_H_ -#define GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_H_ +#define GOOGLETEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_H_ -#endif // GTEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_INTERNAL_CUSTOM_GTEST_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-death-test-internal.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-death-test-internal.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-death-test-internal.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-death-test-internal.h @@ -31,12 +31,12 @@ // // This header file defines internal utilities needed for implementing // death tests. They are subject to change without notice. // GOOGLETEST_CM0001 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_DEATH_TEST_INTERNAL_H_ -#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_DEATH_TEST_INTERNAL_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_DEATH_TEST_INTERNAL_H_ +#define GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_DEATH_TEST_INTERNAL_H_ #include "gtest/gtest-matchers.h" #include "gtest/internal/gtest-internal.h" #include @@ -299,6 +299,6 @@ #endif // GTEST_HAS_DEATH_TEST } // namespace internal } // namespace testing -#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_DEATH_TEST_INTERNAL_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_DEATH_TEST_INTERNAL_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-filepath.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-filepath.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-filepath.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-filepath.h @@ -35,12 +35,12 @@ // This file is #included in gtest/internal/gtest-internal.h. // Do not include this header file separately! // GOOGLETEST_CM0001 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_FILEPATH_H_ -#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_FILEPATH_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_FILEPATH_H_ +#define GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_FILEPATH_H_ #include "gtest/internal/gtest-string.h" GTEST_DISABLE_MSC_WARNINGS_PUSH_(4251 \ /* class A needs to have dll-interface to be used by clients of class B */) @@ -193,11 +193,11 @@ // the primary path separator '\\', so that for example "bar\\/\\foo" becomes // "bar\\foo". void Normalize(); - // Returns a pointer to the last occurence of a valid path separator in + // Returns a pointer to the last occurrence of a valid path separator in // the FilePath. On Windows, for example, both '/' and '\' are valid path // separators. Returns NULL if no path separator was found. const char* FindLastPathSeparator() const; std::string pathname_; @@ -206,6 +206,6 @@ } // namespace internal } // namespace testing GTEST_DISABLE_MSC_WARNINGS_POP_() // 4251 -#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_FILEPATH_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_FILEPATH_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-internal.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-internal.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-internal.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-internal.h @@ -32,12 +32,12 @@ // This header file declares functions and macros used internally by // Google Test. They are subject to change without notice. // GOOGLETEST_CM0001 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_ -#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_ +#define GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_ #include "gtest/internal/gtest-port.h" #if GTEST_OS_LINUX # include @@ -88,11 +88,13 @@ // Complaining about the argument to GTEST_STRINGIFY_ being empty. // This is allowed by the spec. #define GTEST_STRINGIFY_HELPER_(name, ...) #name #define GTEST_STRINGIFY_(...) GTEST_STRINGIFY_HELPER_(__VA_ARGS__, ) -namespace proto2 { class Message; } +namespace proto2 { +class MessageLite; +} namespace testing { // Forward declarations. @@ -283,11 +285,11 @@ // calculations are done with 80-bit precision, while double has 64 // bits. Therefore, 4 should be enough for ordinary use. // // See the following article for more details on ULP: // http://randomascii.wordpress.com/2012/02/25/comparing-floating-point-numbers-2012-edition/ - static const size_t kMaxUlps = 4; + static const uint32_t kMaxUlps = 4; // Constructs a FloatingPoint from a raw floating-point number. // // On an Intel CPU, passing a non-normalized NAN (Not a Number) // around may change its bits, although the new value is guaranteed @@ -516,10 +518,11 @@ using Test = typename std::conditional::type; static SetUpTearDownSuiteFuncType GetSetUpCaseOrSuite(const char* filename, int line_num) { +#ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ SetUpTearDownSuiteFuncType test_case_fp = GetNotDefaultOrNull(&T::SetUpTestCase, &Test::SetUpTestCase); SetUpTearDownSuiteFuncType test_suite_fp = GetNotDefaultOrNull(&T::SetUpTestSuite, &Test::SetUpTestSuite); @@ -527,14 +530,20 @@ << "Test can not provide both SetUpTestSuite and SetUpTestCase, please " "make sure there is only one present at " << filename << ":" << line_num; return test_case_fp != nullptr ? test_case_fp : test_suite_fp; +#else + (void)(filename); + (void)(line_num); + return &T::SetUpTestSuite; +#endif } static SetUpTearDownSuiteFuncType GetTearDownCaseOrSuite(const char* filename, int line_num) { +#ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ SetUpTearDownSuiteFuncType test_case_fp = GetNotDefaultOrNull(&T::TearDownTestCase, &Test::TearDownTestCase); SetUpTearDownSuiteFuncType test_suite_fp = GetNotDefaultOrNull(&T::TearDownTestSuite, &Test::TearDownTestSuite); @@ -542,23 +551,28 @@ << "Test can not provide both TearDownTestSuite and TearDownTestCase," " please make sure there is only one present at" << filename << ":" << line_num; return test_case_fp != nullptr ? test_case_fp : test_suite_fp; +#else + (void)(filename); + (void)(line_num); + return &T::TearDownTestSuite; +#endif } }; // Creates a new TestInfo object and registers it with Google Test; // returns the created object. // // Arguments: // -// test_suite_name: name of the test suite +// test_suite_name: name of the test suite // name: name of the test -// type_param the name of the test's type parameter, or NULL if +// type_param: the name of the test's type parameter, or NULL if // this is not a typed or a type-parameterized test. -// value_param text representation of the test's value parameter, +// value_param: text representation of the test's value parameter, // or NULL if this is not a type-parameterized test. // code_location: code location where the test is defined // fixture_class_id: ID of the test fixture class // set_up_tc: pointer to the function that sets up the test suite // tear_down_tc: pointer to the function that tears down the test suite @@ -574,12 +588,10 @@ // If *pstr starts with the given prefix, modifies *pstr to be right // past the prefix and returns true; otherwise leaves *pstr unchanged // and returns false. None of pstr, *pstr, and prefix can be NULL. GTEST_API_ bool SkipPrefix(const char* prefix, const char** pstr); -#if GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P - GTEST_DISABLE_MSC_WARNINGS_PUSH_(4251 \ /* class A needs to have dll-interface to be used by clients of class B */) // State of the definition of a type-parameterized test suite. class GTEST_API_ TypedTestSuitePState { @@ -807,12 +819,10 @@ std::vector() /*type_names*/) { return true; } }; -#endif // GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P - // Returns the current OS stack trace as an std::string. // // The maximum number of stack frames to be included is specified by // the gtest_stack_trace_depth flag. The skip_count parameter // specifies the number of top frames to be skipped, which doesn't @@ -876,15 +886,38 @@ // Turns const U&, U&, const U, and U all into U. #define GTEST_REMOVE_REFERENCE_AND_CONST_(T) \ typename std::remove_const::type>::type -// IsAProtocolMessage::value is a compile-time bool constant that's -// true if and only if T is type proto2::Message or a subclass of it. +// HasDebugStringAndShortDebugString::value is a compile-time bool constant +// that's true if and only if T has methods DebugString() and ShortDebugString() +// that return std::string. template -struct IsAProtocolMessage - : public std::is_convertible {}; +class HasDebugStringAndShortDebugString { + private: + template + static auto CheckDebugString(C*) -> typename std::is_same< + std::string, decltype(std::declval().DebugString())>::type; + template + static std::false_type CheckDebugString(...); + + template + static auto CheckShortDebugString(C*) -> typename std::is_same< + std::string, decltype(std::declval().ShortDebugString())>::type; + template + static std::false_type CheckShortDebugString(...); + + using HasDebugStringType = decltype(CheckDebugString(nullptr)); + using HasShortDebugStringType = decltype(CheckShortDebugString(nullptr)); + + public: + static constexpr bool value = + HasDebugStringType::value && HasShortDebugStringType::value; +}; + +template +constexpr bool HasDebugStringAndShortDebugString::value; // When the compiler sees expression IsContainerTest(0), if C is an // STL-style container class, the first overload of IsContainerTest // will be viable (since both C::iterator* and C::const_iterator* are // valid types and NULL can be implicitly converted to them). It will @@ -1116,12 +1149,10 @@ } const Element* array_; size_t size_; void (NativeArray::*clone_)(const Element*, size_t); - - GTEST_DISALLOW_ASSIGN_(NativeArray); }; // Backport of std::index_sequence. template struct IndexSequence { @@ -1141,16 +1172,22 @@ }; // Backport of std::make_index_sequence. // It uses O(ln(N)) instantiation depth. template -struct MakeIndexSequence - : DoubleSequence::type, +struct MakeIndexSequenceImpl + : DoubleSequence::type, N / 2>::type {}; template <> -struct MakeIndexSequence<0> : IndexSequence<> {}; +struct MakeIndexSequenceImpl<0> : IndexSequence<> {}; + +template +using MakeIndexSequence = typename MakeIndexSequenceImpl::type; + +template +using IndexSequenceFor = typename MakeIndexSequence::type; template struct Ignore { Ignore(...); // NOLINT }; @@ -1172,21 +1209,25 @@ using type = decltype(ElemFromListImpl::type>::Apply( static_cast(nullptr)...)); }; +struct FlatTupleConstructTag {}; + template class FlatTuple; template struct FlatTupleElemBase; template struct FlatTupleElemBase, I> { using value_type = typename ElemFromList::type; FlatTupleElemBase() = default; - explicit FlatTupleElemBase(value_type t) : value(std::move(t)) {} + template + explicit FlatTupleElemBase(FlatTupleConstructTag, Arg&& t) + : value(std::forward(t)) {} value_type value; }; template struct FlatTupleBase; @@ -1194,12 +1235,34 @@ template struct FlatTupleBase, IndexSequence> : FlatTupleElemBase, Idx>... { using Indices = IndexSequence; FlatTupleBase() = default; - explicit FlatTupleBase(T... t) - : FlatTupleElemBase, Idx>(std::move(t))... {} + template + explicit FlatTupleBase(FlatTupleConstructTag, Args&&... args) + : FlatTupleElemBase, Idx>(FlatTupleConstructTag{}, + std::forward(args))... {} + + template + const typename ElemFromList::type& Get() const { + return FlatTupleElemBase, I>::value; + } + + template + typename ElemFromList::type& Get() { + return FlatTupleElemBase, I>::value; + } + + template + auto Apply(F&& f) -> decltype(std::forward(f)(this->Get()...)) { + return std::forward(f)(Get()...); + } + + template + auto Apply(F&& f) const -> decltype(std::forward(f)(this->Get()...)) { + return std::forward(f)(Get()...); + } }; // Analog to std::tuple but with different tradeoffs. // This class minimizes the template instantiation depth, thus allowing more // elements than std::tuple would. std::tuple has been seen to require an @@ -1216,21 +1279,16 @@ using Indices = typename FlatTupleBase< FlatTuple, typename MakeIndexSequence::type>::Indices; public: FlatTuple() = default; - explicit FlatTuple(T... t) : FlatTuple::FlatTupleBase(std::move(t)...) {} + template + explicit FlatTuple(FlatTupleConstructTag tag, Args&&... args) + : FlatTuple::FlatTupleBase(tag, std::forward(args)...) {} - template - const typename ElemFromList::type& Get() const { - return static_cast*>(this)->value; - } - - template - typename ElemFromList::type& Get() { - return static_cast*>(this)->value; - } + using FlatTuple::FlatTupleBase::Apply; + using FlatTuple::FlatTupleBase::Get; }; // Utility functions to be called with static_assert to induce deprecation // warnings. GTEST_INTERNAL_DEPRECATED( @@ -1259,10 +1317,26 @@ constexpr bool InstantiateTypedTestCase_P_IsDeprecated() { return true; } } // namespace internal } // namespace testing +namespace std { +// Some standard library implementations use `struct tuple_size` and some use +// `class tuple_size`. Clang warns about the mismatch. +// https://reviews.llvm.org/D55466 +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wmismatched-tags" +#endif +template +struct tuple_size> + : std::integral_constant {}; +#ifdef __clang__ +#pragma clang diagnostic pop +#endif +} // namespace std + #define GTEST_MESSAGE_AT_(file, line, message, result_type) \ ::testing::internal::AssertHelper(result_type, file, line, message) \ = ::testing::Message() #define GTEST_MESSAGE_(message, result_type) \ @@ -1281,48 +1355,102 @@ return GTEST_MESSAGE_(message, ::testing::TestPartResult::kSkip) // Suppress MSVC warning 4072 (unreachable code) for the code following // statement if it returns or throws (or doesn't return or throw in some // situations). +// NOTE: The "else" is important to keep this expansion to prevent a top-level +// "else" from attaching to our "if". #define GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement) \ - if (::testing::internal::AlwaysTrue()) { statement; } - -#define GTEST_TEST_THROW_(statement, expected_exception, fail) \ - GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ - if (::testing::internal::ConstCharPtr gtest_msg = "") { \ - bool gtest_caught_expected = false; \ - try { \ - GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ - } \ - catch (expected_exception const&) { \ - gtest_caught_expected = true; \ - } \ - catch (...) { \ - gtest_msg.value = \ - "Expected: " #statement " throws an exception of type " \ - #expected_exception ".\n Actual: it throws a different type."; \ - goto GTEST_CONCAT_TOKEN_(gtest_label_testthrow_, __LINE__); \ - } \ - if (!gtest_caught_expected) { \ - gtest_msg.value = \ - "Expected: " #statement " throws an exception of type " \ - #expected_exception ".\n Actual: it throws nothing."; \ - goto GTEST_CONCAT_TOKEN_(gtest_label_testthrow_, __LINE__); \ - } \ - } else \ - GTEST_CONCAT_TOKEN_(gtest_label_testthrow_, __LINE__): \ - fail(gtest_msg.value) + if (::testing::internal::AlwaysTrue()) { \ + statement; \ + } else /* NOLINT */ \ + static_assert(true, "") // User must have a semicolon after expansion. #if GTEST_HAS_EXCEPTIONS -#define GTEST_TEST_NO_THROW_CATCH_STD_EXCEPTION_() \ - catch (std::exception const& e) { \ - gtest_msg.value = ( \ - "it throws std::exception-derived exception with description: \"" \ - ); \ - gtest_msg.value += e.what(); \ - gtest_msg.value += "\"."; \ +namespace testing { +namespace internal { + +class NeverThrown { + public: + const char* what() const noexcept { + return "this exception should never be thrown"; + } +}; + +} // namespace internal +} // namespace testing + +#if GTEST_HAS_RTTI + +#define GTEST_EXCEPTION_TYPE_(e) ::testing::internal::GetTypeName(typeid(e)) + +#else // GTEST_HAS_RTTI + +#define GTEST_EXCEPTION_TYPE_(e) \ + std::string { "an std::exception-derived error" } + +#endif // GTEST_HAS_RTTI + +#define GTEST_TEST_THROW_CATCH_STD_EXCEPTION_(statement, expected_exception) \ + catch (typename std::conditional< \ + std::is_same::type>::type, \ + std::exception>::value, \ + const ::testing::internal::NeverThrown&, const std::exception&>::type \ + e) { \ + gtest_msg.value = "Expected: " #statement \ + " throws an exception of type " #expected_exception \ + ".\n Actual: it throws "; \ + gtest_msg.value += GTEST_EXCEPTION_TYPE_(e); \ + gtest_msg.value += " with description \""; \ + gtest_msg.value += e.what(); \ + gtest_msg.value += "\"."; \ + goto GTEST_CONCAT_TOKEN_(gtest_label_testthrow_, __LINE__); \ + } + +#else // GTEST_HAS_EXCEPTIONS + +#define GTEST_TEST_THROW_CATCH_STD_EXCEPTION_(statement, expected_exception) + +#endif // GTEST_HAS_EXCEPTIONS + +#define GTEST_TEST_THROW_(statement, expected_exception, fail) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if (::testing::internal::TrueWithString gtest_msg{}) { \ + bool gtest_caught_expected = false; \ + try { \ + GTEST_SUPPRESS_UNREACHABLE_CODE_WARNING_BELOW_(statement); \ + } catch (expected_exception const&) { \ + gtest_caught_expected = true; \ + } \ + GTEST_TEST_THROW_CATCH_STD_EXCEPTION_(statement, expected_exception) \ + catch (...) { \ + gtest_msg.value = "Expected: " #statement \ + " throws an exception of type " #expected_exception \ + ".\n Actual: it throws a different type."; \ + goto GTEST_CONCAT_TOKEN_(gtest_label_testthrow_, __LINE__); \ + } \ + if (!gtest_caught_expected) { \ + gtest_msg.value = "Expected: " #statement \ + " throws an exception of type " #expected_exception \ + ".\n Actual: it throws nothing."; \ + goto GTEST_CONCAT_TOKEN_(gtest_label_testthrow_, __LINE__); \ + } \ + } else /*NOLINT*/ \ + GTEST_CONCAT_TOKEN_(gtest_label_testthrow_, __LINE__) \ + : fail(gtest_msg.value.c_str()) + +#if GTEST_HAS_EXCEPTIONS + +#define GTEST_TEST_NO_THROW_CATCH_STD_EXCEPTION_() \ + catch (std::exception const& e) { \ + gtest_msg.value = "it throws "; \ + gtest_msg.value += GTEST_EXCEPTION_TYPE_(e); \ + gtest_msg.value += " with description \""; \ + gtest_msg.value += e.what(); \ + gtest_msg.value += "\"."; \ goto GTEST_CONCAT_TOKEN_(gtest_label_testnothrow_, __LINE__); \ } #else // GTEST_HAS_EXCEPTIONS @@ -1365,11 +1493,11 @@ " Actual: it doesn't.") // Implements Boolean test assertions such as EXPECT_TRUE. expression can be // either a boolean expression or an AssertionResult. text is a textual -// represenation of expression as it was passed into the EXPECT_TRUE. +// representation of expression as it was passed into the EXPECT_TRUE. #define GTEST_TEST_BOOLEAN_(expression, text, actual, expected, fail) \ GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ if (const ::testing::AssertionResult gtest_ar_ = \ ::testing::AssertionResult(expression)) \ ; \ @@ -1402,11 +1530,11 @@ static_assert(sizeof(GTEST_STRINGIFY_(test_name)) > 1, \ "test_name must not be empty"); \ class GTEST_TEST_CLASS_NAME_(test_suite_name, test_name) \ : public parent_class { \ public: \ - GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)() {} \ + GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)() = default; \ ~GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)() override = default; \ GTEST_DISALLOW_COPY_AND_ASSIGN_(GTEST_TEST_CLASS_NAME_(test_suite_name, \ test_name)); \ GTEST_DISALLOW_MOVE_AND_ASSIGN_(GTEST_TEST_CLASS_NAME_(test_suite_name, \ test_name)); \ @@ -1427,6 +1555,6 @@ parent_class>::GetTearDownCaseOrSuite(__FILE__, __LINE__), \ new ::testing::internal::TestFactoryImpl); \ void GTEST_TEST_CLASS_NAME_(test_suite_name, test_name)::TestBody() -#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_INTERNAL_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-param-util.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-param-util.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-param-util.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-param-util.h @@ -30,12 +30,12 @@ // Type and function utilities for implementing parameterized tests. // GOOGLETEST_CM0001 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_H_ -#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_H_ +#define GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_H_ #include #include #include @@ -457,11 +457,11 @@ public: virtual ~ParameterizedTestSuiteInfoBase() {} // Base part of test suite name for display purposes. virtual const std::string& GetTestSuiteName() const = 0; - // Test case id to verify identity. + // Test suite id to verify identity. virtual TypeId GetTestSuiteTypeId() const = 0; // UnitTest class invokes this method to register tests in this // test suite right before running them in RUN_ALL_TESTS macro. // This method should not be called more than once on any single // instance of a ParameterizedTestSuiteInfoBase derived class. @@ -476,11 +476,11 @@ // INTERNAL IMPLEMENTATION - DO NOT USE IN USER CODE. // // Report a the name of a test_suit as safe to ignore // as the side effect of construction of this type. -struct MarkAsIgnored { +struct GTEST_API_ MarkAsIgnored { explicit MarkAsIgnored(const char* test_suite); }; GTEST_API_ void InsertSyntheticTestCase(const std::string& name, CodeLocation location, bool has_test_p); @@ -505,26 +505,27 @@ explicit ParameterizedTestSuiteInfo(const char* name, CodeLocation code_location) : test_suite_name_(name), code_location_(code_location) {} - // Test case base name for display purposes. + // Test suite base name for display purposes. const std::string& GetTestSuiteName() const override { return test_suite_name_; } - // Test case id to verify identity. + // Test suite id to verify identity. TypeId GetTestSuiteTypeId() const override { return GetTypeId(); } // TEST_P macro uses AddTestPattern() to record information // about a single test in a LocalTestInfo structure. // test_suite_name is the base name of the test suite (without invocation // prefix). test_base_name is the name of an individual test without // parameter index. For the test SequenceA/FooTest.DoBar/1 FooTest is // test suite base name and DoBar is test base name. void AddTestPattern(const char* test_suite_name, const char* test_base_name, - TestMetaFactoryBase* meta_factory) { - tests_.push_back(std::shared_ptr( - new TestInfo(test_suite_name, test_base_name, meta_factory))); + TestMetaFactoryBase* meta_factory, + CodeLocation code_location) { + tests_.push_back(std::shared_ptr(new TestInfo( + test_suite_name, test_base_name, meta_factory, code_location))); } // INSTANTIATE_TEST_SUITE_P macro uses AddGenerator() to record information // about a generator. int AddTestSuiteInstantiation(const std::string& instantiation_name, GeneratorCreationFunc* func, @@ -587,11 +588,11 @@ } test_name_stream << param_name; MakeAndRegisterTestInfo( test_suite_name.c_str(), test_name_stream.GetString().c_str(), nullptr, // No type parameter. - PrintToString(*param_it).c_str(), code_location_, + PrintToString(*param_it).c_str(), test_info->code_location, GetTestSuiteTypeId(), SuiteApiResolver::GetSetUpCaseOrSuite(file, line), SuiteApiResolver::GetTearDownCaseOrSuite(file, line), test_info->test_meta_factory->CreateTestFactory(*param_it)); } // for param_it @@ -608,18 +609,21 @@ private: // LocalTestInfo structure keeps information about a single test registered // with TEST_P macro. struct TestInfo { TestInfo(const char* a_test_suite_base_name, const char* a_test_base_name, - TestMetaFactoryBase* a_test_meta_factory) + TestMetaFactoryBase* a_test_meta_factory, + CodeLocation a_code_location) : test_suite_base_name(a_test_suite_base_name), test_base_name(a_test_base_name), - test_meta_factory(a_test_meta_factory) {} + test_meta_factory(a_test_meta_factory), + code_location(a_code_location) {} const std::string test_suite_base_name; const std::string test_base_name; const std::unique_ptr > test_meta_factory; + const CodeLocation code_location; }; using TestInfoContainer = ::std::vector >; // Records data received from INSTANTIATE_TEST_SUITE_P macros: // @@ -648,11 +652,11 @@ if (name.empty()) return false; // Check for invalid characters for (std::string::size_type index = 0; index < name.size(); ++index) { - if (!isalnum(name[index]) && name[index] != '_') + if (!IsAlNum(name[index]) && name[index] != '_') return false; } return true; } @@ -777,14 +781,19 @@ const Container& container); namespace internal { // Used in the Values() function to provide polymorphic capabilities. +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4100) +#endif + template class ValueArray { public: - ValueArray(Ts... v) : v_{std::move(v)...} {} + explicit ValueArray(Ts... v) : v_(FlatTupleConstructTag{}, std::move(v)...) {} template operator ParamGenerator() const { // NOLINT return ValuesIn(MakeVector(MakeIndexSequence())); } @@ -796,10 +805,14 @@ } FlatTuple v_; }; +#ifdef _MSC_VER +#pragma warning(pop) +#endif + template class CartesianProductGenerator : public ParamGeneratorInterface<::std::tuple> { public: typedef ::std::tuple ParamType; @@ -929,6 +942,6 @@ }; } // namespace internal } // namespace testing -#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_PARAM_UTIL_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-port-arch.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-port-arch.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-port-arch.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-port-arch.h @@ -30,12 +30,12 @@ // The Google C++ Testing and Mocking Framework (Google Test) // // This header file defines the GTEST_OS_* macro. // It is separate from gtest-port.h so that custom/gtest-port.h can include it. -#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_ARCH_H_ -#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_ARCH_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_ARCH_H_ +#define GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_ARCH_H_ // Determines the platform on which Google Test is compiled. #ifdef __CYGWIN__ # define GTEST_OS_CYGWIN 1 # elif defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__) @@ -66,10 +66,11 @@ # endif // _WIN32_WCE #elif defined __OS2__ # define GTEST_OS_OS2 1 #elif defined __APPLE__ # define GTEST_OS_MAC 1 +# include # if TARGET_OS_IPHONE # define GTEST_OS_IOS 1 # endif #elif defined __DragonFly__ # define GTEST_OS_DRAGONFLY 1 @@ -104,8 +105,10 @@ #define GTEST_OS_HAIKU 1 #elif defined ESP8266 #define GTEST_OS_ESP8266 1 #elif defined ESP32 #define GTEST_OS_ESP32 1 +#elif defined(__XTENSA__) +#define GTEST_OS_XTENSA 1 #endif // __CYGWIN__ -#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_ARCH_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_ARCH_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-port.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-port.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-port.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-port.h @@ -38,12 +38,12 @@ // files are expected to #include this. Therefore, it cannot #include // any other Google Test header. // GOOGLETEST_CM0001 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_ -#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_ +#define GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_ // Environment-describing macros // ----------------------------- // // Google Test can be used in many different environments. Macros in @@ -197,13 +197,22 @@ // GTEST_MUST_USE_RESULT_ - declares that a function's result must be used. // GTEST_INTENTIONAL_CONST_COND_PUSH_ - start code section where MSVC C4127 is // suppressed (constant conditional). // GTEST_INTENTIONAL_CONST_COND_POP_ - finish code section where MSVC C4127 // is suppressed. +// GTEST_INTERNAL_HAS_ANY - for enabling UniversalPrinter or +// UniversalPrinter specializations. +// GTEST_INTERNAL_HAS_OPTIONAL - for enabling UniversalPrinter +// or +// UniversalPrinter +// specializations. // GTEST_INTERNAL_HAS_STRING_VIEW - for enabling Matcher or // Matcher // specializations. +// GTEST_INTERNAL_HAS_VARIANT - for enabling UniversalPrinter or +// UniversalPrinter +// specializations. // // Synchronization: // Mutex, MutexLock, ThreadLocal, GetThreadCount() // - synchronization primitives. // @@ -250,10 +259,12 @@ #include // for isspace, etc #include // for ptrdiff_t #include #include #include + +#include #include #include #include #ifndef _WIN32_WCE @@ -265,10 +276,11 @@ # include # include #endif #include // NOLINT +#include #include #include // NOLINT #include #include // NOLINT @@ -345,10 +357,14 @@ // Assume CRITICAL_SECTION is a typedef of _RTL_CRITICAL_SECTION. // This assumption is verified by // WindowsTypesTest.CRITICAL_SECTIONIs_RTL_CRITICAL_SECTION. typedef struct _RTL_CRITICAL_SECTION GTEST_CRITICAL_SECTION; #endif +#elif GTEST_OS_XTENSA +#include +// Xtensa toolchains define strcasecmp in the string.h header instead of +// strings.h. string.h is already included. #else // This assumes that non-Windows OSes provide unistd.h. For OSes where this // is not the case, we need to include headers that provide the functions // mentioned above. # include @@ -365,11 +381,11 @@ #ifndef GTEST_HAS_POSIX_RE # if GTEST_OS_LINUX_ANDROID // On Android, is only available starting with Gingerbread. # define GTEST_HAS_POSIX_RE (__ANDROID_API__ >= 9) # else -# define GTEST_HAS_POSIX_RE (!GTEST_OS_WINDOWS) +#define GTEST_HAS_POSIX_RE (!GTEST_OS_WINDOWS && !GTEST_OS_XTENSA) # endif #endif #if GTEST_USES_PCRE // The appropriate headers have already been included. @@ -450,11 +466,11 @@ // Cygwin 1.7 and below doesn't support ::std::wstring. // Solaris' libc++ doesn't support it either. Android has // no support for it at least as recent as Froyo (2.2). #define GTEST_HAS_STD_WSTRING \ (!(GTEST_OS_LINUX_ANDROID || GTEST_OS_CYGWIN || GTEST_OS_SOLARIS || \ - GTEST_OS_HAIKU || GTEST_OS_ESP32 || GTEST_OS_ESP8266)) + GTEST_OS_HAIKU || GTEST_OS_ESP32 || GTEST_OS_ESP8266 || GTEST_OS_XTENSA)) #endif // GTEST_HAS_STD_WSTRING // Determines whether RTTI is available. #ifndef GTEST_HAS_RTTI @@ -575,11 +591,11 @@ // output correctness and to implement death tests. #ifndef GTEST_HAS_STREAM_REDIRECTION // By default, we assume that stream redirection is supported on all // platforms except known mobile ones. #if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_WINDOWS_PHONE || \ - GTEST_OS_WINDOWS_RT || GTEST_OS_ESP8266 + GTEST_OS_WINDOWS_RT || GTEST_OS_ESP8266 || GTEST_OS_XTENSA # define GTEST_HAS_STREAM_REDIRECTION 0 # else # define GTEST_HAS_STREAM_REDIRECTION 1 # endif // !GTEST_OS_WINDOWS_MOBILE #endif // GTEST_HAS_STREAM_REDIRECTION @@ -677,23 +693,23 @@ type& operator=(type const &) = delete // A macro to disallow copy constructor and operator= // This should be used in the private: declarations for a class. #define GTEST_DISALLOW_COPY_AND_ASSIGN_(type) \ - type(type const &) = delete; \ - GTEST_DISALLOW_ASSIGN_(type) + type(type const&) = delete; \ + type& operator=(type const&) = delete // A macro to disallow move operator= // This should be used in the private: declarations for a class. #define GTEST_DISALLOW_MOVE_ASSIGN_(type) \ type& operator=(type &&) noexcept = delete // A macro to disallow move constructor and operator= // This should be used in the private: declarations for a class. #define GTEST_DISALLOW_MOVE_AND_ASSIGN_(type) \ - type(type &&) noexcept = delete; \ - GTEST_DISALLOW_MOVE_ASSIGN_(type) + type(type&&) noexcept = delete; \ + type& operator=(type&&) noexcept = delete // Tell the compiler to warn about unused return values for functions declared // with this macro. The macro should be used on function declarations // following the argument list: // @@ -916,12 +932,10 @@ # else // GTEST_USES_SIMPLE_RE const char* full_pattern_; // For FullMatch(); # endif - - GTEST_DISALLOW_ASSIGN_(RE); }; #endif // GTEST_USES_PCRE // Formats a source file path and a line number as they would appear @@ -1924,10 +1938,23 @@ return isupper(static_cast(ch)) != 0; } inline bool IsXDigit(char ch) { return isxdigit(static_cast(ch)) != 0; } +#ifdef __cpp_char8_t +inline bool IsXDigit(char8_t ch) { + return isxdigit(static_cast(ch)) != 0; +} +#endif +inline bool IsXDigit(char16_t ch) { + const unsigned char low_byte = static_cast(ch); + return ch == low_byte && isxdigit(low_byte) != 0; +} +inline bool IsXDigit(char32_t ch) { + const unsigned char low_byte = static_cast(ch); + return ch == low_byte && isxdigit(low_byte) != 0; +} inline bool IsXDigit(wchar_t ch) { const unsigned char low_byte = static_cast(ch); return ch == low_byte && isxdigit(low_byte) != 0; } @@ -1958,20 +1985,20 @@ #if GTEST_OS_WINDOWS typedef struct _stat StatStruct; # ifdef __BORLANDC__ -inline int IsATTY(int fd) { return isatty(fd); } +inline int DoIsATTY(int fd) { return isatty(fd); } inline int StrCaseCmp(const char* s1, const char* s2) { return stricmp(s1, s2); } inline char* StrDup(const char* src) { return strdup(src); } # else // !__BORLANDC__ # if GTEST_OS_WINDOWS_MOBILE -inline int IsATTY(int /* fd */) { return 0; } +inline int DoIsATTY(int /* fd */) { return 0; } # else -inline int IsATTY(int fd) { return _isatty(fd); } +inline int DoIsATTY(int fd) { return _isatty(fd); } # endif // GTEST_OS_WINDOWS_MOBILE inline int StrCaseCmp(const char* s1, const char* s2) { return _stricmp(s1, s2); } inline char* StrDup(const char* src) { return _strdup(src); } @@ -1992,11 +2019,11 @@ #elif GTEST_OS_ESP8266 typedef struct stat StatStruct; inline int FileNo(FILE* file) { return fileno(file); } -inline int IsATTY(int fd) { return isatty(fd); } +inline int DoIsATTY(int fd) { return isatty(fd); } inline int Stat(const char* path, StatStruct* buf) { // stat function not implemented on ESP8266 return 0; } inline int StrCaseCmp(const char* s1, const char* s2) { @@ -2009,34 +2036,54 @@ #else typedef struct stat StatStruct; inline int FileNo(FILE* file) { return fileno(file); } -inline int IsATTY(int fd) { return isatty(fd); } +inline int DoIsATTY(int fd) { return isatty(fd); } inline int Stat(const char* path, StatStruct* buf) { return stat(path, buf); } inline int StrCaseCmp(const char* s1, const char* s2) { return strcasecmp(s1, s2); } inline char* StrDup(const char* src) { return strdup(src); } inline int RmDir(const char* dir) { return rmdir(dir); } inline bool IsDir(const StatStruct& st) { return S_ISDIR(st.st_mode); } #endif // GTEST_OS_WINDOWS +inline int IsATTY(int fd) { + // DoIsATTY might change errno (for example ENOTTY in case you redirect stdout + // to a file on Linux), which is unexpected, so save the previous value, and + // restore it after the call. + int savedErrno = errno; + int isAttyValue = DoIsATTY(fd); + errno = savedErrno; + + return isAttyValue; +} + // Functions deprecated by MSVC 8.0. GTEST_DISABLE_MSC_DEPRECATED_PUSH_() // ChDir(), FReopen(), FDOpen(), Read(), Write(), Close(), and // StrError() aren't needed on Windows CE at this time and thus not // defined there. -#if !GTEST_OS_WINDOWS_MOBILE && !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT +#if !GTEST_OS_WINDOWS_MOBILE && !GTEST_OS_WINDOWS_PHONE && \ + !GTEST_OS_WINDOWS_RT && !GTEST_OS_ESP8266 && !GTEST_OS_XTENSA inline int ChDir(const char* dir) { return chdir(dir); } #endif inline FILE* FOpen(const char* path, const char* mode) { +#if GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MINGW + struct wchar_codecvt : public std::codecvt {}; + std::wstring_convert converter; + std::wstring wide_path = converter.from_bytes(path); + std::wstring wide_mode = converter.from_bytes(mode); + return _wfopen(wide_path.c_str(), wide_mode.c_str()); +#else // GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MINGW return fopen(path, mode); +#endif // GTEST_OS_WINDOWS && !GTEST_OS_WINDOWS_MINGW } #if !GTEST_OS_WINDOWS_MOBILE inline FILE *FReopen(const char* path, const char* mode, FILE* stream) { return freopen(path, mode, stream); } @@ -2053,11 +2100,11 @@ inline int Close(int fd) { return close(fd); } inline const char* StrError(int errnum) { return strerror(errnum); } #endif inline const char* GetEnv(const char* name) { #if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_WINDOWS_PHONE || \ - GTEST_OS_WINDOWS_RT || GTEST_OS_ESP8266 + GTEST_OS_WINDOWS_RT || GTEST_OS_ESP8266 || GTEST_OS_XTENSA // We are on an embedded platform, which has no environment variables. static_cast(name); // To prevent 'unused argument' warning. return nullptr; #elif defined(__BORLANDC__) || defined(__SunOS_5_8) || defined(__SunOS_5_9) // Environment variables which we programmatically clear will be set to the @@ -2189,11 +2236,12 @@ #endif // !defined(GTEST_EXCLUSIVE_LOCK_REQUIRED_) // Parses 'str' for a 32-bit signed integer. If successful, writes the result // to *value and returns true; otherwise leaves *value unchanged and returns // false. -bool ParseInt32(const Message& src_text, const char* str, int32_t* value); +GTEST_API_ bool ParseInt32(const Message& src_text, const char* str, + int32_t* value); // Parses a bool/int32_t/string from the environment variable // corresponding to the given Google Test flag. bool BoolFromGTestEnv(const char* flag, bool default_val); GTEST_API_ int32_t Int32FromGTestEnv(const char* flag, int32_t default_val); @@ -2222,10 +2270,68 @@ #endif #endif // !defined(GTEST_INTERNAL_DEPRECATED) #if GTEST_HAS_ABSL +// Always use absl::any for UniversalPrinter<> specializations if googletest +// is built with absl support. +#define GTEST_INTERNAL_HAS_ANY 1 +#include "absl/types/any.h" +namespace testing { +namespace internal { +using Any = ::absl::any; +} // namespace internal +} // namespace testing +#else +#ifdef __has_include +#if __has_include() && __cplusplus >= 201703L +// Otherwise for C++17 and higher use std::any for UniversalPrinter<> +// specializations. +#define GTEST_INTERNAL_HAS_ANY 1 +#include +namespace testing { +namespace internal { +using Any = ::std::any; +} // namespace internal +} // namespace testing +// The case where absl is configured NOT to alias std::any is not +// supported. +#endif // __has_include() && __cplusplus >= 201703L +#endif // __has_include +#endif // GTEST_HAS_ABSL + +#if GTEST_HAS_ABSL +// Always use absl::optional for UniversalPrinter<> specializations if +// googletest is built with absl support. +#define GTEST_INTERNAL_HAS_OPTIONAL 1 +#include "absl/types/optional.h" +namespace testing { +namespace internal { +template +using Optional = ::absl::optional; +} // namespace internal +} // namespace testing +#else +#ifdef __has_include +#if __has_include() && __cplusplus >= 201703L +// Otherwise for C++17 and higher use std::optional for UniversalPrinter<> +// specializations. +#define GTEST_INTERNAL_HAS_OPTIONAL 1 +#include +namespace testing { +namespace internal { +template +using Optional = ::std::optional; +} // namespace internal +} // namespace testing +// The case where absl is configured NOT to alias std::optional is not +// supported. +#endif // __has_include() && __cplusplus >= 201703L +#endif // __has_include +#endif // GTEST_HAS_ABSL + +#if GTEST_HAS_ABSL // Always use absl::string_view for Matcher<> specializations if googletest // is built with absl support. # define GTEST_INTERNAL_HAS_STRING_VIEW 1 #include "absl/strings/string_view.h" namespace testing { @@ -2249,6 +2355,35 @@ // supported. # endif // __has_include() && __cplusplus >= 201703L # endif // __has_include #endif // GTEST_HAS_ABSL -#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_ +#if GTEST_HAS_ABSL +// Always use absl::variant for UniversalPrinter<> specializations if googletest +// is built with absl support. +#define GTEST_INTERNAL_HAS_VARIANT 1 +#include "absl/types/variant.h" +namespace testing { +namespace internal { +template +using Variant = ::absl::variant; +} // namespace internal +} // namespace testing +#else +#ifdef __has_include +#if __has_include() && __cplusplus >= 201703L +// Otherwise for C++17 and higher use std::variant for UniversalPrinter<> +// specializations. +#define GTEST_INTERNAL_HAS_VARIANT 1 +#include +namespace testing { +namespace internal { +template +using Variant = ::std::variant; +} // namespace internal +} // namespace testing +// The case where absl is configured NOT to alias std::variant is not supported. +#endif // __has_include() && __cplusplus >= 201703L +#endif // __has_include +#endif // GTEST_HAS_ABSL + +#endif // GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_PORT_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-string.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-string.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-string.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-string.h @@ -36,12 +36,12 @@ // This header file is #included by gtest-internal.h. // It should not be #included by other files. // GOOGLETEST_CM0001 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_STRING_H_ -#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_STRING_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_STRING_H_ +#define GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_STRING_H_ #ifdef __BORLANDC__ // string.h is not guaranteed to provide strcpy on C++ Builder. # include #endif @@ -147,10 +147,13 @@ const std::string& str, const std::string& suffix); // Formats an int value as "%02d". static std::string FormatIntWidth2(int value); // "%02d" for width == 2 + // Formats an int value to given width with leading zeros. + static std::string FormatIntWidthN(int value, int width); + // Formats an int value as "%X". static std::string FormatHexInt(int value); // Formats an int value as "%X". static std::string FormatHexUInt32(uint32_t value); @@ -167,6 +170,6 @@ GTEST_API_ std::string StringStreamToString(::std::stringstream* stream); } // namespace internal } // namespace testing -#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_STRING_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_STRING_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-type-util.h b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-type-util.h --- a/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-type-util.h +++ b/media/libvpx/libvpx/third_party/googletest/src/include/gtest/internal/gtest-type-util.h @@ -30,12 +30,12 @@ // Type utilities needed for implementing typed and type-parameterized // tests. // GOOGLETEST_CM0001 DO NOT DELETE -#ifndef GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ -#define GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ +#ifndef GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ +#define GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ #include "gtest/internal/gtest-port.h" // #ifdef __GNUC__ is too general here. It is possible to use gcc without using // libstdc++ (which is where cxxabi.h comes from). @@ -62,42 +62,44 @@ } } return s; } -// GetTypeName() returns a human-readable name of type T. +#if GTEST_HAS_RTTI +// GetTypeName(const std::type_info&) returns a human-readable name of type T. +inline std::string GetTypeName(const std::type_info& type) { + const char* const name = type.name(); +#if GTEST_HAS_CXXABI_H_ || defined(__HP_aCC) + int status = 0; + // gcc's implementation of typeid(T).name() mangles the type name, + // so we have to demangle it. +#if GTEST_HAS_CXXABI_H_ + using abi::__cxa_demangle; +#endif // GTEST_HAS_CXXABI_H_ + char* const readable_name = __cxa_demangle(name, nullptr, nullptr, &status); + const std::string name_str(status == 0 ? readable_name : name); + free(readable_name); + return CanonicalizeForStdLibVersioning(name_str); +#else + return name; +#endif // GTEST_HAS_CXXABI_H_ || __HP_aCC +} +#endif // GTEST_HAS_RTTI + +// GetTypeName() returns a human-readable name of type T if and only if +// RTTI is enabled, otherwise it returns a dummy type name. // NB: This function is also used in Google Mock, so don't move it inside of // the typed-test-only section below. template std::string GetTypeName() { -# if GTEST_HAS_RTTI - - const char* const name = typeid(T).name(); -# if GTEST_HAS_CXXABI_H_ || defined(__HP_aCC) - int status = 0; - // gcc's implementation of typeid(T).name() mangles the type name, - // so we have to demangle it. -# if GTEST_HAS_CXXABI_H_ - using abi::__cxa_demangle; -# endif // GTEST_HAS_CXXABI_H_ - char* const readable_name = __cxa_demangle(name, nullptr, nullptr, &status); - const std::string name_str(status == 0 ? readable_name : name); - free(readable_name); - return CanonicalizeForStdLibVersioning(name_str); -# else - return name; -# endif // GTEST_HAS_CXXABI_H_ || __HP_aCC - -# else - +#if GTEST_HAS_RTTI + return GetTypeName(typeid(T)); +#else return ""; - -# endif // GTEST_HAS_RTTI +#endif // GTEST_HAS_RTTI } -#if GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P - // A unique type indicating an empty node struct None {}; # define GTEST_TEMPLATE_ template class @@ -169,15 +171,13 @@ public: using type = typename proxy::type; }; -#endif // GTEST_HAS_TYPED_TEST || GTEST_HAS_TYPED_TEST_P - } // namespace internal template using Types = internal::ProxyTypeList; } // namespace testing -#endif // GTEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ +#endif // GOOGLETEST_INCLUDE_GTEST_INTERNAL_GTEST_TYPE_UTIL_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/src/gtest-death-test.cc b/media/libvpx/libvpx/third_party/googletest/src/src/gtest-death-test.cc --- a/media/libvpx/libvpx/third_party/googletest/src/src/gtest-death-test.cc +++ b/media/libvpx/libvpx/third_party/googletest/src/src/gtest-death-test.cc @@ -30,10 +30,11 @@ // // This file implements death tests. #include "gtest/gtest-death-test.h" +#include #include #include "gtest/internal/gtest-port.h" #include "gtest/internal/custom/gtest.h" @@ -245,11 +246,11 @@ msg << "couldn't detect the number of threads."; } else { msg << "detected " << thread_count << " threads."; } msg << " See " - "https://github.com/google/googletest/blob/master/googletest/docs/" + "https://github.com/google/googletest/blob/master/docs/" "advanced.md#death-tests-and-threads" << " for more explanation and suggested solutions, especially if" << " this is the last message you see before your test times out."; return msg.GetString(); } @@ -862,11 +863,11 @@ char* const* Argv() { return &args_[0]; } int size() { - return args_.size() - 1; + return static_cast(args_.size()) - 1; } private: std::vector args_; }; @@ -888,22 +889,21 @@ status_zx = zx::port::create(0, &port); GTEST_DEATH_TEST_CHECK_(status_zx == ZX_OK); // Register to wait for the child process to terminate. status_zx = child_process_.wait_async( - port, kProcessKey, ZX_PROCESS_TERMINATED, ZX_WAIT_ASYNC_ONCE); + port, kProcessKey, ZX_PROCESS_TERMINATED, 0); GTEST_DEATH_TEST_CHECK_(status_zx == ZX_OK); // Register to wait for the socket to be readable or closed. status_zx = stderr_socket_.wait_async( - port, kSocketKey, ZX_SOCKET_READABLE | ZX_SOCKET_PEER_CLOSED, - ZX_WAIT_ASYNC_ONCE); + port, kSocketKey, ZX_SOCKET_READABLE | ZX_SOCKET_PEER_CLOSED, 0); GTEST_DEATH_TEST_CHECK_(status_zx == ZX_OK); // Register to wait for an exception. status_zx = exception_channel_.wait_async( - port, kExceptionKey, ZX_CHANNEL_READABLE, ZX_WAIT_ASYNC_ONCE); + port, kExceptionKey, ZX_CHANNEL_READABLE, 0); GTEST_DEATH_TEST_CHECK_(status_zx == ZX_OK); bool process_terminated = false; bool socket_closed = false; do { @@ -939,12 +939,11 @@ if (status_zx == ZX_ERR_PEER_CLOSED) { socket_closed = true; } else { GTEST_DEATH_TEST_CHECK_(status_zx == ZX_ERR_SHOULD_WAIT); status_zx = stderr_socket_.wait_async( - port, kSocketKey, ZX_SOCKET_READABLE | ZX_SOCKET_PEER_CLOSED, - ZX_WAIT_ASYNC_ONCE); + port, kSocketKey, ZX_SOCKET_READABLE | ZX_SOCKET_PEER_CLOSED, 0); GTEST_DEATH_TEST_CHECK_(status_zx == ZX_OK); } } else { GTEST_DEATH_TEST_CHECK_(packet.signal.observed & ZX_SOCKET_PEER_CLOSED); socket_closed = true; @@ -953,16 +952,16 @@ } while (!process_terminated && !socket_closed); ReadAndInterpretStatusByte(); zx_info_process_t buffer; - status_zx = child_process_.get_info( - ZX_INFO_PROCESS, &buffer, sizeof(buffer), nullptr, nullptr); + status_zx = child_process_.get_info(ZX_INFO_PROCESS, &buffer, sizeof(buffer), + nullptr, nullptr); GTEST_DEATH_TEST_CHECK_(status_zx == ZX_OK); - GTEST_DEATH_TEST_CHECK_(buffer.exited); - set_status(buffer.return_code); + GTEST_DEATH_TEST_CHECK_(buffer.flags & ZX_INFO_PROCESS_FLAG_EXITED); + set_status(static_cast(buffer.return_code)); return status(); } // The AssumeRole process for a Fuchsia death test. It creates a child // process with the same executable as the current process to run the @@ -1223,25 +1222,13 @@ struct ExecDeathTestArgs { char* const* argv; // Command-line arguments for the child's call to exec int close_fd; // File descriptor to close; the read end of a pipe }; -# if GTEST_OS_MAC -inline char** GetEnviron() { - // When Google Test is built as a framework on MacOS X, the environ variable - // is unavailable. Apple's documentation (man environ) recommends using - // _NSGetEnviron() instead. - return *_NSGetEnviron(); -} -# else -// Some POSIX platforms expect you to declare environ. extern "C" makes -// it reside in the global namespace. +# if GTEST_OS_QNX extern "C" char** environ; -inline char** GetEnviron() { return environ; } -# endif // GTEST_OS_MAC - -# if !GTEST_OS_QNX +# else // GTEST_OS_QNX // The main function for a threadsafe-style death test child process. // This function is called in a clone()-ed process and thus must avoid // any potentially unsafe operations like malloc or libc functions. static int ExecDeathTestChildMain(void* child_arg) { ExecDeathTestArgs* const args = static_cast(child_arg); @@ -1257,22 +1244,22 @@ DeathTestAbort(std::string("chdir(\"") + original_dir + "\") failed: " + GetLastErrnoDescription()); return EXIT_FAILURE; } - // We can safely call execve() as it's a direct system call. We + // We can safely call execv() as it's almost a direct system call. We // cannot use execvp() as it's a libc function and thus potentially - // unsafe. Since execve() doesn't search the PATH, the user must + // unsafe. Since execv() doesn't search the PATH, the user must // invoke the test program via a valid path that contains at least // one path separator. - execve(args->argv[0], args->argv, GetEnviron()); - DeathTestAbort(std::string("execve(") + args->argv[0] + ", ...) in " + + execv(args->argv[0], args->argv); + DeathTestAbort(std::string("execv(") + args->argv[0] + ", ...) in " + original_dir + " failed: " + GetLastErrnoDescription()); return EXIT_FAILURE; } -# endif // !GTEST_OS_QNX +# endif // GTEST_OS_QNX # if GTEST_HAS_CLONE // Two utility routines that together determine the direction the stack // grows. // This could be accomplished more elegantly by a single recursive @@ -1282,23 +1269,28 @@ // GTEST_NO_INLINE_ is required to prevent GCC 4.6 from inlining // StackLowerThanAddress into StackGrowsDown, which then doesn't give // correct answer. static void StackLowerThanAddress(const void* ptr, bool* result) GTEST_NO_INLINE_; +// Make sure sanitizers do not tamper with the stack here. +// Ideally, we want to use `__builtin_frame_address` instead of a local variable +// address with sanitizer disabled, but it does not work when the +// compiler optimizes the stack frame out, which happens on PowerPC targets. // HWAddressSanitizer add a random tag to the MSB of the local variable address, // making comparison result unpredictable. +GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ GTEST_ATTRIBUTE_NO_SANITIZE_HWADDRESS_ static void StackLowerThanAddress(const void* ptr, bool* result) { - int dummy; - *result = (&dummy < ptr); + int dummy = 0; + *result = std::less()(&dummy, ptr); } // Make sure AddressSanitizer does not tamper with the stack here. GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ GTEST_ATTRIBUTE_NO_SANITIZE_HWADDRESS_ static bool StackGrowsDown() { - int dummy; + int dummy = 0; bool result; StackLowerThanAddress(&dummy, &result); return result; } # endif // GTEST_HAS_CLONE @@ -1337,12 +1329,11 @@ GTEST_DEATH_TEST_CHECK_SYSCALL_(fd_flags = fcntl(close_fd, F_GETFD)); GTEST_DEATH_TEST_CHECK_SYSCALL_(fcntl(close_fd, F_SETFD, fd_flags | FD_CLOEXEC)); struct inheritance inherit = {0}; // spawn is a system call. - child_pid = - spawn(args.argv[0], 0, nullptr, &inherit, args.argv, GetEnviron()); + child_pid = spawn(args.argv[0], 0, nullptr, &inherit, args.argv, environ); // Restores the current working directory. GTEST_DEATH_TEST_CHECK_(fchdir(cwd_fd) != -1); GTEST_DEATH_TEST_CHECK_SYSCALL_(close(cwd_fd)); # else // GTEST_OS_QNX diff --git a/media/libvpx/libvpx/third_party/googletest/src/src/gtest-filepath.cc b/media/libvpx/libvpx/third_party/googletest/src/src/gtest-filepath.cc --- a/media/libvpx/libvpx/third_party/googletest/src/src/gtest-filepath.cc +++ b/media/libvpx/libvpx/third_party/googletest/src/src/gtest-filepath.cc @@ -90,12 +90,13 @@ #endif } // Returns the current working directory, or "" if unsuccessful. FilePath FilePath::GetCurrentDir() { -#if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_WINDOWS_PHONE || \ - GTEST_OS_WINDOWS_RT || GTEST_OS_ESP8266 || GTEST_OS_ESP32 +#if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_WINDOWS_PHONE || \ + GTEST_OS_WINDOWS_RT || GTEST_OS_ESP8266 || GTEST_OS_ESP32 || \ + GTEST_OS_XTENSA // These platforms do not have a current directory, so we just return // something reasonable. return FilePath(kCurrentDirectoryString); #elif GTEST_OS_WINDOWS char cwd[GTEST_PATH_MAX_ + 1] = { '\0' }; @@ -207,11 +208,11 @@ LPCWSTR unicode = String::AnsiToUtf16(pathname_.c_str()); const DWORD attributes = GetFileAttributes(unicode); delete [] unicode; return attributes != kInvalidFileAttributes; #else - posix::StatStruct file_stat; + posix::StatStruct file_stat{}; return posix::Stat(pathname_.c_str(), &file_stat) == 0; #endif // GTEST_OS_WINDOWS_MOBILE } // Returns true if pathname describes a directory in the file-system @@ -234,11 +235,11 @@ if ((attributes != kInvalidFileAttributes) && (attributes & FILE_ATTRIBUTE_DIRECTORY)) { result = true; } #else - posix::StatStruct file_stat; + posix::StatStruct file_stat{}; result = posix::Stat(path.c_str(), &file_stat) == 0 && posix::IsDir(file_stat); #endif // GTEST_OS_WINDOWS_MOBILE return result; @@ -321,11 +322,11 @@ LPCWSTR unicode = String::AnsiToUtf16(removed_sep.c_str()); int result = CreateDirectory(unicode, nullptr) ? 0 : -1; delete [] unicode; #elif GTEST_OS_WINDOWS int result = _mkdir(pathname_.c_str()); -#elif GTEST_OS_ESP8266 +#elif GTEST_OS_ESP8266 || GTEST_OS_XTENSA // do nothing int result = 0; #else int result = mkdir(pathname_.c_str(), 0777); #endif // GTEST_OS_WINDOWS_MOBILE @@ -347,36 +348,22 @@ // Removes any redundant separators that might be in the pathname. // For example, "bar///foo" becomes "bar/foo". Does not eliminate other // redundancies that might be in a pathname involving "." or "..". void FilePath::Normalize() { - if (pathname_.c_str() == nullptr) { - pathname_ = ""; - return; - } - const char* src = pathname_.c_str(); - char* const dest = new char[pathname_.length() + 1]; - char* dest_ptr = dest; - memset(dest_ptr, 0, pathname_.length() + 1); + auto out = pathname_.begin(); - while (*src != '\0') { - *dest_ptr = *src; - if (!IsPathSeparator(*src)) { - src++; + for (const char character : pathname_) { + if (!IsPathSeparator(character)) { + *(out++) = character; + } else if (out == pathname_.begin() || *std::prev(out) != kPathSeparator) { + *(out++) = kPathSeparator; } else { -#if GTEST_HAS_ALT_PATH_SEP_ - if (*dest_ptr == kAlternatePathSeparator) { - *dest_ptr = kPathSeparator; - } -#endif - while (IsPathSeparator(*src)) - src++; + continue; } - dest_ptr++; } - *dest_ptr = '\0'; - pathname_ = dest; - delete[] dest; + + pathname_.erase(out, pathname_.end()); } } // namespace internal } // namespace testing diff --git a/media/libvpx/libvpx/third_party/googletest/src/src/gtest-internal-inl.h b/media/libvpx/libvpx/third_party/googletest/src/src/gtest-internal-inl.h --- a/media/libvpx/libvpx/third_party/googletest/src/src/gtest-internal-inl.h +++ b/media/libvpx/libvpx/third_party/googletest/src/src/gtest-internal-inl.h @@ -29,12 +29,12 @@ // Utility functions and classes used by the Google C++ testing framework.// // This file contains purely Google Test's internal implementation. Please // DO NOT #INCLUDE IT IN A USER PROGRAM. -#ifndef GTEST_SRC_GTEST_INTERNAL_INL_H_ -#define GTEST_SRC_GTEST_INTERNAL_INL_H_ +#ifndef GOOGLETEST_SRC_GTEST_INTERNAL_INL_H_ +#define GOOGLETEST_SRC_GTEST_INTERNAL_INL_H_ #ifndef _WIN32_WCE # include #endif // !_WIN32_WCE #include @@ -82,13 +82,15 @@ // Names of the flags (needed for parsing Google Test flags). const char kAlsoRunDisabledTestsFlag[] = "also_run_disabled_tests"; const char kBreakOnFailureFlag[] = "break_on_failure"; const char kCatchExceptionsFlag[] = "catch_exceptions"; const char kColorFlag[] = "color"; +const char kFailFast[] = "fail_fast"; const char kFilterFlag[] = "filter"; const char kListTestsFlag[] = "list_tests"; const char kOutputFlag[] = "output"; +const char kBriefFlag[] = "brief"; const char kPrintTimeFlag[] = "print_time"; const char kPrintUTF8Flag[] = "print_utf8"; const char kRandomSeedFlag[] = "random_seed"; const char kRepeatFlag[] = "repeat"; const char kShuffleFlag[] = "shuffle"; @@ -162,14 +164,16 @@ break_on_failure_ = GTEST_FLAG(break_on_failure); catch_exceptions_ = GTEST_FLAG(catch_exceptions); color_ = GTEST_FLAG(color); death_test_style_ = GTEST_FLAG(death_test_style); death_test_use_fork_ = GTEST_FLAG(death_test_use_fork); + fail_fast_ = GTEST_FLAG(fail_fast); filter_ = GTEST_FLAG(filter); internal_run_death_test_ = GTEST_FLAG(internal_run_death_test); list_tests_ = GTEST_FLAG(list_tests); output_ = GTEST_FLAG(output); + brief_ = GTEST_FLAG(brief); print_time_ = GTEST_FLAG(print_time); print_utf8_ = GTEST_FLAG(print_utf8); random_seed_ = GTEST_FLAG(random_seed); repeat_ = GTEST_FLAG(repeat); shuffle_ = GTEST_FLAG(shuffle); @@ -185,13 +189,15 @@ GTEST_FLAG(catch_exceptions) = catch_exceptions_; GTEST_FLAG(color) = color_; GTEST_FLAG(death_test_style) = death_test_style_; GTEST_FLAG(death_test_use_fork) = death_test_use_fork_; GTEST_FLAG(filter) = filter_; + GTEST_FLAG(fail_fast) = fail_fast_; GTEST_FLAG(internal_run_death_test) = internal_run_death_test_; GTEST_FLAG(list_tests) = list_tests_; GTEST_FLAG(output) = output_; + GTEST_FLAG(brief) = brief_; GTEST_FLAG(print_time) = print_time_; GTEST_FLAG(print_utf8) = print_utf8_; GTEST_FLAG(random_seed) = random_seed_; GTEST_FLAG(repeat) = repeat_; GTEST_FLAG(shuffle) = shuffle_; @@ -206,14 +212,16 @@ bool break_on_failure_; bool catch_exceptions_; std::string color_; std::string death_test_style_; bool death_test_use_fork_; + bool fail_fast_; std::string filter_; std::string internal_run_death_test_; bool list_tests_; std::string output_; + bool brief_; bool print_time_; bool print_utf8_; int32_t random_seed_; int32_t repeat_; bool shuffle_; @@ -384,17 +392,10 @@ // none was explicitly specified. static std::string GetAbsolutePathToOutputFile(); // Functions for processing the gtest_filter flag. - // Returns true if and only if the wildcard pattern matches the string. - // The first ':' or '\0' character in pattern marks the end of it. - // - // This recursive algorithm isn't very efficient, but is clear and - // works well enough for matching test names, which are short. - static bool PatternMatchesString(const char *pattern, const char *str); - // Returns true if and only if the user-specified filter matches the test // suite name and the test name. static bool FilterMatchesTest(const std::string& test_suite_name, const std::string& test_name); @@ -645,14 +646,14 @@ // exist, creates one and returns it. // // Arguments: // // test_suite_name: name of the test suite - // type_param: the name of the test's type parameter, or NULL if - // this is not a typed or a type-parameterized test. - // set_up_tc: pointer to the function that sets up the test suite - // tear_down_tc: pointer to the function that tears down the test suite + // type_param: the name of the test's type parameter, or NULL if + // this is not a typed or a type-parameterized test. + // set_up_tc: pointer to the function that sets up the test suite + // tear_down_tc: pointer to the function that tears down the test suite TestSuite* GetTestSuite(const char* test_suite_name, const char* type_param, internal::SetUpTestSuiteFunc set_up_tc, internal::TearDownTestSuiteFunc tear_down_tc); // Legacy API is deprecated but still available @@ -672,10 +673,11 @@ // tear_down_tc: pointer to the function that tears down the test suite // test_info: the TestInfo object void AddTestInfo(internal::SetUpTestSuiteFunc set_up_tc, internal::TearDownTestSuiteFunc tear_down_tc, TestInfo* test_info) { +#if GTEST_HAS_DEATH_TEST // In order to support thread-safe death tests, we need to // remember the original working directory when the test program // was first invoked. We cannot do this in RUN_ALL_TESTS(), as // the user may have changed the current directory before calling // RUN_ALL_TESTS(). Therefore we capture the current directory in @@ -684,10 +686,11 @@ if (original_working_dir_.IsEmpty()) { original_working_dir_.Set(FilePath::GetCurrentDir()); GTEST_CHECK_(!original_working_dir_.IsEmpty()) << "Failed to get the current working directory."; } +#endif // GTEST_HAS_DEATH_TEST GetTestSuite(test_info->test_suite_name(), test_info->type_param(), set_up_tc, tear_down_tc) ->AddTestInfo(test_info); } @@ -1159,17 +1162,17 @@ FormatBool(unit_test.Passed()) + "&elapsed_time=" + StreamableToString(unit_test.elapsed_time()) + "ms"); } // Note that "event=TestCaseStart" is a wire format and has to remain - // "case" for compatibilty + // "case" for compatibility void OnTestCaseStart(const TestCase& test_case) override { SendLn(std::string("event=TestCaseStart&name=") + test_case.name()); } // Note that "event=TestCaseEnd" is a wire format and has to remain - // "case" for compatibilty + // "case" for compatibility void OnTestCaseEnd(const TestCase& test_case) override { SendLn("event=TestCaseEnd&passed=" + FormatBool(test_case.Passed()) + "&elapsed_time=" + StreamableToString(test_case.elapsed_time()) + "ms"); } @@ -1213,6 +1216,6 @@ } // namespace internal } // namespace testing GTEST_DISABLE_MSC_WARNINGS_POP_() // 4251 -#endif // GTEST_SRC_GTEST_INTERNAL_INL_H_ +#endif // GOOGLETEST_SRC_GTEST_INTERNAL_INL_H_ diff --git a/media/libvpx/libvpx/third_party/googletest/src/src/gtest-port.cc b/media/libvpx/libvpx/third_party/googletest/src/src/gtest-port.cc --- a/media/libvpx/libvpx/third_party/googletest/src/src/gtest-port.cc +++ b/media/libvpx/libvpx/third_party/googletest/src/src/gtest-port.cc @@ -196,21 +196,22 @@ // get number of structs size_t size; if (sysctl(mib, miblen, NULL, &size, NULL, 0)) { return 0; } - mib[5] = size / mib[4]; + + mib[5] = static_cast(size / static_cast(mib[4])); // populate array of structs struct kinfo_proc info[mib[5]]; if (sysctl(mib, miblen, &info, &size, NULL, 0)) { return 0; } // exclude empty members - int nthreads = 0; - for (int i = 0; i < size / mib[4]; i++) { + size_t nthreads = 0; + for (size_t i = 0; i < size / static_cast(mib[4]); i++) { if (info[i].p_tid != -1) nthreads++; } return nthreads; } @@ -685,12 +686,12 @@ static Mutex mutex_; // Protects access to GetThreadMapLocked() and its return value. static Mutex thread_map_mutex_; }; -Mutex ThreadLocalRegistryImpl::mutex_(Mutex::kStaticMutex); -Mutex ThreadLocalRegistryImpl::thread_map_mutex_(Mutex::kStaticMutex); +Mutex ThreadLocalRegistryImpl::mutex_(Mutex::kStaticMutex); // NOLINT +Mutex ThreadLocalRegistryImpl::thread_map_mutex_(Mutex::kStaticMutex); // NOLINT ThreadLocalValueHolderBase* ThreadLocalRegistry::GetValueOnCurrentThread( const ThreadLocalBase* thread_local_instance) { return ThreadLocalRegistryImpl::GetValueOnCurrentThread( thread_local_instance); @@ -1092,13 +1093,13 @@ GTEST_CHECK_(captured_fd != -1) << "Unable to open temporary file " << temp_file_path; filename_ = temp_file_path; # else // There's no guarantee that a test has write access to the current - // directory, so we create the temporary file in the /tmp directory - // instead. We use /tmp on most systems, and /sdcard on Android. - // That's because Android doesn't have /tmp. + // directory, so we create the temporary file in a temporary directory. + std::string name_template; + # if GTEST_OS_LINUX_ANDROID // Note: Android applications are expected to call the framework's // Context.getExternalStorageDirectory() method through JNI to get // the location of the world-writable SD Card directory. However, // this requires a Context handle, which cannot be retrieved @@ -1107,21 +1108,50 @@ // run in a Dalvik process (e.g. when running it through 'adb shell'). // // The location /data/local/tmp is directly accessible from native code. // '/sdcard' and other variants cannot be relied on, as they are not // guaranteed to be mounted, or may have a delay in mounting. - char name_template[] = "/data/local/tmp/gtest_captured_stream.XXXXXX"; + name_template = "/data/local/tmp/"; +# elif GTEST_OS_IOS + char user_temp_dir[PATH_MAX + 1]; + + // Documented alternative to NSTemporaryDirectory() (for obtaining creating + // a temporary directory) at + // https://developer.apple.com/library/archive/documentation/Security/Conceptual/SecureCodingGuide/Articles/RaceConditions.html#//apple_ref/doc/uid/TP40002585-SW10 + // + // _CS_DARWIN_USER_TEMP_DIR (as well as _CS_DARWIN_USER_CACHE_DIR) is not + // documented in the confstr() man page at + // https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/confstr.3.html#//apple_ref/doc/man/3/confstr + // but are still available, according to the WebKit patches at + // https://trac.webkit.org/changeset/262004/webkit + // https://trac.webkit.org/changeset/263705/webkit + // + // The confstr() implementation falls back to getenv("TMPDIR"). See + // https://opensource.apple.com/source/Libc/Libc-1439.100.3/gen/confstr.c.auto.html + ::confstr(_CS_DARWIN_USER_TEMP_DIR, user_temp_dir, sizeof(user_temp_dir)); + + name_template = user_temp_dir; + if (name_template.back() != GTEST_PATH_SEP_[0]) + name_template.push_back(GTEST_PATH_SEP_[0]); # else - char name_template[] = "/tmp/captured_stream.XXXXXX"; -# endif // GTEST_OS_LINUX_ANDROID - const int captured_fd = mkstemp(name_template); + name_template = "/tmp/"; +# endif + name_template.append("gtest_captured_stream.XXXXXX"); + + // mkstemp() modifies the string bytes in place, and does not go beyond the + // string's length. This results in well-defined behavior in C++17. + // + // The const_cast is needed below C++17. The constraints on std::string + // implementations in C++11 and above make assumption behind the const_cast + // fairly safe. + const int captured_fd = ::mkstemp(const_cast(name_template.data())); if (captured_fd == -1) { GTEST_LOG_(WARNING) << "Failed to create tmp file " << name_template << " for test; does the test have access to the /tmp directory?"; } - filename_ = name_template; + filename_ = std::move(name_template); # endif // GTEST_OS_WINDOWS fflush(nullptr); dup2(captured_fd, fd_); close(captured_fd); } diff --git a/media/libvpx/libvpx/third_party/googletest/src/src/gtest-printers.cc b/media/libvpx/libvpx/third_party/googletest/src/src/gtest-printers.cc --- a/media/libvpx/libvpx/third_party/googletest/src/src/gtest-printers.cc +++ b/media/libvpx/libvpx/third_party/googletest/src/src/gtest-printers.cc @@ -40,15 +40,20 @@ // type Foo by defining either operator<<(::std::ostream&, const Foo&) // or void PrintTo(const Foo&, ::std::ostream*) in the namespace that // defines Foo. #include "gtest/gtest-printers.h" + #include + #include +#include #include #include // NOLINT #include +#include + #include "gtest/internal/gtest-port.h" #include "src/gtest-internal-inl.h" namespace testing { @@ -100,13 +105,23 @@ PrintByteSegmentInObjectTo(obj_bytes, resume_pos, count - resume_pos, os); } *os << ">"; } +// Helpers for widening a character to char32_t. Since the standard does not +// specify if char / wchar_t is signed or unsigned, it is important to first +// convert it to the unsigned type of the same width before widening it to +// char32_t. +template +char32_t ToChar32(CharType in) { + return static_cast( + static_cast::type>(in)); +} + } // namespace -namespace internal2 { +namespace internal { // Delegates to PrintBytesInObjectToImpl() to print the bytes in the // given object. The delegation simplifies the implementation, which // uses the << operator and thus is easier done outside of the // ::testing::internal namespace, which contains a << operator that @@ -114,14 +129,10 @@ void PrintBytesInObjectTo(const unsigned char* obj_bytes, size_t count, ostream* os) { PrintBytesInObjectToImpl(obj_bytes, count, os); } -} // namespace internal2 - -namespace internal { - // Depending on the value of a char (or wchar_t), we print it in one // of three formats: // - as is if it's a printable ASCII (e.g. 'a', '2', ' '), // - as a hexadecimal escape sequence (e.g. '\x7F'), or // - as a special escape sequence (e.g. '\r', '\n'). @@ -132,22 +143,19 @@ }; // Returns true if c is a printable ASCII character. We test the // value of c directly instead of calling isprint(), which is buggy on // Windows Mobile. -inline bool IsPrintableAscii(wchar_t c) { - return 0x20 <= c && c <= 0x7E; -} +inline bool IsPrintableAscii(char32_t c) { return 0x20 <= c && c <= 0x7E; } -// Prints a wide or narrow char c as a character literal without the -// quotes, escaping it when necessary; returns how c was formatted. -// The template argument UnsignedChar is the unsigned version of Char, -// which is the type of c. -template +// Prints c (of type char, char8_t, char16_t, char32_t, or wchar_t) as a +// character literal without the quotes, escaping it when necessary; returns how +// c was formatted. +template static CharFormat PrintAsCharLiteralTo(Char c, ostream* os) { - wchar_t w_c = static_cast(c); - switch (w_c) { + const char32_t u_c = ToChar32(c); + switch (u_c) { case L'\0': *os << "\\0"; break; case L'\'': *os << "\\'"; @@ -175,55 +183,96 @@ break; case L'\v': *os << "\\v"; break; default: - if (IsPrintableAscii(w_c)) { + if (IsPrintableAscii(u_c)) { *os << static_cast(c); return kAsIs; } else { ostream::fmtflags flags = os->flags(); - *os << "\\x" << std::hex << std::uppercase - << static_cast(static_cast(c)); + *os << "\\x" << std::hex << std::uppercase << static_cast(u_c); os->flags(flags); return kHexEscape; } } return kSpecialEscape; } -// Prints a wchar_t c as if it's part of a string literal, escaping it when +// Prints a char32_t c as if it's part of a string literal, escaping it when // necessary; returns how c was formatted. -static CharFormat PrintAsStringLiteralTo(wchar_t c, ostream* os) { +static CharFormat PrintAsStringLiteralTo(char32_t c, ostream* os) { switch (c) { case L'\'': *os << "'"; return kAsIs; case L'"': *os << "\\\""; return kSpecialEscape; default: - return PrintAsCharLiteralTo(c, os); + return PrintAsCharLiteralTo(c, os); } } +static const char* GetCharWidthPrefix(char) { + return ""; +} + +static const char* GetCharWidthPrefix(signed char) { + return ""; +} + +static const char* GetCharWidthPrefix(unsigned char) { + return ""; +} + +#ifdef __cpp_char8_t +static const char* GetCharWidthPrefix(char8_t) { + return "u8"; +} +#endif + +static const char* GetCharWidthPrefix(char16_t) { + return "u"; +} + +static const char* GetCharWidthPrefix(char32_t) { + return "U"; +} + +static const char* GetCharWidthPrefix(wchar_t) { + return "L"; +} + // Prints a char c as if it's part of a string literal, escaping it when // necessary; returns how c was formatted. static CharFormat PrintAsStringLiteralTo(char c, ostream* os) { - return PrintAsStringLiteralTo( - static_cast(static_cast(c)), os); + return PrintAsStringLiteralTo(ToChar32(c), os); +} + +#ifdef __cpp_char8_t +static CharFormat PrintAsStringLiteralTo(char8_t c, ostream* os) { + return PrintAsStringLiteralTo(ToChar32(c), os); +} +#endif + +static CharFormat PrintAsStringLiteralTo(char16_t c, ostream* os) { + return PrintAsStringLiteralTo(ToChar32(c), os); } -// Prints a wide or narrow character c and its code. '\0' is printed -// as "'\\0'", other unprintable characters are also properly escaped -// using the standard C++ escape sequence. The template argument -// UnsignedChar is the unsigned version of Char, which is the type of c. -template +static CharFormat PrintAsStringLiteralTo(wchar_t c, ostream* os) { + return PrintAsStringLiteralTo(ToChar32(c), os); +} + +// Prints a character c (of type char, char8_t, char16_t, char32_t, or wchar_t) +// and its code. '\0' is printed as "'\\0'", other unprintable characters are +// also properly escaped using the standard C++ escape sequence. +template void PrintCharAndCodeTo(Char c, ostream* os) { // First, print c as a literal in the most readable form we can find. - *os << ((sizeof(c) > 1) ? "L'" : "'"); - const CharFormat format = PrintAsCharLiteralTo(c, os); + *os << GetCharWidthPrefix(c) << "'"; + const CharFormat format = PrintAsCharLiteralTo(c, os); *os << "'"; // To aid user debugging, we also print c's code in decimal, unless // it's 0 (in which case c was printed as '\\0', making the code // obvious). @@ -240,45 +289,45 @@ *os << ", 0x" << String::FormatHexInt(static_cast(c)); } *os << ")"; } -void PrintTo(unsigned char c, ::std::ostream* os) { - PrintCharAndCodeTo(c, os); -} -void PrintTo(signed char c, ::std::ostream* os) { - PrintCharAndCodeTo(c, os); -} +void PrintTo(unsigned char c, ::std::ostream* os) { PrintCharAndCodeTo(c, os); } +void PrintTo(signed char c, ::std::ostream* os) { PrintCharAndCodeTo(c, os); } // Prints a wchar_t as a symbol if it is printable or as its internal // code otherwise and also as its code. L'\0' is printed as "L'\\0'". -void PrintTo(wchar_t wc, ostream* os) { - PrintCharAndCodeTo(wc, os); +void PrintTo(wchar_t wc, ostream* os) { PrintCharAndCodeTo(wc, os); } + +// TODO(dcheng): Consider making this delegate to PrintCharAndCodeTo() as well. +void PrintTo(char32_t c, ::std::ostream* os) { + *os << std::hex << "U+" << std::uppercase << std::setfill('0') << std::setw(4) + << static_cast(c); } // Prints the given array of characters to the ostream. CharType must be either -// char or wchar_t. +// char, char8_t, char16_t, char32_t, or wchar_t. // The array starts at begin, the length is len, it may include '\0' characters // and may not be NUL-terminated. template GTEST_ATTRIBUTE_NO_SANITIZE_MEMORY_ GTEST_ATTRIBUTE_NO_SANITIZE_ADDRESS_ GTEST_ATTRIBUTE_NO_SANITIZE_HWADDRESS_ GTEST_ATTRIBUTE_NO_SANITIZE_THREAD_ static CharFormat PrintCharsAsStringTo( const CharType* begin, size_t len, ostream* os) { - const char* const kQuoteBegin = sizeof(CharType) == 1 ? "\"" : "L\""; - *os << kQuoteBegin; + const char* const quote_prefix = GetCharWidthPrefix(*begin); + *os << quote_prefix << "\""; bool is_previous_hex = false; CharFormat print_format = kAsIs; for (size_t index = 0; index < len; ++index) { const CharType cur = begin[index]; if (is_previous_hex && IsXDigit(cur)) { // Previous character is of '\x..' form and this character can be // interpreted as another hexadecimal digit in its number. Break string to // disambiguate. - *os << "\" " << kQuoteBegin; + *os << "\" " << quote_prefix << "\""; } is_previous_hex = PrintAsStringLiteralTo(cur, os) == kHexEscape; // Remember if any characters required hex escaping. if (is_previous_hex) { print_format = kHexEscape; @@ -320,42 +369,70 @@ // Prints a (const) char array of 'len' elements, starting at address 'begin'. void UniversalPrintArray(const char* begin, size_t len, ostream* os) { UniversalPrintCharArray(begin, len, os); } +#ifdef __cpp_char8_t +// Prints a (const) char8_t array of 'len' elements, starting at address +// 'begin'. +void UniversalPrintArray(const char8_t* begin, size_t len, ostream* os) { + UniversalPrintCharArray(begin, len, os); +} +#endif + +// Prints a (const) char16_t array of 'len' elements, starting at address +// 'begin'. +void UniversalPrintArray(const char16_t* begin, size_t len, ostream* os) { + UniversalPrintCharArray(begin, len, os); +} + +// Prints a (const) char32_t array of 'len' elements, starting at address +// 'begin'. +void UniversalPrintArray(const char32_t* begin, size_t len, ostream* os) { + UniversalPrintCharArray(begin, len, os); +} + // Prints a (const) wchar_t array of 'len' elements, starting at address // 'begin'. void UniversalPrintArray(const wchar_t* begin, size_t len, ostream* os) { UniversalPrintCharArray(begin, len, os); } -// Prints the given C string to the ostream. -void PrintTo(const char* s, ostream* os) { +namespace { + +// Prints a null-terminated C-style string to the ostream. +template +void PrintCStringTo(const Char* s, ostream* os) { if (s == nullptr) { *os << "NULL"; } else { *os << ImplicitCast_(s) << " pointing to "; - PrintCharsAsStringTo(s, strlen(s), os); + PrintCharsAsStringTo(s, std::char_traits::length(s), os); } } +} // anonymous namespace + +void PrintTo(const char* s, ostream* os) { PrintCStringTo(s, os); } + +#ifdef __cpp_char8_t +void PrintTo(const char8_t* s, ostream* os) { PrintCStringTo(s, os); } +#endif + +void PrintTo(const char16_t* s, ostream* os) { PrintCStringTo(s, os); } + +void PrintTo(const char32_t* s, ostream* os) { PrintCStringTo(s, os); } + // MSVC compiler can be configured to define whar_t as a typedef // of unsigned short. Defining an overload for const wchar_t* in that case // would cause pointers to unsigned shorts be printed as wide strings, // possibly accessing more memory than intended and causing invalid // memory accesses. MSVC defines _NATIVE_WCHAR_T_DEFINED symbol when // wchar_t is implemented as a native type. #if !defined(_MSC_VER) || defined(_NATIVE_WCHAR_T_DEFINED) // Prints the given wide C string to the ostream. -void PrintTo(const wchar_t* s, ostream* os) { - if (s == nullptr) { - *os << "NULL"; - } else { - *os << ImplicitCast_(s) << " pointing to "; - PrintCharsAsStringTo(s, wcslen(s), os); - } -} +void PrintTo(const wchar_t* s, ostream* os) { PrintCStringTo(s, os); } #endif // wchar_t is native namespace { bool ContainsUnprintableControlCodes(const char* str, size_t length) { @@ -429,10 +506,24 @@ ConditionalPrintAsText(s.data(), s.size(), os); } } } +#ifdef __cpp_char8_t +void PrintU8StringTo(const ::std::u8string& s, ostream* os) { + PrintCharsAsStringTo(s.data(), s.size(), os); +} +#endif + +void PrintU16StringTo(const ::std::u16string& s, ostream* os) { + PrintCharsAsStringTo(s.data(), s.size(), os); +} + +void PrintU32StringTo(const ::std::u32string& s, ostream* os) { + PrintCharsAsStringTo(s.data(), s.size(), os); +} + #if GTEST_HAS_STD_WSTRING void PrintWideStringTo(const ::std::wstring& s, ostream* os) { PrintCharsAsStringTo(s.data(), s.size(), os); } #endif // GTEST_HAS_STD_WSTRING diff --git a/media/libvpx/libvpx/third_party/googletest/src/src/gtest-typed-test.cc b/media/libvpx/libvpx/third_party/googletest/src/src/gtest-typed-test.cc --- a/media/libvpx/libvpx/third_party/googletest/src/src/gtest-typed-test.cc +++ b/media/libvpx/libvpx/third_party/googletest/src/src/gtest-typed-test.cc @@ -33,12 +33,10 @@ #include "gtest/gtest.h" namespace testing { namespace internal { -#if GTEST_HAS_TYPED_TEST_P - // Skips to the first non-space char in str. Returns an empty string if str // contains only whitespace characters. static const char* SkipSpaces(const char* str) { while (IsSpace(*str)) str++; @@ -76,21 +74,11 @@ if (tests.count(name) != 0) { errors << "Test " << name << " is listed more than once.\n"; continue; } - bool found = false; - for (RegisteredTestIter it = registered_tests_.begin(); - it != registered_tests_.end(); - ++it) { - if (name == it->first) { - found = true; - break; - } - } - - if (found) { + if (registered_tests_.count(name) != 0) { tests.insert(name); } else { errors << "No test named " << name << " can be found in this test suite.\n"; } @@ -113,9 +101,7 @@ } return registered_tests; } -#endif // GTEST_HAS_TYPED_TEST_P - } // namespace internal } // namespace testing diff --git a/media/libvpx/libvpx/third_party/googletest/src/src/gtest.cc b/media/libvpx/libvpx/third_party/googletest/src/src/gtest.cc --- a/media/libvpx/libvpx/third_party/googletest/src/src/gtest.cc +++ b/media/libvpx/libvpx/third_party/googletest/src/src/gtest.cc @@ -33,19 +33,20 @@ #include "gtest/gtest.h" #include "gtest/internal/custom/gtest.h" #include "gtest/gtest-spi.h" #include -#include #include #include #include #include #include #include #include +#include // NOLINT +#include #include #include #include #include #include @@ -53,12 +54,10 @@ #include #include #if GTEST_OS_LINUX -# define GTEST_HAS_GETTIMEOFDAY_ 1 - # include // NOLINT # include // NOLINT # include // NOLINT // Declares vsnprintf(). This header is not available on Windows. # include // NOLINT @@ -66,11 +65,10 @@ # include // NOLINT # include // NOLINT # include #elif GTEST_OS_ZOS -# define GTEST_HAS_GETTIMEOFDAY_ 1 # include // NOLINT // On z/OS we additionally need strings.h for strcasecmp. # include // NOLINT @@ -84,29 +82,23 @@ # include // NOLINT # undef min #ifdef _MSC_VER # include // NOLINT -# include // NOLINT #endif # include // NOLINT # include // NOLINT # include // NOLINT # include // NOLINT # if GTEST_OS_WINDOWS_MINGW -// MinGW has gettimeofday() but not _ftime64(). -# define GTEST_HAS_GETTIMEOFDAY_ 1 # include // NOLINT # endif // GTEST_OS_WINDOWS_MINGW #else -// Assume other platforms have gettimeofday(). -# define GTEST_HAS_GETTIMEOFDAY_ 1 - // cpplint thinks that the header is already included, so we want to // silence it. # include // NOLINT # include // NOLINT @@ -211,10 +203,25 @@ return testbridge_test_only; } return kUniversalFilter; } +// Bazel passes in the argument to '--test_runner_fail_fast' via the +// TESTBRIDGE_TEST_RUNNER_FAIL_FAST environment variable. +static bool GetDefaultFailFast() { + const char* const testbridge_test_runner_fail_fast = + internal::posix::GetEnv("TESTBRIDGE_TEST_RUNNER_FAIL_FAST"); + if (testbridge_test_runner_fail_fast != nullptr) { + return strcmp(testbridge_test_runner_fail_fast, "1") == 0; + } + return false; +} + +GTEST_DEFINE_bool_( + fail_fast, internal::BoolFromGTestEnv("fail_fast", GetDefaultFailFast()), + "True if and only if a test failure should stop further test execution."); + GTEST_DEFINE_bool_( also_run_disabled_tests, internal::BoolFromGTestEnv("also_run_disabled_tests", false), "Run disabled tests too, in addition to the tests normally being run."); @@ -271,10 +278,14 @@ "If a directory is specified, output files will be created " "within that directory, with file-names based on the test " "executable's name and, if necessary, made unique by adding " "digits."); +GTEST_DEFINE_bool_( + brief, internal::BoolFromGTestEnv("brief", false), + "True if only test failures should be displayed in text output."); + GTEST_DEFINE_bool_(print_time, internal::BoolFromGTestEnv("print_time", true), "True if and only if " GTEST_NAME_ " should display elapsed time in text output."); GTEST_DEFINE_bool_(print_utf8, internal::BoolFromGTestEnv("print_utf8", true), @@ -477,19 +488,19 @@ "code that always depend on code that provides TEST_P. Failing to do " "so is often an indication of dead code, e.g. the last TEST_P was " "removed but the rest got left behind."; std::string message = - "Paramaterized test suite " + name + + "Parameterized test suite " + name + (has_test_p ? kMissingInstantiation : kMissingTestCase) + "\n\n" "To suppress this error for this test suite, insert the following line " "(in a non-header) in the namespace it is defined in:" "\n\n" "GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(" + name + ");"; - std::string full_name = "UninstantiatedParamaterizedTestSuite<" + name + ">"; + std::string full_name = "UninstantiatedParameterizedTestSuite<" + name + ">"; RegisterTest( // "GoogleTestVerification", full_name.c_str(), nullptr, // No type parameter. nullptr, // No value parameter. location.file.c_str(), location.line, [message, location] { @@ -532,27 +543,27 @@ for (const auto& testcase : suites_) { if (testcase.second.instantiated) continue; if (ignored.find(testcase.first) != ignored.end()) continue; std::string message = - "Type paramaterized test suite " + testcase.first + + "Type parameterized test suite " + testcase.first + " is defined via REGISTER_TYPED_TEST_SUITE_P, but never instantiated " "via INSTANTIATE_TYPED_TEST_SUITE_P. None of the test cases will run." "\n\n" "Ideally, TYPED_TEST_P definitions should only ever be included as " "part of binaries that intend to use them. (As opposed to, for " "example, being placed in a library that may be linked in to get other " "utilities.)" "\n\n" "To suppress this error for this test suite, insert the following line " - "(in a non-header) in the namespace it is definedin in:" + "(in a non-header) in the namespace it is defined in:" "\n\n" "GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(" + testcase.first + ");"; std::string full_name = - "UninstantiatedTypeParamaterizedTestSuite<" + testcase.first + ">"; + "UninstantiatedTypeParameterizedTestSuite<" + testcase.first + ">"; RegisterTest( // "GoogleTestVerification", full_name.c_str(), nullptr, // No type parameter. nullptr, // No value parameter. testcase.second.code_location.file.c_str(), @@ -633,51 +644,86 @@ output_name, internal::GetCurrentExecutableName(), GetOutputFormat().c_str())); return result.string(); } -// Returns true if and only if the wildcard pattern matches the string. -// The first ':' or '\0' character in pattern marks the end of it. +// Returns true if and only if the wildcard pattern matches the string. Each +// pattern consists of regular characters, single-character wildcards (?), and +// multi-character wildcards (*). // -// This recursive algorithm isn't very efficient, but is clear and -// works well enough for matching test names, which are short. -bool UnitTestOptions::PatternMatchesString(const char *pattern, - const char *str) { - switch (*pattern) { - case '\0': - case ':': // Either ':' or '\0' marks the end of the pattern. - return *str == '\0'; - case '?': // Matches any single character. - return *str != '\0' && PatternMatchesString(pattern + 1, str + 1); - case '*': // Matches any string (possibly empty) of characters. - return (*str != '\0' && PatternMatchesString(pattern, str + 1)) || - PatternMatchesString(pattern + 1, str); - default: // Non-special character. Matches itself. - return *pattern == *str && - PatternMatchesString(pattern + 1, str + 1); - } -} - -bool UnitTestOptions::MatchesFilter( - const std::string& name, const char* filter) { - const char *cur_pattern = filter; - for (;;) { - if (PatternMatchesString(cur_pattern, name.c_str())) { +// This function implements a linear-time string globbing algorithm based on +// https://research.swtch.com/glob. +static bool PatternMatchesString(const std::string& name_str, + const char* pattern, const char* pattern_end) { + const char* name = name_str.c_str(); + const char* const name_begin = name; + const char* const name_end = name + name_str.size(); + + const char* pattern_next = pattern; + const char* name_next = name; + + while (pattern < pattern_end || name < name_end) { + if (pattern < pattern_end) { + switch (*pattern) { + default: // Match an ordinary character. + if (name < name_end && *name == *pattern) { + ++pattern; + ++name; + continue; + } + break; + case '?': // Match any single character. + if (name < name_end) { + ++pattern; + ++name; + continue; + } + break; + case '*': + // Match zero or more characters. Start by skipping over the wildcard + // and matching zero characters from name. If that fails, restart and + // match one more character than the last attempt. + pattern_next = pattern; + name_next = name + 1; + ++pattern; + continue; + } + } + // Failed to match a character. Restart if possible. + if (name_begin < name_next && name_next <= name_end) { + pattern = pattern_next; + name = name_next; + continue; + } + return false; + } + return true; +} + +bool UnitTestOptions::MatchesFilter(const std::string& name_str, + const char* filter) { + // The filter is a list of patterns separated by colons (:). + const char* pattern = filter; + while (true) { + // Find the bounds of this pattern. + const char* const next_sep = strchr(pattern, ':'); + const char* const pattern_end = + next_sep != nullptr ? next_sep : pattern + strlen(pattern); + + // Check if this pattern matches name_str. + if (PatternMatchesString(name_str, pattern, pattern_end)) { return true; } - // Finds the next pattern in the filter. - cur_pattern = strchr(cur_pattern, ':'); - - // Returns if no more pattern can be found. - if (cur_pattern == nullptr) { + // Give up on this pattern. However, if we found a pattern separator (:), + // advance to the next pattern (skipping over the separator) and restart. + if (next_sep == nullptr) { return false; } - - // Skips the pattern separater (the ':' character). - cur_pattern++; - } + pattern = next_sep + 1; + } + return true; } // Returns true if and only if the user-specified filter matches the test // suite name and the test name. bool UnitTestOptions::FilterMatchesTest(const std::string& test_suite_name, @@ -983,48 +1029,34 @@ // Skips the user-specified number of frames plus this function // itself. ); // NOLINT } -// Returns the current time in milliseconds. +// A helper class for measuring elapsed times. +class Timer { + public: + Timer() : start_(std::chrono::steady_clock::now()) {} + + // Return time elapsed in milliseconds since the timer was created. + TimeInMillis Elapsed() { + return std::chrono::duration_cast( + std::chrono::steady_clock::now() - start_) + .count(); + } + + private: + std::chrono::steady_clock::time_point start_; +}; + +// Returns a timestamp as milliseconds since the epoch. Note this time may jump +// around subject to adjustments by the system, to measure elapsed time use +// Timer instead. TimeInMillis GetTimeInMillis() { -#if GTEST_OS_WINDOWS_MOBILE || defined(__BORLANDC__) - // Difference between 1970-01-01 and 1601-01-01 in milliseconds. - // http://analogous.blogspot.com/2005/04/epoch.html - const TimeInMillis kJavaEpochToWinFileTimeDelta = - static_cast(116444736UL) * 100000UL; - const DWORD kTenthMicrosInMilliSecond = 10000; - - SYSTEMTIME now_systime; - FILETIME now_filetime; - ULARGE_INTEGER now_int64; - GetSystemTime(&now_systime); - if (SystemTimeToFileTime(&now_systime, &now_filetime)) { - now_int64.LowPart = now_filetime.dwLowDateTime; - now_int64.HighPart = now_filetime.dwHighDateTime; - now_int64.QuadPart = (now_int64.QuadPart / kTenthMicrosInMilliSecond) - - kJavaEpochToWinFileTimeDelta; - return now_int64.QuadPart; - } - return 0; -#elif GTEST_OS_WINDOWS && !GTEST_HAS_GETTIMEOFDAY_ - __timeb64 now; - - // MSVC 8 deprecates _ftime64(), so we want to suppress warning 4996 - // (deprecated function) there. - GTEST_DISABLE_MSC_DEPRECATED_PUSH_() - _ftime64(&now); - GTEST_DISABLE_MSC_DEPRECATED_POP_() - - return static_cast(now.time) * 1000 + now.millitm; -#elif GTEST_HAS_GETTIMEOFDAY_ - struct timeval now; - gettimeofday(&now, nullptr); - return static_cast(now.tv_sec) * 1000 + now.tv_usec / 1000; -#else -# error "Don't know how to get the current time on your system." -#endif + return std::chrono::duration_cast( + std::chrono::system_clock::now() - + std::chrono::system_clock::from_time_t(0)) + .count(); } // Utilities // class String. @@ -1535,10 +1567,35 @@ double val2, double abs_error) { const double diff = fabs(val1 - val2); if (diff <= abs_error) return AssertionSuccess(); + // Find the value which is closest to zero. + const double min_abs = std::min(fabs(val1), fabs(val2)); + // Find the distance to the next double from that value. + const double epsilon = + nextafter(min_abs, std::numeric_limits::infinity()) - min_abs; + // Detect the case where abs_error is so small that EXPECT_NEAR is + // effectively the same as EXPECT_EQUAL, and give an informative error + // message so that the situation can be more easily understood without + // requiring exotic floating-point knowledge. + // Don't do an epsilon check if abs_error is zero because that implies + // that an equality check was actually intended. + if (!(std::isnan)(val1) && !(std::isnan)(val2) && abs_error > 0 && + abs_error < epsilon) { + return AssertionFailure() + << "The difference between " << expr1 << " and " << expr2 << " is " + << diff << ", where\n" + << expr1 << " evaluates to " << val1 << ",\n" + << expr2 << " evaluates to " << val2 << ".\nThe abs_error parameter " + << abs_error_expr << " evaluates to " << abs_error + << " which is smaller than the minimum distance between doubles for " + "numbers of this magnitude which is " + << epsilon + << ", thus making this EXPECT_NEAR check equivalent to " + "EXPECT_EQUAL. Consider using EXPECT_DOUBLE_EQ instead."; + } return AssertionFailure() << "The difference between " << expr1 << " and " << expr2 << " is " << diff << ", which exceeds " << abs_error_expr << ", where\n" << expr1 << " evaluates to " << val1 << ",\n" << expr2 << " evaluates to " << val2 << ", and\n" @@ -1597,61 +1654,10 @@ return internal::FloatingPointLE(expr1, expr2, val1, val2); } namespace internal { -// The helper function for {ASSERT|EXPECT}_EQ with int or enum -// arguments. -AssertionResult CmpHelperEQ(const char* lhs_expression, - const char* rhs_expression, - BiggestInt lhs, - BiggestInt rhs) { - if (lhs == rhs) { - return AssertionSuccess(); - } - - return EqFailure(lhs_expression, - rhs_expression, - FormatForComparisonFailureMessage(lhs, rhs), - FormatForComparisonFailureMessage(rhs, lhs), - false); -} - -// A macro for implementing the helper functions needed to implement -// ASSERT_?? and EXPECT_?? with integer or enum arguments. It is here -// just to avoid copy-and-paste of similar code. -#define GTEST_IMPL_CMP_HELPER_(op_name, op)\ -AssertionResult CmpHelper##op_name(const char* expr1, const char* expr2, \ - BiggestInt val1, BiggestInt val2) {\ - if (val1 op val2) {\ - return AssertionSuccess();\ - } else {\ - return AssertionFailure() \ - << "Expected: (" << expr1 << ") " #op " (" << expr2\ - << "), actual: " << FormatForComparisonFailureMessage(val1, val2)\ - << " vs " << FormatForComparisonFailureMessage(val2, val1);\ - }\ -} - -// Implements the helper function for {ASSERT|EXPECT}_NE with int or -// enum arguments. -GTEST_IMPL_CMP_HELPER_(NE, !=) -// Implements the helper function for {ASSERT|EXPECT}_LE with int or -// enum arguments. -GTEST_IMPL_CMP_HELPER_(LE, <=) -// Implements the helper function for {ASSERT|EXPECT}_LT with int or -// enum arguments. -GTEST_IMPL_CMP_HELPER_(LT, < ) -// Implements the helper function for {ASSERT|EXPECT}_GE with int or -// enum arguments. -GTEST_IMPL_CMP_HELPER_(GE, >=) -// Implements the helper function for {ASSERT|EXPECT}_GT with int or -// enum arguments. -GTEST_IMPL_CMP_HELPER_(GT, > ) - -#undef GTEST_IMPL_CMP_HELPER_ - // The helper function for {ASSERT|EXPECT}_STREQ. AssertionResult CmpHelperSTREQ(const char* lhs_expression, const char* rhs_expression, const char* lhs, const char* rhs) { @@ -2121,12 +2127,17 @@ suffix.c_str()); } // Formats an int value as "%02d". std::string String::FormatIntWidth2(int value) { + return FormatIntWidthN(value, 2); +} + +// Formats an int value to given width with leading zeros. +std::string String::FormatIntWidthN(int value, int width) { std::stringstream ss; - ss << std::setfill('0') << std::setw(2) << value; + ss << std::setfill('0') << std::setw(width) << value; return ss.str(); } // Formats an int value as "%X". std::string String::FormatHexUInt32(uint32_t value) { @@ -2174,11 +2185,13 @@ // Appends the user message if it's non-empty. const std::string user_msg_string = user_msg.GetString(); if (user_msg_string.empty()) { return gtest_msg; } - + if (gtest_msg.empty()) { + return user_msg_string; + } return gtest_msg + "\n" + user_msg_string; } } // namespace internal @@ -2226,11 +2239,11 @@ void TestResult::RecordProperty(const std::string& xml_element, const TestProperty& test_property) { if (!ValidateTestProperty(xml_element, test_property)) { return; } - internal::MutexLock lock(&test_properites_mutex_); + internal::MutexLock lock(&test_properties_mutex_); const std::vector::iterator property_with_matching_key = std::find_if(test_properties_.begin(), test_properties_.end(), internal::TestPropertyKeyIs(test_property.key())); if (property_with_matching_key == test_properties_.end()) { test_properties_.push_back(test_property); @@ -2253,11 +2266,12 @@ }; // The list of reserved attributes used in the element of XML // output. static const char* const kReservedTestSuiteAttributes[] = { - "disabled", "errors", "failures", "name", "tests", "time", "timestamp"}; + "disabled", "errors", "failures", "name", + "tests", "time", "timestamp", "skipped"}; // The list of reserved attributes used in the element of XML output. static const char* const kReservedTestCaseAttributes[] = { "classname", "name", "status", "time", "type_param", "value_param", "file", "line"}; @@ -2266,11 +2280,11 @@ // still RecordProperty("result") or "RecordProperty(timestamp") static const char* const kReservedOutputTestCaseAttributes[] = { "classname", "name", "status", "time", "type_param", "value_param", "file", "line", "result", "timestamp"}; -template +template std::vector ArrayAsVector(const char* const (&array)[kSize]) { return std::vector(array, array + kSize); } static std::vector GetReservedAttributesForElement( @@ -2710,10 +2724,11 @@ location_(a_code_location), fixture_class_id_(fixture_class_id), should_run_(false), is_disabled_(false), matches_filter_(false), + is_in_another_shard_(false), factory_(factory), result_() {} // Destructs a TestInfo object. TestInfo::~TestInfo() { delete factory_; } @@ -2723,11 +2738,11 @@ // Creates a new TestInfo object and registers it with Google Test; // returns the created object. // // Arguments: // -// test_suite_name: name of the test suite +// test_suite_name: name of the test suite // name: name of the test // type_param: the name of the test's type parameter, or NULL if // this is not a typed or a type-parameterized test. // value_param: text representation of the test's value parameter, // or NULL if this is not a value-parameterized test. @@ -2825,11 +2840,12 @@ TestEventListener* repeater = UnitTest::GetInstance()->listeners().repeater(); // Notifies the unit test event listeners that a test is about to start. repeater->OnTestStart(*this); - const TimeInMillis start = internal::GetTimeInMillis(); + result_.set_start_timestamp(internal::GetTimeInMillis()); + internal::Timer timer; impl->os_stack_trace_getter()->UponLeavingGTest(); // Creates the test object. Test* const test = internal::HandleExceptionsInMethodIfSupported( @@ -2850,21 +2866,42 @@ impl->os_stack_trace_getter()->UponLeavingGTest(); internal::HandleExceptionsInMethodIfSupported( test, &Test::DeleteSelf_, "the test fixture's destructor"); } - result_.set_start_timestamp(start); - result_.set_elapsed_time(internal::GetTimeInMillis() - start); + result_.set_elapsed_time(timer.Elapsed()); // Notifies the unit test event listener that a test has just finished. repeater->OnTestEnd(*this); // Tells UnitTest to stop associating assertion results to this // test. impl->set_current_test_info(nullptr); } +// Skip and records a skipped test result for this object. +void TestInfo::Skip() { + if (!should_run_) return; + + internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); + impl->set_current_test_info(this); + + TestEventListener* repeater = UnitTest::GetInstance()->listeners().repeater(); + + // Notifies the unit test event listeners that a test is about to start. + repeater->OnTestStart(*this); + + const TestPartResult test_part_result = + TestPartResult(TestPartResult::kSkip, this->file(), this->line(), ""); + impl->GetTestPartResultReporterForCurrentThread()->ReportTestPartResult( + test_part_result); + + // Notifies the unit test event listener that a test has just finished. + repeater->OnTestEnd(*this); + impl->set_current_test_info(nullptr); +} + // class TestSuite // Gets the number of successful tests in this test suite. int TestSuite::successful_test_count() const { return CountIf(test_info_list_, TestPassed); @@ -2907,11 +2944,11 @@ // Creates a TestSuite with the given name. // // Arguments: // -// name: name of the test suite +// a_name: name of the test suite // a_type_param: the name of the test suite's type parameter, or NULL if // this is not a typed or a type-parameterized test suite. // set_up_tc: pointer to the function that sets up the test suite // tear_down_tc: pointer to the function that tears down the test suite TestSuite::TestSuite(const char* a_name, const char* a_type_param, @@ -2962,34 +2999,71 @@ TestEventListener* repeater = UnitTest::GetInstance()->listeners().repeater(); // Call both legacy and the new API repeater->OnTestSuiteStart(*this); // Legacy API is deprecated but still available -#ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI +#ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ repeater->OnTestCaseStart(*this); -#endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI +#endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ impl->os_stack_trace_getter()->UponLeavingGTest(); internal::HandleExceptionsInMethodIfSupported( this, &TestSuite::RunSetUpTestSuite, "SetUpTestSuite()"); start_timestamp_ = internal::GetTimeInMillis(); + internal::Timer timer; for (int i = 0; i < total_test_count(); i++) { GetMutableTestInfo(i)->Run(); - } - elapsed_time_ = internal::GetTimeInMillis() - start_timestamp_; + if (GTEST_FLAG(fail_fast) && GetMutableTestInfo(i)->result()->Failed()) { + for (int j = i + 1; j < total_test_count(); j++) { + GetMutableTestInfo(j)->Skip(); + } + break; + } + } + elapsed_time_ = timer.Elapsed(); impl->os_stack_trace_getter()->UponLeavingGTest(); internal::HandleExceptionsInMethodIfSupported( this, &TestSuite::RunTearDownTestSuite, "TearDownTestSuite()"); // Call both legacy and the new API repeater->OnTestSuiteEnd(*this); // Legacy API is deprecated but still available -#ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI +#ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ repeater->OnTestCaseEnd(*this); -#endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI +#endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ + + impl->set_current_test_suite(nullptr); +} + +// Skips all tests under this TestSuite. +void TestSuite::Skip() { + if (!should_run_) return; + + internal::UnitTestImpl* const impl = internal::GetUnitTestImpl(); + impl->set_current_test_suite(this); + + TestEventListener* repeater = UnitTest::GetInstance()->listeners().repeater(); + + // Call both legacy and the new API + repeater->OnTestSuiteStart(*this); +// Legacy API is deprecated but still available +#ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ + repeater->OnTestCaseStart(*this); +#endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ + + for (int i = 0; i < total_test_count(); i++) { + GetMutableTestInfo(i)->Skip(); + } + + // Call both legacy and the new API + repeater->OnTestSuiteEnd(*this); + // Legacy API is deprecated but still available +#ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ + repeater->OnTestCaseEnd(*this); +#endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ impl->set_current_test_suite(nullptr); } // Clears the results of all tests in this test suite. @@ -3037,11 +3111,11 @@ // to "Failure", as the user usually doesn't care about the difference // between the two when viewing the test result. static const char * TestPartResultTypeToString(TestPartResult::Type type) { switch (type) { case TestPartResult::kSkip: - return "Skipped"; + return "Skipped\n"; case TestPartResult::kSuccess: return "Success"; case TestPartResult::kNonFatalFailure: case TestPartResult::kFatalFailure: @@ -3054,10 +3128,13 @@ return "Unknown result type"; } } namespace internal { +namespace { +enum class GTestColor { kDefault, kRed, kGreen, kYellow }; +} // namespace // Prints a TestPartResult to an std::string. static std::string PrintTestPartResultToString( const TestPartResult& test_part_result) { return (Message() @@ -3091,13 +3168,16 @@ !GTEST_OS_WINDOWS_PHONE && !GTEST_OS_WINDOWS_RT && !GTEST_OS_WINDOWS_MINGW // Returns the character attribute for the given color. static WORD GetColorAttribute(GTestColor color) { switch (color) { - case COLOR_RED: return FOREGROUND_RED; - case COLOR_GREEN: return FOREGROUND_GREEN; - case COLOR_YELLOW: return FOREGROUND_RED | FOREGROUND_GREEN; + case GTestColor::kRed: + return FOREGROUND_RED; + case GTestColor::kGreen: + return FOREGROUND_GREEN; + case GTestColor::kYellow: + return FOREGROUND_RED | FOREGROUND_GREEN; default: return 0; } } static int GetBitOffset(WORD color_mask) { @@ -3131,17 +3211,20 @@ return new_color; } #else -// Returns the ANSI color code for the given color. COLOR_DEFAULT is +// Returns the ANSI color code for the given color. GTestColor::kDefault is // an invalid input. static const char* GetAnsiColorCode(GTestColor color) { switch (color) { - case COLOR_RED: return "1"; - case COLOR_GREEN: return "2"; - case COLOR_YELLOW: return "3"; + case GTestColor::kRed: + return "1"; + case GTestColor::kGreen: + return "2"; + case GTestColor::kYellow: + return "3"; default: return nullptr; } } @@ -3186,21 +3269,23 @@ // Helpers for printing colored strings to stdout. Note that on Windows, we // cannot simply emit special characters and have the terminal change colors. // This routine must actually emit the characters rather than return a string // that would be colored when printed, as can be done on Linux. -void ColoredPrintf(GTestColor color, const char* fmt, ...) { + +GTEST_ATTRIBUTE_PRINTF_(2, 3) +static void ColoredPrintf(GTestColor color, const char *fmt, ...) { va_list args; va_start(args, fmt); #if GTEST_OS_WINDOWS_MOBILE || GTEST_OS_ZOS || GTEST_OS_IOS || \ GTEST_OS_WINDOWS_PHONE || GTEST_OS_WINDOWS_RT || defined(ESP_PLATFORM) const bool use_color = AlwaysFalse(); #else static const bool in_color_mode = ShouldUseColor(posix::IsATTY(posix::FileNo(stdout)) != 0); - const bool use_color = in_color_mode && (color != COLOR_DEFAULT); + const bool use_color = in_color_mode && (color != GTestColor::kDefault); #endif // GTEST_OS_WINDOWS_MOBILE || GTEST_OS_ZOS if (!use_color) { vprintf(fmt, args); va_end(args); @@ -3308,47 +3393,46 @@ const char* const filter = GTEST_FLAG(filter).c_str(); // Prints the filter if it's not *. This reminds the user that some // tests may be skipped. if (!String::CStringEquals(filter, kUniversalFilter)) { - ColoredPrintf(COLOR_YELLOW, - "Note: %s filter = %s\n", GTEST_NAME_, filter); + ColoredPrintf(GTestColor::kYellow, "Note: %s filter = %s\n", GTEST_NAME_, + filter); } if (internal::ShouldShard(kTestTotalShards, kTestShardIndex, false)) { const int32_t shard_index = Int32FromEnvOrDie(kTestShardIndex, -1); - ColoredPrintf(COLOR_YELLOW, - "Note: This is test shard %d of %s.\n", + ColoredPrintf(GTestColor::kYellow, "Note: This is test shard %d of %s.\n", static_cast(shard_index) + 1, internal::posix::GetEnv(kTestTotalShards)); } if (GTEST_FLAG(shuffle)) { - ColoredPrintf(COLOR_YELLOW, + ColoredPrintf(GTestColor::kYellow, "Note: Randomizing tests' orders with a seed of %d .\n", unit_test.random_seed()); } - ColoredPrintf(COLOR_GREEN, "[==========] "); + ColoredPrintf(GTestColor::kGreen, "[==========] "); printf("Running %s from %s.\n", FormatTestCount(unit_test.test_to_run_count()).c_str(), FormatTestSuiteCount(unit_test.test_suite_to_run_count()).c_str()); fflush(stdout); } void PrettyUnitTestResultPrinter::OnEnvironmentsSetUpStart( const UnitTest& /*unit_test*/) { - ColoredPrintf(COLOR_GREEN, "[----------] "); + ColoredPrintf(GTestColor::kGreen, "[----------] "); printf("Global test environment set-up.\n"); fflush(stdout); } #ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ void PrettyUnitTestResultPrinter::OnTestCaseStart(const TestCase& test_case) { const std::string counts = FormatCountableNoun(test_case.test_to_run_count(), "test", "tests"); - ColoredPrintf(COLOR_GREEN, "[----------] "); + ColoredPrintf(GTestColor::kGreen, "[----------] "); printf("%s from %s", counts.c_str(), test_case.name()); if (test_case.type_param() == nullptr) { printf("\n"); } else { printf(", where %s = %s\n", kTypeParamLabel, test_case.type_param()); @@ -3358,11 +3442,11 @@ #else void PrettyUnitTestResultPrinter::OnTestSuiteStart( const TestSuite& test_suite) { const std::string counts = FormatCountableNoun(test_suite.test_to_run_count(), "test", "tests"); - ColoredPrintf(COLOR_GREEN, "[----------] "); + ColoredPrintf(GTestColor::kGreen, "[----------] "); printf("%s from %s", counts.c_str(), test_suite.name()); if (test_suite.type_param() == nullptr) { printf("\n"); } else { printf(", where %s = %s\n", kTypeParamLabel, test_suite.type_param()); @@ -3370,11 +3454,11 @@ fflush(stdout); } #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ void PrettyUnitTestResultPrinter::OnTestStart(const TestInfo& test_info) { - ColoredPrintf(COLOR_GREEN, "[ RUN ] "); + ColoredPrintf(GTestColor::kGreen, "[ RUN ] "); PrintTestName(test_info.test_suite_name(), test_info.name()); printf("\n"); fflush(stdout); } @@ -3393,15 +3477,15 @@ } } void PrettyUnitTestResultPrinter::OnTestEnd(const TestInfo& test_info) { if (test_info.result()->Passed()) { - ColoredPrintf(COLOR_GREEN, "[ OK ] "); + ColoredPrintf(GTestColor::kGreen, "[ OK ] "); } else if (test_info.result()->Skipped()) { - ColoredPrintf(COLOR_GREEN, "[ SKIPPED ] "); + ColoredPrintf(GTestColor::kGreen, "[ SKIPPED ] "); } else { - ColoredPrintf(COLOR_RED, "[ FAILED ] "); + ColoredPrintf(GTestColor::kRed, "[ FAILED ] "); } PrintTestName(test_info.test_suite_name(), test_info.name()); if (test_info.result()->Failed()) PrintFullTestCommentIfPresent(test_info); @@ -3418,39 +3502,39 @@ void PrettyUnitTestResultPrinter::OnTestCaseEnd(const TestCase& test_case) { if (!GTEST_FLAG(print_time)) return; const std::string counts = FormatCountableNoun(test_case.test_to_run_count(), "test", "tests"); - ColoredPrintf(COLOR_GREEN, "[----------] "); + ColoredPrintf(GTestColor::kGreen, "[----------] "); printf("%s from %s (%s ms total)\n\n", counts.c_str(), test_case.name(), internal::StreamableToString(test_case.elapsed_time()).c_str()); fflush(stdout); } #else void PrettyUnitTestResultPrinter::OnTestSuiteEnd(const TestSuite& test_suite) { if (!GTEST_FLAG(print_time)) return; const std::string counts = FormatCountableNoun(test_suite.test_to_run_count(), "test", "tests"); - ColoredPrintf(COLOR_GREEN, "[----------] "); + ColoredPrintf(GTestColor::kGreen, "[----------] "); printf("%s from %s (%s ms total)\n\n", counts.c_str(), test_suite.name(), internal::StreamableToString(test_suite.elapsed_time()).c_str()); fflush(stdout); } #endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ void PrettyUnitTestResultPrinter::OnEnvironmentsTearDownStart( const UnitTest& /*unit_test*/) { - ColoredPrintf(COLOR_GREEN, "[----------] "); + ColoredPrintf(GTestColor::kGreen, "[----------] "); printf("Global test environment tear-down\n"); fflush(stdout); } // Internal helper for printing the list of failed tests. void PrettyUnitTestResultPrinter::PrintFailedTests(const UnitTest& unit_test) { const int failed_test_count = unit_test.failed_test_count(); - ColoredPrintf(COLOR_RED, "[ FAILED ] "); + ColoredPrintf(GTestColor::kRed, "[ FAILED ] "); printf("%s, listed below:\n", FormatTestCount(failed_test_count).c_str()); for (int i = 0; i < unit_test.total_test_suite_count(); ++i) { const TestSuite& test_suite = *unit_test.GetTestSuite(i); if (!test_suite.should_run() || (test_suite.failed_test_count() == 0)) { @@ -3459,11 +3543,11 @@ for (int j = 0; j < test_suite.total_test_count(); ++j) { const TestInfo& test_info = *test_suite.GetTestInfo(j); if (!test_info.should_run() || !test_info.result()->Failed()) { continue; } - ColoredPrintf(COLOR_RED, "[ FAILED ] "); + ColoredPrintf(GTestColor::kRed, "[ FAILED ] "); printf("%s.%s", test_suite.name(), test_info.name()); PrintFullTestCommentIfPresent(test_info); printf("\n"); } } @@ -3480,11 +3564,11 @@ const TestSuite& test_suite = *unit_test.GetTestSuite(i); if (!test_suite.should_run()) { continue; } if (test_suite.ad_hoc_test_result().Failed()) { - ColoredPrintf(COLOR_RED, "[ FAILED ] "); + ColoredPrintf(GTestColor::kRed, "[ FAILED ] "); printf("%s: SetUpTestSuite or TearDownTestSuite\n", test_suite.name()); ++suite_failure_count; } } if (suite_failure_count > 0) { @@ -3508,34 +3592,34 @@ for (int j = 0; j < test_suite.total_test_count(); ++j) { const TestInfo& test_info = *test_suite.GetTestInfo(j); if (!test_info.should_run() || !test_info.result()->Skipped()) { continue; } - ColoredPrintf(COLOR_GREEN, "[ SKIPPED ] "); + ColoredPrintf(GTestColor::kGreen, "[ SKIPPED ] "); printf("%s.%s", test_suite.name(), test_info.name()); printf("\n"); } } } void PrettyUnitTestResultPrinter::OnTestIterationEnd(const UnitTest& unit_test, int /*iteration*/) { - ColoredPrintf(COLOR_GREEN, "[==========] "); + ColoredPrintf(GTestColor::kGreen, "[==========] "); printf("%s from %s ran.", FormatTestCount(unit_test.test_to_run_count()).c_str(), FormatTestSuiteCount(unit_test.test_suite_to_run_count()).c_str()); if (GTEST_FLAG(print_time)) { printf(" (%s ms total)", internal::StreamableToString(unit_test.elapsed_time()).c_str()); } printf("\n"); - ColoredPrintf(COLOR_GREEN, "[ PASSED ] "); + ColoredPrintf(GTestColor::kGreen, "[ PASSED ] "); printf("%s.\n", FormatTestCount(unit_test.successful_test_count()).c_str()); const int skipped_test_count = unit_test.skipped_test_count(); if (skipped_test_count > 0) { - ColoredPrintf(COLOR_GREEN, "[ SKIPPED ] "); + ColoredPrintf(GTestColor::kGreen, "[ SKIPPED ] "); printf("%s, listed below:\n", FormatTestCount(skipped_test_count).c_str()); PrintSkippedTests(unit_test); } if (!unit_test.Passed()) { @@ -3546,21 +3630,123 @@ int num_disabled = unit_test.reportable_disabled_test_count(); if (num_disabled && !GTEST_FLAG(also_run_disabled_tests)) { if (unit_test.Passed()) { printf("\n"); // Add a spacer if no FAILURE banner is displayed. } - ColoredPrintf(COLOR_YELLOW, - " YOU HAVE %d DISABLED %s\n\n", - num_disabled, - num_disabled == 1 ? "TEST" : "TESTS"); + ColoredPrintf(GTestColor::kYellow, " YOU HAVE %d DISABLED %s\n\n", + num_disabled, num_disabled == 1 ? "TEST" : "TESTS"); } // Ensure that Google Test output is printed before, e.g., heapchecker output. fflush(stdout); } // End PrettyUnitTestResultPrinter +// This class implements the TestEventListener interface. +// +// Class BriefUnitTestResultPrinter is copyable. +class BriefUnitTestResultPrinter : public TestEventListener { + public: + BriefUnitTestResultPrinter() {} + static void PrintTestName(const char* test_suite, const char* test) { + printf("%s.%s", test_suite, test); + } + + // The following methods override what's in the TestEventListener class. + void OnTestProgramStart(const UnitTest& /*unit_test*/) override {} + void OnTestIterationStart(const UnitTest& /*unit_test*/, + int /*iteration*/) override {} + void OnEnvironmentsSetUpStart(const UnitTest& /*unit_test*/) override {} + void OnEnvironmentsSetUpEnd(const UnitTest& /*unit_test*/) override {} +#ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ + void OnTestCaseStart(const TestCase& /*test_case*/) override {} +#else + void OnTestSuiteStart(const TestSuite& /*test_suite*/) override {} +#endif // OnTestCaseStart + + void OnTestStart(const TestInfo& /*test_info*/) override {} + + void OnTestPartResult(const TestPartResult& result) override; + void OnTestEnd(const TestInfo& test_info) override; +#ifndef GTEST_REMOVE_LEGACY_TEST_CASEAPI_ + void OnTestCaseEnd(const TestCase& /*test_case*/) override {} +#else + void OnTestSuiteEnd(const TestSuite& /*test_suite*/) override {} +#endif // GTEST_REMOVE_LEGACY_TEST_CASEAPI_ + + void OnEnvironmentsTearDownStart(const UnitTest& /*unit_test*/) override {} + void OnEnvironmentsTearDownEnd(const UnitTest& /*unit_test*/) override {} + void OnTestIterationEnd(const UnitTest& unit_test, int iteration) override; + void OnTestProgramEnd(const UnitTest& /*unit_test*/) override {} +}; + +// Called after an assertion failure. +void BriefUnitTestResultPrinter::OnTestPartResult( + const TestPartResult& result) { + switch (result.type()) { + // If the test part succeeded, we don't need to do anything. + case TestPartResult::kSuccess: + return; + default: + // Print failure message from the assertion + // (e.g. expected this and got that). + PrintTestPartResult(result); + fflush(stdout); + } +} + +void BriefUnitTestResultPrinter::OnTestEnd(const TestInfo& test_info) { + if (test_info.result()->Failed()) { + ColoredPrintf(GTestColor::kRed, "[ FAILED ] "); + PrintTestName(test_info.test_suite_name(), test_info.name()); + PrintFullTestCommentIfPresent(test_info); + + if (GTEST_FLAG(print_time)) { + printf(" (%s ms)\n", + internal::StreamableToString(test_info.result()->elapsed_time()) + .c_str()); + } else { + printf("\n"); + } + fflush(stdout); + } +} + +void BriefUnitTestResultPrinter::OnTestIterationEnd(const UnitTest& unit_test, + int /*iteration*/) { + ColoredPrintf(GTestColor::kGreen, "[==========] "); + printf("%s from %s ran.", + FormatTestCount(unit_test.test_to_run_count()).c_str(), + FormatTestSuiteCount(unit_test.test_suite_to_run_count()).c_str()); + if (GTEST_FLAG(print_time)) { + printf(" (%s ms total)", + internal::StreamableToString(unit_test.elapsed_time()).c_str()); + } + printf("\n"); + ColoredPrintf(GTestColor::kGreen, "[ PASSED ] "); + printf("%s.\n", FormatTestCount(unit_test.successful_test_count()).c_str()); + + const int skipped_test_count = unit_test.skipped_test_count(); + if (skipped_test_count > 0) { + ColoredPrintf(GTestColor::kGreen, "[ SKIPPED ] "); + printf("%s.\n", FormatTestCount(skipped_test_count).c_str()); + } + + int num_disabled = unit_test.reportable_disabled_test_count(); + if (num_disabled && !GTEST_FLAG(also_run_disabled_tests)) { + if (unit_test.Passed()) { + printf("\n"); // Add a spacer if no FAILURE banner is displayed. + } + ColoredPrintf(GTestColor::kYellow, " YOU HAVE %d DISABLED %s\n\n", + num_disabled, num_disabled == 1 ? "TEST" : "TESTS"); + } + // Ensure that Google Test output is printed before, e.g., heapchecker output. + fflush(stdout); +} + +// End BriefUnitTestResultPrinter + // class TestEventRepeater // // This class forwards events to other event listeners. class TestEventRepeater : public TestEventListener { public: @@ -3740,10 +3926,20 @@ const std::string& value); // Streams an XML CDATA section, escaping invalid CDATA sequences as needed. static void OutputXmlCDataSection(::std::ostream* stream, const char* data); + // Streams a test suite XML stanza containing the given test result. + // + // Requires: result.Failed() + static void OutputXmlTestSuiteForTestResult(::std::ostream* stream, + const TestResult& result); + + // Streams an XML representation of a TestResult object. + static void OutputXmlTestResult(::std::ostream* stream, + const TestResult& result); + // Streams an XML representation of a TestInfo object. static void OutputXmlTestInfo(::std::ostream* stream, const char* test_suite_name, const TestInfo& test_info); @@ -3898,10 +4094,14 @@ // Windows' localtime(), which has a thread-local tm buffer. struct tm* tm_ptr = localtime(&seconds); // NOLINT if (tm_ptr == nullptr) return false; *out = *tm_ptr; return true; +#elif defined(__STDC_LIB_EXT1__) + // Uses localtime_s when available as localtime_r is only available from + // C23 standard. + return localtime_s(&seconds, out) != nullptr; #else return localtime_r(&seconds, out) != nullptr; #endif } @@ -3909,17 +4109,18 @@ // 8601 format, without the timezone information. std::string FormatEpochTimeInMillisAsIso8601(TimeInMillis ms) { struct tm time_struct; if (!PortableLocaltime(static_cast(ms / 1000), &time_struct)) return ""; - // YYYY-MM-DDThh:mm:ss + // YYYY-MM-DDThh:mm:ss.sss return StreamableToString(time_struct.tm_year + 1900) + "-" + String::FormatIntWidth2(time_struct.tm_mon + 1) + "-" + String::FormatIntWidth2(time_struct.tm_mday) + "T" + String::FormatIntWidth2(time_struct.tm_hour) + ":" + String::FormatIntWidth2(time_struct.tm_min) + ":" + - String::FormatIntWidth2(time_struct.tm_sec); + String::FormatIntWidth2(time_struct.tm_sec) + "." + + String::FormatIntWidthN(static_cast(ms % 1000), 3); } // Streams an XML CDATA section, escaping invalid CDATA sequences as needed. void XmlUnitTestResultPrinter::OutputXmlCDataSection(::std::ostream* stream, const char* data) { @@ -3954,10 +4155,47 @@ << ">."; *stream << " " << name << "=\"" << EscapeXmlAttribute(value) << "\""; } +// Streams a test suite XML stanza containing the given test result. +void XmlUnitTestResultPrinter::OutputXmlTestSuiteForTestResult( + ::std::ostream* stream, const TestResult& result) { + // Output the boilerplate for a minimal test suite with one test. + *stream << " "; + + // Output the boilerplate for a minimal test case with a single test. + *stream << " \n"; +} + // Prints an XML representation of a TestInfo object. void XmlUnitTestResultPrinter::OutputXmlTestInfo(::std::ostream* stream, const char* test_suite_name, const TestInfo& test_info) { const TestResult& result = *test_info.result(); @@ -3997,34 +4235,53 @@ OutputXmlAttribute( stream, kTestsuite, "timestamp", FormatEpochTimeInMillisAsIso8601(result.start_timestamp())); OutputXmlAttribute(stream, kTestsuite, "classname", test_suite_name); + OutputXmlTestResult(stream, result); +} + +void XmlUnitTestResultPrinter::OutputXmlTestResult(::std::ostream* stream, + const TestResult& result) { int failures = 0; + int skips = 0; for (int i = 0; i < result.total_part_count(); ++i) { const TestPartResult& part = result.GetTestPartResult(i); if (part.failed()) { - if (++failures == 1) { + if (++failures == 1 && skips == 0) { *stream << ">\n"; } const std::string location = internal::FormatCompilerIndependentFileLocation(part.file_name(), part.line_number()); const std::string summary = location + "\n" + part.summary(); *stream << " "; const std::string detail = location + "\n" + part.message(); OutputXmlCDataSection(stream, RemoveInvalidXmlCharacters(detail).c_str()); *stream << "\n"; + } else if (part.skipped()) { + if (++skips == 1 && failures == 0) { + *stream << ">\n"; + } + const std::string location = + internal::FormatCompilerIndependentFileLocation(part.file_name(), + part.line_number()); + const std::string summary = location + "\n" + part.summary(); + *stream << " "; + const std::string detail = location + "\n" + part.message(); + OutputXmlCDataSection(stream, RemoveInvalidXmlCharacters(detail).c_str()); + *stream << "\n"; } } - if (failures == 0 && result.test_property_count() == 0) { + if (failures == 0 && skips == 0 && result.test_property_count() == 0) { *stream << " />\n"; } else { - if (failures == 0) { + if (failures == 0 && skips == 0) { *stream << ">\n"; } OutputXmlTestProperties(stream, result); *stream << " \n"; } @@ -4042,11 +4299,15 @@ OutputXmlAttribute(stream, kTestsuite, "failures", StreamableToString(test_suite.failed_test_count())); OutputXmlAttribute( stream, kTestsuite, "disabled", StreamableToString(test_suite.reportable_disabled_test_count())); + OutputXmlAttribute(stream, kTestsuite, "skipped", + StreamableToString(test_suite.skipped_test_count())); + OutputXmlAttribute(stream, kTestsuite, "errors", "0"); + OutputXmlAttribute(stream, kTestsuite, "time", FormatTimeInMillisAsSeconds(test_suite.elapsed_time())); OutputXmlAttribute( stream, kTestsuite, "timestamp", FormatEpochTimeInMillisAsIso8601(test_suite.start_timestamp())); @@ -4093,10 +4354,17 @@ for (int i = 0; i < unit_test.total_test_suite_count(); ++i) { if (unit_test.GetTestSuite(i)->reportable_test_count() > 0) PrintXmlTestSuite(stream, *unit_test.GetTestSuite(i)); } + + // If there was a test failure outside of one of the test suites (like in a + // test environment) include that in the output. + if (unit_test.ad_hoc_test_result().Failed()) { + OutputXmlTestSuiteForTestResult(stream, unit_test.ad_hoc_test_result()); + } + *stream << "\n"; } void XmlUnitTestResultPrinter::PrintXmlTestsList( std::ostream* stream, const std::vector& test_suites) { @@ -4183,10 +4451,20 @@ const std::string& name, int value, const std::string& indent, bool comma = true); + // Streams a test suite JSON stanza containing the given test result. + // + // Requires: result.Failed() + static void OutputJsonTestSuiteForTestResult(::std::ostream* stream, + const TestResult& result); + + // Streams a JSON representation of a TestResult object. + static void OutputJsonTestResult(::std::ostream* stream, + const TestResult& result); + // Streams a JSON representation of a TestInfo object. static void OutputJsonTestInfo(::std::ostream* stream, const char* test_suite_name, const TestInfo& test_info); @@ -4333,10 +4611,52 @@ *stream << indent << "\"" << name << "\": " << StreamableToString(value); if (comma) *stream << ",\n"; } +// Streams a test suite JSON stanza containing the given test result. +void JsonUnitTestResultPrinter::OutputJsonTestSuiteForTestResult( + ::std::ostream* stream, const TestResult& result) { + // Output the boilerplate for a new test suite. + *stream << Indent(4) << "{\n"; + OutputJsonKey(stream, "testsuite", "name", "NonTestSuiteFailure", Indent(6)); + OutputJsonKey(stream, "testsuite", "tests", 1, Indent(6)); + if (!GTEST_FLAG(list_tests)) { + OutputJsonKey(stream, "testsuite", "failures", 1, Indent(6)); + OutputJsonKey(stream, "testsuite", "disabled", 0, Indent(6)); + OutputJsonKey(stream, "testsuite", "skipped", 0, Indent(6)); + OutputJsonKey(stream, "testsuite", "errors", 0, Indent(6)); + OutputJsonKey(stream, "testsuite", "time", + FormatTimeInMillisAsDuration(result.elapsed_time()), + Indent(6)); + OutputJsonKey(stream, "testsuite", "timestamp", + FormatEpochTimeInMillisAsRFC3339(result.start_timestamp()), + Indent(6)); + } + *stream << Indent(6) << "\"testsuite\": [\n"; + + // Output the boilerplate for a new test case. + *stream << Indent(8) << "{\n"; + OutputJsonKey(stream, "testcase", "name", "", Indent(10)); + OutputJsonKey(stream, "testcase", "status", "RUN", Indent(10)); + OutputJsonKey(stream, "testcase", "result", "COMPLETED", Indent(10)); + OutputJsonKey(stream, "testcase", "timestamp", + FormatEpochTimeInMillisAsRFC3339(result.start_timestamp()), + Indent(10)); + OutputJsonKey(stream, "testcase", "time", + FormatTimeInMillisAsDuration(result.elapsed_time()), + Indent(10)); + OutputJsonKey(stream, "testcase", "classname", "", Indent(10), false); + *stream << TestPropertiesAsJson(result, Indent(10)); + + // Output the actual test result. + OutputJsonTestResult(stream, result); + + // Finish the test suite. + *stream << "\n" << Indent(6) << "]\n" << Indent(4) << "}"; +} + // Prints a JSON representation of a TestInfo object. void JsonUnitTestResultPrinter::OutputJsonTestInfo(::std::ostream* stream, const char* test_suite_name, const TestInfo& test_info) { const TestResult& result = *test_info.result(); @@ -4375,10 +4695,17 @@ FormatTimeInMillisAsDuration(result.elapsed_time()), kIndent); OutputJsonKey(stream, kTestsuite, "classname", test_suite_name, kIndent, false); *stream << TestPropertiesAsJson(result, kIndent); + OutputJsonTestResult(stream, result); +} + +void JsonUnitTestResultPrinter::OutputJsonTestResult(::std::ostream* stream, + const TestResult& result) { + const std::string kIndent = Indent(10); + int failures = 0; for (int i = 0; i < result.total_part_count(); ++i) { const TestPartResult& part = result.GetTestPartResult(i); if (part.failed()) { *stream << ",\n"; @@ -4485,10 +4812,16 @@ } PrintJsonTestSuite(stream, *unit_test.GetTestSuite(i)); } } + // If there was a test failure outside of one of the test suites (like in a + // test environment) include that in the output. + if (unit_test.ad_hoc_test_result().Failed()) { + OutputJsonTestSuiteForTestResult(stream, unit_test.ad_hoc_test_result()); + } + *stream << "\n" << kIndent << "]\n" << "}\n"; } void JsonUnitTestResultPrinter::PrintJsonTestList( std::ostream* stream, const std::vector& test_suites) { @@ -5307,10 +5640,14 @@ // Configures listeners for XML output. This makes it possible for users // to shut down the default XML output before invoking RUN_ALL_TESTS. ConfigureXmlOutput(); + if (GTEST_FLAG(brief)) { + listeners()->SetDefaultResultPrinter(new BriefUnitTestResultPrinter); + } + #if GTEST_CAN_STREAM_RESULTS_ // Configures listeners for streaming test results to the specified server. ConfigureStreamingOutput(); #endif // GTEST_CAN_STREAM_RESULTS_ @@ -5352,14 +5689,14 @@ // TESTS ARE NOT SHUFFLED. // // Arguments: // // test_suite_name: name of the test suite -// type_param: the name of the test suite's type parameter, or NULL if -// this is not a typed or a type-parameterized test suite. -// set_up_tc: pointer to the function that sets up the test suite -// tear_down_tc: pointer to the function that tears down the test suite +// type_param: the name of the test suite's type parameter, or NULL if +// this is not a typed or a type-parameterized test suite. +// set_up_tc: pointer to the function that sets up the test suite +// tear_down_tc: pointer to the function that tears down the test suite TestSuite* UnitTestImpl::GetTestSuite( const char* test_suite_name, const char* type_param, internal::SetUpTestSuiteFunc set_up_tc, internal::TearDownTestSuiteFunc tear_down_tc) { // Can we find a TestSuite with the given name? @@ -5473,11 +5810,11 @@ for (int i = 0; gtest_repeat_forever || i != repeat; i++) { // We want to preserve failures generated by ad-hoc test // assertions executed before RUN_ALL_TESTS(). ClearNonAdHocTestResult(); - const TimeInMillis start = GetTimeInMillis(); + Timer timer; // Shuffles test suites and tests if requested. if (has_tests_to_run && GTEST_FLAG(shuffle)) { random()->Reseed(static_cast(random_seed_)); // This should be done before calling OnTestIterationStart(), @@ -5514,21 +5851,36 @@ fflush(stdout); } else if (!Test::HasFatalFailure()) { for (int test_index = 0; test_index < total_test_suite_count(); test_index++) { GetMutableSuiteCase(test_index)->Run(); + if (GTEST_FLAG(fail_fast) && + GetMutableSuiteCase(test_index)->Failed()) { + for (int j = test_index + 1; j < total_test_suite_count(); j++) { + GetMutableSuiteCase(j)->Skip(); + } + break; + } + } + } else if (Test::HasFatalFailure()) { + // If there was a fatal failure during the global setup then we know we + // aren't going to run any tests. Explicitly mark all of the tests as + // skipped to make this obvious in the output. + for (int test_index = 0; test_index < total_test_suite_count(); + test_index++) { + GetMutableSuiteCase(test_index)->Skip(); } } // Tears down all environments in reverse order afterwards. repeater->OnEnvironmentsTearDownStart(*parent_); std::for_each(environments_.rbegin(), environments_.rend(), TearDownEnvironment); repeater->OnEnvironmentsTearDownEnd(*parent_); } - elapsed_time_ = GetTimeInMillis() - start; + elapsed_time_ = timer.Elapsed(); // Tells the unit test event listener that the tests have just finished. repeater->OnTestIterationEnd(*parent_, i); // Gets the result and clears it. @@ -5552,18 +5904,18 @@ repeater->OnTestProgramEnd(*parent_); if (!gtest_is_initialized_before_run_all_tests) { ColoredPrintf( - COLOR_RED, + GTestColor::kRed, "\nIMPORTANT NOTICE - DO NOT IGNORE:\n" "This test program did NOT call " GTEST_INIT_GOOGLE_TEST_NAME_ "() before calling RUN_ALL_TESTS(). This is INVALID. Soon " GTEST_NAME_ " will start to enforce the valid usage. " "Please fix it ASAP, or IT WILL START TO FAIL.\n"); // NOLINT #if GTEST_FOR_GOOGLE_ - ColoredPrintf(COLOR_RED, + ColoredPrintf(GTestColor::kRed, "For more details, see http://wiki/Main/ValidGUnitMain.\n"); #endif // GTEST_FOR_GOOGLE_ } return !failed; @@ -5576,11 +5928,11 @@ void WriteToShardStatusFileIfNeeded() { const char* const test_shard_file = posix::GetEnv(kTestShardStatusFile); if (test_shard_file != nullptr) { FILE* const file = posix::FOpen(test_shard_file, "w"); if (file == nullptr) { - ColoredPrintf(COLOR_RED, + ColoredPrintf(GTestColor::kRed, "Could not write to the test shard status file \"%s\" " "specified by the %s environment variable.\n", test_shard_file, kTestShardStatusFile); fflush(stdout); exit(EXIT_FAILURE); @@ -5610,28 +5962,28 @@ } else if (total_shards == -1 && shard_index != -1) { const Message msg = Message() << "Invalid environment variables: you have " << kTestShardIndex << " = " << shard_index << ", but have left " << kTestTotalShards << " unset.\n"; - ColoredPrintf(COLOR_RED, "%s", msg.GetString().c_str()); + ColoredPrintf(GTestColor::kRed, "%s", msg.GetString().c_str()); fflush(stdout); exit(EXIT_FAILURE); } else if (total_shards != -1 && shard_index == -1) { const Message msg = Message() << "Invalid environment variables: you have " << kTestTotalShards << " = " << total_shards << ", but have left " << kTestShardIndex << " unset.\n"; - ColoredPrintf(COLOR_RED, "%s", msg.GetString().c_str()); + ColoredPrintf(GTestColor::kRed, "%s", msg.GetString().c_str()); fflush(stdout); exit(EXIT_FAILURE); } else if (shard_index < 0 || shard_index >= total_shards) { const Message msg = Message() << "Invalid environment variables: we require 0 <= " << kTestShardIndex << " < " << kTestTotalShards << ", but you have " << kTestShardIndex << "=" << shard_index << ", " << kTestTotalShards << "=" << total_shards << ".\n"; - ColoredPrintf(COLOR_RED, "%s", msg.GetString().c_str()); + ColoredPrintf(GTestColor::kRed, "%s", msg.GetString().c_str()); fflush(stdout); exit(EXIT_FAILURE); } return total_shards > 1; @@ -6017,11 +6369,11 @@ // @G changes the color to green. // @Y changes the color to yellow. // @D changes to the default terminal text color. // static void PrintColorEncoded(const char* str) { - GTestColor color = COLOR_DEFAULT; // The current color. + GTestColor color = GTestColor::kDefault; // The current color. // Conceptually, we split the string into segments divided by escape // sequences. Then we print one segment at a time. At the end of // each iteration, the str pointer advances to the beginning of the // next segment. @@ -6037,116 +6389,144 @@ const char ch = p[1]; str = p + 2; if (ch == '@') { ColoredPrintf(color, "@"); } else if (ch == 'D') { - color = COLOR_DEFAULT; + color = GTestColor::kDefault; } else if (ch == 'R') { - color = COLOR_RED; + color = GTestColor::kRed; } else if (ch == 'G') { - color = COLOR_GREEN; + color = GTestColor::kGreen; } else if (ch == 'Y') { - color = COLOR_YELLOW; + color = GTestColor::kYellow; } else { --str; } } } static const char kColorEncodedHelpMessage[] = -"This program contains tests written using " GTEST_NAME_ ". You can use the\n" -"following command line flags to control its behavior:\n" -"\n" -"Test Selection:\n" -" @G--" GTEST_FLAG_PREFIX_ "list_tests@D\n" -" List the names of all tests instead of running them. The name of\n" -" TEST(Foo, Bar) is \"Foo.Bar\".\n" -" @G--" GTEST_FLAG_PREFIX_ "filter=@YPOSTIVE_PATTERNS" + "This program contains tests written using " GTEST_NAME_ + ". You can use the\n" + "following command line flags to control its behavior:\n" + "\n" + "Test Selection:\n" + " @G--" GTEST_FLAG_PREFIX_ + "list_tests@D\n" + " List the names of all tests instead of running them. The name of\n" + " TEST(Foo, Bar) is \"Foo.Bar\".\n" + " @G--" GTEST_FLAG_PREFIX_ + "filter=@YPOSITIVE_PATTERNS" "[@G-@YNEGATIVE_PATTERNS]@D\n" -" Run only the tests whose name matches one of the positive patterns but\n" -" none of the negative patterns. '?' matches any single character; '*'\n" -" matches any substring; ':' separates two patterns.\n" -" @G--" GTEST_FLAG_PREFIX_ "also_run_disabled_tests@D\n" -" Run all disabled tests too.\n" -"\n" -"Test Execution:\n" -" @G--" GTEST_FLAG_PREFIX_ "repeat=@Y[COUNT]@D\n" -" Run the tests repeatedly; use a negative count to repeat forever.\n" -" @G--" GTEST_FLAG_PREFIX_ "shuffle@D\n" -" Randomize tests' orders on every iteration.\n" -" @G--" GTEST_FLAG_PREFIX_ "random_seed=@Y[NUMBER]@D\n" -" Random number seed to use for shuffling test orders (between 1 and\n" -" 99999, or 0 to use a seed based on the current time).\n" -"\n" -"Test Output:\n" -" @G--" GTEST_FLAG_PREFIX_ "color=@Y(@Gyes@Y|@Gno@Y|@Gauto@Y)@D\n" -" Enable/disable colored output. The default is @Gauto@D.\n" -" -@G-" GTEST_FLAG_PREFIX_ "print_time=0@D\n" -" Don't print the elapsed time of each test.\n" -" @G--" GTEST_FLAG_PREFIX_ "output=@Y(@Gjson@Y|@Gxml@Y)[@G:@YDIRECTORY_PATH@G" - GTEST_PATH_SEP_ "@Y|@G:@YFILE_PATH]@D\n" -" Generate a JSON or XML report in the given directory or with the given\n" -" file name. @YFILE_PATH@D defaults to @Gtest_detail.xml@D.\n" + " Run only the tests whose name matches one of the positive patterns " + "but\n" + " none of the negative patterns. '?' matches any single character; " + "'*'\n" + " matches any substring; ':' separates two patterns.\n" + " @G--" GTEST_FLAG_PREFIX_ + "also_run_disabled_tests@D\n" + " Run all disabled tests too.\n" + "\n" + "Test Execution:\n" + " @G--" GTEST_FLAG_PREFIX_ + "repeat=@Y[COUNT]@D\n" + " Run the tests repeatedly; use a negative count to repeat forever.\n" + " @G--" GTEST_FLAG_PREFIX_ + "shuffle@D\n" + " Randomize tests' orders on every iteration.\n" + " @G--" GTEST_FLAG_PREFIX_ + "random_seed=@Y[NUMBER]@D\n" + " Random number seed to use for shuffling test orders (between 1 and\n" + " 99999, or 0 to use a seed based on the current time).\n" + "\n" + "Test Output:\n" + " @G--" GTEST_FLAG_PREFIX_ + "color=@Y(@Gyes@Y|@Gno@Y|@Gauto@Y)@D\n" + " Enable/disable colored output. The default is @Gauto@D.\n" + " @G--" GTEST_FLAG_PREFIX_ + "brief=1@D\n" + " Only print test failures.\n" + " @G--" GTEST_FLAG_PREFIX_ + "print_time=0@D\n" + " Don't print the elapsed time of each test.\n" + " @G--" GTEST_FLAG_PREFIX_ + "output=@Y(@Gjson@Y|@Gxml@Y)[@G:@YDIRECTORY_PATH@G" GTEST_PATH_SEP_ + "@Y|@G:@YFILE_PATH]@D\n" + " Generate a JSON or XML report in the given directory or with the " + "given\n" + " file name. @YFILE_PATH@D defaults to @Gtest_detail.xml@D.\n" # if GTEST_CAN_STREAM_RESULTS_ -" @G--" GTEST_FLAG_PREFIX_ "stream_result_to=@YHOST@G:@YPORT@D\n" -" Stream test results to the given server.\n" + " @G--" GTEST_FLAG_PREFIX_ + "stream_result_to=@YHOST@G:@YPORT@D\n" + " Stream test results to the given server.\n" # endif // GTEST_CAN_STREAM_RESULTS_ -"\n" -"Assertion Behavior:\n" + "\n" + "Assertion Behavior:\n" # if GTEST_HAS_DEATH_TEST && !GTEST_OS_WINDOWS -" @G--" GTEST_FLAG_PREFIX_ "death_test_style=@Y(@Gfast@Y|@Gthreadsafe@Y)@D\n" -" Set the default death test style.\n" + " @G--" GTEST_FLAG_PREFIX_ + "death_test_style=@Y(@Gfast@Y|@Gthreadsafe@Y)@D\n" + " Set the default death test style.\n" # endif // GTEST_HAS_DEATH_TEST && !GTEST_OS_WINDOWS -" @G--" GTEST_FLAG_PREFIX_ "break_on_failure@D\n" -" Turn assertion failures into debugger break-points.\n" -" @G--" GTEST_FLAG_PREFIX_ "throw_on_failure@D\n" -" Turn assertion failures into C++ exceptions for use by an external\n" -" test framework.\n" -" @G--" GTEST_FLAG_PREFIX_ "catch_exceptions=0@D\n" -" Do not report exceptions as test failures. Instead, allow them\n" -" to crash the program or throw a pop-up (on Windows).\n" -"\n" -"Except for @G--" GTEST_FLAG_PREFIX_ "list_tests@D, you can alternatively set " + " @G--" GTEST_FLAG_PREFIX_ + "break_on_failure@D\n" + " Turn assertion failures into debugger break-points.\n" + " @G--" GTEST_FLAG_PREFIX_ + "throw_on_failure@D\n" + " Turn assertion failures into C++ exceptions for use by an external\n" + " test framework.\n" + " @G--" GTEST_FLAG_PREFIX_ + "catch_exceptions=0@D\n" + " Do not report exceptions as test failures. Instead, allow them\n" + " to crash the program or throw a pop-up (on Windows).\n" + "\n" + "Except for @G--" GTEST_FLAG_PREFIX_ + "list_tests@D, you can alternatively set " "the corresponding\n" -"environment variable of a flag (all letters in upper-case). For example, to\n" -"disable colored text output, you can either specify @G--" GTEST_FLAG_PREFIX_ + "environment variable of a flag (all letters in upper-case). For example, " + "to\n" + "disable colored text output, you can either specify " + "@G--" GTEST_FLAG_PREFIX_ "color=no@D or set\n" -"the @G" GTEST_FLAG_PREFIX_UPPER_ "COLOR@D environment variable to @Gno@D.\n" -"\n" -"For more information, please read the " GTEST_NAME_ " documentation at\n" -"@G" GTEST_PROJECT_URL_ "@D. If you find a bug in " GTEST_NAME_ "\n" -"(not one in your own code or tests), please report it to\n" -"@G<" GTEST_DEV_EMAIL_ ">@D.\n"; + "the @G" GTEST_FLAG_PREFIX_UPPER_ + "COLOR@D environment variable to @Gno@D.\n" + "\n" + "For more information, please read the " GTEST_NAME_ + " documentation at\n" + "@G" GTEST_PROJECT_URL_ "@D. If you find a bug in " GTEST_NAME_ + "\n" + "(not one in your own code or tests), please report it to\n" + "@G<" GTEST_DEV_EMAIL_ ">@D.\n"; static bool ParseGoogleTestFlag(const char* const arg) { return ParseBoolFlag(arg, kAlsoRunDisabledTestsFlag, >EST_FLAG(also_run_disabled_tests)) || - ParseBoolFlag(arg, kBreakOnFailureFlag, - >EST_FLAG(break_on_failure)) || - ParseBoolFlag(arg, kCatchExceptionsFlag, - >EST_FLAG(catch_exceptions)) || - ParseStringFlag(arg, kColorFlag, >EST_FLAG(color)) || - ParseStringFlag(arg, kDeathTestStyleFlag, - >EST_FLAG(death_test_style)) || - ParseBoolFlag(arg, kDeathTestUseFork, - >EST_FLAG(death_test_use_fork)) || - ParseStringFlag(arg, kFilterFlag, >EST_FLAG(filter)) || - ParseStringFlag(arg, kInternalRunDeathTestFlag, - >EST_FLAG(internal_run_death_test)) || - ParseBoolFlag(arg, kListTestsFlag, >EST_FLAG(list_tests)) || - ParseStringFlag(arg, kOutputFlag, >EST_FLAG(output)) || - ParseBoolFlag(arg, kPrintTimeFlag, >EST_FLAG(print_time)) || - ParseBoolFlag(arg, kPrintUTF8Flag, >EST_FLAG(print_utf8)) || - ParseInt32Flag(arg, kRandomSeedFlag, >EST_FLAG(random_seed)) || - ParseInt32Flag(arg, kRepeatFlag, >EST_FLAG(repeat)) || - ParseBoolFlag(arg, kShuffleFlag, >EST_FLAG(shuffle)) || - ParseInt32Flag(arg, kStackTraceDepthFlag, - >EST_FLAG(stack_trace_depth)) || - ParseStringFlag(arg, kStreamResultToFlag, - >EST_FLAG(stream_result_to)) || - ParseBoolFlag(arg, kThrowOnFailureFlag, - >EST_FLAG(throw_on_failure)); + ParseBoolFlag(arg, kBreakOnFailureFlag, + >EST_FLAG(break_on_failure)) || + ParseBoolFlag(arg, kCatchExceptionsFlag, + >EST_FLAG(catch_exceptions)) || + ParseStringFlag(arg, kColorFlag, >EST_FLAG(color)) || + ParseStringFlag(arg, kDeathTestStyleFlag, + >EST_FLAG(death_test_style)) || + ParseBoolFlag(arg, kDeathTestUseFork, + >EST_FLAG(death_test_use_fork)) || + ParseBoolFlag(arg, kFailFast, >EST_FLAG(fail_fast)) || + ParseStringFlag(arg, kFilterFlag, >EST_FLAG(filter)) || + ParseStringFlag(arg, kInternalRunDeathTestFlag, + >EST_FLAG(internal_run_death_test)) || + ParseBoolFlag(arg, kListTestsFlag, >EST_FLAG(list_tests)) || + ParseStringFlag(arg, kOutputFlag, >EST_FLAG(output)) || + ParseBoolFlag(arg, kBriefFlag, >EST_FLAG(brief)) || + ParseBoolFlag(arg, kPrintTimeFlag, >EST_FLAG(print_time)) || + ParseBoolFlag(arg, kPrintUTF8Flag, >EST_FLAG(print_utf8)) || + ParseInt32Flag(arg, kRandomSeedFlag, >EST_FLAG(random_seed)) || + ParseInt32Flag(arg, kRepeatFlag, >EST_FLAG(repeat)) || + ParseBoolFlag(arg, kShuffleFlag, >EST_FLAG(shuffle)) || + ParseInt32Flag(arg, kStackTraceDepthFlag, + >EST_FLAG(stack_trace_depth)) || + ParseStringFlag(arg, kStreamResultToFlag, + >EST_FLAG(stream_result_to)) || + ParseBoolFlag(arg, kThrowOnFailureFlag, >EST_FLAG(throw_on_failure)); } #if GTEST_USE_OWN_FLAGFILE_FLAG_ static void LoadFlagsFromFile(const std::string& path) { FILE* flagfile = posix::FOpen(path.c_str(), "r"); @@ -6312,28 +6692,35 @@ } std::string TempDir() { #if defined(GTEST_CUSTOM_TEMPDIR_FUNCTION_) return GTEST_CUSTOM_TEMPDIR_FUNCTION_(); -#endif - -#if GTEST_OS_WINDOWS_MOBILE +#elif GTEST_OS_WINDOWS_MOBILE return "\\temp\\"; #elif GTEST_OS_WINDOWS const char* temp_dir = internal::posix::GetEnv("TEMP"); - if (temp_dir == nullptr || temp_dir[0] == '\0') + if (temp_dir == nullptr || temp_dir[0] == '\0') { return "\\temp\\"; - else if (temp_dir[strlen(temp_dir) - 1] == '\\') + } else if (temp_dir[strlen(temp_dir) - 1] == '\\') { return temp_dir; - else + } else { return std::string(temp_dir) + "\\"; + } #elif GTEST_OS_LINUX_ANDROID const char* temp_dir = internal::posix::GetEnv("TEST_TMPDIR"); - if (temp_dir == nullptr || temp_dir[0] == '\0') + if (temp_dir == nullptr || temp_dir[0] == '\0') { return "/data/local/tmp/"; - else + } else { return temp_dir; + } +#elif GTEST_OS_LINUX + const char* temp_dir = internal::posix::GetEnv("TEST_TMPDIR"); + if (temp_dir == nullptr || temp_dir[0] == '\0') { + return "/tmp/"; + } else { + return temp_dir; + } #else return "/tmp/"; #endif // GTEST_OS_WINDOWS_MOBILE } diff --git a/media/libvpx/libvpx/tools_common.h b/media/libvpx/libvpx/tools_common.h --- a/media/libvpx/libvpx/tools_common.h +++ b/media/libvpx/libvpx/tools_common.h @@ -114,16 +114,28 @@ #define VPX_NO_RETURN __declspec(noreturn) #else #define VPX_NO_RETURN #endif +// Tells the compiler to perform `printf` format string checking if the +// compiler supports it; see the 'format' attribute in +// . +#define VPX_TOOLS_FORMAT_PRINTF(string_index, first_to_check) +#if defined(__has_attribute) +#if __has_attribute(format) +#undef VPX_TOOLS_FORMAT_PRINTF +#define VPX_TOOLS_FORMAT_PRINTF(string_index, first_to_check) \ + __attribute__((__format__(__printf__, string_index, first_to_check))) +#endif +#endif + /* Sets a stdio stream into binary mode */ FILE *set_binary_mode(FILE *stream); -VPX_NO_RETURN void die(const char *fmt, ...); -VPX_NO_RETURN void fatal(const char *fmt, ...); -void warn(const char *fmt, ...); +VPX_NO_RETURN void die(const char *fmt, ...) VPX_TOOLS_FORMAT_PRINTF(1, 2); +VPX_NO_RETURN void fatal(const char *fmt, ...) VPX_TOOLS_FORMAT_PRINTF(1, 2); +void warn(const char *fmt, ...) VPX_TOOLS_FORMAT_PRINTF(1, 2); VPX_NO_RETURN void die_codec(vpx_codec_ctx_t *ctx, const char *s); /* The tool including this file must define usage_exit() */ VPX_NO_RETURN void usage_exit(void); diff --git a/media/libvpx/libvpx/vp8/common/blockd.h b/media/libvpx/libvpx/vp8/common/blockd.h --- a/media/libvpx/libvpx/vp8/common/blockd.h +++ b/media/libvpx/libvpx/vp8/common/blockd.h @@ -56,11 +56,11 @@ } ENTROPY_CONTEXT_PLANES; extern const unsigned char vp8_block2left[25]; extern const unsigned char vp8_block2above[25]; -#define VP8_COMBINEENTROPYCONTEXTS(Dest, A, B) Dest = (A) + (B); +#define VP8_COMBINEENTROPYCONTEXTS(Dest, A, B) Dest = (A) + (B) typedef enum { KEY_FRAME = 0, INTER_FRAME = 1 } FRAME_TYPE; typedef enum { DC_PRED, /* average of above and left pixels */ diff --git a/media/libvpx/libvpx/vp8/common/common.h b/media/libvpx/libvpx/vp8/common/common.h --- a/media/libvpx/libvpx/vp8/common/common.h +++ b/media/libvpx/libvpx/vp8/common/common.h @@ -22,26 +22,26 @@ #endif /* Only need this for fixed-size arrays, for structs just assign. */ #define vp8_copy(Dest, Src) \ - { \ + do { \ assert(sizeof(Dest) == sizeof(Src)); \ memcpy(Dest, Src, sizeof(Src)); \ - } + } while (0) /* Use this for variably-sized arrays. */ #define vp8_copy_array(Dest, Src, N) \ - { \ + do { \ assert(sizeof(*(Dest)) == sizeof(*(Src))); \ memcpy(Dest, Src, (N) * sizeof(*(Src))); \ - } + } while (0) -#define vp8_zero(Dest) memset(&(Dest), 0, sizeof(Dest)); +#define vp8_zero(Dest) memset(&(Dest), 0, sizeof(Dest)) -#define vp8_zero_array(Dest, N) memset(Dest, 0, (N) * sizeof(*(Dest))); +#define vp8_zero_array(Dest, N) memset(Dest, 0, (N) * sizeof(*(Dest))) #ifdef __cplusplus } // extern "C" #endif diff --git a/media/libvpx/libvpx/vp8/common/generic/systemdependent.c b/media/libvpx/libvpx/vp8/common/generic/systemdependent.c --- a/media/libvpx/libvpx/vp8/common/generic/systemdependent.c +++ b/media/libvpx/libvpx/vp8/common/generic/systemdependent.c @@ -16,10 +16,12 @@ #include "vpx_ports/x86.h" #elif VPX_ARCH_PPC #include "vpx_ports/ppc.h" #elif VPX_ARCH_MIPS #include "vpx_ports/mips.h" +#elif VPX_ARCH_LOONGARCH +#include "vpx_ports/loongarch.h" #endif #include "vp8/common/onyxc_int.h" #include "vp8/common/systemdependent.h" #if CONFIG_MULTITHREAD @@ -98,10 +100,12 @@ ctx->cpu_caps = x86_simd_caps(); #elif VPX_ARCH_PPC ctx->cpu_caps = ppc_simd_caps(); #elif VPX_ARCH_MIPS ctx->cpu_caps = mips_cpu_caps(); +#elif VPX_ARCH_LOONGARCH + ctx->cpu_caps = loongarch_cpu_caps(); #else // generic-gnu targets. ctx->cpu_caps = 0; #endif } diff --git a/media/libvpx/libvpx/vp8/common/loongarch/idct_lsx.c b/media/libvpx/libvpx/vp8/common/loongarch/idct_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vp8/common/loongarch/idct_lsx.c @@ -0,0 +1,325 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "./vp8_rtcd.h" +#include "vp8/common/blockd.h" +#include "vpx_util/loongson_intrinsics.h" + +static const int32_t cospi8sqrt2minus1 = 20091; +static const int32_t sinpi8sqrt2 = 35468; + +#define TRANSPOSE8X4_SH_SH(in0, in1, in2, in3, out0, out1, out2, out3) \ + { \ + __m128i tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ + \ + DUP2_ARG2(__lsx_vilvl_h, in1, in0, in3, in2, tmp0_m, tmp1_m); \ + DUP2_ARG2(__lsx_vilvh_h, in1, in0, in3, in2, tmp2_m, tmp3_m); \ + DUP2_ARG2(__lsx_vilvl_w, tmp1_m, tmp0_m, tmp3_m, tmp2_m, out0, out2); \ + DUP2_ARG2(__lsx_vilvh_w, tmp1_m, tmp0_m, tmp3_m, tmp2_m, out1, out3); \ + } + +#define TRANSPOSE_TWO_4x4_H(in0, in1, in2, in3, out0, out1, out2, out3) \ + { \ + __m128i s4_m, s5_m, s6_m, s7_m; \ + \ + TRANSPOSE8X4_SH_SH(in0, in1, in2, in3, s4_m, s5_m, s6_m, s7_m); \ + DUP2_ARG2(__lsx_vilvl_d, s6_m, s4_m, s7_m, s5_m, out0, out2); \ + out1 = __lsx_vilvh_d(s6_m, s4_m); \ + out3 = __lsx_vilvh_d(s7_m, s5_m); \ + } + +#define EXPAND_TO_H_MULTIPLY_SINPI8SQRT2_PCK_TO_W(in) \ + ({ \ + __m128i out_m; \ + __m128i zero_m = __lsx_vldi(0); \ + __m128i tmp1_m, tmp2_m; \ + __m128i sinpi8_sqrt2_m = __lsx_vreplgr2vr_w(sinpi8sqrt2); \ + \ + tmp1_m = __lsx_vilvl_h(in, zero_m); \ + tmp2_m = __lsx_vilvh_h(in, zero_m); \ + tmp1_m = __lsx_vsrai_w(tmp1_m, 16); \ + tmp2_m = __lsx_vsrai_w(tmp2_m, 16); \ + tmp1_m = __lsx_vmul_w(tmp1_m, sinpi8_sqrt2_m); \ + tmp1_m = __lsx_vsrai_w(tmp1_m, 16); \ + tmp2_m = __lsx_vmul_w(tmp2_m, sinpi8_sqrt2_m); \ + tmp2_m = __lsx_vsrai_w(tmp2_m, 16); \ + out_m = __lsx_vpickev_h(tmp2_m, tmp1_m); \ + \ + out_m; \ + }) + +#define VP8_IDCT_1D_H(in0, in1, in2, in3, out0, out1, out2, out3) \ + { \ + __m128i a1_m, b1_m, c1_m, d1_m; \ + __m128i c_tmp1_m, c_tmp2_m; \ + __m128i d_tmp1_m, d_tmp2_m; \ + __m128i const_cospi8sqrt2minus1_m; \ + \ + const_cospi8sqrt2minus1_m = __lsx_vreplgr2vr_h(cospi8sqrt2minus1); \ + a1_m = __lsx_vadd_h(in0, in2); \ + b1_m = __lsx_vsub_h(in0, in2); \ + c_tmp1_m = EXPAND_TO_H_MULTIPLY_SINPI8SQRT2_PCK_TO_W(in1); \ + \ + c_tmp2_m = __lsx_vmuh_h(in3, const_cospi8sqrt2minus1_m); \ + c_tmp2_m = __lsx_vslli_h(c_tmp2_m, 1); \ + c_tmp2_m = __lsx_vsrai_h(c_tmp2_m, 1); \ + c_tmp2_m = __lsx_vadd_h(in3, c_tmp2_m); \ + c1_m = __lsx_vsub_h(c_tmp1_m, c_tmp2_m); \ + \ + d_tmp1_m = __lsx_vmuh_h(in1, const_cospi8sqrt2minus1_m); \ + d_tmp1_m = __lsx_vslli_h(d_tmp1_m, 1); \ + d_tmp1_m = __lsx_vsrai_h(d_tmp1_m, 1); \ + d_tmp1_m = __lsx_vadd_h(in1, d_tmp1_m); \ + d_tmp2_m = EXPAND_TO_H_MULTIPLY_SINPI8SQRT2_PCK_TO_W(in3); \ + d1_m = __lsx_vadd_h(d_tmp1_m, d_tmp2_m); \ + LSX_BUTTERFLY_4_H(a1_m, b1_m, c1_m, d1_m, out0, out1, out2, out3); \ + } + +#define VP8_IDCT_1D_W(in0, in1, in2, in3, out0, out1, out2, out3) \ + { \ + __m128i a1_m, b1_m, c1_m, d1_m; \ + __m128i c_tmp1_m, c_tmp2_m, d_tmp1_m, d_tmp2_m; \ + __m128i const_cospi8sqrt2minus1_m, sinpi8_sqrt2_m; \ + \ + const_cospi8sqrt2minus1_m = __lsx_vreplgr2vr_w(cospi8sqrt2minus1); \ + sinpi8_sqrt2_m = __lsx_vreplgr2vr_w(sinpi8sqrt2); \ + a1_m = __lsx_vadd_w(in0, in2); \ + b1_m = __lsx_vsub_w(in0, in2); \ + c_tmp1_m = __lsx_vmul_w(in1, sinpi8_sqrt2_m); \ + c_tmp1_m = __lsx_vsrai_w(c_tmp1_m, 16); \ + c_tmp2_m = __lsx_vmul_w(in3, const_cospi8sqrt2minus1_m); \ + c_tmp2_m = __lsx_vsrai_w(c_tmp2_m, 16); \ + c_tmp2_m = __lsx_vadd_w(in3, c_tmp2_m); \ + c1_m = __lsx_vsub_w(c_tmp1_m, c_tmp2_m); \ + d_tmp1_m = __lsx_vmul_w(in1, const_cospi8sqrt2minus1_m); \ + d_tmp1_m = __lsx_vsrai_w(d_tmp1_m, 16); \ + d_tmp1_m = __lsx_vadd_w(in1, d_tmp1_m); \ + d_tmp2_m = __lsx_vmul_w(in3, sinpi8_sqrt2_m); \ + d_tmp2_m = __lsx_vsrai_w(d_tmp2_m, 16); \ + d1_m = __lsx_vadd_w(d_tmp1_m, d_tmp2_m); \ + LSX_BUTTERFLY_4_W(a1_m, b1_m, c1_m, d1_m, out0, out1, out2, out3); \ + } + +#define UNPCK_SH_SW(in, out0, out1) \ + { \ + out0 = __lsx_vsllwil_w_h(in, 0); \ + out1 = __lsx_vexth_w_h(in); \ + } + +static void idct4x4_addconst_lsx(int16_t in_dc, uint8_t *pred, + int32_t pred_stride, uint8_t *dest, + int32_t dest_stride) { + __m128i vec, res0, res1, res2, res3, dst0, dst1; + __m128i pred0, pred1, pred2, pred3; + __m128i zero = __lsx_vldi(0); + + int32_t pred_stride2 = pred_stride << 1; + int32_t pred_stride3 = pred_stride2 + pred_stride; + + vec = __lsx_vreplgr2vr_h(in_dc); + vec = __lsx_vsrari_h(vec, 3); + pred0 = __lsx_vld(pred, 0); + DUP2_ARG2(__lsx_vldx, pred, pred_stride, pred, pred_stride2, pred1, pred2); + pred3 = __lsx_vldx(pred, pred_stride3); + DUP4_ARG2(__lsx_vilvl_b, zero, pred0, zero, pred1, zero, pred2, zero, pred3, + res0, res1, res2, res3); + DUP4_ARG2(__lsx_vadd_h, res0, vec, res1, vec, res2, vec, res3, vec, res0, + res1, res2, res3); + res0 = __lsx_vclip255_h(res0); + res1 = __lsx_vclip255_h(res1); + res2 = __lsx_vclip255_h(res2); + res3 = __lsx_vclip255_h(res3); + + DUP2_ARG2(__lsx_vpickev_b, res1, res0, res3, res2, dst0, dst1); + dst0 = __lsx_vpickev_w(dst1, dst0); + __lsx_vstelm_w(dst0, dest, 0, 0); + dest += dest_stride; + __lsx_vstelm_w(dst0, dest, 0, 1); + dest += dest_stride; + __lsx_vstelm_w(dst0, dest, 0, 2); + dest += dest_stride; + __lsx_vstelm_w(dst0, dest, 0, 3); +} + +void vp8_dc_only_idct_add_lsx(int16_t input_dc, uint8_t *pred_ptr, + int32_t pred_stride, uint8_t *dst_ptr, + int32_t dst_stride) { + idct4x4_addconst_lsx(input_dc, pred_ptr, pred_stride, dst_ptr, dst_stride); +} + +static void dequant_idct4x4_addblk_2x_lsx(int16_t *input, + int16_t *dequant_input, uint8_t *dest, + int32_t dest_stride) { + __m128i dest0, dest1, dest2, dest3; + __m128i in0, in1, in2, in3, mul0, mul1, mul2, mul3, dequant_in0, dequant_in1; + __m128i hz0, hz1, hz2, hz3, vt0, vt1, vt2, vt3, res0, res1, res2, res3; + __m128i hz0l, hz1l, hz2l, hz3l, hz0r, hz1r, hz2r, hz3r; + __m128i vt0l, vt1l, vt2l, vt3l, vt0r, vt1r, vt2r, vt3r; + __m128i zero = __lsx_vldi(0); + + int32_t dest_stride2 = dest_stride << 1; + int32_t dest_stride3 = dest_stride2 + dest_stride; + + DUP4_ARG2(__lsx_vld, input, 0, input, 16, input, 32, input, 48, in0, in1, in2, + in3); + DUP2_ARG2(__lsx_vld, dequant_input, 0, dequant_input, 16, dequant_in0, + dequant_in1); + + DUP4_ARG2(__lsx_vmul_h, in0, dequant_in0, in1, dequant_in1, in2, dequant_in0, + in3, dequant_in1, mul0, mul1, mul2, mul3); + DUP2_ARG2(__lsx_vpickev_d, mul2, mul0, mul3, mul1, in0, in2); + DUP2_ARG2(__lsx_vpickod_d, mul2, mul0, mul3, mul1, in1, in3); + + VP8_IDCT_1D_H(in0, in1, in2, in3, hz0, hz1, hz2, hz3); + TRANSPOSE_TWO_4x4_H(hz0, hz1, hz2, hz3, hz0, hz1, hz2, hz3); + UNPCK_SH_SW(hz0, hz0r, hz0l); + UNPCK_SH_SW(hz1, hz1r, hz1l); + UNPCK_SH_SW(hz2, hz2r, hz2l); + UNPCK_SH_SW(hz3, hz3r, hz3l); + VP8_IDCT_1D_W(hz0l, hz1l, hz2l, hz3l, vt0l, vt1l, vt2l, vt3l); + DUP4_ARG2(__lsx_vsrari_w, vt0l, 3, vt1l, 3, vt2l, 3, vt3l, 3, vt0l, vt1l, + vt2l, vt3l); + VP8_IDCT_1D_W(hz0r, hz1r, hz2r, hz3r, vt0r, vt1r, vt2r, vt3r); + DUP4_ARG2(__lsx_vsrari_w, vt0r, 3, vt1r, 3, vt2r, 3, vt3r, 3, vt0r, vt1r, + vt2r, vt3r); + DUP4_ARG2(__lsx_vpickev_h, vt0l, vt0r, vt1l, vt1r, vt2l, vt2r, vt3l, vt3r, + vt0, vt1, vt2, vt3); + TRANSPOSE_TWO_4x4_H(vt0, vt1, vt2, vt3, vt0, vt1, vt2, vt3); + dest0 = __lsx_vld(dest, 0); + DUP2_ARG2(__lsx_vldx, dest, dest_stride, dest, dest_stride2, dest1, dest2); + dest3 = __lsx_vldx(dest, dest_stride3); + DUP4_ARG2(__lsx_vilvl_b, zero, dest0, zero, dest1, zero, dest2, zero, dest3, + res0, res1, res2, res3); + DUP4_ARG2(__lsx_vadd_h, res0, vt0, res1, vt1, res2, vt2, res3, vt3, res0, + res1, res2, res3); + + res0 = __lsx_vclip255_h(res0); + res1 = __lsx_vclip255_h(res1); + res2 = __lsx_vclip255_h(res2); + res3 = __lsx_vclip255_h(res3); + DUP2_ARG2(__lsx_vpickev_b, res1, res0, res3, res2, vt0l, vt1l); + + __lsx_vstelm_d(vt0l, dest, 0, 0); + __lsx_vstelm_d(vt0l, dest + dest_stride, 0, 1); + __lsx_vstelm_d(vt1l, dest + dest_stride2, 0, 0); + __lsx_vstelm_d(vt1l, dest + dest_stride3, 0, 1); + + __lsx_vst(zero, input, 0); + __lsx_vst(zero, input, 16); + __lsx_vst(zero, input, 32); + __lsx_vst(zero, input, 48); +} + +static void dequant_idct_addconst_2x_lsx(int16_t *input, int16_t *dequant_input, + uint8_t *dest, int32_t dest_stride) { + __m128i input_dc0, input_dc1, vec, res0, res1, res2, res3; + __m128i dest0, dest1, dest2, dest3; + __m128i zero = __lsx_vldi(0); + int32_t dest_stride2 = dest_stride << 1; + int32_t dest_stride3 = dest_stride2 + dest_stride; + + input_dc0 = __lsx_vreplgr2vr_h(input[0] * dequant_input[0]); + input_dc1 = __lsx_vreplgr2vr_h(input[16] * dequant_input[0]); + DUP2_ARG2(__lsx_vsrari_h, input_dc0, 3, input_dc1, 3, input_dc0, input_dc1); + vec = __lsx_vpickev_d(input_dc1, input_dc0); + input[0] = 0; + input[16] = 0; + dest0 = __lsx_vld(dest, 0); + DUP2_ARG2(__lsx_vldx, dest, dest_stride, dest, dest_stride2, dest1, dest2); + dest3 = __lsx_vldx(dest, dest_stride3); + DUP4_ARG2(__lsx_vilvl_b, zero, dest0, zero, dest1, zero, dest2, zero, dest3, + res0, res1, res2, res3); + DUP4_ARG2(__lsx_vadd_h, res0, vec, res1, vec, res2, vec, res3, vec, res0, + res1, res2, res3); + res0 = __lsx_vclip255_h(res0); + res1 = __lsx_vclip255_h(res1); + res2 = __lsx_vclip255_h(res2); + res3 = __lsx_vclip255_h(res3); + + DUP2_ARG2(__lsx_vpickev_b, res1, res0, res3, res2, res0, res1); + __lsx_vstelm_d(res0, dest, 0, 0); + __lsx_vstelm_d(res0, dest + dest_stride, 0, 1); + __lsx_vstelm_d(res1, dest + dest_stride2, 0, 0); + __lsx_vstelm_d(res1, dest + dest_stride3, 0, 1); +} + +void vp8_dequant_idct_add_y_block_lsx(int16_t *q, int16_t *dq, uint8_t *dst, + int32_t stride, char *eobs) { + int16_t *eobs_h = (int16_t *)eobs; + uint8_t i; + + for (i = 4; i--;) { + if (eobs_h[0]) { + if (eobs_h[0] & 0xfefe) { + dequant_idct4x4_addblk_2x_lsx(q, dq, dst, stride); + } else { + dequant_idct_addconst_2x_lsx(q, dq, dst, stride); + } + } + + q += 32; + + if (eobs_h[1]) { + if (eobs_h[1] & 0xfefe) { + dequant_idct4x4_addblk_2x_lsx(q, dq, dst + 8, stride); + } else { + dequant_idct_addconst_2x_lsx(q, dq, dst + 8, stride); + } + } + + q += 32; + dst += (4 * stride); + eobs_h += 2; + } +} + +void vp8_dequant_idct_add_uv_block_lsx(int16_t *q, int16_t *dq, uint8_t *dst_u, + uint8_t *dst_v, int32_t stride, + char *eobs) { + int16_t *eobs_h = (int16_t *)eobs; + if (eobs_h[0]) { + if (eobs_h[0] & 0xfefe) { + dequant_idct4x4_addblk_2x_lsx(q, dq, dst_u, stride); + } else { + dequant_idct_addconst_2x_lsx(q, dq, dst_u, stride); + } + } + + q += 32; + dst_u += (stride * 4); + + if (eobs_h[1]) { + if (eobs_h[1] & 0xfefe) { + dequant_idct4x4_addblk_2x_lsx(q, dq, dst_u, stride); + } else { + dequant_idct_addconst_2x_lsx(q, dq, dst_u, stride); + } + } + + q += 32; + + if (eobs_h[2]) { + if (eobs_h[2] & 0xfefe) { + dequant_idct4x4_addblk_2x_lsx(q, dq, dst_v, stride); + } else { + dequant_idct_addconst_2x_lsx(q, dq, dst_v, stride); + } + } + q += 32; + dst_v += (stride * 4); + + if (eobs_h[3]) { + if (eobs_h[3] & 0xfefe) { + dequant_idct4x4_addblk_2x_lsx(q, dq, dst_v, stride); + } else { + dequant_idct_addconst_2x_lsx(q, dq, dst_v, stride); + } + } +} diff --git a/media/libvpx/libvpx/vp8/common/loongarch/loopfilter_filters_lsx.c b/media/libvpx/libvpx/vp8/common/loongarch/loopfilter_filters_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vp8/common/loongarch/loopfilter_filters_lsx.c @@ -0,0 +1,743 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Lu Wang + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "./vp8_rtcd.h" +#include "vp8/common/loopfilter.h" +#include "vpx_util/loongson_intrinsics.h" + +#define VP8_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev) \ + { \ + __m128i p1_m, p0_m, q0_m, q1_m, filt, q0_sub_p0, t1, t2; \ + const __m128i cnst4b = __lsx_vldi(4); \ + const __m128i cnst3b = __lsx_vldi(3); \ + \ + p1_m = __lsx_vxori_b(p1, 0x80); \ + p0_m = __lsx_vxori_b(p0, 0x80); \ + q0_m = __lsx_vxori_b(q0, 0x80); \ + q1_m = __lsx_vxori_b(q1, 0x80); \ + \ + filt = __lsx_vssub_b(p1_m, q1_m); \ + filt = __lsx_vand_v(filt, hev); \ + q0_sub_p0 = __lsx_vssub_b(q0_m, p0_m); \ + filt = __lsx_vsadd_b(filt, q0_sub_p0); \ + filt = __lsx_vsadd_b(filt, q0_sub_p0); \ + filt = __lsx_vsadd_b(filt, q0_sub_p0); \ + filt = __lsx_vand_v(filt, mask); \ + t1 = __lsx_vsadd_b(filt, cnst4b); \ + t1 = __lsx_vsra_b(t1, cnst3b); \ + t2 = __lsx_vsadd_b(filt, cnst3b); \ + t2 = __lsx_vsra_b(t2, cnst3b); \ + q0_m = __lsx_vssub_b(q0_m, t1); \ + q0 = __lsx_vxori_b(q0_m, 0x80); \ + p0_m = __lsx_vsadd_b(p0_m, t2); \ + p0 = __lsx_vxori_b(p0_m, 0x80); \ + filt = __lsx_vsrari_b(t1, 1); \ + hev = __lsx_vxori_b(hev, 0xff); \ + filt = __lsx_vand_v(filt, hev); \ + q1_m = __lsx_vssub_b(q1_m, filt); \ + q1 = __lsx_vxori_b(q1_m, 0x80); \ + p1_m = __lsx_vsadd_b(p1_m, filt); \ + p1 = __lsx_vxori_b(p1_m, 0x80); \ + } + +#define VP8_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev) \ + { \ + __m128i p2_m, p1_m, p0_m, q2_m, q1_m, q0_m; \ + __m128i u, filt, t1, t2, filt_sign, q0_sub_p0; \ + __m128i filt_r, filt_l; \ + __m128i temp0, temp1, temp2, temp3; \ + const __m128i cnst4b = __lsx_vldi(4); \ + const __m128i cnst3b = __lsx_vldi(3); \ + const __m128i cnst9h = __lsx_vldi(1033); \ + const __m128i cnst63h = __lsx_vldi(1087); \ + \ + p2_m = __lsx_vxori_b(p2, 0x80); \ + p1_m = __lsx_vxori_b(p1, 0x80); \ + p0_m = __lsx_vxori_b(p0, 0x80); \ + q0_m = __lsx_vxori_b(q0, 0x80); \ + q1_m = __lsx_vxori_b(q1, 0x80); \ + q2_m = __lsx_vxori_b(q2, 0x80); \ + \ + filt = __lsx_vssub_b(p1_m, q1_m); \ + q0_sub_p0 = __lsx_vssub_b(q0_m, p0_m); \ + filt = __lsx_vsadd_b(filt, q0_sub_p0); \ + filt = __lsx_vsadd_b(filt, q0_sub_p0); \ + filt = __lsx_vsadd_b(filt, q0_sub_p0); \ + filt = __lsx_vand_v(filt, mask); \ + \ + t2 = __lsx_vand_v(filt, hev); \ + hev = __lsx_vxori_b(hev, 0xff); \ + filt = __lsx_vand_v(hev, filt); \ + t1 = __lsx_vsadd_b(t2, cnst4b); \ + t1 = __lsx_vsra_b(t1, cnst3b); \ + t2 = __lsx_vsadd_b(t2, cnst3b); \ + t2 = __lsx_vsra_b(t2, cnst3b); \ + q0_m = __lsx_vssub_b(q0_m, t1); \ + p0_m = __lsx_vsadd_b(p0_m, t2); \ + filt_sign = __lsx_vslti_b(filt, 0); \ + filt_r = __lsx_vilvl_b(filt_sign, filt); \ + filt_l = __lsx_vilvh_b(filt_sign, filt); \ + temp0 = __lsx_vmul_h(filt_r, cnst9h); \ + temp1 = __lsx_vadd_h(temp0, cnst63h); \ + temp2 = __lsx_vmul_h(filt_l, cnst9h); \ + temp3 = __lsx_vadd_h(temp2, cnst63h); \ + \ + u = __lsx_vssrani_b_h(temp3, temp1, 7); \ + q2_m = __lsx_vssub_b(q2_m, u); \ + p2_m = __lsx_vsadd_b(p2_m, u); \ + q2 = __lsx_vxori_b(q2_m, 0x80); \ + p2 = __lsx_vxori_b(p2_m, 0x80); \ + \ + temp1 = __lsx_vadd_h(temp1, temp0); \ + temp3 = __lsx_vadd_h(temp3, temp2); \ + \ + u = __lsx_vssrani_b_h(temp3, temp1, 7); \ + q1_m = __lsx_vssub_b(q1_m, u); \ + p1_m = __lsx_vsadd_b(p1_m, u); \ + q1 = __lsx_vxori_b(q1_m, 0x80); \ + p1 = __lsx_vxori_b(p1_m, 0x80); \ + \ + temp1 = __lsx_vadd_h(temp1, temp0); \ + temp3 = __lsx_vadd_h(temp3, temp2); \ + \ + u = __lsx_vssrani_b_h(temp3, temp1, 7); \ + q0_m = __lsx_vssub_b(q0_m, u); \ + p0_m = __lsx_vsadd_b(p0_m, u); \ + q0 = __lsx_vxori_b(q0_m, 0x80); \ + p0 = __lsx_vxori_b(p0_m, 0x80); \ + } + +#define LPF_MASK_HEV(p3_in, p2_in, p1_in, p0_in, q0_in, q1_in, q2_in, q3_in, \ + limit_in, b_limit_in, thresh_in, hev_out, mask_out, \ + flat_out) \ + { \ + __m128i p3_asub_p2_m, p2_asub_p1_m, p1_asub_p0_m, q1_asub_q0_m; \ + __m128i p1_asub_q1_m, p0_asub_q0_m, q3_asub_q2_m, q2_asub_q1_m; \ + \ + p3_asub_p2_m = __lsx_vabsd_bu(p3_in, p2_in); \ + p2_asub_p1_m = __lsx_vabsd_bu(p2_in, p1_in); \ + p1_asub_p0_m = __lsx_vabsd_bu(p1_in, p0_in); \ + q1_asub_q0_m = __lsx_vabsd_bu(q1_in, q0_in); \ + q2_asub_q1_m = __lsx_vabsd_bu(q2_in, q1_in); \ + q3_asub_q2_m = __lsx_vabsd_bu(q3_in, q2_in); \ + p0_asub_q0_m = __lsx_vabsd_bu(p0_in, q0_in); \ + p1_asub_q1_m = __lsx_vabsd_bu(p1_in, q1_in); \ + flat_out = __lsx_vmax_bu(p1_asub_p0_m, q1_asub_q0_m); \ + hev_out = __lsx_vslt_bu(thresh_in, flat_out); \ + p0_asub_q0_m = __lsx_vsadd_bu(p0_asub_q0_m, p0_asub_q0_m); \ + p1_asub_q1_m = __lsx_vsrli_b(p1_asub_q1_m, 1); \ + p0_asub_q0_m = __lsx_vsadd_bu(p0_asub_q0_m, p1_asub_q1_m); \ + mask_out = __lsx_vslt_bu(b_limit_in, p0_asub_q0_m); \ + mask_out = __lsx_vmax_bu(flat_out, mask_out); \ + p3_asub_p2_m = __lsx_vmax_bu(p3_asub_p2_m, p2_asub_p1_m); \ + mask_out = __lsx_vmax_bu(p3_asub_p2_m, mask_out); \ + q2_asub_q1_m = __lsx_vmax_bu(q2_asub_q1_m, q3_asub_q2_m); \ + mask_out = __lsx_vmax_bu(q2_asub_q1_m, mask_out); \ + mask_out = __lsx_vslt_bu(limit_in, mask_out); \ + mask_out = __lsx_vxori_b(mask_out, 0xff); \ + } + +#define VP8_ST6x1_B(in0, in0_idx, in1, in1_idx, pdst, stride) \ + { \ + __lsx_vstelm_w(in0, pdst, 0, in0_idx); \ + __lsx_vstelm_h(in1, pdst + stride, 0, in1_idx); \ + } + +static void loop_filter_horizontal_4_dual_lsx(uint8_t *src, int32_t pitch, + const uint8_t *b_limit0_ptr, + const uint8_t *limit0_ptr, + const uint8_t *thresh0_ptr, + const uint8_t *b_limit1_ptr, + const uint8_t *limit1_ptr, + const uint8_t *thresh1_ptr) { + int32_t pitch_x2 = pitch << 1; + int32_t pitch_x3 = pitch_x2 + pitch; + int32_t pitch_x4 = pitch << 2; + + __m128i mask, hev, flat; + __m128i thresh0, b_limit0, limit0, thresh1, b_limit1, limit1; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + + DUP4_ARG2(__lsx_vldx, src, -pitch_x4, src, -pitch_x3, src, -pitch_x2, src, + -pitch, p3, p2, p1, p0); + q0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, pitch, src, pitch_x2, q1, q2); + q3 = __lsx_vldx(src, pitch_x3); + + thresh0 = __lsx_vreplgr2vr_b(*thresh0_ptr); + thresh1 = __lsx_vreplgr2vr_b(*thresh1_ptr); + thresh0 = __lsx_vilvl_d(thresh1, thresh0); + + b_limit0 = __lsx_vreplgr2vr_b(*b_limit0_ptr); + b_limit1 = __lsx_vreplgr2vr_b(*b_limit1_ptr); + b_limit0 = __lsx_vilvl_d(b_limit1, b_limit0); + + limit0 = __lsx_vreplgr2vr_b(*limit0_ptr); + limit1 = __lsx_vreplgr2vr_b(*limit1_ptr); + limit0 = __lsx_vilvl_d(limit1, limit0); + + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit0, b_limit0, thresh0, hev, + mask, flat); + VP8_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev); + + __lsx_vstx(p1, src, -pitch_x2); + __lsx_vstx(p0, src, -pitch); + __lsx_vst(q0, src, 0); + __lsx_vstx(q1, src, pitch); +} + +static void loop_filter_vertical_4_dual_lsx(uint8_t *src, int32_t pitch, + const uint8_t *b_limit0_ptr, + const uint8_t *limit0_ptr, + const uint8_t *thresh0_ptr, + const uint8_t *b_limit1_ptr, + const uint8_t *limit1_ptr, + const uint8_t *thresh1_ptr) { + uint8_t *src_tmp0 = src - 4; + int32_t pitch_x2 = pitch << 1; + int32_t pitch_x3 = pitch_x2 + pitch; + int32_t pitch_x4 = pitch << 2; + __m128i mask, hev, flat; + __m128i thresh0, b_limit0, limit0, thresh1, b_limit1, limit1; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i row0, row1, row2, row3, row4, row5, row6, row7; + __m128i row8, row9, row10, row11, row12, row13, row14, row15; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + + row0 = __lsx_vld(src_tmp0, 0); + DUP2_ARG2(__lsx_vldx, src_tmp0, pitch, src_tmp0, pitch_x2, row1, row2); + row3 = __lsx_vldx(src_tmp0, pitch_x3); + src_tmp0 += pitch_x4; + row4 = __lsx_vld(src_tmp0, 0); + DUP2_ARG2(__lsx_vldx, src_tmp0, pitch, src_tmp0, pitch_x2, row5, row6); + row7 = __lsx_vldx(src_tmp0, pitch_x3); + src_tmp0 += pitch_x4; + + row8 = __lsx_vld(src_tmp0, 0); + DUP2_ARG2(__lsx_vldx, src_tmp0, pitch, src_tmp0, pitch_x2, row9, row10); + row11 = __lsx_vldx(src_tmp0, pitch_x3); + src_tmp0 += pitch_x4; + row12 = __lsx_vld(src_tmp0, 0); + DUP2_ARG2(__lsx_vldx, src_tmp0, pitch, src_tmp0, pitch_x2, row13, row14); + row15 = __lsx_vldx(src_tmp0, pitch_x3); + + LSX_TRANSPOSE16x8_B(row0, row1, row2, row3, row4, row5, row6, row7, row8, + row9, row10, row11, row12, row13, row14, row15, p3, p2, + p1, p0, q0, q1, q2, q3); + + thresh0 = __lsx_vldrepl_b(thresh0_ptr, 0); + thresh1 = __lsx_vldrepl_b(thresh1_ptr, 0); + thresh0 = __lsx_vilvl_d(thresh1, thresh0); + + b_limit0 = __lsx_vldrepl_b(b_limit0_ptr, 0); + b_limit1 = __lsx_vldrepl_b(b_limit1_ptr, 0); + b_limit0 = __lsx_vilvl_d(b_limit1, b_limit0); + + limit0 = __lsx_vldrepl_b(limit0_ptr, 0); + limit1 = __lsx_vldrepl_b(limit1_ptr, 0); + limit0 = __lsx_vilvl_d(limit1, limit0); + + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit0, b_limit0, thresh0, hev, + mask, flat); + VP8_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev); + + DUP2_ARG2(__lsx_vilvl_b, p0, p1, q1, q0, tmp0, tmp1); + tmp2 = __lsx_vilvl_h(tmp1, tmp0); + tmp3 = __lsx_vilvh_h(tmp1, tmp0); + DUP2_ARG2(__lsx_vilvh_b, p0, p1, q1, q0, tmp0, tmp1); + tmp4 = __lsx_vilvl_h(tmp1, tmp0); + tmp5 = __lsx_vilvh_h(tmp1, tmp0); + + src -= 2; + __lsx_vstelm_w(tmp2, src, 0, 0); + src += pitch; + __lsx_vstelm_w(tmp2, src, 0, 1); + src += pitch; + __lsx_vstelm_w(tmp2, src, 0, 2); + src += pitch; + __lsx_vstelm_w(tmp2, src, 0, 3); + src += pitch; + + __lsx_vstelm_w(tmp3, src, 0, 0); + src += pitch; + __lsx_vstelm_w(tmp3, src, 0, 1); + src += pitch; + __lsx_vstelm_w(tmp3, src, 0, 2); + src += pitch; + __lsx_vstelm_w(tmp3, src, 0, 3); + src += pitch; + + __lsx_vstelm_w(tmp4, src, 0, 0); + src += pitch; + __lsx_vstelm_w(tmp4, src, 0, 1); + src += pitch; + __lsx_vstelm_w(tmp4, src, 0, 2); + src += pitch; + __lsx_vstelm_w(tmp4, src, 0, 3); + src += pitch; + + __lsx_vstelm_w(tmp5, src, 0, 0); + src += pitch; + __lsx_vstelm_w(tmp5, src, 0, 1); + src += pitch; + __lsx_vstelm_w(tmp5, src, 0, 2); + src += pitch; + __lsx_vstelm_w(tmp5, src, 0, 3); +} + +static void loop_filter_horizontal_edge_uv_lsx(uint8_t *src_u, uint8_t *src_v, + int32_t pitch, + const uint8_t b_limit_in, + const uint8_t limit_in, + const uint8_t thresh_in) { + int32_t pitch_x2 = pitch << 1; + int32_t pitch_x3 = pitch_x2 + pitch; + int32_t pitch_x4 = pitch << 2; + + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i mask, hev, flat, thresh, limit, b_limit; + __m128i p3_u, p2_u, p1_u, p0_u, q3_u, q2_u, q1_u, q0_u; + __m128i p3_v, p2_v, p1_v, p0_v, q3_v, q2_v, q1_v, q0_v; + + thresh = __lsx_vreplgr2vr_b(thresh_in); + limit = __lsx_vreplgr2vr_b(limit_in); + b_limit = __lsx_vreplgr2vr_b(b_limit_in); + + DUP4_ARG2(__lsx_vldx, src_u, -pitch_x4, src_u, -pitch_x3, src_u, -pitch_x2, + src_u, -pitch, p3_u, p2_u, p1_u, p0_u); + q0_u = __lsx_vld(src_u, 0); + DUP2_ARG2(__lsx_vldx, src_u, pitch, src_u, pitch_x2, q1_u, q2_u); + q3_u = __lsx_vldx(src_u, pitch_x3); + + DUP4_ARG2(__lsx_vldx, src_v, -pitch_x4, src_v, -pitch_x3, src_v, -pitch_x2, + src_v, -pitch, p3_v, p2_v, p1_v, p0_v); + q0_v = __lsx_vld(src_v, 0); + DUP2_ARG2(__lsx_vldx, src_v, pitch, src_v, pitch_x2, q1_v, q2_v); + q3_v = __lsx_vldx(src_v, pitch_x3); + + /* right 8 element of p3 are u pixel and + left 8 element of p3 are v pixel */ + DUP4_ARG2(__lsx_vilvl_d, p3_v, p3_u, p2_v, p2_u, p1_v, p1_u, p0_v, p0_u, p3, + p2, p1, p0); + DUP4_ARG2(__lsx_vilvl_d, q0_v, q0_u, q1_v, q1_u, q2_v, q2_u, q3_v, q3_u, q0, + q1, q2, q3); + + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + VP8_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev); + + __lsx_vstelm_d(q1, src_u + pitch, 0, 0); + __lsx_vstelm_d(q0, src_u, 0, 0); + __lsx_vstelm_d(p0, src_u - pitch, 0, 0); + __lsx_vstelm_d(p1, src_u - pitch_x2, 0, 0); + + __lsx_vstelm_d(q1, src_v + pitch, 0, 1); + __lsx_vstelm_d(q0, src_v, 0, 1); + __lsx_vstelm_d(p0, src_v - pitch, 0, 1); + __lsx_vstelm_d(p1, src_v - pitch_x2, 0, 1); +} + +static void loop_filter_vertical_edge_uv_lsx(uint8_t *src_u, uint8_t *src_v, + int32_t pitch, + const uint8_t b_limit_in, + const uint8_t limit_in, + const uint8_t thresh_in) { + uint8_t *src_u_tmp, *src_v_tmp; + int32_t pitch_x2 = pitch << 1; + int32_t pitch_x3 = pitch_x2 + pitch; + int32_t pitch_x4 = pitch << 2; + + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i mask, hev, flat, thresh, limit, b_limit; + __m128i row0, row1, row2, row3, row4, row5, row6, row7, row8; + __m128i row9, row10, row11, row12, row13, row14, row15; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + + thresh = __lsx_vreplgr2vr_b(thresh_in); + limit = __lsx_vreplgr2vr_b(limit_in); + b_limit = __lsx_vreplgr2vr_b(b_limit_in); + + src_u_tmp = src_u - 4; + row0 = __lsx_vld(src_u_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_u_tmp, pitch, src_u_tmp, pitch_x2, row1, row2); + row3 = __lsx_vldx(src_u_tmp, pitch_x3); + src_u_tmp += pitch_x4; + row4 = __lsx_vld(src_u_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_u_tmp, pitch, src_u_tmp, pitch_x2, row5, row6); + row7 = __lsx_vldx(src_u_tmp, pitch_x3); + + src_v_tmp = src_v - 4; + row8 = __lsx_vld(src_v_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_v_tmp, pitch, src_v_tmp, pitch_x2, row9, row10); + row11 = __lsx_vldx(src_v_tmp, pitch_x3); + src_v_tmp += pitch_x4; + row12 = __lsx_vld(src_v_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_v_tmp, pitch, src_v_tmp, pitch_x2, row13, row14); + row15 = __lsx_vldx(src_v_tmp, pitch_x3); + + LSX_TRANSPOSE16x8_B(row0, row1, row2, row3, row4, row5, row6, row7, row8, + row9, row10, row11, row12, row13, row14, row15, p3, p2, + p1, p0, q0, q1, q2, q3); + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + VP8_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev); + + DUP2_ARG2(__lsx_vilvl_b, p0, p1, q1, q0, tmp0, tmp1); + tmp2 = __lsx_vilvl_h(tmp1, tmp0); + tmp3 = __lsx_vilvh_h(tmp1, tmp0); + + tmp0 = __lsx_vilvh_b(p0, p1); + tmp1 = __lsx_vilvh_b(q1, q0); + tmp4 = __lsx_vilvl_h(tmp1, tmp0); + tmp5 = __lsx_vilvh_h(tmp1, tmp0); + + src_u_tmp += 2; + __lsx_vstelm_w(tmp2, src_u_tmp - pitch_x4, 0, 0); + __lsx_vstelm_w(tmp2, src_u_tmp - pitch_x3, 0, 1); + __lsx_vstelm_w(tmp2, src_u_tmp - pitch_x2, 0, 2); + __lsx_vstelm_w(tmp2, src_u_tmp - pitch, 0, 3); + + __lsx_vstelm_w(tmp3, src_u_tmp, 0, 0); + __lsx_vstelm_w(tmp3, src_u_tmp + pitch, 0, 1); + __lsx_vstelm_w(tmp3, src_u_tmp + pitch_x2, 0, 2); + __lsx_vstelm_w(tmp3, src_u_tmp + pitch_x3, 0, 3); + + src_v_tmp += 2; + __lsx_vstelm_w(tmp4, src_v_tmp - pitch_x4, 0, 0); + __lsx_vstelm_w(tmp4, src_v_tmp - pitch_x3, 0, 1); + __lsx_vstelm_w(tmp4, src_v_tmp - pitch_x2, 0, 2); + __lsx_vstelm_w(tmp4, src_v_tmp - pitch, 0, 3); + + __lsx_vstelm_w(tmp5, src_v_tmp, 0, 0); + __lsx_vstelm_w(tmp5, src_v_tmp + pitch, 0, 1); + __lsx_vstelm_w(tmp5, src_v_tmp + pitch_x2, 0, 2); + __lsx_vstelm_w(tmp5, src_v_tmp + pitch_x3, 0, 3); +} + +static inline void mbloop_filter_horizontal_edge_y_lsx( + uint8_t *src, int32_t pitch, const uint8_t b_limit_in, + const uint8_t limit_in, const uint8_t thresh_in) { + uint8_t *temp_src; + int32_t pitch_x2 = pitch << 1; + int32_t pitch_x3 = pitch_x2 + pitch; + int32_t pitch_x4 = pitch << 2; + + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i mask, hev, flat, thresh, limit, b_limit; + + DUP2_ARG2(__lsx_vldrepl_b, &b_limit_in, 0, &limit_in, 0, b_limit, limit); + thresh = __lsx_vldrepl_b(&thresh_in, 0); + + temp_src = src - pitch_x4; + DUP4_ARG2(__lsx_vldx, temp_src, 0, temp_src, pitch, temp_src, pitch_x2, + temp_src, pitch_x3, p3, p2, p1, p0); + temp_src += pitch_x4; + DUP4_ARG2(__lsx_vldx, temp_src, 0, temp_src, pitch, temp_src, pitch_x2, + temp_src, pitch_x3, q0, q1, q2, q3); + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + VP8_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev); + + temp_src = src - pitch_x3; + __lsx_vstx(p2, temp_src, 0); + __lsx_vstx(p1, temp_src, pitch); + __lsx_vstx(p0, temp_src, pitch_x2); + __lsx_vstx(q0, temp_src, pitch_x3); + temp_src += pitch_x4; + __lsx_vstx(q1, temp_src, 0); + __lsx_vstx(q2, temp_src, pitch); +} + +static inline void mbloop_filter_horizontal_edge_uv_lsx( + uint8_t *src_u, uint8_t *src_v, int32_t pitch, const uint8_t b_limit_in, + const uint8_t limit_in, const uint8_t thresh_in) { + uint8_t *temp_src; + int32_t pitch_x2 = pitch << 1; + int32_t pitch_x3 = pitch_x2 + pitch; + int32_t pitch_x4 = pitch << 2; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i mask, hev, flat, thresh, limit, b_limit; + __m128i p3_u, p2_u, p1_u, p0_u, q3_u, q2_u, q1_u, q0_u; + __m128i p3_v, p2_v, p1_v, p0_v, q3_v, q2_v, q1_v, q0_v; + + DUP2_ARG2(__lsx_vldrepl_b, &b_limit_in, 0, &limit_in, 0, b_limit, limit); + thresh = __lsx_vldrepl_b(&thresh_in, 0); + + temp_src = src_u - pitch_x4; + DUP4_ARG2(__lsx_vldx, temp_src, 0, temp_src, pitch, temp_src, pitch_x2, + temp_src, pitch_x3, p3_u, p2_u, p1_u, p0_u); + temp_src += pitch_x4; + DUP4_ARG2(__lsx_vldx, temp_src, 0, temp_src, pitch, temp_src, pitch_x2, + temp_src, pitch_x3, q0_u, q1_u, q2_u, q3_u); + temp_src = src_v - pitch_x4; + DUP4_ARG2(__lsx_vldx, temp_src, 0, temp_src, pitch, temp_src, pitch_x2, + temp_src, pitch_x3, p3_v, p2_v, p1_v, p0_v); + temp_src += pitch_x4; + DUP4_ARG2(__lsx_vldx, temp_src, 0, temp_src, pitch, temp_src, pitch_x2, + temp_src, pitch_x3, q0_v, q1_v, q2_v, q3_v); + + DUP4_ARG2(__lsx_vilvl_d, p3_v, p3_u, p2_v, p2_u, p1_v, p1_u, p0_v, p0_u, p3, + p2, p1, p0); + DUP4_ARG2(__lsx_vilvl_d, q0_v, q0_u, q1_v, q1_u, q2_v, q2_u, q3_v, q3_u, q0, + q1, q2, q3); + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + VP8_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev); + + src_u -= pitch_x3; + __lsx_vstelm_d(p2, src_u, 0, 0); + __lsx_vstelm_d(p1, src_u + pitch, 0, 0); + __lsx_vstelm_d(p0, src_u + pitch_x2, 0, 0); + __lsx_vstelm_d(q0, src_u + pitch_x3, 0, 0); + src_u += pitch_x4; + __lsx_vstelm_d(q1, src_u, 0, 0); + src_u += pitch; + __lsx_vstelm_d(q2, src_u, 0, 0); + + src_v -= pitch_x3; + __lsx_vstelm_d(p2, src_v, 0, 1); + __lsx_vstelm_d(p1, src_v + pitch, 0, 1); + __lsx_vstelm_d(p0, src_v + pitch_x2, 0, 1); + __lsx_vstelm_d(q0, src_v + pitch_x3, 0, 1); + src_v += pitch_x4; + __lsx_vstelm_d(q1, src_v, 0, 1); + src_v += pitch; + __lsx_vstelm_d(q2, src_v, 0, 1); +} + +static inline void mbloop_filter_vertical_edge_y_lsx(uint8_t *src, + int32_t pitch, + const uint8_t b_limit_in, + const uint8_t limit_in, + const uint8_t thresh_in) { + uint8_t *temp_src; + int32_t pitch_x2 = pitch << 1; + int32_t pitch_x3 = pitch_x2 + pitch; + int32_t pitch_x4 = pitch << 2; + + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i mask, hev, flat, thresh, limit, b_limit; + __m128i row0, row1, row2, row3, row4, row5, row6, row7, row8; + __m128i row9, row10, row11, row12, row13, row14, row15; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + + DUP2_ARG2(__lsx_vldrepl_b, &b_limit_in, 0, &limit_in, 0, b_limit, limit); + thresh = __lsx_vldrepl_b(&thresh_in, 0); + temp_src = src - 4; + DUP4_ARG2(__lsx_vldx, temp_src, 0, temp_src, pitch, temp_src, pitch_x2, + temp_src, pitch_x3, row0, row1, row2, row3); + temp_src += pitch_x4; + DUP4_ARG2(__lsx_vldx, temp_src, 0, temp_src, pitch, temp_src, pitch_x2, + temp_src, pitch_x3, row4, row5, row6, row7); + temp_src += pitch_x4; + DUP4_ARG2(__lsx_vldx, temp_src, 0, temp_src, pitch, temp_src, pitch_x2, + temp_src, pitch_x3, row8, row9, row10, row11); + temp_src += pitch_x4; + DUP4_ARG2(__lsx_vldx, temp_src, 0, temp_src, pitch, temp_src, pitch_x2, + temp_src, pitch_x3, row12, row13, row14, row15); + temp_src -= pitch_x4; + LSX_TRANSPOSE16x8_B(row0, row1, row2, row3, row4, row5, row6, row7, row8, + row9, row10, row11, row12, row13, row14, row15, p3, p2, + p1, p0, q0, q1, q2, q3); + + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + VP8_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev); + DUP2_ARG2(__lsx_vilvl_b, p1, p2, q0, p0, tmp0, tmp1); + tmp3 = __lsx_vilvl_h(tmp1, tmp0); + tmp4 = __lsx_vilvh_h(tmp1, tmp0); + DUP2_ARG2(__lsx_vilvh_b, p1, p2, q0, p0, tmp0, tmp1); + tmp6 = __lsx_vilvl_h(tmp1, tmp0); + tmp7 = __lsx_vilvh_h(tmp1, tmp0); + tmp2 = __lsx_vilvl_b(q2, q1); + tmp5 = __lsx_vilvh_b(q2, q1); + + temp_src = src - 3; + VP8_ST6x1_B(tmp3, 0, tmp2, 0, temp_src, 4); + temp_src += pitch; + VP8_ST6x1_B(tmp3, 1, tmp2, 1, temp_src, 4); + temp_src += pitch; + VP8_ST6x1_B(tmp3, 2, tmp2, 2, temp_src, 4); + temp_src += pitch; + VP8_ST6x1_B(tmp3, 3, tmp2, 3, temp_src, 4); + temp_src += pitch; + VP8_ST6x1_B(tmp4, 0, tmp2, 4, temp_src, 4); + temp_src += pitch; + VP8_ST6x1_B(tmp4, 1, tmp2, 5, temp_src, 4); + temp_src += pitch; + VP8_ST6x1_B(tmp4, 2, tmp2, 6, temp_src, 4); + temp_src += pitch; + VP8_ST6x1_B(tmp4, 3, tmp2, 7, temp_src, 4); + temp_src += pitch; + VP8_ST6x1_B(tmp6, 0, tmp5, 0, temp_src, 4); + temp_src += pitch; + VP8_ST6x1_B(tmp6, 1, tmp5, 1, temp_src, 4); + temp_src += pitch; + VP8_ST6x1_B(tmp6, 2, tmp5, 2, temp_src, 4); + temp_src += pitch; + VP8_ST6x1_B(tmp6, 3, tmp5, 3, temp_src, 4); + temp_src += pitch; + VP8_ST6x1_B(tmp7, 0, tmp5, 4, temp_src, 4); + temp_src += pitch; + VP8_ST6x1_B(tmp7, 1, tmp5, 5, temp_src, 4); + temp_src += pitch; + VP8_ST6x1_B(tmp7, 2, tmp5, 6, temp_src, 4); + temp_src += pitch; + VP8_ST6x1_B(tmp7, 3, tmp5, 7, temp_src, 4); +} + +static inline void mbloop_filter_vertical_edge_uv_lsx( + uint8_t *src_u, uint8_t *src_v, int32_t pitch, const uint8_t b_limit_in, + const uint8_t limit_in, const uint8_t thresh_in) { + int32_t pitch_x2 = pitch << 1; + int32_t pitch_x3 = pitch_x2 + pitch; + int32_t pitch_x4 = pitch << 2; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i mask, hev, flat, thresh, limit, b_limit; + __m128i row0, row1, row2, row3, row4, row5, row6, row7, row8; + __m128i row9, row10, row11, row12, row13, row14, row15; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + + DUP2_ARG2(__lsx_vldrepl_b, &b_limit_in, 0, &limit_in, 0, b_limit, limit); + thresh = __lsx_vldrepl_b(&thresh_in, 0); + + src_u -= 4; + DUP4_ARG2(__lsx_vldx, src_u, 0, src_u, pitch, src_u, pitch_x2, src_u, + pitch_x3, row0, row1, row2, row3); + src_u += pitch_x4; + DUP4_ARG2(__lsx_vldx, src_u, 0, src_u, pitch, src_u, pitch_x2, src_u, + pitch_x3, row4, row5, row6, row7); + src_v -= 4; + DUP4_ARG2(__lsx_vldx, src_v, 0, src_v, pitch, src_v, pitch_x2, src_v, + pitch_x3, row8, row9, row10, row11); + src_v += pitch_x4; + DUP4_ARG2(__lsx_vldx, src_v, 0, src_v, pitch, src_v, pitch_x2, src_v, + pitch_x3, row12, row13, row14, row15); + LSX_TRANSPOSE16x8_B(row0, row1, row2, row3, row4, row5, row6, row7, row8, + row9, row10, row11, row12, row13, row14, row15, p3, p2, + p1, p0, q0, q1, q2, q3); + + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + VP8_MBFILTER(p2, p1, p0, q0, q1, q2, mask, hev); + + DUP2_ARG2(__lsx_vilvl_b, p1, p2, q0, p0, tmp0, tmp1); + tmp3 = __lsx_vilvl_h(tmp1, tmp0); + tmp4 = __lsx_vilvh_h(tmp1, tmp0); + DUP2_ARG2(__lsx_vilvh_b, p1, p2, q0, p0, tmp0, tmp1); + tmp6 = __lsx_vilvl_h(tmp1, tmp0); + tmp7 = __lsx_vilvh_h(tmp1, tmp0); + tmp2 = __lsx_vilvl_b(q2, q1); + tmp5 = __lsx_vilvh_b(q2, q1); + + src_u += 1 - pitch_x4; + VP8_ST6x1_B(tmp3, 0, tmp2, 0, src_u, 4); + src_u += pitch; + VP8_ST6x1_B(tmp3, 1, tmp2, 1, src_u, 4); + src_u += pitch; + VP8_ST6x1_B(tmp3, 2, tmp2, 2, src_u, 4); + src_u += pitch; + VP8_ST6x1_B(tmp3, 3, tmp2, 3, src_u, 4); + src_u += pitch; + VP8_ST6x1_B(tmp4, 0, tmp2, 4, src_u, 4); + src_u += pitch; + VP8_ST6x1_B(tmp4, 1, tmp2, 5, src_u, 4); + src_u += pitch; + VP8_ST6x1_B(tmp4, 2, tmp2, 6, src_u, 4); + src_u += pitch; + VP8_ST6x1_B(tmp4, 3, tmp2, 7, src_u, 4); + + src_v += 1 - pitch_x4; + VP8_ST6x1_B(tmp6, 0, tmp5, 0, src_v, 4); + src_v += pitch; + VP8_ST6x1_B(tmp6, 1, tmp5, 1, src_v, 4); + src_v += pitch; + VP8_ST6x1_B(tmp6, 2, tmp5, 2, src_v, 4); + src_v += pitch; + VP8_ST6x1_B(tmp6, 3, tmp5, 3, src_v, 4); + src_v += pitch; + VP8_ST6x1_B(tmp7, 0, tmp5, 4, src_v, 4); + src_v += pitch; + VP8_ST6x1_B(tmp7, 1, tmp5, 5, src_v, 4); + src_v += pitch; + VP8_ST6x1_B(tmp7, 2, tmp5, 6, src_v, 4); + src_v += pitch; + VP8_ST6x1_B(tmp7, 3, tmp5, 7, src_v, 4); +} + +void vp8_loop_filter_mbh_lsx(uint8_t *src_y, uint8_t *src_u, uint8_t *src_v, + int32_t pitch_y, int32_t pitch_u_v, + loop_filter_info *lpf_info_ptr) { + mbloop_filter_horizontal_edge_y_lsx(src_y, pitch_y, *lpf_info_ptr->mblim, + *lpf_info_ptr->lim, + *lpf_info_ptr->hev_thr); + if (src_u) { + mbloop_filter_horizontal_edge_uv_lsx( + src_u, src_v, pitch_u_v, *lpf_info_ptr->mblim, *lpf_info_ptr->lim, + *lpf_info_ptr->hev_thr); + } +} + +void vp8_loop_filter_mbv_lsx(uint8_t *src_y, uint8_t *src_u, uint8_t *src_v, + int32_t pitch_y, int32_t pitch_u_v, + loop_filter_info *lpf_info_ptr) { + mbloop_filter_vertical_edge_y_lsx(src_y, pitch_y, *lpf_info_ptr->mblim, + *lpf_info_ptr->lim, *lpf_info_ptr->hev_thr); + if (src_u) { + mbloop_filter_vertical_edge_uv_lsx(src_u, src_v, pitch_u_v, + *lpf_info_ptr->mblim, *lpf_info_ptr->lim, + *lpf_info_ptr->hev_thr); + } +} + +void vp8_loop_filter_bh_lsx(uint8_t *src_y, uint8_t *src_u, uint8_t *src_v, + int32_t pitch_y, int32_t pitch_u_v, + loop_filter_info *lpf_info_ptr) { + loop_filter_horizontal_4_dual_lsx(src_y + 4 * pitch_y, pitch_y, + lpf_info_ptr->blim, lpf_info_ptr->lim, + lpf_info_ptr->hev_thr, lpf_info_ptr->blim, + lpf_info_ptr->lim, lpf_info_ptr->hev_thr); + loop_filter_horizontal_4_dual_lsx(src_y + 8 * pitch_y, pitch_y, + lpf_info_ptr->blim, lpf_info_ptr->lim, + lpf_info_ptr->hev_thr, lpf_info_ptr->blim, + lpf_info_ptr->lim, lpf_info_ptr->hev_thr); + loop_filter_horizontal_4_dual_lsx(src_y + 12 * pitch_y, pitch_y, + lpf_info_ptr->blim, lpf_info_ptr->lim, + lpf_info_ptr->hev_thr, lpf_info_ptr->blim, + lpf_info_ptr->lim, lpf_info_ptr->hev_thr); + if (src_u) { + loop_filter_horizontal_edge_uv_lsx( + src_u + (4 * pitch_u_v), src_v + (4 * pitch_u_v), pitch_u_v, + *lpf_info_ptr->blim, *lpf_info_ptr->lim, *lpf_info_ptr->hev_thr); + } +} + +void vp8_loop_filter_bv_lsx(uint8_t *src_y, uint8_t *src_u, uint8_t *src_v, + int32_t pitch_y, int32_t pitch_u_v, + loop_filter_info *lpf_info_ptr) { + loop_filter_vertical_4_dual_lsx(src_y + 4, pitch_y, lpf_info_ptr->blim, + lpf_info_ptr->lim, lpf_info_ptr->hev_thr, + lpf_info_ptr->blim, lpf_info_ptr->lim, + lpf_info_ptr->hev_thr); + loop_filter_vertical_4_dual_lsx(src_y + 8, pitch_y, lpf_info_ptr->blim, + lpf_info_ptr->lim, lpf_info_ptr->hev_thr, + lpf_info_ptr->blim, lpf_info_ptr->lim, + lpf_info_ptr->hev_thr); + loop_filter_vertical_4_dual_lsx(src_y + 12, pitch_y, lpf_info_ptr->blim, + lpf_info_ptr->lim, lpf_info_ptr->hev_thr, + lpf_info_ptr->blim, lpf_info_ptr->lim, + lpf_info_ptr->hev_thr); + if (src_u) { + loop_filter_vertical_edge_uv_lsx(src_u + 4, src_v + 4, pitch_u_v, + *lpf_info_ptr->blim, *lpf_info_ptr->lim, + *lpf_info_ptr->hev_thr); + } +} diff --git a/media/libvpx/libvpx/vp8/common/loongarch/sixtap_filter_lsx.c b/media/libvpx/libvpx/vp8/common/loongarch/sixtap_filter_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vp8/common/loongarch/sixtap_filter_lsx.c @@ -0,0 +1,1164 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Lu Wang + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "./vp8_rtcd.h" +#include "vp8/common/filter.h" +#include "vpx_ports/mem.h" +#include "vpx_util/loongson_intrinsics.h" + +DECLARE_ALIGNED(16, static const int8_t, vp8_subpel_filters_lsx[7][8]) = { + { 0, -6, 123, 12, -1, 0, 0, 0 }, + { 2, -11, 108, 36, -8, 1, 0, 0 }, /* New 1/4 pel 6 tap filter */ + { 0, -9, 93, 50, -6, 0, 0, 0 }, + { 3, -16, 77, 77, -16, 3, 0, 0 }, /* New 1/2 pel 6 tap filter */ + { 0, -6, 50, 93, -9, 0, 0, 0 }, + { 1, -8, 36, 108, -11, 2, 0, 0 }, /* New 1/4 pel 6 tap filter */ + { 0, -1, 12, 123, -6, 0, 0, 0 }, +}; + +static const uint8_t vp8_mc_filt_mask_arr[16 * 3] = { + /* 8 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, + /* 4 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20, + /* 4 width cases */ + 8, 9, 9, 10, 10, 11, 11, 12, 24, 25, 25, 26, 26, 27, 27, 28 +}; + +#define DPADD_H3(in0, in1, in2, coeff0, coeff1, coeff2) \ + ({ \ + __m128i out0_m; \ + \ + out0_m = __lsx_vdp2_h_b(in0, coeff0); \ + out0_m = __lsx_vdp2add_h_b(out0_m, in1, coeff1); \ + out0_m = __lsx_vdp2add_h_b(out0_m, in2, coeff2); \ + \ + out0_m; \ + }) + +#define HORIZ_6TAP_FILT(src0, src1, mask0, mask1, mask2, filt_h0, filt_h1, \ + filt_h2) \ + ({ \ + __m128i vec0_m, vec1_m, vec2_m; \ + __m128i hz_out_m; \ + \ + DUP2_ARG3(__lsx_vshuf_b, src0, src1, mask0, src0, src1, mask1, vec0_m, \ + vec1_m); \ + vec2_m = __lsx_vshuf_b(src0, src1, mask2); \ + hz_out_m = DPADD_H3(vec0_m, vec1_m, vec2_m, filt_h0, filt_h1, filt_h2); \ + \ + hz_out_m = __lsx_vsrari_h(hz_out_m, VP8_FILTER_SHIFT); \ + hz_out_m = __lsx_vsat_h(hz_out_m, 7); \ + \ + hz_out_m; \ + }) + +#define HORIZ_6TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, \ + mask2, filt0, filt1, filt2, out0, out1, \ + out2, out3) \ + ({ \ + __m128i vec0_m, vec1_m, vec2_m, vec3_m, vec4_m, vec5_m, vec6_m, vec7_m; \ + \ + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src1, mask0, vec0_m, \ + vec1_m); \ + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask0, src3, src3, mask0, vec2_m, \ + vec3_m); \ + DUP4_ARG2(__lsx_vdp2_h_b, vec0_m, filt0, vec1_m, filt0, vec2_m, filt0, \ + vec3_m, filt0, out0, out1, out2, out3); \ + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask1, src1, src1, mask1, vec0_m, \ + vec1_m); \ + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask1, src3, src3, mask1, vec2_m, \ + vec3_m); \ + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask2, src1, src1, mask2, vec4_m, \ + vec5_m); \ + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask2, src3, src3, mask2, vec6_m, \ + vec7_m); \ + DUP4_ARG3(__lsx_vdp2add_h_b, out0, vec0_m, filt1, out1, vec1_m, filt1, \ + out2, vec2_m, filt1, out3, vec3_m, filt1, out0, out1, out2, \ + out3); \ + DUP4_ARG3(__lsx_vdp2add_h_b, out0, vec4_m, filt2, out1, vec5_m, filt2, \ + out2, vec6_m, filt2, out3, vec7_m, filt2, out0, out1, out2, \ + out3); \ + }) + +#define FILT_4TAP_DPADD_H(vec0, vec1, filt0, filt1) \ + ({ \ + __m128i tmp0; \ + \ + tmp0 = __lsx_vdp2_h_b(vec0, filt0); \ + tmp0 = __lsx_vdp2add_h_b(tmp0, vec1, filt1); \ + \ + tmp0; \ + }) + +#define HORIZ_4TAP_FILT(src0, src1, mask0, mask1, filt_h0, filt_h1) \ + ({ \ + __m128i vec0_m, vec1_m; \ + __m128i hz_out_m; \ + \ + DUP2_ARG3(__lsx_vshuf_b, src0, src1, mask0, src0, src1, mask1, vec0_m, \ + vec1_m); \ + hz_out_m = FILT_4TAP_DPADD_H(vec0_m, vec1_m, filt_h0, filt_h1); \ + hz_out_m = __lsx_vsrari_h(hz_out_m, VP8_FILTER_SHIFT); \ + hz_out_m = __lsx_vsat_h(hz_out_m, 7); \ + \ + hz_out_m; \ + }) + +#define HORIZ_4TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, \ + filt0, filt1, out0, out1, out2, out3) \ + ({ \ + __m128i vec0_m, vec1_m, vec2_m, vec3_m; \ + \ + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src1, mask0, vec0_m, \ + vec1_m); \ + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask0, src3, src3, mask0, vec2_m, \ + vec3_m); \ + DUP4_ARG2(__lsx_vdp2_h_b, vec0_m, filt0, vec1_m, filt0, vec2_m, filt0, \ + vec3_m, filt0, out0, out1, out2, out3); \ + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask1, src1, src1, mask1, vec0_m, \ + vec1_m); \ + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask1, src3, src3, mask1, vec2_m, \ + vec3_m); \ + DUP4_ARG3(__lsx_vdp2add_h_b, out0, vec0_m, filt1, out1, vec1_m, filt1, \ + out2, vec2_m, filt1, out3, vec3_m, filt1, out0, out1, out2, \ + out3); \ + }) + +static void common_hz_6t_8w_lsx(uint8_t *RESTRICT src, int32_t src_stride, + uint8_t *RESTRICT dst, int32_t dst_stride, + const int8_t *filter, int32_t height) { + uint32_t loop_cnt; + int32_t src_stride_x2 = src_stride << 1; + int32_t src_stride_x3 = src_stride_x2 + src_stride; + int32_t src_stride_x4 = src_stride << 2; + int32_t dst_stride_x2 = dst_stride << 1; + int32_t dst_stride_x3 = dst_stride_x2 + dst_stride; + int32_t dst_stride_x4 = dst_stride << 2; + __m128i src0, src1, src2, src3, filt0, filt1, filt2; + __m128i mask0, mask1, mask2, tmp0, tmp1; + __m128i filt, out0, out1, out2, out3; + + mask0 = __lsx_vld(vp8_mc_filt_mask_arr, 0); + src -= 2; + + filt = __lsx_vld(filter, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt0, filt1); + filt2 = __lsx_vreplvei_h(filt, 2); + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src0, src1, src2, src3); + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + src += src_stride_x4; + HORIZ_6TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, filt0, + filt1, filt2, out0, out1, out2, out3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, VP8_FILTER_SHIFT, out3, out2, + VP8_FILTER_SHIFT, tmp0, tmp1); + DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + __lsx_vstelm_d(tmp0, dst, 0, 0); + __lsx_vstelm_d(tmp0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(tmp1, dst + dst_stride_x2, 0, 0); + __lsx_vstelm_d(tmp1, dst + dst_stride_x3, 0, 1); + dst += dst_stride_x4; + + for (loop_cnt = (height >> 2) - 1; loop_cnt--;) { + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src0, src1, src2, src3); + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + src += src_stride_x4; + HORIZ_6TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, + filt0, filt1, filt2, out0, out1, out2, out3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, VP8_FILTER_SHIFT, out3, out2, + VP8_FILTER_SHIFT, tmp0, tmp1); + DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + __lsx_vstelm_d(tmp0, dst, 0, 0); + __lsx_vstelm_d(tmp0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(tmp1, dst + dst_stride_x2, 0, 0); + __lsx_vstelm_d(tmp1, dst + dst_stride_x3, 0, 1); + dst += dst_stride_x4; + } +} + +static void common_hz_6t_16w_lsx(uint8_t *RESTRICT src, int32_t src_stride, + uint8_t *RESTRICT dst, int32_t dst_stride, + const int8_t *filter, int32_t height) { + uint32_t loop_cnt; + int32_t src_stride_x2 = src_stride << 1; + int32_t src_stride_x3 = src_stride_x2 + src_stride; + int32_t src_stride_x4 = src_stride << 2; + int32_t dst_stride_x2 = dst_stride << 1; + int32_t dst_stride_x3 = dst_stride_x2 + dst_stride; + int32_t dst_stride_x4 = dst_stride << 2; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, filt0, filt1, filt2; + __m128i mask0, mask1, mask2, out; + __m128i filt, out0, out1, out2, out3, out4, out5, out6, out7; + + mask0 = __lsx_vld(vp8_mc_filt_mask_arr, 0); + src -= 2; + + filt = __lsx_vld(filter, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt0, filt1); + filt2 = __lsx_vreplvei_h(filt, 2); + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src0, src2, src4, src6); + src += 8; + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src1, src3, src5, src7); + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + DUP4_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src6, 128, src7, 128, src4, + src5, src6, src7); + src += src_stride_x4 - 8; + + HORIZ_6TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, + filt0, filt1, filt2, out0, out1, out2, out3); + HORIZ_6TAP_8WID_4VECS_FILT(src4, src5, src6, src7, mask0, mask1, mask2, + filt0, filt1, filt2, out4, out5, out6, out7); + DUP4_ARG2(__lsx_vsrari_h, out0, VP8_FILTER_SHIFT, out1, VP8_FILTER_SHIFT, + out2, VP8_FILTER_SHIFT, out3, VP8_FILTER_SHIFT, out0, out1, out2, + out3); + DUP4_ARG2(__lsx_vsrari_h, out4, VP8_FILTER_SHIFT, out5, VP8_FILTER_SHIFT, + out6, VP8_FILTER_SHIFT, out7, VP8_FILTER_SHIFT, out4, out5, out6, + out7); + DUP4_ARG2(__lsx_vsat_h, out0, 7, out1, 7, out2, 7, out3, 7, out0, out1, + out2, out3); + DUP4_ARG2(__lsx_vsat_h, out4, 7, out5, 7, out6, 7, out7, 7, out4, out5, + out6, out7); + out = __lsx_vpickev_b(out1, out0); + out = __lsx_vxori_b(out, 128); + __lsx_vst(out, dst, 0); + out = __lsx_vpickev_b(out3, out2); + out = __lsx_vxori_b(out, 128); + __lsx_vstx(out, dst, dst_stride); + out = __lsx_vpickev_b(out5, out4); + out = __lsx_vxori_b(out, 128); + __lsx_vstx(out, dst, dst_stride_x2); + out = __lsx_vpickev_b(out7, out6); + out = __lsx_vxori_b(out, 128); + __lsx_vstx(out, dst, dst_stride_x3); + dst += dst_stride_x4; + } +} + +static void common_vt_6t_8w_lsx(uint8_t *RESTRICT src, int32_t src_stride, + uint8_t *RESTRICT dst, int32_t dst_stride, + const int8_t *filter, int32_t height) { + uint32_t loop_cnt; + int32_t src_stride_x2 = src_stride << 1; + int32_t src_stride_x3 = src_stride_x2 + src_stride; + int32_t src_stride_x4 = src_stride << 2; + int32_t dst_stride_x2 = dst_stride << 1; + int32_t dst_stride_x3 = dst_stride_x2 + dst_stride; + int32_t dst_stride_x4 = dst_stride << 2; + __m128i src0, src1, src2, src3, src4, src7, src8, src9, src10; + __m128i src10_r, src32_r, src76_r, src98_r, src21_r, src43_r, src87_r; + __m128i src109_r, filt0, filt1, filt2; + __m128i tmp0, tmp1; + __m128i filt, out0_r, out1_r, out2_r, out3_r; + + src -= src_stride_x2; + filt = __lsx_vld(filter, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt0, filt1); + filt2 = __lsx_vreplvei_h(filt, 2); + + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src0, src1, src2, src3); + src += src_stride_x4; + src4 = __lsx_vld(src, 0); + src += src_stride; + + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + src4 = __lsx_vxori_b(src4, 128); + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src2, src1, src4, src3, + src10_r, src32_r, src21_r, src43_r); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src7, src8, src9, src10); + DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7, + src8, src9, src10); + src += src_stride_x4; + + DUP4_ARG2(__lsx_vilvl_b, src7, src4, src8, src7, src9, src8, src10, src9, + src76_r, src87_r, src98_r, src109_r); + out0_r = DPADD_H3(src10_r, src32_r, src76_r, filt0, filt1, filt2); + out1_r = DPADD_H3(src21_r, src43_r, src87_r, filt0, filt1, filt2); + out2_r = DPADD_H3(src32_r, src76_r, src98_r, filt0, filt1, filt2); + out3_r = DPADD_H3(src43_r, src87_r, src109_r, filt0, filt1, filt2); + DUP2_ARG3(__lsx_vssrarni_b_h, out1_r, out0_r, VP8_FILTER_SHIFT, out3_r, + out2_r, VP8_FILTER_SHIFT, tmp0, tmp1); + DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + __lsx_vstelm_d(tmp0, dst, 0, 0); + __lsx_vstelm_d(tmp0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(tmp1, dst + dst_stride_x2, 0, 0); + __lsx_vstelm_d(tmp1, dst + dst_stride_x3, 0, 1); + dst += dst_stride_x4; + + src10_r = src76_r; + src32_r = src98_r; + src21_r = src87_r; + src43_r = src109_r; + src4 = src10; + } +} + +static void common_vt_6t_16w_lsx(uint8_t *RESTRICT src, int32_t src_stride, + uint8_t *RESTRICT dst, int32_t dst_stride, + const int8_t *filter, int32_t height) { + uint32_t loop_cnt; + int32_t src_stride_x2 = src_stride << 1; + int32_t src_stride_x3 = src_stride_x2 + src_stride; + int32_t src_stride_x4 = src_stride << 2; + int32_t dst_stride_x2 = dst_stride << 1; + int32_t dst_stride_x3 = dst_stride_x2 + dst_stride; + int32_t dst_stride_x4 = dst_stride << 2; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8; + __m128i src10_r, src32_r, src54_r, src76_r, src21_r, src43_r, src65_r; + __m128i src87_r, src10_l, src32_l, src54_l, src76_l, src21_l, src43_l; + __m128i src65_l, src87_l, filt0, filt1, filt2; + __m128i tmp0, tmp1, tmp2, tmp3; + __m128i filt, out0_r, out1_r, out2_r, out3_r, out0_l, out1_l, out2_l, out3_l; + + src -= src_stride_x2; + filt = __lsx_vld(filter, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt0, filt1); + filt2 = __lsx_vreplvei_h(filt, 2); + + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src0, src1, src2, src3); + src += src_stride_x4; + src4 = __lsx_vldx(src, 0); + src += src_stride; + + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + src4 = __lsx_vxori_b(src4, 128); + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src4, src3, src2, src1, + src10_r, src32_r, src43_r, src21_r); + DUP4_ARG2(__lsx_vilvh_b, src1, src0, src3, src2, src4, src3, src2, src1, + src10_l, src32_l, src43_l, src21_l); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src5, src6, src7, src8); + src += src_stride_x4; + + DUP4_ARG2(__lsx_vxori_b, src5, 128, src6, 128, src7, 128, src8, 128, src5, + src6, src7, src8); + DUP4_ARG2(__lsx_vilvl_b, src5, src4, src6, src5, src7, src6, src8, src7, + src54_r, src65_r, src76_r, src87_r); + DUP4_ARG2(__lsx_vilvh_b, src5, src4, src6, src5, src7, src6, src8, src7, + src54_l, src65_l, src76_l, src87_l); + out0_r = DPADD_H3(src10_r, src32_r, src54_r, filt0, filt1, filt2); + out1_r = DPADD_H3(src21_r, src43_r, src65_r, filt0, filt1, filt2); + out2_r = DPADD_H3(src32_r, src54_r, src76_r, filt0, filt1, filt2); + out3_r = DPADD_H3(src43_r, src65_r, src87_r, filt0, filt1, filt2); + out0_l = DPADD_H3(src10_l, src32_l, src54_l, filt0, filt1, filt2); + out1_l = DPADD_H3(src21_l, src43_l, src65_l, filt0, filt1, filt2); + out2_l = DPADD_H3(src32_l, src54_l, src76_l, filt0, filt1, filt2); + out3_l = DPADD_H3(src43_l, src65_l, src87_l, filt0, filt1, filt2); + DUP4_ARG3(__lsx_vssrarni_b_h, out0_l, out0_r, VP8_FILTER_SHIFT, out1_l, + out1_r, VP8_FILTER_SHIFT, out2_l, out2_r, VP8_FILTER_SHIFT, + out3_l, out3_r, VP8_FILTER_SHIFT, tmp0, tmp1, tmp2, tmp3); + DUP4_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp2, 128, tmp3, 128, tmp0, + tmp1, tmp2, tmp3); + __lsx_vstx(tmp0, dst, 0); + __lsx_vstx(tmp1, dst, dst_stride); + __lsx_vstx(tmp2, dst, dst_stride_x2); + __lsx_vstx(tmp3, dst, dst_stride_x3); + dst += dst_stride_x4; + + src10_r = src54_r; + src32_r = src76_r; + src21_r = src65_r; + src43_r = src87_r; + src10_l = src54_l; + src32_l = src76_l; + src21_l = src65_l; + src43_l = src87_l; + src4 = src8; + } +} + +static void common_hv_6ht_6vt_8w_lsx(uint8_t *RESTRICT src, int32_t src_stride, + uint8_t *RESTRICT dst, int32_t dst_stride, + const int8_t *filter_horiz, + const int8_t *filter_vert, + int32_t height) { + uint32_t loop_cnt; + int32_t src_stride_x2 = src_stride << 1; + int32_t src_stride_x3 = src_stride_x2 + src_stride; + int32_t src_stride_x4 = src_stride << 2; + int32_t dst_stride_x2 = dst_stride << 1; + int32_t dst_stride_x3 = dst_stride_x2 + dst_stride; + int32_t dst_stride_x4 = dst_stride << 2; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8; + __m128i filt_hz0, filt_hz1, filt_hz2; + __m128i mask0, mask1, mask2, vec0, vec1; + __m128i filt, filt_vt0, filt_vt1, filt_vt2; + __m128i hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6; + __m128i hz_out7, hz_out8, out0, out1, out2, out3, out4, out5, out6, out7; + __m128i tmp0, tmp1, tmp2, tmp3; + + mask0 = __lsx_vld(vp8_mc_filt_mask_arr, 0); + src -= (2 + src_stride_x2); + + filt = __lsx_vld(filter_horiz, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt_hz0, filt_hz1); + filt_hz2 = __lsx_vreplvei_h(filt, 2); + + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src0, src1, src2, src3); + src += src_stride_x4; + src4 = __lsx_vldx(src, 0); + src += src_stride; + + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + src4 = __lsx_vxori_b(src4, 128); + + hz_out0 = HORIZ_6TAP_FILT(src0, src0, mask0, mask1, mask2, filt_hz0, filt_hz1, + filt_hz2); + hz_out1 = HORIZ_6TAP_FILT(src1, src1, mask0, mask1, mask2, filt_hz0, filt_hz1, + filt_hz2); + hz_out2 = HORIZ_6TAP_FILT(src2, src2, mask0, mask1, mask2, filt_hz0, filt_hz1, + filt_hz2); + hz_out3 = HORIZ_6TAP_FILT(src3, src3, mask0, mask1, mask2, filt_hz0, filt_hz1, + filt_hz2); + hz_out4 = HORIZ_6TAP_FILT(src4, src4, mask0, mask1, mask2, filt_hz0, filt_hz1, + filt_hz2); + filt = __lsx_vld(filter_vert, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt_vt0, filt_vt1); + filt_vt2 = __lsx_vreplvei_h(filt, 2); + + DUP4_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, hz_out2, + hz_out1, hz_out4, hz_out3, out0, out1, out3, out4); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src5, src6, src7, src8); + src += src_stride_x4; + + DUP4_ARG2(__lsx_vxori_b, src5, 128, src6, 128, src7, 128, src8, 128, src5, + src6, src7, src8); + hz_out5 = HORIZ_6TAP_FILT(src5, src5, mask0, mask1, mask2, filt_hz0, + filt_hz1, filt_hz2); + out2 = __lsx_vpackev_b(hz_out5, hz_out4); + tmp0 = DPADD_H3(out0, out1, out2, filt_vt0, filt_vt1, filt_vt2); + + hz_out6 = HORIZ_6TAP_FILT(src6, src6, mask0, mask1, mask2, filt_hz0, + filt_hz1, filt_hz2); + out5 = __lsx_vpackev_b(hz_out6, hz_out5); + tmp1 = DPADD_H3(out3, out4, out5, filt_vt0, filt_vt1, filt_vt2); + + hz_out7 = HORIZ_6TAP_FILT(src7, src7, mask0, mask1, mask2, filt_hz0, + filt_hz1, filt_hz2); + out7 = __lsx_vpackev_b(hz_out7, hz_out6); + tmp2 = DPADD_H3(out1, out2, out7, filt_vt0, filt_vt1, filt_vt2); + + hz_out8 = HORIZ_6TAP_FILT(src8, src8, mask0, mask1, mask2, filt_hz0, + filt_hz1, filt_hz2); + out6 = __lsx_vpackev_b(hz_out8, hz_out7); + tmp3 = DPADD_H3(out4, out5, out6, filt_vt0, filt_vt1, filt_vt2); + + DUP2_ARG3(__lsx_vssrarni_b_h, tmp1, tmp0, VP8_FILTER_SHIFT, tmp3, tmp2, + VP8_FILTER_SHIFT, vec0, vec1); + DUP2_ARG2(__lsx_vxori_b, vec0, 128, vec1, 128, vec0, vec1); + + __lsx_vstelm_d(vec0, dst, 0, 0); + __lsx_vstelm_d(vec0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(vec1, dst + dst_stride_x2, 0, 0); + __lsx_vstelm_d(vec1, dst + dst_stride_x3, 0, 1); + dst += dst_stride_x4; + + hz_out4 = hz_out8; + out0 = out2; + out1 = out7; + out3 = out5; + out4 = out6; + } +} + +static void common_hv_6ht_6vt_16w_lsx(uint8_t *RESTRICT src, int32_t src_stride, + uint8_t *RESTRICT dst, int32_t dst_stride, + const int8_t *filter_horiz, + const int8_t *filter_vert, + int32_t height) { + common_hv_6ht_6vt_8w_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert, height); + common_hv_6ht_6vt_8w_lsx(src + 8, src_stride, dst + 8, dst_stride, + filter_horiz, filter_vert, height); +} + +static void common_hz_4t_8w_lsx(uint8_t *RESTRICT src, int32_t src_stride, + uint8_t *RESTRICT dst, int32_t dst_stride, + const int8_t *filter, int32_t height) { + uint32_t loop_cnt; + int32_t src_stride_x2 = src_stride << 1; + int32_t src_stride_x3 = src_stride_x2 + src_stride; + int32_t src_stride_x4 = src_stride << 2; + int32_t dst_stride_x2 = dst_stride << 1; + int32_t dst_stride_x3 = dst_stride_x2 + dst_stride; + int32_t dst_stride_x4 = dst_stride << 2; + __m128i src0, src1, src2, src3, filt0, filt1, mask0, mask1; + __m128i tmp0, tmp1; + __m128i filt, out0, out1, out2, out3; + + mask0 = __lsx_vld(vp8_mc_filt_mask_arr, 0); + src -= 1; + + filt = __lsx_vld(filter, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt0, filt1); + mask1 = __lsx_vaddi_bu(mask0, 2); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src0, src1, src2, src3); + src += src_stride_x4; + + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_4TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, filt0, + filt1, out0, out1, out2, out3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, VP8_FILTER_SHIFT, out3, out2, + VP8_FILTER_SHIFT, tmp0, tmp1); + DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + __lsx_vstelm_d(tmp0, dst, 0, 0); + __lsx_vstelm_d(tmp0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(tmp1, dst + dst_stride_x2, 0, 0); + __lsx_vstelm_d(tmp1, dst + dst_stride_x3, 0, 1); + dst += dst_stride_x4; + } +} + +static void common_hz_4t_16w_lsx(uint8_t *RESTRICT src, int32_t src_stride, + uint8_t *RESTRICT dst, int32_t dst_stride, + const int8_t *filter, int32_t height) { + uint32_t loop_cnt; + int32_t src_stride_x2 = src_stride << 1; + int32_t src_stride_x3 = src_stride_x2 + src_stride; + int32_t src_stride_x4 = src_stride << 2; + int32_t dst_stride_x2 = dst_stride << 1; + int32_t dst_stride_x3 = dst_stride_x2 + dst_stride; + int32_t dst_stride_x4 = dst_stride << 2; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i filt0, filt1, mask0, mask1; + __m128i filt, out0, out1, out2, out3, out4, out5, out6, out7; + + mask0 = __lsx_vld(vp8_mc_filt_mask_arr, 0); + src -= 1; + + filt = __lsx_vld(filter, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt0, filt1); + mask1 = __lsx_vaddi_bu(mask0, 2); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src0, src2, src4, src6); + src += 8; + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src1, src3, src5, src7); + src += src_stride_x4 - 8; + + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + DUP4_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src6, 128, src7, 128, src4, + src5, src6, src7); + HORIZ_4TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, filt0, + filt1, out0, out1, out2, out3); + HORIZ_4TAP_8WID_4VECS_FILT(src4, src5, src6, src7, mask0, mask1, filt0, + filt1, out4, out5, out6, out7); + DUP4_ARG3(__lsx_vssrarni_b_h, out1, out0, VP8_FILTER_SHIFT, out3, out2, + VP8_FILTER_SHIFT, out5, out4, VP8_FILTER_SHIFT, out7, out6, + VP8_FILTER_SHIFT, out0, out1, out2, out3); + DUP4_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out2, 128, out3, 128, out0, + out1, out2, out3); + __lsx_vstx(out0, dst, 0); + __lsx_vstx(out1, dst, dst_stride); + __lsx_vstx(out2, dst, dst_stride_x2); + __lsx_vstx(out3, dst, dst_stride_x3); + dst += dst_stride_x4; + } +} + +static void common_vt_4t_8w_lsx(uint8_t *RESTRICT src, int32_t src_stride, + uint8_t *RESTRICT dst, int32_t dst_stride, + const int8_t *filter, int32_t height) { + uint32_t loop_cnt; + int32_t src_stride_x2 = src_stride << 1; + int32_t src_stride_x3 = src_stride_x2 + src_stride; + int32_t src_stride_x4 = src_stride << 2; + int32_t dst_stride_x2 = dst_stride << 1; + int32_t dst_stride_x3 = dst_stride_x2 + dst_stride; + int32_t dst_stride_x4 = dst_stride << 2; + __m128i src0, src1, src2, src7, src8, src9, src10; + __m128i src10_r, src72_r, src98_r, src21_r, src87_r, src109_r, filt0, filt1; + __m128i tmp0, tmp1; + __m128i filt, out0_r, out1_r, out2_r, out3_r; + + src -= src_stride; + filt = __lsx_vld(filter, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt0, filt1); + + DUP2_ARG2(__lsx_vldx, src, 0, src, src_stride, src0, src1); + src2 = __lsx_vldx(src, src_stride_x2); + src += src_stride_x3; + + DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + src2 = __lsx_vxori_b(src2, 128); + DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src10_r, src21_r); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src7, src8, src9, src10); + src += src_stride_x4; + + DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7, + src8, src9, src10); + DUP4_ARG2(__lsx_vilvl_b, src7, src2, src8, src7, src9, src8, src10, src9, + src72_r, src87_r, src98_r, src109_r); + out0_r = FILT_4TAP_DPADD_H(src10_r, src72_r, filt0, filt1); + out1_r = FILT_4TAP_DPADD_H(src21_r, src87_r, filt0, filt1); + out2_r = FILT_4TAP_DPADD_H(src72_r, src98_r, filt0, filt1); + out3_r = FILT_4TAP_DPADD_H(src87_r, src109_r, filt0, filt1); + DUP2_ARG3(__lsx_vssrarni_b_h, out1_r, out0_r, VP8_FILTER_SHIFT, out3_r, + out2_r, VP8_FILTER_SHIFT, tmp0, tmp1); + DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + __lsx_vstelm_d(tmp0, dst, 0, 0); + __lsx_vstelm_d(tmp0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(tmp1, dst + dst_stride_x2, 0, 0); + __lsx_vstelm_d(tmp1, dst + dst_stride_x3, 0, 1); + dst += dst_stride_x4; + + src10_r = src98_r; + src21_r = src109_r; + src2 = src10; + } +} + +static void common_vt_4t_16w_lsx(uint8_t *RESTRICT src, int32_t src_stride, + uint8_t *RESTRICT dst, int32_t dst_stride, + const int8_t *filter, int32_t height) { + uint32_t loop_cnt; + int32_t src_stride_x2 = src_stride << 1; + int32_t src_stride_x3 = src_stride_x2 + src_stride; + int32_t src_stride_x4 = src_stride << 2; + int32_t dst_stride_x2 = dst_stride << 1; + int32_t dst_stride_x3 = dst_stride_x2 + dst_stride; + int32_t dst_stride_x4 = dst_stride << 2; + __m128i src0, src1, src2, src3, src4, src5, src6; + __m128i src10_r, src32_r, src54_r, src21_r, src43_r, src65_r, src10_l; + __m128i src32_l, src54_l, src21_l, src43_l, src65_l, filt0, filt1; + __m128i tmp0, tmp1, tmp2, tmp3; + __m128i filt, out0_r, out1_r, out2_r, out3_r, out0_l, out1_l, out2_l, out3_l; + + src -= src_stride; + filt = __lsx_vld(filter, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt0, filt1); + + DUP2_ARG2(__lsx_vldx, src, 0, src, src_stride, src0, src1); + src2 = __lsx_vldx(src, src_stride_x2); + src += src_stride_x3; + + DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + src2 = __lsx_vxori_b(src2, 128); + DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src10_r, src21_r); + DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, src10_l, src21_l); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src3, src4, src5, src6); + src += src_stride_x4; + + DUP4_ARG2(__lsx_vxori_b, src3, 128, src4, 128, src5, 128, src6, 128, src3, + src4, src5, src6); + DUP4_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, src5, src4, src6, src5, + src32_r, src43_r, src54_r, src65_r); + DUP4_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, src5, src4, src6, src5, + src32_l, src43_l, src54_l, src65_l); + out0_r = FILT_4TAP_DPADD_H(src10_r, src32_r, filt0, filt1); + out1_r = FILT_4TAP_DPADD_H(src21_r, src43_r, filt0, filt1); + out2_r = FILT_4TAP_DPADD_H(src32_r, src54_r, filt0, filt1); + out3_r = FILT_4TAP_DPADD_H(src43_r, src65_r, filt0, filt1); + out0_l = FILT_4TAP_DPADD_H(src10_l, src32_l, filt0, filt1); + out1_l = FILT_4TAP_DPADD_H(src21_l, src43_l, filt0, filt1); + out2_l = FILT_4TAP_DPADD_H(src32_l, src54_l, filt0, filt1); + out3_l = FILT_4TAP_DPADD_H(src43_l, src65_l, filt0, filt1); + DUP4_ARG3(__lsx_vssrarni_b_h, out0_l, out0_r, VP8_FILTER_SHIFT, out1_l, + out1_r, VP8_FILTER_SHIFT, out2_l, out2_r, VP8_FILTER_SHIFT, + out3_l, out3_r, VP8_FILTER_SHIFT, tmp0, tmp1, tmp2, tmp3); + DUP4_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp2, 128, tmp3, 128, tmp0, + tmp1, tmp2, tmp3); + __lsx_vstx(tmp0, dst, 0); + __lsx_vstx(tmp1, dst, dst_stride); + __lsx_vstx(tmp2, dst, dst_stride_x2); + __lsx_vstx(tmp3, dst, dst_stride_x3); + dst += dst_stride_x4; + + src10_r = src54_r; + src21_r = src65_r; + src10_l = src54_l; + src21_l = src65_l; + src2 = src6; + } +} + +static inline void common_hv_4ht_4vt_8w_lsx( + uint8_t *RESTRICT src, int32_t src_stride, uint8_t *RESTRICT dst, + int32_t dst_stride, const int8_t *filter_horiz, const int8_t *filter_vert, + int32_t height) { + uint32_t loop_cnt; + int32_t src_stride_x2 = src_stride << 1; + int32_t src_stride_x3 = src_stride_x2 + src_stride; + int32_t src_stride_x4 = src_stride << 2; + int32_t dst_stride_x2 = dst_stride << 1; + int32_t dst_stride_x3 = dst_stride_x2 + dst_stride; + int32_t dst_stride_x4 = dst_stride << 2; + __m128i src0, src1, src2, src3, src4, src5, src6, filt_hz0, filt_hz1; + __m128i mask0, mask1, out0, out1; + __m128i filt, filt_vt0, filt_vt1, tmp0, tmp1, tmp2, tmp3; + __m128i hz_out0, hz_out1, hz_out2, hz_out3; + __m128i vec0, vec1, vec2, vec3, vec4; + + mask0 = __lsx_vld(vp8_mc_filt_mask_arr, 0); + src -= 1 + src_stride; + + filt = __lsx_vld(filter_horiz, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt_hz0, filt_hz1); + mask1 = __lsx_vaddi_bu(mask0, 2); + + DUP2_ARG2(__lsx_vldx, src, 0, src, src_stride, src0, src1); + src2 = __lsx_vldx(src, src_stride_x2); + src += src_stride_x3; + + DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + src2 = __lsx_vxori_b(src2, 128); + hz_out0 = HORIZ_4TAP_FILT(src0, src0, mask0, mask1, filt_hz0, filt_hz1); + hz_out1 = HORIZ_4TAP_FILT(src1, src1, mask0, mask1, filt_hz0, filt_hz1); + hz_out2 = HORIZ_4TAP_FILT(src2, src2, mask0, mask1, filt_hz0, filt_hz1); + DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out2, hz_out1, vec0, vec2); + + filt = __lsx_vld(filter_vert, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt_vt0, filt_vt1); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src3, src4, src5, src6); + src += src_stride_x4; + + DUP4_ARG2(__lsx_vxori_b, src3, 128, src4, 128, src5, 128, src6, 128, src3, + src4, src5, src6); + hz_out3 = HORIZ_4TAP_FILT(src3, src3, mask0, mask1, filt_hz0, filt_hz1); + vec1 = __lsx_vpackev_b(hz_out3, hz_out2); + tmp0 = FILT_4TAP_DPADD_H(vec0, vec1, filt_vt0, filt_vt1); + + hz_out0 = HORIZ_4TAP_FILT(src4, src4, mask0, mask1, filt_hz0, filt_hz1); + vec3 = __lsx_vpackev_b(hz_out0, hz_out3); + tmp1 = FILT_4TAP_DPADD_H(vec2, vec3, filt_vt0, filt_vt1); + + hz_out1 = HORIZ_4TAP_FILT(src5, src5, mask0, mask1, filt_hz0, filt_hz1); + vec4 = __lsx_vpackev_b(hz_out1, hz_out0); + tmp2 = FILT_4TAP_DPADD_H(vec1, vec4, filt_vt0, filt_vt1); + + hz_out2 = HORIZ_4TAP_FILT(src6, src6, mask0, mask1, filt_hz0, filt_hz1); + DUP2_ARG2(__lsx_vpackev_b, hz_out0, hz_out3, hz_out2, hz_out1, vec0, vec1); + tmp3 = FILT_4TAP_DPADD_H(vec0, vec1, filt_vt0, filt_vt1); + + DUP2_ARG3(__lsx_vssrarni_b_h, tmp1, tmp0, 7, tmp3, tmp2, 7, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + __lsx_vstelm_d(out0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(out1, dst + dst_stride_x2, 0, 0); + __lsx_vstelm_d(out1, dst + dst_stride_x3, 0, 1); + dst += dst_stride_x4; + + vec0 = vec4; + vec2 = vec1; + } +} + +static void common_hv_4ht_4vt_16w_lsx(uint8_t *RESTRICT src, int32_t src_stride, + uint8_t *RESTRICT dst, int32_t dst_stride, + const int8_t *filter_horiz, + const int8_t *filter_vert, + int32_t height) { + common_hv_4ht_4vt_8w_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert, height); + common_hv_4ht_4vt_8w_lsx(src + 8, src_stride, dst + 8, dst_stride, + filter_horiz, filter_vert, height); +} + +static inline void common_hv_6ht_4vt_8w_lsx( + uint8_t *RESTRICT src, int32_t src_stride, uint8_t *RESTRICT dst, + int32_t dst_stride, const int8_t *filter_horiz, const int8_t *filter_vert, + int32_t height) { + uint32_t loop_cnt; + int32_t src_stride_x2 = src_stride << 1; + int32_t src_stride_x3 = src_stride_x2 + src_stride; + int32_t src_stride_x4 = src_stride << 2; + int32_t dst_stride_x2 = dst_stride << 1; + int32_t dst_stride_x3 = dst_stride_x2 + dst_stride; + int32_t dst_stride_x4 = dst_stride << 2; + + __m128i src0, src1, src2, src3, src4, src5, src6; + __m128i filt_hz0, filt_hz1, filt_hz2, mask0, mask1, mask2; + __m128i filt, filt_vt0, filt_vt1, hz_out0, hz_out1, hz_out2, hz_out3; + __m128i tmp0, tmp1, tmp2, tmp3, vec0, vec1, vec2, vec3; + __m128i out0, out1; + + mask0 = __lsx_vld(vp8_mc_filt_mask_arr, 0); + src -= (2 + src_stride); + + filt = __lsx_vld(filter_horiz, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt_hz0, filt_hz1); + filt_hz2 = __lsx_vreplvei_h(filt, 2); + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + + DUP2_ARG2(__lsx_vldx, src, 0, src, src_stride, src0, src1); + src2 = __lsx_vldx(src, src_stride_x2); + src += src_stride_x3; + + DUP2_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src0, src1); + src2 = __lsx_vxori_b(src2, 128); + hz_out0 = HORIZ_6TAP_FILT(src0, src0, mask0, mask1, mask2, filt_hz0, filt_hz1, + filt_hz2); + hz_out1 = HORIZ_6TAP_FILT(src1, src1, mask0, mask1, mask2, filt_hz0, filt_hz1, + filt_hz2); + hz_out2 = HORIZ_6TAP_FILT(src2, src2, mask0, mask1, mask2, filt_hz0, filt_hz1, + filt_hz2); + DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out2, hz_out1, vec0, vec2); + + filt = __lsx_vld(filter_vert, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt_vt0, filt_vt1); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src3, src4, src5, src6); + src += src_stride_x4; + DUP4_ARG2(__lsx_vxori_b, src3, 128, src4, 128, src5, 128, src6, 128, src3, + src4, src5, src6); + + hz_out3 = HORIZ_6TAP_FILT(src3, src3, mask0, mask1, mask2, filt_hz0, + filt_hz1, filt_hz2); + vec1 = __lsx_vpackev_b(hz_out3, hz_out2); + tmp0 = FILT_4TAP_DPADD_H(vec0, vec1, filt_vt0, filt_vt1); + + hz_out0 = HORIZ_6TAP_FILT(src4, src4, mask0, mask1, mask2, filt_hz0, + filt_hz1, filt_hz2); + vec3 = __lsx_vpackev_b(hz_out0, hz_out3); + tmp1 = FILT_4TAP_DPADD_H(vec2, vec3, filt_vt0, filt_vt1); + + hz_out1 = HORIZ_6TAP_FILT(src5, src5, mask0, mask1, mask2, filt_hz0, + filt_hz1, filt_hz2); + vec0 = __lsx_vpackev_b(hz_out1, hz_out0); + tmp2 = FILT_4TAP_DPADD_H(vec1, vec0, filt_vt0, filt_vt1); + + hz_out2 = HORIZ_6TAP_FILT(src6, src6, mask0, mask1, mask2, filt_hz0, + filt_hz1, filt_hz2); + DUP2_ARG2(__lsx_vpackev_b, hz_out0, hz_out3, hz_out2, hz_out1, vec1, vec2); + tmp3 = FILT_4TAP_DPADD_H(vec1, vec2, filt_vt0, filt_vt1); + + DUP2_ARG3(__lsx_vssrarni_b_h, tmp1, tmp0, 7, tmp3, tmp2, 7, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + __lsx_vstelm_d(out0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(out1, dst + dst_stride_x2, 0, 0); + __lsx_vstelm_d(out1, dst + dst_stride_x3, 0, 1); + dst += dst_stride_x4; + } +} + +static void common_hv_6ht_4vt_16w_lsx(uint8_t *RESTRICT src, int32_t src_stride, + uint8_t *RESTRICT dst, int32_t dst_stride, + const int8_t *filter_horiz, + const int8_t *filter_vert, + int32_t height) { + common_hv_6ht_4vt_8w_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert, height); + common_hv_6ht_4vt_8w_lsx(src + 8, src_stride, dst + 8, dst_stride, + filter_horiz, filter_vert, height); +} + +static inline void common_hv_4ht_6vt_8w_lsx( + uint8_t *RESTRICT src, int32_t src_stride, uint8_t *RESTRICT dst, + int32_t dst_stride, const int8_t *filter_horiz, const int8_t *filter_vert, + int32_t height) { + uint32_t loop_cnt; + int32_t src_stride_x2 = src_stride << 1; + int32_t src_stride_x3 = src_stride_x2 + src_stride; + int32_t src_stride_x4 = src_stride << 2; + int32_t dst_stride_x2 = dst_stride << 1; + int32_t dst_stride_x3 = dst_stride_x2 + dst_stride; + int32_t dst_stride_x4 = dst_stride << 2; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8; + __m128i filt_hz0, filt_hz1, mask0, mask1; + __m128i filt, filt_vt0, filt_vt1, filt_vt2, tmp0, tmp1, tmp2, tmp3; + __m128i hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6; + __m128i hz_out7, hz_out8, out0, out1, out2, out3, out4, out5, out6, out7; + __m128i vec0, vec1; + + mask0 = __lsx_vld(vp8_mc_filt_mask_arr, 0); + src -= 1 + src_stride_x2; + + filt = __lsx_vld(filter_horiz, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt_hz0, filt_hz1); + mask1 = __lsx_vaddi_bu(mask0, 2); + + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src0, src1, src2, src3); + src += src_stride_x4; + src4 = __lsx_vld(src, 0); + src += src_stride; + + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + src4 = __lsx_vxori_b(src4, 128); + hz_out0 = HORIZ_4TAP_FILT(src0, src0, mask0, mask1, filt_hz0, filt_hz1); + hz_out1 = HORIZ_4TAP_FILT(src1, src1, mask0, mask1, filt_hz0, filt_hz1); + hz_out2 = HORIZ_4TAP_FILT(src2, src2, mask0, mask1, filt_hz0, filt_hz1); + hz_out3 = HORIZ_4TAP_FILT(src3, src3, mask0, mask1, filt_hz0, filt_hz1); + hz_out4 = HORIZ_4TAP_FILT(src4, src4, mask0, mask1, filt_hz0, filt_hz1); + DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, out0, out1); + DUP2_ARG2(__lsx_vpackev_b, hz_out2, hz_out1, hz_out4, hz_out3, out3, out4); + + filt = __lsx_vld(filter_vert, 0); + DUP2_ARG2(__lsx_vreplvei_h, filt, 0, filt, 1, filt_vt0, filt_vt1); + filt_vt2 = __lsx_vreplvei_h(filt, 2); + + for (loop_cnt = (height >> 2); loop_cnt--;) { + DUP4_ARG2(__lsx_vldx, src, 0, src, src_stride, src, src_stride_x2, src, + src_stride_x3, src5, src6, src7, src8); + src += src_stride_x4; + + DUP4_ARG2(__lsx_vxori_b, src5, 128, src6, 128, src7, 128, src8, 128, src5, + src6, src7, src8); + hz_out5 = HORIZ_4TAP_FILT(src5, src5, mask0, mask1, filt_hz0, filt_hz1); + out2 = __lsx_vpackev_b(hz_out5, hz_out4); + tmp0 = DPADD_H3(out0, out1, out2, filt_vt0, filt_vt1, filt_vt2); + + hz_out6 = HORIZ_4TAP_FILT(src6, src6, mask0, mask1, filt_hz0, filt_hz1); + out5 = __lsx_vpackev_b(hz_out6, hz_out5); + tmp1 = DPADD_H3(out3, out4, out5, filt_vt0, filt_vt1, filt_vt2); + + hz_out7 = HORIZ_4TAP_FILT(src7, src7, mask0, mask1, filt_hz0, filt_hz1); + out6 = __lsx_vpackev_b(hz_out7, hz_out6); + tmp2 = DPADD_H3(out1, out2, out6, filt_vt0, filt_vt1, filt_vt2); + + hz_out8 = HORIZ_4TAP_FILT(src8, src8, mask0, mask1, filt_hz0, filt_hz1); + out7 = __lsx_vpackev_b(hz_out8, hz_out7); + tmp3 = DPADD_H3(out4, out5, out7, filt_vt0, filt_vt1, filt_vt2); + DUP2_ARG3(__lsx_vssrarni_b_h, tmp1, tmp0, 7, tmp3, tmp2, 7, vec0, vec1); + DUP2_ARG2(__lsx_vxori_b, vec0, 128, vec1, 128, vec0, vec1); + __lsx_vstelm_d(vec0, dst, 0, 0); + __lsx_vstelm_d(vec0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(vec1, dst + dst_stride_x2, 0, 0); + __lsx_vstelm_d(vec1, dst + dst_stride_x3, 0, 1); + dst += dst_stride_x4; + hz_out4 = hz_out8; + out0 = out2; + out1 = out6; + out3 = out5; + out4 = out7; + } +} + +static void common_hv_4ht_6vt_16w_lsx(uint8_t *RESTRICT src, int32_t src_stride, + uint8_t *RESTRICT dst, int32_t dst_stride, + const int8_t *filter_horiz, + const int8_t *filter_vert, + int32_t height) { + common_hv_4ht_6vt_8w_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert, height); + common_hv_4ht_6vt_8w_lsx(src + 8, src_stride, dst + 8, dst_stride, + filter_horiz, filter_vert, height); +} + +typedef void (*PVp8SixtapPredictFunc1)( + uint8_t *RESTRICT src, int32_t src_stride, uint8_t *RESTRICT dst, + int32_t dst_stride, const int8_t *filter_horiz, const int8_t *filter_vert, + int32_t height); + +typedef void (*PVp8SixtapPredictFunc2)(uint8_t *RESTRICT src, + int32_t src_stride, + uint8_t *RESTRICT dst, + int32_t dst_stride, const int8_t *filter, + int32_t height); + +void vp8_sixtap_predict8x8_lsx(uint8_t *RESTRICT src, int32_t src_stride, + int32_t xoffset, int32_t yoffset, + uint8_t *RESTRICT dst, int32_t dst_stride) { + const int8_t *h_filter = vp8_subpel_filters_lsx[xoffset - 1]; + const int8_t *v_filter = vp8_subpel_filters_lsx[yoffset - 1]; + + static PVp8SixtapPredictFunc1 Predict8x8Funcs1[4] = { + common_hv_6ht_6vt_8w_lsx, + common_hv_6ht_4vt_8w_lsx, + common_hv_4ht_6vt_8w_lsx, + common_hv_4ht_4vt_8w_lsx, + }; + + static PVp8SixtapPredictFunc2 Predict8x8Funcs2[4] = { common_vt_6t_8w_lsx, + common_vt_4t_8w_lsx, + common_hz_6t_8w_lsx, + common_hz_4t_8w_lsx }; + + if (yoffset < 8 && xoffset < 8) { + if (yoffset) { + if (xoffset) { + switch (xoffset & 1) { + case 0: + switch (yoffset & 1) { + case 0: + Predict8x8Funcs1[0](src, src_stride, dst, dst_stride, h_filter, + v_filter, 8); + break; + + case 1: + Predict8x8Funcs1[1](src, src_stride, dst, dst_stride, h_filter, + v_filter + 1, 8); + break; + } + break; + + case 1: + switch (yoffset & 1) { + case 0: + Predict8x8Funcs1[2](src, src_stride, dst, dst_stride, + h_filter + 1, v_filter, 8); + break; + + case 1: + Predict8x8Funcs1[3](src, src_stride, dst, dst_stride, + h_filter + 1, v_filter + 1, 8); + break; + } + break; + } + } else { + switch (yoffset & 1) { + case 0: + Predict8x8Funcs2[0](src, src_stride, dst, dst_stride, v_filter, 8); + break; + + case 1: + Predict8x8Funcs2[1](src, src_stride, dst, dst_stride, v_filter + 1, + 8); + break; + } + } + } else { + switch (xoffset & 1) { + case 1: + Predict8x8Funcs2[3](src, src_stride, dst, dst_stride, h_filter + 1, + 8); + break; + } + switch (xoffset) { + case 0: vp8_copy_mem8x8(src, src_stride, dst, dst_stride); break; + case 2: + case 4: + case 6: + Predict8x8Funcs2[2](src, src_stride, dst, dst_stride, h_filter, 8); + break; + } + } + } +} + +void vp8_sixtap_predict16x16_lsx(uint8_t *RESTRICT src, int32_t src_stride, + int32_t xoffset, int32_t yoffset, + uint8_t *RESTRICT dst, int32_t dst_stride) { + const int8_t *h_filter = vp8_subpel_filters_lsx[xoffset - 1]; + const int8_t *v_filter = vp8_subpel_filters_lsx[yoffset - 1]; + + static PVp8SixtapPredictFunc1 Predict16x16Funcs1[4] = { + common_hv_6ht_6vt_16w_lsx, + common_hv_6ht_4vt_16w_lsx, + common_hv_4ht_6vt_16w_lsx, + common_hv_4ht_4vt_16w_lsx, + }; + + static PVp8SixtapPredictFunc2 Predict16x16Funcs2[4] = { + common_vt_6t_16w_lsx, common_vt_4t_16w_lsx, common_hz_6t_16w_lsx, + common_hz_4t_16w_lsx + }; + + if (yoffset < 8 && xoffset < 8) { + if (yoffset) { + if (xoffset) { + switch (xoffset & 1) { + case 0: + switch (yoffset & 1) { + case 0: + Predict16x16Funcs1[0](src, src_stride, dst, dst_stride, + h_filter, v_filter, 16); + break; + + case 1: + Predict16x16Funcs1[1](src, src_stride, dst, dst_stride, + h_filter, v_filter + 1, 16); + break; + } + break; + + case 1: + switch (yoffset & 1) { + case 0: + Predict16x16Funcs1[2](src, src_stride, dst, dst_stride, + h_filter + 1, v_filter, 16); + break; + + case 1: + Predict16x16Funcs1[3](src, src_stride, dst, dst_stride, + h_filter, v_filter + 1, 16); + break; + } + break; + } + } else { + switch (yoffset & 1) { + case 0: + Predict16x16Funcs2[0](src, src_stride, dst, dst_stride, v_filter, + 16); + break; + + case 1: + Predict16x16Funcs2[1](src, src_stride, dst, dst_stride, + v_filter + 1, 16); + break; + } + } + } else { + switch (xoffset & 1) { + case 1: + Predict16x16Funcs2[3](src, src_stride, dst, dst_stride, h_filter + 1, + 16); + break; + } + switch (xoffset) { + case 0: vp8_copy_mem16x16(src, src_stride, dst, dst_stride); break; + case 2: + case 4: + case 6: + Predict16x16Funcs2[2](src, src_stride, dst, dst_stride, h_filter, 16); + break; + } + } + } +} diff --git a/media/libvpx/libvpx/vp8/common/rtcd_defs.pl b/media/libvpx/libvpx/vp8/common/rtcd_defs.pl --- a/media/libvpx/libvpx/vp8/common/rtcd_defs.pl +++ b/media/libvpx/libvpx/vp8/common/rtcd_defs.pl @@ -36,29 +36,29 @@ add_proto qw/void vp8_dequant_idct_add/, "short *input, short *dq, unsigned char *dest, int stride"; specialize qw/vp8_dequant_idct_add mmx neon dspr2 msa mmi/; add_proto qw/void vp8_dequant_idct_add_y_block/, "short *q, short *dq, unsigned char *dst, int stride, char *eobs"; -specialize qw/vp8_dequant_idct_add_y_block sse2 neon dspr2 msa mmi/; +specialize qw/vp8_dequant_idct_add_y_block sse2 neon dspr2 msa mmi lsx/; add_proto qw/void vp8_dequant_idct_add_uv_block/, "short *q, short *dq, unsigned char *dst_u, unsigned char *dst_v, int stride, char *eobs"; -specialize qw/vp8_dequant_idct_add_uv_block sse2 neon dspr2 msa mmi/; +specialize qw/vp8_dequant_idct_add_uv_block sse2 neon dspr2 msa mmi lsx/; # # Loopfilter # add_proto qw/void vp8_loop_filter_mbv/, "unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi"; -specialize qw/vp8_loop_filter_mbv sse2 neon dspr2 msa mmi/; +specialize qw/vp8_loop_filter_mbv sse2 neon dspr2 msa mmi lsx/; add_proto qw/void vp8_loop_filter_bv/, "unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi"; -specialize qw/vp8_loop_filter_bv sse2 neon dspr2 msa mmi/; +specialize qw/vp8_loop_filter_bv sse2 neon dspr2 msa mmi lsx/; add_proto qw/void vp8_loop_filter_mbh/, "unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi"; -specialize qw/vp8_loop_filter_mbh sse2 neon dspr2 msa mmi/; +specialize qw/vp8_loop_filter_mbh sse2 neon dspr2 msa mmi lsx/; add_proto qw/void vp8_loop_filter_bh/, "unsigned char *y_ptr, unsigned char *u_ptr, unsigned char *v_ptr, int y_stride, int uv_stride, struct loop_filter_info *lfi"; -specialize qw/vp8_loop_filter_bh sse2 neon dspr2 msa mmi/; +specialize qw/vp8_loop_filter_bh sse2 neon dspr2 msa mmi lsx/; add_proto qw/void vp8_loop_filter_simple_mbv/, "unsigned char *y_ptr, int y_stride, const unsigned char *blimit"; specialize qw/vp8_loop_filter_simple_mbv sse2 neon msa mmi/; $vp8_loop_filter_simple_mbv_c=vp8_loop_filter_simple_vertical_edge_c; @@ -106,11 +106,11 @@ add_proto qw/void vp8_short_inv_walsh4x4/, "short *input, short *mb_dqcoeff"; specialize qw/vp8_short_inv_walsh4x4 sse2 neon dspr2 msa mmi/; #idct1_scalar_add add_proto qw/void vp8_dc_only_idct_add/, "short input_dc, unsigned char *pred_ptr, int pred_stride, unsigned char *dst_ptr, int dst_stride"; -specialize qw/vp8_dc_only_idct_add mmx neon dspr2 msa mmi/; +specialize qw/vp8_dc_only_idct_add mmx neon dspr2 msa mmi lsx/; # # RECON # add_proto qw/void vp8_copy_mem16x16/, "unsigned char *src, int src_stride, unsigned char *dst, int dst_stride"; @@ -144,14 +144,14 @@ # # Subpixel # add_proto qw/void vp8_sixtap_predict16x16/, "unsigned char *src_ptr, int src_pixels_per_line, int xoffset, int yoffset, unsigned char *dst_ptr, int dst_pitch"; -specialize qw/vp8_sixtap_predict16x16 sse2 ssse3 neon dspr2 msa mmi/; +specialize qw/vp8_sixtap_predict16x16 sse2 ssse3 neon dspr2 msa mmi lsx/; add_proto qw/void vp8_sixtap_predict8x8/, "unsigned char *src_ptr, int src_pixels_per_line, int xoffset, int yoffset, unsigned char *dst_ptr, int dst_pitch"; -specialize qw/vp8_sixtap_predict8x8 sse2 ssse3 neon dspr2 msa mmi/; +specialize qw/vp8_sixtap_predict8x8 sse2 ssse3 neon dspr2 msa mmi lsx/; add_proto qw/void vp8_sixtap_predict8x4/, "unsigned char *src_ptr, int src_pixels_per_line, int xoffset, int yoffset, unsigned char *dst_ptr, int dst_pitch"; specialize qw/vp8_sixtap_predict8x4 sse2 ssse3 neon dspr2 msa mmi/; add_proto qw/void vp8_sixtap_predict4x4/, "unsigned char *src_ptr, int src_pixels_per_line, int xoffset, int yoffset, unsigned char *dst_ptr, int dst_pitch"; @@ -214,15 +214,10 @@ specialize qw/vp8_mbuverror sse2 msa/; # # Motion search # -add_proto qw/int vp8_full_search_sad/, "struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int sad_per_bit, int distance, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv"; -specialize qw/vp8_full_search_sad sse3 sse4_1/; -$vp8_full_search_sad_sse3=vp8_full_search_sadx3; -$vp8_full_search_sad_sse4_1=vp8_full_search_sadx8; - add_proto qw/int vp8_refining_search_sad/, "struct macroblock *x, struct block *b, struct blockd *d, union int_mv *ref_mv, int error_per_bit, int search_range, struct variance_vtable *fn_ptr, int *mvcost[2], union int_mv *center_mv"; specialize qw/vp8_refining_search_sad sse2 msa/; $vp8_refining_search_sad_sse2=vp8_refining_search_sadx4; $vp8_refining_search_sad_msa=vp8_refining_search_sadx4; diff --git a/media/libvpx/libvpx/vp8/common/x86/bilinear_filter_sse2.c b/media/libvpx/libvpx/vp8/common/x86/bilinear_filter_sse2.c --- a/media/libvpx/libvpx/vp8/common/x86/bilinear_filter_sse2.c +++ b/media/libvpx/libvpx/vp8/common/x86/bilinear_filter_sse2.c @@ -311,14 +311,14 @@ const __m128i row_1_filtered = _mm_mullo_epi16(row_1, vfilter_1); const __m128i sum = _mm_add_epi16(row_0_filtered, row_1_filtered); const __m128i compensated = _mm_add_epi16(sum, round_factor); const __m128i shifted = _mm_srai_epi16(compensated, VP8_FILTER_SHIFT); __m128i packed = _mm_packus_epi16(shifted, shifted); - storeu_uint32(dst, _mm_cvtsi128_si32(packed)); + storeu_int32(dst, _mm_cvtsi128_si32(packed)); packed = _mm_srli_si128(packed, 4); dst += stride; - storeu_uint32(dst, _mm_cvtsi128_si32(packed)); + storeu_int32(dst, _mm_cvtsi128_si32(packed)); dst += stride; src += 8; } } } diff --git a/media/libvpx/libvpx/vp8/decoder/decodeframe.c b/media/libvpx/libvpx/vp8/decoder/decodeframe.c --- a/media/libvpx/libvpx/vp8/decoder/decodeframe.c +++ b/media/libvpx/libvpx/vp8/decoder/decodeframe.c @@ -870,12 +870,12 @@ xd->frame_type = pc->frame_type; xd->mode_info_context->mbmi.mode = DC_PRED; xd->mode_info_stride = pc->mode_info_stride; xd->corrupted = 0; /* init without corruption */ - xd->fullpixel_mask = 0xffffffff; - if (pc->full_pixel) xd->fullpixel_mask = 0xfffffff8; + xd->fullpixel_mask = ~0; + if (pc->full_pixel) xd->fullpixel_mask = ~7; } int vp8_decode_frame(VP8D_COMP *pbi) { vp8_reader *const bc = &pbi->mbc[8]; VP8_COMMON *const pc = &pbi->common; diff --git a/media/libvpx/libvpx/vp8/decoder/decodemv.c b/media/libvpx/libvpx/vp8/decoder/decodemv.c --- a/media/libvpx/libvpx/vp8/decoder/decodemv.c +++ b/media/libvpx/libvpx/vp8/decoder/decodemv.c @@ -171,11 +171,12 @@ { 223, 1, 34 }, /* SUBMVREF_LEFT_ABOVE_SAME */ { 179, 121, 1 }, /* SUBMVREF_ABOVE_ZED */ { 208, 1, 1 } /* SUBMVREF_LEFT_ABOVE_ZED */ }; -static const vp8_prob *get_sub_mv_ref_prob(const int left, const int above) { +static const vp8_prob *get_sub_mv_ref_prob(const uint32_t left, + const uint32_t above) { int lez = (left == 0); int aez = (above == 0); int lea = (left == above); const vp8_prob *prob; diff --git a/media/libvpx/libvpx/vp8/decoder/threading.c b/media/libvpx/libvpx/vp8/decoder/threading.c --- a/media/libvpx/libvpx/vp8/decoder/threading.c +++ b/media/libvpx/libvpx/vp8/decoder/threading.c @@ -72,13 +72,13 @@ memcpy(mbd->dequant_y1_dc, xd->dequant_y1_dc, sizeof(xd->dequant_y1_dc)); memcpy(mbd->dequant_y1, xd->dequant_y1, sizeof(xd->dequant_y1)); memcpy(mbd->dequant_y2, xd->dequant_y2, sizeof(xd->dequant_y2)); memcpy(mbd->dequant_uv, xd->dequant_uv, sizeof(xd->dequant_uv)); - mbd->fullpixel_mask = 0xffffffff; + mbd->fullpixel_mask = ~0; - if (pc->full_pixel) mbd->fullpixel_mask = 0xfffffff8; + if (pc->full_pixel) mbd->fullpixel_mask = ~7; } for (i = 0; i < pc->mb_rows; ++i) vpx_atomic_store_release(&pbi->mt_current_mb_col[i], -1); } diff --git a/media/libvpx/libvpx/vp8/encoder/bitstream.c b/media/libvpx/libvpx/vp8/encoder/bitstream.c --- a/media/libvpx/libvpx/vp8/encoder/bitstream.c +++ b/media/libvpx/libvpx/vp8/encoder/bitstream.c @@ -170,13 +170,12 @@ } validate_buffer(w->buffer + w->pos, 1, w->buffer_end, w->error); w->buffer[w->pos++] = (lowvalue >> (24 - offset)) & 0xff; - lowvalue <<= offset; shift = count; - lowvalue &= 0xffffff; + lowvalue = (int)(((uint64_t)lowvalue << offset) & 0xffffff); count -= 8; } lowvalue <<= shift; } while (n); @@ -221,13 +220,12 @@ } validate_buffer(w->buffer + w->pos, 1, w->buffer_end, w->error); w->buffer[w->pos++] = (lowvalue >> (24 - offset)) & 0xff; - lowvalue <<= offset; shift = count; - lowvalue &= 0xffffff; + lowvalue = (int)(((uint64_t)lowvalue << offset) & 0xffffff); count -= 8; } lowvalue <<= shift; } while (n2); diff --git a/media/libvpx/libvpx/vp8/encoder/block.h b/media/libvpx/libvpx/vp8/encoder/block.h --- a/media/libvpx/libvpx/vp8/encoder/block.h +++ b/media/libvpx/libvpx/vp8/encoder/block.h @@ -90,12 +90,11 @@ int *mb_norm_activity_ptr; signed int act_zbin_adj; signed int last_act_zbin_adj; int *mvcost[2]; - /* MSVC generates code that thinks this is 16-byte aligned */ - DECLARE_ALIGNED(16, int*, mvsadcost[2]); + int *mvsadcost[2]; int (*mbmode_cost)[MB_MODE_COUNT]; int (*intra_uv_mode_cost)[MB_MODE_COUNT]; int (*bmode_costs)[10][10]; int *inter_bmode_costs; int (*token_costs)[COEF_BANDS][PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS]; diff --git a/media/libvpx/libvpx/vp8/encoder/boolhuff.h b/media/libvpx/libvpx/vp8/encoder/boolhuff.h --- a/media/libvpx/libvpx/vp8/encoder/boolhuff.h +++ b/media/libvpx/libvpx/vp8/encoder/boolhuff.h @@ -92,13 +92,12 @@ } validate_buffer(bc->buffer + bc->pos, 1, bc->buffer_end, bc->error); bc->buffer[bc->pos++] = (lowvalue >> (24 - offset) & 0xff); - lowvalue <<= offset; shift = count; - lowvalue &= 0xffffff; + lowvalue = (int)(((uint64_t)lowvalue << offset) & 0xffffff); count -= 8; } lowvalue <<= shift; bc->count = count; diff --git a/media/libvpx/libvpx/vp8/encoder/encodeframe.c b/media/libvpx/libvpx/vp8/encoder/encodeframe.c --- a/media/libvpx/libvpx/vp8/encoder/encodeframe.c +++ b/media/libvpx/libvpx/vp8/encoder/encodeframe.c @@ -67,14 +67,13 @@ static const unsigned char VP8_VAR_OFFS[16] = { 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128 }; /* Original activity measure from Tim T's code. */ -static unsigned int tt_activity_measure(VP8_COMP *cpi, MACROBLOCK *x) { +static unsigned int tt_activity_measure(MACROBLOCK *x) { unsigned int act; unsigned int sse; - (void)cpi; /* TODO: This could also be done over smaller areas (8x8), but that would * require extensive changes elsewhere, as lambda is assumed to be fixed * over an entire MB in most of the code. * Another option is to compute four 8x8 variances, and pick a single * lambda using a non-linear combination (e.g., the smallest, or second @@ -88,32 +87,25 @@ if (act < 8 << 12) act = act < 5 << 12 ? act : 5 << 12; return act; } -/* Stub for alternative experimental activity measures. */ -static unsigned int alt_activity_measure(VP8_COMP *cpi, MACROBLOCK *x, - int use_dc_pred) { - return vp8_encode_intra(cpi, x, use_dc_pred); -} - /* Measure the activity of the current macroblock * What we measure here is TBD so abstracted to this function */ #define ALT_ACT_MEASURE 1 -static unsigned int mb_activity_measure(VP8_COMP *cpi, MACROBLOCK *x, - int mb_row, int mb_col) { +static unsigned int mb_activity_measure(MACROBLOCK *x, int mb_row, int mb_col) { unsigned int mb_activity; if (ALT_ACT_MEASURE) { int use_dc_pred = (mb_col || mb_row) && (!mb_col || !mb_row); - /* Or use and alternative. */ - mb_activity = alt_activity_measure(cpi, x, use_dc_pred); + /* Or use an alternative. */ + mb_activity = vp8_encode_intra(x, use_dc_pred); } else { /* Original activity measure from Tim T's code. */ - mb_activity = tt_activity_measure(cpi, x); + mb_activity = tt_activity_measure(x); } if (mb_activity < VP8_ACTIVITY_AVG_MIN) mb_activity = VP8_ACTIVITY_AVG_MIN; return mb_activity; @@ -262,11 +254,11 @@ #endif /* Copy current mb to a buffer */ vp8_copy_mem16x16(x->src.y_buffer, x->src.y_stride, x->thismb, 16); /* measure activity */ - mb_activity = mb_activity_measure(cpi, x, mb_row, mb_col); + mb_activity = mb_activity_measure(x, mb_row, mb_col); /* Keep frame sum */ activity_sum += mb_activity; /* Store MB level activity details. */ @@ -632,16 +624,17 @@ } else { vp8_calc_ref_frame_costs(x->ref_frame_cost, cpi->prob_intra_coded, cpi->prob_last_coded, cpi->prob_gf_coded); } - xd->fullpixel_mask = 0xffffffff; - if (cm->full_pixel) xd->fullpixel_mask = 0xfffffff8; + xd->fullpixel_mask = ~0; + if (cm->full_pixel) xd->fullpixel_mask = ~7; vp8_zero(x->coef_counts); vp8_zero(x->ymode_count); - vp8_zero(x->uv_mode_count) x->prediction_error = 0; + vp8_zero(x->uv_mode_count); + x->prediction_error = 0; x->intra_error = 0; vp8_zero(x->count_mb_ref_frame_usage); } #if CONFIG_MULTITHREAD @@ -764,16 +757,16 @@ sem_post(&cpi->h_event_start_encoding[i]); } for (mb_row = 0; mb_row < cm->mb_rows; mb_row += (cpi->encoding_thread_count + 1)) { - vp8_zero(cm->left_context) + vp8_zero(cm->left_context); #if CONFIG_REALTIME_ONLY & CONFIG_ONTHEFLY_BITPACKING - tp = cpi->tok; + tp = cpi->tok; #else - tp = cpi->tok + mb_row * (cm->mb_cols * 16 * 24); + tp = cpi->tok + mb_row * (cm->mb_cols * 16 * 24); #endif encode_mb_row(cpi, cm, mb_row, x, xd, &tp, segment_counts, &totalrate); /* adjust to the next row of mbs */ @@ -856,14 +849,14 @@ #endif // CONFIG_MULTITHREAD { /* for each macroblock row in image */ for (mb_row = 0; mb_row < cm->mb_rows; ++mb_row) { - vp8_zero(cm->left_context) + vp8_zero(cm->left_context); #if CONFIG_REALTIME_ONLY & CONFIG_ONTHEFLY_BITPACKING - tp = cpi->tok; + tp = cpi->tok; #endif encode_mb_row(cpi, cm, mb_row, x, xd, &tp, segment_counts, &totalrate); /* adjust to the next row of mbs */ diff --git a/media/libvpx/libvpx/vp8/encoder/encodeintra.h b/media/libvpx/libvpx/vp8/encoder/encodeintra.h --- a/media/libvpx/libvpx/vp8/encoder/encodeintra.h +++ b/media/libvpx/libvpx/vp8/encoder/encodeintra.h @@ -14,11 +14,11 @@ #ifdef __cplusplus extern "C" { #endif -int vp8_encode_intra(VP8_COMP *cpi, MACROBLOCK *x, int use_dc_pred); +int vp8_encode_intra(MACROBLOCK *x, int use_dc_pred); void vp8_encode_intra16x16mby(MACROBLOCK *x); void vp8_encode_intra16x16mbuv(MACROBLOCK *x); void vp8_encode_intra4x4mby(MACROBLOCK *mb); void vp8_encode_intra4x4block(MACROBLOCK *x, int ib); #ifdef __cplusplus diff --git a/media/libvpx/libvpx/vp8/encoder/encodeintra.c b/media/libvpx/libvpx/vp8/encoder/encodeintra.c --- a/media/libvpx/libvpx/vp8/encoder/encodeintra.c +++ b/media/libvpx/libvpx/vp8/encoder/encodeintra.c @@ -16,14 +16,13 @@ #include "vp8/common/reconintra4x4.h" #include "encodemb.h" #include "vp8/common/invtrans.h" #include "encodeintra.h" -int vp8_encode_intra(VP8_COMP *cpi, MACROBLOCK *x, int use_dc_pred) { +int vp8_encode_intra(MACROBLOCK *x, int use_dc_pred) { int i; int intra_pred_var = 0; - (void)cpi; if (use_dc_pred) { x->e_mbd.mode_info_context->mbmi.mode = DC_PRED; x->e_mbd.mode_info_context->mbmi.uv_mode = DC_PRED; x->e_mbd.mode_info_context->mbmi.ref_frame = INTRA_FRAME; diff --git a/media/libvpx/libvpx/vp8/encoder/encodemv.c b/media/libvpx/libvpx/vp8/encoder/encodemv.c --- a/media/libvpx/libvpx/vp8/encoder/encodemv.c +++ b/media/libvpx/libvpx/vp8/encoder/encodemv.c @@ -158,11 +158,11 @@ static void calc_prob(vp8_prob *p, const unsigned int ct[2]) { const unsigned int tot = ct[0] + ct[1]; if (tot) { - const vp8_prob x = ((ct[0] * 255) / tot) & -2; + const vp8_prob x = ((ct[0] * 255) / tot) & ~1u; *p = x ? x : 1; } } static void update(vp8_writer *const w, const unsigned int ct[2], @@ -203,12 +203,15 @@ vp8_prob Pnew[MVPcount]; (void)rc; vp8_copy_array(Pnew, default_mvc, MVPcount); - vp8_zero(is_short_ct) vp8_zero(sign_ct) vp8_zero(bit_ct) vp8_zero(short_ct) - vp8_zero(short_bct) + vp8_zero(is_short_ct); + vp8_zero(sign_ct); + vp8_zero(bit_ct); + vp8_zero(short_ct); + vp8_zero(short_bct); /* j=0 */ { const int c = events[mv_max]; diff --git a/media/libvpx/libvpx/vp8/encoder/ethreading.c b/media/libvpx/libvpx/vp8/encoder/ethreading.c --- a/media/libvpx/libvpx/vp8/encoder/ethreading.c +++ b/media/libvpx/libvpx/vp8/encoder/ethreading.c @@ -468,12 +468,12 @@ mbd->left_context = &cm->left_context; mb->mvc = cm->fc.mvc; setup_mbby_copy(&mbr_ei[i].mb, x); - mbd->fullpixel_mask = 0xffffffff; - if (cm->full_pixel) mbd->fullpixel_mask = 0xfffffff8; + mbd->fullpixel_mask = ~0; + if (cm->full_pixel) mbd->fullpixel_mask = ~7; vp8_zero(mb->coef_counts); vp8_zero(x->ymode_count); mb->skip_true_count = 0; vp8_zero(mb->MVcount); diff --git a/media/libvpx/libvpx/vp8/encoder/firstpass.c b/media/libvpx/libvpx/vp8/encoder/firstpass.c --- a/media/libvpx/libvpx/vp8/encoder/firstpass.c +++ b/media/libvpx/libvpx/vp8/encoder/firstpass.c @@ -565,11 +565,11 @@ /* Copy current mb to a buffer */ vp8_copy_mem16x16(x->src.y_buffer, x->src.y_stride, x->thismb, 16); /* do intra 16x16 prediction */ - this_error = vp8_encode_intra(cpi, x, use_dc_pred); + this_error = vp8_encode_intra(x, use_dc_pred); /* "intrapenalty" below deals with situations where the intra * and inter error scores are very low (eg a plain black frame) * We do not have special cases in first pass for 0,0 and * nearest etc so all inter modes carry an overhead cost @@ -1629,11 +1629,10 @@ double loop_decay_rate = 1.00; /* Starting decay rate */ double this_frame_mv_in_out = 0.0; double mv_in_out_accumulator = 0.0; double abs_mv_in_out_accumulator = 0.0; - double mod_err_per_mb_accumulator = 0.0; int max_bits = frame_max_bits(cpi); /* Max for a single frame */ unsigned int allow_alt_ref = cpi->oxcf.play_alternate && cpi->oxcf.lag_in_frames; @@ -1680,13 +1679,10 @@ /* Accumulate error score of frames in this gf group */ mod_frame_err = calculate_modified_err(cpi, this_frame); gf_group_err += mod_frame_err; - mod_err_per_mb_accumulator += - mod_frame_err / DOUBLE_DIVIDE_CHECK((double)cpi->common.MBs); - if (EOF == input_stats(cpi, &next_frame)) break; /* Test for the case where there is a brief flash but the prediction * quality back to an earlier frame is then restored. */ diff --git a/media/libvpx/libvpx/vp8/encoder/lookahead.c b/media/libvpx/libvpx/vp8/encoder/lookahead.c --- a/media/libvpx/libvpx/vp8/encoder/lookahead.c +++ b/media/libvpx/libvpx/vp8/encoder/lookahead.c @@ -64,12 +64,12 @@ /* Keep last frame in lookahead buffer by increasing depth by 1.*/ depth += 1; /* Align the buffer dimensions */ - width = (width + 15) & ~15; - height = (height + 15) & ~15; + width = (width + 15) & ~15u; + height = (height + 15) & ~15u; /* Allocate the lookahead structures */ ctx = calloc(1, sizeof(*ctx)); if (ctx) { ctx->max_sz = depth; diff --git a/media/libvpx/libvpx/vp8/encoder/mcomp.h b/media/libvpx/libvpx/vp8/encoder/mcomp.h --- a/media/libvpx/libvpx/vp8/encoder/mcomp.h +++ b/media/libvpx/libvpx/vp8/encoder/mcomp.h @@ -48,14 +48,14 @@ fractional_mv_step_fp vp8_find_best_sub_pixel_step_iteratively; fractional_mv_step_fp vp8_find_best_sub_pixel_step; fractional_mv_step_fp vp8_find_best_half_pixel_step; fractional_mv_step_fp vp8_skip_fractional_mv_step; -typedef int (*vp8_full_search_fn_t)(MACROBLOCK *x, BLOCK *b, BLOCKD *d, - int_mv *ref_mv, int sad_per_bit, - int distance, vp8_variance_fn_ptr_t *fn_ptr, - int *mvcost[2], int_mv *center_mv); +int vp8_full_search_sad(MACROBLOCK *x, BLOCK *b, BLOCKD *d, int_mv *ref_mv, + int sad_per_bit, int distance, + vp8_variance_fn_ptr_t *fn_ptr, int *mvcost[2], + int_mv *center_mv); typedef int (*vp8_refining_search_fn_t)(MACROBLOCK *x, BLOCK *b, BLOCKD *d, int_mv *ref_mv, int sad_per_bit, int distance, vp8_variance_fn_ptr_t *fn_ptr, diff --git a/media/libvpx/libvpx/vp8/encoder/mcomp.c b/media/libvpx/libvpx/vp8/encoder/mcomp.c --- a/media/libvpx/libvpx/vp8/encoder/mcomp.c +++ b/media/libvpx/libvpx/vp8/encoder/mcomp.c @@ -202,23 +202,25 @@ #define IFMVCV(r, c, s, e) \ if (c >= minc && c <= maxc && r >= minr && r <= maxr) s else e; /* returns distortion + motion vector cost */ #define ERR(r, c) (MVC(r, c) + DIST(r, c)) /* checks if (r,c) has better score than previous best */ -#define CHECK_BETTER(v, r, c) \ - IFMVCV(r, c, \ - { \ - thismse = DIST(r, c); \ - if ((v = (MVC(r, c) + thismse)) < besterr) { \ - besterr = v; \ - br = r; \ - bc = c; \ - *distortion = thismse; \ - *sse1 = sse; \ - } \ - }, \ - v = UINT_MAX;) +#define CHECK_BETTER(v, r, c) \ + do { \ + IFMVCV(r, c, \ + { \ + thismse = DIST(r, c); \ + if ((v = (MVC(r, c) + thismse)) < besterr) { \ + besterr = v; \ + br = r; \ + bc = c; \ + *distortion = thismse; \ + *sse1 = sse; \ + } \ + }, \ + v = UINT_MAX;) \ + } while (0) int vp8_find_best_sub_pixel_step_iteratively(MACROBLOCK *x, BLOCK *b, BLOCKD *d, int_mv *bestmv, int_mv *ref_mv, int error_per_bit, const vp8_variance_fn_ptr_t *vfp, @@ -798,37 +800,37 @@ return bestmse; } #define CHECK_BOUNDS(range) \ - { \ + do { \ all_in = 1; \ all_in &= ((br - range) >= x->mv_row_min); \ all_in &= ((br + range) <= x->mv_row_max); \ all_in &= ((bc - range) >= x->mv_col_min); \ all_in &= ((bc + range) <= x->mv_col_max); \ - } + } while (0) #define CHECK_POINT \ { \ if (this_mv.as_mv.col < x->mv_col_min) continue; \ if (this_mv.as_mv.col > x->mv_col_max) continue; \ if (this_mv.as_mv.row < x->mv_row_min) continue; \ if (this_mv.as_mv.row > x->mv_row_max) continue; \ } #define CHECK_BETTER \ - { \ + do { \ if (thissad < bestsad) { \ thissad += \ mvsad_err_cost(&this_mv, &fcenter_mv, mvsadcost, sad_per_bit); \ if (thissad < bestsad) { \ bestsad = thissad; \ best_site = i; \ } \ } \ - } + } while (0) static const MV next_chkpts[6][3] = { { { -2, 0 }, { -1, -2 }, { 1, -2 } }, { { -1, -2 }, { 1, -2 }, { 2, 0 } }, { { 1, -2 }, { 2, 0 }, { 1, 2 } }, { { 2, 0 }, { 1, 2 }, { -1, 2 } }, { { 1, 2 }, { -1, 2 }, { -2, 0 } }, { { -1, 2 }, { -2, 0 }, { -1, -2 } } @@ -899,30 +901,30 @@ #else (void)search_param; #endif /* hex search */ - CHECK_BOUNDS(2) + CHECK_BOUNDS(2); if (all_in) { for (i = 0; i < 6; ++i) { this_mv.as_mv.row = br + hex[i].row; this_mv.as_mv.col = bc + hex[i].col; this_offset = base_offset + (this_mv.as_mv.row * in_what_stride) + this_mv.as_mv.col; thissad = vfp->sdf(what, what_stride, this_offset, in_what_stride); - CHECK_BETTER + CHECK_BETTER; } } else { for (i = 0; i < 6; ++i) { this_mv.as_mv.row = br + hex[i].row; this_mv.as_mv.col = bc + hex[i].col; CHECK_POINT this_offset = base_offset + (this_mv.as_mv.row * in_what_stride) + this_mv.as_mv.col; thissad = vfp->sdf(what, what_stride, this_offset, in_what_stride); - CHECK_BETTER + CHECK_BETTER; } } if (best_site == -1) { goto cal_neighbors; @@ -932,30 +934,30 @@ k = best_site; } for (j = 1; j < hex_range; ++j) { best_site = -1; - CHECK_BOUNDS(2) + CHECK_BOUNDS(2); if (all_in) { for (i = 0; i < 3; ++i) { this_mv.as_mv.row = br + next_chkpts[k][i].row; this_mv.as_mv.col = bc + next_chkpts[k][i].col; this_offset = base_offset + (this_mv.as_mv.row * (in_what_stride)) + this_mv.as_mv.col; thissad = vfp->sdf(what, what_stride, this_offset, in_what_stride); - CHECK_BETTER + CHECK_BETTER; } } else { for (i = 0; i < 3; ++i) { this_mv.as_mv.row = br + next_chkpts[k][i].row; this_mv.as_mv.col = bc + next_chkpts[k][i].col; CHECK_POINT this_offset = base_offset + (this_mv.as_mv.row * (in_what_stride)) + this_mv.as_mv.col; thissad = vfp->sdf(what, what_stride, this_offset, in_what_stride); - CHECK_BETTER + CHECK_BETTER; } } if (best_site == -1) { break; @@ -973,30 +975,30 @@ /* check 4 1-away neighbors */ cal_neighbors: for (j = 0; j < dia_range; ++j) { best_site = -1; - CHECK_BOUNDS(1) + CHECK_BOUNDS(1); if (all_in) { for (i = 0; i < 4; ++i) { this_mv.as_mv.row = br + neighbors[i].row; this_mv.as_mv.col = bc + neighbors[i].col; this_offset = base_offset + (this_mv.as_mv.row * (in_what_stride)) + this_mv.as_mv.col; thissad = vfp->sdf(what, what_stride, this_offset, in_what_stride); - CHECK_BETTER + CHECK_BETTER; } } else { for (i = 0; i < 4; ++i) { this_mv.as_mv.row = br + neighbors[i].row; this_mv.as_mv.col = bc + neighbors[i].col; CHECK_POINT this_offset = base_offset + (this_mv.as_mv.row * (in_what_stride)) + this_mv.as_mv.col; thissad = vfp->sdf(what, what_stride, this_offset, in_what_stride); - CHECK_BETTER + CHECK_BETTER; } } if (best_site == -1) { break; @@ -1276,14 +1278,14 @@ return fn_ptr->vf(what, what_stride, best_address, in_what_stride, &thissad) + mv_err_cost(&this_mv, center_mv, mvcost, x->errorperbit); } #endif // HAVE_SSE2 || HAVE_MSA -int vp8_full_search_sad_c(MACROBLOCK *x, BLOCK *b, BLOCKD *d, int_mv *ref_mv, - int sad_per_bit, int distance, - vp8_variance_fn_ptr_t *fn_ptr, int *mvcost[2], - int_mv *center_mv) { +int vp8_full_search_sad(MACROBLOCK *x, BLOCK *b, BLOCKD *d, int_mv *ref_mv, + int sad_per_bit, int distance, + vp8_variance_fn_ptr_t *fn_ptr, int *mvcost[2], + int_mv *center_mv) { unsigned char *what = (*(b->base_src) + b->src); int what_stride = b->src_stride; unsigned char *in_what; int pre_stride = x->e_mbd.pre.y_stride; unsigned char *base_pre = x->e_mbd.pre.y_buffer; @@ -1323,12 +1325,12 @@ /* Baseline value at the centre */ bestsad = fn_ptr->sdf(what, what_stride, bestaddress, in_what_stride) + mvsad_err_cost(best_mv, &fcenter_mv, mvsadcost, sad_per_bit); - /* Apply further limits to prevent us looking using vectors that - * stretch beyiond the UMV border + /* Apply further limits to prevent us looking using vectors that stretch + * beyond the UMV border */ if (col_min < x->mv_col_min) col_min = x->mv_col_min; if (col_max > x->mv_col_max) col_max = x->mv_col_max; @@ -1341,126 +1343,10 @@ check_here = r * mv_stride + in_what + col_min; for (c = col_min; c < col_max; ++c) { thissad = fn_ptr->sdf(what, what_stride, check_here, in_what_stride); - this_mv.as_mv.col = c; - thissad += mvsad_err_cost(&this_mv, &fcenter_mv, mvsadcost, sad_per_bit); - - if (thissad < bestsad) { - bestsad = thissad; - best_mv->as_mv.row = r; - best_mv->as_mv.col = c; - bestaddress = check_here; - } - - check_here++; - } - } - - this_mv.as_mv.row = best_mv->as_mv.row << 3; - this_mv.as_mv.col = best_mv->as_mv.col << 3; - - return fn_ptr->vf(what, what_stride, bestaddress, in_what_stride, &thissad) + - mv_err_cost(&this_mv, center_mv, mvcost, x->errorperbit); -} - -#if HAVE_SSSE3 -int vp8_full_search_sadx3(MACROBLOCK *x, BLOCK *b, BLOCKD *d, int_mv *ref_mv, - int sad_per_bit, int distance, - vp8_variance_fn_ptr_t *fn_ptr, int *mvcost[2], - int_mv *center_mv) { - unsigned char *what = (*(b->base_src) + b->src); - int what_stride = b->src_stride; - unsigned char *in_what; - int pre_stride = x->e_mbd.pre.y_stride; - unsigned char *base_pre = x->e_mbd.pre.y_buffer; - int in_what_stride = pre_stride; - int mv_stride = pre_stride; - unsigned char *bestaddress; - int_mv *best_mv = &d->bmi.mv; - int_mv this_mv; - unsigned int bestsad; - unsigned int thissad; - int r, c; - - unsigned char *check_here; - - int ref_row = ref_mv->as_mv.row; - int ref_col = ref_mv->as_mv.col; - - int row_min = ref_row - distance; - int row_max = ref_row + distance; - int col_min = ref_col - distance; - int col_max = ref_col + distance; - - unsigned int sad_array[3]; - - int *mvsadcost[2]; - int_mv fcenter_mv; - - mvsadcost[0] = x->mvsadcost[0]; - mvsadcost[1] = x->mvsadcost[1]; - fcenter_mv.as_mv.row = center_mv->as_mv.row >> 3; - fcenter_mv.as_mv.col = center_mv->as_mv.col >> 3; - - /* Work out the mid point for the search */ - in_what = base_pre + d->offset; - bestaddress = in_what + (ref_row * pre_stride) + ref_col; - - best_mv->as_mv.row = ref_row; - best_mv->as_mv.col = ref_col; - - /* Baseline value at the centre */ - bestsad = fn_ptr->sdf(what, what_stride, bestaddress, in_what_stride) + - mvsad_err_cost(best_mv, &fcenter_mv, mvsadcost, sad_per_bit); - - /* Apply further limits to prevent us looking using vectors that stretch - * beyond the UMV border - */ - if (col_min < x->mv_col_min) col_min = x->mv_col_min; - - if (col_max > x->mv_col_max) col_max = x->mv_col_max; - - if (row_min < x->mv_row_min) row_min = x->mv_row_min; - - if (row_max > x->mv_row_max) row_max = x->mv_row_max; - - for (r = row_min; r < row_max; ++r) { - this_mv.as_mv.row = r; - check_here = r * mv_stride + in_what + col_min; - c = col_min; - - while ((c + 2) < col_max) { - int i; - - fn_ptr->sdx3f(what, what_stride, check_here, in_what_stride, sad_array); - - for (i = 0; i < 3; ++i) { - thissad = sad_array[i]; - - if (thissad < bestsad) { - this_mv.as_mv.col = c; - thissad += - mvsad_err_cost(&this_mv, &fcenter_mv, mvsadcost, sad_per_bit); - - if (thissad < bestsad) { - bestsad = thissad; - best_mv->as_mv.row = r; - best_mv->as_mv.col = c; - bestaddress = check_here; - } - } - - check_here++; - c++; - } - } - - while (c < col_max) { - thissad = fn_ptr->sdf(what, what_stride, check_here, in_what_stride); - if (thissad < bestsad) { this_mv.as_mv.col = c; thissad += mvsad_err_cost(&this_mv, &fcenter_mv, mvsadcost, sad_per_bit); @@ -1471,169 +1357,19 @@ bestaddress = check_here; } } check_here++; - c++; - } - } - - this_mv.as_mv.row = best_mv->as_mv.row << 3; - this_mv.as_mv.col = best_mv->as_mv.col << 3; - - return fn_ptr->vf(what, what_stride, bestaddress, in_what_stride, &thissad) + - mv_err_cost(&this_mv, center_mv, mvcost, x->errorperbit); -} -#endif // HAVE_SSSE3 - -#if HAVE_SSE4_1 -int vp8_full_search_sadx8(MACROBLOCK *x, BLOCK *b, BLOCKD *d, int_mv *ref_mv, - int sad_per_bit, int distance, - vp8_variance_fn_ptr_t *fn_ptr, int *mvcost[2], - int_mv *center_mv) { - unsigned char *what = (*(b->base_src) + b->src); - int what_stride = b->src_stride; - int pre_stride = x->e_mbd.pre.y_stride; - unsigned char *base_pre = x->e_mbd.pre.y_buffer; - unsigned char *in_what; - int in_what_stride = pre_stride; - int mv_stride = pre_stride; - unsigned char *bestaddress; - int_mv *best_mv = &d->bmi.mv; - int_mv this_mv; - unsigned int bestsad; - unsigned int thissad; - int r, c; - - unsigned char *check_here; - - int ref_row = ref_mv->as_mv.row; - int ref_col = ref_mv->as_mv.col; - - int row_min = ref_row - distance; - int row_max = ref_row + distance; - int col_min = ref_col - distance; - int col_max = ref_col + distance; - - DECLARE_ALIGNED(16, unsigned int, sad_array8[8]); - unsigned int sad_array[3]; - - int *mvsadcost[2]; - int_mv fcenter_mv; - - mvsadcost[0] = x->mvsadcost[0]; - mvsadcost[1] = x->mvsadcost[1]; - fcenter_mv.as_mv.row = center_mv->as_mv.row >> 3; - fcenter_mv.as_mv.col = center_mv->as_mv.col >> 3; - - /* Work out the mid point for the search */ - in_what = base_pre + d->offset; - bestaddress = in_what + (ref_row * pre_stride) + ref_col; - - best_mv->as_mv.row = ref_row; - best_mv->as_mv.col = ref_col; - - /* Baseline value at the centre */ - bestsad = fn_ptr->sdf(what, what_stride, bestaddress, in_what_stride) + - mvsad_err_cost(best_mv, &fcenter_mv, mvsadcost, sad_per_bit); - - /* Apply further limits to prevent us looking using vectors that stretch - * beyond the UMV border - */ - if (col_min < x->mv_col_min) col_min = x->mv_col_min; - - if (col_max > x->mv_col_max) col_max = x->mv_col_max; - - if (row_min < x->mv_row_min) row_min = x->mv_row_min; - - if (row_max > x->mv_row_max) row_max = x->mv_row_max; - - for (r = row_min; r < row_max; ++r) { - this_mv.as_mv.row = r; - check_here = r * mv_stride + in_what + col_min; - c = col_min; - - while ((c + 7) < col_max) { - int i; - - fn_ptr->sdx8f(what, what_stride, check_here, in_what_stride, sad_array8); - - for (i = 0; i < 8; ++i) { - thissad = sad_array8[i]; - - if (thissad < bestsad) { - this_mv.as_mv.col = c; - thissad += - mvsad_err_cost(&this_mv, &fcenter_mv, mvsadcost, sad_per_bit); - - if (thissad < bestsad) { - bestsad = thissad; - best_mv->as_mv.row = r; - best_mv->as_mv.col = c; - bestaddress = check_here; - } - } - - check_here++; - c++; - } - } - - while ((c + 2) < col_max) { - int i; - - fn_ptr->sdx3f(what, what_stride, check_here, in_what_stride, sad_array); - - for (i = 0; i < 3; ++i) { - thissad = sad_array[i]; - - if (thissad < bestsad) { - this_mv.as_mv.col = c; - thissad += - mvsad_err_cost(&this_mv, &fcenter_mv, mvsadcost, sad_per_bit); - - if (thissad < bestsad) { - bestsad = thissad; - best_mv->as_mv.row = r; - best_mv->as_mv.col = c; - bestaddress = check_here; - } - } - - check_here++; - c++; - } - } - - while (c < col_max) { - thissad = fn_ptr->sdf(what, what_stride, check_here, in_what_stride); - - if (thissad < bestsad) { - this_mv.as_mv.col = c; - thissad += - mvsad_err_cost(&this_mv, &fcenter_mv, mvsadcost, sad_per_bit); - - if (thissad < bestsad) { - bestsad = thissad; - best_mv->as_mv.row = r; - best_mv->as_mv.col = c; - bestaddress = check_here; - } - } - - check_here++; - c++; } } this_mv.as_mv.row = best_mv->as_mv.row * 8; this_mv.as_mv.col = best_mv->as_mv.col * 8; return fn_ptr->vf(what, what_stride, bestaddress, in_what_stride, &thissad) + mv_err_cost(&this_mv, center_mv, mvcost, x->errorperbit); } -#endif // HAVE_SSE4_1 int vp8_refining_search_sad_c(MACROBLOCK *x, BLOCK *b, BLOCKD *d, int_mv *ref_mv, int error_per_bit, int search_range, vp8_variance_fn_ptr_t *fn_ptr, int *mvcost[2], int_mv *center_mv) { diff --git a/media/libvpx/libvpx/vp8/encoder/onyx_if.c b/media/libvpx/libvpx/vp8/encoder/onyx_if.c --- a/media/libvpx/libvpx/vp8/encoder/onyx_if.c +++ b/media/libvpx/libvpx/vp8/encoder/onyx_if.c @@ -34,10 +34,11 @@ #include "vpx_mem/vpx_mem.h" #include "vp8/common/reconintra.h" #include "vp8/common/swapyv12buffer.h" #include "vp8/common/threading.h" #include "vpx_ports/system_state.h" +#include "vpx_ports/vpx_once.h" #include "vpx_ports/vpx_timer.h" #include "vpx_util/vpx_write_yuv_frame.h" #if VPX_ARCH_ARM #include "vpx_ports/arm.h" #endif @@ -181,11 +182,11 @@ #ifdef PACKET_TESTING extern FILE *vpxlogc; #endif -static void save_layer_context(VP8_COMP *cpi) { +void vp8_save_layer_context(VP8_COMP *cpi) { LAYER_CONTEXT *lc = &cpi->layer_context[cpi->current_layer]; /* Save layer dependent coding state */ lc->target_bandwidth = cpi->target_bandwidth; lc->starting_buffer_level = cpi->oxcf.starting_buffer_level; @@ -220,11 +221,11 @@ memcpy(lc->count_mb_ref_frame_usage, cpi->mb.count_mb_ref_frame_usage, sizeof(cpi->mb.count_mb_ref_frame_usage)); } -static void restore_layer_context(VP8_COMP *cpi, const int layer) { +void vp8_restore_layer_context(VP8_COMP *cpi, const int layer) { LAYER_CONTEXT *lc = &cpi->layer_context[layer]; /* Restore layer dependent coding state */ cpi->current_layer = layer; cpi->target_bandwidth = lc->target_bandwidth; @@ -267,13 +268,13 @@ int64_t llval = val; return (int)(llval * llnum / llden); } -static void init_temporal_layer_context(VP8_COMP *cpi, VP8_CONFIG *oxcf, - const int layer, - double prev_layer_framerate) { +void vp8_init_temporal_layer_context(VP8_COMP *cpi, VP8_CONFIG *oxcf, + const int layer, + double prev_layer_framerate) { LAYER_CONTEXT *lc = &cpi->layer_context[layer]; lc->framerate = cpi->output_framerate / cpi->oxcf.rate_decimator[layer]; lc->target_bandwidth = cpi->oxcf.target_bitrate[layer] * 1000; @@ -334,16 +335,16 @@ const int curr_num_layers = cpi->oxcf.number_of_layers; // If the previous state was 1 layer, get current layer context from cpi. // We need this to set the layer context for the new layers below. if (prev_num_layers == 1) { cpi->current_layer = 0; - save_layer_context(cpi); + vp8_save_layer_context(cpi); } for (i = 0; i < curr_num_layers; ++i) { LAYER_CONTEXT *lc = &cpi->layer_context[i]; if (i >= prev_num_layers) { - init_temporal_layer_context(cpi, oxcf, i, prev_layer_framerate); + vp8_init_temporal_layer_context(cpi, oxcf, i, prev_layer_framerate); } // The initial buffer levels are set based on their starting levels. // We could set the buffer levels based on the previous state (normalized // properly by the layer bandwidths) but we would need to keep track of // the previous set of layer bandwidths (i.e., target_bitrate[i]) @@ -354,19 +355,19 @@ // TDOD(marpan): Should we set the rate_correction_factor and // active_worst/best_quality to values derived from the previous layer // state (to smooth-out quality dips/rate fluctuation at transition)? // We need to treat the 1 layer case separately: oxcf.target_bitrate[i] - // is not set for 1 layer, and the restore_layer_context/save_context() + // is not set for 1 layer, and the vp8_restore_layer_context/save_context() // are not called in the encoding loop, so we need to call it here to // pass the layer context state to |cpi|. if (curr_num_layers == 1) { lc->target_bandwidth = cpi->oxcf.target_bandwidth; lc->buffer_level = cpi->oxcf.starting_buffer_level_in_ms * lc->target_bandwidth / 1000; lc->bits_off_target = lc->buffer_level; - restore_layer_context(cpi, 0); + vp8_restore_layer_context(cpi, 0); } prev_layer_framerate = cpi->output_framerate / cpi->oxcf.rate_decimator[i]; } } @@ -392,20 +393,17 @@ set_default_lf_deltas(cpi); } static void dealloc_raw_frame_buffers(VP8_COMP *cpi); -void vp8_initialize_enc(void) { - static volatile int init_done = 0; - - if (!init_done) { - vpx_dsp_rtcd(); - vp8_init_intra_predictors(); - init_done = 1; - } +static void initialize_enc(void) { + vpx_dsp_rtcd(); + vp8_init_intra_predictors(); } +void vp8_initialize_enc(void) { once(initialize_enc); } + static void dealloc_compressor_data(VP8_COMP *cpi) { vpx_free(cpi->tplist); cpi->tplist = NULL; /* Delete last frame MV storage buffers */ @@ -1021,11 +1019,11 @@ /* This has a big hit on quality. Last resort */ if (Speed >= 15) sf->half_pixel_search = 0; memset(cpi->mb.error_bins, 0, sizeof(cpi->mb.error_bins)); - }; /* switch */ + } /* switch */ /* Slow quant, dct and trellis not worthwhile for first pass * so make sure they are always turned off. */ if (cpi->pass == 1) { @@ -1272,11 +1270,11 @@ if (framerate < .1) framerate = 30; cpi->framerate = framerate; cpi->output_framerate = framerate; cpi->per_frame_bandwidth = - (int)(cpi->oxcf.target_bandwidth / cpi->output_framerate); + (int)round(cpi->oxcf.target_bandwidth / cpi->output_framerate); cpi->av_per_frame_bandwidth = cpi->per_frame_bandwidth; cpi->min_frame_bandwidth = (int)(cpi->av_per_frame_bandwidth * cpi->oxcf.two_pass_vbrmin_section / 100); /* Set Maximum gf/arf interval */ @@ -1363,11 +1361,11 @@ if (cpi->oxcf.number_of_layers > 1) { unsigned int i; double prev_layer_framerate = 0; for (i = 0; i < cpi->oxcf.number_of_layers; ++i) { - init_temporal_layer_context(cpi, oxcf, i, prev_layer_framerate); + vp8_init_temporal_layer_context(cpi, oxcf, i, prev_layer_framerate); prev_layer_framerate = cpi->output_framerate / cpi->oxcf.rate_decimator[i]; } } @@ -1380,11 +1378,11 @@ for (i = 1; i < 512; ++i) cpi->fixed_divide[i] = 0x80000 / i; } #endif } -static void update_layer_contexts(VP8_COMP *cpi) { +void vp8_update_layer_contexts(VP8_COMP *cpi) { VP8_CONFIG *oxcf = &cpi->oxcf; /* Update snapshots of the layer contexts to reflect new parameters */ if (oxcf->number_of_layers > 1) { unsigned int i; @@ -1415,12 +1413,12 @@ } /* Work out the average size of a frame within this layer */ if (i > 0) { lc->avg_frame_size_for_layer = - (int)((oxcf->target_bitrate[i] - oxcf->target_bitrate[i - 1]) * - 1000 / (lc->framerate - prev_layer_framerate)); + (int)round((oxcf->target_bitrate[i] - oxcf->target_bitrate[i - 1]) * + 1000 / (lc->framerate - prev_layer_framerate)); } prev_layer_framerate = lc->framerate; } } @@ -1908,10 +1906,11 @@ cpi->source_alt_ref_active = 0; cpi->common.refresh_alt_ref_frame = 0; cpi->force_maxqp = 0; cpi->frames_since_last_drop_overshoot = 0; + cpi->rt_always_update_correction_factor = 0; cpi->b_calculate_psnr = CONFIG_INTERNAL_STATS; #if CONFIG_INTERNAL_STATS cpi->b_calculate_ssimg = 0; @@ -2011,51 +2010,40 @@ #endif cpi->fn_ptr[BLOCK_16X16].sdf = vpx_sad16x16; cpi->fn_ptr[BLOCK_16X16].vf = vpx_variance16x16; cpi->fn_ptr[BLOCK_16X16].svf = vpx_sub_pixel_variance16x16; - cpi->fn_ptr[BLOCK_16X16].sdx3f = vpx_sad16x16x3; - cpi->fn_ptr[BLOCK_16X16].sdx8f = vpx_sad16x16x8; cpi->fn_ptr[BLOCK_16X16].sdx4df = vpx_sad16x16x4d; cpi->fn_ptr[BLOCK_16X8].sdf = vpx_sad16x8; cpi->fn_ptr[BLOCK_16X8].vf = vpx_variance16x8; cpi->fn_ptr[BLOCK_16X8].svf = vpx_sub_pixel_variance16x8; - cpi->fn_ptr[BLOCK_16X8].sdx3f = vpx_sad16x8x3; - cpi->fn_ptr[BLOCK_16X8].sdx8f = vpx_sad16x8x8; cpi->fn_ptr[BLOCK_16X8].sdx4df = vpx_sad16x8x4d; cpi->fn_ptr[BLOCK_8X16].sdf = vpx_sad8x16; cpi->fn_ptr[BLOCK_8X16].vf = vpx_variance8x16; cpi->fn_ptr[BLOCK_8X16].svf = vpx_sub_pixel_variance8x16; - cpi->fn_ptr[BLOCK_8X16].sdx3f = vpx_sad8x16x3; - cpi->fn_ptr[BLOCK_8X16].sdx8f = vpx_sad8x16x8; cpi->fn_ptr[BLOCK_8X16].sdx4df = vpx_sad8x16x4d; cpi->fn_ptr[BLOCK_8X8].sdf = vpx_sad8x8; cpi->fn_ptr[BLOCK_8X8].vf = vpx_variance8x8; cpi->fn_ptr[BLOCK_8X8].svf = vpx_sub_pixel_variance8x8; - cpi->fn_ptr[BLOCK_8X8].sdx3f = vpx_sad8x8x3; - cpi->fn_ptr[BLOCK_8X8].sdx8f = vpx_sad8x8x8; cpi->fn_ptr[BLOCK_8X8].sdx4df = vpx_sad8x8x4d; cpi->fn_ptr[BLOCK_4X4].sdf = vpx_sad4x4; cpi->fn_ptr[BLOCK_4X4].vf = vpx_variance4x4; cpi->fn_ptr[BLOCK_4X4].svf = vpx_sub_pixel_variance4x4; - cpi->fn_ptr[BLOCK_4X4].sdx3f = vpx_sad4x4x3; - cpi->fn_ptr[BLOCK_4X4].sdx8f = vpx_sad4x4x8; cpi->fn_ptr[BLOCK_4X4].sdx4df = vpx_sad4x4x4d; #if VPX_ARCH_X86 || VPX_ARCH_X86_64 cpi->fn_ptr[BLOCK_16X16].copymem = vp8_copy32xn; cpi->fn_ptr[BLOCK_16X8].copymem = vp8_copy32xn; cpi->fn_ptr[BLOCK_8X16].copymem = vp8_copy32xn; cpi->fn_ptr[BLOCK_8X8].copymem = vp8_copy32xn; cpi->fn_ptr[BLOCK_4X4].copymem = vp8_copy32xn; #endif - cpi->full_search_sad = vp8_full_search_sad; cpi->diamond_search_sad = vp8_diamond_search_sad; cpi->refining_search_sad = vp8_refining_search_sad; /* make sure frame 1 is okay */ cpi->mb.error_bins[0] = cpi->common.MBs; @@ -3258,11 +3246,11 @@ } break; #endif // !CONFIG_REALTIME_ONLY default: cpi->per_frame_bandwidth = - (int)(cpi->target_bandwidth / cpi->output_framerate); + (int)round(cpi->target_bandwidth / cpi->output_framerate); break; } /* Default turn off buffer to buffer copying */ cm->copy_buffer_to_gf = 0; @@ -3478,11 +3466,11 @@ * at each frame. * * Note that dropping a key frame can be problematic if spatial * resampling is also active */ - if (cpi->decimation_factor > 0) { + if (cpi->decimation_factor > 0 && cpi->drop_frames_allowed) { switch (cpi->decimation_factor) { case 1: cpi->per_frame_bandwidth = cpi->per_frame_bandwidth * 3 / 2; break; case 2: @@ -4014,11 +4002,12 @@ vpx_clear_system_state(); if (frame_over_shoot_limit == 0) frame_over_shoot_limit = 1; /* Are we are overshooting and up against the limit of active max Q. */ - if (((cpi->pass != 2) || + if (!cpi->rt_always_update_correction_factor && + ((cpi->pass != 2) || (cpi->oxcf.end_usage == USAGE_STREAM_FROM_SERVER)) && (Q == cpi->active_worst_quality) && (cpi->active_worst_quality < cpi->worst_quality) && (cpi->projected_frame_size > frame_over_shoot_limit)) { int over_size_percent = @@ -4512,14 +4501,14 @@ /* Clip the buffer level to the maximum specified buffer size */ if (cpi->bits_off_target > cpi->oxcf.maximum_buffer_size) { cpi->bits_off_target = cpi->oxcf.maximum_buffer_size; } - // If the frame dropper is not enabled, don't let the buffer level go below - // some threshold, given here by -|maximum_buffer_size|. For now we only do - // this for screen content input. - if (cpi->drop_frames_allowed == 0 && cpi->oxcf.screen_content_mode && + // Don't let the buffer level go below some threshold, given here + // by -|maximum_buffer_size|. For now we only do this for + // screen content input. + if (cpi->oxcf.screen_content_mode && cpi->bits_off_target < -cpi->oxcf.maximum_buffer_size) { cpi->bits_off_target = -cpi->oxcf.maximum_buffer_size; } /* Rolling monitors of whether we are over or underspending used to @@ -4550,12 +4539,12 @@ if (cpi->oxcf.number_of_layers > 1) { unsigned int i; for (i = cpi->current_layer + 1; i < cpi->oxcf.number_of_layers; ++i) { LAYER_CONTEXT *lc = &cpi->layer_context[i]; - int bits_off_for_this_layer = (int)(lc->target_bandwidth / lc->framerate - - cpi->projected_frame_size); + int bits_off_for_this_layer = (int)round( + lc->target_bandwidth / lc->framerate - cpi->projected_frame_size); lc->bits_off_target += bits_off_for_this_layer; /* Clip buffer level to maximum buffer size for the layer */ if (lc->bits_off_target > lc->maximum_buffer_size) { @@ -4988,21 +4977,21 @@ } if (cpi->oxcf.number_of_layers > 1) { int layer; - update_layer_contexts(cpi); + vp8_update_layer_contexts(cpi); /* Restore layer specific context & set frame rate */ if (cpi->temporal_layer_id >= 0) { layer = cpi->temporal_layer_id; } else { layer = cpi->oxcf .layer_id[cpi->temporal_pattern_counter % cpi->oxcf.periodicity]; } - restore_layer_context(cpi, layer); + vp8_restore_layer_context(cpi, layer); vp8_new_framerate(cpi, cpi->layer_context[layer].framerate); } if (cpi->compressor_speed == 2) { vpx_usec_timer_start(&tsctimer); @@ -5129,11 +5118,11 @@ cm->refresh_last_frame = 1; cm->frame_type = INTER_FRAME; } /* Save layer specific state */ - if (cpi->oxcf.number_of_layers > 1) save_layer_context(cpi); + if (cpi->oxcf.number_of_layers > 1) vp8_save_layer_context(cpi); vpx_usec_timer_mark(&cmptimer); cpi->time_compress_data += vpx_usec_timer_elapsed(&cmptimer); if (cpi->b_calculate_psnr && cpi->pass != 1 && cm->show_frame) { diff --git a/media/libvpx/libvpx/vp8/encoder/onyx_int.h b/media/libvpx/libvpx/vp8/encoder/onyx_int.h --- a/media/libvpx/libvpx/vp8/encoder/onyx_int.h +++ b/media/libvpx/libvpx/vp8/encoder/onyx_int.h @@ -547,11 +547,10 @@ unsigned int partition_sz[MAX_PARTITIONS]; unsigned char *partition_d[MAX_PARTITIONS]; unsigned char *partition_d_end[MAX_PARTITIONS]; fractional_mv_step_fp *find_fractional_mv_step; - vp8_full_search_fn_t full_search_sad; vp8_refining_search_fn_t refining_search_sad; vp8_diamond_search_fn_t diamond_search_sad; vp8_variance_fn_ptr_t fn_ptr[BLOCK_MAX_SEGMENTS]; uint64_t time_receive_data; uint64_t time_compress_data; @@ -700,16 +699,26 @@ // Use the static threshold from ROI settings. int use_roi_static_threshold; int ext_refresh_frame_flags_pending; + + // Always update correction factor used for rate control after each frame for + // realtime encoding. + int rt_always_update_correction_factor; } VP8_COMP; void vp8_initialize_enc(void); void vp8_alloc_compressor_data(VP8_COMP *cpi); int vp8_reverse_trans(int x); +void vp8_init_temporal_layer_context(VP8_COMP *cpi, VP8_CONFIG *oxcf, + const int layer, + double prev_layer_framerate); +void vp8_update_layer_contexts(VP8_COMP *cpi); +void vp8_save_layer_context(VP8_COMP *cpi); +void vp8_restore_layer_context(VP8_COMP *cpi, const int layer); void vp8_new_framerate(VP8_COMP *cpi, double framerate); void vp8_loopfilter_frame(VP8_COMP *cpi, VP8_COMMON *cm); void vp8_pack_bitstream(VP8_COMP *cpi, unsigned char *dest, unsigned char *dest_end, size_t *size); diff --git a/media/libvpx/libvpx/vp8/encoder/ratectrl.c b/media/libvpx/libvpx/vp8/encoder/ratectrl.c --- a/media/libvpx/libvpx/vp8/encoder/ratectrl.c +++ b/media/libvpx/libvpx/vp8/encoder/ratectrl.c @@ -312,11 +312,11 @@ else if (cpi->common.current_video_frame == 0) { /* 1 Pass there is no information on which to base size so use * bandwidth per second * fraction of the initial buffer * level */ - target = cpi->oxcf.starting_buffer_level / 2; + target = (uint64_t)cpi->oxcf.starting_buffer_level / 2; if (target > cpi->oxcf.target_bandwidth * 3 / 2) { target = cpi->oxcf.target_bandwidth * 3 / 2; } } else { @@ -325,11 +325,12 @@ : cpi->ni_av_qi; int initial_boost = 32; /* |3.0 * per_frame_bandwidth| */ /* Boost depends somewhat on frame rate: only used for 1 layer case. */ if (cpi->oxcf.number_of_layers == 1) { - kf_boost = VPXMAX(initial_boost, (int)(2 * cpi->output_framerate - 16)); + kf_boost = + VPXMAX(initial_boost, (int)round(2 * cpi->output_framerate - 16)); } else { /* Initial factor: set target size to: |3.0 * per_frame_bandwidth|. */ kf_boost = initial_boost; } @@ -778,10 +779,11 @@ } else { cpi->active_worst_quality = cpi->worst_quality; } } else { int percent_high = 0; + int64_t target = cpi->this_frame_target; if ((cpi->oxcf.end_usage == USAGE_STREAM_FROM_SERVER) && (cpi->buffer_level > cpi->oxcf.optimal_buffer_level)) { percent_high = (int)((cpi->buffer_level - cpi->oxcf.optimal_buffer_level) / @@ -795,11 +797,13 @@ percent_high = cpi->oxcf.over_shoot_pct; } else if (percent_high < 0) { percent_high = 0; } - cpi->this_frame_target += (cpi->this_frame_target * percent_high) / 200; + target += (target * percent_high) / 200; + target = VPXMIN(target, INT_MAX); + cpi->this_frame_target = (int)target; /* Are we allowing control of active_worst_allowed_q according * to buffer level. */ if (cpi->auto_worst_q && cpi->ni_frames > 150) { @@ -1076,12 +1080,12 @@ } } /* Work out a size correction factor. */ if (projected_size_based_on_q > 0) { - correction_factor = - (100 * cpi->projected_frame_size) / projected_size_based_on_q; + correction_factor = (int)((100 * (int64_t)cpi->projected_frame_size) / + projected_size_based_on_q); } /* More heavily damped adjustment used if we have been oscillating * either side of target */ diff --git a/media/libvpx/libvpx/vp8/encoder/rdopt.c b/media/libvpx/libvpx/vp8/encoder/rdopt.c --- a/media/libvpx/libvpx/vp8/encoder/rdopt.c +++ b/media/libvpx/libvpx/vp8/encoder/rdopt.c @@ -1095,12 +1095,12 @@ if ((cpi->compressor_speed == 0) && (bestsme >> sseshift) > 4000) { /* Check if mvp_full is within the range. */ vp8_clamp_mv(&mvp_full, x->mv_col_min, x->mv_col_max, x->mv_row_min, x->mv_row_max); - thissme = cpi->full_search_sad(x, c, e, &mvp_full, sadpb, 16, - v_fn_ptr, x->mvcost, bsi->ref_mv); + thissme = vp8_full_search_sad(x, c, e, &mvp_full, sadpb, 16, + v_fn_ptr, x->mvcost, bsi->ref_mv); if (thissme < bestsme) { bestsme = thissme; mode_mv[NEW4X4].as_int = e->bmi.mv.as_int; } else { diff --git a/media/libvpx/libvpx/vp8/encoder/x86/quantize_sse4.c b/media/libvpx/libvpx/vp8/encoder/x86/quantize_sse4.c --- a/media/libvpx/libvpx/vp8/encoder/x86/quantize_sse4.c +++ b/media/libvpx/libvpx/vp8/encoder/x86/quantize_sse4.c @@ -9,32 +9,18 @@ */ #include /* SSE4.1 */ #include "./vp8_rtcd.h" -#include "vp8/common/entropy.h" /* vp8_default_inv_zig_zag */ #include "vp8/encoder/block.h" - -#define SELECT_EOB(i, z, x, y, q) \ - do { \ - short boost = *zbin_boost_ptr; \ - /* Technically _mm_extract_epi16() returns an int: */ \ - /* https://bugs.llvm.org/show_bug.cgi?id=41657 */ \ - short x_z = (short)_mm_extract_epi16(x, z); \ - short y_z = (short)_mm_extract_epi16(y, z); \ - int cmp = (x_z < boost) | (y_z == 0); \ - zbin_boost_ptr++; \ - if (cmp) break; \ - q = _mm_insert_epi16(q, y_z, z); \ - eob = i; \ - zbin_boost_ptr = b->zrun_zbin_boost; \ - } while (0) +#include "vpx_ports/bitops.h" /* get_lsb */ void vp8_regular_quantize_b_sse4_1(BLOCK *b, BLOCKD *d) { - char eob = 0; + int eob = -1; short *zbin_boost_ptr = b->zrun_zbin_boost; - + __m128i zbin_boost0 = _mm_load_si128((__m128i *)(zbin_boost_ptr)); + __m128i zbin_boost1 = _mm_load_si128((__m128i *)(zbin_boost_ptr + 8)); __m128i x0, x1, y0, y1, x_minus_zbin0, x_minus_zbin1, dqcoeff0, dqcoeff1; __m128i quant_shift0 = _mm_load_si128((__m128i *)(b->quant_shift)); __m128i quant_shift1 = _mm_load_si128((__m128i *)(b->quant_shift + 8)); __m128i z0 = _mm_load_si128((__m128i *)(b->coeff)); __m128i z1 = _mm_load_si128((__m128i *)(b->coeff + 8)); @@ -45,12 +31,16 @@ __m128i round1 = _mm_load_si128((__m128i *)(b->round + 8)); __m128i quant0 = _mm_load_si128((__m128i *)(b->quant)); __m128i quant1 = _mm_load_si128((__m128i *)(b->quant + 8)); __m128i dequant0 = _mm_load_si128((__m128i *)(d->dequant)); __m128i dequant1 = _mm_load_si128((__m128i *)(d->dequant + 8)); - __m128i qcoeff0 = _mm_setzero_si128(); - __m128i qcoeff1 = _mm_setzero_si128(); + __m128i qcoeff0, qcoeff1, t0, t1, x_shuf0, x_shuf1; + uint32_t mask, ymask; + DECLARE_ALIGNED(16, static const uint8_t, + zig_zag_mask[16]) = { 0, 1, 4, 8, 5, 2, 3, 6, + 9, 12, 13, 10, 7, 11, 14, 15 }; + DECLARE_ALIGNED(16, uint16_t, qcoeff[16]) = { 0 }; /* Duplicate to all lanes. */ zbin_extra = _mm_shufflelo_epi16(zbin_extra, 0); zbin_extra = _mm_unpacklo_epi16(zbin_extra, zbin_extra); @@ -86,34 +76,63 @@ /* Restore the sign. */ y0 = _mm_sign_epi16(y0, z0); y1 = _mm_sign_epi16(y1, z1); - /* The loop gets unrolled anyway. Avoid the vp8_default_zig_zag1d lookup. */ - SELECT_EOB(1, 0, x_minus_zbin0, y0, qcoeff0); - SELECT_EOB(2, 1, x_minus_zbin0, y0, qcoeff0); - SELECT_EOB(3, 4, x_minus_zbin0, y0, qcoeff0); - SELECT_EOB(4, 0, x_minus_zbin1, y1, qcoeff1); - SELECT_EOB(5, 5, x_minus_zbin0, y0, qcoeff0); - SELECT_EOB(6, 2, x_minus_zbin0, y0, qcoeff0); - SELECT_EOB(7, 3, x_minus_zbin0, y0, qcoeff0); - SELECT_EOB(8, 6, x_minus_zbin0, y0, qcoeff0); - SELECT_EOB(9, 1, x_minus_zbin1, y1, qcoeff1); - SELECT_EOB(10, 4, x_minus_zbin1, y1, qcoeff1); - SELECT_EOB(11, 5, x_minus_zbin1, y1, qcoeff1); - SELECT_EOB(12, 2, x_minus_zbin1, y1, qcoeff1); - SELECT_EOB(13, 7, x_minus_zbin0, y0, qcoeff0); - SELECT_EOB(14, 3, x_minus_zbin1, y1, qcoeff1); - SELECT_EOB(15, 6, x_minus_zbin1, y1, qcoeff1); - SELECT_EOB(16, 7, x_minus_zbin1, y1, qcoeff1); + { + const __m128i zig_zag_i16_0 = + _mm_setr_epi8(0, 1, 2, 3, 8, 9, 14, 15, 10, 11, 4, 5, 6, 7, 12, 13); + const __m128i zig_zag_i16_1 = + _mm_setr_epi8(0, 1, 6, 7, 8, 9, 2, 3, 14, 15, 4, 5, 10, 11, 12, 13); + + /* The first part of the zig zag needs a value + * from x_minus_zbin1 and vice versa. */ + t1 = _mm_alignr_epi8(x_minus_zbin1, x_minus_zbin1, 2); + t0 = _mm_blend_epi16(x_minus_zbin0, t1, 0x80); + t1 = _mm_blend_epi16(t1, x_minus_zbin0, 0x80); + x_shuf0 = _mm_shuffle_epi8(t0, zig_zag_i16_0); + x_shuf1 = _mm_shuffle_epi8(t1, zig_zag_i16_1); + } + + /* Check if y is nonzero and put it in zig zag order. */ + t0 = _mm_packs_epi16(y0, y1); + t0 = _mm_cmpeq_epi8(t0, _mm_setzero_si128()); + t0 = _mm_shuffle_epi8(t0, _mm_load_si128((const __m128i *)zig_zag_mask)); + ymask = _mm_movemask_epi8(t0) ^ 0xffff; + + for (;;) { + t0 = _mm_cmpgt_epi16(zbin_boost0, x_shuf0); + t1 = _mm_cmpgt_epi16(zbin_boost1, x_shuf1); + t0 = _mm_packs_epi16(t0, t1); + mask = _mm_movemask_epi8(t0); + mask = ~mask & ymask; + if (!mask) break; + /* |eob| will contain the index of the next found element where: + * boost[i - old_eob - 1] <= x[zigzag[i]] && y[zigzag[i]] != 0 */ + eob = get_lsb(mask); + /* Need to clear the mask from processed elements so that + * they are no longer counted in the next iteration. */ + ymask &= ~1U << eob; + /* It's safe to read ahead of this buffer if struct VP8_COMP has at + * least 32 bytes before the zrun_zbin_boost_* fields (it has 384). + * Any data read outside of the buffer is masked by the updated |ymask|. */ + zbin_boost0 = _mm_loadu_si128((__m128i *)(zbin_boost_ptr - eob - 1)); + zbin_boost1 = _mm_loadu_si128((__m128i *)(zbin_boost_ptr - eob + 7)); + qcoeff[zig_zag_mask[eob]] = 0xffff; + } + + qcoeff0 = _mm_load_si128((__m128i *)(qcoeff)); + qcoeff1 = _mm_load_si128((__m128i *)(qcoeff + 8)); + qcoeff0 = _mm_and_si128(qcoeff0, y0); + qcoeff1 = _mm_and_si128(qcoeff1, y1); _mm_store_si128((__m128i *)(d->qcoeff), qcoeff0); _mm_store_si128((__m128i *)(d->qcoeff + 8), qcoeff1); dqcoeff0 = _mm_mullo_epi16(qcoeff0, dequant0); dqcoeff1 = _mm_mullo_epi16(qcoeff1, dequant1); _mm_store_si128((__m128i *)(d->dqcoeff), dqcoeff0); _mm_store_si128((__m128i *)(d->dqcoeff + 8), dqcoeff1); - *d->eob = eob; + *d->eob = eob + 1; } diff --git a/media/libvpx/libvpx/vp8/encoder/x86/vp8_quantize_ssse3.c b/media/libvpx/libvpx/vp8/encoder/x86/vp8_quantize_ssse3.c --- a/media/libvpx/libvpx/vp8/encoder/x86/vp8_quantize_ssse3.c +++ b/media/libvpx/libvpx/vp8/encoder/x86/vp8_quantize_ssse3.c @@ -10,35 +10,11 @@ #include /* SSSE3 */ #include "./vp8_rtcd.h" #include "vp8/encoder/block.h" - -/* bitscan reverse (bsr) */ -#if defined(_MSC_VER) -#include -#pragma intrinsic(_BitScanReverse) -static int bsr(int mask) { - unsigned long eob; - _BitScanReverse(&eob, mask); - eob++; - if (mask == 0) eob = 0; - return eob; -} -#else -static int bsr(int mask) { - int eob; -#if defined(__GNUC__) && __GNUC__ - __asm__ __volatile__("bsr %1, %0" : "=r"(eob) : "r"(mask) : "flags"); -#elif defined(__SUNPRO_C) || defined(__SUNPRO_CC) - asm volatile("bsr %1, %0" : "=r"(eob) : "r"(mask) : "flags"); -#endif - eob++; - if (mask == 0) eob = 0; - return eob; -} -#endif +#include "vpx_ports/bitops.h" /* get_msb */ void vp8_fast_quantize_b_ssse3(BLOCK *b, BLOCKD *d) { int eob, mask; __m128i z0 = _mm_load_si128((__m128i *)(b->coeff)); @@ -106,9 +82,12 @@ x = _mm_shuffle_epi8(x, zig_zag); mask = _mm_movemask_epi8(x); - eob = bsr(mask); + /* x2 is needed to increase the result from non-zero masks by 1, + * +1 is needed to mask undefined behavior for a null argument, + * the result of get_msb(1) is 0 */ + eob = get_msb(mask * 2 + 1); - *d->eob = 0xFF & eob; + *d->eob = eob; } diff --git a/media/libvpx/libvpx/vp8/vp8_common.mk b/media/libvpx/libvpx/vp8/vp8_common.mk --- a/media/libvpx/libvpx/vp8/vp8_common.mk +++ b/media/libvpx/libvpx/vp8/vp8_common.mk @@ -122,10 +122,15 @@ ifeq ($(CONFIG_POSTPROC),yes) VP8_COMMON_SRCS-$(HAVE_MSA) += common/mips/msa/mfqe_msa.c endif +# common (loongarch LSX intrinsics) +VP8_COMMON_SRCS-$(HAVE_LSX) += common/loongarch/loopfilter_filters_lsx.c +VP8_COMMON_SRCS-$(HAVE_LSX) += common/loongarch/sixtap_filter_lsx.c +VP8_COMMON_SRCS-$(HAVE_LSX) += common/loongarch/idct_lsx.c + # common (neon intrinsics) VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/loopfilter_arm.c VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/loopfilter_arm.h VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/bilinearpredict_neon.c VP8_COMMON_SRCS-$(HAVE_NEON) += common/arm/neon/copymem_neon.c diff --git a/media/libvpx/libvpx/vp8/vp8_cx_iface.c b/media/libvpx/libvpx/vp8/vp8_cx_iface.c --- a/media/libvpx/libvpx/vp8/vp8_cx_iface.c +++ b/media/libvpx/libvpx/vp8/vp8_cx_iface.c @@ -16,11 +16,10 @@ #include "vpx/internal/vpx_codec_internal.h" #include "vpx_version.h" #include "vpx_mem/vpx_mem.h" #include "vpx_ports/static_assert.h" #include "vpx_ports/system_state.h" -#include "vpx_ports/vpx_once.h" #include "vpx_util/vpx_timestamp.h" #include "vp8/encoder/onyx_int.h" #include "vpx/vp8cx.h" #include "vp8/encoder/firstpass.h" #include "vp8/common/onyx.h" @@ -337,11 +336,13 @@ oxcf->end_usage = USAGE_CONSTRAINED_QUALITY; } else if (cfg.rc_end_usage == VPX_Q) { oxcf->end_usage = USAGE_CONSTANT_QUALITY; } - oxcf->target_bandwidth = cfg.rc_target_bitrate; + // Cap the target rate to 1000 Mbps to avoid some integer overflows in + // target bandwidth calculations. + oxcf->target_bandwidth = VPXMIN(cfg.rc_target_bitrate, 1000000); oxcf->rc_max_intra_bitrate_pct = vp8_cfg.rc_max_intra_bitrate_pct; oxcf->gf_cbr_boost_pct = vp8_cfg.gf_cbr_boost_pct; oxcf->best_allowed_q = cfg.rc_min_quantizer; oxcf->worst_allowed_q = cfg.rc_max_quantizer; @@ -603,10 +604,21 @@ struct vp8_extracfg extra_cfg = ctx->vp8_cfg; extra_cfg.screen_content_mode = CAST(VP8E_SET_SCREEN_CONTENT_MODE, args); return update_extracfg(ctx, &extra_cfg); } +static vpx_codec_err_t ctrl_set_rtc_external_ratectrl(vpx_codec_alg_priv_t *ctx, + va_list args) { + VP8_COMP *cpi = ctx->cpi; + const unsigned int data = CAST(VP8E_SET_GF_CBR_BOOST_PCT, args); + if (data) { + cpi->cyclic_refresh_mode_enabled = 0; + cpi->rt_always_update_correction_factor = 1; + } + return VPX_CODEC_OK; +} + static vpx_codec_err_t vp8e_mr_alloc_mem(const vpx_codec_enc_cfg_t *cfg, void **mem_loc) { vpx_codec_err_t res = VPX_CODEC_OK; #if CONFIG_MULTI_RES_ENCODING @@ -679,11 +691,11 @@ ctx->priv->enc.total_encoders = mr_cfg->mr_total_resolutions; } else { ctx->priv->enc.total_encoders = 1; } - once(vp8_initialize_enc); + vp8_initialize_enc(); res = validate_config(priv, &priv->cfg, &priv->vp8_cfg, 0); if (!res) { priv->pts_offset_initialized = 0; @@ -924,23 +936,14 @@ ctx->timestamp_ratio.num / ctx->timestamp_ratio.den; if (img != NULL) { res = image2yuvconfig(img, &sd); - if (sd.y_width != ctx->cfg.g_w || sd.y_height != ctx->cfg.g_h) { - /* from vpx_encoder.h for g_w/g_h: - "Note that the frames passed as input to the encoder must have this - resolution" - */ - ctx->base.err_detail = "Invalid input frame resolution"; - res = VPX_CODEC_INVALID_PARAM; - } else { - if (vp8_receive_raw_frame(ctx->cpi, ctx->next_frame_flag | lib_flags, - &sd, dst_time_stamp, dst_end_time_stamp)) { - VP8_COMP *cpi = (VP8_COMP *)ctx->cpi; - res = update_error_state(ctx, &cpi->common.error); - } + if (vp8_receive_raw_frame(ctx->cpi, ctx->next_frame_flag | lib_flags, &sd, + dst_time_stamp, dst_end_time_stamp)) { + VP8_COMP *cpi = (VP8_COMP *)ctx->cpi; + res = update_error_state(ctx, &cpi->common.error); } /* reset for next frame */ ctx->next_frame_flag = 0; } @@ -1250,10 +1253,11 @@ { VP8E_SET_TUNING, set_tuning }, { VP8E_SET_CQ_LEVEL, set_cq_level }, { VP8E_SET_MAX_INTRA_BITRATE_PCT, set_rc_max_intra_bitrate_pct }, { VP8E_SET_SCREEN_CONTENT_MODE, set_screen_content_mode }, { VP8E_SET_GF_CBR_BOOST_PCT, ctrl_set_rc_gf_cbr_boost_pct }, + { VP8E_SET_RTC_EXTERNAL_RATECTRL, ctrl_set_rtc_external_ratectrl }, { -1, NULL }, }; static vpx_codec_enc_cfg_map_t vp8e_usage_cfg_map[] = { { 0, diff --git a/media/libvpx/libvpx/vp8/vp8_dx_iface.c b/media/libvpx/libvpx/vp8/vp8_dx_iface.c --- a/media/libvpx/libvpx/vp8/vp8_dx_iface.c +++ b/media/libvpx/libvpx/vp8/vp8_dx_iface.c @@ -369,12 +369,10 @@ int i; #endif pc->Width = ctx->si.w; pc->Height = ctx->si.h; { - int prev_mb_rows = pc->mb_rows; - if (setjmp(pbi->common.error.jmp)) { pbi->common.error.setjmp = 0; /* on failure clear the cached resolution to ensure a full * reallocation is attempted on resync. */ ctx->si.w = 0; @@ -396,10 +394,16 @@ pc->Height = h; vpx_internal_error(&pc->error, VPX_CODEC_CORRUPT_FRAME, "Invalid frame height"); } +#if CONFIG_MULTITHREAD + if (vpx_atomic_load_acquire(&pbi->b_multithreaded_rd)) { + vp8mt_de_alloc_temp_buffers(pbi, pc->mb_rows); + } +#endif + if (vp8_alloc_frame_buffers(pc, pc->Width, pc->Height)) { vpx_internal_error(&pc->error, VPX_CODEC_MEM_ERROR, "Failed to allocate frame buffers"); } @@ -440,14 +444,12 @@ #endif #if CONFIG_MULTITHREAD if (vpx_atomic_load_acquire(&pbi->b_multithreaded_rd)) { - vp8mt_alloc_temp_buffers(pbi, pc->Width, prev_mb_rows); + vp8mt_alloc_temp_buffers(pbi, pc->Width, 0); } -#else - (void)prev_mb_rows; #endif } pbi->common.error.setjmp = 0; diff --git a/media/libvpx/libvpx/vp8/vp8_ratectrl_rtc.h b/media/libvpx/libvpx/vp8/vp8_ratectrl_rtc.h new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vp8/vp8_ratectrl_rtc.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2021 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VPX_VP8_RATECTRL_RTC_H_ +#define VPX_VP8_RATECTRL_RTC_H_ + +#include +#include + +#include "vp8/encoder/onyx_int.h" +#include "vp8/common/common.h" +#include "vpx/internal/vpx_ratectrl_rtc.h" + +namespace libvpx { +struct VP8RateControlRtcConfig : public VpxRateControlRtcConfig { + public: + VP8RateControlRtcConfig() { + vp8_zero(layer_target_bitrate); + vp8_zero(ts_rate_decimator); + } +}; + +struct VP8FrameParamsQpRTC { + FRAME_TYPE frame_type; + int temporal_layer_id; +}; + +class VP8RateControlRTC { + public: + static std::unique_ptr Create( + const VP8RateControlRtcConfig &cfg); + ~VP8RateControlRTC() { + if (cpi_) { + vpx_free(cpi_->gf_active_flags); + vpx_free(cpi_); + } + } + + void UpdateRateControl(const VP8RateControlRtcConfig &rc_cfg); + // GetQP() needs to be called after ComputeQP() to get the latest QP + int GetQP() const; + // int GetLoopfilterLevel() const; + void ComputeQP(const VP8FrameParamsQpRTC &frame_params); + // Feedback to rate control with the size of current encoded frame + void PostEncodeUpdate(uint64_t encoded_frame_size); + + private: + VP8RateControlRTC() {} + void InitRateControl(const VP8RateControlRtcConfig &cfg); + VP8_COMP *cpi_; + int q_; +}; + +} // namespace libvpx + +#endif // VPX_VP8_RATECTRL_RTC_H_ diff --git a/media/libvpx/libvpx/vp8/vp8_ratectrl_rtc.cc b/media/libvpx/libvpx/vp8/vp8_ratectrl_rtc.cc new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vp8/vp8_ratectrl_rtc.cc @@ -0,0 +1,347 @@ +/* + * Copyright (c) 2021 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include +#include "vp8/vp8_ratectrl_rtc.h" +#include "vp8/encoder/ratectrl.h" +#include "vpx_ports/system_state.h" + +namespace libvpx { +/* Quant MOD */ +static const int kQTrans[] = { + 0, 1, 2, 3, 4, 5, 7, 8, 9, 10, 12, 13, 15, 17, 18, 19, + 20, 21, 23, 24, 25, 26, 27, 28, 29, 30, 31, 33, 35, 37, 39, 41, + 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 64, 67, 70, 73, 76, 79, + 82, 85, 88, 91, 94, 97, 100, 103, 106, 109, 112, 115, 118, 121, 124, 127, +}; + +static const unsigned char kf_high_motion_minq[QINDEX_RANGE] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, + 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 5, + 5, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 8, 8, 8, 8, 9, 9, 10, 10, + 10, 10, 11, 11, 11, 11, 12, 12, 13, 13, 13, 13, 14, 14, 15, 15, 15, 15, 16, + 16, 16, 16, 17, 17, 18, 18, 18, 18, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, + 22, 22, 23, 23, 24, 25, 25, 26, 26, 27, 28, 28, 29, 30 +}; + +static const unsigned char inter_minq[QINDEX_RANGE] = { + 0, 0, 1, 1, 2, 3, 3, 4, 4, 5, 6, 6, 7, 8, 8, 9, 9, 10, 11, + 11, 12, 13, 13, 14, 15, 15, 16, 17, 17, 18, 19, 20, 20, 21, 22, 22, 23, 24, + 24, 25, 26, 27, 27, 28, 29, 30, 30, 31, 32, 33, 33, 34, 35, 36, 36, 37, 38, + 39, 39, 40, 41, 42, 42, 43, 44, 45, 46, 46, 47, 48, 49, 50, 50, 51, 52, 53, + 54, 55, 55, 56, 57, 58, 59, 60, 60, 61, 62, 63, 64, 65, 66, 67, 67, 68, 69, + 70, 71, 72, 73, 74, 75, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 86, + 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100 +}; + +static int rescale(int val, int num, int denom) { + int64_t llnum = num; + int64_t llden = denom; + int64_t llval = val; + + return (int)(llval * llnum / llden); +} + +std::unique_ptr VP8RateControlRTC::Create( + const VP8RateControlRtcConfig &cfg) { + std::unique_ptr rc_api(new (std::nothrow) + VP8RateControlRTC()); + if (!rc_api) return nullptr; + rc_api->cpi_ = static_cast(vpx_memalign(32, sizeof(*cpi_))); + if (!rc_api->cpi_) return nullptr; + vp8_zero(*rc_api->cpi_); + + rc_api->InitRateControl(cfg); + + return rc_api; +} + +void VP8RateControlRTC::InitRateControl(const VP8RateControlRtcConfig &rc_cfg) { + VP8_COMMON *cm = &cpi_->common; + VP8_CONFIG *oxcf = &cpi_->oxcf; + oxcf->end_usage = USAGE_STREAM_FROM_SERVER; + cpi_->pass = 0; + cm->show_frame = 1; + oxcf->drop_frames_water_mark = 0; + cm->current_video_frame = 0; + cpi_->auto_gold = 1; + cpi_->key_frame_count = 1; + cpi_->rate_correction_factor = 1.0; + cpi_->key_frame_rate_correction_factor = 1.0; + cpi_->cyclic_refresh_mode_enabled = 0; + cpi_->auto_worst_q = 1; + cpi_->kf_overspend_bits = 0; + cpi_->kf_bitrate_adjustment = 0; + cpi_->gf_overspend_bits = 0; + cpi_->non_gf_bitrate_adjustment = 0; + UpdateRateControl(rc_cfg); + cpi_->buffer_level = oxcf->starting_buffer_level; + cpi_->bits_off_target = oxcf->starting_buffer_level; +} + +void VP8RateControlRTC::UpdateRateControl( + const VP8RateControlRtcConfig &rc_cfg) { + VP8_COMMON *cm = &cpi_->common; + VP8_CONFIG *oxcf = &cpi_->oxcf; + vpx_clear_system_state(); + cm->Width = rc_cfg.width; + cm->Height = rc_cfg.height; + oxcf->Width = rc_cfg.width; + oxcf->Height = rc_cfg.height; + oxcf->worst_allowed_q = kQTrans[rc_cfg.max_quantizer]; + oxcf->best_allowed_q = kQTrans[rc_cfg.min_quantizer]; + cpi_->worst_quality = oxcf->worst_allowed_q; + cpi_->best_quality = oxcf->best_allowed_q; + cpi_->output_framerate = rc_cfg.framerate; + oxcf->target_bandwidth = + static_cast(1000 * rc_cfg.target_bandwidth); + cpi_->ref_framerate = cpi_->output_framerate; + oxcf->fixed_q = -1; + oxcf->error_resilient_mode = 1; + oxcf->starting_buffer_level_in_ms = rc_cfg.buf_initial_sz; + oxcf->optimal_buffer_level_in_ms = rc_cfg.buf_optimal_sz; + oxcf->maximum_buffer_size_in_ms = rc_cfg.buf_sz; + oxcf->starting_buffer_level = rc_cfg.buf_initial_sz; + oxcf->optimal_buffer_level = rc_cfg.buf_optimal_sz; + oxcf->maximum_buffer_size = rc_cfg.buf_sz; + oxcf->number_of_layers = rc_cfg.ts_number_layers; + cpi_->buffered_mode = oxcf->optimal_buffer_level > 0; + oxcf->under_shoot_pct = rc_cfg.undershoot_pct; + oxcf->over_shoot_pct = rc_cfg.overshoot_pct; + cpi_->oxcf.rc_max_intra_bitrate_pct = rc_cfg.max_intra_bitrate_pct; + cpi_->framerate = rc_cfg.framerate; + for (int i = 0; i < KEY_FRAME_CONTEXT; ++i) { + cpi_->prior_key_frame_distance[i] = + static_cast(cpi_->output_framerate); + } + + if (oxcf->number_of_layers > 1) { + memcpy(oxcf->target_bitrate, rc_cfg.layer_target_bitrate, + sizeof(rc_cfg.layer_target_bitrate)); + memcpy(oxcf->rate_decimator, rc_cfg.ts_rate_decimator, + sizeof(rc_cfg.ts_rate_decimator)); + oxcf->periodicity = 2; + + double prev_layer_framerate = 0; + for (unsigned int i = 0; i < oxcf->number_of_layers; ++i) { + vp8_init_temporal_layer_context(cpi_, oxcf, i, prev_layer_framerate); + prev_layer_framerate = cpi_->output_framerate / oxcf->rate_decimator[i]; + } + } + + cpi_->total_actual_bits = 0; + cpi_->total_target_vs_actual = 0; + + cm->mb_rows = cm->Height >> 4; + cm->mb_cols = cm->Width >> 4; + cm->MBs = cm->mb_rows * cm->mb_cols; + cm->mode_info_stride = cm->mb_cols + 1; + + oxcf->starting_buffer_level = + rescale((int)oxcf->starting_buffer_level, oxcf->target_bandwidth, 1000); + /* Set or reset optimal and maximum buffer levels. */ + if (oxcf->optimal_buffer_level == 0) { + oxcf->optimal_buffer_level = oxcf->target_bandwidth / 8; + } else { + oxcf->optimal_buffer_level = + rescale((int)oxcf->optimal_buffer_level, oxcf->target_bandwidth, 1000); + } + if (oxcf->maximum_buffer_size == 0) { + oxcf->maximum_buffer_size = oxcf->target_bandwidth / 8; + } else { + oxcf->maximum_buffer_size = + rescale((int)oxcf->maximum_buffer_size, oxcf->target_bandwidth, 1000); + } + + if (cpi_->bits_off_target > oxcf->maximum_buffer_size) { + cpi_->bits_off_target = oxcf->maximum_buffer_size; + cpi_->buffer_level = cpi_->bits_off_target; + } + + vp8_new_framerate(cpi_, cpi_->framerate); + vpx_clear_system_state(); +} + +void VP8RateControlRTC::ComputeQP(const VP8FrameParamsQpRTC &frame_params) { + VP8_COMMON *const cm = &cpi_->common; + vpx_clear_system_state(); + if (cpi_->oxcf.number_of_layers > 1) { + cpi_->temporal_layer_id = frame_params.temporal_layer_id; + const int layer = frame_params.temporal_layer_id; + vp8_update_layer_contexts(cpi_); + /* Restore layer specific context & set frame rate */ + vp8_restore_layer_context(cpi_, layer); + vp8_new_framerate(cpi_, cpi_->layer_context[layer].framerate); + } + cm->frame_type = frame_params.frame_type; + cm->refresh_golden_frame = (cm->frame_type == KEY_FRAME) ? 1 : 0; + cm->refresh_alt_ref_frame = (cm->frame_type == KEY_FRAME) ? 1 : 0; + if (cm->frame_type == KEY_FRAME && cpi_->common.current_video_frame > 0) { + cpi_->common.frame_flags |= FRAMEFLAGS_KEY; + } + + vp8_pick_frame_size(cpi_); + + if (cpi_->buffer_level >= cpi_->oxcf.optimal_buffer_level && + cpi_->buffered_mode) { + /* Max adjustment is 1/4 */ + int Adjustment = cpi_->active_worst_quality / 4; + if (Adjustment) { + int buff_lvl_step; + if (cpi_->buffer_level < cpi_->oxcf.maximum_buffer_size) { + buff_lvl_step = (int)((cpi_->oxcf.maximum_buffer_size - + cpi_->oxcf.optimal_buffer_level) / + Adjustment); + if (buff_lvl_step) { + Adjustment = + (int)((cpi_->buffer_level - cpi_->oxcf.optimal_buffer_level) / + buff_lvl_step); + } else { + Adjustment = 0; + } + } + cpi_->active_worst_quality -= Adjustment; + if (cpi_->active_worst_quality < cpi_->active_best_quality) { + cpi_->active_worst_quality = cpi_->active_best_quality; + } + } + } + + if (cpi_->ni_frames > 150) { + int q = cpi_->active_worst_quality; + if (cm->frame_type == KEY_FRAME) { + cpi_->active_best_quality = kf_high_motion_minq[q]; + } else { + cpi_->active_best_quality = inter_minq[q]; + } + + if (cpi_->buffer_level >= cpi_->oxcf.maximum_buffer_size) { + cpi_->active_best_quality = cpi_->best_quality; + + } else if (cpi_->buffer_level > cpi_->oxcf.optimal_buffer_level) { + int Fraction = + (int)(((cpi_->buffer_level - cpi_->oxcf.optimal_buffer_level) * 128) / + (cpi_->oxcf.maximum_buffer_size - + cpi_->oxcf.optimal_buffer_level)); + int min_qadjustment = + ((cpi_->active_best_quality - cpi_->best_quality) * Fraction) / 128; + + cpi_->active_best_quality -= min_qadjustment; + } + } + + /* Clip the active best and worst quality values to limits */ + if (cpi_->active_worst_quality > cpi_->worst_quality) { + cpi_->active_worst_quality = cpi_->worst_quality; + } + if (cpi_->active_best_quality < cpi_->best_quality) { + cpi_->active_best_quality = cpi_->best_quality; + } + if (cpi_->active_worst_quality < cpi_->active_best_quality) { + cpi_->active_worst_quality = cpi_->active_best_quality; + } + + q_ = vp8_regulate_q(cpi_, cpi_->this_frame_target); + vp8_set_quantizer(cpi_, q_); + vpx_clear_system_state(); +} + +int VP8RateControlRTC::GetQP() const { return q_; } + +void VP8RateControlRTC::PostEncodeUpdate(uint64_t encoded_frame_size) { + VP8_COMMON *const cm = &cpi_->common; + vpx_clear_system_state(); + cpi_->total_byte_count += encoded_frame_size; + cpi_->projected_frame_size = static_cast(encoded_frame_size << 3); + if (cpi_->oxcf.number_of_layers > 1) { + for (unsigned int i = cpi_->current_layer + 1; + i < cpi_->oxcf.number_of_layers; ++i) { + cpi_->layer_context[i].total_byte_count += encoded_frame_size; + } + } + + vp8_update_rate_correction_factors(cpi_, 2); + + cpi_->last_q[cm->frame_type] = cm->base_qindex; + + if (cm->frame_type == KEY_FRAME) { + vp8_adjust_key_frame_context(cpi_); + } + + /* Keep a record of ambient average Q. */ + if (cm->frame_type != KEY_FRAME) { + cpi_->avg_frame_qindex = + (2 + 3 * cpi_->avg_frame_qindex + cm->base_qindex) >> 2; + } + /* Keep a record from which we can calculate the average Q excluding + * key frames. + */ + if (cm->frame_type != KEY_FRAME) { + cpi_->ni_frames++; + /* Damp value for first few frames */ + if (cpi_->ni_frames > 150) { + cpi_->ni_tot_qi += q_; + cpi_->ni_av_qi = (cpi_->ni_tot_qi / cpi_->ni_frames); + } else { + cpi_->ni_tot_qi += q_; + cpi_->ni_av_qi = + ((cpi_->ni_tot_qi / cpi_->ni_frames) + cpi_->worst_quality + 1) / 2; + } + + /* If the average Q is higher than what was used in the last + * frame (after going through the recode loop to keep the frame + * size within range) then use the last frame value - 1. The -1 + * is designed to stop Q and hence the data rate, from + * progressively falling away during difficult sections, but at + * the same time reduce the number of itterations around the + * recode loop. + */ + if (q_ > cpi_->ni_av_qi) cpi_->ni_av_qi = q_ - 1; + } + + cpi_->bits_off_target += + cpi_->av_per_frame_bandwidth - cpi_->projected_frame_size; + if (cpi_->bits_off_target > cpi_->oxcf.maximum_buffer_size) { + cpi_->bits_off_target = cpi_->oxcf.maximum_buffer_size; + } + + cpi_->total_actual_bits += cpi_->projected_frame_size; + cpi_->buffer_level = cpi_->bits_off_target; + + /* Propagate values to higher temporal layers */ + if (cpi_->oxcf.number_of_layers > 1) { + for (unsigned int i = cpi_->current_layer + 1; + i < cpi_->oxcf.number_of_layers; ++i) { + LAYER_CONTEXT *lc = &cpi_->layer_context[i]; + int bits_off_for_this_layer = (int)round( + lc->target_bandwidth / lc->framerate - cpi_->projected_frame_size); + + lc->bits_off_target += bits_off_for_this_layer; + + /* Clip buffer level to maximum buffer size for the layer */ + if (lc->bits_off_target > lc->maximum_buffer_size) { + lc->bits_off_target = lc->maximum_buffer_size; + } + + lc->total_actual_bits += cpi_->projected_frame_size; + lc->total_target_vs_actual += bits_off_for_this_layer; + lc->buffer_level = lc->bits_off_target; + } + } + + cpi_->common.current_video_frame++; + cpi_->frames_since_key++; + + if (cpi_->oxcf.number_of_layers > 1) vp8_save_layer_context(cpi_); + vpx_clear_system_state(); +} +} // namespace libvpx diff --git a/media/libvpx/libvpx/vp9/common/vp9_alloccommon.c b/media/libvpx/libvpx/vp9/common/vp9_alloccommon.c --- a/media/libvpx/libvpx/vp9/common/vp9_alloccommon.c +++ b/media/libvpx/libvpx/vp9/common/vp9_alloccommon.c @@ -71,10 +71,12 @@ } void vp9_free_ref_frame_buffers(BufferPool *pool) { int i; + if (!pool) return; + for (i = 0; i < FRAME_BUFFERS; ++i) { if (!pool->frame_bufs[i].released && pool->frame_bufs[i].raw_frame_buffer.data != NULL) { pool->release_fb_cb(pool->cb_priv, &pool->frame_bufs[i].raw_frame_buffer); pool->frame_bufs[i].ref_count = 0; @@ -98,11 +100,11 @@ (void)cm; #endif } void vp9_free_context_buffers(VP9_COMMON *cm) { - cm->free_mi(cm); + if (cm->free_mi) cm->free_mi(cm); free_seg_map(cm); vpx_free(cm->above_context); cm->above_context = NULL; vpx_free(cm->above_seg_context); cm->above_seg_context = NULL; diff --git a/media/libvpx/libvpx/vp9/common/vp9_common.h b/media/libvpx/libvpx/vp9/common/vp9_common.h --- a/media/libvpx/libvpx/vp9/common/vp9_common.h +++ b/media/libvpx/libvpx/vp9/common/vp9_common.h @@ -25,14 +25,14 @@ extern "C" { #endif // Only need this for fixed-size arrays, for structs just assign. #define vp9_copy(dest, src) \ - { \ + do { \ assert(sizeof(dest) == sizeof(src)); \ memcpy(dest, src, sizeof(src)); \ - } + } while (0) // Use this for variably-sized arrays. #define vp9_copy_array(dest, src, n) \ { \ assert(sizeof(*(dest)) == sizeof(*(src))); \ diff --git a/media/libvpx/libvpx/vp9/common/vp9_frame_buffers.c b/media/libvpx/libvpx/vp9/common/vp9_frame_buffers.c --- a/media/libvpx/libvpx/vp9/common/vp9_frame_buffers.c +++ b/media/libvpx/libvpx/vp9/common/vp9_frame_buffers.c @@ -12,18 +12,21 @@ #include "vp9/common/vp9_frame_buffers.h" #include "vpx_mem/vpx_mem.h" int vp9_alloc_internal_frame_buffers(InternalFrameBufferList *list) { + const int num_buffers = VP9_MAXIMUM_REF_BUFFERS + VPX_MAXIMUM_WORK_BUFFERS; assert(list != NULL); vp9_free_internal_frame_buffers(list); - list->num_internal_frame_buffers = - VP9_MAXIMUM_REF_BUFFERS + VPX_MAXIMUM_WORK_BUFFERS; - list->int_fb = (InternalFrameBuffer *)vpx_calloc( - list->num_internal_frame_buffers, sizeof(*list->int_fb)); - return (list->int_fb == NULL); + list->int_fb = + (InternalFrameBuffer *)vpx_calloc(num_buffers, sizeof(*list->int_fb)); + if (list->int_fb) { + list->num_internal_frame_buffers = num_buffers; + return 0; + } + return -1; } void vp9_free_internal_frame_buffers(InternalFrameBufferList *list) { int i; @@ -33,10 +36,11 @@ vpx_free(list->int_fb[i].data); list->int_fb[i].data = NULL; } vpx_free(list->int_fb); list->int_fb = NULL; + list->num_internal_frame_buffers = 0; } int vp9_get_frame_buffer(void *cb_priv, size_t min_size, vpx_codec_frame_buffer_t *fb) { int i; diff --git a/media/libvpx/libvpx/vp9/common/vp9_postproc.c b/media/libvpx/libvpx/vp9/common/vp9_postproc.c --- a/media/libvpx/libvpx/vp9/common/vp9_postproc.c +++ b/media/libvpx/libvpx/vp9/common/vp9_postproc.c @@ -358,10 +358,11 @@ if (flags & (VP9D_DEMACROBLOCK | VP9D_DEBLOCK)) { if (!cm->postproc_state.limits) { cm->postproc_state.limits = vpx_calloc(unscaled_width, sizeof(*cm->postproc_state.limits)); + if (!cm->postproc_state.limits) return 1; } } if (flags & VP9D_ADDNOISE) { if (!cm->postproc_state.generated_noise) { diff --git a/media/libvpx/libvpx/vp9/common/vp9_rtcd.c b/media/libvpx/libvpx/vp9/common/vp9_rtcd.c --- a/media/libvpx/libvpx/vp9/common/vp9_rtcd.c +++ b/media/libvpx/libvpx/vp9/common/vp9_rtcd.c @@ -10,10 +10,6 @@ #include "./vpx_config.h" #define RTCD_C #include "./vp9_rtcd.h" #include "vpx_ports/vpx_once.h" -void vp9_rtcd() { - // TODO(JBB): Remove this once, by insuring that both the encoder and - // decoder setup functions are protected by once(); - once(setup_rtcd_internal); -} +void vp9_rtcd() { once(setup_rtcd_internal); } diff --git a/media/libvpx/libvpx/vp9/common/vp9_rtcd_defs.pl b/media/libvpx/libvpx/vp9/common/vp9_rtcd_defs.pl --- a/media/libvpx/libvpx/vp9/common/vp9_rtcd_defs.pl +++ b/media/libvpx/libvpx/vp9/common/vp9_rtcd_defs.pl @@ -126,14 +126,14 @@ add_proto qw/int64_t vp9_block_error/, "const tran_low_t *coeff, const tran_low_t *dqcoeff, intptr_t block_size, int64_t *ssz"; add_proto qw/int64_t vp9_block_error_fp/, "const tran_low_t *coeff, const tran_low_t *dqcoeff, int block_size"; -add_proto qw/void vp9_quantize_fp/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; +add_proto qw/void vp9_quantize_fp/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; specialize qw/vp9_quantize_fp neon sse2 avx2 vsx/, "$ssse3_x86_64"; -add_proto qw/void vp9_quantize_fp_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; +add_proto qw/void vp9_quantize_fp_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; specialize qw/vp9_quantize_fp_32x32 neon vsx/, "$ssse3_x86_64"; if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { specialize qw/vp9_block_error avx2 sse2/; @@ -157,13 +157,13 @@ add_proto qw/void vp9_fwht4x4/, "const int16_t *input, tran_low_t *output, int stride"; # Note that there are more specializations appended when CONFIG_VP9_HIGHBITDEPTH # is off. -specialize qw/vp9_fht4x4 sse2/; -specialize qw/vp9_fht8x8 sse2/; -specialize qw/vp9_fht16x16 sse2/; +specialize qw/vp9_fht4x4 sse2 neon/; +specialize qw/vp9_fht8x8 sse2 neon/; +specialize qw/vp9_fht16x16 sse2 neon/; specialize qw/vp9_fwht4x4 sse2/; if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") ne "yes") { # Note that these specializations are appended to the above ones. specialize qw/vp9_fht4x4 msa/; specialize qw/vp9_fht8x8 msa/; @@ -173,11 +173,11 @@ # # Motion search # add_proto qw/int vp9_diamond_search_sad/, "const struct macroblock *x, const struct search_site_config *cfg, struct mv *ref_mv, struct mv *best_mv, int search_param, int sad_per_bit, int *num00, const struct vp9_variance_vtable *fn_ptr, const struct mv *center_mv"; -specialize qw/vp9_diamond_search_sad avx/; +specialize qw/vp9_diamond_search_sad avx neon/; # # Apply temporal filter # if (vpx_config("CONFIG_REALTIME_ONLY") ne "yes") { @@ -193,13 +193,13 @@ if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { # ENCODEMB INVOKE - add_proto qw/void vp9_highbd_quantize_fp/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; + add_proto qw/void vp9_highbd_quantize_fp/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; - add_proto qw/void vp9_highbd_quantize_fp_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan" ; + add_proto qw/void vp9_highbd_quantize_fp_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan" ; # fdct functions add_proto qw/void vp9_highbd_fht4x4/, "const int16_t *input, tran_low_t *output, int stride, int tx_type"; add_proto qw/void vp9_highbd_fht8x8/, "const int16_t *input, tran_low_t *output, int stride, int tx_type"; diff --git a/media/libvpx/libvpx/vp9/common/vp9_seg_common.h b/media/libvpx/libvpx/vp9/common/vp9_seg_common.h --- a/media/libvpx/libvpx/vp9/common/vp9_seg_common.h +++ b/media/libvpx/libvpx/vp9/common/vp9_seg_common.h @@ -23,10 +23,15 @@ #define MAX_SEGMENTS 8 #define SEG_TREE_PROBS (MAX_SEGMENTS - 1) #define PREDICTION_PROBS 3 +// Segment ID used to skip background encoding +#define BACKGROUND_SEG_SKIP_ID 3 +// Number of frames that don't skip after a key frame +#define FRAMES_NO_SKIPPING_AFTER_KEY 20 + // Segment level features. typedef enum { SEG_LVL_ALT_Q = 0, // Use alternate Quantizer .... SEG_LVL_ALT_LF = 1, // Use alternate loop filter value... SEG_LVL_REF_FRAME = 2, // Optional Segment reference frame diff --git a/media/libvpx/libvpx/vp9/encoder/arm/neon/vp9_dct_neon.c b/media/libvpx/libvpx/vp9/encoder/arm/neon/vp9_dct_neon.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vp9/encoder/arm/neon/vp9_dct_neon.c @@ -0,0 +1,1460 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "./vpx_config.h" +#include "./vp9_rtcd.h" +#include "./vpx_dsp_rtcd.h" + +#include "vpx_dsp/txfm_common.h" +#include "vpx_dsp/arm/mem_neon.h" +#include "vpx_dsp/arm/transpose_neon.h" +#include "vpx_dsp/arm/fdct_neon.h" + +static INLINE void load_buffer_4x4(const int16_t *input, int16x8_t *in, + int stride) { + // { 0, 1, 1, 1, 1, 1, 1, 1 }; + const int16x8_t nonzero_bias_a = vextq_s16(vdupq_n_s16(0), vdupq_n_s16(1), 7); + // { 1, 0, 0, 0, 0, 0, 0, 0 }; + const int16x8_t nonzero_bias_b = vextq_s16(vdupq_n_s16(1), vdupq_n_s16(0), 7); + int16x8_t mask; + + int16x4_t input_0 = vshl_n_s16(vld1_s16(input + 0 * stride), 4); + int16x4_t input_1 = vshl_n_s16(vld1_s16(input + 1 * stride), 4); + int16x4_t input_2 = vshl_n_s16(vld1_s16(input + 2 * stride), 4); + int16x4_t input_3 = vshl_n_s16(vld1_s16(input + 3 * stride), 4); + + in[0] = vcombine_s16(input_0, input_1); + in[1] = vcombine_s16(input_2, input_3); + + // Copy the SSE method, use a mask to avoid an 'if' branch here to increase by + // one non-zero first elements + mask = vreinterpretq_s16_u16(vceqq_s16(in[0], nonzero_bias_a)); + in[0] = vaddq_s16(in[0], mask); + in[0] = vaddq_s16(in[0], nonzero_bias_b); +} + +static INLINE void write_buffer_4x4(tran_low_t *output, int16x8_t *res) { + const int16x8_t one_s16 = vdupq_n_s16(1); + res[0] = vaddq_s16(res[0], one_s16); + res[1] = vaddq_s16(res[1], one_s16); + res[0] = vshrq_n_s16(res[0], 2); + res[1] = vshrq_n_s16(res[1], 2); + store_s16q_to_tran_low(output + 0 * 8, res[0]); + store_s16q_to_tran_low(output + 1 * 8, res[1]); +} + +static INLINE void fadst4x4_neon(int16x8_t *in) { + int32x4_t u0, u1, u2, u3; + int16x4_t out_0, out_1, out_2, out_3; + const int32x4_t k__DCT_CONST_ROUNDING = vdupq_n_s32(DCT_CONST_ROUNDING); + + const int16x4_t s0 = vget_low_s16(in[0]); // | x_00 | x_01 | x_02 | x_03 | + const int16x4_t s1 = vget_high_s16(in[0]); // | x_10 | x_11 | x_12 | x_13 | + const int16x4_t s2 = vget_low_s16(in[1]); // | x_20 | x_21 | x_22 | x_23 | + const int16x4_t s3 = vget_high_s16(in[1]); // | x_30 | x_31 | x_32 | x_33 | + + // s0 * sinpi_1_9, s0 * sinpi_4_9 + // Must expand all elements to s32. See 'needs32' comment in fwd_txfm.c. + const int32x4_t s0s1_9 = vmull_n_s16(s0, sinpi_1_9); + const int32x4_t s0s4_9 = vmull_n_s16(s0, sinpi_4_9); + // s1 * sinpi_1_9, s1 * sinpi_2_9 + const int32x4_t s1s1_9 = vmull_n_s16(s1, sinpi_1_9); + const int32x4_t s1s2_9 = vmull_n_s16(s1, sinpi_2_9); + // s2 * sinpi_3_9 + const int32x4_t s2s3_9 = vmull_n_s16(s2, sinpi_3_9); + // s3 * sinpi_2_9, s3 * sinpi_4_9 + const int32x4_t s3s2_9 = vmull_n_s16(s3, sinpi_2_9); + const int32x4_t s3s4_9 = vmull_n_s16(s3, sinpi_4_9); + + // (s0 + s1) * sinpi_3_9 + const int32x4_t s0_p_s1 = vaddl_s16(s0, s1); + const int32x4_t s0_p_s1_m_s3 = vsubw_s16(s0_p_s1, s3); + + // s_0 * sinpi_1_9 + s_1 * sinpi_2_9 + // s_0 * sinpi_4_9 - s_1 * sinpi_1_9 + const int32x4_t s0s1_9_p_s1s2_9 = vaddq_s32(s0s1_9, s1s2_9); + const int32x4_t s0s4_9_m_s1s1_9 = vsubq_s32(s0s4_9, s1s1_9); + /* + * t0 = s0s1_9 + s1s2_9 + s3s4_9 + * t1 = (s0 + s1) * sinpi_3_9 - s3 * sinpi_3_9 + * t2 = s0s4_9 - s1s1_9 + s3s2_9 + * t3 = s2s3_9 + */ + const int32x4_t t0 = vaddq_s32(s0s1_9_p_s1s2_9, s3s4_9); + const int32x4_t t1 = vmulq_n_s32(s0_p_s1_m_s3, sinpi_3_9); + const int32x4_t t2 = vaddq_s32(s0s4_9_m_s1s1_9, s3s2_9); + const int32x4_t t3 = s2s3_9; + /* + * u0 = t0 + t3 + * u1 = t1 + * u2 = t2 - t3 + * u3 = t2 - t0 + t3 + */ + u0 = vaddq_s32(t0, t3); + u1 = t1; + u2 = vsubq_s32(t2, t3); + u3 = vaddq_s32(vsubq_s32(t2, t0), t3); + + // fdct_round_shift + u0 = vaddq_s32(u0, k__DCT_CONST_ROUNDING); + u1 = vaddq_s32(u1, k__DCT_CONST_ROUNDING); + u2 = vaddq_s32(u2, k__DCT_CONST_ROUNDING); + u3 = vaddq_s32(u3, k__DCT_CONST_ROUNDING); + + out_0 = vshrn_n_s32(u0, DCT_CONST_BITS); + out_1 = vshrn_n_s32(u1, DCT_CONST_BITS); + out_2 = vshrn_n_s32(u2, DCT_CONST_BITS); + out_3 = vshrn_n_s32(u3, DCT_CONST_BITS); + + transpose_s16_4x4d(&out_0, &out_1, &out_2, &out_3); + + in[0] = vcombine_s16(out_0, out_1); + in[1] = vcombine_s16(out_2, out_3); +} + +void vp9_fht4x4_neon(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + int16x8_t in[2]; + + switch (tx_type) { + case DCT_DCT: vpx_fdct4x4_neon(input, output, stride); break; + case ADST_DCT: + load_buffer_4x4(input, in, stride); + fadst4x4_neon(in); + vpx_fdct4x4_pass1_neon((int16x4_t *)in); + write_buffer_4x4(output, in); + break; + case DCT_ADST: + load_buffer_4x4(input, in, stride); + vpx_fdct4x4_pass1_neon((int16x4_t *)in); + fadst4x4_neon(in); + write_buffer_4x4(output, in); + break; + default: + assert(tx_type == ADST_ADST); + load_buffer_4x4(input, in, stride); + fadst4x4_neon(in); + fadst4x4_neon(in); + write_buffer_4x4(output, in); + break; + } +} + +static INLINE void load_buffer_8x8(const int16_t *input, int16x8_t *in, + int stride) { + in[0] = vshlq_n_s16(vld1q_s16(input + 0 * stride), 2); + in[1] = vshlq_n_s16(vld1q_s16(input + 1 * stride), 2); + in[2] = vshlq_n_s16(vld1q_s16(input + 2 * stride), 2); + in[3] = vshlq_n_s16(vld1q_s16(input + 3 * stride), 2); + in[4] = vshlq_n_s16(vld1q_s16(input + 4 * stride), 2); + in[5] = vshlq_n_s16(vld1q_s16(input + 5 * stride), 2); + in[6] = vshlq_n_s16(vld1q_s16(input + 6 * stride), 2); + in[7] = vshlq_n_s16(vld1q_s16(input + 7 * stride), 2); +} + +/* right shift and rounding + * first get the sign bit (bit 15). + * If bit == 1, it's the simple case of shifting right by one bit. + * If bit == 2, it essentially computes the expression: + * + * out[j * 16 + i] = (temp_out[j] + 1 + (temp_out[j] < 0)) >> 2; + * + * for each row. + */ +static INLINE void right_shift_8x8(int16x8_t *res, const int bit) { + int16x8_t sign0 = vshrq_n_s16(res[0], 15); + int16x8_t sign1 = vshrq_n_s16(res[1], 15); + int16x8_t sign2 = vshrq_n_s16(res[2], 15); + int16x8_t sign3 = vshrq_n_s16(res[3], 15); + int16x8_t sign4 = vshrq_n_s16(res[4], 15); + int16x8_t sign5 = vshrq_n_s16(res[5], 15); + int16x8_t sign6 = vshrq_n_s16(res[6], 15); + int16x8_t sign7 = vshrq_n_s16(res[7], 15); + + if (bit == 2) { + const int16x8_t const_rounding = vdupq_n_s16(1); + res[0] = vaddq_s16(res[0], const_rounding); + res[1] = vaddq_s16(res[1], const_rounding); + res[2] = vaddq_s16(res[2], const_rounding); + res[3] = vaddq_s16(res[3], const_rounding); + res[4] = vaddq_s16(res[4], const_rounding); + res[5] = vaddq_s16(res[5], const_rounding); + res[6] = vaddq_s16(res[6], const_rounding); + res[7] = vaddq_s16(res[7], const_rounding); + } + + res[0] = vsubq_s16(res[0], sign0); + res[1] = vsubq_s16(res[1], sign1); + res[2] = vsubq_s16(res[2], sign2); + res[3] = vsubq_s16(res[3], sign3); + res[4] = vsubq_s16(res[4], sign4); + res[5] = vsubq_s16(res[5], sign5); + res[6] = vsubq_s16(res[6], sign6); + res[7] = vsubq_s16(res[7], sign7); + + if (bit == 1) { + res[0] = vshrq_n_s16(res[0], 1); + res[1] = vshrq_n_s16(res[1], 1); + res[2] = vshrq_n_s16(res[2], 1); + res[3] = vshrq_n_s16(res[3], 1); + res[4] = vshrq_n_s16(res[4], 1); + res[5] = vshrq_n_s16(res[5], 1); + res[6] = vshrq_n_s16(res[6], 1); + res[7] = vshrq_n_s16(res[7], 1); + } else { + res[0] = vshrq_n_s16(res[0], 2); + res[1] = vshrq_n_s16(res[1], 2); + res[2] = vshrq_n_s16(res[2], 2); + res[3] = vshrq_n_s16(res[3], 2); + res[4] = vshrq_n_s16(res[4], 2); + res[5] = vshrq_n_s16(res[5], 2); + res[6] = vshrq_n_s16(res[6], 2); + res[7] = vshrq_n_s16(res[7], 2); + } +} + +static INLINE void write_buffer_8x8(tran_low_t *output, int16x8_t *res, + int stride) { + store_s16q_to_tran_low(output + 0 * stride, res[0]); + store_s16q_to_tran_low(output + 1 * stride, res[1]); + store_s16q_to_tran_low(output + 2 * stride, res[2]); + store_s16q_to_tran_low(output + 3 * stride, res[3]); + store_s16q_to_tran_low(output + 4 * stride, res[4]); + store_s16q_to_tran_low(output + 5 * stride, res[5]); + store_s16q_to_tran_low(output + 6 * stride, res[6]); + store_s16q_to_tran_low(output + 7 * stride, res[7]); +} + +static INLINE void fadst8x8_neon(int16x8_t *in) { + int16x4_t x0_lo, x0_hi, x1_lo, x1_hi, x2_lo, x2_hi, x3_lo, x3_hi, x4_lo, + x4_hi, x5_lo, x5_hi, x6_lo, x6_hi, x7_lo, x7_hi; + int32x4_t s0_lo, s0_hi, s1_lo, s1_hi, s2_lo, s2_hi, s3_lo, s3_hi, s4_lo, + s4_hi, s5_lo, s5_hi, s6_lo, s6_hi, s7_lo, s7_hi; + int32x4_t t0_lo, t0_hi, t1_lo, t1_hi, t2_lo, t2_hi, t3_lo, t3_hi, t4_lo, + t4_hi, t5_lo, t5_hi, t6_lo, t6_hi, t7_lo, t7_hi; + const int32x4_t k__DCT_CONST_ROUNDING = vdupq_n_s32(DCT_CONST_ROUNDING); + + x0_lo = vget_low_s16(in[7]); + x0_hi = vget_high_s16(in[7]); + x1_lo = vget_low_s16(in[0]); + x1_hi = vget_high_s16(in[0]); + x2_lo = vget_low_s16(in[5]); + x2_hi = vget_high_s16(in[5]); + x3_lo = vget_low_s16(in[2]); + x3_hi = vget_high_s16(in[2]); + x4_lo = vget_low_s16(in[3]); + x4_hi = vget_high_s16(in[3]); + x5_lo = vget_low_s16(in[4]); + x5_hi = vget_high_s16(in[4]); + x6_lo = vget_low_s16(in[1]); + x6_hi = vget_high_s16(in[1]); + x7_lo = vget_low_s16(in[6]); + x7_hi = vget_high_s16(in[6]); + + // stage 1 + // s0 = cospi_2_64 * x0 + cospi_30_64 * x1; + s0_lo = vaddq_s32(vmull_n_s16(x0_lo, cospi_2_64), + vmull_n_s16(x1_lo, cospi_30_64)); + s0_hi = vaddq_s32(vmull_n_s16(x0_hi, cospi_2_64), + vmull_n_s16(x1_hi, cospi_30_64)); + // s1 = cospi_30_64 * x0 - cospi_2_64 * x1; + s1_lo = vsubq_s32(vmull_n_s16(x0_lo, cospi_30_64), + vmull_n_s16(x1_lo, cospi_2_64)); + s1_hi = vsubq_s32(vmull_n_s16(x0_hi, cospi_30_64), + vmull_n_s16(x1_hi, cospi_2_64)); + // s2 = cospi_10_64 * x2 + cospi_22_64 * x3; + s2_lo = vaddq_s32(vmull_n_s16(x2_lo, cospi_10_64), + vmull_n_s16(x3_lo, cospi_22_64)); + s2_hi = vaddq_s32(vmull_n_s16(x2_hi, cospi_10_64), + vmull_n_s16(x3_hi, cospi_22_64)); + // s3 = cospi_22_64 * x2 - cospi_10_64 * x3; + s3_lo = vsubq_s32(vmull_n_s16(x2_lo, cospi_22_64), + vmull_n_s16(x3_lo, cospi_10_64)); + s3_hi = vsubq_s32(vmull_n_s16(x2_hi, cospi_22_64), + vmull_n_s16(x3_hi, cospi_10_64)); + // s4 = cospi_18_64 * x4 + cospi_14_64 * x5; + s4_lo = vaddq_s32(vmull_n_s16(x4_lo, cospi_18_64), + vmull_n_s16(x5_lo, cospi_14_64)); + s4_hi = vaddq_s32(vmull_n_s16(x4_hi, cospi_18_64), + vmull_n_s16(x5_hi, cospi_14_64)); + // s5 = cospi_14_64 * x4 - cospi_18_64 * x5; + s5_lo = vsubq_s32(vmull_n_s16(x4_lo, cospi_14_64), + vmull_n_s16(x5_lo, cospi_18_64)); + s5_hi = vsubq_s32(vmull_n_s16(x4_hi, cospi_14_64), + vmull_n_s16(x5_hi, cospi_18_64)); + // s6 = cospi_26_64 * x6 + cospi_6_64 * x7; + s6_lo = vaddq_s32(vmull_n_s16(x6_lo, cospi_26_64), + vmull_n_s16(x7_lo, cospi_6_64)); + s6_hi = vaddq_s32(vmull_n_s16(x6_hi, cospi_26_64), + vmull_n_s16(x7_hi, cospi_6_64)); + // s7 = cospi_6_64 * x6 - cospi_26_64 * x7; + s7_lo = vsubq_s32(vmull_n_s16(x6_lo, cospi_6_64), + vmull_n_s16(x7_lo, cospi_26_64)); + s7_hi = vsubq_s32(vmull_n_s16(x6_hi, cospi_6_64), + vmull_n_s16(x7_hi, cospi_26_64)); + + // fdct_round_shift + t0_lo = vaddq_s32(s0_lo, s4_lo); + t0_hi = vaddq_s32(s0_hi, s4_hi); + t1_lo = vaddq_s32(s1_lo, s5_lo); + t1_hi = vaddq_s32(s1_hi, s5_hi); + t2_lo = vaddq_s32(s2_lo, s6_lo); + t2_hi = vaddq_s32(s2_hi, s6_hi); + t3_lo = vaddq_s32(s3_lo, s7_lo); + t3_hi = vaddq_s32(s3_hi, s7_hi); + t4_lo = vsubq_s32(s0_lo, s4_lo); + t4_hi = vsubq_s32(s0_hi, s4_hi); + t5_lo = vsubq_s32(s1_lo, s5_lo); + t5_hi = vsubq_s32(s1_hi, s5_hi); + t6_lo = vsubq_s32(s2_lo, s6_lo); + t6_hi = vsubq_s32(s2_hi, s6_hi); + t7_lo = vsubq_s32(s3_lo, s7_lo); + t7_hi = vsubq_s32(s3_hi, s7_hi); + + t0_lo = vaddq_s32(t0_lo, k__DCT_CONST_ROUNDING); + t0_hi = vaddq_s32(t0_hi, k__DCT_CONST_ROUNDING); + t1_lo = vaddq_s32(t1_lo, k__DCT_CONST_ROUNDING); + t1_hi = vaddq_s32(t1_hi, k__DCT_CONST_ROUNDING); + t2_lo = vaddq_s32(t2_lo, k__DCT_CONST_ROUNDING); + t2_hi = vaddq_s32(t2_hi, k__DCT_CONST_ROUNDING); + t3_lo = vaddq_s32(t3_lo, k__DCT_CONST_ROUNDING); + t3_hi = vaddq_s32(t3_hi, k__DCT_CONST_ROUNDING); + t4_lo = vaddq_s32(t4_lo, k__DCT_CONST_ROUNDING); + t4_hi = vaddq_s32(t4_hi, k__DCT_CONST_ROUNDING); + t5_lo = vaddq_s32(t5_lo, k__DCT_CONST_ROUNDING); + t5_hi = vaddq_s32(t5_hi, k__DCT_CONST_ROUNDING); + t6_lo = vaddq_s32(t6_lo, k__DCT_CONST_ROUNDING); + t6_hi = vaddq_s32(t6_hi, k__DCT_CONST_ROUNDING); + t7_lo = vaddq_s32(t7_lo, k__DCT_CONST_ROUNDING); + t7_hi = vaddq_s32(t7_hi, k__DCT_CONST_ROUNDING); + + t0_lo = vshrq_n_s32(t0_lo, DCT_CONST_BITS); + t0_hi = vshrq_n_s32(t0_hi, DCT_CONST_BITS); + t1_lo = vshrq_n_s32(t1_lo, DCT_CONST_BITS); + t1_hi = vshrq_n_s32(t1_hi, DCT_CONST_BITS); + t2_lo = vshrq_n_s32(t2_lo, DCT_CONST_BITS); + t2_hi = vshrq_n_s32(t2_hi, DCT_CONST_BITS); + t3_lo = vshrq_n_s32(t3_lo, DCT_CONST_BITS); + t3_hi = vshrq_n_s32(t3_hi, DCT_CONST_BITS); + t4_lo = vshrq_n_s32(t4_lo, DCT_CONST_BITS); + t4_hi = vshrq_n_s32(t4_hi, DCT_CONST_BITS); + t5_lo = vshrq_n_s32(t5_lo, DCT_CONST_BITS); + t5_hi = vshrq_n_s32(t5_hi, DCT_CONST_BITS); + t6_lo = vshrq_n_s32(t6_lo, DCT_CONST_BITS); + t6_hi = vshrq_n_s32(t6_hi, DCT_CONST_BITS); + t7_lo = vshrq_n_s32(t7_lo, DCT_CONST_BITS); + t7_hi = vshrq_n_s32(t7_hi, DCT_CONST_BITS); + + // stage 2 + s0_lo = t0_lo; + s0_hi = t0_hi; + s1_lo = t1_lo; + s1_hi = t1_hi; + s2_lo = t2_lo; + s2_hi = t2_hi; + s3_lo = t3_lo; + s3_hi = t3_hi; + s4_lo = vaddq_s32(vmulq_n_s32(t4_lo, cospi_8_64), + vmulq_n_s32(t5_lo, cospi_24_64)); + s4_hi = vaddq_s32(vmulq_n_s32(t4_hi, cospi_8_64), + vmulq_n_s32(t5_hi, cospi_24_64)); + s5_lo = vsubq_s32(vmulq_n_s32(t4_lo, cospi_24_64), + vmulq_n_s32(t5_lo, cospi_8_64)); + s5_hi = vsubq_s32(vmulq_n_s32(t4_hi, cospi_24_64), + vmulq_n_s32(t5_hi, cospi_8_64)); + s6_lo = vaddq_s32(vmulq_n_s32(t6_lo, -cospi_24_64), + vmulq_n_s32(t7_lo, cospi_8_64)); + s6_hi = vaddq_s32(vmulq_n_s32(t6_hi, -cospi_24_64), + vmulq_n_s32(t7_hi, cospi_8_64)); + s7_lo = vaddq_s32(vmulq_n_s32(t6_lo, cospi_8_64), + vmulq_n_s32(t7_lo, cospi_24_64)); + s7_hi = vaddq_s32(vmulq_n_s32(t6_hi, cospi_8_64), + vmulq_n_s32(t7_hi, cospi_24_64)); + + // s0 + s2 + t0_lo = vaddq_s32(s0_lo, s2_lo); + t0_hi = vaddq_s32(s0_hi, s2_hi); + // s1 + s3 + t1_lo = vaddq_s32(s1_lo, s3_lo); + t1_hi = vaddq_s32(s1_hi, s3_hi); + // s0 - s2 + t2_lo = vsubq_s32(s0_lo, s2_lo); + t2_hi = vsubq_s32(s0_hi, s2_hi); + // s1 - s3 + t3_lo = vsubq_s32(s1_lo, s3_lo); + t3_hi = vsubq_s32(s1_hi, s3_hi); + // s4 + s6 + t4_lo = vaddq_s32(s4_lo, s6_lo); + t4_hi = vaddq_s32(s4_hi, s6_hi); + // s5 + s7 + t5_lo = vaddq_s32(s5_lo, s7_lo); + t5_hi = vaddq_s32(s5_hi, s7_hi); + // s4 - s6 + t6_lo = vsubq_s32(s4_lo, s6_lo); + t6_hi = vsubq_s32(s4_hi, s6_hi); + // s5 - s7 + t7_lo = vsubq_s32(s5_lo, s7_lo); + t7_hi = vsubq_s32(s5_hi, s7_hi); + + // fdct_round_shift + t4_lo = vaddq_s32(t4_lo, k__DCT_CONST_ROUNDING); + t4_hi = vaddq_s32(t4_hi, k__DCT_CONST_ROUNDING); + t5_lo = vaddq_s32(t5_lo, k__DCT_CONST_ROUNDING); + t5_hi = vaddq_s32(t5_hi, k__DCT_CONST_ROUNDING); + t6_lo = vaddq_s32(t6_lo, k__DCT_CONST_ROUNDING); + t6_hi = vaddq_s32(t6_hi, k__DCT_CONST_ROUNDING); + t7_lo = vaddq_s32(t7_lo, k__DCT_CONST_ROUNDING); + t7_hi = vaddq_s32(t7_hi, k__DCT_CONST_ROUNDING); + t4_lo = vshrq_n_s32(t4_lo, DCT_CONST_BITS); + t4_hi = vshrq_n_s32(t4_hi, DCT_CONST_BITS); + t5_lo = vshrq_n_s32(t5_lo, DCT_CONST_BITS); + t5_hi = vshrq_n_s32(t5_hi, DCT_CONST_BITS); + t6_lo = vshrq_n_s32(t6_lo, DCT_CONST_BITS); + t6_hi = vshrq_n_s32(t6_hi, DCT_CONST_BITS); + t7_lo = vshrq_n_s32(t7_lo, DCT_CONST_BITS); + t7_hi = vshrq_n_s32(t7_hi, DCT_CONST_BITS); + + // stage 3 + // cospi_16_64 * (x2 + x3) + s2_lo = vmulq_n_s32(vaddq_s32(t2_lo, t3_lo), cospi_16_64); + s2_hi = vmulq_n_s32(vaddq_s32(t2_hi, t3_hi), cospi_16_64); + // cospi_16_64 * (x2 - x3) + s3_lo = vmulq_n_s32(vsubq_s32(t2_lo, t3_lo), cospi_16_64); + s3_hi = vmulq_n_s32(vsubq_s32(t2_hi, t3_hi), cospi_16_64); + // cospi_16_64 * (x6 + x7) + s6_lo = vmulq_n_s32(vaddq_s32(t6_lo, t7_lo), cospi_16_64); + s6_hi = vmulq_n_s32(vaddq_s32(t6_hi, t7_hi), cospi_16_64); + // cospi_16_64 * (x2 - x3) + s7_lo = vmulq_n_s32(vsubq_s32(t6_lo, t7_lo), cospi_16_64); + s7_hi = vmulq_n_s32(vsubq_s32(t6_hi, t7_hi), cospi_16_64); + + // final fdct_round_shift + t2_lo = vaddq_s32(s2_lo, k__DCT_CONST_ROUNDING); + t2_hi = vaddq_s32(s2_hi, k__DCT_CONST_ROUNDING); + t3_lo = vaddq_s32(s3_lo, k__DCT_CONST_ROUNDING); + t3_hi = vaddq_s32(s3_hi, k__DCT_CONST_ROUNDING); + t6_lo = vaddq_s32(s6_lo, k__DCT_CONST_ROUNDING); + t6_hi = vaddq_s32(s6_hi, k__DCT_CONST_ROUNDING); + t7_lo = vaddq_s32(s7_lo, k__DCT_CONST_ROUNDING); + t7_hi = vaddq_s32(s7_hi, k__DCT_CONST_ROUNDING); + + x2_lo = vshrn_n_s32(t2_lo, DCT_CONST_BITS); + x2_hi = vshrn_n_s32(t2_hi, DCT_CONST_BITS); + x3_lo = vshrn_n_s32(t3_lo, DCT_CONST_BITS); + x3_hi = vshrn_n_s32(t3_hi, DCT_CONST_BITS); + x6_lo = vshrn_n_s32(t6_lo, DCT_CONST_BITS); + x6_hi = vshrn_n_s32(t6_hi, DCT_CONST_BITS); + x7_lo = vshrn_n_s32(t7_lo, DCT_CONST_BITS); + x7_hi = vshrn_n_s32(t7_hi, DCT_CONST_BITS); + + // x0, x1, x4, x5 narrow down to 16-bits directly + x0_lo = vmovn_s32(t0_lo); + x0_hi = vmovn_s32(t0_hi); + x1_lo = vmovn_s32(t1_lo); + x1_hi = vmovn_s32(t1_hi); + x4_lo = vmovn_s32(t4_lo); + x4_hi = vmovn_s32(t4_hi); + x5_lo = vmovn_s32(t5_lo); + x5_hi = vmovn_s32(t5_hi); + + in[0] = vcombine_s16(x0_lo, x0_hi); + in[1] = vnegq_s16(vcombine_s16(x4_lo, x4_hi)); + in[2] = vcombine_s16(x6_lo, x6_hi); + in[3] = vnegq_s16(vcombine_s16(x2_lo, x2_hi)); + in[4] = vcombine_s16(x3_lo, x3_hi); + in[5] = vnegq_s16(vcombine_s16(x7_lo, x7_hi)); + in[6] = vcombine_s16(x5_lo, x5_hi); + in[7] = vnegq_s16(vcombine_s16(x1_lo, x1_hi)); + + transpose_s16_8x8(&in[0], &in[1], &in[2], &in[3], &in[4], &in[5], &in[6], + &in[7]); +} + +void vp9_fht8x8_neon(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + int16x8_t in[8]; + + switch (tx_type) { + case DCT_DCT: vpx_fdct8x8_neon(input, output, stride); break; + case ADST_DCT: + load_buffer_8x8(input, in, stride); + fadst8x8_neon(in); + vpx_fdct8x8_pass1_neon(in); + right_shift_8x8(in, 1); + write_buffer_8x8(output, in, 8); + break; + case DCT_ADST: + load_buffer_8x8(input, in, stride); + vpx_fdct8x8_pass1_neon(in); + fadst8x8_neon(in); + right_shift_8x8(in, 1); + write_buffer_8x8(output, in, 8); + break; + default: + assert(tx_type == ADST_ADST); + load_buffer_8x8(input, in, stride); + fadst8x8_neon(in); + fadst8x8_neon(in); + right_shift_8x8(in, 1); + write_buffer_8x8(output, in, 8); + break; + } +} + +static INLINE void load_buffer_16x16(const int16_t *input, int16x8_t *in0, + int16x8_t *in1, int stride) { + // load first 8 columns + load_buffer_8x8(input, in0, stride); + load_buffer_8x8(input + 8 * stride, in0 + 8, stride); + + input += 8; + // load second 8 columns + load_buffer_8x8(input, in1, stride); + load_buffer_8x8(input + 8 * stride, in1 + 8, stride); +} + +static INLINE void write_buffer_16x16(tran_low_t *output, int16x8_t *in0, + int16x8_t *in1, int stride) { + // write first 8 columns + write_buffer_8x8(output, in0, stride); + write_buffer_8x8(output + 8 * stride, in0 + 8, stride); + + // write second 8 columns + output += 8; + write_buffer_8x8(output, in1, stride); + write_buffer_8x8(output + 8 * stride, in1 + 8, stride); +} + +static INLINE void right_shift_16x16(int16x8_t *res0, int16x8_t *res1) { + // perform rounding operations + right_shift_8x8(res0, 2); + right_shift_8x8(res0 + 8, 2); + right_shift_8x8(res1, 2); + right_shift_8x8(res1 + 8, 2); +} + +static void fdct16_8col(int16x8_t *in) { + // perform 16x16 1-D DCT for 8 columns + int16x8_t i[8], s1[8], s2[8], s3[8], t[8]; + int16x4_t t_lo[8], t_hi[8]; + int32x4_t u_lo[8], u_hi[8]; + const int32x4_t k__DCT_CONST_ROUNDING = vdupq_n_s32(DCT_CONST_ROUNDING); + + // stage 1 + i[0] = vaddq_s16(in[0], in[15]); + i[1] = vaddq_s16(in[1], in[14]); + i[2] = vaddq_s16(in[2], in[13]); + i[3] = vaddq_s16(in[3], in[12]); + i[4] = vaddq_s16(in[4], in[11]); + i[5] = vaddq_s16(in[5], in[10]); + i[6] = vaddq_s16(in[6], in[9]); + i[7] = vaddq_s16(in[7], in[8]); + + vpx_fdct8x8_pass1_neon(i); + transpose_s16_8x8(&i[0], &i[1], &i[2], &i[3], &i[4], &i[5], &i[6], &i[7]); + + // step 2 + s1[0] = vsubq_s16(in[7], in[8]); + s1[1] = vsubq_s16(in[6], in[9]); + s1[2] = vsubq_s16(in[5], in[10]); + s1[3] = vsubq_s16(in[4], in[11]); + s1[4] = vsubq_s16(in[3], in[12]); + s1[5] = vsubq_s16(in[2], in[13]); + s1[6] = vsubq_s16(in[1], in[14]); + s1[7] = vsubq_s16(in[0], in[15]); + + t[2] = vsubq_s16(s1[5], s1[2]); + t[3] = vsubq_s16(s1[4], s1[3]); + t[4] = vaddq_s16(s1[4], s1[3]); + t[5] = vaddq_s16(s1[5], s1[2]); + + t_lo[2] = vget_low_s16(t[2]); + t_hi[2] = vget_high_s16(t[2]); + t_lo[3] = vget_low_s16(t[3]); + t_hi[3] = vget_high_s16(t[3]); + t_lo[4] = vget_low_s16(t[4]); + t_hi[4] = vget_high_s16(t[4]); + t_lo[5] = vget_low_s16(t[5]); + t_hi[5] = vget_high_s16(t[5]); + + u_lo[2] = vmull_n_s16(t_lo[2], cospi_16_64); + u_hi[2] = vmull_n_s16(t_hi[2], cospi_16_64); + u_lo[3] = vmull_n_s16(t_lo[3], cospi_16_64); + u_hi[3] = vmull_n_s16(t_hi[3], cospi_16_64); + u_lo[4] = vmull_n_s16(t_lo[4], cospi_16_64); + u_hi[4] = vmull_n_s16(t_hi[4], cospi_16_64); + u_lo[5] = vmull_n_s16(t_lo[5], cospi_16_64); + u_hi[5] = vmull_n_s16(t_hi[5], cospi_16_64); + + u_lo[2] = vaddq_s32(u_lo[2], k__DCT_CONST_ROUNDING); + u_hi[2] = vaddq_s32(u_hi[2], k__DCT_CONST_ROUNDING); + u_lo[3] = vaddq_s32(u_lo[3], k__DCT_CONST_ROUNDING); + u_hi[3] = vaddq_s32(u_hi[3], k__DCT_CONST_ROUNDING); + u_lo[4] = vaddq_s32(u_lo[4], k__DCT_CONST_ROUNDING); + u_hi[4] = vaddq_s32(u_hi[4], k__DCT_CONST_ROUNDING); + u_lo[5] = vaddq_s32(u_lo[5], k__DCT_CONST_ROUNDING); + u_hi[5] = vaddq_s32(u_hi[5], k__DCT_CONST_ROUNDING); + + t_lo[2] = vshrn_n_s32(u_lo[2], DCT_CONST_BITS); + t_hi[2] = vshrn_n_s32(u_hi[2], DCT_CONST_BITS); + t_lo[3] = vshrn_n_s32(u_lo[3], DCT_CONST_BITS); + t_hi[3] = vshrn_n_s32(u_hi[3], DCT_CONST_BITS); + t_lo[4] = vshrn_n_s32(u_lo[4], DCT_CONST_BITS); + t_hi[4] = vshrn_n_s32(u_hi[4], DCT_CONST_BITS); + t_lo[5] = vshrn_n_s32(u_lo[5], DCT_CONST_BITS); + t_hi[5] = vshrn_n_s32(u_hi[5], DCT_CONST_BITS); + + s2[2] = vcombine_s16(t_lo[2], t_hi[2]); + s2[3] = vcombine_s16(t_lo[3], t_hi[3]); + s2[4] = vcombine_s16(t_lo[4], t_hi[4]); + s2[5] = vcombine_s16(t_lo[5], t_hi[5]); + + // step 3 + s3[0] = vaddq_s16(s1[0], s2[3]); + s3[1] = vaddq_s16(s1[1], s2[2]); + s3[2] = vsubq_s16(s1[1], s2[2]); + s3[3] = vsubq_s16(s1[0], s2[3]); + s3[4] = vsubq_s16(s1[7], s2[4]); + s3[5] = vsubq_s16(s1[6], s2[5]); + s3[6] = vaddq_s16(s1[6], s2[5]); + s3[7] = vaddq_s16(s1[7], s2[4]); + + // step 4 + t_lo[0] = vget_low_s16(s3[0]); + t_hi[0] = vget_high_s16(s3[0]); + t_lo[1] = vget_low_s16(s3[1]); + t_hi[1] = vget_high_s16(s3[1]); + t_lo[2] = vget_low_s16(s3[2]); + t_hi[2] = vget_high_s16(s3[2]); + t_lo[3] = vget_low_s16(s3[3]); + t_hi[3] = vget_high_s16(s3[3]); + t_lo[4] = vget_low_s16(s3[4]); + t_hi[4] = vget_high_s16(s3[4]); + t_lo[5] = vget_low_s16(s3[5]); + t_hi[5] = vget_high_s16(s3[5]); + t_lo[6] = vget_low_s16(s3[6]); + t_hi[6] = vget_high_s16(s3[6]); + t_lo[7] = vget_low_s16(s3[7]); + t_hi[7] = vget_high_s16(s3[7]); + + u_lo[1] = vaddq_s32(vmull_n_s16(t_lo[1], -cospi_8_64), + vmull_n_s16(t_lo[6], cospi_24_64)); + u_hi[1] = vaddq_s32(vmull_n_s16(t_hi[1], -cospi_8_64), + vmull_n_s16(t_hi[6], cospi_24_64)); + u_lo[2] = vaddq_s32(vmull_n_s16(t_lo[2], cospi_24_64), + vmull_n_s16(t_lo[5], cospi_8_64)); + u_hi[2] = vaddq_s32(vmull_n_s16(t_hi[2], cospi_24_64), + vmull_n_s16(t_hi[5], cospi_8_64)); + u_lo[5] = vaddq_s32(vmull_n_s16(t_lo[2], cospi_8_64), + vmull_n_s16(t_lo[5], -cospi_24_64)); + u_hi[5] = vaddq_s32(vmull_n_s16(t_hi[2], cospi_8_64), + vmull_n_s16(t_hi[5], -cospi_24_64)); + u_lo[6] = vaddq_s32(vmull_n_s16(t_lo[1], cospi_24_64), + vmull_n_s16(t_lo[6], cospi_8_64)); + u_hi[6] = vaddq_s32(vmull_n_s16(t_hi[1], cospi_24_64), + vmull_n_s16(t_hi[6], cospi_8_64)); + + u_lo[1] = vaddq_s32(u_lo[1], k__DCT_CONST_ROUNDING); + u_hi[1] = vaddq_s32(u_hi[1], k__DCT_CONST_ROUNDING); + u_lo[2] = vaddq_s32(u_lo[2], k__DCT_CONST_ROUNDING); + u_hi[2] = vaddq_s32(u_hi[2], k__DCT_CONST_ROUNDING); + u_lo[5] = vaddq_s32(u_lo[5], k__DCT_CONST_ROUNDING); + u_hi[5] = vaddq_s32(u_hi[5], k__DCT_CONST_ROUNDING); + u_lo[6] = vaddq_s32(u_lo[6], k__DCT_CONST_ROUNDING); + u_hi[6] = vaddq_s32(u_hi[6], k__DCT_CONST_ROUNDING); + + t_lo[1] = vshrn_n_s32(u_lo[1], DCT_CONST_BITS); + t_hi[1] = vshrn_n_s32(u_hi[1], DCT_CONST_BITS); + t_lo[2] = vshrn_n_s32(u_lo[2], DCT_CONST_BITS); + t_hi[2] = vshrn_n_s32(u_hi[2], DCT_CONST_BITS); + t_lo[5] = vshrn_n_s32(u_lo[5], DCT_CONST_BITS); + t_hi[5] = vshrn_n_s32(u_hi[5], DCT_CONST_BITS); + t_lo[6] = vshrn_n_s32(u_lo[6], DCT_CONST_BITS); + t_hi[6] = vshrn_n_s32(u_hi[6], DCT_CONST_BITS); + + s2[1] = vcombine_s16(t_lo[1], t_hi[1]); + s2[2] = vcombine_s16(t_lo[2], t_hi[2]); + s2[5] = vcombine_s16(t_lo[5], t_hi[5]); + s2[6] = vcombine_s16(t_lo[6], t_hi[6]); + + // step 5 + s1[0] = vaddq_s16(s3[0], s2[1]); + s1[1] = vsubq_s16(s3[0], s2[1]); + s1[2] = vaddq_s16(s3[3], s2[2]); + s1[3] = vsubq_s16(s3[3], s2[2]); + s1[4] = vsubq_s16(s3[4], s2[5]); + s1[5] = vaddq_s16(s3[4], s2[5]); + s1[6] = vsubq_s16(s3[7], s2[6]); + s1[7] = vaddq_s16(s3[7], s2[6]); + + // step 6 + t_lo[0] = vget_low_s16(s1[0]); + t_hi[0] = vget_high_s16(s1[0]); + t_lo[1] = vget_low_s16(s1[1]); + t_hi[1] = vget_high_s16(s1[1]); + t_lo[2] = vget_low_s16(s1[2]); + t_hi[2] = vget_high_s16(s1[2]); + t_lo[3] = vget_low_s16(s1[3]); + t_hi[3] = vget_high_s16(s1[3]); + t_lo[4] = vget_low_s16(s1[4]); + t_hi[4] = vget_high_s16(s1[4]); + t_lo[5] = vget_low_s16(s1[5]); + t_hi[5] = vget_high_s16(s1[5]); + t_lo[6] = vget_low_s16(s1[6]); + t_hi[6] = vget_high_s16(s1[6]); + t_lo[7] = vget_low_s16(s1[7]); + t_hi[7] = vget_high_s16(s1[7]); + + // step1[0] * cospi_30_64 + step1[7] * cospi_2_64; + u_lo[0] = vaddq_s32(vmull_n_s16(t_lo[0], cospi_30_64), + vmull_n_s16(t_lo[7], cospi_2_64)); + u_hi[0] = vaddq_s32(vmull_n_s16(t_hi[0], cospi_30_64), + vmull_n_s16(t_hi[7], cospi_2_64)); + + // step1[1] * cospi_14_64 + step1[6] * cospi_18_64; + u_lo[1] = vaddq_s32(vmull_n_s16(t_lo[1], cospi_14_64), + vmull_n_s16(t_lo[6], cospi_18_64)); + u_hi[1] = vaddq_s32(vmull_n_s16(t_hi[1], cospi_14_64), + vmull_n_s16(t_hi[6], cospi_18_64)); + + // step1[2] * cospi_22_64 + step1[5] * cospi_10_64; + u_lo[2] = vaddq_s32(vmull_n_s16(t_lo[2], cospi_22_64), + vmull_n_s16(t_lo[5], cospi_10_64)); + u_hi[2] = vaddq_s32(vmull_n_s16(t_hi[2], cospi_22_64), + vmull_n_s16(t_hi[5], cospi_10_64)); + + // step1[3] * cospi_6_64 + step1[4] * cospi_26_64; + u_lo[3] = vaddq_s32(vmull_n_s16(t_lo[3], cospi_6_64), + vmull_n_s16(t_lo[4], cospi_26_64)); + u_hi[3] = vaddq_s32(vmull_n_s16(t_hi[3], cospi_6_64), + vmull_n_s16(t_hi[4], cospi_26_64)); + + // step1[3] * -cospi_26_64 + step1[4] * cospi_6_64; + u_lo[4] = vaddq_s32(vmull_n_s16(t_lo[3], -cospi_26_64), + vmull_n_s16(t_lo[4], cospi_6_64)); + u_hi[4] = vaddq_s32(vmull_n_s16(t_hi[3], -cospi_26_64), + vmull_n_s16(t_hi[4], cospi_6_64)); + + // step1[2] * -cospi_10_64 + step1[5] * cospi_22_64; + u_lo[5] = vaddq_s32(vmull_n_s16(t_lo[2], -cospi_10_64), + vmull_n_s16(t_lo[5], cospi_22_64)); + u_hi[5] = vaddq_s32(vmull_n_s16(t_hi[2], -cospi_10_64), + vmull_n_s16(t_hi[5], cospi_22_64)); + + // step1[1] * -cospi_18_64 + step1[6] * cospi_14_64; + u_lo[6] = vaddq_s32(vmull_n_s16(t_lo[1], -cospi_18_64), + vmull_n_s16(t_lo[6], cospi_14_64)); + u_hi[6] = vaddq_s32(vmull_n_s16(t_hi[1], -cospi_18_64), + vmull_n_s16(t_hi[6], cospi_14_64)); + + // step1[0] * -cospi_2_64 + step1[7] * cospi_30_64; + u_lo[7] = vaddq_s32(vmull_n_s16(t_lo[0], -cospi_2_64), + vmull_n_s16(t_lo[7], cospi_30_64)); + u_hi[7] = vaddq_s32(vmull_n_s16(t_hi[0], -cospi_2_64), + vmull_n_s16(t_hi[7], cospi_30_64)); + + // final fdct_round_shift + u_lo[0] = vaddq_s32(u_lo[0], k__DCT_CONST_ROUNDING); + u_hi[0] = vaddq_s32(u_hi[0], k__DCT_CONST_ROUNDING); + u_lo[1] = vaddq_s32(u_lo[1], k__DCT_CONST_ROUNDING); + u_hi[1] = vaddq_s32(u_hi[1], k__DCT_CONST_ROUNDING); + u_lo[2] = vaddq_s32(u_lo[2], k__DCT_CONST_ROUNDING); + u_hi[2] = vaddq_s32(u_hi[2], k__DCT_CONST_ROUNDING); + u_lo[3] = vaddq_s32(u_lo[3], k__DCT_CONST_ROUNDING); + u_hi[3] = vaddq_s32(u_hi[3], k__DCT_CONST_ROUNDING); + u_lo[4] = vaddq_s32(u_lo[4], k__DCT_CONST_ROUNDING); + u_hi[4] = vaddq_s32(u_hi[4], k__DCT_CONST_ROUNDING); + u_lo[5] = vaddq_s32(u_lo[5], k__DCT_CONST_ROUNDING); + u_hi[5] = vaddq_s32(u_hi[5], k__DCT_CONST_ROUNDING); + u_lo[6] = vaddq_s32(u_lo[6], k__DCT_CONST_ROUNDING); + u_hi[6] = vaddq_s32(u_hi[6], k__DCT_CONST_ROUNDING); + u_lo[7] = vaddq_s32(u_lo[7], k__DCT_CONST_ROUNDING); + u_hi[7] = vaddq_s32(u_hi[7], k__DCT_CONST_ROUNDING); + + t_lo[0] = vshrn_n_s32(u_lo[0], DCT_CONST_BITS); + t_hi[0] = vshrn_n_s32(u_hi[0], DCT_CONST_BITS); + t_lo[1] = vshrn_n_s32(u_lo[1], DCT_CONST_BITS); + t_hi[1] = vshrn_n_s32(u_hi[1], DCT_CONST_BITS); + t_lo[2] = vshrn_n_s32(u_lo[2], DCT_CONST_BITS); + t_hi[2] = vshrn_n_s32(u_hi[2], DCT_CONST_BITS); + t_lo[3] = vshrn_n_s32(u_lo[3], DCT_CONST_BITS); + t_hi[3] = vshrn_n_s32(u_hi[3], DCT_CONST_BITS); + t_lo[4] = vshrn_n_s32(u_lo[4], DCT_CONST_BITS); + t_hi[4] = vshrn_n_s32(u_hi[4], DCT_CONST_BITS); + t_lo[5] = vshrn_n_s32(u_lo[5], DCT_CONST_BITS); + t_hi[5] = vshrn_n_s32(u_hi[5], DCT_CONST_BITS); + t_lo[6] = vshrn_n_s32(u_lo[6], DCT_CONST_BITS); + t_hi[6] = vshrn_n_s32(u_hi[6], DCT_CONST_BITS); + t_lo[7] = vshrn_n_s32(u_lo[7], DCT_CONST_BITS); + t_hi[7] = vshrn_n_s32(u_hi[7], DCT_CONST_BITS); + + in[0] = i[0]; + in[2] = i[1]; + in[4] = i[2]; + in[6] = i[3]; + in[8] = i[4]; + in[10] = i[5]; + in[12] = i[6]; + in[14] = i[7]; + in[1] = vcombine_s16(t_lo[0], t_hi[0]); + in[3] = vcombine_s16(t_lo[4], t_hi[4]); + in[5] = vcombine_s16(t_lo[2], t_hi[2]); + in[7] = vcombine_s16(t_lo[6], t_hi[6]); + in[9] = vcombine_s16(t_lo[1], t_hi[1]); + in[11] = vcombine_s16(t_lo[5], t_hi[5]); + in[13] = vcombine_s16(t_lo[3], t_hi[3]); + in[15] = vcombine_s16(t_lo[7], t_hi[7]); +} + +static void fadst16_8col(int16x8_t *in) { + // perform 16x16 1-D ADST for 8 columns + int16x4_t x_lo[16], x_hi[16]; + int32x4_t s_lo[16], s_hi[16]; + int32x4_t t_lo[16], t_hi[16]; + const int32x4_t k__DCT_CONST_ROUNDING = vdupq_n_s32(DCT_CONST_ROUNDING); + + x_lo[0] = vget_low_s16(in[15]); + x_hi[0] = vget_high_s16(in[15]); + x_lo[1] = vget_low_s16(in[0]); + x_hi[1] = vget_high_s16(in[0]); + x_lo[2] = vget_low_s16(in[13]); + x_hi[2] = vget_high_s16(in[13]); + x_lo[3] = vget_low_s16(in[2]); + x_hi[3] = vget_high_s16(in[2]); + x_lo[4] = vget_low_s16(in[11]); + x_hi[4] = vget_high_s16(in[11]); + x_lo[5] = vget_low_s16(in[4]); + x_hi[5] = vget_high_s16(in[4]); + x_lo[6] = vget_low_s16(in[9]); + x_hi[6] = vget_high_s16(in[9]); + x_lo[7] = vget_low_s16(in[6]); + x_hi[7] = vget_high_s16(in[6]); + x_lo[8] = vget_low_s16(in[7]); + x_hi[8] = vget_high_s16(in[7]); + x_lo[9] = vget_low_s16(in[8]); + x_hi[9] = vget_high_s16(in[8]); + x_lo[10] = vget_low_s16(in[5]); + x_hi[10] = vget_high_s16(in[5]); + x_lo[11] = vget_low_s16(in[10]); + x_hi[11] = vget_high_s16(in[10]); + x_lo[12] = vget_low_s16(in[3]); + x_hi[12] = vget_high_s16(in[3]); + x_lo[13] = vget_low_s16(in[12]); + x_hi[13] = vget_high_s16(in[12]); + x_lo[14] = vget_low_s16(in[1]); + x_hi[14] = vget_high_s16(in[1]); + x_lo[15] = vget_low_s16(in[14]); + x_hi[15] = vget_high_s16(in[14]); + + // stage 1 + // s0 = cospi_1_64 * x0 + cospi_31_64 * x1; + s_lo[0] = vaddq_s32(vmull_n_s16(x_lo[0], cospi_1_64), + vmull_n_s16(x_lo[1], cospi_31_64)); + s_hi[0] = vaddq_s32(vmull_n_s16(x_hi[0], cospi_1_64), + vmull_n_s16(x_hi[1], cospi_31_64)); + // s1 = cospi_31_64 * x0 - cospi_1_64 * x1; + s_lo[1] = vsubq_s32(vmull_n_s16(x_lo[0], cospi_31_64), + vmull_n_s16(x_lo[1], cospi_1_64)); + s_hi[1] = vsubq_s32(vmull_n_s16(x_hi[0], cospi_31_64), + vmull_n_s16(x_hi[1], cospi_1_64)); + // s2 = cospi_5_64 * x2 + cospi_27_64 * x3; + s_lo[2] = vaddq_s32(vmull_n_s16(x_lo[2], cospi_5_64), + vmull_n_s16(x_lo[3], cospi_27_64)); + s_hi[2] = vaddq_s32(vmull_n_s16(x_hi[2], cospi_5_64), + vmull_n_s16(x_hi[3], cospi_27_64)); + // s3 = cospi_27_64 * x2 - cospi_5_64 * x3; + s_lo[3] = vsubq_s32(vmull_n_s16(x_lo[2], cospi_27_64), + vmull_n_s16(x_lo[3], cospi_5_64)); + s_hi[3] = vsubq_s32(vmull_n_s16(x_hi[2], cospi_27_64), + vmull_n_s16(x_hi[3], cospi_5_64)); + // s4 = cospi_9_64 * x4 + cospi_23_64 * x5; + s_lo[4] = vaddq_s32(vmull_n_s16(x_lo[4], cospi_9_64), + vmull_n_s16(x_lo[5], cospi_23_64)); + s_hi[4] = vaddq_s32(vmull_n_s16(x_hi[4], cospi_9_64), + vmull_n_s16(x_hi[5], cospi_23_64)); + // s5 = cospi_23_64 * x4 - cospi_9_64 * x5; + s_lo[5] = vsubq_s32(vmull_n_s16(x_lo[4], cospi_23_64), + vmull_n_s16(x_lo[5], cospi_9_64)); + s_hi[5] = vsubq_s32(vmull_n_s16(x_hi[4], cospi_23_64), + vmull_n_s16(x_hi[5], cospi_9_64)); + // s6 = cospi_13_64 * x6 + cospi_19_64 * x7; + s_lo[6] = vaddq_s32(vmull_n_s16(x_lo[6], cospi_13_64), + vmull_n_s16(x_lo[7], cospi_19_64)); + s_hi[6] = vaddq_s32(vmull_n_s16(x_hi[6], cospi_13_64), + vmull_n_s16(x_hi[7], cospi_19_64)); + // s7 = cospi_19_64 * x6 - cospi_13_64 * x7; + s_lo[7] = vsubq_s32(vmull_n_s16(x_lo[6], cospi_19_64), + vmull_n_s16(x_lo[7], cospi_13_64)); + s_hi[7] = vsubq_s32(vmull_n_s16(x_hi[6], cospi_19_64), + vmull_n_s16(x_hi[7], cospi_13_64)); + // s8 = cospi_17_64 * x8 + cospi_15_64 * x9; + s_lo[8] = vaddq_s32(vmull_n_s16(x_lo[8], cospi_17_64), + vmull_n_s16(x_lo[9], cospi_15_64)); + s_hi[8] = vaddq_s32(vmull_n_s16(x_hi[8], cospi_17_64), + vmull_n_s16(x_hi[9], cospi_15_64)); + // s9 = cospi_15_64 * x8 - cospi_17_64 * x9; + s_lo[9] = vsubq_s32(vmull_n_s16(x_lo[8], cospi_15_64), + vmull_n_s16(x_lo[9], cospi_17_64)); + s_hi[9] = vsubq_s32(vmull_n_s16(x_hi[8], cospi_15_64), + vmull_n_s16(x_hi[9], cospi_17_64)); + // s10 = cospi_21_64 * x10 + cospi_11_64 * x11; + s_lo[10] = vaddq_s32(vmull_n_s16(x_lo[10], cospi_21_64), + vmull_n_s16(x_lo[11], cospi_11_64)); + s_hi[10] = vaddq_s32(vmull_n_s16(x_hi[10], cospi_21_64), + vmull_n_s16(x_hi[11], cospi_11_64)); + // s11 = cospi_11_64 * x10 - cospi_21_64 * x11; + s_lo[11] = vsubq_s32(vmull_n_s16(x_lo[10], cospi_11_64), + vmull_n_s16(x_lo[11], cospi_21_64)); + s_hi[11] = vsubq_s32(vmull_n_s16(x_hi[10], cospi_11_64), + vmull_n_s16(x_hi[11], cospi_21_64)); + // s12 = cospi_25_64 * x12 + cospi_7_64 * x13; + s_lo[12] = vaddq_s32(vmull_n_s16(x_lo[12], cospi_25_64), + vmull_n_s16(x_lo[13], cospi_7_64)); + s_hi[12] = vaddq_s32(vmull_n_s16(x_hi[12], cospi_25_64), + vmull_n_s16(x_hi[13], cospi_7_64)); + // s13 = cospi_7_64 * x12 - cospi_25_64 * x13; + s_lo[13] = vsubq_s32(vmull_n_s16(x_lo[12], cospi_7_64), + vmull_n_s16(x_lo[13], cospi_25_64)); + s_hi[13] = vsubq_s32(vmull_n_s16(x_hi[12], cospi_7_64), + vmull_n_s16(x_hi[13], cospi_25_64)); + // s14 = cospi_29_64 * x14 + cospi_3_64 * x15; + s_lo[14] = vaddq_s32(vmull_n_s16(x_lo[14], cospi_29_64), + vmull_n_s16(x_lo[15], cospi_3_64)); + s_hi[14] = vaddq_s32(vmull_n_s16(x_hi[14], cospi_29_64), + vmull_n_s16(x_hi[15], cospi_3_64)); + // s15 = cospi_3_64 * x14 - cospi_29_64 * x15; + s_lo[15] = vsubq_s32(vmull_n_s16(x_lo[14], cospi_3_64), + vmull_n_s16(x_lo[15], cospi_29_64)); + s_hi[15] = vsubq_s32(vmull_n_s16(x_hi[14], cospi_3_64), + vmull_n_s16(x_hi[15], cospi_29_64)); + + // fdct_round_shift + t_lo[0] = vaddq_s32(s_lo[0], s_lo[8]); + t_hi[0] = vaddq_s32(s_hi[0], s_hi[8]); + t_lo[1] = vaddq_s32(s_lo[1], s_lo[9]); + t_hi[1] = vaddq_s32(s_hi[1], s_hi[9]); + t_lo[2] = vaddq_s32(s_lo[2], s_lo[10]); + t_hi[2] = vaddq_s32(s_hi[2], s_hi[10]); + t_lo[3] = vaddq_s32(s_lo[3], s_lo[11]); + t_hi[3] = vaddq_s32(s_hi[3], s_hi[11]); + t_lo[4] = vaddq_s32(s_lo[4], s_lo[12]); + t_hi[4] = vaddq_s32(s_hi[4], s_hi[12]); + t_lo[5] = vaddq_s32(s_lo[5], s_lo[13]); + t_hi[5] = vaddq_s32(s_hi[5], s_hi[13]); + t_lo[6] = vaddq_s32(s_lo[6], s_lo[14]); + t_hi[6] = vaddq_s32(s_hi[6], s_hi[14]); + t_lo[7] = vaddq_s32(s_lo[7], s_lo[15]); + t_hi[7] = vaddq_s32(s_hi[7], s_hi[15]); + t_lo[8] = vsubq_s32(s_lo[0], s_lo[8]); + t_hi[8] = vsubq_s32(s_hi[0], s_hi[8]); + t_lo[9] = vsubq_s32(s_lo[1], s_lo[9]); + t_hi[9] = vsubq_s32(s_hi[1], s_hi[9]); + t_lo[10] = vsubq_s32(s_lo[2], s_lo[10]); + t_hi[10] = vsubq_s32(s_hi[2], s_hi[10]); + t_lo[11] = vsubq_s32(s_lo[3], s_lo[11]); + t_hi[11] = vsubq_s32(s_hi[3], s_hi[11]); + t_lo[12] = vsubq_s32(s_lo[4], s_lo[12]); + t_hi[12] = vsubq_s32(s_hi[4], s_hi[12]); + t_lo[13] = vsubq_s32(s_lo[5], s_lo[13]); + t_hi[13] = vsubq_s32(s_hi[5], s_hi[13]); + t_lo[14] = vsubq_s32(s_lo[6], s_lo[14]); + t_hi[14] = vsubq_s32(s_hi[6], s_hi[14]); + t_lo[15] = vsubq_s32(s_lo[7], s_lo[15]); + t_hi[15] = vsubq_s32(s_hi[7], s_hi[15]); + + t_lo[0] = vaddq_s32(t_lo[0], k__DCT_CONST_ROUNDING); + t_hi[0] = vaddq_s32(t_hi[0], k__DCT_CONST_ROUNDING); + t_lo[1] = vaddq_s32(t_lo[1], k__DCT_CONST_ROUNDING); + t_hi[1] = vaddq_s32(t_hi[1], k__DCT_CONST_ROUNDING); + t_lo[2] = vaddq_s32(t_lo[2], k__DCT_CONST_ROUNDING); + t_hi[2] = vaddq_s32(t_hi[2], k__DCT_CONST_ROUNDING); + t_lo[3] = vaddq_s32(t_lo[3], k__DCT_CONST_ROUNDING); + t_hi[3] = vaddq_s32(t_hi[3], k__DCT_CONST_ROUNDING); + t_lo[4] = vaddq_s32(t_lo[4], k__DCT_CONST_ROUNDING); + t_hi[4] = vaddq_s32(t_hi[4], k__DCT_CONST_ROUNDING); + t_lo[5] = vaddq_s32(t_lo[5], k__DCT_CONST_ROUNDING); + t_hi[5] = vaddq_s32(t_hi[5], k__DCT_CONST_ROUNDING); + t_lo[6] = vaddq_s32(t_lo[6], k__DCT_CONST_ROUNDING); + t_hi[6] = vaddq_s32(t_hi[6], k__DCT_CONST_ROUNDING); + t_lo[7] = vaddq_s32(t_lo[7], k__DCT_CONST_ROUNDING); + t_hi[7] = vaddq_s32(t_hi[7], k__DCT_CONST_ROUNDING); + t_lo[8] = vaddq_s32(t_lo[8], k__DCT_CONST_ROUNDING); + t_hi[8] = vaddq_s32(t_hi[8], k__DCT_CONST_ROUNDING); + t_lo[9] = vaddq_s32(t_lo[9], k__DCT_CONST_ROUNDING); + t_hi[9] = vaddq_s32(t_hi[9], k__DCT_CONST_ROUNDING); + t_lo[10] = vaddq_s32(t_lo[10], k__DCT_CONST_ROUNDING); + t_hi[10] = vaddq_s32(t_hi[10], k__DCT_CONST_ROUNDING); + t_lo[11] = vaddq_s32(t_lo[11], k__DCT_CONST_ROUNDING); + t_hi[11] = vaddq_s32(t_hi[11], k__DCT_CONST_ROUNDING); + t_lo[12] = vaddq_s32(t_lo[12], k__DCT_CONST_ROUNDING); + t_hi[12] = vaddq_s32(t_hi[12], k__DCT_CONST_ROUNDING); + t_lo[13] = vaddq_s32(t_lo[13], k__DCT_CONST_ROUNDING); + t_hi[13] = vaddq_s32(t_hi[13], k__DCT_CONST_ROUNDING); + t_lo[14] = vaddq_s32(t_lo[14], k__DCT_CONST_ROUNDING); + t_hi[14] = vaddq_s32(t_hi[14], k__DCT_CONST_ROUNDING); + t_lo[15] = vaddq_s32(t_lo[15], k__DCT_CONST_ROUNDING); + t_hi[15] = vaddq_s32(t_hi[15], k__DCT_CONST_ROUNDING); + + t_lo[0] = vshrq_n_s32(t_lo[0], DCT_CONST_BITS); + t_hi[0] = vshrq_n_s32(t_hi[0], DCT_CONST_BITS); + t_lo[1] = vshrq_n_s32(t_lo[1], DCT_CONST_BITS); + t_hi[1] = vshrq_n_s32(t_hi[1], DCT_CONST_BITS); + t_lo[2] = vshrq_n_s32(t_lo[2], DCT_CONST_BITS); + t_hi[2] = vshrq_n_s32(t_hi[2], DCT_CONST_BITS); + t_lo[3] = vshrq_n_s32(t_lo[3], DCT_CONST_BITS); + t_hi[3] = vshrq_n_s32(t_hi[3], DCT_CONST_BITS); + t_lo[4] = vshrq_n_s32(t_lo[4], DCT_CONST_BITS); + t_hi[4] = vshrq_n_s32(t_hi[4], DCT_CONST_BITS); + t_lo[5] = vshrq_n_s32(t_lo[5], DCT_CONST_BITS); + t_hi[5] = vshrq_n_s32(t_hi[5], DCT_CONST_BITS); + t_lo[6] = vshrq_n_s32(t_lo[6], DCT_CONST_BITS); + t_hi[6] = vshrq_n_s32(t_hi[6], DCT_CONST_BITS); + t_lo[7] = vshrq_n_s32(t_lo[7], DCT_CONST_BITS); + t_hi[7] = vshrq_n_s32(t_hi[7], DCT_CONST_BITS); + t_lo[8] = vshrq_n_s32(t_lo[8], DCT_CONST_BITS); + t_hi[8] = vshrq_n_s32(t_hi[8], DCT_CONST_BITS); + t_lo[9] = vshrq_n_s32(t_lo[9], DCT_CONST_BITS); + t_hi[9] = vshrq_n_s32(t_hi[9], DCT_CONST_BITS); + t_lo[10] = vshrq_n_s32(t_lo[10], DCT_CONST_BITS); + t_hi[10] = vshrq_n_s32(t_hi[10], DCT_CONST_BITS); + t_lo[11] = vshrq_n_s32(t_lo[11], DCT_CONST_BITS); + t_hi[11] = vshrq_n_s32(t_hi[11], DCT_CONST_BITS); + t_lo[12] = vshrq_n_s32(t_lo[12], DCT_CONST_BITS); + t_hi[12] = vshrq_n_s32(t_hi[12], DCT_CONST_BITS); + t_lo[13] = vshrq_n_s32(t_lo[13], DCT_CONST_BITS); + t_hi[13] = vshrq_n_s32(t_hi[13], DCT_CONST_BITS); + t_lo[14] = vshrq_n_s32(t_lo[14], DCT_CONST_BITS); + t_hi[14] = vshrq_n_s32(t_hi[14], DCT_CONST_BITS); + t_lo[15] = vshrq_n_s32(t_lo[15], DCT_CONST_BITS); + t_hi[15] = vshrq_n_s32(t_hi[15], DCT_CONST_BITS); + + // stage 2 + s_lo[0] = t_lo[0]; + s_hi[0] = t_hi[0]; + s_lo[1] = t_lo[1]; + s_hi[1] = t_hi[1]; + s_lo[2] = t_lo[2]; + s_hi[2] = t_hi[2]; + s_lo[3] = t_lo[3]; + s_hi[3] = t_hi[3]; + s_lo[4] = t_lo[4]; + s_hi[4] = t_hi[4]; + s_lo[5] = t_lo[5]; + s_hi[5] = t_hi[5]; + s_lo[6] = t_lo[6]; + s_hi[6] = t_hi[6]; + s_lo[7] = t_lo[7]; + s_hi[7] = t_hi[7]; + // s8 = x8 * cospi_4_64 + x9 * cospi_28_64; + s_lo[8] = vaddq_s32(vmulq_n_s32(t_lo[8], cospi_4_64), + vmulq_n_s32(t_lo[9], cospi_28_64)); + s_hi[8] = vaddq_s32(vmulq_n_s32(t_hi[8], cospi_4_64), + vmulq_n_s32(t_hi[9], cospi_28_64)); + // s9 = x8 * cospi_28_64 - x9 * cospi_4_64; + s_lo[9] = vsubq_s32(vmulq_n_s32(t_lo[8], cospi_28_64), + vmulq_n_s32(t_lo[9], cospi_4_64)); + s_hi[9] = vsubq_s32(vmulq_n_s32(t_hi[8], cospi_28_64), + vmulq_n_s32(t_hi[9], cospi_4_64)); + // s10 = x10 * cospi_20_64 + x11 * cospi_12_64; + s_lo[10] = vaddq_s32(vmulq_n_s32(t_lo[10], cospi_20_64), + vmulq_n_s32(t_lo[11], cospi_12_64)); + s_hi[10] = vaddq_s32(vmulq_n_s32(t_hi[10], cospi_20_64), + vmulq_n_s32(t_hi[11], cospi_12_64)); + // s11 = x10 * cospi_12_64 - x11 * cospi_20_64; + s_lo[11] = vsubq_s32(vmulq_n_s32(t_lo[10], cospi_12_64), + vmulq_n_s32(t_lo[11], cospi_20_64)); + s_hi[11] = vsubq_s32(vmulq_n_s32(t_hi[10], cospi_12_64), + vmulq_n_s32(t_hi[11], cospi_20_64)); + // s12 = -x12 * cospi_28_64 + x13 * cospi_4_64; + s_lo[12] = vaddq_s32(vmulq_n_s32(t_lo[12], -cospi_28_64), + vmulq_n_s32(t_lo[13], cospi_4_64)); + s_hi[12] = vaddq_s32(vmulq_n_s32(t_hi[12], -cospi_28_64), + vmulq_n_s32(t_hi[13], cospi_4_64)); + // s13 = x12 * cospi_4_64 + x13 * cospi_28_64; + s_lo[13] = vaddq_s32(vmulq_n_s32(t_lo[12], cospi_4_64), + vmulq_n_s32(t_lo[13], cospi_28_64)); + s_hi[13] = vaddq_s32(vmulq_n_s32(t_hi[12], cospi_4_64), + vmulq_n_s32(t_hi[13], cospi_28_64)); + // s14 = -x14 * cospi_12_64 + x15 * cospi_20_64; + s_lo[14] = vaddq_s32(vmulq_n_s32(t_lo[14], -cospi_12_64), + vmulq_n_s32(t_lo[15], cospi_20_64)); + s_hi[14] = vaddq_s32(vmulq_n_s32(t_hi[14], -cospi_12_64), + vmulq_n_s32(t_hi[15], cospi_20_64)); + // s15 = x14 * cospi_20_64 + x15 * cospi_12_64; + s_lo[15] = vaddq_s32(vmulq_n_s32(t_lo[14], cospi_20_64), + vmulq_n_s32(t_lo[15], cospi_12_64)); + s_hi[15] = vaddq_s32(vmulq_n_s32(t_hi[14], cospi_20_64), + vmulq_n_s32(t_hi[15], cospi_12_64)); + + // s0 + s4 + t_lo[0] = vaddq_s32(s_lo[0], s_lo[4]); + t_hi[0] = vaddq_s32(s_hi[0], s_hi[4]); + // s1 + s5 + t_lo[1] = vaddq_s32(s_lo[1], s_lo[5]); + t_hi[1] = vaddq_s32(s_hi[1], s_hi[5]); + // s2 + s6 + t_lo[2] = vaddq_s32(s_lo[2], s_lo[6]); + t_hi[2] = vaddq_s32(s_hi[2], s_hi[6]); + // s3 + s7 + t_lo[3] = vaddq_s32(s_lo[3], s_lo[7]); + t_hi[3] = vaddq_s32(s_hi[3], s_hi[7]); + // s0 - s4 + t_lo[4] = vsubq_s32(s_lo[0], s_lo[4]); + t_hi[4] = vsubq_s32(s_hi[0], s_hi[4]); + // s1 - s7 + t_lo[5] = vsubq_s32(s_lo[1], s_lo[5]); + t_hi[5] = vsubq_s32(s_hi[1], s_hi[5]); + // s2 - s6 + t_lo[6] = vsubq_s32(s_lo[2], s_lo[6]); + t_hi[6] = vsubq_s32(s_hi[2], s_hi[6]); + // s3 - s7 + t_lo[7] = vsubq_s32(s_lo[3], s_lo[7]); + t_hi[7] = vsubq_s32(s_hi[3], s_hi[7]); + // s8 + s12 + t_lo[8] = vaddq_s32(s_lo[8], s_lo[12]); + t_hi[8] = vaddq_s32(s_hi[8], s_hi[12]); + // s9 + s13 + t_lo[9] = vaddq_s32(s_lo[9], s_lo[13]); + t_hi[9] = vaddq_s32(s_hi[9], s_hi[13]); + // s10 + s14 + t_lo[10] = vaddq_s32(s_lo[10], s_lo[14]); + t_hi[10] = vaddq_s32(s_hi[10], s_hi[14]); + // s11 + s15 + t_lo[11] = vaddq_s32(s_lo[11], s_lo[15]); + t_hi[11] = vaddq_s32(s_hi[11], s_hi[15]); + // s8 + s12 + t_lo[12] = vsubq_s32(s_lo[8], s_lo[12]); + t_hi[12] = vsubq_s32(s_hi[8], s_hi[12]); + // s9 + s13 + t_lo[13] = vsubq_s32(s_lo[9], s_lo[13]); + t_hi[13] = vsubq_s32(s_hi[9], s_hi[13]); + // s10 + s14 + t_lo[14] = vsubq_s32(s_lo[10], s_lo[14]); + t_hi[14] = vsubq_s32(s_hi[10], s_hi[14]); + // s11 + s15 + t_lo[15] = vsubq_s32(s_lo[11], s_lo[15]); + t_hi[15] = vsubq_s32(s_hi[11], s_hi[15]); + + t_lo[8] = vaddq_s32(t_lo[8], k__DCT_CONST_ROUNDING); + t_hi[8] = vaddq_s32(t_hi[8], k__DCT_CONST_ROUNDING); + t_lo[9] = vaddq_s32(t_lo[9], k__DCT_CONST_ROUNDING); + t_hi[9] = vaddq_s32(t_hi[9], k__DCT_CONST_ROUNDING); + t_lo[10] = vaddq_s32(t_lo[10], k__DCT_CONST_ROUNDING); + t_hi[10] = vaddq_s32(t_hi[10], k__DCT_CONST_ROUNDING); + t_lo[11] = vaddq_s32(t_lo[11], k__DCT_CONST_ROUNDING); + t_hi[11] = vaddq_s32(t_hi[11], k__DCT_CONST_ROUNDING); + t_lo[12] = vaddq_s32(t_lo[12], k__DCT_CONST_ROUNDING); + t_hi[12] = vaddq_s32(t_hi[12], k__DCT_CONST_ROUNDING); + t_lo[13] = vaddq_s32(t_lo[13], k__DCT_CONST_ROUNDING); + t_hi[13] = vaddq_s32(t_hi[13], k__DCT_CONST_ROUNDING); + t_lo[14] = vaddq_s32(t_lo[14], k__DCT_CONST_ROUNDING); + t_hi[14] = vaddq_s32(t_hi[14], k__DCT_CONST_ROUNDING); + t_lo[15] = vaddq_s32(t_lo[15], k__DCT_CONST_ROUNDING); + t_hi[15] = vaddq_s32(t_hi[15], k__DCT_CONST_ROUNDING); + t_lo[8] = vshrq_n_s32(t_lo[8], DCT_CONST_BITS); + t_hi[8] = vshrq_n_s32(t_hi[8], DCT_CONST_BITS); + t_lo[9] = vshrq_n_s32(t_lo[9], DCT_CONST_BITS); + t_hi[9] = vshrq_n_s32(t_hi[9], DCT_CONST_BITS); + t_lo[10] = vshrq_n_s32(t_lo[10], DCT_CONST_BITS); + t_hi[10] = vshrq_n_s32(t_hi[10], DCT_CONST_BITS); + t_lo[11] = vshrq_n_s32(t_lo[11], DCT_CONST_BITS); + t_hi[11] = vshrq_n_s32(t_hi[11], DCT_CONST_BITS); + t_lo[12] = vshrq_n_s32(t_lo[12], DCT_CONST_BITS); + t_hi[12] = vshrq_n_s32(t_hi[12], DCT_CONST_BITS); + t_lo[13] = vshrq_n_s32(t_lo[13], DCT_CONST_BITS); + t_hi[13] = vshrq_n_s32(t_hi[13], DCT_CONST_BITS); + t_lo[14] = vshrq_n_s32(t_lo[14], DCT_CONST_BITS); + t_hi[14] = vshrq_n_s32(t_hi[14], DCT_CONST_BITS); + t_lo[15] = vshrq_n_s32(t_lo[15], DCT_CONST_BITS); + t_hi[15] = vshrq_n_s32(t_hi[15], DCT_CONST_BITS); + + // stage 3 + s_lo[0] = t_lo[0]; + s_hi[0] = t_hi[0]; + s_lo[1] = t_lo[1]; + s_hi[1] = t_hi[1]; + s_lo[2] = t_lo[2]; + s_hi[2] = t_hi[2]; + s_lo[3] = t_lo[3]; + s_hi[3] = t_hi[3]; + // s4 = x4 * cospi_8_64 + x5 * cospi_24_64; + s_lo[4] = vaddq_s32(vmulq_n_s32(t_lo[4], cospi_8_64), + vmulq_n_s32(t_lo[5], cospi_24_64)); + s_hi[4] = vaddq_s32(vmulq_n_s32(t_hi[4], cospi_8_64), + vmulq_n_s32(t_hi[5], cospi_24_64)); + // s5 = x4 * cospi_24_64 - x5 * cospi_8_64; + s_lo[5] = vaddq_s32(vmulq_n_s32(t_lo[4], cospi_24_64), + vmulq_n_s32(t_lo[5], -cospi_8_64)); + s_hi[5] = vaddq_s32(vmulq_n_s32(t_hi[4], cospi_24_64), + vmulq_n_s32(t_hi[5], -cospi_8_64)); + // s6 = -x6 * cospi_24_64 + x7 * cospi_8_64; + s_lo[6] = vaddq_s32(vmulq_n_s32(t_lo[6], -cospi_24_64), + vmulq_n_s32(t_lo[7], cospi_8_64)); + s_hi[6] = vaddq_s32(vmulq_n_s32(t_hi[6], -cospi_24_64), + vmulq_n_s32(t_hi[7], cospi_8_64)); + // s7 = x6 * cospi_8_64 + x7 * cospi_24_64; + s_lo[7] = vaddq_s32(vmulq_n_s32(t_lo[6], cospi_8_64), + vmulq_n_s32(t_lo[7], cospi_24_64)); + s_hi[7] = vaddq_s32(vmulq_n_s32(t_hi[6], cospi_8_64), + vmulq_n_s32(t_hi[7], cospi_24_64)); + s_lo[8] = t_lo[8]; + s_hi[8] = t_hi[8]; + s_lo[9] = t_lo[9]; + s_hi[9] = t_hi[9]; + s_lo[10] = t_lo[10]; + s_hi[10] = t_hi[10]; + s_lo[11] = t_lo[11]; + s_hi[11] = t_hi[11]; + // s12 = x12 * cospi_8_64 + x13 * cospi_24_64; + s_lo[12] = vaddq_s32(vmulq_n_s32(t_lo[12], cospi_8_64), + vmulq_n_s32(t_lo[13], cospi_24_64)); + s_hi[12] = vaddq_s32(vmulq_n_s32(t_hi[12], cospi_8_64), + vmulq_n_s32(t_hi[13], cospi_24_64)); + // s13 = x12 * cospi_24_64 - x13 * cospi_8_64; + s_lo[13] = vaddq_s32(vmulq_n_s32(t_lo[12], cospi_24_64), + vmulq_n_s32(t_lo[13], -cospi_8_64)); + s_hi[13] = vaddq_s32(vmulq_n_s32(t_hi[12], cospi_24_64), + vmulq_n_s32(t_hi[13], -cospi_8_64)); + // s14 = -x14 * cospi_24_64 + x15 * cospi_8_64; + s_lo[14] = vaddq_s32(vmulq_n_s32(t_lo[14], -cospi_24_64), + vmulq_n_s32(t_lo[15], cospi_8_64)); + s_hi[14] = vaddq_s32(vmulq_n_s32(t_hi[14], -cospi_24_64), + vmulq_n_s32(t_hi[15], cospi_8_64)); + // s15 = x14 * cospi_8_64 + x15 * cospi_24_64; + s_lo[15] = vaddq_s32(vmulq_n_s32(t_lo[14], cospi_8_64), + vmulq_n_s32(t_lo[15], cospi_24_64)); + s_hi[15] = vaddq_s32(vmulq_n_s32(t_hi[14], cospi_8_64), + vmulq_n_s32(t_hi[15], cospi_24_64)); + + // s0 + s4 + t_lo[0] = vaddq_s32(s_lo[0], s_lo[2]); + t_hi[0] = vaddq_s32(s_hi[0], s_hi[2]); + // s1 + s3 + t_lo[1] = vaddq_s32(s_lo[1], s_lo[3]); + t_hi[1] = vaddq_s32(s_hi[1], s_hi[3]); + // s0 - s4 + t_lo[2] = vsubq_s32(s_lo[0], s_lo[2]); + t_hi[2] = vsubq_s32(s_hi[0], s_hi[2]); + // s1 - s3 + t_lo[3] = vsubq_s32(s_lo[1], s_lo[3]); + t_hi[3] = vsubq_s32(s_hi[1], s_hi[3]); + // s4 + s6 + t_lo[4] = vaddq_s32(s_lo[4], s_lo[6]); + t_hi[4] = vaddq_s32(s_hi[4], s_hi[6]); + // s5 + s7 + t_lo[5] = vaddq_s32(s_lo[5], s_lo[7]); + t_hi[5] = vaddq_s32(s_hi[5], s_hi[7]); + // s4 - s6 + t_lo[6] = vsubq_s32(s_lo[4], s_lo[6]); + t_hi[6] = vsubq_s32(s_hi[4], s_hi[6]); + // s5 - s7 + t_lo[7] = vsubq_s32(s_lo[5], s_lo[7]); + t_hi[7] = vsubq_s32(s_hi[5], s_hi[7]); + // s8 + s10 + t_lo[8] = vaddq_s32(s_lo[8], s_lo[10]); + t_hi[8] = vaddq_s32(s_hi[8], s_hi[10]); + // s9 + s11 + t_lo[9] = vaddq_s32(s_lo[9], s_lo[11]); + t_hi[9] = vaddq_s32(s_hi[9], s_hi[11]); + // s8 - s10 + t_lo[10] = vsubq_s32(s_lo[8], s_lo[10]); + t_hi[10] = vsubq_s32(s_hi[8], s_hi[10]); + // s9 - s11 + t_lo[11] = vsubq_s32(s_lo[9], s_lo[11]); + t_hi[11] = vsubq_s32(s_hi[9], s_hi[11]); + // s12 + s14 + t_lo[12] = vaddq_s32(s_lo[12], s_lo[14]); + t_hi[12] = vaddq_s32(s_hi[12], s_hi[14]); + // s13 + s15 + t_lo[13] = vaddq_s32(s_lo[13], s_lo[15]); + t_hi[13] = vaddq_s32(s_hi[13], s_hi[15]); + // s12 - s14 + t_lo[14] = vsubq_s32(s_lo[12], s_lo[14]); + t_hi[14] = vsubq_s32(s_hi[12], s_hi[14]); + // s13 - s15 + t_lo[15] = vsubq_s32(s_lo[13], s_lo[15]); + t_hi[15] = vsubq_s32(s_hi[13], s_hi[15]); + + t_lo[4] = vaddq_s32(t_lo[4], k__DCT_CONST_ROUNDING); + t_hi[4] = vaddq_s32(t_hi[4], k__DCT_CONST_ROUNDING); + t_lo[5] = vaddq_s32(t_lo[5], k__DCT_CONST_ROUNDING); + t_hi[5] = vaddq_s32(t_hi[5], k__DCT_CONST_ROUNDING); + t_lo[6] = vaddq_s32(t_lo[6], k__DCT_CONST_ROUNDING); + t_hi[6] = vaddq_s32(t_hi[6], k__DCT_CONST_ROUNDING); + t_lo[7] = vaddq_s32(t_lo[7], k__DCT_CONST_ROUNDING); + t_hi[7] = vaddq_s32(t_hi[7], k__DCT_CONST_ROUNDING); + t_lo[12] = vaddq_s32(t_lo[12], k__DCT_CONST_ROUNDING); + t_hi[12] = vaddq_s32(t_hi[12], k__DCT_CONST_ROUNDING); + t_lo[13] = vaddq_s32(t_lo[13], k__DCT_CONST_ROUNDING); + t_hi[13] = vaddq_s32(t_hi[13], k__DCT_CONST_ROUNDING); + t_lo[14] = vaddq_s32(t_lo[14], k__DCT_CONST_ROUNDING); + t_hi[14] = vaddq_s32(t_hi[14], k__DCT_CONST_ROUNDING); + t_lo[15] = vaddq_s32(t_lo[15], k__DCT_CONST_ROUNDING); + t_hi[15] = vaddq_s32(t_hi[15], k__DCT_CONST_ROUNDING); + t_lo[4] = vshrq_n_s32(t_lo[4], DCT_CONST_BITS); + t_hi[4] = vshrq_n_s32(t_hi[4], DCT_CONST_BITS); + t_lo[5] = vshrq_n_s32(t_lo[5], DCT_CONST_BITS); + t_hi[5] = vshrq_n_s32(t_hi[5], DCT_CONST_BITS); + t_lo[6] = vshrq_n_s32(t_lo[6], DCT_CONST_BITS); + t_hi[6] = vshrq_n_s32(t_hi[6], DCT_CONST_BITS); + t_lo[7] = vshrq_n_s32(t_lo[7], DCT_CONST_BITS); + t_hi[7] = vshrq_n_s32(t_hi[7], DCT_CONST_BITS); + t_lo[12] = vshrq_n_s32(t_lo[12], DCT_CONST_BITS); + t_hi[12] = vshrq_n_s32(t_hi[12], DCT_CONST_BITS); + t_lo[13] = vshrq_n_s32(t_lo[13], DCT_CONST_BITS); + t_hi[13] = vshrq_n_s32(t_hi[13], DCT_CONST_BITS); + t_lo[14] = vshrq_n_s32(t_lo[14], DCT_CONST_BITS); + t_hi[14] = vshrq_n_s32(t_hi[14], DCT_CONST_BITS); + t_lo[15] = vshrq_n_s32(t_lo[15], DCT_CONST_BITS); + t_hi[15] = vshrq_n_s32(t_hi[15], DCT_CONST_BITS); + + // stage 4 + // s2 = (-cospi_16_64) * (x2 + x3); + s_lo[2] = vmulq_n_s32(vaddq_s32(t_lo[2], t_lo[3]), -cospi_16_64); + s_hi[2] = vmulq_n_s32(vaddq_s32(t_hi[2], t_hi[3]), -cospi_16_64); + // s3 = cospi_16_64 * (x2 - x3); + s_lo[3] = vmulq_n_s32(vsubq_s32(t_lo[2], t_lo[3]), cospi_16_64); + s_hi[3] = vmulq_n_s32(vsubq_s32(t_hi[2], t_hi[3]), cospi_16_64); + // s6 = cospi_16_64 * (x6 + x7); + s_lo[6] = vmulq_n_s32(vaddq_s32(t_lo[6], t_lo[7]), cospi_16_64); + s_hi[6] = vmulq_n_s32(vaddq_s32(t_hi[6], t_hi[7]), cospi_16_64); + // s7 = cospi_16_64 * (-x6 + x7); + s_lo[7] = vmulq_n_s32(vsubq_s32(t_lo[7], t_lo[6]), cospi_16_64); + s_hi[7] = vmulq_n_s32(vsubq_s32(t_hi[7], t_hi[6]), cospi_16_64); + // s10 = cospi_16_64 * (x10 + x11); + s_lo[10] = vmulq_n_s32(vaddq_s32(t_lo[10], t_lo[11]), cospi_16_64); + s_hi[10] = vmulq_n_s32(vaddq_s32(t_hi[10], t_hi[11]), cospi_16_64); + // s11 = cospi_16_64 * (-x10 + x11); + s_lo[11] = vmulq_n_s32(vsubq_s32(t_lo[11], t_lo[10]), cospi_16_64); + s_hi[11] = vmulq_n_s32(vsubq_s32(t_hi[11], t_hi[10]), cospi_16_64); + // s14 = (-cospi_16_64) * (x14 + x15); + s_lo[14] = vmulq_n_s32(vaddq_s32(t_lo[14], t_lo[15]), -cospi_16_64); + s_hi[14] = vmulq_n_s32(vaddq_s32(t_hi[14], t_hi[15]), -cospi_16_64); + // s15 = cospi_16_64 * (x14 - x15); + s_lo[15] = vmulq_n_s32(vsubq_s32(t_lo[14], t_lo[15]), cospi_16_64); + s_hi[15] = vmulq_n_s32(vsubq_s32(t_hi[14], t_hi[15]), cospi_16_64); + + // final fdct_round_shift + t_lo[2] = vaddq_s32(s_lo[2], k__DCT_CONST_ROUNDING); + t_hi[2] = vaddq_s32(s_hi[2], k__DCT_CONST_ROUNDING); + t_lo[3] = vaddq_s32(s_lo[3], k__DCT_CONST_ROUNDING); + t_hi[3] = vaddq_s32(s_hi[3], k__DCT_CONST_ROUNDING); + t_lo[6] = vaddq_s32(s_lo[6], k__DCT_CONST_ROUNDING); + t_hi[6] = vaddq_s32(s_hi[6], k__DCT_CONST_ROUNDING); + t_lo[7] = vaddq_s32(s_lo[7], k__DCT_CONST_ROUNDING); + t_hi[7] = vaddq_s32(s_hi[7], k__DCT_CONST_ROUNDING); + t_lo[10] = vaddq_s32(s_lo[10], k__DCT_CONST_ROUNDING); + t_hi[10] = vaddq_s32(s_hi[10], k__DCT_CONST_ROUNDING); + t_lo[11] = vaddq_s32(s_lo[11], k__DCT_CONST_ROUNDING); + t_hi[11] = vaddq_s32(s_hi[11], k__DCT_CONST_ROUNDING); + t_lo[14] = vaddq_s32(s_lo[14], k__DCT_CONST_ROUNDING); + t_hi[14] = vaddq_s32(s_hi[14], k__DCT_CONST_ROUNDING); + t_lo[15] = vaddq_s32(s_lo[15], k__DCT_CONST_ROUNDING); + t_hi[15] = vaddq_s32(s_hi[15], k__DCT_CONST_ROUNDING); + + x_lo[2] = vshrn_n_s32(t_lo[2], DCT_CONST_BITS); + x_hi[2] = vshrn_n_s32(t_hi[2], DCT_CONST_BITS); + x_lo[3] = vshrn_n_s32(t_lo[3], DCT_CONST_BITS); + x_hi[3] = vshrn_n_s32(t_hi[3], DCT_CONST_BITS); + x_lo[6] = vshrn_n_s32(t_lo[6], DCT_CONST_BITS); + x_hi[6] = vshrn_n_s32(t_hi[6], DCT_CONST_BITS); + x_lo[7] = vshrn_n_s32(t_lo[7], DCT_CONST_BITS); + x_hi[7] = vshrn_n_s32(t_hi[7], DCT_CONST_BITS); + x_lo[10] = vshrn_n_s32(t_lo[10], DCT_CONST_BITS); + x_hi[10] = vshrn_n_s32(t_hi[10], DCT_CONST_BITS); + x_lo[11] = vshrn_n_s32(t_lo[11], DCT_CONST_BITS); + x_hi[11] = vshrn_n_s32(t_hi[11], DCT_CONST_BITS); + x_lo[14] = vshrn_n_s32(t_lo[14], DCT_CONST_BITS); + x_hi[14] = vshrn_n_s32(t_hi[14], DCT_CONST_BITS); + x_lo[15] = vshrn_n_s32(t_lo[15], DCT_CONST_BITS); + x_hi[15] = vshrn_n_s32(t_hi[15], DCT_CONST_BITS); + + // x0, x1, x4, x5, x8, x9, x12, x13 narrow down to 16-bits directly + x_lo[0] = vmovn_s32(t_lo[0]); + x_hi[0] = vmovn_s32(t_hi[0]); + x_lo[1] = vmovn_s32(t_lo[1]); + x_hi[1] = vmovn_s32(t_hi[1]); + x_lo[4] = vmovn_s32(t_lo[4]); + x_hi[4] = vmovn_s32(t_hi[4]); + x_lo[5] = vmovn_s32(t_lo[5]); + x_hi[5] = vmovn_s32(t_hi[5]); + x_lo[8] = vmovn_s32(t_lo[8]); + x_hi[8] = vmovn_s32(t_hi[8]); + x_lo[9] = vmovn_s32(t_lo[9]); + x_hi[9] = vmovn_s32(t_hi[9]); + x_lo[12] = vmovn_s32(t_lo[12]); + x_hi[12] = vmovn_s32(t_hi[12]); + x_lo[13] = vmovn_s32(t_lo[13]); + x_hi[13] = vmovn_s32(t_hi[13]); + + in[0] = vcombine_s16(x_lo[0], x_hi[0]); + in[1] = vnegq_s16(vcombine_s16(x_lo[8], x_hi[8])); + in[2] = vcombine_s16(x_lo[12], x_hi[12]); + in[3] = vnegq_s16(vcombine_s16(x_lo[4], x_hi[4])); + in[4] = vcombine_s16(x_lo[6], x_hi[6]); + in[5] = vcombine_s16(x_lo[14], x_hi[14]); + in[6] = vcombine_s16(x_lo[10], x_hi[10]); + in[7] = vcombine_s16(x_lo[2], x_hi[2]); + in[8] = vcombine_s16(x_lo[3], x_hi[3]); + in[9] = vcombine_s16(x_lo[11], x_hi[11]); + in[10] = vcombine_s16(x_lo[15], x_hi[15]); + in[11] = vcombine_s16(x_lo[7], x_hi[7]); + in[12] = vcombine_s16(x_lo[5], x_hi[5]); + in[13] = vnegq_s16(vcombine_s16(x_lo[13], x_hi[13])); + in[14] = vcombine_s16(x_lo[9], x_hi[9]); + in[15] = vnegq_s16(vcombine_s16(x_lo[1], x_hi[1])); +} + +static void fdct16x16_neon(int16x8_t *in0, int16x8_t *in1) { + // Left half. + fdct16_8col(in0); + // Right half. + fdct16_8col(in1); + transpose_s16_16x16(in0, in1); +} + +static void fadst16x16_neon(int16x8_t *in0, int16x8_t *in1) { + fadst16_8col(in0); + fadst16_8col(in1); + transpose_s16_16x16(in0, in1); +} + +void vp9_fht16x16_neon(const int16_t *input, tran_low_t *output, int stride, + int tx_type) { + int16x8_t in0[16], in1[16]; + + switch (tx_type) { + case DCT_DCT: vpx_fdct16x16_neon(input, output, stride); break; + case ADST_DCT: + load_buffer_16x16(input, in0, in1, stride); + fadst16x16_neon(in0, in1); + right_shift_16x16(in0, in1); + fdct16x16_neon(in0, in1); + write_buffer_16x16(output, in0, in1, 16); + break; + case DCT_ADST: + load_buffer_16x16(input, in0, in1, stride); + fdct16x16_neon(in0, in1); + right_shift_16x16(in0, in1); + fadst16x16_neon(in0, in1); + write_buffer_16x16(output, in0, in1, 16); + break; + default: + assert(tx_type == ADST_ADST); + load_buffer_16x16(input, in0, in1, stride); + fadst16x16_neon(in0, in1); + right_shift_16x16(in0, in1); + fadst16x16_neon(in0, in1); + write_buffer_16x16(output, in0, in1, 16); + break; + } +} diff --git a/media/libvpx/libvpx/vp9/encoder/arm/neon/vp9_diamond_search_sad_neon.c b/media/libvpx/libvpx/vp9/encoder/arm/neon/vp9_diamond_search_sad_neon.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vp9/encoder/arm/neon/vp9_diamond_search_sad_neon.c @@ -0,0 +1,322 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include "vpx_dsp/vpx_dsp_common.h" +#include "vp9/encoder/vp9_encoder.h" +#include "vpx_ports/mem.h" + +#ifdef __GNUC__ +#define LIKELY(v) __builtin_expect(v, 1) +#define UNLIKELY(v) __builtin_expect(v, 0) +#else +#define LIKELY(v) (v) +#define UNLIKELY(v) (v) +#endif + +static INLINE int_mv pack_int_mv(int16_t row, int16_t col) { + int_mv result; + result.as_mv.row = row; + result.as_mv.col = col; + return result; +} + +static INLINE MV_JOINT_TYPE get_mv_joint(const int_mv mv) { + // This is simplified from the C implementation to utilise that + // x->nmvjointsadcost[1] == x->nmvjointsadcost[2] and + // x->nmvjointsadcost[1] == x->nmvjointsadcost[3] + return mv.as_int == 0 ? 0 : 1; +} + +static INLINE int mv_cost(const int_mv mv, const int *joint_cost, + int *const comp_cost[2]) { + assert(mv.as_mv.row >= -MV_MAX && mv.as_mv.row < MV_MAX); + assert(mv.as_mv.col >= -MV_MAX && mv.as_mv.col < MV_MAX); + return joint_cost[get_mv_joint(mv)] + comp_cost[0][mv.as_mv.row] + + comp_cost[1][mv.as_mv.col]; +} + +static int mvsad_err_cost(const MACROBLOCK *x, const int_mv mv, const MV *ref, + int sad_per_bit) { + const int_mv diff = + pack_int_mv(mv.as_mv.row - ref->row, mv.as_mv.col - ref->col); + return ROUND_POWER_OF_TWO( + (unsigned)mv_cost(diff, x->nmvjointsadcost, x->nmvsadcost) * sad_per_bit, + VP9_PROB_COST_SHIFT); +} + +/***************************************************************************** + * This function utilizes 3 properties of the cost function lookup tables, * + * constructed in using 'cal_nmvjointsadcost' and 'cal_nmvsadcosts' in * + * vp9_encoder.c. * + * For the joint cost: * + * - mvjointsadcost[1] == mvjointsadcost[2] == mvjointsadcost[3] * + * For the component costs: * + * - For all i: mvsadcost[0][i] == mvsadcost[1][i] * + * (Equal costs for both components) * + * - For all i: mvsadcost[0][i] == mvsadcost[0][-i] * + * (Cost function is even) * + * If these do not hold, then this function cannot be used without * + * modification, in which case you can revert to using the C implementation, * + * which does not rely on these properties. * + *****************************************************************************/ +int vp9_diamond_search_sad_neon(const MACROBLOCK *x, + const search_site_config *cfg, MV *ref_mv, + MV *best_mv, int search_param, int sad_per_bit, + int *num00, const vp9_variance_fn_ptr_t *fn_ptr, + const MV *center_mv) { + static const uint32_t data[4] = { 0, 1, 2, 3 }; + const uint32x4_t v_idx_d = vld1q_u32((const uint32_t *)data); + + const int32x4_t zero_s32 = vdupq_n_s32(0); + const int_mv maxmv = pack_int_mv(x->mv_limits.row_max, x->mv_limits.col_max); + const int16x8_t v_max_mv_w = vreinterpretq_s16_s32(vdupq_n_s32(maxmv.as_int)); + const int_mv minmv = pack_int_mv(x->mv_limits.row_min, x->mv_limits.col_min); + const int16x8_t v_min_mv_w = vreinterpretq_s16_s32(vdupq_n_s32(minmv.as_int)); + + const int32x4_t v_spb_d = vdupq_n_s32(sad_per_bit); + + const int32x4_t v_joint_cost_0_d = vdupq_n_s32(x->nmvjointsadcost[0]); + const int32x4_t v_joint_cost_1_d = vdupq_n_s32(x->nmvjointsadcost[1]); + + // search_param determines the length of the initial step and hence the number + // of iterations. + // 0 = initial step (MAX_FIRST_STEP) pel + // 1 = (MAX_FIRST_STEP/2) pel, + // 2 = (MAX_FIRST_STEP/4) pel... + const MV *ss_mv = &cfg->ss_mv[cfg->searches_per_step * search_param]; + const intptr_t *ss_os = &cfg->ss_os[cfg->searches_per_step * search_param]; + const int tot_steps = cfg->total_steps - search_param; + + const int_mv fcenter_mv = + pack_int_mv(center_mv->row >> 3, center_mv->col >> 3); + const int16x8_t vfcmv = vdupq_n_s16(fcenter_mv.as_int); + + const int ref_row = clamp(ref_mv->row, minmv.as_mv.row, maxmv.as_mv.row); + const int ref_col = clamp(ref_mv->col, minmv.as_mv.col, maxmv.as_mv.col); + + int_mv bmv = pack_int_mv(ref_row, ref_col); + int_mv new_bmv = bmv; + int16x8_t v_bmv_w = vreinterpretq_s16_s32(vdupq_n_s32(bmv.as_int)); + + const int what_stride = x->plane[0].src.stride; + const int in_what_stride = x->e_mbd.plane[0].pre[0].stride; + const uint8_t *const what = x->plane[0].src.buf; + const uint8_t *const in_what = + x->e_mbd.plane[0].pre[0].buf + ref_row * in_what_stride + ref_col; + + // Work out the start point for the search + const uint8_t *best_address = in_what; + const uint8_t *new_best_address = best_address; +#if defined(__aarch64__) + int64x2_t v_ba_q = vdupq_n_s64((intptr_t)best_address); +#else + int32x4_t v_ba_d = vdupq_n_s32((intptr_t)best_address); +#endif + unsigned int best_sad = INT_MAX; + int i, j, step; + + // Check the prerequisite cost function properties that are easy to check + // in an assert. See the function-level documentation for details on all + // prerequisites. + assert(x->nmvjointsadcost[1] == x->nmvjointsadcost[2]); + assert(x->nmvjointsadcost[1] == x->nmvjointsadcost[3]); + + // Check the starting position + best_sad = fn_ptr->sdf(what, what_stride, in_what, in_what_stride); + best_sad += mvsad_err_cost(x, bmv, &fcenter_mv.as_mv, sad_per_bit); + + *num00 = 0; + + for (i = 0, step = 0; step < tot_steps; step++) { + for (j = 0; j < cfg->searches_per_step; j += 4, i += 4) { + int16x8_t v_diff_mv_w; + int8x16_t v_inside_d; + uint32x4_t v_outside_d; + int32x4_t v_cost_d, v_sad_d; +#if defined(__aarch64__) + int64x2_t v_blocka[2]; +#else + int32x4_t v_blocka[1]; + uint32x2_t horiz_max_0, horiz_max_1; +#endif + + uint32_t horiz_max; + // Compute the candidate motion vectors + const int16x8_t v_ss_mv_w = vld1q_s16((const int16_t *)&ss_mv[i]); + const int16x8_t v_these_mv_w = vaddq_s16(v_bmv_w, v_ss_mv_w); + // Clamp them to the search bounds + int16x8_t v_these_mv_clamp_w = v_these_mv_w; + v_these_mv_clamp_w = vminq_s16(v_these_mv_clamp_w, v_max_mv_w); + v_these_mv_clamp_w = vmaxq_s16(v_these_mv_clamp_w, v_min_mv_w); + // The ones that did not change are inside the search area + v_inside_d = vreinterpretq_s8_u32( + vceqq_s32(vreinterpretq_s32_s16(v_these_mv_clamp_w), + vreinterpretq_s32_s16(v_these_mv_w))); + + // If none of them are inside, then move on +#if defined(__aarch64__) + horiz_max = vmaxvq_u32(vreinterpretq_u32_s8(v_inside_d)); +#else + horiz_max_0 = vmax_u32(vget_low_u32(vreinterpretq_u32_s8(v_inside_d)), + vget_high_u32(vreinterpretq_u32_s8(v_inside_d))); + horiz_max_1 = vpmax_u32(horiz_max_0, horiz_max_0); + vst1_lane_u32(&horiz_max, horiz_max_1, 0); +#endif + if (LIKELY(horiz_max == 0)) { + continue; + } + + // The inverse mask indicates which of the MVs are outside + v_outside_d = + vreinterpretq_u32_s8(veorq_s8(v_inside_d, vdupq_n_s8((int8_t)0xff))); + // Shift right to keep the sign bit clear, we will use this later + // to set the cost to the maximum value. + v_outside_d = vshrq_n_u32(v_outside_d, 1); + + // Compute the difference MV + v_diff_mv_w = vsubq_s16(v_these_mv_clamp_w, vfcmv); + // We utilise the fact that the cost function is even, and use the + // absolute difference. This allows us to use unsigned indexes later + // and reduces cache pressure somewhat as only a half of the table + // is ever referenced. + v_diff_mv_w = vabsq_s16(v_diff_mv_w); + + // Compute the SIMD pointer offsets. + { +#if defined(__aarch64__) // sizeof(intptr_t) == 8 + // Load the offsets + int64x2_t v_bo10_q = vld1q_s64((const int64_t *)&ss_os[i + 0]); + int64x2_t v_bo32_q = vld1q_s64((const int64_t *)&ss_os[i + 2]); + // Set the ones falling outside to zero + v_bo10_q = vandq_s64( + v_bo10_q, + vmovl_s32(vget_low_s32(vreinterpretq_s32_s8(v_inside_d)))); + v_bo32_q = vandq_s64( + v_bo32_q, + vmovl_s32(vget_high_s32(vreinterpretq_s32_s8(v_inside_d)))); + // Compute the candidate addresses + v_blocka[0] = vaddq_s64(v_ba_q, v_bo10_q); + v_blocka[1] = vaddq_s64(v_ba_q, v_bo32_q); +#else // sizeof(intptr_t) == 4 + int32x4_t v_bo_d = vld1q_s32((const int32_t *)&ss_os[i]); + v_bo_d = vandq_s32(v_bo_d, vreinterpretq_s32_s8(v_inside_d)); + v_blocka[0] = vaddq_s32(v_ba_d, v_bo_d); +#endif + } + + fn_ptr->sdx4df(what, what_stride, (const uint8_t **)&v_blocka[0], + in_what_stride, (uint32_t *)&v_sad_d); + + // Look up the component cost of the residual motion vector + { + uint32_t cost[4]; + int16_t __attribute__((aligned(16))) rowcol[8]; + vst1q_s16(rowcol, v_diff_mv_w); + + // Note: This is a use case for gather instruction + cost[0] = x->nmvsadcost[0][rowcol[0]] + x->nmvsadcost[0][rowcol[1]]; + cost[1] = x->nmvsadcost[0][rowcol[2]] + x->nmvsadcost[0][rowcol[3]]; + cost[2] = x->nmvsadcost[0][rowcol[4]] + x->nmvsadcost[0][rowcol[5]]; + cost[3] = x->nmvsadcost[0][rowcol[6]] + x->nmvsadcost[0][rowcol[7]]; + + v_cost_d = vld1q_s32((int32_t *)cost); + } + + // Now add in the joint cost + { + const uint32x4_t v_sel_d = + vceqq_s32(vreinterpretq_s32_s16(v_diff_mv_w), zero_s32); + const int32x4_t v_joint_cost_d = vreinterpretq_s32_u8( + vbslq_u8(vreinterpretq_u8_u32(v_sel_d), + vreinterpretq_u8_s32(v_joint_cost_0_d), + vreinterpretq_u8_s32(v_joint_cost_1_d))); + v_cost_d = vaddq_s32(v_cost_d, v_joint_cost_d); + } + + // Multiply by sad_per_bit + v_cost_d = vmulq_s32(v_cost_d, v_spb_d); + // ROUND_POWER_OF_TWO(v_cost_d, VP9_PROB_COST_SHIFT) + v_cost_d = + vaddq_s32(v_cost_d, vdupq_n_s32(1 << (VP9_PROB_COST_SHIFT - 1))); + v_cost_d = vshrq_n_s32(v_cost_d, VP9_PROB_COST_SHIFT); + // Add the cost to the sad + v_sad_d = vaddq_s32(v_sad_d, v_cost_d); + + // Make the motion vectors outside the search area have max cost + // by or'ing in the comparison mask, this way the minimum search won't + // pick them. + v_sad_d = vorrq_s32(v_sad_d, vreinterpretq_s32_u32(v_outside_d)); + + // Find the minimum value and index horizontally in v_sad_d + { + uint32_t local_best_sad; +#if defined(__aarch64__) + local_best_sad = vminvq_u32(vreinterpretq_u32_s32(v_sad_d)); +#else + uint32x2_t horiz_min_0 = + vmin_u32(vget_low_u32(vreinterpretq_u32_s32(v_sad_d)), + vget_high_u32(vreinterpretq_u32_s32(v_sad_d))); + uint32x2_t horiz_min_1 = vpmin_u32(horiz_min_0, horiz_min_0); + vst1_lane_u32(&local_best_sad, horiz_min_1, 0); +#endif + + // Update the global minimum if the local minimum is smaller + if (LIKELY(local_best_sad < best_sad)) { +#if defined(__GNUC__) && __GNUC__ >= 4 && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif + uint32_t local_best_idx; + const uint32x4_t v_sel_d = + vceqq_s32(v_sad_d, vdupq_n_s32(local_best_sad)); + uint32x4_t v_mask_d = vandq_u32(v_sel_d, v_idx_d); + v_mask_d = vbslq_u32(v_sel_d, v_mask_d, vdupq_n_u32(0xffffffff)); + +#if defined(__aarch64__) + local_best_idx = vminvq_u32(v_mask_d); +#else + horiz_min_0 = + vmin_u32(vget_low_u32(v_mask_d), vget_high_u32(v_mask_d)); + horiz_min_1 = vpmin_u32(horiz_min_0, horiz_min_0); + vst1_lane_u32(&local_best_idx, horiz_min_1, 0); +#endif + + new_bmv = ((const int_mv *)&v_these_mv_w)[local_best_idx]; +#if defined(__GNUC__) && __GNUC__ >= 4 && !defined(__clang__) +#pragma GCC diagnostic pop +#endif + new_best_address = ((const uint8_t **)v_blocka)[local_best_idx]; + + best_sad = local_best_sad; + } + } + } + + bmv = new_bmv; + best_address = new_best_address; + + v_bmv_w = vreinterpretq_s16_s32(vdupq_n_s32(bmv.as_int)); +#if defined(__aarch64__) + v_ba_q = vdupq_n_s64((intptr_t)best_address); +#else + v_ba_d = vdupq_n_s32((intptr_t)best_address); +#endif + + if (UNLIKELY(best_address == in_what)) { + (*num00)++; + } + } + + *best_mv = bmv.as_mv; + return best_sad; +} diff --git a/media/libvpx/libvpx/vp9/encoder/arm/neon/vp9_quantize_neon.c b/media/libvpx/libvpx/vp9/encoder/arm/neon/vp9_quantize_neon.c --- a/media/libvpx/libvpx/vp9/encoder/arm/neon/vp9_quantize_neon.c +++ b/media/libvpx/libvpx/vp9/encoder/arm/neon/vp9_quantize_neon.c @@ -41,15 +41,14 @@ vst1q_s16(dqcoeff, vcombine_s16(vmovn_s32(dqcoeff_0), vmovn_s32(dqcoeff_1))); #endif // CONFIG_VP9_HIGHBITDEPTH } void vp9_quantize_fp_neon(const tran_low_t *coeff_ptr, intptr_t count, - int skip_block, const int16_t *round_ptr, - const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, - uint16_t *eob_ptr, const int16_t *scan, - const int16_t *iscan) { + const int16_t *round_ptr, const int16_t *quant_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { // Quantization pass: All coefficients with index >= zero_flag are // skippable. Note: zero_flag can be zero. int i; const int16x8_t v_zero = vdupq_n_s16(0); const int16x8_t v_one = vdupq_n_s16(1); @@ -57,12 +56,10 @@ int16x8_t v_round = vmovq_n_s16(round_ptr[1]); int16x8_t v_quant = vmovq_n_s16(quant_ptr[1]); int16x8_t v_dequant = vmovq_n_s16(dequant_ptr[1]); (void)scan; - (void)skip_block; - assert(!skip_block); // adjust for dc v_round = vsetq_lane_s16(round_ptr[0], v_round, 0); v_quant = vsetq_lane_s16(quant_ptr[0], v_quant, 0); v_dequant = vsetq_lane_s16(dequant_ptr[0], v_dequant, 0); @@ -136,11 +133,11 @@ static INLINE int32x4_t extract_sign_bit(int32x4_t a) { return vreinterpretq_s32_u32(vshrq_n_u32(vreinterpretq_u32_s32(a), 31)); } void vp9_quantize_fp_32x32_neon(const tran_low_t *coeff_ptr, intptr_t count, - int skip_block, const int16_t *round_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan) { const int16x8_t one = vdupq_n_s16(1); @@ -165,12 +162,10 @@ int16x8_t qcoeff = vqaddq_s16(coeff_abs, round); int32x4_t dqcoeff_0, dqcoeff_1; uint16x8_t eob_max; (void)scan; (void)count; - (void)skip_block; - assert(!skip_block); // coeff * quant_ptr[]) >> 15 qcoeff = vqdmulhq_s16(qcoeff, quant); // Restore sign. diff --git a/media/libvpx/libvpx/vp9/encoder/ppc/vp9_quantize_vsx.c b/media/libvpx/libvpx/vp9/encoder/ppc/vp9_quantize_vsx.c --- a/media/libvpx/libvpx/vp9/encoder/ppc/vp9_quantize_vsx.c +++ b/media/libvpx/libvpx/vp9/encoder/ppc/vp9_quantize_vsx.c @@ -37,15 +37,14 @@ a = vec_max(a, vec_perm(a, a, vec_perm32)); return vec_max(a, vec_perm(a, a, vec_perm16)); } void vp9_quantize_fp_vsx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *round_ptr, - const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, - uint16_t *eob_ptr, const int16_t *scan, - const int16_t *iscan) { + const int16_t *round_ptr, const int16_t *quant_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { int16x8_t qcoeff0, qcoeff1, dqcoeff0, dqcoeff1, eob; bool16x8_t zero_coeff0, zero_coeff1; int16x8_t round = vec_vsx_ld(0, round_ptr); int16x8_t quant = vec_vsx_ld(0, quant_ptr); @@ -54,12 +53,10 @@ int16x8_t coeff1 = vec_vsx_ld(16, coeff_ptr); int16x8_t scan0 = vec_vsx_ld(0, iscan); int16x8_t scan1 = vec_vsx_ld(16, iscan); (void)scan; - (void)skip_block; - assert(!skip_block); // First set of 8 coeff starts with DC + 7 AC qcoeff0 = vec_mulhi(vec_vaddshs(vec_abs(coeff0), round), quant); zero_coeff0 = vec_cmpeq(qcoeff0, vec_zeros_s16); qcoeff0 = vec_sign(qcoeff0, coeff0); @@ -163,11 +160,11 @@ dqcoeffo = vec_sra(dqcoeffo, vec_ones_u32); return (int16x8_t)vec_perm(dqcoeffe, dqcoeffo, vec_perm_odd_even_pack); } void vp9_quantize_fp_32x32_vsx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *round_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan) { // In stage 1, we quantize 16 coeffs (DC + 15 AC) @@ -192,13 +189,11 @@ int16x8_t thres = vec_sra(dequant, vec_splats((uint16_t)2)); int16x8_t abs_coeff0 = vec_abs(coeff0); int16x8_t abs_coeff1 = vec_abs(coeff1); (void)scan; - (void)skip_block; (void)n_coeffs; - assert(!skip_block); mask0 = vec_cmpge(abs_coeff0, thres); round = vec_sra(vec_add(round, vec_ones_s16), vec_ones_u16); // First set of 8 coeff starts with DC + 7 AC qcoeff0 = vec_madds(vec_vaddshs(abs_coeff0, round), quant, vec_zeros_s16); diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_aq_cyclicrefresh.c b/media/libvpx/libvpx/vp9/encoder/vp9_aq_cyclicrefresh.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_aq_cyclicrefresh.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_aq_cyclicrefresh.c @@ -495,11 +495,13 @@ cpi->svc.layer_context[cpi->svc.temporal_layer_id].is_key_frame) || (!cpi->use_svc && cr->content_mode && rc->avg_frame_low_motion < thresh_low_motion && rc->frames_since_key > 40) || (!cpi->use_svc && rc->avg_frame_qindex[INTER_FRAME] > qp_max_thresh && - rc->frames_since_key > 20)) { + rc->frames_since_key > 20) || + (cpi->roi.enabled && cpi->roi.skip[BACKGROUND_SEG_SKIP_ID] && + rc->frames_since_key > FRAMES_NO_SKIPPING_AFTER_KEY)) { cr->apply_cyclic_refresh = 0; return; } cr->percent_refresh = 10; if (cr->reduce_refresh) cr->percent_refresh = 5; diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_bitstream.c b/media/libvpx/libvpx/vp9/encoder/vp9_bitstream.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_bitstream.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_bitstream.c @@ -234,15 +234,15 @@ } } } } -static void pack_inter_mode_mvs( - VP9_COMP *cpi, const MACROBLOCKD *const xd, - const MB_MODE_INFO_EXT *const mbmi_ext, vpx_writer *w, - unsigned int *const max_mv_magnitude, - int interp_filter_selected[MAX_REF_FRAMES][SWITCHABLE]) { +static void pack_inter_mode_mvs(VP9_COMP *cpi, const MACROBLOCKD *const xd, + const MB_MODE_INFO_EXT *const mbmi_ext, + vpx_writer *w, + unsigned int *const max_mv_magnitude, + int interp_filter_selected[][SWITCHABLE]) { VP9_COMMON *const cm = &cpi->common; const nmv_context *nmvc = &cm->fc->nmvc; const struct segmentation *const seg = &cm->seg; const MODE_INFO *const mi = xd->mi[0]; const PREDICTION_MODE mode = mi->mode; @@ -371,15 +371,16 @@ } write_intra_mode(w, mi->uv_mode, vp9_kf_uv_mode_prob[mi->mode]); } -static void write_modes_b( - VP9_COMP *cpi, MACROBLOCKD *const xd, const TileInfo *const tile, - vpx_writer *w, TOKENEXTRA **tok, const TOKENEXTRA *const tok_end, - int mi_row, int mi_col, unsigned int *const max_mv_magnitude, - int interp_filter_selected[MAX_REF_FRAMES][SWITCHABLE]) { +static void write_modes_b(VP9_COMP *cpi, MACROBLOCKD *const xd, + const TileInfo *const tile, vpx_writer *w, + TOKENEXTRA **tok, const TOKENEXTRA *const tok_end, + int mi_row, int mi_col, + unsigned int *const max_mv_magnitude, + int interp_filter_selected[][SWITCHABLE]) { const VP9_COMMON *const cm = &cpi->common; const MB_MODE_INFO_EXT *const mbmi_ext = cpi->td.mb.mbmi_ext_base + (mi_row * cm->mi_cols + mi_col); MODE_INFO *m; @@ -420,16 +421,16 @@ } else { assert(p == PARTITION_SPLIT); } } -static void write_modes_sb( - VP9_COMP *cpi, MACROBLOCKD *const xd, const TileInfo *const tile, - vpx_writer *w, TOKENEXTRA **tok, const TOKENEXTRA *const tok_end, - int mi_row, int mi_col, BLOCK_SIZE bsize, - unsigned int *const max_mv_magnitude, - int interp_filter_selected[MAX_REF_FRAMES][SWITCHABLE]) { +static void write_modes_sb(VP9_COMP *cpi, MACROBLOCKD *const xd, + const TileInfo *const tile, vpx_writer *w, + TOKENEXTRA **tok, const TOKENEXTRA *const tok_end, + int mi_row, int mi_col, BLOCK_SIZE bsize, + unsigned int *const max_mv_magnitude, + int interp_filter_selected[][SWITCHABLE]) { const VP9_COMMON *const cm = &cpi->common; const int bsl = b_width_log2_lookup[bsize]; const int bs = (1 << bsl) / 4; PARTITION_TYPE partition; BLOCK_SIZE subsize; @@ -483,15 +484,14 @@ if (bsize >= BLOCK_8X8 && (bsize == BLOCK_8X8 || partition != PARTITION_SPLIT)) update_partition_context(xd, mi_row, mi_col, subsize, bsize); } -static void write_modes( - VP9_COMP *cpi, MACROBLOCKD *const xd, const TileInfo *const tile, - vpx_writer *w, int tile_row, int tile_col, - unsigned int *const max_mv_magnitude, - int interp_filter_selected[MAX_REF_FRAMES][SWITCHABLE]) { +static void write_modes(VP9_COMP *cpi, MACROBLOCKD *const xd, + const TileInfo *const tile, vpx_writer *w, int tile_row, + int tile_col, unsigned int *const max_mv_magnitude, + int interp_filter_selected[][SWITCHABLE]) { const VP9_COMMON *const cm = &cpi->common; int mi_row, mi_col, tile_sb_row; TOKENEXTRA *tok = NULL; TOKENEXTRA *tok_end = NULL; @@ -552,20 +552,20 @@ int stepsize = cpi->sf.coeff_prob_appx_step; switch (cpi->sf.use_fast_coef_updates) { case TWO_LOOP: { /* dry run to see if there is any update at all needed */ - int savings = 0; + int64_t savings = 0; int update[2] = { 0, 0 }; for (i = 0; i < PLANE_TYPES; ++i) { for (j = 0; j < REF_TYPES; ++j) { for (k = 0; k < COEF_BANDS; ++k) { for (l = 0; l < BAND_COEFF_CONTEXTS(k); ++l) { for (t = 0; t < entropy_nodes_update; ++t) { vpx_prob newp = new_coef_probs[i][j][k][l][t]; const vpx_prob oldp = old_coef_probs[i][j][k][l][t]; - int s; + int64_t s; int u = 0; if (t == PIVOT_NODE) s = vp9_prob_diff_update_savings_search_model( frame_branch_ct[i][j][k][l][0], oldp, &newp, upd, stepsize); @@ -598,11 +598,11 @@ // calc probs and branch cts for this frame only for (t = 0; t < entropy_nodes_update; ++t) { vpx_prob newp = new_coef_probs[i][j][k][l][t]; vpx_prob *oldp = old_coef_probs[i][j][k][l] + t; const vpx_prob upd = DIFF_UPDATE_PROB; - int s; + int64_t s; int u = 0; if (t == PIVOT_NODE) s = vp9_prob_diff_update_savings_search_model( frame_branch_ct[i][j][k][l][0], *oldp, &newp, upd, stepsize); @@ -634,11 +634,11 @@ for (l = 0; l < BAND_COEFF_CONTEXTS(k); ++l) { // calc probs and branch cts for this frame only for (t = 0; t < entropy_nodes_update; ++t) { vpx_prob newp = new_coef_probs[i][j][k][l][t]; vpx_prob *oldp = old_coef_probs[i][j][k][l] + t; - int s; + int64_t s; int u = 0; if (t == PIVOT_NODE) { s = vp9_prob_diff_update_savings_search_model( frame_branch_ct[i][j][k][l][0], *oldp, &newp, upd, @@ -961,25 +961,24 @@ vpx_free(cpi->vp9_bitstream_worker_data); cpi->vp9_bitstream_worker_data = NULL; } } -static int encode_tiles_buffer_alloc(VP9_COMP *const cpi) { +static void encode_tiles_buffer_alloc(VP9_COMP *const cpi) { + VP9_COMMON *const cm = &cpi->common; int i; const size_t worker_data_size = cpi->num_workers * sizeof(*cpi->vp9_bitstream_worker_data); - cpi->vp9_bitstream_worker_data = vpx_memalign(16, worker_data_size); + CHECK_MEM_ERROR(cm, cpi->vp9_bitstream_worker_data, + vpx_memalign(16, worker_data_size)); memset(cpi->vp9_bitstream_worker_data, 0, worker_data_size); - if (!cpi->vp9_bitstream_worker_data) return 1; for (i = 1; i < cpi->num_workers; ++i) { cpi->vp9_bitstream_worker_data[i].dest_size = cpi->oxcf.width * cpi->oxcf.height; - cpi->vp9_bitstream_worker_data[i].dest = - vpx_malloc(cpi->vp9_bitstream_worker_data[i].dest_size); - if (!cpi->vp9_bitstream_worker_data[i].dest) return 1; + CHECK_MEM_ERROR(cm, cpi->vp9_bitstream_worker_data[i].dest, + vpx_malloc(cpi->vp9_bitstream_worker_data[i].dest_size)); } - return 0; } static size_t encode_tiles_mt(VP9_COMP *cpi, uint8_t *data_ptr) { const VPxWorkerInterface *const winterface = vpx_get_worker_interface(); VP9_COMMON *const cm = &cpi->common; @@ -990,11 +989,11 @@ if (!cpi->vp9_bitstream_worker_data || cpi->vp9_bitstream_worker_data[1].dest_size > (cpi->oxcf.width * cpi->oxcf.height)) { vp9_bitstream_encode_tiles_buffer_dealloc(cpi); - if (encode_tiles_buffer_alloc(cpi)) return 0; + encode_tiles_buffer_alloc(cpi); } while (tile_col < tile_cols) { int i, j; for (i = 0; i < num_workers && tile_col < tile_cols; ++i) { diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_cost.h b/media/libvpx/libvpx/vp9/encoder/vp9_cost.h --- a/media/libvpx/libvpx/vp9/encoder/vp9_cost.h +++ b/media/libvpx/libvpx/vp9/encoder/vp9_cost.h @@ -27,13 +27,12 @@ #define vp9_cost_one(prob) vp9_cost_zero(256 - (prob)) #define vp9_cost_bit(prob, bit) vp9_cost_zero((bit) ? 256 - (prob) : (prob)) -static INLINE unsigned int cost_branch256(const unsigned int ct[2], - vpx_prob p) { - return ct[0] * vp9_cost_zero(p) + ct[1] * vp9_cost_one(p); +static INLINE uint64_t cost_branch256(const unsigned int ct[2], vpx_prob p) { + return (uint64_t)ct[0] * vp9_cost_zero(p) + (uint64_t)ct[1] * vp9_cost_one(p); } static INLINE int treed_cost(vpx_tree tree, const vpx_prob *probs, int bits, int len) { int cost = 0; diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_encodeframe.c b/media/libvpx/libvpx/vp9/encoder/vp9_encodeframe.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_encodeframe.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_encodeframe.c @@ -1903,17 +1903,21 @@ NULL, x->e_mbd.plane[i].subsampling_x, x->e_mbd.plane[i].subsampling_y); } static void set_mode_info_seg_skip(MACROBLOCK *x, TX_MODE tx_mode, + INTERP_FILTER interp_filter, RD_COST *rd_cost, BLOCK_SIZE bsize) { MACROBLOCKD *const xd = &x->e_mbd; MODE_INFO *const mi = xd->mi[0]; INTERP_FILTER filter_ref; filter_ref = get_pred_context_switchable_interp(xd); - if (filter_ref == SWITCHABLE_FILTERS) filter_ref = EIGHTTAP; + if (interp_filter == BILINEAR) + filter_ref = BILINEAR; + else if (filter_ref == SWITCHABLE_FILTERS) + filter_ref = EIGHTTAP; mi->sb_type = bsize; mi->mode = ZEROMV; mi->tx_size = VPXMIN(max_txsize_lookup[bsize], tx_mode_to_biggest_tx_size[tx_mode]); @@ -4680,11 +4684,11 @@ hybrid_intra_mode_search(cpi, x, rd_cost, bsize, ctx); else if (cpi->svc.layer_context[cpi->svc.temporal_layer_id].is_key_frame) hybrid_search_svc_baseiskey(cpi, x, rd_cost, bsize, ctx, tile_data, mi_row, mi_col); else if (segfeature_active(&cm->seg, mi->segment_id, SEG_LVL_SKIP)) - set_mode_info_seg_skip(x, cm->tx_mode, rd_cost, bsize); + set_mode_info_seg_skip(x, cm->tx_mode, cm->interp_filter, rd_cost, bsize); else if (bsize >= BLOCK_8X8) { if (cpi->rc.hybrid_intra_scene_change) hybrid_search_scene_change(cpi, x, rd_cost, bsize, ctx, tile_data, mi_row, mi_col); else @@ -5511,20 +5515,10 @@ x->sb_mvrow_part = 0; x->sb_pickmode_part = 0; x->arf_frame_usage = 0; x->lastgolden_frame_usage = 0; - if (seg->enabled) { - const uint8_t *const map = - seg->update_map ? cpi->segmentation_map : cm->last_frame_seg_map; - int segment_id = get_segment_id(cm, map, BLOCK_64X64, mi_row, mi_col); - seg_skip = segfeature_active(seg, segment_id, SEG_LVL_SKIP); - if (seg_skip) { - partition_search_type = FIXED_PARTITION; - } - } - if (cpi->compute_source_sad_onepass && cpi->sf.use_source_sad) { int shift = cpi->Source->y_stride * (mi_row << 3) + (mi_col << 3); int sb_offset2 = ((cm->mi_cols + 7) >> 3) * (mi_row >> 3) + (mi_col >> 3); int64_t source_sad = avg_source_sad(cpi, x, shift, sb_offset2); if (sf->adapt_partition_source_sad && @@ -5532,10 +5526,42 @@ source_sad > sf->adapt_partition_thresh && (cpi->refresh_golden_frame || cpi->refresh_alt_ref_frame))) partition_search_type = REFERENCE_PARTITION; } + if (seg->enabled) { + const uint8_t *const map = + seg->update_map ? cpi->segmentation_map : cm->last_frame_seg_map; + int segment_id = get_segment_id(cm, map, BLOCK_64X64, mi_row, mi_col); + seg_skip = segfeature_active(seg, segment_id, SEG_LVL_SKIP); + + if (cpi->roi.enabled && cpi->roi.skip[BACKGROUND_SEG_SKIP_ID] && + cpi->rc.frames_since_key > FRAMES_NO_SKIPPING_AFTER_KEY && + x->content_state_sb > kLowSadLowSumdiff) { + // For ROI with skip, force segment = 0 (no skip) over whole + // superblock to avoid artifacts if temporal change in source_sad is + // not 0. + int xi, yi; + const int bw = num_8x8_blocks_wide_lookup[BLOCK_64X64]; + const int bh = num_8x8_blocks_high_lookup[BLOCK_64X64]; + const int xmis = VPXMIN(cm->mi_cols - mi_col, bw); + const int ymis = VPXMIN(cm->mi_rows - mi_row, bh); + const int block_index = mi_row * cm->mi_cols + mi_col; + set_mode_info_offsets(cm, x, xd, mi_row, mi_col); + for (yi = 0; yi < ymis; yi++) + for (xi = 0; xi < xmis; xi++) { + int map_offset = block_index + yi * cm->mi_cols + xi; + cpi->segmentation_map[map_offset] = 0; + } + set_segment_index(cpi, x, mi_row, mi_col, BLOCK_64X64, 0); + seg_skip = 0; + } + if (seg_skip) { + partition_search_type = FIXED_PARTITION; + } + } + // Set the partition type of the 64X64 block switch (partition_search_type) { case VAR_BASED_PARTITION: // TODO(jingning, marpan): The mode decision and encoding process // support both intra and inter sub8x8 block coding for RTC mode. diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_encodemb.c b/media/libvpx/libvpx/vp9/encoder/vp9_encodemb.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_encodemb.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_encodemb.c @@ -364,64 +364,61 @@ #if CONFIG_VP9_HIGHBITDEPTH if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { switch (tx_size) { case TX_32X32: highbd_fdct32x32(x->use_lp32x32fdct, src_diff, coeff, diff_stride); - vp9_highbd_quantize_fp_32x32(coeff, 1024, x->skip_block, p->round_fp, - p->quant_fp, qcoeff, dqcoeff, pd->dequant, - eob, scan_order->scan, scan_order->iscan); + vp9_highbd_quantize_fp_32x32(coeff, 1024, p->round_fp, p->quant_fp, + qcoeff, dqcoeff, pd->dequant, eob, + scan_order->scan, scan_order->iscan); break; case TX_16X16: vpx_highbd_fdct16x16(src_diff, coeff, diff_stride); - vp9_highbd_quantize_fp(coeff, 256, x->skip_block, p->round_fp, - p->quant_fp, qcoeff, dqcoeff, pd->dequant, eob, - scan_order->scan, scan_order->iscan); + vp9_highbd_quantize_fp(coeff, 256, p->round_fp, p->quant_fp, qcoeff, + dqcoeff, pd->dequant, eob, scan_order->scan, + scan_order->iscan); break; case TX_8X8: vpx_highbd_fdct8x8(src_diff, coeff, diff_stride); - vp9_highbd_quantize_fp(coeff, 64, x->skip_block, p->round_fp, - p->quant_fp, qcoeff, dqcoeff, pd->dequant, eob, - scan_order->scan, scan_order->iscan); + vp9_highbd_quantize_fp(coeff, 64, p->round_fp, p->quant_fp, qcoeff, + dqcoeff, pd->dequant, eob, scan_order->scan, + scan_order->iscan); break; default: assert(tx_size == TX_4X4); x->fwd_txfm4x4(src_diff, coeff, diff_stride); - vp9_highbd_quantize_fp(coeff, 16, x->skip_block, p->round_fp, - p->quant_fp, qcoeff, dqcoeff, pd->dequant, eob, - scan_order->scan, scan_order->iscan); + vp9_highbd_quantize_fp(coeff, 16, p->round_fp, p->quant_fp, qcoeff, + dqcoeff, pd->dequant, eob, scan_order->scan, + scan_order->iscan); break; } return; } #endif // CONFIG_VP9_HIGHBITDEPTH switch (tx_size) { case TX_32X32: fdct32x32(x->use_lp32x32fdct, src_diff, coeff, diff_stride); - vp9_quantize_fp_32x32(coeff, 1024, x->skip_block, p->round_fp, - p->quant_fp, qcoeff, dqcoeff, pd->dequant, eob, - scan_order->scan, scan_order->iscan); + vp9_quantize_fp_32x32(coeff, 1024, p->round_fp, p->quant_fp, qcoeff, + dqcoeff, pd->dequant, eob, scan_order->scan, + scan_order->iscan); break; case TX_16X16: vpx_fdct16x16(src_diff, coeff, diff_stride); - vp9_quantize_fp(coeff, 256, x->skip_block, p->round_fp, p->quant_fp, - qcoeff, dqcoeff, pd->dequant, eob, scan_order->scan, - scan_order->iscan); + vp9_quantize_fp(coeff, 256, p->round_fp, p->quant_fp, qcoeff, dqcoeff, + pd->dequant, eob, scan_order->scan, scan_order->iscan); break; case TX_8X8: vpx_fdct8x8(src_diff, coeff, diff_stride); - vp9_quantize_fp(coeff, 64, x->skip_block, p->round_fp, p->quant_fp, - qcoeff, dqcoeff, pd->dequant, eob, scan_order->scan, - scan_order->iscan); + vp9_quantize_fp(coeff, 64, p->round_fp, p->quant_fp, qcoeff, dqcoeff, + pd->dequant, eob, scan_order->scan, scan_order->iscan); break; default: assert(tx_size == TX_4X4); x->fwd_txfm4x4(src_diff, coeff, diff_stride); - vp9_quantize_fp(coeff, 16, x->skip_block, p->round_fp, p->quant_fp, - qcoeff, dqcoeff, pd->dequant, eob, scan_order->scan, - scan_order->iscan); + vp9_quantize_fp(coeff, 16, p->round_fp, p->quant_fp, qcoeff, dqcoeff, + pd->dequant, eob, scan_order->scan, scan_order->iscan); break; } } void vp9_xform_quant_dc(MACROBLOCK *x, int plane, int block, int row, int col, @@ -442,59 +439,55 @@ #if CONFIG_VP9_HIGHBITDEPTH if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { switch (tx_size) { case TX_32X32: vpx_highbd_fdct32x32_1(src_diff, coeff, diff_stride); - vpx_highbd_quantize_dc_32x32(coeff, x->skip_block, p->round, - p->quant_fp[0], qcoeff, dqcoeff, - pd->dequant[0], eob); + vpx_highbd_quantize_dc_32x32(coeff, p->round, p->quant_fp[0], qcoeff, + dqcoeff, pd->dequant[0], eob); break; case TX_16X16: vpx_highbd_fdct16x16_1(src_diff, coeff, diff_stride); - vpx_highbd_quantize_dc(coeff, 256, x->skip_block, p->round, - p->quant_fp[0], qcoeff, dqcoeff, pd->dequant[0], - eob); + vpx_highbd_quantize_dc(coeff, 256, p->round, p->quant_fp[0], qcoeff, + dqcoeff, pd->dequant[0], eob); break; case TX_8X8: vpx_highbd_fdct8x8_1(src_diff, coeff, diff_stride); - vpx_highbd_quantize_dc(coeff, 64, x->skip_block, p->round, - p->quant_fp[0], qcoeff, dqcoeff, pd->dequant[0], - eob); + vpx_highbd_quantize_dc(coeff, 64, p->round, p->quant_fp[0], qcoeff, + dqcoeff, pd->dequant[0], eob); break; default: assert(tx_size == TX_4X4); x->fwd_txfm4x4(src_diff, coeff, diff_stride); - vpx_highbd_quantize_dc(coeff, 16, x->skip_block, p->round, - p->quant_fp[0], qcoeff, dqcoeff, pd->dequant[0], - eob); + vpx_highbd_quantize_dc(coeff, 16, p->round, p->quant_fp[0], qcoeff, + dqcoeff, pd->dequant[0], eob); break; } return; } #endif // CONFIG_VP9_HIGHBITDEPTH switch (tx_size) { case TX_32X32: vpx_fdct32x32_1(src_diff, coeff, diff_stride); - vpx_quantize_dc_32x32(coeff, x->skip_block, p->round, p->quant_fp[0], - qcoeff, dqcoeff, pd->dequant[0], eob); + vpx_quantize_dc_32x32(coeff, p->round, p->quant_fp[0], qcoeff, dqcoeff, + pd->dequant[0], eob); break; case TX_16X16: vpx_fdct16x16_1(src_diff, coeff, diff_stride); - vpx_quantize_dc(coeff, 256, x->skip_block, p->round, p->quant_fp[0], - qcoeff, dqcoeff, pd->dequant[0], eob); + vpx_quantize_dc(coeff, 256, p->round, p->quant_fp[0], qcoeff, dqcoeff, + pd->dequant[0], eob); break; case TX_8X8: vpx_fdct8x8_1(src_diff, coeff, diff_stride); - vpx_quantize_dc(coeff, 64, x->skip_block, p->round, p->quant_fp[0], - qcoeff, dqcoeff, pd->dequant[0], eob); + vpx_quantize_dc(coeff, 64, p->round, p->quant_fp[0], qcoeff, dqcoeff, + pd->dequant[0], eob); break; default: assert(tx_size == TX_4X4); x->fwd_txfm4x4(src_diff, coeff, diff_stride); - vpx_quantize_dc(coeff, 16, x->skip_block, p->round, p->quant_fp[0], - qcoeff, dqcoeff, pd->dequant[0], eob); + vpx_quantize_dc(coeff, 16, p->round, p->quant_fp[0], qcoeff, dqcoeff, + pd->dequant[0], eob); break; } } void vp9_xform_quant(MACROBLOCK *x, int plane, int block, int row, int col, @@ -516,68 +509,63 @@ #if CONFIG_VP9_HIGHBITDEPTH if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { switch (tx_size) { case TX_32X32: highbd_fdct32x32(x->use_lp32x32fdct, src_diff, coeff, diff_stride); - vpx_highbd_quantize_b_32x32(coeff, 1024, x->skip_block, p->zbin, - p->round, p->quant, p->quant_shift, qcoeff, - dqcoeff, pd->dequant, eob, scan_order->scan, - scan_order->iscan); + vpx_highbd_quantize_b_32x32( + coeff, 1024, p->zbin, p->round, p->quant, p->quant_shift, qcoeff, + dqcoeff, pd->dequant, eob, scan_order->scan, scan_order->iscan); break; case TX_16X16: vpx_highbd_fdct16x16(src_diff, coeff, diff_stride); - vpx_highbd_quantize_b(coeff, 256, x->skip_block, p->zbin, p->round, - p->quant, p->quant_shift, qcoeff, dqcoeff, - pd->dequant, eob, scan_order->scan, - scan_order->iscan); + vpx_highbd_quantize_b(coeff, 256, p->zbin, p->round, p->quant, + p->quant_shift, qcoeff, dqcoeff, pd->dequant, eob, + scan_order->scan, scan_order->iscan); break; case TX_8X8: vpx_highbd_fdct8x8(src_diff, coeff, diff_stride); - vpx_highbd_quantize_b(coeff, 64, x->skip_block, p->zbin, p->round, - p->quant, p->quant_shift, qcoeff, dqcoeff, - pd->dequant, eob, scan_order->scan, - scan_order->iscan); + vpx_highbd_quantize_b(coeff, 64, p->zbin, p->round, p->quant, + p->quant_shift, qcoeff, dqcoeff, pd->dequant, eob, + scan_order->scan, scan_order->iscan); break; default: assert(tx_size == TX_4X4); x->fwd_txfm4x4(src_diff, coeff, diff_stride); - vpx_highbd_quantize_b(coeff, 16, x->skip_block, p->zbin, p->round, - p->quant, p->quant_shift, qcoeff, dqcoeff, - pd->dequant, eob, scan_order->scan, - scan_order->iscan); + vpx_highbd_quantize_b(coeff, 16, p->zbin, p->round, p->quant, + p->quant_shift, qcoeff, dqcoeff, pd->dequant, eob, + scan_order->scan, scan_order->iscan); break; } return; } #endif // CONFIG_VP9_HIGHBITDEPTH switch (tx_size) { case TX_32X32: fdct32x32(x->use_lp32x32fdct, src_diff, coeff, diff_stride); - vpx_quantize_b_32x32(coeff, 1024, x->skip_block, p->zbin, p->round, - p->quant, p->quant_shift, qcoeff, dqcoeff, - pd->dequant, eob, scan_order->scan, - scan_order->iscan); + vpx_quantize_b_32x32(coeff, 1024, p->zbin, p->round, p->quant, + p->quant_shift, qcoeff, dqcoeff, pd->dequant, eob, + scan_order->scan, scan_order->iscan); break; case TX_16X16: vpx_fdct16x16(src_diff, coeff, diff_stride); - vpx_quantize_b(coeff, 256, x->skip_block, p->zbin, p->round, p->quant, - p->quant_shift, qcoeff, dqcoeff, pd->dequant, eob, - scan_order->scan, scan_order->iscan); + vpx_quantize_b(coeff, 256, p->zbin, p->round, p->quant, p->quant_shift, + qcoeff, dqcoeff, pd->dequant, eob, scan_order->scan, + scan_order->iscan); break; case TX_8X8: vpx_fdct8x8(src_diff, coeff, diff_stride); - vpx_quantize_b(coeff, 64, x->skip_block, p->zbin, p->round, p->quant, - p->quant_shift, qcoeff, dqcoeff, pd->dequant, eob, - scan_order->scan, scan_order->iscan); + vpx_quantize_b(coeff, 64, p->zbin, p->round, p->quant, p->quant_shift, + qcoeff, dqcoeff, pd->dequant, eob, scan_order->scan, + scan_order->iscan); break; default: assert(tx_size == TX_4X4); x->fwd_txfm4x4(src_diff, coeff, diff_stride); - vpx_quantize_b(coeff, 16, x->skip_block, p->zbin, p->round, p->quant, - p->quant_shift, qcoeff, dqcoeff, pd->dequant, eob, - scan_order->scan, scan_order->iscan); + vpx_quantize_b(coeff, 16, p->zbin, p->round, p->quant, p->quant_shift, + qcoeff, dqcoeff, pd->dequant, eob, scan_order->scan, + scan_order->iscan); break; } } static void encode_block(int plane, int block, int row, int col, @@ -867,14 +855,13 @@ case TX_32X32: if (!x->skip_recode) { vpx_highbd_subtract_block(32, 32, src_diff, diff_stride, src, src_stride, dst, dst_stride, xd->bd); highbd_fdct32x32(x->use_lp32x32fdct, src_diff, coeff, diff_stride); - vpx_highbd_quantize_b_32x32(coeff, 1024, x->skip_block, p->zbin, - p->round, p->quant, p->quant_shift, - qcoeff, dqcoeff, pd->dequant, eob, - scan_order->scan, scan_order->iscan); + vpx_highbd_quantize_b_32x32( + coeff, 1024, p->zbin, p->round, p->quant, p->quant_shift, qcoeff, + dqcoeff, pd->dequant, eob, scan_order->scan, scan_order->iscan); } if (args->enable_coeff_opt && !x->skip_recode) { *a = *l = vp9_optimize_b(x, plane, block, tx_size, entropy_ctx) > 0; } if (!x->skip_encode && *eob) { @@ -887,14 +874,13 @@ src_stride, dst, dst_stride, xd->bd); if (tx_type == DCT_DCT) vpx_highbd_fdct16x16(src_diff, coeff, diff_stride); else vp9_highbd_fht16x16(src_diff, coeff, diff_stride, tx_type); - vpx_highbd_quantize_b(coeff, 256, x->skip_block, p->zbin, p->round, - p->quant, p->quant_shift, qcoeff, dqcoeff, - pd->dequant, eob, scan_order->scan, - scan_order->iscan); + vpx_highbd_quantize_b(coeff, 256, p->zbin, p->round, p->quant, + p->quant_shift, qcoeff, dqcoeff, pd->dequant, + eob, scan_order->scan, scan_order->iscan); } if (args->enable_coeff_opt && !x->skip_recode) { *a = *l = vp9_optimize_b(x, plane, block, tx_size, entropy_ctx) > 0; } if (!x->skip_encode && *eob) { @@ -908,14 +894,13 @@ src_stride, dst, dst_stride, xd->bd); if (tx_type == DCT_DCT) vpx_highbd_fdct8x8(src_diff, coeff, diff_stride); else vp9_highbd_fht8x8(src_diff, coeff, diff_stride, tx_type); - vpx_highbd_quantize_b(coeff, 64, x->skip_block, p->zbin, p->round, - p->quant, p->quant_shift, qcoeff, dqcoeff, - pd->dequant, eob, scan_order->scan, - scan_order->iscan); + vpx_highbd_quantize_b(coeff, 64, p->zbin, p->round, p->quant, + p->quant_shift, qcoeff, dqcoeff, pd->dequant, + eob, scan_order->scan, scan_order->iscan); } if (args->enable_coeff_opt && !x->skip_recode) { *a = *l = vp9_optimize_b(x, plane, block, tx_size, entropy_ctx) > 0; } if (!x->skip_encode && *eob) { @@ -930,14 +915,13 @@ src_stride, dst, dst_stride, xd->bd); if (tx_type != DCT_DCT) vp9_highbd_fht4x4(src_diff, coeff, diff_stride, tx_type); else x->fwd_txfm4x4(src_diff, coeff, diff_stride); - vpx_highbd_quantize_b(coeff, 16, x->skip_block, p->zbin, p->round, - p->quant, p->quant_shift, qcoeff, dqcoeff, - pd->dequant, eob, scan_order->scan, - scan_order->iscan); + vpx_highbd_quantize_b(coeff, 16, p->zbin, p->round, p->quant, + p->quant_shift, qcoeff, dqcoeff, pd->dequant, + eob, scan_order->scan, scan_order->iscan); } if (args->enable_coeff_opt && !x->skip_recode) { *a = *l = vp9_optimize_b(x, plane, block, tx_size, entropy_ctx) > 0; } if (!x->skip_encode && *eob) { @@ -962,14 +946,13 @@ case TX_32X32: if (!x->skip_recode) { vpx_subtract_block(32, 32, src_diff, diff_stride, src, src_stride, dst, dst_stride); fdct32x32(x->use_lp32x32fdct, src_diff, coeff, diff_stride); - vpx_quantize_b_32x32(coeff, 1024, x->skip_block, p->zbin, p->round, - p->quant, p->quant_shift, qcoeff, dqcoeff, - pd->dequant, eob, scan_order->scan, - scan_order->iscan); + vpx_quantize_b_32x32(coeff, 1024, p->zbin, p->round, p->quant, + p->quant_shift, qcoeff, dqcoeff, pd->dequant, eob, + scan_order->scan, scan_order->iscan); } if (args->enable_coeff_opt && !x->skip_recode) { *a = *l = vp9_optimize_b(x, plane, block, tx_size, entropy_ctx) > 0; } if (!x->skip_encode && *eob) @@ -978,13 +961,13 @@ case TX_16X16: if (!x->skip_recode) { vpx_subtract_block(16, 16, src_diff, diff_stride, src, src_stride, dst, dst_stride); vp9_fht16x16(src_diff, coeff, diff_stride, tx_type); - vpx_quantize_b(coeff, 256, x->skip_block, p->zbin, p->round, p->quant, - p->quant_shift, qcoeff, dqcoeff, pd->dequant, eob, - scan_order->scan, scan_order->iscan); + vpx_quantize_b(coeff, 256, p->zbin, p->round, p->quant, p->quant_shift, + qcoeff, dqcoeff, pd->dequant, eob, scan_order->scan, + scan_order->iscan); } if (args->enable_coeff_opt && !x->skip_recode) { *a = *l = vp9_optimize_b(x, plane, block, tx_size, entropy_ctx) > 0; } if (!x->skip_encode && *eob) @@ -993,13 +976,13 @@ case TX_8X8: if (!x->skip_recode) { vpx_subtract_block(8, 8, src_diff, diff_stride, src, src_stride, dst, dst_stride); vp9_fht8x8(src_diff, coeff, diff_stride, tx_type); - vpx_quantize_b(coeff, 64, x->skip_block, p->zbin, p->round, p->quant, - p->quant_shift, qcoeff, dqcoeff, pd->dequant, eob, - scan_order->scan, scan_order->iscan); + vpx_quantize_b(coeff, 64, p->zbin, p->round, p->quant, p->quant_shift, + qcoeff, dqcoeff, pd->dequant, eob, scan_order->scan, + scan_order->iscan); } if (args->enable_coeff_opt && !x->skip_recode) { *a = *l = vp9_optimize_b(x, plane, block, tx_size, entropy_ctx) > 0; } if (!x->skip_encode && *eob) @@ -1012,13 +995,13 @@ dst_stride); if (tx_type != DCT_DCT) vp9_fht4x4(src_diff, coeff, diff_stride, tx_type); else x->fwd_txfm4x4(src_diff, coeff, diff_stride); - vpx_quantize_b(coeff, 16, x->skip_block, p->zbin, p->round, p->quant, - p->quant_shift, qcoeff, dqcoeff, pd->dequant, eob, - scan_order->scan, scan_order->iscan); + vpx_quantize_b(coeff, 16, p->zbin, p->round, p->quant, p->quant_shift, + qcoeff, dqcoeff, pd->dequant, eob, scan_order->scan, + scan_order->iscan); } if (args->enable_coeff_opt && !x->skip_recode) { *a = *l = vp9_optimize_b(x, plane, block, tx_size, entropy_ctx) > 0; } if (!x->skip_encode && *eob) { diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_encoder.h b/media/libvpx/libvpx/vp9/encoder/vp9_encoder.h --- a/media/libvpx/libvpx/vp9/encoder/vp9_encoder.h +++ b/media/libvpx/libvpx/vp9/encoder/vp9_encoder.h @@ -1194,10 +1194,17 @@ static INLINE int frame_is_kf_gf_arf(const VP9_COMP *cpi) { return frame_is_intra_only(&cpi->common) || cpi->refresh_alt_ref_frame || (cpi->refresh_golden_frame && !cpi->rc.is_src_frame_alt_ref); } +static INLINE int ref_frame_to_flag(int8_t ref_frame) { + static const int kVp9RefFlagList[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG, + VP9_ALT_FLAG }; + assert(ref_frame >= LAST_FRAME && ref_frame <= ALTREF_FRAME); + return kVp9RefFlagList[ref_frame]; +} + static INLINE int get_ref_frame_map_idx(const VP9_COMP *cpi, MV_REFERENCE_FRAME ref_frame) { if (ref_frame == LAST_FRAME) { return cpi->lst_fb_idx; } else if (ref_frame == GOLDEN_FRAME) { diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_encoder.c b/media/libvpx/libvpx/vp9/encoder/vp9_encoder.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_encoder.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_encoder.c @@ -23,10 +23,11 @@ #if CONFIG_INTERNAL_STATS #include "vpx_dsp/ssim.h" #endif #include "vpx_ports/mem.h" #include "vpx_ports/system_state.h" +#include "vpx_ports/vpx_once.h" #include "vpx_ports/vpx_timer.h" #if CONFIG_BITSTREAM_DEBUG || CONFIG_MISMATCH_DEBUG #include "vpx_util/vpx_debug_util.h" #endif // CONFIG_BITSTREAM_DEBUG || CONFIG_MISMATCH_DEBUG @@ -583,12 +584,10 @@ const int *delta_lf = roi->delta_lf; const int *skip = roi->skip; int ref_frame[8]; int internal_delta_q[MAX_SEGMENTS]; int i; - static const int flag_list[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG, - VP9_ALT_FLAG }; // TODO(jianj): Investigate why ROI not working in speed < 5 or in non // realtime mode. if (cpi->oxcf.mode != REALTIME || cpi->oxcf.speed < 5) return; if (!roi->enabled) return; @@ -616,20 +615,20 @@ vp9_enable_segfeature(seg, i, SEG_LVL_ALT_LF); vp9_set_segdata(seg, i, SEG_LVL_ALT_LF, delta_lf[i]); } if (skip[i] != 0) { vp9_enable_segfeature(seg, i, SEG_LVL_SKIP); - vp9_set_segdata(seg, i, SEG_LVL_SKIP, skip[i]); + vp9_set_segdata(seg, i, SEG_LVL_SKIP, 0); } if (ref_frame[i] >= 0) { int valid_ref = 1; // ALTREF is not used as reference for nonrd_pickmode with 0 lag. if (ref_frame[i] == ALTREF_FRAME && cpi->sf.use_nonrd_pick_mode) valid_ref = 0; // If GOLDEN is selected, make sure it's set as reference. if (ref_frame[i] == GOLDEN_FRAME && - !(cpi->ref_frame_flags & flag_list[ref_frame[i]])) { + !(cpi->ref_frame_flags & ref_frame_to_flag(ref_frame[i]))) { valid_ref = 0; } // GOLDEN was updated in previous encoded frame, so GOLDEN and LAST are // same reference. if (ref_frame[i] == GOLDEN_FRAME && cpi->rc.frames_since_golden == 0) @@ -927,27 +926,24 @@ cm->mi_grid_base = temp_base; cm->mi_grid_visible = cm->mi_grid_base + cm->mi_stride + 1; cm->prev_mi_grid_visible = cm->prev_mi_grid_base + cm->mi_stride + 1; } -void vp9_initialize_enc(void) { - static volatile int init_done = 0; - - if (!init_done) { - vp9_rtcd(); - vpx_dsp_rtcd(); - vpx_scale_rtcd(); - vp9_init_intra_predictors(); - vp9_init_me_luts(); - vp9_rc_init_minq_luts(); - vp9_entropy_mv_init(); +static void initialize_enc(void) { + vp9_rtcd(); + vpx_dsp_rtcd(); + vpx_scale_rtcd(); + vp9_init_intra_predictors(); + vp9_init_me_luts(); + vp9_rc_init_minq_luts(); + vp9_entropy_mv_init(); #if !CONFIG_REALTIME_ONLY - vp9_temporal_filter_init(); -#endif - init_done = 1; - } -} + vp9_temporal_filter_init(); +#endif +} + +void vp9_initialize_enc(void) { once(initialize_enc); } static void dealloc_compressor_data(VP9_COMP *cpi) { VP9_COMMON *const cm = &cpi->common; int i; @@ -1381,25 +1377,26 @@ vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR, "Failed to allocate scaled raw source frame buffer"); #endif } -static int alloc_context_buffers_ext(VP9_COMP *cpi) { +static void alloc_context_buffers_ext(VP9_COMP *cpi) { VP9_COMMON *cm = &cpi->common; int mi_size = cm->mi_cols * cm->mi_rows; - cpi->mbmi_ext_base = vpx_calloc(mi_size, sizeof(*cpi->mbmi_ext_base)); - if (!cpi->mbmi_ext_base) return 1; - - return 0; + CHECK_MEM_ERROR(cm, cpi->mbmi_ext_base, + vpx_calloc(mi_size, sizeof(*cpi->mbmi_ext_base))); } static void alloc_compressor_data(VP9_COMP *cpi) { VP9_COMMON *cm = &cpi->common; int sb_rows; - vp9_alloc_context_buffers(cm, cm->width, cm->height); + if (vp9_alloc_context_buffers(cm, cm->width, cm->height)) { + vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR, + "Failed to allocate context buffers"); + } alloc_context_buffers_ext(cpi); vpx_free(cpi->tile_tok[0][0]); @@ -1571,19 +1568,17 @@ rc->bits_off_target = VPXMIN(rc->bits_off_target, rc->maximum_buffer_size); rc->buffer_level = VPXMIN(rc->buffer_level, rc->maximum_buffer_size); } #if CONFIG_VP9_HIGHBITDEPTH -// TODO(angiebird): make sdx8f available for highbitdepth if needed #define HIGHBD_BFP(BT, SDF, SDAF, VF, SVF, SVAF, SDX4DF) \ cpi->fn_ptr[BT].sdf = SDF; \ cpi->fn_ptr[BT].sdaf = SDAF; \ cpi->fn_ptr[BT].vf = VF; \ cpi->fn_ptr[BT].svf = SVF; \ cpi->fn_ptr[BT].svaf = SVAF; \ - cpi->fn_ptr[BT].sdx4df = SDX4DF; \ - cpi->fn_ptr[BT].sdx8f = NULL; + cpi->fn_ptr[BT].sdx4df = SDX4DF; #define MAKE_BFP_SAD_WRAPPER(fnname) \ static unsigned int fnname##_bits8(const uint8_t *src_ptr, \ int source_stride, \ const uint8_t *ref_ptr, int ref_stride) { \ @@ -2221,20 +2216,22 @@ unsigned int *subsampling_x, unsigned int *subsampling_y) { switch (fmt) { case VPX_IMG_FMT_I420: case VPX_IMG_FMT_YV12: + case VPX_IMG_FMT_NV12: case VPX_IMG_FMT_I422: case VPX_IMG_FMT_I42016: case VPX_IMG_FMT_I42216: *subsampling_x = 1; break; default: *subsampling_x = 0; break; } switch (fmt) { case VPX_IMG_FMT_I420: case VPX_IMG_FMT_I440: case VPX_IMG_FMT_YV12: + case VPX_IMG_FMT_NV12: case VPX_IMG_FMT_I42016: case VPX_IMG_FMT_I44016: *subsampling_y = 1; break; default: *subsampling_y = 0; break; } } @@ -2561,71 +2558,65 @@ // Allocate memory to store variances for a frame. CHECK_MEM_ERROR(cm, cpi->source_diff_var, vpx_calloc(cm->MBs, sizeof(diff))); cpi->source_var_thresh = 0; cpi->frames_till_next_var_check = 0; -#define BFP(BT, SDF, SDAF, VF, SVF, SVAF, SDX4DF, SDX8F) \ - cpi->fn_ptr[BT].sdf = SDF; \ - cpi->fn_ptr[BT].sdaf = SDAF; \ - cpi->fn_ptr[BT].vf = VF; \ - cpi->fn_ptr[BT].svf = SVF; \ - cpi->fn_ptr[BT].svaf = SVAF; \ - cpi->fn_ptr[BT].sdx4df = SDX4DF; \ - cpi->fn_ptr[BT].sdx8f = SDX8F; - - // TODO(angiebird): make sdx8f available for every block size +#define BFP(BT, SDF, SDAF, VF, SVF, SVAF, SDX4DF) \ + cpi->fn_ptr[BT].sdf = SDF; \ + cpi->fn_ptr[BT].sdaf = SDAF; \ + cpi->fn_ptr[BT].vf = VF; \ + cpi->fn_ptr[BT].svf = SVF; \ + cpi->fn_ptr[BT].svaf = SVAF; \ + cpi->fn_ptr[BT].sdx4df = SDX4DF; + BFP(BLOCK_32X16, vpx_sad32x16, vpx_sad32x16_avg, vpx_variance32x16, vpx_sub_pixel_variance32x16, vpx_sub_pixel_avg_variance32x16, - vpx_sad32x16x4d, NULL) + vpx_sad32x16x4d) BFP(BLOCK_16X32, vpx_sad16x32, vpx_sad16x32_avg, vpx_variance16x32, vpx_sub_pixel_variance16x32, vpx_sub_pixel_avg_variance16x32, - vpx_sad16x32x4d, NULL) + vpx_sad16x32x4d) BFP(BLOCK_64X32, vpx_sad64x32, vpx_sad64x32_avg, vpx_variance64x32, vpx_sub_pixel_variance64x32, vpx_sub_pixel_avg_variance64x32, - vpx_sad64x32x4d, NULL) + vpx_sad64x32x4d) BFP(BLOCK_32X64, vpx_sad32x64, vpx_sad32x64_avg, vpx_variance32x64, vpx_sub_pixel_variance32x64, vpx_sub_pixel_avg_variance32x64, - vpx_sad32x64x4d, NULL) + vpx_sad32x64x4d) BFP(BLOCK_32X32, vpx_sad32x32, vpx_sad32x32_avg, vpx_variance32x32, vpx_sub_pixel_variance32x32, vpx_sub_pixel_avg_variance32x32, - vpx_sad32x32x4d, vpx_sad32x32x8) + vpx_sad32x32x4d) BFP(BLOCK_64X64, vpx_sad64x64, vpx_sad64x64_avg, vpx_variance64x64, vpx_sub_pixel_variance64x64, vpx_sub_pixel_avg_variance64x64, - vpx_sad64x64x4d, NULL) + vpx_sad64x64x4d) BFP(BLOCK_16X16, vpx_sad16x16, vpx_sad16x16_avg, vpx_variance16x16, vpx_sub_pixel_variance16x16, vpx_sub_pixel_avg_variance16x16, - vpx_sad16x16x4d, vpx_sad16x16x8) + vpx_sad16x16x4d) BFP(BLOCK_16X8, vpx_sad16x8, vpx_sad16x8_avg, vpx_variance16x8, vpx_sub_pixel_variance16x8, vpx_sub_pixel_avg_variance16x8, - vpx_sad16x8x4d, vpx_sad16x8x8) + vpx_sad16x8x4d) BFP(BLOCK_8X16, vpx_sad8x16, vpx_sad8x16_avg, vpx_variance8x16, vpx_sub_pixel_variance8x16, vpx_sub_pixel_avg_variance8x16, - vpx_sad8x16x4d, vpx_sad8x16x8) + vpx_sad8x16x4d) BFP(BLOCK_8X8, vpx_sad8x8, vpx_sad8x8_avg, vpx_variance8x8, - vpx_sub_pixel_variance8x8, vpx_sub_pixel_avg_variance8x8, vpx_sad8x8x4d, - vpx_sad8x8x8) + vpx_sub_pixel_variance8x8, vpx_sub_pixel_avg_variance8x8, vpx_sad8x8x4d) BFP(BLOCK_8X4, vpx_sad8x4, vpx_sad8x4_avg, vpx_variance8x4, - vpx_sub_pixel_variance8x4, vpx_sub_pixel_avg_variance8x4, vpx_sad8x4x4d, - NULL) + vpx_sub_pixel_variance8x4, vpx_sub_pixel_avg_variance8x4, vpx_sad8x4x4d) BFP(BLOCK_4X8, vpx_sad4x8, vpx_sad4x8_avg, vpx_variance4x8, - vpx_sub_pixel_variance4x8, vpx_sub_pixel_avg_variance4x8, vpx_sad4x8x4d, - NULL) + vpx_sub_pixel_variance4x8, vpx_sub_pixel_avg_variance4x8, vpx_sad4x8x4d) BFP(BLOCK_4X4, vpx_sad4x4, vpx_sad4x4_avg, vpx_variance4x4, - vpx_sub_pixel_variance4x4, vpx_sub_pixel_avg_variance4x4, vpx_sad4x4x4d, - vpx_sad4x4x8) + vpx_sub_pixel_variance4x4, vpx_sub_pixel_avg_variance4x4, vpx_sad4x4x4d) #if CONFIG_VP9_HIGHBITDEPTH highbd_set_var_fns(cpi); #endif @@ -2674,11 +2665,10 @@ static void free_tpl_buffer(VP9_COMP *cpi); void vp9_remove_compressor(VP9_COMP *cpi) { VP9_COMMON *cm; unsigned int i; - int t; if (!cpi) return; #if CONFIG_INTERNAL_STATS vpx_free(cpi->ssim_vars); @@ -2787,32 +2777,14 @@ vpx_free(cpi->kmeans_data_arr); } free_tpl_buffer(cpi); - for (t = 0; t < cpi->num_workers; ++t) { - VPxWorker *const worker = &cpi->workers[t]; - EncWorkerData *const thread_data = &cpi->tile_thr_data[t]; - - // Deallocate allocated threads. - vpx_get_worker_interface()->end(worker); - - // Deallocate allocated thread data. - if (t < cpi->num_workers - 1) { - vpx_free(thread_data->td->counts); - vp9_free_pc_tree(thread_data->td); - vpx_free(thread_data->td); - } - } - vpx_free(cpi->tile_thr_data); - vpx_free(cpi->workers); + vp9_loop_filter_dealloc(&cpi->lf_row_sync); + vp9_bitstream_encode_tiles_buffer_dealloc(cpi); vp9_row_mt_mem_dealloc(cpi); - - if (cpi->num_workers > 1) { - vp9_loop_filter_dealloc(&cpi->lf_row_sync); - vp9_bitstream_encode_tiles_buffer_dealloc(cpi); - } + vp9_encode_free_mt_data(cpi); #if !CONFIG_REALTIME_ONLY vp9_alt_ref_aq_destroy(cpi->alt_ref_aq); #endif @@ -3710,13 +3682,13 @@ case 4: case 5: l = 100; break; case 6: l = 150; break; } if (!cpi->common.postproc_state.limits) { - cpi->common.postproc_state.limits = - vpx_calloc(cpi->un_scaled_source->y_width, - sizeof(*cpi->common.postproc_state.limits)); + CHECK_MEM_ERROR(cm, cpi->common.postproc_state.limits, + vpx_calloc(cpi->un_scaled_source->y_width, + sizeof(*cpi->common.postproc_state.limits))); } vp9_denoise(&cpi->common, cpi->Source, cpi->Source, l, cpi->common.postproc_state.limits); } #endif // CONFIG_VP9_POSTPROC @@ -4135,15 +4107,26 @@ // it may be pretty bad for rate-control, // and I should handle it somehow vp9_alt_ref_aq_setup_map(cpi->alt_ref_aq, cpi); } else { #endif - if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ) { + // If ROI is enabled and skip feature is used for segmentation, apply cyclic + // refresh but not apply ROI for skip for the first 20 frames (defined by + // FRAMES_NO_SKIPPING_AFTER_KEY) after key frame to improve quality. + if (cpi->roi.enabled && !frame_is_intra_only(cm)) { + if (cpi->roi.skip[BACKGROUND_SEG_SKIP_ID]) { + if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ) + vp9_cyclic_refresh_setup(cpi); + if (cpi->rc.frames_since_key > FRAMES_NO_SKIPPING_AFTER_KEY) + apply_roi_map(cpi); + } else { + apply_roi_map(cpi); + } + } else if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ) { vp9_cyclic_refresh_setup(cpi); - } else if (cpi->roi.enabled && !frame_is_intra_only(cm)) { - apply_roi_map(cpi); - } + } + #if !CONFIG_REALTIME_ONLY } #endif apply_active_map(cpi); @@ -6628,23 +6611,26 @@ const scan_order *const scan_order = &vp9_default_scan_orders[tx_size]; uint16_t eob; int pix_num = 1 << num_pels_log2_lookup[txsize_to_bsize[tx_size]]; const int shift = tx_size == TX_32X32 ? 0 : 2; + // skip block condition should be handled before this is called. + assert(!x->skip_block); + #if CONFIG_VP9_HIGHBITDEPTH if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { - vp9_highbd_quantize_fp_32x32(coeff, pix_num, x->skip_block, p->round_fp, - p->quant_fp, qcoeff, dqcoeff, pd->dequant, - &eob, scan_order->scan, scan_order->iscan); + vp9_highbd_quantize_fp_32x32(coeff, pix_num, p->round_fp, p->quant_fp, + qcoeff, dqcoeff, pd->dequant, &eob, + scan_order->scan, scan_order->iscan); } else { - vp9_quantize_fp_32x32(coeff, pix_num, x->skip_block, p->round_fp, - p->quant_fp, qcoeff, dqcoeff, pd->dequant, &eob, - scan_order->scan, scan_order->iscan); + vp9_quantize_fp_32x32(coeff, pix_num, p->round_fp, p->quant_fp, qcoeff, + dqcoeff, pd->dequant, &eob, scan_order->scan, + scan_order->iscan); } #else - vp9_quantize_fp_32x32(coeff, pix_num, x->skip_block, p->round_fp, p->quant_fp, - qcoeff, dqcoeff, pd->dequant, &eob, scan_order->scan, + vp9_quantize_fp_32x32(coeff, pix_num, p->round_fp, p->quant_fp, qcoeff, + dqcoeff, pd->dequant, &eob, scan_order->scan, scan_order->iscan); #endif // CONFIG_VP9_HIGHBITDEPTH *recon_error = vp9_block_error(coeff, dqcoeff, pix_num, sse) >> shift; *recon_error = VPXMAX(*recon_error, 1); diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_ethread.h b/media/libvpx/libvpx/vp9/encoder/vp9_ethread.h --- a/media/libvpx/libvpx/vp9/encoder/vp9_ethread.h +++ b/media/libvpx/libvpx/vp9/encoder/vp9_ethread.h @@ -40,10 +40,15 @@ int *cur_col; int sync_range; int rows; } VP9RowMTSync; +// Frees EncWorkerData related allocations made by vp9_encode_*_mt(). +// row_mt specific data is freed with vp9_row_mt_mem_dealloc() and is not +// called by this function. +void vp9_encode_free_mt_data(struct VP9_COMP *cpi); + void vp9_encode_tiles_mt(struct VP9_COMP *cpi); void vp9_encode_tiles_row_mt(struct VP9_COMP *cpi); void vp9_encode_fp_row_mt(struct VP9_COMP *cpi); diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_ethread.c b/media/libvpx/libvpx/vp9/encoder/vp9_ethread.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_ethread.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_ethread.c @@ -6,10 +6,12 @@ * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ +#include "vp9/common/vp9_thread_common.h" +#include "vp9/encoder/vp9_bitstream.h" #include "vp9/encoder/vp9_encodeframe.h" #include "vp9/encoder/vp9_encoder.h" #include "vp9/encoder/vp9_ethread.h" #include "vp9/encoder/vp9_firstpass.h" #include "vp9/encoder/vp9_multi_thread.h" @@ -77,64 +79,63 @@ static void create_enc_workers(VP9_COMP *cpi, int num_workers) { VP9_COMMON *const cm = &cpi->common; const VPxWorkerInterface *const winterface = vpx_get_worker_interface(); int i; - - // Only run once to create threads and allocate thread data. - if (cpi->num_workers == 0) { - int allocated_workers = num_workers; - - // While using SVC, we need to allocate threads according to the highest - // resolution. When row based multithreading is enabled, it is OK to - // allocate more threads than the number of max tile columns. - if (cpi->use_svc && !cpi->row_mt) { - int max_tile_cols = get_max_tile_cols(cpi); - allocated_workers = VPXMIN(cpi->oxcf.max_threads, max_tile_cols); - } + // While using SVC, we need to allocate threads according to the highest + // resolution. When row based multithreading is enabled, it is OK to + // allocate more threads than the number of max tile columns. + if (cpi->use_svc && !cpi->row_mt) { + int max_tile_cols = get_max_tile_cols(cpi); + num_workers = VPXMIN(cpi->oxcf.max_threads, max_tile_cols); + } + assert(num_workers > 0); + if (num_workers == cpi->num_workers) return; + vp9_loop_filter_dealloc(&cpi->lf_row_sync); + vp9_bitstream_encode_tiles_buffer_dealloc(cpi); + vp9_encode_free_mt_data(cpi); - CHECK_MEM_ERROR(cm, cpi->workers, - vpx_malloc(allocated_workers * sizeof(*cpi->workers))); + CHECK_MEM_ERROR(cm, cpi->workers, + vpx_malloc(num_workers * sizeof(*cpi->workers))); - CHECK_MEM_ERROR(cm, cpi->tile_thr_data, - vpx_calloc(allocated_workers, sizeof(*cpi->tile_thr_data))); + CHECK_MEM_ERROR(cm, cpi->tile_thr_data, + vpx_calloc(num_workers, sizeof(*cpi->tile_thr_data))); - for (i = 0; i < allocated_workers; i++) { - VPxWorker *const worker = &cpi->workers[i]; - EncWorkerData *thread_data = &cpi->tile_thr_data[i]; + for (i = 0; i < num_workers; i++) { + VPxWorker *const worker = &cpi->workers[i]; + EncWorkerData *thread_data = &cpi->tile_thr_data[i]; - ++cpi->num_workers; - winterface->init(worker); + ++cpi->num_workers; + winterface->init(worker); - if (i < allocated_workers - 1) { - thread_data->cpi = cpi; + if (i < num_workers - 1) { + thread_data->cpi = cpi; - // Allocate thread data. - CHECK_MEM_ERROR(cm, thread_data->td, - vpx_memalign(32, sizeof(*thread_data->td))); - vp9_zero(*thread_data->td); + // Allocate thread data. + CHECK_MEM_ERROR(cm, thread_data->td, + vpx_memalign(32, sizeof(*thread_data->td))); + vp9_zero(*thread_data->td); - // Set up pc_tree. - thread_data->td->leaf_tree = NULL; - thread_data->td->pc_tree = NULL; - vp9_setup_pc_tree(cm, thread_data->td); + // Set up pc_tree. + thread_data->td->leaf_tree = NULL; + thread_data->td->pc_tree = NULL; + vp9_setup_pc_tree(cm, thread_data->td); - // Allocate frame counters in thread data. - CHECK_MEM_ERROR(cm, thread_data->td->counts, - vpx_calloc(1, sizeof(*thread_data->td->counts))); + // Allocate frame counters in thread data. + CHECK_MEM_ERROR(cm, thread_data->td->counts, + vpx_calloc(1, sizeof(*thread_data->td->counts))); - // Create threads - if (!winterface->reset(worker)) - vpx_internal_error(&cm->error, VPX_CODEC_ERROR, - "Tile encoder thread creation failed"); - } else { - // Main thread acts as a worker and uses the thread data in cpi. - thread_data->cpi = cpi; - thread_data->td = &cpi->td; - } - winterface->sync(worker); + // Create threads + if (!winterface->reset(worker)) + vpx_internal_error(&cm->error, VPX_CODEC_ERROR, + "Tile encoder thread creation failed"); + } else { + // Main thread acts as a worker and uses the thread data in cpi. + thread_data->cpi = cpi; + thread_data->td = &cpi->td; } + winterface->sync(worker); } } static void launch_enc_workers(VP9_COMP *cpi, VPxWorkerHook hook, void *data2, int num_workers) { @@ -167,10 +168,31 @@ VPxWorker *const worker = &cpi->workers[i]; winterface->sync(worker); } } +void vp9_encode_free_mt_data(struct VP9_COMP *cpi) { + int t; + for (t = 0; t < cpi->num_workers; ++t) { + VPxWorker *const worker = &cpi->workers[t]; + EncWorkerData *const thread_data = &cpi->tile_thr_data[t]; + + // Deallocate allocated threads. + vpx_get_worker_interface()->end(worker); + + // Deallocate allocated thread data. + if (t < cpi->num_workers - 1) { + vpx_free(thread_data->td->counts); + vp9_free_pc_tree(thread_data->td); + vpx_free(thread_data->td); + } + } + vpx_free(cpi->tile_thr_data); + vpx_free(cpi->workers); + cpi->num_workers = 0; +} + void vp9_encode_tiles_mt(VP9_COMP *cpi) { VP9_COMMON *const cm = &cpi->common; const int tile_cols = 1 << cm->log2_tile_cols; const int num_workers = VPXMIN(cpi->oxcf.max_threads, tile_cols); int i; diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_mbgraph.c b/media/libvpx/libvpx/vp9/encoder/vp9_mbgraph.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_mbgraph.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_mbgraph.c @@ -217,11 +217,11 @@ MACROBLOCK *const x = &cpi->td.mb; MACROBLOCKD *const xd = &x->e_mbd; VP9_COMMON *const cm = &cpi->common; int mb_col, mb_row, offset = 0; - int mb_y_offset = 0, arf_y_offset = 0, gld_y_offset = 0; + int mb_y_offset = 0; MV gld_top_mv = { 0, 0 }; MODE_INFO mi_local; MODE_INFO mi_above, mi_left; vp9_zero(mi_local); @@ -241,12 +241,10 @@ mi_local.ref_frame[1] = NONE; for (mb_row = 0; mb_row < cm->mb_rows; mb_row++) { MV gld_left_mv = gld_top_mv; int mb_y_in_offset = mb_y_offset; - int arf_y_in_offset = arf_y_offset; - int gld_y_in_offset = gld_y_offset; // Set up limit values for motion vectors to prevent them extending outside // the UMV borders. x->mv_limits.col_min = -BORDER_MV_PIXELS_B16; x->mv_limits.col_max = (cm->mb_cols - 1) * 8 + BORDER_MV_PIXELS_B16; @@ -264,22 +262,18 @@ } // Signal to vp9_predict_intra_block() that left is available xd->left_mi = &mi_left; mb_y_in_offset += 16; - gld_y_in_offset += 16; - arf_y_in_offset += 16; x->mv_limits.col_min -= 16; x->mv_limits.col_max -= 16; } // Signal to vp9_predict_intra_block() that above is available xd->above_mi = &mi_above; mb_y_offset += buf->y_stride * 16; - gld_y_offset += golden_ref->y_stride * 16; - if (alt_ref) arf_y_offset += alt_ref->y_stride * 16; x->mv_limits.row_min -= 16; x->mv_limits.row_max -= 16; offset += cm->mb_cols; } } diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_mcomp.h b/media/libvpx/libvpx/vp9/encoder/vp9_mcomp.h --- a/media/libvpx/libvpx/vp9/encoder/vp9_mcomp.h +++ b/media/libvpx/libvpx/vp9/encoder/vp9_mcomp.h @@ -91,20 +91,10 @@ extern fractional_mv_step_fp vp9_find_best_sub_pixel_tree_pruned_evenmore; extern fractional_mv_step_fp vp9_skip_sub_pixel_tree; extern fractional_mv_step_fp vp9_return_max_sub_pixel_mv; extern fractional_mv_step_fp vp9_return_min_sub_pixel_mv; -typedef int (*vp9_full_search_fn_t)(const MACROBLOCK *x, const MV *ref_mv, - int sad_per_bit, int distance, - const vp9_variance_fn_ptr_t *fn_ptr, - const MV *center_mv, MV *best_mv); - -typedef int (*vp9_refining_search_fn_t)(const MACROBLOCK *x, MV *ref_mv, - int sad_per_bit, int distance, - const vp9_variance_fn_ptr_t *fn_ptr, - const MV *center_mv); - typedef int (*vp9_diamond_search_fn_t)( const MACROBLOCK *x, const search_site_config *cfg, MV *ref_mv, MV *best_mv, int search_param, int sad_per_bit, int *num00, const vp9_variance_fn_ptr_t *fn_ptr, const MV *center_mv); diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_mcomp.c b/media/libvpx/libvpx/vp9/encoder/vp9_mcomp.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_mcomp.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_mcomp.c @@ -157,63 +157,67 @@ return &buf[(r >> 3) * stride + (c >> 3)]; } #if CONFIG_VP9_HIGHBITDEPTH /* checks if (r, c) has better score than previous best */ -#define CHECK_BETTER(v, r, c) \ - if (c >= minc && c <= maxc && r >= minr && r <= maxr) { \ - int64_t tmpmse; \ - const MV mv = { r, c }; \ - const MV ref_mv = { rr, rc }; \ - if (second_pred == NULL) { \ - thismse = vfp->svf(pre(y, y_stride, r, c), y_stride, sp(c), sp(r), z, \ - src_stride, &sse); \ - } else { \ - thismse = vfp->svaf(pre(y, y_stride, r, c), y_stride, sp(c), sp(r), z, \ - src_stride, &sse, second_pred); \ - } \ - tmpmse = thismse; \ - tmpmse += mv_err_cost(&mv, &ref_mv, mvjcost, mvcost, error_per_bit); \ - if (tmpmse >= INT_MAX) { \ - v = INT_MAX; \ - } else if ((v = (uint32_t)tmpmse) < besterr) { \ - besterr = v; \ - br = r; \ - bc = c; \ - *distortion = thismse; \ - *sse1 = sse; \ - } \ - } else { \ - v = INT_MAX; \ - } +#define CHECK_BETTER(v, r, c) \ + do { \ + if (c >= minc && c <= maxc && r >= minr && r <= maxr) { \ + int64_t tmpmse; \ + const MV mv = { r, c }; \ + const MV ref_mv = { rr, rc }; \ + if (second_pred == NULL) { \ + thismse = vfp->svf(pre(y, y_stride, r, c), y_stride, sp(c), sp(r), z, \ + src_stride, &sse); \ + } else { \ + thismse = vfp->svaf(pre(y, y_stride, r, c), y_stride, sp(c), sp(r), z, \ + src_stride, &sse, second_pred); \ + } \ + tmpmse = thismse; \ + tmpmse += mv_err_cost(&mv, &ref_mv, mvjcost, mvcost, error_per_bit); \ + if (tmpmse >= INT_MAX) { \ + v = INT_MAX; \ + } else if ((v = (uint32_t)tmpmse) < besterr) { \ + besterr = v; \ + br = r; \ + bc = c; \ + *distortion = thismse; \ + *sse1 = sse; \ + } \ + } else { \ + v = INT_MAX; \ + } \ + } while (0) #else /* checks if (r, c) has better score than previous best */ -#define CHECK_BETTER(v, r, c) \ - if (c >= minc && c <= maxc && r >= minr && r <= maxr) { \ - const MV mv = { r, c }; \ - const MV ref_mv = { rr, rc }; \ - if (second_pred == NULL) \ - thismse = vfp->svf(pre(y, y_stride, r, c), y_stride, sp(c), sp(r), z, \ - src_stride, &sse); \ - else \ - thismse = vfp->svaf(pre(y, y_stride, r, c), y_stride, sp(c), sp(r), z, \ - src_stride, &sse, second_pred); \ - if ((v = mv_err_cost(&mv, &ref_mv, mvjcost, mvcost, error_per_bit) + \ - thismse) < besterr) { \ - besterr = v; \ - br = r; \ - bc = c; \ - *distortion = thismse; \ - *sse1 = sse; \ - } \ - } else { \ - v = INT_MAX; \ - } +#define CHECK_BETTER(v, r, c) \ + do { \ + if (c >= minc && c <= maxc && r >= minr && r <= maxr) { \ + const MV mv = { r, c }; \ + const MV ref_mv = { rr, rc }; \ + if (second_pred == NULL) \ + thismse = vfp->svf(pre(y, y_stride, r, c), y_stride, sp(c), sp(r), z, \ + src_stride, &sse); \ + else \ + thismse = vfp->svaf(pre(y, y_stride, r, c), y_stride, sp(c), sp(r), z, \ + src_stride, &sse, second_pred); \ + if ((v = mv_err_cost(&mv, &ref_mv, mvjcost, mvcost, error_per_bit) + \ + thismse) < besterr) { \ + besterr = v; \ + br = r; \ + bc = c; \ + *distortion = thismse; \ + *sse1 = sse; \ + } \ + } else { \ + v = INT_MAX; \ + } \ + } while (0) #endif #define FIRST_LEVEL_CHECKS \ - { \ + do { \ unsigned int left, right, up, down, diag; \ CHECK_BETTER(left, tr, tc - hstep); \ CHECK_BETTER(right, tr, tc + hstep); \ CHECK_BETTER(up, tr - hstep, tc); \ CHECK_BETTER(down, tr + hstep, tc); \ @@ -222,14 +226,14 @@ case 0: CHECK_BETTER(diag, tr - hstep, tc - hstep); break; \ case 1: CHECK_BETTER(diag, tr - hstep, tc + hstep); break; \ case 2: CHECK_BETTER(diag, tr + hstep, tc - hstep); break; \ case 3: CHECK_BETTER(diag, tr + hstep, tc + hstep); break; \ } \ - } + } while (0) #define SECOND_LEVEL_CHECKS \ - { \ + do { \ int kr, kc; \ unsigned int second; \ if (tr != br && tc != bc) { \ kr = br - tr; \ kc = bc - tc; \ @@ -254,11 +258,11 @@ case 2: CHECK_BETTER(second, tr + kr, tc + hstep); break; \ case 1: \ case 3: CHECK_BETTER(second, tr + kr, tc - hstep); break; \ } \ } \ - } + } while (0) #define SETUP_SUBPEL_SEARCH \ const uint8_t *const z = x->plane[0].src.buf; \ const int src_stride = x->plane[0].src.stride; \ const MACROBLOCKD *xd = &x->e_mbd; \ @@ -288,11 +292,11 @@ maxc = subpel_mv_limits.col_max; \ minr = subpel_mv_limits.row_min; \ maxr = subpel_mv_limits.row_max; \ \ bestmv->row *= 8; \ - bestmv->col *= 8; + bestmv->col *= 8 static unsigned int setup_center_error( const MACROBLOCKD *xd, const MV *bestmv, const MV *ref_mv, int error_per_bit, const vp9_variance_fn_ptr_t *vfp, const uint8_t *const src, const int src_stride, const uint8_t *const y, @@ -676,52 +680,56 @@ } // TODO(yunqing): this part can be further refactored. #if CONFIG_VP9_HIGHBITDEPTH /* checks if (r, c) has better score than previous best */ -#define CHECK_BETTER1(v, r, c) \ - if (c >= minc && c <= maxc && r >= minr && r <= maxr) { \ - int64_t tmpmse; \ - const MV mv = { r, c }; \ - const MV ref_mv = { rr, rc }; \ - thismse = \ - accurate_sub_pel_search(xd, &mv, x->me_sf, kernel, vfp, z, src_stride, \ - y, y_stride, second_pred, w, h, &sse); \ - tmpmse = thismse; \ - tmpmse += mv_err_cost(&mv, &ref_mv, mvjcost, mvcost, error_per_bit); \ - if (tmpmse >= INT_MAX) { \ - v = INT_MAX; \ - } else if ((v = (uint32_t)tmpmse) < besterr) { \ - besterr = v; \ - br = r; \ - bc = c; \ - *distortion = thismse; \ - *sse1 = sse; \ - } \ - } else { \ - v = INT_MAX; \ - } +#define CHECK_BETTER1(v, r, c) \ + do { \ + if (c >= minc && c <= maxc && r >= minr && r <= maxr) { \ + int64_t tmpmse; \ + const MV mv = { r, c }; \ + const MV ref_mv = { rr, rc }; \ + thismse = accurate_sub_pel_search(xd, &mv, x->me_sf, kernel, vfp, z, \ + src_stride, y, y_stride, second_pred, \ + w, h, &sse); \ + tmpmse = thismse; \ + tmpmse += mv_err_cost(&mv, &ref_mv, mvjcost, mvcost, error_per_bit); \ + if (tmpmse >= INT_MAX) { \ + v = INT_MAX; \ + } else if ((v = (uint32_t)tmpmse) < besterr) { \ + besterr = v; \ + br = r; \ + bc = c; \ + *distortion = thismse; \ + *sse1 = sse; \ + } \ + } else { \ + v = INT_MAX; \ + } \ + } while (0) #else /* checks if (r, c) has better score than previous best */ -#define CHECK_BETTER1(v, r, c) \ - if (c >= minc && c <= maxc && r >= minr && r <= maxr) { \ - const MV mv = { r, c }; \ - const MV ref_mv = { rr, rc }; \ - thismse = \ - accurate_sub_pel_search(xd, &mv, x->me_sf, kernel, vfp, z, src_stride, \ - y, y_stride, second_pred, w, h, &sse); \ - if ((v = mv_err_cost(&mv, &ref_mv, mvjcost, mvcost, error_per_bit) + \ - thismse) < besterr) { \ - besterr = v; \ - br = r; \ - bc = c; \ - *distortion = thismse; \ - *sse1 = sse; \ - } \ - } else { \ - v = INT_MAX; \ - } +#define CHECK_BETTER1(v, r, c) \ + do { \ + if (c >= minc && c <= maxc && r >= minr && r <= maxr) { \ + const MV mv = { r, c }; \ + const MV ref_mv = { rr, rc }; \ + thismse = accurate_sub_pel_search(xd, &mv, x->me_sf, kernel, vfp, z, \ + src_stride, y, y_stride, second_pred, \ + w, h, &sse); \ + if ((v = mv_err_cost(&mv, &ref_mv, mvjcost, mvcost, error_per_bit) + \ + thismse) < besterr) { \ + besterr = v; \ + br = r; \ + bc = c; \ + *distortion = thismse; \ + *sse1 = sse; \ + } \ + } else { \ + v = INT_MAX; \ + } \ + } while (0) #endif uint32_t vp9_find_best_sub_pixel_tree( const MACROBLOCK *x, MV *bestmv, const MV *ref_mv, int allow_hp, @@ -1786,33 +1794,10 @@ start_col = VPXMAX(center_mv->col - range, mv_limits->col_min); end_row = VPXMIN(center_mv->row + range, mv_limits->row_max); end_col = VPXMIN(center_mv->col + range, mv_limits->col_max); for (r = start_row; r <= end_row; r += 1) { c = start_col; - // sdx8f may not be available some block size - if (fn_ptr->sdx8f) { - while (c + 7 <= end_col) { - unsigned int sads[8]; - const MV mv = { r, c }; - const uint8_t *buf = get_buf_from_mv(pre, &mv); - fn_ptr->sdx8f(src->buf, src->stride, buf, pre->stride, sads); - - for (i = 0; i < 8; ++i) { - int64_t sad = (int64_t)sads[i] << LOG2_PRECISION; - if (sad < best_sad) { - const MV mv = { r, c + i }; - sad += lambda * - vp9_nb_mvs_inconsistency(&mv, nb_full_mvs, full_mv_num); - if (sad < best_sad) { - best_sad = sad; - *best_mv = mv; - } - } - } - c += 8; - } - } while (c + 3 <= end_col) { unsigned int sads[4]; const uint8_t *addrs[4]; for (i = 0; i < 4; ++i) { const MV mv = { r, c + i }; @@ -2960,11 +2945,11 @@ (void)tr; \ (void)tc; \ (void)sse; \ (void)thismse; \ (void)cost_list; \ - (void)use_accurate_subpel_search; + (void)use_accurate_subpel_search // Return the maximum MV. uint32_t vp9_return_max_sub_pixel_mv( const MACROBLOCK *x, MV *bestmv, const MV *ref_mv, int allow_hp, int error_per_bit, const vp9_variance_fn_ptr_t *vfp, int forced_stop, diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_multi_thread.c b/media/libvpx/libvpx/vp9/encoder/vp9_multi_thread.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_multi_thread.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_multi_thread.c @@ -34,11 +34,11 @@ // lock the mutex for queue access #if CONFIG_MULTITHREAD pthread_mutex_lock(mutex_handle); #endif next = job_queue_hdl->next; - if (NULL != next) { + if (next != NULL) { JobQueue *job_queue = (JobQueue *)next; job_info = &job_queue->job_info; // Update the next job in the queue job_queue_hdl->next = job_queue->next; job_queue_hdl->num_jobs_acquired++; @@ -56,13 +56,14 @@ VP9_COMMON *const cm = &cpi->common; const int sb_rows = (mi_cols_aligned_to_sb(cm->mi_rows) >> MI_BLOCK_SIZE_LOG2) + 1; int i; - this_tile->row_base_thresh_freq_fact = + CHECK_MEM_ERROR( + cm, this_tile->row_base_thresh_freq_fact, (int *)vpx_calloc(sb_rows * BLOCK_SIZES * MAX_MODES, - sizeof(*(this_tile->row_base_thresh_freq_fact))); + sizeof(*(this_tile->row_base_thresh_freq_fact)))); for (i = 0; i < sb_rows * BLOCK_SIZES * MAX_MODES; i++) this_tile->row_base_thresh_freq_fact[i] = RD_THRESH_INIT_FACT; } void vp9_row_mt_mem_alloc(VP9_COMP *cpi) { @@ -82,12 +83,12 @@ multi_thread_ctxt->allocated_tile_cols = tile_cols; multi_thread_ctxt->allocated_tile_rows = tile_rows; multi_thread_ctxt->allocated_vert_unit_rows = jobs_per_tile_col; - multi_thread_ctxt->job_queue = - (JobQueue *)vpx_memalign(32, total_jobs * sizeof(JobQueue)); + CHECK_MEM_ERROR(cm, multi_thread_ctxt->job_queue, + (JobQueue *)vpx_memalign(32, total_jobs * sizeof(JobQueue))); #if CONFIG_MULTITHREAD // Create mutex for each tile for (tile_col = 0; tile_col < tile_cols; tile_col++) { RowMTInfo *row_mt_info = &multi_thread_ctxt->row_mt_info[tile_col]; diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_non_greedy_mv.c b/media/libvpx/libvpx/vp9/encoder/vp9_non_greedy_mv.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_non_greedy_mv.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_non_greedy_mv.c @@ -176,10 +176,11 @@ vp9_free_motion_field_info(motion_field_info); } motion_field_info->frame_num = frame_num; motion_field_info->motion_field_array = vpx_calloc(frame_num, sizeof(*motion_field_info->motion_field_array)); + if (!motion_field_info->motion_field_array) return STATUS_FAILED; for (frame_idx = 0; frame_idx < frame_num; ++frame_idx) { for (rf_idx = 0; rf_idx < MAX_INTER_REF_FRAMES; ++rf_idx) { for (square_block_idx = 0; square_block_idx < SQUARE_BLOCK_SIZES; ++square_block_idx) { BLOCK_SIZE bsize = square_block_idx_to_bsize(square_block_idx); @@ -420,10 +421,11 @@ FloatMV *output = (FloatMV *)malloc(rows * cols * sizeof(FloatMV)); int idx; int row, col; int bw = 4 << b_width_log2_lookup[bsize]; int bh = 4 << b_height_log2_lookup[bsize]; + if (!(input && output)) goto fail; // copy search results to input buffer for (idx = 0; idx < rows * cols; ++idx) { input[idx].row = (float)search_mf[idx].row / bh; input[idx].col = (float)search_mf[idx].col / bw; } @@ -448,10 +450,11 @@ // copy smoothed results to output for (idx = 0; idx < rows * cols; ++idx) { smooth_mf[idx].row = (int)(input[idx].row * bh); smooth_mf[idx].col = (int)(input[idx].col * bw); } +fail: free(input); free(output); } void vp9_get_local_structure(const YV12_BUFFER_CONFIG *cur_frame, diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_pickmode.c b/media/libvpx/libvpx/vp9/encoder/vp9_pickmode.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_pickmode.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_pickmode.c @@ -769,28 +769,31 @@ uint16_t *const eob = &p->eobs[block]; const int diff_stride = bw; const int16_t *src_diff; src_diff = &p->src_diff[(r * diff_stride + c) << 2]; + // skip block condition should be handled before this is called. + assert(!x->skip_block); + switch (tx_size) { case TX_16X16: vpx_hadamard_16x16(src_diff, diff_stride, coeff); - vp9_quantize_fp(coeff, 256, x->skip_block, p->round_fp, p->quant_fp, - qcoeff, dqcoeff, pd->dequant, eob, scan_order->scan, + vp9_quantize_fp(coeff, 256, p->round_fp, p->quant_fp, qcoeff, + dqcoeff, pd->dequant, eob, scan_order->scan, scan_order->iscan); break; case TX_8X8: vpx_hadamard_8x8(src_diff, diff_stride, coeff); - vp9_quantize_fp(coeff, 64, x->skip_block, p->round_fp, p->quant_fp, - qcoeff, dqcoeff, pd->dequant, eob, scan_order->scan, + vp9_quantize_fp(coeff, 64, p->round_fp, p->quant_fp, qcoeff, + dqcoeff, pd->dequant, eob, scan_order->scan, scan_order->iscan); break; default: assert(tx_size == TX_4X4); x->fwd_txfm4x4(src_diff, coeff, diff_stride); - vp9_quantize_fp(coeff, 16, x->skip_block, p->round_fp, p->quant_fp, - qcoeff, dqcoeff, pd->dequant, eob, scan_order->scan, + vp9_quantize_fp(coeff, 16, p->round_fp, p->quant_fp, qcoeff, + dqcoeff, pd->dequant, eob, scan_order->scan, scan_order->iscan); break; } *skippable &= (*eob == 0); eob_cost += 1; @@ -1245,11 +1248,11 @@ static INLINE void find_predictors( VP9_COMP *cpi, MACROBLOCK *x, MV_REFERENCE_FRAME ref_frame, int_mv frame_mv[MB_MODE_COUNT][MAX_REF_FRAMES], int const_motion[MAX_REF_FRAMES], int *ref_frame_skip_mask, - const int flag_list[4], TileDataEnc *tile_data, int mi_row, int mi_col, + TileDataEnc *tile_data, int mi_row, int mi_col, struct buf_2d yv12_mb[4][MAX_MB_PLANE], BLOCK_SIZE bsize, int force_skip_low_temp_var, int comp_pred_allowed) { VP9_COMMON *const cm = &cpi->common; MACROBLOCKD *const xd = &x->e_mbd; const YV12_BUFFER_CONFIG *yv12 = get_ref_frame_buffer(cpi, ref_frame); @@ -1257,11 +1260,11 @@ // TODO(jingning) placeholder for inter-frame non-RD mode decision. x->pred_mv_sad[ref_frame] = INT_MAX; frame_mv[NEWMV][ref_frame].as_int = INVALID_MV; frame_mv[ZEROMV][ref_frame].as_int = 0; // this needs various further optimizations. to be continued.. - if ((cpi->ref_frame_flags & flag_list[ref_frame]) && (yv12 != NULL)) { + if ((cpi->ref_frame_flags & ref_frame_to_flag(ref_frame)) && (yv12 != NULL)) { int_mv *const candidates = x->mbmi_ext->ref_mvs[ref_frame]; const struct scale_factors *const sf = &cm->frame_refs[ref_frame - 1].sf; vp9_setup_pred_block(xd, yv12_mb[ref_frame], yv12, mi_row, mi_col, sf, sf); if (cm->use_prev_frame_mvs || comp_pred_allowed) { vp9_find_mv_refs(cm, xd, xd->mi[0], ref_frame, candidates, mi_row, mi_col, @@ -1688,12 +1691,10 @@ MV_REFERENCE_FRAME ref_frame; MV_REFERENCE_FRAME usable_ref_frame, second_ref_frame; int_mv frame_mv[MB_MODE_COUNT][MAX_REF_FRAMES]; uint8_t mode_checked[MB_MODE_COUNT][MAX_REF_FRAMES]; struct buf_2d yv12_mb[4][MAX_MB_PLANE]; - static const int flag_list[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG, - VP9_ALT_FLAG }; RD_COST this_rdc, best_rdc; // var_y and sse_y are saved to be used in skipping checking unsigned int var_y = UINT_MAX; unsigned int sse_y = UINT_MAX; const int intra_cost_penalty = @@ -1923,18 +1924,18 @@ // For svc mode, on spatial_layer_id > 0: if the reference has different scale // constrain the inter mode to only test zero motion. if (cpi->use_svc && svc->force_zero_mode_spatial_ref && svc->spatial_layer_id > 0 && !gf_temporal_ref) { - if (cpi->ref_frame_flags & flag_list[LAST_FRAME]) { + if (cpi->ref_frame_flags & VP9_LAST_FLAG) { struct scale_factors *const sf = &cm->frame_refs[LAST_FRAME - 1].sf; if (vp9_is_scaled(sf)) { svc_force_zero_mode[LAST_FRAME - 1] = 1; inter_layer_ref = LAST_FRAME; } } - if (cpi->ref_frame_flags & flag_list[GOLDEN_FRAME]) { + if (cpi->ref_frame_flags & VP9_GOLD_FLAG) { struct scale_factors *const sf = &cm->frame_refs[GOLDEN_FRAME - 1].sf; if (vp9_is_scaled(sf)) { svc_force_zero_mode[GOLDEN_FRAME - 1] = 1; inter_layer_ref = GOLDEN_FRAME; } @@ -1955,11 +1956,11 @@ if (sf->disable_golden_ref && (x->content_state_sb != kVeryHighSad || cpi->rc.avg_frame_low_motion < 60)) usable_ref_frame = LAST_FRAME; - if (!((cpi->ref_frame_flags & flag_list[GOLDEN_FRAME]) && + if (!((cpi->ref_frame_flags & VP9_GOLD_FLAG) && !svc_force_zero_mode[GOLDEN_FRAME - 1] && !force_skip_low_temp_var)) use_golden_nonzeromv = 0; if (cpi->oxcf.speed >= 8 && !cpi->use_svc && ((cpi->rc.frames_since_golden + 1) < x->last_sb_high_content || @@ -1983,16 +1984,15 @@ for (ref_frame = LAST_FRAME; ref_frame <= usable_ref_frame; ++ref_frame) { // Skip find_predictor if the reference frame is not in the // ref_frame_flags (i.e., not used as a reference for this frame). skip_ref_find_pred[ref_frame] = - !(cpi->ref_frame_flags & flag_list[ref_frame]); + !(cpi->ref_frame_flags & ref_frame_to_flag(ref_frame)); if (!skip_ref_find_pred[ref_frame]) { find_predictors(cpi, x, ref_frame, frame_mv, const_motion, - &ref_frame_skip_mask, flag_list, tile_data, mi_row, - mi_col, yv12_mb, bsize, force_skip_low_temp_var, - comp_modes > 0); + &ref_frame_skip_mask, tile_data, mi_row, mi_col, yv12_mb, + bsize, force_skip_low_temp_var, comp_modes > 0); } } if (cpi->use_svc || cpi->oxcf.speed <= 7 || bsize < BLOCK_32X32) x->sb_use_mv_part = 0; @@ -2012,11 +2012,11 @@ // For SVC with quality layers, when QP of lower layer is lower // than current layer: force check of GF-ZEROMV before early exit // due to skip flag. if (svc->spatial_layer_id > 0 && no_scaling && - (cpi->ref_frame_flags & flag_list[GOLDEN_FRAME]) && + (cpi->ref_frame_flags & VP9_GOLD_FLAG) && cm->base_qindex > svc->lower_layer_qindex + 10) force_test_gf_zeromv = 1; // For low motion content use x->sb_is_skin in addition to VeryHighSad // for setting large_block. @@ -2092,11 +2092,12 @@ } if (comp_pred) { if (!cpi->allow_comp_inter_inter) continue; // Skip compound inter modes if ARF is not available. - if (!(cpi->ref_frame_flags & flag_list[second_ref_frame])) continue; + if (!(cpi->ref_frame_flags & ref_frame_to_flag(second_ref_frame))) + continue; // Do not allow compound prediction if the segment level reference frame // feature is in use as in this case there can only be one reference. if (segfeature_active(seg, mi->segment_id, SEG_LVL_REF_FRAME)) continue; } @@ -2105,11 +2106,11 @@ if (ref_frame == GOLDEN_FRAME && cpi->oxcf.rc_mode == VPX_CBR && ((cpi->use_svc && sse_zeromv_normalized < thresh_svc_skip_golden) || (!cpi->use_svc && sse_zeromv_normalized < thresh_skip_golden))) continue; - if (!(cpi->ref_frame_flags & flag_list[ref_frame])) continue; + if (!(cpi->ref_frame_flags & ref_frame_to_flag(ref_frame))) continue; // For screen content. If zero_temp_sad source is computed: skip // non-zero motion check for stationary blocks. If the superblock is // non-stationary then for flat blocks skip the zero last check (keep golden // as it may be inter-layer reference). Otherwise (if zero_temp_sad_source @@ -2188,22 +2189,22 @@ !(frame_mv[this_mode][ref_frame].as_int == 0 && ref_frame == LAST_FRAME)) { if (usable_ref_frame < ALTREF_FRAME) { if (!force_skip_low_temp_var && usable_ref_frame > LAST_FRAME) { i = (ref_frame == LAST_FRAME) ? GOLDEN_FRAME : LAST_FRAME; - if ((cpi->ref_frame_flags & flag_list[i])) + if ((cpi->ref_frame_flags & ref_frame_to_flag(i))) if (x->pred_mv_sad[ref_frame] > (x->pred_mv_sad[i] << 1)) ref_frame_skip_mask |= (1 << ref_frame); } } else if (!cpi->rc.is_src_frame_alt_ref && !(frame_mv[this_mode][ref_frame].as_int == 0 && ref_frame == ALTREF_FRAME)) { int ref1 = (ref_frame == GOLDEN_FRAME) ? LAST_FRAME : GOLDEN_FRAME; int ref2 = (ref_frame == ALTREF_FRAME) ? LAST_FRAME : ALTREF_FRAME; - if (((cpi->ref_frame_flags & flag_list[ref1]) && + if (((cpi->ref_frame_flags & ref_frame_to_flag(ref1)) && (x->pred_mv_sad[ref_frame] > (x->pred_mv_sad[ref1] << 1))) || - ((cpi->ref_frame_flags & flag_list[ref2]) && + ((cpi->ref_frame_flags & ref_frame_to_flag(ref2)) && (x->pred_mv_sad[ref_frame] > (x->pred_mv_sad[ref2] << 1)))) ref_frame_skip_mask |= (1 << ref_frame); } } if (ref_frame_skip_mask & (1 << ref_frame)) continue; @@ -2486,11 +2487,11 @@ // temporal layer. if (svc->spatial_layer_id && !gf_temporal_ref) { perform_intra_pred = svc->temporal_layer_id == 0 || svc->layer_context[svc->temporal_layer_id].is_key_frame || - !(cpi->ref_frame_flags & flag_list[GOLDEN_FRAME]) || + !(cpi->ref_frame_flags & VP9_GOLD_FLAG) || (!svc->layer_context[svc->temporal_layer_id].is_key_frame && svc_force_zero_mode[best_pickmode.best_ref_frame - 1]); inter_mode_thresh = (inter_mode_thresh << 1) + inter_mode_thresh; } if ((cpi->oxcf.lag_in_frames > 0 && cpi->oxcf.rc_mode == VPX_VBR && @@ -2745,12 +2746,10 @@ const struct segmentation *const seg = &cm->seg; MV_REFERENCE_FRAME ref_frame, second_ref_frame = NONE; MV_REFERENCE_FRAME best_ref_frame = NONE; unsigned char segment_id = mi->segment_id; struct buf_2d yv12_mb[4][MAX_MB_PLANE]; - static const int flag_list[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG, - VP9_ALT_FLAG }; int64_t best_rd = INT64_MAX; b_mode_info bsi[MAX_REF_FRAMES][4]; int ref_frame_skip_mask = 0; const int num_4x4_blocks_wide = num_4x4_blocks_wide_lookup[bsize]; const int num_4x4_blocks_high = num_4x4_blocks_high_lookup[bsize]; @@ -2762,11 +2761,12 @@ for (ref_frame = LAST_FRAME; ref_frame <= GOLDEN_FRAME; ++ref_frame) { const YV12_BUFFER_CONFIG *yv12 = get_ref_frame_buffer(cpi, ref_frame); int_mv dummy_mv[2]; x->pred_mv_sad[ref_frame] = INT_MAX; - if ((cpi->ref_frame_flags & flag_list[ref_frame]) && (yv12 != NULL)) { + if ((cpi->ref_frame_flags & ref_frame_to_flag(ref_frame)) && + (yv12 != NULL)) { int_mv *const candidates = mbmi_ext->ref_mvs[ref_frame]; const struct scale_factors *const sf = &cm->frame_refs[ref_frame - 1].sf; vp9_setup_pred_block(xd, yv12_mb[ref_frame], yv12, mi_row, mi_col, sf, sf); vp9_find_mv_refs(cm, xd, xd->mi[0], ref_frame, candidates, mi_row, mi_col, diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_quantize.c b/media/libvpx/libvpx/vp9/encoder/vp9_quantize.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_quantize.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_quantize.c @@ -20,19 +20,16 @@ #include "vp9/encoder/vp9_encoder.h" #include "vp9/encoder/vp9_quantize.h" #include "vp9/encoder/vp9_rd.h" void vp9_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *round_ptr, - const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, - uint16_t *eob_ptr, const int16_t *scan, - const int16_t *iscan) { + const int16_t *round_ptr, const int16_t *quant_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { int i, eob = -1; (void)iscan; - (void)skip_block; - assert(!skip_block); memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); // Quantization pass: All coefficients with index >= zero_flag are @@ -54,21 +51,19 @@ *eob_ptr = eob + 1; } #if CONFIG_VP9_HIGHBITDEPTH void vp9_highbd_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *round_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan) { int i; int eob = -1; (void)iscan; - (void)skip_block; - assert(!skip_block); memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); // Quantization pass: All coefficients with index >= zero_flag are @@ -89,19 +84,16 @@ #endif // TODO(jingning) Refactor this file and combine functions with similar // operations. void vp9_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *round_ptr, - const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan) { int i, eob = -1; (void)iscan; - (void)skip_block; - assert(!skip_block); memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); for (i = 0; i < n_coeffs; i++) { @@ -124,19 +116,17 @@ *eob_ptr = eob + 1; } #if CONFIG_VP9_HIGHBITDEPTH void vp9_highbd_quantize_fp_32x32_c( - const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, - const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, - const int16_t *scan, const int16_t *iscan) { + const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, + const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, + const int16_t *iscan) { int i, eob = -1; (void)iscan; - (void)skip_block; - assert(!skip_block); memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); for (i = 0; i < n_coeffs; i++) { @@ -174,20 +164,19 @@ return; } #if CONFIG_VP9_HIGHBITDEPTH if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) { - vpx_highbd_quantize_b(BLOCK_OFFSET(p->coeff, block), n_coeffs, - x->skip_block, p->zbin, p->round, p->quant, - p->quant_shift, qcoeff, dqcoeff, pd->dequant, - &p->eobs[block], scan, iscan); + vpx_highbd_quantize_b(BLOCK_OFFSET(p->coeff, block), n_coeffs, p->zbin, + p->round, p->quant, p->quant_shift, qcoeff, dqcoeff, + pd->dequant, &p->eobs[block], scan, iscan); return; } #endif - vpx_quantize_b(BLOCK_OFFSET(p->coeff, block), n_coeffs, x->skip_block, - p->zbin, p->round, p->quant, p->quant_shift, qcoeff, dqcoeff, - pd->dequant, &p->eobs[block], scan, iscan); + vpx_quantize_b(BLOCK_OFFSET(p->coeff, block), n_coeffs, p->zbin, p->round, + p->quant, p->quant_shift, qcoeff, dqcoeff, pd->dequant, + &p->eobs[block], scan, iscan); } static void invert_quant(int16_t *quant, int16_t *shift, int d) { unsigned t; int l, m; diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_ratectrl.c b/media/libvpx/libvpx/vp9/encoder/vp9_ratectrl.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_ratectrl.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_ratectrl.c @@ -221,24 +221,25 @@ // Clip the frame target to the maximum allowed value. if (target > rc->max_frame_bandwidth) target = rc->max_frame_bandwidth; if (oxcf->rc_max_inter_bitrate_pct) { - const int max_rate = - rc->avg_frame_bandwidth * oxcf->rc_max_inter_bitrate_pct / 100; - target = VPXMIN(target, max_rate); + const int64_t max_rate = + (int64_t)rc->avg_frame_bandwidth * oxcf->rc_max_inter_bitrate_pct / 100; + // target is of type int and VPXMIN cannot evaluate to larger than target + target = (int)VPXMIN(target, max_rate); } return target; } int vp9_rc_clamp_iframe_target_size(const VP9_COMP *const cpi, int target) { const RATE_CONTROL *rc = &cpi->rc; const VP9EncoderConfig *oxcf = &cpi->oxcf; if (oxcf->rc_max_intra_bitrate_pct) { - const int max_rate = - rc->avg_frame_bandwidth * oxcf->rc_max_intra_bitrate_pct / 100; - target = VPXMIN(target, max_rate); + const int64_t max_rate = + (int64_t)rc->avg_frame_bandwidth * oxcf->rc_max_intra_bitrate_pct / 100; + target = (int)VPXMIN(target, max_rate); } if (target > rc->max_frame_bandwidth) target = rc->max_frame_bandwidth; return target; } @@ -275,13 +276,13 @@ if (use_timestamp && cpi->svc.use_set_ref_frame_config && svc->number_temporal_layers == 1 && ts_delta > 0 && svc->current_superframe > 0) { // TODO(marpan): This may need to be modified for temporal layers. const double framerate_pts = 10000000.0 / ts_delta; - lrc->bits_off_target += (int)(lc->target_bandwidth / framerate_pts); + lrc->bits_off_target += (int)round(lc->target_bandwidth / framerate_pts); } else { - lrc->bits_off_target += (int)(lc->target_bandwidth / lc->framerate); + lrc->bits_off_target += (int)round(lc->target_bandwidth / lc->framerate); } // Clip buffer level to maximum buffer size for the layer. lrc->bits_off_target = VPXMIN(lrc->bits_off_target, lrc->maximum_buffer_size); lrc->buffer_level = lrc->bits_off_target; @@ -2211,11 +2212,10 @@ // Also if intra-only is inserted on very first frame, don't allow if // if number of temporal layers > 1. This is because on intra-only frame // only 3 reference buffers can be updated, but for temporal layers > 1 // we generally need to use buffer slots 4 and 5. if ((cm->current_video_frame == 0 && svc->number_temporal_layers > 1) || - svc->temporal_layering_mode == VP9E_TEMPORAL_LAYERING_MODE_BYPASS || svc->number_spatial_layers > 3 || svc->number_temporal_layers > 3 || svc->number_spatial_layers == 1) return; cm->show_frame = 0; cm->intra_only = 1; @@ -2232,23 +2232,33 @@ int i; int count = 0; cpi->lst_fb_idx = -1; cpi->gld_fb_idx = -1; cpi->alt_fb_idx = -1; + svc->update_buffer_slot[0] = 0; // For intra-only frame we need to refresh all slots that were // being used for the base layer (fb_idx_base[i] == 1). // Start with assigning last first, then golden and then alt. for (i = 0; i < REF_FRAMES; ++i) { - if (svc->fb_idx_base[i] == 1) count++; + if (svc->fb_idx_base[i] == 1) { + svc->update_buffer_slot[0] |= 1 << i; + count++; + } if (count == 1 && cpi->lst_fb_idx == -1) cpi->lst_fb_idx = i; if (count == 2 && cpi->gld_fb_idx == -1) cpi->gld_fb_idx = i; if (count == 3 && cpi->alt_fb_idx == -1) cpi->alt_fb_idx = i; } // If golden or alt is not being used for base layer, then set them // to the lst_fb_idx. if (cpi->gld_fb_idx == -1) cpi->gld_fb_idx = cpi->lst_fb_idx; if (cpi->alt_fb_idx == -1) cpi->alt_fb_idx = cpi->lst_fb_idx; + if (svc->temporal_layering_mode == VP9E_TEMPORAL_LAYERING_MODE_BYPASS) { + cpi->ext_refresh_last_frame = 0; + cpi->ext_refresh_golden_frame = 0; + cpi->ext_refresh_alt_ref_frame = 0; + cpi->ref_frame_flags = 0; + } } } void vp9_rc_get_svc_params(VP9_COMP *cpi) { VP9_COMMON *const cm = &cpi->common; @@ -2387,10 +2397,13 @@ } if (svc->set_intra_only_frame) { set_intra_only_frame(cpi); target = vp9_calc_iframe_target_size_one_pass_cbr(cpi); } + // Overlay frame predicts from LAST (intra-only) + if (svc->previous_frame_is_intra_only) cpi->ref_frame_flags |= VP9_LAST_FLAG; + // Any update/change of global cyclic refresh parameters (amount/delta-qp) // should be done here, before the frame qp is selected. if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ) vp9_cyclic_refresh_update_parameters(cpi); diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_rdopt.c b/media/libvpx/libvpx/vp9/encoder/vp9_rdopt.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_rdopt.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_rdopt.c @@ -3313,12 +3313,10 @@ int_mv frame_mv[MB_MODE_COUNT][MAX_REF_FRAMES]; struct buf_2d yv12_mb[4][MAX_MB_PLANE]; int_mv single_newmv[MAX_REF_FRAMES] = { { 0 } }; INTERP_FILTER single_inter_filter[MB_MODE_COUNT][MAX_REF_FRAMES]; int single_skippable[MB_MODE_COUNT][MAX_REF_FRAMES]; - static const int flag_list[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG, - VP9_ALT_FLAG }; int64_t best_rd = best_rd_so_far; int64_t best_pred_diff[REFERENCE_MODES]; int64_t best_pred_rd[REFERENCE_MODES]; int64_t best_filter_rd[SWITCHABLE_FILTER_CONTEXTS]; int64_t best_filter_diff[SWITCHABLE_FILTER_CONTEXTS]; @@ -3390,22 +3388,22 @@ rd_cost->rate = INT_MAX; for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { x->pred_mv_sad[ref_frame] = INT_MAX; - if ((cpi->ref_frame_flags & flag_list[ref_frame]) && + if ((cpi->ref_frame_flags & ref_frame_to_flag(ref_frame)) && !(is_rect_partition && (ctx->skip_ref_frame_mask & (1 << ref_frame)))) { assert(get_ref_frame_buffer(cpi, ref_frame) != NULL); setup_buffer_inter(cpi, x, ref_frame, bsize, mi_row, mi_col, frame_mv[NEARESTMV], frame_mv[NEARMV], yv12_mb); } frame_mv[NEWMV][ref_frame].as_int = INVALID_MV; frame_mv[ZEROMV][ref_frame].as_int = 0; } for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { - if (!(cpi->ref_frame_flags & flag_list[ref_frame])) { + if (!(cpi->ref_frame_flags & ref_frame_to_flag(ref_frame))) { // Skip checking missing references in both single and compound reference // modes. Note that a mode will be skipped if both reference frames // are masked out. ref_frame_skip_mask[0] |= (1 << ref_frame); ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK; @@ -3607,11 +3605,12 @@ if (cm->ref_frame_sign_bias[ref_frame] == cm->ref_frame_sign_bias[second_ref_frame]) continue; // Skip compound inter modes if ARF is not available. - if (!(cpi->ref_frame_flags & flag_list[second_ref_frame])) continue; + if (!(cpi->ref_frame_flags & ref_frame_to_flag(second_ref_frame))) + continue; // Do not allow compound prediction if the segment level reference frame // feature is in use as in this case there can only be one reference. if (segfeature_active(seg, segment_id, SEG_LVL_REF_FRAME)) continue; @@ -4138,12 +4137,10 @@ MV_REFERENCE_FRAME ref_frame, second_ref_frame; unsigned char segment_id = mi->segment_id; int comp_pred, i; int_mv frame_mv[MB_MODE_COUNT][MAX_REF_FRAMES]; struct buf_2d yv12_mb[4][MAX_MB_PLANE]; - static const int flag_list[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG, - VP9_ALT_FLAG }; int64_t best_rd = best_rd_so_far; int64_t best_yrd = best_rd_so_far; // FIXME(rbultje) more precise int64_t best_pred_diff[REFERENCE_MODES]; int64_t best_pred_rd[REFERENCE_MODES]; int64_t best_filter_rd[SWITCHABLE_FILTER_CONTEXTS]; @@ -4189,11 +4186,11 @@ rate_uv_intra = INT_MAX; rd_cost->rate = INT_MAX; for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ref_frame++) { - if (cpi->ref_frame_flags & flag_list[ref_frame]) { + if (cpi->ref_frame_flags & ref_frame_to_flag(ref_frame)) { setup_buffer_inter(cpi, x, ref_frame, bsize, mi_row, mi_col, frame_mv[NEARESTMV], frame_mv[NEARMV], yv12_mb); } else { ref_frame_skip_mask[0] |= (1 << ref_frame); ref_frame_skip_mask[1] |= SECOND_REF_FRAME_MASK; @@ -4274,11 +4271,12 @@ if (cm->ref_frame_sign_bias[ref_frame] == cm->ref_frame_sign_bias[second_ref_frame]) continue; - if (!(cpi->ref_frame_flags & flag_list[second_ref_frame])) continue; + if (!(cpi->ref_frame_flags & ref_frame_to_flag(second_ref_frame))) + continue; // Do not allow compound prediction if the segment level reference frame // feature is in use as in this case there can only be one reference. if (segfeature_active(seg, segment_id, SEG_LVL_REF_FRAME)) continue; if ((sf->mode_search_skip_flags & FLAG_SKIP_COMP_BESTINTRA) && diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_speed_features.c b/media/libvpx/libvpx/vp9/encoder/vp9_speed_features.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_speed_features.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_speed_features.c @@ -493,15 +493,14 @@ sf->reference_masking = (svc->number_spatial_layers == 1); if (sf->reference_masking == 1 && (cpi->external_resize == 1 || cpi->oxcf.resize_mode == RESIZE_DYNAMIC)) { MV_REFERENCE_FRAME ref_frame; - static const int flag_list[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG, - VP9_ALT_FLAG }; for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { const YV12_BUFFER_CONFIG *yv12 = get_ref_frame_buffer(cpi, ref_frame); - if (yv12 != NULL && (cpi->ref_frame_flags & flag_list[ref_frame])) { + if (yv12 != NULL && + (cpi->ref_frame_flags & ref_frame_to_flag(ref_frame))) { const struct scale_factors *const scale_fac = &cm->frame_refs[ref_frame - 1].sf; if (vp9_is_scaled(scale_fac)) sf->reference_masking = 0; } } @@ -651,12 +650,14 @@ sf->adapt_partition_thresh = (cm->width * cm->height <= 640 * 360) ? 40000 : 60000; if (cpi->content_state_sb_fd == NULL && (!cpi->use_svc || svc->spatial_layer_id == svc->number_spatial_layers - 1)) { - cpi->content_state_sb_fd = (uint8_t *)vpx_calloc( - (cm->mi_stride >> 3) * ((cm->mi_rows >> 3) + 1), sizeof(uint8_t)); + CHECK_MEM_ERROR(cm, cpi->content_state_sb_fd, + (uint8_t *)vpx_calloc( + (cm->mi_stride >> 3) * ((cm->mi_rows >> 3) + 1), + sizeof(uint8_t))); } } if (cpi->oxcf.rc_mode == VPX_CBR && content != VP9E_CONTENT_SCREEN) { // Enable short circuit for low temporal variance. sf->short_circuit_low_temp_var = 1; @@ -803,18 +804,21 @@ if (sf->use_altref_onepass) { if (cpi->rc.is_src_frame_alt_ref && cm->frame_type != KEY_FRAME) { sf->partition_search_type = FIXED_PARTITION; sf->always_this_block_size = BLOCK_64X64; } - if (cpi->count_arf_frame_usage == NULL) - cpi->count_arf_frame_usage = + if (cpi->count_arf_frame_usage == NULL) { + CHECK_MEM_ERROR( + cm, cpi->count_arf_frame_usage, (uint8_t *)vpx_calloc((cm->mi_stride >> 3) * ((cm->mi_rows >> 3) + 1), - sizeof(*cpi->count_arf_frame_usage)); + sizeof(*cpi->count_arf_frame_usage))); + } if (cpi->count_lastgolden_frame_usage == NULL) - cpi->count_lastgolden_frame_usage = + CHECK_MEM_ERROR( + cm, cpi->count_lastgolden_frame_usage, (uint8_t *)vpx_calloc((cm->mi_stride >> 3) * ((cm->mi_rows >> 3) + 1), - sizeof(*cpi->count_lastgolden_frame_usage)); + sizeof(*cpi->count_lastgolden_frame_usage))); } if (svc->previous_frame_is_intra_only) { sf->partition_search_type = FIXED_PARTITION; sf->always_this_block_size = BLOCK_64X64; } diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_subexp.h b/media/libvpx/libvpx/vp9/encoder/vp9_subexp.h --- a/media/libvpx/libvpx/vp9/encoder/vp9_subexp.h +++ b/media/libvpx/libvpx/vp9/encoder/vp9_subexp.h @@ -23,17 +23,18 @@ vpx_prob oldp); void vp9_cond_prob_diff_update(struct vpx_writer *w, vpx_prob *oldp, const unsigned int ct[2]); -int vp9_prob_diff_update_savings_search(const unsigned int *ct, vpx_prob oldp, - vpx_prob *bestp, vpx_prob upd); +int64_t vp9_prob_diff_update_savings_search(const unsigned int *ct, + vpx_prob oldp, vpx_prob *bestp, + vpx_prob upd); -int vp9_prob_diff_update_savings_search_model(const unsigned int *ct, - const vpx_prob oldp, - vpx_prob *bestp, vpx_prob upd, - int stepsize); +int64_t vp9_prob_diff_update_savings_search_model(const unsigned int *ct, + const vpx_prob oldp, + vpx_prob *bestp, vpx_prob upd, + int stepsize); #ifdef __cplusplus } // extern "C" #endif diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_subexp.c b/media/libvpx/libvpx/vp9/encoder/vp9_subexp.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_subexp.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_subexp.c @@ -112,42 +112,43 @@ void vp9_write_prob_diff_update(vpx_writer *w, vpx_prob newp, vpx_prob oldp) { const int delp = remap_prob(newp, oldp); encode_term_subexp(w, delp); } -int vp9_prob_diff_update_savings_search(const unsigned int *ct, vpx_prob oldp, - vpx_prob *bestp, vpx_prob upd) { - const int old_b = cost_branch256(ct, oldp); - int bestsavings = 0; +int64_t vp9_prob_diff_update_savings_search(const unsigned int *ct, + vpx_prob oldp, vpx_prob *bestp, + vpx_prob upd) { + const int64_t old_b = cost_branch256(ct, oldp); + int64_t bestsavings = 0; vpx_prob newp, bestnewp = oldp; const int step = *bestp > oldp ? -1 : 1; const int upd_cost = vp9_cost_one(upd) - vp9_cost_zero(upd); if (old_b > upd_cost + (MIN_DELP_BITS << VP9_PROB_COST_SHIFT)) { for (newp = *bestp; newp != oldp; newp += step) { - const int new_b = cost_branch256(ct, newp); - const int update_b = prob_diff_update_cost(newp, oldp) + upd_cost; - const int savings = old_b - new_b - update_b; + const int64_t new_b = cost_branch256(ct, newp); + const int64_t update_b = prob_diff_update_cost(newp, oldp) + upd_cost; + const int64_t savings = old_b - new_b - update_b; if (savings > bestsavings) { bestsavings = savings; bestnewp = newp; } } } *bestp = bestnewp; return bestsavings; } -int vp9_prob_diff_update_savings_search_model(const unsigned int *ct, - const vpx_prob oldp, - vpx_prob *bestp, vpx_prob upd, - int stepsize) { - int i, old_b, new_b, update_b, savings, bestsavings; - int newp; - const int step_sign = *bestp > oldp ? -1 : 1; - const int step = stepsize * step_sign; - const int upd_cost = vp9_cost_one(upd) - vp9_cost_zero(upd); +int64_t vp9_prob_diff_update_savings_search_model(const unsigned int *ct, + const vpx_prob oldp, + vpx_prob *bestp, vpx_prob upd, + int stepsize) { + int64_t i, old_b, new_b, update_b, savings, bestsavings; + int64_t newp; + const int64_t step_sign = *bestp > oldp ? -1 : 1; + const int64_t step = stepsize * step_sign; + const int64_t upd_cost = vp9_cost_one(upd) - vp9_cost_zero(upd); const vpx_prob *newplist, *oldplist; vpx_prob bestnewp; oldplist = vp9_pareto8_full[oldp - 1]; old_b = cost_branch256(ct + 2 * PIVOT_NODE, oldp); for (i = UNCONSTRAINED_NODES; i < ENTROPY_NODES; ++i) @@ -160,18 +161,18 @@ if (old_b > upd_cost + (MIN_DELP_BITS << VP9_PROB_COST_SHIFT)) { for (newp = *bestp; (newp - oldp) * step_sign < 0; newp += step) { if (newp < 1 || newp > 255) continue; newplist = vp9_pareto8_full[newp - 1]; - new_b = cost_branch256(ct + 2 * PIVOT_NODE, newp); + new_b = cost_branch256(ct + 2 * PIVOT_NODE, (vpx_prob)newp); for (i = UNCONSTRAINED_NODES; i < ENTROPY_NODES; ++i) new_b += cost_branch256(ct + 2 * i, newplist[i - UNCONSTRAINED_NODES]); - update_b = prob_diff_update_cost(newp, oldp) + upd_cost; + update_b = prob_diff_update_cost((vpx_prob)newp, oldp) + upd_cost; savings = old_b - new_b - update_b; if (savings > bestsavings) { bestsavings = savings; - bestnewp = newp; + bestnewp = (vpx_prob)newp; } } } *bestp = bestnewp; @@ -180,11 +181,11 @@ void vp9_cond_prob_diff_update(vpx_writer *w, vpx_prob *oldp, const unsigned int ct[2]) { const vpx_prob upd = DIFF_UPDATE_PROB; vpx_prob newp = get_binary_prob(ct[0], ct[1]); - const int savings = + const int64_t savings = vp9_prob_diff_update_savings_search(ct, *oldp, &newp, upd); assert(newp >= 1); if (savings > 0) { vpx_write(w, 1, upd); vp9_write_prob_diff_update(w, newp, *oldp); diff --git a/media/libvpx/libvpx/vp9/encoder/vp9_svc_layercontext.c b/media/libvpx/libvpx/vp9/encoder/vp9_svc_layercontext.c --- a/media/libvpx/libvpx/vp9/encoder/vp9_svc_layercontext.c +++ b/media/libvpx/libvpx/vp9/encoder/vp9_svc_layercontext.c @@ -460,34 +460,31 @@ static void reset_fb_idx_unused(VP9_COMP *const cpi) { // If a reference frame is not referenced or refreshed, then set the // fb_idx for that reference to the first one used/referenced. // This is to avoid setting fb_idx for a reference to a slot that is not // used/needed (i.e., since that reference is not referenced or refreshed). - static const int flag_list[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG, - VP9_ALT_FLAG }; MV_REFERENCE_FRAME ref_frame; MV_REFERENCE_FRAME first_ref = 0; int first_fb_idx = 0; int fb_idx[3] = { cpi->lst_fb_idx, cpi->gld_fb_idx, cpi->alt_fb_idx }; for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ref_frame++) { - if (cpi->ref_frame_flags & flag_list[ref_frame]) { + if (cpi->ref_frame_flags & ref_frame_to_flag(ref_frame)) { first_ref = ref_frame; first_fb_idx = fb_idx[ref_frame - 1]; break; } } if (first_ref > 0) { - if (first_ref != LAST_FRAME && - !(cpi->ref_frame_flags & flag_list[LAST_FRAME]) && + if (first_ref != LAST_FRAME && !(cpi->ref_frame_flags & VP9_LAST_FLAG) && !cpi->ext_refresh_last_frame) cpi->lst_fb_idx = first_fb_idx; else if (first_ref != GOLDEN_FRAME && - !(cpi->ref_frame_flags & flag_list[GOLDEN_FRAME]) && + !(cpi->ref_frame_flags & VP9_GOLD_FLAG) && !cpi->ext_refresh_golden_frame) cpi->gld_fb_idx = first_fb_idx; else if (first_ref != ALTREF_FRAME && - !(cpi->ref_frame_flags & flag_list[ALTREF_FRAME]) && + !(cpi->ref_frame_flags & VP9_ALT_FLAG) && !cpi->ext_refresh_alt_ref_frame) cpi->alt_fb_idx = first_fb_idx; } } @@ -728,12 +725,10 @@ if (svc->reference_altref[sl]) cpi->ref_frame_flags |= VP9_ALT_FLAG; } void vp9_copy_flags_ref_update_idx(VP9_COMP *const cpi) { SVC *const svc = &cpi->svc; - static const int flag_list[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG, - VP9_ALT_FLAG }; int sl = svc->spatial_layer_id; svc->lst_fb_idx[sl] = cpi->lst_fb_idx; svc->gld_fb_idx[sl] = cpi->gld_fb_idx; svc->alt_fb_idx[sl] = cpi->alt_fb_idx; // For the fixed SVC mode: pass the refresh_lst/gld/alt_frame flags to the @@ -752,16 +747,13 @@ // TODO(jianj): Remove these 3, deprecated. svc->update_last[sl] = (uint8_t)cpi->refresh_last_frame; svc->update_golden[sl] = (uint8_t)cpi->refresh_golden_frame; svc->update_altref[sl] = (uint8_t)cpi->refresh_alt_ref_frame; - svc->reference_last[sl] = - (uint8_t)(cpi->ref_frame_flags & flag_list[LAST_FRAME]); - svc->reference_golden[sl] = - (uint8_t)(cpi->ref_frame_flags & flag_list[GOLDEN_FRAME]); - svc->reference_altref[sl] = - (uint8_t)(cpi->ref_frame_flags & flag_list[ALTREF_FRAME]); + svc->reference_last[sl] = (uint8_t)(cpi->ref_frame_flags & VP9_LAST_FLAG); + svc->reference_golden[sl] = (uint8_t)(cpi->ref_frame_flags & VP9_GOLD_FLAG); + svc->reference_altref[sl] = (uint8_t)(cpi->ref_frame_flags & VP9_ALT_FLAG); } int vp9_one_pass_cbr_svc_start_layer(VP9_COMP *const cpi) { int width = 0, height = 0; SVC *const svc = &cpi->svc; @@ -1078,19 +1070,18 @@ !svc->layer_context[svc->temporal_layer_id].is_key_frame && !svc->superframe_has_layer_sync) || svc->disable_inter_layer_pred == INTER_LAYER_PRED_OFF || svc->drop_spatial_layer[sl - 1]) { MV_REFERENCE_FRAME ref_frame; - static const int flag_list[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG, - VP9_ALT_FLAG }; for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { const YV12_BUFFER_CONFIG *yv12 = get_ref_frame_buffer(cpi, ref_frame); - if (yv12 != NULL && (cpi->ref_frame_flags & flag_list[ref_frame])) { + if (yv12 != NULL && + (cpi->ref_frame_flags & ref_frame_to_flag(ref_frame))) { const struct scale_factors *const scale_fac = &cm->frame_refs[ref_frame - 1].sf; if (vp9_is_scaled(scale_fac)) { - cpi->ref_frame_flags &= (~flag_list[ref_frame]); + cpi->ref_frame_flags &= (~ref_frame_to_flag(ref_frame)); // Point golden/altref frame buffer index to last. if (!svc->simulcast_mode) { if (ref_frame == GOLDEN_FRAME) cpi->gld_fb_idx = cpi->lst_fb_idx; else if (ref_frame == ALTREF_FRAME) @@ -1241,18 +1232,24 @@ } } void vp9_svc_update_ref_frame_buffer_idx(VP9_COMP *const cpi) { SVC *const svc = &cpi->svc; + int i = 0; // Update the usage of frame buffer index for base spatial layers. if (svc->spatial_layer_id == 0) { if ((cpi->ref_frame_flags & VP9_LAST_FLAG) || cpi->refresh_last_frame) svc->fb_idx_base[cpi->lst_fb_idx] = 1; if ((cpi->ref_frame_flags & VP9_GOLD_FLAG) || cpi->refresh_golden_frame) svc->fb_idx_base[cpi->gld_fb_idx] = 1; if ((cpi->ref_frame_flags & VP9_ALT_FLAG) || cpi->refresh_alt_ref_frame) svc->fb_idx_base[cpi->alt_fb_idx] = 1; + // For bypass/flexible mode: check for refresh slots. + if (svc->temporal_layering_mode == VP9E_TEMPORAL_LAYERING_MODE_BYPASS) { + for (i = 0; i < REF_FRAMES; ++i) + if (svc->update_buffer_slot[0] & (1 << i)) svc->fb_idx_base[i] = 1; + } } } static void vp9_svc_update_ref_frame_bypass_mode(VP9_COMP *const cpi) { // For non-flexible/bypass SVC mode: check for refreshing other buffer diff --git a/media/libvpx/libvpx/vp9/encoder/x86/temporal_filter_sse4.c b/media/libvpx/libvpx/vp9/encoder/x86/temporal_filter_sse4.c --- a/media/libvpx/libvpx/vp9/encoder/x86/temporal_filter_sse4.c +++ b/media/libvpx/libvpx/vp9/encoder/x86/temporal_filter_sse4.c @@ -268,17 +268,15 @@ // Apply temporal filter to the luma components. This performs temporal // filtering on a luma block of 16 X block_height. Use blk_fw as an array of // size 4 for the weights for each of the 4 subblocks if blk_fw is not NULL, // else use top_weight for top half, and bottom weight for bottom half. static void vp9_apply_temporal_filter_luma_16( - const uint8_t *y_src, int y_src_stride, const uint8_t *y_pre, - int y_pre_stride, const uint8_t *u_src, const uint8_t *v_src, - int uv_src_stride, const uint8_t *u_pre, const uint8_t *v_pre, - int uv_pre_stride, unsigned int block_width, unsigned int block_height, - int ss_x, int ss_y, int strength, int use_whole_blk, uint32_t *y_accum, - uint16_t *y_count, const uint16_t *y_dist, const uint16_t *u_dist, - const uint16_t *v_dist, const int16_t *const *neighbors_first, + const uint8_t *y_pre, int y_pre_stride, unsigned int block_width, + unsigned int block_height, int ss_x, int ss_y, int strength, + int use_whole_blk, uint32_t *y_accum, uint16_t *y_count, + const uint16_t *y_dist, const uint16_t *u_dist, const uint16_t *v_dist, + const int16_t *const *neighbors_first, const int16_t *const *neighbors_second, int top_weight, int bottom_weight, const int *blk_fw) { const int rounding = (1 << strength) >> 1; __m128i weight_first, weight_second; @@ -299,11 +297,10 @@ assert(strength >= 0); assert(strength <= 6); assert(block_width == 16); - (void)block_width; // Initialize the weights if (blk_fw) { weight_first = _mm_set1_epi16(blk_fw[0]); @@ -340,21 +337,16 @@ sum_row_second = average_8(sum_row_second, &mul_second, strength, rounding, &weight_second); accumulate_and_store_16(sum_row_first, sum_row_second, y_pre, y_count, y_accum); - y_src += y_src_stride; y_pre += y_pre_stride; y_count += y_pre_stride; y_accum += y_pre_stride; y_dist += DIST_STRIDE; - u_src += uv_src_stride; - u_pre += uv_pre_stride; u_dist += DIST_STRIDE; - v_src += uv_src_stride; - v_pre += uv_pre_stride; v_dist += DIST_STRIDE; // Then all the rows except the last one mul_first = _mm_load_si128((const __m128i *)neighbors_first[1]); mul_second = _mm_load_si128((const __m128i *)neighbors_second[1]); @@ -390,15 +382,11 @@ // Only calculate the new chroma distortion if we are at a pixel that // corresponds to a new chroma row read_chroma_dist_row_16(ss_x, u_dist, v_dist, &u_first, &u_second, &v_first, &v_second); - u_src += uv_src_stride; - u_pre += uv_pre_stride; u_dist += DIST_STRIDE; - v_src += uv_src_stride; - v_pre += uv_pre_stride; v_dist += DIST_STRIDE; } sum_row_first = _mm_adds_epu16(sum_row_first, u_first); sum_row_second = _mm_adds_epu16(sum_row_second, u_second); @@ -411,11 +399,10 @@ sum_row_second = average_8(sum_row_second, &mul_second, strength, rounding, &weight_second); accumulate_and_store_16(sum_row_first, sum_row_second, y_pre, y_count, y_accum); - y_src += y_src_stride; y_pre += y_pre_stride; y_count += y_pre_stride; y_accum += y_pre_stride; y_dist += DIST_STRIDE; } @@ -456,17 +443,14 @@ y_accum); } // Perform temporal filter for the luma component. static void vp9_apply_temporal_filter_luma( - const uint8_t *y_src, int y_src_stride, const uint8_t *y_pre, - int y_pre_stride, const uint8_t *u_src, const uint8_t *v_src, - int uv_src_stride, const uint8_t *u_pre, const uint8_t *v_pre, - int uv_pre_stride, unsigned int block_width, unsigned int block_height, - int ss_x, int ss_y, int strength, const int *blk_fw, int use_whole_blk, - uint32_t *y_accum, uint16_t *y_count, const uint16_t *y_dist, - const uint16_t *u_dist, const uint16_t *v_dist) { + const uint8_t *y_pre, int y_pre_stride, unsigned int block_width, + unsigned int block_height, int ss_x, int ss_y, int strength, + const int *blk_fw, int use_whole_blk, uint32_t *y_accum, uint16_t *y_count, + const uint16_t *y_dist, const uint16_t *u_dist, const uint16_t *v_dist) { unsigned int blk_col = 0, uv_blk_col = 0; const unsigned int blk_col_step = 16, uv_blk_col_step = 16 >> ss_x; const unsigned int mid_width = block_width >> 1, last_width = block_width - blk_col_step; int top_weight = blk_fw[0], @@ -474,43 +458,36 @@ const int16_t *const *neighbors_first; const int16_t *const *neighbors_second; if (block_width == 16) { // Special Case: The blockwidth is 16 and we are operating on a row of 16 - // chroma pixels. In this case, we can't use the usualy left-midle-right + // chroma pixels. In this case, we can't use the usual left-middle-right // pattern. We also don't support splitting now. neighbors_first = LUMA_LEFT_COLUMN_NEIGHBORS; neighbors_second = LUMA_RIGHT_COLUMN_NEIGHBORS; if (use_whole_blk) { vp9_apply_temporal_filter_luma_16( - y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride, - u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, - u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, 16, - block_height, ss_x, ss_y, strength, use_whole_blk, y_accum + blk_col, - y_count + blk_col, y_dist + blk_col, u_dist + uv_blk_col, - v_dist + uv_blk_col, neighbors_first, neighbors_second, top_weight, - bottom_weight, NULL); + y_pre + blk_col, y_pre_stride, 16, block_height, ss_x, ss_y, strength, + use_whole_blk, y_accum + blk_col, y_count + blk_col, y_dist + blk_col, + u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors_first, + neighbors_second, top_weight, bottom_weight, NULL); } else { vp9_apply_temporal_filter_luma_16( - y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride, - u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, - u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, 16, - block_height, ss_x, ss_y, strength, use_whole_blk, y_accum + blk_col, - y_count + blk_col, y_dist + blk_col, u_dist + uv_blk_col, - v_dist + uv_blk_col, neighbors_first, neighbors_second, 0, 0, blk_fw); + y_pre + blk_col, y_pre_stride, 16, block_height, ss_x, ss_y, strength, + use_whole_blk, y_accum + blk_col, y_count + blk_col, y_dist + blk_col, + u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors_first, + neighbors_second, 0, 0, blk_fw); } return; } // Left neighbors_first = LUMA_LEFT_COLUMN_NEIGHBORS; neighbors_second = LUMA_MIDDLE_COLUMN_NEIGHBORS; vp9_apply_temporal_filter_luma_16( - y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride, - u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, u_pre + uv_blk_col, - v_pre + uv_blk_col, uv_pre_stride, 16, block_height, ss_x, ss_y, strength, + y_pre + blk_col, y_pre_stride, 16, block_height, ss_x, ss_y, strength, use_whole_blk, y_accum + blk_col, y_count + blk_col, y_dist + blk_col, u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors_first, neighbors_second, top_weight, bottom_weight, NULL); blk_col += blk_col_step; @@ -519,17 +496,14 @@ // Middle First neighbors_first = LUMA_MIDDLE_COLUMN_NEIGHBORS; for (; blk_col < mid_width; blk_col += blk_col_step, uv_blk_col += uv_blk_col_step) { vp9_apply_temporal_filter_luma_16( - y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride, - u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, - u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, 16, block_height, - ss_x, ss_y, strength, use_whole_blk, y_accum + blk_col, - y_count + blk_col, y_dist + blk_col, u_dist + uv_blk_col, - v_dist + uv_blk_col, neighbors_first, neighbors_second, top_weight, - bottom_weight, NULL); + y_pre + blk_col, y_pre_stride, 16, block_height, ss_x, ss_y, strength, + use_whole_blk, y_accum + blk_col, y_count + blk_col, y_dist + blk_col, + u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors_first, + neighbors_second, top_weight, bottom_weight, NULL); } if (!use_whole_blk) { top_weight = blk_fw[1]; bottom_weight = blk_fw[3]; @@ -537,39 +511,31 @@ // Middle Second for (; blk_col < last_width; blk_col += blk_col_step, uv_blk_col += uv_blk_col_step) { vp9_apply_temporal_filter_luma_16( - y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride, - u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, - u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, 16, block_height, - ss_x, ss_y, strength, use_whole_blk, y_accum + blk_col, - y_count + blk_col, y_dist + blk_col, u_dist + uv_blk_col, - v_dist + uv_blk_col, neighbors_first, neighbors_second, top_weight, - bottom_weight, NULL); + y_pre + blk_col, y_pre_stride, 16, block_height, ss_x, ss_y, strength, + use_whole_blk, y_accum + blk_col, y_count + blk_col, y_dist + blk_col, + u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors_first, + neighbors_second, top_weight, bottom_weight, NULL); } // Right neighbors_second = LUMA_RIGHT_COLUMN_NEIGHBORS; vp9_apply_temporal_filter_luma_16( - y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride, - u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, u_pre + uv_blk_col, - v_pre + uv_blk_col, uv_pre_stride, 16, block_height, ss_x, ss_y, strength, + y_pre + blk_col, y_pre_stride, 16, block_height, ss_x, ss_y, strength, use_whole_blk, y_accum + blk_col, y_count + blk_col, y_dist + blk_col, u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors_first, neighbors_second, top_weight, bottom_weight, NULL); } // Apply temporal filter to the chroma components. This performs temporal // filtering on a chroma block of 8 X uv_height. If blk_fw is not NULL, use // blk_fw as an array of size 4 for the weights for each of the 4 subblocks, // else use top_weight for top half, and bottom weight for bottom half. static void vp9_apply_temporal_filter_chroma_8( - const uint8_t *y_src, int y_src_stride, const uint8_t *y_pre, - int y_pre_stride, const uint8_t *u_src, const uint8_t *v_src, - int uv_src_stride, const uint8_t *u_pre, const uint8_t *v_pre, - int uv_pre_stride, unsigned int uv_block_width, + const uint8_t *u_pre, const uint8_t *v_pre, int uv_pre_stride, unsigned int uv_block_height, int ss_x, int ss_y, int strength, uint32_t *u_accum, uint16_t *u_count, uint32_t *v_accum, uint16_t *v_count, const uint16_t *y_dist, const uint16_t *u_dist, const uint16_t *v_dist, const int16_t *const *neighbors, int top_weight, int bottom_weight, const int *blk_fw) { @@ -585,13 +551,11 @@ __m128i u_sum_row, v_sum_row; // Loop variable unsigned int h; - (void)uv_block_width; - - // Initilize weight + // Initialize weight if (blk_fw) { weight = _mm_setr_epi16(blk_fw[0], blk_fw[0], blk_fw[0], blk_fw[0], blk_fw[1], blk_fw[1], blk_fw[1], blk_fw[1]); } else { weight = _mm_set1_epi16(top_weight); @@ -619,23 +583,19 @@ v_sum_row = average_8(v_sum_row, &mul, strength, rounding, &weight); accumulate_and_store_8(u_sum_row, u_pre, u_count, u_accum); accumulate_and_store_8(v_sum_row, v_pre, v_count, v_accum); - u_src += uv_src_stride; u_pre += uv_pre_stride; u_dist += DIST_STRIDE; - v_src += uv_src_stride; v_pre += uv_pre_stride; v_dist += DIST_STRIDE; u_count += uv_pre_stride; u_accum += uv_pre_stride; v_count += uv_pre_stride; v_accum += uv_pre_stride; - y_src += y_src_stride * (1 + ss_y); - y_pre += y_pre_stride * (1 + ss_y); y_dist += DIST_STRIDE * (1 + ss_y); // Then all the rows except the last one mul = _mm_load_si128((const __m128i *)neighbors[1]); @@ -674,23 +634,19 @@ v_sum_row = average_8(v_sum_row, &mul, strength, rounding, &weight); accumulate_and_store_8(u_sum_row, u_pre, u_count, u_accum); accumulate_and_store_8(v_sum_row, v_pre, v_count, v_accum); - u_src += uv_src_stride; u_pre += uv_pre_stride; u_dist += DIST_STRIDE; - v_src += uv_src_stride; v_pre += uv_pre_stride; v_dist += DIST_STRIDE; u_count += uv_pre_stride; u_accum += uv_pre_stride; v_count += uv_pre_stride; v_accum += uv_pre_stride; - y_src += y_src_stride * (1 + ss_y); - y_pre += y_pre_stride * (1 + ss_y); y_dist += DIST_STRIDE * (1 + ss_y); } // The last row mul = _mm_load_si128((const __m128i *)neighbors[0]); @@ -717,16 +673,14 @@ accumulate_and_store_8(v_sum_row, v_pre, v_count, v_accum); } // Perform temporal filter for the chroma components. static void vp9_apply_temporal_filter_chroma( - const uint8_t *y_src, int y_src_stride, const uint8_t *y_pre, - int y_pre_stride, const uint8_t *u_src, const uint8_t *v_src, - int uv_src_stride, const uint8_t *u_pre, const uint8_t *v_pre, - int uv_pre_stride, unsigned int block_width, unsigned int block_height, - int ss_x, int ss_y, int strength, const int *blk_fw, int use_whole_blk, - uint32_t *u_accum, uint16_t *u_count, uint32_t *v_accum, uint16_t *v_count, + const uint8_t *u_pre, const uint8_t *v_pre, int uv_pre_stride, + unsigned int block_width, unsigned int block_height, int ss_x, int ss_y, + int strength, const int *blk_fw, int use_whole_blk, uint32_t *u_accum, + uint16_t *u_count, uint32_t *v_accum, uint16_t *v_count, const uint16_t *y_dist, const uint16_t *u_dist, const uint16_t *v_dist) { const unsigned int uv_width = block_width >> ss_x, uv_height = block_height >> ss_y; unsigned int blk_col = 0, uv_blk_col = 0; @@ -749,26 +703,21 @@ neighbors = CHROMA_SINGLE_SS_SINGLE_COLUMN_NEIGHBORS; } if (use_whole_blk) { vp9_apply_temporal_filter_chroma_8( - y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride, - u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, - u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, uv_width, - uv_height, ss_x, ss_y, strength, u_accum + uv_blk_col, - u_count + uv_blk_col, v_accum + uv_blk_col, v_count + uv_blk_col, - y_dist + blk_col, u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors, - top_weight, bottom_weight, NULL); + u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, uv_height, + ss_x, ss_y, strength, u_accum + uv_blk_col, u_count + uv_blk_col, + v_accum + uv_blk_col, v_count + uv_blk_col, y_dist + blk_col, + u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors, top_weight, + bottom_weight, NULL); } else { vp9_apply_temporal_filter_chroma_8( - y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride, - u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, - u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, uv_width, - uv_height, ss_x, ss_y, strength, u_accum + uv_blk_col, - u_count + uv_blk_col, v_accum + uv_blk_col, v_count + uv_blk_col, - y_dist + blk_col, u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors, - 0, 0, blk_fw); + u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, uv_height, + ss_x, ss_y, strength, u_accum + uv_blk_col, u_count + uv_blk_col, + v_accum + uv_blk_col, v_count + uv_blk_col, y_dist + blk_col, + u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors, 0, 0, blk_fw); } return; } @@ -780,14 +729,12 @@ } else { neighbors = CHROMA_NO_SS_LEFT_COLUMN_NEIGHBORS; } vp9_apply_temporal_filter_chroma_8( - y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride, - u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, u_pre + uv_blk_col, - v_pre + uv_blk_col, uv_pre_stride, uv_width, uv_height, ss_x, ss_y, - strength, u_accum + uv_blk_col, u_count + uv_blk_col, + u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, uv_height, ss_x, + ss_y, strength, u_accum + uv_blk_col, u_count + uv_blk_col, v_accum + uv_blk_col, v_count + uv_blk_col, y_dist + blk_col, u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors, top_weight, bottom_weight, NULL); blk_col += blk_col_step; @@ -803,17 +750,15 @@ } for (; uv_blk_col < uv_mid_width; blk_col += blk_col_step, uv_blk_col += uv_blk_col_step) { vp9_apply_temporal_filter_chroma_8( - y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride, - u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, - u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, uv_width, - uv_height, ss_x, ss_y, strength, u_accum + uv_blk_col, - u_count + uv_blk_col, v_accum + uv_blk_col, v_count + uv_blk_col, - y_dist + blk_col, u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors, - top_weight, bottom_weight, NULL); + u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, uv_height, ss_x, + ss_y, strength, u_accum + uv_blk_col, u_count + uv_blk_col, + v_accum + uv_blk_col, v_count + uv_blk_col, y_dist + blk_col, + u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors, top_weight, + bottom_weight, NULL); } if (!use_whole_blk) { top_weight = blk_fw[1]; bottom_weight = blk_fw[3]; @@ -821,17 +766,15 @@ // Middle Second for (; uv_blk_col < uv_last_width; blk_col += blk_col_step, uv_blk_col += uv_blk_col_step) { vp9_apply_temporal_filter_chroma_8( - y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride, - u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, - u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, uv_width, - uv_height, ss_x, ss_y, strength, u_accum + uv_blk_col, - u_count + uv_blk_col, v_accum + uv_blk_col, v_count + uv_blk_col, - y_dist + blk_col, u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors, - top_weight, bottom_weight, NULL); + u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, uv_height, ss_x, + ss_y, strength, u_accum + uv_blk_col, u_count + uv_blk_col, + v_accum + uv_blk_col, v_count + uv_blk_col, y_dist + blk_col, + u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors, top_weight, + bottom_weight, NULL); } // Right if (ss_x && ss_y) { neighbors = CHROMA_DOUBLE_SS_RIGHT_COLUMN_NEIGHBORS; @@ -840,14 +783,12 @@ } else { neighbors = CHROMA_NO_SS_RIGHT_COLUMN_NEIGHBORS; } vp9_apply_temporal_filter_chroma_8( - y_src + blk_col, y_src_stride, y_pre + blk_col, y_pre_stride, - u_src + uv_blk_col, v_src + uv_blk_col, uv_src_stride, u_pre + uv_blk_col, - v_pre + uv_blk_col, uv_pre_stride, uv_width, uv_height, ss_x, ss_y, - strength, u_accum + uv_blk_col, u_count + uv_blk_col, + u_pre + uv_blk_col, v_pre + uv_blk_col, uv_pre_stride, uv_height, ss_x, + ss_y, strength, u_accum + uv_blk_col, u_count + uv_blk_col, v_accum + uv_blk_col, v_count + uv_blk_col, y_dist + blk_col, u_dist + uv_blk_col, v_dist + uv_blk_col, neighbors, top_weight, bottom_weight, NULL); } @@ -884,16 +825,16 @@ assert(strength >= 0 && strength <= 6 && "invalid temporal filter strength"); assert(blk_fw[0] >= 0 && "filter weight must be positive"); assert( (use_whole_blk || (blk_fw[1] >= 0 && blk_fw[2] >= 0 && blk_fw[3] >= 0)) && "subblock filter weight must be positive"); - assert(blk_fw[0] <= 2 && "sublock filter weight must be less than 2"); + assert(blk_fw[0] <= 2 && "subblock filter weight must be less than 2"); assert( (use_whole_blk || (blk_fw[1] <= 2 && blk_fw[2] <= 2 && blk_fw[3] <= 2)) && "subblock filter weight must be less than 2"); - // Precompute the difference sqaured + // Precompute the difference squared for (row = 0; row < block_height; row++) { for (blk_col = 0; blk_col < block_width; blk_col += 16) { store_dist_16(y_src_ptr + blk_col, y_pre_ptr + blk_col, y_dist_ptr + blk_col); } @@ -920,17 +861,15 @@ y_dist_ptr = y_dist + 1; u_dist_ptr = u_dist + 1; v_dist_ptr = v_dist + 1; - vp9_apply_temporal_filter_luma( - y_src, y_src_stride, y_pre, y_pre_stride, u_src, v_src, uv_src_stride, - u_pre, v_pre, uv_pre_stride, block_width, block_height, ss_x, ss_y, - strength, blk_fw_ptr, use_whole_blk, y_accum, y_count, y_dist_ptr, - u_dist_ptr, v_dist_ptr); + vp9_apply_temporal_filter_luma(y_pre, y_pre_stride, block_width, block_height, + ss_x, ss_y, strength, blk_fw_ptr, + use_whole_blk, y_accum, y_count, y_dist_ptr, + u_dist_ptr, v_dist_ptr); vp9_apply_temporal_filter_chroma( - y_src, y_src_stride, y_pre, y_pre_stride, u_src, v_src, uv_src_stride, u_pre, v_pre, uv_pre_stride, block_width, block_height, ss_x, ss_y, strength, blk_fw_ptr, use_whole_blk, u_accum, u_count, v_accum, v_count, y_dist_ptr, u_dist_ptr, v_dist_ptr); } diff --git a/media/libvpx/libvpx/vp9/encoder/x86/vp9_diamond_search_sad_avx.c b/media/libvpx/libvpx/vp9/encoder/x86/vp9_diamond_search_sad_avx.c --- a/media/libvpx/libvpx/vp9/encoder/x86/vp9_diamond_search_sad_avx.c +++ b/media/libvpx/libvpx/vp9/encoder/x86/vp9_diamond_search_sad_avx.c @@ -280,11 +280,18 @@ local_best_idx = _mm_extract_epi32(v_loidx_d, 0); } // Update the global minimum if the local minimum is smaller if (LIKELY(local_best_sad < best_sad)) { +#if defined(__GNUC__) && __GNUC__ >= 4 && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif new_bmv = ((const int_mv *)&v_these_mv_w)[local_best_idx]; +#if defined(__GNUC__) && __GNUC__ >= 4 && !defined(__clang__) +#pragma GCC diagnostic pop +#endif new_best_address = ((const uint8_t **)v_blocka)[local_best_idx]; best_sad = local_best_sad; } } diff --git a/media/libvpx/libvpx/vp9/encoder/x86/vp9_quantize_avx2.c b/media/libvpx/libvpx/vp9/encoder/x86/vp9_quantize_avx2.c --- a/media/libvpx/libvpx/vp9/encoder/x86/vp9_quantize_avx2.c +++ b/media/libvpx/libvpx/vp9/encoder/x86/vp9_quantize_avx2.c @@ -45,22 +45,19 @@ const __m256i iscan_plus_one = _mm256_sub_epi16(iscan, nzero_coeff0); return _mm256_and_si256(iscan_plus_one, nzero_coeff0); } void vp9_quantize_fp_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *round_ptr, - const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, - uint16_t *eob_ptr, const int16_t *scan, - const int16_t *iscan) { + const int16_t *round_ptr, const int16_t *quant_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { __m128i eob; __m256i round256, quant256, dequant256; __m256i eob256, thr256; (void)scan; - (void)skip_block; - assert(!skip_block); coeff_ptr += n_coeffs; iscan += n_coeffs; qcoeff_ptr += n_coeffs; dqcoeff_ptr += n_coeffs; diff --git a/media/libvpx/libvpx/vp9/encoder/x86/vp9_quantize_sse2.c b/media/libvpx/libvpx/vp9/encoder/x86/vp9_quantize_sse2.c --- a/media/libvpx/libvpx/vp9/encoder/x86/vp9_quantize_sse2.c +++ b/media/libvpx/libvpx/vp9/encoder/x86/vp9_quantize_sse2.c @@ -16,24 +16,21 @@ #include "vpx/vpx_integer.h" #include "vpx_dsp/vpx_dsp_common.h" #include "vpx_dsp/x86/bitdepth_conversion_sse2.h" void vp9_quantize_fp_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *round_ptr, - const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, - uint16_t *eob_ptr, const int16_t *scan, - const int16_t *iscan) { + const int16_t *round_ptr, const int16_t *quant_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { __m128i zero; __m128i thr; int nzflag; __m128i eob; __m128i round, quant, dequant; (void)scan; - (void)skip_block; - assert(!skip_block); coeff_ptr += n_coeffs; iscan += n_coeffs; qcoeff_ptr += n_coeffs; dqcoeff_ptr += n_coeffs; diff --git a/media/libvpx/libvpx/vp9/encoder/x86/vp9_quantize_ssse3_x86_64.asm b/media/libvpx/libvpx/vp9/encoder/x86/vp9_quantize_ssse3_x86_64.asm --- a/media/libvpx/libvpx/vp9/encoder/x86/vp9_quantize_ssse3_x86_64.asm +++ b/media/libvpx/libvpx/vp9/encoder/x86/vp9_quantize_ssse3_x86_64.asm @@ -17,22 +17,22 @@ pw_1: times 8 dw 1 SECTION .text %macro QUANTIZE_FP 2 -cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, skip, round, quant, \ +cglobal quantize_%1, 0, %2, 15, coeff, ncoeff, round, quant, \ qcoeff, dqcoeff, dequant, \ eob, scan, iscan ; actual quantize loop - setup pointers, rounders, etc. movifnidn coeffq, coeffmp movifnidn ncoeffq, ncoeffmp - mov r2, dequantmp movifnidn roundq, roundmp movifnidn quantq, quantmp mova m1, [roundq] ; m1 = round mova m2, [quantq] ; m2 = quant + mov r2, dequantmp %ifidn %1, fp_32x32 pcmpeqw m5, m5 psrlw m5, 15 paddw m1, m5 psrlw m1, 1 ; m1 = (m1 + 1) / 2 diff --git a/media/libvpx/libvpx/vp9/ratectrl_rtc.h b/media/libvpx/libvpx/vp9/ratectrl_rtc.h --- a/media/libvpx/libvpx/vp9/ratectrl_rtc.h +++ b/media/libvpx/libvpx/vp9/ratectrl_rtc.h @@ -20,83 +20,48 @@ #include "vp9/vp9_iface_common.h" #include "vp9/encoder/vp9_aq_cyclicrefresh.h" #include "vp9/encoder/vp9_encoder.h" #include "vp9/encoder/vp9_firstpass.h" #include "vp9/vp9_cx_iface.h" +#include "vpx/internal/vpx_ratectrl_rtc.h" #include "vpx_mem/vpx_mem.h" namespace libvpx { -struct VP9RateControlRtcConfig { +struct VP9RateControlRtcConfig : public VpxRateControlRtcConfig { public: VP9RateControlRtcConfig() { - width = 1280; - height = 720; - max_quantizer = 63; - min_quantizer = 2; - target_bandwidth = 1000; - buf_initial_sz = 600; - buf_optimal_sz = 600; - buf_sz = 1000; - undershoot_pct = overshoot_pct = 50; - max_intra_bitrate_pct = 50; - max_inter_bitrate_pct = 0; - framerate = 30.0; - ss_number_layers = ts_number_layers = 1; - rc_mode = VPX_CBR; - aq_mode = 0; vp9_zero(max_quantizers); vp9_zero(min_quantizers); vp9_zero(scaling_factor_den); vp9_zero(scaling_factor_num); vp9_zero(layer_target_bitrate); vp9_zero(ts_rate_decimator); scaling_factor_num[0] = 1; scaling_factor_den[0] = 1; - layer_target_bitrate[0] = static_cast(target_bandwidth); max_quantizers[0] = max_quantizer; min_quantizers[0] = min_quantizer; - ts_rate_decimator[0] = 1; } - int width; - int height; - // 0-63 - int max_quantizer; - int min_quantizer; - int64_t target_bandwidth; - int64_t buf_initial_sz; - int64_t buf_optimal_sz; - int64_t buf_sz; - int undershoot_pct; - int overshoot_pct; - int max_intra_bitrate_pct; - int max_inter_bitrate_pct; - double framerate; // Number of spatial layers int ss_number_layers; // Number of temporal layers int ts_number_layers; int max_quantizers[VPX_MAX_LAYERS]; int min_quantizers[VPX_MAX_LAYERS]; int scaling_factor_num[VPX_SS_MAX_LAYERS]; int scaling_factor_den[VPX_SS_MAX_LAYERS]; - int layer_target_bitrate[VPX_MAX_LAYERS]; - int ts_rate_decimator[VPX_TS_MAX_LAYERS]; - // vbr, cbr - enum vpx_rc_mode rc_mode; - int aq_mode; }; struct VP9FrameParamsQpRTC { FRAME_TYPE frame_type; int spatial_layer_id; int temporal_layer_id; }; // This interface allows using VP9 real-time rate control without initializing -// the encoder. To use this interface, you need to link with libvp9rc.a. +// the encoder. To use this interface, you need to link with libvpxrc.a. // // #include "vp9/ratectrl_rtc.h" // VP9RateControlRTC rc_api; // VP9RateControlRtcConfig cfg; // VP9FrameParamsQpRTC frame_params; diff --git a/media/libvpx/libvpx/vp9/ratectrl_rtc.cc b/media/libvpx/libvpx/vp9/ratectrl_rtc.cc --- a/media/libvpx/libvpx/vp9/ratectrl_rtc.cc +++ b/media/libvpx/libvpx/vp9/ratectrl_rtc.cc @@ -23,19 +23,26 @@ const VP9RateControlRtcConfig &cfg) { std::unique_ptr rc_api(new (std::nothrow) VP9RateControlRTC()); if (!rc_api) return nullptr; rc_api->cpi_ = static_cast(vpx_memalign(32, sizeof(*cpi_))); - if (!rc_api->cpi_) return nullptr; + if (!rc_api->cpi_) { + rc_api.reset(); + return nullptr; + } vp9_zero(*rc_api->cpi_); rc_api->InitRateControl(cfg); if (cfg.aq_mode) { VP9_COMP *const cpi = rc_api->cpi_; cpi->segmentation_map = static_cast( vpx_calloc(cpi->common.mi_rows * cpi->common.mi_cols, sizeof(*cpi->segmentation_map))); + if (!cpi->segmentation_map) { + rc_api.reset(); + return nullptr; + } cpi->cyclic_refresh = vp9_cyclic_refresh_alloc(cpi->common.mi_rows, cpi->common.mi_cols); cpi->cyclic_refresh->content_mode = 0; } return rc_api; @@ -103,11 +110,11 @@ cpi_->oxcf.rc_max_intra_bitrate_pct = rc_cfg.max_intra_bitrate_pct; cpi_->oxcf.rc_max_inter_bitrate_pct = rc_cfg.max_inter_bitrate_pct; cpi_->framerate = rc_cfg.framerate; cpi_->svc.number_spatial_layers = rc_cfg.ss_number_layers; cpi_->svc.number_temporal_layers = rc_cfg.ts_number_layers; - + vp9_set_mb_mi(cm, cm->width, cm->height); for (int sl = 0; sl < cpi_->svc.number_spatial_layers; ++sl) { for (int tl = 0; tl < cpi_->svc.number_temporal_layers; ++tl) { const int layer = LAYER_IDS_TO_IDX(sl, tl, cpi_->svc.number_temporal_layers); LAYER_CONTEXT *lc = &cpi_->svc.layer_context[layer]; diff --git a/media/libvpx/libvpx/vp9/simple_encode.h b/media/libvpx/libvpx/vp9/simple_encode.h --- a/media/libvpx/libvpx/vp9/simple_encode.h +++ b/media/libvpx/libvpx/vp9/simple_encode.h @@ -42,10 +42,30 @@ kRefFrameTypeFuture = 2, kRefFrameTypeMax = 3, kRefFrameTypeNone = -1, }; +enum VP9_LEVEL { + LEVEL_UNKNOWN = 0, + LEVEL_AUTO = 1, + LEVEL_1 = 10, + LEVEL_1_1 = 11, + LEVEL_2 = 20, + LEVEL_2_1 = 21, + LEVEL_3 = 30, + LEVEL_3_1 = 31, + LEVEL_4 = 40, + LEVEL_4_1 = 41, + LEVEL_5 = 50, + LEVEL_5_1 = 51, + LEVEL_5_2 = 52, + LEVEL_6 = 60, + LEVEL_6_1 = 61, + LEVEL_6_2 = 62, + LEVEL_MAX = 255 +}; + enum GopMapFlag { kGopMapFlagStart = 1 << 0, // Indicate this location is the start of a group of pictures. kGopMapFlagUseAltRef = 1 << 1, // Indicate this group of pictures will use an alt ref. Only set @@ -341,11 +361,12 @@ public: // When outfile_path is set, the encoder will output the bitstream in ivf // format. SimpleEncode(int frame_width, int frame_height, int frame_rate_num, int frame_rate_den, int target_bitrate, int num_frames, - const char *infile_path, const char *outfile_path = nullptr); + int target_level, const char *infile_path, + const char *outfile_path = nullptr); ~SimpleEncode(); SimpleEncode(SimpleEncode &) = delete; SimpleEncode &operator=(const SimpleEncode &) = delete; // Adjusts the encoder's coding speed. @@ -511,10 +532,11 @@ int frame_rate_num_; int frame_rate_den_; int target_bitrate_; int num_frames_; int encode_speed_; + int target_level_; std::FILE *in_file_; std::FILE *out_file_; std::unique_ptr impl_ptr_; diff --git a/media/libvpx/libvpx/vp9/simple_encode.cc b/media/libvpx/libvpx/vp9/simple_encode.cc --- a/media/libvpx/libvpx/vp9/simple_encode.cc +++ b/media/libvpx/libvpx/vp9/simple_encode.cc @@ -108,10 +108,11 @@ static VP9_COMP *init_encoder(const VP9EncoderConfig *oxcf, vpx_img_fmt_t img_fmt) { VP9_COMP *cpi; BufferPool *buffer_pool = (BufferPool *)vpx_calloc(1, sizeof(*buffer_pool)); + if (!buffer_pool) return NULL; vp9_initialize_enc(); cpi = vp9_create_compressor(oxcf, buffer_pool); vp9_update_compressor_with_img_fmt(cpi, img_fmt); return cpi; } @@ -780,15 +781,16 @@ } } static VP9EncoderConfig GetEncodeConfig( int frame_width, int frame_height, vpx_rational_t frame_rate, - int target_bitrate, int encode_speed, vpx_enc_pass enc_pass, + int target_bitrate, int encode_speed, int target_level, + vpx_enc_pass enc_pass, const std::vector &encode_config_list) { - VP9EncoderConfig oxcf = - vp9_get_encoder_config(frame_width, frame_height, frame_rate, - target_bitrate, encode_speed, enc_pass); + VP9EncoderConfig oxcf = vp9_get_encoder_config( + frame_width, frame_height, frame_rate, target_bitrate, encode_speed, + target_level, enc_pass); for (const auto &config : encode_config_list) { UpdateEncodeConfig(config, &oxcf); } if (enc_pass == VPX_RC_FIRST_PASS) { oxcf.lag_in_frames = 0; @@ -797,20 +799,21 @@ return oxcf; } SimpleEncode::SimpleEncode(int frame_width, int frame_height, int frame_rate_num, int frame_rate_den, - int target_bitrate, int num_frames, + int target_bitrate, int num_frames, int target_level, const char *infile_path, const char *outfile_path) { impl_ptr_ = std::unique_ptr(new EncodeImpl()); frame_width_ = frame_width; frame_height_ = frame_height; frame_rate_num_ = frame_rate_num; frame_rate_den_ = frame_rate_den; target_bitrate_ = target_bitrate; num_frames_ = num_frames; encode_speed_ = 0; + target_level_ = target_level; frame_coding_index_ = 0; show_frame_count_ = 0; key_frame_group_index_ = 0; @@ -858,23 +861,23 @@ } else { enc_pass = VPX_RC_LAST_PASS; } const vpx_rational_t frame_rate = make_vpx_rational(frame_rate_num_, frame_rate_den_); - const VP9EncoderConfig oxcf = - GetEncodeConfig(frame_width_, frame_height_, frame_rate, target_bitrate_, - encode_speed_, enc_pass, impl_ptr_->encode_config_list); + const VP9EncoderConfig oxcf = GetEncodeConfig( + frame_width_, frame_height_, frame_rate, target_bitrate_, encode_speed_, + target_level_, enc_pass, impl_ptr_->encode_config_list); vp9_dump_encoder_config(&oxcf, fp); return StatusOk; } void SimpleEncode::ComputeFirstPassStats() { vpx_rational_t frame_rate = make_vpx_rational(frame_rate_num_, frame_rate_den_); const VP9EncoderConfig oxcf = GetEncodeConfig( frame_width_, frame_height_, frame_rate, target_bitrate_, encode_speed_, - VPX_RC_FIRST_PASS, impl_ptr_->encode_config_list); + target_level_, VPX_RC_FIRST_PASS, impl_ptr_->encode_config_list); impl_ptr_->cpi = init_encoder(&oxcf, impl_ptr_->img_fmt); struct lookahead_ctx *lookahead = impl_ptr_->cpi->lookahead; int i; int use_highbitdepth = 0; const int num_rows_16x16 = get_num_unit_16x16(frame_height_); @@ -1036,11 +1039,11 @@ assert(impl_ptr_->first_pass_stats.size() > 0); vpx_rational_t frame_rate = make_vpx_rational(frame_rate_num_, frame_rate_den_); VP9EncoderConfig oxcf = GetEncodeConfig( frame_width_, frame_height_, frame_rate, target_bitrate_, encode_speed_, - VPX_RC_LAST_PASS, impl_ptr_->encode_config_list); + target_level_, VPX_RC_LAST_PASS, impl_ptr_->encode_config_list); vpx_fixed_buf_t stats; stats.buf = GetVectorData(impl_ptr_->first_pass_stats); stats.sz = sizeof(impl_ptr_->first_pass_stats[0]) * impl_ptr_->first_pass_stats.size(); @@ -1264,11 +1267,11 @@ const int allow_alt_ref = 1; vpx_rational_t frame_rate = make_vpx_rational(frame_rate_num_, frame_rate_den_); const VP9EncoderConfig oxcf = GetEncodeConfig( frame_width_, frame_height_, frame_rate, target_bitrate_, encode_speed_, - VPX_RC_LAST_PASS, impl_ptr_->encode_config_list); + target_level_, VPX_RC_LAST_PASS, impl_ptr_->encode_config_list); FRAME_INFO frame_info = vp9_get_frame_info(&oxcf); fps_init_first_pass_info(&twopass.first_pass_info, GetVectorData(impl_ptr_->first_pass_stats), num_frames_); unsigned int screen_area = frame_info.frame_width * frame_info.frame_height; @@ -1283,11 +1286,11 @@ static_cast(num_frames_) + 1); vpx_rational_t frame_rate = make_vpx_rational(frame_rate_num_, frame_rate_den_); const VP9EncoderConfig oxcf = GetEncodeConfig( frame_width_, frame_height_, frame_rate, target_bitrate_, encode_speed_, - VPX_RC_LAST_PASS, impl_ptr_->encode_config_list); + target_level_, VPX_RC_LAST_PASS, impl_ptr_->encode_config_list); TWO_PASS twopass; fps_init_first_pass_info(&twopass.first_pass_info, GetVectorData(impl_ptr_->first_pass_stats), num_frames_); std::vector key_frame_map(num_frames_, 0); diff --git a/media/libvpx/libvpx/vp9/vp9_cx_iface.h b/media/libvpx/libvpx/vp9/vp9_cx_iface.h --- a/media/libvpx/libvpx/vp9/vp9_cx_iface.h +++ b/media/libvpx/libvpx/vp9/vp9_cx_iface.h @@ -18,10 +18,11 @@ #endif VP9EncoderConfig vp9_get_encoder_config(int frame_width, int frame_height, vpx_rational_t frame_rate, int target_bitrate, int encode_speed, + int target_level, vpx_enc_pass enc_pass); void vp9_dump_encoder_config(const VP9EncoderConfig *oxcf, FILE *fp); FRAME_INFO vp9_get_frame_info(const VP9EncoderConfig *oxcf); diff --git a/media/libvpx/libvpx/vp9/vp9_cx_iface.c b/media/libvpx/libvpx/vp9/vp9_cx_iface.c --- a/media/libvpx/libvpx/vp9/vp9_cx_iface.c +++ b/media/libvpx/libvpx/vp9/vp9_cx_iface.c @@ -13,11 +13,10 @@ #include "./vpx_config.h" #include "vpx/vpx_encoder.h" #include "vpx/vpx_ext_ratectrl.h" #include "vpx_dsp/psnr.h" -#include "vpx_ports/vpx_once.h" #include "vpx_ports/static_assert.h" #include "vpx_ports/system_state.h" #include "vpx_util/vpx_timestamp.h" #include "vpx/internal/vpx_codec_internal.h" #include "./vpx_version.h" @@ -379,12 +378,12 @@ case VPX_IMG_FMT_I422: case VPX_IMG_FMT_I444: case VPX_IMG_FMT_I440: if (ctx->cfg.g_profile != (unsigned int)PROFILE_1) { ERROR( - "Invalid image format. I422, I444, I440, NV12 images are " - "not supported in profile."); + "Invalid image format. I422, I444, I440 images are not supported " + "in profile."); } break; case VPX_IMG_FMT_I42216: case VPX_IMG_FMT_I44416: case VPX_IMG_FMT_I44016: @@ -395,12 +394,12 @@ "not supported in profile."); } break; default: ERROR( - "Invalid image format. Only YV12, I420, I422, I444 images are " - "supported."); + "Invalid image format. Only YV12, I420, I422, I444, I440, NV12 " + "images are supported."); break; } if (img->d_w != ctx->cfg.g_w || img->d_h != ctx->cfg.g_h) ERROR("Image size must match encoder init configuration size"); @@ -521,12 +520,13 @@ oxcf->rc_mode = cfg->rc_end_usage; raw_target_rate = (unsigned int)((int64_t)oxcf->width * oxcf->height * oxcf->bit_depth * 3 * oxcf->init_framerate / 1000); - // Cap target bitrate to raw rate - cfg->rc_target_bitrate = VPXMIN(raw_target_rate, cfg->rc_target_bitrate); + // Cap target bitrate to raw rate or 1000Mbps, whichever is less + cfg->rc_target_bitrate = + VPXMIN(VPXMIN(raw_target_rate, cfg->rc_target_bitrate), 1000000); // Convert target bandwidth from Kbit/s to Bit/s oxcf->target_bandwidth = 1000 * (int64_t)cfg->rc_target_bitrate; oxcf->rc_max_intra_bitrate_pct = extra_cfg->rc_max_intra_bitrate_pct; oxcf->rc_max_inter_bitrate_pct = extra_cfg->rc_max_inter_bitrate_pct; @@ -1093,11 +1093,11 @@ priv->cfg = *ctx->config.enc; ctx->config.enc = &priv->cfg; } priv->extra_cfg = default_extra_cfg; - once(vp9_initialize_enc); + vp9_initialize_enc(); res = validate_config(priv, &priv->cfg, &priv->extra_cfg); if (res == VPX_CODEC_OK) { priv->pts_offset_initialized = 0; @@ -1356,24 +1356,15 @@ if (ctx->base.init_flags & VPX_CODEC_USE_PSNR) cpi->b_calculate_psnr = 1; if (img != NULL) { res = image2yuvconfig(img, &sd); - if (sd.y_width != ctx->cfg.g_w || sd.y_height != ctx->cfg.g_h) { - /* from vpx_encoder.h for g_w/g_h: - "Note that the frames passed as input to the encoder must have this - resolution" - */ - ctx->base.err_detail = "Invalid input frame resolution"; - res = VPX_CODEC_INVALID_PARAM; - } else { - // Store the original flags in to the frame buffer. Will extract the - // key frame flag when we actually encode this frame. - if (vp9_receive_raw_frame(cpi, flags | ctx->next_frame_flags, &sd, - dst_time_stamp, dst_end_time_stamp)) { - res = update_error_state(ctx, &cpi->common.error); - } + // Store the original flags in to the frame buffer. Will extract the + // key frame flag when we actually encode this frame. + if (vp9_receive_raw_frame(cpi, flags | ctx->next_frame_flags, &sd, + dst_time_stamp, dst_end_time_stamp)) { + res = update_error_state(ctx, &cpi->common.error); } ctx->next_frame_flags = 0; } cx_data = ctx->cx_data; @@ -2150,10 +2141,11 @@ } VP9EncoderConfig vp9_get_encoder_config(int frame_width, int frame_height, vpx_rational_t frame_rate, int target_bitrate, int encode_speed, + int target_level, vpx_enc_pass enc_pass) { /* This function will generate the same VP9EncoderConfig used by the * vpxenc command given below. * The configs in the vpxenc command corresponds to parameters of * vp9_get_encoder_config() as follows. @@ -2161,10 +2153,11 @@ * WIDTH: frame_width * HEIGHT: frame_height * FPS: frame_rate * BITRATE: target_bitrate * CPU_USED:encode_speed + * TARGET_LEVEL: target_level * * INPUT, OUTPUT, LIMIT will not affect VP9EncoderConfig * * vpxenc command: * INPUT=bus_cif.y4m @@ -2173,19 +2166,20 @@ * HEIGHT=288 * BITRATE=600 * FPS=30/1 * LIMIT=150 * CPU_USED=0 + * TARGET_LEVEL=0 * ./vpxenc --limit=$LIMIT --width=$WIDTH --height=$HEIGHT --fps=$FPS * --lag-in-frames=25 \ * --codec=vp9 --good --cpu-used=CPU_USED --threads=0 --profile=0 \ * --min-q=0 --max-q=63 --auto-alt-ref=1 --passes=2 --kf-max-dist=150 \ * --kf-min-dist=0 --drop-frame=0 --static-thresh=0 --bias-pct=50 \ * --minsection-pct=0 --maxsection-pct=150 --arnr-maxframes=7 --psnr \ * --arnr-strength=5 --sharpness=0 --undershoot-pct=100 --overshoot-pct=100 \ * --frame-parallel=0 --tile-columns=0 --cpu-used=0 --end-usage=vbr \ - * --target-bitrate=$BITRATE -o $OUTPUT $INPUT + * --target-bitrate=$BITRATE --target-level=0 -o $OUTPUT $INPUT */ VP9EncoderConfig oxcf; vp9_extracfg extra_cfg = get_extra_cfg(); vpx_codec_enc_cfg_t enc_cfg = get_enc_cfg( @@ -2199,10 +2193,11 @@ oxcf.max_threads = 0; oxcf.tile_columns = 0; oxcf.frame_parallel_decoding_mode = 0; oxcf.two_pass_vbrmax_section = 150; oxcf.speed = abs(encode_speed); + oxcf.target_level = target_level; return oxcf; } #define DUMP_STRUCT_VALUE(fp, structure, value) \ fprintf(fp, #value " %" PRId64 "\n", (int64_t)(structure)->value) diff --git a/media/libvpx/libvpx/vp9/vp9_dx_iface.c b/media/libvpx/libvpx/vp9/vp9_dx_iface.c --- a/media/libvpx/libvpx/vp9/vp9_dx_iface.c +++ b/media/libvpx/libvpx/vp9/vp9_dx_iface.c @@ -199,11 +199,11 @@ set_error_detail(ctx, error->has_detail ? error->detail : NULL); return error->error_code; } -static void init_buffer_callbacks(vpx_codec_alg_priv_t *ctx) { +static vpx_codec_err_t init_buffer_callbacks(vpx_codec_alg_priv_t *ctx) { VP9_COMMON *const cm = &ctx->pbi->common; BufferPool *const pool = cm->buffer_pool; cm->new_fb_idx = INVALID_IDX; cm->byte_alignment = ctx->byte_alignment; @@ -215,16 +215,20 @@ pool->cb_priv = ctx->ext_priv; } else { pool->get_fb_cb = vp9_get_frame_buffer; pool->release_fb_cb = vp9_release_frame_buffer; - if (vp9_alloc_internal_frame_buffers(&pool->int_frame_buffers)) + if (vp9_alloc_internal_frame_buffers(&pool->int_frame_buffers)) { vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR, "Failed to initialize internal frame buffers"); + return VPX_CODEC_MEM_ERROR; + } pool->cb_priv = &pool->int_frame_buffers; } + + return VPX_CODEC_OK; } static void set_default_ppflags(vp8_postproc_cfg_t *cfg) { cfg->post_proc_flag = VP8_DEBLOCK | VP8_DEMACROBLOCK; cfg->deblocking_level = 4; @@ -276,13 +280,11 @@ // If postprocessing was enabled by the application and a // configuration has not been provided, default it. if (!ctx->postproc_cfg_set && (ctx->base.init_flags & VPX_CODEC_USE_POSTPROC)) set_default_ppflags(&ctx->postproc_cfg); - init_buffer_callbacks(ctx); - - return VPX_CODEC_OK; + return init_buffer_callbacks(ctx); } static INLINE void check_resync(vpx_codec_alg_priv_t *const ctx, const VP9Decoder *const pbi) { // Clear resync flag if the decoder got a key frame or intra only frame. diff --git a/media/libvpx/libvpx/vp9/vp9cx.mk b/media/libvpx/libvpx/vp9/vp9cx.mk --- a/media/libvpx/libvpx/vp9/vp9cx.mk +++ b/media/libvpx/libvpx/vp9/vp9cx.mk @@ -111,10 +111,11 @@ VP9_CX_SRCS-$(HAVE_SSE4_1) += encoder/x86/temporal_filter_constants.h VP9_CX_SRCS-$(HAVE_SSE2) += encoder/x86/vp9_quantize_sse2.c VP9_CX_SRCS-$(HAVE_AVX2) += encoder/x86/vp9_quantize_avx2.c VP9_CX_SRCS-$(HAVE_AVX) += encoder/x86/vp9_diamond_search_sad_avx.c +VP9_CX_SRCS-$(HAVE_NEON) += encoder/arm/neon/vp9_diamond_search_sad_neon.c ifeq ($(CONFIG_VP9_HIGHBITDEPTH),yes) VP9_CX_SRCS-$(HAVE_SSE2) += encoder/x86/vp9_highbd_block_error_intrin_sse2.c VP9_CX_SRCS-$(HAVE_SSE4_1) += encoder/x86/highbd_temporal_filter_sse4.c endif @@ -125,10 +126,11 @@ VP9_CX_SRCS-$(HAVE_SSSE3) += encoder/x86/vp9_quantize_ssse3_x86_64.asm endif VP9_CX_SRCS-$(HAVE_SSE2) += encoder/x86/vp9_dct_intrin_sse2.c VP9_CX_SRCS-$(HAVE_SSSE3) += encoder/x86/vp9_frame_scale_ssse3.c +VP9_CX_SRCS-$(HAVE_NEON) += encoder/arm/neon/vp9_dct_neon.c ifeq ($(CONFIG_VP9_TEMPORAL_DENOISING),yes) VP9_CX_SRCS-$(HAVE_SSE2) += encoder/x86/vp9_denoiser_sse2.c VP9_CX_SRCS-$(HAVE_NEON) += encoder/arm/neon/vp9_denoiser_neon.c endif diff --git a/media/libvpx/libvpx/vpx/internal/vpx_codec_internal.h b/media/libvpx/libvpx/vpx/internal/vpx_codec_internal.h --- a/media/libvpx/libvpx/vpx/internal/vpx_codec_internal.h +++ b/media/libvpx/libvpx/vpx/internal/vpx_codec_internal.h @@ -433,13 +433,25 @@ #undef CLANG_ANALYZER_NORETURN #define CLANG_ANALYZER_NORETURN __attribute__((analyzer_noreturn)) #endif #endif +// Tells the compiler to perform `printf` format string checking if the +// compiler supports it; see the 'format' attribute in +// . +#define LIBVPX_FORMAT_PRINTF(string_index, first_to_check) +#if defined(__has_attribute) +#if __has_attribute(format) +#undef LIBVPX_FORMAT_PRINTF +#define LIBVPX_FORMAT_PRINTF(string_index, first_to_check) \ + __attribute__((__format__(__printf__, string_index, first_to_check))) +#endif +#endif + void vpx_internal_error(struct vpx_internal_error_info *info, - vpx_codec_err_t error, const char *fmt, - ...) CLANG_ANALYZER_NORETURN; + vpx_codec_err_t error, const char *fmt, ...) + LIBVPX_FORMAT_PRINTF(3, 4) CLANG_ANALYZER_NORETURN; #ifdef __cplusplus } // extern "C" #endif diff --git a/media/libvpx/libvpx/vpx/internal/vpx_ratectrl_rtc.h b/media/libvpx/libvpx/vpx/internal/vpx_ratectrl_rtc.h new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx/internal/vpx_ratectrl_rtc.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2021 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VPX_VPX_INTERNAL_VPX_RATECTRL_RTC_H_ +#define VPX_VPX_INTERNAL_VPX_RATECTRL_RTC_H_ + +#include "vpx/vpx_encoder.h" + +namespace libvpx { +struct VpxRateControlRtcConfig { + public: + VpxRateControlRtcConfig() { + width = 1280; + height = 720; + max_quantizer = 63; + min_quantizer = 2; + target_bandwidth = 1000; + buf_initial_sz = 600; + buf_optimal_sz = 600; + buf_sz = 1000; + undershoot_pct = overshoot_pct = 50; + max_intra_bitrate_pct = 50; + max_inter_bitrate_pct = 0; + framerate = 30.0; + ts_number_layers = 1; + rc_mode = VPX_CBR; + aq_mode = 0; + layer_target_bitrate[0] = static_cast(target_bandwidth); + ts_rate_decimator[0] = 1; + } + + int width; + int height; + // 0-63 + int max_quantizer; + int min_quantizer; + int64_t target_bandwidth; + int64_t buf_initial_sz; + int64_t buf_optimal_sz; + int64_t buf_sz; + int undershoot_pct; + int overshoot_pct; + int max_intra_bitrate_pct; + int max_inter_bitrate_pct; + double framerate; + // Number of temporal layers + int ts_number_layers; + int layer_target_bitrate[VPX_MAX_LAYERS]; + int ts_rate_decimator[VPX_TS_MAX_LAYERS]; + // vbr, cbr + enum vpx_rc_mode rc_mode; + int aq_mode; +}; +} // namespace libvpx +#endif // VPX_VPX_INTERNAL_VPX_RATECTRL_RTC_H_ diff --git a/media/libvpx/libvpx/vpx/src/vpx_encoder.c b/media/libvpx/libvpx/vpx/src/vpx_encoder.c --- a/media/libvpx/libvpx/vpx/src/vpx_encoder.c +++ b/media/libvpx/libvpx/vpx/src/vpx_encoder.c @@ -171,11 +171,11 @@ * consistency with the SSE unit's 64 bit precision. */ #include "vpx_ports/x86.h" #define FLOATING_POINT_INIT() \ do { \ - unsigned short x87_orig_mode = x87_set_double_precision(); + unsigned short x87_orig_mode = x87_set_double_precision() #define FLOATING_POINT_RESTORE() \ x87_set_control_word(x87_orig_mode); \ } \ while (0) diff --git a/media/libvpx/libvpx/vpx/vp8cx.h b/media/libvpx/libvpx/vpx/vp8cx.h --- a/media/libvpx/libvpx/vpx/vp8cx.h +++ b/media/libvpx/libvpx/vpx/vp8cx.h @@ -31,20 +31,36 @@ /*!\name Algorithm interface for VP8 * * This interface provides the capability to encode raw VP8 streams. * @{ */ + +/*!\brief A single instance of the VP8 encoder. + *\deprecated This access mechanism is provided for backwards compatibility; + * prefer vpx_codec_vp8_cx(). + */ extern vpx_codec_iface_t vpx_codec_vp8_cx_algo; + +/*!\brief The interface to the VP8 encoder. + */ extern vpx_codec_iface_t *vpx_codec_vp8_cx(void); /*!@} - end algorithm interface member group*/ /*!\name Algorithm interface for VP9 * * This interface provides the capability to encode raw VP9 streams. * @{ */ + +/*!\brief A single instance of the VP9 encoder. + *\deprecated This access mechanism is provided for backwards compatibility; + * prefer vpx_codec_vp9_cx(). + */ extern vpx_codec_iface_t vpx_codec_vp9_cx_algo; + +/*!\brief The interface to the VP9 encoder. + */ extern vpx_codec_iface_t *vpx_codec_vp9_cx(void); /*!@} - end algorithm interface member group*/ /* * Algorithm Flags @@ -492,22 +508,10 @@ * * Supported in codecs: VP9 */ VP9E_SET_COLOR_SPACE, - /*!\brief Codec control function to set temporal layering mode. - * \note Valid ranges: 0..3, default is "0" - * (VP9E_TEMPORAL_LAYERING_MODE_NOLAYERING). - * 0 = VP9E_TEMPORAL_LAYERING_MODE_NOLAYERING - * 1 = VP9E_TEMPORAL_LAYERING_MODE_BYPASS - * 2 = VP9E_TEMPORAL_LAYERING_MODE_0101 - * 3 = VP9E_TEMPORAL_LAYERING_MODE_0212 - * - * Supported in codecs: VP9 - */ - VP9E_SET_TEMPORAL_LAYERING_MODE, - /*!\brief Codec control function to set minimum interval between GF/ARF frames * * By default the value is set as 4. * * Supported in codecs: VP9 @@ -740,10 +744,21 @@ * The size of the array passed in should be #VPX_SS_MAX_LAYERS. * * Supported in codecs: VP9 */ VP9E_GET_LAST_QUANTIZER_SVC_LAYERS, + + /*!\brief Codec control to disable internal features in rate control. + * + * This will turn off cyclic refresh for vp8. + * + * With this, the rate control is expected to work exactly the same as the + * interface provided in vp8_ratectrl_rtc.cc/h + * + * Supported in codecs: VP8 + */ + VP8E_SET_RTC_EXTERNAL_RATECTRL, }; /*!\brief vpx 1-D scaling mode * * This set of constants define 1-D vpx scaling modes @@ -795,12 +810,12 @@ * region within a frame. */ unsigned char *roi_map; unsigned int rows; /**< Number of rows. */ unsigned int cols; /**< Number of columns. */ /*! VP8 only uses the first 4 segments. VP9 uses 8 segments. */ - int delta_q[8]; /**< Quantizer deltas. */ - int delta_lf[8]; /**< Loop filter deltas. */ + int delta_q[8]; /**< Quantizer deltas. Valid range: [-63, 63].*/ + int delta_lf[8]; /**< Loop filter deltas. Valid range: [-63, 63].*/ /*! skip and ref frame segment is only used in VP9. */ int skip[8]; /**< Skip this block. */ int ref_frame[8]; /**< Reference frame for this block. */ /*! Static breakout threshold for each segment. Only used in VP8. */ unsigned int static_threshold[4]; @@ -939,32 +954,16 @@ * Defines the data types that VP8E control functions take. Note that * additional common controls are defined in vp8.h * */ -VPX_CTRL_USE_TYPE(VP8E_SET_FRAME_FLAGS, int) -#define VPX_CTRL_VP8E_SET_FRAME_FLAGS -VPX_CTRL_USE_TYPE(VP8E_SET_TEMPORAL_LAYER_ID, int) -#define VPX_CTRL_VP8E_SET_TEMPORAL_LAYER_ID VPX_CTRL_USE_TYPE(VP8E_SET_ROI_MAP, vpx_roi_map_t *) #define VPX_CTRL_VP8E_SET_ROI_MAP -VPX_CTRL_USE_TYPE(VP9E_SET_ROI_MAP, vpx_roi_map_t *) -#define VPX_CTRL_VP9E_SET_ROI_MAP VPX_CTRL_USE_TYPE(VP8E_SET_ACTIVEMAP, vpx_active_map_t *) #define VPX_CTRL_VP8E_SET_ACTIVEMAP VPX_CTRL_USE_TYPE(VP8E_SET_SCALEMODE, vpx_scaling_mode_t *) #define VPX_CTRL_VP8E_SET_SCALEMODE - -VPX_CTRL_USE_TYPE(VP9E_SET_SVC, int) -#define VPX_CTRL_VP9E_SET_SVC -VPX_CTRL_USE_TYPE(VP9E_SET_SVC_PARAMETERS, void *) -#define VPX_CTRL_VP9E_SET_SVC_PARAMETERS -VPX_CTRL_USE_TYPE(VP9E_REGISTER_CX_CALLBACK, void *) -#define VPX_CTRL_VP9E_REGISTER_CX_CALLBACK -VPX_CTRL_USE_TYPE(VP9E_SET_SVC_LAYER_ID, vpx_svc_layer_id_t *) -#define VPX_CTRL_VP9E_SET_SVC_LAYER_ID - VPX_CTRL_USE_TYPE(VP8E_SET_CPUUSED, int) #define VPX_CTRL_VP8E_SET_CPUUSED VPX_CTRL_USE_TYPE(VP8E_SET_ENABLEAUTOALTREF, unsigned int) #define VPX_CTRL_VP8E_SET_ENABLEAUTOALTREF VPX_CTRL_USE_TYPE(VP8E_SET_NOISE_SENSITIVITY, unsigned int) @@ -973,144 +972,121 @@ #define VPX_CTRL_VP8E_SET_SHARPNESS VPX_CTRL_USE_TYPE(VP8E_SET_STATIC_THRESHOLD, unsigned int) #define VPX_CTRL_VP8E_SET_STATIC_THRESHOLD VPX_CTRL_USE_TYPE(VP8E_SET_TOKEN_PARTITIONS, int) /* vp8e_token_partitions */ #define VPX_CTRL_VP8E_SET_TOKEN_PARTITIONS - +VPX_CTRL_USE_TYPE(VP8E_GET_LAST_QUANTIZER, int *) +#define VPX_CTRL_VP8E_GET_LAST_QUANTIZER +VPX_CTRL_USE_TYPE(VP8E_GET_LAST_QUANTIZER_64, int *) +#define VPX_CTRL_VP8E_GET_LAST_QUANTIZER_64 VPX_CTRL_USE_TYPE(VP8E_SET_ARNR_MAXFRAMES, unsigned int) #define VPX_CTRL_VP8E_SET_ARNR_MAXFRAMES VPX_CTRL_USE_TYPE(VP8E_SET_ARNR_STRENGTH, unsigned int) #define VPX_CTRL_VP8E_SET_ARNR_STRENGTH VPX_CTRL_USE_TYPE_DEPRECATED(VP8E_SET_ARNR_TYPE, unsigned int) #define VPX_CTRL_VP8E_SET_ARNR_TYPE VPX_CTRL_USE_TYPE(VP8E_SET_TUNING, int) /* vp8e_tuning */ #define VPX_CTRL_VP8E_SET_TUNING VPX_CTRL_USE_TYPE(VP8E_SET_CQ_LEVEL, unsigned int) #define VPX_CTRL_VP8E_SET_CQ_LEVEL - +VPX_CTRL_USE_TYPE(VP8E_SET_MAX_INTRA_BITRATE_PCT, unsigned int) +#define VPX_CTRL_VP8E_SET_MAX_INTRA_BITRATE_PCT +VPX_CTRL_USE_TYPE(VP8E_SET_FRAME_FLAGS, int) +#define VPX_CTRL_VP8E_SET_FRAME_FLAGS +VPX_CTRL_USE_TYPE(VP9E_SET_MAX_INTER_BITRATE_PCT, unsigned int) +#define VPX_CTRL_VP9E_SET_MAX_INTER_BITRATE_PCT +VPX_CTRL_USE_TYPE(VP9E_SET_GF_CBR_BOOST_PCT, unsigned int) +#define VPX_CTRL_VP9E_SET_GF_CBR_BOOST_PCT +VPX_CTRL_USE_TYPE(VP8E_SET_TEMPORAL_LAYER_ID, int) +#define VPX_CTRL_VP8E_SET_TEMPORAL_LAYER_ID +VPX_CTRL_USE_TYPE(VP8E_SET_SCREEN_CONTENT_MODE, unsigned int) +#define VPX_CTRL_VP8E_SET_SCREEN_CONTENT_MODE +VPX_CTRL_USE_TYPE(VP9E_SET_LOSSLESS, unsigned int) +#define VPX_CTRL_VP9E_SET_LOSSLESS VPX_CTRL_USE_TYPE(VP9E_SET_TILE_COLUMNS, int) #define VPX_CTRL_VP9E_SET_TILE_COLUMNS VPX_CTRL_USE_TYPE(VP9E_SET_TILE_ROWS, int) #define VPX_CTRL_VP9E_SET_TILE_ROWS - -VPX_CTRL_USE_TYPE(VP9E_SET_TPL, int) -#define VPX_CTRL_VP9E_SET_TPL - -VPX_CTRL_USE_TYPE(VP8E_GET_LAST_QUANTIZER, int *) -#define VPX_CTRL_VP8E_GET_LAST_QUANTIZER -VPX_CTRL_USE_TYPE(VP8E_GET_LAST_QUANTIZER_64, int *) -#define VPX_CTRL_VP8E_GET_LAST_QUANTIZER_64 -VPX_CTRL_USE_TYPE(VP9E_GET_LAST_QUANTIZER_SVC_LAYERS, int *) -#define VPX_CTRL_VP9E_GET_LAST_QUANTIZER_SVC_LAYERS - -VPX_CTRL_USE_TYPE(VP9E_GET_SVC_LAYER_ID, vpx_svc_layer_id_t *) -#define VPX_CTRL_VP9E_GET_SVC_LAYER_ID - -VPX_CTRL_USE_TYPE(VP8E_SET_MAX_INTRA_BITRATE_PCT, unsigned int) -#define VPX_CTRL_VP8E_SET_MAX_INTRA_BITRATE_PCT -VPX_CTRL_USE_TYPE(VP9E_SET_MAX_INTER_BITRATE_PCT, unsigned int) -#define VPX_CTRL_VP9E_SET_MAX_INTER_BITRATE_PCT - -VPX_CTRL_USE_TYPE(VP8E_SET_GF_CBR_BOOST_PCT, unsigned int) -#define VPX_CTRL_VP8E_SET_GF_CBR_BOOST_PCT - -VPX_CTRL_USE_TYPE(VP8E_SET_SCREEN_CONTENT_MODE, unsigned int) -#define VPX_CTRL_VP8E_SET_SCREEN_CONTENT_MODE - -VPX_CTRL_USE_TYPE(VP9E_SET_GF_CBR_BOOST_PCT, unsigned int) -#define VPX_CTRL_VP9E_SET_GF_CBR_BOOST_PCT - -VPX_CTRL_USE_TYPE(VP9E_SET_LOSSLESS, unsigned int) -#define VPX_CTRL_VP9E_SET_LOSSLESS - VPX_CTRL_USE_TYPE(VP9E_SET_FRAME_PARALLEL_DECODING, unsigned int) #define VPX_CTRL_VP9E_SET_FRAME_PARALLEL_DECODING - VPX_CTRL_USE_TYPE(VP9E_SET_AQ_MODE, unsigned int) #define VPX_CTRL_VP9E_SET_AQ_MODE - -VPX_CTRL_USE_TYPE(VP9E_SET_ALT_REF_AQ, int) -#define VPX_CTRL_VP9E_SET_ALT_REF_AQ - VPX_CTRL_USE_TYPE(VP9E_SET_FRAME_PERIODIC_BOOST, unsigned int) #define VPX_CTRL_VP9E_SET_FRAME_PERIODIC_BOOST - VPX_CTRL_USE_TYPE(VP9E_SET_NOISE_SENSITIVITY, unsigned int) #define VPX_CTRL_VP9E_SET_NOISE_SENSITIVITY - +VPX_CTRL_USE_TYPE(VP9E_SET_SVC, int) +#define VPX_CTRL_VP9E_SET_SVC +VPX_CTRL_USE_TYPE(VP9E_SET_ROI_MAP, vpx_roi_map_t *) +#define VPX_CTRL_VP9E_SET_ROI_MAP +VPX_CTRL_USE_TYPE(VP9E_SET_SVC_PARAMETERS, void *) +#define VPX_CTRL_VP9E_SET_SVC_PARAMETERS +VPX_CTRL_USE_TYPE(VP9E_SET_SVC_LAYER_ID, vpx_svc_layer_id_t *) +#define VPX_CTRL_VP9E_SET_SVC_LAYER_ID VPX_CTRL_USE_TYPE(VP9E_SET_TUNE_CONTENT, int) /* vp9e_tune_content */ #define VPX_CTRL_VP9E_SET_TUNE_CONTENT - +VPX_CTRL_USE_TYPE(VP9E_GET_SVC_LAYER_ID, vpx_svc_layer_id_t *) +#define VPX_CTRL_VP9E_GET_SVC_LAYER_ID +VPX_CTRL_USE_TYPE(VP9E_REGISTER_CX_CALLBACK, void *) +#define VPX_CTRL_VP9E_REGISTER_CX_CALLBACK VPX_CTRL_USE_TYPE(VP9E_SET_COLOR_SPACE, int) #define VPX_CTRL_VP9E_SET_COLOR_SPACE - VPX_CTRL_USE_TYPE(VP9E_SET_MIN_GF_INTERVAL, unsigned int) #define VPX_CTRL_VP9E_SET_MIN_GF_INTERVAL - VPX_CTRL_USE_TYPE(VP9E_SET_MAX_GF_INTERVAL, unsigned int) #define VPX_CTRL_VP9E_SET_MAX_GF_INTERVAL - VPX_CTRL_USE_TYPE(VP9E_GET_ACTIVEMAP, vpx_active_map_t *) #define VPX_CTRL_VP9E_GET_ACTIVEMAP - VPX_CTRL_USE_TYPE(VP9E_SET_COLOR_RANGE, int) #define VPX_CTRL_VP9E_SET_COLOR_RANGE - VPX_CTRL_USE_TYPE(VP9E_SET_SVC_REF_FRAME_CONFIG, vpx_svc_ref_frame_config_t *) #define VPX_CTRL_VP9E_SET_SVC_REF_FRAME_CONFIG - VPX_CTRL_USE_TYPE(VP9E_SET_RENDER_SIZE, int *) #define VPX_CTRL_VP9E_SET_RENDER_SIZE - VPX_CTRL_USE_TYPE(VP9E_SET_TARGET_LEVEL, unsigned int) #define VPX_CTRL_VP9E_SET_TARGET_LEVEL - VPX_CTRL_USE_TYPE(VP9E_SET_ROW_MT, unsigned int) #define VPX_CTRL_VP9E_SET_ROW_MT - VPX_CTRL_USE_TYPE(VP9E_GET_LEVEL, int *) #define VPX_CTRL_VP9E_GET_LEVEL - -VPX_CTRL_USE_TYPE(VP9E_GET_LOOPFILTER_LEVEL, int *) -#define VPX_CTRL_VP9E_GET_LOOPFILTER_LEVEL - +VPX_CTRL_USE_TYPE(VP9E_SET_ALT_REF_AQ, int) +#define VPX_CTRL_VP9E_SET_ALT_REF_AQ +VPX_CTRL_USE_TYPE(VP8E_SET_GF_CBR_BOOST_PCT, unsigned int) +#define VPX_CTRL_VP8E_SET_GF_CBR_BOOST_PCT VPX_CTRL_USE_TYPE(VP9E_ENABLE_MOTION_VECTOR_UNIT_TEST, unsigned int) #define VPX_CTRL_VP9E_ENABLE_MOTION_VECTOR_UNIT_TEST - VPX_CTRL_USE_TYPE(VP9E_SET_SVC_INTER_LAYER_PRED, unsigned int) #define VPX_CTRL_VP9E_SET_SVC_INTER_LAYER_PRED - VPX_CTRL_USE_TYPE(VP9E_SET_SVC_FRAME_DROP_LAYER, vpx_svc_frame_drop_t *) #define VPX_CTRL_VP9E_SET_SVC_FRAME_DROP_LAYER - VPX_CTRL_USE_TYPE(VP9E_GET_SVC_REF_FRAME_CONFIG, vpx_svc_ref_frame_config_t *) #define VPX_CTRL_VP9E_GET_SVC_REF_FRAME_CONFIG - VPX_CTRL_USE_TYPE(VP9E_SET_SVC_GF_TEMPORAL_REF, unsigned int) #define VPX_CTRL_VP9E_SET_SVC_GF_TEMPORAL_REF - VPX_CTRL_USE_TYPE(VP9E_SET_SVC_SPATIAL_LAYER_SYNC, vpx_svc_spatial_layer_sync_t *) #define VPX_CTRL_VP9E_SET_SVC_SPATIAL_LAYER_SYNC - +VPX_CTRL_USE_TYPE(VP9E_SET_TPL, int) +#define VPX_CTRL_VP9E_SET_TPL VPX_CTRL_USE_TYPE(VP9E_SET_POSTENCODE_DROP, unsigned int) #define VPX_CTRL_VP9E_SET_POSTENCODE_DROP - VPX_CTRL_USE_TYPE(VP9E_SET_DELTA_Q_UV, int) #define VPX_CTRL_VP9E_SET_DELTA_Q_UV - VPX_CTRL_USE_TYPE(VP9E_SET_DISABLE_OVERSHOOT_MAXQ_CBR, int) #define VPX_CTRL_VP9E_SET_DISABLE_OVERSHOOT_MAXQ_CBR - VPX_CTRL_USE_TYPE(VP9E_SET_DISABLE_LOOPFILTER, int) #define VPX_CTRL_VP9E_SET_DISABLE_LOOPFILTER - +VPX_CTRL_USE_TYPE(VP9E_SET_EXTERNAL_RATE_CONTROL, vpx_rc_funcs_t *) +#define VPX_CTRL_VP9E_SET_EXTERNAL_RATE_CONTROL VPX_CTRL_USE_TYPE(VP9E_SET_RTC_EXTERNAL_RATECTRL, int) #define VPX_CTRL_VP9E_SET_RTC_EXTERNAL_RATECTRL - -VPX_CTRL_USE_TYPE(VP9E_SET_EXTERNAL_RATE_CONTROL, vpx_rc_funcs_t *) -#define VPX_CTRL_VP9E_SET_EXTERNAL_RATE_CONTROL +VPX_CTRL_USE_TYPE(VP9E_GET_LOOPFILTER_LEVEL, int *) +#define VPX_CTRL_VP9E_GET_LOOPFILTER_LEVEL +VPX_CTRL_USE_TYPE(VP9E_GET_LAST_QUANTIZER_SVC_LAYERS, int *) +#define VPX_CTRL_VP9E_GET_LAST_QUANTIZER_SVC_LAYERS +VPX_CTRL_USE_TYPE(VP8E_SET_RTC_EXTERNAL_RATECTRL, int) +#define VPX_CTRL_VP8E_SET_RTC_EXTERNAL_RATECTRL /*!\endcond */ /*! @} - end defgroup vp8_encoder */ #ifdef __cplusplus } // extern "C" diff --git a/media/libvpx/libvpx/vpx/vp8dx.h b/media/libvpx/libvpx/vpx/vp8dx.h --- a/media/libvpx/libvpx/vpx/vp8dx.h +++ b/media/libvpx/libvpx/vpx/vp8dx.h @@ -30,20 +30,36 @@ /*!\name Algorithm interface for VP8 * * This interface provides the capability to decode VP8 streams. * @{ */ + +/*!\brief A single instance of the VP8 decoder. + *\deprecated This access mechanism is provided for backwards compatibility; + * prefer vpx_codec_vp8_dx(). + */ extern vpx_codec_iface_t vpx_codec_vp8_dx_algo; + +/*!\brief The interface to the VP8 decoder. + */ extern vpx_codec_iface_t *vpx_codec_vp8_dx(void); /*!@} - end algorithm interface member group*/ /*!\name Algorithm interface for VP9 * * This interface provides the capability to decode VP9 streams. * @{ */ + +/*!\brief A single instance of the VP9 decoder. + *\deprecated This access mechanism is provided for backwards compatibility; + * prefer vpx_codec_vp9_dx(). + */ extern vpx_codec_iface_t vpx_codec_vp9_dx_algo; + +/*!\brief The interface to the VP9 decoder. + */ extern vpx_codec_iface_t *vpx_codec_vp9_dx(void); /*!@} - end algorithm interface member group*/ /*!\enum vp8_dec_control_id * \brief VP8 decoder control functions @@ -175,32 +191,34 @@ #define VPX_CTRL_VP8D_GET_LAST_REF_UPDATES VPX_CTRL_USE_TYPE(VP8D_GET_FRAME_CORRUPTED, int *) #define VPX_CTRL_VP8D_GET_FRAME_CORRUPTED VPX_CTRL_USE_TYPE(VP8D_GET_LAST_REF_USED, int *) #define VPX_CTRL_VP8D_GET_LAST_REF_USED -VPX_CTRL_USE_TYPE(VPXD_GET_LAST_QUANTIZER, int *) -#define VPX_CTRL_VPXD_GET_LAST_QUANTIZER VPX_CTRL_USE_TYPE(VPXD_SET_DECRYPTOR, vpx_decrypt_init *) #define VPX_CTRL_VPXD_SET_DECRYPTOR VPX_CTRL_USE_TYPE(VP8D_SET_DECRYPTOR, vpx_decrypt_init *) #define VPX_CTRL_VP8D_SET_DECRYPTOR +VPX_CTRL_USE_TYPE(VP9D_GET_FRAME_SIZE, int *) +#define VPX_CTRL_VP9D_GET_FRAME_SIZE VPX_CTRL_USE_TYPE(VP9D_GET_DISPLAY_SIZE, int *) #define VPX_CTRL_VP9D_GET_DISPLAY_SIZE VPX_CTRL_USE_TYPE(VP9D_GET_BIT_DEPTH, unsigned int *) #define VPX_CTRL_VP9D_GET_BIT_DEPTH -VPX_CTRL_USE_TYPE(VP9D_GET_FRAME_SIZE, int *) -#define VPX_CTRL_VP9D_GET_FRAME_SIZE +VPX_CTRL_USE_TYPE(VP9_SET_BYTE_ALIGNMENT, int) +#define VPX_CTRL_VP9_SET_BYTE_ALIGNMENT VPX_CTRL_USE_TYPE(VP9_INVERT_TILE_DECODE_ORDER, int) #define VPX_CTRL_VP9_INVERT_TILE_DECODE_ORDER -#define VPX_CTRL_VP9_DECODE_SVC_SPATIAL_LAYER -VPX_CTRL_USE_TYPE(VP9_DECODE_SVC_SPATIAL_LAYER, int) +VPX_CTRL_USE_TYPE(VP9_SET_SKIP_LOOP_FILTER, int) #define VPX_CTRL_VP9_SET_SKIP_LOOP_FILTER -VPX_CTRL_USE_TYPE(VP9_SET_SKIP_LOOP_FILTER, int) -#define VPX_CTRL_VP9_DECODE_SET_ROW_MT +VPX_CTRL_USE_TYPE(VP9_DECODE_SVC_SPATIAL_LAYER, int) +#define VPX_CTRL_VP9_DECODE_SVC_SPATIAL_LAYER +VPX_CTRL_USE_TYPE(VPXD_GET_LAST_QUANTIZER, int *) +#define VPX_CTRL_VPXD_GET_LAST_QUANTIZER VPX_CTRL_USE_TYPE(VP9D_SET_ROW_MT, int) +#define VPX_CTRL_VP9_DECODE_SET_ROW_MT +VPX_CTRL_USE_TYPE(VP9D_SET_LOOP_FILTER_OPT, int) #define VPX_CTRL_VP9_SET_LOOP_FILTER_OPT -VPX_CTRL_USE_TYPE(VP9D_SET_LOOP_FILTER_OPT, int) /*!\endcond */ /*! @} - end defgroup vp8_decoder */ #ifdef __cplusplus diff --git a/media/libvpx/libvpx/vpx/vpx_codec.mk b/media/libvpx/libvpx/vpx/vpx_codec.mk --- a/media/libvpx/libvpx/vpx/vpx_codec.mk +++ b/media/libvpx/libvpx/vpx/vpx_codec.mk @@ -31,10 +31,11 @@ API_SRCS-yes += src/vpx_decoder.c API_SRCS-yes += vpx_decoder.h API_SRCS-yes += src/vpx_encoder.c API_SRCS-yes += vpx_encoder.h API_SRCS-yes += internal/vpx_codec_internal.h +API_SRCS-yes += internal/vpx_ratectrl_rtc.h API_SRCS-yes += src/vpx_codec.c API_SRCS-yes += src/vpx_image.c API_SRCS-yes += vpx_codec.h API_SRCS-yes += vpx_codec.mk API_SRCS-yes += vpx_frame_buffer.h diff --git a/media/libvpx/libvpx/vpx_dsp/arm/fdct16x16_neon.h b/media/libvpx/libvpx/vpx_dsp/arm/fdct16x16_neon.h new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/arm/fdct16x16_neon.h @@ -0,0 +1,327 @@ +/* + * Copyright (c) 2017 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VPX_VPX_DSP_ARM_FDCT16X16_NEON_H_ +#define VPX_VPX_DSP_ARM_FDCT16X16_NEON_H_ + +#include + +static INLINE void load(const int16_t *a, int stride, int16x8_t *b /*[16]*/) { + b[0] = vld1q_s16(a); + a += stride; + b[1] = vld1q_s16(a); + a += stride; + b[2] = vld1q_s16(a); + a += stride; + b[3] = vld1q_s16(a); + a += stride; + b[4] = vld1q_s16(a); + a += stride; + b[5] = vld1q_s16(a); + a += stride; + b[6] = vld1q_s16(a); + a += stride; + b[7] = vld1q_s16(a); + a += stride; + b[8] = vld1q_s16(a); + a += stride; + b[9] = vld1q_s16(a); + a += stride; + b[10] = vld1q_s16(a); + a += stride; + b[11] = vld1q_s16(a); + a += stride; + b[12] = vld1q_s16(a); + a += stride; + b[13] = vld1q_s16(a); + a += stride; + b[14] = vld1q_s16(a); + a += stride; + b[15] = vld1q_s16(a); +} + +// Store 8 16x8 values, assuming stride == 16. +static INLINE void store(tran_low_t *a, const int16x8_t *b /*[8]*/) { + store_s16q_to_tran_low(a, b[0]); + a += 16; + store_s16q_to_tran_low(a, b[1]); + a += 16; + store_s16q_to_tran_low(a, b[2]); + a += 16; + store_s16q_to_tran_low(a, b[3]); + a += 16; + store_s16q_to_tran_low(a, b[4]); + a += 16; + store_s16q_to_tran_low(a, b[5]); + a += 16; + store_s16q_to_tran_low(a, b[6]); + a += 16; + store_s16q_to_tran_low(a, b[7]); +} + +// Load step of each pass. Add and subtract clear across the input, requiring +// all 16 values to be loaded. For the first pass it also multiplies by 4. + +// To maybe reduce register usage this could be combined with the load() step to +// get the first 4 and last 4 values, cross those, then load the middle 8 values +// and cross them. +static INLINE void cross_input(const int16x8_t *a /*[16]*/, + int16x8_t *b /*[16]*/, const int pass) { + if (pass == 0) { + b[0] = vshlq_n_s16(vaddq_s16(a[0], a[15]), 2); + b[1] = vshlq_n_s16(vaddq_s16(a[1], a[14]), 2); + b[2] = vshlq_n_s16(vaddq_s16(a[2], a[13]), 2); + b[3] = vshlq_n_s16(vaddq_s16(a[3], a[12]), 2); + b[4] = vshlq_n_s16(vaddq_s16(a[4], a[11]), 2); + b[5] = vshlq_n_s16(vaddq_s16(a[5], a[10]), 2); + b[6] = vshlq_n_s16(vaddq_s16(a[6], a[9]), 2); + b[7] = vshlq_n_s16(vaddq_s16(a[7], a[8]), 2); + + b[8] = vshlq_n_s16(vsubq_s16(a[7], a[8]), 2); + b[9] = vshlq_n_s16(vsubq_s16(a[6], a[9]), 2); + b[10] = vshlq_n_s16(vsubq_s16(a[5], a[10]), 2); + b[11] = vshlq_n_s16(vsubq_s16(a[4], a[11]), 2); + b[12] = vshlq_n_s16(vsubq_s16(a[3], a[12]), 2); + b[13] = vshlq_n_s16(vsubq_s16(a[2], a[13]), 2); + b[14] = vshlq_n_s16(vsubq_s16(a[1], a[14]), 2); + b[15] = vshlq_n_s16(vsubq_s16(a[0], a[15]), 2); + } else { + b[0] = vaddq_s16(a[0], a[15]); + b[1] = vaddq_s16(a[1], a[14]); + b[2] = vaddq_s16(a[2], a[13]); + b[3] = vaddq_s16(a[3], a[12]); + b[4] = vaddq_s16(a[4], a[11]); + b[5] = vaddq_s16(a[5], a[10]); + b[6] = vaddq_s16(a[6], a[9]); + b[7] = vaddq_s16(a[7], a[8]); + + b[8] = vsubq_s16(a[7], a[8]); + b[9] = vsubq_s16(a[6], a[9]); + b[10] = vsubq_s16(a[5], a[10]); + b[11] = vsubq_s16(a[4], a[11]); + b[12] = vsubq_s16(a[3], a[12]); + b[13] = vsubq_s16(a[2], a[13]); + b[14] = vsubq_s16(a[1], a[14]); + b[15] = vsubq_s16(a[0], a[15]); + } +} + +// Quarter round at the beginning of the second pass. Can't use vrshr (rounding) +// because this only adds 1, not 1 << 2. +static INLINE void partial_round_shift(int16x8_t *a /*[16]*/) { + const int16x8_t one = vdupq_n_s16(1); + a[0] = vshrq_n_s16(vaddq_s16(a[0], one), 2); + a[1] = vshrq_n_s16(vaddq_s16(a[1], one), 2); + a[2] = vshrq_n_s16(vaddq_s16(a[2], one), 2); + a[3] = vshrq_n_s16(vaddq_s16(a[3], one), 2); + a[4] = vshrq_n_s16(vaddq_s16(a[4], one), 2); + a[5] = vshrq_n_s16(vaddq_s16(a[5], one), 2); + a[6] = vshrq_n_s16(vaddq_s16(a[6], one), 2); + a[7] = vshrq_n_s16(vaddq_s16(a[7], one), 2); + a[8] = vshrq_n_s16(vaddq_s16(a[8], one), 2); + a[9] = vshrq_n_s16(vaddq_s16(a[9], one), 2); + a[10] = vshrq_n_s16(vaddq_s16(a[10], one), 2); + a[11] = vshrq_n_s16(vaddq_s16(a[11], one), 2); + a[12] = vshrq_n_s16(vaddq_s16(a[12], one), 2); + a[13] = vshrq_n_s16(vaddq_s16(a[13], one), 2); + a[14] = vshrq_n_s16(vaddq_s16(a[14], one), 2); + a[15] = vshrq_n_s16(vaddq_s16(a[15], one), 2); +} + +// fdct_round_shift((a +/- b) * c) +static INLINE void butterfly_one_coeff(const int16x8_t a, const int16x8_t b, + const tran_high_t c, int16x8_t *add, + int16x8_t *sub) { + const int32x4_t a0 = vmull_n_s16(vget_low_s16(a), c); + const int32x4_t a1 = vmull_n_s16(vget_high_s16(a), c); + const int32x4_t sum0 = vmlal_n_s16(a0, vget_low_s16(b), c); + const int32x4_t sum1 = vmlal_n_s16(a1, vget_high_s16(b), c); + const int32x4_t diff0 = vmlsl_n_s16(a0, vget_low_s16(b), c); + const int32x4_t diff1 = vmlsl_n_s16(a1, vget_high_s16(b), c); + const int16x4_t rounded0 = vqrshrn_n_s32(sum0, 14); + const int16x4_t rounded1 = vqrshrn_n_s32(sum1, 14); + const int16x4_t rounded2 = vqrshrn_n_s32(diff0, 14); + const int16x4_t rounded3 = vqrshrn_n_s32(diff1, 14); + *add = vcombine_s16(rounded0, rounded1); + *sub = vcombine_s16(rounded2, rounded3); +} + +// fdct_round_shift(a * c0 +/- b * c1) +static INLINE void butterfly_two_coeff(const int16x8_t a, const int16x8_t b, + const tran_coef_t c0, + const tran_coef_t c1, int16x8_t *add, + int16x8_t *sub) { + const int32x4_t a0 = vmull_n_s16(vget_low_s16(a), c0); + const int32x4_t a1 = vmull_n_s16(vget_high_s16(a), c0); + const int32x4_t a2 = vmull_n_s16(vget_low_s16(a), c1); + const int32x4_t a3 = vmull_n_s16(vget_high_s16(a), c1); + const int32x4_t sum0 = vmlal_n_s16(a2, vget_low_s16(b), c0); + const int32x4_t sum1 = vmlal_n_s16(a3, vget_high_s16(b), c0); + const int32x4_t diff0 = vmlsl_n_s16(a0, vget_low_s16(b), c1); + const int32x4_t diff1 = vmlsl_n_s16(a1, vget_high_s16(b), c1); + const int16x4_t rounded0 = vqrshrn_n_s32(sum0, 14); + const int16x4_t rounded1 = vqrshrn_n_s32(sum1, 14); + const int16x4_t rounded2 = vqrshrn_n_s32(diff0, 14); + const int16x4_t rounded3 = vqrshrn_n_s32(diff1, 14); + *add = vcombine_s16(rounded0, rounded1); + *sub = vcombine_s16(rounded2, rounded3); +} + +// Transpose 8x8 to a new location. Don't use transpose_neon.h because those +// are all in-place. +static INLINE void transpose_8x8(const int16x8_t *a /*[8]*/, + int16x8_t *b /*[8]*/) { + // Swap 16 bit elements. + const int16x8x2_t c0 = vtrnq_s16(a[0], a[1]); + const int16x8x2_t c1 = vtrnq_s16(a[2], a[3]); + const int16x8x2_t c2 = vtrnq_s16(a[4], a[5]); + const int16x8x2_t c3 = vtrnq_s16(a[6], a[7]); + + // Swap 32 bit elements. + const int32x4x2_t d0 = vtrnq_s32(vreinterpretq_s32_s16(c0.val[0]), + vreinterpretq_s32_s16(c1.val[0])); + const int32x4x2_t d1 = vtrnq_s32(vreinterpretq_s32_s16(c0.val[1]), + vreinterpretq_s32_s16(c1.val[1])); + const int32x4x2_t d2 = vtrnq_s32(vreinterpretq_s32_s16(c2.val[0]), + vreinterpretq_s32_s16(c3.val[0])); + const int32x4x2_t d3 = vtrnq_s32(vreinterpretq_s32_s16(c2.val[1]), + vreinterpretq_s32_s16(c3.val[1])); + + // Swap 64 bit elements + const int16x8x2_t e0 = vpx_vtrnq_s64_to_s16(d0.val[0], d2.val[0]); + const int16x8x2_t e1 = vpx_vtrnq_s64_to_s16(d1.val[0], d3.val[0]); + const int16x8x2_t e2 = vpx_vtrnq_s64_to_s16(d0.val[1], d2.val[1]); + const int16x8x2_t e3 = vpx_vtrnq_s64_to_s16(d1.val[1], d3.val[1]); + + b[0] = e0.val[0]; + b[1] = e1.val[0]; + b[2] = e2.val[0]; + b[3] = e3.val[0]; + b[4] = e0.val[1]; + b[5] = e1.val[1]; + b[6] = e2.val[1]; + b[7] = e3.val[1]; +} + +// Main body of fdct16x16. +static void vpx_fdct16x16_body(const int16x8_t *in /*[16]*/, + int16x8_t *out /*[16]*/) { + int16x8_t s[8]; + int16x8_t x[4]; + int16x8_t step[8]; + + // stage 1 + // From fwd_txfm.c: Work on the first eight values; fdct8(input, + // even_results);" + s[0] = vaddq_s16(in[0], in[7]); + s[1] = vaddq_s16(in[1], in[6]); + s[2] = vaddq_s16(in[2], in[5]); + s[3] = vaddq_s16(in[3], in[4]); + s[4] = vsubq_s16(in[3], in[4]); + s[5] = vsubq_s16(in[2], in[5]); + s[6] = vsubq_s16(in[1], in[6]); + s[7] = vsubq_s16(in[0], in[7]); + + // fdct4(step, step); + x[0] = vaddq_s16(s[0], s[3]); + x[1] = vaddq_s16(s[1], s[2]); + x[2] = vsubq_s16(s[1], s[2]); + x[3] = vsubq_s16(s[0], s[3]); + + // out[0] = fdct_round_shift((x0 + x1) * cospi_16_64) + // out[8] = fdct_round_shift((x0 - x1) * cospi_16_64) + butterfly_one_coeff(x[0], x[1], cospi_16_64, &out[0], &out[8]); + // out[4] = fdct_round_shift(x3 * cospi_8_64 + x2 * cospi_24_64); + // out[12] = fdct_round_shift(x3 * cospi_24_64 - x2 * cospi_8_64); + butterfly_two_coeff(x[3], x[2], cospi_24_64, cospi_8_64, &out[4], &out[12]); + + // Stage 2 + // Re-using source s5/s6 + // s5 = fdct_round_shift((s6 - s5) * cospi_16_64) + // s6 = fdct_round_shift((s6 + s5) * cospi_16_64) + butterfly_one_coeff(s[6], s[5], cospi_16_64, &s[6], &s[5]); + + // Stage 3 + x[0] = vaddq_s16(s[4], s[5]); + x[1] = vsubq_s16(s[4], s[5]); + x[2] = vsubq_s16(s[7], s[6]); + x[3] = vaddq_s16(s[7], s[6]); + + // Stage 4 + // out[2] = fdct_round_shift(x0 * cospi_28_64 + x3 * cospi_4_64) + // out[14] = fdct_round_shift(x3 * cospi_28_64 + x0 * -cospi_4_64) + butterfly_two_coeff(x[3], x[0], cospi_28_64, cospi_4_64, &out[2], &out[14]); + // out[6] = fdct_round_shift(x1 * cospi_12_64 + x2 * cospi_20_64) + // out[10] = fdct_round_shift(x2 * cospi_12_64 + x1 * -cospi_20_64) + butterfly_two_coeff(x[2], x[1], cospi_12_64, cospi_20_64, &out[10], &out[6]); + + // step 2 + // From fwd_txfm.c: Work on the next eight values; step1 -> odd_results" + // That file distinguished between "in_high" and "step1" but the only + // difference is that "in_high" is the first 8 values and "step 1" is the + // second. Here, since they are all in one array, "step1" values are += 8. + + // step2[2] = fdct_round_shift((step1[5] - step1[2]) * cospi_16_64) + // step2[3] = fdct_round_shift((step1[4] - step1[3]) * cospi_16_64) + // step2[4] = fdct_round_shift((step1[4] + step1[3]) * cospi_16_64) + // step2[5] = fdct_round_shift((step1[5] + step1[2]) * cospi_16_64) + butterfly_one_coeff(in[13], in[10], cospi_16_64, &s[5], &s[2]); + butterfly_one_coeff(in[12], in[11], cospi_16_64, &s[4], &s[3]); + + // step 3 + s[0] = vaddq_s16(in[8], s[3]); + s[1] = vaddq_s16(in[9], s[2]); + x[0] = vsubq_s16(in[9], s[2]); + x[1] = vsubq_s16(in[8], s[3]); + x[2] = vsubq_s16(in[15], s[4]); + x[3] = vsubq_s16(in[14], s[5]); + s[6] = vaddq_s16(in[14], s[5]); + s[7] = vaddq_s16(in[15], s[4]); + + // step 4 + // step2[1] = fdct_round_shift(step3[1] *-cospi_8_64 + step3[6] * cospi_24_64) + // step2[6] = fdct_round_shift(step3[1] * cospi_24_64 + step3[6] * cospi_8_64) + butterfly_two_coeff(s[6], s[1], cospi_24_64, cospi_8_64, &s[6], &s[1]); + + // step2[2] = fdct_round_shift(step3[2] * cospi_24_64 + step3[5] * cospi_8_64) + // step2[5] = fdct_round_shift(step3[2] * cospi_8_64 - step3[5] * cospi_24_64) + butterfly_two_coeff(x[0], x[3], cospi_8_64, cospi_24_64, &s[2], &s[5]); + + // step 5 + step[0] = vaddq_s16(s[0], s[1]); + step[1] = vsubq_s16(s[0], s[1]); + step[2] = vaddq_s16(x[1], s[2]); + step[3] = vsubq_s16(x[1], s[2]); + step[4] = vsubq_s16(x[2], s[5]); + step[5] = vaddq_s16(x[2], s[5]); + step[6] = vsubq_s16(s[7], s[6]); + step[7] = vaddq_s16(s[7], s[6]); + + // step 6 + // out[1] = fdct_round_shift(step1[0] * cospi_30_64 + step1[7] * cospi_2_64) + // out[9] = fdct_round_shift(step1[1] * cospi_14_64 + step1[6] * cospi_18_64) + // out[5] = fdct_round_shift(step1[2] * cospi_22_64 + step1[5] * cospi_10_64) + // out[13] = fdct_round_shift(step1[3] * cospi_6_64 + step1[4] * cospi_26_64) + // out[3] = fdct_round_shift(step1[3] * -cospi_26_64 + step1[4] * cospi_6_64) + // out[11] = fdct_round_shift(step1[2] * -cospi_10_64 + step1[5] * + // cospi_22_64) + // out[7] = fdct_round_shift(step1[1] * -cospi_18_64 + step1[6] * cospi_14_64) + // out[15] = fdct_round_shift(step1[0] * -cospi_2_64 + step1[7] * cospi_30_64) + butterfly_two_coeff(step[6], step[1], cospi_14_64, cospi_18_64, &out[9], + &out[7]); + butterfly_two_coeff(step[7], step[0], cospi_30_64, cospi_2_64, &out[1], + &out[15]); + butterfly_two_coeff(step[4], step[3], cospi_6_64, cospi_26_64, &out[13], + &out[3]); + butterfly_two_coeff(step[5], step[2], cospi_22_64, cospi_10_64, &out[5], + &out[11]); +} + +#endif // VPX_VPX_DSP_ARM_FDCT16X16_NEON_H_ diff --git a/media/libvpx/libvpx/vpx_dsp/arm/fdct16x16_neon.c b/media/libvpx/libvpx/vpx_dsp/arm/fdct16x16_neon.c --- a/media/libvpx/libvpx/vpx_dsp/arm/fdct16x16_neon.c +++ b/media/libvpx/libvpx/vpx_dsp/arm/fdct16x16_neon.c @@ -13,10 +13,11 @@ #include "./vpx_config.h" #include "./vpx_dsp_rtcd.h" #include "vpx_dsp/txfm_common.h" #include "vpx_dsp/arm/mem_neon.h" #include "vpx_dsp/arm/transpose_neon.h" +#include "vpx_dsp/arm/fdct16x16_neon.h" // Some builds of gcc 4.9.2 and .3 have trouble with some of the inline // functions. #if !defined(__clang__) && !defined(__ANDROID__) && defined(__GNUC__) && \ __GNUC__ == 4 && __GNUC_MINOR__ == 9 && __GNUC_PATCHLEVEL__ < 4 @@ -25,343 +26,33 @@ vpx_fdct16x16_c(input, output, stride); } #else -static INLINE void load(const int16_t *a, int stride, int16x8_t *b /*[16]*/) { - b[0] = vld1q_s16(a); - a += stride; - b[1] = vld1q_s16(a); - a += stride; - b[2] = vld1q_s16(a); - a += stride; - b[3] = vld1q_s16(a); - a += stride; - b[4] = vld1q_s16(a); - a += stride; - b[5] = vld1q_s16(a); - a += stride; - b[6] = vld1q_s16(a); - a += stride; - b[7] = vld1q_s16(a); - a += stride; - b[8] = vld1q_s16(a); - a += stride; - b[9] = vld1q_s16(a); - a += stride; - b[10] = vld1q_s16(a); - a += stride; - b[11] = vld1q_s16(a); - a += stride; - b[12] = vld1q_s16(a); - a += stride; - b[13] = vld1q_s16(a); - a += stride; - b[14] = vld1q_s16(a); - a += stride; - b[15] = vld1q_s16(a); -} - -// Store 8 16x8 values, assuming stride == 16. -static INLINE void store(tran_low_t *a, const int16x8_t *b /*[8]*/) { - store_s16q_to_tran_low(a, b[0]); - a += 16; - store_s16q_to_tran_low(a, b[1]); - a += 16; - store_s16q_to_tran_low(a, b[2]); - a += 16; - store_s16q_to_tran_low(a, b[3]); - a += 16; - store_s16q_to_tran_low(a, b[4]); - a += 16; - store_s16q_to_tran_low(a, b[5]); - a += 16; - store_s16q_to_tran_low(a, b[6]); - a += 16; - store_s16q_to_tran_low(a, b[7]); -} - -// Load step of each pass. Add and subtract clear across the input, requiring -// all 16 values to be loaded. For the first pass it also multiplies by 4. - -// To maybe reduce register usage this could be combined with the load() step to -// get the first 4 and last 4 values, cross those, then load the middle 8 values -// and cross them. -static INLINE void cross_input(const int16x8_t *a /*[16]*/, - int16x8_t *b /*[16]*/, const int pass) { - if (pass == 0) { - b[0] = vshlq_n_s16(vaddq_s16(a[0], a[15]), 2); - b[1] = vshlq_n_s16(vaddq_s16(a[1], a[14]), 2); - b[2] = vshlq_n_s16(vaddq_s16(a[2], a[13]), 2); - b[3] = vshlq_n_s16(vaddq_s16(a[3], a[12]), 2); - b[4] = vshlq_n_s16(vaddq_s16(a[4], a[11]), 2); - b[5] = vshlq_n_s16(vaddq_s16(a[5], a[10]), 2); - b[6] = vshlq_n_s16(vaddq_s16(a[6], a[9]), 2); - b[7] = vshlq_n_s16(vaddq_s16(a[7], a[8]), 2); - - b[8] = vshlq_n_s16(vsubq_s16(a[7], a[8]), 2); - b[9] = vshlq_n_s16(vsubq_s16(a[6], a[9]), 2); - b[10] = vshlq_n_s16(vsubq_s16(a[5], a[10]), 2); - b[11] = vshlq_n_s16(vsubq_s16(a[4], a[11]), 2); - b[12] = vshlq_n_s16(vsubq_s16(a[3], a[12]), 2); - b[13] = vshlq_n_s16(vsubq_s16(a[2], a[13]), 2); - b[14] = vshlq_n_s16(vsubq_s16(a[1], a[14]), 2); - b[15] = vshlq_n_s16(vsubq_s16(a[0], a[15]), 2); - } else { - b[0] = vaddq_s16(a[0], a[15]); - b[1] = vaddq_s16(a[1], a[14]); - b[2] = vaddq_s16(a[2], a[13]); - b[3] = vaddq_s16(a[3], a[12]); - b[4] = vaddq_s16(a[4], a[11]); - b[5] = vaddq_s16(a[5], a[10]); - b[6] = vaddq_s16(a[6], a[9]); - b[7] = vaddq_s16(a[7], a[8]); - - b[8] = vsubq_s16(a[7], a[8]); - b[9] = vsubq_s16(a[6], a[9]); - b[10] = vsubq_s16(a[5], a[10]); - b[11] = vsubq_s16(a[4], a[11]); - b[12] = vsubq_s16(a[3], a[12]); - b[13] = vsubq_s16(a[2], a[13]); - b[14] = vsubq_s16(a[1], a[14]); - b[15] = vsubq_s16(a[0], a[15]); - } -} - -// Quarter round at the beginning of the second pass. Can't use vrshr (rounding) -// because this only adds 1, not 1 << 2. -static INLINE void partial_round_shift(int16x8_t *a /*[16]*/) { - const int16x8_t one = vdupq_n_s16(1); - a[0] = vshrq_n_s16(vaddq_s16(a[0], one), 2); - a[1] = vshrq_n_s16(vaddq_s16(a[1], one), 2); - a[2] = vshrq_n_s16(vaddq_s16(a[2], one), 2); - a[3] = vshrq_n_s16(vaddq_s16(a[3], one), 2); - a[4] = vshrq_n_s16(vaddq_s16(a[4], one), 2); - a[5] = vshrq_n_s16(vaddq_s16(a[5], one), 2); - a[6] = vshrq_n_s16(vaddq_s16(a[6], one), 2); - a[7] = vshrq_n_s16(vaddq_s16(a[7], one), 2); - a[8] = vshrq_n_s16(vaddq_s16(a[8], one), 2); - a[9] = vshrq_n_s16(vaddq_s16(a[9], one), 2); - a[10] = vshrq_n_s16(vaddq_s16(a[10], one), 2); - a[11] = vshrq_n_s16(vaddq_s16(a[11], one), 2); - a[12] = vshrq_n_s16(vaddq_s16(a[12], one), 2); - a[13] = vshrq_n_s16(vaddq_s16(a[13], one), 2); - a[14] = vshrq_n_s16(vaddq_s16(a[14], one), 2); - a[15] = vshrq_n_s16(vaddq_s16(a[15], one), 2); -} - -// fdct_round_shift((a +/- b) * c) -static INLINE void butterfly_one_coeff(const int16x8_t a, const int16x8_t b, - const tran_high_t c, int16x8_t *add, - int16x8_t *sub) { - const int32x4_t a0 = vmull_n_s16(vget_low_s16(a), c); - const int32x4_t a1 = vmull_n_s16(vget_high_s16(a), c); - const int32x4_t sum0 = vmlal_n_s16(a0, vget_low_s16(b), c); - const int32x4_t sum1 = vmlal_n_s16(a1, vget_high_s16(b), c); - const int32x4_t diff0 = vmlsl_n_s16(a0, vget_low_s16(b), c); - const int32x4_t diff1 = vmlsl_n_s16(a1, vget_high_s16(b), c); - const int16x4_t rounded0 = vqrshrn_n_s32(sum0, 14); - const int16x4_t rounded1 = vqrshrn_n_s32(sum1, 14); - const int16x4_t rounded2 = vqrshrn_n_s32(diff0, 14); - const int16x4_t rounded3 = vqrshrn_n_s32(diff1, 14); - *add = vcombine_s16(rounded0, rounded1); - *sub = vcombine_s16(rounded2, rounded3); -} - -// fdct_round_shift(a * c0 +/- b * c1) -static INLINE void butterfly_two_coeff(const int16x8_t a, const int16x8_t b, - const tran_coef_t c0, - const tran_coef_t c1, int16x8_t *add, - int16x8_t *sub) { - const int32x4_t a0 = vmull_n_s16(vget_low_s16(a), c0); - const int32x4_t a1 = vmull_n_s16(vget_high_s16(a), c0); - const int32x4_t a2 = vmull_n_s16(vget_low_s16(a), c1); - const int32x4_t a3 = vmull_n_s16(vget_high_s16(a), c1); - const int32x4_t sum0 = vmlal_n_s16(a2, vget_low_s16(b), c0); - const int32x4_t sum1 = vmlal_n_s16(a3, vget_high_s16(b), c0); - const int32x4_t diff0 = vmlsl_n_s16(a0, vget_low_s16(b), c1); - const int32x4_t diff1 = vmlsl_n_s16(a1, vget_high_s16(b), c1); - const int16x4_t rounded0 = vqrshrn_n_s32(sum0, 14); - const int16x4_t rounded1 = vqrshrn_n_s32(sum1, 14); - const int16x4_t rounded2 = vqrshrn_n_s32(diff0, 14); - const int16x4_t rounded3 = vqrshrn_n_s32(diff1, 14); - *add = vcombine_s16(rounded0, rounded1); - *sub = vcombine_s16(rounded2, rounded3); -} - -// Transpose 8x8 to a new location. Don't use transpose_neon.h because those -// are all in-place. -static INLINE void transpose_8x8(const int16x8_t *a /*[8]*/, - int16x8_t *b /*[8]*/) { - // Swap 16 bit elements. - const int16x8x2_t c0 = vtrnq_s16(a[0], a[1]); - const int16x8x2_t c1 = vtrnq_s16(a[2], a[3]); - const int16x8x2_t c2 = vtrnq_s16(a[4], a[5]); - const int16x8x2_t c3 = vtrnq_s16(a[6], a[7]); - - // Swap 32 bit elements. - const int32x4x2_t d0 = vtrnq_s32(vreinterpretq_s32_s16(c0.val[0]), - vreinterpretq_s32_s16(c1.val[0])); - const int32x4x2_t d1 = vtrnq_s32(vreinterpretq_s32_s16(c0.val[1]), - vreinterpretq_s32_s16(c1.val[1])); - const int32x4x2_t d2 = vtrnq_s32(vreinterpretq_s32_s16(c2.val[0]), - vreinterpretq_s32_s16(c3.val[0])); - const int32x4x2_t d3 = vtrnq_s32(vreinterpretq_s32_s16(c2.val[1]), - vreinterpretq_s32_s16(c3.val[1])); - - // Swap 64 bit elements - const int16x8x2_t e0 = vpx_vtrnq_s64_to_s16(d0.val[0], d2.val[0]); - const int16x8x2_t e1 = vpx_vtrnq_s64_to_s16(d1.val[0], d3.val[0]); - const int16x8x2_t e2 = vpx_vtrnq_s64_to_s16(d0.val[1], d2.val[1]); - const int16x8x2_t e3 = vpx_vtrnq_s64_to_s16(d1.val[1], d3.val[1]); - - b[0] = e0.val[0]; - b[1] = e1.val[0]; - b[2] = e2.val[0]; - b[3] = e3.val[0]; - b[4] = e0.val[1]; - b[5] = e1.val[1]; - b[6] = e2.val[1]; - b[7] = e3.val[1]; -} - -// Main body of fdct16x16. -static void dct_body(const int16x8_t *in /*[16]*/, int16x8_t *out /*[16]*/) { - int16x8_t s[8]; - int16x8_t x[4]; - int16x8_t step[8]; - - // stage 1 - // From fwd_txfm.c: Work on the first eight values; fdct8(input, - // even_results);" - s[0] = vaddq_s16(in[0], in[7]); - s[1] = vaddq_s16(in[1], in[6]); - s[2] = vaddq_s16(in[2], in[5]); - s[3] = vaddq_s16(in[3], in[4]); - s[4] = vsubq_s16(in[3], in[4]); - s[5] = vsubq_s16(in[2], in[5]); - s[6] = vsubq_s16(in[1], in[6]); - s[7] = vsubq_s16(in[0], in[7]); - - // fdct4(step, step); - x[0] = vaddq_s16(s[0], s[3]); - x[1] = vaddq_s16(s[1], s[2]); - x[2] = vsubq_s16(s[1], s[2]); - x[3] = vsubq_s16(s[0], s[3]); - - // out[0] = fdct_round_shift((x0 + x1) * cospi_16_64) - // out[8] = fdct_round_shift((x0 - x1) * cospi_16_64) - butterfly_one_coeff(x[0], x[1], cospi_16_64, &out[0], &out[8]); - // out[4] = fdct_round_shift(x3 * cospi_8_64 + x2 * cospi_24_64); - // out[12] = fdct_round_shift(x3 * cospi_24_64 - x2 * cospi_8_64); - butterfly_two_coeff(x[3], x[2], cospi_24_64, cospi_8_64, &out[4], &out[12]); - - // Stage 2 - // Re-using source s5/s6 - // s5 = fdct_round_shift((s6 - s5) * cospi_16_64) - // s6 = fdct_round_shift((s6 + s5) * cospi_16_64) - butterfly_one_coeff(s[6], s[5], cospi_16_64, &s[6], &s[5]); - - // Stage 3 - x[0] = vaddq_s16(s[4], s[5]); - x[1] = vsubq_s16(s[4], s[5]); - x[2] = vsubq_s16(s[7], s[6]); - x[3] = vaddq_s16(s[7], s[6]); - - // Stage 4 - // out[2] = fdct_round_shift(x0 * cospi_28_64 + x3 * cospi_4_64) - // out[14] = fdct_round_shift(x3 * cospi_28_64 + x0 * -cospi_4_64) - butterfly_two_coeff(x[3], x[0], cospi_28_64, cospi_4_64, &out[2], &out[14]); - // out[6] = fdct_round_shift(x1 * cospi_12_64 + x2 * cospi_20_64) - // out[10] = fdct_round_shift(x2 * cospi_12_64 + x1 * -cospi_20_64) - butterfly_two_coeff(x[2], x[1], cospi_12_64, cospi_20_64, &out[10], &out[6]); - - // step 2 - // From fwd_txfm.c: Work on the next eight values; step1 -> odd_results" - // That file distinguished between "in_high" and "step1" but the only - // difference is that "in_high" is the first 8 values and "step 1" is the - // second. Here, since they are all in one array, "step1" values are += 8. - - // step2[2] = fdct_round_shift((step1[5] - step1[2]) * cospi_16_64) - // step2[3] = fdct_round_shift((step1[4] - step1[3]) * cospi_16_64) - // step2[4] = fdct_round_shift((step1[4] + step1[3]) * cospi_16_64) - // step2[5] = fdct_round_shift((step1[5] + step1[2]) * cospi_16_64) - butterfly_one_coeff(in[13], in[10], cospi_16_64, &s[5], &s[2]); - butterfly_one_coeff(in[12], in[11], cospi_16_64, &s[4], &s[3]); - - // step 3 - s[0] = vaddq_s16(in[8], s[3]); - s[1] = vaddq_s16(in[9], s[2]); - x[0] = vsubq_s16(in[9], s[2]); - x[1] = vsubq_s16(in[8], s[3]); - x[2] = vsubq_s16(in[15], s[4]); - x[3] = vsubq_s16(in[14], s[5]); - s[6] = vaddq_s16(in[14], s[5]); - s[7] = vaddq_s16(in[15], s[4]); - - // step 4 - // step2[1] = fdct_round_shift(step3[1] *-cospi_8_64 + step3[6] * cospi_24_64) - // step2[6] = fdct_round_shift(step3[1] * cospi_24_64 + step3[6] * cospi_8_64) - butterfly_two_coeff(s[6], s[1], cospi_24_64, cospi_8_64, &s[6], &s[1]); - - // step2[2] = fdct_round_shift(step3[2] * cospi_24_64 + step3[5] * cospi_8_64) - // step2[5] = fdct_round_shift(step3[2] * cospi_8_64 - step3[5] * cospi_24_64) - butterfly_two_coeff(x[0], x[3], cospi_8_64, cospi_24_64, &s[2], &s[5]); - - // step 5 - step[0] = vaddq_s16(s[0], s[1]); - step[1] = vsubq_s16(s[0], s[1]); - step[2] = vaddq_s16(x[1], s[2]); - step[3] = vsubq_s16(x[1], s[2]); - step[4] = vsubq_s16(x[2], s[5]); - step[5] = vaddq_s16(x[2], s[5]); - step[6] = vsubq_s16(s[7], s[6]); - step[7] = vaddq_s16(s[7], s[6]); - - // step 6 - // out[1] = fdct_round_shift(step1[0] * cospi_30_64 + step1[7] * cospi_2_64) - // out[9] = fdct_round_shift(step1[1] * cospi_14_64 + step1[6] * cospi_18_64) - // out[5] = fdct_round_shift(step1[2] * cospi_22_64 + step1[5] * cospi_10_64) - // out[13] = fdct_round_shift(step1[3] * cospi_6_64 + step1[4] * cospi_26_64) - // out[3] = fdct_round_shift(step1[3] * -cospi_26_64 + step1[4] * cospi_6_64) - // out[11] = fdct_round_shift(step1[2] * -cospi_10_64 + step1[5] * - // cospi_22_64) - // out[7] = fdct_round_shift(step1[1] * -cospi_18_64 + step1[6] * cospi_14_64) - // out[15] = fdct_round_shift(step1[0] * -cospi_2_64 + step1[7] * cospi_30_64) - butterfly_two_coeff(step[6], step[1], cospi_14_64, cospi_18_64, &out[9], - &out[7]); - butterfly_two_coeff(step[7], step[0], cospi_30_64, cospi_2_64, &out[1], - &out[15]); - butterfly_two_coeff(step[4], step[3], cospi_6_64, cospi_26_64, &out[13], - &out[3]); - butterfly_two_coeff(step[5], step[2], cospi_22_64, cospi_10_64, &out[5], - &out[11]); -} - void vpx_fdct16x16_neon(const int16_t *input, tran_low_t *output, int stride) { int16x8_t temp0[16]; int16x8_t temp1[16]; int16x8_t temp2[16]; int16x8_t temp3[16]; // Left half. load(input, stride, temp0); cross_input(temp0, temp1, 0); - dct_body(temp1, temp0); + vpx_fdct16x16_body(temp1, temp0); // Right half. load(input + 8, stride, temp1); cross_input(temp1, temp2, 0); - dct_body(temp2, temp1); + vpx_fdct16x16_body(temp2, temp1); // Transpose top left and top right quarters into one contiguous location to // process to the top half. transpose_8x8(&temp0[0], &temp2[0]); transpose_8x8(&temp1[0], &temp2[8]); partial_round_shift(temp2); cross_input(temp2, temp3, 1); - dct_body(temp3, temp2); + vpx_fdct16x16_body(temp3, temp2); transpose_s16_8x8(&temp2[0], &temp2[1], &temp2[2], &temp2[3], &temp2[4], &temp2[5], &temp2[6], &temp2[7]); transpose_s16_8x8(&temp2[8], &temp2[9], &temp2[10], &temp2[11], &temp2[12], &temp2[13], &temp2[14], &temp2[15]); store(output, temp2); @@ -373,11 +64,11 @@ transpose_8x8(&temp0[8], &temp1[0]); transpose_s16_8x8(&temp1[8], &temp1[9], &temp1[10], &temp1[11], &temp1[12], &temp1[13], &temp1[14], &temp1[15]); partial_round_shift(temp1); cross_input(temp1, temp0, 1); - dct_body(temp0, temp1); + vpx_fdct16x16_body(temp0, temp1); transpose_s16_8x8(&temp1[0], &temp1[1], &temp1[2], &temp1[3], &temp1[4], &temp1[5], &temp1[6], &temp1[7]); transpose_s16_8x8(&temp1[8], &temp1[9], &temp1[10], &temp1[11], &temp1[12], &temp1[13], &temp1[14], &temp1[15]); store(output, temp1); diff --git a/media/libvpx/libvpx/vpx_dsp/arm/fdct32x32_neon.c b/media/libvpx/libvpx/vpx_dsp/arm/fdct32x32_neon.c --- a/media/libvpx/libvpx/vpx_dsp/arm/fdct32x32_neon.c +++ b/media/libvpx/libvpx/vpx_dsp/arm/fdct32x32_neon.c @@ -151,11 +151,11 @@ #define STORE_S16(src, index, dest) \ do { \ store_s16q_to_tran_low(dest, src[index]); \ dest += 8; \ - } while (0); + } while (0) // Store 32 16x8 values, assuming stride == 32. // Slight twist: store horizontally in blocks of 8. static INLINE void store(tran_low_t *a, const int16x8_t *b) { STORE_S16(b, 0, a); diff --git a/media/libvpx/libvpx/vpx_dsp/arm/fdct_neon.h b/media/libvpx/libvpx/vpx_dsp/arm/fdct_neon.h new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/arm/fdct_neon.h @@ -0,0 +1,213 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VPX_VPX_DSP_ARM_FDCT_NEON_H_ +#define VPX_VPX_DSP_ARM_FDCT_NEON_H_ + +#include + +static INLINE void vpx_fdct4x4_pass1_neon(int16x4_t *in) { + const int16x8_t input_01 = vcombine_s16(in[0], in[1]); + const int16x8_t input_32 = vcombine_s16(in[3], in[2]); + + // in_0 +/- in_3, in_1 +/- in_2 + const int16x8_t s_01 = vaddq_s16(input_01, input_32); + const int16x8_t s_32 = vsubq_s16(input_01, input_32); + + // step_0 +/- step_1, step_2 +/- step_3 + const int16x4_t s_0 = vget_low_s16(s_01); + const int16x4_t s_1 = vget_high_s16(s_01); + const int16x4_t s_2 = vget_high_s16(s_32); + const int16x4_t s_3 = vget_low_s16(s_32); + + // (s_0 +/- s_1) * cospi_16_64 + // Must expand all elements to s32. See 'needs32' comment in fwd_txfm.c. + const int32x4_t s_0_p_s_1 = vaddl_s16(s_0, s_1); + const int32x4_t s_0_m_s_1 = vsubl_s16(s_0, s_1); + const int32x4_t temp1 = vmulq_n_s32(s_0_p_s_1, cospi_16_64); + const int32x4_t temp2 = vmulq_n_s32(s_0_m_s_1, cospi_16_64); + + // fdct_round_shift + int16x4_t out_0 = vrshrn_n_s32(temp1, DCT_CONST_BITS); + int16x4_t out_2 = vrshrn_n_s32(temp2, DCT_CONST_BITS); + + // s_3 * cospi_8_64 + s_2 * cospi_24_64 + // s_3 * cospi_24_64 - s_2 * cospi_8_64 + const int32x4_t s_3_cospi_8_64 = vmull_n_s16(s_3, cospi_8_64); + const int32x4_t s_3_cospi_24_64 = vmull_n_s16(s_3, cospi_24_64); + + const int32x4_t temp3 = vmlal_n_s16(s_3_cospi_8_64, s_2, cospi_24_64); + const int32x4_t temp4 = vmlsl_n_s16(s_3_cospi_24_64, s_2, cospi_8_64); + + // fdct_round_shift + int16x4_t out_1 = vrshrn_n_s32(temp3, DCT_CONST_BITS); + int16x4_t out_3 = vrshrn_n_s32(temp4, DCT_CONST_BITS); + + transpose_s16_4x4d(&out_0, &out_1, &out_2, &out_3); + + in[0] = out_0; + in[1] = out_1; + in[2] = out_2; + in[3] = out_3; +} + +static INLINE void vpx_fdct8x8_pass1_notranspose_neon(int16x8_t *in, + int16x8_t *out) { + const int16x8_t v_s0 = vaddq_s16(in[0], in[7]); + const int16x8_t v_s1 = vaddq_s16(in[1], in[6]); + const int16x8_t v_s2 = vaddq_s16(in[2], in[5]); + const int16x8_t v_s3 = vaddq_s16(in[3], in[4]); + const int16x8_t v_s4 = vsubq_s16(in[3], in[4]); + const int16x8_t v_s5 = vsubq_s16(in[2], in[5]); + const int16x8_t v_s6 = vsubq_s16(in[1], in[6]); + const int16x8_t v_s7 = vsubq_s16(in[0], in[7]); + // fdct4(step, step); + int16x8_t v_x0 = vaddq_s16(v_s0, v_s3); + int16x8_t v_x1 = vaddq_s16(v_s1, v_s2); + int16x8_t v_x2 = vsubq_s16(v_s1, v_s2); + int16x8_t v_x3 = vsubq_s16(v_s0, v_s3); + // fdct4(step, step); + int32x4_t v_t0_lo = vaddl_s16(vget_low_s16(v_x0), vget_low_s16(v_x1)); + int32x4_t v_t0_hi = vaddl_s16(vget_high_s16(v_x0), vget_high_s16(v_x1)); + int32x4_t v_t1_lo = vsubl_s16(vget_low_s16(v_x0), vget_low_s16(v_x1)); + int32x4_t v_t1_hi = vsubl_s16(vget_high_s16(v_x0), vget_high_s16(v_x1)); + int32x4_t v_t2_lo = vmull_n_s16(vget_low_s16(v_x2), cospi_24_64); + int32x4_t v_t2_hi = vmull_n_s16(vget_high_s16(v_x2), cospi_24_64); + int32x4_t v_t3_lo = vmull_n_s16(vget_low_s16(v_x3), cospi_24_64); + int32x4_t v_t3_hi = vmull_n_s16(vget_high_s16(v_x3), cospi_24_64); + v_t2_lo = vmlal_n_s16(v_t2_lo, vget_low_s16(v_x3), cospi_8_64); + v_t2_hi = vmlal_n_s16(v_t2_hi, vget_high_s16(v_x3), cospi_8_64); + v_t3_lo = vmlsl_n_s16(v_t3_lo, vget_low_s16(v_x2), cospi_8_64); + v_t3_hi = vmlsl_n_s16(v_t3_hi, vget_high_s16(v_x2), cospi_8_64); + v_t0_lo = vmulq_n_s32(v_t0_lo, cospi_16_64); + v_t0_hi = vmulq_n_s32(v_t0_hi, cospi_16_64); + v_t1_lo = vmulq_n_s32(v_t1_lo, cospi_16_64); + v_t1_hi = vmulq_n_s32(v_t1_hi, cospi_16_64); + { + const int16x4_t a = vrshrn_n_s32(v_t0_lo, DCT_CONST_BITS); + const int16x4_t b = vrshrn_n_s32(v_t0_hi, DCT_CONST_BITS); + const int16x4_t c = vrshrn_n_s32(v_t1_lo, DCT_CONST_BITS); + const int16x4_t d = vrshrn_n_s32(v_t1_hi, DCT_CONST_BITS); + const int16x4_t e = vrshrn_n_s32(v_t2_lo, DCT_CONST_BITS); + const int16x4_t f = vrshrn_n_s32(v_t2_hi, DCT_CONST_BITS); + const int16x4_t g = vrshrn_n_s32(v_t3_lo, DCT_CONST_BITS); + const int16x4_t h = vrshrn_n_s32(v_t3_hi, DCT_CONST_BITS); + out[0] = vcombine_s16(a, c); // 00 01 02 03 40 41 42 43 + out[2] = vcombine_s16(e, g); // 20 21 22 23 60 61 62 63 + out[4] = vcombine_s16(b, d); // 04 05 06 07 44 45 46 47 + out[6] = vcombine_s16(f, h); // 24 25 26 27 64 65 66 67 + } + // Stage 2 + v_x0 = vsubq_s16(v_s6, v_s5); + v_x1 = vaddq_s16(v_s6, v_s5); + v_t0_lo = vmull_n_s16(vget_low_s16(v_x0), cospi_16_64); + v_t0_hi = vmull_n_s16(vget_high_s16(v_x0), cospi_16_64); + v_t1_lo = vmull_n_s16(vget_low_s16(v_x1), cospi_16_64); + v_t1_hi = vmull_n_s16(vget_high_s16(v_x1), cospi_16_64); + { + const int16x4_t a = vrshrn_n_s32(v_t0_lo, DCT_CONST_BITS); + const int16x4_t b = vrshrn_n_s32(v_t0_hi, DCT_CONST_BITS); + const int16x4_t c = vrshrn_n_s32(v_t1_lo, DCT_CONST_BITS); + const int16x4_t d = vrshrn_n_s32(v_t1_hi, DCT_CONST_BITS); + const int16x8_t ab = vcombine_s16(a, b); + const int16x8_t cd = vcombine_s16(c, d); + // Stage 3 + v_x0 = vaddq_s16(v_s4, ab); + v_x1 = vsubq_s16(v_s4, ab); + v_x2 = vsubq_s16(v_s7, cd); + v_x3 = vaddq_s16(v_s7, cd); + } + // Stage 4 + v_t0_lo = vmull_n_s16(vget_low_s16(v_x3), cospi_4_64); + v_t0_hi = vmull_n_s16(vget_high_s16(v_x3), cospi_4_64); + v_t0_lo = vmlal_n_s16(v_t0_lo, vget_low_s16(v_x0), cospi_28_64); + v_t0_hi = vmlal_n_s16(v_t0_hi, vget_high_s16(v_x0), cospi_28_64); + v_t1_lo = vmull_n_s16(vget_low_s16(v_x1), cospi_12_64); + v_t1_hi = vmull_n_s16(vget_high_s16(v_x1), cospi_12_64); + v_t1_lo = vmlal_n_s16(v_t1_lo, vget_low_s16(v_x2), cospi_20_64); + v_t1_hi = vmlal_n_s16(v_t1_hi, vget_high_s16(v_x2), cospi_20_64); + v_t2_lo = vmull_n_s16(vget_low_s16(v_x2), cospi_12_64); + v_t2_hi = vmull_n_s16(vget_high_s16(v_x2), cospi_12_64); + v_t2_lo = vmlsl_n_s16(v_t2_lo, vget_low_s16(v_x1), cospi_20_64); + v_t2_hi = vmlsl_n_s16(v_t2_hi, vget_high_s16(v_x1), cospi_20_64); + v_t3_lo = vmull_n_s16(vget_low_s16(v_x3), cospi_28_64); + v_t3_hi = vmull_n_s16(vget_high_s16(v_x3), cospi_28_64); + v_t3_lo = vmlsl_n_s16(v_t3_lo, vget_low_s16(v_x0), cospi_4_64); + v_t3_hi = vmlsl_n_s16(v_t3_hi, vget_high_s16(v_x0), cospi_4_64); + { + const int16x4_t a = vrshrn_n_s32(v_t0_lo, DCT_CONST_BITS); + const int16x4_t b = vrshrn_n_s32(v_t0_hi, DCT_CONST_BITS); + const int16x4_t c = vrshrn_n_s32(v_t1_lo, DCT_CONST_BITS); + const int16x4_t d = vrshrn_n_s32(v_t1_hi, DCT_CONST_BITS); + const int16x4_t e = vrshrn_n_s32(v_t2_lo, DCT_CONST_BITS); + const int16x4_t f = vrshrn_n_s32(v_t2_hi, DCT_CONST_BITS); + const int16x4_t g = vrshrn_n_s32(v_t3_lo, DCT_CONST_BITS); + const int16x4_t h = vrshrn_n_s32(v_t3_hi, DCT_CONST_BITS); + out[1] = vcombine_s16(a, c); // 10 11 12 13 50 51 52 53 + out[3] = vcombine_s16(e, g); // 30 31 32 33 70 71 72 73 + out[5] = vcombine_s16(b, d); // 14 15 16 17 54 55 56 57 + out[7] = vcombine_s16(f, h); // 34 35 36 37 74 75 76 77 + } +} + +static INLINE void vpx_fdct8x8_pass1_neon(int16x8_t *in) { + int16x8_t out[8]; + vpx_fdct8x8_pass1_notranspose_neon(in, out); + // transpose 8x8 + // Can't use transpose_s16_8x8() because the values are arranged in two 4x8 + // columns. + { + // 00 01 02 03 40 41 42 43 + // 10 11 12 13 50 51 52 53 + // 20 21 22 23 60 61 62 63 + // 30 31 32 33 70 71 72 73 + // 04 05 06 07 44 45 46 47 + // 14 15 16 17 54 55 56 57 + // 24 25 26 27 64 65 66 67 + // 34 35 36 37 74 75 76 77 + const int32x4x2_t r02_s32 = + vtrnq_s32(vreinterpretq_s32_s16(out[0]), vreinterpretq_s32_s16(out[2])); + const int32x4x2_t r13_s32 = + vtrnq_s32(vreinterpretq_s32_s16(out[1]), vreinterpretq_s32_s16(out[3])); + const int32x4x2_t r46_s32 = + vtrnq_s32(vreinterpretq_s32_s16(out[4]), vreinterpretq_s32_s16(out[6])); + const int32x4x2_t r57_s32 = + vtrnq_s32(vreinterpretq_s32_s16(out[5]), vreinterpretq_s32_s16(out[7])); + const int16x8x2_t r01_s16 = + vtrnq_s16(vreinterpretq_s16_s32(r02_s32.val[0]), + vreinterpretq_s16_s32(r13_s32.val[0])); + const int16x8x2_t r23_s16 = + vtrnq_s16(vreinterpretq_s16_s32(r02_s32.val[1]), + vreinterpretq_s16_s32(r13_s32.val[1])); + const int16x8x2_t r45_s16 = + vtrnq_s16(vreinterpretq_s16_s32(r46_s32.val[0]), + vreinterpretq_s16_s32(r57_s32.val[0])); + const int16x8x2_t r67_s16 = + vtrnq_s16(vreinterpretq_s16_s32(r46_s32.val[1]), + vreinterpretq_s16_s32(r57_s32.val[1])); + in[0] = r01_s16.val[0]; + in[1] = r01_s16.val[1]; + in[2] = r23_s16.val[0]; + in[3] = r23_s16.val[1]; + in[4] = r45_s16.val[0]; + in[5] = r45_s16.val[1]; + in[6] = r67_s16.val[0]; + in[7] = r67_s16.val[1]; + // 00 10 20 30 40 50 60 70 + // 01 11 21 31 41 51 61 71 + // 02 12 22 32 42 52 62 72 + // 03 13 23 33 43 53 63 73 + // 04 14 24 34 44 54 64 74 + // 05 15 25 35 45 55 65 75 + // 06 16 26 36 46 56 66 76 + // 07 17 27 37 47 57 67 77 + } +} +#endif // VPX_VPX_DSP_ARM_FDCT_NEON_H_ diff --git a/media/libvpx/libvpx/vpx_dsp/arm/fdct_neon.c b/media/libvpx/libvpx/vpx_dsp/arm/fdct_neon.c --- a/media/libvpx/libvpx/vpx_dsp/arm/fdct_neon.c +++ b/media/libvpx/libvpx/vpx_dsp/arm/fdct_neon.c @@ -13,78 +13,37 @@ #include "./vpx_config.h" #include "./vpx_dsp_rtcd.h" #include "vpx_dsp/txfm_common.h" #include "vpx_dsp/vpx_dsp_common.h" #include "vpx_dsp/arm/idct_neon.h" +#include "vpx_dsp/arm/fdct_neon.h" #include "vpx_dsp/arm/mem_neon.h" #include "vpx_dsp/arm/transpose_neon.h" void vpx_fdct4x4_neon(const int16_t *input, tran_low_t *final_output, int stride) { int i; // input[M * stride] * 16 - int16x4_t input_0 = vshl_n_s16(vld1_s16(input + 0 * stride), 4); - int16x4_t input_1 = vshl_n_s16(vld1_s16(input + 1 * stride), 4); - int16x4_t input_2 = vshl_n_s16(vld1_s16(input + 2 * stride), 4); - int16x4_t input_3 = vshl_n_s16(vld1_s16(input + 3 * stride), 4); + int16x4_t in[4]; + in[0] = vshl_n_s16(vld1_s16(input + 0 * stride), 4); + in[1] = vshl_n_s16(vld1_s16(input + 1 * stride), 4); + in[2] = vshl_n_s16(vld1_s16(input + 2 * stride), 4); + in[3] = vshl_n_s16(vld1_s16(input + 3 * stride), 4); // If the very first value != 0, then add 1. if (input[0] != 0) { const int16x4_t one = vreinterpret_s16_s64(vdup_n_s64(1)); - input_0 = vadd_s16(input_0, one); + in[0] = vadd_s16(in[0], one); } - for (i = 0; i < 2; ++i) { - const int16x8_t input_01 = vcombine_s16(input_0, input_1); - const int16x8_t input_32 = vcombine_s16(input_3, input_2); - - // in_0 +/- in_3, in_1 +/- in_2 - const int16x8_t s_01 = vaddq_s16(input_01, input_32); - const int16x8_t s_32 = vsubq_s16(input_01, input_32); - - // step_0 +/- step_1, step_2 +/- step_3 - const int16x4_t s_0 = vget_low_s16(s_01); - const int16x4_t s_1 = vget_high_s16(s_01); - const int16x4_t s_2 = vget_high_s16(s_32); - const int16x4_t s_3 = vget_low_s16(s_32); - - // (s_0 +/- s_1) * cospi_16_64 - // Must expand all elements to s32. See 'needs32' comment in fwd_txfm.c. - const int32x4_t s_0_p_s_1 = vaddl_s16(s_0, s_1); - const int32x4_t s_0_m_s_1 = vsubl_s16(s_0, s_1); - const int32x4_t temp1 = vmulq_n_s32(s_0_p_s_1, cospi_16_64); - const int32x4_t temp2 = vmulq_n_s32(s_0_m_s_1, cospi_16_64); - - // fdct_round_shift - int16x4_t out_0 = vrshrn_n_s32(temp1, DCT_CONST_BITS); - int16x4_t out_2 = vrshrn_n_s32(temp2, DCT_CONST_BITS); - - // s_3 * cospi_8_64 + s_2 * cospi_24_64 - // s_3 * cospi_24_64 - s_2 * cospi_8_64 - const int32x4_t s_3_cospi_8_64 = vmull_n_s16(s_3, cospi_8_64); - const int32x4_t s_3_cospi_24_64 = vmull_n_s16(s_3, cospi_24_64); - - const int32x4_t temp3 = vmlal_n_s16(s_3_cospi_8_64, s_2, cospi_24_64); - const int32x4_t temp4 = vmlsl_n_s16(s_3_cospi_24_64, s_2, cospi_8_64); - - // fdct_round_shift - int16x4_t out_1 = vrshrn_n_s32(temp3, DCT_CONST_BITS); - int16x4_t out_3 = vrshrn_n_s32(temp4, DCT_CONST_BITS); - - transpose_s16_4x4d(&out_0, &out_1, &out_2, &out_3); - - input_0 = out_0; - input_1 = out_1; - input_2 = out_2; - input_3 = out_3; + vpx_fdct4x4_pass1_neon(in); } - { // Not quite a rounding shift. Only add 1 despite shifting by 2. const int16x8_t one = vdupq_n_s16(1); - int16x8_t out_01 = vcombine_s16(input_0, input_1); - int16x8_t out_23 = vcombine_s16(input_2, input_3); + int16x8_t out_01 = vcombine_s16(in[0], in[1]); + int16x8_t out_23 = vcombine_s16(in[2], in[3]); out_01 = vshrq_n_s16(vaddq_s16(out_01, one), 2); out_23 = vshrq_n_s16(vaddq_s16(out_23, one), 2); store_s16q_to_tran_low(final_output + 0 * 8, out_01); store_s16q_to_tran_low(final_output + 1 * 8, out_23); } diff --git a/media/libvpx/libvpx/vpx_dsp/arm/fwd_txfm_neon.c b/media/libvpx/libvpx/vpx_dsp/arm/fwd_txfm_neon.c --- a/media/libvpx/libvpx/vpx_dsp/arm/fwd_txfm_neon.c +++ b/media/libvpx/libvpx/vpx_dsp/arm/fwd_txfm_neon.c @@ -13,198 +13,56 @@ #include "./vpx_config.h" #include "./vpx_dsp_rtcd.h" #include "vpx_dsp/txfm_common.h" #include "vpx_dsp/vpx_dsp_common.h" #include "vpx_dsp/arm/idct_neon.h" +#include "vpx_dsp/arm/fdct_neon.h" #include "vpx_dsp/arm/mem_neon.h" void vpx_fdct8x8_neon(const int16_t *input, tran_low_t *final_output, int stride) { int i; // stage 1 - int16x8_t input_0 = vshlq_n_s16(vld1q_s16(&input[0 * stride]), 2); - int16x8_t input_1 = vshlq_n_s16(vld1q_s16(&input[1 * stride]), 2); - int16x8_t input_2 = vshlq_n_s16(vld1q_s16(&input[2 * stride]), 2); - int16x8_t input_3 = vshlq_n_s16(vld1q_s16(&input[3 * stride]), 2); - int16x8_t input_4 = vshlq_n_s16(vld1q_s16(&input[4 * stride]), 2); - int16x8_t input_5 = vshlq_n_s16(vld1q_s16(&input[5 * stride]), 2); - int16x8_t input_6 = vshlq_n_s16(vld1q_s16(&input[6 * stride]), 2); - int16x8_t input_7 = vshlq_n_s16(vld1q_s16(&input[7 * stride]), 2); + int16x8_t in[8]; + in[0] = vshlq_n_s16(vld1q_s16(&input[0 * stride]), 2); + in[1] = vshlq_n_s16(vld1q_s16(&input[1 * stride]), 2); + in[2] = vshlq_n_s16(vld1q_s16(&input[2 * stride]), 2); + in[3] = vshlq_n_s16(vld1q_s16(&input[3 * stride]), 2); + in[4] = vshlq_n_s16(vld1q_s16(&input[4 * stride]), 2); + in[5] = vshlq_n_s16(vld1q_s16(&input[5 * stride]), 2); + in[6] = vshlq_n_s16(vld1q_s16(&input[6 * stride]), 2); + in[7] = vshlq_n_s16(vld1q_s16(&input[7 * stride]), 2); for (i = 0; i < 2; ++i) { - int16x8_t out_0, out_1, out_2, out_3, out_4, out_5, out_6, out_7; - const int16x8_t v_s0 = vaddq_s16(input_0, input_7); - const int16x8_t v_s1 = vaddq_s16(input_1, input_6); - const int16x8_t v_s2 = vaddq_s16(input_2, input_5); - const int16x8_t v_s3 = vaddq_s16(input_3, input_4); - const int16x8_t v_s4 = vsubq_s16(input_3, input_4); - const int16x8_t v_s5 = vsubq_s16(input_2, input_5); - const int16x8_t v_s6 = vsubq_s16(input_1, input_6); - const int16x8_t v_s7 = vsubq_s16(input_0, input_7); - // fdct4(step, step); - int16x8_t v_x0 = vaddq_s16(v_s0, v_s3); - int16x8_t v_x1 = vaddq_s16(v_s1, v_s2); - int16x8_t v_x2 = vsubq_s16(v_s1, v_s2); - int16x8_t v_x3 = vsubq_s16(v_s0, v_s3); - // fdct4(step, step); - int32x4_t v_t0_lo = vaddl_s16(vget_low_s16(v_x0), vget_low_s16(v_x1)); - int32x4_t v_t0_hi = vaddl_s16(vget_high_s16(v_x0), vget_high_s16(v_x1)); - int32x4_t v_t1_lo = vsubl_s16(vget_low_s16(v_x0), vget_low_s16(v_x1)); - int32x4_t v_t1_hi = vsubl_s16(vget_high_s16(v_x0), vget_high_s16(v_x1)); - int32x4_t v_t2_lo = vmull_n_s16(vget_low_s16(v_x2), cospi_24_64); - int32x4_t v_t2_hi = vmull_n_s16(vget_high_s16(v_x2), cospi_24_64); - int32x4_t v_t3_lo = vmull_n_s16(vget_low_s16(v_x3), cospi_24_64); - int32x4_t v_t3_hi = vmull_n_s16(vget_high_s16(v_x3), cospi_24_64); - v_t2_lo = vmlal_n_s16(v_t2_lo, vget_low_s16(v_x3), cospi_8_64); - v_t2_hi = vmlal_n_s16(v_t2_hi, vget_high_s16(v_x3), cospi_8_64); - v_t3_lo = vmlsl_n_s16(v_t3_lo, vget_low_s16(v_x2), cospi_8_64); - v_t3_hi = vmlsl_n_s16(v_t3_hi, vget_high_s16(v_x2), cospi_8_64); - v_t0_lo = vmulq_n_s32(v_t0_lo, cospi_16_64); - v_t0_hi = vmulq_n_s32(v_t0_hi, cospi_16_64); - v_t1_lo = vmulq_n_s32(v_t1_lo, cospi_16_64); - v_t1_hi = vmulq_n_s32(v_t1_hi, cospi_16_64); - { - const int16x4_t a = vrshrn_n_s32(v_t0_lo, DCT_CONST_BITS); - const int16x4_t b = vrshrn_n_s32(v_t0_hi, DCT_CONST_BITS); - const int16x4_t c = vrshrn_n_s32(v_t1_lo, DCT_CONST_BITS); - const int16x4_t d = vrshrn_n_s32(v_t1_hi, DCT_CONST_BITS); - const int16x4_t e = vrshrn_n_s32(v_t2_lo, DCT_CONST_BITS); - const int16x4_t f = vrshrn_n_s32(v_t2_hi, DCT_CONST_BITS); - const int16x4_t g = vrshrn_n_s32(v_t3_lo, DCT_CONST_BITS); - const int16x4_t h = vrshrn_n_s32(v_t3_hi, DCT_CONST_BITS); - out_0 = vcombine_s16(a, c); // 00 01 02 03 40 41 42 43 - out_2 = vcombine_s16(e, g); // 20 21 22 23 60 61 62 63 - out_4 = vcombine_s16(b, d); // 04 05 06 07 44 45 46 47 - out_6 = vcombine_s16(f, h); // 24 25 26 27 64 65 66 67 - } - // Stage 2 - v_x0 = vsubq_s16(v_s6, v_s5); - v_x1 = vaddq_s16(v_s6, v_s5); - v_t0_lo = vmull_n_s16(vget_low_s16(v_x0), cospi_16_64); - v_t0_hi = vmull_n_s16(vget_high_s16(v_x0), cospi_16_64); - v_t1_lo = vmull_n_s16(vget_low_s16(v_x1), cospi_16_64); - v_t1_hi = vmull_n_s16(vget_high_s16(v_x1), cospi_16_64); - { - const int16x4_t a = vrshrn_n_s32(v_t0_lo, DCT_CONST_BITS); - const int16x4_t b = vrshrn_n_s32(v_t0_hi, DCT_CONST_BITS); - const int16x4_t c = vrshrn_n_s32(v_t1_lo, DCT_CONST_BITS); - const int16x4_t d = vrshrn_n_s32(v_t1_hi, DCT_CONST_BITS); - const int16x8_t ab = vcombine_s16(a, b); - const int16x8_t cd = vcombine_s16(c, d); - // Stage 3 - v_x0 = vaddq_s16(v_s4, ab); - v_x1 = vsubq_s16(v_s4, ab); - v_x2 = vsubq_s16(v_s7, cd); - v_x3 = vaddq_s16(v_s7, cd); - } - // Stage 4 - v_t0_lo = vmull_n_s16(vget_low_s16(v_x3), cospi_4_64); - v_t0_hi = vmull_n_s16(vget_high_s16(v_x3), cospi_4_64); - v_t0_lo = vmlal_n_s16(v_t0_lo, vget_low_s16(v_x0), cospi_28_64); - v_t0_hi = vmlal_n_s16(v_t0_hi, vget_high_s16(v_x0), cospi_28_64); - v_t1_lo = vmull_n_s16(vget_low_s16(v_x1), cospi_12_64); - v_t1_hi = vmull_n_s16(vget_high_s16(v_x1), cospi_12_64); - v_t1_lo = vmlal_n_s16(v_t1_lo, vget_low_s16(v_x2), cospi_20_64); - v_t1_hi = vmlal_n_s16(v_t1_hi, vget_high_s16(v_x2), cospi_20_64); - v_t2_lo = vmull_n_s16(vget_low_s16(v_x2), cospi_12_64); - v_t2_hi = vmull_n_s16(vget_high_s16(v_x2), cospi_12_64); - v_t2_lo = vmlsl_n_s16(v_t2_lo, vget_low_s16(v_x1), cospi_20_64); - v_t2_hi = vmlsl_n_s16(v_t2_hi, vget_high_s16(v_x1), cospi_20_64); - v_t3_lo = vmull_n_s16(vget_low_s16(v_x3), cospi_28_64); - v_t3_hi = vmull_n_s16(vget_high_s16(v_x3), cospi_28_64); - v_t3_lo = vmlsl_n_s16(v_t3_lo, vget_low_s16(v_x0), cospi_4_64); - v_t3_hi = vmlsl_n_s16(v_t3_hi, vget_high_s16(v_x0), cospi_4_64); - { - const int16x4_t a = vrshrn_n_s32(v_t0_lo, DCT_CONST_BITS); - const int16x4_t b = vrshrn_n_s32(v_t0_hi, DCT_CONST_BITS); - const int16x4_t c = vrshrn_n_s32(v_t1_lo, DCT_CONST_BITS); - const int16x4_t d = vrshrn_n_s32(v_t1_hi, DCT_CONST_BITS); - const int16x4_t e = vrshrn_n_s32(v_t2_lo, DCT_CONST_BITS); - const int16x4_t f = vrshrn_n_s32(v_t2_hi, DCT_CONST_BITS); - const int16x4_t g = vrshrn_n_s32(v_t3_lo, DCT_CONST_BITS); - const int16x4_t h = vrshrn_n_s32(v_t3_hi, DCT_CONST_BITS); - out_1 = vcombine_s16(a, c); // 10 11 12 13 50 51 52 53 - out_3 = vcombine_s16(e, g); // 30 31 32 33 70 71 72 73 - out_5 = vcombine_s16(b, d); // 14 15 16 17 54 55 56 57 - out_7 = vcombine_s16(f, h); // 34 35 36 37 74 75 76 77 - } - // transpose 8x8 - // Can't use transpose_s16_8x8() because the values are arranged in two 4x8 - // columns. - { - // 00 01 02 03 40 41 42 43 - // 10 11 12 13 50 51 52 53 - // 20 21 22 23 60 61 62 63 - // 30 31 32 33 70 71 72 73 - // 04 05 06 07 44 45 46 47 - // 14 15 16 17 54 55 56 57 - // 24 25 26 27 64 65 66 67 - // 34 35 36 37 74 75 76 77 - const int32x4x2_t r02_s32 = - vtrnq_s32(vreinterpretq_s32_s16(out_0), vreinterpretq_s32_s16(out_2)); - const int32x4x2_t r13_s32 = - vtrnq_s32(vreinterpretq_s32_s16(out_1), vreinterpretq_s32_s16(out_3)); - const int32x4x2_t r46_s32 = - vtrnq_s32(vreinterpretq_s32_s16(out_4), vreinterpretq_s32_s16(out_6)); - const int32x4x2_t r57_s32 = - vtrnq_s32(vreinterpretq_s32_s16(out_5), vreinterpretq_s32_s16(out_7)); - const int16x8x2_t r01_s16 = - vtrnq_s16(vreinterpretq_s16_s32(r02_s32.val[0]), - vreinterpretq_s16_s32(r13_s32.val[0])); - const int16x8x2_t r23_s16 = - vtrnq_s16(vreinterpretq_s16_s32(r02_s32.val[1]), - vreinterpretq_s16_s32(r13_s32.val[1])); - const int16x8x2_t r45_s16 = - vtrnq_s16(vreinterpretq_s16_s32(r46_s32.val[0]), - vreinterpretq_s16_s32(r57_s32.val[0])); - const int16x8x2_t r67_s16 = - vtrnq_s16(vreinterpretq_s16_s32(r46_s32.val[1]), - vreinterpretq_s16_s32(r57_s32.val[1])); - input_0 = r01_s16.val[0]; - input_1 = r01_s16.val[1]; - input_2 = r23_s16.val[0]; - input_3 = r23_s16.val[1]; - input_4 = r45_s16.val[0]; - input_5 = r45_s16.val[1]; - input_6 = r67_s16.val[0]; - input_7 = r67_s16.val[1]; - // 00 10 20 30 40 50 60 70 - // 01 11 21 31 41 51 61 71 - // 02 12 22 32 42 52 62 72 - // 03 13 23 33 43 53 63 73 - // 04 14 24 34 44 54 64 74 - // 05 15 25 35 45 55 65 75 - // 06 16 26 36 46 56 66 76 - // 07 17 27 37 47 57 67 77 - } + vpx_fdct8x8_pass1_neon(in); } // for { // from vpx_dct_sse2.c // Post-condition (division by two) // division of two 16 bits signed numbers using shifts // n / 2 = (n - (n >> 15)) >> 1 - const int16x8_t sign_in0 = vshrq_n_s16(input_0, 15); - const int16x8_t sign_in1 = vshrq_n_s16(input_1, 15); - const int16x8_t sign_in2 = vshrq_n_s16(input_2, 15); - const int16x8_t sign_in3 = vshrq_n_s16(input_3, 15); - const int16x8_t sign_in4 = vshrq_n_s16(input_4, 15); - const int16x8_t sign_in5 = vshrq_n_s16(input_5, 15); - const int16x8_t sign_in6 = vshrq_n_s16(input_6, 15); - const int16x8_t sign_in7 = vshrq_n_s16(input_7, 15); - input_0 = vhsubq_s16(input_0, sign_in0); - input_1 = vhsubq_s16(input_1, sign_in1); - input_2 = vhsubq_s16(input_2, sign_in2); - input_3 = vhsubq_s16(input_3, sign_in3); - input_4 = vhsubq_s16(input_4, sign_in4); - input_5 = vhsubq_s16(input_5, sign_in5); - input_6 = vhsubq_s16(input_6, sign_in6); - input_7 = vhsubq_s16(input_7, sign_in7); + const int16x8_t sign_in0 = vshrq_n_s16(in[0], 15); + const int16x8_t sign_in1 = vshrq_n_s16(in[1], 15); + const int16x8_t sign_in2 = vshrq_n_s16(in[2], 15); + const int16x8_t sign_in3 = vshrq_n_s16(in[3], 15); + const int16x8_t sign_in4 = vshrq_n_s16(in[4], 15); + const int16x8_t sign_in5 = vshrq_n_s16(in[5], 15); + const int16x8_t sign_in6 = vshrq_n_s16(in[6], 15); + const int16x8_t sign_in7 = vshrq_n_s16(in[7], 15); + in[0] = vhsubq_s16(in[0], sign_in0); + in[1] = vhsubq_s16(in[1], sign_in1); + in[2] = vhsubq_s16(in[2], sign_in2); + in[3] = vhsubq_s16(in[3], sign_in3); + in[4] = vhsubq_s16(in[4], sign_in4); + in[5] = vhsubq_s16(in[5], sign_in5); + in[6] = vhsubq_s16(in[6], sign_in6); + in[7] = vhsubq_s16(in[7], sign_in7); // store results - store_s16q_to_tran_low(final_output + 0 * 8, input_0); - store_s16q_to_tran_low(final_output + 1 * 8, input_1); - store_s16q_to_tran_low(final_output + 2 * 8, input_2); - store_s16q_to_tran_low(final_output + 3 * 8, input_3); - store_s16q_to_tran_low(final_output + 4 * 8, input_4); - store_s16q_to_tran_low(final_output + 5 * 8, input_5); - store_s16q_to_tran_low(final_output + 6 * 8, input_6); - store_s16q_to_tran_low(final_output + 7 * 8, input_7); + store_s16q_to_tran_low(final_output + 0 * 8, in[0]); + store_s16q_to_tran_low(final_output + 1 * 8, in[1]); + store_s16q_to_tran_low(final_output + 2 * 8, in[2]); + store_s16q_to_tran_low(final_output + 3 * 8, in[3]); + store_s16q_to_tran_low(final_output + 4 * 8, in[4]); + store_s16q_to_tran_low(final_output + 5 * 8, in[5]); + store_s16q_to_tran_low(final_output + 6 * 8, in[6]); + store_s16q_to_tran_low(final_output + 7 * 8, in[7]); } } diff --git a/media/libvpx/libvpx/vpx_dsp/arm/idct4x4_add_neon.asm b/media/libvpx/libvpx/vpx_dsp/arm/idct4x4_add_neon.asm --- a/media/libvpx/libvpx/vpx_dsp/arm/idct4x4_add_neon.asm +++ b/media/libvpx/libvpx/vpx_dsp/arm/idct4x4_add_neon.asm @@ -15,11 +15,11 @@ AREA ||.text||, CODE, READONLY, ALIGN=2 INCLUDE vpx_dsp/arm/idct_neon.asm.S - AREA Block, CODE, READONLY ; name this block of code + AREA Block, CODE, READONLY ;void vpx_idct4x4_16_add_neon(int16_t *input, uint8_t *dest, int stride) ; ; r0 int16_t input ; r1 uint8_t *dest ; r2 int stride) diff --git a/media/libvpx/libvpx/vpx_dsp/arm/quantize_neon.c b/media/libvpx/libvpx/vpx_dsp/arm/quantize_neon.c --- a/media/libvpx/libvpx/vpx_dsp/arm/quantize_neon.c +++ b/media/libvpx/libvpx/vpx_dsp/arm/quantize_neon.c @@ -30,22 +30,20 @@ vst1q_s16(dqcoeff, vcombine_s16(vmovn_s32(dqcoeff_0), vmovn_s32(dqcoeff_1))); #endif // CONFIG_VP9_HIGHBITDEPTH } void vpx_quantize_b_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *zbin_ptr, - const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *zbin_ptr, const int16_t *round_ptr, + const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan) { const int16x8_t one = vdupq_n_s16(1); const int16x8_t neg_one = vdupq_n_s16(-1); uint16x8_t eob_max; (void)scan; - (void)skip_block; - assert(!skip_block); // Process first 8 values which include a dc component. { // Only the first element of each vector is DC. const int16x8_t zbin = vld1q_s16(zbin_ptr); @@ -187,11 +185,11 @@ } // Main difference is that zbin values are halved before comparison and dqcoeff // values are divided by 2. zbin is rounded but dqcoeff is not. void vpx_quantize_b_32x32_neon(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *zbin_ptr, + const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, @@ -200,12 +198,10 @@ const int16x8_t neg_one = vdupq_n_s16(-1); uint16x8_t eob_max; int i; (void)scan; (void)n_coeffs; // Because we will always calculate 32*32. - (void)skip_block; - assert(!skip_block); // Process first 8 values which include a dc component. { // Only the first element of each vector is DC. const int16x8_t zbin = vrshrq_n_s16(vld1q_s16(zbin_ptr), 1); diff --git a/media/libvpx/libvpx/vpx_dsp/arm/sad4d_neon.c b/media/libvpx/libvpx/vpx_dsp/arm/sad4d_neon.c --- a/media/libvpx/libvpx/vpx_dsp/arm/sad4d_neon.c +++ b/media/libvpx/libvpx/vpx_dsp/arm/sad4d_neon.c @@ -29,11 +29,11 @@ } static INLINE void sad4x_4d(const uint8_t *const src_ptr, const int src_stride, const uint8_t *const ref_array[4], const int ref_stride, const int height, - uint32_t *const res) { + uint32_t sad_array[4]) { int i; uint16x8_t abs[2] = { vdupq_n_u16(0), vdupq_n_u16(0) }; #if !defined(__aarch64__) uint16x4_t a[2]; #endif @@ -59,30 +59,30 @@ #else a[0] = vpadd_u16(vget_low_u16(abs[0]), vget_high_u16(abs[0])); a[1] = vpadd_u16(vget_low_u16(abs[1]), vget_high_u16(abs[1])); r = vpaddlq_u16(vcombine_u16(a[0], a[1])); #endif - vst1q_u32(res, r); + vst1q_u32(sad_array, r); } void vpx_sad4x4x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { - sad4x_4d(src_ptr, src_stride, ref_array, ref_stride, 4, res); + uint32_t sad_array[4]) { + sad4x_4d(src_ptr, src_stride, ref_array, ref_stride, 4, sad_array); } void vpx_sad4x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { - sad4x_4d(src_ptr, src_stride, ref_array, ref_stride, 8, res); + uint32_t sad_array[4]) { + sad4x_4d(src_ptr, src_stride, ref_array, ref_stride, 8, sad_array); } //////////////////////////////////////////////////////////////////////////////// // Can handle 512 pixels' sad sum (such as 16x32 or 32x16) -static INLINE void sad_512_pel_final_neon(const uint16x8_t *sum /*[4]*/, - uint32_t *const res) { +static INLINE void sad_512_pel_final_neon(const uint16x8_t sum[4], + uint32_t sad_array[4]) { #if defined(__aarch64__) const uint16x8_t a0 = vpaddq_u16(sum[0], sum[1]); const uint16x8_t a1 = vpaddq_u16(sum[2], sum[3]); const uint16x8_t b0 = vpaddq_u16(a0, a1); const uint32x4_t r = vpaddlq_u16(b0); @@ -93,50 +93,50 @@ const uint16x4_t a3 = vadd_u16(vget_low_u16(sum[3]), vget_high_u16(sum[3])); const uint16x4_t b0 = vpadd_u16(a0, a1); const uint16x4_t b1 = vpadd_u16(a2, a3); const uint32x4_t r = vpaddlq_u16(vcombine_u16(b0, b1)); #endif - vst1q_u32(res, r); + vst1q_u32(sad_array, r); } #if defined(__arm__) || !defined(__ARM_FEATURE_DOTPROD) // Can handle 1024 pixels' sad sum (such as 32x32) -static INLINE void sad_1024_pel_final_neon(const uint16x8_t *sum /*[4]*/, - uint32_t *const res) { +static INLINE void sad_1024_pel_final_neon(const uint16x8_t sum[4], + uint32_t sad_array[4]) { #if defined(__aarch64__) const uint16x8_t a0 = vpaddq_u16(sum[0], sum[1]); const uint16x8_t a1 = vpaddq_u16(sum[2], sum[3]); const uint32x4_t b0 = vpaddlq_u16(a0); const uint32x4_t b1 = vpaddlq_u16(a1); const uint32x4_t r = vpaddq_u32(b0, b1); - vst1q_u32(res, r); + vst1q_u32(sad_array, r); #else const uint16x4_t a0 = vpadd_u16(vget_low_u16(sum[0]), vget_high_u16(sum[0])); const uint16x4_t a1 = vpadd_u16(vget_low_u16(sum[1]), vget_high_u16(sum[1])); const uint16x4_t a2 = vpadd_u16(vget_low_u16(sum[2]), vget_high_u16(sum[2])); const uint16x4_t a3 = vpadd_u16(vget_low_u16(sum[3]), vget_high_u16(sum[3])); const uint32x4_t b0 = vpaddlq_u16(vcombine_u16(a0, a1)); const uint32x4_t b1 = vpaddlq_u16(vcombine_u16(a2, a3)); const uint32x2_t c0 = vpadd_u32(vget_low_u32(b0), vget_high_u32(b0)); const uint32x2_t c1 = vpadd_u32(vget_low_u32(b1), vget_high_u32(b1)); - vst1q_u32(res, vcombine_u32(c0, c1)); + vst1q_u32(sad_array, vcombine_u32(c0, c1)); #endif } // Can handle 2048 pixels' sad sum (such as 32x64 or 64x32) -static INLINE void sad_2048_pel_final_neon(const uint16x8_t *sum /*[4]*/, - uint32_t *const res) { +static INLINE void sad_2048_pel_final_neon(const uint16x8_t sum[4], + uint32_t sad_array[4]) { #if defined(__aarch64__) const uint32x4_t a0 = vpaddlq_u16(sum[0]); const uint32x4_t a1 = vpaddlq_u16(sum[1]); const uint32x4_t a2 = vpaddlq_u16(sum[2]); const uint32x4_t a3 = vpaddlq_u16(sum[3]); const uint32x4_t b0 = vpaddq_u32(a0, a1); const uint32x4_t b1 = vpaddq_u32(a2, a3); const uint32x4_t r = vpaddq_u32(b0, b1); - vst1q_u32(res, r); + vst1q_u32(sad_array, r); #else const uint32x4_t a0 = vpaddlq_u16(sum[0]); const uint32x4_t a1 = vpaddlq_u16(sum[1]); const uint32x4_t a2 = vpaddlq_u16(sum[2]); const uint32x4_t a3 = vpaddlq_u16(sum[3]); @@ -144,17 +144,17 @@ const uint32x2_t b1 = vadd_u32(vget_low_u32(a1), vget_high_u32(a1)); const uint32x2_t b2 = vadd_u32(vget_low_u32(a2), vget_high_u32(a2)); const uint32x2_t b3 = vadd_u32(vget_low_u32(a3), vget_high_u32(a3)); const uint32x2_t c0 = vpadd_u32(b0, b1); const uint32x2_t c1 = vpadd_u32(b2, b3); - vst1q_u32(res, vcombine_u32(c0, c1)); + vst1q_u32(sad_array, vcombine_u32(c0, c1)); #endif } // Can handle 4096 pixels' sad sum (such as 64x64) -static INLINE void sad_4096_pel_final_neon(const uint16x8_t *sum /*[8]*/, - uint32_t *const res) { +static INLINE void sad_4096_pel_final_neon(const uint16x8_t sum[8], + uint32_t sad_array[4]) { #if defined(__aarch64__) const uint32x4_t a0 = vpaddlq_u16(sum[0]); const uint32x4_t a1 = vpaddlq_u16(sum[1]); const uint32x4_t a2 = vpaddlq_u16(sum[2]); const uint32x4_t a3 = vpaddlq_u16(sum[3]); @@ -167,11 +167,11 @@ const uint32x4_t b2 = vaddq_u32(a4, a5); const uint32x4_t b3 = vaddq_u32(a6, a7); const uint32x4_t c0 = vpaddq_u32(b0, b1); const uint32x4_t c1 = vpaddq_u32(b2, b3); const uint32x4_t r = vpaddq_u32(c0, c1); - vst1q_u32(res, r); + vst1q_u32(sad_array, r); #else const uint32x4_t a0 = vpaddlq_u16(sum[0]); const uint32x4_t a1 = vpaddlq_u16(sum[1]); const uint32x4_t a2 = vpaddlq_u16(sum[2]); const uint32x4_t a3 = vpaddlq_u16(sum[3]); @@ -187,19 +187,19 @@ const uint32x2_t c1 = vadd_u32(vget_low_u32(b1), vget_high_u32(b1)); const uint32x2_t c2 = vadd_u32(vget_low_u32(b2), vget_high_u32(b2)); const uint32x2_t c3 = vadd_u32(vget_low_u32(b3), vget_high_u32(b3)); const uint32x2_t d0 = vpadd_u32(c0, c1); const uint32x2_t d1 = vpadd_u32(c2, c3); - vst1q_u32(res, vcombine_u32(d0, d1)); + vst1q_u32(sad_array, vcombine_u32(d0, d1)); #endif } #endif static INLINE void sad8x_4d(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res, const int height) { + uint32_t sad_array[4], const int height) { int i, j; const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2], ref_array[3] }; uint16x8_t sum[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0) }; @@ -212,29 +212,29 @@ ref_loop[j] += ref_stride; sum[j] = vabal_u8(sum[j], s, b_u8); } } - sad_512_pel_final_neon(sum, res); + sad_512_pel_final_neon(sum, sad_array); } void vpx_sad8x4x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { - sad8x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 4); + uint32_t sad_array[4]) { + sad8x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 4); } void vpx_sad8x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { - sad8x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 8); + uint32_t sad_array[4]) { + sad8x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 8); } void vpx_sad8x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { - sad8x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 16); + uint32_t sad_array[4]) { + sad8x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 16); } //////////////////////////////////////////////////////////////////////////////// #if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \ @@ -247,11 +247,11 @@ *sum = vdotq_u32(*sum, diff, vdupq_n_u8(1)); } static INLINE void sad16x_4d(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res, const int height) { + uint32_t sad_array[4], const int height) { int i; uint32x4_t r0, r1; const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2], ref_array[3] }; uint32x4_t sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0), @@ -265,11 +265,11 @@ sad16_neon(ref_loop[3] + i * ref_stride, s, &sum[3]); } r0 = vpaddq_u32(sum[0], sum[1]); r1 = vpaddq_u32(sum[2], sum[3]); - vst1q_u32(res, vpaddq_u32(r0, r1)); + vst1q_u32(sad_array, vpaddq_u32(r0, r1)); } #else static INLINE void sad16_neon(const uint8_t *ref_ptr, const uint8x16_t src_ptr, @@ -279,11 +279,11 @@ *sum = vabal_u8(*sum, vget_high_u8(src_ptr), vget_high_u8(r)); } static INLINE void sad16x_4d(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res, const int height) { + uint32_t sad_array[4], const int height) { int i; const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2], ref_array[3] }; uint16x8_t sum[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0) }; @@ -300,41 +300,41 @@ ref_loop[2] += ref_stride; sad16_neon(ref_loop[3], s, &sum[3]); ref_loop[3] += ref_stride; } - sad_512_pel_final_neon(sum, res); + sad_512_pel_final_neon(sum, sad_array); } #endif void vpx_sad16x8x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { - sad16x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 8); + uint32_t sad_array[4]) { + sad16x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 8); } void vpx_sad16x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { - sad16x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 16); + uint32_t sad_array[4]) { + sad16x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 16); } void vpx_sad16x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { - sad16x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 32); + uint32_t sad_array[4]) { + sad16x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 32); } //////////////////////////////////////////////////////////////////////////////// #if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \ (__ARM_FEATURE_DOTPROD == 1) static INLINE void sad32x_4d(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res, const int height) { + uint32_t sad_array[4], const int height) { int i; uint32x4_t r0, r1; const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2], ref_array[3] }; @@ -363,29 +363,29 @@ ref_loop[3] += ref_stride; } r0 = vpaddq_u32(sum[0], sum[1]); r1 = vpaddq_u32(sum[2], sum[3]); - vst1q_u32(res, vpaddq_u32(r0, r1)); + vst1q_u32(sad_array, vpaddq_u32(r0, r1)); } void vpx_sad32x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { - sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 16); + uint32_t sad_array[4]) { + sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 16); } void vpx_sad32x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { - sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 32); + uint32_t sad_array[4]) { + sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 32); } void vpx_sad32x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { - sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, res, 64); + uint32_t sad_array[4]) { + sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, sad_array, 64); } #else static INLINE void sad32x_4d(const uint8_t *src_ptr, int src_stride, @@ -420,30 +420,30 @@ } } void vpx_sad32x16x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { + uint32_t sad_array[4]) { uint16x8_t sum[4]; sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, 16, sum); - sad_512_pel_final_neon(sum, res); + sad_512_pel_final_neon(sum, sad_array); } void vpx_sad32x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { + uint32_t sad_array[4]) { uint16x8_t sum[4]; sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, 32, sum); - sad_1024_pel_final_neon(sum, res); + sad_1024_pel_final_neon(sum, sad_array); } void vpx_sad32x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { + uint32_t sad_array[4]) { uint16x8_t sum[4]; sad32x_4d(src_ptr, src_stride, ref_array, ref_stride, 64, sum); - sad_2048_pel_final_neon(sum, res); + sad_2048_pel_final_neon(sum, sad_array); } #endif //////////////////////////////////////////////////////////////////////////////// @@ -451,11 +451,11 @@ #if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \ (__ARM_FEATURE_DOTPROD == 1) void vpx_sad64x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { + uint32_t sad_array[4]) { int i; uint32x4_t r0, r1; const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2], ref_array[3] }; uint32x4_t sum[4] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0), @@ -495,16 +495,16 @@ ref_loop[3] += ref_stride; } r0 = vpaddq_u32(sum[0], sum[1]); r1 = vpaddq_u32(sum[2], sum[3]); - vst1q_u32(res, vpaddq_u32(r0, r1)); + vst1q_u32(sad_array, vpaddq_u32(r0, r1)); } void vpx_sad64x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { + uint32_t sad_array[4]) { int i; uint32x4_t r0, r1, r2, r3; const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2], ref_array[3] }; uint32x4_t sum[8] = { vdupq_n_u32(0), vdupq_n_u32(0), vdupq_n_u32(0), @@ -549,18 +549,18 @@ r1 = vpaddq_u32(sum[2], sum[3]); r2 = vpaddq_u32(sum[4], sum[5]); r3 = vpaddq_u32(sum[6], sum[7]); r0 = vpaddq_u32(r0, r1); r1 = vpaddq_u32(r2, r3); - vst1q_u32(res, vpaddq_u32(r0, r1)); + vst1q_u32(sad_array, vpaddq_u32(r0, r1)); } #else void vpx_sad64x32x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { + uint32_t sad_array[4]) { int i; const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2], ref_array[3] }; uint16x8_t sum[4] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0) }; @@ -597,16 +597,16 @@ ref_loop[1] += ref_stride; ref_loop[2] += ref_stride; ref_loop[3] += ref_stride; } - sad_2048_pel_final_neon(sum, res); + sad_2048_pel_final_neon(sum, sad_array); } void vpx_sad64x64x4d_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t *res) { + uint32_t sad_array[4]) { int i; const uint8_t *ref_loop[4] = { ref_array[0], ref_array[1], ref_array[2], ref_array[3] }; uint16x8_t sum[8] = { vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0), vdupq_n_u16(0), @@ -644,9 +644,9 @@ ref_loop[1] += ref_stride; ref_loop[2] += ref_stride; ref_loop[3] += ref_stride; } - sad_4096_pel_final_neon(sum, res); + sad_4096_pel_final_neon(sum, sad_array); } #endif diff --git a/media/libvpx/libvpx/vpx_dsp/arm/sad_neon.c b/media/libvpx/libvpx/vpx_dsp/arm/sad_neon.c --- a/media/libvpx/libvpx/vpx_dsp/arm/sad_neon.c +++ b/media/libvpx/libvpx/vpx_dsp/arm/sad_neon.c @@ -108,11 +108,11 @@ abs = vabal_u8(abs, a_u8, avg); } return abs; } -#define sad8xN(n) \ +#define SAD8XN(n) \ uint32_t vpx_sad8x##n##_neon(const uint8_t *src_ptr, int src_stride, \ const uint8_t *ref_ptr, int ref_stride) { \ const uint16x8_t abs = sad8x(src_ptr, src_stride, ref_ptr, ref_stride, n); \ return horizontal_add_uint16x8(abs); \ } \ @@ -123,13 +123,13 @@ const uint16x8_t abs = \ sad8x_avg(src_ptr, src_stride, ref_ptr, ref_stride, second_pred, n); \ return horizontal_add_uint16x8(abs); \ } -sad8xN(4); -sad8xN(8); -sad8xN(16); +SAD8XN(4) +SAD8XN(8) +SAD8XN(16) static INLINE uint16x8_t sad16x(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const int height) { int i; @@ -165,11 +165,11 @@ abs = vabal_u8(abs, vget_high_u8(a_u8), vget_high_u8(avg)); } return abs; } -#define sad16xN(n) \ +#define SAD16XN(n) \ uint32_t vpx_sad16x##n##_neon(const uint8_t *src_ptr, int src_stride, \ const uint8_t *ref_ptr, int ref_stride) { \ const uint16x8_t abs = \ sad16x(src_ptr, src_stride, ref_ptr, ref_stride, n); \ return horizontal_add_uint16x8(abs); \ @@ -181,13 +181,13 @@ const uint16x8_t abs = \ sad16x_avg(src_ptr, src_stride, ref_ptr, ref_stride, second_pred, n); \ return horizontal_add_uint16x8(abs); \ } -sad16xN(8); -sad16xN(16); -sad16xN(32); +SAD16XN(8) +SAD16XN(16) +SAD16XN(32) static INLINE uint16x8_t sad32x(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const int height) { int i; @@ -233,11 +233,11 @@ abs = vabal_u8(abs, vget_high_u8(a_hi), vget_high_u8(avg_hi)); } return abs; } -#define sad32xN(n) \ +#define SAD32XN(n) \ uint32_t vpx_sad32x##n##_neon(const uint8_t *src_ptr, int src_stride, \ const uint8_t *ref_ptr, int ref_stride) { \ const uint16x8_t abs = \ sad32x(src_ptr, src_stride, ref_ptr, ref_stride, n); \ return horizontal_add_uint16x8(abs); \ @@ -249,13 +249,13 @@ const uint16x8_t abs = \ sad32x_avg(src_ptr, src_stride, ref_ptr, ref_stride, second_pred, n); \ return horizontal_add_uint16x8(abs); \ } -sad32xN(16); -sad32xN(32); -sad32xN(64); +SAD32XN(16) +SAD32XN(32) +SAD32XN(64) static INLINE uint32x4_t sad64x(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const int height) { int i; @@ -331,11 +331,11 @@ const uint32x4_t sum = vpaddlq_u16(abs_0); return vpadalq_u16(sum, abs_1); } } -#define sad64xN(n) \ +#define SAD64XN(n) \ uint32_t vpx_sad64x##n##_neon(const uint8_t *src_ptr, int src_stride, \ const uint8_t *ref_ptr, int ref_stride) { \ const uint32x4_t abs = \ sad64x(src_ptr, src_stride, ref_ptr, ref_stride, n); \ return horizontal_add_uint32x4(abs); \ @@ -347,7 +347,7 @@ const uint32x4_t abs = \ sad64x_avg(src_ptr, src_stride, ref_ptr, ref_stride, second_pred, n); \ return horizontal_add_uint32x4(abs); \ } -sad64xN(32); -sad64xN(64); +SAD64XN(32) +SAD64XN(64) diff --git a/media/libvpx/libvpx/vpx_dsp/arm/subpel_variance_neon.c b/media/libvpx/libvpx/vpx_dsp/arm/subpel_variance_neon.c --- a/media/libvpx/libvpx/vpx_dsp/arm/subpel_variance_neon.c +++ b/media/libvpx/libvpx/vpx_dsp/arm/subpel_variance_neon.c @@ -95,11 +95,11 @@ } } // 4xM filter writes an extra row to fdata because it processes two rows at a // time. -#define sub_pixel_varianceNxM(n, m) \ +#define SUB_PIXEL_VARIANCENXM(n, m) \ uint32_t vpx_sub_pixel_variance##n##x##m##_neon( \ const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, \ const uint8_t *ref_ptr, int ref_stride, uint32_t *sse) { \ uint8_t temp0[n * (m + (n == 4 ? 2 : 1))]; \ uint8_t temp1[n * m]; \ @@ -121,27 +121,27 @@ bilinear_filters[y_offset]); \ } \ return vpx_variance##n##x##m(temp1, n, ref_ptr, ref_stride, sse); \ } -sub_pixel_varianceNxM(4, 4); -sub_pixel_varianceNxM(4, 8); -sub_pixel_varianceNxM(8, 4); -sub_pixel_varianceNxM(8, 8); -sub_pixel_varianceNxM(8, 16); -sub_pixel_varianceNxM(16, 8); -sub_pixel_varianceNxM(16, 16); -sub_pixel_varianceNxM(16, 32); -sub_pixel_varianceNxM(32, 16); -sub_pixel_varianceNxM(32, 32); -sub_pixel_varianceNxM(32, 64); -sub_pixel_varianceNxM(64, 32); -sub_pixel_varianceNxM(64, 64); +SUB_PIXEL_VARIANCENXM(4, 4) +SUB_PIXEL_VARIANCENXM(4, 8) +SUB_PIXEL_VARIANCENXM(8, 4) +SUB_PIXEL_VARIANCENXM(8, 8) +SUB_PIXEL_VARIANCENXM(8, 16) +SUB_PIXEL_VARIANCENXM(16, 8) +SUB_PIXEL_VARIANCENXM(16, 16) +SUB_PIXEL_VARIANCENXM(16, 32) +SUB_PIXEL_VARIANCENXM(32, 16) +SUB_PIXEL_VARIANCENXM(32, 32) +SUB_PIXEL_VARIANCENXM(32, 64) +SUB_PIXEL_VARIANCENXM(64, 32) +SUB_PIXEL_VARIANCENXM(64, 64) // 4xM filter writes an extra row to fdata because it processes two rows at a // time. -#define sub_pixel_avg_varianceNxM(n, m) \ +#define SUB_PIXEL_AVG_VARIANCENXM(n, m) \ uint32_t vpx_sub_pixel_avg_variance##n##x##m##_neon( \ const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, \ const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, \ const uint8_t *second_pred) { \ uint8_t temp0[n * (m + (n == 4 ? 2 : 1))]; \ @@ -167,18 +167,18 @@ vpx_comp_avg_pred(temp0, second_pred, n, m, temp1, n); \ \ return vpx_variance##n##x##m(temp0, n, ref_ptr, ref_stride, sse); \ } -sub_pixel_avg_varianceNxM(4, 4); -sub_pixel_avg_varianceNxM(4, 8); -sub_pixel_avg_varianceNxM(8, 4); -sub_pixel_avg_varianceNxM(8, 8); -sub_pixel_avg_varianceNxM(8, 16); -sub_pixel_avg_varianceNxM(16, 8); -sub_pixel_avg_varianceNxM(16, 16); -sub_pixel_avg_varianceNxM(16, 32); -sub_pixel_avg_varianceNxM(32, 16); -sub_pixel_avg_varianceNxM(32, 32); -sub_pixel_avg_varianceNxM(32, 64); -sub_pixel_avg_varianceNxM(64, 32); -sub_pixel_avg_varianceNxM(64, 64); +SUB_PIXEL_AVG_VARIANCENXM(4, 4) +SUB_PIXEL_AVG_VARIANCENXM(4, 8) +SUB_PIXEL_AVG_VARIANCENXM(8, 4) +SUB_PIXEL_AVG_VARIANCENXM(8, 8) +SUB_PIXEL_AVG_VARIANCENXM(8, 16) +SUB_PIXEL_AVG_VARIANCENXM(16, 8) +SUB_PIXEL_AVG_VARIANCENXM(16, 16) +SUB_PIXEL_AVG_VARIANCENXM(16, 32) +SUB_PIXEL_AVG_VARIANCENXM(32, 16) +SUB_PIXEL_AVG_VARIANCENXM(32, 32) +SUB_PIXEL_AVG_VARIANCENXM(32, 64) +SUB_PIXEL_AVG_VARIANCENXM(64, 32) +SUB_PIXEL_AVG_VARIANCENXM(64, 64) diff --git a/media/libvpx/libvpx/vpx_dsp/arm/transpose_neon.h b/media/libvpx/libvpx/vpx_dsp/arm/transpose_neon.h --- a/media/libvpx/libvpx/vpx_dsp/arm/transpose_neon.h +++ b/media/libvpx/libvpx/vpx_dsp/arm/transpose_neon.h @@ -1182,10 +1182,49 @@ *o13 = e5.val[1]; *o14 = e6.val[1]; *o15 = e7.val[1]; } +static INLINE void transpose_s16_16x16(int16x8_t *in0, int16x8_t *in1) { + int16x8_t t[8]; + + // transpose the 4 8x8 quadrants separately but first swap quadrants 2 and 3. + t[0] = in0[8]; + t[1] = in0[9]; + t[2] = in0[10]; + t[3] = in0[11]; + t[4] = in0[12]; + t[5] = in0[13]; + t[6] = in0[14]; + t[7] = in0[15]; + in0[8] = in1[0]; + in0[9] = in1[1]; + in0[10] = in1[2]; + in0[11] = in1[3]; + in0[12] = in1[4]; + in0[13] = in1[5]; + in0[14] = in1[6]; + in0[15] = in1[7]; + in1[0] = t[0]; + in1[1] = t[1]; + in1[2] = t[2]; + in1[3] = t[3]; + in1[4] = t[4]; + in1[5] = t[5]; + in1[6] = t[6]; + in1[7] = t[7]; + + transpose_s16_8x8(&in0[0], &in0[1], &in0[2], &in0[3], &in0[4], &in0[5], + &in0[6], &in0[7]); + transpose_s16_8x8(&in0[8], &in0[9], &in0[10], &in0[11], &in0[12], &in0[13], + &in0[14], &in0[15]); + transpose_s16_8x8(&in1[0], &in1[1], &in1[2], &in1[3], &in1[4], &in1[5], + &in1[6], &in1[7]); + transpose_s16_8x8(&in1[8], &in1[9], &in1[10], &in1[11], &in1[12], &in1[13], + &in1[14], &in1[15]); +} + static INLINE void load_and_transpose_u8_4x8(const uint8_t *a, const int a_stride, uint8x8_t *a0, uint8x8_t *a1, uint8x8_t *a2, uint8x8_t *a3) { uint8x8_t a4, a5, a6, a7; diff --git a/media/libvpx/libvpx/vpx_dsp/arm/variance_neon.c b/media/libvpx/libvpx/vpx_dsp/arm/variance_neon.c --- a/media/libvpx/libvpx/vpx_dsp/arm/variance_neon.c +++ b/media/libvpx/libvpx/vpx_dsp/arm/variance_neon.c @@ -266,11 +266,11 @@ const uint8_t *ref_ptr, int ref_stride, unsigned int *sse, int *sum) { variance_neon_w16(src_ptr, src_stride, ref_ptr, ref_stride, 16, 16, sse, sum); } -#define varianceNxM(n, m, shift) \ +#define VARIANCENXM(n, m, shift) \ unsigned int vpx_variance##n##x##m##_neon( \ const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \ int ref_stride, unsigned int *sse) { \ int sum; \ if (n == 4) \ @@ -286,20 +286,20 @@ return *sse - ((sum * sum) >> shift); \ else \ return *sse - (uint32_t)(((int64_t)sum * sum) >> shift); \ } -varianceNxM(4, 4, 4); -varianceNxM(4, 8, 5); -varianceNxM(8, 4, 5); -varianceNxM(8, 8, 6); -varianceNxM(8, 16, 7); -varianceNxM(16, 8, 7); -varianceNxM(16, 16, 8); -varianceNxM(16, 32, 9); -varianceNxM(32, 16, 9); -varianceNxM(32, 32, 10); +VARIANCENXM(4, 4, 4) +VARIANCENXM(4, 8, 5) +VARIANCENXM(8, 4, 5) +VARIANCENXM(8, 8, 6) +VARIANCENXM(8, 16, 7) +VARIANCENXM(16, 8, 7) +VARIANCENXM(16, 16, 8) +VARIANCENXM(16, 32, 9) +VARIANCENXM(32, 16, 9) +VARIANCENXM(32, 32, 10) unsigned int vpx_variance32x64_neon(const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse) { int sum1, sum2; diff --git a/media/libvpx/libvpx/vpx_dsp/arm/vpx_convolve8_neon_asm.h b/media/libvpx/libvpx/vpx_dsp/arm/vpx_convolve8_neon_asm.h --- a/media/libvpx/libvpx/vpx_dsp/arm/vpx_convolve8_neon_asm.h +++ b/media/libvpx/libvpx/vpx_dsp/arm/vpx_convolve8_neon_asm.h @@ -15,15 +15,15 @@ void vpx_convolve8_##dir##_filter_##type##_neon( \ const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, \ ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, \ int x_step_q4, int y0_q4, int y_step_q4, int w, int h); -DECLARE_FILTER(horiz, type1); -DECLARE_FILTER(avg_horiz, type1); -DECLARE_FILTER(horiz, type2); -DECLARE_FILTER(avg_horiz, type2); -DECLARE_FILTER(vert, type1); -DECLARE_FILTER(avg_vert, type1); -DECLARE_FILTER(vert, type2); -DECLARE_FILTER(avg_vert, type2); +DECLARE_FILTER(horiz, type1) +DECLARE_FILTER(avg_horiz, type1) +DECLARE_FILTER(horiz, type2) +DECLARE_FILTER(avg_horiz, type2) +DECLARE_FILTER(vert, type1) +DECLARE_FILTER(avg_vert, type1) +DECLARE_FILTER(vert, type2) +DECLARE_FILTER(avg_vert, type2) #endif // VPX_VPX_DSP_ARM_VPX_CONVOLVE8_NEON_ASM_H_ diff --git a/media/libvpx/libvpx/vpx_dsp/arm/vpx_convolve8_neon_asm.c b/media/libvpx/libvpx/vpx_dsp/arm/vpx_convolve8_neon_asm.c --- a/media/libvpx/libvpx/vpx_dsp/arm/vpx_convolve8_neon_asm.c +++ b/media/libvpx/libvpx/vpx_dsp/arm/vpx_convolve8_neon_asm.c @@ -33,9 +33,9 @@ src, src_stride, dst, dst_stride, filter, x0_q4, x_step_q4, y0_q4, \ y_step_q4, w, h); \ } \ } -DEFINE_FILTER(horiz); -DEFINE_FILTER(avg_horiz); -DEFINE_FILTER(vert); -DEFINE_FILTER(avg_vert); +DEFINE_FILTER(horiz) +DEFINE_FILTER(avg_horiz) +DEFINE_FILTER(vert) +DEFINE_FILTER(avg_vert) diff --git a/media/libvpx/libvpx/vpx_dsp/fastssim.c b/media/libvpx/libvpx/vpx_dsp/fastssim.c --- a/media/libvpx/libvpx/vpx_dsp/fastssim.c +++ b/media/libvpx/libvpx/vpx_dsp/fastssim.c @@ -45,11 +45,11 @@ fs_level *level; int nlevels; unsigned *col_buf; }; -static void fs_ctx_init(fs_ctx *_ctx, int _w, int _h, int _nlevels) { +static int fs_ctx_init(fs_ctx *_ctx, int _w, int _h, int _nlevels) { unsigned char *data; size_t data_size; int lw; int lh; int l; @@ -69,10 +69,11 @@ data_size += level_size; lw = (lw + 1) >> 1; lh = (lh + 1) >> 1; } data = (unsigned char *)malloc(data_size); + if (!data) return -1; _ctx->level = (fs_level *)data; _ctx->nlevels = _nlevels; data += _nlevels * sizeof(*_ctx->level); lw = (_w + 1) >> 1; lh = (_h + 1) >> 1; @@ -93,10 +94,11 @@ data += im_size * sizeof(*_ctx->level[l].ssim); lw = (lw + 1) >> 1; lh = (lh + 1) >> 1; } _ctx->col_buf = (unsigned *)data; + return 0; } static void fs_ctx_clear(fs_ctx *_ctx) { free(_ctx->level); } static void fs_downsample_level(fs_ctx *_ctx, int _l) { @@ -454,11 +456,11 @@ uint32_t _shift) { fs_ctx ctx; double ret; int l; ret = 1; - fs_ctx_init(&ctx, _w, _h, FS_NLEVELS); + if (fs_ctx_init(&ctx, _w, _h, FS_NLEVELS)) return 99.0; fs_downsample_level0(&ctx, _src, _systride, _dst, _dystride, _w, _h, _bd, _shift); for (l = 0; l < FS_NLEVELS - 1; l++) { fs_calc_structure(&ctx, l, _bd); ret *= fs_average(&ctx, l); diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/avg_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/avg_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/avg_lsx.c @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include + +#include "./vpx_dsp_rtcd.h" +#include "vpx_dsp/loongarch/bitdepth_conversion_lsx.h" + +void vpx_hadamard_8x8_lsx(const int16_t *src, ptrdiff_t src_stride, + tran_low_t *dst) { + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + ptrdiff_t src_stride2 = src_stride << 1; + ptrdiff_t src_stride3 = src_stride2 + src_stride; + ptrdiff_t src_stride4 = src_stride2 << 1; + ptrdiff_t src_stride6 = src_stride3 << 1; + + int16_t *src_tmp = (int16_t *)src; + src0 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, src_stride2, src_tmp, src_stride4, src1, src2); + src3 = __lsx_vldx(src_tmp, src_stride6); + src_tmp += src_stride4; + src4 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, src_stride2, src_tmp, src_stride4, src5, src6); + src7 = __lsx_vldx(src_tmp, src_stride6); + + LSX_BUTTERFLY_8_H(src0, src2, src4, src6, src7, src5, src3, src1, tmp0, tmp2, + tmp4, tmp6, tmp7, tmp5, tmp3, tmp1); + LSX_BUTTERFLY_8_H(tmp0, tmp1, tmp4, tmp5, tmp7, tmp6, tmp3, tmp2, src0, src1, + src4, src5, src7, src6, src3, src2); + LSX_BUTTERFLY_8_H(src0, src1, src2, src3, src7, src6, src5, src4, tmp0, tmp7, + tmp3, tmp4, tmp5, tmp1, tmp6, tmp2); + LSX_TRANSPOSE8x8_H(tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, src0, src1, + src2, src3, src4, src5, src6, src7); + LSX_BUTTERFLY_8_H(src0, src2, src4, src6, src7, src5, src3, src1, tmp0, tmp2, + tmp4, tmp6, tmp7, tmp5, tmp3, tmp1); + LSX_BUTTERFLY_8_H(tmp0, tmp1, tmp4, tmp5, tmp7, tmp6, tmp3, tmp2, src0, src1, + src4, src5, src7, src6, src3, src2); + LSX_BUTTERFLY_8_H(src0, src1, src2, src3, src7, src6, src5, src4, tmp0, tmp7, + tmp3, tmp4, tmp5, tmp1, tmp6, tmp2); + store_tran_low(tmp0, dst, 0); + store_tran_low(tmp1, dst, 8); + store_tran_low(tmp2, dst, 16); + store_tran_low(tmp3, dst, 24); + store_tran_low(tmp4, dst, 32); + store_tran_low(tmp5, dst, 40); + store_tran_low(tmp6, dst, 48); + store_tran_low(tmp7, dst, 56); +} + +void vpx_hadamard_16x16_lsx(const int16_t *src, ptrdiff_t src_stride, + tran_low_t *dst) { + int i; + __m128i a0, a1, a2, a3, b0, b1, b2, b3; + + /* Rearrange 16x16 to 8x32 and remove stride. + * Top left first. */ + vpx_hadamard_8x8_lsx(src + 0 + 0 * src_stride, src_stride, dst + 0); + /* Top right. */ + vpx_hadamard_8x8_lsx(src + 8 + 0 * src_stride, src_stride, dst + 64); + /* Bottom left. */ + vpx_hadamard_8x8_lsx(src + 0 + 8 * src_stride, src_stride, dst + 128); + /* Bottom right. */ + vpx_hadamard_8x8_lsx(src + 8 + 8 * src_stride, src_stride, dst + 192); + + for (i = 0; i < 64; i += 8) { + a0 = load_tran_low(dst); + a1 = load_tran_low(dst + 64); + a2 = load_tran_low(dst + 128); + a3 = load_tran_low(dst + 192); + + LSX_BUTTERFLY_4_H(a0, a2, a3, a1, b0, b2, b3, b1); + DUP4_ARG2(__lsx_vsrai_h, b0, 1, b1, 1, b2, 1, b3, 1, b0, b1, b2, b3); + LSX_BUTTERFLY_4_H(b0, b1, b3, b2, a0, a1, a3, a2); + + store_tran_low(a0, dst, 0); + store_tran_low(a1, dst, 64); + store_tran_low(a2, dst, 128); + store_tran_low(a3, dst, 192); + + dst += 8; + } +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/avg_pred_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/avg_pred_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/avg_pred_lsx.c @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include "./vpx_dsp_rtcd.h" +#include "vpx/vpx_integer.h" +#include "vpx_util/loongson_intrinsics.h" + +void vpx_comp_avg_pred_lsx(uint8_t *comp_pred, const uint8_t *pred, int width, + int height, const uint8_t *ref, int ref_stride) { + // width > 8 || width == 8 || width == 4 + if (width > 8) { + int i, j; + for (i = 0; i < height; ++i) { + for (j = 0; j < width; j += 16) { + __m128i p, r, avg; + + p = __lsx_vld(pred + j, 0); + r = __lsx_vld(ref + j, 0); + avg = __lsx_vavgr_bu(p, r); + __lsx_vst(avg, comp_pred + j, 0); + } + comp_pred += width; + pred += width; + ref += ref_stride; + } + } else if (width == 8) { + int i = height * width; + do { + __m128i p, r, r_0, r_1; + + p = __lsx_vld(pred, 0); + r_0 = __lsx_vld(ref, 0); + ref += ref_stride; + r_1 = __lsx_vld(ref, 0); + ref += ref_stride; + r = __lsx_vilvl_d(r_1, r_0); + r = __lsx_vavgr_bu(p, r); + + __lsx_vst(r, comp_pred, 0); + + pred += 16; + comp_pred += 16; + i -= 16; + } while (i); + } else { // width = 4 + int i = height * width; + assert(width == 4); + do { + __m128i p, r, r_0, r_1, r_2, r_3; + p = __lsx_vld(pred, 0); + + if (width == ref_stride) { + r = __lsx_vld(ref, 0); + ref += 16; + } else { + r_0 = __lsx_vld(ref, 0); + ref += ref_stride; + r_1 = __lsx_vld(ref, 0); + ref += ref_stride; + r_2 = __lsx_vld(ref, 0); + ref += ref_stride; + r_3 = __lsx_vld(ref, 0); + ref += ref_stride; + DUP2_ARG2(__lsx_vilvl_w, r_1, r_0, r_3, r_2, r_0, r_2); + r = __lsx_vilvl_d(r_2, r_0); + } + r = __lsx_vavgr_bu(p, r); + + __lsx_vst(r, comp_pred, 0); + comp_pred += 16; + pred += 16; + i -= 16; + } while (i); + } +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/bitdepth_conversion_lsx.h b/media/libvpx/libvpx/vpx_dsp/loongarch/bitdepth_conversion_lsx.h new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/bitdepth_conversion_lsx.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VPX_VPX_DSP_LOONGARCH_BITDEPTH_CONVERSION_LSX_H_ +#define VPX_VPX_DSP_LOONGARCH_BITDEPTH_CONVERSION_LSX_H_ + +#include "./vpx_config.h" +#include "vpx/vpx_integer.h" +#include "vpx_dsp/vpx_dsp_common.h" +#include "vpx_util/loongson_intrinsics.h" + +#if CONFIG_VP9_HIGHBITDEPTH +#define load_tran_low(s) \ + ({ \ + __m128i res0_m; \ + __m128i v0_m = __lsx_vld(s, 0); \ + __m128i v1_m = __lsx_vld(s + 4, 0); \ + res0_m = __lsx_vsrlni_h_w(v0_m, v1_m, 0); \ + res0_m; \ + }) + +#define store_tran_low(v, s, c) \ + { \ + __m128i v0_m, v1_m; \ + v1_m = __lsx_vexth_w_h(v); \ + v0_m = __lsx_vsllwil_w_h(v, 0); \ + __lsx_vst(v0_m, s + c, 0); \ + __lsx_vst(v1_m, s + c + 4, 0); \ + } +#else +#define load_tran_low(s) \ + ({ \ + __m128i res0_m; \ + res0_m = __lsx_vld(s, 0); \ + res0_m; \ + }) + +#define store_tran_low(v, s, c) __lsx_vst(v, s + c, 0) +#endif // CONFIG_VP9_HIGHBITDEPTH + +#endif // VPX_VPX_DSP_LOONGARCH_BITDEPTH_CONVERSION_LSX_H_ diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/fwd_dct32x32_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/fwd_dct32x32_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/fwd_dct32x32_lsx.c @@ -0,0 +1,1176 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "./vpx_dsp_rtcd.h" +#include "vpx_dsp/loongarch/fwd_txfm_lsx.h" +#include "vpx_dsp/fwd_txfm.h" + +#define UNPCK_SH_SW(in, out0, out1) \ + { \ + out0 = __lsx_vsllwil_w_h(in, 0); \ + out1 = __lsx_vexth_w_h(in); \ + } + +static void fdct8x32_1d_column_load_butterfly(const int16_t *input, + int32_t src_stride, + int16_t *temp_buff) { + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i step0, step1, step2, step3; + __m128i in0_1, in1_1, in2_1, in3_1, in4_1, in5_1, in6_1, in7_1; + __m128i step0_1, step1_1, step2_1, step3_1; + + int32_t stride = src_stride << 1; + int32_t stride2 = stride << 1; + int32_t stride3 = stride2 + stride; + const int16_t *input_tmp = (int16_t *)input; + + in0 = __lsx_vld(input_tmp, 0); + DUP2_ARG2(__lsx_vldx, input_tmp, stride, input_tmp, stride2, in1, in2); + in3 = __lsx_vldx(input_tmp, stride3); + + input_tmp += stride2; + in0_1 = __lsx_vld(input_tmp, 0); + DUP2_ARG2(__lsx_vldx, input_tmp, stride, input_tmp, stride2, in1_1, in2_1); + in3_1 = __lsx_vldx(input_tmp, stride3); + + input_tmp = input + (src_stride * 24); + in4_1 = __lsx_vld(input_tmp, 0); + DUP2_ARG2(__lsx_vldx, input_tmp, stride, input_tmp, stride2, in5_1, in6_1); + in7_1 = __lsx_vldx(input_tmp, stride3); + + input_tmp += stride2; + in4 = __lsx_vld(input_tmp, 0); + DUP2_ARG2(__lsx_vldx, input_tmp, stride, input_tmp, stride2, in5, in6); + in7 = __lsx_vldx(input_tmp, stride3); + + DUP4_ARG2(__lsx_vslli_h, in0, 2, in1, 2, in2, 2, in3, 2, in0, in1, in2, in3); + DUP4_ARG2(__lsx_vslli_h, in4, 2, in5, 2, in6, 2, in7, 2, in4, in5, in6, in7); + DUP4_ARG2(__lsx_vslli_h, in0_1, 2, in1_1, 2, in2_1, 2, in3_1, 2, in0_1, in1_1, + in2_1, in3_1); + DUP4_ARG2(__lsx_vslli_h, in4_1, 2, in5_1, 2, in6_1, 2, in7_1, 2, in4_1, in5_1, + in6_1, in7_1); + LSX_BUTTERFLY_8_H(in0, in1, in2, in3, in4, in5, in6, in7, step0, step1, step2, + step3, in4, in5, in6, in7); + LSX_BUTTERFLY_8_H(in0_1, in1_1, in2_1, in3_1, in4_1, in5_1, in6_1, in7_1, + step0_1, step1_1, step2_1, step3_1, in4_1, in5_1, in6_1, + in7_1); + + __lsx_vst(step0, temp_buff, 0); + __lsx_vst(step1, temp_buff, 16); + __lsx_vst(step2, temp_buff, 32); + __lsx_vst(step3, temp_buff, 48); + + __lsx_vst(in4, temp_buff, 448); + __lsx_vst(in5, temp_buff, 464); + __lsx_vst(in6, temp_buff, 480); + __lsx_vst(in7, temp_buff, 496); + + __lsx_vst(step0_1, temp_buff, 64); + __lsx_vst(step1_1, temp_buff, 80); + __lsx_vst(step2_1, temp_buff, 96); + __lsx_vst(step3_1, temp_buff, 112); + + __lsx_vst(in4_1, temp_buff, 384); + __lsx_vst(in5_1, temp_buff, 400); + __lsx_vst(in6_1, temp_buff, 416); + __lsx_vst(in7_1, temp_buff, 432); + + /* 3rd and 4th set */ + input_tmp = input + (src_stride * 8); + in0 = __lsx_vld(input_tmp, 0); + DUP2_ARG2(__lsx_vldx, input_tmp, stride, input_tmp, stride2, in1, in2); + in3 = __lsx_vldx(input_tmp, stride3); + + input_tmp += stride2; + in0_1 = __lsx_vld(input_tmp, 0); + DUP2_ARG2(__lsx_vldx, input_tmp, stride, input_tmp, stride2, in1_1, in2_1); + in3_1 = __lsx_vldx(input_tmp, stride3); + + input_tmp += stride2; + in4_1 = __lsx_vld(input_tmp, 0); + DUP2_ARG2(__lsx_vldx, input_tmp, stride, input_tmp, stride2, in5_1, in6_1); + in7_1 = __lsx_vldx(input_tmp, stride3); + + input_tmp += stride2; + in4 = __lsx_vld(input_tmp, 0); + DUP2_ARG2(__lsx_vldx, input_tmp, stride, input_tmp, stride2, in5, in6); + in7 = __lsx_vldx(input_tmp, stride3); + DUP4_ARG2(__lsx_vslli_h, in0, 2, in1, 2, in2, 2, in3, 2, in0, in1, in2, in3); + DUP4_ARG2(__lsx_vslli_h, in4, 2, in5, 2, in6, 2, in7, 2, in4, in5, in6, in7); + DUP4_ARG2(__lsx_vslli_h, in0_1, 2, in1_1, 2, in2_1, 2, in3_1, 2, in0_1, in1_1, + in2_1, in3_1); + DUP4_ARG2(__lsx_vslli_h, in4_1, 2, in5_1, 2, in6_1, 2, in7_1, 2, in4_1, in5_1, + in6_1, in7_1); + + LSX_BUTTERFLY_8_H(in0, in1, in2, in3, in4, in5, in6, in7, step0, step1, step2, + step3, in4, in5, in6, in7); + LSX_BUTTERFLY_8_H(in0_1, in1_1, in2_1, in3_1, in4_1, in5_1, in6_1, in7_1, + step0_1, step1_1, step2_1, step3_1, in4_1, in5_1, in6_1, + in7_1); + + __lsx_vst(step0, temp_buff, 128); + __lsx_vst(step1, temp_buff, 144); + __lsx_vst(step2, temp_buff, 160); + __lsx_vst(step3, temp_buff, 176); + + __lsx_vst(in4, temp_buff, 320); + __lsx_vst(in5, temp_buff, 336); + __lsx_vst(in6, temp_buff, 352); + __lsx_vst(in7, temp_buff, 368); + + __lsx_vst(step0_1, temp_buff, 192); + __lsx_vst(step1_1, temp_buff, 208); + __lsx_vst(step2_1, temp_buff, 224); + __lsx_vst(step3_1, temp_buff, 240); + + __lsx_vst(in4_1, temp_buff, 256); + __lsx_vst(in5_1, temp_buff, 272); + __lsx_vst(in6_1, temp_buff, 288); + __lsx_vst(in7_1, temp_buff, 304); +} + +static void fdct8x32_1d_column_even_store(int16_t *input, int16_t *temp) { + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i in8, in9, in10, in11, in12, in13, in14, in15; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i temp0, temp1; + + /* fdct even */ + DUP4_ARG2(__lsx_vld, input, 0, input, 16, input, 32, input, 48, in0, in1, in2, + in3); + DUP4_ARG2(__lsx_vld, input, 192, input, 208, input, 224, input, 240, in12, + in13, in14, in15); + LSX_BUTTERFLY_8_H(in0, in1, in2, in3, in12, in13, in14, in15, vec0, vec1, + vec2, vec3, in12, in13, in14, in15); + DUP4_ARG2(__lsx_vld, input, 64, input, 80, input, 96, input, 112, in4, in5, + in6, in7); + DUP4_ARG2(__lsx_vld, input, 128, input, 144, input, 160, input, 176, in8, in9, + in10, in11); + LSX_BUTTERFLY_8_H(in4, in5, in6, in7, in8, in9, in10, in11, vec4, vec5, vec6, + vec7, in8, in9, in10, in11); + + /* Stage 3 */ + DUP4_ARG2(__lsx_vadd_h, vec0, vec7, vec1, vec6, vec2, vec5, vec3, vec4, in0, + in1, in2, in3); + LSX_BUTTERFLY_4_H(in0, in1, in2, in3, temp0, in4, in1, in0); + DOTP_CONST_PAIR(temp0, in4, cospi_16_64, cospi_16_64, temp1, temp0); + FDCT32_POSTPROC_2V_POS_H(temp0, temp1); + __lsx_vst(temp0, temp, 0); + __lsx_vst(temp1, temp, 1024); + + DOTP_CONST_PAIR(in0, in1, cospi_24_64, cospi_8_64, temp1, temp0); + FDCT32_POSTPROC_2V_POS_H(temp0, temp1); + __lsx_vst(temp0, temp, 512); + __lsx_vst(temp1, temp, 1536); + + DUP4_ARG2(__lsx_vsub_h, vec0, vec7, vec1, vec6, vec2, vec5, vec3, vec4, vec7, + vec6, vec5, vec4); + DOTP_CONST_PAIR(vec6, vec5, cospi_16_64, cospi_16_64, vec5, vec6); + DUP2_ARG2(__lsx_vadd_h, vec4, vec5, vec7, vec6, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_28_64, cospi_4_64, temp1, temp0); + FDCT32_POSTPROC_2V_POS_H(temp0, temp1); + __lsx_vst(temp0, temp, 256); + __lsx_vst(temp1, temp, 1792); + + DUP2_ARG2(__lsx_vsub_h, vec4, vec5, vec7, vec6, vec4, vec7); + DOTP_CONST_PAIR(vec7, vec4, cospi_12_64, cospi_20_64, temp1, temp0); + FDCT32_POSTPROC_2V_POS_H(temp0, temp1); + __lsx_vst(temp0, temp, 1280); + __lsx_vst(temp1, temp, 768); + + DOTP_CONST_PAIR(in13, in10, cospi_16_64, cospi_16_64, vec2, vec5); + DOTP_CONST_PAIR(in12, in11, cospi_16_64, cospi_16_64, vec3, vec4); + DUP4_ARG2(__lsx_vadd_h, in8, vec3, in9, vec2, in14, vec5, in15, vec4, in0, + vec1, vec6, in2); + DOTP_CONST_PAIR(vec6, vec1, cospi_24_64, cospi_8_64, in1, in3); + DUP2_ARG2(__lsx_vadd_h, in0, in1, in2, in3, vec0, vec7); + DOTP_CONST_PAIR(vec7, vec0, cospi_30_64, cospi_2_64, temp1, temp0); + FDCT32_POSTPROC_2V_POS_H(temp0, temp1); + __lsx_vst(temp0, temp, 128); + __lsx_vst(temp1, temp, 1920); + + DUP2_ARG2(__lsx_vsub_h, in0, in1, in2, in3, in0, in2); + DOTP_CONST_PAIR(in2, in0, cospi_14_64, cospi_18_64, temp1, temp0); + FDCT32_POSTPROC_2V_POS_H(temp0, temp1); + __lsx_vst(temp0, temp, 1152); + __lsx_vst(temp1, temp, 896); + + DUP2_ARG2(__lsx_vsub_h, in9, vec2, in14, vec5, vec2, vec5); + temp0 = __lsx_vneg_h(vec2); + DOTP_CONST_PAIR(temp0, vec5, cospi_24_64, cospi_8_64, in2, in1); + DUP4_ARG2(__lsx_vsub_h, in8, vec3, in15, vec4, in3, in2, in0, in1, in3, in0, + vec2, vec5); + DOTP_CONST_PAIR(vec5, vec2, cospi_22_64, cospi_10_64, temp1, temp0); + FDCT32_POSTPROC_2V_POS_H(temp0, temp1); + __lsx_vst(temp0, temp, 640); + __lsx_vst(temp1, temp, 1408); + + DUP2_ARG2(__lsx_vadd_h, in3, in2, in0, in1, vec3, vec4); + DOTP_CONST_PAIR(vec4, vec3, cospi_6_64, cospi_26_64, temp0, temp1); + FDCT32_POSTPROC_2V_POS_H(temp0, temp1); + __lsx_vst(temp0, temp, 384); + __lsx_vst(temp1, temp, 1664); +} + +static void fdct8x32_1d_column_odd_store(int16_t *input, int16_t *temp_ptr) { + __m128i in16, in17, in18, in19, in20, in21, in22, in23; + __m128i in24, in25, in26, in27, in28, in29, in30, in31, vec4, vec5; + __m128i tmp0, tmp1; + + DUP4_ARG2(__lsx_vld, input, 64, input, 80, input, 160, input, 176, in20, in21, + in26, in27); + + DOTP_CONST_PAIR(in27, in20, cospi_16_64, cospi_16_64, in20, in27); + DOTP_CONST_PAIR(in26, in21, cospi_16_64, cospi_16_64, in21, in26); + + DUP4_ARG2(__lsx_vld, input, 32, input, 48, input, 192, input, 208, in18, in19, + in28, in29); + + vec4 = __lsx_vsub_h(in19, in20); + __lsx_vst(vec4, input, 64); + vec4 = __lsx_vsub_h(in18, in21); + __lsx_vst(vec4, input, 80); + vec4 = __lsx_vsub_h(in29, in26); + __lsx_vst(vec4, input, 160); + vec4 = __lsx_vsub_h(in28, in27); + __lsx_vst(vec4, input, 176); + + in21 = __lsx_vadd_h(in18, in21); + in20 = __lsx_vadd_h(in19, in20); + in27 = __lsx_vadd_h(in28, in27); + in26 = __lsx_vadd_h(in29, in26); + + DUP4_ARG2(__lsx_vld, input, 96, input, 112, input, 128, input, 144, in22, + in23, in24, in25); + DOTP_CONST_PAIR(in25, in22, cospi_16_64, cospi_16_64, in22, in25); + DOTP_CONST_PAIR(in24, in23, cospi_16_64, cospi_16_64, in23, in24); + + DUP4_ARG2(__lsx_vld, input, 0, input, 16, input, 224, input, 240, in16, in17, + in30, in31); + + vec4 = __lsx_vsub_h(in17, in22); + __lsx_vst(vec4, input, 32); + vec4 = __lsx_vsub_h(in16, in23); + __lsx_vst(vec4, input, 48); + vec4 = __lsx_vsub_h(in31, in24); + __lsx_vst(vec4, input, 192); + vec4 = __lsx_vsub_h(in30, in25); + __lsx_vst(vec4, input, 208); + + DUP4_ARG2(__lsx_vadd_h, in16, in23, in17, in22, in30, in25, in31, in24, in16, + in17, in30, in31); + DOTP_CONST_PAIR(in26, in21, cospi_24_64, cospi_8_64, in18, in29); + DOTP_CONST_PAIR(in27, in20, cospi_24_64, cospi_8_64, in19, in28); + DUP4_ARG2(__lsx_vadd_h, in16, in19, in17, in18, in30, in29, in31, in28, in27, + in22, in21, in25); + DOTP_CONST_PAIR(in21, in22, cospi_28_64, cospi_4_64, in26, in24); + DUP2_ARG2(__lsx_vadd_h, in27, in26, in25, in24, in23, in20); + DOTP_CONST_PAIR(in20, in23, cospi_31_64, cospi_1_64, vec4, vec5); + FDCT32_POSTPROC_2V_POS_H(vec5, vec4); + __lsx_vst(vec5, temp_ptr, 0); + __lsx_vst(vec4, temp_ptr, 1920); + + DUP2_ARG2(__lsx_vsub_h, in27, in26, in25, in24, in22, in21); + DOTP_CONST_PAIR(in21, in22, cospi_15_64, cospi_17_64, vec5, vec4); + FDCT32_POSTPROC_2V_POS_H(vec5, vec4); + __lsx_vst(vec5, temp_ptr, 896); + __lsx_vst(vec4, temp_ptr, 1024); + + DUP4_ARG2(__lsx_vsub_h, in17, in18, in16, in19, in31, in28, in30, in29, in23, + in26, in24, in20); + tmp0 = __lsx_vneg_h(in23); + DOTP_CONST_PAIR(tmp0, in20, cospi_28_64, cospi_4_64, in27, in25); + DUP2_ARG2(__lsx_vsub_h, in26, in27, in24, in25, in23, in20); + DOTP_CONST_PAIR(in20, in23, cospi_23_64, cospi_9_64, vec4, vec5); + FDCT32_POSTPROC_2V_POS_H(vec5, vec4); + __lsx_vst(vec4, temp_ptr, 1408); + __lsx_vst(vec5, temp_ptr, 512); + + DUP2_ARG2(__lsx_vadd_h, in26, in27, in24, in25, in22, in21); + DOTP_CONST_PAIR(in21, in22, cospi_7_64, cospi_25_64, vec4, vec5); + FDCT32_POSTPROC_2V_POS_H(vec5, vec4); + __lsx_vst(vec4, temp_ptr, 384); + __lsx_vst(vec5, temp_ptr, 1536); + + DUP4_ARG2(__lsx_vld, input, 32, input, 48, input, 64, input, 80, in22, in23, + in20, in21); + DUP4_ARG2(__lsx_vld, input, 160, input, 176, input, 192, input, 208, in26, + in27, in24, in25); + in16 = in20; + in17 = in21; + DUP2_ARG1(__lsx_vneg_h, in16, in17, tmp0, tmp1); + DOTP_CONST_PAIR(tmp0, in27, cospi_24_64, cospi_8_64, in20, in27); + DOTP_CONST_PAIR(tmp1, in26, cospi_24_64, cospi_8_64, in21, in26); + DUP4_ARG2(__lsx_vsub_h, in23, in20, in22, in21, in25, in26, in24, in27, in28, + in17, in18, in31); + DOTP_CONST_PAIR(in18, in17, cospi_12_64, cospi_20_64, in29, in30); + DUP2_ARG2(__lsx_vadd_h, in28, in29, in31, in30, in16, in19); + DOTP_CONST_PAIR(in19, in16, cospi_27_64, cospi_5_64, vec5, vec4); + FDCT32_POSTPROC_2V_POS_H(vec5, vec4); + __lsx_vst(vec5, temp_ptr, 1664); + __lsx_vst(vec4, temp_ptr, 256); + + DUP2_ARG2(__lsx_vsub_h, in28, in29, in31, in30, in17, in18); + DOTP_CONST_PAIR(in18, in17, cospi_11_64, cospi_21_64, vec5, vec4); + FDCT32_POSTPROC_2V_POS_H(vec5, vec4); + __lsx_vst(vec5, temp_ptr, 640); + __lsx_vst(vec4, temp_ptr, 1280); + + DUP4_ARG2(__lsx_vadd_h, in22, in21, in23, in20, in24, in27, in25, in26, in16, + in29, in30, in19); + tmp0 = __lsx_vneg_h(in16); + DOTP_CONST_PAIR(tmp0, in19, cospi_12_64, cospi_20_64, in28, in31); + DUP2_ARG2(__lsx_vsub_h, in29, in28, in30, in31, in16, in19); + DOTP_CONST_PAIR(in19, in16, cospi_19_64, cospi_13_64, vec5, vec4); + FDCT32_POSTPROC_2V_POS_H(vec5, vec4); + __lsx_vst(vec5, temp_ptr, 1152); + __lsx_vst(vec4, temp_ptr, 768); + + DUP2_ARG2(__lsx_vadd_h, in29, in28, in30, in31, in17, in18); + DOTP_CONST_PAIR(in18, in17, cospi_3_64, cospi_29_64, vec5, vec4); + FDCT32_POSTPROC_2V_POS_H(vec5, vec4); + __lsx_vst(vec5, temp_ptr, 128); + __lsx_vst(vec4, temp_ptr, 1792); +} + +static void fdct8x32_1d_column(const int16_t *input, int32_t src_stride, + int16_t *tmp_buf, int16_t *tmp_buf_big) { + fdct8x32_1d_column_load_butterfly(input, src_stride, tmp_buf); + fdct8x32_1d_column_even_store(tmp_buf, tmp_buf_big); + fdct8x32_1d_column_odd_store(tmp_buf + 128, (tmp_buf_big + 32)); +} + +static void fdct8x32_1d_row_load_butterfly(int16_t *temp_buff, + int16_t *output) { + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i in8, in9, in10, in11, in12, in13, in14, in15; + __m128i step0, step1, step2, step3, step4, step5, step6, step7; + + DUP4_ARG2(__lsx_vld, temp_buff, 0, temp_buff, 64, temp_buff, 128, temp_buff, + 192, in0, in1, in2, in3); + DUP4_ARG2(__lsx_vld, temp_buff, 256, temp_buff, 320, temp_buff, 384, + temp_buff, 448, in4, in5, in6, in7); + DUP4_ARG2(__lsx_vld, temp_buff, 48, temp_buff, 112, temp_buff, 176, temp_buff, + 240, in8, in9, in10, in11); + DUP4_ARG2(__lsx_vld, temp_buff, 304, temp_buff, 368, temp_buff, 432, + temp_buff, 496, in12, in13, in14, in15); + LSX_TRANSPOSE8x8_H(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + LSX_TRANSPOSE8x8_H(in8, in9, in10, in11, in12, in13, in14, in15, in8, in9, + in10, in11, in12, in13, in14, in15); + LSX_BUTTERFLY_16_H(in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, in10, + in11, in12, in13, in14, in15, step0, step1, step2, step3, + step4, step5, step6, step7, in8, in9, in10, in11, in12, + in13, in14, in15); + + __lsx_vst(step0, output, 0); + __lsx_vst(step1, output, 16); + __lsx_vst(step2, output, 32); + __lsx_vst(step3, output, 48); + __lsx_vst(step4, output, 64); + __lsx_vst(step5, output, 80); + __lsx_vst(step6, output, 96); + __lsx_vst(step7, output, 112); + + __lsx_vst(in8, output, 384); + __lsx_vst(in9, output, 400); + __lsx_vst(in10, output, 416); + __lsx_vst(in11, output, 432); + __lsx_vst(in12, output, 448); + __lsx_vst(in13, output, 464); + __lsx_vst(in14, output, 480); + __lsx_vst(in15, output, 496); + + /* 2nd set */ + DUP4_ARG2(__lsx_vld, temp_buff, 16, temp_buff, 80, temp_buff, 144, temp_buff, + 208, in0, in1, in2, in3); + DUP4_ARG2(__lsx_vld, temp_buff, 272, temp_buff, 336, temp_buff, 400, + temp_buff, 464, in4, in5, in6, in7); + DUP4_ARG2(__lsx_vld, temp_buff, 32, temp_buff, 96, temp_buff, 160, temp_buff, + 224, in8, in9, in10, in11); + DUP4_ARG2(__lsx_vld, temp_buff, 288, temp_buff, 352, temp_buff, 416, + temp_buff, 480, in12, in13, in14, in15); + LSX_TRANSPOSE8x8_H(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + LSX_TRANSPOSE8x8_H(in8, in9, in10, in11, in12, in13, in14, in15, in8, in9, + in10, in11, in12, in13, in14, in15); + LSX_BUTTERFLY_16_H(in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, in10, + in11, in12, in13, in14, in15, step0, step1, step2, step3, + step4, step5, step6, step7, in8, in9, in10, in11, in12, + in13, in14, in15); + + __lsx_vst(step0, output, 128); + __lsx_vst(step1, output, 144); + __lsx_vst(step2, output, 160); + __lsx_vst(step3, output, 176); + __lsx_vst(step4, output, 192); + __lsx_vst(step5, output, 208); + __lsx_vst(step6, output, 224); + __lsx_vst(step7, output, 240); + + __lsx_vst(in8, output, 256); + __lsx_vst(in9, output, 272); + __lsx_vst(in10, output, 288); + __lsx_vst(in11, output, 304); + __lsx_vst(in12, output, 320); + __lsx_vst(in13, output, 336); + __lsx_vst(in14, output, 352); + __lsx_vst(in15, output, 368); +} + +static void fdct8x32_1d_row_even_4x(int16_t *input, int16_t *interm_ptr, + int16_t *out) { + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i in8, in9, in10, in11, in12, in13, in14, in15; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i vec0_l, vec1_l, vec2_l, vec3_l, vec4_l, vec5_l, vec6_l, vec7_l; + __m128i vec0_r, vec1_r, vec2_r, vec3_r, vec4_r, vec5_r, vec6_r, vec7_r; + __m128i tmp0_w, tmp1_w, tmp2_w, tmp3_w; + + /* fdct32 even */ + /* stage 2 */ + DUP4_ARG2(__lsx_vld, input, 0, input, 16, input, 32, input, 48, in0, in1, in2, + in3); + DUP4_ARG2(__lsx_vld, input, 64, input, 80, input, 96, input, 112, in4, in5, + in6, in7); + DUP4_ARG2(__lsx_vld, input, 128, input, 144, input, 160, input, 176, in8, in9, + in10, in11); + DUP4_ARG2(__lsx_vld, input, 192, input, 208, input, 224, input, 240, in12, + in13, in14, in15); + + LSX_BUTTERFLY_16_H(in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, in10, + in11, in12, in13, in14, in15, vec0, vec1, vec2, vec3, vec4, + vec5, vec6, vec7, in8, in9, in10, in11, in12, in13, in14, + in15); + + __lsx_vst(vec0, interm_ptr, 0); + __lsx_vst(vec1, interm_ptr, 16); + __lsx_vst(vec2, interm_ptr, 32); + __lsx_vst(vec3, interm_ptr, 48); + __lsx_vst(vec4, interm_ptr, 64); + __lsx_vst(vec5, interm_ptr, 80); + __lsx_vst(vec6, interm_ptr, 96); + __lsx_vst(vec7, interm_ptr, 112); + + __lsx_vst(in8, interm_ptr, 128); + __lsx_vst(in9, interm_ptr, 144); + __lsx_vst(in10, interm_ptr, 160); + __lsx_vst(in11, interm_ptr, 176); + __lsx_vst(in12, interm_ptr, 192); + __lsx_vst(in13, interm_ptr, 208); + __lsx_vst(in14, interm_ptr, 224); + __lsx_vst(in15, interm_ptr, 240); + + /* Stage 3 */ + UNPCK_SH_SW(vec0, vec0_l, vec0_r); + UNPCK_SH_SW(vec1, vec1_l, vec1_r); + UNPCK_SH_SW(vec2, vec2_l, vec2_r); + UNPCK_SH_SW(vec3, vec3_l, vec3_r); + UNPCK_SH_SW(vec4, vec4_l, vec4_r); + UNPCK_SH_SW(vec5, vec5_l, vec5_r); + UNPCK_SH_SW(vec6, vec6_l, vec6_r); + UNPCK_SH_SW(vec7, vec7_l, vec7_r); + DUP4_ARG2(__lsx_vadd_w, vec0_r, vec7_r, vec1_r, vec6_r, vec2_r, vec5_r, + vec3_r, vec4_r, tmp0_w, tmp1_w, tmp2_w, tmp3_w); + LSX_BUTTERFLY_4_W(tmp0_w, tmp1_w, tmp2_w, tmp3_w, vec4_r, vec6_r, vec7_r, + vec5_r); + DUP4_ARG2(__lsx_vadd_w, vec0_l, vec7_l, vec1_l, vec6_l, vec2_l, vec5_l, + vec3_l, vec4_l, vec0_r, vec1_r, vec2_r, vec3_r); + + tmp3_w = __lsx_vadd_w(vec0_r, vec3_r); + vec0_r = __lsx_vsub_w(vec0_r, vec3_r); + vec3_r = __lsx_vadd_w(vec1_r, vec2_r); + vec1_r = __lsx_vsub_w(vec1_r, vec2_r); + + DOTP_CONST_PAIR_W(vec4_r, vec6_r, tmp3_w, vec3_r, cospi_16_64, cospi_16_64, + vec4_r, tmp3_w, vec6_r, vec3_r); + FDCT32_POSTPROC_NEG_W(vec4_r); + FDCT32_POSTPROC_NEG_W(tmp3_w); + FDCT32_POSTPROC_NEG_W(vec6_r); + FDCT32_POSTPROC_NEG_W(vec3_r); + DUP2_ARG2(__lsx_vpickev_h, vec4_r, tmp3_w, vec6_r, vec3_r, vec4, vec5); + __lsx_vst(vec5, out, 0); + __lsx_vst(vec4, out, 16); + + DOTP_CONST_PAIR_W(vec5_r, vec7_r, vec0_r, vec1_r, cospi_24_64, cospi_8_64, + vec4_r, tmp3_w, vec6_r, vec3_r); + FDCT32_POSTPROC_NEG_W(vec4_r); + FDCT32_POSTPROC_NEG_W(tmp3_w); + FDCT32_POSTPROC_NEG_W(vec6_r); + FDCT32_POSTPROC_NEG_W(vec3_r); + DUP2_ARG2(__lsx_vpickev_h, vec4_r, tmp3_w, vec6_r, vec3_r, vec4, vec5); + __lsx_vst(vec5, out, 32); + __lsx_vst(vec4, out, 48); + + DUP4_ARG2(__lsx_vld, interm_ptr, 0, interm_ptr, 16, interm_ptr, 32, + interm_ptr, 48, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vld, interm_ptr, 64, interm_ptr, 80, interm_ptr, 96, + interm_ptr, 112, vec4, vec5, vec6, vec7); + DUP4_ARG2(__lsx_vsub_h, vec3, vec4, vec2, vec5, vec1, vec6, vec0, vec7, vec4, + vec5, vec6, vec7); + DOTP_CONST_PAIR(vec6, vec5, cospi_16_64, cospi_16_64, vec5, vec6); + DUP2_ARG2(__lsx_vadd_h, vec4, vec5, vec7, vec6, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_28_64, cospi_4_64, in5, in4); + FDCT_POSTPROC_2V_NEG_H(in4, in5); + __lsx_vst(in4, out, 64); + __lsx_vst(in5, out, 112); + + DUP2_ARG2(__lsx_vsub_h, vec4, vec5, vec7, vec6, vec4, vec7); + DOTP_CONST_PAIR(vec7, vec4, cospi_12_64, cospi_20_64, in5, in4); + FDCT_POSTPROC_2V_NEG_H(in4, in5); + __lsx_vst(in4, out, 80); + __lsx_vst(in5, out, 96); + + DUP4_ARG2(__lsx_vld, interm_ptr, 128, interm_ptr, 144, interm_ptr, 160, + interm_ptr, 176, in8, in9, in10, in11); + DUP4_ARG2(__lsx_vld, interm_ptr, 192, interm_ptr, 208, interm_ptr, 224, + interm_ptr, 240, in12, in13, in14, in15); + DOTP_CONST_PAIR(in13, in10, cospi_16_64, cospi_16_64, vec2, vec5); + DOTP_CONST_PAIR(in12, in11, cospi_16_64, cospi_16_64, vec3, vec4); + DUP4_ARG2(__lsx_vadd_h, in8, vec3, in9, vec2, in14, vec5, in15, vec4, in0, + vec1, vec6, in2); + DOTP_CONST_PAIR(vec6, vec1, cospi_24_64, cospi_8_64, in1, in3); + DUP2_ARG2(__lsx_vadd_h, in0, in1, in2, in3, vec0, vec7); + DOTP_CONST_PAIR(vec7, vec0, cospi_30_64, cospi_2_64, in5, in4); + FDCT_POSTPROC_2V_NEG_H(in4, in5); + __lsx_vst(in4, out, 128); + __lsx_vst(in5, out, 240); + + DUP2_ARG2(__lsx_vsub_h, in0, in1, in2, in3, in0, in2); + DOTP_CONST_PAIR(in2, in0, cospi_14_64, cospi_18_64, in5, in4); + FDCT_POSTPROC_2V_NEG_H(in4, in5); + __lsx_vst(in4, out, 144); + __lsx_vst(in5, out, 224); + + DUP2_ARG2(__lsx_vsub_h, in9, vec2, in14, vec5, vec2, vec5); + tmp0_w = __lsx_vneg_h(vec2); + DOTP_CONST_PAIR(tmp0_w, vec5, cospi_24_64, cospi_8_64, in2, in1); + DUP4_ARG2(__lsx_vsub_h, in8, vec3, in15, vec4, in3, in2, in0, in1, in3, in0, + vec2, vec5); + DOTP_CONST_PAIR(vec5, vec2, cospi_22_64, cospi_10_64, in5, in4); + FDCT_POSTPROC_2V_NEG_H(in4, in5); + __lsx_vst(in4, out, 160); + __lsx_vst(in5, out, 208); + + DUP2_ARG2(__lsx_vadd_h, in3, in2, in0, in1, vec3, vec4); + DOTP_CONST_PAIR(vec4, vec3, cospi_6_64, cospi_26_64, in4, in5); + FDCT_POSTPROC_2V_NEG_H(in4, in5); + __lsx_vst(in4, out, 192); + __lsx_vst(in5, out, 176); +} + +static void fdct8x32_1d_row_even(int16_t *temp, int16_t *out) { + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i in8, in9, in10, in11, in12, in13, in14, in15; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, temp0, temp1; + + /* fdct32 even */ + /* stage 2 */ + DUP4_ARG2(__lsx_vld, temp, 0, temp, 16, temp, 32, temp, 48, in0, in1, in2, + in3); + DUP4_ARG2(__lsx_vld, temp, 64, temp, 80, temp, 96, temp, 112, in4, in5, in6, + in7); + DUP4_ARG2(__lsx_vld, temp, 128, temp, 144, temp, 160, temp, 176, in8, in9, + in10, in11); + DUP4_ARG2(__lsx_vld, temp, 192, temp, 208, temp, 224, temp, 240, in12, in13, + in14, in15); + + LSX_BUTTERFLY_16_H(in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, in10, + in11, in12, in13, in14, in15, vec0, vec1, vec2, vec3, vec4, + vec5, vec6, vec7, in8, in9, in10, in11, in12, in13, in14, + in15); + /* Stage 3 */ + DUP4_ARG2(__lsx_vadd_h, vec0, vec7, vec1, vec6, vec2, vec5, vec3, vec4, in0, + in1, in2, in3); + LSX_BUTTERFLY_4_H(in0, in1, in2, in3, temp0, in4, in1, in0); + DOTP_CONST_PAIR(temp0, in4, cospi_16_64, cospi_16_64, temp1, temp0); + FDCT_POSTPROC_2V_NEG_H(temp0, temp1); + __lsx_vst(temp0, out, 0); + __lsx_vst(temp1, out, 16); + + DOTP_CONST_PAIR(in0, in1, cospi_24_64, cospi_8_64, temp1, temp0); + FDCT_POSTPROC_2V_NEG_H(temp0, temp1); + __lsx_vst(temp0, out, 32); + __lsx_vst(temp1, out, 48); + + DUP4_ARG2(__lsx_vsub_h, vec3, vec4, vec2, vec5, vec1, vec6, vec0, vec7, vec4, + vec5, vec6, vec7); + DOTP_CONST_PAIR(vec6, vec5, cospi_16_64, cospi_16_64, vec5, vec6); + DUP2_ARG2(__lsx_vadd_h, vec4, vec5, vec7, vec6, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_28_64, cospi_4_64, temp1, temp0); + FDCT_POSTPROC_2V_NEG_H(temp0, temp1); + __lsx_vst(temp0, out, 64); + __lsx_vst(temp1, out, 112); + + DUP2_ARG2(__lsx_vsub_h, vec4, vec5, vec7, vec6, vec4, vec7); + DOTP_CONST_PAIR(vec7, vec4, cospi_12_64, cospi_20_64, temp1, temp0); + FDCT_POSTPROC_2V_NEG_H(temp0, temp1); + __lsx_vst(temp0, out, 80); + __lsx_vst(temp1, out, 96); + + DOTP_CONST_PAIR(in13, in10, cospi_16_64, cospi_16_64, vec2, vec5); + DOTP_CONST_PAIR(in12, in11, cospi_16_64, cospi_16_64, vec3, vec4); + DUP4_ARG2(__lsx_vadd_h, in8, vec3, in9, vec2, in14, vec5, in15, vec4, in0, + vec1, vec6, in2); + DOTP_CONST_PAIR(vec6, vec1, cospi_24_64, cospi_8_64, in1, in3); + DUP2_ARG2(__lsx_vadd_h, in0, in1, in2, in3, vec0, vec7); + DOTP_CONST_PAIR(vec7, vec0, cospi_30_64, cospi_2_64, temp1, temp0); + FDCT_POSTPROC_2V_NEG_H(temp0, temp1); + __lsx_vst(temp0, out, 128); + __lsx_vst(temp1, out, 240); + + DUP2_ARG2(__lsx_vsub_h, in0, in1, in2, in3, in0, in2); + DOTP_CONST_PAIR(in2, in0, cospi_14_64, cospi_18_64, temp1, temp0); + FDCT_POSTPROC_2V_NEG_H(temp0, temp1); + __lsx_vst(temp0, out, 144); + __lsx_vst(temp1, out, 224); + + DUP2_ARG2(__lsx_vsub_h, in9, vec2, in14, vec5, vec2, vec5); + temp0 = __lsx_vneg_h(vec2); + DOTP_CONST_PAIR(temp0, vec5, cospi_24_64, cospi_8_64, in2, in1); + DUP4_ARG2(__lsx_vsub_h, in8, vec3, in15, vec4, in3, in2, in0, in1, in3, in0, + vec2, vec5) + DOTP_CONST_PAIR(vec5, vec2, cospi_22_64, cospi_10_64, temp1, temp0); + FDCT_POSTPROC_2V_NEG_H(temp0, temp1); + __lsx_vst(temp0, out, 160); + __lsx_vst(temp1, out, 208); + + DUP2_ARG2(__lsx_vadd_h, in3, in2, in0, in1, vec3, vec4); + DOTP_CONST_PAIR(vec4, vec3, cospi_6_64, cospi_26_64, temp0, temp1); + FDCT_POSTPROC_2V_NEG_H(temp0, temp1); + __lsx_vst(temp0, out, 192); + __lsx_vst(temp1, out, 176); +} + +static void fdct8x32_1d_row_odd(int16_t *temp, int16_t *interm_ptr, + int16_t *out) { + __m128i in16, in17, in18, in19, in20, in21, in22, in23; + __m128i in24, in25, in26, in27, in28, in29, in30, in31, vec4, vec5; + __m128i tmp0, tmp1; + + in20 = __lsx_vld(temp, 64); + in21 = __lsx_vld(temp, 80); + in26 = __lsx_vld(temp, 160); + in27 = __lsx_vld(temp, 176); + + DOTP_CONST_PAIR(in27, in20, cospi_16_64, cospi_16_64, in20, in27); + DOTP_CONST_PAIR(in26, in21, cospi_16_64, cospi_16_64, in21, in26); + + in18 = __lsx_vld(temp, 32); + in19 = __lsx_vld(temp, 48); + in28 = __lsx_vld(temp, 192); + in29 = __lsx_vld(temp, 208); + + vec4 = __lsx_vsub_h(in19, in20); + __lsx_vst(vec4, interm_ptr, 64); + vec4 = __lsx_vsub_h(in18, in21); + __lsx_vst(vec4, interm_ptr, 176); + vec4 = __lsx_vsub_h(in28, in27); + __lsx_vst(vec4, interm_ptr, 112); + vec4 = __lsx_vsub_h(in29, in26); + __lsx_vst(vec4, interm_ptr, 128); + + DUP4_ARG2(__lsx_vadd_h, in18, in21, in19, in20, in28, in27, in29, in26, in21, + in20, in27, in26); + + in22 = __lsx_vld(temp, 96); + in23 = __lsx_vld(temp, 112); + in24 = __lsx_vld(temp, 128); + in25 = __lsx_vld(temp, 144); + + DOTP_CONST_PAIR(in25, in22, cospi_16_64, cospi_16_64, in22, in25); + DOTP_CONST_PAIR(in24, in23, cospi_16_64, cospi_16_64, in23, in24); + + in16 = __lsx_vld(temp, 0); + in17 = __lsx_vld(temp, 16); + in30 = __lsx_vld(temp, 224); + in31 = __lsx_vld(temp, 240); + + vec4 = __lsx_vsub_h(in17, in22); + __lsx_vst(vec4, interm_ptr, 80); + vec4 = __lsx_vsub_h(in30, in25); + __lsx_vst(vec4, interm_ptr, 96); + vec4 = __lsx_vsub_h(in31, in24); + __lsx_vst(vec4, interm_ptr, 144); + vec4 = __lsx_vsub_h(in16, in23); + __lsx_vst(vec4, interm_ptr, 160); + + DUP4_ARG2(__lsx_vadd_h, in16, in23, in17, in22, in30, in25, in31, in24, in16, + in17, in30, in31); + DOTP_CONST_PAIR(in26, in21, cospi_24_64, cospi_8_64, in18, in29); + DOTP_CONST_PAIR(in27, in20, cospi_24_64, cospi_8_64, in19, in28); + + DUP4_ARG2(__lsx_vadd_h, in16, in19, in17, in18, in30, in29, in31, in28, in27, + in22, in21, in25); + DOTP_CONST_PAIR(in21, in22, cospi_28_64, cospi_4_64, in26, in24); + DUP2_ARG2(__lsx_vadd_h, in27, in26, in25, in24, in23, in20); + + DOTP_CONST_PAIR(in20, in23, cospi_31_64, cospi_1_64, vec4, vec5); + FDCT_POSTPROC_2V_NEG_H(vec5, vec4); + __lsx_vst(vec5, out, 0); + __lsx_vst(vec4, out, 240); + + DUP2_ARG2(__lsx_vsub_h, in27, in26, in25, in24, in22, in21); + + DOTP_CONST_PAIR(in21, in22, cospi_15_64, cospi_17_64, vec5, vec4); + FDCT_POSTPROC_2V_NEG_H(vec5, vec4); + __lsx_vst(vec5, out, 224); + __lsx_vst(vec4, out, 16); + + DUP4_ARG2(__lsx_vsub_h, in17, in18, in16, in19, in31, in28, in30, in29, in23, + in26, in24, in20); + tmp0 = __lsx_vneg_h(in23); + DOTP_CONST_PAIR(tmp0, in20, cospi_28_64, cospi_4_64, in27, in25); + DUP2_ARG2(__lsx_vsub_h, in26, in27, in24, in25, in23, in20); + + DOTP_CONST_PAIR(in20, in23, cospi_23_64, cospi_9_64, vec4, vec5); + FDCT_POSTPROC_2V_NEG_H(vec5, vec4); + __lsx_vst(vec4, out, 32); + __lsx_vst(vec5, out, 208); + + DUP2_ARG2(__lsx_vadd_h, in26, in27, in24, in25, in22, in21); + DOTP_CONST_PAIR(in21, in22, cospi_7_64, cospi_25_64, vec4, vec5); + FDCT_POSTPROC_2V_NEG_H(vec5, vec4); + __lsx_vst(vec4, out, 48); + __lsx_vst(vec5, out, 192); + + in20 = __lsx_vld(interm_ptr, 64); + in21 = __lsx_vld(interm_ptr, 176); + in27 = __lsx_vld(interm_ptr, 112); + in26 = __lsx_vld(interm_ptr, 128); + + in16 = in20; + in17 = in21; + DUP2_ARG1(__lsx_vneg_h, in16, in17, tmp0, tmp1); + DOTP_CONST_PAIR(tmp0, in27, cospi_24_64, cospi_8_64, in20, in27); + DOTP_CONST_PAIR(tmp1, in26, cospi_24_64, cospi_8_64, in21, in26); + + in22 = __lsx_vld(interm_ptr, 80); + in25 = __lsx_vld(interm_ptr, 96); + in24 = __lsx_vld(interm_ptr, 144); + in23 = __lsx_vld(interm_ptr, 160); + + DUP4_ARG2(__lsx_vsub_h, in23, in20, in22, in21, in25, in26, in24, in27, in28, + in17, in18, in31); + DOTP_CONST_PAIR(in18, in17, cospi_12_64, cospi_20_64, in29, in30); + DUP2_ARG2(__lsx_vadd_h, in28, in29, in31, in30, in16, in19); + DOTP_CONST_PAIR(in19, in16, cospi_27_64, cospi_5_64, vec5, vec4); + FDCT_POSTPROC_2V_NEG_H(vec5, vec4); + __lsx_vst(vec5, out, 64); + __lsx_vst(vec4, out, 176); + + DUP2_ARG2(__lsx_vsub_h, in28, in29, in31, in30, in17, in18); + DOTP_CONST_PAIR(in18, in17, cospi_11_64, cospi_21_64, vec5, vec4); + FDCT_POSTPROC_2V_NEG_H(vec5, vec4); + __lsx_vst(vec5, out, 80); + __lsx_vst(vec4, out, 160); + + DUP4_ARG2(__lsx_vadd_h, in22, in21, in23, in20, in24, in27, in25, in26, in16, + in29, in30, in19); + tmp0 = __lsx_vneg_h(in16); + DOTP_CONST_PAIR(tmp0, in19, cospi_12_64, cospi_20_64, in28, in31); + DUP2_ARG2(__lsx_vsub_h, in29, in28, in30, in31, in16, in19); + + DOTP_CONST_PAIR(in19, in16, cospi_19_64, cospi_13_64, vec5, vec4); + FDCT_POSTPROC_2V_NEG_H(vec5, vec4); + __lsx_vst(vec5, out, 144); + __lsx_vst(vec4, out, 96); + + DUP2_ARG2(__lsx_vadd_h, in29, in28, in30, in31, in17, in18); + + DOTP_CONST_PAIR(in18, in17, cospi_3_64, cospi_29_64, vec5, vec4); + FDCT_POSTPROC_2V_NEG_H(vec5, vec4); + __lsx_vst(vec4, out, 112); + __lsx_vst(vec5, out, 128); +} + +static void fdct8x32_1d_row_transpose_store(int16_t *temp, int16_t *output) { + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i in0_1, in1_1, in2_1, in3_1, in4_1, in5_1, in6_1, in7_1; + + /* 1st set */ + in0 = __lsx_vld(temp, 0); + in4 = __lsx_vld(temp, 64); + in2 = __lsx_vld(temp, 128); + in6 = __lsx_vld(temp, 192); + in1 = __lsx_vld(temp, 256); + in7 = __lsx_vld(temp, 304); + in3 = __lsx_vld(temp, 384); + in5 = __lsx_vld(temp, 432); + + LSX_TRANSPOSE8x8_H(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + + /* 2nd set */ + in0_1 = __lsx_vld(temp, 32); + in1_1 = __lsx_vld(temp, 464); + in2_1 = __lsx_vld(temp, 160); + in3_1 = __lsx_vld(temp, 336); + in4_1 = __lsx_vld(temp, 96); + in5_1 = __lsx_vld(temp, 352); + in6_1 = __lsx_vld(temp, 224); + in7_1 = __lsx_vld(temp, 480); + + __lsx_vst(in0, output, 0); + __lsx_vst(in1, output, 64); + __lsx_vst(in2, output, 128); + __lsx_vst(in3, output, 192); + __lsx_vst(in4, output, 256); + __lsx_vst(in5, output, 320); + __lsx_vst(in6, output, 384); + __lsx_vst(in7, output, 448); + + LSX_TRANSPOSE8x8_H(in0_1, in1_1, in2_1, in3_1, in4_1, in5_1, in6_1, in7_1, + in0_1, in1_1, in2_1, in3_1, in4_1, in5_1, in6_1, in7_1); + + /* 3rd set */ + in0 = __lsx_vld(temp, 16); + in1 = __lsx_vld(temp, 272); + in2 = __lsx_vld(temp, 144); + in3 = __lsx_vld(temp, 400); + in4 = __lsx_vld(temp, 80); + in5 = __lsx_vld(temp, 416); + in6 = __lsx_vld(temp, 208); + in7 = __lsx_vld(temp, 288); + + __lsx_vst(in0_1, output, 16); + __lsx_vst(in1_1, output, 80); + __lsx_vst(in2_1, output, 144); + __lsx_vst(in3_1, output, 208); + __lsx_vst(in4_1, output, 272); + __lsx_vst(in5_1, output, 336); + __lsx_vst(in6_1, output, 400); + __lsx_vst(in7_1, output, 464); + + LSX_TRANSPOSE8x8_H(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + + __lsx_vst(in0, output, 32); + __lsx_vst(in1, output, 96); + __lsx_vst(in2, output, 160); + __lsx_vst(in3, output, 224); + __lsx_vst(in4, output, 288); + __lsx_vst(in5, output, 352); + __lsx_vst(in6, output, 416); + __lsx_vst(in7, output, 480); + + /* 4th set */ + in0_1 = __lsx_vld(temp, 48); + in1_1 = __lsx_vld(temp, 448); + in2_1 = __lsx_vld(temp, 176); + in3_1 = __lsx_vld(temp, 320); + in4_1 = __lsx_vld(temp, 112); + in5_1 = __lsx_vld(temp, 368); + in6_1 = __lsx_vld(temp, 240); + in7_1 = __lsx_vld(temp, 496); + + LSX_TRANSPOSE8x8_H(in0_1, in1_1, in2_1, in3_1, in4_1, in5_1, in6_1, in7_1, + in0_1, in1_1, in2_1, in3_1, in4_1, in5_1, in6_1, in7_1); + + __lsx_vst(in0_1, output, 48); + __lsx_vst(in1_1, output, 112); + __lsx_vst(in2_1, output, 176); + __lsx_vst(in3_1, output, 240); + __lsx_vst(in4_1, output, 304); + __lsx_vst(in5_1, output, 368); + __lsx_vst(in6_1, output, 432); + __lsx_vst(in7_1, output, 496); +} + +static void fdct32x8_1d_row(int16_t *temp, int16_t *temp_buf, int16_t *output) { + fdct8x32_1d_row_load_butterfly(temp, temp_buf); + fdct8x32_1d_row_even(temp_buf, temp_buf); + fdct8x32_1d_row_odd(temp_buf + 128, temp, temp_buf + 128); + fdct8x32_1d_row_transpose_store(temp_buf, output); +} + +static void fdct32x8_1d_row_4x(int16_t *tmp_buf_big, int16_t *tmp_buf, + int16_t *output) { + fdct8x32_1d_row_load_butterfly(tmp_buf_big, tmp_buf); + fdct8x32_1d_row_even_4x(tmp_buf, tmp_buf_big, tmp_buf); + fdct8x32_1d_row_odd(tmp_buf + 128, tmp_buf_big, tmp_buf + 128); + fdct8x32_1d_row_transpose_store(tmp_buf, output); +} + +void vpx_fdct32x32_lsx(const int16_t *input, int16_t *output, + int32_t src_stride) { + int i; + DECLARE_ALIGNED(32, int16_t, tmp_buf_big[1024]); + DECLARE_ALIGNED(32, int16_t, tmp_buf[256]); + + /* column transform */ + for (i = 0; i < 4; ++i) { + fdct8x32_1d_column(input + (8 * i), src_stride, tmp_buf, + tmp_buf_big + (8 * i)); + } + + /* row transform */ + fdct32x8_1d_row_4x(tmp_buf_big, tmp_buf, output); + + /* row transform */ + for (i = 1; i < 4; ++i) { + fdct32x8_1d_row(tmp_buf_big + (i * 256), tmp_buf, output + (i * 256)); + } +} + +static void fdct8x32_1d_row_even_rd(int16_t *temp, int16_t *out) { + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i in8, in9, in10, in11, in12, in13, in14, in15; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, temp0, temp1; + + /* fdct32 even */ + /* stage 2 */ + DUP4_ARG2(__lsx_vld, temp, 0, temp, 16, temp, 32, temp, 48, in0, in1, in2, + in3); + DUP4_ARG2(__lsx_vld, temp, 64, temp, 80, temp, 96, temp, 112, in4, in5, in6, + in7); + DUP4_ARG2(__lsx_vld, temp, 128, temp, 144, temp, 160, temp, 176, in8, in9, + in10, in11); + DUP4_ARG2(__lsx_vld, temp, 192, temp, 208, temp, 224, temp, 240, in12, in13, + in14, in15); + LSX_BUTTERFLY_16_H(in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, in10, + in11, in12, in13, in14, in15, vec0, vec1, vec2, vec3, vec4, + vec5, vec6, vec7, in8, in9, in10, in11, in12, in13, in14, + in15); + + FDCT_POSTPROC_2V_NEG_H(vec0, vec1); + FDCT_POSTPROC_2V_NEG_H(vec2, vec3); + FDCT_POSTPROC_2V_NEG_H(vec4, vec5); + FDCT_POSTPROC_2V_NEG_H(vec6, vec7); + FDCT_POSTPROC_2V_NEG_H(in8, in9); + FDCT_POSTPROC_2V_NEG_H(in10, in11); + FDCT_POSTPROC_2V_NEG_H(in12, in13); + FDCT_POSTPROC_2V_NEG_H(in14, in15); + + /* Stage 3 */ + DUP4_ARG2(__lsx_vadd_h, vec0, vec7, vec1, vec6, vec2, vec5, vec3, vec4, in0, + in1, in2, in3); + + temp0 = __lsx_vadd_h(in0, in3); + in0 = __lsx_vsub_h(in0, in3); + in3 = __lsx_vadd_h(in1, in2); + in1 = __lsx_vsub_h(in1, in2); + + DOTP_CONST_PAIR(temp0, in3, cospi_16_64, cospi_16_64, temp1, temp0); + __lsx_vst(temp0, out, 0); + __lsx_vst(temp1, out, 16); + + DOTP_CONST_PAIR(in0, in1, cospi_24_64, cospi_8_64, temp1, temp0); + __lsx_vst(temp0, out, 32); + __lsx_vst(temp1, out, 48); + + DUP4_ARG2(__lsx_vsub_h, vec3, vec4, vec2, vec5, vec1, vec6, vec0, vec7, vec4, + vec5, vec6, vec7); + DOTP_CONST_PAIR(vec6, vec5, cospi_16_64, cospi_16_64, vec5, vec6); + DUP2_ARG2(__lsx_vadd_h, vec4, vec5, vec7, vec6, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_28_64, cospi_4_64, temp1, temp0); + __lsx_vst(temp0, out, 64); + __lsx_vst(temp1, out, 112); + + DUP2_ARG2(__lsx_vsub_h, vec4, vec5, vec7, vec6, vec4, vec7); + DOTP_CONST_PAIR(vec7, vec4, cospi_12_64, cospi_20_64, temp1, temp0); + __lsx_vst(temp0, out, 80); + __lsx_vst(temp1, out, 96); + + DOTP_CONST_PAIR(in13, in10, cospi_16_64, cospi_16_64, vec2, vec5); + DOTP_CONST_PAIR(in12, in11, cospi_16_64, cospi_16_64, vec3, vec4); + DUP4_ARG2(__lsx_vadd_h, in8, vec3, in9, vec2, in14, vec5, in15, vec4, in0, + vec1, vec6, in2); + DOTP_CONST_PAIR(vec6, vec1, cospi_24_64, cospi_8_64, in1, in3); + DUP2_ARG2(__lsx_vadd_h, in0, in1, in2, in3, vec0, vec7); + DOTP_CONST_PAIR(vec7, vec0, cospi_30_64, cospi_2_64, temp1, temp0); + __lsx_vst(temp0, out, 128); + __lsx_vst(temp1, out, 240); + + DUP2_ARG2(__lsx_vsub_h, in0, in1, in2, in3, in0, in2); + DOTP_CONST_PAIR(in2, in0, cospi_14_64, cospi_18_64, temp1, temp0); + __lsx_vst(temp0, out, 144); + __lsx_vst(temp1, out, 224); + + DUP2_ARG2(__lsx_vsub_h, in9, vec2, in14, vec5, vec2, vec5); + temp0 = __lsx_vneg_h(vec2); + DOTP_CONST_PAIR(temp0, vec5, cospi_24_64, cospi_8_64, in2, in1); + DUP4_ARG2(__lsx_vsub_h, in8, vec3, in15, vec4, in3, in2, in0, in1, in3, in0, + vec2, vec5); + DOTP_CONST_PAIR(vec5, vec2, cospi_22_64, cospi_10_64, temp1, temp0); + __lsx_vst(temp0, out, 160); + __lsx_vst(temp1, out, 208); + + DUP2_ARG2(__lsx_vadd_h, in3, in2, in0, in1, vec3, vec4); + DOTP_CONST_PAIR(vec4, vec3, cospi_6_64, cospi_26_64, temp0, temp1); + __lsx_vst(temp0, out, 192); + __lsx_vst(temp1, out, 176); +} + +static void fdct8x32_1d_row_odd_rd(int16_t *temp, int16_t *interm_ptr, + int16_t *out) { + __m128i in16, in17, in18, in19, in20, in21, in22, in23; + __m128i in24, in25, in26, in27, in28, in29, in30, in31; + __m128i vec4, vec5, tmp0, tmp1; + + in20 = __lsx_vld(temp, 64); + in21 = __lsx_vld(temp, 80); + in26 = __lsx_vld(temp, 160); + in27 = __lsx_vld(temp, 176); + + DOTP_CONST_PAIR(in27, in20, cospi_16_64, cospi_16_64, in20, in27); + DOTP_CONST_PAIR(in26, in21, cospi_16_64, cospi_16_64, in21, in26); + + FDCT_POSTPROC_2V_NEG_H(in20, in21); + FDCT_POSTPROC_2V_NEG_H(in26, in27); + + in18 = __lsx_vld(temp, 32); + in19 = __lsx_vld(temp, 48); + in28 = __lsx_vld(temp, 192); + in29 = __lsx_vld(temp, 208); + + FDCT_POSTPROC_2V_NEG_H(in18, in19); + FDCT_POSTPROC_2V_NEG_H(in28, in29); + + vec4 = __lsx_vsub_h(in19, in20); + __lsx_vst(vec4, interm_ptr, 64); + vec4 = __lsx_vsub_h(in18, in21); + __lsx_vst(vec4, interm_ptr, 176); + vec4 = __lsx_vsub_h(in29, in26); + __lsx_vst(vec4, interm_ptr, 128); + vec4 = __lsx_vsub_h(in28, in27); + __lsx_vst(vec4, interm_ptr, 112); + + DUP4_ARG2(__lsx_vadd_h, in18, in21, in19, in20, in28, in27, in29, in26, in21, + in20, in27, in26); + + in22 = __lsx_vld(temp, 96); + in23 = __lsx_vld(temp, 112); + in24 = __lsx_vld(temp, 128); + in25 = __lsx_vld(temp, 144); + + DOTP_CONST_PAIR(in25, in22, cospi_16_64, cospi_16_64, in22, in25); + DOTP_CONST_PAIR(in24, in23, cospi_16_64, cospi_16_64, in23, in24); + FDCT_POSTPROC_2V_NEG_H(in22, in23); + FDCT_POSTPROC_2V_NEG_H(in24, in25); + + in16 = __lsx_vld(temp, 0); + in17 = __lsx_vld(temp, 16); + in30 = __lsx_vld(temp, 224); + in31 = __lsx_vld(temp, 240); + + FDCT_POSTPROC_2V_NEG_H(in16, in17); + FDCT_POSTPROC_2V_NEG_H(in30, in31); + + vec4 = __lsx_vsub_h(in17, in22); + __lsx_vst(vec4, interm_ptr, 80); + vec4 = __lsx_vsub_h(in30, in25); + __lsx_vst(vec4, interm_ptr, 96); + vec4 = __lsx_vsub_h(in31, in24); + __lsx_vst(vec4, interm_ptr, 144); + vec4 = __lsx_vsub_h(in16, in23); + __lsx_vst(vec4, interm_ptr, 160); + + DUP4_ARG2(__lsx_vadd_h, in16, in23, in17, in22, in30, in25, in31, in24, in16, + in17, in30, in31); + DOTP_CONST_PAIR(in26, in21, cospi_24_64, cospi_8_64, in18, in29); + DOTP_CONST_PAIR(in27, in20, cospi_24_64, cospi_8_64, in19, in28); + DUP4_ARG2(__lsx_vadd_h, in16, in19, in17, in18, in30, in29, in31, in28, in27, + in22, in21, in25); + DOTP_CONST_PAIR(in21, in22, cospi_28_64, cospi_4_64, in26, in24); + DUP2_ARG2(__lsx_vadd_h, in27, in26, in25, in24, in23, in20); + DOTP_CONST_PAIR(in20, in23, cospi_31_64, cospi_1_64, vec4, vec5); + __lsx_vst(vec5, out, 0); + __lsx_vst(vec4, out, 240); + + DUP2_ARG2(__lsx_vsub_h, in27, in26, in25, in24, in22, in21); + DOTP_CONST_PAIR(in21, in22, cospi_15_64, cospi_17_64, vec5, vec4); + __lsx_vst(vec5, out, 224); + __lsx_vst(vec4, out, 16); + + DUP4_ARG2(__lsx_vsub_h, in17, in18, in16, in19, in31, in28, in30, in29, in23, + in26, in24, in20); + tmp0 = __lsx_vneg_h(in23); + DOTP_CONST_PAIR(tmp0, in20, cospi_28_64, cospi_4_64, in27, in25); + DUP2_ARG2(__lsx_vsub_h, in26, in27, in24, in25, in23, in20); + DOTP_CONST_PAIR(in20, in23, cospi_23_64, cospi_9_64, vec4, vec5); + __lsx_vst(vec4, out, 32); + __lsx_vst(vec5, out, 208); + + DUP2_ARG2(__lsx_vadd_h, in26, in27, in24, in25, in22, in21); + DOTP_CONST_PAIR(in21, in22, cospi_7_64, cospi_25_64, vec4, vec5); + __lsx_vst(vec4, out, 48); + __lsx_vst(vec5, out, 192); + + in20 = __lsx_vld(interm_ptr, 64); + in21 = __lsx_vld(interm_ptr, 176); + in27 = __lsx_vld(interm_ptr, 112); + in26 = __lsx_vld(interm_ptr, 128); + + in16 = in20; + in17 = in21; + DUP2_ARG1(__lsx_vneg_h, in16, in17, tmp0, tmp1); + DOTP_CONST_PAIR(tmp0, in27, cospi_24_64, cospi_8_64, in20, in27); + DOTP_CONST_PAIR(tmp1, in26, cospi_24_64, cospi_8_64, in21, in26); + + in22 = __lsx_vld(interm_ptr, 80); + in25 = __lsx_vld(interm_ptr, 96); + in24 = __lsx_vld(interm_ptr, 144); + in23 = __lsx_vld(interm_ptr, 160); + + DUP4_ARG2(__lsx_vsub_h, in23, in20, in22, in21, in25, in26, in24, in27, in28, + in17, in18, in31); + DOTP_CONST_PAIR(in18, in17, cospi_12_64, cospi_20_64, in29, in30); + in16 = __lsx_vadd_h(in28, in29); + in19 = __lsx_vadd_h(in31, in30); + DOTP_CONST_PAIR(in19, in16, cospi_27_64, cospi_5_64, vec5, vec4); + __lsx_vst(vec5, out, 64); + __lsx_vst(vec4, out, 176); + + DUP2_ARG2(__lsx_vsub_h, in28, in29, in31, in30, in17, in18); + DOTP_CONST_PAIR(in18, in17, cospi_11_64, cospi_21_64, vec5, vec4); + __lsx_vst(vec5, out, 80); + __lsx_vst(vec4, out, 160); + + DUP4_ARG2(__lsx_vadd_h, in22, in21, in23, in20, in24, in27, in25, in26, in16, + in29, in30, in19); + tmp0 = __lsx_vneg_h(in16); + DOTP_CONST_PAIR(tmp0, in19, cospi_12_64, cospi_20_64, in28, in31); + DUP2_ARG2(__lsx_vsub_h, in29, in28, in30, in31, in16, in19); + DOTP_CONST_PAIR(in19, in16, cospi_19_64, cospi_13_64, vec5, vec4); + __lsx_vst(vec5, out, 144); + __lsx_vst(vec4, out, 96); + + DUP2_ARG2(__lsx_vadd_h, in29, in28, in30, in31, in17, in18); + DOTP_CONST_PAIR(in18, in17, cospi_3_64, cospi_29_64, vec5, vec4); + __lsx_vst(vec4, out, 112); + __lsx_vst(vec5, out, 128); +} + +static void fdct32x8_1d_row_rd(int16_t *tmp_buf_big, int16_t *tmp_buf, + int16_t *output) { + fdct8x32_1d_row_load_butterfly(tmp_buf_big, tmp_buf); + fdct8x32_1d_row_even_rd(tmp_buf, tmp_buf); + fdct8x32_1d_row_odd_rd((tmp_buf + 128), tmp_buf_big, (tmp_buf + 128)); + fdct8x32_1d_row_transpose_store(tmp_buf, output); +} + +void vpx_fdct32x32_rd_lsx(const int16_t *input, int16_t *out, + int32_t src_stride) { + int32_t i; + DECLARE_ALIGNED(32, int16_t, tmp_buf_big[1024]); + DECLARE_ALIGNED(32, int16_t, tmp_buf[256]); + + /* column transform */ + for (i = 0; i < 4; ++i) { + fdct8x32_1d_column(input + (8 * i), src_stride, &tmp_buf[0], + &tmp_buf_big[0] + (8 * i)); + } + /* row transform */ + for (i = 0; i < 4; ++i) { + fdct32x8_1d_row_rd(&tmp_buf_big[0] + (8 * i * 32), &tmp_buf[0], + out + (8 * i * 32)); + } +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/fwd_txfm_lsx.h b/media/libvpx/libvpx/vpx_dsp/loongarch/fwd_txfm_lsx.h new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/fwd_txfm_lsx.h @@ -0,0 +1,381 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VPX_VPX_DSP_LOONGARCH_FWD_TXFM_LSX_H_ +#define VPX_VPX_DSP_LOONGARCH_FWD_TXFM_LSX_H_ + +#include "vpx_dsp/loongarch/txfm_macros_lsx.h" +#include "vpx_dsp/txfm_common.h" + +#define VP9_FDCT4(in0, in1, in2, in3, out0, out1, out2, out3) \ + { \ + __m128i cnst0_m, cnst1_m, cnst2_m, cnst3_m; \ + __m128i vec0_m, vec1_m, vec2_m, vec3_m; \ + __m128i vec4_m, vec5_m, vec6_m, vec7_m; \ + __m128i coeff_m = { 0x187e3b21d2bf2d41, 0x000000000000c4df }; \ + \ + LSX_BUTTERFLY_4_H(in0, in1, in2, in3, vec0_m, vec1_m, vec2_m, vec3_m); \ + DUP2_ARG2(__lsx_vilvl_h, vec1_m, vec0_m, vec3_m, vec2_m, vec0_m, vec2_m); \ + DUP2_ARG2(__lsx_vreplvei_h, coeff_m, 0, coeff_m, 1, cnst0_m, cnst1_m); \ + cnst1_m = __lsx_vpackev_h(cnst1_m, cnst0_m); \ + vec5_m = __lsx_vdp2_w_h(vec0_m, cnst1_m); \ + DUP2_ARG2(__lsx_vreplvei_h, coeff_m, 4, coeff_m, 3, cnst2_m, cnst3_m); \ + cnst2_m = __lsx_vpackev_h(cnst3_m, cnst2_m); \ + vec7_m = __lsx_vdp2_w_h(vec2_m, cnst2_m); \ + \ + vec4_m = __lsx_vdp2_w_h(vec0_m, cnst0_m); \ + cnst2_m = __lsx_vreplvei_h(coeff_m, 2); \ + cnst2_m = __lsx_vpackev_h(cnst2_m, cnst3_m); \ + vec6_m = __lsx_vdp2_w_h(vec2_m, cnst2_m); \ + \ + DUP4_ARG3(__lsx_vssrarni_h_w, vec4_m, vec4_m, DCT_CONST_BITS, vec5_m, \ + vec5_m, DCT_CONST_BITS, vec6_m, vec6_m, DCT_CONST_BITS, vec7_m, \ + vec7_m, DCT_CONST_BITS, out0, out2, out1, out3); \ + } + +#define VP9_FDCT8(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, out2, \ + out3, out4, out5, out6, out7) \ + { \ + __m128i s0_m, s1_m, s2_m, s3_m, s4_m, s5_m, s6_m; \ + __m128i s7_m, x0_m, x1_m, x2_m, x3_m; \ + __m128i coeff_m = { 0x187e3b21d2bf2d41, 0x238e35370c7c3ec5 }; \ + \ + /* FDCT stage1 */ \ + LSX_BUTTERFLY_8_H(in0, in1, in2, in3, in4, in5, in6, in7, s0_m, s1_m, \ + s2_m, s3_m, s4_m, s5_m, s6_m, s7_m); \ + LSX_BUTTERFLY_4_H(s0_m, s1_m, s2_m, s3_m, x0_m, x1_m, x2_m, x3_m); \ + DUP2_ARG2(__lsx_vilvh_h, x1_m, x0_m, x3_m, x2_m, s0_m, s2_m); \ + DUP2_ARG2(__lsx_vilvl_h, x1_m, x0_m, x3_m, x2_m, s1_m, s3_m); \ + DUP2_ARG2(__lsx_vreplvei_h, coeff_m, 0, coeff_m, 1, x0_m, x1_m); \ + x1_m = __lsx_vpackev_h(x1_m, x0_m); \ + DOT_SHIFT_RIGHT_PCK_H(s0_m, s1_m, x1_m, out4); \ + \ + DUP2_ARG2(__lsx_vreplvei_h, coeff_m, 2, coeff_m, 3, x2_m, x3_m); \ + x2_m = __lsx_vneg_h(x2_m); \ + x2_m = __lsx_vpackev_h(x3_m, x2_m); \ + DOT_SHIFT_RIGHT_PCK_H(s2_m, s3_m, x2_m, out6); \ + \ + DOT_SHIFT_RIGHT_PCK_H(s0_m, s1_m, x0_m, out0); \ + x2_m = __lsx_vreplvei_h(coeff_m, 2); \ + x2_m = __lsx_vpackev_h(x2_m, x3_m); \ + DOT_SHIFT_RIGHT_PCK_H(s2_m, s3_m, x2_m, out2); \ + \ + /* stage2 */ \ + s1_m = __lsx_vilvl_h(s5_m, s6_m); \ + s0_m = __lsx_vilvh_h(s5_m, s6_m); \ + \ + DOT_SHIFT_RIGHT_PCK_H(s0_m, s1_m, x0_m, s6_m); \ + DOT_SHIFT_RIGHT_PCK_H(s0_m, s1_m, x1_m, s5_m); \ + \ + /* stage3 */ \ + LSX_BUTTERFLY_4_H(s4_m, s7_m, s6_m, s5_m, x0_m, x3_m, x2_m, x1_m); \ + \ + /* stage4 */ \ + DUP2_ARG2(__lsx_vilvh_h, x3_m, x0_m, x2_m, x1_m, s4_m, s6_m); \ + DUP2_ARG2(__lsx_vilvl_h, x3_m, x0_m, x2_m, x1_m, s5_m, s7_m); \ + \ + DUP2_ARG2(__lsx_vreplvei_h, coeff_m, 4, coeff_m, 5, x0_m, x1_m); \ + x1_m = __lsx_vpackev_h(x0_m, x1_m); \ + DOT_SHIFT_RIGHT_PCK_H(s4_m, s5_m, x1_m, out1); \ + \ + DUP2_ARG2(__lsx_vreplvei_h, coeff_m, 6, coeff_m, 7, x2_m, x3_m); \ + x2_m = __lsx_vpackev_h(x3_m, x2_m); \ + DOT_SHIFT_RIGHT_PCK_H(s6_m, s7_m, x2_m, out5); \ + \ + x1_m = __lsx_vreplvei_h(coeff_m, 5); \ + x0_m = __lsx_vneg_h(x0_m); \ + x0_m = __lsx_vpackev_h(x1_m, x0_m); \ + DOT_SHIFT_RIGHT_PCK_H(s4_m, s5_m, x0_m, out7); \ + x2_m = __lsx_vreplvei_h(coeff_m, 6); \ + x3_m = __lsx_vneg_h(x3_m); \ + x2_m = __lsx_vpackev_h(x2_m, x3_m); \ + DOT_SHIFT_RIGHT_PCK_H(s6_m, s7_m, x2_m, out3); \ + } + +#define SRLI_AVE_S_4V_H(in0, in1, in2, in3, in4, in5, in6, in7) \ + { \ + __m128i vec0_m, vec1_m, vec2_m, vec3_m, vec4_m, vec5_m, vec6_m, vec7_m; \ + \ + DUP4_ARG2(__lsx_vsrli_h, in0, 15, in1, 15, in2, 15, in3, 15, vec0_m, \ + vec1_m, vec2_m, vec3_m); \ + DUP4_ARG2(__lsx_vsrli_h, in4, 15, in5, 15, in6, 15, in7, 15, vec4_m, \ + vec5_m, vec6_m, vec7_m); \ + DUP4_ARG2(__lsx_vavg_h, vec0_m, in0, vec1_m, in1, vec2_m, in2, vec3_m, \ + in3, in0, in1, in2, in3); \ + DUP4_ARG2(__lsx_vavg_h, vec4_m, in4, vec5_m, in5, vec6_m, in6, vec7_m, \ + in7, in4, in5, in6, in7); \ + } + +#define FDCT32_POSTPROC_2V_POS_H(vec0, vec1) \ + { \ + __m128i tp0_m, tp1_m; \ + __m128i one = __lsx_vreplgr2vr_h(1); \ + \ + tp0_m = __lsx_vslei_h(vec0, 0); \ + tp1_m = __lsx_vslei_h(vec1, 0); \ + tp0_m = __lsx_vxori_b(tp0_m, 255); \ + tp1_m = __lsx_vxori_b(tp1_m, 255); \ + vec0 = __lsx_vadd_h(vec0, one); \ + vec1 = __lsx_vadd_h(vec1, one); \ + tp0_m = __lsx_vand_v(one, tp0_m); \ + tp1_m = __lsx_vand_v(one, tp1_m); \ + vec0 = __lsx_vadd_h(vec0, tp0_m); \ + vec1 = __lsx_vadd_h(vec1, tp1_m); \ + vec0 = __lsx_vsrai_h(vec0, 2); \ + vec1 = __lsx_vsrai_h(vec1, 2); \ + } + +#define FDCT_POSTPROC_2V_NEG_H(vec0, vec1) \ + { \ + __m128i tp0_m, tp1_m; \ + __m128i one_m = __lsx_vldi(0x401); \ + \ + tp0_m = __lsx_vslti_h(vec0, 0); \ + tp1_m = __lsx_vslti_h(vec1, 0); \ + vec0 = __lsx_vadd_h(vec0, one_m); \ + vec1 = __lsx_vadd_h(vec1, one_m); \ + tp0_m = __lsx_vand_v(one_m, tp0_m); \ + tp1_m = __lsx_vand_v(one_m, tp1_m); \ + vec0 = __lsx_vadd_h(vec0, tp0_m); \ + vec1 = __lsx_vadd_h(vec1, tp1_m); \ + vec0 = __lsx_vsrai_h(vec0, 2); \ + vec1 = __lsx_vsrai_h(vec1, 2); \ + } + +#define FDCT32_POSTPROC_NEG_W(vec) \ + { \ + __m128i temp_m; \ + __m128i one_m = __lsx_vreplgr2vr_w(1); \ + \ + temp_m = __lsx_vslti_w(vec, 0); \ + vec = __lsx_vadd_w(vec, one_m); \ + temp_m = __lsx_vand_v(one_m, temp_m); \ + vec = __lsx_vadd_w(vec, temp_m); \ + vec = __lsx_vsrai_w(vec, 2); \ + } + +#define DOTP_CONST_PAIR_W(reg0_left, reg1_left, reg0_right, reg1_right, \ + const0, const1, out0, out1, out2, out3) \ + { \ + __m128i s0_m, s1_m, s2_m, s3_m, s4_m, s5_m, s6_m, s7_m; \ + __m128i tp0_m, tp1_m, tp2_m, tp3_m, _tmp0, _tmp1; \ + __m128i k0_m = __lsx_vreplgr2vr_w((int32_t)const0); \ + \ + s0_m = __lsx_vreplgr2vr_w((int32_t)const1); \ + k0_m = __lsx_vpackev_w(s0_m, k0_m); \ + \ + DUP2_ARG1(__lsx_vneg_w, reg1_left, reg1_right, _tmp0, _tmp1); \ + s1_m = __lsx_vilvl_w(_tmp0, reg0_left); \ + s0_m = __lsx_vilvh_w(_tmp0, reg0_left); \ + s3_m = __lsx_vilvl_w(reg0_left, reg1_left); \ + s2_m = __lsx_vilvh_w(reg0_left, reg1_left); \ + s5_m = __lsx_vilvl_w(_tmp1, reg0_right); \ + s4_m = __lsx_vilvh_w(_tmp1, reg0_right); \ + s7_m = __lsx_vilvl_w(reg0_right, reg1_right); \ + s6_m = __lsx_vilvh_w(reg0_right, reg1_right); \ + DUP2_ARG2(__lsx_vdp2_d_w, s0_m, k0_m, s1_m, k0_m, tp0_m, tp1_m); \ + DUP2_ARG2(__lsx_vdp2_d_w, s4_m, k0_m, s5_m, k0_m, tp2_m, tp3_m); \ + DUP2_ARG3(__lsx_vssrarni_w_d, tp0_m, tp1_m, DCT_CONST_BITS, tp2_m, tp3_m, \ + DCT_CONST_BITS, out0, out1); \ + DUP2_ARG2(__lsx_vdp2_d_w, s2_m, k0_m, s3_m, k0_m, tp0_m, tp1_m); \ + DUP2_ARG2(__lsx_vdp2_d_w, s6_m, k0_m, s7_m, k0_m, tp2_m, tp3_m); \ + DUP2_ARG3(__lsx_vssrarni_w_d, tp0_m, tp1_m, DCT_CONST_BITS, tp2_m, tp3_m, \ + DCT_CONST_BITS, out2, out3); \ + } + +#define VP9_ADDBLK_ST8x4_UB(dst, _stride, _stride2, _stride3, in0, in1, in2, \ + in3) \ + { \ + __m128i dst0_m, dst1_m, dst2_m, dst3_m; \ + __m128i tmp0_m, tmp1_m; \ + __m128i res0_m, res1_m, res2_m, res3_m; \ + \ + dst0_m = __lsx_vld(dst, 0); \ + DUP2_ARG2(__lsx_vldx, dst, _stride, dst, _stride2, dst1_m, dst2_m); \ + dst3_m = __lsx_vldx(dst, _stride3); \ + DUP4_ARG2(__lsx_vsllwil_hu_bu, dst0_m, 0, dst1_m, 0, dst2_m, 0, dst3_m, 0, \ + res0_m, res1_m, res2_m, res3_m); \ + DUP4_ARG2(__lsx_vadd_h, res0_m, in0, res1_m, in1, res2_m, in2, res3_m, \ + in3, res0_m, res1_m, res2_m, res3_m); \ + DUP2_ARG3(__lsx_vssrarni_bu_h, res1_m, res0_m, 0, res3_m, res2_m, 0, \ + tmp0_m, tmp1_m); \ + __lsx_vstelm_d(tmp0_m, dst, 0, 0); \ + __lsx_vstelm_d(tmp0_m, dst + _stride, 0, 1); \ + __lsx_vstelm_d(tmp1_m, dst + _stride2, 0, 0); \ + __lsx_vstelm_d(tmp1_m, dst + _stride3, 0, 1); \ + } + +#define FDCT8x16_EVEN(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, \ + out2, out3, out4, out5, out6, out7) \ + { \ + __m128i s0_m, s1_m, s2_m, s3_m, s4_m, s5_m, s6_m, s7_m; \ + __m128i x0_m, x1_m, x2_m, x3_m; \ + __m128i coeff_m = { 0x187e3b21d2bf2d41, 0x238e35370c7c3ec5 }; \ + \ + /* FDCT stage1 */ \ + LSX_BUTTERFLY_8_H(in0, in1, in2, in3, in4, in5, in6, in7, s0_m, s1_m, \ + s2_m, s3_m, s4_m, s5_m, s6_m, s7_m); \ + LSX_BUTTERFLY_4_H(s0_m, s1_m, s2_m, s3_m, x0_m, x1_m, x2_m, x3_m); \ + DUP2_ARG2(__lsx_vilvh_h, x1_m, x0_m, x3_m, x2_m, s0_m, s2_m); \ + DUP2_ARG2(__lsx_vilvl_h, x1_m, x0_m, x3_m, x2_m, s1_m, s3_m); \ + DUP2_ARG2(__lsx_vreplvei_h, coeff_m, 0, coeff_m, 1, x0_m, x1_m); \ + x1_m = __lsx_vpackev_h(x1_m, x0_m); \ + DOT_SHIFT_RIGHT_PCK_H(s0_m, s1_m, x1_m, out4); \ + \ + DUP2_ARG2(__lsx_vreplvei_h, coeff_m, 2, coeff_m, 3, x2_m, x3_m); \ + x2_m = __lsx_vneg_h(x2_m); \ + x2_m = __lsx_vpackev_h(x3_m, x2_m); \ + DOT_SHIFT_RIGHT_PCK_H(s2_m, s3_m, x2_m, out6); \ + \ + DOT_SHIFT_RIGHT_PCK_H(s0_m, s1_m, x0_m, out0); \ + x2_m = __lsx_vreplvei_h(coeff_m, 2); \ + x2_m = __lsx_vpackev_h(x2_m, x3_m); \ + DOT_SHIFT_RIGHT_PCK_H(s2_m, s3_m, x2_m, out2); \ + \ + /* stage2 */ \ + s1_m = __lsx_vilvl_h(s5_m, s6_m); \ + s0_m = __lsx_vilvh_h(s5_m, s6_m); \ + \ + DOT_SHIFT_RIGHT_PCK_H(s0_m, s1_m, x0_m, s6_m); \ + DOT_SHIFT_RIGHT_PCK_H(s0_m, s1_m, x1_m, s5_m); \ + \ + /* stage3 */ \ + LSX_BUTTERFLY_4_H(s4_m, s7_m, s6_m, s5_m, x0_m, x3_m, x2_m, x1_m); \ + \ + /* stage4 */ \ + DUP2_ARG2(__lsx_vilvh_h, x3_m, x0_m, x2_m, x1_m, s4_m, s6_m); \ + DUP2_ARG2(__lsx_vilvl_h, x3_m, x0_m, x2_m, x1_m, s5_m, s7_m); \ + \ + DUP2_ARG2(__lsx_vreplvei_h, coeff_m, 4, coeff_m, 5, x0_m, x1_m); \ + x1_m = __lsx_vpackev_h(x0_m, x1_m); \ + DOT_SHIFT_RIGHT_PCK_H(s4_m, s5_m, x1_m, out1); \ + \ + DUP2_ARG2(__lsx_vreplvei_h, coeff_m, 6, coeff_m, 7, x2_m, x3_m); \ + x2_m = __lsx_vpackev_h(x3_m, x2_m); \ + DOT_SHIFT_RIGHT_PCK_H(s6_m, s7_m, x2_m, out5); \ + \ + x1_m = __lsx_vreplvei_h(coeff_m, 5); \ + x0_m = __lsx_vneg_h(x0_m); \ + x0_m = __lsx_vpackev_h(x1_m, x0_m); \ + DOT_SHIFT_RIGHT_PCK_H(s4_m, s5_m, x0_m, out7); \ + \ + x2_m = __lsx_vreplvei_h(coeff_m, 6); \ + x3_m = __lsx_vneg_h(x3_m); \ + x2_m = __lsx_vpackev_h(x2_m, x3_m); \ + DOT_SHIFT_RIGHT_PCK_H(s6_m, s7_m, x2_m, out3); \ + } + +#define FDCT8x16_ODD(input0, input1, input2, input3, input4, input5, input6, \ + input7, out1, out3, out5, out7, out9, out11, out13, \ + out15) \ + { \ + __m128i stp21_m, stp22_m, stp23_m, stp24_m, stp25_m, stp26_m; \ + __m128i stp30_m, stp31_m, stp32_m, stp33_m, stp34_m, stp35_m; \ + __m128i stp36_m, stp37_m, vec0_m, vec1_m; \ + __m128i vec2_m, vec3_m, vec4_m, vec5_m, vec6_m; \ + __m128i cnst0_m, cnst1_m, cnst4_m, cnst5_m; \ + __m128i coeff_m = { 0x187e3b21d2bf2d41, 0x238e3537e782c4df }; \ + __m128i coeff1_m = { 0x289a317906463fb1, 0x12943d3f1e2b3871 }; \ + __m128i coeff2_m = { 0xed6cd766c78fc04f, 0x0 }; \ + \ + /* stp 1 */ \ + DUP2_ARG2(__lsx_vilvh_h, input2, input5, input3, input4, vec2_m, vec4_m); \ + DUP2_ARG2(__lsx_vilvl_h, input2, input5, input3, input4, vec3_m, vec5_m); \ + \ + cnst4_m = __lsx_vreplvei_h(coeff_m, 0); \ + DOT_SHIFT_RIGHT_PCK_H(vec2_m, vec3_m, cnst4_m, stp25_m); \ + \ + cnst5_m = __lsx_vreplvei_h(coeff_m, 1); \ + cnst5_m = __lsx_vpackev_h(cnst5_m, cnst4_m); \ + DOT_SHIFT_RIGHT_PCK_H(vec2_m, vec3_m, cnst5_m, stp22_m); \ + DOT_SHIFT_RIGHT_PCK_H(vec4_m, vec5_m, cnst4_m, stp24_m); \ + DOT_SHIFT_RIGHT_PCK_H(vec4_m, vec5_m, cnst5_m, stp23_m); \ + \ + /* stp2 */ \ + LSX_BUTTERFLY_4_H(input0, input1, stp22_m, stp23_m, stp30_m, stp31_m, \ + stp32_m, stp33_m); \ + LSX_BUTTERFLY_4_H(input7, input6, stp25_m, stp24_m, stp37_m, stp36_m, \ + stp35_m, stp34_m); \ + \ + DUP2_ARG2(__lsx_vilvh_h, stp36_m, stp31_m, stp35_m, stp32_m, vec2_m, \ + vec4_m); \ + DUP2_ARG2(__lsx_vilvl_h, stp36_m, stp31_m, stp35_m, stp32_m, vec3_m, \ + vec5_m); \ + \ + DUP2_ARG2(__lsx_vreplvei_h, coeff_m, 2, coeff_m, 3, cnst0_m, cnst1_m); \ + cnst0_m = __lsx_vpackev_h(cnst0_m, cnst1_m); \ + DOT_SHIFT_RIGHT_PCK_H(vec2_m, vec3_m, cnst0_m, stp26_m); \ + \ + cnst0_m = __lsx_vreplvei_h(coeff_m, 4); \ + cnst1_m = __lsx_vpackev_h(cnst1_m, cnst0_m); \ + DOT_SHIFT_RIGHT_PCK_H(vec2_m, vec3_m, cnst1_m, stp21_m); \ + \ + DUP2_ARG2(__lsx_vreplvei_h, coeff_m, 5, coeff_m, 2, cnst0_m, cnst1_m); \ + cnst1_m = __lsx_vpackev_h(cnst0_m, cnst1_m); \ + DOT_SHIFT_RIGHT_PCK_H(vec4_m, vec5_m, cnst1_m, stp25_m); \ + \ + cnst0_m = __lsx_vreplvei_h(coeff_m, 3); \ + cnst1_m = __lsx_vpackev_h(cnst1_m, cnst0_m); \ + DOT_SHIFT_RIGHT_PCK_H(vec4_m, vec5_m, cnst1_m, stp22_m); \ + \ + /* stp4 */ \ + LSX_BUTTERFLY_4_H(stp30_m, stp37_m, stp26_m, stp21_m, vec6_m, vec2_m, \ + vec4_m, vec5_m); \ + LSX_BUTTERFLY_4_H(stp33_m, stp34_m, stp25_m, stp22_m, stp21_m, stp23_m, \ + stp24_m, stp31_m); \ + \ + vec1_m = __lsx_vilvl_h(vec2_m, vec6_m); \ + vec0_m = __lsx_vilvh_h(vec2_m, vec6_m); \ + DUP2_ARG2(__lsx_vreplvei_h, coeff1_m, 0, coeff1_m, 1, cnst0_m, cnst1_m); \ + cnst0_m = __lsx_vpackev_h(cnst0_m, cnst1_m); \ + \ + DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst0_m, out1); \ + \ + cnst0_m = __lsx_vreplvei_h(coeff2_m, 0); \ + cnst0_m = __lsx_vpackev_h(cnst1_m, cnst0_m); \ + DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst0_m, out15); \ + \ + vec1_m = __lsx_vilvl_h(vec4_m, vec5_m); \ + vec0_m = __lsx_vilvh_h(vec4_m, vec5_m); \ + DUP2_ARG2(__lsx_vreplvei_h, coeff1_m, 2, coeff1_m, 3, cnst0_m, cnst1_m); \ + cnst1_m = __lsx_vpackev_h(cnst1_m, cnst0_m); \ + \ + DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst1_m, out9); \ + \ + cnst1_m = __lsx_vreplvei_h(coeff2_m, 2); \ + cnst0_m = __lsx_vpackev_h(cnst0_m, cnst1_m); \ + DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst0_m, out7); \ + \ + vec1_m = __lsx_vilvl_h(stp23_m, stp21_m); \ + vec0_m = __lsx_vilvh_h(stp23_m, stp21_m); \ + DUP2_ARG2(__lsx_vreplvei_h, coeff1_m, 4, coeff1_m, 5, cnst0_m, cnst1_m); \ + cnst0_m = __lsx_vpackev_h(cnst0_m, cnst1_m); \ + DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst0_m, out5); \ + \ + cnst0_m = __lsx_vreplvei_h(coeff2_m, 1); \ + cnst0_m = __lsx_vpackev_h(cnst1_m, cnst0_m); \ + DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst0_m, out11); \ + \ + vec1_m = __lsx_vilvl_h(stp24_m, stp31_m); \ + vec0_m = __lsx_vilvh_h(stp24_m, stp31_m); \ + DUP2_ARG2(__lsx_vreplvei_h, coeff1_m, 6, coeff1_m, 7, cnst0_m, cnst1_m); \ + cnst1_m = __lsx_vpackev_h(cnst1_m, cnst0_m); \ + \ + DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst1_m, out13); \ + \ + cnst1_m = __lsx_vreplvei_h(coeff2_m, 3); \ + cnst0_m = __lsx_vpackev_h(cnst0_m, cnst1_m); \ + DOT_SHIFT_RIGHT_PCK_H(vec0_m, vec1_m, cnst0_m, out3); \ + } + +void fdct8x16_1d_column(const int16_t *input, int16_t *tmp_ptr, + int32_t src_stride); +void fdct16x8_1d_row(int16_t *input, int16_t *output); +#endif // VPX_VPX_DSP_LOONGARCH_FWD_TXFM_LSX_H_ diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/fwd_txfm_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/fwd_txfm_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/fwd_txfm_lsx.c @@ -0,0 +1,350 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "./vpx_dsp_rtcd.h" +#include "vpx_dsp/loongarch/fwd_txfm_lsx.h" + +#define LSX_TRANSPOSE4x4_H(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ + { \ + __m128i _s0, _s1, _s2, _s3, _t0, _t1, _t2, _t3; \ + \ + DUP2_ARG2(__lsx_vilvl_h, _in2, _in0, _in3, _in1, _s0, _s1); \ + DUP2_ARG2(__lsx_vilvh_h, _in2, _in0, _in3, _in1, _s2, _s3); \ + _t0 = __lsx_vilvl_h(_s1, _s0); \ + _t1 = __lsx_vilvh_h(_s1, _s0); \ + _t2 = __lsx_vilvl_h(_s3, _s2); \ + _t3 = __lsx_vilvh_h(_s3, _s2); \ + DUP2_ARG2(__lsx_vpickev_d, _t2, _t0, _t3, _t1, _out0, _out2); \ + DUP2_ARG2(__lsx_vpickod_d, _t2, _t0, _t3, _t1, _out1, _out3); \ + } + +#if !CONFIG_VP9_HIGHBITDEPTH +void fdct8x16_1d_column(const int16_t *input, int16_t *tmp_ptr, + int32_t src_stride) { + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i in8, in9, in10, in11, in12, in13, in14, in15; + __m128i stp21, stp22, stp23, stp24, stp25, stp26, stp30; + __m128i stp31, stp32, stp33, stp34, stp35, stp36, stp37; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, cnst0, cnst1, cnst4, cnst5; + __m128i coeff = { 0x187e3b21d2bf2d41, 0x238e3537e782c4df }; + __m128i coeff1 = { 0x289a317906463fb1, 0x12943d3f1e2b3871 }; + __m128i coeff2 = { 0xed6cd766c78fc04f, 0x0 }; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride4 = src_stride2 << 1; + int32_t src_stride6 = src_stride4 + src_stride2; + int32_t src_stride8 = src_stride4 << 1; + int16_t *input_tmp = (int16_t *)input; + in0 = __lsx_vld(input_tmp, 0); + DUP4_ARG2(__lsx_vldx, input_tmp, src_stride2, input_tmp, src_stride4, + input_tmp, src_stride6, input_tmp, src_stride8, in1, in2, in3, in4); + input_tmp += src_stride4; + DUP4_ARG2(__lsx_vldx, input_tmp, src_stride2, input_tmp, src_stride4, + input_tmp, src_stride6, input_tmp, src_stride8, in5, in6, in7, in8); + input_tmp += src_stride4; + DUP4_ARG2(__lsx_vldx, input_tmp, src_stride2, input_tmp, src_stride4, + input_tmp, src_stride6, input_tmp, src_stride8, in9, in10, in11, + in12); + input_tmp += src_stride4; + DUP2_ARG2(__lsx_vldx, input_tmp, src_stride2, input_tmp, src_stride4, in13, + in14); + input_tmp += src_stride2; + in15 = __lsx_vldx(input_tmp, src_stride2); + + DUP4_ARG2(__lsx_vslli_h, in0, 2, in1, 2, in2, 2, in3, 2, in0, in1, in2, in3); + DUP4_ARG2(__lsx_vslli_h, in4, 2, in5, 2, in6, 2, in7, 2, in4, in5, in6, in7); + DUP4_ARG2(__lsx_vslli_h, in8, 2, in9, 2, in10, 2, in11, 2, in8, in9, in10, + in11); + DUP4_ARG2(__lsx_vslli_h, in12, 2, in13, 2, in14, 2, in15, 2, in12, in13, in14, + in15); + DUP4_ARG2(__lsx_vadd_h, in0, in15, in1, in14, in2, in13, in3, in12, tmp0, + tmp1, tmp2, tmp3); + DUP4_ARG2(__lsx_vadd_h, in4, in11, in5, in10, in6, in9, in7, in8, tmp4, tmp5, + tmp6, tmp7); + FDCT8x16_EVEN(tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp0, tmp1, + tmp2, tmp3, tmp4, tmp5, tmp6, tmp7); + __lsx_vst(tmp0, tmp_ptr, 0); + __lsx_vst(tmp1, tmp_ptr, 64); + __lsx_vst(tmp2, tmp_ptr, 128); + __lsx_vst(tmp3, tmp_ptr, 192); + __lsx_vst(tmp4, tmp_ptr, 256); + __lsx_vst(tmp5, tmp_ptr, 320); + __lsx_vst(tmp6, tmp_ptr, 384); + __lsx_vst(tmp7, tmp_ptr, 448); + DUP4_ARG2(__lsx_vsub_h, in0, in15, in1, in14, in2, in13, in3, in12, in15, + in14, in13, in12); + DUP4_ARG2(__lsx_vsub_h, in4, in11, in5, in10, in6, in9, in7, in8, in11, in10, + in9, in8); + + tmp_ptr += 16; + + /* stp 1 */ + DUP2_ARG2(__lsx_vilvh_h, in10, in13, in11, in12, vec2, vec4); + DUP2_ARG2(__lsx_vilvl_h, in10, in13, in11, in12, vec3, vec5); + + cnst4 = __lsx_vreplvei_h(coeff, 0); + DOT_SHIFT_RIGHT_PCK_H(vec2, vec3, cnst4, stp25); + + cnst5 = __lsx_vreplvei_h(coeff, 1); + cnst5 = __lsx_vpackev_h(cnst5, cnst4); + DOT_SHIFT_RIGHT_PCK_H(vec2, vec3, cnst5, stp22); + DOT_SHIFT_RIGHT_PCK_H(vec4, vec5, cnst4, stp24); + DOT_SHIFT_RIGHT_PCK_H(vec4, vec5, cnst5, stp23); + + /* stp2 */ + LSX_BUTTERFLY_4_H(in8, in9, stp22, stp23, stp30, stp31, stp32, stp33); + LSX_BUTTERFLY_4_H(in15, in14, stp25, stp24, stp37, stp36, stp35, stp34); + DUP2_ARG2(__lsx_vilvh_h, stp36, stp31, stp35, stp32, vec2, vec4); + DUP2_ARG2(__lsx_vilvl_h, stp36, stp31, stp35, stp32, vec3, vec5); + DUP2_ARG2(__lsx_vreplvei_h, coeff, 2, coeff, 3, cnst0, cnst1); + cnst0 = __lsx_vpackev_h(cnst0, cnst1); + DOT_SHIFT_RIGHT_PCK_H(vec2, vec3, cnst0, stp26); + + cnst0 = __lsx_vreplvei_h(coeff, 4); + cnst1 = __lsx_vpackev_h(cnst1, cnst0); + DOT_SHIFT_RIGHT_PCK_H(vec2, vec3, cnst1, stp21); + + LSX_BUTTERFLY_4_H(stp30, stp37, stp26, stp21, in8, in15, in14, in9); + vec1 = __lsx_vilvl_h(in15, in8); + vec0 = __lsx_vilvh_h(in15, in8); + + DUP2_ARG2(__lsx_vreplvei_h, coeff1, 0, coeff1, 1, cnst0, cnst1); + cnst0 = __lsx_vpackev_h(cnst0, cnst1); + + DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst0, in8); + __lsx_vst(in8, tmp_ptr, 0); + + cnst0 = __lsx_vreplvei_h(coeff2, 0); + cnst0 = __lsx_vpackev_h(cnst1, cnst0); + DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst0, in8); + __lsx_vst(in8, tmp_ptr, 448); + + vec1 = __lsx_vilvl_h(in14, in9); + vec0 = __lsx_vilvh_h(in14, in9); + DUP2_ARG2(__lsx_vreplvei_h, coeff1, 2, coeff1, 3, cnst0, cnst1); + cnst1 = __lsx_vpackev_h(cnst1, cnst0); + + DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst1, in8); + __lsx_vst(in8, tmp_ptr, 256); + + cnst1 = __lsx_vreplvei_h(coeff2, 2); + cnst0 = __lsx_vpackev_h(cnst0, cnst1); + DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst0, in8); + __lsx_vst(in8, tmp_ptr, 192); + + DUP2_ARG2(__lsx_vreplvei_h, coeff, 2, coeff, 5, cnst0, cnst1); + cnst1 = __lsx_vpackev_h(cnst1, cnst0); + DOT_SHIFT_RIGHT_PCK_H(vec4, vec5, cnst1, stp25); + + cnst1 = __lsx_vreplvei_h(coeff, 3); + cnst1 = __lsx_vpackev_h(cnst0, cnst1); + DOT_SHIFT_RIGHT_PCK_H(vec4, vec5, cnst1, stp22); + + /* stp4 */ + DUP2_ARG2(__lsx_vadd_h, stp34, stp25, stp33, stp22, in13, in10); + + vec1 = __lsx_vilvl_h(in13, in10); + vec0 = __lsx_vilvh_h(in13, in10); + DUP2_ARG2(__lsx_vreplvei_h, coeff1, 4, coeff1, 5, cnst0, cnst1); + cnst0 = __lsx_vpackev_h(cnst0, cnst1); + DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst0, in8); + __lsx_vst(in8, tmp_ptr, 128); + + cnst0 = __lsx_vreplvei_h(coeff2, 1); + cnst0 = __lsx_vpackev_h(cnst1, cnst0); + DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst0, in8); + __lsx_vst(in8, tmp_ptr, 320); + + DUP2_ARG2(__lsx_vsub_h, stp34, stp25, stp33, stp22, in12, in11); + vec1 = __lsx_vilvl_h(in12, in11); + vec0 = __lsx_vilvh_h(in12, in11); + DUP2_ARG2(__lsx_vreplvei_h, coeff1, 6, coeff1, 7, cnst0, cnst1); + cnst1 = __lsx_vpackev_h(cnst1, cnst0); + + DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst1, in8); + __lsx_vst(in8, tmp_ptr, 384); + + cnst1 = __lsx_vreplvei_h(coeff2, 3); + cnst0 = __lsx_vpackev_h(cnst0, cnst1); + DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst0, in8); + __lsx_vst(in8, tmp_ptr, 64); +} + +void fdct16x8_1d_row(int16_t *input, int16_t *output) { + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + __m128i in8, in9, in10, in11, in12, in13, in14, in15; + int16_t *input_tmp = input; + + DUP4_ARG2(__lsx_vld, input, 0, input, 32, input, 64, input, 96, in0, in1, in2, + in3); + DUP4_ARG2(__lsx_vld, input, 128, input, 160, input, 192, input, 224, in4, in5, + in6, in7); + DUP4_ARG2(__lsx_vld, input_tmp, 16, input_tmp, 48, input_tmp, 80, input_tmp, + 112, in8, in9, in10, in11); + DUP4_ARG2(__lsx_vld, input_tmp, 144, input_tmp, 176, input_tmp, 208, + input_tmp, 240, in12, in13, in14, in15); + + LSX_TRANSPOSE8x8_H(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + LSX_TRANSPOSE8x8_H(in8, in9, in10, in11, in12, in13, in14, in15, in8, in9, + in10, in11, in12, in13, in14, in15); + DUP4_ARG2(__lsx_vaddi_hu, in0, 1, in1, 1, in2, 1, in3, 1, in0, in1, in2, in3); + DUP4_ARG2(__lsx_vaddi_hu, in4, 1, in5, 1, in6, 1, in7, 1, in4, in5, in6, in7); + DUP4_ARG2(__lsx_vaddi_hu, in8, 1, in9, 1, in10, 1, in11, 1, in8, in9, in10, + in11); + DUP4_ARG2(__lsx_vaddi_hu, in12, 1, in13, 1, in14, 1, in15, 1, in12, in13, + in14, in15); + + DUP4_ARG2(__lsx_vsrai_h, in0, 2, in1, 2, in2, 2, in3, 2, in0, in1, in2, in3); + DUP4_ARG2(__lsx_vsrai_h, in4, 2, in5, 2, in6, 2, in7, 2, in4, in5, in6, in7); + DUP4_ARG2(__lsx_vsrai_h, in8, 2, in9, 2, in10, 2, in11, 2, in8, in9, in10, + in11); + DUP4_ARG2(__lsx_vsrai_h, in12, 2, in13, 2, in14, 2, in15, 2, in12, in13, in14, + in15); + LSX_BUTTERFLY_16_H(in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, in10, + in11, in12, in13, in14, in15, tmp0, tmp1, tmp2, tmp3, tmp4, + tmp5, tmp6, tmp7, in8, in9, in10, in11, in12, in13, in14, + in15); + __lsx_vst(in8, input, 0); + __lsx_vst(in9, input, 32); + __lsx_vst(in10, input, 64); + __lsx_vst(in11, input, 96); + __lsx_vst(in12, input, 128); + __lsx_vst(in13, input, 160); + __lsx_vst(in14, input, 192); + __lsx_vst(in15, input, 224); + + FDCT8x16_EVEN(tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp0, tmp1, + tmp2, tmp3, tmp4, tmp5, tmp6, tmp7); + DUP4_ARG2(__lsx_vld, input, 0, input, 32, input, 64, input, 96, in8, in9, + in10, in11); + DUP4_ARG2(__lsx_vld, input, 128, input, 160, input, 192, input, 224, in12, + in13, in14, in15); + FDCT8x16_ODD(in8, in9, in10, in11, in12, in13, in14, in15, in0, in1, in2, in3, + in4, in5, in6, in7); + LSX_TRANSPOSE8x8_H(tmp0, in0, tmp1, in1, tmp2, in2, tmp3, in3, tmp0, in0, + tmp1, in1, tmp2, in2, tmp3, in3); + __lsx_vst(tmp0, output, 0); + __lsx_vst(in0, output, 32); + __lsx_vst(tmp1, output, 64); + __lsx_vst(in1, output, 96); + __lsx_vst(tmp2, output, 128); + __lsx_vst(in2, output, 160); + __lsx_vst(tmp3, output, 192); + __lsx_vst(in3, output, 224); + + LSX_TRANSPOSE8x8_H(tmp4, in4, tmp5, in5, tmp6, in6, tmp7, in7, tmp4, in4, + tmp5, in5, tmp6, in6, tmp7, in7); + __lsx_vst(tmp4, output, 16); + __lsx_vst(in4, output, 48); + __lsx_vst(tmp5, output, 80); + __lsx_vst(in5, output, 112); + __lsx_vst(tmp6, output, 144); + __lsx_vst(in6, output, 176); + __lsx_vst(tmp7, output, 208); + __lsx_vst(in7, output, 240); +} + +void vpx_fdct4x4_lsx(const int16_t *input, int16_t *output, + int32_t src_stride) { + __m128i in0, in1, in2, in3; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride4 = src_stride2 << 1; + int32_t src_stride6 = src_stride4 + src_stride2; + + in0 = __lsx_vld(input, 0); + DUP2_ARG2(__lsx_vldx, input, src_stride2, input, src_stride4, in1, in2); + in3 = __lsx_vldx(input, src_stride6); + + /* fdct4 pre-process */ + { + __m128i vec, mask; + __m128i zero = __lsx_vldi(0); + + mask = __lsx_vinsgr2vr_b(zero, 1, 0); + DUP4_ARG2(__lsx_vslli_h, in0, 4, in1, 4, in2, 4, in3, 4, in0, in1, in2, + in3); + vec = __lsx_vseqi_h(in0, 0); + vec = __lsx_vxori_b(vec, 255); + vec = __lsx_vand_v(mask, vec); + in0 = __lsx_vadd_h(in0, vec); + } + + VP9_FDCT4(in0, in1, in2, in3, in0, in1, in2, in3); + LSX_TRANSPOSE4x4_H(in0, in1, in2, in3, in0, in1, in2, in3); + VP9_FDCT4(in0, in1, in2, in3, in0, in1, in2, in3); + LSX_TRANSPOSE4x4_H(in0, in1, in2, in3, in0, in1, in2, in3); + DUP4_ARG2(__lsx_vaddi_hu, in0, 1, in1, 1, in2, 1, in3, 1, in0, in1, in2, in3); + DUP4_ARG2(__lsx_vsrai_h, in0, 2, in1, 2, in2, 2, in3, 2, in0, in1, in2, in3); + DUP2_ARG2(__lsx_vpickev_d, in1, in0, in3, in2, in0, in2); + __lsx_vst(in0, output, 0); + __lsx_vst(in2, output, 16); +} + +void vpx_fdct8x8_lsx(const int16_t *input, int16_t *output, + int32_t src_stride) { + __m128i in0, in1, in2, in3, in4, in5, in6, in7; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride4 = src_stride2 << 1; + int32_t src_stride6 = src_stride4 + src_stride2; + int16_t *input_tmp = (int16_t *)input; + + in0 = __lsx_vld(input_tmp, 0); + DUP2_ARG2(__lsx_vldx, input_tmp, src_stride2, input_tmp, src_stride4, in1, + in2); + in3 = __lsx_vldx(input_tmp, src_stride6); + input_tmp += src_stride4; + in4 = __lsx_vld(input_tmp, 0); + DUP2_ARG2(__lsx_vldx, input_tmp, src_stride2, input_tmp, src_stride4, in5, + in6); + in7 = __lsx_vldx(input_tmp, src_stride6); + + DUP4_ARG2(__lsx_vslli_h, in0, 2, in1, 2, in2, 2, in3, 2, in0, in1, in2, in3); + DUP4_ARG2(__lsx_vslli_h, in4, 2, in5, 2, in6, 2, in7, 2, in4, in5, in6, in7); + + VP9_FDCT8(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, in4, + in5, in6, in7); + LSX_TRANSPOSE8x8_H(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + VP9_FDCT8(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, in4, + in5, in6, in7); + LSX_TRANSPOSE8x8_H(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, + in4, in5, in6, in7); + SRLI_AVE_S_4V_H(in0, in1, in2, in3, in4, in5, in6, in7); + + __lsx_vst(in0, output, 0); + __lsx_vst(in1, output, 16); + __lsx_vst(in2, output, 32); + __lsx_vst(in3, output, 48); + __lsx_vst(in4, output, 64); + __lsx_vst(in5, output, 80); + __lsx_vst(in6, output, 96); + __lsx_vst(in7, output, 112); +} + +void vpx_fdct16x16_lsx(const int16_t *input, int16_t *output, + int32_t src_stride) { + int32_t i; + DECLARE_ALIGNED(32, int16_t, tmp_buf[16 * 16]); + + /* column transform */ + for (i = 0; i < 2; ++i) { + fdct8x16_1d_column((input + 8 * i), (&tmp_buf[0] + 8 * i), src_stride); + } + + /* row transform */ + for (i = 0; i < 2; ++i) { + fdct16x8_1d_row((&tmp_buf[0] + (128 * i)), (output + (128 * i))); + } +} +#endif // !CONFIG_VP9_HIGHBITDEPTH diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/idct32x32_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/idct32x32_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/idct32x32_lsx.c @@ -0,0 +1,834 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "./vpx_dsp_rtcd.h" +#include "vpx_dsp/loongarch/fwd_txfm_lsx.h" + +#define UNPCK_UB_SH(_in, _out0, _out1) \ + { \ + _out0 = __lsx_vsllwil_hu_bu(_in, 0); \ + _out1 = __lsx_vexth_hu_bu(_in); \ + } + +static void idct32x8_row_transpose_store(const int16_t *input, + int16_t *tmp_buf) { + __m128i m0, m1, m2, m3, m4, m5, m6, m7; + __m128i n0, n1, n2, n3, n4, n5, n6, n7; + + /* 1st & 2nd 8x8 */ + DUP4_ARG2(__lsx_vld, input, 0, input, 64, input, 128, input, 192, m0, n0, m1, + n1); + DUP4_ARG2(__lsx_vld, input, 256, input, 320, input, 384, input, 448, m2, n2, + m3, n3); + DUP4_ARG2(__lsx_vld, input, 16, input, 80, input, 144, input, 208, m4, n4, m5, + n5); + DUP4_ARG2(__lsx_vld, input, 272, input, 336, input, 400, input, 464, m6, n6, + m7, n7); + + LSX_TRANSPOSE8x8_H(m0, n0, m1, n1, m2, n2, m3, n3, m0, n0, m1, n1, m2, n2, m3, + n3); + LSX_TRANSPOSE8x8_H(m4, n4, m5, n5, m6, n6, m7, n7, m4, n4, m5, n5, m6, n6, m7, + n7); + + __lsx_vst(m0, tmp_buf, 0); + __lsx_vst(n0, tmp_buf, 16); + __lsx_vst(m1, tmp_buf, 32); + __lsx_vst(n1, tmp_buf, 48); + __lsx_vst(m2, tmp_buf, 64); + __lsx_vst(n2, tmp_buf, 80); + __lsx_vst(m3, tmp_buf, 96); + __lsx_vst(n3, tmp_buf, 112); + __lsx_vst(m4, tmp_buf, 128); + __lsx_vst(n4, tmp_buf, 144); + __lsx_vst(m5, tmp_buf, 160); + __lsx_vst(n5, tmp_buf, 176); + __lsx_vst(m6, tmp_buf, 192); + __lsx_vst(n6, tmp_buf, 208); + __lsx_vst(m7, tmp_buf, 224); + __lsx_vst(n7, tmp_buf, 240); + + /* 3rd & 4th 8x8 */ + DUP4_ARG2(__lsx_vld, input, 32, input, 96, input, 160, input, 224, m0, n0, m1, + n1); + DUP4_ARG2(__lsx_vld, input, 288, input, 352, input, 416, input, 480, m2, n2, + m3, n3); + DUP4_ARG2(__lsx_vld, input, 48, input, 112, input, 176, input, 240, m4, n4, + m5, n5); + DUP4_ARG2(__lsx_vld, input, 304, input, 368, input, 432, input, 496, m6, n6, + m7, n7); + + LSX_TRANSPOSE8x8_H(m0, n0, m1, n1, m2, n2, m3, n3, m0, n0, m1, n1, m2, n2, m3, + n3); + LSX_TRANSPOSE8x8_H(m4, n4, m5, n5, m6, n6, m7, n7, m4, n4, m5, n5, m6, n6, m7, + n7); + + __lsx_vst(m0, tmp_buf, 256); + __lsx_vst(n0, tmp_buf, 272); + __lsx_vst(m1, tmp_buf, 288); + __lsx_vst(n1, tmp_buf, 304); + __lsx_vst(m2, tmp_buf, 320); + __lsx_vst(n2, tmp_buf, 336); + __lsx_vst(m3, tmp_buf, 352); + __lsx_vst(n3, tmp_buf, 368); + __lsx_vst(m4, tmp_buf, 384); + __lsx_vst(n4, tmp_buf, 400); + __lsx_vst(m5, tmp_buf, 416); + __lsx_vst(n5, tmp_buf, 432); + __lsx_vst(m6, tmp_buf, 448); + __lsx_vst(n6, tmp_buf, 464); + __lsx_vst(m7, tmp_buf, 480); + __lsx_vst(n7, tmp_buf, 496); +} + +static void idct32x8_row_even_process_store(int16_t *tmp_buf, + int16_t *tmp_eve_buf) { + __m128i vec0, vec1, vec2, vec3, loc0, loc1, loc2, loc3; + __m128i reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7; + __m128i stp0, stp1, stp2, stp3, stp4, stp5, stp6, stp7; + __m128i tmp0; + + /* Even stage 1 */ + DUP4_ARG2(__lsx_vld, tmp_buf, 0, tmp_buf, 64, tmp_buf, 128, tmp_buf, 192, + reg0, reg1, reg2, reg3); + DUP4_ARG2(__lsx_vld, tmp_buf, 256, tmp_buf, 320, tmp_buf, 384, tmp_buf, 448, + reg4, reg5, reg6, reg7); + + DOTP_CONST_PAIR(reg1, reg7, cospi_28_64, cospi_4_64, reg1, reg7); + DOTP_CONST_PAIR(reg5, reg3, cospi_12_64, cospi_20_64, reg5, reg3); + LSX_BUTTERFLY_4_H(reg1, reg7, reg3, reg5, vec1, vec3, vec2, vec0); + DOTP_CONST_PAIR(vec2, vec0, cospi_16_64, cospi_16_64, loc2, loc3); + + loc1 = vec3; + loc0 = vec1; + + DOTP_CONST_PAIR(reg0, reg4, cospi_16_64, cospi_16_64, reg0, reg4); + DOTP_CONST_PAIR(reg2, reg6, cospi_24_64, cospi_8_64, reg2, reg6); + LSX_BUTTERFLY_4_H(reg4, reg0, reg2, reg6, vec1, vec3, vec2, vec0); + LSX_BUTTERFLY_4_H(vec0, vec1, loc1, loc0, stp3, stp0, stp7, stp4); + LSX_BUTTERFLY_4_H(vec2, vec3, loc3, loc2, stp2, stp1, stp6, stp5); + + /* Even stage 2 */ + DUP4_ARG2(__lsx_vld, tmp_buf, 32, tmp_buf, 96, tmp_buf, 160, tmp_buf, 224, + reg0, reg1, reg2, reg3); + DUP4_ARG2(__lsx_vld, tmp_buf, 288, tmp_buf, 352, tmp_buf, 416, tmp_buf, 480, + reg4, reg5, reg6, reg7); + DOTP_CONST_PAIR(reg0, reg7, cospi_30_64, cospi_2_64, reg0, reg7); + DOTP_CONST_PAIR(reg4, reg3, cospi_14_64, cospi_18_64, reg4, reg3); + DOTP_CONST_PAIR(reg2, reg5, cospi_22_64, cospi_10_64, reg2, reg5); + DOTP_CONST_PAIR(reg6, reg1, cospi_6_64, cospi_26_64, reg6, reg1); + + vec0 = __lsx_vadd_h(reg0, reg4); + reg0 = __lsx_vsub_h(reg0, reg4); + reg4 = __lsx_vadd_h(reg6, reg2); + reg6 = __lsx_vsub_h(reg6, reg2); + reg2 = __lsx_vadd_h(reg1, reg5); + reg1 = __lsx_vsub_h(reg1, reg5); + reg5 = __lsx_vadd_h(reg7, reg3); + reg7 = __lsx_vsub_h(reg7, reg3); + reg3 = vec0; + + vec1 = reg2; + reg2 = __lsx_vadd_h(reg3, reg4); + reg3 = __lsx_vsub_h(reg3, reg4); + reg4 = __lsx_vsub_h(reg5, vec1); + reg5 = __lsx_vadd_h(reg5, vec1); + + tmp0 = __lsx_vneg_h(reg6); + DOTP_CONST_PAIR(reg7, reg0, cospi_24_64, cospi_8_64, reg0, reg7); + DOTP_CONST_PAIR(tmp0, reg1, cospi_24_64, cospi_8_64, reg6, reg1); + + vec0 = __lsx_vsub_h(reg0, reg6); + reg0 = __lsx_vadd_h(reg0, reg6); + vec1 = __lsx_vsub_h(reg7, reg1); + reg7 = __lsx_vadd_h(reg7, reg1); + + DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, reg6, reg1); + DOTP_CONST_PAIR(reg4, reg3, cospi_16_64, cospi_16_64, reg3, reg4); + + /* Even stage 3 : Dependency on Even stage 1 & Even stage 2 */ + LSX_BUTTERFLY_4_H(stp0, stp1, reg7, reg5, loc1, loc3, loc2, loc0); + __lsx_vst(loc0, tmp_eve_buf, 240); + __lsx_vst(loc1, tmp_eve_buf, 0); + __lsx_vst(loc2, tmp_eve_buf, 224); + __lsx_vst(loc3, tmp_eve_buf, 16); + + LSX_BUTTERFLY_4_H(stp2, stp3, reg4, reg1, loc1, loc3, loc2, loc0); + __lsx_vst(loc0, tmp_eve_buf, 208); + __lsx_vst(loc1, tmp_eve_buf, 32); + __lsx_vst(loc2, tmp_eve_buf, 192); + __lsx_vst(loc3, tmp_eve_buf, 48); + + /* Store 8 */ + LSX_BUTTERFLY_4_H(stp4, stp5, reg6, reg3, loc1, loc3, loc2, loc0); + __lsx_vst(loc0, tmp_eve_buf, 176); + __lsx_vst(loc1, tmp_eve_buf, 64); + __lsx_vst(loc2, tmp_eve_buf, 160); + __lsx_vst(loc3, tmp_eve_buf, 80); + + LSX_BUTTERFLY_4_H(stp6, stp7, reg2, reg0, loc1, loc3, loc2, loc0); + __lsx_vst(loc0, tmp_eve_buf, 144); + __lsx_vst(loc1, tmp_eve_buf, 96); + __lsx_vst(loc2, tmp_eve_buf, 128); + __lsx_vst(loc3, tmp_eve_buf, 112); +} + +static void idct32x8_row_odd_process_store(int16_t *tmp_buf, + int16_t *tmp_odd_buf) { + __m128i vec0, vec1, vec2, vec3, loc0, loc1, loc2, loc3; + __m128i reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7; + + /* Odd stage 1 */ + DUP4_ARG2(__lsx_vld, tmp_buf, 16, tmp_buf, 112, tmp_buf, 144, tmp_buf, 240, + reg0, reg1, reg2, reg3); + DUP4_ARG2(__lsx_vld, tmp_buf, 272, tmp_buf, 368, tmp_buf, 400, tmp_buf, 496, + reg4, reg5, reg6, reg7); + + DOTP_CONST_PAIR(reg0, reg7, cospi_31_64, cospi_1_64, reg0, reg7); + DOTP_CONST_PAIR(reg4, reg3, cospi_15_64, cospi_17_64, reg3, reg4); + DOTP_CONST_PAIR(reg2, reg5, cospi_23_64, cospi_9_64, reg2, reg5); + DOTP_CONST_PAIR(reg6, reg1, cospi_7_64, cospi_25_64, reg1, reg6); + + vec0 = __lsx_vadd_h(reg0, reg3); + reg0 = __lsx_vsub_h(reg0, reg3); + reg3 = __lsx_vadd_h(reg7, reg4); + reg7 = __lsx_vsub_h(reg7, reg4); + reg4 = __lsx_vadd_h(reg1, reg2); + reg1 = __lsx_vsub_h(reg1, reg2); + reg2 = __lsx_vadd_h(reg6, reg5); + reg6 = __lsx_vsub_h(reg6, reg5); + reg5 = vec0; + + /* 4 Stores */ + DUP2_ARG2(__lsx_vadd_h, reg5, reg4, reg3, reg2, vec0, vec1); + __lsx_vst(vec0, tmp_odd_buf, 64); + __lsx_vst(vec1, tmp_odd_buf, 80); + + DUP2_ARG2(__lsx_vsub_h, reg5, reg4, reg3, reg2, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_24_64, cospi_8_64, vec0, vec1); + __lsx_vst(vec0, tmp_odd_buf, 0); + __lsx_vst(vec1, tmp_odd_buf, 16); + + /* 4 Stores */ + DOTP_CONST_PAIR(reg7, reg0, cospi_28_64, cospi_4_64, reg0, reg7); + DOTP_CONST_PAIR(reg6, reg1, -cospi_4_64, cospi_28_64, reg1, reg6); + LSX_BUTTERFLY_4_H(reg0, reg7, reg6, reg1, vec0, vec1, vec2, vec3); + __lsx_vst(vec0, tmp_odd_buf, 96); + __lsx_vst(vec1, tmp_odd_buf, 112); + + DOTP_CONST_PAIR(vec2, vec3, cospi_24_64, cospi_8_64, vec2, vec3); + __lsx_vst(vec2, tmp_odd_buf, 32); + __lsx_vst(vec3, tmp_odd_buf, 48); + + /* Odd stage 2 */ + /* 8 loads */ + DUP4_ARG2(__lsx_vld, tmp_buf, 48, tmp_buf, 80, tmp_buf, 176, tmp_buf, 208, + reg0, reg1, reg2, reg3); + DUP4_ARG2(__lsx_vld, tmp_buf, 304, tmp_buf, 336, tmp_buf, 432, tmp_buf, 464, + reg4, reg5, reg6, reg7); + + DOTP_CONST_PAIR(reg1, reg6, cospi_27_64, cospi_5_64, reg1, reg6); + DOTP_CONST_PAIR(reg5, reg2, cospi_11_64, cospi_21_64, reg2, reg5); + DOTP_CONST_PAIR(reg3, reg4, cospi_19_64, cospi_13_64, reg3, reg4); + DOTP_CONST_PAIR(reg7, reg0, cospi_3_64, cospi_29_64, reg0, reg7); + + /* 4 Stores */ + DUP4_ARG2(__lsx_vsub_h, reg1, reg2, reg6, reg5, reg0, reg3, reg7, reg4, vec0, + vec1, vec2, vec3); + DOTP_CONST_PAIR(vec1, vec0, cospi_12_64, cospi_20_64, loc0, loc1); + DOTP_CONST_PAIR(vec3, vec2, -cospi_20_64, cospi_12_64, loc2, loc3); + + LSX_BUTTERFLY_4_H(loc3, loc2, loc0, loc1, vec1, vec0, vec2, vec3); + __lsx_vst(vec0, tmp_odd_buf, 192); + __lsx_vst(vec1, tmp_odd_buf, 240); + + DOTP_CONST_PAIR(vec3, vec2, -cospi_8_64, cospi_24_64, vec0, vec1); + __lsx_vst(vec0, tmp_odd_buf, 160); + __lsx_vst(vec1, tmp_odd_buf, 176); + + /* 4 Stores */ + DUP4_ARG2(__lsx_vadd_h, reg1, reg2, reg6, reg5, reg0, reg3, reg7, reg4, vec1, + vec2, vec0, vec3); + LSX_BUTTERFLY_4_H(vec0, vec3, vec2, vec1, reg0, reg1, reg3, reg2); + __lsx_vst(reg0, tmp_odd_buf, 208); + __lsx_vst(reg1, tmp_odd_buf, 224); + + DOTP_CONST_PAIR(reg3, reg2, -cospi_8_64, cospi_24_64, reg0, reg1); + __lsx_vst(reg0, tmp_odd_buf, 128); + __lsx_vst(reg1, tmp_odd_buf, 144); + + /* Odd stage 3 : Dependency on Odd stage 1 & Odd stage 2 */ + + /* Load 8 & Store 8 */ + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 0, tmp_odd_buf, 16, tmp_odd_buf, 32, + tmp_odd_buf, 48, reg0, reg1, reg2, reg3); + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 128, tmp_odd_buf, 144, tmp_odd_buf, 160, + tmp_odd_buf, 176, reg4, reg5, reg6, reg7); + DUP4_ARG2(__lsx_vadd_h, reg0, reg4, reg1, reg5, reg2, reg6, reg3, reg7, loc0, + loc1, loc2, loc3); + __lsx_vst(loc0, tmp_odd_buf, 0); + __lsx_vst(loc1, tmp_odd_buf, 16); + __lsx_vst(loc2, tmp_odd_buf, 32); + __lsx_vst(loc3, tmp_odd_buf, 48); + + DUP2_ARG2(__lsx_vsub_h, reg0, reg4, reg1, reg5, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc0, loc1); + + DUP2_ARG2(__lsx_vsub_h, reg2, reg6, reg3, reg7, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc2, loc3); + __lsx_vst(loc0, tmp_odd_buf, 128); + __lsx_vst(loc1, tmp_odd_buf, 144); + __lsx_vst(loc2, tmp_odd_buf, 160); + __lsx_vst(loc3, tmp_odd_buf, 176); + + /* Load 8 & Store 8 */ + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 64, tmp_odd_buf, 80, tmp_odd_buf, 96, + tmp_odd_buf, 112, reg1, reg2, reg0, reg3); + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 192, tmp_odd_buf, 208, tmp_odd_buf, 224, + tmp_odd_buf, 240, reg4, reg5, reg6, reg7); + + DUP4_ARG2(__lsx_vadd_h, reg0, reg4, reg1, reg5, reg2, reg6, reg3, reg7, loc0, + loc1, loc2, loc3); + __lsx_vst(loc0, tmp_odd_buf, 64); + __lsx_vst(loc1, tmp_odd_buf, 80); + __lsx_vst(loc2, tmp_odd_buf, 96); + __lsx_vst(loc3, tmp_odd_buf, 112); + + DUP2_ARG2(__lsx_vsub_h, reg0, reg4, reg3, reg7, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc0, loc1); + DUP2_ARG2(__lsx_vsub_h, reg1, reg5, reg2, reg6, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc2, loc3); + __lsx_vst(loc0, tmp_odd_buf, 192); + __lsx_vst(loc1, tmp_odd_buf, 208); + __lsx_vst(loc2, tmp_odd_buf, 224); + __lsx_vst(loc3, tmp_odd_buf, 240); +} + +static void idct_butterfly_transpose_store(int16_t *tmp_buf, + int16_t *tmp_eve_buf, + int16_t *tmp_odd_buf, int16_t *dst) { + __m128i vec0, vec1, vec2, vec3, loc0, loc1, loc2, loc3; + __m128i m0, m1, m2, m3, m4, m5, m6, m7; + __m128i n0, n1, n2, n3, n4, n5, n6, n7; + __m128i reg0, reg1, reg2, reg3; + + /* FINAL BUTTERFLY : Dependency on Even & Odd */ + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 0, tmp_odd_buf, 144, tmp_odd_buf, 224, + tmp_odd_buf, 96, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vld, tmp_eve_buf, 0, tmp_eve_buf, 128, tmp_eve_buf, 64, + tmp_eve_buf, 192, loc0, loc1, loc2, loc3); + + DUP4_ARG2(__lsx_vadd_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, m0, + m4, m2, m6); + DUP4_ARG2(__lsx_vsub_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, reg0, + reg1, reg2, reg3); + __lsx_vst(reg0, tmp_buf, 496); + __lsx_vst(reg1, tmp_buf, 368); + __lsx_vst(reg2, tmp_buf, 432); + __lsx_vst(reg3, tmp_buf, 304); + + /* Load 8 & Store 8 */ + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 64, tmp_odd_buf, 208, tmp_odd_buf, 160, + tmp_odd_buf, 48, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vld, tmp_eve_buf, 32, tmp_eve_buf, 160, tmp_eve_buf, 96, + tmp_eve_buf, 224, loc0, loc1, loc2, loc3); + + DUP4_ARG2(__lsx_vadd_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, m1, + m5, m3, m7); + DUP4_ARG2(__lsx_vsub_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, reg0, + reg1, reg2, reg3); + __lsx_vst(reg0, tmp_buf, 464); + __lsx_vst(reg1, tmp_buf, 336); + __lsx_vst(reg2, tmp_buf, 400); + __lsx_vst(reg3, tmp_buf, 272); + + /* Load 8 & Store 8 */ + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 32, tmp_odd_buf, 176, tmp_odd_buf, 192, + tmp_odd_buf, 112, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vld, tmp_eve_buf, 16, tmp_eve_buf, 144, tmp_eve_buf, 80, + tmp_eve_buf, 208, loc0, loc1, loc2, loc3); + + DUP4_ARG2(__lsx_vadd_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, n0, + n4, n2, n6); + DUP4_ARG2(__lsx_vsub_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, reg0, + reg1, reg2, reg3); + __lsx_vst(reg0, tmp_buf, 480); + __lsx_vst(reg1, tmp_buf, 352); + __lsx_vst(reg2, tmp_buf, 416); + __lsx_vst(reg3, tmp_buf, 288); + + /* Load 8 & Store 8 */ + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 80, tmp_odd_buf, 240, tmp_odd_buf, 128, + tmp_odd_buf, 16, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vld, tmp_eve_buf, 48, tmp_eve_buf, 176, tmp_eve_buf, 112, + tmp_eve_buf, 240, loc0, loc1, loc2, loc3); + DUP4_ARG2(__lsx_vadd_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, n1, + n5, n3, n7); + DUP4_ARG2(__lsx_vsub_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, reg0, + reg1, reg2, reg3); + __lsx_vst(reg0, tmp_buf, 448); + __lsx_vst(reg1, tmp_buf, 320); + __lsx_vst(reg2, tmp_buf, 384); + __lsx_vst(reg3, tmp_buf, 256); + + /* Transpose : 16 vectors */ + /* 1st & 2nd 8x8 */ + LSX_TRANSPOSE8x8_H(m0, n0, m1, n1, m2, n2, m3, n3, m0, n0, m1, n1, m2, n2, m3, + n3); + __lsx_vst(m0, dst, 0); + __lsx_vst(n0, dst, 64); + __lsx_vst(m1, dst, 128); + __lsx_vst(n1, dst, 192); + __lsx_vst(m2, dst, 256); + __lsx_vst(n2, dst, 320); + __lsx_vst(m3, dst, 384); + __lsx_vst(n3, dst, 448); + + LSX_TRANSPOSE8x8_H(m4, n4, m5, n5, m6, n6, m7, n7, m4, n4, m5, n5, m6, n6, m7, + n7); + __lsx_vst(m4, dst, 16); + __lsx_vst(n4, dst, 80); + __lsx_vst(m5, dst, 144); + __lsx_vst(n5, dst, 208); + __lsx_vst(m6, dst, 272); + __lsx_vst(n6, dst, 336); + __lsx_vst(m7, dst, 400); + __lsx_vst(n7, dst, 464); + + /* 3rd & 4th 8x8 */ + DUP4_ARG2(__lsx_vld, tmp_buf, 256, tmp_buf, 272, tmp_buf, 288, tmp_buf, 304, + m0, n0, m1, n1); + DUP4_ARG2(__lsx_vld, tmp_buf, 320, tmp_buf, 336, tmp_buf, 352, tmp_buf, 368, + m2, n2, m3, n3); + DUP4_ARG2(__lsx_vld, tmp_buf, 384, tmp_buf, 400, tmp_buf, 416, tmp_buf, 432, + m4, n4, m5, n5); + DUP4_ARG2(__lsx_vld, tmp_buf, 448, tmp_buf, 464, tmp_buf, 480, tmp_buf, 496, + m6, n6, m7, n7); + LSX_TRANSPOSE8x8_H(m0, n0, m1, n1, m2, n2, m3, n3, m0, n0, m1, n1, m2, n2, m3, + n3); + LSX_TRANSPOSE8x8_H(m4, n4, m5, n5, m6, n6, m7, n7, m4, n4, m5, n5, m6, n6, m7, + n7); + __lsx_vst(m0, dst, 32); + __lsx_vst(n0, dst, 96); + __lsx_vst(m1, dst, 160); + __lsx_vst(n1, dst, 224); + __lsx_vst(m2, dst, 288); + __lsx_vst(n2, dst, 352); + __lsx_vst(m3, dst, 416); + __lsx_vst(n3, dst, 480); + __lsx_vst(m4, dst, 48); + __lsx_vst(n4, dst, 112); + __lsx_vst(m5, dst, 176); + __lsx_vst(n5, dst, 240); + __lsx_vst(m6, dst, 304); + __lsx_vst(n6, dst, 368); + __lsx_vst(m7, dst, 432); + __lsx_vst(n7, dst, 496); +} + +static void idct32x8_1d_rows_lsx(const int16_t *input, int16_t *output) { + DECLARE_ALIGNED(32, int16_t, tmp_buf[8 * 32]); + DECLARE_ALIGNED(32, int16_t, tmp_odd_buf[16 * 8]); + DECLARE_ALIGNED(32, int16_t, tmp_eve_buf[16 * 8]); + + idct32x8_row_transpose_store(input, &tmp_buf[0]); + idct32x8_row_even_process_store(&tmp_buf[0], &tmp_eve_buf[0]); + idct32x8_row_odd_process_store(&tmp_buf[0], &tmp_odd_buf[0]); + idct_butterfly_transpose_store(&tmp_buf[0], &tmp_eve_buf[0], &tmp_odd_buf[0], + output); +} + +static void idct8x32_column_even_process_store(int16_t *tmp_buf, + int16_t *tmp_eve_buf) { + __m128i vec0, vec1, vec2, vec3, loc0, loc1, loc2, loc3; + __m128i reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7; + __m128i stp0, stp1, stp2, stp3, stp4, stp5, stp6, stp7; + __m128i tmp0; + + /* Even stage 1 */ + DUP4_ARG2(__lsx_vld, tmp_buf, 0, tmp_buf, 256, tmp_buf, 512, tmp_buf, 768, + reg0, reg1, reg2, reg3); + DUP4_ARG2(__lsx_vld, tmp_buf, 1024, tmp_buf, 1280, tmp_buf, 1536, tmp_buf, + 1792, reg4, reg5, reg6, reg7); + tmp_buf += 64; + + DOTP_CONST_PAIR(reg1, reg7, cospi_28_64, cospi_4_64, reg1, reg7); + DOTP_CONST_PAIR(reg5, reg3, cospi_12_64, cospi_20_64, reg5, reg3); + LSX_BUTTERFLY_4_H(reg1, reg7, reg3, reg5, vec1, vec3, vec2, vec0); + DOTP_CONST_PAIR(vec2, vec0, cospi_16_64, cospi_16_64, loc2, loc3); + + loc1 = vec3; + loc0 = vec1; + + DOTP_CONST_PAIR(reg0, reg4, cospi_16_64, cospi_16_64, reg0, reg4); + DOTP_CONST_PAIR(reg2, reg6, cospi_24_64, cospi_8_64, reg2, reg6); + LSX_BUTTERFLY_4_H(reg4, reg0, reg2, reg6, vec1, vec3, vec2, vec0); + LSX_BUTTERFLY_4_H(vec0, vec1, loc1, loc0, stp3, stp0, stp7, stp4); + LSX_BUTTERFLY_4_H(vec2, vec3, loc3, loc2, stp2, stp1, stp6, stp5); + + /* Even stage 2 */ + /* Load 8 */ + DUP4_ARG2(__lsx_vld, tmp_buf, 0, tmp_buf, 256, tmp_buf, 512, tmp_buf, 768, + reg0, reg1, reg2, reg3); + DUP4_ARG2(__lsx_vld, tmp_buf, 1024, tmp_buf, 1280, tmp_buf, 1536, tmp_buf, + 1792, reg4, reg5, reg6, reg7); + DOTP_CONST_PAIR(reg0, reg7, cospi_30_64, cospi_2_64, reg0, reg7); + DOTP_CONST_PAIR(reg4, reg3, cospi_14_64, cospi_18_64, reg4, reg3); + DOTP_CONST_PAIR(reg2, reg5, cospi_22_64, cospi_10_64, reg2, reg5); + DOTP_CONST_PAIR(reg6, reg1, cospi_6_64, cospi_26_64, reg6, reg1); + + vec0 = __lsx_vadd_h(reg0, reg4); + reg0 = __lsx_vsub_h(reg0, reg4); + reg4 = __lsx_vadd_h(reg6, reg2); + reg6 = __lsx_vsub_h(reg6, reg2); + reg2 = __lsx_vadd_h(reg1, reg5); + reg1 = __lsx_vsub_h(reg1, reg5); + reg5 = __lsx_vadd_h(reg7, reg3); + reg7 = __lsx_vsub_h(reg7, reg3); + reg3 = vec0; + + vec1 = reg2; + reg2 = __lsx_vadd_h(reg3, reg4); + reg3 = __lsx_vsub_h(reg3, reg4); + reg4 = __lsx_vsub_h(reg5, vec1); + reg5 = __lsx_vadd_h(reg5, vec1); + + tmp0 = __lsx_vneg_h(reg6); + DOTP_CONST_PAIR(reg7, reg0, cospi_24_64, cospi_8_64, reg0, reg7); + DOTP_CONST_PAIR(tmp0, reg1, cospi_24_64, cospi_8_64, reg6, reg1); + + vec0 = __lsx_vsub_h(reg0, reg6); + reg0 = __lsx_vadd_h(reg0, reg6); + vec1 = __lsx_vsub_h(reg7, reg1); + reg7 = __lsx_vadd_h(reg7, reg1); + + DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, reg6, reg1); + DOTP_CONST_PAIR(reg4, reg3, cospi_16_64, cospi_16_64, reg3, reg4); + + /* Even stage 3 : Dependency on Even stage 1 & Even stage 2 */ + /* Store 8 */ + LSX_BUTTERFLY_4_H(stp0, stp1, reg7, reg5, loc1, loc3, loc2, loc0); + __lsx_vst(loc1, tmp_eve_buf, 0); + __lsx_vst(loc3, tmp_eve_buf, 16); + __lsx_vst(loc2, tmp_eve_buf, 224); + __lsx_vst(loc0, tmp_eve_buf, 240); + + LSX_BUTTERFLY_4_H(stp2, stp3, reg4, reg1, loc1, loc3, loc2, loc0); + __lsx_vst(loc1, tmp_eve_buf, 32); + __lsx_vst(loc3, tmp_eve_buf, 48); + __lsx_vst(loc2, tmp_eve_buf, 192); + __lsx_vst(loc0, tmp_eve_buf, 208); + + /* Store 8 */ + LSX_BUTTERFLY_4_H(stp4, stp5, reg6, reg3, loc1, loc3, loc2, loc0); + __lsx_vst(loc1, tmp_eve_buf, 64); + __lsx_vst(loc3, tmp_eve_buf, 80); + __lsx_vst(loc2, tmp_eve_buf, 160); + __lsx_vst(loc0, tmp_eve_buf, 176); + + LSX_BUTTERFLY_4_H(stp6, stp7, reg2, reg0, loc1, loc3, loc2, loc0); + __lsx_vst(loc1, tmp_eve_buf, 96); + __lsx_vst(loc3, tmp_eve_buf, 112); + __lsx_vst(loc2, tmp_eve_buf, 128); + __lsx_vst(loc0, tmp_eve_buf, 144); +} + +static void idct8x32_column_odd_process_store(int16_t *tmp_buf, + int16_t *tmp_odd_buf) { + __m128i vec0, vec1, vec2, vec3, loc0, loc1, loc2, loc3; + __m128i reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7; + + /* Odd stage 1 */ + DUP4_ARG2(__lsx_vld, tmp_buf, 64, tmp_buf, 448, tmp_buf, 576, tmp_buf, 960, + reg0, reg1, reg2, reg3); + DUP4_ARG2(__lsx_vld, tmp_buf, 1088, tmp_buf, 1472, tmp_buf, 1600, tmp_buf, + 1984, reg4, reg5, reg6, reg7); + + DOTP_CONST_PAIR(reg0, reg7, cospi_31_64, cospi_1_64, reg0, reg7); + DOTP_CONST_PAIR(reg4, reg3, cospi_15_64, cospi_17_64, reg3, reg4); + DOTP_CONST_PAIR(reg2, reg5, cospi_23_64, cospi_9_64, reg2, reg5); + DOTP_CONST_PAIR(reg6, reg1, cospi_7_64, cospi_25_64, reg1, reg6); + + vec0 = __lsx_vadd_h(reg0, reg3); + reg0 = __lsx_vsub_h(reg0, reg3); + reg3 = __lsx_vadd_h(reg7, reg4); + reg7 = __lsx_vsub_h(reg7, reg4); + reg4 = __lsx_vadd_h(reg1, reg2); + reg1 = __lsx_vsub_h(reg1, reg2); + reg2 = __lsx_vadd_h(reg6, reg5); + reg6 = __lsx_vsub_h(reg6, reg5); + reg5 = vec0; + + /* 4 Stores */ + DUP2_ARG2(__lsx_vadd_h, reg5, reg4, reg3, reg2, vec0, vec1); + __lsx_vst(vec0, tmp_odd_buf, 64); + __lsx_vst(vec1, tmp_odd_buf, 80); + DUP2_ARG2(__lsx_vsub_h, reg5, reg4, reg3, reg2, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_24_64, cospi_8_64, vec0, vec1); + __lsx_vst(vec0, tmp_odd_buf, 0); + __lsx_vst(vec1, tmp_odd_buf, 16); + + /* 4 Stores */ + DOTP_CONST_PAIR(reg7, reg0, cospi_28_64, cospi_4_64, reg0, reg7); + DOTP_CONST_PAIR(reg6, reg1, -cospi_4_64, cospi_28_64, reg1, reg6); + LSX_BUTTERFLY_4_H(reg0, reg7, reg6, reg1, vec0, vec1, vec2, vec3); + DOTP_CONST_PAIR(vec2, vec3, cospi_24_64, cospi_8_64, vec2, vec3); + __lsx_vst(vec0, tmp_odd_buf, 96); + __lsx_vst(vec1, tmp_odd_buf, 112); + __lsx_vst(vec2, tmp_odd_buf, 32); + __lsx_vst(vec3, tmp_odd_buf, 48); + + /* Odd stage 2 */ + /* 8 loads */ + DUP4_ARG2(__lsx_vld, tmp_buf, 192, tmp_buf, 320, tmp_buf, 704, tmp_buf, 832, + reg0, reg1, reg2, reg3); + DUP4_ARG2(__lsx_vld, tmp_buf, 1216, tmp_buf, 1344, tmp_buf, 1728, tmp_buf, + 1856, reg4, reg5, reg6, reg7); + DOTP_CONST_PAIR(reg1, reg6, cospi_27_64, cospi_5_64, reg1, reg6); + DOTP_CONST_PAIR(reg5, reg2, cospi_11_64, cospi_21_64, reg2, reg5); + DOTP_CONST_PAIR(reg3, reg4, cospi_19_64, cospi_13_64, reg3, reg4); + DOTP_CONST_PAIR(reg7, reg0, cospi_3_64, cospi_29_64, reg0, reg7); + + /* 4 Stores */ + DUP4_ARG2(__lsx_vsub_h, reg1, reg2, reg6, reg5, reg0, reg3, reg7, reg4, vec0, + vec1, vec2, vec3); + DOTP_CONST_PAIR(vec1, vec0, cospi_12_64, cospi_20_64, loc0, loc1); + DOTP_CONST_PAIR(vec3, vec2, -cospi_20_64, cospi_12_64, loc2, loc3); + LSX_BUTTERFLY_4_H(loc2, loc3, loc1, loc0, vec0, vec1, vec3, vec2); + __lsx_vst(vec0, tmp_odd_buf, 192); + __lsx_vst(vec1, tmp_odd_buf, 240); + DOTP_CONST_PAIR(vec3, vec2, -cospi_8_64, cospi_24_64, vec0, vec1); + __lsx_vst(vec0, tmp_odd_buf, 160); + __lsx_vst(vec1, tmp_odd_buf, 176); + + /* 4 Stores */ + DUP4_ARG2(__lsx_vadd_h, reg0, reg3, reg1, reg2, reg5, reg6, reg4, reg7, vec0, + vec1, vec2, vec3); + LSX_BUTTERFLY_4_H(vec0, vec3, vec2, vec1, reg0, reg1, reg3, reg2); + __lsx_vst(reg0, tmp_odd_buf, 208); + __lsx_vst(reg1, tmp_odd_buf, 224); + DOTP_CONST_PAIR(reg3, reg2, -cospi_8_64, cospi_24_64, reg0, reg1); + __lsx_vst(reg0, tmp_odd_buf, 128); + __lsx_vst(reg1, tmp_odd_buf, 144); + + /* Odd stage 3 : Dependency on Odd stage 1 & Odd stage 2 */ + /* Load 8 & Store 8 */ + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 0, tmp_odd_buf, 16, tmp_odd_buf, 32, + tmp_odd_buf, 48, reg0, reg1, reg2, reg3); + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 128, tmp_odd_buf, 144, tmp_odd_buf, 160, + tmp_odd_buf, 176, reg4, reg5, reg6, reg7); + DUP4_ARG2(__lsx_vadd_h, reg0, reg4, reg1, reg5, reg2, reg6, reg3, reg7, loc0, + loc1, loc2, loc3); + __lsx_vst(loc0, tmp_odd_buf, 0); + __lsx_vst(loc1, tmp_odd_buf, 16); + __lsx_vst(loc2, tmp_odd_buf, 32); + __lsx_vst(loc3, tmp_odd_buf, 48); + + DUP2_ARG2(__lsx_vsub_h, reg0, reg4, reg1, reg5, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc0, loc1); + DUP2_ARG2(__lsx_vsub_h, reg2, reg6, reg3, reg7, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc2, loc3); + __lsx_vst(loc0, tmp_odd_buf, 128); + __lsx_vst(loc1, tmp_odd_buf, 144); + __lsx_vst(loc2, tmp_odd_buf, 160); + __lsx_vst(loc3, tmp_odd_buf, 176); + + /* Load 8 & Store 8 */ + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 64, tmp_odd_buf, 80, tmp_odd_buf, 96, + tmp_odd_buf, 112, reg1, reg2, reg0, reg3); + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 192, tmp_odd_buf, 208, tmp_odd_buf, 224, + tmp_odd_buf, 240, reg4, reg5, reg6, reg7); + DUP4_ARG2(__lsx_vadd_h, reg0, reg4, reg1, reg5, reg2, reg6, reg3, reg7, loc0, + loc1, loc2, loc3); + __lsx_vst(loc0, tmp_odd_buf, 64); + __lsx_vst(loc1, tmp_odd_buf, 80); + __lsx_vst(loc2, tmp_odd_buf, 96); + __lsx_vst(loc3, tmp_odd_buf, 112); + + DUP2_ARG2(__lsx_vsub_h, reg0, reg4, reg3, reg7, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc0, loc1); + DUP2_ARG2(__lsx_vsub_h, reg1, reg5, reg2, reg6, vec0, vec1); + DOTP_CONST_PAIR(vec1, vec0, cospi_16_64, cospi_16_64, loc2, loc3); + __lsx_vst(loc0, tmp_odd_buf, 192); + __lsx_vst(loc1, tmp_odd_buf, 208); + __lsx_vst(loc2, tmp_odd_buf, 224); + __lsx_vst(loc3, tmp_odd_buf, 240); +} + +static void idct8x32_column_butterfly_addblk(int16_t *tmp_eve_buf, + int16_t *tmp_odd_buf, uint8_t *dst, + int32_t dst_stride) { + __m128i vec0, vec1, vec2, vec3, loc0, loc1, loc2, loc3; + __m128i m0, m1, m2, m3, m4, m5, m6, m7; + __m128i n0, n1, n2, n3, n4, n5, n6, n7; + int32_t stride = dst_stride << 2; + int32_t stride2 = stride << 1; + int32_t stride3 = stride + stride2; + + /* FINAL BUTTERFLY : Dependency on Even & Odd */ + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 0, tmp_odd_buf, 144, tmp_odd_buf, 224, + tmp_odd_buf, 96, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vld, tmp_eve_buf, 0, tmp_eve_buf, 128, tmp_eve_buf, 64, + tmp_eve_buf, 192, loc0, loc1, loc2, loc3); + + DUP4_ARG2(__lsx_vadd_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, m0, + m4, m2, m6); + DUP4_ARG2(__lsx_vsrari_h, m0, 6, m2, 6, m4, 6, m6, 6, m0, m2, m4, m6); + VP9_ADDBLK_ST8x4_UB(dst, stride, stride2, stride3, m0, m2, m4, m6); + DUP4_ARG2(__lsx_vsub_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, m6, + m2, m4, m0); + DUP4_ARG2(__lsx_vsrari_h, m0, 6, m2, 6, m4, 6, m6, 6, m0, m2, m4, m6); + VP9_ADDBLK_ST8x4_UB((dst + 19 * dst_stride), stride, stride2, stride3, m0, m2, + m4, m6); + + /* Load 8 & Store 8 */ + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 64, tmp_odd_buf, 208, tmp_odd_buf, 160, + tmp_odd_buf, 48, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vld, tmp_eve_buf, 32, tmp_eve_buf, 160, tmp_eve_buf, 96, + tmp_eve_buf, 224, loc0, loc1, loc2, loc3); + + DUP4_ARG2(__lsx_vadd_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, m1, + m5, m3, m7); + DUP4_ARG2(__lsx_vsrari_h, m1, 6, m3, 6, m5, 6, m7, 6, m1, m3, m5, m7); + VP9_ADDBLK_ST8x4_UB((dst + 2 * dst_stride), stride, stride2, stride3, m1, m3, + m5, m7); + DUP4_ARG2(__lsx_vsub_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, m7, + m3, m5, m1); + DUP4_ARG2(__lsx_vsrari_h, m1, 6, m3, 6, m5, 6, m7, 6, m1, m3, m5, m7); + VP9_ADDBLK_ST8x4_UB((dst + 17 * dst_stride), stride, stride2, stride3, m1, m3, + m5, m7); + + /* Load 8 & Store 8 */ + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 32, tmp_odd_buf, 176, tmp_odd_buf, 192, + tmp_odd_buf, 112, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vld, tmp_eve_buf, 16, tmp_eve_buf, 144, tmp_eve_buf, 80, + tmp_eve_buf, 208, loc0, loc1, loc2, loc3); + DUP4_ARG2(__lsx_vadd_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, n0, + n4, n2, n6); + DUP4_ARG2(__lsx_vsrari_h, n0, 6, n2, 6, n4, 6, n6, 6, n0, n2, n4, n6); + VP9_ADDBLK_ST8x4_UB((dst + dst_stride), stride, stride2, stride3, n0, n2, n4, + n6); + DUP4_ARG2(__lsx_vsub_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, n6, + n2, n4, n0); + DUP4_ARG2(__lsx_vsrari_h, n0, 6, n2, 6, n4, 6, n6, 6, n0, n2, n4, n6); + VP9_ADDBLK_ST8x4_UB((dst + 18 * dst_stride), stride, stride2, stride3, n0, n2, + n4, n6); + + /* Load 8 & Store 8 */ + DUP4_ARG2(__lsx_vld, tmp_odd_buf, 80, tmp_odd_buf, 240, tmp_odd_buf, 128, + tmp_odd_buf, 16, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vld, tmp_eve_buf, 48, tmp_eve_buf, 176, tmp_eve_buf, 112, + tmp_eve_buf, 240, loc0, loc1, loc2, loc3); + DUP4_ARG2(__lsx_vadd_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, n1, + n5, n3, n7); + DUP4_ARG2(__lsx_vsrari_h, n1, 6, n3, 6, n5, 6, n7, 6, n1, n3, n5, n7); + VP9_ADDBLK_ST8x4_UB((dst + 3 * dst_stride), stride, stride2, stride3, n1, n3, + n5, n7); + DUP4_ARG2(__lsx_vsub_h, loc0, vec3, loc1, vec2, loc2, vec1, loc3, vec0, n7, + n3, n5, n1); + DUP4_ARG2(__lsx_vsrari_h, n1, 6, n3, 6, n5, 6, n7, 6, n1, n3, n5, n7); + VP9_ADDBLK_ST8x4_UB((dst + 16 * dst_stride), stride, stride2, stride3, n1, n3, + n5, n7); +} + +static void idct8x32_1d_columns_addblk_lsx(int16_t *input, uint8_t *dst, + int32_t dst_stride) { + DECLARE_ALIGNED(32, int16_t, tmp_odd_buf[16 * 8]); + DECLARE_ALIGNED(32, int16_t, tmp_eve_buf[16 * 8]); + + idct8x32_column_even_process_store(input, &tmp_eve_buf[0]); + idct8x32_column_odd_process_store(input, &tmp_odd_buf[0]); + idct8x32_column_butterfly_addblk(&tmp_eve_buf[0], &tmp_odd_buf[0], dst, + dst_stride); +} + +void vpx_idct32x32_1024_add_lsx(const int16_t *input, uint8_t *dst, + int32_t dst_stride) { + int32_t i; + DECLARE_ALIGNED(32, int16_t, out_arr[32 * 32]); + int16_t *out_ptr = out_arr; + + /* transform rows */ + for (i = 0; i < 4; ++i) { + /* process 32 * 8 block */ + idct32x8_1d_rows_lsx((input + (i << 8)), (out_ptr + (i << 8))); + } + + for (i = 0; i < 4; ++i) { + /* process 8 * 32 block */ + idct8x32_1d_columns_addblk_lsx((out_ptr + (i << 3)), (dst + (i << 3)), + dst_stride); + } +} + +void vpx_idct32x32_34_add_lsx(const int16_t *input, uint8_t *dst, + int32_t dst_stride) { + int32_t i; + DECLARE_ALIGNED(32, int16_t, out_arr[32 * 32]); + int16_t *out_ptr = out_arr; + __m128i zero = __lsx_vldi(0); + + for (i = 32; i--;) { + __lsx_vst(zero, out_ptr, 0); + __lsx_vst(zero, out_ptr, 16); + __lsx_vst(zero, out_ptr, 32); + __lsx_vst(zero, out_ptr, 48); + out_ptr += 32; + } + + out_ptr = out_arr; + + /* rows: only upper-left 8x8 has non-zero coeff */ + idct32x8_1d_rows_lsx(input, out_ptr); + + /* transform columns */ + for (i = 0; i < 4; ++i) { + /* process 8 * 32 block */ + idct8x32_1d_columns_addblk_lsx((out_ptr + (i << 3)), (dst + (i << 3)), + dst_stride); + } +} + +void vpx_idct32x32_1_add_lsx(const int16_t *input, uint8_t *dst, + int32_t dst_stride) { + int32_t i; + int16_t out; + __m128i dst0, dst1, dst2, dst3, tmp0, tmp1, tmp2, tmp3; + __m128i res0, res1, res2, res3, res4, res5, res6, res7, vec; + + out = ROUND_POWER_OF_TWO((input[0] * cospi_16_64), DCT_CONST_BITS); + out = ROUND_POWER_OF_TWO((out * cospi_16_64), DCT_CONST_BITS); + out = ROUND_POWER_OF_TWO(out, 6); + + vec = __lsx_vreplgr2vr_h(out); + + for (i = 16; i--;) { + DUP2_ARG2(__lsx_vld, dst, 0, dst, 16, dst0, dst1); + dst2 = __lsx_vldx(dst, dst_stride); + dst3 = __lsx_vldx(dst + 16, dst_stride); + + UNPCK_UB_SH(dst0, res0, res4); + UNPCK_UB_SH(dst1, res1, res5); + UNPCK_UB_SH(dst2, res2, res6); + UNPCK_UB_SH(dst3, res3, res7); + + DUP4_ARG2(__lsx_vadd_h, res0, vec, res1, vec, res2, vec, res3, vec, res0, + res1, res2, res3); + DUP4_ARG2(__lsx_vadd_h, res4, vec, res5, vec, res6, vec, res7, vec, res4, + res5, res6, res7); + DUP4_ARG3(__lsx_vssrarni_bu_h, res4, res0, 0, res5, res1, 0, res6, res2, 0, + res7, res3, 0, tmp0, tmp1, tmp2, tmp3); + __lsx_vst(tmp0, dst, 0); + __lsx_vst(tmp1, dst, 16); + dst += dst_stride; + __lsx_vst(tmp2, dst, 0); + __lsx_vst(tmp3, dst, 16); + dst += dst_stride; + } +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/intrapred_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/intrapred_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/intrapred_lsx.c @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Lu Wang + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "./vpx_dsp_rtcd.h" +#include "vpx_util/loongson_intrinsics.h" + +static inline void intra_predict_dc_8x8_lsx(const uint8_t *src_top, + const uint8_t *src_left, + uint8_t *dst, int32_t dst_stride) { + uint64_t val0, val1; + int32_t dst_stride_x2 = dst_stride << 1; + int32_t dst_stride_x3 = dst_stride_x2 + dst_stride; + int32_t dst_stride_x4 = dst_stride << 2; + __m128i store, sum_h, sum_w, sum_d; + __m128i src = { 0 }; + + val0 = *(const uint64_t *)src_top; + val1 = *(const uint64_t *)src_left; + DUP2_ARG3(__lsx_vinsgr2vr_d, src, val0, 0, src, val1, 1, src, src); + sum_h = __lsx_vhaddw_hu_bu(src, src); + sum_w = __lsx_vhaddw_wu_hu(sum_h, sum_h); + sum_d = __lsx_vhaddw_du_wu(sum_w, sum_w); + sum_w = __lsx_vpickev_w(sum_d, sum_d); + sum_d = __lsx_vhaddw_du_wu(sum_w, sum_w); + sum_w = __lsx_vsrari_w(sum_d, 4); + store = __lsx_vreplvei_b(sum_w, 0); + + __lsx_vstelm_d(store, dst, 0, 0); + __lsx_vstelm_d(store, dst + dst_stride, 0, 0); + __lsx_vstelm_d(store, dst + dst_stride_x2, 0, 0); + __lsx_vstelm_d(store, dst + dst_stride_x3, 0, 0); + dst += dst_stride_x4; + __lsx_vstelm_d(store, dst, 0, 0); + __lsx_vstelm_d(store, dst + dst_stride, 0, 0); + __lsx_vstelm_d(store, dst + dst_stride_x2, 0, 0); + __lsx_vstelm_d(store, dst + dst_stride_x3, 0, 0); +} + +static inline void intra_predict_dc_16x16_lsx(const uint8_t *src_top, + const uint8_t *src_left, + uint8_t *dst, + int32_t dst_stride) { + int32_t dst_stride_x2 = dst_stride << 1; + int32_t dst_stride_x3 = dst_stride_x2 + dst_stride; + int32_t dst_stride_x4 = dst_stride << 2; + __m128i top, left, out; + __m128i sum_h, sum_top, sum_left; + __m128i sum_w; + __m128i sum_d; + + DUP2_ARG2(__lsx_vld, src_top, 0, src_left, 0, top, left); + DUP2_ARG2(__lsx_vhaddw_hu_bu, top, top, left, left, sum_top, sum_left); + sum_h = __lsx_vadd_h(sum_top, sum_left); + sum_w = __lsx_vhaddw_wu_hu(sum_h, sum_h); + sum_d = __lsx_vhaddw_du_wu(sum_w, sum_w); + sum_w = __lsx_vpickev_w(sum_d, sum_d); + sum_d = __lsx_vhaddw_du_wu(sum_w, sum_w); + sum_w = __lsx_vsrari_w(sum_d, 5); + out = __lsx_vreplvei_b(sum_w, 0); + + __lsx_vstx(out, dst, 0); + __lsx_vstx(out, dst, dst_stride); + __lsx_vstx(out, dst, dst_stride_x2); + __lsx_vstx(out, dst, dst_stride_x3); + dst += dst_stride_x4; + __lsx_vstx(out, dst, 0); + __lsx_vstx(out, dst, dst_stride); + __lsx_vstx(out, dst, dst_stride_x2); + __lsx_vstx(out, dst, dst_stride_x3); + dst += dst_stride_x4; + __lsx_vstx(out, dst, 0); + __lsx_vstx(out, dst, dst_stride); + __lsx_vstx(out, dst, dst_stride_x2); + __lsx_vstx(out, dst, dst_stride_x3); + dst += dst_stride_x4; + __lsx_vstx(out, dst, 0); + __lsx_vstx(out, dst, dst_stride); + __lsx_vstx(out, dst, dst_stride_x2); + __lsx_vstx(out, dst, dst_stride_x3); +} + +void vpx_dc_predictor_8x8_lsx(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left) { + intra_predict_dc_8x8_lsx(above, left, dst, y_stride); +} + +void vpx_dc_predictor_16x16_lsx(uint8_t *dst, ptrdiff_t y_stride, + const uint8_t *above, const uint8_t *left) { + intra_predict_dc_16x16_lsx(above, left, dst, y_stride); +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/loopfilter_16_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/loopfilter_16_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/loopfilter_16_lsx.c @@ -0,0 +1,1330 @@ +/* + * Copyright (c) 2022 Loongson Technology Corporation Limited + * Contributed by Hecai Yuan + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "./vpx_dsp_rtcd.h" +#include "vpx_dsp/loongarch/loopfilter_lsx.h" +#include "vpx_ports/mem.h" + +#define LSX_LD_8(_src, _stride, _stride2, _stride3, _stride4, _in0, _in1, \ + _in2, _in3, _in4, _in5, _in6, _in7) \ + { \ + _in0 = __lsx_vld(_src, 0); \ + _in1 = __lsx_vldx(_src, _stride); \ + _in2 = __lsx_vldx(_src, _stride2); \ + _in3 = __lsx_vldx(_src, _stride3); \ + _src += _stride4; \ + _in4 = __lsx_vld(_src, 0); \ + _in5 = __lsx_vldx(_src, _stride); \ + _in6 = __lsx_vldx(_src, _stride2); \ + _in7 = __lsx_vldx(_src, _stride3); \ + } + +#define LSX_ST_8(_dst0, _dst1, _dst2, _dst3, _dst4, _dst5, _dst6, _dst7, _dst, \ + _stride, _stride2, _stride3, _stride4) \ + { \ + __lsx_vst(_dst0, _dst, 0); \ + __lsx_vstx(_dst1, _dst, _stride); \ + __lsx_vstx(_dst2, _dst, _stride2); \ + __lsx_vstx(_dst3, _dst, _stride3); \ + _dst += _stride4; \ + __lsx_vst(_dst4, _dst, 0); \ + __lsx_vstx(_dst5, _dst, _stride); \ + __lsx_vstx(_dst6, _dst, _stride2); \ + __lsx_vstx(_dst7, _dst, _stride3); \ + } + +static int32_t hz_lpf_t4_and_t8_16w(uint8_t *dst, int32_t stride, + uint8_t *filter48, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr) { + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i p2_out, p1_out, p0_out, q0_out, q1_out, q2_out; + __m128i flat, mask, hev, thresh, b_limit, limit; + __m128i p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l; + __m128i p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h; + __m128i p2_filt8_l, p1_filt8_l, p0_filt8_l; + __m128i q0_filt8_l, q1_filt8_l, q2_filt8_l; + __m128i p2_filt8_h, p1_filt8_h, p0_filt8_h; + __m128i q0_filt8_h, q1_filt8_h, q2_filt8_h; + __m128i zero = __lsx_vldi(0); + + int32_t stride2 = stride << 1; + int32_t stride3 = stride2 + stride; + int32_t stride4 = stride2 << 1; + + /* load vector elements */ + DUP4_ARG2(__lsx_vldx, dst, -stride4, dst, -stride3, dst, -stride2, dst, + -stride, p3, p2, p1, p0); + + q0 = __lsx_vld(dst, 0); + DUP2_ARG2(__lsx_vldx, dst, stride, dst, stride2, q1, q2); + q3 = __lsx_vldx(dst, stride3); + + thresh = __lsx_vreplgr2vr_b(*thresh_ptr); + b_limit = __lsx_vreplgr2vr_b(*b_limit_ptr); + limit = __lsx_vreplgr2vr_b(*limit_ptr); + /* mask and hev */ + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + VP9_FLAT4(p3, p2, p0, q0, q2, q3, flat); + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, q1_out); + + if (__lsx_bz_v(flat)) { + __lsx_vstx(p1_out, dst, -stride2); + __lsx_vstx(p0_out, dst, -stride); + __lsx_vst(q0_out, dst, 0); + __lsx_vstx(q1_out, dst, stride); + + return 1; + } + + DUP4_ARG2(__lsx_vilvl_b, zero, p3, zero, p2, zero, p1, zero, p0, p3_l, p2_l, + p1_l, p0_l); + DUP4_ARG2(__lsx_vilvl_b, zero, q0, zero, q1, zero, q2, zero, q3, q0_l, q1_l, + q2_l, q3_l); + + VP9_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l, q1_filt8_l, q2_filt8_l); + + DUP4_ARG2(__lsx_vilvh_b, zero, p3, zero, p2, zero, p1, zero, p0, p3_h, p2_h, + p1_h, p0_h); + DUP4_ARG2(__lsx_vilvh_b, zero, q0, zero, q1, zero, q2, zero, q3, q0_h, q1_h, + q2_h, q3_h); + VP9_FILTER8(p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h, p2_filt8_h, + p1_filt8_h, p0_filt8_h, q0_filt8_h, q1_filt8_h, q2_filt8_h); + + /* convert 16 bit output data into 8 bit */ + DUP4_ARG2(__lsx_vpickev_b, p2_filt8_h, p2_filt8_l, p1_filt8_h, p1_filt8_l, + p0_filt8_h, p0_filt8_l, q0_filt8_h, q0_filt8_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l); + DUP2_ARG2(__lsx_vpickev_b, q1_filt8_h, q1_filt8_l, q2_filt8_h, q2_filt8_l, + q1_filt8_l, q2_filt8_l); + + /* store pixel values */ + DUP4_ARG3(__lsx_vbitsel_v, p2, p2_filt8_l, flat, p1_out, p1_filt8_l, flat, + p0_out, p0_filt8_l, flat, q0_out, q0_filt8_l, flat, p2_out, p1_out, + p0_out, q0_out); + DUP2_ARG3(__lsx_vbitsel_v, q1_out, q1_filt8_l, flat, q2, q2_filt8_l, flat, + q1_out, q2_out); + + __lsx_vst(p2_out, filter48, 0); + __lsx_vst(p1_out, filter48, 16); + __lsx_vst(p0_out, filter48, 32); + __lsx_vst(q0_out, filter48, 48); + __lsx_vst(q1_out, filter48, 64); + __lsx_vst(q2_out, filter48, 80); + __lsx_vst(flat, filter48, 96); + + return 0; +} + +static void hz_lpf_t16_16w(uint8_t *dst, int32_t stride, uint8_t *filter48) { + int32_t stride2 = stride << 1; + int32_t stride3 = stride2 + stride; + int32_t stride4 = stride2 << 1; + uint8_t *dst_tmp0 = dst - stride4; + uint8_t *dst_tmp1 = dst + stride4; + + __m128i flat, flat2, filter8; + __m128i zero = __lsx_vldi(0); + __m128i p7, p6, p5, p4, p3, p2, p1, p0, q0, q1, q2, q3, q4, q5, q6, q7; + __m128i out_h, out_l; + v8u16 p7_l_in, p6_l_in, p5_l_in, p4_l_in; + v8u16 p3_l_in, p2_l_in, p1_l_in, p0_l_in; + v8u16 q7_l_in, q6_l_in, q5_l_in, q4_l_in; + v8u16 q3_l_in, q2_l_in, q1_l_in, q0_l_in; + v8u16 p7_h_in, p6_h_in, p5_h_in, p4_h_in; + v8u16 p3_h_in, p2_h_in, p1_h_in, p0_h_in; + v8u16 q7_h_in, q6_h_in, q5_h_in, q4_h_in; + v8u16 q3_h_in, q2_h_in, q1_h_in, q0_h_in; + v8u16 tmp0_l, tmp1_l, tmp0_h, tmp1_h; + + flat = __lsx_vld(filter48, 96); + + DUP4_ARG2(__lsx_vldx, dst_tmp0, -stride4, dst_tmp0, -stride3, dst_tmp0, + -stride2, dst_tmp0, -stride, p7, p6, p5, p4); + + p3 = __lsx_vld(dst_tmp0, 0); + DUP2_ARG2(__lsx_vldx, dst_tmp0, stride, dst_tmp0, stride2, p2, p1); + p0 = __lsx_vldx(dst_tmp0, stride3); + + q0 = __lsx_vld(dst, 0); + DUP2_ARG2(__lsx_vldx, dst, stride, dst, stride2, q1, q2); + q3 = __lsx_vldx(dst, stride3); + + q4 = __lsx_vld(dst_tmp1, 0); + DUP2_ARG2(__lsx_vldx, dst_tmp1, stride, dst_tmp1, stride2, q5, q6); + q7 = __lsx_vldx(dst_tmp1, stride3); + + VP9_FLAT5(p7, p6, p5, p4, p0, q0, q4, q5, q6, q7, flat, flat2); + + if (__lsx_bz_v(flat2)) { + DUP4_ARG2(__lsx_vld, filter48, 0, filter48, 16, filter48, 32, filter48, 48, + p2, p1, p0, q0); + DUP2_ARG2(__lsx_vld, filter48, 64, filter48, 80, q1, q2); + __lsx_vstx(p2, dst, -stride3); + __lsx_vstx(p1, dst, -stride2); + __lsx_vstx(p0, dst, -stride); + __lsx_vst(q0, dst, 0); + __lsx_vstx(q1, dst, stride); + __lsx_vstx(q2, dst, stride2); + } else { + dst = dst_tmp0 - stride3; + + p7_l_in = (v8u16)__lsx_vilvl_b(zero, p7); + p6_l_in = (v8u16)__lsx_vilvl_b(zero, p6); + p5_l_in = (v8u16)__lsx_vilvl_b(zero, p5); + p4_l_in = (v8u16)__lsx_vilvl_b(zero, p4); + p3_l_in = (v8u16)__lsx_vilvl_b(zero, p3); + p2_l_in = (v8u16)__lsx_vilvl_b(zero, p2); + p1_l_in = (v8u16)__lsx_vilvl_b(zero, p1); + p0_l_in = (v8u16)__lsx_vilvl_b(zero, p0); + + q0_l_in = (v8u16)__lsx_vilvl_b(zero, q0); + + tmp0_l = p7_l_in << 3; + tmp0_l -= p7_l_in; + tmp0_l += p6_l_in; + tmp0_l += q0_l_in; + tmp1_l = p6_l_in + p5_l_in; + tmp1_l += p4_l_in; + tmp1_l += p3_l_in; + tmp1_l += p2_l_in; + tmp1_l += p1_l_in; + tmp1_l += p0_l_in; + tmp1_l += tmp0_l; + + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + p7_h_in = (v8u16)__lsx_vilvh_b(zero, p7); + p6_h_in = (v8u16)__lsx_vilvh_b(zero, p6); + p5_h_in = (v8u16)__lsx_vilvh_b(zero, p5); + p4_h_in = (v8u16)__lsx_vilvh_b(zero, p4); + + p3_h_in = (v8u16)__lsx_vilvh_b(zero, p3); + p2_h_in = (v8u16)__lsx_vilvh_b(zero, p2); + p1_h_in = (v8u16)__lsx_vilvh_b(zero, p1); + p0_h_in = (v8u16)__lsx_vilvh_b(zero, p0); + q0_h_in = (v8u16)__lsx_vilvh_b(zero, q0); + + tmp0_h = p7_h_in << 3; + tmp0_h -= p7_h_in; + tmp0_h += p6_h_in; + tmp0_h += q0_h_in; + tmp1_h = p6_h_in + p5_h_in; + tmp1_h += p4_h_in; + tmp1_h += p3_h_in; + tmp1_h += p2_h_in; + tmp1_h += p1_h_in; + tmp1_h += p0_h_in; + tmp1_h += tmp0_h; + + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + p6 = __lsx_vbitsel_v(p6, out_l, flat2); + __lsx_vst(p6, dst, 0); + dst += stride; + + /* p5 */ + q1_l_in = (v8u16)__lsx_vilvl_b(zero, q1); + tmp0_l = p5_l_in - p6_l_in; + tmp0_l += q1_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + q1_h_in = (v8u16)__lsx_vilvh_b(zero, q1); + tmp0_h = p5_h_in - p6_h_in; + tmp0_h += q1_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + p5 = __lsx_vbitsel_v(p5, out_l, flat2); + __lsx_vst(p5, dst, 0); + dst += stride; + + /* p4 */ + q2_l_in = (v8u16)__lsx_vilvl_b(zero, q2); + tmp0_l = p4_l_in - p5_l_in; + tmp0_l += q2_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + q2_h_in = (v8u16)__lsx_vilvh_b(zero, q2); + tmp0_h = p4_h_in - p5_h_in; + tmp0_h += q2_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + p4 = __lsx_vbitsel_v(p4, out_l, flat2); + __lsx_vst(p4, dst, 0); + dst += stride; + + /* p3 */ + q3_l_in = (v8u16)__lsx_vilvl_b(zero, q3); + tmp0_l = p3_l_in - p4_l_in; + tmp0_l += q3_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + q3_h_in = (v8u16)__lsx_vilvh_b(zero, q3); + tmp0_h = p3_h_in - p4_h_in; + tmp0_h += q3_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + p3 = __lsx_vbitsel_v(p3, out_l, flat2); + __lsx_vst(p3, dst, 0); + dst += stride; + + /* p2 */ + q4_l_in = (v8u16)__lsx_vilvl_b(zero, q4); + filter8 = __lsx_vld(filter48, 0); + tmp0_l = p2_l_in - p3_l_in; + tmp0_l += q4_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + q4_h_in = (v8u16)__lsx_vilvh_b(zero, q4); + tmp0_h = p2_h_in - p3_h_in; + tmp0_h += q4_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 0); + dst += stride; + + /* p1 */ + q5_l_in = (v8u16)__lsx_vilvl_b(zero, q5); + filter8 = __lsx_vld(filter48, 16); + tmp0_l = p1_l_in - p2_l_in; + tmp0_l += q5_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + q5_h_in = (v8u16)__lsx_vilvh_b(zero, q5); + tmp0_h = p1_h_in - p2_h_in; + tmp0_h += q5_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 0); + dst += stride; + + /* p0 */ + q6_l_in = (v8u16)__lsx_vilvl_b(zero, q6); + filter8 = __lsx_vld(filter48, 32); + tmp0_l = p0_l_in - p1_l_in; + tmp0_l += q6_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + q6_h_in = (v8u16)__lsx_vilvh_b(zero, q6); + tmp0_h = p0_h_in - p1_h_in; + tmp0_h += q6_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 0); + dst += stride; + + /* q0 */ + q7_l_in = (v8u16)__lsx_vilvl_b(zero, q7); + filter8 = __lsx_vld(filter48, 48); + tmp0_l = q7_l_in - p0_l_in; + tmp0_l += q0_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + q7_h_in = (v8u16)__lsx_vilvh_b(zero, q7); + tmp0_h = q7_h_in - p0_h_in; + tmp0_h += q0_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 0); + dst += stride; + + /* q1 */ + filter8 = __lsx_vld(filter48, 64); + tmp0_l = q7_l_in - q0_l_in; + tmp0_l += q1_l_in; + tmp0_l -= p6_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + tmp0_h = q7_h_in - q0_h_in; + tmp0_h += q1_h_in; + tmp0_h -= p6_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 0); + dst += stride; + + /* q2 */ + filter8 = __lsx_vld(filter48, 80); + tmp0_l = q7_l_in - q1_l_in; + tmp0_l += q2_l_in; + tmp0_l -= p5_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + tmp0_h = q7_h_in - q1_h_in; + tmp0_h += q2_h_in; + tmp0_h -= p5_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 0); + dst += stride; + + /* q3 */ + tmp0_l = q7_l_in - q2_l_in; + tmp0_l += q3_l_in; + tmp0_l -= p4_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + tmp0_h = q7_h_in - q2_h_in; + tmp0_h += q3_h_in; + tmp0_h -= p4_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + q3 = __lsx_vbitsel_v(q3, out_l, flat2); + __lsx_vst(q3, dst, 0); + dst += stride; + + /* q4 */ + tmp0_l = q7_l_in - q3_l_in; + tmp0_l += q4_l_in; + tmp0_l -= p3_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + tmp0_h = q7_h_in - q3_h_in; + tmp0_h += q4_h_in; + tmp0_h -= p3_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + q4 = __lsx_vbitsel_v(q4, out_l, flat2); + __lsx_vst(q4, dst, 0); + dst += stride; + + /* q5 */ + tmp0_l = q7_l_in - q4_l_in; + tmp0_l += q5_l_in; + tmp0_l -= p2_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + tmp0_h = q7_h_in - q4_h_in; + tmp0_h += q5_h_in; + tmp0_h -= p2_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + q5 = __lsx_vbitsel_v(q5, out_l, flat2); + __lsx_vst(q5, dst, 0); + dst += stride; + + /* q6 */ + tmp0_l = q7_l_in - q5_l_in; + tmp0_l += q6_l_in; + tmp0_l -= p1_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + + tmp0_h = q7_h_in - q5_h_in; + tmp0_h += q6_h_in; + tmp0_h -= p1_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + q6 = __lsx_vbitsel_v(q6, out_l, flat2); + __lsx_vst(q6, dst, 0); + } +} + +static void mb_lpf_horizontal_edge_dual(uint8_t *dst, int32_t stride, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr) { + DECLARE_ALIGNED(16, uint8_t, filter48[16 * 8]); + uint8_t early_exit = 0; + + early_exit = hz_lpf_t4_and_t8_16w(dst, stride, &filter48[0], b_limit_ptr, + limit_ptr, thresh_ptr); + + if (early_exit == 0) { + hz_lpf_t16_16w(dst, stride, filter48); + } +} + +static void mb_lpf_horizontal_edge(uint8_t *dst, int32_t stride, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr, int32_t count) { + if (count == 1) { + __m128i flat2, mask, hev, flat, thresh, b_limit, limit; + __m128i p3, p2, p1, p0, q3, q2, q1, q0, p7, p6, p5, p4, q4, q5, q6, q7; + __m128i p2_out, p1_out, p0_out, q0_out, q1_out, q2_out; + __m128i p0_filter16, p1_filter16; + __m128i p2_filter8, p1_filter8, p0_filter8; + __m128i q0_filter8, q1_filter8, q2_filter8; + __m128i p7_l, p6_l, p5_l, p4_l, q7_l, q6_l, q5_l, q4_l; + __m128i p3_l, p2_l, p1_l, p0_l, q3_l, q2_l, q1_l, q0_l; + __m128i zero = __lsx_vldi(0); + __m128i tmp0, tmp1, tmp2; + + int32_t stride2 = stride << 1; + int32_t stride3 = 2 + stride; + int32_t stride4 = stride << 2; + uint8_t *dst_tmp0 = dst - stride4; + uint8_t *dst_tmp1 = dst + stride4; + + /* load vector elements */ + DUP4_ARG2(__lsx_vldx, dst, -stride4, dst, -stride3, dst, -stride2, dst, + -stride, p3, p2, p1, p0); + q0 = __lsx_vld(dst, 0); + DUP2_ARG2(__lsx_vldx, dst, stride, dst, stride2, q1, q2); + q3 = __lsx_vldx(dst, stride3); + + thresh = __lsx_vreplgr2vr_b(*thresh_ptr); + b_limit = __lsx_vreplgr2vr_b(*b_limit_ptr); + limit = __lsx_vreplgr2vr_b(*limit_ptr); + + /* filter_mask* */ + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + VP9_FLAT4(p3, p2, p0, q0, q2, q3, flat); + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, + q1_out); + flat = __lsx_vilvl_d(zero, flat); + if (__lsx_bz_v(flat)) { + __lsx_vstelm_d(p1_out, dst - stride2, 0, 0); + __lsx_vstelm_d(p0_out, dst - stride, 0, 0); + __lsx_vstelm_d(q0_out, dst, 0, 0); + __lsx_vstelm_d(q1_out, dst + stride, 0, 0); + } else { + /* convert 8 bit input data into 16 bit */ + DUP4_ARG2(__lsx_vilvl_b, zero, p3, zero, p2, zero, p1, zero, p0, p3_l, + p2_l, p1_l, p0_l); + DUP4_ARG2(__lsx_vilvl_b, zero, q0, zero, q1, zero, q2, zero, q3, q0_l, + q1_l, q2_l, q3_l); + VP9_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filter8, + p1_filter8, p0_filter8, q0_filter8, q1_filter8, q2_filter8); + + /* convert 16 bit output data into 8 bit */ + DUP4_ARG2(__lsx_vpickev_b, zero, p2_filter8, zero, p1_filter8, zero, + p0_filter8, zero, q0_filter8, p2_filter8, p1_filter8, + p0_filter8, q0_filter8); + DUP2_ARG2(__lsx_vpickev_b, zero, q1_filter8, zero, q2_filter8, q1_filter8, + q2_filter8); + + /* store pixel values */ + p2_out = __lsx_vbitsel_v(p2, p2_filter8, flat); + p1_out = __lsx_vbitsel_v(p1_out, p1_filter8, flat); + p0_out = __lsx_vbitsel_v(p0_out, p0_filter8, flat); + q0_out = __lsx_vbitsel_v(q0_out, q0_filter8, flat); + q1_out = __lsx_vbitsel_v(q1_out, q1_filter8, flat); + q2_out = __lsx_vbitsel_v(q2, q2_filter8, flat); + + /* load 16 vector elements */ + DUP4_ARG2(__lsx_vldx, dst_tmp0, -stride4, dst_tmp0, -stride3, dst_tmp0, + -stride2, dst_tmp0, -stride, p7, p6, p5, p4); + q4 = __lsx_vld(dst_tmp1, 0); + DUP2_ARG2(__lsx_vldx, dst_tmp1, stride, dst_tmp1, stride2, q5, q6); + q7 = __lsx_vldx(dst_tmp1, stride3); + + VP9_FLAT5(p7, p6, p5, p4, p0, q0, q4, q5, q6, q7, flat, flat2); + + if (__lsx_bz_v(flat2)) { + dst -= stride3; + __lsx_vstelm_d(p2_out, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(p1_out, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(p0_out, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(q0_out, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(q1_out, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(q2_out, dst, 0, 0); + } else { + /* LSB(right) 8 pixel operation */ + DUP4_ARG2(__lsx_vilvl_b, zero, p7, zero, p6, zero, p5, zero, p4, p7_l, + p6_l, p5_l, p4_l); + DUP4_ARG2(__lsx_vilvl_b, zero, q4, zero, q5, zero, q6, zero, q7, q4_l, + q5_l, q6_l, q7_l); + + tmp0 = __lsx_vslli_h(p7_l, 3); + tmp0 = __lsx_vsub_h(tmp0, p7_l); + tmp0 = __lsx_vadd_h(tmp0, p6_l); + tmp0 = __lsx_vadd_h(tmp0, q0_l); + + dst = dst_tmp0 - stride3; + + /* calculation of p6 and p5 */ + tmp1 = __lsx_vadd_h(p6_l, p5_l); + tmp1 = __lsx_vadd_h(tmp1, p4_l); + tmp1 = __lsx_vadd_h(tmp1, p3_l); + tmp1 = __lsx_vadd_h(tmp1, p2_l); + tmp1 = __lsx_vadd_h(tmp1, p1_l); + tmp1 = __lsx_vadd_h(tmp1, p0_l); + tmp1 = __lsx_vadd_h(tmp1, tmp0); + p0_filter16 = __lsx_vsrari_h(tmp1, 4); + tmp0 = __lsx_vsub_h(p5_l, p6_l); + tmp0 = __lsx_vadd_h(tmp0, q1_l); + tmp0 = __lsx_vsub_h(tmp0, p7_l); + tmp1 = __lsx_vadd_h(tmp1, tmp0); + p1_filter16 = __lsx_vsrari_h(tmp1, 4); + DUP2_ARG2(__lsx_vpickev_b, zero, p0_filter16, zero, p1_filter16, + p0_filter16, p1_filter16); + DUP2_ARG3(__lsx_vbitsel_v, p6, p0_filter16, flat2, p5, p1_filter16, + flat2, p0_filter16, p1_filter16); + __lsx_vstelm_d(p0_filter16, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(p1_filter16, dst, 0, 0); + dst += stride; + + /* calculation of p4 and p3 */ + tmp0 = __lsx_vsub_h(p4_l, p5_l); + tmp0 = __lsx_vadd_h(tmp0, q2_l); + tmp0 = __lsx_vsub_h(tmp0, p7_l); + tmp2 = __lsx_vsub_h(p3_l, p4_l); + tmp2 = __lsx_vadd_h(tmp2, q3_l); + tmp2 = __lsx_vsub_h(tmp2, p7_l); + tmp1 = __lsx_vadd_h(tmp1, tmp0); + p0_filter16 = __lsx_vsrari_h(tmp1, 4); + tmp1 = __lsx_vadd_h(tmp1, tmp2); + p1_filter16 = __lsx_vsrari_h(tmp1, 4); + DUP2_ARG2(__lsx_vpickev_b, zero, p0_filter16, zero, p1_filter16, + p0_filter16, p1_filter16); + DUP2_ARG3(__lsx_vbitsel_v, p4, p0_filter16, flat2, p3, p1_filter16, + flat2, p0_filter16, p1_filter16); + __lsx_vstelm_d(p0_filter16, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(p1_filter16, dst, 0, 0); + dst += stride; + + /* calculation of p2 and p1 */ + tmp0 = __lsx_vsub_h(p2_l, p3_l); + tmp0 = __lsx_vadd_h(tmp0, q4_l); + tmp0 = __lsx_vsub_h(tmp0, p7_l); + tmp2 = __lsx_vsub_h(p1_l, p2_l); + tmp2 = __lsx_vadd_h(tmp2, q5_l); + tmp2 = __lsx_vsub_h(tmp2, p7_l); + tmp1 = __lsx_vadd_h(tmp1, tmp0); + p0_filter16 = __lsx_vsrari_h(tmp1, 4); + tmp1 = __lsx_vadd_h(tmp1, tmp2); + p1_filter16 = __lsx_vsrari_h(tmp1, 4); + DUP2_ARG2(__lsx_vpickev_b, zero, p0_filter16, zero, p1_filter16, + p0_filter16, p1_filter16); + DUP2_ARG3(__lsx_vbitsel_v, p2_out, p0_filter16, flat2, p1_out, + p1_filter16, flat2, p0_filter16, p1_filter16); + __lsx_vstelm_d(p0_filter16, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(p1_filter16, dst, 0, 0); + dst += stride; + + /* calculation of p0 and q0 */ + tmp0 = __lsx_vsub_h(p0_l, p1_l); + tmp0 = __lsx_vadd_h(tmp0, q6_l); + tmp0 = __lsx_vsub_h(tmp0, p7_l); + tmp2 = __lsx_vsub_h(q7_l, p0_l); + tmp2 = __lsx_vadd_h(tmp2, q0_l); + tmp2 = __lsx_vsub_h(tmp2, p7_l); + tmp1 = __lsx_vadd_h(tmp1, tmp0); + p0_filter16 = __lsx_vsrari_h(tmp1, 4); + tmp1 = __lsx_vadd_h(tmp1, tmp2); + p1_filter16 = __lsx_vsrari_h(tmp1, 4); + DUP2_ARG2(__lsx_vpickev_b, zero, p0_filter16, zero, p1_filter16, + p0_filter16, p1_filter16); + DUP2_ARG3(__lsx_vbitsel_v, p0_out, p0_filter16, flat2, q0_out, + p1_filter16, flat2, p0_filter16, p1_filter16); + __lsx_vstelm_d(p0_filter16, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(p1_filter16, dst, 0, 0); + dst += stride; + + /* calculation of q1 and q2 */ + tmp0 = __lsx_vsub_h(q7_l, q0_l); + tmp0 = __lsx_vadd_h(tmp0, q1_l); + tmp0 = __lsx_vsub_h(tmp0, p6_l); + tmp2 = __lsx_vsub_h(q7_l, q1_l); + tmp2 = __lsx_vadd_h(tmp2, q2_l); + tmp2 = __lsx_vsub_h(tmp2, p5_l); + tmp1 = __lsx_vadd_h(tmp1, tmp0); + p0_filter16 = __lsx_vsrari_h(tmp1, 4); + tmp1 = __lsx_vadd_h(tmp1, tmp2); + p1_filter16 = __lsx_vsrari_h(tmp1, 4); + DUP2_ARG2(__lsx_vpickev_b, zero, p0_filter16, zero, p1_filter16, + p0_filter16, p1_filter16); + DUP2_ARG3(__lsx_vbitsel_v, q1_out, p0_filter16, flat2, q2_out, + p1_filter16, flat2, p0_filter16, p1_filter16); + __lsx_vstelm_d(p0_filter16, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(p1_filter16, dst, 0, 0); + dst += stride; + + /* calculation of q3 and q4 */ + tmp0 = __lsx_vsub_h(q7_l, q2_l); + tmp0 = __lsx_vadd_h(tmp0, q3_l); + tmp0 = __lsx_vsub_h(tmp0, p4_l); + tmp2 = __lsx_vsub_h(q7_l, q3_l); + tmp2 = __lsx_vadd_h(tmp2, q4_l); + tmp2 = __lsx_vsub_h(tmp2, p3_l); + tmp1 = __lsx_vadd_h(tmp1, tmp0); + p0_filter16 = __lsx_vsrari_h(tmp1, 4); + tmp1 = __lsx_vadd_h(tmp1, tmp2); + p1_filter16 = __lsx_vsrari_h(tmp1, 4); + DUP2_ARG2(__lsx_vpickev_b, zero, p0_filter16, zero, p1_filter16, + p0_filter16, p1_filter16); + DUP2_ARG3(__lsx_vbitsel_v, q3, p0_filter16, flat2, q4, p1_filter16, + flat2, p0_filter16, p1_filter16); + __lsx_vstelm_d(p0_filter16, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(p1_filter16, dst, 0, 0); + dst += stride; + + /* calculation of q5 and q6 */ + tmp0 = __lsx_vsub_h(q7_l, q4_l); + tmp0 = __lsx_vadd_h(tmp0, q5_l); + tmp0 = __lsx_vsub_h(tmp0, p2_l); + tmp2 = __lsx_vsub_h(q7_l, q5_l); + tmp2 = __lsx_vadd_h(tmp2, q6_l); + tmp2 = __lsx_vsub_h(tmp2, p1_l); + tmp1 = __lsx_vadd_h(tmp1, tmp0); + p0_filter16 = __lsx_vsrari_h(tmp1, 4); + tmp1 = __lsx_vadd_h(tmp1, tmp2); + p1_filter16 = __lsx_vsrari_h(tmp1, 4); + DUP2_ARG2(__lsx_vpickev_b, zero, p0_filter16, zero, p1_filter16, + p0_filter16, p1_filter16); + DUP2_ARG3(__lsx_vbitsel_v, q5, p0_filter16, flat2, q6, p1_filter16, + flat2, p0_filter16, p1_filter16); + __lsx_vstelm_d(p0_filter16, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(p1_filter16, dst, 0, 0); + } + } + } else { + mb_lpf_horizontal_edge_dual(dst, stride, b_limit_ptr, limit_ptr, + thresh_ptr); + } +} + +void vpx_lpf_horizontal_16_dual_lsx(uint8_t *dst, int32_t stride, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr) { + mb_lpf_horizontal_edge(dst, stride, b_limit_ptr, limit_ptr, thresh_ptr, 2); +} + +static void transpose_16x16(uint8_t *input, int32_t in_stride, uint8_t *output, + int32_t out_stride) { + __m128i row0, row1, row2, row3, row4, row5, row6, row7; + __m128i row8, row9, row10, row11, row12, row13, row14, row15; + __m128i tmp0, tmp1, tmp4, tmp5, tmp6, tmp7; + __m128i tmp2, tmp3; + __m128i p7, p6, p5, p4, p3, p2, p1, p0, q0, q1, q2, q3, q4, q5, q6, q7; + int32_t in_stride2 = in_stride << 1; + int32_t in_stride3 = in_stride2 + in_stride; + int32_t in_stride4 = in_stride2 << 1; + int32_t out_stride2 = out_stride << 1; + int32_t out_stride3 = out_stride2 + out_stride; + int32_t out_stride4 = out_stride2 << 1; + + LSX_LD_8(input, in_stride, in_stride2, in_stride3, in_stride4, row0, row1, + row2, row3, row4, row5, row6, row7); + input += in_stride4; + LSX_LD_8(input, in_stride, in_stride2, in_stride3, in_stride4, row8, row9, + row10, row11, row12, row13, row14, row15); + + LSX_TRANSPOSE16x8_B(row0, row1, row2, row3, row4, row5, row6, row7, row8, + row9, row10, row11, row12, row13, row14, row15, p7, p6, + p5, p4, p3, p2, p1, p0); + + /* transpose 16x8 matrix into 8x16 */ + /* total 8 intermediate register and 32 instructions */ + q7 = __lsx_vpackod_d(row8, row0); + q6 = __lsx_vpackod_d(row9, row1); + q5 = __lsx_vpackod_d(row10, row2); + q4 = __lsx_vpackod_d(row11, row3); + q3 = __lsx_vpackod_d(row12, row4); + q2 = __lsx_vpackod_d(row13, row5); + q1 = __lsx_vpackod_d(row14, row6); + q0 = __lsx_vpackod_d(row15, row7); + + DUP2_ARG2(__lsx_vpackev_b, q6, q7, q4, q5, tmp0, tmp1); + DUP2_ARG2(__lsx_vpackod_b, q6, q7, q4, q5, tmp4, tmp5); + + DUP2_ARG2(__lsx_vpackev_b, q2, q3, q0, q1, q5, q7); + DUP2_ARG2(__lsx_vpackod_b, q2, q3, q0, q1, tmp6, tmp7); + + DUP2_ARG2(__lsx_vpackev_h, tmp1, tmp0, q7, q5, tmp2, tmp3); + q0 = __lsx_vpackev_w(tmp3, tmp2); + q4 = __lsx_vpackod_w(tmp3, tmp2); + + tmp2 = __lsx_vpackod_h(tmp1, tmp0); + tmp3 = __lsx_vpackod_h(q7, q5); + q2 = __lsx_vpackev_w(tmp3, tmp2); + q6 = __lsx_vpackod_w(tmp3, tmp2); + + DUP2_ARG2(__lsx_vpackev_h, tmp5, tmp4, tmp7, tmp6, tmp2, tmp3); + q1 = __lsx_vpackev_w(tmp3, tmp2); + q5 = __lsx_vpackod_w(tmp3, tmp2); + + tmp2 = __lsx_vpackod_h(tmp5, tmp4); + tmp3 = __lsx_vpackod_h(tmp7, tmp6); + q3 = __lsx_vpackev_w(tmp3, tmp2); + q7 = __lsx_vpackod_w(tmp3, tmp2); + + LSX_ST_8(p7, p6, p5, p4, p3, p2, p1, p0, output, out_stride, out_stride2, + out_stride3, out_stride4); + output += out_stride4; + LSX_ST_8(q0, q1, q2, q3, q4, q5, q6, q7, output, out_stride, out_stride2, + out_stride3, out_stride4); +} + +static int32_t vt_lpf_t4_and_t8_16w(uint8_t *dst, uint8_t *filter48, + uint8_t *dst_org, int32_t stride, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr) { + int32_t stride2 = stride << 1; + int32_t stride3 = stride2 + stride; + int32_t stride4 = stride2 << 1; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i p2_out, p1_out, p0_out, q0_out, q1_out, q2_out; + __m128i flat, mask, hev, thresh, b_limit, limit; + __m128i p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l; + __m128i p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h; + __m128i p2_filt8_l, p1_filt8_l, p0_filt8_l; + __m128i q0_filt8_l, q1_filt8_l, q2_filt8_l; + __m128i p2_filt8_h, p1_filt8_h, p0_filt8_h; + __m128i q0_filt8_h, q1_filt8_h, q2_filt8_h; + __m128i vec0, vec1, vec2, vec3, vec4, vec5; + __m128i zero = __lsx_vldi(0); + + /* load vector elements */ + DUP4_ARG2(__lsx_vld, dst, -64, dst, -48, dst, -32, dst, -16, p3, p2, p1, p0); + DUP4_ARG2(__lsx_vld, dst, 0, dst, 16, dst, 32, dst, 48, q0, q1, q2, q3); + + thresh = __lsx_vreplgr2vr_b(*thresh_ptr); + b_limit = __lsx_vreplgr2vr_b(*b_limit_ptr); + limit = __lsx_vreplgr2vr_b(*limit_ptr); + + /* mask and hev */ + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + /* flat4 */ + VP9_FLAT4(p3, p2, p0, q0, q2, q3, flat); + /* filter4 */ + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, q1_out); + + /* if flat is zero for all pixels, then no need to calculate other filter */ + if (__lsx_bz_v(flat)) { + DUP2_ARG2(__lsx_vilvl_b, p0_out, p1_out, q1_out, q0_out, vec0, vec1); + vec2 = __lsx_vilvl_h(vec1, vec0); + vec3 = __lsx_vilvh_h(vec1, vec0); + DUP2_ARG2(__lsx_vilvh_b, p0_out, p1_out, q1_out, q0_out, vec0, vec1); + vec4 = __lsx_vilvl_h(vec1, vec0); + vec5 = __lsx_vilvh_h(vec1, vec0); + + dst_org -= 2; + __lsx_vstelm_w(vec2, dst_org, 0, 0); + __lsx_vstelm_w(vec2, dst_org + stride, 0, 1); + __lsx_vstelm_w(vec2, dst_org + stride2, 0, 2); + __lsx_vstelm_w(vec2, dst_org + stride3, 0, 3); + dst_org += stride4; + __lsx_vstelm_w(vec3, dst_org, 0, 0); + __lsx_vstelm_w(vec3, dst_org + stride, 0, 1); + __lsx_vstelm_w(vec3, dst_org + stride2, 0, 2); + __lsx_vstelm_w(vec3, dst_org + stride3, 0, 3); + dst_org += stride4; + __lsx_vstelm_w(vec4, dst_org, 0, 0); + __lsx_vstelm_w(vec4, dst_org + stride, 0, 1); + __lsx_vstelm_w(vec4, dst_org + stride2, 0, 2); + __lsx_vstelm_w(vec4, dst_org + stride3, 0, 3); + dst_org += stride4; + __lsx_vstelm_w(vec5, dst_org, 0, 0); + __lsx_vstelm_w(vec5, dst_org + stride, 0, 1); + __lsx_vstelm_w(vec5, dst_org + stride2, 0, 2); + __lsx_vstelm_w(vec5, dst_org + stride3, 0, 3); + + return 1; + } + + DUP4_ARG2(__lsx_vilvl_b, zero, p3, zero, p2, zero, p1, zero, p0, p3_l, p2_l, + p1_l, p0_l); + DUP4_ARG2(__lsx_vilvl_b, zero, q0, zero, q1, zero, q2, zero, q3, q0_l, q1_l, + q2_l, q3_l); + VP9_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l, q1_filt8_l, q2_filt8_l); + DUP4_ARG2(__lsx_vilvh_b, zero, p3, zero, p2, zero, p1, zero, p0, p3_h, p2_h, + p1_h, p0_h); + DUP4_ARG2(__lsx_vilvh_b, zero, q0, zero, q1, zero, q2, zero, q3, q0_h, q1_h, + q2_h, q3_h); + VP9_FILTER8(p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h, p2_filt8_h, + p1_filt8_h, p0_filt8_h, q0_filt8_h, q1_filt8_h, q2_filt8_h); + + /* convert 16 bit output data into 8 bit */ + DUP4_ARG2(__lsx_vpickev_b, p2_filt8_h, p2_filt8_l, p1_filt8_h, p1_filt8_l, + p0_filt8_h, p0_filt8_l, q0_filt8_h, q0_filt8_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l); + DUP2_ARG2(__lsx_vpickev_b, q1_filt8_h, q1_filt8_l, q2_filt8_h, q2_filt8_l, + q1_filt8_l, q2_filt8_l); + + /* store pixel values */ + p2_out = __lsx_vbitsel_v(p2, p2_filt8_l, flat); + p1_out = __lsx_vbitsel_v(p1_out, p1_filt8_l, flat); + p0_out = __lsx_vbitsel_v(p0_out, p0_filt8_l, flat); + q0_out = __lsx_vbitsel_v(q0_out, q0_filt8_l, flat); + q1_out = __lsx_vbitsel_v(q1_out, q1_filt8_l, flat); + q2_out = __lsx_vbitsel_v(q2, q2_filt8_l, flat); + + __lsx_vst(p2_out, filter48, 0); + __lsx_vst(p1_out, filter48, 16); + __lsx_vst(p0_out, filter48, 32); + __lsx_vst(q0_out, filter48, 48); + __lsx_vst(q1_out, filter48, 64); + __lsx_vst(q2_out, filter48, 80); + __lsx_vst(flat, filter48, 96); + + return 0; +} + +static int32_t vt_lpf_t16_16w(uint8_t *dst, uint8_t *dst_org, int32_t stride, + uint8_t *filter48) { + __m128i zero = __lsx_vldi(0); + __m128i flat, flat2, filter8; + __m128i p7, p6, p5, p4, p3, p2, p1, p0, q0, q1, q2, q3, q4, q5, q6, q7; + __m128i out_l, out_h; + v8u16 p7_l_in, p6_l_in, p5_l_in, p4_l_in; + v8u16 p3_l_in, p2_l_in, p1_l_in, p0_l_in; + v8u16 q7_l_in, q6_l_in, q5_l_in, q4_l_in; + v8u16 q3_l_in, q2_l_in, q1_l_in, q0_l_in; + v8u16 p7_h_in, p6_h_in, p5_h_in, p4_h_in; + v8u16 p3_h_in, p2_h_in, p1_h_in, p0_h_in; + v8u16 q7_h_in, q6_h_in, q5_h_in, q4_h_in; + v8u16 q3_h_in, q2_h_in, q1_h_in, q0_h_in; + v8u16 tmp0_l, tmp1_l, tmp0_h, tmp1_h; + uint8_t *dst_tmp = dst - 128; + + flat = __lsx_vld(filter48, 96); + + DUP4_ARG2(__lsx_vld, dst_tmp, 0, dst_tmp, 16, dst_tmp, 32, dst_tmp, 48, p7, + p6, p5, p4); + DUP4_ARG2(__lsx_vld, dst_tmp, 64, dst_tmp, 80, dst_tmp, 96, dst_tmp, 112, p3, + p2, p1, p0); + DUP4_ARG2(__lsx_vld, dst, 0, dst, 16, dst, 32, dst, 48, q0, q1, q2, q3); + DUP4_ARG2(__lsx_vld, dst, 64, dst, 80, dst, 96, dst, 112, q4, q5, q6, q7); + + VP9_FLAT5(p7, p6, p5, p4, p0, q0, q4, q5, q6, q7, flat, flat2); + /* if flat2 is zero for all pixels, then no need to calculate other filter */ + if (__lsx_bz_v(flat2)) { + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + + DUP4_ARG2(__lsx_vld, filter48, 0, filter48, 16, filter48, 32, filter48, 48, + p2, p1, p0, q0); + DUP2_ARG2(__lsx_vld, filter48, 64, filter48, 80, q1, q2); + + DUP2_ARG2(__lsx_vilvl_b, p1, p2, q0, p0, vec0, vec1); + vec3 = __lsx_vilvl_h(vec1, vec0); + vec4 = __lsx_vilvh_h(vec1, vec0); + DUP2_ARG2(__lsx_vilvh_b, p1, p2, q0, p0, vec0, vec1); + vec6 = __lsx_vilvl_h(vec1, vec0); + vec7 = __lsx_vilvh_h(vec1, vec0); + vec2 = __lsx_vilvl_b(q2, q1); + vec5 = __lsx_vilvh_b(q2, q1); + + dst_org -= 3; + __lsx_vstelm_w(vec3, dst_org, 0, 0); + __lsx_vstelm_h(vec2, dst_org, 4, 0); + dst_org += stride; + __lsx_vstelm_w(vec3, dst_org, 0, 1); + __lsx_vstelm_h(vec2, dst_org, 4, 1); + dst_org += stride; + __lsx_vstelm_w(vec3, dst_org, 0, 2); + __lsx_vstelm_h(vec2, dst_org, 4, 2); + dst_org += stride; + __lsx_vstelm_w(vec3, dst_org, 0, 3); + __lsx_vstelm_h(vec2, dst_org, 4, 3); + dst_org += stride; + __lsx_vstelm_w(vec4, dst_org, 0, 0); + __lsx_vstelm_h(vec2, dst_org, 4, 4); + dst_org += stride; + __lsx_vstelm_w(vec4, dst_org, 0, 1); + __lsx_vstelm_h(vec2, dst_org, 4, 5); + dst_org += stride; + __lsx_vstelm_w(vec4, dst_org, 0, 2); + __lsx_vstelm_h(vec2, dst_org, 4, 6); + dst_org += stride; + __lsx_vstelm_w(vec4, dst_org, 0, 3); + __lsx_vstelm_h(vec2, dst_org, 4, 7); + dst_org += stride; + __lsx_vstelm_w(vec6, dst_org, 0, 0); + __lsx_vstelm_h(vec5, dst_org, 4, 0); + dst_org += stride; + __lsx_vstelm_w(vec6, dst_org, 0, 1); + __lsx_vstelm_h(vec5, dst_org, 4, 1); + dst_org += stride; + __lsx_vstelm_w(vec6, dst_org, 0, 2); + __lsx_vstelm_h(vec5, dst_org, 4, 2); + dst_org += stride; + __lsx_vstelm_w(vec6, dst_org, 0, 3); + __lsx_vstelm_h(vec5, dst_org, 4, 3); + dst_org += stride; + __lsx_vstelm_w(vec7, dst_org, 0, 0); + __lsx_vstelm_h(vec5, dst_org, 4, 4); + dst_org += stride; + __lsx_vstelm_w(vec7, dst_org, 0, 1); + __lsx_vstelm_h(vec5, dst_org, 4, 5); + dst_org += stride; + __lsx_vstelm_w(vec7, dst_org, 0, 2); + __lsx_vstelm_h(vec5, dst_org, 4, 6); + dst_org += stride; + __lsx_vstelm_w(vec7, dst_org, 0, 3); + __lsx_vstelm_h(vec5, dst_org, 4, 7); + + return 1; + } + + dst -= 7 * 16; + + p7_l_in = (v8u16)__lsx_vilvl_b(zero, p7); + p6_l_in = (v8u16)__lsx_vilvl_b(zero, p6); + p5_l_in = (v8u16)__lsx_vilvl_b(zero, p5); + p4_l_in = (v8u16)__lsx_vilvl_b(zero, p4); + p3_l_in = (v8u16)__lsx_vilvl_b(zero, p3); + p2_l_in = (v8u16)__lsx_vilvl_b(zero, p2); + p1_l_in = (v8u16)__lsx_vilvl_b(zero, p1); + p0_l_in = (v8u16)__lsx_vilvl_b(zero, p0); + q0_l_in = (v8u16)__lsx_vilvl_b(zero, q0); + + tmp0_l = p7_l_in << 3; + tmp0_l -= p7_l_in; + tmp0_l += p6_l_in; + tmp0_l += q0_l_in; + tmp1_l = p6_l_in + p5_l_in; + tmp1_l += p4_l_in; + tmp1_l += p3_l_in; + tmp1_l += p2_l_in; + tmp1_l += p1_l_in; + tmp1_l += p0_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + p7_h_in = (v8u16)__lsx_vilvh_b(zero, p7); + p6_h_in = (v8u16)__lsx_vilvh_b(zero, p6); + p5_h_in = (v8u16)__lsx_vilvh_b(zero, p5); + p4_h_in = (v8u16)__lsx_vilvh_b(zero, p4); + p3_h_in = (v8u16)__lsx_vilvh_b(zero, p3); + p2_h_in = (v8u16)__lsx_vilvh_b(zero, p2); + p1_h_in = (v8u16)__lsx_vilvh_b(zero, p1); + p0_h_in = (v8u16)__lsx_vilvh_b(zero, p0); + q0_h_in = (v8u16)__lsx_vilvh_b(zero, q0); + + tmp0_h = p7_h_in << 3; + tmp0_h -= p7_h_in; + tmp0_h += p6_h_in; + tmp0_h += q0_h_in; + tmp1_h = p6_h_in + p5_h_in; + tmp1_h += p4_h_in; + tmp1_h += p3_h_in; + tmp1_h += p2_h_in; + tmp1_h += p1_h_in; + tmp1_h += p0_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + + out_l = __lsx_vpickev_b(out_h, out_l); + p6 = __lsx_vbitsel_v(p6, out_l, flat2); + __lsx_vst(p6, dst, 0); + + /* p5 */ + q1_l_in = (v8u16)__lsx_vilvl_b(zero, q1); + tmp0_l = p5_l_in - p6_l_in; + tmp0_l += q1_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + q1_h_in = (v8u16)__lsx_vilvh_b(zero, q1); + tmp0_h = p5_h_in - p6_h_in; + tmp0_h += q1_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + p5 = __lsx_vbitsel_v(p5, out_l, flat2); + __lsx_vst(p5, dst, 16); + + /* p4 */ + q2_l_in = (v8u16)__lsx_vilvl_b(zero, q2); + tmp0_l = p4_l_in - p5_l_in; + tmp0_l += q2_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + q2_h_in = (v8u16)__lsx_vilvh_b(zero, q2); + tmp0_h = p4_h_in - p5_h_in; + tmp0_h += q2_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + p4 = __lsx_vbitsel_v(p4, out_l, flat2); + __lsx_vst(p4, dst, 16 * 2); + + /* p3 */ + q3_l_in = (v8u16)__lsx_vilvl_b(zero, q3); + tmp0_l = p3_l_in - p4_l_in; + tmp0_l += q3_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + q3_h_in = (v8u16)__lsx_vilvh_b(zero, q3); + tmp0_h = p3_h_in - p4_h_in; + tmp0_h += q3_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + p3 = __lsx_vbitsel_v(p3, out_l, flat2); + __lsx_vst(p3, dst, 16 * 3); + + /* p2 */ + q4_l_in = (v8u16)__lsx_vilvl_b(zero, q4); + filter8 = __lsx_vld(filter48, 0); + tmp0_l = p2_l_in - p3_l_in; + tmp0_l += q4_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + q4_h_in = (v8u16)__lsx_vilvh_b(zero, q4); + tmp0_h = p2_h_in - p3_h_in; + tmp0_h += q4_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 16 * 4); + + /* p1 */ + q5_l_in = (v8u16)__lsx_vilvl_b(zero, q5); + filter8 = __lsx_vld(filter48, 16); + tmp0_l = p1_l_in - p2_l_in; + tmp0_l += q5_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + q5_h_in = (v8u16)__lsx_vilvh_b(zero, q5); + tmp0_h = p1_h_in - p2_h_in; + tmp0_h += q5_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)(tmp1_h), 4); + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 16 * 5); + + /* p0 */ + q6_l_in = (v8u16)__lsx_vilvl_b(zero, q6); + filter8 = __lsx_vld(filter48, 32); + tmp0_l = p0_l_in - p1_l_in; + tmp0_l += q6_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + q6_h_in = (v8u16)__lsx_vilvh_b(zero, q6); + tmp0_h = p0_h_in - p1_h_in; + tmp0_h += q6_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 16 * 6); + + /* q0 */ + q7_l_in = (v8u16)__lsx_vilvl_b(zero, q7); + filter8 = __lsx_vld(filter48, 48); + tmp0_l = q7_l_in - p0_l_in; + tmp0_l += q0_l_in; + tmp0_l -= p7_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + q7_h_in = (v8u16)__lsx_vilvh_b(zero, q7); + tmp0_h = q7_h_in - p0_h_in; + tmp0_h += q0_h_in; + tmp0_h -= p7_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 16 * 7); + + /* q1 */ + filter8 = __lsx_vld(filter48, 64); + tmp0_l = q7_l_in - q0_l_in; + tmp0_l += q1_l_in; + tmp0_l -= p6_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + tmp0_h = q7_h_in - q0_h_in; + tmp0_h += q1_h_in; + tmp0_h -= p6_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 16 * 8); + + /* q2 */ + filter8 = __lsx_vld(filter48, 80); + tmp0_l = q7_l_in - q1_l_in; + tmp0_l += q2_l_in; + tmp0_l -= p5_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + tmp0_h = q7_h_in - q1_h_in; + tmp0_h += q2_h_in; + tmp0_h -= p5_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + filter8 = __lsx_vbitsel_v(filter8, out_l, flat2); + __lsx_vst(filter8, dst, 16 * 9); + + /* q3 */ + tmp0_l = q7_l_in - q2_l_in; + tmp0_l += q3_l_in; + tmp0_l -= p4_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + tmp0_h = q7_h_in - q2_h_in; + tmp0_h += q3_h_in; + tmp0_h -= p4_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + q3 = __lsx_vbitsel_v(q3, out_l, flat2); + __lsx_vst(q3, dst, 16 * 10); + + /* q4 */ + tmp0_l = q7_l_in - q3_l_in; + tmp0_l += q4_l_in; + tmp0_l -= p3_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + tmp0_h = q7_h_in - q3_h_in; + tmp0_h += q4_h_in; + tmp0_h -= p3_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + q4 = __lsx_vbitsel_v(q4, out_l, flat2); + __lsx_vst(q4, dst, 16 * 11); + + /* q5 */ + tmp0_l = q7_l_in - q4_l_in; + tmp0_l += q5_l_in; + tmp0_l -= p2_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + tmp0_h = q7_h_in - q4_h_in; + tmp0_h += q5_h_in; + tmp0_h -= p2_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + q5 = __lsx_vbitsel_v(q5, out_l, flat2); + __lsx_vst(q5, dst, 16 * 12); + + /* q6 */ + tmp0_l = q7_l_in - q5_l_in; + tmp0_l += q6_l_in; + tmp0_l -= p1_l_in; + tmp1_l += tmp0_l; + out_l = __lsx_vsrari_h((__m128i)tmp1_l, 4); + tmp0_h = q7_h_in - q5_h_in; + tmp0_h += q6_h_in; + tmp0_h -= p1_h_in; + tmp1_h += tmp0_h; + out_h = __lsx_vsrari_h((__m128i)tmp1_h, 4); + out_l = __lsx_vpickev_b(out_h, out_l); + q6 = __lsx_vbitsel_v(q6, out_l, flat2); + __lsx_vst(q6, dst, 16 * 13); + + return 0; +} + +void vpx_lpf_vertical_16_dual_lsx(uint8_t *src, int32_t pitch, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr) { + uint8_t early_exit = 0; + DECLARE_ALIGNED(16, uint8_t, transposed_input[16 * 24]); + uint8_t *filter48 = &transposed_input[16 * 16]; + + transpose_16x16((src - 8), pitch, &transposed_input[0], 16); + + early_exit = + vt_lpf_t4_and_t8_16w((transposed_input + 16 * 8), &filter48[0], src, + pitch, b_limit_ptr, limit_ptr, thresh_ptr); + + if (early_exit == 0) { + early_exit = + vt_lpf_t16_16w((transposed_input + 16 * 8), src, pitch, &filter48[0]); + + if (early_exit == 0) { + transpose_16x16(transposed_input, 16, (src - 8), pitch); + } + } +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/loopfilter_4_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/loopfilter_4_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/loopfilter_4_lsx.c @@ -0,0 +1,214 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "./vpx_dsp_rtcd.h" +#include "vpx_dsp/loongarch/loopfilter_lsx.h" + +void vpx_lpf_horizontal_4_lsx(uint8_t *src, int32_t pitch, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr) { + __m128i mask, hev, flat, thresh, b_limit, limit; + __m128i p3, p2, p1, p0, q3, q2, q1, q0, p1_out, p0_out, q0_out, q1_out; + int32_t pitch2 = pitch << 1; + int32_t pitch3 = pitch2 + pitch; + int32_t pitch4 = pitch2 << 1; + + DUP4_ARG2(__lsx_vldx, src, -pitch4, src, -pitch3, src, -pitch2, src, -pitch, + p3, p2, p1, p0); + q0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, pitch, src, pitch2, q1, q2); + q3 = __lsx_vldx(src, pitch3); + + thresh = __lsx_vreplgr2vr_b(*thresh_ptr); + b_limit = __lsx_vreplgr2vr_b(*b_limit_ptr); + limit = __lsx_vreplgr2vr_b(*limit_ptr); + + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, q1_out); + + __lsx_vstelm_d(p1_out, src - pitch2, 0, 0); + __lsx_vstelm_d(p0_out, src - pitch, 0, 0); + __lsx_vstelm_d(q0_out, src, 0, 0); + __lsx_vstelm_d(q1_out, src + pitch, 0, 0); +} + +void vpx_lpf_horizontal_4_dual_lsx(uint8_t *src, int32_t pitch, + const uint8_t *b_limit0_ptr, + const uint8_t *limit0_ptr, + const uint8_t *thresh0_ptr, + const uint8_t *b_limit1_ptr, + const uint8_t *limit1_ptr, + const uint8_t *thresh1_ptr) { + __m128i mask, hev, flat, thresh0, b_limit0, limit0, thresh1, b_limit1, limit1; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + int32_t pitch2 = pitch << 1; + int32_t pitch3 = pitch2 + pitch; + int32_t pitch4 = pitch2 << 1; + + DUP4_ARG2(__lsx_vldx, src, -pitch4, src, -pitch3, src, -pitch2, src, -pitch, + p3, p2, p1, p0); + q0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, pitch, src, pitch2, q1, q2); + q3 = __lsx_vldx(src, pitch3); + + thresh0 = __lsx_vreplgr2vr_b(*thresh0_ptr); + thresh1 = __lsx_vreplgr2vr_b(*thresh1_ptr); + thresh0 = __lsx_vilvl_d(thresh1, thresh0); + + b_limit0 = __lsx_vreplgr2vr_b(*b_limit0_ptr); + b_limit1 = __lsx_vreplgr2vr_b(*b_limit1_ptr); + b_limit0 = __lsx_vilvl_d(b_limit1, b_limit0); + + limit0 = __lsx_vreplgr2vr_b(*limit0_ptr); + limit1 = __lsx_vreplgr2vr_b(*limit1_ptr); + limit0 = __lsx_vilvl_d(limit1, limit0); + + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit0, b_limit0, thresh0, hev, + mask, flat); + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1, p0, q0, q1); + + __lsx_vstx(p1, src, -pitch2); + __lsx_vstx(p0, src, -pitch); + __lsx_vst(q0, src, 0); + __lsx_vstx(q1, src, pitch); +} + +void vpx_lpf_vertical_4_lsx(uint8_t *src, int32_t pitch, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr) { + __m128i mask, hev, flat, limit, thresh, b_limit; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i vec0, vec1, vec2, vec3; + int32_t pitch2 = pitch << 1; + int32_t pitch3 = pitch2 + pitch; + int32_t pitch4 = pitch2 << 1; + uint8_t *src_tmp = src - 4; + + p3 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, pitch, src_tmp, pitch2, p2, p1); + p0 = __lsx_vldx(src_tmp, pitch3); + src_tmp += pitch4; + q0 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, pitch, src_tmp, pitch2, q1, q2); + q3 = __lsx_vldx(src_tmp, pitch3); + + thresh = __lsx_vreplgr2vr_b(*thresh_ptr); + b_limit = __lsx_vreplgr2vr_b(*b_limit_ptr); + limit = __lsx_vreplgr2vr_b(*limit_ptr); + + LSX_TRANSPOSE8x8_B(p3, p2, p1, p0, q0, q1, q2, q3, p3, p2, p1, p0, q0, q1, q2, + q3); + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1, p0, q0, q1); + DUP2_ARG2(__lsx_vilvl_b, p0, p1, q1, q0, vec0, vec1); + vec2 = __lsx_vilvl_h(vec1, vec0); + vec3 = __lsx_vilvh_h(vec1, vec0); + + src -= 2; + __lsx_vstelm_w(vec2, src, 0, 0); + src += pitch; + __lsx_vstelm_w(vec2, src, 0, 1); + src += pitch; + __lsx_vstelm_w(vec2, src, 0, 2); + src += pitch; + __lsx_vstelm_w(vec2, src, 0, 3); + src += pitch; + + __lsx_vstelm_w(vec3, src, 0, 0); + __lsx_vstelm_w(vec3, src + pitch, 0, 1); + __lsx_vstelm_w(vec3, src + pitch2, 0, 2); + __lsx_vstelm_w(vec3, src + pitch3, 0, 3); +} + +void vpx_lpf_vertical_4_dual_lsx(uint8_t *src, int32_t pitch, + const uint8_t *b_limit0_ptr, + const uint8_t *limit0_ptr, + const uint8_t *thresh0_ptr, + const uint8_t *b_limit1_ptr, + const uint8_t *limit1_ptr, + const uint8_t *thresh1_ptr) { + __m128i mask, hev, flat; + __m128i thresh0, b_limit0, limit0, thresh1, b_limit1, limit1; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i row0, row1, row2, row3, row4, row5, row6, row7; + __m128i row8, row9, row10, row11, row12, row13, row14, row15; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + int32_t pitch2 = pitch << 1; + int32_t pitch3 = pitch2 + pitch; + int32_t pitch4 = pitch2 << 1; + uint8_t *src_tmp = src - 4; + + row0 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, pitch, src_tmp, pitch2, row1, row2); + row3 = __lsx_vldx(src_tmp, pitch3); + src_tmp += pitch4; + row4 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, pitch, src_tmp, pitch2, row5, row6); + row7 = __lsx_vldx(src_tmp, pitch3); + src_tmp += pitch4; + row8 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, pitch, src_tmp, pitch2, row9, row10); + row11 = __lsx_vldx(src_tmp, pitch3); + src_tmp += pitch4; + row12 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, pitch, src_tmp, pitch2, row13, row14); + row15 = __lsx_vldx(src_tmp, pitch3); + + LSX_TRANSPOSE16x8_B(row0, row1, row2, row3, row4, row5, row6, row7, row8, + row9, row10, row11, row12, row13, row14, row15, p3, p2, + p1, p0, q0, q1, q2, q3); + + thresh0 = __lsx_vreplgr2vr_b(*thresh0_ptr); + thresh1 = __lsx_vreplgr2vr_b(*thresh1_ptr); + thresh0 = __lsx_vilvl_d(thresh1, thresh0); + + b_limit0 = __lsx_vreplgr2vr_b(*b_limit0_ptr); + b_limit1 = __lsx_vreplgr2vr_b(*b_limit1_ptr); + b_limit0 = __lsx_vilvl_d(b_limit1, b_limit0); + + limit0 = __lsx_vreplgr2vr_b(*limit0_ptr); + limit1 = __lsx_vreplgr2vr_b(*limit1_ptr); + limit0 = __lsx_vilvl_d(limit1, limit0); + + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit0, b_limit0, thresh0, hev, + mask, flat); + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1, p0, q0, q1); + DUP2_ARG2(__lsx_vilvl_b, p0, p1, q1, q0, tmp0, tmp1); + tmp2 = __lsx_vilvl_h(tmp1, tmp0); + tmp3 = __lsx_vilvh_h(tmp1, tmp0); + DUP2_ARG2(__lsx_vilvh_b, p0, p1, q1, q0, tmp0, tmp1); + tmp4 = __lsx_vilvl_h(tmp1, tmp0); + tmp5 = __lsx_vilvh_h(tmp1, tmp0); + + src -= 2; + __lsx_vstelm_w(tmp2, src, 0, 0); + __lsx_vstelm_w(tmp2, src + pitch, 0, 1); + __lsx_vstelm_w(tmp2, src + pitch2, 0, 2); + __lsx_vstelm_w(tmp2, src + pitch3, 0, 3); + src += pitch4; + __lsx_vstelm_w(tmp3, src, 0, 0); + __lsx_vstelm_w(tmp3, src + pitch, 0, 1); + __lsx_vstelm_w(tmp3, src + pitch2, 0, 2); + __lsx_vstelm_w(tmp3, src + pitch3, 0, 3); + src += pitch4; + __lsx_vstelm_w(tmp4, src, 0, 0); + __lsx_vstelm_w(tmp4, src + pitch, 0, 1); + __lsx_vstelm_w(tmp4, src + pitch2, 0, 2); + __lsx_vstelm_w(tmp4, src + pitch3, 0, 3); + src += pitch4; + __lsx_vstelm_w(tmp5, src, 0, 0); + __lsx_vstelm_w(tmp5, src + pitch, 0, 1); + __lsx_vstelm_w(tmp5, src + pitch2, 0, 2); + __lsx_vstelm_w(tmp5, src + pitch3, 0, 3); +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/loopfilter_8_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/loopfilter_8_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/loopfilter_8_lsx.c @@ -0,0 +1,469 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "./vpx_dsp_rtcd.h" +#include "vpx_dsp/loongarch/loopfilter_lsx.h" + +void vpx_lpf_horizontal_8_lsx(uint8_t *dst, int32_t stride, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr) { + __m128i mask, hev, flat, thresh, b_limit, limit; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i p2_out, p1_out, p0_out, q0_out, q1_out, q2_out; + __m128i p2_filter8, p1_filter8, p0_filter8; + __m128i q0_filter8, q1_filter8, q2_filter8; + __m128i p3_l, p2_l, p1_l, p0_l, q3_l, q2_l, q1_l, q0_l; + __m128i zero = __lsx_vldi(0); + + int32_t stride2 = stride << 1; + int32_t stride3 = stride2 + stride; + int32_t stride4 = stride2 << 1; + + /* load vector elements */ + DUP4_ARG2(__lsx_vldx, dst, -stride4, dst, -stride3, dst, -stride2, dst, + -stride, p3, p2, p1, p0); + q0 = __lsx_vld(dst, 0); + DUP2_ARG2(__lsx_vldx, dst, stride, dst, stride2, q1, q2); + q3 = __lsx_vldx(dst, stride3); + + thresh = __lsx_vreplgr2vr_b(*thresh_ptr); + b_limit = __lsx_vreplgr2vr_b(*b_limit_ptr); + limit = __lsx_vreplgr2vr_b(*limit_ptr); + + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + VP9_FLAT4(p3, p2, p0, q0, q2, q3, flat); + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, q1_out); + + flat = __lsx_vilvl_d(zero, flat); + + if (__lsx_bz_v(flat)) { + __lsx_vstelm_d(p1_out, dst - stride2, 0, 0); + __lsx_vstelm_d(p0_out, dst - stride, 0, 0); + __lsx_vstelm_d(q0_out, dst, 0, 0); + __lsx_vstelm_d(q1_out, dst + stride, 0, 0); + } else { + DUP4_ARG2(__lsx_vilvl_b, zero, p3, zero, p2, zero, p1, zero, p0, p3_l, p2_l, + p1_l, p0_l); + DUP4_ARG2(__lsx_vilvl_b, zero, q0, zero, q1, zero, q2, zero, q3, q0_l, q1_l, + q2_l, q3_l); + VP9_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filter8, + p1_filter8, p0_filter8, q0_filter8, q1_filter8, q2_filter8); + + /* convert 16 bit output data into 8 bit */ + DUP4_ARG2(__lsx_vpickev_b, zero, p2_filter8, zero, p1_filter8, zero, + p0_filter8, zero, q0_filter8, p2_filter8, p1_filter8, p0_filter8, + q0_filter8); + DUP2_ARG2(__lsx_vpickev_b, zero, q1_filter8, zero, q2_filter8, q1_filter8, + q2_filter8); + DUP4_ARG3(__lsx_vbitsel_v, p2, p2_filter8, flat, p1_out, p1_filter8, flat, + p0_out, p0_filter8, flat, q0_out, q0_filter8, flat, p2_out, + p1_out, p0_out, q0_out); + DUP2_ARG3(__lsx_vbitsel_v, q1_out, q1_filter8, flat, q2, q2_filter8, flat, + q1_out, q2_out); + dst -= stride3; + + __lsx_vstelm_d(p2_out, dst, 0, 0); + __lsx_vstelm_d(p1_out, dst + stride, 0, 0); + __lsx_vstelm_d(p0_out, dst + stride2, 0, 0); + __lsx_vstelm_d(q0_out, dst + stride3, 0, 0); + + dst += stride4; + __lsx_vstelm_d(q1_out, dst, 0, 0); + dst += stride; + __lsx_vstelm_d(q2_out, dst, 0, 0); + } +} + +void vpx_lpf_horizontal_8_dual_lsx( + uint8_t *dst, int32_t stride, const uint8_t *b_limit0, + const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *b_limit1, + const uint8_t *limit1, const uint8_t *thresh1) { + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i p2_out, p1_out, p0_out, q0_out, q1_out, q2_out; + __m128i flat, mask, hev, tmp, thresh, b_limit, limit; + __m128i p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l; + __m128i p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h; + __m128i p2_filt8_l, p1_filt8_l, p0_filt8_l; + __m128i q0_filt8_l, q1_filt8_l, q2_filt8_l; + __m128i p2_filt8_h, p1_filt8_h, p0_filt8_h; + __m128i q0_filt8_h, q1_filt8_h, q2_filt8_h; + __m128i zero = __lsx_vldi(0); + + int32_t stride2 = stride << 1; + int32_t stride3 = stride2 + stride; + int32_t stride4 = stride2 << 1; + + DUP4_ARG2(__lsx_vldx, dst, -stride4, dst, -stride3, dst, -stride2, dst, + -stride, p3, p2, p1, p0); + q0 = __lsx_vld(dst, 0); + DUP2_ARG2(__lsx_vldx, dst, stride, dst, stride2, q1, q2); + q3 = __lsx_vldx(dst, stride3); + + thresh = __lsx_vreplgr2vr_b(*thresh0); + tmp = __lsx_vreplgr2vr_b(*thresh1); + thresh = __lsx_vilvl_d(tmp, thresh); + + b_limit = __lsx_vreplgr2vr_b(*b_limit0); + tmp = __lsx_vreplgr2vr_b(*b_limit1); + b_limit = __lsx_vilvl_d(tmp, b_limit); + + limit = __lsx_vreplgr2vr_b(*limit0); + tmp = __lsx_vreplgr2vr_b(*limit1); + limit = __lsx_vilvl_d(tmp, limit); + + /* mask and hev */ + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + VP9_FLAT4(p3, p2, p0, q0, q2, q3, flat); + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, q1_out); + + if (__lsx_bz_v(flat)) { + __lsx_vst(p1_out, dst - stride2, 0); + __lsx_vst(p0_out, dst - stride, 0); + __lsx_vst(q0_out, dst, 0); + __lsx_vst(q1_out, dst + stride, 0); + } else { + DUP4_ARG2(__lsx_vilvl_b, zero, p3, zero, p2, zero, p1, zero, p0, p3_l, p2_l, + p1_l, p0_l); + DUP4_ARG2(__lsx_vilvl_b, zero, q0, zero, q1, zero, q2, zero, q3, q0_l, q1_l, + q2_l, q3_l); + VP9_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l, q1_filt8_l, q2_filt8_l); + + DUP4_ARG2(__lsx_vilvh_b, zero, p3, zero, p2, zero, p1, zero, p0, p3_h, p2_h, + p1_h, p0_h); + DUP4_ARG2(__lsx_vilvh_b, zero, q0, zero, q1, zero, q2, zero, q3, q0_h, q1_h, + q2_h, q3_h); + VP9_FILTER8(p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h, p2_filt8_h, + p1_filt8_h, p0_filt8_h, q0_filt8_h, q1_filt8_h, q2_filt8_h); + + /* convert 16 bit output data into 8 bit */ + DUP4_ARG2(__lsx_vpickev_b, p2_filt8_h, p2_filt8_l, p1_filt8_h, p1_filt8_l, + p0_filt8_h, p0_filt8_l, q0_filt8_h, q0_filt8_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l); + DUP2_ARG2(__lsx_vpickev_b, q1_filt8_h, q1_filt8_l, q2_filt8_h, q2_filt8_l, + q1_filt8_l, q2_filt8_l); + + /* store pixel values */ + p2_out = __lsx_vbitsel_v(p2, p2_filt8_l, flat); + p1_out = __lsx_vbitsel_v(p1_out, p1_filt8_l, flat); + p0_out = __lsx_vbitsel_v(p0_out, p0_filt8_l, flat); + q0_out = __lsx_vbitsel_v(q0_out, q0_filt8_l, flat); + q1_out = __lsx_vbitsel_v(q1_out, q1_filt8_l, flat); + q2_out = __lsx_vbitsel_v(q2, q2_filt8_l, flat); + + __lsx_vst(p2_out, dst - stride3, 0); + __lsx_vst(p1_out, dst - stride2, 0); + __lsx_vst(p0_out, dst - stride, 0); + __lsx_vst(q0_out, dst, 0); + __lsx_vst(q1_out, dst + stride, 0); + __lsx_vst(q2_out, dst + stride2, 0); + } +} + +void vpx_lpf_vertical_8_lsx(uint8_t *dst, int32_t stride, + const uint8_t *b_limit_ptr, + const uint8_t *limit_ptr, + const uint8_t *thresh_ptr) { + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i p1_out, p0_out, q0_out, q1_out; + __m128i flat, mask, hev, thresh, b_limit, limit; + __m128i p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l; + __m128i p2_filt8_l, p1_filt8_l, p0_filt8_l; + __m128i q0_filt8_l, q1_filt8_l, q2_filt8_l; + __m128i vec0, vec1, vec2, vec3, vec4; + __m128i zero = __lsx_vldi(0); + + int32_t stride2 = stride << 1; + int32_t stride3 = stride2 + stride; + int32_t stride4 = stride2 << 1; + uint8_t *dst_tmp = dst - 4; + + /* load vector elements */ + p3 = __lsx_vld(dst_tmp, 0); + DUP2_ARG2(__lsx_vldx, dst_tmp, stride, dst_tmp, stride2, p2, p1); + p0 = __lsx_vldx(dst_tmp, stride3); + dst_tmp += stride4; + q0 = __lsx_vld(dst_tmp, 0); + DUP2_ARG2(__lsx_vldx, dst_tmp, stride, dst_tmp, stride2, q1, q2); + q3 = __lsx_vldx(dst_tmp, stride3); + + LSX_TRANSPOSE8x8_B(p3, p2, p1, p0, q0, q1, q2, q3, p3, p2, p1, p0, q0, q1, q2, + q3); + + thresh = __lsx_vreplgr2vr_b(*thresh_ptr); + b_limit = __lsx_vreplgr2vr_b(*b_limit_ptr); + limit = __lsx_vreplgr2vr_b(*limit_ptr); + + /* mask and hev */ + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + /* flat4 */ + VP9_FLAT4(p3, p2, p0, q0, q2, q3, flat); + /* filter4 */ + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, q1_out); + + flat = __lsx_vilvl_d(zero, flat); + + /* if flat is zero for all pixels, then no need to calculate other filter */ + if (__lsx_bz_v(flat)) { + /* Store 4 pixels p1-_q1 */ + DUP2_ARG2(__lsx_vilvl_b, p0_out, p1_out, q1_out, q0_out, vec0, vec1); + vec2 = __lsx_vilvl_h(vec1, vec0); + vec3 = __lsx_vilvh_h(vec1, vec0); + + dst -= 2; + __lsx_vstelm_w(vec2, dst, 0, 0); + __lsx_vstelm_w(vec2, dst + stride, 0, 1); + __lsx_vstelm_w(vec2, dst + stride2, 0, 2); + __lsx_vstelm_w(vec2, dst + stride3, 0, 3); + dst += stride4; + __lsx_vstelm_w(vec3, dst, 0, 0); + __lsx_vstelm_w(vec3, dst + stride, 0, 1); + __lsx_vstelm_w(vec3, dst + stride2, 0, 2); + __lsx_vstelm_w(vec3, dst + stride3, 0, 3); + } else { + DUP4_ARG2(__lsx_vilvl_b, zero, p3, zero, p2, zero, p1, zero, p0, p3_l, p2_l, + p1_l, p0_l); + DUP4_ARG2(__lsx_vilvl_b, zero, q0, zero, q1, zero, q2, zero, q3, q0_l, q1_l, + q2_l, q3_l); + VP9_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l, q1_filt8_l, q2_filt8_l); + /* convert 16 bit output data into 8 bit */ + DUP4_ARG2(__lsx_vpickev_b, p2_filt8_l, p2_filt8_l, p1_filt8_l, p1_filt8_l, + p0_filt8_l, p0_filt8_l, q0_filt8_l, q0_filt8_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l); + DUP2_ARG2(__lsx_vpickev_b, q1_filt8_l, q1_filt8_l, q2_filt8_l, q2_filt8_l, + q1_filt8_l, q2_filt8_l); + /* store pixel values */ + p2 = __lsx_vbitsel_v(p2, p2_filt8_l, flat); + p1 = __lsx_vbitsel_v(p1_out, p1_filt8_l, flat); + p0 = __lsx_vbitsel_v(p0_out, p0_filt8_l, flat); + q0 = __lsx_vbitsel_v(q0_out, q0_filt8_l, flat); + q1 = __lsx_vbitsel_v(q1_out, q1_filt8_l, flat); + q2 = __lsx_vbitsel_v(q2, q2_filt8_l, flat); + + /* Store 6 pixels p2-_q2 */ + DUP2_ARG2(__lsx_vilvl_b, p1, p2, q0, p0, vec0, vec1); + vec2 = __lsx_vilvl_h(vec1, vec0); + vec3 = __lsx_vilvh_h(vec1, vec0); + vec4 = __lsx_vilvl_b(q2, q1); + + dst -= 3; + __lsx_vstelm_w(vec2, dst, 0, 0); + __lsx_vstelm_h(vec4, dst, 4, 0); + dst += stride; + __lsx_vstelm_w(vec2, dst, 0, 1); + __lsx_vstelm_h(vec4, dst, 4, 1); + dst += stride; + __lsx_vstelm_w(vec2, dst, 0, 2); + __lsx_vstelm_h(vec4, dst, 4, 2); + dst += stride; + __lsx_vstelm_w(vec2, dst, 0, 3); + __lsx_vstelm_h(vec4, dst, 4, 3); + dst += stride; + __lsx_vstelm_w(vec3, dst, 0, 0); + __lsx_vstelm_h(vec4, dst, 4, 4); + dst += stride; + __lsx_vstelm_w(vec3, dst, 0, 1); + __lsx_vstelm_h(vec4, dst, 4, 5); + dst += stride; + __lsx_vstelm_w(vec3, dst, 0, 2); + __lsx_vstelm_h(vec4, dst, 4, 6); + dst += stride; + __lsx_vstelm_w(vec3, dst, 0, 3); + __lsx_vstelm_h(vec4, dst, 4, 7); + } +} + +void vpx_lpf_vertical_8_dual_lsx(uint8_t *dst, int32_t stride, + const uint8_t *b_limit0, const uint8_t *limit0, + const uint8_t *thresh0, + const uint8_t *b_limit1, const uint8_t *limit1, + const uint8_t *thresh1) { + uint8_t *dst_tmp = dst - 4; + __m128i p3, p2, p1, p0, q3, q2, q1, q0; + __m128i p1_out, p0_out, q0_out, q1_out; + __m128i flat, mask, hev, thresh, b_limit, limit; + __m128i row4, row5, row6, row7, row12, row13, row14, row15; + __m128i p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l; + __m128i p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h; + __m128i p2_filt8_l, p1_filt8_l, p0_filt8_l; + __m128i q0_filt8_l, q1_filt8_l, q2_filt8_l; + __m128i p2_filt8_h, p1_filt8_h, p0_filt8_h; + __m128i q0_filt8_h, q1_filt8_h, q2_filt8_h; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i zero = __lsx_vldi(0); + int32_t stride2 = stride << 1; + int32_t stride3 = stride2 + stride; + int32_t stride4 = stride2 << 1; + + p0 = __lsx_vld(dst_tmp, 0); + DUP2_ARG2(__lsx_vldx, dst_tmp, stride, dst_tmp, stride2, p1, p2); + p3 = __lsx_vldx(dst_tmp, stride3); + dst_tmp += stride4; + row4 = __lsx_vld(dst_tmp, 0); + DUP2_ARG2(__lsx_vldx, dst_tmp, stride, dst_tmp, stride2, row5, row6); + row7 = __lsx_vldx(dst_tmp, stride3); + dst_tmp += stride4; + + q3 = __lsx_vld(dst_tmp, 0); + DUP2_ARG2(__lsx_vldx, dst_tmp, stride, dst_tmp, stride2, q2, q1); + q0 = __lsx_vldx(dst_tmp, stride3); + dst_tmp += stride4; + row12 = __lsx_vld(dst_tmp, 0); + DUP2_ARG2(__lsx_vldx, dst_tmp, stride, dst_tmp, stride2, row13, row14); + row15 = __lsx_vldx(dst_tmp, stride3); + + /* transpose 16x8 matrix into 8x16 */ + LSX_TRANSPOSE16x8_B(p0, p1, p2, p3, row4, row5, row6, row7, q3, q2, q1, q0, + row12, row13, row14, row15, p3, p2, p1, p0, q0, q1, q2, + q3); + + thresh = __lsx_vreplgr2vr_b(*thresh0); + vec0 = __lsx_vreplgr2vr_b(*thresh1); + thresh = __lsx_vilvl_d(vec0, thresh); + + b_limit = __lsx_vreplgr2vr_b(*b_limit0); + vec0 = __lsx_vreplgr2vr_b(*b_limit1); + b_limit = __lsx_vilvl_d(vec0, b_limit); + + limit = __lsx_vreplgr2vr_b(*limit0); + vec0 = __lsx_vreplgr2vr_b(*limit1); + limit = __lsx_vilvl_d(vec0, limit); + + /* mask and hev */ + LPF_MASK_HEV(p3, p2, p1, p0, q0, q1, q2, q3, limit, b_limit, thresh, hev, + mask, flat); + /* flat4 */ + VP9_FLAT4(p3, p2, p0, q0, q2, q3, flat); + /* filter4 */ + VP9_LPF_FILTER4_4W(p1, p0, q0, q1, mask, hev, p1_out, p0_out, q0_out, q1_out); + /* if flat is zero for all pixels, then no need to calculate other filter */ + if (__lsx_bz_v(flat)) { + DUP2_ARG2(__lsx_vilvl_b, p0_out, p1_out, q1_out, q0_out, vec0, vec1); + vec2 = __lsx_vilvl_h(vec1, vec0); + vec3 = __lsx_vilvh_h(vec1, vec0); + DUP2_ARG2(__lsx_vilvh_b, p0_out, p1_out, q1_out, q0_out, vec0, vec1); + vec4 = __lsx_vilvl_h(vec1, vec0); + vec5 = __lsx_vilvh_h(vec1, vec0); + + dst -= 2; + __lsx_vstelm_w(vec2, dst, 0, 0); + __lsx_vstelm_w(vec2, dst + stride, 0, 1); + __lsx_vstelm_w(vec2, dst + stride2, 0, 2); + __lsx_vstelm_w(vec2, dst + stride3, 0, 3); + dst += stride4; + __lsx_vstelm_w(vec3, dst, 0, 0); + __lsx_vstelm_w(vec3, dst + stride, 0, 1); + __lsx_vstelm_w(vec3, dst + stride2, 0, 2); + __lsx_vstelm_w(vec3, dst + stride3, 0, 3); + dst += stride4; + __lsx_vstelm_w(vec4, dst, 0, 0); + __lsx_vstelm_w(vec4, dst + stride, 0, 1); + __lsx_vstelm_w(vec4, dst + stride2, 0, 2); + __lsx_vstelm_w(vec4, dst + stride3, 0, 3); + dst += stride4; + __lsx_vstelm_w(vec5, dst, 0, 0); + __lsx_vstelm_w(vec5, dst + stride, 0, 1); + __lsx_vstelm_w(vec5, dst + stride2, 0, 2); + __lsx_vstelm_w(vec5, dst + stride3, 0, 3); + } else { + DUP4_ARG2(__lsx_vilvl_b, zero, p3, zero, p2, zero, p1, zero, p0, p3_l, p2_l, + p1_l, p0_l); + DUP4_ARG2(__lsx_vilvl_b, zero, q0, zero, q1, zero, q2, zero, q3, q0_l, q1_l, + q2_l, q3_l); + VP9_FILTER8(p3_l, p2_l, p1_l, p0_l, q0_l, q1_l, q2_l, q3_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l, q1_filt8_l, q2_filt8_l); + DUP4_ARG2(__lsx_vilvh_b, zero, p3, zero, p2, zero, p1, zero, p0, p3_h, p2_h, + p1_h, p0_h); + DUP4_ARG2(__lsx_vilvh_b, zero, q0, zero, q1, zero, q2, zero, q3, q0_h, q1_h, + q2_h, q3_h); + + /* filter8 */ + VP9_FILTER8(p3_h, p2_h, p1_h, p0_h, q0_h, q1_h, q2_h, q3_h, p2_filt8_h, + p1_filt8_h, p0_filt8_h, q0_filt8_h, q1_filt8_h, q2_filt8_h); + + /* convert 16 bit output data into 8 bit */ + DUP4_ARG2(__lsx_vpickev_b, p2_filt8_h, p2_filt8_l, p1_filt8_h, p1_filt8_l, + p0_filt8_h, p0_filt8_l, q0_filt8_h, q0_filt8_l, p2_filt8_l, + p1_filt8_l, p0_filt8_l, q0_filt8_l); + DUP2_ARG2(__lsx_vpickev_b, q1_filt8_h, q1_filt8_l, q2_filt8_h, q2_filt8_l, + q1_filt8_l, q2_filt8_l); + + /* store pixel values */ + p2 = __lsx_vbitsel_v(p2, p2_filt8_l, flat); + p1 = __lsx_vbitsel_v(p1_out, p1_filt8_l, flat); + p0 = __lsx_vbitsel_v(p0_out, p0_filt8_l, flat); + q0 = __lsx_vbitsel_v(q0_out, q0_filt8_l, flat); + q1 = __lsx_vbitsel_v(q1_out, q1_filt8_l, flat); + q2 = __lsx_vbitsel_v(q2, q2_filt8_l, flat); + + DUP2_ARG2(__lsx_vilvl_b, p1, p2, q0, p0, vec0, vec1); + vec3 = __lsx_vilvl_h(vec1, vec0); + vec4 = __lsx_vilvh_h(vec1, vec0); + DUP2_ARG2(__lsx_vilvh_b, p1, p2, q0, p0, vec0, vec1); + vec6 = __lsx_vilvl_h(vec1, vec0); + vec7 = __lsx_vilvh_h(vec1, vec0); + vec2 = __lsx_vilvl_b(q2, q1); + vec5 = __lsx_vilvh_b(q2, q1); + + dst -= 3; + __lsx_vstelm_w(vec3, dst, 0, 0); + __lsx_vstelm_h(vec2, dst, 4, 0); + dst += stride; + __lsx_vstelm_w(vec3, dst, 0, 1); + __lsx_vstelm_h(vec2, dst, 4, 1); + dst += stride; + __lsx_vstelm_w(vec3, dst, 0, 2); + __lsx_vstelm_h(vec2, dst, 4, 2); + dst += stride; + __lsx_vstelm_w(vec3, dst, 0, 3); + __lsx_vstelm_h(vec2, dst, 4, 3); + dst += stride; + __lsx_vstelm_w(vec4, dst, 0, 0); + __lsx_vstelm_h(vec2, dst, 4, 4); + dst += stride; + __lsx_vstelm_w(vec4, dst, 0, 1); + __lsx_vstelm_h(vec2, dst, 4, 5); + dst += stride; + __lsx_vstelm_w(vec4, dst, 0, 2); + __lsx_vstelm_h(vec2, dst, 4, 6); + dst += stride; + __lsx_vstelm_w(vec4, dst, 0, 3); + __lsx_vstelm_h(vec2, dst, 4, 7); + dst += stride; + __lsx_vstelm_w(vec6, dst, 0, 0); + __lsx_vstelm_h(vec5, dst, 4, 0); + dst += stride; + __lsx_vstelm_w(vec6, dst, 0, 1); + __lsx_vstelm_h(vec5, dst, 4, 1); + dst += stride; + __lsx_vstelm_w(vec6, dst, 0, 2); + __lsx_vstelm_h(vec5, dst, 4, 2); + dst += stride; + __lsx_vstelm_w(vec6, dst, 0, 3); + __lsx_vstelm_h(vec5, dst, 4, 3); + dst += stride; + __lsx_vstelm_w(vec7, dst, 0, 0); + __lsx_vstelm_h(vec5, dst, 4, 4); + dst += stride; + __lsx_vstelm_w(vec7, dst, 0, 1); + __lsx_vstelm_h(vec5, dst, 4, 5); + dst += stride; + __lsx_vstelm_w(vec7, dst, 0, 2); + __lsx_vstelm_h(vec5, dst, 4, 6); + dst += stride; + __lsx_vstelm_w(vec7, dst, 0, 3); + __lsx_vstelm_h(vec5, dst, 4, 7); + } +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/loopfilter_lsx.h b/media/libvpx/libvpx/vpx_dsp/loongarch/loopfilter_lsx.h new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/loopfilter_lsx.h @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VPX_VPX_DSP_LOONGARCH_LOOPFILTER_LSX_H_ +#define VPX_VPX_DSP_LOONGARCH_LOOPFILTER_LSX_H_ + +#include "vpx_util/loongson_intrinsics.h" + +#define LPF_MASK_HEV(p3_in, p2_in, p1_in, p0_in, q0_in, q1_in, q2_in, q3_in, \ + limit_in, b_limit_in, thresh_in, hev_out, mask_out, \ + flat_out) \ + { \ + __m128i p3_asub_p2_m, p2_asub_p1_m, p1_asub_p0_m, q1_asub_q0_m; \ + __m128i p1_asub_q1_m, p0_asub_q0_m, q3_asub_q2_m, q2_asub_q1_m; \ + \ + /* absolute subtraction of pixel values */ \ + p3_asub_p2_m = __lsx_vabsd_bu(p3_in, p2_in); \ + p2_asub_p1_m = __lsx_vabsd_bu(p2_in, p1_in); \ + p1_asub_p0_m = __lsx_vabsd_bu(p1_in, p0_in); \ + q1_asub_q0_m = __lsx_vabsd_bu(q1_in, q0_in); \ + q2_asub_q1_m = __lsx_vabsd_bu(q2_in, q1_in); \ + q3_asub_q2_m = __lsx_vabsd_bu(q3_in, q2_in); \ + p0_asub_q0_m = __lsx_vabsd_bu(p0_in, q0_in); \ + p1_asub_q1_m = __lsx_vabsd_bu(p1_in, q1_in); \ + \ + /* calculation of hev */ \ + flat_out = __lsx_vmax_bu(p1_asub_p0_m, q1_asub_q0_m); \ + hev_out = __lsx_vslt_bu(thresh_in, flat_out); \ + \ + /* calculation of mask */ \ + p0_asub_q0_m = __lsx_vsadd_bu(p0_asub_q0_m, p0_asub_q0_m); \ + p1_asub_q1_m = __lsx_vsrli_b(p1_asub_q1_m, 1); \ + p0_asub_q0_m = __lsx_vsadd_bu(p0_asub_q0_m, p1_asub_q1_m); \ + mask_out = __lsx_vslt_bu(b_limit_in, p0_asub_q0_m); \ + mask_out = __lsx_vmax_bu(flat_out, mask_out); \ + p3_asub_p2_m = __lsx_vmax_bu(p3_asub_p2_m, p2_asub_p1_m); \ + mask_out = __lsx_vmax_bu(p3_asub_p2_m, mask_out); \ + q2_asub_q1_m = __lsx_vmax_bu(q2_asub_q1_m, q3_asub_q2_m); \ + mask_out = __lsx_vmax_bu(q2_asub_q1_m, mask_out); \ + \ + mask_out = __lsx_vslt_bu(limit_in, mask_out); \ + mask_out = __lsx_vxori_b(mask_out, 0xff); \ + } + +#define VP9_FLAT4(p3_in, p2_in, p0_in, q0_in, q2_in, q3_in, flat_out) \ + { \ + __m128i p2_asub_p0, q2_asub_q0, p3_asub_p0, q3_asub_q0; \ + __m128i flat4_tmp = __lsx_vldi(1); \ + \ + DUP4_ARG2(__lsx_vabsd_bu, p2_in, p0_in, q2_in, q0_in, p3_in, p0_in, q3_in, \ + q0_in, p2_asub_p0, q2_asub_q0, p3_asub_p0, q3_asub_q0); \ + p2_asub_p0 = __lsx_vmax_bu(p2_asub_p0, q2_asub_q0); \ + flat_out = __lsx_vmax_bu(p2_asub_p0, flat_out); \ + p3_asub_p0 = __lsx_vmax_bu(p3_asub_p0, q3_asub_q0); \ + flat_out = __lsx_vmax_bu(p3_asub_p0, flat_out); \ + \ + flat_out = __lsx_vslt_bu(flat4_tmp, flat_out); \ + flat_out = __lsx_vxori_b(flat_out, 0xff); \ + flat_out = flat_out & (mask); \ + } + +#define VP9_FLAT5(p7_in, p6_in, p5_in, p4_in, p0_in, q0_in, q4_in, q5_in, \ + q6_in, q7_in, flat_in, flat2_out) \ + { \ + __m128i flat5_tmp = __lsx_vldi(1); \ + __m128i p4_asub_p0, q4_asub_q0, p5_asub_p0, q5_asub_q0; \ + __m128i p6_asub_p0, q6_asub_q0, p7_asub_p0, q7_asub_q0; \ + DUP4_ARG2(__lsx_vabsd_bu, p4_in, p0_in, q4_in, q0_in, p5_in, p0_in, q5_in, \ + q0_in, p4_asub_p0, q4_asub_q0, p5_asub_p0, q5_asub_q0); \ + DUP4_ARG2(__lsx_vabsd_bu, p6_in, p0_in, q6_in, q0_in, p7_in, p0_in, q7_in, \ + q0_in, p6_asub_p0, q6_asub_q0, p7_asub_p0, q7_asub_q0); \ + \ + DUP2_ARG2(__lsx_vmax_bu, p4_asub_p0, q4_asub_q0, p5_asub_p0, q5_asub_q0, \ + p4_asub_p0, flat2_out); \ + flat2_out = __lsx_vmax_bu(p4_asub_p0, flat2_out); \ + p6_asub_p0 = __lsx_vmax_bu(p6_asub_p0, q6_asub_q0); \ + flat2_out = __lsx_vmax_bu(p6_asub_p0, flat2_out); \ + p7_asub_p0 = __lsx_vmax_bu(p7_asub_p0, q7_asub_q0); \ + flat2_out = __lsx_vmax_bu(p7_asub_p0, flat2_out); \ + flat2_out = __lsx_vslt_bu(flat5_tmp, flat2_out); \ + flat2_out = __lsx_vxori_b(flat2_out, 0xff); \ + flat2_out = flat2_out & flat_in; \ + } + +#define VP9_LPF_FILTER4_4W(p1_in, p0_in, q0_in, q1_in, mask, hev, p1_out, \ + p0_out, q0_out, q1_out) \ + { \ + __m128i p1_m, p0_m, q0_m, q1_m, filt, q0_sub_p0, t1, t2; \ + const __m128i cnst4b = __lsx_vldi(4); \ + const __m128i cnst3b = __lsx_vldi(3); \ + DUP4_ARG2(__lsx_vxori_b, p1_in, 0x80, p0_in, 0x80, q0_in, 0x80, q1_in, \ + 0x80, p1_m, p0_m, q0_m, q1_m); \ + filt = __lsx_vssub_b(p1_m, q1_m); \ + filt &= hev; \ + \ + q0_sub_p0 = __lsx_vssub_b(q0_m, p0_m); \ + filt = __lsx_vsadd_b(filt, q0_sub_p0); \ + filt = __lsx_vsadd_b(filt, q0_sub_p0); \ + filt = __lsx_vsadd_b(filt, q0_sub_p0); \ + filt &= mask; \ + DUP2_ARG2(__lsx_vsadd_b, filt, cnst4b, filt, cnst3b, t1, t2); \ + DUP2_ARG2(__lsx_vsrai_b, t1, 3, t2, 3, t1, t2); \ + \ + q0_m = __lsx_vssub_b(q0_m, t1); \ + p0_m = __lsx_vsadd_b(p0_m, t2); \ + DUP2_ARG2(__lsx_vxori_b, q0_m, 0x80, p0_m, 0x80, q0_out, p0_out); \ + \ + filt = __lsx_vsrari_b(t1, 1); \ + hev = __lsx_vxori_b(hev, 0xff); \ + filt &= hev; \ + q1_m = __lsx_vssub_b(q1_m, filt); \ + p1_m = __lsx_vsadd_b(p1_m, filt); \ + DUP2_ARG2(__lsx_vxori_b, q1_m, 0x80, p1_m, 0x80, q1_out, p1_out); \ + } + +#define VP9_FILTER8(p3_in, p2_in, p1_in, p0_in, q0_in, q1_in, q2_in, q3_in, \ + p2_filt8_out, p1_filt8_out, p0_filt8_out, q0_filt8_out, \ + q1_filt8_out, q2_filt8_out) \ + { \ + __m128i tmp_filt8_0, tmp_filt8_1, tmp_filt8_2; \ + \ + tmp_filt8_2 = __lsx_vadd_h(p2_in, p1_in); \ + tmp_filt8_2 = __lsx_vadd_h(tmp_filt8_2, p0_in); \ + tmp_filt8_0 = __lsx_vslli_h(p3_in, 1); \ + \ + tmp_filt8_0 = __lsx_vadd_h(tmp_filt8_0, tmp_filt8_2); \ + tmp_filt8_0 = __lsx_vadd_h(tmp_filt8_0, q0_in); \ + tmp_filt8_1 = __lsx_vadd_h(tmp_filt8_0, p3_in); \ + tmp_filt8_1 = __lsx_vadd_h(tmp_filt8_1, p2_in); \ + p2_filt8_out = __lsx_vsrari_h(tmp_filt8_1, 3); \ + \ + tmp_filt8_1 = __lsx_vadd_h(tmp_filt8_0, p1_in); \ + tmp_filt8_1 = __lsx_vadd_h(tmp_filt8_1, q1_in); \ + p1_filt8_out = __lsx_vsrari_h(tmp_filt8_1, 3); \ + \ + tmp_filt8_1 = __lsx_vadd_h(q2_in, q1_in); \ + tmp_filt8_1 = __lsx_vadd_h(tmp_filt8_1, q0_in); \ + tmp_filt8_2 = __lsx_vadd_h(tmp_filt8_2, tmp_filt8_1); \ + tmp_filt8_0 = __lsx_vadd_h(tmp_filt8_2, p0_in); \ + tmp_filt8_0 = __lsx_vadd_h(tmp_filt8_0, p3_in); \ + p0_filt8_out = __lsx_vsrari_h(tmp_filt8_0, 3); \ + \ + tmp_filt8_0 = __lsx_vadd_h(q2_in, q3_in); \ + tmp_filt8_0 = __lsx_vadd_h(p0_in, tmp_filt8_0); \ + tmp_filt8_0 = __lsx_vadd_h(tmp_filt8_0, tmp_filt8_1); \ + tmp_filt8_1 = __lsx_vadd_h(q3_in, q3_in); \ + tmp_filt8_1 = __lsx_vadd_h(tmp_filt8_1, tmp_filt8_0); \ + q2_filt8_out = __lsx_vsrari_h(tmp_filt8_1, 3); \ + \ + tmp_filt8_0 = __lsx_vadd_h(tmp_filt8_2, q3_in); \ + tmp_filt8_1 = __lsx_vadd_h(tmp_filt8_0, q0_in); \ + q0_filt8_out = __lsx_vsrari_h(tmp_filt8_1, 3); \ + \ + tmp_filt8_1 = __lsx_vsub_h(tmp_filt8_0, p2_in); \ + tmp_filt8_0 = __lsx_vadd_h(q1_in, q3_in); \ + tmp_filt8_1 = __lsx_vadd_h(tmp_filt8_0, tmp_filt8_1); \ + q1_filt8_out = __lsx_vsrari_h(tmp_filt8_1, 3); \ + } + +#endif // VPX_VPX_DSP_LOONGARCH_LOOPFILTER_LSX_H_ diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/sad_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/sad_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/sad_lsx.c @@ -0,0 +1,672 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "./vpx_dsp_rtcd.h" +#include "vpx_util/loongson_intrinsics.h" + +#define SAD_UB2_UH(in0, in1, ref0, ref1) \ + ({ \ + __m128i diff0_m, diff1_m, sad_m0; \ + __m128i sad_m = __lsx_vldi(0); \ + \ + diff0_m = __lsx_vabsd_bu(in0, ref0); \ + diff1_m = __lsx_vabsd_bu(in1, ref1); \ + \ + sad_m0 = __lsx_vhaddw_hu_bu(diff0_m, diff0_m); \ + sad_m = __lsx_vadd_h(sad_m, sad_m0); \ + sad_m0 = __lsx_vhaddw_hu_bu(diff1_m, diff1_m); \ + sad_m = __lsx_vadd_h(sad_m, sad_m0); \ + \ + sad_m; \ + }) + +#define HADD_UW_U32(in) \ + ({ \ + __m128i res0_m; \ + uint32_t sum_m; \ + res0_m = __lsx_vhaddw_du_wu(in, in); \ + res0_m = __lsx_vhaddw_qu_du(res0_m, res0_m); \ + sum_m = __lsx_vpickve2gr_w(res0_m, 0); \ + sum_m; \ + }) + +#define HADD_UH_U32(in) \ + ({ \ + __m128i res_m; \ + uint32_t sum_m; \ + res_m = __lsx_vhaddw_wu_hu(in, in); \ + sum_m = HADD_UW_U32(res_m); \ + sum_m; \ + }) + +#define HADD_SW_S32(in) \ + ({ \ + __m128i res0_m; \ + int32_t sum_m; \ + \ + res0_m = __lsx_vhaddw_d_w(in, in); \ + res0_m = __lsx_vhaddw_q_d(res0_m, res0_m); \ + sum_m = __lsx_vpickve2gr_w(res0_m, 0); \ + sum_m; \ + }) + +static uint32_t sad_16width_lsx(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + int32_t height) { + int32_t ht_cnt = (height >> 2); + __m128i src0, src1, ref0, ref1, sad_tmp; + __m128i sad = __lsx_vldi(0); + int32_t src_stride2 = src_stride << 1; + int32_t ref_stride2 = ref_stride << 1; + + for (; ht_cnt--;) { + DUP2_ARG2(__lsx_vld, src, 0, ref, 0, src0, ref0); + DUP2_ARG2(__lsx_vldx, src, src_stride, ref, ref_stride, src1, ref1); + src += src_stride2; + ref += ref_stride2; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad = __lsx_vadd_h(sad, sad_tmp); + + DUP2_ARG2(__lsx_vld, src, 0, ref, 0, src0, ref0); + DUP2_ARG2(__lsx_vldx, src, src_stride, ref, ref_stride, src1, ref1); + src += src_stride2; + ref += ref_stride2; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad = __lsx_vadd_h(sad, sad_tmp); + } + return HADD_UH_U32(sad); +} + +static uint32_t sad_32width_lsx(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + int32_t height) { + int32_t ht_cnt = (height >> 2); + __m128i src0, src1, ref0, ref1; + __m128i sad_tmp; + __m128i sad = __lsx_vldi(0); + + for (; ht_cnt--;) { + DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1); + src += src_stride; + DUP2_ARG2(__lsx_vld, ref, 0, ref, 16, ref0, ref1); + ref += ref_stride; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad = __lsx_vadd_h(sad, sad_tmp); + + DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1); + src += src_stride; + DUP2_ARG2(__lsx_vld, ref, 0, ref, 16, ref0, ref1); + ref += ref_stride; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad = __lsx_vadd_h(sad, sad_tmp); + + DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1); + src += src_stride; + DUP2_ARG2(__lsx_vld, ref, 0, ref, 16, ref0, ref1); + ref += ref_stride; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad = __lsx_vadd_h(sad, sad_tmp); + + DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1); + src += src_stride; + DUP2_ARG2(__lsx_vld, ref, 0, ref, 16, ref0, ref1); + ref += ref_stride; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad = __lsx_vadd_h(sad, sad_tmp); + } + return HADD_UH_U32(sad); +} + +static uint32_t sad_64width_lsx(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + int32_t height) { + int32_t ht_cnt = (height >> 1); + uint32_t sad = 0; + __m128i src0, src1, src2, src3; + __m128i ref0, ref1, ref2, ref3; + __m128i sad_tmp; + __m128i sad0 = __lsx_vldi(0); + __m128i sad1 = sad0; + + for (; ht_cnt--;) { + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2, + src3); + src += src_stride; + DUP4_ARG2(__lsx_vld, ref, 0, ref, 16, ref, 32, ref, 48, ref0, ref1, ref2, + ref3); + ref += ref_stride; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad0 = __lsx_vadd_h(sad0, sad_tmp); + sad_tmp = SAD_UB2_UH(src2, src3, ref2, ref3); + sad1 = __lsx_vadd_h(sad1, sad_tmp); + + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2, + src3); + src += src_stride; + DUP4_ARG2(__lsx_vld, ref, 0, ref, 16, ref, 32, ref, 48, ref0, ref1, ref2, + ref3); + ref += ref_stride; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad0 = __lsx_vadd_h(sad0, sad_tmp); + sad_tmp = SAD_UB2_UH(src2, src3, ref2, ref3); + sad1 = __lsx_vadd_h(sad1, sad_tmp); + } + + sad = HADD_UH_U32(sad0); + sad += HADD_UH_U32(sad1); + + return sad; +} + +static void sad_8width_x4d_lsx(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *const aref_ptr[], + int32_t ref_stride, int32_t height, + uint32_t *sad_array) { + int32_t ht_cnt = (height >> 2); + uint8_t *ref0_ptr, *ref1_ptr, *ref2_ptr, *ref3_ptr; + __m128i src0, src1, src2, src3, sad_tmp; + __m128i ref0, ref1, ref2, ref3, ref4, ref5, ref6, ref7; + __m128i ref8, ref9, ref10, ref11, ref12, ref13, ref14, ref15; + __m128i sad0 = __lsx_vldi(0); + __m128i sad1 = sad0; + __m128i sad2 = sad0; + __m128i sad3 = sad0; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + int32_t ref_stride2 = ref_stride << 1; + int32_t ref_stride3 = ref_stride2 + ref_stride; + int32_t ref_stride4 = ref_stride2 << 1; + + ref0_ptr = aref_ptr[0]; + ref1_ptr = aref_ptr[1]; + ref2_ptr = aref_ptr[2]; + ref3_ptr = aref_ptr[3]; + + for (; ht_cnt--;) { + src0 = __lsx_vld(src_ptr, 0); + DUP2_ARG2(__lsx_vldx, src_ptr, src_stride, src_ptr, src_stride2, src1, + src2); + src3 = __lsx_vldx(src_ptr, src_stride3); + src_ptr += src_stride4; + ref0 = __lsx_vld(ref0_ptr, 0); + DUP2_ARG2(__lsx_vldx, ref0_ptr, ref_stride, ref0_ptr, ref_stride2, ref1, + ref2); + ref3 = __lsx_vldx(ref0_ptr, ref_stride3); + ref0_ptr += ref_stride4; + ref4 = __lsx_vld(ref1_ptr, 0); + DUP2_ARG2(__lsx_vldx, ref1_ptr, ref_stride, ref1_ptr, ref_stride2, ref5, + ref6); + ref7 = __lsx_vldx(ref1_ptr, ref_stride3); + ref1_ptr += ref_stride4; + ref8 = __lsx_vld(ref2_ptr, 0); + DUP2_ARG2(__lsx_vldx, ref2_ptr, ref_stride, ref2_ptr, ref_stride2, ref9, + ref10); + ref11 = __lsx_vldx(ref2_ptr, ref_stride3); + ref2_ptr += ref_stride4; + ref12 = __lsx_vld(ref3_ptr, 0); + DUP2_ARG2(__lsx_vldx, ref3_ptr, ref_stride, ref3_ptr, ref_stride2, ref13, + ref14); + ref15 = __lsx_vldx(ref3_ptr, ref_stride3); + ref3_ptr += ref_stride4; + + DUP2_ARG2(__lsx_vpickev_d, src1, src0, src3, src2, src0, src1); + DUP2_ARG2(__lsx_vpickev_d, ref1, ref0, ref3, ref2, ref0, ref1); + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad0 = __lsx_vadd_h(sad0, sad_tmp); + + DUP2_ARG2(__lsx_vpickev_d, ref5, ref4, ref7, ref6, ref0, ref1); + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad1 = __lsx_vadd_h(sad1, sad_tmp); + + DUP2_ARG2(__lsx_vpickev_d, ref9, ref8, ref11, ref10, ref0, ref1); + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad2 = __lsx_vadd_h(sad2, sad_tmp); + + DUP2_ARG2(__lsx_vpickev_d, ref13, ref12, ref15, ref14, ref0, ref1); + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad3 = __lsx_vadd_h(sad3, sad_tmp); + } + sad_array[0] = HADD_UH_U32(sad0); + sad_array[1] = HADD_UH_U32(sad1); + sad_array[2] = HADD_UH_U32(sad2); + sad_array[3] = HADD_UH_U32(sad3); +} + +static void sad_16width_x4d_lsx(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *const aref_ptr[], + int32_t ref_stride, int32_t height, + uint32_t *sad_array) { + int32_t ht_cnt = (height >> 1); + const uint8_t *ref0_ptr, *ref1_ptr, *ref2_ptr, *ref3_ptr; + __m128i src, ref0, ref1, ref2, ref3, diff, sad_tmp; + __m128i sad0 = __lsx_vldi(0); + __m128i sad1 = sad0; + __m128i sad2 = sad0; + __m128i sad3 = sad0; + + ref0_ptr = aref_ptr[0]; + ref1_ptr = aref_ptr[1]; + ref2_ptr = aref_ptr[2]; + ref3_ptr = aref_ptr[3]; + + for (; ht_cnt--;) { + src = __lsx_vld(src_ptr, 0); + src_ptr += src_stride; + ref0 = __lsx_vld(ref0_ptr, 0); + ref0_ptr += ref_stride; + ref1 = __lsx_vld(ref1_ptr, 0); + ref1_ptr += ref_stride; + ref2 = __lsx_vld(ref2_ptr, 0); + ref2_ptr += ref_stride; + ref3 = __lsx_vld(ref3_ptr, 0); + ref3_ptr += ref_stride; + + diff = __lsx_vabsd_bu(src, ref0); + sad_tmp = __lsx_vhaddw_hu_bu(diff, diff); + sad0 = __lsx_vadd_h(sad0, sad_tmp); + diff = __lsx_vabsd_bu(src, ref1); + sad_tmp = __lsx_vhaddw_hu_bu(diff, diff); + sad1 = __lsx_vadd_h(sad1, sad_tmp); + diff = __lsx_vabsd_bu(src, ref2); + sad_tmp = __lsx_vhaddw_hu_bu(diff, diff); + sad2 = __lsx_vadd_h(sad2, sad_tmp); + diff = __lsx_vabsd_bu(src, ref3); + sad_tmp = __lsx_vhaddw_hu_bu(diff, diff); + sad3 = __lsx_vadd_h(sad3, sad_tmp); + + src = __lsx_vld(src_ptr, 0); + src_ptr += src_stride; + ref0 = __lsx_vld(ref0_ptr, 0); + ref0_ptr += ref_stride; + ref1 = __lsx_vld(ref1_ptr, 0); + ref1_ptr += ref_stride; + ref2 = __lsx_vld(ref2_ptr, 0); + ref2_ptr += ref_stride; + ref3 = __lsx_vld(ref3_ptr, 0); + ref3_ptr += ref_stride; + + diff = __lsx_vabsd_bu(src, ref0); + sad_tmp = __lsx_vhaddw_hu_bu(diff, diff); + sad0 = __lsx_vadd_h(sad0, sad_tmp); + diff = __lsx_vabsd_bu(src, ref1); + sad_tmp = __lsx_vhaddw_hu_bu(diff, diff); + sad1 = __lsx_vadd_h(sad1, sad_tmp); + diff = __lsx_vabsd_bu(src, ref2); + sad_tmp = __lsx_vhaddw_hu_bu(diff, diff); + sad2 = __lsx_vadd_h(sad2, sad_tmp); + diff = __lsx_vabsd_bu(src, ref3); + sad_tmp = __lsx_vhaddw_hu_bu(diff, diff); + sad3 = __lsx_vadd_h(sad3, sad_tmp); + } + sad_array[0] = HADD_UH_U32(sad0); + sad_array[1] = HADD_UH_U32(sad1); + sad_array[2] = HADD_UH_U32(sad2); + sad_array[3] = HADD_UH_U32(sad3); +} + +static void sad_32width_x4d_lsx(const uint8_t *src, int32_t src_stride, + const uint8_t *const aref_ptr[], + int32_t ref_stride, int32_t height, + uint32_t *sad_array) { + const uint8_t *ref0_ptr, *ref1_ptr, *ref2_ptr, *ref3_ptr; + int32_t ht_cnt = height; + __m128i src0, src1, ref0, ref1, sad_tmp; + __m128i sad0 = __lsx_vldi(0); + __m128i sad1 = sad0; + __m128i sad2 = sad0; + __m128i sad3 = sad0; + + ref0_ptr = aref_ptr[0]; + ref1_ptr = aref_ptr[1]; + ref2_ptr = aref_ptr[2]; + ref3_ptr = aref_ptr[3]; + + for (; ht_cnt--;) { + DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1); + src += src_stride; + + DUP2_ARG2(__lsx_vld, ref0_ptr, 0, ref0_ptr, 16, ref0, ref1); + ref0_ptr += ref_stride; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad0 = __lsx_vadd_h(sad0, sad_tmp); + + DUP2_ARG2(__lsx_vld, ref1_ptr, 0, ref1_ptr, 16, ref0, ref1); + ref1_ptr += ref_stride; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad1 = __lsx_vadd_h(sad1, sad_tmp); + + DUP2_ARG2(__lsx_vld, ref2_ptr, 0, ref2_ptr, 16, ref0, ref1); + ref2_ptr += ref_stride; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad2 = __lsx_vadd_h(sad2, sad_tmp); + + DUP2_ARG2(__lsx_vld, ref3_ptr, 0, ref3_ptr, 16, ref0, ref1); + ref3_ptr += ref_stride; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad3 = __lsx_vadd_h(sad3, sad_tmp); + } + sad_array[0] = HADD_UH_U32(sad0); + sad_array[1] = HADD_UH_U32(sad1); + sad_array[2] = HADD_UH_U32(sad2); + sad_array[3] = HADD_UH_U32(sad3); +} + +static void sad_64width_x4d_lsx(const uint8_t *src, int32_t src_stride, + const uint8_t *const aref_ptr[], + int32_t ref_stride, int32_t height, + uint32_t *sad_array) { + const uint8_t *ref0_ptr, *ref1_ptr, *ref2_ptr, *ref3_ptr; + int32_t ht_cnt = height; + __m128i src0, src1, src2, src3; + __m128i ref0, ref1, ref2, ref3; + __m128i sad, sad_tmp; + + __m128i sad0_0 = __lsx_vldi(0); + __m128i sad0_1 = sad0_0; + __m128i sad1_0 = sad0_0; + __m128i sad1_1 = sad0_0; + __m128i sad2_0 = sad0_0; + __m128i sad2_1 = sad0_0; + __m128i sad3_0 = sad0_0; + __m128i sad3_1 = sad0_0; + + ref0_ptr = aref_ptr[0]; + ref1_ptr = aref_ptr[1]; + ref2_ptr = aref_ptr[2]; + ref3_ptr = aref_ptr[3]; + + for (; ht_cnt--;) { + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2, + src3); + src += src_stride; + + DUP4_ARG2(__lsx_vld, ref0_ptr, 0, ref0_ptr, 16, ref0_ptr, 32, ref0_ptr, 48, + ref0, ref1, ref2, ref3); + ref0_ptr += ref_stride; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad0_0 = __lsx_vadd_h(sad0_0, sad_tmp); + sad_tmp = SAD_UB2_UH(src2, src3, ref2, ref3); + sad0_1 = __lsx_vadd_h(sad0_1, sad_tmp); + + DUP4_ARG2(__lsx_vld, ref1_ptr, 0, ref1_ptr, 16, ref1_ptr, 32, ref1_ptr, 48, + ref0, ref1, ref2, ref3); + ref1_ptr += ref_stride; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad1_0 = __lsx_vadd_h(sad1_0, sad_tmp); + sad_tmp = SAD_UB2_UH(src2, src3, ref2, ref3); + sad1_1 = __lsx_vadd_h(sad1_1, sad_tmp); + + DUP4_ARG2(__lsx_vld, ref2_ptr, 0, ref2_ptr, 16, ref2_ptr, 32, ref2_ptr, 48, + ref0, ref1, ref2, ref3); + ref2_ptr += ref_stride; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad2_0 = __lsx_vadd_h(sad2_0, sad_tmp); + sad_tmp = SAD_UB2_UH(src2, src3, ref2, ref3); + sad2_1 = __lsx_vadd_h(sad2_1, sad_tmp); + + DUP4_ARG2(__lsx_vld, ref3_ptr, 0, ref3_ptr, 16, ref3_ptr, 32, ref3_ptr, 48, + ref0, ref1, ref2, ref3); + ref3_ptr += ref_stride; + sad_tmp = SAD_UB2_UH(src0, src1, ref0, ref1); + sad3_0 = __lsx_vadd_h(sad3_0, sad_tmp); + sad_tmp = SAD_UB2_UH(src2, src3, ref2, ref3); + sad3_1 = __lsx_vadd_h(sad3_1, sad_tmp); + } + sad = __lsx_vhaddw_wu_hu(sad0_0, sad0_0); + sad_tmp = __lsx_vhaddw_wu_hu(sad0_1, sad0_1); + sad = __lsx_vadd_w(sad, sad_tmp); + sad_array[0] = HADD_UW_U32(sad); + + sad = __lsx_vhaddw_wu_hu(sad1_0, sad1_0); + sad_tmp = __lsx_vhaddw_wu_hu(sad1_1, sad1_1); + sad = __lsx_vadd_w(sad, sad_tmp); + sad_array[1] = HADD_UW_U32(sad); + + sad = __lsx_vhaddw_wu_hu(sad2_0, sad2_0); + sad_tmp = __lsx_vhaddw_wu_hu(sad2_1, sad2_1); + sad = __lsx_vadd_w(sad, sad_tmp); + sad_array[2] = HADD_UW_U32(sad); + + sad = __lsx_vhaddw_wu_hu(sad3_0, sad3_0); + sad_tmp = __lsx_vhaddw_wu_hu(sad3_1, sad3_1); + sad = __lsx_vadd_w(sad, sad_tmp); + sad_array[3] = HADD_UW_U32(sad); +} + +static uint32_t avgsad_32width_lsx(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + int32_t height, const uint8_t *sec_pred) { + int32_t ht_cnt = (height >> 2); + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i ref0, ref1, ref2, ref3, ref4, ref5, ref6, ref7; + __m128i pred0, pred1, pred2, pred3, pred4, pred5, pred6, pred7; + __m128i comp0, comp1, sad_tmp; + __m128i sad = __lsx_vldi(0); + uint8_t *src_tmp, *ref_tmp; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + int32_t ref_stride2 = ref_stride << 1; + int32_t ref_stride3 = ref_stride2 + ref_stride; + int32_t ref_stride4 = ref_stride2 << 1; + + for (; ht_cnt--;) { + src_tmp = (uint8_t *)src + 16; + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src4); + src6 = __lsx_vldx(src, src_stride3); + src1 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src3, + src5); + src7 = __lsx_vldx(src_tmp, src_stride3); + src += src_stride4; + + ref_tmp = (uint8_t *)ref + 16; + ref0 = __lsx_vld(ref, 0); + DUP2_ARG2(__lsx_vldx, ref, ref_stride, ref, ref_stride2, ref2, ref4); + ref6 = __lsx_vldx(ref, ref_stride3); + ref1 = __lsx_vld(ref_tmp, 0); + DUP2_ARG2(__lsx_vldx, ref_tmp, ref_stride, ref_tmp, ref_stride2, ref3, + ref5); + ref7 = __lsx_vldx(ref_tmp, ref_stride3); + ref += ref_stride4; + + DUP4_ARG2(__lsx_vld, sec_pred, 0, sec_pred, 32, sec_pred, 64, sec_pred, 96, + pred0, pred2, pred4, pred6); + DUP4_ARG2(__lsx_vld, sec_pred, 16, sec_pred, 48, sec_pred, 80, sec_pred, + 112, pred1, pred3, pred5, pred7); + sec_pred += 128; + + DUP2_ARG2(__lsx_vavgr_bu, pred0, ref0, pred1, ref1, comp0, comp1); + sad_tmp = SAD_UB2_UH(src0, src1, comp0, comp1); + sad = __lsx_vadd_h(sad, sad_tmp); + DUP2_ARG2(__lsx_vavgr_bu, pred2, ref2, pred3, ref3, comp0, comp1); + sad_tmp = SAD_UB2_UH(src2, src3, comp0, comp1); + sad = __lsx_vadd_h(sad, sad_tmp); + DUP2_ARG2(__lsx_vavgr_bu, pred4, ref4, pred5, ref5, comp0, comp1); + sad_tmp = SAD_UB2_UH(src4, src5, comp0, comp1); + sad = __lsx_vadd_h(sad, sad_tmp); + DUP2_ARG2(__lsx_vavgr_bu, pred6, ref6, pred7, ref7, comp0, comp1); + sad_tmp = SAD_UB2_UH(src6, src7, comp0, comp1); + sad = __lsx_vadd_h(sad, sad_tmp); + } + + return HADD_UH_U32(sad); +} + +static uint32_t avgsad_64width_lsx(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + int32_t height, const uint8_t *sec_pred) { + int32_t ht_cnt = (height >> 2); + __m128i src0, src1, src2, src3, ref0, ref1, ref2, ref3; + __m128i comp0, comp1, comp2, comp3, pred0, pred1, pred2, pred3; + __m128i sad, sad_tmp; + __m128i sad0 = __lsx_vldi(0); + __m128i sad1 = sad0; + + for (; ht_cnt--;) { + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2, + src3); + src += src_stride; + DUP4_ARG2(__lsx_vld, ref, 0, ref, 16, ref, 32, ref, 48, ref0, ref1, ref2, + ref3); + ref += ref_stride; + DUP4_ARG2(__lsx_vld, sec_pred, 0, sec_pred, 16, sec_pred, 32, sec_pred, 48, + pred0, pred1, pred2, pred3); + sec_pred += 64; + DUP4_ARG2(__lsx_vavgr_bu, pred0, ref0, pred1, ref1, pred2, ref2, pred3, + ref3, comp0, comp1, comp2, comp3); + sad_tmp = SAD_UB2_UH(src0, src1, comp0, comp1); + sad0 = __lsx_vadd_h(sad0, sad_tmp); + sad_tmp = SAD_UB2_UH(src2, src3, comp2, comp3); + sad1 = __lsx_vadd_h(sad1, sad_tmp); + + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2, + src3); + src += src_stride; + DUP4_ARG2(__lsx_vld, ref, 0, ref, 16, ref, 32, ref, 48, ref0, ref1, ref2, + ref3); + ref += ref_stride; + DUP4_ARG2(__lsx_vld, sec_pred, 0, sec_pred, 16, sec_pred, 32, sec_pred, 48, + pred0, pred1, pred2, pred3); + sec_pred += 64; + DUP4_ARG2(__lsx_vavgr_bu, pred0, ref0, pred1, ref1, pred2, ref2, pred3, + ref3, comp0, comp1, comp2, comp3); + sad_tmp = SAD_UB2_UH(src0, src1, comp0, comp1); + sad0 = __lsx_vadd_h(sad0, sad_tmp); + sad_tmp = SAD_UB2_UH(src2, src3, comp2, comp3); + sad1 = __lsx_vadd_h(sad1, sad_tmp); + + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2, + src3); + src += src_stride; + DUP4_ARG2(__lsx_vld, ref, 0, ref, 16, ref, 32, ref, 48, ref0, ref1, ref2, + ref3); + ref += ref_stride; + DUP4_ARG2(__lsx_vld, sec_pred, 0, sec_pred, 16, sec_pred, 32, sec_pred, 48, + pred0, pred1, pred2, pred3); + sec_pred += 64; + DUP4_ARG2(__lsx_vavgr_bu, pred0, ref0, pred1, ref1, pred2, ref2, pred3, + ref3, comp0, comp1, comp2, comp3); + sad_tmp = SAD_UB2_UH(src0, src1, comp0, comp1); + sad0 = __lsx_vadd_h(sad0, sad_tmp); + sad_tmp = SAD_UB2_UH(src2, src3, comp2, comp3); + sad1 = __lsx_vadd_h(sad1, sad_tmp); + + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2, + src3); + src += src_stride; + DUP4_ARG2(__lsx_vld, ref, 0, ref, 16, ref, 32, ref, 48, ref0, ref1, ref2, + ref3); + ref += ref_stride; + DUP4_ARG2(__lsx_vld, sec_pred, 0, sec_pred, 16, sec_pred, 32, sec_pred, 48, + pred0, pred1, pred2, pred3); + sec_pred += 64; + DUP4_ARG2(__lsx_vavgr_bu, pred0, ref0, pred1, ref1, pred2, ref2, pred3, + ref3, comp0, comp1, comp2, comp3); + sad_tmp = SAD_UB2_UH(src0, src1, comp0, comp1); + sad0 = __lsx_vadd_h(sad0, sad_tmp); + sad_tmp = SAD_UB2_UH(src2, src3, comp2, comp3); + sad1 = __lsx_vadd_h(sad1, sad_tmp); + } + sad = __lsx_vhaddw_wu_hu(sad0, sad0); + sad_tmp = __lsx_vhaddw_wu_hu(sad1, sad1); + sad = __lsx_vadd_w(sad, sad_tmp); + + return HADD_SW_S32(sad); +} + +#define VPX_SAD_16xHT_LSX(height) \ + uint32_t vpx_sad16x##height##_lsx(const uint8_t *src, int32_t src_stride, \ + const uint8_t *ref, int32_t ref_stride) { \ + return sad_16width_lsx(src, src_stride, ref, ref_stride, height); \ + } + +#define VPX_SAD_32xHT_LSX(height) \ + uint32_t vpx_sad32x##height##_lsx(const uint8_t *src, int32_t src_stride, \ + const uint8_t *ref, int32_t ref_stride) { \ + return sad_32width_lsx(src, src_stride, ref, ref_stride, height); \ + } + +#define VPX_SAD_64xHT_LSX(height) \ + uint32_t vpx_sad64x##height##_lsx(const uint8_t *src, int32_t src_stride, \ + const uint8_t *ref, int32_t ref_stride) { \ + return sad_64width_lsx(src, src_stride, ref, ref_stride, height); \ + } + +#define VPX_SAD_8xHTx4D_LSX(height) \ + void vpx_sad8x##height##x4d_lsx(const uint8_t *src, int32_t src_stride, \ + const uint8_t *const refs[4], \ + int32_t ref_stride, uint32_t sads[4]) { \ + sad_8width_x4d_lsx(src, src_stride, refs, ref_stride, height, sads); \ + } + +#define VPX_SAD_16xHTx4D_LSX(height) \ + void vpx_sad16x##height##x4d_lsx(const uint8_t *src, int32_t src_stride, \ + const uint8_t *const refs[], \ + int32_t ref_stride, uint32_t *sads) { \ + sad_16width_x4d_lsx(src, src_stride, refs, ref_stride, height, sads); \ + } + +#define VPX_SAD_32xHTx4D_LSX(height) \ + void vpx_sad32x##height##x4d_lsx(const uint8_t *src, int32_t src_stride, \ + const uint8_t *const refs[], \ + int32_t ref_stride, uint32_t *sads) { \ + sad_32width_x4d_lsx(src, src_stride, refs, ref_stride, height, sads); \ + } + +#define VPX_SAD_64xHTx4D_LSX(height) \ + void vpx_sad64x##height##x4d_lsx(const uint8_t *src, int32_t src_stride, \ + const uint8_t *const refs[], \ + int32_t ref_stride, uint32_t *sads) { \ + sad_64width_x4d_lsx(src, src_stride, refs, ref_stride, height, sads); \ + } + +#define VPX_AVGSAD_32xHT_LSX(height) \ + uint32_t vpx_sad32x##height##_avg_lsx( \ + const uint8_t *src, int32_t src_stride, const uint8_t *ref, \ + int32_t ref_stride, const uint8_t *second_pred) { \ + return avgsad_32width_lsx(src, src_stride, ref, ref_stride, height, \ + second_pred); \ + } + +#define VPX_AVGSAD_64xHT_LSX(height) \ + uint32_t vpx_sad64x##height##_avg_lsx( \ + const uint8_t *src, int32_t src_stride, const uint8_t *ref, \ + int32_t ref_stride, const uint8_t *second_pred) { \ + return avgsad_64width_lsx(src, src_stride, ref, ref_stride, height, \ + second_pred); \ + } + +#define SAD64 \ + VPX_SAD_64xHT_LSX(64) VPX_SAD_64xHTx4D_LSX(64) VPX_SAD_64xHTx4D_LSX(32) \ + VPX_AVGSAD_64xHT_LSX(64) + +SAD64 + +#define SAD32 \ + VPX_SAD_32xHT_LSX(32) VPX_SAD_32xHTx4D_LSX(32) VPX_SAD_32xHTx4D_LSX(64) \ + VPX_AVGSAD_32xHT_LSX(32) + +SAD32 + +#define SAD16 VPX_SAD_16xHT_LSX(16) VPX_SAD_16xHTx4D_LSX(16) + +SAD16 + +#define SAD8 VPX_SAD_8xHTx4D_LSX(8) + +SAD8 + +#undef SAD64 +#undef SAD32 +#undef SAD16 +#undef SAD8 diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/sub_pixel_variance_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/sub_pixel_variance_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/sub_pixel_variance_lsx.c @@ -0,0 +1,750 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "./vpx_dsp_rtcd.h" +#include "vpx_ports/mem.h" +#include "vpx_util/loongson_intrinsics.h" +#include "vpx_dsp/variance.h" + +#define HADD_SW_S32(in0, in1) \ + do { \ + __m128i res0_m; \ + \ + res0_m = __lsx_vhaddw_d_w(in0, in0); \ + res0_m = __lsx_vhaddw_q_d(res0_m, res0_m); \ + in1 = __lsx_vpickve2gr_w(res0_m, 0); \ + } while (0) + +#define HORIZ_2TAP_FILT_UH(in0, in1, mask, coeff, shift, in2) \ + do { \ + __m128i tmp0_m, tmp1_m; \ + \ + tmp0_m = __lsx_vshuf_b(in1, in0, mask); \ + tmp1_m = __lsx_vdp2_h_bu(tmp0_m, coeff); \ + in2 = __lsx_vsrari_h(tmp1_m, shift); \ + } while (0) + +#define CALC_MSE_AVG_B(src, ref, var, sub) \ + { \ + __m128i src_l0_m, src_l1_m; \ + __m128i res_l0_m, res_l1_m; \ + \ + src_l0_m = __lsx_vilvl_b(src, ref); \ + src_l1_m = __lsx_vilvh_b(src, ref); \ + DUP2_ARG2(__lsx_vhsubw_hu_bu, src_l0_m, src_l0_m, src_l1_m, src_l1_m, \ + res_l0_m, res_l1_m); \ + var = __lsx_vdp2add_w_h(var, res_l0_m, res_l0_m); \ + var = __lsx_vdp2add_w_h(var, res_l1_m, res_l1_m); \ + sub = __lsx_vadd_h(sub, res_l0_m); \ + sub = __lsx_vadd_h(sub, res_l1_m); \ + } + +static const uint8_t bilinear_filters_lsx[8][2] = { + { 128, 0 }, { 112, 16 }, { 96, 32 }, { 80, 48 }, + { 64, 64 }, { 48, 80 }, { 32, 96 }, { 16, 112 }, +}; + +#define VARIANCE_LARGE_WxH(sse, diff, shift) \ + (sse) - (((int64_t)(diff) * (diff)) >> (shift)) + +static uint32_t avg_sse_diff_64x64_lsx(const uint8_t *src_ptr, + int32_t src_stride, + const uint8_t *ref_ptr, + int32_t ref_stride, + const uint8_t *sec_pred, int32_t *diff) { + int32_t ht_cnt = 32; + uint32_t res; + __m128i src0, src1, src2, src3, ref0, ref1, ref2, ref3; + __m128i pred0, pred1, pred2, pred3, vec, vec_tmp; + __m128i avg0, avg1, avg2, avg3; + __m128i var = __lsx_vldi(0); + + avg0 = var; + avg1 = var; + avg2 = var; + avg3 = var; + + for (; ht_cnt--;) { + DUP4_ARG2(__lsx_vld, sec_pred, 0, sec_pred, 16, sec_pred, 32, sec_pred, 48, + pred0, pred1, pred2, pred3); + sec_pred += 64; + DUP4_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src_ptr, 32, src_ptr, 48, + src0, src1, src2, src3); + src_ptr += src_stride; + DUP4_ARG2(__lsx_vld, ref_ptr, 0, ref_ptr, 16, ref_ptr, 32, ref_ptr, 48, + ref0, ref1, ref2, ref3); + ref_ptr += ref_stride; + + DUP4_ARG2(__lsx_vavgr_bu, src0, pred0, src1, pred1, src2, pred2, src3, + pred3, src0, src1, src2, src3); + CALC_MSE_AVG_B(src0, ref0, var, avg0); + CALC_MSE_AVG_B(src1, ref1, var, avg1); + CALC_MSE_AVG_B(src2, ref2, var, avg2); + CALC_MSE_AVG_B(src3, ref3, var, avg3); + + DUP4_ARG2(__lsx_vld, sec_pred, 0, sec_pred, 16, sec_pred, 32, sec_pred, 48, + pred0, pred1, pred2, pred3); + sec_pred += 64; + DUP4_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src_ptr, 32, src_ptr, 48, + src0, src1, src2, src3); + src_ptr += src_stride; + DUP4_ARG2(__lsx_vld, ref_ptr, 0, ref_ptr, 16, ref_ptr, 32, ref_ptr, 48, + ref0, ref1, ref2, ref3); + ref_ptr += ref_stride; + + DUP4_ARG2(__lsx_vavgr_bu, src0, pred0, src1, pred1, src2, pred2, src3, + pred3, src0, src1, src2, src3); + CALC_MSE_AVG_B(src0, ref0, var, avg0); + CALC_MSE_AVG_B(src1, ref1, var, avg1); + CALC_MSE_AVG_B(src2, ref2, var, avg2); + CALC_MSE_AVG_B(src3, ref3, var, avg3); + } + vec = __lsx_vhaddw_w_h(avg0, avg0); + vec_tmp = __lsx_vhaddw_w_h(avg1, avg1); + vec = __lsx_vadd_w(vec, vec_tmp); + vec_tmp = __lsx_vhaddw_w_h(avg2, avg2); + vec = __lsx_vadd_w(vec, vec_tmp); + vec_tmp = __lsx_vhaddw_w_h(avg3, avg3); + vec = __lsx_vadd_w(vec, vec_tmp); + HADD_SW_S32(vec, *diff); + HADD_SW_S32(var, res); + + return res; +} + +static uint32_t sub_pixel_sse_diff_16width_h_lsx( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *filter, int32_t height, int32_t *diff) { + uint32_t loop_cnt = (height >> 2); + uint32_t res; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i dst0, dst1, dst2, dst3, filt0; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i out0, out1, out2, out3, out4, out5, out6, out7; + __m128i vec, var = __lsx_vldi(0); + __m128i avg = var; + __m128i mask = { 0x0403030202010100, 0x0807070606050504 }; + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + int32_t dst_stride4 = dst_stride2 << 1; + + filt0 = __lsx_vldrepl_h(filter, 0); + + for (; loop_cnt--;) { + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src0, src1); + src += src_stride; + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src2, src3); + src += src_stride; + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src4, src5); + src += src_stride; + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src6, src7); + src += src_stride; + + dst0 = __lsx_vld(dst, 0); + DUP2_ARG2(__lsx_vldx, dst, dst_stride, dst, dst_stride2, dst1, dst2); + dst3 = __lsx_vldx(dst, dst_stride3); + dst += dst_stride4; + + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1); + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3); + DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask, src5, src5, mask, vec4, vec5); + DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask, src7, src7, mask, vec6, vec7); + + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, + filt0, out0, out1, out2, out3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7, + filt0, out4, out5, out6, out7); + DUP4_ARG3(__lsx_vssrarni_bu_h, out1, out0, FILTER_BITS, out3, out2, + FILTER_BITS, out5, out4, FILTER_BITS, out7, out6, FILTER_BITS, + src0, src1, src2, src3); + CALC_MSE_AVG_B(src0, dst0, var, avg); + CALC_MSE_AVG_B(src1, dst1, var, avg); + CALC_MSE_AVG_B(src2, dst2, var, avg); + CALC_MSE_AVG_B(src3, dst3, var, avg); + } + vec = __lsx_vhaddw_w_h(avg, avg); + HADD_SW_S32(vec, *diff); + HADD_SW_S32(var, res); + + return res; +} + +static uint32_t sub_pixel_sse_diff_32width_h_lsx( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *filter, int32_t height, int32_t *diff) { + uint32_t sse = 0; + int32_t diff0[2]; + + sse += sub_pixel_sse_diff_16width_h_lsx(src, src_stride, dst, dst_stride, + filter, height, &diff0[0]); + src += 16; + dst += 16; + + sse += sub_pixel_sse_diff_16width_h_lsx(src, src_stride, dst, dst_stride, + filter, height, &diff0[1]); + + *diff = diff0[0] + diff0[1]; + + return sse; +} + +static uint32_t sub_pixel_sse_diff_16width_v_lsx( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *filter, int32_t height, int32_t *diff) { + uint32_t loop_cnt = (height >> 2); + uint32_t res; + __m128i ref0, ref1, ref2, ref3, src0, src1, src2, src3, src4; + __m128i out0, out1, out2, out3, tmp0, tmp1, filt0, vec; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i var = __lsx_vldi(0); + __m128i avg = var; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + int32_t dst_stride4 = dst_stride2 << 1; + + filt0 = __lsx_vldrepl_h(filter, 0); + + src0 = __lsx_vld(src, 0); + src += src_stride; + + for (; loop_cnt--;) { + src1 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src3); + src4 = __lsx_vldx(src, src_stride3); + src += src_stride4; + ref0 = __lsx_vld(dst, 0); + DUP2_ARG2(__lsx_vldx, dst, dst_stride, dst, dst_stride2, ref1, ref2); + ref3 = __lsx_vldx(dst, dst_stride3); + dst += dst_stride4; + + DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, vec0, vec2); + DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, vec1, vec3); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1); + out0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + + DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, vec4, vec6); + DUP2_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, vec5, vec7); + DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp0, tmp1); + out1 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + + DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1); + out2 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp0, tmp1); + out3 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + + src0 = src4; + + CALC_MSE_AVG_B(out0, ref0, var, avg); + CALC_MSE_AVG_B(out1, ref1, var, avg); + CALC_MSE_AVG_B(out2, ref2, var, avg); + CALC_MSE_AVG_B(out3, ref3, var, avg); + } + vec = __lsx_vhaddw_w_h(avg, avg); + HADD_SW_S32(vec, *diff); + HADD_SW_S32(var, res); + + return res; +} + +static uint32_t sub_pixel_sse_diff_32width_v_lsx( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *filter, int32_t height, int32_t *diff) { + uint32_t sse = 0; + int32_t diff0[2]; + + sse += sub_pixel_sse_diff_16width_v_lsx(src, src_stride, dst, dst_stride, + filter, height, &diff0[0]); + src += 16; + dst += 16; + + sse += sub_pixel_sse_diff_16width_v_lsx(src, src_stride, dst, dst_stride, + filter, height, &diff0[1]); + + *diff = diff0[0] + diff0[1]; + + return sse; +} + +static uint32_t sub_pixel_sse_diff_16width_hv_lsx( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *filter_horiz, const uint8_t *filter_vert, + int32_t height, int32_t *diff) { + uint32_t loop_cnt = (height >> 2); + uint32_t res; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i ref0, ref1, ref2, ref3, filt_hz, filt_vt, vec0, vec1; + __m128i hz_out0, hz_out1, hz_out2, hz_out3, tmp0, tmp1, vec; + __m128i var = __lsx_vldi(0); + __m128i avg = var; + __m128i mask = { 0x0403030202010100, 0x0807070606050504 }; + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + int32_t dst_stride4 = dst_stride2 << 1; + + filt_hz = __lsx_vldrepl_h(filter_horiz, 0); + filt_vt = __lsx_vldrepl_h(filter_vert, 0); + + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src0, src1); + src += src_stride; + + HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS, hz_out0); + HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS, hz_out2); + + for (; loop_cnt--;) { + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src0, src1); + src += src_stride; + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src2, src3); + src += src_stride; + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src4, src5); + src += src_stride; + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src6, src7); + src += src_stride; + + ref0 = __lsx_vld(dst, 0); + DUP2_ARG2(__lsx_vldx, dst, dst_stride, dst, dst_stride2, ref1, ref2); + ref3 = __lsx_vldx(dst, dst_stride3); + dst += dst_stride4; + + HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS, hz_out1); + HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS, hz_out3); + DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1); + src0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + + HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS, hz_out0); + HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS, hz_out2); + DUP2_ARG2(__lsx_vpackev_b, hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1); + src1 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + + HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS, hz_out1); + HORIZ_2TAP_FILT_UH(src5, src5, mask, filt_hz, FILTER_BITS, hz_out3); + DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1); + src2 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + + HORIZ_2TAP_FILT_UH(src6, src6, mask, filt_hz, FILTER_BITS, hz_out0); + HORIZ_2TAP_FILT_UH(src7, src7, mask, filt_hz, FILTER_BITS, hz_out2); + DUP2_ARG2(__lsx_vpackev_b, hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1); + src3 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + + CALC_MSE_AVG_B(src0, ref0, var, avg); + CALC_MSE_AVG_B(src1, ref1, var, avg); + CALC_MSE_AVG_B(src2, ref2, var, avg); + CALC_MSE_AVG_B(src3, ref3, var, avg); + } + vec = __lsx_vhaddw_w_h(avg, avg); + HADD_SW_S32(vec, *diff); + HADD_SW_S32(var, res); + + return res; +} + +static uint32_t sub_pixel_sse_diff_32width_hv_lsx( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *filter_horiz, const uint8_t *filter_vert, + int32_t height, int32_t *diff) { + uint32_t sse = 0; + int32_t diff0[2]; + + sse += sub_pixel_sse_diff_16width_hv_lsx(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, height, + &diff0[0]); + src += 16; + dst += 16; + + sse += sub_pixel_sse_diff_16width_hv_lsx(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, height, + &diff0[1]); + + *diff = diff0[0] + diff0[1]; + + return sse; +} + +static uint32_t subpel_avg_ssediff_16w_h_lsx( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *sec_pred, const uint8_t *filter, + int32_t height, int32_t *diff, int32_t width) { + uint32_t loop_cnt = (height >> 2); + uint32_t res; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i dst0, dst1, dst2, dst3, tmp0, tmp1, tmp2, tmp3; + __m128i pred0, pred1, pred2, pred3, filt0, vec; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i out0, out1, out2, out3, out4, out5, out6, out7; + __m128i mask = { 0x403030202010100, 0x807070606050504 }; + __m128i avg = __lsx_vldi(0); + __m128i var = avg; + + filt0 = __lsx_vldrepl_h(filter, 0); + + for (; loop_cnt--;) { + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src0, src1); + src += src_stride; + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src2, src3); + src += src_stride; + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src4, src5); + src += src_stride; + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src6, src7); + src += src_stride; + + dst0 = __lsx_vld(dst, 0); + dst += dst_stride; + dst1 = __lsx_vld(dst, 0); + dst += dst_stride; + dst2 = __lsx_vld(dst, 0); + dst += dst_stride; + dst3 = __lsx_vld(dst, 0); + dst += dst_stride; + + pred0 = __lsx_vld(sec_pred, 0); + sec_pred += width; + pred1 = __lsx_vld(sec_pred, 0); + sec_pred += width; + pred2 = __lsx_vld(sec_pred, 0); + sec_pred += width; + pred3 = __lsx_vld(sec_pred, 0); + sec_pred += width; + + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1); + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3); + DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask, src5, src5, mask, vec4, vec5); + DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask, src7, src7, mask, vec6, vec7); + + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, + filt0, out0, out1, out2, out3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7, + filt0, out4, out5, out6, out7); + DUP4_ARG3(__lsx_vssrarni_bu_h, out1, out0, FILTER_BITS, out3, out2, + FILTER_BITS, out5, out4, FILTER_BITS, out7, out6, FILTER_BITS, + tmp0, tmp1, tmp2, tmp3); + DUP4_ARG2(__lsx_vavgr_bu, tmp0, pred0, tmp1, pred1, tmp2, pred2, tmp3, + pred3, tmp0, tmp1, tmp2, tmp3); + + CALC_MSE_AVG_B(tmp0, dst0, var, avg); + CALC_MSE_AVG_B(tmp1, dst1, var, avg); + CALC_MSE_AVG_B(tmp2, dst2, var, avg); + CALC_MSE_AVG_B(tmp3, dst3, var, avg); + } + vec = __lsx_vhaddw_w_h(avg, avg); + HADD_SW_S32(vec, *diff); + HADD_SW_S32(var, res); + + return res; +} + +static uint32_t subpel_avg_ssediff_16w_v_lsx( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *sec_pred, const uint8_t *filter, + int32_t height, int32_t *diff, int32_t width) { + uint32_t loop_cnt = (height >> 2); + uint32_t res; + __m128i ref0, ref1, ref2, ref3, pred0, pred1, pred2, pred3; + __m128i src0, src1, src2, src3, src4, out0, out1, out2, out3; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i tmp0, tmp1, vec, filt0; + __m128i avg = __lsx_vldi(0); + __m128i var = avg; + + filt0 = __lsx_vldrepl_h(filter, 0); + + src0 = __lsx_vld(src, 0); + src += src_stride; + + for (; loop_cnt--;) { + src1 = __lsx_vld(src, 0); + src += src_stride; + src2 = __lsx_vld(src, 0); + src += src_stride; + src3 = __lsx_vld(src, 0); + src += src_stride; + src4 = __lsx_vld(src, 0); + src += src_stride; + + pred0 = __lsx_vld(sec_pred, 0); + sec_pred += width; + pred1 = __lsx_vld(sec_pred, 0); + sec_pred += width; + pred2 = __lsx_vld(sec_pred, 0); + sec_pred += width; + pred3 = __lsx_vld(sec_pred, 0); + sec_pred += width; + + DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, vec0, vec2); + DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, vec1, vec3); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1); + out0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + + DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, vec4, vec6); + DUP2_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, vec5, vec7); + DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp0, tmp1); + out1 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + + DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1); + out2 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + + DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp0, tmp1); + out3 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + + src0 = src4; + ref0 = __lsx_vld(dst, 0); + dst += dst_stride; + ref1 = __lsx_vld(dst, 0); + dst += dst_stride; + ref2 = __lsx_vld(dst, 0); + dst += dst_stride; + ref3 = __lsx_vld(dst, 0); + dst += dst_stride; + + DUP4_ARG2(__lsx_vavgr_bu, out0, pred0, out1, pred1, out2, pred2, out3, + pred3, out0, out1, out2, out3); + + CALC_MSE_AVG_B(out0, ref0, var, avg); + CALC_MSE_AVG_B(out1, ref1, var, avg); + CALC_MSE_AVG_B(out2, ref2, var, avg); + CALC_MSE_AVG_B(out3, ref3, var, avg); + } + vec = __lsx_vhaddw_w_h(avg, avg); + HADD_SW_S32(vec, *diff); + HADD_SW_S32(var, res); + return res; +} + +static uint32_t subpel_avg_ssediff_16w_hv_lsx( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *sec_pred, const uint8_t *filter_horiz, + const uint8_t *filter_vert, int32_t height, int32_t *diff, int32_t width) { + uint32_t loop_cnt = (height >> 2); + uint32_t res; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i ref0, ref1, ref2, ref3, pred0, pred1, pred2, pred3; + __m128i hz_out0, hz_out1, hz_out2, hz_out3, tmp0, tmp1; + __m128i out0, out1, out2, out3, filt_hz, filt_vt, vec, vec0, vec1; + __m128i mask = { 0x403030202010100, 0x807070606050504 }; + __m128i avg = __lsx_vldi(0); + __m128i var = avg; + + filt_hz = __lsx_vldrepl_h(filter_horiz, 0); + filt_vt = __lsx_vldrepl_h(filter_vert, 0); + + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src0, src1); + src += src_stride; + + HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS, hz_out0); + HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS, hz_out2); + + for (; loop_cnt--;) { + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src0, src1); + src += src_stride; + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src2, src3); + src += src_stride; + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src4, src5); + src += src_stride; + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src6, src7); + src += src_stride; + + pred0 = __lsx_vld(sec_pred, 0); + sec_pred += width; + pred1 = __lsx_vld(sec_pred, 0); + sec_pred += width; + pred2 = __lsx_vld(sec_pred, 0); + sec_pred += width; + pred3 = __lsx_vld(sec_pred, 0); + sec_pred += width; + + HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS, hz_out1); + HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS, hz_out3); + DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1); + out0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + + HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS, hz_out0); + HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS, hz_out2); + DUP2_ARG2(__lsx_vpackev_b, hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1); + out1 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + + HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS, hz_out1); + HORIZ_2TAP_FILT_UH(src5, src5, mask, filt_hz, FILTER_BITS, hz_out3); + DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1); + out2 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + + HORIZ_2TAP_FILT_UH(src6, src6, mask, filt_hz, FILTER_BITS, hz_out0); + HORIZ_2TAP_FILT_UH(src7, src7, mask, filt_hz, FILTER_BITS, hz_out2); + DUP2_ARG2(__lsx_vpackev_b, hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1); + out3 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + + ref0 = __lsx_vld(dst, 0); + dst += dst_stride; + ref1 = __lsx_vld(dst, 0); + dst += dst_stride; + ref2 = __lsx_vld(dst, 0); + dst += dst_stride; + ref3 = __lsx_vld(dst, 0); + dst += dst_stride; + + DUP4_ARG2(__lsx_vavgr_bu, out0, pred0, out1, pred1, out2, pred2, out3, + pred3, out0, out1, out2, out3); + + CALC_MSE_AVG_B(out0, ref0, var, avg); + CALC_MSE_AVG_B(out1, ref1, var, avg); + CALC_MSE_AVG_B(out2, ref2, var, avg); + CALC_MSE_AVG_B(out3, ref3, var, avg); + } + vec = __lsx_vhaddw_w_h(avg, avg); + HADD_SW_S32(vec, *diff); + HADD_SW_S32(var, res); + return res; +} + +static uint32_t sub_pixel_avg_sse_diff_64width_h_lsx( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *sec_pred, const uint8_t *filter, + int32_t height, int32_t *diff) { + uint32_t loop_cnt, sse = 0; + int32_t diff0[4]; + + for (loop_cnt = 0; loop_cnt < 4; ++loop_cnt) { + sse += + subpel_avg_ssediff_16w_h_lsx(src, src_stride, dst, dst_stride, sec_pred, + filter, height, &diff0[loop_cnt], 64); + src += 16; + dst += 16; + sec_pred += 16; + } + + *diff = diff0[0] + diff0[1] + diff0[2] + diff0[3]; + + return sse; +} + +static uint32_t sub_pixel_avg_sse_diff_64width_v_lsx( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *sec_pred, const uint8_t *filter, + int32_t height, int32_t *diff) { + uint32_t loop_cnt, sse = 0; + int32_t diff0[4]; + + for (loop_cnt = 0; loop_cnt < 4; ++loop_cnt) { + sse += + subpel_avg_ssediff_16w_v_lsx(src, src_stride, dst, dst_stride, sec_pred, + filter, height, &diff0[loop_cnt], 64); + src += 16; + dst += 16; + sec_pred += 16; + } + + *diff = diff0[0] + diff0[1] + diff0[2] + diff0[3]; + + return sse; +} + +static uint32_t sub_pixel_avg_sse_diff_64width_hv_lsx( + const uint8_t *src, int32_t src_stride, const uint8_t *dst, + int32_t dst_stride, const uint8_t *sec_pred, const uint8_t *filter_horiz, + const uint8_t *filter_vert, int32_t height, int32_t *diff) { + uint32_t loop_cnt, sse = 0; + int32_t diff0[4]; + + for (loop_cnt = 0; loop_cnt < 4; ++loop_cnt) { + sse += subpel_avg_ssediff_16w_hv_lsx(src, src_stride, dst, dst_stride, + sec_pred, filter_horiz, filter_vert, + height, &diff0[loop_cnt], 64); + src += 16; + dst += 16; + sec_pred += 16; + } + + *diff = diff0[0] + diff0[1] + diff0[2] + diff0[3]; + + return sse; +} + +#define VARIANCE_32Wx32H(sse, diff) VARIANCE_LARGE_WxH(sse, diff, 10) +#define VARIANCE_64Wx64H(sse, diff) VARIANCE_LARGE_WxH(sse, diff, 12) + +#define VPX_SUB_PIXEL_VARIANCE_WDXHT_LSX(wd, ht) \ + uint32_t vpx_sub_pixel_variance##wd##x##ht##_lsx( \ + const uint8_t *src, int32_t src_stride, int32_t x_offset, \ + int32_t y_offset, const uint8_t *ref, int32_t ref_stride, \ + uint32_t *sse) { \ + int32_t diff; \ + uint32_t var; \ + const uint8_t *h_filter = bilinear_filters_lsx[x_offset]; \ + const uint8_t *v_filter = bilinear_filters_lsx[y_offset]; \ + \ + if (y_offset) { \ + if (x_offset) { \ + *sse = sub_pixel_sse_diff_##wd##width_hv_lsx( \ + src, src_stride, ref, ref_stride, h_filter, v_filter, ht, &diff); \ + } else { \ + *sse = sub_pixel_sse_diff_##wd##width_v_lsx( \ + src, src_stride, ref, ref_stride, v_filter, ht, &diff); \ + } \ + \ + var = VARIANCE_##wd##Wx##ht##H(*sse, diff); \ + } else { \ + if (x_offset) { \ + *sse = sub_pixel_sse_diff_##wd##width_h_lsx( \ + src, src_stride, ref, ref_stride, h_filter, ht, &diff); \ + \ + var = VARIANCE_##wd##Wx##ht##H(*sse, diff); \ + } else { \ + var = vpx_variance##wd##x##ht##_lsx(src, src_stride, ref, ref_stride, \ + sse); \ + } \ + } \ + \ + return var; \ + } + +VPX_SUB_PIXEL_VARIANCE_WDXHT_LSX(32, 32) + +#define VPX_SUB_PIXEL_AVG_VARIANCE64XHEIGHT_LSX(ht) \ + uint32_t vpx_sub_pixel_avg_variance64x##ht##_lsx( \ + const uint8_t *src_ptr, int32_t src_stride, int32_t x_offset, \ + int32_t y_offset, const uint8_t *ref_ptr, int32_t ref_stride, \ + uint32_t *sse, const uint8_t *sec_pred) { \ + int32_t diff; \ + const uint8_t *h_filter = bilinear_filters_lsx[x_offset]; \ + const uint8_t *v_filter = bilinear_filters_lsx[y_offset]; \ + \ + if (y_offset) { \ + if (x_offset) { \ + *sse = sub_pixel_avg_sse_diff_64width_hv_lsx( \ + src_ptr, src_stride, ref_ptr, ref_stride, sec_pred, h_filter, \ + v_filter, ht, &diff); \ + } else { \ + *sse = sub_pixel_avg_sse_diff_64width_v_lsx( \ + src_ptr, src_stride, ref_ptr, ref_stride, sec_pred, v_filter, ht, \ + &diff); \ + } \ + } else { \ + if (x_offset) { \ + *sse = sub_pixel_avg_sse_diff_64width_h_lsx( \ + src_ptr, src_stride, ref_ptr, ref_stride, sec_pred, h_filter, ht, \ + &diff); \ + } else { \ + *sse = avg_sse_diff_64x##ht##_lsx(src_ptr, src_stride, ref_ptr, \ + ref_stride, sec_pred, &diff); \ + } \ + } \ + \ + return VARIANCE_64Wx##ht##H(*sse, diff); \ + } + +VPX_SUB_PIXEL_AVG_VARIANCE64XHEIGHT_LSX(64) diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/txfm_macros_lsx.h b/media/libvpx/libvpx/vpx_dsp/loongarch/txfm_macros_lsx.h new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/txfm_macros_lsx.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VPX_VPX_DSP_LOONGARCH_TXFM_MACROS_LSX_H_ +#define VPX_VPX_DSP_LOONGARCH_TXFM_MACROS_LSX_H_ + +#include "vpx_util/loongson_intrinsics.h" + +#define DOTP_CONST_PAIR(reg0, reg1, cnst0, cnst1, out0, out1) \ + { \ + __m128i s0_m, s1_m, s2_m, s3_m, s4_m, s5_m; \ + __m128i k0_m, k1_m, k2_m, k3_m; \ + __m128i zero = __lsx_vldi(0); \ + \ + k0_m = __lsx_vreplgr2vr_h(cnst0); \ + k1_m = __lsx_vreplgr2vr_h(cnst1); \ + k2_m = __lsx_vpackev_h(k1_m, k0_m); \ + k0_m = __lsx_vpackev_h(zero, k0_m); \ + k1_m = __lsx_vpackev_h(k1_m, zero); \ + \ + s5_m = __lsx_vilvl_h(reg1, reg0); \ + s4_m = __lsx_vilvh_h(reg1, reg0); \ + s3_m = __lsx_vilvl_h(reg0, reg1); \ + s2_m = __lsx_vilvh_h(reg0, reg1); \ + \ + s1_m = __lsx_vdp2_w_h(s5_m, k0_m); \ + s0_m = __lsx_vdp2_w_h(s4_m, k0_m); \ + k3_m = __lsx_vdp2_w_h(s5_m, k1_m); \ + s1_m = __lsx_vsub_w(s1_m, k3_m); \ + k3_m = __lsx_vdp2_w_h(s4_m, k1_m); \ + s0_m = __lsx_vsub_w(s0_m, k3_m); \ + \ + out0 = __lsx_vssrarni_h_w(s0_m, s1_m, DCT_CONST_BITS); \ + \ + s1_m = __lsx_vdp2_w_h(s3_m, k2_m); \ + s0_m = __lsx_vdp2_w_h(s2_m, k2_m); \ + out1 = __lsx_vssrarni_h_w(s0_m, s1_m, DCT_CONST_BITS); \ + } + +#define DOT_SHIFT_RIGHT_PCK_H(in0, in1, in2, in3) \ + do { \ + __m128i tp0_m, tp1_m; \ + \ + DUP2_ARG2(__lsx_vdp2_w_h, in0, in2, in1, in2, tp1_m, tp0_m); \ + in3 = __lsx_vssrarni_h_w(tp1_m, tp0_m, DCT_CONST_BITS); \ + } while (0) + +#endif // VPX_VPX_DSP_LOONGARCH_TXFM_MACROS_LSX_H_ diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/variance_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/variance_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/variance_lsx.c @@ -0,0 +1,249 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "./vpx_dsp_rtcd.h" +#include "vpx_util/loongson_intrinsics.h" + +#define HADD_SW_S32(in) \ + ({ \ + __m128i res0_m; \ + int32_t sum_m; \ + \ + res0_m = __lsx_vhaddw_d_w(in, in); \ + res0_m = __lsx_vhaddw_q_d(res0_m, res0_m); \ + sum_m = __lsx_vpickve2gr_w(res0_m, 0); \ + sum_m; \ + }) + +#define CALC_MSE_AVG_B(src, ref, var, sub) \ + { \ + __m128i src_l0_m, src_l1_m; \ + __m128i res_l0_m, res_l1_m; \ + \ + src_l0_m = __lsx_vilvl_b(src, ref); \ + src_l1_m = __lsx_vilvh_b(src, ref); \ + DUP2_ARG2(__lsx_vhsubw_hu_bu, src_l0_m, src_l0_m, src_l1_m, src_l1_m, \ + res_l0_m, res_l1_m); \ + var = __lsx_vdp2add_w_h(var, res_l0_m, res_l0_m); \ + var = __lsx_vdp2add_w_h(var, res_l1_m, res_l1_m); \ + sub = __lsx_vadd_h(sub, res_l0_m); \ + sub = __lsx_vadd_h(sub, res_l1_m); \ + } + +#define VARIANCE_WxH(sse, diff, shift) \ + (sse) - (((uint32_t)(diff) * (diff)) >> (shift)) + +#define VARIANCE_LARGE_WxH(sse, diff, shift) \ + (sse) - (((int64_t)(diff) * (diff)) >> (shift)) + +static uint32_t sse_diff_8width_lsx(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *ref_ptr, int32_t ref_stride, + int32_t height, int32_t *diff) { + int32_t ht_cnt = (height >> 2); + __m128i src0, src1, src2, src3, ref0, ref1, ref2, ref3, vec; + __m128i avg = __lsx_vldi(0); + __m128i var = avg; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + int32_t ref_stride2 = ref_stride << 1; + int32_t ref_stride3 = ref_stride2 + ref_stride; + int32_t ref_stride4 = ref_stride2 << 1; + + for (; ht_cnt--;) { + DUP4_ARG2(__lsx_vld, src_ptr, 0, src_ptr + src_stride, 0, + src_ptr + src_stride2, 0, src_ptr + src_stride3, 0, src0, src1, + src2, src3); + src_ptr += src_stride4; + DUP4_ARG2(__lsx_vld, ref_ptr, 0, ref_ptr + ref_stride, 0, + ref_ptr + ref_stride2, 0, ref_ptr + ref_stride3, 0, ref0, ref1, + ref2, ref3); + ref_ptr += ref_stride4; + + DUP4_ARG2(__lsx_vpickev_d, src1, src0, src3, src2, ref1, ref0, ref3, ref2, + src0, src1, ref0, ref1); + CALC_MSE_AVG_B(src0, ref0, var, avg); + CALC_MSE_AVG_B(src1, ref1, var, avg); + } + + vec = __lsx_vhaddw_w_h(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t sse_diff_16width_lsx(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *ref_ptr, int32_t ref_stride, + int32_t height, int32_t *diff) { + int32_t ht_cnt = (height >> 2); + __m128i src, ref, vec; + __m128i avg = __lsx_vldi(0); + __m128i var = avg; + + for (; ht_cnt--;) { + src = __lsx_vld(src_ptr, 0); + src_ptr += src_stride; + ref = __lsx_vld(ref_ptr, 0); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src, ref, var, avg); + + src = __lsx_vld(src_ptr, 0); + src_ptr += src_stride; + ref = __lsx_vld(ref_ptr, 0); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src, ref, var, avg); + src = __lsx_vld(src_ptr, 0); + src_ptr += src_stride; + ref = __lsx_vld(ref_ptr, 0); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src, ref, var, avg); + + src = __lsx_vld(src_ptr, 0); + src_ptr += src_stride; + ref = __lsx_vld(ref_ptr, 0); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src, ref, var, avg); + } + vec = __lsx_vhaddw_w_h(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t sse_diff_32width_lsx(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *ref_ptr, int32_t ref_stride, + int32_t height, int32_t *diff) { + int32_t ht_cnt = (height >> 2); + __m128i avg = __lsx_vldi(0); + __m128i src0, src1, ref0, ref1; + __m128i vec; + __m128i var = avg; + + for (; ht_cnt--;) { + DUP2_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src0, src1); + src_ptr += src_stride; + DUP2_ARG2(__lsx_vld, ref_ptr, 0, ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src0, ref0, var, avg); + CALC_MSE_AVG_B(src1, ref1, var, avg); + + DUP2_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src0, src1); + src_ptr += src_stride; + DUP2_ARG2(__lsx_vld, ref_ptr, 0, ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src0, ref0, var, avg); + CALC_MSE_AVG_B(src1, ref1, var, avg); + + DUP2_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src0, src1); + src_ptr += src_stride; + DUP2_ARG2(__lsx_vld, ref_ptr, 0, ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src0, ref0, var, avg); + CALC_MSE_AVG_B(src1, ref1, var, avg); + + DUP2_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src0, src1); + src_ptr += src_stride; + DUP2_ARG2(__lsx_vld, ref_ptr, 0, ref_ptr, 16, ref0, ref1); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src0, ref0, var, avg); + CALC_MSE_AVG_B(src1, ref1, var, avg); + } + + vec = __lsx_vhaddw_w_h(avg, avg); + *diff = HADD_SW_S32(vec); + + return HADD_SW_S32(var); +} + +static uint32_t sse_diff_64x64_lsx(const uint8_t *src_ptr, int32_t src_stride, + const uint8_t *ref_ptr, int32_t ref_stride, + int32_t *diff) { + int32_t ht_cnt = 32; + __m128i avg0 = __lsx_vldi(0); + __m128i src0, src1, src2, src3; + __m128i ref0, ref1, ref2, ref3; + __m128i vec0, vec1; + __m128i avg1 = avg0; + __m128i avg2 = avg0; + __m128i avg3 = avg0; + __m128i var = avg0; + + for (; ht_cnt--;) { + DUP4_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src_ptr, 32, src_ptr, 48, + src0, src1, src2, src3); + src_ptr += src_stride; + DUP4_ARG2(__lsx_vld, ref_ptr, 0, ref_ptr, 16, ref_ptr, 32, ref_ptr, 48, + ref0, ref1, ref2, ref3); + ref_ptr += ref_stride; + + CALC_MSE_AVG_B(src0, ref0, var, avg0); + CALC_MSE_AVG_B(src1, ref1, var, avg1); + CALC_MSE_AVG_B(src2, ref2, var, avg2); + CALC_MSE_AVG_B(src3, ref3, var, avg3); + DUP4_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src_ptr, 32, src_ptr, 48, + src0, src1, src2, src3); + src_ptr += src_stride; + DUP4_ARG2(__lsx_vld, ref_ptr, 0, ref_ptr, 16, ref_ptr, 32, ref_ptr, 48, + ref0, ref1, ref2, ref3); + ref_ptr += ref_stride; + CALC_MSE_AVG_B(src0, ref0, var, avg0); + CALC_MSE_AVG_B(src1, ref1, var, avg1); + CALC_MSE_AVG_B(src2, ref2, var, avg2); + CALC_MSE_AVG_B(src3, ref3, var, avg3); + } + vec0 = __lsx_vhaddw_w_h(avg0, avg0); + vec1 = __lsx_vhaddw_w_h(avg1, avg1); + vec0 = __lsx_vadd_w(vec0, vec1); + vec1 = __lsx_vhaddw_w_h(avg2, avg2); + vec0 = __lsx_vadd_w(vec0, vec1); + vec1 = __lsx_vhaddw_w_h(avg3, avg3); + vec0 = __lsx_vadd_w(vec0, vec1); + *diff = HADD_SW_S32(vec0); + + return HADD_SW_S32(var); +} + +#define VARIANCE_8Wx8H(sse, diff) VARIANCE_WxH(sse, diff, 6); +#define VARIANCE_16Wx16H(sse, diff) VARIANCE_WxH(sse, diff, 8) + +#define VARIANCE_32Wx32H(sse, diff) VARIANCE_LARGE_WxH(sse, diff, 10) +#define VARIANCE_64Wx64H(sse, diff) VARIANCE_LARGE_WxH(sse, diff, 12) + +#define VPX_VARIANCE_WDXHT_LSX(wd, ht) \ + uint32_t vpx_variance##wd##x##ht##_lsx( \ + const uint8_t *src, int32_t src_stride, const uint8_t *ref, \ + int32_t ref_stride, uint32_t *sse) { \ + int32_t diff; \ + \ + *sse = \ + sse_diff_##wd##width_lsx(src, src_stride, ref, ref_stride, ht, &diff); \ + \ + return VARIANCE_##wd##Wx##ht##H(*sse, diff); \ + } + +VPX_VARIANCE_WDXHT_LSX(8, 8) +VPX_VARIANCE_WDXHT_LSX(16, 16) +VPX_VARIANCE_WDXHT_LSX(32, 32) + +uint32_t vpx_variance64x64_lsx(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, + uint32_t *sse) { + int32_t diff; + + *sse = sse_diff_64x64_lsx(src, src_stride, ref, ref_stride, &diff); + + return VARIANCE_64Wx64H(*sse, diff); +} + +void vpx_get16x16var_lsx(const uint8_t *src, int32_t src_stride, + const uint8_t *ref, int32_t ref_stride, uint32_t *sse, + int32_t *sum) { + *sse = sse_diff_16width_lsx(src, src_stride, ref, ref_stride, 16, sum); +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_avg_horiz_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_avg_horiz_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_avg_horiz_lsx.c @@ -0,0 +1,972 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include "./vpx_dsp_rtcd.h" +#include "vpx_dsp/loongarch/vpx_convolve_lsx.h" + +static const uint8_t mc_filt_mask_arr[16 * 3] = { + /* 8 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, + /* 4 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20, + /* 4 width cases */ + 8, 9, 9, 10, 10, 11, 11, 12, 24, 25, 25, 26, 26, 27, 27, 28 +}; + +static void common_hz_8t_and_aver_dst_4x4_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter) { + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3; + __m128i filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3; + __m128i tmp0, tmp1; + __m128i dst0, dst1, dst2, dst3; + + mask0 = __lsx_vld(mc_filt_mask_arr, 16); + src -= 3; + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filter0, filter1, filter2, filter3, tmp0, tmp1); + dst0 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_w(dst_tmp, 0); + dst0 = __lsx_vilvl_w(dst1, dst0); + dst1 = __lsx_vilvl_w(dst3, dst2); + dst0 = __lsx_vilvl_d(dst1, dst0); + tmp0 = __lsx_vssrarni_b_h(tmp1, tmp0, 7); + tmp0 = __lsx_vxori_b(tmp0, 128); + dst0 = __lsx_vavgr_bu(tmp0, dst0); + __lsx_vstelm_w(dst0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(dst0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(dst0, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(dst0, dst, 0, 3); +} + +static void common_hz_8t_and_aver_dst_4x8_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter) { + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3, filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3, tmp0, tmp1, tmp2, tmp3; + __m128i dst0, dst1; + + mask0 = __lsx_vld(mc_filt_mask_arr, 16); + src -= 3; + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + src += src_stride; + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + tmp0 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + tmp1 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + tmp2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + tmp3 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + tmp0 = __lsx_vilvl_w(tmp1, tmp0); + tmp1 = __lsx_vilvl_w(tmp3, tmp2); + dst0 = __lsx_vilvl_d(tmp1, tmp0); + + tmp0 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + tmp1 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + tmp2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + tmp3 = __lsx_vldrepl_w(dst_tmp, 0); + tmp0 = __lsx_vilvl_w(tmp1, tmp0); + tmp1 = __lsx_vilvl_w(tmp3, tmp2); + dst1 = __lsx_vilvl_d(tmp1, tmp0); + + HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filter0, filter1, filter2, filter3, tmp0, tmp1); + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filter0, filter1, filter2, filter3, tmp2, tmp3); + DUP4_ARG3(__lsx_vssrarni_b_h, tmp0, tmp0, 7, tmp1, tmp1, 7, tmp2, tmp2, 7, + tmp3, tmp3, 7, tmp0, tmp1, tmp2, tmp3); + DUP2_ARG2(__lsx_vilvl_d, tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + DUP2_ARG2(__lsx_vavgr_bu, tmp0, dst0, tmp1, dst1, dst0, dst1); + __lsx_vstelm_w(dst0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(dst0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(dst0, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(dst0, dst, 0, 3); + dst += dst_stride; + __lsx_vstelm_w(dst1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(dst1, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(dst1, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(dst1, dst, 0, 3); +} + +static void common_hz_8t_and_aver_dst_4w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, int8_t *filter, + int32_t height) { + if (height == 4) { + common_hz_8t_and_aver_dst_4x4_lsx(src, src_stride, dst, dst_stride, filter); + } else if (height == 8) { + common_hz_8t_and_aver_dst_4x8_lsx(src, src_stride, dst, dst_stride, filter); + } +} + +static void common_hz_8t_and_aver_dst_8w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, int8_t *filter, + int32_t height) { + int32_t loop_cnt = height >> 2; + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3, filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3; + __m128i tmp0, tmp1, tmp2, tmp3; + __m128i dst0, dst1, dst2, dst3; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + uint8_t *_src = (uint8_t *)src - 3; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + for (; loop_cnt--;) { + src0 = __lsx_vld(_src, 0); + DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src1, src2); + src3 = __lsx_vldx(_src, src_stride3); + _src += src_stride4; + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, + mask3, filter0, filter1, filter2, filter3, tmp0, + tmp1, tmp2, tmp3); + dst0 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1); + DUP2_ARG3(__lsx_vssrarni_b_h, tmp1, tmp0, 7, tmp3, tmp2, 7, tmp0, tmp1); + DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + DUP2_ARG2(__lsx_vavgr_bu, tmp0, dst0, tmp1, dst1, dst0, dst1); + __lsx_vstelm_d(dst0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(dst0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(dst1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(dst1, dst, 0, 1); + dst += dst_stride; + } +} + +static void common_hz_8t_and_aver_dst_16w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter, int32_t height) { + int32_t loop_cnt = height >> 1; + int32_t dst_stride2 = dst_stride << 1; + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3, filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3, dst0, dst1, dst2, dst3; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + __m128i tmp8, tmp9, tmp10, tmp11, tmp12, tmp13, tmp14, tmp15; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + src -= 3; + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + for (; loop_cnt--;) { + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src0, src1); + src += src_stride; + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src2, src3); + src += src_stride; + dst0 = __lsx_vld(dst_tmp, 0); + dst1 = __lsx_vldx(dst_tmp, dst_stride); + dst_tmp += dst_stride2; + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src1, mask0, src2, src2, + mask0, src3, src3, mask0, tmp0, tmp1, tmp2, tmp3); + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask1, src1, src1, mask1, src2, src2, + mask1, src3, src3, mask1, tmp4, tmp5, tmp6, tmp7); + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask2, src1, src1, mask2, src2, src2, + mask2, src3, src3, mask2, tmp8, tmp9, tmp10, tmp11); + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask3, src1, src1, mask3, src2, src2, + mask3, src3, src3, mask3, tmp12, tmp13, tmp14, tmp15); + DUP4_ARG2(__lsx_vdp2_h_b, tmp0, filter0, tmp1, filter0, tmp2, filter0, tmp3, + filter0, tmp0, tmp1, tmp2, tmp3); + DUP4_ARG2(__lsx_vdp2_h_b, tmp8, filter2, tmp9, filter2, tmp10, filter2, + tmp11, filter2, tmp8, tmp9, tmp10, tmp11); + DUP4_ARG3(__lsx_vdp2add_h_b, tmp0, tmp4, filter1, tmp1, tmp5, filter1, tmp2, + tmp6, filter1, tmp3, tmp7, filter1, tmp0, tmp1, tmp2, tmp3); + DUP4_ARG3(__lsx_vdp2add_h_b, tmp8, tmp12, filter3, tmp9, tmp13, filter3, + tmp10, tmp14, filter3, tmp11, tmp15, filter3, tmp4, tmp5, tmp6, + tmp7); + DUP4_ARG2(__lsx_vsadd_h, tmp0, tmp4, tmp1, tmp5, tmp2, tmp6, tmp3, tmp7, + tmp0, tmp1, tmp2, tmp3); + DUP2_ARG3(__lsx_vssrarni_b_h, tmp1, tmp0, 7, tmp3, tmp2, 7, dst2, dst3); + DUP2_ARG2(__lsx_vxori_b, dst2, 128, dst3, 128, dst2, dst3); + DUP2_ARG2(__lsx_vavgr_bu, dst0, dst2, dst1, dst3, dst0, dst1); + __lsx_vst(dst0, dst, 0); + __lsx_vstx(dst1, dst, dst_stride); + dst += dst_stride2; + } +} + +static void common_hz_8t_and_aver_dst_32w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = height; + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3, filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3, dst0, dst1; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + __m128i tmp8, tmp9, tmp10, tmp11, tmp12, tmp13, tmp14, tmp15; + __m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 }; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + src -= 3; + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + for (; loop_cnt--;) { + DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src2); + src3 = __lsx_vld(src, 24); + src1 = __lsx_vshuf_b(src2, src0, shuff); + src += src_stride; + DUP2_ARG2(__lsx_vld, dst_tmp, 0, dst, 16, dst0, dst1); + dst_tmp += dst_stride; + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src1, mask0, src2, src2, + mask0, src3, src3, mask0, tmp0, tmp1, tmp2, tmp3); + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask1, src1, src1, mask1, src2, src2, + mask1, src3, src3, mask1, tmp4, tmp5, tmp6, tmp7); + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask2, src1, src1, mask2, src2, src2, + mask2, src3, src3, mask2, tmp8, tmp9, tmp10, tmp11); + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask3, src1, src1, mask3, src2, src2, + mask3, src3, src3, mask3, tmp12, tmp13, tmp14, tmp15); + DUP4_ARG2(__lsx_vdp2_h_b, tmp0, filter0, tmp1, filter0, tmp2, filter0, tmp3, + filter0, tmp0, tmp1, tmp2, tmp3); + DUP4_ARG2(__lsx_vdp2_h_b, tmp8, filter2, tmp9, filter2, tmp10, filter2, + tmp11, filter2, tmp8, tmp9, tmp10, tmp11); + DUP4_ARG3(__lsx_vdp2add_h_b, tmp0, tmp4, filter1, tmp1, tmp5, filter1, tmp2, + tmp6, filter1, tmp3, tmp7, filter1, tmp0, tmp1, tmp2, tmp3); + DUP4_ARG3(__lsx_vdp2add_h_b, tmp8, tmp12, filter3, tmp9, tmp13, filter3, + tmp10, tmp14, filter3, tmp11, tmp15, filter3, tmp4, tmp5, tmp6, + tmp7); + DUP4_ARG2(__lsx_vsadd_h, tmp0, tmp4, tmp1, tmp5, tmp2, tmp6, tmp3, tmp7, + tmp0, tmp1, tmp2, tmp3); + DUP2_ARG3(__lsx_vssrarni_b_h, tmp1, tmp0, 7, tmp3, tmp2, 7, tmp0, tmp1); + DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + DUP2_ARG2(__lsx_vavgr_bu, dst0, tmp0, dst1, tmp1, dst0, dst1); + __lsx_vst(dst0, dst, 0); + __lsx_vst(dst1, dst, 16); + dst += dst_stride; + } +} + +static void common_hz_8t_and_aver_dst_64w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter, int32_t height) { + int32_t loop_cnt = height; + __m128i src0, src1, src2, src3; + __m128i filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3; + __m128i out0, out1, out2, out3, dst0, dst1; + __m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 }; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + src -= 3; + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + for (; loop_cnt--;) { + DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src2); + src3 = __lsx_vld(src, 24); + src1 = __lsx_vshuf_b(src2, src0, shuff); + DUP2_ARG2(__lsx_vld, dst, 0, dst, 16, dst0, dst1); + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, + mask3, filter0, filter1, filter2, filter3, out0, + out1, out2, out3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + DUP2_ARG2(__lsx_vavgr_bu, out0, dst0, out1, dst1, out0, out1); + __lsx_vst(out0, dst, 0); + __lsx_vst(out1, dst, 16); + + DUP2_ARG2(__lsx_vld, src, 32, src, 48, src0, src2); + src3 = __lsx_vld(src, 56); + src1 = __lsx_vshuf_b(src2, src0, shuff); + DUP2_ARG2(__lsx_vld, dst, 32, dst, 48, dst0, dst1); + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, + mask3, filter0, filter1, filter2, filter3, out0, + out1, out2, out3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + DUP2_ARG2(__lsx_vavgr_bu, out0, dst0, out1, dst1, out0, out1); + __lsx_vst(out0, dst, 32); + __lsx_vst(out1, dst, 48); + src += src_stride; + dst += dst_stride; + } +} + +static void common_hz_2t_and_aver_dst_4x4_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter) { + __m128i src0, src1, src2, src3, mask; + __m128i dst0, dst1, dst2, dst3, vec0, vec1, filt0; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + uint8_t *dst_tmp = dst; + + mask = __lsx_vld(mc_filt_mask_arr, 16); + + /* rearranging filter */ + filt0 = __lsx_vldrepl_h(filter, 0); + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + dst0 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_w, dst1, dst0, dst3, dst2, dst0, dst1); + dst0 = __lsx_vilvl_d(dst1, dst0); + DUP2_ARG3(__lsx_vshuf_b, src1, src0, mask, src3, src2, mask, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec0, vec1); + vec0 = __lsx_vssrarni_bu_h(vec1, vec0, FILTER_BITS); + vec0 = __lsx_vavgr_bu(vec0, dst0); + __lsx_vstelm_w(vec0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(vec0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(vec0, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(vec0, dst, 0, 3); +} + +static void common_hz_2t_and_aver_dst_4x8_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter) { + __m128i src0, src1, src2, src3, src4, src5, src6, src7, mask; + __m128i filt0, vec0, vec1, vec2, vec3, res0, res1, res2, res3; + __m128i dst0, dst1, dst2, dst3, dst4; + __m128i vec4, vec5, vec6, vec7; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + uint8_t *src_tmp1 = (uint8_t *)src + src_stride4; + uint8_t *dst_tmp = dst; + + mask = __lsx_vld(mc_filt_mask_arr, 16); + + /* rearranging filter */ + filt0 = __lsx_vldrepl_h(filter, 0); + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + + src4 = __lsx_vld(src_tmp1, 0); + DUP2_ARG2(__lsx_vldx, src_tmp1, src_stride, src_tmp1, src_stride2, src5, + src6); + src7 = __lsx_vldx(src_tmp1, src_stride3); + + dst0 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_w, dst1, dst0, dst3, dst2, dst0, dst1); + dst0 = __lsx_vilvl_d(dst1, dst0); + + dst1 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst4 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_w, dst2, dst1, dst4, dst3, dst1, dst2); + dst1 = __lsx_vilvl_d(dst2, dst1); + + DUP2_ARG3(__lsx_vshuf_b, src1, src0, mask, src3, src2, mask, vec0, vec1); + DUP2_ARG3(__lsx_vshuf_b, src5, src4, mask, src7, src6, mask, vec2, vec3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0, + vec4, vec5, vec6, vec7); + DUP4_ARG3(__lsx_vssrarni_bu_h, vec4, vec4, FILTER_BITS, vec5, vec5, + FILTER_BITS, vec6, vec6, FILTER_BITS, vec7, vec7, FILTER_BITS, res0, + res1, res2, res3); + DUP2_ARG2(__lsx_vilvl_d, res1, res0, res3, res2, res0, res2); + DUP2_ARG2(__lsx_vavgr_bu, res0, dst0, res2, dst1, res0, res2); + + __lsx_vstelm_w(res0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(res0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(res0, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(res0, dst, 0, 3); + dst += dst_stride; + + __lsx_vstelm_w(res2, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(res2, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(res2, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(res2, dst, 0, 3); + dst += dst_stride; +} + +static void common_hz_2t_and_aver_dst_4w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, int8_t *filter, + int32_t height) { + if (height == 4) { + common_hz_2t_and_aver_dst_4x4_lsx(src, src_stride, dst, dst_stride, filter); + } else if (height == 8) { + common_hz_2t_and_aver_dst_4x8_lsx(src, src_stride, dst, dst_stride, filter); + } +} + +static void common_hz_2t_and_aver_dst_8x4_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter) { + __m128i src0, src1, src2, src3, mask; + __m128i filt0, dst0, dst1, dst2, dst3; + __m128i vec0, vec1, vec2, vec3; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + uint8_t *dst_tmp = dst; + + mask = __lsx_vld(mc_filt_mask_arr, 0); + + /* rearranging filter */ + filt0 = __lsx_vldrepl_h(filter, 0); + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1); + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0, + vec0, vec1, vec2, vec3); + DUP2_ARG3(__lsx_vssrarni_bu_h, vec1, vec0, FILTER_BITS, vec3, vec2, + FILTER_BITS, vec0, vec1); + dst0 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + + DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1); + DUP2_ARG2(__lsx_vavgr_bu, vec0, dst0, vec1, dst1, vec0, vec1); + __lsx_vstelm_d(vec0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(vec0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(vec1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(vec1, dst, 0, 1); +} + +static void common_hz_2t_and_aver_dst_8x8mult_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + __m128i src0, src1, src2, src3, mask; + __m128i filt0, dst0, dst1, dst2, dst3; + __m128i vec0, vec1, vec2, vec3; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + uint8_t *dst_tmp = dst; + + mask = __lsx_vld(mc_filt_mask_arr, 0); + + /* rearranging filter */ + filt0 = __lsx_vldrepl_h(filter, 0); + + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + src += src_stride; + + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1); + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0, + vec0, vec1, vec2, vec3); + DUP2_ARG3(__lsx_vssrarni_bu_h, vec1, vec0, FILTER_BITS, vec3, vec2, + FILTER_BITS, vec0, vec2); + dst0 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1); + + DUP2_ARG2(__lsx_vavgr_bu, vec0, dst0, vec2, dst1, vec0, vec2); + __lsx_vstelm_d(vec0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(vec0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(vec2, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(vec2, dst, 0, 1); + dst += dst_stride; + + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + src += src_stride; + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1); + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3); + + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0, + vec0, vec1, vec2, vec3); + DUP2_ARG3(__lsx_vssrarni_bu_h, vec1, vec0, FILTER_BITS, vec3, vec2, + FILTER_BITS, vec0, vec2); + dst0 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1); + DUP2_ARG2(__lsx_vavgr_bu, vec0, dst0, vec2, dst1, vec0, vec2); + __lsx_vstelm_d(vec0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(vec0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(vec2, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(vec2, dst, 0, 1); + dst += dst_stride; + + if (height == 16) { + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + src += src_stride; + + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1); + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, + filt0, vec0, vec1, vec2, vec3); + DUP2_ARG3(__lsx_vssrarni_bu_h, vec1, vec0, FILTER_BITS, vec3, vec2, + FILTER_BITS, vec0, vec2); + dst0 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1); + DUP2_ARG2(__lsx_vavgr_bu, vec0, dst0, vec2, dst1, vec0, vec2); + __lsx_vstelm_d(vec0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(vec0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(vec2, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(vec2, dst, 0, 1); + dst += dst_stride; + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1); + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, + filt0, vec0, vec1, vec2, vec3); + DUP2_ARG3(__lsx_vssrarni_bu_h, vec1, vec0, FILTER_BITS, vec3, vec2, + FILTER_BITS, vec0, vec2); + dst0 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1); + DUP2_ARG2(__lsx_vavgr_bu, vec0, dst0, vec2, dst1, vec0, vec2); + __lsx_vstelm_d(vec0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(vec0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(vec2, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(vec2, dst, 0, 1); + dst += dst_stride; + } +} + +static void common_hz_2t_and_aver_dst_8w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, int8_t *filter, + int32_t height) { + if (height == 4) { + common_hz_2t_and_aver_dst_8x4_lsx(src, src_stride, dst, dst_stride, filter); + } else { + common_hz_2t_and_aver_dst_8x8mult_lsx(src, src_stride, dst, dst_stride, + filter, height); + } +} + +static void common_hz_2t_and_aver_dst_16w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = (height >> 2) - 1; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, mask; + __m128i filt0, dst0; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i res0, res1, res2, res3, res4, res5, res6, res7; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + uint8_t *src_tmp1 = (uint8_t *)src + 8; + + mask = __lsx_vld(mc_filt_mask_arr, 0); + + /* rearranging filter */ + filt0 = __lsx_vldrepl_h(filter, 0); + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src4); + src6 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + src1 = __lsx_vld(src_tmp1, 0); + DUP2_ARG2(__lsx_vldx, src_tmp1, src_stride, src_tmp1, src_stride2, src3, + src5); + src7 = __lsx_vldx(src_tmp1, src_stride3); + src_tmp1 += src_stride4; + + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1); + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3); + DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask, src5, src5, mask, vec4, vec5); + DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask, src7, src7, mask, vec6, vec7); + + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0, + res0, res1, res2, res3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7, filt0, + res4, res5, res6, res7); + DUP4_ARG3(__lsx_vssrarni_bu_h, res1, res0, FILTER_BITS, res3, res2, + FILTER_BITS, res5, res4, FILTER_BITS, res7, res6, FILTER_BITS, res0, + res2, res4, res6); + dst0 = __lsx_vld(dst, 0); + res0 = __lsx_vavgr_bu(res0, dst0); + __lsx_vst(res0, dst, 0); + dst += dst_stride; + + dst0 = __lsx_vld(dst, 0); + res2 = __lsx_vavgr_bu(res2, dst0); + __lsx_vst(res2, dst, 0); + dst += dst_stride; + + dst0 = __lsx_vld(dst, 0); + res4 = __lsx_vavgr_bu(res4, dst0); + __lsx_vst(res4, dst, 0); + dst += dst_stride; + + dst0 = __lsx_vld(dst, 0); + res6 = __lsx_vavgr_bu(res6, dst0); + __lsx_vst(res6, dst, 0); + dst += dst_stride; + + for (; loop_cnt--;) { + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src4); + src6 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + src1 = __lsx_vld(src_tmp1, 0); + DUP2_ARG2(__lsx_vldx, src_tmp1, src_stride, src_tmp1, src_stride2, src3, + src5); + src7 = __lsx_vldx(src_tmp1, src_stride3); + src_tmp1 += src_stride4; + + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1); + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3); + DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask, src5, src5, mask, vec4, vec5); + DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask, src7, src7, mask, vec6, vec7); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, + filt0, res0, res1, res2, res3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7, + filt0, res4, res5, res6, res7); + + DUP4_ARG3(__lsx_vssrarni_bu_h, res1, res0, FILTER_BITS, res3, res2, + FILTER_BITS, res5, res4, FILTER_BITS, res7, res6, FILTER_BITS, + res0, res2, res4, res6); + dst0 = __lsx_vld(dst, 0); + res0 = __lsx_vavgr_bu(res0, dst0); + __lsx_vst(res0, dst, 0); + dst += dst_stride; + + dst0 = __lsx_vld(dst, 0); + res2 = __lsx_vavgr_bu(res2, dst0); + __lsx_vst(res2, dst, 0); + dst += dst_stride; + + dst0 = __lsx_vld(dst, 0); + res4 = __lsx_vavgr_bu(res4, dst0); + __lsx_vst(res4, dst, 0); + dst += dst_stride; + + dst0 = __lsx_vld(dst, 0); + res6 = __lsx_vavgr_bu(res6, dst0); + __lsx_vst(res6, dst, 0); + dst += dst_stride; + } +} + +static void common_hz_2t_and_aver_dst_32w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = (height >> 1); + __m128i src0, src1, src2, src3, src4, src5, src6, src7, mask; + __m128i filt0, dst0, dst1; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i res0, res1, res2, res3, res4, res5, res6, res7; + __m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 }; + + mask = __lsx_vld(mc_filt_mask_arr, 0); + + /* rearranging filter */ + filt0 = __lsx_vldrepl_h(filter, 0); + + for (; loop_cnt--;) { + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vld, src, 16, src, 24, src2, src3); + src1 = __lsx_vshuf_b(src2, src0, shuff); + src += src_stride; + src4 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vld, src, 16, src, 24, src6, src7); + src5 = __lsx_vshuf_b(src6, src4, shuff); + src += src_stride; + + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1); + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3); + DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask, src5, src5, mask, vec4, vec5); + DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask, src7, src7, mask, vec6, vec7); + + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, + filt0, res0, res1, res2, res3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7, + filt0, res4, res5, res6, res7); + DUP4_ARG3(__lsx_vssrarni_bu_h, res1, res0, FILTER_BITS, res3, res2, + FILTER_BITS, res5, res4, FILTER_BITS, res7, res6, FILTER_BITS, + res0, res2, res4, res6); + + DUP2_ARG2(__lsx_vld, dst, 0, dst, 16, dst0, dst1); + res0 = __lsx_vavgr_bu(res0, dst0); + __lsx_vst(res0, dst, 0); + res2 = __lsx_vavgr_bu(res2, dst1); + __lsx_vst(res2, dst, 16); + dst += dst_stride; + + DUP2_ARG2(__lsx_vld, dst, 0, dst, 16, dst0, dst1); + res4 = __lsx_vavgr_bu(res4, dst0); + __lsx_vst(res4, dst, 0); + res6 = __lsx_vavgr_bu(res6, dst1); + __lsx_vst(res6, dst, 16); + dst += dst_stride; + } +} + +static void common_hz_2t_and_aver_dst_64w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = height; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, mask; + __m128i filt0, dst0, dst1, dst2, dst3; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i out0, out1, out2, out3, out4, out5, out6, out7; + __m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 }; + + mask = __lsx_vld(mc_filt_mask_arr, 0); + + /* rearranging filter */ + filt0 = __lsx_vldrepl_h(filter, 0); + + for (; loop_cnt--;) { + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src2, src4, + src6); + src7 = __lsx_vld(src, 56); + DUP2_ARG3(__lsx_vshuf_b, src2, src0, shuff, src4, src2, shuff, src1, src3); + src5 = __lsx_vshuf_b(src6, src4, shuff); + src += src_stride; + + DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, vec0, vec1); + DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask, src3, src3, mask, vec2, vec3); + DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask, src5, src5, mask, vec4, vec5); + DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask, src7, src7, mask, vec6, vec7); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, + filt0, out0, out1, out2, out3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7, + filt0, out4, out5, out6, out7); + + DUP4_ARG3(__lsx_vssrarni_bu_h, out1, out0, FILTER_BITS, out3, out2, + FILTER_BITS, out5, out4, FILTER_BITS, out7, out6, FILTER_BITS, + out0, out2, out4, out6); + + DUP4_ARG2(__lsx_vld, dst, 0, dst, 16, dst, 32, dst, 48, dst0, dst1, dst2, + dst3); + out0 = __lsx_vavgr_bu(out0, dst0); + __lsx_vst(out0, dst, 0); + out2 = __lsx_vavgr_bu(out2, dst1); + __lsx_vst(out2, dst, 16); + out4 = __lsx_vavgr_bu(out4, dst2); + __lsx_vst(out4, dst, 32); + out6 = __lsx_vavgr_bu(out6, dst3); + __lsx_vst(out6, dst, 48); + dst += dst_stride; + } +} + +void vpx_convolve8_avg_horiz_lsx(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *filter, int x0_q4, + int x_step_q4, int y0_q4, int y_step_q4, int w, + int h) { + const int16_t *const filter_x = filter[x0_q4]; + int8_t cnt, filt_hor[8]; + + assert(x_step_q4 == 16); + assert(((const int32_t *)filter_x)[1] != 0x800000); + + for (cnt = 0; cnt < 8; ++cnt) { + filt_hor[cnt] = filter_x[cnt]; + } + + if (vpx_get_filter_taps(filter_x) == 2) { + switch (w) { + case 4: + common_hz_2t_and_aver_dst_4w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], h); + break; + case 8: + common_hz_2t_and_aver_dst_8w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], h); + break; + case 16: + common_hz_2t_and_aver_dst_16w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], h); + break; + + case 32: + common_hz_2t_and_aver_dst_32w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], h); + break; + case 64: + common_hz_2t_and_aver_dst_64w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], h); + break; + default: + vpx_convolve8_avg_horiz_c(src, src_stride, dst, dst_stride, filter, + x0_q4, x_step_q4, y0_q4, y_step_q4, w, h); + break; + } + } else { + switch (w) { + case 4: + common_hz_8t_and_aver_dst_4w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, h); + break; + case 8: + common_hz_8t_and_aver_dst_8w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, h); + break; + case 16: + common_hz_8t_and_aver_dst_16w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, h); + break; + case 32: + common_hz_8t_and_aver_dst_32w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, h); + break; + case 64: + common_hz_8t_and_aver_dst_64w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, h); + break; + default: + vpx_convolve8_avg_horiz_c(src, src_stride, dst, dst_stride, filter, + x0_q4, x_step_q4, y0_q4, y_step_q4, w, h); + break; + } + } +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_avg_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_avg_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_avg_lsx.c @@ -0,0 +1,748 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include "./vpx_dsp_rtcd.h" +#include "vpx_dsp/loongarch/vpx_convolve_lsx.h" + +static const uint8_t mc_filt_mask_arr[16 * 3] = { + /* 8 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, + /* 4 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20, + /* 4 width cases */ + 8, 9, 9, 10, 10, 11, 11, 12, 24, 25, 25, 26, 26, 27, 27, 28 +}; + +static void common_hv_8ht_8vt_and_aver_dst_4w_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + const int8_t *filter_horiz, const int8_t *filter_vert, int32_t height) { + uint32_t loop_cnt = height >> 2; + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i filt_hz0, filt_hz1, filt_hz2, filt_hz3; + __m128i filt_vt0, filt_vt1, filt_vt2, filt_vt3; + __m128i mask0, mask1, mask2, mask3; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + __m128i out0, out1; + __m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 }; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride + src_stride2; + int32_t src_stride4 = src_stride2 << 1; + uint8_t *_src = (uint8_t *)src - 3 - src_stride3; + + mask0 = __lsx_vld(mc_filt_mask_arr, 16); + DUP4_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_horiz, 2, filter_horiz, 4, + filter_horiz, 6, filt_hz0, filt_hz1, filt_hz2, filt_hz3); + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + + src0 = __lsx_vld(_src, 0); + DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src1, src2); + src3 = __lsx_vldx(_src, src_stride3); + _src += src_stride4; + src4 = __lsx_vld(_src, 0); + DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src5, src6); + _src += src_stride3; + + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + + tmp0 = HORIZ_8TAP_FILT(src0, src1, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + tmp2 = HORIZ_8TAP_FILT(src2, src3, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + tmp4 = HORIZ_8TAP_FILT(src4, src5, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + tmp5 = HORIZ_8TAP_FILT(src5, src6, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + DUP2_ARG3(__lsx_vshuf_b, tmp2, tmp0, shuff, tmp4, tmp2, shuff, tmp1, tmp3); + DUP4_ARG2(__lsx_vldrepl_h, filter_vert, 0, filter_vert, 2, filter_vert, 4, + filter_vert, 6, filt_vt0, filt_vt1, filt_vt2, filt_vt3); + DUP2_ARG2(__lsx_vpackev_b, tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + tmp2 = __lsx_vpackev_b(tmp5, tmp4); + for (; loop_cnt--;) { + src7 = __lsx_vld(_src, 0); + DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src8, src9); + src10 = __lsx_vldx(_src, src_stride3); + _src += src_stride4; + src2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + src3 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + src4 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + src5 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_w, src3, src2, src5, src4, src2, src3); + src2 = __lsx_vilvl_d(src3, src2); + DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7, + src8, src9, src10); + tmp3 = HORIZ_8TAP_FILT(src7, src8, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + tmp4 = __lsx_vshuf_b(tmp3, tmp5, shuff); + tmp4 = __lsx_vpackev_b(tmp3, tmp4); + out0 = FILT_8TAP_DPADD_S_H(tmp0, tmp1, tmp2, tmp4, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + src1 = HORIZ_8TAP_FILT(src9, src10, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src0 = __lsx_vshuf_b(src1, tmp3, shuff); + src0 = __lsx_vpackev_b(src1, src0); + out1 = FILT_8TAP_DPADD_S_H(tmp1, tmp2, tmp4, src0, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + out0 = __lsx_vssrarni_b_h(out1, out0, FILTER_BITS); + out0 = __lsx_vxori_b(out0, 128); + out0 = __lsx_vavgr_bu(out0, src2); + __lsx_vstelm_w(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 3); + dst += dst_stride; + + tmp5 = src1; + tmp0 = tmp2; + tmp1 = tmp4; + tmp2 = src0; + } +} + +static void common_hv_8ht_8vt_and_aver_dst_8w_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + const int8_t *filter_horiz, const int8_t *filter_vert, int32_t height) { + uint32_t loop_cnt = height >> 2; + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i filt_hz0, filt_hz1, filt_hz2, filt_hz3; + __m128i filt_vt0, filt_vt1, filt_vt2, filt_vt3; + __m128i mask0, mask1, mask2, mask3; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6; + __m128i out0, out1; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride + src_stride2; + int32_t src_stride4 = src_stride2 << 1; + uint8_t *_src = (uint8_t *)src - 3 - src_stride3; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + DUP4_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_horiz, 2, filter_horiz, 4, + filter_horiz, 6, filt_hz0, filt_hz1, filt_hz2, filt_hz3); + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + + src0 = __lsx_vld(_src, 0); + DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src1, src2); + src3 = __lsx_vldx(_src, src_stride3); + _src += src_stride4; + src4 = __lsx_vld(_src, 0); + DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src5, src6); + _src += src_stride3; + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + + src0 = HORIZ_8TAP_FILT(src0, src0, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src1 = HORIZ_8TAP_FILT(src1, src1, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src2 = HORIZ_8TAP_FILT(src2, src2, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src3 = HORIZ_8TAP_FILT(src3, src3, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src4 = HORIZ_8TAP_FILT(src4, src4, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src5 = HORIZ_8TAP_FILT(src5, src5, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src6 = HORIZ_8TAP_FILT(src6, src6, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + + DUP4_ARG2(__lsx_vldrepl_h, filter_vert, 0, filter_vert, 2, filter_vert, 4, + filter_vert, 6, filt_vt0, filt_vt1, filt_vt2, filt_vt3); + DUP4_ARG2(__lsx_vpackev_b, src1, src0, src3, src2, src5, src4, src2, src1, + tmp0, tmp1, tmp2, tmp4); + DUP2_ARG2(__lsx_vpackev_b, src4, src3, src6, src5, tmp5, tmp6); + + for (; loop_cnt--;) { + src7 = __lsx_vld(_src, 0); + DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src8, src9); + src10 = __lsx_vldx(_src, src_stride3); + _src += src_stride4; + + DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7, + src8, src9, src10); + src7 = HORIZ_8TAP_FILT(src7, src7, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + tmp3 = __lsx_vpackev_b(src7, src6); + out0 = FILT_8TAP_DPADD_S_H(tmp0, tmp1, tmp2, tmp3, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + src8 = HORIZ_8TAP_FILT(src8, src8, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src0 = __lsx_vpackev_b(src8, src7); + out1 = FILT_8TAP_DPADD_S_H(tmp4, tmp5, tmp6, src0, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + src9 = HORIZ_8TAP_FILT(src9, src9, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src1 = __lsx_vpackev_b(src9, src8); + src3 = FILT_8TAP_DPADD_S_H(tmp1, tmp2, tmp3, src1, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + src10 = HORIZ_8TAP_FILT(src10, src10, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src2 = __lsx_vpackev_b(src10, src9); + src4 = FILT_8TAP_DPADD_S_H(tmp5, tmp6, src0, src2, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, FILTER_BITS, src4, src3, + FILTER_BITS, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + src5 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + src7 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + src8 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + src9 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_d, src7, src5, src9, src8, src5, src7); + DUP2_ARG2(__lsx_vavgr_bu, out0, src5, out1, src7, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 1); + dst += dst_stride; + + src6 = src10; + tmp0 = tmp2; + tmp1 = tmp3; + tmp2 = src1; + tmp4 = tmp6; + tmp5 = src0; + tmp6 = src2; + } +} + +static void common_hv_8ht_8vt_and_aver_dst_16w_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + const int8_t *filter_horiz, const int8_t *filter_vert, int32_t height) { + common_hv_8ht_8vt_and_aver_dst_8w_lsx(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, height); + src += 8; + dst += 8; + + common_hv_8ht_8vt_and_aver_dst_8w_lsx(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, height); +} + +static void common_hv_8ht_8vt_and_aver_dst_32w_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + const int8_t *filter_horiz, const int8_t *filter_vert, int32_t height) { + int32_t multiple8_cnt; + + for (multiple8_cnt = 4; multiple8_cnt--;) { + common_hv_8ht_8vt_and_aver_dst_8w_lsx(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, height); + src += 8; + dst += 8; + } +} + +static void common_hv_8ht_8vt_and_aver_dst_64w_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + const int8_t *filter_horiz, const int8_t *filter_vert, int32_t height) { + int32_t multiple8_cnt; + + for (multiple8_cnt = 8; multiple8_cnt--;) { + common_hv_8ht_8vt_and_aver_dst_8w_lsx(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, height); + + src += 8; + dst += 8; + } +} + +static void common_hv_2ht_2vt_and_aver_dst_4x4_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert) { + __m128i src0, src1, src2, src3, src4, mask; + __m128i filt_hz, filt_vt, vec0, vec1; + __m128i dst0, dst1, dst2, dst3; + __m128i hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, tmp0, tmp1, out; + __m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 }; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + + mask = __lsx_vld(mc_filt_mask_arr, 16); + /* rearranging filter */ + DUP2_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_vert, 0, filt_hz, filt_vt); + + src0 = __lsx_vld(src, 0); + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src1, src2, src3, src4); + + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src1, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src2, src3, mask, filt_hz, FILTER_BITS); + hz_out4 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + hz_out1 = __lsx_vshuf_b(hz_out2, hz_out0, shuff); + hz_out3 = __lsx_vpickod_d(hz_out4, hz_out2); + DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1); + + dst0 = __lsx_vldrepl_w(dst, 0); + dst1 = __lsx_vldrepl_w(dst + dst_stride, 0); + dst2 = __lsx_vldrepl_w(dst + dst_stride2, 0); + dst3 = __lsx_vldrepl_w(dst + dst_stride3, 0); + dst0 = __lsx_vilvl_w(dst1, dst0); + dst1 = __lsx_vilvl_w(dst3, dst2); + dst0 = __lsx_vilvl_d(dst1, dst0); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1); + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + out = __lsx_vpickev_b(tmp1, tmp0); + out = __lsx_vavgr_bu(out, dst0); + __lsx_vstelm_w(out, dst, 0, 0); + __lsx_vstelm_w(out, dst + dst_stride, 0, 1); + __lsx_vstelm_w(out, dst + dst_stride2, 0, 2); + __lsx_vstelm_w(out, dst + dst_stride3, 0, 3); +} + +static void common_hv_2ht_2vt_and_aver_dst_4x8_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert) { + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, mask; + __m128i filt_hz, filt_vt, vec0, vec1, vec2, vec3, res0, res1; + __m128i hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6; + __m128i hz_out7, hz_out8, tmp0, tmp1, tmp2, tmp3; + __m128i dst0, dst1, dst2, dst3, dst4; + __m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 }; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + mask = __lsx_vld(mc_filt_mask_arr, 16); + + /* rearranging filter */ + filt_hz = __lsx_vldrepl_h(filter_horiz, 0); + filt_vt = __lsx_vldrepl_h(filter_vert, 0); + + src0 = __lsx_vld(src, 0); + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src1, src2, src3, src4); + src += src_stride4; + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src5, src6, src7, src8); + src += src_stride4; + + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src1, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src2, src3, mask, filt_hz, FILTER_BITS); + hz_out4 = HORIZ_2TAP_FILT_UH(src4, src5, mask, filt_hz, FILTER_BITS); + hz_out6 = HORIZ_2TAP_FILT_UH(src6, src7, mask, filt_hz, FILTER_BITS); + hz_out8 = HORIZ_2TAP_FILT_UH(src8, src8, mask, filt_hz, FILTER_BITS); + DUP2_ARG3(__lsx_vshuf_b, hz_out2, hz_out0, shuff, hz_out4, hz_out2, shuff, + hz_out1, hz_out3); + hz_out5 = __lsx_vshuf_b(hz_out6, hz_out4, shuff); + hz_out7 = __lsx_vpickod_d(hz_out8, hz_out6); + + dst0 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst0 = __lsx_vilvl_w(dst1, dst0); + dst1 = __lsx_vilvl_w(dst3, dst2); + dst0 = __lsx_vilvl_d(dst1, dst0); + + dst1 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst4 = __lsx_vldrepl_w(dst_tmp, 0); + dst1 = __lsx_vilvl_w(dst2, dst1); + dst2 = __lsx_vilvl_w(dst4, dst3); + dst1 = __lsx_vilvl_d(dst2, dst1); + + DUP4_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, hz_out5, + hz_out4, hz_out7, hz_out6, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, vec2, filt_vt, vec3, + filt_vt, tmp0, tmp1, tmp2, tmp3); + DUP4_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp2, + FILTER_BITS, tmp3, FILTER_BITS, tmp0, tmp1, tmp2, tmp3); + DUP2_ARG2(__lsx_vpickev_b, tmp1, tmp0, tmp3, tmp2, res0, res1); + DUP2_ARG2(__lsx_vavgr_bu, res0, dst0, res1, dst1, res0, res1); + + __lsx_vstelm_w(res0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(res0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(res0, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(res0, dst, 0, 3); + dst += dst_stride; + + __lsx_vstelm_w(res1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(res1, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(res1, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(res1, dst, 0, 3); +} + +static void common_hv_2ht_2vt_and_aver_dst_4w_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, int32_t height) { + if (height == 4) { + common_hv_2ht_2vt_and_aver_dst_4x4_lsx(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert); + } else if (height == 8) { + common_hv_2ht_2vt_and_aver_dst_4x8_lsx(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert); + } +} + +static void common_hv_2ht_2vt_and_aver_dst_8x4_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert) { + __m128i src0, src1, src2, src3, src4, mask; + __m128i filt_hz, filt_vt, vec0, vec1, vec2, vec3; + __m128i hz_out0, hz_out1, tmp0, tmp1, tmp2, tmp3; + __m128i dst0, dst1, dst2, dst3; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + uint8_t *dst_tmp = dst; + mask = __lsx_vld(mc_filt_mask_arr, 0); + /* rearranging filter */ + filt_hz = __lsx_vldrepl_h(filter_horiz, 0); + filt_vt = __lsx_vldrepl_h(filter_vert, 0); + + src0 = __lsx_vld(src, 0); + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src1, src2, src3, src4); + + dst0 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_d(dst_tmp, 0); + DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1); + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS); + hz_out1 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + vec0 = __lsx_vpackev_b(hz_out1, hz_out0); + tmp0 = __lsx_vdp2_h_bu(vec0, filt_vt); + + hz_out0 = HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS); + vec1 = __lsx_vpackev_b(hz_out0, hz_out1); + tmp1 = __lsx_vdp2_h_bu(vec1, filt_vt); + + hz_out1 = HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS); + vec2 = __lsx_vpackev_b(hz_out1, hz_out0); + tmp2 = __lsx_vdp2_h_bu(vec2, filt_vt); + + hz_out0 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + vec3 = __lsx_vpackev_b(hz_out0, hz_out1); + tmp3 = __lsx_vdp2_h_bu(vec3, filt_vt); + + DUP4_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp2, + FILTER_BITS, tmp3, FILTER_BITS, tmp0, tmp1, tmp2, tmp3); + PCKEV_AVG_ST4_D(tmp0, tmp1, tmp2, tmp3, dst0, dst1, dst, dst_stride); +} + +static void common_hv_2ht_2vt_and_aver_dst_8x8mult_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, int32_t height) { + uint32_t loop_cnt = (height >> 2); + __m128i src0, src1, src2, src3, src4, mask; + __m128i filt_hz, filt_vt, vec0; + __m128i hz_out0, hz_out1, tmp0, tmp1, tmp2, tmp3; + __m128i dst0, dst1, dst2, dst3; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + uint8_t *dst_tmp = dst; + + /* rearranging filter */ + mask = __lsx_vld(mc_filt_mask_arr, 0); + + filt_hz = __lsx_vldrepl_h(filter_horiz, 0); + filt_vt = __lsx_vldrepl_h(filter_vert, 0); + + src0 = __lsx_vld(src, 0); + src += src_stride; + + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS); + + for (; loop_cnt--;) { + src1 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src3); + src4 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + hz_out1 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + vec0 = __lsx_vpackev_b(hz_out1, hz_out0); + tmp0 = __lsx_vdp2_h_bu(vec0, filt_vt); + + hz_out0 = HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS); + vec0 = __lsx_vpackev_b(hz_out0, hz_out1); + tmp1 = __lsx_vdp2_h_bu(vec0, filt_vt); + + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + + hz_out1 = HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS); + vec0 = __lsx_vpackev_b(hz_out1, hz_out0); + tmp2 = __lsx_vdp2_h_bu(vec0, filt_vt); + + hz_out0 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + vec0 = __lsx_vpackev_b(hz_out0, hz_out1); + tmp3 = __lsx_vdp2_h_bu(vec0, filt_vt); + + DUP2_ARG2(__lsx_vsrari_h, tmp2, FILTER_BITS, tmp3, FILTER_BITS, tmp2, tmp3); + + dst0 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1); + PCKEV_AVG_ST4_D(tmp0, tmp1, tmp2, tmp3, dst0, dst1, dst, dst_stride); + dst += dst_stride; + } +} + +static void common_hv_2ht_2vt_and_aver_dst_8w_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, int32_t height) { + if (height == 4) { + common_hv_2ht_2vt_and_aver_dst_8x4_lsx(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert); + } else { + common_hv_2ht_2vt_and_aver_dst_8x8mult_lsx( + src, src_stride, dst, dst_stride, filter_horiz, filter_vert, height); + } +} + +static void common_hv_2ht_2vt_and_aver_dst_16w_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, int32_t height) { + uint8_t *src_tmp1; + uint32_t loop_cnt = (height >> 2); + __m128i src0, src1, src2, src3, src4, src5, src6, src7, mask; + __m128i filt_hz, filt_vt, vec0, vec1, dst0, dst1, dst2, dst3; + __m128i hz_out0, hz_out1, hz_out2, hz_out3, tmp0, tmp1, tmp3; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + int32_t dst_stride4 = dst_stride << 2; + + mask = __lsx_vld(mc_filt_mask_arr, 0); + /* rearranging filter */ + filt_hz = __lsx_vldrepl_h(filter_horiz, 0); + filt_vt = __lsx_vldrepl_h(filter_vert, 0); + + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src0, src1); + src += src_stride; + + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + + for (; loop_cnt--;) { + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src4); + src6 = __lsx_vldx(src, src_stride3); + src_tmp1 = (uint8_t *)(src + 8); + src1 = __lsx_vld(src_tmp1, 0); + DUP2_ARG2(__lsx_vldx, src_tmp1, src_stride, src_tmp1, src_stride2, src3, + src5); + src7 = __lsx_vldx(src_tmp1, src_stride3); + src += src_stride4; + dst0 = __lsx_vld(dst, 0); + DUP2_ARG2(__lsx_vldx, dst, dst_stride, dst, dst_stride2, dst1, dst2); + dst3 = __lsx_vldx(dst, dst_stride3); + + hz_out1 = HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS); + hz_out3 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1); + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + tmp3 = __lsx_vpickev_b(tmp1, tmp0); + tmp3 = __lsx_vavgr_bu(tmp3, dst0); + __lsx_vst(tmp3, dst, 0); + + hz_out0 = HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS); + DUP2_ARG2(__lsx_vpackev_b, hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1); + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + tmp3 = __lsx_vpickev_b(tmp1, tmp0); + tmp3 = __lsx_vavgr_bu(tmp3, dst1); + __lsx_vstx(tmp3, dst, dst_stride); + + hz_out1 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + hz_out3 = HORIZ_2TAP_FILT_UH(src5, src5, mask, filt_hz, FILTER_BITS); + DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1); + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + tmp3 = __lsx_vpickev_b(tmp1, tmp0); + tmp3 = __lsx_vavgr_bu(tmp3, dst2); + __lsx_vstx(tmp3, dst, dst_stride2); + + hz_out0 = HORIZ_2TAP_FILT_UH(src6, src6, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src7, src7, mask, filt_hz, FILTER_BITS); + DUP2_ARG2(__lsx_vpackev_b, hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1); + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + tmp3 = __lsx_vpickev_b(tmp1, tmp0); + tmp3 = __lsx_vavgr_bu(tmp3, dst3); + __lsx_vstx(tmp3, dst, dst_stride3); + dst += dst_stride4; + } +} + +static void common_hv_2ht_2vt_and_aver_dst_32w_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, int32_t height) { + common_hv_2ht_2vt_and_aver_dst_16w_lsx(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, height); + src += 16; + dst += 16; + + common_hv_2ht_2vt_and_aver_dst_16w_lsx(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, height); + src += 16; + dst += 16; +} + +static void common_hv_2ht_2vt_and_aver_dst_64w_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, int32_t height) { + int32_t multiple8_cnt; + for (multiple8_cnt = 4; multiple8_cnt--;) { + common_hv_2ht_2vt_and_aver_dst_16w_lsx(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, height); + src += 16; + dst += 16; + } +} + +void vpx_convolve8_avg_lsx(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *filter, int x0_q4, int x_step_q4, + int y0_q4, int y_step_q4, int w, int h) { + const int16_t *const filter_x = filter[x0_q4]; + const int16_t *const filter_y = filter[y0_q4]; + int8_t cnt, filt_hor[8], filt_ver[8]; + + assert(x_step_q4 == 16); + assert(y_step_q4 == 16); + assert(((const int32_t *)filter_x)[1] != 0x800000); + assert(((const int32_t *)filter_y)[1] != 0x800000); + + for (cnt = 0; cnt < 8; ++cnt) { + filt_hor[cnt] = filter_x[cnt]; + filt_ver[cnt] = filter_y[cnt]; + } + if (vpx_get_filter_taps(filter_x) == 2 && + vpx_get_filter_taps(filter_y) == 2) { + switch (w) { + case 4: + common_hv_2ht_2vt_and_aver_dst_4w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], + &filt_ver[3], h); + break; + case 8: + common_hv_2ht_2vt_and_aver_dst_8w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], + &filt_ver[3], h); + break; + case 16: + common_hv_2ht_2vt_and_aver_dst_16w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, + &filt_hor[3], &filt_ver[3], h); + break; + case 32: + common_hv_2ht_2vt_and_aver_dst_32w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, + &filt_hor[3], &filt_ver[3], h); + break; + case 64: + common_hv_2ht_2vt_and_aver_dst_64w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, + &filt_hor[3], &filt_ver[3], h); + break; + default: + vpx_convolve8_avg_c(src, src_stride, dst, dst_stride, filter, x0_q4, + x_step_q4, y0_q4, y_step_q4, w, h); + break; + } + } else if (vpx_get_filter_taps(filter_x) == 2 || + vpx_get_filter_taps(filter_y) == 2) { + vpx_convolve8_avg_c(src, src_stride, dst, dst_stride, filter, x0_q4, + x_step_q4, y0_q4, y_step_q4, w, h); + } else { + switch (w) { + case 4: + common_hv_8ht_8vt_and_aver_dst_4w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, + filt_ver, h); + break; + case 8: + common_hv_8ht_8vt_and_aver_dst_8w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, + filt_ver, h); + break; + case 16: + common_hv_8ht_8vt_and_aver_dst_16w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, + filt_ver, h); + break; + case 32: + common_hv_8ht_8vt_and_aver_dst_32w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, + filt_ver, h); + break; + case 64: + common_hv_8ht_8vt_and_aver_dst_64w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, + filt_ver, h); + break; + default: + vpx_convolve8_avg_c(src, src_stride, dst, dst_stride, filter, x0_q4, + x_step_q4, y0_q4, y_step_q4, w, h); + break; + } + } +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_avg_vert_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_avg_vert_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_avg_vert_lsx.c @@ -0,0 +1,918 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include "./vpx_dsp_rtcd.h" +#include "vpx_dsp/loongarch/vpx_convolve_lsx.h" + +static void common_vt_8t_and_aver_dst_4w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) { + uint32_t loop_cnt = (height >> 2); + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + __m128i reg0, reg1, reg2, reg3, reg4; + __m128i filter0, filter1, filter2, filter3; + __m128i out0, out1; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride + src_stride2; + int32_t src_stride4 = src_stride2 << 1; + uint8_t *src_tmp0 = (uint8_t *)src - src_stride3; + + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + src0 = __lsx_vld(src_tmp0, 0); + DUP2_ARG2(__lsx_vldx, src_tmp0, src_stride, src_tmp0, src_stride2, src1, + src2); + src3 = __lsx_vldx(src_tmp0, src_stride3); + src_tmp0 += src_stride4; + src4 = __lsx_vld(src_tmp0, 0); + DUP2_ARG2(__lsx_vldx, src_tmp0, src_stride, src_tmp0, src_stride2, src5, + src6); + src_tmp0 += src_stride3; + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, tmp0, + tmp1, tmp2, tmp3); + DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, tmp4, tmp5); + DUP2_ARG2(__lsx_vilvl_d, tmp3, tmp0, tmp4, tmp1, reg0, reg1); + reg2 = __lsx_vilvl_d(tmp5, tmp2); + DUP2_ARG2(__lsx_vxori_b, reg0, 128, reg1, 128, reg0, reg1); + reg2 = __lsx_vxori_b(reg2, 128); + + for (; loop_cnt--;) { + src7 = __lsx_vld(src_tmp0, 0); + DUP2_ARG2(__lsx_vldx, src_tmp0, src_stride, src_tmp0, src_stride2, src8, + src9); + src10 = __lsx_vldx(src_tmp0, src_stride3); + src_tmp0 += src_stride4; + src0 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + src1 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + src2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + src3 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_w, src1, src0, src3, src2, src0, src1); + src0 = __lsx_vilvl_d(src1, src0); + DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9, + tmp0, tmp1, tmp2, tmp3); + DUP2_ARG2(__lsx_vilvl_d, tmp1, tmp0, tmp3, tmp2, reg3, reg4); + DUP2_ARG2(__lsx_vxori_b, reg3, 128, reg4, 128, reg3, reg4); + out0 = FILT_8TAP_DPADD_S_H(reg0, reg1, reg2, reg3, filter0, filter1, + filter2, filter3); + out1 = FILT_8TAP_DPADD_S_H(reg1, reg2, reg3, reg4, filter0, filter1, + filter2, filter3); + out0 = __lsx_vssrarni_b_h(out1, out0, 7); + out0 = __lsx_vxori_b(out0, 128); + out0 = __lsx_vavgr_bu(out0, src0); + __lsx_vstelm_w(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 3); + dst += dst_stride; + reg0 = reg2; + reg1 = reg3; + reg2 = reg4; + src6 = src10; + } +} + +static void common_vt_8t_and_aver_dst_8w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) { + uint32_t loop_cnt = height >> 2; + uint8_t *dst_tmp = dst; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i tmp0, tmp1, tmp2, tmp3; + __m128i reg0, reg1, reg2, reg3, reg4, reg5; + __m128i filter0, filter1, filter2, filter3; + __m128i out0, out1, out2, out3; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride + src_stride2; + int32_t src_stride4 = src_stride2 << 1; + uint8_t *src_tmp0 = (uint8_t *)src - src_stride3; + + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + src0 = __lsx_vld(src_tmp0, 0); + DUP2_ARG2(__lsx_vldx, src_tmp0, src_stride, src_tmp0, src_stride2, src1, + src2); + src3 = __lsx_vldx(src_tmp0, src_stride3); + src_tmp0 += src_stride4; + src4 = __lsx_vld(src_tmp0, 0); + DUP2_ARG2(__lsx_vldx, src_tmp0, src_stride, src_tmp0, src_stride2, src5, + src6); + src_tmp0 += src_stride3; + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, reg0, + reg1, reg2, reg3); + DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, reg4, reg5); + + for (; loop_cnt--;) { + src7 = __lsx_vld(src_tmp0, 0); + DUP2_ARG2(__lsx_vldx, src_tmp0, src_stride, src_tmp0, src_stride2, src8, + src9); + src10 = __lsx_vldx(src_tmp0, src_stride3); + src_tmp0 += src_stride4; + src0 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + src1 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + src2 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + src3 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_d, src1, src0, src3, src2, src0, src1); + DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7, + src8, src9, src10); + DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9, + tmp0, tmp1, tmp2, tmp3); + out0 = FILT_8TAP_DPADD_S_H(reg0, reg1, reg2, tmp0, filter0, filter1, + filter2, filter3); + out1 = FILT_8TAP_DPADD_S_H(reg3, reg4, reg5, tmp1, filter0, filter1, + filter2, filter3); + out2 = FILT_8TAP_DPADD_S_H(reg1, reg2, tmp0, tmp2, filter0, filter1, + filter2, filter3); + out3 = FILT_8TAP_DPADD_S_H(reg4, reg5, tmp1, tmp3, filter0, filter1, + filter2, filter3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + DUP2_ARG2(__lsx_vavgr_bu, out0, src0, out1, src1, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 1); + dst += dst_stride; + reg0 = reg2; + reg1 = tmp0; + reg2 = tmp2; + reg3 = reg5; + reg4 = tmp1; + reg5 = tmp3; + src6 = src10; + } +} + +static void common_vt_8t_and_aver_dst_16w_mult_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height, int32_t width) { + uint8_t *src_tmp; + uint32_t cnt = width >> 4; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i filter0, filter1, filter2, filter3; + __m128i reg0, reg1, reg2, reg3, reg4, reg5; + __m128i reg6, reg7, reg8, reg9, reg10, reg11; + __m128i tmp0, tmp1, tmp2, tmp3; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride + src_stride2; + int32_t src_stride4 = src_stride2 << 1; + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + int32_t dst_stride4 = dst_stride2 << 1; + uint8_t *src_tmp0 = (uint8_t *)src - src_stride3; + + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + for (; cnt--;) { + uint32_t loop_cnt = height >> 2; + uint8_t *dst_reg = dst; + + src_tmp = src_tmp0; + src0 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src1, + src2); + src3 = __lsx_vldx(src_tmp, src_stride3); + src_tmp += src_stride4; + src4 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src5, + src6); + src_tmp += src_stride3; + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, + reg0, reg1, reg2, reg3); + DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, reg4, reg5); + DUP4_ARG2(__lsx_vilvh_b, src1, src0, src3, src2, src5, src4, src2, src1, + reg6, reg7, reg8, reg9); + DUP2_ARG2(__lsx_vilvh_b, src4, src3, src6, src5, reg10, reg11); + for (; loop_cnt--;) { + src7 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src8, + src9); + src10 = __lsx_vldx(src_tmp, src_stride3); + src_tmp += src_stride4; + DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, + src7, src8, src9, src10); + DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9, + src0, src1, src2, src3); + DUP4_ARG2(__lsx_vilvh_b, src7, src6, src8, src7, src9, src8, src10, src9, + src4, src5, src7, src8); + tmp0 = FILT_8TAP_DPADD_S_H(reg0, reg1, reg2, src0, filter0, filter1, + filter2, filter3); + tmp1 = FILT_8TAP_DPADD_S_H(reg3, reg4, reg5, src1, filter0, filter1, + filter2, filter3); + tmp2 = FILT_8TAP_DPADD_S_H(reg6, reg7, reg8, src4, filter0, filter1, + filter2, filter3); + tmp3 = FILT_8TAP_DPADD_S_H(reg9, reg10, reg11, src5, filter0, filter1, + filter2, filter3); + DUP2_ARG3(__lsx_vssrarni_b_h, tmp2, tmp0, 7, tmp3, tmp1, 7, tmp0, tmp1); + DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + tmp2 = __lsx_vld(dst_reg, 0); + tmp3 = __lsx_vldx(dst_reg, dst_stride); + DUP2_ARG2(__lsx_vavgr_bu, tmp0, tmp2, tmp1, tmp3, tmp0, tmp1); + __lsx_vst(tmp0, dst_reg, 0); + __lsx_vstx(tmp1, dst_reg, dst_stride); + tmp0 = FILT_8TAP_DPADD_S_H(reg1, reg2, src0, src2, filter0, filter1, + filter2, filter3); + tmp1 = FILT_8TAP_DPADD_S_H(reg4, reg5, src1, src3, filter0, filter1, + filter2, filter3); + tmp2 = FILT_8TAP_DPADD_S_H(reg7, reg8, src4, src7, filter0, filter1, + filter2, filter3); + tmp3 = FILT_8TAP_DPADD_S_H(reg10, reg11, src5, src8, filter0, filter1, + filter2, filter3); + DUP2_ARG3(__lsx_vssrarni_b_h, tmp2, tmp0, 7, tmp3, tmp1, 7, tmp0, tmp1); + DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + tmp2 = __lsx_vldx(dst_reg, dst_stride2); + tmp3 = __lsx_vldx(dst_reg, dst_stride3); + DUP2_ARG2(__lsx_vavgr_bu, tmp0, tmp2, tmp1, tmp3, tmp0, tmp1); + __lsx_vstx(tmp0, dst_reg, dst_stride2); + __lsx_vstx(tmp1, dst_reg, dst_stride3); + dst_reg += dst_stride4; + + reg0 = reg2; + reg1 = src0; + reg2 = src2; + reg3 = reg5; + reg4 = src1; + reg5 = src3; + reg6 = reg8; + reg7 = src4; + reg8 = src7; + reg9 = reg11; + reg10 = src5; + reg11 = src8; + src6 = src10; + } + src_tmp0 += 16; + dst += 16; + } +} + +static void common_vt_8t_and_aver_dst_16w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) { + common_vt_8t_and_aver_dst_16w_mult_lsx(src, src_stride, dst, dst_stride, + filter, height, 16); +} + +static void common_vt_8t_and_aver_dst_32w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) { + common_vt_8t_and_aver_dst_16w_mult_lsx(src, src_stride, dst, dst_stride, + filter, height, 32); +} + +static void common_vt_8t_and_aver_dst_64w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + const int8_t *filter, + int32_t height) { + common_vt_8t_and_aver_dst_16w_mult_lsx(src, src_stride, dst, dst_stride, + filter, height, 64); +} + +static void common_vt_2t_and_aver_dst_4x4_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter) { + __m128i src0, src1, src2, src3, src4; + __m128i dst0, dst1, dst2, dst3, out, filt0, src2110, src4332; + __m128i src10_r, src32_r, src21_r, src43_r; + __m128i tmp0, tmp1; + uint8_t *dst_tmp = dst; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + filt0 = __lsx_vldrepl_h(filter, 0); + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + src += src_stride4; + src4 = __lsx_vld(src, 0); + src += src_stride; + + dst0 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_w(dst_tmp, 0); + dst0 = __lsx_vilvl_w(dst1, dst0); + dst1 = __lsx_vilvl_w(dst3, dst2); + dst0 = __lsx_vilvl_d(dst1, dst0); + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src3, src2, src4, src3, + src10_r, src21_r, src32_r, src43_r); + DUP2_ARG2(__lsx_vilvl_d, src21_r, src10_r, src43_r, src32_r, src2110, + src4332); + DUP2_ARG2(__lsx_vdp2_h_bu, src2110, filt0, src4332, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + out = __lsx_vavgr_bu(tmp0, dst0); + __lsx_vstelm_w(out, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(out, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(out, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(out, dst, 0, 3); + dst += dst_stride; +} + +static void common_vt_2t_and_aver_dst_4x8_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter) { + __m128i dst0, dst1, dst2, dst3, dst4; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src87_r; + __m128i src10_r, src32_r, src54_r, src76_r, src21_r, src43_r, src65_r; + __m128i src2110, src4332, src6554, src8776, filt0; + __m128i tmp0, tmp1, tmp2, tmp3; + uint8_t *dst_tmp = dst; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + filt0 = __lsx_vldrepl_h(filter, 0); + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + src += src_stride4; + src4 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src5, src6); + src7 = __lsx_vldx(src, src_stride3); + src += src_stride4; + src8 = __lsx_vld(src, 0); + + dst0 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst0 = __lsx_vilvl_w(dst1, dst0); + dst1 = __lsx_vilvl_w(dst3, dst2); + dst0 = __lsx_vilvl_d(dst1, dst0); + + dst1 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_w(dst_tmp, 0); + dst_tmp += dst_stride; + dst4 = __lsx_vldrepl_w(dst_tmp, 0); + dst1 = __lsx_vilvl_w(dst2, dst1); + dst2 = __lsx_vilvl_w(dst4, dst3); + dst1 = __lsx_vilvl_d(dst2, dst1); + + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src3, src2, src4, src3, + src10_r, src21_r, src32_r, src43_r); + DUP4_ARG2(__lsx_vilvl_b, src5, src4, src6, src5, src7, src6, src8, src7, + src54_r, src65_r, src76_r, src87_r); + DUP4_ARG2(__lsx_vilvl_d, src21_r, src10_r, src43_r, src32_r, src65_r, src54_r, + src87_r, src76_r, src2110, src4332, src6554, src8776); + DUP4_ARG2(__lsx_vdp2_h_bu, src2110, filt0, src4332, filt0, src6554, filt0, + src8776, filt0, tmp0, tmp1, tmp2, tmp3); + DUP2_ARG3(__lsx_vssrarni_bu_h, tmp1, tmp0, FILTER_BITS, tmp3, tmp2, + FILTER_BITS, tmp0, tmp2); + DUP2_ARG2(__lsx_vavgr_bu, tmp0, dst0, tmp2, dst1, tmp0, tmp2); + __lsx_vstelm_w(tmp0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(tmp0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(tmp0, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(tmp0, dst, 0, 3); + dst += dst_stride; + + __lsx_vstelm_w(tmp2, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(tmp2, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(tmp2, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(tmp2, dst, 0, 3); +} + +static void common_vt_2t_and_aver_dst_4w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, int8_t *filter, + int32_t height) { + if (height == 4) { + common_vt_2t_and_aver_dst_4x4_lsx(src, src_stride, dst, dst_stride, filter); + } else if (height == 8) { + common_vt_2t_and_aver_dst_4x8_lsx(src, src_stride, dst, dst_stride, filter); + } +} + +static void common_vt_2t_and_aver_dst_8x4_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter) { + __m128i src0, src1, src2, src3, src4; + __m128i dst0, dst1, dst2, dst3, vec0, vec1, vec2, vec3, filt0; + __m128i tmp0, tmp1, tmp2, tmp3; + uint8_t *dst_tmp = dst; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + filt0 = __lsx_vldrepl_h(filter, 0); + src0 = __lsx_vld(src, 0); + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src1, src2, src3, src4); + dst0 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1); + DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, vec0, vec1); + DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, vec2, vec3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0, + tmp0, tmp1, tmp2, tmp3); + DUP2_ARG3(__lsx_vssrarni_bu_h, tmp1, tmp0, FILTER_BITS, tmp3, tmp2, + FILTER_BITS, tmp0, tmp2); + DUP2_ARG2(__lsx_vavgr_bu, tmp0, dst0, tmp2, dst1, tmp0, tmp2); + __lsx_vstelm_d(tmp0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(tmp0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(tmp2, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(tmp2, dst, 0, 1); +} + +static void common_vt_2t_and_aver_dst_8x8mult_lsx( + const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = (height >> 3); + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8; + __m128i dst0, dst1, dst2, dst3, dst4, dst5; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, filt0; + __m128i tmp0, tmp1, tmp2, tmp3; + uint8_t *dst_tmp = dst; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + filt0 = __lsx_vldrepl_h(filter, 0); + src0 = __lsx_vld(src, 0); + src += src_stride; + + for (; loop_cnt--;) { + src1 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src3); + src4 = __lsx_vldx(src, src_stride3); + src += src_stride4; + src5 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src6, src7); + src8 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + dst0 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst1 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst2 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_d, dst1, dst0, dst3, dst2, dst0, dst1); + + dst2 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst3 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst4 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + dst5 = __lsx_vldrepl_d(dst_tmp, 0); + dst_tmp += dst_stride; + DUP2_ARG2(__lsx_vilvl_d, dst3, dst2, dst5, dst4, dst2, dst3); + + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src3, src2, src4, src3, + vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vilvl_b, src5, src4, src6, src5, src7, src6, src8, src7, + vec4, vec5, vec6, vec7); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, + filt0, tmp0, tmp1, tmp2, tmp3); + + DUP2_ARG3(__lsx_vssrarni_bu_h, tmp1, tmp0, FILTER_BITS, tmp3, tmp2, + FILTER_BITS, tmp0, tmp2); + DUP2_ARG2(__lsx_vavgr_bu, tmp0, dst0, tmp2, dst1, tmp0, tmp2); + __lsx_vstelm_d(tmp0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(tmp0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(tmp2, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(tmp2, dst, 0, 1); + dst += dst_stride; + + DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7, + filt0, tmp0, tmp1, tmp2, tmp3); + DUP2_ARG3(__lsx_vssrarni_bu_h, tmp1, tmp0, FILTER_BITS, tmp3, tmp2, + FILTER_BITS, tmp0, tmp2); + DUP2_ARG2(__lsx_vavgr_bu, tmp0, dst2, tmp2, dst3, tmp0, tmp2); + __lsx_vstelm_d(tmp0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(tmp0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(tmp2, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(tmp2, dst, 0, 1); + dst += dst_stride; + + src0 = src8; + } +} + +static void common_vt_2t_and_aver_dst_8w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, int8_t *filter, + int32_t height) { + if (height == 4) { + common_vt_2t_and_aver_dst_8x4_lsx(src, src_stride, dst, dst_stride, filter); + } else { + common_vt_2t_and_aver_dst_8x8mult_lsx(src, src_stride, dst, dst_stride, + filter, height); + } +} + +static void common_vt_2t_and_aver_dst_16w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = (height >> 2); + __m128i src0, src1, src2, src3, src4, dst0, dst1, dst2, dst3, filt0; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i tmp0, tmp1; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + + filt0 = __lsx_vldrepl_h(filter, 0); + src0 = __lsx_vld(src, 0); + src += src_stride; + + for (; loop_cnt--;) { + src1 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src3); + src4 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + dst0 = __lsx_vld(dst, 0); + DUP2_ARG2(__lsx_vldx, dst, dst_stride, dst, dst_stride2, dst1, dst2); + dst3 = __lsx_vldx(dst, dst_stride3); + + DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, vec0, vec2); + DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, vec1, vec3); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst0); + __lsx_vst(tmp0, dst, 0); + dst += dst_stride; + + DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, vec4, vec6); + DUP2_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, vec5, vec7); + DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst1); + __lsx_vst(tmp0, dst, 0); + dst += dst_stride; + + DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst2); + __lsx_vst(tmp0, dst, 0); + dst += dst_stride; + + DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst3); + __lsx_vst(tmp0, dst, 0); + dst += dst_stride; + + src0 = src4; + } +} + +static void common_vt_2t_and_aver_dst_32w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = (height >> 2); + uint8_t *src_tmp1; + uint8_t *dst_tmp1; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, filt0; + __m128i tmp0, tmp1; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + + filt0 = __lsx_vldrepl_h(filter, 0); + DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src5); + src += src_stride; + + for (; loop_cnt--;) { + src1 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src3); + src4 = __lsx_vldx(src, src_stride3); + + dst0 = __lsx_vld(dst, 0); + DUP2_ARG2(__lsx_vldx, dst, dst_stride, dst, dst_stride2, dst1, dst2); + dst3 = __lsx_vldx(dst, dst_stride3); + + DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, vec0, vec2); + DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, vec1, vec3); + + src_tmp1 = src + 16; + src6 = __lsx_vld(src_tmp1, 0); + DUP2_ARG2(__lsx_vldx, src_tmp1, src_stride, src_tmp1, src_stride2, src7, + src8); + src9 = __lsx_vldx(src_tmp1, src_stride3); + + dst_tmp1 = dst + 16; + dst4 = __lsx_vld(dst_tmp1, 0); + DUP2_ARG2(__lsx_vldx, dst_tmp1, dst_stride, dst_tmp1, dst_stride2, dst5, + dst6); + dst7 = __lsx_vldx(dst_tmp1, dst_stride3); + src += src_stride4; + + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst0); + __lsx_vst(tmp0, dst, 0); + + DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst1); + __lsx_vstx(tmp0, dst, dst_stride); + + DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, vec4, vec6); + DUP2_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, vec5, vec7); + DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst2); + __lsx_vstx(tmp0, dst, dst_stride2); + + DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst3); + __lsx_vstx(tmp0, dst, dst_stride3); + + DUP2_ARG2(__lsx_vilvl_b, src6, src5, src7, src6, vec0, vec2); + DUP2_ARG2(__lsx_vilvh_b, src6, src5, src7, src6, vec1, vec3); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst4); + __lsx_vst(tmp0, dst, 16); + + DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst5); + dst += dst_stride; + __lsx_vst(tmp0, dst, 16); + + DUP2_ARG2(__lsx_vilvl_b, src8, src7, src9, src8, vec4, vec6); + DUP2_ARG2(__lsx_vilvh_b, src8, src7, src9, src8, vec5, vec7); + DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst6); + dst += dst_stride; + __lsx_vst(tmp0, dst, 16); + + DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst7); + dst += dst_stride; + __lsx_vst(tmp0, dst, 16); + dst += dst_stride; + + src0 = src4; + src5 = src9; + } +} + +static void common_vt_2t_and_aver_dst_64w_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = (height >> 1); + int32_t src_stride2 = src_stride << 1; + int32_t dst_stride2 = dst_stride << 1; + uint8_t *src_tmp1; + uint8_t *dst_tmp1; + __m128i src0, src1, src2, src3, src4, src5; + __m128i src6, src7, src8, src9, src10, src11, filt0; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i tmp0, tmp1; + + filt0 = __lsx_vldrepl_h(filter, 0); + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src3, src6, + src9); + src += src_stride; + + for (; loop_cnt--;) { + src2 = __lsx_vldx(src, src_stride); + dst1 = __lsx_vldx(dst, dst_stride); + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src1, src4, src7, + src10); + DUP4_ARG2(__lsx_vld, dst, 0, dst, 16, dst, 32, dst, 48, dst0, dst2, dst4, + dst6); + src_tmp1 = (uint8_t *)src + 16; + src5 = __lsx_vldx(src_tmp1, src_stride); + src_tmp1 = src_tmp1 + 16; + src8 = __lsx_vldx(src_tmp1, src_stride); + src_tmp1 = src_tmp1 + 16; + src11 = __lsx_vldx(src_tmp1, src_stride); + + dst_tmp1 = dst + 16; + dst3 = __lsx_vldx(dst_tmp1, dst_stride); + dst_tmp1 = dst + 32; + dst5 = __lsx_vldx(dst_tmp1, dst_stride); + dst_tmp1 = dst + 48; + dst7 = __lsx_vldx(dst_tmp1, dst_stride); + src += src_stride2; + + DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, vec0, vec2); + DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, vec1, vec3); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst0); + __lsx_vst(tmp0, dst, 0); + + DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst1); + __lsx_vstx(tmp0, dst, dst_stride); + + DUP2_ARG2(__lsx_vilvl_b, src4, src3, src5, src4, vec4, vec6); + DUP2_ARG2(__lsx_vilvh_b, src4, src3, src5, src4, vec5, vec7); + DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst2); + __lsx_vst(tmp0, dst, 16); + + dst_tmp1 = dst + 16; + DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst3); + __lsx_vstx(tmp0, dst_tmp1, dst_stride); + + DUP2_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, vec0, vec2); + DUP2_ARG2(__lsx_vilvh_b, src7, src6, src8, src7, vec1, vec3); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst4); + __lsx_vst(tmp0, dst, 32); + + dst_tmp1 = dst_tmp1 + 16; + DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst5); + __lsx_vstx(tmp0, dst_tmp1, dst_stride); + + DUP2_ARG2(__lsx_vilvl_b, src10, src9, src11, src10, vec4, vec6); + DUP2_ARG2(__lsx_vilvh_b, src10, src9, src11, src10, vec5, vec7); + DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst6); + __lsx_vst(tmp0, dst, 48); + + dst_tmp1 = dst_tmp1 + 16; + DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp0, tmp1); + tmp0 = __lsx_vssrarni_bu_h(tmp1, tmp0, FILTER_BITS); + tmp0 = __lsx_vavgr_bu(tmp0, dst7); + __lsx_vstx(tmp0, dst_tmp1, dst_stride); + dst += dst_stride2; + + src0 = src2; + src3 = src5; + src6 = src8; + src9 = src11; + } +} + +void vpx_convolve8_avg_vert_lsx(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *filter, int x0_q4, + int x_step_q4, int y0_q4, int y_step_q4, int w, + int h) { + const int16_t *const filter_y = filter[y0_q4]; + int8_t cnt, filt_ver[8]; + + assert(y_step_q4 == 16); + assert(((const int32_t *)filter_y)[1] != 0x800000); + + for (cnt = 0; cnt < 8; ++cnt) { + filt_ver[cnt] = filter_y[cnt]; + } + + if (vpx_get_filter_taps(filter_y) == 2) { + switch (w) { + case 4: + common_vt_2t_and_aver_dst_4w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_ver[3], h); + break; + case 8: + common_vt_2t_and_aver_dst_8w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_ver[3], h); + break; + case 16: + common_vt_2t_and_aver_dst_16w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_ver[3], h); + break; + case 32: + common_vt_2t_and_aver_dst_32w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_ver[3], h); + break; + case 64: + common_vt_2t_and_aver_dst_64w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_ver[3], h); + break; + default: + vpx_convolve8_avg_vert_c(src, src_stride, dst, dst_stride, filter, + x0_q4, x_step_q4, y0_q4, y_step_q4, w, h); + break; + } + } else { + switch (w) { + case 4: + common_vt_8t_and_aver_dst_4w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_ver, h); + break; + case 8: + common_vt_8t_and_aver_dst_8w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_ver, h); + break; + case 16: + common_vt_8t_and_aver_dst_16w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_ver, h); + + break; + case 32: + common_vt_8t_and_aver_dst_32w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_ver, h); + break; + case 64: + common_vt_8t_and_aver_dst_64w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_ver, h); + break; + default: + vpx_convolve8_avg_vert_c(src, src_stride, dst, dst_stride, filter, + x0_q4, x_step_q4, y0_q4, y_step_q4, w, h); + break; + } + } +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_horiz_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_horiz_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_horiz_lsx.c @@ -0,0 +1,841 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include "./vpx_dsp_rtcd.h" +#include "vpx_dsp/loongarch/vpx_convolve_lsx.h" + +static const uint8_t mc_filt_mask_arr[16 * 3] = { + /* 8 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, + /* 4 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20, + /* 4 width cases */ + 8, 9, 9, 10, 10, 11, 11, 12, 24, 25, 25, 26, 26, 27, 27, 28 +}; + +static void common_hz_8t_4x4_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter) { + __m128i src0, src1, src2, src3; + __m128i filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3; + __m128i out, out0, out1; + + mask0 = __lsx_vld(mc_filt_mask_arr, 16); + src -= 3; + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filter0, filter1, filter2, filter3, out0, out1); + out = __lsx_vssrarni_b_h(out1, out0, 7); + out = __lsx_vxori_b(out, 128); + __lsx_vstelm_w(out, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(out, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(out, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(out, dst, 0, 3); +} + +static void common_hz_8t_4x8_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter) { + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride + src_stride2; + int32_t src_stride4 = src_stride2 << 1; + __m128i src0, src1, src2, src3; + __m128i filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3; + __m128i out0, out1, out2, out3; + uint8_t *_src = (uint8_t *)src - 3; + + mask0 = __lsx_vld(mc_filt_mask_arr, 16); + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + src0 = __lsx_vld(_src, 0); + DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src1, src2); + src3 = __lsx_vldx(_src, src_stride3); + _src += src_stride4; + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filter0, filter1, filter2, filter3, out0, out1); + src0 = __lsx_vld(_src, 0); + DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src1, src2); + src3 = __lsx_vldx(_src, src_stride3); + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filter0, filter1, filter2, filter3, out2, out3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + + __lsx_vstelm_w(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 3); + dst += dst_stride; + __lsx_vstelm_w(out1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(out1, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(out1, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(out1, dst, 0, 3); +} + +static void common_hz_8t_4w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) { + if (height == 4) { + common_hz_8t_4x4_lsx(src, src_stride, dst, dst_stride, filter); + } else if (height == 8) { + common_hz_8t_4x8_lsx(src, src_stride, dst, dst_stride, filter); + } +} + +static void common_hz_8t_8x4_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter) { + __m128i src0, src1, src2, src3; + __m128i filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3; + __m128i out0, out1, out2, out3; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + src -= 3; + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, mask3, + filter0, filter1, filter2, filter3, out0, out1, + out2, out3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 1); +} + +static void common_hz_8t_8x8mult_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) { + uint32_t loop_cnt = height >> 2; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride + src_stride2; + int32_t src_stride4 = src_stride2 << 1; + __m128i src0, src1, src2, src3; + __m128i filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3; + __m128i out0, out1, out2, out3; + uint8_t *_src = (uint8_t *)src - 3; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + for (; loop_cnt--;) { + src0 = __lsx_vld(_src, 0); + DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src1, src2); + src3 = __lsx_vldx(_src, src_stride3); + _src += src_stride4; + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, + mask3, filter0, filter1, filter2, filter3, out0, + out1, out2, out3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 1); + dst += dst_stride; + } +} + +static void common_hz_8t_8w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) { + if (height == 4) { + common_hz_8t_8x4_lsx(src, src_stride, dst, dst_stride, filter); + } else { + common_hz_8t_8x8mult_lsx(src, src_stride, dst, dst_stride, filter, height); + } +} + +static void common_hz_8t_16w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) { + uint32_t loop_cnt = height >> 1; + int32_t stride = src_stride << 1; + __m128i src0, src1, src2, src3; + __m128i filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3; + __m128i out0, out1, out2, out3; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + src -= 3; + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + for (; loop_cnt--;) { + const uint8_t *_src = src + src_stride; + DUP2_ARG2(__lsx_vld, src, 0, _src, 0, src0, src2); + DUP2_ARG2(__lsx_vld, src, 8, _src, 8, src1, src3); + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, + mask3, filter0, filter1, filter2, filter3, out0, + out1, out2, out3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vst(out0, dst, 0); + dst += dst_stride; + __lsx_vst(out1, dst, 0); + dst += dst_stride; + src += stride; + } +} + +static void common_hz_8t_32w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) { + uint32_t loop_cnt = height >> 1; + __m128i src0, src1, src2, src3; + __m128i filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3; + __m128i out0, out1, out2, out3; + __m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 }; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + src -= 3; + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + for (; loop_cnt--;) { + DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src2); + src3 = __lsx_vld(src, 24); + src1 = __lsx_vshuf_b(src2, src0, shuff); + src += src_stride; + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, + mask3, filter0, filter1, filter2, filter3, out0, + out1, out2, out3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vst(out0, dst, 0); + __lsx_vst(out1, dst, 16); + + DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src2); + src3 = __lsx_vld(src, 24); + src1 = __lsx_vshuf_b(src2, src0, shuff); + src += src_stride; + + dst += dst_stride; + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, + mask3, filter0, filter1, filter2, filter3, out0, + out1, out2, out3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vst(out0, dst, 0); + __lsx_vst(out1, dst, 16); + dst += dst_stride; + } +} + +static void common_hz_8t_64w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + const int8_t *filter, int32_t height) { + int32_t loop_cnt = height; + __m128i src0, src1, src2, src3; + __m128i filter0, filter1, filter2, filter3; + __m128i mask0, mask1, mask2, mask3; + __m128i out0, out1, out2, out3; + __m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 }; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + src -= 3; + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + for (; loop_cnt--;) { + DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src2); + src3 = __lsx_vld(src, 24); + src1 = __lsx_vshuf_b(src2, src0, shuff); + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, + mask3, filter0, filter1, filter2, filter3, out0, + out1, out2, out3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vst(out0, dst, 0); + __lsx_vst(out1, dst, 16); + + DUP2_ARG2(__lsx_vld, src, 32, src, 48, src0, src2); + src3 = __lsx_vld(src, 56); + src1 = __lsx_vshuf_b(src2, src0, shuff); + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2, + mask3, filter0, filter1, filter2, filter3, out0, + out1, out2, out3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vst(out0, dst, 32); + __lsx_vst(out1, dst, 48); + src += src_stride; + dst += dst_stride; + } +} + +static void common_hz_2t_4x4_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter) { + __m128i src0, src1, src2, src3, mask; + __m128i filt0, vec0, vec1, res0, res1; + __m128i vec2, vec3; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride + src_stride2; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride + dst_stride2; + + mask = __lsx_vld(mc_filt_mask_arr, 16); + /* rearranging filter */ + filt0 = __lsx_vldrepl_h(filter, 0); + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + DUP2_ARG3(__lsx_vshuf_b, src1, src0, mask, src3, src2, mask, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, vec3); + DUP2_ARG2(__lsx_vsrari_h, vec2, FILTER_BITS, vec3, FILTER_BITS, vec2, vec3); + DUP2_ARG2(__lsx_vpickev_b, vec2, vec2, vec3, vec3, res0, res1); + + __lsx_vstelm_w(res0, dst, 0, 0); + __lsx_vstelm_w(res0, dst + dst_stride, 0, 1); + __lsx_vstelm_w(res1, dst + dst_stride2, 0, 0); + __lsx_vstelm_w(res1, dst + dst_stride3, 0, 1); +} + +static void common_hz_2t_4x8_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter) { + __m128i vec0, vec1, vec2, vec3, filt0; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, mask; + __m128i res0, res1, res2, res3; + __m128i vec4, vec5, vec6, vec7; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride + src_stride2; + int32_t src_stride4 = src_stride2 << 1; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride + dst_stride2; + + uint8_t *src_tmp1 = src + src_stride4; + + mask = __lsx_vld(mc_filt_mask_arr, 16); + + /* rearranging filter */ + filt0 = __lsx_vldrepl_h(filter, 0); + + src0 = __lsx_vld(src, 0); + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src1, src2, src3, src4); + DUP2_ARG2(__lsx_vldx, src_tmp1, src_stride, src_tmp1, src_stride2, src5, + src6); + src7 = __lsx_vldx(src_tmp1, src_stride3); + + DUP4_ARG3(__lsx_vshuf_b, src1, src0, mask, src3, src2, mask, src5, src4, mask, + src7, src6, mask, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0, + vec4, vec5, vec6, vec7); + DUP4_ARG2(__lsx_vsrari_h, vec4, FILTER_BITS, vec5, FILTER_BITS, vec6, + FILTER_BITS, vec7, FILTER_BITS, vec4, vec5, vec6, vec7); + DUP4_ARG2(__lsx_vpickev_b, vec4, vec4, vec5, vec5, vec6, vec6, vec7, vec7, + res0, res1, res2, res3); + __lsx_vstelm_w(res0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(res0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(res1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(res1, dst, 0, 1); + dst += dst_stride; + + __lsx_vstelm_w(res2, dst, 0, 0); + __lsx_vstelm_w(res2, dst + dst_stride, 0, 1); + __lsx_vstelm_w(res3, dst + dst_stride2, 0, 0); + __lsx_vstelm_w(res3, dst + dst_stride3, 0, 1); +} + +static void common_hz_2t_4w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + if (height == 4) { + common_hz_2t_4x4_lsx(src, src_stride, dst, dst_stride, filter); + } else if (height == 8) { + common_hz_2t_4x8_lsx(src, src_stride, dst, dst_stride, filter); + } +} + +static void common_hz_2t_8x4_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter) { + __m128i filt0, mask; + __m128i src0, src1, src2, src3; + __m128i vec0, vec1, vec2, vec3; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + + mask = __lsx_vld(mc_filt_mask_arr, 0); + + /* rearranging filter */ + filt0 = __lsx_vldrepl_h(filter, 0); + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, src2, src2, mask, + src3, src3, mask, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0, + vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vsrari_h, vec0, FILTER_BITS, vec1, FILTER_BITS, vec2, + FILTER_BITS, vec3, FILTER_BITS, vec0, vec1, vec2, vec3); + DUP2_ARG2(__lsx_vpickev_b, vec1, vec0, vec3, vec2, src0, src1); + + __lsx_vstelm_d(src0, dst, 0, 0); + __lsx_vstelm_d(src0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(src1, dst + dst_stride2, 0, 0); + __lsx_vstelm_d(src1, dst + dst_stride3, 0, 1); +} + +static void common_hz_2t_8x8mult_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + __m128i filt0, mask; + __m128i src0, src1, src2, src3, out0, out1; + __m128i vec0, vec1, vec2, vec3; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + int32_t dst_stride4 = dst_stride2 << 1; + + mask = __lsx_vld(mc_filt_mask_arr, 0); + + /* rearranging filter */ + filt0 = __lsx_vldrepl_h(filter, 0); + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, src2, src2, mask, + src3, src3, mask, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0, + vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vsrari_h, vec0, FILTER_BITS, vec1, FILTER_BITS, vec2, + FILTER_BITS, vec3, FILTER_BITS, vec0, vec1, vec2, vec3); + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + DUP2_ARG2(__lsx_vpickev_b, vec1, vec0, vec3, vec2, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 1); + dst += dst_stride; + + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, src2, src2, mask, + src3, src3, mask, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0, + vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vsrari_h, vec0, FILTER_BITS, vec1, FILTER_BITS, vec2, + FILTER_BITS, vec3, FILTER_BITS, vec0, vec1, vec2, vec3); + DUP2_ARG2(__lsx_vpickev_b, vec1, vec0, vec3, vec2, out0, out1); + + __lsx_vstelm_d(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 1); + dst += dst_stride; + + if (height == 16) { + uint8_t *dst_tmp1 = dst + dst_stride4; + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, src2, src2, + mask, src3, src3, mask, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, + filt0, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vsrari_h, vec0, FILTER_BITS, vec1, FILTER_BITS, vec2, + FILTER_BITS, vec3, FILTER_BITS, vec0, vec1, vec2, vec3); + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + DUP2_ARG2(__lsx_vpickev_b, vec1, vec0, vec3, vec2, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + __lsx_vstelm_d(out0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(out1, dst + dst_stride2, 0, 0); + __lsx_vstelm_d(out1, dst + dst_stride3, 0, 1); + + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, src2, src2, + mask, src3, src3, mask, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, + filt0, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vsrari_h, vec0, FILTER_BITS, vec1, FILTER_BITS, vec2, + FILTER_BITS, vec3, FILTER_BITS, vec0, vec1, vec2, vec3); + DUP2_ARG2(__lsx_vpickev_b, vec1, vec0, vec3, vec2, out0, out1); + + __lsx_vstelm_d(out0, dst_tmp1, 0, 0); + __lsx_vstelm_d(out0, dst_tmp1 + dst_stride, 0, 1); + __lsx_vstelm_d(out1, dst_tmp1 + dst_stride2, 0, 0); + __lsx_vstelm_d(out1, dst_tmp1 + dst_stride3, 0, 1); + } +} + +static void common_hz_2t_8w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + if (height == 4) { + common_hz_2t_8x4_lsx(src, src_stride, dst, dst_stride, filter); + } else { + common_hz_2t_8x8mult_lsx(src, src_stride, dst, dst_stride, filter, height); + } +} + +static void common_hz_2t_16w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = (height >> 2) - 1; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, mask; + __m128i filt0, vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i out0, out1, out2, out3, out4, out5, out6, out7, tmp; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + uint8_t *src_tmp1 = src + 8; + mask = __lsx_vld(mc_filt_mask_arr, 0); + filt0 = __lsx_vldrepl_h(filter, 0); + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src4); + src6 = __lsx_vldx(src, src_stride3); + src1 = __lsx_vld(src_tmp1, 0); + DUP2_ARG2(__lsx_vldx, src_tmp1, src_stride, src_tmp1, src_stride2, src3, + src5); + src7 = __lsx_vldx(src_tmp1, src_stride3); + src += src_stride4; + + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, src2, src2, mask, + src3, src3, mask, vec0, vec1, vec2, vec3); + DUP4_ARG3(__lsx_vshuf_b, src4, src4, mask, src5, src5, mask, src6, src6, mask, + src7, src7, mask, vec4, vec5, vec6, vec7); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0, + out0, out1, out2, out3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7, filt0, + out4, out5, out6, out7); + DUP4_ARG2(__lsx_vsrari_h, out0, FILTER_BITS, out1, FILTER_BITS, out2, + FILTER_BITS, out3, FILTER_BITS, out0, out1, out2, out3); + DUP4_ARG2(__lsx_vsrari_h, out4, FILTER_BITS, out5, FILTER_BITS, out6, + FILTER_BITS, out7, FILTER_BITS, out4, out5, out6, out7); + + tmp = __lsx_vpickev_b(out1, out0); + __lsx_vst(tmp, dst, 0); + dst += dst_stride; + tmp = __lsx_vpickev_b(out3, out2); + __lsx_vst(tmp, dst, 0); + dst += dst_stride; + tmp = __lsx_vpickev_b(out5, out4); + __lsx_vst(tmp, dst, 0); + dst += dst_stride; + tmp = __lsx_vpickev_b(out7, out6); + __lsx_vst(tmp, dst, 0); + dst += dst_stride; + + for (; loop_cnt--;) { + src_tmp1 += src_stride4; + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src4); + src6 = __lsx_vldx(src, src_stride3); + + src1 = __lsx_vld(src_tmp1, 0); + DUP2_ARG2(__lsx_vldx, src_tmp1, src_stride, src_tmp1, src_stride2, src3, + src5); + src7 = __lsx_vldx(src_tmp1, src_stride3); + src += src_stride4; + + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, src2, src2, + mask, src3, src3, mask, vec0, vec1, vec2, vec3); + DUP4_ARG3(__lsx_vshuf_b, src4, src4, mask, src5, src5, mask, src6, src6, + mask, src7, src7, mask, vec4, vec5, vec6, vec7); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, + filt0, out0, out1, out2, out3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7, + filt0, out4, out5, out6, out7); + DUP4_ARG2(__lsx_vsrari_h, out0, FILTER_BITS, out1, FILTER_BITS, out2, + FILTER_BITS, out3, FILTER_BITS, out0, out1, out2, out3); + DUP4_ARG2(__lsx_vsrari_h, out4, FILTER_BITS, out5, FILTER_BITS, out6, + FILTER_BITS, out7, FILTER_BITS, out4, out5, out6, out7); + + tmp = __lsx_vpickev_b(out1, out0); + __lsx_vst(tmp, dst, 0); + dst += dst_stride; + tmp = __lsx_vpickev_b(out3, out2); + __lsx_vst(tmp, dst, 0); + dst += dst_stride; + tmp = __lsx_vpickev_b(out5, out4); + __lsx_vst(tmp, dst, 0); + dst += dst_stride; + tmp = __lsx_vpickev_b(out7, out6); + __lsx_vst(tmp, dst, 0); + dst += dst_stride; + } +} + +static void common_hz_2t_32w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = (height >> 1); + __m128i src0, src1, src2, src3, src4, src5, src6, src7, mask; + __m128i filt0, vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i out0, out1, out2, out3, out4, out5, out6, out7, tmp; + __m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 }; + + mask = __lsx_vld(mc_filt_mask_arr, 0); + /* rearranging filter */ + filt0 = __lsx_vldrepl_h(filter, 0); + + for (; loop_cnt--;) { + DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src2); + src3 = __lsx_vld(src, 24); + src1 = __lsx_vshuf_b(src2, src0, shuff); + src += src_stride; + DUP2_ARG2(__lsx_vld, src, 0, src, 16, src4, src6); + src7 = __lsx_vld(src, 24); + src5 = __lsx_vshuf_b(src6, src4, shuff); + src += src_stride; + + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, src2, src2, + mask, src3, src3, mask, vec0, vec1, vec2, vec3); + DUP4_ARG3(__lsx_vshuf_b, src4, src4, mask, src5, src5, mask, src6, src6, + mask, src7, src7, mask, vec4, vec5, vec6, vec7); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, + filt0, out0, out1, out2, out3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7, + filt0, out4, out5, out6, out7); + DUP4_ARG2(__lsx_vsrari_h, out0, FILTER_BITS, out1, FILTER_BITS, out2, + FILTER_BITS, out3, FILTER_BITS, out0, out1, out2, out3); + DUP4_ARG2(__lsx_vsrari_h, out4, FILTER_BITS, out5, FILTER_BITS, out6, + FILTER_BITS, out7, FILTER_BITS, out4, out5, out6, out7); + + tmp = __lsx_vpickev_b(out1, out0); + __lsx_vst(tmp, dst, 0); + tmp = __lsx_vpickev_b(out3, out2); + __lsx_vst(tmp, dst, 16); + dst += dst_stride; + + tmp = __lsx_vpickev_b(out5, out4); + __lsx_vst(tmp, dst, 0); + tmp = __lsx_vpickev_b(out7, out6); + __lsx_vst(tmp, dst, 16); + dst += dst_stride; + } +} + +static void common_hz_2t_64w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = height; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, mask; + __m128i filt0, vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7; + __m128i out0, out1, out2, out3, out4, out5, out6, out7, tmp; + __m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 }; + + mask = __lsx_vld(mc_filt_mask_arr, 0); + + /* rearranging filter */ + filt0 = __lsx_vldrepl_h(filter, 0); + + for (; loop_cnt--;) { + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src2, src4, + src6); + src7 = __lsx_vld(src, 56); + DUP2_ARG3(__lsx_vshuf_b, src2, src0, shuff, src4, src2, shuff, src1, src3); + src5 = __lsx_vshuf_b(src6, src4, shuff); + src += src_stride; + + DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask, src1, src1, mask, src2, src2, + mask, src3, src3, mask, vec0, vec1, vec2, vec3); + DUP4_ARG3(__lsx_vshuf_b, src4, src4, mask, src5, src5, mask, src6, src6, + mask, src7, src7, mask, vec4, vec5, vec6, vec7); + + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, + filt0, out0, out1, out2, out3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7, + filt0, out4, out5, out6, out7); + DUP4_ARG2(__lsx_vsrari_h, out0, FILTER_BITS, out1, FILTER_BITS, out2, + FILTER_BITS, out3, FILTER_BITS, out0, out1, out2, out3); + DUP4_ARG2(__lsx_vsrari_h, out4, FILTER_BITS, out5, FILTER_BITS, out6, + FILTER_BITS, out7, FILTER_BITS, out4, out5, out6, out7); + + tmp = __lsx_vpickev_b(out1, out0); + __lsx_vst(tmp, dst, 0); + tmp = __lsx_vpickev_b(out3, out2); + __lsx_vst(tmp, dst, 16); + tmp = __lsx_vpickev_b(out5, out4); + __lsx_vst(tmp, dst, 32); + tmp = __lsx_vpickev_b(out7, out6); + __lsx_vst(tmp, dst, 48); + dst += dst_stride; + } +} + +void vpx_convolve8_horiz_lsx(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *filter, int x0_q4, + int x_step_q4, int y0_q4, int y_step_q4, int w, + int h) { + const int16_t *const filter_x = filter[x0_q4]; + int8_t cnt, filt_hor[8]; + + assert(x_step_q4 == 16); + assert(((const int32_t *)filter_x)[1] != 0x800000); + + for (cnt = 0; cnt < 8; ++cnt) { + filt_hor[cnt] = filter_x[cnt]; + } + if (vpx_get_filter_taps(filter_x) == 2) { + switch (w) { + case 4: + common_hz_2t_4w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + &filt_hor[3], h); + break; + case 8: + common_hz_2t_8w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + &filt_hor[3], h); + break; + case 16: + common_hz_2t_16w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + &filt_hor[3], h); + break; + case 32: + common_hz_2t_32w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + &filt_hor[3], h); + break; + case 64: + common_hz_2t_64w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + &filt_hor[3], h); + break; + default: + vpx_convolve8_horiz_c(src, src_stride, dst, dst_stride, filter, x0_q4, + x_step_q4, y0_q4, y_step_q4, w, h); + break; + } + } else { + switch (w) { + case 4: + common_hz_8t_4w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + filt_hor, h); + break; + case 8: + common_hz_8t_8w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + filt_hor, h); + break; + + case 16: + common_hz_8t_16w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + filt_hor, h); + break; + + case 32: + common_hz_8t_32w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + filt_hor, h); + break; + + case 64: + common_hz_8t_64w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + filt_hor, h); + break; + default: + vpx_convolve8_horiz_c(src, src_stride, dst, dst_stride, filter, x0_q4, + x_step_q4, y0_q4, y_step_q4, w, h); + break; + } + } +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_lsx.c @@ -0,0 +1,707 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include "./vpx_dsp_rtcd.h" +#include "vpx_dsp/loongarch/vpx_convolve_lsx.h" + +static const uint8_t mc_filt_mask_arr[16 * 3] = { + /* 8 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, + /* 4 width cases */ + 0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20, + /* 4 width cases */ + 8, 9, 9, 10, 10, 11, 11, 12, 24, 25, 25, 26, 26, 27, 27, 28 +}; + +static void common_hv_8ht_8vt_4w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, + int32_t height) { + uint32_t loop_cnt = (height >> 2); + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i filt_hz0, filt_hz1, filt_hz2, filt_hz3; + __m128i filt_vt0, filt_vt1, filt_vt2, filt_vt3; + __m128i mask0, mask1, mask2, mask3; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + __m128i out0, out1; + __m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 }; + + mask0 = __lsx_vld(mc_filt_mask_arr, 16); + src -= (3 + 3 * src_stride); + DUP4_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_horiz, 2, filter_horiz, 4, + filter_horiz, 6, filt_hz0, filt_hz1, filt_hz2, filt_hz3); + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + src += src_stride; + src4 = __lsx_vld(src, 0); + src += src_stride; + src5 = __lsx_vld(src, 0); + src += src_stride; + src6 = __lsx_vld(src, 0); + src += src_stride; + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + + tmp0 = HORIZ_8TAP_FILT(src0, src1, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + tmp2 = HORIZ_8TAP_FILT(src2, src3, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + tmp4 = HORIZ_8TAP_FILT(src4, src5, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + tmp5 = HORIZ_8TAP_FILT(src5, src6, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + DUP2_ARG3(__lsx_vshuf_b, tmp2, tmp0, shuff, tmp4, tmp2, shuff, tmp1, tmp3); + DUP4_ARG2(__lsx_vldrepl_h, filter_vert, 0, filter_vert, 2, filter_vert, 4, + filter_vert, 6, filt_vt0, filt_vt1, filt_vt2, filt_vt3); + DUP2_ARG2(__lsx_vpackev_b, tmp1, tmp0, tmp3, tmp2, tmp0, tmp1); + tmp2 = __lsx_vpackev_b(tmp5, tmp4); + + for (; loop_cnt--;) { + LSX_LD_4(src, src_stride, src7, src8, src9, src10); + src += src_stride; + DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7, + src8, src9, src10); + tmp3 = HORIZ_8TAP_FILT(src7, src8, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + tmp4 = __lsx_vshuf_b(tmp3, tmp5, shuff); + tmp4 = __lsx_vpackev_b(tmp3, tmp4); + out0 = FILT_8TAP_DPADD_S_H(tmp0, tmp1, tmp2, tmp4, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + src1 = HORIZ_8TAP_FILT(src9, src10, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src0 = __lsx_vshuf_b(src1, tmp3, shuff); + src0 = __lsx_vpackev_b(src1, src0); + out1 = FILT_8TAP_DPADD_S_H(tmp1, tmp2, tmp4, src0, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + out0 = __lsx_vssrarni_b_h(out1, out0, 7); + out0 = __lsx_vxori_b(out0, 128); + __lsx_vstelm_w(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 3); + dst += dst_stride; + + tmp5 = src1; + tmp0 = tmp2; + tmp1 = tmp4; + tmp2 = src0; + } +} + +static void common_hv_8ht_8vt_8w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, + int32_t height) { + uint32_t loop_cnt = (height >> 2); + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i filt_hz0, filt_hz1, filt_hz2, filt_hz3; + __m128i filt_vt0, filt_vt1, filt_vt2, filt_vt3; + __m128i mask0, mask1, mask2, mask3; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6; + __m128i out0, out1; + + mask0 = __lsx_vld(mc_filt_mask_arr, 0); + src -= (3 + 3 * src_stride); + DUP4_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_horiz, 2, filter_horiz, 4, + filter_horiz, 6, filt_hz0, filt_hz1, filt_hz2, filt_hz3); + DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2); + mask3 = __lsx_vaddi_bu(mask0, 6); + + LSX_LD_4(src, src_stride, src0, src1, src2, src3); + src += src_stride; + src4 = __lsx_vld(src, 0); + src += src_stride; + src5 = __lsx_vld(src, 0); + src += src_stride; + src6 = __lsx_vld(src, 0); + src += src_stride; + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + + src0 = HORIZ_8TAP_FILT(src0, src0, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src1 = HORIZ_8TAP_FILT(src1, src1, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src2 = HORIZ_8TAP_FILT(src2, src2, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src3 = HORIZ_8TAP_FILT(src3, src3, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src4 = HORIZ_8TAP_FILT(src4, src4, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src5 = HORIZ_8TAP_FILT(src5, src5, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src6 = HORIZ_8TAP_FILT(src6, src6, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + + DUP4_ARG2(__lsx_vldrepl_h, filter_vert, 0, filter_vert, 2, filter_vert, 4, + filter_vert, 6, filt_vt0, filt_vt1, filt_vt2, filt_vt3); + DUP4_ARG2(__lsx_vpackev_b, src1, src0, src3, src2, src5, src4, src2, src1, + tmp0, tmp1, tmp2, tmp4); + DUP2_ARG2(__lsx_vpackev_b, src4, src3, src6, src5, tmp5, tmp6); + + for (; loop_cnt--;) { + LSX_LD_4(src, src_stride, src7, src8, src9, src10); + src += src_stride; + DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7, + src8, src9, src10); + src7 = HORIZ_8TAP_FILT(src7, src7, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + tmp3 = __lsx_vpackev_b(src7, src6); + out0 = FILT_8TAP_DPADD_S_H(tmp0, tmp1, tmp2, tmp3, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + src8 = HORIZ_8TAP_FILT(src8, src8, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src0 = __lsx_vpackev_b(src8, src7); + out1 = FILT_8TAP_DPADD_S_H(tmp4, tmp5, tmp6, src0, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + src9 = HORIZ_8TAP_FILT(src9, src9, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src1 = __lsx_vpackev_b(src9, src8); + src3 = FILT_8TAP_DPADD_S_H(tmp1, tmp2, tmp3, src1, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + src10 = HORIZ_8TAP_FILT(src10, src10, mask0, mask1, mask2, mask3, filt_hz0, + filt_hz1, filt_hz2, filt_hz3); + src2 = __lsx_vpackev_b(src10, src9); + src4 = FILT_8TAP_DPADD_S_H(tmp5, tmp6, src0, src2, filt_vt0, filt_vt1, + filt_vt2, filt_vt3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, src4, src3, 7, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 1); + dst += dst_stride; + + src6 = src10; + tmp0 = tmp2; + tmp1 = tmp3; + tmp2 = src1; + tmp4 = tmp6; + tmp5 = src0; + tmp6 = src2; + } +} + +static void common_hv_8ht_8vt_16w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, + int32_t height) { + common_hv_8ht_8vt_8w_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert, height); + src += 8; + dst += 8; + + common_hv_8ht_8vt_8w_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert, height); + src += 8; + dst += 8; +} + +static void common_hv_8ht_8vt_32w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, + int32_t height) { + int32_t multiple8_cnt; + for (multiple8_cnt = 4; multiple8_cnt--;) { + common_hv_8ht_8vt_8w_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert, height); + src += 8; + dst += 8; + } +} + +static void common_hv_8ht_8vt_64w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, + int32_t height) { + int32_t multiple8_cnt; + for (multiple8_cnt = 8; multiple8_cnt--;) { + common_hv_8ht_8vt_8w_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert, height); + src += 8; + dst += 8; + } +} + +static void common_hv_2ht_2vt_4x4_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, + int8_t *filter_vert) { + __m128i src0, src1, src2, src3, src4, mask; + __m128i filt_vt, filt_hz, vec0, vec1, res0, res1; + __m128i hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, tmp0, tmp1; + __m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 }; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride + src_stride2; + int32_t src_stride4 = src_stride2 << 1; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + mask = __lsx_vld(mc_filt_mask_arr, 16); + + /* rearranging filter */ + filt_hz = __lsx_vldrepl_h(filter_horiz, 0); + filt_vt = __lsx_vldrepl_h(filter_vert, 0); + + src0 = __lsx_vld(src, 0); + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src1, src2, src3, src4); + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src1, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src2, src3, mask, filt_hz, FILTER_BITS); + hz_out4 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + + hz_out1 = __lsx_vshuf_b(hz_out2, hz_out0, shuff); + hz_out3 = __lsx_vpickod_d(hz_out4, hz_out2); + + DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp0, tmp1); + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + DUP2_ARG2(__lsx_vpickev_b, tmp0, tmp0, tmp1, tmp1, res0, res1); + + __lsx_vstelm_w(res0, dst, 0, 0); + __lsx_vstelm_w(res0, dst + dst_stride, 0, 1); + __lsx_vstelm_w(res1, dst + dst_stride2, 0, 0); + __lsx_vstelm_w(res1, dst + dst_stride3, 0, 1); +} + +static void common_hv_2ht_2vt_4x8_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, + int8_t *filter_vert) { + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, mask; + __m128i res0, res1, res2, res3; + __m128i filt_hz, filt_vt, vec0, vec1, vec2, vec3; + __m128i hz_out0, hz_out1, hz_out2, hz_out3, hz_out4, hz_out5, hz_out6; + __m128i hz_out7, hz_out8, vec4, vec5, vec6, vec7; + __m128i shuff = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 }; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + int32_t dst_stride4 = dst_stride2 << 1; + + mask = __lsx_vld(mc_filt_mask_arr, 16); + + /* rearranging filter */ + DUP2_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_vert, 0, filt_hz, filt_vt); + + src0 = __lsx_vld(src, 0); + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src1, src2, src3, src4); + src += src_stride4; + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src5, src6, src7, src8); + src += src_stride4; + + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src1, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src2, src3, mask, filt_hz, FILTER_BITS); + hz_out4 = HORIZ_2TAP_FILT_UH(src4, src5, mask, filt_hz, FILTER_BITS); + hz_out6 = HORIZ_2TAP_FILT_UH(src6, src7, mask, filt_hz, FILTER_BITS); + hz_out8 = HORIZ_2TAP_FILT_UH(src8, src8, mask, filt_hz, FILTER_BITS); + + DUP2_ARG3(__lsx_vshuf_b, hz_out2, hz_out0, shuff, hz_out4, hz_out2, shuff, + hz_out1, hz_out3); + hz_out5 = __lsx_vshuf_b(hz_out6, hz_out4, shuff); + hz_out7 = __lsx_vpickod_d(hz_out8, hz_out6); + DUP4_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, hz_out5, + hz_out4, hz_out7, hz_out6, vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, vec2, filt_vt, vec3, + filt_vt, vec4, vec5, vec6, vec7); + DUP4_ARG2(__lsx_vsrari_h, vec4, FILTER_BITS, vec5, FILTER_BITS, vec6, + FILTER_BITS, vec7, FILTER_BITS, vec4, vec5, vec6, vec7); + DUP4_ARG2(__lsx_vpickev_b, vec4, vec4, vec5, vec5, vec6, vec6, vec7, vec7, + res0, res1, res2, res3); + + __lsx_vstelm_w(res0, dst, 0, 0); + __lsx_vstelm_w(res0, dst + dst_stride, 0, 1); + __lsx_vstelm_w(res1, dst + dst_stride2, 0, 0); + __lsx_vstelm_w(res1, dst + dst_stride3, 0, 1); + dst += dst_stride4; + __lsx_vstelm_w(res2, dst, 0, 0); + __lsx_vstelm_w(res2, dst + dst_stride, 0, 1); + __lsx_vstelm_w(res3, dst + dst_stride2, 0, 0); + __lsx_vstelm_w(res3, dst + dst_stride3, 0, 1); +} + +static void common_hv_2ht_2vt_4w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, + int32_t height) { + if (height == 4) { + common_hv_2ht_2vt_4x4_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert); + } else if (height == 8) { + common_hv_2ht_2vt_4x8_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert); + } +} + +static void common_hv_2ht_2vt_8x4_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, + int8_t *filter_vert) { + __m128i src0, src1, src2, src3, src4, mask, out0, out1; + __m128i filt_hz, filt_vt, vec0, vec1, vec2, vec3; + __m128i hz_out0, hz_out1, tmp0, tmp1, tmp2, tmp3; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + + mask = __lsx_vld(mc_filt_mask_arr, 0); + + /* rearranging filter */ + DUP2_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_vert, 0, filt_hz, filt_vt); + + src0 = __lsx_vld(src, 0); + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src1, src2, src3, src4); + + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS); + hz_out1 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + vec0 = __lsx_vpackev_b(hz_out1, hz_out0); + tmp0 = __lsx_vdp2_h_bu(vec0, filt_vt); + + hz_out0 = HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS); + vec1 = __lsx_vpackev_b(hz_out0, hz_out1); + tmp1 = __lsx_vdp2_h_bu(vec1, filt_vt); + + hz_out1 = HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS); + vec2 = __lsx_vpackev_b(hz_out1, hz_out0); + tmp2 = __lsx_vdp2_h_bu(vec2, filt_vt); + + hz_out0 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + vec3 = __lsx_vpackev_b(hz_out0, hz_out1); + tmp3 = __lsx_vdp2_h_bu(vec3, filt_vt); + + DUP4_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp2, + FILTER_BITS, tmp3, FILTER_BITS, tmp0, tmp1, tmp2, tmp3); + DUP2_ARG2(__lsx_vpickev_b, tmp1, tmp0, tmp3, tmp2, out0, out1); + + __lsx_vstelm_d(out0, dst, 0, 0); + __lsx_vstelm_d(out0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(out1, dst + dst_stride2, 0, 0); + __lsx_vstelm_d(out1, dst + dst_stride3, 0, 1); +} + +static void common_hv_2ht_2vt_8x8mult_lsx(const uint8_t *src, + int32_t src_stride, uint8_t *dst, + int32_t dst_stride, + int8_t *filter_horiz, + int8_t *filter_vert, int32_t height) { + uint32_t loop_cnt = (height >> 3); + __m128i src0, src1, src2, src3, src4, mask, out0, out1; + __m128i filt_hz, filt_vt, vec0; + __m128i hz_out0, hz_out1, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp8; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + mask = __lsx_vld(mc_filt_mask_arr, 0); + + /* rearranging filter */ + DUP2_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_vert, 0, filt_hz, filt_vt); + + src0 = __lsx_vld(src, 0); + src += src_stride; + + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS); + + for (; loop_cnt--;) { + src1 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src3); + src4 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + hz_out1 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + vec0 = __lsx_vpackev_b(hz_out1, hz_out0); + tmp1 = __lsx_vdp2_h_bu(vec0, filt_vt); + + hz_out0 = HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS); + vec0 = __lsx_vpackev_b(hz_out0, hz_out1); + tmp2 = __lsx_vdp2_h_bu(vec0, filt_vt); + + DUP2_ARG2(__lsx_vsrari_h, tmp1, FILTER_BITS, tmp2, FILTER_BITS, tmp1, tmp2); + + hz_out1 = HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS); + vec0 = __lsx_vpackev_b(hz_out1, hz_out0); + tmp3 = __lsx_vdp2_h_bu(vec0, filt_vt); + + hz_out0 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + src1 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src3); + src4 = __lsx_vldx(src, src_stride3); + src += src_stride4; + vec0 = __lsx_vpackev_b(hz_out0, hz_out1); + tmp4 = __lsx_vdp2_h_bu(vec0, filt_vt); + + DUP2_ARG2(__lsx_vsrari_h, tmp3, FILTER_BITS, tmp4, FILTER_BITS, tmp3, tmp4); + DUP2_ARG2(__lsx_vpickev_b, tmp2, tmp1, tmp4, tmp3, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 1); + dst += dst_stride; + + hz_out1 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + vec0 = __lsx_vpackev_b(hz_out1, hz_out0); + tmp5 = __lsx_vdp2_h_bu(vec0, filt_vt); + + hz_out0 = HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS); + vec0 = __lsx_vpackev_b(hz_out0, hz_out1); + tmp6 = __lsx_vdp2_h_bu(vec0, filt_vt); + + hz_out1 = HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS); + vec0 = __lsx_vpackev_b(hz_out1, hz_out0); + tmp7 = __lsx_vdp2_h_bu(vec0, filt_vt); + + hz_out0 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + vec0 = __lsx_vpackev_b(hz_out0, hz_out1); + tmp8 = __lsx_vdp2_h_bu(vec0, filt_vt); + + DUP4_ARG2(__lsx_vsrari_h, tmp5, FILTER_BITS, tmp6, FILTER_BITS, tmp7, + FILTER_BITS, tmp8, FILTER_BITS, tmp5, tmp6, tmp7, tmp8); + DUP2_ARG2(__lsx_vpickev_b, tmp6, tmp5, tmp8, tmp7, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 1); + dst += dst_stride; + } +} + +static void common_hv_2ht_2vt_8w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, + int32_t height) { + if (height == 4) { + common_hv_2ht_2vt_8x4_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert); + } else { + common_hv_2ht_2vt_8x8mult_lsx(src, src_stride, dst, dst_stride, + filter_horiz, filter_vert, height); + } +} + +static void common_hv_2ht_2vt_16w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, + int32_t height) { + uint32_t loop_cnt = (height >> 2); + __m128i src0, src1, src2, src3, src4, src5, src6, src7, mask; + __m128i filt_hz, filt_vt, vec0, vec1; + __m128i tmp, tmp1, tmp2, hz_out0, hz_out1, hz_out2, hz_out3; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + mask = __lsx_vld(mc_filt_mask_arr, 0); + + /* rearranging filter */ + DUP2_ARG2(__lsx_vldrepl_h, filter_horiz, 0, filter_vert, 0, filt_hz, filt_vt); + + DUP2_ARG2(__lsx_vld, src, 0, src, 8, src0, src1); + src += src_stride; + + hz_out0 = HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + + for (; loop_cnt--;) { + uint8_t *src_tmp0 = src + 8; + + DUP2_ARG2(__lsx_vld, src, 0, src_tmp0, 0, src0, src1); + DUP4_ARG2(__lsx_vldx, src, src_stride, src_tmp0, src_stride, src, + src_stride2, src_tmp0, src_stride2, src2, src3, src4, src5); + DUP2_ARG2(__lsx_vldx, src, src_stride3, src_tmp0, src_stride3, src6, src7); + src += src_stride4; + + hz_out1 = HORIZ_2TAP_FILT_UH(src0, src0, mask, filt_hz, FILTER_BITS); + hz_out3 = HORIZ_2TAP_FILT_UH(src1, src1, mask, filt_hz, FILTER_BITS); + DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp1, tmp2); + DUP2_ARG2(__lsx_vsrari_h, tmp1, FILTER_BITS, tmp2, FILTER_BITS, tmp1, tmp2); + tmp = __lsx_vpickev_b(tmp2, tmp1); + __lsx_vst(tmp, dst, 0); + dst += dst_stride; + + hz_out0 = HORIZ_2TAP_FILT_UH(src2, src2, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src3, src3, mask, filt_hz, FILTER_BITS); + DUP2_ARG2(__lsx_vpackev_b, hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp1, tmp2); + DUP2_ARG2(__lsx_vsrari_h, tmp1, FILTER_BITS, tmp2, FILTER_BITS, tmp1, tmp2); + tmp = __lsx_vpickev_b(tmp2, tmp1); + __lsx_vst(tmp, dst, 0); + dst += dst_stride; + + hz_out1 = HORIZ_2TAP_FILT_UH(src4, src4, mask, filt_hz, FILTER_BITS); + hz_out3 = HORIZ_2TAP_FILT_UH(src5, src5, mask, filt_hz, FILTER_BITS); + DUP2_ARG2(__lsx_vpackev_b, hz_out1, hz_out0, hz_out3, hz_out2, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp1, tmp2); + DUP2_ARG2(__lsx_vsrari_h, tmp1, FILTER_BITS, tmp2, FILTER_BITS, tmp1, tmp2); + tmp = __lsx_vpickev_b(tmp2, tmp1); + __lsx_vst(tmp, dst, 0); + dst += dst_stride; + + hz_out0 = HORIZ_2TAP_FILT_UH(src6, src6, mask, filt_hz, FILTER_BITS); + hz_out2 = HORIZ_2TAP_FILT_UH(src7, src7, mask, filt_hz, FILTER_BITS); + DUP2_ARG2(__lsx_vpackev_b, hz_out0, hz_out1, hz_out2, hz_out3, vec0, vec1); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt_vt, vec1, filt_vt, tmp1, tmp2); + DUP2_ARG2(__lsx_vsrari_h, tmp1, FILTER_BITS, tmp2, FILTER_BITS, tmp1, tmp2); + tmp = __lsx_vpickev_b(tmp2, tmp1); + __lsx_vst(tmp, dst, 0); + dst += dst_stride; + } +} + +static void common_hv_2ht_2vt_32w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, + int32_t height) { + common_hv_2ht_2vt_16w_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert, height); + src += 16; + dst += 16; + + common_hv_2ht_2vt_16w_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert, height); + src += 16; + dst += 16; +} + +static void common_hv_2ht_2vt_64w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter_horiz, int8_t *filter_vert, + int32_t height) { + int32_t multiple8_cnt; + for (multiple8_cnt = 4; multiple8_cnt--;) { + common_hv_2ht_2vt_16w_lsx(src, src_stride, dst, dst_stride, filter_horiz, + filter_vert, height); + src += 16; + dst += 16; + } +} + +void vpx_convolve8_lsx(const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, + ptrdiff_t dst_stride, const InterpKernel *filter, + int x0_q4, int32_t x_step_q4, int y0_q4, + int32_t y_step_q4, int32_t w, int32_t h) { + const int16_t *const filter_x = filter[x0_q4]; + const int16_t *const filter_y = filter[y0_q4]; + int8_t cnt, filt_hor[8], filt_ver[8]; + + assert(x_step_q4 == 16); + assert(y_step_q4 == 16); + assert(((const int32_t *)filter_x)[1] != 0x800000); + assert(((const int32_t *)filter_y)[1] != 0x800000); + + for (cnt = 0; cnt < 8; ++cnt) { + filt_hor[cnt] = filter_x[cnt]; + filt_ver[cnt] = filter_y[cnt]; + } + + if (vpx_get_filter_taps(filter_x) == 2 && + vpx_get_filter_taps(filter_y) == 2) { + switch (w) { + case 4: + common_hv_2ht_2vt_4w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], + &filt_ver[3], (int32_t)h); + break; + case 8: + common_hv_2ht_2vt_8w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], + &filt_ver[3], (int32_t)h); + break; + case 16: + common_hv_2ht_2vt_16w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], + &filt_ver[3], (int32_t)h); + break; + case 32: + common_hv_2ht_2vt_32w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], + &filt_ver[3], (int32_t)h); + break; + case 64: + common_hv_2ht_2vt_64w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, &filt_hor[3], + &filt_ver[3], (int32_t)h); + break; + default: + vpx_convolve8_c(src, src_stride, dst, dst_stride, filter, x0_q4, + x_step_q4, y0_q4, y_step_q4, w, h); + break; + } + } else if (vpx_get_filter_taps(filter_x) == 2 || + vpx_get_filter_taps(filter_y) == 2) { + vpx_convolve8_c(src, src_stride, dst, dst_stride, filter, x0_q4, x_step_q4, + y0_q4, y_step_q4, w, h); + } else { + switch (w) { + case 4: + common_hv_8ht_8vt_4w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, filt_ver, + (int32_t)h); + break; + case 8: + common_hv_8ht_8vt_8w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, filt_ver, + (int32_t)h); + break; + case 16: + common_hv_8ht_8vt_16w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, filt_ver, + (int32_t)h); + break; + case 32: + common_hv_8ht_8vt_32w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, filt_ver, + (int32_t)h); + break; + case 64: + common_hv_8ht_8vt_64w_lsx(src, (int32_t)src_stride, dst, + (int32_t)dst_stride, filt_hor, filt_ver, + (int32_t)h); + break; + default: + vpx_convolve8_c(src, src_stride, dst, dst_stride, filter, x0_q4, + x_step_q4, y0_q4, y_step_q4, w, h); + break; + } + } +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_vert_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_vert_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve8_vert_lsx.c @@ -0,0 +1,870 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include "./vpx_dsp_rtcd.h" +#include "vpx_dsp/loongarch/vpx_convolve_lsx.h" + +static void common_vt_8t_4w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = height >> 2; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride + src_stride2; + int32_t src_stride4 = src_stride2 << 1; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5; + __m128i reg0, reg1, reg2, reg3, reg4; + __m128i filter0, filter1, filter2, filter3; + __m128i out0, out1; + uint8_t *_src = (uint8_t *)src - src_stride3; + + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + src0 = __lsx_vld(_src, 0); + DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src1, src2); + src3 = __lsx_vldx(_src, src_stride3); + _src += src_stride4; + src4 = __lsx_vld(_src, 0); + DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src5, src6); + _src += src_stride3; + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, tmp0, + tmp1, tmp2, tmp3); + DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, tmp4, tmp5); + DUP2_ARG2(__lsx_vilvl_d, tmp3, tmp0, tmp4, tmp1, reg0, reg1); + reg2 = __lsx_vilvl_d(tmp5, tmp2); + DUP2_ARG2(__lsx_vxori_b, reg0, 128, reg1, 128, reg0, reg1); + reg2 = __lsx_vxori_b(reg2, 128); + + for (; loop_cnt--;) { + src7 = __lsx_vld(_src, 0); + DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride2, src8, src9); + src10 = __lsx_vldx(_src, src_stride3); + _src += src_stride4; + DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9, + tmp0, tmp1, tmp2, tmp3); + DUP2_ARG2(__lsx_vilvl_d, tmp1, tmp0, tmp3, tmp2, reg3, reg4); + DUP2_ARG2(__lsx_vxori_b, reg3, 128, reg4, 128, reg3, reg4); + out0 = FILT_8TAP_DPADD_S_H(reg0, reg1, reg2, reg3, filter0, filter1, + filter2, filter3); + out1 = FILT_8TAP_DPADD_S_H(reg1, reg2, reg3, reg4, filter0, filter1, + filter2, filter3); + out0 = __lsx_vssrarni_b_h(out1, out0, 7); + out0 = __lsx_vxori_b(out0, 128); + __lsx_vstelm_w(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 2); + dst += dst_stride; + __lsx_vstelm_w(out0, dst, 0, 3); + dst += dst_stride; + + reg0 = reg2; + reg1 = reg3; + reg2 = reg4; + src6 = src10; + } +} + +static void common_vt_8t_8w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = height >> 2; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i tmp0, tmp1, tmp2, tmp3; + __m128i reg0, reg1, reg2, reg3, reg4, reg5; + __m128i filter0, filter1, filter2, filter3; + __m128i out0, out1, out2, out3; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride + src_stride2; + int32_t src_stride4 = src_stride2 << 1; + src = src - src_stride3; + + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + src += src_stride4; + src4 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src5, src6); + src += src_stride3; + + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, reg0, + reg1, reg2, reg3); + DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, reg4, reg5); + + for (; loop_cnt--;) { + src7 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src8, src9); + src10 = __lsx_vldx(src, src_stride3); + src += src_stride4; + DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7, + src8, src9, src10); + DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9, + tmp0, tmp1, tmp2, tmp3); + out0 = FILT_8TAP_DPADD_S_H(reg0, reg1, reg2, tmp0, filter0, filter1, + filter2, filter3); + out1 = FILT_8TAP_DPADD_S_H(reg3, reg4, reg5, tmp1, filter0, filter1, + filter2, filter3); + out2 = FILT_8TAP_DPADD_S_H(reg1, reg2, tmp0, tmp2, filter0, filter1, + filter2, filter3); + out3 = FILT_8TAP_DPADD_S_H(reg4, reg5, tmp1, tmp3, filter0, filter1, + filter2, filter3); + DUP2_ARG3(__lsx_vssrarni_b_h, out1, out0, 7, out3, out2, 7, out0, out1); + DUP2_ARG2(__lsx_vxori_b, out0, 128, out1, 128, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out0, dst, 0, 1); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(out1, dst, 0, 1); + dst += dst_stride; + + reg0 = reg2; + reg1 = tmp0; + reg2 = tmp2; + reg3 = reg5; + reg4 = tmp1; + reg5 = tmp3; + src6 = src10; + } +} + +static void common_vt_8t_16w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = height >> 2; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i filter0, filter1, filter2, filter3; + __m128i reg0, reg1, reg2, reg3, reg4, reg5; + __m128i reg6, reg7, reg8, reg9, reg10, reg11; + __m128i tmp0, tmp1, tmp2, tmp3; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride + src_stride2; + int32_t src_stride4 = src_stride2 << 1; + // uint8_t *_src = (uint8_t *)src - src_stride3; + src -= src_stride3; + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + src0 = __lsx_vld(src, 0); + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src1, src2, src3, src4); + src += src_stride4; + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src5, src6); + src += src_stride3; + + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, reg0, + reg1, reg2, reg3); + DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, reg4, reg5); + DUP4_ARG2(__lsx_vilvh_b, src1, src0, src3, src2, src5, src4, src2, src1, reg6, + reg7, reg8, reg9); + DUP2_ARG2(__lsx_vilvh_b, src4, src3, src6, src5, reg10, reg11); + + for (; loop_cnt--;) { + src7 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src8, src9); + src10 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, src7, + src8, src9, src10); + DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9, + src0, src1, src2, src3); + DUP4_ARG2(__lsx_vilvh_b, src7, src6, src8, src7, src9, src8, src10, src9, + src4, src5, src7, src8); + tmp0 = FILT_8TAP_DPADD_S_H(reg0, reg1, reg2, src0, filter0, filter1, + filter2, filter3); + tmp1 = FILT_8TAP_DPADD_S_H(reg3, reg4, reg5, src1, filter0, filter1, + filter2, filter3); + tmp2 = FILT_8TAP_DPADD_S_H(reg6, reg7, reg8, src4, filter0, filter1, + filter2, filter3); + tmp3 = FILT_8TAP_DPADD_S_H(reg9, reg10, reg11, src5, filter0, filter1, + filter2, filter3); + DUP2_ARG3(__lsx_vssrarni_b_h, tmp2, tmp0, 7, tmp3, tmp1, 7, tmp0, tmp1); + DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + __lsx_vst(tmp0, dst, 0); + dst += dst_stride; + __lsx_vst(tmp1, dst, 0); + dst += dst_stride; + tmp0 = FILT_8TAP_DPADD_S_H(reg1, reg2, src0, src2, filter0, filter1, + filter2, filter3); + tmp1 = FILT_8TAP_DPADD_S_H(reg4, reg5, src1, src3, filter0, filter1, + filter2, filter3); + tmp2 = FILT_8TAP_DPADD_S_H(reg7, reg8, src4, src7, filter0, filter1, + filter2, filter3); + tmp3 = FILT_8TAP_DPADD_S_H(reg10, reg11, src5, src8, filter0, filter1, + filter2, filter3); + DUP2_ARG3(__lsx_vssrarni_b_h, tmp2, tmp0, 7, tmp3, tmp1, 7, tmp0, tmp1); + DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + __lsx_vst(tmp0, dst, 0); + dst += dst_stride; + __lsx_vst(tmp1, dst, 0); + dst += dst_stride; + + reg0 = reg2; + reg1 = src0; + reg2 = src2; + reg3 = reg5; + reg4 = src1; + reg5 = src3; + reg6 = reg8; + reg7 = src4; + reg8 = src7; + reg9 = reg11; + reg10 = src5; + reg11 = src8; + src6 = src10; + } +} + +static void common_vt_8t_16w_mult_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height, + int32_t width) { + uint8_t *src_tmp; + uint8_t *dst_tmp; + uint32_t cnt = width >> 4; + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i filter0, filter1, filter2, filter3; + __m128i reg0, reg1, reg2, reg3, reg4, reg5; + __m128i reg6, reg7, reg8, reg9, reg10, reg11; + __m128i tmp0, tmp1, tmp2, tmp3; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride + src_stride2; + int32_t src_stride4 = src_stride2 << 1; + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + int32_t dst_stride4 = dst_stride2 << 1; + src -= src_stride3; + DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6, + filter0, filter1, filter2, filter3); + + for (; cnt--;) { + uint32_t loop_cnt = height >> 2; + + src_tmp = src; + dst_tmp = dst; + + src0 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src1, + src2); + src3 = __lsx_vldx(src_tmp, src_stride3); + src_tmp += src_stride4; + src4 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src5, + src6); + src_tmp += src_stride3; + + DUP4_ARG2(__lsx_vxori_b, src0, 128, src1, 128, src2, 128, src3, 128, src0, + src1, src2, src3); + DUP2_ARG2(__lsx_vxori_b, src4, 128, src5, 128, src4, src5); + src6 = __lsx_vxori_b(src6, 128); + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1, + reg0, reg1, reg2, reg3); + DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, reg4, reg5); + DUP4_ARG2(__lsx_vilvh_b, src1, src0, src3, src2, src5, src4, src2, src1, + reg6, reg7, reg8, reg9); + DUP2_ARG2(__lsx_vilvh_b, src4, src3, src6, src5, reg10, reg11); + + for (; loop_cnt--;) { + src7 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src8, + src9); + src10 = __lsx_vldx(src_tmp, src_stride3); + src_tmp += src_stride4; + DUP4_ARG2(__lsx_vxori_b, src7, 128, src8, 128, src9, 128, src10, 128, + src7, src8, src9, src10); + DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9, + src0, src1, src2, src3); + DUP4_ARG2(__lsx_vilvh_b, src7, src6, src8, src7, src9, src8, src10, src9, + src4, src5, src7, src8); + tmp0 = FILT_8TAP_DPADD_S_H(reg0, reg1, reg2, src0, filter0, filter1, + filter2, filter3); + tmp1 = FILT_8TAP_DPADD_S_H(reg3, reg4, reg5, src1, filter0, filter1, + filter2, filter3); + tmp2 = FILT_8TAP_DPADD_S_H(reg6, reg7, reg8, src4, filter0, filter1, + filter2, filter3); + tmp3 = FILT_8TAP_DPADD_S_H(reg9, reg10, reg11, src5, filter0, filter1, + filter2, filter3); + DUP2_ARG3(__lsx_vssrarni_b_h, tmp2, tmp0, 7, tmp3, tmp1, 7, tmp0, tmp1); + DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + __lsx_vst(tmp0, dst_tmp, 0); + __lsx_vstx(tmp1, dst_tmp, dst_stride); + tmp0 = FILT_8TAP_DPADD_S_H(reg1, reg2, src0, src2, filter0, filter1, + filter2, filter3); + tmp1 = FILT_8TAP_DPADD_S_H(reg4, reg5, src1, src3, filter0, filter1, + filter2, filter3); + tmp2 = FILT_8TAP_DPADD_S_H(reg7, reg8, src4, src7, filter0, filter1, + filter2, filter3); + tmp3 = FILT_8TAP_DPADD_S_H(reg10, reg11, src5, src8, filter0, filter1, + filter2, filter3); + DUP2_ARG3(__lsx_vssrarni_b_h, tmp2, tmp0, 7, tmp3, tmp1, 7, tmp0, tmp1); + DUP2_ARG2(__lsx_vxori_b, tmp0, 128, tmp1, 128, tmp0, tmp1); + __lsx_vstx(tmp0, dst_tmp, dst_stride2); + __lsx_vstx(tmp1, dst_tmp, dst_stride3); + dst_tmp += dst_stride4; + + reg0 = reg2; + reg1 = src0; + reg2 = src2; + reg3 = reg5; + reg4 = src1; + reg5 = src3; + reg6 = reg8; + reg7 = src4; + reg8 = src7; + reg9 = reg11; + reg10 = src5; + reg11 = src8; + src6 = src10; + } + src += 16; + dst += 16; + } +} + +static void common_vt_8t_32w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + common_vt_8t_16w_mult_lsx(src, src_stride, dst, dst_stride, filter, height, + 32); +} + +static void common_vt_8t_64w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + common_vt_8t_16w_mult_lsx(src, src_stride, dst, dst_stride, filter, height, + 64); +} + +static void common_vt_2t_4x4_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter) { + __m128i src0, src1, src2, src3, src4; + __m128i src10_l, src32_l, src21_l, src43_l, src2110, src4332; + __m128i filt0, tmp0, tmp1; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + + filt0 = __lsx_vldrepl_h(filter, 0); + + src0 = __lsx_vld(src, 0); + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src1, src2, src3, src4); + src += (src_stride4 + src_stride); + + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src3, src2, src4, src3, + src10_l, src21_l, src32_l, src43_l); + DUP2_ARG2(__lsx_vilvl_d, src21_l, src10_l, src43_l, src32_l, src2110, + src4332); + DUP2_ARG2(__lsx_vdp2_h_bu, src2110, filt0, src4332, filt0, tmp0, tmp1); + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + src2110 = __lsx_vpickev_b(tmp1, tmp0); + + __lsx_vstelm_w(src2110, dst, 0, 0); + __lsx_vstelm_w(src2110, dst + dst_stride, 0, 1); + __lsx_vstelm_w(src2110, dst + dst_stride2, 0, 2); + __lsx_vstelm_w(src2110, dst + dst_stride3, 0, 3); +} + +static void common_vt_2t_4x8_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter) { + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8; + __m128i src10_l, src32_l, src54_l, src76_l, src21_l, src43_l; + __m128i src65_l, src87_l, src2110, src4332, src6554, src8776; + __m128i tmp0, tmp1, tmp2, tmp3; + __m128i filt0; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + int32_t dst_stride4 = dst_stride2 << 1; + + uint8_t *dst_tmp1 = dst + dst_stride4; + + filt0 = __lsx_vldrepl_h(filter, 0); + src0 = __lsx_vld(src, 0); + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src1, src2, src3, src4); + src += src_stride4; + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src5, src6, src7, src8); + src += (src_stride4 + src_stride); + + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src3, src2, src4, src3, + src10_l, src21_l, src32_l, src43_l); + DUP4_ARG2(__lsx_vilvl_b, src5, src4, src6, src5, src7, src6, src8, src7, + src54_l, src65_l, src76_l, src87_l); + DUP4_ARG2(__lsx_vilvl_d, src21_l, src10_l, src43_l, src32_l, src65_l, src54_l, + src87_l, src76_l, src2110, src4332, src6554, src8776); + DUP4_ARG2(__lsx_vdp2_h_bu, src2110, filt0, src4332, filt0, src6554, filt0, + src8776, filt0, tmp0, tmp1, tmp2, tmp3); + DUP4_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp2, + FILTER_BITS, tmp3, FILTER_BITS, tmp0, tmp1, tmp2, tmp3); + DUP2_ARG2(__lsx_vpickev_b, tmp1, tmp0, tmp3, tmp2, src2110, src4332); + + __lsx_vstelm_w(src2110, dst, 0, 0); + __lsx_vstelm_w(src2110, dst + dst_stride, 0, 1); + __lsx_vstelm_w(src2110, dst + dst_stride2, 0, 2); + __lsx_vstelm_w(src2110, dst + dst_stride3, 0, 3); + + __lsx_vstelm_w(src4332, dst_tmp1, 0, 0); + __lsx_vstelm_w(src4332, dst_tmp1 + dst_stride, 0, 1); + __lsx_vstelm_w(src4332, dst_tmp1 + dst_stride2, 0, 2); + __lsx_vstelm_w(src4332, dst_tmp1 + dst_stride3, 0, 3); +} + +static void common_vt_2t_4w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + if (height == 4) { + common_vt_2t_4x4_lsx(src, src_stride, dst, dst_stride, filter); + } else if (height == 8) { + common_vt_2t_4x8_lsx(src, src_stride, dst, dst_stride, filter); + } +} + +static void common_vt_2t_8x4_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter) { + __m128i src0, src1, src2, src3, src4, vec0, vec1, vec2, vec3, filt0; + __m128i out0, out1; + __m128i tmp0, tmp1, tmp2, tmp3; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + + /* rearranging filter_y */ + filt0 = __lsx_vldrepl_h(filter, 0); + + src0 = __lsx_vld(src, 0); + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src1, src2, src3, src4); + + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src3, src2, src4, src3, vec0, + vec1, vec2, vec3); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, filt0, + tmp0, tmp1, tmp2, tmp3); + DUP4_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp2, + FILTER_BITS, tmp3, FILTER_BITS, tmp0, tmp1, tmp2, tmp3); + DUP2_ARG2(__lsx_vpickev_b, tmp1, tmp0, tmp3, tmp2, out0, out1); + + __lsx_vstelm_d(out0, dst, 0, 0); + __lsx_vstelm_d(out0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(out1, dst + dst_stride2, 0, 0); + __lsx_vstelm_d(out1, dst + dst_stride3, 0, 1); +} + +static void common_vt_2t_8x8mult_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = (height >> 3); + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, filt0; + __m128i out0, out1; + __m128i tmp0, tmp1, tmp2, tmp3; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + int32_t dst_stride4 = dst_stride2 << 1; + + filt0 = __lsx_vldrepl_h(filter, 0); + src0 = __lsx_vld(src, 0); + src += src_stride; + + for (; loop_cnt--;) { + src1 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src3); + src4 = __lsx_vldx(src, src_stride3); + src += src_stride4; + src5 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src6, src7) + src8 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + DUP4_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src3, src2, src4, src3, + vec0, vec1, vec2, vec3); + DUP4_ARG2(__lsx_vilvl_b, src5, src4, src6, src5, src7, src6, src8, src7, + vec4, vec5, vec6, vec7); + DUP4_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, vec2, filt0, vec3, + filt0, tmp0, tmp1, tmp2, tmp3); + DUP4_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp2, + FILTER_BITS, tmp3, FILTER_BITS, tmp0, tmp1, tmp2, tmp3); + DUP2_ARG2(__lsx_vpickev_b, tmp1, tmp0, tmp3, tmp2, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + __lsx_vstelm_d(out0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(out1, dst + dst_stride2, 0, 0); + __lsx_vstelm_d(out1, dst + dst_stride3, 0, 1); + dst += dst_stride4; + + DUP4_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, vec6, filt0, vec7, + filt0, tmp0, tmp1, tmp2, tmp3); + DUP4_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp2, + FILTER_BITS, tmp3, FILTER_BITS, tmp0, tmp1, tmp2, tmp3); + DUP2_ARG2(__lsx_vpickev_b, tmp1, tmp0, tmp3, tmp2, out0, out1); + __lsx_vstelm_d(out0, dst, 0, 0); + __lsx_vstelm_d(out0, dst + dst_stride, 0, 1); + __lsx_vstelm_d(out1, dst + dst_stride2, 0, 0); + __lsx_vstelm_d(out1, dst + dst_stride3, 0, 1); + dst += dst_stride4; + + src0 = src8; + } +} + +static void common_vt_2t_8w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + if (height == 4) { + common_vt_2t_8x4_lsx(src, src_stride, dst, dst_stride, filter); + } else { + common_vt_2t_8x8mult_lsx(src, src_stride, dst, dst_stride, filter, height); + } +} + +static void common_vt_2t_16w_lsx(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *filter, int y0_q4, + int y_step_q4, int w, int height) { + uint32_t loop_cnt = (height >> 2); + __m128i src0, src1, src2, src3, src4; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, filt0; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + const int16_t *const filter_y = filter[y0_q4]; + int8_t cnt, filt_ver[8]; + + assert(y_step_q4 == 16); + assert(((const int32_t *)filter_y)[1] != 0x800000); + + for (cnt = 8; cnt--;) { + filt_ver[cnt] = filter_y[cnt]; + } + + filt0 = __lsx_vldrepl_h(&filt_ver[3], 0); + + src0 = __lsx_vld(src, 0); + src += src_stride; + + for (; loop_cnt--;) { + src1 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src2, src3); + src4 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, vec0, vec2); + DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, vec1, vec3); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1); + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + tmp4 = __lsx_vpickev_b(tmp1, tmp0); + __lsx_vst(tmp4, dst, 0); + dst += dst_stride; + + DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, vec4, vec6); + DUP2_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, vec5, vec7); + DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp2, tmp3); + DUP2_ARG2(__lsx_vsrari_h, tmp2, FILTER_BITS, tmp3, FILTER_BITS, tmp2, tmp3); + tmp4 = __lsx_vpickev_b(tmp3, tmp2); + __lsx_vst(tmp4, dst, 0); + dst += dst_stride; + + DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1); + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + tmp4 = __lsx_vpickev_b(tmp1, tmp0); + __lsx_vst(tmp4, dst, 0); + dst += dst_stride; + + DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp2, tmp3); + DUP2_ARG2(__lsx_vsrari_h, tmp2, FILTER_BITS, tmp3, FILTER_BITS, tmp2, tmp3); + tmp4 = __lsx_vpickev_b(tmp3, tmp2); + __lsx_vst(tmp4, dst, 0); + dst += dst_stride; + + src0 = src4; + } +} + +static void common_vt_2t_32w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = (height >> 2); + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9; + __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, filt0; + __m128i tmp0, tmp1, tmp2, tmp3, tmp4; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + + uint8_t *src_tmp; + filt0 = __lsx_vldrepl_h(filter, 0); + + src0 = __lsx_vld(src, 0); + src5 = __lsx_vld(src, 16); + src += src_stride; + src_tmp = src + 16; + + for (; loop_cnt--;) { + DUP2_ARG2(__lsx_vld, src, 0, src_tmp, 0, src1, src6); + DUP4_ARG2(__lsx_vldx, src, src_stride, src_tmp, src_stride, src, + src_stride2, src_tmp, src_stride2, src2, src7, src3, src8); + DUP2_ARG2(__lsx_vldx, src, src_stride3, src_tmp, src_stride3, src4, src9); + DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, vec0, vec2); + DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, vec1, vec3); + src += src_stride4; + src_tmp += src_stride4; + + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1); + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + tmp4 = __lsx_vpickev_b(tmp1, tmp0); + __lsx_vst(tmp4, dst, 0); + + DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp2, tmp3); + DUP2_ARG2(__lsx_vsrari_h, tmp2, FILTER_BITS, tmp3, FILTER_BITS, tmp2, tmp3); + tmp4 = __lsx_vpickev_b(tmp3, tmp2); + __lsx_vstx(tmp4, dst, dst_stride); + + DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, vec4, vec6); + DUP2_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, vec5, vec7); + DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1); + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + tmp4 = __lsx_vpickev_b(tmp1, tmp0); + __lsx_vstx(tmp4, dst, dst_stride2); + + DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp2, tmp3); + DUP2_ARG2(__lsx_vsrari_h, tmp2, FILTER_BITS, tmp3, FILTER_BITS, tmp2, tmp3); + tmp4 = __lsx_vpickev_b(tmp3, tmp2); + __lsx_vstx(tmp4, dst, dst_stride3); + + DUP2_ARG2(__lsx_vilvl_b, src6, src5, src7, src6, vec0, vec2); + DUP2_ARG2(__lsx_vilvh_b, src6, src5, src7, src6, vec1, vec3); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1); + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + tmp4 = __lsx_vpickev_b(tmp1, tmp0); + __lsx_vst(tmp4, dst, 16); + + DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp2, tmp3); + DUP2_ARG2(__lsx_vsrari_h, tmp2, FILTER_BITS, tmp3, FILTER_BITS, tmp2, tmp3); + dst += dst_stride; + tmp4 = __lsx_vpickev_b(tmp3, tmp2); + __lsx_vst(tmp4, dst, 16); + + DUP2_ARG2(__lsx_vilvl_b, src8, src7, src9, src8, vec4, vec6); + DUP2_ARG2(__lsx_vilvh_b, src8, src7, src9, src8, vec5, vec7); + DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp0, tmp1); + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + dst += dst_stride; + tmp4 = __lsx_vpickev_b(tmp1, tmp0); + __lsx_vst(tmp4, dst, 16); + + DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp2, tmp3); + DUP2_ARG2(__lsx_vsrari_h, tmp2, FILTER_BITS, tmp3, FILTER_BITS, tmp2, tmp3); + dst += dst_stride; + tmp4 = __lsx_vpickev_b(tmp3, tmp2); + __lsx_vst(tmp4, dst, 16); + + dst += dst_stride; + + src0 = src4; + src5 = src9; + } +} + +static void common_vt_2t_64w_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int8_t *filter, int32_t height) { + uint32_t loop_cnt = (height >> 1); + __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10; + __m128i src11, vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, filt0; + __m128i tmp, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; + + int32_t src_stride2 = src_stride << 1; + int32_t dst_stride2 = dst_stride << 1; + uint8_t *dst_tmp1 = dst + dst_stride; + + filt0 = __lsx_vldrepl_h(filter, 0); + + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src3, src6, + src9); + src += src_stride; + + for (; loop_cnt--;) { + uint8_t *src_tmp0 = src + src_stride; + + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src1, src4, src7, + src10); + DUP4_ARG2(__lsx_vld, src_tmp0, 0, src_tmp0, 16, src_tmp0, 32, src_tmp0, 48, + src2, src5, src8, src11); + src += src_stride2; + + DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, vec0, vec2); + DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, vec1, vec3); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1); + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + tmp = __lsx_vpickev_b(tmp1, tmp0); + __lsx_vst(tmp, dst, 0); + + DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp2, tmp3); + DUP2_ARG2(__lsx_vsrari_h, tmp2, FILTER_BITS, tmp3, FILTER_BITS, tmp2, tmp3); + tmp = __lsx_vpickev_b(tmp3, tmp2); + __lsx_vst(tmp, dst_tmp1, 0); + + DUP2_ARG2(__lsx_vilvl_b, src4, src3, src5, src4, vec4, vec6); + DUP2_ARG2(__lsx_vilvh_b, src4, src3, src5, src4, vec5, vec7); + DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp4, tmp5); + DUP2_ARG2(__lsx_vsrari_h, tmp4, FILTER_BITS, tmp5, FILTER_BITS, tmp4, tmp5); + tmp = __lsx_vpickev_b(tmp5, tmp4); + __lsx_vst(tmp, dst, 16); + + DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp6, tmp7); + DUP2_ARG2(__lsx_vsrari_h, tmp6, FILTER_BITS, tmp7, FILTER_BITS, tmp6, tmp7); + tmp = __lsx_vpickev_b(tmp7, tmp6); + __lsx_vst(tmp, dst_tmp1, 16); + + DUP2_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, vec0, vec2); + DUP2_ARG2(__lsx_vilvh_b, src7, src6, src8, src7, vec1, vec3); + DUP2_ARG2(__lsx_vdp2_h_bu, vec0, filt0, vec1, filt0, tmp0, tmp1); + DUP2_ARG2(__lsx_vsrari_h, tmp0, FILTER_BITS, tmp1, FILTER_BITS, tmp0, tmp1); + tmp = __lsx_vpickev_b(tmp1, tmp0); + __lsx_vst(tmp, dst, 32); + + DUP2_ARG2(__lsx_vdp2_h_bu, vec2, filt0, vec3, filt0, tmp2, tmp3); + DUP2_ARG2(__lsx_vsrari_h, tmp2, FILTER_BITS, tmp3, FILTER_BITS, tmp2, tmp3); + tmp = __lsx_vpickev_b(tmp3, tmp2); + __lsx_vst(tmp, dst_tmp1, 32); + + DUP2_ARG2(__lsx_vilvl_b, src10, src9, src11, src10, vec4, vec6); + DUP2_ARG2(__lsx_vilvh_b, src10, src9, src11, src10, vec5, vec7); + DUP2_ARG2(__lsx_vdp2_h_bu, vec4, filt0, vec5, filt0, tmp4, tmp5); + DUP2_ARG2(__lsx_vsrari_h, tmp4, FILTER_BITS, tmp5, FILTER_BITS, tmp4, tmp5); + tmp = __lsx_vpickev_b(tmp5, tmp4); + __lsx_vst(tmp, dst, 48); + + DUP2_ARG2(__lsx_vdp2_h_bu, vec6, filt0, vec7, filt0, tmp6, tmp7); + DUP2_ARG2(__lsx_vsrari_h, tmp6, FILTER_BITS, tmp7, FILTER_BITS, tmp6, tmp7); + tmp = __lsx_vpickev_b(tmp7, tmp6); + __lsx_vst(tmp, dst_tmp1, 48); + dst += dst_stride2; + dst_tmp1 += dst_stride2; + + src0 = src2; + src3 = src5; + src6 = src8; + src9 = src11; + } +} + +void vpx_convolve8_vert_lsx(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *filter, int x0_q4, + int32_t x_step_q4, int y0_q4, int y_step_q4, int w, + int h) { + const int16_t *const filter_y = filter[y0_q4]; + int8_t cnt, filt_ver[8]; + + assert(y_step_q4 == 16); + assert(((const int32_t *)filter_y)[1] != 0x800000); + + for (cnt = 8; cnt--;) { + filt_ver[cnt] = filter_y[cnt]; + } + + if (vpx_get_filter_taps(filter_y) == 2) { + switch (w) { + case 4: + common_vt_2t_4w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + &filt_ver[3], h); + break; + case 8: + common_vt_2t_8w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + &filt_ver[3], h); + break; + case 16: + common_vt_2t_16w_lsx(src, src_stride, dst, dst_stride, filter, y0_q4, + y_step_q4, w, h); + break; + case 32: + common_vt_2t_32w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + &filt_ver[3], h); + break; + case 64: + common_vt_2t_64w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + &filt_ver[3], h); + break; + default: + vpx_convolve8_vert_c(src, src_stride, dst, dst_stride, filter, x0_q4, + x_step_q4, y0_q4, y_step_q4, w, h); + break; + } + } else { + switch (w) { + case 4: + common_vt_8t_4w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + filt_ver, h); + break; + case 8: + common_vt_8t_8w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + filt_ver, h); + break; + case 16: + common_vt_8t_16w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + filt_ver, h); + break; + case 32: + common_vt_8t_32w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + filt_ver, h); + break; + case 64: + common_vt_8t_64w_lsx(src, (int32_t)src_stride, dst, (int32_t)dst_stride, + filt_ver, h); + break; + default: + vpx_convolve8_vert_c(src, src_stride, dst, dst_stride, filter, x0_q4, + x_step_q4, y0_q4, y_step_q4, w, h); + break; + } + } +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve_avg_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve_avg_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve_avg_lsx.c @@ -0,0 +1,321 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "./vpx_dsp_rtcd.h" +#include "vpx_util/loongson_intrinsics.h" + +static void avg_width4_lsx(const uint8_t *src, int32_t src_stride, uint8_t *dst, + int32_t dst_stride, int32_t height) { + int32_t cnt; + __m128i src0, src1; + __m128i dst0, dst1; + + int32_t src_stride2 = src_stride << 1; + + if ((height % 2) == 0) { + for (cnt = (height / 2); cnt--;) { + src0 = __lsx_vld(src, 0); + src1 = __lsx_vldx(src, src_stride); + src += src_stride2; + + dst0 = __lsx_vld(dst, 0); + dst1 = __lsx_vldx(dst, dst_stride); + DUP2_ARG2(__lsx_vavgr_bu, src0, dst0, src1, dst1, dst0, dst1); + + __lsx_vstelm_w(dst0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_w(dst1, dst, 0, 0); + dst += dst_stride; + } + } +} + +static void avg_width8_lsx(const uint8_t *src, int32_t src_stride, uint8_t *dst, + int32_t dst_stride, int32_t height) { + int32_t cnt = (height / 4); + __m128i src0, src1, src2, src3; + __m128i dst0, dst1, dst2, dst3; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + + for (; cnt--;) { + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + dst0 = __lsx_vld(dst, 0); + DUP2_ARG2(__lsx_vldx, dst, dst_stride, dst, dst_stride2, dst1, dst2); + dst3 = __lsx_vldx(dst, dst_stride3); + + DUP4_ARG2(__lsx_vavgr_bu, src0, dst0, src1, dst1, src2, dst2, src3, dst3, + dst0, dst1, dst2, dst3); + + __lsx_vstelm_d(dst0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(dst1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(dst2, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(dst3, dst, 0, 0); + dst += dst_stride; + } +} + +static void avg_width16_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, int32_t height) { + int32_t cnt = (height / 8); + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + int32_t dst_stride4 = dst_stride2 << 1; + + for (; cnt--;) { + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + src += src_stride4; + src4 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src5, src6); + src7 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + dst0 = __lsx_vld(dst, 0); + DUP2_ARG2(__lsx_vldx, dst, dst_stride, dst, dst_stride2, dst1, dst2); + dst3 = __lsx_vldx(dst, dst_stride3); + dst += dst_stride4; + dst4 = __lsx_vld(dst, 0); + DUP2_ARG2(__lsx_vldx, dst, dst_stride, dst, dst_stride2, dst5, dst6); + dst7 = __lsx_vldx(dst, dst_stride3); + dst -= dst_stride4; + + DUP4_ARG2(__lsx_vavgr_bu, src0, dst0, src1, dst1, src2, dst2, src3, dst3, + dst0, dst1, dst2, dst3); + DUP4_ARG2(__lsx_vavgr_bu, src4, dst4, src5, dst5, src6, dst6, src7, dst7, + dst4, dst5, dst6, dst7); + + __lsx_vst(dst0, dst, 0); + __lsx_vstx(dst1, dst, dst_stride); + __lsx_vstx(dst2, dst, dst_stride2); + __lsx_vstx(dst3, dst, dst_stride3); + dst += dst_stride4; + __lsx_vst(dst4, dst, 0); + __lsx_vstx(dst5, dst, dst_stride); + __lsx_vstx(dst6, dst, dst_stride2); + __lsx_vstx(dst7, dst, dst_stride3); + dst += dst_stride4; + } +} + +static void avg_width32_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, int32_t height) { + int32_t cnt = (height / 8); + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i src8, src9, src10, src11, src12, src13, src14, src15; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + __m128i dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15; + + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + int32_t dst_stride2 = dst_stride << 1; + int32_t dst_stride3 = dst_stride2 + dst_stride; + int32_t dst_stride4 = dst_stride2 << 1; + + for (; cnt--;) { + uint8_t *dst_tmp = dst; + uint8_t *dst_tmp1 = dst_tmp + 16; + uint8_t *src_tmp = src + 16; + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vld, src, 0, src_tmp, 0, src0, src1); + DUP4_ARG2(__lsx_vldx, src, src_stride, src_tmp, src_stride, src, + src_stride2, src_tmp, src_stride2, src2, src3, src4, src5); + DUP2_ARG2(__lsx_vldx, src, src_stride3, src_tmp, src_stride3, src6, src7); + src += src_stride4; + + DUP2_ARG2(__lsx_vld, dst_tmp, 0, dst_tmp1, 0, dst0, dst1); + DUP4_ARG2(__lsx_vldx, dst_tmp, dst_stride, dst_tmp1, dst_stride, dst_tmp, + dst_stride2, dst_tmp1, dst_stride2, dst2, dst3, dst4, dst5); + DUP2_ARG2(__lsx_vldx, dst_tmp, dst_stride3, dst_tmp1, dst_stride3, dst6, + dst7); + dst_tmp += dst_stride4; + dst_tmp1 += dst_stride4; + + src_tmp = src + 16; + DUP2_ARG2(__lsx_vld, src, 0, src_tmp, 0, src8, src9); + DUP4_ARG2(__lsx_vldx, src, src_stride, src_tmp, src_stride, src, + src_stride2, src_tmp, src_stride2, src10, src11, src12, src13); + DUP2_ARG2(__lsx_vldx, src, src_stride3, src_tmp, src_stride3, src14, src15); + src += src_stride4; + + DUP2_ARG2(__lsx_vld, dst_tmp, 0, dst_tmp1, 0, dst8, dst9); + DUP4_ARG2(__lsx_vldx, dst_tmp, dst_stride, dst_tmp1, dst_stride, dst_tmp, + dst_stride2, dst_tmp1, dst_stride2, dst10, dst11, dst12, dst13); + DUP2_ARG2(__lsx_vldx, dst_tmp, dst_stride3, dst_tmp1, dst_stride3, dst14, + dst15); + DUP4_ARG2(__lsx_vavgr_bu, src0, dst0, src1, dst1, src2, dst2, src3, dst3, + dst0, dst1, dst2, dst3); + DUP4_ARG2(__lsx_vavgr_bu, src4, dst4, src5, dst5, src6, dst6, src7, dst7, + dst4, dst5, dst6, dst7); + DUP4_ARG2(__lsx_vavgr_bu, src8, dst8, src9, dst9, src10, dst10, src11, + dst11, dst8, dst9, dst10, dst11); + DUP4_ARG2(__lsx_vavgr_bu, src12, dst12, src13, dst13, src14, dst14, src15, + dst15, dst12, dst13, dst14, dst15); + + dst_tmp = dst + 16; + __lsx_vst(dst0, dst, 0); + __lsx_vstx(dst2, dst, dst_stride); + __lsx_vstx(dst4, dst, dst_stride2); + __lsx_vstx(dst6, dst, dst_stride3); + __lsx_vst(dst1, dst_tmp, 0); + __lsx_vstx(dst3, dst_tmp, dst_stride); + __lsx_vstx(dst5, dst_tmp, dst_stride2); + __lsx_vstx(dst7, dst_tmp, dst_stride3); + dst += dst_stride4; + + __lsx_vst(dst8, dst, 0); + __lsx_vstx(dst10, dst, dst_stride); + __lsx_vstx(dst12, dst, dst_stride2); + __lsx_vstx(dst14, dst, dst_stride3); + __lsx_vst(dst9, dst_tmp1, 0); + __lsx_vstx(dst11, dst_tmp1, dst_stride); + __lsx_vstx(dst13, dst_tmp1, dst_stride2); + __lsx_vstx(dst15, dst_tmp1, dst_stride3); + dst += dst_stride4; + } +} + +static void avg_width64_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, int32_t height) { + int32_t cnt = (height / 4); + uint8_t *dst_tmp = dst; + + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + __m128i src8, src9, src10, src11, src12, src13, src14, src15; + __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7; + __m128i dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15; + + for (; cnt--;) { + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2, + src3); + src += src_stride; + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src4, src5, src6, + src7); + src += src_stride; + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src8, src9, src10, + src11); + src += src_stride; + DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src12, src13, src14, + src15); + src += src_stride; + + DUP4_ARG2(__lsx_vld, dst_tmp, 0, dst_tmp, 16, dst_tmp, 32, dst_tmp, 48, + dst0, dst1, dst2, dst3); + dst_tmp += dst_stride; + DUP4_ARG2(__lsx_vld, dst_tmp, 0, dst_tmp, 16, dst_tmp, 32, dst_tmp, 48, + dst4, dst5, dst6, dst7); + dst_tmp += dst_stride; + DUP4_ARG2(__lsx_vld, dst_tmp, 0, dst_tmp, 16, dst_tmp, 32, dst_tmp, 48, + dst8, dst9, dst10, dst11); + dst_tmp += dst_stride; + DUP4_ARG2(__lsx_vld, dst_tmp, 0, dst_tmp, 16, dst_tmp, 32, dst_tmp, 48, + dst12, dst13, dst14, dst15); + dst_tmp += dst_stride; + + DUP4_ARG2(__lsx_vavgr_bu, src0, dst0, src1, dst1, src2, dst2, src3, dst3, + dst0, dst1, dst2, dst3); + DUP4_ARG2(__lsx_vavgr_bu, src4, dst4, src5, dst5, src6, dst6, src7, dst7, + dst4, dst5, dst6, dst7); + DUP4_ARG2(__lsx_vavgr_bu, src8, dst8, src9, dst9, src10, dst10, src11, + dst11, dst8, dst9, dst10, dst11); + DUP4_ARG2(__lsx_vavgr_bu, src12, dst12, src13, dst13, src14, dst14, src15, + dst15, dst12, dst13, dst14, dst15); + + __lsx_vst(dst0, dst, 0); + __lsx_vst(dst1, dst, 16); + __lsx_vst(dst2, dst, 32); + __lsx_vst(dst3, dst, 48); + dst += dst_stride; + __lsx_vst(dst4, dst, 0); + __lsx_vst(dst5, dst, 16); + __lsx_vst(dst6, dst, 32); + __lsx_vst(dst7, dst, 48); + dst += dst_stride; + __lsx_vst(dst8, dst, 0); + __lsx_vst(dst9, dst, 16); + __lsx_vst(dst10, dst, 32); + __lsx_vst(dst11, dst, 48); + dst += dst_stride; + __lsx_vst(dst12, dst, 0); + __lsx_vst(dst13, dst, 16); + __lsx_vst(dst14, dst, 32); + __lsx_vst(dst15, dst, 48); + dst += dst_stride; + } +} + +void vpx_convolve_avg_lsx(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *filter, int x0_q4, + int32_t x_step_q4, int y0_q4, int32_t y_step_q4, + int32_t w, int32_t h) { + (void)filter; + (void)x0_q4; + (void)x_step_q4; + (void)y0_q4; + (void)y_step_q4; + switch (w) { + case 4: { + avg_width4_lsx(src, src_stride, dst, dst_stride, h); + break; + } + + case 8: { + avg_width8_lsx(src, src_stride, dst, dst_stride, h); + break; + } + case 16: { + avg_width16_lsx(src, src_stride, dst, dst_stride, h); + break; + } + case 32: { + avg_width32_lsx(src, src_stride, dst, dst_stride, h); + break; + } + case 64: { + avg_width64_lsx(src, src_stride, dst, dst_stride, h); + break; + } + default: { + int32_t lp, cnt; + for (cnt = h; cnt--;) { + for (lp = 0; lp < w; ++lp) { + dst[lp] = (((dst[lp] + src[lp]) + 1) >> 1); + } + src += src_stride; + dst += dst_stride; + } + break; + } + } +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve_copy_lsx.c b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve_copy_lsx.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve_copy_lsx.c @@ -0,0 +1,438 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include "./vpx_dsp_rtcd.h" +#include "vpx_util/loongson_intrinsics.h" + +static void copy_width8_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, int32_t height) { + int32_t cnt; + uint64_t out0, out1, out2, out3, out4, out5, out6, out7; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + if ((height % 12) == 0) { + for (cnt = (height / 12); cnt--;) { + src0 = __lsx_vld(src, 0); + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src1, src2, src3, src4); + src += src_stride4; + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src5, src6); + src += src_stride2; + src7 = __lsx_vldx(src, src_stride); + src += src_stride2; + + __lsx_vstelm_d(src0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src2, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src3, dst, 0, 0); + dst += dst_stride; + + __lsx_vstelm_d(src4, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src5, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src6, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src7, dst, 0, 0); + dst += dst_stride; + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + __lsx_vstelm_d(src0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src2, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src3, dst, 0, 0); + dst += dst_stride; + } + } else if ((height % 8) == 0) { + for (cnt = height >> 3; cnt--;) { + src0 = __lsx_vld(src, 0); + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src1, src2, src3, src4); + src += src_stride4; + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src5, src6); + src += src_stride2; + src7 = __lsx_vldx(src, src_stride); + src += src_stride2; + + __lsx_vstelm_d(src0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src2, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src3, dst, 0, 0); + dst += dst_stride; + + __lsx_vstelm_d(src4, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src5, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src6, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src7, dst, 0, 0); + dst += dst_stride; + } + } else if ((height % 4) == 0) { + for (cnt = (height / 4); cnt--;) { + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + __lsx_vstelm_d(src0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src1, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src2, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src3, dst, 0, 0); + dst += dst_stride; + } + } else if ((height % 2) == 0) { + for (cnt = (height / 2); cnt--;) { + src0 = __lsx_vld(src, 0); + src1 = __lsx_vldx(src, src_stride); + src += src_stride2; + + __lsx_vstelm_d(src0, dst, 0, 0); + dst += dst_stride; + __lsx_vstelm_d(src1, dst, 0, 0); + dst += dst_stride; + } + } +} + +static void copy_16multx8mult_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, + int32_t height, int32_t width) { + int32_t cnt, loop_cnt; + uint8_t *src_tmp; + uint8_t *dst_tmp; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + for (cnt = (width >> 4); cnt--;) { + src_tmp = (uint8_t *)src; + dst_tmp = dst; + + for (loop_cnt = (height >> 3); loop_cnt--;) { + src0 = __lsx_vld(src_tmp, 0); + DUP4_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src_tmp, + src_stride3, src_tmp, src_stride4, src1, src2, src3, src4); + src_tmp += src_stride4; + DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src5, + src6); + src_tmp += src_stride2; + src7 = __lsx_vldx(src_tmp, src_stride); + src_tmp += src_stride2; + + __lsx_vst(src0, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src1, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src2, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src3, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src4, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src5, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src6, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src7, dst_tmp, 0); + dst_tmp += dst_stride; + } + src += 16; + dst += 16; + } +} + +static void copy_width16_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, int32_t height) { + int32_t cnt; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + if ((height % 12) == 0) { + for (cnt = (height / 12); cnt--;) { + src0 = __lsx_vld(src, 0); + DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src, src_stride3, + src, src_stride4, src1, src2, src3, src4); + src += src_stride4; + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src5, src6); + src += src_stride2; + src7 = __lsx_vldx(src, src_stride); + src += src_stride2; + + __lsx_vst(src0, dst, 0); + dst += dst_stride; + __lsx_vst(src1, dst, 0); + dst += dst_stride; + __lsx_vst(src2, dst, 0); + dst += dst_stride; + __lsx_vst(src3, dst, 0); + dst += dst_stride; + __lsx_vst(src4, dst, 0); + dst += dst_stride; + __lsx_vst(src5, dst, 0); + dst += dst_stride; + __lsx_vst(src6, dst, 0); + dst += dst_stride; + __lsx_vst(src7, dst, 0); + dst += dst_stride; + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + __lsx_vst(src0, dst, 0); + dst += dst_stride; + __lsx_vst(src1, dst, 0); + dst += dst_stride; + __lsx_vst(src2, dst, 0); + dst += dst_stride; + __lsx_vst(src3, dst, 0); + dst += dst_stride; + } + } else if ((height % 8) == 0) { + copy_16multx8mult_lsx(src, src_stride, dst, dst_stride, height, 16); + } else if ((height % 4) == 0) { + for (cnt = (height >> 2); cnt--;) { + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + src += src_stride4; + + __lsx_vst(src0, dst, 0); + dst += dst_stride; + __lsx_vst(src1, dst, 0); + dst += dst_stride; + __lsx_vst(src2, dst, 0); + dst += dst_stride; + __lsx_vst(src3, dst, 0); + dst += dst_stride; + } + } +} + +static void copy_width32_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, int32_t height) { + int32_t cnt; + uint8_t *src_tmp; + uint8_t *dst_tmp; + __m128i src0, src1, src2, src3, src4, src5, src6, src7; + int32_t src_stride2 = src_stride << 1; + int32_t src_stride3 = src_stride2 + src_stride; + int32_t src_stride4 = src_stride2 << 1; + + if ((height % 12) == 0) { + for (cnt = (height / 12); cnt--;) { + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + + src_tmp = (uint8_t *)src + 16; + src4 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src5, + src6); + src7 = __lsx_vldx(src_tmp, src_stride3); + src += src_stride4; + + __lsx_vst(src0, dst, 0); + dst += dst_stride; + __lsx_vst(src1, dst, 0); + dst += dst_stride; + __lsx_vst(src2, dst, 0); + dst += dst_stride; + __lsx_vst(src3, dst, 0); + dst += dst_stride; + + dst_tmp = dst + 16; + __lsx_vst(src4, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src5, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src6, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src7, dst_tmp, 0); + dst_tmp += dst_stride; + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + + src_tmp = (uint8_t *)src + 16; + src4 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src5, + src6); + src7 = __lsx_vldx(src_tmp, src_stride3); + src += src_stride4; + + __lsx_vst(src0, dst, 0); + dst += dst_stride; + __lsx_vst(src1, dst, 0); + dst += dst_stride; + __lsx_vst(src2, dst, 0); + dst += dst_stride; + __lsx_vst(src3, dst, 0); + dst += dst_stride; + + dst_tmp = dst + 16; + __lsx_vst(src4, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src5, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src6, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src7, dst_tmp, 0); + dst_tmp += dst_stride; + + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + + src_tmp = (uint8_t *)src + 16; + src4 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src5, + src6); + src7 = __lsx_vldx(src_tmp, src_stride3); + src += src_stride4; + + __lsx_vst(src0, dst, 0); + dst += dst_stride; + __lsx_vst(src1, dst, 0); + dst += dst_stride; + __lsx_vst(src2, dst, 0); + dst += dst_stride; + __lsx_vst(src3, dst, 0); + dst += dst_stride; + + dst_tmp = dst + 16; + __lsx_vst(src4, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src5, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src6, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src7, dst_tmp, 0); + dst_tmp += dst_stride; + } + } else if ((height % 8) == 0) { + copy_16multx8mult_lsx(src, src_stride, dst, dst_stride, height, 32); + } else if ((height % 4) == 0) { + for (cnt = (height >> 2); cnt--;) { + src0 = __lsx_vld(src, 0); + DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride2, src1, src2); + src3 = __lsx_vldx(src, src_stride3); + + src_tmp = (uint8_t *)src + 16; + src4 = __lsx_vld(src_tmp, 0); + DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride2, src5, + src6); + src7 = __lsx_vldx(src_tmp, src_stride3); + src += src_stride4; + + __lsx_vst(src0, dst, 0); + dst += dst_stride; + __lsx_vst(src1, dst, 0); + dst += dst_stride; + __lsx_vst(src2, dst, 0); + dst += dst_stride; + __lsx_vst(src3, dst, 0); + dst += dst_stride; + + dst_tmp = dst + 16; + __lsx_vst(src4, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src5, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src6, dst_tmp, 0); + dst_tmp += dst_stride; + __lsx_vst(src7, dst_tmp, 0); + dst_tmp += dst_stride; + } + } +} + +static void copy_width64_lsx(const uint8_t *src, int32_t src_stride, + uint8_t *dst, int32_t dst_stride, int32_t height) { + copy_16multx8mult_lsx(src, src_stride, dst, dst_stride, height, 64); +} + +void vpx_convolve_copy_lsx(const uint8_t *src, ptrdiff_t src_stride, + uint8_t *dst, ptrdiff_t dst_stride, + const InterpKernel *filter, int x0_q4, + int32_t x_step_q4, int y0_q4, int32_t y_step_q4, + int32_t w, int32_t h) { + (void)filter; + (void)x0_q4; + (void)x_step_q4; + (void)y0_q4; + (void)y_step_q4; + + switch (w) { + case 4: { + uint32_t cnt; + __m128i tmp; + for (cnt = h; cnt--;) { + tmp = __lsx_vldrepl_w(src, 0); + __lsx_vstelm_w(tmp, dst, 0, 0); + src += src_stride; + dst += dst_stride; + } + break; + } + case 8: { + copy_width8_lsx(src, src_stride, dst, dst_stride, h); + break; + } + case 16: { + copy_width16_lsx(src, src_stride, dst, dst_stride, h); + break; + } + case 32: { + copy_width32_lsx(src, src_stride, dst, dst_stride, h); + break; + } + case 64: { + copy_width64_lsx(src, src_stride, dst, dst_stride, h); + break; + } + default: { + uint32_t cnt; + for (cnt = h; cnt--;) { + memcpy(dst, src, w); + src += src_stride; + dst += dst_stride; + } + break; + } + } +} diff --git a/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve_lsx.h b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve_lsx.h new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_dsp/loongarch/vpx_convolve_lsx.h @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VPX_VPX_DSP_LOONGARCH_VPX_CONVOLVE_LSX_H_ +#define VPX_VPX_DSP_LOONGARCH_VPX_CONVOLVE_LSX_H_ + +#include "vpx_util/loongson_intrinsics.h" +#include "vpx_dsp/vpx_filter.h" + +#define LSX_LD_4(_src, _stride, _src0, _src1, _src2, _src3) \ + { \ + _src0 = __lsx_vld(_src, 0); \ + _src += _stride; \ + _src1 = __lsx_vld(_src, 0); \ + _src += _stride; \ + _src2 = __lsx_vld(_src, 0); \ + _src += _stride; \ + _src3 = __lsx_vld(_src, 0); \ + } + +#define FILT_8TAP_DPADD_S_H(_reg0, _reg1, _reg2, _reg3, _filter0, _filter1, \ + _filter2, _filter3) \ + ({ \ + __m128i _vec0, _vec1; \ + \ + _vec0 = __lsx_vdp2_h_b(_reg0, _filter0); \ + _vec0 = __lsx_vdp2add_h_b(_vec0, _reg1, _filter1); \ + _vec1 = __lsx_vdp2_h_b(_reg2, _filter2); \ + _vec1 = __lsx_vdp2add_h_b(_vec1, _reg3, _filter3); \ + _vec0 = __lsx_vsadd_h(_vec0, _vec1); \ + \ + _vec0; \ + }) + +#define HORIZ_8TAP_FILT(_src0, _src1, _mask0, _mask1, _mask2, _mask3, \ + _filt_h0, _filt_h1, _filt_h2, _filt_h3) \ + ({ \ + __m128i _tmp0, _tmp1, _tmp2, _tmp3; \ + __m128i _out; \ + \ + DUP4_ARG3(__lsx_vshuf_b, _src1, _src0, _mask0, _src1, _src0, _mask1, \ + _src1, _src0, _mask2, _src1, _src0, _mask3, _tmp0, _tmp1, _tmp2, \ + _tmp3); \ + _out = FILT_8TAP_DPADD_S_H(_tmp0, _tmp1, _tmp2, _tmp3, _filt_h0, _filt_h1, \ + _filt_h2, _filt_h3); \ + _out = __lsx_vsrari_h(_out, FILTER_BITS); \ + _out = __lsx_vsat_h(_out, 7); \ + \ + _out; \ + }) + +#define HORIZ_8TAP_4WID_4VECS_FILT(_src0, _src1, _src2, _src3, _mask0, _mask1, \ + _mask2, _mask3, _filter0, _filter1, \ + _filter2, _filter3, _out0, _out1) \ + { \ + __m128i _tmp0, _tmp1, _tmp2, _tmp3, _tmp4, _tmp5, _tmp6, _tmp7; \ + __m128i _reg0, _reg1, _reg2, _reg3; \ + \ + DUP2_ARG3(__lsx_vshuf_b, _src1, _src0, _mask0, _src3, _src2, _mask0, \ + _tmp0, _tmp1); \ + DUP2_ARG2(__lsx_vdp2_h_b, _tmp0, _filter0, _tmp1, _filter0, _reg0, _reg1); \ + DUP2_ARG3(__lsx_vshuf_b, _src1, _src0, _mask1, _src3, _src2, _mask1, \ + _tmp2, _tmp3); \ + DUP2_ARG3(__lsx_vdp2add_h_b, _reg0, _tmp2, _filter1, _reg1, _tmp3, \ + _filter1, _reg0, _reg1); \ + DUP2_ARG3(__lsx_vshuf_b, _src1, _src0, _mask2, _src3, _src2, _mask2, \ + _tmp4, _tmp5); \ + DUP2_ARG2(__lsx_vdp2_h_b, _tmp4, _filter2, _tmp5, _filter2, _reg2, _reg3); \ + DUP2_ARG3(__lsx_vshuf_b, _src1, _src0, _mask3, _src3, _src2, _mask3, \ + _tmp6, _tmp7); \ + DUP2_ARG3(__lsx_vdp2add_h_b, _reg2, _tmp6, _filter3, _reg3, _tmp7, \ + _filter3, _reg2, _reg3); \ + DUP2_ARG2(__lsx_vsadd_h, _reg0, _reg2, _reg1, _reg3, _out0, _out1); \ + } + +#define HORIZ_8TAP_8WID_4VECS_FILT( \ + _src0, _src1, _src2, _src3, _mask0, _mask1, _mask2, _mask3, _filter0, \ + _filter1, _filter2, _filter3, _out0, _out1, _out2, _out3) \ + { \ + __m128i _tmp0, _tmp1, _tmp2, _tmp3, _tmp4, _tmp5, _tmp6, _tmp7; \ + __m128i _reg0, _reg1, _reg2, _reg3, _reg4, _reg5, _reg6, _reg7; \ + \ + DUP4_ARG3(__lsx_vshuf_b, _src0, _src0, _mask0, _src1, _src1, _mask0, \ + _src2, _src2, _mask0, _src3, _src3, _mask0, _tmp0, _tmp1, _tmp2, \ + _tmp3); \ + DUP4_ARG2(__lsx_vdp2_h_b, _tmp0, _filter0, _tmp1, _filter0, _tmp2, \ + _filter0, _tmp3, _filter0, _reg0, _reg1, _reg2, _reg3); \ + DUP4_ARG3(__lsx_vshuf_b, _src0, _src0, _mask2, _src1, _src1, _mask2, \ + _src2, _src2, _mask2, _src3, _src3, _mask2, _tmp0, _tmp1, _tmp2, \ + _tmp3); \ + DUP4_ARG2(__lsx_vdp2_h_b, _tmp0, _filter2, _tmp1, _filter2, _tmp2, \ + _filter2, _tmp3, _filter2, _reg4, _reg5, _reg6, _reg7); \ + DUP4_ARG3(__lsx_vshuf_b, _src0, _src0, _mask1, _src1, _src1, _mask1, \ + _src2, _src2, _mask1, _src3, _src3, _mask1, _tmp4, _tmp5, _tmp6, \ + _tmp7); \ + DUP4_ARG3(__lsx_vdp2add_h_b, _reg0, _tmp4, _filter1, _reg1, _tmp5, \ + _filter1, _reg2, _tmp6, _filter1, _reg3, _tmp7, _filter1, _reg0, \ + _reg1, _reg2, _reg3); \ + DUP4_ARG3(__lsx_vshuf_b, _src0, _src0, _mask3, _src1, _src1, _mask3, \ + _src2, _src2, _mask3, _src3, _src3, _mask3, _tmp4, _tmp5, _tmp6, \ + _tmp7); \ + DUP4_ARG3(__lsx_vdp2add_h_b, _reg4, _tmp4, _filter3, _reg5, _tmp5, \ + _filter3, _reg6, _tmp6, _filter3, _reg7, _tmp7, _filter3, _reg4, \ + _reg5, _reg6, _reg7); \ + DUP4_ARG2(__lsx_vsadd_h, _reg0, _reg4, _reg1, _reg5, _reg2, _reg6, _reg3, \ + _reg7, _out0, _out1, _out2, _out3); \ + } + +#define HORIZ_2TAP_FILT_UH(in0, in1, mask, coeff, shift) \ + ({ \ + __m128i tmp0_m; \ + __m128i tmp1_m; \ + \ + tmp0_m = __lsx_vshuf_b(in1, in0, mask); \ + tmp1_m = __lsx_vdp2_h_bu(tmp0_m, coeff); \ + tmp1_m = __lsx_vsrari_h(tmp1_m, shift); \ + \ + tmp1_m; \ + }) + +#define PCKEV_AVG_ST4_D(in0, in1, in2, in3, dst0, dst1, pdst, stride) \ + { \ + __m128i tmp0_m, tmp1_m; \ + \ + DUP2_ARG2(__lsx_vpickev_b, in1, in0, in3, in2, tmp0_m, tmp1_m); \ + DUP2_ARG2(__lsx_vavgr_bu, tmp0_m, dst0, tmp1_m, dst1, tmp0_m, tmp1_m); \ + __lsx_vstelm_d(tmp0_m, pdst, 0, 0); \ + pdst += stride; \ + __lsx_vstelm_d(tmp0_m, pdst, 0, 1); \ + pdst += stride; \ + __lsx_vstelm_d(tmp1_m, pdst, 0, 0); \ + pdst += stride; \ + __lsx_vstelm_d(tmp1_m, pdst, 0, 1); \ + } + +#endif // VPX_VPX_DSP_LOONGARCH_VPX_CONVOLVE_LSX_H_ diff --git a/media/libvpx/libvpx/vpx_dsp/mips/sad_mmi.c b/media/libvpx/libvpx/vpx_dsp/mips/sad_mmi.c --- a/media/libvpx/libvpx/vpx_dsp/mips/sad_mmi.c +++ b/media/libvpx/libvpx/vpx_dsp/mips/sad_mmi.c @@ -332,23 +332,10 @@ "mthc1 $0, %[ftmp1] \n\t" \ "biadd %[ftmp1], %[ftmp1] \n\t" \ "paddw %[ftmp3], %[ftmp3], %[ftmp1] \n\t" #endif /* _MIPS_SIM == _ABIO32 */ -// depending on call sites, pass **ref_array to avoid & in subsequent call and -// de-dup with 4D below. -#define sadMxNxK_mmi(m, n, k) \ - void vpx_sad##m##x##n##x##k##_mmi(const uint8_t *src, int src_stride, \ - const uint8_t *ref_array, int ref_stride, \ - uint32_t *sad_array) { \ - int i; \ - for (i = 0; i < (k); ++i) \ - sad_array[i] = \ - vpx_sad##m##x##n##_mmi(src, src_stride, &ref_array[i], ref_stride); \ - } - -// This appears to be equivalent to the above when k == 4 and refs is const #define sadMxNx4D_mmi(m, n) \ void vpx_sad##m##x##n##x4d_mmi(const uint8_t *src, int src_stride, \ const uint8_t *const ref_array[], \ int ref_stride, uint32_t *sad_array) { \ int i; \ @@ -581,14 +568,10 @@ } vpx_sad16xN(32); vpx_sad16xN(16); vpx_sad16xN(8); -sadMxNxK_mmi(16, 16, 3); -sadMxNxK_mmi(16, 16, 8); -sadMxNxK_mmi(16, 8, 3); -sadMxNxK_mmi(16, 8, 8); sadMxNx4D_mmi(16, 32); sadMxNx4D_mmi(16, 16); sadMxNx4D_mmi(16, 8); static inline unsigned int vpx_sad_avg16x(const uint8_t *src, int src_stride, @@ -679,14 +662,10 @@ } vpx_sad8xN(16); vpx_sad8xN(8); vpx_sad8xN(4); -sadMxNxK_mmi(8, 16, 3); -sadMxNxK_mmi(8, 16, 8); -sadMxNxK_mmi(8, 8, 3); -sadMxNxK_mmi(8, 8, 8); sadMxNx4D_mmi(8, 16); sadMxNx4D_mmi(8, 8); sadMxNx4D_mmi(8, 4); static inline unsigned int vpx_sad_avg8x(const uint8_t *src, int src_stride, @@ -775,12 +754,10 @@ return vpx_sad4x(src, src_stride, ref, ref_stride, H); \ } vpx_sad4xN(8); vpx_sad4xN(4); -sadMxNxK_mmi(4, 4, 3); -sadMxNxK_mmi(4, 4, 8); sadMxNx4D_mmi(4, 8); sadMxNx4D_mmi(4, 4); static inline unsigned int vpx_sad_avg4x(const uint8_t *src, int src_stride, const uint8_t *ref, int ref_stride, diff --git a/media/libvpx/libvpx/vpx_dsp/mips/sad_msa.c b/media/libvpx/libvpx/vpx_dsp/mips/sad_msa.c --- a/media/libvpx/libvpx/vpx_dsp/mips/sad_msa.c +++ b/media/libvpx/libvpx/vpx_dsp/mips/sad_msa.c @@ -157,384 +157,10 @@ sad += HADD_UH_U32(sad1); return sad; } -static void sad_4width_x3_msa(const uint8_t *src_ptr, int32_t src_stride, - const uint8_t *ref_ptr, int32_t ref_stride, - int32_t height, uint32_t *sad_array) { - int32_t ht_cnt; - uint32_t src0, src1, src2, src3; - v16u8 src = { 0 }; - v16u8 ref = { 0 }; - v16u8 ref0, ref1, ref2, ref3, diff; - v8u16 sad0 = { 0 }; - v8u16 sad1 = { 0 }; - v8u16 sad2 = { 0 }; - - for (ht_cnt = (height >> 2); ht_cnt--;) { - LW4(src_ptr, src_stride, src0, src1, src2, src3); - src_ptr += (4 * src_stride); - INSERT_W4_UB(src0, src1, src2, src3, src); - - LD_UB4(ref_ptr, ref_stride, ref0, ref1, ref2, ref3); - ref_ptr += (4 * ref_stride); - SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); - diff = __msa_asub_u_b(src, ref); - sad0 += __msa_hadd_u_h(diff, diff); - - SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1); - SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1); - SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); - diff = __msa_asub_u_b(src, ref); - sad1 += __msa_hadd_u_h(diff, diff); - - SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1); - SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1); - SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); - diff = __msa_asub_u_b(src, ref); - sad2 += __msa_hadd_u_h(diff, diff); - } - - sad_array[0] = HADD_UH_U32(sad0); - sad_array[1] = HADD_UH_U32(sad1); - sad_array[2] = HADD_UH_U32(sad2); -} - -static void sad_8width_x3_msa(const uint8_t *src, int32_t src_stride, - const uint8_t *ref, int32_t ref_stride, - int32_t height, uint32_t *sad_array) { - int32_t ht_cnt; - v16u8 src0, src1, src2, src3; - v16u8 ref0, ref1, ref00, ref11, ref22, ref33; - v8u16 sad0 = { 0 }; - v8u16 sad1 = { 0 }; - v8u16 sad2 = { 0 }; - - for (ht_cnt = (height >> 2); ht_cnt--;) { - LD_UB4(src, src_stride, src0, src1, src2, src3); - src += (4 * src_stride); - LD_UB4(ref, ref_stride, ref00, ref11, ref22, ref33); - ref += (4 * ref_stride); - PCKEV_D4_UB(src1, src0, src3, src2, ref11, ref00, ref33, ref22, src0, src1, - ref0, ref1); - sad0 += SAD_UB2_UH(src0, src1, ref0, ref1); - - SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1); - SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1); - PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1); - sad1 += SAD_UB2_UH(src0, src1, ref0, ref1); - - SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1); - SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1); - PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1); - sad2 += SAD_UB2_UH(src0, src1, ref0, ref1); - } - - sad_array[0] = HADD_UH_U32(sad0); - sad_array[1] = HADD_UH_U32(sad1); - sad_array[2] = HADD_UH_U32(sad2); -} - -static void sad_16width_x3_msa(const uint8_t *src_ptr, int32_t src_stride, - const uint8_t *ref_ptr, int32_t ref_stride, - int32_t height, uint32_t *sad_array) { - int32_t ht_cnt; - v16u8 src, ref, ref0, ref1, diff; - v8u16 sad0 = { 0 }; - v8u16 sad1 = { 0 }; - v8u16 sad2 = { 0 }; - - for (ht_cnt = (height >> 1); ht_cnt--;) { - src = LD_UB(src_ptr); - src_ptr += src_stride; - LD_UB2(ref_ptr, 16, ref0, ref1); - ref_ptr += ref_stride; - - diff = __msa_asub_u_b(src, ref0); - sad0 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 1); - diff = __msa_asub_u_b(src, ref); - sad1 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 2); - diff = __msa_asub_u_b(src, ref); - sad2 += __msa_hadd_u_h(diff, diff); - - src = LD_UB(src_ptr); - src_ptr += src_stride; - LD_UB2(ref_ptr, 16, ref0, ref1); - ref_ptr += ref_stride; - - diff = __msa_asub_u_b(src, ref0); - sad0 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 1); - diff = __msa_asub_u_b(src, ref); - sad1 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 2); - diff = __msa_asub_u_b(src, ref); - sad2 += __msa_hadd_u_h(diff, diff); - } - - sad_array[0] = HADD_UH_U32(sad0); - sad_array[1] = HADD_UH_U32(sad1); - sad_array[2] = HADD_UH_U32(sad2); -} - -static void sad_4width_x8_msa(const uint8_t *src_ptr, int32_t src_stride, - const uint8_t *ref_ptr, int32_t ref_stride, - int32_t height, uint32_t *sad_array) { - int32_t ht_cnt; - uint32_t src0, src1, src2, src3; - v16u8 ref0, ref1, ref2, ref3, diff; - v16u8 src = { 0 }; - v16u8 ref = { 0 }; - v8u16 sad0 = { 0 }; - v8u16 sad1 = { 0 }; - v8u16 sad2 = { 0 }; - v8u16 sad3 = { 0 }; - v8u16 sad4 = { 0 }; - v8u16 sad5 = { 0 }; - v8u16 sad6 = { 0 }; - v8u16 sad7 = { 0 }; - - for (ht_cnt = (height >> 2); ht_cnt--;) { - LW4(src_ptr, src_stride, src0, src1, src2, src3); - INSERT_W4_UB(src0, src1, src2, src3, src); - src_ptr += (4 * src_stride); - LD_UB4(ref_ptr, ref_stride, ref0, ref1, ref2, ref3); - ref_ptr += (4 * ref_stride); - - SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); - diff = __msa_asub_u_b(src, ref); - sad0 += __msa_hadd_u_h(diff, diff); - - SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1); - SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1); - SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); - diff = __msa_asub_u_b(src, ref); - sad1 += __msa_hadd_u_h(diff, diff); - - SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1); - SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1); - SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); - diff = __msa_asub_u_b(src, ref); - sad2 += __msa_hadd_u_h(diff, diff); - - SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1); - SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1); - SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); - diff = __msa_asub_u_b(src, ref); - sad3 += __msa_hadd_u_h(diff, diff); - - SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1); - SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1); - SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); - diff = __msa_asub_u_b(src, ref); - sad4 += __msa_hadd_u_h(diff, diff); - - SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1); - SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1); - SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); - diff = __msa_asub_u_b(src, ref); - sad5 += __msa_hadd_u_h(diff, diff); - - SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1); - SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1); - SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); - diff = __msa_asub_u_b(src, ref); - sad6 += __msa_hadd_u_h(diff, diff); - - SLDI_B2_UB(ref0, ref1, ref0, ref1, ref0, ref1, 1); - SLDI_B2_UB(ref2, ref3, ref2, ref3, ref2, ref3, 1); - SAD_INSVE_W4_UB(ref0, ref1, ref2, ref3, ref); - diff = __msa_asub_u_b(src, ref); - sad7 += __msa_hadd_u_h(diff, diff); - } - - sad_array[0] = HADD_UH_U32(sad0); - sad_array[1] = HADD_UH_U32(sad1); - sad_array[2] = HADD_UH_U32(sad2); - sad_array[3] = HADD_UH_U32(sad3); - sad_array[4] = HADD_UH_U32(sad4); - sad_array[5] = HADD_UH_U32(sad5); - sad_array[6] = HADD_UH_U32(sad6); - sad_array[7] = HADD_UH_U32(sad7); -} - -static void sad_8width_x8_msa(const uint8_t *src, int32_t src_stride, - const uint8_t *ref, int32_t ref_stride, - int32_t height, uint32_t *sad_array) { - int32_t ht_cnt; - v16u8 src0, src1, src2, src3; - v16u8 ref0, ref1, ref00, ref11, ref22, ref33; - v8u16 sad0 = { 0 }; - v8u16 sad1 = { 0 }; - v8u16 sad2 = { 0 }; - v8u16 sad3 = { 0 }; - v8u16 sad4 = { 0 }; - v8u16 sad5 = { 0 }; - v8u16 sad6 = { 0 }; - v8u16 sad7 = { 0 }; - - for (ht_cnt = (height >> 2); ht_cnt--;) { - LD_UB4(src, src_stride, src0, src1, src2, src3); - src += (4 * src_stride); - LD_UB4(ref, ref_stride, ref00, ref11, ref22, ref33); - ref += (4 * ref_stride); - PCKEV_D4_UB(src1, src0, src3, src2, ref11, ref00, ref33, ref22, src0, src1, - ref0, ref1); - sad0 += SAD_UB2_UH(src0, src1, ref0, ref1); - - SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1); - SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1); - PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1); - sad1 += SAD_UB2_UH(src0, src1, ref0, ref1); - - SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1); - SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1); - PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1); - sad2 += SAD_UB2_UH(src0, src1, ref0, ref1); - - SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1); - SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1); - PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1); - sad3 += SAD_UB2_UH(src0, src1, ref0, ref1); - - SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1); - SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1); - PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1); - sad4 += SAD_UB2_UH(src0, src1, ref0, ref1); - - SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1); - SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1); - PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1); - sad5 += SAD_UB2_UH(src0, src1, ref0, ref1); - - SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1); - SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1); - PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1); - sad6 += SAD_UB2_UH(src0, src1, ref0, ref1); - - SLDI_B2_UB(ref00, ref11, ref00, ref11, ref00, ref11, 1); - SLDI_B2_UB(ref22, ref33, ref22, ref33, ref22, ref33, 1); - PCKEV_D2_UB(ref11, ref00, ref33, ref22, ref0, ref1); - sad7 += SAD_UB2_UH(src0, src1, ref0, ref1); - } - - sad_array[0] = HADD_UH_U32(sad0); - sad_array[1] = HADD_UH_U32(sad1); - sad_array[2] = HADD_UH_U32(sad2); - sad_array[3] = HADD_UH_U32(sad3); - sad_array[4] = HADD_UH_U32(sad4); - sad_array[5] = HADD_UH_U32(sad5); - sad_array[6] = HADD_UH_U32(sad6); - sad_array[7] = HADD_UH_U32(sad7); -} - -static void sad_16width_x8_msa(const uint8_t *src_ptr, int32_t src_stride, - const uint8_t *ref_ptr, int32_t ref_stride, - int32_t height, uint32_t *sad_array) { - int32_t ht_cnt; - v16u8 src, ref0, ref1, ref; - v16u8 diff; - v8u16 sad0 = { 0 }; - v8u16 sad1 = { 0 }; - v8u16 sad2 = { 0 }; - v8u16 sad3 = { 0 }; - v8u16 sad4 = { 0 }; - v8u16 sad5 = { 0 }; - v8u16 sad6 = { 0 }; - v8u16 sad7 = { 0 }; - - for (ht_cnt = (height >> 1); ht_cnt--;) { - src = LD_UB(src_ptr); - src_ptr += src_stride; - LD_UB2(ref_ptr, 16, ref0, ref1); - ref_ptr += ref_stride; - - diff = __msa_asub_u_b(src, ref0); - sad0 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 1); - diff = __msa_asub_u_b(src, ref); - sad1 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 2); - diff = __msa_asub_u_b(src, ref); - sad2 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 3); - diff = __msa_asub_u_b(src, ref); - sad3 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 4); - diff = __msa_asub_u_b(src, ref); - sad4 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 5); - diff = __msa_asub_u_b(src, ref); - sad5 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 6); - diff = __msa_asub_u_b(src, ref); - sad6 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 7); - diff = __msa_asub_u_b(src, ref); - sad7 += __msa_hadd_u_h(diff, diff); - - src = LD_UB(src_ptr); - src_ptr += src_stride; - LD_UB2(ref_ptr, 16, ref0, ref1); - ref_ptr += ref_stride; - - diff = __msa_asub_u_b(src, ref0); - sad0 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 1); - diff = __msa_asub_u_b(src, ref); - sad1 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 2); - diff = __msa_asub_u_b(src, ref); - sad2 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 3); - diff = __msa_asub_u_b(src, ref); - sad3 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 4); - diff = __msa_asub_u_b(src, ref); - sad4 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 5); - diff = __msa_asub_u_b(src, ref); - sad5 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 6); - diff = __msa_asub_u_b(src, ref); - sad6 += __msa_hadd_u_h(diff, diff); - - ref = (v16u8)__msa_sldi_b((v16i8)ref1, (v16i8)ref0, 7); - diff = __msa_asub_u_b(src, ref); - sad7 += __msa_hadd_u_h(diff, diff); - } - - sad_array[0] = HADD_UH_U32(sad0); - sad_array[1] = HADD_UH_U32(sad1); - sad_array[2] = HADD_UH_U32(sad2); - sad_array[3] = HADD_UH_U32(sad3); - sad_array[4] = HADD_UH_U32(sad4); - sad_array[5] = HADD_UH_U32(sad5); - sad_array[6] = HADD_UH_U32(sad6); - sad_array[7] = HADD_UH_U32(sad7); -} - static void sad_4width_x4d_msa(const uint8_t *src_ptr, int32_t src_stride, const uint8_t *const aref_ptr[], int32_t ref_stride, int32_t height, uint32_t *sad_array) { const uint8_t *ref0_ptr, *ref1_ptr, *ref2_ptr, *ref3_ptr; @@ -1035,84 +661,42 @@ uint32_t vpx_sad64x##height##_msa(const uint8_t *src, int32_t src_stride, \ const uint8_t *ref, int32_t ref_stride) { \ return sad_64width_msa(src, src_stride, ref, ref_stride, height); \ } -#define VPX_SAD_4xHEIGHTx3_MSA(height) \ - void vpx_sad4x##height##x3_msa(const uint8_t *src, int32_t src_stride, \ - const uint8_t *ref, int32_t ref_stride, \ - uint32_t *sads) { \ - sad_4width_x3_msa(src, src_stride, ref, ref_stride, height, sads); \ - } - -#define VPX_SAD_8xHEIGHTx3_MSA(height) \ - void vpx_sad8x##height##x3_msa(const uint8_t *src, int32_t src_stride, \ - const uint8_t *ref, int32_t ref_stride, \ - uint32_t *sads) { \ - sad_8width_x3_msa(src, src_stride, ref, ref_stride, height, sads); \ - } - -#define VPX_SAD_16xHEIGHTx3_MSA(height) \ - void vpx_sad16x##height##x3_msa(const uint8_t *src, int32_t src_stride, \ - const uint8_t *ref, int32_t ref_stride, \ - uint32_t *sads) { \ - sad_16width_x3_msa(src, src_stride, ref, ref_stride, height, sads); \ - } - -#define VPX_SAD_4xHEIGHTx8_MSA(height) \ - void vpx_sad4x##height##x8_msa(const uint8_t *src, int32_t src_stride, \ - const uint8_t *ref, int32_t ref_stride, \ - uint32_t *sads) { \ - sad_4width_x8_msa(src, src_stride, ref, ref_stride, height, sads); \ - } - -#define VPX_SAD_8xHEIGHTx8_MSA(height) \ - void vpx_sad8x##height##x8_msa(const uint8_t *src, int32_t src_stride, \ - const uint8_t *ref, int32_t ref_stride, \ - uint32_t *sads) { \ - sad_8width_x8_msa(src, src_stride, ref, ref_stride, height, sads); \ - } - -#define VPX_SAD_16xHEIGHTx8_MSA(height) \ - void vpx_sad16x##height##x8_msa(const uint8_t *src, int32_t src_stride, \ - const uint8_t *ref, int32_t ref_stride, \ - uint32_t *sads) { \ - sad_16width_x8_msa(src, src_stride, ref, ref_stride, height, sads); \ - } - #define VPX_SAD_4xHEIGHTx4D_MSA(height) \ void vpx_sad4x##height##x4d_msa(const uint8_t *src, int32_t src_stride, \ - const uint8_t *const refs[], \ - int32_t ref_stride, uint32_t *sads) { \ + const uint8_t *const refs[4], \ + int32_t ref_stride, uint32_t sads[4]) { \ sad_4width_x4d_msa(src, src_stride, refs, ref_stride, height, sads); \ } #define VPX_SAD_8xHEIGHTx4D_MSA(height) \ void vpx_sad8x##height##x4d_msa(const uint8_t *src, int32_t src_stride, \ - const uint8_t *const refs[], \ - int32_t ref_stride, uint32_t *sads) { \ + const uint8_t *const refs[4], \ + int32_t ref_stride, uint32_t sads[4]) { \ sad_8width_x4d_msa(src, src_stride, refs, ref_stride, height, sads); \ } #define VPX_SAD_16xHEIGHTx4D_MSA(height) \ void vpx_sad16x##height##x4d_msa(const uint8_t *src, int32_t src_stride, \ - const uint8_t *const refs[], \ - int32_t ref_stride, uint32_t *sads) { \ + const uint8_t *const refs[4], \ + int32_t ref_stride, uint32_t sads[4]) { \ sad_16width_x4d_msa(src, src_stride, refs, ref_stride, height, sads); \ } #define VPX_SAD_32xHEIGHTx4D_MSA(height) \ void vpx_sad32x##height##x4d_msa(const uint8_t *src, int32_t src_stride, \ - const uint8_t *const refs[], \ - int32_t ref_stride, uint32_t *sads) { \ + const uint8_t *const refs[4], \ + int32_t ref_stride, uint32_t sads[4]) { \ sad_32width_x4d_msa(src, src_stride, refs, ref_stride, height, sads); \ } #define VPX_SAD_64xHEIGHTx4D_MSA(height) \ void vpx_sad64x##height##x4d_msa(const uint8_t *src, int32_t src_stride, \ - const uint8_t *const refs[], \ - int32_t ref_stride, uint32_t *sads) { \ + const uint8_t *const refs[4], \ + int32_t ref_stride, uint32_t sads[4]) { \ sad_64width_x4d_msa(src, src_stride, refs, ref_stride, height, sads); \ } #define VPX_AVGSAD_4xHEIGHT_MSA(height) \ uint32_t vpx_sad4x##height##_avg_msa(const uint8_t *src, int32_t src_stride, \ @@ -1184,33 +768,25 @@ VPX_SAD_16xHEIGHTx4D_MSA(32); VPX_AVGSAD_16xHEIGHT_MSA(32); // 16x16 VPX_SAD_16xHEIGHT_MSA(16); -VPX_SAD_16xHEIGHTx3_MSA(16); -VPX_SAD_16xHEIGHTx8_MSA(16); VPX_SAD_16xHEIGHTx4D_MSA(16); VPX_AVGSAD_16xHEIGHT_MSA(16); // 16x8 VPX_SAD_16xHEIGHT_MSA(8); -VPX_SAD_16xHEIGHTx3_MSA(8); -VPX_SAD_16xHEIGHTx8_MSA(8); VPX_SAD_16xHEIGHTx4D_MSA(8); VPX_AVGSAD_16xHEIGHT_MSA(8); // 8x16 VPX_SAD_8xHEIGHT_MSA(16); -VPX_SAD_8xHEIGHTx3_MSA(16); -VPX_SAD_8xHEIGHTx8_MSA(16); VPX_SAD_8xHEIGHTx4D_MSA(16); VPX_AVGSAD_8xHEIGHT_MSA(16); // 8x8 VPX_SAD_8xHEIGHT_MSA(8); -VPX_SAD_8xHEIGHTx3_MSA(8); -VPX_SAD_8xHEIGHTx8_MSA(8); VPX_SAD_8xHEIGHTx4D_MSA(8); VPX_AVGSAD_8xHEIGHT_MSA(8); // 8x4 VPX_SAD_8xHEIGHT_MSA(4); @@ -1222,9 +798,7 @@ VPX_SAD_4xHEIGHTx4D_MSA(8); VPX_AVGSAD_4xHEIGHT_MSA(8); // 4x4 VPX_SAD_4xHEIGHT_MSA(4); -VPX_SAD_4xHEIGHTx3_MSA(4); -VPX_SAD_4xHEIGHTx8_MSA(4); VPX_SAD_4xHEIGHTx4D_MSA(4); VPX_AVGSAD_4xHEIGHT_MSA(4); diff --git a/media/libvpx/libvpx/vpx_dsp/ppc/quantize_vsx.c b/media/libvpx/libvpx/vpx_dsp/ppc/quantize_vsx.c --- a/media/libvpx/libvpx/vpx_dsp/ppc/quantize_vsx.c +++ b/media/libvpx/libvpx/vpx_dsp/ppc/quantize_vsx.c @@ -93,12 +93,12 @@ a = vec_max(a, vec_perm(a, a, vec_perm32)); return vec_max(a, vec_perm(a, a, vec_perm16)); } void vpx_quantize_b_vsx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *zbin_ptr, - const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *zbin_ptr, const int16_t *round_ptr, + const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan_ptr, const int16_t *iscan_ptr) { int16x8_t qcoeff0, qcoeff1, dqcoeff0, dqcoeff1, eob; @@ -120,12 +120,10 @@ zero_mask0 = vec_cmpge(coeff0_abs, zbin); zbin = vec_splat(zbin, 1); zero_mask1 = vec_cmpge(coeff1_abs, zbin); (void)scan_ptr; - (void)skip_block; - assert(!skip_block); qcoeff0 = quantize_coeff(coeff0, coeff0_abs, round, quant, quant_shift, zero_mask0); vec_vsx_st(qcoeff0, 0, qcoeff_ptr); round = vec_splat(round, 1); @@ -194,16 +192,18 @@ eob = vec_max_across(eob); *eob_ptr = eob[0]; } -void vpx_quantize_b_32x32_vsx( - const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, - const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, - const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, - const int16_t *scan_ptr, const int16_t *iscan_ptr) { +void vpx_quantize_b_32x32_vsx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + const int16_t *zbin_ptr, const int16_t *round_ptr, + const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan_ptr, + const int16_t *iscan_ptr) { // In stage 1, we quantize 16 coeffs (DC + 15 AC) // In stage 2, we loop 42 times and quantize 24 coeffs per iteration // (32 * 32 - 16) / 24 = 42 int num_itr = 42; // Offsets are in bytes, 16 coeffs = 32 bytes @@ -225,13 +225,11 @@ int16x8_t coeff0_abs = vec_abs(coeff0); int16x8_t coeff1_abs = vec_abs(coeff1); (void)scan_ptr; - (void)skip_block; (void)n_coeffs; - assert(!skip_block); // 32x32 quantization requires that zbin and round be divided by 2 zbin = vec_sra(vec_add(zbin, vec_ones_s16), vec_ones_u16); round = vec_sra(vec_add(round, vec_ones_s16), vec_ones_u16); diff --git a/media/libvpx/libvpx/vpx_dsp/quantize.h b/media/libvpx/libvpx/vpx_dsp/quantize.h --- a/media/libvpx/libvpx/vpx_dsp/quantize.h +++ b/media/libvpx/libvpx/vpx_dsp/quantize.h @@ -16,26 +16,25 @@ #ifdef __cplusplus extern "C" { #endif -void vpx_quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs, int skip_block, +void vpx_quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs, const int16_t *round_ptr, const int16_t quant, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t dequant, uint16_t *eob_ptr); -void vpx_quantize_dc_32x32(const tran_low_t *coeff_ptr, int skip_block, +void vpx_quantize_dc_32x32(const tran_low_t *coeff_ptr, const int16_t *round_ptr, const int16_t quant, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t dequant, uint16_t *eob_ptr); #if CONFIG_VP9_HIGHBITDEPTH void vpx_highbd_quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs, - int skip_block, const int16_t *round_ptr, - const int16_t quant, tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, const int16_t dequant, - uint16_t *eob_ptr); -void vpx_highbd_quantize_dc_32x32(const tran_low_t *coeff_ptr, int skip_block, + const int16_t *round_ptr, const int16_t quant, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t dequant, uint16_t *eob_ptr); +void vpx_highbd_quantize_dc_32x32(const tran_low_t *coeff_ptr, const int16_t *round_ptr, const int16_t quant, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t dequant, uint16_t *eob_ptr); #endif diff --git a/media/libvpx/libvpx/vpx_dsp/quantize.c b/media/libvpx/libvpx/vpx_dsp/quantize.c --- a/media/libvpx/libvpx/vpx_dsp/quantize.c +++ b/media/libvpx/libvpx/vpx_dsp/quantize.c @@ -13,11 +13,11 @@ #include "./vpx_dsp_rtcd.h" #include "vpx_dsp/quantize.h" #include "vpx_dsp/vpx_dsp_common.h" #include "vpx_mem/vpx_mem.h" -void vpx_quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs, int skip_block, +void vpx_quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs, const int16_t *round_ptr, const int16_t quant, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t dequant, uint16_t *eob_ptr) { const int rc = 0; const int coeff = coeff_ptr[rc]; @@ -26,46 +26,45 @@ int tmp, eob = -1; memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); - if (!skip_block) { - tmp = clamp(abs_coeff + round_ptr[rc != 0], INT16_MIN, INT16_MAX); - tmp = (tmp * quant) >> 16; - qcoeff_ptr[rc] = (tmp ^ coeff_sign) - coeff_sign; - dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant; - if (tmp) eob = 0; - } + tmp = clamp(abs_coeff + round_ptr[rc != 0], INT16_MIN, INT16_MAX); + tmp = (tmp * quant) >> 16; + qcoeff_ptr[rc] = (tmp ^ coeff_sign) - coeff_sign; + dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant; + if (tmp) eob = 0; + *eob_ptr = eob + 1; } #if CONFIG_VP9_HIGHBITDEPTH void vpx_highbd_quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs, - int skip_block, const int16_t *round_ptr, - const int16_t quant, tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, const int16_t dequant, - uint16_t *eob_ptr) { + const int16_t *round_ptr, const int16_t quant, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t dequant, uint16_t *eob_ptr) { int eob = -1; memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); - if (!skip_block) { + { const int coeff = coeff_ptr[0]; const int coeff_sign = (coeff >> 31); const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; const int64_t tmp = abs_coeff + round_ptr[0]; const int abs_qcoeff = (int)((tmp * quant) >> 16); qcoeff_ptr[0] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); dqcoeff_ptr[0] = qcoeff_ptr[0] * dequant; if (abs_qcoeff) eob = 0; } + *eob_ptr = eob + 1; } #endif -void vpx_quantize_dc_32x32(const tran_low_t *coeff_ptr, int skip_block, +void vpx_quantize_dc_32x32(const tran_low_t *coeff_ptr, const int16_t *round_ptr, const int16_t quant, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t dequant, uint16_t *eob_ptr) { const int n_coeffs = 1024; const int rc = 0; @@ -75,60 +74,57 @@ int tmp, eob = -1; memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); - if (!skip_block) { - tmp = clamp(abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1), - INT16_MIN, INT16_MAX); - tmp = (tmp * quant) >> 15; - qcoeff_ptr[rc] = (tmp ^ coeff_sign) - coeff_sign; - dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant / 2; - if (tmp) eob = 0; - } + tmp = clamp(abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], 1), INT16_MIN, + INT16_MAX); + tmp = (tmp * quant) >> 15; + qcoeff_ptr[rc] = (tmp ^ coeff_sign) - coeff_sign; + dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant / 2; + if (tmp) eob = 0; + *eob_ptr = eob + 1; } #if CONFIG_VP9_HIGHBITDEPTH -void vpx_highbd_quantize_dc_32x32(const tran_low_t *coeff_ptr, int skip_block, +void vpx_highbd_quantize_dc_32x32(const tran_low_t *coeff_ptr, const int16_t *round_ptr, const int16_t quant, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t dequant, uint16_t *eob_ptr) { const int n_coeffs = 1024; int eob = -1; memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); - if (!skip_block) { + { const int coeff = coeff_ptr[0]; const int coeff_sign = (coeff >> 31); const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign; const int64_t tmp = abs_coeff + ROUND_POWER_OF_TWO(round_ptr[0], 1); const int abs_qcoeff = (int)((tmp * quant) >> 15); qcoeff_ptr[0] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); dqcoeff_ptr[0] = qcoeff_ptr[0] * dequant / 2; if (abs_qcoeff) eob = 0; } + *eob_ptr = eob + 1; } #endif void vpx_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *zbin_ptr, - const int16_t *round_ptr, const int16_t *quant_ptr, - const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, - tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, - uint16_t *eob_ptr, const int16_t *scan, - const int16_t *iscan) { + const int16_t *zbin_ptr, const int16_t *round_ptr, + const int16_t *quant_ptr, const int16_t *quant_shift_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan, const int16_t *iscan) { int i, non_zero_count = (int)n_coeffs, eob = -1; const int zbins[2] = { zbin_ptr[0], zbin_ptr[1] }; const int nzbins[2] = { zbins[0] * -1, zbins[1] * -1 }; (void)iscan; - (void)skip_block; - assert(!skip_block); memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); // Pre-scan pass @@ -164,22 +160,20 @@ *eob_ptr = eob + 1; } #if CONFIG_VP9_HIGHBITDEPTH void vpx_highbd_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *zbin_ptr, - const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *zbin_ptr, const int16_t *round_ptr, + const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan) { int i, non_zero_count = (int)n_coeffs, eob = -1; const int zbins[2] = { zbin_ptr[0], zbin_ptr[1] }; const int nzbins[2] = { zbins[0] * -1, zbins[1] * -1 }; (void)iscan; - (void)skip_block; - assert(!skip_block); memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); // Pre-scan pass @@ -213,12 +207,12 @@ *eob_ptr = eob + 1; } #endif void vpx_quantize_b_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *zbin_ptr, - const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *zbin_ptr, const int16_t *round_ptr, + const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan) { const int zbins[2] = { ROUND_POWER_OF_TWO(zbin_ptr[0], 1), @@ -227,12 +221,10 @@ int idx = 0; int idx_arr[1024]; int i, eob = -1; (void)iscan; - (void)skip_block; - assert(!skip_block); memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); // Pre-scan pass @@ -275,12 +267,12 @@ *eob_ptr = eob + 1; } #if CONFIG_VP9_HIGHBITDEPTH void vpx_highbd_quantize_b_32x32_c( - const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, - const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, + const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan) { const int zbins[2] = { ROUND_POWER_OF_TWO(zbin_ptr[0], 1), ROUND_POWER_OF_TWO(zbin_ptr[1], 1) }; @@ -288,12 +280,10 @@ int idx = 0; int idx_arr[1024]; int i, eob = -1; (void)iscan; - (void)skip_block; - assert(!skip_block); memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr)); // Pre-scan pass diff --git a/media/libvpx/libvpx/vpx_dsp/sad.c b/media/libvpx/libvpx/vpx_dsp/sad.c --- a/media/libvpx/libvpx/vpx_dsp/sad.c +++ b/media/libvpx/libvpx/vpx_dsp/sad.c @@ -43,27 +43,15 @@ DECLARE_ALIGNED(16, uint8_t, comp_pred[m * n]); \ vpx_comp_avg_pred_c(comp_pred, second_pred, m, n, ref_ptr, ref_stride); \ return sad(src_ptr, src_stride, comp_pred, m, m, n); \ } -// depending on call sites, pass **ref_array to avoid & in subsequent call and -// de-dup with 4D below. -#define sadMxNxK(m, n, k) \ - void vpx_sad##m##x##n##x##k##_c(const uint8_t *src_ptr, int src_stride, \ - const uint8_t *ref_ptr, int ref_stride, \ - uint32_t *sad_array) { \ - int i; \ - for (i = 0; i < k; ++i) \ - sad_array[i] = \ - vpx_sad##m##x##n##_c(src_ptr, src_stride, &ref_ptr[i], ref_stride); \ - } - -// This appears to be equivalent to the above when k == 4 and refs is const +// Compare |src_ptr| to 4 distinct references in |ref_array[4]| #define sadMxNx4D(m, n) \ void vpx_sad##m##x##n##x4d_c(const uint8_t *src_ptr, int src_stride, \ - const uint8_t *const ref_array[], \ - int ref_stride, uint32_t *sad_array) { \ + const uint8_t *const ref_array[4], \ + int ref_stride, uint32_t sad_array[4]) { \ int i; \ for (i = 0; i < 4; ++i) \ sad_array[i] = \ vpx_sad##m##x##n##_c(src_ptr, src_stride, ref_array[i], ref_stride); \ } @@ -81,11 +69,10 @@ sadMxN(32, 64) sadMxNx4D(32, 64) // 32x32 sadMxN(32, 32) -sadMxNxK(32, 32, 8) sadMxNx4D(32, 32) // 32x16 sadMxN(32, 16) sadMxNx4D(32, 16) @@ -94,30 +81,22 @@ sadMxN(16, 32) sadMxNx4D(16, 32) // 16x16 sadMxN(16, 16) -sadMxNxK(16, 16, 3) -sadMxNxK(16, 16, 8) sadMxNx4D(16, 16) // 16x8 sadMxN(16, 8) -sadMxNxK(16, 8, 3) -sadMxNxK(16, 8, 8) sadMxNx4D(16, 8) // 8x16 sadMxN(8, 16) -sadMxNxK(8, 16, 3) -sadMxNxK(8, 16, 8) sadMxNx4D(8, 16) // 8x8 sadMxN(8, 8) -sadMxNxK(8, 8, 3) -sadMxNxK(8, 8, 8) sadMxNx4D(8, 8) // 8x4 sadMxN(8, 4) sadMxNx4D(8, 4) @@ -126,12 +105,10 @@ sadMxN(4, 8) sadMxNx4D(4, 8) // 4x4 sadMxN(4, 4) -sadMxNxK(4, 4, 3) -sadMxNxK(4, 4, 8) sadMxNx4D(4, 4) /* clang-format on */ #if CONFIG_VP9_HIGHBITDEPTH static INLINE @@ -179,19 +156,19 @@ vpx_highbd_comp_avg_pred_c(comp_pred, CONVERT_TO_SHORTPTR(second_pred), m, \ n, CONVERT_TO_SHORTPTR(ref_ptr), ref_stride); \ return highbd_sadb(src_ptr, src_stride, comp_pred, m, m, n); \ } -#define highbd_sadMxNx4D(m, n) \ - void vpx_highbd_sad##m##x##n##x4d_c(const uint8_t *src_ptr, int src_stride, \ - const uint8_t *const ref_array[], \ - int ref_stride, uint32_t *sad_array) { \ - int i; \ - for (i = 0; i < 4; ++i) { \ - sad_array[i] = vpx_highbd_sad##m##x##n##_c(src_ptr, src_stride, \ - ref_array[i], ref_stride); \ - } \ +#define highbd_sadMxNx4D(m, n) \ + void vpx_highbd_sad##m##x##n##x4d_c(const uint8_t *src_ptr, int src_stride, \ + const uint8_t *const ref_array[4], \ + int ref_stride, uint32_t sad_array[4]) { \ + int i; \ + for (i = 0; i < 4; ++i) { \ + sad_array[i] = vpx_highbd_sad##m##x##n##_c(src_ptr, src_stride, \ + ref_array[i], ref_stride); \ + } \ } /* clang-format off */ // 64x64 highbd_sadMxN(64, 64) diff --git a/media/libvpx/libvpx/vpx_dsp/variance.h b/media/libvpx/libvpx/vpx_dsp/variance.h --- a/media/libvpx/libvpx/vpx_dsp/variance.h +++ b/media/libvpx/libvpx/vpx_dsp/variance.h @@ -57,12 +57,10 @@ #if CONFIG_VP8 typedef struct variance_vtable { vpx_sad_fn_t sdf; vpx_variance_fn_t vf; vpx_subpixvariance_fn_t svf; - vpx_sad_multi_fn_t sdx3f; - vpx_sad_multi_fn_t sdx8f; vpx_sad_multi_d_fn_t sdx4df; #if VPX_ARCH_X86 || VPX_ARCH_X86_64 vp8_copy32xn_fn_t copymem; #endif } vp8_variance_fn_ptr_t; @@ -74,11 +72,10 @@ vpx_sad_avg_fn_t sdaf; vpx_variance_fn_t vf; vpx_subpixvariance_fn_t svf; vpx_subp_avg_variance_fn_t svaf; vpx_sad_multi_d_fn_t sdx4df; - vpx_sad_multi_fn_t sdx8f; } vp9_variance_fn_ptr_t; #endif // CONFIG_VP9 #ifdef __cplusplus } // extern "C" diff --git a/media/libvpx/libvpx/vpx_dsp/vpx_dsp.mk b/media/libvpx/libvpx/vpx_dsp/vpx_dsp.mk --- a/media/libvpx/libvpx/vpx_dsp/vpx_dsp.mk +++ b/media/libvpx/libvpx/vpx_dsp/vpx_dsp.mk @@ -72,10 +72,11 @@ endif # CONFIG_POSTPROC DSP_SRCS-$(HAVE_NEON_ASM) += arm/intrapred_neon_asm$(ASM) DSP_SRCS-$(HAVE_NEON) += arm/intrapred_neon.c DSP_SRCS-$(HAVE_MSA) += mips/intrapred_msa.c +DSP_SRCS-$(HAVE_LSX) += loongarch/intrapred_lsx.c DSP_SRCS-$(HAVE_DSPR2) += mips/intrapred4_dspr2.c DSP_SRCS-$(HAVE_DSPR2) += mips/intrapred8_dspr2.c DSP_SRCS-$(HAVE_DSPR2) += mips/intrapred16_dspr2.c DSP_SRCS-$(HAVE_DSPR2) += mips/common_dspr2.h @@ -160,14 +161,25 @@ DSP_SRCS-$(HAVE_DSPR2) += mips/convolve8_horiz_dspr2.c DSP_SRCS-$(HAVE_DSPR2) += mips/convolve8_vert_dspr2.c DSP_SRCS-$(HAVE_VSX) += ppc/vpx_convolve_vsx.c +# common (lsx) +DSP_SRCS-$(HAVE_LSX) += loongarch/vpx_convolve8_avg_horiz_lsx.c +DSP_SRCS-$(HAVE_LSX) += loongarch/vpx_convolve8_avg_vert_lsx.c +DSP_SRCS-$(HAVE_LSX) += loongarch/vpx_convolve8_horiz_lsx.c +DSP_SRCS-$(HAVE_LSX) += loongarch/vpx_convolve8_vert_lsx.c +DSP_SRCS-$(HAVE_LSX) += loongarch/vpx_convolve8_lsx.c +DSP_SRCS-$(HAVE_LSX) += loongarch/vpx_convolve8_avg_lsx.c +DSP_SRCS-$(HAVE_LSX) += loongarch/vpx_convolve_avg_lsx.c +DSP_SRCS-$(HAVE_LSX) += loongarch/vpx_convolve_copy_lsx.c +DSP_SRCS-$(HAVE_LSX) += loongarch/vpx_convolve_lsx.h + # loop filters DSP_SRCS-yes += loopfilter.c -DSP_SRCS-$(HAVE_SSE2) += x86/loopfilter_intrin_sse2.c +DSP_SRCS-$(HAVE_SSE2) += x86/loopfilter_sse2.c DSP_SRCS-$(HAVE_AVX2) += x86/loopfilter_avx2.c ifeq ($(HAVE_NEON_ASM),yes) DSP_SRCS-yes += arm/loopfilter_16_neon$(ASM) DSP_SRCS-yes += arm/loopfilter_8_neon$(ASM) @@ -186,19 +198,25 @@ DSP_SRCS-$(HAVE_DSPR2) += mips/loopfilter_masks_dspr2.h DSP_SRCS-$(HAVE_DSPR2) += mips/loopfilter_mb_dspr2.c DSP_SRCS-$(HAVE_DSPR2) += mips/loopfilter_mb_horiz_dspr2.c DSP_SRCS-$(HAVE_DSPR2) += mips/loopfilter_mb_vert_dspr2.c +DSP_SRCS-$(HAVE_LSX) += loongarch/loopfilter_lsx.h +DSP_SRCS-$(HAVE_LSX) += loongarch/loopfilter_16_lsx.c +DSP_SRCS-$(HAVE_LSX) += loongarch/loopfilter_8_lsx.c +DSP_SRCS-$(HAVE_LSX) += loongarch/loopfilter_4_lsx.c + ifeq ($(CONFIG_VP9_HIGHBITDEPTH),yes) DSP_SRCS-$(HAVE_NEON) += arm/highbd_loopfilter_neon.c DSP_SRCS-$(HAVE_SSE2) += x86/highbd_loopfilter_sse2.c endif # CONFIG_VP9_HIGHBITDEPTH endif # CONFIG_VP9 DSP_SRCS-yes += txfm_common.h DSP_SRCS-$(HAVE_SSE2) += x86/txfm_common_sse2.h DSP_SRCS-$(HAVE_MSA) += mips/txfm_macros_msa.h +DSP_SRCS-$(HAVE_LSX) += loongarch/txfm_macros_lsx.h # forward transform ifeq ($(CONFIG_VP9_ENCODER),yes) DSP_SRCS-yes += fwd_txfm.c DSP_SRCS-yes += fwd_txfm.h DSP_SRCS-$(HAVE_SSE2) += x86/fwd_txfm_sse2.h @@ -215,13 +233,16 @@ DSP_SRCS-$(HAVE_NEON) += arm/fdct32x32_neon.c DSP_SRCS-$(HAVE_NEON) += arm/fdct_partial_neon.c DSP_SRCS-$(HAVE_NEON) += arm/fwd_txfm_neon.c DSP_SRCS-$(HAVE_MSA) += mips/fwd_txfm_msa.h DSP_SRCS-$(HAVE_MSA) += mips/fwd_txfm_msa.c +DSP_SRCS-$(HAVE_LSX) += loongarch/fwd_txfm_lsx.h +DSP_SRCS-$(HAVE_LSX) += loongarch/fwd_txfm_lsx.c ifneq ($(CONFIG_VP9_HIGHBITDEPTH),yes) DSP_SRCS-$(HAVE_MSA) += mips/fwd_dct32x32_msa.c +DSP_SRCS-$(HAVE_LSX) += loongarch/fwd_dct32x32_lsx.c endif # !CONFIG_VP9_HIGHBITDEPTH DSP_SRCS-$(HAVE_VSX) += ppc/fdct32x32_vsx.c endif # CONFIG_VP9_ENCODER @@ -250,10 +271,12 @@ DSP_SRCS-$(HAVE_DSPR2) += mips/itrans4_dspr2.c DSP_SRCS-$(HAVE_DSPR2) += mips/itrans8_dspr2.c DSP_SRCS-$(HAVE_DSPR2) += mips/itrans16_dspr2.c DSP_SRCS-$(HAVE_DSPR2) += mips/itrans32_dspr2.c DSP_SRCS-$(HAVE_DSPR2) += mips/itrans32_cols_dspr2.c + +DSP_SRCS-$(HAVE_LSX) += loongarch/idct32x32_lsx.c else # CONFIG_VP9_HIGHBITDEPTH DSP_SRCS-$(HAVE_NEON) += arm/highbd_idct4x4_add_neon.c DSP_SRCS-$(HAVE_NEON) += arm/highbd_idct8x8_add_neon.c DSP_SRCS-$(HAVE_NEON) += arm/highbd_idct16x16_add_neon.c DSP_SRCS-$(HAVE_NEON) += arm/highbd_idct32x32_add_neon.c @@ -314,10 +337,11 @@ DSP_SRCS-$(HAVE_SSE2) += x86/avg_intrin_sse2.c DSP_SRCS-$(HAVE_AVX2) += x86/avg_intrin_avx2.c DSP_SRCS-$(HAVE_NEON) += arm/avg_neon.c DSP_SRCS-$(HAVE_NEON) += arm/hadamard_neon.c DSP_SRCS-$(HAVE_MSA) += mips/avg_msa.c +DSP_SRCS-$(HAVE_LSX) += loongarch/avg_lsx.c ifeq ($(VPX_ARCH_X86_64),yes) DSP_SRCS-$(HAVE_SSSE3) += x86/avg_ssse3_x86_64.asm endif DSP_SRCS-$(HAVE_VSX) += ppc/hadamard_vsx.c @@ -340,16 +364,15 @@ DSP_SRCS-$(HAVE_NEON) += arm/subtract_neon.c DSP_SRCS-$(HAVE_MSA) += mips/sad_msa.c DSP_SRCS-$(HAVE_MSA) += mips/subtract_msa.c +DSP_SRCS-$(HAVE_LSX) += loongarch/sad_lsx.c + DSP_SRCS-$(HAVE_MMI) += mips/sad_mmi.c DSP_SRCS-$(HAVE_MMI) += mips/subtract_mmi.c -DSP_SRCS-$(HAVE_SSE3) += x86/sad_sse3.asm -DSP_SRCS-$(HAVE_SSSE3) += x86/sad_ssse3.asm -DSP_SRCS-$(HAVE_SSE4_1) += x86/sad_sse4.asm DSP_SRCS-$(HAVE_AVX2) += x86/sad4d_avx2.c DSP_SRCS-$(HAVE_AVX2) += x86/sad_avx2.c DSP_SRCS-$(HAVE_AVX512) += x86/sad4d_avx512.c DSP_SRCS-$(HAVE_SSE2) += x86/sad4d_sse2.asm @@ -375,10 +398,14 @@ DSP_SRCS-$(HAVE_NEON) += arm/variance_neon.c DSP_SRCS-$(HAVE_MSA) += mips/variance_msa.c DSP_SRCS-$(HAVE_MSA) += mips/sub_pixel_variance_msa.c +DSP_SRCS-$(HAVE_LSX) += loongarch/variance_lsx.c +DSP_SRCS-$(HAVE_LSX) += loongarch/sub_pixel_variance_lsx.c +DSP_SRCS-$(HAVE_LSX) += loongarch/avg_pred_lsx.c + DSP_SRCS-$(HAVE_MMI) += mips/variance_mmi.c DSP_SRCS-$(HAVE_SSE2) += x86/avg_pred_sse2.c DSP_SRCS-$(HAVE_SSE2) += x86/variance_sse2.c # Contains SSE2 and SSSE3 DSP_SRCS-$(HAVE_AVX2) += x86/variance_avx2.c @@ -411,10 +438,13 @@ # X86 utilities DSP_SRCS-$(HAVE_SSE2) += x86/mem_sse2.h DSP_SRCS-$(HAVE_SSE2) += x86/transpose_sse2.h +# LSX utilities +DSP_SRCS-$(HAVE_LSX) += loongarch/bitdepth_conversion_lsx.h + DSP_SRCS-no += $(DSP_SRCS_REMOVE-yes) DSP_SRCS-yes += vpx_dsp_rtcd.c DSP_SRCS-yes += vpx_dsp_rtcd_defs.pl diff --git a/media/libvpx/libvpx/vpx_dsp/vpx_dsp_rtcd_defs.pl b/media/libvpx/libvpx/vpx_dsp/vpx_dsp_rtcd_defs.pl --- a/media/libvpx/libvpx/vpx_dsp/vpx_dsp_rtcd_defs.pl +++ b/media/libvpx/libvpx/vpx_dsp/vpx_dsp_rtcd_defs.pl @@ -115,11 +115,11 @@ # TODO(crbug.com/webm/1522): Re-enable vsx implementation. specialize qw/vpx_tm_predictor_8x8 neon dspr2 msa sse2/; add_proto qw/void vpx_dc_predictor_8x8/, "uint8_t *dst, ptrdiff_t stride, const uint8_t *above, const uint8_t *left"; # TODO(crbug.com/webm/1522): Re-enable vsx implementation. -specialize qw/vpx_dc_predictor_8x8 dspr2 neon msa sse2/; +specialize qw/vpx_dc_predictor_8x8 dspr2 neon msa sse2 lsx/; add_proto qw/void vpx_dc_top_predictor_8x8/, "uint8_t *dst, ptrdiff_t stride, const uint8_t *above, const uint8_t *left"; specialize qw/vpx_dc_top_predictor_8x8 neon msa sse2/; add_proto qw/void vpx_dc_left_predictor_8x8/, "uint8_t *dst, ptrdiff_t stride, const uint8_t *above, const uint8_t *left"; @@ -153,11 +153,11 @@ add_proto qw/void vpx_tm_predictor_16x16/, "uint8_t *dst, ptrdiff_t stride, const uint8_t *above, const uint8_t *left"; specialize qw/vpx_tm_predictor_16x16 neon msa sse2 vsx/; add_proto qw/void vpx_dc_predictor_16x16/, "uint8_t *dst, ptrdiff_t stride, const uint8_t *above, const uint8_t *left"; -specialize qw/vpx_dc_predictor_16x16 dspr2 neon msa sse2 vsx/; +specialize qw/vpx_dc_predictor_16x16 dspr2 neon msa sse2 vsx lsx/; add_proto qw/void vpx_dc_top_predictor_16x16/, "uint8_t *dst, ptrdiff_t stride, const uint8_t *above, const uint8_t *left"; specialize qw/vpx_dc_top_predictor_16x16 neon msa sse2 vsx/; add_proto qw/void vpx_dc_left_predictor_16x16/, "uint8_t *dst, ptrdiff_t stride, const uint8_t *above, const uint8_t *left"; @@ -366,32 +366,32 @@ if (vpx_config("CONFIG_VP9") eq "yes") { # # Sub Pixel Filters # add_proto qw/void vpx_convolve_copy/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h"; -specialize qw/vpx_convolve_copy neon dspr2 msa sse2 vsx/; +specialize qw/vpx_convolve_copy neon dspr2 msa sse2 vsx lsx/; add_proto qw/void vpx_convolve_avg/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h"; -specialize qw/vpx_convolve_avg neon dspr2 msa sse2 vsx mmi/; +specialize qw/vpx_convolve_avg neon dspr2 msa sse2 vsx mmi lsx/; add_proto qw/void vpx_convolve8/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h"; -specialize qw/vpx_convolve8 sse2 ssse3 avx2 neon dspr2 msa vsx mmi/; +specialize qw/vpx_convolve8 sse2 ssse3 avx2 neon dspr2 msa vsx mmi lsx/; add_proto qw/void vpx_convolve8_horiz/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h"; -specialize qw/vpx_convolve8_horiz sse2 ssse3 avx2 neon dspr2 msa vsx mmi/; +specialize qw/vpx_convolve8_horiz sse2 ssse3 avx2 neon dspr2 msa vsx mmi lsx/; add_proto qw/void vpx_convolve8_vert/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h"; -specialize qw/vpx_convolve8_vert sse2 ssse3 avx2 neon dspr2 msa vsx mmi/; +specialize qw/vpx_convolve8_vert sse2 ssse3 avx2 neon dspr2 msa vsx mmi lsx/; add_proto qw/void vpx_convolve8_avg/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h"; -specialize qw/vpx_convolve8_avg sse2 ssse3 avx2 neon dspr2 msa vsx mmi/; +specialize qw/vpx_convolve8_avg sse2 ssse3 avx2 neon dspr2 msa vsx mmi lsx/; add_proto qw/void vpx_convolve8_avg_horiz/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h"; -specialize qw/vpx_convolve8_avg_horiz sse2 ssse3 avx2 neon dspr2 msa vsx mmi/; +specialize qw/vpx_convolve8_avg_horiz sse2 ssse3 avx2 neon dspr2 msa vsx mmi lsx/; add_proto qw/void vpx_convolve8_avg_vert/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h"; -specialize qw/vpx_convolve8_avg_vert sse2 ssse3 avx2 neon dspr2 msa vsx mmi/; +specialize qw/vpx_convolve8_avg_vert sse2 ssse3 avx2 neon dspr2 msa vsx mmi lsx/; add_proto qw/void vpx_scaled_2d/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h"; specialize qw/vpx_scaled_2d ssse3 neon msa/; add_proto qw/void vpx_scaled_horiz/, "const uint8_t *src, ptrdiff_t src_stride, uint8_t *dst, ptrdiff_t dst_stride, const InterpKernel *filter, int x0_q4, int x_step_q4, int y0_q4, int y_step_q4, int w, int h"; @@ -440,41 +440,41 @@ # add_proto qw/void vpx_lpf_vertical_16/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh"; specialize qw/vpx_lpf_vertical_16 sse2 neon dspr2 msa/; add_proto qw/void vpx_lpf_vertical_16_dual/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh"; -specialize qw/vpx_lpf_vertical_16_dual sse2 neon dspr2 msa/; +specialize qw/vpx_lpf_vertical_16_dual sse2 neon dspr2 msa lsx/; add_proto qw/void vpx_lpf_vertical_8/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh"; -specialize qw/vpx_lpf_vertical_8 sse2 neon dspr2 msa/; +specialize qw/vpx_lpf_vertical_8 sse2 neon dspr2 msa lsx/; add_proto qw/void vpx_lpf_vertical_8_dual/, "uint8_t *s, int pitch, const uint8_t *blimit0, const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, const uint8_t *thresh1"; -specialize qw/vpx_lpf_vertical_8_dual sse2 neon dspr2 msa/; +specialize qw/vpx_lpf_vertical_8_dual sse2 neon dspr2 msa lsx/; add_proto qw/void vpx_lpf_vertical_4/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh"; -specialize qw/vpx_lpf_vertical_4 sse2 neon dspr2 msa/; +specialize qw/vpx_lpf_vertical_4 sse2 neon dspr2 msa lsx/; add_proto qw/void vpx_lpf_vertical_4_dual/, "uint8_t *s, int pitch, const uint8_t *blimit0, const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, const uint8_t *thresh1"; -specialize qw/vpx_lpf_vertical_4_dual sse2 neon dspr2 msa/; +specialize qw/vpx_lpf_vertical_4_dual sse2 neon dspr2 msa lsx/; add_proto qw/void vpx_lpf_horizontal_16/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh"; specialize qw/vpx_lpf_horizontal_16 sse2 avx2 neon dspr2 msa/; add_proto qw/void vpx_lpf_horizontal_16_dual/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh"; -specialize qw/vpx_lpf_horizontal_16_dual sse2 avx2 neon dspr2 msa/; +specialize qw/vpx_lpf_horizontal_16_dual sse2 avx2 neon dspr2 msa lsx/; add_proto qw/void vpx_lpf_horizontal_8/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh"; -specialize qw/vpx_lpf_horizontal_8 sse2 neon dspr2 msa/; +specialize qw/vpx_lpf_horizontal_8 sse2 neon dspr2 msa lsx/; add_proto qw/void vpx_lpf_horizontal_8_dual/, "uint8_t *s, int pitch, const uint8_t *blimit0, const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, const uint8_t *thresh1"; -specialize qw/vpx_lpf_horizontal_8_dual sse2 neon dspr2 msa/; +specialize qw/vpx_lpf_horizontal_8_dual sse2 neon dspr2 msa lsx/; add_proto qw/void vpx_lpf_horizontal_4/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh"; -specialize qw/vpx_lpf_horizontal_4 sse2 neon dspr2 msa/; +specialize qw/vpx_lpf_horizontal_4 sse2 neon dspr2 msa lsx/; add_proto qw/void vpx_lpf_horizontal_4_dual/, "uint8_t *s, int pitch, const uint8_t *blimit0, const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, const uint8_t *thresh1"; -specialize qw/vpx_lpf_horizontal_4_dual sse2 neon dspr2 msa/; +specialize qw/vpx_lpf_horizontal_4_dual sse2 neon dspr2 msa lsx/; } #CONFIG_VP9 if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { add_proto qw/void vpx_highbd_lpf_vertical_16/, "uint16_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh, int bd"; specialize qw/vpx_highbd_lpf_vertical_16 sse2 neon/; @@ -571,32 +571,32 @@ specialize qw/vpx_highbd_fdct32x32_rd sse2/; add_proto qw/void vpx_highbd_fdct32x32_1/, "const int16_t *input, tran_low_t *output, int stride"; } else { add_proto qw/void vpx_fdct4x4/, "const int16_t *input, tran_low_t *output, int stride"; - specialize qw/vpx_fdct4x4 neon sse2 msa/; + specialize qw/vpx_fdct4x4 neon sse2 msa lsx/; add_proto qw/void vpx_fdct4x4_1/, "const int16_t *input, tran_low_t *output, int stride"; specialize qw/vpx_fdct4x4_1 sse2 neon/; add_proto qw/void vpx_fdct8x8/, "const int16_t *input, tran_low_t *output, int stride"; - specialize qw/vpx_fdct8x8 sse2 neon msa/, "$ssse3_x86_64"; + specialize qw/vpx_fdct8x8 sse2 neon msa lsx/, "$ssse3_x86_64"; add_proto qw/void vpx_fdct8x8_1/, "const int16_t *input, tran_low_t *output, int stride"; specialize qw/vpx_fdct8x8_1 sse2 neon msa/; add_proto qw/void vpx_fdct16x16/, "const int16_t *input, tran_low_t *output, int stride"; - specialize qw/vpx_fdct16x16 neon sse2 msa/; + specialize qw/vpx_fdct16x16 neon sse2 msa lsx/; add_proto qw/void vpx_fdct16x16_1/, "const int16_t *input, tran_low_t *output, int stride"; specialize qw/vpx_fdct16x16_1 sse2 neon msa/; add_proto qw/void vpx_fdct32x32/, "const int16_t *input, tran_low_t *output, int stride"; - specialize qw/vpx_fdct32x32 neon sse2 avx2 msa/; + specialize qw/vpx_fdct32x32 neon sse2 avx2 msa lsx/; add_proto qw/void vpx_fdct32x32_rd/, "const int16_t *input, tran_low_t *output, int stride"; - specialize qw/vpx_fdct32x32_rd sse2 avx2 neon msa vsx/; + specialize qw/vpx_fdct32x32_rd sse2 avx2 neon msa vsx lsx/; add_proto qw/void vpx_fdct32x32_1/, "const int16_t *input, tran_low_t *output, int stride"; specialize qw/vpx_fdct32x32_1 sse2 neon msa/; } # CONFIG_VP9_HIGHBITDEPTH } # CONFIG_VP9_ENCODER @@ -650,16 +650,17 @@ specialize qw/vpx_idct16x16_38_add dspr2 msa/; $vpx_idct16x16_38_add_dspr2=vpx_idct16x16_256_add_dspr2; $vpx_idct16x16_38_add_msa=vpx_idct16x16_256_add_msa; specialize qw/vpx_idct16x16_10_add dspr2 msa/; specialize qw/vpx_idct16x16_1_add dspr2 msa/; - specialize qw/vpx_idct32x32_1024_add dspr2 msa/; + specialize qw/vpx_idct32x32_1024_add dspr2 msa lsx/; specialize qw/vpx_idct32x32_135_add dspr2 msa/; $vpx_idct32x32_135_add_dspr2=vpx_idct32x32_1024_add_dspr2; $vpx_idct32x32_135_add_msa=vpx_idct32x32_1024_add_msa; - specialize qw/vpx_idct32x32_34_add dspr2 msa/; - specialize qw/vpx_idct32x32_1_add dspr2 msa/; + $vpx_idct32x32_135_add_lsx=vpx_idct32x32_1024_add_lsx; + specialize qw/vpx_idct32x32_34_add dspr2 msa lsx/; + specialize qw/vpx_idct32x32_1_add dspr2 msa lsx/; specialize qw/vpx_iwht4x4_16_add msa/; specialize qw/vpx_iwht4x4_1_add msa/; } # !CONFIG_VP9_HIGHBITDEPTH } # !CONFIG_EMULATE_HARDWARE @@ -707,21 +708,21 @@ # # Quantization # if (vpx_config("CONFIG_VP9_ENCODER") eq "yes") { - add_proto qw/void vpx_quantize_b/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; + add_proto qw/void vpx_quantize_b/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; specialize qw/vpx_quantize_b neon sse2 ssse3 avx vsx/; - add_proto qw/void vpx_quantize_b_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; + add_proto qw/void vpx_quantize_b_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; specialize qw/vpx_quantize_b_32x32 neon ssse3 avx vsx/; if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { - add_proto qw/void vpx_highbd_quantize_b/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; + add_proto qw/void vpx_highbd_quantize_b/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; specialize qw/vpx_highbd_quantize_b sse2/; - add_proto qw/void vpx_highbd_quantize_b_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; + add_proto qw/void vpx_highbd_quantize_b_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; specialize qw/vpx_highbd_quantize_b_32x32 sse2/; } # CONFIG_VP9_HIGHBITDEPTH } # CONFIG_VP9_ENCODER if (vpx_config("CONFIG_ENCODERS") eq "yes") { @@ -733,29 +734,29 @@ # # Single block SAD # add_proto qw/unsigned int vpx_sad64x64/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride"; -specialize qw/vpx_sad64x64 neon avx2 msa sse2 vsx mmi/; +specialize qw/vpx_sad64x64 neon avx2 msa sse2 vsx mmi lsx/; add_proto qw/unsigned int vpx_sad64x32/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride"; specialize qw/vpx_sad64x32 neon avx2 msa sse2 vsx mmi/; add_proto qw/unsigned int vpx_sad32x64/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride"; specialize qw/vpx_sad32x64 neon avx2 msa sse2 vsx mmi/; add_proto qw/unsigned int vpx_sad32x32/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride"; -specialize qw/vpx_sad32x32 neon avx2 msa sse2 vsx mmi/; +specialize qw/vpx_sad32x32 neon avx2 msa sse2 vsx mmi lsx/; add_proto qw/unsigned int vpx_sad32x16/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride"; specialize qw/vpx_sad32x16 neon avx2 msa sse2 vsx mmi/; add_proto qw/unsigned int vpx_sad16x32/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride"; specialize qw/vpx_sad16x32 neon msa sse2 vsx mmi/; add_proto qw/unsigned int vpx_sad16x16/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride"; -specialize qw/vpx_sad16x16 neon msa sse2 vsx mmi/; +specialize qw/vpx_sad16x16 neon msa sse2 vsx mmi lsx/; add_proto qw/unsigned int vpx_sad16x8/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride"; specialize qw/vpx_sad16x8 neon msa sse2 vsx mmi/; add_proto qw/unsigned int vpx_sad8x16/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride"; @@ -786,14 +787,14 @@ add_proto qw/void vpx_minmax_8x8/, "const uint8_t *s, int p, const uint8_t *d, int dp, int *min, int *max"; specialize qw/vpx_minmax_8x8 sse2 neon msa/; if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") { add_proto qw/void vpx_hadamard_8x8/, "const int16_t *src_diff, ptrdiff_t src_stride, tran_low_t *coeff"; - specialize qw/vpx_hadamard_8x8 sse2 neon vsx/, "$ssse3_x86_64"; + specialize qw/vpx_hadamard_8x8 sse2 neon vsx lsx/, "$ssse3_x86_64"; add_proto qw/void vpx_hadamard_16x16/, "const int16_t *src_diff, ptrdiff_t src_stride, tran_low_t *coeff"; - specialize qw/vpx_hadamard_16x16 avx2 sse2 neon vsx/; + specialize qw/vpx_hadamard_16x16 avx2 sse2 neon vsx lsx/; add_proto qw/void vpx_hadamard_32x32/, "const int16_t *src_diff, ptrdiff_t src_stride, tran_low_t *coeff"; specialize qw/vpx_hadamard_32x32 sse2 avx2/; add_proto qw/void vpx_highbd_hadamard_8x8/, "const int16_t *src_diff, ptrdiff_t src_stride, tran_low_t *coeff"; @@ -810,43 +811,43 @@ add_proto qw/int vpx_highbd_satd/, "const tran_low_t *coeff, int length"; specialize qw/vpx_highbd_satd avx2/; } else { add_proto qw/void vpx_hadamard_8x8/, "const int16_t *src_diff, ptrdiff_t src_stride, int16_t *coeff"; - specialize qw/vpx_hadamard_8x8 sse2 neon msa vsx/, "$ssse3_x86_64"; + specialize qw/vpx_hadamard_8x8 sse2 neon msa vsx lsx/, "$ssse3_x86_64"; add_proto qw/void vpx_hadamard_16x16/, "const int16_t *src_diff, ptrdiff_t src_stride, int16_t *coeff"; - specialize qw/vpx_hadamard_16x16 avx2 sse2 neon msa vsx/; + specialize qw/vpx_hadamard_16x16 avx2 sse2 neon msa vsx lsx/; add_proto qw/void vpx_hadamard_32x32/, "const int16_t *src_diff, ptrdiff_t src_stride, int16_t *coeff"; specialize qw/vpx_hadamard_32x32 sse2 avx2/; add_proto qw/int vpx_satd/, "const int16_t *coeff, int length"; specialize qw/vpx_satd avx2 sse2 neon msa/; } - add_proto qw/void vpx_int_pro_row/, "int16_t *hbuf, const uint8_t *ref, const int ref_stride, const int height"; + add_proto qw/void vpx_int_pro_row/, "int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height"; specialize qw/vpx_int_pro_row sse2 neon msa/; add_proto qw/int16_t vpx_int_pro_col/, "const uint8_t *ref, const int width"; specialize qw/vpx_int_pro_col sse2 neon msa/; add_proto qw/int vpx_vector_var/, "const int16_t *ref, const int16_t *src, const int bwl"; specialize qw/vpx_vector_var neon sse2 msa/; } # CONFIG_VP9_ENCODER add_proto qw/unsigned int vpx_sad64x64_avg/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred"; -specialize qw/vpx_sad64x64_avg neon avx2 msa sse2 vsx mmi/; +specialize qw/vpx_sad64x64_avg neon avx2 msa sse2 vsx mmi lsx/; add_proto qw/unsigned int vpx_sad64x32_avg/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred"; specialize qw/vpx_sad64x32_avg neon avx2 msa sse2 vsx mmi/; add_proto qw/unsigned int vpx_sad32x64_avg/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred"; specialize qw/vpx_sad32x64_avg neon avx2 msa sse2 vsx mmi/; add_proto qw/unsigned int vpx_sad32x32_avg/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred"; -specialize qw/vpx_sad32x32_avg neon avx2 msa sse2 vsx mmi/; +specialize qw/vpx_sad32x32_avg neon avx2 msa sse2 vsx mmi lsx/; add_proto qw/unsigned int vpx_sad32x16_avg/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred"; specialize qw/vpx_sad32x16_avg neon avx2 msa sse2 vsx mmi/; add_proto qw/unsigned int vpx_sad16x32_avg/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred"; @@ -872,87 +873,49 @@ add_proto qw/unsigned int vpx_sad4x4_avg/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred"; specialize qw/vpx_sad4x4_avg neon msa sse2 mmi/; # -# Multi-block SAD, comparing a reference to N blocks 1 pixel apart horizontally -# -# Blocks of 3 -add_proto qw/void vpx_sad16x16x3/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad16x16x3 sse3 ssse3 msa mmi/; - -add_proto qw/void vpx_sad16x8x3/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad16x8x3 sse3 ssse3 msa mmi/; - -add_proto qw/void vpx_sad8x16x3/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad8x16x3 sse3 msa mmi/; - -add_proto qw/void vpx_sad8x8x3/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad8x8x3 sse3 msa mmi/; - -add_proto qw/void vpx_sad4x4x3/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad4x4x3 sse3 msa mmi/; - -# Blocks of 8 -add_proto qw/void vpx_sad32x32x8/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad32x32x8 avx2/; - -add_proto qw/void vpx_sad16x16x8/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad16x16x8 sse4_1 msa mmi/; - -add_proto qw/void vpx_sad16x8x8/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad16x8x8 sse4_1 msa mmi/; - -add_proto qw/void vpx_sad8x16x8/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad8x16x8 sse4_1 msa mmi/; - -add_proto qw/void vpx_sad8x8x8/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad8x8x8 sse4_1 msa mmi/; - -add_proto qw/void vpx_sad4x4x8/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad4x4x8 sse4_1 msa mmi/; - -# # Multi-block SAD, comparing a reference to N independent blocks # -add_proto qw/void vpx_sad64x64x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad64x64x4d avx512 avx2 neon msa sse2 vsx mmi/; +add_proto qw/void vpx_sad64x64x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]"; +specialize qw/vpx_sad64x64x4d avx512 avx2 neon msa sse2 vsx mmi lsx/; -add_proto qw/void vpx_sad64x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad64x32x4d neon msa sse2 vsx mmi/; +add_proto qw/void vpx_sad64x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]"; +specialize qw/vpx_sad64x32x4d neon msa sse2 vsx mmi lsx/; -add_proto qw/void vpx_sad32x64x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad32x64x4d neon msa sse2 vsx mmi/; +add_proto qw/void vpx_sad32x64x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]"; +specialize qw/vpx_sad32x64x4d neon msa sse2 vsx mmi lsx/; -add_proto qw/void vpx_sad32x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad32x32x4d avx2 neon msa sse2 vsx mmi/; +add_proto qw/void vpx_sad32x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]"; +specialize qw/vpx_sad32x32x4d avx2 neon msa sse2 vsx mmi lsx/; -add_proto qw/void vpx_sad32x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array"; +add_proto qw/void vpx_sad32x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_sad32x16x4d neon msa sse2 vsx mmi/; -add_proto qw/void vpx_sad16x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array"; +add_proto qw/void vpx_sad16x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_sad16x32x4d neon msa sse2 vsx mmi/; -add_proto qw/void vpx_sad16x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad16x16x4d neon msa sse2 vsx mmi/; +add_proto qw/void vpx_sad16x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]"; +specialize qw/vpx_sad16x16x4d neon msa sse2 vsx mmi lsx/; -add_proto qw/void vpx_sad16x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array"; +add_proto qw/void vpx_sad16x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_sad16x8x4d neon msa sse2 vsx mmi/; -add_proto qw/void vpx_sad8x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array"; +add_proto qw/void vpx_sad8x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_sad8x16x4d neon msa sse2 mmi/; -add_proto qw/void vpx_sad8x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array"; -specialize qw/vpx_sad8x8x4d neon msa sse2 mmi/; +add_proto qw/void vpx_sad8x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]"; +specialize qw/vpx_sad8x8x4d neon msa sse2 mmi lsx/; -add_proto qw/void vpx_sad8x4x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array"; +add_proto qw/void vpx_sad8x4x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_sad8x4x4d neon msa sse2 mmi/; -add_proto qw/void vpx_sad4x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array"; +add_proto qw/void vpx_sad4x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_sad4x8x4d neon msa sse2 mmi/; -add_proto qw/void vpx_sad4x4x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[], int ref_stride, uint32_t *sad_array"; +add_proto qw/void vpx_sad4x4x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t * const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_sad4x4x4d neon msa sse2 mmi/; add_proto qw/uint64_t vpx_sum_squares_2d_i16/, "const int16_t *src, int stride, int size"; specialize qw/vpx_sum_squares_2d_i16 neon sse2 msa/; @@ -1062,47 +1025,47 @@ add_proto qw/unsigned int vpx_highbd_sad4x4_avg/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, const uint8_t *second_pred"; # # Multi-block SAD, comparing a reference to N independent blocks # - add_proto qw/void vpx_highbd_sad64x64x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array"; + add_proto qw/void vpx_highbd_sad64x64x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_highbd_sad64x64x4d sse2/; - add_proto qw/void vpx_highbd_sad64x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array"; + add_proto qw/void vpx_highbd_sad64x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_highbd_sad64x32x4d sse2/; - add_proto qw/void vpx_highbd_sad32x64x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array"; + add_proto qw/void vpx_highbd_sad32x64x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_highbd_sad32x64x4d sse2/; - add_proto qw/void vpx_highbd_sad32x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array"; + add_proto qw/void vpx_highbd_sad32x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_highbd_sad32x32x4d sse2/; - add_proto qw/void vpx_highbd_sad32x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array"; + add_proto qw/void vpx_highbd_sad32x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_highbd_sad32x16x4d sse2/; - add_proto qw/void vpx_highbd_sad16x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array"; + add_proto qw/void vpx_highbd_sad16x32x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_highbd_sad16x32x4d sse2/; - add_proto qw/void vpx_highbd_sad16x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array"; + add_proto qw/void vpx_highbd_sad16x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_highbd_sad16x16x4d sse2/; - add_proto qw/void vpx_highbd_sad16x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array"; + add_proto qw/void vpx_highbd_sad16x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_highbd_sad16x8x4d sse2/; - add_proto qw/void vpx_highbd_sad8x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array"; + add_proto qw/void vpx_highbd_sad8x16x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_highbd_sad8x16x4d sse2/; - add_proto qw/void vpx_highbd_sad8x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array"; + add_proto qw/void vpx_highbd_sad8x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_highbd_sad8x8x4d sse2/; - add_proto qw/void vpx_highbd_sad8x4x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array"; + add_proto qw/void vpx_highbd_sad8x4x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_highbd_sad8x4x4d sse2/; - add_proto qw/void vpx_highbd_sad4x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array"; + add_proto qw/void vpx_highbd_sad4x8x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_highbd_sad4x8x4d sse2/; - add_proto qw/void vpx_highbd_sad4x4x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[], int ref_stride, uint32_t *sad_array"; + add_proto qw/void vpx_highbd_sad4x4x4d/, "const uint8_t *src_ptr, int src_stride, const uint8_t* const ref_array[4], int ref_stride, uint32_t sad_array[4]"; specialize qw/vpx_highbd_sad4x4x4d sse2/; # # Structured Similarity (SSIM) # @@ -1116,38 +1079,38 @@ # # Variance # add_proto qw/unsigned int vpx_variance64x64/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse"; - specialize qw/vpx_variance64x64 sse2 avx2 neon msa mmi vsx/; + specialize qw/vpx_variance64x64 sse2 avx2 neon msa mmi vsx lsx/; add_proto qw/unsigned int vpx_variance64x32/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse"; specialize qw/vpx_variance64x32 sse2 avx2 neon msa mmi vsx/; add_proto qw/unsigned int vpx_variance32x64/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse"; specialize qw/vpx_variance32x64 sse2 avx2 neon msa mmi vsx/; add_proto qw/unsigned int vpx_variance32x32/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse"; - specialize qw/vpx_variance32x32 sse2 avx2 neon msa mmi vsx/; + specialize qw/vpx_variance32x32 sse2 avx2 neon msa mmi vsx lsx/; add_proto qw/unsigned int vpx_variance32x16/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse"; specialize qw/vpx_variance32x16 sse2 avx2 neon msa mmi vsx/; add_proto qw/unsigned int vpx_variance16x32/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse"; specialize qw/vpx_variance16x32 sse2 avx2 neon msa mmi vsx/; add_proto qw/unsigned int vpx_variance16x16/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse"; - specialize qw/vpx_variance16x16 sse2 avx2 neon msa mmi vsx/; + specialize qw/vpx_variance16x16 sse2 avx2 neon msa mmi vsx lsx/; add_proto qw/unsigned int vpx_variance16x8/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse"; specialize qw/vpx_variance16x8 sse2 avx2 neon msa mmi vsx/; add_proto qw/unsigned int vpx_variance8x16/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse"; specialize qw/vpx_variance8x16 sse2 neon msa mmi vsx/; add_proto qw/unsigned int vpx_variance8x8/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse"; - specialize qw/vpx_variance8x8 sse2 neon msa mmi vsx/; + specialize qw/vpx_variance8x8 sse2 neon msa mmi vsx lsx/; add_proto qw/unsigned int vpx_variance8x4/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse"; specialize qw/vpx_variance8x4 sse2 neon msa mmi vsx/; add_proto qw/unsigned int vpx_variance4x8/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse"; @@ -1158,11 +1121,11 @@ # # Specialty Variance # add_proto qw/void vpx_get16x16var/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse, int *sum"; - specialize qw/vpx_get16x16var sse2 avx2 neon msa vsx/; + specialize qw/vpx_get16x16var sse2 avx2 neon msa vsx lsx/; add_proto qw/void vpx_get8x8var/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse, int *sum"; specialize qw/vpx_get8x8var sse2 neon msa vsx/; add_proto qw/unsigned int vpx_mse16x16/, "const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, int ref_stride, unsigned int *sse"; @@ -1182,11 +1145,11 @@ add_proto qw/unsigned int vpx_get4x4sse_cs/, "const unsigned char *src_ptr, int src_stride, const unsigned char *ref_ptr, int ref_stride"; specialize qw/vpx_get4x4sse_cs neon msa vsx/; add_proto qw/void vpx_comp_avg_pred/, "uint8_t *comp_pred, const uint8_t *pred, int width, int height, const uint8_t *ref, int ref_stride"; - specialize qw/vpx_comp_avg_pred neon sse2 vsx/; + specialize qw/vpx_comp_avg_pred neon sse2 vsx lsx/; # # Subpixel Variance # add_proto qw/uint32_t vpx_sub_pixel_variance64x64/, "const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse"; @@ -1197,11 +1160,11 @@ add_proto qw/uint32_t vpx_sub_pixel_variance32x64/, "const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse"; specialize qw/vpx_sub_pixel_variance32x64 neon msa mmi sse2 ssse3/; add_proto qw/uint32_t vpx_sub_pixel_variance32x32/, "const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse"; - specialize qw/vpx_sub_pixel_variance32x32 avx2 neon msa mmi sse2 ssse3/; + specialize qw/vpx_sub_pixel_variance32x32 avx2 neon msa mmi sse2 ssse3 lsx/; add_proto qw/uint32_t vpx_sub_pixel_variance32x16/, "const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse"; specialize qw/vpx_sub_pixel_variance32x16 neon msa mmi sse2 ssse3/; add_proto qw/uint32_t vpx_sub_pixel_variance16x32/, "const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse"; @@ -1227,11 +1190,11 @@ add_proto qw/uint32_t vpx_sub_pixel_variance4x4/, "const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse"; specialize qw/vpx_sub_pixel_variance4x4 neon msa mmi sse2 ssse3/; add_proto qw/uint32_t vpx_sub_pixel_avg_variance64x64/, "const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred"; - specialize qw/vpx_sub_pixel_avg_variance64x64 neon avx2 msa mmi sse2 ssse3/; + specialize qw/vpx_sub_pixel_avg_variance64x64 neon avx2 msa mmi sse2 ssse3 lsx/; add_proto qw/uint32_t vpx_sub_pixel_avg_variance64x32/, "const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred"; specialize qw/vpx_sub_pixel_avg_variance64x32 neon msa mmi sse2 ssse3/; add_proto qw/uint32_t vpx_sub_pixel_avg_variance32x64/, "const uint8_t *src_ptr, int src_stride, int x_offset, int y_offset, const uint8_t *ref_ptr, int ref_stride, uint32_t *sse, const uint8_t *second_pred"; diff --git a/media/libvpx/libvpx/vpx_dsp/x86/avg_intrin_sse2.c b/media/libvpx/libvpx/vpx_dsp/x86/avg_intrin_sse2.c --- a/media/libvpx/libvpx/vpx_dsp/x86/avg_intrin_sse2.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/avg_intrin_sse2.c @@ -462,11 +462,11 @@ } return _mm_cvtsi128_si32(accum); } -void vpx_int_pro_row_sse2(int16_t *hbuf, const uint8_t *ref, +void vpx_int_pro_row_sse2(int16_t hbuf[16], const uint8_t *ref, const int ref_stride, const int height) { int idx; __m128i zero = _mm_setzero_si128(); __m128i src_line = _mm_loadu_si128((const __m128i *)ref); __m128i s0 = _mm_unpacklo_epi8(src_line, zero); diff --git a/media/libvpx/libvpx/vpx_dsp/x86/avg_pred_sse2.c b/media/libvpx/libvpx/vpx_dsp/x86/avg_pred_sse2.c --- a/media/libvpx/libvpx/vpx_dsp/x86/avg_pred_sse2.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/avg_pred_sse2.c @@ -44,13 +44,13 @@ __m128i avg; if (width == ref_stride) { r = _mm_loadu_si128((const __m128i *)ref); ref += 16; } else if (width == 4) { - r = _mm_set_epi32(loadu_uint32(ref + 3 * ref_stride), - loadu_uint32(ref + 2 * ref_stride), - loadu_uint32(ref + ref_stride), loadu_uint32(ref)); + r = _mm_set_epi32(loadu_int32(ref + 3 * ref_stride), + loadu_int32(ref + 2 * ref_stride), + loadu_int32(ref + ref_stride), loadu_int32(ref)); ref += 4 * ref_stride; } else { const __m128i r_0 = _mm_loadl_epi64((const __m128i *)ref); assert(width == 8); diff --git a/media/libvpx/libvpx/vpx_dsp/x86/highbd_convolve_avx2.c b/media/libvpx/libvpx/vpx_dsp/x86/highbd_convolve_avx2.c --- a/media/libvpx/libvpx/vpx_dsp/x86/highbd_convolve_avx2.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/highbd_convolve_avx2.c @@ -1463,14 +1463,14 @@ #define vpx_highbd_filter_block1d4_v4_avg_avx2 \ vpx_highbd_filter_block1d4_v8_avg_avx2 #define vpx_highbd_filter_block1d4_h4_avg_avx2 \ vpx_highbd_filter_block1d4_h8_avg_avx2 -HIGH_FUN_CONV_1D(horiz, x0_q4, x_step_q4, h, src, , avx2, 0); +HIGH_FUN_CONV_1D(horiz, x0_q4, x_step_q4, h, src, , avx2, 0) HIGH_FUN_CONV_1D(vert, y0_q4, y_step_q4, v, - src - src_stride * (num_taps / 2 - 1), , avx2, 0); -HIGH_FUN_CONV_2D(, avx2, 0); + src - src_stride * (num_taps / 2 - 1), , avx2, 0) +HIGH_FUN_CONV_2D(, avx2, 0) // From vpx_dsp/x86/vpx_high_subpixel_8t_sse2.asm. highbd_filter8_1dfunction vpx_highbd_filter_block1d4_h8_avg_sse2; highbd_filter8_1dfunction vpx_highbd_filter_block1d4_v8_avg_sse2; @@ -1485,11 +1485,11 @@ #define vpx_highbd_filter_block1d4_v8_avg_avx2 \ vpx_highbd_filter_block1d4_v8_avg_sse2 #define vpx_highbd_filter_block1d4_v2_avg_avx2 \ vpx_highbd_filter_block1d4_v2_avg_sse2 -HIGH_FUN_CONV_1D(avg_horiz, x0_q4, x_step_q4, h, src, avg_, avx2, 1); +HIGH_FUN_CONV_1D(avg_horiz, x0_q4, x_step_q4, h, src, avg_, avx2, 1) HIGH_FUN_CONV_1D(avg_vert, y0_q4, y_step_q4, v, - src - src_stride * (num_taps / 2 - 1), avg_, avx2, 1); -HIGH_FUN_CONV_2D(avg_, avx2, 1); + src - src_stride * (num_taps / 2 - 1), avg_, avx2, 1) +HIGH_FUN_CONV_2D(avg_, avx2, 1) #undef HIGHBD_FUNC diff --git a/media/libvpx/libvpx/vpx_dsp/x86/highbd_quantize_intrin_sse2.c b/media/libvpx/libvpx/vpx_dsp/x86/highbd_quantize_intrin_sse2.c --- a/media/libvpx/libvpx/vpx_dsp/x86/highbd_quantize_intrin_sse2.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/highbd_quantize_intrin_sse2.c @@ -16,11 +16,11 @@ #include "vpx_mem/vpx_mem.h" #include "vpx_ports/mem.h" #if CONFIG_VP9_HIGHBITDEPTH void vpx_highbd_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t count, - int skip_block, const int16_t *zbin_ptr, + const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, @@ -37,12 +37,10 @@ nzbins[1] = _mm_setzero_si128(); nzbins[0] = _mm_sub_epi32(nzbins[0], zbins[0]); nzbins[1] = _mm_sub_epi32(nzbins[1], zbins[1]); (void)scan; - (void)skip_block; - assert(!skip_block); memset(qcoeff_ptr, 0, count * sizeof(*qcoeff_ptr)); memset(dqcoeff_ptr, 0, count * sizeof(*dqcoeff_ptr)); // Pre-scan pass @@ -92,12 +90,12 @@ } *eob_ptr = eob_i + 1; } void vpx_highbd_quantize_b_32x32_sse2( - const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, - const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, + const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan) { __m128i zbins[2]; __m128i nzbins[2]; @@ -105,12 +103,10 @@ int idx_arr[1024]; int i, eob = -1; const int zbin0_tmp = ROUND_POWER_OF_TWO(zbin_ptr[0], 1); const int zbin1_tmp = ROUND_POWER_OF_TWO(zbin_ptr[1], 1); (void)scan; - (void)skip_block; - assert(!skip_block); zbins[0] = _mm_set_epi32(zbin1_tmp, zbin1_tmp, zbin1_tmp, zbin0_tmp); zbins[1] = _mm_set1_epi32(zbin1_tmp); nzbins[0] = _mm_setzero_si128(); diff --git a/media/libvpx/libvpx/vpx_dsp/x86/highbd_variance_sse2.c b/media/libvpx/libvpx/vpx_dsp/x86/highbd_variance_sse2.c --- a/media/libvpx/libvpx/vpx_dsp/x86/highbd_variance_sse2.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/highbd_variance_sse2.c @@ -119,12 +119,12 @@ sum); \ *sum = ROUND_POWER_OF_TWO(*sum, 4); \ *sse = ROUND_POWER_OF_TWO(*sse, 8); \ } -HIGH_GET_VAR(16); -HIGH_GET_VAR(8); +HIGH_GET_VAR(16) +HIGH_GET_VAR(8) #undef HIGH_GET_VAR #define VAR_FN(w, h, block_size, shift) \ uint32_t vpx_highbd_8_variance##w##x##h##_sse2( \ @@ -165,20 +165,20 @@ vpx_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ var = (int64_t)(*sse) - (((int64_t)sum * sum) >> (shift)); \ return (var >= 0) ? (uint32_t)var : 0; \ } -VAR_FN(64, 64, 16, 12); -VAR_FN(64, 32, 16, 11); -VAR_FN(32, 64, 16, 11); -VAR_FN(32, 32, 16, 10); -VAR_FN(32, 16, 16, 9); -VAR_FN(16, 32, 16, 9); -VAR_FN(16, 16, 16, 8); -VAR_FN(16, 8, 8, 7); -VAR_FN(8, 16, 8, 7); -VAR_FN(8, 8, 8, 6); +VAR_FN(64, 64, 16, 12) +VAR_FN(64, 32, 16, 11) +VAR_FN(32, 64, 16, 11) +VAR_FN(32, 32, 16, 10) +VAR_FN(32, 16, 16, 9) +VAR_FN(16, 32, 16, 9) +VAR_FN(16, 16, 16, 8) +VAR_FN(16, 8, 8, 7) +VAR_FN(8, 16, 8, 7) +VAR_FN(8, 8, 8, 6) #undef VAR_FN unsigned int vpx_highbd_8_mse16x16_sse2(const uint8_t *src8, int src_stride, const uint8_t *ref8, int ref_stride, @@ -253,14 +253,14 @@ int vpx_highbd_sub_pixel_variance##w##xh_##opt( \ const uint16_t *src, ptrdiff_t src_stride, int x_offset, int y_offset, \ const uint16_t *ref, ptrdiff_t ref_stride, int height, \ unsigned int *sse, void *unused0, void *unused); #define DECLS(opt) \ - DECL(8, opt); \ + DECL(8, opt) \ DECL(16, opt) -DECLS(sse2); +DECLS(sse2) #undef DECLS #undef DECL #define FN(w, h, wf, wlog2, hlog2, opt, cast) \ @@ -381,24 +381,24 @@ *sse_ptr = sse; \ var = (int64_t)(sse) - ((cast se * se) >> (wlog2 + hlog2)); \ return (var >= 0) ? (uint32_t)var : 0; \ } -#define FNS(opt) \ - FN(64, 64, 16, 6, 6, opt, (int64_t)); \ - FN(64, 32, 16, 6, 5, opt, (int64_t)); \ - FN(32, 64, 16, 5, 6, opt, (int64_t)); \ - FN(32, 32, 16, 5, 5, opt, (int64_t)); \ - FN(32, 16, 16, 5, 4, opt, (int64_t)); \ - FN(16, 32, 16, 4, 5, opt, (int64_t)); \ - FN(16, 16, 16, 4, 4, opt, (int64_t)); \ - FN(16, 8, 16, 4, 3, opt, (int64_t)); \ - FN(8, 16, 8, 3, 4, opt, (int64_t)); \ - FN(8, 8, 8, 3, 3, opt, (int64_t)); \ - FN(8, 4, 8, 3, 2, opt, (int64_t)); +#define FNS(opt) \ + FN(64, 64, 16, 6, 6, opt, (int64_t)) \ + FN(64, 32, 16, 6, 5, opt, (int64_t)) \ + FN(32, 64, 16, 5, 6, opt, (int64_t)) \ + FN(32, 32, 16, 5, 5, opt, (int64_t)) \ + FN(32, 16, 16, 5, 4, opt, (int64_t)) \ + FN(16, 32, 16, 4, 5, opt, (int64_t)) \ + FN(16, 16, 16, 4, 4, opt, (int64_t)) \ + FN(16, 8, 16, 4, 3, opt, (int64_t)) \ + FN(8, 16, 8, 3, 4, opt, (int64_t)) \ + FN(8, 8, 8, 3, 3, opt, (int64_t)) \ + FN(8, 4, 8, 3, 2, opt, (int64_t)) -FNS(sse2); +FNS(sse2) #undef FNS #undef FN // The 2 unused parameters are place holders for PIC enabled build. @@ -410,11 +410,11 @@ void *unused); #define DECLS(opt1) \ DECL(16, opt1) \ DECL(8, opt1) -DECLS(sse2); +DECLS(sse2) #undef DECL #undef DECLS #define FN(w, h, wf, wlog2, hlog2, opt, cast) \ uint32_t vpx_highbd_8_sub_pixel_avg_variance##w##x##h##_##opt( \ @@ -540,22 +540,22 @@ *sse_ptr = sse; \ var = (int64_t)(sse) - ((cast se * se) >> (wlog2 + hlog2)); \ return (var >= 0) ? (uint32_t)var : 0; \ } -#define FNS(opt1) \ - FN(64, 64, 16, 6, 6, opt1, (int64_t)); \ - FN(64, 32, 16, 6, 5, opt1, (int64_t)); \ - FN(32, 64, 16, 5, 6, opt1, (int64_t)); \ - FN(32, 32, 16, 5, 5, opt1, (int64_t)); \ - FN(32, 16, 16, 5, 4, opt1, (int64_t)); \ - FN(16, 32, 16, 4, 5, opt1, (int64_t)); \ - FN(16, 16, 16, 4, 4, opt1, (int64_t)); \ - FN(16, 8, 16, 4, 3, opt1, (int64_t)); \ - FN(8, 16, 8, 4, 3, opt1, (int64_t)); \ - FN(8, 8, 8, 3, 3, opt1, (int64_t)); \ - FN(8, 4, 8, 3, 2, opt1, (int64_t)); +#define FNS(opt1) \ + FN(64, 64, 16, 6, 6, opt1, (int64_t)) \ + FN(64, 32, 16, 6, 5, opt1, (int64_t)) \ + FN(32, 64, 16, 5, 6, opt1, (int64_t)) \ + FN(32, 32, 16, 5, 5, opt1, (int64_t)) \ + FN(32, 16, 16, 5, 4, opt1, (int64_t)) \ + FN(16, 32, 16, 4, 5, opt1, (int64_t)) \ + FN(16, 16, 16, 4, 4, opt1, (int64_t)) \ + FN(16, 8, 16, 4, 3, opt1, (int64_t)) \ + FN(8, 16, 8, 4, 3, opt1, (int64_t)) \ + FN(8, 8, 8, 3, 3, opt1, (int64_t)) \ + FN(8, 4, 8, 3, 2, opt1, (int64_t)) -FNS(sse2); +FNS(sse2) #undef FNS #undef FN diff --git a/media/libvpx/libvpx/vpx_dsp/x86/loopfilter_intrin_sse2.c b/media/libvpx/libvpx/vpx_dsp/x86/loopfilter_intrin_sse2.c --- a/media/libvpx/libvpx/vpx_dsp/x86/loopfilter_intrin_sse2.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/loopfilter_intrin_sse2.c @@ -209,25 +209,25 @@ // 04 14 24 34 05 15 25 35 06 16 26 36 07 17 27 37 qs1qs0 = _mm_unpackhi_epi8(ps1ps0, x0); // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33 ps1ps0 = _mm_unpacklo_epi8(ps1ps0, x0); - storeu_uint32(s + 0 * pitch - 2, _mm_cvtsi128_si32(ps1ps0)); + storeu_int32(s + 0 * pitch - 2, _mm_cvtsi128_si32(ps1ps0)); ps1ps0 = _mm_srli_si128(ps1ps0, 4); - storeu_uint32(s + 1 * pitch - 2, _mm_cvtsi128_si32(ps1ps0)); + storeu_int32(s + 1 * pitch - 2, _mm_cvtsi128_si32(ps1ps0)); ps1ps0 = _mm_srli_si128(ps1ps0, 4); - storeu_uint32(s + 2 * pitch - 2, _mm_cvtsi128_si32(ps1ps0)); + storeu_int32(s + 2 * pitch - 2, _mm_cvtsi128_si32(ps1ps0)); ps1ps0 = _mm_srli_si128(ps1ps0, 4); - storeu_uint32(s + 3 * pitch - 2, _mm_cvtsi128_si32(ps1ps0)); + storeu_int32(s + 3 * pitch - 2, _mm_cvtsi128_si32(ps1ps0)); - storeu_uint32(s + 4 * pitch - 2, _mm_cvtsi128_si32(qs1qs0)); + storeu_int32(s + 4 * pitch - 2, _mm_cvtsi128_si32(qs1qs0)); qs1qs0 = _mm_srli_si128(qs1qs0, 4); - storeu_uint32(s + 5 * pitch - 2, _mm_cvtsi128_si32(qs1qs0)); + storeu_int32(s + 5 * pitch - 2, _mm_cvtsi128_si32(qs1qs0)); qs1qs0 = _mm_srli_si128(qs1qs0, 4); - storeu_uint32(s + 6 * pitch - 2, _mm_cvtsi128_si32(qs1qs0)); + storeu_int32(s + 6 * pitch - 2, _mm_cvtsi128_si32(qs1qs0)); qs1qs0 = _mm_srli_si128(qs1qs0, 4); - storeu_uint32(s + 7 * pitch - 2, _mm_cvtsi128_si32(qs1qs0)); + storeu_int32(s + 7 * pitch - 2, _mm_cvtsi128_si32(qs1qs0)); } void vpx_lpf_horizontal_16_sse2(unsigned char *s, int pitch, const unsigned char *blimit, const unsigned char *limit, diff --git a/media/libvpx/libvpx/vpx_dsp/x86/mem_sse2.h b/media/libvpx/libvpx/vpx_dsp/x86/mem_sse2.h --- a/media/libvpx/libvpx/vpx_dsp/x86/mem_sse2.h +++ b/media/libvpx/libvpx/vpx_dsp/x86/mem_sse2.h @@ -14,16 +14,16 @@ #include // SSE2 #include #include "./vpx_config.h" -static INLINE void storeu_uint32(void *dst, uint32_t v) { +static INLINE void storeu_int32(void *dst, int32_t v) { memcpy(dst, &v, sizeof(v)); } -static INLINE uint32_t loadu_uint32(const void *src) { - uint32_t v; +static INLINE int32_t loadu_int32(const void *src) { + int32_t v; memcpy(&v, src, sizeof(v)); return v; } static INLINE __m128i load_unaligned_u32(const void *a) { diff --git a/media/libvpx/libvpx/vpx_dsp/x86/quantize_avx.c b/media/libvpx/libvpx/vpx_dsp/x86/quantize_avx.c --- a/media/libvpx/libvpx/vpx_dsp/x86/quantize_avx.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/quantize_avx.c @@ -19,12 +19,12 @@ #include "vpx_dsp/x86/bitdepth_conversion_sse2.h" #include "vpx_dsp/x86/quantize_sse2.h" #include "vpx_dsp/x86/quantize_ssse3.h" void vpx_quantize_b_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *zbin_ptr, - const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *zbin_ptr, const int16_t *round_ptr, + const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan) { const __m128i zero = _mm_setzero_si128(); @@ -37,12 +37,10 @@ __m128i cmp_mask0, cmp_mask1; __m128i all_zero; __m128i eob = zero, eob0; (void)scan; - (void)skip_block; - assert(!skip_block); *eob_ptr = 0; load_b_values(zbin_ptr, &zbin, round_ptr, &round, quant_ptr, &quant, dequant_ptr, &dequant, quant_shift_ptr, &shift); @@ -143,12 +141,11 @@ *eob_ptr = accumulate_eob(eob); } void vpx_quantize_b_32x32_avx(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *zbin_ptr, - const int16_t *round_ptr, + const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan) { @@ -164,12 +161,10 @@ __m128i all_zero; __m128i eob = zero, eob0; (void)scan; (void)n_coeffs; - (void)skip_block; - assert(!skip_block); // Setup global values. // The 32x32 halves zbin and round. zbin = _mm_load_si128((const __m128i *)zbin_ptr); // Shift with rounding. diff --git a/media/libvpx/libvpx/vpx_dsp/x86/quantize_sse2.c b/media/libvpx/libvpx/vpx_dsp/x86/quantize_sse2.c --- a/media/libvpx/libvpx/vpx_dsp/x86/quantize_sse2.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/quantize_sse2.c @@ -16,12 +16,12 @@ #include "vpx/vpx_integer.h" #include "vpx_dsp/x86/bitdepth_conversion_sse2.h" #include "vpx_dsp/x86/quantize_sse2.h" void vpx_quantize_b_sse2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *zbin_ptr, - const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *zbin_ptr, const int16_t *round_ptr, + const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan) { const __m128i zero = _mm_setzero_si128(); @@ -32,12 +32,10 @@ __m128i qcoeff0, qcoeff1; __m128i cmp_mask0, cmp_mask1; __m128i eob, eob0; (void)scan; - (void)skip_block; - assert(!skip_block); // Setup global values. load_b_values(zbin_ptr, &zbin, round_ptr, &round, quant_ptr, &quant, dequant_ptr, &dequant, quant_shift_ptr, &shift); diff --git a/media/libvpx/libvpx/vpx_dsp/x86/quantize_ssse3.c b/media/libvpx/libvpx/vpx_dsp/x86/quantize_ssse3.c --- a/media/libvpx/libvpx/vpx_dsp/x86/quantize_ssse3.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/quantize_ssse3.c @@ -16,12 +16,12 @@ #include "vpx_dsp/x86/bitdepth_conversion_sse2.h" #include "vpx_dsp/x86/quantize_sse2.h" #include "vpx_dsp/x86/quantize_ssse3.h" void vpx_quantize_b_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *zbin_ptr, - const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *zbin_ptr, const int16_t *round_ptr, + const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan) { const __m128i zero = _mm_setzero_si128(); @@ -32,12 +32,10 @@ __m128i qcoeff0, qcoeff1; __m128i cmp_mask0, cmp_mask1; __m128i eob, eob0; (void)scan; - (void)skip_block; - assert(!skip_block); load_b_values(zbin_ptr, &zbin, round_ptr, &round, quant_ptr, &quant, dequant_ptr, &dequant, quant_shift_ptr, &shift); // Do DC and first 15 AC. @@ -109,11 +107,11 @@ *eob_ptr = accumulate_eob(eob); } void vpx_quantize_b_32x32_ssse3(const tran_low_t *coeff_ptr, intptr_t n_coeffs, - int skip_block, const int16_t *zbin_ptr, + const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, @@ -129,12 +127,10 @@ __m128i all_zero; __m128i eob = zero, eob0; (void)scan; (void)n_coeffs; - (void)skip_block; - assert(!skip_block); // Setup global values. // The 32x32 halves zbin and round. zbin = _mm_load_si128((const __m128i *)zbin_ptr); // Shift with rounding. diff --git a/media/libvpx/libvpx/vpx_dsp/x86/sad4d_avx2.c b/media/libvpx/libvpx/vpx_dsp/x86/sad4d_avx2.c --- a/media/libvpx/libvpx/vpx_dsp/x86/sad4d_avx2.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/sad4d_avx2.c @@ -9,12 +9,16 @@ */ #include // AVX2 #include "./vpx_dsp_rtcd.h" #include "vpx/vpx_integer.h" +// Note with sums[4] some versions of Visual Studio may fail due to parameter +// alignment, though the functions should be equivalent: +// error C2719: 'sums': formal parameter with requested alignment of 32 won't be +// aligned static INLINE void calc_final_4(const __m256i *const sums /*[4]*/, - uint32_t *sad_array) { + uint32_t sad_array[4]) { const __m256i t0 = _mm256_hadd_epi32(sums[0], sums[1]); const __m256i t1 = _mm256_hadd_epi32(sums[2], sums[3]); const __m256i t2 = _mm256_hadd_epi32(t0, t1); const __m128i sum = _mm_add_epi32(_mm256_castsi256_si128(t2), _mm256_extractf128_si256(t2, 1)); @@ -67,67 +71,10 @@ } calc_final_4(sums, sad_array); } -void vpx_sad32x32x8_avx2(const uint8_t *src_ptr, int src_stride, - const uint8_t *ref_ptr, int ref_stride, - uint32_t *sad_array) { - int i; - __m256i sums[8]; - - sums[0] = _mm256_setzero_si256(); - sums[1] = _mm256_setzero_si256(); - sums[2] = _mm256_setzero_si256(); - sums[3] = _mm256_setzero_si256(); - sums[4] = _mm256_setzero_si256(); - sums[5] = _mm256_setzero_si256(); - sums[6] = _mm256_setzero_si256(); - sums[7] = _mm256_setzero_si256(); - - for (i = 0; i < 32; i++) { - __m256i r[8]; - - // load src and all ref[] - const __m256i s = _mm256_load_si256((const __m256i *)src_ptr); - r[0] = _mm256_loadu_si256((const __m256i *)&ref_ptr[0]); - r[1] = _mm256_loadu_si256((const __m256i *)&ref_ptr[1]); - r[2] = _mm256_loadu_si256((const __m256i *)&ref_ptr[2]); - r[3] = _mm256_loadu_si256((const __m256i *)&ref_ptr[3]); - r[4] = _mm256_loadu_si256((const __m256i *)&ref_ptr[4]); - r[5] = _mm256_loadu_si256((const __m256i *)&ref_ptr[5]); - r[6] = _mm256_loadu_si256((const __m256i *)&ref_ptr[6]); - r[7] = _mm256_loadu_si256((const __m256i *)&ref_ptr[7]); - - // sum of the absolute differences between every ref[] to src - r[0] = _mm256_sad_epu8(r[0], s); - r[1] = _mm256_sad_epu8(r[1], s); - r[2] = _mm256_sad_epu8(r[2], s); - r[3] = _mm256_sad_epu8(r[3], s); - r[4] = _mm256_sad_epu8(r[4], s); - r[5] = _mm256_sad_epu8(r[5], s); - r[6] = _mm256_sad_epu8(r[6], s); - r[7] = _mm256_sad_epu8(r[7], s); - - // sum every ref[] - sums[0] = _mm256_add_epi32(sums[0], r[0]); - sums[1] = _mm256_add_epi32(sums[1], r[1]); - sums[2] = _mm256_add_epi32(sums[2], r[2]); - sums[3] = _mm256_add_epi32(sums[3], r[3]); - sums[4] = _mm256_add_epi32(sums[4], r[4]); - sums[5] = _mm256_add_epi32(sums[5], r[5]); - sums[6] = _mm256_add_epi32(sums[6], r[6]); - sums[7] = _mm256_add_epi32(sums[7], r[7]); - - src_ptr += src_stride; - ref_ptr += ref_stride; - } - - calc_final_4(sums, sad_array); - calc_final_4(sums + 4, sad_array + 4); -} - void vpx_sad64x64x4d_avx2(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, uint32_t sad_array[4]) { __m256i sums[4]; int i; diff --git a/media/libvpx/libvpx/vpx_dsp/x86/sad4d_avx512.c b/media/libvpx/libvpx/vpx_dsp/x86/sad4d_avx512.c --- a/media/libvpx/libvpx/vpx_dsp/x86/sad4d_avx512.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/sad4d_avx512.c @@ -11,11 +11,11 @@ #include "./vpx_dsp_rtcd.h" #include "vpx/vpx_integer.h" void vpx_sad64x64x4d_avx512(const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], int ref_stride, - uint32_t res[4]) { + uint32_t sad_array[4]) { __m512i src_reg, ref0_reg, ref1_reg, ref2_reg, ref3_reg; __m512i sum_ref0, sum_ref1, sum_ref2, sum_ref3; __m512i sum_mlow, sum_mhigh; int i; const uint8_t *ref0, *ref1, *ref2, *ref3; @@ -76,8 +76,8 @@ sum256 = _mm256_add_epi32(_mm512_castsi512_si256(sum_mlow), _mm512_extracti32x8_epi32(sum_mlow, 1)); sum128 = _mm_add_epi32(_mm256_castsi256_si128(sum256), _mm256_extractf128_si256(sum256, 1)); - _mm_storeu_si128((__m128i *)(res), sum128); + _mm_storeu_si128((__m128i *)(sad_array), sum128); } } diff --git a/media/libvpx/libvpx/vpx_dsp/x86/sad_avx2.c b/media/libvpx/libvpx/vpx_dsp/x86/sad_avx2.c --- a/media/libvpx/libvpx/vpx_dsp/x86/sad_avx2.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/sad_avx2.c @@ -69,21 +69,21 @@ sum_sad128 = _mm_add_epi32(_mm256_castsi256_si128(sum_sad), sum_sad128); \ res = _mm_cvtsi128_si32(sum_sad128); \ return res; \ } -#define FSAD64 \ - FSAD64_H(64); \ - FSAD64_H(32); +#define FSAD64 \ + FSAD64_H(64) \ + FSAD64_H(32) -#define FSAD32 \ - FSAD32_H(64); \ - FSAD32_H(32); \ - FSAD32_H(16); +#define FSAD32 \ + FSAD32_H(64) \ + FSAD32_H(32) \ + FSAD32_H(16) -FSAD64; -FSAD32; +FSAD64 +FSAD32 #undef FSAD64 #undef FSAD32 #undef FSAD64_H #undef FSAD32_H @@ -158,21 +158,21 @@ sum_sad128 = _mm_add_epi32(_mm256_castsi256_si128(sum_sad), sum_sad128); \ res = _mm_cvtsi128_si32(sum_sad128); \ return res; \ } -#define FSADAVG64 \ - FSADAVG64_H(64); \ - FSADAVG64_H(32); +#define FSADAVG64 \ + FSADAVG64_H(64) \ + FSADAVG64_H(32) -#define FSADAVG32 \ - FSADAVG32_H(64); \ - FSADAVG32_H(32); \ - FSADAVG32_H(16); +#define FSADAVG32 \ + FSADAVG32_H(64) \ + FSADAVG32_H(32) \ + FSADAVG32_H(16) -FSADAVG64; -FSADAVG32; +FSADAVG64 +FSADAVG32 #undef FSADAVG64 #undef FSADAVG32 #undef FSADAVG64_H #undef FSADAVG32_H diff --git a/media/libvpx/libvpx/vpx_dsp/x86/sad_sse3.asm b/media/libvpx/libvpx/vpx_dsp/x86/sad_sse3.asm deleted file mode 100644 --- a/media/libvpx/libvpx/vpx_dsp/x86/sad_sse3.asm +++ /dev/null @@ -1,376 +0,0 @@ -; -; Copyright (c) 2010 The WebM project authors. All Rights Reserved. -; -; Use of this source code is governed by a BSD-style license -; that can be found in the LICENSE file in the root of the source -; tree. An additional intellectual property rights grant can be found -; in the file PATENTS. All contributing project authors may -; be found in the AUTHORS file in the root of the source tree. -; - -%include "vpx_ports/x86_abi_support.asm" - -%macro STACK_FRAME_CREATE_X3 0 -%if ABI_IS_32BIT - %define src_ptr rsi - %define src_stride rax - %define ref_ptr rdi - %define ref_stride rdx - %define end_ptr rcx - %define ret_var rbx - %define result_ptr arg(4) - %define height dword ptr arg(4) - push rbp - mov rbp, rsp - push rsi - push rdi - push rbx - - mov rsi, arg(0) ; src_ptr - mov rdi, arg(2) ; ref_ptr - - movsxd rax, dword ptr arg(1) ; src_stride - movsxd rdx, dword ptr arg(3) ; ref_stride -%else - %if LIBVPX_YASM_WIN64 - SAVE_XMM 7, u - %define src_ptr rcx - %define src_stride rdx - %define ref_ptr r8 - %define ref_stride r9 - %define end_ptr r10 - %define ret_var r11 - %define result_ptr [rsp+xmm_stack_space+8+4*8] - %define height dword ptr [rsp+xmm_stack_space+8+4*8] - %else - %define src_ptr rdi - %define src_stride rsi - %define ref_ptr rdx - %define ref_stride rcx - %define end_ptr r9 - %define ret_var r10 - %define result_ptr r8 - %define height r8 - %endif -%endif - -%endmacro - -%macro STACK_FRAME_DESTROY_X3 0 - %define src_ptr - %define src_stride - %define ref_ptr - %define ref_stride - %define end_ptr - %define ret_var - %define result_ptr - %define height - -%if ABI_IS_32BIT - pop rbx - pop rdi - pop rsi - pop rbp -%else - %if LIBVPX_YASM_WIN64 - RESTORE_XMM - %endif -%endif - ret -%endmacro - -%macro PROCESS_16X2X3 5 -%if %1==0 - movdqa xmm0, XMMWORD PTR [%2] - lddqu xmm5, XMMWORD PTR [%3] - lddqu xmm6, XMMWORD PTR [%3+1] - lddqu xmm7, XMMWORD PTR [%3+2] - - psadbw xmm5, xmm0 - psadbw xmm6, xmm0 - psadbw xmm7, xmm0 -%else - movdqa xmm0, XMMWORD PTR [%2] - lddqu xmm1, XMMWORD PTR [%3] - lddqu xmm2, XMMWORD PTR [%3+1] - lddqu xmm3, XMMWORD PTR [%3+2] - - psadbw xmm1, xmm0 - psadbw xmm2, xmm0 - psadbw xmm3, xmm0 - - paddw xmm5, xmm1 - paddw xmm6, xmm2 - paddw xmm7, xmm3 -%endif - movdqa xmm0, XMMWORD PTR [%2+%4] - lddqu xmm1, XMMWORD PTR [%3+%5] - lddqu xmm2, XMMWORD PTR [%3+%5+1] - lddqu xmm3, XMMWORD PTR [%3+%5+2] - -%if %1==0 || %1==1 - lea %2, [%2+%4*2] - lea %3, [%3+%5*2] -%endif - - psadbw xmm1, xmm0 - psadbw xmm2, xmm0 - psadbw xmm3, xmm0 - - paddw xmm5, xmm1 - paddw xmm6, xmm2 - paddw xmm7, xmm3 -%endmacro - -%macro PROCESS_8X2X3 5 -%if %1==0 - movq mm0, QWORD PTR [%2] - movq mm5, QWORD PTR [%3] - movq mm6, QWORD PTR [%3+1] - movq mm7, QWORD PTR [%3+2] - - psadbw mm5, mm0 - psadbw mm6, mm0 - psadbw mm7, mm0 -%else - movq mm0, QWORD PTR [%2] - movq mm1, QWORD PTR [%3] - movq mm2, QWORD PTR [%3+1] - movq mm3, QWORD PTR [%3+2] - - psadbw mm1, mm0 - psadbw mm2, mm0 - psadbw mm3, mm0 - - paddw mm5, mm1 - paddw mm6, mm2 - paddw mm7, mm3 -%endif - movq mm0, QWORD PTR [%2+%4] - movq mm1, QWORD PTR [%3+%5] - movq mm2, QWORD PTR [%3+%5+1] - movq mm3, QWORD PTR [%3+%5+2] - -%if %1==0 || %1==1 - lea %2, [%2+%4*2] - lea %3, [%3+%5*2] -%endif - - psadbw mm1, mm0 - psadbw mm2, mm0 - psadbw mm3, mm0 - - paddw mm5, mm1 - paddw mm6, mm2 - paddw mm7, mm3 -%endmacro - -SECTION .text - -;void int vpx_sad16x16x3_sse3( -; unsigned char *src_ptr, -; int src_stride, -; unsigned char *ref_ptr, -; int ref_stride, -; int *results) -globalsym(vpx_sad16x16x3_sse3) -sym(vpx_sad16x16x3_sse3): - - STACK_FRAME_CREATE_X3 - - PROCESS_16X2X3 0, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_16X2X3 2, src_ptr, ref_ptr, src_stride, ref_stride - - mov rcx, result_ptr - - movq xmm0, xmm5 - psrldq xmm5, 8 - - paddw xmm0, xmm5 - movd [rcx], xmm0 -;- - movq xmm0, xmm6 - psrldq xmm6, 8 - - paddw xmm0, xmm6 - movd [rcx+4], xmm0 -;- - movq xmm0, xmm7 - psrldq xmm7, 8 - - paddw xmm0, xmm7 - movd [rcx+8], xmm0 - - STACK_FRAME_DESTROY_X3 - -;void int vpx_sad16x8x3_sse3( -; unsigned char *src_ptr, -; int src_stride, -; unsigned char *ref_ptr, -; int ref_stride, -; int *results) -globalsym(vpx_sad16x8x3_sse3) -sym(vpx_sad16x8x3_sse3): - - STACK_FRAME_CREATE_X3 - - PROCESS_16X2X3 0, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_16X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_16X2X3 2, src_ptr, ref_ptr, src_stride, ref_stride - - mov rcx, result_ptr - - movq xmm0, xmm5 - psrldq xmm5, 8 - - paddw xmm0, xmm5 - movd [rcx], xmm0 -;- - movq xmm0, xmm6 - psrldq xmm6, 8 - - paddw xmm0, xmm6 - movd [rcx+4], xmm0 -;- - movq xmm0, xmm7 - psrldq xmm7, 8 - - paddw xmm0, xmm7 - movd [rcx+8], xmm0 - - STACK_FRAME_DESTROY_X3 - -;void int vpx_sad8x16x3_sse3( -; unsigned char *src_ptr, -; int src_stride, -; unsigned char *ref_ptr, -; int ref_stride, -; int *results) -globalsym(vpx_sad8x16x3_sse3) -sym(vpx_sad8x16x3_sse3): - - STACK_FRAME_CREATE_X3 - - PROCESS_8X2X3 0, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_8X2X3 2, src_ptr, ref_ptr, src_stride, ref_stride - - mov rcx, result_ptr - - punpckldq mm5, mm6 - - movq [rcx], mm5 - movd [rcx+8], mm7 - - STACK_FRAME_DESTROY_X3 - -;void int vpx_sad8x8x3_sse3( -; unsigned char *src_ptr, -; int src_stride, -; unsigned char *ref_ptr, -; int ref_stride, -; int *results) -globalsym(vpx_sad8x8x3_sse3) -sym(vpx_sad8x8x3_sse3): - - STACK_FRAME_CREATE_X3 - - PROCESS_8X2X3 0, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_8X2X3 1, src_ptr, ref_ptr, src_stride, ref_stride - PROCESS_8X2X3 2, src_ptr, ref_ptr, src_stride, ref_stride - - mov rcx, result_ptr - - punpckldq mm5, mm6 - - movq [rcx], mm5 - movd [rcx+8], mm7 - - STACK_FRAME_DESTROY_X3 - -;void int vpx_sad4x4x3_sse3( -; unsigned char *src_ptr, -; int src_stride, -; unsigned char *ref_ptr, -; int ref_stride, -; int *results) -globalsym(vpx_sad4x4x3_sse3) -sym(vpx_sad4x4x3_sse3): - - STACK_FRAME_CREATE_X3 - - movd mm0, DWORD PTR [src_ptr] - movd mm1, DWORD PTR [ref_ptr] - - movd mm2, DWORD PTR [src_ptr+src_stride] - movd mm3, DWORD PTR [ref_ptr+ref_stride] - - punpcklbw mm0, mm2 - punpcklbw mm1, mm3 - - movd mm4, DWORD PTR [ref_ptr+1] - movd mm5, DWORD PTR [ref_ptr+2] - - movd mm2, DWORD PTR [ref_ptr+ref_stride+1] - movd mm3, DWORD PTR [ref_ptr+ref_stride+2] - - psadbw mm1, mm0 - - punpcklbw mm4, mm2 - punpcklbw mm5, mm3 - - psadbw mm4, mm0 - psadbw mm5, mm0 - - lea src_ptr, [src_ptr+src_stride*2] - lea ref_ptr, [ref_ptr+ref_stride*2] - - movd mm0, DWORD PTR [src_ptr] - movd mm2, DWORD PTR [ref_ptr] - - movd mm3, DWORD PTR [src_ptr+src_stride] - movd mm6, DWORD PTR [ref_ptr+ref_stride] - - punpcklbw mm0, mm3 - punpcklbw mm2, mm6 - - movd mm3, DWORD PTR [ref_ptr+1] - movd mm7, DWORD PTR [ref_ptr+2] - - psadbw mm2, mm0 - - paddw mm1, mm2 - - movd mm2, DWORD PTR [ref_ptr+ref_stride+1] - movd mm6, DWORD PTR [ref_ptr+ref_stride+2] - - punpcklbw mm3, mm2 - punpcklbw mm7, mm6 - - psadbw mm3, mm0 - psadbw mm7, mm0 - - paddw mm3, mm4 - paddw mm7, mm5 - - mov rcx, result_ptr - - punpckldq mm1, mm3 - - movq [rcx], mm1 - movd [rcx+8], mm7 - - STACK_FRAME_DESTROY_X3 diff --git a/media/libvpx/libvpx/vpx_dsp/x86/sad_sse4.asm b/media/libvpx/libvpx/vpx_dsp/x86/sad_sse4.asm deleted file mode 100644 --- a/media/libvpx/libvpx/vpx_dsp/x86/sad_sse4.asm +++ /dev/null @@ -1,361 +0,0 @@ -; -; Copyright (c) 2010 The WebM project authors. All Rights Reserved. -; -; Use of this source code is governed by a BSD-style license -; that can be found in the LICENSE file in the root of the source -; tree. An additional intellectual property rights grant can be found -; in the file PATENTS. All contributing project authors may -; be found in the AUTHORS file in the root of the source tree. -; - - -%include "vpx_ports/x86_abi_support.asm" - -%macro PROCESS_16X2X8 1 -%if %1 - movdqa xmm0, XMMWORD PTR [rsi] - movq xmm1, MMWORD PTR [rdi] - movq xmm3, MMWORD PTR [rdi+8] - movq xmm2, MMWORD PTR [rdi+16] - punpcklqdq xmm1, xmm3 - punpcklqdq xmm3, xmm2 - - movdqa xmm2, xmm1 - mpsadbw xmm1, xmm0, 0x0 - mpsadbw xmm2, xmm0, 0x5 - - psrldq xmm0, 8 - - movdqa xmm4, xmm3 - mpsadbw xmm3, xmm0, 0x0 - mpsadbw xmm4, xmm0, 0x5 - - paddw xmm1, xmm2 - paddw xmm1, xmm3 - paddw xmm1, xmm4 -%else - movdqa xmm0, XMMWORD PTR [rsi] - movq xmm5, MMWORD PTR [rdi] - movq xmm3, MMWORD PTR [rdi+8] - movq xmm2, MMWORD PTR [rdi+16] - punpcklqdq xmm5, xmm3 - punpcklqdq xmm3, xmm2 - - movdqa xmm2, xmm5 - mpsadbw xmm5, xmm0, 0x0 - mpsadbw xmm2, xmm0, 0x5 - - psrldq xmm0, 8 - - movdqa xmm4, xmm3 - mpsadbw xmm3, xmm0, 0x0 - mpsadbw xmm4, xmm0, 0x5 - - paddw xmm5, xmm2 - paddw xmm5, xmm3 - paddw xmm5, xmm4 - - paddw xmm1, xmm5 -%endif - movdqa xmm0, XMMWORD PTR [rsi + rax] - movq xmm5, MMWORD PTR [rdi+ rdx] - movq xmm3, MMWORD PTR [rdi+ rdx+8] - movq xmm2, MMWORD PTR [rdi+ rdx+16] - punpcklqdq xmm5, xmm3 - punpcklqdq xmm3, xmm2 - - lea rsi, [rsi+rax*2] - lea rdi, [rdi+rdx*2] - - movdqa xmm2, xmm5 - mpsadbw xmm5, xmm0, 0x0 - mpsadbw xmm2, xmm0, 0x5 - - psrldq xmm0, 8 - movdqa xmm4, xmm3 - mpsadbw xmm3, xmm0, 0x0 - mpsadbw xmm4, xmm0, 0x5 - - paddw xmm5, xmm2 - paddw xmm5, xmm3 - paddw xmm5, xmm4 - - paddw xmm1, xmm5 -%endmacro - -%macro PROCESS_8X2X8 1 -%if %1 - movq xmm0, MMWORD PTR [rsi] - movq xmm1, MMWORD PTR [rdi] - movq xmm3, MMWORD PTR [rdi+8] - punpcklqdq xmm1, xmm3 - - movdqa xmm2, xmm1 - mpsadbw xmm1, xmm0, 0x0 - mpsadbw xmm2, xmm0, 0x5 - paddw xmm1, xmm2 -%else - movq xmm0, MMWORD PTR [rsi] - movq xmm5, MMWORD PTR [rdi] - movq xmm3, MMWORD PTR [rdi+8] - punpcklqdq xmm5, xmm3 - - movdqa xmm2, xmm5 - mpsadbw xmm5, xmm0, 0x0 - mpsadbw xmm2, xmm0, 0x5 - paddw xmm5, xmm2 - - paddw xmm1, xmm5 -%endif - movq xmm0, MMWORD PTR [rsi + rax] - movq xmm5, MMWORD PTR [rdi+ rdx] - movq xmm3, MMWORD PTR [rdi+ rdx+8] - punpcklqdq xmm5, xmm3 - - lea rsi, [rsi+rax*2] - lea rdi, [rdi+rdx*2] - - movdqa xmm2, xmm5 - mpsadbw xmm5, xmm0, 0x0 - mpsadbw xmm2, xmm0, 0x5 - paddw xmm5, xmm2 - - paddw xmm1, xmm5 -%endmacro - -%macro PROCESS_4X2X8 1 -%if %1 - movd xmm0, [rsi] - movq xmm1, MMWORD PTR [rdi] - movq xmm3, MMWORD PTR [rdi+8] - punpcklqdq xmm1, xmm3 - - mpsadbw xmm1, xmm0, 0x0 -%else - movd xmm0, [rsi] - movq xmm5, MMWORD PTR [rdi] - movq xmm3, MMWORD PTR [rdi+8] - punpcklqdq xmm5, xmm3 - - mpsadbw xmm5, xmm0, 0x0 - - paddw xmm1, xmm5 -%endif - movd xmm0, [rsi + rax] - movq xmm5, MMWORD PTR [rdi+ rdx] - movq xmm3, MMWORD PTR [rdi+ rdx+8] - punpcklqdq xmm5, xmm3 - - lea rsi, [rsi+rax*2] - lea rdi, [rdi+rdx*2] - - mpsadbw xmm5, xmm0, 0x0 - - paddw xmm1, xmm5 -%endmacro - -%macro WRITE_AS_INTS 0 - mov rdi, arg(4) ;Results - pxor xmm0, xmm0 - movdqa xmm2, xmm1 - punpcklwd xmm1, xmm0 - punpckhwd xmm2, xmm0 - - movdqa [rdi], xmm1 - movdqa [rdi + 16], xmm2 -%endmacro - -SECTION .text - -;void vpx_sad16x16x8_sse4_1( -; const unsigned char *src_ptr, -; int src_stride, -; const unsigned char *ref_ptr, -; int ref_stride, -; unsigned short *sad_array); -globalsym(vpx_sad16x16x8_sse4_1) -sym(vpx_sad16x16x8_sse4_1): - push rbp - mov rbp, rsp - SHADOW_ARGS_TO_STACK 5 - push rsi - push rdi - ; end prolog - - mov rsi, arg(0) ;src_ptr - mov rdi, arg(2) ;ref_ptr - - movsxd rax, dword ptr arg(1) ;src_stride - movsxd rdx, dword ptr arg(3) ;ref_stride - - PROCESS_16X2X8 1 - PROCESS_16X2X8 0 - PROCESS_16X2X8 0 - PROCESS_16X2X8 0 - PROCESS_16X2X8 0 - PROCESS_16X2X8 0 - PROCESS_16X2X8 0 - PROCESS_16X2X8 0 - - WRITE_AS_INTS - - ; begin epilog - pop rdi - pop rsi - UNSHADOW_ARGS - pop rbp - ret - - -;void vpx_sad16x8x8_sse4_1( -; const unsigned char *src_ptr, -; int src_stride, -; const unsigned char *ref_ptr, -; int ref_stride, -; unsigned short *sad_array -;); -globalsym(vpx_sad16x8x8_sse4_1) -sym(vpx_sad16x8x8_sse4_1): - push rbp - mov rbp, rsp - SHADOW_ARGS_TO_STACK 5 - push rsi - push rdi - ; end prolog - - mov rsi, arg(0) ;src_ptr - mov rdi, arg(2) ;ref_ptr - - movsxd rax, dword ptr arg(1) ;src_stride - movsxd rdx, dword ptr arg(3) ;ref_stride - - PROCESS_16X2X8 1 - PROCESS_16X2X8 0 - PROCESS_16X2X8 0 - PROCESS_16X2X8 0 - - WRITE_AS_INTS - - ; begin epilog - pop rdi - pop rsi - UNSHADOW_ARGS - pop rbp - ret - - -;void vpx_sad8x8x8_sse4_1( -; const unsigned char *src_ptr, -; int src_stride, -; const unsigned char *ref_ptr, -; int ref_stride, -; unsigned short *sad_array -;); -globalsym(vpx_sad8x8x8_sse4_1) -sym(vpx_sad8x8x8_sse4_1): - push rbp - mov rbp, rsp - SHADOW_ARGS_TO_STACK 5 - push rsi - push rdi - ; end prolog - - mov rsi, arg(0) ;src_ptr - mov rdi, arg(2) ;ref_ptr - - movsxd rax, dword ptr arg(1) ;src_stride - movsxd rdx, dword ptr arg(3) ;ref_stride - - PROCESS_8X2X8 1 - PROCESS_8X2X8 0 - PROCESS_8X2X8 0 - PROCESS_8X2X8 0 - - WRITE_AS_INTS - - ; begin epilog - pop rdi - pop rsi - UNSHADOW_ARGS - pop rbp - ret - - -;void vpx_sad8x16x8_sse4_1( -; const unsigned char *src_ptr, -; int src_stride, -; const unsigned char *ref_ptr, -; int ref_stride, -; unsigned short *sad_array -;); -globalsym(vpx_sad8x16x8_sse4_1) -sym(vpx_sad8x16x8_sse4_1): - push rbp - mov rbp, rsp - SHADOW_ARGS_TO_STACK 5 - push rsi - push rdi - ; end prolog - - mov rsi, arg(0) ;src_ptr - mov rdi, arg(2) ;ref_ptr - - movsxd rax, dword ptr arg(1) ;src_stride - movsxd rdx, dword ptr arg(3) ;ref_stride - - PROCESS_8X2X8 1 - PROCESS_8X2X8 0 - PROCESS_8X2X8 0 - PROCESS_8X2X8 0 - PROCESS_8X2X8 0 - PROCESS_8X2X8 0 - PROCESS_8X2X8 0 - PROCESS_8X2X8 0 - - WRITE_AS_INTS - - ; begin epilog - pop rdi - pop rsi - UNSHADOW_ARGS - pop rbp - ret - - -;void vpx_sad4x4x8_sse4_1( -; const unsigned char *src_ptr, -; int src_stride, -; const unsigned char *ref_ptr, -; int ref_stride, -; unsigned short *sad_array -;); -globalsym(vpx_sad4x4x8_sse4_1) -sym(vpx_sad4x4x8_sse4_1): - push rbp - mov rbp, rsp - SHADOW_ARGS_TO_STACK 5 - push rsi - push rdi - ; end prolog - - mov rsi, arg(0) ;src_ptr - mov rdi, arg(2) ;ref_ptr - - movsxd rax, dword ptr arg(1) ;src_stride - movsxd rdx, dword ptr arg(3) ;ref_stride - - PROCESS_4X2X8 1 - PROCESS_4X2X8 0 - - WRITE_AS_INTS - - ; begin epilog - pop rdi - pop rsi - UNSHADOW_ARGS - pop rbp - ret - - - - diff --git a/media/libvpx/libvpx/vpx_dsp/x86/sad_ssse3.asm b/media/libvpx/libvpx/vpx_dsp/x86/sad_ssse3.asm deleted file mode 100644 --- a/media/libvpx/libvpx/vpx_dsp/x86/sad_ssse3.asm +++ /dev/null @@ -1,372 +0,0 @@ -; -; Copyright (c) 2010 The WebM project authors. All Rights Reserved. -; -; Use of this source code is governed by a BSD-style license -; that can be found in the LICENSE file in the root of the source -; tree. An additional intellectual property rights grant can be found -; in the file PATENTS. All contributing project authors may -; be found in the AUTHORS file in the root of the source tree. -; - - -%include "vpx_ports/x86_abi_support.asm" - -%macro PROCESS_16X2X3 1 -%if %1 - movdqa xmm0, XMMWORD PTR [rsi] - lddqu xmm5, XMMWORD PTR [rdi] - lddqu xmm6, XMMWORD PTR [rdi+1] - lddqu xmm7, XMMWORD PTR [rdi+2] - - psadbw xmm5, xmm0 - psadbw xmm6, xmm0 - psadbw xmm7, xmm0 -%else - movdqa xmm0, XMMWORD PTR [rsi] - lddqu xmm1, XMMWORD PTR [rdi] - lddqu xmm2, XMMWORD PTR [rdi+1] - lddqu xmm3, XMMWORD PTR [rdi+2] - - psadbw xmm1, xmm0 - psadbw xmm2, xmm0 - psadbw xmm3, xmm0 - - paddw xmm5, xmm1 - paddw xmm6, xmm2 - paddw xmm7, xmm3 -%endif - movdqa xmm0, XMMWORD PTR [rsi+rax] - lddqu xmm1, XMMWORD PTR [rdi+rdx] - lddqu xmm2, XMMWORD PTR [rdi+rdx+1] - lddqu xmm3, XMMWORD PTR [rdi+rdx+2] - - lea rsi, [rsi+rax*2] - lea rdi, [rdi+rdx*2] - - psadbw xmm1, xmm0 - psadbw xmm2, xmm0 - psadbw xmm3, xmm0 - - paddw xmm5, xmm1 - paddw xmm6, xmm2 - paddw xmm7, xmm3 -%endmacro - -%macro PROCESS_16X2X3_OFFSET 2 -%if %1 - movdqa xmm0, XMMWORD PTR [rsi] - movdqa xmm4, XMMWORD PTR [rdi] - movdqa xmm7, XMMWORD PTR [rdi+16] - - movdqa xmm5, xmm7 - palignr xmm5, xmm4, %2 - - movdqa xmm6, xmm7 - palignr xmm6, xmm4, (%2+1) - - palignr xmm7, xmm4, (%2+2) - - psadbw xmm5, xmm0 - psadbw xmm6, xmm0 - psadbw xmm7, xmm0 -%else - movdqa xmm0, XMMWORD PTR [rsi] - movdqa xmm4, XMMWORD PTR [rdi] - movdqa xmm3, XMMWORD PTR [rdi+16] - - movdqa xmm1, xmm3 - palignr xmm1, xmm4, %2 - - movdqa xmm2, xmm3 - palignr xmm2, xmm4, (%2+1) - - palignr xmm3, xmm4, (%2+2) - - psadbw xmm1, xmm0 - psadbw xmm2, xmm0 - psadbw xmm3, xmm0 - - paddw xmm5, xmm1 - paddw xmm6, xmm2 - paddw xmm7, xmm3 -%endif - movdqa xmm0, XMMWORD PTR [rsi+rax] - movdqa xmm4, XMMWORD PTR [rdi+rdx] - movdqa xmm3, XMMWORD PTR [rdi+rdx+16] - - movdqa xmm1, xmm3 - palignr xmm1, xmm4, %2 - - movdqa xmm2, xmm3 - palignr xmm2, xmm4, (%2+1) - - palignr xmm3, xmm4, (%2+2) - - lea rsi, [rsi+rax*2] - lea rdi, [rdi+rdx*2] - - psadbw xmm1, xmm0 - psadbw xmm2, xmm0 - psadbw xmm3, xmm0 - - paddw xmm5, xmm1 - paddw xmm6, xmm2 - paddw xmm7, xmm3 -%endmacro - -%macro PROCESS_16X16X3_OFFSET 2 -%2_aligned_by_%1: - - sub rdi, %1 - - PROCESS_16X2X3_OFFSET 1, %1 - PROCESS_16X2X3_OFFSET 0, %1 - PROCESS_16X2X3_OFFSET 0, %1 - PROCESS_16X2X3_OFFSET 0, %1 - PROCESS_16X2X3_OFFSET 0, %1 - PROCESS_16X2X3_OFFSET 0, %1 - PROCESS_16X2X3_OFFSET 0, %1 - PROCESS_16X2X3_OFFSET 0, %1 - - jmp %2_store_off - -%endmacro - -%macro PROCESS_16X8X3_OFFSET 2 -%2_aligned_by_%1: - - sub rdi, %1 - - PROCESS_16X2X3_OFFSET 1, %1 - PROCESS_16X2X3_OFFSET 0, %1 - PROCESS_16X2X3_OFFSET 0, %1 - PROCESS_16X2X3_OFFSET 0, %1 - - jmp %2_store_off - -%endmacro - -SECTION .text - -;void int vpx_sad16x16x3_ssse3( -; unsigned char *src_ptr, -; int src_stride, -; unsigned char *ref_ptr, -; int ref_stride, -; int *results) -globalsym(vpx_sad16x16x3_ssse3) -sym(vpx_sad16x16x3_ssse3): - push rbp - mov rbp, rsp - SHADOW_ARGS_TO_STACK 5 - SAVE_XMM 7 - push rsi - push rdi - push rcx - ; end prolog - - mov rsi, arg(0) ;src_ptr - mov rdi, arg(2) ;ref_ptr - - mov rdx, 0xf - and rdx, rdi - - jmp .vpx_sad16x16x3_ssse3_skiptable -.vpx_sad16x16x3_ssse3_jumptable: - dd .vpx_sad16x16x3_ssse3_aligned_by_0 - .vpx_sad16x16x3_ssse3_do_jump - dd .vpx_sad16x16x3_ssse3_aligned_by_1 - .vpx_sad16x16x3_ssse3_do_jump - dd .vpx_sad16x16x3_ssse3_aligned_by_2 - .vpx_sad16x16x3_ssse3_do_jump - dd .vpx_sad16x16x3_ssse3_aligned_by_3 - .vpx_sad16x16x3_ssse3_do_jump - dd .vpx_sad16x16x3_ssse3_aligned_by_4 - .vpx_sad16x16x3_ssse3_do_jump - dd .vpx_sad16x16x3_ssse3_aligned_by_5 - .vpx_sad16x16x3_ssse3_do_jump - dd .vpx_sad16x16x3_ssse3_aligned_by_6 - .vpx_sad16x16x3_ssse3_do_jump - dd .vpx_sad16x16x3_ssse3_aligned_by_7 - .vpx_sad16x16x3_ssse3_do_jump - dd .vpx_sad16x16x3_ssse3_aligned_by_8 - .vpx_sad16x16x3_ssse3_do_jump - dd .vpx_sad16x16x3_ssse3_aligned_by_9 - .vpx_sad16x16x3_ssse3_do_jump - dd .vpx_sad16x16x3_ssse3_aligned_by_10 - .vpx_sad16x16x3_ssse3_do_jump - dd .vpx_sad16x16x3_ssse3_aligned_by_11 - .vpx_sad16x16x3_ssse3_do_jump - dd .vpx_sad16x16x3_ssse3_aligned_by_12 - .vpx_sad16x16x3_ssse3_do_jump - dd .vpx_sad16x16x3_ssse3_aligned_by_13 - .vpx_sad16x16x3_ssse3_do_jump - dd .vpx_sad16x16x3_ssse3_aligned_by_14 - .vpx_sad16x16x3_ssse3_do_jump - dd .vpx_sad16x16x3_ssse3_aligned_by_15 - .vpx_sad16x16x3_ssse3_do_jump -.vpx_sad16x16x3_ssse3_skiptable: - - call .vpx_sad16x16x3_ssse3_do_jump -.vpx_sad16x16x3_ssse3_do_jump: - pop rcx ; get the address of do_jump - mov rax, .vpx_sad16x16x3_ssse3_jumptable - .vpx_sad16x16x3_ssse3_do_jump - add rax, rcx ; get the absolute address of vpx_sad16x16x3_ssse3_jumptable - - movsxd rax, dword [rax + 4*rdx] ; get the 32 bit offset from the jumptable - add rcx, rax - - movsxd rax, dword ptr arg(1) ;src_stride - movsxd rdx, dword ptr arg(3) ;ref_stride - - jmp rcx - - PROCESS_16X16X3_OFFSET 0, .vpx_sad16x16x3_ssse3 - PROCESS_16X16X3_OFFSET 1, .vpx_sad16x16x3_ssse3 - PROCESS_16X16X3_OFFSET 2, .vpx_sad16x16x3_ssse3 - PROCESS_16X16X3_OFFSET 3, .vpx_sad16x16x3_ssse3 - PROCESS_16X16X3_OFFSET 4, .vpx_sad16x16x3_ssse3 - PROCESS_16X16X3_OFFSET 5, .vpx_sad16x16x3_ssse3 - PROCESS_16X16X3_OFFSET 6, .vpx_sad16x16x3_ssse3 - PROCESS_16X16X3_OFFSET 7, .vpx_sad16x16x3_ssse3 - PROCESS_16X16X3_OFFSET 8, .vpx_sad16x16x3_ssse3 - PROCESS_16X16X3_OFFSET 9, .vpx_sad16x16x3_ssse3 - PROCESS_16X16X3_OFFSET 10, .vpx_sad16x16x3_ssse3 - PROCESS_16X16X3_OFFSET 11, .vpx_sad16x16x3_ssse3 - PROCESS_16X16X3_OFFSET 12, .vpx_sad16x16x3_ssse3 - PROCESS_16X16X3_OFFSET 13, .vpx_sad16x16x3_ssse3 - PROCESS_16X16X3_OFFSET 14, .vpx_sad16x16x3_ssse3 - -.vpx_sad16x16x3_ssse3_aligned_by_15: - PROCESS_16X2X3 1 - PROCESS_16X2X3 0 - PROCESS_16X2X3 0 - PROCESS_16X2X3 0 - PROCESS_16X2X3 0 - PROCESS_16X2X3 0 - PROCESS_16X2X3 0 - PROCESS_16X2X3 0 - -.vpx_sad16x16x3_ssse3_store_off: - mov rdi, arg(4) ;Results - - movq xmm0, xmm5 - psrldq xmm5, 8 - - paddw xmm0, xmm5 - movd [rdi], xmm0 -;- - movq xmm0, xmm6 - psrldq xmm6, 8 - - paddw xmm0, xmm6 - movd [rdi+4], xmm0 -;- - movq xmm0, xmm7 - psrldq xmm7, 8 - - paddw xmm0, xmm7 - movd [rdi+8], xmm0 - - ; begin epilog - pop rcx - pop rdi - pop rsi - RESTORE_XMM - UNSHADOW_ARGS - pop rbp - ret - -;void int vpx_sad16x8x3_ssse3( -; unsigned char *src_ptr, -; int src_stride, -; unsigned char *ref_ptr, -; int ref_stride, -; int *results) -globalsym(vpx_sad16x8x3_ssse3) -sym(vpx_sad16x8x3_ssse3): - push rbp - mov rbp, rsp - SHADOW_ARGS_TO_STACK 5 - SAVE_XMM 7 - push rsi - push rdi - push rcx - ; end prolog - - mov rsi, arg(0) ;src_ptr - mov rdi, arg(2) ;ref_ptr - - mov rdx, 0xf - and rdx, rdi - - jmp .vpx_sad16x8x3_ssse3_skiptable -.vpx_sad16x8x3_ssse3_jumptable: - dd .vpx_sad16x8x3_ssse3_aligned_by_0 - .vpx_sad16x8x3_ssse3_do_jump - dd .vpx_sad16x8x3_ssse3_aligned_by_1 - .vpx_sad16x8x3_ssse3_do_jump - dd .vpx_sad16x8x3_ssse3_aligned_by_2 - .vpx_sad16x8x3_ssse3_do_jump - dd .vpx_sad16x8x3_ssse3_aligned_by_3 - .vpx_sad16x8x3_ssse3_do_jump - dd .vpx_sad16x8x3_ssse3_aligned_by_4 - .vpx_sad16x8x3_ssse3_do_jump - dd .vpx_sad16x8x3_ssse3_aligned_by_5 - .vpx_sad16x8x3_ssse3_do_jump - dd .vpx_sad16x8x3_ssse3_aligned_by_6 - .vpx_sad16x8x3_ssse3_do_jump - dd .vpx_sad16x8x3_ssse3_aligned_by_7 - .vpx_sad16x8x3_ssse3_do_jump - dd .vpx_sad16x8x3_ssse3_aligned_by_8 - .vpx_sad16x8x3_ssse3_do_jump - dd .vpx_sad16x8x3_ssse3_aligned_by_9 - .vpx_sad16x8x3_ssse3_do_jump - dd .vpx_sad16x8x3_ssse3_aligned_by_10 - .vpx_sad16x8x3_ssse3_do_jump - dd .vpx_sad16x8x3_ssse3_aligned_by_11 - .vpx_sad16x8x3_ssse3_do_jump - dd .vpx_sad16x8x3_ssse3_aligned_by_12 - .vpx_sad16x8x3_ssse3_do_jump - dd .vpx_sad16x8x3_ssse3_aligned_by_13 - .vpx_sad16x8x3_ssse3_do_jump - dd .vpx_sad16x8x3_ssse3_aligned_by_14 - .vpx_sad16x8x3_ssse3_do_jump - dd .vpx_sad16x8x3_ssse3_aligned_by_15 - .vpx_sad16x8x3_ssse3_do_jump -.vpx_sad16x8x3_ssse3_skiptable: - - call .vpx_sad16x8x3_ssse3_do_jump -.vpx_sad16x8x3_ssse3_do_jump: - pop rcx ; get the address of do_jump - mov rax, .vpx_sad16x8x3_ssse3_jumptable - .vpx_sad16x8x3_ssse3_do_jump - add rax, rcx ; get the absolute address of vpx_sad16x8x3_ssse3_jumptable - - movsxd rax, dword [rax + 4*rdx] ; get the 32 bit offset from the jumptable - add rcx, rax - - movsxd rax, dword ptr arg(1) ;src_stride - movsxd rdx, dword ptr arg(3) ;ref_stride - - jmp rcx - - PROCESS_16X8X3_OFFSET 0, .vpx_sad16x8x3_ssse3 - PROCESS_16X8X3_OFFSET 1, .vpx_sad16x8x3_ssse3 - PROCESS_16X8X3_OFFSET 2, .vpx_sad16x8x3_ssse3 - PROCESS_16X8X3_OFFSET 3, .vpx_sad16x8x3_ssse3 - PROCESS_16X8X3_OFFSET 4, .vpx_sad16x8x3_ssse3 - PROCESS_16X8X3_OFFSET 5, .vpx_sad16x8x3_ssse3 - PROCESS_16X8X3_OFFSET 6, .vpx_sad16x8x3_ssse3 - PROCESS_16X8X3_OFFSET 7, .vpx_sad16x8x3_ssse3 - PROCESS_16X8X3_OFFSET 8, .vpx_sad16x8x3_ssse3 - PROCESS_16X8X3_OFFSET 9, .vpx_sad16x8x3_ssse3 - PROCESS_16X8X3_OFFSET 10, .vpx_sad16x8x3_ssse3 - PROCESS_16X8X3_OFFSET 11, .vpx_sad16x8x3_ssse3 - PROCESS_16X8X3_OFFSET 12, .vpx_sad16x8x3_ssse3 - PROCESS_16X8X3_OFFSET 13, .vpx_sad16x8x3_ssse3 - PROCESS_16X8X3_OFFSET 14, .vpx_sad16x8x3_ssse3 - -.vpx_sad16x8x3_ssse3_aligned_by_15: - - PROCESS_16X2X3 1 - PROCESS_16X2X3 0 - PROCESS_16X2X3 0 - PROCESS_16X2X3 0 - -.vpx_sad16x8x3_ssse3_store_off: - mov rdi, arg(4) ;Results - - movq xmm0, xmm5 - psrldq xmm5, 8 - - paddw xmm0, xmm5 - movd [rdi], xmm0 -;- - movq xmm0, xmm6 - psrldq xmm6, 8 - - paddw xmm0, xmm6 - movd [rdi+4], xmm0 -;- - movq xmm0, xmm7 - psrldq xmm7, 8 - - paddw xmm0, xmm7 - movd [rdi+8], xmm0 - - ; begin epilog - pop rcx - pop rdi - pop rsi - RESTORE_XMM - UNSHADOW_ARGS - pop rbp - ret diff --git a/media/libvpx/libvpx/vpx_dsp/x86/variance_sse2.c b/media/libvpx/libvpx/vpx_dsp/x86/variance_sse2.c --- a/media/libvpx/libvpx/vpx_dsp/x86/variance_sse2.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/variance_sse2.c @@ -34,12 +34,12 @@ return add32x4_sse2(vsum); } static INLINE __m128i load4x2_sse2(const uint8_t *const p, const int stride) { - const __m128i p0 = _mm_cvtsi32_si128(loadu_uint32(p + 0 * stride)); - const __m128i p1 = _mm_cvtsi32_si128(loadu_uint32(p + 1 * stride)); + const __m128i p0 = _mm_cvtsi32_si128(loadu_int32(p + 0 * stride)); + const __m128i p1 = _mm_cvtsi32_si128(loadu_int32(p + 1 * stride)); const __m128i p01 = _mm_unpacklo_epi32(p0, p1); return _mm_unpacklo_epi8(p01, _mm_setzero_si128()); } static INLINE void variance_kernel_sse2(const __m128i src_ptr, @@ -469,27 +469,27 @@ *sse = sse_tmp; \ return sse_tmp - \ (unsigned int)(cast_prod(cast se * se) >> (wlog2 + hlog2)); \ } -#define FNS(opt1, opt2) \ - FN(64, 64, 16, 6, 6, opt1, (int64_t), (int64_t)); \ - FN(64, 32, 16, 6, 5, opt1, (int64_t), (int64_t)); \ - FN(32, 64, 16, 5, 6, opt1, (int64_t), (int64_t)); \ - FN(32, 32, 16, 5, 5, opt1, (int64_t), (int64_t)); \ - FN(32, 16, 16, 5, 4, opt1, (int64_t), (int64_t)); \ - FN(16, 32, 16, 4, 5, opt1, (int64_t), (int64_t)); \ - FN(16, 16, 16, 4, 4, opt1, (uint32_t), (int64_t)); \ - FN(16, 8, 16, 4, 3, opt1, (int32_t), (int32_t)); \ - FN(8, 16, 8, 3, 4, opt1, (int32_t), (int32_t)); \ - FN(8, 8, 8, 3, 3, opt1, (int32_t), (int32_t)); \ - FN(8, 4, 8, 3, 2, opt1, (int32_t), (int32_t)); \ - FN(4, 8, 4, 2, 3, opt1, (int32_t), (int32_t)); \ +#define FNS(opt1, opt2) \ + FN(64, 64, 16, 6, 6, opt1, (int64_t), (int64_t)) \ + FN(64, 32, 16, 6, 5, opt1, (int64_t), (int64_t)) \ + FN(32, 64, 16, 5, 6, opt1, (int64_t), (int64_t)) \ + FN(32, 32, 16, 5, 5, opt1, (int64_t), (int64_t)) \ + FN(32, 16, 16, 5, 4, opt1, (int64_t), (int64_t)) \ + FN(16, 32, 16, 4, 5, opt1, (int64_t), (int64_t)) \ + FN(16, 16, 16, 4, 4, opt1, (uint32_t), (int64_t)) \ + FN(16, 8, 16, 4, 3, opt1, (int32_t), (int32_t)) \ + FN(8, 16, 8, 3, 4, opt1, (int32_t), (int32_t)) \ + FN(8, 8, 8, 3, 3, opt1, (int32_t), (int32_t)) \ + FN(8, 4, 8, 3, 2, opt1, (int32_t), (int32_t)) \ + FN(4, 8, 4, 2, 3, opt1, (int32_t), (int32_t)) \ FN(4, 4, 4, 2, 2, opt1, (int32_t), (int32_t)) -FNS(sse2, sse2); -FNS(ssse3, ssse3); +FNS(sse2, sse2) +FNS(ssse3, ssse3) #undef FNS #undef FN // The 2 unused parameters are place holders for PIC enabled build. @@ -541,25 +541,25 @@ *sse = sse_tmp; \ return sse_tmp - \ (unsigned int)(cast_prod(cast se * se) >> (wlog2 + hlog2)); \ } -#define FNS(opt1, opt2) \ - FN(64, 64, 16, 6, 6, opt1, (int64_t), (int64_t)); \ - FN(64, 32, 16, 6, 5, opt1, (int64_t), (int64_t)); \ - FN(32, 64, 16, 5, 6, opt1, (int64_t), (int64_t)); \ - FN(32, 32, 16, 5, 5, opt1, (int64_t), (int64_t)); \ - FN(32, 16, 16, 5, 4, opt1, (int64_t), (int64_t)); \ - FN(16, 32, 16, 4, 5, opt1, (int64_t), (int64_t)); \ - FN(16, 16, 16, 4, 4, opt1, (uint32_t), (int64_t)); \ - FN(16, 8, 16, 4, 3, opt1, (uint32_t), (int32_t)); \ - FN(8, 16, 8, 3, 4, opt1, (uint32_t), (int32_t)); \ - FN(8, 8, 8, 3, 3, opt1, (uint32_t), (int32_t)); \ - FN(8, 4, 8, 3, 2, opt1, (uint32_t), (int32_t)); \ - FN(4, 8, 4, 2, 3, opt1, (uint32_t), (int32_t)); \ +#define FNS(opt1, opt2) \ + FN(64, 64, 16, 6, 6, opt1, (int64_t), (int64_t)) \ + FN(64, 32, 16, 6, 5, opt1, (int64_t), (int64_t)) \ + FN(32, 64, 16, 5, 6, opt1, (int64_t), (int64_t)) \ + FN(32, 32, 16, 5, 5, opt1, (int64_t), (int64_t)) \ + FN(32, 16, 16, 5, 4, opt1, (int64_t), (int64_t)) \ + FN(16, 32, 16, 4, 5, opt1, (int64_t), (int64_t)) \ + FN(16, 16, 16, 4, 4, opt1, (uint32_t), (int64_t)) \ + FN(16, 8, 16, 4, 3, opt1, (uint32_t), (int32_t)) \ + FN(8, 16, 8, 3, 4, opt1, (uint32_t), (int32_t)) \ + FN(8, 8, 8, 3, 3, opt1, (uint32_t), (int32_t)) \ + FN(8, 4, 8, 3, 2, opt1, (uint32_t), (int32_t)) \ + FN(4, 8, 4, 2, 3, opt1, (uint32_t), (int32_t)) \ FN(4, 4, 4, 2, 2, opt1, (uint32_t), (int32_t)) -FNS(sse2, sse); -FNS(ssse3, ssse3); +FNS(sse2, sse) +FNS(ssse3, ssse3) #undef FNS #undef FN diff --git a/media/libvpx/libvpx/vpx_dsp/x86/vpx_subpixel_4t_intrin_sse2.c b/media/libvpx/libvpx/vpx_dsp/x86/vpx_subpixel_4t_intrin_sse2.c --- a/media/libvpx/libvpx/vpx_dsp/x86/vpx_subpixel_4t_intrin_sse2.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/vpx_subpixel_4t_intrin_sse2.c @@ -1038,16 +1038,16 @@ // void vpx_convolve8_avg_vert_sse2(const uint8_t *src, ptrdiff_t src_stride, // uint8_t *dst, ptrdiff_t dst_stride, // const InterpKernel *filter, int x0_q4, // int32_t x_step_q4, int y0_q4, int y_step_q4, // int w, int h); -FUN_CONV_1D(horiz, x0_q4, x_step_q4, h, src, , sse2, 0); +FUN_CONV_1D(horiz, x0_q4, x_step_q4, h, src, , sse2, 0) FUN_CONV_1D(vert, y0_q4, y_step_q4, v, src - (num_taps / 2 - 1) * src_stride, , - sse2, 0); -FUN_CONV_1D(avg_horiz, x0_q4, x_step_q4, h, src, avg_, sse2, 1); + sse2, 0) +FUN_CONV_1D(avg_horiz, x0_q4, x_step_q4, h, src, avg_, sse2, 1) FUN_CONV_1D(avg_vert, y0_q4, y_step_q4, v, - src - (num_taps / 2 - 1) * src_stride, avg_, sse2, 1); + src - (num_taps / 2 - 1) * src_stride, avg_, sse2, 1) // void vpx_convolve8_sse2(const uint8_t *src, ptrdiff_t src_stride, // uint8_t *dst, ptrdiff_t dst_stride, // const InterpKernel *filter, int x0_q4, // int32_t x_step_q4, int y0_q4, int y_step_q4, @@ -1055,12 +1055,12 @@ // void vpx_convolve8_avg_sse2(const uint8_t *src, ptrdiff_t src_stride, // uint8_t *dst, ptrdiff_t dst_stride, // const InterpKernel *filter, int x0_q4, // int32_t x_step_q4, int y0_q4, int y_step_q4, // int w, int h); -FUN_CONV_2D(, sse2, 0); -FUN_CONV_2D(avg_, sse2, 1); +FUN_CONV_2D(, sse2, 0) +FUN_CONV_2D(avg_, sse2, 1) #if CONFIG_VP9_HIGHBITDEPTH && VPX_ARCH_X86_64 // From vpx_dsp/x86/vpx_high_subpixel_8t_sse2.asm. highbd_filter8_1dfunction vpx_highbd_filter_block1d16_v8_sse2; highbd_filter8_1dfunction vpx_highbd_filter_block1d16_h8_sse2; @@ -1137,16 +1137,16 @@ // const int16_t *filter_x, // int x_step_q4, // const int16_t *filter_y, // int y_step_q4, // int w, int h, int bd); -HIGH_FUN_CONV_1D(horiz, x0_q4, x_step_q4, h, src, , sse2, 0); +HIGH_FUN_CONV_1D(horiz, x0_q4, x_step_q4, h, src, , sse2, 0) HIGH_FUN_CONV_1D(vert, y0_q4, y_step_q4, v, - src - src_stride * (num_taps / 2 - 1), , sse2, 0); -HIGH_FUN_CONV_1D(avg_horiz, x0_q4, x_step_q4, h, src, avg_, sse2, 1); + src - src_stride * (num_taps / 2 - 1), , sse2, 0) +HIGH_FUN_CONV_1D(avg_horiz, x0_q4, x_step_q4, h, src, avg_, sse2, 1) HIGH_FUN_CONV_1D(avg_vert, y0_q4, y_step_q4, v, - src - src_stride * (num_taps / 2 - 1), avg_, sse2, 1); + src - src_stride * (num_taps / 2 - 1), avg_, sse2, 1) // void vpx_highbd_convolve8_sse2(const uint8_t *src, ptrdiff_t src_stride, // uint8_t *dst, ptrdiff_t dst_stride, // const InterpKernel *filter, int x0_q4, // int32_t x_step_q4, int y0_q4, int y_step_q4, @@ -1154,8 +1154,8 @@ // void vpx_highbd_convolve8_avg_sse2(const uint8_t *src, ptrdiff_t src_stride, // uint8_t *dst, ptrdiff_t dst_stride, // const InterpKernel *filter, int x0_q4, // int32_t x_step_q4, int y0_q4, // int y_step_q4, int w, int h, int bd); -HIGH_FUN_CONV_2D(, sse2, 0); -HIGH_FUN_CONV_2D(avg_, sse2, 1); +HIGH_FUN_CONV_2D(, sse2, 0) +HIGH_FUN_CONV_2D(avg_, sse2, 1) #endif // CONFIG_VP9_HIGHBITDEPTH && VPX_ARCH_X86_64 diff --git a/media/libvpx/libvpx/vpx_dsp/x86/vpx_subpixel_8t_intrin_avx2.c b/media/libvpx/libvpx/vpx_dsp/x86/vpx_subpixel_8t_intrin_avx2.c --- a/media/libvpx/libvpx/vpx_dsp/x86/vpx_subpixel_8t_intrin_avx2.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/vpx_subpixel_8t_intrin_avx2.c @@ -967,16 +967,16 @@ // void vpx_convolve8_avg_vert_avx2(const uint8_t *src, ptrdiff_t src_stride, // uint8_t *dst, ptrdiff_t dst_stride, // const InterpKernel *filter, int x0_q4, // int32_t x_step_q4, int y0_q4, // int y_step_q4, int w, int h); -FUN_CONV_1D(horiz, x0_q4, x_step_q4, h, src, , avx2, 0); +FUN_CONV_1D(horiz, x0_q4, x_step_q4, h, src, , avx2, 0) FUN_CONV_1D(vert, y0_q4, y_step_q4, v, src - src_stride * (num_taps / 2 - 1), , - avx2, 0); -FUN_CONV_1D(avg_horiz, x0_q4, x_step_q4, h, src, avg_, avx2, 1); + avx2, 0) +FUN_CONV_1D(avg_horiz, x0_q4, x_step_q4, h, src, avg_, avx2, 1) FUN_CONV_1D(avg_vert, y0_q4, y_step_q4, v, - src - src_stride * (num_taps / 2 - 1), avg_, avx2, 1); + src - src_stride * (num_taps / 2 - 1), avg_, avx2, 1) // void vpx_convolve8_avx2(const uint8_t *src, ptrdiff_t src_stride, // uint8_t *dst, ptrdiff_t dst_stride, // const InterpKernel *filter, int x0_q4, // int32_t x_step_q4, int y0_q4, int y_step_q4, @@ -984,8 +984,8 @@ // void vpx_convolve8_avg_avx2(const uint8_t *src, ptrdiff_t src_stride, // uint8_t *dst, ptrdiff_t dst_stride, // const InterpKernel *filter, int x0_q4, // int32_t x_step_q4, int y0_q4, int y_step_q4, // int w, int h); -FUN_CONV_2D(, avx2, 0); -FUN_CONV_2D(avg_, avx2, 1); +FUN_CONV_2D(, avx2, 0) +FUN_CONV_2D(avg_, avx2, 1) #endif // HAVE_AX2 && HAVE_SSSE3 diff --git a/media/libvpx/libvpx/vpx_dsp/x86/vpx_subpixel_8t_intrin_ssse3.c b/media/libvpx/libvpx/vpx_dsp/x86/vpx_subpixel_8t_intrin_ssse3.c --- a/media/libvpx/libvpx/vpx_dsp/x86/vpx_subpixel_8t_intrin_ssse3.c +++ b/media/libvpx/libvpx/vpx_dsp/x86/vpx_subpixel_8t_intrin_ssse3.c @@ -729,16 +729,16 @@ // void vpx_convolve8_avg_vert_ssse3(const uint8_t *src, ptrdiff_t src_stride, // uint8_t *dst, ptrdiff_t dst_stride, // const InterpKernel *filter, int x0_q4, // int32_t x_step_q4, int y0_q4, // int y_step_q4, int w, int h); -FUN_CONV_1D(horiz, x0_q4, x_step_q4, h, src, , ssse3, 0); +FUN_CONV_1D(horiz, x0_q4, x_step_q4, h, src, , ssse3, 0) FUN_CONV_1D(vert, y0_q4, y_step_q4, v, src - src_stride * (num_taps / 2 - 1), , - ssse3, 0); -FUN_CONV_1D(avg_horiz, x0_q4, x_step_q4, h, src, avg_, ssse3, 1); + ssse3, 0) +FUN_CONV_1D(avg_horiz, x0_q4, x_step_q4, h, src, avg_, ssse3, 1) FUN_CONV_1D(avg_vert, y0_q4, y_step_q4, v, - src - src_stride * (num_taps / 2 - 1), avg_, ssse3, 1); + src - src_stride * (num_taps / 2 - 1), avg_, ssse3, 1) static void filter_horiz_w8_ssse3(const uint8_t *const src, const ptrdiff_t src_stride, uint8_t *const dst, const int16_t *const x_filter) { @@ -1081,7 +1081,7 @@ // void vpx_convolve8_avg_ssse3(const uint8_t *src, ptrdiff_t src_stride, // uint8_t *dst, ptrdiff_t dst_stride, // const InterpKernel *filter, int x0_q4, // int32_t x_step_q4, int y0_q4, int y_step_q4, // int w, int h); -FUN_CONV_2D(, ssse3, 0); -FUN_CONV_2D(avg_, ssse3, 1); +FUN_CONV_2D(, ssse3, 0) +FUN_CONV_2D(avg_, ssse3, 1) diff --git a/media/libvpx/libvpx/vpx_ports/bitops.h b/media/libvpx/libvpx/vpx_ports/bitops.h --- a/media/libvpx/libvpx/vpx_ports/bitops.h +++ b/media/libvpx/libvpx/vpx_ports/bitops.h @@ -24,32 +24,51 @@ #ifdef __cplusplus extern "C" { #endif -// These versions of get_msb() are only valid when n != 0 because all -// of the optimized versions are undefined when n == 0: +// These versions of get_lsb() and get_msb() are only valid when n != 0 +// because all of the optimized versions are undefined when n == 0: // https://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html // use GNU builtins where available. #if defined(__GNUC__) && \ ((__GNUC__ == 3 && __GNUC_MINOR__ >= 4) || __GNUC__ >= 4) +static INLINE int get_lsb(unsigned int n) { + assert(n != 0); + return __builtin_ctz(n); +} + static INLINE int get_msb(unsigned int n) { assert(n != 0); return 31 ^ __builtin_clz(n); } #elif defined(USE_MSC_INTRINSICS) +#pragma intrinsic(_BitScanForward) #pragma intrinsic(_BitScanReverse) +static INLINE int get_lsb(unsigned int n) { + unsigned long first_set_bit; // NOLINT(runtime/int) + _BitScanForward(&first_set_bit, n); + return first_set_bit; +} + static INLINE int get_msb(unsigned int n) { unsigned long first_set_bit; assert(n != 0); _BitScanReverse(&first_set_bit, n); return first_set_bit; } #undef USE_MSC_INTRINSICS #else +static INLINE int get_lsb(unsigned int n) { + int i; + assert(n != 0); + for (i = 0; i < 32 && !(n & 1); ++i) n >>= 1; + return i; +} + // Returns (int)floor(log2(n)). n must be > 0. static INLINE int get_msb(unsigned int n) { int log = 0; unsigned int value = n; int i; diff --git a/media/libvpx/libvpx/vpx_ports/loongarch.h b/media/libvpx/libvpx/vpx_ports/loongarch.h new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_ports/loongarch.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Jin Bo + * Contributed by Lu Wang + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef VPX_VPX_PORTS_LOONGARCH_H_ +#define VPX_VPX_PORTS_LOONGARCH_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#define HAS_LSX 0x01 +#define HAS_LASX 0x02 + +int loongarch_cpu_caps(void); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // VPX_VPX_PORTS_LOONGARCH_H_ diff --git a/media/libvpx/libvpx/vpx_ports/loongarch_cpudetect.c b/media/libvpx/libvpx/vpx_ports/loongarch_cpudetect.c new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_ports/loongarch_cpudetect.c @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * Contributed by Jin Bo + * Contributed by Lu Wang + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "./vpx_config.h" +#include "vpx_ports/loongarch.h" + +#define LOONGARCH_CFG2 0x02 +#define LOONGARCH_CFG2_LSX (1 << 6) +#define LOONGARCH_CFG2_LASX (1 << 7) + +#if CONFIG_RUNTIME_CPU_DETECT +#if defined(__loongarch__) && defined(__linux__) +int loongarch_cpu_caps(void) { + int reg = 0; + int flag = 0; + + __asm__ volatile("cpucfg %0, %1 \n\t" : "+&r"(reg) : "r"(LOONGARCH_CFG2)); + if (reg & LOONGARCH_CFG2_LSX) flag |= HAS_LSX; + + if (reg & LOONGARCH_CFG2_LASX) flag |= HAS_LASX; + + return flag; +} +#else /* end __loongarch__ && __linux__ */ +#error \ + "--enable-runtime-cpu-detect selected, but no CPU detection method " \ +"available for your platform. Reconfigure with --disable-runtime-cpu-detect." +#endif +#else /* end CONFIG_RUNTIME_CPU_DETECT */ +int loongarch_cpu_caps(void) { return 0; } +#endif diff --git a/media/libvpx/libvpx/vpx_ports/mips.h b/media/libvpx/libvpx/vpx_ports/mips.h --- a/media/libvpx/libvpx/vpx_ports/mips.h +++ b/media/libvpx/libvpx/vpx_ports/mips.h @@ -6,12 +6,12 @@ * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ -#ifndef VPX_PORTS_MIPS_H_ -#define VPX_PORTS_MIPS_H_ +#ifndef VPX_VPX_PORTS_MIPS_H_ +#define VPX_VPX_PORTS_MIPS_H_ #ifdef __cplusplus extern "C" { #endif @@ -22,6 +22,6 @@ #ifdef __cplusplus } // extern "C" #endif -#endif // VPX_PORTS_MIPS_H_ +#endif // VPX_VPX_PORTS_MIPS_H_ diff --git a/media/libvpx/libvpx/vpx_ports/vpx_once.h b/media/libvpx/libvpx/vpx_ports/vpx_once.h --- a/media/libvpx/libvpx/vpx_ports/vpx_once.h +++ b/media/libvpx/libvpx/vpx_ports/vpx_once.h @@ -93,11 +93,11 @@ #elif CONFIG_MULTITHREAD && defined(__OS2__) #define INCL_DOS #include static void once(void (*func)(void)) { - static int done; + static volatile int done; /* If the initialization is complete, return early. */ if (done) return; /* Causes all other threads in the process to block themselves @@ -126,11 +126,11 @@ * so as long as your platform provides atomic loads/stores of pointers * no synchronization is strictly necessary. */ static void once(void (*func)(void)) { - static int done; + static volatile int done; if (!done) { func(); done = 1; } diff --git a/media/libvpx/libvpx/vpx_ports/vpx_ports.mk b/media/libvpx/libvpx/vpx_ports/vpx_ports.mk --- a/media/libvpx/libvpx/vpx_ports/vpx_ports.mk +++ b/media/libvpx/libvpx/vpx_ports/vpx_ports.mk @@ -26,11 +26,11 @@ # Visual Studio x64 does not support the _mm_empty() intrinsic. PORTS_SRCS-$(HAVE_MMX) += emms_mmx.asm endif ifeq ($(VPX_ARCH_X86_64),yes) -PORTS_SRCS-yes += float_control_word.asm +PORTS_SRCS-$(CONFIG_MSVS) += float_control_word.asm endif ifeq ($(VPX_ARCH_X86)$(VPX_ARCH_X86_64),yes) PORTS_SRCS-yes += x86.h PORTS_SRCS-yes += x86_abi_support.asm @@ -43,8 +43,11 @@ PORTS_SRCS-$(VPX_ARCH_PPC) += ppc.h PORTS_SRCS-$(VPX_ARCH_MIPS) += mips_cpudetect.c PORTS_SRCS-$(VPX_ARCH_MIPS) += mips.h +PORTS_SRCS-$(VPX_ARCH_LOONGARCH) += loongarch_cpudetect.c +PORTS_SRCS-$(VPX_ARCH_LOONGARCH) += loongarch.h + ifeq ($(VPX_ARCH_MIPS), yes) PORTS_SRCS-yes += asmdefs_mmi.h endif diff --git a/media/libvpx/libvpx/vpx_ports/x86.h b/media/libvpx/libvpx/vpx_ports/x86.h --- a/media/libvpx/libvpx/vpx_ports/x86.h +++ b/media/libvpx/libvpx/vpx_ports/x86.h @@ -45,19 +45,19 @@ #if defined(__GNUC__) && __GNUC__ || defined(__ANDROID__) #if VPX_ARCH_X86_64 #define cpuid(func, func2, ax, bx, cx, dx) \ __asm__ __volatile__("cpuid \n\t" \ : "=a"(ax), "=b"(bx), "=c"(cx), "=d"(dx) \ - : "a"(func), "c"(func2)); + : "a"(func), "c"(func2)) #else #define cpuid(func, func2, ax, bx, cx, dx) \ __asm__ __volatile__( \ "mov %%ebx, %%edi \n\t" \ "cpuid \n\t" \ "xchg %%edi, %%ebx \n\t" \ : "=a"(ax), "=D"(bx), "=c"(cx), "=d"(dx) \ - : "a"(func), "c"(func2)); + : "a"(func), "c"(func2)) #endif #elif defined(__SUNPRO_C) || \ defined(__SUNPRO_CC) /* end __GNUC__ or __ANDROID__*/ #if VPX_ARCH_X86_64 #define cpuid(func, func2, ax, bx, cx, dx) \ @@ -65,20 +65,20 @@ "xchg %rsi, %rbx \n\t" \ "cpuid \n\t" \ "movl %ebx, %edi \n\t" \ "xchg %rsi, %rbx \n\t" \ : "=a"(ax), "=D"(bx), "=c"(cx), "=d"(dx) \ - : "a"(func), "c"(func2)); + : "a"(func), "c"(func2)) #else #define cpuid(func, func2, ax, bx, cx, dx) \ asm volatile( \ "pushl %ebx \n\t" \ "cpuid \n\t" \ "movl %ebx, %edi \n\t" \ "popl %ebx \n\t" \ : "=a"(ax), "=D"(bx), "=c"(cx), "=d"(dx) \ - : "a"(func), "c"(func2)); + : "a"(func), "c"(func2)) #endif #else /* end __SUNPRO__ */ #if VPX_ARCH_X86_64 #if defined(_MSC_VER) && _MSC_VER > 1500 #define cpuid(func, func2, a, b, c, d) \ @@ -389,11 +389,11 @@ // Precision PC Field // Single Precision (24-Bits) 00B // Reserved 01B // Double Precision (53-Bits) 10B // Extended Precision (64-Bits) 11B - x87_set_control_word((mode & ~0x300) | 0x200); + x87_set_control_word((mode & ~0x300u) | 0x200u); return mode; } #ifdef __cplusplus } // extern "C" diff --git a/media/libvpx/libvpx/vpx_scale/generic/yv12config.c b/media/libvpx/libvpx/vpx_scale/generic/yv12config.c --- a/media/libvpx/libvpx/vpx_scale/generic/yv12config.c +++ b/media/libvpx/libvpx/vpx_scale/generic/yv12config.c @@ -62,10 +62,14 @@ int uvplane_size = (uv_height + border) * uv_stride; const size_t frame_size = yplane_size + 2 * uvplane_size; if (!ybf->buffer_alloc) { ybf->buffer_alloc = (uint8_t *)vpx_memalign(32, frame_size); + if (!ybf->buffer_alloc) { + ybf->buffer_alloc_sz = 0; + return -1; + } #if defined(__has_feature) #if __has_feature(memory_sanitizer) // This memset is needed for fixing the issue of using uninitialized // value in msan test. It will cause a perf loss, so only do this for // msan test. @@ -73,11 +77,11 @@ #endif #endif ybf->buffer_alloc_sz = frame_size; } - if (!ybf->buffer_alloc || ybf->buffer_alloc_sz < frame_size) return -1; + if (ybf->buffer_alloc_sz < frame_size) return -1; /* Only support allocating buffers that have a border that's a multiple * of 32. The border restriction is required to get 16-byte alignment of * the start of the chroma rows without introducing an arbitrary gap * between planes, which would break the semantics of things like diff --git a/media/libvpx/libvpx/vpx_util/loongson_intrinsics.h b/media/libvpx/libvpx/vpx_util/loongson_intrinsics.h new file mode 100644 --- /dev/null +++ b/media/libvpx/libvpx/vpx_util/loongson_intrinsics.h @@ -0,0 +1,2090 @@ +/* + * Copyright (c) 2022 The WebM project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#ifndef VPX_VPX_UTIL_LOONGSON_INTRINSICS_H_ +#define VPX_VPX_UTIL_LOONGSON_INTRINSICS_H_ + +/* + * Copyright (c) 2021 Loongson Technology Corporation Limited + * All rights reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + * Contributed by Shiyou Yin + * Xiwei Gu + * Lu Wang + * + * This file is a header file for loongarch builtin extension. + * + */ + +#ifndef LOONGSON_INTRINSICS_H +#define LOONGSON_INTRINSICS_H + +/** + * MAJOR version: Macro usage changes. + * MINOR version: Add new functions, or bug fixes. + * MICRO version: Comment changes or implementation changes. + */ +#define LSOM_VERSION_MAJOR 1 +#define LSOM_VERSION_MINOR 2 +#define LSOM_VERSION_MICRO 1 + +#define DUP2_ARG1(_INS, _IN0, _IN1, _OUT0, _OUT1) \ + { \ + _OUT0 = _INS(_IN0); \ + _OUT1 = _INS(_IN1); \ + } + +#define DUP2_ARG2(_INS, _IN0, _IN1, _IN2, _IN3, _OUT0, _OUT1) \ + { \ + _OUT0 = _INS(_IN0, _IN1); \ + _OUT1 = _INS(_IN2, _IN3); \ + } + +#define DUP2_ARG3(_INS, _IN0, _IN1, _IN2, _IN3, _IN4, _IN5, _OUT0, _OUT1) \ + { \ + _OUT0 = _INS(_IN0, _IN1, _IN2); \ + _OUT1 = _INS(_IN3, _IN4, _IN5); \ + } + +#define DUP4_ARG1(_INS, _IN0, _IN1, _IN2, _IN3, _OUT0, _OUT1, _OUT2, _OUT3) \ + { \ + DUP2_ARG1(_INS, _IN0, _IN1, _OUT0, _OUT1); \ + DUP2_ARG1(_INS, _IN2, _IN3, _OUT2, _OUT3); \ + } + +#define DUP4_ARG2(_INS, _IN0, _IN1, _IN2, _IN3, _IN4, _IN5, _IN6, _IN7, _OUT0, \ + _OUT1, _OUT2, _OUT3) \ + { \ + DUP2_ARG2(_INS, _IN0, _IN1, _IN2, _IN3, _OUT0, _OUT1); \ + DUP2_ARG2(_INS, _IN4, _IN5, _IN6, _IN7, _OUT2, _OUT3); \ + } + +#define DUP4_ARG3(_INS, _IN0, _IN1, _IN2, _IN3, _IN4, _IN5, _IN6, _IN7, _IN8, \ + _IN9, _IN10, _IN11, _OUT0, _OUT1, _OUT2, _OUT3) \ + { \ + DUP2_ARG3(_INS, _IN0, _IN1, _IN2, _IN3, _IN4, _IN5, _OUT0, _OUT1); \ + DUP2_ARG3(_INS, _IN6, _IN7, _IN8, _IN9, _IN10, _IN11, _OUT2, _OUT3); \ + } + +#ifdef __loongarch_sx +#include +/* + * ============================================================================= + * Description : Dot product & addition of byte vector elements + * Arguments : Inputs - in_c, in_h, in_l + * Outputs - out + * Return Type - halfword + * Details : Signed byte elements from in_h are multiplied by + * signed byte elements from in_l, and then added adjacent to + * each other to get a result twice the size of input. Then + * the results are added to signed half-word elements from in_c. + * Example : out = __lsx_vdp2add_h_b(in_c, in_h, in_l) + * in_c : 1,2,3,4, 1,2,3,4 + * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * in_l : 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,1 + * out : 23,40,41,26, 23,40,41,26 + * ============================================================================= + */ +static inline __m128i __lsx_vdp2add_h_b(__m128i in_c, __m128i in_h, + __m128i in_l) { + __m128i out; + + out = __lsx_vmaddwev_h_b(in_c, in_h, in_l); + out = __lsx_vmaddwod_h_b(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product & addition of byte vector elements + * Arguments : Inputs - in_c, in_h, in_l + * Outputs - out + * Return Type - halfword + * Details : Unsigned byte elements from in_h are multiplied by + * unsigned byte elements from in_l, and then added adjacent to + * each other to get a result twice the size of input. + * The results are added to signed half-word elements from in_c. + * Example : out = __lsx_vdp2add_h_bu(in_c, in_h, in_l) + * in_c : 1,2,3,4, 1,2,3,4 + * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * in_l : 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,1 + * out : 23,40,41,26, 23,40,41,26 + * ============================================================================= + */ +static inline __m128i __lsx_vdp2add_h_bu(__m128i in_c, __m128i in_h, + __m128i in_l) { + __m128i out; + + out = __lsx_vmaddwev_h_bu(in_c, in_h, in_l); + out = __lsx_vmaddwod_h_bu(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product & addition of byte vector elements + * Arguments : Inputs - in_c, in_h, in_l + * Outputs - out + * Return Type - halfword + * Details : Unsigned byte elements from in_h are multiplied by + * signed byte elements from in_l, and then added adjacent to + * each other to get a result twice the size of input. + * The results are added to signed half-word elements from in_c. + * Example : out = __lsx_vdp2add_h_bu_b(in_c, in_h, in_l) + * in_c : 1,1,1,1, 1,1,1,1 + * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * in_l : -1,-2,-3,-4, -5,-6,-7,-8, 1,2,3,4, 5,6,7,8 + * out : -4,-24,-60,-112, 6,26,62,114 + * ============================================================================= + */ +static inline __m128i __lsx_vdp2add_h_bu_b(__m128i in_c, __m128i in_h, + __m128i in_l) { + __m128i out; + + out = __lsx_vmaddwev_h_bu_b(in_c, in_h, in_l); + out = __lsx_vmaddwod_h_bu_b(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product & addition of half-word vector elements + * Arguments : Inputs - in_c, in_h, in_l + * Outputs - out + * Return Type - __m128i + * Details : Signed half-word elements from in_h are multiplied by + * signed half-word elements from in_l, and then added adjacent to + * each other to get a result twice the size of input. + * Then the results are added to signed word elements from in_c. + * Example : out = __lsx_vdp2add_h_b(in_c, in_h, in_l) + * in_c : 1,2,3,4 + * in_h : 1,2,3,4, 5,6,7,8 + * in_l : 8,7,6,5, 4,3,2,1 + * out : 23,40,41,26 + * ============================================================================= + */ +static inline __m128i __lsx_vdp2add_w_h(__m128i in_c, __m128i in_h, + __m128i in_l) { + __m128i out; + + out = __lsx_vmaddwev_w_h(in_c, in_h, in_l); + out = __lsx_vmaddwod_w_h(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of byte vector elements + * Arguments : Inputs - in_h, in_l + * Outputs - out + * Return Type - halfword + * Details : Signed byte elements from in_h are multiplied by + * signed byte elements from in_l, and then added adjacent to + * each other to get a result twice the size of input. + * Example : out = __lsx_vdp2_h_b(in_h, in_l) + * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * in_l : 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,1 + * out : 22,38,38,22, 22,38,38,22 + * ============================================================================= + */ +static inline __m128i __lsx_vdp2_h_b(__m128i in_h, __m128i in_l) { + __m128i out; + + out = __lsx_vmulwev_h_b(in_h, in_l); + out = __lsx_vmaddwod_h_b(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of byte vector elements + * Arguments : Inputs - in_h, in_l + * Outputs - out + * Return Type - halfword + * Details : Unsigned byte elements from in_h are multiplied by + * unsigned byte elements from in_l, and then added adjacent to + * each other to get a result twice the size of input. + * Example : out = __lsx_vdp2_h_bu(in_h, in_l) + * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * in_l : 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,1 + * out : 22,38,38,22, 22,38,38,22 + * ============================================================================= + */ +static inline __m128i __lsx_vdp2_h_bu(__m128i in_h, __m128i in_l) { + __m128i out; + + out = __lsx_vmulwev_h_bu(in_h, in_l); + out = __lsx_vmaddwod_h_bu(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of byte vector elements + * Arguments : Inputs - in_h, in_l + * Outputs - out + * Return Type - halfword + * Details : Unsigned byte elements from in_h are multiplied by + * signed byte elements from in_l, and then added adjacent to + * each other to get a result twice the size of input. + * Example : out = __lsx_vdp2_h_bu_b(in_h, in_l) + * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * in_l : 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,-1 + * out : 22,38,38,22, 22,38,38,6 + * ============================================================================= + */ +static inline __m128i __lsx_vdp2_h_bu_b(__m128i in_h, __m128i in_l) { + __m128i out; + + out = __lsx_vmulwev_h_bu_b(in_h, in_l); + out = __lsx_vmaddwod_h_bu_b(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of byte vector elements + * Arguments : Inputs - in_h, in_l + * Outputs - out + * Return Type - halfword + * Details : Signed byte elements from in_h are multiplied by + * signed byte elements from in_l, and then added adjacent to + * each other to get a result twice the size of input. + * Example : out = __lsx_vdp2_w_h(in_h, in_l) + * in_h : 1,2,3,4, 5,6,7,8 + * in_l : 8,7,6,5, 4,3,2,1 + * out : 22,38,38,22 + * ============================================================================= + */ +static inline __m128i __lsx_vdp2_w_h(__m128i in_h, __m128i in_l) { + __m128i out; + + out = __lsx_vmulwev_w_h(in_h, in_l); + out = __lsx_vmaddwod_w_h(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of byte vector elements + * Arguments : Inputs - in_h, in_l + * Outputs - out + * Return Type - double + * Details : Signed byte elements from in_h are multiplied by + * signed byte elements from in_l, and then added adjacent to + * each other to get a result twice the size of input. + * Example : out = __lsx_vdp2_d_w(in_h, in_l) + * in_h : 1,2,3,4 + * in_l : 8,7,6,5 + * out : 22,38 + * ============================================================================= + */ +static inline __m128i __lsx_vdp2_d_w(__m128i in_h, __m128i in_l) { + __m128i out; + + out = __lsx_vmulwev_d_w(in_h, in_l); + out = __lsx_vmaddwod_d_w(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Clip all halfword elements of input vector between min & max + * out = ((_in) < (min)) ? (min) : (((_in) > (max)) ? (max) : + * (_in)) + * Arguments : Inputs - _in (input vector) + * - min (min threshold) + * - max (max threshold) + * Outputs - out (output vector with clipped elements) + * Return Type - signed halfword + * Example : out = __lsx_vclip_h(_in) + * _in : -8,2,280,249, -8,255,280,249 + * min : 1,1,1,1, 1,1,1,1 + * max : 9,9,9,9, 9,9,9,9 + * out : 1,2,9,9, 1,9,9,9 + * ============================================================================= + */ +static inline __m128i __lsx_vclip_h(__m128i _in, __m128i min, __m128i max) { + __m128i out; + + out = __lsx_vmax_h(min, _in); + out = __lsx_vmin_h(max, out); + return out; +} + +/* + * ============================================================================= + * Description : Set each element of vector between 0 and 255 + * Arguments : Inputs - _in + * Outputs - out + * Return Type - halfword + * Details : Signed byte elements from _in are clamped between 0 and 255. + * Example : out = __lsx_vclip255_h(_in) + * _in : -8,255,280,249, -8,255,280,249 + * out : 0,255,255,249, 0,255,255,249 + * ============================================================================= + */ +static inline __m128i __lsx_vclip255_h(__m128i _in) { + __m128i out; + + out = __lsx_vmaxi_h(_in, 0); + out = __lsx_vsat_hu(out, 7); + return out; +} + +/* + * ============================================================================= + * Description : Set each element of vector between 0 and 255 + * Arguments : Inputs - _in + * Outputs - out + * Return Type - word + * Details : Signed byte elements from _in are clamped between 0 and 255. + * Example : out = __lsx_vclip255_w(_in) + * _in : -8,255,280,249 + * out : 0,255,255,249 + * ============================================================================= + */ +static inline __m128i __lsx_vclip255_w(__m128i _in) { + __m128i out; + + out = __lsx_vmaxi_w(_in, 0); + out = __lsx_vsat_wu(out, 7); + return out; +} + +/* + * ============================================================================= + * Description : Swap two variables + * Arguments : Inputs - _in0, _in1 + * Outputs - _in0, _in1 (in-place) + * Details : Swapping of two input variables using xor + * Example : LSX_SWAP(_in0, _in1) + * _in0 : 1,2,3,4 + * _in1 : 5,6,7,8 + * _in0(out) : 5,6,7,8 + * _in1(out) : 1,2,3,4 + * ============================================================================= + */ +#define LSX_SWAP(_in0, _in1) \ + { \ + _in0 = __lsx_vxor_v(_in0, _in1); \ + _in1 = __lsx_vxor_v(_in0, _in1); \ + _in0 = __lsx_vxor_v(_in0, _in1); \ + } + +/* + * ============================================================================= + * Description : Transpose 4x4 block with word elements in vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1, out2, out3 + * Details : + * Example : + * 1, 2, 3, 4 1, 5, 9,13 + * 5, 6, 7, 8 to 2, 6,10,14 + * 9,10,11,12 =====> 3, 7,11,15 + * 13,14,15,16 4, 8,12,16 + * ============================================================================= + */ +#define LSX_TRANSPOSE4x4_W(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ + { \ + __m128i _t0, _t1, _t2, _t3; \ + \ + _t0 = __lsx_vilvl_w(_in1, _in0); \ + _t1 = __lsx_vilvh_w(_in1, _in0); \ + _t2 = __lsx_vilvl_w(_in3, _in2); \ + _t3 = __lsx_vilvh_w(_in3, _in2); \ + _out0 = __lsx_vilvl_d(_t2, _t0); \ + _out1 = __lsx_vilvh_d(_t2, _t0); \ + _out2 = __lsx_vilvl_d(_t3, _t1); \ + _out3 = __lsx_vilvh_d(_t3, _t1); \ + } + +/* + * ============================================================================= + * Description : Transpose 8x8 block with byte elements in vectors + * Arguments : Inputs - _in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7 + * Outputs - _out0, _out1, _out2, _out3, _out4, _out5, _out6, + * _out7 + * Details : The rows of the matrix become columns, and the columns + * become rows. + * Example : LSX_TRANSPOSE8x8_B + * _in0 : 00,01,02,03,04,05,06,07, 00,00,00,00,00,00,00,00 + * _in1 : 10,11,12,13,14,15,16,17, 00,00,00,00,00,00,00,00 + * _in2 : 20,21,22,23,24,25,26,27, 00,00,00,00,00,00,00,00 + * _in3 : 30,31,32,33,34,35,36,37, 00,00,00,00,00,00,00,00 + * _in4 : 40,41,42,43,44,45,46,47, 00,00,00,00,00,00,00,00 + * _in5 : 50,51,52,53,54,55,56,57, 00,00,00,00,00,00,00,00 + * _in6 : 60,61,62,63,64,65,66,67, 00,00,00,00,00,00,00,00 + * _in7 : 70,71,72,73,74,75,76,77, 00,00,00,00,00,00,00,00 + * + * _ out0 : 00,10,20,30,40,50,60,70, 00,00,00,00,00,00,00,00 + * _ out1 : 01,11,21,31,41,51,61,71, 00,00,00,00,00,00,00,00 + * _ out2 : 02,12,22,32,42,52,62,72, 00,00,00,00,00,00,00,00 + * _ out3 : 03,13,23,33,43,53,63,73, 00,00,00,00,00,00,00,00 + * _ out4 : 04,14,24,34,44,54,64,74, 00,00,00,00,00,00,00,00 + * _ out5 : 05,15,25,35,45,55,65,75, 00,00,00,00,00,00,00,00 + * _ out6 : 06,16,26,36,46,56,66,76, 00,00,00,00,00,00,00,00 + * _ out7 : 07,17,27,37,47,57,67,77, 00,00,00,00,00,00,00,00 + * ============================================================================= + */ +#define LSX_TRANSPOSE8x8_B(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ + _out7) \ + { \ + __m128i zero = { 0 }; \ + __m128i shuf8 = { 0x0F0E0D0C0B0A0908, 0x1716151413121110 }; \ + __m128i _t0, _t1, _t2, _t3, _t4, _t5, _t6, _t7; \ + \ + _t0 = __lsx_vilvl_b(_in2, _in0); \ + _t1 = __lsx_vilvl_b(_in3, _in1); \ + _t2 = __lsx_vilvl_b(_in6, _in4); \ + _t3 = __lsx_vilvl_b(_in7, _in5); \ + _t4 = __lsx_vilvl_b(_t1, _t0); \ + _t5 = __lsx_vilvh_b(_t1, _t0); \ + _t6 = __lsx_vilvl_b(_t3, _t2); \ + _t7 = __lsx_vilvh_b(_t3, _t2); \ + _out0 = __lsx_vilvl_w(_t6, _t4); \ + _out2 = __lsx_vilvh_w(_t6, _t4); \ + _out4 = __lsx_vilvl_w(_t7, _t5); \ + _out6 = __lsx_vilvh_w(_t7, _t5); \ + _out1 = __lsx_vshuf_b(zero, _out0, shuf8); \ + _out3 = __lsx_vshuf_b(zero, _out2, shuf8); \ + _out5 = __lsx_vshuf_b(zero, _out4, shuf8); \ + _out7 = __lsx_vshuf_b(zero, _out6, shuf8); \ + } + +/* + * ============================================================================= + * Description : Transpose 8x8 block with half-word elements in vectors + * Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7 + * Outputs - out0, out1, out2, out3, out4, out5, out6, out7 + * Details : + * Example : + * 00,01,02,03,04,05,06,07 00,10,20,30,40,50,60,70 + * 10,11,12,13,14,15,16,17 01,11,21,31,41,51,61,71 + * 20,21,22,23,24,25,26,27 02,12,22,32,42,52,62,72 + * 30,31,32,33,34,35,36,37 to 03,13,23,33,43,53,63,73 + * 40,41,42,43,44,45,46,47 ======> 04,14,24,34,44,54,64,74 + * 50,51,52,53,54,55,56,57 05,15,25,35,45,55,65,75 + * 60,61,62,63,64,65,66,67 06,16,26,36,46,56,66,76 + * 70,71,72,73,74,75,76,77 07,17,27,37,47,57,67,77 + * ============================================================================= + */ +#define LSX_TRANSPOSE8x8_H(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ + _out7) \ + { \ + __m128i _s0, _s1, _t0, _t1, _t2, _t3, _t4, _t5, _t6, _t7; \ + \ + _s0 = __lsx_vilvl_h(_in6, _in4); \ + _s1 = __lsx_vilvl_h(_in7, _in5); \ + _t0 = __lsx_vilvl_h(_s1, _s0); \ + _t1 = __lsx_vilvh_h(_s1, _s0); \ + _s0 = __lsx_vilvh_h(_in6, _in4); \ + _s1 = __lsx_vilvh_h(_in7, _in5); \ + _t2 = __lsx_vilvl_h(_s1, _s0); \ + _t3 = __lsx_vilvh_h(_s1, _s0); \ + _s0 = __lsx_vilvl_h(_in2, _in0); \ + _s1 = __lsx_vilvl_h(_in3, _in1); \ + _t4 = __lsx_vilvl_h(_s1, _s0); \ + _t5 = __lsx_vilvh_h(_s1, _s0); \ + _s0 = __lsx_vilvh_h(_in2, _in0); \ + _s1 = __lsx_vilvh_h(_in3, _in1); \ + _t6 = __lsx_vilvl_h(_s1, _s0); \ + _t7 = __lsx_vilvh_h(_s1, _s0); \ + \ + _out0 = __lsx_vpickev_d(_t0, _t4); \ + _out2 = __lsx_vpickev_d(_t1, _t5); \ + _out4 = __lsx_vpickev_d(_t2, _t6); \ + _out6 = __lsx_vpickev_d(_t3, _t7); \ + _out1 = __lsx_vpickod_d(_t0, _t4); \ + _out3 = __lsx_vpickod_d(_t1, _t5); \ + _out5 = __lsx_vpickod_d(_t2, _t6); \ + _out7 = __lsx_vpickod_d(_t3, _t7); \ + } + +/* + * ============================================================================= + * Description : Transpose input 8x4 byte block into 4x8 + * Arguments : Inputs - _in0, _in1, _in2, _in3 (input 8x4 byte block) + * Outputs - _out0, _out1, _out2, _out3 (output 4x8 byte block) + * Return Type - as per RTYPE + * Details : The rows of the matrix become columns, and the columns become + * rows. + * Example : LSX_TRANSPOSE8x4_B + * _in0 : 00,01,02,03,00,00,00,00, 00,00,00,00,00,00,00,00 + * _in1 : 10,11,12,13,00,00,00,00, 00,00,00,00,00,00,00,00 + * _in2 : 20,21,22,23,00,00,00,00, 00,00,00,00,00,00,00,00 + * _in3 : 30,31,32,33,00,00,00,00, 00,00,00,00,00,00,00,00 + * _in4 : 40,41,42,43,00,00,00,00, 00,00,00,00,00,00,00,00 + * _in5 : 50,51,52,53,00,00,00,00, 00,00,00,00,00,00,00,00 + * _in6 : 60,61,62,63,00,00,00,00, 00,00,00,00,00,00,00,00 + * _in7 : 70,71,72,73,00,00,00,00, 00,00,00,00,00,00,00,00 + * + * _out0 : 00,10,20,30,40,50,60,70, 00,00,00,00,00,00,00,00 + * _out1 : 01,11,21,31,41,51,61,71, 00,00,00,00,00,00,00,00 + * _out2 : 02,12,22,32,42,52,62,72, 00,00,00,00,00,00,00,00 + * _out3 : 03,13,23,33,43,53,63,73, 00,00,00,00,00,00,00,00 + * ============================================================================= + */ +#define LSX_TRANSPOSE8x4_B(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3) \ + { \ + __m128i _tmp0_m, _tmp1_m, _tmp2_m, _tmp3_m; \ + \ + _tmp0_m = __lsx_vpackev_w(_in4, _in0); \ + _tmp1_m = __lsx_vpackev_w(_in5, _in1); \ + _tmp2_m = __lsx_vilvl_b(_tmp1_m, _tmp0_m); \ + _tmp0_m = __lsx_vpackev_w(_in6, _in2); \ + _tmp1_m = __lsx_vpackev_w(_in7, _in3); \ + \ + _tmp3_m = __lsx_vilvl_b(_tmp1_m, _tmp0_m); \ + _tmp0_m = __lsx_vilvl_h(_tmp3_m, _tmp2_m); \ + _tmp1_m = __lsx_vilvh_h(_tmp3_m, _tmp2_m); \ + \ + _out0 = __lsx_vilvl_w(_tmp1_m, _tmp0_m); \ + _out2 = __lsx_vilvh_w(_tmp1_m, _tmp0_m); \ + _out1 = __lsx_vilvh_d(_out2, _out0); \ + _out3 = __lsx_vilvh_d(_out0, _out2); \ + } + +/* + * ============================================================================= + * Description : Transpose 16x8 block with byte elements in vectors + * Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7, in8 + * in9, in10, in11, in12, in13, in14, in15 + * Outputs - out0, out1, out2, out3, out4, out5, out6, out7 + * Details : + * Example : + * 000,001,002,003,004,005,006,007 + * 008,009,010,011,012,013,014,015 + * 016,017,018,019,020,021,022,023 + * 024,025,026,027,028,029,030,031 + * 032,033,034,035,036,037,038,039 + * 040,041,042,043,044,045,046,047 000,008,...,112,120 + * 048,049,050,051,052,053,054,055 001,009,...,113,121 + * 056,057,058,059,060,061,062,063 to 002,010,...,114,122 + * 064,068,066,067,068,069,070,071 =====> 003,011,...,115,123 + * 072,073,074,075,076,077,078,079 004,012,...,116,124 + * 080,081,082,083,084,085,086,087 005,013,...,117,125 + * 088,089,090,091,092,093,094,095 006,014,...,118,126 + * 096,097,098,099,100,101,102,103 007,015,...,119,127 + * 104,105,106,107,108,109,110,111 + * 112,113,114,115,116,117,118,119 + * 120,121,122,123,124,125,126,127 + * ============================================================================= + */ +#define LSX_TRANSPOSE16x8_B(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _in8, _in9, _in10, _in11, _in12, _in13, _in14, \ + _in15, _out0, _out1, _out2, _out3, _out4, _out5, \ + _out6, _out7) \ + { \ + __m128i _tmp0, _tmp1, _tmp2, _tmp3, _tmp4, _tmp5, _tmp6, _tmp7; \ + __m128i _t0, _t1, _t2, _t3, _t4, _t5, _t6, _t7; \ + DUP4_ARG2(__lsx_vilvl_b, _in2, _in0, _in3, _in1, _in6, _in4, _in7, _in5, \ + _tmp0, _tmp1, _tmp2, _tmp3); \ + DUP4_ARG2(__lsx_vilvl_b, _in10, _in8, _in11, _in9, _in14, _in12, _in15, \ + _in13, _tmp4, _tmp5, _tmp6, _tmp7); \ + DUP2_ARG2(__lsx_vilvl_b, _tmp1, _tmp0, _tmp3, _tmp2, _t0, _t2); \ + DUP2_ARG2(__lsx_vilvh_b, _tmp1, _tmp0, _tmp3, _tmp2, _t1, _t3); \ + DUP2_ARG2(__lsx_vilvl_b, _tmp5, _tmp4, _tmp7, _tmp6, _t4, _t6); \ + DUP2_ARG2(__lsx_vilvh_b, _tmp5, _tmp4, _tmp7, _tmp6, _t5, _t7); \ + DUP2_ARG2(__lsx_vilvl_w, _t2, _t0, _t3, _t1, _tmp0, _tmp4); \ + DUP2_ARG2(__lsx_vilvh_w, _t2, _t0, _t3, _t1, _tmp2, _tmp6); \ + DUP2_ARG2(__lsx_vilvl_w, _t6, _t4, _t7, _t5, _tmp1, _tmp5); \ + DUP2_ARG2(__lsx_vilvh_w, _t6, _t4, _t7, _t5, _tmp3, _tmp7); \ + DUP2_ARG2(__lsx_vilvl_d, _tmp1, _tmp0, _tmp3, _tmp2, _out0, _out2); \ + DUP2_ARG2(__lsx_vilvh_d, _tmp1, _tmp0, _tmp3, _tmp2, _out1, _out3); \ + DUP2_ARG2(__lsx_vilvl_d, _tmp5, _tmp4, _tmp7, _tmp6, _out4, _out6); \ + DUP2_ARG2(__lsx_vilvh_d, _tmp5, _tmp4, _tmp7, _tmp6, _out5, _out7); \ + } + +/* + * ============================================================================= + * Description : Butterfly of 4 input vectors + * Arguments : Inputs - in0, in1, in2, in3 + * Outputs - out0, out1, out2, out3 + * Details : Butterfly operation + * Example : + * out0 = in0 + in3; + * out1 = in1 + in2; + * out2 = in1 - in2; + * out3 = in0 - in3; + * ============================================================================= + */ +#define LSX_BUTTERFLY_4_B(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ + { \ + _out0 = __lsx_vadd_b(_in0, _in3); \ + _out1 = __lsx_vadd_b(_in1, _in2); \ + _out2 = __lsx_vsub_b(_in1, _in2); \ + _out3 = __lsx_vsub_b(_in0, _in3); \ + } +#define LSX_BUTTERFLY_4_H(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ + { \ + _out0 = __lsx_vadd_h(_in0, _in3); \ + _out1 = __lsx_vadd_h(_in1, _in2); \ + _out2 = __lsx_vsub_h(_in1, _in2); \ + _out3 = __lsx_vsub_h(_in0, _in3); \ + } +#define LSX_BUTTERFLY_4_W(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ + { \ + _out0 = __lsx_vadd_w(_in0, _in3); \ + _out1 = __lsx_vadd_w(_in1, _in2); \ + _out2 = __lsx_vsub_w(_in1, _in2); \ + _out3 = __lsx_vsub_w(_in0, _in3); \ + } +#define LSX_BUTTERFLY_4_D(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ + { \ + _out0 = __lsx_vadd_d(_in0, _in3); \ + _out1 = __lsx_vadd_d(_in1, _in2); \ + _out2 = __lsx_vsub_d(_in1, _in2); \ + _out3 = __lsx_vsub_d(_in0, _in3); \ + } + +/* + * ============================================================================= + * Description : Butterfly of 8 input vectors + * Arguments : Inputs - _in0, _in1, _in2, _in3, ~ + * Outputs - _out0, _out1, _out2, _out3, ~ + * Details : Butterfly operation + * Example : + * _out0 = _in0 + _in7; + * _out1 = _in1 + _in6; + * _out2 = _in2 + _in5; + * _out3 = _in3 + _in4; + * _out4 = _in3 - _in4; + * _out5 = _in2 - _in5; + * _out6 = _in1 - _in6; + * _out7 = _in0 - _in7; + * ============================================================================= + */ +#define LSX_BUTTERFLY_8_B(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ + _out7) \ + { \ + _out0 = __lsx_vadd_b(_in0, _in7); \ + _out1 = __lsx_vadd_b(_in1, _in6); \ + _out2 = __lsx_vadd_b(_in2, _in5); \ + _out3 = __lsx_vadd_b(_in3, _in4); \ + _out4 = __lsx_vsub_b(_in3, _in4); \ + _out5 = __lsx_vsub_b(_in2, _in5); \ + _out6 = __lsx_vsub_b(_in1, _in6); \ + _out7 = __lsx_vsub_b(_in0, _in7); \ + } + +#define LSX_BUTTERFLY_8_H(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ + _out7) \ + { \ + _out0 = __lsx_vadd_h(_in0, _in7); \ + _out1 = __lsx_vadd_h(_in1, _in6); \ + _out2 = __lsx_vadd_h(_in2, _in5); \ + _out3 = __lsx_vadd_h(_in3, _in4); \ + _out4 = __lsx_vsub_h(_in3, _in4); \ + _out5 = __lsx_vsub_h(_in2, _in5); \ + _out6 = __lsx_vsub_h(_in1, _in6); \ + _out7 = __lsx_vsub_h(_in0, _in7); \ + } + +#define LSX_BUTTERFLY_8_W(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ + _out7) \ + { \ + _out0 = __lsx_vadd_w(_in0, _in7); \ + _out1 = __lsx_vadd_w(_in1, _in6); \ + _out2 = __lsx_vadd_w(_in2, _in5); \ + _out3 = __lsx_vadd_w(_in3, _in4); \ + _out4 = __lsx_vsub_w(_in3, _in4); \ + _out5 = __lsx_vsub_w(_in2, _in5); \ + _out6 = __lsx_vsub_w(_in1, _in6); \ + _out7 = __lsx_vsub_w(_in0, _in7); \ + } + +#define LSX_BUTTERFLY_8_D(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ + _out7) \ + { \ + _out0 = __lsx_vadd_d(_in0, _in7); \ + _out1 = __lsx_vadd_d(_in1, _in6); \ + _out2 = __lsx_vadd_d(_in2, _in5); \ + _out3 = __lsx_vadd_d(_in3, _in4); \ + _out4 = __lsx_vsub_d(_in3, _in4); \ + _out5 = __lsx_vsub_d(_in2, _in5); \ + _out6 = __lsx_vsub_d(_in1, _in6); \ + _out7 = __lsx_vsub_d(_in0, _in7); \ + } + +/* + * ============================================================================= + * Description : Butterfly of 16 input vectors + * Arguments : Inputs - _in0, _in1, _in2, _in3, ~ + * Outputs - _out0, _out1, _out2, _out3, ~ + * Details : Butterfly operation + * Example : + * _out0 = _in0 + _in15; + * _out1 = _in1 + _in14; + * _out2 = _in2 + _in13; + * _out3 = _in3 + _in12; + * _out4 = _in4 + _in11; + * _out5 = _in5 + _in10; + * _out6 = _in6 + _in9; + * _out7 = _in7 + _in8; + * _out8 = _in7 - _in8; + * _out9 = _in6 - _in9; + * _out10 = _in5 - _in10; + * _out11 = _in4 - _in11; + * _out12 = _in3 - _in12; + * _out13 = _in2 - _in13; + * _out14 = _in1 - _in14; + * _out15 = _in0 - _in15; + * ============================================================================= + */ + +#define LSX_BUTTERFLY_16_B(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _in8, _in9, _in10, _in11, _in12, _in13, _in14, \ + _in15, _out0, _out1, _out2, _out3, _out4, _out5, \ + _out6, _out7, _out8, _out9, _out10, _out11, _out12, \ + _out13, _out14, _out15) \ + { \ + _out0 = __lsx_vadd_b(_in0, _in15); \ + _out1 = __lsx_vadd_b(_in1, _in14); \ + _out2 = __lsx_vadd_b(_in2, _in13); \ + _out3 = __lsx_vadd_b(_in3, _in12); \ + _out4 = __lsx_vadd_b(_in4, _in11); \ + _out5 = __lsx_vadd_b(_in5, _in10); \ + _out6 = __lsx_vadd_b(_in6, _in9); \ + _out7 = __lsx_vadd_b(_in7, _in8); \ + \ + _out8 = __lsx_vsub_b(_in7, _in8); \ + _out9 = __lsx_vsub_b(_in6, _in9); \ + _out10 = __lsx_vsub_b(_in5, _in10); \ + _out11 = __lsx_vsub_b(_in4, _in11); \ + _out12 = __lsx_vsub_b(_in3, _in12); \ + _out13 = __lsx_vsub_b(_in2, _in13); \ + _out14 = __lsx_vsub_b(_in1, _in14); \ + _out15 = __lsx_vsub_b(_in0, _in15); \ + } + +#define LSX_BUTTERFLY_16_H(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _in8, _in9, _in10, _in11, _in12, _in13, _in14, \ + _in15, _out0, _out1, _out2, _out3, _out4, _out5, \ + _out6, _out7, _out8, _out9, _out10, _out11, _out12, \ + _out13, _out14, _out15) \ + { \ + _out0 = __lsx_vadd_h(_in0, _in15); \ + _out1 = __lsx_vadd_h(_in1, _in14); \ + _out2 = __lsx_vadd_h(_in2, _in13); \ + _out3 = __lsx_vadd_h(_in3, _in12); \ + _out4 = __lsx_vadd_h(_in4, _in11); \ + _out5 = __lsx_vadd_h(_in5, _in10); \ + _out6 = __lsx_vadd_h(_in6, _in9); \ + _out7 = __lsx_vadd_h(_in7, _in8); \ + \ + _out8 = __lsx_vsub_h(_in7, _in8); \ + _out9 = __lsx_vsub_h(_in6, _in9); \ + _out10 = __lsx_vsub_h(_in5, _in10); \ + _out11 = __lsx_vsub_h(_in4, _in11); \ + _out12 = __lsx_vsub_h(_in3, _in12); \ + _out13 = __lsx_vsub_h(_in2, _in13); \ + _out14 = __lsx_vsub_h(_in1, _in14); \ + _out15 = __lsx_vsub_h(_in0, _in15); \ + } + +#define LSX_BUTTERFLY_16_W(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _in8, _in9, _in10, _in11, _in12, _in13, _in14, \ + _in15, _out0, _out1, _out2, _out3, _out4, _out5, \ + _out6, _out7, _out8, _out9, _out10, _out11, _out12, \ + _out13, _out14, _out15) \ + { \ + _out0 = __lsx_vadd_w(_in0, _in15); \ + _out1 = __lsx_vadd_w(_in1, _in14); \ + _out2 = __lsx_vadd_w(_in2, _in13); \ + _out3 = __lsx_vadd_w(_in3, _in12); \ + _out4 = __lsx_vadd_w(_in4, _in11); \ + _out5 = __lsx_vadd_w(_in5, _in10); \ + _out6 = __lsx_vadd_w(_in6, _in9); \ + _out7 = __lsx_vadd_w(_in7, _in8); \ + \ + _out8 = __lsx_vsub_w(_in7, _in8); \ + _out9 = __lsx_vsub_w(_in6, _in9); \ + _out10 = __lsx_vsub_w(_in5, _in10); \ + _out11 = __lsx_vsub_w(_in4, _in11); \ + _out12 = __lsx_vsub_w(_in3, _in12); \ + _out13 = __lsx_vsub_w(_in2, _in13); \ + _out14 = __lsx_vsub_w(_in1, _in14); \ + _out15 = __lsx_vsub_w(_in0, _in15); \ + } + +#define LSX_BUTTERFLY_16_D(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _in8, _in9, _in10, _in11, _in12, _in13, _in14, \ + _in15, _out0, _out1, _out2, _out3, _out4, _out5, \ + _out6, _out7, _out8, _out9, _out10, _out11, _out12, \ + _out13, _out14, _out15) \ + { \ + _out0 = __lsx_vadd_d(_in0, _in15); \ + _out1 = __lsx_vadd_d(_in1, _in14); \ + _out2 = __lsx_vadd_d(_in2, _in13); \ + _out3 = __lsx_vadd_d(_in3, _in12); \ + _out4 = __lsx_vadd_d(_in4, _in11); \ + _out5 = __lsx_vadd_d(_in5, _in10); \ + _out6 = __lsx_vadd_d(_in6, _in9); \ + _out7 = __lsx_vadd_d(_in7, _in8); \ + \ + _out8 = __lsx_vsub_d(_in7, _in8); \ + _out9 = __lsx_vsub_d(_in6, _in9); \ + _out10 = __lsx_vsub_d(_in5, _in10); \ + _out11 = __lsx_vsub_d(_in4, _in11); \ + _out12 = __lsx_vsub_d(_in3, _in12); \ + _out13 = __lsx_vsub_d(_in2, _in13); \ + _out14 = __lsx_vsub_d(_in1, _in14); \ + _out15 = __lsx_vsub_d(_in0, _in15); \ + } + +#endif // LSX + +#ifdef __loongarch_asx +#include +/* + * ============================================================================= + * Description : Dot product of byte vector elements + * Arguments : Inputs - in_h, in_l + * Output - out + * Return Type - signed halfword + * Details : Unsigned byte elements from in_h are multiplied with + * unsigned byte elements from in_l producing a result + * twice the size of input i.e. signed halfword. + * Then these multiplied results of adjacent odd-even elements + * are added to the out vector + * Example : See out = __lasx_xvdp2_w_h(in_h, in_l) + * ============================================================================= + */ +static inline __m256i __lasx_xvdp2_h_bu(__m256i in_h, __m256i in_l) { + __m256i out; + + out = __lasx_xvmulwev_h_bu(in_h, in_l); + out = __lasx_xvmaddwod_h_bu(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of byte vector elements + * Arguments : Inputs - in_h, in_l + * Output - out + * Return Type - signed halfword + * Details : Signed byte elements from in_h are multiplied with + * signed byte elements from in_l producing a result + * twice the size of input i.e. signed halfword. + * Then these multiplication results of adjacent odd-even elements + * are added to the out vector + * Example : See out = __lasx_xvdp2_w_h(in_h, in_l) + * ============================================================================= + */ +static inline __m256i __lasx_xvdp2_h_b(__m256i in_h, __m256i in_l) { + __m256i out; + + out = __lasx_xvmulwev_h_b(in_h, in_l); + out = __lasx_xvmaddwod_h_b(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of halfword vector elements + * Arguments : Inputs - in_h, in_l + * Output - out + * Return Type - signed word + * Details : Signed halfword elements from in_h are multiplied with + * signed halfword elements from in_l producing a result + * twice the size of input i.e. signed word. + * Then these multiplied results of adjacent odd-even elements + * are added to the out vector. + * Example : out = __lasx_xvdp2_w_h(in_h, in_l) + * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * in_l : 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,1 + * out : 22,38,38,22, 22,38,38,22 + * ============================================================================= + */ +static inline __m256i __lasx_xvdp2_w_h(__m256i in_h, __m256i in_l) { + __m256i out; + + out = __lasx_xvmulwev_w_h(in_h, in_l); + out = __lasx_xvmaddwod_w_h(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of word vector elements + * Arguments : Inputs - in_h, in_l + * Output - out + * Return Type - signed double + * Details : Signed word elements from in_h are multiplied with + * signed word elements from in_l producing a result + * twice the size of input i.e. signed double-word. + * Then these multiplied results of adjacent odd-even elements + * are added to the out vector. + * Example : See out = __lasx_xvdp2_w_h(in_h, in_l) + * ============================================================================= + */ +static inline __m256i __lasx_xvdp2_d_w(__m256i in_h, __m256i in_l) { + __m256i out; + + out = __lasx_xvmulwev_d_w(in_h, in_l); + out = __lasx_xvmaddwod_d_w(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of halfword vector elements + * Arguments : Inputs - in_h, in_l + * Output - out + * Return Type - signed word + * Details : Unsigned halfword elements from in_h are multiplied with + * signed halfword elements from in_l producing a result + * twice the size of input i.e. unsigned word. + * Multiplication result of adjacent odd-even elements + * are added to the out vector + * Example : See out = __lasx_xvdp2_w_h(in_h, in_l) + * ============================================================================= + */ +static inline __m256i __lasx_xvdp2_w_hu_h(__m256i in_h, __m256i in_l) { + __m256i out; + + out = __lasx_xvmulwev_w_hu_h(in_h, in_l); + out = __lasx_xvmaddwod_w_hu_h(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product & addition of byte vector elements + * Arguments : Inputs - in_h, in_l + * Output - out + * Return Type - halfword + * Details : Signed byte elements from in_h are multiplied with + * signed byte elements from in_l producing a result + * twice the size of input i.e. signed halfword. + * Then these multiplied results of adjacent odd-even elements + * are added to the in_c vector. + * Example : See out = __lasx_xvdp2add_w_h(in_c, in_h, in_l) + * ============================================================================= + */ +static inline __m256i __lasx_xvdp2add_h_b(__m256i in_c, __m256i in_h, + __m256i in_l) { + __m256i out; + + out = __lasx_xvmaddwev_h_b(in_c, in_h, in_l); + out = __lasx_xvmaddwod_h_b(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product & addition of byte vector elements + * Arguments : Inputs - in_h, in_l + * Output - out + * Return Type - halfword + * Details : Unsigned byte elements from in_h are multiplied with + * unsigned byte elements from in_l producing a result + * twice the size of input i.e. signed halfword. + * Then these multiplied results of adjacent odd-even elements + * are added to the in_c vector. + * Example : See out = __lasx_xvdp2add_w_h(in_c, in_h, in_l) + * ============================================================================= + */ +static inline __m256i __lasx_xvdp2add_h_bu(__m256i in_c, __m256i in_h, + __m256i in_l) { + __m256i out; + + out = __lasx_xvmaddwev_h_bu(in_c, in_h, in_l); + out = __lasx_xvmaddwod_h_bu(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product & addition of byte vector elements + * Arguments : Inputs - in_h, in_l + * Output - out + * Return Type - halfword + * Details : Unsigned byte elements from in_h are multiplied with + * signed byte elements from in_l producing a result + * twice the size of input i.e. signed halfword. + * Then these multiplied results of adjacent odd-even elements + * are added to the in_c vector. + * Example : See out = __lasx_xvdp2add_w_h(in_c, in_h, in_l) + * ============================================================================= + */ +static inline __m256i __lasx_xvdp2add_h_bu_b(__m256i in_c, __m256i in_h, + __m256i in_l) { + __m256i out; + + out = __lasx_xvmaddwev_h_bu_b(in_c, in_h, in_l); + out = __lasx_xvmaddwod_h_bu_b(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of halfword vector elements + * Arguments : Inputs - in_c, in_h, in_l + * Output - out + * Return Type - per RTYPE + * Details : Signed halfword elements from in_h are multiplied with + * signed halfword elements from in_l producing a result + * twice the size of input i.e. signed word. + * Multiplication result of adjacent odd-even elements + * are added to the in_c vector. + * Example : out = __lasx_xvdp2add_w_h(in_c, in_h, in_l) + * in_c : 1,2,3,4, 1,2,3,4 + * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8, + * in_l : 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,1, + * out : 23,40,41,26, 23,40,41,26 + * ============================================================================= + */ +static inline __m256i __lasx_xvdp2add_w_h(__m256i in_c, __m256i in_h, + __m256i in_l) { + __m256i out; + + out = __lasx_xvmaddwev_w_h(in_c, in_h, in_l); + out = __lasx_xvmaddwod_w_h(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of halfword vector elements + * Arguments : Inputs - in_c, in_h, in_l + * Output - out + * Return Type - signed word + * Details : Unsigned halfword elements from in_h are multiplied with + * unsigned halfword elements from in_l producing a result + * twice the size of input i.e. signed word. + * Multiplication result of adjacent odd-even elements + * are added to the in_c vector. + * Example : See out = __lasx_xvdp2add_w_h(in_c, in_h, in_l) + * ============================================================================= + */ +static inline __m256i __lasx_xvdp2add_w_hu(__m256i in_c, __m256i in_h, + __m256i in_l) { + __m256i out; + + out = __lasx_xvmaddwev_w_hu(in_c, in_h, in_l); + out = __lasx_xvmaddwod_w_hu(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of halfword vector elements + * Arguments : Inputs - in_c, in_h, in_l + * Output - out + * Return Type - signed word + * Details : Unsigned halfword elements from in_h are multiplied with + * signed halfword elements from in_l producing a result + * twice the size of input i.e. signed word. + * Multiplication result of adjacent odd-even elements + * are added to the in_c vector + * Example : See out = __lasx_xvdp2add_w_h(in_c, in_h, in_l) + * ============================================================================= + */ +static inline __m256i __lasx_xvdp2add_w_hu_h(__m256i in_c, __m256i in_h, + __m256i in_l) { + __m256i out; + + out = __lasx_xvmaddwev_w_hu_h(in_c, in_h, in_l); + out = __lasx_xvmaddwod_w_hu_h(out, in_h, in_l); + return out; +} + +/* + * ============================================================================= + * Description : Vector Unsigned Dot Product and Subtract + * Arguments : Inputs - in_c, in_h, in_l + * Output - out + * Return Type - signed halfword + * Details : Unsigned byte elements from in_h are multiplied with + * unsigned byte elements from in_l producing a result + * twice the size of input i.e. signed halfword. + * Multiplication result of adjacent odd-even elements + * are added together and subtracted from double width elements + * in_c vector. + * Example : See out = __lasx_xvdp2sub_w_h(in_c, in_h, in_l) + * ============================================================================= + */ +static inline __m256i __lasx_xvdp2sub_h_bu(__m256i in_c, __m256i in_h, + __m256i in_l) { + __m256i out; + + out = __lasx_xvmulwev_h_bu(in_h, in_l); + out = __lasx_xvmaddwod_h_bu(out, in_h, in_l); + out = __lasx_xvsub_h(in_c, out); + return out; +} + +/* + * ============================================================================= + * Description : Vector Signed Dot Product and Subtract + * Arguments : Inputs - in_c, in_h, in_l + * Output - out + * Return Type - signed word + * Details : Signed halfword elements from in_h are multiplied with + * Signed halfword elements from in_l producing a result + * twice the size of input i.e. signed word. + * Multiplication result of adjacent odd-even elements + * are added together and subtracted from double width elements + * in_c vector. + * Example : out = __lasx_xvdp2sub_w_h(in_c, in_h, in_l) + * in_c : 0,0,0,0, 0,0,0,0 + * in_h : 3,1,3,0, 0,0,0,1, 0,0,1,1, 0,0,0,1 + * in_l : 2,1,1,0, 1,0,0,0, 0,0,1,0, 1,0,0,1 + * out : -7,-3,0,0, 0,-1,0,-1 + * ============================================================================= + */ +static inline __m256i __lasx_xvdp2sub_w_h(__m256i in_c, __m256i in_h, + __m256i in_l) { + __m256i out; + + out = __lasx_xvmulwev_w_h(in_h, in_l); + out = __lasx_xvmaddwod_w_h(out, in_h, in_l); + out = __lasx_xvsub_w(in_c, out); + return out; +} + +/* + * ============================================================================= + * Description : Dot product of halfword vector elements + * Arguments : Inputs - in_h, in_l + * Output - out + * Return Type - signed word + * Details : Signed halfword elements from in_h are multiplied with + * signed halfword elements from in_l producing a result + * four times the size of input i.e. signed doubleword. + * Then these multiplication results of four adjacent elements + * are added together and stored to the out vector. + * Example : out = __lasx_xvdp4_d_h(in_h, in_l) + * in_h : 3,1,3,0, 0,0,0,1, 0,0,1,-1, 0,0,0,1 + * in_l : -2,1,1,0, 1,0,0,0, 0,0,1, 0, 1,0,0,1 + * out : -2,0,1,1 + * ============================================================================= + */ +static inline __m256i __lasx_xvdp4_d_h(__m256i in_h, __m256i in_l) { + __m256i out; + + out = __lasx_xvmulwev_w_h(in_h, in_l); + out = __lasx_xvmaddwod_w_h(out, in_h, in_l); + out = __lasx_xvhaddw_d_w(out, out); + return out; +} + +/* + * ============================================================================= + * Description : The high half of the vector elements are expanded and + * added after being doubled. + * Arguments : Inputs - in_h, in_l + * Output - out + * Details : The in_h vector and the in_l vector are added after the + * higher half of the two-fold sign extension (signed byte + * to signed halfword) and stored to the out vector. + * Example : See out = __lasx_xvaddwh_w_h(in_h, in_l) + * ============================================================================= + */ +static inline __m256i __lasx_xvaddwh_h_b(__m256i in_h, __m256i in_l) { + __m256i out; + + out = __lasx_xvilvh_b(in_h, in_l); + out = __lasx_xvhaddw_h_b(out, out); + return out; +} + +/* + * ============================================================================= + * Description : The high half of the vector elements are expanded and + * added after being doubled. + * Arguments : Inputs - in_h, in_l + * Output - out + * Details : The in_h vector and the in_l vector are added after the + * higher half of the two-fold sign extension (signed halfword + * to signed word) and stored to the out vector. + * Example : out = __lasx_xvaddwh_w_h(in_h, in_l) + * in_h : 3, 0,3,0, 0,0,0,-1, 0,0,1,-1, 0,0,0,1 + * in_l : 2,-1,1,2, 1,0,0, 0, 1,0,1, 0, 1,0,0,1 + * out : 1,0,0,-1, 1,0,0, 2 + * ============================================================================= + */ +static inline __m256i __lasx_xvaddwh_w_h(__m256i in_h, __m256i in_l) { + __m256i out; + + out = __lasx_xvilvh_h(in_h, in_l); + out = __lasx_xvhaddw_w_h(out, out); + return out; +} + +/* + * ============================================================================= + * Description : The low half of the vector elements are expanded and + * added after being doubled. + * Arguments : Inputs - in_h, in_l + * Output - out + * Details : The in_h vector and the in_l vector are added after the + * lower half of the two-fold sign extension (signed byte + * to signed halfword) and stored to the out vector. + * Example : See out = __lasx_xvaddwl_w_h(in_h, in_l) + * ============================================================================= + */ +static inline __m256i __lasx_xvaddwl_h_b(__m256i in_h, __m256i in_l) { + __m256i out; + + out = __lasx_xvilvl_b(in_h, in_l); + out = __lasx_xvhaddw_h_b(out, out); + return out; +} + +/* + * ============================================================================= + * Description : The low half of the vector elements are expanded and + * added after being doubled. + * Arguments : Inputs - in_h, in_l + * Output - out + * Details : The in_h vector and the in_l vector are added after the + * lower half of the two-fold sign extension (signed halfword + * to signed word) and stored to the out vector. + * Example : out = __lasx_xvaddwl_w_h(in_h, in_l) + * in_h : 3, 0,3,0, 0,0,0,-1, 0,0,1,-1, 0,0,0,1 + * in_l : 2,-1,1,2, 1,0,0, 0, 1,0,1, 0, 1,0,0,1 + * out : 5,-1,4,2, 1,0,2,-1 + * ============================================================================= + */ +static inline __m256i __lasx_xvaddwl_w_h(__m256i in_h, __m256i in_l) { + __m256i out; + + out = __lasx_xvilvl_h(in_h, in_l); + out = __lasx_xvhaddw_w_h(out, out); + return out; +} + +/* + * ============================================================================= + * Description : The low half of the vector elements are expanded and + * added after being doubled. + * Arguments : Inputs - in_h, in_l + * Output - out + * Details : The out vector and the out vector are added after the + * lower half of the two-fold zero extension (unsigned byte + * to unsigned halfword) and stored to the out vector. + * Example : See out = __lasx_xvaddwl_w_h(in_h, in_l) + * ============================================================================= + */ +static inline __m256i __lasx_xvaddwl_h_bu(__m256i in_h, __m256i in_l) { + __m256i out; + + out = __lasx_xvilvl_b(in_h, in_l); + out = __lasx_xvhaddw_hu_bu(out, out); + return out; +} + +/* + * ============================================================================= + * Description : The low half of the vector elements are expanded and + * added after being doubled. + * Arguments : Inputs - in_h, in_l + * Output - out + * Details : The in_l vector after double zero extension (unsigned byte to + * signed halfword)，added to the in_h vector. + * Example : See out = __lasx_xvaddw_w_w_h(in_h, in_l) + * ============================================================================= + */ +static inline __m256i __lasx_xvaddw_h_h_bu(__m256i in_h, __m256i in_l) { + __m256i out; + + out = __lasx_xvsllwil_hu_bu(in_l, 0); + out = __lasx_xvadd_h(in_h, out); + return out; +} + +/* + * ============================================================================= + * Description : The low half of the vector elements are expanded and + * added after being doubled. + * Arguments : Inputs - in_h, in_l + * Output - out + * Details : The in_l vector after double sign extension (signed halfword to + * signed word), added to the in_h vector. + * Example : out = __lasx_xvaddw_w_w_h(in_h, in_l) + * in_h : 0, 1,0,0, -1,0,0,1, + * in_l : 2,-1,1,2, 1,0,0,0, 0,0,1,0, 1,0,0,1, + * out : 2, 0,1,2, -1,0,1,1, + * ============================================================================= + */ +static inline __m256i __lasx_xvaddw_w_w_h(__m256i in_h, __m256i in_l) { + __m256i out; + + out = __lasx_xvsllwil_w_h(in_l, 0); + out = __lasx_xvadd_w(in_h, out); + return out; +} + +/* + * ============================================================================= + * Description : Multiplication and addition calculation after expansion + * of the lower half of the vector. + * Arguments : Inputs - in_c, in_h, in_l + * Output - out + * Details : The in_h vector and the in_l vector are multiplied after + * the lower half of the two-fold sign extension (signed halfword + * to signed word), and the result is added to the vector in_c, + * then stored to the out vector. + * Example : out = __lasx_xvmaddwl_w_h(in_c, in_h, in_l) + * in_c : 1,2,3,4, 5,6,7,8 + * in_h : 1,2,3,4, 1,2,3,4, 5,6,7,8, 5,6,7,8 + * in_l : 200, 300, 400, 500, 2000, 3000, 4000, 5000, + * -200,-300,-400,-500, -2000,-3000,-4000,-5000 + * out : 201, 602,1203,2004, -995, -1794,-2793,-3992 + * ============================================================================= + */ +static inline __m256i __lasx_xvmaddwl_w_h(__m256i in_c, __m256i in_h, + __m256i in_l) { + __m256i tmp0, tmp1, out; + + tmp0 = __lasx_xvsllwil_w_h(in_h, 0); + tmp1 = __lasx_xvsllwil_w_h(in_l, 0); + tmp0 = __lasx_xvmul_w(tmp0, tmp1); + out = __lasx_xvadd_w(tmp0, in_c); + return out; +} + +/* + * ============================================================================= + * Description : Multiplication and addition calculation after expansion + * of the higher half of the vector. + * Arguments : Inputs - in_c, in_h, in_l + * Output - out + * Details : The in_h vector and the in_l vector are multiplied after + * the higher half of the two-fold sign extension (signed + * halfword to signed word), and the result is added to + * the vector in_c, then stored to the out vector. + * Example : See out = __lasx_xvmaddwl_w_h(in_c, in_h, in_l) + * ============================================================================= + */ +static inline __m256i __lasx_xvmaddwh_w_h(__m256i in_c, __m256i in_h, + __m256i in_l) { + __m256i tmp0, tmp1, out; + + tmp0 = __lasx_xvilvh_h(in_h, in_h); + tmp1 = __lasx_xvilvh_h(in_l, in_l); + tmp0 = __lasx_xvmulwev_w_h(tmp0, tmp1); + out = __lasx_xvadd_w(tmp0, in_c); + return out; +} + +/* + * ============================================================================= + * Description : Multiplication calculation after expansion of the lower + * half of the vector. + * Arguments : Inputs - in_h, in_l + * Output - out + * Details : The in_h vector and the in_l vector are multiplied after + * the lower half of the two-fold sign extension (signed + * halfword to signed word), then stored to the out vector. + * Example : out = __lasx_xvmulwl_w_h(in_h, in_l) + * in_h : 3,-1,3,0, 0,0,0,-1, 0,0,1,-1, 0,0,0,1 + * in_l : 2,-1,1,2, 1,0,0, 0, 0,0,1, 0, 1,0,0,1 + * out : 6,1,3,0, 0,0,1,0 + * ============================================================================= + */ +static inline __m256i __lasx_xvmulwl_w_h(__m256i in_h, __m256i in_l) { + __m256i tmp0, tmp1, out; + + tmp0 = __lasx_xvsllwil_w_h(in_h, 0); + tmp1 = __lasx_xvsllwil_w_h(in_l, 0); + out = __lasx_xvmul_w(tmp0, tmp1); + return out; +} + +/* + * ============================================================================= + * Description : Multiplication calculation after expansion of the lower + * half of the vector. + * Arguments : Inputs - in_h, in_l + * Output - out + * Details : The in_h vector and the in_l vector are multiplied after + * the lower half of the two-fold sign extension (signed + * halfword to signed word), then stored to the out vector. + * Example : out = __lasx_xvmulwh_w_h(in_h, in_l) + * in_h : 3,-1,3,0, 0,0,0,-1, 0,0,1,-1, 0,0,0,1 + * in_l : 2,-1,1,2, 1,0,0, 0, 0,0,1, 0, 1,0,0,1 + * out : 0,0,0,0, 0,0,0,1 + * ============================================================================= + */ +static inline __m256i __lasx_xvmulwh_w_h(__m256i in_h, __m256i in_l) { + __m256i tmp0, tmp1, out; + + tmp0 = __lasx_xvilvh_h(in_h, in_h); + tmp1 = __lasx_xvilvh_h(in_l, in_l); + out = __lasx_xvmulwev_w_h(tmp0, tmp1); + return out; +} + +/* + * ============================================================================= + * Description : The low half of the vector elements are added to the high half + * after being doubled, then saturated. + * Arguments : Inputs - in_h, in_l + * Output - out + * Details : The in_h vector adds the in_l vector after the lower half of + * the two-fold zero extension (unsigned byte to unsigned + * halfword) and then saturated. The results are stored to the out + * vector. + * Example : out = __lasx_xvsaddw_hu_hu_bu(in_h, in_l) + * in_h : 2,65532,1,2, 1,0,0,0, 0,0,1,0, 1,0,0,1 + * in_l : 3,6,3,0, 0,0,0,1, 0,0,1,1, 0,0,0,1, 3,18,3,0, 0,0,0,1, 0,0,1,1, + * 0,0,0,1 + * out : 5,65535,4,2, 1,0,0,1, 3,18,4,0, 1,0,0,2, + * ============================================================================= + */ +static inline __m256i __lasx_xvsaddw_hu_hu_bu(__m256i in_h, __m256i in_l) { + __m256i tmp1, out; + __m256i zero = { 0 }; + + tmp1 = __lasx_xvilvl_b(zero, in_l); + out = __lasx_xvsadd_hu(in_h, tmp1); + return out; +} + +/* + * ============================================================================= + * Description : Clip all halfword elements of input vector between min & max + * out = ((in) < (min)) ? (min) : (((in) > (max)) ? (max) : (in)) + * Arguments : Inputs - in (input vector) + * - min (min threshold) + * - max (max threshold) + * Outputs - in (output vector with clipped elements) + * Return Type - signed halfword + * Example : out = __lasx_xvclip_h(in, min, max) + * in : -8,2,280,249, -8,255,280,249, 4,4,4,4, 5,5,5,5 + * min : 1,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1 + * max : 9,9,9,9, 9,9,9,9, 9,9,9,9, 9,9,9,9 + * out : 1,2,9,9, 1,9,9,9, 4,4,4,4, 5,5,5,5 + * ============================================================================= + */ +static inline __m256i __lasx_xvclip_h(__m256i in, __m256i min, __m256i max) { + __m256i out; + + out = __lasx_xvmax_h(min, in); + out = __lasx_xvmin_h(max, out); + return out; +} + +/* + * ============================================================================= + * Description : Clip all signed halfword elements of input vector + * between 0 & 255 + * Arguments : Inputs - in (input vector) + * Outputs - out (output vector with clipped elements) + * Return Type - signed halfword + * Example : See out = __lasx_xvclip255_w(in) + * ============================================================================= + */ +static inline __m256i __lasx_xvclip255_h(__m256i in) { + __m256i out; + + out = __lasx_xvmaxi_h(in, 0); + out = __lasx_xvsat_hu(out, 7); + return out; +} + +/* + * ============================================================================= + * Description : Clip all signed word elements of input vector + * between 0 & 255 + * Arguments : Inputs - in (input vector) + * Output - out (output vector with clipped elements) + * Return Type - signed word + * Example : out = __lasx_xvclip255_w(in) + * in : -8,255,280,249, -8,255,280,249 + * out : 0,255,255,249, 0,255,255,249 + * ============================================================================= + */ +static inline __m256i __lasx_xvclip255_w(__m256i in) { + __m256i out; + + out = __lasx_xvmaxi_w(in, 0); + out = __lasx_xvsat_wu(out, 7); + return out; +} + +/* + * ============================================================================= + * Description : Indexed halfword element values are replicated to all + * elements in output vector. If 'idx < 8' use xvsplati_l_*, + * if 'idx >= 8' use xvsplati_h_*. + * Arguments : Inputs - in, idx + * Output - out + * Details : Idx element value from in vector is replicated to all + * elements in out vector. + * Valid index range for halfword operation is 0-7 + * Example : out = __lasx_xvsplati_l_h(in, idx) + * in : 20,10,11,12, 13,14,15,16, 0,0,2,0, 0,0,0,0 + * idx : 0x02 + * out : 11,11,11,11, 11,11,11,11, 11,11,11,11, 11,11,11,11 + * ============================================================================= + */ +static inline __m256i __lasx_xvsplati_l_h(__m256i in, int idx) { + __m256i out; + + out = __lasx_xvpermi_q(in, in, 0x02); + out = __lasx_xvreplve_h(out, idx); + return out; +} + +/* + * ============================================================================= + * Description : Indexed halfword element values are replicated to all + * elements in output vector. If 'idx < 8' use xvsplati_l_*, + * if 'idx >= 8' use xvsplati_h_*. + * Arguments : Inputs - in, idx + * Output - out + * Details : Idx element value from in vector is replicated to all + * elements in out vector. + * Valid index range for halfword operation is 0-7 + * Example : out = __lasx_xvsplati_h_h(in, idx) + * in : 20,10,11,12, 13,14,15,16, 0,2,0,0, 0,0,0,0 + * idx : 0x09 + * out : 2,2,2,2, 2,2,2,2, 2,2,2,2, 2,2,2,2 + * ============================================================================= + */ +static inline __m256i __lasx_xvsplati_h_h(__m256i in, int idx) { + __m256i out; + + out = __lasx_xvpermi_q(in, in, 0x13); + out = __lasx_xvreplve_h(out, idx); + return out; +} + +/* + * ============================================================================= + * Description : Transpose 4x4 block with double-word elements in vectors + * Arguments : Inputs - _in0, _in1, _in2, _in3 + * Outputs - _out0, _out1, _out2, _out3 + * Example : LASX_TRANSPOSE4x4_D + * _in0 : 1,2,3,4 + * _in1 : 1,2,3,4 + * _in2 : 1,2,3,4 + * _in3 : 1,2,3,4 + * + * _out0 : 1,1,1,1 + * _out1 : 2,2,2,2 + * _out2 : 3,3,3,3 + * _out3 : 4,4,4,4 + * ============================================================================= + */ +#define LASX_TRANSPOSE4x4_D(_in0, _in1, _in2, _in3, _out0, _out1, _out2, \ + _out3) \ + { \ + __m256i _tmp0, _tmp1, _tmp2, _tmp3; \ + _tmp0 = __lasx_xvilvl_d(_in1, _in0); \ + _tmp1 = __lasx_xvilvh_d(_in1, _in0); \ + _tmp2 = __lasx_xvilvl_d(_in3, _in2); \ + _tmp3 = __lasx_xvilvh_d(_in3, _in2); \ + _out0 = __lasx_xvpermi_q(_tmp2, _tmp0, 0x20); \ + _out2 = __lasx_xvpermi_q(_tmp2, _tmp0, 0x31); \ + _out1 = __lasx_xvpermi_q(_tmp3, _tmp1, 0x20); \ + _out3 = __lasx_xvpermi_q(_tmp3, _tmp1, 0x31); \ + } + +/* + * ============================================================================= + * Description : Transpose 8x8 block with word elements in vectors + * Arguments : Inputs - _in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7 + * Outputs - _out0, _out1, _out2, _out3, _out4, _out5, _out6, + * _out7 + * Example : LASX_TRANSPOSE8x8_W + * _in0 : 1,2,3,4,5,6,7,8 + * _in1 : 2,2,3,4,5,6,7,8 + * _in2 : 3,2,3,4,5,6,7,8 + * _in3 : 4,2,3,4,5,6,7,8 + * _in4 : 5,2,3,4,5,6,7,8 + * _in5 : 6,2,3,4,5,6,7,8 + * _in6 : 7,2,3,4,5,6,7,8 + * _in7 : 8,2,3,4,5,6,7,8 + * + * _out0 : 1,2,3,4,5,6,7,8 + * _out1 : 2,2,2,2,2,2,2,2 + * _out2 : 3,3,3,3,3,3,3,3 + * _out3 : 4,4,4,4,4,4,4,4 + * _out4 : 5,5,5,5,5,5,5,5 + * _out5 : 6,6,6,6,6,6,6,6 + * _out6 : 7,7,7,7,7,7,7,7 + * _out7 : 8,8,8,8,8,8,8,8 + * ============================================================================= + */ +#define LASX_TRANSPOSE8x8_W(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ + _out7) \ + { \ + __m256i _s0_m, _s1_m; \ + __m256i _tmp0_m, _tmp1_m, _tmp2_m, _tmp3_m; \ + __m256i _tmp4_m, _tmp5_m, _tmp6_m, _tmp7_m; \ + \ + _s0_m = __lasx_xvilvl_w(_in2, _in0); \ + _s1_m = __lasx_xvilvl_w(_in3, _in1); \ + _tmp0_m = __lasx_xvilvl_w(_s1_m, _s0_m); \ + _tmp1_m = __lasx_xvilvh_w(_s1_m, _s0_m); \ + _s0_m = __lasx_xvilvh_w(_in2, _in0); \ + _s1_m = __lasx_xvilvh_w(_in3, _in1); \ + _tmp2_m = __lasx_xvilvl_w(_s1_m, _s0_m); \ + _tmp3_m = __lasx_xvilvh_w(_s1_m, _s0_m); \ + _s0_m = __lasx_xvilvl_w(_in6, _in4); \ + _s1_m = __lasx_xvilvl_w(_in7, _in5); \ + _tmp4_m = __lasx_xvilvl_w(_s1_m, _s0_m); \ + _tmp5_m = __lasx_xvilvh_w(_s1_m, _s0_m); \ + _s0_m = __lasx_xvilvh_w(_in6, _in4); \ + _s1_m = __lasx_xvilvh_w(_in7, _in5); \ + _tmp6_m = __lasx_xvilvl_w(_s1_m, _s0_m); \ + _tmp7_m = __lasx_xvilvh_w(_s1_m, _s0_m); \ + _out0 = __lasx_xvpermi_q(_tmp4_m, _tmp0_m, 0x20); \ + _out1 = __lasx_xvpermi_q(_tmp5_m, _tmp1_m, 0x20); \ + _out2 = __lasx_xvpermi_q(_tmp6_m, _tmp2_m, 0x20); \ + _out3 = __lasx_xvpermi_q(_tmp7_m, _tmp3_m, 0x20); \ + _out4 = __lasx_xvpermi_q(_tmp4_m, _tmp0_m, 0x31); \ + _out5 = __lasx_xvpermi_q(_tmp5_m, _tmp1_m, 0x31); \ + _out6 = __lasx_xvpermi_q(_tmp6_m, _tmp2_m, 0x31); \ + _out7 = __lasx_xvpermi_q(_tmp7_m, _tmp3_m, 0x31); \ + } + +/* + * ============================================================================= + * Description : Transpose input 16x8 byte block + * Arguments : Inputs - _in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, + * _in8, _in9, _in10, _in11, _in12, _in13, _in14, _in15 + * (input 16x8 byte block) + * Outputs - _out0, _out1, _out2, _out3, _out4, _out5, _out6, + * _out7 (output 8x16 byte block) + * Details : The rows of the matrix become columns, and the columns become + * rows. + * Example : See LASX_TRANSPOSE16x8_H + * ============================================================================= + */ +#define LASX_TRANSPOSE16x8_B(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _in8, _in9, _in10, _in11, _in12, _in13, _in14, \ + _in15, _out0, _out1, _out2, _out3, _out4, _out5, \ + _out6, _out7) \ + { \ + __m256i _tmp0_m, _tmp1_m, _tmp2_m, _tmp3_m; \ + __m256i _tmp4_m, _tmp5_m, _tmp6_m, _tmp7_m; \ + \ + _tmp0_m = __lasx_xvilvl_b(_in2, _in0); \ + _tmp1_m = __lasx_xvilvl_b(_in3, _in1); \ + _tmp2_m = __lasx_xvilvl_b(_in6, _in4); \ + _tmp3_m = __lasx_xvilvl_b(_in7, _in5); \ + _tmp4_m = __lasx_xvilvl_b(_in10, _in8); \ + _tmp5_m = __lasx_xvilvl_b(_in11, _in9); \ + _tmp6_m = __lasx_xvilvl_b(_in14, _in12); \ + _tmp7_m = __lasx_xvilvl_b(_in15, _in13); \ + _out0 = __lasx_xvilvl_b(_tmp1_m, _tmp0_m); \ + _out1 = __lasx_xvilvh_b(_tmp1_m, _tmp0_m); \ + _out2 = __lasx_xvilvl_b(_tmp3_m, _tmp2_m); \ + _out3 = __lasx_xvilvh_b(_tmp3_m, _tmp2_m); \ + _out4 = __lasx_xvilvl_b(_tmp5_m, _tmp4_m); \ + _out5 = __lasx_xvilvh_b(_tmp5_m, _tmp4_m); \ + _out6 = __lasx_xvilvl_b(_tmp7_m, _tmp6_m); \ + _out7 = __lasx_xvilvh_b(_tmp7_m, _tmp6_m); \ + _tmp0_m = __lasx_xvilvl_w(_out2, _out0); \ + _tmp2_m = __lasx_xvilvh_w(_out2, _out0); \ + _tmp4_m = __lasx_xvilvl_w(_out3, _out1); \ + _tmp6_m = __lasx_xvilvh_w(_out3, _out1); \ + _tmp1_m = __lasx_xvilvl_w(_out6, _out4); \ + _tmp3_m = __lasx_xvilvh_w(_out6, _out4); \ + _tmp5_m = __lasx_xvilvl_w(_out7, _out5); \ + _tmp7_m = __lasx_xvilvh_w(_out7, _out5); \ + _out0 = __lasx_xvilvl_d(_tmp1_m, _tmp0_m); \ + _out1 = __lasx_xvilvh_d(_tmp1_m, _tmp0_m); \ + _out2 = __lasx_xvilvl_d(_tmp3_m, _tmp2_m); \ + _out3 = __lasx_xvilvh_d(_tmp3_m, _tmp2_m); \ + _out4 = __lasx_xvilvl_d(_tmp5_m, _tmp4_m); \ + _out5 = __lasx_xvilvh_d(_tmp5_m, _tmp4_m); \ + _out6 = __lasx_xvilvl_d(_tmp7_m, _tmp6_m); \ + _out7 = __lasx_xvilvh_d(_tmp7_m, _tmp6_m); \ + } + +/* + * ============================================================================= + * Description : Transpose input 16x8 byte block + * Arguments : Inputs - _in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, + * _in8, _in9, _in10, _in11, _in12, _in13, _in14, _in15 + * (input 16x8 byte block) + * Outputs - _out0, _out1, _out2, _out3, _out4, _out5, _out6, + * _out7 (output 8x16 byte block) + * Details : The rows of the matrix become columns, and the columns become + * rows. + * Example : LASX_TRANSPOSE16x8_H + * _in0 : 1,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * _in1 : 2,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * _in2 : 3,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * _in3 : 4,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * _in4 : 5,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * _in5 : 6,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * _in6 : 7,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * _in7 : 8,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * _in8 : 9,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * _in9 : 1,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * _in10 : 0,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * _in11 : 2,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * _in12 : 3,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * _in13 : 7,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * _in14 : 5,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * _in15 : 6,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 + * + * _out0 : 1,2,3,4,5,6,7,8,9,1,0,2,3,7,5,6 + * _out1 : 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2 + * _out2 : 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3 + * _out3 : 4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4 + * _out4 : 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 + * _out5 : 6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6 + * _out6 : 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7 + * _out7 : 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8 + * ============================================================================= + */ +#define LASX_TRANSPOSE16x8_H(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _in8, _in9, _in10, _in11, _in12, _in13, _in14, \ + _in15, _out0, _out1, _out2, _out3, _out4, _out5, \ + _out6, _out7) \ + { \ + __m256i _tmp0_m, _tmp1_m, _tmp2_m, _tmp3_m; \ + __m256i _tmp4_m, _tmp5_m, _tmp6_m, _tmp7_m; \ + __m256i _t0, _t1, _t2, _t3, _t4, _t5, _t6, _t7; \ + \ + _tmp0_m = __lasx_xvilvl_h(_in2, _in0); \ + _tmp1_m = __lasx_xvilvl_h(_in3, _in1); \ + _tmp2_m = __lasx_xvilvl_h(_in6, _in4); \ + _tmp3_m = __lasx_xvilvl_h(_in7, _in5); \ + _tmp4_m = __lasx_xvilvl_h(_in10, _in8); \ + _tmp5_m = __lasx_xvilvl_h(_in11, _in9); \ + _tmp6_m = __lasx_xvilvl_h(_in14, _in12); \ + _tmp7_m = __lasx_xvilvl_h(_in15, _in13); \ + _t0 = __lasx_xvilvl_h(_tmp1_m, _tmp0_m); \ + _t1 = __lasx_xvilvh_h(_tmp1_m, _tmp0_m); \ + _t2 = __lasx_xvilvl_h(_tmp3_m, _tmp2_m); \ + _t3 = __lasx_xvilvh_h(_tmp3_m, _tmp2_m); \ + _t4 = __lasx_xvilvl_h(_tmp5_m, _tmp4_m); \ + _t5 = __lasx_xvilvh_h(_tmp5_m, _tmp4_m); \ + _t6 = __lasx_xvilvl_h(_tmp7_m, _tmp6_m); \ + _t7 = __lasx_xvilvh_h(_tmp7_m, _tmp6_m); \ + _tmp0_m = __lasx_xvilvl_d(_t2, _t0); \ + _tmp2_m = __lasx_xvilvh_d(_t2, _t0); \ + _tmp4_m = __lasx_xvilvl_d(_t3, _t1); \ + _tmp6_m = __lasx_xvilvh_d(_t3, _t1); \ + _tmp1_m = __lasx_xvilvl_d(_t6, _t4); \ + _tmp3_m = __lasx_xvilvh_d(_t6, _t4); \ + _tmp5_m = __lasx_xvilvl_d(_t7, _t5); \ + _tmp7_m = __lasx_xvilvh_d(_t7, _t5); \ + _out0 = __lasx_xvpermi_q(_tmp1_m, _tmp0_m, 0x20); \ + _out1 = __lasx_xvpermi_q(_tmp3_m, _tmp2_m, 0x20); \ + _out2 = __lasx_xvpermi_q(_tmp5_m, _tmp4_m, 0x20); \ + _out3 = __lasx_xvpermi_q(_tmp7_m, _tmp6_m, 0x20); \ + \ + _tmp0_m = __lasx_xvilvh_h(_in2, _in0); \ + _tmp1_m = __lasx_xvilvh_h(_in3, _in1); \ + _tmp2_m = __lasx_xvilvh_h(_in6, _in4); \ + _tmp3_m = __lasx_xvilvh_h(_in7, _in5); \ + _tmp4_m = __lasx_xvilvh_h(_in10, _in8); \ + _tmp5_m = __lasx_xvilvh_h(_in11, _in9); \ + _tmp6_m = __lasx_xvilvh_h(_in14, _in12); \ + _tmp7_m = __lasx_xvilvh_h(_in15, _in13); \ + _t0 = __lasx_xvilvl_h(_tmp1_m, _tmp0_m); \ + _t1 = __lasx_xvilvh_h(_tmp1_m, _tmp0_m); \ + _t2 = __lasx_xvilvl_h(_tmp3_m, _tmp2_m); \ + _t3 = __lasx_xvilvh_h(_tmp3_m, _tmp2_m); \ + _t4 = __lasx_xvilvl_h(_tmp5_m, _tmp4_m); \ + _t5 = __lasx_xvilvh_h(_tmp5_m, _tmp4_m); \ + _t6 = __lasx_xvilvl_h(_tmp7_m, _tmp6_m); \ + _t7 = __lasx_xvilvh_h(_tmp7_m, _tmp6_m); \ + _tmp0_m = __lasx_xvilvl_d(_t2, _t0); \ + _tmp2_m = __lasx_xvilvh_d(_t2, _t0); \ + _tmp4_m = __lasx_xvilvl_d(_t3, _t1); \ + _tmp6_m = __lasx_xvilvh_d(_t3, _t1); \ + _tmp1_m = __lasx_xvilvl_d(_t6, _t4); \ + _tmp3_m = __lasx_xvilvh_d(_t6, _t4); \ + _tmp5_m = __lasx_xvilvl_d(_t7, _t5); \ + _tmp7_m = __lasx_xvilvh_d(_t7, _t5); \ + _out4 = __lasx_xvpermi_q(_tmp1_m, _tmp0_m, 0x20); \ + _out5 = __lasx_xvpermi_q(_tmp3_m, _tmp2_m, 0x20); \ + _out6 = __lasx_xvpermi_q(_tmp5_m, _tmp4_m, 0x20); \ + _out7 = __lasx_xvpermi_q(_tmp7_m, _tmp6_m, 0x20); \ + } + +/* + * ============================================================================= + * Description : Transpose 4x4 block with halfword elements in vectors + * Arguments : Inputs - _in0, _in1, _in2, _in3 + * Outputs - _out0, _out1, _out2, _out3 + * Return Type - signed halfword + * Details : The rows of the matrix become columns, and the columns become + * rows. + * Example : See LASX_TRANSPOSE8x8_H + * ============================================================================= + */ +#define LASX_TRANSPOSE4x4_H(_in0, _in1, _in2, _in3, _out0, _out1, _out2, \ + _out3) \ + { \ + __m256i _s0_m, _s1_m; \ + \ + _s0_m = __lasx_xvilvl_h(_in1, _in0); \ + _s1_m = __lasx_xvilvl_h(_in3, _in2); \ + _out0 = __lasx_xvilvl_w(_s1_m, _s0_m); \ + _out2 = __lasx_xvilvh_w(_s1_m, _s0_m); \ + _out1 = __lasx_xvilvh_d(_out0, _out0); \ + _out3 = __lasx_xvilvh_d(_out2, _out2); \ + } + +/* + * ============================================================================= + * Description : Transpose input 8x8 byte block + * Arguments : Inputs - _in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7 + * (input 8x8 byte block) + * Outputs - _out0, _out1, _out2, _out3, _out4, _out5, _out6, + * _out7 (output 8x8 byte block) + * Example : See LASX_TRANSPOSE8x8_H + * ============================================================================= + */ +#define LASX_TRANSPOSE8x8_B(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ + _out7) \ + { \ + __m256i _tmp0_m, _tmp1_m, _tmp2_m, _tmp3_m; \ + __m256i _tmp4_m, _tmp5_m, _tmp6_m, _tmp7_m; \ + _tmp0_m = __lasx_xvilvl_b(_in2, _in0); \ + _tmp1_m = __lasx_xvilvl_b(_in3, _in1); \ + _tmp2_m = __lasx_xvilvl_b(_in6, _in4); \ + _tmp3_m = __lasx_xvilvl_b(_in7, _in5); \ + _tmp4_m = __lasx_xvilvl_b(_tmp1_m, _tmp0_m); \ + _tmp5_m = __lasx_xvilvh_b(_tmp1_m, _tmp0_m); \ + _tmp6_m = __lasx_xvilvl_b(_tmp3_m, _tmp2_m); \ + _tmp7_m = __lasx_xvilvh_b(_tmp3_m, _tmp2_m); \ + _out0 = __lasx_xvilvl_w(_tmp6_m, _tmp4_m); \ + _out2 = __lasx_xvilvh_w(_tmp6_m, _tmp4_m); \ + _out4 = __lasx_xvilvl_w(_tmp7_m, _tmp5_m); \ + _out6 = __lasx_xvilvh_w(_tmp7_m, _tmp5_m); \ + _out1 = __lasx_xvbsrl_v(_out0, 8); \ + _out3 = __lasx_xvbsrl_v(_out2, 8); \ + _out5 = __lasx_xvbsrl_v(_out4, 8); \ + _out7 = __lasx_xvbsrl_v(_out6, 8); \ + } + +/* + * ============================================================================= + * Description : Transpose 8x8 block with halfword elements in vectors. + * Arguments : Inputs - _in0, _in1, ~ + * Outputs - _out0, _out1, ~ + * Details : The rows of the matrix become columns, and the columns become + * rows. + * Example : LASX_TRANSPOSE8x8_H + * _in0 : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * _in1 : 8,2,3,4, 5,6,7,8, 8,2,3,4, 5,6,7,8 + * _in2 : 8,2,3,4, 5,6,7,8, 8,2,3,4, 5,6,7,8 + * _in3 : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * _in4 : 9,2,3,4, 5,6,7,8, 9,2,3,4, 5,6,7,8 + * _in5 : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * _in6 : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 + * _in7 : 9,2,3,4, 5,6,7,8, 9,2,3,4, 5,6,7,8 + * + * _out0 : 1,8,8,1, 9,1,1,9, 1,8,8,1, 9,1,1,9 + * _out1 : 2,2,2,2, 2,2,2,2, 2,2,2,2, 2,2,2,2 + * _out2 : 3,3,3,3, 3,3,3,3, 3,3,3,3, 3,3,3,3 + * _out3 : 4,4,4,4, 4,4,4,4, 4,4,4,4, 4,4,4,4 + * _out4 : 5,5,5,5, 5,5,5,5, 5,5,5,5, 5,5,5,5 + * _out5 : 6,6,6,6, 6,6,6,6, 6,6,6,6, 6,6,6,6 + * _out6 : 7,7,7,7, 7,7,7,7, 7,7,7,7, 7,7,7,7 + * _out7 : 8,8,8,8, 8,8,8,8, 8,8,8,8, 8,8,8,8 + * ============================================================================= + */ +#define LASX_TRANSPOSE8x8_H(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ + _out7) \ + { \ + __m256i _s0_m, _s1_m; \ + __m256i _tmp0_m, _tmp1_m, _tmp2_m, _tmp3_m; \ + __m256i _tmp4_m, _tmp5_m, _tmp6_m, _tmp7_m; \ + \ + _s0_m = __lasx_xvilvl_h(_in6, _in4); \ + _s1_m = __lasx_xvilvl_h(_in7, _in5); \ + _tmp0_m = __lasx_xvilvl_h(_s1_m, _s0_m); \ + _tmp1_m = __lasx_xvilvh_h(_s1_m, _s0_m); \ + _s0_m = __lasx_xvilvh_h(_in6, _in4); \ + _s1_m = __lasx_xvilvh_h(_in7, _in5); \ + _tmp2_m = __lasx_xvilvl_h(_s1_m, _s0_m); \ + _tmp3_m = __lasx_xvilvh_h(_s1_m, _s0_m); \ + \ + _s0_m = __lasx_xvilvl_h(_in2, _in0); \ + _s1_m = __lasx_xvilvl_h(_in3, _in1); \ + _tmp4_m = __lasx_xvilvl_h(_s1_m, _s0_m); \ + _tmp5_m = __lasx_xvilvh_h(_s1_m, _s0_m); \ + _s0_m = __lasx_xvilvh_h(_in2, _in0); \ + _s1_m = __lasx_xvilvh_h(_in3, _in1); \ + _tmp6_m = __lasx_xvilvl_h(_s1_m, _s0_m); \ + _tmp7_m = __lasx_xvilvh_h(_s1_m, _s0_m); \ + \ + _out0 = __lasx_xvpickev_d(_tmp0_m, _tmp4_m); \ + _out2 = __lasx_xvpickev_d(_tmp1_m, _tmp5_m); \ + _out4 = __lasx_xvpickev_d(_tmp2_m, _tmp6_m); \ + _out6 = __lasx_xvpickev_d(_tmp3_m, _tmp7_m); \ + _out1 = __lasx_xvpickod_d(_tmp0_m, _tmp4_m); \ + _out3 = __lasx_xvpickod_d(_tmp1_m, _tmp5_m); \ + _out5 = __lasx_xvpickod_d(_tmp2_m, _tmp6_m); \ + _out7 = __lasx_xvpickod_d(_tmp3_m, _tmp7_m); \ + } + +/* + * ============================================================================= + * Description : Butterfly of 4 input vectors + * Arguments : Inputs - _in0, _in1, _in2, _in3 + * Outputs - _out0, _out1, _out2, _out3 + * Details : Butterfly operation + * Example : LASX_BUTTERFLY_4 + * _out0 = _in0 + _in3; + * _out1 = _in1 + _in2; + * _out2 = _in1 - _in2; + * _out3 = _in0 - _in3; + * ============================================================================= + */ +#define LASX_BUTTERFLY_4_B(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ + { \ + _out0 = __lasx_xvadd_b(_in0, _in3); \ + _out1 = __lasx_xvadd_b(_in1, _in2); \ + _out2 = __lasx_xvsub_b(_in1, _in2); \ + _out3 = __lasx_xvsub_b(_in0, _in3); \ + } +#define LASX_BUTTERFLY_4_H(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ + { \ + _out0 = __lasx_xvadd_h(_in0, _in3); \ + _out1 = __lasx_xvadd_h(_in1, _in2); \ + _out2 = __lasx_xvsub_h(_in1, _in2); \ + _out3 = __lasx_xvsub_h(_in0, _in3); \ + } +#define LASX_BUTTERFLY_4_W(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ + { \ + _out0 = __lasx_xvadd_w(_in0, _in3); \ + _out1 = __lasx_xvadd_w(_in1, _in2); \ + _out2 = __lasx_xvsub_w(_in1, _in2); \ + _out3 = __lasx_xvsub_w(_in0, _in3); \ + } +#define LASX_BUTTERFLY_4_D(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ + { \ + _out0 = __lasx_xvadd_d(_in0, _in3); \ + _out1 = __lasx_xvadd_d(_in1, _in2); \ + _out2 = __lasx_xvsub_d(_in1, _in2); \ + _out3 = __lasx_xvsub_d(_in0, _in3); \ + } + +/* + * ============================================================================= + * Description : Butterfly of 8 input vectors + * Arguments : Inputs - _in0, _in1, _in2, _in3, ~ + * Outputs - _out0, _out1, _out2, _out3, ~ + * Details : Butterfly operation + * Example : LASX_BUTTERFLY_8 + * _out0 = _in0 + _in7; + * _out1 = _in1 + _in6; + * _out2 = _in2 + _in5; + * _out3 = _in3 + _in4; + * _out4 = _in3 - _in4; + * _out5 = _in2 - _in5; + * _out6 = _in1 - _in6; + * _out7 = _in0 - _in7; + * ============================================================================= + */ +#define LASX_BUTTERFLY_8_B(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ + _out7) \ + { \ + _out0 = __lasx_xvadd_b(_in0, _in7); \ + _out1 = __lasx_xvadd_b(_in1, _in6); \ + _out2 = __lasx_xvadd_b(_in2, _in5); \ + _out3 = __lasx_xvadd_b(_in3, _in4); \ + _out4 = __lasx_xvsub_b(_in3, _in4); \ + _out5 = __lasx_xvsub_b(_in2, _in5); \ + _out6 = __lasx_xvsub_b(_in1, _in6); \ + _out7 = __lasx_xvsub_b(_in0, _in7); \ + } + +#define LASX_BUTTERFLY_8_H(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ + _out7) \ + { \ + _out0 = __lasx_xvadd_h(_in0, _in7); \ + _out1 = __lasx_xvadd_h(_in1, _in6); \ + _out2 = __lasx_xvadd_h(_in2, _in5); \ + _out3 = __lasx_xvadd_h(_in3, _in4); \ + _out4 = __lasx_xvsub_h(_in3, _in4); \ + _out5 = __lasx_xvsub_h(_in2, _in5); \ + _out6 = __lasx_xvsub_h(_in1, _in6); \ + _out7 = __lasx_xvsub_h(_in0, _in7); \ + } + +#define LASX_BUTTERFLY_8_W(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ + _out7) \ + { \ + _out0 = __lasx_xvadd_w(_in0, _in7); \ + _out1 = __lasx_xvadd_w(_in1, _in6); \ + _out2 = __lasx_xvadd_w(_in2, _in5); \ + _out3 = __lasx_xvadd_w(_in3, _in4); \ + _out4 = __lasx_xvsub_w(_in3, _in4); \ + _out5 = __lasx_xvsub_w(_in2, _in5); \ + _out6 = __lasx_xvsub_w(_in1, _in6); \ + _out7 = __lasx_xvsub_w(_in0, _in7); \ + } + +#define LASX_BUTTERFLY_8_D(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ + _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ + _out7) \ + { \ + _out0 = __lasx_xvadd_d(_in0, _in7); \ + _out1 = __lasx_xvadd_d(_in1, _in6); \ + _out2 = __lasx_xvadd_d(_in2, _in5); \ + _out3 = __lasx_xvadd_d(_in3, _in4); \ + _out4 = __lasx_xvsub_d(_in3, _in4); \ + _out5 = __lasx_xvsub_d(_in2, _in5); \ + _out6 = __lasx_xvsub_d(_in1, _in6); \ + _out7 = __lasx_xvsub_d(_in0, _in7); \ + } + +#endif // LASX + +/* + * ============================================================================= + * Description : Print out elements in vector. + * Arguments : Inputs - RTYPE, _element_num, _in0, _enter + * Outputs - + * Details : Print out '_element_num' elements in 'RTYPE' vector '_in0', if + * '_enter' is TRUE, prefix "\nVP:" will be added first. + * Example : VECT_PRINT(v4i32,4,in0,1); // in0: 1,2,3,4 + * VP:1,2,3,4, + * ============================================================================= + */ +#define VECT_PRINT(RTYPE, element_num, in0, enter) \ + { \ + RTYPE _tmp0 = (RTYPE)in0; \ + int _i = 0; \ + if (enter) printf("\nVP:"); \ + for (_i = 0; _i < element_num; _i++) printf("%d,", _tmp0[_i]); \ + } + +#endif /* LOONGSON_INTRINSICS_H */ +#endif /* VPX_VPX_UTIL_LOONGSON_INTRINSICS_H_ */ diff --git a/media/libvpx/libvpx/vpxdec.c b/media/libvpx/libvpx/vpxdec.c --- a/media/libvpx/libvpx/vpxdec.c +++ b/media/libvpx/libvpx/vpxdec.c @@ -579,11 +579,14 @@ input.vpx_input_ctx = &vpx_input_ctx; /* Parse command line */ exec_name = argv_[0]; argv = argv_dup(argc - 1, argv_ + 1); - + if (!argv) { + fprintf(stderr, "Error allocating argument list\n"); + return EXIT_FAILURE; + } for (argi = argj = argv; (*argj = *argi); argi += arg.argv_step) { memset(&arg, 0, sizeof(arg)); arg.argv_step = 1; if (arg_match(&arg, &help, argi)) { @@ -813,10 +816,14 @@ if (num_external_frame_buffers > 0) { ext_fb_list.num_external_frame_buffers = num_external_frame_buffers; ext_fb_list.ext_fb = (struct ExternalFrameBuffer *)calloc( num_external_frame_buffers, sizeof(*ext_fb_list.ext_fb)); + if (!ext_fb_list.ext_fb) { + fprintf(stderr, "Failed to allocate ExternalFrameBuffer\n"); + goto fail; + } if (vpx_codec_set_frame_buffer_functions(&decoder, get_vp9_frame_buffer, release_vp9_frame_buffer, &ext_fb_list)) { fprintf(stderr, "Failed to configure external frame buffers: %s\n", vpx_codec_error(&decoder)); @@ -928,10 +935,15 @@ render_height = render_size[1]; } } scaled_img = vpx_img_alloc(NULL, img->fmt, render_width, render_height, 16); + if (!scaled_img) { + fprintf(stderr, "Failed to allocate scaled image (%d x %d)\n", + render_width, render_height); + goto fail; + } scaled_img->bit_depth = img->bit_depth; } if (img->d_w != scaled_img->d_w || img->d_h != scaled_img->d_h) { #if CONFIG_LIBYUV @@ -964,10 +976,14 @@ img_shifted = NULL; } if (!img_shifted) { img_shifted = vpx_img_alloc(NULL, shifted_fmt, img->d_w, img->d_h, 16); + if (!img_shifted) { + fprintf(stderr, "Failed to allocate image\n"); + goto fail; + } img_shifted->bit_depth = output_bit_depth; } if (output_bit_depth > img->bit_depth) { vpx_img_upshift(img_shifted, img, output_bit_depth - img->bit_depth); } else { @@ -1108,10 +1124,14 @@ char **argv, **argi, **argj; struct arg arg; int error = 0; argv = argv_dup(argc - 1, argv_ + 1); + if (!argv) { + fprintf(stderr, "Error allocating argument list\n"); + return EXIT_FAILURE; + } for (argi = argj = argv; (*argj = *argi); argi += arg.argv_step) { memset(&arg, 0, sizeof(arg)); arg.argv_step = 1; if (arg_match(&arg, &looparg, argi)) { diff --git a/media/libvpx/libvpx/vpxenc.c b/media/libvpx/libvpx/vpxenc.c --- a/media/libvpx/libvpx/vpxenc.c +++ b/media/libvpx/libvpx/vpxenc.c @@ -56,12 +56,12 @@ } #define fwrite wrap_fwrite static const char *exec_name; -static void warn_or_exit_on_errorv(vpx_codec_ctx_t *ctx, int fatal, - const char *s, va_list ap) { +static VPX_TOOLS_FORMAT_PRINTF(3, 0) void warn_or_exit_on_errorv( + vpx_codec_ctx_t *ctx, int fatal, const char *s, va_list ap) { if (ctx->err) { const char *detail = vpx_codec_error_detail(ctx); vfprintf(stderr, s, ap); fprintf(stderr, ": %s\n", vpx_codec_error(ctx)); @@ -70,20 +70,22 @@ if (fatal) exit(EXIT_FAILURE); } } -static void ctx_exit_on_error(vpx_codec_ctx_t *ctx, const char *s, ...) { +static VPX_TOOLS_FORMAT_PRINTF(2, + 3) void ctx_exit_on_error(vpx_codec_ctx_t *ctx, + const char *s, ...) { va_list ap; va_start(ap, s); warn_or_exit_on_errorv(ctx, 1, s, ap); va_end(ap); } -static void warn_or_exit_on_error(vpx_codec_ctx_t *ctx, int fatal, - const char *s, ...) { +static VPX_TOOLS_FORMAT_PRINTF(3, 4) void warn_or_exit_on_error( + vpx_codec_ctx_t *ctx, int fatal, const char *s, ...) { va_list ap; va_start(ap, s); warn_or_exit_on_errorv(ctx, fatal, s, ap); va_end(ap); @@ -1699,10 +1701,14 @@ /* First parse the global configuration values, because we want to apply * other parameters on top of the default configuration provided by the * codec. */ argv = argv_dup(argc - 1, argv_ + 1); + if (!argv) { + fprintf(stderr, "Error allocating argument list\n"); + return EXIT_FAILURE; + } parse_global_config(&global, argv); if (argc < 3) usage_exit(); switch (global.color_type) { diff --git a/media/libvpx/libvpx/vpxstats.c b/media/libvpx/libvpx/vpxstats.c --- a/media/libvpx/libvpx/vpxstats.c +++ b/media/libvpx/libvpx/vpxstats.c @@ -39,11 +39,11 @@ rewind(stats->file); stats->buf.buf = malloc(stats->buf_alloc_sz); if (!stats->buf.buf) - fatal("Failed to allocate first-pass stats buffer (%lu bytes)", + fatal("Failed to allocate first-pass stats buffer (%u bytes)", (unsigned int)stats->buf_alloc_sz); nbytes = fread(stats->buf.buf, 1, stats->buf.sz, stats->file); res = (nbytes == stats->buf.sz); } diff --git a/media/libvpx/libvpx/warnings.c b/media/libvpx/libvpx/warnings.c --- a/media/libvpx/libvpx/warnings.c +++ b/media/libvpx/libvpx/warnings.c @@ -96,11 +96,11 @@ stream_config->g_lag_in_frames, global_config->deadline, stream_config->rc_end_usage, &warning_list); /* Count and print warnings. */ for (warning = warning_list.warning_node; warning != NULL; warning = warning->next_warning, ++num_warnings) { - warn(warning->warning_string); + warn("%s", warning->warning_string); } free_warning_list(&warning_list); if (num_warnings) { diff --git a/media/libvpx/libvpx/webmdec.cc b/media/libvpx/libvpx/webmdec.cc --- a/media/libvpx/libvpx/webmdec.cc +++ b/media/libvpx/libvpx/webmdec.cc @@ -208,10 +208,12 @@ } vpx_ctx->framerate.numerator = (i - 1) * 1000000; vpx_ctx->framerate.denominator = static_cast(webm_ctx->timestamp_ns / 1000); delete[] buffer; + // webm_ctx->buffer is assigned to the buffer pointer in webm_read_frame(). + webm_ctx->buffer = nullptr; get_first_cluster(webm_ctx); webm_ctx->block = nullptr; webm_ctx->block_entry = nullptr; webm_ctx->block_frame_index = 0; diff --git a/media/libvpx/libvpx/y4minput.c b/media/libvpx/libvpx/y4minput.c --- a/media/libvpx/libvpx/y4minput.c +++ b/media/libvpx/libvpx/y4minput.c @@ -1085,13 +1085,19 @@ ((y4m_ctx->pic_h + y4m_ctx->dst_c_dec_v - 1) / y4m_ctx->dst_c_dec_v); if (y4m_ctx->bit_depth == 8) y4m_ctx->dst_buf = (unsigned char *)malloc(y4m_ctx->dst_buf_sz); else y4m_ctx->dst_buf = (unsigned char *)malloc(2 * y4m_ctx->dst_buf_sz); + if (!y4m_ctx->dst_buf) return -1; - if (y4m_ctx->aux_buf_sz > 0) + if (y4m_ctx->aux_buf_sz > 0) { y4m_ctx->aux_buf = (unsigned char *)malloc(y4m_ctx->aux_buf_sz); + if (!y4m_ctx->aux_buf) { + free(y4m_ctx->dst_buf); + return -1; + } + } return 0; } void y4m_input_close(y4m_input *_y4m) { free(_y4m->dst_buf); diff --git a/media/libvpx/moz.yaml b/media/libvpx/moz.yaml --- a/media/libvpx/moz.yaml +++ b/media/libvpx/moz.yaml @@ -18,15 +18,15 @@ # Usually different from repository url url: https://chromium.googlesource.com/webm/libvpx # Human-readable identifier for this version/release # Generally "version NNN", "tag SSS", "bookmark SSS" - release: 626ff35955c2c35b806b3e0ecf551a1a8611cdbf (Wed Oct 06 17:41:19 2021). + release: 40a46f2034906b3d645ff2890741028458764e57 (Fri May 13 19:15:59 2022). # Revision to pull in # Must be a long or short commit SHA (long preferred) - revision: 626ff35955c2c35b806b3e0ecf551a1a8611cdbf + revision: 40a46f2034906b3d645ff2890741028458764e57 # The package's license, where possible using the mnemonic from # https://spdx.org/licenses/ # Multiple licenses can be specified (as a YAML list) # A "LICENSE" file must exist containing the full license text diff --git a/media/libvpx/sources.mozbuild b/media/libvpx/sources.mozbuild --- a/media/libvpx/sources.mozbuild +++ b/media/libvpx/sources.mozbuild @@ -233,22 +233,19 @@ 'libvpx/vpx_dsp/x86/intrapred_ssse3.asm', 'libvpx/vpx_dsp/x86/inv_txfm_sse2.c', 'libvpx/vpx_dsp/x86/inv_txfm_ssse3.c', 'libvpx/vpx_dsp/x86/inv_wht_sse2.asm', 'libvpx/vpx_dsp/x86/loopfilter_avx2.c', - 'libvpx/vpx_dsp/x86/loopfilter_intrin_sse2.c', + 'libvpx/vpx_dsp/x86/loopfilter_sse2.c', 'libvpx/vpx_dsp/x86/post_proc_sse2.c', 'libvpx/vpx_dsp/x86/quantize_avx.c', 'libvpx/vpx_dsp/x86/quantize_sse2.c', 'libvpx/vpx_dsp/x86/quantize_ssse3.c', 'libvpx/vpx_dsp/x86/sad4d_avx2.c', 'libvpx/vpx_dsp/x86/sad4d_sse2.asm', 'libvpx/vpx_dsp/x86/sad_avx2.c', 'libvpx/vpx_dsp/x86/sad_sse2.asm', - 'libvpx/vpx_dsp/x86/sad_sse3.asm', - 'libvpx/vpx_dsp/x86/sad_sse4.asm', - 'libvpx/vpx_dsp/x86/sad_ssse3.asm', 'libvpx/vpx_dsp/x86/ssim_opt_x86_64.asm', 'libvpx/vpx_dsp/x86/subpel_variance_sse2.asm', 'libvpx/vpx_dsp/x86/subtract_sse2.asm', 'libvpx/vpx_dsp/x86/sum_squares_sse2.c', 'libvpx/vpx_dsp/x86/variance_avx2.c', @@ -261,11 +258,10 @@ 'libvpx/vpx_dsp/x86/vpx_subpixel_8t_ssse3.asm', 'libvpx/vpx_dsp/x86/vpx_subpixel_bilinear_sse2.asm', 'libvpx/vpx_dsp/x86/vpx_subpixel_bilinear_ssse3.asm', 'libvpx/vpx_mem/vpx_mem.c', 'libvpx/vpx_ports/emms_mmx.asm', - 'libvpx/vpx_ports/float_control_word.asm', 'libvpx/vpx_scale/generic/gen_scalers.c', 'libvpx/vpx_scale/generic/vpx_scale.c', 'libvpx/vpx_scale/generic/yv12config.c', 'libvpx/vpx_scale/generic/yv12extend.c', 'libvpx/vpx_scale/vpx_scale_rtcd.c', @@ -500,22 +496,19 @@ 'libvpx/vpx_dsp/x86/intrapred_ssse3.asm', 'libvpx/vpx_dsp/x86/inv_txfm_sse2.c', 'libvpx/vpx_dsp/x86/inv_txfm_ssse3.c', 'libvpx/vpx_dsp/x86/inv_wht_sse2.asm', 'libvpx/vpx_dsp/x86/loopfilter_avx2.c', - 'libvpx/vpx_dsp/x86/loopfilter_intrin_sse2.c', + 'libvpx/vpx_dsp/x86/loopfilter_sse2.c', 'libvpx/vpx_dsp/x86/post_proc_sse2.c', 'libvpx/vpx_dsp/x86/quantize_avx.c', 'libvpx/vpx_dsp/x86/quantize_sse2.c', 'libvpx/vpx_dsp/x86/quantize_ssse3.c', 'libvpx/vpx_dsp/x86/sad4d_avx2.c', 'libvpx/vpx_dsp/x86/sad4d_sse2.asm', 'libvpx/vpx_dsp/x86/sad_avx2.c', 'libvpx/vpx_dsp/x86/sad_sse2.asm', - 'libvpx/vpx_dsp/x86/sad_sse3.asm', - 'libvpx/vpx_dsp/x86/sad_sse4.asm', - 'libvpx/vpx_dsp/x86/sad_ssse3.asm', 'libvpx/vpx_dsp/x86/subpel_variance_sse2.asm', 'libvpx/vpx_dsp/x86/subtract_sse2.asm', 'libvpx/vpx_dsp/x86/sum_squares_sse2.c', 'libvpx/vpx_dsp/x86/variance_avx2.c', 'libvpx/vpx_dsp/x86/variance_sse2.c', @@ -663,10 +656,12 @@ 'libvpx/vp9/decoder/vp9_decodemv.c', 'libvpx/vp9/decoder/vp9_decoder.c', 'libvpx/vp9/decoder/vp9_detokenize.c', 'libvpx/vp9/decoder/vp9_dsubexp.c', 'libvpx/vp9/decoder/vp9_job_queue.c', + 'libvpx/vp9/encoder/arm/neon/vp9_dct_neon.c', + 'libvpx/vp9/encoder/arm/neon/vp9_diamond_search_sad_neon.c', 'libvpx/vp9/encoder/arm/neon/vp9_error_neon.c', 'libvpx/vp9/encoder/arm/neon/vp9_frame_scale_neon.c', 'libvpx/vp9/encoder/arm/neon/vp9_quantize_neon.c', 'libvpx/vp9/encoder/vp9_aq_cyclicrefresh.c', 'libvpx/vp9/encoder/vp9_bitstream.c', @@ -906,10 +901,12 @@ 'libvpx/vp9/decoder/vp9_decodemv.c', 'libvpx/vp9/decoder/vp9_decoder.c', 'libvpx/vp9/decoder/vp9_detokenize.c', 'libvpx/vp9/decoder/vp9_dsubexp.c', 'libvpx/vp9/decoder/vp9_job_queue.c', + 'libvpx/vp9/encoder/arm/neon/vp9_dct_neon.c', + 'libvpx/vp9/encoder/arm/neon/vp9_diamond_search_sad_neon.c', 'libvpx/vp9/encoder/arm/neon/vp9_error_neon.c', 'libvpx/vp9/encoder/arm/neon/vp9_frame_scale_neon.c', 'libvpx/vp9/encoder/arm/neon/vp9_quantize_neon.c', 'libvpx/vp9/encoder/vp9_aq_cyclicrefresh.c', 'libvpx/vp9/encoder/vp9_bitstream.c', diff --git a/third_party/jpeg-xl/.github/workflows/build_test.yml b/third_party/jpeg-xl/.github/workflows/build_test.yml --- a/third_party/jpeg-xl/.github/workflows/build_test.yml +++ b/third_party/jpeg-xl/.github/workflows/build_test.yml @@ -453,14 +453,13 @@ wasm32_build: name: WASM wasm32/${{ matrix.variant }} runs-on: ubuntu-latest env: CCACHE_DIR: ${{ github.workspace }}/.ccache - EM_VERSION: 3.1.4 - V8_VERSION: 9.8.177 - V8: ${{ github.workspace }}/.jsvu/v8 BUILD_TARGET: wasm32 + EM_VERSION: 3.1.1 + NODE_VERSION: 18 strategy: matrix: include: - variant: scalar @@ -497,39 +496,43 @@ path: ${{ env.CCACHE_DIR }} key: ${{ runner.os }}-${{ github.sha }}-${{ matrix.variant }} restore-keys: | ${{ runner.os }}-${{ steps.git-env.outputs.parent }}-${{ matrix.variant }} + - name: Install node + uses: actions/setup-node@v3 + with: + node-version: ${{env.NODE_VERSION}} + + - name: Get non-EMSDK node path + run: which node >> $HOME/.base_node_path + - name: Install emsdk uses: mymindstorm/setup-emsdk@v11 # TODO(deymo): We could cache this action but it doesn't work when running # in a matrix. with: version: ${{env.EM_VERSION}} no-cache: true - - name: Install v8 + + - name: Set EMSDK node version run: | - npm install jsvu -g - HOME="${{ github.workspace }}" jsvu --os=linux64 "v8@${V8_VERSION}" - rm -f "${{ github.workspace }}/.jsvu/v8" - ln -s "${{ github.workspace }}/.jsvu/v8-${V8_VERSION}" \ - "${{ github.workspace }}/.jsvu/v8" + echo "NODE_JS='$(cat $HOME/.base_node_path)'" >> $EM_CONFIG + emsdk construct_env + # TODO(deymo): Build and install other dependencies like libpng, libjpeg, # etc. - name: Build run: | mkdir -p ${CCACHE_DIR} echo "max_size = 200M" > ${CCACHE_DIR}/ccache.conf if [[ "${{ matrix.variant }}" == "simd" ]]; then export ENABLE_WASM_SIMD=1 fi - emconfigure ./ci.sh release \ + ./ci.sh release \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON \ - -DBUILD_SHARED_LIBS=OFF \ - -DJPEGXL_ENABLE_JNI=OFF + -DCMAKE_C_COMPILER_LAUNCHER=ccache env: SKIP_TEST: 1 - name: ccache stats run: ccache --show-stats @@ -537,6 +540,6 @@ if: | github.event_name == 'push' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.names, 'CI:full')) run: | - emconfigure ./ci.sh test + ./ci.sh test diff --git a/third_party/jpeg-xl/AUTHORS b/third_party/jpeg-xl/AUTHORS --- a/third_party/jpeg-xl/AUTHORS +++ b/third_party/jpeg-xl/AUTHORS @@ -23,16 +23,18 @@ CanadianBaconBoi Daniel Novomeský David Burnett Dirk Lemstra Don Olmstead +Even Rouault Heiko Becker Jon Sneyers Kleis Auke Wolthuizen L. E. Segovia Leo Izen Lovell Fuller +Maarten DB Marcin Konicki Martin Strunz Mathieu Malaterre Misaki Kasumi Petr Diblík diff --git a/third_party/jpeg-xl/CMakeLists.txt b/third_party/jpeg-xl/CMakeLists.txt --- a/third_party/jpeg-xl/CMakeLists.txt +++ b/third_party/jpeg-xl/CMakeLists.txt @@ -232,10 +232,16 @@ "-Wl,--whole-archive;-lpthread;-Wl,--no-whole-archive") endif() endif() endif() # JPEGXL_STATIC +if (JPEGXL_EMSCRIPTEN) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -pthread") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -pthread") +endif() + if ("${CXX_MACRO_PREFIX_MAP}") add_compile_options(-fmacro-prefix-map=${CMAKE_CURRENT_SOURCE_DIR}=.) endif() if (CXX_NO_RTTI_SUPPORTED) @@ -316,11 +322,11 @@ # Enable tests regardless of where they are defined. enable_testing() include(CTest) # Specify default location of `testdata`: if(NOT DEFINED JPEGXL_TEST_DATA_PATH) - set(JPEGXL_TEST_DATA_PATH "${PROJECT_SOURCE_DIR}/third_party/testdata") + set(JPEGXL_TEST_DATA_PATH "${PROJECT_SOURCE_DIR}/testdata") endif() # Libraries. add_subdirectory(lib) diff --git a/third_party/jpeg-xl/bash_test.sh b/third_party/jpeg-xl/bash_test.sh --- a/third_party/jpeg-xl/bash_test.sh +++ b/third_party/jpeg-xl/bash_test.sh @@ -101,10 +101,16 @@ echo "Don't use '%zu' or '%zd' in a format string, instead use " \ "'%\" PRIuS \"' or '%\" PRIdS \"'." >&2 ret=1 fi + if grep -n -E 'gmock\.h' \ + $(git ls-files | grep -E '(\.c|\.cc|\.cpp|\.h)$' | grep -v -F /test_utils.h); then + echo "Don't include gmock directly, instead include 'test_utils.h'. " >&2 + ret=1 + fi + local f for f in $(git ls-files | grep -E "\.cc$" | xargs grep 'PRI[udx]S' | cut -f 1 -d : | uniq); do if [ ! -e "$f" ]; then continue @@ -114,11 +120,11 @@ "types for code outside lib/jxl library." >&2 ret=1 fi done - for f in $(git ls-files | grep -E "\.h$" | grep -v -F printf_macros.h | + for f in $(git ls-files | grep -E "\.h$" | grep -v -E '(printf_macros\.h|test_utils\.h)' | xargs grep -n 'PRI[udx]S'); do # Having PRIuS / PRIdS in a header file means that printf_macros.h may # be included before a system header, in particular before gtest headers. # those may re-define PRIuS unconditionally causing a compile error. echo "$f: Don't use PRI.S in header files. Sorry." diff --git a/third_party/jpeg-xl/ci.sh b/third_party/jpeg-xl/ci.sh --- a/third_party/jpeg-xl/ci.sh +++ b/third_party/jpeg-xl/ci.sh @@ -39,26 +39,10 @@ # Time limit for the "fuzz" command in seconds (0 means no limit). FUZZER_MAX_TIME="${FUZZER_MAX_TIME:-0}" SANITIZER="none" -if [[ "${BUILD_TARGET}" == wasm* ]]; then - # Check that environment is setup for the WASM build target. - if [[ -z "${EMSCRIPTEN}" ]]; then - echo "'EMSCRIPTEN' is not defined. Use 'emconfigure' wrapper to setup WASM build environment" >&2 - return 1 - fi - # Remove the side-effect of "emconfigure" wrapper - it considers NodeJS environment. - unset EMMAKEN_JUST_CONFIGURE - EMS_TOOLCHAIN_FILE="${EMSCRIPTEN}/cmake/Modules/Platform/Emscripten.cmake" - if [[ -f "${EMS_TOOLCHAIN_FILE}" ]]; then - CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE:-${EMS_TOOLCHAIN_FILE}} - else - echo "Warning: EMSCRIPTEN CMake module not found" >&2 - fi - CMAKE_CROSSCOMPILING_EMULATOR="${MYDIR}/js-wasm-wrapper.sh" -fi if [[ "${BUILD_TARGET%%-*}" == "x86_64" || "${BUILD_TARGET%%-*}" == "i686" ]]; then # Default to building all targets, even if compiler baseline is SSE4 HWY_BASELINE_TARGETS=${HWY_BASELINE_TARGETS:-HWY_SCALAR} @@ -348,11 +332,10 @@ -B"${BUILD_DIR}" -H"${MYDIR}" -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" -G Ninja -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" - -DCMAKE_TOOLCHAIN_FILE="${CMAKE_TOOLCHAIN_FILE}" -DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}" -DCMAKE_MODULE_LINKER_FLAGS="${CMAKE_MODULE_LINKER_FLAGS}" -DCMAKE_SHARED_LINKER_FLAGS="${CMAKE_SHARED_LINKER_FLAGS}" -DJPEGXL_VERSION="${JPEGXL_VERSION}" -DSANITIZER="${SANITIZER}" @@ -390,15 +373,18 @@ -DCMAKE_C_COMPILER_TARGET="${BUILD_TARGET}" -DCMAKE_CXX_COMPILER_TARGET="${BUILD_TARGET}" # Only the first element of the target triplet. -DCMAKE_SYSTEM_PROCESSOR="${BUILD_TARGET%%-*}" -DCMAKE_SYSTEM_NAME="${system_name}" + -DCMAKE_TOOLCHAIN_FILE="${CMAKE_TOOLCHAIN_FILE}" ) else - # sjpeg confuses WASM SIMD with SSE. args+=( + # sjpeg confuses WASM SIMD with SSE. -DSJPEG_ENABLE_SIMD=OFF + # Building shared libs is not very useful for WASM. + -DBUILD_SHARED_LIBS=OFF ) fi args+=( # These are needed to make googletest work when cross-compiling. -DCMAKE_CROSSCOMPILING=1 @@ -455,11 +441,15 @@ if [[ -n "${CMAKE_MAKE_PROGRAM}" ]]; then args+=( -DCMAKE_MAKE_PROGRAM="${CMAKE_MAKE_PROGRAM}" ) fi - cmake "${args[@]}" "$@" + if [[ "${BUILD_TARGET}" == wasm* ]]; then + emcmake cmake "${args[@]}" "$@" + else + cmake "${args[@]}" "$@" + fi } cmake_build_and_test() { # gtest_discover_tests() runs the test binaries to discover the list of tests # at build time, which fails under qemu. @@ -492,11 +482,11 @@ # Configure the build to strip unused functions. This considerably reduces the # output size, specially for tests which only use a small part of the whole # library. strip_dead_code() { # Emscripten does tree shaking without any extra flags. - if [[ "${CMAKE_TOOLCHAIN_FILE##*/}" == "Emscripten.cmake" ]]; then + if [[ "${BUILD_TARGET}" == wasm* ]]; then return 0 fi # -ffunction-sections, -fdata-sections and -Wl,--gc-sections effectively # discard all unreachable code, reducing the code size. For this to work, we # need to also pass --no-export-dynamic to prevent it from exporting all the @@ -1018,15 +1008,15 @@ local jxl_jpeg_benchmarks=( "--num_reps=3" ) local images=( - "third_party/testdata/third_party/imagecompression.info/flower_foveon.png" + "testdata/jxl/flower/flower.png" ) local jpg_images=( - "third_party/testdata/third_party/imagecompression.info/flower_foveon.png.im_q85_420.jpg" + "testdata/jxl/flower/flower.png.im_q85_420.jpg" ) if [[ "${SKIP_CPUSET:-}" == "1" ]]; then # Use a single cpu config in this case. local cpu_confs=("?") diff --git a/third_party/jpeg-xl/debian/copyright b/third_party/jpeg-xl/debian/copyright --- a/third_party/jpeg-xl/debian/copyright +++ b/third_party/jpeg-xl/debian/copyright @@ -36,47 +36,27 @@ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -Files: third_party/testdata/third_party/imagecompression.info/* -Copyright: their respective owners. -License: License without any prohibitive copyright restrictions. - See https://imagecompression.info/test_images/ for details. - . - These Images are available without any prohibitive copyright restrictions. - . - These images are (c) there respective owners. You are granted full - redistribution and publication rights on these images provided: - . - 1. The origin of the pictures must not be misrepresented; you must not claim - that you took the original pictures. If you use, publish or redistribute them, - an acknowledgment would be appreciated but is not required. - 2. Altered versions must be plainly marked as such, and must not be - misinterpreted as being the originals. - 3. No payment is required for distribution of this material, it must be - available freely under the conditions stated here. That is, it is prohibited to - sell the material. - 4. This notice may not be removed or altered from any distribution. - -Files: third_party/testdata/third_party/pngsuite/* +Files: testdata/third_party/pngsuite/* Copyright: Willem van Schaik, 1996, 2011 License: PngSuite License See http://www.schaik.com/pngsuite/ for details. . Permission to use, copy, modify and distribute these images for any purpose and without fee is hereby granted. -Files: third_party/testdata/raw.pixls/* +Files: testdata/raw.pixls/* Copyright: their respective owners listed in https://raw.pixls.us/ License: CC0-1.0 -Files: third_party/testdata/raw.pixls/* +Files: testdata/raw.pixls/* Copyright: their respective owners listed in https://www.wesaturate.com/ License: CC0-1.0 -Files: third_party/testdata/third_party/wide-gamut-tests/ +Files: testdata/third_party/wide-gamut-tests/ Copyright: github.com/codelogic/wide-gamut-tests authors. License: Apache-2.0 License: Apache-2.0 Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/third_party/jpeg-xl/js-wasm-wrapper.sh b/third_party/jpeg-xl/js-wasm-wrapper.sh deleted file mode 100755 --- a/third_party/jpeg-xl/js-wasm-wrapper.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) the JPEG XL Project Authors. All rights reserved. -# -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -# Continuous integration helper module. This module is meant to be called from -# the .gitlab-ci.yml file during the continuous integration build, as well as -# from the command line for developers. - -# This wrapper is used to enable WASM SIMD when running tests. -# Unfortunately, it is impossible to pass the option directly via the -# CMAKE_CROSSCOMPILING_EMULATOR variable. - -# Fallback to default v8 binary, if override is not set. -V8="${V8:-$(which v8)}" -SCRIPT="$1" -shift -"${V8}" --experimental-wasm-simd "${SCRIPT}" -- "$@" diff --git a/third_party/jpeg-xl/lib/extras/codec.h b/third_party/jpeg-xl/lib/extras/codec.h --- a/third_party/jpeg-xl/lib/extras/codec.h +++ b/third_party/jpeg-xl/lib/extras/codec.h @@ -47,11 +47,11 @@ // Replaces "bytes" with an encoding of pixels transformed from c_current // color space to c_desired. Status Encode(const CodecInOut& io, extras::Codec codec, const ColorEncoding& c_desired, size_t bits_per_sample, - PaddedBytes* bytes, ThreadPool* pool = nullptr); + std::vector* bytes, ThreadPool* pool = nullptr); // Deduces codec, calls Encode and writes to file. Status EncodeToFile(const CodecInOut& io, const ColorEncoding& c_desired, size_t bits_per_sample, const std::string& pathname, ThreadPool* pool = nullptr); diff --git a/third_party/jpeg-xl/lib/extras/codec.cc b/third_party/jpeg-xl/lib/extras/codec.cc --- a/third_party/jpeg-xl/lib/extras/codec.cc +++ b/third_party/jpeg-xl/lib/extras/codec.cc @@ -50,20 +50,20 @@ } Status SetFromFile(const std::string& pathname, const extras::ColorHints& color_hints, CodecInOut* io, ThreadPool* pool, extras::Codec* orig_codec) { - PaddedBytes encoded; + std::vector encoded; JXL_RETURN_IF_ERROR(ReadFile(pathname, &encoded)); JXL_RETURN_IF_ERROR(SetFromBytes(Span(encoded), color_hints, io, pool, orig_codec)); return true; } Status Encode(const CodecInOut& io, const extras::Codec codec, const ColorEncoding& c_desired, size_t bits_per_sample, - PaddedBytes* bytes, ThreadPool* pool) { + std::vector* bytes, ThreadPool* pool) { JXL_CHECK(!io.Main().c_current().ICC().empty()); JXL_CHECK(!c_desired.ICC().empty()); io.CheckMetadata(); if (io.Main().IsJPEG()) { JXL_WARNING("Writing JPEG data as pixels"); @@ -161,11 +161,11 @@ if (bits_per_sample > 16 && codec == extras::Codec::kPNG) { JXL_WARNING("PNG only supports up to 16 bits per sample"); bits_per_sample = 16; } - PaddedBytes encoded; + std::vector encoded; return Encode(io, codec, c_desired, bits_per_sample, &encoded, pool) && WriteFile(encoded, pathname); } Status EncodeToFile(const CodecInOut& io, const std::string& pathname, diff --git a/third_party/jpeg-xl/lib/extras/codec_test.cc b/third_party/jpeg-xl/lib/extras/codec_test.cc --- a/third_party/jpeg-xl/lib/extras/codec_test.cc +++ b/third_party/jpeg-xl/lib/extras/codec_test.cc @@ -10,27 +10,36 @@ #include #include #include -#include "gtest/gtest.h" #include "lib/extras/dec/pgx.h" #include "lib/extras/dec/pnm.h" +#include "lib/extras/enc/encode.h" +#include "lib/extras/packed_image_convert.h" #include "lib/jxl/base/printf_macros.h" #include "lib/jxl/base/random.h" #include "lib/jxl/base/thread_pool_internal.h" #include "lib/jxl/color_management.h" #include "lib/jxl/enc_color_management.h" #include "lib/jxl/image.h" #include "lib/jxl/image_bundle.h" #include "lib/jxl/image_test_utils.h" +#include "lib/jxl/test_utils.h" #include "lib/jxl/testdata.h" namespace jxl { namespace extras { namespace { +using ::testing::AllOf; +using ::testing::Contains; +using ::testing::Field; +using ::testing::IsEmpty; +using ::testing::NotNull; +using ::testing::SizeIs; + std::string ExtensionFromCodec(Codec codec, const bool is_gray, const bool has_alpha, const size_t bits_per_sample) { switch (codec) { case Codec::kJPG: @@ -124,11 +133,11 @@ const CodecInOut io = CreateTestImage(xsize, ysize, is_gray, add_alpha, bits_per_sample, c_native); const ImageBundle& ib1 = io.Main(); - PaddedBytes encoded; + std::vector encoded; JXL_CHECK(Encode(io, codec, c_external, bits_per_sample, &encoded, pool)); CodecInOut io2; ColorHints color_hints; // Only for PNM because PNG will warn about ignoring them. @@ -215,11 +224,11 @@ JXL_CHECK( SetFromBytes(Span(orig), color_hints, &io, pool, nullptr)); const ImageBundle& ib1 = io.Main(); // Encode/Decode again to make sure Encode carries through all metadata. - PaddedBytes encoded; + std::vector encoded; JXL_CHECK(Encode(io, Codec::kPNG, io.metadata.m.color_encoding, io.metadata.m.bit_depth.bits_per_sample, &encoded, pool)); CodecInOut io2; JXL_CHECK(SetFromBytes(Span(encoded), color_hints, &io2, pool, @@ -394,8 +403,77 @@ // Primaries::k2100, &pool); } TEST(CodecTest, TestPNM) { TestCodecPNM(); } +TEST(CodecTest, FormatNegotiation) { + const std::vector accepted_formats = { + {/*num_channels=*/4, + /*data_type=*/JXL_TYPE_UINT16, + /*endianness=*/JXL_NATIVE_ENDIAN, + /*align=*/0}, + {/*num_channels=*/3, + /*data_type=*/JXL_TYPE_UINT8, + /*endianness=*/JXL_NATIVE_ENDIAN, + /*align=*/0}, + {/*num_channels=*/3, + /*data_type=*/JXL_TYPE_UINT16, + /*endianness=*/JXL_NATIVE_ENDIAN, + /*align=*/0}, + {/*num_channels=*/1, + /*data_type=*/JXL_TYPE_UINT8, + /*endianness=*/JXL_NATIVE_ENDIAN, + /*align=*/0}, + }; + + JxlBasicInfo info; + JxlEncoderInitBasicInfo(&info); + info.bits_per_sample = 12; + info.num_color_channels = 2; + + JxlPixelFormat format; + EXPECT_FALSE(SelectFormat(accepted_formats, info, &format)); + + info.num_color_channels = 3; + ASSERT_TRUE(SelectFormat(accepted_formats, info, &format)); + EXPECT_EQ(format.num_channels, info.num_color_channels); + // 16 is the smallest accepted format that can accommodate the 12-bit data. + EXPECT_EQ(format.data_type, JXL_TYPE_UINT16); +} + +TEST(CodecTest, EncodeToPNG) { + ThreadPool* const pool = nullptr; + + std::unique_ptr png_encoder = Encoder::FromExtension(".png"); + ASSERT_THAT(png_encoder, NotNull()); + + const PaddedBytes original_png = ReadTestData( + "third_party/wesaturate/500px/tmshre_riaphotographs_srgb8.png"); + PackedPixelFile ppf; + ASSERT_TRUE(extras::DecodeBytes(Span(original_png), + ColorHints(), SizeConstraints(), &ppf)); + + const JxlPixelFormat& format = ppf.frames.front().color.format; + ASSERT_THAT( + png_encoder->AcceptedFormats(), + Contains(AllOf(Field(&JxlPixelFormat::num_channels, format.num_channels), + Field(&JxlPixelFormat::data_type, format.data_type), + Field(&JxlPixelFormat::endianness, format.endianness)))); + EncodedImage encoded_png; + ASSERT_TRUE(png_encoder->Encode(ppf, &encoded_png, pool)); + EXPECT_THAT(encoded_png.icc, IsEmpty()); + ASSERT_THAT(encoded_png.bitstreams, SizeIs(1)); + + PackedPixelFile decoded_ppf; + ASSERT_TRUE( + extras::DecodeBytes(Span(encoded_png.bitstreams.front()), + ColorHints(), SizeConstraints(), &decoded_ppf)); + + CodecInOut io1, io2; + ASSERT_TRUE(ConvertPackedPixelFileToCodecInOut(ppf, pool, &io1)); + ASSERT_TRUE(ConvertPackedPixelFileToCodecInOut(decoded_ppf, pool, &io2)); + VerifyEqual(*io1.Main().color(), *io2.Main().color()); +} + } // namespace } // namespace extras } // namespace jxl diff --git a/third_party/jpeg-xl/lib/extras/enc/apng.h b/third_party/jpeg-xl/lib/extras/enc/apng.h --- a/third_party/jpeg-xl/lib/extras/enc/apng.h +++ b/third_party/jpeg-xl/lib/extras/enc/apng.h @@ -6,23 +6,26 @@ #ifndef LIB_EXTRAS_ENC_APNG_H_ #define LIB_EXTRAS_ENC_APNG_H_ // Encodes APNG images in memory. +#include "lib/extras/enc/encode.h" #include "lib/jxl/base/data_parallel.h" #include "lib/jxl/base/padded_bytes.h" #include "lib/jxl/base/span.h" #include "lib/jxl/base/status.h" #include "lib/jxl/codec_in_out.h" namespace jxl { namespace extras { +std::unique_ptr GetAPNGEncoder(); + // Encodes `io` into `bytes`. Status EncodeImageAPNG(const CodecInOut* io, const ColorEncoding& c_desired, size_t bits_per_sample, ThreadPool* pool, - PaddedBytes* bytes); + std::vector* bytes); } // namespace extras } // namespace jxl #endif // LIB_EXTRAS_ENC_APNG_H_ diff --git a/third_party/jpeg-xl/lib/extras/enc/apng.cc b/third_party/jpeg-xl/lib/extras/enc/apng.cc --- a/third_party/jpeg-xl/lib/extras/enc/apng.cc +++ b/third_party/jpeg-xl/lib/extras/enc/apng.cc @@ -41,10 +41,11 @@ #include #include #include "jxl/encode.h" +#include "lib/extras/packed_image_convert.h" #include "lib/jxl/base/compiler_specific.h" #include "lib/jxl/base/printf_macros.h" #include "lib/jxl/color_encoding_internal.h" #include "lib/jxl/dec_external_image.h" #include "lib/jxl/enc_color_management.h" @@ -59,13 +60,42 @@ namespace jxl { namespace extras { namespace { +class APNGEncoder : public Encoder { + public: + std::vector AcceptedFormats() const override { + std::vector formats; + for (const uint32_t num_channels : {1, 2, 3, 4}) { + for (const JxlDataType data_type : {JXL_TYPE_UINT8, JXL_TYPE_UINT16}) { + for (JxlEndianness endianness : {JXL_BIG_ENDIAN, JXL_LITTLE_ENDIAN}) { + formats.push_back(JxlPixelFormat{/*num_channels=*/num_channels, + /*data_type=*/data_type, + /*endianness=*/endianness, + /*align=*/0}); + } + } + } + return formats; + } + Status Encode(const PackedPixelFile& ppf, EncodedImage* encoded_image, + ThreadPool* pool) const override { + encoded_image->icc.clear(); + encoded_image->bitstreams.resize(1); + CodecInOut io; + JXL_RETURN_IF_ERROR(ConvertPackedPixelFileToCodecInOut(ppf, pool, &io)); + return EncodeImageAPNG(&io, io.metadata.m.color_encoding, + ppf.info.bits_per_sample, pool, + &encoded_image->bitstreams.front()); + } +}; + static void PngWrite(png_structp png_ptr, png_bytep data, png_size_t length) { - PaddedBytes* bytes = static_cast(png_get_io_ptr(png_ptr)); - bytes->append(data, data + length); + std::vector* bytes = + static_cast*>(png_get_io_ptr(png_ptr)); + bytes->insert(bytes->end(), data, data + length); } // Stores XMP and EXIF/IPTC into key/value strings for PNG class BlobsWriterPNG { public: @@ -122,13 +152,17 @@ } }; } // namespace +std::unique_ptr GetAPNGEncoder() { + return jxl::make_unique(); +} + Status EncodeImageAPNG(const CodecInOut* io, const ColorEncoding& c_desired, size_t bits_per_sample, ThreadPool* pool, - PaddedBytes* bytes) { + std::vector* bytes) { if (bits_per_sample > 8) { bits_per_sample = 16; } else if (bits_per_sample < 8) { // PNG can also do 4, 2, and 1 bits per sample, but it isn't implemented bits_per_sample = 8; diff --git a/third_party/jpeg-xl/lib/extras/enc/encode.h b/third_party/jpeg-xl/lib/extras/enc/encode.h new file mode 100644 --- /dev/null +++ b/third_party/jpeg-xl/lib/extras/enc/encode.h @@ -0,0 +1,69 @@ +// Copyright (c) the JPEG XL Project Authors. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#ifndef LIB_EXTRAS_ENC_ENCODE_H_ +#define LIB_EXTRAS_ENC_ENCODE_H_ + +// Facade for image encoders. + +#include +#include + +#include "lib/extras/dec/decode.h" +#include "lib/jxl/base/compiler_specific.h" +#include "lib/jxl/base/data_parallel.h" +#include "lib/jxl/base/padded_bytes.h" +#include "lib/jxl/base/span.h" +#include "lib/jxl/base/status.h" +#include "lib/jxl/codec_in_out.h" +#include "lib/jxl/field_encodings.h" // MakeBit + +namespace jxl { +namespace extras { + +struct EncodedImage { + // One (if the format supports animations or the image has only one frame) or + // more sequential bitstreams. + std::vector> bitstreams; + + // If the format does not support embedding color profiles into the bitstreams + // above, it will be present here, to be written as a separate file. If it + // does support them, this field will be empty. + std::vector icc; +}; + +class Encoder { + public: + static std::unique_ptr FromExtension(std::string extension); + + virtual ~Encoder() = default; + + virtual std::vector AcceptedFormats() const = 0; + + // Any existing data in encoded_image is discarded. + virtual Status Encode(const PackedPixelFile& ppf, EncodedImage* encoded_image, + ThreadPool* pool = nullptr) const = 0; + + void SetOption(std::string name, std::string value) { + options_[std::move(name)] = std::move(value); + } + + protected: + const std::unordered_map& options() const { + return options_; + } + + private: + std::unordered_map options_; +}; + +// TODO(sboukortt): consider exposing this as part of the C API. +Status SelectFormat(const std::vector& accepted_formats, + const JxlBasicInfo& basic_info, JxlPixelFormat* format); + +} // namespace extras +} // namespace jxl + +#endif // LIB_EXTRAS_ENC_ENCODE_H_ diff --git a/third_party/jpeg-xl/lib/extras/enc/encode.cc b/third_party/jpeg-xl/lib/extras/enc/encode.cc new file mode 100644 --- /dev/null +++ b/third_party/jpeg-xl/lib/extras/enc/encode.cc @@ -0,0 +1,84 @@ +// Copyright (c) the JPEG XL Project Authors. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "lib/extras/enc/encode.h" + +#include + +#if JPEGXL_ENABLE_APNG +#include "lib/extras/enc/apng.h" +#endif +#if JPEGXL_ENABLE_EXR +#include "lib/extras/enc/exr.h" +#endif +#if JPEGXL_ENABLE_JPEG +#include "lib/extras/enc/jpg.h" +#endif +#include "lib/extras/enc/pgx.h" +#include "lib/extras/enc/pnm.h" +#include "lib/jxl/base/printf_macros.h" + +namespace jxl { +namespace extras { + +Status SelectFormat(const std::vector& accepted_formats, + const JxlBasicInfo& basic_info, JxlPixelFormat* format) { + const size_t original_bit_depth = basic_info.bits_per_sample; + const auto num_channels = + basic_info.num_color_channels + (basic_info.alpha_bits != 0 ? 1 : 0); + size_t current_bit_depth = 0; + for (const JxlPixelFormat& candidate : accepted_formats) { + if (candidate.num_channels != num_channels) continue; + const size_t candidate_bit_depth = + PackedImage::BitsPerChannel(candidate.data_type); + if ( + // Candidate bit depth is less than what we have and still enough + (original_bit_depth <= candidate_bit_depth && + candidate_bit_depth < current_bit_depth) || + // Or larger than the too-small bit depth we currently have + (current_bit_depth < candidate_bit_depth && + current_bit_depth < original_bit_depth)) { + *format = candidate; + current_bit_depth = candidate_bit_depth; + } + } + if (current_bit_depth == 0) { + return JXL_FAILURE("no appropriate format found"); + } + if (current_bit_depth < original_bit_depth) { + JXL_WARNING("encoding %" PRIuS "-bit original to %" PRIuS " bits", + original_bit_depth, current_bit_depth); + } + return true; +} + +std::unique_ptr Encoder::FromExtension(std::string extension) { + std::transform( + extension.begin(), extension.end(), extension.begin(), + [](char c) { return std::tolower(c, std::locale::classic()); }); +#if JPEGXL_ENABLE_APNG + if (extension == ".png") return GetAPNGEncoder(); +#endif + +#if JPEGXL_ENABLE_JPEG + if (extension == ".jpg") return GetJPEGEncoder(); + if (extension == ".jpeg") return GetJPEGEncoder(); +#endif + + if (extension == ".pgx") return GetPGXEncoder(); + + if (extension == ".pgm") return GetPGMEncoder(); + if (extension == ".ppm") return GetPPMEncoder(); + if (extension == ".pfm") return GetPFMEncoder(); + +#if JPEGXL_ENABLE_EXR + if (extension == ".exr") return GetEXREncoder(); +#endif + + return nullptr; +} + +} // namespace extras +} // namespace jxl diff --git a/third_party/jpeg-xl/lib/extras/enc/exr.h b/third_party/jpeg-xl/lib/extras/enc/exr.h --- a/third_party/jpeg-xl/lib/extras/enc/exr.h +++ b/third_party/jpeg-xl/lib/extras/enc/exr.h @@ -6,10 +6,11 @@ #ifndef LIB_EXTRAS_ENC_EXR_H_ #define LIB_EXTRAS_ENC_EXR_H_ // Encodes OpenEXR images in memory. +#include "lib/extras/enc/encode.h" #include "lib/extras/packed_image.h" #include "lib/jxl/base/data_parallel.h" #include "lib/jxl/base/padded_bytes.h" #include "lib/jxl/base/span.h" #include "lib/jxl/base/status.h" @@ -17,14 +18,16 @@ #include "lib/jxl/color_encoding_internal.h" namespace jxl { namespace extras { +std::unique_ptr GetEXREncoder(); + // Transforms from io->c_current to `c_desired` (with the transfer function set // to linear as that is the OpenEXR convention) and encodes into `bytes`. Status EncodeImageEXR(const CodecInOut* io, const ColorEncoding& c_desired, - ThreadPool* pool, PaddedBytes* bytes); + ThreadPool* pool, std::vector* bytes); } // namespace extras } // namespace jxl #endif // LIB_EXTRAS_ENC_EXR_H_ diff --git a/third_party/jpeg-xl/lib/extras/enc/exr.cc b/third_party/jpeg-xl/lib/extras/enc/exr.cc --- a/third_party/jpeg-xl/lib/extras/enc/exr.cc +++ b/third_party/jpeg-xl/lib/extras/enc/exr.cc @@ -10,10 +10,11 @@ #include #include #include +#include "lib/extras/packed_image_convert.h" #include "lib/jxl/alpha.h" #include "lib/jxl/color_encoding_internal.h" #include "lib/jxl/color_management.h" #include "lib/jxl/enc_color_management.h" #include "lib/jxl/enc_image_bundle.h" @@ -45,11 +46,11 @@ } class InMemoryOStream : public OpenEXR::OStream { public: // `bytes` must outlive the InMemoryOStream. - explicit InMemoryOStream(PaddedBytes* const bytes) + explicit InMemoryOStream(std::vector* const bytes) : OStream(/*fileName=*/""), bytes_(*bytes) {} void write(const char c[], const int n) override { if (bytes_.size() < pos_ + n) { bytes_.resize(pos_ + n); @@ -65,93 +66,90 @@ } pos_ = pos; } private: - PaddedBytes& bytes_; + std::vector& bytes_; size_t pos_ = 0; }; -} // namespace - -Status EncodeImageEXR(const CodecInOut* io, const ColorEncoding& c_desired, - ThreadPool* pool, PaddedBytes* bytes) { +Status EncodeImageEXR(const ImageBundle& ib, const ColorEncoding& c_desired, + ThreadPool* pool, std::vector* bytes) { // As in `DecodeImageEXR`, `pool` is only used for pixel conversion, not for // actual OpenEXR I/O. OpenEXR::setGlobalThreadCount(GetNumThreads(pool)); ColorEncoding c_linear = c_desired; c_linear.tf.SetTransferFunction(TransferFunction::kLinear); JXL_RETURN_IF_ERROR(c_linear.CreateICC()); - ImageMetadata metadata = io->metadata.m; + ImageMetadata metadata = *ib.metadata(); ImageBundle store(&metadata); const ImageBundle* linear; - JXL_RETURN_IF_ERROR(TransformIfNeeded(io->Main(), c_linear, GetJxlCms(), pool, - &store, &linear)); + JXL_RETURN_IF_ERROR( + TransformIfNeeded(ib, c_linear, GetJxlCms(), pool, &store, &linear)); - const bool has_alpha = io->Main().HasAlpha(); - const bool alpha_is_premultiplied = io->Main().AlphaIsPremultiplied(); + const bool has_alpha = ib.HasAlpha(); + const bool alpha_is_premultiplied = ib.AlphaIsPremultiplied(); - OpenEXR::Header header(io->xsize(), io->ysize()); + OpenEXR::Header header(ib.xsize(), ib.ysize()); const PrimariesCIExy& primaries = c_linear.HasPrimaries() ? c_linear.GetPrimaries() : ColorEncoding::SRGB().GetPrimaries(); OpenEXR::Chromaticities chromaticities; chromaticities.red = Imath::V2f(primaries.r.x, primaries.r.y); chromaticities.green = Imath::V2f(primaries.g.x, primaries.g.y); chromaticities.blue = Imath::V2f(primaries.b.x, primaries.b.y); chromaticities.white = Imath::V2f(c_linear.GetWhitePoint().x, c_linear.GetWhitePoint().y); OpenEXR::addChromaticities(header, chromaticities); - OpenEXR::addWhiteLuminance(header, io->metadata.m.IntensityTarget()); + OpenEXR::addWhiteLuminance(header, ib.metadata()->IntensityTarget()); // Ensure that the destructor of RgbaOutputFile has run before we look at the // size of `bytes`. { InMemoryOStream os(bytes); OpenEXR::RgbaOutputFile output( os, header, has_alpha ? OpenEXR::WRITE_RGBA : OpenEXR::WRITE_RGB); // How many rows to write at once. Again, the OpenEXR documentation // recommends writing the whole image in one call. - const int y_chunk_size = io->ysize(); - std::vector output_rows(io->xsize() * y_chunk_size); + const int y_chunk_size = ib.ysize(); + std::vector output_rows(ib.xsize() * y_chunk_size); - for (size_t start_y = 0; start_y < io->ysize(); start_y += y_chunk_size) { + for (size_t start_y = 0; start_y < ib.ysize(); start_y += y_chunk_size) { // Inclusive. - const size_t end_y = - std::min(start_y + y_chunk_size - 1, io->ysize() - 1); - output.setFrameBuffer(output_rows.data() - start_y * io->xsize(), - /*xStride=*/1, /*yStride=*/io->xsize()); + const size_t end_y = std::min(start_y + y_chunk_size - 1, ib.ysize() - 1); + output.setFrameBuffer(output_rows.data() - start_y * ib.xsize(), + /*xStride=*/1, /*yStride=*/ib.xsize()); JXL_RETURN_IF_ERROR(RunOnPool( pool, start_y, end_y + 1, ThreadPool::NoInit, [&](const uint32_t y, size_t /* thread */) { const float* const JXL_RESTRICT input_rows[] = { linear->color().ConstPlaneRow(0, y), linear->color().ConstPlaneRow(1, y), linear->color().ConstPlaneRow(2, y), }; OpenEXR::Rgba* const JXL_RESTRICT row_data = - &output_rows[(y - start_y) * io->xsize()]; + &output_rows[(y - start_y) * ib.xsize()]; if (has_alpha) { const float* const JXL_RESTRICT alpha_row = - io->Main().alpha().ConstRow(y); + ib.alpha().ConstRow(y); if (alpha_is_premultiplied) { - for (size_t x = 0; x < io->xsize(); ++x) { + for (size_t x = 0; x < ib.xsize(); ++x) { row_data[x] = OpenEXR::Rgba(input_rows[0][x], input_rows[1][x], input_rows[2][x], alpha_row[x]); } } else { - for (size_t x = 0; x < io->xsize(); ++x) { + for (size_t x = 0; x < ib.xsize(); ++x) { row_data[x] = OpenEXR::Rgba(alpha_row[x] * input_rows[0][x], alpha_row[x] * input_rows[1][x], alpha_row[x] * input_rows[2][x], alpha_row[x]); } } } else { - for (size_t x = 0; x < io->xsize(); ++x) { + for (size_t x = 0; x < ib.xsize(); ++x) { row_data[x] = OpenEXR::Rgba(input_rows[0][x], input_rows[1][x], input_rows[2][x], 1.f); } } }, @@ -161,7 +159,49 @@ } return true; } +class EXREncoder : public Encoder { + std::vector AcceptedFormats() const override { + std::vector formats; + for (const uint32_t num_channels : {1, 2, 3, 4}) { + for (const JxlDataType data_type : {JXL_TYPE_FLOAT, JXL_TYPE_FLOAT16}) { + for (JxlEndianness endianness : {JXL_BIG_ENDIAN, JXL_LITTLE_ENDIAN}) { + formats.push_back(JxlPixelFormat{/*num_channels=*/num_channels, + /*data_type=*/data_type, + /*endianness=*/endianness, + /*align=*/0}); + } + } + } + return formats; + } + Status Encode(const PackedPixelFile& ppf, EncodedImage* encoded_image, + ThreadPool* pool = nullptr) const override { + encoded_image->icc.clear(); + CodecInOut io; + JXL_RETURN_IF_ERROR(ConvertPackedPixelFileToCodecInOut(ppf, pool, &io)); + encoded_image->bitstreams.clear(); + encoded_image->bitstreams.reserve(io.frames.size()); + for (const ImageBundle& ib : io.frames) { + encoded_image->bitstreams.emplace_back(); + JXL_RETURN_IF_ERROR(EncodeImageEXR(ib, io.metadata.m.color_encoding, pool, + &encoded_image->bitstreams.back())); + } + return true; + } +}; + +} // namespace + +std::unique_ptr GetEXREncoder() { + return jxl::make_unique(); +} + +Status EncodeImageEXR(const CodecInOut* io, const ColorEncoding& c_desired, + ThreadPool* pool, std::vector* bytes) { + return EncodeImageEXR(io->Main(), c_desired, pool, bytes); +} + } // namespace extras } // namespace jxl diff --git a/third_party/jpeg-xl/lib/extras/enc/jpg.h b/third_party/jpeg-xl/lib/extras/enc/jpg.h --- a/third_party/jpeg-xl/lib/extras/enc/jpg.h +++ b/third_party/jpeg-xl/lib/extras/enc/jpg.h @@ -8,11 +8,11 @@ // Encodes JPG pixels and metadata in memory. #include -#include "lib/extras/codec.h" +#include "lib/extras/enc/encode.h" #include "lib/jxl/base/data_parallel.h" #include "lib/jxl/base/padded_bytes.h" #include "lib/jxl/base/span.h" #include "lib/jxl/base/status.h" #include "lib/jxl/codec_in_out.h" @@ -23,14 +23,16 @@ enum class JpegEncoder { kLibJpeg, kSJpeg, }; +std::unique_ptr GetJPEGEncoder(); + // Encodes into `bytes`. Status EncodeImageJPG(const CodecInOut* io, JpegEncoder encoder, size_t quality, YCbCrChromaSubsampling chroma_subsampling, - ThreadPool* pool, PaddedBytes* bytes); + ThreadPool* pool, std::vector* bytes); } // namespace extras } // namespace jxl #endif // LIB_EXTRAS_ENC_JPG_H_ diff --git a/third_party/jpeg-xl/lib/extras/enc/jpg.cc b/third_party/jpeg-xl/lib/extras/enc/jpg.cc --- a/third_party/jpeg-xl/lib/extras/enc/jpg.cc +++ b/third_party/jpeg-xl/lib/extras/enc/jpg.cc @@ -10,13 +10,15 @@ #include #include #include #include +#include #include #include +#include "lib/extras/packed_image_convert.h" #include "lib/jxl/base/compiler_specific.h" #include "lib/jxl/base/status.h" #include "lib/jxl/color_encoding_internal.h" #include "lib/jxl/common.h" #include "lib/jxl/dec_external_image.h" @@ -91,29 +93,27 @@ chroma_subsampling.VShift(i < 2 ? i ^ 1 : i)); } return true; } -} // namespace - -Status EncodeWithLibJpeg(const ImageBundle* ib, const CodecInOut* io, +Status EncodeWithLibJpeg(const ImageBundle& ib, std::vector exif, size_t quality, const YCbCrChromaSubsampling& chroma_subsampling, - PaddedBytes* bytes) { + std::vector* bytes) { jpeg_compress_struct cinfo; // cinfo is initialized by libjpeg, which we are not instrumenting with // msan. msan::UnpoisonMemory(&cinfo, sizeof(cinfo)); jpeg_error_mgr jerr; cinfo.err = jpeg_std_error(&jerr); jpeg_create_compress(&cinfo); unsigned char* buffer = nullptr; unsigned long size = 0; jpeg_mem_dest(&cinfo, &buffer, &size); - cinfo.image_width = ib->oriented_xsize(); - cinfo.image_height = ib->oriented_ysize(); - if (ib->IsGray()) { + cinfo.image_width = ib.oriented_xsize(); + cinfo.image_height = ib.oriented_ysize(); + if (ib.IsGray()) { cinfo.input_components = 1; cinfo.in_color_space = JCS_GRAYSCALE; } else { cinfo.input_components = 3; cinfo.in_color_space = JCS_RGB; @@ -123,30 +123,29 @@ if (cinfo.input_components == 3) { JXL_RETURN_IF_ERROR(SetChromaSubsampling(chroma_subsampling, &cinfo)); } jpeg_set_quality(&cinfo, quality, TRUE); jpeg_start_compress(&cinfo, TRUE); - if (!ib->IsSRGB()) { - WriteICCProfile(&cinfo, ib->c_current().ICC()); + if (!ib.IsSRGB()) { + WriteICCProfile(&cinfo, ib.c_current().ICC()); } - if (!io->blobs.exif.empty()) { - std::vector exif = io->blobs.exif; + if (!exif.empty()) { ResetExifOrientation(exif); WriteExif(&cinfo, exif); } if (cinfo.input_components > 3 || cinfo.input_components < 0) return JXL_FAILURE("invalid numbers of components"); size_t stride = - ib->oriented_xsize() * cinfo.input_components * sizeof(JSAMPLE); - PaddedBytes raw_bytes(stride * ib->oriented_ysize()); + ib.oriented_xsize() * cinfo.input_components * sizeof(JSAMPLE); + std::vector raw_bytes(stride * ib.oriented_ysize()); JXL_RETURN_IF_ERROR(ConvertToExternal( - *ib, BITS_IN_JSAMPLE, /*float_out=*/false, cinfo.input_components, + ib, BITS_IN_JSAMPLE, /*float_out=*/false, cinfo.input_components, JXL_BIG_ENDIAN, stride, nullptr, raw_bytes.data(), raw_bytes.size(), - /*out_callback=*/{}, ib->metadata()->GetOrientation())); + /*out_callback=*/{}, ib.metadata()->GetOrientation())); - for (size_t y = 0; y < ib->oriented_ysize(); ++y) { + for (size_t y = 0; y < ib.oriented_ysize(); ++y) { JSAMPROW row[] = {raw_bytes.data() + y * stride}; jpeg_write_scanlines(&cinfo, row, 1); } jpeg_finish_compress(&cinfo); @@ -158,23 +157,22 @@ std::copy_n(buffer, size, bytes->data()); std::free(buffer); return true; } -Status EncodeWithSJpeg(const ImageBundle* ib, const CodecInOut* io, +Status EncodeWithSJpeg(const ImageBundle& ib, std::vector exif, size_t quality, const YCbCrChromaSubsampling& chroma_subsampling, - PaddedBytes* bytes) { + std::vector* bytes) { #if !JPEGXL_ENABLE_SJPEG return JXL_FAILURE("JPEG XL was built without sjpeg support"); #else sjpeg::EncoderParam param(quality); - if (!ib->IsSRGB()) { - param.iccp.assign(ib->metadata()->color_encoding.ICC().begin(), - ib->metadata()->color_encoding.ICC().end()); + if (!ib.IsSRGB()) { + param.iccp.assign(ib.metadata()->color_encoding.ICC().begin(), + ib.metadata()->color_encoding.ICC().end()); } - std::vector exif = io->blobs.exif; if (!exif.empty()) { ResetExifOrientation(exif); param.exif.assign(exif.begin(), exif.end()); } if (chroma_subsampling.Is444()) { @@ -182,58 +180,143 @@ } else if (chroma_subsampling.Is420()) { param.yuv_mode = SJPEG_YUV_SHARP; } else { return JXL_FAILURE("sjpeg does not support this chroma subsampling mode"); } - size_t stride = ib->oriented_xsize() * 3; - PaddedBytes rgb(ib->xsize() * ib->ysize() * 3); + size_t stride = ib.oriented_xsize() * 3; + std::vector rgb(ib.xsize() * ib.ysize() * 3); JXL_RETURN_IF_ERROR( - ConvertToExternal(*ib, 8, /*float_out=*/false, 3, JXL_BIG_ENDIAN, stride, + ConvertToExternal(ib, 8, /*float_out=*/false, 3, JXL_BIG_ENDIAN, stride, nullptr, rgb.data(), rgb.size(), - /*out_callback=*/{}, ib->metadata()->GetOrientation())); + /*out_callback=*/{}, ib.metadata()->GetOrientation())); std::string output; - JXL_RETURN_IF_ERROR(sjpeg::Encode(rgb.data(), ib->oriented_xsize(), - ib->oriented_ysize(), stride, param, + JXL_RETURN_IF_ERROR(sjpeg::Encode(rgb.data(), ib.oriented_xsize(), + ib.oriented_ysize(), stride, param, &output)); bytes->assign( reinterpret_cast(output.data()), reinterpret_cast(output.data() + output.size())); return true; #endif } -Status EncodeImageJPG(const CodecInOut* io, JpegEncoder encoder, size_t quality, +Status EncodeImageJPG(const ImageBundle& ib, std::vector exif, + JpegEncoder encoder, size_t quality, YCbCrChromaSubsampling chroma_subsampling, - ThreadPool* pool, PaddedBytes* bytes) { - if (io->Main().HasAlpha()) { + ThreadPool* pool, std::vector* bytes) { + if (ib.HasAlpha()) { return JXL_FAILURE("alpha is not supported"); } if (quality > 100) { return JXL_FAILURE("please specify a 0-100 JPEG quality"); } - const ImageBundle* ib; - ImageMetadata metadata = io->metadata.m; + const ImageBundle* transformed; + ImageMetadata metadata = *ib.metadata(); ImageBundle ib_store(&metadata); - JXL_RETURN_IF_ERROR(TransformIfNeeded(io->Main(), - io->metadata.m.color_encoding, - GetJxlCms(), pool, &ib_store, &ib)); + JXL_RETURN_IF_ERROR(TransformIfNeeded( + ib, metadata.color_encoding, GetJxlCms(), pool, &ib_store, &transformed)); switch (encoder) { case JpegEncoder::kLibJpeg: - JXL_RETURN_IF_ERROR( - EncodeWithLibJpeg(ib, io, quality, chroma_subsampling, bytes)); + JXL_RETURN_IF_ERROR(EncodeWithLibJpeg(ib, std::move(exif), quality, + chroma_subsampling, bytes)); break; case JpegEncoder::kSJpeg: - JXL_RETURN_IF_ERROR( - EncodeWithSJpeg(ib, io, quality, chroma_subsampling, bytes)); + JXL_RETURN_IF_ERROR(EncodeWithSJpeg(ib, std::move(exif), quality, + chroma_subsampling, bytes)); break; default: return JXL_FAILURE("tried to use an unknown JPEG encoder"); } return true; } +Status ParseChromaSubsampling(const std::string& param, + YCbCrChromaSubsampling* subsampling) { + const std::pair, std::array>> + options[] = {{"444", {{{1, 1, 1}}, {{1, 1, 1}}}}, + {"420", {{{2, 1, 1}}, {{2, 1, 1}}}}, + {"422", {{{2, 1, 1}}, {{1, 1, 1}}}}, + {"440", {{{1, 1, 1}}, {{2, 1, 1}}}}}; + for (const auto& option : options) { + if (param == option.first) { + return subsampling->Set(option.second.first.data(), + option.second.second.data()); + } + } + return false; +} + +class JPEGEncoder : public Encoder { + std::vector AcceptedFormats() const override { + std::vector formats; + for (const uint32_t num_channels : {1, 3}) { + for (JxlEndianness endianness : {JXL_BIG_ENDIAN, JXL_LITTLE_ENDIAN}) { + formats.push_back(JxlPixelFormat{/*num_channels=*/num_channels, + /*data_type=*/JXL_TYPE_UINT8, + /*endianness=*/endianness, + /*align=*/0}); + } + } + return formats; + } + Status Encode(const PackedPixelFile& ppf, EncodedImage* encoded_image, + ThreadPool* pool = nullptr) const override { + const auto& options = this->options(); + int quality = 100; + auto it_quality = options.find("q"); + if (it_quality != options.end()) { + std::istringstream is(it_quality->second); + JXL_RETURN_IF_ERROR(static_cast(is >> quality)); + } + YCbCrChromaSubsampling chroma_subsampling; + auto it_chroma_subsampling = options.find("chroma_subsampling"); + if (it_chroma_subsampling != options.end()) { + JXL_RETURN_IF_ERROR(ParseChromaSubsampling(it_chroma_subsampling->second, + &chroma_subsampling)); + } + JpegEncoder jpeg_encoder = JpegEncoder::kLibJpeg; + auto it_encoder = options.find("jpeg_encoder"); + if (it_encoder != options.end()) { + if (it_encoder->second == "libjpeg") { + jpeg_encoder = JpegEncoder::kLibJpeg; + } else if (it_encoder->second == "sjpeg") { + jpeg_encoder = JpegEncoder::kSJpeg; + } else { + return JXL_FAILURE("unknown jpeg encoder \"%s\"", + it_encoder->second.c_str()); + } + } + CodecInOut io; + JXL_RETURN_IF_ERROR(ConvertPackedPixelFileToCodecInOut(ppf, pool, &io)); + encoded_image->icc = ppf.icc; + encoded_image->bitstreams.clear(); + encoded_image->bitstreams.reserve(io.frames.size()); + for (const ImageBundle& ib : io.frames) { + encoded_image->bitstreams.emplace_back(); + JXL_RETURN_IF_ERROR(EncodeImageJPG(ib, ppf.metadata.exif, jpeg_encoder, + quality, chroma_subsampling, pool, + &encoded_image->bitstreams.back())); + } + return true; + } +}; + +} // namespace + +std::unique_ptr GetJPEGEncoder() { + return jxl::make_unique(); +} + +Status EncodeImageJPG(const CodecInOut* io, JpegEncoder encoder, size_t quality, + YCbCrChromaSubsampling chroma_subsampling, + ThreadPool* pool, std::vector* bytes) { + return EncodeImageJPG(io->Main(), io->blobs.exif, encoder, quality, + chroma_subsampling, pool, bytes); +} + } // namespace extras } // namespace jxl diff --git a/third_party/jpeg-xl/lib/extras/enc/pgx.h b/third_party/jpeg-xl/lib/extras/enc/pgx.h --- a/third_party/jpeg-xl/lib/extras/enc/pgx.h +++ b/third_party/jpeg-xl/lib/extras/enc/pgx.h @@ -9,10 +9,11 @@ // Encodes PGX pixels in memory. #include #include +#include "lib/extras/enc/encode.h" #include "lib/extras/packed_image.h" #include "lib/jxl/base/data_parallel.h" #include "lib/jxl/base/padded_bytes.h" #include "lib/jxl/base/span.h" #include "lib/jxl/base/status.h" @@ -20,14 +21,16 @@ #include "lib/jxl/color_encoding_internal.h" namespace jxl { namespace extras { +std::unique_ptr GetPGXEncoder(); + // Transforms from io->c_current to `c_desired` and encodes into `bytes`. Status EncodeImagePGX(const CodecInOut* io, const ColorEncoding& c_desired, size_t bits_per_sample, ThreadPool* pool, - PaddedBytes* bytes); + std::vector* bytes); } // namespace extras } // namespace jxl #endif // LIB_EXTRAS_ENC_PGX_H_ diff --git a/third_party/jpeg-xl/lib/extras/enc/pgx.cc b/third_party/jpeg-xl/lib/extras/enc/pgx.cc --- a/third_party/jpeg-xl/lib/extras/enc/pgx.cc +++ b/third_party/jpeg-xl/lib/extras/enc/pgx.cc @@ -6,10 +6,11 @@ #include "lib/extras/enc/pgx.h" #include #include +#include "lib/extras/packed_image_convert.h" #include "lib/jxl/base/bits.h" #include "lib/jxl/base/compiler_specific.h" #include "lib/jxl/base/file_io.h" #include "lib/jxl/base/printf_macros.h" #include "lib/jxl/color_management.h" @@ -45,33 +46,29 @@ JXL_RETURN_IF_ERROR(static_cast(*chars_written) < kMaxHeaderSize); return true; } -} // namespace - -Status EncodeImagePGX(const CodecInOut* io, const ColorEncoding& c_desired, +Status EncodeImagePGX(const ImageBundle& ib, const ColorEncoding& c_desired, size_t bits_per_sample, ThreadPool* pool, - PaddedBytes* bytes) { - if (!Bundle::AllDefault(io->metadata.m)) { + std::vector* bytes) { + if (!Bundle::AllDefault(*ib.metadata())) { JXL_WARNING("PGX encoder ignoring metadata - use a different codec"); } if (!c_desired.IsSRGB()) { JXL_WARNING( "PGX encoder cannot store custom ICC profile; decoder\n" "will need hint key=color_space to get the same values"); } - ImageBundle ib = io->Main().Copy(); - - ImageMetadata metadata = io->metadata.m; + ImageMetadata metadata = *ib.metadata(); ImageBundle store(&metadata); const ImageBundle* transformed; JXL_RETURN_IF_ERROR(TransformIfNeeded(ib, c_desired, GetJxlCms(), pool, &store, &transformed)); - PaddedBytes pixels(ib.xsize() * ib.ysize() * - (bits_per_sample / kBitsPerByte)); + std::vector pixels(ib.xsize() * ib.ysize() * + (bits_per_sample / kBitsPerByte)); size_t stride = ib.xsize() * (bits_per_sample / kBitsPerByte); JXL_RETURN_IF_ERROR( ConvertToExternal(*transformed, bits_per_sample, /*float_out=*/false, /*num_channels=*/1, JXL_BIG_ENDIAN, stride, pool, @@ -87,7 +84,51 @@ memcpy(bytes->data() + header_size, pixels.data(), pixels.size()); return true; } +class PGXEncoder : public Encoder { + public: + std::vector AcceptedFormats() const override { + std::vector formats; + for (const JxlDataType data_type : {JXL_TYPE_UINT8, JXL_TYPE_UINT16}) { + for (JxlEndianness endianness : {JXL_BIG_ENDIAN, JXL_LITTLE_ENDIAN}) { + formats.push_back(JxlPixelFormat{/*num_channels=*/1, + /*data_type=*/data_type, + /*endianness=*/endianness, + /*align=*/0}); + } + } + return formats; + } + Status Encode(const PackedPixelFile& ppf, EncodedImage* encoded_image, + ThreadPool* pool) const override { + CodecInOut io; + JXL_RETURN_IF_ERROR(ConvertPackedPixelFileToCodecInOut(ppf, pool, &io)); + const PaddedBytes& icc = io.Main().c_current().ICC(); + encoded_image->icc.assign(icc.begin(), icc.end()); + encoded_image->bitstreams.clear(); + encoded_image->bitstreams.reserve(io.frames.size()); + for (const ImageBundle& ib : io.frames) { + encoded_image->bitstreams.emplace_back(); + JXL_RETURN_IF_ERROR(EncodeImagePGX(ib, io.metadata.m.color_encoding, + ppf.info.bits_per_sample, pool, + &encoded_image->bitstreams.back())); + } + return true; + } +}; + +} // namespace + +std::unique_ptr GetPGXEncoder() { + return jxl::make_unique(); +} + +Status EncodeImagePGX(const CodecInOut* io, const ColorEncoding& c_desired, + size_t bits_per_sample, ThreadPool* pool, + std::vector* bytes) { + return EncodeImagePGX(io->Main(), c_desired, bits_per_sample, pool, bytes); +} + } // namespace extras } // namespace jxl diff --git a/third_party/jpeg-xl/lib/extras/enc/pnm.h b/third_party/jpeg-xl/lib/extras/enc/pnm.h --- a/third_party/jpeg-xl/lib/extras/enc/pnm.h +++ b/third_party/jpeg-xl/lib/extras/enc/pnm.h @@ -9,20 +9,25 @@ // Encodes/decodes PBM/PGM/PPM/PFM pixels in memory. // TODO(janwas): workaround for incorrect Win64 codegen (cause unknown) #include +#include "lib/extras/enc/encode.h" #include "lib/extras/packed_image.h" #include "lib/jxl/base/data_parallel.h" #include "lib/jxl/base/padded_bytes.h" #include "lib/jxl/base/status.h" #include "lib/jxl/codec_in_out.h" #include "lib/jxl/color_encoding_internal.h" namespace jxl { namespace extras { +std::unique_ptr GetPGMEncoder(); +std::unique_ptr GetPPMEncoder(); +std::unique_ptr GetPFMEncoder(); + // Transforms from io->c_current to `c_desired` and encodes into `bytes`. Status EncodeImagePNM(const PackedPixelFile& ppf, size_t bits_per_sample, ThreadPool* pool, size_t frame_index, std::vector* bytes); diff --git a/third_party/jpeg-xl/lib/extras/enc/pnm.cc b/third_party/jpeg-xl/lib/extras/enc/pnm.cc --- a/third_party/jpeg-xl/lib/extras/enc/pnm.cc +++ b/third_party/jpeg-xl/lib/extras/enc/pnm.cc @@ -30,10 +30,78 @@ namespace extras { namespace { constexpr size_t kMaxHeaderSize = 200; +class PNMEncoder : public Encoder { + public: + Status Encode(const PackedPixelFile& ppf, EncodedImage* encoded_image, + ThreadPool* pool = nullptr) const override { + encoded_image->icc = ppf.icc; + encoded_image->bitstreams.clear(); + encoded_image->bitstreams.reserve(ppf.frames.size()); + for (size_t i = 0; i < ppf.frames.size(); ++i) { + encoded_image->bitstreams.emplace_back(); + JXL_RETURN_IF_ERROR(EncodeImagePNM(ppf, ppf.info.bits_per_sample, pool, + /*frame_index=*/i, + &encoded_image->bitstreams.back())); + } + return true; + } +}; + +class PPMEncoder : public PNMEncoder { + public: + std::vector AcceptedFormats() const override { + std::vector formats; + for (const uint32_t num_channels : {1, 2, 3, 4}) { + for (const JxlDataType data_type : {JXL_TYPE_UINT8, JXL_TYPE_UINT16}) { + for (JxlEndianness endianness : {JXL_BIG_ENDIAN, JXL_LITTLE_ENDIAN}) { + formats.push_back(JxlPixelFormat{/*num_channels=*/num_channels, + /*data_type=*/data_type, + /*endianness=*/endianness, + /*align=*/0}); + } + } + } + return formats; + } +}; + +class PFMEncoder : public PNMEncoder { + public: + std::vector AcceptedFormats() const override { + std::vector formats; + for (const uint32_t num_channels : {1, 3}) { + for (const JxlDataType data_type : {JXL_TYPE_FLOAT16, JXL_TYPE_FLOAT}) { + for (JxlEndianness endianness : {JXL_BIG_ENDIAN, JXL_LITTLE_ENDIAN}) { + formats.push_back(JxlPixelFormat{/*num_channels=*/num_channels, + /*data_type=*/data_type, + /*endianness=*/endianness, + /*align=*/0}); + } + } + } + return formats; + } +}; + +class PGMEncoder : public PPMEncoder { + public: + std::vector AcceptedFormats() const override { + std::vector formats = PPMEncoder::AcceptedFormats(); + for (auto it = formats.begin(); it != formats.end();) { + if (it->num_channels > 2) { + it = formats.erase(it); + } else { + ++it; + } + } + return formats; + } +}; + Status EncodeHeader(const PackedPixelFile& ppf, const size_t bits_per_sample, const bool little_endian, char* header, int* JXL_RESTRICT chars_written) { bool is_gray = ppf.info.num_color_channels <= 2; size_t oriented_xsize = @@ -94,10 +162,22 @@ } } } // namespace +std::unique_ptr GetPPMEncoder() { + return jxl::make_unique(); +} + +std::unique_ptr GetPFMEncoder() { + return jxl::make_unique(); +} + +std::unique_ptr GetPGMEncoder() { + return jxl::make_unique(); +} + Status EncodeImagePNM(const PackedPixelFile& ppf, size_t bits_per_sample, ThreadPool* pool, size_t frame_index, std::vector* bytes) { const bool floating_point = bits_per_sample > 16; // Choose native for PFM; PGM/PPM require big-endian diff --git a/third_party/jpeg-xl/lib/extras/tone_mapping_gbench.cc b/third_party/jpeg-xl/lib/extras/tone_mapping_gbench.cc --- a/third_party/jpeg-xl/lib/extras/tone_mapping_gbench.cc +++ b/third_party/jpeg-xl/lib/extras/tone_mapping_gbench.cc @@ -11,12 +11,11 @@ namespace jxl { static void BM_ToneMapping(benchmark::State& state) { CodecInOut image; - const PaddedBytes image_bytes = - ReadTestData("third_party/imagecompression.info/flower_foveon.png"); + const PaddedBytes image_bytes = ReadTestData("jxl/flower/flower.png"); JXL_CHECK(SetFromBytes(Span(image_bytes), &image)); // Convert to linear Rec. 2020 so that `ToneMapTo` doesn't have to and we // mainly measure the tone mapping itself. ColorEncoding linear_rec2020; diff --git a/third_party/jpeg-xl/lib/include/jxl/color_encoding.h b/third_party/jpeg-xl/lib/include/jxl/color_encoding.h --- a/third_party/jpeg-xl/lib/include/jxl/color_encoding.h +++ b/third_party/jpeg-xl/lib/include/jxl/color_encoding.h @@ -88,11 +88,11 @@ JXL_TRANSFER_FUNCTION_UNKNOWN = 2, /** The gamma exponent is 1 */ JXL_TRANSFER_FUNCTION_LINEAR = 8, /** As specified in IEC 61966-2-1 sRGB */ JXL_TRANSFER_FUNCTION_SRGB = 13, - /** As specified in SMPTE ST 428-1 */ + /** As specified in SMPTE ST 2084 */ JXL_TRANSFER_FUNCTION_PQ = 16, /** As specified in SMPTE ST 428-1 */ JXL_TRANSFER_FUNCTION_DCI = 17, /** As specified in Rec. ITU-R BT.2100-1 (HLG) */ JXL_TRANSFER_FUNCTION_HLG = 18, diff --git a/third_party/jpeg-xl/lib/include/jxl/decode.h b/third_party/jpeg-xl/lib/include/jxl/decode.h --- a/third_party/jpeg-xl/lib/include/jxl/decode.h +++ b/third_party/jpeg-xl/lib/include/jxl/decode.h @@ -1277,21 +1277,20 @@ JXL_EXPORT JxlDecoderStatus JxlDecoderGetBoxSizeRaw(const JxlDecoder* dec, uint64_t* size); /** * Configures at which progressive steps in frame decoding the @ref - * JXL_DEC_FRAME_PROGRESSION event occurs. By default, this is 0. The detail - * values mean: 0 = only trigger for the DC image, the 8x8th lower resolution - * image. 1 = also trigger when a full pass of groups is ready. Higher values - * indicate more steps but are not yet implemented. Higher values always include - * the events of lower values as well. + * JXL_DEC_FRAME_PROGRESSION event occurs. The default value for the level of + * detail if this function is never called is `kDC`. * * @param dec decoder object * @param detail at which level of detail to trigger JXL_DEC_FRAME_PROGRESSION + * @return JXL_DEC_SUCCESS on success, JXL_DEC_ERROR on error, such as an + * invalid value for the progressive detail. */ -JXL_EXPORT void JxlDecoderSetProgressiveDetail(JxlDecoder* dec, - uint32_t detail); +JXL_EXPORT JxlDecoderStatus +JxlDecoderSetProgressiveDetail(JxlDecoder* dec, JxlProgressiveDetail detail); /** * Outputs progressive step towards the decoded image so far when only partial * input was received. If the flush was successful, the buffer set with * JxlDecoderSetImageOutBuffer will contain partial image data. diff --git a/third_party/jpeg-xl/lib/include/jxl/encode.h b/third_party/jpeg-xl/lib/include/jxl/encode.h --- a/third_party/jpeg-xl/lib/include/jxl/encode.h +++ b/third_party/jpeg-xl/lib/include/jxl/encode.h @@ -71,19 +71,64 @@ */ JXL_ENC_NEED_MORE_OUTPUT = 2, /** DEPRECATED: the encoder does not return this status and there is no need * to handle or expect it. + * Instead, JXL_ENC_ERROR is returned with error condition + * JXL_ENC_ERR_NOT_SUPPORTED. */ JXL_ENC_NOT_SUPPORTED = 3, } JxlEncoderStatus; /** - * Id of encoder options for a frame. This includes options such as the - * image quality and compression speed for this frame. This does not include - * non-frame related encoder options such as for boxes. + * Error conditions: + * API usage errors have the 0x80 bit set to 1 + * Other errors have the 0x80 bit set to 0 + */ +typedef enum { + /** No error + */ + JXL_ENC_ERR_OK = 0, + + /** Generic encoder error due to unspecified cause + */ + JXL_ENC_ERR_GENERIC = 1, + + /** Out of memory + * TODO(jon): actually catch this and return this error + */ + JXL_ENC_ERR_OOM = 2, + + /** JPEG bitstream reconstruction data could not be + * represented (e.g. too much tail data) + */ + JXL_ENC_ERR_JBRD = 3, + + /** Input is invalid (e.g. corrupt JPEG file or ICC profile) + */ + JXL_ENC_ERR_BAD_INPUT = 4, + + /** The encoder doesn't (yet) support this. Either no version of libjxl + * supports this, and the API is used incorrectly, or the libjxl version + * should have been checked before trying to do this. + */ + JXL_ENC_ERR_NOT_SUPPORTED = 0x80, + + /** The encoder API is used in an incorrect way. + * In this case, a debug build of libjxl should output a specific error + * message. (if not, please open an issue about it) + */ + JXL_ENC_ERR_API_USAGE = 0x81, + +} JxlEncoderError; + +/** + * Id of encoder options for a frame. This includes options such as setting + * encoding effort/speed or overriding the use of certain coding tools, for this + * frame. This does not include non-frame related encoder options such as for + * boxes. */ typedef enum { /** Sets encoder effort/speed level without affecting decoding speed. Valid * values are, from faster to slower speed: 1:lightning 2:thunder 3:falcon * 4:cheetah 5:hare 6:wombat 7:squirrel 8:kitten 9:tortoise. @@ -234,13 +279,16 @@ * YCbCr both perform no transform, but YCbCr is used to indicate that the * encoded data losslessly represents YCbCr values. */ JXL_ENC_FRAME_SETTING_COLOR_TRANSFORM = 24, - /** Color space for modular encoding: -1=default, 0-35=reverse color transform + /** Reversible color transform for modular encoding: -1=default, 0-48=RCT * index, e.g. index 0 = none, index 6 = YCoCg. - * The default behavior is to try several, depending on the speed setting. + * If this option is set to a non-default value, the RCT will be globally + * applied to the whole frame. + * The default behavior is to try several RCTs locally per modular group, + * depending on the speed and distance setting. */ JXL_ENC_FRAME_SETTING_MODULAR_COLOR_SPACE = 25, /** Group size for modular encoding: -1=default, 0=128, 1=256, 2=512, 3=1024. */ @@ -353,10 +401,19 @@ JXL_EXPORT JxlEncoderStatus JxlEncoderSetParallelRunner(JxlEncoder* enc, JxlParallelRunner parallel_runner, void* parallel_runner_opaque); /** + * Get the (last) error code in case JXL_ENC_ERROR was returned. + * + * @param enc encoder object. + * @return the JxlEncoderError that caused the (last) JXL_ENC_ERROR to be + * returned. + */ +JXL_EXPORT JxlEncoderError JxlEncoderGetError(JxlEncoder* enc); + +/** * Encodes JPEG XL file using the available bytes. @p *avail_out indicates how * many output bytes are available, and @p *next_out points to the input bytes. * *avail_out will be decremented by the amount of bytes that have been * processed by the encoder and *next_out will be incremented by the same * amount, so *next_out will now point at the amount of *avail_out unprocessed @@ -713,10 +770,11 @@ /** * Sets the original color encoding of the image encoded by this encoder. This * is an alternative to JxlEncoderSetICCProfile and only one of these two must * be used. This one sets the color encoding as a @ref JxlColorEncoding, while * the other sets it as ICC binary data. + * Must be called after JxlEncoderSetBasicInfo. * * @param enc encoder object. * @param color color encoding. Object owned by the caller and its contents are * copied internally. * @return JXL_ENC_SUCCESS if the operation was successful, JXL_ENC_ERROR or @@ -728,10 +786,11 @@ /** * Sets the original color encoding of the image encoded by this encoder as an * ICC color profile. This is an alternative to JxlEncoderSetColorEncoding and * only one of these two must be used. This one sets the color encoding as ICC * binary data, while the other defines it as a @ref JxlColorEncoding. + * Must be called after JxlEncoderSetBasicInfo. * * @param enc encoder object. * @param icc_profile bytes of the original ICC profile * @param size size of the icc_profile buffer in bytes * @return JXL_ENC_SUCCESS if the operation was successful, JXL_ENC_ERROR or diff --git a/third_party/jpeg-xl/lib/include/jxl/types.h b/third_party/jpeg-xl/lib/include/jxl/types.h --- a/third_party/jpeg-xl/lib/include/jxl/types.h +++ b/third_party/jpeg-xl/lib/include/jxl/types.h @@ -113,10 +113,36 @@ /** Data type holding the 4-character type name of an ISOBMFF box. */ typedef char JxlBoxType[4]; +/** Types of progressive detail. + * Setting a progressive detail with value N implies all progressive details + * with smaller or equal value. Currently only the following level of + * progressive detail is implemented: + * - kDC (which implies kFrames) + * - kPasses (which implies kLastPasses, kDC and kFrames) + */ +typedef enum { + // after completed kRegularFrames + kFrames = 0, + // after completed DC (1:8) + kDC = 1, + // after completed AC passes that are the last pass for their resolution + // target. + kLastPasses = 2, + // after completed AC passes that are not the last pass for their resolution + // target. + kPasses = 3, + // during DC frame when lower resolution are completed (1:32, 1:16) + kDCProgressive = 4, + // after completed groups + kDCGroups = 5, + // after completed groups + kGroups = 6, +} JxlProgressiveDetail; + #if defined(__cplusplus) || defined(c_plusplus) } #endif #endif /* JXL_TYPES_H_ */ diff --git a/third_party/jpeg-xl/lib/jxl.cmake b/third_party/jpeg-xl/lib/jxl.cmake --- a/third_party/jpeg-xl/lib/jxl.cmake +++ b/third_party/jpeg-xl/lib/jxl.cmake @@ -398,14 +398,14 @@ add_library(jxl_dec-obj OBJECT ${JPEGXL_INTERNAL_SOURCES_DEC}) target_compile_options(jxl_dec-obj PRIVATE ${JPEGXL_INTERNAL_FLAGS}) target_compile_options(jxl_dec-obj PUBLIC ${JPEGXL_COVERAGE_FLAGS}) set_property(TARGET jxl_dec-obj PROPERTY POSITION_INDEPENDENT_CODE ON) target_include_directories(jxl_dec-obj PUBLIC - ${PROJECT_SOURCE_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/include - $ - $ + "$" + "$" + "$>" + "$>" ) target_compile_definitions(jxl_dec-obj PUBLIC ${OBJ_COMPILE_DEFINITIONS} ) if (JPEGXL_ENABLE_PROFILER) @@ -468,13 +468,13 @@ $ ) target_link_libraries(jxl_dec-static PUBLIC ${JPEGXL_COVERAGE_FLAGS} ${JPEGXL_DEC_INTERNAL_LIBS}) target_include_directories(jxl_dec-static PUBLIC - "${PROJECT_SOURCE_DIR}" - "${CMAKE_CURRENT_SOURCE_DIR}/include" - "${CMAKE_CURRENT_BINARY_DIR}/include") + "$" + "$" + "$") # The list of objects in the static and shared libraries. set(JPEGXL_INTERNAL_OBJECTS $ $ @@ -489,13 +489,13 @@ # to do, remove $ here and depend on jxl_dec-static add_library(jxl-static STATIC ${JPEGXL_INTERNAL_OBJECTS}) target_link_libraries(jxl-static PUBLIC ${JPEGXL_COVERAGE_FLAGS} ${JPEGXL_INTERNAL_LIBS}) target_include_directories(jxl-static PUBLIC - "${PROJECT_SOURCE_DIR}" - "${CMAKE_CURRENT_SOURCE_DIR}/include" - "${CMAKE_CURRENT_BINARY_DIR}/include") + "$" + "$" + "$") # JXL_EXPORT is defined to "__declspec(dllimport)" automatically by CMake # in Windows builds when including headers from the C API and compiling from # outside the jxl library. This is required when using the shared library, # however in windows this causes the function to not be found when linking @@ -541,12 +541,12 @@ strip_static(JPEGXL_INTERNAL_SHARED_LIBS JPEGXL_INTERNAL_LIBS) target_link_libraries(jxl PUBLIC ${JPEGXL_COVERAGE_FLAGS}) target_link_libraries(jxl PRIVATE ${JPEGXL_INTERNAL_SHARED_LIBS}) # Shared library include path contains only the "include/" paths. target_include_directories(jxl PUBLIC - "${CMAKE_CURRENT_SOURCE_DIR}/include" - "${CMAKE_CURRENT_BINARY_DIR}/include") + "$" + "$") set_target_properties(jxl PROPERTIES VERSION ${JPEGXL_LIBRARY_VERSION} SOVERSION ${JPEGXL_LIBRARY_SOVERSION} LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") @@ -614,9 +614,23 @@ set(JPEGXL_LIBRARY_REQUIRES "libhwy libbrotlicommon libbrotlienc libbrotlidec") if(NOT JPEGXL_ENABLE_SKCMS) set(JPEGXL_LIBRARY_REQUIRES "${JPEGXL_LIBRARY_REQUIRES} lcms2") endif() + +# Allow adding prefix if CMAKE_INSTALL_INCLUDEDIR not absolute. +if(IS_ABSOLUTE "${CMAKE_INSTALL_INCLUDEDIR}") + set(PKGCONFIG_TARGET_INCLUDES "${CMAKE_INSTALL_INCLUDEDIR}") +else() + set(PKGCONFIG_TARGET_INCLUDES "\${prefix}/${CMAKE_INSTALL_INCLUDEDIR}") +endif() +# Allow adding prefix if CMAKE_INSTALL_LIBDIR not absolute. +if(IS_ABSOLUTE "${CMAKE_INSTALL_LIBDIR}") + set(PKGCONFIG_TARGET_LIBS "${CMAKE_INSTALL_LIBDIR}") +else() + set(PKGCONFIG_TARGET_LIBS "\${exec_prefix}/${CMAKE_INSTALL_LIBDIR}") +endif() + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/jxl/libjxl.pc.in" "libjxl.pc" @ONLY) install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libjxl.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") diff --git a/third_party/jpeg-xl/lib/jxl/alpha_test.cc b/third_party/jpeg-xl/lib/jxl/alpha_test.cc --- a/third_party/jpeg-xl/lib/jxl/alpha_test.cc +++ b/third_party/jpeg-xl/lib/jxl/alpha_test.cc @@ -3,12 +3,11 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. #include "lib/jxl/alpha.h" -#include "gmock/gmock.h" -#include "gtest/gtest.h" +#include "lib/jxl/test_utils.h" namespace jxl { namespace { using ::testing::_; diff --git a/third_party/jpeg-xl/lib/jxl/base/file_io.h b/third_party/jpeg-xl/lib/jxl/base/file_io.h --- a/third_party/jpeg-xl/lib/jxl/base/file_io.h +++ b/third_party/jpeg-xl/lib/jxl/base/file_io.h @@ -75,11 +75,12 @@ template static inline Status ReadFile(const std::string& pathname, ContainerType* JXL_RESTRICT bytes) { FileWrapper f(pathname, "rb"); - if (f == nullptr) return JXL_FAILURE("Failed to open file for reading"); + if (f == nullptr) + return JXL_FAILURE("Failed to open file for reading: %s", pathname.c_str()); // Get size of file in bytes const int64_t size = f.size(); if (size < 0) { // Size is unknown, loop reading chunks until EOF. diff --git a/third_party/jpeg-xl/lib/jxl/blending_test.cc b/third_party/jpeg-xl/lib/jxl/blending_test.cc --- a/third_party/jpeg-xl/lib/jxl/blending_test.cc +++ b/third_party/jpeg-xl/lib/jxl/blending_test.cc @@ -1,15 +1,14 @@ // Copyright (c) the JPEG XL Project Authors. All rights reserved. // // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -#include "gmock/gmock.h" -#include "gtest/gtest.h" #include "lib/extras/codec.h" #include "lib/jxl/dec_file.h" #include "lib/jxl/image_test_utils.h" +#include "lib/jxl/test_utils.h" #include "lib/jxl/testdata.h" namespace jxl { namespace { diff --git a/third_party/jpeg-xl/lib/jxl/color_management_test.cc b/third_party/jpeg-xl/lib/jxl/color_management_test.cc --- a/third_party/jpeg-xl/lib/jxl/color_management_test.cc +++ b/third_party/jpeg-xl/lib/jxl/color_management_test.cc @@ -11,12 +11,10 @@ #include #include #include #include -#include "gmock/gmock.h" -#include "gtest/gtest.h" #include "lib/jxl/base/compiler_specific.h" #include "lib/jxl/base/data_parallel.h" #include "lib/jxl/base/file_io.h" #include "lib/jxl/base/thread_pool_internal.h" #include "lib/jxl/enc_color_management.h" diff --git a/third_party/jpeg-xl/lib/jxl/dec_cache.h b/third_party/jpeg-xl/lib/jxl/dec_cache.h --- a/third_party/jpeg-xl/lib/jxl/dec_cache.h +++ b/third_party/jpeg-xl/lib/jxl/dec_cache.h @@ -6,10 +6,11 @@ #ifndef LIB_JXL_DEC_CACHE_H_ #define LIB_JXL_DEC_CACHE_H_ #include +#include #include // HWY_ALIGN_MAX #include "jxl/decode.h" #include "lib/jxl/ac_strategy.h" #include "lib/jxl/base/profiler.h" diff --git a/third_party/jpeg-xl/lib/jxl/dec_cache.cc b/third_party/jpeg-xl/lib/jxl/dec_cache.cc --- a/third_party/jpeg-xl/lib/jxl/dec_cache.cc +++ b/third_party/jpeg-xl/lib/jxl/dec_cache.cc @@ -87,11 +87,13 @@ } } } if ((frame_header.flags & FrameHeader::kPatches) != 0) { - builder.AddStage(GetPatchesStage(&shared->image_features.patches)); + builder.AddStage( + GetPatchesStage(&shared->image_features.patches, + 3 + shared->metadata->m.num_extra_channels)); } if ((frame_header.flags & FrameHeader::kSplines) != 0) { builder.AddStage(GetSplineStage(&shared->image_features.splines)); } diff --git a/third_party/jpeg-xl/lib/jxl/dec_frame.h b/third_party/jpeg-xl/lib/jxl/dec_frame.h --- a/third_party/jpeg-xl/lib/jxl/dec_frame.h +++ b/third_party/jpeg-xl/lib/jxl/dec_frame.h @@ -7,10 +7,11 @@ #define LIB_JXL_DEC_FRAME_H_ #include #include "jxl/decode.h" +#include "jxl/types.h" #include "lib/jxl/base/compiler_specific.h" #include "lib/jxl/base/data_parallel.h" #include "lib/jxl/base/span.h" #include "lib/jxl/base/status.h" #include "lib/jxl/blending.h" @@ -133,17 +134,25 @@ // Returns whether a DC image has been decoded, accessible at low resolution // at passes.shared_storage.dc_storage bool HasDecodedDC() const { return finalized_dc_; } bool HasDecodedAll() const { return NumSections() == num_sections_done_; } + size_t NumCompletePasses() const { + return *std::min_element(decoded_passes_per_ac_group_.begin(), + decoded_passes_per_ac_group_.end()); + }; + // If enabled, ProcessSections will stop and return true when the DC // sections have been processed, instead of starting the AC sections. This // will only occur if supported (that is, flushing will produce a valid // 1/8th*1/8th resolution image). The return value of true then does not mean // all sections have been processed, use HasDecodedDC and HasDecodedAll // to check the true finished state. - void SetPauseAtProgressive() { pause_at_progressive_ = true; } + void SetPauseAtProgressive(JxlProgressiveDetail prog_detail) { + pause_at_progressive_ = true; + progressive_detail_ = prog_detail; + } // Sets the buffer to which uint8 sRGB pixels will be decoded. This is not // supported for all images. If it succeeds, HasRGBBuffer() will return true. // If it does not succeed, the image is decoded to the ImageBundle passed to // InitFrame instead. @@ -284,10 +293,11 @@ // Testing setting: whether or not to use the slow rendering pipeline. bool use_slow_rendering_pipeline_; bool pause_at_progressive_ = false; + JxlProgressiveDetail progressive_detail_ = kDC; }; } // namespace jxl #endif // LIB_JXL_DEC_FRAME_H_ diff --git a/third_party/jpeg-xl/lib/jxl/dec_frame.cc b/third_party/jpeg-xl/lib/jxl/dec_frame.cc --- a/third_party/jpeg-xl/lib/jxl/dec_frame.cc +++ b/third_party/jpeg-xl/lib/jxl/dec_frame.cc @@ -13,10 +13,11 @@ #include #include #include #include +#include "jxl/types.h" #include "lib/jxl/ac_context.h" #include "lib/jxl/ac_strategy.h" #include "lib/jxl/ans_params.h" #include "lib/jxl/base/bits.h" #include "lib/jxl/base/compiler_specific.h" @@ -654,21 +655,23 @@ size_t ac_global_sec = num; std::vector dc_group_sec(frame_dim_.num_dc_groups, num); std::vector> ac_group_sec( frame_dim_.num_groups, std::vector(frame_header_.passes.num_passes, num)); - std::vector num_ac_passes(frame_dim_.num_groups); + // This keeps track of the number of ac passes we want to process during this + // call of ProcessSections. + std::vector desired_num_ac_passes(frame_dim_.num_groups); bool single_section = frame_dim_.num_groups == 1 && frame_header_.passes.num_passes == 1; if (single_section) { JXL_ASSERT(num == 1); JXL_ASSERT(sections[0].id == 0); if (processed_section_[0] == false) { processed_section_[0] = true; ac_group_sec[0].resize(1); dc_global_sec = ac_global_sec = dc_group_sec[0] = ac_group_sec[0][0] = 0; - num_ac_passes[0] = 1; + desired_num_ac_passes[0] = 1; } else { section_status[0] = SectionStatus::kDuplicate; } } else { size_t ac_global_index = frame_dim_.num_dc_groups + 1; @@ -704,11 +707,11 @@ for (; j + decoded_passes_per_ac_group_[g] < max_passes_; j++) { if (ac_group_sec[g][j + decoded_passes_per_ac_group_[g]] == num) { break; } } - num_ac_passes[g] = j; + desired_num_ac_passes[g] = j; } } if (dc_global_sec != num) { Status dc_global_status = ProcessDCGlobal(sections[dc_global_sec].br); if (dc_global_status.IsFatalError()) return dc_global_status; @@ -735,59 +738,72 @@ }, "DecodeDCGroup")); } if (has_error) return JXL_FAILURE("Error in DC group"); + bool do_progressive = false; + if (pause_at_progressive_ && (frame_header_.frame_type != kSkipProgressive)) { + do_progressive = true; + if (single_section) { + // If there's only one group and one pass, there is no separate section + // for DC and the entire full resolution image is available at once. + do_progressive = false; + } + if (!decoded_->metadata()->extra_channel_info.empty()) { + // If extra channels are encoded with modular without squeeze, they + // don't support DC. If the are encoded with squeeze, DC works in theory + // but the implementation may not yet correctly support this for Flush. + // Therefore, can't correctly pause for a progressive step if there is + // an extra channel (including alpha channel) + // TOOD(firsching): Check if this is still the case. + do_progressive = false; + } + if (frame_header_.encoding != FrameEncoding::kVarDCT) { + // DC is not guaranteed to be available in modular mode and may be a + // black image. If squeeze is used, it may be available depending on the + // current implementation. + // TODO(lode): do return DC if it's known that flushing at this point will + // produce a valid 1/8th downscaled image with modular encoding. + do_progressive = false; + } + } if (*std::min_element(decoded_dc_groups_.begin(), decoded_dc_groups_.end()) && !finalized_dc_) { PassesDecoderState::PipelineOptions pipeline_options; pipeline_options.use_slow_render_pipeline = use_slow_rendering_pipeline_; pipeline_options.coalescing = coalescing_; pipeline_options.render_spotcolors = render_spotcolors_; JXL_RETURN_IF_ERROR( dec_state_->PreparePipeline(decoded_, pipeline_options)); FinalizeDC(); JXL_RETURN_IF_ERROR(AllocateOutput()); - if (pause_at_progressive_ && !single_section) { - bool can_return_dc = true; - if (single_section) { - // If there's only one group and one pass, there is no separate section - // for DC and the entire full resolution image is available at once. - can_return_dc = false; - } - if (!decoded_->metadata()->extra_channel_info.empty()) { - // If extra channels are encoded with modular without squeeze, they - // don't support DC. If the are encoded with squeeze, DC works in theory - // but the implementation may not yet correctly support this for Flush. - // Therefore, can't correctly pause for a progressive step if there is - // an extra channel (including alpha channel) - can_return_dc = false; - } - if (frame_header_.encoding != FrameEncoding::kVarDCT) { - // DC is not guaranteed to be available in modular mode and may be a - // black image. If squeeze is used, it may be available depending on the - // current implementation. - // TODO(lode): do return DC if it's known that flushing at this point - // will produce a valid 1/8th downscaled image with modular encoding. - can_return_dc = false; - } - if (can_return_dc) { - MarkSections(sections, num, section_status); - return true; - } + if (do_progressive && progressive_detail_ >= JxlProgressiveDetail::kDC) { + MarkSections(sections, num, section_status); + return true; } } if (finalized_dc_ && ac_global_sec != num && !decoded_ac_global_) { JXL_RETURN_IF_ERROR(ProcessACGlobal(sections[ac_global_sec].br)); section_status[ac_global_sec] = SectionStatus::kDone; } + if (do_progressive && progressive_detail_ >= JxlProgressiveDetail::kPasses) { + // Mark that we only want the next pass. + size_t num_complete_passes = NumCompletePasses(); + for (size_t i = 0; i < ac_group_sec.size(); i++) { + desired_num_ac_passes[i] = + std::min(desired_num_ac_passes[i], + 1 + num_complete_passes - decoded_passes_per_ac_group_[i]); + } + } + if (decoded_ac_global_) { // Mark all the AC groups that we received as not complete yet. for (size_t i = 0; i < ac_group_sec.size(); i++) { - if (num_ac_passes[i] == 0 && !modular_frame_decoder_.UsesFullImage()) { + if (desired_num_ac_passes[i] == 0 && + !modular_frame_decoder_.UsesFullImage()) { continue; } dec_state_->render_pipeline->ClearDone(i); } @@ -795,28 +811,29 @@ pool_, 0, ac_group_sec.size(), [this](size_t num_threads) { return PrepareStorage(num_threads, decoded_passes_per_ac_group_.size()); }, - [this, &ac_group_sec, &num_ac_passes, &num, §ions, §ion_status, - &has_error](size_t g, size_t thread) { - if (num_ac_passes[g] == 0) { // no new AC pass, nothing to do. + [this, &ac_group_sec, &desired_num_ac_passes, &num, §ions, + §ion_status, &has_error](size_t g, size_t thread) { + if (desired_num_ac_passes[g] == 0) { + // no new AC pass, nothing to do return; } (void)num; size_t first_pass = decoded_passes_per_ac_group_[g]; BitReader* JXL_RESTRICT readers[kMaxNumPasses]; - for (size_t i = 0; i < num_ac_passes[g]; i++) { + for (size_t i = 0; i < desired_num_ac_passes[g]; i++) { JXL_ASSERT(ac_group_sec[g][first_pass + i] != num); readers[i] = sections[ac_group_sec[g][first_pass + i]].br; } - if (!ProcessACGroup(g, readers, num_ac_passes[g], + if (!ProcessACGroup(g, readers, desired_num_ac_passes[g], GetStorageLocation(thread, g), /*force_draw=*/false, /*dc_only=*/false)) { has_error = true; } else { - for (size_t i = 0; i < num_ac_passes[g]; i++) { + for (size_t i = 0; i < desired_num_ac_passes[g]; i++) { section_status[ac_group_sec[g][first_pass + i]] = SectionStatus::kDone; } } }, diff --git a/third_party/jpeg-xl/lib/jxl/dec_modular.cc b/third_party/jpeg-xl/lib/jxl/dec_modular.cc --- a/third_party/jpeg-xl/lib/jxl/dec_modular.cc +++ b/third_party/jpeg-xl/lib/jxl/dec_modular.cc @@ -5,10 +5,11 @@ #include "lib/jxl/dec_modular.h" #include +#include #include #include "lib/jxl/frame_header.h" #undef HWY_TARGET_INCLUDE diff --git a/third_party/jpeg-xl/lib/jxl/decode.cc b/third_party/jpeg-xl/lib/jxl/decode.cc --- a/third_party/jpeg-xl/lib/jxl/decode.cc +++ b/third_party/jpeg-xl/lib/jxl/decode.cc @@ -3,19 +3,21 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. #include "jxl/decode.h" +#include "jxl/types.h" #include "lib/jxl/base/byte_order.h" #include "lib/jxl/base/span.h" #include "lib/jxl/base/status.h" #include "lib/jxl/box_content_decoder.h" #include "lib/jxl/dec_external_image.h" #include "lib/jxl/dec_frame.h" #include "lib/jxl/dec_modular.h" #include "lib/jxl/decode_to_jpeg.h" #include "lib/jxl/fields.h" +#include "lib/jxl/frame_header.h" #include "lib/jxl/headers.h" #include "lib/jxl/icc_codec.h" #include "lib/jxl/image_bundle.h" #include "lib/jxl/loop_filter.h" #include "lib/jxl/memory_manager_internal.h" @@ -449,10 +451,13 @@ // Fields for reading the basic info from the header. size_t basic_info_size_hint; bool have_container; size_t box_count; + // The level of progressive detail in frame decoding. + JxlProgressiveDetail prog_detail = kDC; + // Whether the preview out buffer was set. It is possible for the buffer to // be nullptr and buffer_set to be true, indicating it was deliberately // set to nullptr. bool preview_out_buffer_set; // Idem for the image buffer. @@ -1393,11 +1398,11 @@ dec->passes_state.get(), dec->metadata, dec->thread_pool.get(), /*use_slow_rendering_pipeline=*/false)); dec->frame_dec->SetRenderSpotcolors(dec->render_spotcolors); dec->frame_dec->SetCoalescing(dec->coalescing); if (dec->events_wanted & JXL_DEC_FRAME_PROGRESSION) { - dec->frame_dec->SetPauseAtProgressive(); + dec->frame_dec->SetPauseAtProgressive(dec->prog_detail); } // If JPEG reconstruction is wanted and possible, set the jpeg_data of // the ImageBundle. if (!dec->jpeg_decoder.SetImageBundleJpegData(dec->ib.get())) @@ -1427,10 +1432,11 @@ } bool return_full_image = false; if (dec->frame_stage == FrameStage::kFull) { + size_t num_complete_passes = 0; if (dec->events_wanted & JXL_DEC_FULL_IMAGE) { if (!dec->image_out_buffer_set && (!dec->jpeg_decoder.IsOutputSet() || dec->ib->jpeg_data == nullptr) && dec->is_last_of_still) { @@ -1510,11 +1516,32 @@ bool got_dc_only = !!status && !all_sections_done && dec->frame_dec->HasDecodedDC(); if ((dec->events_wanted & JXL_DEC_FRAME_PROGRESSION) && got_dc_only) { - dec->events_wanted &= ~JXL_DEC_FRAME_PROGRESSION; + if (dec->prog_detail <= JxlProgressiveDetail::kDC) { + dec->events_wanted &= ~JXL_DEC_FRAME_PROGRESSION; + } + if (dec->frame_header->frame_type != kSkipProgressive) { + return JXL_DEC_FRAME_PROGRESSION; + } + } + + size_t new_num_complete_passes = dec->frame_dec->NumCompletePasses(); + + bool new_group_done = num_complete_passes < new_num_complete_passes; + if (new_group_done) { + num_complete_passes = new_num_complete_passes; + } + + bool got_complete_group_only = + !!status && !all_sections_done && new_group_done; + + if ((dec->events_wanted & JXL_DEC_FRAME_PROGRESSION) && + got_complete_group_only && + (dec->frame_header->frame_type != kSkipProgressive) && + (dec->prog_detail <= JxlProgressiveDetail::kPasses)) { return JXL_DEC_FRAME_PROGRESSION; } if (!all_sections_done) { // Not all sections have been processed yet @@ -2179,10 +2206,12 @@ JxlDecoderStatus JxlDecoderGetBasicInfo(const JxlDecoder* dec, JxlBasicInfo* info) { if (!dec->got_basic_info) return JXL_DEC_NEED_MORE_INPUT; if (info) { + memset(info, 0, sizeof(*info)); + const jxl::ImageMetadata& meta = dec->metadata.m; info->have_container = dec->have_container; info->xsize = dec->metadata.size.xsize(); info->ysize = dec->metadata.size.ysize(); @@ -2699,10 +2728,11 @@ JxlFrameHeader* header) { if (!dec->frame_header || dec->frame_stage == FrameStage::kHeader) { return JXL_API_ERROR("no frame header available"); } const auto& metadata = dec->metadata.m; + memset(header, 0, sizeof(*header)); if (metadata.have_animation) { header->duration = dec->frame_header->animation_frame.duration; if (metadata.animation.have_timecodes) { header->timecode = dec->frame_header->animation_frame.timecode; } @@ -2880,5 +2910,17 @@ if (size) { *size = dec->box_size; } return JXL_DEC_SUCCESS; } + +JxlDecoderStatus JxlDecoderSetProgressiveDetail(JxlDecoder* dec, + JxlProgressiveDetail detail) { + if (detail != kDC && detail != kPasses) { + return JXL_API_ERROR( + "Values other than kDC (%d) and kPasses (%d), like %d are not " + "implemented.", + kDC, kPasses, detail); + } + dec->prog_detail = detail; + return JXL_DEC_SUCCESS; +} diff --git a/third_party/jpeg-xl/lib/jxl/decode_test.cc b/third_party/jpeg-xl/lib/jxl/decode_test.cc --- a/third_party/jpeg-xl/lib/jxl/decode_test.cc +++ b/third_party/jpeg-xl/lib/jxl/decode_test.cc @@ -15,10 +15,11 @@ #include "gtest/gtest.h" #include "jxl/decode_cxx.h" #include "jxl/resizable_parallel_runner_cxx.h" #include "jxl/thread_parallel_runner_cxx.h" +#include "jxl/types.h" #include "lib/extras/enc/jpg.h" #include "lib/jxl/base/byte_order.h" #include "lib/jxl/base/file_io.h" #include "lib/jxl/base/span.h" #include "lib/jxl/base/status.h" @@ -201,11 +202,11 @@ /*alpha_is_premultiplied=*/false, /*bits_per_sample=*/16, JXL_BIG_ENDIAN, /*flipped_y=*/false, &pool, &io.Main(), /*float_in=*/false, /*align=*/0)); jxl::PaddedBytes jpeg_data; if (jpeg_codestream != nullptr) { #if JPEGXL_ENABLE_JPEG - jxl::PaddedBytes jpeg_bytes; + std::vector jpeg_bytes; EXPECT_TRUE(EncodeImageJPG(&io, jxl::extras::JpegEncoder::kLibJpeg, /*quality=*/70, jxl::YCbCrChromaSubsampling(), &pool, &jpeg_bytes)); jpeg_codestream->append(jpeg_bytes.data(), jpeg_bytes.data() + jpeg_bytes.size()); @@ -3391,126 +3392,163 @@ JxlDecoderDestroy(dec); } TEST(DecodeTest, ProgressiveEventTest) { + std::set progressive_details = {kDC, kPasses}; for (int single_group = 0; single_group <= 1; ++single_group) { for (int lossless = 0; lossless <= 1; ++lossless) { for (uint32_t num_channels = 3; num_channels < 4; ++num_channels) { - // Only few combinations are expected to support outputting the DC. The - // test can be updated if more cases are expected to support it. - bool expect_dc = !single_group && (num_channels & 1) && !lossless; - size_t xsize, ysize; - if (single_group) { - // An image smaller than 256x256 ensures it contains only 1 group. - xsize = 99; - ysize = 100; - } else { - xsize = 257; - ysize = 280; - } - std::vector pixels = - jxl::test::GetSomeTestImage(xsize, ysize, num_channels, 0); - jxl::CompressParams cparams; - if (lossless) { - cparams.SetLossless(); - } else { - cparams.butteraugli_distance = 0.5f; - } - - jxl::PaddedBytes data = jxl::CreateTestJXLCodestream( - jxl::Span(pixels.data(), pixels.size()), xsize, - ysize, num_channels, cparams, kCSBF_None, JXL_ORIENT_IDENTITY, - false, false); - JxlPixelFormat format = {num_channels, JXL_TYPE_UINT16, JXL_BIG_ENDIAN, - 0}; - - std::vector pixels2; - pixels2.resize(pixels.size()); - std::vector dc; - dc.resize(pixels.size()); - - JxlDecoder* dec = JxlDecoderCreate(nullptr); - - EXPECT_EQ(JXL_DEC_SUCCESS, - JxlDecoderSubscribeEvents( - dec, JXL_DEC_BASIC_INFO | JXL_DEC_FRAME | - JXL_DEC_FULL_IMAGE | JXL_DEC_FRAME_PROGRESSION)); - - EXPECT_EQ(JXL_DEC_SUCCESS, - JxlDecoderSetInput(dec, data.data(), data.size())); - - EXPECT_EQ(JXL_DEC_BASIC_INFO, JxlDecoderProcessInput(dec)); - JxlBasicInfo info; - EXPECT_EQ(JXL_DEC_SUCCESS, JxlDecoderGetBasicInfo(dec, &info)); - EXPECT_EQ(info.xsize, xsize); - EXPECT_EQ(info.ysize, ysize); - - EXPECT_EQ(JXL_DEC_FRAME, JxlDecoderProcessInput(dec)); - - size_t buffer_size; - EXPECT_EQ(JXL_DEC_SUCCESS, - JxlDecoderImageOutBufferSize(dec, &format, &buffer_size)); - EXPECT_EQ(pixels2.size(), buffer_size); - EXPECT_EQ(JXL_DEC_SUCCESS, - JxlDecoderSetImageOutBuffer(dec, &format, pixels2.data(), - pixels2.size())); - - if (expect_dc) { - EXPECT_EQ(JXL_DEC_FRAME_PROGRESSION, JxlDecoderProcessInput(dec)); - EXPECT_EQ(JXL_DEC_SUCCESS, JxlDecoderFlushImage(dec)); - dc = pixels2; - } - - EXPECT_EQ(JXL_DEC_FULL_IMAGE, JxlDecoderProcessInput(dec)); - EXPECT_EQ(JXL_DEC_SUCCESS, JxlDecoderProcessInput(dec)); - - jxl::Image3F imagef_pixels(xsize, ysize); - jxl::Image3F imagef_pixels2(xsize, ysize); - jxl::Image3F imagef_dc(xsize, ysize); - for (size_t c = 0; c < 3; c++) { - for (size_t y = 0; y < ysize; y++) { - float* row_pixels = imagef_pixels.PlaneRow(c, y); - float* row_pixels2 = imagef_pixels2.PlaneRow(c, y); - float* row_dc = imagef_dc.PlaneRow(c, y); - for (size_t x = 0; x < xsize; x++) { - size_t pixel_index = (y * xsize + x) * num_channels * 2 + c * 2; - row_pixels[x] = pixels[pixel_index] / 255.0f; - row_pixels2[x] = pixels2[pixel_index] / 255.0f; - row_dc[x] = dc[pixel_index] / 255.0f; + for (auto prog_detail : progressive_details) { + // Only few combinations are expected to support outputting + // intermediate flushes for complete DC and complete passes. + // The test can be updated if more cases are expected to support it. + bool expect_flush = (num_channels & 1) && !lossless; + size_t xsize, ysize; + if (single_group) { + // An image smaller than 256x256 ensures it contains only 1 group. + xsize = 99; + ysize = 100; + } else { + xsize = 277; + ysize = 280; + } + std::vector pixels = + jxl::test::GetSomeTestImage(xsize, ysize, num_channels, 0); + jxl::CompressParams cparams; + if (lossless) { + cparams.SetLossless(); + } else { + cparams.butteraugli_distance = 0.5f; + } + cparams.qprogressive_mode = true; + jxl::PaddedBytes data = jxl::CreateTestJXLCodestream( + jxl::Span(pixels.data(), pixels.size()), xsize, + ysize, num_channels, cparams, kCSBF_None, JXL_ORIENT_IDENTITY, + false, false); + JxlPixelFormat format = {num_channels, JXL_TYPE_UINT16, + JXL_BIG_ENDIAN, 0}; + + std::vector pixels2; + pixels2.resize(pixels.size()); + std::vector dc; + dc.resize(pixels.size()); + std::vector pass; + pass.resize(pixels.size()); + + JxlDecoder* dec = JxlDecoderCreate(nullptr); + + EXPECT_EQ( + JXL_DEC_SUCCESS, + JxlDecoderSubscribeEvents( + dec, JXL_DEC_BASIC_INFO | JXL_DEC_FRAME | JXL_DEC_FULL_IMAGE | + JXL_DEC_FRAME_PROGRESSION)); + EXPECT_EQ(JXL_DEC_ERROR, + JxlDecoderSetProgressiveDetail(dec, kFrames)); + EXPECT_EQ(JXL_DEC_ERROR, + JxlDecoderSetProgressiveDetail(dec, kLastPasses)); + EXPECT_EQ(JXL_DEC_ERROR, + JxlDecoderSetProgressiveDetail(dec, kDCProgressive)); + EXPECT_EQ(JXL_DEC_ERROR, + JxlDecoderSetProgressiveDetail(dec, kDCGroups)); + EXPECT_EQ(JXL_DEC_ERROR, + JxlDecoderSetProgressiveDetail(dec, kGroups)); + EXPECT_EQ(JXL_DEC_SUCCESS, + JxlDecoderSetProgressiveDetail(dec, prog_detail)); + EXPECT_EQ(JXL_DEC_SUCCESS, + JxlDecoderSetInput(dec, data.data(), data.size())); + + EXPECT_EQ(JXL_DEC_BASIC_INFO, JxlDecoderProcessInput(dec)); + JxlBasicInfo info; + EXPECT_EQ(JXL_DEC_SUCCESS, JxlDecoderGetBasicInfo(dec, &info)); + EXPECT_EQ(info.xsize, xsize); + EXPECT_EQ(info.ysize, ysize); + + EXPECT_EQ(JXL_DEC_FRAME, JxlDecoderProcessInput(dec)); + + size_t buffer_size; + EXPECT_EQ(JXL_DEC_SUCCESS, + JxlDecoderImageOutBufferSize(dec, &format, &buffer_size)); + EXPECT_EQ(pixels2.size(), buffer_size); + EXPECT_EQ(JXL_DEC_SUCCESS, + JxlDecoderSetImageOutBuffer(dec, &format, pixels2.data(), + pixels2.size())); + + if (expect_flush) { + EXPECT_EQ(JXL_DEC_FRAME_PROGRESSION, JxlDecoderProcessInput(dec)); + EXPECT_EQ(JXL_DEC_SUCCESS, JxlDecoderFlushImage(dec)); + dc = pixels2; + if (prog_detail == kPasses) { + EXPECT_EQ(JXL_DEC_FRAME_PROGRESSION, JxlDecoderProcessInput(dec)); + EXPECT_EQ(JXL_DEC_SUCCESS, JxlDecoderFlushImage(dec)); + pass = pixels2; } } - } - - jxl::CodecInOut io_pixels; - io_pixels.SetFromImage(std::move(imagef_pixels), + + EXPECT_EQ(JXL_DEC_FULL_IMAGE, JxlDecoderProcessInput(dec)); + EXPECT_EQ(JXL_DEC_SUCCESS, JxlDecoderProcessInput(dec)); + + jxl::Image3F imagef_pixels(xsize, ysize); + jxl::Image3F imagef_pixels2(xsize, ysize); + jxl::Image3F imagef_dc(xsize, ysize); + jxl::Image3F imagef_pass(xsize, ysize); + for (size_t c = 0; c < 3; c++) { + for (size_t y = 0; y < ysize; y++) { + float* row_pixels = imagef_pixels.PlaneRow(c, y); + float* row_pixels2 = imagef_pixels2.PlaneRow(c, y); + float* row_dc = imagef_dc.PlaneRow(c, y); + float* row_pass = imagef_pass.PlaneRow(c, y); + for (size_t x = 0; x < xsize; x++) { + size_t pixel_index = (y * xsize + x) * num_channels * 2 + c * 2; + row_pixels[x] = pixels[pixel_index] / 255.0f; + row_pixels2[x] = pixels2[pixel_index] / 255.0f; + row_dc[x] = dc[pixel_index] / 255.0f; + row_pass[x] = pass[pixel_index] / 255.0f; + } + } + } + + jxl::CodecInOut io_pixels; + io_pixels.SetFromImage(std::move(imagef_pixels), + jxl::ColorEncoding::SRGB(false)); + jxl::CodecInOut io_pixels2; + io_pixels2.SetFromImage(std::move(imagef_pixels2), + jxl::ColorEncoding::SRGB(false)); + jxl::CodecInOut io_dc; + io_dc.SetFromImage(std::move(imagef_dc), + jxl::ColorEncoding::SRGB(false)); + jxl::CodecInOut io_pass; + io_pass.SetFromImage(std::move(imagef_pass), jxl::ColorEncoding::SRGB(false)); - jxl::CodecInOut io_pixels2; - io_pixels2.SetFromImage(std::move(imagef_pixels2), - jxl::ColorEncoding::SRGB(false)); - jxl::CodecInOut io_dc; - io_dc.SetFromImage(std::move(imagef_dc), - jxl::ColorEncoding::SRGB(false)); - - jxl::ButteraugliParams ba; - float distance_full = - ButteraugliDistance(io_pixels, io_pixels2, ba, jxl::GetJxlCms(), - /*distmap=*/nullptr, nullptr); - - if (expect_dc) { - float distance_dc = - ButteraugliDistance(io_pixels, io_dc, ba, jxl::GetJxlCms(), + + jxl::ButteraugliParams ba; + float distance_full = + ButteraugliDistance(io_pixels, io_pixels2, ba, jxl::GetJxlCms(), /*distmap=*/nullptr, nullptr); EXPECT_LT(distance_full, 2.0f); - EXPECT_LT(distance_dc, 30.0f); - // Verify that the returned DC image is actually DC, by checking that - // it has worse butteraugli score than the full image - EXPECT_LT(distance_full + 1.0f, distance_dc); - } else { - EXPECT_LT(distance_full, 2.0f); + + if (expect_flush) { + float distance_dc = + ButteraugliDistance(io_pixels, io_dc, ba, jxl::GetJxlCms(), + /*distmap=*/nullptr, nullptr); + EXPECT_LT(distance_dc, 30.0f); + // Verify that the returned DC image is actually DC, by checking + // that it has much worse butteraugli score than the full image. + EXPECT_LT(distance_full * 5, distance_dc); + if (prog_detail == kPasses) { + float distance_pass = + ButteraugliDistance(io_pixels, io_pass, ba, jxl::GetJxlCms(), + /*distmap=*/nullptr, nullptr); + EXPECT_LT(distance_pass, 3.0f); + // Verify that the returned pass image is actually not the full + // image, by checking that it has a bit worse butteraugli score + // than the full image. + EXPECT_LT(distance_full * 1.1, distance_pass); + } + } + JxlDecoderDestroy(dec); } - JxlDecoderDestroy(dec); } } } } @@ -3562,12 +3600,11 @@ VerifyJPEGReconstruction(compressed, jpeg_codestream); } #endif // JPEGXL_ENABLE_JPEG TEST(DecodeTest, JXL_TRANSCODE_JPEG_TEST(JPEGReconstructionTest)) { - const std::string jpeg_path = - "third_party/imagecompression.info/flower_foveon.png.im_q85_420.jpg"; + const std::string jpeg_path = "jxl/flower/flower.png.im_q85_420.jpg"; const jxl::PaddedBytes orig = jxl::ReadTestData(jpeg_path); jxl::CodecInOut orig_io; ASSERT_TRUE( jxl::jpeg::DecodeImageJPG(jxl::Span(orig), &orig_io)); orig_io.metadata.m.xyb_encoded = false; diff --git a/third_party/jpeg-xl/lib/jxl/enc_image_bundle.cc b/third_party/jpeg-xl/lib/jxl/enc_image_bundle.cc --- a/third_party/jpeg-xl/lib/jxl/enc_image_bundle.cc +++ b/third_party/jpeg-xl/lib/jxl/enc_image_bundle.cc @@ -3,10 +3,11 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. #include "lib/jxl/enc_image_bundle.h" +#include #include #include #include "lib/jxl/alpha.h" #include "lib/jxl/base/byte_order.h" diff --git a/third_party/jpeg-xl/lib/jxl/enc_modular.cc b/third_party/jpeg-xl/lib/jxl/enc_modular.cc --- a/third_party/jpeg-xl/lib/jxl/enc_modular.cc +++ b/third_party/jpeg-xl/lib/jxl/enc_modular.cc @@ -7,10 +7,11 @@ #include #include #include +#include #include #include #include #include @@ -673,10 +674,12 @@ !cparams.lossy_palette && (cparams.speed_tier <= SpeedTier::kThunder || (do_color && metadata.bit_depth.bits_per_sample > 8))) { // single channel palette (like FLIF's ChannelCompact) size_t nb_channels = gi.channel.size() - gi.nb_meta_channels; + int orig_bitdepth = max_bitdepth; + max_bitdepth = 0; for (size_t i = 0; i < nb_channels; i++) { int min, max; compute_minmax(gi.channel[gi.nb_meta_channels + i], &min, &max); int64_t colors = max - min + 1; JXL_DEBUG_V(10, "Channel %" PRIuS ": range=%i..%i", i, min, max); @@ -692,11 +695,15 @@ (int)(cparams.channel_colors_pre_transform_percent / 100. * colors)); if (do_transform(gi, maybe_palette_1, weighted::Header(), pool)) { // effective bit depth is lower, adjust quantization accordingly compute_minmax(gi.channel[gi.nb_meta_channels + i], &min, &max); if (max < maxval) maxval = max; - } + int ch_bitdepth = + (max > 0 ? CeilLog2Nonzero(static_cast(max)) : 0); + if (ch_bitdepth > max_bitdepth) max_bitdepth = ch_bitdepth; + } else + max_bitdepth = orig_bitdepth; } } // Global palette if ((cparams.palette_colors != 0 || cparams.lossy_palette) && @@ -736,25 +743,24 @@ cparams.options.zero_tokens); } } // don't do an RCT if we're short on bits - if (cparams.color_transform == ColorTransform::kNone && do_color && !fp && + if (cparams.color_transform == ColorTransform::kNone && do_color && gi.channel.size() - gi.nb_meta_channels >= 3 && max_bitdepth + 1 < level_max_bitdepth) { - if (cparams.colorspace == 1 || - (cparams.colorspace < 0 && - (!cparams.IsLossless() || cparams.speed_tier > SpeedTier::kHare))) { + if (cparams.colorspace < 0 && + (!cparams.IsLossless() || cparams.speed_tier > SpeedTier::kHare)) { Transform ycocg{TransformId::kRCT}; ycocg.rct_type = 6; ycocg.begin_c = gi.nb_meta_channels; do_transform(gi, ycocg, weighted::Header(), pool); max_bitdepth++; - } else if (cparams.colorspace >= 2) { + } else if (cparams.colorspace > 0) { Transform sg(TransformId::kRCT); sg.begin_c = gi.nb_meta_channels; - sg.rct_type = cparams.colorspace - 2; + sg.rct_type = cparams.colorspace; do_transform(gi, sg, weighted::Header(), pool); max_bitdepth++; } } diff --git a/third_party/jpeg-xl/lib/jxl/enc_patch_dictionary.cc b/third_party/jpeg-xl/lib/jxl/enc_patch_dictionary.cc --- a/third_party/jpeg-xl/lib/jxl/enc_patch_dictionary.cc +++ b/third_party/jpeg-xl/lib/jxl/enc_patch_dictionary.cc @@ -8,10 +8,11 @@ #include #include #include #include +#include #include #include #include #include diff --git a/third_party/jpeg-xl/lib/jxl/enc_photon_noise_test.cc b/third_party/jpeg-xl/lib/jxl/enc_photon_noise_test.cc --- a/third_party/jpeg-xl/lib/jxl/enc_photon_noise_test.cc +++ b/third_party/jpeg-xl/lib/jxl/enc_photon_noise_test.cc @@ -3,11 +3,11 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. #include "lib/jxl/enc_photon_noise.h" -#include "gmock/gmock.h" +#include "lib/jxl/test_utils.h" namespace jxl { namespace { using ::testing::FloatNear; diff --git a/third_party/jpeg-xl/lib/jxl/encode.cc b/third_party/jpeg-xl/lib/jxl/encode.cc --- a/third_party/jpeg-xl/lib/jxl/encode.cc +++ b/third_party/jpeg-xl/lib/jxl/encode.cc @@ -25,18 +25,26 @@ #include "lib/jxl/sanitizers.h" // Debug-printing failure macro similar to JXL_FAILURE, but for the status code // JXL_ENC_ERROR #ifdef JXL_CRASH_ON_ERROR -#define JXL_API_ERROR(format, ...) \ +#define JXL_API_ERROR(enc, error_code, format, ...) \ + (enc->error = error_code, \ + ::jxl::Debug(("%s:%d: " format "\n"), __FILE__, __LINE__, ##__VA_ARGS__), \ + ::jxl::Abort(), JXL_ENC_ERROR) +#define JXL_API_ERROR_NOSET(format, ...) \ (::jxl::Debug(("%s:%d: " format "\n"), __FILE__, __LINE__, ##__VA_ARGS__), \ ::jxl::Abort(), JXL_ENC_ERROR) #else // JXL_CRASH_ON_ERROR -#define JXL_API_ERROR(format, ...) \ - (((JXL_DEBUG_ON_ERROR) && \ +#define JXL_API_ERROR(enc, error_code, format, ...) \ + (enc->error = error_code, \ + ((JXL_DEBUG_ON_ERROR) && \ ::jxl::Debug(("%s:%d: " format "\n"), __FILE__, __LINE__, ##__VA_ARGS__)), \ JXL_ENC_ERROR) +#define JXL_API_ERROR_NOSET(format, ...) \ + (::jxl::Debug(("%s:%d: " format "\n"), __FILE__, __LINE__, ##__VA_ARGS__), \ + JXL_ENC_ERROR) #endif // JXL_CRASH_ON_ERROR namespace jxl {} // namespace jxl uint32_t JxlEncoderVersion(void) { @@ -78,11 +86,11 @@ JxlEncoderStatus BrotliCompress(int quality, const uint8_t* in, size_t in_size, jxl::PaddedBytes* out) { std::unique_ptr enc(BrotliEncoderCreateInstance(nullptr, nullptr, nullptr), BrotliEncoderDestroyInstance); - if (!enc) return JXL_API_ERROR("BrotliEncoderCreateInstance failed"); + if (!enc) return JXL_API_ERROR_NOSET("BrotliEncoderCreateInstance failed"); BrotliEncoderSetParameter(enc.get(), BROTLI_PARAM_QUALITY, quality); BrotliEncoderSetParameter(enc.get(), BROTLI_PARAM_SIZE_HINT, in_size); constexpr size_t kBufferSize = 128 * 1024; @@ -98,11 +106,11 @@ uint8_t* next_out = temp_buffer.data(); jxl::msan::MemoryIsInitialized(next_in, avail_in); if (!BrotliEncoderCompressStream(enc.get(), BROTLI_OPERATION_FINISH, &avail_in, &next_in, &avail_out, &next_out, &total_out)) { - return JXL_API_ERROR("Brotli compression failed"); + return JXL_API_ERROR_NOSET("Brotli compression failed"); } size_t out_size = next_out - temp_buffer.data(); jxl::msan::UnpoisonMemory(next_out - out_size, out_size); out->resize(out->size() + out_size); memcpy(out->data() + out->size() - out_size, temp_buffer.data(), out_size); @@ -191,16 +199,16 @@ uint32_t exponent_bits_per_sample) { if (!exponent_bits_per_sample) { // The spec allows up to 31 for bits_per_sample here, but // the code does not (yet) support it. if (!(bits_per_sample > 0 && bits_per_sample <= 24)) { - return JXL_API_ERROR("Invalid value for bits_per_sample"); + return JXL_API_ERROR_NOSET("Invalid value for bits_per_sample"); } } else if ((exponent_bits_per_sample > 8) || (bits_per_sample > 24 + exponent_bits_per_sample) || (bits_per_sample < 3 + exponent_bits_per_sample)) { - return JXL_API_ERROR("Invalid float description"); + return JXL_API_ERROR_NOSET("Invalid float description"); } return JXL_ENC_SUCCESS; } } // namespace @@ -225,30 +233,33 @@ if (codestream_level == 5 && required_level != 5) { // If the required level is 10, return error rather than automatically // setting the level to 10, to avoid inadvertently creating a level 10 // JXL file while intending to target a level 5 decoder. return JXL_API_ERROR( - "%s", + this, JXL_ENC_ERR_API_USAGE, "%s", ("Codestream level verification for level 5 failed: " + level_message) .c_str()); } if (codestream_level == 10 && required_level == -1) { return JXL_API_ERROR( - "%s", ("Codestream level verification for level 10 failed: " + - level_message) - .c_str()); + this, JXL_ENC_ERR_API_USAGE, "%s", + ("Codestream level verification for level 10 failed: " + + level_message) + .c_str()); } jxl::BitWriter writer; if (!WriteHeaders(&metadata, &writer, nullptr)) { - return JXL_API_ERROR("Failed to write codestream header"); + return JXL_API_ERROR(this, JXL_ENC_ERR_GENERIC, + "Failed to write codestream header"); } // Only send ICC (at least several hundred bytes) if fields aren't enough. if (metadata.m.color_encoding.WantICC()) { if (!jxl::WriteICC(metadata.m.color_encoding.ICC(), &writer, jxl::kLayerHeader, nullptr)) { - return JXL_API_ERROR("Failed to write ICC profile"); + return JXL_API_ERROR(this, JXL_ENC_ERR_GENERIC, + "Failed to write ICC profile"); } } // TODO(lode): preview should be added here if a preview image is added writer.ZeroPadToByte(); @@ -309,11 +320,12 @@ std::move(input.frame); input_queue.erase(input_queue.begin()); num_queued_frames--; for (unsigned idx = 0; idx < input_frame->ec_initialized.size(); idx++) { if (!input_frame->ec_initialized[idx]) { - return JXL_API_ERROR("Extra channel %u is not initialized", idx); + return JXL_API_ERROR(this, JXL_ENC_ERR_API_USAGE, + "Extra channel %u is not initialized", idx); } } // TODO(zond): If the input queue is empty and the frames_closed is true, // then mark this frame as the last. @@ -396,11 +408,11 @@ JXL_ASSERT(writer.BitsWritten() == 0); if (!jxl::EncodeFrame(input_frame->option_values.cparams, frame_info, &metadata, input_frame->frame, &enc_state, cms, thread_pool.get(), &writer, /*aux_out=*/nullptr)) { - return JXL_API_ERROR("Failed to encode frame"); + return JXL_API_ERROR(this, JXL_ENC_ERR_GENERIC, "Failed to encode frame"); } codestream_bytes_written_beginning_of_frame = codestream_bytes_written_end_of_frame; codestream_bytes_written_end_of_frame += jxl::DivCeil(writer.BitsWritten(), 8); @@ -441,11 +453,12 @@ } if (JXL_ENC_SUCCESS != BrotliCompress((brotli_effort >= 0 ? brotli_effort : 4), box->contents.data(), box->contents.size(), &compressed)) { - return JXL_API_ERROR("Brotli compression for brob box failed"); + return JXL_API_ERROR(this, JXL_ENC_ERR_GENERIC, + "Brotli compression for brob box failed"); } jxl::AppendBoxHeader(jxl::MakeBoxType("brob"), compressed.size(), false, &output_byte_queue); output_byte_queue.insert(output_byte_queue.end(), compressed.data(), compressed.data() + compressed.size()); @@ -460,17 +473,32 @@ return JXL_ENC_SUCCESS; } JxlEncoderStatus JxlEncoderSetColorEncoding(JxlEncoder* enc, const JxlColorEncoding* color) { + if (!enc->basic_info_set) { + return JXL_API_ERROR(enc, JXL_ENC_ERR_API_USAGE, "Basic info not yet set"); + } if (enc->color_encoding_set) { - // Already set - return JXL_ENC_ERROR; + return JXL_API_ERROR(enc, JXL_ENC_ERR_API_USAGE, + "Color encoding is already set"); } if (!jxl::ConvertExternalToInternalColorEncoding( *color, &enc->metadata.m.color_encoding)) { - return JXL_ENC_ERROR; + return JXL_API_ERROR(enc, JXL_ENC_ERR_GENERIC, "Error in color conversion"); + } + if (enc->metadata.m.color_encoding.GetColorSpace() == + jxl::ColorSpace::kGray) { + if (enc->basic_info.num_color_channels != 1) + return JXL_API_ERROR( + enc, JXL_ENC_ERR_API_USAGE, + "Cannot use grayscale color encoding with num_color_channels != 1"); + } else { + if (enc->basic_info.num_color_channels != 3) + return JXL_API_ERROR( + enc, JXL_ENC_ERR_API_USAGE, + "Cannot use RGB color encoding with num_color_channels != 3"); } enc->color_encoding_set = true; if (!enc->intensity_target_set) { jxl::SetIntensityTarget(&enc->metadata.m); } @@ -478,18 +506,36 @@ } JxlEncoderStatus JxlEncoderSetICCProfile(JxlEncoder* enc, const uint8_t* icc_profile, size_t size) { + if (!enc->basic_info_set) { + return JXL_API_ERROR(enc, JXL_ENC_ERR_API_USAGE, "Basic info not yet set"); + } if (enc->color_encoding_set) { - // Already set - return JXL_ENC_ERROR; + return JXL_API_ERROR(enc, JXL_ENC_ERR_API_USAGE, + "ICC profile is already set"); } jxl::PaddedBytes icc; icc.assign(icc_profile, icc_profile + size); if (!enc->metadata.m.color_encoding.SetICC(std::move(icc))) { - return JXL_ENC_ERROR; + return JXL_API_ERROR(enc, JXL_ENC_ERR_BAD_INPUT, + "ICC profile could not be set"); + } + if (enc->metadata.m.color_encoding.GetColorSpace() == + jxl::ColorSpace::kGray) { + if (enc->basic_info.num_color_channels != 1) + return JXL_API_ERROR( + enc, JXL_ENC_ERR_BAD_INPUT, + "Cannot use grayscale ICC profile with num_color_channels != 1"); + } else { + if (enc->basic_info.num_color_channels != 3) + return JXL_API_ERROR( + enc, JXL_ENC_ERR_BAD_INPUT, + "Cannot use RGB ICC profile with num_color_channels != 3"); + // TODO(jon): also check that a kBlack extra channel is provided in the CMYK + // case } enc->color_encoding_set = true; if (!enc->intensity_target_set) { jxl::SetIntensityTarget(&enc->metadata.m); } @@ -564,15 +610,15 @@ } JxlEncoderStatus JxlEncoderSetBasicInfo(JxlEncoder* enc, const JxlBasicInfo* info) { if (!enc->metadata.size.Set(info->xsize, info->ysize)) { - return JXL_ENC_ERROR; + return JXL_API_ERROR(enc, JXL_ENC_ERR_API_USAGE, "Invalid dimensions"); } if (JXL_ENC_SUCCESS != CheckValidBitdepth(info->bits_per_sample, info->exponent_bits_per_sample)) { - return JXL_ENC_ERROR; + return JXL_API_ERROR(enc, JXL_ENC_ERR_API_USAGE, "Invalid bit depth"); } enc->metadata.m.bit_depth.bits_per_sample = info->bits_per_sample; enc->metadata.m.bit_depth.exponent_bits_per_sample = info->exponent_bits_per_sample; enc->metadata.m.bit_depth.floating_point_sample = @@ -585,10 +631,11 @@ // RGBA with no other extra channels, has exactly num_extra_channels == 1 enc->metadata.m.num_extra_channels = info->num_extra_channels; enc->metadata.m.extra_channel_info.resize(enc->metadata.m.num_extra_channels); if (info->num_extra_channels == 0 && info->alpha_bits) { return JXL_API_ERROR( + enc, JXL_ENC_ERR_API_USAGE, "when alpha_bits is non-zero, the number of channels must be at least " "1"); } // If the user provides non-zero alpha_bits, we make the channel info at index // zero the appropriate alpha channel. @@ -596,19 +643,25 @@ JxlExtraChannelInfo channel_info; JxlEncoderInitExtraChannelInfo(JXL_CHANNEL_ALPHA, &channel_info); channel_info.bits_per_sample = info->alpha_bits; channel_info.exponent_bits_per_sample = info->alpha_exponent_bits; if (JxlEncoderSetExtraChannelInfo(enc, 0, &channel_info)) { - return JXL_ENC_ERROR; + return JXL_API_ERROR(enc, JXL_ENC_ERR_API_USAGE, + "Problem setting extra channel info for alpha"); } } enc->metadata.m.xyb_encoded = !info->uses_original_profile; if (info->orientation > 0 && info->orientation <= 8) { enc->metadata.m.orientation = info->orientation; } else { - return JXL_API_ERROR("Invalid value for orientation field"); + return JXL_API_ERROR(enc, JXL_ENC_ERR_API_USAGE, + "Invalid value for orientation field"); + } + if (info->num_color_channels != 1 && info->num_color_channels != 3) { + return JXL_API_ERROR(enc, JXL_ENC_ERR_API_USAGE, + "Invalid number of color channels"); } if (info->intensity_target != 0) { enc->metadata.m.SetIntensityTarget(info->intensity_target); enc->intensity_target_set = true; } else if (enc->color_encoding_set || enc->metadata.m.xyb_encoded) { @@ -620,34 +673,38 @@ } enc->metadata.m.tone_mapping.min_nits = info->min_nits; enc->metadata.m.tone_mapping.relative_to_max_display = info->relative_to_max_display; enc->metadata.m.tone_mapping.linear_below = info->linear_below; + enc->basic_info = *info; enc->basic_info_set = true; enc->metadata.m.have_animation = info->have_animation; if (info->have_animation) { if (info->animation.tps_denominator < 1) { return JXL_API_ERROR( + enc, JXL_ENC_ERR_API_USAGE, "If animation is used, tps_denominator must be >= 1"); } if (info->animation.tps_numerator < 1) { - return JXL_API_ERROR("If animation is used, tps_numerator must be >= 1"); + return JXL_API_ERROR(enc, JXL_ENC_ERR_API_USAGE, + "If animation is used, tps_numerator must be >= 1"); } enc->metadata.m.animation.tps_numerator = info->animation.tps_numerator; enc->metadata.m.animation.tps_denominator = info->animation.tps_denominator; enc->metadata.m.animation.num_loops = info->animation.num_loops; enc->metadata.m.animation.have_timecodes = info->animation.have_timecodes; } std::string level_message; int required_level = VerifyLevelSettings(enc, &level_message); if (required_level == -1 || static_cast(enc->codestream_level) < required_level) { - return JXL_API_ERROR("%s", ("Codestream level verification for level " + - std::to_string(enc->codestream_level) + - " failed: " + level_message) - .c_str()); + return JXL_API_ERROR( + enc, JXL_ENC_ERR_API_USAGE, "%s", + ("Codestream level verification for level " + + std::to_string(enc->codestream_level) + " failed: " + level_message) + .c_str()); } return JXL_ENC_SUCCESS; } void JxlEncoderInitExtraChannelInfo(JxlExtraChannelType type, @@ -666,15 +723,16 @@ } JXL_EXPORT JxlEncoderStatus JxlEncoderSetExtraChannelInfo( JxlEncoder* enc, size_t index, const JxlExtraChannelInfo* info) { if (index >= enc->metadata.m.num_extra_channels) { - return JXL_API_ERROR("Invalid value for the index of extra channel"); + return JXL_API_ERROR(enc, JXL_ENC_ERR_API_USAGE, + "Invalid value for the index of extra channel"); } if (JXL_ENC_SUCCESS != CheckValidBitdepth(info->bits_per_sample, info->exponent_bits_per_sample)) { - return JXL_ENC_ERROR; + return JXL_API_ERROR(enc, JXL_ENC_ERR_API_USAGE, "Invalid bit depth"); } jxl::ExtraChannelInfo& channel = enc->metadata.m.extra_channel_info[index]; channel.type = static_cast(info->type); channel.bit_depth.bits_per_sample = info->bits_per_sample; @@ -692,24 +750,26 @@ channel.spot_color[3] = info->spot_color[3]; std::string level_message; int required_level = VerifyLevelSettings(enc, &level_message); if (required_level == -1 || static_cast(enc->codestream_level) < required_level) { - return JXL_API_ERROR("%s", ("Codestream level verification for level " + - std::to_string(enc->codestream_level) + - " failed: " + level_message) - .c_str()); + return JXL_API_ERROR( + enc, JXL_ENC_ERR_API_USAGE, "%s", + ("Codestream level verification for level " + + std::to_string(enc->codestream_level) + " failed: " + level_message) + .c_str()); } return JXL_ENC_SUCCESS; } JXL_EXPORT JxlEncoderStatus JxlEncoderSetExtraChannelName(JxlEncoder* enc, size_t index, const char* name, size_t size) { if (index >= enc->metadata.m.num_extra_channels) { - return JXL_API_ERROR("Invalid value for the index of extra channel"); + return JXL_API_ERROR(enc, JXL_ENC_ERR_API_USAGE, + "Invalid value for the index of extra channel"); } enc->metadata.m.extra_channel_info[index].name = std::string(name, name + size); return JXL_ENC_SUCCESS; } @@ -739,11 +799,12 @@ JxlEncoderStatus JxlEncoderSetFrameLossless( JxlEncoderFrameSettings* frame_settings, const JXL_BOOL lossless) { if (lossless && frame_settings->enc->basic_info_set && frame_settings->enc->metadata.m.xyb_encoded) { - return JXL_API_ERROR("Set use_original_profile=true for lossless encoding"); + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "Set use_original_profile=true for lossless encoding"); } frame_settings->values.lossless = lossless; return JXL_ENC_SUCCESS; } @@ -760,11 +821,12 @@ } JxlEncoderStatus JxlEncoderSetFrameDistance( JxlEncoderFrameSettings* frame_settings, float distance) { if (distance < 0.f || distance > 25.f) { - return JXL_ENC_ERROR; + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "Distance has to be in [0.0..25.0]"); } if (distance > 0.f && distance < 0.01f) { distance = 0.01f; } frame_settings->values.cparams.butteraugli_distance = distance; @@ -784,36 +846,64 @@ } JxlEncoderStatus JxlEncoderFrameSettingsSetOption( JxlEncoderFrameSettings* frame_settings, JxlEncoderFrameSettingId option, int32_t value) { + // check if value is -1, 0 or 1 for Override-type options + switch (option) { + case JXL_ENC_FRAME_SETTING_NOISE: + case JXL_ENC_FRAME_SETTING_DOTS: + case JXL_ENC_FRAME_SETTING_PATCHES: + case JXL_ENC_FRAME_SETTING_GABORISH: + case JXL_ENC_FRAME_SETTING_MODULAR: + case JXL_ENC_FRAME_SETTING_KEEP_INVISIBLE: + case JXL_ENC_FRAME_SETTING_GROUP_ORDER: + case JXL_ENC_FRAME_SETTING_RESPONSIVE: + case JXL_ENC_FRAME_SETTING_PROGRESSIVE_AC: + case JXL_ENC_FRAME_SETTING_QPROGRESSIVE_AC: + case JXL_ENC_FRAME_SETTING_LOSSY_PALETTE: + case JXL_ENC_FRAME_SETTING_JPEG_RECON_CFL: + if (value < -1 || value > 1) { + return JXL_API_ERROR( + frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "Option value has to be -1 (default), 0 (off) or 1 (on)"); + } + break; + default: + break; + } + switch (option) { case JXL_ENC_FRAME_SETTING_EFFORT: if (value < 1 || value > 9) { - return JXL_ENC_ERROR; + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_NOT_SUPPORTED, + "Encode effort has to be in [1..9]"); } frame_settings->values.cparams.speed_tier = static_cast(10 - value); return JXL_ENC_SUCCESS; case JXL_ENC_FRAME_SETTING_BROTLI_EFFORT: if (value < -1 || value > 11) { - return JXL_ENC_ERROR; + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "Brotli effort has to be in [-1..11]"); } // set cparams for brotli use in JPEG frames frame_settings->values.cparams.brotli_effort = value; // set enc option for brotli use in brob boxes frame_settings->enc->brotli_effort = value; return JXL_ENC_SUCCESS; case JXL_ENC_FRAME_SETTING_DECODING_SPEED: if (value < 0 || value > 4) { - return JXL_ENC_ERROR; + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_NOT_SUPPORTED, + "Decoding speed has to be in [0..4]"); } frame_settings->values.cparams.decoding_speed_tier = value; return JXL_ENC_SUCCESS; case JXL_ENC_FRAME_SETTING_RESAMPLING: if (value != -1 && value != 1 && value != 2 && value != 4 && value != 8) { - return JXL_ENC_ERROR; + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "Resampling factor has to be 1, 2, 4 or 8"); } frame_settings->values.cparams.resampling = value; return JXL_ENC_SUCCESS; case JXL_ENC_FRAME_SETTING_EXTRA_CHANNEL_RESAMPLING: // TOOD(lode): the jxl codestream allows choosing a different resampling @@ -821,11 +911,12 @@ // option to a JxlEncoderFrameSettings-option that can be set per extra // channel, so needs its own function rather than // JxlEncoderFrameSettingsSetOption due to the extra channel index // argument required. if (value != -1 && value != 1 && value != 2 && value != 4 && value != 8) { - return JXL_ENC_ERROR; + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "Resampling factor has to be 1, 2, 4 or 8"); } frame_settings->values.cparams.ec_resampling = value; return JXL_ENC_SUCCESS; case JXL_ENC_FRAME_SETTING_ALREADY_DOWNSAMPLED: if (value < 0 || value > 1) { @@ -839,124 +930,138 @@ // the noise synthesis parameters per frame for more fine grained control. frame_settings->values.cparams.photon_noise_iso = static_cast(value); return JXL_ENC_SUCCESS; case JXL_ENC_FRAME_SETTING_NOISE: - if (value < -1 || value > 1) return JXL_ENC_ERROR; frame_settings->values.cparams.noise = static_cast(value); return JXL_ENC_SUCCESS; case JXL_ENC_FRAME_SETTING_DOTS: - if (value < -1 || value > 1) return JXL_ENC_ERROR; frame_settings->values.cparams.dots = static_cast(value); return JXL_ENC_SUCCESS; case JXL_ENC_FRAME_SETTING_PATCHES: - if (value < -1 || value > 1) return JXL_ENC_ERROR; frame_settings->values.cparams.patches = static_cast(value); return JXL_ENC_SUCCESS; case JXL_ENC_FRAME_SETTING_EPF: - if (value < -1 || value > 3) return JXL_ENC_ERROR; + if (value < -1 || value > 3) { + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "EPF value has to be in [-1..3]"); + } frame_settings->values.cparams.epf = static_cast(value); return JXL_ENC_SUCCESS; case JXL_ENC_FRAME_SETTING_GABORISH: - if (value < -1 || value > 1) return JXL_ENC_ERROR; frame_settings->values.cparams.gaborish = static_cast(value); return JXL_ENC_SUCCESS; case JXL_ENC_FRAME_SETTING_MODULAR: - if (value == 1) { - frame_settings->values.cparams.modular_mode = true; - } else if (value == -1 || value == 0) { - frame_settings->values.cparams.modular_mode = false; - } else { - return JXL_ENC_ERROR; - } + frame_settings->values.cparams.modular_mode = (value == 1); return JXL_ENC_SUCCESS; case JXL_ENC_FRAME_SETTING_KEEP_INVISIBLE: - if (value < -1 || value > 1) return JXL_ENC_ERROR; frame_settings->values.cparams.keep_invisible = static_cast(value); return JXL_ENC_SUCCESS; case JXL_ENC_FRAME_SETTING_GROUP_ORDER: - if (value < -1 || value > 1) return JXL_ENC_ERROR; frame_settings->values.cparams.centerfirst = (value == 1); return JXL_ENC_SUCCESS; case JXL_ENC_FRAME_SETTING_GROUP_ORDER_CENTER_X: - if (value < -1) return JXL_ENC_ERROR; + if (value < -1) { + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "Center x coordinate has to be -1 or positive"); + } frame_settings->values.cparams.center_x = static_cast(value); return JXL_ENC_SUCCESS; case JXL_ENC_FRAME_SETTING_GROUP_ORDER_CENTER_Y: - if (value < -1) return JXL_ENC_ERROR; + if (value < -1) { + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "Center y coordinate has to be -1 or positive"); + } frame_settings->values.cparams.center_y = static_cast(value); return JXL_ENC_SUCCESS; case JXL_ENC_FRAME_SETTING_RESPONSIVE: - if (value < -1 || value > 1) return JXL_ENC_ERROR; frame_settings->values.cparams.responsive = value; return JXL_ENC_SUCCESS; case JXL_ENC_FRAME_SETTING_PROGRESSIVE_AC: - if (value < -1 || value > 1) return JXL_ENC_ERROR; frame_settings->values.cparams.progressive_mode = value; return JXL_ENC_SUCCESS; case JXL_ENC_FRAME_SETTING_QPROGRESSIVE_AC: - if (value < -1 || value > 1) return JXL_ENC_ERROR; frame_settings->values.cparams.qprogressive_mode = value; return JXL_ENC_SUCCESS; case JXL_ENC_FRAME_SETTING_PROGRESSIVE_DC: - if (value < -1 || value > 2) return JXL_ENC_ERROR; + if (value < -1 || value > 2) { + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "Progressive DC has to be in [-1..2]"); + } frame_settings->values.cparams.progressive_dc = value; return JXL_ENC_SUCCESS; case JXL_ENC_FRAME_SETTING_CHANNEL_COLORS_GLOBAL_PERCENT: - if (value < -1 || value > 100) return JXL_ENC_ERROR; + if (value < -1 || value > 100) { + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "Option value has to be in [-1..100]"); + } if (value == -1) { frame_settings->values.cparams.channel_colors_pre_transform_percent = 95.0f; } else { frame_settings->values.cparams.channel_colors_pre_transform_percent = static_cast(value); } return JXL_ENC_SUCCESS; case JXL_ENC_FRAME_SETTING_CHANNEL_COLORS_GROUP_PERCENT: - if (value < -1 || value > 100) return JXL_ENC_ERROR; + if (value < -1 || value > 100) { + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "Option value has to be in [-1..100]"); + } if (value == -1) { frame_settings->values.cparams.channel_colors_percent = 80.0f; } else { frame_settings->values.cparams.channel_colors_percent = static_cast(value); } return JXL_ENC_SUCCESS; case JXL_ENC_FRAME_SETTING_PALETTE_COLORS: - if (value < -1 || value > 70913) return JXL_ENC_ERROR; + if (value < -1 || value > 70913) { + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "Option value has to be in [-1..70913]"); + } if (value == -1) { frame_settings->values.cparams.palette_colors = 1 << 10; } else { frame_settings->values.cparams.palette_colors = value; } return JXL_ENC_SUCCESS; case JXL_ENC_FRAME_SETTING_LOSSY_PALETTE: - if (value < -1 || value > 1) return JXL_ENC_ERROR; // TODO(lode): the defaults of some palette settings depend on others. // See the logic in cjxl. Similar for other settings. This should be // handled in the encoder during JxlEncoderProcessOutput (or, // alternatively, in the cjxl binary like now) frame_settings->values.cparams.lossy_palette = (value == 1); return JXL_ENC_SUCCESS; case JXL_ENC_FRAME_SETTING_COLOR_TRANSFORM: - if (value < -1 || value > 2) return JXL_ENC_ERROR; + if (value < -1 || value > 2) { + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "Option value has to be in [-1..2]"); + } if (value == -1) { frame_settings->values.cparams.color_transform = jxl::ColorTransform::kXYB; } else { frame_settings->values.cparams.color_transform = static_cast(value); } return JXL_ENC_SUCCESS; case JXL_ENC_FRAME_SETTING_MODULAR_COLOR_SPACE: - if (value < -1 || value > 35) return JXL_ENC_ERROR; - frame_settings->values.cparams.colorspace = value + 2; + if (value < -1 || value > 48) { + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "Option value has to be in [-1..48]"); + } + frame_settings->values.cparams.colorspace = value; return JXL_ENC_SUCCESS; case JXL_ENC_FRAME_SETTING_MODULAR_GROUP_SIZE: - if (value < -1 || value > 3) return JXL_ENC_ERROR; + if (value < -1 || value > 3) { + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "Option value has to be in [-1..3]"); + } // TODO(lode): the default behavior of this parameter for cjxl is // to choose 1 or 2 depending on the situation. This behavior needs to be // implemented either in the C++ library by allowing to set this to -1, or // kept in cjxl and set it to 1 or 2 using this API. if (value == -1) { @@ -964,16 +1069,22 @@ } else { frame_settings->values.cparams.modular_group_size_shift = value; } return JXL_ENC_SUCCESS; case JXL_ENC_FRAME_SETTING_MODULAR_PREDICTOR: - if (value < -1 || value > 15) return JXL_ENC_ERROR; + if (value < -1 || value > 15) { + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "Option value has to be in [-1..15]"); + } frame_settings->values.cparams.options.predictor = static_cast(value); return JXL_ENC_SUCCESS; case JXL_ENC_FRAME_SETTING_MODULAR_MA_TREE_LEARNING_PERCENT: - if (value < -1) return JXL_ENC_ERROR; + if (value < -1) { + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "Option value has to be in [-1..100]"); + } // This value is called "iterations" or "nb_repeats" in cjxl, but is in // fact a fraction in range 0.0-1.0, with the defautl value 0.5. // Convert from integer percentage to floating point fraction here. if (value == -1) { // TODO(lode): for this and many other settings, avoid duplicating the @@ -987,27 +1098,31 @@ case JXL_ENC_FRAME_SETTING_MODULAR_NB_PREV_CHANNELS: // The max allowed value can in theory be higher. However, it depends on // the effort setting. 11 is the highest safe value that doesn't cause // tree_samples to be >= 64 in the encoder. The specification may allow // more than this. With more fine tuning higher values could be allowed. - if (value < -1 || value > 11) return JXL_ENC_ERROR; + // For N-channel images, the largest useful value is N-1. + if (value < -1 || value > 11) { + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "Option value has to be in [-1..11]"); + } if (value == -1) { frame_settings->values.cparams.options.max_properties = 0; } else { frame_settings->values.cparams.options.max_properties = value; } return JXL_ENC_SUCCESS; case JXL_ENC_FRAME_SETTING_JPEG_RECON_CFL: - if (value < -1 || value > 1) return JXL_ENC_ERROR; if (value == -1) { frame_settings->values.cparams.force_cfl_jpeg_recompression = true; } else { frame_settings->values.cparams.force_cfl_jpeg_recompression = value; } return JXL_ENC_SUCCESS; default: - return JXL_ENC_ERROR; + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_NOT_SUPPORTED, + "Unknown option"); } } JxlEncoder* JxlEncoderCreate(const JxlMemoryManager* memory_manager) { JxlMemoryManager local_memory_manager; @@ -1048,10 +1163,11 @@ enc->color_encoding_set = false; enc->intensity_target_set = false; enc->use_container = false; enc->use_boxes = false; enc->codestream_level = 5; + JxlEncoderInitBasicInfo(&enc->basic_info); } void JxlEncoderDestroy(JxlEncoder* enc) { if (enc) { JxlMemoryManager local_memory_manager = enc->memory_manager; @@ -1059,32 +1175,39 @@ enc->~JxlEncoder(); jxl::MemoryManagerFree(&local_memory_manager, enc); } } +JxlEncoderError JxlEncoderGetError(JxlEncoder* enc) { return enc->error; } + JxlEncoderStatus JxlEncoderUseContainer(JxlEncoder* enc, JXL_BOOL use_container) { if (enc->wrote_bytes) { - return JXL_API_ERROR("this setting can only be set at the beginning"); + return JXL_API_ERROR(enc, JXL_ENC_ERR_API_USAGE, + "this setting can only be set at the beginning"); } enc->use_container = static_cast(use_container); return JXL_ENC_SUCCESS; } JxlEncoderStatus JxlEncoderStoreJPEGMetadata(JxlEncoder* enc, JXL_BOOL store_jpeg_metadata) { if (enc->wrote_bytes) { - return JXL_API_ERROR("this setting can only be set at the beginning"); + return JXL_API_ERROR(enc, JXL_ENC_ERR_API_USAGE, + "this setting can only be set at the beginning"); } enc->store_jpeg_metadata = static_cast(store_jpeg_metadata); return JXL_ENC_SUCCESS; } JxlEncoderStatus JxlEncoderSetCodestreamLevel(JxlEncoder* enc, int level) { - if (level != 5 && level != 10) return JXL_API_ERROR("invalid level"); + if (level != 5 && level != 10) { + return JXL_API_ERROR(enc, JXL_ENC_ERR_NOT_SUPPORTED, "invalid level"); + } if (enc->wrote_bytes) { - return JXL_API_ERROR("this setting can only be set at the beginning"); + return JXL_API_ERROR(enc, JXL_ENC_ERR_API_USAGE, + "this setting can only be set at the beginning"); } enc->codestream_level = level; return JXL_ENC_SUCCESS; } @@ -1098,15 +1221,19 @@ } JxlEncoderStatus JxlEncoderSetParallelRunner(JxlEncoder* enc, JxlParallelRunner parallel_runner, void* parallel_runner_opaque) { - if (enc->thread_pool) return JXL_API_ERROR("parallel runner already set"); + if (enc->thread_pool) { + return JXL_API_ERROR(enc, JXL_ENC_ERR_API_USAGE, + "parallel runner already set"); + } enc->thread_pool = jxl::MemoryManagerMakeUnique( &enc->memory_manager, parallel_runner, parallel_runner_opaque); if (!enc->thread_pool) { - return JXL_ENC_ERROR; + return JXL_API_ERROR(enc, JXL_ENC_ERR_GENERIC, + "error setting parallel runner"); } return JXL_ENC_SUCCESS; } namespace { @@ -1123,33 +1250,37 @@ size_t factor = frame_settings->values.cparams.resampling; xsize = jxl::DivCeil(xsize, factor); ysize = jxl::DivCeil(ysize, factor); } if (xsize == 0 || ysize == 0) { - return JXL_API_ERROR("zero-sized frame is not allowed"); + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "zero-sized frame is not allowed"); } return JXL_ENC_SUCCESS; } } // namespace JxlEncoderStatus JxlEncoderAddJPEGFrame( const JxlEncoderFrameSettings* frame_settings, const uint8_t* buffer, size_t size) { if (frame_settings->enc->frames_closed) { - return JXL_ENC_ERROR; + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "Frame input is already closed"); } jxl::CodecInOut io; if (!jxl::jpeg::DecodeImageJPG(jxl::Span(buffer, size), &io)) { - return JXL_ENC_ERROR; + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_BAD_INPUT, + "Error during decode of input JPEG"); } if (!frame_settings->enc->color_encoding_set) { if (!SetColorEncodingFromJpegData( *io.Main().jpeg_data, &frame_settings->enc->metadata.m.color_encoding)) { - return JXL_ENC_ERROR; + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_BAD_INPUT, + "Error in input JPEG color space"); } } if (!frame_settings->enc->basic_info_set) { JxlBasicInfo basic_info; @@ -1157,25 +1288,51 @@ basic_info.xsize = io.Main().jpeg_data->width; basic_info.ysize = io.Main().jpeg_data->height; basic_info.uses_original_profile = true; if (JxlEncoderSetBasicInfo(frame_settings->enc, &basic_info) != JXL_ENC_SUCCESS) { - return JXL_ENC_ERROR; + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_GENERIC, + "Error setting basic info"); } } if (frame_settings->enc->metadata.m.xyb_encoded) { - // Can't XYB encode a lossless JPEG. - return JXL_ENC_ERROR; + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "Can't XYB encode a lossless JPEG"); } - + if (!io.blobs.exif.empty()) { + size_t exif_size = io.blobs.exif.size(); + // Exif data in JPEG is limited to 64k + if (exif_size > 0xFFFF) { + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_GENERIC, + "Exif larger than possible in JPEG?"); + } + exif_size += 4; // prefix 4 zero bytes for tiff offset + std::vector exif(exif_size); + memcpy(exif.data() + 4, io.blobs.exif.data(), io.blobs.exif.size()); + JxlEncoderUseBoxes(frame_settings->enc); + JxlEncoderAddBox(frame_settings->enc, "Exif", exif.data(), exif_size, + /*compress_box=*/JXL_TRUE); + } + if (!io.blobs.xmp.empty()) { + JxlEncoderUseBoxes(frame_settings->enc); + JxlEncoderAddBox(frame_settings->enc, "xml ", io.blobs.xmp.data(), + io.blobs.xmp.size(), /*compress_box=*/JXL_TRUE); + } + if (!io.blobs.jumbf.empty()) { + JxlEncoderUseBoxes(frame_settings->enc); + JxlEncoderAddBox(frame_settings->enc, "jumb", io.blobs.jumbf.data(), + io.blobs.jumbf.size(), /*compress_box=*/JXL_TRUE); + } if (frame_settings->enc->store_jpeg_metadata) { jxl::jpeg::JPEGData data_in = *io.Main().jpeg_data; jxl::PaddedBytes jpeg_data; if (!jxl::jpeg::EncodeJPEGData(data_in, &jpeg_data, frame_settings->values.cparams)) { - return JXL_ENC_ERROR; + return JXL_API_ERROR( + frame_settings->enc, JXL_ENC_ERR_JBRD, + "JPEG bitstream reconstruction data cannot be encoded"); } frame_settings->enc->jpeg_metadata = std::vector( jpeg_data.data(), jpeg_data.data() + jpeg_data.size()); } @@ -1186,21 +1343,25 @@ jxl::JxlEncoderQueuedFrame{ frame_settings->values, jxl::ImageBundle(&frame_settings->enc->metadata.m), {}}); if (!queued_frame) { - return JXL_ENC_ERROR; + // TODO(jon): when can this happen? is this an API usage error? + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_GENERIC, + "No frame queued?"); } queued_frame->frame.SetFromImage(std::move(*io.Main().color()), io.Main().c_current()); size_t xsize, ysize; if (GetCurrentDimensions(frame_settings, xsize, ysize) != JXL_ENC_SUCCESS) { - return JXL_API_ERROR("bad dimensions"); + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_GENERIC, + "bad dimensions"); } if (xsize != static_cast(io.Main().jpeg_data->width) || ysize != static_cast(io.Main().jpeg_data->height)) { - return JXL_API_ERROR("JPEG dimensions don't match frame dimensions"); + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_GENERIC, + "JPEG dimensions don't match frame dimensions"); } std::vector extra_channels( frame_settings->enc->metadata.m.num_extra_channels); for (auto& extra_channel : extra_channels) { extra_channel = jxl::ImageF(xsize, ysize); @@ -1222,15 +1383,28 @@ (!frame_settings->enc->color_encoding_set && !frame_settings->enc->metadata.m.xyb_encoded)) { // Basic Info must be set, and color encoding must be set directly, // or set to XYB via JxlBasicInfo.uses_original_profile = JXL_FALSE // Otherwise, this is an API misuse. - return JXL_ENC_ERROR; + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "Basic info or color encoding not set yet"); } if (frame_settings->enc->frames_closed) { - return JXL_ENC_ERROR; + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "Frame input already closed"); + } + if (pixel_format->num_channels < 3) { + if (frame_settings->enc->basic_info.num_color_channels != 1) { + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "Grayscale pixel format input for an RGB image"); + } + } else { + if (frame_settings->enc->basic_info.num_color_channels != 3) { + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "RGB pixel format input for a grayscale image"); + } } auto queued_frame = jxl::MemoryManagerMakeUnique( &frame_settings->enc->memory_manager, // JxlEncoderQueuedFrame is a struct with no constructors, so we use the @@ -1239,11 +1413,13 @@ frame_settings->values, jxl::ImageBundle(&frame_settings->enc->metadata.m), {}}); if (!queued_frame) { - return JXL_ENC_ERROR; + // TODO(jon): when can this happen? is this an API usage error? + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_GENERIC, + "No frame queued?"); } jxl::ColorEncoding c_current; if (!frame_settings->enc->color_encoding_set) { if ((pixel_format->data_type == JXL_TYPE_FLOAT) || @@ -1259,15 +1435,18 @@ uint32_t num_channels = pixel_format->num_channels; size_t has_interleaved_alpha = static_cast(num_channels == 2 || num_channels == 4); if (has_interleaved_alpha > frame_settings->enc->metadata.m.num_extra_channels) { - return JXL_API_ERROR("number of extra channels mismatch"); + return JXL_API_ERROR( + frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "number of extra channels mismatch (need 1 extra channel for alpha)"); } size_t xsize, ysize; if (GetCurrentDimensions(frame_settings, xsize, ysize) != JXL_ENC_SUCCESS) { - return JXL_API_ERROR("bad dimensions"); + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_GENERIC, + "bad dimensions"); } std::vector extra_channels( frame_settings->enc->metadata.m.num_extra_channels); for (auto& extra_channel : extra_channels) { extra_channel = jxl::ImageF(xsize, ysize); @@ -1296,51 +1475,58 @@ frame_settings->values.header.layer_info.blend_info.source > 0; if (!jxl::BufferToImageBundle(*pixel_format, xsize, ysize, buffer, size, frame_settings->enc->thread_pool.get(), c_current, &(queued_frame->frame))) { - return JXL_ENC_ERROR; + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "Invalid input buffer"); } if (frame_settings->values.lossless && frame_settings->enc->metadata.m.xyb_encoded) { - return JXL_API_ERROR("Set use_original_profile=true for lossless encoding"); + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "Set use_original_profile=true for lossless encoding"); } queued_frame->option_values.cparams.level = frame_settings->enc->codestream_level; QueueFrame(frame_settings, queued_frame); return JXL_ENC_SUCCESS; } JxlEncoderStatus JxlEncoderUseBoxes(JxlEncoder* enc) { if (enc->wrote_bytes) { - return JXL_API_ERROR("this setting can only be set at the beginning"); + return JXL_API_ERROR(enc, JXL_ENC_ERR_API_USAGE, + "this setting can only be set at the beginning"); } enc->use_boxes = true; return JXL_ENC_SUCCESS; } JxlEncoderStatus JxlEncoderAddBox(JxlEncoder* enc, const JxlBoxType type, const uint8_t* contents, size_t size, JXL_BOOL compress_box) { if (!enc->use_boxes) { return JXL_API_ERROR( + enc, JXL_ENC_ERR_API_USAGE, "must set JxlEncoderUseBoxes at the beginning to add boxes"); } if (compress_box) { if (memcmp("jxl", type, 3) == 0) { return JXL_API_ERROR( + enc, JXL_ENC_ERR_API_USAGE, "brob box may not contain a type starting with \"jxl\""); } if (memcmp("jbrd", type, 4) == 0) { - return JXL_API_ERROR("jbrd box may not be brob compressed"); + return JXL_API_ERROR(enc, JXL_ENC_ERR_API_USAGE, + "jbrd box may not be brob compressed"); } if (memcmp("brob", type, 4) == 0) { // The compress_box will compress an existing non-brob box into a brob // box. If already giving a valid brotli-compressed brob box, set // compress_box to false since it is already compressed. - return JXL_API_ERROR("a brob box cannot contain another brob box"); + return JXL_API_ERROR(enc, JXL_ENC_ERR_API_USAGE, + "a brob box cannot contain another brob box"); } } auto box = jxl::MemoryManagerMakeUnique( &enc->memory_manager); @@ -1354,31 +1540,37 @@ JXL_EXPORT JxlEncoderStatus JxlEncoderSetExtraChannelBuffer( const JxlEncoderOptions* frame_settings, const JxlPixelFormat* pixel_format, const void* buffer, size_t size, uint32_t index) { if (index >= frame_settings->enc->metadata.m.num_extra_channels) { - return JXL_API_ERROR("Invalid value for the index of extra channel"); + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "Invalid value for the index of extra channel"); } if (!frame_settings->enc->basic_info_set || !frame_settings->enc->color_encoding_set) { - return JXL_ENC_ERROR; + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "Basic info has to be set first"); } if (frame_settings->enc->input_queue.empty()) { - return JXL_ENC_ERROR; + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "First add image frame, then extra channels"); } if (frame_settings->enc->frames_closed) { - return JXL_ENC_ERROR; + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "Frame input already closed"); } size_t xsize, ysize; if (GetCurrentDimensions(frame_settings, xsize, ysize) != JXL_ENC_SUCCESS) { - return JXL_API_ERROR("bad dimensions"); + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_GENERIC, + "bad dimensions"); } if (!jxl::BufferToImageF(*pixel_format, xsize, ysize, buffer, size, frame_settings->enc->thread_pool.get(), &frame_settings->enc->input_queue.back() .frame->frame.extra_channels()[index])) { - return JXL_API_ERROR("Failed to set buffer for extra channel"); + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "Failed to set buffer for extra channel"); } frame_settings->enc->input_queue.back().frame->ec_initialized[index] = 1; return JXL_ENC_SUCCESS; } @@ -1416,17 +1608,19 @@ } JxlEncoderStatus JxlEncoderSetFrameHeader(JxlEncoderOptions* frame_settings, const JxlFrameHeader* frame_header) { if (frame_header->layer_info.blend_info.source > 3) { - return JXL_API_ERROR("invalid blending source index"); + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "invalid blending source index"); } // If there are no extra channels, it's ok for the value to be 0. if (frame_header->layer_info.blend_info.alpha != 0 && frame_header->layer_info.blend_info.alpha >= frame_settings->enc->metadata.m.extra_channel_info.size()) { - return JXL_API_ERROR("alpha blend channel index out of bounds"); + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "alpha blend channel index out of bounds"); } frame_settings->values.header = *frame_header; // Setting the frame header resets the frame name, it must be set again with // JxlEncoderSetFrameName if desired. @@ -1437,11 +1631,12 @@ JxlEncoderStatus JxlEncoderSetExtraChannelBlendInfo( JxlEncoderOptions* frame_settings, size_t index, const JxlBlendInfo* blend_info) { if (index >= frame_settings->enc->metadata.m.num_extra_channels) { - return JXL_API_ERROR("Invalid value for the index of extra channel"); + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "Invalid value for the index of extra channel"); } if (frame_settings->values.extra_channel_blend_info.size() != frame_settings->enc->metadata.m.num_extra_channels) { JxlBlendInfo default_blend_info; @@ -1455,11 +1650,12 @@ JxlEncoderStatus JxlEncoderSetFrameName(JxlEncoderFrameSettings* frame_settings, const char* frame_name) { std::string str = frame_name ? frame_name : ""; if (str.size() > 1071) { - return JXL_API_ERROR("frame name can be max 1071 bytes long"); + return JXL_API_ERROR(frame_settings->enc, JXL_ENC_ERR_API_USAGE, + "frame name can be max 1071 bytes long"); } frame_settings->values.frame_name = str; frame_settings->values.header.name_length = str.size(); return JXL_ENC_SUCCESS; } diff --git a/third_party/jpeg-xl/lib/jxl/encode_internal.h b/third_party/jpeg-xl/lib/jxl/encode_internal.h --- a/third_party/jpeg-xl/lib/jxl/encode_internal.h +++ b/third_party/jpeg-xl/lib/jxl/encode_internal.h @@ -104,10 +104,11 @@ } // namespace jxl // Internal use only struct, can only be initialized correctly by // JxlEncoderCreate. struct JxlEncoderStruct { + JxlEncoderError error = JxlEncoderError::JXL_ENC_ERR_OK; JxlMemoryManager memory_manager; jxl::MemoryManagerUniquePtr thread_pool{ nullptr, jxl::MemoryManagerDeleteHelper(&memory_manager)}; JxlCmsInterface cms; std::vector> @@ -143,10 +144,11 @@ // Wrote any output at all, so wrote the data before the first user added // frame or box, such as signature, basic info, ICC profile or jpeg // reconstruction box. bool wrote_bytes; jxl::CompressParams last_used_cparams; + JxlBasicInfo basic_info; // Encoder wrote a jxlp (partial codestream) box, so any next codestream // parts must also be written in jxlp boxes, a single jxlc box cannot be // used. The counter is used for the 4-byte jxlp box index header. size_t jxlp_counter; diff --git a/third_party/jpeg-xl/lib/jxl/encode_test.cc b/third_party/jpeg-xl/lib/jxl/encode_test.cc --- a/third_party/jpeg-xl/lib/jxl/encode_test.cc +++ b/third_party/jpeg-xl/lib/jxl/encode_test.cc @@ -56,12 +56,11 @@ JxlEncoderPtr enc = JxlEncoderMake(nullptr); EXPECT_NE(nullptr, enc.get()); JxlEncoderCloseInput(enc.get()); - const std::string jpeg_path = - "third_party/imagecompression.info/flower_foveon.png.im_q85_420.jpg"; + const std::string jpeg_path = "jxl/flower/flower.png.im_q85_420.jpg"; const jxl::PaddedBytes orig = jxl::ReadTestData(jpeg_path); JxlEncoderFrameSettings* frame_settings = JxlEncoderFrameSettingsCreate(enc.get(), NULL); @@ -108,11 +107,11 @@ jxl::test::SomeTestImageToCodecInOut(pixels, 4, xsize, ysize); JxlColorEncoding color_encoding; JxlColorEncodingSetToSRGB(&color_encoding, /*is_gray=*/pixel_format.num_channels < 3); - EXPECT_EQ(JXL_ENC_SUCCESS, + EXPECT_EQ(JXL_ENC_ERROR, JxlEncoderSetColorEncoding(enc.get(), &color_encoding)); JxlEncoderFrameSettings* frame_settings = JxlEncoderFrameSettingsCreate(enc.get(), NULL); EXPECT_EQ(JXL_ENC_ERROR, JxlEncoderAddImageFrame(frame_settings, &pixel_format, @@ -177,13 +176,19 @@ } // 16-bit alpha means this requires level 10 EXPECT_EQ(JXL_ENC_SUCCESS, JxlEncoderSetCodestreamLevel(enc, 10)); EXPECT_EQ(JXL_ENC_SUCCESS, JxlEncoderSetBasicInfo(enc, &basic_info)); JxlColorEncoding color_encoding; - JxlColorEncodingSetToSRGB(&color_encoding, - /*is_gray=*/pixel_format.num_channels < 3); + JxlColorEncodingSetToSRGB(&color_encoding, true); + EXPECT_EQ(JXL_ENC_ERROR, JxlEncoderSetColorEncoding(enc, &color_encoding)); + JxlColorEncodingSetToSRGB(&color_encoding, false); EXPECT_EQ(JXL_ENC_SUCCESS, JxlEncoderSetColorEncoding(enc, &color_encoding)); + pixel_format.num_channels = 1; + EXPECT_EQ(JXL_ENC_ERROR, + JxlEncoderAddImageFrame(frame_settings, &pixel_format, + pixels.data(), pixels.size())); + pixel_format.num_channels = 4; EXPECT_EQ(JXL_ENC_SUCCESS, JxlEncoderAddImageFrame(frame_settings, &pixel_format, pixels.data(), pixels.size())); JxlEncoderCloseInput(enc); @@ -449,14 +454,11 @@ EXPECT_EQ( JXL_ENC_SUCCESS, JxlEncoderFrameSettingsSetOption( frame_settings, JXL_ENC_FRAME_SETTING_MODULAR_NB_PREV_CHANNELS, 7)); VerifyFrameEncoding(enc.get(), frame_settings); - // It was set to 30, but becomes 32 because in the C++ implementation, the - // numerical RCT values are shifted 2 compared to the specification. The - // API uses the values seen in the JXL specification. - EXPECT_EQ(32, enc->last_used_cparams.colorspace); + EXPECT_EQ(30, enc->last_used_cparams.colorspace); EXPECT_EQ(2, enc->last_used_cparams.modular_group_size_shift); EXPECT_EQ(jxl::Predictor::Best, enc->last_used_cparams.options.predictor); EXPECT_EQ(0.77f, enc->last_used_cparams.options.nb_repeats); EXPECT_EQ(7, enc->last_used_cparams.options.max_properties); } @@ -769,12 +771,11 @@ basic_info.xsize = 1ull << 31ull; EXPECT_EQ(JXL_ENC_ERROR, JxlEncoderSetBasicInfo(enc.get(), &basic_info)); } TEST(EncodeTest, JXL_TRANSCODE_JPEG_TEST(JPEGReconstructionTest)) { - const std::string jpeg_path = - "third_party/imagecompression.info/flower_foveon.png.im_q85_420.jpg"; + const std::string jpeg_path = "jxl/flower/flower.png.im_q85_420.jpg"; const jxl::PaddedBytes orig = jxl::ReadTestData(jpeg_path); JxlEncoderPtr enc = JxlEncoderMake(nullptr); JxlEncoderFrameSettings* frame_settings = JxlEncoderFrameSettingsCreate(enc.get(), NULL); @@ -1304,12 +1305,13 @@ #if JPEGXL_ENABLE_JPEG // Loading .jpg files requires libjpeg support. TEST(EncodeTest, JXL_TRANSCODE_JPEG_TEST(JPEGFrameTest)) { for (int skip_basic_info = 0; skip_basic_info < 2; skip_basic_info++) { for (int skip_color_encoding = 0; skip_color_encoding < 2; skip_color_encoding++) { - const std::string jpeg_path = - "third_party/imagecompression.info/flower_foveon_cropped.jpg"; + // cannot set color encoding if basic info is not set + if (skip_basic_info && !skip_color_encoding) continue; + const std::string jpeg_path = "jxl/flower/flower_cropped.jpg"; const jxl::PaddedBytes orig = jxl::ReadTestData(jpeg_path); jxl::CodecInOut orig_io; ASSERT_TRUE(SetFromBytes(jxl::Span(orig), &orig_io, /*pool=*/nullptr)); diff --git a/third_party/jpeg-xl/lib/jxl/image.h b/third_party/jpeg-xl/lib/jxl/image.h --- a/third_party/jpeg-xl/lib/jxl/image.h +++ b/third_party/jpeg-xl/lib/jxl/image.h @@ -298,10 +298,36 @@ size_t xsize() const { return xsize_; } size_t ysize() const { return ysize_; } T x1() const { return x0_ + xsize_; } T y1() const { return y0_ + ysize_; } + RectT ShiftLeft(size_t shiftx, size_t shifty) const { + return RectT(x0_ * (1 << shiftx), y0_ * (1 << shifty), xsize_ << shiftx, + ysize_ << shifty); + } + RectT ShiftLeft(size_t shift) const { return ShiftLeft(shift, shift); } + + // Requires x0(), y0() to be multiples of 1< CeilShiftRight(size_t shiftx, size_t shifty) const { + JXL_ASSERT(x0_ % (1 << shiftx) == 0); + JXL_ASSERT(y0_ % (1 << shifty) == 0); + return RectT(x0_ / (1 << shiftx), y0_ / (1 << shifty), + DivCeil(xsize_, T{1} << shiftx), + DivCeil(ysize_, T{1} << shifty)); + } + RectT CeilShiftRight(std::pair shift) const { + return CeilShiftRight(shift.first, shift.second); + } + RectT CeilShiftRight(size_t shift) const { + return CeilShiftRight(shift, shift); + } + + template + RectT As() const { + return RectT(U(x0_), U(y0_), U(xsize_), U(ysize_)); + } + private: // Returns size_max, or whatever is left in [begin, end). static constexpr size_t ClampedSize(T begin, size_t size_max, T end) { return (static_cast(begin + size_max) <= end) ? size_max diff --git a/third_party/jpeg-xl/lib/jxl/jpeg/jpeg_data.h b/third_party/jpeg-xl/lib/jxl/jpeg/jpeg_data.h --- a/third_party/jpeg-xl/lib/jxl/jpeg/jpeg_data.h +++ b/third_party/jpeg-xl/lib/jxl/jpeg/jpeg_data.h @@ -103,10 +103,11 @@ EMPTY_DQT, OUT_OF_BAND_COEFF, EOB_RUN_TOO_LONG, IMAGE_TOO_LARGE, INVALID_QUANT_TBL_PRECISION, + TAIL_DATA_TOO_LARGE }; // Quantization values for an 8x8 pixel block. struct JPEGQuantTable { std::array values; diff --git a/third_party/jpeg-xl/lib/jxl/jpeg/jpeg_data.cc b/third_party/jpeg-xl/lib/jxl/jpeg/jpeg_data.cc --- a/third_party/jpeg-xl/lib/jxl/jpeg/jpeg_data.cc +++ b/third_party/jpeg-xl/lib/jxl/jpeg/jpeg_data.cc @@ -363,10 +363,15 @@ uint32_t len = visitor->IsReading() ? 0 : inter_marker_data[i].size(); JXL_RETURN_IF_ERROR(visitor->Bits(16, 0, &len)); if (visitor->IsReading()) inter_marker_data_sizes.emplace_back(len); } uint32_t tail_data_len = tail_data.size(); + if (!visitor->IsReading() && tail_data_len > 4260096) { + error = JPEGReadError::TAIL_DATA_TOO_LARGE; + return JXL_FAILURE("Tail data too large (max size = 4260096, size = %u).", + tail_data_len); + } JXL_RETURN_IF_ERROR(visitor->U32(Val(0), BitsOffset(8, 1), BitsOffset(16, 257), BitsOffset(22, 65793), 0, &tail_data_len)); JXL_RETURN_IF_ERROR(visitor->Bool(false, &has_zero_padding_bit)); diff --git a/third_party/jpeg-xl/lib/jxl/jxl_test.cc b/third_party/jpeg-xl/lib/jxl/jxl_test.cc --- a/third_party/jpeg-xl/lib/jxl/jxl_test.cc +++ b/third_party/jpeg-xl/lib/jxl/jxl_test.cc @@ -229,34 +229,32 @@ IsSlightlyBelow(90)); } TEST(JxlTest, RoundtripResample2MT) { ThreadPoolInternal pool(4); - const PaddedBytes orig = - ReadTestData("third_party/imagecompression.info/flower_foveon.png"); + const PaddedBytes orig = ReadTestData("jxl/flower/flower.png"); // image has to be large enough to have multiple groups after downsampling CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); CompressParams cparams; cparams.resampling = 2; cparams.speed_tier = SpeedTier::kFalcon; DecompressParams dparams; CodecInOut io2; // TODO(veluca): Figure out why msan and release produce different // file size. - EXPECT_LE(Roundtrip(&io, cparams, dparams, &pool, &io2), 64500u); + EXPECT_LE(Roundtrip(&io, cparams, dparams, &pool, &io2), 200000u); EXPECT_THAT(ComputeDistance2(io.Main(), io2.Main(), GetJxlCms()), - IsSlightlyBelow(320)); + IsSlightlyBelow(340)); } // Roundtrip the image using a parallel runner that executes single-threaded but // in random order. TEST(JxlTest, RoundtripOutOfOrderProcessing) { FakeParallelRunner fake_pool(/*order_seed=*/123, /*num_threads=*/8); ThreadPool pool(&JxlFakeParallelRunner, &fake_pool); - const PaddedBytes orig = - ReadTestData("third_party/imagecompression.info/flower_foveon.png"); + const PaddedBytes orig = ReadTestData("jxl/flower/flower.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); // Image size is selected so that the block border needed is larger than the // amount of pixels available on the next block. io.ShrinkTo(513, 515); @@ -274,12 +272,11 @@ } TEST(JxlTest, RoundtripOutOfOrderProcessingBorder) { FakeParallelRunner fake_pool(/*order_seed=*/47, /*num_threads=*/8); ThreadPool pool(&JxlFakeParallelRunner, &fake_pool); - const PaddedBytes orig = - ReadTestData("third_party/imagecompression.info/flower_foveon.png"); + const PaddedBytes orig = ReadTestData("jxl/flower/flower.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); // Image size is selected so that the block border needed is larger than the // amount of pixels available on the next block. io.ShrinkTo(513, 515); @@ -352,12 +349,11 @@ #if JXL_TEST_NL TEST(JxlTest, RoundtripMultiGroupNL) { ThreadPoolInternal pool(4); - const PaddedBytes orig = - ReadTestData("third_party/imagecompression.info/flower_foveon.png"); + const PaddedBytes orig = ReadTestData("jxl/flower/flower.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); io.ShrinkTo(600, 1024); // partial X, full Y group CompressParams cparams; @@ -381,47 +377,45 @@ #endif TEST(JxlTest, RoundtripMultiGroup) { ThreadPoolInternal pool(4); - const PaddedBytes orig = - ReadTestData("third_party/imagecompression.info/flower_foveon.png"); + const PaddedBytes orig = ReadTestData("jxl/flower/flower.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); io.ShrinkTo(600, 1024); CompressParams cparams; DecompressParams dparams; cparams.butteraugli_distance = 1.0f; cparams.speed_tier = SpeedTier::kKitten; CodecInOut io2; - EXPECT_LE(Roundtrip(&io, cparams, dparams, &pool, &io2), 40000u); + EXPECT_LE(Roundtrip(&io, cparams, dparams, &pool, &io2), 55000u); EXPECT_THAT(ComputeDistance2(io.Main(), io2.Main(), GetJxlCms()), - IsSlightlyBelow(15)); + IsSlightlyBelow(11)); cparams.butteraugli_distance = 2.0f; cparams.speed_tier = SpeedTier::kWombat; CodecInOut io3; - EXPECT_LE(Roundtrip(&io, cparams, dparams, &pool, &io3), 22100u); + EXPECT_LE(Roundtrip(&io, cparams, dparams, &pool, &io3), 34000u); EXPECT_THAT(ComputeDistance2(io.Main(), io3.Main(), GetJxlCms()), - IsSlightlyBelow(24)); + IsSlightlyBelow(18)); } TEST(JxlTest, RoundtripLargeFast) { ThreadPoolInternal pool(8); - const PaddedBytes orig = - ReadTestData("third_party/imagecompression.info/flower_foveon.png"); + const PaddedBytes orig = ReadTestData("jxl/flower/flower.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); CompressParams cparams; cparams.speed_tier = SpeedTier::kSquirrel; DecompressParams dparams; CodecInOut io2; - EXPECT_LE(Roundtrip(&io, cparams, dparams, &pool, &io2), 265000u); + EXPECT_LE(Roundtrip(&io, cparams, dparams, &pool, &io2), 450000u); } TEST(JxlTest, RoundtripDotsForceEpf) { ThreadPoolInternal pool(8); const PaddedBytes orig = @@ -434,20 +428,19 @@ cparams.dots = Override::kOn; cparams.speed_tier = SpeedTier::kSquirrel; DecompressParams dparams; CodecInOut io2; - EXPECT_LE(Roundtrip(&io, cparams, dparams, &pool, &io2), 265000u); + EXPECT_LE(Roundtrip(&io, cparams, dparams, &pool, &io2), 450000u); } // Checks for differing size/distance in two consecutive runs of distance 2, // which involves additional processing including adaptive reconstruction. // Failing this may be a sign of race conditions or invalid memory accesses. TEST(JxlTest, RoundtripD2Consistent) { ThreadPoolInternal pool(8); - const PaddedBytes orig = - ReadTestData("third_party/imagecompression.info/flower_foveon.png"); + const PaddedBytes orig = ReadTestData("jxl/flower/flower.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); CompressParams cparams; cparams.speed_tier = SpeedTier::kSquirrel; @@ -475,12 +468,11 @@ } // Same as above, but for full image, testing multiple groups. TEST(JxlTest, RoundtripLargeConsistent) { ThreadPoolInternal pool(8); - const PaddedBytes orig = - ReadTestData("third_party/imagecompression.info/flower_foveon.png"); + const PaddedBytes orig = ReadTestData("jxl/flower/flower.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); CompressParams cparams; cparams.speed_tier = SpeedTier::kSquirrel; @@ -1218,16 +1210,14 @@ EXPECT_TRUE(SamePixels(*io.Main().alpha(), *io2.Main().alpha())); } TEST(JxlTest, RoundtripYCbCr420) { ThreadPool* pool = nullptr; - const PaddedBytes orig = - ReadTestData("third_party/imagecompression.info/flower_foveon.png"); + const PaddedBytes orig = ReadTestData("jxl/flower/flower.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, pool)); - const PaddedBytes yuv420 = ReadTestData( - "third_party/imagecompression.info/flower_foveon.png.ffmpeg.y4m"); + const PaddedBytes yuv420 = ReadTestData("jxl/flower/flower.png.ffmpeg.y4m"); CodecInOut io2; ASSERT_TRUE(test::DecodeImageY4M(Span(yuv420), &io2)); CompressParams cparams = CParamsForLossless(); cparams.speed_tier = SpeedTier::kThunder; @@ -1239,15 +1229,15 @@ EXPECT_TRUE(EncodeFile(cparams, &io2, &enc_state, &compressed, GetJxlCms(), aux_out, pool)); CodecInOut io3; EXPECT_TRUE(DecodeFile(dparams, compressed, &io3, pool)); - EXPECT_LE(compressed.size(), 1325000u); + EXPECT_LE(compressed.size(), 2000000u); // we're comparing an original PNG with a YCbCr 4:2:0 version EXPECT_THAT(ComputeDistance2(io.Main(), io3.Main(), GetJxlCms()), - IsSlightlyBelow(8.5)); + IsSlightlyBelow(4.3)); } TEST(JxlTest, RoundtripDots) { ThreadPool* pool = nullptr; const PaddedBytes orig = @@ -1472,22 +1462,20 @@ return compressed.size(); } TEST(JxlTest, JXL_TRANSCODE_JPEG_TEST(RoundtripJpegRecompression444)) { ThreadPoolInternal pool(8); - const PaddedBytes orig = ReadTestData( - "third_party/imagecompression.info/flower_foveon.png.im_q85_444.jpg"); - // JPEG size is 326'916 bytes. - EXPECT_LE(RoundtripJpeg(orig, &pool), 256000u); + const PaddedBytes orig = ReadTestData("jxl/flower/flower.png.im_q85_444.jpg"); + // JPEG size is 696,659 bytes. + EXPECT_LE(RoundtripJpeg(orig, &pool), 570000u); } #if JPEGXL_ENABLE_JPEG TEST(JxlTest, JXL_TRANSCODE_JPEG_TEST(RoundtripJpegRecompressionToPixels)) { ThreadPoolInternal pool(8); - const PaddedBytes orig = ReadTestData( - "third_party/imagecompression.info/flower_foveon.png.im_q85_444.jpg"); + const PaddedBytes orig = ReadTestData("jxl/flower/flower.png.im_q85_444.jpg"); CodecInOut io; ASSERT_TRUE(jpeg::DecodeImageJPG(Span(orig), &io)); CodecInOut io2; ASSERT_TRUE(SetFromBytes(Span(orig), &io2, &pool)); @@ -1506,12 +1494,11 @@ IsSlightlyBelow(12)); } TEST(JxlTest, JXL_TRANSCODE_JPEG_TEST(RoundtripJpegRecompressionToPixels420)) { ThreadPoolInternal pool(8); - const PaddedBytes orig = ReadTestData( - "third_party/imagecompression.info/flower_foveon.png.im_q85_420.jpg"); + const PaddedBytes orig = ReadTestData("jxl/flower/flower.png.im_q85_420.jpg"); CodecInOut io; ASSERT_TRUE(jpeg::DecodeImageJPG(Span(orig), &io)); CodecInOut io2; ASSERT_TRUE(SetFromBytes(Span(orig), &io2, &pool)); @@ -1529,12 +1516,11 @@ } TEST(JxlTest, JXL_TRANSCODE_JPEG_TEST(RoundtripJpegRecompressionToPixels420EarlyFlush)) { ThreadPoolInternal pool(8); - const PaddedBytes orig = ReadTestData( - "third_party/imagecompression.info/flower_foveon.png.im_q85_420.jpg"); + const PaddedBytes orig = ReadTestData("jxl/flower/flower.png.im_q85_420.jpg"); CodecInOut io; ASSERT_TRUE(jpeg::DecodeImageJPG(Span(orig), &io)); CodecInOut io2; ASSERT_TRUE(SetFromBytes(Span(orig), &io2, &pool)); @@ -1547,18 +1533,17 @@ CodecInOut io3; Roundtrip(&io, cparams, dparams, &pool, &io3); EXPECT_THAT(ComputeDistance2(io2.Main(), io3.Main(), GetJxlCms()), - IsSlightlyBelow(650)); + IsSlightlyBelow(4410)); } TEST(JxlTest, JXL_TRANSCODE_JPEG_TEST(RoundtripJpegRecompressionToPixels420Mul16)) { ThreadPoolInternal pool(8); - const PaddedBytes orig = ReadTestData( - "third_party/imagecompression.info/flower_foveon_cropped.jpg"); + const PaddedBytes orig = ReadTestData("jxl/flower/flower_cropped.jpg"); CodecInOut io; ASSERT_TRUE(jpeg::DecodeImageJPG(Span(orig), &io)); CodecInOut io2; ASSERT_TRUE(SetFromBytes(Span(orig), &io2, &pool)); @@ -1576,13 +1561,12 @@ } TEST(JxlTest, JXL_TRANSCODE_JPEG_TEST(RoundtripJpegRecompressionToPixels_asymmetric)) { ThreadPoolInternal pool(8); - const PaddedBytes orig = ReadTestData( - "third_party/imagecompression.info/" - "flower_foveon.png.im_q85_asymmetric.jpg"); + const PaddedBytes orig = + ReadTestData("jxl/flower/flower.png.im_q85_asymmetric.jpg"); CodecInOut io; ASSERT_TRUE(jpeg::DecodeImageJPG(Span(orig), &io)); CodecInOut io2; ASSERT_TRUE(SetFromBytes(Span(orig), &io2, &pool)); @@ -1601,82 +1585,76 @@ #endif TEST(JxlTest, JXL_TRANSCODE_JPEG_TEST(RoundtripJpegRecompressionGray)) { ThreadPoolInternal pool(8); - const PaddedBytes orig = ReadTestData( - "third_party/imagecompression.info/flower_foveon.png.im_q85_gray.jpg"); - // JPEG size is 167'025 bytes. - EXPECT_LE(RoundtripJpeg(orig, &pool), 140000u); + const PaddedBytes orig = + ReadTestData("jxl/flower/flower.png.im_q85_gray.jpg"); + // JPEG size is 456,528 bytes. + EXPECT_LE(RoundtripJpeg(orig, &pool), 390000u); } TEST(JxlTest, JXL_TRANSCODE_JPEG_TEST(RoundtripJpegRecompression420)) { ThreadPoolInternal pool(8); - const PaddedBytes orig = ReadTestData( - "third_party/imagecompression.info/flower_foveon.png.im_q85_420.jpg"); - // JPEG size is 226'018 bytes. - EXPECT_LE(RoundtripJpeg(orig, &pool), 181050u); + const PaddedBytes orig = ReadTestData("jxl/flower/flower.png.im_q85_420.jpg"); + // JPEG size is 546,797 bytes. + EXPECT_LE(RoundtripJpeg(orig, &pool), 460000u); } TEST(JxlTest, JXL_TRANSCODE_JPEG_TEST(RoundtripJpegRecompression_luma_subsample)) { ThreadPoolInternal pool(8); - const PaddedBytes orig = ReadTestData( - "third_party/imagecompression.info/" - "flower_foveon.png.im_q85_luma_subsample.jpg"); - // JPEG size is 216'069 bytes. - EXPECT_LE(RoundtripJpeg(orig, &pool), 181000u); + const PaddedBytes orig = + ReadTestData("jxl/flower/flower.png.im_q85_luma_subsample.jpg"); + // JPEG size is 400,724 bytes. + EXPECT_LE(RoundtripJpeg(orig, &pool), 330000u); } TEST(JxlTest, JXL_TRANSCODE_JPEG_TEST(RoundtripJpegRecompression444_12)) { // 444 JPEG that has an interesting sampling-factor (1x2, 1x2, 1x2). ThreadPoolInternal pool(8); - const PaddedBytes orig = ReadTestData( - "third_party/imagecompression.info/flower_foveon.png.im_q85_444_1x2.jpg"); - // JPEG size is 329'942 bytes. - EXPECT_LE(RoundtripJpeg(orig, &pool), 256000u); + const PaddedBytes orig = + ReadTestData("jxl/flower/flower.png.im_q85_444_1x2.jpg"); + // JPEG size is 703,874 bytes. + EXPECT_LE(RoundtripJpeg(orig, &pool), 570000u); } TEST(JxlTest, JXL_TRANSCODE_JPEG_TEST(RoundtripJpegRecompression422)) { ThreadPoolInternal pool(8); - const PaddedBytes orig = ReadTestData( - "third_party/imagecompression.info/flower_foveon.png.im_q85_422.jpg"); - // JPEG size is 265'590 bytes. - EXPECT_LE(RoundtripJpeg(orig, &pool), 209000u); + const PaddedBytes orig = ReadTestData("jxl/flower/flower.png.im_q85_422.jpg"); + // JPEG size is 522,057 bytes. + EXPECT_LE(RoundtripJpeg(orig, &pool), 500000u); } TEST(JxlTest, JXL_TRANSCODE_JPEG_TEST(RoundtripJpegRecompression440)) { ThreadPoolInternal pool(8); - const PaddedBytes orig = ReadTestData( - "third_party/imagecompression.info/flower_foveon.png.im_q85_440.jpg"); - // JPEG size is 262'249 bytes. - EXPECT_LE(RoundtripJpeg(orig, &pool), 209000u); + const PaddedBytes orig = ReadTestData("jxl/flower/flower.png.im_q85_440.jpg"); + // JPEG size is 603,623 bytes. + EXPECT_LE(RoundtripJpeg(orig, &pool), 510000u); } TEST(JxlTest, JXL_TRANSCODE_JPEG_TEST(RoundtripJpegRecompression_asymmetric)) { // 2x vertical downsample of one chroma channel, 2x horizontal downsample of // the other. ThreadPoolInternal pool(8); - const PaddedBytes orig = ReadTestData( - "third_party/imagecompression.info/" - "flower_foveon.png.im_q85_asymmetric.jpg"); - // JPEG size is 262'249 bytes. - EXPECT_LE(RoundtripJpeg(orig, &pool), 209000u); + const PaddedBytes orig = + ReadTestData("jxl/flower/flower.png.im_q85_asymmetric.jpg"); + // JPEG size is 604,601 bytes. + EXPECT_LE(RoundtripJpeg(orig, &pool), 510000u); } TEST(JxlTest, JXL_TRANSCODE_JPEG_TEST(RoundtripJpegRecompression420Progr)) { ThreadPoolInternal pool(8); - const PaddedBytes orig = ReadTestData( - "third_party/imagecompression.info/" - "flower_foveon.png.im_q85_420_progr.jpg"); - EXPECT_LE(RoundtripJpeg(orig, &pool), 181000u); + const PaddedBytes orig = + ReadTestData("jxl/flower/flower.png.im_q85_420_progr.jpg"); + // JPEG size is 522,057 bytes. + EXPECT_LE(RoundtripJpeg(orig, &pool), 460000u); } TEST(JxlTest, RoundtripProgressive) { ThreadPoolInternal pool(4); - const PaddedBytes orig = - ReadTestData("third_party/imagecompression.info/flower_foveon.png"); + const PaddedBytes orig = ReadTestData("jxl/flower/flower.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); io.ShrinkTo(600, 1024); CompressParams cparams; @@ -1685,13 +1663,13 @@ cparams.butteraugli_distance = 1.0f; cparams.progressive_dc = true; cparams.responsive = true; cparams.progressive_mode = true; CodecInOut io2; - EXPECT_LE(Roundtrip(&io, cparams, dparams, &pool, &io2), 40000u); + EXPECT_LE(Roundtrip(&io, cparams, dparams, &pool, &io2), 61000u); EXPECT_THAT(ButteraugliDistance(io, io2, cparams.ba_params, GetJxlCms(), /*distmap=*/nullptr, &pool), - IsSlightlyBelow(1.1f)); + IsSlightlyBelow(1.2f)); } } // namespace } // namespace jxl diff --git a/third_party/jpeg-xl/lib/jxl/libjxl.pc.in b/third_party/jpeg-xl/lib/jxl/libjxl.pc.in --- a/third_party/jpeg-xl/lib/jxl/libjxl.pc.in +++ b/third_party/jpeg-xl/lib/jxl/libjxl.pc.in @@ -1,9 +1,9 @@ prefix=@CMAKE_INSTALL_PREFIX@ exec_prefix=${prefix} -libdir=${exec_prefix}/@CMAKE_INSTALL_LIBDIR@ -includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ +libdir=@PKGCONFIG_TARGET_LIBS@ +includedir=@PKGCONFIG_TARGET_INCLUDES@ Name: libjxl Description: Loads and saves JPEG XL files Version: @JPEGXL_LIBRARY_VERSION@ Requires.private: @JPEGXL_LIBRARY_REQUIRES@ diff --git a/third_party/jpeg-xl/lib/jxl/modular/transform/enc_palette.cc b/third_party/jpeg-xl/lib/jxl/modular/transform/enc_palette.cc --- a/third_party/jpeg-xl/lib/jxl/modular/transform/enc_palette.cc +++ b/third_party/jpeg-xl/lib/jxl/modular/transform/enc_palette.cc @@ -177,11 +177,47 @@ pixel_type minval, maxval; compute_minmax(input.channel[begin_c], &minval, &maxval); size_t lookup_table_size = static_cast(maxval) - static_cast(minval) + 1; if (lookup_table_size > palette_internal::kMaxPaletteLookupTableSize) { - return false; // too large lookup table + // a lookup table would use too much memory, instead use a slower approach + // with std::set + std::set chpalette; + pixel_type idx = 0; + for (size_t y = 0; y < h; y++) { + const pixel_type *p = input.channel[begin_c].Row(y); + for (size_t x = 0; x < w; x++) { + const bool new_color = chpalette.insert(p[x]).second; + if (new_color) { + idx++; + if (idx > (int)nb_colors) return false; + } + } + } + JXL_DEBUG_V(6, "Channel %i uses only %i colors.", begin_c, idx); + Channel pch(idx, 1); + pch.hshift = -1; + nb_colors = idx; + idx = 0; + pixel_type *JXL_RESTRICT p_palette = pch.Row(0); + for (pixel_type p : chpalette) { + p_palette[idx++] = p; + } + for (size_t y = 0; y < h; y++) { + pixel_type *p = input.channel[begin_c].Row(y); + for (size_t x = 0; x < w; x++) { + for (idx = 0; p[x] != p_palette[idx] && idx < (int)nb_colors; idx++) { + } + JXL_DASSERT(idx < (int)nb_colors); + p[x] = idx; + } + } + predictor = Predictor::Zero; + input.nb_meta_channels++; + input.channel.insert(input.channel.begin(), std::move(pch)); + + return true; } lookup.resize(lookup_table_size, 0); pixel_type idx = 0; for (size_t y = 0; y < h; y++) { const pixel_type *p = input.channel[begin_c].Row(y); diff --git a/third_party/jpeg-xl/lib/jxl/modular/transform/palette.h b/third_party/jpeg-xl/lib/jxl/modular/transform/palette.h --- a/third_party/jpeg-xl/lib/jxl/modular/transform/palette.h +++ b/third_party/jpeg-xl/lib/jxl/modular/transform/palette.h @@ -4,10 +4,12 @@ // license that can be found in the LICENSE file. #ifndef LIB_JXL_MODULAR_TRANSFORM_PALETTE_H_ #define LIB_JXL_MODULAR_TRANSFORM_PALETTE_H_ +#include + #include "lib/jxl/base/data_parallel.h" #include "lib/jxl/base/status.h" #include "lib/jxl/common.h" #include "lib/jxl/modular/encoding/context_predict.h" #include "lib/jxl/modular/modular_image.h" @@ -82,11 +84,11 @@ index %= 1 + 2 * (kDeltaPalette.size() - 1); static constexpr int kMultiplier[] = {-1, 1}; pixel_type result = kDeltaPalette[((index + 1) >> 1)][c] * kMultiplier[index & 1]; if (bit_depth > 8) { - result *= static_cast(1) << (bit_depth - 8); + result *= static_cast(1) << (std::min(bit_depth, 24) - 8); } return result; } else if (palette_size <= index && index < palette_size + kLargeCubeOffset) { if (c >= kRgbChannels) return 0; index -= palette_size; diff --git a/third_party/jpeg-xl/lib/jxl/modular/transform/transform.cc b/third_party/jpeg-xl/lib/jxl/modular/transform/transform.cc --- a/third_party/jpeg-xl/lib/jxl/modular/transform/transform.cc +++ b/third_party/jpeg-xl/lib/jxl/modular/transform/transform.cc @@ -66,13 +66,10 @@ case TransformId::kPalette: JXL_DEBUG_V(2, "Transform: kPalette, begin_c=%" PRIu32 ", num_c=%" PRIu32 ", nb_colors=%" PRIu32 ", nb_deltas=%" PRIu32, begin_c, num_c, nb_colors, nb_deltas); - if (input.bitdepth > 24) { - return JXL_FAILURE("Palette is not allowed for bitdepth > 24"); - } return MetaPalette(input, begin_c, begin_c + num_c - 1, nb_colors, nb_deltas, lossy_palette); default: return JXL_FAILURE("Unknown transformation (ID=%u)", static_cast(id)); diff --git a/third_party/jpeg-xl/lib/jxl/modular_test.cc b/third_party/jpeg-xl/lib/jxl/modular_test.cc --- a/third_party/jpeg-xl/lib/jxl/modular_test.cc +++ b/third_party/jpeg-xl/lib/jxl/modular_test.cc @@ -43,12 +43,11 @@ namespace { using test::Roundtrip; void TestLosslessGroups(size_t group_size_shift) { ThreadPool* pool = nullptr; - const PaddedBytes orig = - ReadTestData("third_party/imagecompression.info/flower_foveon.png"); + const PaddedBytes orig = ReadTestData("jxl/flower/flower.png"); CompressParams cparams; cparams.SetLossless(); cparams.modular_group_size_shift = group_size_shift; DecompressParams dparams; diff --git a/third_party/jpeg-xl/lib/jxl/passes_test.cc b/third_party/jpeg-xl/lib/jxl/passes_test.cc --- a/third_party/jpeg-xl/lib/jxl/passes_test.cc +++ b/third_party/jpeg-xl/lib/jxl/passes_test.cc @@ -74,12 +74,11 @@ IsSlightlyBelow(1.6)); } TEST(PassesTest, RoundtripMultiGroupPasses) { ThreadPoolInternal pool(4); - const PaddedBytes orig = - ReadTestData("third_party/imagecompression.info/flower_foveon.png"); + const PaddedBytes orig = ReadTestData("jxl/flower/flower.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); io.ShrinkTo(600, 1024); // partial X, full Y group CompressParams cparams; @@ -89,24 +88,23 @@ cparams.progressive_mode = true; CodecInOut io2; Roundtrip(&io, cparams, dparams, &pool, &io2); EXPECT_THAT(ButteraugliDistance(io, io2, cparams.ba_params, GetJxlCms(), /*distmap=*/nullptr, &pool), - IsSlightlyBelow(1.2f)); + IsSlightlyBelow(1.3f)); cparams.butteraugli_distance = 2.0f; CodecInOut io3; Roundtrip(&io, cparams, dparams, &pool, &io3); EXPECT_THAT(ButteraugliDistance(io, io3, cparams.ba_params, GetJxlCms(), /*distmap=*/nullptr, &pool), - IsSlightlyBelow(2.0f)); + IsSlightlyBelow(2.3f)); } TEST(PassesTest, RoundtripLargeFastPasses) { ThreadPoolInternal pool(8); - const PaddedBytes orig = - ReadTestData("third_party/imagecompression.info/flower_foveon.png"); + const PaddedBytes orig = ReadTestData("jxl/flower/flower.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); CompressParams cparams; cparams.speed_tier = SpeedTier::kSquirrel; @@ -120,12 +118,11 @@ // Checks for differing size/distance in two consecutive runs of distance 2, // which involves additional processing including adaptive reconstruction. // Failing this may be a sign of race conditions or invalid memory accesses. TEST(PassesTest, RoundtripProgressiveConsistent) { ThreadPoolInternal pool(8); - const PaddedBytes orig = - ReadTestData("third_party/imagecompression.info/flower_foveon.png"); + const PaddedBytes orig = ReadTestData("jxl/flower/flower.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); CompressParams cparams; cparams.speed_tier = SpeedTier::kSquirrel; @@ -299,12 +296,11 @@ EXPECT_GE(butteraugli_distance_down2_full, 1.0f); } TEST(PassesTest, ProgressiveDownsample2DegradesCorrectly) { ThreadPoolInternal pool(8); - const PaddedBytes orig = - ReadTestData("third_party/imagecompression.info/flower_foveon.png"); + const PaddedBytes orig = ReadTestData("jxl/flower/flower.png"); CodecInOut io_orig; ASSERT_TRUE(SetFromBytes(Span(orig), &io_orig, &pool)); Rect rect(0, 0, io_orig.xsize(), 128); // need 2 DC groups for the DC frame to actually be progressive. Image3F large(4242, rect.ysize()); @@ -346,12 +342,11 @@ EXPECT_GE(butteraugli_distance_down2_full, 1.0f); } TEST(PassesTest, NonProgressiveDCImage) { ThreadPoolInternal pool(8); - const PaddedBytes orig = - ReadTestData("third_party/imagecompression.info/flower_foveon.png"); + const PaddedBytes orig = ReadTestData("jxl/flower/flower.png"); CodecInOut io; ASSERT_TRUE(SetFromBytes(Span(orig), &io, &pool)); PaddedBytes compressed; AuxOut aux; diff --git a/third_party/jpeg-xl/lib/jxl/render_pipeline/low_memory_render_pipeline.cc b/third_party/jpeg-xl/lib/jxl/render_pipeline/low_memory_render_pipeline.cc --- a/third_party/jpeg-xl/lib/jxl/render_pipeline/low_memory_render_pipeline.cc +++ b/third_party/jpeg-xl/lib/jxl/render_pipeline/low_memory_render_pipeline.cc @@ -301,10 +301,21 @@ input_sizes[c] = {full_image_xsize_, full_image_ysize_}; } stages_[i]->SetInputSizes(input_sizes); } + // Check that kInput stages appear after the switch to image dimensions (if + // any). + for (size_t i = 0; i < stages_.size(); i++) { + for (size_t c = 0; c < shifts.size(); c++) { + if (stages_[i]->GetChannelMode(c) == RenderPipelineChannelMode::kInput) { + JXL_ASSERT(first_image_dim_stage_ == stages_.size() || + i >= first_image_dim_stage_); + } + } + } + anyc_.resize(stages_.size()); for (size_t i = 0; i < stages_.size(); i++) { for (size_t c = 0; c < shifts.size(); c++) { if (stages_[i]->GetChannelMode(c) != RenderPipelineChannelMode::kIgnored) { @@ -476,76 +487,135 @@ } } } } +// Information about where the *output* of each stage is stored. +class Rows { + public: + Rows(const std::vector>& stages, + const Rect data_max_color_channel_rect, int group_data_x_border, + int group_data_y_border, + const std::vector>& group_data_shift, + size_t base_color_shift, std::vector>& thread_data, + std::vector& input_data) { + size_t num_stages = stages.size(); + size_t num_channels = input_data.size(); + + JXL_ASSERT(thread_data.size() == num_channels); + JXL_ASSERT(group_data_shift.size() == num_channels); + +#if JXL_ENABLE_ASSERT + for (const auto& td : thread_data) { + JXL_ASSERT(td.size() == num_stages); + } +#endif + + rows_.resize(num_stages + 1, std::vector(num_channels)); + + for (size_t i = 0; i < num_stages; i++) { + for (size_t c = 0; c < input_data.size(); c++) { + if (stages[i]->GetChannelMode(c) == RenderPipelineChannelMode::kInOut) { + rows_[i + 1][c].ymod_minus_1 = thread_data[c][i].ysize() - 1; + rows_[i + 1][c].base_ptr = thread_data[c][i].Row(0); + rows_[i + 1][c].stride = thread_data[c][i].PixelsPerRow(); + } + } + } + + for (size_t c = 0; c < input_data.size(); c++) { + auto channel_group_data_rect = + data_max_color_channel_rect.As() + .Translate(-group_data_x_border, -group_data_y_border) + .ShiftLeft(base_color_shift) + .CeilShiftRight(group_data_shift[c]) + .Translate(group_data_x_border - ssize_t(kRenderPipelineXOffset), + group_data_y_border); + rows_[0][c].base_ptr = channel_group_data_rect.Row(&input_data[c], 0); + rows_[0][c].stride = input_data[c].PixelsPerRow(); + rows_[0][c].ymod_minus_1 = -1; + } + } + + // Stage -1 refers to the input data; all other values must be nonnegative and + // refer to the data for the output of that stage. + JXL_INLINE float* GetBuffer(int stage, int y, size_t c) const { + JXL_DASSERT(stage >= -1); + const RowInfo& info = rows_[stage + 1][c]; + return info.base_ptr + ssize_t(info.stride) * (y & info.ymod_minus_1); + } + + private: + struct RowInfo { + // Pointer to beginning of the first row. + float* base_ptr; + // Modulo value for the y axis minus 1 (ymod is guaranteed to be a power of + // 2, which allows efficient mod computation by masking). + int ymod_minus_1; + // Number of floats per row. + size_t stride; + }; + std::vector> rows_; +}; + } // namespace void LowMemoryRenderPipeline::RenderRect(size_t thread_id, std::vector& input_data, Rect data_max_color_channel_rect, Rect image_max_color_channel_rect) { // For each stage, the rect corresponding to the image area currently being - // processed. + // processed, in the coordinates of that stage (i.e. with the scaling factor + // that that stage has). std::vector group_rect; group_rect.resize(stages_.size()); + Rect image_area_rect = + image_max_color_channel_rect.ShiftLeft(base_color_shift_) + .Crop(frame_dimensions_.xsize_upsampled, + frame_dimensions_.ysize_upsampled); for (size_t i = 0; i < stages_.size(); i++) { - size_t x0 = (image_max_color_channel_rect.x0() << base_color_shift_) >> - channel_shifts_[i][anyc_[i]].first; - size_t x1 = DivCeil(std::min(frame_dimensions_.xsize_upsampled, - (image_max_color_channel_rect.x0() + - image_max_color_channel_rect.xsize()) - << base_color_shift_), - 1 << channel_shifts_[i][anyc_[i]].first); - size_t y0 = (image_max_color_channel_rect.y0() << base_color_shift_) >> - channel_shifts_[i][anyc_[i]].second; - size_t y1 = DivCeil(std::min(frame_dimensions_.ysize_upsampled, - (image_max_color_channel_rect.y0() + - image_max_color_channel_rect.ysize()) - << base_color_shift_), - 1 << channel_shifts_[i][anyc_[i]].second); - group_rect[i] = Rect(x0, y0, x1 - x0, y1 - y0); + group_rect[i] = + image_area_rect.CeilShiftRight(channel_shifts_[i][anyc_[i]]); } - struct RowInfo { - float* base_ptr; - int ymod_minus_1; - size_t stride; - }; - - std::vector> rows( - stages_.size() + 1, std::vector(input_data.size())); - - for (size_t i = 0; i < stages_.size(); i++) { - for (size_t c = 0; c < input_data.size(); c++) { - if (stages_[i]->GetChannelMode(c) == RenderPipelineChannelMode::kInOut) { - rows[i + 1][c].ymod_minus_1 = stage_data_[thread_id][c][i].ysize() - 1; - rows[i + 1][c].base_ptr = stage_data_[thread_id][c][i].Row(0); - rows[i + 1][c].stride = stage_data_[thread_id][c][i].PixelsPerRow(); - } - } - } + ssize_t frame_x0 = + first_image_dim_stage_ == stages_.size() ? 0 : frame_origin_.x0; + ssize_t frame_y0 = + first_image_dim_stage_ == stages_.size() ? 0 : frame_origin_.y0; + size_t full_image_xsize = first_image_dim_stage_ == stages_.size() + ? frame_dimensions_.xsize_upsampled + : full_image_xsize_; + size_t full_image_ysize = first_image_dim_stage_ == stages_.size() + ? frame_dimensions_.ysize_upsampled + : full_image_ysize_; - for (size_t c = 0; c < input_data.size(); c++) { - int xoff = int(data_max_color_channel_rect.x0()) - - int(LowMemoryRenderPipeline::group_data_x_border_); - xoff = xoff * (1 << base_color_shift_) >> channel_shifts_[0][c].first; - xoff += LowMemoryRenderPipeline::group_data_x_border_; - int yoff = int(data_max_color_channel_rect.y0()) - - int(LowMemoryRenderPipeline::group_data_y_border_); - yoff = yoff * (1 << base_color_shift_) >> channel_shifts_[0][c].second; - yoff += LowMemoryRenderPipeline::group_data_y_border_; - rows[0][c].base_ptr = - input_data[c].Row(yoff) + xoff - kRenderPipelineXOffset; - rows[0][c].stride = input_data[c].PixelsPerRow(); - rows[0][c].ymod_minus_1 = -1; + // Compute actual x-axis bounds for the current image area in the context of + // the full image this frame is part of. As the left boundary may be negative, + // we also create the x_pixels_skip value, defined as follows: + // - both x_pixels_skip and full_image_x0 are >= 0, and at least one is 0; + // - full_image_x0 - x_pixels_skip is the position of the current frame area + // in the full image. + ssize_t full_image_x0 = frame_x0 + image_area_rect.x0(); + ssize_t x_pixels_skip = 0; + if (full_image_x0 < 0) { + x_pixels_skip = -full_image_x0; + full_image_x0 = 0; } + ssize_t full_image_x1 = frame_x0 + image_area_rect.x1(); + full_image_x1 = std::min(full_image_x1, full_image_xsize); - auto get_row_buffer = [&](int stage, int y, size_t c) { - const RowInfo& info = rows[stage + 1][c]; - return info.base_ptr + ssize_t(info.stride) * (y & info.ymod_minus_1); - }; + // If the current image area is entirely outside of the visible image, there + // is no point in proceeding. Note: this uses the assumption that if there is + // a stage with observable effects (i.e. a kInput stage), it only appears + // after the stage that switches to image dimensions. + if (full_image_x1 <= full_image_x0) return; + + // Data structures to hold information about input/output rows and their + // buffers. + Rows rows(stages_, data_max_color_channel_rect, group_data_x_border_, + group_data_y_border_, channel_shifts_[0], base_color_shift_, + stage_data_[thread_id], input_data); std::vector input_rows(first_trailing_stage_ + 1); for (size_t i = 0; i < first_trailing_stage_; i++) { input_rows[i].resize(input_data.size()); @@ -555,133 +625,133 @@ // Maximum possible shift is 3. RenderPipelineStage::RowInfo output_rows(input_data.size(), std::vector(8)); + // Fills in input_rows and output_rows for a given y value (relative to the + // start of the group, measured in actual pixels at the appropriate vertical + // scaling factor) and a given stage, applying mirroring if necessary. This + // function is somewhat inefficient for trailing kInOut or kInput stages, + // where just filling the input row once ought to be sufficient. + auto prepare_io_rows = [&](int y, size_t i) { + ssize_t bordery = stages_[i]->settings_.border_y; + size_t shifty = stages_[i]->settings_.shift_y; + auto make_row = [&](size_t c, ssize_t iy) { + size_t mirrored_y = GetMirroredY(y + iy - bordery, group_rect[i].y0(), + image_rect_[i].ysize()); + input_rows[i][c][iy] = + rows.GetBuffer(stage_input_for_channel_[i][c], mirrored_y, c); + ApplyXMirroring(input_rows[i][c][iy], stages_[i]->settings_.border_x, + group_rect[i].x0(), group_rect[i].xsize(), + image_rect_[i].xsize()); + }; + for (size_t c = 0; c < input_data.size(); c++) { + RenderPipelineChannelMode mode = stages_[i]->GetChannelMode(c); + if (mode == RenderPipelineChannelMode::kIgnored) { + continue; + } + // If we already have rows from a previous iteration, we can just shift + // the rows by 1 and insert the new one. + if (input_rows[i][c].size() == 2 * size_t(bordery) + 1) { + for (ssize_t iy = 0; iy < 2 * bordery; iy++) { + input_rows[i][c][iy] = input_rows[i][c][iy + 1]; + } + make_row(c, bordery * 2); + } else { + input_rows[i][c].resize(2 * bordery + 1); + for (ssize_t iy = 0; iy < 2 * bordery + 1; iy++) { + make_row(c, iy); + } + } + + // If necessary, get the output buffers. + if (mode == RenderPipelineChannelMode::kInOut) { + for (size_t iy = 0; iy < (1u << shifty); iy++) { + output_rows[c][iy] = rows.GetBuffer(i, y * (1 << shifty) + iy, c); + } + } + } + }; + // We pretend that every stage has a vertical shift of 0, i.e. it is as tall // as the final image. // We call each such row a "virtual" row, because it may or may not correspond // to an actual row of the current processing stage; actual processing happens // when vy % (1<settings_.border_y; - size_t shifty = stages_[i]->settings_.shift_y; int y = stage_vy >> channel_shifts_[i][anyc_[i]].second; ssize_t image_y = ssize_t(group_rect[i].y0()) + y; // Do not produce rows in out-of-bounds areas. if (image_y < 0 || image_y >= ssize_t(image_rect_[i].ysize())) { continue; } - // Get the actual input rows and potentially apply mirroring. - for (size_t c = 0; c < input_data.size(); c++) { - RenderPipelineChannelMode mode = stages_[i]->GetChannelMode(c); - if (mode == RenderPipelineChannelMode::kIgnored) { - continue; - } - auto make_row = [&](ssize_t iy) { - size_t mirrored_y = GetMirroredY(y + iy - bordery, group_rect[i].y0(), - image_rect_[i].ysize()); - input_rows[i][c][iy] = - get_row_buffer(stage_input_for_channel_[i][c], mirrored_y, c); - ApplyXMirroring(input_rows[i][c][iy], stages_[i]->settings_.border_x, - group_rect[i].x0(), group_rect[i].xsize(), - image_rect_[i].xsize()); - }; - // If we already have rows from a previous iteration, we can just shift - // the rows by 1 and insert the new one. - if (input_rows[i][c].size() == 2 * size_t(bordery) + 1) { - for (ssize_t iy = 0; iy < 2 * bordery; iy++) { - input_rows[i][c][iy] = input_rows[i][c][iy + 1]; - } - make_row(bordery * 2); - } else { - input_rows[i][c].resize(2 * bordery + 1); - for (ssize_t iy = 0; iy < 2 * bordery + 1; iy++) { - make_row(iy); - } - } + // Get the input/output rows and potentially apply mirroring to the input. + prepare_io_rows(y, i); - // If necessary, get the output buffers. - if (mode == RenderPipelineChannelMode::kInOut) { - for (size_t iy = 0; iy < (1u << shifty); iy++) { - output_rows[c][iy] = get_row_buffer(i, y * (1 << shifty) + iy, c); - } - } - } // Produce output rows. stages_[i]->ProcessRow(input_rows[i], output_rows, xpadding_for_output_[i], group_rect[i].xsize(), - group_rect[i].x0(), group_rect[i].y0() + y, - thread_id); + group_rect[i].x0(), image_y, thread_id); } // Process trailing stages, i.e. the final set of non-kInOut stages; they // all have the same input buffer and no need to use any mirroring. int y = vy - num_extra_rows; - if (y < 0 || y >= ssize_t(frame_dimensions_.ysize_upsampled)) continue; for (size_t c = 0; c < input_data.size(); c++) { - input_rows[first_trailing_stage_][c][0] = get_row_buffer( - stage_input_for_channel_[first_trailing_stage_][c], y, c); + // Skip pixels that are not part of the actual final image area. + input_rows[first_trailing_stage_][c][0] = + rows.GetBuffer(stage_input_for_channel_[first_trailing_stage_][c], y, + c) + + x_pixels_skip; } - for (size_t i = first_trailing_stage_; i < first_image_dim_stage_; i++) { - stages_[i]->ProcessRow(input_rows[first_trailing_stage_], output_rows, - /*xextra=*/0, group_rect[i].xsize(), - group_rect[i].x0(), group_rect[i].y0() + y, - thread_id); - } - - if (first_image_dim_stage_ == stages_.size()) continue; - - ssize_t full_image_y = - y + frame_origin_.y0 + group_rect[first_image_dim_stage_].y0(); - if (full_image_y < 0 || full_image_y >= ssize_t(full_image_ysize_)) { + // Check that we are not outside of the bounds for the current rendering + // rect. Not doing so might result in overwriting some rows that have been + // written (or will be written) by other threads. + if (y < 0 || y >= ssize_t(image_area_rect.ysize())) { continue; } - ssize_t full_image_x0 = - frame_origin_.x0 + group_rect[first_image_dim_stage_].x0(); - - if (full_image_x0 < 0) { - // Skip pixels. - for (size_t c = 0; c < input_data.size(); c++) { - input_rows[first_trailing_stage_][c][0] -= full_image_x0; - } - full_image_x0 = 0; + // Avoid running pipeline stages on pixels that are outside the full image + // area. As trailing stages have no borders, this is a free optimization + // (and may be necessary for correctness, as some stages assume coordinates + // are within bounds). + ssize_t full_image_y = frame_y0 + image_area_rect.y0() + y; + if (full_image_y < 0 || full_image_y >= ssize_t(full_image_ysize)) { + continue; } - ssize_t full_image_x1 = frame_origin_.x0 + - group_rect[first_image_dim_stage_].x0() + - group_rect[first_image_dim_stage_].xsize(); - full_image_x1 = std::min(full_image_x1, full_image_xsize_); - if (full_image_x1 <= full_image_x0) continue; - - for (size_t i = first_image_dim_stage_; i < stages_.size(); i++) { + for (size_t i = first_trailing_stage_; i < stages_.size(); i++) { + // Before the first_image_dim_stage_, coordinates are relative to the + // current frame. + size_t x0 = + i < first_image_dim_stage_ ? full_image_x0 - frame_x0 : full_image_x0; + size_t y = + i < first_image_dim_stage_ ? full_image_y - frame_y0 : full_image_y; stages_[i]->ProcessRow(input_rows[first_trailing_stage_], output_rows, - /*xextra=*/0, full_image_x1 - full_image_x0, - full_image_x0, full_image_y, thread_id); + /*xextra=*/0, full_image_x1 - full_image_x0, x0, y, + thread_id); } } } void LowMemoryRenderPipeline::RenderPadding(size_t thread_id, Rect rect) { diff --git a/third_party/jpeg-xl/lib/jxl/render_pipeline/render_pipeline_stage.h b/third_party/jpeg-xl/lib/jxl/render_pipeline/render_pipeline_stage.h --- a/third_party/jpeg-xl/lib/jxl/render_pipeline/render_pipeline_stage.h +++ b/third_party/jpeg-xl/lib/jxl/render_pipeline/render_pipeline_stage.h @@ -29,11 +29,14 @@ kIgnored = 0, // This channel is modified in-place. kInPlace = 1, // This channel is modified and written to a new buffer. kInOut = 2, - // This channel is only read. + // This channel is only read. These are the only stages that are assumed to + // have observable effects, i.e. calls to ProcessRow for other stages may be + // omitted if it can be shown they can't affect any kInput stage ProcessRow + // call that happens inside image boundaries. kInput = 3, }; class RenderPipeline; @@ -98,10 +101,14 @@ // of floats; concurrent calls will have different buffers. virtual void ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows, size_t xextra, size_t xsize, size_t xpos, size_t ypos, size_t thread_id) const = 0; + // How each channel will be processed. Channels are numbered starting from + // color channels (always 3) and followed by all other channels. + virtual RenderPipelineChannelMode GetChannelMode(size_t c) const = 0; + protected: explicit RenderPipelineStage(Settings settings) : settings_(settings) {} virtual Status IsInitialized() const { return true; } @@ -110,14 +117,10 @@ virtual void SetInputSizes( const std::vector>& input_sizes) {} virtual Status PrepareForThreads(size_t num_threads) { return true; } - // How each channel will be processed. Channels are numbered starting from - // color channels (always 3) and followed by all other channels. - virtual RenderPipelineChannelMode GetChannelMode(size_t c) const = 0; - // Returns a pointer to the input row of channel `c` with offset `y`. // `y` must be in [-settings_.border_y, settings_.border_y]. `c` must be such // that `GetChannelMode(c) != kIgnored`. The returned pointer points to the // offset-ed row (i.e. kRenderPipelineXOffset has been applied). float* GetInputRow(const RowInfo& input_rows, size_t c, int offset) const { @@ -138,10 +141,12 @@ } // Indicates whether, from this stage on, the pipeline will operate on an // image- rather than frame-sized buffer. Only one stage in the pipeline // should return true, and it should implement ProcessPaddingRow below too. + // It is assumed that, if there is a SwitchToImageDimensions() == true stage, + // all kInput stages appear after it. virtual bool SwitchToImageDimensions() const { return false; } // If SwitchToImageDimensions returns true, then this should set xsize and // ysize to the image size, and frame_origin to the location of the frame // within the image. Otherwise, this is not called at all. diff --git a/third_party/jpeg-xl/lib/jxl/render_pipeline/render_pipeline_test.cc b/third_party/jpeg-xl/lib/jxl/render_pipeline/render_pipeline_test.cc --- a/third_party/jpeg-xl/lib/jxl/render_pipeline/render_pipeline_test.cc +++ b/third_party/jpeg-xl/lib/jxl/render_pipeline/render_pipeline_test.cc @@ -220,11 +220,11 @@ {3, 8}, {128, 128}, {256, 256}, {258, 258}, {533, 401}, {777, 777}, }; for (auto size : sizes) { RenderPipelineTestInputSettings settings; - settings.input_path = "third_party/imagecompression.info/flower_foveon.png"; + settings.input_path = "jxl/flower/flower.png"; settings.xsize = size.first; settings.ysize = size.second; // Base settings. settings.cparams.butteraugli_distance = 1.0; @@ -352,40 +352,36 @@ all_tests.push_back(s); } { auto s = settings; - s.input_path = - "third_party/imagecompression.info/flower_foveon_alpha.png"; + s.input_path = "jxl/flower/flower_alpha.png"; s.cparams_descr = "AlphaVarDCT"; all_tests.push_back(s); } { auto s = settings; - s.input_path = - "third_party/imagecompression.info/flower_foveon_alpha.png"; + s.input_path = "jxl/flower/flower_alpha.png"; s.cparams_descr = "AlphaVarDCTUpsamplingEPF"; s.cparams.epf = 1; s.cparams.ec_resampling = 2; all_tests.push_back(s); } { auto s = settings; s.cparams.modular_mode = true; s.cparams.butteraugli_distance = 0; - s.input_path = - "third_party/imagecompression.info/flower_foveon_alpha.png"; + s.input_path = "jxl/flower/flower_alpha.png"; s.cparams_descr = "AlphaLossless"; all_tests.push_back(s); } { auto s = settings; - s.input_path = - "third_party/imagecompression.info/flower_foveon_alpha.png"; + s.input_path = "jxl/flower/flower_alpha.png"; s.cparams_descr = "AlphaDownsample"; s.cparams.ec_resampling = 2; all_tests.push_back(s); } @@ -396,15 +392,14 @@ all_tests.push_back(s); } } #if JPEGXL_ENABLE_TRANSCODE_JPEG - for (const char* input : - {"third_party/imagecompression.info/flower_foveon.png.im_q85_444.jpg", - "third_party/imagecompression.info/flower_foveon.png.im_q85_420.jpg", - "third_party/imagecompression.info/flower_foveon.png.im_q85_422.jpg", - "third_party/imagecompression.info/flower_foveon.png.im_q85_440.jpg"}) { + for (const char* input : {"jxl/flower/flower.png.im_q85_444.jpg", + "jxl/flower/flower.png.im_q85_420.jpg", + "jxl/flower/flower.png.im_q85_422.jpg", + "jxl/flower/flower.png.im_q85_440.jpg"}) { RenderPipelineTestInputSettings settings; settings.input_path = input; settings.jpeg_transcode = true; settings.xsize = 2268; settings.ysize = 1512; @@ -421,10 +416,30 @@ settings.ysize = 277; settings.cparams_descr = "Patches"; all_tests.push_back(settings); } + { + RenderPipelineTestInputSettings settings; + settings.input_path = "jxl/grayscale_patches.png"; + settings.xsize = 1011; + settings.ysize = 277; + settings.cparams.photon_noise_iso = 1000; + settings.cparams_descr = "PatchesAndNoise"; + all_tests.push_back(settings); + } + + { + RenderPipelineTestInputSettings settings; + settings.input_path = "jxl/grayscale_patches.png"; + settings.xsize = 1011; + settings.ysize = 277; + settings.cparams.resampling = 2; + settings.cparams_descr = "PatchesAndUps2"; + all_tests.push_back(settings); + } + return all_tests; } std::ostream& operator<<(std::ostream& os, const RenderPipelineTestInputSettings& c) { diff --git a/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_patches.h b/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_patches.h --- a/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_patches.h +++ b/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_patches.h @@ -13,10 +13,10 @@ namespace jxl { // Draws patches if applicable. std::unique_ptr GetPatchesStage( - const PatchDictionary* patches); + const PatchDictionary* patches, size_t num_channels); } // namespace jxl #endif // LIB_JXL_RENDER_PIPELINE_STAGE_PATCHES_H_ diff --git a/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_patches.cc b/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_patches.cc --- a/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_patches.cc +++ b/third_party/jpeg-xl/lib/jxl/render_pipeline/stage_patches.cc @@ -7,38 +7,42 @@ namespace jxl { namespace { class PatchDictionaryStage : public RenderPipelineStage { public: - explicit PatchDictionaryStage(const PatchDictionary* patches) + PatchDictionaryStage(const PatchDictionary* patches, size_t num_channels) : RenderPipelineStage(RenderPipelineStage::Settings()), - patches_(*patches) {} + patches_(*patches), + num_channels_(num_channels) {} void ProcessRow(const RowInfo& input_rows, const RowInfo& output_rows, size_t xextra, size_t xsize, size_t xpos, size_t ypos, size_t thread_id) const final { PROFILER_ZONE("RenderPatches"); - std::vector row_ptrs(input_rows.size()); - for (size_t i = 0; i < row_ptrs.size(); i++) { - row_ptrs[i] = GetInputRow(input_rows, i, 0) - xextra; + JXL_ASSERT(xpos == 0 || xpos >= xextra); + size_t x0 = xpos ? xpos - xextra : 0; + std::vector row_ptrs(num_channels_); + for (size_t i = 0; i < num_channels_; i++) { + row_ptrs[i] = GetInputRow(input_rows, i, 0) + x0 - xpos; } - patches_.AddOneRow(row_ptrs.data(), ypos, xpos - xextra, - xsize + 2 * xextra); + patches_.AddOneRow(row_ptrs.data(), ypos, x0, xsize + xextra + xpos - x0); } RenderPipelineChannelMode GetChannelMode(size_t c) const final { - return RenderPipelineChannelMode::kInPlace; + return c < num_channels_ ? RenderPipelineChannelMode::kInPlace + : RenderPipelineChannelMode::kIgnored; } const char* GetName() const override { return "Patches"; } private: const PatchDictionary& patches_; + const size_t num_channels_; }; } // namespace std::unique_ptr GetPatchesStage( - const PatchDictionary* patches) { - return jxl::make_unique(patches); + const PatchDictionary* patches, size_t num_channels) { + return jxl::make_unique(patches, num_channels); } } // namespace jxl diff --git a/third_party/jpeg-xl/lib/jxl/roundtrip_test.cc b/third_party/jpeg-xl/lib/jxl/roundtrip_test.cc --- a/third_party/jpeg-xl/lib/jxl/roundtrip_test.cc +++ b/third_party/jpeg-xl/lib/jxl/roundtrip_test.cc @@ -798,12 +798,11 @@ JxlDecoderDestroy(dec); } #if JPEGXL_ENABLE_JPEG // Loading .jpg files requires libjpeg support. TEST(RoundtripTest, JXL_TRANSCODE_JPEG_TEST(TestJPEGReconstruction)) { - const std::string jpeg_path = - "third_party/imagecompression.info/flower_foveon.png.im_q85_420.jpg"; + const std::string jpeg_path = "jxl/flower/flower.png.im_q85_420.jpg"; const jxl::PaddedBytes orig = jxl::ReadTestData(jpeg_path); jxl::CodecInOut orig_io; ASSERT_TRUE( SetFromBytes(jxl::Span(orig), &orig_io, /*pool=*/nullptr)); diff --git a/third_party/jpeg-xl/lib/jxl/splines_test.cc b/third_party/jpeg-xl/lib/jxl/splines_test.cc --- a/third_party/jpeg-xl/lib/jxl/splines_test.cc +++ b/third_party/jpeg-xl/lib/jxl/splines_test.cc @@ -3,19 +3,18 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. #include "lib/jxl/splines.h" -#include "gmock/gmock.h" -#include "gtest/gtest.h" #include "lib/extras/codec.h" #include "lib/jxl/base/printf_macros.h" #include "lib/jxl/dec_file.h" #include "lib/jxl/enc_butteraugli_comparator.h" #include "lib/jxl/enc_color_management.h" #include "lib/jxl/enc_splines.h" #include "lib/jxl/image_test_utils.h" +#include "lib/jxl/test_utils.h" #include "lib/jxl/testdata.h" namespace jxl { std::ostream& operator<<(std::ostream& os, const Spline::Point& p) { diff --git a/third_party/jpeg-xl/lib/jxl/test_utils.h b/third_party/jpeg-xl/lib/jxl/test_utils.h --- a/third_party/jpeg-xl/lib/jxl/test_utils.h +++ b/third_party/jpeg-xl/lib/jxl/test_utils.h @@ -6,11 +6,18 @@ #ifndef LIB_JXL_TEST_UTILS_H_ #define LIB_JXL_TEST_UTILS_H_ // Macros and functions useful for tests. +// gmock unconditionally redefines those macros (to wrong values). +// Lets include it only here and mitigate the problem. +#pragma push_macro("PRIdS") +#pragma push_macro("PRIuS") #include "gmock/gmock.h" +#pragma pop_macro("PRIuS") +#pragma pop_macro("PRIdS") + #include "gtest/gtest.h" #include "jxl/codestream_header.h" #include "jxl/encode.h" #include "lib/jxl/aux_out_fwd.h" #include "lib/jxl/base/data_parallel.h" @@ -79,10 +86,15 @@ basic_info->exponent_bits_per_sample = 0; break; default: JXL_ABORT("Unhandled JxlDataType"); } + if (pixel_format->num_channels < 3) { + basic_info->num_color_channels = 1; + } else { + basic_info->num_color_channels = 3; + } if (pixel_format->num_channels == 2 || pixel_format->num_channels == 4) { basic_info->alpha_exponent_bits = basic_info->exponent_bits_per_sample; basic_info->alpha_bits = basic_info->bits_per_sample; basic_info->num_extra_channels = 1; } else { diff --git a/third_party/jpeg-xl/lib/jxl/testdata.h b/third_party/jpeg-xl/lib/jxl/testdata.h --- a/third_party/jpeg-xl/lib/jxl/testdata.h +++ b/third_party/jpeg-xl/lib/jxl/testdata.h @@ -17,43 +17,11 @@ namespace jxl { static inline PaddedBytes ReadTestData(const std::string& filename) { std::string full_path = std::string(TEST_DATA_PATH "/") + filename; PaddedBytes data; - bool ok = ReadFile(full_path, &data); -#ifdef __EMSCRIPTEN__ - // Fallback in case FS is not supported in current JS engine. - if (!ok) { - // {size_t size, uint8_t* bytes} pair. - uint32_t size_bytes[2] = {0, 0}; - EM_ASM( - { - let buffer = null; - try { - buffer = readbuffer(UTF8ToString($0)); - } catch { - } - if (!buffer) return; - let bytes = new Uint8Array(buffer); - let size = bytes.length; - let out = _malloc(size); - if (!out) return; - HEAP8.set(bytes, out); - HEAP32[$1 >> 2] = size; - HEAP32[($1 + 4) >> 2] = out; - }, - full_path.c_str(), size_bytes); - size_t size = size_bytes[0]; - uint8_t* bytes = reinterpret_cast(size_bytes[1]); - if (size) { - data.append(bytes, bytes + size); - free(reinterpret_cast(bytes)); - ok = true; - } - } -#endif - JXL_CHECK(ok); + JXL_CHECK(ReadFile(full_path, &data)); return data; } } // namespace jxl diff --git a/third_party/jpeg-xl/lib/jxl_extras.cmake b/third_party/jpeg-xl/lib/jxl_extras.cmake --- a/third_party/jpeg-xl/lib/jxl_extras.cmake +++ b/third_party/jpeg-xl/lib/jxl_extras.cmake @@ -14,10 +14,12 @@ extras/dec/decode.h extras/dec/pgx.cc extras/dec/pgx.h extras/dec/pnm.cc extras/dec/pnm.h + extras/enc/encode.cc + extras/enc/encode.h extras/enc/pgx.cc extras/enc/pgx.h extras/enc/pnm.cc extras/enc/pnm.h extras/hlg.cc diff --git a/third_party/jpeg-xl/lib/jxl_tests.cmake b/third_party/jpeg-xl/lib/jxl_tests.cmake --- a/third_party/jpeg-xl/lib/jxl_tests.cmake +++ b/third_party/jpeg-xl/lib/jxl_tests.cmake @@ -103,10 +103,14 @@ set_target_properties(${TESTNAME} PROPERTIES LINK_FLAGS "\ -O1 \ -s USE_LIBPNG=1 \ -s TOTAL_MEMORY=1536MB \ -s SINGLE_FILE=1 \ + -s PROXY_TO_PTHREAD \ + -s EXIT_RUNTIME=1 \ + -s USE_PTHREADS=1 \ + -s NODERAWFS=1 \ ") endif() target_compile_options(${TESTNAME} PRIVATE ${JPEGXL_INTERNAL_FLAGS} # Add coverage flags to the test binary so code in the private headers of diff --git a/third_party/jpeg-xl/lib/jxl_threads.cmake b/third_party/jpeg-xl/lib/jxl_threads.cmake --- a/third_party/jpeg-xl/lib/jxl_threads.cmake +++ b/third_party/jpeg-xl/lib/jxl_threads.cmake @@ -107,10 +107,24 @@ endif() # TARGET_SUPPORTS_SHARED_LIBS AND NOT JPEGXL_STATIC AND # BUILD_SHARED_LIBS ### Add a pkg-config file for libjxl_threads. + +# Allow adding prefix if CMAKE_INSTALL_INCLUDEDIR not absolute. +if(IS_ABSOLUTE "${CMAKE_INSTALL_INCLUDEDIR}") + set(PKGCONFIG_TARGET_INCLUDES "${CMAKE_INSTALL_INCLUDEDIR}") +else() + set(PKGCONFIG_TARGET_INCLUDES "\${prefix}/${CMAKE_INSTALL_INCLUDEDIR}") +endif() +# Allow adding prefix if CMAKE_INSTALL_LIBDIR not absolute. +if(IS_ABSOLUTE "${CMAKE_INSTALL_LIBDIR}") + set(PKGCONFIG_TARGET_LIBS "${CMAKE_INSTALL_LIBDIR}") +else() + set(PKGCONFIG_TARGET_LIBS "\${exec_prefix}/${CMAKE_INSTALL_LIBDIR}") +endif() + set(JPEGXL_THREADS_LIBRARY_REQUIRES "") configure_file("${CMAKE_CURRENT_SOURCE_DIR}/threads/libjxl_threads.pc.in" "libjxl_threads.pc" @ONLY) install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libjxl_threads.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") diff --git a/third_party/jpeg-xl/lib/lib.gni b/third_party/jpeg-xl/lib/lib.gni --- a/third_party/jpeg-xl/lib/lib.gni +++ b/third_party/jpeg-xl/lib/lib.gni @@ -446,10 +446,12 @@ "extras/dec/decode.h", "extras/dec/pgx.cc", "extras/dec/pgx.h", "extras/dec/pnm.cc", "extras/dec/pnm.h", + "extras/enc/encode.cc", + "extras/enc/encode.h", "extras/enc/pgx.cc", "extras/enc/pgx.h", "extras/enc/pnm.cc", "extras/enc/pnm.h", "extras/hlg.cc", diff --git a/third_party/jpeg-xl/lib/threads/libjxl_threads.pc.in b/third_party/jpeg-xl/lib/threads/libjxl_threads.pc.in --- a/third_party/jpeg-xl/lib/threads/libjxl_threads.pc.in +++ b/third_party/jpeg-xl/lib/threads/libjxl_threads.pc.in @@ -1,9 +1,9 @@ prefix=@CMAKE_INSTALL_PREFIX@ exec_prefix=${prefix} -libdir=${exec_prefix}/@CMAKE_INSTALL_LIBDIR@ -includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ +libdir=@PKGCONFIG_TARGET_LIBS@ +includedir=@PKGCONFIG_TARGET_INCLUDES@ Name: libjxl_threads Description: JPEG XL multi-thread runner using std::threads. Version: @JPEGXL_LIBRARY_VERSION@ Requires.private: @JPEGXL_THREADS_LIBRARY_REQUIRES@ diff --git a/third_party/jpeg-xl/lib/threads/thread_parallel_runner_internal.cc b/third_party/jpeg-xl/lib/threads/thread_parallel_runner_internal.cc --- a/third_party/jpeg-xl/lib/threads/thread_parallel_runner_internal.cc +++ b/third_party/jpeg-xl/lib/threads/thread_parallel_runner_internal.cc @@ -171,18 +171,12 @@ } } } ThreadParallelRunner::ThreadParallelRunner(const int num_worker_threads) -#if defined(__EMSCRIPTEN__) - : num_worker_threads_(0), num_threads_(1) { - // TODO(eustas): find out if pthreads would work for us. - (void)num_worker_threads; -#else : num_worker_threads_(num_worker_threads), num_threads_(std::max(num_worker_threads, 1)) { -#endif PROFILER_ZONE("ThreadParallelRunner ctor"); threads_.reserve(num_worker_threads_); // Suppress "unused-private-field" warning. diff --git a/third_party/jpeg-xl/lib/threads/thread_parallel_runner_test.cc b/third_party/jpeg-xl/lib/threads/thread_parallel_runner_test.cc --- a/third_party/jpeg-xl/lib/threads/thread_parallel_runner_test.cc +++ b/third_party/jpeg-xl/lib/threads/thread_parallel_runner_test.cc @@ -1,10 +1,12 @@ // Copyright (c) the JPEG XL Project Authors. All rights reserved. // // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +#include + #include "gtest/gtest.h" #include "lib/jxl/base/data_parallel.h" #include "lib/jxl/base/thread_pool_internal.h" namespace jpegxl { diff --git a/third_party/jpeg-xl/plugins/gdk-pixbuf/CMakeLists.txt b/third_party/jpeg-xl/plugins/gdk-pixbuf/CMakeLists.txt --- a/third_party/jpeg-xl/plugins/gdk-pixbuf/CMakeLists.txt +++ b/third_party/jpeg-xl/plugins/gdk-pixbuf/CMakeLists.txt @@ -69,11 +69,11 @@ add_test( NAME pixbufloader_test_jxl COMMAND ${XVFB_PROGRAM_PREFIX} $ "${CMAKE_CURRENT_SOURCE_DIR}/loaders_test.cache" - "${CMAKE_SOURCE_DIR}/third_party/testdata/jxl/blending/cropped_traffic_light.jxl" + "${CMAKE_SOURCE_DIR}/testdata/jxl/blending/cropped_traffic_light.jxl" WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} ) set_tests_properties(pixbufloader_test_jxl PROPERTIES SKIP_RETURN_CODE 254) endif() endif() # Gdk_FOUND