diff --git a/media/libdav1d/asm/moz.build b/media/libdav1d/asm/moz.build --- a/media/libdav1d/asm/moz.build +++ b/media/libdav1d/asm/moz.build @@ -86,10 +86,11 @@ SOURCES += [ '../../../third_party/dav1d/src/x86/cdef16_avx2.asm', # moved from autovendored '../../../third_party/dav1d/src/x86/cdef_avx2.asm', '../../../third_party/dav1d/src/x86/cdef_avx512.asm', '../../../third_party/dav1d/src/x86/filmgrain16_avx2.asm', + '../../../third_party/dav1d/src/x86/filmgrain16_avx512.asm', '../../../third_party/dav1d/src/x86/filmgrain_avx2.asm', '../../../third_party/dav1d/src/x86/filmgrain_avx512.asm', # '../../../third_party/dav1d/src/x86/filmgrain_common.asm', # ^ filmgrain_common.asm must *not* be in SOURCES because it's only # used as an %include for other .asm files. Trying to assemble it diff --git a/media/libdav1d/moz.yaml b/media/libdav1d/moz.yaml --- a/media/libdav1d/moz.yaml +++ b/media/libdav1d/moz.yaml @@ -18,15 +18,15 @@ # Usually different from repository url url: https://code.videolan.org/videolan/dav1d # Human-readable identifier for this version/release # Generally "version NNN", "tag SSS", "bookmark SSS" - release: commit 28a9c46e1c36540d3276299f2e284ece1d2386be (2022-03-10T14:49:23.000+00:00). + release: 9833c92807715d5b6ca0f88d13081e25744d65e0 (2022-05-07T16:51:25.000+02:00). # Revision to pull in # Must be a long or short commit SHA (long preferred) - revision: 28a9c46e1c36540d3276299f2e284ece1d2386be + revision: 9833c92807715d5b6ca0f88d13081e25744d65e0 # The package's license, where possible using the mnemonic from # https://spdx.org/licenses/ # Multiple licenses can be specified (as a YAML list) # A "LICENSE" file must exist containing the full license text diff --git a/media/libdav1d/vcs_version.h b/media/libdav1d/vcs_version.h --- a/media/libdav1d/vcs_version.h +++ b/media/libdav1d/vcs_version.h @@ -1,2 +1,2 @@ /* auto-generated, do not edit */ -#define DAV1D_VERSION "@VCS_TAG@" +#define DAV1D_VERSION "9833c92807715d5b6ca0f88d13081e25744d65e0" diff --git a/media/libdav1d/version.h b/media/libdav1d/version.h --- a/media/libdav1d/version.h +++ b/media/libdav1d/version.h @@ -26,9 +26,9 @@ #ifndef DAV1D_VERSION_H #define DAV1D_VERSION_H #define DAV1D_API_VERSION_MAJOR 6 -#define DAV1D_API_VERSION_MINOR 6 +#define DAV1D_API_VERSION_MINOR 7 #define DAV1D_API_VERSION_PATCH 0 #endif /* DAV1D_VERSION_H */ diff --git a/third_party/dav1d/NEWS b/third_party/dav1d/NEWS --- a/third_party/dav1d/NEWS +++ b/third_party/dav1d/NEWS @@ -7,11 +7,12 @@ an automatic thread management. It also adds support for AVX-512 acceleration, and adds speedups to existing x86 code (from SSE2 to AVX2). -1.0.0 adds new grain API to ease acceleration on the GPU. +1.0.0 adds new grain API to ease acceleration on the GPU, and adds an API call +to get information of which frame failed to decode, in error cases. Finally, 1.0.0 fixes numerous small bugs that were reported since the beginning of the project to have a proper release. .''. diff --git a/third_party/dav1d/include/compat/gcc/stdatomic.h b/third_party/dav1d/include/compat/gcc/stdatomic.h --- a/third_party/dav1d/include/compat/gcc/stdatomic.h +++ b/third_party/dav1d/include/compat/gcc/stdatomic.h @@ -38,10 +38,11 @@ #define atomic_init(p_a, v) do { *(p_a) = (v); } while(0) #define atomic_store(p_a, v) __atomic_store_n(p_a, v, __ATOMIC_SEQ_CST) #define atomic_load(p_a) __atomic_load_n(p_a, __ATOMIC_SEQ_CST) #define atomic_load_explicit(p_a, mo) __atomic_load_n(p_a, mo) #define atomic_fetch_add(p_a, inc) __atomic_fetch_add(p_a, inc, __ATOMIC_SEQ_CST) +#define atomic_fetch_add_explicit(p_a, inc, mo) __atomic_fetch_add(p_a, inc, mo) #define atomic_fetch_sub(p_a, dec) __atomic_fetch_sub(p_a, dec, __ATOMIC_SEQ_CST) #define atomic_exchange(p_a, v) __atomic_exchange_n(p_a, v, __ATOMIC_SEQ_CST) #define atomic_fetch_or(p_a, v) __atomic_fetch_or(p_a, v, __ATOMIC_SEQ_CST) #endif /* !defined(__cplusplus) */ diff --git a/third_party/dav1d/include/compat/msvc/stdatomic.h b/third_party/dav1d/include/compat/msvc/stdatomic.h --- a/third_party/dav1d/include/compat/msvc/stdatomic.h +++ b/third_party/dav1d/include/compat/msvc/stdatomic.h @@ -60,10 +60,11 @@ * using InterlockedIncrement/InterlockedDecrement */ #define atomic_fetch_add(p_a, inc) InterlockedExchangeAdd(p_a, inc) #define atomic_fetch_sub(p_a, dec) InterlockedExchangeAdd(p_a, -(dec)) #define atomic_fetch_or(p_a, v) InterlockedOr(p_a, v) +#define atomic_fetch_add_explicit(p_a, inc, mo) atomic_fetch_add(p_a, inc) #endif /* ! stdatomic.h */ #pragma warning(pop) diff --git a/third_party/dav1d/include/dav1d/dav1d.h b/third_party/dav1d/include/dav1d/dav1d.h --- a/third_party/dav1d/include/dav1d/dav1d.h +++ b/third_party/dav1d/include/dav1d/dav1d.h @@ -285,10 +285,25 @@ * * @return 0 on success, or < 0 (a negative DAV1D_ERR code) on error. */ DAV1D_API int dav1d_get_decode_error_data_props(Dav1dContext *c, Dav1dDataProps *out); +/** + * Get the decoder delay, which is the number of internally buffered frames, not + * including reference frames. + * This value is guaranteed to be >= 1 and <= max_frame_delay. + * + * @param s Input settings context. + * + * @return Decoder frame delay on success, or < 0 (a negative DAV1D_ERR code) on + * error. + * + * @note The returned delay is valid only for a Dav1dContext initialized with the + * provided Dav1dSettings. + */ +DAV1D_API int dav1d_get_frame_delay(const Dav1dSettings *s); + # ifdef __cplusplus } # endif #endif /* DAV1D_H */ diff --git a/third_party/dav1d/meson.build b/third_party/dav1d/meson.build --- a/third_party/dav1d/meson.build +++ b/third_party/dav1d/meson.build @@ -21,18 +21,18 @@ # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. project('dav1d', ['c'], - version: '0.9.2', + version: '1.0.0', default_options: ['c_std=c99', 'warning_level=2', 'buildtype=release', 'b_ndebug=if-release'], meson_version: '>= 0.49.0') -dav1d_soname_version = '6.6.0' +dav1d_soname_version = '6.7.0' dav1d_api_version_array = dav1d_soname_version.split('.') dav1d_api_version_major = dav1d_api_version_array[0] dav1d_api_version_minor = dav1d_api_version_array[1] dav1d_api_version_revision = dav1d_api_version_array[2] diff --git a/third_party/dav1d/src/arm/32/film_grain.S b/third_party/dav1d/src/arm/32/film_grain.S deleted file mode 100644 --- a/third_party/dav1d/src/arm/32/film_grain.S +++ /dev/null @@ -1,2039 +0,0 @@ -/* - * Copyright © 2021, VideoLAN and dav1d authors - * Copyright © 2021, Martin Storsjo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include "src/arm/asm.S" -#include "util.S" -#include "src/arm/asm-offsets.h" - -#define GRAIN_WIDTH 82 -#define GRAIN_HEIGHT 73 - -#define SUB_GRAIN_WIDTH 44 -#define SUB_GRAIN_HEIGHT 38 - -.macro increment_seed steps, shift=1 - lsr r11, r2, #3 - lsr r12, r2, #12 - lsr lr, r2, #1 - eor r11, r2, r11 // (r >> 0) ^ (r >> 3) - eor r12, r12, lr // (r >> 12) ^ (r >> 1) - eor r11, r11, r12 // (r >> 0) ^ (r >> 3) ^ (r >> 12) ^ (r >> 1) -.if \shift - lsr r2, r2, #\steps -.endif - and r11, r11, #((1 << \steps) - 1) // bit -.if \shift - orr r2, r2, r11, lsl #(16 - \steps) // *state -.else - orr r2, r2, r11, lsl #16 // *state -.endif -.endm - -.macro read_rand dest, bits, age - ubfx \dest, r2, #16 - \bits - \age, #\bits -.endm - -.macro read_shift_rand dest, bits - ubfx \dest, r2, #17 - \bits, #\bits - lsr r2, r2, #1 -.endm - -// special calling convention: -// r2 holds seed -// r3 holds dav1d_gaussian_sequence -// clobbers r11-r12 -// returns in d0-d1 -function get_gaussian_neon - push {r5-r6,lr} - increment_seed 4 - read_rand r5, 11, 3 - read_rand r6, 11, 2 - add r5, r3, r5, lsl #1 - add r6, r3, r6, lsl #1 - vld1.16 {d0[0]}, [r5] - read_rand r5, 11, 1 - vld1.16 {d0[1]}, [r6] - add r5, r3, r5, lsl #1 - read_rand r6, 11, 0 - increment_seed 4 - add r6, r3, r6, lsl #1 - vld1.16 {d0[2]}, [r5] - read_rand r5, 11, 3 - vld1.16 {d0[3]}, [r6] - add r5, r3, r5, lsl #1 - read_rand r6, 11, 2 - vld1.16 {d1[0]}, [r5] - add r6, r3, r6, lsl #1 - read_rand r5, 11, 1 - vld1.16 {d1[1]}, [r6] - read_rand r6, 11, 0 - add r5, r3, r5, lsl #1 - add r6, r3, r6, lsl #1 - vld1.16 {d1[2]}, [r5] - vld1.16 {d1[3]}, [r6] - pop {r5-r6,pc} -endfunc - -.macro get_grain_row r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10 - bl get_gaussian_neon - vrshl.s16 q0, q0, q15 - vmovn.i16 \r0, q0 - bl get_gaussian_neon - vrshl.s16 q0, q0, q15 - vmovn.i16 \r1, q0 - bl get_gaussian_neon - vrshl.s16 q0, q0, q15 - vmovn.i16 \r2, q0 - bl get_gaussian_neon - vrshl.s16 q0, q0, q15 - vmovn.i16 \r3, q0 - bl get_gaussian_neon - vrshl.s16 q0, q0, q15 - vmovn.i16 \r4, q0 - bl get_gaussian_neon - vrshl.s16 q0, q0, q15 - vmovn.i16 \r5, q0 - bl get_gaussian_neon - vrshl.s16 q0, q0, q15 - vmovn.i16 \r6, q0 - bl get_gaussian_neon - vrshl.s16 q0, q0, q15 - vmovn.i16 \r7, q0 - bl get_gaussian_neon - vrshl.s16 q0, q0, q15 - vmovn.i16 \r8, q0 - bl get_gaussian_neon - vrshl.s16 q0, q0, q15 - vmovn.i16 \r9, q0 - increment_seed 2 - read_rand r11, 11, 1 - read_rand r12, 11, 0 - add r11, r3, r11, lsl #1 - add r12, r3, r12, lsl #1 - vld1.16 {d0[0]}, [r11] - vld1.16 {d0[1]}, [r12] - vrshl.s16 d0, d0, d30 - vmovn.i16 \r10, q0 -.endm - -.macro store_grain_row r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10 - vst1.16 {\r0, \r1, \r2, \r3}, [r0]! - vst1.16 {\r4, \r5, \r6, \r7}, [r0]! - vst1.16 {\r8, \r9}, [r0]! - vst1.16 {\r10[0]}, [r0]! -.endm - -.macro get_grain_row_44 r0, r1, r2, r3, r4, r5 - bl get_gaussian_neon - vrshl.s16 q0, q0, q15 - vmovn.i16 \r0, q0 - bl get_gaussian_neon - vrshl.s16 q0, q0, q15 - vmovn.i16 \r1, q0 - bl get_gaussian_neon - vrshl.s16 q0, q0, q15 - vmovn.i16 \r2, q0 - bl get_gaussian_neon - vrshl.s16 q0, q0, q15 - vmovn.i16 \r3, q0 - bl get_gaussian_neon - vrshl.s16 q0, q0, q15 - vmovn.i16 \r4, q0 - increment_seed 4 - read_rand r11, 11, 3 - read_rand r12, 11, 2 - add r11, r3, r11, lsl #1 - add r12, r3, r12, lsl #1 - vld1.16 {d0[]}, [r11] - read_rand r11, 11, 1 - vld1.16 {d0[1]}, [r12] - add r11, r3, r11, lsl #1 - read_rand r12, 11, 0 - vld1.16 {d0[2]}, [r11] - add r12, r3, r12, lsl #1 - vld1.16 {d0[3]}, [r12] - vrshl.s16 d0, d0, d30 - vmovn.i16 \r5, q0 -.endm - -.macro store_grain_row_44 r0, r1, r2, r3, r4, r5 - vst1.16 {\r0, \r1, \r2, \r3}, [r0]! - vst1.16 {\r4, \r5}, [r0] - add r0, r0, #GRAIN_WIDTH-32 -.endm - -function get_grain_2_neon - push {r11,lr} - increment_seed 2 - read_rand r11, 11, 1 - read_rand r12, 11, 0 - add r11, r3, r11, lsl #1 - add r12, r3, r12, lsl #1 - vld1.16 {d0[0]}, [r11] - vld1.16 {d0[1]}, [r12] - vrshl.s16 d0, d0, d30 - vmovn.i16 d0, q0 - pop {r11,pc} -endfunc - -.macro get_grain_2 dst - bl get_grain_2_neon -.ifnc \dst, d0 - vmov \dst, d0 -.endif -.endm - -// r1 holds the number of entries to produce -// r6, r8 and r10 hold the previous output entries -// q0 holds the vector of produced entries -// q1 holds the input vector of sums from above -.macro output_lag n -function output_lag\n\()_neon - push {r0, lr} -.if \n == 1 - mov lr, #-128 -.else - mov r0, #1 - mov lr, #1 - sub r7, r7, #1 - sub r9, r9, #1 - lsl r0, r0, r7 - lsl lr, lr, r9 - add r7, r7, #1 - add r9, r9, #1 -.endif -1: - read_shift_rand r12, 11 - vmov.32 r11, d2[0] - lsl r12, r12, #1 - vext.8 q0, q0, q0, #1 - ldrsh r12, [r3, r12] -.if \n == 1 - mla r11, r6, r4, r11 // sum (above) + *coeff * prev output - add r6, r11, r8 // 1 << (ar_coeff_shift - 1) - add r12, r12, r10 - asr r6, r6, r7 // >> ar_coeff_shift - asr r12, r12, r9 // >> (4 + grain_scale_shift) - add r6, r6, r12 - cmp r6, r5 -.elseif \n == 2 - mla r11, r8, r4, r11 // sum (above) + *coeff * prev output 1 - mla r11, r6, r10, r11 // += *coeff * prev output 2 - mov r8, r6 - add r6, r11, r0 // 1 << (ar_coeff_shift - 1) - add r12, r12, lr // 1 << (4 + grain_scale_shift - 1) - asr r6, r6, r7 // >> ar_coeff_shift - asr r12, r12, r9 // >> (4 + grain_scale_shift) - add r6, r6, r12 - push {lr} - cmp r6, r5 - mov lr, #-128 -.else - push {r1-r3} - sbfx r1, r4, #0, #8 - sbfx r2, r4, #8, #8 - sbfx r3, r4, #16, #8 - mla r11, r10, r1, r11 // sum (above) + *coeff * prev output 1 - mla r11, r8, r2, r11 // sum (above) + *coeff * prev output 2 - mla r11, r6, r3, r11 // += *coeff * prev output 3 - pop {r1-r3} - mov r10, r8 - mov r8, r6 - - add r6, r11, r0 // 1 << (ar_coeff_shift - 1) - add r12, r12, lr // 1 << (4 + grain_scale_shift - 1) - asr r6, r6, r7 // >> ar_coeff_shift - asr r12, r12, r9 // >> (4 + grain_scale_shift) - add r6, r6, r12 - push {lr} - cmp r6, r5 - mov lr, #-128 -.endif - it gt - movgt r6, r5 - cmp r6, lr - it lt - movlt r6, lr -.if \n >= 2 - pop {lr} -.endif - subs r1, r1, #1 - vext.8 q1, q1, q1, #4 - vmov.8 d1[7], r6 - bgt 1b - pop {r0, pc} -endfunc -.endm - -output_lag 1 -output_lag 2 -output_lag 3 - - -function sum_lag1_above_neon - vmull.s8 q2, d6, d28 - vmull.s8 q3, d7, d28 - vmull.s8 q4, d0, d27 - vmull.s8 q5, d1, d27 - - vaddl.s16 q0, d4, d8 - vaddl.s16 q2, d5, d9 - vaddl.s16 q4, d6, d10 - vaddl.s16 q5, d7, d11 - - vmull.s8 q3, d3, d29 - vmull.s8 q1, d2, d29 - - vaddw.s16 q4, q4, d6 - vaddw.s16 q5, q5, d7 - vaddw.s16 q3, q2, d3 - vaddw.s16 q2, q0, d2 - bx lr -endfunc - -.macro sum_lag_n_body lag, type, uv_layout, edge, elems, store, uv_coeff -.ifc \lag\()_\edge, lag3_left - bl sum_lag3_left_above_neon -.else - bl sum_\lag\()_above_neon -.endif -.ifc \type, uv_420 - vpush {q6-q7} - add r12, r11, #GRAIN_WIDTH - vld1.16 {q0, q1}, [r11]! - vld1.16 {q6, q7}, [r12]! - vpaddl.s8 q0, q0 - vpaddl.s8 q1, q1 - vpaddl.s8 q6, q6 - vpaddl.s8 q7, q7 - vadd.i16 q0, q0, q6 - vadd.i16 q1, q1, q7 - vpop {q6-q7} - vrshrn.s16 d0, q0, #2 - vrshrn.s16 d1, q1, #2 -.endif -.ifc \type, uv_422 - vld1.8 {q0, q1}, [r11]! - vpaddl.s8 q0, q0 - vpaddl.s8 q1, q1 - vrshrn.s16 d0, q0, #1 - vrshrn.s16 d1, q1, #1 -.endif -.ifc \type, uv_444 - vld1.8 {q0}, [r11]! -.endif -.if \uv_layout -.ifnb \uv_coeff - vdup.8 d13, \uv_coeff -.endif - vmull.s8 q1, d0, d13 - vmull.s8 q0, d1, d13 - vaddw.s16 q2, q2, d2 - vaddw.s16 q3, q3, d3 - vaddw.s16 q4, q4, d0 - vaddw.s16 q5, q5, d1 -.endif -.if \uv_layout && \elems == 16 - b sum_\lag\()_y_\edge\()_start -.elseif \uv_layout == 444 && \elems == 15 - b sum_\lag\()_y_\edge\()_start -.elseif \uv_layout == 422 && \elems == 9 - b sum_\lag\()_uv_420_\edge\()_start -.else -sum_\lag\()_\type\()_\edge\()_start: - push {r11} -.ifc \edge, left - increment_seed 4 - read_rand r11, 11, 3 - read_rand r12, 11, 2 - add r11, r3, r11, lsl #1 - add r12, r3, r12, lsl #1 - vld1.16 {d1[1]}, [r11] - read_rand r11, 11, 1 - vld1.16 {d1[2]}, [r12] - add r11, r3, r11, lsl #1 - vld1.16 {d1[3]}, [r11] - lsl r2, r2, #1 // shift back the state as if we'd done increment_seed with shift=0 - vrshl.s16 d1, d1, d30 - vmovn.i16 d1, q0 - vext.8 q2, q2, q2, #12 -.ifc \lag, lag3 - vmov.s8 r10, d1[5] -.endif -.ifnc \lag, lag1 - vmov.s8 r8, d1[6] -.endif - vmov.s8 r6, d1[7] - - vmov q1, q2 - mov r1, #1 - bl output_\lag\()_neon -.else - increment_seed 4, shift=0 - vmov q1, q2 - mov r1, #4 - bl output_\lag\()_neon -.endif - - increment_seed 4, shift=0 - vmov q1, q3 - mov r1, #4 - bl output_\lag\()_neon - - increment_seed 4, shift=0 - vmov q1, q4 -.if \elems == 9 - mov r1, #1 - bl output_\lag\()_neon - lsr r2, r2, #3 - - read_rand r11, 11, 2 - read_rand r12, 11, 1 - add r11, r3, r11, lsl #1 - add r12, r3, r12, lsl #1 - vld1.16 {d2[0]}, [r11] - read_rand r11, 11, 0 - vld1.16 {d2[1]}, [r12] - add r11, r3, r11, lsl #1 - vld1.16 {d2[2]}, [r11] - vrshl.s16 d2, d2, d30 - vmovn.i16 d2, q1 - vext.8 q0, q0, q1, #7 -.else - mov r1, #4 - bl output_\lag\()_neon - - increment_seed 4, shift=0 - vmov q1, q5 - -.ifc \edge, right - mov r1, #3 - bl output_\lag\()_neon - read_shift_rand r11, 11 - add r11, r3, r11, lsl #1 - vld1.16 {d2[0]}, [r11] - vrshl.s16 d2, d2, d30 - vext.8 q0, q0, q1, #1 -.else - mov r1, #4 - bl output_\lag\()_neon -.endif -.endif -.if \store - vst1.8 {q0}, [r0]! -.endif - pop {r11} - pop {r1, pc} -.endif -.endm - -.macro sum_lag1_func type, uv_layout, edge, elems=16 -function sum_\type\()_lag1_\edge\()_neon - push {r1, lr} - sum_lag_n_body lag1, \type, \uv_layout, \edge, \elems, store=0 -endfunc -.endm - -sum_lag1_func y, 0, left -sum_lag1_func y, 0, mid -sum_lag1_func y, 0, right, 15 -sum_lag1_func uv_444, 444, left -sum_lag1_func uv_444, 444, mid -sum_lag1_func uv_444, 444, right, 15 -sum_lag1_func uv_422, 422, left -sum_lag1_func uv_422, 422, mid -sum_lag1_func uv_422, 422, right, 9 -sum_lag1_func uv_420, 420, left -sum_lag1_func uv_420, 420, mid -sum_lag1_func uv_420, 420, right, 9 - -.macro sum_lag1 type, dst, left, mid, right, edge=mid - vmov q3, \mid - vext.8 q0, \left, \mid, #15 - vext.8 q1, \mid, \right, #1 - bl sum_\type\()_lag1_\edge\()_neon - vmov \dst, q0 -.endm - -.macro sum_y_lag1 dst, left, mid, right, edge=mid - sum_lag1 y, \dst, \left, \mid, \right, \edge -.endm - -.macro sum_uv_444_lag1 dst, left, mid, right, edge=mid - sum_lag1 uv_444, \dst, \left, \mid, \right, \edge -.endm - -.macro sum_uv_422_lag1 dst, left, mid, right, edge=mid - sum_lag1 uv_422, \dst, \left, \mid, \right, \edge -.endm - -.macro sum_uv_420_lag1 dst, left, mid, right, edge=mid - sum_lag1 uv_420, \dst, \left, \mid, \right, \edge -.endm - - -function sum_lag2_above_neon - push {lr} - sub r12, r0, #2*GRAIN_WIDTH - 16 - sub lr, r0, #1*GRAIN_WIDTH - 16 - vld1.8 {q10}, [r12] // load top right - vld1.8 {q13}, [lr] - - vext.8 q6, q8, q9, #14 // top left, top mid - vdup.8 d14, d28[0] - vext.8 q8, q8, q9, #15 - vdup.8 d15, d28[1] - - vmull.s8 q0, d12, d14 - vmull.s8 q1, d13, d14 - vmull.s8 q6, d16, d15 - vmull.s8 q8, d17, d15 - - vaddl.s16 q2, d0, d12 - vaddl.s16 q3, d1, d13 - vaddl.s16 q4, d2, d16 - vaddl.s16 q5, d3, d17 - - vext.8 q6, q9, q10, #1 // top mid, top right - vdup.8 d14, d28[3] - vext.8 q8, q9, q10, #2 - vdup.8 d15, d28[4] - - vmull.s8 q0, d12, d14 - vmull.s8 q1, d13, d14 - vmull.s8 q6, d16, d15 - vmull.s8 q8, d17, d15 - - vaddl.s16 q7, d0, d12 - vaddl.s16 q0, d1, d13 - vaddl.s16 q6, d2, d16 - vaddl.s16 q1, d3, d17 - - vadd.i32 q2, q2, q7 - vadd.i32 q3, q3, q0 - vadd.i32 q4, q4, q6 - vadd.i32 q5, q5, q1 - - vext.8 q6, q11, q12, #14 // top left, top mid - vdup.8 d14, d28[5] - vext.8 q8, q11, q12, #15 - vdup.8 d15, d28[6] - - vmull.s8 q0, d12, d14 - vmull.s8 q1, d13, d14 - vmull.s8 q6, d16, d15 - vmull.s8 q8, d17, d15 - - vaddl.s16 q7, d0, d12 - vaddl.s16 q0, d1, d13 - vaddl.s16 q6, d2, d16 - vaddl.s16 q1, d3, d17 - - vadd.i32 q2, q2, q7 - vadd.i32 q3, q3, q0 - vadd.i32 q4, q4, q6 - vadd.i32 q5, q5, q1 - - vext.8 q6, q12, q13, #1 // top mid, top right - vdup.8 d14, d29[0] - vext.8 q8, q12, q13, #2 - vdup.8 d15, d29[1] - - vmull.s8 q0, d12, d14 - vmull.s8 q1, d13, d14 - vmull.s8 q6, d16, d15 - vmull.s8 q8, d17, d15 - - vaddl.s16 q7, d0, d12 - vaddl.s16 q0, d1, d13 - vaddl.s16 q6, d2, d16 - vaddl.s16 q1, d3, d17 - - vadd.i32 q2, q2, q7 - vadd.i32 q3, q3, q0 - vadd.i32 q4, q4, q6 - vadd.i32 q5, q5, q1 - - vdup.8 d14, d28[2] - vdup.8 d15, d28[7] - - vmull.s8 q0, d18, d14 - vmull.s8 q1, d19, d14 - vmull.s8 q6, d24, d15 - vmull.s8 q8, d25, d15 - - vaddl.s16 q7, d0, d12 - vaddl.s16 q0, d1, d13 - vaddl.s16 q6, d2, d16 - vaddl.s16 q1, d3, d17 - - vmov q8, q9 - vmov q9, q10 - - vadd.i32 q2, q2, q7 - vadd.i32 q3, q3, q0 - vadd.i32 q4, q4, q6 - vadd.i32 q5, q5, q1 - - vmov q11, q12 - vmov q12, q13 - - pop {pc} -endfunc - -.macro sum_lag2_func type, uv_layout, edge, elems=16 -function sum_\type\()_lag2_\edge\()_neon - push {r1, lr} -.ifc \edge, left - sub r12, r0, #2*GRAIN_WIDTH - sub lr, r0, #1*GRAIN_WIDTH - vld1.8 {q9}, [r12] // load the previous block right above - vld1.8 {q12}, [lr] -.endif - sum_lag_n_body lag2, \type, \uv_layout, \edge, \elems, store=1, uv_coeff=d29[4] -endfunc -.endm - -sum_lag2_func y, 0, left -sum_lag2_func y, 0, mid -sum_lag2_func y, 0, right, 15 -sum_lag2_func uv_444, 444, left -sum_lag2_func uv_444, 444, mid -sum_lag2_func uv_444, 444, right, 15 -sum_lag2_func uv_422, 422, left -sum_lag2_func uv_422, 422, mid -sum_lag2_func uv_422, 422, right, 9 -sum_lag2_func uv_420, 420, left -sum_lag2_func uv_420, 420, mid -sum_lag2_func uv_420, 420, right, 9 - - -function sum_lag3_left_above_neon - // A separate codepath for the left edge, to avoid reading outside - // of the edge of the buffer. - sub r12, r0, #3*GRAIN_WIDTH - vld1.8 {q11, q12}, [r12] - vext.8 q12, q11, q12, #13 - vext.8 q11, q11, q11, #13 - b sum_lag3_above_start -endfunc - -function sum_lag3_above_neon - sub r12, r0, #3*GRAIN_WIDTH + 3 - vld1.8 {q11, q12}, [r12] - -sum_lag3_above_start: - vdup.8 d20, d26[0] - vext.8 q9, q11, q12, #1 - vdup.8 d21, d26[1] - - vmull.s8 q0, d22, d20 - vmull.s8 q1, d23, d20 - vmull.s8 q6, d18, d21 - vmull.s8 q7, d19, d21 - - vext.8 q8, q11, q12, #2 - vdup.8 d20, d26[2] - vext.8 q9, q11, q12, #3 - vdup.8 d21, d26[3] - - vaddl.s16 q2, d0, d12 - vaddl.s16 q3, d1, d13 - vaddl.s16 q4, d2, d14 - vaddl.s16 q5, d3, d15 - - vmull.s8 q0, d16, d20 - vmull.s8 q1, d17, d20 - vmull.s8 q6, d18, d21 - vmull.s8 q7, d19, d21 - - vaddl.s16 q8, d0, d12 - vaddl.s16 q9, d1, d13 - vaddl.s16 q0, d2, d14 - vaddl.s16 q1, d3, d15 - - vext.8 q6, q11, q12, #4 - vdup.8 d20, d26[4] - vext.8 q7, q11, q12, #5 - vdup.8 d21, d26[5] - - vadd.i32 q2, q2, q8 - vadd.i32 q3, q3, q9 - vadd.i32 q4, q4, q0 - vadd.i32 q5, q5, q1 - - vmull.s8 q0, d12, d20 - vmull.s8 q1, d13, d20 - vmull.s8 q8, d14, d21 - vmull.s8 q9, d15, d21 - - sub r12, r0, #2*GRAIN_WIDTH + 3 - - vaddl.s16 q6, d0, d16 - vaddl.s16 q7, d1, d17 - vaddl.s16 q0, d2, d18 - vaddl.s16 q1, d3, d19 - - vext.8 q8, q11, q12, #6 - vld1.8 {q11, q12}, [r12] - vdup.8 d20, d26[6] - vdup.8 d21, d26[7] - - vadd.i32 q2, q2, q6 - vadd.i32 q3, q3, q7 - vadd.i32 q4, q4, q0 - vadd.i32 q5, q5, q1 - - vmull.s8 q0, d16, d20 - vmull.s8 q1, d17, d20 - vmull.s8 q6, d22, d21 - vmull.s8 q7, d23, d21 - - vaddl.s16 q8, d0, d12 - vaddl.s16 q9, d1, d13 - vaddl.s16 q0, d2, d14 - vaddl.s16 q1, d3, d15 - - vext.8 q6, q11, q12, #1 - vdup.8 d20, d27[0] - vext.8 q7, q11, q12, #2 - vdup.8 d21, d27[1] - - vadd.i32 q2, q2, q8 - vadd.i32 q3, q3, q9 - vadd.i32 q4, q4, q0 - vadd.i32 q5, q5, q1 - - vmull.s8 q0, d12, d20 - vmull.s8 q1, d13, d20 - vmull.s8 q8, d14, d21 - vmull.s8 q9, d15, d21 - - vaddl.s16 q6, d0, d16 - vaddl.s16 q7, d1, d17 - vaddl.s16 q0, d2, d18 - vaddl.s16 q1, d3, d19 - - vext.8 q8, q11, q12, #3 - vdup.8 d20, d27[2] - vext.8 q9, q11, q12, #4 - vdup.8 d21, d27[3] - - vadd.i32 q2, q2, q6 - vadd.i32 q3, q3, q7 - vadd.i32 q4, q4, q0 - vadd.i32 q5, q5, q1 - - vmull.s8 q0, d16, d20 - vmull.s8 q1, d17, d20 - vmull.s8 q6, d18, d21 - vmull.s8 q7, d19, d21 - - sub r12, r0, #1*GRAIN_WIDTH + 3 - - vaddl.s16 q8, d0, d12 - vaddl.s16 q9, d1, d13 - vaddl.s16 q0, d2, d14 - vaddl.s16 q1, d3, d15 - - vext.8 q6, q11, q12, #5 - vdup.8 d20, d27[4] - vext.8 q7, q11, q12, #6 - vdup.8 d21, d27[5] - - vld1.8 {q11, q12}, [r12] - - vadd.i32 q2, q2, q8 - vadd.i32 q3, q3, q9 - vadd.i32 q4, q4, q0 - vadd.i32 q5, q5, q1 - - vmull.s8 q0, d12, d20 - vmull.s8 q1, d13, d20 - vmull.s8 q8, d14, d21 - vmull.s8 q9, d15, d21 - - vaddl.s16 q6, d0, d16 - vaddl.s16 q7, d1, d17 - vaddl.s16 q0, d2, d18 - vaddl.s16 q1, d3, d19 - - vdup.8 d20, d27[6] - vext.8 q9, q11, q12, #1 - vdup.8 d21, d27[7] - - vadd.i32 q2, q2, q6 - vadd.i32 q3, q3, q7 - vadd.i32 q4, q4, q0 - vadd.i32 q5, q5, q1 - - vmull.s8 q0, d22, d20 - vmull.s8 q1, d23, d20 - vmull.s8 q6, d18, d21 - vmull.s8 q7, d19, d21 - - vaddl.s16 q8, d0, d12 - vaddl.s16 q9, d1, d13 - vaddl.s16 q0, d2, d14 - vaddl.s16 q1, d3, d15 - - vext.8 q6, q11, q12, #2 - vdup.8 d20, d28[0] - vext.8 q7, q11, q12, #3 - vdup.8 d21, d28[1] - - vadd.i32 q2, q2, q8 - vadd.i32 q3, q3, q9 - vadd.i32 q4, q4, q0 - vadd.i32 q5, q5, q1 - - vmull.s8 q0, d12, d20 - vmull.s8 q1, d13, d20 - vmull.s8 q8, d14, d21 - vmull.s8 q9, d15, d21 - - vaddl.s16 q6, d0, d16 - vaddl.s16 q7, d1, d17 - vaddl.s16 q0, d2, d18 - vaddl.s16 q1, d3, d19 - - vext.8 q8, q11, q12, #4 - vdup.8 d20, d28[2] - vext.8 q9, q11, q12, #5 - vdup.8 d21, d28[3] - - vadd.i32 q2, q2, q6 - vadd.i32 q3, q3, q7 - vadd.i32 q4, q4, q0 - vadd.i32 q5, q5, q1 - - vmull.s8 q0, d16, d20 - vmull.s8 q1, d17, d20 - vmull.s8 q6, d18, d21 - vmull.s8 q7, d19, d21 - - vaddl.s16 q8, d0, d12 - vaddl.s16 q9, d1, d13 - vaddl.s16 q0, d2, d14 - vaddl.s16 q1, d3, d15 - - vext.8 q6, q11, q12, #6 - vdup.8 d20, d28[4] - - vadd.i32 q2, q2, q8 - vadd.i32 q3, q3, q9 - vadd.i32 q4, q4, q0 - vadd.i32 q5, q5, q1 - - vmull.s8 q0, d12, d20 - vmull.s8 q1, d13, d20 - - vaddw.s16 q2, q2, d0 - vaddw.s16 q3, q3, d1 - vaddw.s16 q4, q4, d2 - vaddw.s16 q5, q5, d3 - - bx lr -endfunc - -.macro sum_lag3_func type, uv_layout, edge, elems=16 -function sum_\type\()_lag3_\edge\()_neon - push {r1, lr} - sum_lag_n_body lag3, \type, \uv_layout, \edge, \elems, store=1, uv_coeff=d29[0] -endfunc -.endm - -sum_lag3_func y, 0, left -sum_lag3_func y, 0, mid -sum_lag3_func y, 0, right, 15 -sum_lag3_func uv_444, 444, left -sum_lag3_func uv_444, 444, mid -sum_lag3_func uv_444, 444, right, 15 -sum_lag3_func uv_422, 422, left -sum_lag3_func uv_422, 422, mid -sum_lag3_func uv_422, 422, right, 9 -sum_lag3_func uv_420, 420, left -sum_lag3_func uv_420, 420, mid -sum_lag3_func uv_420, 420, right, 9 - -function generate_grain_rows_neon - push {r11,lr} -1: - get_grain_row d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26 - subs r1, r1, #1 - store_grain_row d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26 - bgt 1b - pop {r11,pc} -endfunc - -function generate_grain_rows_44_neon - push {r11,lr} -1: - get_grain_row_44 d16, d17, d18, d19, d20, d21 - subs r1, r1, #1 - store_grain_row_44 d16, d17, d18, d19, d20, d21 - bgt 1b - pop {r11,pc} -endfunc - -function gen_grain_uv_444_lag0_neon - vld1.8 {q3}, [r11]! - push {r11,lr} - bl get_gaussian_neon - vrshl.s16 q8, q0, q15 - bl get_gaussian_neon - vrshl.s16 q9, q0, q15 - vqmovn.s16 d0, q8 - vqmovn.s16 d1, q9 - - vand q3, q3, q1 - vmull.s8 q2, d6, d22 - vmull.s8 q3, d7, d22 - vrshl.s16 q2, q2, q12 - vrshl.s16 q3, q3, q12 - vaddw.s8 q2, q2, d0 - vaddw.s8 q3, q3, d1 - vqmovn.s16 d4, q2 - vqmovn.s16 d5, q3 - vst1.8 {q2}, [r0]! - pop {r11,pc} -endfunc - -function get_grain_row_44_neon - push {r11,lr} - get_grain_row_44 d16, d17, d18, d19, d20, d21 - pop {r11,pc} -endfunc - -function add_uv_420_coeff_lag0_neon - vld1.16 {q2, q3}, [r11]! - vld1.16 {q4, q5}, [r12]! - vpaddl.s8 q2, q2 - vpaddl.s8 q3, q3 - vpaddl.s8 q4, q4 - vpaddl.s8 q5, q5 - vadd.i16 q2, q2, q4 - vadd.i16 q3, q3, q5 - vrshrn.s16 d4, q2, #2 - vrshrn.s16 d5, q3, #2 - b add_coeff_lag0_start -endfunc - -function add_uv_422_coeff_lag0_neon - vld1.16 {q2, q3}, [r11]! - vpaddl.s8 q2, q2 - vpaddl.s8 q3, q3 - vrshrn.s16 d4, q2, #1 - vrshrn.s16 d5, q3, #1 - -add_coeff_lag0_start: - vand q3, q2, q1 - vmull.s8 q2, d6, d22 - vmull.s8 q3, d7, d22 - vrshl.s16 q2, q2, q12 - vrshl.s16 q3, q3, q12 - vaddw.s8 q2, q2, d0 - vaddw.s8 q3, q3, d1 - vqmovn.s16 d4, q2 - vqmovn.s16 d5, q3 - bx lr -endfunc - -.macro gen_grain_82 type -function generate_grain_\type\()_8bpc_neon, export=1 - push {r4-r11,lr} - -.ifc \type, uv_444 - mov r12, r3 - mov lr, #28 - add r11, r1, #3*GRAIN_WIDTH - mov r1, r2 - mul r12, r12, lr -.endif - movrel r3, X(gaussian_sequence) - ldr r2, [r1, #FGD_SEED] - ldr r9, [r1, #FGD_GRAIN_SCALE_SHIFT] -.ifc \type, y - add r4, r1, #FGD_AR_COEFFS_Y -.else - add r4, r1, #FGD_AR_COEFFS_UV -.endif - adr r5, L(gen_grain_\type\()_tbl) - ldr r6, [r1, #FGD_AR_COEFF_LAG] - add r9, r9, #4 - ldr r6, [r5, r6, lsl #2] - vdup.16 q15, r9 // 4 + data->grain_scale_shift - add r5, r5, r6 - vneg.s16 q15, q15 - -.ifc \type, uv_444 - cmp r12, #0 - movw r10, #0x49d8 - movw lr, #0xb524 - // Intentionally using a separate register instead of moveq with an - // immediate constant, to avoid armv8 deprecated it instruction forms. - it eq - moveq r10, lr - add r4, r4, r12 // Add offset to ar_coeffs_uv[1] - eor r2, r2, r10 -.endif - - ldr r7, [r1, #FGD_AR_COEFF_SHIFT] - mov r8, #1 - mov r10, #1 - lsl r8, r8, r7 // 1 << ar_coeff_shift - lsl r10, r10, r9 // 1 << (4 + data->grain_scale_shift) - lsr r8, r8, #1 // 1 << (ar_coeff_shift - 1) - lsr r10, r10, #1 // 1 << (4 + data->grain_scale_shift - 1) - - bx r5 - - .align 2 -L(gen_grain_\type\()_tbl): - .word L(generate_grain_\type\()_lag0) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB - .word L(generate_grain_\type\()_lag1) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB - .word L(generate_grain_\type\()_lag2) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB - .word L(generate_grain_\type\()_lag3) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB - -L(generate_grain_\type\()_lag0): -.ifc \type, y - mov r1, #GRAIN_HEIGHT - bl generate_grain_rows_neon -.else - - mov r1, #3 - bl generate_grain_rows_neon - mov r1, #GRAIN_HEIGHT-3 - - vdup.16 q12, r7 - vld1.8 {d22[]}, [r4] // ar_coeffs_uv[0] - vmov.i8 q0, #0 - vmov.i8 q1, #255 - vext.8 q13, q0, q1, #13 - vext.8 q14, q1, q0, #1 - vneg.s16 q12, q12 - -1: - vmov q1, q13 - bl gen_grain_uv_444_lag0_neon // 16 - vmov.i8 q1, #255 - bl gen_grain_uv_444_lag0_neon // 32 - bl gen_grain_uv_444_lag0_neon // 48 - bl gen_grain_uv_444_lag0_neon // 64 - vmov q1, q14 - bl gen_grain_uv_444_lag0_neon // 80 - get_grain_2 d16 - subs r1, r1, #1 - add r11, r11, #2 - vst1.16 {d16[0]}, [r0]! - bgt 1b -.endif - pop {r4-r11,pc} - -L(generate_grain_\type\()_lag1): - vpush {q4-q7} - mov r5, #127 - vld1.8 {d27[]}, [r4]! // ar_coeffs_y[0] - vld1.8 {d28[]}, [r4]! // ar_coeffs_y[1] - vld1.8 {d29[]}, [r4] // ar_coeffs_y[2] -.ifc \type, y - ldrsb r4, [r4, #1] // ar_coeffs_y[3] -.else - add r4, r4, #2 -.endif - - mov r1, #3 -.ifc \type, uv_444 - vld1.8 {d13[]}, [r4] // ar_coeffs_uv[4] - ldrsb r4, [r4, #-1] // ar_coeffs_uv[3] -.endif - bl generate_grain_rows_neon - - mov r1, #GRAIN_HEIGHT - 3 -1: - sum_\type\()_lag1 q7, q8, q8, q9, left - sum_\type\()_lag1 q8, q8, q9, q10 - sum_\type\()_lag1 q9, q9, q10, q11 - sum_\type\()_lag1 q10, q10, q11, q12 - sum_\type\()_lag1 q12, q11, q12, q13, right - get_grain_2 d26 - subs r1, r1, #1 -.ifc \type, uv_444 - add r11, r11, #2 -.endif - store_grain_row d14, d15, d16, d17, d18, d19, d20, d21, d24, d25, d26 - vmov q11, q10 - vmov q10, q9 - vmov q9, q8 - vmov q8, q7 - bgt 1b - - vpop {q4-q7} - pop {r4-r11,pc} - -L(generate_grain_\type\()_lag2): - vpush {q4-q7} - mov r5, #127 - vld1.8 {d28,d29}, [r4] // ar_coeffs_y[0-11], ar_coeffs_uv[0-12] - - vmov.s8 r4, d29[2] - vmov.s8 r10, d29[3] - - mov r1, #3 - bl generate_grain_rows_neon - - mov r1, #GRAIN_HEIGHT - 3 -1: - bl sum_\type\()_lag2_left_neon - bl sum_\type\()_lag2_mid_neon - bl sum_\type\()_lag2_mid_neon - bl sum_\type\()_lag2_mid_neon - bl sum_\type\()_lag2_right_neon - get_grain_2 d16 - subs r1, r1, #1 -.ifc \type, uv_444 - add r11, r11, #2 -.endif - vst1.16 {d16[0]}, [r0]! - bgt 1b - - vpop {q4-q7} - pop {r4-r11,pc} - -L(generate_grain_\type\()_lag3): - vpush {q4-q7} - mov r5, #127 - vld1.8 {q13, q14}, [r4] // ar_coeffs_y[0-23], ar_coeffs_uv[0-24] - - vmov.u8 r4, d28[5] - vmov.u8 r10, d28[6] - vmov.u8 r12, d28[7] - - orr r4, r4, r10, lsl #8 - orr r4, r4, r12, lsl #16 - - mov r1, #3 - vpush {d26} - bl generate_grain_rows_neon - vpop {d26} - - mov r1, #GRAIN_HEIGHT - 3 -1: - bl sum_\type\()_lag3_left_neon - bl sum_\type\()_lag3_mid_neon - bl sum_\type\()_lag3_mid_neon - bl sum_\type\()_lag3_mid_neon - bl sum_\type\()_lag3_right_neon - get_grain_2 d16 - subs r1, r1, #1 -.ifc \type, uv_444 - add r11, r11, #2 -.endif - vst1.16 {d16[0]}, [r0]! - bgt 1b - - vpop {q4-q7} - pop {r4-r11,pc} -endfunc -.endm - -gen_grain_82 y -gen_grain_82 uv_444 - -.macro set_height dst, type -.ifc \type, uv_420 - mov \dst, #SUB_GRAIN_HEIGHT-3 -.else - mov \dst, #GRAIN_HEIGHT-3 -.endif -.endm - -.macro increment_y_ptr reg, type -.ifc \type, uv_420 - add \reg, \reg, #2*GRAIN_WIDTH-(3*32) -.else - sub \reg, \reg, #3*32-GRAIN_WIDTH -.endif -.endm - -.macro gen_grain_44 type -function generate_grain_\type\()_8bpc_neon, export=1 - push {r4-r11,lr} - - mov r12, r3 - mov lr, #28 - add r11, r1, #3*GRAIN_WIDTH-3 - mov r1, r2 - mul r12, r12, lr - - movrel r3, X(gaussian_sequence) - ldr r2, [r1, #FGD_SEED] - ldr r9, [r1, #FGD_GRAIN_SCALE_SHIFT] - add r4, r1, #FGD_AR_COEFFS_UV - adr r5, L(gen_grain_\type\()_tbl) - ldr r6, [r1, #FGD_AR_COEFF_LAG] - add r9, r9, #4 - ldr r6, [r5, r6, lsl #2] - vdup.16 q15, r9 // 4 + data->grain_scale_shift - add r5, r5, r6 - vneg.s16 q15, q15 - - cmp r12, #0 - movw r10, #0x49d8 - movw lr, #0xb524 - // Intentionally using a separate register instead of moveq with an - // immediate constant, to avoid armv8 deprecated it instruction forms. - it eq - moveq r10, lr - add r4, r4, r12 // Add offset to ar_coeffs_uv[1] - eor r2, r2, r10 - - ldr r7, [r1, #FGD_AR_COEFF_SHIFT] - mov r8, #1 - mov r10, #1 - lsl r8, r8, r7 // 1 << ar_coeff_shift - lsl r10, r10, r9 // 1 << (4 + data->grain_scale_shift) - lsr r8, r8, #1 // 1 << (ar_coeff_shift - 1) - lsr r10, r10, #1 // 1 << (4 + data->grain_scale_shift - 1) - bx r5 - - .align 2 -L(gen_grain_\type\()_tbl): - .word L(generate_grain_\type\()_lag0) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB - .word L(generate_grain_\type\()_lag1) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB - .word L(generate_grain_\type\()_lag2) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB - .word L(generate_grain_\type\()_lag3) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB - -L(generate_grain_\type\()_lag0): -.ifc \type, uv_420 - vpush {q4-q5} -.endif - mov r1, #3 - bl generate_grain_rows_44_neon - set_height r1, \type - - vdup.16 q12, r7 - vld1.8 {d22[]}, [r4] // ar_coeffs_uv[0] - vmov.i8 q0, #0 - vmov.i8 q1, #255 - vext.8 q13, q0, q1, #13 - vext.8 q14, q1, q0, #7 - vneg.s16 q12, q12 - -1: - bl get_grain_row_44_neon -.ifc \type, uv_420 - add r12, r11, #GRAIN_WIDTH -.endif - vmov q1, q13 - vmov q0, q8 - bl add_\type\()_coeff_lag0_neon - vmov.i8 q1, #255 - vmov q0, q9 - vmov q8, q2 - bl add_\type\()_coeff_lag0_neon - vmov.i8 q1, q14 - vmov q0, q10 - vmov q9, q2 - bl add_\type\()_coeff_lag0_neon - vmov q10, q2 - subs r1, r1, #1 - increment_y_ptr r11, \type - store_grain_row_44 d16, d17, d18, d19, d20, d21 - bgt 1b - -.ifc \type, uv_420 - vpop {q4-q5} -.endif - pop {r4-r11,pc} - -L(generate_grain_\type\()_lag1): - vpush {q4-q7} - mov r5, #127 - vld1.8 {d27[]}, [r4]! // ar_coeffs_uv[0] - vld1.8 {d28[]}, [r4]! // ar_coeffs_uv[1] - vld1.8 {d29[]}, [r4] // ar_coeffs_uv[2] - add r4, r4, #2 - - mov r1, #3 - vld1.8 {d13[]}, [r4] // ar_coeffs_uv[4] - ldrsb r4, [r4, #-1] // ar_coeffs_uv[3] - bl generate_grain_rows_44_neon - - set_height r1, \type -1: - sum_\type\()_lag1 q7, q8, q8, q9, left - sum_\type\()_lag1 q8, q8, q9, q10 - sum_\type\()_lag1 q10, q9, q10, q11, right - subs r1, r1, #1 - increment_y_ptr r11, \type - store_grain_row_44 d14, d15, d16, d17, d20, d21 - vmov q9, q8 - vmov q8, q7 - bgt 1b - - vpop {q4-q7} - pop {r4-r11,pc} - -L(generate_grain_\type\()_lag2): - vpush {q4-q7} - mov r5, #127 - vld1.8 {d28,d29}, [r4] // ar_coeffs_uv[0-12] - - vmov.s8 r4, d29[2] - vmov.s8 r10, d29[3] - - mov r1, #3 - bl generate_grain_rows_44_neon - - set_height r1, \type -1: - bl sum_\type\()_lag2_left_neon - bl sum_\type\()_lag2_mid_neon - bl sum_\type\()_lag2_right_neon - subs r1, r1, #1 - increment_y_ptr r11, \type - add r0, r0, #GRAIN_WIDTH-48 - bgt 1b - - vpop {q4-q7} - pop {r4-r11,pc} - -L(generate_grain_\type\()_lag3): - vpush {q4-q7} - mov r5, #127 - vld1.8 {q13, q14}, [r4] // ar_coeffs_y[0-23], ar_coeffs_uv[0-24] - - vmov.u8 r4, d28[5] - vmov.u8 r10, d28[6] - vmov.u8 r12, d28[7] - - orr r4, r4, r10, lsl #8 - orr r4, r4, r12, lsl #16 - - mov r1, #3 - bl generate_grain_rows_44_neon - - set_height r1, \type -1: - bl sum_\type\()_lag3_left_neon - bl sum_\type\()_lag3_mid_neon - bl sum_\type\()_lag3_right_neon - subs r1, r1, #1 - increment_y_ptr r11, \type - add r0, r0, #GRAIN_WIDTH-48 - bgt 1b - - vpop {q4-q7} - pop {r4-r11,pc} -endfunc -.endm - -gen_grain_44 uv_420 -gen_grain_44 uv_422 - -.macro gather_interleaved dst1, dst2, src1, src2, off - vmov.u8 r11, \src1[0+\off] - vmov.u8 r12, \src2[0+\off] - add r11, r11, r3 - vmov.u8 lr, \src1[2+\off] - add r12, r12, r3 - vld1.8 {\dst1[0+\off]}, [r11] - vmov.u8 r11, \src2[2+\off] - add lr, lr, r3 - vld1.8 {\dst2[0+\off]}, [r12] - vmov.u8 r12, \src1[4+\off] - add r11, r11, r3 - vld1.8 {\dst1[2+\off]}, [lr] - vmov.u8 lr, \src2[4+\off] - add r12, r12, r3 - vld1.8 {\dst2[2+\off]}, [r11] - vmov.u8 r11, \src1[6+\off] - add lr, lr, r3 - vld1.8 {\dst1[4+\off]}, [r12] - vmov.u8 r12, \src2[6+\off] - add r11, r11, r3 - vld1.8 {\dst2[4+\off]}, [lr] - add r12, r12, r3 - vld1.8 {\dst1[6+\off]}, [r11] - vld1.8 {\dst2[6+\off]}, [r12] -.endm - -.macro gather dst1, dst2, dst3, dst4, src1, src2, src3, src4 - gather_interleaved \dst1, \dst3, \src1, \src3, 0 - gather_interleaved \dst1, \dst3, \src1, \src3, 1 - gather_interleaved \dst2, \dst4, \src2, \src4, 0 - gather_interleaved \dst2, \dst4, \src2, \src4, 1 -.endm - -function gather32_neon - push {r11-r12,lr} - gather d8, d9, d10, d11, d0, d1, d2, d3 - pop {r11-r12,pc} -endfunc - -function gather16_neon - push {r11-r12,lr} - gather_interleaved d8, d9, d0, d1, 0 - gather_interleaved d8, d9, d0, d1, 1 - pop {r11-r12,pc} -endfunc - -const overlap_coeffs_0, align=4 - .byte 27, 17, 0, 0, 0, 0, 0, 0 - .byte 17, 27, 32, 32, 32, 32, 32, 32 -endconst - -const overlap_coeffs_1, align=4 - .byte 23, 0, 0, 0, 0, 0, 0, 0 - .byte 22, 32, 32, 32, 32, 32, 32, 32 -endconst - -.macro calc_offset offx, offy, src, sx, sy - and \offy, \src, #0xF // randval & 0xF - lsr \offx, \src, #4 // randval >> 4 -.if \sy == 0 - add \offy, \offy, \offy // 2 * (randval & 0xF) -.endif -.if \sx == 0 - add \offx, \offx, \offx // 2 * (randval >> 4) -.endif -.endm - -.macro add_offset dst, offx, offy, src, stride - mla \dst, \stride, \offy, \src // grain_lut += grain_stride * offy - add \dst, \dst, \offx // grain_lut += offx -.endm - -// void dav1d_fgy_32x32_8bpc_neon(pixel *const dst, const pixel *const src, -// const ptrdiff_t stride, -// const uint8_t scaling[SCALING_SIZE], -// const int scaling_shift, -// const entry grain_lut[][GRAIN_WIDTH], -// const int offsets[][2], -// const int h, const ptrdiff_t clip, -// const ptrdiff_t type); -function fgy_32x32_8bpc_neon, export=1 - push {r4-r11,lr} - vpush {q4-q7} - ldrd r4, r5, [sp, #100] // scaling_shift, grain_lut - ldrd r6, r7, [sp, #108] // offsets, h - ldr r8, [sp, #116] // clip - mov r9, #GRAIN_WIDTH // grain_lut stride - - neg r4, r4 - vdup.16 q13, r4 // -scaling_shift - cmp r8, #0 - - movrel_local r12, overlap_coeffs_0 - - beq 1f - // clip - vmov.i8 q14, #16 - vmov.i8 q15, #235 - b 2f -1: - // no clip - vmov.i8 q14, #0 - vmov.i8 q15, #255 -2: - - vld1.8 {d24, d25}, [r12, :128] // overlap_coeffs - - add r5, r5, #9 // grain_lut += 9 - add r5, r5, r9, lsl #3 // grain_lut += 8 * grain_stride - add r5, r5, r9 // grain_lut += grain_stride - - ldr r10, [r6, #8] // offsets[1][0] - calc_offset r10, r4, r10, 0, 0 - add_offset r4, r10, r4, r5, r9 - ldr r10, [r6, #4] // offsets[0][1] - calc_offset r10, r11, r10, 0, 0 - add_offset r11, r10, r11, r5, r9 - ldr r10, [r6, #12] // offsets[1][1] - calc_offset r10, r8, r10, 0, 0 - add_offset r8, r10, r8, r5, r9 - ldr r6, [r6] // offsets[0][0] - calc_offset r6, lr, r6, 0, 0 - add_offset r5, r6, lr, r5, r9 - - add r4, r4, #32 // grain_lut += BLOCK_SIZE * bx - add r6, r11, r9, lsl #5 // grain_lut += grain_stride * BLOCK_SIZE * by - - ldr r10, [sp, #120] // type - adr r11, L(fgy_loop_tbl) - - tst r10, #1 - ldr r10, [r11, r10, lsl #2] - - add r8, r8, r9, lsl #5 // grain_lut += grain_stride * BLOCK_SIZE * by - add r8, r8, #32 // grain_lut += BLOCK_SIZE * bx - - add r11, r11, r10 - - beq 1f - // y overlap - vdup.8 d14, d24[0] - vdup.8 d15, d24[1] - mov r10, r7 // backup actual h - mov r7, #2 -1: - bx r11 -endfunc - -function fgy_loop_neon -L(fgy_loop_tbl): - .word L(loop_00) - L(fgy_loop_tbl) + CONFIG_THUMB - .word L(loop_01) - L(fgy_loop_tbl) + CONFIG_THUMB - .word L(loop_10) - L(fgy_loop_tbl) + CONFIG_THUMB - .word L(loop_11) - L(fgy_loop_tbl) + CONFIG_THUMB - -.macro fgy ox, oy -L(loop_\ox\oy): -1: -.if \ox - vld1.8 {d8}, [r4], r9 // grain_lut old -.endif -.if \oy - vld1.8 {q2, q3}, [r6], r9 // grain_lut top -.endif -.if \ox && \oy - vld1.8 {d10}, [r8], r9 // grain_lut top old -.endif - vld1.8 {q0, q1}, [r1, :128], r2 // src - vld1.8 {q10, q11}, [r5], r9 // grain_lut - -.if \ox - vmull.s8 q4, d8, d24 - vmlal.s8 q4, d20, d25 -.endif - -.if \oy -.if \ox - vmull.s8 q5, d10, d24 - vmlal.s8 q5, d4, d25 - vqrshrn.s16 d20, q4, #5 - vqrshrn.s16 d4, q5, #5 -.endif - - vmull.s8 q4, d20, d15 - vmull.s8 q5, d21, d15 - vmull.s8 q8, d22, d15 - vmull.s8 q9, d23, d15 - vmlal.s8 q4, d4, d14 - vmlal.s8 q5, d5, d14 - vmlal.s8 q8, d6, d14 - vmlal.s8 q9, d7, d14 - vqrshrn.s16 d20, q4, #5 - vqrshrn.s16 d21, q5, #5 - vqrshrn.s16 d22, q8, #5 - vqrshrn.s16 d23, q9, #5 -.elseif \ox - vqrshrn.s16 d20, q4, #5 -.endif - - bl gather32_neon - - vmovl.s8 q8, d20 // grain - vmovl.s8 q9, d21 - vmovl.s8 q10, d22 - vmovl.s8 q11, d23 - - vmovl.u8 q2, d8 // scaling - vmovl.u8 q3, d9 - vmovl.u8 q4, d10 - vmovl.u8 q5, d11 - - vmul.i16 q8, q8, q2 // scaling * grain - vmul.i16 q9, q9, q3 - vmul.i16 q10, q10, q4 - vmul.i16 q11, q11, q5 - - vrshl.s16 q8, q8, q13 // round2(scaling * grain, scaling_shift) - vrshl.s16 q9, q9, q13 - vrshl.s16 q10, q10, q13 - vrshl.s16 q11, q11, q13 - - vaddw.u8 q8, q8, d0 // *src + noise - vaddw.u8 q9, q9, d1 - vaddw.u8 q10, q10, d2 - vaddw.u8 q11, q11, d3 - - vqmovun.s16 d0, q8 - vqmovun.s16 d1, q9 - vqmovun.s16 d2, q10 - vqmovun.s16 d3, q11 - - vmax.u8 q0, q0, q14 - vmax.u8 q1, q1, q14 - vmin.u8 q0, q0, q15 - vmin.u8 q1, q1, q15 - - subs r7, r7, #1 -.if \oy - vdup.8 d14, d25[0] - vdup.8 d15, d25[1] -.endif - vst1.8 {q0, q1}, [r0, :128], r2 // dst - bgt 1b - -.if \oy - cmp r10, #2 - sub r7, r10, #2 // restore actual remaining h - bgt L(loop_\ox\()0) -.endif - vpop {q4-q7} - pop {r4-r11,pc} -.endm - - fgy 0, 0 - fgy 0, 1 - fgy 1, 0 - fgy 1, 1 -endfunc - -// void dav1d_fguv_32x32_420_8bpc_neon(pixel *const dst, -// const pixel *const src, -// const ptrdiff_t stride, -// const uint8_t scaling[SCALING_SIZE], -// const Dav1dFilmGrainData *const data, -// const entry grain_lut[][GRAIN_WIDTH], -// const pixel *const luma_row, -// const ptrdiff_t luma_stride, -// const int offsets[][2], -// const ptrdiff_t h, const ptrdiff_t uv, -// const ptrdiff_t is_id, -// const ptrdiff_t type); -.macro fguv layout, sx, sy -function fguv_32x32_\layout\()_8bpc_neon, export=1 - push {r4-r11,lr} - vpush {q4-q7} - ldrd r4, r5, [sp, #100] // data, grain_lut - ldrd r6, r7, [sp, #108] // luma_row, luma_stride - ldrd r8, r9, [sp, #116] // offsets, h - ldrd r10, r11, [sp, #124] // uv, is_id - - // !csfl - add r10, r4, r10, lsl #2 // + 4*uv - add r12, r10, #FGD_UV_LUMA_MULT - add lr, r10, #FGD_UV_MULT - add r10, r10, #FGD_UV_OFFSET - vld1.16 {d4[]}, [r12] // uv_luma_mult - vld1.16 {d4[2]}, [r10] // uv_offset - vld1.16 {d4[1]}, [lr] // uv_mult - - ldr lr, [r4, #FGD_SCALING_SHIFT] - ldr r12, [r4, #FGD_CLIP_TO_RESTRICTED_RANGE] - neg lr, lr // -scaling_shift - - cmp r12, #0 - vdup.16 q13, lr // -scaling_shift - - beq 1f - // clip - cmp r11, #0 - vmov.i8 q14, #16 - vmov.i8 q15, #240 - beq 2f - // is_id - vmov.i8 q15, #235 - b 2f -1: - // no clip - vmov.i8 q14, #0 - vmov.i8 q15, #255 -2: - - mov r10, #GRAIN_WIDTH // grain_lut stride - - add r5, r5, #(3 + (2 >> \sx)*3) // grain_lut += 9 or 6 -.if \sy - add r5, r5, r10, lsl #2 // grain_lut += 4 * grain_stride - add r5, r5, r10, lsl #1 // grain_lut += 2 * grain_stride -.else - add r5, r5, r10, lsl #3 // grain_lut += 8 * grain_stride - add r5, r5, r10 // grain_lut += grain_stride -.endif - - ldr r12, [r8, #8] // offsets[1][0] - calc_offset r12, r4, r12, \sx, \sy - add_offset r4, r12, r4, r5, r10 - - ldr r12, [r8, #4] // offsets[0][1] - calc_offset r12, lr, r12, \sx, \sy - add_offset lr, r12, lr, r5, r10 - - ldr r12, [r8, #12] // offsets[1][1] - calc_offset r12, r11, r12, \sx, \sy - add_offset r11, r12, r11, r5, r10 - - ldr r8, [r8] // offsets[0][0] - calc_offset r8, r12, r8, \sx, \sy - add_offset r5, r8, r12, r5, r10 - - add r4, r4, #(32 >> \sx) // grain_lut += BLOCK_SIZE * bx - add r8, lr, r10, lsl #(5 - \sy) // grain_lut += grain_stride * BLOCK_SIZE * by - add r11, r11, r10, lsl #(5 - \sy) // grain_lut += grain_stride * BLOCK_SIZE * by - add r11, r11, #(32 >> \sx) // grain_lut += BLOCK_SIZE * bx - - movrel_local r12, overlap_coeffs_\sx - ldr lr, [sp, #132] // type - - vld1.8 {d24, d25}, [r12, :128] // overlap_coeffs - - movrel_local r12, L(fguv_loop_sx\sx\()_tbl) -#if CONFIG_THUMB - // This uses movrel_local instead of adr above, because the target - // can be out of range for adr. But movrel_local leaves the thumb bit - // set on COFF (but probably wouldn't if building for thumb on ELF), - // thus try to clear the bit for robustness. - bic r12, r12, #1 -#endif - - tst lr, #1 - ldr lr, [r12, lr, lsl #2] - - add r12, r12, lr - - beq 1f - // y overlap - sub lr, r9, #(2 >> \sy) // backup remaining h - mov r9, #(2 >> \sy) - -1: - -.if \sy - vmov.i8 d6, #23 - vmov.i8 d7, #22 -.else - vmov.i8 d6, #27 - vmov.i8 d7, #17 -.endif - -.if \sy - add r7, r7, r7 // luma_stride *= 2 -.endif - - bx r12 -endfunc -.endm - -fguv 420, 1, 1 -fguv 422, 1, 0 -fguv 444, 0, 0 - -function fguv_loop_sx0_neon -L(fguv_loop_sx0_tbl): - .word L(fguv_loop_sx0_csfl0_00) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB - .word L(fguv_loop_sx0_csfl0_01) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB - .word L(fguv_loop_sx0_csfl0_10) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB - .word L(fguv_loop_sx0_csfl0_11) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB - .word L(fguv_loop_sx0_csfl1_00) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB - .word L(fguv_loop_sx0_csfl1_01) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB - .word L(fguv_loop_sx0_csfl1_10) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB - .word L(fguv_loop_sx0_csfl1_11) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB - -.macro fguv_loop_sx0 csfl, ox, oy -L(fguv_loop_sx0_csfl\csfl\()_\ox\oy): -.if \oy - mov r12, lr -.endif -1: -.if \ox - vld1.8 {d8}, [r4], r10 // grain_lut old -.endif -.if \oy - vld1.8 {q8, q9}, [r8], r10 // grain_lut top -.endif -.if \ox && \oy - vld1.8 {d10}, [r11], r10 // grain_lut top old -.endif - vld1.8 {q0, q1}, [r6, :128], r7 // luma - vld1.8 {q10, q11}, [r5], r10 // grain_lut - -.if \ox - vmull.s8 q4, d8, d24 - vmlal.s8 q4, d20, d25 -.endif - -.if \oy -.if \ox - vmull.s8 q5, d10, d24 - vmlal.s8 q5, d16, d25 - vqrshrn.s16 d20, q4, #5 - vqrshrn.s16 d16, q5, #5 -.endif - - vmull.s8 q4, d20, d7 - vmull.s8 q5, d21, d7 - vmull.s8 q6, d22, d7 - vmull.s8 q7, d23, d7 - vmlal.s8 q4, d16, d6 - vmlal.s8 q5, d17, d6 - vmlal.s8 q6, d18, d6 - vmlal.s8 q7, d19, d6 - vqrshrn.s16 d20, q4, #5 - vqrshrn.s16 d21, q5, #5 - vqrshrn.s16 d22, q6, #5 - vqrshrn.s16 d23, q7, #5 -.elseif \ox - vqrshrn.s16 d20, q4, #5 -.endif -.if !\csfl - vld1.8 {q8, q9}, [r1, :128] // src - vmovl.u8 q4, d0 - vmovl.u8 q5, d1 - vmovl.u8 q6, d2 - vmovl.u8 q7, d3 - vmovl.u8 q0, d16 - vmovl.u8 q1, d17 - vmovl.u8 q8, d18 - vmovl.u8 q9, d19 - vmul.i16 q4, q4, d4[0] - vmul.i16 q5, q5, d4[0] - vmul.i16 q6, q6, d4[0] - vmul.i16 q7, q7, d4[0] - vmul.i16 q0, q0, d4[1] - vmul.i16 q1, q1, d4[1] - vmul.i16 q8, q8, d4[1] - vmul.i16 q9, q9, d4[1] - vqadd.s16 q4, q4, q0 - vqadd.s16 q5, q5, q1 - vqadd.s16 q6, q6, q8 - vqadd.s16 q7, q7, q9 - vdup.16 q0, d4[2] - vshr.s16 q4, q4, #6 - vshr.s16 q5, q5, #6 - vshr.s16 q6, q6, #6 - vshr.s16 q7, q7, #6 - vadd.i16 q4, q4, q0 - vadd.i16 q5, q5, q0 - vadd.i16 q6, q6, q0 - vadd.i16 q7, q7, q0 - vqmovun.s16 d0, q4 - vqmovun.s16 d1, q5 - vqmovun.s16 d2, q6 - vqmovun.s16 d3, q7 -.endif - - bl gather32_neon - - vld1.8 {q0, q1}, [r1, :128], r2 // src - - vmovl.s8 q8, d20 // grain - vmovl.s8 q9, d21 - vmovl.s8 q10, d22 - vmovl.s8 q11, d23 - - vmovl.u8 q6, d8 // scaling - vmovl.u8 q7, d9 - vmovl.u8 q4, d10 - vmovl.u8 q5, d11 - - vmul.i16 q8, q8, q6 // scaling * grain - vmul.i16 q9, q9, q7 - vmul.i16 q10, q10, q4 - vmul.i16 q11, q11, q5 - - vrshl.s16 q8, q8, q13 // round2(scaling * grain, scaling_shift) - vrshl.s16 q9, q9, q13 - vrshl.s16 q10, q10, q13 - vrshl.s16 q11, q11, q13 - - vaddw.u8 q8, q8, d0 // *src + noise - vaddw.u8 q9, q9, d1 - vaddw.u8 q10, q10, d2 - vaddw.u8 q11, q11, d3 - - vqmovun.s16 d0, q8 - vqmovun.s16 d1, q9 - vqmovun.s16 d2, q10 - vqmovun.s16 d3, q11 - - vmax.u8 q0, q0, q14 - vmax.u8 q1, q1, q14 - vmin.u8 q0, q0, q15 - vmin.u8 q1, q1, q15 - - subs r9, r9, #1 -.if \oy - vdup.8 d6, d25[0] - vdup.8 d7, d25[1] -.endif - - vst1.8 {q0, q1}, [r0, :128], r2 // dst - bgt 1b - -.if \oy - cmp r12, #0 - mov r9, r12 // restore actual remaining h - bgt L(fguv_loop_sx0_csfl\csfl\()_\ox\()0) -.endif - b 9f -.endm - fguv_loop_sx0 0, 0, 0 - fguv_loop_sx0 0, 0, 1 - fguv_loop_sx0 0, 1, 0 - fguv_loop_sx0 0, 1, 1 - fguv_loop_sx0 1, 0, 0 - fguv_loop_sx0 1, 0, 1 - fguv_loop_sx0 1, 1, 0 - fguv_loop_sx0 1, 1, 1 - -9: - vpop {q4-q7} - pop {r4-r11,pc} -endfunc - -function fguv_loop_sx1_neon -L(fguv_loop_sx1_tbl): - .word L(fguv_loop_sx1_csfl0_00) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB - .word L(fguv_loop_sx1_csfl0_01) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB - .word L(fguv_loop_sx1_csfl0_10) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB - .word L(fguv_loop_sx1_csfl0_11) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB - .word L(fguv_loop_sx1_csfl1_00) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB - .word L(fguv_loop_sx1_csfl1_01) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB - .word L(fguv_loop_sx1_csfl1_10) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB - .word L(fguv_loop_sx1_csfl1_11) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB - -.macro fguv_loop_sx1 csfl, ox, oy -L(fguv_loop_sx1_csfl\csfl\()_\ox\oy): -.if \oy - mov r12, lr -.endif -1: -.if \ox - vld1.8 {d8}, [r4], r10 // grain_lut old -.endif -.if \oy - vld1.8 {q8}, [r8], r10 // grain_lut top -.endif -.if \ox && \oy - vld1.8 {d10}, [r11], r10 // grain_lut top old -.endif - vld1.8 {q0, q1}, [r6, :128], r7 // luma - vld1.8 {q10}, [r5], r10 // grain_lut - vld1.8 {q11}, [r1, :128], r2 // src - -.if \ox - vmull.s8 q4, d8, d24 - vmlal.s8 q4, d20, d25 -.endif - - vpaddl.u8 q0, q0 - vpaddl.u8 q1, q1 -.if \oy -.if \ox - vmull.s8 q5, d10, d24 - vmlal.s8 q5, d16, d25 - vqrshrn.s16 d20, q4, #5 - vqrshrn.s16 d16, q5, #5 -.endif - - vmull.s8 q4, d20, d7 - vmull.s8 q5, d21, d7 - vmlal.s8 q4, d16, d6 - vmlal.s8 q5, d17, d6 - vqrshrn.s16 d20, q4, #5 - vqrshrn.s16 d21, q5, #5 -.elseif \ox - vqrshrn.s16 d20, q4, #5 -.endif -.if \csfl - vrshrn.u16 d0, q0, #1 - vrshrn.u16 d1, q1, #1 -.else - vrshr.u16 q4, q0, #1 - vrshr.u16 q5, q1, #1 - vmovl.u8 q0, d22 - vmovl.u8 q1, d23 - vmul.i16 q4, q4, d4[0] - vmul.i16 q5, q5, d4[0] - vmul.i16 q0, q0, d4[1] - vmul.i16 q1, q1, d4[1] - vqadd.s16 q4, q4, q0 - vqadd.s16 q5, q5, q1 - vdup.16 q0, d4[2] - vshr.s16 q4, q4, #6 - vshr.s16 q5, q5, #6 - vadd.i16 q4, q4, q0 - vadd.i16 q5, q5, q0 - vqmovun.s16 d0, q4 - vqmovun.s16 d1, q5 -.endif - - bl gather16_neon - - vmovl.s8 q8, d20 // grain - vmovl.s8 q9, d21 - - vmovl.u8 q6, d8 // scaling - vmovl.u8 q7, d9 - - vmul.i16 q8, q8, q6 // scaling * grain - vmul.i16 q9, q9, q7 - - vrshl.s16 q8, q8, q13 // round2(scaling * grain, scaling_shift) - vrshl.s16 q9, q9, q13 - - vaddw.u8 q8, q8, d22 // *src + noise - vaddw.u8 q9, q9, d23 - - vqmovun.s16 d0, q8 - vqmovun.s16 d1, q9 - - vmax.u8 q0, q0, q14 - vmin.u8 q0, q0, q15 - - subs r9, r9, #1 -.if \oy - vswp d6, d7 -.endif - vst1.8 {q0}, [r0, :128], r2 // dst - bgt 1b - -.if \oy - cmp r12, #0 - mov r9, r12 // restore actual remaining h - bgt L(fguv_loop_sx1_csfl\csfl\()_\ox\()0) -.endif - - b 9f -.endm - fguv_loop_sx1 0, 0, 0 - fguv_loop_sx1 0, 0, 1 - fguv_loop_sx1 0, 1, 0 - fguv_loop_sx1 0, 1, 1 - fguv_loop_sx1 1, 0, 0 - fguv_loop_sx1 1, 0, 1 - fguv_loop_sx1 1, 1, 0 - fguv_loop_sx1 1, 1, 1 - -9: - vpop {q4-q7} - pop {r4-r11,pc} -endfunc diff --git a/third_party/dav1d/src/arm/32/film_grain16.S b/third_party/dav1d/src/arm/32/film_grain16.S deleted file mode 100644 --- a/third_party/dav1d/src/arm/32/film_grain16.S +++ /dev/null @@ -1,2137 +0,0 @@ -/* - * Copyright © 2021, VideoLAN and dav1d authors - * Copyright © 2021, Martin Storsjo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include "src/arm/asm.S" -#include "util.S" -#include "src/arm/asm-offsets.h" - -#define GRAIN_WIDTH 82 -#define GRAIN_HEIGHT 73 - -#define SUB_GRAIN_WIDTH 44 -#define SUB_GRAIN_HEIGHT 38 - -.macro increment_seed steps, shift=1 - lsr r11, r2, #3 - lsr r12, r2, #12 - lsr lr, r2, #1 - eor r11, r2, r11 // (r >> 0) ^ (r >> 3) - eor r12, r12, lr // (r >> 12) ^ (r >> 1) - eor r11, r11, r12 // (r >> 0) ^ (r >> 3) ^ (r >> 12) ^ (r >> 1) -.if \shift - lsr r2, r2, #\steps -.endif - and r11, r11, #((1 << \steps) - 1) // bit -.if \shift - orr r2, r2, r11, lsl #(16 - \steps) // *state -.else - orr r2, r2, r11, lsl #16 // *state -.endif -.endm - -.macro read_rand dest, bits, age - ubfx \dest, r2, #16 - \bits - \age, #\bits -.endm - -.macro read_shift_rand dest, bits - ubfx \dest, r2, #17 - \bits, #\bits - lsr r2, r2, #1 -.endm - -// special calling convention: -// r2 holds seed -// r3 holds dav1d_gaussian_sequence -// clobbers r11-r12 -// returns in d0-d1 -function get_gaussian_neon - push {r5-r6,lr} - increment_seed 4 - read_rand r5, 11, 3 - read_rand r6, 11, 2 - add r5, r3, r5, lsl #1 - add r6, r3, r6, lsl #1 - vld1.16 {d0[0]}, [r5] - read_rand r5, 11, 1 - vld1.16 {d0[1]}, [r6] - add r5, r3, r5, lsl #1 - read_rand r6, 11, 0 - increment_seed 4 - add r6, r3, r6, lsl #1 - vld1.16 {d0[2]}, [r5] - read_rand r5, 11, 3 - vld1.16 {d0[3]}, [r6] - add r5, r3, r5, lsl #1 - read_rand r6, 11, 2 - vld1.16 {d1[0]}, [r5] - add r6, r3, r6, lsl #1 - read_rand r5, 11, 1 - vld1.16 {d1[1]}, [r6] - read_rand r6, 11, 0 - add r5, r3, r5, lsl #1 - add r6, r3, r6, lsl #1 - vld1.16 {d1[2]}, [r5] - vld1.16 {d1[3]}, [r6] - pop {r5-r6,pc} -endfunc - -function get_grain_2_neon - push {r11,lr} - increment_seed 2 - read_rand r11, 11, 1 - read_rand r12, 11, 0 - add r11, r3, r11, lsl #1 - add r12, r3, r12, lsl #1 - vld1.16 {d0[0]}, [r11] - vld1.16 {d0[1]}, [r12] - vrshl.s16 d0, d0, d30 - pop {r11,pc} -endfunc - -.macro get_grain_2 dst - bl get_grain_2_neon -.ifnc \dst, d0 - vmov \dst, d0 -.endif -.endm - -function get_grain_4_neon - push {r11,lr} - increment_seed 4 - read_rand r11, 11, 3 - read_rand r12, 11, 2 - add r11, r3, r11, lsl #1 - add r12, r3, r12, lsl #1 - vld1.16 {d0[0]}, [r11] - read_rand r11, 11, 1 - vld1.16 {d0[1]}, [r12] - read_rand r12, 11, 0 - add r11, r3, r11, lsl #1 - add r12, r3, r12, lsl #1 - vld1.16 {d0[2]}, [r11] - vld1.16 {d0[3]}, [r12] - vrshl.s16 d0, d0, d30 - pop {r11,pc} -endfunc - -.macro get_grain_4 dst - bl get_grain_4_neon -.ifnc \dst, d0 - vmov \dst, d0 -.endif -.endm - -// r1 holds the number of entries to produce -// r6, r8 and r10 hold the previous output entries -// q0 holds the vector of produced entries -// q1 holds the input vector of sums from above -.macro output_lag n -function output_lag\n\()_neon - push {r0, lr} -.if \n == 1 - mvn lr, r5 // grain_min = ~grain_max -.else - mov r0, #1 - mov lr, #1 - sub r7, r7, #1 - sub r9, r9, #1 - lsl r0, r0, r7 - lsl lr, lr, r9 - add r7, r7, #1 - add r9, r9, #1 -.endif -1: - read_shift_rand r12, 11 - vmov.32 r11, d2[0] - lsl r12, r12, #1 - vext.8 q0, q0, q0, #2 - ldrsh r12, [r3, r12] -.if \n == 1 - mla r11, r6, r4, r11 // sum (above) + *coeff * prev output - add r6, r11, r8 // 1 << (ar_coeff_shift - 1) - add r12, r12, r10 - asr r6, r6, r7 // >> ar_coeff_shift - asr r12, r12, r9 // >> (4 - bitdepth_min_8 + grain_scale_shift) - add r6, r6, r12 - cmp r6, r5 -.elseif \n == 2 - mla r11, r8, r4, r11 // sum (above) + *coeff * prev output 1 - mla r11, r6, r10, r11 // += *coeff * prev output 2 - mov r8, r6 - add r6, r11, r0 // 1 << (ar_coeff_shift - 1) - add r12, r12, lr // 1 << (4 - bitdepth_min_8 + grain_scale_shift - 1) - asr r6, r6, r7 // >> ar_coeff_shift - asr r12, r12, r9 // >> (4 - bitdepth_min_8 + grain_scale_shift) - add r6, r6, r12 - push {lr} - cmp r6, r5 - mvn lr, r5 // grain_min = ~grain_max -.else - push {r1-r3} - sbfx r1, r4, #0, #8 - sbfx r2, r4, #8, #8 - sbfx r3, r4, #16, #8 - mla r11, r10, r1, r11 // sum (above) + *coeff * prev output 1 - mla r11, r8, r2, r11 // sum (above) + *coeff * prev output 2 - mla r11, r6, r3, r11 // += *coeff * prev output 3 - pop {r1-r3} - mov r10, r8 - mov r8, r6 - - add r6, r11, r0 // 1 << (ar_coeff_shift - 1) - add r12, r12, lr // 1 << (4 - bitdepth_min_8 + grain_scale_shift - 1) - asr r6, r6, r7 // >> ar_coeff_shift - asr r12, r12, r9 // >> (4 - bitdepth_min_8 + grain_scale_shift) - add r6, r6, r12 - push {lr} - cmp r6, r5 - mvn lr, r5 // grain_min = ~grain_max -.endif - it gt - movgt r6, r5 - cmp r6, lr - it lt - movlt r6, lr -.if \n >= 2 - pop {lr} -.endif - subs r1, r1, #1 - vext.8 q1, q1, q1, #4 - vmov.16 d1[3], r6 - bgt 1b - pop {r0, pc} -endfunc -.endm - -output_lag 1 -output_lag 2 -output_lag 3 - - -function sum_lag1_above_neon - sub r12, r0, #1*GRAIN_WIDTH*2 - 16 - vld1.16 {q10}, [r12] // load top right - - vext.8 q0, q8, q9, #14 // top left, top mid - vext.8 q1, q9, q10, #2 // top left, top mid - - vmull.s16 q2, d18, d28 - vmlal.s16 q2, d0, d27 - vmlal.s16 q2, d2, d29 - vmull.s16 q3, d19, d28 - vmlal.s16 q3, d1, d27 - vmlal.s16 q3, d3, d29 - - vmov q8, q9 - vmov q9, q10 - - bx lr -endfunc - -.macro sum_lag_n_body lag, type, uv_layout, edge, elems, uv_coeff -.ifc \lag\()_\edge, lag3_left - bl sum_lag3_left_above_neon -.else - bl sum_\lag\()_above_neon -.endif -.ifc \type, uv_420 - vpush {q6-q7} - add r12, r11, #GRAIN_WIDTH*2 - vld1.16 {q0, q1}, [r11]! - vld1.16 {q6, q7}, [r12]! - vpadd.i16 d0, d0, d1 - vpadd.i16 d1, d2, d3 - vpadd.i16 d12, d12, d13 - vpadd.i16 d13, d14, d15 - vadd.i16 q0, q0, q6 - vpop {q6-q7} - vrshr.s16 q0, q0, #2 -.endif -.ifc \type, uv_422 - vld1.16 {q0, q1}, [r11]! - vpadd.i16 d0, d0, d1 - vpadd.i16 d1, d2, d3 - vrshr.s16 q0, q0, #1 -.endif -.ifc \type, uv_444 - vld1.16 {q0}, [r11]! -.endif -.if \uv_layout -.ifnb \uv_coeff - vdup.8 d13, \uv_coeff - vmovl.s8 q6, d13 -.endif - vmlal.s16 q2, d0, d13 - vmlal.s16 q3, d1, d13 -.endif -.if \uv_layout && \elems == 8 - b sum_\lag\()_y_\edge\()_start -.elseif \uv_layout == 444 && \elems == 7 - b sum_\lag\()_y_\edge\()_start -.elseif \uv_layout == 422 && \elems == 1 - b sum_\lag\()_uv_420_\edge\()_start -.else -sum_\lag\()_\type\()_\edge\()_start: - push {r11} -.if \elems > 4 -.ifc \edge, left - increment_seed 4 - read_rand r11, 11, 3 - read_rand r12, 11, 2 - add r11, r3, r11, lsl #1 - add r12, r3, r12, lsl #1 - vld1.16 {d1[1]}, [r11] - read_rand r11, 11, 1 - vld1.16 {d1[2]}, [r12] - add r11, r3, r11, lsl #1 - vld1.16 {d1[3]}, [r11] - lsl r2, r2, #1 // shift back the state as if we'd done increment_seed with shift=0 - vrshl.s16 d1, d1, d30 - vext.8 q2, q2, q2, #12 -.ifc \lag, lag3 - vmov.s16 r10, d1[1] -.endif -.ifnc \lag, lag1 - vmov.s16 r8, d1[2] -.endif - vmov.s16 r6, d1[3] - - vmov q1, q2 - mov r1, #1 - bl output_\lag\()_neon -.else - increment_seed 4, shift=0 - vmov q1, q2 - mov r1, #4 - bl output_\lag\()_neon -.endif - - increment_seed 4, shift=0 - vmov q1, q3 -.ifc \edge, right - mov r1, #3 - bl output_\lag\()_neon - read_shift_rand r12, 11 - add r12, r3, r12, lsl #1 - vld1.16 {d2[0]}, [r12] - vrshl.s16 d2, d2, d30 - vext.8 q0, q0, q1, #2 -.else - mov r1, #4 - bl output_\lag\()_neon -.endif -.else - // elems == 1 - increment_seed 4, shift=0 - vmov q1, q2 - mov r1, #1 - bl output_\lag\()_neon - lsr r2, r2, #3 - - read_rand r11, 11, 2 - read_rand r12, 11, 1 - add r11, r3, r11, lsl #1 - add r12, r3, r12, lsl #1 - vld1.16 {d2[0]}, [r11] - read_rand r11, 11, 0 - vld1.16 {d2[1]}, [r12] - add r11, r3, r11, lsl #1 - vld1.16 {d2[2]}, [r11] - vrshl.s16 d2, d2, d30 - vext.8 q0, q0, q1, #14 -.endif - vst1.16 {q0}, [r0]! - pop {r11} - pop {r1, pc} -.endif -.endm - -.macro sum_lag1_func type, uv_layout, edge, elems=8 -function sum_\type\()_lag1_\edge\()_neon - push {r1, lr} -.ifc \edge, left - sub r12, r0, #1*GRAIN_WIDTH*2 - vld1.8 {q9}, [r12] // load the previous block right above -.endif - sum_lag_n_body lag1, \type, \uv_layout, \edge, \elems -endfunc -.endm - -sum_lag1_func y, 0, left -sum_lag1_func y, 0, mid -sum_lag1_func y, 0, right, 7 -sum_lag1_func uv_444, 444, left -sum_lag1_func uv_444, 444, mid -sum_lag1_func uv_444, 444, right, 7 -sum_lag1_func uv_422, 422, left -sum_lag1_func uv_422, 422, mid -sum_lag1_func uv_422, 422, right, 1 -sum_lag1_func uv_420, 420, left -sum_lag1_func uv_420, 420, mid -sum_lag1_func uv_420, 420, right, 1 - - -function sum_lag2_above_neon - push {lr} - sub r12, r0, #2*GRAIN_WIDTH*2 - 16 - sub lr, r0, #1*GRAIN_WIDTH*2 - 16 - vld1.16 {q10}, [r12] // load top right - vld1.16 {q13}, [lr] - - vdup.8 d10, d28[0] - vext.8 q0, q8, q9, #12 // top left, top mid - vdup.8 d12, d28[1] - vext.8 q1, q8, q9, #14 - vdup.8 d14, d28[3] - vext.8 q4, q9, q10, #2 // top mid, top right - vmovl.s8 q5, d10 - vmovl.s8 q6, d12 - vmovl.s8 q7, d14 - - vmull.s16 q2, d0, d10 - vmlal.s16 q2, d2, d12 - vmlal.s16 q2, d8, d14 - vmull.s16 q3, d1, d10 - vmlal.s16 q3, d3, d12 - vmlal.s16 q3, d9, d14 - - vdup.8 d10, d28[4] - vext.8 q0, q9, q10, #4 // top mid, top right - vdup.8 d12, d28[5] - vext.8 q1, q11, q12, #12 // top left, top mid - vdup.8 d14, d28[6] - vext.8 q4, q11, q12, #14 - vmovl.s8 q5, d10 - vmovl.s8 q6, d12 - vmovl.s8 q7, d14 - - vmlal.s16 q2, d0, d10 - vmlal.s16 q2, d2, d12 - vmlal.s16 q2, d8, d14 - vmlal.s16 q3, d1, d10 - vmlal.s16 q3, d3, d12 - vmlal.s16 q3, d9, d14 - - vdup.8 d10, d29[0] - vext.8 q0, q12, q13, #2 // top mid, top right - vdup.8 d12, d29[1] - vext.8 q1, q12, q13, #4 - - vdup.8 d14, d28[2] - vdup.8 d8, d28[7] - - vmovl.s8 q5, d10 - vmovl.s8 q6, d12 - vmovl.s8 q7, d14 - vmovl.s8 q4, d8 - - vmlal.s16 q2, d0, d10 - vmlal.s16 q2, d2, d12 - vmlal.s16 q2, d18, d14 - vmlal.s16 q2, d24, d8 - vmlal.s16 q3, d1, d10 - vmlal.s16 q3, d3, d12 - vmlal.s16 q3, d19, d14 - vmlal.s16 q3, d25, d8 - - vmov q8, q9 - vmov q9, q10 - - vmov q11, q12 - vmov q12, q13 - - pop {pc} -endfunc - -.macro sum_lag2_func type, uv_layout, edge, elems=8 -function sum_\type\()_lag2_\edge\()_neon - push {r1, lr} -.ifc \edge, left - sub r12, r0, #2*GRAIN_WIDTH*2 - sub lr, r0, #1*GRAIN_WIDTH*2 - vld1.16 {q9}, [r12] // load the previous block right above - vld1.16 {q12}, [lr] -.endif - sum_lag_n_body lag2, \type, \uv_layout, \edge, \elems, uv_coeff=d29[4] -endfunc -.endm - -sum_lag2_func y, 0, left -sum_lag2_func y, 0, mid -sum_lag2_func y, 0, right, 7 -sum_lag2_func uv_444, 444, left -sum_lag2_func uv_444, 444, mid -sum_lag2_func uv_444, 444, right, 7 -sum_lag2_func uv_422, 422, left -sum_lag2_func uv_422, 422, mid -sum_lag2_func uv_422, 422, right, 1 -sum_lag2_func uv_420, 420, left -sum_lag2_func uv_420, 420, mid -sum_lag2_func uv_420, 420, right, 1 - - -function sum_lag3_left_above_neon - // A separate codepath for the left edge, to avoid reading outside - // of the edge of the buffer. - sub r12, r0, #3*GRAIN_WIDTH*2 - vld1.8 {q11, q12}, [r12] - vext.8 q12, q11, q12, #10 - vext.8 q11, q11, q11, #10 - b sum_lag3_above_start -endfunc - -function sum_lag3_above_neon - movw r12, #(3*GRAIN_WIDTH + 3)*2 - sub r12, r0, r12 - vld1.8 {q11, q12}, [r12] - -sum_lag3_above_start: - vdup.8 d12, d26[0] - vext.8 q1, q11, q12, #2 - vdup.8 d14, d26[1] - vext.8 q4, q11, q12, #4 - vdup.8 d16, d26[2] - vext.8 q5, q11, q12, #6 - vdup.8 d18, d26[3] - vmovl.s8 q6, d12 - vmovl.s8 q7, d14 - vmovl.s8 q8, d16 - vmovl.s8 q9, d18 - - movw r12, #(2*GRAIN_WIDTH + 3)*2 - sub r12, r0, r12 - - vmull.s16 q2, d22, d12 - vmlal.s16 q2, d2, d14 - vmlal.s16 q2, d8, d16 - vmlal.s16 q2, d10, d18 - vmull.s16 q3, d23, d12 - vmlal.s16 q3, d3, d14 - vmlal.s16 q3, d9, d16 - vmlal.s16 q3, d11, d18 - - vdup.8 d12, d26[4] - vext.8 q0, q11, q12, #8 - vdup.8 d14, d26[5] - vext.8 q1, q11, q12, #10 - vdup.8 d16, d26[6] - vext.8 q4, q11, q12, #12 - vld1.8 {q11, q12}, [r12] - vdup.8 d18, d26[7] - vmovl.s8 q6, d12 - vmovl.s8 q7, d14 - vmovl.s8 q8, d16 - vmovl.s8 q9, d18 - - vmlal.s16 q2, d0, d12 - vmlal.s16 q2, d2, d14 - vmlal.s16 q2, d8, d16 - vmlal.s16 q2, d22, d18 - vmlal.s16 q3, d1, d12 - vmlal.s16 q3, d3, d14 - vmlal.s16 q3, d9, d16 - vmlal.s16 q3, d23, d18 - - vdup.8 d12, d27[0] - vext.8 q0, q11, q12, #2 - vdup.8 d14, d27[1] - vext.8 q1, q11, q12, #4 - vdup.8 d16, d27[2] - vext.8 q4, q11, q12, #6 - vdup.8 d18, d27[3] - vext.8 q5, q11, q12, #8 - vmovl.s8 q6, d12 - vmovl.s8 q7, d14 - vmovl.s8 q8, d16 - vmovl.s8 q9, d18 - - sub r12, r0, #(1*GRAIN_WIDTH + 3)*2 - - vmlal.s16 q2, d0, d12 - vmlal.s16 q2, d2, d14 - vmlal.s16 q2, d8, d16 - vmlal.s16 q2, d10, d18 - vmlal.s16 q3, d1, d12 - vmlal.s16 q3, d3, d14 - vmlal.s16 q3, d9, d16 - vmlal.s16 q3, d11, d18 - - vdup.8 d12, d27[4] - vext.8 q0, q11, q12, #10 - vdup.8 d14, d27[5] - vext.8 q1, q11, q12, #12 - vld1.8 {q11, q12}, [r12] - vdup.8 d16, d27[6] - vdup.8 d18, d27[7] - vmovl.s8 q6, d12 - vmovl.s8 q7, d14 - vext.8 q5, q11, q12, #2 - vmovl.s8 q8, d16 - vmovl.s8 q9, d18 - - vmlal.s16 q2, d0, d12 - vmlal.s16 q2, d2, d14 - vmlal.s16 q2, d22, d16 - vmlal.s16 q2, d10, d18 - vmlal.s16 q3, d1, d12 - vmlal.s16 q3, d3, d14 - vmlal.s16 q3, d23, d16 - vmlal.s16 q3, d11, d18 - - vdup.8 d12, d28[0] - vext.8 q0, q11, q12, #4 - vdup.8 d14, d28[1] - vext.8 q1, q11, q12, #6 - vdup.8 d16, d28[2] - vext.8 q4, q11, q12, #8 - vdup.8 d18, d28[3] - vext.8 q5, q11, q12, #10 - vmovl.s8 q6, d12 - vmovl.s8 q7, d14 - vmovl.s8 q8, d16 - vmovl.s8 q9, d18 - - vmlal.s16 q2, d0, d12 - vmlal.s16 q2, d2, d14 - vmlal.s16 q2, d8, d16 - vmlal.s16 q2, d10, d18 - vmlal.s16 q3, d1, d12 - vmlal.s16 q3, d3, d14 - vmlal.s16 q3, d9, d16 - vmlal.s16 q3, d11, d18 - - vdup.8 d12, d28[4] - vext.8 q0, q11, q12, #12 - vmovl.s8 q6, d12 - - vmlal.s16 q2, d0, d12 - vmlal.s16 q3, d1, d12 - - bx lr -endfunc - -.macro sum_lag3_func type, uv_layout, edge, elems=8 -function sum_\type\()_lag3_\edge\()_neon - push {r1, lr} - sum_lag_n_body lag3, \type, \uv_layout, \edge, \elems, uv_coeff=d29[0] -endfunc -.endm - -sum_lag3_func y, 0, left -sum_lag3_func y, 0, mid -sum_lag3_func y, 0, right, 7 -sum_lag3_func uv_444, 444, left -sum_lag3_func uv_444, 444, mid -sum_lag3_func uv_444, 444, right, 7 -sum_lag3_func uv_422, 422, left -sum_lag3_func uv_422, 422, mid -sum_lag3_func uv_422, 422, right, 1 -sum_lag3_func uv_420, 420, left -sum_lag3_func uv_420, 420, mid -sum_lag3_func uv_420, 420, right, 1 - -function generate_grain_rows_neon - push {r10-r11,lr} -1: - mov r10, #80 -2: - bl get_gaussian_neon - vrshl.s16 q0, q0, q15 - subs r10, r10, #8 - vst1.16 {q0}, [r0]! - bgt 2b - get_grain_2 d0 - subs r1, r1, #1 - vst1.32 {d0[0]}, [r0]! - bgt 1b - pop {r10-r11,pc} -endfunc - -function generate_grain_rows_44_neon - push {r10-r11,lr} -1: - mov r10, #40 -2: - bl get_gaussian_neon - vrshl.s16 q0, q0, q15 - subs r10, r10, #8 - vst1.16 {q0}, [r0]! - bgt 2b - get_grain_4 d0 - subs r1, r1, #1 - vst1.16 {d0}, [r0] - add r0, r0, #GRAIN_WIDTH*2-80 - bgt 1b - pop {r10-r11,pc} -endfunc - -function gen_grain_uv_444_lag0_neon - vld1.16 {q3}, [r11]! -gen_grain_uv_lag0_8_start: - push {r11,lr} - bl get_gaussian_neon - vrshl.s16 q0, q0, q15 -gen_grain_uv_lag0_8_add: - vand q3, q3, q1 - vmull.s16 q2, d6, d22 - vmull.s16 q3, d7, d22 - vrshl.s32 q2, q2, q12 - vrshl.s32 q3, q3, q12 - vqmovn.s32 d4, q2 - vqmovn.s32 d5, q3 - vqadd.s16 q2, q2, q0 - vmin.s16 q2, q2, q9 - vmax.s16 q2, q2, q10 - vst1.16 {q2}, [r0]! - pop {r11,pc} -endfunc - -function gen_grain_uv_420_lag0_8_neon - add r12, r11, #GRAIN_WIDTH*2 - vld1.16 {q2,q3}, [r11]! - vld1.16 {q4,q5}, [r12] - vpadd.i16 d4, d4, d5 - vpadd.i16 d5, d6, d7 - vpadd.i16 d8, d8, d9 - vpadd.i16 d9, d10, d11 - vadd.i16 q2, q2, q4 - vrshr.s16 q3, q2, #2 - b gen_grain_uv_lag0_8_start -endfunc - -function gen_grain_uv_422_lag0_8_neon - vld1.16 {q2,q3}, [r11]! - vpadd.i16 d4, d4, d5 - vpadd.i16 d5, d6, d7 - vrshr.s16 q3, q2, #1 - b gen_grain_uv_lag0_8_start -endfunc - -function gen_grain_uv_420_lag0_4_neon - add r12, r11, #GRAIN_WIDTH*2 - vld1.16 {q2}, [r11] - vld1.16 {q0}, [r12] - add r11, r11, #32 - vpadd.i16 d4, d4, d5 - vpadd.i16 d0, d0, d1 - vadd.i16 d4, d4, d0 - vrshr.s16 d6, d4, #2 - push {r11,lr} - get_grain_4 d0 - b gen_grain_uv_lag0_8_add -endfunc - -function gen_grain_uv_422_lag0_4_neon - vld1.16 {q2}, [r11] - add r11, r11, #32 - vpadd.i16 d4, d4, d5 - vrshr.s16 d6, d4, #1 - push {r11,lr} - get_grain_4 d0 - b gen_grain_uv_lag0_8_add -endfunc - -.macro gen_grain_82 type -function generate_grain_\type\()_16bpc_neon, export=1 - push {r4-r11,lr} - -.ifc \type, uv_444 - ldr r4, [sp, #36] - mov r12, r3 - mov lr, #28 - add r11, r1, #3*GRAIN_WIDTH*2 - mov r1, r2 - mul r12, r12, lr - clz lr, r4 -.else - clz lr, r2 -.endif - movrel r3, X(gaussian_sequence) - sub lr, lr, #24 // -bitdepth_min_8 - ldr r2, [r1, #FGD_SEED] - ldr r9, [r1, #FGD_GRAIN_SCALE_SHIFT] -.ifc \type, y - add r4, r1, #FGD_AR_COEFFS_Y -.else - add r4, r1, #FGD_AR_COEFFS_UV -.endif - add r9, r9, lr // grain_scale_shift - bitdepth_min_8 - adr r5, L(gen_grain_\type\()_tbl) - ldr r6, [r1, #FGD_AR_COEFF_LAG] - add r9, r9, #4 - ldr r6, [r5, r6, lsl #2] - vdup.16 q15, r9 // 4 - bitdepth_min_8 + data->grain_scale_shift - add r5, r5, r6 - vneg.s16 q15, q15 - -.ifc \type, uv_444 - push {lr} - cmp r12, #0 - movw r10, #0x49d8 - movw lr, #0xb524 - // Intentionally using a separate register instead of moveq with an - // immediate constant, to avoid armv8 deprecated it instruction forms. - it eq - moveq r10, lr - add r4, r4, r12 // Add offset to ar_coeffs_uv[1] - eor r2, r2, r10 - pop {lr} -.endif - - ldr r7, [r1, #FGD_AR_COEFF_SHIFT] - neg lr, lr // bitdepth_min_8 - mov r8, #1 - mov r10, #1 - lsl r8, r8, r7 // 1 << ar_coeff_shift - lsl r10, r10, r9 // 1 << (4 + data->grain_scale_shift) - lsr r8, r8, #1 // 1 << (ar_coeff_shift - 1) - lsr r10, r10, #1 // 1 << (4 + data->grain_scale_shift - 1) - - bx r5 - - .align 2 -L(gen_grain_\type\()_tbl): - .word L(generate_grain_\type\()_lag0) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB - .word L(generate_grain_\type\()_lag1) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB - .word L(generate_grain_\type\()_lag2) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB - .word L(generate_grain_\type\()_lag3) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB - -L(generate_grain_\type\()_lag0): -.ifc \type, y - mov r1, #GRAIN_HEIGHT - bl generate_grain_rows_neon -.else - mov r5, #128 - lsl r5, r5, lr // 128 << bitdepth_min_8 - sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 - mvn r6, r5 // grain_min = ~grain_max - - mov r1, #3 - bl generate_grain_rows_neon - mov r1, #GRAIN_HEIGHT-3 - - vdup.32 q12, r7 - vld1.8 {d22[]}, [r4] // ar_coeffs_uv[0] - vmov.i8 q0, #0 - vmov.i8 q1, #255 - vdup.16 q9, r5 - vdup.16 q10, r6 - vext.8 q13, q0, q1, #10 - vext.8 q14, q1, q0, #2 - vneg.s32 q12, q12 - vmovl.s8 q11, d22 - -1: - vmov q1, q13 - bl gen_grain_uv_444_lag0_neon // 8 - vmov.i8 q1, #255 - bl gen_grain_uv_444_lag0_neon // 16 - bl gen_grain_uv_444_lag0_neon // 24 - bl gen_grain_uv_444_lag0_neon // 32 - bl gen_grain_uv_444_lag0_neon // 40 - bl gen_grain_uv_444_lag0_neon // 48 - bl gen_grain_uv_444_lag0_neon // 56 - bl gen_grain_uv_444_lag0_neon // 64 - bl gen_grain_uv_444_lag0_neon // 72 - vmov q1, q14 - bl gen_grain_uv_444_lag0_neon // 80 - get_grain_2 d16 - subs r1, r1, #1 - add r11, r11, #4 - vst1.32 {d16[0]}, [r0]! - bgt 1b -.endif - pop {r4-r11,pc} - -L(generate_grain_\type\()_lag1): - vpush {q4-q7} - mov r5, #128 - lsl r5, r5, lr // 128 << bitdepth_min_8 - sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 - vld1.8 {d27[]}, [r4]! // ar_coeffs_y[0] - vld1.8 {d28[]}, [r4]! // ar_coeffs_y[1] - vld1.8 {d29[]}, [r4] // ar_coeffs_y[2] -.ifc \type, y - ldrsb r4, [r4, #1] // ar_coeffs_y[3] -.else - add r4, r4, #2 -.endif - - mov r1, #3 -.ifc \type, uv_444 - vld1.8 {d13[]}, [r4] // ar_coeffs_uv[4] - ldrsb r4, [r4, #-1] // ar_coeffs_uv[3] -.endif - bl generate_grain_rows_neon - vmovl.s8 q13, d27 - vmovl.s8 q12, d29 - vmovl.s8 q14, d28 - vmov d29, d24 -.ifc \type, uv_444 - vmovl.s8 q6, d13 -.endif - - mov r1, #GRAIN_HEIGHT - 3 -1: - bl sum_\type\()_lag1_left_neon // 8 - bl sum_\type\()_lag1_mid_neon // 16 - bl sum_\type\()_lag1_mid_neon // 24 - bl sum_\type\()_lag1_mid_neon // 32 - bl sum_\type\()_lag1_mid_neon // 40 - bl sum_\type\()_lag1_mid_neon // 48 - bl sum_\type\()_lag1_mid_neon // 56 - bl sum_\type\()_lag1_mid_neon // 64 - bl sum_\type\()_lag1_mid_neon // 72 - bl sum_\type\()_lag1_right_neon // 80 - get_grain_2 d16 - subs r1, r1, #1 -.ifc \type, uv_444 - add r11, r11, #4 -.endif - vst1.32 {d16[0]}, [r0]! - bgt 1b - - vpop {q4-q7} - pop {r4-r11,pc} - -L(generate_grain_\type\()_lag2): - vpush {q4-q7} - mov r5, #128 - lsl r5, r5, lr // 128 << bitdepth_min_8 - sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 - vld1.8 {d28,d29}, [r4] // ar_coeffs_y[0-11], ar_coeffs_uv[0-12] - - vmov.s8 r4, d29[2] - vmov.s8 r10, d29[3] - - mov r1, #3 - bl generate_grain_rows_neon - - mov r1, #GRAIN_HEIGHT - 3 -1: - bl sum_\type\()_lag2_left_neon // 8 - bl sum_\type\()_lag2_mid_neon // 16 - bl sum_\type\()_lag2_mid_neon // 24 - bl sum_\type\()_lag2_mid_neon // 32 - bl sum_\type\()_lag2_mid_neon // 40 - bl sum_\type\()_lag2_mid_neon // 48 - bl sum_\type\()_lag2_mid_neon // 56 - bl sum_\type\()_lag2_mid_neon // 64 - bl sum_\type\()_lag2_mid_neon // 72 - bl sum_\type\()_lag2_right_neon // 80 - get_grain_2 d16 - subs r1, r1, #1 -.ifc \type, uv_444 - add r11, r11, #4 -.endif - vst1.32 {d16[0]}, [r0]! - bgt 1b - - vpop {q4-q7} - pop {r4-r11,pc} - -L(generate_grain_\type\()_lag3): - vpush {q4-q7} - mov r5, #128 - lsl r5, r5, lr // 128 << bitdepth_min_8 - sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 - vld1.8 {q13, q14}, [r4] // ar_coeffs_y[0-23], ar_coeffs_uv[0-24] - - vmov.u8 r4, d28[5] - vmov.u8 r10, d28[6] - vmov.u8 r12, d28[7] - - orr r4, r4, r10, lsl #8 - orr r4, r4, r12, lsl #16 - - mov r1, #3 - vpush {d26} - bl generate_grain_rows_neon - vpop {d26} - - mov r1, #GRAIN_HEIGHT - 3 -1: - bl sum_\type\()_lag3_left_neon // 8 - bl sum_\type\()_lag3_mid_neon // 16 - bl sum_\type\()_lag3_mid_neon // 24 - bl sum_\type\()_lag3_mid_neon // 32 - bl sum_\type\()_lag3_mid_neon // 40 - bl sum_\type\()_lag3_mid_neon // 48 - bl sum_\type\()_lag3_mid_neon // 56 - bl sum_\type\()_lag3_mid_neon // 64 - bl sum_\type\()_lag3_mid_neon // 72 - bl sum_\type\()_lag3_right_neon // 80 - get_grain_2 d16 - subs r1, r1, #1 -.ifc \type, uv_444 - add r11, r11, #4 -.endif - vst1.32 {d16[0]}, [r0]! - bgt 1b - - vpop {q4-q7} - pop {r4-r11,pc} -endfunc -.endm - -gen_grain_82 y -gen_grain_82 uv_444 - -.macro set_height dst, type -.ifc \type, uv_420 - mov \dst, #SUB_GRAIN_HEIGHT-3 -.else - mov \dst, #GRAIN_HEIGHT-3 -.endif -.endm - -.macro increment_y_ptr reg, type -.ifc \type, uv_420 - add \reg, \reg, #2*GRAIN_WIDTH*2-(6*32) -.else - sub \reg, \reg, #6*32-GRAIN_WIDTH*2 -.endif -.endm - -.macro gen_grain_44 type -function generate_grain_\type\()_16bpc_neon, export=1 - push {r4-r11,lr} - - ldr r4, [sp, #36] - mov r12, r3 - movw r11, #(3*GRAIN_WIDTH-3)*2 - mov lr, #28 - add r11, r1, r11 - mov r1, r2 - mul r12, r12, lr - clz lr, r4 - - movrel r3, X(gaussian_sequence) - sub lr, lr, #24 // -bitdepth_min_8 - ldr r2, [r1, #FGD_SEED] - ldr r9, [r1, #FGD_GRAIN_SCALE_SHIFT] - add r4, r1, #FGD_AR_COEFFS_UV - add r9, r9, lr // grain_scale_shift - bitdepth_min_8 - adr r5, L(gen_grain_\type\()_tbl) - ldr r6, [r1, #FGD_AR_COEFF_LAG] - add r9, r9, #4 - ldr r6, [r5, r6, lsl #2] - vdup.16 q15, r9 // 4 - bitdepth_min_8 + data->grain_scale_shift - add r5, r5, r6 - vneg.s16 q15, q15 - - push {lr} - cmp r12, #0 - movw r10, #0x49d8 - movw lr, #0xb524 - // Intentionally using a separate register instead of moveq with an - // immediate constant, to avoid armv8 deprecated it instruction forms. - it eq - moveq r10, lr - add r4, r4, r12 // Add offset to ar_coeffs_uv[1] - eor r2, r2, r10 - pop {lr} - - ldr r7, [r1, #FGD_AR_COEFF_SHIFT] - neg lr, lr - mov r8, #1 - mov r10, #1 - lsl r8, r8, r7 // 1 << ar_coeff_shift - lsl r10, r10, r9 // 1 << (4 + data->grain_scale_shift) - lsr r8, r8, #1 // 1 << (ar_coeff_shift - 1) - lsr r10, r10, #1 // 1 << (4 + data->grain_scale_shift - 1) - bx r5 - - .align 2 -L(gen_grain_\type\()_tbl): - .word L(generate_grain_\type\()_lag0) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB - .word L(generate_grain_\type\()_lag1) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB - .word L(generate_grain_\type\()_lag2) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB - .word L(generate_grain_\type\()_lag3) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB - -L(generate_grain_\type\()_lag0): -.ifc \type, uv_420 - vpush {q4-q5} -.endif - mov r5, #128 - lsl r5, r5, lr // 128 << bitdepth_min_8 - sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 - mvn r6, r5 // grain_min = ~grain_max - - mov r1, #3 - bl generate_grain_rows_44_neon - set_height r1, \type - - vdup.32 q12, r7 - vld1.8 {d22[]}, [r4] // ar_coeffs_uv[0] - vmov.i8 q0, #0 - vmov.i8 q1, #255 - vdup.16 q9, r5 - vdup.16 q10, r6 - vext.8 q13, q0, q1, #10 - vext.8 q14, q1, q0, #14 - vneg.s32 q12, q12 - vmovl.s8 q11, d22 - -1: - vmov q1, q13 - bl gen_grain_\type\()_lag0_8_neon // 8 - vmov.i8 q1, #255 - bl gen_grain_\type\()_lag0_8_neon // 16 - bl gen_grain_\type\()_lag0_8_neon // 24 - bl gen_grain_\type\()_lag0_8_neon // 32 - bl gen_grain_\type\()_lag0_8_neon // 40 - vmov q1, q14 - bl gen_grain_\type\()_lag0_4_neon // 44 - subs r1, r1, #1 - increment_y_ptr r11, \type - add r0, r0, #GRAIN_WIDTH*2-6*16 - bgt 1b - -.ifc \type, uv_420 - vpop {q4-q5} -.endif - pop {r4-r11,pc} - -L(generate_grain_\type\()_lag1): - vpush {q4-q7} - mov r5, #128 - lsl r5, r5, lr // 128 << bitdepth_min_8 - sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 - vld1.8 {d27[]}, [r4]! // ar_coeffs_uv[0] - vld1.8 {d28[]}, [r4]! // ar_coeffs_uv[1] - vld1.8 {d29[]}, [r4] // ar_coeffs_uv[2] - add r4, r4, #2 - - mov r1, #3 - vld1.8 {d13[]}, [r4] // ar_coeffs_uv[4] - ldrsb r4, [r4, #-1] // ar_coeffs_uv[3] - bl generate_grain_rows_44_neon - vmovl.s8 q13, d27 - vmovl.s8 q12, d29 - vmovl.s8 q14, d28 - vmov d29, d24 - vmovl.s8 q6, d13 - - set_height r1, \type -1: - bl sum_\type\()_lag1_left_neon // 8 - bl sum_\type\()_lag1_mid_neon // 16 - bl sum_\type\()_lag1_mid_neon // 24 - bl sum_\type\()_lag1_mid_neon // 32 - bl sum_\type\()_lag1_mid_neon // 40 - bl sum_\type\()_lag1_right_neon // 44 - subs r1, r1, #1 - increment_y_ptr r11, \type - add r0, r0, #GRAIN_WIDTH*2-6*16 - bgt 1b - - vpop {q4-q7} - pop {r4-r11,pc} - -L(generate_grain_\type\()_lag2): - vpush {q4-q7} - mov r5, #128 - lsl r5, r5, lr // 128 << bitdepth_min_8 - sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 - vld1.8 {d28,d29}, [r4] // ar_coeffs_uv[0-12] - - vmov.s8 r4, d29[2] - vmov.s8 r10, d29[3] - - mov r1, #3 - bl generate_grain_rows_44_neon - - set_height r1, \type -1: - bl sum_\type\()_lag2_left_neon // 8 - bl sum_\type\()_lag2_mid_neon // 16 - bl sum_\type\()_lag2_mid_neon // 24 - bl sum_\type\()_lag2_mid_neon // 32 - bl sum_\type\()_lag2_mid_neon // 40 - bl sum_\type\()_lag2_right_neon // 44 - subs r1, r1, #1 - increment_y_ptr r11, \type - add r0, r0, #GRAIN_WIDTH*2-6*16 - bgt 1b - - vpop {q4-q7} - pop {r4-r11,pc} - -L(generate_grain_\type\()_lag3): - vpush {q4-q7} - mov r5, #128 - lsl r5, r5, lr // 128 << bitdepth_min_8 - sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 - vld1.8 {q13, q14}, [r4] // ar_coeffs_y[0-23], ar_coeffs_uv[0-24] - - vmov.u8 r4, d28[5] - vmov.u8 r10, d28[6] - vmov.u8 r12, d28[7] - - orr r4, r4, r10, lsl #8 - orr r4, r4, r12, lsl #16 - - mov r1, #3 - bl generate_grain_rows_44_neon - - set_height r1, \type -1: - bl sum_\type\()_lag3_left_neon // 8 - bl sum_\type\()_lag3_mid_neon // 16 - bl sum_\type\()_lag3_mid_neon // 24 - bl sum_\type\()_lag3_mid_neon // 32 - bl sum_\type\()_lag3_mid_neon // 40 - bl sum_\type\()_lag3_right_neon // 44 - subs r1, r1, #1 - increment_y_ptr r11, \type - add r0, r0, #GRAIN_WIDTH*2-6*16 - bgt 1b - - vpop {q4-q7} - pop {r4-r11,pc} -endfunc -.endm - -gen_grain_44 uv_420 -gen_grain_44 uv_422 - -.macro gather_interleaved dst1, dst2, src1, src2, src3, src4, off - vmov.u16 r11, \src1[0+\off] - vmov.u16 r12, \src3[0+\off] - add r11, r11, r3 - vmov.u16 lr, \src1[2+\off] - add r12, r12, r3 - vld1.8 {\dst1[0+\off]}, [r11] - vmov.u16 r11, \src3[2+\off] - add lr, lr, r3 - vld1.8 {\dst2[0+\off]}, [r12] - vmov.u16 r12, \src2[0+\off] - add r11, r11, r3 - vld1.8 {\dst1[2+\off]}, [lr] - vmov.u16 lr, \src4[0+\off] - add r12, r12, r3 - vld1.8 {\dst2[2+\off]}, [r11] - vmov.u16 r11, \src2[2+\off] - add lr, lr, r3 - vld1.8 {\dst1[4+\off]}, [r12] - vmov.u16 r12, \src4[2+\off] - add r11, r11, r3 - vld1.8 {\dst2[4+\off]}, [lr] - add r12, r12, r3 - vld1.8 {\dst1[6+\off]}, [r11] - vld1.8 {\dst2[6+\off]}, [r12] -.endm - -.macro gather dst1, dst2, dst3, dst4, src1, src2, src3, src4, src5, src6, src7, src8 - gather_interleaved \dst1, \dst3, \src1, \src2, \src5, \src6, 0 - gather_interleaved \dst1, \dst3, \src1, \src2, \src5, \src6, 1 - gather_interleaved \dst2, \dst4, \src3, \src4, \src7, \src8, 0 - gather_interleaved \dst2, \dst4, \src3, \src4, \src7, \src8, 1 -.endm - -function gather32_neon - push {r11-r12,lr} - gather d8, d9, d10, d11, d0, d1, d2, d3, d4, d5, d6, d7 - pop {r11-r12,pc} -endfunc - -function gather16_neon - push {r11-r12,lr} - gather_interleaved d8, d9, d0, d1, d2, d3, 0 - gather_interleaved d8, d9, d0, d1, d2, d3, 1 - pop {r11-r12,pc} -endfunc - -const overlap_coeffs_0, align=4 - .short 27, 17, 0, 0 - .short 17, 27, 32, 32 -endconst - -const overlap_coeffs_1, align=4 - .short 23, 0, 0, 0 - .short 22, 32, 32, 32 -endconst - -.macro calc_offset offx, offy, src, sx, sy - and \offy, \src, #0xF // randval & 0xF - lsr \offx, \src, #4 // randval >> 4 -.if \sy == 0 - add \offy, \offy, \offy // 2 * (randval & 0xF) -.endif -.if \sx == 0 - add \offx, \offx, \offx // 2 * (randval >> 4) -.endif -.endm - -.macro add_offset dst, offx, offy, src, stride - mla \dst, \stride, \offy, \src // grain_lut += grain_stride * offy - add \dst, \dst, \offx, lsl #1 // grain_lut += offx -.endm - -// void dav1d_fgy_32x32_16bpc_neon(pixel *const dst, const pixel *const src, -// const ptrdiff_t stride, -// const uint8_t scaling[SCALING_SIZE], -// const int scaling_shift, -// const entry grain_lut[][GRAIN_WIDTH], -// const int offsets[][2], -// const int h, const ptrdiff_t clip, -// const ptrdiff_t type, -// const int bitdepth_max); -function fgy_32x32_16bpc_neon, export=1 - push {r4-r11,lr} - vpush {q4-q7} - ldrd r4, r5, [sp, #100] // scaling_shift, grain_lut - ldrd r6, r7, [sp, #108] // offsets, h - ldr r8, [sp, #116] // clip - mov r9, #GRAIN_WIDTH*2 // grain_lut stride - ldr r10, [sp, #124] // bitdepth_max - - eor r4, r4, #15 // 15 - scaling_shift - vdup.16 q6, r10 // bitdepth_max - clz r10, r10 - vdup.16 q13, r4 // 15 - scaling_shift - rsb r10, r10, #24 // bitdepth_min_8 - cmp r8, #0 - vdup.16 q12, r10 // bitdepth_min_8 - - movrel_local r12, overlap_coeffs_0 - - beq 1f - // clip - vmov.i16 q14, #16 - vmov.i16 q15, #235 - vshl.s16 q14, q14, q12 - vshl.s16 q15, q15, q12 - b 2f -1: - // no clip - vmov.i16 q14, #0 - vmov q15, q6 -2: - vshr.u16 q6, q6, #1 // grain_max - - vld1.16 {d24, d25}, [r12, :128] // overlap_coeffs - - add r5, r5, #18 // grain_lut += 9 - add r5, r5, r9, lsl #3 // grain_lut += 8 * grain_stride - add r5, r5, r9 // grain_lut += grain_stride - - ldr r10, [r6, #8] // offsets[1][0] - calc_offset r10, r4, r10, 0, 0 - add_offset r4, r10, r4, r5, r9 - ldr r10, [r6, #4] // offsets[0][1] - calc_offset r10, r11, r10, 0, 0 - add_offset r11, r10, r11, r5, r9 - ldr r10, [r6, #12] // offsets[1][1] - calc_offset r10, r8, r10, 0, 0 - add_offset r8, r10, r8, r5, r9 - ldr r6, [r6] // offsets[0][0] - calc_offset r6, lr, r6, 0, 0 - add_offset r5, r6, lr, r5, r9 - - add r4, r4, #32*2 // grain_lut += BLOCK_SIZE * bx - add r6, r11, r9, lsl #5 // grain_lut += grain_stride * BLOCK_SIZE * by - - ldr r10, [sp, #120] // type - adr r11, L(fgy_loop_tbl) - - tst r10, #1 - ldr r10, [r11, r10, lsl #2] - - add r8, r8, r9, lsl #5 // grain_lut += grain_stride * BLOCK_SIZE * by - add r8, r8, #32*2 // grain_lut += BLOCK_SIZE * bx - - add r11, r11, r10 - - beq 1f - // y overlap - vdup.16 d14, d24[0] - vdup.16 d15, d24[1] - mov r10, r7 // backup actual h - mov r7, #2 -1: - sub r2, r2, #32 // src_stride -= 32 - sub r9, r9, #32 // grain_stride -= 32 - bx r11 -endfunc - -function fgy_loop_neon -L(fgy_loop_tbl): - .word L(loop_00) - L(fgy_loop_tbl) + CONFIG_THUMB - .word L(loop_01) - L(fgy_loop_tbl) + CONFIG_THUMB - .word L(loop_10) - L(fgy_loop_tbl) + CONFIG_THUMB - .word L(loop_11) - L(fgy_loop_tbl) + CONFIG_THUMB - -.macro fgy ox, oy -L(loop_\ox\oy): -1: -.if \ox - vld1.16 {d0}, [r4], r9 // grain_lut old -.endif -.if \oy - vld1.16 {q2, q3}, [r6]! // grain_lut top -.endif -.if \ox && \oy - vld1.16 {d2}, [r8], r9 // grain_lut top old -.endif -.if \oy - vld1.16 {q4, q5}, [r6], r9 // grain_lut top -.endif -.if !\ox && !\oy - vld1.16 {q0, q1}, [r1, :128]! // src -.endif - vld1.16 {q8, q9}, [r5]! // grain_lut -.if !\ox && !\oy - vld1.16 {q2, q3}, [r1, :128], r2 // src -.endif -.if !\oy - vmvn.i16 q5, #0xf000 // 0x0fff -.endif - vld1.16 {q10, q11}, [r5], r9 // grain_lut - -.if \ox - add r4, r4, #32 - vmull.s16 q0, d0, d24 - vmlal.s16 q0, d16, d25 -.endif - -.if \oy -.if \ox - add r8, r8, #32 - vmull.s16 q1, d2, d24 - vmlal.s16 q1, d4, d25 - vqrshrn.s32 d16, q0, #5 - vmvn d0, d12 // grain_min - vqrshrn.s32 d4, q1, #5 - vmin.s16 d16, d16, d12 - vmin.s16 d4, d4, d12 - vmax.s16 d16, d16, d0 - vmax.s16 d4, d4, d0 -.endif - - vmull.s16 q0, d4, d14 - vmull.s16 q1, d5, d14 - vmull.s16 q2, d6, d14 - vmull.s16 q3, d7, d14 - vmlal.s16 q0, d16, d15 - vmlal.s16 q1, d17, d15 - vmlal.s16 q2, d18, d15 - vmlal.s16 q3, d19, d15 - vmull.s16 q8, d20, d15 - vmull.s16 q9, d21, d15 - vmull.s16 q10, d22, d15 - vmull.s16 q11, d23, d15 - vmlal.s16 q8, d8, d14 - vmlal.s16 q9, d9, d14 - vmlal.s16 q10, d10, d14 - vmlal.s16 q11, d11, d14 - vmvn q4, q6 // grain_min - vqrshrn.s32 d0, q0, #5 - vqrshrn.s32 d1, q1, #5 - vqrshrn.s32 d2, q2, #5 - vqrshrn.s32 d3, q3, #5 - vqrshrn.s32 d4, q8, #5 - vqrshrn.s32 d5, q9, #5 - vqrshrn.s32 d6, q10, #5 - vqrshrn.s32 d7, q11, #5 - vmin.s16 q8, q0, q6 - vmin.s16 q9, q1, q6 - vld1.16 {q0, q1}, [r1, :128]! // src - vmin.s16 q10, q2, q6 - vmin.s16 q11, q3, q6 - vmax.s16 q8, q8, q4 - vmax.s16 q9, q9, q4 - vld1.16 {q2, q3}, [r1, :128], r2 // src - vmvn.i16 q5, #0xf000 // 0x0fff - vmax.s16 q10, q10, q4 - vmax.s16 q11, q11, q4 -.elseif \ox - vmvn d4, d12 // grain_min - vqrshrn.s32 d16, q0, #5 - vld1.16 {q0, q1}, [r1, :128]! // src - vmin.s16 d16, d16, d12 - vmax.s16 d16, d16, d4 - vld1.16 {q2, q3}, [r1, :128], r2 // src -.endif - - // Make sure that uninitialized pixels out of range past the right - // edge are in range; their actual values shouldn't matter. - vand q0, q0, q5 - vand q1, q1, q5 - vand q2, q2, q5 - vand q3, q3, q5 - - bl gather32_neon - -.if \ox || \oy - vpush {q6-q7} -.endif - - vmovl.u8 q6, d8 // scaling - vmovl.u8 q7, d9 - vmovl.u8 q4, d10 - vmovl.u8 q5, d11 - - vshl.u16 q6, q6, q13 // scaling << (15 - scaling_shift) - vshl.u16 q7, q7, q13 - vshl.u16 q4, q4, q13 - vshl.u16 q5, q5, q13 - - vqrdmulh.s16 q8, q8, q6 // round2((scaling << (15 - scaling_shift) * grain, 15) - vqrdmulh.s16 q9, q9, q7 - vqrdmulh.s16 q10, q10, q4 - vqrdmulh.s16 q11, q11, q5 - -.if \ox || \oy - vpop {q6-q7} -.endif - - vqadd.s16 q0, q0, q8 // *src + noise - vqadd.s16 q1, q1, q9 - vqadd.s16 q2, q2, q10 - vqadd.s16 q3, q3, q11 - - vmax.s16 q0, q0, q14 - vmax.s16 q1, q1, q14 - vmax.s16 q2, q2, q14 - vmax.s16 q3, q3, q14 - vmin.s16 q0, q0, q15 - vmin.s16 q1, q1, q15 - vmin.s16 q2, q2, q15 - vmin.s16 q3, q3, q15 - - vst1.16 {q0, q1}, [r0, :128]! // dst - subs r7, r7, #1 -.if \oy - vdup.16 d14, d25[0] - vdup.16 d15, d25[1] -.endif - vst1.16 {q2, q3}, [r0, :128], r2 // dst - bgt 1b - -.if \oy - cmp r10, #2 - sub r7, r10, #2 // restore actual remaining h - bgt L(loop_\ox\()0) -.endif - vpop {q4-q7} - pop {r4-r11,pc} -.endm - - fgy 0, 0 - fgy 0, 1 - fgy 1, 0 - fgy 1, 1 -endfunc - -// void dav1d_fguv_32x32_420_16bpc_neon(pixel *const dst, -// const pixel *const src, -// const ptrdiff_t stride, -// const uint8_t scaling[SCALING_SIZE], -// const Dav1dFilmGrainData *const data, -// const entry grain_lut[][GRAIN_WIDTH], -// const pixel *const luma_row, -// const ptrdiff_t luma_stride, -// const int offsets[][2], -// const ptrdiff_t h, const ptrdiff_t uv, -// const ptrdiff_t is_id, -// const ptrdiff_t type, -// const int bitdepth_max); -.macro fguv layout, sx, sy -function fguv_32x32_\layout\()_16bpc_neon, export=1 - push {r4-r11,lr} - vpush {q4-q7} - ldrd r4, r5, [sp, #100] // data, grain_lut - ldrd r10, r11, [sp, #124] // uv, is_id - ldr r6, [sp, #136] // bitdepth_max - - clz r7, r6 - rsb r7, r7, #24 // bitdepth_min_8 - - // !csfl - add r10, r4, r10, lsl #2 // + 4*uv - add r12, r10, #FGD_UV_LUMA_MULT - add lr, r10, #FGD_UV_MULT - ldrh r10, [r10, #FGD_UV_OFFSET] // uv_offset - vld1.16 {d30[]}, [r12] // uv_luma_mult - lsl r10, r10, r7 // uv_offset << bitdepth_min_8 - vld1.16 {d30[1]}, [lr] // uv_mult - - ldr lr, [r4, #FGD_SCALING_SHIFT] - ldr r12, [r4, #FGD_CLIP_TO_RESTRICTED_RANGE] - eor lr, lr, #15 // 15 - scaling_shift - - vmov.16 d30[2], r10 // uv_offset << bitdepth_min_8 - - cmp r12, #0 - vdup.16 q13, lr // 15 - scaling_shift - - beq 1f - // clip - cmp r11, #0 - mov r8, #16 - mov r9, #240 - lsl r8, r8, r7 - lsl r9, r9, r7 - beq 2f - // is_id - mov r9, #235 - lsl r9, r9, r7 - b 2f -1: - // no clip - mov r8, #0 - mov r9, r6 // bitdepth_max -2: - vmov.16 d30[3], r6 // bitdepth_max - vdup.16 d31, r8 // clip_min - - mov r10, #GRAIN_WIDTH*2 // grain_lut stride - -.if \sy - mov r6, #23 - mov r7, #22 -.else - mov r6, #27 - mov r7, #17 -.endif - vmov.16 d31[1], r9 // clip_max - - ldrd r8, r9, [sp, #116] // offsets, h - - add r5, r5, #(2*(3 + (2 >> \sx)*3)) // grain_lut += 9 or 6 -.if \sy - add r5, r5, r10, lsl #2 // grain_lut += 4 * grain_stride - add r5, r5, r10, lsl #1 // grain_lut += 2 * grain_stride -.else - add r5, r5, r10, lsl #3 // grain_lut += 8 * grain_stride - add r5, r5, r10 // grain_lut += grain_stride -.endif - vmov.16 d31[2], r6 // overlap y [0] - - ldr r12, [r8, #8] // offsets[1][0] - calc_offset r12, r4, r12, \sx, \sy - add_offset r4, r12, r4, r5, r10 - - ldr r12, [r8, #4] // offsets[0][1] - calc_offset r12, lr, r12, \sx, \sy - add_offset lr, r12, lr, r5, r10 - - ldr r12, [r8, #12] // offsets[1][1] - calc_offset r12, r11, r12, \sx, \sy - add_offset r11, r12, r11, r5, r10 - - ldr r8, [r8] // offsets[0][0] - calc_offset r8, r12, r8, \sx, \sy - add_offset r5, r8, r12, r5, r10 - - vmov.16 d31[3], r7 // overlap y [1] - - add r4, r4, #2*(32 >> \sx) // grain_lut += BLOCK_SIZE * bx - add r8, lr, r10, lsl #(5 - \sy) // grain_lut += grain_stride * BLOCK_SIZE * by - add r11, r11, r10, lsl #(5 - \sy) // grain_lut += grain_stride * BLOCK_SIZE * by - add r11, r11, #2*(32 >> \sx) // grain_lut += BLOCK_SIZE * bx - - movrel_local r12, overlap_coeffs_\sx - ldr lr, [sp, #132] // type - ldrd r6, r7, [sp, #108] // luma_row, luma_stride - - vld1.16 {d24, d25}, [r12, :128] // overlap_coeffs - - movrel_local r12, L(fguv_loop_sx\sx\()_tbl) -#if CONFIG_THUMB - // This uses movrel_local instead of adr above, because the target - // can be out of range for adr. But movrel_local leaves the thumb bit - // set on COFF (but probably wouldn't if building for thumb on ELF), - // thus try to clear the bit for robustness. - bic r12, r12, #1 -#endif - - tst lr, #1 - ldr lr, [r12, lr, lsl #2] - - add r12, r12, lr - - beq 1f - // y overlap - sub lr, r9, #(2 >> \sy) // backup remaining h - mov r9, #(2 >> \sy) - -1: -.if \sy - add r7, r7, r7 // luma_stride *= 2 -.endif - sub r7, r7, #32 // luma_stride -= 32 - - bx r12 -endfunc -.endm - -fguv 420, 1, 1 -fguv 422, 1, 0 -fguv 444, 0, 0 - -function fguv_loop_sx0_neon -L(fguv_loop_sx0_tbl): - .word L(fguv_loop_sx0_csfl0_00) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB - .word L(fguv_loop_sx0_csfl0_01) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB - .word L(fguv_loop_sx0_csfl0_10) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB - .word L(fguv_loop_sx0_csfl0_11) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB - .word L(fguv_loop_sx0_csfl1_00) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB - .word L(fguv_loop_sx0_csfl1_01) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB - .word L(fguv_loop_sx0_csfl1_10) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB - .word L(fguv_loop_sx0_csfl1_11) - L(fguv_loop_sx0_tbl) + CONFIG_THUMB - -.macro fguv_loop_sx0 csfl, ox, oy -L(fguv_loop_sx0_csfl\csfl\()_\ox\oy): - sub r2, r2, #32 // src_stride -= 32 - sub r10, r10, #32 // grain_stride -= 32 -.if \oy - mov r12, lr -.endif -L(fguv_loop_sx0_csfl\csfl\()_\ox\oy\()_loopstart): -1: -.if \ox - vld1.16 {d0}, [r4], r10 // grain_lut old -.endif -.if \oy - vld1.16 {q2, q3}, [r8]! // grain_lut top -.endif -.if \ox && \oy - vld1.16 {d2}, [r11], r10 // grain_lut top old -.endif -.if !\ox && !\oy - vld1.16 {q0, q1}, [r6, :128]! // luma -.endif - vld1.16 {q8, q9}, [r5]! // grain_lut -.if \oy - vld1.16 {q4, q5}, [r8], r10 // grain_lut top -.endif -.if !\ox && !\oy - vld1.16 {q2, q3}, [r6, :128], r7 // luma -.endif -.if \oy - vdup.16 d28, d31[2] // overlap y coeff - vdup.16 d29, d31[3] // overlap y coeff -.endif - vld1.16 {q10, q11}, [r5], r10 // grain_lut - -.if \ox - vdup.16 q7, d30[3] // bitdepth_max - add r4, r4, #32 - vmull.s16 q0, d0, d24 - vshr.u16 q7, q7, #1 // grain_max - vmlal.s16 q0, d16, d25 - vmvn q6, q7 // grain_min -.endif - -.if \oy -.if \ox - add r11, r11, #32 - vmull.s16 q1, d2, d24 - vmlal.s16 q1, d4, d25 - vqrshrn.s32 d16, q0, #5 - vqrshrn.s32 d4, q1, #5 - vmin.s16 d4, d4, d14 - vmin.s16 d16, d16, d14 - vmax.s16 d4, d4, d12 - vmax.s16 d16, d16, d12 -.endif - - vmull.s16 q0, d4, d28 - vmull.s16 q1, d5, d28 - vmull.s16 q2, d6, d28 - vmull.s16 q3, d7, d28 -.if !\ox - vdup.16 q7, d30[3] // bitdepth_max -.endif - vmlal.s16 q0, d16, d29 - vmlal.s16 q1, d17, d29 - vmlal.s16 q2, d18, d29 - vmlal.s16 q3, d19, d29 -.if !\ox - vshr.u16 q7, q7, #1 // grain_max -.endif - vmull.s16 q8, d20, d29 - vmull.s16 q9, d21, d29 - vmull.s16 q10, d22, d29 - vmull.s16 q11, d23, d29 -.if !\ox - vmvn q6, q7 // grain_min -.endif - vmlal.s16 q8, d8, d28 - vmlal.s16 q9, d9, d28 - vmlal.s16 q10, d10, d28 - vmlal.s16 q11, d11, d28 - vqrshrn.s32 d0, q0, #5 - vqrshrn.s32 d1, q1, #5 - vqrshrn.s32 d2, q2, #5 - vqrshrn.s32 d3, q3, #5 - vqrshrn.s32 d4, q8, #5 - vqrshrn.s32 d5, q9, #5 - vqrshrn.s32 d6, q10, #5 - vqrshrn.s32 d7, q11, #5 - vmin.s16 q8, q0, q7 - vmin.s16 q9, q1, q7 - vld1.16 {q0, q1}, [r6, :128]! // luma - vmin.s16 q10, q2, q7 - vmin.s16 q11, q3, q7 - vmax.s16 q8, q8, q6 - vmax.s16 q9, q9, q6 - vld1.16 {q2, q3}, [r6, :128], r7 // luma - vmax.s16 q10, q10, q6 - vmax.s16 q11, q11, q6 -.elseif \ox - vqrshrn.s32 d16, q0, #5 - vld1.16 {q0, q1}, [r6, :128]! // luma - vmin.s16 d16, d16, d14 - vld1.16 {q2, q3}, [r6, :128], r7 // luma - vmax.s16 d16, d16, d12 -.endif - -.if !\csfl - vdup.16 d28, d30[0] // uv_luma_mult - vld1.16 {q4, q5}, [r1, :128]! // src - vdup.16 d29, d30[1] // uv_mult - vmull.s16 q6, d0, d28 - vmull.s16 q7, d1, d28 - vmull.s16 q0, d2, d28 - vmull.s16 q1, d3, d28 - vmlal.s16 q6, d8, d29 - vmlal.s16 q7, d9, d29 - vmlal.s16 q0, d10, d29 - vmlal.s16 q1, d11, d29 - vld1.16 {q4, q5}, [r1, :128] // src - sub r1, r1, #32 - vshrn.s32 d12, q6, #6 - vshrn.s32 d13, q7, #6 - vshrn.s32 d14, q0, #6 - vshrn.s32 d15, q1, #6 - vmull.s16 q0, d4, d28 - vmull.s16 q1, d5, d28 - vmull.s16 q2, d6, d28 - vmull.s16 q3, d7, d28 - vmlal.s16 q0, d8, d29 - vmlal.s16 q1, d9, d29 - vmlal.s16 q2, d10, d29 - vmlal.s16 q3, d11, d29 - vdup.16 q14, d30[2] // uv_offset - vshrn.s32 d0, q0, #6 - vshrn.s32 d1, q1, #6 - vshrn.s32 d2, q2, #6 - vshrn.s32 d3, q3, #6 - vdup.16 q4, d30[3] // bitdepth_max - vmov.i16 q5, #0 - vadd.i16 q6, q6, q14 - vadd.i16 q7, q7, q14 - vadd.i16 q2, q0, q14 - vadd.i16 q3, q1, q14 - vmin.s16 q0, q6, q4 - vmin.s16 q1, q7, q4 - vmin.s16 q2, q2, q4 - vmin.s16 q3, q3, q4 - vmax.s16 q0, q0, q5 - vmax.s16 q1, q1, q5 - vmax.s16 q2, q2, q5 - vmax.s16 q3, q3, q5 -.else - vdup.16 q14, d30[3] // bitdepth_max - // Make sure that uninitialized pixels out of range past the right - // edge are in range; their actual values shouldn't matter. - vand q0, q0, q14 - vand q1, q1, q14 - vand q2, q2, q14 - vand q3, q3, q14 -.endif - - bl gather32_neon - - vld1.16 {q0, q1}, [r1, :128]! // src - - vmovl.u8 q6, d8 // scaling - vmovl.u8 q7, d9 - vmovl.u8 q4, d10 - vmovl.u8 q5, d11 - - vld1.16 {q2, q3}, [r1, :128], r2 // src - - vshl.u16 q6, q6, q13 // scaling << (15 - scaling_shift) - vshl.u16 q7, q7, q13 - vshl.u16 q4, q4, q13 - vshl.u16 q5, q5, q13 - - vqrdmulh.s16 q8, q8, q6 // round2((scaling << (15 - scaling_shift) * grain, 15) - vqrdmulh.s16 q9, q9, q7 - vqrdmulh.s16 q10, q10, q4 - vqrdmulh.s16 q11, q11, q5 - - - vdup.16 q4, d31[0] // clip_min - vdup.16 q5, d31[1] // clip_max - - vqadd.s16 q0, q0, q8 // *src + noise - vqadd.s16 q1, q1, q9 - vqadd.s16 q2, q2, q10 - vqadd.s16 q3, q3, q11 - -.if \oy - vmov.32 lr, d25[0] // 2 first 16 bit coeffs from overlap x -.endif - - vmax.s16 q0, q0, q4 - vmax.s16 q1, q1, q4 - vmax.s16 q2, q2, q4 - vmax.s16 q3, q3, q4 - vmin.s16 q0, q0, q5 - vmin.s16 q1, q1, q5 - vmin.s16 q2, q2, q5 - vmin.s16 q3, q3, q5 - - vst1.16 {q0, q1}, [r0, :128]! // dst - - subs r9, r9, #1 -.if \oy - vmov.32 d31[1], lr // new coeffs for overlap y -.endif - - vst1.16 {q2, q3}, [r0, :128], r2 // dst - bgt 1b - -.if \oy - cmp r12, #0 - mov r9, r12 // restore actual remaining h - bgt L(fguv_loop_sx0_csfl\csfl\()_\ox\()0_loopstart) -.endif - b 9f -.endm - fguv_loop_sx0 0, 0, 0 - fguv_loop_sx0 0, 0, 1 - fguv_loop_sx0 0, 1, 0 - fguv_loop_sx0 0, 1, 1 - fguv_loop_sx0 1, 0, 0 - fguv_loop_sx0 1, 0, 1 - fguv_loop_sx0 1, 1, 0 - fguv_loop_sx0 1, 1, 1 - -9: - vpop {q4-q7} - pop {r4-r11,pc} -endfunc - -function fguv_loop_sx1_neon -L(fguv_loop_sx1_tbl): - .word L(fguv_loop_sx1_csfl0_00) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB - .word L(fguv_loop_sx1_csfl0_01) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB - .word L(fguv_loop_sx1_csfl0_10) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB - .word L(fguv_loop_sx1_csfl0_11) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB - .word L(fguv_loop_sx1_csfl1_00) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB - .word L(fguv_loop_sx1_csfl1_01) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB - .word L(fguv_loop_sx1_csfl1_10) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB - .word L(fguv_loop_sx1_csfl1_11) - L(fguv_loop_sx1_tbl) + CONFIG_THUMB - -.macro fguv_loop_sx1 csfl, ox, oy -L(fguv_loop_sx1_csfl\csfl\()_\ox\oy): -.if \oy - mov r12, lr -.endif -1: -.if \ox - vld1.16 {d0}, [r4], r10 // grain_lut old -.endif -.if \ox && \oy - vld1.16 {d2}, [r11], r10 // grain_lut top old -.endif -.if \oy - vld1.16 {q2, q3}, [r8], r10 // grain_lut top -.endif -.if !\ox && !\oy - vld1.16 {q0, q1}, [r6, :128]! // luma -.endif - vld1.16 {q8, q9}, [r5], r10 // grain_lut -.if \oy - vdup.16 d28, d31[2] // overlap y coeff - vdup.16 d29, d31[3] // overlap y coeff -.endif -.if !\ox && !\oy - vld1.16 {q2, q3}, [r6, :128], r7 // luma -.endif - -.if \ox - vdup.16 q7, d30[3] // bitdepth_max - vmull.s16 q0, d0, d24 - vshr.u16 q7, q7, #1 // grain_max - vmlal.s16 q0, d16, d25 - vmvn q6, q7 // grain_min -.endif - -.if \oy -.if \ox - vmull.s16 q1, d2, d24 - vmlal.s16 q1, d4, d25 - vqrshrn.s32 d16, q0, #5 - vqrshrn.s32 d4, q1, #5 - vmin.s16 d4, d4, d14 - vmin.s16 d16, d16, d14 - vmax.s16 d4, d4, d12 - vmax.s16 d16, d16, d12 -.endif - - vmull.s16 q0, d4, d28 - vmull.s16 q1, d5, d28 - vmull.s16 q2, d6, d28 - vmull.s16 q3, d7, d28 -.if !\ox - vdup.16 q7, d30[3] // bitdepth_max -.endif - vmlal.s16 q0, d16, d29 - vmlal.s16 q1, d17, d29 - vmlal.s16 q2, d18, d29 - vmlal.s16 q3, d19, d29 -.if !\ox - vshr.u16 q7, q7, #1 // grain_max -.endif - vqrshrn.s32 d16, q0, #5 - vqrshrn.s32 d17, q1, #5 - vqrshrn.s32 d18, q2, #5 - vqrshrn.s32 d19, q3, #5 -.if !\ox - vmvn q6, q7 // grain_min -.endif - vld1.16 {q0, q1}, [r6, :128]! // luma - vmin.s16 q8, q8, q7 - vmin.s16 q9, q9, q7 - vmax.s16 q8, q8, q6 - vmax.s16 q9, q9, q6 - vld1.16 {q2, q3}, [r6, :128], r7 // luma -.elseif \ox - vqrshrn.s32 d16, q0, #5 - vld1.16 {q0, q1}, [r6, :128]! // luma - vmin.s16 d16, d16, d14 - vld1.16 {q2, q3}, [r6, :128], r7 // luma - vmax.s16 d16, d16, d12 -.endif - - vpadd.i16 d0, d0, d1 - vpadd.i16 d1, d2, d3 - vpadd.i16 d2, d4, d5 - vpadd.i16 d3, d6, d7 - vrshr.u16 q0, q0, #1 - vrshr.u16 q1, q1, #1 -.if !\csfl - vdup.16 d28, d30[0] // uv_luma_mult - vld1.16 {q2, q3}, [r1, :128], r2 // src - vdup.16 d29, d30[1] // uv_mult - vmull.s16 q6, d0, d28 - vmull.s16 q7, d1, d28 - vmull.s16 q0, d2, d28 - vmull.s16 q1, d3, d28 - vmlal.s16 q6, d4, d29 - vmlal.s16 q7, d5, d29 - vmlal.s16 q0, d6, d29 - vmlal.s16 q1, d7, d29 - vshrn.s32 d12, q6, #6 - vshrn.s32 d13, q7, #6 - vshrn.s32 d14, q0, #6 - vshrn.s32 d15, q1, #6 - vdup.16 q14, d30[2] // uv_offset - vdup.16 q4, d30[3] // bitdepth_max - vmov.i16 q5, #0 - vadd.i16 q6, q6, q14 - vadd.i16 q7, q7, q14 - vmin.s16 q0, q6, q4 - vmin.s16 q1, q7, q4 - vmax.s16 q0, q0, q5 - vmax.s16 q1, q1, q5 -.else - vdup.16 q14, d30[3] // bitdepth_max - vld1.16 {q2, q3}, [r1, :128], r2 // src - - // Make sure that uninitialized pixels out of range past the right - // edge are in range; their actual values shouldn't matter. - vand q0, q0, q14 - vand q1, q1, q14 -.endif - - bl gather16_neon - - vmovl.u8 q6, d8 // scaling - vmovl.u8 q7, d9 - - vshl.u16 q6, q6, q13 // scaling << (15 - scaling_shift) - vshl.u16 q7, q7, q13 - - vqrdmulh.s16 q8, q8, q6 // round2((scaling << (15 - scaling_shift) * grain, 15) - vqrdmulh.s16 q9, q9, q7 - - - vdup.16 q4, d31[0] // clip_min - vdup.16 q5, d31[1] // clip_max - - vqadd.s16 q0, q2, q8 // *src + noise - vqadd.s16 q1, q3, q9 - -.if \oy - // Swap the two last coefficients of d31, place them first in d28 - vrev64.16 d28, d31 -.endif - - vmax.s16 q0, q0, q4 - vmax.s16 q1, q1, q4 - vmin.s16 q0, q0, q5 - vmin.s16 q1, q1, q5 - - subs r9, r9, #1 -.if \oy - // Take the first two 16 bit coefficients of d28 and place them at the - // end of d31 - vtrn.32 d31, d28 -.endif - - vst1.16 {q0, q1}, [r0, :128], r2 // dst - bgt 1b - -.if \oy - cmp r12, #0 - mov r9, r12 // restore actual remaining h - bgt L(fguv_loop_sx1_csfl\csfl\()_\ox\()0) -.endif - - b 9f -.endm - fguv_loop_sx1 0, 0, 0 - fguv_loop_sx1 0, 0, 1 - fguv_loop_sx1 0, 1, 0 - fguv_loop_sx1 0, 1, 1 - fguv_loop_sx1 1, 0, 0 - fguv_loop_sx1 1, 0, 1 - fguv_loop_sx1 1, 1, 0 - fguv_loop_sx1 1, 1, 1 - -9: - vpop {q4-q7} - pop {r4-r11,pc} -endfunc diff --git a/third_party/dav1d/src/arm/64/film_grain.S b/third_party/dav1d/src/arm/64/film_grain.S deleted file mode 100644 --- a/third_party/dav1d/src/arm/64/film_grain.S +++ /dev/null @@ -1,1983 +0,0 @@ -/* - * Copyright © 2021, VideoLAN and dav1d authors - * Copyright © 2021, Martin Storsjo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include "src/arm/asm.S" -#include "util.S" -#include "src/arm/asm-offsets.h" - -#define GRAIN_WIDTH 82 -#define GRAIN_HEIGHT 73 - -#define SUB_GRAIN_WIDTH 44 -#define SUB_GRAIN_HEIGHT 38 - -.macro increment_seed steps, shift=1 - lsr w11, w2, #3 - lsr w12, w2, #12 - lsr w13, w2, #1 - eor w11, w2, w11 // (r >> 0) ^ (r >> 3) - eor w12, w12, w13 // (r >> 12) ^ (r >> 1) - eor w11, w11, w12 // (r >> 0) ^ (r >> 3) ^ (r >> 12) ^ (r >> 1) -.if \shift - lsr w2, w2, #\steps -.endif - and w11, w11, #((1 << \steps) - 1) // bit -.if \shift - orr w2, w2, w11, lsl #(16 - \steps) // *state -.else - orr w2, w2, w11, lsl #16 // *state -.endif -.endm - -.macro read_rand dest, bits, age - ubfx \dest, x2, #16 - \bits - \age, #\bits -.endm - -.macro read_shift_rand dest, bits - ubfx \dest, x2, #17 - \bits, #\bits - lsr w2, w2, #1 -.endm - -// special calling convention: -// w2 holds seed -// x3 holds dav1d_gaussian_sequence -// clobbers x11-x15 -// returns in v0.8h -function get_gaussian_neon - increment_seed 4 - read_rand x14, 11, 3 - read_rand x15, 11, 2 - add x14, x3, x14, lsl #1 - add x15, x3, x15, lsl #1 - ld1 {v0.h}[0], [x14] - read_rand x14, 11, 1 - ld1 {v0.h}[1], [x15] - add x14, x3, x14, lsl #1 - read_rand x15, 11, 0 - increment_seed 4 - add x15, x3, x15, lsl #1 - ld1 {v0.h}[2], [x14] - read_rand x14, 11, 3 - ld1 {v0.h}[3], [x15] - add x14, x3, x14, lsl #1 - read_rand x15, 11, 2 - ld1 {v0.h}[4], [x14] - add x15, x3, x15, lsl #1 - read_rand x14, 11, 1 - ld1 {v0.h}[5], [x15] - read_rand x15, 11, 0 - add x14, x3, x14, lsl #1 - add x15, x3, x15, lsl #1 - ld1 {v0.h}[6], [x14] - ld1 {v0.h}[7], [x15] - ret -endfunc - -.macro get_grain_row r0, r1, r2, r3, r4, r5 - bl get_gaussian_neon - srshl \r5\().8h, v0.8h, v31.8h - xtn \r0\().8b, \r5\().8h - bl get_gaussian_neon - srshl \r5\().8h, v0.8h, v31.8h - xtn2 \r0\().16b, \r5\().8h - bl get_gaussian_neon - srshl \r5\().8h, v0.8h, v31.8h - xtn \r1\().8b, \r5\().8h - bl get_gaussian_neon - srshl \r5\().8h, v0.8h, v31.8h - xtn2 \r1\().16b, \r5\().8h - bl get_gaussian_neon - srshl \r5\().8h, v0.8h, v31.8h - xtn \r2\().8b, \r5\().8h - bl get_gaussian_neon - srshl \r5\().8h, v0.8h, v31.8h - xtn2 \r2\().16b, \r5\().8h - bl get_gaussian_neon - srshl \r5\().8h, v0.8h, v31.8h - xtn \r3\().8b, \r5\().8h - bl get_gaussian_neon - srshl \r5\().8h, v0.8h, v31.8h - xtn2 \r3\().16b, \r5\().8h - bl get_gaussian_neon - srshl \r5\().8h, v0.8h, v31.8h - xtn \r4\().8b, \r5\().8h - bl get_gaussian_neon - srshl \r5\().8h, v0.8h, v31.8h - xtn2 \r4\().16b, \r5\().8h - increment_seed 2 - read_rand x14, 11, 1 - read_rand x15, 11, 0 - add x14, x3, x14, lsl #1 - add x15, x3, x15, lsl #1 - ld1 {\r5\().h}[0], [x14] - ld1 {\r5\().h}[1], [x15] - srshl v0.4h, \r5\().4h, v31.4h - xtn \r5\().8b, v0.8h -.endm - -.macro store_grain_row r0, r1, r2, r3, r4, r5 - st1 {\r0\().16b,\r1\().16b}, [x0], #32 - st1 {\r2\().16b,\r3\().16b}, [x0], #32 - st1 {\r4\().16b}, [x0], #16 - st1 {\r5\().h}[0], [x0], #2 -.endm - -.macro get_grain_row_44 r0, r1, r2 - bl get_gaussian_neon - srshl \r2\().8h, v0.8h, v31.8h - xtn \r0\().8b, \r2\().8h - bl get_gaussian_neon - srshl \r2\().8h, v0.8h, v31.8h - xtn2 \r0\().16b, \r2\().8h - bl get_gaussian_neon - srshl \r2\().8h, v0.8h, v31.8h - xtn \r1\().8b, \r2\().8h - bl get_gaussian_neon - srshl \r2\().8h, v0.8h, v31.8h - xtn2 \r1\().16b, \r2\().8h - bl get_gaussian_neon - srshl \r2\().8h, v0.8h, v31.8h - xtn \r2\().8b, \r2\().8h - - increment_seed 4 - read_rand x14, 11, 3 - read_rand x15, 11, 2 - add x14, x3, x14, lsl #1 - add x15, x3, x15, lsl #1 - ld1 {v0.h}[0], [x14] - read_rand x14, 11, 1 - ld1 {v0.h}[1], [x15] - read_rand x15, 11, 0 - add x14, x3, x14, lsl #1 - add x15, x3, x15, lsl #1 - ld1 {v0.h}[2], [x14] - ld1 {v0.h}[3], [x15] - srshl v0.4h, v0.4h, v31.4h - xtn2 \r2\().16b, v0.8h -.endm - -.macro store_grain_row_44 r0, r1, r2 - st1 {\r0\().16b,\r1\().16b}, [x0], #32 - st1 {\r2\().16b}, [x0] - add x0, x0, #GRAIN_WIDTH-32 -.endm - -function get_grain_2_neon - increment_seed 2 - read_rand x14, 11, 1 - read_rand x15, 11, 0 - add x14, x3, x14, lsl #1 - add x15, x3, x15, lsl #1 - ld1 {v0.h}[0], [x14] - ld1 {v0.h}[1], [x15] - srshl v0.4h, v0.4h, v31.4h - xtn v0.8b, v0.8h - ret -endfunc - -.macro get_grain_2 dst - bl get_grain_2_neon -.ifnc \dst, v0 - mov \dst\().8b, v0.8b -.endif -.endm - -// w15 holds the number of entries to produce -// w14, w16 and w17 hold the previous output entries -// v0 holds the vector of produced entries -// v1 holds the input vector of sums from above -.macro output_lag n -function output_lag\n\()_neon -1: - read_shift_rand x13, 11 - mov w11, v1.s[0] - ldrsh w12, [x3, x13, lsl #1] - ext v0.16b, v0.16b, v0.16b, #1 -.if \n == 1 - madd w11, w14, w4, w11 // sum (above) + *coeff * prev output -.elseif \n == 2 - madd w11, w16, w4, w11 // sum (above) + *coeff * prev output 1 - madd w11, w14, w17, w11 // += *coeff * prev output 2 - mov w16, w14 -.else - madd w11, w17, w4, w11 // sum (above) + *coeff * prev output 1 - madd w11, w16, w20, w11 // sum (above) + *coeff * prev output 2 - madd w11, w14, w21, w11 // += *coeff * prev output 3 - mov w17, w16 - mov w16, w14 -.endif - add w14, w11, w8 // 1 << (ar_coeff_shift - 1) - add w12, w12, w10 // 1 << (4 + grain_scale_shift - 1) - asr w14, w14, w7 // >> ar_coeff_shift - asr w12, w12, w9 // >> (4 + grain_scale_shift) - add w14, w14, w12 - cmp w14, w5 - csel w14, w14, w5, le - cmp w14, w6 - csel w14, w14, w6, ge - subs w15, w15, #1 - ext v1.16b, v1.16b, v1.16b, #4 - ins v0.b[15], w14 - b.gt 1b - ret -endfunc -.endm - -output_lag 1 -output_lag 2 -output_lag 3 - - -function sum_lag1_above_neon - smull v2.8h, v3.8b, v28.8b - smull2 v3.8h, v3.16b, v28.16b - smull v4.8h, v0.8b, v27.8b - smull2 v5.8h, v0.16b, v27.16b - smull v6.8h, v1.8b, v29.8b - smull2 v7.8h, v1.16b, v29.16b - saddl v0.4s, v2.4h, v4.4h - saddl2 v1.4s, v2.8h, v4.8h - saddl v2.4s, v3.4h, v5.4h - saddl2 v3.4s, v3.8h, v5.8h - saddw v4.4s, v0.4s, v6.4h - saddw2 v5.4s, v1.4s, v6.8h - saddw v6.4s, v2.4s, v7.4h - saddw2 v7.4s, v3.4s, v7.8h - ret -endfunc - -.macro sum_lag_n_body lag, type, uv_layout, edge, elems, store, uv_coeff - bl sum_\lag\()_above_neon -.ifc \type, uv_420 - add x12, x19, #GRAIN_WIDTH - ld1 {v22.16b, v23.16b}, [x19], #32 - ld1 {v24.16b, v25.16b}, [x12] - saddlp v22.8h, v22.16b - saddlp v23.8h, v23.16b - saddlp v24.8h, v24.16b - saddlp v25.8h, v25.16b - add v22.8h, v22.8h, v24.8h - add v23.8h, v23.8h, v25.8h - rshrn v0.8b, v22.8h, #2 - rshrn2 v0.16b, v23.8h, #2 -.endif -.ifc \type, uv_422 - ld1 {v22.16b, v23.16b}, [x19], #32 - saddlp v22.8h, v22.16b - saddlp v23.8h, v23.16b - rshrn v0.8b, v22.8h, #1 - rshrn2 v0.16b, v23.8h, #1 -.endif -.ifc \type, uv_444 - ld1 {v0.16b}, [x19], #16 -.endif -.if \uv_layout -.ifnb \uv_coeff - dup v1.16b, \uv_coeff - smull v2.8h, v0.8b, v1.8b - smull2 v3.8h, v0.16b, v1.16b -.else - smull v2.8h, v0.8b, v30.8b - smull2 v3.8h, v0.16b, v30.16b -.endif - saddw v4.4s, v4.4s, v2.4h - saddw2 v5.4s, v5.4s, v2.8h - saddw v6.4s, v6.4s, v3.4h - saddw2 v7.4s, v7.4s, v3.8h -.endif -.if \uv_layout && \elems == 16 - b sum_\lag\()_y_\edge\()_start -.elseif \uv_layout == 444 && \elems == 15 - b sum_\lag\()_y_\edge\()_start -.elseif \uv_layout == 422 && \elems == 9 - b sum_\lag\()_uv_420_\edge\()_start -.else -sum_\lag\()_\type\()_\edge\()_start: -.ifc \edge, left - increment_seed 4 - read_rand x12, 11, 3 - read_rand x13, 11, 2 - read_rand x14, 11, 1 - add x12, x3, x12, lsl #1 - add x13, x3, x13, lsl #1 - add x14, x3, x14, lsl #1 - ld1 {v0.h}[5], [x12] - ld1 {v0.h}[6], [x13] - ld1 {v0.h}[7], [x14] - lsl x2, x2, #1 // shift back the state as if we'd done increment_seed with shift=0 - srshl v0.8h, v0.8h, v31.8h - xtn2 v0.16b, v0.8h - ext v4.16b, v4.16b, v4.16b, #12 -.ifc \lag, lag3 - smov w17, v0.b[13] -.endif -.ifnc \lag, lag1 - smov w16, v0.b[14] -.endif - smov w14, v0.b[15] - - mov v1.16b, v4.16b - mov w15, #1 - bl output_\lag\()_neon -.else - increment_seed 4, shift=0 - mov v1.16b, v4.16b - mov w15, #4 - bl output_\lag\()_neon -.endif - - increment_seed 4, shift=0 - mov v1.16b, v5.16b - mov w15, #4 - bl output_\lag\()_neon - - increment_seed 4, shift=0 - mov v1.16b, v6.16b -.if \elems == 9 - mov w15, #1 - bl output_\lag\()_neon - lsr w2, w2, #3 - - read_rand x12, 11, 2 - read_rand x13, 11, 1 - read_rand x14, 11, 0 - add x12, x3, x12, lsl #1 - add x13, x3, x13, lsl #1 - add x14, x3, x14, lsl #1 - ld1 {v1.h}[0], [x12] - ld1 {v1.h}[1], [x13] - ld1 {v1.h}[2], [x14] - srshl v1.4h, v1.4h, v31.4h - xtn v1.8b, v1.8h - ext v0.16b, v0.16b, v1.16b, #7 -.else - mov w15, #4 - bl output_\lag\()_neon - - increment_seed 4, shift=0 - mov v1.16b, v7.16b - -.ifc \edge, right - mov w15, #3 - bl output_\lag\()_neon - read_shift_rand x15, 11 - add x15, x3, x15, lsl #1 - ld1 {v1.h}[0], [x15] - srshl v1.4h, v1.4h, v31.4h - ext v0.16b, v0.16b, v1.16b, #1 -.else - mov w15, #4 - bl output_\lag\()_neon -.endif -.endif -.if \store - st1 {v0.16b}, [x0], #16 -.endif - ldr x30, [sp], #16 - ret -.endif -.endm - -.macro sum_lag1_func type, uv_layout, edge, elems=16 -function sum_\type\()_lag1_\edge\()_neon - str x30, [sp, #-16]! - sum_lag_n_body lag1, \type, \uv_layout, \edge, \elems, store=0 -endfunc -.endm - -sum_lag1_func y, 0, left -sum_lag1_func y, 0, mid -sum_lag1_func y, 0, right, 15 -sum_lag1_func uv_444, 444, left -sum_lag1_func uv_444, 444, mid -sum_lag1_func uv_444, 444, right, 15 -sum_lag1_func uv_422, 422, left -sum_lag1_func uv_422, 422, mid -sum_lag1_func uv_422, 422, right, 9 -sum_lag1_func uv_420, 420, left -sum_lag1_func uv_420, 420, mid -sum_lag1_func uv_420, 420, right, 9 - -.macro sum_lag1 type, dst, left, mid, right, edge=mid - mov v3.16b, \mid\().16b - ext v0.16b, \left\().16b, \mid\().16b, #15 - ext v1.16b, \mid\().16b, \right\().16b, #1 - bl sum_\type\()_lag1_\edge\()_neon - mov \dst\().16b, v0.16b -.endm - -.macro sum_y_lag1 dst, left, mid, right, edge=mid - sum_lag1 y, \dst, \left, \mid, \right, \edge -.endm - -.macro sum_uv_444_lag1 dst, left, mid, right, edge=mid - sum_lag1 uv_444, \dst, \left, \mid, \right, \edge -.endm - -.macro sum_uv_422_lag1 dst, left, mid, right, edge=mid - sum_lag1 uv_422, \dst, \left, \mid, \right, \edge -.endm - -.macro sum_uv_420_lag1 dst, left, mid, right, edge=mid - sum_lag1 uv_420, \dst, \left, \mid, \right, \edge -.endm - - -function sum_lag2_above_neon - sub x12, x0, #2*GRAIN_WIDTH - 16 - sub x13, x0, #1*GRAIN_WIDTH - 16 - ld1 {v18.16b}, [x12] // load top right - ld1 {v21.16b}, [x13] - - ext v22.16b, v16.16b, v17.16b, #14 // top left, top mid - dup v26.16b, v30.b[0] - ext v23.16b, v16.16b, v17.16b, #15 - dup v27.16b, v30.b[1] - ext v0.16b, v17.16b, v18.16b, #1 // top mid, top right - dup v28.16b, v30.b[3] - ext v1.16b, v17.16b, v18.16b, #2 - dup v29.16b, v30.b[4] - - smull v2.8h, v22.8b, v26.8b - smull2 v3.8h, v22.16b, v26.16b - smull v4.8h, v23.8b, v27.8b - smull2 v5.8h, v23.16b, v27.16b - smull v6.8h, v0.8b, v28.8b - smull2 v7.8h, v0.16b, v28.16b - smull v0.8h, v1.8b, v29.8b - smull2 v1.8h, v1.16b, v29.16b - saddl v22.4s, v2.4h, v4.4h - saddl2 v23.4s, v2.8h, v4.8h - saddl v26.4s, v3.4h, v5.4h - saddl2 v27.4s, v3.8h, v5.8h - saddl v2.4s, v0.4h, v6.4h - saddl2 v3.4s, v0.8h, v6.8h - saddl v6.4s, v1.4h, v7.4h - saddl2 v7.4s, v1.8h, v7.8h - add v4.4s, v22.4s, v2.4s - add v5.4s, v23.4s, v3.4s - add v6.4s, v26.4s, v6.4s - add v7.4s, v27.4s, v7.4s - - ext v22.16b, v19.16b, v20.16b, #14 // top left, top mid - dup v26.16b, v30.b[5] - ext v23.16b, v19.16b, v20.16b, #15 - dup v27.16b, v30.b[6] - ext v0.16b, v20.16b, v21.16b, #1 // top mid, top right - dup v28.16b, v30.b[8] - ext v1.16b, v20.16b, v21.16b, #2 - dup v29.16b, v30.b[9] - - smull v2.8h, v22.8b, v26.8b - smull2 v3.8h, v22.16b, v26.16b - smull v22.8h, v23.8b, v27.8b - smull2 v23.8h, v23.16b, v27.16b - smull v26.8h, v0.8b, v28.8b - smull2 v27.8h, v0.16b, v28.16b - smull v28.8h, v1.8b, v29.8b - smull2 v29.8h, v1.16b, v29.16b - saddl v0.4s, v2.4h, v22.4h - saddl2 v1.4s, v2.8h, v22.8h - saddl v2.4s, v3.4h, v23.4h - saddl2 v3.4s, v3.8h, v23.8h - saddl v22.4s, v26.4h, v28.4h - saddl2 v23.4s, v26.8h, v28.8h - saddl v26.4s, v27.4h, v29.4h - saddl2 v27.4s, v27.8h, v29.8h - add v0.4s, v0.4s, v22.4s - add v1.4s, v1.4s, v23.4s - add v2.4s, v2.4s, v26.4s - add v3.4s, v3.4s, v27.4s - dup v26.16b, v30.b[2] - dup v27.16b, v30.b[7] - smull v22.8h, v17.8b, v26.8b - smull2 v23.8h, v17.16b, v26.16b - smull v24.8h, v20.8b, v27.8b - smull2 v25.8h, v20.16b, v27.16b - add v4.4s, v4.4s, v0.4s - add v5.4s, v5.4s, v1.4s - add v6.4s, v6.4s, v2.4s - add v7.4s, v7.4s, v3.4s - - mov v16.16b, v17.16b - mov v17.16b, v18.16b - - saddl v0.4s, v22.4h, v24.4h - saddl2 v1.4s, v22.8h, v24.8h - saddl v2.4s, v23.4h, v25.4h - saddl2 v3.4s, v23.8h, v25.8h - mov v19.16b, v20.16b - mov v20.16b, v21.16b - add v4.4s, v4.4s, v0.4s - add v5.4s, v5.4s, v1.4s - add v6.4s, v6.4s, v2.4s - add v7.4s, v7.4s, v3.4s - ret -endfunc - -.macro sum_lag2_func type, uv_layout, edge, elems=16 -function sum_\type\()_lag2_\edge\()_neon - str x30, [sp, #-16]! -.ifc \edge, left - sub x12, x0, #2*GRAIN_WIDTH - sub x13, x0, #1*GRAIN_WIDTH - ld1 {v17.16b}, [x12] // load the previous block right above - ld1 {v20.16b}, [x13] -.endif - sum_lag_n_body lag2, \type, \uv_layout, \edge, \elems, store=1, uv_coeff=v30.b[12] -endfunc -.endm - -sum_lag2_func y, 0, left -sum_lag2_func y, 0, mid -sum_lag2_func y, 0, right, 15 -sum_lag2_func uv_444, 444, left -sum_lag2_func uv_444, 444, mid -sum_lag2_func uv_444, 444, right, 15 -sum_lag2_func uv_422, 422, left -sum_lag2_func uv_422, 422, mid -sum_lag2_func uv_422, 422, right, 9 -sum_lag2_func uv_420, 420, left -sum_lag2_func uv_420, 420, mid -sum_lag2_func uv_420, 420, right, 9 - - -function sum_lag3_above_neon - sub x11, x0, #3*GRAIN_WIDTH - 16 - sub x12, x0, #2*GRAIN_WIDTH - 16 - sub x13, x0, #1*GRAIN_WIDTH - 16 - ld1 {v15.16b}, [x11] // load top right - ld1 {v18.16b}, [x12] - ld1 {v21.16b}, [x13] - - ext v8.16b, v13.16b, v14.16b, #13 // top left, top mid - dup v22.16b, v29.b[0] - ext v9.16b, v13.16b, v14.16b, #14 - dup v23.16b, v29.b[1] - ext v10.16b, v13.16b, v14.16b, #15 - dup v24.16b, v29.b[2] - dup v25.16b, v29.b[3] - ext v11.16b, v14.16b, v15.16b, #1 // top mid, top right - dup v26.16b, v29.b[4] - ext v12.16b, v14.16b, v15.16b, #2 - dup v27.16b, v29.b[5] - ext v13.16b, v14.16b, v15.16b, #3 - dup v28.16b, v29.b[6] - - smull v0.8h, v8.8b, v22.8b - smull2 v1.8h, v8.16b, v22.16b - smull v2.8h, v9.8b, v23.8b - smull2 v3.8h, v9.16b, v23.16b - smull v8.8h, v10.8b, v24.8b - smull2 v9.8h, v10.16b, v24.16b - smull v10.8h, v11.8b, v26.8b - smull2 v11.8h, v11.16b, v26.16b - saddl v22.4s, v0.4h, v2.4h - saddl2 v23.4s, v0.8h, v2.8h - saddl v24.4s, v1.4h, v3.4h - saddl2 v26.4s, v1.8h, v3.8h - saddl v0.4s, v8.4h, v10.4h - saddl2 v1.4s, v8.8h, v10.8h - saddl v2.4s, v9.4h, v11.4h - saddl2 v3.4s, v9.8h, v11.8h - smull v8.8h, v12.8b, v27.8b - smull2 v9.8h, v12.16b, v27.16b - smull v10.8h, v13.8b, v28.8b - smull2 v11.8h, v13.16b, v28.16b - smull v12.8h, v14.8b, v25.8b - smull2 v13.8h, v14.16b, v25.16b - add v4.4s, v22.4s, v0.4s - add v5.4s, v23.4s, v1.4s - add v6.4s, v24.4s, v2.4s - add v7.4s, v26.4s, v3.4s - saddl v0.4s, v8.4h, v10.4h - saddl2 v1.4s, v8.8h, v10.8h - saddl v2.4s, v9.4h, v11.4h - saddl2 v3.4s, v9.8h, v11.8h - add v4.4s, v4.4s, v0.4s - add v5.4s, v5.4s, v1.4s - add v6.4s, v6.4s, v2.4s - add v7.4s, v7.4s, v3.4s - saddw v4.4s, v4.4s, v12.4h - saddw2 v5.4s, v5.4s, v12.8h - saddw v6.4s, v6.4s, v13.4h - saddw2 v7.4s, v7.4s, v13.8h - - ext v8.16b, v16.16b, v17.16b, #13 // top left, top mid - dup v22.16b, v29.b[7] - ext v9.16b, v16.16b, v17.16b, #14 - dup v23.16b, v29.b[8] - ext v10.16b, v16.16b, v17.16b, #15 - dup v24.16b, v29.b[9] - dup v25.16b, v29.b[10] - ext v11.16b, v17.16b, v18.16b, #1 // top mid, top right - dup v26.16b, v29.b[11] - ext v12.16b, v17.16b, v18.16b, #2 - dup v27.16b, v29.b[12] - ext v13.16b, v17.16b, v18.16b, #3 - dup v28.16b, v29.b[13] - - smull v0.8h, v8.8b, v22.8b - smull2 v1.8h, v8.16b, v22.16b - smull v2.8h, v9.8b, v23.8b - smull2 v3.8h, v9.16b, v23.16b - smull v8.8h, v10.8b, v24.8b - smull2 v9.8h, v10.16b, v24.16b - smull v10.8h, v11.8b, v26.8b - smull2 v11.8h, v11.16b, v26.16b - saddl v22.4s, v0.4h, v2.4h - saddl2 v23.4s, v0.8h, v2.8h - saddl v24.4s, v1.4h, v3.4h - saddl2 v26.4s, v1.8h, v3.8h - saddl v0.4s, v8.4h, v10.4h - saddl2 v1.4s, v8.8h, v10.8h - saddl v2.4s, v9.4h, v11.4h - saddl2 v3.4s, v9.8h, v11.8h - smull v8.8h, v12.8b, v27.8b - smull2 v9.8h, v12.16b, v27.16b - smull v10.8h, v13.8b, v28.8b - smull2 v11.8h, v13.16b, v28.16b - smull v12.8h, v17.8b, v25.8b - smull2 v13.8h, v17.16b, v25.16b - add v22.4s, v22.4s, v0.4s - add v23.4s, v23.4s, v1.4s - add v24.4s, v24.4s, v2.4s - add v26.4s, v26.4s, v3.4s - saddl v0.4s, v8.4h, v10.4h - saddl2 v1.4s, v8.8h, v10.8h - saddl v2.4s, v9.4h, v11.4h - saddl2 v3.4s, v9.8h, v11.8h - add v4.4s, v4.4s, v22.4s - add v5.4s, v5.4s, v23.4s - add v6.4s, v6.4s, v24.4s - add v7.4s, v7.4s, v26.4s - add v4.4s, v4.4s, v0.4s - add v5.4s, v5.4s, v1.4s - add v6.4s, v6.4s, v2.4s - add v7.4s, v7.4s, v3.4s - saddw v4.4s, v4.4s, v12.4h - saddw2 v5.4s, v5.4s, v12.8h - saddw v6.4s, v6.4s, v13.4h - saddw2 v7.4s, v7.4s, v13.8h - - ext v8.16b, v19.16b, v20.16b, #13 // top left, top mid - dup v22.16b, v29.b[14] - ext v9.16b, v19.16b, v20.16b, #14 - dup v23.16b, v29.b[15] - ext v10.16b, v19.16b, v20.16b, #15 - dup v24.16b, v30.b[0] - dup v25.16b, v30.b[1] - ext v11.16b, v20.16b, v21.16b, #1 // top mid, top right - dup v26.16b, v30.b[2] - ext v12.16b, v20.16b, v21.16b, #2 - dup v27.16b, v30.b[3] - ext v13.16b, v20.16b, v21.16b, #3 - dup v28.16b, v30.b[4] - - smull v0.8h, v8.8b, v22.8b - smull2 v1.8h, v8.16b, v22.16b - smull v2.8h, v9.8b, v23.8b - smull2 v3.8h, v9.16b, v23.16b - smull v8.8h, v10.8b, v24.8b - smull2 v9.8h, v10.16b, v24.16b - smull v10.8h, v11.8b, v26.8b - smull2 v11.8h, v11.16b, v26.16b - saddl v22.4s, v0.4h, v2.4h - saddl2 v23.4s, v0.8h, v2.8h - saddl v24.4s, v1.4h, v3.4h - saddl2 v26.4s, v1.8h, v3.8h - saddl v0.4s, v8.4h, v10.4h - saddl2 v1.4s, v8.8h, v10.8h - saddl v2.4s, v9.4h, v11.4h - saddl2 v3.4s, v9.8h, v11.8h - smull v8.8h, v12.8b, v27.8b - smull2 v9.8h, v12.16b, v27.16b - smull v10.8h, v13.8b, v28.8b - smull2 v11.8h, v13.16b, v28.16b - smull v12.8h, v20.8b, v25.8b - smull2 v19.8h, v20.16b, v25.16b - add v22.4s, v22.4s, v0.4s - add v23.4s, v23.4s, v1.4s - add v24.4s, v24.4s, v2.4s - add v26.4s, v26.4s, v3.4s - saddl v0.4s, v8.4h, v10.4h - saddl2 v1.4s, v8.8h, v10.8h - saddl v2.4s, v9.4h, v11.4h - saddl2 v3.4s, v9.8h, v11.8h - add v4.4s, v4.4s, v22.4s - add v5.4s, v5.4s, v23.4s - add v6.4s, v6.4s, v24.4s - add v7.4s, v7.4s, v26.4s - mov v13.16b, v14.16b - mov v14.16b, v15.16b - add v4.4s, v4.4s, v0.4s - add v5.4s, v5.4s, v1.4s - add v6.4s, v6.4s, v2.4s - add v7.4s, v7.4s, v3.4s - mov v16.16b, v17.16b - mov v17.16b, v18.16b - saddw v4.4s, v4.4s, v12.4h - saddw2 v5.4s, v5.4s, v12.8h - saddw v6.4s, v6.4s, v19.4h - saddw2 v7.4s, v7.4s, v19.8h - - mov v19.16b, v20.16b - mov v20.16b, v21.16b - ret -endfunc - -.macro sum_lag3_func type, uv_layout, edge, elems=16 -function sum_\type\()_lag3_\edge\()_neon - str x30, [sp, #-16]! -.ifc \edge, left - sub x11, x0, #3*GRAIN_WIDTH - sub x12, x0, #2*GRAIN_WIDTH - sub x13, x0, #1*GRAIN_WIDTH - ld1 {v14.16b}, [x11] // load the previous block right above - ld1 {v17.16b}, [x12] - ld1 {v20.16b}, [x13] -.endif - sum_lag_n_body lag3, \type, \uv_layout, \edge, \elems, store=1, uv_coeff=v30.b[8] -endfunc -.endm - -sum_lag3_func y, 0, left -sum_lag3_func y, 0, mid -sum_lag3_func y, 0, right, 15 -sum_lag3_func uv_444, 444, left -sum_lag3_func uv_444, 444, mid -sum_lag3_func uv_444, 444, right, 15 -sum_lag3_func uv_422, 422, left -sum_lag3_func uv_422, 422, mid -sum_lag3_func uv_422, 422, right, 9 -sum_lag3_func uv_420, 420, left -sum_lag3_func uv_420, 420, mid -sum_lag3_func uv_420, 420, right, 9 - -function generate_grain_rows_neon - str x30, [sp, #-16]! -1: - get_grain_row v16, v17, v18, v19, v20, v21 - subs w1, w1, #1 - store_grain_row v16, v17, v18, v19, v20, v21 - b.gt 1b - ldr x30, [sp], #16 - ret -endfunc - -function generate_grain_rows_44_neon - str x30, [sp, #-16]! -1: - get_grain_row_44 v16, v17, v18 - subs w1, w1, #1 - store_grain_row_44 v16, v17, v18 - b.gt 1b - ldr x30, [sp], #16 - ret -endfunc - -function get_grain_row_neon - str x30, [sp, #-16]! - get_grain_row v16, v17, v18, v19, v20, v21 - ldr x30, [sp], #16 - ret -endfunc - -function get_grain_row_44_neon - str x30, [sp, #-16]! - get_grain_row_44 v16, v17, v18 - ldr x30, [sp], #16 - ret -endfunc - -function add_uv_444_coeff_lag0_neon -add_coeff_lag0_start: - smull v2.8h, v0.8b, v27.8b - smull2 v3.8h, v0.16b, v27.16b - srshl v2.8h, v2.8h, v28.8h - srshl v3.8h, v3.8h, v28.8h - saddw v2.8h, v2.8h, v1.8b - saddw2 v3.8h, v3.8h, v1.16b - sqxtn v2.8b, v2.8h - sqxtn2 v2.16b, v3.8h - ret -endfunc - -function add_uv_420_coeff_lag0_neon - ld1 {v4.16b, v5.16b}, [x19], #32 - ld1 {v6.16b, v7.16b}, [x12], #32 - saddlp v4.8h, v4.16b - saddlp v5.8h, v5.16b - saddlp v6.8h, v6.16b - saddlp v7.8h, v7.16b - add v4.8h, v4.8h, v6.8h - add v5.8h, v5.8h, v7.8h - rshrn v4.8b, v4.8h, #2 - rshrn2 v4.16b, v5.8h, #2 - and v0.16b, v4.16b, v0.16b - b add_coeff_lag0_start -endfunc - -function add_uv_422_coeff_lag0_neon - ld1 {v4.16b, v5.16b}, [x19], #32 - saddlp v4.8h, v4.16b - saddlp v5.8h, v5.16b - rshrn v4.8b, v4.8h, #1 - rshrn2 v4.16b, v5.8h, #1 - and v0.16b, v4.16b, v0.16b - b add_coeff_lag0_start -endfunc - -.macro gen_grain_82 type -function generate_grain_\type\()_8bpc_neon, export=1 - stp x30, x19, [sp, #-96]! - -.ifc \type, uv_444 - mov w13, w3 - mov w14, #28 - add x19, x1, #3*GRAIN_WIDTH - mov x1, x2 - mul w13, w13, w14 -.endif - movrel x3, X(gaussian_sequence) - ldr w2, [x1, #FGD_SEED] - ldr w9, [x1, #FGD_GRAIN_SCALE_SHIFT] -.ifc \type, y - add x4, x1, #FGD_AR_COEFFS_Y -.else - add x4, x1, #FGD_AR_COEFFS_UV -.endif - adr x16, L(gen_grain_\type\()_tbl) - ldr w17, [x1, #FGD_AR_COEFF_LAG] - add w9, w9, #4 - ldrh w17, [x16, w17, uxtw #1] - dup v31.8h, w9 // 4 + data->grain_scale_shift - sub x16, x16, w17, uxtw - neg v31.8h, v31.8h - -.ifc \type, uv_444 - cmp w13, #0 - mov w11, #0x49d8 - mov w14, #0xb524 - add x4, x4, w13, uxtw // Add offset to ar_coeffs_uv[1] - csel w11, w11, w14, ne -.endif - - ldr w7, [x1, #FGD_AR_COEFF_SHIFT] - mov w8, #1 - mov w10, #1 - lsl w8, w8, w7 // 1 << ar_coeff_shift - lsl w10, w10, w9 // 1 << (4 + data->grain_scale_shift) - lsr w8, w8, #1 // 1 << (ar_coeff_shift - 1) - lsr w10, w10, #1 // 1 << (4 + data->grain_scale_shift - 1) - mov w5, #127 - mov w6, #-128 - -.ifc \type, uv_444 - eor w2, w2, w11 -.endif - - br x16 - -L(generate_grain_\type\()_lag0): - AARCH64_VALID_JUMP_TARGET -.ifc \type, y - mov w1, #GRAIN_HEIGHT - bl generate_grain_rows_neon -.else - dup v28.8h, w7 - ld1r {v27.16b}, [x4] // ar_coeffs_uv[0] - movi v0.16b, #0 - movi v1.16b, #255 - ext v29.16b, v0.16b, v1.16b, #13 - ext v30.16b, v1.16b, v0.16b, #1 - neg v28.8h, v28.8h - - mov w1, #3 - bl generate_grain_rows_neon - mov w1, #GRAIN_HEIGHT-3 -1: - ld1 {v22.16b, v23.16b, v24.16b, v25.16b}, [x19], #64 - bl get_grain_row_neon - and v0.16b, v22.16b, v29.16b - mov v1.16b, v16.16b - bl add_uv_444_coeff_lag0_neon - mov v0.16b, v23.16b - mov v1.16b, v17.16b - mov v16.16b, v2.16b - bl add_uv_444_coeff_lag0_neon - ld1 {v26.16b}, [x19], #16 - mov v0.16b, v24.16b - mov v1.16b, v18.16b - mov v17.16b, v2.16b - bl add_uv_444_coeff_lag0_neon - add x19, x19, #2 - mov v0.16b, v25.16b - mov v1.16b, v19.16b - mov v18.16b, v2.16b - bl add_uv_444_coeff_lag0_neon - and v0.16b, v26.16b, v30.16b - mov v1.16b, v20.16b - mov v19.16b, v2.16b - bl add_uv_444_coeff_lag0_neon - mov v20.16b, v2.16b - subs w1, w1, #1 - store_grain_row v16, v17, v18, v19, v20, v21 - b.gt 1b -.endif - ldp x30, x19, [sp], #96 - ret - -L(generate_grain_\type\()_lag1): - AARCH64_VALID_JUMP_TARGET - ld1r {v27.16b}, [x4], #1 // ar_coeffs_y[0] - ld1r {v28.16b}, [x4], #1 // ar_coeffs_y[1] - ld1r {v29.16b}, [x4] // ar_coeffs_y[2] -.ifc \type, y - ldrsb w4, [x4, #1] // ar_coeffs_y[3] -.else - add x4, x4, #2 -.endif - - mov w1, #3 -.ifc \type, uv_444 - ld1r {v30.16b}, [x4] // ar_coeffs_uv[4] - ldursb w4, [x4, #-1] // ar_coeffs_uv[3] -.endif - bl generate_grain_rows_neon - - mov w1, #GRAIN_HEIGHT - 3 -1: - sum_\type\()_lag1 v22, v16, v16, v17, left - sum_\type\()_lag1 v23, v16, v17, v18 - sum_\type\()_lag1 v24, v17, v18, v19 - sum_\type\()_lag1 v25, v18, v19, v20 - sum_\type\()_lag1 v20, v19, v20, v21, right - get_grain_2 v21 - subs w1, w1, #1 -.ifc \type, uv_444 - add x19, x19, #2 -.endif - store_grain_row v22, v23, v24, v25, v20, v21 - mov v16.16b, v22.16b - mov v17.16b, v23.16b - mov v18.16b, v24.16b - mov v19.16b, v25.16b - b.gt 1b - - ldp x30, x19, [sp], #96 - ret - -L(generate_grain_\type\()_lag2): - AARCH64_VALID_JUMP_TARGET - ld1 {v30.16b}, [x4] // ar_coeffs_y[0-11], ar_coeffs_uv[0-12] - - smov w4, v30.b[10] - smov w17, v30.b[11] - - mov w1, #3 - bl generate_grain_rows_neon - - mov w1, #GRAIN_HEIGHT - 3 -1: - bl sum_\type\()_lag2_left_neon - bl sum_\type\()_lag2_mid_neon - bl sum_\type\()_lag2_mid_neon - bl sum_\type\()_lag2_mid_neon - bl sum_\type\()_lag2_right_neon - get_grain_2 v16 - subs w1, w1, #1 -.ifc \type, uv_444 - add x19, x19, #2 -.endif - st1 {v16.h}[0], [x0], #2 - b.gt 1b - - ldp x30, x19, [sp], #96 - ret - -L(generate_grain_\type\()_lag3): - AARCH64_VALID_JUMP_TARGET - ld1 {v29.16b, v30.16b}, [x4] // ar_coeffs_y[0-23], ar_coeffs_uv[0-24] - stp d8, d9, [sp, #16] - stp d10, d11, [sp, #32] - stp d12, d13, [sp, #48] - stp d14, d15, [sp, #64] - stp x20, x21, [sp, #80] - - smov w4, v30.b[5] - smov w20, v30.b[6] - smov w21, v30.b[7] - - mov w1, #3 - bl generate_grain_rows_neon - - mov w1, #GRAIN_HEIGHT - 3 -1: - bl sum_\type\()_lag3_left_neon - bl sum_\type\()_lag3_mid_neon - bl sum_\type\()_lag3_mid_neon - bl sum_\type\()_lag3_mid_neon - bl sum_\type\()_lag3_right_neon - get_grain_2 v16 - subs w1, w1, #1 -.ifc \type, uv_444 - add x19, x19, #2 -.endif - st1 {v16.h}[0], [x0], #2 - b.gt 1b - - ldp x20, x21, [sp, #80] - ldp d14, d15, [sp, #64] - ldp d12, d13, [sp, #48] - ldp d10, d11, [sp, #32] - ldp d8, d9, [sp, #16] - ldp x30, x19, [sp], #96 - ret - -L(gen_grain_\type\()_tbl): - .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag0) - .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag1) - .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag2) - .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag3) -endfunc -.endm - -gen_grain_82 y -gen_grain_82 uv_444 - -.macro set_height dst, type -.ifc \type, uv_420 - mov \dst, #SUB_GRAIN_HEIGHT-3 -.else - mov \dst, #GRAIN_HEIGHT-3 -.endif -.endm - -.macro increment_y_ptr reg, type -.ifc \type, uv_420 - add \reg, \reg, #2*GRAIN_WIDTH-(3*32) -.else - sub \reg, \reg, #3*32-GRAIN_WIDTH -.endif -.endm - -.macro gen_grain_44 type -function generate_grain_\type\()_8bpc_neon, export=1 - stp x30, x19, [sp, #-96]! - - mov w13, w3 - mov w14, #28 - add x19, x1, #3*GRAIN_WIDTH-3 - mov x1, x2 - mul w13, w13, w14 - - movrel x3, X(gaussian_sequence) - ldr w2, [x1, #FGD_SEED] - ldr w9, [x1, #FGD_GRAIN_SCALE_SHIFT] - add x4, x1, #FGD_AR_COEFFS_UV - adr x16, L(gen_grain_\type\()_tbl) - ldr w17, [x1, #FGD_AR_COEFF_LAG] - add w9, w9, #4 - ldrh w17, [x16, w17, uxtw #1] - dup v31.8h, w9 // 4 + data->grain_scale_shift - sub x16, x16, w17, uxtw - neg v31.8h, v31.8h - - cmp w13, #0 - mov w11, #0x49d8 - mov w14, #0xb524 - add x4, x4, w13, uxtw // Add offset to ar_coeffs_uv[1] - csel w11, w11, w14, ne - - ldr w7, [x1, #FGD_AR_COEFF_SHIFT] - mov w8, #1 - mov w10, #1 - lsl w8, w8, w7 // 1 << ar_coeff_shift - lsl w10, w10, w9 // 1 << (4 + data->grain_scale_shift) - lsr w8, w8, #1 // 1 << (ar_coeff_shift - 1) - lsr w10, w10, #1 // 1 << (4 + data->grain_scale_shift - 1) - mov w5, #127 - mov w6, #-128 - - eor w2, w2, w11 - - br x16 - -L(generate_grain_\type\()_lag0): - AARCH64_VALID_JUMP_TARGET - dup v28.8h, w7 - ld1r {v27.16b}, [x4] // ar_coeffs_uv[0] - movi v0.16b, #0 - movi v1.16b, #255 - ext v29.16b, v0.16b, v1.16b, #13 - ext v30.16b, v1.16b, v0.16b, #7 - neg v28.8h, v28.8h - - mov w1, #3 - bl generate_grain_rows_44_neon - set_height w1, \type -1: - bl get_grain_row_44_neon -.ifc \type, uv_420 - add x12, x19, #GRAIN_WIDTH -.endif - mov v0.16b, v29.16b - mov v1.16b, v16.16b - bl add_\type\()_coeff_lag0_neon - movi v0.16b, #255 - mov v1.16b, v17.16b - mov v16.16b, v2.16b - bl add_\type\()_coeff_lag0_neon - mov v0.16b, v30.16b - mov v1.16b, v18.16b - mov v17.16b, v2.16b - bl add_\type\()_coeff_lag0_neon - mov v18.16b, v2.16b - subs w1, w1, #1 - increment_y_ptr x19, \type - store_grain_row_44 v16, v17, v18 - b.gt 1b - - ldp x30, x19, [sp], #96 - ret - -L(generate_grain_\type\()_lag1): - AARCH64_VALID_JUMP_TARGET - ld1r {v27.16b}, [x4], #1 // ar_coeffs_uv[0] - ld1r {v28.16b}, [x4], #1 // ar_coeffs_uv[1] - ld1r {v29.16b}, [x4] // ar_coeffs_uv[2] - add x4, x4, #2 - - mov w1, #3 - ld1r {v30.16b}, [x4] // ar_coeffs_u4[4] - ldursb w4, [x4, #-1] // ar_coeffs_uv[3] - bl generate_grain_rows_44_neon - - set_height w1, \type -1: - sum_\type\()_lag1 v20, v16, v16, v17, left - sum_\type\()_lag1 v21, v16, v17, v18 - sum_\type\()_lag1 v18, v17, v18, v18, right - subs w1, w1, #1 - increment_y_ptr x19, \type - store_grain_row_44 v20, v21, v18 - mov v16.16b, v20.16b - mov v17.16b, v21.16b - b.gt 1b - - ldp x30, x19, [sp], #96 - ret - -L(generate_grain_\type\()_lag2): - AARCH64_VALID_JUMP_TARGET - ld1 {v30.16b}, [x4] // ar_coeffs_uv[0-12] - - smov w4, v30.b[10] - smov w17, v30.b[11] - - mov w1, #3 - bl generate_grain_rows_44_neon - - set_height w1, \type -1: - bl sum_\type\()_lag2_left_neon - bl sum_\type\()_lag2_mid_neon - bl sum_\type\()_lag2_right_neon - subs w1, w1, #1 - increment_y_ptr x19, \type - add x0, x0, #GRAIN_WIDTH-48 - b.gt 1b - - ldp x30, x19, [sp], #96 - ret - -L(generate_grain_\type\()_lag3): - AARCH64_VALID_JUMP_TARGET - ldr q29, [x4] // ar_coeffs_uv[0-15] - ldr q30, [x4, #16] // ar_coeffs_uv[16-24] - stp d8, d9, [sp, #16] - stp d10, d11, [sp, #32] - stp d12, d13, [sp, #48] - stp d14, d15, [sp, #64] - stp x20, x21, [sp, #80] - - smov w4, v30.b[5] - smov w20, v30.b[6] - smov w21, v30.b[7] - - mov w1, #3 - bl generate_grain_rows_44_neon - - set_height w1, \type -1: - bl sum_\type\()_lag3_left_neon - bl sum_\type\()_lag3_mid_neon - bl sum_\type\()_lag3_right_neon - subs w1, w1, #1 - increment_y_ptr x19, \type - add x0, x0, #GRAIN_WIDTH-48 - b.gt 1b - - ldp x20, x21, [sp, #80] - ldp d14, d15, [sp, #64] - ldp d12, d13, [sp, #48] - ldp d10, d11, [sp, #32] - ldp d8, d9, [sp, #16] - ldp x30, x19, [sp], #96 - ret - -L(gen_grain_\type\()_tbl): - .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag0) - .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag1) - .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag2) - .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag3) -endfunc -.endm - -gen_grain_44 uv_420 -gen_grain_44 uv_422 - -.macro gather_interleaved dst1, dst2, src1, src2, off - umov w14, \src1[0+\off] - umov w15, \src2[8+\off] - umov w16, \src1[2+\off] - add x14, x14, x3 - umov w17, \src2[10+\off] - add x15, x15, x3 - ld1 {\dst1}[0+\off], [x14] - umov w14, \src1[4+\off] - add x16, x16, x3 - ld1 {\dst2}[8+\off], [x15] - umov w15, \src2[12+\off] - add x17, x17, x3 - ld1 {\dst1}[2+\off], [x16] - umov w16, \src1[6+\off] - add x14, x14, x3 - ld1 {\dst2}[10+\off], [x17] - umov w17, \src2[14+\off] - add x15, x15, x3 - ld1 {\dst1}[4+\off], [x14] - add x16, x16, x3 - ld1 {\dst2}[12+\off], [x15] - add x17, x17, x3 - ld1 {\dst1}[6+\off], [x16] - ld1 {\dst2}[14+\off], [x17] -.endm - -.macro gather dst1, dst2, src1, src2 - gather_interleaved \dst1, \dst2, \src1, \src2, 0 - gather_interleaved \dst2, \dst1, \src2, \src1, 0 - gather_interleaved \dst1, \dst2, \src1, \src2, 1 - gather_interleaved \dst2, \dst1, \src2, \src1, 1 -.endm - -function gather32_neon - gather v4.b, v5.b, v0.b, v1.b - ret -endfunc - -function gather16_neon - gather_interleaved v4.b, v5.b, v0.b, v0.b, 0 - gather_interleaved v4.b, v5.b, v0.b, v0.b, 1 - ins v4.d[1], v5.d[1] - ret -endfunc - -const overlap_coeffs_0, align=4 - .byte 27, 17, 0, 0, 0, 0, 0, 0 - .byte 17, 27, 32, 32, 32, 32, 32, 32 -endconst - -const overlap_coeffs_1, align=4 - .byte 23, 0, 0, 0, 0, 0, 0, 0 - .byte 22, 32, 32, 32, 32, 32, 32, 32 -endconst - -.macro calc_offset offx, offy, src, sx, sy - and \offy, \src, #0xF // randval & 0xF - lsr \offx, \src, #4 // randval >> 4 -.if \sy == 0 - add \offy, \offy, \offy // 2 * (randval & 0xF) -.endif -.if \sx == 0 - add \offx, \offx, \offx // 2 * (randval >> 4) -.endif -.endm - -.macro add_offset dst, offx, offy, src, stride - madd \dst, \stride, \offy, \src // grain_lut += grain_stride * offy - add \dst, \dst, \offx, uxtw // grain_lut += offx -.endm - -// void dav1d_fgy_32x32_8bpc_neon(pixel *const dst, const pixel *const src, -// const ptrdiff_t stride, -// const uint8_t scaling[SCALING_SIZE], -// const int scaling_shift, -// const entry grain_lut[][GRAIN_WIDTH], -// const int offsets[][2], -// const int h, const ptrdiff_t clip, -// const ptrdiff_t type); -function fgy_32x32_8bpc_neon, export=1 - str x30, [sp, #-16]! - ldr w11, [x6, #8] // offsets[1][0] - ldr w13, [x6, #4] // offsets[0][1] - ldr w15, [x6, #12] // offsets[1][1] - ldr w6, [x6] // offsets[0][0] - ldr w8, [sp, #16] // clip - mov x9, #GRAIN_WIDTH // grain_lut stride - - neg w4, w4 - dup v29.8h, w4 // -scaling_shift - - movrel x16, overlap_coeffs_0 - - cbz w8, 1f - // clip - movi v30.16b, #16 - movi v31.16b, #235 - b 2f -1: - // no clip - movi v30.16b, #0 - movi v31.16b, #255 -2: - - ld1 {v27.8b, v28.8b}, [x16] // overlap_coeffs - - add x5, x5, #9 // grain_lut += 9 - add x5, x5, x9, lsl #3 // grain_lut += 8 * grain_stride - add x5, x5, x9 // grain_lut += grain_stride - - calc_offset w11, w12, w11, 0, 0 - calc_offset w13, w14, w13, 0, 0 - calc_offset w15, w16, w15, 0, 0 - calc_offset w6, w10, w6, 0, 0 - - add_offset x12, w11, x12, x5, x9 - add_offset x14, w13, x14, x5, x9 - add_offset x16, w15, x16, x5, x9 - add_offset x5, w6, x10, x5, x9 - - ldr w11, [sp, #24] // type - adr x13, L(fgy_loop_tbl) - - add x4, x12, #32 // grain_lut += BLOCK_SIZE * bx - add x6, x14, x9, lsl #5 // grain_lut += grain_stride * BLOCK_SIZE * by - - tst w11, #1 - ldrh w11, [x13, w11, uxtw #1] - - add x8, x16, x9, lsl #5 // grain_lut += grain_stride * BLOCK_SIZE * by - add x8, x8, #32 // grain_lut += BLOCK_SIZE * bx - - sub x11, x13, w11, uxtw - - b.eq 1f - // y overlap - dup v6.16b, v27.b[0] - dup v7.16b, v27.b[1] - mov w10, w7 // backup actual h - mov w7, #2 -1: - br x11 -endfunc - -function fgy_loop_neon -.macro fgy ox, oy -L(loop_\ox\oy): - AARCH64_VALID_JUMP_TARGET -1: - ld1 {v0.16b, v1.16b}, [x1], x2 // src -.if \ox - ld1 {v20.8b}, [x4], x9 // grain_lut old -.endif -.if \oy - ld1 {v22.16b, v23.16b}, [x6], x9 // grain_lut top -.endif -.if \ox && \oy - ld1 {v21.8b}, [x8], x9 // grain_lut top old -.endif - ld1 {v18.16b, v19.16b}, [x5], x9 // grain_lut - - bl gather32_neon - -.if \ox - smull v20.8h, v20.8b, v27.8b - smlal v20.8h, v18.8b, v28.8b -.endif - -.if \oy -.if \ox - smull v21.8h, v21.8b, v27.8b - smlal v21.8h, v22.8b, v28.8b - sqrshrn v20.8b, v20.8h, #5 - sqrshrn v21.8b, v21.8h, #5 -.endif - -.if \ox - smull v16.8h, v20.8b, v7.8b -.else - smull v16.8h, v18.8b, v7.8b -.endif - smull2 v17.8h, v18.16b, v7.16b - smull v18.8h, v19.8b, v7.8b - smull2 v19.8h, v19.16b, v7.16b -.if \ox - smlal v16.8h, v21.8b, v6.8b -.else - smlal v16.8h, v22.8b, v6.8b -.endif - smlal2 v17.8h, v22.16b, v6.16b - smlal v18.8h, v23.8b, v6.8b - smlal2 v19.8h, v23.16b, v6.16b - sqrshrn v22.8b, v16.8h, #5 - sqrshrn2 v22.16b, v17.8h, #5 - sqrshrn v23.8b, v18.8h, #5 - sqrshrn2 v23.16b, v19.8h, #5 -.endif - - // sxtl of grain -.if \oy - sxtl v16.8h, v22.8b - sxtl2 v17.8h, v22.16b - sxtl v18.8h, v23.8b - sxtl2 v19.8h, v23.16b -.elseif \ox - sqrshrn v20.8b, v20.8h, #5 - sxtl2 v17.8h, v18.16b - sxtl v18.8h, v19.8b - sxtl2 v19.8h, v19.16b - sxtl v16.8h, v20.8b -.else - sxtl v16.8h, v18.8b - sxtl2 v17.8h, v18.16b - sxtl v18.8h, v19.8b - sxtl2 v19.8h, v19.16b -.endif - - uxtl v2.8h, v4.8b // scaling - uxtl2 v3.8h, v4.16b - uxtl v4.8h, v5.8b - uxtl2 v5.8h, v5.16b - - mul v16.8h, v16.8h, v2.8h // scaling * grain - mul v17.8h, v17.8h, v3.8h - mul v18.8h, v18.8h, v4.8h - mul v19.8h, v19.8h, v5.8h - - srshl v16.8h, v16.8h, v29.8h // round2(scaling * grain, scaling_shift) - srshl v17.8h, v17.8h, v29.8h - srshl v18.8h, v18.8h, v29.8h - srshl v19.8h, v19.8h, v29.8h - - uaddw v16.8h, v16.8h, v0.8b // *src + noise - uaddw2 v17.8h, v17.8h, v0.16b - uaddw v18.8h, v18.8h, v1.8b - uaddw2 v19.8h, v19.8h, v1.16b - - sqxtun v0.8b, v16.8h - sqxtun2 v0.16b, v17.8h - sqxtun v1.8b, v18.8h - sqxtun2 v1.16b, v19.8h - - umax v0.16b, v0.16b, v30.16b - umax v1.16b, v1.16b, v30.16b - umin v0.16b, v0.16b, v31.16b - umin v1.16b, v1.16b, v31.16b - - subs w7, w7, #1 -.if \oy - dup v6.16b, v28.b[0] - dup v7.16b, v28.b[1] -.endif - st1 {v0.16b, v1.16b}, [x0], x2 // dst - b.gt 1b - -.if \oy - cmp w10, #2 - sub w7, w10, #2 // restore actual remaining h - b.gt L(loop_\ox\()0) -.endif - ldr x30, [sp], #16 - ret -.endm - - fgy 0, 0 - fgy 0, 1 - fgy 1, 0 - fgy 1, 1 - -L(fgy_loop_tbl): - .hword L(fgy_loop_tbl) - L(loop_00) - .hword L(fgy_loop_tbl) - L(loop_01) - .hword L(fgy_loop_tbl) - L(loop_10) - .hword L(fgy_loop_tbl) - L(loop_11) -endfunc - -// void dav1d_fguv_32x32_420_8bpc_neon(pixel *const dst, -// const pixel *const src, -// const ptrdiff_t stride, -// const uint8_t scaling[SCALING_SIZE], -// const Dav1dFilmGrainData *const data, -// const entry grain_lut[][GRAIN_WIDTH], -// const pixel *const luma_row, -// const ptrdiff_t luma_stride, -// const int offsets[][2], -// const ptrdiff_t h, const ptrdiff_t uv, -// const ptrdiff_t is_id, -// const ptrdiff_t type); -.macro fguv layout, sx, sy -function fguv_32x32_\layout\()_8bpc_neon, export=1 - str x30, [sp, #-32]! - str d8, [sp, #16] - ldp x8, x9, [sp, #32] // offsets, h - ldp x10, x11, [sp, #48] // uv, is_id - - ldr w13, [x4, #FGD_SCALING_SHIFT] - ldr w12, [x4, #FGD_CLIP_TO_RESTRICTED_RANGE] - neg w13, w13 // -scaling_shift - - // !csfl - add x10, x4, x10, lsl #2 // + 4*uv - add x14, x10, #FGD_UV_LUMA_MULT - add x15, x10, #FGD_UV_MULT - add x10, x10, #FGD_UV_OFFSET - ld1 {v8.h}[0], [x14] // uv_luma_mult - ld1r {v24.8h}, [x10] // uv_offset - ld1 {v8.h}[1], [x15] // uv_mult - - dup v29.8h, w13 // -scaling_shift - - cbz w12, 1f - // clip - movi v30.16b, #16 - movi v31.16b, #240 - cbz w11, 2f - // is_id - movi v31.16b, #235 - b 2f -1: - // no clip - movi v30.16b, #0 - movi v31.16b, #255 -2: - - ldr w12, [x8, #8] // offsets[1][0] - ldr w14, [x8, #4] // offsets[0][1] - ldr w16, [x8, #12] // offsets[1][1] - ldr w8, [x8] // offsets[0][0] - - mov x10, #GRAIN_WIDTH // grain_lut stride - - add x5, x5, #(3 + (2 >> \sx)*3) // grain_lut += 9 or 6 -.if \sy - add x5, x5, x10, lsl #2 // grain_lut += 4 * grain_stride - add x5, x5, x10, lsl #1 // grain_lut += 2 * grain_stride -.else - add x5, x5, x10, lsl #3 // grain_lut += 8 * grain_stride - add x5, x5, x10 // grain_lut += grain_stride -.endif - - calc_offset w12, w13, w12, \sx, \sy - calc_offset w14, w15, w14, \sx, \sy - calc_offset w16, w17, w16, \sx, \sy - calc_offset w8, w11, w8, \sx, \sy - - add_offset x13, w12, x13, x5, x10 - add_offset x15, w14, x15, x5, x10 - add_offset x17, w16, x17, x5, x10 - add_offset x5, w8, x11, x5, x10 - - add x4, x13, #(32 >> \sx) // grain_lut += BLOCK_SIZE * bx - add x8, x15, x10, lsl #(5 - \sy) // grain_lut += grain_stride * BLOCK_SIZE * by - add x11, x17, x10, lsl #(5 - \sy) // grain_lut += grain_stride * BLOCK_SIZE * by - add x11, x11, #(32 >> \sx) // grain_lut += BLOCK_SIZE * bx - - ldr w13, [sp, #64] // type - - movrel x16, overlap_coeffs_\sx - adr x14, L(fguv_loop_sx\sx\()_tbl) - - ld1 {v27.8b, v28.8b}, [x16] // overlap_coeffs - tst w13, #1 - ldrh w13, [x14, w13, uxtw #1] - - b.eq 1f - // y overlap - sub w12, w9, #(2 >> \sy) // backup remaining h - mov w9, #(2 >> \sy) - -1: - sub x13, x14, w13, uxtw - -.if \sy - movi v25.16b, #23 - movi v26.16b, #22 -.else - movi v25.16b, #27 - movi v26.16b, #17 -.endif - -.if \sy - add x7, x7, x7 // luma_stride *= 2 -.endif - - br x13 -endfunc -.endm - -fguv 420, 1, 1 -fguv 422, 1, 0 -fguv 444, 0, 0 - -function fguv_loop_sx0_neon -.macro fguv_loop_sx0 csfl, ox, oy -L(fguv_loop_sx0_csfl\csfl\()_\ox\oy): - AARCH64_VALID_JUMP_TARGET -1: - ld1 {v0.16b, v1.16b}, [x6], x7 // luma - ld1 {v6.16b, v7.16b}, [x1], x2 // src -.if \ox - ld1 {v20.8b}, [x4], x10 // grain_lut old -.endif -.if \oy - ld1 {v22.16b, v23.16b}, [x8], x10 // grain_lut top -.endif -.if \ox && \oy - ld1 {v21.8b}, [x11], x10 // grain_lut top old -.endif - ld1 {v18.16b, v19.16b}, [x5], x10 // grain_lut - -.if !\csfl - uxtl v2.8h, v0.8b - uxtl2 v3.8h, v0.16b - uxtl v4.8h, v1.8b - uxtl2 v5.8h, v1.16b - uxtl v0.8h, v6.8b - uxtl2 v1.8h, v6.16b - uxtl v16.8h, v7.8b - uxtl2 v17.8h, v7.16b - mul v2.8h, v2.8h, v8.h[0] - mul v3.8h, v3.8h, v8.h[0] - mul v4.8h, v4.8h, v8.h[0] - mul v5.8h, v5.8h, v8.h[0] - mul v0.8h, v0.8h, v8.h[1] - mul v1.8h, v1.8h, v8.h[1] - mul v16.8h, v16.8h, v8.h[1] - mul v17.8h, v17.8h, v8.h[1] - sqadd v2.8h, v2.8h, v0.8h - sqadd v3.8h, v3.8h, v1.8h - sqadd v4.8h, v4.8h, v16.8h - sqadd v5.8h, v5.8h, v17.8h - sshr v2.8h, v2.8h, #6 - sshr v3.8h, v3.8h, #6 - sshr v4.8h, v4.8h, #6 - sshr v5.8h, v5.8h, #6 - add v2.8h, v2.8h, v24.8h - add v3.8h, v3.8h, v24.8h - add v4.8h, v4.8h, v24.8h - add v5.8h, v5.8h, v24.8h - sqxtun v0.8b, v2.8h - sqxtun2 v0.16b, v3.8h - sqxtun v1.8b, v4.8h - sqxtun2 v1.16b, v5.8h -.endif - - bl gather32_neon - -.if \ox - smull v20.8h, v20.8b, v27.8b - smlal v20.8h, v18.8b, v28.8b -.endif - -.if \oy -.if \ox - smull v21.8h, v21.8b, v27.8b - smlal v21.8h, v22.8b, v28.8b - sqrshrn v20.8b, v20.8h, #5 - sqrshrn v21.8b, v21.8h, #5 -.endif - -.if \ox - smull v16.8h, v20.8b, v26.8b -.else - smull v16.8h, v18.8b, v26.8b -.endif - smull2 v17.8h, v18.16b, v26.16b - smull v18.8h, v19.8b, v26.8b - smull2 v19.8h, v19.16b, v26.16b -.if \ox - smlal v16.8h, v21.8b, v25.8b -.else - smlal v16.8h, v22.8b, v25.8b -.endif - smlal2 v17.8h, v22.16b, v25.16b - smlal v18.8h, v23.8b, v25.8b - smlal2 v19.8h, v23.16b, v25.16b - sqrshrn v22.8b, v16.8h, #5 - sqrshrn2 v22.16b, v17.8h, #5 - sqrshrn v23.8b, v18.8h, #5 - sqrshrn2 v23.16b, v19.8h, #5 -.endif - - // sxtl of grain -.if \oy - sxtl v16.8h, v22.8b - sxtl2 v17.8h, v22.16b - sxtl v18.8h, v23.8b - sxtl2 v19.8h, v23.16b -.elseif \ox - sqrshrn v20.8b, v20.8h, #5 - sxtl2 v17.8h, v18.16b - sxtl v18.8h, v19.8b - sxtl2 v19.8h, v19.16b - sxtl v16.8h, v20.8b -.else - sxtl v16.8h, v18.8b - sxtl2 v17.8h, v18.16b - sxtl v18.8h, v19.8b - sxtl2 v19.8h, v19.16b -.endif - - uxtl v2.8h, v4.8b // scaling - uxtl2 v3.8h, v4.16b - uxtl v4.8h, v5.8b - uxtl2 v5.8h, v5.16b - - mul v16.8h, v16.8h, v2.8h // scaling * grain - mul v17.8h, v17.8h, v3.8h - mul v18.8h, v18.8h, v4.8h - mul v19.8h, v19.8h, v5.8h - - srshl v16.8h, v16.8h, v29.8h // round2(scaling * grain, scaling_shift) - srshl v17.8h, v17.8h, v29.8h - srshl v18.8h, v18.8h, v29.8h - srshl v19.8h, v19.8h, v29.8h - - uaddw v16.8h, v16.8h, v6.8b // *src + noise - uaddw2 v17.8h, v17.8h, v6.16b - uaddw v18.8h, v18.8h, v7.8b - uaddw2 v19.8h, v19.8h, v7.16b - - sqxtun v0.8b, v16.8h - sqxtun2 v0.16b, v17.8h - sqxtun v1.8b, v18.8h - sqxtun2 v1.16b, v19.8h - - umax v0.16b, v0.16b, v30.16b - umax v1.16b, v1.16b, v30.16b - umin v0.16b, v0.16b, v31.16b - umin v1.16b, v1.16b, v31.16b - - subs w9, w9, #1 -.if \oy - dup v25.16b, v28.b[0] - dup v26.16b, v28.b[1] -.endif - st1 {v0.16b, v1.16b}, [x0], x2 // dst - b.gt 1b - -.if \oy - cmp w12, #0 - mov w9, w12 // restore actual remaining h - b.gt L(fguv_loop_sx0_csfl\csfl\()_\ox\()0) -.endif - b 9f -.endm - fguv_loop_sx0 0, 0, 0 - fguv_loop_sx0 0, 0, 1 - fguv_loop_sx0 0, 1, 0 - fguv_loop_sx0 0, 1, 1 - fguv_loop_sx0 1, 0, 0 - fguv_loop_sx0 1, 0, 1 - fguv_loop_sx0 1, 1, 0 - fguv_loop_sx0 1, 1, 1 - -9: - ldr d8, [sp, #16] - ldr x30, [sp], #32 - ret - -L(fguv_loop_sx0_tbl): - .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl0_00) - .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl0_01) - .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl0_10) - .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl0_11) - .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl1_00) - .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl1_01) - .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl1_10) - .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl1_11) -endfunc - -function fguv_loop_sx1_neon -.macro fguv_loop_sx1 csfl, ox, oy -L(fguv_loop_sx1_csfl\csfl\()_\ox\oy): - AARCH64_VALID_JUMP_TARGET -1: - ld1 {v0.16b, v1.16b}, [x6], x7 // luma - ld1 {v6.16b}, [x1], x2 // src -.if \ox - ld1 {v20.8b}, [x4], x10 // grain_lut old -.endif -.if \oy - ld1 {v22.16b}, [x8], x10 // grain_lut top -.endif -.if \ox && \oy - ld1 {v21.8b}, [x11], x10 // grain_lut top old -.endif - ld1 {v18.16b}, [x5], x10 // grain_lut - - uaddlp v2.8h, v0.16b - uaddlp v3.8h, v1.16b -.if \csfl - rshrn v0.8b, v2.8h, #1 - rshrn2 v0.16b, v3.8h, #1 -.else - urshr v2.8h, v2.8h, #1 - urshr v3.8h, v3.8h, #1 - uxtl v0.8h, v6.8b - uxtl2 v1.8h, v6.16b - mul v2.8h, v2.8h, v8.h[0] - mul v3.8h, v3.8h, v8.h[0] - mul v0.8h, v0.8h, v8.h[1] - mul v1.8h, v1.8h, v8.h[1] - sqadd v2.8h, v2.8h, v0.8h - sqadd v3.8h, v3.8h, v1.8h - sshr v2.8h, v2.8h, #6 - sshr v3.8h, v3.8h, #6 - add v2.8h, v2.8h, v24.8h - add v3.8h, v3.8h, v24.8h - sqxtun v0.8b, v2.8h - sqxtun2 v0.16b, v3.8h -.endif - - bl gather16_neon - -.if \ox - smull v20.8h, v20.8b, v27.8b - smlal v20.8h, v18.8b, v28.8b -.endif - -.if \oy -.if \ox - smull v21.8h, v21.8b, v27.8b - smlal v21.8h, v22.8b, v28.8b - sqrshrn v20.8b, v20.8h, #5 - sqrshrn v21.8b, v21.8h, #5 -.endif - -.if \ox - smull v16.8h, v20.8b, v26.8b -.else - smull v16.8h, v18.8b, v26.8b -.endif - smull2 v17.8h, v18.16b, v26.16b -.if \ox - smlal v16.8h, v21.8b, v25.8b -.else - smlal v16.8h, v22.8b, v25.8b -.endif - smlal2 v17.8h, v22.16b, v25.16b - sqrshrn v22.8b, v16.8h, #5 - sqrshrn2 v22.16b, v17.8h, #5 -.endif - - // sxtl of grain -.if \oy - sxtl v16.8h, v22.8b - sxtl2 v17.8h, v22.16b -.elseif \ox - sqrshrn v20.8b, v20.8h, #5 - sxtl2 v17.8h, v18.16b - sxtl v16.8h, v20.8b -.else - sxtl v16.8h, v18.8b - sxtl2 v17.8h, v18.16b -.endif - - uxtl v2.8h, v4.8b // scaling - uxtl2 v3.8h, v4.16b - - mul v16.8h, v16.8h, v2.8h // scaling * grain - mul v17.8h, v17.8h, v3.8h - - srshl v16.8h, v16.8h, v29.8h // round2(scaling * grain, scaling_shift) - srshl v17.8h, v17.8h, v29.8h - - uaddw v16.8h, v16.8h, v6.8b // *src + noise - uaddw2 v17.8h, v17.8h, v6.16b - - sqxtun v0.8b, v16.8h - sqxtun2 v0.16b, v17.8h - - umax v0.16b, v0.16b, v30.16b - umin v0.16b, v0.16b, v31.16b - -.if \oy - mov v16.16b, v25.16b -.endif - subs w9, w9, #1 -.if \oy - mov v25.16b, v26.16b - mov v26.16b, v16.16b -.endif - st1 {v0.16b}, [x0], x2 // dst - b.gt 1b - -.if \oy - cmp w12, #0 - mov w9, w12 // restore actual remaining h - b.gt L(fguv_loop_sx1_csfl\csfl\()_\ox\()0) -.endif - - b 9f -.endm - fguv_loop_sx1 0, 0, 0 - fguv_loop_sx1 0, 0, 1 - fguv_loop_sx1 0, 1, 0 - fguv_loop_sx1 0, 1, 1 - fguv_loop_sx1 1, 0, 0 - fguv_loop_sx1 1, 0, 1 - fguv_loop_sx1 1, 1, 0 - fguv_loop_sx1 1, 1, 1 - -9: - ldr d8, [sp, #16] - ldr x30, [sp], #32 - ret - -L(fguv_loop_sx1_tbl): - .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl0_00) - .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl0_01) - .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl0_10) - .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl0_11) - .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl1_00) - .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl1_01) - .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl1_10) - .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl1_11) -endfunc diff --git a/third_party/dav1d/src/arm/64/film_grain16.S b/third_party/dav1d/src/arm/64/film_grain16.S deleted file mode 100644 --- a/third_party/dav1d/src/arm/64/film_grain16.S +++ /dev/null @@ -1,1968 +0,0 @@ -/* - * Copyright © 2021, VideoLAN and dav1d authors - * Copyright © 2021, Martin Storsjo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include "src/arm/asm.S" -#include "util.S" -#include "src/arm/asm-offsets.h" - -#define GRAIN_WIDTH 82 -#define GRAIN_HEIGHT 73 - -#define SUB_GRAIN_WIDTH 44 -#define SUB_GRAIN_HEIGHT 38 - -.macro increment_seed steps, shift=1 - lsr w11, w2, #3 - lsr w12, w2, #12 - lsr w13, w2, #1 - eor w11, w2, w11 // (r >> 0) ^ (r >> 3) - eor w12, w12, w13 // (r >> 12) ^ (r >> 1) - eor w11, w11, w12 // (r >> 0) ^ (r >> 3) ^ (r >> 12) ^ (r >> 1) -.if \shift - lsr w2, w2, #\steps -.endif - and w11, w11, #((1 << \steps) - 1) // bit -.if \shift - orr w2, w2, w11, lsl #(16 - \steps) // *state -.else - orr w2, w2, w11, lsl #16 // *state -.endif -.endm - -.macro read_rand dest, bits, age - ubfx \dest, x2, #16 - \bits - \age, #\bits -.endm - -.macro read_shift_rand dest, bits - ubfx \dest, x2, #17 - \bits, #\bits - lsr w2, w2, #1 -.endm - -// special calling convention: -// w2 holds seed -// x3 holds dav1d_gaussian_sequence -// clobbers x11-x15 -// returns in v0.8h -function get_gaussian_neon - increment_seed 4 - read_rand x14, 11, 3 - read_rand x15, 11, 2 - add x14, x3, x14, lsl #1 - add x15, x3, x15, lsl #1 - ld1 {v0.h}[0], [x14] - read_rand x14, 11, 1 - ld1 {v0.h}[1], [x15] - add x14, x3, x14, lsl #1 - read_rand x15, 11, 0 - increment_seed 4 - add x15, x3, x15, lsl #1 - ld1 {v0.h}[2], [x14] - read_rand x14, 11, 3 - ld1 {v0.h}[3], [x15] - add x14, x3, x14, lsl #1 - read_rand x15, 11, 2 - ld1 {v0.h}[4], [x14] - add x15, x3, x15, lsl #1 - read_rand x14, 11, 1 - ld1 {v0.h}[5], [x15] - read_rand x15, 11, 0 - add x14, x3, x14, lsl #1 - add x15, x3, x15, lsl #1 - ld1 {v0.h}[6], [x14] - ld1 {v0.h}[7], [x15] - ret -endfunc - -.macro store_grain_row r0, r1, r2, r3, r4, r5 - st1 {\r0\().16b,\r1\().16b}, [x0], #32 - st1 {\r2\().16b,\r3\().16b}, [x0], #32 - st1 {\r4\().16b}, [x0], #16 - st1 {\r5\().h}[0], [x0], #2 -.endm - -function get_grain_2_neon - increment_seed 2 - read_rand x14, 11, 1 - read_rand x15, 11, 0 - add x14, x3, x14, lsl #1 - add x15, x3, x15, lsl #1 - ld1 {v0.h}[0], [x14] - ld1 {v0.h}[1], [x15] - srshl v0.4h, v0.4h, v31.4h - ret -endfunc - -.macro get_grain_2 dst - bl get_grain_2_neon -.ifnc \dst, v0 - mov \dst\().8b, v0.8b -.endif -.endm - -function get_grain_4_neon - increment_seed 4 - read_rand x14, 11, 3 - read_rand x15, 11, 2 - add x14, x3, x14, lsl #1 - add x15, x3, x15, lsl #1 - ld1 {v0.h}[0], [x14] - read_rand x14, 11, 1 - ld1 {v0.h}[1], [x15] - add x14, x3, x14, lsl #1 - read_rand x15, 11, 0 - add x15, x3, x15, lsl #1 - ld1 {v0.h}[2], [x14] - ld1 {v0.h}[3], [x15] - srshl v0.4h, v0.4h, v31.4h - ret -endfunc - -.macro get_grain_4 dst - bl get_grain_4_neon -.ifnc \dst, v0 - mov \dst\().8b, v0.8b -.endif -.endm - -// w15 holds the number of entries to produce -// w14, w16 and w17 hold the previous output entries -// v0 holds the vector of produced entries -// v1 holds the input vector of sums from above -.macro output_lag n -function output_lag\n\()_neon -1: - read_shift_rand x13, 11 - mov w11, v1.s[0] - ldrsh w12, [x3, x13, lsl #1] - ext v0.16b, v0.16b, v0.16b, #2 -.if \n == 1 - madd w11, w14, w4, w11 // sum (above) + *coeff * prev output -.elseif \n == 2 - madd w11, w16, w4, w11 // sum (above) + *coeff * prev output 1 - madd w11, w14, w17, w11 // += *coeff * prev output 2 - mov w16, w14 -.else - madd w11, w17, w4, w11 // sum (above) + *coeff * prev output 1 - madd w11, w16, w20, w11 // sum (above) + *coeff * prev output 2 - madd w11, w14, w21, w11 // += *coeff * prev output 3 - mov w17, w16 - mov w16, w14 -.endif - add w14, w11, w8 // 1 << (ar_coeff_shift - 1) - add w12, w12, w10 // 1 << (4 - bitdepth_min_8 + grain_scale_shift - 1) - asr w14, w14, w7 // >> ar_coeff_shift - asr w12, w12, w9 // >> (4 - bitdepth_min_8 + grain_scale_shift) - add w14, w14, w12 - cmp w14, w5 - csel w14, w14, w5, le - cmp w14, w6 - csel w14, w14, w6, ge - subs w15, w15, #1 - ext v1.16b, v1.16b, v1.16b, #4 - ins v0.h[7], w14 - b.gt 1b - ret -endfunc -.endm - -output_lag 1 -output_lag 2 -output_lag 3 - - -function sum_lag1_above_neon - sub x12, x0, #1*GRAIN_WIDTH*2 - 16 - ld1 {v18.8h}, [x12] // load top right - - ext v0.16b, v16.16b, v17.16b, #14 // top left, top mid - ext v1.16b, v17.16b, v18.16b, #2 // top mid, top right - - smull v4.4s, v17.4h, v28.4h - smlal v4.4s, v0.4h, v27.4h - smlal v4.4s, v1.4h, v29.4h - smull2 v5.4s, v17.8h, v28.8h - smlal2 v5.4s, v0.8h, v27.8h - smlal2 v5.4s, v1.8h, v29.8h - - mov v16.16b, v17.16b - mov v17.16b, v18.16b - - ret -endfunc - -.macro sum_lag_n_body lag, type, uv_layout, edge, elems, uv_coeff - bl sum_\lag\()_above_neon -.ifc \type, uv_420 - add x12, x19, #GRAIN_WIDTH*2 - ld1 {v22.8h, v23.8h}, [x19], #32 - ld1 {v24.8h, v25.8h}, [x12] - addp v22.8h, v22.8h, v23.8h - addp v23.8h, v24.8h, v25.8h - add v22.8h, v22.8h, v23.8h - srshr v0.8h, v22.8h, #2 -.endif -.ifc \type, uv_422 - ld1 {v22.8h, v23.8h}, [x19], #32 - addp v22.8h, v22.8h, v23.8h - srshr v0.8h, v22.8h, #1 -.endif -.ifc \type, uv_444 - ld1 {v0.8h}, [x19], #16 -.endif -.if \uv_layout -.ifnb \uv_coeff - dup v1.8b, \uv_coeff - sxtl v1.8h, v1.8b - smlal v4.4s, v0.4h, v1.4h - smlal2 v5.4s, v0.8h, v1.8h -.else - smlal v4.4s, v0.4h, v30.4h - smlal2 v5.4s, v0.8h, v30.8h -.endif -.endif -.if \uv_layout && \elems == 8 - b sum_\lag\()_y_\edge\()_start -.elseif \uv_layout == 444 && \elems == 7 - b sum_\lag\()_y_\edge\()_start -.elseif \uv_layout == 422 && \elems == 1 - b sum_\lag\()_uv_420_\edge\()_start -.else -sum_\lag\()_\type\()_\edge\()_start: -.if \elems > 4 -.ifc \edge, left - increment_seed 4 - read_rand x12, 11, 3 - read_rand x13, 11, 2 - read_rand x14, 11, 1 - add x12, x3, x12, lsl #1 - add x13, x3, x13, lsl #1 - add x14, x3, x14, lsl #1 - ld1 {v0.h}[5], [x12] - ld1 {v0.h}[6], [x13] - ld1 {v0.h}[7], [x14] - lsl x2, x2, #1 // shift back the state as if we'd done increment_seed with shift=0 - srshl v0.8h, v0.8h, v31.8h - ext v4.16b, v4.16b, v4.16b, #12 -.ifc \lag, lag3 - smov w17, v0.h[5] -.endif -.ifnc \lag, lag1 - smov w16, v0.h[6] -.endif - smov w14, v0.h[7] - - mov v1.16b, v4.16b - mov w15, #1 - bl output_\lag\()_neon -.else - increment_seed 4, shift=0 - mov v1.16b, v4.16b - mov w15, #4 - bl output_\lag\()_neon -.endif - - increment_seed 4, shift=0 - mov v1.16b, v5.16b -.ifc \edge, right - mov w15, #3 - bl output_\lag\()_neon - read_shift_rand x15, 11 - add x15, x3, x15, lsl #1 - ld1 {v1.h}[0], [x15] - srshl v1.4h, v1.4h, v31.4h - ext v0.16b, v0.16b, v1.16b, #2 -.else - mov w15, #4 - bl output_\lag\()_neon -.endif -.else - // elems == 1 - increment_seed 4, shift=0 - mov v1.16b, v4.16b - mov w15, #1 - bl output_\lag\()_neon - lsr w2, w2, #3 - - read_rand x12, 11, 2 - read_rand x13, 11, 1 - read_rand x14, 11, 0 - add x12, x3, x12, lsl #1 - add x13, x3, x13, lsl #1 - add x14, x3, x14, lsl #1 - ld1 {v1.h}[0], [x12] - ld1 {v1.h}[1], [x13] - ld1 {v1.h}[2], [x14] - srshl v1.4h, v1.4h, v31.4h - ext v0.16b, v0.16b, v1.16b, #14 -.endif - st1 {v0.8h}, [x0], #16 - ldr x30, [sp], #16 - ret -.endif -.endm - -.macro sum_lag1_func type, uv_layout, edge, elems=8 -function sum_\type\()_lag1_\edge\()_neon - str x30, [sp, #-16]! -.ifc \edge, left - sub x12, x0, #1*GRAIN_WIDTH*2 - ld1 {v17.8h}, [x12] // load the previous block right above -.endif - sum_lag_n_body lag1, \type, \uv_layout, \edge, \elems -endfunc -.endm - -sum_lag1_func y, 0, left -sum_lag1_func y, 0, mid -sum_lag1_func y, 0, right, 7 -sum_lag1_func uv_444, 444, left -sum_lag1_func uv_444, 444, mid -sum_lag1_func uv_444, 444, right, 7 -sum_lag1_func uv_422, 422, left -sum_lag1_func uv_422, 422, mid -sum_lag1_func uv_422, 422, right, 1 -sum_lag1_func uv_420, 420, left -sum_lag1_func uv_420, 420, mid -sum_lag1_func uv_420, 420, right, 1 - - -function sum_lag2_above_neon - sub x12, x0, #2*GRAIN_WIDTH*2 - 16 - sub x13, x0, #1*GRAIN_WIDTH*2 - 16 - ld1 {v18.8h}, [x12] // load top right - ld1 {v21.8h}, [x13] - - dup v26.8b, v30.b[0] - ext v22.16b, v16.16b, v17.16b, #12 // top left, top mid - dup v27.8b, v30.b[1] - ext v23.16b, v16.16b, v17.16b, #14 - sxtl v26.8h, v26.8b - dup v28.8b, v30.b[3] - ext v0.16b, v17.16b, v18.16b, #2 // top mid, top right - sxtl v27.8h, v27.8b - dup v29.8b, v30.b[4] - ext v1.16b, v17.16b, v18.16b, #4 - sxtl v28.8h, v28.8b - sxtl v29.8h, v29.8b - - smull v4.4s, v22.4h, v26.4h - smlal v4.4s, v23.4h, v27.4h - smlal v4.4s, v0.4h, v28.4h - smlal v4.4s, v1.4h, v29.4h - smull2 v5.4s, v22.8h, v26.8h - smlal2 v5.4s, v23.8h, v27.8h - smlal2 v5.4s, v0.8h, v28.8h - smlal2 v5.4s, v1.8h, v29.8h - - dup v26.16b, v30.b[5] - ext v22.16b, v19.16b, v20.16b, #12 // top left, top mid - dup v27.16b, v30.b[6] - ext v23.16b, v19.16b, v20.16b, #14 - sxtl v26.8h, v26.8b - dup v28.16b, v30.b[8] - ext v0.16b, v20.16b, v21.16b, #2 // top mid, top right - sxtl v27.8h, v27.8b - dup v29.16b, v30.b[9] - ext v1.16b, v20.16b, v21.16b, #4 - sxtl v28.8h, v28.8b - sxtl v29.8h, v29.8b - - smlal v4.4s, v22.4h, v26.4h - smlal v4.4s, v23.4h, v27.4h - smlal v4.4s, v0.4h, v28.4h - smlal v4.4s, v1.4h, v29.4h - smlal2 v5.4s, v22.8h, v26.8h - smlal2 v5.4s, v23.8h, v27.8h - smlal2 v5.4s, v0.8h, v28.8h - smlal2 v5.4s, v1.8h, v29.8h - - dup v26.16b, v30.b[2] - dup v27.16b, v30.b[7] - sxtl v26.8h, v26.8b - sxtl v27.8h, v27.8b - - smlal v4.4s, v17.4h, v26.4h - smlal v4.4s, v20.4h, v27.4h - smlal2 v5.4s, v17.8h, v26.8h - smlal2 v5.4s, v20.8h, v27.8h - mov v16.16b, v17.16b - mov v17.16b, v18.16b - - mov v19.16b, v20.16b - mov v20.16b, v21.16b - ret -endfunc - -.macro sum_lag2_func type, uv_layout, edge, elems=8 -function sum_\type\()_lag2_\edge\()_neon - str x30, [sp, #-16]! -.ifc \edge, left - sub x12, x0, #2*GRAIN_WIDTH*2 - sub x13, x0, #1*GRAIN_WIDTH*2 - ld1 {v17.8h}, [x12] // load the previous block right above - ld1 {v20.8h}, [x13] -.endif - sum_lag_n_body lag2, \type, \uv_layout, \edge, \elems, v30.b[12] -endfunc -.endm - -sum_lag2_func y, 0, left -sum_lag2_func y, 0, mid -sum_lag2_func y, 0, right, 7 -sum_lag2_func uv_444, 444, left -sum_lag2_func uv_444, 444, mid -sum_lag2_func uv_444, 444, right, 7 -sum_lag2_func uv_422, 422, left -sum_lag2_func uv_422, 422, mid -sum_lag2_func uv_422, 422, right, 1 -sum_lag2_func uv_420, 420, left -sum_lag2_func uv_420, 420, mid -sum_lag2_func uv_420, 420, right, 1 - - -function sum_lag3_above_neon - sub x11, x0, #3*GRAIN_WIDTH*2 - 16 - sub x12, x0, #2*GRAIN_WIDTH*2 - 16 - sub x13, x0, #1*GRAIN_WIDTH*2 - 16 - ld1 {v15.8h}, [x11] // load top right - ld1 {v18.8h}, [x12] - ld1 {v21.8h}, [x13] - - dup v22.8b, v29.b[0] - ext v8.16b, v13.16b, v14.16b, #10 // top left, top mid - dup v23.8b, v29.b[1] - ext v9.16b, v13.16b, v14.16b, #12 - sxtl v22.8h, v22.8b - dup v24.8b, v29.b[2] - sxtl v23.8h, v23.8b - dup v25.8b, v29.b[3] - ext v10.16b, v13.16b, v14.16b, #14 - sxtl v24.8h, v24.8b - dup v26.8b, v29.b[4] - ext v11.16b, v14.16b, v15.16b, #2 // top mid, top right - sxtl v25.8h, v25.8b - dup v27.8b, v29.b[5] - ext v12.16b, v14.16b, v15.16b, #4 - sxtl v26.8h, v26.8b - dup v28.8b, v29.b[6] - ext v13.16b, v14.16b, v15.16b, #6 - sxtl v27.8h, v27.8b - sxtl v28.8h, v28.8b - - smull v4.4s, v8.4h, v22.4h - smlal v4.4s, v9.4h, v23.4h - smlal v4.4s, v10.4h, v24.4h - smlal v4.4s, v11.4h, v26.4h - smlal v4.4s, v12.4h, v27.4h - smlal v4.4s, v13.4h, v28.4h - smlal v4.4s, v14.4h, v25.4h - smull2 v5.4s, v8.8h, v22.8h - smlal2 v5.4s, v9.8h, v23.8h - smlal2 v5.4s, v10.8h, v24.8h - smlal2 v5.4s, v11.8h, v26.8h - smlal2 v5.4s, v12.8h, v27.8h - smlal2 v5.4s, v13.8h, v28.8h - smlal2 v5.4s, v14.8h, v25.8h - - dup v22.8b, v29.b[7] - ext v8.16b, v16.16b, v17.16b, #10 // top left, top mid - dup v23.8b, v29.b[8] - ext v9.16b, v16.16b, v17.16b, #12 - sxtl v22.8h, v22.8b - dup v24.8b, v29.b[9] - sxtl v23.8h, v23.8b - dup v25.8b, v29.b[10] - ext v10.16b, v16.16b, v17.16b, #14 - sxtl v24.8h, v24.8b - dup v26.8b, v29.b[11] - ext v11.16b, v17.16b, v18.16b, #2 // top mid, top right - sxtl v25.8h, v25.8b - dup v27.8b, v29.b[12] - ext v12.16b, v17.16b, v18.16b, #4 - sxtl v26.8h, v26.8b - dup v28.8b, v29.b[13] - ext v13.16b, v17.16b, v18.16b, #6 - sxtl v27.8h, v27.8b - sxtl v28.8h, v28.8b - - smlal v4.4s, v8.4h, v22.4h - smlal v4.4s, v9.4h, v23.4h - smlal v4.4s, v10.4h, v24.4h - smlal v4.4s, v11.4h, v26.4h - smlal v4.4s, v12.4h, v27.4h - smlal v4.4s, v13.4h, v28.4h - smlal v4.4s, v17.4h, v25.4h - smlal2 v5.4s, v8.8h, v22.8h - smlal2 v5.4s, v9.8h, v23.8h - smlal2 v5.4s, v10.8h, v24.8h - smlal2 v5.4s, v11.8h, v26.8h - smlal2 v5.4s, v12.8h, v27.8h - smlal2 v5.4s, v13.8h, v28.8h - smlal2 v5.4s, v17.8h, v25.8h - - dup v22.8b, v29.b[14] - ext v8.16b, v19.16b, v20.16b, #10 // top left, top mid - dup v23.8b, v29.b[15] - ext v9.16b, v19.16b, v20.16b, #12 - sxtl v22.8h, v22.8b - dup v24.8b, v30.b[0] - sxtl v23.8h, v23.8b - dup v25.8b, v30.b[1] - ext v10.16b, v19.16b, v20.16b, #14 - sxtl v24.8h, v24.8b - dup v26.8b, v30.b[2] - ext v11.16b, v20.16b, v21.16b, #2 // top mid, top right - sxtl v25.8h, v25.8b - dup v27.8b, v30.b[3] - ext v12.16b, v20.16b, v21.16b, #4 - sxtl v26.8h, v26.8b - dup v28.8b, v30.b[4] - ext v13.16b, v20.16b, v21.16b, #6 - sxtl v27.8h, v27.8b - sxtl v28.8h, v28.8b - - smlal v4.4s, v8.4h, v22.4h - smlal v4.4s, v9.4h, v23.4h - smlal v4.4s, v10.4h, v24.4h - smlal v4.4s, v11.4h, v26.4h - smlal v4.4s, v12.4h, v27.4h - smlal v4.4s, v13.4h, v28.4h - smlal v4.4s, v20.4h, v25.4h - mov v16.16b, v17.16b - mov v17.16b, v18.16b - smlal2 v5.4s, v8.8h, v22.8h - smlal2 v5.4s, v9.8h, v23.8h - smlal2 v5.4s, v10.8h, v24.8h - smlal2 v5.4s, v11.8h, v26.8h - smlal2 v5.4s, v12.8h, v27.8h - smlal2 v5.4s, v13.8h, v28.8h - smlal2 v5.4s, v20.8h, v25.8h - - mov v13.16b, v14.16b - mov v14.16b, v15.16b - - mov v19.16b, v20.16b - mov v20.16b, v21.16b - ret -endfunc - -.macro sum_lag3_func type, uv_layout, edge, elems=8 -function sum_\type\()_lag3_\edge\()_neon - str x30, [sp, #-16]! -.ifc \edge, left - sub x11, x0, #3*GRAIN_WIDTH*2 - sub x12, x0, #2*GRAIN_WIDTH*2 - sub x13, x0, #1*GRAIN_WIDTH*2 - ld1 {v14.8h}, [x11] // load the previous block right above - ld1 {v17.8h}, [x12] - ld1 {v20.8h}, [x13] -.endif - sum_lag_n_body lag3, \type, \uv_layout, \edge, \elems, v30.b[8] -endfunc -.endm - -sum_lag3_func y, 0, left -sum_lag3_func y, 0, mid -sum_lag3_func y, 0, right, 7 -sum_lag3_func uv_444, 444, left -sum_lag3_func uv_444, 444, mid -sum_lag3_func uv_444, 444, right, 7 -sum_lag3_func uv_422, 422, left -sum_lag3_func uv_422, 422, mid -sum_lag3_func uv_422, 422, right, 1 -sum_lag3_func uv_420, 420, left -sum_lag3_func uv_420, 420, mid -sum_lag3_func uv_420, 420, right, 1 - -function generate_grain_rows_neon - str x30, [sp, #-16]! -1: - mov w16, #80 -2: - bl get_gaussian_neon - srshl v0.8h, v0.8h, v31.8h - subs w16, w16, #8 - st1 {v0.8h}, [x0], #16 - b.gt 2b - get_grain_2 v0 - subs w1, w1, #1 - st1 {v0.s}[0], [x0], #4 - b.gt 1b - ldr x30, [sp], #16 - ret -endfunc - -function generate_grain_rows_44_neon - str x30, [sp, #-16]! -1: - mov w16, #40 -2: - bl get_gaussian_neon - srshl v0.8h, v0.8h, v31.8h - subs w16, w16, #8 - st1 {v0.8h}, [x0], #16 - b.gt 2b - get_grain_4 v0 - subs w1, w1, #1 - st1 {v0.4h}, [x0] - add x0, x0, #GRAIN_WIDTH*2-80 - b.gt 1b - ldr x30, [sp], #16 - ret -endfunc - -function gen_grain_uv_444_lag0_neon - str x30, [sp, #-16]! - ld1 {v4.8h}, [x19], #16 -gen_grain_uv_lag0_8_start: - bl get_gaussian_neon - srshl v0.8h, v0.8h, v31.8h -gen_grain_uv_lag0_8_add: - and v4.16b, v4.16b, v1.16b - smull v2.4s, v4.4h, v27.4h - smull2 v3.4s, v4.8h, v27.8h - srshl v2.4s, v2.4s, v28.4s - srshl v3.4s, v3.4s, v28.4s - sqxtn v2.4h, v2.4s - sqxtn2 v2.8h, v3.4s - sqadd v2.8h, v2.8h, v0.8h - smin v2.8h, v2.8h, v25.8h - smax v2.8h, v2.8h, v26.8h - st1 {v2.8h}, [x0], #16 - ldr x30, [sp], #16 - ret -endfunc - -function gen_grain_uv_420_lag0_8_neon - add x12, x19, #GRAIN_WIDTH*2 - str x30, [sp, #-16]! - ld1 {v16.8h, v17.8h}, [x19], #32 - ld1 {v18.8h, v19.8h}, [x12] - addp v16.8h, v16.8h, v17.8h - addp v17.8h, v18.8h, v19.8h - add v16.8h, v16.8h, v17.8h - srshr v4.8h, v16.8h, #2 - b gen_grain_uv_lag0_8_start -endfunc - -function gen_grain_uv_422_lag0_8_neon - str x30, [sp, #-16]! - ld1 {v16.8h, v17.8h}, [x19], #32 - addp v16.8h, v16.8h, v17.8h - srshr v4.8h, v16.8h, #1 - b gen_grain_uv_lag0_8_start -endfunc - -function gen_grain_uv_420_lag0_4_neon - add x12, x19, #GRAIN_WIDTH*2 - str x30, [sp, #-16]! - ld1 {v16.4h, v17.4h}, [x19] - ld1 {v18.4h, v19.4h}, [x12] - add x19, x19, #32 - addp v16.4h, v16.4h, v17.4h - addp v17.4h, v18.4h, v19.4h - add v16.4h, v16.4h, v17.4h - srshr v4.4h, v16.4h, #2 - get_grain_4 v0 - b gen_grain_uv_lag0_8_add -endfunc - -function gen_grain_uv_422_lag0_4_neon - str x30, [sp, #-16]! - ld1 {v16.4h, v17.4h}, [x19] - add x19, x19, #32 - addp v16.4h, v16.4h, v17.4h - srshr v4.4h, v16.4h, #1 - get_grain_4 v0 - b gen_grain_uv_lag0_8_add -endfunc - -.macro gen_grain_82 type -function generate_grain_\type\()_16bpc_neon, export=1 - stp x30, x19, [sp, #-96]! - -.ifc \type, uv_444 - mov w13, w3 - mov w14, #28 - add x19, x1, #3*GRAIN_WIDTH*2 - mov x1, x2 - mul w13, w13, w14 - clz w15, w4 -.else - clz w15, w2 -.endif - movrel x3, X(gaussian_sequence) - sub w15, w15, #24 // -bitdepth_min_8 - ldr w2, [x1, #FGD_SEED] - ldr w9, [x1, #FGD_GRAIN_SCALE_SHIFT] -.ifc \type, y - add x4, x1, #FGD_AR_COEFFS_Y -.else - add x4, x1, #FGD_AR_COEFFS_UV -.endif - add w9, w9, w15 // grain_scale_shift - bitdepth_min_8 - adr x16, L(gen_grain_\type\()_tbl) - ldr w17, [x1, #FGD_AR_COEFF_LAG] - add w9, w9, #4 - ldrh w17, [x16, w17, uxtw #1] - dup v31.8h, w9 // 4 - bitdepth_min_8 + data->grain_scale_shift - sub x16, x16, w17, uxtw - neg v31.8h, v31.8h - -.ifc \type, uv_444 - cmp w13, #0 - mov w11, #0x49d8 - mov w14, #0xb524 - add x4, x4, w13, uxtw // Add offset to ar_coeffs_uv[1] - csel w11, w11, w14, ne -.endif - - ldr w7, [x1, #FGD_AR_COEFF_SHIFT] - neg w15, w15 // bitdepth_min_8 - mov w8, #1 - mov w10, #1 - lsl w8, w8, w7 // 1 << ar_coeff_shift - lsl w10, w10, w9 // 1 << (4 + data->grain_scale_shift) - lsr w8, w8, #1 // 1 << (ar_coeff_shift - 1) - lsr w10, w10, #1 // 1 << (4 + data->grain_scale_shift - 1) - mov w5, #128 - lsl w5, w5, w15 // 128 << bitdepth_min_8 - neg w6, w5 // -(128 << bitpdeth_min_8) - sub w5, w5, #1 // (128 << bitdepth_min_8) - 1 - -.ifc \type, uv_444 - eor w2, w2, w11 -.endif - - br x16 - -L(generate_grain_\type\()_lag0): - AARCH64_VALID_JUMP_TARGET -.ifc \type, y - mov w1, #GRAIN_HEIGHT - bl generate_grain_rows_neon -.else - dup v28.4s, w7 - ld1r {v27.8b}, [x4] // ar_coeffs_uv[0] - movi v0.16b, #0 - movi v1.16b, #255 - dup v25.8h, w5 - dup v26.8h, w6 - ext v29.16b, v0.16b, v1.16b, #10 - ext v30.16b, v1.16b, v0.16b, #2 - neg v28.4s, v28.4s - sxtl v27.8h, v27.8b - - mov w1, #3 - bl generate_grain_rows_neon - mov w1, #GRAIN_HEIGHT-3 -1: - mov v1.16b, v29.16b - bl gen_grain_uv_444_lag0_neon // 8 - movi v1.16b, #255 - bl gen_grain_uv_444_lag0_neon // 16 - bl gen_grain_uv_444_lag0_neon // 24 - bl gen_grain_uv_444_lag0_neon // 32 - bl gen_grain_uv_444_lag0_neon // 40 - bl gen_grain_uv_444_lag0_neon // 48 - bl gen_grain_uv_444_lag0_neon // 56 - bl gen_grain_uv_444_lag0_neon // 64 - bl gen_grain_uv_444_lag0_neon // 72 - mov v1.16b, v30.16b - bl gen_grain_uv_444_lag0_neon // 80 - get_grain_2 v16 - subs w1, w1, #1 - add x19, x19, #4 - st1 {v16.s}[0], [x0], #4 - b.gt 1b -.endif - ldp x30, x19, [sp], #96 - ret - -L(generate_grain_\type\()_lag1): - AARCH64_VALID_JUMP_TARGET - ld1r {v27.8b}, [x4], #1 // ar_coeffs_y[0] - ld1r {v28.8b}, [x4], #1 // ar_coeffs_y[1] - ld1r {v29.8b}, [x4] // ar_coeffs_y[2] -.ifc \type, y - ldrsb w4, [x4, #1] // ar_coeffs_y[3] -.else - add x4, x4, #2 -.endif - - mov w1, #3 -.ifc \type, uv_444 - ld1r {v30.8b}, [x4] // ar_coeffs_uv[4] - ldursb w4, [x4, #-1] // ar_coeffs_uv[3] -.endif - bl generate_grain_rows_neon - sxtl v27.8h, v27.8b - sxtl v28.8h, v28.8b - sxtl v29.8h, v29.8b -.ifc \type, uv_444 - sxtl v30.8h, v30.8b -.endif - - mov w1, #GRAIN_HEIGHT - 3 -1: - bl sum_\type\()_lag1_left_neon // 8 - bl sum_\type\()_lag1_mid_neon // 16 - bl sum_\type\()_lag1_mid_neon // 24 - bl sum_\type\()_lag1_mid_neon // 32 - bl sum_\type\()_lag1_mid_neon // 40 - bl sum_\type\()_lag1_mid_neon // 48 - bl sum_\type\()_lag1_mid_neon // 56 - bl sum_\type\()_lag1_mid_neon // 64 - bl sum_\type\()_lag1_mid_neon // 72 - bl sum_\type\()_lag1_right_neon // 80 - get_grain_2 v16 - subs w1, w1, #1 -.ifc \type, uv_444 - add x19, x19, #4 -.endif - st1 {v16.s}[0], [x0], #4 - b.gt 1b - - ldp x30, x19, [sp], #96 - ret - -L(generate_grain_\type\()_lag2): - AARCH64_VALID_JUMP_TARGET - ld1 {v30.16b}, [x4] // ar_coeffs_y[0-11], ar_coeffs_uv[0-12] - - smov w4, v30.b[10] - smov w17, v30.b[11] - - mov w1, #3 - bl generate_grain_rows_neon - - mov w1, #GRAIN_HEIGHT - 3 -1: - bl sum_\type\()_lag2_left_neon // 8 - bl sum_\type\()_lag2_mid_neon // 16 - bl sum_\type\()_lag2_mid_neon // 24 - bl sum_\type\()_lag2_mid_neon // 32 - bl sum_\type\()_lag2_mid_neon // 40 - bl sum_\type\()_lag2_mid_neon // 48 - bl sum_\type\()_lag2_mid_neon // 56 - bl sum_\type\()_lag2_mid_neon // 64 - bl sum_\type\()_lag2_mid_neon // 72 - bl sum_\type\()_lag2_right_neon // 80 - get_grain_2 v16 - subs w1, w1, #1 -.ifc \type, uv_444 - add x19, x19, #4 -.endif - st1 {v16.s}[0], [x0], #4 - b.gt 1b - - ldp x30, x19, [sp], #96 - ret - -L(generate_grain_\type\()_lag3): - AARCH64_VALID_JUMP_TARGET - ld1 {v29.16b, v30.16b}, [x4] // ar_coeffs_y[0-23], ar_coeffs_uv[0-24] - stp d8, d9, [sp, #16] - stp d10, d11, [sp, #32] - stp d12, d13, [sp, #48] - stp d14, d15, [sp, #64] - stp x20, x21, [sp, #80] - - smov w4, v30.b[5] - smov w20, v30.b[6] - smov w21, v30.b[7] - - mov w1, #3 - bl generate_grain_rows_neon - - mov w1, #GRAIN_HEIGHT - 3 -1: - bl sum_\type\()_lag3_left_neon // 8 - bl sum_\type\()_lag3_mid_neon // 16 - bl sum_\type\()_lag3_mid_neon // 24 - bl sum_\type\()_lag3_mid_neon // 32 - bl sum_\type\()_lag3_mid_neon // 40 - bl sum_\type\()_lag3_mid_neon // 48 - bl sum_\type\()_lag3_mid_neon // 56 - bl sum_\type\()_lag3_mid_neon // 64 - bl sum_\type\()_lag3_mid_neon // 72 - bl sum_\type\()_lag3_right_neon // 80 - get_grain_2 v16 - subs w1, w1, #1 -.ifc \type, uv_444 - add x19, x19, #4 -.endif - st1 {v16.s}[0], [x0], #4 - b.gt 1b - - ldp x20, x21, [sp, #80] - ldp d14, d15, [sp, #64] - ldp d12, d13, [sp, #48] - ldp d10, d11, [sp, #32] - ldp d8, d9, [sp, #16] - ldp x30, x19, [sp], #96 - ret - -L(gen_grain_\type\()_tbl): - .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag0) - .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag1) - .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag2) - .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag3) -endfunc -.endm - -gen_grain_82 y -gen_grain_82 uv_444 - -.macro set_height dst, type -.ifc \type, uv_420 - mov \dst, #SUB_GRAIN_HEIGHT-3 -.else - mov \dst, #GRAIN_HEIGHT-3 -.endif -.endm - -.macro increment_y_ptr reg, type -.ifc \type, uv_420 - add \reg, \reg, #2*GRAIN_WIDTH*2-(6*32) -.else - sub \reg, \reg, #6*32-GRAIN_WIDTH*2 -.endif -.endm - -.macro gen_grain_44 type -function generate_grain_\type\()_16bpc_neon, export=1 - stp x30, x19, [sp, #-96]! - - mov w13, w3 - mov w14, #28 - add x19, x1, #(3*GRAIN_WIDTH-3)*2 - mov x1, x2 - mul w13, w13, w14 - clz w15, w4 - - movrel x3, X(gaussian_sequence) - sub w15, w15, #24 // -bitdepth_min_8 - ldr w2, [x1, #FGD_SEED] - ldr w9, [x1, #FGD_GRAIN_SCALE_SHIFT] - add x4, x1, #FGD_AR_COEFFS_UV - add w9, w9, w15 // grain_scale_shift - bitdepth_min_8 - adr x16, L(gen_grain_\type\()_tbl) - ldr w17, [x1, #FGD_AR_COEFF_LAG] - add w9, w9, #4 - ldrh w17, [x16, w17, uxtw #1] - dup v31.8h, w9 // 4 - bitdepth_min_8 + data->grain_scale_shift - sub x16, x16, w17, uxtw - neg v31.8h, v31.8h - - cmp w13, #0 - mov w11, #0x49d8 - mov w14, #0xb524 - add x4, x4, w13, uxtw // Add offset to ar_coeffs_uv[1] - csel w11, w11, w14, ne - - ldr w7, [x1, #FGD_AR_COEFF_SHIFT] - neg w15, w15 // bitdepth_min_8 - mov w8, #1 - mov w10, #1 - lsl w8, w8, w7 // 1 << ar_coeff_shift - lsl w10, w10, w9 // 1 << (4 + data->grain_scale_shift) - lsr w8, w8, #1 // 1 << (ar_coeff_shift - 1) - lsr w10, w10, #1 // 1 << (4 + data->grain_scale_shift - 1) - mov w5, #128 - lsl w5, w5, w15 // 128 << bitdepth_min_8 - neg w6, w5 // -(128 << bitpdeth_min_8) - sub w5, w5, #1 // (128 << bitdepth_min_8) - 1 - - eor w2, w2, w11 - - br x16 - -L(generate_grain_\type\()_lag0): - AARCH64_VALID_JUMP_TARGET - dup v28.4s, w7 - ld1r {v27.8b}, [x4] // ar_coeffs_uv[0] - movi v0.16b, #0 - movi v1.16b, #255 - dup v25.8h, w5 - dup v26.8h, w6 - ext v29.16b, v0.16b, v1.16b, #10 - ext v30.16b, v1.16b, v0.16b, #14 - neg v28.4s, v28.4s - sxtl v27.8h, v27.8b - - mov w1, #3 - bl generate_grain_rows_44_neon - set_height w1, \type -1: - mov v1.16b, v29.16b - bl gen_grain_\type\()_lag0_8_neon // 8 - movi v1.16b, #255 - bl gen_grain_\type\()_lag0_8_neon // 16 - bl gen_grain_\type\()_lag0_8_neon // 24 - bl gen_grain_\type\()_lag0_8_neon // 32 - bl gen_grain_\type\()_lag0_8_neon // 40 - mov v1.16b, v30.16b - bl gen_grain_\type\()_lag0_4_neon // 44 - subs w1, w1, #1 - increment_y_ptr x19, \type - add x0, x0, #GRAIN_WIDTH*2-6*16 - b.gt 1b - - ldp x30, x19, [sp], #96 - ret - -L(generate_grain_\type\()_lag1): - AARCH64_VALID_JUMP_TARGET - ld1r {v27.8b}, [x4], #1 // ar_coeffs_uv[0] - ld1r {v28.8b}, [x4], #1 // ar_coeffs_uv[1] - ld1r {v29.8b}, [x4] // ar_coeffs_uv[2] - add x4, x4, #2 - - mov w1, #3 - ld1r {v30.8b}, [x4] // ar_coeffs_u4[4] - ldursb w4, [x4, #-1] // ar_coeffs_uv[3] - bl generate_grain_rows_44_neon - - sxtl v27.8h, v27.8b - sxtl v28.8h, v28.8b - sxtl v29.8h, v29.8b - sxtl v30.8h, v30.8b - set_height w1, \type -1: - bl sum_\type\()_lag1_left_neon // 8 - bl sum_\type\()_lag1_mid_neon // 16 - bl sum_\type\()_lag1_mid_neon // 24 - bl sum_\type\()_lag1_mid_neon // 32 - bl sum_\type\()_lag1_mid_neon // 40 - bl sum_\type\()_lag1_right_neon // 44 - subs w1, w1, #1 - increment_y_ptr x19, \type - add x0, x0, #GRAIN_WIDTH*2-6*16 - b.gt 1b - - ldp x30, x19, [sp], #96 - ret - -L(generate_grain_\type\()_lag2): - AARCH64_VALID_JUMP_TARGET - ld1 {v30.16b}, [x4] // ar_coeffs_uv[0-12] - - smov w4, v30.b[10] - smov w17, v30.b[11] - - mov w1, #3 - bl generate_grain_rows_44_neon - - set_height w1, \type -1: - bl sum_\type\()_lag2_left_neon // 8 - bl sum_\type\()_lag2_mid_neon // 16 - bl sum_\type\()_lag2_mid_neon // 24 - bl sum_\type\()_lag2_mid_neon // 32 - bl sum_\type\()_lag2_mid_neon // 40 - bl sum_\type\()_lag2_right_neon // 44 - subs w1, w1, #1 - increment_y_ptr x19, \type - add x0, x0, #GRAIN_WIDTH*2-6*16 - b.gt 1b - - ldp x30, x19, [sp], #96 - ret - -L(generate_grain_\type\()_lag3): - AARCH64_VALID_JUMP_TARGET - ldr q29, [x4] // ar_coeffs_uv[0-15] - ldr q30, [x4, #16] // ar_coeffs_uv[16-24] - stp d8, d9, [sp, #16] - stp d10, d11, [sp, #32] - stp d12, d13, [sp, #48] - stp d14, d15, [sp, #64] - stp x20, x21, [sp, #80] - - smov w4, v30.b[5] - smov w20, v30.b[6] - smov w21, v30.b[7] - - mov w1, #3 - bl generate_grain_rows_44_neon - - set_height w1, \type -1: - bl sum_\type\()_lag3_left_neon // 8 - bl sum_\type\()_lag3_mid_neon // 16 - bl sum_\type\()_lag3_mid_neon // 24 - bl sum_\type\()_lag3_mid_neon // 32 - bl sum_\type\()_lag3_mid_neon // 40 - bl sum_\type\()_lag3_right_neon // 44 - subs w1, w1, #1 - increment_y_ptr x19, \type - add x0, x0, #GRAIN_WIDTH*2-6*16 - b.gt 1b - - ldp x20, x21, [sp, #80] - ldp d14, d15, [sp, #64] - ldp d12, d13, [sp, #48] - ldp d10, d11, [sp, #32] - ldp d8, d9, [sp, #16] - ldp x30, x19, [sp], #96 - ret - -L(gen_grain_\type\()_tbl): - .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag0) - .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag1) - .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag2) - .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag3) -endfunc -.endm - -gen_grain_44 uv_420 -gen_grain_44 uv_422 - -.macro gather_interleaved dst1, dst2, src1, src2, off - umov w14, \src1[0] - umov w15, \src2[1] - umov w16, \src1[2] - add x14, x14, x3 - umov w17, \src2[3] - add x15, x15, x3 - ld1 {\dst1}[0+\off], [x14] - umov w14, \src1[4] - add x16, x16, x3 - ld1 {\dst2}[1+\off], [x15] - umov w15, \src2[5] - add x17, x17, x3 - ld1 {\dst1}[2+\off], [x16] - umov w16, \src1[6] - add x14, x14, x3 - ld1 {\dst2}[3+\off], [x17] - umov w17, \src2[7] - add x15, x15, x3 - ld1 {\dst1}[4+\off], [x14] - add x16, x16, x3 - ld1 {\dst2}[5+\off], [x15] - add x17, x17, x3 - ld1 {\dst1}[6+\off], [x16] - ld1 {\dst2}[7+\off], [x17] -.endm - -.macro gather dst1, dst2, src1, src2, src3, src4 - gather_interleaved \dst1, \dst2, \src1, \src3, 0 - gather_interleaved \dst2, \dst1, \src3, \src1, 0 - gather_interleaved \dst1, \dst2, \src2, \src4, 8 - gather_interleaved \dst2, \dst1, \src4, \src2, 8 -.endm - -function gather32_neon - gather v6.b, v7.b, v0.h, v1.h, v2.h, v3.h - ret -endfunc - -function gather16_neon - gather_interleaved v6.b, v7.b, v0.h, v1.h, 0 - gather_interleaved v7.b, v6.b, v1.h, v0.h, 0 - ins v6.d[1], v7.d[0] - ret -endfunc - -const overlap_coeffs_0, align=4 - .short 27, 17, 0, 0 - .short 17, 27, 32, 32 -endconst - -const overlap_coeffs_1, align=4 - .short 23, 0, 0, 0 - .short 22, 32, 32, 32 -endconst - -.macro calc_offset offx, offy, src, sx, sy - and \offy, \src, #0xF // randval & 0xF - lsr \offx, \src, #4 // randval >> 4 -.if \sy == 0 - add \offy, \offy, \offy // 2 * (randval & 0xF) -.endif -.if \sx == 0 - add \offx, \offx, \offx // 2 * (randval >> 4) -.endif -.endm - -.macro add_offset dst, offx, offy, src, stride - madd \dst, \stride, \offy, \src // grain_lut += grain_stride * offy - add \dst, \dst, \offx, uxtw #1 // grain_lut += offx -.endm - -// void dav1d_fgy_32x32_16bpc_neon(pixel *const dst, const pixel *const src, -// const ptrdiff_t stride, -// const uint8_t scaling[SCALING_SIZE], -// const int scaling_shift, -// const entry grain_lut[][GRAIN_WIDTH], -// const int offsets[][2], -// const int h, const ptrdiff_t clip, -// const ptrdiff_t type, -// const int bitdepth_max); -function fgy_32x32_16bpc_neon, export=1 - str x30, [sp, #-80]! - stp d8, d9, [sp, #16] - stp d10, d11, [sp, #32] - stp d12, d13, [sp, #48] - str d14, [sp, #64] - eor w4, w4, #15 // 15 - scaling_shift - ldr w11, [x6, #8] // offsets[1][0] - ldr w13, [x6, #4] // offsets[0][1] - ldr w15, [x6, #12] // offsets[1][1] - ldr w10, [sp, #96] // bitdepth_max - ldr w6, [x6] // offsets[0][0] - dup v26.8h, w10 // bitdepth_max - clz w10, w10 - ldr w8, [sp, #80] // clip - sub w10, w10, #24 // -bitdepth_min_8 - mov x9, #GRAIN_WIDTH*2 // grain_lut stride - neg w10, w10 // bitdepth_min_8 - - dup v29.8h, w4 // 15 - scaling_shift - dup v27.8h, w10 // bitdepth_min_8 - - movrel x16, overlap_coeffs_0 - - cbz w8, 1f - // clip - movi v30.8h, #16 - movi v31.8h, #235 - sshl v30.8h, v30.8h, v27.8h - sshl v31.8h, v31.8h, v27.8h - b 2f -1: - // no clip - movi v30.8h, #0 - mov v31.16b, v26.16b // bitdepth_max -2: - - ushr v26.8h, v26.8h, #1 // grain_max - not v25.16b, v26.16b // grain_min - - ld1 {v27.4h, v28.4h}, [x16] // overlap_coeffs - - add x5, x5, #18 // grain_lut += 9 - add x5, x5, x9, lsl #3 // grain_lut += 8 * grain_stride - add x5, x5, x9 // grain_lut += grain_stride - - calc_offset w11, w12, w11, 0, 0 - calc_offset w13, w14, w13, 0, 0 - calc_offset w15, w16, w15, 0, 0 - calc_offset w6, w10, w6, 0, 0 - - add_offset x12, w11, x12, x5, x9 - add_offset x14, w13, x14, x5, x9 - add_offset x16, w15, x16, x5, x9 - add_offset x5, w6, x10, x5, x9 - - ldr w11, [sp, #88] // type - adr x13, L(fgy_loop_tbl) - - add x4, x12, #32*2 // grain_lut += BLOCK_SIZE * bx - add x6, x14, x9, lsl #5 // grain_lut += grain_stride * BLOCK_SIZE * by - - tst w11, #1 - ldrh w11, [x13, w11, uxtw #1] - - add x8, x16, x9, lsl #5 // grain_lut += grain_stride * BLOCK_SIZE * by - add x8, x8, #32*2 // grain_lut += BLOCK_SIZE * bx - - sub x11, x13, w11, uxtw - - b.eq 1f - // y overlap - dup v8.8h, v27.h[0] - dup v9.8h, v27.h[1] - mov w10, w7 // backup actual h - mov w7, #2 -1: - br x11 -endfunc - -function fgy_loop_neon -.macro fgy ox, oy -L(loop_\ox\oy): - AARCH64_VALID_JUMP_TARGET -1: - ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x1], x2 // src -.if \ox - ld1 {v20.4h}, [x4], x9 // grain_lut old -.endif -.if \oy - ld1 {v21.8h, v22.8h, v23.8h, v24.8h}, [x6], x9 // grain_lut top -.endif -.if \ox && \oy - ld1 {v14.4h}, [x8], x9 // grain_lut top old -.endif - mvni v4.8h, #0xf0, lsl #8 // 0x0fff - ld1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x5], x9 // grain_lut - - // Make sure that uninitialized pixels out of range past the right - // edge are in range; their actual values shouldn't matter. - and v0.16b, v0.16b, v4.16b - and v1.16b, v1.16b, v4.16b - and v2.16b, v2.16b, v4.16b - and v3.16b, v3.16b, v4.16b - bl gather32_neon - -.if \ox - smull v20.4s, v20.4h, v27.4h - smlal v20.4s, v16.4h, v28.4h -.endif - -.if \oy -.if \ox - smull v14.4s, v14.4h, v27.4h - smlal v14.4s, v21.4h, v28.4h - sqrshrn v20.4h, v20.4s, #5 - sqrshrn v14.4h, v14.4s, #5 - smin v20.4h, v20.4h, v26.4h - smin v14.4h, v14.4h, v26.4h - smax v20.4h, v20.4h, v25.4h - smax v14.4h, v14.4h, v25.4h -.endif - -.if \ox - smull v10.4s, v20.4h, v9.4h -.else - smull v10.4s, v16.4h, v9.4h -.endif - smull2 v11.4s, v16.8h, v9.8h - smull v12.4s, v17.4h, v9.4h - smull2 v13.4s, v17.8h, v9.8h - smull v16.4s, v18.4h, v9.4h - smull2 v17.4s, v18.8h, v9.8h - smull v18.4s, v19.4h, v9.4h - smull2 v19.4s, v19.8h, v9.8h -.if \ox - smlal v10.4s, v14.4h, v8.4h -.else - smlal v10.4s, v21.4h, v8.4h -.endif - smlal2 v11.4s, v21.8h, v8.8h - smlal v12.4s, v22.4h, v8.4h - smlal2 v13.4s, v22.8h, v8.8h - smlal v16.4s, v23.4h, v8.4h - smlal2 v17.4s, v23.8h, v8.8h - smlal v18.4s, v24.4h, v8.4h - smlal2 v19.4s, v24.8h, v8.8h - sqrshrn v10.4h, v10.4s, #5 - sqrshrn2 v10.8h, v11.4s, #5 - sqrshrn v11.4h, v12.4s, #5 - sqrshrn2 v11.8h, v13.4s, #5 - sqrshrn v12.4h, v16.4s, #5 - sqrshrn2 v12.8h, v17.4s, #5 - sqrshrn v13.4h, v18.4s, #5 - sqrshrn2 v13.8h, v19.4s, #5 - smin v16.8h, v10.8h, v26.8h - smin v17.8h, v11.8h, v26.8h - smin v18.8h, v12.8h, v26.8h - smin v19.8h, v13.8h, v26.8h - smax v16.8h, v16.8h, v25.8h - smax v17.8h, v17.8h, v25.8h - smax v18.8h, v18.8h, v25.8h - smax v19.8h, v19.8h, v25.8h -.endif - - uxtl v4.8h, v6.8b // scaling -.if \ox && !\oy - sqrshrn v20.4h, v20.4s, #5 -.endif - uxtl2 v5.8h, v6.16b -.if \ox && !\oy - smin v20.4h, v20.4h, v26.4h -.endif - uxtl v6.8h, v7.8b -.if \ox && !\oy - smax v20.4h, v20.4h, v25.4h -.endif - uxtl2 v7.8h, v7.16b -.if \ox && !\oy - ins v16.d[0], v20.d[0] -.endif - ushl v4.8h, v4.8h, v29.8h // scaling << (15 - scaling_shift) - ushl v5.8h, v5.8h, v29.8h - ushl v6.8h, v6.8h, v29.8h - ushl v7.8h, v7.8h, v29.8h - - sqrdmulh v20.8h, v16.8h, v4.8h // round2((scaling << (15 - scaling_shift) * grain, 15) - sqrdmulh v21.8h, v17.8h, v5.8h - sqrdmulh v22.8h, v18.8h, v6.8h - sqrdmulh v23.8h, v19.8h, v7.8h - - usqadd v0.8h, v20.8h // *src + noise - usqadd v1.8h, v21.8h - usqadd v2.8h, v22.8h - usqadd v3.8h, v23.8h - - umax v0.8h, v0.8h, v30.8h - umax v1.8h, v1.8h, v30.8h - umax v2.8h, v2.8h, v30.8h - umax v3.8h, v3.8h, v30.8h - umin v0.8h, v0.8h, v31.8h - umin v1.8h, v1.8h, v31.8h - umin v2.8h, v2.8h, v31.8h - umin v3.8h, v3.8h, v31.8h - - subs w7, w7, #1 -.if \oy - dup v8.8h, v28.h[0] - dup v9.8h, v28.h[1] -.endif - st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x2 // dst - b.gt 1b - -.if \oy - cmp w10, #2 - sub w7, w10, #2 // restore actual remaining h - b.gt L(loop_\ox\()0) -.endif - ldr d14, [sp, #64] - ldp d12, d13, [sp, #48] - ldp d10, d11, [sp, #32] - ldp d8, d9, [sp, #16] - ldr x30, [sp], #80 - ret -.endm - - fgy 0, 0 - fgy 0, 1 - fgy 1, 0 - fgy 1, 1 - -L(fgy_loop_tbl): - .hword L(fgy_loop_tbl) - L(loop_00) - .hword L(fgy_loop_tbl) - L(loop_01) - .hword L(fgy_loop_tbl) - L(loop_10) - .hword L(fgy_loop_tbl) - L(loop_11) -endfunc - -// void dav1d_fguv_32x32_420_16bpc_neon(pixel *const dst, -// const pixel *const src, -// const ptrdiff_t stride, -// const uint8_t scaling[SCALING_SIZE], -// const Dav1dFilmGrainData *const data, -// const entry grain_lut[][GRAIN_WIDTH], -// const pixel *const luma_row, -// const ptrdiff_t luma_stride, -// const int offsets[][2], -// const ptrdiff_t h, const ptrdiff_t uv, -// const ptrdiff_t is_id, -// const ptrdiff_t type, -// const int bitdepth_max); -.macro fguv layout, sx, sy -function fguv_32x32_\layout\()_16bpc_neon, export=1 - str x30, [sp, #-80]! - stp d8, d9, [sp, #16] - stp d10, d11, [sp, #32] - stp d12, d13, [sp, #48] - stp d14, d15, [sp, #64] - - ldp x8, x9, [sp, #80] // offsets, h - ldp x10, x11, [sp, #96] // uv, is_id - ldr w16, [sp, #120] // bitdepth_max - - ldr w13, [x4, #FGD_SCALING_SHIFT] - ldr w12, [x4, #FGD_CLIP_TO_RESTRICTED_RANGE] - dup v23.8h, w16 // bitdepth_max - clz w16, w16 - eor w13, w13, #15 // 15 - scaling_shift - sub w16, w16, #24 // -bitdepth_min_8 - - // !csfl - add x10, x4, x10, lsl #2 // + 4*uv - add x14, x10, #FGD_UV_LUMA_MULT - add x15, x10, #FGD_UV_MULT - add x10, x10, #FGD_UV_OFFSET - neg w16, w16 // bitdepth_min_8 - ld1r {v8.8h}, [x14] // uv_luma_mult - ld1r {v24.8h}, [x10] // uv_offset - ld1r {v9.8h}, [x15] // uv_mult - - dup v29.8h, w13 // 15 - scaling_shift - dup v27.8h, w16 // bitdepth_min_8 - - cbz w12, 1f - // clip - movi v30.8h, #16 - movi v31.8h, #240 - sshl v30.8h, v30.8h, v27.8h - sshl v31.8h, v31.8h, v27.8h - cbz w11, 2f - // is_id - movi v31.8h, #235 - sshl v31.8h, v31.8h, v27.8h - b 2f -1: - // no clip - movi v30.8h, #0 - mov v31.16b, v23.16b // bitdepth_max -2: - - ushr v15.8h, v23.8h, #1 // grain_max - sshl v24.8h, v24.8h, v27.8h // uv_offset << bitdepth_min_8 - not v14.16b, v15.16b // grain_min - - ldr w12, [x8, #8] // offsets[1][0] - ldr w14, [x8, #4] // offsets[0][1] - ldr w16, [x8, #12] // offsets[1][1] - ldr w8, [x8] // offsets[0][0] - - mov x10, #GRAIN_WIDTH*2 // grain_lut stride - - add x5, x5, #(2*(3 + (2 >> \sx)*3)) // grain_lut += 9 or 6 -.if \sy - add x5, x5, x10, lsl #2 // grain_lut += 4 * grain_stride - add x5, x5, x10, lsl #1 // grain_lut += 2 * grain_stride -.else - add x5, x5, x10, lsl #3 // grain_lut += 8 * grain_stride - add x5, x5, x10 // grain_lut += grain_stride -.endif - - calc_offset w12, w13, w12, \sx, \sy - calc_offset w14, w15, w14, \sx, \sy - calc_offset w16, w17, w16, \sx, \sy - calc_offset w8, w11, w8, \sx, \sy - - add_offset x13, w12, x13, x5, x10 - add_offset x15, w14, x15, x5, x10 - add_offset x17, w16, x17, x5, x10 - add_offset x5, w8, x11, x5, x10 - - add x4, x13, #2*(32 >> \sx) // grain_lut += BLOCK_SIZE * bx - add x8, x15, x10, lsl #(5 - \sy) // grain_lut += grain_stride * BLOCK_SIZE * by - add x11, x17, x10, lsl #(5 - \sy) // grain_lut += grain_stride * BLOCK_SIZE * by - add x11, x11, #2*(32 >> \sx) // grain_lut += BLOCK_SIZE * bx - - ldr w13, [sp, #112] // type - - movrel x16, overlap_coeffs_\sx - adr x14, L(fguv_loop_sx\sx\()_tbl) - - ld1 {v27.4h, v28.4h}, [x16] // overlap_coeffs - tst w13, #1 - ldrh w13, [x14, w13, uxtw #1] - - b.eq 1f - // y overlap - sub w12, w9, #(2 >> \sy) // backup remaining h - mov w9, #(2 >> \sy) - -1: - sub x13, x14, w13, uxtw - -.if \sy - movi v25.8h, #23 - movi v26.8h, #22 -.else - movi v25.8h, #27 - movi v26.8h, #17 -.endif - -.if \sy - add x7, x7, x7 // luma_stride *= 2 -.endif - - br x13 -endfunc -.endm - -fguv 420, 1, 1 -fguv 422, 1, 0 -fguv 444, 0, 0 - -function fguv_loop_sx0_neon -.macro fguv_loop_sx0 csfl, ox, oy -L(fguv_loop_sx0_csfl\csfl\()_\ox\oy): - AARCH64_VALID_JUMP_TARGET -1: -.if \ox - ld1 {v4.4h}, [x4], x10 // grain_lut old -.endif -.if \oy - ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x8], x10 // grain_lut top -.endif -.if \ox && \oy - ld1 {v5.4h}, [x11], x10 // grain_lut top old -.endif - ld1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x5], x10 // grain_lut - -.if \ox - smull v4.4s, v4.4h, v27.4h - smlal v4.4s, v16.4h, v28.4h -.endif - -.if \oy -.if \ox - smull v5.4s, v5.4h, v27.4h - smlal v5.4s, v0.4h, v28.4h - sqrshrn v4.4h, v4.4s, #5 - sqrshrn v5.4h, v5.4s, #5 - smin v4.4h, v4.4h, v15.4h - smin v5.4h, v5.4h, v15.4h - smax v4.4h, v4.4h, v14.4h - smax v5.4h, v5.4h, v14.4h - ins v16.d[0], v4.d[0] - ins v0.d[0], v5.d[0] -.endif - - smull v6.4s, v16.4h, v26.4h - smull2 v7.4s, v16.8h, v26.8h - smull v10.4s, v17.4h, v26.4h - smull2 v11.4s, v17.8h, v26.8h - smull v16.4s, v18.4h, v26.4h - smull2 v17.4s, v18.8h, v26.8h - smull v18.4s, v19.4h, v26.4h - smull2 v19.4s, v19.8h, v26.8h - smlal v6.4s, v0.4h, v25.4h - smlal2 v7.4s, v0.8h, v25.8h - smlal v10.4s, v1.4h, v25.4h - smlal2 v11.4s, v1.8h, v25.8h - smlal v16.4s, v2.4h, v25.4h - smlal2 v17.4s, v2.8h, v25.8h - smlal v18.4s, v3.4h, v25.4h - smlal2 v19.4s, v3.8h, v25.8h - sqrshrn v6.4h, v6.4s, #5 - sqrshrn2 v6.8h, v7.4s, #5 - sqrshrn v7.4h, v10.4s, #5 - sqrshrn2 v7.8h, v11.4s, #5 - sqrshrn v10.4h, v16.4s, #5 - sqrshrn2 v10.8h, v17.4s, #5 - sqrshrn v11.4h, v18.4s, #5 - sqrshrn2 v11.8h, v19.4s, #5 -.endif - -.if \ox && !\oy - sqrshrn v4.4h, v4.4s, #5 - smin v4.4h, v4.4h, v15.4h -.endif - ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x7 // luma -.if \oy - smin v16.8h, v6.8h, v15.8h - smin v17.8h, v7.8h, v15.8h - smin v18.8h, v10.8h, v15.8h - smin v19.8h, v11.8h, v15.8h - smax v16.8h, v16.8h, v14.8h - smax v17.8h, v17.8h, v14.8h - smax v18.8h, v18.8h, v14.8h - smax v19.8h, v19.8h, v14.8h -.endif - -.if \ox && !\oy - smax v4.4h, v4.4h, v14.4h -.endif - ld1 {v10.8h, v11.8h, v12.8h, v13.8h}, [x1], x2 // src -.if \ox && !\oy - ins v16.d[0], v4.d[0] -.endif - -.if !\csfl - smull v4.4s, v0.4h, v8.4h - smull2 v5.4s, v0.8h, v8.8h - smull v6.4s, v1.4h, v8.4h - smull2 v7.4s, v1.8h, v8.8h - smull v0.4s, v2.4h, v8.4h - smull2 v1.4s, v2.8h, v8.8h - smull v2.4s, v3.4h, v8.4h - smull2 v3.4s, v3.8h, v8.8h - smlal v4.4s, v10.4h, v9.4h - smlal2 v5.4s, v10.8h, v9.8h - smlal v6.4s, v11.4h, v9.4h - smlal2 v7.4s, v11.8h, v9.8h - smlal v0.4s, v12.4h, v9.4h - smlal2 v1.4s, v12.8h, v9.8h - smlal v2.4s, v13.4h, v9.4h - smlal2 v3.4s, v13.8h, v9.8h - shrn v4.4h, v4.4s, #6 - shrn2 v4.8h, v5.4s, #6 - shrn v5.4h, v6.4s, #6 - shrn2 v5.8h, v7.4s, #6 - shrn v6.4h, v0.4s, #6 - shrn2 v6.8h, v1.4s, #6 - shrn v7.4h, v2.4s, #6 - shrn2 v7.8h, v3.4s, #6 - add v0.8h, v4.8h, v24.8h - add v1.8h, v5.8h, v24.8h - add v2.8h, v6.8h, v24.8h - add v3.8h, v7.8h, v24.8h - movi v20.8h, #0 - smin v0.8h, v0.8h, v23.8h - smin v1.8h, v1.8h, v23.8h - smin v2.8h, v2.8h, v23.8h - smin v3.8h, v3.8h, v23.8h - smax v0.8h, v0.8h, v20.8h - smax v1.8h, v1.8h, v20.8h - smax v2.8h, v2.8h, v20.8h - smax v3.8h, v3.8h, v20.8h -.else - // Make sure that uninitialized pixels out of range past the right - // edge are in range; their actual values shouldn't matter. - and v0.16b, v0.16b, v23.16b - and v1.16b, v1.16b, v23.16b - and v2.16b, v2.16b, v23.16b - and v3.16b, v3.16b, v23.16b -.endif - - bl gather32_neon - - uxtl v4.8h, v6.8b // scaling - uxtl2 v5.8h, v6.16b - uxtl v6.8h, v7.8b - uxtl2 v7.8h, v7.16b - - ushl v4.8h, v4.8h, v29.8h // scaling << (15 - scaling_shift) - ushl v5.8h, v5.8h, v29.8h - ushl v6.8h, v6.8h, v29.8h - ushl v7.8h, v7.8h, v29.8h - - sqrdmulh v16.8h, v16.8h, v4.8h // round2((scaling << (15 - scaling_shift) * grain, 15) - sqrdmulh v17.8h, v17.8h, v5.8h - sqrdmulh v18.8h, v18.8h, v6.8h - sqrdmulh v19.8h, v19.8h, v7.8h - - usqadd v10.8h, v16.8h // *src + noise - usqadd v11.8h, v17.8h - usqadd v12.8h, v18.8h - usqadd v13.8h, v19.8h - - umax v0.8h, v10.8h, v30.8h - umax v1.8h, v11.8h, v30.8h - umax v2.8h, v12.8h, v30.8h - umax v3.8h, v13.8h, v30.8h - umin v0.8h, v0.8h, v31.8h - umin v1.8h, v1.8h, v31.8h - umin v2.8h, v2.8h, v31.8h - umin v3.8h, v3.8h, v31.8h - - subs w9, w9, #1 -.if \oy - dup v25.8h, v28.h[0] - dup v26.8h, v28.h[1] -.endif - st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x2 // dst - b.gt 1b - -.if \oy - cmp w12, #0 - mov w9, w12 // restore actual remaining h - b.gt L(fguv_loop_sx0_csfl\csfl\()_\ox\()0) -.endif - b 9f -.endm - fguv_loop_sx0 0, 0, 0 - fguv_loop_sx0 0, 0, 1 - fguv_loop_sx0 0, 1, 0 - fguv_loop_sx0 0, 1, 1 - fguv_loop_sx0 1, 0, 0 - fguv_loop_sx0 1, 0, 1 - fguv_loop_sx0 1, 1, 0 - fguv_loop_sx0 1, 1, 1 - -9: - ldp d14, d15, [sp, #64] - ldp d12, d13, [sp, #48] - ldp d10, d11, [sp, #32] - ldp d8, d9, [sp, #16] - ldr x30, [sp], #80 - ret - -L(fguv_loop_sx0_tbl): - .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl0_00) - .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl0_01) - .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl0_10) - .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl0_11) - .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl1_00) - .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl1_01) - .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl1_10) - .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl1_11) -endfunc - -function fguv_loop_sx1_neon -.macro fguv_loop_sx1 csfl, ox, oy -L(fguv_loop_sx1_csfl\csfl\()_\ox\oy): - AARCH64_VALID_JUMP_TARGET -1: -.if \ox - ld1 {v18.4h}, [x4], x10 // grain_lut old -.endif -.if \oy - ld1 {v20.8h, v21.8h}, [x8], x10 // grain_lut top -.endif -.if \ox && \oy - ld1 {v19.4h}, [x11], x10 // grain_lut top old -.endif - ld1 {v16.8h, v17.8h}, [x5], x10 // grain_lut - -.if \ox - smull v18.4s, v18.4h, v27.4h - smlal v18.4s, v16.4h, v28.4h -.endif - -.if \oy -.if \ox - smull v19.4s, v19.4h, v27.4h - smlal v19.4s, v20.4h, v28.4h - sqrshrn v18.4h, v18.4s, #5 - sqrshrn v19.4h, v19.4s, #5 - smin v18.4h, v18.4h, v15.4h - smin v19.4h, v19.4h, v15.4h - smax v18.4h, v18.4h, v14.4h - smax v19.4h, v19.4h, v14.4h - ins v16.d[0], v18.d[0] - ins v20.d[0], v19.d[0] -.endif - - smull v0.4s, v16.4h, v26.4h - smull2 v1.4s, v16.8h, v26.8h - smull v2.4s, v17.4h, v26.4h - smull2 v3.4s, v17.8h, v26.8h - smlal v0.4s, v20.4h, v25.4h - smlal2 v1.4s, v20.8h, v25.8h - smlal v2.4s, v21.4h, v25.4h - smlal2 v3.4s, v21.8h, v25.8h - sqrshrn v16.4h, v0.4s, #5 - sqrshrn2 v16.8h, v1.4s, #5 - sqrshrn v17.4h, v2.4s, #5 - sqrshrn2 v17.8h, v3.4s, #5 -.endif - -.if \ox && !\oy - sqrshrn v18.4h, v18.4s, #5 - smin v18.4h, v18.4h, v15.4h -.endif - ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x7 // luma -.if \oy - smin v16.8h, v16.8h, v15.8h - smin v17.8h, v17.8h, v15.8h - smax v16.8h, v16.8h, v14.8h - smax v17.8h, v17.8h, v14.8h -.endif - -.if \ox && !\oy - smax v18.4h, v18.4h, v14.4h -.endif - ld1 {v10.8h, v11.8h}, [x1], x2 // src -.if \ox && !\oy - ins v16.d[0], v18.d[0] -.endif - addp v0.8h, v0.8h, v1.8h - addp v1.8h, v2.8h, v3.8h - urshr v0.8h, v0.8h, #1 - urshr v1.8h, v1.8h, #1 -.if !\csfl - smull v2.4s, v0.4h, v8.4h - smull2 v3.4s, v0.8h, v8.8h - smull v0.4s, v1.4h, v8.4h - smull2 v1.4s, v1.8h, v8.8h - smlal v2.4s, v10.4h, v9.4h - smlal2 v3.4s, v10.8h, v9.8h - smlal v0.4s, v11.4h, v9.4h - smlal2 v1.4s, v11.8h, v9.8h - shrn v2.4h, v2.4s, #6 - shrn2 v2.8h, v3.4s, #6 - shrn v3.4h, v0.4s, #6 - shrn2 v3.8h, v1.4s, #6 - add v0.8h, v2.8h, v24.8h - add v1.8h, v3.8h, v24.8h - movi v2.8h, #0 - smin v0.8h, v0.8h, v23.8h - smin v1.8h, v1.8h, v23.8h - smax v0.8h, v0.8h, v2.8h - smax v1.8h, v1.8h, v2.8h -.else - // Make sure that uninitialized pixels out of range past the right - // edge are in range; their actual values shouldn't matter. - and v0.16b, v0.16b, v23.16b - and v1.16b, v1.16b, v23.16b -.endif - - bl gather16_neon - - uxtl v4.8h, v6.8b // scaling - uxtl2 v5.8h, v6.16b - - ushl v4.8h, v4.8h, v29.8h // scaling << (15 - scaling_shift) - ushl v5.8h, v5.8h, v29.8h - - sqrdmulh v16.8h, v16.8h, v4.8h // round2((scaling << (15 - scaling_shift) * grain, 15) - sqrdmulh v17.8h, v17.8h, v5.8h - - usqadd v10.8h, v16.8h // *src + noise - usqadd v11.8h, v17.8h - - umax v0.8h, v10.8h, v30.8h - umax v1.8h, v11.8h, v30.8h - umin v0.8h, v0.8h, v31.8h - umin v1.8h, v1.8h, v31.8h - -.if \oy - mov v16.16b, v25.16b -.endif - subs w9, w9, #1 -.if \oy - mov v25.16b, v26.16b - mov v26.16b, v16.16b -.endif - st1 {v0.8h, v1.8h}, [x0], x2 // dst - b.gt 1b - -.if \oy - cmp w12, #0 - mov w9, w12 // restore actual remaining h - b.gt L(fguv_loop_sx1_csfl\csfl\()_\ox\()0) -.endif - - b 9f -.endm - fguv_loop_sx1 0, 0, 0 - fguv_loop_sx1 0, 0, 1 - fguv_loop_sx1 0, 1, 0 - fguv_loop_sx1 0, 1, 1 - fguv_loop_sx1 1, 0, 0 - fguv_loop_sx1 1, 0, 1 - fguv_loop_sx1 1, 1, 0 - fguv_loop_sx1 1, 1, 1 - -9: - ldp d14, d15, [sp, #64] - ldp d12, d13, [sp, #48] - ldp d10, d11, [sp, #32] - ldp d8, d9, [sp, #16] - ldr x30, [sp], #80 - ret - -L(fguv_loop_sx1_tbl): - .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl0_00) - .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl0_01) - .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl0_10) - .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl0_11) - .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl1_00) - .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl1_01) - .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl1_10) - .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl1_11) -endfunc diff --git a/third_party/dav1d/src/cdf.c b/third_party/dav1d/src/cdf.c --- a/third_party/dav1d/src/cdf.c +++ b/third_party/dav1d/src/cdf.c @@ -4116,9 +4116,8 @@ if (src->ref) dav1d_ref_inc(src->ref); } void dav1d_cdf_thread_unref(CdfThreadContext *const cdf) { - if (cdf->ref) - dav1d_ref_dec(&cdf->ref); - memset(cdf, 0, sizeof(*cdf)); + memset(&cdf->data, 0, sizeof(*cdf) - offsetof(CdfThreadContext, data)); + dav1d_ref_dec(&cdf->ref); } diff --git a/third_party/dav1d/src/cpu.c b/third_party/dav1d/src/cpu.c --- a/third_party/dav1d/src/cpu.c +++ b/third_party/dav1d/src/cpu.c @@ -97,8 +97,9 @@ if (!sysctlbyname("hw.logicalcpu", &num_processors, &length, NULL, 0)) return num_processors; #elif defined(_SC_NPROCESSORS_ONLN) return (int)sysconf(_SC_NPROCESSORS_ONLN); #endif - dav1d_log(c, "Unable to detect thread count, defaulting to single-threaded mode\n"); + if (c) + dav1d_log(c, "Unable to detect thread count, defaulting to single-threaded mode\n"); return 1; } diff --git a/third_party/dav1d/src/decode.c b/third_party/dav1d/src/decode.c --- a/third_party/dav1d/src/decode.c +++ b/third_party/dav1d/src/decode.c @@ -3441,13 +3441,11 @@ if (!res) { if (f->c->n_tc > 1) { pthread_mutex_lock(&f->task_thread.ttd->lock); res = dav1d_task_create_tile_sbrow(f, 0, 1); if (!res) { - const int uses_2pass = f->c->n_fc > 1; while (!f->task_thread.done[0] || - (uses_2pass && !f->task_thread.done[1]) || f->task_thread.task_counter > 0) { pthread_cond_wait(&f->task_thread.cond, &f->task_thread.ttd->lock); } @@ -3461,10 +3459,11 @@ &f->ts[f->frame_hdr->tiling.update].cdf); } } } dav1d_decode_frame_exit(f, res); + f->n_tile_data = 0; return res; } static int get_upscale_x0(const int in_w, const int out_w, const int step) { const int err = out_w * step - (in_w << 14); diff --git a/third_party/dav1d/src/film_grain.h b/third_party/dav1d/src/film_grain.h deleted file mode 100644 --- a/third_party/dav1d/src/film_grain.h +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright © 2018-2021, VideoLAN and dav1d authors - * Copyright © 2018, Two Orioles, LLC - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef DAV1D_SRC_FILM_GRAIN_H -#define DAV1D_SRC_FILM_GRAIN_H - -#include "common/bitdepth.h" - -#include "src/levels.h" - -#define GRAIN_WIDTH 82 -#define GRAIN_HEIGHT 73 -#define BLOCK_SIZE 32 -#if !defined(BITDEPTH) || BITDEPTH == 8 -#define SCALING_SIZE 256 -typedef int8_t entry; -#else -#define SCALING_SIZE 4096 -typedef int16_t entry; -#endif - -#define decl_generate_grain_y_fn(name) \ -void (name)(entry buf[][GRAIN_WIDTH], \ - const Dav1dFilmGrainData *const data HIGHBD_DECL_SUFFIX) -typedef decl_generate_grain_y_fn(*generate_grain_y_fn); - -#define decl_generate_grain_uv_fn(name) \ -void (name)(entry buf[][GRAIN_WIDTH], \ - const entry buf_y[][GRAIN_WIDTH], \ - const Dav1dFilmGrainData *const data, const intptr_t uv HIGHBD_DECL_SUFFIX) -typedef decl_generate_grain_uv_fn(*generate_grain_uv_fn); - -#define decl_fgy_32x32xn_fn(name) \ -void (name)(pixel *dst_row, const pixel *src_row, ptrdiff_t stride, \ - const Dav1dFilmGrainData *data, \ - size_t pw, const uint8_t scaling[SCALING_SIZE], \ - const entry grain_lut[][GRAIN_WIDTH], \ - int bh, int row_num HIGHBD_DECL_SUFFIX) -typedef decl_fgy_32x32xn_fn(*fgy_32x32xn_fn); - -#define decl_fguv_32x32xn_fn(name) \ -void (name)(pixel *dst_row, const pixel *src_row, ptrdiff_t stride, \ - const Dav1dFilmGrainData *data, int pw, \ - const uint8_t scaling[SCALING_SIZE], \ - const entry grain_lut[][GRAIN_WIDTH], int bh, int row_num, \ - const pixel *luma_row, ptrdiff_t luma_stride, \ - int uv_pl, int is_id HIGHBD_DECL_SUFFIX) -typedef decl_fguv_32x32xn_fn(*fguv_32x32xn_fn); - -typedef struct Dav1dFilmGrainDSPContext { - generate_grain_y_fn generate_grain_y; - generate_grain_uv_fn generate_grain_uv[3]; - - fgy_32x32xn_fn fgy_32x32xn; - fguv_32x32xn_fn fguv_32x32xn[3]; -} Dav1dFilmGrainDSPContext; - -bitfn_decls(void dav1d_film_grain_dsp_init, Dav1dFilmGrainDSPContext *c); -bitfn_decls(void dav1d_film_grain_dsp_init_arm, Dav1dFilmGrainDSPContext *c); -bitfn_decls(void dav1d_film_grain_dsp_init_x86, Dav1dFilmGrainDSPContext *c); - -#endif /* DAV1D_SRC_FILM_GRAIN_H */ diff --git a/third_party/dav1d/src/film_grain_tmpl.c b/third_party/dav1d/src/film_grain_tmpl.c deleted file mode 100644 --- a/third_party/dav1d/src/film_grain_tmpl.c +++ /dev/null @@ -1,433 +0,0 @@ -/* - * Copyright © 2018, Niklas Haas - * Copyright © 2018, VideoLAN and dav1d authors - * Copyright © 2018, Two Orioles, LLC - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR - * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include "common/attributes.h" -#include "common/intops.h" - -#include "film_grain.h" -#include "tables.h" - -#define SUB_GRAIN_WIDTH 44 -#define SUB_GRAIN_HEIGHT 38 - -static inline int get_random_number(const int bits, unsigned *const state) { - const int r = *state; - unsigned bit = ((r >> 0) ^ (r >> 1) ^ (r >> 3) ^ (r >> 12)) & 1; - *state = (r >> 1) | (bit << 15); - - return (*state >> (16 - bits)) & ((1 << bits) - 1); -} - -static inline int round2(const int x, const uint64_t shift) { - return (x + ((1 << shift) >> 1)) >> shift; -} - -static void generate_grain_y_c(entry buf[][GRAIN_WIDTH], - const Dav1dFilmGrainData *const data - HIGHBD_DECL_SUFFIX) -{ - const int bitdepth_min_8 = bitdepth_from_max(bitdepth_max) - 8; - unsigned seed = data->seed; - const int shift = 4 - bitdepth_min_8 + data->grain_scale_shift; - const int grain_ctr = 128 << bitdepth_min_8; - const int grain_min = -grain_ctr, grain_max = grain_ctr - 1; - - for (int y = 0; y < GRAIN_HEIGHT; y++) { - for (int x = 0; x < GRAIN_WIDTH; x++) { - const int value = get_random_number(11, &seed); - buf[y][x] = round2(dav1d_gaussian_sequence[ value ], shift); - } - } - - const int ar_pad = 3; - const int ar_lag = data->ar_coeff_lag; - - for (int y = ar_pad; y < GRAIN_HEIGHT; y++) { - for (int x = ar_pad; x < GRAIN_WIDTH - ar_pad; x++) { - const int8_t *coeff = data->ar_coeffs_y; - int sum = 0; - for (int dy = -ar_lag; dy <= 0; dy++) { - for (int dx = -ar_lag; dx <= ar_lag; dx++) { - if (!dx && !dy) - break; - sum += *(coeff++) * buf[y + dy][x + dx]; - } - } - - const int grain = buf[y][x] + round2(sum, data->ar_coeff_shift); - buf[y][x] = iclip(grain, grain_min, grain_max); - } - } -} - -static NOINLINE void -generate_grain_uv_c(entry buf[][GRAIN_WIDTH], - const entry buf_y[][GRAIN_WIDTH], - const Dav1dFilmGrainData *const data, const intptr_t uv, - const int subx, const int suby HIGHBD_DECL_SUFFIX) -{ - const int bitdepth_min_8 = bitdepth_from_max(bitdepth_max) - 8; - unsigned seed = data->seed ^ (uv ? 0x49d8 : 0xb524); - const int shift = 4 - bitdepth_min_8 + data->grain_scale_shift; - const int grain_ctr = 128 << bitdepth_min_8; - const int grain_min = -grain_ctr, grain_max = grain_ctr - 1; - - const int chromaW = subx ? SUB_GRAIN_WIDTH : GRAIN_WIDTH; - const int chromaH = suby ? SUB_GRAIN_HEIGHT : GRAIN_HEIGHT; - - for (int y = 0; y < chromaH; y++) { - for (int x = 0; x < chromaW; x++) { - const int value = get_random_number(11, &seed); - buf[y][x] = round2(dav1d_gaussian_sequence[ value ], shift); - } - } - - const int ar_pad = 3; - const int ar_lag = data->ar_coeff_lag; - - for (int y = ar_pad; y < chromaH; y++) { - for (int x = ar_pad; x < chromaW - ar_pad; x++) { - const int8_t *coeff = data->ar_coeffs_uv[uv]; - int sum = 0; - for (int dy = -ar_lag; dy <= 0; dy++) { - for (int dx = -ar_lag; dx <= ar_lag; dx++) { - // For the final (current) pixel, we need to add in the - // contribution from the luma grain texture - if (!dx && !dy) { - if (!data->num_y_points) - break; - int luma = 0; - const int lumaX = ((x - ar_pad) << subx) + ar_pad; - const int lumaY = ((y - ar_pad) << suby) + ar_pad; - for (int i = 0; i <= suby; i++) { - for (int j = 0; j <= subx; j++) { - luma += buf_y[lumaY + i][lumaX + j]; - } - } - luma = round2(luma, subx + suby); - sum += luma * (*coeff); - break; - } - - sum += *(coeff++) * buf[y + dy][x + dx]; - } - } - - const int grain = buf[y][x] + round2(sum, data->ar_coeff_shift); - buf[y][x] = iclip(grain, grain_min, grain_max); - } - } -} - -#define gnuv_ss_fn(nm, ss_x, ss_y) \ -static decl_generate_grain_uv_fn(generate_grain_uv_##nm##_c) { \ - generate_grain_uv_c(buf, buf_y, data, uv, ss_x, ss_y HIGHBD_TAIL_SUFFIX); \ -} - -gnuv_ss_fn(420, 1, 1); -gnuv_ss_fn(422, 1, 0); -gnuv_ss_fn(444, 0, 0); - -// samples from the correct block of a grain LUT, while taking into account the -// offsets provided by the offsets cache -static inline entry sample_lut(const entry grain_lut[][GRAIN_WIDTH], - const int offsets[2][2], const int subx, const int suby, - const int bx, const int by, const int x, const int y) -{ - const int randval = offsets[bx][by]; - const int offx = 3 + (2 >> subx) * (3 + (randval >> 4)); - const int offy = 3 + (2 >> suby) * (3 + (randval & 0xF)); - return grain_lut[offy + y + (BLOCK_SIZE >> suby) * by] - [offx + x + (BLOCK_SIZE >> subx) * bx]; -} - -static void fgy_32x32xn_c(pixel *const dst_row, const pixel *const src_row, - const ptrdiff_t stride, - const Dav1dFilmGrainData *const data, const size_t pw, - const uint8_t scaling[SCALING_SIZE], - const entry grain_lut[][GRAIN_WIDTH], - const int bh, const int row_num HIGHBD_DECL_SUFFIX) -{ - const int rows = 1 + (data->overlap_flag && row_num > 0); - const int bitdepth_min_8 = bitdepth_from_max(bitdepth_max) - 8; - const int grain_ctr = 128 << bitdepth_min_8; - const int grain_min = -grain_ctr, grain_max = grain_ctr - 1; - - int min_value, max_value; - if (data->clip_to_restricted_range) { - min_value = 16 << bitdepth_min_8; - max_value = 235 << bitdepth_min_8; - } else { - min_value = 0; - max_value = BITDEPTH_MAX; - } - - // seed[0] contains the current row, seed[1] contains the previous - unsigned seed[2]; - for (int i = 0; i < rows; i++) { - seed[i] = data->seed; - seed[i] ^= (((row_num - i) * 37 + 178) & 0xFF) << 8; - seed[i] ^= (((row_num - i) * 173 + 105) & 0xFF); - } - - assert(stride % (BLOCK_SIZE * sizeof(pixel)) == 0); - - int offsets[2 /* col offset */][2 /* row offset */]; - - // process this row in BLOCK_SIZE^2 blocks - for (unsigned bx = 0; bx < pw; bx += BLOCK_SIZE) { - const int bw = imin(BLOCK_SIZE, (int) pw - bx); - - if (data->overlap_flag && bx) { - // shift previous offsets left - for (int i = 0; i < rows; i++) - offsets[1][i] = offsets[0][i]; - } - - // update current offsets - for (int i = 0; i < rows; i++) - offsets[0][i] = get_random_number(8, &seed[i]); - - // x/y block offsets to compensate for overlapped regions - const int ystart = data->overlap_flag && row_num ? imin(2, bh) : 0; - const int xstart = data->overlap_flag && bx ? imin(2, bw) : 0; - - static const int w[2][2] = { { 27, 17 }, { 17, 27 } }; - -#define add_noise_y(x, y, grain) \ - const pixel *const src = src_row + (y) * PXSTRIDE(stride) + (x) + bx; \ - pixel *const dst = dst_row + (y) * PXSTRIDE(stride) + (x) + bx; \ - const int noise = round2(scaling[ *src ] * (grain), data->scaling_shift); \ - *dst = iclip(*src + noise, min_value, max_value); - - for (int y = ystart; y < bh; y++) { - // Non-overlapped image region (straightforward) - for (int x = xstart; x < bw; x++) { - int grain = sample_lut(grain_lut, offsets, 0, 0, 0, 0, x, y); - add_noise_y(x, y, grain); - } - - // Special case for overlapped column - for (int x = 0; x < xstart; x++) { - int grain = sample_lut(grain_lut, offsets, 0, 0, 0, 0, x, y); - int old = sample_lut(grain_lut, offsets, 0, 0, 1, 0, x, y); - grain = round2(old * w[x][0] + grain * w[x][1], 5); - grain = iclip(grain, grain_min, grain_max); - add_noise_y(x, y, grain); - } - } - - for (int y = 0; y < ystart; y++) { - // Special case for overlapped row (sans corner) - for (int x = xstart; x < bw; x++) { - int grain = sample_lut(grain_lut, offsets, 0, 0, 0, 0, x, y); - int old = sample_lut(grain_lut, offsets, 0, 0, 0, 1, x, y); - grain = round2(old * w[y][0] + grain * w[y][1], 5); - grain = iclip(grain, grain_min, grain_max); - add_noise_y(x, y, grain); - } - - // Special case for doubly-overlapped corner - for (int x = 0; x < xstart; x++) { - // Blend the top pixel with the top left block - int top = sample_lut(grain_lut, offsets, 0, 0, 0, 1, x, y); - int old = sample_lut(grain_lut, offsets, 0, 0, 1, 1, x, y); - top = round2(old * w[x][0] + top * w[x][1], 5); - top = iclip(top, grain_min, grain_max); - - // Blend the current pixel with the left block - int grain = sample_lut(grain_lut, offsets, 0, 0, 0, 0, x, y); - old = sample_lut(grain_lut, offsets, 0, 0, 1, 0, x, y); - grain = round2(old * w[x][0] + grain * w[x][1], 5); - grain = iclip(grain, grain_min, grain_max); - - // Mix the row rows together and apply grain - grain = round2(top * w[y][0] + grain * w[y][1], 5); - grain = iclip(grain, grain_min, grain_max); - add_noise_y(x, y, grain); - } - } - } -} - -static NOINLINE void -fguv_32x32xn_c(pixel *const dst_row, const pixel *const src_row, - const ptrdiff_t stride, const Dav1dFilmGrainData *const data, - const int pw, const uint8_t scaling[SCALING_SIZE], - const entry grain_lut[][GRAIN_WIDTH], const int bh, - const int row_num, const pixel *const luma_row, - const ptrdiff_t luma_stride, const int uv, const int is_id, - const int sx, const int sy HIGHBD_DECL_SUFFIX) -{ - const int rows = 1 + (data->overlap_flag && row_num > 0); - const int bitdepth_min_8 = bitdepth_from_max(bitdepth_max) - 8; - const int grain_ctr = 128 << bitdepth_min_8; - const int grain_min = -grain_ctr, grain_max = grain_ctr - 1; - - int min_value, max_value; - if (data->clip_to_restricted_range) { - min_value = 16 << bitdepth_min_8; - max_value = (is_id ? 235 : 240) << bitdepth_min_8; - } else { - min_value = 0; - max_value = BITDEPTH_MAX; - } - - // seed[0] contains the current row, seed[1] contains the previous - unsigned seed[2]; - for (int i = 0; i < rows; i++) { - seed[i] = data->seed; - seed[i] ^= (((row_num - i) * 37 + 178) & 0xFF) << 8; - seed[i] ^= (((row_num - i) * 173 + 105) & 0xFF); - } - - assert(stride % (BLOCK_SIZE * sizeof(pixel)) == 0); - - int offsets[2 /* col offset */][2 /* row offset */]; - - // process this row in BLOCK_SIZE^2 blocks (subsampled) - for (int bx = 0; bx < pw; bx += BLOCK_SIZE >> sx) { - const int bw = imin(BLOCK_SIZE >> sx, pw - bx); - if (data->overlap_flag && bx) { - // shift previous offsets left - for (int i = 0; i < rows; i++) - offsets[1][i] = offsets[0][i]; - } - - // update current offsets - for (int i = 0; i < rows; i++) - offsets[0][i] = get_random_number(8, &seed[i]); - - // x/y block offsets to compensate for overlapped regions - const int ystart = data->overlap_flag && row_num ? imin(2 >> sy, bh) : 0; - const int xstart = data->overlap_flag && bx ? imin(2 >> sx, bw) : 0; - - static const int w[2 /* sub */][2 /* off */][2] = { - { { 27, 17 }, { 17, 27 } }, - { { 23, 22 } }, - }; - -#define add_noise_uv(x, y, grain) \ - const int lx = (bx + x) << sx; \ - const int ly = y << sy; \ - const pixel *const luma = luma_row + ly * PXSTRIDE(luma_stride) + lx; \ - pixel avg = luma[0]; \ - if (sx) \ - avg = (avg + luma[1] + 1) >> 1; \ - const pixel *const src = src_row + (y) * PXSTRIDE(stride) + (bx + (x)); \ - pixel *const dst = dst_row + (y) * PXSTRIDE(stride) + (bx + (x)); \ - int val = avg; \ - if (!data->chroma_scaling_from_luma) { \ - const int combined = avg * data->uv_luma_mult[uv] + \ - *src * data->uv_mult[uv]; \ - val = iclip_pixel( (combined >> 6) + \ - (data->uv_offset[uv] * (1 << bitdepth_min_8)) ); \ - } \ - const int noise = round2(scaling[ val ] * (grain), data->scaling_shift); \ - *dst = iclip(*src + noise, min_value, max_value); - - for (int y = ystart; y < bh; y++) { - // Non-overlapped image region (straightforward) - for (int x = xstart; x < bw; x++) { - int grain = sample_lut(grain_lut, offsets, sx, sy, 0, 0, x, y); - add_noise_uv(x, y, grain); - } - - // Special case for overlapped column - for (int x = 0; x < xstart; x++) { - int grain = sample_lut(grain_lut, offsets, sx, sy, 0, 0, x, y); - int old = sample_lut(grain_lut, offsets, sx, sy, 1, 0, x, y); - grain = round2(old * w[sx][x][0] + grain * w[sx][x][1], 5); - grain = iclip(grain, grain_min, grain_max); - add_noise_uv(x, y, grain); - } - } - - for (int y = 0; y < ystart; y++) { - // Special case for overlapped row (sans corner) - for (int x = xstart; x < bw; x++) { - int grain = sample_lut(grain_lut, offsets, sx, sy, 0, 0, x, y); - int old = sample_lut(grain_lut, offsets, sx, sy, 0, 1, x, y); - grain = round2(old * w[sy][y][0] + grain * w[sy][y][1], 5); - grain = iclip(grain, grain_min, grain_max); - add_noise_uv(x, y, grain); - } - - // Special case for doubly-overlapped corner - for (int x = 0; x < xstart; x++) { - // Blend the top pixel with the top left block - int top = sample_lut(grain_lut, offsets, sx, sy, 0, 1, x, y); - int old = sample_lut(grain_lut, offsets, sx, sy, 1, 1, x, y); - top = round2(old * w[sx][x][0] + top * w[sx][x][1], 5); - top = iclip(top, grain_min, grain_max); - - // Blend the current pixel with the left block - int grain = sample_lut(grain_lut, offsets, sx, sy, 0, 0, x, y); - old = sample_lut(grain_lut, offsets, sx, sy, 1, 0, x, y); - grain = round2(old * w[sx][x][0] + grain * w[sx][x][1], 5); - grain = iclip(grain, grain_min, grain_max); - - // Mix the row rows together and apply to image - grain = round2(top * w[sy][y][0] + grain * w[sy][y][1], 5); - grain = iclip(grain, grain_min, grain_max); - add_noise_uv(x, y, grain); - } - } - } -} - -#define fguv_ss_fn(nm, ss_x, ss_y) \ -static decl_fguv_32x32xn_fn(fguv_32x32xn_##nm##_c) { \ - fguv_32x32xn_c(dst_row, src_row, stride, data, pw, scaling, grain_lut, bh, \ - row_num, luma_row, luma_stride, uv_pl, is_id, ss_x, ss_y \ - HIGHBD_TAIL_SUFFIX); \ -} - -fguv_ss_fn(420, 1, 1); -fguv_ss_fn(422, 1, 0); -fguv_ss_fn(444, 0, 0); - -COLD void bitfn(dav1d_film_grain_dsp_init)(Dav1dFilmGrainDSPContext *const c) { - c->generate_grain_y = generate_grain_y_c; - c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I420 - 1] = generate_grain_uv_420_c; - c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I422 - 1] = generate_grain_uv_422_c; - c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I444 - 1] = generate_grain_uv_444_c; - - c->fgy_32x32xn = fgy_32x32xn_c; - c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I420 - 1] = fguv_32x32xn_420_c; - c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I422 - 1] = fguv_32x32xn_422_c; - c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I444 - 1] = fguv_32x32xn_444_c; - -#if HAVE_ASM -#if ARCH_AARCH64 || ARCH_ARM - bitfn(dav1d_film_grain_dsp_init_arm)(c); -#elif ARCH_X86 - bitfn(dav1d_film_grain_dsp_init_x86)(c); -#endif -#endif -} diff --git a/third_party/dav1d/src/lib.c b/third_party/dav1d/src/lib.c --- a/third_party/dav1d/src/lib.c +++ b/third_party/dav1d/src/lib.c @@ -95,10 +95,41 @@ return get_minstack(thread_attr) - PTHREAD_STACK_MIN; #endif return 0; } +static COLD void get_num_threads(Dav1dContext *const c, const Dav1dSettings *const s, + unsigned *n_tc, unsigned *n_fc) +{ + /* ceil(sqrt(n)) */ + static const uint8_t fc_lut[49] = { + 1, /* 1 */ + 2, 2, 2, /* 2- 4 */ + 3, 3, 3, 3, 3, /* 5- 9 */ + 4, 4, 4, 4, 4, 4, 4, /* 10-16 */ + 5, 5, 5, 5, 5, 5, 5, 5, 5, /* 17-25 */ + 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, /* 26-36 */ + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, /* 37-49 */ + }; + *n_tc = s->n_threads ? s->n_threads : + iclip(dav1d_num_logical_processors(c), 1, DAV1D_MAX_THREADS); + *n_fc = s->max_frame_delay ? umin(s->max_frame_delay, *n_tc) : + *n_tc < 50 ? fc_lut[*n_tc - 1] : 8; // min(8, ceil(sqrt(n))) +} + +COLD int dav1d_get_frame_delay(const Dav1dSettings *const s) { + unsigned n_tc, n_fc; + validate_input_or_ret(s != NULL, DAV1D_ERR(EINVAL)); + validate_input_or_ret(s->n_threads >= 0 && + s->n_threads <= DAV1D_MAX_THREADS, DAV1D_ERR(EINVAL)); + validate_input_or_ret(s->max_frame_delay >= 0 && + s->max_frame_delay <= DAV1D_MAX_FRAME_DELAY, DAV1D_ERR(EINVAL)); + + get_num_threads(NULL, s, &n_tc, &n_fc); + return n_fc; +} + COLD int dav1d_open(Dav1dContext **const c_out, const Dav1dSettings *const s) { static pthread_once_t initted = PTHREAD_ONCE_INIT; pthread_once(&initted, init_internal); validate_input_or_ret(c_out != NULL, DAV1D_ERR(EINVAL)); @@ -169,24 +200,11 @@ } c->flush = &c->flush_mem; atomic_init(c->flush, 0); - c->n_tc = s->n_threads ? s->n_threads : - iclip(dav1d_num_logical_processors(c), 1, DAV1D_MAX_THREADS); - /* ceil(sqrt(n)) */ - static const uint8_t fc_lut[49] = { - 1, /* 1 */ - 2, 2, 2, /* 2- 4 */ - 3, 3, 3, 3, 3, /* 5- 9 */ - 4, 4, 4, 4, 4, 4, 4, /* 10-16 */ - 5, 5, 5, 5, 5, 5, 5, 5, 5, /* 17-25 */ - 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, /* 26-36 */ - 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, /* 37-49 */ - }; - c->n_fc = s->max_frame_delay ? umin(s->max_frame_delay, c->n_tc) : - c->n_tc < 50 ? fc_lut[c->n_tc - 1] : 8; // min(8, ceil(sqrt(n))) + get_num_threads(c, s, &c->n_tc, &c->n_fc); c->fc = dav1d_alloc_aligned(sizeof(*c->fc) * c->n_fc, 32); if (!c->fc) goto error; memset(c->fc, 0, sizeof(*c->fc) * c->n_fc); diff --git a/third_party/dav1d/src/meson.build b/third_party/dav1d/src/meson.build --- a/third_party/dav1d/src/meson.build +++ b/third_party/dav1d/src/meson.build @@ -224,10 +224,11 @@ ) endif if dav1d_bitdepths.contains('16') libdav1d_sources_asm += files( + 'x86/filmgrain16_avx512.asm', 'x86/ipred16_avx512.asm', 'x86/looprestoration16_avx512.asm', 'x86/mc16_avx512.asm', 'x86/cdef16_avx2.asm', 'x86/filmgrain16_avx2.asm', diff --git a/third_party/dav1d/src/obu.c b/third_party/dav1d/src/obu.c --- a/third_party/dav1d/src/obu.c +++ b/third_party/dav1d/src/obu.c @@ -394,10 +394,11 @@ hdr->frame_type = seqhdr->reduced_still_picture_header ? DAV1D_FRAME_TYPE_KEY : dav1d_get_bits(gb, 2); hdr->show_frame = seqhdr->reduced_still_picture_header || dav1d_get_bits(gb, 1); if (hdr->show_frame) { if (seqhdr->decoder_model_info_present && !seqhdr->equal_picture_interval) hdr->frame_presentation_delay = dav1d_get_bits(gb, seqhdr->frame_presentation_delay_length); + hdr->showable_frame = hdr->frame_type != DAV1D_FRAME_TYPE_KEY; } else hdr->showable_frame = dav1d_get_bits(gb, 1); hdr->error_resilient_mode = (hdr->frame_type == DAV1D_FRAME_TYPE_KEY && hdr->show_frame) || hdr->frame_type == DAV1D_FRAME_TYPE_SWITCH || @@ -452,10 +453,15 @@ hdr->refresh_frame_flags = (hdr->frame_type == DAV1D_FRAME_TYPE_KEY && hdr->show_frame) ? 0xff : dav1d_get_bits(gb, 8); if (hdr->refresh_frame_flags != 0xff && hdr->error_resilient_mode && seqhdr->order_hint) for (int i = 0; i < 8; i++) dav1d_get_bits(gb, seqhdr->order_hint_n_bits); + if (c->strict_std_compliance && + hdr->frame_type == DAV1D_FRAME_TYPE_INTRA && hdr->refresh_frame_flags == 0xff) + { + goto error; + } if (read_frame_size(c, gb, 0) < 0) goto error; hdr->allow_intrabc = hdr->allow_screen_content_tools && !hdr->super_res.enabled && dav1d_get_bits(gb, 1); hdr->use_ref_frame_mvs = 0; } else { @@ -1549,10 +1555,15 @@ } if (c->seq_hdr && c->frame_hdr) { if (c->frame_hdr->show_existing_frame) { if (!c->refs[c->frame_hdr->existing_frame_idx].p.p.data[0]) goto error; + if (c->strict_std_compliance && + !c->refs[c->frame_hdr->existing_frame_idx].p.showable) + { + goto error; + } if (c->n_fc == 1) { dav1d_thread_picture_ref(&c->out, &c->refs[c->frame_hdr->existing_frame_idx].p); dav1d_data_props_copy(&c->out.p.m, &in->m); c->event_flags |= dav1d_picture_get_event_flags(&c->refs[c->frame_hdr->existing_frame_idx].p); @@ -1600,10 +1611,11 @@ dav1d_data_props_copy(&out_delayed->p.m, &in->m); pthread_mutex_unlock(&c->task_thread.lock); } if (c->refs[c->frame_hdr->existing_frame_idx].p.p.frame_hdr->frame_type == DAV1D_FRAME_TYPE_KEY) { const int r = c->frame_hdr->existing_frame_idx; + c->refs[r].p.showable = 0; for (int i = 0; i < 8; i++) { if (i == r) continue; if (c->refs[i].p.p.data[0]) dav1d_thread_picture_unref(&c->refs[i].p); diff --git a/third_party/dav1d/src/picture.h b/third_party/dav1d/src/picture.h --- a/third_party/dav1d/src/picture.h +++ b/third_party/dav1d/src/picture.h @@ -50,10 +50,14 @@ }; typedef struct Dav1dThreadPicture { Dav1dPicture p; int visible; + // This can be set for inter frames, non-key intra frames, or for invisible + // keyframes that have not yet been made visible using the show-existing-frame + // mechanism. + int showable; enum PictureFlags flags; // [0] block data (including segmentation map and motion vectors) // [1] pixel data atomic_uint *progress; } Dav1dThreadPicture; diff --git a/third_party/dav1d/src/picture.c b/third_party/dav1d/src/picture.c --- a/third_party/dav1d/src/picture.c +++ b/third_party/dav1d/src/picture.c @@ -192,14 +192,19 @@ // Must be removed from the context after being attached to the frame dav1d_ref_dec(&c->itut_t35_ref); c->itut_t35 = NULL; + // Don't clear these flags from c->frame_flags if the frame is not visible. + // This way they will be added to the next visible frame too. + const int flags_mask = (f->frame_hdr->show_frame || c->output_invisible_frames) + ? 0 : (PICTURE_FLAG_NEW_SEQUENCE | PICTURE_FLAG_NEW_OP_PARAMS_INFO); p->flags = c->frame_flags; - c->frame_flags = 0; + c->frame_flags &= flags_mask; p->visible = f->frame_hdr->show_frame; + p->showable = f->frame_hdr->showable_frame; if (have_frame_mt) { atomic_init(&p->progress[0], 0); atomic_init(&p->progress[1], 0); } return res; @@ -253,19 +258,21 @@ void dav1d_thread_picture_ref(Dav1dThreadPicture *const dst, const Dav1dThreadPicture *const src) { dav1d_picture_ref(&dst->p, &src->p); dst->visible = src->visible; + dst->showable = src->showable; dst->progress = src->progress; dst->flags = src->flags; } void dav1d_thread_picture_move_ref(Dav1dThreadPicture *const dst, Dav1dThreadPicture *const src) { dav1d_picture_move_ref(&dst->p, &src->p); dst->visible = src->visible; + dst->showable = src->showable; dst->progress = src->progress; dst->flags = src->flags; memset(src, 0, sizeof(*src)); } diff --git a/third_party/dav1d/src/ref.h b/third_party/dav1d/src/ref.h --- a/third_party/dav1d/src/ref.h +++ b/third_party/dav1d/src/ref.h @@ -48,11 +48,13 @@ Dav1dRef *dav1d_ref_create(size_t size); Dav1dRef *dav1d_ref_create_using_pool(Dav1dMemPool *pool, size_t size); Dav1dRef *dav1d_ref_wrap(const uint8_t *ptr, void (*free_callback)(const uint8_t *data, void *user_data), void *user_data); -void dav1d_ref_inc(Dav1dRef *ref); void dav1d_ref_dec(Dav1dRef **ref); - int dav1d_ref_is_writable(Dav1dRef *ref); +static inline void dav1d_ref_inc(Dav1dRef *const ref) { + atomic_fetch_add_explicit(&ref->ref_cnt, 1, memory_order_relaxed); +} + #endif /* DAV1D_SRC_REF_H */ diff --git a/third_party/dav1d/src/ref.c b/third_party/dav1d/src/ref.c --- a/third_party/dav1d/src/ref.c +++ b/third_party/dav1d/src/ref.c @@ -86,26 +86,22 @@ res->user_data = user_data; return res; } -void dav1d_ref_inc(Dav1dRef *const ref) { - atomic_fetch_add(&ref->ref_cnt, 1); -} - void dav1d_ref_dec(Dav1dRef **const pref) { assert(pref != NULL); Dav1dRef *const ref = *pref; if (!ref) return; + *pref = NULL; if (atomic_fetch_sub(&ref->ref_cnt, 1) == 1) { const int free_ref = ref->free_ref; ref->free_callback(ref->const_data, ref->user_data); if (free_ref) free(ref); } - *pref = NULL; } int dav1d_ref_is_writable(Dav1dRef *const ref) { return atomic_load(&ref->ref_cnt) == 1 && ref->data; } diff --git a/third_party/dav1d/src/x86/filmgrain16_avx2.asm b/third_party/dav1d/src/x86/filmgrain16_avx2.asm --- a/third_party/dav1d/src/x86/filmgrain16_avx2.asm +++ b/third_party/dav1d/src/x86/filmgrain16_avx2.asm @@ -27,11 +27,11 @@ %include "ext/x86/x86inc.asm" %include "x86/filmgrain_common.asm" %if ARCH_X86_64 -SECTION_RODATA 32 +SECTION_RODATA 16 pb_mask: db 0,128,128, 0,128, 0, 0,128,128, 0, 0,128, 0,128,128, 0 gen_shufA: db 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9 gen_shufB: db 4, 5, 6, 7, 6, 7, 8, 9, 8, 9, 10, 11, 10, 11, 12, 13 next_upperbit_mask: dw 0x100B, 0x2016, 0x402C, 0x8058 pw_27_17_17_27: dw 27, 17, 17, 27 @@ -875,11 +875,11 @@ %define base r11-grain_min lea r11, [grain_min] mov r6d, r9m ; bdmax mov r9d, [fg_dataq+FGData.clip_to_restricted_range] mov r7d, [fg_dataq+FGData.scaling_shift] - movifnidn sbyd, sbym + mov sbyd, sbym vpbroadcastd m8, r9m shr r6d, 11 ; is_12bpc vpbroadcastd m9, [base+grain_min+r6*4] shlx r10d, r9d, r6d vpbroadcastd m10, [base+grain_max+r6*4] @@ -888,11 +888,11 @@ vpbroadcastd m12, [base+fg_min+r10*4] vpbroadcastd m13, [base+fg_max+r9*4] test sbyd, sbyd setnz r7b vpbroadcastd m14, [base+pd_16] - test r7b, byte [fg_dataq+FGData.overlap_flag] + test r7b, [fg_dataq+FGData.overlap_flag] jnz .vertical_overlap imul seed, sbyd, (173 << 24) | 37 add seed, (105 << 24) | 178 rorx seed, seed, 24 @@ -984,11 +984,11 @@ cmovp seed, r6d ; updated seed DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ offx, offy, see, src_bak, left_offxy - lea left_offxyd, [offyd+32] ; previous column's offy*stride+offx + lea left_offxyd, [offyq+32] ; previous column's offy*stride+offx rorx offyd, seed, 8 rorx offxq, seeq, 12 and offyd, 0xf imul offyd, 164 lea offyd, [offyq+offxq*2+747] ; offy*stride+offx @@ -1057,12 +1057,10 @@ jge .end lea srcq, [src_bakq+wq*2] cmp dword r8m, 0 ; sby jne .loop_x_hv_overlap jmp .loop_x_h_overlap -.end: - RET .vertical_overlap: DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused1, \ sby, see, src_bak @@ -1192,11 +1190,11 @@ add hd, 0x80000000 jnc .loop_y_v_overlap jmp .loop_y .end_y_v_overlap: add wq, 32 - jge .end_hv + jge .end lea srcq, [src_bakq+wq*2] ; since fg_dataq.overlap is guaranteed to be set, we never jump ; back to .loop_x_v_overlap, and instead always fall-through to ; h+v overlap @@ -1218,12 +1216,12 @@ rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ offx, offy, see, src_bak, left_offxy, top_offxy, topleft_offxy - lea topleft_offxyq, [top_offxyq+32] - lea left_offxyq, [offyq+32] + lea topleft_offxyd, [top_offxyq+32] + lea left_offxyd, [offyq+32] rorx offyd, seed, 8 rorx offxd, seed, 12 and offyd, 0xf000f and offxd, 0xf000f imul offyd, 164 @@ -1269,11 +1267,11 @@ punpcklqdq xm6, xm4 movddup xm4, [pw_27_17_17_27] pmaddwd xm6, xm4 paddd xm6, xm14 psrad xm6, 5 - packssdw xm6, xm4 + packssdw xm6, xm6 pmaxsw xm6, xm9 pminsw xm6, xm10 pshuflw xm4, xm6, q1032 vpblendd m6, m7, 0xfe vpblendd m4, m5, 0xfe @@ -1291,11 +1289,10 @@ packssdw m4, m5 punpcklwd m5, m6, m7 punpckhwd m6, m7 pmaddwd m5, m15 pmaddwd m6, m15 - vpbroadcastd m7, r9m paddd m5, m14 paddd m6, m14 psrad m5, 5 psrad m6, 5 packssdw m5, m6 @@ -1335,22 +1332,22 @@ jmp .loop_y_h_overlap .end_y_hv_overlap: add wq, 32 lea srcq, [src_bakq+wq*2] jl .loop_x_hv_overlap -.end_hv: +.end: RET %macro FGUV_FN 3 ; name, ss_hor, ss_ver cglobal fguv_32x32xn_i%1_16bpc, 6, 15, 16, dst, src, stride, fg_data, w, scaling, \ grain_lut, h, sby, luma, lstride, uv_pl, is_id %define base r12-grain_min lea r12, [grain_min] mov r9d, r13m ; bdmax mov r7d, [fg_dataq+FGData.scaling_shift] mov r11d, is_idm - movifnidn sbyd, sbym + mov sbyd, sbym vpbroadcastw m11, [base+mul_bits+r7*2-12] mov r6d, [fg_dataq+FGData.clip_to_restricted_range] shr r9d, 11 ; is_12bpc vpbroadcastd m8, [base+grain_min+r9*4] shlx r10d, r6d, r9d @@ -1383,11 +1380,11 @@ %else vpbroadcastq m15, [base+pw_27_17_17_27] %endif vpbroadcastd m14, [base+pd_16] %endif - test r7b, byte [fg_dataq+FGData.overlap_flag] + test r7b, [fg_dataq+FGData.overlap_flag] jnz %%vertical_overlap imul seed, sbyd, (173 << 24) | 37 add seed, (105 << 24) | 178 rorx seed, seed, 24 @@ -1536,11 +1533,11 @@ %else dec hb %endif jg %%loop_y add wq, 32>>%2 - jge %%end + jge .end mov srcq, r9mp mov dstq, r11mp mov lumaq, r12mp lea srcq, [srcq+wq*2] lea dstq, [dstq+wq*2] @@ -1559,11 +1556,11 @@ cmovp seed, r6d ; updated seed DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ offx, offy, see, left_offxy, unused1, unused2, luma, lstride - lea left_offxyd, [offyd+(32>>%2)] ; previous column's offy*stride+offx + lea left_offxyd, [offyq+(32>>%2)] ; previous column's offy*stride+offx rorx offyd, seed, 8 rorx offxq, seeq, 12 and offyd, 0xf imul offyd, 164>>%3 lea offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx @@ -1715,22 +1712,20 @@ %else dec hb %endif jg %%loop_y_h_overlap add wq, 32>>%2 - jge %%end + jge .end mov srcq, r9mp mov dstq, r11mp mov lumaq, r12mp lea srcq, [srcq+wq*2] lea dstq, [dstq+wq*2] lea lumaq, [lumaq+wq*(2<<%2)] cmp dword r8m, 0 ; sby jne %%loop_x_hv_overlap jmp %%loop_x_h_overlap -%%end: - RET %%vertical_overlap: DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused, \ sby, see, unused1, unused2, unused3, lstride @@ -1955,11 +1950,11 @@ add r10, 4 jmp %%loop_y_v_overlap %endif %%end_y_v_overlap: add wq, 32>>%2 - jge %%end_hv + jge .end mov srcq, r9mp mov dstq, r11mp mov lumaq, r12mp lea srcq, [srcq+wq*2] lea dstq, [dstq+wq*2] @@ -2224,25 +2219,25 @@ add r14, 4 jmp %%loop_y_hv_overlap %endif %%end_y_hv_overlap: add wq, 32>>%2 - jge %%end_hv + jge .end mov srcq, r9mp mov dstq, r11mp mov lumaq, r12mp lea srcq, [srcq+wq*2] lea dstq, [dstq+wq*2] lea lumaq, [lumaq+wq*(2<<%2)] jmp %%loop_x_hv_overlap -%%end_hv: - RET %endmacro %%FGUV_32x32xN_LOOP 1, %2, %3 .csfl: %%FGUV_32x32xN_LOOP 0, %2, %3 +.end: + RET %endmacro GEN_GRAIN_UV_FN 420, 1, 1 FGUV_FN 420, 1, 1 GEN_GRAIN_UV_FN 422, 1, 0 diff --git a/third_party/dav1d/src/x86/filmgrain16_avx512.asm b/third_party/dav1d/src/x86/filmgrain16_avx512.asm new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/x86/filmgrain16_avx512.asm @@ -0,0 +1,932 @@ +; Copyright © 2022, VideoLAN and dav1d authors +; Copyright © 2022, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" +%include "x86/filmgrain_common.asm" + +%if ARCH_X86_64 + +SECTION_RODATA 64 +pb_0to63: db 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + db 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + db 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47 + db 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63 +scale_mask: db -1, -1, 0, -1, -1, -1, 4, -1, -1, -1, 8, -1, -1, -1, 12, -1 +scale_shift: dw 7, 7, 6, 6, 5, 5, 4, 4 +pw_27_17_17_27: dw 108, 68, 68, 108, 27, 17, 17, 27 +pw_23_22: dw 92, 88, 0, 128, 23, 22, 0, 32 +fg_min: times 2 dw 0 + times 2 dw 64 + times 2 dw 256 +fg_max: times 2 dw 1023 + times 2 dw 4095 + times 2 dw 960 + times 2 dw 3840 + times 2 dw 940 + times 2 dw 3760 +scale_rnd: dd 64 + dd 16 +uv_offset_mul: dd 256 + dd 1024 +pb_8_9_0_1: db 8, 9, 0, 1 + +SECTION .text + +INIT_ZMM avx512icl +cglobal fgy_32x32xn_16bpc, 6, 15, 21, dst, src, stride, fg_data, w, scaling, \ + grain_lut, offx, sby, see, offy, src_bak +%define base r11-fg_min + lea r11, [fg_min] + mov r6d, r9m ; bdmax + mov r9d, [fg_dataq+FGData.clip_to_restricted_range] + mov r7d, [fg_dataq+FGData.scaling_shift] + mov sbyd, sbym + vpbroadcastd m6, r9m + shr r6d, 11 ; is_12bpc + vbroadcasti32x4 m7, [base+scale_mask] + shlx r10d, r9d, r6d + vpbroadcastd m10, [base+scale_shift+r7*4-32] + lea r9d, [r6+r9*4] + vpbroadcastd m8, [base+fg_min+r10*4] + kxnorw k1, k1, k1 ; 0xffff + vpbroadcastd m9, [base+fg_max+r9*4] + mov r12, 0xeeeeeeeeeeeeeeee + vpbroadcastd m19, [base+scale_rnd+r6*4] + kshiftrb k2, k1, 4 ; 0xf + vpbroadcastq xm20, [base+pw_27_17_17_27+r6*8] + kmovq k3, r12 + vpbroadcastd m11, [base+scale_shift+r6*8+4] + test sbyd, sbyd + setnz r7b + vpbroadcastd m12, [base+pw_27_17_17_27+r6*8+0] + vpbroadcastd m13, [base+pw_27_17_17_27+r6*8+4] + test r7b, [fg_dataq+FGData.overlap_flag] + jnz .v_overlap + + imul seed, sbyd, (173 << 24) | 37 + add seed, (105 << 24) | 178 + rorx seed, seed, 24 + movzx seed, seew + xor seed, [fg_dataq+FGData.seed] + lea src_bakq, [srcq+wq*2] + neg wq + sub dstq, srcq + +.loop_x: + rorx r6, seeq, 1 + or seed, 0xeff4 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + rorx offyd, seed, 8 + rorx offxq, seeq, 12 + and offyd, 0xf + imul offyd, 164 + lea offyd, [offyq+offxq*2+747] ; offy*stride+offx + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, h, \ + sby, see, offxy, src_bak + + mov grain_lutq, grain_lutmp + mov hd, hm +.loop_y: + movu m4, [grain_lutq+offxyq*2+82*0] + movu m5, [grain_lutq+offxyq*2+82*2] + call .add_noise + sub hb, 2 + jg .loop_y + add wq, 32 + jge .end + lea srcq, [src_bakq+wq*2] + cmp byte [fg_dataq+FGData.overlap_flag], 0 + je .loop_x + test sbyd, sbyd + jnz .hv_overlap + + ; horizontal overlap (without vertical overlap) +.loop_x_h_overlap: + rorx r6, seeq, 1 + or seed, 0xeff4 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, offx, \ + sby, see, offy, src_bak, left_offxy + + lea left_offxyd, [offyq+73] ; previous column's offy*stride+offx + rorx offyd, seed, 8 + rorx offxq, seeq, 12 + and offyd, 0xf + imul offyd, 164 + lea offyd, [offyq+offxq*2+747] ; offy*stride+offx + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, h, \ + sby, see, offxy, src_bak, left_offxy + + mov grain_lutq, grain_lutmp + mov hd, hm +.loop_y_h_overlap: + movu m4, [grain_lutq+offxyq*2+82*0] + movu m5, [grain_lutq+offxyq*2+82*2] + movd xm17, [grain_lutq+left_offxyq*2-82*1] + pinsrd xm17, [grain_lutq+left_offxyq*2+82*1], 1 + punpckldq xm16, xm4, xm5 + punpcklwd xm17, xm16 + mova xm16, xm19 + vpdpwssd xm16, xm20, xm17 + psrad xm16, 1 + packssdw xm16, xm16 + vpsravw xm16, xm11 + vmovdqu8 m4{k2}, m16 + vpalignr m5{k2}, m16, m16, 4 + call .add_noise + sub hb, 2 + jg .loop_y_h_overlap + add wq, 32 + jge .end + lea srcq, [src_bakq+wq*2] + test sbyd, sbyd + jnz .hv_overlap + jmp .loop_x_h_overlap + +.v_overlap: + movzx sbyd, sbyb + imul seed, [fg_dataq+FGData.seed], 0x00010001 + imul r7d, sbyd, 173 * 0x00010001 + imul sbyd, 37 * 0x01000100 + add r7d, (105 << 16) | 188 + add sbyd, (178 << 24) | (141 << 8) + and r7d, 0x00ff00ff + and sbyd, 0xff00ff00 + xor seed, r7d + xor seed, sbyd ; (cur_seed << 16) | top_seed + lea src_bakq, [srcq+wq*2] + neg wq + sub dstq, srcq + + ; we assume from the block above that bits 8-15 of r7d are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp r7b ; parity of top_seed + shr seed, 16 + shl r7d, 16 + test seeb, seeh + setp r7b ; parity of cur_seed + or r6d, 0x00010001 + xor r7d, r6d + rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, offx, \ + sby, see, offy, src_bak, _, top_offxy + + rorx offyd, seed, 8 + rorx offxd, seed, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyd, [offyq+offxq*2+0x10001*747+32*82] + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, h, \ + sby, see, offxy, src_bak, _, top_offxy + + mov grain_lutq, grain_lutmp + mov hd, hm + movzx top_offxyd, offxyw + shr offxyd, 16 + + movu m16, [grain_lutq+offxyq*2+82*0] + movu m0, [grain_lutq+top_offxyq*2+82*0] + movu m17, [grain_lutq+offxyq*2+82*2] + movu m1, [grain_lutq+top_offxyq*2+82*2] + punpckhwd m4, m0, m16 + punpcklwd m0, m16 + punpckhwd m5, m1, m17 + punpcklwd m1, m17 + call .add_noise_v + sub hb, 2 + jg .loop_y + add wq, 32 + jge .end + lea srcq, [src_bakq+wq*2] + + ; since fg_dataq.overlap is guaranteed to be set, we never jump back + ; to .v_overlap, and instead always fall-through to .hv_overlap +.hv_overlap: + ; we assume from the block above that bits 8-15 of r7d are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp r7b ; parity of top_seed + shr seed, 16 + shl r7d, 16 + test seeb, seeh + setp r7b ; parity of cur_seed + or r6d, 0x00010001 + xor r7d, r6d + rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, offx, \ + sby, see, offy, src_bak, left_offxy, top_offxy, topleft_offxy + + lea topleft_offxyd, [top_offxyq+73] + lea left_offxyd, [offyq+73] + rorx offyd, seed, 8 + rorx offxd, seed, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyd, [offyq+offxq*2+0x10001*747+32*82] + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, h, \ + sby, see, offxy, src_bak, left_offxy, top_offxy, topleft_offxy + + mov grain_lutq, grain_lutmp + mov hd, hm + movzx top_offxyd, offxyw + shr offxyd, 16 + + movu m5, [grain_lutq+offxyq*2+82*0] + movu m0, [grain_lutq+top_offxyq*2+82*0] + movd xm17, [grain_lutq+left_offxyq*2-82*1] + pinsrd xm17, [grain_lutq+topleft_offxyq*2-82*1], 1 + movu m2, [grain_lutq+offxyq*2+82*2] + movu m1, [grain_lutq+top_offxyq*2+82*2] + movd xm18, [grain_lutq+left_offxyq*2+82*1] + pinsrd xm18, [grain_lutq+topleft_offxyq*2+82*1], 1 + punpckldq xm16, xm5, xm0 + punpcklwd xm17, xm16 + mova xm16, xm19 + vpdpwssd xm16, xm20, xm17 + punpckldq xm17, xm2, xm1 + punpcklwd xm18, xm17 + mova xm17, xm19 + vpdpwssd xm17, xm20, xm18 + punpckhwd m4, m0, m5 + punpcklwd m0, m5 + punpckhwd m5, m1, m2 + punpcklwd m1, m2 + psrad xm16, 1 + psrad xm17, 1 + packssdw xm16, xm17 + vpsravw xm16, xm11 + vpshuflw m0{k2}, m16, q1302 + punpckhqdq xm16, xm16 + vpshuflw m1{k2}, m16, q1302 + call .add_noise_v + sub hb, 2 + jg .loop_y_h_overlap + add wq, 32 + lea srcq, [src_bakq+wq*2] + jl .hv_overlap +.end: + RET +ALIGN function_align +.add_noise_v: + mova m2, m19 + vpdpwssd m2, m12, m4 + mova m3, m19 + vpdpwssd m3, m13, m5 + mova m4, m19 + vpdpwssd m4, m12, m0 + mova m5, m19 + vpdpwssd m5, m13, m1 + REPX {psrad x, 1}, m2, m3, m4, m5 + packssdw m4, m2 + packssdw m5, m3 + vpsravw m4, m11 + vpsravw m5, m11 +.add_noise: + mova m0, [srcq+strideq*0] + mova m1, [srcq+strideq*1] + kmovw k4, k1 + pand m16, m6, m0 + psrld m3, m0, 16 + vpgatherdd m2{k4}, [scalingq+m16] + vpcmpud k4, m3, m6, 2 ; px <= bdmax + vpgatherdd m16{k4}, [scalingq+m3] + kmovw k4, k1 + pand m17, m6, m1 + vpgatherdd m3{k4}, [scalingq+m17] + vpshufb m2{k3}, m16, m7 + psrld m16, m1, 16 + vpcmpud k4, m16, m6, 2 + vpgatherdd m17{k4}, [scalingq+m16] + vpshufb m3{k3}, m17, m7 + vpsllvw m2, m10 + vpsllvw m3, m10 + pmulhrsw m4, m2 + pmulhrsw m5, m3 + add grain_lutq, 82*4 + paddw m0, m4 + paddw m1, m5 + pmaxsw m0, m8 + pmaxsw m1, m8 + pminsw m0, m9 + pminsw m1, m9 + mova [dstq+srcq], m0 + add srcq, strideq + mova [dstq+srcq], m1 + add srcq, strideq + ret + +%macro FGUV_FN 3 ; name, ss_hor, ss_ver +cglobal fguv_32x32xn_i%1_16bpc, 6, 15, 22, dst, src, stride, fg_data, w, scaling, \ + grain_lut, h, sby, luma, lstride, uv_pl, is_id +%define base r12-fg_min + lea r12, [fg_min] + mov r9d, r13m ; bdmax + mov r7d, [fg_dataq+FGData.scaling_shift] + mov r6d, [fg_dataq+FGData.clip_to_restricted_range] + mov r11d, is_idm + kxnorw k1, k1, k1 ; 0xffff + vpbroadcastd m5, r13m + mov r13, 0xeeeeeeeeeeeeeeee + vbroadcasti32x4 m6, [base+scale_mask] + shr r9d, 11 ; is_12bpc + vpbroadcastd m7, [base+scale_shift+r7*4-32] + shlx r10d, r6d, r9d + mov sbyd, sbym + shlx r6d, r6d, r11d + vpbroadcastd m8, [base+fg_min+r10*4] + lea r6d, [r9+r6*2] + vpbroadcastd m9, [base+fg_max+r6*4] + kmovq k2, r13 + vpbroadcastd m20, [base+scale_rnd+r9*4] + packssdw m4, m5, m5 + vpbroadcastd m21, [base+scale_shift+r9*8+4] +%if %2 + mova m12, [base+pb_0to63] ; pw_even + mov r13d, 0x0101 + vpbroadcastq m10, [base+pw_23_22+r9*8] + kmovw k3, r13d +%if %3 + pshufd m11, m10, q0000 +%else + vpbroadcastd ym16, [base+pw_27_17_17_27+r9*8+0] + vpbroadcastd m11, [base+pw_27_17_17_27+r9*8+4] + vmovdqu16 m11{k1}, m16 +%endif + psrlw m13, m12, 8 ; pw_odd +%else + vpbroadcastq m10, [base+pw_27_17_17_27+r9*8] + kshiftrb k3, k1, 7 ; 0x01 + kshiftrb k4, k1, 4 ; 0x0f + pshufd m11, m10, q0000 +%endif + mov lstrideq, r10mp + test sbyd, sbyd + setnz r7b + cmp byte [fg_dataq+FGData.chroma_scaling_from_luma], 0 + jne .csfl + +%macro %%FGUV_32x32xN_LOOP 3 ; not-csfl, ss_hor, ss_ver + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + _, sby, see, lstride + +%if %1 + mov r6d, r11m + vpbroadcastd m0, [base+uv_offset_mul+r9*4] + vpbroadcastd m1, [base+pb_8_9_0_1] + vpbroadcastd m14, [fg_dataq+FGData.uv_offset+r6*4] + vbroadcasti32x4 m15, [fg_dataq+FGData.uv_mult+r6*4] + pmaddwd m14, m0 + pshufb m15, m1 ; { uv_luma_mult, uv_mult } +%endif + test r7b, [fg_dataq+FGData.overlap_flag] + jnz %%v_overlap + + imul seed, sbyd, (173 << 24) | 37 + add seed, (105 << 24) | 178 + rorx seed, seed, 24 + movzx seed, seew + xor seed, [fg_dataq+FGData.seed] + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, lstride, luma + + mov lumaq, r9mp + lea r12, [srcq+wq*2] + lea r13, [dstq+wq*2] + lea r14, [lumaq+wq*(2<<%2)] + mov r9mp, r12 + mov r10mp, r13 + mov r11mp, r14 + neg wq + +%%loop_x: + rorx r6, seeq, 1 + or seed, 0xeff4 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + rorx offyd, seed, 8 + rorx offxq, seeq, 12 + and offyd, 0xf + imul offyd, 164>>%3 + lea offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+(3+(6>>%2))] ; offy*stride+offx + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, lstride, luma + + mov grain_lutq, grain_lutmp + mov hd, hm +%%loop_y: +%if %2 + movu ym18, [grain_lutq+offxyq*2+82*0] + vinserti32x8 m18, [grain_lutq+offxyq*2+82*2], 1 + movu ym19, [grain_lutq+offxyq*2+82*4] + vinserti32x8 m19, [grain_lutq+offxyq*2+82*6], 1 +%else + movu m18, [grain_lutq+offxyq*2+82*0] + movu m19, [grain_lutq+offxyq*2+82*2] +%endif + call %%add_noise + sub hb, 2<<%2 + jg %%loop_y + add wq, 32>>%2 + jge .end + mov srcq, r9mp + mov dstq, r10mp + mov lumaq, r11mp + lea srcq, [srcq+wq*2] + lea dstq, [dstq+wq*2] + lea lumaq, [lumaq+wq*(2<<%2)] + cmp byte [fg_dataq+FGData.overlap_flag], 0 + je %%loop_x + cmp dword r8m, 0 ; sby + jne %%hv_overlap + + ; horizontal overlap (without vertical overlap) +%%loop_x_h_overlap: + rorx r6, seeq, 1 + or seed, 0xEFF4 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, lstride, luma, left_offxy + + lea left_offxyd, [offyq+(32>>%2)] ; previous column's offy*stride+offx + rorx offyd, seed, 8 + rorx offxq, seeq, 12 + and offyd, 0xf + imul offyd, 164>>%3 + lea offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, lstride, luma, left_offxy + + mov grain_lutq, grain_lutmp + mov hd, hm +%%loop_y_h_overlap: +%if %2 + movu ym18, [grain_lutq+offxyq*2+82*0] + vinserti32x8 m18, [grain_lutq+offxyq*2+82*2], 1 + movu ym19, [grain_lutq+offxyq*2+82*4] + vinserti32x8 m19, [grain_lutq+offxyq*2+82*6], 1 + movd xm16, [grain_lutq+left_offxyq*2+82*0] + vinserti32x4 m16, [grain_lutq+left_offxyq*2+82*2], 2 + movd xm17, [grain_lutq+left_offxyq*2+82*4] + vinserti32x4 m17, [grain_lutq+left_offxyq*2+82*6], 2 + punpckldq m16, m17 + punpckldq m17, m18, m19 + punpcklwd m16, m17 + mova m17, m20 + vpdpwssd m17, m16, m10 + psrad m17, 1 + packssdw m17, m17 + vpsravw m17, m21 +%else + movu m18, [grain_lutq+offxyq*2+82*0] + movu m19, [grain_lutq+offxyq*2+82*2] + movd xm16, [grain_lutq+left_offxyq*2+82*0] + pinsrd xm16, [grain_lutq+left_offxyq*2+82*2], 1 + punpckldq xm17, xm18, xm19 + punpcklwd xm16, xm17 + mova xm17, xm20 + vpdpwssd xm17, xm16, xm10 + psrad xm17, 1 + packssdw xm17, xm17 + vpsravw xm17, xm21 +%endif + vmovdqa32 m18{k3}, m17 + vpshufd m19{k3}, m17, q0321 + call %%add_noise + sub hb, 2<<%2 + jg %%loop_y_h_overlap + add wq, 32>>%2 + jge .end + mov srcq, r9mp + mov dstq, r10mp + mov lumaq, r11mp + lea srcq, [srcq+wq*2] + lea dstq, [dstq+wq*2] + lea lumaq, [lumaq+wq*(2<<%2)] + cmp dword r8m, 0 ; sby + jne %%hv_overlap + jmp %%loop_x_h_overlap + +%%v_overlap: + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + _, sby, see, lstride + + movzx sbyd, sbyb + imul seed, [fg_dataq+FGData.seed], 0x00010001 + imul r7d, sbyd, 173 * 0x00010001 + imul sbyd, 37 * 0x01000100 + add r7d, (105 << 16) | 188 + add sbyd, (178 << 24) | (141 << 8) + and r7d, 0x00ff00ff + and sbyd, 0xff00ff00 + xor seed, r7d + xor seed, sbyd ; (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, lstride, luma, _, top_offxy + + mov lumaq, r9mp + lea r12, [srcq+wq*2] + lea r13, [dstq+wq*2] + lea r14, [lumaq+wq*(2<<%2)] + mov r9mp, r12 + mov r10mp, r13 + mov r11mp, r14 + neg wq + + ; we assume from the block above that bits 8-15 of r7d are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp r7b ; parity of top_seed + shr seed, 16 + shl r7d, 16 + test seeb, seeh + setp r7b ; parity of cur_seed + or r6d, 0x00010001 + xor r7d, r6d + rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed + + rorx offyd, seed, 8 + rorx offxd, seed, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164>>%3 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyd, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, lstride, luma, _, top_offxy + + mov grain_lutq, grain_lutmp + mov hd, hm + movzx top_offxyd, offxyw + shr offxyd, 16 + +%if %3 + movu ym16, [grain_lutq+offxyq*2+82*0] + movu ym1, [grain_lutq+top_offxyq*2+82*0] + vbroadcasti32x8 m18, [grain_lutq+offxyq*2+82*2] + movu ym19, [grain_lutq+offxyq*2+82*4] + vinserti32x8 m19, [grain_lutq+offxyq*2+82*6], 1 + punpcklwd ym17, ym1, ym16 + punpckhwd ym1, ym16 +%elif %2 + movu ym18, [grain_lutq+offxyq*2+82*0] + vinserti32x8 m18, [grain_lutq+offxyq*2+82*2], 1 + movu ym17, [grain_lutq+top_offxyq*2+82*0] + vinserti32x8 m17, [grain_lutq+top_offxyq*2+82*2], 1 + movu ym19, [grain_lutq+offxyq*2+82*4] + vinserti32x8 m19, [grain_lutq+offxyq*2+82*6], 1 + punpcklwd m16, m17, m18 + punpckhwd m17, m18 +%else + movu m18, [grain_lutq+offxyq*2+82*0] + movu m19, [grain_lutq+top_offxyq*2+82*0] + movu m2, [grain_lutq+offxyq*2+82*2] + movu m16, [grain_lutq+top_offxyq*2+82*2] + punpckhwd m1, m19, m18 + punpcklwd m19, m18 + punpckhwd m18, m2, m16 + punpcklwd m2, m16 +%endif + call %%add_noise_v + sub hb, 2<<%2 + jg %%loop_y + add wq, 32>>%2 + jge .end + mov srcq, r9mp + mov dstq, r10mp + mov lumaq, r11mp + lea srcq, [srcq+wq*2] + lea dstq, [dstq+wq*2] + lea lumaq, [lumaq+wq*(2<<%2)] + + ; since fg_dataq.overlap is guaranteed to be set, we never jump back + ; to %%v_overlap, and instead always fall-through to %%hv_overlap +%%hv_overlap: + ; we assume from the block above that bits 8-15 of r7d are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp r7b ; parity of top_seed + shr seed, 16 + shl r7d, 16 + test seeb, seeh + setp r7b ; parity of cur_seed + or r6d, 0x00010001 + xor r7d, r6d + rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, lstride, luma, left_offxy, top_offxy, topleft_offxy + + lea topleft_offxyq, [top_offxyq+(32>>%2)] + lea left_offxyq, [offyq+(32>>%2)] + rorx offyd, seed, 8 + rorx offxd, seed, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164>>%3 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyd, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, lstride, luma, left_offxy, top_offxy, topleft_offxy + + mov grain_lutq, grain_lutmp + mov hd, hm + movzx top_offxyd, offxyw + shr offxyd, 16 + + ; grain = grain_lut[offy+y][offx+x] +%if %2 + movd xm16, [grain_lutq+left_offxyq*2+82*0] + vinserti32x4 m16, [grain_lutq+left_offxyq*2+82*2], 2 + movd xm17, [grain_lutq+left_offxyq*2+82*4] + vinserti32x4 m17, [grain_lutq+left_offxyq*2+82*6], 2 + movu ym18, [grain_lutq+offxyq*2+82*0] + vinserti32x8 m18, [grain_lutq+offxyq*2+82*2], 1 + movu ym19, [grain_lutq+offxyq*2+82*4] + vinserti32x8 m19, [grain_lutq+offxyq*2+82*6], 1 + punpckldq m16, m17 + punpckldq m17, m18, m19 + punpcklwd m16, m17 + movu ym1, [grain_lutq+top_offxyq*2+82*0] + movd xm17, [grain_lutq+topleft_offxyq*2+82*0] + mova m0, m20 + vpdpwssd m0, m16, m10 +%if %3 + punpcklwd xm17, xm1 + mova xm16, xm20 + vpdpwssd xm16, xm17, xm10 + psrad xm16, 1 +%else + vinserti32x8 m1, [grain_lutq+top_offxyq*2+82*2], 1 + vinserti32x4 m17, [grain_lutq+topleft_offxyq*2+82*2], 2 + punpcklwd m17, m1 + mova m16, m20 + vpdpwssd m16, m17, m10 + psrad m16, 1 +%endif + psrad m0, 1 + packssdw m0, m16 + vpsravw m0, m21 + vmovdqa32 m18{k3}, m0 + vpshufd m19{k3}, m0, q0321 +%if %3 + vpunpckhdq ym1{k3}, ym0, ym0 + punpcklwd ym17, ym1, ym18 + punpckhwd ym1, ym18 +%else + vpunpckhdq m1{k3}, m0, m0 + punpcklwd m16, m1, m18 + punpckhwd m17, m1, m18 +%endif +%else + movu m18, [grain_lutq+offxyq*2+82*0] + movu m19, [grain_lutq+top_offxyq*2+82*0] + movd xm17, [grain_lutq+left_offxyq*2+82*0] + pinsrd xm17, [grain_lutq+topleft_offxyq*2+82*0], 1 + punpckldq xm16, xm18, xm19 + punpcklwd xm17, xm16 + movu m2, [grain_lutq+offxyq*2+82*2] + movu m0, [grain_lutq+top_offxyq*2+82*2] + movd xm16, [grain_lutq+left_offxyq*2+82*2] + pinsrd xm16, [grain_lutq+topleft_offxyq*2+82*2], 1 + punpckldq xm1, xm2, xm0 + punpcklwd xm1, xm16, xm1 + mova xm16, xm20 + vpdpwssd xm16, xm17, xm10 + mova xm17, xm20 + vpdpwssd xm17, xm1, xm10 + punpckhwd m1, m19, m18 + punpcklwd m19, m18 + punpckhwd m18, m2, m0 + punpcklwd m2, m0 + psrad xm16, 1 + psrad xm17, 1 + packssdw xm16, xm17 + vpsravw xm16, xm21 + vpshuflw m19{k4}, m16, q1302 + punpckhqdq xm16, xm16 + vpshuflw m2{k4}, m16, q3120 +%endif + call %%add_noise_v + sub hb, 2<<%2 + jg %%loop_y_h_overlap + add wq, 32>>%2 + jge .end + mov srcq, r9mp + mov dstq, r10mp + mov lumaq, r11mp + lea srcq, [srcq+wq*2] + lea dstq, [dstq+wq*2] + lea lumaq, [lumaq+wq*(2<<%2)] + jmp %%hv_overlap + +ALIGN function_align +%%add_noise_v: +%if %3 + mova ym16, ym20 + vpdpwssd ym16, ym17, ym11 + mova ym17, ym20 + vpdpwssd ym17, ym1, ym11 + psrad ym16, 1 + psrad ym17, 1 + packssdw ym16, ym17 + vpsravw m18{k1}, m16, m21 +%elif %2 + mova m18, m20 + vpdpwssd m18, m16, m11 + mova m16, m20 + vpdpwssd m16, m17, m11 + psrad m18, 1 + psrad m16, 1 + packssdw m18, m16 + vpsravw m18, m21 +%else + mova m16, m20 + vpdpwssd m16, m1, m11 + mova m17, m20 + vpdpwssd m17, m18, m11 + mova m18, m20 + vpdpwssd m18, m19, m11 + mova m19, m20 + vpdpwssd m19, m2, m11 + REPX {psrad x, 1}, m16, m17, m18, m19 + packssdw m18, m16 + packssdw m19, m17 + vpsravw m18, m21 + vpsravw m19, m21 +%endif +%%add_noise: +%if %2 + mova m2, [lumaq+lstrideq*(0<<%3)] + mova m0, [lumaq+lstrideq*(1<<%3)] + lea lumaq, [lumaq+lstrideq*(2<<%3)] + mova m3, [lumaq+lstrideq*(0<<%3)] + mova m1, [lumaq+lstrideq*(1<<%3)] + mova m16, m12 + vpermi2w m16, m2, m0 + vpermt2w m2, m13, m0 + mova m17, m12 + vpermi2w m17, m3, m1 + vpermt2w m3, m13, m1 + pavgw m2, m16 + pavgw m3, m17 +%elif %1 + mova m2, [lumaq+lstrideq*0] + mova m3, [lumaq+lstrideq*1] +%endif +%if %2 + mova ym16, [srcq+strideq*0] + vinserti32x8 m16, [srcq+strideq*1], 1 + lea srcq, [srcq+strideq*2] +%else + mova m16, [srcq+strideq*0] +%endif +%if %1 + punpckhwd m17, m2, m16 + mova m0, m14 + vpdpwssd m0, m17, m15 + punpcklwd m17, m2, m16 + mova m2, m14 + vpdpwssd m2, m17, m15 +%endif +%if %2 + mova ym17, [srcq+strideq*0] + vinserti32x8 m17, [srcq+strideq*1], 1 +%else + mova m17, [srcq+strideq*1] +%endif +%if %1 + psrad m0, 6 + psrad m2, 6 + packusdw m2, m0 + punpckhwd m0, m3, m17 + mova m1, m14 + vpdpwssd m1, m15, m0 + punpcklwd m0, m3, m17 + mova m3, m14 + vpdpwssd m3, m15, m0 + psrad m1, 6 + psrad m3, 6 + packusdw m3, m1 + pminuw m2, m4 + pminuw m3, m4 + +.add_noise_main: + ; scaling[luma_src] + kmovw k5, k1 + pand m1, m5, m2 + vpgatherdd m0{k5}, [scalingq+m1] + kmovw k5, k1 + psrld m2, 16 + vpgatherdd m1{k5}, [scalingq+m2] + vpshufb m0{k2}, m1, m6 + kmovw k5, k1 + psrld m1, m3, 16 + vpgatherdd m2{k5}, [scalingq+m1] + kmovw k5, k1 + pand m3, m5 + vpgatherdd m1{k5}, [scalingq+m3] + vpshufb m1{k2}, m2, m6 + + ; noise = round2(scaling[luma_src] * grain, scaling_shift) + vpsllvw m0, m7 + vpsllvw m1, m7 + pmulhrsw m18, m0 + pmulhrsw m19, m1 + add grain_lutq, 82*(4<<%2) + lea lumaq, [lumaq+lstrideq*(2<<%3)] + lea srcq, [srcq+strideq*2] + paddw m16, m18 + paddw m17, m19 + pmaxsw m16, m8 + pmaxsw m17, m8 + pminsw m16, m9 + pminsw m17, m9 +%if %2 + mova [dstq+strideq*0], ym16 + vextracti32x8 [dstq+strideq*1], m16, 1 + lea dstq, [dstq+strideq*2] + mova [dstq+strideq*0], ym17 + vextracti32x8 [dstq+strideq*1], m17, 1 +%else + mova [dstq+strideq*0], m16 + mova [dstq+strideq*1], m17 +%endif + lea dstq, [dstq+strideq*2] + ret +%else +%if %2 + pand m2, m4 + pand m3, m4 +%else + pand m2, m4, [lumaq+lstrideq*0] + pand m3, m4, [lumaq+lstrideq*1] +%endif + jmp .add_noise_main +%endif +%endmacro + + %%FGUV_32x32xN_LOOP 1, %2, %3 +.csfl: + %%FGUV_32x32xN_LOOP 0, %2, %3 +.end: + RET +%endmacro + +FGUV_FN 420, 1, 1 +FGUV_FN 422, 1, 0 +FGUV_FN 444, 0, 0 + +%endif diff --git a/third_party/dav1d/src/x86/filmgrain_avx2.asm b/third_party/dav1d/src/x86/filmgrain_avx2.asm --- a/third_party/dav1d/src/x86/filmgrain_avx2.asm +++ b/third_party/dav1d/src/x86/filmgrain_avx2.asm @@ -1486,11 +1486,11 @@ add grain_lutq, 82<<%2 sub hb, 1+%2 jg %%loop_y add wq, 32>>%2 - jge %%end + jge .end mov srcq, r11mp mov dstq, r12mp lea lumaq, [r14+wq*(1+%2)] add srcq, wq add dstq, wq @@ -1645,11 +1645,11 @@ add grain_lutq, 82*(1+%2) sub hb, 1+%2 jg %%loop_y_h_overlap add wq, 32>>%2 - jge %%end + jge .end mov srcq, r11mp mov dstq, r12mp lea lumaq, [r14+wq*(1+%2)] add srcq, wq add dstq, wq @@ -1850,11 +1850,11 @@ %endif jmp %%loop_y %%end_y_v_overlap: add wq, 32>>%2 - jge %%end + jge .end mov srcq, r11mp mov dstq, r12mp lea lumaq, [r14+wq*(1+%2)] add srcq, wq add dstq, wq @@ -2079,24 +2079,24 @@ jmp %%loop_y_h_overlap %endif %%end_y_hv_overlap: add wq, 32>>%2 - jge %%end + jge .end mov srcq, r11mp mov dstq, r12mp lea lumaq, [r14+wq*(1+%2)] add srcq, wq add dstq, wq jmp %%loop_x_hv_overlap -%%end: - RET %endmacro %%FGUV_32x32xN_LOOP 1, %2, %3 .csfl: %%FGUV_32x32xN_LOOP 0, %2, %3 +.end: + RET %endmacro GEN_GRAIN_UV_FN 420, 1, 1 FGUV_FN 420, 1, 1 GEN_GRAIN_UV_FN 422, 1, 0 diff --git a/third_party/dav1d/src/x86/filmgrain_avx512.asm b/third_party/dav1d/src/x86/filmgrain_avx512.asm --- a/third_party/dav1d/src/x86/filmgrain_avx512.asm +++ b/third_party/dav1d/src/x86/filmgrain_avx512.asm @@ -117,38 +117,13 @@ h, sby, see, overlap mov grain_lutq, grain_lutmp mov hd, hm .loop_y: - mova ym18, [srcq+strideq*0] - vinserti32x8 m18, [srcq+strideq*1], 1 movu ym21, [grain_lutq+offxyq-82] vinserti32x8 m21, [grain_lutq+offxyq+ 0], 1 - mova m19, m0 - vpmovb2m k2, m18 - punpcklbw m16, m18, m5 - punpckhbw m17, m18, m5 - vpermt2b m19, m18, m1 ; scaling[ 0..127] - vpermi2b m18, m2, m3 ; scaling[128..255] - punpcklbw m20, m5, m21 ; grain - punpckhbw m21, m5 - vmovdqu8 m19{k2}, m18 ; scaling[src] - pshufb m19, m4 - pmaddubsw m18, m19, m20 - pmaddubsw m19, m21 - add grain_lutq, 82*2 - pmulhrsw m18, m6 ; noise - pmulhrsw m19, m6 - paddw m16, m18 - paddw m17, m19 - packuswb m16, m17 - pmaxub m16, m7 - pminub m16, m8 - mova [dstq+srcq], ym16 - add srcq, strideq - vextracti32x8 [dstq+srcq], m16, 1 - add srcq, strideq + call .add_noise sub hb, 2 jg .loop_y add wq, 32 jge .end lea srcq, [src_bakq+wq] @@ -180,42 +155,19 @@ mov grain_lutq, grain_lutmp mov hd, hm .loop_y_h_overlap: movu ym20, [grain_lutq+offxyq-82] vinserti32x8 m20, [grain_lutq+offxyq+ 0], 1 - movd xm21, [grain_lutq+left_offxyq-50] - vinserti32x4 m21, [grain_lutq+left_offxyq+32], 2 - mova ym18, [srcq+strideq*0] - vinserti32x8 m18, [srcq+strideq*1], 1 - mova m19, m0 - punpcklbw m21, m20 - vpmovb2m k2, m18 - punpcklbw m16, m18, m5 - punpckhbw m17, m18, m5 - pmaddubsw m21, m10, m21 - vpermt2b m19, m18, m1 - vpermi2b m18, m2, m3 - pmulhrsw m21, m9 - vmovdqu8 m19{k2}, m18 ; scaling[src] - punpckhbw m18, m20, m5 - pshufb m19, m4 - packsswb m20{k1}, m21, m21 - punpcklbw m20, m5, m20 ; grain - pmaddubsw m18, m19, m18 - pmaddubsw m19, m20 - add grain_lutq, 82*2 - pmulhrsw m18, m6 ; noise - pmulhrsw m19, m6 - paddw m17, m18 - paddw m16, m19 - packuswb m16, m17 - pmaxub m16, m7 - pminub m16, m8 - mova [dstq+srcq], ym16 - add srcq, strideq - vextracti32x8 [dstq+srcq], m16, 1 - add srcq, strideq + movd xm19, [grain_lutq+left_offxyq-50] + vinserti32x4 m19, [grain_lutq+left_offxyq+32], 2 + punpcklbw m19, m20 + pmaddubsw m19, m10, m19 + pmulhrsw m19, m9 + punpckhbw m21, m20, m5 + packsswb m20{k1}, m19, m19 + punpcklbw m20, m5, m20 + call .add_noise_h sub hb, 2 jg .loop_y_h_overlap add wq, 32 jge .end lea srcq, [src_bakq+wq] @@ -270,47 +222,17 @@ mov grain_lutq, grain_lutmp mov hd, hm movzx top_offxyd, offxyw shr offxyd, 16 - movu ym16, [grain_lutq+offxyq-82] - vinserti32x8 m16, [grain_lutq+offxyq+ 0], 1 + movu ym19, [grain_lutq+offxyq-82] + vinserti32x8 m19, [grain_lutq+offxyq+ 0], 1 movu ym21, [grain_lutq+top_offxyq-82] vinserti32x8 m21, [grain_lutq+top_offxyq+ 0], 1 - mova ym18, [srcq+strideq*0] - vinserti32x8 m18, [srcq+strideq*1], 1 - mova m19, m0 - punpcklbw m20, m21, m16 - punpckhbw m21, m16 - vpmovb2m k2, m18 - pmaddubsw m20, m12, m20 - pmaddubsw m21, m12, m21 - punpcklbw m16, m18, m5 - punpckhbw m17, m18, m5 - vpermt2b m19, m18, m1 - vpermi2b m18, m2, m3 - pmulhrsw m20, m9 - pmulhrsw m21, m9 - vmovdqu8 m19{k2}, m18 ; scaling[src] - pshufb m19, m4 - packsswb m20, m21 - punpcklbw m18, m5, m20 ; grain - punpckhbw m20, m5 - pmaddubsw m18, m19, m18 - pmaddubsw m19, m20 - add grain_lutq, 82*2 - pmulhrsw m18, m6 ; noise - pmulhrsw m19, m6 - paddw m16, m18 - paddw m17, m19 - packuswb m16, m17 - pmaxub m16, m7 - pminub m16, m8 - mova [dstq+srcq], ym16 - add srcq, strideq - vextracti32x8 [dstq+srcq], m16, 1 - add srcq, strideq + punpckhbw m20, m21, m19 + punpcklbw m21, m19 + call .add_noise_v sub hb, 2 jg .loop_y add wq, 32 jge .end lea srcq, [src_bakq+wq] @@ -357,39 +279,51 @@ vinserti32x4 m16, [grain_lutq+left_offxyq+32], 2 movu ym21, [grain_lutq+top_offxyq-82] vinserti32x8 m21, [grain_lutq+top_offxyq+ 0], 1 movd xm17, [grain_lutq+topleft_offxyq-50] vinserti32x4 m17, [grain_lutq+topleft_offxyq+32], 2 - mova ym18, [srcq+strideq*0] - vinserti32x8 m18, [srcq+strideq*1], 1 ; do h interpolation first (so top | top/left -> top, left | cur -> cur) punpcklbw m16, m19 + pmaddubsw m16, m10, m16 punpcklbw m17, m21 - pmaddubsw m16, m10, m16 pmaddubsw m17, m10, m17 punpckhbw m20, m21, m19 - vpmovb2m k2, m18 pmulhrsw m16, m9 pmulhrsw m17, m9 packsswb m19{k1}, m16, m16 packsswb m21{k1}, m17, m17 ; followed by v interpolation (top | cur -> cur) punpcklbw m21, m19 - mova m19, m0 + call .add_noise_v + sub hb, 2 + jg .loop_y_h_overlap + add wq, 32 + lea srcq, [src_bakq+wq] + jl .hv_overlap +.end: + RET +ALIGN function_align +.add_noise_v: pmaddubsw m20, m12, m20 pmaddubsw m21, m12, m21 - punpcklbw m16, m18, m5 - punpckhbw m17, m18, m5 - vpermt2b m19, m18, m1 ; scaling[ 0..127] - vpermi2b m18, m2, m3 ; scaling[128..255] pmulhrsw m20, m9 pmulhrsw m21, m9 + packsswb m21, m20 +.add_noise: + punpcklbw m20, m5, m21 + punpckhbw m21, m5 +.add_noise_h: + mova ym18, [srcq+strideq*0] + vinserti32x8 m18, [srcq+strideq*1], 1 + mova m19, m0 + punpcklbw m16, m18, m5 + vpermt2b m19, m18, m1 ; scaling[ 0..127] + vpmovb2m k2, m18 + punpckhbw m17, m18, m5 + vpermi2b m18, m2, m3 ; scaling[128..255] vmovdqu8 m19{k2}, m18 ; scaling[src] pshufb m19, m4 - packsswb m21, m20 - punpcklbw m20, m5, m21 - punpckhbw m21, m5 pmaddubsw m18, m19, m20 pmaddubsw m19, m21 add grain_lutq, 82*2 pmulhrsw m18, m6 ; noise pmulhrsw m19, m6 @@ -400,17 +334,11 @@ pminub m16, m8 mova [dstq+srcq], ym16 add srcq, strideq vextracti32x8 [dstq+srcq], m16, 1 add srcq, strideq - sub hb, 2 - jg .loop_y_h_overlap - add wq, 32 - lea srcq, [src_bakq+wq] - jl .hv_overlap -.end: - RET + ret %macro FGUV_FN 3 ; name, ss_hor, ss_ver cglobal fguv_32x32xn_i%1_8bpc, 6, 14+%2, 22, dst, src, stride, fg_data, w, \ scaling, grain_lut, h, sby, luma, \ overlap, uv_pl, is_id, _, stride3 @@ -494,81 +422,24 @@ h, offxy, see, overlap, _, _, _, stride3 mov grain_lutq, grain_lutmp mov hd, hm %%loop_y: - mova ym18, [lumaq+lstrideq*(0<<%3)] - vinserti32x8 m18, [lumaq+lstrideq*(1<<%3)], 1 - lea lumaq, [lumaq+lstrideq*(2<<%3)] %if %2 - mova ym20, [lumaq+lstrideq*(0<<%3)] - vinserti32x8 m20, [lumaq+lstrideq*(1<<%3)], 1 - lea lumaq, [lumaq+lstrideq*(2<<%3)] - mova xm17, [srcq+strideq*0] movu xm21, [grain_lutq+offxyq+82*0] - vinserti128 ym17, [srcq+strideq*1], 1 vinserti128 ym21, [grain_lutq+offxyq+82*1], 1 - mova m19, m11 - vpermi2b m19, m18, m20 - vpermt2b m18, m12, m20 - vinserti32x4 m17, [srcq+strideq*2], 2 vinserti32x4 m21, [grain_lutq+offxyq+82*2], 2 - pavgb m18, m19 - vinserti32x4 m17, [srcq+stride3q ], 3 vinserti32x4 m21, [grain_lutq+offxyq+82*3], 3 %else - mova ym17, [srcq+strideq*0] - vinserti32x8 m17, [srcq+strideq*1], 1 movu ym21, [grain_lutq+offxyq+82*0] vinserti32x8 m21, [grain_lutq+offxyq+82*1], 1 %endif - lea srcq, [srcq+strideq*(2<<%2)] -%if %1 - punpckhbw m19, m18, m17 - punpcklbw m18, m17 ; { luma, chroma } - pmaddubsw m19, m14 - pmaddubsw m18, m14 - psraw m19, 6 - psraw m18, 6 - paddw m19, m15 - paddw m18, m15 - packuswb m18, m19 -%endif - mova m19, m0 - vpmovb2m k2, m18 - vpermt2b m19, m18, m1 ; scaling[ 0..127] - vpermi2b m18, m2, m3 ; scaling[128..255] - punpcklbw m20, m5, m21 ; grain - punpckhbw m21, m5 - vmovdqu8 m19{k2}, m18 ; scaling[src] - pshufb m19, m4 - pmaddubsw m18, m19, m20 - pmaddubsw m19, m21 - add grain_lutq, 82*2<<%2 - pmulhrsw m18, m6 ; noise - pmulhrsw m19, m6 - punpcklbw m16, m17, m5 ; chroma - punpckhbw m17, m5 - paddw m16, m18 - paddw m17, m19 - packuswb m16, m17 - pmaxub m16, m7 - pminub m16, m8 -%if %2 - mova [dstq+strideq*0], xm16 - vextracti128 [dstq+strideq*1], ym16, 1 - vextracti32x4 [dstq+strideq*2], m16, 2 - vextracti32x4 [dstq+stride3q ], m16, 3 -%else - mova [dstq+strideq*0], ym16 - vextracti32x8 [dstq+strideq*1], m16, 1 -%endif - lea dstq, [dstq+strideq*(2<<%2)] + call %%add_noise sub hb, 2<<%2 jg %%loop_y add wq, 32>>%2 - jge %%end + jge .end mov srcq, r11mp mov dstq, r12mp lea lumaq, [r13+wq*(1<<%2)] add srcq, wq add dstq, wq @@ -599,98 +470,36 @@ h, offxy, see, left_offxy, _, _, _, stride3 mov grain_lutq, grain_lutmp mov hd, hm %%loop_y_h_overlap: - ; src -%if %2 - mova ym18, [lumaq+lstrideq*(0<<%3)] - vinserti32x8 m18, [lumaq+lstrideq*(1<<%3)], 1 - lea lumaq, [lumaq+lstrideq*(2<<%3)] - mova ym20, [lumaq+lstrideq*(0<<%3)] - vinserti32x8 m20, [lumaq+lstrideq*(1<<%3)], 1 - lea lumaq, [lumaq+lstrideq*(2<<%3)] - mova xm17, [srcq+strideq*0] - vinserti128 ym17, [srcq+strideq*1], 1 - mova m19, m11 - vpermi2b m19, m18, m20 - vpermt2b m18, m12, m20 - vinserti32x4 m17, [srcq+strideq*2], 2 - pavgb m18, m19 - vinserti32x4 m17, [srcq+stride3q ], 3 -%else - mova ym18, [lumaq+lstrideq*0] - vinserti32x8 m18, [lumaq+lstrideq*1], 1 - mova ym17, [srcq+strideq*0] - vinserti32x8 m17, [srcq+strideq*1], 1 - lea lumaq, [lumaq+lstrideq*2] -%endif - lea srcq, [srcq+strideq*(2<<%2)] -%if %1 - punpckhbw m19, m18, m17 - punpcklbw m18, m17 ; { luma, chroma } - pmaddubsw m19, m14 - pmaddubsw m18, m14 - psraw m19, 6 - psraw m18, 6 - paddw m19, m15 - paddw m18, m15 - packuswb m18, m19 -%endif - mova m19, m0 - vpmovb2m k2, m18 - vpermt2b m19, m18, m1 ; scaling[ 0..127] - vpermi2b m18, m2, m3 ; scaling[128..255] - vmovdqu8 m19{k2}, m18 ; scaling[src] %if %2 movu xm20, [grain_lutq+offxyq +82*0] - movd xm18, [grain_lutq+left_offxyq+82*0] + movd xm19, [grain_lutq+left_offxyq+82*0] vinserti32x4 ym20, [grain_lutq+offxyq +82*1], 1 - vinserti32x4 ym18, [grain_lutq+left_offxyq+82*1], 1 + vinserti32x4 ym19, [grain_lutq+left_offxyq+82*1], 1 vinserti32x4 m20, [grain_lutq+offxyq +82*2], 2 - vinserti32x4 m18, [grain_lutq+left_offxyq+82*2], 2 + vinserti32x4 m19, [grain_lutq+left_offxyq+82*2], 2 vinserti32x4 m20, [grain_lutq+offxyq +82*3], 3 - vinserti32x4 m18, [grain_lutq+left_offxyq+82*3], 3 + vinserti32x4 m19, [grain_lutq+left_offxyq+82*3], 3 %else movu ym20, [grain_lutq+offxyq + 0] - movd xm18, [grain_lutq+left_offxyq+ 0] + movd xm19, [grain_lutq+left_offxyq+ 0] vinserti32x8 m20, [grain_lutq+offxyq +82], 1 - vinserti32x4 m18, [grain_lutq+left_offxyq+82], 2 + vinserti32x4 m19, [grain_lutq+left_offxyq+82], 2 %endif - punpcklbw m18, m20 - pmaddubsw m18, m10, m18 + punpcklbw m19, m20 + pmaddubsw m19, m10, m19 punpckhbw m21, m20, m5 - pshufb m19, m4 - pmulhrsw m18, m9 - vpacksswb m20{k1}, m18, m18 + pmulhrsw m19, m9 + vpacksswb m20{k1}, m19, m19 punpcklbw m20, m5, m20 - pmaddubsw m18, m19, m20 - pmaddubsw m19, m21 - add grain_lutq, 82*2<<%2 - pmulhrsw m18, m6 ; noise - pmulhrsw m19, m6 - punpcklbw m16, m17, m5 ; chroma - punpckhbw m17, m5 - paddw m16, m18 - paddw m17, m19 - packuswb m16, m17 - pmaxub m16, m7 - pminub m16, m8 -%if %2 - mova [dstq+strideq*0], xm16 - vextracti128 [dstq+strideq*1], ym16, 1 - vextracti32x4 [dstq+strideq*2], m16, 2 - vextracti32x4 [dstq+stride3q ], m16, 3 -%else - mova [dstq+strideq*0], ym16 - vextracti32x8 [dstq+strideq*1], m16, 1 -%endif - lea dstq, [dstq+strideq*(2<<%2)] + call %%add_noise_h sub hb, 2<<%2 jg %%loop_y_h_overlap add wq, 32>>%2 - jge %%end + jge .end mov srcq, r11mp mov dstq, r12mp lea lumaq, [r13+wq*(1<<%2)] add srcq, wq add dstq, wq @@ -763,119 +572,39 @@ mov grain_lutq, grain_lutmp mov hd, hm movzx top_offxyd, offxyw shr offxyd, 16 -%if %2 - mova ym18, [lumaq+lstrideq*(0<<%3)] - vinserti32x8 m18, [lumaq+lstrideq*(1<<%3)], 1 - lea lumaq, [lumaq+lstrideq*(2<<%3)] - mova ym20, [lumaq+lstrideq*(0<<%3)] - vinserti32x8 m20, [lumaq+lstrideq*(1<<%3)], 1 - lea lumaq, [lumaq+lstrideq*(2<<%3)] - mova xm17, [srcq+strideq*0] - vinserti128 ym17, [srcq+strideq*1], 1 - mova m19, m11 - vpermi2b m19, m18, m20 - vpermt2b m18, m12, m20 - vinserti32x4 m17, [srcq+strideq*2], 2 - pavgb m18, m19 - vinserti32x4 m17, [srcq+stride3q ], 3 -%else - mova ym18, [lumaq+lstrideq*0] - vinserti32x8 m18, [lumaq+lstrideq*1], 1 - mova ym17, [srcq+strideq*0] - vinserti32x8 m17, [srcq+strideq*1], 1 - lea lumaq, [lumaq+lstrideq*2] -%endif - lea srcq, [srcq+strideq*(2<<%2)] -%if %1 - punpckhbw m19, m18, m17 - punpcklbw m18, m17 ; { luma, chroma } - pmaddubsw m19, m14 - pmaddubsw m18, m14 - psraw m19, 6 - psraw m18, 6 - paddw m19, m15 - paddw m18, m15 - packuswb m18, m19 -%endif - mova m19, m0 - vpmovb2m k2, m18 - vpermt2b m19, m18, m1 ; scaling[ 0..127] - vpermi2b m18, m2, m3 ; scaling[128..255] %if %3 - movu xm21, [grain_lutq+offxyq+82*0] - movu xm16, [grain_lutq+top_offxyq+82*0] - punpcklbw xm20, xm16, xm21 - punpckhbw xm16, xm21 - pmaddubsw xm20, xm13, xm20 - pmaddubsw xm16, xm13, xm16 + movu xm18, [grain_lutq+offxyq+82*0] + movu xm20, [grain_lutq+top_offxyq+82*0] ; only interpolate first line, insert remaining line unmodified vbroadcasti128 ym21, [grain_lutq+offxyq+82*1] vinserti32x4 m21, [grain_lutq+offxyq+82*2], 2 vinserti32x4 m21, [grain_lutq+offxyq+82*3], 3 - pmulhrsw xm20, xm9 - pmulhrsw xm16, xm9 - vpacksswb m21{k3}, m20, m16 + punpcklbw xm19, xm20, xm18 + punpckhbw xm20, xm18 %elif %2 - movu xm21, [grain_lutq+offxyq+82*0] - vinserti128 ym21, [grain_lutq+offxyq+82*1], 1 - movu xm16, [grain_lutq+top_offxyq+82*0] - vinserti32x4 ym16, [grain_lutq+top_offxyq+82*1], 1 - punpcklbw ym20, ym16, ym21 - punpckhbw ym16, ym21 - pmaddubsw ym20, ym13, ym20 - pmaddubsw ym16, ym13, ym16 + movu xm18, [grain_lutq+offxyq+82*0] + vinserti128 ym18, [grain_lutq+offxyq+82*1], 1 + movu xm20, [grain_lutq+top_offxyq+82*0] + vinserti32x4 ym20, [grain_lutq+top_offxyq+82*1], 1 vbroadcasti32x4 m21, [grain_lutq+offxyq+82*2] vinserti32x4 m21, [grain_lutq+offxyq+82*3], 3 - pmulhrsw ym20, ym9 - pmulhrsw ym16, ym9 - packsswb m21{k3}, m20, m16 + punpcklbw ym19, ym20, ym18 + punpckhbw ym20, ym18 %else - movu ym16, [grain_lutq+offxyq+82*0] - vinserti32x8 m16, [grain_lutq+offxyq+82*1], 1 + movu ym21, [grain_lutq+offxyq+82*0] + vinserti32x8 m21, [grain_lutq+offxyq+82*1], 1 movu ym20, [grain_lutq+top_offxyq+82*0] vinserti32x8 m20, [grain_lutq+top_offxyq+82*1], 1 - punpcklbw m21, m20, m16 - punpckhbw m20, m16 - pmaddubsw m21, m13, m21 - pmaddubsw m20, m13, m20 - pmulhrsw m21, m9 - pmulhrsw m20, m9 - packsswb m21, m20 %endif - vmovdqu8 m19{k2}, m18 ; scaling[src] - pshufb m19, m4 - punpcklbw m20, m5, m21 - punpckhbw m21, m5 - pmaddubsw m18, m19, m20 - pmaddubsw m19, m21 - add grain_lutq, 82*2<<%2 - pmulhrsw m18, m6 ; noise - pmulhrsw m19, m6 - punpcklbw m16, m17, m5 ; chroma - punpckhbw m17, m5 - paddw m16, m18 - paddw m17, m19 - packuswb m16, m17 - pmaxub m16, m7 - pminub m16, m8 -%if %2 - mova [dstq+strideq*0], xm16 - vextracti128 [dstq+strideq*1], ym16, 1 - vextracti32x4 [dstq+strideq*2], m16, 2 - vextracti32x4 [dstq+stride3q ], m16, 3 -%else - mova [dstq+strideq*0], ym16 - vextracti32x8 [dstq+strideq*1], m16, 1 -%endif - lea dstq, [dstq+strideq*(2<<%2)] + call %%add_noise_v sub hb, 2<<%2 jg %%loop_y add wq, 32>>%2 - jge %%end + jge .end mov srcq, r11mp mov dstq, r12mp lea lumaq, [r13+wq*(1<<%2)] add srcq, wq add dstq, wq @@ -937,16 +666,20 @@ %endif punpcklqdq m16, m18 pmaddubsw m16, m10, m16 pmulhrsw m16, m9 packsswb m16, m16 + vmovdqu8 m21{k1}, m16 %if %3 vpalignr xm20{k1}, xm16, xm16, 4 + punpcklbw xm19, xm20, xm21 + punpckhbw xm20, xm21 %else vpalignr ym20{k1}, ym16, ym16, 4 + punpcklbw ym19, ym20, ym21 + punpckhbw ym20, ym21 %endif - vmovdqu8 m21{k1}, m16 %else movu ym21, [grain_lutq+offxyq+82*0] vinserti32x8 m21, [grain_lutq+offxyq+82*1], 1 movd xm16, [grain_lutq+left_offxyq+82*0] vinserti32x4 m16, [grain_lutq+left_offxyq+82*1], 2 @@ -961,82 +694,89 @@ pmulhrsw m16, m9 packsswb m16, m16 vpalignr m20{k1}, m16, m16, 4 vmovdqu8 m21{k1}, m16 %endif -%if %2 + call %%add_noise_v + sub hb, 2<<%2 + jg %%loop_y_h_overlap + add wq, 32>>%2 + jge .end + mov srcq, r11mp + mov dstq, r12mp + lea lumaq, [r13+wq*(1<<%2)] + add srcq, wq + add dstq, wq + jmp %%hv_overlap +ALIGN function_align +%%add_noise_v: +%if %3 + pmaddubsw xm19, xm13, xm19 + pmaddubsw xm20, xm13, xm20 + pmulhrsw xm19, xm9 + pmulhrsw xm20, xm9 + vpacksswb m21{k3}, m19, m20 +%elif %2 + pmaddubsw ym19, ym13, ym19 + pmaddubsw ym20, ym13, ym20 + pmulhrsw ym19, ym9 + pmulhrsw ym20, ym9 + vpacksswb m21{k3}, m19, m20 +%else + punpcklbw m19, m20, m21 + punpckhbw m20, m21 + pmaddubsw m19, m13, m19 + pmaddubsw m20, m13, m20 + pmulhrsw m19, m9 + pmulhrsw m20, m9 + packsswb m21, m19, m20 +%endif +%%add_noise: + punpcklbw m20, m5, m21 + punpckhbw m21, m5 +%%add_noise_h: mova ym18, [lumaq+lstrideq*(0<<%3)] vinserti32x8 m18, [lumaq+lstrideq*(1<<%3)], 1 +%if %2 lea lumaq, [lumaq+lstrideq*(2<<%3)] mova ym16, [lumaq+lstrideq*(0<<%3)] vinserti32x8 m16, [lumaq+lstrideq*(1<<%3)], 1 - lea lumaq, [lumaq+lstrideq*(2<<%3)] mova xm17, [srcq+strideq*0] - vinserti128 ym17, [srcq+strideq*1], 1 mova m19, m11 vpermi2b m19, m18, m16 + vinserti128 ym17, [srcq+strideq*1], 1 vpermt2b m18, m12, m16 vinserti32x4 m17, [srcq+strideq*2], 2 pavgb m18, m19 vinserti32x4 m17, [srcq+stride3q ], 3 %else - mova ym18, [lumaq+lstrideq*0] - vinserti32x8 m18, [lumaq+lstrideq*1], 1 mova ym17, [srcq+strideq*0] vinserti32x8 m17, [srcq+strideq*1], 1 - lea lumaq, [lumaq+lstrideq*2] %endif - lea srcq, [srcq+strideq*(2<<%2)] %if %1 punpckhbw m19, m18, m17 punpcklbw m18, m17 ; { luma, chroma } pmaddubsw m19, m14 pmaddubsw m18, m14 psraw m19, 6 psraw m18, 6 paddw m19, m15 paddw m18, m15 packuswb m18, m19 -%endif +.add_noise_main: mova m19, m0 + vpermt2b m19, m18, m1 ; scaling[ 0..127] vpmovb2m k2, m18 - vpermt2b m19, m18, m1 ; scaling[ 0..127] vpermi2b m18, m2, m3 ; scaling[128..255] - ; followed by v interpolation (top | cur -> cur) -%if %3 - punpcklbw xm16, xm20, xm21 - punpckhbw xm20, xm21 - pmaddubsw xm16, xm13, xm16 - pmaddubsw xm20, xm13, xm20 - pmulhrsw xm16, xm9 - pmulhrsw xm20, xm9 - vpacksswb m21{k3}, m16, m20 -%elif %2 - punpcklbw ym16, ym20, ym21 - punpckhbw ym20, ym21 - pmaddubsw ym16, ym13, ym16 - pmaddubsw ym20, ym13, ym20 - pmulhrsw ym16, ym9 - pmulhrsw ym20, ym9 - vpacksswb m21{k3}, m16, m20 -%else - punpcklbw m16, m20, m21 - punpckhbw m20, m21 - pmaddubsw m16, m13, m16 - pmaddubsw m20, m13, m20 - pmulhrsw m16, m9 - pmulhrsw m20, m9 - packsswb m21, m16, m20 -%endif vmovdqu8 m19{k2}, m18 ; scaling[src] pshufb m19, m4 - punpcklbw m20, m5, m21 - punpckhbw m21, m5 pmaddubsw m18, m19, m20 pmaddubsw m19, m21 add grain_lutq, 82*2<<%2 - pmulhrsw m18, m6 ; grain + lea lumaq, [lumaq+lstrideq*(2<<%3)] + lea srcq, [srcq+strideq*(2<<%2)] + pmulhrsw m18, m6 ; noise pmulhrsw m19, m6 punpcklbw m16, m17, m5 ; chroma punpckhbw m17, m5 paddw m16, m18 paddw m17, m19 @@ -1051,27 +791,21 @@ %else mova [dstq+strideq*0], ym16 vextracti32x8 [dstq+strideq*1], m16, 1 %endif lea dstq, [dstq+strideq*(2<<%2)] - sub hb, 2<<%2 - jg %%loop_y_h_overlap - add wq, 32>>%2 - jge %%end - mov srcq, r11mp - mov dstq, r12mp - lea lumaq, [r13+wq*(1<<%2)] - add srcq, wq - add dstq, wq - jmp %%hv_overlap -%%end: - RET + ret +%else + jmp .add_noise_main +%endif %endmacro %%FGUV_32x32xN_LOOP 1, %2, %3 .csfl: %%FGUV_32x32xN_LOOP 0, %2, %3 +.end: + RET %endmacro FGUV_FN 420, 1, 1 FGUV_FN 422, 1, 0 FGUV_FN 444, 0, 0 diff --git a/third_party/dav1d/src/x86/filmgrain_init_tmpl.c b/third_party/dav1d/src/x86/filmgrain_init_tmpl.c --- a/third_party/dav1d/src/x86/filmgrain_init_tmpl.c +++ b/third_party/dav1d/src/x86/filmgrain_init_tmpl.c @@ -69,15 +69,13 @@ c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I420 - 1] = BF(dav1d_fguv_32x32xn_i420, avx2); c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I422 - 1] = BF(dav1d_fguv_32x32xn_i422, avx2); c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I444 - 1] = BF(dav1d_fguv_32x32xn_i444, avx2); } -#if BITDEPTH == 8 if (!(flags & DAV1D_X86_CPU_FLAG_AVX512ICL)) return; c->fgy_32x32xn = BF(dav1d_fgy_32x32xn, avx512icl); c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I420 - 1] = BF(dav1d_fguv_32x32xn_i420, avx512icl); c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I422 - 1] = BF(dav1d_fguv_32x32xn_i422, avx512icl); c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I444 - 1] = BF(dav1d_fguv_32x32xn_i444, avx512icl); #endif -#endif } diff --git a/third_party/dav1d/src/x86/itx16_avx2.asm b/third_party/dav1d/src/x86/itx16_avx2.asm --- a/third_party/dav1d/src/x86/itx16_avx2.asm +++ b/third_party/dav1d/src/x86/itx16_avx2.asm @@ -683,10 +683,13 @@ jmp m(iadst_4x4_internal_12bpc).end %macro INV_TXFM_4X8_FN 2-3 10 ; type1, type2, bitdepth INV_TXFM_FN %1, %2, 0, 4x8, %3 %ifidn %1_%2, dct_dct + vpbroadcastd xm3, [pixel_%3bpc_max] +%if %3 = 10 +.dconly: imul r6d, [cq], 2896 mov [cq], eobd ; 0 mov r3d, 8 add r6d, 2048 sar r6d, 12 @@ -697,11 +700,10 @@ imul r6d, 2896 add r6d, 34816 sar r6d, 16 movd xm0, r6d vpbroadcastw xm0, xm0 - vpbroadcastd xm3, [pixel_%3bpc_max] pxor xm2, xm2 .end_loop: movq xm1, [dstq+strideq*0] movhps xm1, [dstq+strideq*1] paddw xm1, xm0 @@ -711,10 +713,13 @@ movhps [dstq+strideq*1], xm1 lea dstq, [dstq+strideq*2] sub r3d, 2 jg .end_loop WRAP_XMM RET +%else + jmp m(inv_txfm_add_dct_dct_4x8_10bpc).dconly +%endif %endif %endmacro %macro IDCT4_1D 8 ; src[1-4], tmp[1-3], rnd ITX_MULSUB_2D %2, %4, %5, %6, %7, %8, 1567, 3784 ; t2, t3 @@ -1178,16 +1183,17 @@ RET %macro INV_TXFM_4X16_FN 2-3 10 ; type1, type2, bitdepth INV_TXFM_FN %1, %2, 0, 4x16, %3 %ifidn %1_%2, dct_dct + vpbroadcastd xm3, [pixel_%3bpc_max] imul r6d, [cq], 2896 mov [cq], eobd ; 0 mov r3d, 16 add r6d, 6144 sar r6d, 13 - jmp m(inv_txfm_add_dct_dct_4x8_%3bpc).end + jmp m(inv_txfm_add_dct_dct_4x8_10bpc).end %endif %endmacro INV_TXFM_4X16_FN dct, dct INV_TXFM_4X16_FN dct, identity @@ -1793,10 +1799,13 @@ RET %macro INV_TXFM_8X4_FN 2-3 10 ; type1, type2, bitdepth INV_TXFM_FN %1, %2, 0, 8x4, %3 %ifidn %1_%2, dct_dct + vpbroadcastd m4, [pixel_%3bpc_max] +%if %3 = 10 +.dconly: imul r6d, [cq], 2896 mov [cq], eobd ; 0 add r6d, 2048 sar r6d, 12 imul r6d, 2896 @@ -1806,11 +1815,10 @@ add r6d, 34816 sar r6d, 16 movd xm0, r6d vpbroadcastw m0, xm0 .end: - vpbroadcastd m4, [pixel_%3bpc_max] pxor m3, m3 mova xm1, [dstq+strideq*0] vinserti128 m1, [dstq+strideq*1], 1 lea r6, [dstq+strideq*2] mova xm2, [r6 +strideq*0] @@ -1824,10 +1832,13 @@ mova [dstq+strideq*0], xm1 vextracti128 [dstq+strideq*1], m1, 1 mova [r6 +strideq*0], xm2 vextracti128 [r6 +strideq*1], m2, 1 RET +%else + jmp m(inv_txfm_add_dct_dct_8x4_10bpc).dconly +%endif %endif %endmacro INV_TXFM_8X4_FN dct, dct INV_TXFM_8X4_FN dct, identity @@ -2195,23 +2206,25 @@ jmp m(iidentity_8x4_internal_10bpc).pass2_end %macro INV_TXFM_8X8_FN 2-3 10 ; type1, type2, bitdepth INV_TXFM_FN %1, %2, 0, 8x8, %3 %ifidn %1_%2, dct_dct + vpbroadcastd m3, [pixel_%3bpc_max] +%if %3 = 10 +.dconly: imul r6d, [cq], 2896 mov [cq], eobd ; 0 mov r3d, 8 -.dconly: +.dconly2: add r6d, 6144 sar r6d, 13 -.dconly2: +.dconly3: imul r6d, 2896 add r6d, 34816 sar r6d, 16 movd xm0, r6d vpbroadcastw m0, xm0 - vpbroadcastd m3, [pixel_%3bpc_max] pxor m2, m2 .dconly_loop: mova xm1, [dstq+strideq*0] vinserti128 m1, [dstq+strideq*1], 1 paddw m1, m0 @@ -2221,10 +2234,13 @@ vextracti128 [dstq+strideq*1], m1, 1 lea dstq, [dstq+strideq*2] sub r3d, 2 jg .dconly_loop RET +%else + jmp m(inv_txfm_add_dct_dct_8x8_10bpc).dconly +%endif %endif %endmacro %macro IADST8_1D 14 ; src[1-8], tmp[1-3], pd_2048, clip[1-2] ITX_MULSUB_2D %8, %1, %9, %10, %11, %12, 401, 4076 ; t1a, t0a @@ -2727,17 +2743,18 @@ jmp m(iidentity_8x8_internal_10bpc).pass2_main %macro INV_TXFM_8X16_FN 2-4 0,10 ; type1, type2, eob_offset, bitdepth INV_TXFM_FN %1, %2, %3, 8x16, %4 %ifidn %1_%2, dct_dct + vpbroadcastd m3, [pixel_%4bpc_max] imul r6d, [cq], 2896 mov [cq], eobd ; 0 mov r3d, 16 add r6d, 2048 sar r6d, 12 imul r6d, 2896 - jmp m(inv_txfm_add_dct_dct_8x8_%4bpc).dconly + jmp m(inv_txfm_add_dct_dct_8x8_10bpc).dconly2 %endif %endmacro INV_TXFM_8X16_FN dct, dct INV_TXFM_8X16_FN dct, identity, 35 @@ -3371,23 +3388,25 @@ ret %macro INV_TXFM_16X4_FN 2-3 10 ; type1, type2, bitdepth INV_TXFM_FN %1, %2, 0, 16x4, %3 %ifidn %1_%2, dct_dct + vpbroadcastd m4, [pixel_%3bpc_max] +%if %3 = 10 +.dconly: imul r6d, [cq], 2896 mov [cq], eobd ; 0 mov r3d, 4 -.dconly: +.dconly2: add r6d, 6144 sar r6d, 13 -.dconly2: +.dconly3: imul r6d, 2896 add r6d, 34816 sar r6d, 16 movd xm0, r6d vpbroadcastw m0, xm0 - vpbroadcastd m4, [pixel_%3bpc_max] pxor m3, m3 .dconly_loop: paddw m1, m0, [dstq+strideq*0] paddw m2, m0, [dstq+strideq*1] pmaxsw m1, m3 @@ -3398,10 +3417,13 @@ mova [dstq+strideq*1], m2 lea dstq, [dstq+strideq*2] sub r3d, 2 jg .dconly_loop RET +%else + jmp m(inv_txfm_add_dct_dct_16x4_10bpc).dconly +%endif %endif %endmacro INV_TXFM_16X4_FN dct, dct INV_TXFM_16X4_FN dct, identity @@ -3842,17 +3864,18 @@ jmp m(idct_16x4_internal_10bpc).end2 %macro INV_TXFM_16X8_FN 2-3 10 ; type1, type2, bitdepth INV_TXFM_FN %1, %2, 0, 16x8, %3 %ifidn %1_%2, dct_dct + vpbroadcastd m4, [pixel_%3bpc_max] imul r6d, [cq], 2896 mov [cq], eobd ; 0 mov r3d, 8 add r6d, 2048 sar r6d, 12 imul r6d, 2896 - jmp m(inv_txfm_add_dct_dct_16x4_%3bpc).dconly + jmp m(inv_txfm_add_dct_dct_16x4_10bpc).dconly2 %endif %endmacro INV_TXFM_16X8_FN dct, dct INV_TXFM_16X8_FN dct, identity @@ -4338,10 +4361,14 @@ cglobal idct_16x8_internal_12bpc, 0, 7, 16, 32*8, dst, stride, c, eob, tx2 vpbroadcastd m12, [clip_20b_min] vpbroadcastd m13, [clip_20b_max] jmp m(idct_16x8_internal_10bpc).pass1 .pass2: + call .pass2_main + RET +ALIGN function_align +.pass2_main: call m(idct_8x16_internal_12bpc).transpose vpbroadcastd m12, [clip_18b_min] vpbroadcastd m13, [clip_18b_max] vpbroadcastd m11, [pd_2048] REPX {pmaxsd x, m12}, m0, m1, m2, m3, m4, m5, m6, m7 @@ -4381,12 +4408,11 @@ call m(idct_16x8_internal_10bpc).write_16x4_zero vpermq m0, m4, q3120 vpermq m1, m5, q3120 vpermq m2, m6, q3120 vpermq m3, m7, q3120 - call m(idct_16x8_internal_10bpc).write_16x4_zero - RET + jmp m(idct_16x8_internal_10bpc).write_16x4_zero ALIGN function_align .write_16x4_start: vpbroadcastd m9, [pixel_12bpc_max] lea r3, [strideq*3] pxor m8, m8 @@ -4401,11 +4427,12 @@ vpbroadcastd m13, [clip_20b_min] vpbroadcastd m14, [clip_20b_max] jmp m(iadst_16x8_internal_10bpc).pass1 .pass2: call .pass2_main - jmp m(idct_16x8_internal_12bpc).end + call m(idct_16x8_internal_12bpc).end + RET ALIGN function_align .pass2_main: call m(idct_8x16_internal_12bpc).transpose vpbroadcastd m12, [clip_18b_min] vpbroadcastd m13, [clip_18b_max] @@ -4481,16 +4508,17 @@ jmp m(idct_16x8_internal_10bpc).end2 %macro INV_TXFM_16X16_FN 2-4 0,10 ; type1, type2, eob_offset, bitdepth INV_TXFM_FN %1, %2, %3, 16x16, %4 %ifidn %1_%2, dct_dct + vpbroadcastd m4, [pixel_%4bpc_max] imul r6d, [cq], 2896 mov [cq], eobd ; 0 mov r3d, 16 add r6d, 10240 sar r6d, 14 - jmp m(inv_txfm_add_dct_dct_16x4_%4bpc).dconly2 + jmp m(inv_txfm_add_dct_dct_16x4_10bpc).dconly3 %endif %endmacro INV_TXFM_16X16_FN dct, dct INV_TXFM_16X16_FN dct, identity, 28 @@ -5427,11 +5455,11 @@ mova m7, [cq+32*15] .pass2_fast: call m(idct_16x16_internal_12bpc).write_16x16 RET -%macro IDCT32_END 6 ; in/out1, out2, tmp[1-3], shift +%macro IDCT32_END 6-7 1 ; in/out1, out2, tmp[1-3], shift, pack mova m%4, [r6+32*(%1-4)] mova m%2, [r5+32*(3-%1)] mova m%5, [r4+32*(%1-4)] psubd m%3, m%1, m%4 ; idct16 out15 - n paddd m%1, m%4 ; idct16 out0 + n @@ -5444,12 +5472,14 @@ psubd m%4, m%1, m%2 ; out31 - n paddd m%1, m%2 ; out0 + n paddd m%2, m%3, m%5 ; out15 - n psubd m%3, m%5 ; out16 + n REPX {psrad x, %6}, m%1, m%3, m%2, m%4 +%if %7 & 1 packssdw m%1, m%3 ; out0 + n, out16 + n packssdw m%2, m%4 ; out15 - n, out31 - n +%endif %endmacro cglobal inv_txfm_add_dct_dct_8x32_10bpc, 4, 7, 0, dst, stride, c, eob test eobd, eobd jz .dconly @@ -5572,27 +5602,27 @@ pmulhrsw m0, m12 pmulhrsw m1, m12 call m(idct_8x8_internal_10bpc).write_8x4 RET .dconly: + vpbroadcastd m3, [pixel_10bpc_max] imul r6d, [cq], 2896 mov [cq], eobd ; 0 mov r3d, 32 add r6d, 10240 sar r6d, 14 - jmp m(inv_txfm_add_dct_dct_8x8_10bpc).dconly2 + jmp m(inv_txfm_add_dct_dct_8x8_10bpc).dconly3 ALIGN function_align -.pass1_main: +.pass1_main_part1: mova m0, [cq+128*0] mova m1, [cq+128*1] mova m2, [cq+128*2] mova m3, [cq+128*3] mova m4, [cq+128*4] mova m5, [cq+128*5] mova m6, [cq+128*6] mova m7, [cq+128*7] - add cq, 32 call m(idct_8x8_internal_10bpc).main psrld m1, m11, 10 ; pd_2 REPX {paddd x, m1}, m0, m6, m5, m3 paddd m1, m6, m7 ; out1 psubd m6, m7 ; out6 @@ -5601,10 +5631,15 @@ paddd m2, m5, m4 ; out2 psubd m5, m4 ; out5 psubd m4, m3, m8 ; out4 paddd m3, m8 ; out3 REPX {psrad x, 2 }, m0, m1, m2, m3, m4, m5, m6, m7 + ret +ALIGN function_align +.pass1_main: + call .pass1_main_part1 + add cq, 32 packssdw m0, m1 packssdw m2, m3 packssdw m4, m5 packssdw m6, m7 pshufb m0, m14 @@ -5879,12 +5914,13 @@ vperm2i128 m5, m1, m15, 0x31 vinserti128 m1, xm15, 1 ret cglobal inv_txfm_add_identity_identity_8x32_10bpc, 4, 7, 8, dst, stride, c, eob + vpbroadcastd m7, [pixel_10bpc_max] +.pass1: vpbroadcastd m5, [pw_5] - vpbroadcastd m7, [pixel_10bpc_max] pxor m6, m6 mov r6d, eobd add eobb, 21 cmovc eobd, r6d ; 43, 107, 171 -> 64, 128, 192 lea r6, [strideq*3] @@ -5945,26 +5981,260 @@ vextracti128 [dstq+r6*2 ], m2, 1 mova [dstq+r6 ], xm3 vextracti128 [dstq+r4 ], m3, 1 ret +cglobal inv_txfm_add_dct_dct_8x32_12bpc, 4, 7, 0, dst, stride, c, eob + test eobd, eobd + jz .dconly + PROLOGUE 0, 7, 16, 32*24, dst, stride, c, eob +%undef cmp + vpbroadcastd m11, [pd_2048] + vpbroadcastd m12, [clip_20b_min] + vpbroadcastd m13, [clip_20b_max] + mov r4, cq + lea r6, [rsp+32*4] + call .pass1_main + cmp eobd, 43 + jge .eob43 + jmp .pass2_fast +.eob43: + call .pass1_main + cmp eobd, 107 + jge .eob107 +.pass2_fast: + mov cq, r4 + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] + pmaxsd m0, m12, [cq+128*1+ 0] + pmaxsd m1, m12, [cq+128*7+ 0] + pmaxsd m2, m12, [cq+128*1+32] + pmaxsd m3, m12, [cq+128*7+32] + REPX {pminsd x, m13}, m0, m1, m2, m3 + vpbroadcastd m14, [pd_2896] + call m(inv_txfm_add_dct_dct_8x32_10bpc).main_oddhalf_part1_fast + pmaxsd m0, m12, [cq+128*3+ 0] + pmaxsd m1, m12, [cq+128*5+ 0] + pmaxsd m2, m12, [cq+128*3+32] + pmaxsd m3, m12, [cq+128*5+32] + REPX {pminsd x, m13}, m0, m1, m2, m3 + call m(inv_txfm_add_dct_dct_8x32_10bpc).main_oddhalf_part2_fast + pmaxsd m0, m12, [cq+128*2+ 0] + pmaxsd m1, m12, [cq+128*6+ 0] + pmaxsd m2, m12, [cq+128*2+32] + pmaxsd m3, m12, [cq+128*6+32] + REPX {pminsd x, m13}, m0, m1, m2, m3 + call m(idct_8x16_internal_10bpc).main_oddhalf_fast + pmaxsd m0, m12, [cq+128*0+ 0] + pmaxsd m1, m12, [cq+128*4+ 0] + pmaxsd m2, m12, [cq+128*0+32] + pmaxsd m3, m12, [cq+128*4+32] + REPX {pminsd x, m13}, m0, m1, m2, m3 + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + call m(idct_8x8_internal_10bpc).main + call m(idct_8x16_internal_10bpc).main_evenhalf + jmp .pass2_end +.eob107: + call .pass1_main + cmp eobd, 171 + jge .eob171 + jmp .pass2 +.eob171: + call .pass1_main +.pass2: + mov cq, r4 + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] + pmaxsd m0, m12, [cq+128*1+ 0] + pmaxsd m1, m12, [cq+128*7+ 0] + pmaxsd m2, m12, [cq+128*1+32] + pmaxsd m3, m12, [cq+128*7+32] + pmaxsd m4, m12, [cq+128*1+64] + pmaxsd m5, m12, [cq+128*7+64] + pmaxsd m6, m12, [cq+128*1+96] + pmaxsd m7, m12, [cq+128*7+96] + REPX {pminsd x, m13}, m0, m1, m2, m3, m4, m5, m6, m7 + vpbroadcastd m14, [pd_2896] + call m(inv_txfm_add_dct_dct_8x32_10bpc).main_oddhalf_part1 + pmaxsd m0, m12, [cq+128*3+ 0] + pmaxsd m1, m12, [cq+128*5+ 0] + pmaxsd m2, m12, [cq+128*3+32] + pmaxsd m3, m12, [cq+128*5+32] + pmaxsd m4, m12, [cq+128*3+64] + pmaxsd m5, m12, [cq+128*5+64] + pmaxsd m6, m12, [cq+128*3+96] + pmaxsd m7, m12, [cq+128*5+96] + REPX {pminsd x, m13}, m0, m1, m2, m3, m4, m5, m6, m7 + call m(inv_txfm_add_dct_dct_8x32_10bpc).main_oddhalf_part2 + pmaxsd m0, m12, [cq+128*2+ 0] + pmaxsd m1, m12, [cq+128*6+ 0] + pmaxsd m2, m12, [cq+128*2+32] + pmaxsd m3, m12, [cq+128*6+32] + pmaxsd m4, m12, [cq+128*2+64] + pmaxsd m5, m12, [cq+128*6+64] + pmaxsd m6, m12, [cq+128*2+96] + pmaxsd m7, m12, [cq+128*6+96] + REPX {pminsd x, m13}, m0, m1, m2, m3, m4, m5, m6, m7 + call m(idct_8x16_internal_10bpc).main_oddhalf + pmaxsd m0, m12, [cq+128*0+ 0] + pmaxsd m1, m12, [cq+128*4+ 0] + pmaxsd m2, m12, [cq+128*0+32] + pmaxsd m3, m12, [cq+128*4+32] + pmaxsd m4, m12, [cq+128*0+64] + pmaxsd m5, m12, [cq+128*4+64] + pmaxsd m6, m12, [cq+128*0+96] + pmaxsd m7, m12, [cq+128*4+96] + REPX {pminsd x, m13}, m0, m1, m2, m3, m4, m5, m6, m7 + call m(idct_8x8_internal_10bpc).main + call m(idct_8x16_internal_10bpc).main_evenhalf +.pass2_end: + psrld m11, 8 ; pd_8 + IDCT32_END 0, 15, 8, 9, 10, 4 + IDCT32_END 1, 14, 8, 9, 10, 4 + punpckhqdq m8, m0, m1 ; 16 17 (interleaved) + punpcklqdq m0, m1 ; 0 1 (interleaved) + punpcklqdq m1, m14, m15 ; 14 15 (interleaved) + punpckhqdq m14, m15 ; 30 31 (interleaved) + mova [r5+32*3], m8 + mova [r5+32*2], m14 + IDCT32_END 2, 15, 8, 9, 10, 4 + IDCT32_END 3, 14, 8, 9, 10, 4 + punpckhqdq m8, m2, m3 ; 18 19 (interleaved) + punpcklqdq m2, m3 ; 2 3 (interleaved) + punpcklqdq m3, m14, m15 ; 12 13 (interleaved) + punpckhqdq m14, m15 ; 28 29 (interleaved) + mova [r5+32*1], m8 + mova [r5+32*0], m14 + IDCT32_END 4, 15, 8, 9, 10, 4 + IDCT32_END 5, 14, 8, 9, 10, 4 + punpckhqdq m8, m4, m5 ; 20 21 (interleaved) + punpcklqdq m4, m5 ; 4 5 (interleaved) + punpcklqdq m5, m14, m15 ; 10 11 (interleaved) + punpckhqdq m14, m15 ; 26 27 (interleaved) + mova [r5-32*1], m8 + mova [r5-32*2], m14 + IDCT32_END 6, 15, 8, 9, 10, 4 + IDCT32_END 7, 14, 8, 9, 10, 4 + punpckhqdq m8, m6, m7 ; 22 23 (interleaved) + punpcklqdq m6, m7 ; 6 7 (interleaved) + punpcklqdq m7, m14, m15 ; 8 9 (interleaved) + punpckhqdq m14, m15 ; 24 25 (interleaved) + mova [r5-32*3], m8 + mova [r5-32*4], m14 + mova m15, m1 +.end: + vpermq m0, m0, q3120 + vpermq m1, m2, q3120 + call m(idct_8x8_internal_12bpc).write_8x4_start + call m(idct_8x8_internal_10bpc).write_8x4 + vpermq m0, m4, q3120 + vpermq m1, m6, q3120 + call m(idct_8x8_internal_10bpc).write_8x4 + vpermq m0, m7, q3120 + vpermq m1, m5, q3120 + call m(idct_8x8_internal_10bpc).write_8x4 + vpermq m0, m3, q3120 + vpermq m1, m15, q3120 + call m(idct_8x8_internal_10bpc).write_8x4 + vpermq m0, [r5+32*3], q3120 + vpermq m1, [r5+32*1], q3120 + call m(idct_8x8_internal_10bpc).write_8x4 + vpermq m0, [r5-32*1], q3120 + vpermq m1, [r5-32*3], q3120 + call m(idct_8x8_internal_10bpc).write_8x4 + vpermq m0, [r5-32*4], q3120 + vpermq m1, [r5-32*2], q3120 + call m(idct_8x8_internal_10bpc).write_8x4 + vpermq m0, [r5+32*0], q3120 + vpermq m1, [r5+32*2], q3120 + call m(idct_8x8_internal_10bpc).write_8x4 + RET +.dconly: + vpbroadcastd m3, [pixel_12bpc_max] + imul r6d, [cq], 2896 + mov [cq], eobd ; 0 + mov r3d, 32 + add r6d, 10240 + sar r6d, 14 + jmp m(inv_txfm_add_dct_dct_8x8_10bpc).dconly3 +ALIGN function_align +.pass1_main: + call m(inv_txfm_add_dct_dct_8x32_10bpc).pass1_main_part1 + TRANSPOSE_8X8_DWORD 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15 + mova [cq+128*0], m0 + mova [cq+128*1], m1 + mova [cq+128*2], m2 + mova [cq+128*3], m3 + mova [cq+128*4], m4 + mova [cq+128*5], m5 + mova [cq+128*6], m6 + mova [cq+128*7], m7 + add cq, 32 + ret +ALIGN function_align +.main_end: + psrld m11, 10 ; pd_2 + IDCT32_END 0, 15, 8, 9, 10, 2, 0 + mova [cq+32*16], m8 + mova [cq+32*31], m9 + IDCT32_END 1, 14, 8, 9, 10, 2, 0 + mova [cq+32*17], m8 + mova [cq+32*30], m9 + mova [cq+32*14], m14 + IDCT32_END 2, 14, 8, 9, 10, 2, 0 + mova [cq+32*18], m8 + mova [cq+32*29], m9 + mova [cq+32*13], m14 + IDCT32_END 3, 14, 8, 9, 10, 2, 0 + mova [cq+32*19], m8 + mova [cq+32*28], m9 + mova [cq+32*12], m14 + IDCT32_END 4, 14, 8, 9, 10, 2, 0 + mova [cq+32*20], m8 + mova [cq+32*27], m9 + mova [cq+32* 0], m0 + mova [cq+32* 1], m1 + mova [cq+32* 2], m2 + IDCT32_END 5, 10, 0, 1, 2, 2, 0 + mova [cq+32*21], m0 + mova [cq+32*26], m1 + IDCT32_END 6, 9, 0, 1, 2, 2, 0 + mova [cq+32*22], m0 + mova [cq+32*25], m1 + IDCT32_END 7, 8, 0, 1, 2, 2, 0 + mova [cq+32*23], m0 + mova [cq+32*24], m1 + mova m0, [cq+32* 0] + mova m1, [cq+32* 1] + mova m2, [cq+32* 2] + mova m11, m14 + mova m12, [cq+32*12] + mova m13, [cq+32*13] + mova m14, [cq+32*14] + ret + +cglobal inv_txfm_add_identity_identity_8x32_12bpc, 4, 7, 8, dst, stride, c, eob + vpbroadcastd m7, [pixel_12bpc_max] + jmp m(inv_txfm_add_identity_identity_8x32_10bpc).pass1 + cglobal inv_txfm_add_dct_dct_32x8_10bpc, 4, 7, 0, dst, stride, c, eob test eobd, eobd jnz .full imul r6d, [cq], 2896 mov [cq], eobd ; 0 mov r3d, 8 + vpbroadcastd m4, [pixel_10bpc_max] .dconly: add r6d, 10240 sar r6d, 14 .dconly2: imul r6d, 2896 add r6d, 34816 sar r6d, 16 movd xm0, r6d vpbroadcastw m0, xm0 - vpbroadcastd m4, [pixel_10bpc_max] pxor m3, m3 .dconly_loop: paddw m1, m0, [dstq+32*0] paddw m2, m0, [dstq+32*1] pmaxsw m1, m3 @@ -5977,23 +6247,53 @@ dec r3d jg .dconly_loop RET .full: PROLOGUE 0, 7, 16, 32*24, dst, stride, c, eob + lea r6, [rsp+32*4] + vpbroadcastd m12, [clip_18b_min] + vpbroadcastd m13, [clip_18b_max] + call .pass1 + call m(inv_txfm_add_dct_dct_8x32_10bpc).main_end + lea r6, [deint_shuf+128] + vpbroadcastd m11, [pw_2048] + mov r4, dstq + call .pass2 + mova m0, [r5+32*3] ; 16 17 + mova m1, [r5+32*2] ; 30 31 + mova m2, [r5+32*1] ; 18 19 + mova m3, [r5+32*0] ; 28 29 + mova m4, [r5-32*1] ; 20 21 + mova m5, [r5-32*2] ; 26 27 + mova m6, [r5-32*3] ; 22 23 + mova m7, [r5-32*4] ; 24 25 + call m(inv_txfm_add_dct_dct_8x32_10bpc).transpose + lea dstq, [r4+32] + call .pass2 + RET +ALIGN function_align +.pass2: + call m(idct_16x8_internal_8bpc).main + REPX {pmulhrsw x, m11}, m0, m1, m2, m3 + call m(idct_16x8_internal_10bpc).write_16x4_start + pmulhrsw m0, m11, m4 + pmulhrsw m1, m11, m5 + pmulhrsw m2, m11, m6 + pmulhrsw m3, m11, m7 + jmp m(idct_16x8_internal_10bpc).write_16x4_zero +ALIGN function_align +.pass1: mova m0, [cq+32* 1] mova m1, [cq+32* 7] mova m2, [cq+32* 9] mova m3, [cq+32*15] mova m4, [cq+32*17] mova m5, [cq+32*23] mova m6, [cq+32*25] mova m7, [cq+32*31] vpbroadcastd m11, [pd_2048] - vpbroadcastd m12, [clip_18b_min] - vpbroadcastd m13, [clip_18b_max] vpbroadcastd m14, [pd_2896] - lea r6, [rsp+32*4] call m(inv_txfm_add_dct_dct_8x32_10bpc).main_oddhalf_part1 mova m0, [cq+32* 3] mova m1, [cq+32* 5] mova m2, [cq+32*11] mova m3, [cq+32*13] @@ -6019,41 +6319,16 @@ mova m5, [cq+32*20] mova m6, [cq+32*24] mova m7, [cq+32*28] call m(idct_8x8_internal_10bpc).main call m(idct_8x16_internal_10bpc).main_evenhalf - call m(inv_txfm_add_dct_dct_8x32_10bpc).main_end - lea r6, [deint_shuf+128] - vpbroadcastd m11, [pw_2048] - mov r4, dstq - call .pass2 - mova m0, [r5+32*3] ; 16 17 - mova m1, [r5+32*2] ; 30 31 - mova m2, [r5+32*1] ; 18 19 - mova m3, [r5+32*0] ; 28 29 - mova m4, [r5-32*1] ; 20 21 - mova m5, [r5-32*2] ; 26 27 - mova m6, [r5-32*3] ; 22 23 - mova m7, [r5-32*4] ; 24 25 - call m(inv_txfm_add_dct_dct_8x32_10bpc).transpose - lea dstq, [r4+32] - call .pass2 - RET -ALIGN function_align -.pass2: - call m(idct_16x8_internal_8bpc).main - REPX {pmulhrsw x, m11}, m0, m1, m2, m3 - call m(idct_16x8_internal_10bpc).write_16x4_start - pmulhrsw m0, m11, m4 - pmulhrsw m1, m11, m5 - pmulhrsw m2, m11, m6 - pmulhrsw m3, m11, m7 - jmp m(idct_16x8_internal_10bpc).write_16x4_zero + ret cglobal inv_txfm_add_identity_identity_32x8_10bpc, 4, 7, 8, dst, stride, c, eob + vpbroadcastd m7, [pixel_10bpc_max] +.pass1: vpbroadcastd m5, [pw_4096] - vpbroadcastd m7, [pixel_10bpc_max] pxor m6, m6 mov r6d, eobd add eobb, 21 cmovc eobd, r6d lea r6, [strideq*3] @@ -6076,10 +6351,51 @@ add dstq, 16 sub eobd, 64 jge .loop RET +cglobal inv_txfm_add_dct_dct_32x8_12bpc, 4, 7, 0, dst, stride, c, eob + test eobd, eobd + jnz .full + imul r6d, [cq], 2896 + mov [cq], eobd ; 0 + mov r3d, 8 + vpbroadcastd m4, [pixel_12bpc_max] + jmp m(inv_txfm_add_dct_dct_32x8_10bpc).dconly +.full: + PROLOGUE 0, 7, 16, 32*24, dst, stride, c, eob + lea r6, [rsp+32*4] + vpbroadcastd m12, [clip_20b_min] + vpbroadcastd m13, [clip_20b_max] + call m(inv_txfm_add_dct_dct_32x8_10bpc).pass1 + call m(inv_txfm_add_dct_dct_8x32_12bpc).main_end + mov r4, dstq + call m(idct_16x8_internal_12bpc).pass2_main + mova m0, [cq+32* 0] ; 16 + mova m1, [cq+32* 1] ; 17 + mova m2, [cq+32* 2] ; 18 + mova m3, [cq+32* 3] ; 19 + mova m4, [cq+32* 4] ; 20 + mova m5, [cq+32* 5] ; 21 + mova m6, [cq+32* 6] ; 22 + mova m7, [cq+32* 7] ; 23 + mova m8, [cq+32* 8] ; 24 + mova m9, [cq+32* 9] ; 25 + mova m10, [cq+32*10] ; 26 + mova m11, [cq+32*11] ; 27 + mova m12, [cq+32*12] ; 28 + mova m13, [cq+32*13] ; 29 + mova m14, [cq+32*14] ; 30 + mova m15, [cq+32*15] ; 31 + lea dstq, [r4+32] + call m(idct_16x8_internal_12bpc).pass2_main + RET + +cglobal inv_txfm_add_identity_identity_32x8_12bpc, 4, 7, 8, dst, stride, c, eob + vpbroadcastd m7, [pixel_12bpc_max] + jmp m(inv_txfm_add_identity_identity_32x8_10bpc).pass1 + %macro IDCT32_PASS2_END 6 ; coefs[1-2], tmp[1-2], offset[1-2] mova m%4, [%2] paddsw m%3, m%1, m%4 psubsw m%1, m%4 %if %1 == 0 @@ -6119,17 +6435,18 @@ pxor m4, m4 REPX {mova x, m4}, m5, m6, m7 REPX {mova [r6+32*x], m4}, 0, 1, 2, 3 jmp .fast .dconly: + vpbroadcastd m4, [pixel_10bpc_max] imul r6d, [cq], 2896 mov [cq], eobd ; 0 mov r3d, 32 add r6d, 2048 sar r6d, 12 imul r6d, 2896 - jmp m(inv_txfm_add_dct_dct_16x4_10bpc).dconly + jmp m(inv_txfm_add_dct_dct_16x4_10bpc).dconly2 .eob44: mova [r4+16*0], xm0 mova [r4+16*1], xm3 mova [r4+16*2], xm1 mova [r4+16*3], xm4 @@ -6478,10 +6795,11 @@ add r6d, 2048 sar r6d, 12 imul r6d, 2896 add r6d, 6144 sar r6d, 13 + vpbroadcastd m4, [pixel_10bpc_max] jmp m(inv_txfm_add_dct_dct_32x8_10bpc).dconly2 .full: add cq, 32 mova [r4+32*3], m0 mova [r4+32*2], m1 @@ -6743,10 +7061,11 @@ jmp .pass2 .dconly: imul r6d, [cq], 2896 mov [cq], eobd ; 0 mov r3d, 32 + vpbroadcastd m4, [pixel_10bpc_max] jmp m(inv_txfm_add_dct_dct_32x8_10bpc).dconly .fast: lea r4, [rsp+32*71] pxor m0, m0 .fast_loop: @@ -7017,16 +7336,17 @@ sub eobd, 128 jl .fast call .main jmp .pass2 .dconly: + vpbroadcastd m4, [pixel_10bpc_max] imul r6d, [cq], 2896 mov [cq], eobd ; 0 mov r3d, 64 add r6d, 10240 sar r6d, 14 - jmp m(inv_txfm_add_dct_dct_16x4_10bpc).dconly2 + jmp m(inv_txfm_add_dct_dct_16x4_10bpc).dconly3 .fast: lea r4, [rsp+32*38] pxor m0, m0 .fast_loop: REPX {mova [r6+32*x], m0}, -4, -3, -2, -1, 0, 1, 2, 3 @@ -7364,10 +7684,11 @@ add r6d, 2048 sar r6d, 12 imul r6d, 2896 add r6d, 6144 sar r6d, 13 + vpbroadcastd m4, [pixel_10bpc_max] jmp m(inv_txfm_add_dct_dct_32x8_10bpc).dconly2 .fast: lea r4, [rsp+32*70] pxor m0, m0 .fast_loop: diff --git a/third_party/dav1d/src/x86/itx_init_tmpl.c b/third_party/dav1d/src/x86/itx_init_tmpl.c --- a/third_party/dav1d/src/x86/itx_init_tmpl.c +++ b/third_party/dav1d/src/x86/itx_init_tmpl.c @@ -269,13 +269,15 @@ assign_itx16_bpc_fn(R, 4, 8, 12, avx2); assign_itx16_bpc_fn(R, 4, 16, 12, avx2); assign_itx16_bpc_fn(R, 8, 4, 12, avx2); assign_itx16_bpc_fn( , 8, 8, 12, avx2); assign_itx16_bpc_fn(R, 8, 16, 12, avx2); + assign_itx2_bpc_fn (R, 8, 32, 12, avx2); assign_itx16_bpc_fn(R, 16, 4, 12, avx2); assign_itx16_bpc_fn(R, 16, 8, 12, avx2); assign_itx12_bpc_fn( , 16, 16, 12, avx2); + assign_itx2_bpc_fn (R, 32, 8, 12, avx2); #endif if (bpc > 10) return; #if BITDEPTH == 8 diff --git a/third_party/dav1d/tests/checkasm/filmgrain.c b/third_party/dav1d/tests/checkasm/filmgrain.c --- a/third_party/dav1d/tests/checkasm/filmgrain.c +++ b/third_party/dav1d/tests/checkasm/filmgrain.c @@ -81,13 +81,13 @@ report("gen_grain_y"); } static void check_gen_grnuv(const Dav1dFilmGrainDSPContext *const dsp) { - entry grain_lut_y[GRAIN_HEIGHT + 1][GRAIN_WIDTH]; - entry grain_lut_c[GRAIN_HEIGHT][GRAIN_WIDTH]; - entry grain_lut_a[GRAIN_HEIGHT + 1][GRAIN_WIDTH]; + ALIGN_STK_16(entry, grain_lut_y, GRAIN_HEIGHT + 1,[GRAIN_WIDTH]); + ALIGN_STK_16(entry, grain_lut_c, GRAIN_HEIGHT, [GRAIN_WIDTH]); + ALIGN_STK_16(entry, grain_lut_a, GRAIN_HEIGHT + 1,[GRAIN_WIDTH]); declare_func(void, entry grain_lut[][GRAIN_WIDTH], const entry grain_lut_y[][GRAIN_WIDTH], const Dav1dFilmGrainData *data, intptr_t uv HIGHBD_DECL_SUFFIX); @@ -153,20 +153,20 @@ const entry grain_lut[][GRAIN_WIDTH], int bh, int row_num HIGHBD_DECL_SUFFIX); if (check_func(dsp->fgy_32x32xn, "fgy_32x32xn_%dbpc", BITDEPTH)) { ALIGN_STK_16(Dav1dFilmGrainData, fg_data, 16,); + ALIGN_STK_16(entry, grain_lut, GRAIN_HEIGHT + 1,[GRAIN_WIDTH]); ALIGN_STK_64(uint8_t, scaling, SCALING_SIZE,); fg_data[0].seed = rnd() & 0xFFFF; #if BITDEPTH == 16 const int bitdepth_max = rnd() & 1 ? 0x3ff : 0xfff; #else const int bitdepth_max = 0xff; #endif - entry grain_lut[GRAIN_HEIGHT + 1][GRAIN_WIDTH]; fg_data[0].grain_scale_shift = rnd() & 3; fg_data[0].ar_coeff_shift = (rnd() & 3) + 6; fg_data[0].ar_coeff_lag = rnd() & 3; const int num_y_pos = 2 * fg_data[0].ar_coeff_lag * (fg_data[0].ar_coeff_lag + 1); for (int n = 0; n < num_y_pos; n++) @@ -265,10 +265,11 @@ if (check_func(dsp->fguv_32x32xn[layout_idx], "fguv_32x32xn_%dbpc_%s_csfl%d", BITDEPTH, ss_name[layout_idx], csfl)) { ALIGN_STK_16(Dav1dFilmGrainData, fg_data, 1,); + ALIGN_STK_16(entry, grain_lut, 2,[GRAIN_HEIGHT + 1][GRAIN_WIDTH]); ALIGN_STK_64(uint8_t, scaling, SCALING_SIZE,); fg_data[0].seed = rnd() & 0xFFFF; #if BITDEPTH == 16 @@ -277,11 +278,10 @@ const int bitdepth_max = 0xff; #endif const int uv_pl = rnd() & 1; const int is_identity = rnd() & 1; - entry grain_lut[2][GRAIN_HEIGHT + 1][GRAIN_WIDTH]; fg_data[0].grain_scale_shift = rnd() & 3; fg_data[0].ar_coeff_shift = (rnd() & 3) + 6; fg_data[0].ar_coeff_lag = rnd() & 3; const int num_y_pos = 2 * fg_data[0].ar_coeff_lag * (fg_data[0].ar_coeff_lag + 1); for (int n = 0; n < num_y_pos; n++)