diff --git a/media/libdav1d/asm/moz.build b/media/libdav1d/asm/moz.build --- a/media/libdav1d/asm/moz.build +++ b/media/libdav1d/asm/moz.build @@ -86,10 +86,11 @@ SOURCES += [ '../../../third_party/dav1d/src/x86/cdef16_avx2.asm', # moved from autovendored '../../../third_party/dav1d/src/x86/cdef_avx2.asm', '../../../third_party/dav1d/src/x86/cdef_avx512.asm', '../../../third_party/dav1d/src/x86/filmgrain16_avx2.asm', + '../../../third_party/dav1d/src/x86/filmgrain16_avx512.asm', '../../../third_party/dav1d/src/x86/filmgrain_avx2.asm', '../../../third_party/dav1d/src/x86/filmgrain_avx512.asm', # '../../../third_party/dav1d/src/x86/filmgrain_common.asm', # ^ filmgrain_common.asm must *not* be in SOURCES because it's only # used as an %include for other .asm files. Trying to assemble it diff --git a/media/libdav1d/moz.yaml b/media/libdav1d/moz.yaml --- a/media/libdav1d/moz.yaml +++ b/media/libdav1d/moz.yaml @@ -18,15 +18,15 @@ # Usually different from repository url url: https://code.videolan.org/videolan/dav1d # Human-readable identifier for this version/release # Generally "version NNN", "tag SSS", "bookmark SSS" - release: commit 28a9c46e1c36540d3276299f2e284ece1d2386be (2022-03-10T14:49:23.000+00:00). + release: commit 87f9a81cd770e49394a45deca7a3df41243de00b (2022-03-19T03:09:31.000+01:00). # Revision to pull in # Must be a long or short commit SHA (long preferred) - revision: 28a9c46e1c36540d3276299f2e284ece1d2386be + revision: 87f9a81cd770e49394a45deca7a3df41243de00b # The package's license, where possible using the mnemonic from # https://spdx.org/licenses/ # Multiple licenses can be specified (as a YAML list) # A "LICENSE" file must exist containing the full license text diff --git a/media/libdav1d/vcs_version.h b/media/libdav1d/vcs_version.h --- a/media/libdav1d/vcs_version.h +++ b/media/libdav1d/vcs_version.h @@ -1,2 +1,2 @@ /* auto-generated, do not edit */ -#define DAV1D_VERSION "0.9.2-170-g28a9c46" +#define DAV1D_VERSION "87f9a81cd770e49394a45deca7a3df41243de00b" diff --git a/third_party/dav1d/NEWS b/third_party/dav1d/NEWS --- a/third_party/dav1d/NEWS +++ b/third_party/dav1d/NEWS @@ -7,11 +7,12 @@ an automatic thread management. It also adds support for AVX-512 acceleration, and adds speedups to existing x86 code (from SSE2 to AVX2). -1.0.0 adds new grain API to ease acceleration on the GPU. +1.0.0 adds new grain API to ease acceleration on the GPU, and adds an API call +to get information of which frame failed to decode, in error cases. Finally, 1.0.0 fixes numerous small bugs that were reported since the beginning of the project to have a proper release. .''. diff --git a/third_party/dav1d/meson.build b/third_party/dav1d/meson.build --- a/third_party/dav1d/meson.build +++ b/third_party/dav1d/meson.build @@ -21,11 +21,11 @@ # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. project('dav1d', ['c'], - version: '0.9.2', + version: '1.0.0', default_options: ['c_std=c99', 'warning_level=2', 'buildtype=release', 'b_ndebug=if-release'], meson_version: '>= 0.49.0') diff --git a/third_party/dav1d/src/decode.c b/third_party/dav1d/src/decode.c --- a/third_party/dav1d/src/decode.c +++ b/third_party/dav1d/src/decode.c @@ -3461,10 +3461,11 @@ &f->ts[f->frame_hdr->tiling.update].cdf); } } } dav1d_decode_frame_exit(f, res); + f->n_tile_data = 0; return res; } static int get_upscale_x0(const int in_w, const int out_w, const int step) { const int err = out_w * step - (in_w << 14); diff --git a/third_party/dav1d/src/meson.build b/third_party/dav1d/src/meson.build --- a/third_party/dav1d/src/meson.build +++ b/third_party/dav1d/src/meson.build @@ -224,10 +224,11 @@ ) endif if dav1d_bitdepths.contains('16') libdav1d_sources_asm += files( + 'x86/filmgrain16_avx512.asm', 'x86/ipred16_avx512.asm', 'x86/looprestoration16_avx512.asm', 'x86/mc16_avx512.asm', 'x86/cdef16_avx2.asm', 'x86/filmgrain16_avx2.asm', diff --git a/third_party/dav1d/src/x86/filmgrain16_avx2.asm b/third_party/dav1d/src/x86/filmgrain16_avx2.asm --- a/third_party/dav1d/src/x86/filmgrain16_avx2.asm +++ b/third_party/dav1d/src/x86/filmgrain16_avx2.asm @@ -27,11 +27,11 @@ %include "ext/x86/x86inc.asm" %include "x86/filmgrain_common.asm" %if ARCH_X86_64 -SECTION_RODATA 32 +SECTION_RODATA 16 pb_mask: db 0,128,128, 0,128, 0, 0,128,128, 0, 0,128, 0,128,128, 0 gen_shufA: db 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9 gen_shufB: db 4, 5, 6, 7, 6, 7, 8, 9, 8, 9, 10, 11, 10, 11, 12, 13 next_upperbit_mask: dw 0x100B, 0x2016, 0x402C, 0x8058 pw_27_17_17_27: dw 27, 17, 17, 27 @@ -875,11 +875,11 @@ %define base r11-grain_min lea r11, [grain_min] mov r6d, r9m ; bdmax mov r9d, [fg_dataq+FGData.clip_to_restricted_range] mov r7d, [fg_dataq+FGData.scaling_shift] - movifnidn sbyd, sbym + mov sbyd, sbym vpbroadcastd m8, r9m shr r6d, 11 ; is_12bpc vpbroadcastd m9, [base+grain_min+r6*4] shlx r10d, r9d, r6d vpbroadcastd m10, [base+grain_max+r6*4] @@ -888,11 +888,11 @@ vpbroadcastd m12, [base+fg_min+r10*4] vpbroadcastd m13, [base+fg_max+r9*4] test sbyd, sbyd setnz r7b vpbroadcastd m14, [base+pd_16] - test r7b, byte [fg_dataq+FGData.overlap_flag] + test r7b, [fg_dataq+FGData.overlap_flag] jnz .vertical_overlap imul seed, sbyd, (173 << 24) | 37 add seed, (105 << 24) | 178 rorx seed, seed, 24 @@ -984,11 +984,11 @@ cmovp seed, r6d ; updated seed DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ offx, offy, see, src_bak, left_offxy - lea left_offxyd, [offyd+32] ; previous column's offy*stride+offx + lea left_offxyd, [offyq+32] ; previous column's offy*stride+offx rorx offyd, seed, 8 rorx offxq, seeq, 12 and offyd, 0xf imul offyd, 164 lea offyd, [offyq+offxq*2+747] ; offy*stride+offx @@ -1057,12 +1057,10 @@ jge .end lea srcq, [src_bakq+wq*2] cmp dword r8m, 0 ; sby jne .loop_x_hv_overlap jmp .loop_x_h_overlap -.end: - RET .vertical_overlap: DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused1, \ sby, see, src_bak @@ -1192,11 +1190,11 @@ add hd, 0x80000000 jnc .loop_y_v_overlap jmp .loop_y .end_y_v_overlap: add wq, 32 - jge .end_hv + jge .end lea srcq, [src_bakq+wq*2] ; since fg_dataq.overlap is guaranteed to be set, we never jump ; back to .loop_x_v_overlap, and instead always fall-through to ; h+v overlap @@ -1218,12 +1216,12 @@ rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ offx, offy, see, src_bak, left_offxy, top_offxy, topleft_offxy - lea topleft_offxyq, [top_offxyq+32] - lea left_offxyq, [offyq+32] + lea topleft_offxyd, [top_offxyq+32] + lea left_offxyd, [offyq+32] rorx offyd, seed, 8 rorx offxd, seed, 12 and offyd, 0xf000f and offxd, 0xf000f imul offyd, 164 @@ -1269,11 +1267,11 @@ punpcklqdq xm6, xm4 movddup xm4, [pw_27_17_17_27] pmaddwd xm6, xm4 paddd xm6, xm14 psrad xm6, 5 - packssdw xm6, xm4 + packssdw xm6, xm6 pmaxsw xm6, xm9 pminsw xm6, xm10 pshuflw xm4, xm6, q1032 vpblendd m6, m7, 0xfe vpblendd m4, m5, 0xfe @@ -1291,11 +1289,10 @@ packssdw m4, m5 punpcklwd m5, m6, m7 punpckhwd m6, m7 pmaddwd m5, m15 pmaddwd m6, m15 - vpbroadcastd m7, r9m paddd m5, m14 paddd m6, m14 psrad m5, 5 psrad m6, 5 packssdw m5, m6 @@ -1335,22 +1332,22 @@ jmp .loop_y_h_overlap .end_y_hv_overlap: add wq, 32 lea srcq, [src_bakq+wq*2] jl .loop_x_hv_overlap -.end_hv: +.end: RET %macro FGUV_FN 3 ; name, ss_hor, ss_ver cglobal fguv_32x32xn_i%1_16bpc, 6, 15, 16, dst, src, stride, fg_data, w, scaling, \ grain_lut, h, sby, luma, lstride, uv_pl, is_id %define base r12-grain_min lea r12, [grain_min] mov r9d, r13m ; bdmax mov r7d, [fg_dataq+FGData.scaling_shift] mov r11d, is_idm - movifnidn sbyd, sbym + mov sbyd, sbym vpbroadcastw m11, [base+mul_bits+r7*2-12] mov r6d, [fg_dataq+FGData.clip_to_restricted_range] shr r9d, 11 ; is_12bpc vpbroadcastd m8, [base+grain_min+r9*4] shlx r10d, r6d, r9d @@ -1383,11 +1380,11 @@ %else vpbroadcastq m15, [base+pw_27_17_17_27] %endif vpbroadcastd m14, [base+pd_16] %endif - test r7b, byte [fg_dataq+FGData.overlap_flag] + test r7b, [fg_dataq+FGData.overlap_flag] jnz %%vertical_overlap imul seed, sbyd, (173 << 24) | 37 add seed, (105 << 24) | 178 rorx seed, seed, 24 @@ -1536,11 +1533,11 @@ %else dec hb %endif jg %%loop_y add wq, 32>>%2 - jge %%end + jge .end mov srcq, r9mp mov dstq, r11mp mov lumaq, r12mp lea srcq, [srcq+wq*2] lea dstq, [dstq+wq*2] @@ -1559,11 +1556,11 @@ cmovp seed, r6d ; updated seed DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ offx, offy, see, left_offxy, unused1, unused2, luma, lstride - lea left_offxyd, [offyd+(32>>%2)] ; previous column's offy*stride+offx + lea left_offxyd, [offyq+(32>>%2)] ; previous column's offy*stride+offx rorx offyd, seed, 8 rorx offxq, seeq, 12 and offyd, 0xf imul offyd, 164>>%3 lea offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx @@ -1715,22 +1712,20 @@ %else dec hb %endif jg %%loop_y_h_overlap add wq, 32>>%2 - jge %%end + jge .end mov srcq, r9mp mov dstq, r11mp mov lumaq, r12mp lea srcq, [srcq+wq*2] lea dstq, [dstq+wq*2] lea lumaq, [lumaq+wq*(2<<%2)] cmp dword r8m, 0 ; sby jne %%loop_x_hv_overlap jmp %%loop_x_h_overlap -%%end: - RET %%vertical_overlap: DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused, \ sby, see, unused1, unused2, unused3, lstride @@ -1955,11 +1950,11 @@ add r10, 4 jmp %%loop_y_v_overlap %endif %%end_y_v_overlap: add wq, 32>>%2 - jge %%end_hv + jge .end mov srcq, r9mp mov dstq, r11mp mov lumaq, r12mp lea srcq, [srcq+wq*2] lea dstq, [dstq+wq*2] @@ -2224,25 +2219,25 @@ add r14, 4 jmp %%loop_y_hv_overlap %endif %%end_y_hv_overlap: add wq, 32>>%2 - jge %%end_hv + jge .end mov srcq, r9mp mov dstq, r11mp mov lumaq, r12mp lea srcq, [srcq+wq*2] lea dstq, [dstq+wq*2] lea lumaq, [lumaq+wq*(2<<%2)] jmp %%loop_x_hv_overlap -%%end_hv: - RET %endmacro %%FGUV_32x32xN_LOOP 1, %2, %3 .csfl: %%FGUV_32x32xN_LOOP 0, %2, %3 +.end: + RET %endmacro GEN_GRAIN_UV_FN 420, 1, 1 FGUV_FN 420, 1, 1 GEN_GRAIN_UV_FN 422, 1, 0 diff --git a/third_party/dav1d/src/x86/filmgrain16_avx512.asm b/third_party/dav1d/src/x86/filmgrain16_avx512.asm new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/x86/filmgrain16_avx512.asm @@ -0,0 +1,932 @@ +; Copyright © 2022, VideoLAN and dav1d authors +; Copyright © 2022, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" +%include "x86/filmgrain_common.asm" + +%if ARCH_X86_64 + +SECTION_RODATA 64 +pb_0to63: db 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + db 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + db 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47 + db 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63 +scale_mask: db -1, -1, 0, -1, -1, -1, 4, -1, -1, -1, 8, -1, -1, -1, 12, -1 +scale_shift: dw 7, 7, 6, 6, 5, 5, 4, 4 +pw_27_17_17_27: dw 108, 68, 68, 108, 27, 17, 17, 27 +pw_23_22: dw 92, 88, 0, 128, 23, 22, 0, 32 +fg_min: times 2 dw 0 + times 2 dw 64 + times 2 dw 256 +fg_max: times 2 dw 1023 + times 2 dw 4095 + times 2 dw 960 + times 2 dw 3840 + times 2 dw 940 + times 2 dw 3760 +scale_rnd: dd 64 + dd 16 +uv_offset_mul: dd 256 + dd 1024 +pb_8_9_0_1: db 8, 9, 0, 1 + +SECTION .text + +INIT_ZMM avx512icl +cglobal fgy_32x32xn_16bpc, 6, 15, 21, dst, src, stride, fg_data, w, scaling, \ + grain_lut, offx, sby, see, offy, src_bak +%define base r11-fg_min + lea r11, [fg_min] + mov r6d, r9m ; bdmax + mov r9d, [fg_dataq+FGData.clip_to_restricted_range] + mov r7d, [fg_dataq+FGData.scaling_shift] + mov sbyd, sbym + vpbroadcastd m6, r9m + shr r6d, 11 ; is_12bpc + vbroadcasti32x4 m7, [base+scale_mask] + shlx r10d, r9d, r6d + vpbroadcastd m10, [base+scale_shift+r7*4-32] + lea r9d, [r6+r9*4] + vpbroadcastd m8, [base+fg_min+r10*4] + kxnorw k1, k1, k1 ; 0xffff + vpbroadcastd m9, [base+fg_max+r9*4] + mov r12, 0xeeeeeeeeeeeeeeee + vpbroadcastd m19, [base+scale_rnd+r6*4] + kshiftrb k2, k1, 4 ; 0xf + vpbroadcastq xm20, [base+pw_27_17_17_27+r6*8] + kmovq k3, r12 + vpbroadcastd m11, [base+scale_shift+r6*8+4] + test sbyd, sbyd + setnz r7b + vpbroadcastd m12, [base+pw_27_17_17_27+r6*8+0] + vpbroadcastd m13, [base+pw_27_17_17_27+r6*8+4] + test r7b, [fg_dataq+FGData.overlap_flag] + jnz .v_overlap + + imul seed, sbyd, (173 << 24) | 37 + add seed, (105 << 24) | 178 + rorx seed, seed, 24 + movzx seed, seew + xor seed, [fg_dataq+FGData.seed] + lea src_bakq, [srcq+wq*2] + neg wq + sub dstq, srcq + +.loop_x: + rorx r6, seeq, 1 + or seed, 0xeff4 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + rorx offyd, seed, 8 + rorx offxq, seeq, 12 + and offyd, 0xf + imul offyd, 164 + lea offyd, [offyq+offxq*2+747] ; offy*stride+offx + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, h, \ + sby, see, offxy, src_bak + + mov grain_lutq, grain_lutmp + mov hd, hm +.loop_y: + movu m4, [grain_lutq+offxyq*2+82*0] + movu m5, [grain_lutq+offxyq*2+82*2] + call .add_noise + sub hb, 2 + jg .loop_y + add wq, 32 + jge .end + lea srcq, [src_bakq+wq*2] + cmp byte [fg_dataq+FGData.overlap_flag], 0 + je .loop_x + test sbyd, sbyd + jnz .hv_overlap + + ; horizontal overlap (without vertical overlap) +.loop_x_h_overlap: + rorx r6, seeq, 1 + or seed, 0xeff4 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, offx, \ + sby, see, offy, src_bak, left_offxy + + lea left_offxyd, [offyq+73] ; previous column's offy*stride+offx + rorx offyd, seed, 8 + rorx offxq, seeq, 12 + and offyd, 0xf + imul offyd, 164 + lea offyd, [offyq+offxq*2+747] ; offy*stride+offx + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, h, \ + sby, see, offxy, src_bak, left_offxy + + mov grain_lutq, grain_lutmp + mov hd, hm +.loop_y_h_overlap: + movu m4, [grain_lutq+offxyq*2+82*0] + movu m5, [grain_lutq+offxyq*2+82*2] + movd xm17, [grain_lutq+left_offxyq*2-82*1] + pinsrd xm17, [grain_lutq+left_offxyq*2+82*1], 1 + punpckldq xm16, xm4, xm5 + punpcklwd xm17, xm16 + mova xm16, xm19 + vpdpwssd xm16, xm20, xm17 + psrad xm16, 1 + packssdw xm16, xm16 + vpsravw xm16, xm11 + vmovdqu8 m4{k2}, m16 + vpalignr m5{k2}, m16, m16, 4 + call .add_noise + sub hb, 2 + jg .loop_y_h_overlap + add wq, 32 + jge .end + lea srcq, [src_bakq+wq*2] + test sbyd, sbyd + jnz .hv_overlap + jmp .loop_x_h_overlap + +.v_overlap: + movzx sbyd, sbyb + imul seed, [fg_dataq+FGData.seed], 0x00010001 + imul r7d, sbyd, 173 * 0x00010001 + imul sbyd, 37 * 0x01000100 + add r7d, (105 << 16) | 188 + add sbyd, (178 << 24) | (141 << 8) + and r7d, 0x00ff00ff + and sbyd, 0xff00ff00 + xor seed, r7d + xor seed, sbyd ; (cur_seed << 16) | top_seed + lea src_bakq, [srcq+wq*2] + neg wq + sub dstq, srcq + + ; we assume from the block above that bits 8-15 of r7d are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp r7b ; parity of top_seed + shr seed, 16 + shl r7d, 16 + test seeb, seeh + setp r7b ; parity of cur_seed + or r6d, 0x00010001 + xor r7d, r6d + rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, offx, \ + sby, see, offy, src_bak, _, top_offxy + + rorx offyd, seed, 8 + rorx offxd, seed, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyd, [offyq+offxq*2+0x10001*747+32*82] + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, h, \ + sby, see, offxy, src_bak, _, top_offxy + + mov grain_lutq, grain_lutmp + mov hd, hm + movzx top_offxyd, offxyw + shr offxyd, 16 + + movu m16, [grain_lutq+offxyq*2+82*0] + movu m0, [grain_lutq+top_offxyq*2+82*0] + movu m17, [grain_lutq+offxyq*2+82*2] + movu m1, [grain_lutq+top_offxyq*2+82*2] + punpckhwd m4, m0, m16 + punpcklwd m0, m16 + punpckhwd m5, m1, m17 + punpcklwd m1, m17 + call .add_noise_v + sub hb, 2 + jg .loop_y + add wq, 32 + jge .end + lea srcq, [src_bakq+wq*2] + + ; since fg_dataq.overlap is guaranteed to be set, we never jump back + ; to .v_overlap, and instead always fall-through to .hv_overlap +.hv_overlap: + ; we assume from the block above that bits 8-15 of r7d are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp r7b ; parity of top_seed + shr seed, 16 + shl r7d, 16 + test seeb, seeh + setp r7b ; parity of cur_seed + or r6d, 0x00010001 + xor r7d, r6d + rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, offx, \ + sby, see, offy, src_bak, left_offxy, top_offxy, topleft_offxy + + lea topleft_offxyd, [top_offxyq+73] + lea left_offxyd, [offyq+73] + rorx offyd, seed, 8 + rorx offxd, seed, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyd, [offyq+offxq*2+0x10001*747+32*82] + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, h, \ + sby, see, offxy, src_bak, left_offxy, top_offxy, topleft_offxy + + mov grain_lutq, grain_lutmp + mov hd, hm + movzx top_offxyd, offxyw + shr offxyd, 16 + + movu m5, [grain_lutq+offxyq*2+82*0] + movu m0, [grain_lutq+top_offxyq*2+82*0] + movd xm17, [grain_lutq+left_offxyq*2-82*1] + pinsrd xm17, [grain_lutq+topleft_offxyq*2-82*1], 1 + movu m2, [grain_lutq+offxyq*2+82*2] + movu m1, [grain_lutq+top_offxyq*2+82*2] + movd xm18, [grain_lutq+left_offxyq*2+82*1] + pinsrd xm18, [grain_lutq+topleft_offxyq*2+82*1], 1 + punpckldq xm16, xm5, xm0 + punpcklwd xm17, xm16 + mova xm16, xm19 + vpdpwssd xm16, xm20, xm17 + punpckldq xm17, xm2, xm1 + punpcklwd xm18, xm17 + mova xm17, xm19 + vpdpwssd xm17, xm20, xm18 + punpckhwd m4, m0, m5 + punpcklwd m0, m5 + punpckhwd m5, m1, m2 + punpcklwd m1, m2 + psrad xm16, 1 + psrad xm17, 1 + packssdw xm16, xm17 + vpsravw xm16, xm11 + vpshuflw m0{k2}, m16, q1302 + punpckhqdq xm16, xm16 + vpshuflw m1{k2}, m16, q1302 + call .add_noise_v + sub hb, 2 + jg .loop_y_h_overlap + add wq, 32 + lea srcq, [src_bakq+wq*2] + jl .hv_overlap +.end: + RET +ALIGN function_align +.add_noise_v: + mova m2, m19 + vpdpwssd m2, m12, m4 + mova m3, m19 + vpdpwssd m3, m13, m5 + mova m4, m19 + vpdpwssd m4, m12, m0 + mova m5, m19 + vpdpwssd m5, m13, m1 + REPX {psrad x, 1}, m2, m3, m4, m5 + packssdw m4, m2 + packssdw m5, m3 + vpsravw m4, m11 + vpsravw m5, m11 +.add_noise: + mova m0, [srcq+strideq*0] + mova m1, [srcq+strideq*1] + kmovw k4, k1 + pand m16, m6, m0 + psrld m3, m0, 16 + vpgatherdd m2{k4}, [scalingq+m16] + vpcmpud k4, m3, m6, 2 ; px <= bdmax + vpgatherdd m16{k4}, [scalingq+m3] + kmovw k4, k1 + pand m17, m6, m1 + vpgatherdd m3{k4}, [scalingq+m17] + vpshufb m2{k3}, m16, m7 + psrld m16, m1, 16 + vpcmpud k4, m16, m6, 2 + vpgatherdd m17{k4}, [scalingq+m16] + vpshufb m3{k3}, m17, m7 + vpsllvw m2, m10 + vpsllvw m3, m10 + pmulhrsw m4, m2 + pmulhrsw m5, m3 + add grain_lutq, 82*4 + paddw m0, m4 + paddw m1, m5 + pmaxsw m0, m8 + pmaxsw m1, m8 + pminsw m0, m9 + pminsw m1, m9 + mova [dstq+srcq], m0 + add srcq, strideq + mova [dstq+srcq], m1 + add srcq, strideq + ret + +%macro FGUV_FN 3 ; name, ss_hor, ss_ver +cglobal fguv_32x32xn_i%1_16bpc, 6, 15, 22, dst, src, stride, fg_data, w, scaling, \ + grain_lut, h, sby, luma, lstride, uv_pl, is_id +%define base r12-fg_min + lea r12, [fg_min] + mov r9d, r13m ; bdmax + mov r7d, [fg_dataq+FGData.scaling_shift] + mov r6d, [fg_dataq+FGData.clip_to_restricted_range] + mov r11d, is_idm + kxnorw k1, k1, k1 ; 0xffff + vpbroadcastd m5, r13m + mov r13, 0xeeeeeeeeeeeeeeee + vbroadcasti32x4 m6, [base+scale_mask] + shr r9d, 11 ; is_12bpc + vpbroadcastd m7, [base+scale_shift+r7*4-32] + shlx r10d, r6d, r9d + mov sbyd, sbym + shlx r6d, r6d, r11d + vpbroadcastd m8, [base+fg_min+r10*4] + lea r6d, [r9+r6*2] + vpbroadcastd m9, [base+fg_max+r6*4] + kmovq k2, r13 + vpbroadcastd m20, [base+scale_rnd+r9*4] + packssdw m4, m5, m5 + vpbroadcastd m21, [base+scale_shift+r9*8+4] +%if %2 + mova m12, [base+pb_0to63] ; pw_even + mov r13d, 0x0101 + vpbroadcastq m10, [base+pw_23_22+r9*8] + kmovw k3, r13d +%if %3 + pshufd m11, m10, q0000 +%else + vpbroadcastd ym16, [base+pw_27_17_17_27+r9*8+0] + vpbroadcastd m11, [base+pw_27_17_17_27+r9*8+4] + vmovdqu16 m11{k1}, m16 +%endif + psrlw m13, m12, 8 ; pw_odd +%else + vpbroadcastq m10, [base+pw_27_17_17_27+r9*8] + kshiftrb k3, k1, 7 ; 0x01 + kshiftrb k4, k1, 4 ; 0x0f + pshufd m11, m10, q0000 +%endif + mov lstrideq, r10mp + test sbyd, sbyd + setnz r7b + cmp byte [fg_dataq+FGData.chroma_scaling_from_luma], 0 + jne .csfl + +%macro %%FGUV_32x32xN_LOOP 3 ; not-csfl, ss_hor, ss_ver + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + _, sby, see, lstride + +%if %1 + mov r6d, r11m + vpbroadcastd m0, [base+uv_offset_mul+r9*4] + vpbroadcastd m1, [base+pb_8_9_0_1] + vpbroadcastd m14, [fg_dataq+FGData.uv_offset+r6*4] + vbroadcasti32x4 m15, [fg_dataq+FGData.uv_mult+r6*4] + pmaddwd m14, m0 + pshufb m15, m1 ; { uv_luma_mult, uv_mult } +%endif + test r7b, [fg_dataq+FGData.overlap_flag] + jnz %%v_overlap + + imul seed, sbyd, (173 << 24) | 37 + add seed, (105 << 24) | 178 + rorx seed, seed, 24 + movzx seed, seew + xor seed, [fg_dataq+FGData.seed] + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, lstride, luma + + mov lumaq, r9mp + lea r12, [srcq+wq*2] + lea r13, [dstq+wq*2] + lea r14, [lumaq+wq*(2<<%2)] + mov r9mp, r12 + mov r10mp, r13 + mov r11mp, r14 + neg wq + +%%loop_x: + rorx r6, seeq, 1 + or seed, 0xeff4 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + rorx offyd, seed, 8 + rorx offxq, seeq, 12 + and offyd, 0xf + imul offyd, 164>>%3 + lea offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+(3+(6>>%2))] ; offy*stride+offx + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, lstride, luma + + mov grain_lutq, grain_lutmp + mov hd, hm +%%loop_y: +%if %2 + movu ym18, [grain_lutq+offxyq*2+82*0] + vinserti32x8 m18, [grain_lutq+offxyq*2+82*2], 1 + movu ym19, [grain_lutq+offxyq*2+82*4] + vinserti32x8 m19, [grain_lutq+offxyq*2+82*6], 1 +%else + movu m18, [grain_lutq+offxyq*2+82*0] + movu m19, [grain_lutq+offxyq*2+82*2] +%endif + call %%add_noise + sub hb, 2<<%2 + jg %%loop_y + add wq, 32>>%2 + jge .end + mov srcq, r9mp + mov dstq, r10mp + mov lumaq, r11mp + lea srcq, [srcq+wq*2] + lea dstq, [dstq+wq*2] + lea lumaq, [lumaq+wq*(2<<%2)] + cmp byte [fg_dataq+FGData.overlap_flag], 0 + je %%loop_x + cmp dword r8m, 0 ; sby + jne %%hv_overlap + + ; horizontal overlap (without vertical overlap) +%%loop_x_h_overlap: + rorx r6, seeq, 1 + or seed, 0xEFF4 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, lstride, luma, left_offxy + + lea left_offxyd, [offyq+(32>>%2)] ; previous column's offy*stride+offx + rorx offyd, seed, 8 + rorx offxq, seeq, 12 + and offyd, 0xf + imul offyd, 164>>%3 + lea offyd, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, lstride, luma, left_offxy + + mov grain_lutq, grain_lutmp + mov hd, hm +%%loop_y_h_overlap: +%if %2 + movu ym18, [grain_lutq+offxyq*2+82*0] + vinserti32x8 m18, [grain_lutq+offxyq*2+82*2], 1 + movu ym19, [grain_lutq+offxyq*2+82*4] + vinserti32x8 m19, [grain_lutq+offxyq*2+82*6], 1 + movd xm16, [grain_lutq+left_offxyq*2+82*0] + vinserti32x4 m16, [grain_lutq+left_offxyq*2+82*2], 2 + movd xm17, [grain_lutq+left_offxyq*2+82*4] + vinserti32x4 m17, [grain_lutq+left_offxyq*2+82*6], 2 + punpckldq m16, m17 + punpckldq m17, m18, m19 + punpcklwd m16, m17 + mova m17, m20 + vpdpwssd m17, m16, m10 + psrad m17, 1 + packssdw m17, m17 + vpsravw m17, m21 +%else + movu m18, [grain_lutq+offxyq*2+82*0] + movu m19, [grain_lutq+offxyq*2+82*2] + movd xm16, [grain_lutq+left_offxyq*2+82*0] + pinsrd xm16, [grain_lutq+left_offxyq*2+82*2], 1 + punpckldq xm17, xm18, xm19 + punpcklwd xm16, xm17 + mova xm17, xm20 + vpdpwssd xm17, xm16, xm10 + psrad xm17, 1 + packssdw xm17, xm17 + vpsravw xm17, xm21 +%endif + vmovdqa32 m18{k3}, m17 + vpshufd m19{k3}, m17, q0321 + call %%add_noise + sub hb, 2<<%2 + jg %%loop_y_h_overlap + add wq, 32>>%2 + jge .end + mov srcq, r9mp + mov dstq, r10mp + mov lumaq, r11mp + lea srcq, [srcq+wq*2] + lea dstq, [dstq+wq*2] + lea lumaq, [lumaq+wq*(2<<%2)] + cmp dword r8m, 0 ; sby + jne %%hv_overlap + jmp %%loop_x_h_overlap + +%%v_overlap: + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + _, sby, see, lstride + + movzx sbyd, sbyb + imul seed, [fg_dataq+FGData.seed], 0x00010001 + imul r7d, sbyd, 173 * 0x00010001 + imul sbyd, 37 * 0x01000100 + add r7d, (105 << 16) | 188 + add sbyd, (178 << 24) | (141 << 8) + and r7d, 0x00ff00ff + and sbyd, 0xff00ff00 + xor seed, r7d + xor seed, sbyd ; (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, lstride, luma, _, top_offxy + + mov lumaq, r9mp + lea r12, [srcq+wq*2] + lea r13, [dstq+wq*2] + lea r14, [lumaq+wq*(2<<%2)] + mov r9mp, r12 + mov r10mp, r13 + mov r11mp, r14 + neg wq + + ; we assume from the block above that bits 8-15 of r7d are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp r7b ; parity of top_seed + shr seed, 16 + shl r7d, 16 + test seeb, seeh + setp r7b ; parity of cur_seed + or r6d, 0x00010001 + xor r7d, r6d + rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed + + rorx offyd, seed, 8 + rorx offxd, seed, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164>>%3 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyd, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, lstride, luma, _, top_offxy + + mov grain_lutq, grain_lutmp + mov hd, hm + movzx top_offxyd, offxyw + shr offxyd, 16 + +%if %3 + movu ym16, [grain_lutq+offxyq*2+82*0] + movu ym1, [grain_lutq+top_offxyq*2+82*0] + vbroadcasti32x8 m18, [grain_lutq+offxyq*2+82*2] + movu ym19, [grain_lutq+offxyq*2+82*4] + vinserti32x8 m19, [grain_lutq+offxyq*2+82*6], 1 + punpcklwd ym17, ym1, ym16 + punpckhwd ym1, ym16 +%elif %2 + movu ym18, [grain_lutq+offxyq*2+82*0] + vinserti32x8 m18, [grain_lutq+offxyq*2+82*2], 1 + movu ym17, [grain_lutq+top_offxyq*2+82*0] + vinserti32x8 m17, [grain_lutq+top_offxyq*2+82*2], 1 + movu ym19, [grain_lutq+offxyq*2+82*4] + vinserti32x8 m19, [grain_lutq+offxyq*2+82*6], 1 + punpcklwd m16, m17, m18 + punpckhwd m17, m18 +%else + movu m18, [grain_lutq+offxyq*2+82*0] + movu m19, [grain_lutq+top_offxyq*2+82*0] + movu m2, [grain_lutq+offxyq*2+82*2] + movu m16, [grain_lutq+top_offxyq*2+82*2] + punpckhwd m1, m19, m18 + punpcklwd m19, m18 + punpckhwd m18, m2, m16 + punpcklwd m2, m16 +%endif + call %%add_noise_v + sub hb, 2<<%2 + jg %%loop_y + add wq, 32>>%2 + jge .end + mov srcq, r9mp + mov dstq, r10mp + mov lumaq, r11mp + lea srcq, [srcq+wq*2] + lea dstq, [dstq+wq*2] + lea lumaq, [lumaq+wq*(2<<%2)] + + ; since fg_dataq.overlap is guaranteed to be set, we never jump back + ; to %%v_overlap, and instead always fall-through to %%hv_overlap +%%hv_overlap: + ; we assume from the block above that bits 8-15 of r7d are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp r7b ; parity of top_seed + shr seed, 16 + shl r7d, 16 + test seeb, seeh + setp r7b ; parity of cur_seed + or r6d, 0x00010001 + xor r7d, r6d + rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, lstride, luma, left_offxy, top_offxy, topleft_offxy + + lea topleft_offxyq, [top_offxyq+(32>>%2)] + lea left_offxyq, [offyq+(32>>%2)] + rorx offyd, seed, 8 + rorx offxd, seed, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164>>%3 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyd, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, lstride, luma, left_offxy, top_offxy, topleft_offxy + + mov grain_lutq, grain_lutmp + mov hd, hm + movzx top_offxyd, offxyw + shr offxyd, 16 + + ; grain = grain_lut[offy+y][offx+x] +%if %2 + movd xm16, [grain_lutq+left_offxyq*2+82*0] + vinserti32x4 m16, [grain_lutq+left_offxyq*2+82*2], 2 + movd xm17, [grain_lutq+left_offxyq*2+82*4] + vinserti32x4 m17, [grain_lutq+left_offxyq*2+82*6], 2 + movu ym18, [grain_lutq+offxyq*2+82*0] + vinserti32x8 m18, [grain_lutq+offxyq*2+82*2], 1 + movu ym19, [grain_lutq+offxyq*2+82*4] + vinserti32x8 m19, [grain_lutq+offxyq*2+82*6], 1 + punpckldq m16, m17 + punpckldq m17, m18, m19 + punpcklwd m16, m17 + movu ym1, [grain_lutq+top_offxyq*2+82*0] + movd xm17, [grain_lutq+topleft_offxyq*2+82*0] + mova m0, m20 + vpdpwssd m0, m16, m10 +%if %3 + punpcklwd xm17, xm1 + mova xm16, xm20 + vpdpwssd xm16, xm17, xm10 + psrad xm16, 1 +%else + vinserti32x8 m1, [grain_lutq+top_offxyq*2+82*2], 1 + vinserti32x4 m17, [grain_lutq+topleft_offxyq*2+82*2], 2 + punpcklwd m17, m1 + mova m16, m20 + vpdpwssd m16, m17, m10 + psrad m16, 1 +%endif + psrad m0, 1 + packssdw m0, m16 + vpsravw m0, m21 + vmovdqa32 m18{k3}, m0 + vpshufd m19{k3}, m0, q0321 +%if %3 + vpunpckhdq ym1{k3}, ym0, ym0 + punpcklwd ym17, ym1, ym18 + punpckhwd ym1, ym18 +%else + vpunpckhdq m1{k3}, m0, m0 + punpcklwd m16, m1, m18 + punpckhwd m17, m1, m18 +%endif +%else + movu m18, [grain_lutq+offxyq*2+82*0] + movu m19, [grain_lutq+top_offxyq*2+82*0] + movd xm17, [grain_lutq+left_offxyq*2+82*0] + pinsrd xm17, [grain_lutq+topleft_offxyq*2+82*0], 1 + punpckldq xm16, xm18, xm19 + punpcklwd xm17, xm16 + movu m2, [grain_lutq+offxyq*2+82*2] + movu m0, [grain_lutq+top_offxyq*2+82*2] + movd xm16, [grain_lutq+left_offxyq*2+82*2] + pinsrd xm16, [grain_lutq+topleft_offxyq*2+82*2], 1 + punpckldq xm1, xm2, xm0 + punpcklwd xm1, xm16, xm1 + mova xm16, xm20 + vpdpwssd xm16, xm17, xm10 + mova xm17, xm20 + vpdpwssd xm17, xm1, xm10 + punpckhwd m1, m19, m18 + punpcklwd m19, m18 + punpckhwd m18, m2, m0 + punpcklwd m2, m0 + psrad xm16, 1 + psrad xm17, 1 + packssdw xm16, xm17 + vpsravw xm16, xm21 + vpshuflw m19{k4}, m16, q1302 + punpckhqdq xm16, xm16 + vpshuflw m2{k4}, m16, q3120 +%endif + call %%add_noise_v + sub hb, 2<<%2 + jg %%loop_y_h_overlap + add wq, 32>>%2 + jge .end + mov srcq, r9mp + mov dstq, r10mp + mov lumaq, r11mp + lea srcq, [srcq+wq*2] + lea dstq, [dstq+wq*2] + lea lumaq, [lumaq+wq*(2<<%2)] + jmp %%hv_overlap + +ALIGN function_align +%%add_noise_v: +%if %3 + mova ym16, ym20 + vpdpwssd ym16, ym17, ym11 + mova ym17, ym20 + vpdpwssd ym17, ym1, ym11 + psrad ym16, 1 + psrad ym17, 1 + packssdw ym16, ym17 + vpsravw m18{k1}, m16, m21 +%elif %2 + mova m18, m20 + vpdpwssd m18, m16, m11 + mova m16, m20 + vpdpwssd m16, m17, m11 + psrad m18, 1 + psrad m16, 1 + packssdw m18, m16 + vpsravw m18, m21 +%else + mova m16, m20 + vpdpwssd m16, m1, m11 + mova m17, m20 + vpdpwssd m17, m18, m11 + mova m18, m20 + vpdpwssd m18, m19, m11 + mova m19, m20 + vpdpwssd m19, m2, m11 + REPX {psrad x, 1}, m16, m17, m18, m19 + packssdw m18, m16 + packssdw m19, m17 + vpsravw m18, m21 + vpsravw m19, m21 +%endif +%%add_noise: +%if %2 + mova m2, [lumaq+lstrideq*(0<<%3)] + mova m0, [lumaq+lstrideq*(1<<%3)] + lea lumaq, [lumaq+lstrideq*(2<<%3)] + mova m3, [lumaq+lstrideq*(0<<%3)] + mova m1, [lumaq+lstrideq*(1<<%3)] + mova m16, m12 + vpermi2w m16, m2, m0 + vpermt2w m2, m13, m0 + mova m17, m12 + vpermi2w m17, m3, m1 + vpermt2w m3, m13, m1 + pavgw m2, m16 + pavgw m3, m17 +%elif %1 + mova m2, [lumaq+lstrideq*0] + mova m3, [lumaq+lstrideq*1] +%endif +%if %2 + mova ym16, [srcq+strideq*0] + vinserti32x8 m16, [srcq+strideq*1], 1 + lea srcq, [srcq+strideq*2] +%else + mova m16, [srcq+strideq*0] +%endif +%if %1 + punpckhwd m17, m2, m16 + mova m0, m14 + vpdpwssd m0, m17, m15 + punpcklwd m17, m2, m16 + mova m2, m14 + vpdpwssd m2, m17, m15 +%endif +%if %2 + mova ym17, [srcq+strideq*0] + vinserti32x8 m17, [srcq+strideq*1], 1 +%else + mova m17, [srcq+strideq*1] +%endif +%if %1 + psrad m0, 6 + psrad m2, 6 + packusdw m2, m0 + punpckhwd m0, m3, m17 + mova m1, m14 + vpdpwssd m1, m15, m0 + punpcklwd m0, m3, m17 + mova m3, m14 + vpdpwssd m3, m15, m0 + psrad m1, 6 + psrad m3, 6 + packusdw m3, m1 + pminuw m2, m4 + pminuw m3, m4 + +.add_noise_main: + ; scaling[luma_src] + kmovw k5, k1 + pand m1, m5, m2 + vpgatherdd m0{k5}, [scalingq+m1] + kmovw k5, k1 + psrld m2, 16 + vpgatherdd m1{k5}, [scalingq+m2] + vpshufb m0{k2}, m1, m6 + kmovw k5, k1 + psrld m1, m3, 16 + vpgatherdd m2{k5}, [scalingq+m1] + kmovw k5, k1 + pand m3, m5 + vpgatherdd m1{k5}, [scalingq+m3] + vpshufb m1{k2}, m2, m6 + + ; noise = round2(scaling[luma_src] * grain, scaling_shift) + vpsllvw m0, m7 + vpsllvw m1, m7 + pmulhrsw m18, m0 + pmulhrsw m19, m1 + add grain_lutq, 82*(4<<%2) + lea lumaq, [lumaq+lstrideq*(2<<%3)] + lea srcq, [srcq+strideq*2] + paddw m16, m18 + paddw m17, m19 + pmaxsw m16, m8 + pmaxsw m17, m8 + pminsw m16, m9 + pminsw m17, m9 +%if %2 + mova [dstq+strideq*0], ym16 + vextracti32x8 [dstq+strideq*1], m16, 1 + lea dstq, [dstq+strideq*2] + mova [dstq+strideq*0], ym17 + vextracti32x8 [dstq+strideq*1], m17, 1 +%else + mova [dstq+strideq*0], m16 + mova [dstq+strideq*1], m17 +%endif + lea dstq, [dstq+strideq*2] + ret +%else +%if %2 + pand m2, m4 + pand m3, m4 +%else + pand m2, m4, [lumaq+lstrideq*0] + pand m3, m4, [lumaq+lstrideq*1] +%endif + jmp .add_noise_main +%endif +%endmacro + + %%FGUV_32x32xN_LOOP 1, %2, %3 +.csfl: + %%FGUV_32x32xN_LOOP 0, %2, %3 +.end: + RET +%endmacro + +FGUV_FN 420, 1, 1 +FGUV_FN 422, 1, 0 +FGUV_FN 444, 0, 0 + +%endif diff --git a/third_party/dav1d/src/x86/filmgrain_avx2.asm b/third_party/dav1d/src/x86/filmgrain_avx2.asm --- a/third_party/dav1d/src/x86/filmgrain_avx2.asm +++ b/third_party/dav1d/src/x86/filmgrain_avx2.asm @@ -1486,11 +1486,11 @@ add grain_lutq, 82<<%2 sub hb, 1+%2 jg %%loop_y add wq, 32>>%2 - jge %%end + jge .end mov srcq, r11mp mov dstq, r12mp lea lumaq, [r14+wq*(1+%2)] add srcq, wq add dstq, wq @@ -1645,11 +1645,11 @@ add grain_lutq, 82*(1+%2) sub hb, 1+%2 jg %%loop_y_h_overlap add wq, 32>>%2 - jge %%end + jge .end mov srcq, r11mp mov dstq, r12mp lea lumaq, [r14+wq*(1+%2)] add srcq, wq add dstq, wq @@ -1850,11 +1850,11 @@ %endif jmp %%loop_y %%end_y_v_overlap: add wq, 32>>%2 - jge %%end + jge .end mov srcq, r11mp mov dstq, r12mp lea lumaq, [r14+wq*(1+%2)] add srcq, wq add dstq, wq @@ -2079,24 +2079,24 @@ jmp %%loop_y_h_overlap %endif %%end_y_hv_overlap: add wq, 32>>%2 - jge %%end + jge .end mov srcq, r11mp mov dstq, r12mp lea lumaq, [r14+wq*(1+%2)] add srcq, wq add dstq, wq jmp %%loop_x_hv_overlap -%%end: - RET %endmacro %%FGUV_32x32xN_LOOP 1, %2, %3 .csfl: %%FGUV_32x32xN_LOOP 0, %2, %3 +.end: + RET %endmacro GEN_GRAIN_UV_FN 420, 1, 1 FGUV_FN 420, 1, 1 GEN_GRAIN_UV_FN 422, 1, 0 diff --git a/third_party/dav1d/src/x86/filmgrain_avx512.asm b/third_party/dav1d/src/x86/filmgrain_avx512.asm --- a/third_party/dav1d/src/x86/filmgrain_avx512.asm +++ b/third_party/dav1d/src/x86/filmgrain_avx512.asm @@ -117,38 +117,13 @@ h, sby, see, overlap mov grain_lutq, grain_lutmp mov hd, hm .loop_y: - mova ym18, [srcq+strideq*0] - vinserti32x8 m18, [srcq+strideq*1], 1 movu ym21, [grain_lutq+offxyq-82] vinserti32x8 m21, [grain_lutq+offxyq+ 0], 1 - mova m19, m0 - vpmovb2m k2, m18 - punpcklbw m16, m18, m5 - punpckhbw m17, m18, m5 - vpermt2b m19, m18, m1 ; scaling[ 0..127] - vpermi2b m18, m2, m3 ; scaling[128..255] - punpcklbw m20, m5, m21 ; grain - punpckhbw m21, m5 - vmovdqu8 m19{k2}, m18 ; scaling[src] - pshufb m19, m4 - pmaddubsw m18, m19, m20 - pmaddubsw m19, m21 - add grain_lutq, 82*2 - pmulhrsw m18, m6 ; noise - pmulhrsw m19, m6 - paddw m16, m18 - paddw m17, m19 - packuswb m16, m17 - pmaxub m16, m7 - pminub m16, m8 - mova [dstq+srcq], ym16 - add srcq, strideq - vextracti32x8 [dstq+srcq], m16, 1 - add srcq, strideq + call .add_noise sub hb, 2 jg .loop_y add wq, 32 jge .end lea srcq, [src_bakq+wq] @@ -180,42 +155,19 @@ mov grain_lutq, grain_lutmp mov hd, hm .loop_y_h_overlap: movu ym20, [grain_lutq+offxyq-82] vinserti32x8 m20, [grain_lutq+offxyq+ 0], 1 - movd xm21, [grain_lutq+left_offxyq-50] - vinserti32x4 m21, [grain_lutq+left_offxyq+32], 2 - mova ym18, [srcq+strideq*0] - vinserti32x8 m18, [srcq+strideq*1], 1 - mova m19, m0 - punpcklbw m21, m20 - vpmovb2m k2, m18 - punpcklbw m16, m18, m5 - punpckhbw m17, m18, m5 - pmaddubsw m21, m10, m21 - vpermt2b m19, m18, m1 - vpermi2b m18, m2, m3 - pmulhrsw m21, m9 - vmovdqu8 m19{k2}, m18 ; scaling[src] - punpckhbw m18, m20, m5 - pshufb m19, m4 - packsswb m20{k1}, m21, m21 - punpcklbw m20, m5, m20 ; grain - pmaddubsw m18, m19, m18 - pmaddubsw m19, m20 - add grain_lutq, 82*2 - pmulhrsw m18, m6 ; noise - pmulhrsw m19, m6 - paddw m17, m18 - paddw m16, m19 - packuswb m16, m17 - pmaxub m16, m7 - pminub m16, m8 - mova [dstq+srcq], ym16 - add srcq, strideq - vextracti32x8 [dstq+srcq], m16, 1 - add srcq, strideq + movd xm19, [grain_lutq+left_offxyq-50] + vinserti32x4 m19, [grain_lutq+left_offxyq+32], 2 + punpcklbw m19, m20 + pmaddubsw m19, m10, m19 + pmulhrsw m19, m9 + punpckhbw m21, m20, m5 + packsswb m20{k1}, m19, m19 + punpcklbw m20, m5, m20 + call .add_noise_h sub hb, 2 jg .loop_y_h_overlap add wq, 32 jge .end lea srcq, [src_bakq+wq] @@ -270,47 +222,17 @@ mov grain_lutq, grain_lutmp mov hd, hm movzx top_offxyd, offxyw shr offxyd, 16 - movu ym16, [grain_lutq+offxyq-82] - vinserti32x8 m16, [grain_lutq+offxyq+ 0], 1 + movu ym19, [grain_lutq+offxyq-82] + vinserti32x8 m19, [grain_lutq+offxyq+ 0], 1 movu ym21, [grain_lutq+top_offxyq-82] vinserti32x8 m21, [grain_lutq+top_offxyq+ 0], 1 - mova ym18, [srcq+strideq*0] - vinserti32x8 m18, [srcq+strideq*1], 1 - mova m19, m0 - punpcklbw m20, m21, m16 - punpckhbw m21, m16 - vpmovb2m k2, m18 - pmaddubsw m20, m12, m20 - pmaddubsw m21, m12, m21 - punpcklbw m16, m18, m5 - punpckhbw m17, m18, m5 - vpermt2b m19, m18, m1 - vpermi2b m18, m2, m3 - pmulhrsw m20, m9 - pmulhrsw m21, m9 - vmovdqu8 m19{k2}, m18 ; scaling[src] - pshufb m19, m4 - packsswb m20, m21 - punpcklbw m18, m5, m20 ; grain - punpckhbw m20, m5 - pmaddubsw m18, m19, m18 - pmaddubsw m19, m20 - add grain_lutq, 82*2 - pmulhrsw m18, m6 ; noise - pmulhrsw m19, m6 - paddw m16, m18 - paddw m17, m19 - packuswb m16, m17 - pmaxub m16, m7 - pminub m16, m8 - mova [dstq+srcq], ym16 - add srcq, strideq - vextracti32x8 [dstq+srcq], m16, 1 - add srcq, strideq + punpckhbw m20, m21, m19 + punpcklbw m21, m19 + call .add_noise_v sub hb, 2 jg .loop_y add wq, 32 jge .end lea srcq, [src_bakq+wq] @@ -357,39 +279,51 @@ vinserti32x4 m16, [grain_lutq+left_offxyq+32], 2 movu ym21, [grain_lutq+top_offxyq-82] vinserti32x8 m21, [grain_lutq+top_offxyq+ 0], 1 movd xm17, [grain_lutq+topleft_offxyq-50] vinserti32x4 m17, [grain_lutq+topleft_offxyq+32], 2 - mova ym18, [srcq+strideq*0] - vinserti32x8 m18, [srcq+strideq*1], 1 ; do h interpolation first (so top | top/left -> top, left | cur -> cur) punpcklbw m16, m19 + pmaddubsw m16, m10, m16 punpcklbw m17, m21 - pmaddubsw m16, m10, m16 pmaddubsw m17, m10, m17 punpckhbw m20, m21, m19 - vpmovb2m k2, m18 pmulhrsw m16, m9 pmulhrsw m17, m9 packsswb m19{k1}, m16, m16 packsswb m21{k1}, m17, m17 ; followed by v interpolation (top | cur -> cur) punpcklbw m21, m19 - mova m19, m0 + call .add_noise_v + sub hb, 2 + jg .loop_y_h_overlap + add wq, 32 + lea srcq, [src_bakq+wq] + jl .hv_overlap +.end: + RET +ALIGN function_align +.add_noise_v: pmaddubsw m20, m12, m20 pmaddubsw m21, m12, m21 - punpcklbw m16, m18, m5 - punpckhbw m17, m18, m5 - vpermt2b m19, m18, m1 ; scaling[ 0..127] - vpermi2b m18, m2, m3 ; scaling[128..255] pmulhrsw m20, m9 pmulhrsw m21, m9 + packsswb m21, m20 +.add_noise: + punpcklbw m20, m5, m21 + punpckhbw m21, m5 +.add_noise_h: + mova ym18, [srcq+strideq*0] + vinserti32x8 m18, [srcq+strideq*1], 1 + mova m19, m0 + punpcklbw m16, m18, m5 + vpermt2b m19, m18, m1 ; scaling[ 0..127] + vpmovb2m k2, m18 + punpckhbw m17, m18, m5 + vpermi2b m18, m2, m3 ; scaling[128..255] vmovdqu8 m19{k2}, m18 ; scaling[src] pshufb m19, m4 - packsswb m21, m20 - punpcklbw m20, m5, m21 - punpckhbw m21, m5 pmaddubsw m18, m19, m20 pmaddubsw m19, m21 add grain_lutq, 82*2 pmulhrsw m18, m6 ; noise pmulhrsw m19, m6 @@ -400,17 +334,11 @@ pminub m16, m8 mova [dstq+srcq], ym16 add srcq, strideq vextracti32x8 [dstq+srcq], m16, 1 add srcq, strideq - sub hb, 2 - jg .loop_y_h_overlap - add wq, 32 - lea srcq, [src_bakq+wq] - jl .hv_overlap -.end: - RET + ret %macro FGUV_FN 3 ; name, ss_hor, ss_ver cglobal fguv_32x32xn_i%1_8bpc, 6, 14+%2, 22, dst, src, stride, fg_data, w, \ scaling, grain_lut, h, sby, luma, \ overlap, uv_pl, is_id, _, stride3 @@ -494,81 +422,24 @@ h, offxy, see, overlap, _, _, _, stride3 mov grain_lutq, grain_lutmp mov hd, hm %%loop_y: - mova ym18, [lumaq+lstrideq*(0<<%3)] - vinserti32x8 m18, [lumaq+lstrideq*(1<<%3)], 1 - lea lumaq, [lumaq+lstrideq*(2<<%3)] %if %2 - mova ym20, [lumaq+lstrideq*(0<<%3)] - vinserti32x8 m20, [lumaq+lstrideq*(1<<%3)], 1 - lea lumaq, [lumaq+lstrideq*(2<<%3)] - mova xm17, [srcq+strideq*0] movu xm21, [grain_lutq+offxyq+82*0] - vinserti128 ym17, [srcq+strideq*1], 1 vinserti128 ym21, [grain_lutq+offxyq+82*1], 1 - mova m19, m11 - vpermi2b m19, m18, m20 - vpermt2b m18, m12, m20 - vinserti32x4 m17, [srcq+strideq*2], 2 vinserti32x4 m21, [grain_lutq+offxyq+82*2], 2 - pavgb m18, m19 - vinserti32x4 m17, [srcq+stride3q ], 3 vinserti32x4 m21, [grain_lutq+offxyq+82*3], 3 %else - mova ym17, [srcq+strideq*0] - vinserti32x8 m17, [srcq+strideq*1], 1 movu ym21, [grain_lutq+offxyq+82*0] vinserti32x8 m21, [grain_lutq+offxyq+82*1], 1 %endif - lea srcq, [srcq+strideq*(2<<%2)] -%if %1 - punpckhbw m19, m18, m17 - punpcklbw m18, m17 ; { luma, chroma } - pmaddubsw m19, m14 - pmaddubsw m18, m14 - psraw m19, 6 - psraw m18, 6 - paddw m19, m15 - paddw m18, m15 - packuswb m18, m19 -%endif - mova m19, m0 - vpmovb2m k2, m18 - vpermt2b m19, m18, m1 ; scaling[ 0..127] - vpermi2b m18, m2, m3 ; scaling[128..255] - punpcklbw m20, m5, m21 ; grain - punpckhbw m21, m5 - vmovdqu8 m19{k2}, m18 ; scaling[src] - pshufb m19, m4 - pmaddubsw m18, m19, m20 - pmaddubsw m19, m21 - add grain_lutq, 82*2<<%2 - pmulhrsw m18, m6 ; noise - pmulhrsw m19, m6 - punpcklbw m16, m17, m5 ; chroma - punpckhbw m17, m5 - paddw m16, m18 - paddw m17, m19 - packuswb m16, m17 - pmaxub m16, m7 - pminub m16, m8 -%if %2 - mova [dstq+strideq*0], xm16 - vextracti128 [dstq+strideq*1], ym16, 1 - vextracti32x4 [dstq+strideq*2], m16, 2 - vextracti32x4 [dstq+stride3q ], m16, 3 -%else - mova [dstq+strideq*0], ym16 - vextracti32x8 [dstq+strideq*1], m16, 1 -%endif - lea dstq, [dstq+strideq*(2<<%2)] + call %%add_noise sub hb, 2<<%2 jg %%loop_y add wq, 32>>%2 - jge %%end + jge .end mov srcq, r11mp mov dstq, r12mp lea lumaq, [r13+wq*(1<<%2)] add srcq, wq add dstq, wq @@ -599,98 +470,36 @@ h, offxy, see, left_offxy, _, _, _, stride3 mov grain_lutq, grain_lutmp mov hd, hm %%loop_y_h_overlap: - ; src -%if %2 - mova ym18, [lumaq+lstrideq*(0<<%3)] - vinserti32x8 m18, [lumaq+lstrideq*(1<<%3)], 1 - lea lumaq, [lumaq+lstrideq*(2<<%3)] - mova ym20, [lumaq+lstrideq*(0<<%3)] - vinserti32x8 m20, [lumaq+lstrideq*(1<<%3)], 1 - lea lumaq, [lumaq+lstrideq*(2<<%3)] - mova xm17, [srcq+strideq*0] - vinserti128 ym17, [srcq+strideq*1], 1 - mova m19, m11 - vpermi2b m19, m18, m20 - vpermt2b m18, m12, m20 - vinserti32x4 m17, [srcq+strideq*2], 2 - pavgb m18, m19 - vinserti32x4 m17, [srcq+stride3q ], 3 -%else - mova ym18, [lumaq+lstrideq*0] - vinserti32x8 m18, [lumaq+lstrideq*1], 1 - mova ym17, [srcq+strideq*0] - vinserti32x8 m17, [srcq+strideq*1], 1 - lea lumaq, [lumaq+lstrideq*2] -%endif - lea srcq, [srcq+strideq*(2<<%2)] -%if %1 - punpckhbw m19, m18, m17 - punpcklbw m18, m17 ; { luma, chroma } - pmaddubsw m19, m14 - pmaddubsw m18, m14 - psraw m19, 6 - psraw m18, 6 - paddw m19, m15 - paddw m18, m15 - packuswb m18, m19 -%endif - mova m19, m0 - vpmovb2m k2, m18 - vpermt2b m19, m18, m1 ; scaling[ 0..127] - vpermi2b m18, m2, m3 ; scaling[128..255] - vmovdqu8 m19{k2}, m18 ; scaling[src] %if %2 movu xm20, [grain_lutq+offxyq +82*0] - movd xm18, [grain_lutq+left_offxyq+82*0] + movd xm19, [grain_lutq+left_offxyq+82*0] vinserti32x4 ym20, [grain_lutq+offxyq +82*1], 1 - vinserti32x4 ym18, [grain_lutq+left_offxyq+82*1], 1 + vinserti32x4 ym19, [grain_lutq+left_offxyq+82*1], 1 vinserti32x4 m20, [grain_lutq+offxyq +82*2], 2 - vinserti32x4 m18, [grain_lutq+left_offxyq+82*2], 2 + vinserti32x4 m19, [grain_lutq+left_offxyq+82*2], 2 vinserti32x4 m20, [grain_lutq+offxyq +82*3], 3 - vinserti32x4 m18, [grain_lutq+left_offxyq+82*3], 3 + vinserti32x4 m19, [grain_lutq+left_offxyq+82*3], 3 %else movu ym20, [grain_lutq+offxyq + 0] - movd xm18, [grain_lutq+left_offxyq+ 0] + movd xm19, [grain_lutq+left_offxyq+ 0] vinserti32x8 m20, [grain_lutq+offxyq +82], 1 - vinserti32x4 m18, [grain_lutq+left_offxyq+82], 2 + vinserti32x4 m19, [grain_lutq+left_offxyq+82], 2 %endif - punpcklbw m18, m20 - pmaddubsw m18, m10, m18 + punpcklbw m19, m20 + pmaddubsw m19, m10, m19 punpckhbw m21, m20, m5 - pshufb m19, m4 - pmulhrsw m18, m9 - vpacksswb m20{k1}, m18, m18 + pmulhrsw m19, m9 + vpacksswb m20{k1}, m19, m19 punpcklbw m20, m5, m20 - pmaddubsw m18, m19, m20 - pmaddubsw m19, m21 - add grain_lutq, 82*2<<%2 - pmulhrsw m18, m6 ; noise - pmulhrsw m19, m6 - punpcklbw m16, m17, m5 ; chroma - punpckhbw m17, m5 - paddw m16, m18 - paddw m17, m19 - packuswb m16, m17 - pmaxub m16, m7 - pminub m16, m8 -%if %2 - mova [dstq+strideq*0], xm16 - vextracti128 [dstq+strideq*1], ym16, 1 - vextracti32x4 [dstq+strideq*2], m16, 2 - vextracti32x4 [dstq+stride3q ], m16, 3 -%else - mova [dstq+strideq*0], ym16 - vextracti32x8 [dstq+strideq*1], m16, 1 -%endif - lea dstq, [dstq+strideq*(2<<%2)] + call %%add_noise_h sub hb, 2<<%2 jg %%loop_y_h_overlap add wq, 32>>%2 - jge %%end + jge .end mov srcq, r11mp mov dstq, r12mp lea lumaq, [r13+wq*(1<<%2)] add srcq, wq add dstq, wq @@ -763,119 +572,39 @@ mov grain_lutq, grain_lutmp mov hd, hm movzx top_offxyd, offxyw shr offxyd, 16 -%if %2 - mova ym18, [lumaq+lstrideq*(0<<%3)] - vinserti32x8 m18, [lumaq+lstrideq*(1<<%3)], 1 - lea lumaq, [lumaq+lstrideq*(2<<%3)] - mova ym20, [lumaq+lstrideq*(0<<%3)] - vinserti32x8 m20, [lumaq+lstrideq*(1<<%3)], 1 - lea lumaq, [lumaq+lstrideq*(2<<%3)] - mova xm17, [srcq+strideq*0] - vinserti128 ym17, [srcq+strideq*1], 1 - mova m19, m11 - vpermi2b m19, m18, m20 - vpermt2b m18, m12, m20 - vinserti32x4 m17, [srcq+strideq*2], 2 - pavgb m18, m19 - vinserti32x4 m17, [srcq+stride3q ], 3 -%else - mova ym18, [lumaq+lstrideq*0] - vinserti32x8 m18, [lumaq+lstrideq*1], 1 - mova ym17, [srcq+strideq*0] - vinserti32x8 m17, [srcq+strideq*1], 1 - lea lumaq, [lumaq+lstrideq*2] -%endif - lea srcq, [srcq+strideq*(2<<%2)] -%if %1 - punpckhbw m19, m18, m17 - punpcklbw m18, m17 ; { luma, chroma } - pmaddubsw m19, m14 - pmaddubsw m18, m14 - psraw m19, 6 - psraw m18, 6 - paddw m19, m15 - paddw m18, m15 - packuswb m18, m19 -%endif - mova m19, m0 - vpmovb2m k2, m18 - vpermt2b m19, m18, m1 ; scaling[ 0..127] - vpermi2b m18, m2, m3 ; scaling[128..255] %if %3 - movu xm21, [grain_lutq+offxyq+82*0] - movu xm16, [grain_lutq+top_offxyq+82*0] - punpcklbw xm20, xm16, xm21 - punpckhbw xm16, xm21 - pmaddubsw xm20, xm13, xm20 - pmaddubsw xm16, xm13, xm16 + movu xm18, [grain_lutq+offxyq+82*0] + movu xm20, [grain_lutq+top_offxyq+82*0] ; only interpolate first line, insert remaining line unmodified vbroadcasti128 ym21, [grain_lutq+offxyq+82*1] vinserti32x4 m21, [grain_lutq+offxyq+82*2], 2 vinserti32x4 m21, [grain_lutq+offxyq+82*3], 3 - pmulhrsw xm20, xm9 - pmulhrsw xm16, xm9 - vpacksswb m21{k3}, m20, m16 + punpcklbw xm19, xm20, xm18 + punpckhbw xm20, xm18 %elif %2 - movu xm21, [grain_lutq+offxyq+82*0] - vinserti128 ym21, [grain_lutq+offxyq+82*1], 1 - movu xm16, [grain_lutq+top_offxyq+82*0] - vinserti32x4 ym16, [grain_lutq+top_offxyq+82*1], 1 - punpcklbw ym20, ym16, ym21 - punpckhbw ym16, ym21 - pmaddubsw ym20, ym13, ym20 - pmaddubsw ym16, ym13, ym16 + movu xm18, [grain_lutq+offxyq+82*0] + vinserti128 ym18, [grain_lutq+offxyq+82*1], 1 + movu xm20, [grain_lutq+top_offxyq+82*0] + vinserti32x4 ym20, [grain_lutq+top_offxyq+82*1], 1 vbroadcasti32x4 m21, [grain_lutq+offxyq+82*2] vinserti32x4 m21, [grain_lutq+offxyq+82*3], 3 - pmulhrsw ym20, ym9 - pmulhrsw ym16, ym9 - packsswb m21{k3}, m20, m16 + punpcklbw ym19, ym20, ym18 + punpckhbw ym20, ym18 %else - movu ym16, [grain_lutq+offxyq+82*0] - vinserti32x8 m16, [grain_lutq+offxyq+82*1], 1 + movu ym21, [grain_lutq+offxyq+82*0] + vinserti32x8 m21, [grain_lutq+offxyq+82*1], 1 movu ym20, [grain_lutq+top_offxyq+82*0] vinserti32x8 m20, [grain_lutq+top_offxyq+82*1], 1 - punpcklbw m21, m20, m16 - punpckhbw m20, m16 - pmaddubsw m21, m13, m21 - pmaddubsw m20, m13, m20 - pmulhrsw m21, m9 - pmulhrsw m20, m9 - packsswb m21, m20 %endif - vmovdqu8 m19{k2}, m18 ; scaling[src] - pshufb m19, m4 - punpcklbw m20, m5, m21 - punpckhbw m21, m5 - pmaddubsw m18, m19, m20 - pmaddubsw m19, m21 - add grain_lutq, 82*2<<%2 - pmulhrsw m18, m6 ; noise - pmulhrsw m19, m6 - punpcklbw m16, m17, m5 ; chroma - punpckhbw m17, m5 - paddw m16, m18 - paddw m17, m19 - packuswb m16, m17 - pmaxub m16, m7 - pminub m16, m8 -%if %2 - mova [dstq+strideq*0], xm16 - vextracti128 [dstq+strideq*1], ym16, 1 - vextracti32x4 [dstq+strideq*2], m16, 2 - vextracti32x4 [dstq+stride3q ], m16, 3 -%else - mova [dstq+strideq*0], ym16 - vextracti32x8 [dstq+strideq*1], m16, 1 -%endif - lea dstq, [dstq+strideq*(2<<%2)] + call %%add_noise_v sub hb, 2<<%2 jg %%loop_y add wq, 32>>%2 - jge %%end + jge .end mov srcq, r11mp mov dstq, r12mp lea lumaq, [r13+wq*(1<<%2)] add srcq, wq add dstq, wq @@ -937,16 +666,20 @@ %endif punpcklqdq m16, m18 pmaddubsw m16, m10, m16 pmulhrsw m16, m9 packsswb m16, m16 + vmovdqu8 m21{k1}, m16 %if %3 vpalignr xm20{k1}, xm16, xm16, 4 + punpcklbw xm19, xm20, xm21 + punpckhbw xm20, xm21 %else vpalignr ym20{k1}, ym16, ym16, 4 + punpcklbw ym19, ym20, ym21 + punpckhbw ym20, ym21 %endif - vmovdqu8 m21{k1}, m16 %else movu ym21, [grain_lutq+offxyq+82*0] vinserti32x8 m21, [grain_lutq+offxyq+82*1], 1 movd xm16, [grain_lutq+left_offxyq+82*0] vinserti32x4 m16, [grain_lutq+left_offxyq+82*1], 2 @@ -961,82 +694,89 @@ pmulhrsw m16, m9 packsswb m16, m16 vpalignr m20{k1}, m16, m16, 4 vmovdqu8 m21{k1}, m16 %endif -%if %2 + call %%add_noise_v + sub hb, 2<<%2 + jg %%loop_y_h_overlap + add wq, 32>>%2 + jge .end + mov srcq, r11mp + mov dstq, r12mp + lea lumaq, [r13+wq*(1<<%2)] + add srcq, wq + add dstq, wq + jmp %%hv_overlap +ALIGN function_align +%%add_noise_v: +%if %3 + pmaddubsw xm19, xm13, xm19 + pmaddubsw xm20, xm13, xm20 + pmulhrsw xm19, xm9 + pmulhrsw xm20, xm9 + vpacksswb m21{k3}, m19, m20 +%elif %2 + pmaddubsw ym19, ym13, ym19 + pmaddubsw ym20, ym13, ym20 + pmulhrsw ym19, ym9 + pmulhrsw ym20, ym9 + vpacksswb m21{k3}, m19, m20 +%else + punpcklbw m19, m20, m21 + punpckhbw m20, m21 + pmaddubsw m19, m13, m19 + pmaddubsw m20, m13, m20 + pmulhrsw m19, m9 + pmulhrsw m20, m9 + packsswb m21, m19, m20 +%endif +%%add_noise: + punpcklbw m20, m5, m21 + punpckhbw m21, m5 +%%add_noise_h: mova ym18, [lumaq+lstrideq*(0<<%3)] vinserti32x8 m18, [lumaq+lstrideq*(1<<%3)], 1 +%if %2 lea lumaq, [lumaq+lstrideq*(2<<%3)] mova ym16, [lumaq+lstrideq*(0<<%3)] vinserti32x8 m16, [lumaq+lstrideq*(1<<%3)], 1 - lea lumaq, [lumaq+lstrideq*(2<<%3)] mova xm17, [srcq+strideq*0] - vinserti128 ym17, [srcq+strideq*1], 1 mova m19, m11 vpermi2b m19, m18, m16 + vinserti128 ym17, [srcq+strideq*1], 1 vpermt2b m18, m12, m16 vinserti32x4 m17, [srcq+strideq*2], 2 pavgb m18, m19 vinserti32x4 m17, [srcq+stride3q ], 3 %else - mova ym18, [lumaq+lstrideq*0] - vinserti32x8 m18, [lumaq+lstrideq*1], 1 mova ym17, [srcq+strideq*0] vinserti32x8 m17, [srcq+strideq*1], 1 - lea lumaq, [lumaq+lstrideq*2] %endif - lea srcq, [srcq+strideq*(2<<%2)] %if %1 punpckhbw m19, m18, m17 punpcklbw m18, m17 ; { luma, chroma } pmaddubsw m19, m14 pmaddubsw m18, m14 psraw m19, 6 psraw m18, 6 paddw m19, m15 paddw m18, m15 packuswb m18, m19 -%endif +.add_noise_main: mova m19, m0 + vpermt2b m19, m18, m1 ; scaling[ 0..127] vpmovb2m k2, m18 - vpermt2b m19, m18, m1 ; scaling[ 0..127] vpermi2b m18, m2, m3 ; scaling[128..255] - ; followed by v interpolation (top | cur -> cur) -%if %3 - punpcklbw xm16, xm20, xm21 - punpckhbw xm20, xm21 - pmaddubsw xm16, xm13, xm16 - pmaddubsw xm20, xm13, xm20 - pmulhrsw xm16, xm9 - pmulhrsw xm20, xm9 - vpacksswb m21{k3}, m16, m20 -%elif %2 - punpcklbw ym16, ym20, ym21 - punpckhbw ym20, ym21 - pmaddubsw ym16, ym13, ym16 - pmaddubsw ym20, ym13, ym20 - pmulhrsw ym16, ym9 - pmulhrsw ym20, ym9 - vpacksswb m21{k3}, m16, m20 -%else - punpcklbw m16, m20, m21 - punpckhbw m20, m21 - pmaddubsw m16, m13, m16 - pmaddubsw m20, m13, m20 - pmulhrsw m16, m9 - pmulhrsw m20, m9 - packsswb m21, m16, m20 -%endif vmovdqu8 m19{k2}, m18 ; scaling[src] pshufb m19, m4 - punpcklbw m20, m5, m21 - punpckhbw m21, m5 pmaddubsw m18, m19, m20 pmaddubsw m19, m21 add grain_lutq, 82*2<<%2 - pmulhrsw m18, m6 ; grain + lea lumaq, [lumaq+lstrideq*(2<<%3)] + lea srcq, [srcq+strideq*(2<<%2)] + pmulhrsw m18, m6 ; noise pmulhrsw m19, m6 punpcklbw m16, m17, m5 ; chroma punpckhbw m17, m5 paddw m16, m18 paddw m17, m19 @@ -1051,27 +791,21 @@ %else mova [dstq+strideq*0], ym16 vextracti32x8 [dstq+strideq*1], m16, 1 %endif lea dstq, [dstq+strideq*(2<<%2)] - sub hb, 2<<%2 - jg %%loop_y_h_overlap - add wq, 32>>%2 - jge %%end - mov srcq, r11mp - mov dstq, r12mp - lea lumaq, [r13+wq*(1<<%2)] - add srcq, wq - add dstq, wq - jmp %%hv_overlap -%%end: - RET + ret +%else + jmp .add_noise_main +%endif %endmacro %%FGUV_32x32xN_LOOP 1, %2, %3 .csfl: %%FGUV_32x32xN_LOOP 0, %2, %3 +.end: + RET %endmacro FGUV_FN 420, 1, 1 FGUV_FN 422, 1, 0 FGUV_FN 444, 0, 0 diff --git a/third_party/dav1d/src/x86/filmgrain_init_tmpl.c b/third_party/dav1d/src/x86/filmgrain_init_tmpl.c --- a/third_party/dav1d/src/x86/filmgrain_init_tmpl.c +++ b/third_party/dav1d/src/x86/filmgrain_init_tmpl.c @@ -69,15 +69,13 @@ c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I420 - 1] = BF(dav1d_fguv_32x32xn_i420, avx2); c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I422 - 1] = BF(dav1d_fguv_32x32xn_i422, avx2); c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I444 - 1] = BF(dav1d_fguv_32x32xn_i444, avx2); } -#if BITDEPTH == 8 if (!(flags & DAV1D_X86_CPU_FLAG_AVX512ICL)) return; c->fgy_32x32xn = BF(dav1d_fgy_32x32xn, avx512icl); c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I420 - 1] = BF(dav1d_fguv_32x32xn_i420, avx512icl); c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I422 - 1] = BF(dav1d_fguv_32x32xn_i422, avx512icl); c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I444 - 1] = BF(dav1d_fguv_32x32xn_i444, avx512icl); #endif -#endif } diff --git a/third_party/dav1d/tests/checkasm/filmgrain.c b/third_party/dav1d/tests/checkasm/filmgrain.c --- a/third_party/dav1d/tests/checkasm/filmgrain.c +++ b/third_party/dav1d/tests/checkasm/filmgrain.c @@ -81,13 +81,13 @@ report("gen_grain_y"); } static void check_gen_grnuv(const Dav1dFilmGrainDSPContext *const dsp) { - entry grain_lut_y[GRAIN_HEIGHT + 1][GRAIN_WIDTH]; - entry grain_lut_c[GRAIN_HEIGHT][GRAIN_WIDTH]; - entry grain_lut_a[GRAIN_HEIGHT + 1][GRAIN_WIDTH]; + ALIGN_STK_16(entry, grain_lut_y, GRAIN_HEIGHT + 1,[GRAIN_WIDTH]); + ALIGN_STK_16(entry, grain_lut_c, GRAIN_HEIGHT, [GRAIN_WIDTH]); + ALIGN_STK_16(entry, grain_lut_a, GRAIN_HEIGHT + 1,[GRAIN_WIDTH]); declare_func(void, entry grain_lut[][GRAIN_WIDTH], const entry grain_lut_y[][GRAIN_WIDTH], const Dav1dFilmGrainData *data, intptr_t uv HIGHBD_DECL_SUFFIX); @@ -153,20 +153,20 @@ const entry grain_lut[][GRAIN_WIDTH], int bh, int row_num HIGHBD_DECL_SUFFIX); if (check_func(dsp->fgy_32x32xn, "fgy_32x32xn_%dbpc", BITDEPTH)) { ALIGN_STK_16(Dav1dFilmGrainData, fg_data, 16,); + ALIGN_STK_16(entry, grain_lut, GRAIN_HEIGHT + 1,[GRAIN_WIDTH]); ALIGN_STK_64(uint8_t, scaling, SCALING_SIZE,); fg_data[0].seed = rnd() & 0xFFFF; #if BITDEPTH == 16 const int bitdepth_max = rnd() & 1 ? 0x3ff : 0xfff; #else const int bitdepth_max = 0xff; #endif - entry grain_lut[GRAIN_HEIGHT + 1][GRAIN_WIDTH]; fg_data[0].grain_scale_shift = rnd() & 3; fg_data[0].ar_coeff_shift = (rnd() & 3) + 6; fg_data[0].ar_coeff_lag = rnd() & 3; const int num_y_pos = 2 * fg_data[0].ar_coeff_lag * (fg_data[0].ar_coeff_lag + 1); for (int n = 0; n < num_y_pos; n++) @@ -265,10 +265,11 @@ if (check_func(dsp->fguv_32x32xn[layout_idx], "fguv_32x32xn_%dbpc_%s_csfl%d", BITDEPTH, ss_name[layout_idx], csfl)) { ALIGN_STK_16(Dav1dFilmGrainData, fg_data, 1,); + ALIGN_STK_16(entry, grain_lut, 2,[GRAIN_HEIGHT + 1][GRAIN_WIDTH]); ALIGN_STK_64(uint8_t, scaling, SCALING_SIZE,); fg_data[0].seed = rnd() & 0xFFFF; #if BITDEPTH == 16 @@ -277,11 +278,10 @@ const int bitdepth_max = 0xff; #endif const int uv_pl = rnd() & 1; const int is_identity = rnd() & 1; - entry grain_lut[2][GRAIN_HEIGHT + 1][GRAIN_WIDTH]; fg_data[0].grain_scale_shift = rnd() & 3; fg_data[0].ar_coeff_shift = (rnd() & 3) + 6; fg_data[0].ar_coeff_lag = rnd() & 3; const int num_y_pos = 2 * fg_data[0].ar_coeff_lag * (fg_data[0].ar_coeff_lag + 1); for (int n = 0; n < num_y_pos; n++)