diff --git a/media/libdav1d/moz.yaml b/media/libdav1d/moz.yaml --- a/media/libdav1d/moz.yaml +++ b/media/libdav1d/moz.yaml @@ -18,15 +18,15 @@ # Usually different from repository url url: https://code.videolan.org/videolan/dav1d # Human-readable identifier for this version/release # Generally "version NNN", "tag SSS", "bookmark SSS" - release: 4b9f5b704e299543afcea87f375a308b90ef6c70 (2022-11-10T00:58:40.000+00:00). + release: 77b3955537c716acbbdf4bfc46ec2f6b0ccd683a (2023-01-31T15:33:58.000+02:00). # Revision to pull in # Must be a long or short commit SHA (long preferred) - revision: 4b9f5b704e299543afcea87f375a308b90ef6c70 + revision: 77b3955537c716acbbdf4bfc46ec2f6b0ccd683a # The package's license, where possible using the mnemonic from # https://spdx.org/licenses/ # Multiple licenses can be specified (as a YAML list) # A "LICENSE" file must exist containing the full license text diff --git a/media/libdav1d/vcs_version.h b/media/libdav1d/vcs_version.h --- a/media/libdav1d/vcs_version.h +++ b/media/libdav1d/vcs_version.h @@ -1,2 +1,2 @@ /* auto-generated, do not edit */ -#define DAV1D_VERSION "4b9f5b704e299543afcea87f375a308b90ef6c70" +#define DAV1D_VERSION "77b3955537c716acbbdf4bfc46ec2f6b0ccd683a" diff --git a/media/libdav1d/version.h b/media/libdav1d/version.h --- a/media/libdav1d/version.h +++ b/media/libdav1d/version.h @@ -26,9 +26,9 @@ #ifndef DAV1D_VERSION_H #define DAV1D_VERSION_H #define DAV1D_API_VERSION_MAJOR 6 -#define DAV1D_API_VERSION_MINOR 7 +#define DAV1D_API_VERSION_MINOR 8 #define DAV1D_API_VERSION_PATCH 0 #endif /* DAV1D_VERSION_H */ diff --git a/third_party/dav1d/include/dav1d/dav1d.h b/third_party/dav1d/include/dav1d/dav1d.h --- a/third_party/dav1d/include/dav1d/dav1d.h +++ b/third_party/dav1d/include/dav1d/dav1d.h @@ -66,10 +66,17 @@ DAV1D_INLOOPFILTER_ALL = DAV1D_INLOOPFILTER_DEBLOCK | DAV1D_INLOOPFILTER_CDEF | DAV1D_INLOOPFILTER_RESTORATION, }; +enum Dav1dDecodeFrameType { + DAV1D_DECODEFRAMETYPE_ALL = 0, ///< decode and return all frames + DAV1D_DECODEFRAMETYPE_REFERENCE = 1,///< decode and return frames referenced by other frames only + DAV1D_DECODEFRAMETYPE_INTRA = 2, ///< decode and return intra frames only (includes keyframes) + DAV1D_DECODEFRAMETYPE_KEY = 3, ///< decode and return keyframes only +}; + typedef struct Dav1dSettings { int n_threads; ///< number of threads (0 = number of logical cores in host system, default 0) int max_frame_delay; ///< Set to 1 for low-latency decoding (0 = ceil(sqrt(n_threads)), default 0) int apply_grain; ///< whether to apply film grain on output frames (default 1) int operating_point; ///< select an operating point for scalable AV1 bitstreams (0 - 31, default 0) @@ -84,11 +91,13 @@ ///< to all visible frames. Because of show-existing-frame, this ///< means some frames may appear twice (once when coded, ///< once when shown, default 0) enum Dav1dInloopFilterType inloop_filters; ///< postfilters to enable during decoding (default ///< DAV1D_INLOOPFILTER_ALL) - uint8_t reserved[20]; ///< reserved for future use + enum Dav1dDecodeFrameType decode_frame_type; ///< frame types to decode (default + ///< DAV1D_DECODEFRAMETYPE_ALL) + uint8_t reserved[16]; ///< reserved for future use } Dav1dSettings; /** * Get library version. */ diff --git a/third_party/dav1d/meson.build b/third_party/dav1d/meson.build --- a/third_party/dav1d/meson.build +++ b/third_party/dav1d/meson.build @@ -28,11 +28,11 @@ 'warning_level=2', 'buildtype=release', 'b_ndebug=if-release'], meson_version: '>= 0.49.0') -dav1d_soname_version = '6.7.0' +dav1d_soname_version = '6.8.0' dav1d_api_version_array = dav1d_soname_version.split('.') dav1d_api_version_major = dav1d_api_version_array[0] dav1d_api_version_minor = dav1d_api_version_array[1] dav1d_api_version_revision = dav1d_api_version_array[2] @@ -252,10 +252,13 @@ #endif ''' if cc.has_function('pthread_getaffinity_np', prefix : pthread_np_prefix, args : test_args, dependencies : thread_dependency) cdata.set('HAVE_PTHREAD_GETAFFINITY_NP', 1) endif +if cc.has_function('pthread_setaffinity_np', prefix : pthread_np_prefix, args : test_args, dependencies : thread_dependency) + cdata.set('HAVE_PTHREAD_SETAFFINITY_NP', 1) +endif if cc.compiles('int x = _Generic(0, default: 0);', name: '_Generic', args: test_args) cdata.set('HAVE_C11_GENERIC', 1) endif @@ -317,49 +320,21 @@ endif fuzzer_args = ['-fsanitize=fuzzer-no-link', '-fsanitize=fuzzer'] add_project_arguments(cc.first_supported_argument(fuzzer_args), language : 'c') endif -# Stack alignments flags - -stackalign_flag = [] -stackrealign_flag = [] - cdata.set10('ENDIANNESS_BIG', host_machine.endian() == 'big') if host_machine.cpu_family().startswith('x86') if get_option('stack_alignment') > 0 stack_alignment = get_option('stack_alignment') - elif host_machine.cpu_family() == 'x86_64' - if cc.has_argument('-mpreferred-stack-boundary=6') - stackalign_flag = ['-mpreferred-stack-boundary=6'] - stackrealign_flag = ['-mincoming-stack-boundary=4'] - stack_alignment = 32 - elif cc.has_argument('-mstack-alignment=64') - stackalign_flag = ['-mstack-alignment=64'] - stackrealign_flag = ['-mstackrealign'] - stack_alignment = 32 - else - stack_alignment = 16 - endif + elif host_machine.cpu_family() == 'x86_64' or host_machine.system() in ['linux', 'darwin', 'ios', 'tvos'] + stack_alignment = 16 else - if host_machine.system() == 'linux' or host_machine.system() in ['darwin', 'ios', 'tvos'] - stack_alignment = 16 - elif cc.has_argument('-mpreferred-stack-boundary=4') - stackalign_flag = ['-mpreferred-stack-boundary=4'] - stackrealign_flag = ['-mincoming-stack-boundary=2'] - stack_alignment = 16 - elif cc.has_argument('-mstack-alignment=16') - stackalign_flag = ['-mstack-alignment=16'] - stackrealign_flag = ['-mstackrealign'] - stack_alignment = 16 - else - stack_alignment = 4 - endif + stack_alignment = 4 endif cdata_asm.set('STACK_ALIGNMENT', stack_alignment) - cdata.set('STACK_ALIGNMENT', stack_alignment) endif cdata.set10('ARCH_AARCH64', host_machine.cpu_family() == 'aarch64' or host_machine.cpu() == 'arm64') cdata.set10('ARCH_ARM', host_machine.cpu_family().startswith('arm') and host_machine.cpu() != 'arm64') if (is_asm_enabled and diff --git a/third_party/dav1d/src/arm/64/ipred.S b/third_party/dav1d/src/arm/64/ipred.S --- a/third_party/dav1d/src/arm/64/ipred.S +++ b/third_party/dav1d/src/arm/64/ipred.S @@ -1373,10 +1373,1117 @@ .hword L(ipred_smooth_h_tbl) - 160b .hword L(ipred_smooth_h_tbl) - 80b .hword L(ipred_smooth_h_tbl) - 40b endfunc +const padding_mask_buf + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + .byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +padding_mask: + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff +endconst + +// void ipred_z1_upsample_edge_8bpc_neon(pixel *out, const int hsz, +// const pixel *const in, const int end); +function ipred_z1_upsample_edge_8bpc_neon, export=1 + movrel x4, padding_mask + ld1 {v0.16b}, [x2] // in[] + add x5, x2, w3, uxtw // in[end] + sub x4, x4, w3, uxtw + + ld1r {v1.16b}, [x5] // padding + ld1 {v3.16b}, [x4] // padding_mask + + movi v31.8h, #9 + + bit v0.16b, v1.16b, v3.16b // padded in[] + + ext v4.16b, v0.16b, v1.16b, #1 + ext v5.16b, v0.16b, v1.16b, #2 + ext v6.16b, v0.16b, v1.16b, #3 + + uaddl v16.8h, v4.8b, v5.8b // in[i+1] + in[i+2] + uaddl2 v17.8h, v4.16b, v5.16b + uaddl v18.8h, v0.8b, v6.8b // in[i+0] + in[i+3] + uaddl2 v19.8h, v0.16b, v6.16b + mul v16.8h, v16.8h, v31.8h // 9*(in[i+1] + in[i+2]) + mul v17.8h, v17.8h, v31.8h + sub v16.8h, v16.8h, v18.8h + sub v17.8h, v17.8h, v19.8h + + sqrshrun v16.8b, v16.8h, #4 + sqrshrun2 v16.16b, v17.8h, #4 + + zip1 v0.16b, v4.16b, v16.16b + zip2 v1.16b, v4.16b, v16.16b + + st1 {v0.16b, v1.16b}, [x0] + + ret +endfunc + +const edge_filter + .byte 0, 4, 8, 0 + .byte 0, 5, 6, 0 +// Leaving out the coeffs for strength=3 +// .byte 2, 4, 4, 0 +endconst + +// void ipred_z1_filter_edge_8bpc_neon(pixel *out, const int sz, +// const pixel *const in, const int end, +// const int strength); +function ipred_z1_filter_edge_8bpc_neon, export=1 + cmp w4, #3 + b.eq L(fivetap) // if (strength == 3) goto fivetap + + movrel x5, edge_filter, -3 + add x5, x5, w4, uxtw #2 // edge_filter + (strength - 1)*4 + 1 + + ld1 {v31.h}[0], [x5] // kernel[1-2] + + ld1 {v0.16b}, [x2], #16 + + dup v30.16b, v31.b[0] + dup v31.16b, v31.b[1] +1: + // in[end], is the last valid pixel. We produce 16 pixels out by + // using 18 pixels in - the last pixel used is [17] of the ones + // read/buffered. + cmp w3, #17 + ld1 {v1.16b}, [x2], #16 + b.lt 2f + ext v2.16b, v0.16b, v1.16b, #1 + ext v3.16b, v0.16b, v1.16b, #2 + umull v4.8h, v0.8b, v30.8b + umlal v4.8h, v2.8b, v31.8b + umlal v4.8h, v3.8b, v30.8b + umull2 v5.8h, v0.16b, v30.16b + umlal2 v5.8h, v2.16b, v31.16b + umlal2 v5.8h, v3.16b, v30.16b + subs w1, w1, #16 + mov v0.16b, v1.16b + rshrn v4.8b, v4.8h, #4 + rshrn2 v4.16b, v5.8h, #4 + sub w3, w3, #16 + st1 {v4.16b}, [x0], #16 + b.gt 1b + ret +2: + // Right padding + + // x2[w3-32] is the padding pixel (x2 points 32 bytes ahead) + movrel x5, padding_mask + sub w6, w3, #32 + sub x5, x5, w3, uxtw + add x6, x2, w6, sxtw + + ld1 {v2.16b, v3.16b}, [x5] // padding_mask + + ld1r {v4.16b}, [x6] + bit v0.16b, v4.16b, v2.16b // Pad v0-v1 + bit v1.16b, v4.16b, v3.16b + + // Filter one block + ext v2.16b, v0.16b, v1.16b, #1 + ext v3.16b, v0.16b, v1.16b, #2 + umull v4.8h, v0.8b, v30.8b + umlal v4.8h, v2.8b, v31.8b + umlal v4.8h, v3.8b, v30.8b + umull2 v5.8h, v0.16b, v30.16b + umlal2 v5.8h, v2.16b, v31.16b + umlal2 v5.8h, v3.16b, v30.16b + subs w1, w1, #16 + rshrn v4.8b, v4.8h, #4 + rshrn2 v4.16b, v5.8h, #4 + st1 {v4.16b}, [x0], #16 + b.le 9f +5: + // After one block, any remaining output would only be filtering + // padding - thus just store the padding. + subs w1, w1, #16 + st1 {v1.16b}, [x0], #16 + b.gt 5b +9: + ret + +L(fivetap): + sub x2, x2, #1 // topleft -= 1 + movi v29.16b, #2 + ld1 {v0.16b}, [x2], #16 + movi v30.16b, #4 + movi v31.16b, #4 + ins v0.b[0], v0.b[1] +1: + // in[end+1], is the last valid pixel. We produce 16 pixels out by + // using 20 pixels in - the last pixel used is [19] of the ones + // read/buffered. + cmp w3, #18 + ld1 {v1.16b}, [x2], #16 + b.lt 2f // if (end + 1 < 19) + ext v2.16b, v0.16b, v1.16b, #1 + ext v3.16b, v0.16b, v1.16b, #2 + ext v4.16b, v0.16b, v1.16b, #3 + ext v5.16b, v0.16b, v1.16b, #4 + umull v6.8h, v0.8b, v29.8b + umlal v6.8h, v2.8b, v30.8b + umlal v6.8h, v3.8b, v31.8b + umlal v6.8h, v4.8b, v30.8b + umlal v6.8h, v5.8b, v29.8b + umull2 v7.8h, v0.16b, v29.16b + umlal2 v7.8h, v2.16b, v30.16b + umlal2 v7.8h, v3.16b, v31.16b + umlal2 v7.8h, v4.16b, v30.16b + umlal2 v7.8h, v5.16b, v29.16b + subs w1, w1, #16 + mov v0.16b, v1.16b + rshrn v6.8b, v6.8h, #4 + rshrn2 v6.16b, v7.8h, #4 + sub w3, w3, #16 + st1 {v6.16b}, [x0], #16 + b.gt 1b + ret +2: + // Right padding + + // x2[w3+1-32] is the padding pixel (x2 points 32 bytes ahead) + movrel x5, padding_mask, -1 + sub w6, w3, #31 + sub x5, x5, w3, uxtw + add x6, x2, w6, sxtw + + ld1 {v2.16b, v3.16b}, [x5] // padding_mask + + ld1r {v28.16b}, [x6] + bit v0.16b, v28.16b, v2.16b // Pad v0-v1 + bit v1.16b, v28.16b, v3.16b +4: + // Filter one block + ext v2.16b, v0.16b, v1.16b, #1 + ext v3.16b, v0.16b, v1.16b, #2 + ext v4.16b, v0.16b, v1.16b, #3 + ext v5.16b, v0.16b, v1.16b, #4 + umull v6.8h, v0.8b, v29.8b + umlal v6.8h, v2.8b, v30.8b + umlal v6.8h, v3.8b, v31.8b + umlal v6.8h, v4.8b, v30.8b + umlal v6.8h, v5.8b, v29.8b + umull2 v7.8h, v0.16b, v29.16b + umlal2 v7.8h, v2.16b, v30.16b + umlal2 v7.8h, v3.16b, v31.16b + umlal2 v7.8h, v4.16b, v30.16b + umlal2 v7.8h, v5.16b, v29.16b + subs w1, w1, #16 + mov v0.16b, v1.16b + mov v1.16b, v28.16b + rshrn v6.8b, v6.8h, #4 + rshrn2 v6.16b, v7.8h, #4 + sub w3, w3, #16 + st1 {v6.16b}, [x0], #16 + b.le 9f + // v0-v1[w3+1] is the last valid pixel; if (w3 + 1 > 0) we need to + // filter properly once more - aka (w3 >= 0). + cmp w3, #0 + b.ge 4b +5: + // When w3 <= 0, all remaining pixels in v0-v1 are equal to the + // last valid pixel - thus just output that without filtering. + subs w1, w1, #16 + st1 {v1.16b}, [x0], #16 + b.gt 5b +9: + ret +endfunc + +// void ipred_z1_fill1_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const top, +// const int width, const int height, +// const int dx, const int max_base_x); +function ipred_z1_fill1_8bpc_neon, export=1 + clz w9, w3 + adr x8, L(ipred_z1_fill1_tbl) + sub w9, w9, #25 + ldrh w9, [x8, w9, uxtw #1] + add x10, x2, w6, uxtw // top[max_base_x] + sub x8, x8, w9, uxtw + ld1r {v31.16b}, [x10] // padding + mov w7, w5 + mov w15, #64 + br x8 +40: + AARCH64_VALID_JUMP_TARGET +4: + lsr w8, w7, #6 // base + and w9, w7, #0x3e // frac + add w7, w7, w5 // xpos += dx + cmp w8, w6 // base >= max_base_x + lsr w10, w7, #6 // base + and w11, w7, #0x3e // frac + b.ge 49f + ldr d0, [x2, w8, uxtw] // top[base] + ldr d2, [x2, w10, uxtw] + dup v4.4h, w9 // frac + dup v5.4h, w11 + ext v1.8b, v0.8b, v0.8b, #1 // top[base+1] + ext v3.8b, v2.8b, v2.8b, #1 + usubl v6.8h, v1.8b, v0.8b // top[base+1]-top[base] + usubl v7.8h, v3.8b, v2.8b + ushll v16.8h, v0.8b, #6 // top[base]*64 + ushll v17.8h, v2.8b, #6 + mla v16.4h, v6.4h, v4.4h // + top[base+1]*frac + mla v17.4h, v7.4h, v5.4h + rshrn v16.8b, v16.8h, #6 + rshrn v17.8b, v17.8h, #6 + st1 {v16.s}[0], [x0], x1 + add w7, w7, w5 // xpos += dx + subs w4, w4, #2 + st1 {v17.s}[0], [x0], x1 + b.gt 4b + ret + +49: + st1 {v31.s}[0], [x0], x1 + subs w4, w4, #2 + st1 {v31.s}[0], [x0], x1 + b.gt 49b + ret + +80: + AARCH64_VALID_JUMP_TARGET +8: + lsr w8, w7, #6 // base + and w9, w7, #0x3e // frac + add w7, w7, w5 // xpos += dx + cmp w8, w6 // base >= max_base_x + lsr w10, w7, #6 // base + and w11, w7, #0x3e // frac + b.ge 89f + ldr q0, [x2, w8, uxtw] // top[base] + ldr q2, [x2, w10, uxtw] + dup v4.8b, w9 // frac + dup v5.8b, w11 + sub w9, w15, w9 // 64 - frac + sub w11, w15, w11 + dup v6.8b, w9 // 64 - frac + dup v7.8b, w11 + ext v1.16b, v0.16b, v0.16b, #1 // top[base+1] + ext v3.16b, v2.16b, v2.16b, #1 + umull v16.8h, v1.8b, v4.8b // top[base+1]*frac + umlal v16.8h, v0.8b, v6.8b // + top[base]*(64-frac) + umull v17.8h, v3.8b, v5.8b + umlal v17.8h, v2.8b, v7.8b + rshrn v16.8b, v16.8h, #6 + rshrn v17.8b, v17.8h, #6 + st1 {v16.8b}, [x0], x1 + add w7, w7, w5 // xpos += dx + subs w4, w4, #2 + st1 {v17.8b}, [x0], x1 + b.gt 8b + ret + +89: + st1 {v31.8b}, [x0], x1 + subs w4, w4, #2 + st1 {v31.8b}, [x0], x1 + b.gt 89b + ret + +160: +320: +640: + AARCH64_VALID_JUMP_TARGET + + mov w12, w3 + + add x13, x0, x1 + lsl x1, x1, #1 + sub x1, x1, w3, uxtw +1: + lsr w8, w7, #6 // base + and w9, w7, #0x3e // frac + add w7, w7, w5 // xpos += dx + cmp w8, w6 // base >= max_base_x + lsr w10, w7, #6 // base + and w11, w7, #0x3e // frac + b.ge 169f + add x8, x2, w8, uxtw + add x10, x2, w10, uxtw + dup v4.16b, w9 // frac + dup v5.16b, w11 + ld1 {v0.16b, v1.16b}, [x8], #32 // top[base] + ld1 {v2.16b, v3.16b}, [x10], #32 + sub w9, w15, w9 // 64 - frac + sub w11, w15, w11 + dup v6.16b, w9 // 64 - frac + dup v7.16b, w11 + add w7, w7, w5 // xpos += dx +2: + ext v16.16b, v0.16b, v1.16b, #1 // top[base+1] + ext v17.16b, v2.16b, v3.16b, #1 + subs w3, w3, #16 + umull v18.8h, v16.8b, v4.8b // top[base+1]*frac + umlal v18.8h, v0.8b, v6.8b // + top[base]*(64-frac) + umull2 v19.8h, v16.16b, v4.16b + umlal2 v19.8h, v0.16b, v6.16b + umull v20.8h, v17.8b, v5.8b + umlal v20.8h, v2.8b, v7.8b + umull2 v21.8h, v17.16b, v5.16b + umlal2 v21.8h, v2.16b, v7.16b + rshrn v16.8b, v18.8h, #6 + rshrn2 v16.16b, v19.8h, #6 + rshrn v17.8b, v20.8h, #6 + rshrn2 v17.16b, v21.8h, #6 + st1 {v16.16b}, [x0], #16 + st1 {v17.16b}, [x13], #16 + b.le 3f + mov v0.16b, v1.16b + ld1 {v1.16b}, [x8], #16 // top[base] + mov v2.16b, v3.16b + ld1 {v3.16b}, [x10], #16 + b 2b + +3: + subs w4, w4, #2 + b.le 9f + add x0, x0, x1 + add x13, x13, x1 + mov w3, w12 + b 1b +9: + ret + +169: + st1 {v31.16b}, [x0], #16 + subs w3, w3, #16 + st1 {v31.16b}, [x13], #16 + b.gt 169b + subs w4, w4, #2 + b.le 9b + add x0, x0, x1 + add x13, x13, x1 + mov w3, w12 + b 169b + +L(ipred_z1_fill1_tbl): + .hword L(ipred_z1_fill1_tbl) - 640b + .hword L(ipred_z1_fill1_tbl) - 320b + .hword L(ipred_z1_fill1_tbl) - 160b + .hword L(ipred_z1_fill1_tbl) - 80b + .hword L(ipred_z1_fill1_tbl) - 40b +endfunc + +function ipred_z1_fill2_8bpc_neon, export=1 + cmp w3, #8 + add x10, x2, w6, uxtw // top[max_base_x] + ld1r {v31.16b}, [x10] // padding + mov w7, w5 + mov w15, #64 + b.eq 8f + +4: // w == 4 + lsr w8, w7, #6 // base + and w9, w7, #0x3e // frac + add w7, w7, w5 // xpos += dx + cmp w8, w6 // base >= max_base_x + lsr w10, w7, #6 // base + and w11, w7, #0x3e // frac + b.ge 49f + ldr d0, [x2, w8, uxtw] // top[base] + ldr d2, [x2, w10, uxtw] + dup v4.4h, w9 // frac + dup v5.4h, w11 + uzp2 v1.8b, v0.8b, v0.8b // top[base+1] + uzp1 v0.8b, v0.8b, v0.8b // top[base] + uzp2 v3.8b, v2.8b, v2.8b + uzp1 v2.8b, v2.8b, v2.8b + usubl v6.8h, v1.8b, v0.8b // top[base+1]-top[base] + usubl v7.8h, v3.8b, v2.8b + ushll v16.8h, v0.8b, #6 // top[base]*64 + ushll v17.8h, v2.8b, #6 + mla v16.4h, v6.4h, v4.4h // + top[base+1]*frac + mla v17.4h, v7.4h, v5.4h + rshrn v16.8b, v16.8h, #6 + rshrn v17.8b, v17.8h, #6 + st1 {v16.s}[0], [x0], x1 + add w7, w7, w5 // xpos += dx + subs w4, w4, #2 + st1 {v17.s}[0], [x0], x1 + b.gt 4b + ret + +49: + st1 {v31.s}[0], [x0], x1 + subs w4, w4, #2 + st1 {v31.s}[0], [x0], x1 + b.gt 49b + ret + +8: // w == 8 + lsr w8, w7, #6 // base + and w9, w7, #0x3e // frac + add w7, w7, w5 // xpos += dx + cmp w8, w6 // base >= max_base_x + lsr w10, w7, #6 // base + and w11, w7, #0x3e // frac + b.ge 89f + ldr q0, [x2, w8, uxtw] // top[base] + ldr q2, [x2, w10, uxtw] + dup v4.8b, w9 // frac + dup v5.8b, w11 + sub w9, w15, w9 // 64 - frac + sub w11, w15, w11 + dup v6.8b, w9 // 64 - frac + dup v7.8b, w11 + uzp2 v1.16b, v0.16b, v0.16b // top[base+1] + uzp1 v0.16b, v0.16b, v0.16b // top[base] + uzp2 v3.16b, v2.16b, v2.16b + uzp1 v2.16b, v2.16b, v2.16b + umull v16.8h, v1.8b, v4.8b // top[base+1]*frac + umlal v16.8h, v0.8b, v6.8b // + top[base]*(64-frac) + umull v17.8h, v3.8b, v5.8b + umlal v17.8h, v2.8b, v7.8b + rshrn v16.8b, v16.8h, #6 + rshrn v17.8b, v17.8h, #6 + st1 {v16.8b}, [x0], x1 + add w7, w7, w5 // xpos += dx + subs w4, w4, #2 + st1 {v17.8b}, [x0], x1 + b.gt 8b + ret + +89: + st1 {v31.8b}, [x0], x1 + subs w4, w4, #2 + st1 {v31.8b}, [x0], x1 + b.gt 89b + ret +endfunc + +// void ipred_reverse_8bpc_neon(pixel *dst, const pixel *const src, +// const int n); +function ipred_reverse_8bpc_neon, export=1 + sub x1, x1, #16 + add x3, x0, #8 + mov x4, #16 +1: + ld1 {v0.16b}, [x1] + subs w2, w2, #16 + rev64 v0.16b, v0.16b + sub x1, x1, #16 + st1 {v0.d}[1], [x0], x4 + st1 {v0.d}[0], [x3], x4 + b.gt 1b + ret +endfunc + +const increments + .short 0, 1, 2, 3, 4, 5, 6, 7 +endconst + +// void ipred_z3_fill1_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const left, +// const int width, const int height, +// const int dy, const int max_base_y); +function ipred_z3_fill1_8bpc_neon, export=1 + cmp w6, #64 + clz w9, w3 + adr x8, L(ipred_z3_fill1_tbl) + sub w9, w9, #25 + ldrh w9, [x8, w9, uxtw #1] + add x10, x2, w6, uxtw // left[max_base_y] + sub x8, x8, w9, uxtw + movrel x11, increments + ld1r {v31.16b}, [x10] // padding + ld1 {v30.8h}, [x11] // increments + mov w7, w5 + b.gt L(ipred_z3_fill1_large_w16) + br x8 + +40: + AARCH64_VALID_JUMP_TARGET + dup v29.4h, w5 // dy + + mul v30.4h, v30.4h, v29.4h // {0,1,2,3,4,5,6,7}*dy + movi v23.16b, #0x3e + + ld1 {v0.16b, v1.16b}, [x2] // left[] + add v30.4h, v29.4h, v30.4h // ypos + + movi v22.16b, #64 + movi v20.16b, #1 + movi v21.16b, #2 + + xtn v24.8b, v30.8h // (uint8_t)ypos + uqshrn v26.8b, v30.8h, #6 // base + and v24.8b, v24.8b, v23.8b // frac + + mov v4.8b, v31.8b + uqadd v27.8b, v26.8b, v20.8b // base + 1 + uqadd v28.8b, v26.8b, v21.8b // base + 2 + sub v25.8b, v22.8b, v24.8b // 64 - frac + + tbx v4.8b, {v0.16b, v1.16b}, v26.8b // left[base] + + trn1 v27.2s, v27.2s, v28.2s // base + 1, base + 2 + trn1 v24.2s, v24.2s, v24.2s // frac + trn1 v25.2s, v25.2s, v25.2s // 64 - frac +1: + mov v5.8b, v31.8b + tbx v5.8b, {v0.16b, v1.16b}, v27.8b // left[base+1], left[base+2] + + trn1 v4.2s, v4.2s, v5.2s // left[base], left[base+1] + + umull v16.8h, v4.8b, v25.8b // left[base]*(64-frac) + umlal v16.8h, v5.8b, v24.8b // + left[base+1]*frac + rshrn v16.8b, v16.8h, #6 + st1 {v16.s}[0], [x0], x1 + subs w4, w4, #2 + st1 {v16.s}[1], [x0], x1 + b.le 9f + + ext v4.8b, v5.8b, v5.8b, #4 + uqadd v27.8b, v27.8b, v21.8b // base += 2 + b 1b + +9: + ret + +80: + AARCH64_VALID_JUMP_TARGET + dup v29.8h, w5 // dy + + mul v30.8h, v30.8h, v29.8h // {0,1,2,3,4,5,6,7}*dy + movi v23.16b, #0x3e + + ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2] // left[] + add v30.8h, v29.8h, v30.8h // ypos + + movi v22.16b, #64 + movi v20.16b, #1 + movi v21.16b, #2 + + xtn v24.8b, v30.8h // (uint8_t)ypos + uqshrn v26.8b, v30.8h, #6 // base + and v24.8b, v24.8b, v23.8b // frac + + mov v4.8b, v31.8b + uqadd v27.8b, v26.8b, v20.8b // base + 1 + uqadd v28.8b, v26.8b, v21.8b // base + 2 + sub v25.8b, v22.8b, v24.8b // 64 - frac + + tbx v4.8b, {v0.16b, v1.16b, v2.16b, v3.16b}, v26.8b // left[base] +1: + mov v5.8b, v31.8b + mov v6.8b, v31.8b + tbx v5.8b, {v0.16b, v1.16b, v2.16b, v3.16b}, v27.8b // left[base+1] + tbx v6.8b, {v0.16b, v1.16b, v2.16b, v3.16b}, v28.8b // left[base+2] + + umull v16.8h, v4.8b, v25.8b // left[base]*(64-frac) + umlal v16.8h, v5.8b, v24.8b // + left[base+1]*frac + umull v17.8h, v5.8b, v25.8b + umlal v17.8h, v6.8b, v24.8b + rshrn v16.8b, v16.8h, #6 + rshrn v17.8b, v17.8h, #6 + st1 {v16.8b}, [x0], x1 + subs w4, w4, #2 + st1 {v17.8b}, [x0], x1 + b.le 9f + + mov v4.8b, v6.8b + uqadd v27.8b, v27.8b, v21.8b // base += 2 + uqadd v28.8b, v28.8b, v21.8b // base += 2 + b 1b + +9: + ret + +160: + AARCH64_VALID_JUMP_TARGET + dup v28.8h, w5 // dy + + shl v29.8h, v28.8h, #3 // 8*dy + mul v30.8h, v30.8h, v28.8h // {0,1,2,3,4,5,6,7}*dy + movi v23.16b, #0x3e + + ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2] // left[] + add v28.8h, v28.8h, v30.8h // ypos + + movi v22.16b, #64 + movi v20.16b, #1 + movi v21.16b, #2 + + add v29.8h, v28.8h, v29.8h // ypos + 8*dy + + xtn v24.8b, v28.8h // (uint8_t)ypos + xtn2 v24.16b, v29.8h + uqshrn v26.8b, v28.8h, #6 // base + uqshrn2 v26.16b, v29.8h, #6 + and v24.16b, v24.16b, v23.16b // frac + + mov v4.16b, v31.16b + uqadd v27.16b, v26.16b, v20.16b // base + 1 + uqadd v28.16b, v26.16b, v21.16b // base + 2 + sub v25.16b, v22.16b, v24.16b // 64 - frac + + tbx v4.16b, {v0.16b, v1.16b, v2.16b, v3.16b}, v26.16b // left[base] +1: + mov v5.16b, v31.16b + mov v6.16b, v31.16b + tbx v5.16b, {v0.16b, v1.16b, v2.16b, v3.16b}, v27.16b // left[base+1] + tbx v6.16b, {v0.16b, v1.16b, v2.16b, v3.16b}, v28.16b // left[base+2] + + umull v16.8h, v4.8b, v25.8b // left[base]*(64-frac) + umlal v16.8h, v5.8b, v24.8b // + left[base+1]*frac + umull2 v17.8h, v4.16b, v25.16b + umlal2 v17.8h, v5.16b, v24.16b + umull v18.8h, v5.8b, v25.8b + umlal v18.8h, v6.8b, v24.8b + umull2 v19.8h, v5.16b, v25.16b + umlal2 v19.8h, v6.16b, v24.16b + rshrn v16.8b, v16.8h, #6 + rshrn2 v16.16b, v17.8h, #6 + rshrn v17.8b, v18.8h, #6 + rshrn2 v17.16b, v19.8h, #6 + st1 {v16.16b}, [x0], x1 + subs w4, w4, #2 + st1 {v17.16b}, [x0], x1 + b.le 9f + + mov v4.16b, v6.16b + uqadd v27.16b, v27.16b, v21.16b // base += 2 + uqadd v28.16b, v28.16b, v21.16b // base += 2 + b 1b + +9: + ret +320: +640: + AARCH64_VALID_JUMP_TARGET + dup v28.8h, w5 // dy + mov w12, w3 + + add x13, x0, x1 + + shl v29.8h, v28.8h, #3 // 8*dy + mul v30.8h, v30.8h, v28.8h // {0,1,2,3,4,5,6,7}*dy + movi v23.16b, #0x3e + + lsl x1, x1, #1 + sub x1, x1, w3, uxtw + add v30.8h, v28.8h, v30.8h // ypos + + ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x2] // left[] + + movi v22.16b, #64 + movi v20.16b, #1 + movi v21.16b, #2 + +1: + mov v26.16b, v30.16b // reset ypos + +2: + add v27.8h, v26.8h, v29.8h // ypos + 8*dy + uqshrn v16.8b, v26.8h, #6 // base + uqshrn2 v16.16b, v27.8h, #6 + xtn v24.8b, v26.8h // (uint8_t)ypos + xtn2 v24.16b, v27.8h + umov w14, v16.b[0] + and v24.16b, v24.16b, v23.16b // frac + + uqadd v17.16b, v16.16b, v20.16b // base + 1 + cmp w14, w6 // base >= max_base_y + uqadd v18.16b, v16.16b, v21.16b // base + 2 + sub v25.16b, v22.16b, v24.16b // 64 - frac + + b.ge 4f + + mov v4.16b, v31.16b + mov v5.16b, v31.16b + mov v6.16b, v31.16b + tbx v4.16b, {v0.16b, v1.16b, v2.16b, v3.16b}, v16.16b // left[base] + tbx v5.16b, {v0.16b, v1.16b, v2.16b, v3.16b}, v17.16b // left[base+1] + tbx v6.16b, {v0.16b, v1.16b, v2.16b, v3.16b}, v18.16b // left[base+2] + + subs w3, w3, #16 + umull v16.8h, v4.8b, v25.8b // left[base]*(64-frac) + umlal v16.8h, v5.8b, v24.8b // + left[base+1]*frac + umull2 v17.8h, v4.16b, v25.16b + umlal2 v17.8h, v5.16b, v24.16b + umull v18.8h, v5.8b, v25.8b + umlal v18.8h, v6.8b, v24.8b + umull2 v19.8h, v5.16b, v25.16b + umlal2 v19.8h, v6.16b, v24.16b + rshrn v16.8b, v16.8h, #6 + rshrn2 v16.16b, v17.8h, #6 + rshrn v17.8b, v18.8h, #6 + rshrn2 v17.16b, v19.8h, #6 + st1 {v16.16b}, [x0], #16 + st1 {v17.16b}, [x13], #16 + b.le 3f + add v26.8h, v27.8h, v29.8h // ypos += 16*dy + b 2b + +3: + subs w4, w4, #2 + b.le 9f + movi v16.8h, #128 + add x0, x0, x1 + add x13, x13, x1 + add v30.8h, v30.8h, v16.8h // ypos = dy + y*(1<<6)*2 + mov w3, w12 + b 1b + +4: + subs w3, w3, #16 + st1 {v31.16b}, [x0], #16 + st1 {v31.16b}, [x13], #16 + b.gt 4b + b 3b + +9: + ret + +L(ipred_z3_fill1_large_w16): + // Fallback case for max_base_y > 64; similar to the z1 + // implementation. This does the filtering vertically, filling out + // a 2x pixel column at a time. + mov w15, #64 + add x13, x0, x1 + lsl x1, x1, #1 + + mov w12, w4 +1: + lsr w8, w7, #6 // base + and w9, w7, #0x3e // frac + add w7, w7, w5 // ypos += dy + cmp w8, w6 // base >= max_base_y + lsr w10, w7, #6 // base + and w11, w7, #0x3e // frac + b.ge ipred_z3_fill_padding_neon + add x8, x2, w8, uxtw + add x10, x2, w10, uxtw + dup v4.16b, w9 // frac + dup v5.16b, w11 + ld1 {v0.16b, v1.16b}, [x8], #32 // left[base] + ld1 {v2.16b, v3.16b}, [x10], #32 + sub w9, w15, w9 // 64 - frac + sub w11, w15, w11 + dup v6.16b, w9 // 64 - frac + dup v7.16b, w11 + add w7, w7, w5 // ypos += dy +2: + ext v16.16b, v0.16b, v1.16b, #1 // left[base+1] + ext v17.16b, v2.16b, v3.16b, #1 + subs w4, w4, #16 + umull v18.8h, v16.8b, v4.8b // left[base+1]*frac + umlal v18.8h, v0.8b, v6.8b // + left[base]*(64-frac) + umull2 v19.8h, v16.16b, v4.16b + umlal2 v19.8h, v0.16b, v6.16b + umull v20.8h, v17.8b, v5.8b + umlal v20.8h, v2.8b, v7.8b + umull2 v21.8h, v17.16b, v5.16b + umlal2 v21.8h, v2.16b, v7.16b + rshrn v16.8b, v18.8h, #6 + rshrn2 v16.16b, v19.8h, #6 + rshrn v17.8b, v20.8h, #6 + rshrn2 v17.16b, v21.8h, #6 + zip1 v18.16b, v16.16b, v17.16b + zip2 v19.16b, v16.16b, v17.16b + st1 {v18.h}[0], [x0], x1 + st1 {v18.h}[1], [x13], x1 + st1 {v18.h}[2], [x0], x1 + st1 {v18.h}[3], [x13], x1 + st1 {v18.h}[4], [x0], x1 + st1 {v18.h}[5], [x13], x1 + st1 {v18.h}[6], [x0], x1 + st1 {v18.h}[7], [x13], x1 + st1 {v19.h}[0], [x0], x1 + st1 {v19.h}[1], [x13], x1 + st1 {v19.h}[2], [x0], x1 + st1 {v19.h}[3], [x13], x1 + st1 {v19.h}[4], [x0], x1 + st1 {v19.h}[5], [x13], x1 + st1 {v19.h}[6], [x0], x1 + st1 {v19.h}[7], [x13], x1 + b.le 3f + mov v0.16b, v1.16b + ld1 {v1.16b}, [x8], #16 // left[base] + mov v2.16b, v3.16b + ld1 {v3.16b}, [x10], #16 + b 2b + +3: + subs w3, w3, #2 + b.le 9f + lsr x1, x1, #1 + msub x0, x1, x12, x0 // ptr -= h * stride + msub x13, x1, x12, x13 + lsl x1, x1, #1 + add x0, x0, #2 + add x13, x13, #2 + mov w4, w12 + b 1b +9: + ret + +L(ipred_z3_fill1_tbl): + .hword L(ipred_z3_fill1_tbl) - 640b + .hword L(ipred_z3_fill1_tbl) - 320b + .hword L(ipred_z3_fill1_tbl) - 160b + .hword L(ipred_z3_fill1_tbl) - 80b + .hword L(ipred_z3_fill1_tbl) - 40b +endfunc + +function ipred_z3_fill_padding_neon, export=0 + cmp w3, #16 + adr x8, L(ipred_z3_fill_padding_tbl) + b.gt L(ipred_z3_fill_padding_wide) + // w3 = remaining width, w4 = constant height + mov w12, w4 + +1: + // Fill a WxH rectangle with padding. W can be any number; + // this fills the exact width by filling in the largest + // power of two in the remaining width, and repeating. + clz w9, w3 + sub w9, w9, #25 + ldrh w9, [x8, w9, uxtw #1] + sub x9, x8, w9, uxtw + br x9 + +2: + st1 {v31.h}[0], [x0], x1 + subs w4, w4, #4 + st1 {v31.h}[0], [x13], x1 + st1 {v31.h}[0], [x0], x1 + st1 {v31.h}[0], [x13], x1 + b.gt 2b + subs w3, w3, #2 + lsr x1, x1, #1 + msub x0, x1, x12, x0 // ptr -= h * stride + msub x13, x1, x12, x13 + b.le 9f + lsl x1, x1, #1 + add x0, x0, #2 + add x13, x13, #2 + mov w4, w12 + b 1b + +4: + st1 {v31.s}[0], [x0], x1 + subs w4, w4, #4 + st1 {v31.s}[0], [x13], x1 + st1 {v31.s}[0], [x0], x1 + st1 {v31.s}[0], [x13], x1 + b.gt 4b + subs w3, w3, #4 + lsr x1, x1, #1 + msub x0, x1, x12, x0 // ptr -= h * stride + msub x13, x1, x12, x13 + b.le 9f + lsl x1, x1, #1 + add x0, x0, #4 + add x13, x13, #4 + mov w4, w12 + b 1b + +8: + st1 {v31.8b}, [x0], x1 + subs w4, w4, #4 + st1 {v31.8b}, [x13], x1 + st1 {v31.8b}, [x0], x1 + st1 {v31.8b}, [x13], x1 + b.gt 4b + subs w3, w3, #8 + lsr x1, x1, #1 + msub x0, x1, x12, x0 // ptr -= h * stride + msub x13, x1, x12, x13 + b.le 9f + lsl x1, x1, #1 + add x0, x0, #8 + add x13, x13, #8 + mov w4, w12 + b 1b + +16: +32: +64: + st1 {v31.16b}, [x0], x1 + subs w4, w4, #4 + st1 {v31.16b}, [x13], x1 + st1 {v31.16b}, [x0], x1 + st1 {v31.16b}, [x13], x1 + b.gt 4b + subs w3, w3, #16 + lsr x1, x1, #1 + msub x0, x1, x12, x0 // ptr -= h * stride + msub x13, x1, x12, x13 + b.le 9f + lsl x1, x1, #1 + add x0, x0, #16 + add x13, x13, #16 + mov w4, w12 + b 1b + +9: + ret + +L(ipred_z3_fill_padding_tbl): + .hword L(ipred_z3_fill_padding_tbl) - 64b + .hword L(ipred_z3_fill_padding_tbl) - 32b + .hword L(ipred_z3_fill_padding_tbl) - 16b + .hword L(ipred_z3_fill_padding_tbl) - 8b + .hword L(ipred_z3_fill_padding_tbl) - 4b + .hword L(ipred_z3_fill_padding_tbl) - 2b + +L(ipred_z3_fill_padding_wide): + // Fill a WxH rectangle with padding, with W > 16. + lsr x1, x1, #1 + mov w12, w3 + sub x1, x1, w3, uxtw +1: + ands w5, w3, #15 + b.eq 2f + // If the width isn't aligned to 16, first do one 16 byte write + // and align the start pointer. + sub w3, w3, w5 + st1 {v31.16b}, [x0] + add x0, x0, w5, uxtw +2: + // Fill the rest of the line with aligned 16 byte writes. + subs w3, w3, #16 + st1 {v31.16b}, [x0], #16 + b.gt 2b + subs w4, w4, #1 + add x0, x0, x1 + b.le 9f + mov w3, w12 + b 1b +9: + ret +endfunc + +function ipred_z3_fill2_8bpc_neon, export=1 + adr x8, L(ipred_z3_fill1_tbl) + add x10, x2, w6, uxtw // left[max_base_y] + movrel x11, increments + ld1r {v31.16b}, [x10] // padding + ld1 {v30.8h}, [x11] // increments + mov w7, w5 + + cmp w3, #8 + add x10, x2, w6, uxtw // left[max_base_y] + ld1r {v31.16b}, [x10] // padding + b.eq 80f + +40: // w == 4 + dup v29.4h, w5 // dy + + mul v30.4h, v30.4h, v29.4h // {0,1,2,3,4,5,6,7}*dy + movi v23.16b, #0x3e + + ld1 {v0.16b, v1.16b}, [x2] // left[] + add v30.4h, v29.4h, v30.4h // ypos + + movi v22.16b, #64 + movi v20.16b, #1 + movi v21.16b, #2 + + xtn v24.8b, v30.8h // (uint8_t)ypos + uqshrn v26.8b, v30.8h, #6 // base + and v24.8b, v24.8b, v23.8b // frac + + uqadd v27.8b, v26.8b, v20.8b // base + 1 + uqadd v28.8b, v26.8b, v21.8b // base + 2 + sub v25.8b, v22.8b, v24.8b // 64 - frac + uqadd v29.8b, v27.8b, v21.8b // base + 3 + + trn1 v24.2s, v24.2s, v24.2s // frac + trn1 v26.2s, v26.2s, v28.2s // base + 0, base + 2 + trn1 v27.2s, v27.2s, v29.2s // base + 1, base + 3 + trn1 v25.2s, v25.2s, v25.2s // 64 - frac + + movi v21.16b, #4 +1: + mov v4.8b, v31.8b + mov v5.8b, v31.8b + tbx v4.8b, {v0.16b, v1.16b}, v26.8b // left[base], left[base+2] + tbx v5.8b, {v0.16b, v1.16b}, v27.8b // left[base+1], left[base+3] + + umull v16.8h, v4.8b, v25.8b // left[base]*(64-frac) + umlal v16.8h, v5.8b, v24.8b // + left[base+1]*frac + rshrn v16.8b, v16.8h, #6 + st1 {v16.s}[0], [x0], x1 + subs w4, w4, #2 + st1 {v16.s}[1], [x0], x1 + b.le 9f + + uqadd v26.8b, v26.8b, v21.8b // base += 4 + uqadd v27.8b, v27.8b, v21.8b // base += 4 + b 1b + +9: + ret + +80: // w == 8 + dup v29.8h, w5 // dy + + mul v30.8h, v30.8h, v29.8h // {0,1,2,3,4,5,6,7}*dy + movi v23.16b, #0x3e + + ld1 {v0.16b, v1.16b}, [x2] // left[] + add v30.8h, v29.8h, v30.8h // ypos + + movi v22.16b, #64 + movi v20.16b, #1 + movi v21.16b, #2 + + xtn v24.8b, v30.8h // (uint8_t)ypos + uqshrn v26.8b, v30.8h, #6 // base + and v24.8b, v24.8b, v23.8b // frac + + uqadd v27.8b, v26.8b, v20.8b // base + 1 + uqadd v28.8b, v26.8b, v21.8b // base + 2 + sub v25.8b, v22.8b, v24.8b // 64 - frac + uqadd v29.8b, v27.8b, v21.8b // base + 3 + + trn1 v24.2d, v24.2d, v24.2d // frac + trn1 v26.2d, v26.2d, v28.2d // base + 0, base + 2 + trn1 v27.2d, v27.2d, v29.2d // base + 1, base + 3 + trn1 v25.2d, v25.2d, v25.2d // 64 - frac + + movi v21.16b, #4 +1: + mov v4.16b, v31.16b + mov v5.16b, v31.16b + tbx v4.16b, {v0.16b, v1.16b, v2.16b, v3.16b}, v26.16b // left[base], left[base+2] + tbx v5.16b, {v0.16b, v1.16b, v2.16b, v3.16b}, v27.16b // left[base+1], left[base+3] + + umull v16.8h, v4.8b, v25.8b // left[base]*(64-frac) + umlal v16.8h, v5.8b, v24.8b // + left[base+1]*frac + umull2 v17.8h, v4.16b, v25.16b + umlal2 v17.8h, v5.16b, v24.16b + rshrn v16.8b, v16.8h, #6 + rshrn v17.8b, v17.8h, #6 + st1 {v16.8b}, [x0], x1 + subs w4, w4, #2 + st1 {v17.8b}, [x0], x1 + b.le 9f + + uqadd v26.16b, v26.16b, v21.16b // base += 4 + uqadd v27.16b, v27.16b, v21.16b // base += 4 + b 1b + +9: + ret +endfunc + + // void ipred_filter_8bpc_neon(pixel *dst, const ptrdiff_t stride, // const pixel *const topleft, // const int width, const int height, const int filt_idx, // const int max_width, const int max_height); function ipred_filter_8bpc_neon, export=1 diff --git a/third_party/dav1d/src/arm/ipred.h b/third_party/dav1d/src/arm/ipred.h --- a/third_party/dav1d/src/arm/ipred.h +++ b/third_party/dav1d/src/arm/ipred.h @@ -48,10 +48,142 @@ decl_cfl_ac_fn(BF(dav1d_ipred_cfl_ac_422, neon)); decl_cfl_ac_fn(BF(dav1d_ipred_cfl_ac_444, neon)); decl_pal_pred_fn(BF(dav1d_pal_pred, neon)); +#if ARCH_AARCH64 && BITDEPTH == 8 +void BF(dav1d_ipred_z1_upsample_edge, neon)(pixel *out, const int hsz, + const pixel *const in, + const int end); +void BF(dav1d_ipred_z1_filter_edge, neon)(pixel *out, const int sz, + const pixel *const in, + const int end, const int strength); +void BF(dav1d_ipred_z1_fill1, neon)(pixel *dst, ptrdiff_t stride, + const pixel *const top, const int width, + const int height, const int dx, + const int max_base_x); +void BF(dav1d_ipred_z1_fill2, neon)(pixel *dst, ptrdiff_t stride, + const pixel *const top, const int width, + const int height, const int dx, + const int max_base_x); + +static void ipred_z1_neon(pixel *dst, const ptrdiff_t stride, + const pixel *const topleft_in, + const int width, const int height, int angle, + const int max_width, const int max_height + HIGHBD_DECL_SUFFIX) +{ + const int is_sm = (angle >> 9) & 0x1; + const int enable_intra_edge_filter = angle >> 10; + angle &= 511; + int dx = dav1d_dr_intra_derivative[angle >> 1]; + pixel top_out[64 + 64 + (64+15)*2]; + int max_base_x; + const int upsample_above = enable_intra_edge_filter ? + get_upsample(width + height, 90 - angle, is_sm) : 0; + if (upsample_above) { + BF(dav1d_ipred_z1_upsample_edge, neon)(top_out, width + height, + topleft_in, + width + imin(width, height)); + max_base_x = 2 * (width + height) - 2; + dx <<= 1; + } else { + const int filter_strength = enable_intra_edge_filter ? + get_filter_strength(width + height, 90 - angle, is_sm) : 0; + if (filter_strength) { + BF(dav1d_ipred_z1_filter_edge, neon)(top_out, width + height, + topleft_in, + width + imin(width, height), + filter_strength); + max_base_x = width + height - 1; + } else { + max_base_x = width + imin(width, height) - 1; + memcpy(top_out, &topleft_in[1], (max_base_x + 1) * sizeof(pixel)); + } + } + const int base_inc = 1 + upsample_above; + int pad_pixels = width + 15; // max(dx >> 6) == 15 + pixel_set(&top_out[max_base_x + 1], top_out[max_base_x], pad_pixels * base_inc); + if (upsample_above) + BF(dav1d_ipred_z1_fill2, neon)(dst, stride, top_out, width, height, + dx, max_base_x); + else + BF(dav1d_ipred_z1_fill1, neon)(dst, stride, top_out, width, height, + dx, max_base_x); +} + +void BF(dav1d_ipred_z3_fill1, neon)(pixel *dst, ptrdiff_t stride, + const pixel *const left, const int width, + const int height, const int dy, + const int max_base_y); +void BF(dav1d_ipred_z3_fill2, neon)(pixel *dst, ptrdiff_t stride, + const pixel *const left, const int width, + const int height, const int dy, + const int max_base_y); + +void BF(dav1d_ipred_reverse, neon)(pixel *dst, const pixel *const src, + const int n); + +static void ipred_z3_neon(pixel *dst, const ptrdiff_t stride, + const pixel *const topleft_in, + const int width, const int height, int angle, + const int max_width, const int max_height + HIGHBD_DECL_SUFFIX) +{ + const int is_sm = (angle >> 9) & 0x1; + const int enable_intra_edge_filter = angle >> 10; + angle &= 511; + assert(angle > 180); + int dy = dav1d_dr_intra_derivative[(270 - angle) >> 1]; + pixel flipped[64 + 64 + 16]; + pixel left_out[64 + 64 + (64+15)*2]; + int max_base_y; + const int upsample_left = enable_intra_edge_filter ? + get_upsample(width + height, angle - 180, is_sm) : 0; + if (upsample_left) { + flipped[0] = topleft_in[0]; + BF(dav1d_ipred_reverse, neon)(&flipped[1], &topleft_in[0], + height + imax(width, height)); + BF(dav1d_ipred_z1_upsample_edge, neon)(left_out, width + height, + flipped, + height + imin(width, height)); + max_base_y = 2 * (width + height) - 2; + dy <<= 1; + } else { + const int filter_strength = enable_intra_edge_filter ? + get_filter_strength(width + height, angle - 180, is_sm) : 0; + + if (filter_strength) { + flipped[0] = topleft_in[0]; + BF(dav1d_ipred_reverse, neon)(&flipped[1], &topleft_in[0], + height + imax(width, height)); + BF(dav1d_ipred_z1_filter_edge, neon)(left_out, width + height, + flipped, + height + imin(width, height), + filter_strength); + max_base_y = width + height - 1; + } else { + BF(dav1d_ipred_reverse, neon)(left_out, &topleft_in[0], + height + imin(width, height)); + max_base_y = height + imin(width, height) - 1; + } + } + const int base_inc = 1 + upsample_left; + // The tbx based implementation needs left[] to have 64 bytes intitialized, + // the other implementation can read height + max(dy >> 6) past the end. + int pad_pixels = imax(64 - max_base_y - 1, height + 15); + + pixel_set(&left_out[max_base_y + 1], left_out[max_base_y], pad_pixels * base_inc); + if (upsample_left) + BF(dav1d_ipred_z3_fill2, neon)(dst, stride, left_out, width, height, + dy, max_base_y); + else + BF(dav1d_ipred_z3_fill1, neon)(dst, stride, left_out, width, height, + dy, max_base_y); +} +#endif + static ALWAYS_INLINE void intra_pred_dsp_init_arm(Dav1dIntraPredDSPContext *const c) { const unsigned flags = dav1d_get_cpu_flags(); if (!(flags & DAV1D_ARM_CPU_FLAG_NEON)) return; @@ -63,10 +195,14 @@ c->intra_pred[VERT_PRED] = BF(dav1d_ipred_v, neon); c->intra_pred[PAETH_PRED] = BF(dav1d_ipred_paeth, neon); c->intra_pred[SMOOTH_PRED] = BF(dav1d_ipred_smooth, neon); c->intra_pred[SMOOTH_V_PRED] = BF(dav1d_ipred_smooth_v, neon); c->intra_pred[SMOOTH_H_PRED] = BF(dav1d_ipred_smooth_h, neon); +#if ARCH_AARCH64 && BITDEPTH == 8 + c->intra_pred[Z1_PRED] = ipred_z1_neon; + c->intra_pred[Z3_PRED] = ipred_z3_neon; +#endif c->intra_pred[FILTER_PRED] = BF(dav1d_ipred_filter, neon); c->cfl_pred[DC_PRED] = BF(dav1d_ipred_cfl, neon); c->cfl_pred[DC_128_PRED] = BF(dav1d_ipred_cfl_128, neon); c->cfl_pred[TOP_DC_PRED] = BF(dav1d_ipred_cfl_top, neon); diff --git a/third_party/dav1d/src/decode.c b/third_party/dav1d/src/decode.c --- a/third_party/dav1d/src/decode.c +++ b/third_party/dav1d/src/decode.c @@ -3424,11 +3424,11 @@ if (c->n_fc > 1 && retval && f->frame_thread.cf) { memset(f->frame_thread.cf, 0, (size_t)f->frame_thread.cf_sz * 128 * 128 / 2); } for (int i = 0; i < 7; i++) { - if (f->refp[i].p.data[0]) + if (f->refp[i].p.frame_hdr) dav1d_thread_picture_unref(&f->refp[i]); dav1d_ref_dec(&f->ref_mvs_ref[i]); } dav1d_picture_unref_internal(&f->cur); @@ -3830,11 +3830,11 @@ // update references etc. const unsigned refresh_frame_flags = f->frame_hdr->refresh_frame_flags; for (int i = 0; i < 8; i++) { if (refresh_frame_flags & (1 << i)) { - if (c->refs[i].p.p.data[0]) + if (c->refs[i].p.p.frame_hdr) dav1d_thread_picture_unref(&c->refs[i].p); dav1d_thread_picture_ref(&c->refs[i].p, &f->sr_cur); dav1d_cdf_thread_unref(&c->cdf[i]); if (f->frame_hdr->refresh_context) { @@ -3860,11 +3860,11 @@ if (c->n_fc == 1) { if ((res = dav1d_decode_frame(f)) < 0) { dav1d_thread_picture_unref(&c->out); for (int i = 0; i < 8; i++) { if (refresh_frame_flags & (1 << i)) { - if (c->refs[i].p.p.data[0]) + if (c->refs[i].p.p.frame_hdr) dav1d_thread_picture_unref(&c->refs[i].p); dav1d_cdf_thread_unref(&c->cdf[i]); dav1d_ref_dec(&c->refs[i].segmap); dav1d_ref_dec(&c->refs[i].refmvs); } @@ -3881,11 +3881,11 @@ atomic_init(&f->task_thread.error, 1); dav1d_cdf_thread_unref(&f->in_cdf); if (f->frame_hdr->refresh_context) dav1d_cdf_thread_unref(&f->out_cdf); for (int i = 0; i < 7; i++) { - if (f->refp[i].p.data[0]) + if (f->refp[i].p.frame_hdr) dav1d_thread_picture_unref(&f->refp[i]); dav1d_ref_dec(&f->ref_mvs_ref[i]); } if (c->n_fc == 1) dav1d_thread_picture_unref(&c->out); diff --git a/third_party/dav1d/src/getbits.h b/third_party/dav1d/src/getbits.h --- a/third_party/dav1d/src/getbits.h +++ b/third_party/dav1d/src/getbits.h @@ -30,19 +30,19 @@ #include #include typedef struct GetBits { - int error, eof; uint64_t state; - unsigned bits_left; + int bits_left, error; const uint8_t *ptr, *ptr_start, *ptr_end; } GetBits; void dav1d_init_get_bits(GetBits *c, const uint8_t *data, size_t sz); -unsigned dav1d_get_bits(GetBits *c, unsigned n); -int dav1d_get_sbits(GetBits *c, unsigned n); +unsigned dav1d_get_bit(GetBits *c); +unsigned dav1d_get_bits(GetBits *c, int n); +int dav1d_get_sbits(GetBits *c, int n); unsigned dav1d_get_uleb128(GetBits *c); // Output in range 0..max-1 unsigned dav1d_get_uniform(GetBits *c, unsigned max); unsigned dav1d_get_vlc(GetBits *c); diff --git a/third_party/dav1d/src/getbits.c b/third_party/dav1d/src/getbits.c --- a/third_party/dav1d/src/getbits.c +++ b/third_party/dav1d/src/getbits.c @@ -34,54 +34,65 @@ #include "src/getbits.h" void dav1d_init_get_bits(GetBits *const c, const uint8_t *const data, const size_t sz) { - // If sz were 0, c->eof would need to be initialized to 1. assert(sz); c->ptr = c->ptr_start = data; c->ptr_end = &c->ptr_start[sz]; + c->state = 0; c->bits_left = 0; - c->state = 0; c->error = 0; - c->eof = 0; +} + +unsigned dav1d_get_bit(GetBits *const c) { + if (!c->bits_left) { + if (c->ptr >= c->ptr_end) { + c->error = 1; + } else { + const unsigned state = *c->ptr++; + c->bits_left = 7; + c->state = (uint64_t) state << 57; + return state >> 7; + } + } + + const uint64_t state = c->state; + c->bits_left--; + c->state = state << 1; + return (unsigned) (state >> 63); } -static void refill(GetBits *const c, const unsigned n) { - assert(c->bits_left <= 56); - uint64_t state = 0; +static inline void refill(GetBits *const c, const int n) { + assert(c->bits_left >= 0 && c->bits_left < 32); + unsigned state = 0; do { - state <<= 8; + if (c->ptr >= c->ptr_end) { + c->error = 1; + if (state) break; + return; + } + state = (state << 8) | *c->ptr++; c->bits_left += 8; - if (!c->eof) - state |= *c->ptr++; - if (c->ptr >= c->ptr_end) { - c->error = c->eof; - c->eof = 1; - } } while (n > c->bits_left); - c->state |= state << (64 - c->bits_left); + c->state |= (uint64_t) state << (64 - c->bits_left); } -unsigned dav1d_get_bits(GetBits *const c, const unsigned n) { - assert(n <= 32 /* can go up to 57 if we change return type */); - assert(n /* can't shift state by 64 */); - - if (n > c->bits_left) refill(c, n); - - const uint64_t state = c->state; - c->bits_left -= n; - c->state <<= n; - - return (unsigned) (state >> (64 - n)); +#define GET_BITS(name, type, type64) \ +type name(GetBits *const c, const int n) { \ + assert(n > 0 && n <= 32); \ + /* Unsigned cast avoids refill after eob */ \ + if ((unsigned) n > (unsigned) c->bits_left) \ + refill(c, n); \ + const uint64_t state = c->state; \ + c->bits_left -= n; \ + c->state = state << n; \ + return (type) ((type64) state >> (64 - n)); \ } -int dav1d_get_sbits(GetBits *const c, const unsigned n) { - const int shift = 31 - n; - const int res = dav1d_get_bits(c, n + 1) << shift; - return res >> shift; -} +GET_BITS(dav1d_get_bits, unsigned, uint64_t) +GET_BITS(dav1d_get_sbits, int, int64_t) unsigned dav1d_get_uleb128(GetBits *const c) { uint64_t val = 0; unsigned i = 0, more; @@ -106,19 +117,24 @@ assert(max > 1); const int l = ulog2(max) + 1; assert(l > 1); const unsigned m = (1U << l) - max; const unsigned v = dav1d_get_bits(c, l - 1); - return v < m ? v : (v << 1) - m + dav1d_get_bits(c, 1); + return v < m ? v : (v << 1) - m + dav1d_get_bit(c); } unsigned dav1d_get_vlc(GetBits *const c) { + if (dav1d_get_bit(c)) + return 0; + int n_bits = 0; - while (!dav1d_get_bits(c, 1)) + do { if (++n_bits == 32) return 0xFFFFFFFFU; - return n_bits ? ((1U << n_bits) - 1) + dav1d_get_bits(c, n_bits) : 0; + } while (!dav1d_get_bit(c)); + + return ((1U << n_bits) - 1) + dav1d_get_bits(c, n_bits); } static unsigned get_bits_subexp_u(GetBits *const c, const unsigned ref, const unsigned n) { @@ -130,11 +146,11 @@ if (n < v + 3 * (1 << b)) { v += dav1d_get_uniform(c, n - v + 1); break; } - if (!dav1d_get_bits(c, 1)) { + if (!dav1d_get_bit(c)) { v += dav1d_get_bits(c, b); break; } v += 1 << b; diff --git a/third_party/dav1d/src/internal.h b/third_party/dav1d/src/internal.h --- a/third_party/dav1d/src/internal.h +++ b/third_party/dav1d/src/internal.h @@ -192,10 +192,11 @@ int max_spatial_id; unsigned frame_size_limit; int strict_std_compliance; int output_invisible_frames; enum Dav1dInloopFilterType inloop_filters; + enum Dav1dDecodeFrameType decode_frame_type; int drain; enum PictureFlags frame_flags; enum Dav1dEventFlags event_flags; Dav1dDataProps cached_error_props; int cached_error; diff --git a/third_party/dav1d/src/lib.c b/third_party/dav1d/src/lib.c --- a/third_party/dav1d/src/lib.c +++ b/third_party/dav1d/src/lib.c @@ -75,10 +75,11 @@ s->all_layers = 1; // just until the tests are adjusted s->frame_size_limit = 0; s->strict_std_compliance = 0; s->output_invisible_frames = 0; s->inloop_filters = DAV1D_INLOOPFILTER_ALL; + s->decode_frame_type = DAV1D_DECODEFRAMETYPE_ALL; } static void close_internal(Dav1dContext **const c_out, int flush); NO_SANITIZE("cfi-icall") // CFI is broken with dlsym() @@ -142,10 +143,12 @@ DAV1D_ERR(EINVAL)); validate_input_or_ret(s->allocator.release_picture_callback != NULL, DAV1D_ERR(EINVAL)); validate_input_or_ret(s->operating_point >= 0 && s->operating_point <= 31, DAV1D_ERR(EINVAL)); + validate_input_or_ret(s->decode_frame_type >= DAV1D_DECODEFRAMETYPE_ALL && + s->decode_frame_type <= DAV1D_DECODEFRAMETYPE_KEY, DAV1D_ERR(EINVAL)); pthread_attr_t thread_attr; if (pthread_attr_init(&thread_attr)) return DAV1D_ERR(ENOMEM); size_t stack_size = 1024 * 1024 + get_stack_size_internal(&thread_attr); @@ -162,10 +165,11 @@ c->all_layers = s->all_layers; c->frame_size_limit = s->frame_size_limit; c->strict_std_compliance = s->strict_std_compliance; c->output_invisible_frames = s->output_invisible_frames; c->inloop_filters = s->inloop_filters; + c->decode_frame_type = s->decode_frame_type; dav1d_data_props_set_defaults(&c->cached_error_props); if (dav1d_mem_pool_init(&c->seq_hdr_pool) || dav1d_mem_pool_init(&c->frame_hdr_pool) || @@ -400,23 +404,21 @@ while (f->n_tile_data > 0) pthread_cond_wait(&f->task_thread.cond, &f->task_thread.ttd->lock); Dav1dThreadPicture *const out_delayed = &c->frame_thread.out_delayed[next]; - if (out_delayed->p.data[0] || atomic_load(&f->task_thread.error)) { - unsigned first = atomic_load(&c->task_thread.first); - if (first + 1U < c->n_fc) - atomic_fetch_add(&c->task_thread.first, 1U); - else - atomic_store(&c->task_thread.first, 0); - atomic_compare_exchange_strong(&c->task_thread.reset_task_cur, - &first, UINT_MAX); - if (c->task_thread.cur && c->task_thread.cur < c->n_fc) - c->task_thread.cur--; - } if (++c->frame_thread.next == c->n_fc) c->frame_thread.next = 0; + unsigned first = atomic_load(&c->task_thread.first); + if (first + 1U < c->n_fc) + atomic_fetch_add(&c->task_thread.first, 1U); + else + atomic_store(&c->task_thread.first, 0); + atomic_compare_exchange_strong(&c->task_thread.reset_task_cur, + &first, UINT_MAX); + if (c->task_thread.cur && c->task_thread.cur < c->n_fc) + c->task_thread.cur--; pthread_mutex_unlock(&c->task_thread.lock); const int error = f->task_thread.retval; if (error) { f->task_thread.retval = 0; dav1d_data_props_copy(&c->cached_error_props, &out_delayed->p.m); @@ -559,20 +561,20 @@ return res; } void dav1d_flush(Dav1dContext *const c) { dav1d_data_unref_internal(&c->in); - if (c->out.p.data[0]) + if (c->out.p.frame_hdr) dav1d_thread_picture_unref(&c->out); - if (c->cache.p.data[0]) + if (c->cache.p.frame_hdr) dav1d_thread_picture_unref(&c->cache); c->drain = 0; c->cached_error = 0; for (int i = 0; i < 8; i++) { - if (c->refs[i].p.p.data[0]) + if (c->refs[i].p.p.frame_hdr) dav1d_thread_picture_unref(&c->refs[i].p); dav1d_ref_dec(&c->refs[i].segmap); dav1d_ref_dec(&c->refs[i].refmvs); dav1d_cdf_thread_unref(&c->cdf[i]); } @@ -623,11 +625,11 @@ Dav1dFrameContext *const f = &c->fc[next]; dav1d_decode_frame_exit(f, -1); f->n_tile_data = 0; f->task_thread.retval = 0; Dav1dThreadPicture *out_delayed = &c->frame_thread.out_delayed[next]; - if (out_delayed->p.data[0]) { + if (out_delayed->p.frame_hdr) { dav1d_thread_picture_unref(out_delayed); } } c->frame_thread.next = 0; } @@ -702,20 +704,20 @@ dav1d_free_aligned(f->lf.lr_line_buf); } dav1d_free_aligned(c->fc); if (c->n_fc > 1 && c->frame_thread.out_delayed) { for (unsigned n = 0; n < c->n_fc; n++) - if (c->frame_thread.out_delayed[n].p.data[0]) + if (c->frame_thread.out_delayed[n].p.frame_hdr) dav1d_thread_picture_unref(&c->frame_thread.out_delayed[n]); free(c->frame_thread.out_delayed); } for (int n = 0; n < c->n_tile_data; n++) dav1d_data_unref_internal(&c->tile[n].data); free(c->tile); for (int n = 0; n < 8; n++) { dav1d_cdf_thread_unref(&c->cdf[n]); - if (c->refs[n].p.p.data[0]) + if (c->refs[n].p.p.frame_hdr) dav1d_thread_picture_unref(&c->refs[n].p); dav1d_ref_dec(&c->refs[n].refmvs); dav1d_ref_dec(&c->refs[n].segmap); } dav1d_ref_dec(&c->seq_hdr_ref); diff --git a/third_party/dav1d/src/meson.build b/third_party/dav1d/src/meson.build --- a/third_party/dav1d/src/meson.build +++ b/third_party/dav1d/src/meson.build @@ -35,20 +35,22 @@ 'dequant_tables.c', 'getbits.c', 'intra_edge.c', 'itx_1d.c', 'lf_mask.c', + 'lib.c', 'log.c', 'mem.c', 'msac.c', 'obu.c', 'picture.c', 'qm.c', 'ref.c', 'refmvs.c', 'scan.c', 'tables.c', + 'thread_task.c', 'warpmv.c', 'wedge.c', ) # libdav1d bitdepth source files @@ -72,18 +74,10 @@ libdav1d_arch_tmpl_sources = [] libdav1d_bitdepth_objs = [] -# libdav1d entrypoint source files -# These source files contain library entry points and are -# built with the stack-realign flag set, where necessary. -libdav1d_entrypoints_sources = files( - 'lib.c', - 'thread_task.c' -) - # ASM specific sources libdav1d_asm_objs = [] # Arch-specific flags arch_flags = [] if is_asm_enabled @@ -243,11 +237,11 @@ endif libdav1d_rc_obj = [] -libdav1d_flags = [stackalign_flag] +libdav1d_flags = [] api_export_flags = [] # # Windows .rc file and API export flags # @@ -278,22 +272,10 @@ # # Library definitions # -# Helper library for dav1d entrypoints -libdav1d_entrypoints_objs = static_library('dav1d_entrypoint', - libdav1d_entrypoints_sources, - rev_target, config_h_target, - - include_directories : dav1d_inc_dirs, - dependencies: [stdatomic_dependencies], - c_args : [libdav1d_flags, stackrealign_flag, api_export_flags], - install : false, - build_by_default : false, -).extract_all_objects(recursive: true) - # Helper library for each bitdepth libdav1d_bitdepth_objs = [] foreach bitdepth : dav1d_bitdepths libdav1d_bitdepth_objs += static_library( 'dav1d_bitdepth_@0@'.format(bitdepth), @@ -328,14 +310,15 @@ libdav1d = library('dav1d', libdav1d_sources, libdav1d_asm_objs, libdav1d_rc_obj, + rev_target, + config_h_target, objects : [ libdav1d_bitdepth_objs, - libdav1d_entrypoints_objs ], include_directories : dav1d_inc_dirs, dependencies : [ stdatomic_dependencies, diff --git a/third_party/dav1d/src/obu.c b/third_party/dav1d/src/obu.c --- a/third_party/dav1d/src/obu.c +++ b/third_party/dav1d/src/obu.c @@ -51,91 +51,85 @@ #if DEBUG_SEQ_HDR const unsigned init_bit_pos = dav1d_get_bits_pos(gb); #endif + memset(hdr, 0, sizeof(*hdr)); hdr->profile = dav1d_get_bits(gb, 3); if (hdr->profile > 2) goto error; #if DEBUG_SEQ_HDR printf("SEQHDR: post-profile: off=%u\n", dav1d_get_bits_pos(gb) - init_bit_pos); #endif - hdr->still_picture = dav1d_get_bits(gb, 1); - hdr->reduced_still_picture_header = dav1d_get_bits(gb, 1); + hdr->still_picture = dav1d_get_bit(gb); + hdr->reduced_still_picture_header = dav1d_get_bit(gb); if (hdr->reduced_still_picture_header && !hdr->still_picture) goto error; #if DEBUG_SEQ_HDR printf("SEQHDR: post-stillpicture_flags: off=%u\n", dav1d_get_bits_pos(gb) - init_bit_pos); #endif if (hdr->reduced_still_picture_header) { - hdr->timing_info_present = 0; - hdr->decoder_model_info_present = 0; - hdr->display_model_info_present = 0; hdr->num_operating_points = 1; - hdr->operating_points[0].idc = 0; hdr->operating_points[0].major_level = dav1d_get_bits(gb, 3); hdr->operating_points[0].minor_level = dav1d_get_bits(gb, 2); - hdr->operating_points[0].tier = 0; - hdr->operating_points[0].decoder_model_param_present = 0; - hdr->operating_points[0].display_model_param_present = 0; + hdr->operating_points[0].initial_display_delay = 10; } else { - hdr->timing_info_present = dav1d_get_bits(gb, 1); + hdr->timing_info_present = dav1d_get_bit(gb); if (hdr->timing_info_present) { hdr->num_units_in_tick = dav1d_get_bits(gb, 32); hdr->time_scale = dav1d_get_bits(gb, 32); - hdr->equal_picture_interval = dav1d_get_bits(gb, 1); + hdr->equal_picture_interval = dav1d_get_bit(gb); if (hdr->equal_picture_interval) { const unsigned num_ticks_per_picture = dav1d_get_vlc(gb); if (num_ticks_per_picture == 0xFFFFFFFFU) goto error; hdr->num_ticks_per_picture = num_ticks_per_picture + 1; } - hdr->decoder_model_info_present = dav1d_get_bits(gb, 1); + hdr->decoder_model_info_present = dav1d_get_bit(gb); if (hdr->decoder_model_info_present) { hdr->encoder_decoder_buffer_delay_length = dav1d_get_bits(gb, 5) + 1; hdr->num_units_in_decoding_tick = dav1d_get_bits(gb, 32); hdr->buffer_removal_delay_length = dav1d_get_bits(gb, 5) + 1; hdr->frame_presentation_delay_length = dav1d_get_bits(gb, 5) + 1; } - } else { - hdr->decoder_model_info_present = 0; } #if DEBUG_SEQ_HDR printf("SEQHDR: post-timinginfo: off=%u\n", dav1d_get_bits_pos(gb) - init_bit_pos); #endif - hdr->display_model_info_present = dav1d_get_bits(gb, 1); + hdr->display_model_info_present = dav1d_get_bit(gb); hdr->num_operating_points = dav1d_get_bits(gb, 5) + 1; for (int i = 0; i < hdr->num_operating_points; i++) { struct Dav1dSequenceHeaderOperatingPoint *const op = &hdr->operating_points[i]; op->idc = dav1d_get_bits(gb, 12); if (op->idc && (!(op->idc & 0xff) || !(op->idc & 0xf00))) goto error; op->major_level = 2 + dav1d_get_bits(gb, 3); op->minor_level = dav1d_get_bits(gb, 2); - op->tier = op->major_level > 3 ? dav1d_get_bits(gb, 1) : 0; - op->decoder_model_param_present = - hdr->decoder_model_info_present && dav1d_get_bits(gb, 1); - if (op->decoder_model_param_present) { - struct Dav1dSequenceHeaderOperatingParameterInfo *const opi = - &hdr->operating_parameter_info[i]; - opi->decoder_buffer_delay = - dav1d_get_bits(gb, hdr->encoder_decoder_buffer_delay_length); - opi->encoder_buffer_delay = - dav1d_get_bits(gb, hdr->encoder_decoder_buffer_delay_length); - opi->low_delay_mode = dav1d_get_bits(gb, 1); + if (op->major_level > 3) + op->tier = dav1d_get_bit(gb); + if (hdr->decoder_model_info_present) { + op->decoder_model_param_present = dav1d_get_bit(gb); + if (op->decoder_model_param_present) { + struct Dav1dSequenceHeaderOperatingParameterInfo *const opi = + &hdr->operating_parameter_info[i]; + opi->decoder_buffer_delay = + dav1d_get_bits(gb, hdr->encoder_decoder_buffer_delay_length); + opi->encoder_buffer_delay = + dav1d_get_bits(gb, hdr->encoder_decoder_buffer_delay_length); + opi->low_delay_mode = dav1d_get_bit(gb); + } } - op->display_model_param_present = - hdr->display_model_info_present && dav1d_get_bits(gb, 1); - if (op->display_model_param_present) { - op->initial_display_delay = dav1d_get_bits(gb, 4) + 1; - } + if (hdr->display_model_info_present) + op->display_model_param_present = dav1d_get_bit(gb); + op->initial_display_delay = + op->display_model_param_present ? dav1d_get_bits(gb, 4) + 1 : 10; } #if DEBUG_SEQ_HDR printf("SEQHDR: post-operating-points: off=%u\n", dav1d_get_bits_pos(gb) - init_bit_pos); #endif @@ -153,139 +147,127 @@ hdr->max_height = dav1d_get_bits(gb, hdr->height_n_bits) + 1; #if DEBUG_SEQ_HDR printf("SEQHDR: post-size: off=%u\n", dav1d_get_bits_pos(gb) - init_bit_pos); #endif - hdr->frame_id_numbers_present = - hdr->reduced_still_picture_header ? 0 : dav1d_get_bits(gb, 1); - if (hdr->frame_id_numbers_present) { - hdr->delta_frame_id_n_bits = dav1d_get_bits(gb, 4) + 2; - hdr->frame_id_n_bits = dav1d_get_bits(gb, 3) + hdr->delta_frame_id_n_bits + 1; + if (!hdr->reduced_still_picture_header) { + hdr->frame_id_numbers_present = dav1d_get_bit(gb); + if (hdr->frame_id_numbers_present) { + hdr->delta_frame_id_n_bits = dav1d_get_bits(gb, 4) + 2; + hdr->frame_id_n_bits = dav1d_get_bits(gb, 3) + hdr->delta_frame_id_n_bits + 1; + } } #if DEBUG_SEQ_HDR printf("SEQHDR: post-frame-id-numbers-present: off=%u\n", dav1d_get_bits_pos(gb) - init_bit_pos); #endif - hdr->sb128 = dav1d_get_bits(gb, 1); - hdr->filter_intra = dav1d_get_bits(gb, 1); - hdr->intra_edge_filter = dav1d_get_bits(gb, 1); + hdr->sb128 = dav1d_get_bit(gb); + hdr->filter_intra = dav1d_get_bit(gb); + hdr->intra_edge_filter = dav1d_get_bit(gb); if (hdr->reduced_still_picture_header) { - hdr->inter_intra = 0; - hdr->masked_compound = 0; - hdr->warped_motion = 0; - hdr->dual_filter = 0; - hdr->order_hint = 0; - hdr->jnt_comp = 0; - hdr->ref_frame_mvs = 0; - hdr->order_hint_n_bits = 0; hdr->screen_content_tools = DAV1D_ADAPTIVE; hdr->force_integer_mv = DAV1D_ADAPTIVE; } else { - hdr->inter_intra = dav1d_get_bits(gb, 1); - hdr->masked_compound = dav1d_get_bits(gb, 1); - hdr->warped_motion = dav1d_get_bits(gb, 1); - hdr->dual_filter = dav1d_get_bits(gb, 1); - hdr->order_hint = dav1d_get_bits(gb, 1); + hdr->inter_intra = dav1d_get_bit(gb); + hdr->masked_compound = dav1d_get_bit(gb); + hdr->warped_motion = dav1d_get_bit(gb); + hdr->dual_filter = dav1d_get_bit(gb); + hdr->order_hint = dav1d_get_bit(gb); if (hdr->order_hint) { - hdr->jnt_comp = dav1d_get_bits(gb, 1); - hdr->ref_frame_mvs = dav1d_get_bits(gb, 1); - } else { - hdr->jnt_comp = 0; - hdr->ref_frame_mvs = 0; - hdr->order_hint_n_bits = 0; + hdr->jnt_comp = dav1d_get_bit(gb); + hdr->ref_frame_mvs = dav1d_get_bit(gb); } - hdr->screen_content_tools = dav1d_get_bits(gb, 1) ? DAV1D_ADAPTIVE : dav1d_get_bits(gb, 1); + hdr->screen_content_tools = dav1d_get_bit(gb) ? DAV1D_ADAPTIVE : dav1d_get_bit(gb); #if DEBUG_SEQ_HDR printf("SEQHDR: post-screentools: off=%u\n", dav1d_get_bits_pos(gb) - init_bit_pos); #endif hdr->force_integer_mv = hdr->screen_content_tools ? - dav1d_get_bits(gb, 1) ? DAV1D_ADAPTIVE : dav1d_get_bits(gb, 1) : 2; + dav1d_get_bit(gb) ? DAV1D_ADAPTIVE : dav1d_get_bit(gb) : 2; if (hdr->order_hint) hdr->order_hint_n_bits = dav1d_get_bits(gb, 3) + 1; } - hdr->super_res = dav1d_get_bits(gb, 1); - hdr->cdef = dav1d_get_bits(gb, 1); - hdr->restoration = dav1d_get_bits(gb, 1); + hdr->super_res = dav1d_get_bit(gb); + hdr->cdef = dav1d_get_bit(gb); + hdr->restoration = dav1d_get_bit(gb); #if DEBUG_SEQ_HDR printf("SEQHDR: post-featurebits: off=%u\n", dav1d_get_bits_pos(gb) - init_bit_pos); #endif - hdr->hbd = dav1d_get_bits(gb, 1); - if (hdr->profile == 2 && hdr->hbd) hdr->hbd += dav1d_get_bits(gb, 1); - hdr->monochrome = hdr->profile != 1 ? dav1d_get_bits(gb, 1) : 0; - hdr->color_description_present = dav1d_get_bits(gb, 1); + hdr->hbd = dav1d_get_bit(gb); + if (hdr->profile == 2 && hdr->hbd) + hdr->hbd += dav1d_get_bit(gb); + if (hdr->profile != 1) + hdr->monochrome = dav1d_get_bit(gb); + hdr->color_description_present = dav1d_get_bit(gb); if (hdr->color_description_present) { hdr->pri = dav1d_get_bits(gb, 8); hdr->trc = dav1d_get_bits(gb, 8); hdr->mtrx = dav1d_get_bits(gb, 8); } else { hdr->pri = DAV1D_COLOR_PRI_UNKNOWN; hdr->trc = DAV1D_TRC_UNKNOWN; hdr->mtrx = DAV1D_MC_UNKNOWN; } if (hdr->monochrome) { - hdr->color_range = dav1d_get_bits(gb, 1); + hdr->color_range = dav1d_get_bit(gb); hdr->layout = DAV1D_PIXEL_LAYOUT_I400; hdr->ss_hor = hdr->ss_ver = 1; hdr->chr = DAV1D_CHR_UNKNOWN; - hdr->separate_uv_delta_q = 0; } else if (hdr->pri == DAV1D_COLOR_PRI_BT709 && hdr->trc == DAV1D_TRC_SRGB && hdr->mtrx == DAV1D_MC_IDENTITY) { hdr->layout = DAV1D_PIXEL_LAYOUT_I444; - hdr->ss_hor = hdr->ss_ver = 0; hdr->color_range = 1; if (hdr->profile != 1 && !(hdr->profile == 2 && hdr->hbd == 2)) goto error; } else { - hdr->color_range = dav1d_get_bits(gb, 1); + hdr->color_range = dav1d_get_bit(gb); switch (hdr->profile) { case 0: hdr->layout = DAV1D_PIXEL_LAYOUT_I420; hdr->ss_hor = hdr->ss_ver = 1; break; case 1: hdr->layout = DAV1D_PIXEL_LAYOUT_I444; - hdr->ss_hor = hdr->ss_ver = 0; break; case 2: if (hdr->hbd == 2) { - hdr->ss_hor = dav1d_get_bits(gb, 1); - hdr->ss_ver = hdr->ss_hor && dav1d_get_bits(gb, 1); - } else { + hdr->ss_hor = dav1d_get_bit(gb); + if (hdr->ss_hor) + hdr->ss_ver = dav1d_get_bit(gb); + } else hdr->ss_hor = 1; - hdr->ss_ver = 0; - } hdr->layout = hdr->ss_hor ? hdr->ss_ver ? DAV1D_PIXEL_LAYOUT_I420 : DAV1D_PIXEL_LAYOUT_I422 : DAV1D_PIXEL_LAYOUT_I444; break; } - hdr->chr = hdr->ss_hor == 1 && hdr->ss_ver == 1 ? + hdr->chr = (hdr->ss_hor & hdr->ss_ver) ? dav1d_get_bits(gb, 2) : DAV1D_CHR_UNKNOWN; } if (c->strict_std_compliance && hdr->mtrx == DAV1D_MC_IDENTITY && hdr->layout != DAV1D_PIXEL_LAYOUT_I444) { goto error; } - hdr->separate_uv_delta_q = !hdr->monochrome && dav1d_get_bits(gb, 1); + if (!hdr->monochrome) + hdr->separate_uv_delta_q = dav1d_get_bit(gb); #if DEBUG_SEQ_HDR printf("SEQHDR: post-colorinfo: off=%u\n", dav1d_get_bits_pos(gb) - init_bit_pos); #endif - hdr->film_grain_present = dav1d_get_bits(gb, 1); + hdr->film_grain_present = dav1d_get_bit(gb); #if DEBUG_SEQ_HDR printf("SEQHDR: post-filmgrain: off=%u\n", dav1d_get_bits_pos(gb) - init_bit_pos); #endif - dav1d_get_bits(gb, 1); // dummy bit + dav1d_get_bit(gb); // dummy bit // We needn't bother flushing the OBU here: we'll check we didn't // overrun in the caller and will then discard gb, so there's no // point in setting its position properly. @@ -302,19 +284,19 @@ const Dav1dSequenceHeader *const seqhdr = c->seq_hdr; Dav1dFrameHeader *const hdr = c->frame_hdr; if (use_ref) { for (int i = 0; i < 7; i++) { - if (dav1d_get_bits(gb, 1)) { + if (dav1d_get_bit(gb)) { const Dav1dThreadPicture *const ref = &c->refs[c->frame_hdr->refidx[i]].p; - if (!ref->p.data[0]) return -1; - hdr->width[1] = ref->p.p.w; - hdr->height = ref->p.p.h; + if (!ref->p.frame_hdr) return -1; + hdr->width[1] = ref->p.frame_hdr->width[1]; + hdr->height = ref->p.frame_hdr->height; hdr->render_width = ref->p.frame_hdr->render_width; hdr->render_height = ref->p.frame_hdr->render_height; - hdr->super_res.enabled = seqhdr->super_res && dav1d_get_bits(gb, 1); + hdr->super_res.enabled = seqhdr->super_res && dav1d_get_bit(gb); if (hdr->super_res.enabled) { const int d = hdr->super_res.width_scale_denominator = 9 + dav1d_get_bits(gb, 3); hdr->width[0] = imax((hdr->width[1] * 8 + (d >> 1)) / d, imin(16, hdr->width[1])); @@ -332,19 +314,19 @@ hdr->height = dav1d_get_bits(gb, seqhdr->height_n_bits) + 1; } else { hdr->width[1] = seqhdr->max_width; hdr->height = seqhdr->max_height; } - hdr->super_res.enabled = seqhdr->super_res && dav1d_get_bits(gb, 1); + hdr->super_res.enabled = seqhdr->super_res && dav1d_get_bit(gb); if (hdr->super_res.enabled) { const int d = hdr->super_res.width_scale_denominator = 9 + dav1d_get_bits(gb, 3); hdr->width[0] = imax((hdr->width[1] * 8 + (d >> 1)) / d, imin(16, hdr->width[1])); } else { hdr->super_res.width_scale_denominator = 8; hdr->width[0] = hdr->width[1]; } - hdr->have_render_size = dav1d_get_bits(gb, 1); + hdr->have_render_size = dav1d_get_bit(gb); if (hdr->have_render_size) { hdr->render_width = dav1d_get_bits(gb, 16) + 1; hdr->render_height = dav1d_get_bits(gb, 16) + 1; } else { hdr->render_width = hdr->width[1]; @@ -372,11 +354,11 @@ #endif const Dav1dSequenceHeader *const seqhdr = c->seq_hdr; Dav1dFrameHeader *const hdr = c->frame_hdr; hdr->show_existing_frame = - !seqhdr->reduced_still_picture_header && dav1d_get_bits(gb, 1); + !seqhdr->reduced_still_picture_header && dav1d_get_bit(gb); #if DEBUG_FRAME_HDR printf("HDR: post-show_existing_frame: off=%td\n", (gb->ptr - init_ptr) * 8 - gb->bits_left); #endif if (hdr->show_existing_frame) { @@ -390,53 +372,53 @@ } return 0; } hdr->frame_type = seqhdr->reduced_still_picture_header ? DAV1D_FRAME_TYPE_KEY : dav1d_get_bits(gb, 2); - hdr->show_frame = seqhdr->reduced_still_picture_header || dav1d_get_bits(gb, 1); + hdr->show_frame = seqhdr->reduced_still_picture_header || dav1d_get_bit(gb); if (hdr->show_frame) { if (seqhdr->decoder_model_info_present && !seqhdr->equal_picture_interval) hdr->frame_presentation_delay = dav1d_get_bits(gb, seqhdr->frame_presentation_delay_length); hdr->showable_frame = hdr->frame_type != DAV1D_FRAME_TYPE_KEY; } else - hdr->showable_frame = dav1d_get_bits(gb, 1); + hdr->showable_frame = dav1d_get_bit(gb); hdr->error_resilient_mode = (hdr->frame_type == DAV1D_FRAME_TYPE_KEY && hdr->show_frame) || hdr->frame_type == DAV1D_FRAME_TYPE_SWITCH || - seqhdr->reduced_still_picture_header || dav1d_get_bits(gb, 1); + seqhdr->reduced_still_picture_header || dav1d_get_bit(gb); #if DEBUG_FRAME_HDR printf("HDR: post-frametype_bits: off=%td\n", (gb->ptr - init_ptr) * 8 - gb->bits_left); #endif - hdr->disable_cdf_update = dav1d_get_bits(gb, 1); + hdr->disable_cdf_update = dav1d_get_bit(gb); hdr->allow_screen_content_tools = seqhdr->screen_content_tools == DAV1D_ADAPTIVE ? - dav1d_get_bits(gb, 1) : seqhdr->screen_content_tools; + dav1d_get_bit(gb) : seqhdr->screen_content_tools; if (hdr->allow_screen_content_tools) hdr->force_integer_mv = seqhdr->force_integer_mv == DAV1D_ADAPTIVE ? - dav1d_get_bits(gb, 1) : seqhdr->force_integer_mv; + dav1d_get_bit(gb) : seqhdr->force_integer_mv; else hdr->force_integer_mv = 0; if (IS_KEY_OR_INTRA(hdr)) hdr->force_integer_mv = 1; if (seqhdr->frame_id_numbers_present) hdr->frame_id = dav1d_get_bits(gb, seqhdr->frame_id_n_bits); hdr->frame_size_override = seqhdr->reduced_still_picture_header ? 0 : - hdr->frame_type == DAV1D_FRAME_TYPE_SWITCH ? 1 : dav1d_get_bits(gb, 1); + hdr->frame_type == DAV1D_FRAME_TYPE_SWITCH ? 1 : dav1d_get_bit(gb); #if DEBUG_FRAME_HDR printf("HDR: post-frame_size_override_flag: off=%td\n", (gb->ptr - init_ptr) * 8 - gb->bits_left); #endif hdr->frame_offset = seqhdr->order_hint ? dav1d_get_bits(gb, seqhdr->order_hint_n_bits) : 0; hdr->primary_ref_frame = !hdr->error_resilient_mode && IS_INTER_OR_SWITCH(hdr) ? dav1d_get_bits(gb, 3) : DAV1D_PRIMARY_REF_NONE; if (seqhdr->decoder_model_info_present) { - hdr->buffer_removal_time_present = dav1d_get_bits(gb, 1); + hdr->buffer_removal_time_present = dav1d_get_bit(gb); if (hdr->buffer_removal_time_present) { for (int i = 0; i < c->seq_hdr->num_operating_points; i++) { const struct Dav1dSequenceHeaderOperatingPoint *const seqop = &seqhdr->operating_points[i]; struct Dav1dFrameHeaderOperatingPoint *const op = &hdr->operating_points[i]; if (seqop->decoder_model_param_present) { @@ -460,21 +442,21 @@ { goto error; } if (read_frame_size(c, gb, 0) < 0) goto error; hdr->allow_intrabc = hdr->allow_screen_content_tools && - !hdr->super_res.enabled && dav1d_get_bits(gb, 1); + !hdr->super_res.enabled && dav1d_get_bit(gb); hdr->use_ref_frame_mvs = 0; } else { hdr->allow_intrabc = 0; hdr->refresh_frame_flags = hdr->frame_type == DAV1D_FRAME_TYPE_SWITCH ? 0xff : dav1d_get_bits(gb, 8); if (hdr->error_resilient_mode && seqhdr->order_hint) for (int i = 0; i < 8; i++) dav1d_get_bits(gb, seqhdr->order_hint_n_bits); hdr->frame_ref_short_signaling = - seqhdr->order_hint && dav1d_get_bits(gb, 1); + seqhdr->order_hint && dav1d_get_bit(gb); if (hdr->frame_ref_short_signaling) { // FIXME: Nearly verbatim copy from section 7.8 hdr->refidx[0] = dav1d_get_bits(gb, 3); hdr->refidx[1] = hdr->refidx[2] = -1; hdr->refidx[3] = dav1d_get_bits(gb, 3); hdr->refidx[4] = hdr->refidx[5] = hdr->refidx[6] = -1; @@ -574,32 +556,32 @@ } } const int use_ref = !hdr->error_resilient_mode && hdr->frame_size_override; if (read_frame_size(c, gb, use_ref) < 0) goto error; - hdr->hp = !hdr->force_integer_mv && dav1d_get_bits(gb, 1); - hdr->subpel_filter_mode = dav1d_get_bits(gb, 1) ? DAV1D_FILTER_SWITCHABLE : + hdr->hp = !hdr->force_integer_mv && dav1d_get_bit(gb); + hdr->subpel_filter_mode = dav1d_get_bit(gb) ? DAV1D_FILTER_SWITCHABLE : dav1d_get_bits(gb, 2); - hdr->switchable_motion_mode = dav1d_get_bits(gb, 1); + hdr->switchable_motion_mode = dav1d_get_bit(gb); hdr->use_ref_frame_mvs = !hdr->error_resilient_mode && seqhdr->ref_frame_mvs && seqhdr->order_hint && - IS_INTER_OR_SWITCH(hdr) && dav1d_get_bits(gb, 1); + IS_INTER_OR_SWITCH(hdr) && dav1d_get_bit(gb); } #if DEBUG_FRAME_HDR printf("HDR: post-frametype-specific-bits: off=%td\n", (gb->ptr - init_ptr) * 8 - gb->bits_left); #endif hdr->refresh_context = !seqhdr->reduced_still_picture_header && - !hdr->disable_cdf_update && !dav1d_get_bits(gb, 1); + !hdr->disable_cdf_update && !dav1d_get_bit(gb); #if DEBUG_FRAME_HDR printf("HDR: post-refresh_context: off=%td\n", (gb->ptr - init_ptr) * 8 - gb->bits_left); #endif // tile data - hdr->tiling.uniform = dav1d_get_bits(gb, 1); + hdr->tiling.uniform = dav1d_get_bit(gb); const int sbsz_min1 = (64 << seqhdr->sb128) - 1; const int sbsz_log2 = 6 + seqhdr->sb128; const int sbw = (hdr->width[0] + sbsz_min1) >> sbsz_log2; const int sbh = (hdr->height + sbsz_min1) >> sbsz_log2; const int max_tile_width_sb = 4096 >> sbsz_log2; @@ -609,21 +591,21 @@ hdr->tiling.max_log2_rows = tile_log2(1, imin(sbh, DAV1D_MAX_TILE_ROWS)); const int min_log2_tiles = imax(tile_log2(max_tile_area_sb, sbw * sbh), hdr->tiling.min_log2_cols); if (hdr->tiling.uniform) { for (hdr->tiling.log2_cols = hdr->tiling.min_log2_cols; - hdr->tiling.log2_cols < hdr->tiling.max_log2_cols && dav1d_get_bits(gb, 1); + hdr->tiling.log2_cols < hdr->tiling.max_log2_cols && dav1d_get_bit(gb); hdr->tiling.log2_cols++) ; const int tile_w = 1 + ((sbw - 1) >> hdr->tiling.log2_cols); hdr->tiling.cols = 0; for (int sbx = 0; sbx < sbw; sbx += tile_w, hdr->tiling.cols++) hdr->tiling.col_start_sb[hdr->tiling.cols] = sbx; hdr->tiling.min_log2_rows = imax(min_log2_tiles - hdr->tiling.log2_cols, 0); for (hdr->tiling.log2_rows = hdr->tiling.min_log2_rows; - hdr->tiling.log2_rows < hdr->tiling.max_log2_rows && dav1d_get_bits(gb, 1); + hdr->tiling.log2_rows < hdr->tiling.max_log2_rows && dav1d_get_bit(gb); hdr->tiling.log2_rows++) ; const int tile_h = 1 + ((sbh - 1) >> hdr->tiling.log2_rows); hdr->tiling.rows = 0; for (int sby = 0; sby < sbh; sby += tile_h, hdr->tiling.rows++) hdr->tiling.row_start_sb[hdr->tiling.rows] = sby; @@ -670,31 +652,31 @@ (gb->ptr - init_ptr) * 8 - gb->bits_left); #endif // quant data hdr->quant.yac = dav1d_get_bits(gb, 8); - hdr->quant.ydc_delta = dav1d_get_bits(gb, 1) ? dav1d_get_sbits(gb, 6) : 0; + hdr->quant.ydc_delta = dav1d_get_bit(gb) ? dav1d_get_sbits(gb, 7) : 0; if (!seqhdr->monochrome) { // If the sequence header says that delta_q might be different // for U, V, we must check whether it actually is for this // frame. - const int diff_uv_delta = seqhdr->separate_uv_delta_q ? dav1d_get_bits(gb, 1) : 0; - hdr->quant.udc_delta = dav1d_get_bits(gb, 1) ? dav1d_get_sbits(gb, 6) : 0; - hdr->quant.uac_delta = dav1d_get_bits(gb, 1) ? dav1d_get_sbits(gb, 6) : 0; + const int diff_uv_delta = seqhdr->separate_uv_delta_q ? dav1d_get_bit(gb) : 0; + hdr->quant.udc_delta = dav1d_get_bit(gb) ? dav1d_get_sbits(gb, 7) : 0; + hdr->quant.uac_delta = dav1d_get_bit(gb) ? dav1d_get_sbits(gb, 7) : 0; if (diff_uv_delta) { - hdr->quant.vdc_delta = dav1d_get_bits(gb, 1) ? dav1d_get_sbits(gb, 6) : 0; - hdr->quant.vac_delta = dav1d_get_bits(gb, 1) ? dav1d_get_sbits(gb, 6) : 0; + hdr->quant.vdc_delta = dav1d_get_bit(gb) ? dav1d_get_sbits(gb, 7) : 0; + hdr->quant.vac_delta = dav1d_get_bit(gb) ? dav1d_get_sbits(gb, 7) : 0; } else { hdr->quant.vdc_delta = hdr->quant.udc_delta; hdr->quant.vac_delta = hdr->quant.uac_delta; } } #if DEBUG_FRAME_HDR printf("HDR: post-quant: off=%td\n", (gb->ptr - init_ptr) * 8 - gb->bits_left); #endif - hdr->quant.qm = dav1d_get_bits(gb, 1); + hdr->quant.qm = dav1d_get_bit(gb); if (hdr->quant.qm) { hdr->quant.qm_y = dav1d_get_bits(gb, 4); hdr->quant.qm_u = dav1d_get_bits(gb, 4); hdr->quant.qm_v = seqhdr->separate_uv_delta_q ? (int)dav1d_get_bits(gb, 4) : @@ -704,71 +686,71 @@ printf("HDR: post-qm: off=%td\n", (gb->ptr - init_ptr) * 8 - gb->bits_left); #endif // segmentation data - hdr->segmentation.enabled = dav1d_get_bits(gb, 1); + hdr->segmentation.enabled = dav1d_get_bit(gb); if (hdr->segmentation.enabled) { if (hdr->primary_ref_frame == DAV1D_PRIMARY_REF_NONE) { hdr->segmentation.update_map = 1; hdr->segmentation.temporal = 0; hdr->segmentation.update_data = 1; } else { - hdr->segmentation.update_map = dav1d_get_bits(gb, 1); + hdr->segmentation.update_map = dav1d_get_bit(gb); hdr->segmentation.temporal = - hdr->segmentation.update_map ? dav1d_get_bits(gb, 1) : 0; - hdr->segmentation.update_data = dav1d_get_bits(gb, 1); + hdr->segmentation.update_map ? dav1d_get_bit(gb) : 0; + hdr->segmentation.update_data = dav1d_get_bit(gb); } if (hdr->segmentation.update_data) { hdr->segmentation.seg_data.preskip = 0; hdr->segmentation.seg_data.last_active_segid = -1; for (int i = 0; i < DAV1D_MAX_SEGMENTS; i++) { Dav1dSegmentationData *const seg = &hdr->segmentation.seg_data.d[i]; - if (dav1d_get_bits(gb, 1)) { - seg->delta_q = dav1d_get_sbits(gb, 8); + if (dav1d_get_bit(gb)) { + seg->delta_q = dav1d_get_sbits(gb, 9); hdr->segmentation.seg_data.last_active_segid = i; } else { seg->delta_q = 0; } - if (dav1d_get_bits(gb, 1)) { - seg->delta_lf_y_v = dav1d_get_sbits(gb, 6); + if (dav1d_get_bit(gb)) { + seg->delta_lf_y_v = dav1d_get_sbits(gb, 7); hdr->segmentation.seg_data.last_active_segid = i; } else { seg->delta_lf_y_v = 0; } - if (dav1d_get_bits(gb, 1)) { - seg->delta_lf_y_h = dav1d_get_sbits(gb, 6); + if (dav1d_get_bit(gb)) { + seg->delta_lf_y_h = dav1d_get_sbits(gb, 7); hdr->segmentation.seg_data.last_active_segid = i; } else { seg->delta_lf_y_h = 0; } - if (dav1d_get_bits(gb, 1)) { - seg->delta_lf_u = dav1d_get_sbits(gb, 6); + if (dav1d_get_bit(gb)) { + seg->delta_lf_u = dav1d_get_sbits(gb, 7); hdr->segmentation.seg_data.last_active_segid = i; } else { seg->delta_lf_u = 0; } - if (dav1d_get_bits(gb, 1)) { - seg->delta_lf_v = dav1d_get_sbits(gb, 6); + if (dav1d_get_bit(gb)) { + seg->delta_lf_v = dav1d_get_sbits(gb, 7); hdr->segmentation.seg_data.last_active_segid = i; } else { seg->delta_lf_v = 0; } - if (dav1d_get_bits(gb, 1)) { + if (dav1d_get_bit(gb)) { seg->ref = dav1d_get_bits(gb, 3); hdr->segmentation.seg_data.last_active_segid = i; hdr->segmentation.seg_data.preskip = 1; } else { seg->ref = -1; } - if ((seg->skip = dav1d_get_bits(gb, 1))) { + if ((seg->skip = dav1d_get_bit(gb))) { hdr->segmentation.seg_data.last_active_segid = i; hdr->segmentation.seg_data.preskip = 1; } - if ((seg->globalmv = dav1d_get_bits(gb, 1))) { + if ((seg->globalmv = dav1d_get_bit(gb))) { hdr->segmentation.seg_data.last_active_segid = i; hdr->segmentation.seg_data.preskip = 1; } } } else { @@ -789,16 +771,16 @@ printf("HDR: post-segmentation: off=%td\n", (gb->ptr - init_ptr) * 8 - gb->bits_left); #endif // delta q - hdr->delta.q.present = hdr->quant.yac ? dav1d_get_bits(gb, 1) : 0; + hdr->delta.q.present = hdr->quant.yac ? dav1d_get_bit(gb) : 0; hdr->delta.q.res_log2 = hdr->delta.q.present ? dav1d_get_bits(gb, 2) : 0; hdr->delta.lf.present = hdr->delta.q.present && !hdr->allow_intrabc && - dav1d_get_bits(gb, 1); + dav1d_get_bit(gb); hdr->delta.lf.res_log2 = hdr->delta.lf.present ? dav1d_get_bits(gb, 2) : 0; - hdr->delta.lf.multi = hdr->delta.lf.present ? dav1d_get_bits(gb, 1) : 0; + hdr->delta.lf.multi = hdr->delta.lf.present ? dav1d_get_bit(gb) : 0; #if DEBUG_FRAME_HDR printf("HDR: post-delta_q_lf_flags: off=%td\n", (gb->ptr - init_ptr) * 8 - gb->bits_left); #endif @@ -840,22 +822,22 @@ const int ref = hdr->refidx[hdr->primary_ref_frame]; if (!c->refs[ref].p.p.frame_hdr) goto error; hdr->loopfilter.mode_ref_deltas = c->refs[ref].p.p.frame_hdr->loopfilter.mode_ref_deltas; } - hdr->loopfilter.mode_ref_delta_enabled = dav1d_get_bits(gb, 1); + hdr->loopfilter.mode_ref_delta_enabled = dav1d_get_bit(gb); if (hdr->loopfilter.mode_ref_delta_enabled) { - hdr->loopfilter.mode_ref_delta_update = dav1d_get_bits(gb, 1); + hdr->loopfilter.mode_ref_delta_update = dav1d_get_bit(gb); if (hdr->loopfilter.mode_ref_delta_update) { for (int i = 0; i < 8; i++) - if (dav1d_get_bits(gb, 1)) + if (dav1d_get_bit(gb)) hdr->loopfilter.mode_ref_deltas.ref_delta[i] = - dav1d_get_sbits(gb, 6); + dav1d_get_sbits(gb, 7); for (int i = 0; i < 2; i++) - if (dav1d_get_bits(gb, 1)) + if (dav1d_get_bit(gb)) hdr->loopfilter.mode_ref_deltas.mode_delta[i] = - dav1d_get_sbits(gb, 6); + dav1d_get_sbits(gb, 7); } } } #if DEBUG_FRAME_HDR printf("HDR: post-lpf: off=%td\n", @@ -897,20 +879,20 @@ if (hdr->restoration.type[0] || hdr->restoration.type[1] || hdr->restoration.type[2]) { // Log2 of the restoration unit size. hdr->restoration.unit_size[0] = 6 + seqhdr->sb128; - if (dav1d_get_bits(gb, 1)) { + if (dav1d_get_bit(gb)) { hdr->restoration.unit_size[0]++; if (!seqhdr->sb128) - hdr->restoration.unit_size[0] += dav1d_get_bits(gb, 1); + hdr->restoration.unit_size[0] += dav1d_get_bit(gb); } hdr->restoration.unit_size[1] = hdr->restoration.unit_size[0]; if ((hdr->restoration.type[1] || hdr->restoration.type[2]) && seqhdr->ss_hor == 1 && seqhdr->ss_ver == 1) { - hdr->restoration.unit_size[1] -= dav1d_get_bits(gb, 1); + hdr->restoration.unit_size[1] -= dav1d_get_bit(gb); } } else { hdr->restoration.unit_size[0] = 8; } } else { @@ -922,16 +904,16 @@ printf("HDR: post-restoration: off=%td\n", (gb->ptr - init_ptr) * 8 - gb->bits_left); #endif hdr->txfm_mode = hdr->all_lossless ? DAV1D_TX_4X4_ONLY : - dav1d_get_bits(gb, 1) ? DAV1D_TX_SWITCHABLE : DAV1D_TX_LARGEST; + dav1d_get_bit(gb) ? DAV1D_TX_SWITCHABLE : DAV1D_TX_LARGEST; #if DEBUG_FRAME_HDR printf("HDR: post-txfmmode: off=%td\n", (gb->ptr - init_ptr) * 8 - gb->bits_left); #endif - hdr->switchable_comp_refs = IS_INTER_OR_SWITCH(hdr) ? dav1d_get_bits(gb, 1) : 0; + hdr->switchable_comp_refs = IS_INTER_OR_SWITCH(hdr) ? dav1d_get_bit(gb) : 0; #if DEBUG_FRAME_HDR printf("HDR: post-refmode: off=%td\n", (gb->ptr - init_ptr) * 8 - gb->bits_left); #endif hdr->skip_mode_allowed = 0; @@ -939,11 +921,11 @@ const unsigned poc = hdr->frame_offset; unsigned off_before = 0xFFFFFFFFU; int off_after = -1; int off_before_idx, off_after_idx; for (int i = 0; i < 7; i++) { - if (!c->refs[hdr->refidx[i]].p.p.data[0]) goto error; + if (!c->refs[hdr->refidx[i]].p.p.frame_hdr) goto error; const unsigned refpoc = c->refs[hdr->refidx[i]].p.p.frame_hdr->frame_offset; const int diff = get_poc_diff(seqhdr->order_hint_n_bits, refpoc, poc); if (diff > 0) { if (off_after == -1 || get_poc_diff(seqhdr->order_hint_n_bits, @@ -967,11 +949,11 @@ hdr->skip_mode_allowed = 1; } else if (off_before != 0xFFFFFFFFU) { unsigned off_before2 = 0xFFFFFFFFU; int off_before2_idx; for (int i = 0; i < 7; i++) { - if (!c->refs[hdr->refidx[i]].p.p.data[0]) goto error; + if (!c->refs[hdr->refidx[i]].p.p.frame_hdr) goto error; const unsigned refpoc = c->refs[hdr->refidx[i]].p.p.frame_hdr->frame_offset; if (get_poc_diff(seqhdr->order_hint_n_bits, refpoc, off_before) < 0) { if (off_before2 == 0xFFFFFFFFU || get_poc_diff(seqhdr->order_hint_n_bits, @@ -988,35 +970,35 @@ hdr->skip_mode_refs[1] = imax(off_before_idx, off_before2_idx); hdr->skip_mode_allowed = 1; } } } - hdr->skip_mode_enabled = hdr->skip_mode_allowed ? dav1d_get_bits(gb, 1) : 0; + hdr->skip_mode_enabled = hdr->skip_mode_allowed ? dav1d_get_bit(gb) : 0; #if DEBUG_FRAME_HDR printf("HDR: post-extskip: off=%td\n", (gb->ptr - init_ptr) * 8 - gb->bits_left); #endif hdr->warp_motion = !hdr->error_resilient_mode && IS_INTER_OR_SWITCH(hdr) && - seqhdr->warped_motion && dav1d_get_bits(gb, 1); + seqhdr->warped_motion && dav1d_get_bit(gb); #if DEBUG_FRAME_HDR printf("HDR: post-warpmotionbit: off=%td\n", (gb->ptr - init_ptr) * 8 - gb->bits_left); #endif - hdr->reduced_txtp_set = dav1d_get_bits(gb, 1); + hdr->reduced_txtp_set = dav1d_get_bit(gb); #if DEBUG_FRAME_HDR printf("HDR: post-reducedtxtpset: off=%td\n", (gb->ptr - init_ptr) * 8 - gb->bits_left); #endif for (int i = 0; i < 7; i++) hdr->gmv[i] = dav1d_default_wm_params; if (IS_INTER_OR_SWITCH(hdr)) { for (int i = 0; i < 7; i++) { - hdr->gmv[i].type = !dav1d_get_bits(gb, 1) ? DAV1D_WM_TYPE_IDENTITY : - dav1d_get_bits(gb, 1) ? DAV1D_WM_TYPE_ROT_ZOOM : - dav1d_get_bits(gb, 1) ? DAV1D_WM_TYPE_TRANSLATION : + hdr->gmv[i].type = !dav1d_get_bit(gb) ? DAV1D_WM_TYPE_IDENTITY : + dav1d_get_bit(gb) ? DAV1D_WM_TYPE_ROT_ZOOM : + dav1d_get_bit(gb) ? DAV1D_WM_TYPE_TRANSLATION : DAV1D_WM_TYPE_AFFINE; if (hdr->gmv[i].type == DAV1D_WM_TYPE_IDENTITY) continue; const Dav1dWarpedMotionParams *ref_gmv; @@ -1061,14 +1043,14 @@ (gb->ptr - init_ptr) * 8 - gb->bits_left); #endif hdr->film_grain.present = seqhdr->film_grain_present && (hdr->show_frame || hdr->showable_frame) && - dav1d_get_bits(gb, 1); + dav1d_get_bit(gb); if (hdr->film_grain.present) { const unsigned seed = dav1d_get_bits(gb, 16); - hdr->film_grain.update = hdr->frame_type != DAV1D_FRAME_TYPE_INTER || dav1d_get_bits(gb, 1); + hdr->film_grain.update = hdr->frame_type != DAV1D_FRAME_TYPE_INTER || dav1d_get_bit(gb); if (!hdr->film_grain.update) { const int refidx = dav1d_get_bits(gb, 3); int i; for (i = 0; i < 7; i++) if (hdr->refidx[i] == refidx) @@ -1088,11 +1070,11 @@ goto error; fgd->y_points[i][1] = dav1d_get_bits(gb, 8); } fgd->chroma_scaling_from_luma = - !seqhdr->monochrome && dav1d_get_bits(gb, 1); + !seqhdr->monochrome && dav1d_get_bit(gb); if (seqhdr->monochrome || fgd->chroma_scaling_from_luma || (seqhdr->ss_ver == 1 && seqhdr->ss_hor == 1 && !fgd->num_y_points)) { fgd->num_uv_points[0] = fgd->num_uv_points[1] = 0; } else for (int pl = 0; pl < 2; pl++) { @@ -1132,12 +1114,12 @@ if (fgd->num_uv_points[pl]) { fgd->uv_mult[pl] = dav1d_get_bits(gb, 8) - 128; fgd->uv_luma_mult[pl] = dav1d_get_bits(gb, 8) - 128; fgd->uv_offset[pl] = dav1d_get_bits(gb, 9) - 256; } - fgd->overlap_flag = dav1d_get_bits(gb, 1); - fgd->clip_to_restricted_range = dav1d_get_bits(gb, 1); + fgd->overlap_flag = dav1d_get_bit(gb); + fgd->clip_to_restricted_range = dav1d_get_bit(gb); } } else { memset(&hdr->film_grain.data, 0, sizeof(hdr->film_grain.data)); } #if DEBUG_FRAME_HDR @@ -1152,11 +1134,11 @@ return DAV1D_ERR(EINVAL); } static void parse_tile_hdr(Dav1dContext *const c, GetBits *const gb) { const int n_tiles = c->frame_hdr->tiling.cols * c->frame_hdr->tiling.rows; - const int have_tile_pos = n_tiles > 1 ? dav1d_get_bits(gb, 1) : 0; + const int have_tile_pos = n_tiles > 1 ? dav1d_get_bit(gb) : 0; if (have_tile_pos) { const int n_bits = c->frame_hdr->tiling.log2_cols + c->frame_hdr->tiling.log2_rows; c->tile[c->n_tile_data].start = dav1d_get_bits(gb, n_bits); @@ -1198,15 +1180,15 @@ int res; dav1d_init_get_bits(&gb, in->data, in->sz); // obu header - dav1d_get_bits(&gb, 1); // obu_forbidden_bit + dav1d_get_bit(&gb); // obu_forbidden_bit const enum Dav1dObuType type = dav1d_get_bits(&gb, 4); - const int has_extension = dav1d_get_bits(&gb, 1); - const int has_length_field = dav1d_get_bits(&gb, 1); - dav1d_get_bits(&gb, 1); // reserved + const int has_extension = dav1d_get_bit(&gb); + const int has_length_field = dav1d_get_bit(&gb); + dav1d_get_bit(&gb); // reserved int temporal_id = 0, spatial_id = 0; if (has_extension) { temporal_id = dav1d_get_bits(&gb, 3); spatial_id = dav1d_get_bits(&gb, 2); @@ -1249,11 +1231,10 @@ case DAV1D_OBU_SEQ_HDR: { Dav1dRef *ref = dav1d_ref_create_using_pool(c->seq_hdr_pool, sizeof(Dav1dSequenceHeader)); if (!ref) return DAV1D_ERR(ENOMEM); Dav1dSequenceHeader *seq_hdr = ref->data; - memset(seq_hdr, 0, sizeof(*seq_hdr)); if ((res = parse_seq_hdr(c, &gb, seq_hdr)) < 0) { dav1d_ref_dec(&ref); goto error; } if (check_for_overrun(c, &gb, init_bit_pos, len)) { @@ -1274,11 +1255,11 @@ c->mastering_display = NULL; c->content_light = NULL; dav1d_ref_dec(&c->mastering_display_ref); dav1d_ref_dec(&c->content_light_ref); for (int i = 0; i < 8; i++) { - if (c->refs[i].p.p.data[0]) + if (c->refs[i].p.p.frame_hdr) dav1d_thread_picture_unref(&c->refs[i].p); dav1d_ref_dec(&c->refs[i].segmap); dav1d_ref_dec(&c->refs[i].refmvs); dav1d_cdf_thread_unref(&c->cdf[i]); } @@ -1323,11 +1304,11 @@ c->n_tile_data = 0; c->n_tiles = 0; if (type != DAV1D_OBU_FRAME) { // This is actually a frame header OBU so read the // trailing bit and check for overrun. - dav1d_get_bits(&gb, 1); + dav1d_get_bit(&gb); if (check_for_overrun(c, &gb, init_bit_pos, len)) { c->frame_hdr = NULL; goto error; } } @@ -1423,11 +1404,11 @@ content_light->max_frame_average_light_level, (gb.ptr - init_ptr) * 8 - gb.bits_left); #endif // Skip the trailing bit, align to the next byte boundary and check for overrun. - dav1d_get_bits(&gb, 1); + dav1d_get_bit(&gb); dav1d_bytealign_get_bits(&gb); if (check_for_overrun(c, &gb, init_bit_pos, len)) { dav1d_ref_dec(&ref); goto error; } @@ -1475,11 +1456,11 @@ printf("MDCVOBU: min-luminance: %d [off=%td]\n", mastering_display->min_luminance, (gb.ptr - init_ptr) * 8 - gb.bits_left); #endif // Skip the trailing bit, align to the next byte boundary and check for overrun. - dav1d_get_bits(&gb, 1); + dav1d_get_bit(&gb); dav1d_bytealign_get_bits(&gb); if (check_for_overrun(c, &gb, init_bit_pos, len)) { dav1d_ref_dec(&ref); goto error; } @@ -1554,10 +1535,24 @@ break; } if (c->seq_hdr && c->frame_hdr) { if (c->frame_hdr->show_existing_frame) { + if (!c->refs[c->frame_hdr->existing_frame_idx].p.p.frame_hdr) goto error; + switch (c->refs[c->frame_hdr->existing_frame_idx].p.p.frame_hdr->frame_type) { + case DAV1D_FRAME_TYPE_INTER: + case DAV1D_FRAME_TYPE_SWITCH: + if (c->decode_frame_type > DAV1D_DECODEFRAMETYPE_REFERENCE) + goto skip; + break; + case DAV1D_FRAME_TYPE_INTRA: + if (c->decode_frame_type > DAV1D_DECODEFRAMETYPE_INTRA) + goto skip; + // fall-through + default: + break; + } if (!c->refs[c->frame_hdr->existing_frame_idx].p.p.data[0]) goto error; if (c->strict_std_compliance && !c->refs[c->frame_hdr->existing_frame_idx].p.showable) { goto error; @@ -1618,11 +1613,11 @@ const int r = c->frame_hdr->existing_frame_idx; c->refs[r].p.showable = 0; for (int i = 0; i < 8; i++) { if (i == r) continue; - if (c->refs[i].p.p.data[0]) + if (c->refs[i].p.p.frame_hdr) dav1d_thread_picture_unref(&c->refs[i].p); dav1d_thread_picture_ref(&c->refs[i].p, &c->refs[r].p); dav1d_cdf_thread_unref(&c->cdf[i]); dav1d_cdf_thread_ref(&c->cdf[i], &c->cdf[r]); @@ -1634,10 +1629,27 @@ dav1d_ref_dec(&c->refs[i].refmvs); } } c->frame_hdr = NULL; } else if (c->n_tiles == c->frame_hdr->tiling.cols * c->frame_hdr->tiling.rows) { + switch (c->frame_hdr->frame_type) { + case DAV1D_FRAME_TYPE_INTER: + case DAV1D_FRAME_TYPE_SWITCH: + if (c->decode_frame_type > DAV1D_DECODEFRAMETYPE_REFERENCE || + (c->decode_frame_type == DAV1D_DECODEFRAMETYPE_REFERENCE && + !c->frame_hdr->refresh_frame_flags)) + goto skip; + break; + case DAV1D_FRAME_TYPE_INTRA: + if (c->decode_frame_type > DAV1D_DECODEFRAMETYPE_INTRA || + (c->decode_frame_type == DAV1D_DECODEFRAMETYPE_REFERENCE && + !c->frame_hdr->refresh_frame_flags)) + goto skip; + // fall-through + default: + break; + } if (!c->n_tile_data) goto error; if ((res = dav1d_submit_frame(c)) < 0) return res; assert(!c->n_tile_data); @@ -1646,10 +1658,30 @@ } } return len + init_byte_pos; +skip: + // update refs with only the headers in case we skip the frame + for (int i = 0; i < 8; i++) { + if (c->frame_hdr->refresh_frame_flags & (1 << i)) { + dav1d_thread_picture_unref(&c->refs[i].p); + c->refs[i].p.p.frame_hdr = c->frame_hdr; + c->refs[i].p.p.seq_hdr = c->seq_hdr; + c->refs[i].p.p.frame_hdr_ref = c->frame_hdr_ref; + c->refs[i].p.p.seq_hdr_ref = c->seq_hdr_ref; + dav1d_ref_inc(c->frame_hdr_ref); + dav1d_ref_inc(c->seq_hdr_ref); + } + } + + dav1d_ref_dec(&c->frame_hdr_ref); + c->frame_hdr = NULL; + c->n_tiles = 0; + + return len + init_byte_pos; + error: dav1d_data_props_copy(&c->cached_error_props, &in->m); dav1d_log(c, "Error parsing OBU data\n"); return DAV1D_ERR(EINVAL); } diff --git a/third_party/dav1d/src/picture.c b/third_party/dav1d/src/picture.c --- a/third_party/dav1d/src/picture.c +++ b/third_party/dav1d/src/picture.c @@ -231,17 +231,17 @@ validate_input(src != NULL); if (src->ref) { validate_input(src->data[0] != NULL); dav1d_ref_inc(src->ref); - if (src->frame_hdr_ref) dav1d_ref_inc(src->frame_hdr_ref); - if (src->seq_hdr_ref) dav1d_ref_inc(src->seq_hdr_ref); - if (src->m.user_data.ref) dav1d_ref_inc(src->m.user_data.ref); - if (src->content_light_ref) dav1d_ref_inc(src->content_light_ref); - if (src->mastering_display_ref) dav1d_ref_inc(src->mastering_display_ref); - if (src->itut_t35_ref) dav1d_ref_inc(src->itut_t35_ref); } + if (src->frame_hdr_ref) dav1d_ref_inc(src->frame_hdr_ref); + if (src->seq_hdr_ref) dav1d_ref_inc(src->seq_hdr_ref); + if (src->m.user_data.ref) dav1d_ref_inc(src->m.user_data.ref); + if (src->content_light_ref) dav1d_ref_inc(src->content_light_ref); + if (src->mastering_display_ref) dav1d_ref_inc(src->mastering_display_ref); + if (src->itut_t35_ref) dav1d_ref_inc(src->itut_t35_ref); *dst = *src; } void dav1d_picture_move_ref(Dav1dPicture *const dst, Dav1dPicture *const src) { validate_input(dst != NULL); @@ -280,17 +280,17 @@ validate_input(p != NULL); if (p->ref) { validate_input(p->data[0] != NULL); dav1d_ref_dec(&p->ref); - dav1d_ref_dec(&p->seq_hdr_ref); - dav1d_ref_dec(&p->frame_hdr_ref); - dav1d_ref_dec(&p->m.user_data.ref); - dav1d_ref_dec(&p->content_light_ref); - dav1d_ref_dec(&p->mastering_display_ref); - dav1d_ref_dec(&p->itut_t35_ref); } + dav1d_ref_dec(&p->seq_hdr_ref); + dav1d_ref_dec(&p->frame_hdr_ref); + dav1d_ref_dec(&p->m.user_data.ref); + dav1d_ref_dec(&p->content_light_ref); + dav1d_ref_dec(&p->mastering_display_ref); + dav1d_ref_dec(&p->itut_t35_ref); memset(p, 0, sizeof(*p)); dav1d_data_props_set_defaults(&p->m); } void dav1d_thread_picture_unref(Dav1dThreadPicture *const p) { diff --git a/third_party/dav1d/src/ppc/cdef_tmpl.c b/third_party/dav1d/src/ppc/cdef_tmpl.c --- a/third_party/dav1d/src/ppc/cdef_tmpl.c +++ b/third_party/dav1d/src/ppc/cdef_tmpl.c @@ -80,11 +80,19 @@ } vec_st(l0, 0, tmp + (h + 0) * 8); vec_st(l1, 0, tmp + (h + 1) * 8); - for (int y = 0; y < h; y++) { + int y_with_left_edge = 0; + if (!(edges & CDEF_HAVE_LEFT)) { + u16x8 l = u8h_to_u16(vec_vsx_ld(0, src)); + vec_vsx_st(l, 0, tmp + 2); + + y_with_left_edge = 1; + } + + for (int y = y_with_left_edge; y < h; y++) { u16x8 l = u8h_to_u16(vec_vsx_ld(0, src - 2 + y * src_stride)); vec_st(l, 0, tmp + y * 8); } if (!(edges & CDEF_HAVE_LEFT)) { @@ -158,11 +166,22 @@ vec_st(l0h, 0, tmp + (h + 0) * 16); vec_st(l0l, 0, tmp + (h + 0) * 16 + 8); vec_st(l1h, 0, tmp + (h + 1) * 16); vec_st(l1l, 0, tmp + (h + 1) * 16 + 8); - for (int y = 0; y < h; y++) { + int y_with_left_edge = 0; + if (!(edges & CDEF_HAVE_LEFT)) { + u8x16 l = vec_vsx_ld(0, src); + u16x8 lh = u8h_to_u16(l); + u16x8 ll = u8l_to_u16(l); + vec_vsx_st(lh, 0, tmp + 2); + vec_vsx_st(ll, 0, tmp + 8 + 2); + + y_with_left_edge = 1; + } + + for (int y = y_with_left_edge; y < h; y++) { u8x16 l = vec_vsx_ld(0, src - 2 + y * src_stride); u16x8 lh = u8h_to_u16(l); u16x8 ll = u8l_to_u16(l); vec_st(lh, 0, tmp + y * 16); vec_st(ll, 0, tmp + 8 + y * 16); @@ -454,11 +473,11 @@ const int sec_strength, \ const int dir, \ const int damping, \ const enum CdefEdgeFlags edges) \ { \ - ALIGN_STK_16(uint16_t, tmp_buf, 12 * tmp_stride,); \ + ALIGN_STK_16(uint16_t, tmp_buf, 12 * tmp_stride + 8,); \ uint16_t *tmp = tmp_buf + 2 * tmp_stride + 2; \ filter_##w##xN(dst, dst_stride, left, top, bottom, w, h, pri_strength, \ sec_strength, dir, damping, edges, tmp_stride, tmp); \ } diff --git a/third_party/dav1d/src/ppc/looprestoration_tmpl.c b/third_party/dav1d/src/ppc/looprestoration_tmpl.c --- a/third_party/dav1d/src/ppc/looprestoration_tmpl.c +++ b/third_party/dav1d/src/ppc/looprestoration_tmpl.c @@ -48,16 +48,16 @@ static void wiener_filter_h_vsx(int32_t *hor_ptr, uint8_t *tmp_ptr, const int16_t filterh[8], const int w, const int h) { - static const i32x4 zerov = vec_splats(0); - static const i32x4 seven_vec = vec_splats(7); - static const i32x4 bitdepth_added_vec = vec_splats(1 << 14); - static const i32x4 round_bits_vec = vec_splats(3); - static const i32x4 rounding_off_vec = vec_splats(1<<2); - static const i32x4 clip_limit_v = vec_splats((1 << 13) - 1); + const i32x4 zerov = vec_splats(0); + const i32x4 seven_vec = vec_splats(7); + const i32x4 bitdepth_added_vec = vec_splats(1 << 14); + const i32x4 round_bits_vec = vec_splats(3); + const i32x4 rounding_off_vec = vec_splats(1<<2); + const i32x4 clip_limit_v = vec_splats((1 << 13) - 1); i16x8 filterhvall = vec_vsx_ld(0, filterh); i16x8 filterhv0 = vec_splat( filterhvall, 0); i16x8 filterhv1 = vec_splat( filterhvall, 1); i16x8 filterhv2 = vec_splat( filterhvall, 2); @@ -126,12 +126,12 @@ hor_ptr += REST_UNIT_STRIDE; } } static inline i16x8 iclip_u8_vec(i16x8 v) { - static const i16x8 zerov = vec_splats((int16_t)0); - static const i16x8 maxv = vec_splats((int16_t)255); + const i16x8 zerov = vec_splats((int16_t)0); + const i16x8 maxv = vec_splats((int16_t)255); v = vec_max(zerov, v); v = vec_min(maxv, v); return v; } @@ -173,12 +173,12 @@ const ptrdiff_t stride, const int32_t *hor, const int16_t filterv[8], const int w, const int h) { - static const i32x4 round_bits_vec = vec_splats(11); - static const i32x4 round_vec = vec_splats((1 << 10) - (1 << 18)); + const i32x4 round_bits_vec = vec_splats(11); + const i32x4 round_vec = vec_splats((1 << 10) - (1 << 18)); i32x4 filterv0 = vec_splats((int32_t) filterv[0]); i32x4 filterv1 = vec_splats((int32_t) filterv[1]); i32x4 filterv2 = vec_splats((int32_t) filterv[2]); i32x4 filterv3 = vec_splats((int32_t) filterv[3]); diff --git a/third_party/dav1d/src/x86/cdef_avx2.asm b/third_party/dav1d/src/x86/cdef_avx2.asm --- a/third_party/dav1d/src/x86/cdef_avx2.asm +++ b/third_party/dav1d/src/x86/cdef_avx2.asm @@ -394,51 +394,39 @@ %endif %endmacro %macro CDEF_FILTER 2 ; w, h INIT_YMM avx2 -cglobal cdef_filter_%1x%2_8bpc, 5, 10, 0, dst, stride, left, top, bot, \ +cglobal cdef_filter_%1x%2_8bpc, 5, 11, 0, dst, stride, left, top, bot, \ pri, sec, dir, damping, edge %assign stack_offset_entry stack_offset mov edged, edgem cmp edged, 0xf jne .border_block - PUSH r10 PUSH r11 + PUSH r12 %if %2 == 4 - %assign regs_used 12 - %if STACK_ALIGNMENT < 32 - PUSH r%+regs_used - %assign regs_used regs_used+1 - %endif +%assign regs_used 13 ALLOC_STACK 0x60, 16 pmovzxbw xm0, [leftq+1] vpermq m0, m0, q0110 psrldq m1, m0, 4 vpalignr m2, m0, m0, 12 movu [rsp+0x10], m0 movu [rsp+0x28], m1 movu [rsp+0x40], m2 %elif %1 == 4 - PUSH r12 - %assign regs_used 13 - %if STACK_ALIGNMENT < 32 - PUSH r%+regs_used - %assign regs_used regs_used+1 - %endif +%assign regs_used 14 + PUSH r13 ALLOC_STACK 8*2+%1*%2*1, 16 pmovzxwd m0, [leftq] mova [rsp+0x10], m0 %else - PUSH r12 +%assign regs_used 15 PUSH r13 - %assign regs_used 14 - %if STACK_ALIGNMENT < 32 - PUSH r%+regs_used - %assign regs_used regs_used+1 - %endif + PUSH r14 ALLOC_STACK 8*4+%1*%2*2+32, 16 lea r11, [strideq*3] movu xm4, [dstq+strideq*2] pmovzxwq m0, [leftq+0] pmovzxwq m1, [leftq+8] @@ -1207,15 +1195,11 @@ .border_block: DEFINE_ARGS dst, stride, left, top, bot, pri, sec, stride3, dst4, edge %define rstk rsp %assign stack_offset stack_offset_entry -%assign regs_used 10 -%if STACK_ALIGNMENT < 32 - PUSH r%+regs_used - %assign regs_used regs_used+1 -%endif +%assign regs_used 11 ALLOC_STACK 2*16+(%2+4)*32, 16 %define px rsp+2*16+2*32 pcmpeqw m14, m14 psllw m14, 15 ; 0x8000 diff --git a/third_party/dav1d/src/x86/ipred16_avx2.asm b/third_party/dav1d/src/x86/ipred16_avx2.asm --- a/third_party/dav1d/src/x86/ipred16_avx2.asm +++ b/third_party/dav1d/src/x86/ipred16_avx2.asm @@ -1934,15 +1934,11 @@ ret ALIGN function_align .upsample_left: ; h4/h8 mova xm0, [tlq-16] ; 8 7 6 5 4 3 2 1 movu xm1, [tlq-14] ; 7 6 5 4 3 2 1 0 -%if STACK_ALIGNMENT < 32 vpbroadcastw xm4, r8m ; pixel_max -%else - vpbroadcastw xm4, r9m ; r8m -> r9m due to call -%endif cmp hd, 8 je .upsample_left_h8 pshufhw xm2, xm0, q2100 ; _ _ _ _ 4 4 3 2 pshufhw xm3, xm1, q3321 ; _ _ _ _ 2 1 0 0 jmp .upsample_left_end diff --git a/third_party/dav1d/src/x86/itx.h b/third_party/dav1d/src/x86/itx.h --- a/third_party/dav1d/src/x86/itx.h +++ b/third_party/dav1d/src/x86/itx.h @@ -1,8 +1,8 @@ /* - * Copyright © 2018-2021, VideoLAN and dav1d authors - * Copyright © 2018, Two Orioles, LLC + * Copyright © 2018-2023, VideoLAN and dav1d authors + * Copyright © 2018-2023, Two Orioles, LLC * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * @@ -344,11 +344,16 @@ assign_itx1_fn ( , 64, 64, avx512icl); #else if (bpc == 10) { assign_itx16_bpc_fn( , 8, 8, 10, avx512icl); assign_itx16_bpc_fn(R, 8, 16, 10, avx512icl); + assign_itx2_bpc_fn (R, 8, 32, 10, avx512icl); assign_itx16_bpc_fn(R, 16, 8, 10, avx512icl); assign_itx12_bpc_fn( , 16, 16, 10, avx512icl); + assign_itx2_bpc_fn (R, 16, 32, 10, avx512icl); + assign_itx2_bpc_fn (R, 32, 8, 10, avx512icl); + assign_itx2_bpc_fn (R, 32, 16, 10, avx512icl); + assign_itx2_bpc_fn ( , 32, 32, 10, avx512icl); } #endif #endif } diff --git a/third_party/dav1d/src/x86/itx16_avx2.asm b/third_party/dav1d/src/x86/itx16_avx2.asm --- a/third_party/dav1d/src/x86/itx16_avx2.asm +++ b/third_party/dav1d/src/x86/itx16_avx2.asm @@ -89,10 +89,12 @@ pd_6144: dd 6144 ; 2048 + 4096 pd_17408: dd 17408 ; 1024 + 16384 pixel_10bpc_max: times 2 dw 0x03ff pixel_12bpc_max: times 2 dw 0x0fff +dconly_10bpc: times 2 dw 0x7c00 +dconly_12bpc: times 2 dw 0x7000 clip_18b_min: dd -0x20000 clip_18b_max: dd 0x1ffff clip_20b_min: dd -0x80000 clip_20b_max: dd 0x7ffff @@ -284,32 +286,31 @@ %endmacro %macro INV_TXFM_4X4_FN 2-3 10 ; type1, type2, bitdepth INV_TXFM_FN %1, %2, 0, 4x4, %3 %ifidn %1_%2, dct_dct - vpbroadcastd xm3, [pixel_%3bpc_max] + vpbroadcastd xm2, [dconly_%3bpc] %if %3 = 10 .dconly: imul r6d, [cq], 181 mov [cq], eobd ; 0 - mov r3d, 4 + or r3d, 4 .dconly2: add r6d, 128 sar r6d, 8 .dconly3: imul r6d, 181 add r6d, 2176 sar r6d, 12 movd xm0, r6d + paddsw xm0, xm2 vpbroadcastw xm0, xm0 - pxor xm2, xm2 .dconly_loop: movq xm1, [dstq+strideq*0] movhps xm1, [dstq+strideq*1] - paddw xm1, xm0 - pmaxsw xm1, xm2 - pminsw xm1, xm3 + paddsw xm1, xm0 + psubusw xm1, xm2 movq [dstq+strideq*0], xm1 movhps [dstq+strideq*1], xm1 lea dstq, [dstq+strideq*2] sub r3d, 2 jg .dconly_loop @@ -718,16 +719,16 @@ jmp m(iadst_4x4_internal_12bpc).end %macro INV_TXFM_4X8_FN 2-3 10 ; type1, type2, bitdepth INV_TXFM_FN %1, %2, 0, 4x8, %3 %ifidn %1_%2, dct_dct - vpbroadcastd xm3, [pixel_%3bpc_max] + vpbroadcastd xm2, [dconly_%3bpc] %if %3 = 10 .dconly: imul r6d, [cq], 181 mov [cq], eobd ; 0 - mov r3d, 8 + or r3d, 8 add r6d, 128 sar r6d, 8 imul r6d, 181 jmp m(inv_txfm_add_dct_dct_4x4_10bpc).dconly2 %else @@ -1214,14 +1215,14 @@ RET %macro INV_TXFM_4X16_FN 2-3 10 ; type1, type2, bitdepth INV_TXFM_FN %1, %2, 0, 4x16, %3 %ifidn %1_%2, dct_dct - vpbroadcastd xm3, [pixel_%3bpc_max] imul r6d, [cq], 181 + vpbroadcastd xm2, [dconly_%3bpc] mov [cq], eobd ; 0 - mov r3d, 16 + or r3d, 16 add r6d, 384 sar r6d, 9 jmp m(inv_txfm_add_dct_dct_4x4_10bpc).dconly3 %endif %endmacro @@ -1894,16 +1895,16 @@ RET %macro INV_TXFM_8X4_FN 2-3 10 ; type1, type2, bitdepth INV_TXFM_FN %1, %2, 0, 8x4, %3 %ifidn %1_%2, dct_dct - vpbroadcastd m3, [pixel_%3bpc_max] + vpbroadcastd m2, [dconly_%3bpc] %if %3 = 10 .dconly: imul r6d, [cq], 181 mov [cq], eobd ; 0 - mov r3d, 4 + or r3d, 4 add r6d, 128 sar r6d, 8 imul r6d, 181 add r6d, 128 sar r6d, 8 @@ -2259,32 +2260,31 @@ jmp m(iidentity_8x4_internal_10bpc).pass2_end %macro INV_TXFM_8X8_FN 2-3 10 ; type1, type2, bitdepth INV_TXFM_FN %1, %2, 0, 8x8, %3 %ifidn %1_%2, dct_dct - vpbroadcastd m3, [pixel_%3bpc_max] + vpbroadcastd m2, [dconly_%3bpc] %if %3 = 10 .dconly: imul r6d, [cq], 181 mov [cq], eobd ; 0 - mov r3d, 8 + or r3d, 8 .dconly2: add r6d, 384 sar r6d, 9 .dconly3: imul r6d, 181 add r6d, 2176 sar r6d, 12 movd xm0, r6d + paddsw xm0, xm2 vpbroadcastw m0, xm0 - pxor m2, m2 .dconly_loop: mova xm1, [dstq+strideq*0] vinserti128 m1, [dstq+strideq*1], 1 - paddw m1, m0 - pmaxsw m1, m2 - pminsw m1, m3 + paddsw m1, m0 + psubusw m1, m2 mova [dstq+strideq*0], xm1 vextracti128 [dstq+strideq*1], m1, 1 lea dstq, [dstq+strideq*2] sub r3d, 2 jg .dconly_loop @@ -2796,14 +2796,14 @@ jmp m(iidentity_8x8_internal_10bpc).pass2_main %macro INV_TXFM_8X16_FN 2-4 0,10 ; type1, type2, eob_offset, bitdepth INV_TXFM_FN %1, %2, %3, 8x16, %4 %ifidn %1_%2, dct_dct - vpbroadcastd m3, [pixel_%4bpc_max] imul r6d, [cq], 181 + vpbroadcastd m2, [dconly_%4bpc] mov [cq], eobd ; 0 - mov r3d, 16 + or r3d, 16 add r6d, 128 sar r6d, 8 imul r6d, 181 jmp m(inv_txfm_add_dct_dct_8x8_10bpc).dconly2 %endif @@ -3441,33 +3441,31 @@ ret %macro INV_TXFM_16X4_FN 2-3 10 ; type1, type2, bitdepth INV_TXFM_FN %1, %2, 0, 16x4, %3 %ifidn %1_%2, dct_dct - vpbroadcastd m4, [pixel_%3bpc_max] + vpbroadcastd m3, [dconly_%3bpc] %if %3 = 10 .dconly: imul r6d, [cq], 181 mov [cq], eobd ; 0 - mov r3d, 4 + or r3d, 4 .dconly2: add r6d, 384 sar r6d, 9 .dconly3: imul r6d, 181 add r6d, 2176 sar r6d, 12 movd xm0, r6d + paddsw xm0, xm3 vpbroadcastw m0, xm0 - pxor m3, m3 .dconly_loop: - paddw m1, m0, [dstq+strideq*0] - paddw m2, m0, [dstq+strideq*1] - pmaxsw m1, m3 - pmaxsw m2, m3 - pminsw m1, m4 - pminsw m2, m4 + paddsw m1, m0, [dstq+strideq*0] + paddsw m2, m0, [dstq+strideq*1] + psubusw m1, m3 + psubusw m2, m3 mova [dstq+strideq*0], m1 mova [dstq+strideq*1], m2 lea dstq, [dstq+strideq*2] sub r3d, 2 jg .dconly_loop @@ -3961,14 +3959,14 @@ jmp m(idct_16x4_internal_10bpc).end2 %macro INV_TXFM_16X8_FN 2-3 10 ; type1, type2, bitdepth INV_TXFM_FN %1, %2, 0, 16x8, %3 %ifidn %1_%2, dct_dct - vpbroadcastd m4, [pixel_%3bpc_max] imul r6d, [cq], 181 + vpbroadcastd m3, [dconly_%3bpc] mov [cq], eobd ; 0 - mov r3d, 8 + or r3d, 8 add r6d, 128 sar r6d, 8 imul r6d, 181 jmp m(inv_txfm_add_dct_dct_16x4_10bpc).dconly2 %endif @@ -4600,14 +4598,14 @@ jmp m(idct_16x8_internal_10bpc).end2 %macro INV_TXFM_16X16_FN 2-4 0,10 ; type1, type2, eob_offset, bitdepth INV_TXFM_FN %1, %2, %3, 16x16, %4 %ifidn %1_%2, dct_dct - vpbroadcastd m4, [pixel_%4bpc_max] imul r6d, [cq], 181 + vpbroadcastd m3, [dconly_%4bpc] mov [cq], eobd ; 0 - mov r3d, 16 + or r3d, 16 add r6d, 640 sar r6d, 10 jmp m(inv_txfm_add_dct_dct_16x4_10bpc).dconly3 %endif %endmacro @@ -5758,14 +5756,14 @@ pmulhrsw m0, m12 pmulhrsw m1, m12 call m(idct_8x8_internal_10bpc).write_8x4 RET .dconly: - vpbroadcastd m3, [pixel_10bpc_max] imul r6d, [cq], 181 + vpbroadcastd m2, [dconly_10bpc] mov [cq], eobd ; 0 - mov r3d, 32 + or r3d, 32 add r6d, 640 sar r6d, 10 jmp m(inv_txfm_add_dct_dct_8x8_10bpc).dconly3 ALIGN function_align .pass1_main_part1: @@ -6304,14 +6302,14 @@ vpermq m0, [r5+32*0], q3120 vpermq m1, [r5+32*2], q3120 call m(idct_8x8_internal_10bpc).write_8x4 RET .dconly: - vpbroadcastd m3, [pixel_12bpc_max] imul r6d, [cq], 181 + vpbroadcastd m2, [dconly_12bpc] mov [cq], eobd ; 0 - mov r3d, 32 + or r3d, 32 add r6d, 640 sar r6d, 10 jmp m(inv_txfm_add_dct_dct_8x8_10bpc).dconly3 ALIGN function_align .pass1_main: @@ -6375,30 +6373,28 @@ cglobal inv_txfm_add_dct_dct_32x8_10bpc, 4, 7, 0, dst, stride, c, eob test eobd, eobd jnz .full imul r6d, [cq], 181 + vpbroadcastd m3, [dconly_10bpc] mov [cq], eobd ; 0 - mov r3d, 8 - vpbroadcastd m4, [pixel_10bpc_max] + or r3d, 8 .dconly: add r6d, 640 sar r6d, 10 .dconly2: imul r6d, 181 add r6d, 2176 sar r6d, 12 movd xm0, r6d + paddsw xm0, xm3 vpbroadcastw m0, xm0 - pxor m3, m3 .dconly_loop: - paddw m1, m0, [dstq+32*0] - paddw m2, m0, [dstq+32*1] - pmaxsw m1, m3 - pmaxsw m2, m3 - pminsw m1, m4 - pminsw m2, m4 + paddsw m1, m0, [dstq+32*0] + paddsw m2, m0, [dstq+32*1] + psubusw m1, m3 + psubusw m2, m3 mova [dstq+32*0], m1 mova [dstq+32*1], m2 add dstq, strideq dec r3d jg .dconly_loop @@ -6511,13 +6507,13 @@ cglobal inv_txfm_add_dct_dct_32x8_12bpc, 4, 7, 0, dst, stride, c, eob test eobd, eobd jnz .full imul r6d, [cq], 181 + vpbroadcastd m3, [dconly_12bpc] mov [cq], eobd ; 0 - mov r3d, 8 - vpbroadcastd m4, [pixel_12bpc_max] + or r3d, 8 jmp m(inv_txfm_add_dct_dct_32x8_10bpc).dconly .full: PROLOGUE 0, 7, 16, 32*24, dst, stride, c, eob lea r6, [rsp+32*4] vpbroadcastd m12, [clip_20b_min] @@ -6591,14 +6587,14 @@ pxor m4, m4 REPX {mova x, m4}, m5, m6, m7 REPX {mova [r6+32*x], m4}, 0, 1, 2, 3 jmp .fast .dconly: - vpbroadcastd m4, [pixel_10bpc_max] imul r6d, [cq], 181 + vpbroadcastd m3, [dconly_10bpc] mov [cq], eobd ; 0 - mov r3d, 32 + or r3d, 32 add r6d, 128 sar r6d, 8 imul r6d, 181 jmp m(inv_txfm_add_dct_dct_16x4_10bpc).dconly2 .eob44: @@ -6944,18 +6940,18 @@ pxor m8, m8 REPX {mova x, m8}, m9, m10, m11, m12, m13, m14, [rsp] jmp .end .dconly: imul r6d, [cq], 181 + vpbroadcastd m3, [dconly_10bpc] mov [cq], eobd ; 0 - mov r3d, 16 + or r3d, 16 add r6d, 128 sar r6d, 8 imul r6d, 181 add r6d, 384 sar r6d, 9 - vpbroadcastd m4, [pixel_10bpc_max] jmp m(inv_txfm_add_dct_dct_32x8_10bpc).dconly2 .full: add cq, 32 mova [r4+32*3], m0 mova [r4+32*2], m1 @@ -7215,13 +7211,13 @@ jl .fast call .main jmp .pass2 .dconly: imul r6d, [cq], 181 + vpbroadcastd m3, [dconly_10bpc] mov [cq], eobd ; 0 - mov r3d, 32 - vpbroadcastd m4, [pixel_10bpc_max] + or r3d, 32 jmp m(inv_txfm_add_dct_dct_32x8_10bpc).dconly .fast: lea r4, [rsp+32*71] pxor m0, m0 .fast_loop: @@ -7492,14 +7488,14 @@ sub eobd, 128 jl .fast call .main jmp .pass2 .dconly: - vpbroadcastd m4, [pixel_10bpc_max] imul r6d, [cq], 181 + vpbroadcastd m3, [dconly_10bpc] mov [cq], eobd ; 0 - mov r3d, 64 + or r3d, 64 add r6d, 640 sar r6d, 10 jmp m(inv_txfm_add_dct_dct_16x4_10bpc).dconly3 .fast: lea r4, [rsp+32*38] @@ -7833,18 +7829,18 @@ jl .fast call .main jmp .pass2 .dconly: imul r6d, [cq], 181 + vpbroadcastd m3, [dconly_10bpc] mov [cq], eobd ; 0 - mov r3d, 64 + or r3d, 64 add r6d, 128 sar r6d, 8 imul r6d, 181 add r6d, 384 sar r6d, 9 - vpbroadcastd m4, [pixel_10bpc_max] jmp m(inv_txfm_add_dct_dct_32x8_10bpc).dconly2 .fast: lea r4, [rsp+32*70] pxor m0, m0 .fast_loop: @@ -8017,42 +8013,35 @@ cglobal inv_txfm_add_dct_dct_64x16_10bpc, 4, 7, 0, dst, stride, c, eob test eobd, eobd jnz .normal imul r6d, [cq], 181 mov [cq], eobd ; 0 - mov r3d, 16 + or r3d, 16 .dconly: add r6d, 640 sar r6d, 10 .dconly2: + vpbroadcastd m5, [dconly_10bpc] imul r6d, 181 add r6d, 2176 sar r6d, 12 movd xm0, r6d -%if WIN64 - movaps [rsp+8], xmm6 -%endif + paddsw xm0, xm5 vpbroadcastw m0, xm0 - vpbroadcastd m6, [pixel_10bpc_max] - pxor m5, m5 .dconly_loop: - paddw m1, m0, [dstq+32*0] - paddw m2, m0, [dstq+32*1] - paddw m3, m0, [dstq+32*2] - paddw m4, m0, [dstq+32*3] - REPX {pmaxsw x, m5}, m1, m2, m3, m4 - REPX {pminsw x, m6}, m1, m2, m3, m4 + paddsw m1, m0, [dstq+32*0] + paddsw m2, m0, [dstq+32*1] + paddsw m3, m0, [dstq+32*2] + paddsw m4, m0, [dstq+32*3] + REPX {psubusw x, m5}, m1, m2, m3, m4 mova [dstq+32*0], m1 mova [dstq+32*1], m2 mova [dstq+32*2], m3 mova [dstq+32*3], m4 add dstq, strideq dec r3d jg .dconly_loop -%if WIN64 - movaps xmm6, [rsp+8] -%endif RET .normal: PROLOGUE 0, 8, 16, 32*96, dst, stride, c, eob %undef cmp vpbroadcastd m11, [pd_2048] @@ -8291,11 +8280,11 @@ call .main jmp .pass2 .dconly: imul r6d, [cq], 181 mov [cq], eobd ; 0 - mov r3d, 32 + or r3d, 32 add r6d, 128 sar r6d, 8 imul r6d, 181 add r6d, 384 sar r6d, 9 @@ -8440,11 +8429,11 @@ call .main jmp .pass2 .dconly: imul r6d, [cq], 181 mov [cq], eobd ; 0 - mov r3d, 64 + or r3d, 64 jmp m(inv_txfm_add_dct_dct_64x16_10bpc).dconly .fast: pxor m0, m0 lea r4, [rsp+32*135] .fast_loop: diff --git a/third_party/dav1d/src/x86/itx16_avx512.asm b/third_party/dav1d/src/x86/itx16_avx512.asm --- a/third_party/dav1d/src/x86/itx16_avx512.asm +++ b/third_party/dav1d/src/x86/itx16_avx512.asm @@ -1,7 +1,7 @@ -; Copyright © 2022, VideoLAN and dav1d authors -; Copyright © 2022, Two Orioles, LLC +; Copyright © 2022-2023, VideoLAN and dav1d authors +; Copyright © 2022-2023, Two Orioles, LLC ; All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions are met: ; @@ -56,10 +56,22 @@ db 7, 14, 9, 0, 3, 12, 0, 1, 15, 15, 11, 2, 11, 13, 2, 3 permC: db 0, 9, 0, 0, 0, 1, 4, 4, 2, 11, 2, 2, 2, 3, 6, 6 db 1, 8, 1, 8, 4, 5, 5, 12, 3, 10, 3, 10, 6, 7, 7, 14 db 9, 1, 8, 1, 1, 0, 12, 5, 11, 3, 10, 3, 3, 2, 14, 7 db 8, 0, 9, 9, 5, 4, 13, 13, 10, 2, 11, 11, 7, 6, 15, 15 +idct8x32p: db 0, 1, 4, 5, 16, 17, 20, 21, 32, 33, 36, 37, 48, 49, 52, 53 + db 8, 9, 12, 13, 24, 25, 28, 29, 40, 41, 44, 45, 56, 57, 60, 61 + db 2, 3, 6, 7, 18, 19, 22, 23, 34, 35, 38, 39, 50, 51, 54, 55 + db 10, 11, 14, 15, 26, 27, 30, 31, 42, 43, 46, 47, 58, 59, 62, 63 +idct32x8p: db 2, 18, 0, 16, 3, 19, 1, 17, 10, 26, 8, 24, 11, 27, 9, 25 + db 34, 50, 32, 48, 35, 51, 33, 49, 42, 58, 40, 56, 43, 59, 41, 57 + db 6, 22, 4, 20, 7, 23, 5, 21, 14, 30, 12, 28, 15, 31, 13, 29 + db 38, 54, 36, 52, 39, 55, 37, 53, 46, 62, 44, 60, 47, 63, 45, 61 +idtx32x8p: db 0, 8, 16, 24, 4, 12, 20, 28, 2, 10, 18, 26, 6, 14, 22, 30 + db 32, 40, 48, 56, 36, 44, 52, 60, 34, 42, 50, 58, 38, 46, 54, 62 + db 1, 9, 17, 25, 5, 13, 21, 29, 3, 11, 19, 27, 7, 15, 23, 31 + db 33, 41, 49, 57, 37, 45, 53, 61, 35, 43, 51, 59, 39, 47, 55, 63 pw_2048_m2048: times 16 dw 2048 pw_m2048_2048: times 16 dw -2048 pw_2048: times 16 dw 2048 @@ -88,10 +100,11 @@ COEF_PAIR 401, 1189, 1 COEF_PAIR 401, 1931 COEF_PAIR 401, 3920 COEF_PAIR 799, 2276, 1 COEF_PAIR 799, 3406 +COEF_PAIR 799, 4017 COEF_PAIR 1380, 601 COEF_PAIR 1751, 2440 COEF_PAIR 2598, 1189 COEF_PAIR 2598, 1931, 2 COEF_PAIR 2598, 3612 @@ -109,14 +122,18 @@ COEF_PAIR 4076, 1189 COEF_PAIR 4076, 3612 COEF_PAIR 4076, 3920 COEF_PAIR 4091, 3973 -pw_4096 times 2 dw 4096 +pb_32: times 4 db 32 +pw_5: times 2 dw 5 +pw_4096: times 2 dw 4096 +pw_8192: times 2 dw 8192 pw_1697x16: times 2 dw 1697*16 pw_2896x8: times 2 dw 2896*8 pixel_10bpc_max: times 2 dw 0x03ff +dconly_10bpc: times 2 dw 0x7c00 clip_18b_min: dd -0x20000 clip_18b_max: dd 0x1ffff pd_1: dd 1 pd_2: dd 2 pd_1448: dd 1448 @@ -125,19 +142,40 @@ pd_3072: dd 3072 ; 1024 + 2048 pd_5119: dd 5119 ; 1024 + 4096 - 1 pd_5120: dd 5120 ; 1024 + 4096 pd_5793: dd 5793 +cextern dup16_perm cextern int8_permA cextern idct_8x8_internal_8bpc_avx512icl.main cextern iadst_8x8_internal_8bpc_avx512icl.main_pass2 +cextern idct_8x16_internal_8bpc_avx512icl.main cextern idct_8x16_internal_8bpc_avx512icl.main2 +cextern idct_8x16_internal_8bpc_avx512icl.main_fast +cextern idct_8x16_internal_8bpc_avx512icl.main_fast2 cextern iadst_8x16_internal_8bpc_avx512icl.main2 cextern idct_16x8_internal_8bpc_avx512icl.main cextern iadst_16x8_internal_8bpc_avx512icl.main_pass2 cextern idct_16x16_internal_8bpc_avx512icl.main +cextern idct_16x16_internal_8bpc_avx512icl.main2 +cextern idct_16x16_internal_8bpc_avx512icl.main_fast +cextern idct_16x16_internal_8bpc_avx512icl.main_fast2 cextern iadst_16x16_internal_8bpc_avx512icl.main_pass2b +cextern inv_txfm_add_dct_dct_8x32_8bpc_avx512icl.main +cextern inv_txfm_add_dct_dct_8x32_8bpc_avx512icl.main_fast +cextern inv_txfm_add_dct_dct_8x32_8bpc_avx512icl.main_fast2 +cextern inv_txfm_add_dct_dct_8x32_8bpc_avx512icl.main_end +cextern inv_txfm_add_dct_dct_16x32_8bpc_avx512icl.main_oddhalf +cextern inv_txfm_add_dct_dct_16x32_8bpc_avx512icl.main_oddhalf_fast +cextern inv_txfm_add_dct_dct_16x32_8bpc_avx512icl.main_oddhalf_fast2 +cextern inv_txfm_add_dct_dct_32x8_8bpc_avx512icl.main +cextern inv_txfm_add_dct_dct_32x16_8bpc_avx512icl.main_oddhalf +cextern inv_txfm_add_dct_dct_32x16_8bpc_avx512icl.main_oddhalf_fast +cextern inv_txfm_add_dct_dct_32x16_8bpc_avx512icl.main_oddhalf_fast2 +cextern inv_txfm_add_dct_dct_32x32_8bpc_avx512icl.main_oddhalf +cextern inv_txfm_add_dct_dct_32x32_8bpc_avx512icl.main_oddhalf_fast +cextern inv_txfm_add_dct_dct_32x32_8bpc_avx512icl.main_oddhalf_fast2 SECTION .text %define o_base (pw_2048+4*128) %define o_base_8bpc (int8_permA+64*18) @@ -222,35 +260,30 @@ %macro INV_TXFM_8X8_FN 2-3 0 ; type1, type2, eob_offset INV_TXFM_FN %1, %2, %3, 8x8 %ifidn %1_%2, dct_dct imul r6d, [cq], 181 mov [cq], eobd ; 0 - mov r3d, 8 + or r3d, 8 .dconly: add r6d, 384 sar r6d, 9 +.dconly2: + vpbroadcastd ym2, [o(dconly_10bpc)] imul r6d, 181 - vpbroadcastd m3, [o(pixel_10bpc_max)] - lea r2, [strideq*3] add r6d, 2176 sar r6d, 12 - vpbroadcastw m1, r6d - pxor m2, m2 + vpbroadcastw ym1, r6d + paddsw ym1, ym2 .dconly_loop: mova xm0, [dstq+strideq*0] vinserti32x4 ym0, [dstq+strideq*1], 1 - vinserti32x4 m0, [dstq+strideq*2], 2 - vinserti32x4 m0, [dstq+r2 ], 3 - paddw m0, m1 - pmaxsw m0, m2 - pminsw m0, m3 + paddsw ym0, ym1 + psubusw ym0, ym2 mova [dstq+strideq*0], xm0 vextracti32x4 [dstq+strideq*1], ym0, 1 - vextracti32x4 [dstq+strideq*2], m0, 2 - vextracti32x4 [dstq+r2 ], m0, 3 - lea dstq, [dstq+strideq*4] - sub r3d, 4 + lea dstq, [dstq+strideq*2] + sub r3d, 2 jg .dconly_loop RET %endif %endmacro @@ -325,11 +358,11 @@ vpbroadcastd m12, [o(pd_2896)] mova m3, m1 vpbroadcastd m11, [o(pd_1)] ret ALIGN function_align -.main_fast: +.main_fast: ; bottom half is zero vbroadcasti32x4 m3, [o(pd_4017_3406)] vbroadcasti32x4 m8, [o(pd_799_m2276)] vbroadcasti32x4 m2, [o(pd_2896_3784)] vbroadcasti32x4 m9, [o(pd_2896_1567)] pmulld m3, m1 ; t4a t5a @@ -354,10 +387,11 @@ pmulld m3, m12 paddd m0, m2, m1 ; dct4 out0 out1 psubd m2, m1 ; dct4 out3 out2 REPX {pmaxsd x, m14}, m8, m0, m2 REPX {pminsd x, m15}, m8, m0, m2 +.main3: pshufd m1, m3, q1032 paddd m3, m13 psubd m9, m3, m1 paddd m3, m1 psrad m9, 12 @@ -370,11 +404,11 @@ paddd m2, m11 psubd m3, m0, m1 ; out7 out6 paddd m0, m1 ; out0 out1 paddd m1, m2, m8 ; out3 out2 psubd m2, m8 ; out4 out5 - REPX {psrad x, 1}, m0, m2, m3, m1 + REPX {vpsravd x, m11}, m0, m2, m3, m1 ret INV_TXFM_8X8_FN adst, dct INV_TXFM_8X8_FN adst, flipadst INV_TXFM_8X8_FN adst, identity @@ -517,11 +551,11 @@ %macro INV_TXFM_8X16_FN 2-3 0 ; type1, type2, eob_offset INV_TXFM_FN %1, %2, %3, 8x16 %ifidn %1_%2, dct_dct imul r6d, [cq], 181 mov [cq], eobd ; 0 - mov r3d, 16 + or r3d, 16 add r6d, 128 sar r6d, 8 imul r6d, 181 jmp m(inv_txfm_add_dct_dct_8x8_10bpc).dconly %endif @@ -537,11 +571,10 @@ cmp eobd, 43 jl .fast call .load call .main call .main_end - REPX {psrad x, 1 }, m0, m1, m2, m3, m4, m5, m6, m7 .pass1_end: packssdw m0, m4 packssdw m1, m5 packssdw m2, m6 packssdw m3, m7 @@ -632,60 +665,90 @@ vpbroadcastd m13, [o(pd_2048)] pmulld m4, m12, [cq+64*4] pmulld m5, m12, [cq+64*5] pmulld m6, m12, [cq+64*6] pmulld m7, m12, [cq+64*7] - REPX {paddd x, m13}, m0, m1, m2, m3, m4, m5, m6, m7 - REPX {psrad x, 12 }, m0, m1, m2, m3, m4, m5, m6, m7 +.round: + REPX {paddd x, m13}, m0, m1, m2, m3 + REPX {psrad x, 12 }, m0, m1, m2, m3 + REPX {paddd x, m13}, m4, m5, m6, m7 + REPX {psrad x, 12 }, m4, m5, m6, m7 ret ALIGN function_align +.main_fast_rect2: + REPX {paddd x, m13}, m0, m1, m2, m3 + REPX {psrad x, 12 }, m0, m1, m2, m3 +.main_fast: + pmulld m0, m12 + pmulld m5, m3, [o(pd_2276)] {1to16} ; t5a + pmulld m3, [o(pd_3406)] {1to16} ; t6a + pmulld m7, m1, [o(pd_4017)] {1to16} ; t7a + pmulld m1, [o(pd_799)] {1to16} ; t4a + pmulld m6, m2, [o(pd_3784)] {1to16} ; t3 + pmulld m2, [o(pd_1567)] {1to16} ; t2 + paddd m0, m13 + psubd m5, m13, m5 + psrad m0, 12 ; t0 + mova m9, m0 ; t1 + jmp .main2 +.main_rect2: + call .round .main: - ITX_MULSUB_2D 5, 3, 8, 9, 10, 13, 3406, 2276 ; t5a t6a - ITX_MULSUB_2D 1, 7, 8, 9, 10, 13, 799, 4017 ; t4a t7a pmulld m0, m12 + ITX_MULSUB_2D 5, 3, 8, 9, 10, _, 3406, 2276 ; t5a t6a + ITX_MULSUB_2D 1, 7, 8, 9, 10, _, 799, 4017 ; t4a t7a + ITX_MULSUB_2D 2, 6, 8, 9, 10, _, 1567, 3784 ; t2 t3 pmulld m4, m12 + paddd m0, m13 + paddd m5, m13 + psubd m9, m0, m4 ; t1 + paddd m0, m4 ; t0 + psrad m9, 12 + psrad m0, 12 +.main2: + REPX {paddd x, m13}, m3, m1, m7 + REPX {psrad x, 12 }, m5, m1, m3, m7 paddd m8, m1, m5 ; t4 psubd m1, m5 ; t5a psubd m5, m7, m3 ; t6a paddd m7, m3 ; t7 pmaxsd m5, m14 pmaxsd m1, m14 + paddd m2, m13 + paddd m6, m13 pminsd m5, m15 pminsd m1, m15 pmulld m5, m12 pmulld m1, m12 - ITX_MULSUB_2D 2, 6, 3, 9, 10, 13, 1567, 3784 ; t2 t3 pmaxsd m8, m14 pmaxsd m7, m14 - paddd m0, m13 pminsd m8, m15 - psubd m3, m0, m4 paddd m5, m13 - paddd m0, m4 psubd m4, m5, m1 paddd m5, m1 - REPX {psrad x, 12 }, m3, m5, m0, m4 - paddd m1, m3, m2 ; dct4 out1 - psubd m2, m3, m2 ; dct4 out2 + REPX {psrad x, 12 }, m2, m6, m5, m4 + paddd m1, m9, m2 ; dct4 out1 + psubd m2, m9, m2 ; dct4 out2 psubd m3, m0, m6 ; dct4 out3 paddd m0, m6 ; dct4 out0 pminsd m6, m15, m7 REPX {pmaxsd x, m14}, m0, m1, m2, m3 REPX {pminsd x, m15}, m0, m1, m2, m3 ret .main_end: - vpbroadcastd m7, [o(pd_1)] + vpbroadcastd m11, [o(pd_1)] .main_end2: - REPX {paddd x, m7}, m0, m1, m2, m3 + REPX {paddd x, m11}, m0, m1, m2, m3 psubd m7, m0, m6 ; out7 paddd m0, m6 ; out0 psubd m6, m1, m5 ; out6 paddd m1, m5 ; out1 psubd m5, m2, m4 ; out5 paddd m2, m4 ; out2 psubd m4, m3, m8 ; out4 paddd m3, m8 ; out3 + REPX {vpsravd x, m11}, m0, m1, m2, m3, m4, m5, m6, m7 ret INV_TXFM_8X16_FN adst, dct INV_TXFM_8X16_FN adst, identity, 35 INV_TXFM_8X16_FN adst, flipadst @@ -940,29 +1003,29 @@ %macro INV_TXFM_16X8_FN 2-3 0 ; type1, type2, eob_offset INV_TXFM_FN %1, %2, %3, 16x8 %ifidn %1_%2, dct_dct imul r6d, [cq], 181 mov [cq], eobd ; 0 - mov r3d, 8 + or r3d, 8 +.dconly: add r6d, 128 sar r6d, 8 imul r6d, 181 add r6d, 384 sar r6d, 9 -.dconly: +.dconly2: + vpbroadcastd m2, [o(dconly_10bpc)] imul r6d, 181 - vpbroadcastd m3, [o(pixel_10bpc_max)] add r6d, 2176 sar r6d, 12 vpbroadcastw m1, r6d - pxor m2, m2 + paddsw m1, m2 .dconly_loop: mova ym0, [dstq+strideq*0] vinserti32x8 m0, [dstq+strideq*1], 1 - paddw m0, m1 - pmaxsw m0, m2 - pminsw m0, m3 + paddsw m0, m1 + psubusw m0, m2 mova [dstq+strideq*0], ym0 vextracti32x8 [dstq+strideq*1], m0, 1 lea dstq, [dstq+strideq*2] sub r3d, 2 jg .dconly_loop @@ -1024,95 +1087,93 @@ .pass1_end: call m(idct_8x16_internal_10bpc).main_end mova m8, [o(permA)] psrlq m9, m8, 8 .pass1_end2: - REPX {psrad x, 1 }, m0, m4, m1, m5, m2, m6, m3, m7 mova m10, m9 mova m11, m8 -.pass1_end3: call .transpose_16x8 jmp tx2q .pass2: lea r5, [o_base_8bpc] call m(idct_16x8_internal_8bpc).main movshdup m4, [permC] - vpbroadcastd m13, [pw_2048] + vpbroadcastd m11, [pw_2048] psrlq m5, m4, 8 - vpermq m0, m4, m0 - vpermq m1, m5, m1 - vpermq m2, m4, m2 - vpermq m3, m5, m3 .end: - vpbroadcastd m15, [pixel_10bpc_max] - pxor m14, m14 - pmulhrsw m8, m13, m0 - pmulhrsw m9, m13, m1 + vpbroadcastd m13, [pixel_10bpc_max] + pxor m12, m12 + vpermq m8, m4, m0 + vpermq m9, m5, m1 lea r6, [strideq*3] call .write_16x4 - pmulhrsw m8, m13, m2 - pmulhrsw m9, m13, m3 + vpermq m8, m4, m2 + vpermq m9, m5, m3 .write_16x4: + pmulhrsw m8, m11 + pmulhrsw m9, m11 +.write_16x4_noround: mova ym10, [dstq+strideq*0] vinserti32x8 m10, [dstq+strideq*1], 1 paddw m8, m10 mova ym10, [dstq+strideq*2] vinserti32x8 m10, [dstq+r6 ], 1 paddw m9, m10 - pmaxsw m8, m14 - pmaxsw m9, m14 - pminsw m8, m15 - pminsw m9, m15 + pmaxsw m8, m12 + pmaxsw m9, m12 + pminsw m8, m13 + pminsw m9, m13 mova [dstq+strideq*0], ym8 vextracti32x8 [dstq+strideq*1], m8, 1 mova [dstq+strideq*2], ym9 vextracti32x8 [dstq+r6 ], m9, 1 lea dstq, [dstq+strideq*4] ret ALIGN function_align -.main_fast: - vbroadcasti32x4 m6, [o(pd_4076_3920)] - vbroadcasti32x4 m3, [o(pd_401_m1189)] - vbroadcasti32x4 m5, [o(pd_m2598_1931)] - vbroadcasti32x4 m9, [o(pd_3166_3612)] - pmulld m6, m4 ; t15a t12a - pmulld m4, m3 ; t8a t11a - pmulld m5, m7 ; t9a t10a - pmulld m7, m9 ; t14a t13a +.main_fast: ; bottom half is zero + vbroadcasti32x4 m6, [o(pd_4076_3920)] + vbroadcasti32x4 m3, [o(pd_401_m1189)] + vbroadcasti32x4 m5, [o(pd_m2598_1931)] + vbroadcasti32x4 m9, [o(pd_3166_3612)] + pmulld m6, m4 ; t15a t12a + pmulld m4, m3 ; t8a t11a + pmulld m5, m7 ; t9a t10a + pmulld m7, m9 ; t14a t13a jmp .main2 .main: ITX_MULSUB_2D 4, 6, 3, 9, 10, _, 401_3920, 4076_1189 ITX_MULSUB_2D 5, 7, 3, 9, 10, _, 3166_1931, 2598_3612 .main2: - psubd m3, m0, m1 ; dct8 out7 out6 - paddd m0, m1 ; dct8 out0 out1 - paddd m1, m2, m8 ; dct8 out3 out2 - psubd m2, m8 ; dct8 out4 out5 REPX {paddd x, m13}, m4, m6, m5, m7 REPX {psrad x, 12 }, m4, m5, m6, m7 - paddd m8, m4, m5 ; t8 t11 + paddd m9, m4, m5 ; t8 t11 psubd m4, m5 ; t9 t10 psubd m5, m6, m7 ; t14 t13 paddd m6, m7 ; t15 t12 - REPX {pmaxsd x, m14}, m5, m4 - REPX {pminsd x, m15}, m5, m4 - vbroadcasti32x4 m7, [pd_3784_m3784] + REPX {pmaxsd x, m14}, m5, m4, m9, m6 + REPX {pminsd x, m15}, m5, m4, m9, m6 +.main3: + psubd m3, m0, m1 ; dct8 out7 out6 + paddd m0, m1 ; dct8 out0 out1 + vbroadcasti32x4 m7, [o(pd_3784_m3784)] pmulld m7, m5 - vpmulld m5, [pd_1567] {1to16} - vbroadcasti32x4 m9, [pd_1567_m1567] - pmulld m9, m4 - vpmulld m4, [pd_3784] {1to16} - REPX {pmaxsd x, m14}, m0, m1, m8, m6 - REPX {pminsd x, m15}, m0, m1, m8, m6 + vpmulld m5, [o(pd_1567)] {1to16} + paddd m1, m2, m8 ; dct8 out3 out2 + psubd m2, m8 ; dct8 out4 out5 + vbroadcasti32x4 m8, [o(pd_1567_m1567)] + pmulld m8, m4 + vpmulld m4, [o(pd_3784)] {1to16} + REPX {pmaxsd x, m14}, m0, m1 + REPX {pminsd x, m15}, m0, m1 paddd m7, m13 paddd m5, m13 - paddd m7, m9 + paddd m7, m8 psubd m5, m4 psrad m7, 12 ; t14a t10a psrad m5, 12 ; t9a t13a - punpckhqdq m4, m8, m7 - punpcklqdq m8, m5 + punpckhqdq m4, m9, m7 + punpcklqdq m8, m9, m5 punpckhqdq m5, m6, m5 punpcklqdq m6, m7 psubd m7, m8, m4 ; t11a t10 paddd m8, m4 ; t8a t9 psubd m4, m6, m5 ; t12a t13 @@ -1164,21 +1225,23 @@ psubd m3, m9, m3 paddd m4, m9, m5 psubd m5, m9, m6 paddd m6, m9, m7 psubd m7, m9, m8 +.pass1_end: mova m9, [o(permA)] psrlq m8, m9, 8 + REPX {psrad x, 1}, m0, m4, m1, m5, m2, m6, m3, m7 jmp m(idct_16x8_internal_10bpc).pass1_end2 .pass2: call .main_pass2 - vpermq m8, m13, m0 - vpermq m9, m13, m1 - call m(idct_16x8_internal_10bpc).write_16x4 - vpermq m8, m13, m2 - vpermq m9, m13, m3 - jmp m(idct_16x8_internal_10bpc).write_16x4 + vpermq m8, m11, m0 + vpermq m9, m11, m1 + call m(idct_16x8_internal_10bpc).write_16x4_noround + vpermq m8, m11, m2 + vpermq m9, m11, m3 + jmp m(idct_16x8_internal_10bpc).write_16x4_noround ALIGN function_align .main_pass1: vpbroadcastd m12, [o(pd_2896)] pmulld m2, m12, [cq+64*0] pmulld m7, m12, [cq+64*1] @@ -1308,15 +1371,15 @@ .main_pass2: lea r5, [o_base_8bpc] pshufd m4, m0, q1032 pshufd m5, m1, q1032 call m(iadst_16x8_internal_8bpc).main_pass2 - movshdup m13, [permC] + movshdup m11, [permC] pmulhrsw m0, m6 pmulhrsw m1, m6 - vpbroadcastd m15, [pixel_10bpc_max] - pxor m14, m14 + vpbroadcastd m13, [pixel_10bpc_max] + pxor m12, m12 lea r6, [strideq*3] ret INV_TXFM_16X8_FN flipadst, dct INV_TXFM_16X8_FN flipadst, identity, -21 @@ -1332,58 +1395,55 @@ psubd m2, m9, m6 psubd m6, m9, m1 paddd m1, m9, m7 paddd m7, m9, m0 psubd m0, m9, m8 - mova m9, [o(permA)] - psrlq m8, m9, 8 - jmp m(idct_16x8_internal_10bpc).pass1_end2 + jmp m(iadst_16x8_internal_10bpc).pass1_end .pass2: call m(iadst_16x8_internal_10bpc).main_pass2 - psrlq m13, 8 - vpermq m8, m13, m3 - vpermq m9, m13, m2 - call m(idct_16x8_internal_10bpc).write_16x4 - vpermq m8, m13, m1 - vpermq m9, m13, m0 - jmp m(idct_16x8_internal_10bpc).write_16x4 + psrlq m11, 8 + vpermq m8, m11, m3 + vpermq m9, m11, m2 + call m(idct_16x8_internal_10bpc).write_16x4_noround + vpermq m8, m11, m1 + vpermq m9, m11, m0 + jmp m(idct_16x8_internal_10bpc).write_16x4_noround INV_TXFM_16X8_FN identity, dct INV_TXFM_16X8_FN identity, adst INV_TXFM_16X8_FN identity, flipadst INV_TXFM_16X8_FN identity, identity cglobal iidentity_16x8_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 call m(idct_8x16_internal_10bpc).load2 vpbroadcastd m8, [o(pd_5793)] - vpbroadcastd m9, [o(pd_3072)] + vpbroadcastd m13, [o(pd_3072)] pxor m10, m10 REPX {pmulld x, m8}, m0, m1, m2, m3, m4, m5, m6, m7 - REPX {mova [cq+64*x], m10}, 0, 1, 2, 3 - REPX {paddd x, m9}, m0, m1, m2, m3, m4, m5, m6, m7 - REPX {mova [cq+64*x], m10}, 4, 5, 6, 7 - REPX {psrad x, 12}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {mova [cq+64*x], m10}, 0, 1, 2, 3, 4, 5, 6, 7 + call m(idct_8x16_internal_10bpc).round psrlq m8, [o(permA)], 16 psrlq m9, m8, 8 mova m10, m8 mova m11, m9 - jmp m(idct_16x8_internal_10bpc).pass1_end3 + call m(idct_16x8_internal_10bpc).transpose_16x8 + jmp tx2q .pass2: movshdup m4, [o(permC)] - vpbroadcastd m13, [o(pw_4096)] - REPX {vpermq x, m4, x}, m0, m1, m2, m3 + vpbroadcastd m11, [o(pw_4096)] + mova m5, m4 jmp m(idct_16x8_internal_10bpc).end %macro INV_TXFM_16X16_FN 2-3 0 ; type1, type2, eob_offset INV_TXFM_FN %1, %2, %3, 16x16 %ifidn %1_%2, dct_dct imul r6d, [cq], 181 mov [cq], eobd ; 0 - mov r3d, 16 + or r3d, 16 add r6d, 640 sar r6d, 10 - jmp m(inv_txfm_add_dct_dct_16x8_10bpc).dconly + jmp m(inv_txfm_add_dct_dct_16x8_10bpc).dconly2 %endif %endmacro INV_TXFM_16X16_FN dct, dct INV_TXFM_16X16_FN dct, identity, 28 @@ -1420,56 +1480,17 @@ mova m22, [cq+64*13] mova m23, [cq+64*15] call .main call .main_end .pass1_end: - packssdw m0, m16 - packssdw m1, m17 - packssdw m2, m18 - packssdw m3, m19 - packssdw m4, m20 - packssdw m5, m21 - packssdw m6, m22 - packssdw m7, m23 %if WIN64 movaps xmm6, [cq+16*0] movaps xmm7, [cq+16*1] %endif vzeroupper .pass1_end2: - punpckhwd m8, m0, m1 - punpcklwd m0, m1 - punpckhwd m1, m2, m3 - punpcklwd m2, m3 - punpckhwd m3, m4, m5 - punpcklwd m4, m5 - punpcklwd m5, m6, m7 - punpckhwd m6, m7 - punpckhdq m7, m0, m2 - punpckldq m0, m2 - punpckhdq m2, m8, m1 - punpckldq m8, m1 - punpckhdq m1, m4, m5 - punpckldq m4, m5 - punpckhdq m5, m3, m6 - punpckldq m3, m6 - vshufi32x4 m6, m0, m4, q3232 - vinserti32x8 m0, ym4, 1 - vinserti32x8 m4, m8, ym3, 1 - vshufi32x4 m8, m3, q3232 - vinserti32x8 m3, m7, ym1, 1 - vshufi32x4 m7, m1, q3232 - vshufi32x4 m1, m2, m5, q3232 - vinserti32x8 m2, ym5, 1 - vshufi32x4 m5, m7, m1, q2020 ; 10 11 - vshufi32x4 m7, m1, q3131 ; 14 15 - vshufi32x4 m1, m3, m2, q2020 ; 2 3 - vshufi32x4 m3, m2, q3131 ; 6 7 - vshufi32x4 m2, m0, m4, q3131 ; 4 5 - vshufi32x4 m0, m4, q2020 ; 0 1 - vshufi32x4 m4, m6, m8, q2020 ; 8 9 - vshufi32x4 m6, m8, q3131 ; 12 13 + call .main_end3 .pass1_end3: mov r6d, 64*12 pxor m8, m8 .zero_loop: mova [cq+r6+64*3], m8 @@ -1480,37 +1501,37 @@ jge .zero_loop jmp tx2q .pass2: lea r5, [o_base_8bpc] call m(idct_16x16_internal_8bpc).main - movshdup m10, [permC] - vpbroadcastd m13, [pw_2048] - psrlq m11, m10, 8 - vpermq m8, m10, m0 - vpermq m0, m11, m7 - vpermq m7, m11, m1 - vpermq m1, m10, m6 - vpermq m6, m10, m2 - vpermq m2, m11, m5 - vpermq m5, m11, m3 - vpermq m3, m10, m4 + movshdup m12, [permC] + vpbroadcastd m11, [pw_2048] + psrlq m13, m12, 8 + vpermq m8, m12, m0 + vpermq m0, m13, m7 + vpermq m7, m13, m1 + vpermq m1, m12, m6 + vpermq m6, m12, m2 + vpermq m2, m13, m5 + vpermq m5, m13, m3 + vpermq m3, m12, m4 .pass2_end: lea r6, [strideq*3] - vpbroadcastd m15, [pixel_10bpc_max] - pxor m14, m14 - pmulhrsw m8, m13, m8 - pmulhrsw m9, m13, m7 - call m(idct_16x8_internal_10bpc).write_16x4 - pmulhrsw m8, m13, m6 - pmulhrsw m9, m13, m5 - call m(idct_16x8_internal_10bpc).write_16x4 - pmulhrsw m8, m13, m3 - pmulhrsw m9, m13, m2 - call m(idct_16x8_internal_10bpc).write_16x4 - pmulhrsw m8, m13, m1 - pmulhrsw m9, m13, m0 - jmp m(idct_16x8_internal_10bpc).write_16x4 + vpbroadcastd m13, [pixel_10bpc_max] + pxor m12, m12 + pmulhrsw m8, m11, m8 + pmulhrsw m9, m11, m7 + call m(idct_16x8_internal_10bpc).write_16x4_noround + pmulhrsw m8, m11, m6 + pmulhrsw m9, m11, m5 + call m(idct_16x8_internal_10bpc).write_16x4_noround + pmulhrsw m8, m11, m3 + pmulhrsw m9, m11, m2 + call m(idct_16x8_internal_10bpc).write_16x4_noround + pmulhrsw m8, m11, m1 + pmulhrsw m9, m11, m0 + jmp m(idct_16x8_internal_10bpc).write_16x4_noround .fast: mova ym0, [cq+64*0] mova ym2, [cq+64*4] movshdup m8, [o(permB)] mova ym1, [cq+64*2] @@ -1523,21 +1544,41 @@ vpermt2q m1, m8, m3 ; 2 6 vpermt2q m4, m8, m5 ; 1 3 vpermt2q m7, m8, m6 ; 7 5 call m(idct_8x8_internal_10bpc).main_fast call m(idct_16x8_internal_10bpc).main_fast - vpbroadcastd m7, [o(pd_2)] + vpbroadcastd m11, [o(pd_2)] call m(idct_8x16_internal_10bpc).main_end2 mova m8, [o(permA)] psrlq m9, m8, 8 jmp m(iadst_16x16_internal_10bpc).pass1_fast_end2 ALIGN function_align +.main_fast_rect2: + REPX {paddd x, m13}, m16, m17, m18, m19 + REPX {psrad x, 12 }, m16, m17, m18, m19 +.main_fast: + pmulld m23, m16, [o(pd_4076)] {1to16} ; t15a + pmulld m16, [o(pd_401)] {1to16} ; t8a + pmulld m20, m19, [o(pd_2598)] {1to16} ; t9a + pmulld m19, [o(pd_3166)] {1to16} ; t14a + pmulld m22, m17, [o(pd_1189)] {1to16} ; t11a + pmulld m17, [o(pd_3920)] {1to16} ; t12a + pmulld m21, m18, [o(pd_3612)] {1to16} ; t13a + pmulld m18, [o(pd_1931)] {1to16} ; t10a + psubd m20, m13, m20 + psubd m22, m13, m22 + call .round2 + jmp .main2 +.main_rect2: + call .round .main: - ITX_MULSUB_2D 16, 23, 7, 9, 10, 13, 401, 4076 ; t8a, t15a - ITX_MULSUB_2D 20, 19, 7, 9, 10, 13, 3166, 2598 ; t9a, t14a - ITX_MULSUB_2D 22, 17, 7, 9, 10, 13, 3920, 1189 ; t11a, t12a - ITX_MULSUB_2D 18, 21, 7, 9, 10, 13, 1931, 3612 ; t10a, t13a + ITX_MULSUB_2D 16, 23, 7, 9, 10, _, 401, 4076 ; t8a, t15a + ITX_MULSUB_2D 20, 19, 7, 9, 10, _, 3166, 2598 ; t9a, t14a + ITX_MULSUB_2D 22, 17, 7, 9, 10, _, 3920, 1189 ; t11a, t12a + ITX_MULSUB_2D 18, 21, 7, 9, 10, _, 1931, 3612 ; t10a, t13a + call .round +.main2: paddd m9, m20, m16 ; t8 psubd m20, m16, m20 ; t9 psubd m16, m22, m18 ; t10 paddd m18, m22 ; t11 paddd m22, m23, m19 ; t15 @@ -1584,12 +1625,13 @@ paddd m19, m23 ; t12 REPX {pminsd x, m15}, m22, m21, m16, m9 REPX {psrad x, 12 }, m20, m19, m18, m17 ret .main_end: - vpbroadcastd m23, [o(pd_2)] - REPX {paddd x, m23}, m0, m1, m2, m3, m4, m5, m6, m7 + vpbroadcastd m11, [o(pd_2)] +.main_end2: + REPX {paddd x, m11}, m0, m1, m2, m3, m4, m5, m6, m7 psubd m23, m0, m22 ; out15 paddd m0, m22 ; out0 psubd m22, m1, m21 ; out14 paddd m1, m21 ; out1 psubd m21, m2, m20 ; out13 @@ -1602,12 +1644,66 @@ paddd m5, m17 ; out5 psubd m17, m6, m16 ; out9 paddd m6, m16 ; out6 psubd m16, m7, m9 ; out8 paddd m7, m9 ; out7 - REPX {psrad x, 2 }, m0, m16, m1, m17, m2, m18, m3, m19, \ + REPX {vpsravd x, m11}, m0, m16, m1, m17, m2, m18, m3, m19, \ m4, m20, m5, m21, m6, m22, m7, m23 + packssdw m0, m16 + packssdw m1, m17 + packssdw m2, m18 + packssdw m3, m19 + packssdw m4, m20 + packssdw m5, m21 + packssdw m6, m22 + packssdw m7, m23 + ret +.main_end3: + punpckhwd m8, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m2, m3 + punpcklwd m2, m3 + punpckhwd m3, m4, m5 + punpcklwd m4, m5 + punpcklwd m5, m6, m7 + punpckhwd m6, m7 + punpckhdq m7, m0, m2 + punpckldq m0, m2 + punpckhdq m2, m8, m1 + punpckldq m8, m1 + punpckhdq m1, m4, m5 + punpckldq m4, m5 + punpckhdq m5, m3, m6 + punpckldq m3, m6 + vshufi32x4 m6, m0, m4, q3232 + vinserti32x8 m0, ym4, 1 + vinserti32x8 m4, m8, ym3, 1 + vshufi32x4 m8, m3, q3232 + vinserti32x8 m3, m7, ym1, 1 + vshufi32x4 m7, m1, q3232 + vshufi32x4 m1, m2, m5, q3232 + vinserti32x8 m2, ym5, 1 + vshufi32x4 m5, m7, m1, q2020 ; 10 11 + vshufi32x4 m7, m1, q3131 ; 14 15 + vshufi32x4 m1, m3, m2, q2020 ; 2 3 + vshufi32x4 m3, m2, q3131 ; 6 7 + vshufi32x4 m2, m0, m4, q3131 ; 4 5 + vshufi32x4 m0, m4, q2020 ; 0 1 + vshufi32x4 m4, m6, m8, q2020 ; 8 9 + vshufi32x4 m6, m8, q3131 ; 12 13 + ret +ALIGN function_align +.round: + paddd m20, m13 + paddd m22, m13 +.round2: + paddd m16, m13 + paddd m18, m13 +.round3: + REPX {psrad x, 12 }, m16, m18, m20, m22 + REPX {paddd x, m13}, m17, m19, m21, m23 + REPX {psrad x, 12 }, m17, m19, m21, m23 ret INV_TXFM_16X16_FN adst, dct INV_TXFM_16X16_FN adst, flipadst INV_TXFM_16X16_FN adst, adst @@ -1615,26 +1711,18 @@ cglobal iadst_16x16_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 %undef cmp cmp eobd, 36 jl .fast call .main_pass1 - psrad m0, m1, 2 - psrad m16, 13 - psrad m1, m2, 2 - psrad m17, 13 - psrad m2, m3, 2 - psrad m18, 13 - psrad m3, m4, 2 - psrad m19, 13 - psrad m4, m5, 13 - psrad m20, 2 - psrad m5, m6, 13 - psrad m21, 2 - psrad m6, m7, 13 - psrad m22, 2 - psrad m7, m8, 13 - psrad m23, 2 + packssdw m0, m16 + packssdw m1, m17 + packssdw m2, m18 + packssdw m3, m19 + packssdw m4, m5, m20 + packssdw m5, m6, m21 + packssdw m6, m7, m22 + packssdw m7, m8, m23 jmp m(idct_16x16_internal_10bpc).pass1_end .fast: call .main_pass1_fast vpbroadcastd m9, [o(pd_2)] paddd m0, m9 @@ -1646,33 +1734,33 @@ paddd m6, m9, m7 psubd m7, m9, m8 .pass1_fast_end: mova m9, [o(permA)] psrlq m8, m9, 8 + REPX {psrad x, 2}, m0, m1, m2, m3, m4, m5, m6, m7 .pass1_fast_end2: - REPX {psrad x, 2}, m0, m1, m2, m3, m4, m5, m6, m7 mova m10, m9 mova m11, m8 call m(idct_16x8_internal_10bpc).transpose_16x8 pxor m4, m4 REPX {mova x, m4}, m5, m6, m7 REPX {mova [cq+64*x], ym4}, 0, 1, 2, 3, 4, 5, 6, 7 jmp tx2q .pass2: lea r5, [o_base_8bpc] call m(iadst_16x16_internal_8bpc).main_pass2b - movshdup m10, [permC] - mova m13, [pw_2048_m2048] - psrlq m11, m10, 8 - vpermq m8, m11, m0 - vpermq m0, m10, m7 - vpermq m7, m11, m1 - vpermq m1, m10, m6 - vpermq m6, m11, m2 - vpermq m2, m10, m5 - vpermq m5, m11, m3 - vpermq m3, m10, m4 + movshdup m12, [permC] + mova m11, [pw_2048_m2048] + psrlq m13, m12, 8 + vpermq m8, m13, m0 + vpermq m0, m12, m7 + vpermq m7, m13, m1 + vpermq m1, m12, m6 + vpermq m6, m13, m2 + vpermq m2, m12, m5 + vpermq m5, m13, m3 + vpermq m3, m12, m4 jmp m(idct_16x16_internal_10bpc).pass2_end ALIGN function_align .main_pass1: mova m0, [cq+64* 0] %if WIN64 @@ -1703,110 +1791,112 @@ mova m1, [cq+64* 1] mova m22, [cq+64*14] ITX_MULSUB_2D 1, 22, 8, 9, 10, 13, 4052, 601 ; t15 t14 vpbroadcastd m14, [o(clip_18b_min)] vpbroadcastd m15, [o(clip_18b_max)] + psubd m9, m23, m7 ; t9a + paddd m23, m7 ; t1a + psubd m7, m2, m18 ; t10a + paddd m18, m2 ; t2a + REPX {pmaxsd x, m14}, m9, m23, m7, m18 + psubd m2, m17, m1 ; t15a + paddd m17, m1 ; t7a + REPX {pminsd x, m15}, m9, m23, m7, m18 + psubd m1, m21, m5 ; t11a + paddd m21, m5 ; t3a + REPX {pmaxsd x, m14}, m2, m17, m1, m21 + psubd m5, m4, m20 ; t12a + paddd m4, m20 ; t4a + REPX {pminsd x, m15}, m2, m17, m1, m21 + psubd m20, m19, m3 ; t13a + paddd m19, m3 ; t5a + REPX {pmaxsd x, m14}, m5, m4, m20, m19 psubd m8, m6, m22 ; t14a paddd m6, m22 ; t6a + REPX {pminsd x, m15}, m5, m4, m20, m19 psubd m22, m0, m16 ; t8a paddd m16, m0 ; t0a REPX {pmaxsd x, m14}, m8, m6, m22, m16 - psubd m0, m23, m7 ; t9a - paddd m23, m7 ; t1a - REPX {pminsd x, m15}, m8, m6, m22, m16 - psubd m7, m2, m18 ; t10a - paddd m18, m2 ; t2a - REPX {pmaxsd x, m14}, m0, m23, m7, m18 - psubd m2, m21, m5 ; t11a - paddd m21, m5 ; t3a - REPX {pminsd x, m15}, m0, m23, m7, m18 - psubd m5, m4, m20 ; t12a - paddd m4, m20 ; t4a - REPX {pmaxsd x, m14}, m2, m21, m5, m4 - psubd m20, m19, m3 ; t13a - paddd m19, m3 ; t5a - REPX {pminsd x, m15}, m2, m21, m5, m4 - psubd m3, m17, m1 ; t15a - paddd m17, m1 ; t7a - REPX {pmaxsd x, m14}, m20, m19, m3, m17 vpbroadcastd m11, [o(pd_4017)] vpbroadcastd m10, [o(pd_799)] - REPX {pminsd x, m15}, m20, m19, m3, m17 - ITX_MULSUB_2D 22, 0, 1, 9, _, 13, 10, 11 ; t9 t8 - ITX_MULSUB_2D 20, 5, 1, 9, _, 13, 11, 10 ; t12 t13 + REPX {pminsd x, m15}, m8, m6, m22, m16 + ITX_MULSUB_2D 22, 9, 0, 3, _, 13, 10, 11 ; t9 t8 + ITX_MULSUB_2D 20, 5, 0, 3, _, 13, 11, 10 ; t12 t13 vpbroadcastd m11, [o(pd_2276)] vpbroadcastd m10, [o(pd_3406)] - ITX_MULSUB_2D 7, 2, 1, 9, _, 13, 10, 11 ; t11 t10 - ITX_MULSUB_2D 3, 8, 1, 9, _, 13, 11, 10 ; t14 t15 - paddd m1, m16, m4 ; t0 + ITX_MULSUB_2D 7, 1, 0, 3, _, 13, 10, 11 ; t11 t10 + ITX_MULSUB_2D 2, 8, 0, 3, _, 13, 11, 10 ; t14 t15 + paddd m0, m16, m4 ; t0 psubd m16, m4 ; t4 - psubd m4, m23, m19 ; t5 + psubd m3, m23, m19 ; t5 paddd m23, m19 ; t1 - REPX {pmaxsd x, m14}, m1, m16, m4, m23 + REPX {pmaxsd x, m14}, m0, m16, m3, m23 psubd m19, m18, m6 ; t6 paddd m18, m6 ; t2 - REPX {pminsd x, m15}, m1, m16, m4, m23 + REPX {pminsd x, m15}, m0, m16, m3, m23 psubd m6, m21, m17 ; t7 paddd m21, m17 ; t3 REPX {pmaxsd x, m14}, m19, m18, m6, m21 - paddd m17, m0, m20 ; t8a - psubd m0, m20 ; t12a + paddd m17, m9, m20 ; t8a + psubd m9, m20 ; t12a REPX {pminsd x, m15}, m19, m18, m6, m21 psubd m20, m22, m5 ; t13a paddd m22, m5 ; t9a - REPX {pmaxsd x, m14}, m17, m0, m20, m22 - psubd m5, m2, m3 ; t14a - paddd m2, m3 ; t10a - REPX {pminsd x, m15}, m17, m0, m20, m22 - psubd m3, m7, m8 ; t15a + REPX {pmaxsd x, m14}, m17, m9, m20, m22 + psubd m5, m1, m2 ; t14a + paddd m1, m2 ; t10a + REPX {pminsd x, m15}, m17, m9, m20, m22 + psubd m2, m7, m8 ; t15a paddd m7, m8 ; t11a - REPX {pmaxsd x, m14}, m5, m2, m3, m7 + REPX {pmaxsd x, m14}, m5, m1, m2, m7 vpbroadcastd m11, [o(pd_3784)] vpbroadcastd m10, [o(pd_1567)] - REPX {pminsd x, m15}, m5, m2, m3, m7 - ITX_MULSUB_2D 16, 4, 8, 9, _, 13, 10, 11 ; t5a t4a - ITX_MULSUB_2D 6, 19, 8, 9, _, 13, 11, 10 ; t6a t7a - ITX_MULSUB_2D 0, 20, 8, 9, _, 13, 10, 11 ; t13 t12 - ITX_MULSUB_2D 3, 5, 8, 9, _, 13, 11, 10 ; t14 t15 - psubd m8, m1, m18 ; t2a - paddd m1, m18 ; out0 + REPX {pminsd x, m15}, m5, m1, m2, m7 + ITX_MULSUB_2D 16, 3, 4, 8, _, 13, 10, 11 ; t5a t4a + ITX_MULSUB_2D 6, 19, 4, 8, _, 13, 11, 10 ; t6a t7a + ITX_MULSUB_2D 9, 20, 4, 8, _, 13, 10, 11 ; t13 t12 + ITX_MULSUB_2D 2, 5, 4, 8, _, 13, 11, 10 ; t14 t15 + psubd m8, m0, m18 ; t2a + paddd m0, m18 ; out0 psubd m18, m23, m21 ; t3a paddd m23, m21 ; -out15 - paddd m21, m0, m5 ; -out13 - psubd m0, m5 ; t15a - psubd m5, m4, m6 ; t6 - paddd m4, m6 ; -out3 - REPX {pmaxsd x, m14}, m8, m18, m0, m5 - psubd m6, m20, m3 ; t14a - paddd m3, m20 ; out2 + paddd m21, m9, m5 ; -out13 + psubd m9, m5 ; t15a + psubd m5, m3, m6 ; t6 + paddd m3, m6 ; -out3 + REPX {pmaxsd x, m14}, m8, m18, m9, m5 + psubd m6, m20, m2 ; t14a + paddd m2, m20 ; out2 paddd m20, m16, m19 ; out12 psubd m16, m19 ; t7 - REPX {pminsd x, m15}, m8, m18, m0, m5 + REPX {pminsd x, m15}, m8, m18, m9, m5 psubd m19, m22, m7 ; t11 paddd m22, m7 ; out14 - psubd m7, m17, m2 ; t10 - paddd m2, m17 ; -out1 + psubd m7, m17, m1 ; t10 + paddd m1, m17 ; -out1 REPX {pmaxsd x, m14}, m6, m16, m19, m7 vpbroadcastd m12, [o(pd_1448)] - vpbroadcastd m9, [o(pd_2)] + vpbroadcastd m4, [o(pd_2)] vpbroadcastd m10, [o(pd_5120)] vpbroadcastd m11, [o(pd_5119)] REPX {pminsd x, m15}, m6, m16, m19, m7 psubd m17, m7, m19 ; -out9 paddd m7, m19 ; out6 psubd m19, m5, m16 ; -out11 paddd m5, m16 ; out4 REPX {pmulld x, m12}, m17, m7, m19, m5 psubd m16, m8, m18 ; out8 paddd m8, m18 ; -out7 - psubd m18, m6, m0 ; out10 - paddd m6, m0 ; -out5 + psubd m18, m6, m9 ; out10 + paddd m6, m9 ; -out5 REPX {pmulld x, m12}, m16, m8, m18, m6 - REPX {paddd x, m9 }, m1, m3, m20, m22 - REPX {psubd x, m9, x}, m2, m4, m21, m23 + REPX {paddd x, m4 }, m0, m2, m20, m22 + REPX {psubd x, m4, x}, m1, m3, m21, m23 REPX {paddd x, m10 }, m7, m5, m16, m18 REPX {psubd x, m11, x}, m17, m19, m8, m6 + REPX {psrad x, 2 }, m20, m22, m0, m2, m21, m23, m1, m3 + REPX {psrad x, 13}, m17, m19, m5, m7, m16, m18, m6, m8 ret ALIGN function_align .main_pass1_fast: mova ym0, [cq+64*0] mova ym1, [cq+64*2] @@ -1832,26 +1922,18 @@ cglobal iflipadst_16x16_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 %undef cmp cmp eobd, 36 jl .fast call m(iadst_16x16_internal_10bpc).main_pass1 - psrad m0, m23, 2 - psrad m23, m1, 2 - psrad m1, m22, 2 - psrad m22, m2, 2 - psrad m2, m21, 2 - psrad m21, m3, 2 - psrad m3, m20, 2 - psrad m20, m4, 2 - psrad m4, m19, 13 - psrad m19, m5, 13 - psrad m5, m18, 13 - psrad m18, m6, 13 - psrad m6, m17, 13 - psrad m17, m7, 13 - psrad m7, m16, 13 - psrad m16, m8, 13 + packssdw m4, m19, m3 + packssdw m3, m20, m5 + packssdw m5, m18, m2 + packssdw m2, m21, m6 + packssdw m6, m17, m1 + packssdw m1, m22, m7 + packssdw m7, m16, m0 + packssdw m0, m23, m8 jmp m(idct_16x16_internal_10bpc).pass1_end .fast: call m(iadst_16x16_internal_10bpc).main_pass1_fast vpbroadcastd m9, [o(pd_2)] psubd m4, m9, m3 @@ -1864,21 +1946,21 @@ psubd m0, m9, m8 jmp m(iadst_16x16_internal_10bpc).pass1_fast_end .pass2: lea r5, [o_base_8bpc] call m(iadst_16x16_internal_8bpc).main_pass2b - movshdup m10, [permC] - movu m13, [pw_m2048_2048] - psrlq m11, m10, 8 - vpermq m8, m11, m7 - vpermq m7, m11, m6 - vpermq m6, m11, m5 - vpermq m5, m11, m4 - vpermq m3, m10, m3 - vpermq m2, m10, m2 - vpermq m1, m10, m1 - vpermq m0, m10, m0 + movshdup m12, [permC] + movu m11, [pw_m2048_2048] + psrlq m13, m12, 8 + vpermq m8, m13, m7 + vpermq m7, m13, m6 + vpermq m6, m13, m5 + vpermq m5, m13, m4 + vpermq m3, m12, m3 + vpermq m2, m12, m2 + vpermq m1, m12, m1 + vpermq m0, m12, m0 jmp m(idct_16x16_internal_10bpc).pass2_end INV_TXFM_16X16_FN identity, dct, -92 INV_TXFM_16X16_FN identity, identity @@ -1925,36 +2007,34 @@ vshufi32x4 m3, m1, m4, q3131 vshufi32x4 m1, m4, q2020 REPX {mova x, m7}, m4, m5, m6 jmp m(idct_16x16_internal_10bpc).pass1_end3 .pass2: - movshdup m11, [o(permC)] - vpbroadcastd m12, [o(pw_1697x16)] + movshdup m14, [o(permC)] + vpbroadcastd m15, [o(pw_1697x16)] lea r6, [strideq*3] - vpbroadcastd m13, [o(pw_2048)] - pxor m14, m14 - vpbroadcastd m15, [pixel_10bpc_max] - vpermq m8, m11, m0 - vpermq m9, m11, m1 + vpbroadcastd m11, [o(pw_2048)] + pxor m12, m12 + vpbroadcastd m13, [pixel_10bpc_max] + vpermq m8, m14, m0 + vpermq m9, m14, m1 call .pass2_main - vpermq m8, m11, m2 - vpermq m9, m11, m3 + vpermq m8, m14, m2 + vpermq m9, m14, m3 call .pass2_main - vpermq m8, m11, m4 - vpermq m9, m11, m5 + vpermq m8, m14, m4 + vpermq m9, m14, m5 call .pass2_main - vpermq m8, m11, m6 - vpermq m9, m11, m7 + vpermq m8, m14, m6 + vpermq m9, m14, m7 .pass2_main: - pmulhrsw m0, m12, m8 - pmulhrsw m1, m12, m9 + pmulhrsw m0, m15, m8 + pmulhrsw m1, m15, m9 paddsw m8, m8 paddsw m9, m9 paddsw m8, m0 paddsw m9, m1 - pmulhrsw m8, m13 - pmulhrsw m9, m13 jmp m(idct_16x8_internal_10bpc).write_16x4 ALIGN function_align .pass1_main: pmulld m6, m10, [r6+64*0] pmulld m7, m10, [r6+64*1] @@ -1974,6 +2054,1765 @@ REPX {pmulld x, m10}, m6, m7 REPX {paddd x, m11}, m6, m7 REPX {psrad x, 13 }, m6, m7 ret +cglobal inv_txfm_add_dct_dct_8x32_10bpc, 4, 7, 22, dst, stride, c, eob +%undef cmp + lea r5, [o_base] + test eobd, eobd + jz .dconly + vpbroadcastd m12, [o(pd_2896)] + vpbroadcastd m13, [o(pd_2048)] + vpbroadcastd m14, [o(clip_18b_min)] + vpbroadcastd m15, [o(clip_18b_max)] + vpbroadcastd m11, [o(pd_2)] + mova m20, [o(idct8x32p)] + pxor m21, m21 + cmp eobd, 43 + jl .fast + call .pass1_main + punpcklwd m16, m0, m1 + punpcklwd m17, m2, m3 + punpckhwd m18, m0, m1 + punpckhwd m19, m2, m3 + cmp eobd, 107 + jge .full + punpckldq m0, m16, m17 ; 0 2 + punpckhdq m1, m16, m17 ; 4 6 + punpckldq m2, m18, m19 ; 8 10 + punpckhdq m3, m18, m19 ; 12 14 + lea r5, [o_base_8bpc] + vextracti32x8 ym14, m0, 1 + vextracti32x8 ym15, m1, 1 + vextracti32x8 ym16, m2, 1 + vextracti32x8 ym17, m3, 1 + call m(idct_8x16_internal_8bpc).main_fast + call m(inv_txfm_add_dct_dct_8x32_8bpc).main_fast + jmp .end +.full: + add cq, 64 + call .pass1_main + punpcklwd m5, m0, m1 + punpcklwd m6, m2, m3 + punpckhwd m7, m0, m1 + punpckhwd m8, m2, m3 + punpckldq m0, m16, m17 ; 0 2 + punpckhdq m1, m16, m17 ; 4 6 + punpckldq m2, m18, m19 ; 8 10 + punpckhdq m3, m18, m19 ; 12 14 + punpckldq m4, m5, m6 ; 16 18 + punpckhdq m5, m6 ; 20 22 + punpckldq m6, m7, m8 ; 24 26 + punpckhdq m7, m8 ; 28 30 + lea r5, [o_base_8bpc] + vextracti32x8 ym14, m0, 1 + vextracti32x8 ym15, m1, 1 + vextracti32x8 ym16, m2, 1 + vextracti32x8 ym17, m3, 1 + vextracti32x8 ym18, m4, 1 + vextracti32x8 ym19, m5, 1 + vextracti32x8 ym20, m6, 1 + vextracti32x8 ym21, m7, 1 + call m(idct_8x16_internal_8bpc).main + REPX {pshufd x, x, q1032}, ym18, ym19, ym20, ym21 + call m(inv_txfm_add_dct_dct_8x32_8bpc).main + jmp .end +.fast: + movshdup m8, [o(permB)] + mova ym1, [cq+128*1] + mova ym5, [cq+128*5] + mova ym7, [cq+128*3] + mova ym3, [cq+128*7] + mova ym0, [cq+128*0] + mova ym4, [cq+128*2] + mova ym2, [cq+128*4] + mova ym6, [cq+128*6] + vpermt2q m1, m8, m5 ; 1 5 + vpermt2q m3, m8, m7 ; 7 3 + vpermt2q m0, m8, m4 ; 0 2 + vpermt2q m2, m8, m6 ; 4 6 + mova [cq+128*0], ym21 + REPX {vmovdqa32 [cq+128*x], ym21}, 1, 2, 3, 4, 5, 6, 7 + call m(idct_8x8_internal_10bpc).main + call m(idct_8x8_internal_10bpc).main_end + packssdw m0, m2 + packssdw m1, m3 + vpermb m0, m20, m0 + vprold m20, 16 + vpermb m2, m20, m1 + punpckhdq m1, m0, m2 + punpckldq m0, m2 + lea r5, [o_base_8bpc] + vextracti32x8 ym14, m0, 1 + vextracti32x8 ym15, m1, 1 + call m(idct_8x16_internal_8bpc).main_fast2 + call m(inv_txfm_add_dct_dct_8x32_8bpc).main_fast2 +.end: + call m(inv_txfm_add_dct_dct_8x32_8bpc).main_end ; performs vzeroupper + lea r3, [strideq*2] + vpbroadcastd m12, [pixel_10bpc_max] + lea r6, [strideq*3] + pxor m11, m11 + lea r3, [dstq+r3*8] + pmulhrsw m0, m10 + pmulhrsw m1, m10 + call .write_8x4x2 + pmulhrsw m0, m10, m2 + pmulhrsw m1, m10, m3 + call .write_8x4x2 + pmulhrsw m0, m10, m4 + pmulhrsw m1, m10, m5 + call .write_8x4x2 + pmulhrsw m0, m10, m6 + pmulhrsw m1, m10, m7 +.write_8x4x2: + mova xm8, [dstq+strideq*0] + vinserti32x4 ym8, [dstq+strideq*1], 1 + vinserti32x4 m8, [dstq+strideq*2], 2 + vinserti32x4 m8, [dstq+r6 ], 3 + mova xm9, [r3 +r6 ] + vinserti32x4 ym9, [r3 +strideq*2], 1 + vinserti32x4 m9, [r3 +strideq*1], 2 + vinserti32x4 m9, [r3 +strideq*0], 3 + paddw m8, m0 + paddw m9, m1 + pmaxsw m8, m11 + pmaxsw m9, m11 + pminsw m8, m12 + pminsw m9, m12 + mova [dstq+strideq*0], xm8 + vextracti32x4 [dstq+strideq*1], ym8, 1 + vextracti32x4 [dstq+strideq*2], m8, 2 + vextracti32x4 [dstq+r6 ], m8, 3 + lea dstq, [dstq+strideq*4] + vextracti32x4 [r3 +strideq*0], m9, 3 + vextracti32x4 [r3 +strideq*1], m9, 2 + vextracti32x4 [r3 +strideq*2], ym9, 1 + mova [r3 +r6 ], xm9 + lea r3, [r3+strideq*4] + ret +.dconly: + imul r6d, [cq], 181 + mov [cq], eobd + or r3d, 32 + add r6d, 640 + sar r6d, 10 + jmp m(inv_txfm_add_dct_dct_8x8_10bpc).dconly2 +ALIGN function_align +.pass1_main: + mova m0, [cq+128*0] + mova m1, [cq+128*1] + mova m2, [cq+128*2] + mova m3, [cq+128*3] + mova m4, [cq+128*4] + mova m5, [cq+128*5] + mova m6, [cq+128*6] + mova m7, [cq+128*7] + REPX {mova [cq+128*x], m21}, 0, 1, 2, 3, 4, 5, 6, 7 + call m(idct_8x16_internal_10bpc).main + call m(idct_8x16_internal_10bpc).main_end2 + packssdw m0, m4 + packssdw m1, m5 + packssdw m2, m6 + packssdw m3, m7 + REPX {vpermb x, m20, x}, m0, m1, m2, m3 + ret + +cglobal inv_txfm_add_identity_identity_8x32_10bpc, 4, 8, 12, dst, stride, c, eob + vpbroadcastd m9, [pw_5] + lea r4, [strideq*3] + pxor m10, m10 + lea r5, [strideq*5] + vpbroadcastd m11, [pixel_10bpc_max] + sub eobd, 107 + lea r6, [strideq+r4*2] +.loop: + mova m0, [cq+128*0] + packssdw m0, [cq+128*1] + mova m1, [cq+128*2] + packssdw m1, [cq+128*3] + mova m2, [cq+128*4] + packssdw m2, [cq+128*5] + mova m3, [cq+128*6] + packssdw m3, [cq+128*7] + lea r7, [dstq+strideq*8] + REPX {mova [cq+128*x], m10}, 0, 1, 2, 3 + REPX {paddsw x, m9}, m0, m1, m2, m3 + REPX {mova [cq+128*x], m10}, 4, 5, 6, 7 + REPX {psraw x, 3 }, m0, m1, m2, m3 + add cq, 64 + mova xm4, [dstq+strideq*0] + mova xm5, [dstq+strideq*1] + mova xm6, [dstq+strideq*2] + mova xm7, [dstq+r4 *1] + punpckhwd m8, m0, m1 + vinserti32x4 ym4, [dstq+strideq*4], 1 + punpcklwd m0, m1 + vinserti32x4 ym5, [dstq+r5 *1], 1 + punpckhwd m1, m2, m3 + vinserti32x4 ym6, [dstq+r4 *2], 1 + punpcklwd m2, m3 + vinserti32x4 ym7, [dstq+r6 *1], 1 + punpckhwd m3, m0, m8 + vinserti32x4 m4, [r7 +strideq*0], 2 + punpcklwd m0, m8 + vinserti32x4 m5, [r7 +strideq*1], 2 + punpckhwd m8, m2, m1 + vinserti32x4 m6, [r7 +strideq*2], 2 + punpcklwd m2, m1 + vinserti32x4 m7, [r7 +r4 *1], 2 + punpckhqdq m1, m0, m2 + vinserti32x4 m4, [r7 +strideq*4], 3 + punpcklqdq m0, m2 + vinserti32x4 m5, [r7 +r5 *1], 3 + punpcklqdq m2, m3, m8 + vinserti32x4 m6, [r7 +r4 *2], 3 + punpckhqdq m3, m8 + vinserti32x4 m7, [r7 +r6 *1], 3 + paddw m0, m4 + paddw m1, m5 + paddw m2, m6 + paddw m3, m7 + REPX {pmaxsw x, m10}, m0, m1, m2, m3 + REPX {pminsw x, m11}, m0, m1, m2, m3 + mova [dstq+strideq*0], xm0 + mova [dstq+strideq*1], xm1 + mova [dstq+strideq*2], xm2 + mova [dstq+r4 *1], xm3 + vextracti32x4 [dstq+strideq*4], ym0, 1 + vextracti32x4 [dstq+r5 *1], ym1, 1 + vextracti32x4 [dstq+r4 *2], ym2, 1 + vextracti32x4 [dstq+r6 *1], ym3, 1 + lea dstq, [r7+strideq*8] + vextracti32x4 [r7 +strideq*0], m0, 2 + vextracti32x4 [r7 +strideq*1], m1, 2 + vextracti32x4 [r7 +strideq*2], m2, 2 + vextracti32x4 [r7 +r4 *1], m3, 2 + vextracti32x4 [r7 +strideq*4], m0, 3 + vextracti32x4 [r7 +r5 *1], m1, 3 + vextracti32x4 [r7 +r4 *2], m2, 3 + vextracti32x4 [r7 +r6 *1], m3, 3 + add eobd, 0x80000000 + jnc .loop + RET + +cglobal inv_txfm_add_dct_dct_32x8_10bpc, 4, 7, 0, dst, stride, c, eob +%undef cmp + lea r5, [o_base] + test eobd, eobd + jz .dconly + mova m11, [o(permB)] + mova m0, [cq+64* 0] ; 0 1 + mova m4, [cq+64* 1] ; 2 3 + mova m1, [cq+64* 2] ; 4 5 + mova m8, [cq+64* 3] ; 6 7 + vpbroadcastd m12, [o(pd_2896)] + vpbroadcastd m13, [o(pd_2048)] + vpbroadcastd m14, [o(clip_18b_min)] + vpbroadcastd m15, [o(clip_18b_max)] + psrlq m10, m11, 32 +%if WIN64 + movaps [cq+16*0], xmm6 + movaps [cq+16*1], xmm7 +%endif + mova m16, m11 + vpermi2q m16, m0, m1 ; 1 5 + mova m17, m11 + vpermi2q m17, m8, m4 ; 7 3 + cmp eobd, 43 + jl .fast + mova m18, [cq+64* 4] ; 8 9 + mova m20, [cq+64* 5] ; 10 11 + mova m6, [cq+64* 6] ; 12 13 + mova m7, [cq+64* 7] ; 14 15 + vpermt2q m0, m10, m18 ; 0 8 + vpermt2q m18, m11, m6 ; 9 13 + mova m19, m11 + vpermi2q m19, m7, m20 ; 15 11 + cmp eobd, 107 + jge .full + vpermt2q m1, m10, m6 ; 4 12 + vpermt2q m4, m10, m8 ; 2 6 + vpermt2q m7, m10, m20 ; 14 10 + mov r6d, 64*1 + call m(idct_8x8_internal_10bpc).main_fast + call m(idct_16x8_internal_10bpc).main_fast + call .main_fast + call m(idct_16x16_internal_10bpc).main_end + jmp .end +.full: + mova m2, [cq+64* 8] ; 16 17 + mova m5, [cq+64* 9] ; 18 19 + mova m9, [cq+64*10] ; 20 21 + mova m21, [cq+64*11] ; 22 23 + vpermt2q m1, m10, m9 ; 4 20 + vpermt2q m7, m10, m21 ; 14 22 + vpermt2q m21, m11, m5 ; 23 19 + vpermt2q m5, m10, m20 ; 18 10 + mova m20, m11 + vpermi2q m20, m2, m9 ; 17 21 + mova m22, [cq+64*12] ; 24 25 + mova m9, [cq+64*13] ; 26 27 + mova m3, [cq+64*14] ; 28 29 + mova m23, [cq+64*15] ; 30 31 + vpermt2q m2, m10, m22 ; 16 24 + vpermt2q m22, m11, m3 ; 25 29 + vpermt2q m3, m10, m6 ; 28 12 + vpermt2q m4, m10, m9 ; 2 26 + mova m6, m10 + vpermi2q m6, m23, m8 ; 30 6 + vpermt2q m23, m11, m9 ; 31 27 + mov r6d, 64*3 + call m(idct_8x8_internal_10bpc).main + call m(idct_16x8_internal_10bpc).main + call .main + call m(idct_16x16_internal_10bpc).main_end + jmp .end +.fast: + vpermq m0, m10, m0 ; 0 0 + vpermq m1, m10, m1 ; 4 4 + vpermt2q m4, m10, m8 ; 2 6 + xor r6d, r6d + call .main_fast2 + call m(idct_16x16_internal_10bpc).main_end +.end: +%if WIN64 + movaps xmm6, [cq+16*0] + movaps xmm7, [cq+16*1] +%endif + vzeroupper + call .transpose_8x32 + pxor m14, m14 +.zero_loop: + mova [cq+r6*4+64*3], m14 + mova [cq+r6*4+64*2], m14 + mova [cq+r6*4+64*1], m14 + mova [cq+r6*4+64*0], m14 + sub r6d, 64 + jge .zero_loop + lea r5, [o_base_8bpc] + punpckhqdq m1, m0, m2 + punpcklqdq m0, m2 + punpcklqdq m2, m3, m4 + punpckhqdq m3, m4 + punpcklqdq m4, m5, m7 + punpckhqdq m5, m7 + punpckhqdq m7, m6, m8 + punpcklqdq m6, m8 + call m(inv_txfm_add_dct_dct_32x8_8bpc).main + pxor m12, m12 +.write_32x8_start: + vpbroadcastd m11, [pw_2048] + vpbroadcastd m13, [pixel_10bpc_max] + lea r3, [strideq*3] +.write_32x8: + pmulhrsw m0, m11 + pmulhrsw m1, m11 + pmulhrsw m2, m11 + pmulhrsw m3, m11 + call .write_32x4 + pmulhrsw m0, m11, m4 + pmulhrsw m1, m11, m5 + pmulhrsw m2, m11, m6 + pmulhrsw m3, m11, m7 +.write_32x4: + paddw m0, [dstq+strideq*0] + paddw m1, [dstq+strideq*1] + paddw m2, [dstq+strideq*2] + paddw m3, [dstq+r3 ] + REPX {pmaxsw x, m12}, m0, m1, m2, m3 + REPX {pminsw x, m13}, m0, m1, m2, m3 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + mova [dstq+strideq*2], m2 + mova [dstq+r3 ], m3 + lea dstq, [dstq+strideq*4] + ret +.dconly: + imul r6d, [cq], 181 + mov [cq], eobd + or r3d, 8 + add r6d, 640 + sar r6d, 10 + jmp m(inv_txfm_add_dct_dct_32x16_10bpc).dconly2 +ALIGN function_align +.main_fast2: ; bottom three-quarters are zero + vbroadcasti32x4 m8, [o(pd_799_4017)] + pmulld m8, m1 ; t4 t7 + vpmulld m0, [o(pd_2896)] {1to16} ; t0 t1 + REPX {paddd x, m13}, m8, m0 + REPX {psrad x, 12 }, m8, m0 + pmulld m3, m8, m12 + mova m2, m0 ; t3 t2 + call m(idct_8x8_internal_10bpc).main3 + vbroadcasti32x4 m6, [o(pd_4076_3920)] + vbroadcasti32x4 m3, [o(pd_401_m1189)] + pmulld m6, m4 ; t15 t12 + pmulld m4, m3 ; t9 t10 + REPX {paddd x, m13}, m6, m4 + REPX {psrad x, 12 }, m6, m4 + mova m5, m6 ; t14 t13 + mova m9, m4 ; t8 t11 + call m(idct_16x8_internal_10bpc).main3 + vbroadcasti32x4 m23, [o(pd_4091_3973)] + vbroadcasti32x4 m7, [o(pd_201_995)] + vbroadcasti32x4 m22, [o(pd_1380_601)] + vbroadcasti32x4 m9, [o(pd_3857_4052)] + pmulld m23, m16 ; t16 t20 + pmulld m16, m7 ; t31 t27 + pmulld m22, m17 ; -t19 -t25 + pmulld m17, m9 ; t28 t24 + REPX {paddd x, m13}, m23, m16, m17 + psubd m22, m13, m22 + REPX {psrad x, 12 }, m23, m16, m22, m17 + mova m20, m23 ; t30 t26 + mova m9, m16 ; t17 t21 + mova m19, m22 ; t18 t22 + mova m18, m17 ; t29 t25 + jmp .main3 +.main_fast: ; bottom half is zero + vbroadcasti32x4 m23, [o(pd_4091_3973)] + vbroadcasti32x4 m7, [o(pd_201_995)] + vbroadcasti32x4 m20, [o(pd_2751_2106)] + vbroadcasti32x4 m9, [o(pd_3035_3513)] + vbroadcasti32x4 m21, [o(pd_3703_3290)] + vbroadcasti32x4 m10, [o(pd_1751_2440)] + vbroadcasti32x4 m22, [o(pd_1380_601)] + vbroadcasti32x4 m11, [o(pd_3857_4052)] + pmulld m23, m16 ; t16a t20a + pmulld m16, m7 ; t31a t27a + pmulld m20, m19 ; -t17a -t21a + pmulld m19, m9 ; t30a t26a + pmulld m21, m18 ; t18a t22a + pmulld m18, m10 ; t29a t25a + pmulld m22, m17 ; -t19a -t25a + pmulld m17, m11 ; t28a t24a + psubd m20, m13, m20 + psubd m22, m13, m22 + jmp .main2 +.main: + ITX_MULSUB_2D 16, 23, 7, 9, 10, _, 201_995, 4091_3973 + ITX_MULSUB_2D 20, 19, 7, 9, 10, _, 3035_3513, 2751_2106 + ITX_MULSUB_2D 18, 21, 7, 9, 10, _, 1751_2440, 3703_3290 + ITX_MULSUB_2D 22, 17, 7, 9, 10, _, 3857_4052, 1380_601 + paddd m20, m13 + paddd m22, m13 +.main2: + REPX {paddd x, m13}, m16, m23, m19 + REPX {psrad x, 12 }, m16, m20, m23, m19 + psubd m9, m16, m20 ; t17 t21 + paddd m16, m20 ; t16 t20 + psubd m20, m23, m19 ; t30 t26 + paddd m23, m19 ; t31 t27 + REPX {pmaxsd x, m14}, m9, m16, m20, m23 + REPX {paddd x, m13}, m21, m18, m17 + REPX {psrad x, 12 }, m18, m22, m21, m17 + psubd m19, m22, m18 ; t18 t22 + paddd m22, m18 ; t19 t23 + psubd m18, m17, m21 ; t29 t25 + paddd m17, m21 ; t28 t24 + REPX {pmaxsd x, m14}, m19, m22, m18, m17 + REPX {pminsd x, m15}, m20, m9, m18, m19, m16, m23, m22, m17 +.main3: + vbroadcasti32x4 m11, [o(pd_4017_2276)] + vbroadcasti32x4 m10, [o(pd_799_3406)] + psubd m7, m0, m6 ; dct16 out15 out14 + paddd m0, m6 ; dct16 out0 out1 + psubd m6, m1, m5 ; dct16 out12 out13 + paddd m1, m5 ; dct16 out3 out2 + psubd m5, m2, m4 ; dct16 out11 out10 + paddd m2, m4 ; dct16 out4 out5 + psubd m4, m3, m8 ; dct16 out8 out9 + paddd m3, m8 ; dct16 out7 out6 + ITX_MULSUB_2D 20, 9, 8, 21, _, 13, 10, 11 + ITX_MULSUB_2D 18, 19, 8, 21, _, 13, 10, 11, 2 + REPX {pmaxsd x, m14}, m7, m0, m6, m1, m5, m2, m4, m3 + punpckhqdq m21, m16, m20 ; t20 t21a + punpcklqdq m16, m20 ; t16 t17a + punpcklqdq m20, m22, m19 ; t19 t18a + punpckhqdq m22, m19 ; t23 t22a + REPX {pminsd x, m15}, m0, m1, m2, m3, m4, m5, m6, m7 + punpcklqdq m19, m23, m9 ; t31 t30a + punpckhqdq m23, m9 ; t27 t26a + punpckhqdq m9, m17, m18 ; t24 t25a + punpcklqdq m17, m18 ; t28 t29a + vpbroadcastd m11, [o(pd_3784)] + vpbroadcastd m10, [o(pd_1567)] + psubd m18, m16, m20 ; t19a t18 + paddd m20, m16 ; t16a t17 + psubd m16, m19, m17 ; t28a t29 + paddd m19, m17 ; t31a t30 + psubd m17, m22, m21 ; t20a t21 + paddd m22, m21 ; t23a t22 + psubd m21, m9, m23 ; t27a t26 + paddd m23, m9 ; t24a t25 + REPX {pmaxsd x, m14}, m18, m16, m17, m21 + REPX {pminsd x, m15}, m16, m18, m21, m17 + ITX_MULSUB_2D 16, 18, 8, 9, _, 13, 10, 11 + ITX_MULSUB_2D 21, 17, 8, 9, _, 13, 10, 11, 2 + REPX {pmaxsd x, m14}, m20, m22, m19, m23 + REPX {pminsd x, m15}, m20, m22, m19, m23 + paddd m9, m20, m22 ; t16 t17a + psubd m20, m22 ; t23 t22a + paddd m22, m19, m23 ; t31 t30a + psubd m19, m23 ; t24 t25a + psubd m23, m16, m17 ; t20a t21 + paddd m16, m17 ; t19a t18 + psubd m17, m18, m21 ; t27a t26 + paddd m21, m18 ; t28a t29 + REPX {pmaxsd x, m14}, m20, m19, m23, m17 + REPX {pminsd x, m15}, m19, m20, m17, m23 + REPX {pmulld x, m12}, m19, m20, m17, m23 + REPX {pmaxsd x, m14}, m22, m21, m16, m9 + paddd m19, m13 + paddd m17, m13 + REPX {pminsd x, m15}, m22, m21, m16, m9 + psubd m18, m19, m20 ; t23a t22 + paddd m19, m20 ; t24a t25 + paddd m20, m17, m23 ; t27 t26a + psubd m17, m23 ; t20 t21a + REPX {psrad x, 12 }, m20, m19, m18, m17 + ret +.transpose_8x32: + mova m10, [o(idct32x8p)] + psrlw m8, m10, 8 + mova m9, m8 + vpermi2w m8, m1, m5 + vpermt2w m1, m10, m5 + vprold m5, m9, 16 + vpermi2w m9, m3, m7 + vpermt2w m3, m10, m7 + vprold m10, 16 + mova m7, m5 + vpermi2w m5, m0, m4 + vpermt2w m0, m10, m4 + vpermi2w m7, m2, m6 + vpermt2w m2, m10, m6 + punpckhdq m6, m5, m8 + punpckldq m5, m8 + punpckhdq m8, m7, m9 + punpckldq m7, m9 + punpckhdq m4, m2, m3 + punpckldq m2, m3 + punpckhdq m3, m0, m1 + punpckldq m0, m1 + ret + +cglobal inv_txfm_add_identity_identity_32x8_10bpc, 4, 7, 10, dst, stride, c, eob + vpbroadcastd m5, [pw_4096] + lea r4, [strideq*3] + mova m6, [idtx32x8p] + lea r5, [strideq*5] + vpbroadcastd m9, [pixel_10bpc_max] + lea r6, [strideq+r4*2] + pxor m8, m8 + sub eobd, 107 + psrlw m7, m6, 8 +.loop: + mova m0, [cq+64*0] + packssdw m0, [cq+64*1] ; 02 13 + mova m1, [cq+64*2] + packssdw m1, [cq+64*3] ; 46 57 + mova m2, [cq+64*4] + packssdw m2, [cq+64*5] ; 8a 9b + mova m3, [cq+64*6] + packssdw m3, [cq+64*7] ; ce df + REPX {pmulhrsw x, m5}, m0, m1, m2, m3 + REPX {mova [cq+64*x], m8}, 0, 1, 2, 3 + mova m4, m6 + vpermi2w m4, m1, m3 + vpermt2w m1, m7, m3 + REPX {mova [cq+64*x], m8}, 4, 5, 6, 7 + mova m3, m7 + vpermi2w m3, m0, m2 + vpermt2w m0, m6, m2 + add cq, 64*8 + punpcklqdq m2, m3, m1 ; 4 5 + punpckhqdq m3, m1 ; 6 7 + punpckhqdq m1, m0, m4 ; 2 3 + punpcklqdq m0, m4 ; 0 1 + mova ym4, [dstq+strideq*0] + vinserti32x8 m4, [dstq+strideq*1], 1 + paddw m0, m4 + mova ym4, [dstq+strideq*2] + vinserti32x8 m4, [dstq+r4 *1], 1 + paddw m1, m4 + mova ym4, [dstq+strideq*4] + vinserti32x8 m4, [dstq+r5 *1], 1 + paddw m2, m4 + mova ym4, [dstq+r4 *2] + vinserti32x8 m4, [dstq+r6 *1], 1 + paddw m3, m4 + REPX {pmaxsw x, m8}, m0, m1, m2, m3 + REPX {pminsw x, m9}, m0, m1, m2, m3 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + mova [dstq+strideq*2], ym1 + vextracti32x8 [dstq+r4 *1], m1, 1 + mova [dstq+strideq*4], ym2 + vextracti32x8 [dstq+r5 *1], m2, 1 + mova [dstq+r4 *2], ym3 + vextracti32x8 [dstq+r6 *1], m3, 1 + add dstq, 32 + add eobd, 0x80000000 + jnc .loop + RET + +cglobal inv_txfm_add_dct_dct_16x32_10bpc, 4, 7, 0, dst, stride, c, eob +%undef cmp + lea r5, [o_base] + test eobd, eobd + jz .dconly + vpbroadcastd m12, [o(pd_2896)] + vpbroadcastd m13, [o(pd_2048)] + vpbroadcastd m14, [o(clip_18b_min)] + vpbroadcastd m15, [o(clip_18b_max)] +%if WIN64 + movaps [rsp+ 8], xmm6 + movaps [rsp+24], xmm7 +%endif + cmp eobd, 36 + jl .fast + call .pass1 + cmp eobd, 151 + jge .full + lea r5, [o_base_8bpc] + pxor m9, m9 + punpcklwd m8, m1, m1 ; 2 + punpckhwd m14, m1, m1 ; 3 + punpcklwd m1, m3, m3 ; 6 + punpckhwd m15, m3, m3 ; 7 + punpcklwd m3, m6, m6 ; 12 + punpckhwd m19, m6, m6 ; 13 + punpcklwd m6, m9, m4 ; __ 8 + punpckhwd m20, m4, m4 ; 9 + punpckhwd m16, m5, m5 ; 11 + punpcklwd m5, m5 ; 10 + punpcklwd m9, m0 ; __ 0 + punpckhwd m21, m0, m0 ; 1 + punpcklwd m0, m7, m7 ; 14 + punpckhwd m17, m7, m7 ; 15 + punpcklwd m7, m2, m2 ; 4 + punpckhwd m18, m2, m2 ; 5 + call m(idct_16x16_internal_8bpc).main_fast + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast + mov r6d, 64*3 + pxor m8, m8 +.zero_loop: + REPX {mova [cq+r6*8+128*x], m8}, 3, 2, 1, 0 + sub r6d, 64 + jge .zero_loop + jmp .pass2_end +.full: + mova [cq+128*0], m0 + mova [cq+128*1], m1 + mova [cq+128*2], m2 + mova [cq+128*3], m3 + mova [cq+128*4], m4 + mova [cq+128*5], m5 + mova [cq+128*6], m6 + mova [cq+128*7], m7 + add cq, 64 + call .pass1 + mova m9, [cq-64* 1] ; 0 1 + mova m14, [cq+64* 1] ; 2 3 + mova m18, [cq+64* 3] ; 4 5 + mova m15, [cq+64* 5] ; 6 7 + mova m20, [cq+64* 7] ; 8 9 + mova m16, [cq+64* 9] ; 10 11 + mova m22, [cq+64*11] ; 12 13 + mova m19, [cq+64*13] ; 14 15 + lea r5, [o_base_8bpc] + punpcklwd m8, m7, m14 ; 30 2 + punpckhwd m21, m7, m9 ; 31 1 + punpcklwd m7, m6, m18 ; 28 4 + punpckhwd m14, m6 ; 3 29 + punpcklwd m9, m0, m9 ; 16 0 + punpckhwd m17, m19, m0 ; 15 17 + punpcklwd m0, m19, m1 ; 14 18 + punpckhwd m19, m1, m22 ; 19 13 + punpcklwd m1, m15, m5 ; 6 26 + punpckhwd m18, m5, m18 ; 27 5 + punpcklwd m6, m4, m20 ; 24 8 + punpckhwd m15, m4 ; 7 25 + punpcklwd m5, m3, m16 ; 22 10 + punpckhwd m20, m3, m20 ; 23 9 + punpcklwd m3, m22, m2 ; 12 20 + punpckhwd m16, m2 ; 11 21 + call m(idct_16x16_internal_8bpc).main2 + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf + mov r6d, 32*7 + pxor m8, m8 +.full_zero_loop: + REPX {mova [cq+r6*8+64*x], m8}, 2, 1, 0, -1 + sub r6d, 32 + jge .full_zero_loop + jmp .pass2_end +.fast: + mova ym0, [cq+128*0] + mova ym2, [cq+128*4] + movshdup m8, [o(permB)] + mova ym1, [cq+128*2] + mova ym3, [cq+128*6] + mova ym4, [cq+128*1] + mova ym5, [cq+128*3] + mova ym6, [cq+128*5] + mova ym7, [cq+128*7] + vpermt2q m0, m8, m2 ; 0 4 + vpermt2q m1, m8, m3 ; 2 6 + vpermt2q m4, m8, m5 ; 1 3 + vpermt2q m7, m8, m6 ; 7 5 + REPX {pmulld x, m12}, m0, m1, m4, m7 + pxor ym16, ym16 + mova [cq+128*0], ym16 + REPX {vmovdqa32 [cq+128*x], ym16}, 1, 2, 3, 4, 5, 6, 7 + REPX {paddd x, m13}, m0, m1, m4, m7 + REPX {psrad x, 12 }, m0, m1, m4, m7 + call m(idct_8x8_internal_10bpc).main_fast + call m(idct_16x8_internal_10bpc).main_fast + vpbroadcastd m11, [o(pd_1)] + call m(idct_8x16_internal_10bpc).main_end2 + mova m8, [o(idct8x32p)] + packssdw m0, m4 + packssdw m1, m5 + packssdw m2, m6 + packssdw m3, m7 + mova m6, [dup16_perm] + vpermb m0, m8, m0 + vpermb m2, m8, m2 + vprold m8, 16 + vpermb m1, m8, m1 + vpermb m3, m8, m3 + punpckldq m4, m0, m2 + punpckhdq m0, m2 + punpckldq m2, m1, m3 + punpckhdq m1, m3 + punpckldq m21, m4, m2 + punpckhdq m14, m4, m2 + punpckldq m18, m0, m1 + punpckhdq m15, m0, m1 + vpermb m8, m6, m14 ; 2 + vpermb m1, m6, m15 ; 6 + vpermb m7, m6, m18 ; 4 + pmovzxwd m9, ym21 ; 0 + vpord m6, [o(pb_32)] {1to16} + lea r5, [o_base_8bpc] + vpermb m21, m6, m21 ; 1 + vpermb m15, m6, m15 ; 7 + vpermb m18, m6, m18 ; 5 + vpermb m14, m6, m14 ; 3 + pslld m9, 16 + call m(idct_16x16_internal_8bpc).main_fast2 + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast2 +.pass2_end: + movshdup m22, [permC] + vpbroadcastd m11, [pw_2048] + vpbroadcastd m13, [pixel_10bpc_max] + lea r6, [strideq*3] + pxor m12, m12 + psrlq m23, m22, 8 + vpermq m8, m22, m0 + vpermq m9, m23, m1 + call m(idct_16x8_internal_10bpc).write_16x4 + vpermq m8, m22, m2 + vpermq m9, m23, m3 + call m(idct_16x8_internal_10bpc).write_16x4 + vpermq m8, m22, m4 + vpermq m9, m23, m5 + call m(idct_16x8_internal_10bpc).write_16x4 + vpermq m8, m22, m6 + vpermq m9, m23, m7 + call m(idct_16x8_internal_10bpc).write_16x4 + vpermq m8, m22, m14 + vpermq m9, m23, m15 + call m(idct_16x8_internal_10bpc).write_16x4 + vpermq m8, m22, m16 + vpermq m9, m23, m17 + call m(idct_16x8_internal_10bpc).write_16x4 + vpermq m8, m22, m18 + vpermq m9, m23, m19 + call m(idct_16x8_internal_10bpc).write_16x4 + vpermq m8, m22, m20 + vpermq m9, m23, m21 +%if WIN64 + movaps xmm6, [rsp+ 8] + movaps xmm7, [rsp+24] +%endif + vzeroupper + jmp m(idct_16x8_internal_10bpc).write_16x4 +.pass1: + pmulld m0, m12, [cq+128* 0] + pmulld m1, m12, [cq+128* 2] + pmulld m2, m12, [cq+128* 4] + pmulld m3, m12, [cq+128* 6] + pmulld m4, m12, [cq+128* 8] + pmulld m5, m12, [cq+128*10] + pmulld m6, m12, [cq+128*12] + pmulld m7, m12, [cq+128*14] + call m(idct_8x16_internal_10bpc).main_rect2 + pmulld m16, m12, [cq+128* 1] + pmulld m17, m12, [cq+128* 3] + pmulld m18, m12, [cq+128* 5] + pmulld m19, m12, [cq+128* 7] + pmulld m20, m12, [cq+128* 9] + pmulld m21, m12, [cq+128*11] + pmulld m22, m12, [cq+128*13] + pmulld m23, m12, [cq+128*15] + call m(idct_16x16_internal_10bpc).main_rect2 + vpbroadcastd m11, [o(pd_1)] + call m(idct_16x16_internal_10bpc).main_end2 + jmp m(idct_16x16_internal_10bpc).main_end3 +.dconly: + imul r6d, [cq], 181 + mov [cq], eobd + or r3d, 32 + jmp m(inv_txfm_add_dct_dct_16x8_10bpc).dconly + +cglobal inv_txfm_add_identity_identity_16x32_10bpc, 4, 7, 16, dst, stride, c, eob +%undef cmp + vpbroadcastd m10, [pw_2896x8] + vpbroadcastd m11, [pw_1697x16] + vpbroadcastd m13, [pw_8192] + vpbroadcastd m15, [pixel_10bpc_max] + lea r6, [strideq*9] + pxor m14, m14 + paddw m12, m13, m13 ; pw_16384 + cmp eobd, 151 + jl .main + call .main + add cq, 64-128*4 + lea dstq, [dstq+strideq*8] +.main: + call .main_internal + add cq, 128*4 + pmulhrsw m1, m13, m2 + pmulhrsw m3, m13, m4 + pmulhrsw m5, m13, m6 + pmulhrsw m7, m13, m8 + call .main_internal +.main2: + pmulhrsw m2, m13 + pmulhrsw m4, m13 + pmulhrsw m6, m13 + pmulhrsw m8, m13 + punpcklqdq m0, m1, m2 ; 0 8 + punpckhqdq m1, m2 ; 1 9 + call .write_16x2x2 + punpcklqdq m0, m3, m4 ; 2 10 + punpckhqdq m1, m3, m4 ; 3 11 + call .write_16x2x2 + punpcklqdq m0, m5, m6 ; 4 12 + punpckhqdq m1, m5, m6 ; 5 13 + call .write_16x2x2 + punpcklqdq m0, m7, m8 ; 6 14 + punpckhqdq m1, m7, m8 ; 7 15 +.write_16x2x2: + mova ym2, [dstq+strideq*0] + vinserti32x8 m2, [dstq+strideq*8], 1 + mova ym9, [dstq+strideq*1] + vinserti32x8 m9, [dstq+r6 ], 1 + paddw m0, m2 + paddw m1, m9 + pmaxsw m0, m14 + pmaxsw m1, m14 + pminsw m0, m15 + pminsw m1, m15 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*8], m0, 1 + mova [dstq+strideq*1], ym1 + vextracti32x8 [dstq+r6 ], m1, 1 + lea dstq, [dstq+strideq*2] + ret +.main_internal: + mova m8, [cq+128* 0] + packssdw m8, [cq+128* 8] + mova m6, [cq+128* 1] + packssdw m6, [cq+128* 9] + mova m0, [cq+128* 2] + packssdw m0, [cq+128*10] + mova m2, [cq+128* 3] + packssdw m2, [cq+128*11] + REPX {pmulhrsw x, m10}, m8, m6, m0, m2 + REPX {vpermq x, x, q3120}, m8, m6, m0, m2 + pmulhrsw m4, m11, m8 + pmulhrsw m9, m11, m6 + REPX {mova [cq+128*x], m14}, 0, 1, 2, 3 + pmulhrsw m4, m12 + pmulhrsw m9, m12 + paddsw m8, m4 + paddsw m6, m9 + pmulhrsw m4, m11, m0 + pmulhrsw m9, m11, m2 + REPX {mova [cq+128*x], m14}, 8, 9, 10, 11 + pmulhrsw m4, m12 + pmulhrsw m9, m12 + paddsw m0, m4 + paddsw m2, m9 + punpcklwd m4, m8, m6 + punpckhwd m8, m6 + punpcklwd m6, m0, m2 + punpckhwd m0, m2 + punpckldq m2, m4, m6 ; 0 1 + punpckhdq m4, m6 ; 2 3 + punpckldq m6, m8, m0 ; 4 5 + punpckhdq m8, m0 ; 6 7 + ret + +cglobal inv_txfm_add_dct_dct_32x16_10bpc, 4, 7, 0, dst, stride, c, eob +%undef cmp + lea r5, [o_base] + test eobd, eobd + jz .dconly + vpbroadcastd m12, [o(pd_2896)] + vpbroadcastd m13, [o(pd_2048)] + vpbroadcastd m14, [o(clip_18b_min)] + vpbroadcastd m15, [o(clip_18b_max)] +%if WIN64 + movaps [rsp+ 8], xmm6 + movaps [rsp+24], xmm7 +%endif + mov r6d, 8*12 + cmp eobd, 36 + jl .fast + pmulld m0, m12, [cq+64* 0] + pmulld m1, m12, [cq+64* 4] + pmulld m2, m12, [cq+64* 8] + pmulld m3, m12, [cq+64*12] + pmulld m16, m12, [cq+64* 2] + pmulld m17, m12, [cq+64* 6] + pmulld m18, m12, [cq+64*10] + pmulld m19, m12, [cq+64*14] + cmp eobd, 151 + jge .full + call m(idct_8x16_internal_10bpc).main_fast_rect2 + call m(idct_16x16_internal_10bpc).main_fast_rect2 + call .idct16_sumsub + call .pass1_load_spill + call .main_fast_rect2 + jmp .pass1_end +.full: + pmulld m4, m12, [cq+64*16] + pmulld m5, m12, [cq+64*20] + pmulld m6, m12, [cq+64*24] + pmulld m7, m12, [cq+64*28] + pmulld m20, m12, [cq+64*18] + pmulld m21, m12, [cq+64*22] + pmulld m22, m12, [cq+64*26] + pmulld m23, m12, [cq+64*30] + add r6d, 8*16 + call m(idct_8x16_internal_10bpc).main_rect2 + call m(idct_16x16_internal_10bpc).main_rect2 + call .idct16_sumsub + call .pass1_load_spill + pmulld m16, m12, [cq+64*17] + pmulld m17, m12, [cq+64*19] + pmulld m18, m12, [cq+64*21] + pmulld m19, m12, [cq+64*23] + pmulld m20, m12, [cq+64*25] + pmulld m21, m12, [cq+64*27] + pmulld m22, m12, [cq+64*29] + pmulld m23, m12, [cq+64*31] + call .main_rect2 +.pass1_end: + vpbroadcastd m11, [o(pd_1)] + lea r4, [cq+64] + call .idct32_pass1_end + lea r5, [o_base_8bpc] + punpckhqdq m19, m5, m16 ; 11 + punpcklqdq m5, m16 ; 10 + punpckhqdq m16, m2, m1 ; 5 + punpcklqdq m2, m1 ; 4 + punpcklqdq m1, m15, m4 ; 2 + punpckhqdq m15, m4 ; 3 + punpcklqdq m4, m14, m18 ; 8 + punpckhqdq m18, m14, m18 ; 9 + punpckhqdq m14, m0, m20 ; 1 + punpcklqdq m0, m20 ; 0 + punpckhqdq m20, m6, m17 ; 13 + punpcklqdq m6, m17 ; 12 + punpckhqdq m17, m3, m21 ; 7 + punpcklqdq m3, m21 ; 6 + punpckhqdq m21, m7, m8 ; 15 + punpcklqdq m7, m8 ; 14 + call m(inv_txfm_add_dct_dct_32x8_8bpc).main + call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf + jmp .end +.fast: + pmulld ym0, ym12, [cq+64*0] + pmulld ym1, ym12, [cq+64*4] + movshdup m7, [o(permB)] + mova ym4, [cq+64*2] + mova ym5, [cq+64*6] + mova ym16, [cq+64*1] + mova ym2, [cq+64*5] + mova ym3, [cq+64*3] + mova ym17, [cq+64*7] + vpermt2q m4, m7, m5 ; 2 6 + vpermt2q m16, m7, m2 ; 1 5 + vpermt2q m17, m7, m3 ; 7 3 + paddd ym0, ym13 + paddd ym1, ym13 + psrad ym0, 12 + psrad ym1, 12 + vpermq m0, m7, m0 ; 0 0 + vpermq m1, m7, m1 ; 4 4 + REPX {pmulld x, m12}, m4, m16, m17 + REPX {paddd x, m13}, m4, m16, m17 + REPX {psrad x, 12 }, m4, m16, m17 + call m(inv_txfm_add_dct_dct_32x8_10bpc).main_fast2 + vpbroadcastd m11, [o(pd_1)] + call m(idct_16x16_internal_10bpc).main_end2 + call m(inv_txfm_add_dct_dct_32x8_10bpc).transpose_8x32 + lea r5, [o_base_8bpc] + punpckhqdq m14, m0, m2 ; 1 + punpcklqdq m0, m2 ; 0 + punpcklqdq m1, m3, m4 ; 2 + punpckhqdq m15, m3, m4 ; 3 + punpcklqdq m2, m5, m7 ; 4 + punpckhqdq m16, m5, m7 ; 5 + punpcklqdq m3, m6, m8 ; 6 + punpckhqdq m17, m6, m8 ; 7 + call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast +.end: +%if WIN64 + movaps xmm6, [rsp+ 8] + movaps xmm7, [rsp+24] +%endif + pxor m12, m12 +.zero_loop: + mova [cq+r6*8+64*3], m12 + mova [cq+r6*8+64*2], m12 + mova [cq+r6*8+64*1], m12 + mova [cq+r6*8+64*0], m12 + sub r6d, 8*4 + jge .zero_loop + call m(inv_txfm_add_dct_dct_32x8_10bpc).write_32x8_start + pmulhrsw m0, m11, m14 + pmulhrsw m1, m11, m15 + pmulhrsw m2, m11, m16 + pmulhrsw m3, m11, m17 + call m(inv_txfm_add_dct_dct_32x8_10bpc).write_32x4 + pmulhrsw m0, m11, m18 + pmulhrsw m1, m11, m19 + pmulhrsw m2, m11, m20 + pmulhrsw m3, m11, m21 + vzeroupper + jmp m(inv_txfm_add_dct_dct_32x8_10bpc).write_32x4 +.dconly: + imul r6d, [cq], 181 + mov [cq], eobd + or r3d, 16 + add r6d, 128 + sar r6d, 8 + imul r6d, 181 + add r6d, 384 + sar r6d, 9 +.dconly2: + vpbroadcastd m3, [o(dconly_10bpc)] + imul r6d, 181 + add r6d, 2176 + sar r6d, 12 + vpbroadcastw m2, r6d + paddsw m2, m3 +.dconly_loop: + paddsw m0, m2, [dstq+strideq*0] + paddsw m1, m2, [dstq+strideq*1] + psubusw m0, m3 + psubusw m1, m3 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + lea dstq, [dstq+strideq*2] + sub r3d, 2 + jg .dconly_loop + RET +ALIGN function_align +.idct16_sumsub: + psubd m23, m0, m22 ; t15 + paddd m0, m22 ; t0 + psubd m22, m1, m21 ; t14 + paddd m1, m21 ; t1 + REPX {pmaxsd x, m14}, m23, m0, m22, m1 + psubd m21, m2, m20 ; t13 + paddd m2, m20 ; t2 + REPX {pminsd x, m15}, m23, m0, m22, m1 + psubd m20, m3, m19 ; t12 + paddd m3, m19 ; t3 + REPX {pmaxsd x, m14}, m21, m2, m20, m3 + psubd m19, m4, m18 ; t11 + paddd m4, m18 ; t4 + REPX {pminsd x, m15}, m21, m2, m20, m3 + psubd m18, m5, m17 ; t10 + paddd m5, m17 ; t5 + REPX {pmaxsd x, m14}, m19, m4, m18, m5 + psubd m17, m6, m16 ; t9 + paddd m6, m16 ; t6 + REPX {pminsd x, m15}, m19, m4, m18, m5 + psubd m16, m7, m9 ; t8 + paddd m7, m9 ; t7 + REPX {pmaxsd x, m14}, m17, m6, m16, m7 + REPX {pminsd x, m15}, m17, m6, m16, m7 + ret +.idct32_pass1_end: + psrlq m12, [o(permC)], 24 ; 0 2 8 10 1 3 9 11 + psrlq m13, m12, 32 ; 4 6 12 14 5 7 13 15 +%macro IDCT32_PASS1_END 2 ; low, high + paddd m8, m11, [r4+128*%1] + paddd m9, m11, [cq+128*%1] + psubd m10, m8, m%1 ; out 16+n + paddd m8, m%1 ; out 15-n + paddd m%1, m9, m%2 ; out 0+n + psubd m9, m%2 ; out 31-n + REPX {vpsravd x, m11}, m10, m%1, m8, m9 + packssdw m%1, m10 ; 0+n 16+n + packssdw m%2, m8, m9 ; 15-n 31-n +%endmacro + IDCT32_PASS1_END 0, 23 ; 0 16, 15 31 + IDCT32_PASS1_END 7, 16 ; 7 23, 8 24 + mova m14, m13 + vpermi2q m14, m0, m16 + vpermt2q m0, m12, m16 + IDCT32_PASS1_END 1, 22 ; 1 17, 14 30 + IDCT32_PASS1_END 6, 17 ; 6 22, 9 25 + mova m15, m13 + vpermi2q m15, m1, m17 + vpermt2q m1, m12, m17 + IDCT32_PASS1_END 2, 21 ; 2 18, 13 29 + IDCT32_PASS1_END 5, 18 ; 5 21, 10 26 + mova m16, m13 + vpermi2q m16, m2, m18 + vpermt2q m2, m12, m18 + IDCT32_PASS1_END 3, 20 ; 3 19, 12 28 + IDCT32_PASS1_END 4, 19 ; 4 20, 11 27 + mova m17, m13 + vpermi2q m17, m3, m19 + vpermt2q m3, m12, m19 + mova m18, m13 + vpermi2q m18, m4, m20 + vpermt2q m4, m12, m20 + mova m19, m13 + vpermi2q m19, m5, m21 + vpermt2q m5, m12, m21 + mova m20, m13 + vpermi2q m20, m6, m22 + vpermt2q m6, m12, m22 + mova m21, m13 + vpermi2q m21, m7, m23 + vpermt2q m7, m12, m23 + punpckhwd m8, m2, m3 ; c04 d04 c05 d05 c06 d06 c07 d07 + punpcklwd m2, m3 ; c00 d00 c01 d01 c02 d02 c03 d03 + punpckhwd m3, m0, m1 ; a04 b04 a05 b05 a06 b06 a07 b07 + punpcklwd m0, m1 ; a00 b00 a01 b01 a02 b02 a03 b03 + punpckhwd m1, m4, m5 ; e04 f04 e05 f05 e06 f06 e07 f07 + punpcklwd m4, m5 ; e00 f00 e01 f01 e02 f02 e03 f03 + punpckhwd m5, m6, m7 ; g04 h04 g05 h05 g06 h06 g07 h07 + punpcklwd m6, m7 ; g00 h00 g01 h01 g02 h02 g03 h03 + punpckhwd m7, m14, m15 ; a12 b12 a13 b13 a14 b14 a15 b15 + punpcklwd m14, m15 ; a08 b08 a09 b09 a10 b10 a11 b11 + punpckhwd m15, m16, m17 ; c12 d12 c13 d13 c14 d14 c15 d15 + punpcklwd m16, m17 ; c08 d08 c09 d09 c10 d10 c11 d11 + punpckhwd m17, m18, m19 ; e12 f12 e13 f13 e14 f14 e15 f15 + punpcklwd m18, m19 ; e08 f08 e09 f09 e10 f10 e11 f11 + punpckhwd m19, m20, m21 ; g12 h12 g13 h13 g14 h14 g15 h15 + punpcklwd m20, m21 ; g08 h08 g09 h09 g10 h10 g11 h11 + punpckhdq m21, m1, m5 ; e06 f06 g06 h06 e07 f07 g07 h07 + punpckldq m1, m5 ; e04 f04 g04 h04 e05 f05 g05 h05 + punpckhdq m5, m14, m16 ; a10 b10 c10 d10 a11 b11 c11 d11 + punpckldq m14, m16 ; a08 b08 c08 d08 a09 b09 c09 d09 + punpckhdq m16, m18, m20 ; e10 f10 g10 h10 e11 f11 g11 h11 + punpckldq m18, m20 ; e08 f08 g08 h08 e09 f09 g09 h09 + punpckldq m20, m4, m6 ; e00 f00 g00 h00 e01 f01 g01 h01 + punpckhdq m4, m6 ; e02 f02 g02 h02 e03 f03 g03 h03 + punpckldq m6, m7, m15 ; a12 b12 c12 d12 a13 b13 c13 d13 + punpckhdq m7, m15 ; a14 b14 c14 d14 a15 b15 c15 d15 + punpckhdq m15, m0, m2 ; a02 b02 c02 d02 a03 b03 c03 d03 + punpckldq m0, m2 ; a00 b00 c00 d00 a01 b01 c01 d01 + punpckldq m2, m3, m8 ; a04 b04 c04 d04 a05 b05 c05 d05 + punpckhdq m3, m8 ; a06 b06 c06 d06 a07 b07 c07 d07 + punpckhdq m8, m17, m19 ; e14 f14 g14 h14 e15 f15 g15 h15 + punpckldq m17, m19 ; e12 f12 g12 h12 e13 f13 g13 h13 + ret +.pass1_load_spill: + mova [cq+64* 0], m0 + mova [cq+64* 2], m1 + mova [cq+64* 4], m2 + mova [cq+64* 6], m3 + mova [cq+64* 8], m4 + mova [cq+64*10], m5 + mova [cq+64*12], m6 + mova [cq+64*14], m7 + pmulld m0, m12, [cq+64* 1] + pmulld m1, m12, [cq+64* 3] + pmulld m2, m12, [cq+64* 5] + pmulld m3, m12, [cq+64* 7] + pmulld m4, m12, [cq+64* 9] + pmulld m5, m12, [cq+64*11] + pmulld m6, m12, [cq+64*13] + pmulld m7, m12, [cq+64*15] + mova [cq+64* 1], m23 + mova [cq+64* 3], m22 + mova [cq+64* 5], m21 + mova [cq+64* 7], m20 + mova [cq+64* 9], m19 + mova [cq+64*11], m18 + mova [cq+64*13], m17 + mova [cq+64*15], m16 + ret +.main_fast_rect2: + call m(idct_8x16_internal_10bpc).round +.main_fast: ; bottom half is zero + pmulld m23, m0, [o(pd_4091)] {1to16} ; t31a + pmulld m0, [o(pd_201)] {1to16} ; t16a + pmulld m16, m7, [o(pd_2751)] {1to16} ; t17a + pmulld m7, [o(pd_3035)] {1to16} ; t30a + pmulld m19, m4, [o(pd_3703)] {1to16} ; t29a + pmulld m4, [o(pd_1751)] {1to16} ; t18a + pmulld m20, m3, [o(pd_1380)] {1to16} ; t19a + pmulld m3, [o(pd_3857)] {1to16} ; t28a + pmulld m21, m2, [o(pd_3973)] {1to16} ; t27a + pmulld m2, [o(pd_995)] {1to16} ; t20a + pmulld m18, m5, [o(pd_2106)] {1to16} ; t21a + pmulld m5, [o(pd_3513)] {1to16} ; t26a + pmulld m17, m6, [o(pd_3290)] {1to16} ; t25a + pmulld m6, [o(pd_2440)] {1to16} ; t22a + pmulld m22, m1, [o(pd_601)] {1to16} ; t23a + pmulld m1, [o(pd_4052)] {1to16} ; t24a + REPX {psubd x, m13, x}, m16, m20, m18, m22 + call m(idct_16x16_internal_10bpc).round3 + jmp .main2 +.main_rect2: + call m(idct_8x16_internal_10bpc).round + call m(idct_16x16_internal_10bpc).round +.main: + ITX_MULSUB_2D 0, 23, 8, 9, 10, _, 201, 4091 ; t16a, t31a + ITX_MULSUB_2D 16, 7, 8, 9, 10, _, 3035, 2751 ; t17a, t30a + ITX_MULSUB_2D 4, 19, 8, 9, 10, _, 1751, 3703 ; t18a, t29a + ITX_MULSUB_2D 20, 3, 8, 9, 10, _, 3857, 1380 ; t19a, t28a + ITX_MULSUB_2D 2, 21, 8, 9, 10, _, 995, 3973 ; t20a, t27a + ITX_MULSUB_2D 18, 5, 8, 9, 10, _, 3513, 2106 ; t21a, t26a + ITX_MULSUB_2D 6, 17, 8, 9, 10, _, 2440, 3290 ; t22a, t25a + ITX_MULSUB_2D 22, 1, 8, 9, 10, _, 4052, 601 ; t23a, t24a + call m(idct_16x16_internal_10bpc).round +.main2: + call m(idct_8x16_internal_10bpc).round + psubd m8, m0, m16 ; t17 + paddd m0, m16 ; t16 + psubd m16, m23, m7 ; t30 + paddd m23, m7 ; t31 + REPX {pmaxsd x, m14}, m8, m0, m16, m23 + paddd m7, m20, m4 ; t19 + psubd m20, m4 ; t18 + REPX {pminsd x, m15}, m8, m0, m16, m23 + paddd m4, m3, m19 ; t28 + psubd m3, m19 ; t29 + REPX {pmaxsd x, m14}, m7, m20, m4, m3 + psubd m19, m2, m18 ; t21 + paddd m2, m18 ; t20 + REPX {pminsd x, m15}, m7, m20, m4, m3 + psubd m18, m21, m5 ; t26 + paddd m21, m5 ; t27 + REPX {pmaxsd x, m14}, m19, m2, m18, m21 + psubd m5, m22, m6 ; t22 + paddd m6, m22 ; t23 + REPX {pminsd x, m15}, m19, m2, m18, m21 + psubd m22, m1, m17 ; t25 + paddd m17, m1 ; t24 + REPX {pmaxsd x, m14}, m5, m6, m22, m17 + vpbroadcastd m11, [o(pd_4017)] + vpbroadcastd m10, [o(pd_799)] + REPX {pminsd x, m15}, m5, m6, m22, m17 + ITX_MULSUB_2D 16, 8, 9, 1, _, 13, 10, 11 ; t17a, t30a + ITX_MULSUB_2D 3, 20, 9, 1, _, 13, 10, 11, 2 ; t29a, t18a + vpbroadcastd m11, [o(pd_2276)] + vpbroadcastd m10, [o(pd_3406)] + ITX_MULSUB_2D 18, 19, 9, 1, _, 13, 10, 11 ; t21a, t26a + ITX_MULSUB_2D 22, 5, 9, 1, _, 13, 10, 11, 2 ; t25a, t22a + paddd m1, m6, m2 ; t23a + psubd m6, m2 ; t20a + psubd m2, m17, m21 ; t27a + paddd m17, m21 ; t24a + REPX {pmaxsd x, m14}, m1, m6, m2, m17 + psubd m21, m23, m4 ; t28a + paddd m23, m4 ; t31a + REPX {pminsd x, m15}, m1, m6, m2, m17 + psubd m4, m16, m20 ; t18 + paddd m16, m20 ; t17 + REPX {pmaxsd x, m14}, m21, m23, m4, m16 + psubd m20, m0, m7 ; t19a + paddd m0, m7 ; t16a + REPX {pminsd x, m15}, m21, m23, m4, m16 + psubd m7, m8, m3 ; t29 + paddd m3, m8 ; t30 + REPX {pmaxsd x, m14}, m20, m0, m7, m3 + paddd m8, m5, m18 ; t22 + psubd m5, m18 ; t21 + REPX {pminsd x, m15}, m20, m0, m7, m3 + psubd m18, m22, m19 ; t26 + paddd m22, m19 ; t25 + REPX {pmaxsd x, m14}, m8, m5, m18, m22 + vpbroadcastd m11, [o(pd_3784)] + vpbroadcastd m10, [o(pd_1567)] + REPX {pminsd x, m15}, m8, m5, m18, m22 + ITX_MULSUB_2D 21, 20, 9, 19, _, 13, 10, 11 ; t19, t28 + ITX_MULSUB_2D 2, 6, 9, 19, _, 13, 10, 11, 2 ; t27, t20 + ITX_MULSUB_2D 7, 4, 9, 19, _, 13, 10, 11 ; t18a, t29a + ITX_MULSUB_2D 18, 5, 9, 19, _, 13, 10, 11, 2 ; t26a, t21a + psubd m19, m0, m1 ; t23 + paddd m0, m1 ; t16 + paddd m1, m8, m16 ; t17a + psubd m8, m16, m8 ; t22a + REPX {pmaxsd x, m14}, m19, m0, m1, m8 + psubd m16, m23, m17 ; t24 + paddd m23, m17 ; t31 + REPX {pminsd x, m15}, m19, m0, m1, m8 + psubd m17, m3, m22 ; t25a + paddd m22, m3 ; t30a + REPX {pmaxsd x, m14}, m16, m23, m17, m22 + paddd m3, m6, m21 ; t19a + psubd m6, m21, m6 ; t20a + REPX {pminsd x, m15}, m16, m23, m17, m22 + paddd m21, m18, m4 ; t29 + psubd m18, m4, m18 ; t26 + REPX {pmaxsd x, m14}, m3, m6, m21, m18 + psubd m4, m20, m2 ; t27a + paddd m20, m2 ; t28a + REPX {pminsd x, m15}, m3, m6, m21, m18 + paddd m2, m7, m5 ; t18 + psubd m7, m5 ; t21 + REPX {pmaxsd x, m14}, m4, m20, m2, m7 + REPX {pminsd x, m15}, m4, m20, m2, m7 + REPX {pmulld x, m12}, m18, m16, m4, m17, m7, m19, m6, m8 + REPX {paddd x, m13}, m18, m16, m4, m17 + psubd m5, m18, m7 ; t21a + paddd m18, m7 ; t26a + psubd m7, m16, m19 ; t23a + paddd m16, m19 ; t24a + REPX {psrad x, 12 }, m5, m18, m7, m16 + paddd m19, m4, m6 ; t27 + psubd m4, m6 ; t20 + psubd m6, m17, m8 ; t22 + paddd m17, m8 ; t25 + REPX {psrad x, 12 }, m19, m4, m6, m17 + ret + +cglobal inv_txfm_add_identity_identity_32x16_10bpc, 4, 7, 16, dst, stride, c, eob +%undef cmp + vpbroadcastd m10, [pw_2896x8] + vpbroadcastd m11, [pw_1697x16] + vpbroadcastd m13, [pw_2048] + vpbroadcastd m15, [pixel_10bpc_max] + lea r6, [strideq*9] + pxor m14, m14 + cmp eobd, 151 + jl .main + mov r4, dstq + call .main + add cq, 64*12 + lea dstq, [r4+32] +.main: + call .main_internal + add cq, 64*4 + pmulhrsw m1, m13, m2 + pmulhrsw m3, m13, m4 + pmulhrsw m5, m13, m6 + pmulhrsw m7, m13, m8 + call .main_internal + jmp m(inv_txfm_add_identity_identity_16x32_10bpc).main2 +.main_internal: + mova m8, [cq+64* 0] + packssdw m8, [cq+64* 8] + mova m6, [cq+64* 1] + packssdw m6, [cq+64* 9] + mova m0, [cq+64* 2] + packssdw m0, [cq+64*10] + mova m2, [cq+64* 3] + packssdw m2, [cq+64*11] + REPX {pmulhrsw x, m10}, m8, m6, m0, m2 + REPX {paddsw x, x }, m8, m6, m0, m2 + REPX {vpermq x, x, q3120}, m8, m6, m0, m2 + pmulhrsw m4, m11, m8 + pmulhrsw m9, m11, m6 + paddsw m8, m8 + paddsw m6, m6 + REPX {mova [cq+64*x], m14}, 0, 1, 2, 3 + paddsw m8, m4 + paddsw m6, m9 + pmulhrsw m4, m11, m0 + pmulhrsw m9, m11, m2 + paddsw m0, m0 + paddsw m2, m2 + REPX {mova [cq+64*x], m14}, 8, 9, 10, 11 + paddsw m0, m4 + paddsw m2, m9 + punpcklwd m4, m8, m6 + punpckhwd m8, m6 + punpcklwd m6, m0, m2 + punpckhwd m0, m2 + punpckldq m2, m4, m6 ; 0 1 + punpckhdq m4, m6 ; 2 3 + punpckldq m6, m8, m0 ; 4 5 + punpckhdq m8, m0 ; 6 7 + ret + +cglobal inv_txfm_add_dct_dct_32x32_10bpc, 4, 7, 0, dst, stride, c, eob +%undef cmp + lea r5, [o_base] + test eobd, eobd + jz .dconly + vpbroadcastd m12, [o(pd_2896)] + vpbroadcastd m13, [o(pd_2048)] + vpbroadcastd m14, [o(clip_18b_min)] + vpbroadcastd m15, [o(clip_18b_max)] + WIN64_SPILL_XMM 30 + cmp eobd, 136 + jl .fast + add cq, 64 + cmp eobd, 543 + jge .full + call .pass1_fast ; bottomright 16x16 zero + mov r6d, 16*12 + jmp .lefthalf +.full: + call .pass1 + mov r6d, 16*28 +.lefthalf: + mova [cq+128* 0], m0 + mova [cq+128* 1], m1 + mova [cq+128* 2], m2 + mova [cq+128* 3], m3 + mova [cq+128* 4], m14 + mova [cq+128* 5], m15 + mova [cq+128* 6], m16 + mova [cq+128* 7], m17 + mova [cq+128* 8], m22 + mova [cq+128* 9], m23 + mova [cq+128*10], m24 + mova [cq+128*11], m25 + mova [cq+128*12], m26 + mova [cq+128*13], m27 + mova [cq+128*14], m28 + mova [cq+128*15], m29 + sub cq, 64 + vpbroadcastd m12, [o(pd_2896)] + vpbroadcastd m13, [o(pd_2048)] + vpbroadcastd m14, [o(clip_18b_min)] + vpbroadcastd m15, [o(clip_18b_max)] + call .pass1 + lea r5, [o_base_8bpc] + mova m4, [cq+64+128* 0] + mova m5, [cq+64+128* 1] + mova m6, [cq+64+128* 2] + mova m7, [cq+64+128* 3] + mova m18, [cq+64+128* 4] + mova m19, [cq+64+128* 5] + mova m20, [cq+64+128* 6] + mova m21, [cq+64+128* 7] + call m(inv_txfm_add_dct_dct_32x8_8bpc).main + call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf + mova [cq+128*0], m14 + mova [cq+128*1], m15 + mova [cq+128*2], m16 + mova [cq+128*3], m17 + mova [cq+128*4], m18 + mova [cq+128*5], m19 + mova [cq+128*6], m20 + mova [cq+128*7], m21 + mova m14, [cq+64+128* 8] + mova m15, [cq+64+128* 9] + mova m16, [cq+64+128*10] + mova m17, [cq+64+128*11] + mova m18, [cq+64+128*12] + mova m19, [cq+64+128*13] + mova m20, [cq+64+128*14] + mova m21, [cq+64+128*15] + call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf + pxor m12, m12 +.right_zero_loop: + mova [cq+r6*8+64+128*3], m12 + mova [cq+r6*8+64+128*2], m12 + mova [cq+r6*8+64+128*1], m12 + mova [cq+r6*8+64+128*0], m12 + sub r6d, 16*4 + jge .right_zero_loop + mov r6d, 16*28 + jmp .end2 +.fast: ; topleft 16x16 nonzero + cmp eobd, 36 + jl .fast2 + call .pass1_fast + lea r5, [o_base_8bpc] + call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast + mova [cq+128*0], m14 + mova [cq+128*1], m15 + mova [cq+128*2], m16 + mova [cq+128*3], m17 + mova [cq+128*4], m18 + mova [cq+128*5], m19 + mova [cq+128*6], m20 + mova [cq+128*7], m21 + call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast + jmp .end +.fast2: ; topleft 8x8 nonzero + movshdup m7, [o(permB)] + mova ym0, [cq+128*0] + mova ym1, [cq+128*4] + mova ym4, [cq+128*2] + mova ym5, [cq+128*6] + mova ym16, [cq+128*1] + mova ym2, [cq+128*5] + mova ym3, [cq+128*3] + mova ym17, [cq+128*7] + mov r6d, 16*4 + vpermq m0, m7, m0 ; 0 0 + vpermq m1, m7, m1 ; 4 4 + vpermt2q m4, m7, m5 ; 2 6 + vpermt2q m16, m7, m2 ; 1 5 + vpermt2q m17, m7, m3 ; 7 3 + call m(inv_txfm_add_dct_dct_32x8_10bpc).main_fast2 + call m(idct_16x16_internal_10bpc).main_end + call m(inv_txfm_add_dct_dct_32x8_10bpc).transpose_8x32 + lea r5, [o_base_8bpc] + punpckhqdq m22, m0, m2 ; 1 + punpcklqdq m0, m2 ; 0 + punpcklqdq m1, m5, m7 ; 4 + punpckhqdq m24, m5, m7 ; 5 + punpcklqdq m14, m3, m4 ; 2 + punpckhqdq m23, m3, m4 ; 3 + punpcklqdq m15, m6, m8 ; 6 + punpckhqdq m25, m6, m8 ; 7 + mova m10, m13 + call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast2 + mova [cq+128*0], m14 + mova [cq+128*1], m15 + mova [cq+128*2], m16 + mova [cq+128*3], m17 + mova [cq+128*4], m18 + mova [cq+128*5], m19 + mova [cq+128*6], m20 + mova [cq+128*7], m21 + call m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast2 +.end: + pxor m12, m12 +.end2: + psubsw m9, m0, m29 ; out31 + paddsw m0, m29 ; out0 + psubsw m29, m1, m28 ; out30 + paddsw m1, m28 ; out1 + psubsw m28, m2, m27 ; out29 + paddsw m2, m27 ; out2 + psubsw m27, m3, m26 ; out28 + paddsw m3, m26 ; out3 + psubsw m26, m4, m25 ; out27 + paddsw m4, m25 ; out4 + psubsw m25, m5, m24 ; out26 + paddsw m5, m24 ; out5 + psubsw m24, m6, m23 ; out25 + paddsw m6, m23 ; out6 + psubsw m23, m7, m22 ; out24 + paddsw m7, m22 ; out7 + call m(inv_txfm_add_dct_dct_32x8_10bpc).write_32x8_start + mova m0, [cq+128*0] + mova m1, [cq+128*1] + mova m2, [cq+128*2] + mova m3, [cq+128*3] + mova m4, [cq+128*4] + mova m5, [cq+128*5] + mova m6, [cq+128*6] + mova m7, [cq+128*7] + psubsw m22, m0, m21 ; out23 + paddsw m0, m21 ; out8 + psubsw m21, m1, m20 ; out22 + paddsw m1, m20 ; out9 + psubsw m20, m2, m19 ; out21 + paddsw m2, m19 ; out10 + psubsw m19, m3, m18 ; out20 + paddsw m3, m18 ; out11 + psubsw m18, m4, m17 ; out19 + paddsw m4, m17 ; out12 + psubsw m17, m5, m16 ; out18 + paddsw m5, m16 ; out13 + psubsw m16, m6, m15 ; out17 + paddsw m6, m15 ; out14 + psubsw m15, m7, m14 ; out16 + paddsw m7, m14 ; out15 +.zero_loop: + mova [cq+r6*8+128*3], m12 + mova [cq+r6*8+128*2], m12 + mova [cq+r6*8+128*1], m12 + mova [cq+r6*8+128*0], m12 + sub r6d, 16*4 + jge .zero_loop + call m(inv_txfm_add_dct_dct_32x8_10bpc).write_32x8 + pmulhrsw m0, m11, m15 + pmulhrsw m1, m11, m16 + pmulhrsw m2, m11, m17 + pmulhrsw m3, m11, m18 + call m(inv_txfm_add_dct_dct_32x8_10bpc).write_32x4 + pmulhrsw m0, m11, m19 + pmulhrsw m1, m11, m20 + pmulhrsw m2, m11, m21 + pmulhrsw m3, m11, m22 + call m(inv_txfm_add_dct_dct_32x8_10bpc).write_32x4 + pmulhrsw m0, m11, m23 + pmulhrsw m1, m11, m24 + pmulhrsw m2, m11, m25 + pmulhrsw m3, m11, m26 + call m(inv_txfm_add_dct_dct_32x8_10bpc).write_32x4 + pmulhrsw m0, m11, m27 + pmulhrsw m1, m11, m28 + pmulhrsw m2, m11, m29 + pmulhrsw m3, m11, m9 + WIN64_RESTORE_XMM + vzeroupper + jmp m(inv_txfm_add_dct_dct_32x8_10bpc).write_32x4 +.dconly: + imul r6d, [cq], 181 + mov [cq], eobd + or r3d, 32 + add r6d, 640 + sar r6d, 10 + jmp m(inv_txfm_add_dct_dct_32x16_10bpc).dconly2 +.pass1_fast: + mova m0, [cq+128* 0] + mova m1, [cq+128* 4] + mova m2, [cq+128* 8] + mova m3, [cq+128*12] + mov r6d, 16*12 + call m(idct_8x16_internal_10bpc).main_fast + mova m16, [cq+128* 2] + mova m17, [cq+128* 6] + mova m18, [cq+128*10] + mova m19, [cq+128*14] + call m(idct_16x16_internal_10bpc).main_fast + call .pass1_load_spill + call m(inv_txfm_add_dct_dct_32x16_10bpc).main_fast + jmp .pass1_end +.pass1: + mova m0, [cq+128* 0] + mova m1, [cq+128* 4] + mova m2, [cq+128* 8] + mova m3, [cq+128*12] + mova m4, [cq+128*16] + mova m5, [cq+128*20] + mova m6, [cq+128*24] + mova m7, [cq+128*28] + call m(idct_8x16_internal_10bpc).main + mova m16, [cq+128* 2] + mova m17, [cq+128* 6] + mova m18, [cq+128*10] + mova m19, [cq+128*14] + mova m20, [cq+128*18] + mova m21, [cq+128*22] + mova m22, [cq+128*26] + mova m23, [cq+128*30] + call m(idct_16x16_internal_10bpc).main + call .pass1_load_spill + mova m16, [cq+128*17] + mova m17, [cq+128*19] + mova m18, [cq+128*21] + mova m19, [cq+128*23] + mova m20, [cq+128*25] + mova m21, [cq+128*27] + mova m22, [cq+128*29] + mova m23, [cq+128*31] + call m(inv_txfm_add_dct_dct_32x16_10bpc).main +.pass1_end: + vpbroadcastd m11, [o(pd_2)] + lea r4, [cq+128*8] + call m(inv_txfm_add_dct_dct_32x16_10bpc).idct32_pass1_end + punpckhqdq m22, m0, m20 ; 1 + punpcklqdq m0, m20 ; 0 + punpckhqdq m24, m2, m1 ; 5 + punpcklqdq m1, m2, m1 ; 4 + punpcklqdq m2, m14, m18 ; 8 + punpckhqdq m26, m14, m18 ; 9 + punpcklqdq m14, m15, m4 ; 2 + punpckhqdq m23, m15, m4 ; 3 + punpckhqdq m25, m3, m21 ; 7 + punpcklqdq m15, m3, m21 ; 6 + punpckhqdq m28, m6, m17 ; 13 + punpcklqdq m3, m6, m17 ; 12 + punpckhqdq m27, m5, m16 ; 11 + punpcklqdq m16, m5, m16 ; 10 + punpckhqdq m29, m7, m8 ; 15 + punpcklqdq m17, m7, m8 ; 14 + ret +.pass1_load_spill: + call m(inv_txfm_add_dct_dct_32x16_10bpc).idct16_sumsub + mova [cq+128* 0], m0 + mova m0, [cq+128* 1] + mova [cq+128* 1], m1 + mova [cq+128* 2], m2 + mova m1, [cq+128* 3] + mova m2, [cq+128* 5] + mova [cq+128* 3], m3 + mova [cq+128* 4], m4 + mova m3, [cq+128* 7] + mova m4, [cq+128* 9] + mova [cq+128* 5], m5 + mova [cq+128* 6], m6 + mova [cq+128* 7], m7 + mova m5, [cq+128*11] + mova m6, [cq+128*13] + mova m7, [cq+128*15] + mova [cq+128* 8], m23 + mova [cq+128* 9], m22 + mova [cq+128*10], m21 + mova [cq+128*11], m20 + mova [cq+128*12], m19 + mova [cq+128*13], m18 + mova [cq+128*14], m17 + mova [cq+128*15], m16 + ret + +cglobal inv_txfm_add_identity_identity_32x32_10bpc, 4, 7, 16, dst, stride, c, eob +%undef cmp + vpbroadcastd m13, [pw_8192] + vpbroadcastd m15, [pixel_10bpc_max] + pxor m14, m14 + lea r6, [strideq*9] + cmp eobd, 136 + jl .main + mov r4, dstq + call .main + add cq, 64-128*4 + lea dstq, [dstq+strideq*8] + call .main + add cq, 128*12-64 + lea dstq, [r4+32] + cmp eobd, 543 + jl .main + call .main + add cq, 64-128*4 + lea dstq, [dstq+strideq*8] +.main: + call .main_internal + add cq, 128*4 + pmulhrsw m1, m13, m2 + pmulhrsw m3, m13, m4 + pmulhrsw m5, m13, m6 + pmulhrsw m7, m13, m8 + call .main_internal + jmp m(inv_txfm_add_identity_identity_16x32_10bpc).main2 +.main_internal: + mova m8, [cq+128* 0] + packssdw m8, [cq+128* 8] + mova m6, [cq+128* 1] + packssdw m6, [cq+128* 9] + mova m0, [cq+128* 2] + packssdw m0, [cq+128*10] + mova m2, [cq+128* 3] + packssdw m2, [cq+128*11] + REPX {vpermq x, x, q3120}, m8, m6, m0, m2 + REPX {mova [cq+128*x], m14}, 0, 1, 2, 3 + punpcklwd m4, m8, m6 + punpckhwd m8, m6 + punpcklwd m6, m0, m2 + punpckhwd m0, m2 + REPX {mova [cq+128*x], m14}, 8, 9, 10, 11 + punpckldq m2, m4, m6 ; 0 1 + punpckhdq m4, m6 ; 2 3 + punpckldq m6, m8, m0 ; 4 5 + punpckhdq m8, m0 ; 6 7 + ret + %endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/itx_avx2.asm b/third_party/dav1d/src/x86/itx_avx2.asm --- a/third_party/dav1d/src/x86/itx_avx2.asm +++ b/third_party/dav1d/src/x86/itx_avx2.asm @@ -1192,17 +1192,13 @@ %macro INV_TXFM_8X4_FN 2 ; type1, type2 INV_TXFM_FN %1, %2, 8x4 %ifidn %1_%2, dct_dct movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] - pmulhrsw xm0, xm1 - movd xm2, [o(pw_2048)] + mov [cq], eobd pmulhrsw xm0, xm1 - pmulhrsw xm0, xm2 - vpbroadcastw m0, xm0 - mova m1, m0 - jmp m(iadst_8x4_internal_8bpc).end3 + jmp m(inv_txfm_add_dct_dct_8x8_8bpc).dconly2 %endif %endmacro INV_TXFM_8X4_FN dct, dct INV_TXFM_8X4_FN dct, adst @@ -1338,24 +1334,24 @@ %ifidn %1_%2, dct_dct movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] movd xm2, [o(pw_16384)] mov [cq], eobd + or r3d, 8 +.dconly: pmulhrsw xm0, xm2 - psrlw xm2, 3 ; pw_2048 +.dconly2: + movd xm2, [pw_2048] pmulhrsw xm0, xm1 + lea r2, [strideq*3] pmulhrsw xm0, xm2 vpbroadcastw m0, xm0 -.end: - mov r2d, 2 -.end2: - lea r3, [strideq*3] -.loop: - WRITE_8X4 0, 0, 1, 2 +.dconly_loop: + WRITE_8X4 0, 0, 1, 2, strideq*1, strideq*2, r2 lea dstq, [dstq+strideq*4] - dec r2d - jg .loop + sub r3d, 4 + jg .dconly_loop RET %endif %endmacro INV_TXFM_8X8_FN dct, dct @@ -1541,17 +1537,12 @@ movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] movd xm2, [o(pw_16384)] mov [cq], eobd pmulhrsw xm0, xm1 - pmulhrsw xm0, xm2 - psrlw xm2, 3 ; pw_2048 - pmulhrsw xm0, xm1 - pmulhrsw xm0, xm2 - vpbroadcastw m0, xm0 - mov r2d, 4 - jmp m(inv_txfm_add_dct_dct_8x8_8bpc).end2 + or r3d, 16 + jmp m(inv_txfm_add_dct_dct_8x8_8bpc).dconly %endif %endmacro %macro ITX_8X16_LOAD_COEFS 0 vpbroadcastd m4, [o(pw_2896x8)] @@ -1900,30 +1891,30 @@ %ifidn %1_%2, dct_dct movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] movd xm2, [o(pw_16384)] mov [cq], eobd - mov r2d, 2 + or r3d, 4 .dconly: pmulhrsw xm0, xm2 movd xm2, [pw_2048] ; intentionally rip-relative pmulhrsw xm0, xm1 pmulhrsw xm0, xm2 vpbroadcastw m0, xm0 pxor m3, m3 .dconly_loop: - mova xm1, [dstq] - vinserti128 m1, [dstq+strideq], 1 + mova xm1, [dstq+strideq*0] + vinserti128 m1, [dstq+strideq*1], 1 punpckhbw m2, m1, m3 punpcklbw m1, m3 paddw m2, m0 paddw m1, m0 packuswb m1, m2 - mova [dstq], xm1 - vextracti128 [dstq+strideq], m1, 1 + mova [dstq+strideq*0], xm1 + vextracti128 [dstq+strideq*1], m1, 1 lea dstq, [dstq+strideq*2] - dec r2d + sub r3d, 2 jg .dconly_loop RET %endif %endmacro @@ -2160,11 +2151,11 @@ movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] movd xm2, [o(pw_16384)] mov [cq], eobd pmulhrsw xm0, xm1 - mov r2d, 4 + or r3d, 8 jmp m(inv_txfm_add_dct_dct_16x4_8bpc).dconly %endif %endmacro %macro ITX_16X8_LOAD_COEFS 1 ; shuf_odd @@ -2471,11 +2462,11 @@ %ifidn %1_%2, dct_dct movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] movd xm2, [o(pw_8192)] mov [cq], eobd - mov r2d, 8 + or r3d, 16 jmp m(inv_txfm_add_dct_dct_16x4_8bpc).dconly %endif %endmacro %macro ITX_16X16_LOAD_COEFS 0 @@ -3118,17 +3109,12 @@ .dconly: movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] movd xm2, [o(pw_8192)] mov [cq], eobd - pmulhrsw xm0, xm2 - psrlw xm2, 2 ; pw_2048 - pmulhrsw xm0, xm1 - pmulhrsw xm0, xm2 - vpbroadcastw m0, xm0 - mov r2d, 8 - jmp m(inv_txfm_add_dct_dct_8x8_8bpc).end2 + or r3d, 32 + jmp m(inv_txfm_add_dct_dct_8x8_8bpc).dconly .full: REPX {pmulhrsw x, m9}, m12, m13, m14, m15 pmulhrsw m6, m9, [rsp+32*2] mova [rsp+32*2], m4 pmulhrsw m4, m9, [rsp+32*0] @@ -3288,11 +3274,11 @@ jnz .normal movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] movd xm2, [o(pw_8192)] mov [cq], eobd - mov r2d, 8 + or r3d, 8 .dconly: pmulhrsw xm0, xm2 movd xm2, [pw_2048] ; intentionally rip-relative pmulhrsw xm0, xm1 pmulhrsw xm0, xm2 @@ -3305,11 +3291,11 @@ paddw m2, m0 paddw m1, m0 packuswb m1, m2 mova [dstq], m1 add dstq, strideq - dec r2d + dec r3d jg .dconly_loop RET .normal: PROLOGUE 0, 4, 16, 32*3, dst, stride, c, eob %undef cmp @@ -3670,11 +3656,11 @@ movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] movd xm2, [o(pw_16384)] mov [cq], eobd pmulhrsw xm0, xm1 - mov r2d, 16 + or r3d, 32 jmp m(inv_txfm_add_dct_dct_16x4_8bpc).dconly .full: mova [tmp1q-32*4], m1 mova [tmp1q-32*3], m3 mova [tmp1q-32*2], m5 @@ -3989,11 +3975,11 @@ movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] movd xm2, [o(pw_16384)] mov [cq], eobd pmulhrsw xm0, xm1 - mov r2d, 16 + or r3d, 16 jmp m(inv_txfm_add_dct_dct_32x8_8bpc).dconly .normal: PROLOGUE 0, 6, 16, 32*19, dst, stride, c, eob, tmp1, tmp2 vpbroadcastd m15, [o(pw_2896x8)] pmulhrsw m0, m15, [cq+32* 1] @@ -4220,11 +4206,11 @@ jnz .normal movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] movd xm2, [o(pw_8192)] mov [cq], eobd - mov r2d, 32 + or r3d, 32 jmp m(inv_txfm_add_dct_dct_32x8_8bpc).dconly .normal: PROLOGUE 0, 9, 16, 32*67, dst, stride, c, eob, tmp1, tmp2, \ base, tmp3, tmp4 %undef cmp @@ -4484,11 +4470,11 @@ jnz .normal movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] movd xm2, [o(pw_8192)] mov [cq], eobd - mov r2d, 32 + or r3d, 64 jmp m(inv_txfm_add_dct_dct_16x4_8bpc).dconly .normal: PROLOGUE 0, 10, 16, 32*67, dst, stride, c, eob, tmp1, tmp2 %undef cmp lea tmp1q, [rsp+32*23] @@ -4830,11 +4816,11 @@ jnz .normal movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] movd xm2, [o(pw_8192)] mov [cq], eobd - mov r2d, 16 + or r3d, 16 .dconly: pmulhrsw xm0, xm2 movd xm2, [o(pw_2048)] pmulhrsw xm0, xm1 pmulhrsw xm0, xm2 @@ -4854,11 +4840,11 @@ packuswb m2, m4 packuswb m3, m5 mova [dstq+32*0], m2 mova [dstq+32*1], m3 add dstq, strideq - dec r2d + dec r3d jg .dconly_loop RET .normal: PROLOGUE 0, 7, 16, 32*67, dst, stride, c, eob, tmp1, tmp2 LOAD_8ROWS cq+32*0, 32*4 @@ -4995,11 +4981,11 @@ movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] movd xm2, [o(pw_16384)] mov [cq], eobd pmulhrsw xm0, xm1 - mov r2d, 64 + or r3d, 64 jmp m(inv_txfm_add_dct_dct_32x8_8bpc).dconly .normal: PROLOGUE 0, 11, 16, 32*99, dst, stride, c, eob, tmp1, tmp2 lea tmp1q, [rsp+32*7] lea r10d, [eobq-136] @@ -5198,11 +5184,11 @@ movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] movd xm2, [o(pw_16384)] mov [cq], eobd pmulhrsw xm0, xm1 - mov r2d, 32 + or r3d, 32 jmp m(inv_txfm_add_dct_dct_64x16_8bpc).dconly .normal: PROLOGUE 0, 9, 16, 32*131, dst, stride, c, eob, tmp1, tmp2, \ base, tmp3, tmp4 lea tmp1q, [rsp+32*7] @@ -5379,11 +5365,11 @@ jnz .normal movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] movd xm2, [o(pw_8192)] mov [cq], eobd - mov r2d, 64 + or r3d, 64 jmp m(inv_txfm_add_dct_dct_64x16_8bpc).dconly .normal: PROLOGUE 0, 11, 16, 32*199, dst, stride, c, eob, tmp1, tmp2 lea tmp1q, [rsp+32*71] lea r10d, [eobq-136] diff --git a/third_party/dav1d/src/x86/itx_avx512.asm b/third_party/dav1d/src/x86/itx_avx512.asm --- a/third_party/dav1d/src/x86/itx_avx512.asm +++ b/third_party/dav1d/src/x86/itx_avx512.asm @@ -1,7 +1,7 @@ -; Copyright © 2020, VideoLAN and dav1d authors -; Copyright © 2020, Two Orioles, LLC +; Copyright © 2020-2023, VideoLAN and dav1d authors +; Copyright © 2020-2023, Two Orioles, LLC ; All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions are met: ; @@ -28,10 +28,15 @@ %if ARCH_X86_64 SECTION_RODATA 64 const \ +dup16_perm, db 0, 1, 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7 + db 8, 9, 8, 9, 10, 11, 10, 11, 12, 13, 12, 13, 14, 15, 14, 15 + db 16, 17, 16, 17, 18, 19, 18, 19, 20, 21, 20, 21, 22, 23, 22, 23 + db 24, 25, 24, 25, 26, 27, 26, 27, 28, 29, 28, 29, 30, 31, 30, 31 +const \ int8_permA, db 0, 1, 16, 17, 32, 33, 48, 49, 2, 3, 18, 19, 34, 35, 50, 51 db 4, 5, 20, 21, 36, 37, 52, 53, 6, 7, 22, 23, 38, 39, 54, 55 db 8, 9, 24, 25, 40, 41, 56, 57, 10, 11, 26, 27, 42, 43, 58, 59 db 12, 13, 28, 29, 44, 45, 60, 61, 14, 15, 30, 31, 46, 47, 62, 63 int8_permB: db 0, 1, 16, 17, 32, 33, 48, 49, 2, 3, 18, 19, 34, 35, 50, 51 @@ -40,14 +45,10 @@ db 12, 13, 28, 29, 44, 45, 60, 61, 14, 15, 30, 31, 46, 47, 62, 63 int16_perm: db 0, 1, 32, 33, 2, 3, 34, 35, 4, 5, 36, 37, 6, 7, 38, 39 db 8, 9, 40, 41, 10, 11, 42, 43, 12, 13, 44, 45, 14, 15, 46, 47 db 16, 17, 48, 49, 18, 19, 50, 51, 20, 21, 52, 53, 22, 23, 54, 55 db 24, 25, 56, 57, 26, 27, 58, 59, 28, 29, 60, 61, 30, 31, 62, 63 -dup16_perm: db 0, 1, 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7 - db 8, 9, 8, 9, 10, 11, 10, 11, 12, 13, 12, 13, 14, 15, 14, 15 - db 16, 17, 16, 17, 18, 19, 18, 19, 20, 21, 20, 21, 22, 23, 22, 23 - db 24, 25, 24, 25, 26, 27, 26, 27, 28, 29, 28, 29, 30, 31, 30, 31 idtx_16x4p: db 0, 1, 4, 5, 16, 17, 20, 21, 2, 3, 6, 7, 18, 19, 22, 23 db 32, 33, 36, 37, 48, 49, 52, 53, 34, 35, 38, 39, 50, 51, 54, 55 db 8, 9, 12, 13, 24, 25, 28, 29, 10, 11, 14, 15, 26, 27, 30, 31 db 40, 41, 44, 45, 56, 57, 60, 61, 42, 43, 46, 47, 58, 59, 62, 63 idct_8x32p: db 60, 61, 4, 5, 32, 33, 0, 1, 28, 29, 36, 37, 56, 57, 8, 9 @@ -83,11 +84,11 @@ pd_0to15: dd 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 gather8a: dd 0, 2, 1, 3, 8, 10, 9, 11 gather8b: dd 0, 1, 4, 5, 8, 9, 12, 13 gather8c: dd 0, 4, 2, 6, 12, 8, 14, 10 -gather8d: dd 0, 3, 1, 2, 8, 11, 9, 10 +gather8d: dd 0, 19, 1, 18, 2, 17, 3, 16 int_shuf1: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15 int_shuf2: db 8, 9, 0, 1, 10, 11, 2, 3, 12, 13, 4, 5, 14, 15, 6, 7 int_shuf3: db 0, 1, 8, 9, 4, 5, 12, 13, 2, 3, 10, 11, 6, 7, 14, 15 int_shuf4: db 8, 9, 0, 1, 12, 13, 4, 5, 10, 11, 2, 3, 14, 15, 6, 7 @@ -1498,12 +1499,12 @@ %macro INV_TXFM_8X16_FN 2 ; type1, type2 INV_TXFM_FN %1, %2, 8x16 %ifidn %1_%2, dct_dct movsx r6d, word [cq] mov [cq], eobd + or r3d, 16 imul r6d, 181 - mov r3d, 16 add r6d, 128 sar r6d, 8 jmp m(inv_txfm_add_dct_dct_8x8_8bpc).dconly %endif %endmacro @@ -1607,10 +1608,57 @@ paddw m3, m7 packuswb m2, m3 vpscatterdq [r3+ym8]{k2}, m2 RET ALIGN function_align +cglobal_label .main_fast2 ; bottom three-quarters are zero + vpbroadcastd ym10, [o(pd_2048)] + vpbroadcastq ym13, [o(int_mshift)] + vpbroadcastd ym3, [o(pw_401_4076x8)] + vpbroadcastd ym5, [o(pw_799_4017x8)] + vpbroadcastd ym4, [o(pw_m1189_3920x8)] + pxor ym6, ym6 + punpckhwd ym2, ym0, ym0 + pmulhrsw ym2, ym3 ; t8a t15a + punpcklwd ym7, ym1, ym1 + pmulhrsw ym7, ym5 ; t4a t7a + punpckhwd ym1, ym1 + pmulhrsw ym4, ym1 ; t11a t12a + vpcmpub k7, ym13, ym10, 6 + punpcklwd ym9, ym6, ym0 + psubsw ym0, ym2, ym4 ; t11a t12a + paddsw ym8, ym2, ym4 ; t8a t15a + mova ym1, ym7 + jmp .main5 +ALIGN function_align +cglobal_label .main_fast ; bottom half is zero + vpbroadcastd ym10, [o(pd_2048)] + vpbroadcastq ym13, [o(int_mshift)] + pxor ym6, ym6 + punpckhwd ym8, ym0, ym0 + punpckhwd ym4, ym3, ym3 + punpckhwd ym5, ym2, ym2 + punpcklwd ym7, ym1, ym1 + punpckhwd ym1, ym1 + punpcklwd ym3, ym3 + punpcklwd ym9, ym6, ym0 + punpcklwd ym6, ym2 + vpbroadcastd ym2, [o(pw_401_4076x8)] + vpbroadcastd ym0, [o(pw_m2598_3166x8)] + vpbroadcastd ym11, [o(pw_1931_3612x8)] + vpbroadcastd ym12, [o(pw_m1189_3920x8)] + pmulhrsw ym8, ym2 ; t8a t15a + vpbroadcastd ym2, [o(pw_799_4017x8)] + pmulhrsw ym0, ym4 ; t9a t14a + vpbroadcastd ym4, [o(pw_m2276_3406x8)] + pmulhrsw ym5, ym11 ; t10a t13a + pmulhrsw ym1, ym12 ; t11a t12a + pmulhrsw ym7, ym2 ; t4a t7a + pmulhrsw ym3, ym4 ; t5a t6a + vpcmpub k7, ym13, ym10, 6 + jmp .main4 +ALIGN function_align cglobal_label .main WRAP_YMM IDCT16_1D_PACKED ret INV_TXFM_8X16_FN adst, dct @@ -1976,11 +2024,11 @@ %macro INV_TXFM_16X8_FN 2 ; type1, type2 INV_TXFM_FN %1, %2, 16x8 %ifidn %1_%2, dct_dct movsx r6d, word [cq] mov [cq], eobd - mov r3d, 8 + or r3d, 8 .dconly: imul r6d, 181 add r6d, 128 sar r6d, 8 .dconly2: @@ -2277,12 +2325,12 @@ %macro INV_TXFM_16X16_FN 2 ; type1, type2 INV_TXFM_FN %1, %2, 16x16 %ifidn %1_%2, dct_dct movsx r6d, word [cq] mov [cq], eobd + or r3d, 16 imul r6d, 181 - mov r3d, 16 add r6d, 128+512 sar r6d, 8+2 jmp m(inv_txfm_add_dct_dct_16x8_8bpc).dconly3 %endif %endmacro @@ -2420,11 +2468,11 @@ vextracti32x4 [r6+strideq*1], m2, 3 vextracti32x4 [r6+strideq*2], m3, 2 vextracti32x4 [r6+r3 ], m3, 3 RET ALIGN function_align -.main_fast2: ; bottom three-quarters are zero +cglobal_label .main_fast2 ; bottom three-quarters are zero vpbroadcastd m10, [o(pd_2048)] vpbroadcastq m13, [o(int_mshift)] vpcmpub k7, m13, m10, 6 .main_fast4: vpbroadcastd m2, [o(pw_401_4076x8)] @@ -2437,11 +2485,11 @@ psubsw m0, m2, m4 ; t11a t12a paddsw m8, m2, m4 ; t8a t15a mova m1, m7 jmp .main5 ALIGN function_align -.main_fast: ; bottom half is zero +cglobal_label .main_fast ; bottom half is zero vpbroadcastd m10, [o(pd_2048)] .main_fast3: vpbroadcastq m13, [o(int_mshift)] vpcmpub k7, m13, m10, 6 .main_fast5: @@ -2772,17 +2820,17 @@ vpermt2q m8, m12, m6 vpermi2q m11, m9, m7 vpermt2q m9, m12, m7 jmp m(idct_16x16_internal_8bpc).end -%macro ITX_UNPACK_MULHRSW 7 ; dst1, dst2/src, tmp, coef[1-4] - vpbroadcastd m%3, [o(pw_%4_%5x8)] - punpcklwd m%1, m%2, m%2 - pmulhrsw m%1, m%3 - vpbroadcastd m%3, [o(pw_%6_%7x8)] - punpckhwd m%2, m%2 - pmulhrsw m%2, m%3 +%macro ITX_UNPACK_MULHRSW 8 ; dst[1-2], src, tmp, coef[1-4] + vpbroadcastd m%4, [o(pw_%5_%6x8)] + punpcklwd m%1, m%3, m%3 + pmulhrsw m%1, m%4 + vpbroadcastd m%4, [o(pw_%7_%8x8)] + punpckhwd m%2, m%3, m%3 + pmulhrsw m%2, m%4 %endmacro cglobal inv_txfm_add_dct_dct_8x32_8bpc, 4, 4, 0, dst, stride, c, eob %undef cmp lea r5, [o_base] @@ -2866,86 +2914,90 @@ REPX {pmulhrsw x, m4}, m6, m17, m2, m16 vinserti32x4 ym0, ym2, xm6, 1 ; 0 2 vshufi32x4 ym1, ym2, ym6, 0x03 ; 4 6 vinserti32x4 ym14, ym16, xm17, 1 ; 1 3 vshufi32x4 ym15, ym16, ym17, 0x03 ; 5 7 - pxor ym4, ym4 vpermt2q m2, m5, m6 ; 8 10 vpermt2q m16, m5, m17 ; 9 11 - mova ym5, ym4 - mova ym6, ym4 - mova ym7, ym4 vextracti32x8 ym3, m2, 1 ; 12 14 vextracti32x8 ym17, m16, 1 ; 13 15 - call m(idct_8x16_internal_8bpc).main + call m(idct_8x16_internal_8bpc).main_fast call .main_fast .end: - vpbroadcastd ym12, strided - vpbroadcastd m13, [o(pw_2048)] - pmulld ym7, ym12, [o(gather8d)] - REPX {pmulhrsw x, m13}, m0, m1, m2, m3, m8, m9, m10, m11 + vpbroadcastd ym8, strided + pmulld ym8, [o(gather8d)] + call .main_end lea r3, [dstq+strideq*4] - shl strideq, 4 - lea r4, [dstq+strideq] - add r1, r3 kxnorb k1, k1, k1 - pxor m6, m6 + lea r4, [dstq+strideq*8] + pxor m9, m9 + lea r1, [r3+strideq*8] kmovb k2, k1 - vpgatherdq m12{k1}, [r0+ym7] + vpgatherdq m12{k1}, [r0+ym8] kmovb k1, k2 - vpgatherdq m13{k2}, [r3+ym7] + vpgatherdq m13{k2}, [r3+ym8] kmovb k2, k1 - vpgatherdq m14{k1}, [r4+ym7] + vpgatherdq m14{k1}, [r4+ym8] kmovb k1, k2 - vpgatherdq m15{k2}, [r1+ym7] - REPX {mova [cq+64*x], m6}, 0, 1, 2, 3, 4, 5, 6, 7 - punpcklbw m4, m12, m6 - punpckhbw m12, m6 - paddw m0, m4 + vpgatherdq m15{k2}, [r1+ym8] + REPX {pmulhrsw x, m10}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {mova [cq+64*x], m9}, 0, 1, 2, 3, 4, 5, 6, 7 + punpcklbw m11, m12, m9 + punpckhbw m12, m9 + paddw m0, m11 paddw m1, m12 packuswb m0, m1 kmovb k2, k1 - vpscatterdq [r0+ym7]{k1}, m0 - punpcklbw m4, m13, m6 - punpckhbw m13, m6 - paddw m2, m4 + vpscatterdq [r0+ym8]{k1}, m0 + punpcklbw m12, m13, m9 + punpckhbw m13, m9 + paddw m2, m12 paddw m3, m13 packuswb m2, m3 kmovb k1, k2 - vpscatterdq [r3+ym7]{k2}, m2 - punpcklbw m4, m14, m6 - punpckhbw m14, m6 - paddw m8, m4 - paddw m9, m14 - packuswb m8, m9 + vpscatterdq [r3+ym8]{k2}, m2 + punpcklbw m13, m14, m9 + punpckhbw m14, m9 + paddw m4, m13 + paddw m5, m14 + packuswb m4, m5 kmovb k2, k1 - vpscatterdq [r4+ym7]{k1}, m8 - punpcklbw m4, m15, m6 - punpckhbw m15, m6 - paddw m10, m4 - paddw m11, m15 - packuswb m10, m11 - vpscatterdq [r1+ym7]{k2}, m10 + vpscatterdq [r4+ym8]{k1}, m4 + punpcklbw m14, m15, m9 + punpckhbw m15, m9 + paddw m6, m14 + paddw m7, m15 + packuswb m6, m7 + vpscatterdq [r1+ym8]{k2}, m6 RET .dconly: movsx r6d, word [cq] mov [cq], eobd - mov r3d, 32 + or r3d, 32 imul r6d, 181 add r6d, 128+512 sar r6d, 8+2 jmp m(inv_txfm_add_dct_dct_8x8_8bpc).dconly2 INIT_YMM avx512icl ALIGN function_align -.main_fast: ; bottom half is zero - ITX_UNPACK_MULHRSW 12, 14, 8, 201, 4091, m601, 4052 ; t16a, t31a, t23a, t24a - ITX_UNPACK_MULHRSW 21, 15, 8, 995, 3973, m1380, 3857 ; t20a, t27a, t19a, t28a - ITX_UNPACK_MULHRSW 20, 16, 8, 1751, 3703, m2106, 3513 ; t18a, t29a, t21a, t26a - ITX_UNPACK_MULHRSW 19, 17, 8, 2440, 3290, m2751, 3035 ; t22a, t25a, t17a, t30a +cglobal_label .main_fast2 ; bottom three-quarters are zero + ITX_UNPACK_MULHRSW 12, 14, 14, 8, 201, 4091, m601, 4052 ; t16a, t31a, t23a, t24a + ITX_UNPACK_MULHRSW 21, 20, 15, 8, 995, 3973, m1380, 3857 ; t20a, t27a, t19a, t28a + mova m11, m12 + mova m17, m20 + mova m15, m21 + mova m16, m14 + jmp .main4 +ALIGN function_align +cglobal_label .main_fast ; bottom half is zero + ITX_UNPACK_MULHRSW 12, 14, 14, 8, 201, 4091, m601, 4052 ; t16a, t31a, t23a, t24a + ITX_UNPACK_MULHRSW 21, 15, 15, 8, 995, 3973, m1380, 3857 ; t20a, t27a, t19a, t28a + ITX_UNPACK_MULHRSW 20, 16, 16, 8, 1751, 3703, m2106, 3513 ; t18a, t29a, t21a, t26a + ITX_UNPACK_MULHRSW 19, 17, 17, 8, 2440, 3290, m2751, 3035 ; t22a, t25a, t17a, t30a jmp .main3 ALIGN function_align -.main: +cglobal_label .main punpcklwd m12, m21, m14 ; in31 in1 punpckhwd m14, m21 ; in3 in29 punpcklwd m21, m20, m15 ; in27 in5 punpckhwd m15, m20 ; in7 in25 punpcklwd m20, m19, m16 ; in23 in9 @@ -2968,10 +3020,11 @@ paddsw m20, m15 ; t19 t28 psubsw m15, m21, m16 ; t21 t26 paddsw m21, m16 ; t20 t27 psubsw m16, m14, m19 ; t22 t25 paddsw m14, m19 ; t23 t24 +.main4: ITX_MUL2X_PACK 11, 18, 19, 10, 799, 4017, 5 ; t17a t30a ITX_MUL2X_PACK 17, 18, 19, 10, m4017, 799, 5 ; t18a t29a ITX_MUL2X_PACK 15, 18, 19, 10, 3406, 2276, 5 ; t21a t26a ITX_MUL2X_PACK 16, 18, 19, 10, m2276, 3406, 5 ; t22a t25a vpbroadcastd m8, [o(pw_m3784_1567)] @@ -2999,46 +3052,41 @@ psubsw m17, m19, m12 ; t20a t27a paddsw m19, m12 ; t19a t28a REPX {pshufb x, m18}, m20, m11, m21, m19 ITX_MUL2X_PACK 15, 18, 12, 10, 8, 9, 8 ; t23a t22a ITX_MUL2X_PACK 14, 13, 15, 10, 8, 9, 8 ; t22 t25 - packssdw m18, m13 ; t23a t22 - packssdw m12, m15 ; t24a t25 + packssdw m18, m13 ; t23a t22 + packssdw m12, m15 ; t24a t25 ITX_MUL2X_PACK 16, 13, 15, 10, 8, 9, 8 ; t21a t26a ITX_MUL2X_PACK 17, 16, 14, 10, 8, 9, 8 ; t20 t27 packssdw m16, m13 ; t20 t21a packssdw m14, m15 ; t27 t26a punpcklqdq m13, m19, m21 ; t19a t18 punpckhqdq m19, m21 ; t28a t29 punpcklqdq m21, m20, m11 ; t16 t17a punpckhqdq m20, m11 ; t31 t30a - psubsw m15, m1, m19 ; out28 out29 - paddsw m1, m19 ; out3 out2 - psubsw m9, m6, m13 ; out19 out18 - paddsw m6, m13 ; out12 out13 - psubsw m10, m5, m16 ; out20 out21 - paddsw m5, m16 ; out11 out10 - psubsw m19, m3, m12 ; out24 out25 - paddsw m3, m12 ; out7 out6 - psubsw m8, m7, m21 ; out16 out17 - paddsw m7, m21 ; out15 out14 - psubsw m21, m0, m20 ; out31 out30 - paddsw m0, m20 ; out0 out1 - psubsw m11, m4, m18 ; out23 out22 - paddsw m4, m18 ; out8 out9 - psubsw m18, m2, m14 ; out27 out26 - paddsw m2, m14 ; out4 out5 INIT_ZMM avx512icl - movu m16, [o(permD+3)] - vpermt2q m0, m16, m4 ; 0 1 8 9 - vpermt2q m8, m16, m19 ; 16 17 24 25 - vpermt2q m1, m16, m5 ; 3 2 11 10 - vpermt2q m9, m16, m18 ; 19 18 27 26 - vpermt2q m2, m16, m6 ; 4 5 12 13 - vpermt2q m10, m16, m15 ; 20 21 28 29 - vpermt2q m3, m16, m7 ; 7 6 15 14 - vpermt2q m11, m16, m21 ; 23 22 31 30 + mova m15, [o(permA)] + ret +cglobal_label .main_end + vpbroadcastd m10, [o(pw_2048)] + vpermt2q m0, m15, m1 ; t0 t1 t2 t3 + vpermt2q m20, m15, m19 ; t31 t30a t29 t28a + vpermt2q m2, m15, m3 ; t4 t5 t6 t7 + vpermt2q m14, m15, m12 ; t27 t26a t25 t24a + vpermt2q m4, m15, m5 ; t8 t9 t10 t11 + vpermt2q m18, m15, m16 ; t23a t22 t21a t20 + vpermt2q m6, m15, m7 ; t12 t13 t14 t15 + vpermt2q m13, m15, m21 ; t19a t18 t17a t16 + psubsw m7, m0, m20 ; out31 out30 out29 out28 + paddsw m0, m20 ; out0 out1 out2 out3 + psubsw m5, m2, m14 ; out27 out26 out25 out24 + paddsw m2, m14 ; out4 out5 out6 out7 + psubsw m3, m4, m18 ; out23 out22 out21 out20 + paddsw m4, m18 ; out8 out9 out10 out11 + psubsw m1, m6, m13 ; out19 out18 out17 out16 + paddsw m6, m13 ; out12 out13 out14 out15 vzeroupper ret %macro LOAD_PACKED_16X2 3 ; dst, row[1-2] vbroadcasti32x4 ym%1, [cq+16*%2] @@ -3081,20 +3129,37 @@ mova ym6, ym4 mova ym7, ym4 call m(idct_8x16_internal_8bpc).main call m(inv_txfm_add_dct_dct_8x32_8bpc).main_fast .pass2: - vpbroadcastd m12, [o(pw_8192)] - vshufi32x4 m7, m3, m11, q2020 ; 7 15 23 31 - vshufi32x4 m6, m3, m11, q3131 ; 6 14 22 30 - vshufi32x4 m5, m2, m10, q3131 ; 5 13 21 29 - vshufi32x4 m4, m2, m10, q2020 ; 4 12 20 28 - vshufi32x4 m3, m1, m9, q2020 ; 3 11 19 27 - vshufi32x4 m2, m1, m9, q3131 ; 2 10 18 26 - vshufi32x4 m1, m0, m8, q3131 ; 1 9 17 15 - vshufi32x4 m0, m8, q2020 ; 0 8 16 24 - REPX {pmulhrsw x, m12}, m0, m1, m2, m3, m4, m5, m6, m7 + vpbroadcastd m10, [o(pw_8192)] + vpermt2q m0, m15, m4 ; t0 t1 t9 t8 + vpermt2q m20, m15, m18 ; t31 t30a t23a t22 + vpermt2q m3, m15, m7 ; t7 t6 t14 t15 + vpermt2q m12, m15, m21 ; t25 t24a t17a t16 + vpermt2q m2, m15, m6 ; t4 t5 t13 t12 + vpermt2q m14, m15, m13 ; t23a t22 t21a t20 + vpermt2q m1, m15, m5 ; t3 t2 t10 t11 + vpermt2q m19, m15, m16 ; t27 t26a t19a t18 + psubsw m8, m0, m20 ; out31 out30 out22 out23 + paddsw m0, m20 ; out0 out1 out9 out8 + paddsw m6, m3, m12 ; out7 out6 out14 out15 + psubsw m3, m12 ; out24 out25 out17 out16 + psubsw m5, m2, m14 ; out27 out26 out18 out19 + paddsw m4, m2, m14 ; out4 out5 out13 out12 + psubsw m7, m1, m19 ; out28 out29 out21 out20 + paddsw m2, m1, m19 ; out3 out2 out10 out11 + vzeroupper + vshufi32x4 m1, m0, m3, q1221 ; out1 out9 out17 out25 + vshufi32x4 m0, m3, q0330 ; out0 out8 out16 out24 + vshufi32x4 m3, m2, m5, q0330 ; out3 out11 out19 out27 + vshufi32x4 m2, m5, q1221 ; out2 out10 out18 out26 + vshufi32x4 m5, m4, m7, q1221 ; out5 out13 out21 out29 + vshufi32x4 m4, m7, q0330 ; out4 out12 out20 out28 + vshufi32x4 m7, m6, m8, q0330 ; out7 out15 out23 out31 + vshufi32x4 m6, m8, q1221 ; out6 out14 out22 out30 + REPX {pmulhrsw x, m10}, m0, m1, m2, m3, m4, m5, m6, m7 call m(inv_txfm_add_dct_dct_64x32_8bpc).transpose_8x8 call .main vpbroadcastd m8, [o(pw_2048)] REPX {pmulhrsw x, m8}, m0, m1, m2, m3, m4, m5, m6, m7 lea r2, [strideq*3] @@ -3134,11 +3199,11 @@ vextracti32x8 [r3+r2 ], m6, 1 RET .dconly: movsx r6d, word [cq] mov [cq], eobd - mov r3d, 8 + or r3d, 8 .dconly2: imul r6d, 181 add r6d, 128+512 sar r6d, 8+2 .dconly3: @@ -3160,11 +3225,11 @@ lea dstq, [dstq+strideq*2] sub r3d, 2 jg .dconly_loop RET ALIGN function_align -.main: +cglobal_label .main vpbroadcastd m10, [o(pd_2048)] .main2: ITX_MULSUB_2W 5, 3, 8, 9, 10, 3406, 2276 ; t5a, t6a ITX_MULSUB_2W 1, 7, 8, 9, 10, 799, 4017 ; t4a, t7a ITX_MULSUB_2W 2, 6, 8, 9, 10, 1567, 3784 ; t2, t3 @@ -3537,14 +3602,14 @@ RET ALIGN function_align .dconly: movsx r6d, word [cq] mov [cq], eobd - mov r3d, 32 + or r3d, 32 jmp m(inv_txfm_add_dct_dct_16x8_8bpc).dconly ALIGN function_align -.main_oddhalf_fast2: ; bottom three-quarters are zero +cglobal_label .main_oddhalf_fast2 ; bottom three-quarters are zero vpbroadcastd m8, [o(pw_201_4091x8)] vpbroadcastd m20, [o(pw_m1380_3857x8)] vpbroadcastd m9, [o(pw_995_3973x8)] vpbroadcastd m16, [o(pw_m601_4052x8)] pmulhrsw m21, m8 ; t16a, t31a @@ -3555,11 +3620,11 @@ mova m17, m20 mova m15, m18 mova m16, m14 jmp .main3 ALIGN function_align -.main_oddhalf_fast: ; bottom half is zero +cglobal_label .main_oddhalf_fast ; bottom half is zero vpbroadcastd m8, [o(pw_201_4091x8)] vpbroadcastd m9, [o(pw_m2751_3035x8)] vpbroadcastd m11, [o(pw_1751_3703x8)] vpbroadcastd m12, [o(pw_m1380_3857x8)] pmulhrsw m21, m8 ; t16a, t31a @@ -3574,11 +3639,11 @@ pmulhrsw m16, m9 ; t21a, t26a pmulhrsw m19, m11 ; t22a, t25a pmulhrsw m14, m12 ; t23a, t24a jmp .main2 ALIGN function_align -.main_oddhalf: +cglobal_label .main_oddhalf ITX_MUL2X_PACK 21, 8, 9, 10, 201, 4091, 5 ; t16a, t31a ITX_MUL2X_PACK 17, 8, 9, 10, 3035, 2751, 5 ; t17a, t30a ITX_MUL2X_PACK 20, 8, 9, 10, 1751, 3703, 5 ; t18a, t29a ITX_MUL2X_PACK 15, 8, 9, 10, 3857, 1380, 5 ; t19a, t28a ITX_MUL2X_PACK 18, 8, 9, 10, 995, 3973, 5 ; t20a, t27a @@ -3823,20 +3888,20 @@ RET ALIGN function_align .dconly: movsx r6d, word [cq] mov [cq], eobd + or r3d, 16 imul r6d, 181 - mov r3d, 16 add r6d, 128 sar r6d, 8 imul r6d, 181 add r6d, 128+256 sar r6d, 8+1 jmp m(inv_txfm_add_dct_dct_32x8_8bpc).dconly3 ALIGN function_align -.main_oddhalf_fast2: ; bottom three-quarters are zero +cglobal_label .main_oddhalf_fast2 ; bottom three-quarters are zero vpbroadcastd m9, [o(pw_2896x8)] vpbroadcastd m2, [o(pw_4017x8)] vpbroadcastd m3, [o(pw_799x8)] vpbroadcastd m18, [o(pw_4076x8)] vpbroadcastd m19, [o(pw_401x8)] @@ -3862,11 +3927,11 @@ paddsw m1, m9 ; idct8 out1 psubsw m5, m9, m2 ; idct8 out5 paddsw m2, m9 ; idct8 out2 jmp .main3 ALIGN function_align -.main_oddhalf_fast: ; bottom half is zero +cglobal_label .main_oddhalf_fast ; bottom half is zero vpbroadcastd m5, [o(pw_m2276x8)] vpbroadcastd m11, [o(pw_3406x8)] vpbroadcastd m7, [o(pw_4017x8)] vpbroadcastd m12, [o(pw_799x8)] vpbroadcastd m6, [o(pw_3784x8)] @@ -3900,11 +3965,11 @@ pmulhrsw m16, m11 ; t10a pmulhrsw m20, m15 ; t11a pmulhrsw m15, m12 ; t12a jmp .main2 ALIGN function_align -.main_oddhalf: +cglobal_label .main_oddhalf ITX_MULSUB_2W 14, 21, 8, 9, 10, 401, 4076 ; t8a, t15a ITX_MULSUB_2W 18, 17, 8, 9, 10, 3166, 2598 ; t9a, t14a ITX_MULSUB_2W 16, 19, 8, 9, 10, 1931, 3612 ; t10a, t13a ITX_MULSUB_2W 20, 15, 8, 9, 10, 3920, 1189 ; t11a, t12a .main2: @@ -4605,14 +4670,14 @@ IDCT_32x32_END 14, 15, r4 , strideq*0 RET .dconly: movsx r6d, word [cq] mov [cq], eobd - mov r3d, 32 + or r3d, 32 jmp m(inv_txfm_add_dct_dct_32x8_8bpc).dconly2 ALIGN function_align -.main_oddhalf_fast2: ; bottom three-quarters are zero +cglobal_label .main_oddhalf_fast2 ; bottom three-quarters are zero vpbroadcastd m21, [o(pw_4091x8)] vpbroadcastd m8, [o(pw_201x8)] vpbroadcastd m18, [o(pw_m1380x8)] vpbroadcastd m9, [o(pw_3857x8)] vpbroadcastd m19, [o(pw_3973x8)] @@ -4635,11 +4700,11 @@ mova m26, m24 mova m16, m28 mova m20, m23 jmp .main3 ALIGN function_align -.main_oddhalf_fast: ; bottom half is zero +cglobal_label .main_oddhalf_fast ; bottom half is zero vpbroadcastd m21, [o(pw_4091x8)] vpbroadcastd m8, [o(pw_201x8)] vpbroadcastd m14, [o(pw_m2751x8)] vpbroadcastd m9, [o(pw_3035x8)] vpbroadcastd m17, [o(pw_3703x8)] @@ -4670,11 +4735,11 @@ pmulhrsw m28, m11 ; t22a pmulhrsw m20, m23 ; t23a pmulhrsw m23, m12 ; t24a jmp .main2 ALIGN function_align -.main_oddhalf: +cglobal_label .main_oddhalf ITX_MULSUB_2W 22, 21, 8, 9, 10, 201, 4091 ; t16a, t31a ITX_MULSUB_2W 14, 29, 8, 9, 10, 3035, 2751 ; t17a, t30a ITX_MULSUB_2W 26, 17, 8, 9, 10, 1751, 3703 ; t18a, t29a ITX_MULSUB_2W 18, 25, 8, 9, 10, 3857, 1380 ; t19a, t28a ITX_MULSUB_2W 24, 19, 8, 9, 10, 995, 3973 ; t20a, t27a @@ -5070,12 +5135,12 @@ IDCT_16x64_END 15, 14, 28 RET .dconly: movsx r6d, word [cq] mov [cq], eobd + or r3d, 64 imul r6d, 181 - mov r3d, 64 add r6d, 128+512 sar r6d, 8+2 jmp m(inv_txfm_add_dct_dct_16x8_8bpc).dconly3 ALIGN function_align .main_oddhalf_fast: ; bottom three-quarters are zero @@ -5284,11 +5349,11 @@ lea r5, [o_base] test eobd, eobd jnz .normal movsx r6d, word [cq] mov [cq], eobd - mov r3d, 16 + or r3d, 16 .dconly: imul r6d, 181 add r6d, 128+512 sar r6d, 8+2 .dconly2: @@ -6014,12 +6079,12 @@ IDCT_32x64_END 14, 15, r4 , r5*8, strideq*0, r8 RET .dconly: movsx r6d, word [cq] mov [cq], eobd + or r3d, 64 imul r6d, 181 - mov r3d, 64 add r6d, 128 sar r6d, 8 imul r6d, 181 add r6d, 128+256 sar r6d, 8+1 @@ -6676,12 +6741,12 @@ RET ALIGN function_align .dconly: movsx r6d, word [cq] mov [cq], eobd + or r3d, 32 imul r6d, 181 - mov r3d, 32 add r6d, 128 sar r6d, 8 imul r6d, 181 add r6d, 128+256 sar r6d, 8+1 @@ -7119,11 +7184,11 @@ jb .end_loop RET .dconly: movsx r6d, word [cq] mov [cq], eobd - mov r3d, 64 + or r3d, 64 jmp m(inv_txfm_add_dct_dct_64x16_8bpc).dconly ALIGN function_align .pass2_end: REPX {pmulhrsw x, m13}, m22, m24, m26, m28, m14, m16, m18, m20, m0, m2, m4, m6 mova [r4+64*20], m1 diff --git a/third_party/dav1d/src/x86/mc16_avx2.asm b/third_party/dav1d/src/x86/mc16_avx2.asm --- a/third_party/dav1d/src/x86/mc16_avx2.asm +++ b/third_party/dav1d/src/x86/mc16_avx2.asm @@ -2648,27 +2648,18 @@ %macro MC_8TAP_SCALED 1 %ifidn %1, put %assign isput 1 %assign isprep 0 - %if required_stack_alignment <= STACK_ALIGNMENT -cglobal put_8tap_scaled_16bpc, 4, 15, 16, 0xe0, dst, ds, src, ss, w, h, mx, my, dx, dy, pxmax - %else cglobal put_8tap_scaled_16bpc, 4, 14, 16, 0xe0, dst, ds, src, ss, w, h, mx, my, dx, dy, pxmax - %endif %xdefine base_reg r12 mov r7d, pxmaxm %else %assign isput 0 %assign isprep 1 - %if required_stack_alignment <= STACK_ALIGNMENT -cglobal prep_8tap_scaled_16bpc, 4, 15, 16, 0xe0, tmp, src, ss, w, h, mx, my, dx, dy, pxmax - %xdefine tmp_stridem r14q - %else cglobal prep_8tap_scaled_16bpc, 4, 14, 16, 0xe0, tmp, src, ss, w, h, mx, my, dx, dy, pxmax %define tmp_stridem qword [rsp+0xd0] - %endif %xdefine base_reg r11 %endif lea base_reg, [%1_8tap_scaled_16bpc_avx2] %define base base_reg-%1_8tap_scaled_16bpc_avx2 tzcnt wd, wm @@ -2696,19 +2687,13 @@ %define dxm r8m %else DEFINE_ARGS dst, ds, src, ss, w, h, _, my, dx, dy, ss3 %define hm r6m %endif - %if required_stack_alignment > STACK_ALIGNMENT - %define dsm [rsp+0x98] - %define rX r1 - %define rXd r1d - %else - %define dsm dsq - %define rX r14 - %define rXd r14d - %endif + %define dsm [rsp+0x98] + %define rX r1 + %define rXd r1d %else ; prep %if WIN64 mov r7d, hm DEFINE_ARGS tmp, src, ss, w, _, _, my, h, dy, ss3 %define hm r4m @@ -3578,13 +3563,11 @@ SWAP m10, m12, m1 SWAP m11, m7 ; m1=mx, m7=pxmax, m10=h_rnd, m11=h_sh, m12=free mov myd, mym %if isput - %if required_stack_alignment > STACK_ALIGNMENT - %define dsm [rsp+0xb8] - %endif + %define dsm [rsp+0xb8] movifnidn dsm, dsq mova [rsp+0xc0], xm7 %else %if UNIX64 %define hm [rsp+0xb8] diff --git a/third_party/dav1d/src/x86/mc_avx2.asm b/third_party/dav1d/src/x86/mc_avx2.asm --- a/third_party/dav1d/src/x86/mc_avx2.asm +++ b/third_party/dav1d/src/x86/mc_avx2.asm @@ -2719,26 +2719,17 @@ %endmacro %macro MC_8TAP_SCALED 1 %ifidn %1, put %assign isprep 0 - %if required_stack_alignment <= STACK_ALIGNMENT -cglobal put_8tap_scaled_8bpc, 4, 15, 16, 112, dst, ds, src, ss, w, h, mx, my, dx, dy - %else cglobal put_8tap_scaled_8bpc, 4, 14, 16, 128, dst, ds, src, ss, w, h, mx, my, dx, dy - %endif %xdefine base_reg r12 %define rndshift 10 %else %assign isprep 1 - %if required_stack_alignment <= STACK_ALIGNMENT -cglobal prep_8tap_scaled_8bpc, 4, 15, 16, 128, tmp, src, ss, w, h, mx, my, dx, dy - %xdefine tmp_stridem r14q - %else cglobal prep_8tap_scaled_8bpc, 4, 14, 16, 128, tmp, src, ss, w, h, mx, my, dx, dy - %define tmp_stridem qword [rsp+120] - %endif + %define tmp_stridem qword [rsp+120] %xdefine base_reg r11 %define rndshift 6 %endif lea base_reg, [%1_8tap_scaled_8bpc_avx2] %define base base_reg-%1_8tap_scaled_8bpc_avx2 @@ -2761,19 +2752,13 @@ %define dxm r8m %else DEFINE_ARGS dst, ds, src, ss, w, h, _, my, dx, dy, ss3 %define hm r6m %endif - %if required_stack_alignment > STACK_ALIGNMENT - %define dsm [rsp+112] - %define rX r1 - %define rXd r1d - %else - %define dsm dsq - %define rX r14 - %define rXd r14d - %endif + %define dsm [rsp+112] + %define rX r1 + %define rXd r1d %else ; prep %if WIN64 mov r7d, hm DEFINE_ARGS tmp, src, ss, w, _, _, my, h, dy, ss3 %define hm r4m diff --git a/third_party/dav1d/src/x86/msac.asm b/third_party/dav1d/src/x86/msac.asm --- a/third_party/dav1d/src/x86/msac.asm +++ b/third_party/dav1d/src/x86/msac.asm @@ -617,24 +617,18 @@ vpbroadcastw m3, [t0+msac.dif+6] vbroadcasti128 m4, [rax] mov t3d, [t0+msac.update_cdf] mov t4d, t2d not t2 -%if STACK_ALIGNMENT < 32 mov r5, rsp %if WIN64 and rsp, ~31 sub rsp, 40 %else and r5, ~31 %define buf r5-32 %endif -%elif WIN64 - sub rsp, 64 -%else - %define buf rsp-56 -%endif psrlw m1, m0, 6 movd [buf-4], xm2 pand m2, m4 psllw m1, 7 pmulhuw m1, m2 @@ -664,14 +658,10 @@ mov t4, [t0+msac.dif] movzx t1d, word [buf+rax-0] movzx t2d, word [buf+rax-2] shr eax, 1 %if WIN64 -%if STACK_ALIGNMENT < 32 mov rsp, r5 -%else - add rsp, 64 -%endif %endif vzeroupper jmp m(msac_decode_symbol_adapt4, _sse2).renorm2 %endif diff --git a/third_party/dav1d/tests/checkasm/checkasm.c b/third_party/dav1d/tests/checkasm/checkasm.c --- a/third_party/dav1d/tests/checkasm/checkasm.c +++ b/third_party/dav1d/tests/checkasm/checkasm.c @@ -40,10 +40,14 @@ #define COLOR_YELLOW (FOREGROUND_RED|FOREGROUND_GREEN) #else #include #include #include +#include +#ifdef HAVE_PTHREAD_NP_H +#include +#endif #ifdef __APPLE__ #include #endif #define COLOR_RED 1 #define COLOR_GREEN 2 @@ -554,10 +558,11 @@ if (!strncmp(argv[1], "--help", 6) || !strcmp(argv[1], "-h")) { fprintf(stderr, "checkasm [options] \n" " Numeric value to seed the rng\n" "Options:\n" + " --affinity= Run the process on CPU \n" " --test= Test only \n" " --function= -f Test only the functions matching \n" " --bench -b Benchmark the tested functions\n" " --list-functions List available functions\n" " --list-tests List available tests\n" @@ -591,10 +596,47 @@ for (int i = 0; tests[i].name; i++) printf("%s\n", tests[i].name); return 0; } else if (!strcmp(argv[1], "--verbose") || !strcmp(argv[1], "-v")) { state.verbose = 1; + } else if (!strncmp(argv[1], "--affinity=", 11)) { + unsigned long affinity = strtoul(argv[1] + 11, NULL, 16); +#ifdef _WIN32 + BOOL (WINAPI *spdcs)(HANDLE, const ULONG*, ULONG) = + (void*)GetProcAddress(GetModuleHandleW(L"kernel32.dll"), "SetProcessDefaultCpuSets"); + HANDLE process = GetCurrentProcess(); + int affinity_err; + if (spdcs) { + affinity_err = !spdcs(process, (ULONG[]){ affinity + 256 }, 1); + } else { + if (affinity < sizeof(DWORD_PTR) * 8) + affinity_err = !SetProcessAffinityMask(process, (DWORD_PTR)1 << affinity); + else + affinity_err = 1; + } + if (affinity_err) { + fprintf(stderr, "checkasm: invalid cpu affinity (%lu)\n", affinity); + return 1; + } else { + fprintf(stderr, "checkasm: running on cpu %lu\n", affinity); + } +#elif defined(HAVE_PTHREAD_SETAFFINITY_NP) && defined(CPU_SET) + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(affinity, &set); + if (pthread_setaffinity_np(pthread_self(), sizeof(set), &set)) { + fprintf(stderr, "checkasm: invalid cpu affinity (%lu)\n", affinity); + return 1; + } else { + fprintf(stderr, "checkasm: running on cpu %lu\n", affinity); + } +#else + (void)affinity; + fprintf(stderr, + "checkasm: --affinity is not supported on your system\n"); + return 1; +#endif } else { state.seed = (unsigned) strtoul(argv[1], NULL, 10); } argc--; diff --git a/third_party/dav1d/tests/checkasm/ipred.c b/third_party/dav1d/tests/checkasm/ipred.c --- a/third_party/dav1d/tests/checkasm/ipred.c +++ b/third_party/dav1d/tests/checkasm/ipred.c @@ -88,55 +88,58 @@ { for (int h = imax(w / 4, 4); h <= imin(w * 4, (mode == FILTER_PRED ? 32 : 64)); h <<= 1) { const ptrdiff_t stride = c_dst_stride; + int nb_iters = (mode >= Z1_PRED && mode <= Z3_PRED) ? 5 : 1; - int a = 0, maxw = 0, maxh = 0; - if (mode >= Z1_PRED && mode <= Z3_PRED) { /* angle */ - a = (90 * (mode - Z1_PRED) + z_angles[rnd() % 27]) | - (rnd() & 0x600); - if (mode == Z2_PRED) { - maxw = rnd(), maxh = rnd(); - maxw = 1 + (maxw & (maxw & 4096 ? 4095 : w - 1)); - maxh = 1 + (maxh & (maxh & 4096 ? 4095 : h - 1)); - } - } else if (mode == FILTER_PRED) /* filter_idx */ - a = (rnd() % 5) | (rnd() & ~511); + for (int iter = 0; iter < nb_iters; iter++) { + int a = 0, maxw = 0, maxh = 0; + if (mode >= Z1_PRED && mode <= Z3_PRED) { /* angle */ + a = (90 * (mode - Z1_PRED) + z_angles[rnd() % 27]) | + (rnd() & 0x600); + if (mode == Z2_PRED) { + maxw = rnd(), maxh = rnd(); + maxw = 1 + (maxw & (maxw & 4096 ? 4095 : w - 1)); + maxh = 1 + (maxh & (maxh & 4096 ? 4095 : h - 1)); + } + } else if (mode == FILTER_PRED) /* filter_idx */ + a = (rnd() % 5) | (rnd() & ~511); - int bitdepth_max; - if (bpc == 16) - bitdepth_max = rnd() & 1 ? 0x3ff : 0xfff; - else - bitdepth_max = (1 << bpc) - 1; + int bitdepth_max; + if (bpc == 16) + bitdepth_max = rnd() & 1 ? 0x3ff : 0xfff; + else + bitdepth_max = (1 << bpc) - 1; - for (int i = -h * 2; i <= w * 2; i++) - topleft[i] = rnd() & bitdepth_max; + for (int i = -h * 2; i <= w * 2; i++) + topleft[i] = rnd() & bitdepth_max; - CLEAR_PIXEL_RECT(c_dst); - CLEAR_PIXEL_RECT(a_dst); - call_ref(c_dst, stride, topleft, w, h, a, maxw, maxh - HIGHBD_TAIL_SUFFIX); - call_new(a_dst, stride, topleft, w, h, a, maxw, maxh - HIGHBD_TAIL_SUFFIX); - if (checkasm_check_pixel_padded(c_dst, stride, - a_dst, stride, - w, h, "dst")) - { - if (mode == Z1_PRED || mode == Z3_PRED) - fprintf(stderr, "angle = %d (0x%03x)\n", - a & 0x1ff, a & 0x600); - else if (mode == Z2_PRED) - fprintf(stderr, "angle = %d (0x%03x), " - "max_width = %d, max_height = %d\n", - a & 0x1ff, a & 0x600, maxw, maxh); - else if (mode == FILTER_PRED) - fprintf(stderr, "filter_idx = %d\n", a & 0x1ff); + CLEAR_PIXEL_RECT(c_dst); + CLEAR_PIXEL_RECT(a_dst); + call_ref(c_dst, stride, topleft, w, h, a, maxw, maxh + HIGHBD_TAIL_SUFFIX); + call_new(a_dst, stride, topleft, w, h, a, maxw, maxh + HIGHBD_TAIL_SUFFIX); + if (checkasm_check_pixel_padded(c_dst, stride, + a_dst, stride, + w, h, "dst")) + { + if (mode == Z1_PRED || mode == Z3_PRED) + fprintf(stderr, "angle = %d (0x%03x)\n", + a & 0x1ff, a & 0x600); + else if (mode == Z2_PRED) + fprintf(stderr, "angle = %d (0x%03x), " + "max_width = %d, max_height = %d\n", + a & 0x1ff, a & 0x600, maxw, maxh); + else if (mode == FILTER_PRED) + fprintf(stderr, "filter_idx = %d\n", a & 0x1ff); + } + + bench_new(a_dst, stride, topleft, w, h, a, 128, 128 + HIGHBD_TAIL_SUFFIX); } - - bench_new(a_dst, stride, topleft, w, h, a, 128, 128 - HIGHBD_TAIL_SUFFIX); } } } report("intra_pred"); } diff --git a/third_party/dav1d/tests/checkasm/refmvs.c b/third_party/dav1d/tests/checkasm/refmvs.c --- a/third_party/dav1d/tests/checkasm/refmvs.c +++ b/third_party/dav1d/tests/checkasm/refmvs.c @@ -46,11 +46,11 @@ if (check_func(c->splat_mv, "splat_mv_w%d", w)) { const int h_min = imax(w / 4, 1); const int h_max = imin(w * 4, 32); const int w_uint32 = w * sizeof(refmvs_block) / sizeof(uint32_t); for (int h = h_min; h <= h_max; h *= 2) { - const int offset = (w * rnd()) & 31; + const int offset = (int) ((unsigned) w * rnd()) & 31; union { refmvs_block rmv; uint32_t u32[3]; } ALIGN(tmp, 16); tmp.u32[0] = rnd(); diff --git a/third_party/dav1d/tests/libfuzzer/meson.build b/third_party/dav1d/tests/libfuzzer/meson.build --- a/third_party/dav1d/tests/libfuzzer/meson.build +++ b/third_party/dav1d/tests/libfuzzer/meson.build @@ -49,22 +49,21 @@ endif dav1d_fuzzer = executable('dav1d_fuzzer', dav1d_fuzzer_sources, include_directories: dav1d_inc_dirs, - c_args: [stackalign_flag, stackrealign_flag], link_args: fuzzer_ldflags, link_with : libdav1d, build_by_default: true, dependencies : [thread_dependency], kwargs: fuzzer_link_lang ) dav1d_fuzzer_mt = executable('dav1d_fuzzer_mt', dav1d_fuzzer_sources, include_directories: dav1d_inc_dirs, - c_args: [stackalign_flag, stackrealign_flag, '-DDAV1D_MT_FUZZING'], + c_args: ['-DDAV1D_MT_FUZZING'], link_args: fuzzer_ldflags, link_with : libdav1d, build_by_default: true, dependencies : [thread_dependency], kwargs: fuzzer_link_lang @@ -90,11 +89,11 @@ '@INPUT@', '@OUTPUT@']) dav1d_fuzzer_mem = executable('dav1d_fuzzer_mem', dav1d_fuzzer_sources + ['alloc_fail.c'], include_directories: dav1d_inc_dirs, - c_args: [stackalign_flag, stackrealign_flag, '-DDAV1D_ALLOC_FAIL'], + c_args: ['-DDAV1D_ALLOC_FAIL'], link_args: fuzzer_ldflags + [join_paths(libdav1d_af.full_path())], link_depends: libdav1d_af, build_by_default: false, dependencies : [thread_dependency], kwargs: fuzzer_link_lang diff --git a/third_party/dav1d/tests/meson.build b/third_party/dav1d/tests/meson.build --- a/third_party/dav1d/tests/meson.build +++ b/third_party/dav1d/tests/meson.build @@ -52,11 +52,11 @@ foreach bitdepth : dav1d_bitdepths checkasm_bitdepth_lib = static_library( 'checkasm_bitdepth_@0@'.format(bitdepth), checkasm_tmpl_sources, include_directories: dav1d_inc_dirs, - c_args: ['-DBITDEPTH=@0@'.format(bitdepth), stackalign_flag], + c_args: ['-DBITDEPTH=@0@'.format(bitdepth)], install: false, build_by_default: false, ) checkasm_bitdepth_objs += checkasm_bitdepth_lib.extract_all_objects(recursive: true) endforeach @@ -85,11 +85,10 @@ checkasm_bitdepth_objs, libdav1d.extract_all_objects(recursive: true), ], include_directories: dav1d_inc_dirs, - c_args: [stackalign_flag, stackrealign_flag], build_by_default: false, dependencies : [ thread_dependency, rt_dependency, libdl_dependency,