diff --git a/media/libdav1d/asm/moz.build b/media/libdav1d/asm/moz.build --- a/media/libdav1d/asm/moz.build +++ b/media/libdav1d/asm/moz.build @@ -136,10 +136,11 @@ entry_point='add_define', inputs=[relative_path + f], flags=['BITDEPTH', '8']) elif CONFIG['CPU_ARCH'] == 'arm' or CONFIG['CPU_ARCH'] == 'aarch64': SOURCES += [ + '../../../third_party/dav1d/src/arm/asm-offsets.h', '../../../third_party/dav1d/src/arm/cpu.c', ] EXPORTS += [ '../../../third_party/dav1d/src/arm/cpu.h', '../../../third_party/dav1d/src/arm/msac.h', diff --git a/media/libdav1d/moz.yaml b/media/libdav1d/moz.yaml --- a/media/libdav1d/moz.yaml +++ b/media/libdav1d/moz.yaml @@ -18,15 +18,15 @@ # Usually different from repository url url: https://code.videolan.org/videolan/dav1d # Human-readable identifier for this version/release # Generally "version NNN", "tag SSS", "bookmark SSS" - release: commit 6c6d25d355b78556d231b1a5633ded2ddb9e3774 (2021-03-15T14:38:41.000+02:00). + release: commit 136585101bd5fa8a07a5c7bf66390439a1c7d387 (2021-04-28T17:48:22.000+00:00). # Revision to pull in # Must be a long or short commit SHA (long preferred) - revision: 6c6d25d355b78556d231b1a5633ded2ddb9e3774 + revision: 136585101bd5fa8a07a5c7bf66390439a1c7d387 # The package's license, where possible using the mnemonic from # https://spdx.org/licenses/ # Multiple licenses can be specified (as a YAML list) # A "LICENSE" file must exist containing the full license text diff --git a/media/libdav1d/vcs_version.h b/media/libdav1d/vcs_version.h --- a/media/libdav1d/vcs_version.h +++ b/media/libdav1d/vcs_version.h @@ -1,2 +1,2 @@ /* auto-generated, do not edit */ -#define DAV1D_VERSION "0.8.2-2-g6c6d25d" +#define DAV1D_VERSION "136585101bd5fa8a07a5c7bf66390439a1c7d387" diff --git a/third_party/dav1d/include/common/attributes.h b/third_party/dav1d/include/common/attributes.h --- a/third_party/dav1d/include/common/attributes.h +++ b/third_party/dav1d/include/common/attributes.h @@ -29,10 +29,11 @@ #define DAV1D_COMMON_ATTRIBUTES_H #include "config.h" #include +#include #ifdef __GNUC__ #define ATTR_ALIAS __attribute__((may_alias)) #define ATTR_FORMAT_PRINTF(fmt, attr) __attribute__((__format__(__printf__, fmt, attr))) #define COLD __attribute__((cold)) @@ -101,15 +102,15 @@ #else #define NO_SANITIZE(x) #endif #if defined(NDEBUG) && (defined(__GNUC__) || defined(__clang__)) +#undef assert #define assert(x) do { if (!(x)) __builtin_unreachable(); } while (0) #elif defined(NDEBUG) && defined(_MSC_VER) +#undef assert #define assert __assume -#else -#include #endif #if defined(__GNUC__) && !defined(__INTEL_COMPILER) && !defined(__clang__) # define dav1d_uninit(x) x=x #else @@ -161,6 +162,14 @@ #ifndef __has_feature #define __has_feature(x) 0 #endif +#ifndef static_assert +#define CHECK_OFFSET(type, field, name) \ + struct check_##type##_##field { int x[(name == offsetof(type, field)) ? 1 : -1]; } +#else +#define CHECK_OFFSET(type, field, name) \ + static_assert(name == offsetof(type, field), #field) +#endif + #endif /* DAV1D_COMMON_ATTRIBUTES_H */ diff --git a/third_party/dav1d/include/dav1d/dav1d.h b/third_party/dav1d/include/dav1d/dav1d.h --- a/third_party/dav1d/include/dav1d/dav1d.h +++ b/third_party/dav1d/include/dav1d/dav1d.h @@ -200,10 +200,39 @@ * delivered to dav1d_send_data(). * */ DAV1D_API void dav1d_flush(Dav1dContext *c); +enum Dav1dEventFlags { + /** + * The last returned picture contains a reference to a new Sequence Header, + * either because it's the start of a new coded sequence, or the decoder was + * flushed before it was generated. + */ + DAV1D_EVENT_FLAG_NEW_SEQUENCE = 1 << 0, + /** + * The last returned picture contains a reference to a Sequence Header with + * new operating parameters information for the current coded sequence. + */ + DAV1D_EVENT_FLAG_NEW_OP_PARAMS_INFO = 1 << 1, +}; + +/** + * Fetch a combination of DAV1D_EVENT_FLAG_* event flags generated by the decoding + * process. + * + * @param c Input decoder instance. + * @param flags Where to write the flags. + * + * @return 0 on success, or < 0 (a negative DAV1D_ERR code) on error. + * + * @note Calling this function will clear all the event flags currently stored in + * the decoder. + * + */ +DAV1D_API int dav1d_get_event_flags(Dav1dContext *c, enum Dav1dEventFlags *flags); + # ifdef __cplusplus } # endif #endif /* DAV1D_H */ diff --git a/third_party/dav1d/meson.build b/third_party/dav1d/meson.build --- a/third_party/dav1d/meson.build +++ b/third_party/dav1d/meson.build @@ -28,11 +28,11 @@ 'warning_level=2', 'buildtype=release', 'b_ndebug=if-release'], meson_version: '>= 0.49.0') -dav1d_soname_version = '5.0.1' +dav1d_soname_version = '5.1.0' dav1d_api_version_array = dav1d_soname_version.split('.') dav1d_api_version_major = dav1d_api_version_array[0] dav1d_api_version_minor = dav1d_api_version_array[1] dav1d_api_version_revision = dav1d_api_version_array[2] diff --git a/third_party/dav1d/src/arm/64/film_grain.S b/third_party/dav1d/src/arm/64/film_grain.S --- a/third_party/dav1d/src/arm/64/film_grain.S +++ b/third_party/dav1d/src/arm/64/film_grain.S @@ -25,12 +25,854 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "src/arm/asm.S" #include "util.S" +#include "src/arm/asm-offsets.h" #define GRAIN_WIDTH 82 +#define GRAIN_HEIGHT 73 + +.macro increment_seed steps, shift=1 + lsr w11, w2, #3 + lsr w12, w2, #12 + lsr w13, w2, #1 + eor w11, w2, w11 // (r >> 0) ^ (r >> 3) + eor w12, w12, w13 // (r >> 12) ^ (r >> 1) + eor w11, w11, w12 // (r >> 0) ^ (r >> 3) ^ (r >> 12) ^ (r >> 1) +.if \shift + lsr w2, w2, #\steps +.endif + and w11, w11, #((1 << \steps) - 1) // bit +.if \shift + orr w2, w2, w11, lsl #(16 - \steps) // *state +.else + orr w2, w2, w11, lsl #16 // *state +.endif +.endm + +.macro read_rand dest, bits, age + ubfx \dest, x2, #16 - \bits - \age, #\bits +.endm + +.macro read_shift_rand dest, bits + ubfx \dest, x2, #17 - \bits, #\bits + lsr w2, w2, #1 +.endm + +// special calling convention: +// w2 holds seed +// x3 holds dav1d_gaussian_sequence +// clobbers x11-x15 +// returns in v0.8h +function get_gaussian_neon + increment_seed 4 + read_rand x14, 11, 3 + read_rand x15, 11, 2 + add x14, x3, x14, lsl #1 + add x15, x3, x15, lsl #1 + ld1 {v0.h}[0], [x14] + read_rand x14, 11, 1 + ld1 {v0.h}[1], [x15] + add x14, x3, x14, lsl #1 + read_rand x15, 11, 0 + increment_seed 4 + add x15, x3, x15, lsl #1 + ld1 {v0.h}[2], [x14] + read_rand x14, 11, 3 + ld1 {v0.h}[3], [x15] + add x14, x3, x14, lsl #1 + read_rand x15, 11, 2 + ld1 {v0.h}[4], [x14] + add x15, x3, x15, lsl #1 + read_rand x14, 11, 1 + ld1 {v0.h}[5], [x15] + read_rand x15, 11, 0 + add x14, x3, x14, lsl #1 + add x15, x3, x15, lsl #1 + ld1 {v0.h}[6], [x14] + ld1 {v0.h}[7], [x15] + ret +endfunc + +.macro get_grain_row r0, r1, r2, r3, r4, r5 + bl get_gaussian_neon + srshl \r5\().8h, v0.8h, v31.8h + xtn \r0\().8b, \r5\().8h + bl get_gaussian_neon + srshl \r5\().8h, v0.8h, v31.8h + xtn2 \r0\().16b, \r5\().8h + bl get_gaussian_neon + srshl \r5\().8h, v0.8h, v31.8h + xtn \r1\().8b, \r5\().8h + bl get_gaussian_neon + srshl \r5\().8h, v0.8h, v31.8h + xtn2 \r1\().16b, \r5\().8h + bl get_gaussian_neon + srshl \r5\().8h, v0.8h, v31.8h + xtn \r2\().8b, \r5\().8h + bl get_gaussian_neon + srshl \r5\().8h, v0.8h, v31.8h + xtn2 \r2\().16b, \r5\().8h + bl get_gaussian_neon + srshl \r5\().8h, v0.8h, v31.8h + xtn \r3\().8b, \r5\().8h + bl get_gaussian_neon + srshl \r5\().8h, v0.8h, v31.8h + xtn2 \r3\().16b, \r5\().8h + bl get_gaussian_neon + srshl \r5\().8h, v0.8h, v31.8h + xtn \r4\().8b, \r5\().8h + bl get_gaussian_neon + srshl \r5\().8h, v0.8h, v31.8h + xtn2 \r4\().16b, \r5\().8h + increment_seed 2 + read_rand x14, 11, 1 + read_rand x15, 11, 0 + add x14, x3, x14, lsl #1 + add x15, x3, x15, lsl #1 + ld1 {\r5\().h}[0], [x14] + ld1 {\r5\().h}[1], [x15] + srshl v0.4h, \r5\().4h, v31.4h + xtn \r5\().8b, v0.8h +.endm + +.macro store_grain_row r0, r1, r2, r3, r4, r5 + st1 {\r0\().16b,\r1\().16b}, [x0], #32 + st1 {\r2\().16b,\r3\().16b}, [x0], #32 + st1 {\r4\().16b}, [x0], #16 + st1 {\r5\().h}[0], [x0], #2 +.endm + +.macro get_grain_2 dst + increment_seed 2 + read_rand x14, 11, 1 + read_rand x15, 11, 0 + add x14, x3, x14, lsl #1 + add x15, x3, x15, lsl #1 + ld1 {\dst\().h}[0], [x14] + ld1 {\dst\().h}[1], [x15] + srshl v0.4h, \dst\().4h, v31.4h + xtn \dst\().8b, v0.8h +.endm + +// w15 holds the number of entries to produce +// w14 holds the previous output entry +// v0 holds the vector of produced entries +// v1 holds the input vector of sums from above +function output_y_lag1_neon +1: + read_shift_rand x13, 11 + mov w11, v1.s[0] + ldrsh w12, [x3, x13, lsl #1] + ext v0.16b, v0.16b, v0.16b, #1 + madd w14, w14, w4, w11 // sum (above) + *coeff * prev output + add w14, w14, w8 // 1 << (ar_coeff_shift - 1) + asr w14, w14, w7 // >> ar_coeff_shift + add w12, w12, w10 + asr w12, w12, w9 // >> (4 + grain_scale_shift) + add w14, w14, w12 + cmp w14, w5 + csel w14, w14, w5, le + cmp w14, w6 + csel w14, w14, w6, ge + subs w15, w15, #1 + ext v1.16b, v1.16b, v1.16b, #4 + ins v0.b[15], w14 + b.gt 1b + ret +endfunc + +function sum_y_lag1_above_neon + smull v2.8h, v3.8b, v28.8b + smull2 v3.8h, v3.16b, v28.16b + smull v4.8h, v0.8b, v27.8b + smull2 v5.8h, v0.16b, v27.16b + smull v6.8h, v1.8b, v29.8b + smull2 v7.8h, v1.16b, v29.16b + add v2.8h, v2.8h, v4.8h + add v3.8h, v3.8h, v5.8h + saddl v4.4s, v2.4h, v6.4h + saddl2 v5.4s, v2.8h, v6.8h + saddl v6.4s, v3.4h, v7.4h + saddl2 v7.4s, v3.8h, v7.8h + ret +endfunc + +.macro sum_y_lag1_func edge +function sum_y_lag1_\edge\()_neon + str x30, [sp, #-16]! + bl sum_y_lag1_above_neon +.ifc \edge, left + increment_seed 4 + read_rand x12, 11, 3 + read_rand x13, 11, 2 + read_rand x14, 11, 1 + add x12, x3, x12, lsl #1 + add x13, x3, x13, lsl #1 + add x14, x3, x14, lsl #1 + ld1 {v0.h}[5], [x12] + ld1 {v0.h}[6], [x13] + ld1 {v0.h}[7], [x14] + lsl x2, x2, #1 // shift back the state as if we'd done increment_seed with shift=0 + srshl v0.8h, v0.8h, v31.8h + xtn2 v0.16b, v0.8h + ext v4.16b, v4.16b, v4.16b, #12 + smov w14, v0.b[15] + + mov v1.16b, v4.16b + mov w15, #1 + bl output_y_lag1_neon +.else + increment_seed 4, shift=0 + mov v1.16b, v4.16b + mov w15, #4 + bl output_y_lag1_neon +.endif + + increment_seed 4, shift=0 + mov v1.16b, v5.16b + mov w15, #4 + bl output_y_lag1_neon + + increment_seed 4, shift=0 + mov v1.16b, v6.16b + mov w15, #4 + bl output_y_lag1_neon + + increment_seed 4, shift=0 + mov v1.16b, v7.16b + +.ifc \edge, right + mov w15, #3 + bl output_y_lag1_neon + read_shift_rand x15, 11 + add x15, x3, x15, lsl #1 + ld1 {v1.h}[0], [x15] + srshl v1.4h, v1.4h, v31.4h + ext v0.16b, v0.16b, v1.16b, #1 +.else + mov w15, #4 + bl output_y_lag1_neon +.endif + ldr x30, [sp], #16 + ret +endfunc +.endm + +sum_y_lag1_func left +sum_y_lag1_func mid +sum_y_lag1_func right + +.macro sum_y_lag1 dst, left, mid, right, edge=mid + mov v3.16b, \mid\().16b + ext v0.16b, \left\().16b, \mid\().16b, #15 + ext v1.16b, \mid\().16b, \right\().16b, #1 + bl sum_y_lag1_\edge\()_neon + mov \dst\().16b, v0.16b +.endm + +// w15 holds the number of entries to produce +// w14 and w16 hold the previous output entries +// v0 holds the vector of produced entries +// v1 holds the input vector of sums from above +function output_y_lag2_neon +1: + read_shift_rand x13, 11 + mov w11, v1.s[0] + ldrsh w12, [x3, x13, lsl #1] + ext v0.16b, v0.16b, v0.16b, #1 + madd w11, w16, w4, w11 // sum (above) + *coeff * prev output 1 + madd w11, w14, w17, w11 // += *coeff * prev output 2 + mov w16, w14 + add w14, w11, w8 // 1 << (ar_coeff_shift - 1) + asr w14, w14, w7 // >> ar_coeff_shift + add w12, w12, w10 + asr w12, w12, w9 // >> (4 + grain_scale_shift) + add w14, w14, w12 + cmp w14, w5 + csel w14, w14, w5, le + cmp w14, w6 + csel w14, w14, w6, ge + subs w15, w15, #1 + ext v1.16b, v1.16b, v1.16b, #4 + ins v0.b[15], w14 + b.gt 1b + ret +endfunc + +function sum_y_lag2_above_neon + sub x12, x0, #2*GRAIN_WIDTH - 16 + sub x13, x0, #1*GRAIN_WIDTH - 16 + ld1 {v18.16b}, [x12] // load top right + ld1 {v21.16b}, [x13] + + ext v22.16b, v16.16b, v17.16b, #14 // top left, top mid + dup v26.16b, v30.b[0] + ext v23.16b, v16.16b, v17.16b, #15 + dup v27.16b, v30.b[1] + ext v0.16b, v17.16b, v18.16b, #1 // top mid, top right + dup v28.16b, v30.b[3] + ext v1.16b, v17.16b, v18.16b, #2 + dup v29.16b, v30.b[4] + + smull v2.8h, v22.8b, v26.8b + smull2 v3.8h, v22.16b, v26.16b + smull v4.8h, v23.8b, v27.8b + smull2 v5.8h, v23.16b, v27.16b + smull v6.8h, v0.8b, v28.8b + smull2 v7.8h, v0.16b, v28.16b + smull v0.8h, v1.8b, v29.8b + smull2 v1.8h, v1.16b, v29.16b + add v2.8h, v2.8h, v4.8h + add v3.8h, v3.8h, v5.8h + add v0.8h, v0.8h, v6.8h + add v1.8h, v1.8h, v7.8h + saddl v4.4s, v2.4h, v0.4h + saddl2 v5.4s, v2.8h, v0.8h + saddl v6.4s, v3.4h, v1.4h + saddl2 v7.4s, v3.8h, v1.8h + + ext v22.16b, v19.16b, v20.16b, #14 // top left, top mid + dup v26.16b, v30.b[5] + ext v23.16b, v19.16b, v20.16b, #15 + dup v27.16b, v30.b[6] + ext v0.16b, v20.16b, v21.16b, #1 // top mid, top right + dup v28.16b, v30.b[8] + ext v1.16b, v20.16b, v21.16b, #2 + dup v29.16b, v30.b[9] + + smull v2.8h, v22.8b, v26.8b + smull2 v3.8h, v22.16b, v26.16b + smull v22.8h, v23.8b, v27.8b + smull2 v23.8h, v23.16b, v27.16b + smull v26.8h, v0.8b, v28.8b + smull2 v27.8h, v0.16b, v28.16b + smull v28.8h, v1.8b, v29.8b + smull2 v29.8h, v1.16b, v29.16b + add v2.8h, v2.8h, v22.8h + add v3.8h, v3.8h, v23.8h + add v26.8h, v26.8h, v28.8h + add v27.8h, v27.8h, v29.8h + saddl v0.4s, v2.4h, v26.4h + saddl2 v1.4s, v2.8h, v26.8h + saddl v2.4s, v3.4h, v27.4h + saddl2 v3.4s, v3.8h, v27.8h + dup v26.16b, v30.b[2] + dup v27.16b, v30.b[7] + smull v22.8h, v17.8b, v26.8b + smull2 v23.8h, v17.16b, v26.16b + smull v24.8h, v20.8b, v27.8b + smull2 v25.8h, v20.16b, v27.16b + add v4.4s, v4.4s, v0.4s + add v5.4s, v5.4s, v1.4s + add v6.4s, v6.4s, v2.4s + add v7.4s, v7.4s, v3.4s + + mov v16.16b, v17.16b + mov v17.16b, v18.16b + + add v22.8h, v22.8h, v24.8h + add v23.8h, v23.8h, v25.8h + mov v19.16b, v20.16b + mov v20.16b, v21.16b + saddw v4.4s, v4.4s, v22.4h + saddw2 v5.4s, v5.4s, v22.8h + saddw v6.4s, v6.4s, v23.4h + saddw2 v7.4s, v7.4s, v23.8h + ret +endfunc + +.macro sum_y_lag2_func edge +function sum_y_lag2_\edge\()_neon + str x30, [sp, #-16]! +.ifc \edge, left + sub x12, x0, #2*GRAIN_WIDTH + sub x13, x0, #1*GRAIN_WIDTH + ld1 {v17.16b}, [x12] // load the previous block right above + ld1 {v20.16b}, [x13] +.endif + bl sum_y_lag2_above_neon +.ifc \edge, left + increment_seed 4 + read_rand x12, 11, 3 + read_rand x13, 11, 2 + read_rand x14, 11, 1 + add x12, x3, x12, lsl #1 + add x13, x3, x13, lsl #1 + add x14, x3, x14, lsl #1 + ld1 {v0.h}[5], [x12] + ld1 {v0.h}[6], [x13] + ld1 {v0.h}[7], [x14] + lsl x2, x2, #1 // shift back the state as if we'd done increment_seed with shift=0 + srshl v0.8h, v0.8h, v31.8h + xtn2 v0.16b, v0.8h + ext v4.16b, v4.16b, v4.16b, #12 + smov w16, v0.b[14] + smov w14, v0.b[15] + + mov v1.16b, v4.16b + mov w15, #1 + bl output_y_lag2_neon +.else + increment_seed 4, shift=0 + mov v1.16b, v4.16b + mov w15, #4 + bl output_y_lag2_neon +.endif + + increment_seed 4, shift=0 + mov v1.16b, v5.16b + mov w15, #4 + bl output_y_lag2_neon + + increment_seed 4, shift=0 + mov v1.16b, v6.16b + mov w15, #4 + bl output_y_lag2_neon + + increment_seed 4, shift=0 + mov v1.16b, v7.16b + +.ifc \edge, right + mov w15, #3 + bl output_y_lag2_neon + read_shift_rand x15, 11 + add x15, x3, x15, lsl #1 + ld1 {v1.h}[0], [x15] + srshl v1.4h, v1.4h, v31.4h + ext v0.16b, v0.16b, v1.16b, #1 +.else + mov w15, #4 + bl output_y_lag2_neon +.endif + st1 {v0.16b}, [x0], #16 + ldr x30, [sp], #16 + ret +endfunc +.endm + +sum_y_lag2_func left +sum_y_lag2_func mid +sum_y_lag2_func right + +// w15 holds the number of entries to produce +// w14, w16 and w17 hold the previous output entries +// v0 holds the vector of produced entries +// v1 holds the input vector of sums from above +function output_y_lag3_neon +1: + read_shift_rand x13, 11 + mov w11, v1.s[0] + ldrsh w12, [x3, x13, lsl #1] + ext v0.16b, v0.16b, v0.16b, #1 + madd w11, w17, w4, w11 // sum (above) + *coeff * prev output 1 + madd w11, w16, w19, w11 // sum (above) + *coeff * prev output 2 + madd w11, w14, w20, w11 // += *coeff * prev output 3 + mov w17, w16 + mov w16, w14 + add w14, w11, w8 // 1 << (ar_coeff_shift - 1) + asr w14, w14, w7 // >> ar_coeff_shift + add w12, w12, w10 + asr w12, w12, w9 // >> (4 + grain_scale_shift) + add w14, w14, w12 + cmp w14, w5 + csel w14, w14, w5, le + cmp w14, w6 + csel w14, w14, w6, ge + subs w15, w15, #1 + ext v1.16b, v1.16b, v1.16b, #4 + ins v0.b[15], w14 + b.gt 1b + ret +endfunc + +function sum_y_lag3_above_neon + sub x11, x0, #3*GRAIN_WIDTH - 16 + sub x12, x0, #2*GRAIN_WIDTH - 16 + sub x13, x0, #1*GRAIN_WIDTH - 16 + ld1 {v15.16b}, [x11] // load top right + ld1 {v18.16b}, [x12] + ld1 {v21.16b}, [x13] + + ext v8.16b, v13.16b, v14.16b, #13 // top left, top mid + dup v22.16b, v29.b[0] + ext v9.16b, v13.16b, v14.16b, #14 + dup v23.16b, v29.b[1] + ext v10.16b, v13.16b, v14.16b, #15 + dup v24.16b, v29.b[2] + dup v25.16b, v29.b[3] + ext v11.16b, v14.16b, v15.16b, #1 // top mid, top right + dup v26.16b, v29.b[4] + ext v12.16b, v14.16b, v15.16b, #2 + dup v27.16b, v29.b[5] + ext v13.16b, v14.16b, v15.16b, #3 + dup v28.16b, v29.b[6] + + smull v0.8h, v8.8b, v22.8b + smull2 v1.8h, v8.16b, v22.16b + smull v2.8h, v9.8b, v23.8b + smull2 v3.8h, v9.16b, v23.16b + smull v8.8h, v10.8b, v24.8b + smull2 v9.8h, v10.16b, v24.16b + add v0.8h, v0.8h, v2.8h + add v1.8h, v1.8h, v3.8h + smull v10.8h, v11.8b, v26.8b + smull2 v11.8h, v11.16b, v26.16b + smull v2.8h, v12.8b, v27.8b + smull2 v3.8h, v12.16b, v27.16b + add v8.8h, v8.8h, v10.8h + add v9.8h, v9.8h, v11.8h + smull v10.8h, v13.8b, v28.8b + smull2 v11.8h, v13.16b, v28.16b + saddl v4.4s, v0.4h, v8.4h + saddl2 v5.4s, v0.8h, v8.8h + saddl v6.4s, v1.4h, v9.4h + saddl2 v7.4s, v1.8h, v9.8h + smull v8.8h, v14.8b, v25.8b + smull2 v9.8h, v14.16b, v25.16b + add v2.8h, v2.8h, v10.8h + add v3.8h, v3.8h, v11.8h + saddl v0.4s, v2.4h, v8.4h + saddl2 v1.4s, v2.8h, v8.8h + saddl v2.4s, v3.4h, v9.4h + saddl2 v3.4s, v3.8h, v9.8h + add v4.4s, v4.4s, v0.4s + add v5.4s, v5.4s, v1.4s + add v6.4s, v6.4s, v2.4s + add v7.4s, v7.4s, v3.4s + + ext v8.16b, v16.16b, v17.16b, #13 // top left, top mid + dup v22.16b, v29.b[7] + ext v9.16b, v16.16b, v17.16b, #14 + dup v23.16b, v29.b[8] + ext v10.16b, v16.16b, v17.16b, #15 + dup v24.16b, v29.b[9] + dup v25.16b, v29.b[10] + ext v11.16b, v17.16b, v18.16b, #1 // top mid, top right + dup v26.16b, v29.b[11] + ext v12.16b, v17.16b, v18.16b, #2 + dup v27.16b, v29.b[12] + ext v13.16b, v17.16b, v18.16b, #3 + dup v28.16b, v29.b[13] + + smull v0.8h, v8.8b, v22.8b + smull2 v1.8h, v8.16b, v22.16b + smull v2.8h, v9.8b, v23.8b + smull2 v3.8h, v9.16b, v23.16b + smull v8.8h, v10.8b, v24.8b + smull2 v9.8h, v10.16b, v24.16b + add v0.8h, v0.8h, v2.8h + add v1.8h, v1.8h, v3.8h + smull v10.8h, v11.8b, v26.8b + smull2 v11.8h, v11.16b, v26.16b + smull v2.8h, v12.8b, v27.8b + smull2 v3.8h, v12.16b, v27.16b + add v8.8h, v8.8h, v10.8h + add v9.8h, v9.8h, v11.8h + smull v10.8h, v13.8b, v28.8b + smull2 v11.8h, v13.16b, v28.16b + saddl v12.4s, v0.4h, v8.4h + saddl2 v13.4s, v0.8h, v8.8h + saddl v0.4s, v1.4h, v9.4h + saddl2 v1.4s, v1.8h, v9.8h + smull v8.8h, v17.8b, v25.8b + smull2 v9.8h, v17.16b, v25.16b + add v2.8h, v2.8h, v10.8h + add v3.8h, v3.8h, v11.8h + add v4.4s, v4.4s, v12.4s + add v5.4s, v5.4s, v13.4s + add v6.4s, v6.4s, v0.4s + add v7.4s, v7.4s, v1.4s + saddl v0.4s, v2.4h, v8.4h + saddl2 v1.4s, v2.8h, v8.8h + saddl v2.4s, v3.4h, v9.4h + saddl2 v3.4s, v3.8h, v9.8h + add v4.4s, v4.4s, v0.4s + add v5.4s, v5.4s, v1.4s + add v6.4s, v6.4s, v2.4s + add v7.4s, v7.4s, v3.4s + + ext v8.16b, v19.16b, v20.16b, #13 // top left, top mid + dup v22.16b, v29.b[14] + ext v9.16b, v19.16b, v20.16b, #14 + dup v23.16b, v29.b[15] + ext v10.16b, v19.16b, v20.16b, #15 + dup v24.16b, v30.b[0] + dup v25.16b, v30.b[1] + ext v11.16b, v20.16b, v21.16b, #1 // top mid, top right + dup v26.16b, v30.b[2] + ext v12.16b, v20.16b, v21.16b, #2 + dup v27.16b, v30.b[3] + ext v13.16b, v20.16b, v21.16b, #3 + dup v28.16b, v30.b[4] + + smull v0.8h, v8.8b, v22.8b + smull2 v1.8h, v8.16b, v22.16b + smull v2.8h, v9.8b, v23.8b + smull2 v3.8h, v9.16b, v23.16b + smull v8.8h, v10.8b, v24.8b + smull2 v9.8h, v10.16b, v24.16b + add v0.8h, v0.8h, v2.8h + add v1.8h, v1.8h, v3.8h + smull v10.8h, v11.8b, v26.8b + smull2 v11.8h, v11.16b, v26.16b + smull v2.8h, v12.8b, v27.8b + smull2 v3.8h, v12.16b, v27.16b + add v8.8h, v8.8h, v10.8h + add v9.8h, v9.8h, v11.8h + smull v10.8h, v13.8b, v28.8b + smull2 v11.8h, v13.16b, v28.16b + saddl v12.4s, v0.4h, v8.4h + saddl2 v13.4s, v0.8h, v8.8h + saddl v0.4s, v1.4h, v9.4h + saddl2 v1.4s, v1.8h, v9.8h + smull v8.8h, v20.8b, v25.8b + smull2 v9.8h, v20.16b, v25.16b + add v2.8h, v2.8h, v10.8h + add v3.8h, v3.8h, v11.8h + add v4.4s, v4.4s, v12.4s + add v5.4s, v5.4s, v13.4s + add v6.4s, v6.4s, v0.4s + add v7.4s, v7.4s, v1.4s + saddl v0.4s, v2.4h, v8.4h + saddl2 v1.4s, v2.8h, v8.8h + saddl v2.4s, v3.4h, v9.4h + saddl2 v3.4s, v3.8h, v9.8h + add v4.4s, v4.4s, v0.4s + add v5.4s, v5.4s, v1.4s + add v6.4s, v6.4s, v2.4s + add v7.4s, v7.4s, v3.4s + + mov v13.16b, v14.16b + mov v14.16b, v15.16b + + mov v16.16b, v17.16b + mov v17.16b, v18.16b + + mov v19.16b, v20.16b + mov v20.16b, v21.16b + ret +endfunc + +.macro sum_y_lag3_func edge +function sum_y_lag3_\edge\()_neon + str x30, [sp, #-16]! +.ifc \edge, left + sub x11, x0, #3*GRAIN_WIDTH + sub x12, x0, #2*GRAIN_WIDTH + sub x13, x0, #1*GRAIN_WIDTH + ld1 {v14.16b}, [x11] // load the previous block right above + ld1 {v17.16b}, [x12] + ld1 {v20.16b}, [x13] +.endif + bl sum_y_lag3_above_neon + +.ifc \edge, left + increment_seed 4 + read_rand x12, 11, 3 + read_rand x13, 11, 2 + read_rand x14, 11, 1 + add x12, x3, x12, lsl #1 + add x13, x3, x13, lsl #1 + add x14, x3, x14, lsl #1 + ld1 {v0.h}[5], [x12] + ld1 {v0.h}[6], [x13] + ld1 {v0.h}[7], [x14] + lsl x2, x2, #1 // shift back the state as if we'd done increment_seed with shift=0 + srshl v0.8h, v0.8h, v31.8h + xtn2 v0.16b, v0.8h + ext v4.16b, v4.16b, v4.16b, #12 + smov w17, v0.b[13] + smov w16, v0.b[14] + smov w14, v0.b[15] + + mov v1.16b, v4.16b + mov w15, #1 + bl output_y_lag3_neon +.else + increment_seed 4, shift=0 + mov v1.16b, v4.16b + mov w15, #4 + bl output_y_lag3_neon +.endif + + increment_seed 4, shift=0 + mov v1.16b, v5.16b + mov w15, #4 + bl output_y_lag3_neon + + increment_seed 4, shift=0 + mov v1.16b, v6.16b + mov w15, #4 + bl output_y_lag3_neon + + increment_seed 4, shift=0 + mov v1.16b, v7.16b + +.ifc \edge, right + mov w15, #3 + bl output_y_lag3_neon + read_shift_rand x15, 11 + add x15, x3, x15, lsl #1 + ld1 {v1.h}[0], [x15] + srshl v1.4h, v1.4h, v31.4h + ext v0.16b, v0.16b, v1.16b, #1 +.else + mov w15, #4 + bl output_y_lag3_neon +.endif + st1 {v0.16b}, [x0], #16 + ldr x30, [sp], #16 + ret +endfunc +.endm + +sum_y_lag3_func left +sum_y_lag3_func mid +sum_y_lag3_func right + +function generate_grain_rows_neon + str x30, [sp, #-16]! +1: + get_grain_row v16, v17, v18, v19, v20, v21 + subs w1, w1, #1 + store_grain_row v16, v17, v18, v19, v20, v21 + b.gt 1b + ldr x30, [sp], #16 + ret +endfunc + +function generate_grain_y_8bpc_neon, export=1 + str x30, [sp, #-96]! + + movrel x3, X(gaussian_sequence) + ldr w2, [x1, #FGD_SEED] + ldr w9, [x1, #FGD_GRAIN_SCALE_SHIFT] + add x4, x1, #FGD_AR_COEFFS_Y + adr x16, L(gen_grain_y_tbl) + ldr w17, [x1, #FGD_AR_COEFF_LAG] + add w9, w9, #4 + ldrh w17, [x16, w17, uxtw #1] + dup v31.8h, w9 // 4 + data->grain_scale_shift + sub x16, x16, w17, uxtw + neg v31.8h, v31.8h + + ldr w7, [x1, #FGD_AR_COEFF_SHIFT] + mov w8, #1 + mov w10, #1 + lsl w8, w8, w7 // 1 << ar_coeff_shift + lsl w10, w10, w9 // 1 << (4 + data->grain_scale_shift) + lsr w8, w8, #1 // 1 << (ar_coeff_shift - 1) + lsr w10, w10, #1 // 1 << (4 + data->grain_scale_shift - 1) + mov w5, #127 + mov w6, #-128 + + br x16 + + ret + +L(generate_grain_y_lag0): + mov w1, #GRAIN_HEIGHT + bl generate_grain_rows_neon + ldr x30, [sp], #96 + ret + +L(generate_grain_y_lag1): + ld1r {v27.16b}, [x4], #1 // ar_coeffs_y[0] + ld1r {v28.16b}, [x4], #1 // ar_coeffs_y[1] + ld1r {v29.16b}, [x4], #1 // ar_coeffs_y[2] + ldrsb w4, [x4] // ar_coeffs_y[3] + + mov w1, #3 + bl generate_grain_rows_neon + + mov w1, #GRAIN_HEIGHT - 3 +1: + sum_y_lag1 v22, v16, v16, v17, left + sum_y_lag1 v23, v16, v17, v18 + sum_y_lag1 v24, v17, v18, v19 + sum_y_lag1 v25, v18, v19, v20 + sum_y_lag1 v20, v19, v20, v21, right + get_grain_2 v21 + subs w1, w1, #1 + store_grain_row v22, v23, v24, v25, v20, v21 + mov v16.16b, v22.16b + mov v17.16b, v23.16b + mov v18.16b, v24.16b + mov v19.16b, v25.16b + b.gt 1b + + ldr x30, [sp], #96 + ret + +L(generate_grain_y_lag2): + ld1 {v30.16b}, [x4] // ar_coeffs_y[0-11] + + smov w4, v30.b[10] + smov w17, v30.b[11] + + mov w1, #3 + bl generate_grain_rows_neon + + mov w1, #GRAIN_HEIGHT - 3 +1: + bl sum_y_lag2_left_neon + bl sum_y_lag2_mid_neon + bl sum_y_lag2_mid_neon + bl sum_y_lag2_mid_neon + bl sum_y_lag2_right_neon + get_grain_2 v16 + subs w1, w1, #1 + st1 {v16.h}[0], [x0], #2 + b.gt 1b + + ldr x30, [sp], #96 + ret + +L(generate_grain_y_lag3): + ldr q29, [x4] // ar_coeffs_y[0-15] + ldr d30, [x4, #16] // ar_coeffs_y[16-23] + stp d8, d9, [sp, #16] + stp d10, d11, [sp, #32] + stp d12, d13, [sp, #48] + stp d14, d15, [sp, #64] + stp x19, x20, [sp, #80] + + smov w4, v30.b[5] + smov w19, v30.b[6] + smov w20, v30.b[7] + + mov w1, #3 + bl generate_grain_rows_neon + + mov w1, #GRAIN_HEIGHT - 3 +1: + bl sum_y_lag3_left_neon + bl sum_y_lag3_mid_neon + bl sum_y_lag3_mid_neon + bl sum_y_lag3_mid_neon + bl sum_y_lag3_right_neon + get_grain_2 v16 + subs w1, w1, #1 + st1 {v16.h}[0], [x0], #2 + b.gt 1b + + ldp x19, x20, [sp, #80] + ldp d14, d15, [sp, #64] + ldp d12, d13, [sp, #48] + ldp d10, d11, [sp, #32] + ldp d8, d9, [sp, #16] + ldr x30, [sp], #96 + ret + +L(gen_grain_y_tbl): + .hword L(gen_grain_y_tbl) - L(generate_grain_y_lag0) + .hword L(gen_grain_y_tbl) - L(generate_grain_y_lag1) + .hword L(gen_grain_y_tbl) - L(generate_grain_y_lag2) + .hword L(gen_grain_y_tbl) - L(generate_grain_y_lag3) +endfunc .macro gather_interleaved dst1, dst2, src1, src2, off umov w14, \src1[0+\off] umov w15, \src2[1+\off] umov w16, \src1[2+\off] @@ -67,56 +909,57 @@ function gather_neon gather v4.b, v5.b, v0.b, v1.b ret endfunc -const overlap_coeffs, align=4 +const overlap_coeffs_0, align=4 .byte 27, 17, 0, 0, 0, 0, 0, 0 .byte 17, 27, 32, 32, 32, 32, 32, 32 endconst -.macro calc_offset offx, offy, src +const overlap_coeffs_1, align=4 + .byte 23, 0, 0, 0, 0, 0, 0, 0 + .byte 22, 32, 32, 32, 32, 32, 32, 32 +endconst + +.macro calc_offset offx, offy, src, sx, sy and \offy, \src, #0xF // randval & 0xF lsr \offx, \src, #4 // randval >> 4 +.if \sy == 0 add \offy, \offy, \offy // 2 * (randval & 0xF) +.endif +.if \sx == 0 add \offx, \offx, \offx // 2 * (randval >> 4) +.endif .endm -.macro add_offset dst, offx, offy, src - madd \dst, x9, \offy, \src // grain_lut += grain_stride * offy +.macro add_offset dst, offx, offy, src, stride + madd \dst, \stride, \offy, \src // grain_lut += grain_stride * offy add \dst, \dst, \offx, uxtw // grain_lut += offx .endm -// void dav1d_fgy_32x32_XY_8bpc_neon(pixel *const dst, const pixel *const src, -// const ptrdiff_t stride, -// const uint8_t scaling[SCALING_SIZE], -// const int scaling_shift, -// const entry grain_lut[][GRAIN_WIDTH], -// const int offsets[][2], -// const int h, const ptrdiff_t clip); -.macro fgy ox, oy -function fgy_32x32_\ox\oy\()_8bpc_neon, export=1 +// void dav1d_fgy_32x32_8bpc_neon(pixel *const dst, const pixel *const src, +// const ptrdiff_t stride, +// const uint8_t scaling[SCALING_SIZE], +// const int scaling_shift, +// const entry grain_lut[][GRAIN_WIDTH], +// const int offsets[][2], +// const int h, const ptrdiff_t clip, +// const ptrdiff_t type); +function fgy_32x32_8bpc_neon, export=1 str x30, [sp, #-16]! -.if \ox ldr w11, [x6, #8] // offsets[1][0] -.endif -.if \oy ldr w13, [x6, #4] // offsets[0][1] -.endif -.if \ox && \oy ldr w15, [x6, #12] // offsets[1][1] -.endif ldr w6, [x6] // offsets[0][0] ldr w8, [sp, #16] // clip mov x9, #GRAIN_WIDTH // grain_lut stride neg w4, w4 dup v29.8h, w4 // -scaling_shift -.if \ox || \oy - movrel x16, overlap_coeffs -.endif + movrel x16, overlap_coeffs_0 cbz w8, 1f // clip movi v30.16b, #16 movi v31.16b, #235 @@ -125,58 +968,52 @@ // no clip movi v30.16b, #0 movi v31.16b, #255 2: -.if \ox || \oy ld1 {v27.8b, v28.8b}, [x16] // overlap_coeffs -.endif add x5, x5, #9 // grain_lut += 9 add x5, x5, x9, lsl #3 // grain_lut += 8 * grain_stride add x5, x5, x9 // grain_lut += grain_stride -.if \ox - calc_offset w11, w12, w11 -.endif -.if \oy - calc_offset w13, w14, w13 -.endif -.if \ox && \oy - calc_offset w15, w16, w15 -.endif - calc_offset w6, w10, w6 + calc_offset w11, w12, w11, 0, 0 + calc_offset w13, w14, w13, 0, 0 + calc_offset w15, w16, w15, 0, 0 + calc_offset w6, w10, w6, 0, 0 + + add_offset x12, w11, x12, x5, x9 + add_offset x14, w13, x14, x5, x9 + add_offset x16, w15, x16, x5, x9 + add_offset x5, w6, x10, x5, x9 -.if \ox - add_offset x12, w11, x12, x5 -.endif -.if \oy - add_offset x14, w13, x14, x5 -.endif -.if \ox && \oy - add_offset x16, w15, x16, x5 -.endif - add_offset x5, w6, x10, x5 + ldr w11, [sp, #24] // type + adr x13, L(fgy_loop_tbl) -.if \ox add x4, x12, #32 // grain_lut += BLOCK_SIZE * bx -.endif -.if \oy add x6, x14, x9, lsl #5 // grain_lut += grain_stride * BLOCK_SIZE * by -.endif -.if \ox && \oy + + tst w11, #1 + ldrh w11, [x13, w11, uxtw #1] + add x8, x16, x9, lsl #5 // grain_lut += grain_stride * BLOCK_SIZE * by add x8, x8, #32 // grain_lut += BLOCK_SIZE * bx -.endif + + sub x11, x13, w11, uxtw -.if \oy + b.eq 1f + // y overlap dup v6.16b, v27.b[0] dup v7.16b, v27.b[1] mov w10, w7 // backup actual h mov w7, #2 -.endif +1: + br x11 +endfunc +function fgy_loop_neon +.macro fgy ox, oy L(loop_\ox\oy): 1: ld1 {v0.16b, v1.16b}, [x1], x2 // src .if \ox ld1 {v20.8b}, [x4], x9 // grain_lut old @@ -288,12 +1125,459 @@ sub w7, w10, #2 // restore actual remaining h b.gt L(loop_\ox\()0) .endif ldr x30, [sp], #16 ret +.endm + + fgy 0, 0 + fgy 0, 1 + fgy 1, 0 + fgy 1, 1 + +L(fgy_loop_tbl): + .hword L(fgy_loop_tbl) - L(loop_00) + .hword L(fgy_loop_tbl) - L(loop_01) + .hword L(fgy_loop_tbl) - L(loop_10) + .hword L(fgy_loop_tbl) - L(loop_11) +endfunc + +// void dav1d_fguv_32x32_420_8bpc_neon(pixel *const dst, +// const pixel *const src, +// const ptrdiff_t stride, +// const uint8_t scaling[SCALING_SIZE], +// const Dav1dFilmGrainData *const data, +// const entry grain_lut[][GRAIN_WIDTH], +// const pixel *const luma_row, +// const ptrdiff_t luma_stride, +// const int offsets[][2], +// const ptrdiff_t h, const ptrdiff_t uv, +// const ptrdiff_t is_id, +// const ptrdiff_t type); +.macro fguv layout, sx, sy +function fguv_32x32_\layout\()_8bpc_neon, export=1 + str x30, [sp, #-32]! + stp d8, d9, [sp, #16] + ldp x8, x9, [sp, #32] // offsets, h + ldp x10, x11, [sp, #48] // uv, is_id + + ldr w13, [x4, #FGD_SCALING_SHIFT] + ldr w12, [x4, #FGD_CLIP_TO_RESTRICTED_RANGE] + neg w13, w13 // -scaling_shift + + // !csfl + add x10, x4, x10, lsl #2 // + 4*uv + add x14, x10, #FGD_UV_LUMA_MULT + add x15, x10, #FGD_UV_MULT + add x10, x10, #FGD_UV_OFFSET + ld1 {v8.h}[0], [x14] // uv_luma_mult + ld1r {v24.8h}, [x10] // uv_offset + ld1 {v8.h}[1], [x15] // uv_mult + + dup v29.8h, w13 // -scaling_shift + + cbz w12, 1f + // clip + movi v30.16b, #16 + movi v31.16b, #240 + cbz w11, 2f + // is_id + movi v31.16b, #235 + b 2f +1: + // no clip + movi v30.16b, #0 + movi v31.16b, #255 +2: + + ldr w12, [x8, #8] // offsets[1][0] + ldr w14, [x8, #4] // offsets[0][1] + ldr w16, [x8, #12] // offsets[1][1] + ldr w8, [x8] // offsets[0][0] + + mov x10, #GRAIN_WIDTH // grain_lut stride + + add x5, x5, #(3 + (2 >> \sx)*3) // grain_lut += 9 or 6 +.if \sy + add x5, x5, x10, lsl #2 // grain_lut += 4 * grain_stride + add x5, x5, x10, lsl #1 // grain_lut += 2 * grain_stride +.else + add x5, x5, x10, lsl #3 // grain_lut += 8 * grain_stride + add x5, x5, x10 // grain_lut += grain_stride +.endif + + calc_offset w12, w13, w12, \sx, \sy + calc_offset w14, w15, w14, \sx, \sy + calc_offset w16, w17, w16, \sx, \sy + calc_offset w8, w11, w8, \sx, \sy + + add_offset x13, w12, x13, x5, x10 + add_offset x15, w14, x15, x5, x10 + add_offset x17, w16, x17, x5, x10 + add_offset x5, w8, x11, x5, x10 + + add x4, x13, #(32 >> \sx) // grain_lut += BLOCK_SIZE * bx + add x8, x15, x10, lsl #(5 - \sy) // grain_lut += grain_stride * BLOCK_SIZE * by + add x11, x17, x10, lsl #(5 - \sy) // grain_lut += grain_stride * BLOCK_SIZE * by + add x11, x11, #(32 >> \sx) // grain_lut += BLOCK_SIZE * bx + + ldr w13, [sp, #64] // type + + movrel x16, overlap_coeffs_\sx + adr x14, L(fguv_loop_sx\sx\()_tbl) + + ld1 {v27.8b, v28.8b}, [x16] // overlap_coeffs + tst w13, #1 + ldrh w13, [x14, w13, uxtw #1] + + b.eq 1f + // y overlap + sub w12, w9, #(2 >> \sy) // backup remaining h + mov w9, #(2 >> \sy) + +1: + sub x13, x14, w13, uxtw + +.if \sy + movi v25.16b, #23 + movi v26.16b, #22 +.else + movi v25.16b, #27 + movi v26.16b, #17 +.endif + +.if \sy + add x7, x7, x7 // luma_stride *= 2 +.endif + + br x13 endfunc .endm -fgy 0, 0 -fgy 0, 1 -fgy 1, 0 -fgy 1, 1 +fguv 420, 1, 1 +fguv 422, 1, 0 +fguv 444, 0, 0 + +function fguv_loop_sx0_neon +.macro fguv_loop_sx0 csfl, ox, oy +L(fguv_loop_sx0_csfl\csfl\()_\ox\oy): +1: + ld1 {v0.16b, v1.16b}, [x6], x7 // luma + ld1 {v6.16b, v7.16b}, [x1], x2 // src +.if \ox + ld1 {v20.8b}, [x4], x10 // grain_lut old +.endif +.if \oy + ld1 {v22.16b, v23.16b}, [x8], x10 // grain_lut top +.endif +.if \ox && \oy + ld1 {v21.8b}, [x11], x10 // grain_lut top old +.endif + ld1 {v18.16b, v19.16b}, [x5], x10 // grain_lut + +.if !\csfl + uxtl v2.8h, v0.8b + uxtl2 v3.8h, v0.16b + uxtl v4.8h, v1.8b + uxtl2 v5.8h, v1.16b + uxtl v0.8h, v6.8b + uxtl2 v1.8h, v6.16b + uxtl v16.8h, v7.8b + uxtl2 v17.8h, v7.16b + mul v2.8h, v2.8h, v8.h[0] + mul v3.8h, v3.8h, v8.h[0] + mul v4.8h, v4.8h, v8.h[0] + mul v5.8h, v5.8h, v8.h[0] + mul v0.8h, v0.8h, v8.h[1] + mul v1.8h, v1.8h, v8.h[1] + mul v16.8h, v16.8h, v8.h[1] + mul v17.8h, v17.8h, v8.h[1] + sqadd v2.8h, v2.8h, v0.8h + sqadd v3.8h, v3.8h, v1.8h + sqadd v4.8h, v4.8h, v16.8h + sqadd v5.8h, v5.8h, v17.8h + sshr v2.8h, v2.8h, #6 + sshr v3.8h, v3.8h, #6 + sshr v4.8h, v4.8h, #6 + sshr v5.8h, v5.8h, #6 + add v2.8h, v2.8h, v24.8h + add v3.8h, v3.8h, v24.8h + add v4.8h, v4.8h, v24.8h + add v5.8h, v5.8h, v24.8h + sqxtun v0.8b, v2.8h + sqxtun2 v0.16b, v3.8h + sqxtun v1.8b, v4.8h + sqxtun2 v1.16b, v5.8h +.endif + + bl gather_neon + +.if \ox + smull v20.8h, v20.8b, v27.8b + smlal v20.8h, v18.8b, v28.8b +.endif + +.if \oy +.if \ox + smull v21.8h, v21.8b, v27.8b + smlal v21.8h, v22.8b, v28.8b + sqrshrn v20.8b, v20.8h, #5 + sqrshrn v21.8b, v21.8h, #5 +.endif + +.if \ox + smull v16.8h, v20.8b, v26.8b +.else + smull v16.8h, v18.8b, v26.8b +.endif + smull2 v17.8h, v18.16b, v26.16b + smull v18.8h, v19.8b, v26.8b + smull2 v19.8h, v19.16b, v26.16b +.if \ox + smlal v16.8h, v21.8b, v25.8b +.else + smlal v16.8h, v22.8b, v25.8b +.endif + smlal2 v17.8h, v22.16b, v25.16b + smlal v18.8h, v23.8b, v25.8b + smlal2 v19.8h, v23.16b, v25.16b + sqrshrn v22.8b, v16.8h, #5 + sqrshrn2 v22.16b, v17.8h, #5 + sqrshrn v23.8b, v18.8h, #5 + sqrshrn2 v23.16b, v19.8h, #5 +.endif + + // sxtl of grain +.if \oy + sxtl v16.8h, v22.8b + sxtl2 v17.8h, v22.16b + sxtl v18.8h, v23.8b + sxtl2 v19.8h, v23.16b +.elseif \ox + sqrshrn v20.8b, v20.8h, #5 + sxtl2 v17.8h, v18.16b + sxtl v18.8h, v19.8b + sxtl2 v19.8h, v19.16b + sxtl v16.8h, v20.8b +.else + sxtl v16.8h, v18.8b + sxtl2 v17.8h, v18.16b + sxtl v18.8h, v19.8b + sxtl2 v19.8h, v19.16b +.endif + + uxtl v2.8h, v4.8b // scaling + uxtl2 v3.8h, v4.16b + uxtl v4.8h, v5.8b + uxtl2 v5.8h, v5.16b + + mul v16.8h, v16.8h, v2.8h // scaling * grain + mul v17.8h, v17.8h, v3.8h + mul v18.8h, v18.8h, v4.8h + mul v19.8h, v19.8h, v5.8h + + srshl v16.8h, v16.8h, v29.8h // round2(scaling * grain, scaling_shift) + srshl v17.8h, v17.8h, v29.8h + srshl v18.8h, v18.8h, v29.8h + srshl v19.8h, v19.8h, v29.8h + + uaddw v16.8h, v16.8h, v6.8b // *src + noise + uaddw2 v17.8h, v17.8h, v6.16b + uaddw v18.8h, v18.8h, v7.8b + uaddw2 v19.8h, v19.8h, v7.16b + + sqxtun v0.8b, v16.8h + sqxtun2 v0.16b, v17.8h + sqxtun v1.8b, v18.8h + sqxtun2 v1.16b, v19.8h + + umax v0.16b, v0.16b, v30.16b + umax v1.16b, v1.16b, v30.16b + umin v0.16b, v0.16b, v31.16b + umin v1.16b, v1.16b, v31.16b + + subs w9, w9, #1 +.if \oy + dup v25.16b, v28.b[0] + dup v26.16b, v28.b[1] +.endif + st1 {v0.16b, v1.16b}, [x0], x2 // dst + b.gt 1b + +.if \oy + cmp w12, #0 + mov w9, w12 // restore actual remaining h + b.gt L(fguv_loop_sx0_csfl\csfl\()_\ox\()0) +.endif + b 9f +.endm + fguv_loop_sx0 0, 0, 0 + fguv_loop_sx0 0, 0, 1 + fguv_loop_sx0 0, 1, 0 + fguv_loop_sx0 0, 1, 1 + fguv_loop_sx0 1, 0, 0 + fguv_loop_sx0 1, 0, 1 + fguv_loop_sx0 1, 1, 0 + fguv_loop_sx0 1, 1, 1 + +9: + ldp d8, d9, [sp, #16] + ldr x30, [sp], #32 + ret + +L(fguv_loop_sx0_tbl): + .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl0_00) + .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl0_01) + .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl0_10) + .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl0_11) + .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl1_00) + .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl1_01) + .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl1_10) + .hword L(fguv_loop_sx0_tbl) - L(fguv_loop_sx0_csfl1_11) +endfunc + +function fguv_loop_sx1_neon +.macro fguv_loop_sx1 csfl, ox, oy +L(fguv_loop_sx1_csfl\csfl\()_\ox\oy): +1: + ld1 {v0.16b, v1.16b}, [x6], x7 // luma + ld1 {v6.16b}, [x1], x2 // src +.if \ox + ld1 {v20.8b}, [x4], x10 // grain_lut old +.endif +.if \oy + ld1 {v22.16b}, [x8], x10 // grain_lut top +.endif +.if \ox && \oy + ld1 {v21.8b}, [x11], x10 // grain_lut top old +.endif + ld1 {v18.16b}, [x5], x10 // grain_lut + + uaddlp v2.8h, v0.16b + uaddlp v3.8h, v1.16b +.if \csfl + rshrn v0.8b, v2.8h, #1 + rshrn2 v0.16b, v3.8h, #1 +.else + urshr v2.8h, v2.8h, #1 + urshr v3.8h, v3.8h, #1 + uxtl v0.8h, v6.8b + uxtl2 v1.8h, v6.16b + mul v2.8h, v2.8h, v8.h[0] + mul v3.8h, v3.8h, v8.h[0] + mul v0.8h, v0.8h, v8.h[1] + mul v1.8h, v1.8h, v8.h[1] + sqadd v2.8h, v2.8h, v0.8h + sqadd v3.8h, v3.8h, v1.8h + sshr v2.8h, v2.8h, #6 + sshr v3.8h, v3.8h, #6 + add v2.8h, v2.8h, v24.8h + add v3.8h, v3.8h, v24.8h + sqxtun v0.8b, v2.8h + sqxtun2 v0.16b, v3.8h +.endif + + bl gather_neon + +.if \ox + smull v20.8h, v20.8b, v27.8b + smlal v20.8h, v18.8b, v28.8b +.endif + +.if \oy +.if \ox + smull v21.8h, v21.8b, v27.8b + smlal v21.8h, v22.8b, v28.8b + sqrshrn v20.8b, v20.8h, #5 + sqrshrn v21.8b, v21.8h, #5 +.endif + +.if \ox + smull v16.8h, v20.8b, v26.8b +.else + smull v16.8h, v18.8b, v26.8b +.endif + smull2 v17.8h, v18.16b, v26.16b +.if \ox + smlal v16.8h, v21.8b, v25.8b +.else + smlal v16.8h, v22.8b, v25.8b +.endif + smlal2 v17.8h, v22.16b, v25.16b + sqrshrn v22.8b, v16.8h, #5 + sqrshrn2 v22.16b, v17.8h, #5 +.endif + + // sxtl of grain +.if \oy + sxtl v16.8h, v22.8b + sxtl2 v17.8h, v22.16b +.elseif \ox + sqrshrn v20.8b, v20.8h, #5 + sxtl2 v17.8h, v18.16b + sxtl v16.8h, v20.8b +.else + sxtl v16.8h, v18.8b + sxtl2 v17.8h, v18.16b +.endif + + uxtl v2.8h, v4.8b // scaling + uxtl2 v3.8h, v4.16b + + mul v16.8h, v16.8h, v2.8h // scaling * grain + mul v17.8h, v17.8h, v3.8h + + srshl v16.8h, v16.8h, v29.8h // round2(scaling * grain, scaling_shift) + srshl v17.8h, v17.8h, v29.8h + + uaddw v16.8h, v16.8h, v6.8b // *src + noise + uaddw2 v17.8h, v17.8h, v6.16b + + sqxtun v0.8b, v16.8h + sqxtun2 v0.16b, v17.8h + + umax v0.16b, v0.16b, v30.16b + umin v0.16b, v0.16b, v31.16b + +.if \oy + mov v16.16b, v25.16b +.endif + subs w9, w9, #1 +.if \oy + mov v25.16b, v26.16b + mov v26.16b, v16.16b +.endif + st1 {v0.16b}, [x0], x2 // dst + b.gt 1b + +.if \oy + cmp w12, #0 + mov w9, w12 // restore actual remaining h + b.gt L(fguv_loop_sx1_csfl\csfl\()_\ox\()0) +.endif + + b 9f +.endm + fguv_loop_sx1 0, 0, 0 + fguv_loop_sx1 0, 0, 1 + fguv_loop_sx1 0, 1, 0 + fguv_loop_sx1 0, 1, 1 + fguv_loop_sx1 1, 0, 0 + fguv_loop_sx1 1, 0, 1 + fguv_loop_sx1 1, 1, 0 + fguv_loop_sx1 1, 1, 1 + +9: + ldp d8, d9, [sp, #16] + ldr x30, [sp], #32 + ret + +L(fguv_loop_sx1_tbl): + .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl0_00) + .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl0_01) + .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl0_10) + .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl0_11) + .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl1_00) + .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl1_01) + .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl1_10) + .hword L(fguv_loop_sx1_tbl) - L(fguv_loop_sx1_csfl1_11) +endfunc diff --git a/third_party/dav1d/src/arm/asm-offsets.h b/third_party/dav1d/src/arm/asm-offsets.h new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/arm/asm-offsets.h @@ -0,0 +1,42 @@ +/* + * Copyright © 2021, VideoLAN and dav1d authors + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef ARM_ASM_OFFSETS_H +#define ARM_ASM_OFFSETS_H + +#define FGD_SEED 0 +#define FGD_AR_COEFF_LAG 92 +#define FGD_AR_COEFFS_Y 96 +#define FGD_AR_COEFF_SHIFT 176 +#define FGD_GRAIN_SCALE_SHIFT 184 + +#define FGD_SCALING_SHIFT 88 +#define FGD_UV_MULT 188 +#define FGD_UV_LUMA_MULT 196 +#define FGD_UV_OFFSET 204 +#define FGD_CLIP_TO_RESTRICTED_RANGE 216 + +#endif /* ARM_ASM_OFFSETS_H */ diff --git a/third_party/dav1d/src/arm/film_grain_init_tmpl.c b/third_party/dav1d/src/arm/film_grain_init_tmpl.c --- a/third_party/dav1d/src/arm/film_grain_init_tmpl.c +++ b/third_party/dav1d/src/arm/film_grain_init_tmpl.c @@ -27,30 +27,63 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "src/cpu.h" #include "src/film_grain.h" +#include "asm-offsets.h" #if BITDEPTH == 8 && ARCH_AARCH64 +CHECK_OFFSET(Dav1dFilmGrainData, seed, FGD_SEED); +CHECK_OFFSET(Dav1dFilmGrainData, ar_coeff_lag, FGD_AR_COEFF_LAG); +CHECK_OFFSET(Dav1dFilmGrainData, ar_coeffs_y, FGD_AR_COEFFS_Y); +CHECK_OFFSET(Dav1dFilmGrainData, ar_coeff_shift, FGD_AR_COEFF_SHIFT); +CHECK_OFFSET(Dav1dFilmGrainData, grain_scale_shift, FGD_GRAIN_SCALE_SHIFT); + +CHECK_OFFSET(Dav1dFilmGrainData, scaling_shift, FGD_SCALING_SHIFT); +CHECK_OFFSET(Dav1dFilmGrainData, uv_mult, FGD_UV_MULT); +CHECK_OFFSET(Dav1dFilmGrainData, uv_luma_mult, FGD_UV_LUMA_MULT); +CHECK_OFFSET(Dav1dFilmGrainData, uv_offset, FGD_UV_OFFSET); +CHECK_OFFSET(Dav1dFilmGrainData, clip_to_restricted_range, FGD_CLIP_TO_RESTRICTED_RANGE); + +void BF(dav1d_generate_grain_y, neon)(entry buf[][GRAIN_WIDTH], + const Dav1dFilmGrainData *const data + HIGHBD_DECL_SUFFIX); + // Use ptrdiff_t instead of int for the last few parameters, to get the // same layout of parameters on the stack across platforms. -#define FGY(suff) \ -void BF(dav1d_fgy_32x32_ ## suff, neon)(pixel *const dst, \ - const pixel *const src, \ - const ptrdiff_t stride, \ - const uint8_t scaling[SCALING_SIZE], \ - const int scaling_shift, \ - const entry grain_lut[][GRAIN_WIDTH], \ - const int offsets[][2], \ - const int h, const ptrdiff_t clip \ - HIGHBD_DECL_SUFFIX) +void BF(dav1d_fgy_32x32, neon)(pixel *const dst, + const pixel *const src, + const ptrdiff_t stride, + const uint8_t scaling[SCALING_SIZE], + const int scaling_shift, + const entry grain_lut[][GRAIN_WIDTH], + const int offsets[][2], + const int h, const ptrdiff_t clip, + const ptrdiff_t type + HIGHBD_DECL_SUFFIX); -FGY(00); -FGY(01); -FGY(10); -FGY(11); +// Use ptrdiff_t instead of int for the last few parameters, to get the +// parameters on the stack with the same layout across platforms. +#define FGUV(suff) \ +void BF(dav1d_fguv_32x32_ ## suff, neon)(pixel *const dst, \ + const pixel *const src, \ + const ptrdiff_t stride, \ + const uint8_t scaling[SCALING_SIZE], \ + const Dav1dFilmGrainData *const data, \ + const entry grain_lut[][GRAIN_WIDTH], \ + const pixel *const luma_row, \ + const ptrdiff_t luma_stride, \ + const int offsets[][2], \ + const ptrdiff_t h, const ptrdiff_t uv, \ + const ptrdiff_t is_id, \ + const ptrdiff_t type \ + HIGHBD_DECL_SUFFIX) + +FGUV(420); +FGUV(422); +FGUV(444); static inline int get_random_number(const int bits, unsigned *const state) { const int r = *state; unsigned bit = ((r >> 0) ^ (r >> 1) ^ (r >> 3) ^ (r >> 12)) & 1; *state = (r >> 1) | (bit << 15); @@ -88,46 +121,89 @@ // update current offsets for (int i = 0; i < rows; i++) offsets[0][i] = get_random_number(8, &seed[i]); - if (data->overlap_flag && row_num) { - if (data->overlap_flag && bx) - BF(dav1d_fgy_32x32_11, neon)(dst_row + bx, src_row + bx, stride, - scaling, data->scaling_shift, - grain_lut, offsets, bh, - data->clip_to_restricted_range - HIGHBD_TAIL_SUFFIX); - else - BF(dav1d_fgy_32x32_01, neon)(dst_row + bx, src_row + bx, stride, - scaling, data->scaling_shift, - grain_lut, offsets, bh, - data->clip_to_restricted_range - HIGHBD_TAIL_SUFFIX); - } else { - if (data->overlap_flag && bx) - BF(dav1d_fgy_32x32_10, neon)(dst_row + bx, src_row + bx, stride, - scaling, data->scaling_shift, - grain_lut, offsets, bh, - data->clip_to_restricted_range - HIGHBD_TAIL_SUFFIX); - else - BF(dav1d_fgy_32x32_00, neon)(dst_row + bx, src_row + bx, stride, - scaling, data->scaling_shift, - grain_lut, offsets, bh, - data->clip_to_restricted_range - HIGHBD_TAIL_SUFFIX); - } + int type = 0; + if (data->overlap_flag && row_num) + type |= 1; /* overlap y */ + if (data->overlap_flag && bx) + type |= 2; /* overlap x */ + + BF(dav1d_fgy_32x32, neon)(dst_row + bx, src_row + bx, stride, + scaling, data->scaling_shift, + grain_lut, offsets, bh, + data->clip_to_restricted_range, type + HIGHBD_TAIL_SUFFIX); } } +#define fguv_ss_fn(nm, sx, sy) \ +static void \ +fguv_32x32xn_##nm##_neon(pixel *const dst_row, const pixel *const src_row, \ + const ptrdiff_t stride, const Dav1dFilmGrainData *const data, \ + const int pw, const uint8_t scaling[SCALING_SIZE], \ + const entry grain_lut[][GRAIN_WIDTH], const int bh, \ + const int row_num, const pixel *const luma_row, \ + const ptrdiff_t luma_stride, const int uv, const int is_id \ + HIGHBD_DECL_SUFFIX) \ +{ \ + const int rows = 1 + (data->overlap_flag && row_num > 0); \ + \ + /* seed[0] contains the current row, seed[1] contains the previous */ \ + unsigned seed[2]; \ + for (int i = 0; i < rows; i++) { \ + seed[i] = data->seed; \ + seed[i] ^= (((row_num - i) * 37 + 178) & 0xFF) << 8; \ + seed[i] ^= (((row_num - i) * 173 + 105) & 0xFF); \ + } \ + \ + int offsets[2 /* col offset */][2 /* row offset */]; \ + \ + /* process this row in BLOCK_SIZE^2 blocks (subsampled) */ \ + for (int bx = 0; bx < pw; bx += BLOCK_SIZE >> sx) { \ + if (data->overlap_flag && bx) { \ + /* shift previous offsets left */ \ + for (int i = 0; i < rows; i++) \ + offsets[1][i] = offsets[0][i]; \ + } \ + \ + /* update current offsets */ \ + for (int i = 0; i < rows; i++) \ + offsets[0][i] = get_random_number(8, &seed[i]); \ + \ + int type = 0; \ + if (data->overlap_flag && row_num) \ + type |= 1; /* overlap y */ \ + if (data->overlap_flag && bx) \ + type |= 2; /* overlap x */ \ + if (data->chroma_scaling_from_luma) \ + type |= 4; \ + \ + BF(dav1d_fguv_32x32_##nm, neon)(dst_row + bx, src_row + bx, stride, \ + scaling, data, grain_lut, \ + luma_row + (bx << sx), luma_stride, \ + offsets, bh, uv, is_id, type \ + HIGHBD_TAIL_SUFFIX); \ + } \ +} + +fguv_ss_fn(420, 1, 1); +fguv_ss_fn(422, 1, 0); +fguv_ss_fn(444, 0, 0); + #endif COLD void bitfn(dav1d_film_grain_dsp_init_arm)(Dav1dFilmGrainDSPContext *const c) { const unsigned flags = dav1d_get_cpu_flags(); if (!(flags & DAV1D_ARM_CPU_FLAG_NEON)) return; #if BITDEPTH == 8 && ARCH_AARCH64 + c->generate_grain_y = BF(dav1d_generate_grain_y, neon); + c->fgy_32x32xn = fgy_32x32xn_neon; + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I420 - 1] = fguv_32x32xn_420_neon; + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I422 - 1] = fguv_32x32xn_422_neon; + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I444 - 1] = fguv_32x32xn_444_neon; #endif } diff --git a/third_party/dav1d/src/decode.c b/third_party/dav1d/src/decode.c --- a/third_party/dav1d/src/decode.c +++ b/third_party/dav1d/src/decode.c @@ -3314,12 +3314,14 @@ &f->frame_thread.td.lock); out_delayed = &c->frame_thread.out_delayed[next]; if (out_delayed->p.data[0]) { const unsigned progress = atomic_load_explicit(&out_delayed->progress[1], memory_order_relaxed); - if (out_delayed->visible && progress != FRAME_ERROR) + if (out_delayed->visible && progress != FRAME_ERROR) { dav1d_picture_ref(&c->out, &out_delayed->p); + c->event_flags |= dav1d_picture_get_event_flags(out_delayed); + } dav1d_thread_picture_unref(out_delayed); } } else { f = c->fc; } @@ -3485,12 +3487,14 @@ f->resize_start[1] = get_upscale_x0(in_cw, out_cw, f->resize_step[1]); } // move f->cur into output queue if (c->n_fc == 1) { - if (f->frame_hdr->show_frame) + if (f->frame_hdr->show_frame) { dav1d_picture_ref(&c->out, &f->sr_cur.p); + c->event_flags |= dav1d_picture_get_event_flags(&f->sr_cur); + } } else { dav1d_thread_picture_ref(out_delayed, &f->sr_cur); } f->w4 = (f->frame_hdr->width[0] + 3) >> 2; diff --git a/third_party/dav1d/src/fg_apply_tmpl.c b/third_party/dav1d/src/fg_apply_tmpl.c --- a/third_party/dav1d/src/fg_apply_tmpl.c +++ b/third_party/dav1d/src/fg_apply_tmpl.c @@ -133,11 +133,11 @@ if (in->p.layout != DAV1D_PIXEL_LAYOUT_I400 && !data->chroma_scaling_from_luma) { assert(out->stride[1] == in->stride[1]); const int ss_ver = in->p.layout == DAV1D_PIXEL_LAYOUT_I420; const ptrdiff_t stride = out->stride[1]; - const ptrdiff_t sz = (out->p.h * stride) >> ss_ver; + const ptrdiff_t sz = ((out->p.h + ss_ver) >> ss_ver) * stride; if (sz < 0) { if (!data->num_uv_points[0]) memcpy((uint8_t*) out->data[1] + sz - stride, (uint8_t*) in->data[1] + sz - stride, -sz); if (!data->num_uv_points[1]) diff --git a/third_party/dav1d/src/film_grain_tmpl.c b/third_party/dav1d/src/film_grain_tmpl.c --- a/third_party/dav1d/src/film_grain_tmpl.c +++ b/third_party/dav1d/src/film_grain_tmpl.c @@ -182,15 +182,11 @@ if (data->clip_to_restricted_range) { min_value = 16 << bitdepth_min_8; max_value = 235 << bitdepth_min_8; } else { min_value = 0; -#if BITDEPTH == 8 - max_value = 0xff; -#else - max_value = bitdepth_max; -#endif + max_value = BITDEPTH_MAX; } // seed[0] contains the current row, seed[1] contains the previous unsigned seed[2]; for (int i = 0; i < rows; i++) { @@ -297,15 +293,11 @@ if (data->clip_to_restricted_range) { min_value = 16 << bitdepth_min_8; max_value = (is_id ? 235 : 240) << bitdepth_min_8; } else { min_value = 0; -#if BITDEPTH == 8 - max_value = 0xff; -#else - max_value = bitdepth_max; -#endif + max_value = BITDEPTH_MAX; } // seed[0] contains the current row, seed[1] contains the previous unsigned seed[2]; for (int i = 0; i < rows; i++) { @@ -368,42 +360,42 @@ // Special case for overlapped column for (int x = 0; x < xstart; x++) { int grain = sample_lut(grain_lut, offsets, sx, sy, 0, 0, x, y); int old = sample_lut(grain_lut, offsets, sx, sy, 1, 0, x, y); - grain = (old * w[sx][x][0] + grain * w[sx][x][1] + 16) >> 5; + grain = round2(old * w[sx][x][0] + grain * w[sx][x][1], 5); grain = iclip(grain, grain_min, grain_max); add_noise_uv(x, y, grain); } } for (int y = 0; y < ystart; y++) { // Special case for overlapped row (sans corner) for (int x = xstart; x < bw; x++) { int grain = sample_lut(grain_lut, offsets, sx, sy, 0, 0, x, y); int old = sample_lut(grain_lut, offsets, sx, sy, 0, 1, x, y); - grain = (old * w[sy][y][0] + grain * w[sy][y][1] + 16) >> 5; + grain = round2(old * w[sy][y][0] + grain * w[sy][y][1], 5); grain = iclip(grain, grain_min, grain_max); add_noise_uv(x, y, grain); } // Special case for doubly-overlapped corner for (int x = 0; x < xstart; x++) { // Blend the top pixel with the top left block int top = sample_lut(grain_lut, offsets, sx, sy, 0, 1, x, y); int old = sample_lut(grain_lut, offsets, sx, sy, 1, 1, x, y); - top = (old * w[sx][x][0] + top * w[sx][x][1] + 16) >> 5; + top = round2(old * w[sx][x][0] + top * w[sx][x][1], 5); top = iclip(top, grain_min, grain_max); // Blend the current pixel with the left block int grain = sample_lut(grain_lut, offsets, sx, sy, 0, 0, x, y); old = sample_lut(grain_lut, offsets, sx, sy, 1, 0, x, y); - grain = (old * w[sx][x][0] + grain * w[sx][x][1] + 16) >> 5; + grain = round2(old * w[sx][x][0] + grain * w[sx][x][1], 5); grain = iclip(grain, grain_min, grain_max); // Mix the row rows together and apply to image - grain = (top * w[sy][y][0] + grain * w[sy][y][1] + 16) >> 5; + grain = round2(top * w[sy][y][0] + grain * w[sy][y][1], 5); grain = iclip(grain, grain_min, grain_max); add_noise_uv(x, y, grain); } } } diff --git a/third_party/dav1d/src/internal.h b/third_party/dav1d/src/internal.h --- a/third_party/dav1d/src/internal.h +++ b/third_party/dav1d/src/internal.h @@ -149,10 +149,12 @@ int operating_point; unsigned operating_point_idc; int all_layers; unsigned frame_size_limit; int drain; + enum PictureFlags frame_flags; + enum Dav1dEventFlags event_flags; Dav1dLogger logger; Dav1dMemPool *picture_pool; }; diff --git a/third_party/dav1d/src/lib.c b/third_party/dav1d/src/lib.c --- a/third_party/dav1d/src/lib.c +++ b/third_party/dav1d/src/lib.c @@ -405,12 +405,14 @@ c->frame_thread.next = 0; if (out_delayed->p.data[0]) { const unsigned progress = atomic_load_explicit(&out_delayed->progress[1], memory_order_relaxed); - if (out_delayed->visible && progress != FRAME_ERROR) + if (out_delayed->visible && progress != FRAME_ERROR) { dav1d_picture_ref(&c->out, &out_delayed->p); + c->event_flags |= dav1d_picture_get_event_flags(out_delayed); + } dav1d_thread_picture_unref(out_delayed); if (output_picture_ready(c)) return output_image(c, out, &c->out); } } while (++drain_count < c->n_fc); @@ -687,10 +689,19 @@ dav1d_mem_pool_end(c->picture_pool); dav1d_freep_aligned(c_out); } +int dav1d_get_event_flags(Dav1dContext *const c, enum Dav1dEventFlags *const flags) { + validate_input_or_ret(c != NULL, DAV1D_ERR(EINVAL)); + validate_input_or_ret(flags != NULL, DAV1D_ERR(EINVAL)); + + *flags = c->event_flags; + c->event_flags = 0; + return 0; +} + void dav1d_picture_unref(Dav1dPicture *const p) { dav1d_picture_unref_internal(p); } uint8_t *dav1d_data_create(Dav1dData *const buf, const size_t sz) { diff --git a/third_party/dav1d/src/obu.c b/third_party/dav1d/src/obu.c --- a/third_party/dav1d/src/obu.c +++ b/third_party/dav1d/src/obu.c @@ -1247,15 +1247,17 @@ return DAV1D_ERR(EINVAL); } // If we have read a sequence header which is different from // the old one, this is a new video sequence and can't use any // previous state. Free that state. - if (!c->seq_hdr) + + if (!c->seq_hdr) { c->frame_hdr = NULL; + c->frame_flags |= PICTURE_FLAG_NEW_SEQUENCE; // see 7.5, operating_parameter_info is allowed to change in // sequence headers of a single sequence - else if (memcmp(seq_hdr, c->seq_hdr, offsetof(Dav1dSequenceHeader, operating_parameter_info))) { + } else if (memcmp(seq_hdr, c->seq_hdr, offsetof(Dav1dSequenceHeader, operating_parameter_info))) { c->frame_hdr = NULL; c->mastering_display = NULL; c->content_light = NULL; dav1d_ref_dec(&c->mastering_display_ref); dav1d_ref_dec(&c->content_light_ref); @@ -1264,10 +1266,16 @@ dav1d_thread_picture_unref(&c->refs[i].p); dav1d_ref_dec(&c->refs[i].segmap); dav1d_ref_dec(&c->refs[i].refmvs); dav1d_cdf_thread_unref(&c->cdf[i]); } + c->frame_flags |= PICTURE_FLAG_NEW_SEQUENCE; + // If operating_parameter_info changed, signal it + } else if (memcmp(seq_hdr->operating_parameter_info, c->seq_hdr->operating_parameter_info, + sizeof(seq_hdr->operating_parameter_info))) + { + c->frame_flags |= PICTURE_FLAG_NEW_OP_PARAMS_INFO; } dav1d_ref_dec(&c->seq_hdr_ref); c->seq_hdr_ref = ref; c->seq_hdr = seq_hdr; break; @@ -1535,10 +1543,11 @@ if (!c->refs[c->frame_hdr->existing_frame_idx].p.p.data[0]) return DAV1D_ERR(EINVAL); if (c->n_fc == 1) { dav1d_picture_ref(&c->out, &c->refs[c->frame_hdr->existing_frame_idx].p.p); dav1d_data_props_copy(&c->out.m, &in->m); + c->event_flags |= dav1d_picture_get_event_flags(&c->refs[c->frame_hdr->existing_frame_idx].p); } else { // need to append this to the frame output queue const unsigned next = c->frame_thread.next++; if (c->frame_thread.next == c->n_fc) c->frame_thread.next = 0; @@ -1551,12 +1560,14 @@ Dav1dThreadPicture *const out_delayed = &c->frame_thread.out_delayed[next]; if (out_delayed->p.data[0]) { const unsigned progress = atomic_load_explicit(&out_delayed->progress[1], memory_order_relaxed); - if (out_delayed->visible && progress != FRAME_ERROR) + if (out_delayed->visible && progress != FRAME_ERROR) { dav1d_picture_ref(&c->out, &out_delayed->p); + c->event_flags |= dav1d_picture_get_event_flags(out_delayed); + } dav1d_thread_picture_unref(out_delayed); } dav1d_thread_picture_ref(out_delayed, &c->refs[c->frame_hdr->existing_frame_idx].p); out_delayed->visible = 1; diff --git a/third_party/dav1d/src/picture.h b/third_party/dav1d/src/picture.h --- a/third_party/dav1d/src/picture.h +++ b/third_party/dav1d/src/picture.h @@ -41,13 +41,19 @@ PLANE_TYPE_UV, PLANE_TYPE_BLOCK, PLANE_TYPE_ALL, }; +enum PictureFlags { + PICTURE_FLAG_NEW_SEQUENCE = 1 << 0, + PICTURE_FLAG_NEW_OP_PARAMS_INFO = 1 << 1, +}; + typedef struct Dav1dThreadPicture { Dav1dPicture p; int visible; + enum PictureFlags flags; struct thread_data *t; // [0] block data (including segmentation map and motion vectors) // [1] pixel data atomic_uint *progress; } Dav1dThreadPicture; @@ -112,6 +118,11 @@ int dav1d_default_picture_alloc(Dav1dPicture *p, void *cookie); void dav1d_default_picture_release(Dav1dPicture *p, void *cookie); void dav1d_picture_unref_internal(Dav1dPicture *p); +/** + * Get event flags from picture flags. + */ +enum Dav1dEventFlags dav1d_picture_get_event_flags(const Dav1dThreadPicture *p); + #endif /* DAV1D_SRC_PICTURE_H */ diff --git a/third_party/dav1d/src/picture.c b/third_party/dav1d/src/picture.c --- a/third_party/dav1d/src/picture.c +++ b/third_party/dav1d/src/picture.c @@ -192,10 +192,13 @@ // Must be removed from the context after being attached to the frame dav1d_ref_dec(&c->itut_t35_ref); c->itut_t35 = NULL; + p->flags = c->frame_flags; + c->frame_flags = 0; + p->visible = f->frame_hdr->show_frame; if (p->t) { atomic_init(&p->progress[0], 0); atomic_init(&p->progress[1], 0); } @@ -252,10 +255,11 @@ { dav1d_picture_ref(&dst->p, &src->p); dst->t = src->t; dst->visible = src->visible; dst->progress = src->progress; + dst->flags = src->flags; } void dav1d_picture_unref_internal(Dav1dPicture *const p) { validate_input(p != NULL); @@ -320,5 +324,18 @@ if (plane_type != PLANE_TYPE_BLOCK) atomic_store(&p->progress[1], y); pthread_cond_broadcast(&p->t->cond); pthread_mutex_unlock(&p->t->lock); } + +enum Dav1dEventFlags dav1d_picture_get_event_flags(const Dav1dThreadPicture *const p) { + if (!p->flags) + return 0; + + enum Dav1dEventFlags flags = 0; + if (p->flags & PICTURE_FLAG_NEW_SEQUENCE) + flags |= DAV1D_EVENT_FLAG_NEW_SEQUENCE; + if (p->flags & PICTURE_FLAG_NEW_OP_PARAMS_INFO) + flags |= DAV1D_EVENT_FLAG_NEW_OP_PARAMS_INFO; + + return flags; +} diff --git a/third_party/dav1d/src/x86/film_grain.asm b/third_party/dav1d/src/x86/film_grain.asm --- a/third_party/dav1d/src/x86/film_grain.asm +++ b/third_party/dav1d/src/x86/film_grain.asm @@ -2070,11 +2070,11 @@ packuswb m0, m9 mova [dstq], m0 %endif sub hb, 1+%2 - jl %%end_y_v_overlap + jle %%end_y_v_overlap %if %2 lea srcq, [srcq+strideq*2] lea dstq, [dstq+strideq*2] lea lumaq, [lumaq+lstrideq*(2<<%3)] %else diff --git a/third_party/dav1d/tests/checkasm/checkasm.h b/third_party/dav1d/tests/checkasm/checkasm.h --- a/third_party/dav1d/tests/checkasm/checkasm.h +++ b/third_party/dav1d/tests/checkasm/checkasm.h @@ -305,28 +305,43 @@ } while (0) #else #define bench_new(...) do {} while (0) #endif + +#define PIXEL_RECT(name, w, h) \ + ALIGN_STK_64(pixel, name##_buf, ((h)+32)*((w)+64) + 64,); \ + ptrdiff_t name##_stride = sizeof(pixel)*((w)+64); \ + (void)name##_stride; \ + pixel *name = name##_buf + ((w)+64)*16 + 64 + +#define CLEAR_PIXEL_RECT(name) \ + memset(name##_buf, 0x99, sizeof(name##_buf)) \ + #define DECL_CHECKASM_CHECK_FUNC(type) \ int checkasm_check_##type(const char *const file, const int line, \ const type *const buf1, const ptrdiff_t stride1, \ const type *const buf2, const ptrdiff_t stride2, \ - const int w, const int h, const char *const name) + const int w, const int h, const char *const name, \ + const int align_w, const int align_h, \ + const int padding) +DECL_CHECKASM_CHECK_FUNC(int8_t); DECL_CHECKASM_CHECK_FUNC(uint8_t); DECL_CHECKASM_CHECK_FUNC(uint16_t); DECL_CHECKASM_CHECK_FUNC(int16_t); DECL_CHECKASM_CHECK_FUNC(int32_t); -#define PASTE(a,b) a ## b -#define CONCAT(a,b) PASTE(a,b) +#define CONCAT(a,b) a ## b -#define checkasm_check(prefix, ...) CONCAT(checkasm_check_, prefix)(__FILE__, __LINE__, __VA_ARGS__) +#define checkasm_check2(prefix, ...) CONCAT(checkasm_check_, prefix)(__FILE__, __LINE__, __VA_ARGS__) +#define checkasm_check(prefix, ...) checkasm_check2(prefix, __VA_ARGS__, 0, 0, 0) #ifdef BITDEPTH #define checkasm_check_pixel(...) checkasm_check(PIXEL_TYPE, __VA_ARGS__) +#define checkasm_check_pixel_padded(...) checkasm_check2(PIXEL_TYPE, __VA_ARGS__, 1, 1, 8) +#define checkasm_check_pixel_padded_align(...) checkasm_check2(PIXEL_TYPE, __VA_ARGS__, 8) #define checkasm_check_coef(...) checkasm_check(COEF_TYPE, __VA_ARGS__) #endif #endif /* DAV1D_TESTS_CHECKASM_CHECKASM_H */ diff --git a/third_party/dav1d/tests/checkasm/checkasm.c b/third_party/dav1d/tests/checkasm/checkasm.c --- a/third_party/dav1d/tests/checkasm/checkasm.c +++ b/third_party/dav1d/tests/checkasm/checkasm.c @@ -766,43 +766,95 @@ signal(SIGILL, handler); signal(SIGSEGV, handler); #endif } +static int check_err(const char *const file, const int line, + const char *const name, const int w, const int h, + int *const err) +{ + if (*err) + return 0; + if (!checkasm_fail_func("%s:%d", file, line)) + return 1; + *err = 1; + fprintf(stderr, "%s (%dx%d):\n", name, w, h); + return 0; +} + #define DEF_CHECKASM_CHECK_FUNC(type, fmt) \ int checkasm_check_##type(const char *const file, const int line, \ const type *buf1, ptrdiff_t stride1, \ const type *buf2, ptrdiff_t stride2, \ - const int w, int h, const char *const name) \ + const int w, int h, const char *const name, \ + const int align_w, const int align_h, \ + const int padding) \ { \ + int aligned_w = (w + align_w - 1) & ~(align_w - 1); \ + int aligned_h = (h + align_h - 1) & ~(align_h - 1); \ + int err = 0; \ stride1 /= sizeof(*buf1); \ stride2 /= sizeof(*buf2); \ int y = 0; \ for (y = 0; y < h; y++) \ if (memcmp(&buf1[y*stride1], &buf2[y*stride2], w*sizeof(*buf1))) \ break; \ - if (y == h) \ - return 0; \ - if (!checkasm_fail_func("%s:%d", file, line)) \ - return 1; \ - fprintf(stderr, "%s:\n", name); \ - while (h--) { \ - for (int x = 0; x < w; x++) \ - fprintf(stderr, " " fmt, buf1[x]); \ - fprintf(stderr, " "); \ - for (int x = 0; x < w; x++) \ - fprintf(stderr, " " fmt, buf2[x]); \ - fprintf(stderr, " "); \ - for (int x = 0; x < w; x++) \ - fprintf(stderr, "%c", buf1[x] != buf2[x] ? 'x' : '.'); \ - buf1 += stride1; \ - buf2 += stride2; \ - fprintf(stderr, "\n"); \ + if (y != h) { \ + if (check_err(file, line, name, w, h, &err)) \ + return 1; \ + for (y = 0; y < h; y++) { \ + for (int x = 0; x < w; x++) \ + fprintf(stderr, " " fmt, buf1[x]); \ + fprintf(stderr, " "); \ + for (int x = 0; x < w; x++) \ + fprintf(stderr, " " fmt, buf2[x]); \ + fprintf(stderr, " "); \ + for (int x = 0; x < w; x++) \ + fprintf(stderr, "%c", buf1[x] != buf2[x] ? 'x' : '.'); \ + buf1 += stride1; \ + buf2 += stride2; \ + fprintf(stderr, "\n"); \ + } \ + buf1 -= h*stride1; \ + buf2 -= h*stride2; \ } \ - return 1; \ + for (y = -padding; y < 0; y++) \ + if (memcmp(&buf1[y*stride1 - padding], &buf2[y*stride2 - padding], \ + (w + 2*padding)*sizeof(*buf1))) { \ + if (check_err(file, line, name, w, h, &err)) \ + return 1; \ + fprintf(stderr, " overwrite above\n"); \ + break; \ + } \ + for (y = aligned_h; y < aligned_h + padding; y++) \ + if (memcmp(&buf1[y*stride1 - padding], &buf2[y*stride2 - padding], \ + (w + 2*padding)*sizeof(*buf1))) { \ + if (check_err(file, line, name, w, h, &err)) \ + return 1; \ + fprintf(stderr, " overwrite below\n"); \ + break; \ + } \ + for (y = 0; y < h; y++) \ + if (memcmp(&buf1[y*stride1 - padding], &buf2[y*stride2 - padding], \ + padding*sizeof(*buf1))) { \ + if (check_err(file, line, name, w, h, &err)) \ + return 1; \ + fprintf(stderr, " overwrite left\n"); \ + break; \ + } \ + for (y = 0; y < h; y++) \ + if (memcmp(&buf1[y*stride1 + aligned_w], &buf2[y*stride2 + aligned_w], \ + padding*sizeof(*buf1))) { \ + if (check_err(file, line, name, w, h, &err)) \ + return 1; \ + fprintf(stderr, " overwrite right\n"); \ + break; \ + } \ + return err; \ } +DEF_CHECKASM_CHECK_FUNC(int8_t, "%4d") DEF_CHECKASM_CHECK_FUNC(uint8_t, "%02x") DEF_CHECKASM_CHECK_FUNC(uint16_t, "%04x") DEF_CHECKASM_CHECK_FUNC(int16_t, "%6d") DEF_CHECKASM_CHECK_FUNC(int32_t, "%9d") diff --git a/third_party/dav1d/tests/checkasm/filmgrain.c b/third_party/dav1d/tests/checkasm/filmgrain.c --- a/third_party/dav1d/tests/checkasm/filmgrain.c +++ b/third_party/dav1d/tests/checkasm/filmgrain.c @@ -32,10 +32,16 @@ #include "src/levels.h" #include "src/film_grain.h" #define UNIT_TEST 1 #include "src/fg_apply_tmpl.c" +#if BITDEPTH == 8 +#define checkasm_check_entry(...) checkasm_check(int8_t, __VA_ARGS__) +#else +#define checkasm_check_entry(...) checkasm_check(int16_t, __VA_ARGS__) +#endif + static const char ss_name[][4] = { [DAV1D_PIXEL_LAYOUT_I420 - 1] = "420", [DAV1D_PIXEL_LAYOUT_I422 - 1] = "422", [DAV1D_PIXEL_LAYOUT_I444 - 1] = "444", }; @@ -63,15 +69,13 @@ for (int n = 0; n < num_y_pos; n++) fg_data[0].ar_coeffs_y[n] = (rnd() & 0xff) - 128; call_ref(grain_lut_c, fg_data HIGHBD_TAIL_SUFFIX); call_new(grain_lut_a, fg_data HIGHBD_TAIL_SUFFIX); - if (memcmp(grain_lut_c, grain_lut_a, - GRAIN_WIDTH * GRAIN_HEIGHT * sizeof(entry))) - { - fail(); - } + checkasm_check_entry(grain_lut_c[0], sizeof(entry) * GRAIN_WIDTH, + grain_lut_a[0], sizeof(entry) * GRAIN_WIDTH, + GRAIN_WIDTH, GRAIN_HEIGHT, "grain_lut"); bench_new(grain_lut_a, fg_data HIGHBD_TAIL_SUFFIX); } } @@ -121,28 +125,29 @@ fg_data[0].ar_coeffs_uv[uv][num_uv_pos] = 0; memset(grain_lut_c, 0xff, sizeof(grain_lut_c)); memset(grain_lut_a, 0xff, sizeof(grain_lut_a)); call_ref(grain_lut_c, grain_lut_y, fg_data, uv HIGHBD_TAIL_SUFFIX); call_new(grain_lut_a, grain_lut_y, fg_data, uv HIGHBD_TAIL_SUFFIX); - int diff = 0, w = ss_x ? 44 : GRAIN_WIDTH; - for (int y = 0; y < (ss_y ? 38 : GRAIN_HEIGHT); y++) - diff |= memcmp(grain_lut_a[y], grain_lut_c[y], w * sizeof(entry)); - if (diff) fail(); + int w = ss_x ? 44 : GRAIN_WIDTH; + int h = ss_y ? 38 : GRAIN_HEIGHT; + checkasm_check_entry(grain_lut_c[0], sizeof(entry) * GRAIN_WIDTH, + grain_lut_a[0], sizeof(entry) * GRAIN_WIDTH, + w, h, "grain_lut"); bench_new(grain_lut_a, grain_lut_y, fg_data, uv HIGHBD_TAIL_SUFFIX); } } } report("gen_grain_uv"); } static void check_fgy_sbrow(const Dav1dFilmGrainDSPContext *const dsp) { - ALIGN_STK_64(pixel, c_dst, 128 * 32,); - ALIGN_STK_64(pixel, a_dst, 128 * 32,); - ALIGN_STK_64(pixel, src, 128 * 32,); - const ptrdiff_t stride = 128 * sizeof(pixel); + PIXEL_RECT(c_dst, 128, 32); + PIXEL_RECT(a_dst, 128, 32); + PIXEL_RECT(src, 128, 32); + const ptrdiff_t stride = c_dst_stride; declare_func(void, pixel *dst_row, const pixel *src_row, ptrdiff_t stride, const Dav1dFilmGrainData *data, size_t pw, const uint8_t scaling[SCALING_SIZE], const entry grain_lut[][GRAIN_WIDTH], @@ -176,44 +181,56 @@ fg_data[0].y_points[n][1] = rnd() & 0xff; } generate_scaling(bitdepth_from_max(bitdepth_max), fg_data[0].y_points, fg_data[0].num_y_points, scaling); - const int w = 1 + (rnd() & 127); - const int h = 1 + (rnd() & 31); - for (int y = 0; y < 32; y++) for (int x = 0; x < 128; x++) src[y * PXSTRIDE(stride) + x] = rnd() & bitdepth_max; - const int row_num = rnd() & 1 ? rnd() & 0x7ff : 0; fg_data[0].clip_to_restricted_range = rnd() & 1; fg_data[0].scaling_shift = (rnd() & 3) + 8; for (fg_data[0].overlap_flag = 0; fg_data[0].overlap_flag <= 1; fg_data[0].overlap_flag++) { - call_ref(c_dst, src, stride, fg_data, w, scaling, grain_lut, h, - row_num HIGHBD_TAIL_SUFFIX); - call_new(a_dst, src, stride, fg_data, w, scaling, grain_lut, h, - row_num HIGHBD_TAIL_SUFFIX); + for (int i = 0; i <= fg_data[0].overlap_flag; i++) { + int w, h, row_num; + if (fg_data[0].overlap_flag) { + w = 35 + (rnd() % 93); + h = 3 + (rnd() % 29); + row_num = i ? 1 + (rnd() & 0x7ff) : 0; + } else { + w = 1 + (rnd() & 127); + h = 1 + (rnd() & 31); + row_num = rnd() & 0x7ff; + } - checkasm_check_pixel(c_dst, stride, a_dst, stride, w, h, "dst"); + CLEAR_PIXEL_RECT(c_dst); + CLEAR_PIXEL_RECT(a_dst); + call_ref(c_dst, src, stride, fg_data, w, scaling, grain_lut, h, + row_num HIGHBD_TAIL_SUFFIX); + call_new(a_dst, src, stride, fg_data, w, scaling, grain_lut, h, + row_num HIGHBD_TAIL_SUFFIX); + + checkasm_check_pixel_padded_align(c_dst, stride, a_dst, stride, + w, h, "dst", 32, 2); + } } fg_data[0].overlap_flag = 1; bench_new(a_dst, src, stride, fg_data, 64, scaling, grain_lut, 32, - row_num HIGHBD_TAIL_SUFFIX); + 1 HIGHBD_TAIL_SUFFIX); } report("fgy_32x32xn"); } static void check_fguv_sbrow(const Dav1dFilmGrainDSPContext *const dsp) { - ALIGN_STK_64(pixel, c_dst, 128 * 32,); - ALIGN_STK_64(pixel, a_dst, 128 * 32,); - ALIGN_STK_64(pixel, src, 128 * 32,); - ALIGN_STK_64(pixel, luma_src, 128 * 32,); - const ptrdiff_t lstride = 128 * sizeof(pixel); + PIXEL_RECT(c_dst, 128, 32); + PIXEL_RECT(a_dst, 128, 32); + PIXEL_RECT(src, 128, 32); + PIXEL_RECT(luma_src, 128, 32); + const ptrdiff_t lstride = luma_src_stride; declare_func(void, pixel *dst_row, const pixel *src_row, ptrdiff_t stride, const Dav1dFilmGrainData *data, size_t pw, const uint8_t scaling[SCALING_SIZE], const entry grain_lut[][GRAIN_WIDTH], int bh, int row_num, @@ -222,11 +239,11 @@ for (int layout_idx = 0; layout_idx < 3; layout_idx++) { const enum Dav1dPixelLayout layout = layout_idx + 1; const int ss_x = layout != DAV1D_PIXEL_LAYOUT_I444; const int ss_y = layout == DAV1D_PIXEL_LAYOUT_I420; - const ptrdiff_t stride = (ss_x ? 96 : 128) * sizeof(pixel); + const ptrdiff_t stride = c_dst_stride; for (int csfl = 0; csfl <= 1; csfl++) { if (check_func(dsp->fguv_32x32xn[layout_idx], "fguv_32x32xn_%dbpc_%s_csfl%d", BITDEPTH, ss_name[layout_idx], csfl)) @@ -256,21 +273,16 @@ fg_data[0].ar_coeffs_uv[uv_pl][n] = (rnd() & 0xff) - 128; dsp->generate_grain_y(grain_lut[0], fg_data HIGHBD_TAIL_SUFFIX); dsp->generate_grain_uv[layout_idx](grain_lut[1], grain_lut[0], fg_data, uv_pl HIGHBD_TAIL_SUFFIX); - const int w = 1 + (rnd() & (127 >> ss_x)); - const int h = 1 + (rnd() & (31 >> ss_y)); - for (int y = 0; y < 32; y++) for (int x = 0; x < 128; x++) src[y * PXSTRIDE(stride) + x] = rnd() & bitdepth_max; for (int y = 0; y < 32; y++) for (int x = 0; x < 128; x++) luma_src[y * PXSTRIDE(lstride) + x] = rnd() & bitdepth_max; - const int row_num = rnd() & 1 ? rnd() & 0x7ff : 0; - if (csfl) { fg_data[0].num_y_points = 2 + (rnd() % 13); const int pad = 0xff / fg_data[0].num_y_points; for (int n = 0; n < fg_data[0].num_y_points; n++) { fg_data[0].y_points[n][0] = 0xff * n / fg_data[0].num_y_points; @@ -299,21 +311,39 @@ fg_data[0].scaling_shift = (rnd() & 3) + 8; fg_data[0].chroma_scaling_from_luma = csfl; for (fg_data[0].overlap_flag = 0; fg_data[0].overlap_flag <= 1; fg_data[0].overlap_flag++) { - call_ref(c_dst, src, stride, fg_data, w, scaling, grain_lut[1], h, - row_num, luma_src, lstride, uv_pl, is_identity HIGHBD_TAIL_SUFFIX); - call_new(a_dst, src, stride, fg_data, w, scaling, grain_lut[1], h, - row_num, luma_src, lstride, uv_pl, is_identity HIGHBD_TAIL_SUFFIX); + for (int i = 0; i <= fg_data[0].overlap_flag; i++) { + int w, h, row_num; + if (fg_data[0].overlap_flag) { + w = (36 >> ss_x) + (rnd() % (92 >> ss_x)); + h = (4 >> ss_y) + (rnd() % (28 >> ss_y)); + row_num = i ? 1 + (rnd() & 0x7ff) : 0; + } else { + w = 1 + (rnd() & (127 >> ss_x)); + h = 1 + (rnd() & (31 >> ss_y)); + row_num = rnd() & 0x7ff; + } - checkasm_check_pixel(c_dst, stride, a_dst, stride, w, h, "dst"); + CLEAR_PIXEL_RECT(c_dst); + CLEAR_PIXEL_RECT(a_dst); + call_ref(c_dst, src, stride, fg_data, w, scaling, grain_lut[1], h, + row_num, luma_src, lstride, uv_pl, is_identity HIGHBD_TAIL_SUFFIX); + call_new(a_dst, src, stride, fg_data, w, scaling, grain_lut[1], h, + row_num, luma_src, lstride, uv_pl, is_identity HIGHBD_TAIL_SUFFIX); + + checkasm_check_pixel_padded_align(c_dst, stride, + a_dst, stride, + w, h, "dst", + 32 >> ss_x, 2); + } } fg_data[0].overlap_flag = 1; bench_new(a_dst, src, stride, fg_data, 32, scaling, grain_lut[1], 16, - row_num, luma_src, lstride, uv_pl, is_identity HIGHBD_TAIL_SUFFIX); + 1, luma_src, lstride, uv_pl, is_identity HIGHBD_TAIL_SUFFIX); } } } report("fguv_32x32xn"); diff --git a/third_party/dav1d/tests/checkasm/ipred.c b/third_party/dav1d/tests/checkasm/ipred.c --- a/third_party/dav1d/tests/checkasm/ipred.c +++ b/third_party/dav1d/tests/checkasm/ipred.c @@ -64,12 +64,12 @@ 58, 61, 64, 67, 70, 73, 76, 81, 84, 87 }; static void check_intra_pred(Dav1dIntraPredDSPContext *const c) { - ALIGN_STK_64(pixel, c_dst, 64 * 64,); - ALIGN_STK_64(pixel, a_dst, 64 * 64,); + PIXEL_RECT(c_dst, 64, 64); + PIXEL_RECT(a_dst, 64, 64); ALIGN_STK_64(pixel, topleft_buf, 257,); pixel *const topleft = topleft_buf + 128; declare_func(void, pixel *dst, ptrdiff_t stride, const pixel *topleft, int width, int height, int angle, int max_width, int max_height @@ -87,11 +87,11 @@ intra_pred_mode_names[mode], w, bpc)) { for (int h = imax(w / 4, 4); h <= imin(w * 4, (mode == FILTER_PRED ? 32 : 64)); h <<= 1) { - const ptrdiff_t stride = w * sizeof(pixel); + const ptrdiff_t stride = c_dst_stride; int a = 0, maxw = 0, maxh = 0; if (mode >= Z1_PRED && mode <= Z3_PRED) { /* angle */ a = (90 * (mode - Z1_PRED) + z_angles[rnd() % 27]) | (rnd() & 0x600); @@ -110,16 +110,19 @@ bitdepth_max = (1 << bpc) - 1; for (int i = -h * 2; i <= w * 2; i++) topleft[i] = rnd() & bitdepth_max; + CLEAR_PIXEL_RECT(c_dst); + CLEAR_PIXEL_RECT(a_dst); call_ref(c_dst, stride, topleft, w, h, a, maxw, maxh HIGHBD_TAIL_SUFFIX); call_new(a_dst, stride, topleft, w, h, a, maxw, maxh HIGHBD_TAIL_SUFFIX); - if (checkasm_check_pixel(c_dst, stride, a_dst, stride, - w, h, "dst")) + if (checkasm_check_pixel_padded(c_dst, stride, + a_dst, stride, + w, h, "dst")) { if (mode == Z1_PRED || mode == Z3_PRED) fprintf(stderr, "angle = %d (0x%03x)\n", a & 0x1ff, a & 0x600); else if (mode == Z2_PRED) diff --git a/third_party/dav1d/tools/input/ivf.c b/third_party/dav1d/tools/input/ivf.c --- a/third_party/dav1d/tools/input/ivf.c +++ b/third_party/dav1d/tools/input/ivf.c @@ -64,17 +64,16 @@ } static int ivf_open(IvfInputContext *const c, const char *const file, unsigned fps[2], unsigned *const num_frames, unsigned timebase[2]) { - size_t res; uint8_t hdr[32]; if (!(c->f = fopen(file, "rb"))) { fprintf(stderr, "Failed to open %s: %s\n", file, strerror(errno)); return -1; - } else if ((res = fread(hdr, 32, 1, c->f)) != 1) { + } else if (fread(hdr, 32, 1, c->f) != 1) { fprintf(stderr, "Failed to read stream header: %s\n", strerror(errno)); fclose(c->f); return -1; } else if (memcmp(hdr, "DKIF", 4)) { fprintf(stderr, "%s is not an IVF file [tag=%.4s|0x%02x%02x%02x%02x]\n", @@ -93,13 +92,13 @@ const unsigned duration = rl32(&hdr[24]); uint8_t data[8]; c->broken = 0; for (*num_frames = 0;; (*num_frames)++) { - if ((res = fread(data, 4, 1, c->f)) != 1) break; // EOF + if (fread(data, 4, 1, c->f) != 1) break; // EOF size_t sz = rl32(data); - if ((res = fread(data, 8, 1, c->f)) != 1) break; // EOF + if (fread(data, 8, 1, c->f) != 1) break; // EOF const uint64_t ts = rl64(data); if (*num_frames && ts <= c->last_ts) c->broken = 1; c->last_ts = ts; fseeko(c->f, sz, SEEK_CUR);