diff --git a/media/libdav1d/asm/moz.build b/media/libdav1d/asm/moz.build --- a/media/libdav1d/asm/moz.build +++ b/media/libdav1d/asm/moz.build @@ -85,12 +85,14 @@ SOURCES += [ '../../../third_party/dav1d/src/x86/cdef16_avx2.asm', # moved from autovendored '../../../third_party/dav1d/src/x86/cdef_avx2.asm', '../../../third_party/dav1d/src/x86/cdef_avx512.asm', '../../../third_party/dav1d/src/x86/film_grain16_avx2.asm', + '../../../third_party/dav1d/src/x86/film_grain16_sse.asm', '../../../third_party/dav1d/src/x86/film_grain_avx2.asm', '../../../third_party/dav1d/src/x86/ipred16_avx2.asm', + '../../../third_party/dav1d/src/x86/ipred16_sse.asm', '../../../third_party/dav1d/src/x86/ipred_avx2.asm', '../../../third_party/dav1d/src/x86/itx16_avx2.asm', '../../../third_party/dav1d/src/x86/itx_avx2.asm', '../../../third_party/dav1d/src/x86/loopfilter16_avx2.asm', '../../../third_party/dav1d/src/x86/loopfilter16_sse.asm', @@ -98,10 +100,12 @@ '../../../third_party/dav1d/src/x86/looprestoration16_avx2.asm', '../../../third_party/dav1d/src/x86/looprestoration_avx2.asm', '../../../third_party/dav1d/src/x86/mc16_avx2.asm', '../../../third_party/dav1d/src/x86/mc_avx2.asm', '../../../third_party/dav1d/src/x86/mc_avx512.asm', + '../../../third_party/dav1d/src/x86/refmvs.asm', + '../../../third_party/dav1d/src/x86/refmvs_init.c', ] SOURCES += [ '../../../third_party/dav1d/src/x86/cdef16_sse.asm', '../../../third_party/dav1d/src/x86/cdef_sse.asm', @@ -146,10 +150,11 @@ flags=['BITDEPTH', '8']) elif CONFIG['CPU_ARCH'] == 'arm' or CONFIG['CPU_ARCH'] == 'aarch64': SOURCES += [ '../../../third_party/dav1d/src/arm/cpu.c', + '../../../third_party/dav1d/src/arm/refmvs_init.c', ] EXPORTS += [ '../../../third_party/dav1d/src/arm/asm-offsets.h', '../../../third_party/dav1d/src/arm/cpu.h', '../../../third_party/dav1d/src/arm/msac.h', @@ -201,10 +206,11 @@ '../../../third_party/dav1d/src/arm/64/looprestoration_common.S', '../../../third_party/dav1d/src/arm/64/looprestoration_tmpl.S', '../../../third_party/dav1d/src/arm/64/mc.S', '../../../third_party/dav1d/src/arm/64/mc16.S', '../../../third_party/dav1d/src/arm/64/msac.S', + '../../../third_party/dav1d/src/arm/64/refmvs.S', ] elif CONFIG['CPU_ARCH'] == 'arm': SOURCES += [ '../../../third_party/dav1d/src/arm/32/cdef.S', '../../../third_party/dav1d/src/arm/32/cdef16.S', @@ -222,10 +228,11 @@ '../../../third_party/dav1d/src/arm/32/looprestoration_common.S', '../../../third_party/dav1d/src/arm/32/looprestoration_tmpl.S', '../../../third_party/dav1d/src/arm/32/mc.S', '../../../third_party/dav1d/src/arm/32/mc16.S', '../../../third_party/dav1d/src/arm/32/msac.S', + '../../../third_party/dav1d/src/arm/32/refmvs.S', ] if CONFIG['CPU_ARCH'] in ('x86', 'x86_64'): USE_NASM = True elif CONFIG['CPU_ARCH'] == 'aarch64' and CONFIG['OS_TARGET'] == 'WINNT': diff --git a/media/libdav1d/moz.yaml b/media/libdav1d/moz.yaml --- a/media/libdav1d/moz.yaml +++ b/media/libdav1d/moz.yaml @@ -18,15 +18,15 @@ # Usually different from repository url url: https://code.videolan.org/videolan/dav1d # Human-readable identifier for this version/release # Generally "version NNN", "tag SSS", "bookmark SSS" - release: commit ddbbfde198aced0d02ea739c320d754d43406f7b (2021-06-12T07:58:29.000+00:00). + release: commit d174e6f0fa8a8dcbd5eacd6d0227b1924edf80be (2021-09-21T10:13:44.000+00:00). # Revision to pull in # Must be a long or short commit SHA (long preferred) - revision: ddbbfde198aced0d02ea739c320d754d43406f7b + revision: d174e6f0fa8a8dcbd5eacd6d0227b1924edf80be # The package's license, where possible using the mnemonic from # https://spdx.org/licenses/ # Multiple licenses can be specified (as a YAML list) # A "LICENSE" file must exist containing the full license text @@ -62,5 +62,6 @@ to: '{yaml_dir}/vcs_version.h' - action: replace-in-file pattern: '@VCS_TAG@' with: '{revision}' file: '{yaml_dir}/vcs_version.h' + diff --git a/media/libdav1d/vcs_version.h b/media/libdav1d/vcs_version.h --- a/media/libdav1d/vcs_version.h +++ b/media/libdav1d/vcs_version.h @@ -1,2 +1,2 @@ /* auto-generated, do not edit */ -#define DAV1D_VERSION "0.9.0-24-gddbbfde" +#define DAV1D_VERSION "d174e6f0fa8a8dcbd5eacd6d0227b1924edf80be" diff --git a/third_party/dav1d/NEWS b/third_party/dav1d/NEWS --- a/third_party/dav1d/NEWS +++ b/third_party/dav1d/NEWS @@ -1,5 +1,31 @@ +Changes for 0.9.2 'Golden Eagle': +--------------------------------- + +0.9.2 is a small update of dav1d on the 0.9.x branch: + - x86: SSE4 optimizations of inverse transforms for 10bit for all sizes + - x86: mc.resize optimizations with AVX2/SSSE3 for 10/12b + - x86: SSSE3 optimizations for cdef_filter in 10/12b and mc_w_mask_422/444 in 8b + - ARM NEON optimizations for FilmGrain Gen_grain functions + - Optimizations for splat_mv in SSE2/AVX2 and NEON + - x86: SGR improvements for SSSE3 CPUs + - x86: AVX2 optimizations for cfl_ac + + +Changes for 0.9.1 'Golden Eagle': +--------------------------------- + +0.9.1 is a middle-size revision of dav1d, adding notably 10b acceleration for SSSE3: + - 10/12b SSSE3 optimizations for mc (avg, w_avg, mask, w_mask, emu_edge), + prep/put_bilin, prep/put_8tap, ipred (dc/h/v, paeth, smooth, pal, filter), wiener, + sgr (10b), warp8x8, deblock, film_grain, cfl_ac/pred for 32bit and 64bit x86 processors + - Film grain NEON for fguv 10/12b, fgy/fguv 8b and fgy/fguv 10/12 arm32 + - Fixes for filmgrain on ARM + - itx 10bit optimizations for 4x4/x8/x16, 8x4/x8/x16 for SSE4 + - Misc improvements on SSE2, SSE4 + + Changes for 0.9.0 'Golden Eagle': --------------------------------- 0.9.0 is a major version of dav1d, adding notably 10b acceleration on x64. diff --git a/third_party/dav1d/README.md b/third_party/dav1d/README.md --- a/third_party/dav1d/README.md +++ b/third_party/dav1d/README.md @@ -34,20 +34,20 @@ 5. Make it fast on mobile, by writing asm for ARMv8 chips, 6. Make it fast on older desktop, by writing asm for SSSE3+ chips, 7. Make high bit-depth fast on mobile, by writing asm for ARMv8 chips. 8. Make it fast on older mobile, by writing asm for ARMv7 chips, 9. Make high bit-depth fast on older mobile, by writing asm for ARMv7 chips, +10. Make high bit-depth fast on desktop, by writing asm for AVX2 chips, +11. Make high bit-depth fast on older desktop, by writing asm for SSSE3+ chips, ### On-going -10. Improve C code base with [various tweaks](https://code.videolan.org/videolan/dav1d/wikis/task-list), -11. Accelerate for less common architectures, like PPC, SSE2 or AVX-512. -12. Make high bit-depth fast on desktop, by writing asm for AVX2 chips, +12. Improve C code base with [various tweaks](https://code.videolan.org/videolan/dav1d/wikis/task-list), +13. Accelerate for less common architectures, like PPC, SSE2 or AVX-512. +14. Improve threading. ### After -13. Make high bit-depth fast on older desktop, by writing asm for SSSE3+ chips, -14. Use more GPU decoding, when possible. -15. Improve threading. +15. Use more GPU decoding, when possible. # Contribute Currently, we are looking for help from: - C developers, @@ -58,11 +58,11 @@ Our contributions guidelines are quite strict. We want to build a coherent codebase to simplify maintenance and achieve the highest possible speed. Notably, the codebase is in pure C and asm. -We are on IRC, on the **#dav1d** channel on [*Libera.chat*](http://libera.chat/). If you do not have an IRC Client at hand, use [KiwiIRC Web Interface](https://kiwiirc.com/nextclient/#ircs://irc.libera.chat/#dav1d). +We are on IRC, on the **#dav1d** channel on [*Libera.chat*](http://libera.chat/). If you do not have an IRC Client at hand, use [IRC Web Interface](https://web.libera.chat/#dav1d). See the [contributions document](CONTRIBUTING.md). ## CLA diff --git a/third_party/dav1d/include/common/attributes.h b/third_party/dav1d/include/common/attributes.h --- a/third_party/dav1d/include/common/attributes.h +++ b/third_party/dav1d/include/common/attributes.h @@ -31,10 +31,18 @@ #include "config.h" #include #include +#ifndef __has_attribute +#define __has_attribute(x) 0 +#endif + +#ifndef __has_feature +#define __has_feature(x) 0 +#endif + #ifdef __GNUC__ #define ATTR_ALIAS __attribute__((may_alias)) #define ATTR_FORMAT_PRINTF(fmt, attr) __attribute__((__format__(__printf__, fmt, attr))) #define COLD __attribute__((cold)) #else @@ -91,13 +99,15 @@ * Forbid inlining of a function: * static NOINLINE void func() {} */ #ifdef _MSC_VER #define NOINLINE __declspec(noinline) -#else /* !_MSC_VER */ +#elif __has_attribute(noclone) +#define NOINLINE __attribute__((noinline, noclone)) +#else #define NOINLINE __attribute__((noinline)) -#endif /* !_MSC_VER */ +#endif #ifdef __clang__ #define NO_SANITIZE(x) __attribute__((no_sanitize(x))) #else #define NO_SANITIZE(x) @@ -158,18 +168,20 @@ static inline int clzll(const unsigned long long mask) { return __builtin_clzll(mask); } #endif /* !_MSC_VER */ -#ifndef __has_feature -#define __has_feature(x) 0 -#endif - #ifndef static_assert #define CHECK_OFFSET(type, field, name) \ struct check_##type##_##field { int x[(name == offsetof(type, field)) ? 1 : -1]; } #else #define CHECK_OFFSET(type, field, name) \ static_assert(name == offsetof(type, field), #field) #endif +#ifdef _MSC_VER +#define PACKED(...) __pragma(pack(push, 1)) __VA_ARGS__ __pragma(pack(pop)) +#else +#define PACKED(...) __VA_ARGS__ __attribute__((__packed__)) +#endif + #endif /* DAV1D_COMMON_ATTRIBUTES_H */ diff --git a/third_party/dav1d/include/compat/msvc/stdatomic.h b/third_party/dav1d/include/compat/msvc/stdatomic.h --- a/third_party/dav1d/include/compat/msvc/stdatomic.h +++ b/third_party/dav1d/include/compat/msvc/stdatomic.h @@ -39,29 +39,31 @@ #include #include "common/attributes.h" -typedef volatile LONG __declspec(align(32)) atomic_int; -typedef volatile ULONG __declspec(align(32)) atomic_uint; +typedef volatile LONG atomic_int; +typedef volatile ULONG atomic_uint; typedef enum { memory_order_relaxed, memory_order_acquire } msvc_atomic_memory_order; #define atomic_init(p_a, v) do { *(p_a) = (v); } while(0) #define atomic_store(p_a, v) InterlockedExchange((LONG*)p_a, v) #define atomic_load(p_a) InterlockedCompareExchange((LONG*)p_a, 0, 0) +#define atomic_exchange(p_a, v) InterlockedExchange(p_a, v) #define atomic_load_explicit(p_a, mo) atomic_load(p_a) /* * TODO use a special call to increment/decrement * using InterlockedIncrement/InterlockedDecrement */ #define atomic_fetch_add(p_a, inc) InterlockedExchangeAdd(p_a, inc) #define atomic_fetch_sub(p_a, dec) InterlockedExchangeAdd(p_a, -(dec)) +#define atomic_fetch_or(p_a, v) InterlockedOr(p_a, v) #endif /* ! stdatomic.h */ #pragma warning(pop) diff --git a/third_party/dav1d/include/dav1d/dav1d.h b/third_party/dav1d/include/dav1d/dav1d.h --- a/third_party/dav1d/include/dav1d/dav1d.h +++ b/third_party/dav1d/include/dav1d/dav1d.h @@ -41,13 +41,12 @@ #include "version.h" typedef struct Dav1dContext Dav1dContext; typedef struct Dav1dRef Dav1dRef; -#define DAV1D_MAX_FRAME_THREADS 256 -#define DAV1D_MAX_TILE_THREADS 64 -#define DAV1D_MAX_POSTFILTER_THREADS 256 +#define DAV1D_MAX_THREADS 256 +#define DAV1D_MAX_FRAME_DELAY 256 typedef struct Dav1dLogger { void *cookie; ///< Custom data to pass to the callback. /** * Logger callback. May be NULL to disable logging. @@ -58,20 +57,19 @@ */ void (*callback)(void *cookie, const char *format, va_list ap); } Dav1dLogger; typedef struct Dav1dSettings { - int n_frame_threads; - int n_tile_threads; + int n_threads; ///< number of threads (0 = auto) + int max_frame_delay; ///< Set to 1 for low-latency decoding (0 = auto) int apply_grain; int operating_point; ///< select an operating point for scalable AV1 bitstreams (0 - 31) int all_layers; ///< output all spatial layers of a scalable AV1 biststream unsigned frame_size_limit; ///< maximum frame size, in pixels (0 = unlimited) Dav1dPicAllocator allocator; ///< Picture allocator callback. Dav1dLogger logger; ///< Logger callback. - int n_postfilter_threads; - uint8_t reserved[28]; ///< reserved for future use + uint8_t reserved[32]; ///< reserved for future use } Dav1dSettings; /** * Get library version. */ @@ -103,11 +101,16 @@ * * @param out Output Sequence Header. * @param buf The data to be parser. * @param sz Size of the data. * - * @return 0 on success, or < 0 (a negative DAV1D_ERR code) on error. + * @return + * 0: Success, and out is filled with the parsed Sequence Header + * OBU parameters. + * DAV1D_ERR(ENOENT): No Sequence Header OBUs were found in the buffer. + * other negative DAV1D_ERR codes: Invalid data in the buffer, invalid passed-in + * arguments, and other errors during parsing. * * @note It is safe to feed this function data containing other OBUs than a * Sequence Header, as they will simply be ignored. If there is more than * one Sequence Header OBU present, only the last will be returned. */ diff --git a/third_party/dav1d/meson.build b/third_party/dav1d/meson.build --- a/third_party/dav1d/meson.build +++ b/third_party/dav1d/meson.build @@ -21,18 +21,18 @@ # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. project('dav1d', ['c'], - version: '0.9.0', + version: '0.9.2', default_options: ['c_std=c99', 'warning_level=2', 'buildtype=release', 'b_ndebug=if-release'], meson_version: '>= 0.49.0') -dav1d_soname_version = '5.1.0' +dav1d_soname_version = '6.0.0' dav1d_api_version_array = dav1d_soname_version.split('.') dav1d_api_version_major = dav1d_api_version_array[0] dav1d_api_version_minor = dav1d_api_version_array[1] dav1d_api_version_revision = dav1d_api_version_array[2] @@ -171,27 +171,32 @@ libm_dependency = cc.find_library('m', required: false) # Header checks -stdatomic_dependency = [] +stdatomic_dependencies = [] if not cc.check_header('stdatomic.h') if cc.get_id() == 'msvc' # we have a custom replacement for MSVC - stdatomic_dependency = declare_dependency( + stdatomic_dependencies += declare_dependency( include_directories : include_directories('include/compat/msvc'), ) elif cc.compiles('''int main() { int v = 0; return __atomic_fetch_add(&v, 1, __ATOMIC_SEQ_CST); }''', name : 'GCC-style atomics', args : test_args) - stdatomic_dependency = declare_dependency( + stdatomic_dependencies += declare_dependency( include_directories : include_directories('include/compat/gcc'), ) else error('Atomics not supported') endif endif +if host_machine.cpu_family().startswith('wasm') + # enable atomics + bulk-memory features + stdatomic_dependencies += thread_dependency.partial_dependency(compile_args: true) +endif + if cc.check_header('unistd.h') cdata.set('HAVE_UNISTD_H', 1) endif if cc.check_header('io.h') @@ -245,10 +250,11 @@ '-Wundef', '-Werror=vla', '-Wno-maybe-uninitialized', '-Wno-missing-field-initializers', '-Wno-unused-parameter', + '-Wstrict-prototypes', '-Werror=missing-prototypes', '-Wshorten-64-to-32', ] if host_machine.cpu_family() == 'x86' optional_arguments += [ @@ -367,15 +373,22 @@ if host_machine.cpu_family().startswith('x86') cdata_asm.set('private_prefix', 'dav1d') cdata_asm.set10('ARCH_X86_64', host_machine.cpu_family() == 'x86_64') cdata_asm.set10('ARCH_X86_32', host_machine.cpu_family() == 'x86') cdata_asm.set10('PIC', true) + + # Convert SSE asm into (128-bit) AVX when compiler flags are set to use AVX instructions + cdata_asm.set10('FORCE_VEX_ENCODING', cc.get_define('__AVX__') != '') endif cdata.set10('ARCH_PPC64LE', host_machine.cpu() == 'ppc64le') -if cc.symbols_have_underscore_prefix() +# meson's cc.symbols_have_underscore_prefix() is unfortunately unrelieably +# when additional flags like '-fprofile-instr-generate' are passed via CFLAGS +# see following meson issue https://github.com/mesonbuild/meson/issues/5482 +if (host_machine.system() == 'darwin' or + (host_machine.system() == 'windows' and host_machine.cpu_family() == 'x86')) cdata.set10('PREFIX', true) cdata_asm.set10('PREFIX', true) endif # diff --git a/third_party/dav1d/src/arm/32/film_grain.S b/third_party/dav1d/src/arm/32/film_grain.S --- a/third_party/dav1d/src/arm/32/film_grain.S +++ b/third_party/dav1d/src/arm/32/film_grain.S @@ -28,10 +28,1335 @@ #include "src/arm/asm.S" #include "util.S" #include "src/arm/asm-offsets.h" #define GRAIN_WIDTH 82 +#define GRAIN_HEIGHT 73 + +#define SUB_GRAIN_WIDTH 44 +#define SUB_GRAIN_HEIGHT 38 + +.macro increment_seed steps, shift=1 + lsr r11, r2, #3 + lsr r12, r2, #12 + lsr lr, r2, #1 + eor r11, r2, r11 // (r >> 0) ^ (r >> 3) + eor r12, r12, lr // (r >> 12) ^ (r >> 1) + eor r11, r11, r12 // (r >> 0) ^ (r >> 3) ^ (r >> 12) ^ (r >> 1) +.if \shift + lsr r2, r2, #\steps +.endif + and r11, r11, #((1 << \steps) - 1) // bit +.if \shift + orr r2, r2, r11, lsl #(16 - \steps) // *state +.else + orr r2, r2, r11, lsl #16 // *state +.endif +.endm + +.macro read_rand dest, bits, age + ubfx \dest, r2, #16 - \bits - \age, #\bits +.endm + +.macro read_shift_rand dest, bits + ubfx \dest, r2, #17 - \bits, #\bits + lsr r2, r2, #1 +.endm + +// special calling convention: +// r2 holds seed +// r3 holds dav1d_gaussian_sequence +// clobbers r11-r12 +// returns in d0-d1 +function get_gaussian_neon + push {r5-r6,lr} + increment_seed 4 + read_rand r5, 11, 3 + read_rand r6, 11, 2 + add r5, r3, r5, lsl #1 + add r6, r3, r6, lsl #1 + vld1.16 {d0[0]}, [r5] + read_rand r5, 11, 1 + vld1.16 {d0[1]}, [r6] + add r5, r3, r5, lsl #1 + read_rand r6, 11, 0 + increment_seed 4 + add r6, r3, r6, lsl #1 + vld1.16 {d0[2]}, [r5] + read_rand r5, 11, 3 + vld1.16 {d0[3]}, [r6] + add r5, r3, r5, lsl #1 + read_rand r6, 11, 2 + vld1.16 {d1[0]}, [r5] + add r6, r3, r6, lsl #1 + read_rand r5, 11, 1 + vld1.16 {d1[1]}, [r6] + read_rand r6, 11, 0 + add r5, r3, r5, lsl #1 + add r6, r3, r6, lsl #1 + vld1.16 {d1[2]}, [r5] + vld1.16 {d1[3]}, [r6] + pop {r5-r6,pc} +endfunc + +.macro get_grain_row r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10 + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + vmovn.i16 \r0, q0 + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + vmovn.i16 \r1, q0 + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + vmovn.i16 \r2, q0 + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + vmovn.i16 \r3, q0 + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + vmovn.i16 \r4, q0 + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + vmovn.i16 \r5, q0 + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + vmovn.i16 \r6, q0 + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + vmovn.i16 \r7, q0 + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + vmovn.i16 \r8, q0 + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + vmovn.i16 \r9, q0 + increment_seed 2 + read_rand r11, 11, 1 + read_rand r12, 11, 0 + add r11, r3, r11, lsl #1 + add r12, r3, r12, lsl #1 + vld1.16 {d0[0]}, [r11] + vld1.16 {d0[1]}, [r12] + vrshl.s16 d0, d0, d30 + vmovn.i16 \r10, q0 +.endm + +.macro store_grain_row r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10 + vst1.16 {\r0, \r1, \r2, \r3}, [r0]! + vst1.16 {\r4, \r5, \r6, \r7}, [r0]! + vst1.16 {\r8, \r9}, [r0]! + vst1.16 {\r10[0]}, [r0]! +.endm + +.macro get_grain_row_44 r0, r1, r2, r3, r4, r5 + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + vmovn.i16 \r0, q0 + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + vmovn.i16 \r1, q0 + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + vmovn.i16 \r2, q0 + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + vmovn.i16 \r3, q0 + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + vmovn.i16 \r4, q0 + increment_seed 4 + read_rand r11, 11, 3 + read_rand r12, 11, 2 + add r11, r3, r11, lsl #1 + add r12, r3, r12, lsl #1 + vld1.16 {d0[]}, [r11] + read_rand r11, 11, 1 + vld1.16 {d0[1]}, [r12] + add r11, r3, r11, lsl #1 + read_rand r12, 11, 0 + vld1.16 {d0[2]}, [r11] + add r12, r3, r12, lsl #1 + vld1.16 {d0[3]}, [r12] + vrshl.s16 d0, d0, d30 + vmovn.i16 \r5, q0 +.endm + +.macro store_grain_row_44 r0, r1, r2, r3, r4, r5 + vst1.16 {\r0, \r1, \r2, \r3}, [r0]! + vst1.16 {\r4, \r5}, [r0] + add r0, r0, #GRAIN_WIDTH-32 +.endm + +function get_grain_2_neon + push {r11,lr} + increment_seed 2 + read_rand r11, 11, 1 + read_rand r12, 11, 0 + add r11, r3, r11, lsl #1 + add r12, r3, r12, lsl #1 + vld1.16 {d0[0]}, [r11] + vld1.16 {d0[1]}, [r12] + vrshl.s16 d0, d0, d30 + vmovn.i16 d0, q0 + pop {r11,pc} +endfunc + +.macro get_grain_2 dst + bl get_grain_2_neon +.ifnc \dst, d0 + vmov \dst, d0 +.endif +.endm + +// r1 holds the number of entries to produce +// r6, r8 and r10 hold the previous output entries +// q0 holds the vector of produced entries +// q1 holds the input vector of sums from above +.macro output_lag n +function output_lag\n\()_neon + push {r0, lr} +.if \n == 1 + mov lr, #-128 +.else + mov r0, #1 + mov lr, #1 + sub r7, r7, #1 + sub r9, r9, #1 + lsl r0, r0, r7 + lsl lr, lr, r9 + add r7, r7, #1 + add r9, r9, #1 +.endif +1: + read_shift_rand r12, 11 + vmov.32 r11, d2[0] + lsl r12, r12, #1 + vext.8 q0, q0, q0, #1 + ldrsh r12, [r3, r12] +.if \n == 1 + mla r11, r6, r4, r11 // sum (above) + *coeff * prev output + add r6, r11, r8 // 1 << (ar_coeff_shift - 1) + add r12, r12, r10 + asr r6, r6, r7 // >> ar_coeff_shift + asr r12, r12, r9 // >> (4 + grain_scale_shift) + add r6, r6, r12 + cmp r6, r5 +.elseif \n == 2 + mla r11, r8, r4, r11 // sum (above) + *coeff * prev output 1 + mla r11, r6, r10, r11 // += *coeff * prev output 2 + mov r8, r6 + add r6, r11, r0 // 1 << (ar_coeff_shift - 1) + add r12, r12, lr // 1 << (4 + grain_scale_shift - 1) + asr r6, r6, r7 // >> ar_coeff_shift + asr r12, r12, r9 // >> (4 + grain_scale_shift) + add r6, r6, r12 + push {lr} + cmp r6, r5 + mov lr, #-128 +.else + push {r1-r3} + sbfx r1, r4, #0, #8 + sbfx r2, r4, #8, #8 + sbfx r3, r4, #16, #8 + mla r11, r10, r1, r11 // sum (above) + *coeff * prev output 1 + mla r11, r8, r2, r11 // sum (above) + *coeff * prev output 2 + mla r11, r6, r3, r11 // += *coeff * prev output 3 + pop {r1-r3} + mov r10, r8 + mov r8, r6 + + add r6, r11, r0 // 1 << (ar_coeff_shift - 1) + add r12, r12, lr // 1 << (4 + grain_scale_shift - 1) + asr r6, r6, r7 // >> ar_coeff_shift + asr r12, r12, r9 // >> (4 + grain_scale_shift) + add r6, r6, r12 + push {lr} + cmp r6, r5 + mov lr, #-128 +.endif + it gt + movgt r6, r5 + cmp r6, lr + it lt + movlt r6, lr +.if \n >= 2 + pop {lr} +.endif + subs r1, r1, #1 + vext.8 q1, q1, q1, #4 + vmov.8 d1[7], r6 + bgt 1b + pop {r0, pc} +endfunc +.endm + +output_lag 1 +output_lag 2 +output_lag 3 + + +function sum_lag1_above_neon + vmull.s8 q2, d6, d28 + vmull.s8 q3, d7, d28 + vmull.s8 q4, d0, d27 + vmull.s8 q5, d1, d27 + + vaddl.s16 q0, d4, d8 + vaddl.s16 q2, d5, d9 + vaddl.s16 q4, d6, d10 + vaddl.s16 q5, d7, d11 + + vmull.s8 q3, d3, d29 + vmull.s8 q1, d2, d29 + + vaddw.s16 q4, q4, d6 + vaddw.s16 q5, q5, d7 + vaddw.s16 q3, q2, d3 + vaddw.s16 q2, q0, d2 + bx lr +endfunc + +.macro sum_lag_n_body lag, type, uv_layout, edge, elems, store, uv_coeff +.ifc \lag\()_\edge, lag3_left + bl sum_lag3_left_above_neon +.else + bl sum_\lag\()_above_neon +.endif +.ifc \type, uv_420 + vpush {q6-q7} + add r12, r11, #GRAIN_WIDTH + vld1.16 {q0, q1}, [r11]! + vld1.16 {q6, q7}, [r12]! + vpaddl.s8 q0, q0 + vpaddl.s8 q1, q1 + vpaddl.s8 q6, q6 + vpaddl.s8 q7, q7 + vadd.i16 q0, q0, q6 + vadd.i16 q1, q1, q7 + vpop {q6-q7} + vrshrn.s16 d0, q0, #2 + vrshrn.s16 d1, q1, #2 +.endif +.ifc \type, uv_422 + vld1.8 {q0, q1}, [r11]! + vpaddl.s8 q0, q0 + vpaddl.s8 q1, q1 + vrshrn.s16 d0, q0, #1 + vrshrn.s16 d1, q1, #1 +.endif +.ifc \type, uv_444 + vld1.8 {q0}, [r11]! +.endif +.if \uv_layout +.ifnb \uv_coeff + vdup.8 d13, \uv_coeff +.endif + vmull.s8 q1, d0, d13 + vmull.s8 q0, d1, d13 + vaddw.s16 q2, q2, d2 + vaddw.s16 q3, q3, d3 + vaddw.s16 q4, q4, d0 + vaddw.s16 q5, q5, d1 +.endif +.if \uv_layout && \elems == 16 + b sum_\lag\()_y_\edge\()_start +.elseif \uv_layout == 444 && \elems == 15 + b sum_\lag\()_y_\edge\()_start +.elseif \uv_layout == 422 && \elems == 9 + b sum_\lag\()_uv_420_\edge\()_start +.else +sum_\lag\()_\type\()_\edge\()_start: + push {r11} +.ifc \edge, left + increment_seed 4 + read_rand r11, 11, 3 + read_rand r12, 11, 2 + add r11, r3, r11, lsl #1 + add r12, r3, r12, lsl #1 + vld1.16 {d1[1]}, [r11] + read_rand r11, 11, 1 + vld1.16 {d1[2]}, [r12] + add r11, r3, r11, lsl #1 + vld1.16 {d1[3]}, [r11] + lsl r2, r2, #1 // shift back the state as if we'd done increment_seed with shift=0 + vrshl.s16 d1, d1, d30 + vmovn.i16 d1, q0 + vext.8 q2, q2, q2, #12 +.ifc \lag, lag3 + vmov.s8 r10, d1[5] +.endif +.ifnc \lag, lag1 + vmov.s8 r8, d1[6] +.endif + vmov.s8 r6, d1[7] + + vmov q1, q2 + mov r1, #1 + bl output_\lag\()_neon +.else + increment_seed 4, shift=0 + vmov q1, q2 + mov r1, #4 + bl output_\lag\()_neon +.endif + + increment_seed 4, shift=0 + vmov q1, q3 + mov r1, #4 + bl output_\lag\()_neon + + increment_seed 4, shift=0 + vmov q1, q4 +.if \elems == 9 + mov r1, #1 + bl output_\lag\()_neon + lsr r2, r2, #3 + + read_rand r11, 11, 2 + read_rand r12, 11, 1 + add r11, r3, r11, lsl #1 + add r12, r3, r12, lsl #1 + vld1.16 {d2[0]}, [r11] + read_rand r11, 11, 0 + vld1.16 {d2[1]}, [r12] + add r11, r3, r11, lsl #1 + vld1.16 {d2[2]}, [r11] + vrshl.s16 d2, d2, d30 + vmovn.i16 d2, q1 + vext.8 q0, q0, q1, #7 +.else + mov r1, #4 + bl output_\lag\()_neon + + increment_seed 4, shift=0 + vmov q1, q5 + +.ifc \edge, right + mov r1, #3 + bl output_\lag\()_neon + read_shift_rand r11, 11 + add r11, r3, r11, lsl #1 + vld1.16 {d2[0]}, [r11] + vrshl.s16 d2, d2, d30 + vext.8 q0, q0, q1, #1 +.else + mov r1, #4 + bl output_\lag\()_neon +.endif +.endif +.if \store + vst1.8 {q0}, [r0]! +.endif + pop {r11} + pop {r1, pc} +.endif +.endm + +.macro sum_lag1_func type, uv_layout, edge, elems=16 +function sum_\type\()_lag1_\edge\()_neon + push {r1, lr} + sum_lag_n_body lag1, \type, \uv_layout, \edge, \elems, store=0 +endfunc +.endm + +sum_lag1_func y, 0, left +sum_lag1_func y, 0, mid +sum_lag1_func y, 0, right, 15 +sum_lag1_func uv_444, 444, left +sum_lag1_func uv_444, 444, mid +sum_lag1_func uv_444, 444, right, 15 +sum_lag1_func uv_422, 422, left +sum_lag1_func uv_422, 422, mid +sum_lag1_func uv_422, 422, right, 9 +sum_lag1_func uv_420, 420, left +sum_lag1_func uv_420, 420, mid +sum_lag1_func uv_420, 420, right, 9 + +.macro sum_lag1 type, dst, left, mid, right, edge=mid + vmov q3, \mid + vext.8 q0, \left, \mid, #15 + vext.8 q1, \mid, \right, #1 + bl sum_\type\()_lag1_\edge\()_neon + vmov \dst, q0 +.endm + +.macro sum_y_lag1 dst, left, mid, right, edge=mid + sum_lag1 y, \dst, \left, \mid, \right, \edge +.endm + +.macro sum_uv_444_lag1 dst, left, mid, right, edge=mid + sum_lag1 uv_444, \dst, \left, \mid, \right, \edge +.endm + +.macro sum_uv_422_lag1 dst, left, mid, right, edge=mid + sum_lag1 uv_422, \dst, \left, \mid, \right, \edge +.endm + +.macro sum_uv_420_lag1 dst, left, mid, right, edge=mid + sum_lag1 uv_420, \dst, \left, \mid, \right, \edge +.endm + + +function sum_lag2_above_neon + push {lr} + sub r12, r0, #2*GRAIN_WIDTH - 16 + sub lr, r0, #1*GRAIN_WIDTH - 16 + vld1.8 {q10}, [r12] // load top right + vld1.8 {q13}, [lr] + + vext.8 q6, q8, q9, #14 // top left, top mid + vdup.8 d14, d28[0] + vext.8 q8, q8, q9, #15 + vdup.8 d15, d28[1] + + vmull.s8 q0, d12, d14 + vmull.s8 q1, d13, d14 + vmull.s8 q6, d16, d15 + vmull.s8 q8, d17, d15 + + vaddl.s16 q2, d0, d12 + vaddl.s16 q3, d1, d13 + vaddl.s16 q4, d2, d16 + vaddl.s16 q5, d3, d17 + + vext.8 q6, q9, q10, #1 // top mid, top right + vdup.8 d14, d28[3] + vext.8 q8, q9, q10, #2 + vdup.8 d15, d28[4] + + vmull.s8 q0, d12, d14 + vmull.s8 q1, d13, d14 + vmull.s8 q6, d16, d15 + vmull.s8 q8, d17, d15 + + vaddl.s16 q7, d0, d12 + vaddl.s16 q0, d1, d13 + vaddl.s16 q6, d2, d16 + vaddl.s16 q1, d3, d17 + + vadd.i32 q2, q2, q7 + vadd.i32 q3, q3, q0 + vadd.i32 q4, q4, q6 + vadd.i32 q5, q5, q1 + + vext.8 q6, q11, q12, #14 // top left, top mid + vdup.8 d14, d28[5] + vext.8 q8, q11, q12, #15 + vdup.8 d15, d28[6] + + vmull.s8 q0, d12, d14 + vmull.s8 q1, d13, d14 + vmull.s8 q6, d16, d15 + vmull.s8 q8, d17, d15 + + vaddl.s16 q7, d0, d12 + vaddl.s16 q0, d1, d13 + vaddl.s16 q6, d2, d16 + vaddl.s16 q1, d3, d17 + + vadd.i32 q2, q2, q7 + vadd.i32 q3, q3, q0 + vadd.i32 q4, q4, q6 + vadd.i32 q5, q5, q1 + + vext.8 q6, q12, q13, #1 // top mid, top right + vdup.8 d14, d29[0] + vext.8 q8, q12, q13, #2 + vdup.8 d15, d29[1] + + vmull.s8 q0, d12, d14 + vmull.s8 q1, d13, d14 + vmull.s8 q6, d16, d15 + vmull.s8 q8, d17, d15 + + vaddl.s16 q7, d0, d12 + vaddl.s16 q0, d1, d13 + vaddl.s16 q6, d2, d16 + vaddl.s16 q1, d3, d17 + + vadd.i32 q2, q2, q7 + vadd.i32 q3, q3, q0 + vadd.i32 q4, q4, q6 + vadd.i32 q5, q5, q1 + + vdup.8 d14, d28[2] + vdup.8 d15, d28[7] + + vmull.s8 q0, d18, d14 + vmull.s8 q1, d19, d14 + vmull.s8 q6, d24, d15 + vmull.s8 q8, d25, d15 + + vaddl.s16 q7, d0, d12 + vaddl.s16 q0, d1, d13 + vaddl.s16 q6, d2, d16 + vaddl.s16 q1, d3, d17 + + vmov q8, q9 + vmov q9, q10 + + vadd.i32 q2, q2, q7 + vadd.i32 q3, q3, q0 + vadd.i32 q4, q4, q6 + vadd.i32 q5, q5, q1 + + vmov q11, q12 + vmov q12, q13 + + pop {pc} +endfunc + +.macro sum_lag2_func type, uv_layout, edge, elems=16 +function sum_\type\()_lag2_\edge\()_neon + push {r1, lr} +.ifc \edge, left + sub r12, r0, #2*GRAIN_WIDTH + sub lr, r0, #1*GRAIN_WIDTH + vld1.8 {q9}, [r12] // load the previous block right above + vld1.8 {q12}, [lr] +.endif + sum_lag_n_body lag2, \type, \uv_layout, \edge, \elems, store=1, uv_coeff=d29[4] +endfunc +.endm + +sum_lag2_func y, 0, left +sum_lag2_func y, 0, mid +sum_lag2_func y, 0, right, 15 +sum_lag2_func uv_444, 444, left +sum_lag2_func uv_444, 444, mid +sum_lag2_func uv_444, 444, right, 15 +sum_lag2_func uv_422, 422, left +sum_lag2_func uv_422, 422, mid +sum_lag2_func uv_422, 422, right, 9 +sum_lag2_func uv_420, 420, left +sum_lag2_func uv_420, 420, mid +sum_lag2_func uv_420, 420, right, 9 + + +function sum_lag3_left_above_neon + // A separate codepath for the left edge, to avoid reading outside + // of the edge of the buffer. + sub r12, r0, #3*GRAIN_WIDTH + vld1.8 {q11, q12}, [r12] + vext.8 q12, q11, q12, #13 + vext.8 q11, q11, q11, #13 + b sum_lag3_above_start +endfunc + +function sum_lag3_above_neon + sub r12, r0, #3*GRAIN_WIDTH + 3 + vld1.8 {q11, q12}, [r12] + +sum_lag3_above_start: + vdup.8 d20, d26[0] + vext.8 q9, q11, q12, #1 + vdup.8 d21, d26[1] + + vmull.s8 q0, d22, d20 + vmull.s8 q1, d23, d20 + vmull.s8 q6, d18, d21 + vmull.s8 q7, d19, d21 + + vext.8 q8, q11, q12, #2 + vdup.8 d20, d26[2] + vext.8 q9, q11, q12, #3 + vdup.8 d21, d26[3] + + vaddl.s16 q2, d0, d12 + vaddl.s16 q3, d1, d13 + vaddl.s16 q4, d2, d14 + vaddl.s16 q5, d3, d15 + + vmull.s8 q0, d16, d20 + vmull.s8 q1, d17, d20 + vmull.s8 q6, d18, d21 + vmull.s8 q7, d19, d21 + + vaddl.s16 q8, d0, d12 + vaddl.s16 q9, d1, d13 + vaddl.s16 q0, d2, d14 + vaddl.s16 q1, d3, d15 + + vext.8 q6, q11, q12, #4 + vdup.8 d20, d26[4] + vext.8 q7, q11, q12, #5 + vdup.8 d21, d26[5] + + vadd.i32 q2, q2, q8 + vadd.i32 q3, q3, q9 + vadd.i32 q4, q4, q0 + vadd.i32 q5, q5, q1 + + vmull.s8 q0, d12, d20 + vmull.s8 q1, d13, d20 + vmull.s8 q8, d14, d21 + vmull.s8 q9, d15, d21 + + sub r12, r0, #2*GRAIN_WIDTH + 3 + + vaddl.s16 q6, d0, d16 + vaddl.s16 q7, d1, d17 + vaddl.s16 q0, d2, d18 + vaddl.s16 q1, d3, d19 + + vext.8 q8, q11, q12, #6 + vld1.8 {q11, q12}, [r12] + vdup.8 d20, d26[6] + vdup.8 d21, d26[7] + + vadd.i32 q2, q2, q6 + vadd.i32 q3, q3, q7 + vadd.i32 q4, q4, q0 + vadd.i32 q5, q5, q1 + + vmull.s8 q0, d16, d20 + vmull.s8 q1, d17, d20 + vmull.s8 q6, d22, d21 + vmull.s8 q7, d23, d21 + + vaddl.s16 q8, d0, d12 + vaddl.s16 q9, d1, d13 + vaddl.s16 q0, d2, d14 + vaddl.s16 q1, d3, d15 + + vext.8 q6, q11, q12, #1 + vdup.8 d20, d27[0] + vext.8 q7, q11, q12, #2 + vdup.8 d21, d27[1] + + vadd.i32 q2, q2, q8 + vadd.i32 q3, q3, q9 + vadd.i32 q4, q4, q0 + vadd.i32 q5, q5, q1 + + vmull.s8 q0, d12, d20 + vmull.s8 q1, d13, d20 + vmull.s8 q8, d14, d21 + vmull.s8 q9, d15, d21 + + vaddl.s16 q6, d0, d16 + vaddl.s16 q7, d1, d17 + vaddl.s16 q0, d2, d18 + vaddl.s16 q1, d3, d19 + + vext.8 q8, q11, q12, #3 + vdup.8 d20, d27[2] + vext.8 q9, q11, q12, #4 + vdup.8 d21, d27[3] + + vadd.i32 q2, q2, q6 + vadd.i32 q3, q3, q7 + vadd.i32 q4, q4, q0 + vadd.i32 q5, q5, q1 + + vmull.s8 q0, d16, d20 + vmull.s8 q1, d17, d20 + vmull.s8 q6, d18, d21 + vmull.s8 q7, d19, d21 + + sub r12, r0, #1*GRAIN_WIDTH + 3 + + vaddl.s16 q8, d0, d12 + vaddl.s16 q9, d1, d13 + vaddl.s16 q0, d2, d14 + vaddl.s16 q1, d3, d15 + + vext.8 q6, q11, q12, #5 + vdup.8 d20, d27[4] + vext.8 q7, q11, q12, #6 + vdup.8 d21, d27[5] + + vld1.8 {q11, q12}, [r12] + + vadd.i32 q2, q2, q8 + vadd.i32 q3, q3, q9 + vadd.i32 q4, q4, q0 + vadd.i32 q5, q5, q1 + + vmull.s8 q0, d12, d20 + vmull.s8 q1, d13, d20 + vmull.s8 q8, d14, d21 + vmull.s8 q9, d15, d21 + + vaddl.s16 q6, d0, d16 + vaddl.s16 q7, d1, d17 + vaddl.s16 q0, d2, d18 + vaddl.s16 q1, d3, d19 + + vdup.8 d20, d27[6] + vext.8 q9, q11, q12, #1 + vdup.8 d21, d27[7] + + vadd.i32 q2, q2, q6 + vadd.i32 q3, q3, q7 + vadd.i32 q4, q4, q0 + vadd.i32 q5, q5, q1 + + vmull.s8 q0, d22, d20 + vmull.s8 q1, d23, d20 + vmull.s8 q6, d18, d21 + vmull.s8 q7, d19, d21 + + vaddl.s16 q8, d0, d12 + vaddl.s16 q9, d1, d13 + vaddl.s16 q0, d2, d14 + vaddl.s16 q1, d3, d15 + + vext.8 q6, q11, q12, #2 + vdup.8 d20, d28[0] + vext.8 q7, q11, q12, #3 + vdup.8 d21, d28[1] + + vadd.i32 q2, q2, q8 + vadd.i32 q3, q3, q9 + vadd.i32 q4, q4, q0 + vadd.i32 q5, q5, q1 + + vmull.s8 q0, d12, d20 + vmull.s8 q1, d13, d20 + vmull.s8 q8, d14, d21 + vmull.s8 q9, d15, d21 + + vaddl.s16 q6, d0, d16 + vaddl.s16 q7, d1, d17 + vaddl.s16 q0, d2, d18 + vaddl.s16 q1, d3, d19 + + vext.8 q8, q11, q12, #4 + vdup.8 d20, d28[2] + vext.8 q9, q11, q12, #5 + vdup.8 d21, d28[3] + + vadd.i32 q2, q2, q6 + vadd.i32 q3, q3, q7 + vadd.i32 q4, q4, q0 + vadd.i32 q5, q5, q1 + + vmull.s8 q0, d16, d20 + vmull.s8 q1, d17, d20 + vmull.s8 q6, d18, d21 + vmull.s8 q7, d19, d21 + + vaddl.s16 q8, d0, d12 + vaddl.s16 q9, d1, d13 + vaddl.s16 q0, d2, d14 + vaddl.s16 q1, d3, d15 + + vext.8 q6, q11, q12, #6 + vdup.8 d20, d28[4] + + vadd.i32 q2, q2, q8 + vadd.i32 q3, q3, q9 + vadd.i32 q4, q4, q0 + vadd.i32 q5, q5, q1 + + vmull.s8 q0, d12, d20 + vmull.s8 q1, d13, d20 + + vaddw.s16 q2, q2, d0 + vaddw.s16 q3, q3, d1 + vaddw.s16 q4, q4, d2 + vaddw.s16 q5, q5, d3 + + bx lr +endfunc + +.macro sum_lag3_func type, uv_layout, edge, elems=16 +function sum_\type\()_lag3_\edge\()_neon + push {r1, lr} + sum_lag_n_body lag3, \type, \uv_layout, \edge, \elems, store=1, uv_coeff=d29[0] +endfunc +.endm + +sum_lag3_func y, 0, left +sum_lag3_func y, 0, mid +sum_lag3_func y, 0, right, 15 +sum_lag3_func uv_444, 444, left +sum_lag3_func uv_444, 444, mid +sum_lag3_func uv_444, 444, right, 15 +sum_lag3_func uv_422, 422, left +sum_lag3_func uv_422, 422, mid +sum_lag3_func uv_422, 422, right, 9 +sum_lag3_func uv_420, 420, left +sum_lag3_func uv_420, 420, mid +sum_lag3_func uv_420, 420, right, 9 + +function generate_grain_rows_neon + push {r11,lr} +1: + get_grain_row d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26 + subs r1, r1, #1 + store_grain_row d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26 + bgt 1b + pop {r11,pc} +endfunc + +function generate_grain_rows_44_neon + push {r11,lr} +1: + get_grain_row_44 d16, d17, d18, d19, d20, d21 + subs r1, r1, #1 + store_grain_row_44 d16, d17, d18, d19, d20, d21 + bgt 1b + pop {r11,pc} +endfunc + +function gen_grain_uv_444_lag0_neon + vld1.8 {q3}, [r11]! + push {r11,lr} + bl get_gaussian_neon + vrshl.s16 q8, q0, q15 + bl get_gaussian_neon + vrshl.s16 q9, q0, q15 + vqmovn.s16 d0, q8 + vqmovn.s16 d1, q9 + + vand q3, q3, q1 + vmull.s8 q2, d6, d22 + vmull.s8 q3, d7, d22 + vrshl.s16 q2, q2, q12 + vrshl.s16 q3, q3, q12 + vaddw.s8 q2, q2, d0 + vaddw.s8 q3, q3, d1 + vqmovn.s16 d4, q2 + vqmovn.s16 d5, q3 + vst1.8 {q2}, [r0]! + pop {r11,pc} +endfunc + +function get_grain_row_44_neon + push {r11,lr} + get_grain_row_44 d16, d17, d18, d19, d20, d21 + pop {r11,pc} +endfunc + +function add_uv_420_coeff_lag0_neon + vld1.16 {q2, q3}, [r11]! + vld1.16 {q4, q5}, [r12]! + vpaddl.s8 q2, q2 + vpaddl.s8 q3, q3 + vpaddl.s8 q4, q4 + vpaddl.s8 q5, q5 + vadd.i16 q2, q2, q4 + vadd.i16 q3, q3, q5 + vrshrn.s16 d4, q2, #2 + vrshrn.s16 d5, q3, #2 + b add_coeff_lag0_start +endfunc + +function add_uv_422_coeff_lag0_neon + vld1.16 {q2, q3}, [r11]! + vpaddl.s8 q2, q2 + vpaddl.s8 q3, q3 + vrshrn.s16 d4, q2, #1 + vrshrn.s16 d5, q3, #1 + +add_coeff_lag0_start: + vand q3, q2, q1 + vmull.s8 q2, d6, d22 + vmull.s8 q3, d7, d22 + vrshl.s16 q2, q2, q12 + vrshl.s16 q3, q3, q12 + vaddw.s8 q2, q2, d0 + vaddw.s8 q3, q3, d1 + vqmovn.s16 d4, q2 + vqmovn.s16 d5, q3 + bx lr +endfunc + +.macro gen_grain_82 type +function generate_grain_\type\()_8bpc_neon, export=1 + push {r4-r11,lr} + +.ifc \type, uv_444 + mov r12, r3 + mov lr, #28 + add r11, r1, #3*GRAIN_WIDTH + mov r1, r2 + mul r12, r12, lr +.endif + movrel r3, X(gaussian_sequence) + ldr r2, [r1, #FGD_SEED] + ldr r9, [r1, #FGD_GRAIN_SCALE_SHIFT] +.ifc \type, y + add r4, r1, #FGD_AR_COEFFS_Y +.else + add r4, r1, #FGD_AR_COEFFS_UV +.endif + adr r5, L(gen_grain_\type\()_tbl) + ldr r6, [r1, #FGD_AR_COEFF_LAG] + add r9, r9, #4 + ldr r6, [r5, r6, lsl #2] + vdup.16 q15, r9 // 4 + data->grain_scale_shift + add r5, r5, r6 + vneg.s16 q15, q15 + +.ifc \type, uv_444 + cmp r12, #0 + movw r10, #0x49d8 + movw lr, #0xb524 + // Intentionally using a separate register instead of moveq with an + // immediate constant, to avoid armv8 deprecated it instruction forms. + it eq + moveq r10, lr + add r4, r4, r12 // Add offset to ar_coeffs_uv[1] + eor r2, r2, r10 +.endif + + ldr r7, [r1, #FGD_AR_COEFF_SHIFT] + mov r8, #1 + mov r10, #1 + lsl r8, r8, r7 // 1 << ar_coeff_shift + lsl r10, r10, r9 // 1 << (4 + data->grain_scale_shift) + lsr r8, r8, #1 // 1 << (ar_coeff_shift - 1) + lsr r10, r10, #1 // 1 << (4 + data->grain_scale_shift - 1) + + bx r5 + + .align 2 +L(gen_grain_\type\()_tbl): + .word L(generate_grain_\type\()_lag0) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + .word L(generate_grain_\type\()_lag1) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + .word L(generate_grain_\type\()_lag2) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + .word L(generate_grain_\type\()_lag3) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + +L(generate_grain_\type\()_lag0): +.ifc \type, y + mov r1, #GRAIN_HEIGHT + bl generate_grain_rows_neon +.else + + mov r1, #3 + bl generate_grain_rows_neon + mov r1, #GRAIN_HEIGHT-3 + + vdup.16 q12, r7 + vld1.8 {d22[]}, [r4] // ar_coeffs_uv[0] + vmov.i8 q0, #0 + vmov.i8 q1, #255 + vext.8 q13, q0, q1, #13 + vext.8 q14, q1, q0, #1 + vneg.s16 q12, q12 + +1: + vmov q1, q13 + bl gen_grain_uv_444_lag0_neon // 16 + vmov.i8 q1, #255 + bl gen_grain_uv_444_lag0_neon // 32 + bl gen_grain_uv_444_lag0_neon // 48 + bl gen_grain_uv_444_lag0_neon // 64 + vmov q1, q14 + bl gen_grain_uv_444_lag0_neon // 80 + get_grain_2 d16 + subs r1, r1, #1 + add r11, r11, #2 + vst1.16 {d16[0]}, [r0]! + bgt 1b +.endif + pop {r4-r11,pc} + +L(generate_grain_\type\()_lag1): + vpush {q4-q7} + mov r5, #127 + vld1.8 {d27[]}, [r4]! // ar_coeffs_y[0] + vld1.8 {d28[]}, [r4]! // ar_coeffs_y[1] + vld1.8 {d29[]}, [r4] // ar_coeffs_y[2] +.ifc \type, y + ldrsb r4, [r4, #1] // ar_coeffs_y[3] +.else + add r4, r4, #2 +.endif + + mov r1, #3 +.ifc \type, uv_444 + vld1.8 {d13[]}, [r4] // ar_coeffs_uv[4] + ldrsb r4, [r4, #-1] // ar_coeffs_uv[3] +.endif + bl generate_grain_rows_neon + + mov r1, #GRAIN_HEIGHT - 3 +1: + sum_\type\()_lag1 q7, q8, q8, q9, left + sum_\type\()_lag1 q8, q8, q9, q10 + sum_\type\()_lag1 q9, q9, q10, q11 + sum_\type\()_lag1 q10, q10, q11, q12 + sum_\type\()_lag1 q12, q11, q12, q13, right + get_grain_2 d26 + subs r1, r1, #1 +.ifc \type, uv_444 + add r11, r11, #2 +.endif + store_grain_row d14, d15, d16, d17, d18, d19, d20, d21, d24, d25, d26 + vmov q11, q10 + vmov q10, q9 + vmov q9, q8 + vmov q8, q7 + bgt 1b + + vpop {q4-q7} + pop {r4-r11,pc} + +L(generate_grain_\type\()_lag2): + vpush {q4-q7} + mov r5, #127 + vld1.8 {d28,d29}, [r4] // ar_coeffs_y[0-11], ar_coeffs_uv[0-12] + + vmov.s8 r4, d29[2] + vmov.s8 r10, d29[3] + + mov r1, #3 + bl generate_grain_rows_neon + + mov r1, #GRAIN_HEIGHT - 3 +1: + bl sum_\type\()_lag2_left_neon + bl sum_\type\()_lag2_mid_neon + bl sum_\type\()_lag2_mid_neon + bl sum_\type\()_lag2_mid_neon + bl sum_\type\()_lag2_right_neon + get_grain_2 d16 + subs r1, r1, #1 +.ifc \type, uv_444 + add r11, r11, #2 +.endif + vst1.16 {d16[0]}, [r0]! + bgt 1b + + vpop {q4-q7} + pop {r4-r11,pc} + +L(generate_grain_\type\()_lag3): + vpush {q4-q7} + mov r5, #127 + vld1.8 {q13, q14}, [r4] // ar_coeffs_y[0-23], ar_coeffs_uv[0-24] + + vmov.u8 r4, d28[5] + vmov.u8 r10, d28[6] + vmov.u8 r12, d28[7] + + orr r4, r4, r10, lsl #8 + orr r4, r4, r12, lsl #16 + + mov r1, #3 + vpush {d26} + bl generate_grain_rows_neon + vpop {d26} + + mov r1, #GRAIN_HEIGHT - 3 +1: + bl sum_\type\()_lag3_left_neon + bl sum_\type\()_lag3_mid_neon + bl sum_\type\()_lag3_mid_neon + bl sum_\type\()_lag3_mid_neon + bl sum_\type\()_lag3_right_neon + get_grain_2 d16 + subs r1, r1, #1 +.ifc \type, uv_444 + add r11, r11, #2 +.endif + vst1.16 {d16[0]}, [r0]! + bgt 1b + + vpop {q4-q7} + pop {r4-r11,pc} +endfunc +.endm + +gen_grain_82 y +gen_grain_82 uv_444 + +.macro set_height dst, type +.ifc \type, uv_420 + mov \dst, #SUB_GRAIN_HEIGHT-3 +.else + mov \dst, #GRAIN_HEIGHT-3 +.endif +.endm + +.macro increment_y_ptr reg, type +.ifc \type, uv_420 + add \reg, \reg, #2*GRAIN_WIDTH-(3*32) +.else + sub \reg, \reg, #3*32-GRAIN_WIDTH +.endif +.endm + +.macro gen_grain_44 type +function generate_grain_\type\()_8bpc_neon, export=1 + push {r4-r11,lr} + + mov r12, r3 + mov lr, #28 + add r11, r1, #3*GRAIN_WIDTH-3 + mov r1, r2 + mul r12, r12, lr + + movrel r3, X(gaussian_sequence) + ldr r2, [r1, #FGD_SEED] + ldr r9, [r1, #FGD_GRAIN_SCALE_SHIFT] + add r4, r1, #FGD_AR_COEFFS_UV + adr r5, L(gen_grain_\type\()_tbl) + ldr r6, [r1, #FGD_AR_COEFF_LAG] + add r9, r9, #4 + ldr r6, [r5, r6, lsl #2] + vdup.16 q15, r9 // 4 + data->grain_scale_shift + add r5, r5, r6 + vneg.s16 q15, q15 + + cmp r12, #0 + movw r10, #0x49d8 + movw lr, #0xb524 + // Intentionally using a separate register instead of moveq with an + // immediate constant, to avoid armv8 deprecated it instruction forms. + it eq + moveq r10, lr + add r4, r4, r12 // Add offset to ar_coeffs_uv[1] + eor r2, r2, r10 + + ldr r7, [r1, #FGD_AR_COEFF_SHIFT] + mov r8, #1 + mov r10, #1 + lsl r8, r8, r7 // 1 << ar_coeff_shift + lsl r10, r10, r9 // 1 << (4 + data->grain_scale_shift) + lsr r8, r8, #1 // 1 << (ar_coeff_shift - 1) + lsr r10, r10, #1 // 1 << (4 + data->grain_scale_shift - 1) + bx r5 + + .align 2 +L(gen_grain_\type\()_tbl): + .word L(generate_grain_\type\()_lag0) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + .word L(generate_grain_\type\()_lag1) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + .word L(generate_grain_\type\()_lag2) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + .word L(generate_grain_\type\()_lag3) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + +L(generate_grain_\type\()_lag0): +.ifc \type, uv_420 + vpush {q4-q5} +.endif + mov r1, #3 + bl generate_grain_rows_44_neon + set_height r1, \type + + vdup.16 q12, r7 + vld1.8 {d22[]}, [r4] // ar_coeffs_uv[0] + vmov.i8 q0, #0 + vmov.i8 q1, #255 + vext.8 q13, q0, q1, #13 + vext.8 q14, q1, q0, #7 + vneg.s16 q12, q12 + +1: + bl get_grain_row_44_neon +.ifc \type, uv_420 + add r12, r11, #GRAIN_WIDTH +.endif + vmov q1, q13 + vmov q0, q8 + bl add_\type\()_coeff_lag0_neon + vmov.i8 q1, #255 + vmov q0, q9 + vmov q8, q2 + bl add_\type\()_coeff_lag0_neon + vmov.i8 q1, q14 + vmov q0, q10 + vmov q9, q2 + bl add_\type\()_coeff_lag0_neon + vmov q10, q2 + subs r1, r1, #1 + increment_y_ptr r11, \type + store_grain_row_44 d16, d17, d18, d19, d20, d21 + bgt 1b + +.ifc \type, uv_420 + vpop {q4-q5} +.endif + pop {r4-r11,pc} + +L(generate_grain_\type\()_lag1): + vpush {q4-q7} + mov r5, #127 + vld1.8 {d27[]}, [r4]! // ar_coeffs_uv[0] + vld1.8 {d28[]}, [r4]! // ar_coeffs_uv[1] + vld1.8 {d29[]}, [r4] // ar_coeffs_uv[2] + add r4, r4, #2 + + mov r1, #3 + vld1.8 {d13[]}, [r4] // ar_coeffs_uv[4] + ldrsb r4, [r4, #-1] // ar_coeffs_uv[3] + bl generate_grain_rows_44_neon + + set_height r1, \type +1: + sum_\type\()_lag1 q7, q8, q8, q9, left + sum_\type\()_lag1 q8, q8, q9, q10 + sum_\type\()_lag1 q10, q9, q10, q11, right + subs r1, r1, #1 + increment_y_ptr r11, \type + store_grain_row_44 d14, d15, d16, d17, d20, d21 + vmov q9, q8 + vmov q8, q7 + bgt 1b + + vpop {q4-q7} + pop {r4-r11,pc} + +L(generate_grain_\type\()_lag2): + vpush {q4-q7} + mov r5, #127 + vld1.8 {d28,d29}, [r4] // ar_coeffs_uv[0-12] + + vmov.s8 r4, d29[2] + vmov.s8 r10, d29[3] + + mov r1, #3 + bl generate_grain_rows_44_neon + + set_height r1, \type +1: + bl sum_\type\()_lag2_left_neon + bl sum_\type\()_lag2_mid_neon + bl sum_\type\()_lag2_right_neon + subs r1, r1, #1 + increment_y_ptr r11, \type + add r0, r0, #GRAIN_WIDTH-48 + bgt 1b + + vpop {q4-q7} + pop {r4-r11,pc} + +L(generate_grain_\type\()_lag3): + vpush {q4-q7} + mov r5, #127 + vld1.8 {q13, q14}, [r4] // ar_coeffs_y[0-23], ar_coeffs_uv[0-24] + + vmov.u8 r4, d28[5] + vmov.u8 r10, d28[6] + vmov.u8 r12, d28[7] + + orr r4, r4, r10, lsl #8 + orr r4, r4, r12, lsl #16 + + mov r1, #3 + bl generate_grain_rows_44_neon + + set_height r1, \type +1: + bl sum_\type\()_lag3_left_neon + bl sum_\type\()_lag3_mid_neon + bl sum_\type\()_lag3_right_neon + subs r1, r1, #1 + increment_y_ptr r11, \type + add r0, r0, #GRAIN_WIDTH-48 + bgt 1b + + vpop {q4-q7} + pop {r4-r11,pc} +endfunc +.endm + +gen_grain_44 uv_420 +gen_grain_44 uv_422 .macro gather_interleaved dst1, dst2, src1, src2, off vmov.u8 r11, \src1[0+\off] vmov.u8 r12, \src2[0+\off] add r11, r11, r3 diff --git a/third_party/dav1d/src/arm/32/film_grain16.S b/third_party/dav1d/src/arm/32/film_grain16.S --- a/third_party/dav1d/src/arm/32/film_grain16.S +++ b/third_party/dav1d/src/arm/32/film_grain16.S @@ -28,10 +28,1198 @@ #include "src/arm/asm.S" #include "util.S" #include "src/arm/asm-offsets.h" #define GRAIN_WIDTH 82 +#define GRAIN_HEIGHT 73 + +#define SUB_GRAIN_WIDTH 44 +#define SUB_GRAIN_HEIGHT 38 + +.macro increment_seed steps, shift=1 + lsr r11, r2, #3 + lsr r12, r2, #12 + lsr lr, r2, #1 + eor r11, r2, r11 // (r >> 0) ^ (r >> 3) + eor r12, r12, lr // (r >> 12) ^ (r >> 1) + eor r11, r11, r12 // (r >> 0) ^ (r >> 3) ^ (r >> 12) ^ (r >> 1) +.if \shift + lsr r2, r2, #\steps +.endif + and r11, r11, #((1 << \steps) - 1) // bit +.if \shift + orr r2, r2, r11, lsl #(16 - \steps) // *state +.else + orr r2, r2, r11, lsl #16 // *state +.endif +.endm + +.macro read_rand dest, bits, age + ubfx \dest, r2, #16 - \bits - \age, #\bits +.endm + +.macro read_shift_rand dest, bits + ubfx \dest, r2, #17 - \bits, #\bits + lsr r2, r2, #1 +.endm + +// special calling convention: +// r2 holds seed +// r3 holds dav1d_gaussian_sequence +// clobbers r11-r12 +// returns in d0-d1 +function get_gaussian_neon + push {r5-r6,lr} + increment_seed 4 + read_rand r5, 11, 3 + read_rand r6, 11, 2 + add r5, r3, r5, lsl #1 + add r6, r3, r6, lsl #1 + vld1.16 {d0[0]}, [r5] + read_rand r5, 11, 1 + vld1.16 {d0[1]}, [r6] + add r5, r3, r5, lsl #1 + read_rand r6, 11, 0 + increment_seed 4 + add r6, r3, r6, lsl #1 + vld1.16 {d0[2]}, [r5] + read_rand r5, 11, 3 + vld1.16 {d0[3]}, [r6] + add r5, r3, r5, lsl #1 + read_rand r6, 11, 2 + vld1.16 {d1[0]}, [r5] + add r6, r3, r6, lsl #1 + read_rand r5, 11, 1 + vld1.16 {d1[1]}, [r6] + read_rand r6, 11, 0 + add r5, r3, r5, lsl #1 + add r6, r3, r6, lsl #1 + vld1.16 {d1[2]}, [r5] + vld1.16 {d1[3]}, [r6] + pop {r5-r6,pc} +endfunc + +function get_grain_2_neon + push {r11,lr} + increment_seed 2 + read_rand r11, 11, 1 + read_rand r12, 11, 0 + add r11, r3, r11, lsl #1 + add r12, r3, r12, lsl #1 + vld1.16 {d0[0]}, [r11] + vld1.16 {d0[1]}, [r12] + vrshl.s16 d0, d0, d30 + pop {r11,pc} +endfunc + +.macro get_grain_2 dst + bl get_grain_2_neon +.ifnc \dst, d0 + vmov \dst, d0 +.endif +.endm + +function get_grain_4_neon + push {r11,lr} + increment_seed 4 + read_rand r11, 11, 3 + read_rand r12, 11, 2 + add r11, r3, r11, lsl #1 + add r12, r3, r12, lsl #1 + vld1.16 {d0[0]}, [r11] + read_rand r11, 11, 1 + vld1.16 {d0[1]}, [r12] + read_rand r12, 11, 0 + add r11, r3, r11, lsl #1 + add r12, r3, r12, lsl #1 + vld1.16 {d0[2]}, [r11] + vld1.16 {d0[3]}, [r12] + vrshl.s16 d0, d0, d30 + pop {r11,pc} +endfunc + +.macro get_grain_4 dst + bl get_grain_4_neon +.ifnc \dst, d0 + vmov \dst, d0 +.endif +.endm + +// r1 holds the number of entries to produce +// r6, r8 and r10 hold the previous output entries +// q0 holds the vector of produced entries +// q1 holds the input vector of sums from above +.macro output_lag n +function output_lag\n\()_neon + push {r0, lr} +.if \n == 1 + mvn lr, r5 // grain_min = ~grain_max +.else + mov r0, #1 + mov lr, #1 + sub r7, r7, #1 + sub r9, r9, #1 + lsl r0, r0, r7 + lsl lr, lr, r9 + add r7, r7, #1 + add r9, r9, #1 +.endif +1: + read_shift_rand r12, 11 + vmov.32 r11, d2[0] + lsl r12, r12, #1 + vext.8 q0, q0, q0, #2 + ldrsh r12, [r3, r12] +.if \n == 1 + mla r11, r6, r4, r11 // sum (above) + *coeff * prev output + add r6, r11, r8 // 1 << (ar_coeff_shift - 1) + add r12, r12, r10 + asr r6, r6, r7 // >> ar_coeff_shift + asr r12, r12, r9 // >> (4 - bitdepth_min_8 + grain_scale_shift) + add r6, r6, r12 + cmp r6, r5 +.elseif \n == 2 + mla r11, r8, r4, r11 // sum (above) + *coeff * prev output 1 + mla r11, r6, r10, r11 // += *coeff * prev output 2 + mov r8, r6 + add r6, r11, r0 // 1 << (ar_coeff_shift - 1) + add r12, r12, lr // 1 << (4 - bitdepth_min_8 + grain_scale_shift - 1) + asr r6, r6, r7 // >> ar_coeff_shift + asr r12, r12, r9 // >> (4 - bitdepth_min_8 + grain_scale_shift) + add r6, r6, r12 + push {lr} + cmp r6, r5 + mvn lr, r5 // grain_min = ~grain_max +.else + push {r1-r3} + sbfx r1, r4, #0, #8 + sbfx r2, r4, #8, #8 + sbfx r3, r4, #16, #8 + mla r11, r10, r1, r11 // sum (above) + *coeff * prev output 1 + mla r11, r8, r2, r11 // sum (above) + *coeff * prev output 2 + mla r11, r6, r3, r11 // += *coeff * prev output 3 + pop {r1-r3} + mov r10, r8 + mov r8, r6 + + add r6, r11, r0 // 1 << (ar_coeff_shift - 1) + add r12, r12, lr // 1 << (4 - bitdepth_min_8 + grain_scale_shift - 1) + asr r6, r6, r7 // >> ar_coeff_shift + asr r12, r12, r9 // >> (4 - bitdepth_min_8 + grain_scale_shift) + add r6, r6, r12 + push {lr} + cmp r6, r5 + mvn lr, r5 // grain_min = ~grain_max +.endif + it gt + movgt r6, r5 + cmp r6, lr + it lt + movlt r6, lr +.if \n >= 2 + pop {lr} +.endif + subs r1, r1, #1 + vext.8 q1, q1, q1, #4 + vmov.16 d1[3], r6 + bgt 1b + pop {r0, pc} +endfunc +.endm + +output_lag 1 +output_lag 2 +output_lag 3 + + +function sum_lag1_above_neon + sub r12, r0, #1*GRAIN_WIDTH*2 - 16 + vld1.16 {q10}, [r12] // load top right + + vext.8 q0, q8, q9, #14 // top left, top mid + vext.8 q1, q9, q10, #2 // top left, top mid + + vmull.s16 q2, d18, d28 + vmlal.s16 q2, d0, d27 + vmlal.s16 q2, d2, d29 + vmull.s16 q3, d19, d28 + vmlal.s16 q3, d1, d27 + vmlal.s16 q3, d3, d29 + + vmov q8, q9 + vmov q9, q10 + + bx lr +endfunc + +.macro sum_lag_n_body lag, type, uv_layout, edge, elems, uv_coeff +.ifc \lag\()_\edge, lag3_left + bl sum_lag3_left_above_neon +.else + bl sum_\lag\()_above_neon +.endif +.ifc \type, uv_420 + vpush {q6-q7} + add r12, r11, #GRAIN_WIDTH*2 + vld1.16 {q0, q1}, [r11]! + vld1.16 {q6, q7}, [r12]! + vpadd.i16 d0, d0, d1 + vpadd.i16 d1, d2, d3 + vpadd.i16 d12, d12, d13 + vpadd.i16 d13, d14, d15 + vadd.i16 q0, q0, q6 + vpop {q6-q7} + vrshr.s16 q0, q0, #2 +.endif +.ifc \type, uv_422 + vld1.16 {q0, q1}, [r11]! + vpadd.i16 d0, d0, d1 + vpadd.i16 d1, d2, d3 + vrshr.s16 q0, q0, #1 +.endif +.ifc \type, uv_444 + vld1.16 {q0}, [r11]! +.endif +.if \uv_layout +.ifnb \uv_coeff + vdup.8 d13, \uv_coeff + vmovl.s8 q6, d13 +.endif + vmlal.s16 q2, d0, d13 + vmlal.s16 q3, d1, d13 +.endif +.if \uv_layout && \elems == 8 + b sum_\lag\()_y_\edge\()_start +.elseif \uv_layout == 444 && \elems == 7 + b sum_\lag\()_y_\edge\()_start +.elseif \uv_layout == 422 && \elems == 1 + b sum_\lag\()_uv_420_\edge\()_start +.else +sum_\lag\()_\type\()_\edge\()_start: + push {r11} +.if \elems > 4 +.ifc \edge, left + increment_seed 4 + read_rand r11, 11, 3 + read_rand r12, 11, 2 + add r11, r3, r11, lsl #1 + add r12, r3, r12, lsl #1 + vld1.16 {d1[1]}, [r11] + read_rand r11, 11, 1 + vld1.16 {d1[2]}, [r12] + add r11, r3, r11, lsl #1 + vld1.16 {d1[3]}, [r11] + lsl r2, r2, #1 // shift back the state as if we'd done increment_seed with shift=0 + vrshl.s16 d1, d1, d30 + vext.8 q2, q2, q2, #12 +.ifc \lag, lag3 + vmov.s16 r10, d1[1] +.endif +.ifnc \lag, lag1 + vmov.s16 r8, d1[2] +.endif + vmov.s16 r6, d1[3] + + vmov q1, q2 + mov r1, #1 + bl output_\lag\()_neon +.else + increment_seed 4, shift=0 + vmov q1, q2 + mov r1, #4 + bl output_\lag\()_neon +.endif + + increment_seed 4, shift=0 + vmov q1, q3 +.ifc \edge, right + mov r1, #3 + bl output_\lag\()_neon + read_shift_rand r12, 11 + add r12, r3, r12, lsl #1 + vld1.16 {d2[0]}, [r12] + vrshl.s16 d2, d2, d30 + vext.8 q0, q0, q1, #2 +.else + mov r1, #4 + bl output_\lag\()_neon +.endif +.else + // elems == 1 + increment_seed 4, shift=0 + vmov q1, q2 + mov r1, #1 + bl output_\lag\()_neon + lsr r2, r2, #3 + + read_rand r11, 11, 2 + read_rand r12, 11, 1 + add r11, r3, r11, lsl #1 + add r12, r3, r12, lsl #1 + vld1.16 {d2[0]}, [r11] + read_rand r11, 11, 0 + vld1.16 {d2[1]}, [r12] + add r11, r3, r11, lsl #1 + vld1.16 {d2[2]}, [r11] + vrshl.s16 d2, d2, d30 + vext.8 q0, q0, q1, #14 +.endif + vst1.16 {q0}, [r0]! + pop {r11} + pop {r1, pc} +.endif +.endm + +.macro sum_lag1_func type, uv_layout, edge, elems=8 +function sum_\type\()_lag1_\edge\()_neon + push {r1, lr} +.ifc \edge, left + sub r12, r0, #1*GRAIN_WIDTH*2 + vld1.8 {q9}, [r12] // load the previous block right above +.endif + sum_lag_n_body lag1, \type, \uv_layout, \edge, \elems +endfunc +.endm + +sum_lag1_func y, 0, left +sum_lag1_func y, 0, mid +sum_lag1_func y, 0, right, 7 +sum_lag1_func uv_444, 444, left +sum_lag1_func uv_444, 444, mid +sum_lag1_func uv_444, 444, right, 7 +sum_lag1_func uv_422, 422, left +sum_lag1_func uv_422, 422, mid +sum_lag1_func uv_422, 422, right, 1 +sum_lag1_func uv_420, 420, left +sum_lag1_func uv_420, 420, mid +sum_lag1_func uv_420, 420, right, 1 + + +function sum_lag2_above_neon + push {lr} + sub r12, r0, #2*GRAIN_WIDTH*2 - 16 + sub lr, r0, #1*GRAIN_WIDTH*2 - 16 + vld1.16 {q10}, [r12] // load top right + vld1.16 {q13}, [lr] + + vdup.8 d10, d28[0] + vext.8 q0, q8, q9, #12 // top left, top mid + vdup.8 d12, d28[1] + vext.8 q1, q8, q9, #14 + vdup.8 d14, d28[3] + vext.8 q4, q9, q10, #2 // top mid, top right + vmovl.s8 q5, d10 + vmovl.s8 q6, d12 + vmovl.s8 q7, d14 + + vmull.s16 q2, d0, d10 + vmlal.s16 q2, d2, d12 + vmlal.s16 q2, d8, d14 + vmull.s16 q3, d1, d10 + vmlal.s16 q3, d3, d12 + vmlal.s16 q3, d9, d14 + + vdup.8 d10, d28[4] + vext.8 q0, q9, q10, #4 // top mid, top right + vdup.8 d12, d28[5] + vext.8 q1, q11, q12, #12 // top left, top mid + vdup.8 d14, d28[6] + vext.8 q4, q11, q12, #14 + vmovl.s8 q5, d10 + vmovl.s8 q6, d12 + vmovl.s8 q7, d14 + + vmlal.s16 q2, d0, d10 + vmlal.s16 q2, d2, d12 + vmlal.s16 q2, d8, d14 + vmlal.s16 q3, d1, d10 + vmlal.s16 q3, d3, d12 + vmlal.s16 q3, d9, d14 + + vdup.8 d10, d29[0] + vext.8 q0, q12, q13, #2 // top mid, top right + vdup.8 d12, d29[1] + vext.8 q1, q12, q13, #4 + + vdup.8 d14, d28[2] + vdup.8 d8, d28[7] + + vmovl.s8 q5, d10 + vmovl.s8 q6, d12 + vmovl.s8 q7, d14 + vmovl.s8 q4, d8 + + vmlal.s16 q2, d0, d10 + vmlal.s16 q2, d2, d12 + vmlal.s16 q2, d18, d14 + vmlal.s16 q2, d24, d8 + vmlal.s16 q3, d1, d10 + vmlal.s16 q3, d3, d12 + vmlal.s16 q3, d19, d14 + vmlal.s16 q3, d25, d8 + + vmov q8, q9 + vmov q9, q10 + + vmov q11, q12 + vmov q12, q13 + + pop {pc} +endfunc + +.macro sum_lag2_func type, uv_layout, edge, elems=8 +function sum_\type\()_lag2_\edge\()_neon + push {r1, lr} +.ifc \edge, left + sub r12, r0, #2*GRAIN_WIDTH*2 + sub lr, r0, #1*GRAIN_WIDTH*2 + vld1.16 {q9}, [r12] // load the previous block right above + vld1.16 {q12}, [lr] +.endif + sum_lag_n_body lag2, \type, \uv_layout, \edge, \elems, uv_coeff=d29[4] +endfunc +.endm + +sum_lag2_func y, 0, left +sum_lag2_func y, 0, mid +sum_lag2_func y, 0, right, 7 +sum_lag2_func uv_444, 444, left +sum_lag2_func uv_444, 444, mid +sum_lag2_func uv_444, 444, right, 7 +sum_lag2_func uv_422, 422, left +sum_lag2_func uv_422, 422, mid +sum_lag2_func uv_422, 422, right, 1 +sum_lag2_func uv_420, 420, left +sum_lag2_func uv_420, 420, mid +sum_lag2_func uv_420, 420, right, 1 + + +function sum_lag3_left_above_neon + // A separate codepath for the left edge, to avoid reading outside + // of the edge of the buffer. + sub r12, r0, #3*GRAIN_WIDTH*2 + vld1.8 {q11, q12}, [r12] + vext.8 q12, q11, q12, #10 + vext.8 q11, q11, q11, #10 + b sum_lag3_above_start +endfunc + +function sum_lag3_above_neon + movw r12, #(3*GRAIN_WIDTH + 3)*2 + sub r12, r0, r12 + vld1.8 {q11, q12}, [r12] + +sum_lag3_above_start: + vdup.8 d12, d26[0] + vext.8 q1, q11, q12, #2 + vdup.8 d14, d26[1] + vext.8 q4, q11, q12, #4 + vdup.8 d16, d26[2] + vext.8 q5, q11, q12, #6 + vdup.8 d18, d26[3] + vmovl.s8 q6, d12 + vmovl.s8 q7, d14 + vmovl.s8 q8, d16 + vmovl.s8 q9, d18 + + movw r12, #(2*GRAIN_WIDTH + 3)*2 + sub r12, r0, r12 + + vmull.s16 q2, d22, d12 + vmlal.s16 q2, d2, d14 + vmlal.s16 q2, d8, d16 + vmlal.s16 q2, d10, d18 + vmull.s16 q3, d23, d12 + vmlal.s16 q3, d3, d14 + vmlal.s16 q3, d9, d16 + vmlal.s16 q3, d11, d18 + + vdup.8 d12, d26[4] + vext.8 q0, q11, q12, #8 + vdup.8 d14, d26[5] + vext.8 q1, q11, q12, #10 + vdup.8 d16, d26[6] + vext.8 q4, q11, q12, #12 + vld1.8 {q11, q12}, [r12] + vdup.8 d18, d26[7] + vmovl.s8 q6, d12 + vmovl.s8 q7, d14 + vmovl.s8 q8, d16 + vmovl.s8 q9, d18 + + vmlal.s16 q2, d0, d12 + vmlal.s16 q2, d2, d14 + vmlal.s16 q2, d8, d16 + vmlal.s16 q2, d22, d18 + vmlal.s16 q3, d1, d12 + vmlal.s16 q3, d3, d14 + vmlal.s16 q3, d9, d16 + vmlal.s16 q3, d23, d18 + + vdup.8 d12, d27[0] + vext.8 q0, q11, q12, #2 + vdup.8 d14, d27[1] + vext.8 q1, q11, q12, #4 + vdup.8 d16, d27[2] + vext.8 q4, q11, q12, #6 + vdup.8 d18, d27[3] + vext.8 q5, q11, q12, #8 + vmovl.s8 q6, d12 + vmovl.s8 q7, d14 + vmovl.s8 q8, d16 + vmovl.s8 q9, d18 + + sub r12, r0, #(1*GRAIN_WIDTH + 3)*2 + + vmlal.s16 q2, d0, d12 + vmlal.s16 q2, d2, d14 + vmlal.s16 q2, d8, d16 + vmlal.s16 q2, d10, d18 + vmlal.s16 q3, d1, d12 + vmlal.s16 q3, d3, d14 + vmlal.s16 q3, d9, d16 + vmlal.s16 q3, d11, d18 + + vdup.8 d12, d27[4] + vext.8 q0, q11, q12, #10 + vdup.8 d14, d27[5] + vext.8 q1, q11, q12, #12 + vld1.8 {q11, q12}, [r12] + vdup.8 d16, d27[6] + vdup.8 d18, d27[7] + vmovl.s8 q6, d12 + vmovl.s8 q7, d14 + vext.8 q5, q11, q12, #2 + vmovl.s8 q8, d16 + vmovl.s8 q9, d18 + + vmlal.s16 q2, d0, d12 + vmlal.s16 q2, d2, d14 + vmlal.s16 q2, d22, d16 + vmlal.s16 q2, d10, d18 + vmlal.s16 q3, d1, d12 + vmlal.s16 q3, d3, d14 + vmlal.s16 q3, d23, d16 + vmlal.s16 q3, d11, d18 + + vdup.8 d12, d28[0] + vext.8 q0, q11, q12, #4 + vdup.8 d14, d28[1] + vext.8 q1, q11, q12, #6 + vdup.8 d16, d28[2] + vext.8 q4, q11, q12, #8 + vdup.8 d18, d28[3] + vext.8 q5, q11, q12, #10 + vmovl.s8 q6, d12 + vmovl.s8 q7, d14 + vmovl.s8 q8, d16 + vmovl.s8 q9, d18 + + vmlal.s16 q2, d0, d12 + vmlal.s16 q2, d2, d14 + vmlal.s16 q2, d8, d16 + vmlal.s16 q2, d10, d18 + vmlal.s16 q3, d1, d12 + vmlal.s16 q3, d3, d14 + vmlal.s16 q3, d9, d16 + vmlal.s16 q3, d11, d18 + + vdup.8 d12, d28[4] + vext.8 q0, q11, q12, #12 + vmovl.s8 q6, d12 + + vmlal.s16 q2, d0, d12 + vmlal.s16 q3, d1, d12 + + bx lr +endfunc + +.macro sum_lag3_func type, uv_layout, edge, elems=8 +function sum_\type\()_lag3_\edge\()_neon + push {r1, lr} + sum_lag_n_body lag3, \type, \uv_layout, \edge, \elems, uv_coeff=d29[0] +endfunc +.endm + +sum_lag3_func y, 0, left +sum_lag3_func y, 0, mid +sum_lag3_func y, 0, right, 7 +sum_lag3_func uv_444, 444, left +sum_lag3_func uv_444, 444, mid +sum_lag3_func uv_444, 444, right, 7 +sum_lag3_func uv_422, 422, left +sum_lag3_func uv_422, 422, mid +sum_lag3_func uv_422, 422, right, 1 +sum_lag3_func uv_420, 420, left +sum_lag3_func uv_420, 420, mid +sum_lag3_func uv_420, 420, right, 1 + +function generate_grain_rows_neon + push {r10-r11,lr} +1: + mov r10, #80 +2: + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + subs r10, r10, #8 + vst1.16 {q0}, [r0]! + bgt 2b + get_grain_2 d0 + subs r1, r1, #1 + vst1.32 {d0[0]}, [r0]! + bgt 1b + pop {r10-r11,pc} +endfunc + +function generate_grain_rows_44_neon + push {r10-r11,lr} +1: + mov r10, #40 +2: + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 + subs r10, r10, #8 + vst1.16 {q0}, [r0]! + bgt 2b + get_grain_4 d0 + subs r1, r1, #1 + vst1.16 {d0}, [r0] + add r0, r0, #GRAIN_WIDTH*2-80 + bgt 1b + pop {r10-r11,pc} +endfunc + +function gen_grain_uv_444_lag0_neon + vld1.16 {q3}, [r11]! +gen_grain_uv_lag0_8_start: + push {r11,lr} + bl get_gaussian_neon + vrshl.s16 q0, q0, q15 +gen_grain_uv_lag0_8_add: + vand q3, q3, q1 + vmull.s16 q2, d6, d22 + vmull.s16 q3, d7, d22 + vrshl.s32 q2, q2, q12 + vrshl.s32 q3, q3, q12 + vqmovn.s32 d4, q2 + vqmovn.s32 d5, q3 + vqadd.s16 q2, q2, q0 + vmin.s16 q2, q2, q9 + vmax.s16 q2, q2, q10 + vst1.16 {q2}, [r0]! + pop {r11,pc} +endfunc + +function gen_grain_uv_420_lag0_8_neon + add r12, r11, #GRAIN_WIDTH*2 + vld1.16 {q2,q3}, [r11]! + vld1.16 {q4,q5}, [r12] + vpadd.i16 d4, d4, d5 + vpadd.i16 d5, d6, d7 + vpadd.i16 d8, d8, d9 + vpadd.i16 d9, d10, d11 + vadd.i16 q2, q2, q4 + vrshr.s16 q3, q2, #2 + b gen_grain_uv_lag0_8_start +endfunc + +function gen_grain_uv_422_lag0_8_neon + vld1.16 {q2,q3}, [r11]! + vpadd.i16 d4, d4, d5 + vpadd.i16 d5, d6, d7 + vrshr.s16 q3, q2, #1 + b gen_grain_uv_lag0_8_start +endfunc + +function gen_grain_uv_420_lag0_4_neon + add r12, r11, #GRAIN_WIDTH*2 + vld1.16 {q2}, [r11] + vld1.16 {q0}, [r12] + add r11, r11, #32 + vpadd.i16 d4, d4, d5 + vpadd.i16 d0, d0, d1 + vadd.i16 d4, d4, d0 + vrshr.s16 d6, d4, #2 + push {r11,lr} + get_grain_4 d0 + b gen_grain_uv_lag0_8_add +endfunc + +function gen_grain_uv_422_lag0_4_neon + vld1.16 {q2}, [r11] + add r11, r11, #32 + vpadd.i16 d4, d4, d5 + vrshr.s16 d6, d4, #1 + push {r11,lr} + get_grain_4 d0 + b gen_grain_uv_lag0_8_add +endfunc + +.macro gen_grain_82 type +function generate_grain_\type\()_16bpc_neon, export=1 + push {r4-r11,lr} + +.ifc \type, uv_444 + ldr r4, [sp, #36] + mov r12, r3 + mov lr, #28 + add r11, r1, #3*GRAIN_WIDTH*2 + mov r1, r2 + mul r12, r12, lr + clz lr, r4 +.else + clz lr, r2 +.endif + movrel r3, X(gaussian_sequence) + sub lr, lr, #24 // -bitdepth_min_8 + ldr r2, [r1, #FGD_SEED] + ldr r9, [r1, #FGD_GRAIN_SCALE_SHIFT] +.ifc \type, y + add r4, r1, #FGD_AR_COEFFS_Y +.else + add r4, r1, #FGD_AR_COEFFS_UV +.endif + add r9, r9, lr // grain_scale_shift - bitdepth_min_8 + adr r5, L(gen_grain_\type\()_tbl) + ldr r6, [r1, #FGD_AR_COEFF_LAG] + add r9, r9, #4 + ldr r6, [r5, r6, lsl #2] + vdup.16 q15, r9 // 4 - bitdepth_min_8 + data->grain_scale_shift + add r5, r5, r6 + vneg.s16 q15, q15 + +.ifc \type, uv_444 + push {lr} + cmp r12, #0 + movw r10, #0x49d8 + movw lr, #0xb524 + // Intentionally using a separate register instead of moveq with an + // immediate constant, to avoid armv8 deprecated it instruction forms. + it eq + moveq r10, lr + add r4, r4, r12 // Add offset to ar_coeffs_uv[1] + eor r2, r2, r10 + pop {lr} +.endif + + ldr r7, [r1, #FGD_AR_COEFF_SHIFT] + neg lr, lr // bitdepth_min_8 + mov r8, #1 + mov r10, #1 + lsl r8, r8, r7 // 1 << ar_coeff_shift + lsl r10, r10, r9 // 1 << (4 + data->grain_scale_shift) + lsr r8, r8, #1 // 1 << (ar_coeff_shift - 1) + lsr r10, r10, #1 // 1 << (4 + data->grain_scale_shift - 1) + + bx r5 + + .align 2 +L(gen_grain_\type\()_tbl): + .word L(generate_grain_\type\()_lag0) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + .word L(generate_grain_\type\()_lag1) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + .word L(generate_grain_\type\()_lag2) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + .word L(generate_grain_\type\()_lag3) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + +L(generate_grain_\type\()_lag0): +.ifc \type, y + mov r1, #GRAIN_HEIGHT + bl generate_grain_rows_neon +.else + mov r5, #128 + lsl r5, r5, lr // 128 << bitdepth_min_8 + sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 + mvn r6, r5 // grain_min = ~grain_max + + mov r1, #3 + bl generate_grain_rows_neon + mov r1, #GRAIN_HEIGHT-3 + + vdup.32 q12, r7 + vld1.8 {d22[]}, [r4] // ar_coeffs_uv[0] + vmov.i8 q0, #0 + vmov.i8 q1, #255 + vdup.16 q9, r5 + vdup.16 q10, r6 + vext.8 q13, q0, q1, #10 + vext.8 q14, q1, q0, #2 + vneg.s32 q12, q12 + vmovl.s8 q11, d22 + +1: + vmov q1, q13 + bl gen_grain_uv_444_lag0_neon // 8 + vmov.i8 q1, #255 + bl gen_grain_uv_444_lag0_neon // 16 + bl gen_grain_uv_444_lag0_neon // 24 + bl gen_grain_uv_444_lag0_neon // 32 + bl gen_grain_uv_444_lag0_neon // 40 + bl gen_grain_uv_444_lag0_neon // 48 + bl gen_grain_uv_444_lag0_neon // 56 + bl gen_grain_uv_444_lag0_neon // 64 + bl gen_grain_uv_444_lag0_neon // 72 + vmov q1, q14 + bl gen_grain_uv_444_lag0_neon // 80 + get_grain_2 d16 + subs r1, r1, #1 + add r11, r11, #4 + vst1.32 {d16[0]}, [r0]! + bgt 1b +.endif + pop {r4-r11,pc} + +L(generate_grain_\type\()_lag1): + vpush {q4-q7} + mov r5, #128 + lsl r5, r5, lr // 128 << bitdepth_min_8 + sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 + vld1.8 {d27[]}, [r4]! // ar_coeffs_y[0] + vld1.8 {d28[]}, [r4]! // ar_coeffs_y[1] + vld1.8 {d29[]}, [r4] // ar_coeffs_y[2] +.ifc \type, y + ldrsb r4, [r4, #1] // ar_coeffs_y[3] +.else + add r4, r4, #2 +.endif + + mov r1, #3 +.ifc \type, uv_444 + vld1.8 {d13[]}, [r4] // ar_coeffs_uv[4] + ldrsb r4, [r4, #-1] // ar_coeffs_uv[3] +.endif + bl generate_grain_rows_neon + vmovl.s8 q13, d27 + vmovl.s8 q12, d29 + vmovl.s8 q14, d28 + vmov d29, d24 +.ifc \type, uv_444 + vmovl.s8 q6, d13 +.endif + + mov r1, #GRAIN_HEIGHT - 3 +1: + bl sum_\type\()_lag1_left_neon // 8 + bl sum_\type\()_lag1_mid_neon // 16 + bl sum_\type\()_lag1_mid_neon // 24 + bl sum_\type\()_lag1_mid_neon // 32 + bl sum_\type\()_lag1_mid_neon // 40 + bl sum_\type\()_lag1_mid_neon // 48 + bl sum_\type\()_lag1_mid_neon // 56 + bl sum_\type\()_lag1_mid_neon // 64 + bl sum_\type\()_lag1_mid_neon // 72 + bl sum_\type\()_lag1_right_neon // 80 + get_grain_2 d16 + subs r1, r1, #1 +.ifc \type, uv_444 + add r11, r11, #4 +.endif + vst1.32 {d16[0]}, [r0]! + bgt 1b + + vpop {q4-q7} + pop {r4-r11,pc} + +L(generate_grain_\type\()_lag2): + vpush {q4-q7} + mov r5, #128 + lsl r5, r5, lr // 128 << bitdepth_min_8 + sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 + vld1.8 {d28,d29}, [r4] // ar_coeffs_y[0-11], ar_coeffs_uv[0-12] + + vmov.s8 r4, d29[2] + vmov.s8 r10, d29[3] + + mov r1, #3 + bl generate_grain_rows_neon + + mov r1, #GRAIN_HEIGHT - 3 +1: + bl sum_\type\()_lag2_left_neon // 8 + bl sum_\type\()_lag2_mid_neon // 16 + bl sum_\type\()_lag2_mid_neon // 24 + bl sum_\type\()_lag2_mid_neon // 32 + bl sum_\type\()_lag2_mid_neon // 40 + bl sum_\type\()_lag2_mid_neon // 48 + bl sum_\type\()_lag2_mid_neon // 56 + bl sum_\type\()_lag2_mid_neon // 64 + bl sum_\type\()_lag2_mid_neon // 72 + bl sum_\type\()_lag2_right_neon // 80 + get_grain_2 d16 + subs r1, r1, #1 +.ifc \type, uv_444 + add r11, r11, #4 +.endif + vst1.32 {d16[0]}, [r0]! + bgt 1b + + vpop {q4-q7} + pop {r4-r11,pc} + +L(generate_grain_\type\()_lag3): + vpush {q4-q7} + mov r5, #128 + lsl r5, r5, lr // 128 << bitdepth_min_8 + sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 + vld1.8 {q13, q14}, [r4] // ar_coeffs_y[0-23], ar_coeffs_uv[0-24] + + vmov.u8 r4, d28[5] + vmov.u8 r10, d28[6] + vmov.u8 r12, d28[7] + + orr r4, r4, r10, lsl #8 + orr r4, r4, r12, lsl #16 + + mov r1, #3 + vpush {d26} + bl generate_grain_rows_neon + vpop {d26} + + mov r1, #GRAIN_HEIGHT - 3 +1: + bl sum_\type\()_lag3_left_neon // 8 + bl sum_\type\()_lag3_mid_neon // 16 + bl sum_\type\()_lag3_mid_neon // 24 + bl sum_\type\()_lag3_mid_neon // 32 + bl sum_\type\()_lag3_mid_neon // 40 + bl sum_\type\()_lag3_mid_neon // 48 + bl sum_\type\()_lag3_mid_neon // 56 + bl sum_\type\()_lag3_mid_neon // 64 + bl sum_\type\()_lag3_mid_neon // 72 + bl sum_\type\()_lag3_right_neon // 80 + get_grain_2 d16 + subs r1, r1, #1 +.ifc \type, uv_444 + add r11, r11, #4 +.endif + vst1.32 {d16[0]}, [r0]! + bgt 1b + + vpop {q4-q7} + pop {r4-r11,pc} +endfunc +.endm + +gen_grain_82 y +gen_grain_82 uv_444 + +.macro set_height dst, type +.ifc \type, uv_420 + mov \dst, #SUB_GRAIN_HEIGHT-3 +.else + mov \dst, #GRAIN_HEIGHT-3 +.endif +.endm + +.macro increment_y_ptr reg, type +.ifc \type, uv_420 + add \reg, \reg, #2*GRAIN_WIDTH*2-(6*32) +.else + sub \reg, \reg, #6*32-GRAIN_WIDTH*2 +.endif +.endm + +.macro gen_grain_44 type +function generate_grain_\type\()_16bpc_neon, export=1 + push {r4-r11,lr} + + ldr r4, [sp, #36] + mov r12, r3 + movw r11, #(3*GRAIN_WIDTH-3)*2 + mov lr, #28 + add r11, r1, r11 + mov r1, r2 + mul r12, r12, lr + clz lr, r4 + + movrel r3, X(gaussian_sequence) + sub lr, lr, #24 // -bitdepth_min_8 + ldr r2, [r1, #FGD_SEED] + ldr r9, [r1, #FGD_GRAIN_SCALE_SHIFT] + add r4, r1, #FGD_AR_COEFFS_UV + add r9, r9, lr // grain_scale_shift - bitdepth_min_8 + adr r5, L(gen_grain_\type\()_tbl) + ldr r6, [r1, #FGD_AR_COEFF_LAG] + add r9, r9, #4 + ldr r6, [r5, r6, lsl #2] + vdup.16 q15, r9 // 4 - bitdepth_min_8 + data->grain_scale_shift + add r5, r5, r6 + vneg.s16 q15, q15 + + push {lr} + cmp r12, #0 + movw r10, #0x49d8 + movw lr, #0xb524 + // Intentionally using a separate register instead of moveq with an + // immediate constant, to avoid armv8 deprecated it instruction forms. + it eq + moveq r10, lr + add r4, r4, r12 // Add offset to ar_coeffs_uv[1] + eor r2, r2, r10 + pop {lr} + + ldr r7, [r1, #FGD_AR_COEFF_SHIFT] + neg lr, lr + mov r8, #1 + mov r10, #1 + lsl r8, r8, r7 // 1 << ar_coeff_shift + lsl r10, r10, r9 // 1 << (4 + data->grain_scale_shift) + lsr r8, r8, #1 // 1 << (ar_coeff_shift - 1) + lsr r10, r10, #1 // 1 << (4 + data->grain_scale_shift - 1) + bx r5 + + .align 2 +L(gen_grain_\type\()_tbl): + .word L(generate_grain_\type\()_lag0) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + .word L(generate_grain_\type\()_lag1) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + .word L(generate_grain_\type\()_lag2) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + .word L(generate_grain_\type\()_lag3) - L(gen_grain_\type\()_tbl) + CONFIG_THUMB + +L(generate_grain_\type\()_lag0): +.ifc \type, uv_420 + vpush {q4-q5} +.endif + mov r5, #128 + lsl r5, r5, lr // 128 << bitdepth_min_8 + sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 + mvn r6, r5 // grain_min = ~grain_max + + mov r1, #3 + bl generate_grain_rows_44_neon + set_height r1, \type + + vdup.32 q12, r7 + vld1.8 {d22[]}, [r4] // ar_coeffs_uv[0] + vmov.i8 q0, #0 + vmov.i8 q1, #255 + vdup.16 q9, r5 + vdup.16 q10, r6 + vext.8 q13, q0, q1, #10 + vext.8 q14, q1, q0, #14 + vneg.s32 q12, q12 + vmovl.s8 q11, d22 + +1: + vmov q1, q13 + bl gen_grain_\type\()_lag0_8_neon // 8 + vmov.i8 q1, #255 + bl gen_grain_\type\()_lag0_8_neon // 16 + bl gen_grain_\type\()_lag0_8_neon // 24 + bl gen_grain_\type\()_lag0_8_neon // 32 + bl gen_grain_\type\()_lag0_8_neon // 40 + vmov q1, q14 + bl gen_grain_\type\()_lag0_4_neon // 44 + subs r1, r1, #1 + increment_y_ptr r11, \type + add r0, r0, #GRAIN_WIDTH*2-6*16 + bgt 1b + +.ifc \type, uv_420 + vpop {q4-q5} +.endif + pop {r4-r11,pc} + +L(generate_grain_\type\()_lag1): + vpush {q4-q7} + mov r5, #128 + lsl r5, r5, lr // 128 << bitdepth_min_8 + sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 + vld1.8 {d27[]}, [r4]! // ar_coeffs_uv[0] + vld1.8 {d28[]}, [r4]! // ar_coeffs_uv[1] + vld1.8 {d29[]}, [r4] // ar_coeffs_uv[2] + add r4, r4, #2 + + mov r1, #3 + vld1.8 {d13[]}, [r4] // ar_coeffs_uv[4] + ldrsb r4, [r4, #-1] // ar_coeffs_uv[3] + bl generate_grain_rows_44_neon + vmovl.s8 q13, d27 + vmovl.s8 q12, d29 + vmovl.s8 q14, d28 + vmov d29, d24 + vmovl.s8 q6, d13 + + set_height r1, \type +1: + bl sum_\type\()_lag1_left_neon // 8 + bl sum_\type\()_lag1_mid_neon // 16 + bl sum_\type\()_lag1_mid_neon // 24 + bl sum_\type\()_lag1_mid_neon // 32 + bl sum_\type\()_lag1_mid_neon // 40 + bl sum_\type\()_lag1_right_neon // 44 + subs r1, r1, #1 + increment_y_ptr r11, \type + add r0, r0, #GRAIN_WIDTH*2-6*16 + bgt 1b + + vpop {q4-q7} + pop {r4-r11,pc} + +L(generate_grain_\type\()_lag2): + vpush {q4-q7} + mov r5, #128 + lsl r5, r5, lr // 128 << bitdepth_min_8 + sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 + vld1.8 {d28,d29}, [r4] // ar_coeffs_uv[0-12] + + vmov.s8 r4, d29[2] + vmov.s8 r10, d29[3] + + mov r1, #3 + bl generate_grain_rows_44_neon + + set_height r1, \type +1: + bl sum_\type\()_lag2_left_neon // 8 + bl sum_\type\()_lag2_mid_neon // 16 + bl sum_\type\()_lag2_mid_neon // 24 + bl sum_\type\()_lag2_mid_neon // 32 + bl sum_\type\()_lag2_mid_neon // 40 + bl sum_\type\()_lag2_right_neon // 44 + subs r1, r1, #1 + increment_y_ptr r11, \type + add r0, r0, #GRAIN_WIDTH*2-6*16 + bgt 1b + + vpop {q4-q7} + pop {r4-r11,pc} + +L(generate_grain_\type\()_lag3): + vpush {q4-q7} + mov r5, #128 + lsl r5, r5, lr // 128 << bitdepth_min_8 + sub r5, r5, #1 // (128 << bitdepth_min_8) - 1 + vld1.8 {q13, q14}, [r4] // ar_coeffs_y[0-23], ar_coeffs_uv[0-24] + + vmov.u8 r4, d28[5] + vmov.u8 r10, d28[6] + vmov.u8 r12, d28[7] + + orr r4, r4, r10, lsl #8 + orr r4, r4, r12, lsl #16 + + mov r1, #3 + bl generate_grain_rows_44_neon + + set_height r1, \type +1: + bl sum_\type\()_lag3_left_neon // 8 + bl sum_\type\()_lag3_mid_neon // 16 + bl sum_\type\()_lag3_mid_neon // 24 + bl sum_\type\()_lag3_mid_neon // 32 + bl sum_\type\()_lag3_mid_neon // 40 + bl sum_\type\()_lag3_right_neon // 44 + subs r1, r1, #1 + increment_y_ptr r11, \type + add r0, r0, #GRAIN_WIDTH*2-6*16 + bgt 1b + + vpop {q4-q7} + pop {r4-r11,pc} +endfunc +.endm + +gen_grain_44 uv_420 +gen_grain_44 uv_422 .macro gather_interleaved dst1, dst2, src1, src2, src3, src4, off vmov.u16 r11, \src1[0+\off] vmov.u16 r12, \src3[0+\off] add r11, r11, r3 diff --git a/third_party/dav1d/src/arm/32/refmvs.S b/third_party/dav1d/src/arm/32/refmvs.S new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/arm/32/refmvs.S @@ -0,0 +1,97 @@ +/* + * Copyright © 2021, VideoLAN and dav1d authors + * Copyright © 2021, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +// void dav1d_splat_mv_neon(refmvs_block **rr, const refmvs_block *rmv, +// int bx4, int bw4, int bh4) + +function splat_mv_neon, export=1 + push {r4, lr} + vld1.8 {q3}, [r1] + ldr r4, [sp, #8] + clz r3, r3 + adr lr, L(splat_tbl) + sub r3, r3, #26 + vext.8 q2, q3, q3, #12 + ldr r3, [lr, r3, lsl #2] + add r2, r2, r2, lsl #1 + vext.8 q0, q2, q3, #4 + add r3, lr, r3 + vext.8 q1, q2, q3, #8 + lsl r2, r2, #2 + vext.8 q2, q2, q3, #12 + vmov q3, q0 +1: + ldr r1, [r0], #4 + subs r4, r4, #1 + add r1, r1, r2 + bx r3 + + .align 2 +L(splat_tbl): + .word 320f - L(splat_tbl) + CONFIG_THUMB + .word 160f - L(splat_tbl) + CONFIG_THUMB + .word 80f - L(splat_tbl) + CONFIG_THUMB + .word 40f - L(splat_tbl) + CONFIG_THUMB + .word 20f - L(splat_tbl) + CONFIG_THUMB + .word 10f - L(splat_tbl) + CONFIG_THUMB + +10: + vst1.8 {d0}, [r1] + vstr s2, [r1, #8] + bgt 1b + pop {r4, pc} +20: + vst1.8 {q0}, [r1] + vstr d2, [r1, #16] + bgt 1b + pop {r4, pc} +40: + vst1.8 {q0, q1}, [r1]! + vst1.8 {q2}, [r1] + bgt 1b + pop {r4, pc} +320: + vst1.8 {q0, q1}, [r1]! + vst1.8 {q2, q3}, [r1]! + vst1.8 {q1, q2}, [r1]! + vst1.8 {q0, q1}, [r1]! + vst1.8 {q2, q3}, [r1]! + vst1.8 {q1, q2}, [r1]! +160: + vst1.8 {q0, q1}, [r1]! + vst1.8 {q2, q3}, [r1]! + vst1.8 {q1, q2}, [r1]! +80: + vst1.8 {q0, q1}, [r1]! + vst1.8 {q2, q3}, [r1]! + vst1.8 {q1, q2}, [r1] + bgt 1b + pop {r4, pc} +endfunc diff --git a/third_party/dav1d/src/arm/64/film_grain.S b/third_party/dav1d/src/arm/64/film_grain.S --- a/third_party/dav1d/src/arm/64/film_grain.S +++ b/third_party/dav1d/src/arm/64/film_grain.S @@ -184,36 +184,57 @@ st1 {\r0\().16b,\r1\().16b}, [x0], #32 st1 {\r2\().16b}, [x0] add x0, x0, #GRAIN_WIDTH-32 .endm -.macro get_grain_2 dst +function get_grain_2_neon increment_seed 2 read_rand x14, 11, 1 read_rand x15, 11, 0 add x14, x3, x14, lsl #1 add x15, x3, x15, lsl #1 - ld1 {\dst\().h}[0], [x14] - ld1 {\dst\().h}[1], [x15] - srshl v0.4h, \dst\().4h, v31.4h - xtn \dst\().8b, v0.8h + ld1 {v0.h}[0], [x14] + ld1 {v0.h}[1], [x15] + srshl v0.4h, v0.4h, v31.4h + xtn v0.8b, v0.8h + ret +endfunc + +.macro get_grain_2 dst + bl get_grain_2_neon +.ifnc \dst, v0 + mov \dst\().8b, v0.8b +.endif .endm // w15 holds the number of entries to produce -// w14 holds the previous output entry +// w14, w16 and w17 hold the previous output entries // v0 holds the vector of produced entries // v1 holds the input vector of sums from above -function output_lag1_neon +.macro output_lag n +function output_lag\n\()_neon 1: read_shift_rand x13, 11 mov w11, v1.s[0] ldrsh w12, [x3, x13, lsl #1] ext v0.16b, v0.16b, v0.16b, #1 - madd w14, w14, w4, w11 // sum (above) + *coeff * prev output - add w14, w14, w8 // 1 << (ar_coeff_shift - 1) +.if \n == 1 + madd w11, w14, w4, w11 // sum (above) + *coeff * prev output +.elseif \n == 2 + madd w11, w16, w4, w11 // sum (above) + *coeff * prev output 1 + madd w11, w14, w17, w11 // += *coeff * prev output 2 + mov w16, w14 +.else + madd w11, w17, w4, w11 // sum (above) + *coeff * prev output 1 + madd w11, w16, w20, w11 // sum (above) + *coeff * prev output 2 + madd w11, w14, w21, w11 // += *coeff * prev output 3 + mov w17, w16 + mov w16, w14 +.endif + add w14, w11, w8 // 1 << (ar_coeff_shift - 1) + add w12, w12, w10 // 1 << (4 + grain_scale_shift - 1) asr w14, w14, w7 // >> ar_coeff_shift - add w12, w12, w10 asr w12, w12, w9 // >> (4 + grain_scale_shift) add w14, w14, w12 cmp w14, w5 csel w14, w14, w5, le cmp w14, w6 @@ -222,10 +243,16 @@ ext v1.16b, v1.16b, v1.16b, #4 ins v0.b[15], w14 b.gt 1b ret endfunc +.endm + +output_lag 1 +output_lag 2 +output_lag 3 + function sum_lag1_above_neon smull v2.8h, v3.8b, v28.8b smull2 v3.8h, v3.16b, v28.16b smull v4.8h, v0.8b, v27.8b @@ -241,53 +268,57 @@ saddw v6.4s, v2.4s, v7.4h saddw2 v7.4s, v3.4s, v7.8h ret endfunc -.macro sum_lag1_func type, uv_layout, edge, elems=16 -function sum_\type\()_lag1_\edge\()_neon - str x30, [sp, #-16]! - bl sum_lag1_above_neon +.macro sum_lag_n_body lag, type, uv_layout, edge, elems, store, uv_coeff + bl sum_\lag\()_above_neon .ifc \type, uv_420 add x12, x19, #GRAIN_WIDTH ld1 {v22.16b, v23.16b}, [x19], #32 ld1 {v24.16b, v25.16b}, [x12] saddlp v22.8h, v22.16b saddlp v23.8h, v23.16b saddlp v24.8h, v24.16b saddlp v25.8h, v25.16b add v22.8h, v22.8h, v24.8h add v23.8h, v23.8h, v25.8h - rshrn v0.8b, v22.8h, #2 - rshrn2 v0.16b, v23.8h, #2 + rshrn v0.8b, v22.8h, #2 + rshrn2 v0.16b, v23.8h, #2 .endif .ifc \type, uv_422 ld1 {v22.16b, v23.16b}, [x19], #32 saddlp v22.8h, v22.16b saddlp v23.8h, v23.16b - rshrn v0.8b, v22.8h, #1 - rshrn2 v0.16b, v23.8h, #1 + rshrn v0.8b, v22.8h, #1 + rshrn2 v0.16b, v23.8h, #1 .endif .ifc \type, uv_444 ld1 {v0.16b}, [x19], #16 .endif .if \uv_layout +.ifnb \uv_coeff + dup v1.16b, \uv_coeff + smull v2.8h, v0.8b, v1.8b + smull2 v3.8h, v0.16b, v1.16b +.else smull v2.8h, v0.8b, v30.8b smull2 v3.8h, v0.16b, v30.16b +.endif saddw v4.4s, v4.4s, v2.4h saddw2 v5.4s, v5.4s, v2.8h saddw v6.4s, v6.4s, v3.4h saddw2 v7.4s, v7.4s, v3.8h .endif .if \uv_layout && \elems == 16 - b sum_lag1_y_\edge\()_start + b sum_\lag\()_y_\edge\()_start .elseif \uv_layout == 444 && \elems == 15 - b sum_lag1_y_\edge\()_start + b sum_\lag\()_y_\edge\()_start .elseif \uv_layout == 422 && \elems == 9 - b sum_lag1_uv_420_\edge\()_start + b sum_\lag\()_uv_420_\edge\()_start .else -sum_lag1_\type\()_\edge\()_start: +sum_\lag\()_\type\()_\edge\()_start: .ifc \edge, left increment_seed 4 read_rand x12, 11, 3 read_rand x13, 11, 2 read_rand x14, 11, 1 @@ -299,32 +330,38 @@ ld1 {v0.h}[7], [x14] lsl x2, x2, #1 // shift back the state as if we'd done increment_seed with shift=0 srshl v0.8h, v0.8h, v31.8h xtn2 v0.16b, v0.8h ext v4.16b, v4.16b, v4.16b, #12 +.ifc \lag, lag3 + smov w17, v0.b[13] +.endif +.ifnc \lag, lag1 + smov w16, v0.b[14] +.endif smov w14, v0.b[15] mov v1.16b, v4.16b mov w15, #1 - bl output_lag1_neon + bl output_\lag\()_neon .else increment_seed 4, shift=0 mov v1.16b, v4.16b mov w15, #4 - bl output_lag1_neon + bl output_\lag\()_neon .endif increment_seed 4, shift=0 mov v1.16b, v5.16b mov w15, #4 - bl output_lag1_neon + bl output_\lag\()_neon increment_seed 4, shift=0 mov v1.16b, v6.16b .if \elems == 9 mov w15, #1 - bl output_lag1_neon + bl output_\lag\()_neon lsr w2, w2, #3 read_rand x12, 11, 2 read_rand x13, 11, 1 read_rand x14, 11, 0 @@ -337,31 +374,40 @@ srshl v1.4h, v1.4h, v31.4h xtn v1.8b, v1.8h ext v0.16b, v0.16b, v1.16b, #7 .else mov w15, #4 - bl output_lag1_neon + bl output_\lag\()_neon increment_seed 4, shift=0 mov v1.16b, v7.16b .ifc \edge, right mov w15, #3 - bl output_lag1_neon + bl output_\lag\()_neon read_shift_rand x15, 11 add x15, x3, x15, lsl #1 ld1 {v1.h}[0], [x15] srshl v1.4h, v1.4h, v31.4h ext v0.16b, v0.16b, v1.16b, #1 .else mov w15, #4 - bl output_lag1_neon + bl output_\lag\()_neon +.endif .endif +.if \store + st1 {v0.16b}, [x0], #16 .endif ldr x30, [sp], #16 ret .endif +.endm + +.macro sum_lag1_func type, uv_layout, edge, elems=16 +function sum_\type\()_lag1_\edge\()_neon + str x30, [sp, #-16]! + sum_lag_n_body lag1, \type, \uv_layout, \edge, \elems, store=0 endfunc .endm sum_lag1_func y, 0, left sum_lag1_func y, 0, mid @@ -398,38 +444,10 @@ .macro sum_uv_420_lag1 dst, left, mid, right, edge=mid sum_lag1 uv_420, \dst, \left, \mid, \right, \edge .endm -// w15 holds the number of entries to produce -// w14 and w16 hold the previous output entries -// v0 holds the vector of produced entries -// v1 holds the input vector of sums from above -function output_lag2_neon -1: - read_shift_rand x13, 11 - mov w11, v1.s[0] - ldrsh w12, [x3, x13, lsl #1] - ext v0.16b, v0.16b, v0.16b, #1 - madd w11, w16, w4, w11 // sum (above) + *coeff * prev output 1 - madd w11, w14, w17, w11 // += *coeff * prev output 2 - mov w16, w14 - add w14, w11, w8 // 1 << (ar_coeff_shift - 1) - asr w14, w14, w7 // >> ar_coeff_shift - add w12, w12, w10 - asr w12, w12, w9 // >> (4 + grain_scale_shift) - add w14, w14, w12 - cmp w14, w5 - csel w14, w14, w5, le - cmp w14, w6 - csel w14, w14, w6, ge - subs w15, w15, #1 - ext v1.16b, v1.16b, v1.16b, #4 - ins v0.b[15], w14 - b.gt 1b - ret -endfunc function sum_lag2_above_neon sub x12, x0, #2*GRAIN_WIDTH - 16 sub x13, x0, #1*GRAIN_WIDTH - 16 ld1 {v18.16b}, [x12] // load top right @@ -528,127 +546,11 @@ sub x12, x0, #2*GRAIN_WIDTH sub x13, x0, #1*GRAIN_WIDTH ld1 {v17.16b}, [x12] // load the previous block right above ld1 {v20.16b}, [x13] .endif - bl sum_lag2_above_neon -.ifc \type, uv_420 - add x12, x19, #GRAIN_WIDTH - ld1 {v22.16b, v23.16b}, [x19], #32 - ld1 {v24.16b, v25.16b}, [x12] - saddlp v22.8h, v22.16b - saddlp v23.8h, v23.16b - saddlp v24.8h, v24.16b - saddlp v25.8h, v25.16b - add v22.8h, v22.8h, v24.8h - add v23.8h, v23.8h, v25.8h - rshrn v0.8b, v22.8h, #2 - rshrn2 v0.16b, v23.8h, #2 -.endif -.ifc \type, uv_422 - ld1 {v22.16b, v23.16b}, [x19], #32 - saddlp v22.8h, v22.16b - saddlp v23.8h, v23.16b - rshrn v0.8b, v22.8h, #1 - rshrn2 v0.16b, v23.8h, #1 -.endif -.ifc \type, uv_444 - ld1 {v0.16b}, [x19], #16 -.endif -.if \uv_layout - dup v1.16b, v30.b[12] - smull v2.8h, v0.8b, v1.8b - smull2 v3.8h, v0.16b, v1.16b - saddw v4.4s, v4.4s, v2.4h - saddw2 v5.4s, v5.4s, v2.8h - saddw v6.4s, v6.4s, v3.4h - saddw2 v7.4s, v7.4s, v3.8h -.endif -.if \uv_layout && \elems == 16 - b sum_lag2_y_\edge\()_start -.elseif \uv_layout == 444 && \elems == 15 - b sum_lag2_y_\edge\()_start -.elseif \uv_layout == 422 && \elems == 9 - b sum_lag2_uv_420_\edge\()_start -.else -sum_lag2_\type\()_\edge\()_start: -.ifc \edge, left - increment_seed 4 - read_rand x12, 11, 3 - read_rand x13, 11, 2 - read_rand x14, 11, 1 - add x12, x3, x12, lsl #1 - add x13, x3, x13, lsl #1 - add x14, x3, x14, lsl #1 - ld1 {v0.h}[5], [x12] - ld1 {v0.h}[6], [x13] - ld1 {v0.h}[7], [x14] - lsl x2, x2, #1 // shift back the state as if we'd done increment_seed with shift=0 - srshl v0.8h, v0.8h, v31.8h - xtn2 v0.16b, v0.8h - ext v4.16b, v4.16b, v4.16b, #12 - smov w16, v0.b[14] - smov w14, v0.b[15] - - mov v1.16b, v4.16b - mov w15, #1 - bl output_lag2_neon -.else - increment_seed 4, shift=0 - mov v1.16b, v4.16b - mov w15, #4 - bl output_lag2_neon -.endif - - increment_seed 4, shift=0 - mov v1.16b, v5.16b - mov w15, #4 - bl output_lag2_neon - - increment_seed 4, shift=0 - mov v1.16b, v6.16b -.if \elems == 9 - mov w15, #1 - bl output_lag2_neon - lsr w2, w2, #3 - - read_rand x12, 11, 2 - read_rand x13, 11, 1 - read_rand x14, 11, 0 - add x12, x3, x12, lsl #1 - add x13, x3, x13, lsl #1 - add x14, x3, x14, lsl #1 - ld1 {v1.h}[0], [x12] - ld1 {v1.h}[1], [x13] - ld1 {v1.h}[2], [x14] - srshl v1.4h, v1.4h, v31.4h - xtn v1.8b, v1.8h - ext v0.16b, v0.16b, v1.16b, #7 -.else - mov w15, #4 - bl output_lag2_neon - - increment_seed 4, shift=0 - mov v1.16b, v7.16b - -.ifc \edge, right - mov w15, #3 - bl output_lag2_neon - read_shift_rand x15, 11 - add x15, x3, x15, lsl #1 - ld1 {v1.h}[0], [x15] - srshl v1.4h, v1.4h, v31.4h - ext v0.16b, v0.16b, v1.16b, #1 -.else - mov w15, #4 - bl output_lag2_neon -.endif -.endif - st1 {v0.16b}, [x0], #16 - ldr x30, [sp], #16 - ret -.endif + sum_lag_n_body lag2, \type, \uv_layout, \edge, \elems, store=1, uv_coeff=v30.b[12] endfunc .endm sum_lag2_func y, 0, left sum_lag2_func y, 0, mid @@ -662,41 +564,10 @@ sum_lag2_func uv_420, 420, left sum_lag2_func uv_420, 420, mid sum_lag2_func uv_420, 420, right, 9 -// w15 holds the number of entries to produce -// w14, w16 and w17 hold the previous output entries -// v0 holds the vector of produced entries -// v1 holds the input vector of sums from above -function output_lag3_neon -1: - read_shift_rand x13, 11 - mov w11, v1.s[0] - ldrsh w12, [x3, x13, lsl #1] - ext v0.16b, v0.16b, v0.16b, #1 - madd w11, w17, w4, w11 // sum (above) + *coeff * prev output 1 - madd w11, w16, w20, w11 // sum (above) + *coeff * prev output 2 - madd w11, w14, w21, w11 // += *coeff * prev output 3 - mov w17, w16 - mov w16, w14 - add w14, w11, w8 // 1 << (ar_coeff_shift - 1) - asr w14, w14, w7 // >> ar_coeff_shift - add w12, w12, w10 - asr w12, w12, w9 // >> (4 + grain_scale_shift) - add w14, w14, w12 - cmp w14, w5 - csel w14, w14, w5, le - cmp w14, w6 - csel w14, w14, w6, ge - subs w15, w15, #1 - ext v1.16b, v1.16b, v1.16b, #4 - ins v0.b[15], w14 - b.gt 1b - ret -endfunc - function sum_lag3_above_neon sub x11, x0, #3*GRAIN_WIDTH - 16 sub x12, x0, #2*GRAIN_WIDTH - 16 sub x13, x0, #1*GRAIN_WIDTH - 16 ld1 {v15.16b}, [x11] // load top right @@ -888,128 +759,11 @@ sub x13, x0, #1*GRAIN_WIDTH ld1 {v14.16b}, [x11] // load the previous block right above ld1 {v17.16b}, [x12] ld1 {v20.16b}, [x13] .endif - bl sum_lag3_above_neon -.ifc \type, uv_420 - add x12, x19, #GRAIN_WIDTH - ld1 {v22.16b, v23.16b}, [x19], #32 - ld1 {v24.16b, v25.16b}, [x12] - saddlp v22.8h, v22.16b - saddlp v23.8h, v23.16b - saddlp v24.8h, v24.16b - saddlp v25.8h, v25.16b - add v22.8h, v22.8h, v24.8h - add v23.8h, v23.8h, v25.8h - rshrn v0.8b, v22.8h, #2 - rshrn2 v0.16b, v23.8h, #2 -.endif -.ifc \type, uv_422 - ld1 {v22.16b, v23.16b}, [x19], #32 - saddlp v22.8h, v22.16b - saddlp v23.8h, v23.16b - rshrn v0.8b, v22.8h, #1 - rshrn2 v0.16b, v23.8h, #1 -.endif -.ifc \type, uv_444 - ld1 {v0.16b}, [x19], #16 -.endif -.if \uv_layout - dup v1.16b, v30.b[8] - smull v2.8h, v0.8b, v1.8b - smull2 v3.8h, v0.16b, v1.16b - saddw v4.4s, v4.4s, v2.4h - saddw2 v5.4s, v5.4s, v2.8h - saddw v6.4s, v6.4s, v3.4h - saddw2 v7.4s, v7.4s, v3.8h -.endif -.if \uv_layout && \elems == 16 - b sum_lag3_y_\edge\()_start -.elseif \uv_layout == 444 && \elems == 15 - b sum_lag3_y_\edge\()_start -.elseif \uv_layout == 422 && \elems == 9 - b sum_lag3_uv_420_\edge\()_start -.else -sum_lag3_\type\()_\edge\()_start: -.ifc \edge, left - increment_seed 4 - read_rand x12, 11, 3 - read_rand x13, 11, 2 - read_rand x14, 11, 1 - add x12, x3, x12, lsl #1 - add x13, x3, x13, lsl #1 - add x14, x3, x14, lsl #1 - ld1 {v0.h}[5], [x12] - ld1 {v0.h}[6], [x13] - ld1 {v0.h}[7], [x14] - lsl x2, x2, #1 // shift back the state as if we'd done increment_seed with shift=0 - srshl v0.8h, v0.8h, v31.8h - xtn2 v0.16b, v0.8h - ext v4.16b, v4.16b, v4.16b, #12 - smov w17, v0.b[13] - smov w16, v0.b[14] - smov w14, v0.b[15] - - mov v1.16b, v4.16b - mov w15, #1 - bl output_lag3_neon -.else - increment_seed 4, shift=0 - mov v1.16b, v4.16b - mov w15, #4 - bl output_lag3_neon -.endif - - increment_seed 4, shift=0 - mov v1.16b, v5.16b - mov w15, #4 - bl output_lag3_neon - - increment_seed 4, shift=0 - mov v1.16b, v6.16b -.if \elems == 9 - mov w15, #1 - bl output_lag3_neon - lsr w2, w2, #3 - - read_rand x12, 11, 2 - read_rand x13, 11, 1 - read_rand x14, 11, 0 - add x12, x3, x12, lsl #1 - add x13, x3, x13, lsl #1 - add x14, x3, x14, lsl #1 - ld1 {v1.h}[0], [x12] - ld1 {v1.h}[1], [x13] - ld1 {v1.h}[2], [x14] - srshl v1.4h, v1.4h, v31.4h - xtn v1.8b, v1.8h - ext v0.16b, v0.16b, v1.16b, #7 -.else - mov w15, #4 - bl output_lag3_neon - - increment_seed 4, shift=0 - mov v1.16b, v7.16b - -.ifc \edge, right - mov w15, #3 - bl output_lag3_neon - read_shift_rand x15, 11 - add x15, x3, x15, lsl #1 - ld1 {v1.h}[0], [x15] - srshl v1.4h, v1.4h, v31.4h - ext v0.16b, v0.16b, v1.16b, #1 -.else - mov w15, #4 - bl output_lag3_neon -.endif -.endif - st1 {v0.16b}, [x0], #16 - ldr x30, [sp], #16 - ret -.endif + sum_lag_n_body lag3, \type, \uv_layout, \edge, \elems, store=1, uv_coeff=v30.b[8] endfunc .endm sum_lag3_func y, 0, left sum_lag3_func y, 0, mid @@ -1059,42 +813,38 @@ ldr x30, [sp], #16 ret endfunc function add_uv_444_coeff_lag0_neon - str x30, [sp, #-16]! add_coeff_lag0_start: smull v2.8h, v0.8b, v27.8b smull2 v3.8h, v0.16b, v27.16b srshl v2.8h, v2.8h, v28.8h srshl v3.8h, v3.8h, v28.8h saddw v2.8h, v2.8h, v1.8b saddw2 v3.8h, v3.8h, v1.16b sqxtn v2.8b, v2.8h sqxtn2 v2.16b, v3.8h - ldr x30, [sp], #16 ret endfunc function add_uv_420_coeff_lag0_neon - str x30, [sp, #-16]! ld1 {v4.16b, v5.16b}, [x19], #32 ld1 {v6.16b, v7.16b}, [x12], #32 saddlp v4.8h, v4.16b saddlp v5.8h, v5.16b saddlp v6.8h, v6.16b saddlp v7.8h, v7.16b - add v4.8h, v4.8h, v6.8h - add v5.8h, v5.8h, v7.8h + add v4.8h, v4.8h, v6.8h + add v5.8h, v5.8h, v7.8h rshrn v4.8b, v4.8h, #2 rshrn2 v4.16b, v5.8h, #2 and v0.16b, v4.16b, v0.16b b add_coeff_lag0_start endfunc function add_uv_422_coeff_lag0_neon - str x30, [sp, #-16]! ld1 {v4.16b, v5.16b}, [x19], #32 saddlp v4.8h, v4.16b saddlp v5.8h, v5.16b rshrn v4.8b, v4.8h, #1 rshrn2 v4.16b, v5.8h, #1 @@ -1151,12 +901,10 @@ eor w2, w2, w11 .endif br x16 - ret - L(generate_grain_\type\()_lag0): .ifc \type, y mov w1, #GRAIN_HEIGHT bl generate_grain_rows_neon .else @@ -1206,19 +954,19 @@ L(generate_grain_\type\()_lag1): ld1r {v27.16b}, [x4], #1 // ar_coeffs_y[0] ld1r {v28.16b}, [x4], #1 // ar_coeffs_y[1] ld1r {v29.16b}, [x4] // ar_coeffs_y[2] .ifc \type, y - ldrsb w4, [x4, #1] // ar_coeffs_y[4] + ldrsb w4, [x4, #1] // ar_coeffs_y[3] .else add x4, x4, #2 .endif mov w1, #3 .ifc \type, uv_444 - ld1r {v30.16b}, [x4] // ar_coeffs_uv[5] - ldursb w4, [x4, #-1] // ar_coeffs_uv[4] + ld1r {v30.16b}, [x4] // ar_coeffs_uv[4] + ldursb w4, [x4, #-1] // ar_coeffs_uv[3] .endif bl generate_grain_rows_neon mov w1, #GRAIN_HEIGHT - 3 1: @@ -1268,12 +1016,11 @@ ldp x30, x19, [sp], #96 ret L(generate_grain_\type\()_lag3): - ldr q29, [x4] // ar_coeffs_y[0-15] - ldr q30, [x4, #16] // ar_coeffs_y[16-23], ar_coeffs_uv[16-24] + ld1 {v29.16b, v30.16b}, [x4] // ar_coeffs_y[0-23], ar_coeffs_uv[0-24] stp d8, d9, [sp, #16] stp d10, d11, [sp, #32] stp d12, d13, [sp, #48] stp d14, d15, [sp, #64] stp x20, x21, [sp, #80] @@ -1375,12 +1122,10 @@ eor w2, w2, w11 br x16 - ret - L(generate_grain_\type\()_lag0): dup v28.8h, w7 ld1r {v27.16b}, [x4] // ar_coeffs_uv[0] movi v0.16b, #0 movi v1.16b, #255 @@ -1421,12 +1166,12 @@ ld1r {v28.16b}, [x4], #1 // ar_coeffs_uv[1] ld1r {v29.16b}, [x4] // ar_coeffs_uv[2] add x4, x4, #2 mov w1, #3 - ld1r {v30.16b}, [x4] // ar_coeffs_uv[5] - ldursb w4, [x4, #-1] // ar_coeffs_uv[4] + ld1r {v30.16b}, [x4] // ar_coeffs_u4[4] + ldursb w4, [x4, #-1] // ar_coeffs_uv[3] bl generate_grain_rows_44_neon set_height w1, \type 1: sum_\type\()_lag1 v20, v16, v16, v17, left diff --git a/third_party/dav1d/src/arm/64/film_grain16.S b/third_party/dav1d/src/arm/64/film_grain16.S --- a/third_party/dav1d/src/arm/64/film_grain16.S +++ b/third_party/dav1d/src/arm/64/film_grain16.S @@ -28,10 +28,1114 @@ #include "src/arm/asm.S" #include "util.S" #include "src/arm/asm-offsets.h" #define GRAIN_WIDTH 82 +#define GRAIN_HEIGHT 73 + +#define SUB_GRAIN_WIDTH 44 +#define SUB_GRAIN_HEIGHT 38 + +.macro increment_seed steps, shift=1 + lsr w11, w2, #3 + lsr w12, w2, #12 + lsr w13, w2, #1 + eor w11, w2, w11 // (r >> 0) ^ (r >> 3) + eor w12, w12, w13 // (r >> 12) ^ (r >> 1) + eor w11, w11, w12 // (r >> 0) ^ (r >> 3) ^ (r >> 12) ^ (r >> 1) +.if \shift + lsr w2, w2, #\steps +.endif + and w11, w11, #((1 << \steps) - 1) // bit +.if \shift + orr w2, w2, w11, lsl #(16 - \steps) // *state +.else + orr w2, w2, w11, lsl #16 // *state +.endif +.endm + +.macro read_rand dest, bits, age + ubfx \dest, x2, #16 - \bits - \age, #\bits +.endm + +.macro read_shift_rand dest, bits + ubfx \dest, x2, #17 - \bits, #\bits + lsr w2, w2, #1 +.endm + +// special calling convention: +// w2 holds seed +// x3 holds dav1d_gaussian_sequence +// clobbers x11-x15 +// returns in v0.8h +function get_gaussian_neon + increment_seed 4 + read_rand x14, 11, 3 + read_rand x15, 11, 2 + add x14, x3, x14, lsl #1 + add x15, x3, x15, lsl #1 + ld1 {v0.h}[0], [x14] + read_rand x14, 11, 1 + ld1 {v0.h}[1], [x15] + add x14, x3, x14, lsl #1 + read_rand x15, 11, 0 + increment_seed 4 + add x15, x3, x15, lsl #1 + ld1 {v0.h}[2], [x14] + read_rand x14, 11, 3 + ld1 {v0.h}[3], [x15] + add x14, x3, x14, lsl #1 + read_rand x15, 11, 2 + ld1 {v0.h}[4], [x14] + add x15, x3, x15, lsl #1 + read_rand x14, 11, 1 + ld1 {v0.h}[5], [x15] + read_rand x15, 11, 0 + add x14, x3, x14, lsl #1 + add x15, x3, x15, lsl #1 + ld1 {v0.h}[6], [x14] + ld1 {v0.h}[7], [x15] + ret +endfunc + +.macro store_grain_row r0, r1, r2, r3, r4, r5 + st1 {\r0\().16b,\r1\().16b}, [x0], #32 + st1 {\r2\().16b,\r3\().16b}, [x0], #32 + st1 {\r4\().16b}, [x0], #16 + st1 {\r5\().h}[0], [x0], #2 +.endm + +function get_grain_2_neon + increment_seed 2 + read_rand x14, 11, 1 + read_rand x15, 11, 0 + add x14, x3, x14, lsl #1 + add x15, x3, x15, lsl #1 + ld1 {v0.h}[0], [x14] + ld1 {v0.h}[1], [x15] + srshl v0.4h, v0.4h, v31.4h + ret +endfunc + +.macro get_grain_2 dst + bl get_grain_2_neon +.ifnc \dst, v0 + mov \dst\().8b, v0.8b +.endif +.endm + +function get_grain_4_neon + increment_seed 4 + read_rand x14, 11, 3 + read_rand x15, 11, 2 + add x14, x3, x14, lsl #1 + add x15, x3, x15, lsl #1 + ld1 {v0.h}[0], [x14] + read_rand x14, 11, 1 + ld1 {v0.h}[1], [x15] + add x14, x3, x14, lsl #1 + read_rand x15, 11, 0 + add x15, x3, x15, lsl #1 + ld1 {v0.h}[2], [x14] + ld1 {v0.h}[3], [x15] + srshl v0.4h, v0.4h, v31.4h + ret +endfunc + +.macro get_grain_4 dst + bl get_grain_4_neon +.ifnc \dst, v0 + mov \dst\().8b, v0.8b +.endif +.endm + +// w15 holds the number of entries to produce +// w14, w16 and w17 hold the previous output entries +// v0 holds the vector of produced entries +// v1 holds the input vector of sums from above +.macro output_lag n +function output_lag\n\()_neon +1: + read_shift_rand x13, 11 + mov w11, v1.s[0] + ldrsh w12, [x3, x13, lsl #1] + ext v0.16b, v0.16b, v0.16b, #2 +.if \n == 1 + madd w11, w14, w4, w11 // sum (above) + *coeff * prev output +.elseif \n == 2 + madd w11, w16, w4, w11 // sum (above) + *coeff * prev output 1 + madd w11, w14, w17, w11 // += *coeff * prev output 2 + mov w16, w14 +.else + madd w11, w17, w4, w11 // sum (above) + *coeff * prev output 1 + madd w11, w16, w20, w11 // sum (above) + *coeff * prev output 2 + madd w11, w14, w21, w11 // += *coeff * prev output 3 + mov w17, w16 + mov w16, w14 +.endif + add w14, w11, w8 // 1 << (ar_coeff_shift - 1) + add w12, w12, w10 // 1 << (4 - bitdepth_min_8 + grain_scale_shift - 1) + asr w14, w14, w7 // >> ar_coeff_shift + asr w12, w12, w9 // >> (4 - bitdepth_min_8 + grain_scale_shift) + add w14, w14, w12 + cmp w14, w5 + csel w14, w14, w5, le + cmp w14, w6 + csel w14, w14, w6, ge + subs w15, w15, #1 + ext v1.16b, v1.16b, v1.16b, #4 + ins v0.h[7], w14 + b.gt 1b + ret +endfunc +.endm + +output_lag 1 +output_lag 2 +output_lag 3 + + +function sum_lag1_above_neon + sub x12, x0, #1*GRAIN_WIDTH*2 - 16 + ld1 {v18.8h}, [x12] // load top right + + ext v0.16b, v16.16b, v17.16b, #14 // top left, top mid + ext v1.16b, v17.16b, v18.16b, #2 // top mid, top right + + smull v4.4s, v17.4h, v28.4h + smlal v4.4s, v0.4h, v27.4h + smlal v4.4s, v1.4h, v29.4h + smull2 v5.4s, v17.8h, v28.8h + smlal2 v5.4s, v0.8h, v27.8h + smlal2 v5.4s, v1.8h, v29.8h + + mov v16.16b, v17.16b + mov v17.16b, v18.16b + + ret +endfunc + +.macro sum_lag_n_body lag, type, uv_layout, edge, elems, uv_coeff + bl sum_\lag\()_above_neon +.ifc \type, uv_420 + add x12, x19, #GRAIN_WIDTH*2 + ld1 {v22.8h, v23.8h}, [x19], #32 + ld1 {v24.8h, v25.8h}, [x12] + addp v22.8h, v22.8h, v23.8h + addp v23.8h, v24.8h, v25.8h + add v22.8h, v22.8h, v23.8h + srshr v0.8h, v22.8h, #2 +.endif +.ifc \type, uv_422 + ld1 {v22.8h, v23.8h}, [x19], #32 + addp v22.8h, v22.8h, v23.8h + srshr v0.8h, v22.8h, #1 +.endif +.ifc \type, uv_444 + ld1 {v0.8h}, [x19], #16 +.endif +.if \uv_layout +.ifnb \uv_coeff + dup v1.8b, \uv_coeff + sxtl v1.8h, v1.8b + smlal v4.4s, v0.4h, v1.4h + smlal2 v5.4s, v0.8h, v1.8h +.else + smlal v4.4s, v0.4h, v30.4h + smlal2 v5.4s, v0.8h, v30.8h +.endif +.endif +.if \uv_layout && \elems == 8 + b sum_\lag\()_y_\edge\()_start +.elseif \uv_layout == 444 && \elems == 7 + b sum_\lag\()_y_\edge\()_start +.elseif \uv_layout == 422 && \elems == 1 + b sum_\lag\()_uv_420_\edge\()_start +.else +sum_\lag\()_\type\()_\edge\()_start: +.if \elems > 4 +.ifc \edge, left + increment_seed 4 + read_rand x12, 11, 3 + read_rand x13, 11, 2 + read_rand x14, 11, 1 + add x12, x3, x12, lsl #1 + add x13, x3, x13, lsl #1 + add x14, x3, x14, lsl #1 + ld1 {v0.h}[5], [x12] + ld1 {v0.h}[6], [x13] + ld1 {v0.h}[7], [x14] + lsl x2, x2, #1 // shift back the state as if we'd done increment_seed with shift=0 + srshl v0.8h, v0.8h, v31.8h + ext v4.16b, v4.16b, v4.16b, #12 +.ifc \lag, lag3 + smov w17, v0.h[5] +.endif +.ifnc \lag, lag1 + smov w16, v0.h[6] +.endif + smov w14, v0.h[7] + + mov v1.16b, v4.16b + mov w15, #1 + bl output_\lag\()_neon +.else + increment_seed 4, shift=0 + mov v1.16b, v4.16b + mov w15, #4 + bl output_\lag\()_neon +.endif + + increment_seed 4, shift=0 + mov v1.16b, v5.16b +.ifc \edge, right + mov w15, #3 + bl output_\lag\()_neon + read_shift_rand x15, 11 + add x15, x3, x15, lsl #1 + ld1 {v1.h}[0], [x15] + srshl v1.4h, v1.4h, v31.4h + ext v0.16b, v0.16b, v1.16b, #2 +.else + mov w15, #4 + bl output_\lag\()_neon +.endif +.else + // elems == 1 + increment_seed 4, shift=0 + mov v1.16b, v4.16b + mov w15, #1 + bl output_\lag\()_neon + lsr w2, w2, #3 + + read_rand x12, 11, 2 + read_rand x13, 11, 1 + read_rand x14, 11, 0 + add x12, x3, x12, lsl #1 + add x13, x3, x13, lsl #1 + add x14, x3, x14, lsl #1 + ld1 {v1.h}[0], [x12] + ld1 {v1.h}[1], [x13] + ld1 {v1.h}[2], [x14] + srshl v1.4h, v1.4h, v31.4h + ext v0.16b, v0.16b, v1.16b, #14 +.endif + st1 {v0.8h}, [x0], #16 + ldr x30, [sp], #16 + ret +.endif +.endm + +.macro sum_lag1_func type, uv_layout, edge, elems=8 +function sum_\type\()_lag1_\edge\()_neon + str x30, [sp, #-16]! +.ifc \edge, left + sub x12, x0, #1*GRAIN_WIDTH*2 + ld1 {v17.8h}, [x12] // load the previous block right above +.endif + sum_lag_n_body lag1, \type, \uv_layout, \edge, \elems +endfunc +.endm + +sum_lag1_func y, 0, left +sum_lag1_func y, 0, mid +sum_lag1_func y, 0, right, 7 +sum_lag1_func uv_444, 444, left +sum_lag1_func uv_444, 444, mid +sum_lag1_func uv_444, 444, right, 7 +sum_lag1_func uv_422, 422, left +sum_lag1_func uv_422, 422, mid +sum_lag1_func uv_422, 422, right, 1 +sum_lag1_func uv_420, 420, left +sum_lag1_func uv_420, 420, mid +sum_lag1_func uv_420, 420, right, 1 + + +function sum_lag2_above_neon + sub x12, x0, #2*GRAIN_WIDTH*2 - 16 + sub x13, x0, #1*GRAIN_WIDTH*2 - 16 + ld1 {v18.8h}, [x12] // load top right + ld1 {v21.8h}, [x13] + + dup v26.8b, v30.b[0] + ext v22.16b, v16.16b, v17.16b, #12 // top left, top mid + dup v27.8b, v30.b[1] + ext v23.16b, v16.16b, v17.16b, #14 + sxtl v26.8h, v26.8b + dup v28.8b, v30.b[3] + ext v0.16b, v17.16b, v18.16b, #2 // top mid, top right + sxtl v27.8h, v27.8b + dup v29.8b, v30.b[4] + ext v1.16b, v17.16b, v18.16b, #4 + sxtl v28.8h, v28.8b + sxtl v29.8h, v29.8b + + smull v4.4s, v22.4h, v26.4h + smlal v4.4s, v23.4h, v27.4h + smlal v4.4s, v0.4h, v28.4h + smlal v4.4s, v1.4h, v29.4h + smull2 v5.4s, v22.8h, v26.8h + smlal2 v5.4s, v23.8h, v27.8h + smlal2 v5.4s, v0.8h, v28.8h + smlal2 v5.4s, v1.8h, v29.8h + + dup v26.16b, v30.b[5] + ext v22.16b, v19.16b, v20.16b, #12 // top left, top mid + dup v27.16b, v30.b[6] + ext v23.16b, v19.16b, v20.16b, #14 + sxtl v26.8h, v26.8b + dup v28.16b, v30.b[8] + ext v0.16b, v20.16b, v21.16b, #2 // top mid, top right + sxtl v27.8h, v27.8b + dup v29.16b, v30.b[9] + ext v1.16b, v20.16b, v21.16b, #4 + sxtl v28.8h, v28.8b + sxtl v29.8h, v29.8b + + smlal v4.4s, v22.4h, v26.4h + smlal v4.4s, v23.4h, v27.4h + smlal v4.4s, v0.4h, v28.4h + smlal v4.4s, v1.4h, v29.4h + smlal2 v5.4s, v22.8h, v26.8h + smlal2 v5.4s, v23.8h, v27.8h + smlal2 v5.4s, v0.8h, v28.8h + smlal2 v5.4s, v1.8h, v29.8h + + dup v26.16b, v30.b[2] + dup v27.16b, v30.b[7] + sxtl v26.8h, v26.8b + sxtl v27.8h, v27.8b + + smlal v4.4s, v17.4h, v26.4h + smlal v4.4s, v20.4h, v27.4h + smlal2 v5.4s, v17.8h, v26.8h + smlal2 v5.4s, v20.8h, v27.8h + mov v16.16b, v17.16b + mov v17.16b, v18.16b + + mov v19.16b, v20.16b + mov v20.16b, v21.16b + ret +endfunc + +.macro sum_lag2_func type, uv_layout, edge, elems=8 +function sum_\type\()_lag2_\edge\()_neon + str x30, [sp, #-16]! +.ifc \edge, left + sub x12, x0, #2*GRAIN_WIDTH*2 + sub x13, x0, #1*GRAIN_WIDTH*2 + ld1 {v17.8h}, [x12] // load the previous block right above + ld1 {v20.8h}, [x13] +.endif + sum_lag_n_body lag2, \type, \uv_layout, \edge, \elems, v30.b[12] +endfunc +.endm + +sum_lag2_func y, 0, left +sum_lag2_func y, 0, mid +sum_lag2_func y, 0, right, 7 +sum_lag2_func uv_444, 444, left +sum_lag2_func uv_444, 444, mid +sum_lag2_func uv_444, 444, right, 7 +sum_lag2_func uv_422, 422, left +sum_lag2_func uv_422, 422, mid +sum_lag2_func uv_422, 422, right, 1 +sum_lag2_func uv_420, 420, left +sum_lag2_func uv_420, 420, mid +sum_lag2_func uv_420, 420, right, 1 + + +function sum_lag3_above_neon + sub x11, x0, #3*GRAIN_WIDTH*2 - 16 + sub x12, x0, #2*GRAIN_WIDTH*2 - 16 + sub x13, x0, #1*GRAIN_WIDTH*2 - 16 + ld1 {v15.8h}, [x11] // load top right + ld1 {v18.8h}, [x12] + ld1 {v21.8h}, [x13] + + dup v22.8b, v29.b[0] + ext v8.16b, v13.16b, v14.16b, #10 // top left, top mid + dup v23.8b, v29.b[1] + ext v9.16b, v13.16b, v14.16b, #12 + sxtl v22.8h, v22.8b + dup v24.8b, v29.b[2] + sxtl v23.8h, v23.8b + dup v25.8b, v29.b[3] + ext v10.16b, v13.16b, v14.16b, #14 + sxtl v24.8h, v24.8b + dup v26.8b, v29.b[4] + ext v11.16b, v14.16b, v15.16b, #2 // top mid, top right + sxtl v25.8h, v25.8b + dup v27.8b, v29.b[5] + ext v12.16b, v14.16b, v15.16b, #4 + sxtl v26.8h, v26.8b + dup v28.8b, v29.b[6] + ext v13.16b, v14.16b, v15.16b, #6 + sxtl v27.8h, v27.8b + sxtl v28.8h, v28.8b + + smull v4.4s, v8.4h, v22.4h + smlal v4.4s, v9.4h, v23.4h + smlal v4.4s, v10.4h, v24.4h + smlal v4.4s, v11.4h, v26.4h + smlal v4.4s, v12.4h, v27.4h + smlal v4.4s, v13.4h, v28.4h + smlal v4.4s, v14.4h, v25.4h + smull2 v5.4s, v8.8h, v22.8h + smlal2 v5.4s, v9.8h, v23.8h + smlal2 v5.4s, v10.8h, v24.8h + smlal2 v5.4s, v11.8h, v26.8h + smlal2 v5.4s, v12.8h, v27.8h + smlal2 v5.4s, v13.8h, v28.8h + smlal2 v5.4s, v14.8h, v25.8h + + dup v22.8b, v29.b[7] + ext v8.16b, v16.16b, v17.16b, #10 // top left, top mid + dup v23.8b, v29.b[8] + ext v9.16b, v16.16b, v17.16b, #12 + sxtl v22.8h, v22.8b + dup v24.8b, v29.b[9] + sxtl v23.8h, v23.8b + dup v25.8b, v29.b[10] + ext v10.16b, v16.16b, v17.16b, #14 + sxtl v24.8h, v24.8b + dup v26.8b, v29.b[11] + ext v11.16b, v17.16b, v18.16b, #2 // top mid, top right + sxtl v25.8h, v25.8b + dup v27.8b, v29.b[12] + ext v12.16b, v17.16b, v18.16b, #4 + sxtl v26.8h, v26.8b + dup v28.8b, v29.b[13] + ext v13.16b, v17.16b, v18.16b, #6 + sxtl v27.8h, v27.8b + sxtl v28.8h, v28.8b + + smlal v4.4s, v8.4h, v22.4h + smlal v4.4s, v9.4h, v23.4h + smlal v4.4s, v10.4h, v24.4h + smlal v4.4s, v11.4h, v26.4h + smlal v4.4s, v12.4h, v27.4h + smlal v4.4s, v13.4h, v28.4h + smlal v4.4s, v17.4h, v25.4h + smlal2 v5.4s, v8.8h, v22.8h + smlal2 v5.4s, v9.8h, v23.8h + smlal2 v5.4s, v10.8h, v24.8h + smlal2 v5.4s, v11.8h, v26.8h + smlal2 v5.4s, v12.8h, v27.8h + smlal2 v5.4s, v13.8h, v28.8h + smlal2 v5.4s, v17.8h, v25.8h + + dup v22.8b, v29.b[14] + ext v8.16b, v19.16b, v20.16b, #10 // top left, top mid + dup v23.8b, v29.b[15] + ext v9.16b, v19.16b, v20.16b, #12 + sxtl v22.8h, v22.8b + dup v24.8b, v30.b[0] + sxtl v23.8h, v23.8b + dup v25.8b, v30.b[1] + ext v10.16b, v19.16b, v20.16b, #14 + sxtl v24.8h, v24.8b + dup v26.8b, v30.b[2] + ext v11.16b, v20.16b, v21.16b, #2 // top mid, top right + sxtl v25.8h, v25.8b + dup v27.8b, v30.b[3] + ext v12.16b, v20.16b, v21.16b, #4 + sxtl v26.8h, v26.8b + dup v28.8b, v30.b[4] + ext v13.16b, v20.16b, v21.16b, #6 + sxtl v27.8h, v27.8b + sxtl v28.8h, v28.8b + + smlal v4.4s, v8.4h, v22.4h + smlal v4.4s, v9.4h, v23.4h + smlal v4.4s, v10.4h, v24.4h + smlal v4.4s, v11.4h, v26.4h + smlal v4.4s, v12.4h, v27.4h + smlal v4.4s, v13.4h, v28.4h + smlal v4.4s, v20.4h, v25.4h + mov v16.16b, v17.16b + mov v17.16b, v18.16b + smlal2 v5.4s, v8.8h, v22.8h + smlal2 v5.4s, v9.8h, v23.8h + smlal2 v5.4s, v10.8h, v24.8h + smlal2 v5.4s, v11.8h, v26.8h + smlal2 v5.4s, v12.8h, v27.8h + smlal2 v5.4s, v13.8h, v28.8h + smlal2 v5.4s, v20.8h, v25.8h + + mov v13.16b, v14.16b + mov v14.16b, v15.16b + + mov v19.16b, v20.16b + mov v20.16b, v21.16b + ret +endfunc + +.macro sum_lag3_func type, uv_layout, edge, elems=8 +function sum_\type\()_lag3_\edge\()_neon + str x30, [sp, #-16]! +.ifc \edge, left + sub x11, x0, #3*GRAIN_WIDTH*2 + sub x12, x0, #2*GRAIN_WIDTH*2 + sub x13, x0, #1*GRAIN_WIDTH*2 + ld1 {v14.8h}, [x11] // load the previous block right above + ld1 {v17.8h}, [x12] + ld1 {v20.8h}, [x13] +.endif + sum_lag_n_body lag3, \type, \uv_layout, \edge, \elems, v30.b[8] +endfunc +.endm + +sum_lag3_func y, 0, left +sum_lag3_func y, 0, mid +sum_lag3_func y, 0, right, 7 +sum_lag3_func uv_444, 444, left +sum_lag3_func uv_444, 444, mid +sum_lag3_func uv_444, 444, right, 7 +sum_lag3_func uv_422, 422, left +sum_lag3_func uv_422, 422, mid +sum_lag3_func uv_422, 422, right, 1 +sum_lag3_func uv_420, 420, left +sum_lag3_func uv_420, 420, mid +sum_lag3_func uv_420, 420, right, 1 + +function generate_grain_rows_neon + str x30, [sp, #-16]! +1: + mov w16, #80 +2: + bl get_gaussian_neon + srshl v0.8h, v0.8h, v31.8h + subs w16, w16, #8 + st1 {v0.8h}, [x0], #16 + b.gt 2b + get_grain_2 v0 + subs w1, w1, #1 + st1 {v0.s}[0], [x0], #4 + b.gt 1b + ldr x30, [sp], #16 + ret +endfunc + +function generate_grain_rows_44_neon + str x30, [sp, #-16]! +1: + mov w16, #40 +2: + bl get_gaussian_neon + srshl v0.8h, v0.8h, v31.8h + subs w16, w16, #8 + st1 {v0.8h}, [x0], #16 + b.gt 2b + get_grain_4 v0 + subs w1, w1, #1 + st1 {v0.4h}, [x0] + add x0, x0, #GRAIN_WIDTH*2-80 + b.gt 1b + ldr x30, [sp], #16 + ret +endfunc + +function gen_grain_uv_444_lag0_neon + str x30, [sp, #-16]! + ld1 {v4.8h}, [x19], #16 +gen_grain_uv_lag0_8_start: + bl get_gaussian_neon + srshl v0.8h, v0.8h, v31.8h +gen_grain_uv_lag0_8_add: + and v4.16b, v4.16b, v1.16b + smull v2.4s, v4.4h, v27.4h + smull2 v3.4s, v4.8h, v27.8h + srshl v2.4s, v2.4s, v28.4s + srshl v3.4s, v3.4s, v28.4s + sqxtn v2.4h, v2.4s + sqxtn2 v2.8h, v3.4s + sqadd v2.8h, v2.8h, v0.8h + smin v2.8h, v2.8h, v25.8h + smax v2.8h, v2.8h, v26.8h + st1 {v2.8h}, [x0], #16 + ldr x30, [sp], #16 + ret +endfunc + +function gen_grain_uv_420_lag0_8_neon + add x12, x19, #GRAIN_WIDTH*2 + str x30, [sp, #-16]! + ld1 {v16.8h, v17.8h}, [x19], #32 + ld1 {v18.8h, v19.8h}, [x12] + addp v16.8h, v16.8h, v17.8h + addp v17.8h, v18.8h, v19.8h + add v16.8h, v16.8h, v17.8h + srshr v4.8h, v16.8h, #2 + b gen_grain_uv_lag0_8_start +endfunc + +function gen_grain_uv_422_lag0_8_neon + str x30, [sp, #-16]! + ld1 {v16.8h, v17.8h}, [x19], #32 + addp v16.8h, v16.8h, v17.8h + srshr v4.8h, v16.8h, #1 + b gen_grain_uv_lag0_8_start +endfunc + +function gen_grain_uv_420_lag0_4_neon + add x12, x19, #GRAIN_WIDTH*2 + str x30, [sp, #-16]! + ld1 {v16.4h, v17.4h}, [x19] + ld1 {v18.4h, v19.4h}, [x12] + add x19, x19, #32 + addp v16.4h, v16.4h, v17.4h + addp v17.4h, v18.4h, v19.4h + add v16.4h, v16.4h, v17.4h + srshr v4.4h, v16.4h, #2 + get_grain_4 v0 + b gen_grain_uv_lag0_8_add +endfunc + +function gen_grain_uv_422_lag0_4_neon + str x30, [sp, #-16]! + ld1 {v16.4h, v17.4h}, [x19] + add x19, x19, #32 + addp v16.4h, v16.4h, v17.4h + srshr v4.4h, v16.4h, #1 + get_grain_4 v0 + b gen_grain_uv_lag0_8_add +endfunc + +.macro gen_grain_82 type +function generate_grain_\type\()_16bpc_neon, export=1 + stp x30, x19, [sp, #-96]! + +.ifc \type, uv_444 + mov w13, w3 + mov w14, #28 + add x19, x1, #3*GRAIN_WIDTH*2 + mov x1, x2 + mul w13, w13, w14 + clz w15, w4 +.else + clz w15, w2 +.endif + movrel x3, X(gaussian_sequence) + sub w15, w15, #24 // -bitdepth_min_8 + ldr w2, [x1, #FGD_SEED] + ldr w9, [x1, #FGD_GRAIN_SCALE_SHIFT] +.ifc \type, y + add x4, x1, #FGD_AR_COEFFS_Y +.else + add x4, x1, #FGD_AR_COEFFS_UV +.endif + add w9, w9, w15 // grain_scale_shift - bitdepth_min_8 + adr x16, L(gen_grain_\type\()_tbl) + ldr w17, [x1, #FGD_AR_COEFF_LAG] + add w9, w9, #4 + ldrh w17, [x16, w17, uxtw #1] + dup v31.8h, w9 // 4 - bitdepth_min_8 + data->grain_scale_shift + sub x16, x16, w17, uxtw + neg v31.8h, v31.8h + +.ifc \type, uv_444 + cmp w13, #0 + mov w11, #0x49d8 + mov w14, #0xb524 + add x4, x4, w13, uxtw // Add offset to ar_coeffs_uv[1] + csel w11, w11, w14, ne +.endif + + ldr w7, [x1, #FGD_AR_COEFF_SHIFT] + neg w15, w15 // bitdepth_min_8 + mov w8, #1 + mov w10, #1 + lsl w8, w8, w7 // 1 << ar_coeff_shift + lsl w10, w10, w9 // 1 << (4 + data->grain_scale_shift) + lsr w8, w8, #1 // 1 << (ar_coeff_shift - 1) + lsr w10, w10, #1 // 1 << (4 + data->grain_scale_shift - 1) + mov w5, #128 + lsl w5, w5, w15 // 128 << bitdepth_min_8 + neg w6, w5 // -(128 << bitpdeth_min_8) + sub w5, w5, #1 // (128 << bitdepth_min_8) - 1 + +.ifc \type, uv_444 + eor w2, w2, w11 +.endif + + br x16 + +L(generate_grain_\type\()_lag0): +.ifc \type, y + mov w1, #GRAIN_HEIGHT + bl generate_grain_rows_neon +.else + dup v28.4s, w7 + ld1r {v27.8b}, [x4] // ar_coeffs_uv[0] + movi v0.16b, #0 + movi v1.16b, #255 + dup v25.8h, w5 + dup v26.8h, w6 + ext v29.16b, v0.16b, v1.16b, #10 + ext v30.16b, v1.16b, v0.16b, #2 + neg v28.4s, v28.4s + sxtl v27.8h, v27.8b + + mov w1, #3 + bl generate_grain_rows_neon + mov w1, #GRAIN_HEIGHT-3 +1: + mov v1.16b, v29.16b + bl gen_grain_uv_444_lag0_neon // 8 + movi v1.16b, #255 + bl gen_grain_uv_444_lag0_neon // 16 + bl gen_grain_uv_444_lag0_neon // 24 + bl gen_grain_uv_444_lag0_neon // 32 + bl gen_grain_uv_444_lag0_neon // 40 + bl gen_grain_uv_444_lag0_neon // 48 + bl gen_grain_uv_444_lag0_neon // 56 + bl gen_grain_uv_444_lag0_neon // 64 + bl gen_grain_uv_444_lag0_neon // 72 + mov v1.16b, v30.16b + bl gen_grain_uv_444_lag0_neon // 80 + get_grain_2 v16 + subs w1, w1, #1 + add x19, x19, #4 + st1 {v16.s}[0], [x0], #4 + b.gt 1b +.endif + ldp x30, x19, [sp], #96 + ret + +L(generate_grain_\type\()_lag1): + ld1r {v27.8b}, [x4], #1 // ar_coeffs_y[0] + ld1r {v28.8b}, [x4], #1 // ar_coeffs_y[1] + ld1r {v29.8b}, [x4] // ar_coeffs_y[2] +.ifc \type, y + ldrsb w4, [x4, #1] // ar_coeffs_y[3] +.else + add x4, x4, #2 +.endif + + mov w1, #3 +.ifc \type, uv_444 + ld1r {v30.8b}, [x4] // ar_coeffs_uv[4] + ldursb w4, [x4, #-1] // ar_coeffs_uv[3] +.endif + bl generate_grain_rows_neon + sxtl v27.8h, v27.8b + sxtl v28.8h, v28.8b + sxtl v29.8h, v29.8b +.ifc \type, uv_444 + sxtl v30.8h, v30.8b +.endif + + mov w1, #GRAIN_HEIGHT - 3 +1: + bl sum_\type\()_lag1_left_neon // 8 + bl sum_\type\()_lag1_mid_neon // 16 + bl sum_\type\()_lag1_mid_neon // 24 + bl sum_\type\()_lag1_mid_neon // 32 + bl sum_\type\()_lag1_mid_neon // 40 + bl sum_\type\()_lag1_mid_neon // 48 + bl sum_\type\()_lag1_mid_neon // 56 + bl sum_\type\()_lag1_mid_neon // 64 + bl sum_\type\()_lag1_mid_neon // 72 + bl sum_\type\()_lag1_right_neon // 80 + get_grain_2 v16 + subs w1, w1, #1 +.ifc \type, uv_444 + add x19, x19, #4 +.endif + st1 {v16.s}[0], [x0], #4 + b.gt 1b + + ldp x30, x19, [sp], #96 + ret + +L(generate_grain_\type\()_lag2): + ld1 {v30.16b}, [x4] // ar_coeffs_y[0-11], ar_coeffs_uv[0-12] + + smov w4, v30.b[10] + smov w17, v30.b[11] + + mov w1, #3 + bl generate_grain_rows_neon + + mov w1, #GRAIN_HEIGHT - 3 +1: + bl sum_\type\()_lag2_left_neon // 8 + bl sum_\type\()_lag2_mid_neon // 16 + bl sum_\type\()_lag2_mid_neon // 24 + bl sum_\type\()_lag2_mid_neon // 32 + bl sum_\type\()_lag2_mid_neon // 40 + bl sum_\type\()_lag2_mid_neon // 48 + bl sum_\type\()_lag2_mid_neon // 56 + bl sum_\type\()_lag2_mid_neon // 64 + bl sum_\type\()_lag2_mid_neon // 72 + bl sum_\type\()_lag2_right_neon // 80 + get_grain_2 v16 + subs w1, w1, #1 +.ifc \type, uv_444 + add x19, x19, #4 +.endif + st1 {v16.s}[0], [x0], #4 + b.gt 1b + + ldp x30, x19, [sp], #96 + ret + +L(generate_grain_\type\()_lag3): + ld1 {v29.16b, v30.16b}, [x4] // ar_coeffs_y[0-23], ar_coeffs_uv[0-24] + stp d8, d9, [sp, #16] + stp d10, d11, [sp, #32] + stp d12, d13, [sp, #48] + stp d14, d15, [sp, #64] + stp x20, x21, [sp, #80] + + smov w4, v30.b[5] + smov w20, v30.b[6] + smov w21, v30.b[7] + + mov w1, #3 + bl generate_grain_rows_neon + + mov w1, #GRAIN_HEIGHT - 3 +1: + bl sum_\type\()_lag3_left_neon // 8 + bl sum_\type\()_lag3_mid_neon // 16 + bl sum_\type\()_lag3_mid_neon // 24 + bl sum_\type\()_lag3_mid_neon // 32 + bl sum_\type\()_lag3_mid_neon // 40 + bl sum_\type\()_lag3_mid_neon // 48 + bl sum_\type\()_lag3_mid_neon // 56 + bl sum_\type\()_lag3_mid_neon // 64 + bl sum_\type\()_lag3_mid_neon // 72 + bl sum_\type\()_lag3_right_neon // 80 + get_grain_2 v16 + subs w1, w1, #1 +.ifc \type, uv_444 + add x19, x19, #4 +.endif + st1 {v16.s}[0], [x0], #4 + b.gt 1b + + ldp x20, x21, [sp, #80] + ldp d14, d15, [sp, #64] + ldp d12, d13, [sp, #48] + ldp d10, d11, [sp, #32] + ldp d8, d9, [sp, #16] + ldp x30, x19, [sp], #96 + ret + +L(gen_grain_\type\()_tbl): + .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag0) + .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag1) + .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag2) + .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag3) +endfunc +.endm + +gen_grain_82 y +gen_grain_82 uv_444 + +.macro set_height dst, type +.ifc \type, uv_420 + mov \dst, #SUB_GRAIN_HEIGHT-3 +.else + mov \dst, #GRAIN_HEIGHT-3 +.endif +.endm + +.macro increment_y_ptr reg, type +.ifc \type, uv_420 + add \reg, \reg, #2*GRAIN_WIDTH*2-(6*32) +.else + sub \reg, \reg, #6*32-GRAIN_WIDTH*2 +.endif +.endm + +.macro gen_grain_44 type +function generate_grain_\type\()_16bpc_neon, export=1 + stp x30, x19, [sp, #-96]! + + mov w13, w3 + mov w14, #28 + add x19, x1, #(3*GRAIN_WIDTH-3)*2 + mov x1, x2 + mul w13, w13, w14 + clz w15, w4 + + movrel x3, X(gaussian_sequence) + sub w15, w15, #24 // -bitdepth_min_8 + ldr w2, [x1, #FGD_SEED] + ldr w9, [x1, #FGD_GRAIN_SCALE_SHIFT] + add x4, x1, #FGD_AR_COEFFS_UV + add w9, w9, w15 // grain_scale_shift - bitdepth_min_8 + adr x16, L(gen_grain_\type\()_tbl) + ldr w17, [x1, #FGD_AR_COEFF_LAG] + add w9, w9, #4 + ldrh w17, [x16, w17, uxtw #1] + dup v31.8h, w9 // 4 - bitdepth_min_8 + data->grain_scale_shift + sub x16, x16, w17, uxtw + neg v31.8h, v31.8h + + cmp w13, #0 + mov w11, #0x49d8 + mov w14, #0xb524 + add x4, x4, w13, uxtw // Add offset to ar_coeffs_uv[1] + csel w11, w11, w14, ne + + ldr w7, [x1, #FGD_AR_COEFF_SHIFT] + neg w15, w15 // bitdepth_min_8 + mov w8, #1 + mov w10, #1 + lsl w8, w8, w7 // 1 << ar_coeff_shift + lsl w10, w10, w9 // 1 << (4 + data->grain_scale_shift) + lsr w8, w8, #1 // 1 << (ar_coeff_shift - 1) + lsr w10, w10, #1 // 1 << (4 + data->grain_scale_shift - 1) + mov w5, #128 + lsl w5, w5, w15 // 128 << bitdepth_min_8 + neg w6, w5 // -(128 << bitpdeth_min_8) + sub w5, w5, #1 // (128 << bitdepth_min_8) - 1 + + eor w2, w2, w11 + + br x16 + +L(generate_grain_\type\()_lag0): + dup v28.4s, w7 + ld1r {v27.8b}, [x4] // ar_coeffs_uv[0] + movi v0.16b, #0 + movi v1.16b, #255 + dup v25.8h, w5 + dup v26.8h, w6 + ext v29.16b, v0.16b, v1.16b, #10 + ext v30.16b, v1.16b, v0.16b, #14 + neg v28.4s, v28.4s + sxtl v27.8h, v27.8b + + mov w1, #3 + bl generate_grain_rows_44_neon + set_height w1, \type +1: + mov v1.16b, v29.16b + bl gen_grain_\type\()_lag0_8_neon // 8 + movi v1.16b, #255 + bl gen_grain_\type\()_lag0_8_neon // 16 + bl gen_grain_\type\()_lag0_8_neon // 24 + bl gen_grain_\type\()_lag0_8_neon // 32 + bl gen_grain_\type\()_lag0_8_neon // 40 + mov v1.16b, v30.16b + bl gen_grain_\type\()_lag0_4_neon // 44 + subs w1, w1, #1 + increment_y_ptr x19, \type + add x0, x0, #GRAIN_WIDTH*2-6*16 + b.gt 1b + + ldp x30, x19, [sp], #96 + ret + +L(generate_grain_\type\()_lag1): + ld1r {v27.8b}, [x4], #1 // ar_coeffs_uv[0] + ld1r {v28.8b}, [x4], #1 // ar_coeffs_uv[1] + ld1r {v29.8b}, [x4] // ar_coeffs_uv[2] + add x4, x4, #2 + + mov w1, #3 + ld1r {v30.8b}, [x4] // ar_coeffs_u4[4] + ldursb w4, [x4, #-1] // ar_coeffs_uv[3] + bl generate_grain_rows_44_neon + + sxtl v27.8h, v27.8b + sxtl v28.8h, v28.8b + sxtl v29.8h, v29.8b + sxtl v30.8h, v30.8b + set_height w1, \type +1: + bl sum_\type\()_lag1_left_neon // 8 + bl sum_\type\()_lag1_mid_neon // 16 + bl sum_\type\()_lag1_mid_neon // 24 + bl sum_\type\()_lag1_mid_neon // 32 + bl sum_\type\()_lag1_mid_neon // 40 + bl sum_\type\()_lag1_right_neon // 44 + subs w1, w1, #1 + increment_y_ptr x19, \type + add x0, x0, #GRAIN_WIDTH*2-6*16 + b.gt 1b + + ldp x30, x19, [sp], #96 + ret + +L(generate_grain_\type\()_lag2): + ld1 {v30.16b}, [x4] // ar_coeffs_uv[0-12] + + smov w4, v30.b[10] + smov w17, v30.b[11] + + mov w1, #3 + bl generate_grain_rows_44_neon + + set_height w1, \type +1: + bl sum_\type\()_lag2_left_neon // 8 + bl sum_\type\()_lag2_mid_neon // 16 + bl sum_\type\()_lag2_mid_neon // 24 + bl sum_\type\()_lag2_mid_neon // 32 + bl sum_\type\()_lag2_mid_neon // 40 + bl sum_\type\()_lag2_right_neon // 44 + subs w1, w1, #1 + increment_y_ptr x19, \type + add x0, x0, #GRAIN_WIDTH*2-6*16 + b.gt 1b + + ldp x30, x19, [sp], #96 + ret + +L(generate_grain_\type\()_lag3): + ldr q29, [x4] // ar_coeffs_uv[0-15] + ldr q30, [x4, #16] // ar_coeffs_uv[16-24] + stp d8, d9, [sp, #16] + stp d10, d11, [sp, #32] + stp d12, d13, [sp, #48] + stp d14, d15, [sp, #64] + stp x20, x21, [sp, #80] + + smov w4, v30.b[5] + smov w20, v30.b[6] + smov w21, v30.b[7] + + mov w1, #3 + bl generate_grain_rows_44_neon + + set_height w1, \type +1: + bl sum_\type\()_lag3_left_neon // 8 + bl sum_\type\()_lag3_mid_neon // 16 + bl sum_\type\()_lag3_mid_neon // 24 + bl sum_\type\()_lag3_mid_neon // 32 + bl sum_\type\()_lag3_mid_neon // 40 + bl sum_\type\()_lag3_right_neon // 44 + subs w1, w1, #1 + increment_y_ptr x19, \type + add x0, x0, #GRAIN_WIDTH*2-6*16 + b.gt 1b + + ldp x20, x21, [sp, #80] + ldp d14, d15, [sp, #64] + ldp d12, d13, [sp, #48] + ldp d10, d11, [sp, #32] + ldp d8, d9, [sp, #16] + ldp x30, x19, [sp], #96 + ret + +L(gen_grain_\type\()_tbl): + .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag0) + .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag1) + .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag2) + .hword L(gen_grain_\type\()_tbl) - L(generate_grain_\type\()_lag3) +endfunc +.endm + +gen_grain_44 uv_420 +gen_grain_44 uv_422 .macro gather_interleaved dst1, dst2, src1, src2, off umov w14, \src1[0] umov w15, \src2[1] umov w16, \src1[2] diff --git a/third_party/dav1d/src/arm/64/refmvs.S b/third_party/dav1d/src/arm/64/refmvs.S new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/arm/64/refmvs.S @@ -0,0 +1,85 @@ +/* + * Copyright © 2021, VideoLAN and dav1d authors + * Copyright © 2021, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +// void dav1d_splat_mv_neon(refmvs_block **rr, const refmvs_block *rmv, +// int bx4, int bw4, int bh4) + +function splat_mv_neon, export=1 + ld1 {v3.16b}, [x1] + clz w3, w3 + adr x5, L(splat_tbl) + sub w3, w3, #26 + ext v2.16b, v3.16b, v3.16b, #12 + ldrh w3, [x5, w3, uxtw #1] + add w2, w2, w2, lsl #1 + ext v0.16b, v2.16b, v3.16b, #4 + sub x3, x5, w3, uxtw + ext v1.16b, v2.16b, v3.16b, #8 + lsl w2, w2, #2 + ext v2.16b, v2.16b, v3.16b, #12 +1: + ldr x1, [x0], #8 + subs w4, w4, #1 + add x1, x1, x2 + br x3 + +10: + st1 {v0.8b}, [x1] + str s2, [x1, #8] + b.gt 1b + ret +20: + st1 {v0.16b}, [x1] + str d1, [x1, #16] + b.gt 1b + ret +320: + st1 {v0.16b, v1.16b, v2.16b}, [x1], #48 + st1 {v0.16b, v1.16b, v2.16b}, [x1], #48 + st1 {v0.16b, v1.16b, v2.16b}, [x1], #48 + st1 {v0.16b, v1.16b, v2.16b}, [x1], #48 +160: + st1 {v0.16b, v1.16b, v2.16b}, [x1], #48 + st1 {v0.16b, v1.16b, v2.16b}, [x1], #48 +80: + st1 {v0.16b, v1.16b, v2.16b}, [x1], #48 +40: + st1 {v0.16b, v1.16b, v2.16b}, [x1] + b.gt 1b + ret + +L(splat_tbl): + .hword L(splat_tbl) - 320b + .hword L(splat_tbl) - 160b + .hword L(splat_tbl) - 80b + .hword L(splat_tbl) - 40b + .hword L(splat_tbl) - 20b + .hword L(splat_tbl) - 10b +endfunc diff --git a/third_party/dav1d/src/arm/film_grain_init_tmpl.c b/third_party/dav1d/src/arm/film_grain_init_tmpl.c --- a/third_party/dav1d/src/arm/film_grain_init_tmpl.c +++ b/third_party/dav1d/src/arm/film_grain_init_tmpl.c @@ -29,12 +29,10 @@ #include "src/cpu.h" #include "src/film_grain.h" #include "asm-offsets.h" -#if ARCH_AARCH64 - CHECK_OFFSET(Dav1dFilmGrainData, seed, FGD_SEED); CHECK_OFFSET(Dav1dFilmGrainData, ar_coeff_lag, FGD_AR_COEFF_LAG); CHECK_OFFSET(Dav1dFilmGrainData, ar_coeffs_y, FGD_AR_COEFFS_Y); CHECK_OFFSET(Dav1dFilmGrainData, ar_coeffs_uv, FGD_AR_COEFFS_UV); CHECK_OFFSET(Dav1dFilmGrainData, ar_coeff_shift, FGD_AR_COEFF_SHIFT); @@ -58,11 +56,10 @@ HIGHBD_DECL_SUFFIX) GEN_GRAIN_UV(420); GEN_GRAIN_UV(422); GEN_GRAIN_UV(444); -#endif // Use ptrdiff_t instead of int for the last few parameters, to get the // same layout of parameters on the stack across platforms. void BF(dav1d_fgy_32x32, neon)(pixel *const dst, const pixel *const src, @@ -207,16 +204,14 @@ COLD void bitfn(dav1d_film_grain_dsp_init_arm)(Dav1dFilmGrainDSPContext *const c) { const unsigned flags = dav1d_get_cpu_flags(); if (!(flags & DAV1D_ARM_CPU_FLAG_NEON)) return; -#if ARCH_AARCH64 && BITDEPTH == 8 c->generate_grain_y = BF(dav1d_generate_grain_y, neon); c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I420 - 1] = BF(dav1d_generate_grain_uv_420, neon); c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I422 - 1] = BF(dav1d_generate_grain_uv_422, neon); c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I444 - 1] = BF(dav1d_generate_grain_uv_444, neon); -#endif c->fgy_32x32xn = fgy_32x32xn_neon; c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I420 - 1] = fguv_32x32xn_420_neon; c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I422 - 1] = fguv_32x32xn_422_neon; c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I444 - 1] = fguv_32x32xn_444_neon; diff --git a/third_party/dav1d/src/arm/refmvs_init.c b/third_party/dav1d/src/arm/refmvs_init.c new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/arm/refmvs_init.c @@ -0,0 +1,39 @@ +/* + * Copyright © 2021, VideoLAN and dav1d authors + * Copyright © 2021, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/cpu.h" +#include "src/refmvs.h" + +decl_splat_mv_fn(dav1d_splat_mv_neon); + +COLD void dav1d_refmvs_dsp_init_arm(Dav1dRefmvsDSPContext *const c) { + const unsigned flags = dav1d_get_cpu_flags(); + + if (!(flags & DAV1D_ARM_CPU_FLAG_NEON)) return; + + c->splat_mv = dav1d_splat_mv_neon; +} diff --git a/third_party/dav1d/src/cdef_tmpl.c b/third_party/dav1d/src/cdef_tmpl.c --- a/third_party/dav1d/src/cdef_tmpl.c +++ b/third_party/dav1d/src/cdef_tmpl.c @@ -111,11 +111,11 @@ if (pri_strength) { const int bitdepth_min_8 = bitdepth_from_max(bitdepth_max) - 8; const int pri_tap = 4 - ((pri_strength >> bitdepth_min_8) & 1); const int pri_shift = imax(0, damping - ulog2(pri_strength)); if (sec_strength) { - const int sec_shift = imax(0, damping - ulog2(sec_strength)); + const int sec_shift = damping - ulog2(sec_strength); do { for (int x = 0; x < w; x++) { const int px = dst[x]; int sum = 0; int max = px, min = px; @@ -178,11 +178,11 @@ tmp += tmp_stride; } while (--h); } } else { // sec_strength only assert(sec_strength); - const int sec_shift = imax(0, damping - ulog2(sec_strength)); + const int sec_shift = damping - ulog2(sec_strength); do { for (int x = 0; x < w; x++) { const int px = dst[x]; int sum = 0; for (int k = 0; k < 2; k++) { diff --git a/third_party/dav1d/src/cdf.h b/third_party/dav1d/src/cdf.h --- a/third_party/dav1d/src/cdf.h +++ b/third_party/dav1d/src/cdf.h @@ -133,25 +133,18 @@ Dav1dRef *ref; ///< allocation origin union { CdfContext *cdf; // if ref != NULL unsigned qcat; // if ref == NULL, from static CDF tables } data; - struct thread_data *t; atomic_uint *progress; } CdfThreadContext; void dav1d_cdf_thread_init_static(CdfThreadContext *cdf, int qidx); int dav1d_cdf_thread_alloc(Dav1dContext *c, CdfThreadContext *cdf, - struct thread_data *t); + const int have_frame_mt); void dav1d_cdf_thread_copy(CdfContext *dst, const CdfThreadContext *src); void dav1d_cdf_thread_ref(CdfThreadContext *dst, CdfThreadContext *src); void dav1d_cdf_thread_unref(CdfThreadContext *cdf); void dav1d_cdf_thread_update(const Dav1dFrameHeader *hdr, CdfContext *dst, const CdfContext *src); -/* - * These are binary signals (so a signal is either "done" or "not done"). - */ -void dav1d_cdf_thread_wait(CdfThreadContext *cdf); -void dav1d_cdf_thread_signal(CdfThreadContext *cdf); - #endif /* DAV1D_SRC_CDF_H */ diff --git a/third_party/dav1d/src/cdf.c b/third_party/dav1d/src/cdf.c --- a/third_party/dav1d/src/cdf.c +++ b/third_party/dav1d/src/cdf.c @@ -4094,20 +4094,19 @@ default_mv_component_cdf; } } int dav1d_cdf_thread_alloc(Dav1dContext *const c, CdfThreadContext *const cdf, - struct thread_data *const t) + const int have_frame_mt) { cdf->ref = dav1d_ref_create_using_pool(c->cdf_pool, sizeof(CdfContext) + sizeof(atomic_uint)); if (!cdf->ref) return DAV1D_ERR(ENOMEM); cdf->data.cdf = cdf->ref->data; - if (t) { + if (have_frame_mt) { cdf->progress = (atomic_uint *) &cdf->data.cdf[1]; atomic_init(cdf->progress, 0); - cdf->t = t; } return 0; } void dav1d_cdf_thread_ref(CdfThreadContext *const dst, @@ -4121,24 +4120,5 @@ void dav1d_cdf_thread_unref(CdfThreadContext *const cdf) { if (cdf->ref) dav1d_ref_dec(&cdf->ref); memset(cdf, 0, sizeof(*cdf)); } - -void dav1d_cdf_thread_wait(CdfThreadContext *const cdf) { - if (!cdf->t) return; - - if (atomic_load(cdf->progress)) return; - pthread_mutex_lock(&cdf->t->lock); - while (!atomic_load(cdf->progress)) - pthread_cond_wait(&cdf->t->cond, &cdf->t->lock); - pthread_mutex_unlock(&cdf->t->lock); -} - -void dav1d_cdf_thread_signal(CdfThreadContext *const cdf) { - if (!cdf->t) return; - - pthread_mutex_lock(&cdf->t->lock); - atomic_store(cdf->progress, 1); - pthread_cond_broadcast(&cdf->t->cond); - pthread_mutex_unlock(&cdf->t->lock); -} diff --git a/third_party/dav1d/src/cpu.h b/third_party/dav1d/src/cpu.h --- a/third_party/dav1d/src/cpu.h +++ b/third_party/dav1d/src/cpu.h @@ -31,10 +31,11 @@ #include "config.h" #include "common/attributes.h" #include "dav1d/common.h" +#include "dav1d/dav1d.h" #if ARCH_AARCH64 || ARCH_ARM #include "src/arm/cpu.h" #elif ARCH_PPC64LE #include "src/ppc/cpu.h" @@ -43,7 +44,8 @@ #endif void dav1d_init_cpu(void); unsigned dav1d_get_cpu_flags(void); DAV1D_API void dav1d_set_cpu_flags_mask(unsigned mask); +int dav1d_num_logical_processors(Dav1dContext *c); #endif /* DAV1D_SRC_CPU_H */ diff --git a/third_party/dav1d/src/cpu.c b/third_party/dav1d/src/cpu.c --- a/third_party/dav1d/src/cpu.c +++ b/third_party/dav1d/src/cpu.c @@ -27,10 +27,21 @@ #include "config.h" #include #include "src/cpu.h" +#include "src/log.h" + +#ifdef _WIN32 +#include +#elif defined(__linux__) +#include +#include +#elif defined(__APPLE__) +#include +#include +#endif static unsigned flags = 0; #if __has_feature(memory_sanitizer) // memory sanitizer is inherently incompatible with asm @@ -59,5 +70,38 @@ } COLD void dav1d_set_cpu_flags_mask(const unsigned mask) { flags_mask = mask; } + +COLD int dav1d_num_logical_processors(Dav1dContext *const c) { +#ifdef _WIN32 +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) + GROUP_AFFINITY affinity; + if (GetThreadGroupAffinity(GetCurrentThread(), &affinity)) { + int num_processors = 1; + while (affinity.Mask &= affinity.Mask - 1) + num_processors++; + return num_processors; + } +#else + SYSTEM_INFO system_info; + GetNativeSystemInfo(&system_info); + return system_info.dwNumberOfProcessors; +#endif +#elif defined(__linux__) +#ifdef CPU_COUNT + cpu_set_t affinity; + if (!sched_getaffinity(0, sizeof(affinity), &affinity)) + return CPU_COUNT(&affinity); +#else + return (int)sysconf(_SC_NPROCESSORS_ONLN); +#endif +#elif defined(__APPLE__) + int num_processors; + size_t length = sizeof(num_processors); + if (!sysctlbyname("hw.logicalcpu", &num_processors, &length, NULL, 0)) + return num_processors; +#endif + dav1d_log(c, "Unable to detect thread count, defaulting to single-threaded mode\n"); + return 1; +} diff --git a/third_party/dav1d/src/decode.c b/third_party/dav1d/src/decode.c --- a/third_party/dav1d/src/decode.c +++ b/third_party/dav1d/src/decode.c @@ -71,11 +71,11 @@ dq[i][2][0] = dav1d_dq_tbl[seq_hdr->hbd][vdc][0]; dq[i][2][1] = dav1d_dq_tbl[seq_hdr->hbd][vac][1]; } } -static int read_mv_component_diff(Dav1dTileContext *const t, +static int read_mv_component_diff(Dav1dTaskContext *const t, CdfMvComponent *const mv_comp, const int have_fp) { Dav1dTileState *const ts = t->ts; const Dav1dFrameContext *const f = t->f; @@ -115,11 +115,11 @@ const int diff = ((up << 3) | (fp << 1) | hp) + 1; return sign ? -diff : diff; } -static void read_mv_residual(Dav1dTileContext *const t, mv *const ref_mv, +static void read_mv_residual(Dav1dTaskContext *const t, mv *const ref_mv, CdfMvContext *const mv_cdf, const int have_fp) { switch (dav1d_msac_decode_symbol_adapt4(&t->ts->msac, t->ts->cdf.mv.joint, N_MV_JOINTS - 1)) { @@ -136,11 +136,11 @@ default: break; } } -static void read_tx_tree(Dav1dTileContext *const t, +static void read_tx_tree(Dav1dTaskContext *const t, const enum RectTxfmSize from, const int depth, uint16_t *const masks, const int x_off, const int y_off) { const Dav1dFrameContext *const f = t->f; @@ -214,11 +214,11 @@ } return max - (diff + 1); } } -static void find_matching_ref(const Dav1dTileContext *const t, +static void find_matching_ref(const Dav1dTaskContext *const t, const enum EdgeFlags intra_edge_flags, const int bw4, const int bh4, const int w4, const int h4, const int have_left, const int have_top, const int ref, uint64_t masks[2]) @@ -287,11 +287,11 @@ masks[0] |= 1ULL << 32; } #undef matches } -static void derive_warpmv(const Dav1dTileContext *const t, +static void derive_warpmv(const Dav1dTaskContext *const t, const int bw4, const int bh4, const uint64_t masks[2], const union mv mv, Dav1dWarpedMotionParams *const wmp) { int pts[8][2 /* in, out */][2 /* x, y */], np = 0; @@ -368,11 +368,11 @@ for (int n = 0; n < len; n++) if (!buf[n * 2]) return 1; return 0; } -static void read_pal_plane(Dav1dTileContext *const t, Av1Block *const b, +static void read_pal_plane(Dav1dTaskContext *const t, Av1Block *const b, const int pl, const int sz_ctx, const int bx4, const int by4) { Dav1dTileState *const ts = t->ts; const Dav1dFrameContext *const f = t->f; @@ -423,11 +423,11 @@ if (dav1d_msac_decode_bool_equi(&ts->msac)) used_cache[i++] = cache[n]; const int n_used_cache = i; // parse new entries - uint16_t *const pal = f->frame_thread.pass ? + uint16_t *const pal = t->frame_thread.pass ? f->frame_thread.pal[((t->by >> 1) + (t->bx & 1)) * (f->b4_stride >> 1) + ((t->bx >> 1) + (t->by & 1))][pl] : t->scratch.pal[pl]; if (i < pal_sz) { int prev = pal[i++] = dav1d_msac_decode_bools(&ts->msac, f->cur.p.bpc); @@ -471,19 +471,19 @@ printf("%c%02x", n ? ' ' : '[', pal[n]); printf("]\n"); } } -static void read_pal_uv(Dav1dTileContext *const t, Av1Block *const b, +static void read_pal_uv(Dav1dTaskContext *const t, Av1Block *const b, const int sz_ctx, const int bx4, const int by4) { read_pal_plane(t, b, 1, sz_ctx, bx4, by4); // V pal coding Dav1dTileState *const ts = t->ts; const Dav1dFrameContext *const f = t->f; - uint16_t *const pal = f->frame_thread.pass ? + uint16_t *const pal = t->frame_thread.pass ? f->frame_thread.pal[((t->by >> 1) + (t->bx & 1)) * (f->b4_stride >> 1) + ((t->bx >> 1) + (t->by & 1))][2] : t->scratch.pal[2]; if (dav1d_msac_decode_bool_equi(&ts->msac)) { const int bits = f->cur.p.bpc - 4 + dav1d_msac_decode_bools(&ts->msac, 2); @@ -573,11 +573,11 @@ assert(o_idx == 8); #undef add } } -static void read_pal_indices(Dav1dTileContext *const t, +static void read_pal_indices(Dav1dTaskContext *const t, uint8_t *const pal_idx, const Av1Block *const b, const int pl, const int w4, const int h4, const int bw4, const int bh4) { @@ -610,11 +610,11 @@ for (int y = h4 * 4; y < bh4 * 4; y++) memcpy(&pal_idx[y * stride], src, bw4 * 4); } } -static void read_vartx_tree(Dav1dTileContext *const t, +static void read_vartx_tree(Dav1dTaskContext *const t, Av1Block *const b, const enum BlockSize bs, const int bx4, const int by4) { const Dav1dFrameContext *const f = t->f; const uint8_t *const b_dim = dav1d_block_dimensions[bs]; @@ -672,15 +672,10 @@ const int w4, int h4, const uint8_t *ref_seg_map, const ptrdiff_t stride) { assert(f->frame_hdr->primary_ref_frame != DAV1D_PRIMARY_REF_NONE); - if (dav1d_thread_picture_wait(&f->refp[f->frame_hdr->primary_ref_frame], - (by + h4) * 4, PLANE_TYPE_BLOCK)) - { - return 8; - } unsigned seg_id = 8; ref_seg_map += by * stride + bx; do { for (int x = 0; x < w4; x++) @@ -690,19 +685,172 @@ assert(seg_id < 8); return seg_id; } -static int decode_b(Dav1dTileContext *const t, +static inline void splat_oneref_mv(const Dav1dContext *const c, + Dav1dTaskContext *const t, + const enum BlockSize bs, + const Av1Block *const b, + const int bw4, const int bh4) +{ + const enum InterPredMode mode = b->inter_mode; + const refmvs_block ALIGN(tmpl, 16) = (refmvs_block) { + .ref.ref = { b->ref[0] + 1, b->interintra_type ? 0 : -1 }, + .mv.mv[0] = b->mv[0], + .bs = bs, + .mf = (mode == GLOBALMV && imin(bw4, bh4) >= 2) | ((mode == NEWMV) * 2), + }; + c->refmvs_dsp.splat_mv(&t->rt.r[(t->by & 31) + 5], &tmpl, t->bx, bw4, bh4); +} + +static inline void splat_intrabc_mv(const Dav1dContext *const c, + Dav1dTaskContext *const t, + const enum BlockSize bs, + const Av1Block *const b, + const int bw4, const int bh4) +{ + const refmvs_block ALIGN(tmpl, 16) = (refmvs_block) { + .ref.ref = { 0, -1 }, + .mv.mv[0] = b->mv[0], + .bs = bs, + .mf = 0, + }; + c->refmvs_dsp.splat_mv(&t->rt.r[(t->by & 31) + 5], &tmpl, t->bx, bw4, bh4); +} + +static inline void splat_tworef_mv(const Dav1dContext *const c, + Dav1dTaskContext *const t, + const enum BlockSize bs, + const Av1Block *const b, + const int bw4, const int bh4) +{ + assert(bw4 >= 2 && bh4 >= 2); + const enum CompInterPredMode mode = b->inter_mode; + const refmvs_block ALIGN(tmpl, 16) = (refmvs_block) { + .ref.ref = { b->ref[0] + 1, b->ref[1] + 1 }, + .mv.mv = { b->mv[0], b->mv[1] }, + .bs = bs, + .mf = (mode == GLOBALMV_GLOBALMV) | !!((1 << mode) & (0xbc)) * 2, + }; + c->refmvs_dsp.splat_mv(&t->rt.r[(t->by & 31) + 5], &tmpl, t->bx, bw4, bh4); +} + +static inline void splat_intraref(const Dav1dContext *const c, + Dav1dTaskContext *const t, + const enum BlockSize bs, + const int bw4, const int bh4) +{ + const refmvs_block ALIGN(tmpl, 16) = (refmvs_block) { + .ref.ref = { 0, -1 }, + .mv.mv[0].n = INVALID_MV, + .bs = bs, + .mf = 0, + }; + c->refmvs_dsp.splat_mv(&t->rt.r[(t->by & 31) + 5], &tmpl, t->bx, bw4, bh4); +} + +static inline void mc_lowest_px(int *const dst, const int by4, const int bh4, + const int mvy, const int ss_ver, + const struct ScalableMotionParams *const smp) +{ + const int v_mul = 4 >> ss_ver; + if (!smp->scale) { + const int my = mvy >> (3 + ss_ver), dy = mvy & (15 >> !ss_ver); + *dst = imax(*dst, (by4 + bh4) * v_mul + my + 4 * !!dy); + } else { + int y = (by4 * v_mul << 4) + mvy * (1 << !ss_ver); + const int64_t tmp = (int64_t)(y) * smp->scale + (smp->scale - 0x4000) * 8; + y = apply_sign64((int)((llabs(tmp) + 128) >> 8), tmp) + 32; + const int bottom = ((y + (bh4 * v_mul - 1) * smp->step) >> 10) + 1 + 4; + *dst = imax(*dst, bottom); + } +} + +static inline void affine_lowest_px(Dav1dTaskContext *const t, + int *const dst, const int is_chroma, + const uint8_t *const b_dim, + const Dav1dWarpedMotionParams *const wmp) +{ + const Dav1dFrameContext *const f = t->f; + const int ss_ver = is_chroma && f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; + const int ss_hor = is_chroma && f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I444; + const int h_mul = 4 >> ss_hor, v_mul = 4 >> ss_ver; + assert(!((b_dim[0] * h_mul) & 7) && !((b_dim[1] * v_mul) & 7)); + const int32_t *const mat = wmp->matrix; + const int y = b_dim[1] * v_mul - 8; // lowest y + + const int src_y = t->by * 4 + ((y + 4) << ss_ver); + const int64_t mat5_y = (int64_t) mat[5] * src_y + mat[1]; + // check left- and right-most blocks + for (int x = 0; x < b_dim[0] * h_mul; x += imax(8, b_dim[0] * h_mul - 8)) { + // calculate transformation relative to center of 8x8 block in + // luma pixel units + const int src_x = t->bx * 4 + ((x + 4) << ss_hor); + const int64_t mvy = ((int64_t) mat[4] * src_x + mat5_y) >> ss_ver; + const int dy = (int) (mvy >> 16) - 4; + *dst = imax(*dst, dy + 4 + 8); + } +} + +static void obmc_lowest_px(Dav1dTaskContext *const t, + int (*const dst)[2], const int is_chroma, + const uint8_t *const b_dim, + const int bx4, const int by4, const int w4, const int h4) +{ + assert(!(t->bx & 1) && !(t->by & 1)); + const Dav1dFrameContext *const f = t->f; + /*const*/ refmvs_block **r = &t->rt.r[(t->by & 31) + 5]; + const int ss_ver = is_chroma && f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; + const int ss_hor = is_chroma && f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I444; + const int h_mul = 4 >> ss_hor, v_mul = 4 >> ss_ver; + + if (t->by > t->ts->tiling.row_start && + (!is_chroma || b_dim[0] * h_mul + b_dim[1] * v_mul >= 16)) + { + for (int i = 0, x = 0; x < w4 && i < imin(b_dim[2], 4); ) { + // only odd blocks are considered for overlap handling, hence +1 + const refmvs_block *const a_r = &r[-1][t->bx + x + 1]; + const uint8_t *const a_b_dim = dav1d_block_dimensions[a_r->bs]; + + if (a_r->ref.ref[0] > 0) { + const int oh4 = imin(b_dim[1], 16) >> 1; + mc_lowest_px(&dst[a_r->ref.ref[0] - 1][is_chroma], t->by, + (oh4 * 3 + 3) >> 2, a_r->mv.mv[0].y, ss_ver, + &f->svc[a_r->ref.ref[0] - 1][1]); + i++; + } + x += imax(a_b_dim[0], 2); + } + } + + if (t->bx > t->ts->tiling.col_start) + for (int i = 0, y = 0; y < h4 && i < imin(b_dim[3], 4); ) { + // only odd blocks are considered for overlap handling, hence +1 + const refmvs_block *const l_r = &r[y + 1][t->bx - 1]; + const uint8_t *const l_b_dim = dav1d_block_dimensions[l_r->bs]; + + if (l_r->ref.ref[0] > 0) { + const int oh4 = iclip(l_b_dim[1], 2, b_dim[1]); + mc_lowest_px(&dst[l_r->ref.ref[0] - 1][is_chroma], + t->by + y, oh4, l_r->mv.mv[0].y, ss_ver, + &f->svc[l_r->ref.ref[0] - 1][1]); + i++; + } + y += imax(l_b_dim[1], 2); + } +} + +static int decode_b(Dav1dTaskContext *const t, const enum BlockLevel bl, const enum BlockSize bs, const enum BlockPartition bp, const enum EdgeFlags intra_edge_flags) { Dav1dTileState *const ts = t->ts; const Dav1dFrameContext *const f = t->f; - Av1Block b_mem, *const b = f->frame_thread.pass ? + Av1Block b_mem, *const b = t->frame_thread.pass ? &f->frame_thread.b[t->by * f->b4_stride + t->bx] : &b_mem; const uint8_t *const b_dim = dav1d_block_dimensions[bs]; const int bx4 = t->bx & 31, by4 = t->by & 31; const int ss_ver = f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; const int ss_hor = f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I444; @@ -714,11 +862,11 @@ const int have_top = t->by > ts->tiling.row_start; const int has_chroma = f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I400 && (bw4 > ss_hor || t->bx & 1) && (bh4 > ss_ver || t->by & 1); - if (f->frame_thread.pass == 2) { + if (t->frame_thread.pass == 2) { if (b->intra) { f->bd_fn.recon_b_intra(t, bs, intra_edge_flags, b); const enum IntraPredMode y_mode_nofilt = b->y_mode == FILTER_PRED ? DC_PRED : b->y_mode; @@ -1172,27 +1320,29 @@ b->y_mode, b->y_angle, ts->msac.rng); } if (b->pal_sz[0]) { uint8_t *pal_idx; - if (f->frame_thread.pass) { - assert(ts->frame_thread.pal_idx); - pal_idx = ts->frame_thread.pal_idx; - ts->frame_thread.pal_idx += bw4 * bh4 * 16; + if (t->frame_thread.pass) { + const int p = t->frame_thread.pass & 1; + assert(ts->frame_thread[p].pal_idx); + pal_idx = ts->frame_thread[p].pal_idx; + ts->frame_thread[p].pal_idx += bw4 * bh4 * 16; } else pal_idx = t->scratch.pal_idx; read_pal_indices(t, pal_idx, b, 0, w4, h4, bw4, bh4); if (DEBUG_BLOCK_INFO) printf("Post-y-pal-indices: r=%d\n", ts->msac.rng); } if (has_chroma && b->pal_sz[1]) { uint8_t *pal_idx; - if (f->frame_thread.pass) { - assert(ts->frame_thread.pal_idx); - pal_idx = ts->frame_thread.pal_idx; - ts->frame_thread.pal_idx += cbw4 * cbh4 * 16; + if (t->frame_thread.pass) { + const int p = t->frame_thread.pass & 1; + assert(ts->frame_thread[p].pal_idx); + pal_idx = ts->frame_thread[p].pal_idx; + ts->frame_thread[p].pal_idx += cbw4 * cbh4 * 16; } else pal_idx = &t->scratch.pal_idx[bw4 * bh4 * 16]; read_pal_indices(t, pal_idx, b, 1, cw4, ch4, cbw4, cbh4); if (DEBUG_BLOCK_INFO) printf("Post-uv-pal-indices: r=%d\n", ts->msac.rng); @@ -1220,11 +1370,11 @@ if (DEBUG_BLOCK_INFO) printf("Post-tx[%d]: r=%d\n", b->tx, ts->msac.rng); } // reconstruction - if (f->frame_thread.pass == 1) { + if (t->frame_thread.pass == 1) { f->bd_fn.read_coef_blocks(t, bs, b); } else { f->bd_fn.recon_b_intra(t, bs, intra_edge_flags, b); } @@ -1264,11 +1414,11 @@ b->y_mode == FILTER_PRED ? DC_PRED : b->y_mode; case_set(bh4, l., 1, by4); case_set(bw4, a->, 0, bx4); #undef set_ctx if (b->pal_sz[0]) { - uint16_t *const pal = f->frame_thread.pass ? + uint16_t *const pal = t->frame_thread.pass ? f->frame_thread.pal[((t->by >> 1) + (t->bx & 1)) * (f->b4_stride >> 1) + ((t->bx >> 1) + (t->by & 1))][0] : t->scratch.pal[0]; for (int x = 0; x < bw4; x++) memcpy(t->al_pal[0][bx4 + x][0], pal, 16); for (int y = 0; y < bh4; y++) @@ -1279,11 +1429,11 @@ rep_macro(type, t->dir uvmode, off, mul * b->uv_mode) case_set(cbh4, l., 1, cby4); case_set(cbw4, a->, 0, cbx4); #undef set_ctx if (b->pal_sz[1]) { - const uint16_t (*const pal)[8] = f->frame_thread.pass ? + const uint16_t (*const pal)[8] = t->frame_thread.pass ? f->frame_thread.pal[((t->by >> 1) + (t->bx & 1)) * (f->b4_stride >> 1) + ((t->bx >> 1) + (t->by & 1))] : t->scratch.pal; // see aomedia bug 2183 for why we use luma coordinates here for (int pl = 1; pl <= 2; pl++) { @@ -1292,13 +1442,12 @@ for (int y = 0; y < bh4; y++) memcpy(t->al_pal[1][by4 + y][pl], pal[pl], 16); } } } - if (IS_INTER_OR_SWITCH(f->frame_hdr) || f->frame_hdr->allow_intrabc) { - splat_intraref(&t->rt, t->by, t->bx, bs); - } + if (IS_INTER_OR_SWITCH(f->frame_hdr) || f->frame_hdr->allow_intrabc) + splat_intraref(f->c, t, bs, bw4, bh4); } else if (IS_KEY_OR_INTRA(f->frame_hdr)) { // intra block copy refmvs_candidate mvstack[8]; int n_mvs, ctx; dav1d_refmvs_find(&t->rt, mvstack, &n_mvs, &ctx, @@ -1383,18 +1532,18 @@ b->mv[0].y, b->mv[0].x, ref.y, ref.x, mvstack[0].mv.mv[0].y, mvstack[0].mv.mv[0].x, ts->msac.rng); read_vartx_tree(t, b, bs, bx4, by4); // reconstruction - if (f->frame_thread.pass == 1) { + if (t->frame_thread.pass == 1) { f->bd_fn.read_coef_blocks(t, bs, b); b->filter2d = FILTER_2D_BILINEAR; } else { if (f->bd_fn.recon_b_inter(t, bs, b)) return -1; } - splat_intrabc_mv(&t->rt, t->by, t->bx, bs, b->mv[0]); + splat_intrabc_mv(f->c, t, bs, b, bw4, bh4); #define set_ctx(type, dir, diridx, off, mul, rep_macro) \ rep_macro(type, t->dir tx_intra, off, mul * b_dim[2 + diridx]); \ rep_macro(type, t->dir mode, off, mul * DC_PRED); \ rep_macro(type, t->dir pal_sz, off, 0); \ @@ -1847,11 +1996,11 @@ signabs(t->warpmv.u.p.beta), signabs(t->warpmv.u.p.gamma), signabs(t->warpmv.u.p.delta), b->mv[0].y, b->mv[0].x); #undef signabs - if (f->frame_thread.pass) { + if (t->frame_thread.pass) { if (t->warpmv.type == DAV1D_WM_TYPE_AFFINE) { b->matrix[0] = t->warpmv.matrix[2] - 0x10000; b->matrix[1] = t->warpmv.matrix[3]; b->matrix[2] = t->warpmv.matrix[4]; b->matrix[3] = t->warpmv.matrix[5] - 0x10000; @@ -1860,12 +2009,12 @@ } } } if (DEBUG_BLOCK_INFO) - printf("Post-motionmode[%d]: r=%d [mask: 0x%" PRIu64 "x/0x%" - PRIu64 "x]\n", b->motion_mode, ts->msac.rng, mask[0], + printf("Post-motionmode[%d]: r=%d [mask: 0x%" PRIx64 "/0x%" + PRIx64 "]\n", b->motion_mode, ts->msac.rng, mask[0], mask[1]); } else { b->motion_mode = MM_TRANSLATION; } } @@ -1907,11 +2056,11 @@ b->filter2d = dav1d_filter_2d[filter[1]][filter[0]]; read_vartx_tree(t, b, bs, bx4, by4); // reconstruction - if (f->frame_thread.pass == 1) { + if (t->frame_thread.pass == 1) { f->bd_fn.read_coef_blocks(t, bs, b); } else { if (f->bd_fn.recon_b_inter(t, bs, b)) return -1; } @@ -1932,18 +2081,14 @@ has_chroma ? &t->a->tx_lpf_uv[cbx4] : NULL, has_chroma ? &t->l.tx_lpf_uv[cby4] : NULL); } // context updates - if (is_comp) { - splat_tworef_mv(&t->rt, t->by, t->bx, bs, b->inter_mode, - (refmvs_refpair) { .ref = { b->ref[0], b->ref[1] }}, - (refmvs_mvpair) { .mv = { [0] = b->mv[0], [1] = b->mv[1] }}); - } else { - splat_oneref_mv(&t->rt, t->by, t->bx, bs, b->inter_mode, - b->ref[0], b->mv[0], b->interintra_type); - } + if (is_comp) + splat_tworef_mv(f->c, t, bs, b, bw4, bh4); + else + splat_oneref_mv(f->c, t, bs, b, bw4, bh4); #define set_ctx(type, dir, diridx, off, mul, rep_macro) \ rep_macro(type, t->dir seg_pred, off, mul * seg_pred); \ rep_macro(type, t->dir skip_mode, off, mul * b->skip_mode); \ rep_macro(type, t->dir intra, off, 0); \ @@ -1993,27 +2138,131 @@ if (bw4 == 32) // this should be mask >> 16, but it's 0xffffffff anyway (*noskip_mask)[1] |= mask; } } + if (t->frame_thread.pass == 1 && !b->intra && IS_INTER_OR_SWITCH(f->frame_hdr)) { + const int sby = (t->by - ts->tiling.row_start) >> f->sb_shift; + int (*const lowest_px)[2] = ts->lowest_pixel[sby]; + + // keep track of motion vectors for each reference + if (b->comp_type == COMP_INTER_NONE) { + // y + if (imin(bw4, bh4) > 1 && + ((b->inter_mode == GLOBALMV && f->gmv_warp_allowed[b->ref[0]]) || + (b->motion_mode == MM_WARP && t->warpmv.type > DAV1D_WM_TYPE_TRANSLATION))) + { + affine_lowest_px(t, &lowest_px[b->ref[0]][0], 0, b_dim, + b->motion_mode == MM_WARP ? &t->warpmv : + &f->frame_hdr->gmv[b->ref[0]]); + } else { + mc_lowest_px(&lowest_px[b->ref[0]][0], t->by, bh4, b->mv[0].y, + 0, &f->svc[b->ref[0]][1]); + if (b->motion_mode == MM_OBMC) { + obmc_lowest_px(t, lowest_px, 0, b_dim, bx4, by4, w4, h4); + } + } + + // uv + if (has_chroma) { + // sub8x8 derivation + int is_sub8x8 = bw4 == ss_hor || bh4 == ss_ver; + refmvs_block *const *r; + if (is_sub8x8) { + assert(ss_hor == 1); + r = &t->rt.r[(t->by & 31) + 5]; + if (bw4 == 1) is_sub8x8 &= r[0][t->bx - 1].ref.ref[0] > 0; + if (bh4 == ss_ver) is_sub8x8 &= r[-1][t->bx].ref.ref[0] > 0; + if (bw4 == 1 && bh4 == ss_ver) + is_sub8x8 &= r[-1][t->bx - 1].ref.ref[0] > 0; + } + + // chroma prediction + if (is_sub8x8) { + assert(ss_hor == 1); + if (bw4 == 1 && bh4 == ss_ver) { + const refmvs_block *const rr = &r[-1][t->bx - 1]; + mc_lowest_px(&lowest_px[rr->ref.ref[0] - 1][1], + t->by - 1, bh4, rr->mv.mv[0].y, ss_ver, + &f->svc[rr->ref.ref[0] - 1][1]); + } + if (bw4 == 1) { + const refmvs_block *const rr = &r[0][t->bx - 1]; + mc_lowest_px(&lowest_px[rr->ref.ref[0] - 1][1], + t->by, bh4, rr->mv.mv[0].y, ss_ver, + &f->svc[rr->ref.ref[0] - 1][1]); + } + if (bh4 == ss_ver) { + const refmvs_block *const rr = &r[-1][t->bx]; + mc_lowest_px(&lowest_px[rr->ref.ref[0] - 1][1], + t->by - 1, bh4, rr->mv.mv[0].y, ss_ver, + &f->svc[rr->ref.ref[0] - 1][1]); + } + mc_lowest_px(&lowest_px[b->ref[0]][1], t->by, bh4, + b->mv[0].y, ss_ver, &f->svc[b->ref[0]][1]); + } else { + if (imin(cbw4, cbh4) > 1 && + ((b->inter_mode == GLOBALMV && f->gmv_warp_allowed[b->ref[0]]) || + (b->motion_mode == MM_WARP && t->warpmv.type > DAV1D_WM_TYPE_TRANSLATION))) + { + affine_lowest_px(t, &lowest_px[b->ref[0]][1], 1, b_dim, + b->motion_mode == MM_WARP ? &t->warpmv : + &f->frame_hdr->gmv[b->ref[0]]); + } else { + mc_lowest_px(&lowest_px[b->ref[0]][1], + t->by & ~ss_ver, bh4 << (bh4 == ss_ver), + b->mv[0].y, ss_ver, &f->svc[b->ref[0]][1]); + if (b->motion_mode == MM_OBMC) { + obmc_lowest_px(t, lowest_px, 1, b_dim, bx4, by4, w4, h4); + } + } + } + } + } else { + // y + for (int i = 0; i < 2; i++) { + if (b->inter_mode == GLOBALMV_GLOBALMV && f->gmv_warp_allowed[b->ref[i]]) { + affine_lowest_px(t, &lowest_px[b->ref[i]][0], 0, b_dim, + &f->frame_hdr->gmv[b->ref[i]]); + } else { + mc_lowest_px(&lowest_px[b->ref[i]][0], t->by, bh4, + b->mv[i].y, 0, &f->svc[b->ref[i]][1]); + } + } + + // uv + if (has_chroma) for (int i = 0; i < 2; i++) { + if (b->inter_mode == GLOBALMV_GLOBALMV && + imin(cbw4, cbh4) > 1 && f->gmv_warp_allowed[b->ref[i]]) + { + affine_lowest_px(t, &lowest_px[b->ref[i]][1], 1, b_dim, + &f->frame_hdr->gmv[b->ref[i]]); + } else { + mc_lowest_px(&lowest_px[b->ref[i]][1], t->by, bh4, + b->mv[i].y, ss_ver, &f->svc[b->ref[i]][1]); + } + } + } + } + return 0; } #if __has_feature(memory_sanitizer) #include -static int checked_decode_b(Dav1dTileContext *const t, +static int checked_decode_b(Dav1dTaskContext *const t, const enum BlockLevel bl, const enum BlockSize bs, const enum BlockPartition bp, const enum EdgeFlags intra_edge_flags) { const Dav1dFrameContext *const f = t->f; const int err = decode_b(t, bl, bs, bp, intra_edge_flags); - if (err == 0 && !(f->frame_thread.pass & 1)) { + if (err == 0 && !(t->frame_thread.pass & 1)) { const int ss_ver = f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; const int ss_hor = f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I444; const uint8_t *const b_dim = dav1d_block_dimensions[bs]; const int bw4 = b_dim[0], bh4 = b_dim[1]; const int w4 = imin(bw4, f->bw - t->bx), h4 = imin(bh4, f->bh - t->by); @@ -2049,11 +2298,11 @@ #define decode_b checked_decode_b #endif /* defined(__has_feature) */ -static int decode_sb(Dav1dTileContext *const t, const enum BlockLevel bl, +static int decode_sb(Dav1dTaskContext *const t, const enum BlockLevel bl, const EdgeNode *const node) { const Dav1dFrameContext *const f = t->f; const int hsz = 16 >> bl; const int have_h_split = f->bw > t->bx + hsz; @@ -2065,22 +2314,22 @@ } uint16_t *pc; enum BlockPartition bp; int ctx, bx8, by8; - if (f->frame_thread.pass != 2) { + if (t->frame_thread.pass != 2) { if (0 && bl == BL_64X64) printf("poc=%d,y=%d,x=%d,bl=%d,r=%d\n", f->frame_hdr->frame_offset, t->by, t->bx, bl, t->ts->msac.rng); bx8 = (t->bx & 31) >> 1; by8 = (t->by & 31) >> 1; ctx = get_partition_ctx(t->a, &t->l, bl, by8, bx8); pc = t->ts->cdf.m.partition[bl][ctx]; } if (have_h_split && have_v_split) { - if (f->frame_thread.pass == 2) { + if (t->frame_thread.pass == 2) { const Av1Block *const b = &f->frame_thread.b[t->by * f->b4_stride + t->bx]; bp = b->bl == bl ? b->bp : PARTITION_SPLIT; } else { bp = dav1d_msac_decode_symbol_adapt16(&t->ts->msac, pc, dav1d_partition_type_count[bl]); @@ -2248,11 +2497,11 @@ } default: assert(0); } } else if (have_h_split) { unsigned is_split; - if (f->frame_thread.pass == 2) { + if (t->frame_thread.pass == 2) { const Av1Block *const b = &f->frame_thread.b[t->by * f->b4_stride + t->bx]; is_split = b->bl != bl; } else { is_split = dav1d_msac_decode_bool(&t->ts->msac, gather_top_partition_prob(pc, bl)); @@ -2277,11 +2526,11 @@ return -1; } } else { assert(have_v_split); unsigned is_split; - if (f->frame_thread.pass == 2) { + if (t->frame_thread.pass == 2) { const Av1Block *const b = &f->frame_thread.b[t->by * f->b4_stride + t->bx]; is_split = b->bl != bl; } else { is_split = dav1d_msac_decode_bool(&t->ts->msac, gather_left_partition_prob(pc, bl)); @@ -2307,11 +2556,11 @@ PARTITION_V, node->v[0])) return -1; } } - if (f->frame_thread.pass != 2 && (bp != PARTITION_SPLIT || bl == BL_8X8)) { + if (t->frame_thread.pass != 2 && (bp != PARTITION_SPLIT || bl == BL_8X8)) { #define set_ctx(type, dir, diridx, off, mul, rep_macro) \ rep_macro(type, t->a->partition, bx8, mul * dav1d_al_part_ctx[0][bl][bp]); \ rep_macro(type, t->l.partition, by8, mul * dav1d_al_part_ctx[1][bl][bp]) case_set_upto16(hsz,,,); #undef set_ctx @@ -2367,18 +2616,19 @@ const int row_sb_start = f->frame_hdr->tiling.row_start_sb[tile_row]; const int row_sb_end = f->frame_hdr->tiling.row_start_sb[tile_row + 1]; const int sb_shift = f->sb_shift; const uint8_t *const size_mul = ss_size_mul[f->cur.p.layout]; - ts->frame_thread.pal_idx = f->frame_thread.pal_idx ? - &f->frame_thread.pal_idx[(size_t)tile_start_off * size_mul[1] / 4] : - NULL; - - ts->frame_thread.cf = f->frame_thread.cf ? - (uint8_t*)f->frame_thread.cf + - (((size_t)tile_start_off * size_mul[0]) >> !f->seq_hdr->hbd) : - NULL; + for (int p = 0; p < 2; p++) { + ts->frame_thread[p].pal_idx = f->frame_thread.pal_idx ? + &f->frame_thread.pal_idx[(size_t)tile_start_off * size_mul[1] / 4] : + NULL; + ts->frame_thread[p].cf = f->frame_thread.cf ? + (uint8_t*)f->frame_thread.cf + + (((size_t)tile_start_off * size_mul[0]) >> !f->seq_hdr->hbd) : + NULL; + } dav1d_cdf_thread_copy(&ts->cdf, &f->in_cdf); ts->last_qidx = f->frame_hdr->quant.yac; memset(ts->last_delta_lf, 0, sizeof(ts->last_delta_lf)); @@ -2429,15 +2679,17 @@ ts->lr_ref[p]->filter_h[2] = 15; ts->lr_ref[p]->sgr_weights[0] = -32; ts->lr_ref[p]->sgr_weights[1] = 31; } - if (f->n_tc > 1) - atomic_init(&ts->progress, row_sb_start); + if (f->c->n_tc > 1) { + for (int p = 0; p < 2; p++) + atomic_init(&ts->progress[p], row_sb_start); + } } -static void read_restoration_info(Dav1dTileContext *const t, +static void read_restoration_info(Dav1dTaskContext *const t, Av1RestorationUnit *const lr, const int p, const enum Dav1dRestorationType frame_type) { const Dav1dFrameContext *const f = t->f; Dav1dTileState *const ts = t->ts; @@ -2499,11 +2751,11 @@ p, lr->sgr_idx, lr->sgr_weights[0], lr->sgr_weights[1], ts->msac.rng); } } -int dav1d_decode_tile_sbrow(Dav1dTileContext *const t) { +int dav1d_decode_tile_sbrow(Dav1dTaskContext *const t) { const Dav1dFrameContext *const f = t->f; const enum BlockLevel root_bl = f->seq_hdr->sb128 ? BL_128X128 : BL_64X64; Dav1dTileState *const ts = t->ts; const Dav1dContext *const c = f->c; const int sb_step = f->sb_step; @@ -2513,17 +2765,26 @@ if (IS_INTER_OR_SWITCH(f->frame_hdr) || f->frame_hdr->allow_intrabc) { dav1d_refmvs_tile_sbrow_init(&t->rt, &f->rf, ts->tiling.col_start, ts->tiling.col_end, ts->tiling.row_start, ts->tiling.row_end, t->by >> f->sb_shift, - ts->tiling.row); + ts->tiling.row, t->frame_thread.pass); } - reset_context(&t->l, IS_KEY_OR_INTRA(f->frame_hdr), f->frame_thread.pass); - if (f->frame_thread.pass == 2) { + if (IS_INTER_OR_SWITCH(f->frame_hdr) && c->n_fc > 1) { + const int sby = (t->by - ts->tiling.row_start) >> f->sb_shift; + int (*const lowest_px)[2] = ts->lowest_pixel[sby]; + for (int n = 0; n < 7; n++) + for (int m = 0; m < 2; m++) + lowest_px[n][m] = INT_MIN; + } + + reset_context(&t->l, IS_KEY_OR_INTRA(f->frame_hdr), t->frame_thread.pass); + if (t->frame_thread.pass == 2) { + const int off_2pass = c->n_tc > 1 ? f->sb128w * f->frame_hdr->tiling.rows : 0; for (t->bx = ts->tiling.col_start, - t->a = f->a + col_sb128_start + tile_row * f->sb128w; + t->a = f->a + off_2pass + col_sb128_start + tile_row * f->sb128w; t->bx < ts->tiling.col_end; t->bx += sb_step) { if (atomic_load_explicit(c->flush, memory_order_acquire)) return 1; if (decode_sb(t, root_bl, c->intra_edge.root[root_bl])) @@ -2536,17 +2797,11 @@ } // error out on symbol decoder overread if (ts->msac.cnt < -15) return 1; - if (f->n_tc > 1 && f->frame_hdr->use_ref_frame_mvs) { - if (c->n_fc > 1) for (int n = 0; n < 7; n++) - if (dav1d_thread_picture_wait(&f->refp[n], 4 * (t->by + sb_step), - PLANE_TYPE_BLOCK)) - { - return 1; - } + if (f->c->n_tc > 1 && f->frame_hdr->use_ref_frame_mvs) { dav1d_refmvs_load_tmvs(&f->rf, ts->tiling.row, ts->tiling.col_start >> 1, ts->tiling.col_end >> 1, t->by >> 1, (t->by + sb_step) >> 1); } memset(t->pal_sz_uv[1], 0, sizeof(*t->pal_sz_uv)); @@ -2627,18 +2882,18 @@ t->a++; t->lf_mask++; } } - if (f->n_tc > 1 && IS_INTER_OR_SWITCH(f->frame_hdr)) { + if (f->seq_hdr->ref_frame_mvs && f->c->n_tc > 1 && IS_INTER_OR_SWITCH(f->frame_hdr)) { dav1d_refmvs_save_tmvs(&t->rt, ts->tiling.col_start >> 1, ts->tiling.col_end >> 1, t->by >> 1, (t->by + sb_step) >> 1); } // backup pre-loopfilter pixels for intra prediction of the next sbrow - if (f->frame_thread.pass != 1) + if (t->frame_thread.pass != 1) f->bd_fn.backup_ipred_edge(t); // backup t->a/l.tx_lpf_y/uv at tile boundaries to use them to "fix" // up the initial value in neighbour tiles when running the loopfilter int align_h = (f->bh + 31) & ~31; @@ -2650,102 +2905,52 @@ &t->l.tx_lpf_uv[(t->by & 16) >> ss_ver], sb_step >> ss_ver); return 0; } -int dav1d_decode_frame(Dav1dFrameContext *const f) { +int dav1d_decode_frame_init(Dav1dFrameContext *const f) { const Dav1dContext *const c = f->c; int retval = DAV1D_ERR(ENOMEM); - if (f->n_tc > 1) { - const int titsati_sz = f->frame_hdr->tiling.cols * f->sbh; - if (titsati_sz != f->tile_thread.titsati_sz) { - freep(&f->tile_thread.task_idx_to_sby_and_tile_idx); - f->tile_thread.task_idx_to_sby_and_tile_idx = - malloc(sizeof(*f->tile_thread.task_idx_to_sby_and_tile_idx) * - titsati_sz); - if (!f->tile_thread.task_idx_to_sby_and_tile_idx) { - f->tile_thread.titsati_sz = 0; - goto error; - } - f->tile_thread.titsati_sz = titsati_sz; + if (f->sbh > f->lf.start_of_tile_row_sz) { + free(f->lf.start_of_tile_row); + f->lf.start_of_tile_row = malloc(f->sbh * sizeof(uint8_t)); + if (!f->lf.start_of_tile_row) { + f->lf.start_of_tile_row_sz = 0; + goto error; } - if (f->tile_thread.titsati_init[0] != f->frame_hdr->tiling.cols || - f->tile_thread.titsati_init[1] != f->frame_hdr->tiling.rows || - memcmp(f->frame_hdr->tiling.row_start_sb, f->tile_thread.titsati_index_rows, - sizeof(*f->tile_thread.titsati_index_rows) * - (f->frame_hdr->tiling.rows + 1))) - { - for (int tile_row = 0, task_idx = 0; - tile_row < f->frame_hdr->tiling.rows; tile_row++) - { - for (int sby = f->frame_hdr->tiling.row_start_sb[tile_row]; - sby < f->frame_hdr->tiling.row_start_sb[tile_row + 1]; sby++) - { - for (int tile_col = 0; tile_col < f->frame_hdr->tiling.cols; - tile_col++, task_idx++) - { - f->tile_thread.task_idx_to_sby_and_tile_idx[task_idx][0] = sby; - f->tile_thread.task_idx_to_sby_and_tile_idx[task_idx][1] = - tile_row * f->frame_hdr->tiling.cols + tile_col; - } - } - } - f->tile_thread.titsati_init[0] = f->frame_hdr->tiling.cols; - f->tile_thread.titsati_init[1] = f->frame_hdr->tiling.rows; - memcpy(f->tile_thread.titsati_index_rows, f->frame_hdr->tiling.row_start_sb, - sizeof(*f->tile_thread.titsati_index_rows) * - (f->frame_hdr->tiling.rows + 1)); - } + f->lf.start_of_tile_row_sz = f->sbh; + } + int sby = 0; + for (int tile_row = 0; tile_row < f->frame_hdr->tiling.rows; tile_row++) { + f->lf.start_of_tile_row[sby++] = tile_row; + while (sby < f->frame_hdr->tiling.row_start_sb[tile_row + 1]) + f->lf.start_of_tile_row[sby++] = 0; } const int n_ts = f->frame_hdr->tiling.cols * f->frame_hdr->tiling.rows; if (n_ts != f->n_ts) { if (c->n_fc > 1) { freep(&f->frame_thread.tile_start_off); f->frame_thread.tile_start_off = malloc(sizeof(*f->frame_thread.tile_start_off) * n_ts); if (!f->frame_thread.tile_start_off) { - for (int n = 0; n < f->n_ts; n++) { - Dav1dTileState *const ts = &f->ts[n]; - pthread_cond_destroy(&ts->tile_thread.cond); - pthread_mutex_destroy(&ts->tile_thread.lock); - } f->n_ts = 0; goto error; } } Dav1dTileState *ts_new = dav1d_alloc_aligned(sizeof(*f->ts) * n_ts, 32); if (!ts_new) goto error; - if (n_ts > f->n_ts) { - if (f->ts) { - memcpy(ts_new, f->ts, sizeof(*f->ts) * f->n_ts); - dav1d_free_aligned(f->ts); - } - f->ts = ts_new; - for (int n = f->n_ts; n < n_ts; f->n_ts = ++n) { - Dav1dTileState *const ts = &f->ts[n]; - if (pthread_mutex_init(&ts->tile_thread.lock, NULL)) goto error; - if (pthread_cond_init(&ts->tile_thread.cond, NULL)) { - pthread_mutex_destroy(&ts->tile_thread.lock); - goto error; - } - } - } else { - for (int n = n_ts; n < f->n_ts; n++) { - Dav1dTileState *const ts = &f->ts[n]; - pthread_cond_destroy(&ts->tile_thread.cond); - pthread_mutex_destroy(&ts->tile_thread.lock); - } - memcpy(ts_new, f->ts, sizeof(*f->ts) * n_ts); + if (f->ts) { + memcpy(ts_new, f->ts, sizeof(*f->ts) * imin(n_ts, f->n_ts)); dav1d_free_aligned(f->ts); - f->n_ts = n_ts; - f->ts = ts_new; } + f->n_ts = n_ts; + f->ts = ts_new; } - const int a_sz = f->sb128w * f->frame_hdr->tiling.rows; + const int a_sz = f->sb128w * f->frame_hdr->tiling.rows * (1 + (c->n_fc > 1 && c->n_tc > 1)); if (a_sz != f->a_sz) { freep(&f->a); f->a = malloc(sizeof(*f->a) * a_sz); if (!f->a) { f->a_sz = 0; @@ -2768,10 +2973,33 @@ f->frame_thread.tile_start_off[tile_idx++] = row_off + b_diff * f->frame_hdr->tiling.col_start_sb[tile_col] * f->sb_step * 4; } } + const int lowest_pixel_mem_sz = f->frame_hdr->tiling.cols * f->sbh; + if (lowest_pixel_mem_sz != f->tile_thread.lowest_pixel_mem_sz) { + free(f->tile_thread.lowest_pixel_mem); + f->tile_thread.lowest_pixel_mem = + malloc(lowest_pixel_mem_sz * sizeof(*f->tile_thread.lowest_pixel_mem)); + if (!f->tile_thread.lowest_pixel_mem) { + f->tile_thread.lowest_pixel_mem_sz = 0; + goto error; + } + f->tile_thread.lowest_pixel_mem_sz = lowest_pixel_mem_sz; + } + int (*lowest_pixel_ptr)[7][2] = f->tile_thread.lowest_pixel_mem; + for (int tile_row = 0, tile_row_base = 0; tile_row < f->frame_hdr->tiling.rows; + tile_row++, tile_row_base += f->frame_hdr->tiling.cols) + { + const int tile_row_sb_h = f->frame_hdr->tiling.row_start_sb[tile_row + 1] - + f->frame_hdr->tiling.row_start_sb[tile_row]; + for (int tile_col = 0; tile_col < f->frame_hdr->tiling.cols; tile_col++) { + f->ts[tile_row_base + tile_col].lowest_pixel = lowest_pixel_ptr; + lowest_pixel_ptr += tile_row_sb_h; + } + } + const int cf_sz = (num_sb128 * size_mul[0]) << hbd; if (cf_sz != f->frame_thread.cf_sz) { dav1d_freep_aligned(&f->frame_thread.cf); f->frame_thread.cf = dav1d_alloc_aligned((size_t)cf_sz * 128 * 128 / 2, 32); @@ -2852,27 +3080,28 @@ f->lf.cdef_line_sz[0] = (int) y_stride; f->lf.cdef_line_sz[1] = (int) uv_stride; } + const int num_lines = c->n_tc > 1 ? f->sbh * (4 << f->seq_hdr->sb128) : 12; const int lr_line_sz = ((f->sr_cur.p.p.w + 31) & ~31) << hbd; - if (lr_line_sz != f->lf.lr_line_sz) { + const size_t lr_plane_sz = num_lines * lr_line_sz; + if (lr_plane_sz != f->lf.lr_plane_sz) { dav1d_freep_aligned(&f->lf.lr_lpf_line[0]); - const int num_lines = c->n_pfc > 1 ? f->sbh * (4 << f->seq_hdr->sb128) : 12; // lr simd may overread the input, so slightly over-allocate the lpf buffer - uint8_t *lr_ptr = dav1d_alloc_aligned(lr_line_sz * num_lines * 3 + 64, 32); + uint8_t *lr_ptr = dav1d_alloc_aligned(lr_plane_sz * 3 + 64, 32); if (!lr_ptr) { - f->lf.lr_line_sz = 0; + f->lf.lr_plane_sz = 0; goto error; } for (int pl = 0; pl <= 2; pl++) { f->lf.lr_lpf_line[pl] = lr_ptr; - lr_ptr += lr_line_sz * num_lines; + lr_ptr += lr_plane_sz; } - f->lf.lr_line_sz = lr_line_sz; + f->lf.lr_plane_sz = lr_plane_sz; } // update allocation for loopfilter masks if (num_sb128 != f->lf.mask_sz) { freep(&f->lf.mask); @@ -2950,19 +3179,15 @@ // init ref mvs if (IS_INTER_OR_SWITCH(f->frame_hdr) || f->frame_hdr->allow_intrabc) { const int ret = dav1d_refmvs_init_frame(&f->rf, f->seq_hdr, f->frame_hdr, - f->refpoc, f->mvs, f->refrefpoc, f->ref_mvs, f->n_tc); + f->refpoc, f->mvs, f->refrefpoc, f->ref_mvs, + f->c->n_tc, f->c->n_fc); if (ret < 0) goto error; } - // create post-filtering tasks - if (c->n_pfc > 1) - if (dav1d_task_create_filter_sbrow(f)) - goto error; - retval = DAV1D_ERR(EINVAL); // setup dequant tables init_quant_tables(f->seq_hdr, f->frame_hdr, f->frame_hdr->quant.yac, f->dq); if (f->frame_hdr->quant.qm) @@ -3022,18 +3247,17 @@ f->lf.p[1] = f->cur.data[has_chroma ? 1 : 0]; f->lf.p[2] = f->cur.data[has_chroma ? 2 : 0]; f->lf.sr_p[0] = f->sr_cur.p.data[0]; f->lf.sr_p[1] = f->sr_cur.p.data[has_chroma ? 1 : 0]; f->lf.sr_p[2] = f->sr_cur.p.data[has_chroma ? 2 : 0]; - f->lf.tile_row = 1; - - dav1d_cdf_thread_wait(&f->in_cdf); + if (f->frame_hdr->refresh_context) dav1d_cdf_thread_copy(f->out_cdf.data.cdf, &f->in_cdf); // parse individual tiles per tile group - int update_set = 0, tile_row = 0, tile_col = 0; + int tile_row = 0, tile_col = 0; + f->task_thread.update_set = 0; for (int i = 0; i < f->n_tile_data; i++) { const uint8_t *data = f->tile[i].data.data; size_t size = f->tile[i].data.sz; for (int j = f->tile[i].start; j <= f->tile[i].end; j++) { @@ -3056,240 +3280,138 @@ if (tile_col == f->frame_hdr->tiling.cols) { tile_col = 0; tile_row++; } if (j == f->frame_hdr->tiling.update && f->frame_hdr->refresh_context) - update_set = 1; + f->task_thread.update_set = 1; data += tile_sz; size -= tile_sz; } } - // 2-pass decoding: - // - enabled for frame-threading, so that one frame can do symbol parsing - // as another (or multiple) are doing reconstruction. One advantage here - // is that although reconstruction is limited by reference availability, - // symbol parsing is not. Therefore, symbol parsing can effectively use - // row and col tile threading, but reconstruction only col tile threading; - // - pass 0 means no 2-pass; - // - pass 1 means symbol parsing only; - // - pass 2 means reconstruction and loop filtering. - - const int uses_2pass = c->n_fc > 1 && f->frame_hdr->refresh_context; - for (f->frame_thread.pass = uses_2pass; - f->frame_thread.pass <= 2 * uses_2pass; f->frame_thread.pass++) - { - const enum PlaneType progress_plane_type = - f->frame_thread.pass == 0 ? PLANE_TYPE_ALL : - f->frame_thread.pass == 1 ? PLANE_TYPE_BLOCK : PLANE_TYPE_Y; - - for (int n = 0; n < f->sb128w * f->frame_hdr->tiling.rows; n++) - reset_context(&f->a[n], IS_KEY_OR_INTRA(f->frame_hdr), f->frame_thread.pass); - - if (f->n_tc == 1 || (c->n_pfc > 1 && f->frame_hdr->tiling.cols * f->frame_hdr->tiling.rows == 1)) { - Dav1dTileContext *const t = f->tc; - - // no tile threading - we explicitly interleave tile/sbrow decoding - // and post-filtering, so that the full process runs in-line, so - // that frame threading is still possible - for (int tile_row = 0; tile_row < f->frame_hdr->tiling.rows; tile_row++) { - const int sbh_end = - imin(f->frame_hdr->tiling.row_start_sb[tile_row + 1], f->sbh); - for (int sby = f->frame_hdr->tiling.row_start_sb[tile_row]; - sby < sbh_end; sby++) - { - t->by = sby << (4 + f->seq_hdr->sb128); - const int by_end = (t->by + f->sb_step) >> 1; - if (f->frame_thread.pass <= 1 && f->frame_hdr->use_ref_frame_mvs) { - if (c->n_fc > 1) for (int n = 0; n < 7; n++) - if (dav1d_thread_picture_wait(&f->refp[n], - 4 * (t->by + f->sb_step), - PLANE_TYPE_BLOCK)) - { - goto error; - } - dav1d_refmvs_load_tmvs(&f->rf, tile_row, - 0, f->bw >> 1, t->by >> 1, by_end); - } - for (int tile_col = 0; tile_col < f->frame_hdr->tiling.cols; tile_col++) { - t->ts = &f->ts[tile_row * f->frame_hdr->tiling.cols + tile_col]; - if (dav1d_decode_tile_sbrow(t)) goto error; - } - if (f->frame_thread.pass <= 1 && IS_INTER_OR_SWITCH(f->frame_hdr)) { - dav1d_refmvs_save_tmvs(&t->rt, 0, f->bw >> 1, t->by >> 1, by_end); - } - - // loopfilter + cdef + restoration - if (f->frame_thread.pass != 1) { - if (c->n_pfc == 1) - f->bd_fn.filter_sbrow(f, sby); - else { - pthread_mutex_lock(&f->lf.thread.pftd->lock); - if (f->lf.thread.npf != 0 && !f->lf.thread.done) { - Dav1dTask *const t = &f->lf.thread.tasks[sby * f->lf.thread.npf]; - t->start = 1; - if (t->status == DAV1D_TASK_READY) - dav1d_task_schedule(f->lf.thread.pftd, t); - } - pthread_mutex_unlock(&f->lf.thread.pftd->lock); - } - } - if (c->n_pfc == 1 || f->frame_thread.pass == 1 || f->lf.thread.npf == 0) - dav1d_thread_picture_signal(&f->sr_cur, - (sby + 1) * f->sb_step * 4, - progress_plane_type); - } - } - } else { - // signal available tasks to worker threads - int num_tasks; - - pthread_mutex_lock(&f->tile_thread.lock); - assert(!f->tile_thread.tasks_left); - if (f->frame_thread.pass == 1 || f->n_tc >= f->frame_hdr->tiling.cols) { - // we can (or in fact, if >, we need to) do full tile decoding. - // loopfilter happens below - num_tasks = f->frame_hdr->tiling.cols * f->frame_hdr->tiling.rows; - } else { - // we need to interleave sbrow decoding for all tile cols in a - // tile row, since otherwise subsequent threads will be blocked - // waiting for the post-filter to complete - num_tasks = f->sbh * f->frame_hdr->tiling.cols; + if (c->n_tc > 1) { + const int uses_2pass = c->n_fc > 1; + for (int n = 0; n < f->sb128w * f->frame_hdr->tiling.rows * (1 + uses_2pass); n++) + reset_context(&f->a[n], IS_KEY_OR_INTRA(f->frame_hdr), + uses_2pass ? 1 + (n >= f->sb128w * f->frame_hdr->tiling.rows) : 0); + } + + retval = 0; +error: + return retval; +} + +int dav1d_decode_frame_main(Dav1dFrameContext *const f) { + const Dav1dContext *const c = f->c; + int retval = DAV1D_ERR(ENOMEM); + + assert(f->c->n_tc == 1); + + Dav1dTaskContext *const t = &c->tc[f - c->fc]; + t->f = f; + t->frame_thread.pass = 0; + + for (int n = 0; n < f->sb128w * f->frame_hdr->tiling.rows; n++) + reset_context(&f->a[n], IS_KEY_OR_INTRA(f->frame_hdr), 0); + + // no threading - we explicitly interleave tile/sbrow decoding + // and post-filtering, so that the full process runs in-line + for (int tile_row = 0; tile_row < f->frame_hdr->tiling.rows; tile_row++) { + const int sbh_end = + imin(f->frame_hdr->tiling.row_start_sb[tile_row + 1], f->sbh); + for (int sby = f->frame_hdr->tiling.row_start_sb[tile_row]; + sby < sbh_end; sby++) + { + t->by = sby << (4 + f->seq_hdr->sb128); + const int by_end = (t->by + f->sb_step) >> 1; + if (f->frame_hdr->use_ref_frame_mvs) { + dav1d_refmvs_load_tmvs(&f->rf, tile_row, + 0, f->bw >> 1, t->by >> 1, by_end); } - f->tile_thread.num_tasks = f->tile_thread.tasks_left = num_tasks; - pthread_cond_broadcast(&f->tile_thread.cond); - pthread_mutex_unlock(&f->tile_thread.lock); - - for (int tile_row = 0; tile_row < f->frame_hdr->tiling.rows; tile_row++) { - for (int sby = f->frame_hdr->tiling.row_start_sb[tile_row]; - sby < f->frame_hdr->tiling.row_start_sb[tile_row + 1]; sby++) - { - for (int tile_col = 0; tile_col < f->frame_hdr->tiling.cols; - tile_col++) - { - int progress; - Dav1dTileState *const ts = - &f->ts[tile_row * f->frame_hdr->tiling.cols + tile_col]; - - if ((progress = atomic_load(&ts->progress)) <= sby) { - pthread_mutex_lock(&ts->tile_thread.lock); - while ((progress = atomic_load(&ts->progress)) <= sby) - pthread_cond_wait(&ts->tile_thread.cond, - &ts->tile_thread.lock); - pthread_mutex_unlock(&ts->tile_thread.lock); - } - if (progress == TILE_ERROR) { - dav1d_thread_picture_signal(&f->sr_cur, FRAME_ERROR, - PLANE_TYPE_ALL); - const uint64_t all_mask = ~0ULL >> (64 - f->n_tc); - pthread_mutex_lock(&f->tile_thread.lock); - while (f->tile_thread.available != all_mask) - pthread_cond_wait(&f->tile_thread.icond, &f->tile_thread.lock); - pthread_mutex_unlock(&f->tile_thread.lock); - goto error; - } - } - - // loopfilter + cdef + restoration - if (f->frame_thread.pass != 1) { - if (c->n_pfc == 1) - f->bd_fn.filter_sbrow(f, sby); - else { - pthread_mutex_lock(&f->lf.thread.pftd->lock); - if (f->lf.thread.npf != 0 && !f->lf.thread.done) { - Dav1dTask *const t = &f->lf.thread.tasks[sby * f->lf.thread.npf]; - t->start = 1; - if (t->status == DAV1D_TASK_READY) - dav1d_task_schedule(f->lf.thread.pftd, t); - } - pthread_mutex_unlock(&f->lf.thread.pftd->lock); - } - } - if (c->n_pfc == 1 || f->frame_thread.pass == 1 || f->lf.thread.npf == 0) - dav1d_thread_picture_signal(&f->sr_cur, - (sby + 1) * f->sb_step * 4, - progress_plane_type); - } + for (int tile_col = 0; tile_col < f->frame_hdr->tiling.cols; tile_col++) { + t->ts = &f->ts[tile_row * f->frame_hdr->tiling.cols + tile_col]; + if (dav1d_decode_tile_sbrow(t)) goto error; } - - const uint64_t all_mask = ~0ULL >> (64 - f->n_tc); - pthread_mutex_lock(&f->tile_thread.lock); - while (f->tile_thread.available != all_mask) - pthread_cond_wait(&f->tile_thread.icond, &f->tile_thread.lock); - pthread_mutex_unlock(&f->tile_thread.lock); - } - - if (f->frame_thread.pass <= 1 && f->frame_hdr->refresh_context) { - // cdf update - if (update_set) - dav1d_cdf_thread_update(f->frame_hdr, f->out_cdf.data.cdf, - &f->ts[f->frame_hdr->tiling.update].cdf); - dav1d_cdf_thread_signal(&f->out_cdf); - } - if (f->frame_thread.pass == 1) { - assert(c->n_fc > 1); - for (int tile_idx = 0; - tile_idx < f->frame_hdr->tiling.rows * f->frame_hdr->tiling.cols; - tile_idx++) - { - Dav1dTileState *const ts = &f->ts[tile_idx]; - const size_t tile_start_off = - (size_t) f->frame_thread.tile_start_off[tile_idx]; - ts->frame_thread.pal_idx = f->frame_thread.pal_idx ? - &f->frame_thread.pal_idx[tile_start_off * size_mul[1] / 4] : - NULL; - ts->frame_thread.cf = f->frame_thread.cf ? - (uint8_t*)f->frame_thread.cf + - ((tile_start_off * size_mul[0]) >> !f->seq_hdr->hbd) : - NULL; - if (f->n_tc > 0) { - const unsigned row_sb_start = - f->frame_hdr->tiling.row_start_sb[ts->tiling.row]; - atomic_init(&ts->progress, row_sb_start); - } + if (IS_INTER_OR_SWITCH(f->frame_hdr)) { + dav1d_refmvs_save_tmvs(&t->rt, 0, f->bw >> 1, t->by >> 1, by_end); } + + // loopfilter + cdef + restoration + f->bd_fn.filter_sbrow(f, sby); } } retval = 0; error: - if (c->n_pfc > 1) { - pthread_mutex_lock(&f->lf.thread.pftd->lock); - if (!f->lf.thread.done) { - if (retval != 0) { - f->lf.thread.done = -1; - pthread_cond_signal(&f->lf.thread.pftd->cond); - } - pthread_cond_wait(&f->lf.thread.cond, &f->lf.thread.pftd->lock); - } - pthread_mutex_unlock(&f->lf.thread.pftd->lock); + return retval; +} + +void dav1d_decode_frame_exit(Dav1dFrameContext *const f, const int retval) { + const Dav1dContext *const c = f->c; + + if (f->sr_cur.p.data[0]) + atomic_init(&f->task_thread.error, 0); + + if (c->n_fc > 1 && retval && f->frame_thread.cf) { + memset(f->frame_thread.cf, 0, + (size_t)f->frame_thread.cf_sz * 128 * 128 / 2); } - dav1d_thread_picture_signal(&f->sr_cur, retval == 0 ? UINT_MAX : FRAME_ERROR, - PLANE_TYPE_ALL); for (int i = 0; i < 7; i++) { if (f->refp[i].p.data[0]) dav1d_thread_picture_unref(&f->refp[i]); dav1d_ref_dec(&f->ref_mvs_ref[i]); } dav1d_picture_unref_internal(&f->cur); dav1d_thread_picture_unref(&f->sr_cur); dav1d_cdf_thread_unref(&f->in_cdf); - if (f->frame_hdr->refresh_context) { - dav1d_cdf_thread_signal(&f->out_cdf); + if (f->frame_hdr && f->frame_hdr->refresh_context) { + if (f->out_cdf.progress) + atomic_store(f->out_cdf.progress, retval == 0 ? 1 : TILE_ERROR); dav1d_cdf_thread_unref(&f->out_cdf); } dav1d_ref_dec(&f->cur_segmap_ref); dav1d_ref_dec(&f->prev_segmap_ref); dav1d_ref_dec(&f->mvs_ref); dav1d_ref_dec(&f->seq_hdr_ref); dav1d_ref_dec(&f->frame_hdr_ref); for (int i = 0; i < f->n_tile_data; i++) dav1d_data_unref_internal(&f->tile[i].data); - - return retval; +} + +int dav1d_decode_frame(Dav1dFrameContext *const f) { + assert(f->c->n_fc == 1); + // if n_tc > 1 (but n_fc == 1), we could run init/exit in the task + // threads also. Not sure it makes a measurable difference. + int res = dav1d_decode_frame_init(f); + // wait until all threads have completed + if (!res) { + if (f->c->n_tc > 1) { + pthread_mutex_lock(&f->task_thread.ttd->lock); + res = dav1d_task_create_tile_sbrow(f, 0, 1); + if (!res) { + const int uses_2pass = f->c->n_fc > 1; + while (!f->task_thread.done[0] || + (uses_2pass && !f->task_thread.done[1]) || + f->task_thread.task_counter > 0) + { + pthread_cond_wait(&f->task_thread.cond, + &f->task_thread.ttd->lock); + } + } + pthread_mutex_unlock(&f->task_thread.ttd->lock); + } else { + res = dav1d_decode_frame_main(f); + if (!res && f->frame_hdr->refresh_context && f->task_thread.update_set) { + dav1d_cdf_thread_update(f->frame_hdr, f->out_cdf.data.cdf, + &f->ts[f->frame_hdr->tiling.update].cdf); + } + } + } + dav1d_decode_frame_exit(f, res); + return res; } static int get_upscale_x0(const int in_w, const int out_w, const int step) { const int err = out_w * step - (in_w << 14); const int x0 = (-((out_w - in_w) << 13) + (out_w >> 1)) / out_w + 128 - (err >> 1); @@ -3301,20 +3423,28 @@ int res = -1; // wait for c->out_delayed[next] and move into c->out if visible Dav1dThreadPicture *out_delayed; if (c->n_fc > 1) { + pthread_mutex_lock(&c->task_thread.lock); const unsigned next = c->frame_thread.next++; if (c->frame_thread.next == c->n_fc) c->frame_thread.next = 0; f = &c->fc[next]; - pthread_mutex_lock(&f->frame_thread.td.lock); while (f->n_tile_data > 0) - pthread_cond_wait(&f->frame_thread.td.cond, - &f->frame_thread.td.lock); + pthread_cond_wait(&f->task_thread.cond, + &c->task_thread.lock); out_delayed = &c->frame_thread.out_delayed[next]; + if (out_delayed->p.data[0] || atomic_load(&f->task_thread.error)) { + if (atomic_load(&c->task_thread.first) + 1U < c->n_fc) + atomic_fetch_add(&c->task_thread.first, 1U); + else + atomic_store(&c->task_thread.first, 0); + if (c->task_thread.cur < c->n_fc) + c->task_thread.cur--; + } if (out_delayed->p.data[0]) { const unsigned progress = atomic_load_explicit(&out_delayed->progress[1], memory_order_relaxed); if (out_delayed->visible && progress != FRAME_ERROR) { dav1d_picture_ref(&c->out, &out_delayed->p); @@ -3370,11 +3500,12 @@ #define assign_bitdepth_case(bd) \ f->bd_fn.recon_b_inter = dav1d_recon_b_inter_##bd##bpc; \ f->bd_fn.recon_b_intra = dav1d_recon_b_intra_##bd##bpc; \ f->bd_fn.filter_sbrow = dav1d_filter_sbrow_##bd##bpc; \ - f->bd_fn.filter_sbrow_deblock = dav1d_filter_sbrow_deblock_##bd##bpc; \ + f->bd_fn.filter_sbrow_deblock_cols = dav1d_filter_sbrow_deblock_cols_##bd##bpc; \ + f->bd_fn.filter_sbrow_deblock_rows = dav1d_filter_sbrow_deblock_rows_##bd##bpc; \ f->bd_fn.filter_sbrow_cdef = dav1d_filter_sbrow_cdef_##bd##bpc; \ f->bd_fn.filter_sbrow_resize = dav1d_filter_sbrow_resize_##bd##bpc; \ f->bd_fn.filter_sbrow_lr = dav1d_filter_sbrow_lr_##bd##bpc; \ f->bd_fn.backup_ipred_edge = dav1d_backup_ipred_edge_##bd##bpc; \ f->bd_fn.read_coef_blocks = dav1d_read_coef_blocks_##bd##bpc @@ -3425,11 +3556,11 @@ f->svc[i][1].scale = scale_fac(c->refs[refidx].p.p.p.h, f->frame_hdr->height); f->svc[i][0].step = (f->svc[i][0].scale + 8) >> 4; f->svc[i][1].step = (f->svc[i][1].scale + 8) >> 4; } else { - f->svc[i][0].scale = 0; + f->svc[i][0].scale = f->svc[i][1].scale = 0; } f->gmv_warp_allowed[i] = f->frame_hdr->gmv[i].type > DAV1D_WM_TYPE_TRANSLATION && !f->frame_hdr->force_integer_mv && !dav1d_get_shear_params(&f->frame_hdr->gmv[i]) && !f->svc[i][0].scale; @@ -3442,11 +3573,11 @@ } else { const int pri_ref = f->frame_hdr->refidx[f->frame_hdr->primary_ref_frame]; dav1d_cdf_thread_ref(&f->in_cdf, &c->cdf[pri_ref]); } if (f->frame_hdr->refresh_context) { - res = dav1d_cdf_thread_alloc(c, &f->out_cdf, c->n_fc > 1 ? &f->frame_thread.td : NULL); + res = dav1d_cdf_thread_alloc(c, &f->out_cdf, c->n_fc > 1); if (res < 0) goto error; } // FIXME qsort so tiles are in order (for frame threading) if (f->n_tile_data_alloc < c->n_tile_data) { @@ -3506,10 +3637,15 @@ f->sb_shift = 4 + f->seq_hdr->sb128; f->sb_step = 16 << f->seq_hdr->sb128; f->sbh = (f->bh + f->sb_step - 1) >> f->sb_shift; f->b4_stride = (f->bw + 31) & ~31; f->bitdepth_max = (1 << f->cur.p.bpc) - 1; + atomic_init(&f->task_thread.error, 0); + const int uses_2pass = c->n_fc > 1; + const int cols = f->frame_hdr->tiling.cols; + const int rows = f->frame_hdr->tiling.rows; + f->task_thread.task_counter = (cols * rows + f->sbh) << uses_2pass; // ref_mvs if (IS_INTER_OR_SWITCH(f->frame_hdr) || f->frame_hdr->allow_intrabc) { f->mvs_ref = dav1d_ref_create_using_pool(c->refmvs_pool, sizeof(*f->mvs) * f->sb128h * 16 * (f->b4_stride >> 1)); @@ -3648,16 +3784,17 @@ } } return res; } } else { - pthread_cond_signal(&f->frame_thread.td.cond); - pthread_mutex_unlock(&f->frame_thread.td.lock); + dav1d_task_frame_init(f); + pthread_mutex_unlock(&c->task_thread.lock); } return 0; error: + atomic_init(&f->task_thread.error, 1); dav1d_cdf_thread_unref(&f->in_cdf); if (f->frame_hdr->refresh_context) dav1d_cdf_thread_unref(&f->out_cdf); for (int i = 0; i < 7; i++) { if (f->refp[i].p.data[0]) @@ -3676,12 +3813,10 @@ for (int i = 0; i < f->n_tile_data; i++) dav1d_data_unref_internal(&f->tile[i].data); f->n_tile_data = 0; - if (c->n_fc > 1) { - pthread_cond_signal(&f->frame_thread.td.cond); - pthread_mutex_unlock(&f->frame_thread.td.lock); - } + if (c->n_fc > 1) + pthread_mutex_unlock(&c->task_thread.lock); return res; } diff --git a/third_party/dav1d/src/ext/x86/x86inc.asm b/third_party/dav1d/src/ext/x86/x86inc.asm --- a/third_party/dav1d/src/ext/x86/x86inc.asm +++ b/third_party/dav1d/src/ext/x86/x86inc.asm @@ -77,10 +77,15 @@ %define mangle(x) _ %+ x %else %define mangle(x) x %endif +; Use VEX-encoding even in non-AVX functions +%ifndef FORCE_VEX_ENCODING + %define FORCE_VEX_ENCODING 0 +%endif + %macro SECTION_RODATA 0-1 16 %ifidn __OUTPUT_FORMAT__,win32 SECTION .rdata align=%1 %elif WIN64 SECTION .rdata align=%1 @@ -1006,11 +1011,11 @@ INIT_CPUFLAGS %1 DEFINE_MMREGS mm %endmacro %macro INIT_XMM 0-1+ - %assign avx_enabled 0 + %assign avx_enabled FORCE_VEX_ENCODING %define RESET_MM_PERMUTATION INIT_XMM %1 %define mmsize 16 %define mova movdqa %define movu movdqu %define movh movq @@ -1337,30 +1342,54 @@ %1 %6, __src2 %endif %elif %0 >= 9 __instr %6, %7, %8, %9 %elif %0 == 8 - %if avx_enabled && %5 + %if avx_enabled && __sizeofreg >= 16 && %4 == 0 %xdefine __src1 %7 %xdefine __src2 %8 - %ifnum regnumof%7 - %ifnum regnumof%8 - %if regnumof%7 < 8 && regnumof%8 >= 8 && regnumof%8 < 16 && sizeof%8 <= 32 - ; Most VEX-encoded instructions require an additional byte to encode when - ; src2 is a high register (e.g. m8..15). If the instruction is commutative - ; we can swap src1 and src2 when doing so reduces the instruction length. - %xdefine __src1 %8 - %xdefine __src2 %7 + %if %5 + %ifnum regnumof%7 + %ifnum regnumof%8 + %if regnumof%7 < 8 && regnumof%8 >= 8 && regnumof%8 < 16 && sizeof%8 <= 32 + ; Most VEX-encoded instructions require an additional byte to encode when + ; src2 is a high register (e.g. m8..15). If the instruction is commutative + ; we can swap src1 and src2 when doing so reduces the instruction length. + %xdefine __src1 %8 + %xdefine __src2 %7 + %endif %endif + %elifnum regnumof%8 ; put memory operands in src2 when possible + %xdefine __src1 %8 + %xdefine __src2 %7 + %else + %assign __emulate_avx 1 + %endif + %elifnnum regnumof%7 + ; EVEX allows imm8 shift instructions to be used with memory operands, + ; but VEX does not. This handles those special cases. + %ifnnum %8 + %assign __emulate_avx 1 + %elif notcpuflag(avx512) + %assign __emulate_avx 1 %endif %endif - __instr %6, __src1, __src2 + %if __emulate_avx ; a separate load is required + %if %3 + vmovaps %6, %7 + %else + vmovdqa %6, %7 + %endif + __instr %6, %8 + %else + __instr %6, __src1, __src2 + %endif %else __instr %6, %7, %8 %endif %elif %0 == 7 - %if avx_enabled && %5 + %if avx_enabled && __sizeofreg >= 16 && %5 %xdefine __src1 %6 %xdefine __src2 %7 %ifnum regnumof%6 %ifnum regnumof%7 %if regnumof%6 < 8 && regnumof%7 >= 8 && regnumof%7 < 16 && sizeof%7 <= 32 diff --git a/third_party/dav1d/src/internal.h b/third_party/dav1d/src/internal.h --- a/third_party/dav1d/src/internal.h +++ b/third_party/dav1d/src/internal.h @@ -32,12 +32,11 @@ #include "dav1d/data.h" typedef struct Dav1dFrameContext Dav1dFrameContext; typedef struct Dav1dTileState Dav1dTileState; -typedef struct Dav1dTileContext Dav1dTileContext; -typedef struct Dav1dPostFilterContext Dav1dPostFilterContext; +typedef struct Dav1dTaskContext Dav1dTaskContext; typedef struct Dav1dTask Dav1dTask; #include "common/attributes.h" #include "src/cdef.h" @@ -76,12 +75,12 @@ struct Dav1dContext { Dav1dFrameContext *fc; unsigned n_fc; - Dav1dPostFilterContext *pfc; - unsigned n_pfc; + Dav1dTaskContext *tc; + unsigned n_tc; // cache of OBUs that make up a single frame before we submit them // to a frame worker to be decoded struct Dav1dTileGroup *tile; int n_tile_data_alloc; @@ -110,18 +109,24 @@ struct { Dav1dThreadPicture *out_delayed; unsigned next; } frame_thread; - // postfilter threading (refer to pfc[] for per_thread thingies) - struct PostFilterThreadData { + // task threading (refer to tc[] for per_thread thingies) + struct TaskThreadData { pthread_mutex_t lock; pthread_cond_t cond; - struct Dav1dTask *tasks; - int frame_cnt; + atomic_uint first; + unsigned cur; + // This is used for delayed reset of the task cur pointer when + // such operation is needed but the thread doesn't enter a critical + // section (typically when executing the next sbrow task locklessly). + // See src/thread_task.c:reset_task_cur(). + atomic_uint reset_task_cur; + atomic_int cond_signaled; int inited; - } postfilter_thread; + } task_thread; // reference/entropy state Dav1dMemPool *segmap_pool; Dav1dMemPool *refmvs_pool; struct { @@ -132,10 +137,11 @@ } refs[8]; Dav1dMemPool *cdf_pool; CdfThreadContext cdf[8]; Dav1dDSPContext dsp[3 /* 8, 10, 12 bits/component */]; + Dav1dRefmvsDSPContext refmvs_dsp; // tree to keep track of which edges are available struct { EdgeNode *root[2 /* BL_128X128 vs. BL_64X64 */]; EdgeBranch branch_sb128[1 + 4 + 16 + 64]; @@ -157,10 +163,33 @@ Dav1dLogger logger; Dav1dMemPool *picture_pool; }; +enum TaskType { + DAV1D_TASK_TYPE_INIT, + DAV1D_TASK_TYPE_TILE_ENTROPY, + DAV1D_TASK_TYPE_TILE_RECONSTRUCTION, + DAV1D_TASK_TYPE_DEBLOCK_COLS, + DAV1D_TASK_TYPE_DEBLOCK_ROWS, + DAV1D_TASK_TYPE_CDEF, + DAV1D_TASK_TYPE_SUPER_RESOLUTION, + DAV1D_TASK_TYPE_LOOP_RESTORATION, + DAV1D_TASK_TYPE_ENTROPY_PROGRESS, +}; + +struct Dav1dTask { + unsigned frame_idx; // frame thread id + enum TaskType type; // task work + int sby; // sbrow + + // task dependencies + int recon_progress, deblock_progress, cdef_progress, lr_progress; + int deps_skip; + struct Dav1dTask *next; // only used in task queue +}; + struct Dav1dFrameContext { Dav1dRef *seq_hdr_ref; Dav1dSequenceHeader *seq_hdr; Dav1dRef *frame_hdr_ref; Dav1dFrameHeader *frame_hdr; @@ -186,20 +215,19 @@ int step; } svc[7][2 /* x, y */]; int resize_step[2 /* y, uv */], resize_start[2 /* y, uv */]; const Dav1dContext *c; - Dav1dTileContext *tc; - int n_tc; Dav1dTileState *ts; int n_ts; const Dav1dDSPContext *dsp; struct { recon_b_intra_fn recon_b_intra; recon_b_inter_fn recon_b_inter; filter_sbrow_fn filter_sbrow; - filter_sbrow_fn filter_sbrow_deblock; + filter_sbrow_fn filter_sbrow_deblock_cols; + filter_sbrow_fn filter_sbrow_deblock_rows; filter_sbrow_fn filter_sbrow_cdef; filter_sbrow_fn filter_sbrow_resize; filter_sbrow_fn filter_sbrow_lr; backup_ipred_edge_fn backup_ipred_edge; read_coef_blocks_fn read_coef_blocks; @@ -216,12 +244,13 @@ refmvs_frame rf; uint8_t jnt_weights[7][7]; int bitdepth_max; struct { - struct thread_data td; - int pass, die; + int next_tile_row[2 /* 0: reconstruction, 1: entropy */]; + int entropy_progress; + atomic_int deblock_progress, cdef_progress, lr_progress; // in sby units // indexed using t->by * f->b4_stride + t->bx Av1Block *b; struct CodedBlockInfo { int16_t eob[3 /* plane */]; uint8_t txtp[3 /* plane */]; @@ -241,46 +270,49 @@ uint8_t (*level)[4]; Av1Filter *mask; Av1Restoration *lr_mask; int top_pre_cdef_toggle; int mask_sz /* w*h */, lr_mask_sz, cdef_line_sz[2] /* stride */; - int lr_line_sz, re_sz /* h */; + size_t lr_plane_sz; /* w*sbh*4*is_sb128 if n_tc > 1, else w*12 */ + int re_sz /* h */; ALIGN(Av1FilterLUT lim_lut, 16); int last_sharpness; uint8_t lvl[8 /* seg_id */][4 /* dir */][8 /* ref */][2 /* is_gmv */]; uint8_t *tx_lpf_right_edge[2]; uint8_t *cdef_line_buf; pixel *cdef_line[2 /* pre, post */][3 /* plane */]; pixel *lr_lpf_line[3 /* plane */]; // in-loop filter per-frame state keeping - int tile_row; // for carry-over at tile row edges + uint8_t *start_of_tile_row; + int start_of_tile_row_sz; pixel *p[3], *sr_p[3]; Av1Filter *mask_ptr, *prev_mask_ptr; int restore_planes; // enum LrRestorePlanes + } lf; - struct { - pthread_cond_t cond; - struct PostFilterThreadData *pftd; - struct Dav1dTask *tasks; - int num_tasks; - int npf; - int done; - int inited; - } thread; - } lf; + struct { + pthread_cond_t cond; + struct TaskThreadData *ttd; + struct Dav1dTask *tasks, *tile_tasks[2], init_task; + int num_tasks, num_tile_tasks; + int done[2]; + int update_set; // whether we need to update CDF reference + atomic_int error; + int task_counter; + struct Dav1dTask *task_head, *task_tail; + // Points to the task directly before the cur pointer in the queue. + // This cur pointer is theoretical here, we actually keep track of the + // "prev_t" variable. This is needed to not loose the tasks in + // [head;cur-1] when picking one for execution. + struct Dav1dTask *task_cur_prev; + } task_thread; // threading (refer to tc[] for per-thread things) struct FrameTileThreadData { - uint64_t available; - pthread_mutex_t lock; - pthread_cond_t cond, icond; - int tasks_left, num_tasks; - int (*task_idx_to_sby_and_tile_idx)[2]; - int titsati_sz, titsati_init[2]; - uint16_t titsati_index_rows[1 + DAV1D_MAX_TILE_ROWS]; - int inited; + int (*lowest_pixel_mem)[7][2]; + int lowest_pixel_mem_sz; } tile_thread; }; struct Dav1dTileState { CdfContext cdf; @@ -289,19 +321,20 @@ struct { int col_start, col_end, row_start, row_end; // in 4px units int col, row; // in tile units } tiling; - atomic_int progress; // in sby units, TILE_ERROR after a decoding error - struct { - pthread_mutex_t lock; - pthread_cond_t cond; - } tile_thread; + // in sby units, TILE_ERROR after a decoding error + atomic_int progress[2 /* 0: reconstruction, 1: entropy */]; struct { uint8_t *pal_idx; coef *cf; - } frame_thread; + } frame_thread[2 /* 0: reconstruction, 1: entropy */]; + + // in fullpel units, [0] = Y, [1] = UV, used for progress requirements + // each entry is one tile-sbrow; middle index is refidx + int (*lowest_pixel)[7][2]; uint16_t dqmem[DAV1D_MAX_SEGMENTS][3 /* plane */][2 /* dc/ac */]; const uint16_t (*dq)[3][2]; int last_qidx; @@ -310,11 +343,12 @@ const uint8_t (*lflvl)[4][8][2]; Av1RestorationUnit *lr_ref[3]; }; -struct Dav1dTileContext { +struct Dav1dTaskContext { + const Dav1dContext *c; const Dav1dFrameContext *f; Dav1dTileState *ts; int bx, by; BlockContext l, *a; ALIGN(union, 32) { @@ -374,19 +408,17 @@ // a 4x4 area, but the top/left one can go out of cache already, so this // keeps it accessible enum Filter2d tl_4x4_filter; struct { + int pass; + } frame_thread; + struct { struct thread_data td; + struct TaskThreadData *ttd; struct FrameTileThreadData *fttd; + int flushed; int die; - } tile_thread; -}; - -struct Dav1dPostFilterContext { - Dav1dContext *c; - struct thread_data td; - int flushed; - int die; + } task_thread; }; #endif /* DAV1D_SRC_INTERNAL_H */ diff --git a/third_party/dav1d/src/levels.h b/third_party/dav1d/src/levels.h --- a/third_party/dav1d/src/levels.h +++ b/third_party/dav1d/src/levels.h @@ -29,10 +29,11 @@ #define DAV1D_SRC_LEVELS_H #include #include "dav1d/headers.h" +#include "common/attributes.h" enum ObuMetaType { OBU_META_HDR_CLL = 1, OBU_META_HDR_MDCV = 2, OBU_META_SCALABILITY = 3, diff --git a/third_party/dav1d/src/lf_apply.h b/third_party/dav1d/src/lf_apply.h --- a/third_party/dav1d/src/lf_apply.h +++ b/third_party/dav1d/src/lf_apply.h @@ -33,10 +33,13 @@ #include "common/bitdepth.h" #include "src/internal.h" #include "src/levels.h" -void bytefn(dav1d_loopfilter_sbrow)(const Dav1dFrameContext *f, - pixel *const p[3], Av1Filter *lflvl, - int sby, int start_of_tile_row); +void bytefn(dav1d_loopfilter_sbrow_cols)(const Dav1dFrameContext *f, + pixel *const p[3], Av1Filter *lflvl, + int sby, int start_of_tile_row); +void bytefn(dav1d_loopfilter_sbrow_rows)(const Dav1dFrameContext *f, + pixel *const p[3], Av1Filter *lflvl, + int sby); #endif /* DAV1D_SRC_LF_APPLY_H */ diff --git a/third_party/dav1d/src/lf_apply_tmpl.c b/third_party/dav1d/src/lf_apply_tmpl.c --- a/third_party/dav1d/src/lf_apply_tmpl.c +++ b/third_party/dav1d/src/lf_apply_tmpl.c @@ -168,17 +168,16 @@ (const uint8_t(*)[4]) &lvl[0][3], b4_stride, &f->lf.lim_lut, w HIGHBD_CALL_SUFFIX); } } -void bytefn(dav1d_loopfilter_sbrow)(const Dav1dFrameContext *const f, - pixel *const p[3], Av1Filter *const lflvl, - int sby, const int start_of_tile_row) +void bytefn(dav1d_loopfilter_sbrow_cols)(const Dav1dFrameContext *const f, + pixel *const p[3], Av1Filter *const lflvl, + int sby, const int start_of_tile_row) { int x, have_left; // Don't filter outside the frame - const int have_top = sby > 0; const int is_sb64 = !f->seq_hdr->sb128; const int starty4 = (sby & is_sb64) << 4; const int sbsz = 32 >> is_sb64; const int sbl2 = 5 - is_sb64; const int halign = (f->bh + 31) & ~31; @@ -269,17 +268,10 @@ filter_plane_cols_y(f, have_left, level_ptr, f->b4_stride, lflvl[x].filter_y[0], ptr, f->cur.stride[0], imin(32, f->w4 - x * 32), starty4, endy4); } - level_ptr = f->lf.level + f->b4_stride * sby * sbsz; - for (ptr = p[0], x = 0; x < f->sb128w; x++, ptr += 128, level_ptr += 32) { - filter_plane_rows_y(f, have_top, level_ptr, f->b4_stride, - lflvl[x].filter_y[1], ptr, f->cur.stride[0], - imin(32, f->w4 - x * 32), starty4, endy4); - } - if (!f->frame_hdr->loopfilter.level_u && !f->frame_hdr->loopfilter.level_v) return; ptrdiff_t uv_off; level_ptr = f->lf.level + f->b4_stride * (sby * sbsz >> ss_ver); @@ -290,11 +282,39 @@ lflvl[x].filter_uv[0], &p[1][uv_off], &p[2][uv_off], f->cur.stride[1], (imin(32, f->w4 - x * 32) + ss_hor) >> ss_hor, starty4 >> ss_ver, uv_endy4, ss_ver); } +} +void bytefn(dav1d_loopfilter_sbrow_rows)(const Dav1dFrameContext *const f, + pixel *const p[3], Av1Filter *const lflvl, + int sby) +{ + int x; + // Don't filter outside the frame + const int have_top = sby > 0; + const int is_sb64 = !f->seq_hdr->sb128; + const int starty4 = (sby & is_sb64) << 4; + const int sbsz = 32 >> is_sb64; + const int ss_ver = f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; + const int ss_hor = f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I444; + const unsigned endy4 = starty4 + imin(f->h4 - sby * sbsz, sbsz); + const unsigned uv_endy4 = (endy4 + ss_ver) >> ss_ver; + + pixel *ptr; + uint8_t (*level_ptr)[4] = f->lf.level + f->b4_stride * sby * sbsz; + for (ptr = p[0], x = 0; x < f->sb128w; x++, ptr += 128, level_ptr += 32) { + filter_plane_rows_y(f, have_top, level_ptr, f->b4_stride, + lflvl[x].filter_y[1], ptr, f->cur.stride[0], + imin(32, f->w4 - x * 32), starty4, endy4); + } + + if (!f->frame_hdr->loopfilter.level_u && !f->frame_hdr->loopfilter.level_v) + return; + + ptrdiff_t uv_off; level_ptr = f->lf.level + f->b4_stride * (sby * sbsz >> ss_ver); for (uv_off = 0, x = 0; x < f->sb128w; x++, uv_off += 128 >> ss_hor, level_ptr += 32 >> ss_hor) { filter_plane_rows_uv(f, have_top, level_ptr, f->b4_stride, diff --git a/third_party/dav1d/src/lib.c b/third_party/dav1d/src/lib.c --- a/third_party/dav1d/src/lib.c +++ b/third_party/dav1d/src/lib.c @@ -61,13 +61,12 @@ COLD const char *dav1d_version(void) { return DAV1D_VERSION; } COLD void dav1d_default_settings(Dav1dSettings *const s) { - s->n_frame_threads = 1; - s->n_tile_threads = 1; - s->n_postfilter_threads = 1; + s->n_threads = 0; + s->max_frame_delay = 0; s->apply_grain = 1; s->allocator.cookie = NULL; s->allocator.alloc_picture_callback = dav1d_default_picture_alloc; s->allocator.release_picture_callback = dav1d_default_picture_release; s->logger.cookie = NULL; @@ -99,16 +98,14 @@ static pthread_once_t initted = PTHREAD_ONCE_INIT; pthread_once(&initted, init_internal); validate_input_or_ret(c_out != NULL, DAV1D_ERR(EINVAL)); validate_input_or_ret(s != NULL, DAV1D_ERR(EINVAL)); - validate_input_or_ret(s->n_postfilter_threads >= 1 && - s->n_postfilter_threads <= DAV1D_MAX_POSTFILTER_THREADS, DAV1D_ERR(EINVAL)); - validate_input_or_ret(s->n_tile_threads >= 1 && - s->n_tile_threads <= DAV1D_MAX_TILE_THREADS, DAV1D_ERR(EINVAL)); - validate_input_or_ret(s->n_frame_threads >= 1 && - s->n_frame_threads <= DAV1D_MAX_FRAME_THREADS, DAV1D_ERR(EINVAL)); + validate_input_or_ret(s->n_threads >= 0 && + s->n_threads <= DAV1D_MAX_THREADS, DAV1D_ERR(EINVAL)); + validate_input_or_ret(s->max_frame_delay >= 0 && + s->max_frame_delay <= DAV1D_MAX_FRAME_DELAY, DAV1D_ERR(EINVAL)); validate_input_or_ret(s->allocator.alloc_picture_callback != NULL, DAV1D_ERR(EINVAL)); validate_input_or_ret(s->allocator.release_picture_callback != NULL, DAV1D_ERR(EINVAL)); validate_input_or_ret(s->operating_point >= 0 && @@ -164,110 +161,70 @@ } c->flush = &c->flush_mem; atomic_init(c->flush, 0); - c->n_pfc = s->n_postfilter_threads; - c->n_fc = s->n_frame_threads; - c->fc = dav1d_alloc_aligned(sizeof(*c->fc) * s->n_frame_threads, 32); - if (!c->fc) goto error; - memset(c->fc, 0, sizeof(*c->fc) * s->n_frame_threads); + c->n_tc = s->n_threads ? s->n_threads : + iclip(dav1d_num_logical_processors(c), 1, DAV1D_MAX_THREADS); + c->n_fc = s->max_frame_delay ? umin(s->max_frame_delay, c->n_tc) : + umin(c->n_tc, 8); - if (c->n_pfc > 1) { - c->pfc = dav1d_alloc_aligned(sizeof(*c->pfc) * s->n_postfilter_threads, 32); - if (!c->pfc) goto error; - memset(c->pfc, 0, sizeof(*c->pfc) * s->n_postfilter_threads); - if (pthread_mutex_init(&c->postfilter_thread.lock, NULL)) goto error; - if (pthread_cond_init(&c->postfilter_thread.cond, NULL)) { - pthread_mutex_destroy(&c->postfilter_thread.lock); + c->fc = dav1d_alloc_aligned(sizeof(*c->fc) * c->n_fc, 32); + if (!c->fc) goto error; + memset(c->fc, 0, sizeof(*c->fc) * c->n_fc); + + c->tc = dav1d_alloc_aligned(sizeof(*c->tc) * c->n_tc, 64); + if (!c->tc) goto error; + memset(c->tc, 0, sizeof(*c->tc) * c->n_tc); + if (c->n_tc > 1) { + if (pthread_mutex_init(&c->task_thread.lock, NULL)) goto error; + if (pthread_cond_init(&c->task_thread.cond, NULL)) { + pthread_mutex_destroy(&c->task_thread.lock); goto error; } - c->postfilter_thread.inited = 1; - for (int n = 0; n < s->n_frame_threads; n++) { - Dav1dFrameContext *const f = &c->fc[n]; - if (pthread_cond_init(&f->lf.thread.cond, NULL)) goto error; - f->lf.thread.pftd = &c->postfilter_thread; - f->lf.thread.done = 1; - f->lf.thread.inited = 1; - } - for (int n = 0; n < s->n_postfilter_threads; ++n) { - Dav1dPostFilterContext *const pf = &c->pfc[n]; - pf->c = c; - if (pthread_mutex_init(&pf->td.lock, NULL)) goto error; - if (pthread_cond_init(&pf->td.cond, NULL)) { - pthread_mutex_destroy(&pf->td.lock); - goto error; - } - if (pthread_create(&pf->td.thread, &thread_attr, dav1d_postfilter_task, pf)) { - pthread_cond_destroy(&c->postfilter_thread.cond); - pthread_mutex_destroy(&c->postfilter_thread.lock); - goto error; - } - pf->td.inited = 1; - } + c->task_thread.cur = c->n_fc; + atomic_init(&c->task_thread.reset_task_cur, UINT_MAX); + atomic_init(&c->task_thread.cond_signaled, 0); + c->task_thread.inited = 1; } if (c->n_fc > 1) { c->frame_thread.out_delayed = calloc(c->n_fc, sizeof(*c->frame_thread.out_delayed)); if (!c->frame_thread.out_delayed) goto error; } - for (int n = 0; n < s->n_frame_threads; n++) { + for (unsigned n = 0; n < c->n_fc; n++) { Dav1dFrameContext *const f = &c->fc[n]; + if (c->n_tc > 1) + if (pthread_cond_init(&f->task_thread.cond, NULL)) goto error; f->c = c; + f->task_thread.ttd = &c->task_thread; f->lf.last_sharpness = -1; - f->n_tc = s->n_tile_threads; - f->tc = dav1d_alloc_aligned(sizeof(*f->tc) * s->n_tile_threads, 64); - if (!f->tc) goto error; - memset(f->tc, 0, sizeof(*f->tc) * s->n_tile_threads); - if (f->n_tc > 1) { - if (pthread_mutex_init(&f->tile_thread.lock, NULL)) goto error; - if (pthread_cond_init(&f->tile_thread.cond, NULL)) { - pthread_mutex_destroy(&f->tile_thread.lock); - goto error; - } - if (pthread_cond_init(&f->tile_thread.icond, NULL)) { - pthread_mutex_destroy(&f->tile_thread.lock); - pthread_cond_destroy(&f->tile_thread.cond); + dav1d_refmvs_init(&f->rf); + } + + for (unsigned m = 0; m < c->n_tc; m++) { + Dav1dTaskContext *const t = &c->tc[m]; + t->f = &c->fc[0]; + t->task_thread.ttd = &c->task_thread; + t->c = c; + memset(t->cf_16bpc, 0, sizeof(t->cf_16bpc)); + if (c->n_tc > 1) { + if (pthread_mutex_init(&t->task_thread.td.lock, NULL)) goto error; + if (pthread_cond_init(&t->task_thread.td.cond, NULL)) { + pthread_mutex_destroy(&t->task_thread.td.lock); goto error; } - f->tile_thread.inited = 1; - } - for (int m = 0; m < s->n_tile_threads; m++) { - Dav1dTileContext *const t = &f->tc[m]; - t->f = f; - memset(t->cf_16bpc, 0, sizeof(t->cf_16bpc)); - if (f->n_tc > 1) { - if (pthread_mutex_init(&t->tile_thread.td.lock, NULL)) goto error; - if (pthread_cond_init(&t->tile_thread.td.cond, NULL)) { - pthread_mutex_destroy(&t->tile_thread.td.lock); - goto error; - } - t->tile_thread.fttd = &f->tile_thread; - if (pthread_create(&t->tile_thread.td.thread, &thread_attr, dav1d_tile_task, t)) { - pthread_cond_destroy(&t->tile_thread.td.cond); - pthread_mutex_destroy(&t->tile_thread.td.lock); - goto error; - } - t->tile_thread.td.inited = 1; - } - } - dav1d_refmvs_init(&f->rf); - if (c->n_fc > 1) { - if (pthread_mutex_init(&f->frame_thread.td.lock, NULL)) goto error; - if (pthread_cond_init(&f->frame_thread.td.cond, NULL)) { - pthread_mutex_destroy(&f->frame_thread.td.lock); + if (pthread_create(&t->task_thread.td.thread, &thread_attr, dav1d_worker_task, t)) { + pthread_cond_destroy(&t->task_thread.td.cond); + pthread_mutex_destroy(&t->task_thread.td.lock); goto error; } - if (pthread_create(&f->frame_thread.td.thread, &thread_attr, dav1d_frame_task, f)) { - pthread_cond_destroy(&f->frame_thread.td.cond); - pthread_mutex_destroy(&f->frame_thread.td.lock); - goto error; - } - f->frame_thread.td.inited = 1; + t->task_thread.td.inited = 1; } } + dav1d_refmvs_dsp_init(&c->refmvs_dsp); // intra edge tree c->intra_edge.root[BL_128X128] = &c->intra_edge.branch_sb128[0].node; dav1d_init_mode_tree(c->intra_edge.root[BL_128X128], c->intra_edge.tip_sb128, 1); c->intra_edge.root[BL_64X64] = &c->intra_edge.branch_sb64[0].node; @@ -295,10 +252,11 @@ validate_input_or_ret(out != NULL, DAV1D_ERR(EINVAL)); Dav1dSettings s; dav1d_default_settings(&s); + s.n_threads = 1; s.logger.callback = NULL; Dav1dContext *c; res = dav1d_open(&c, &s); if (res < 0) return res; @@ -316,11 +274,11 @@ buf.sz -= res; buf.data += res; } if (!c->seq_hdr) { - res = DAV1D_ERR(EINVAL); + res = DAV1D_ERR(ENOENT); goto error; } memcpy(out, c->seq_hdr, sizeof(*out)); @@ -392,19 +350,27 @@ static int drain_picture(Dav1dContext *const c, Dav1dPicture *const out) { unsigned drain_count = 0; do { const unsigned next = c->frame_thread.next; Dav1dFrameContext *const f = &c->fc[next]; - pthread_mutex_lock(&f->frame_thread.td.lock); + pthread_mutex_lock(&c->task_thread.lock); while (f->n_tile_data > 0) - pthread_cond_wait(&f->frame_thread.td.cond, - &f->frame_thread.td.lock); - pthread_mutex_unlock(&f->frame_thread.td.lock); + pthread_cond_wait(&f->task_thread.cond, + &f->task_thread.ttd->lock); Dav1dThreadPicture *const out_delayed = &c->frame_thread.out_delayed[next]; + if (out_delayed->p.data[0] || atomic_load(&f->task_thread.error)) { + if (atomic_load(&c->task_thread.first) + 1U < c->n_fc) + atomic_fetch_add(&c->task_thread.first, 1U); + else + atomic_store(&c->task_thread.first, 0); + if (c->task_thread.cur && c->task_thread.cur < c->n_fc) + c->task_thread.cur--; + } if (++c->frame_thread.next == c->n_fc) c->frame_thread.next = 0; + pthread_mutex_unlock(&c->task_thread.lock); if (out_delayed->p.data[0]) { const unsigned progress = atomic_load_explicit(&out_delayed->progress[1], memory_order_relaxed); if (out_delayed->visible && progress != FRAME_ERROR) { @@ -507,55 +473,47 @@ c->itut_t35 = NULL; dav1d_ref_dec(&c->mastering_display_ref); dav1d_ref_dec(&c->content_light_ref); dav1d_ref_dec(&c->itut_t35_ref); - if (c->n_fc == 1 && c->n_pfc == 1) return; + if (c->n_fc == 1 && c->n_tc == 1) return; + atomic_store(c->flush, 1); + + // stop running tasks in worker threads + if (c->n_tc > 1) { + pthread_mutex_lock(&c->task_thread.lock); + for (unsigned i = 0; i < c->n_tc; i++) { + Dav1dTaskContext *const tc = &c->tc[i]; + while (!tc->task_thread.flushed) { + pthread_cond_wait(&tc->task_thread.td.cond, &c->task_thread.lock); + } + } + for (unsigned i = 0; i < c->n_fc; i++) { + c->fc[i].task_thread.task_head = NULL; + c->fc[i].task_thread.task_tail = NULL; + c->fc[i].task_thread.task_cur_prev = NULL; + } + atomic_init(&c->task_thread.first, 0); + c->task_thread.cur = c->n_fc; + atomic_store(&c->task_thread.reset_task_cur, UINT_MAX); + atomic_store(&c->task_thread.cond_signaled, 0); + pthread_mutex_unlock(&c->task_thread.lock); + } // wait for threads to complete flushing - if (c->n_pfc > 1) - pthread_mutex_lock(&c->postfilter_thread.lock); - atomic_store(c->flush, 1); - if (c->n_pfc > 1) { - pthread_cond_broadcast(&c->postfilter_thread.cond); - pthread_mutex_unlock(&c->postfilter_thread.lock); - } - if (c->n_fc == 1) goto skip_ft_flush; - for (unsigned n = 0, next = c->frame_thread.next; n < c->n_fc; n++, next++) { - if (next == c->n_fc) next = 0; - Dav1dFrameContext *const f = &c->fc[next]; - pthread_mutex_lock(&f->frame_thread.td.lock); - if (f->n_tile_data > 0) { - while (f->n_tile_data > 0) - pthread_cond_wait(&f->frame_thread.td.cond, - &f->frame_thread.td.lock); - assert(!f->cur.data[0]); + if (c->n_fc > 1) { + for (unsigned n = 0, next = c->frame_thread.next; n < c->n_fc; n++, next++) { + if (next == c->n_fc) next = 0; + Dav1dFrameContext *const f = &c->fc[next]; + dav1d_decode_frame_exit(f, -1); + f->n_tile_data = 0; + Dav1dThreadPicture *out_delayed = &c->frame_thread.out_delayed[next]; + if (out_delayed->p.data[0]) { + dav1d_thread_picture_unref(out_delayed); + } } - pthread_mutex_unlock(&f->frame_thread.td.lock); - Dav1dThreadPicture *const out_delayed = - &c->frame_thread.out_delayed[next]; - if (out_delayed->p.data[0]) - dav1d_thread_picture_unref(out_delayed); - } - c->frame_thread.next = 0; -skip_ft_flush: - if (c->n_pfc > 1) { - for (unsigned i = 0; i < c->n_pfc; ++i) { - Dav1dPostFilterContext *const pf = &c->pfc[i]; - pthread_mutex_lock(&pf->td.lock); - if (!pf->flushed) - pthread_cond_wait(&pf->td.cond, &pf->td.lock); - pf->flushed = 0; - pthread_mutex_unlock(&pf->td.lock); - } - pthread_mutex_lock(&c->postfilter_thread.lock); - c->postfilter_thread.tasks = NULL; - pthread_mutex_unlock(&c->postfilter_thread.lock); - for (unsigned i = 0; i < c->n_fc; ++i) { - freep(&c->fc[i].lf.thread.tasks); - c->fc[i].lf.thread.num_tasks = 0; - } + c->frame_thread.next = 0; } atomic_store(c->flush, 0); } COLD void dav1d_close(Dav1dContext **const c_out) { @@ -567,93 +525,56 @@ Dav1dContext *const c = *c_out; if (!c) return; if (flush) dav1d_flush(c); - if (c->pfc) { - struct PostFilterThreadData *pftd = &c->postfilter_thread; - if (pftd->inited) { - pthread_mutex_lock(&pftd->lock); - for (unsigned n = 0; n < c->n_pfc && c->pfc[n].td.inited; n++) - c->pfc[n].die = 1; - pthread_cond_broadcast(&pftd->cond); - pthread_mutex_unlock(&pftd->lock); - for (unsigned n = 0; n < c->n_pfc && c->pfc[n].td.inited; n++) { - pthread_join(c->pfc[n].td.thread, NULL); - pthread_cond_destroy(&c->pfc[n].td.cond); - pthread_mutex_destroy(&c->pfc[n].td.lock); + if (c->tc) { + struct TaskThreadData *ttd = &c->task_thread; + if (ttd->inited) { + pthread_mutex_lock(&ttd->lock); + for (unsigned n = 0; n < c->n_tc && c->tc[n].task_thread.td.inited; n++) + c->tc[n].task_thread.die = 1; + pthread_cond_broadcast(&ttd->cond); + pthread_mutex_unlock(&ttd->lock); + for (unsigned n = 0; n < c->n_tc; n++) { + Dav1dTaskContext *const pf = &c->tc[n]; + if (!pf->task_thread.td.inited) break; + pthread_join(pf->task_thread.td.thread, NULL); + pthread_cond_destroy(&pf->task_thread.td.cond); + pthread_mutex_destroy(&pf->task_thread.td.lock); } - pthread_cond_destroy(&pftd->cond); - pthread_mutex_destroy(&pftd->lock); + pthread_cond_destroy(&ttd->cond); + pthread_mutex_destroy(&ttd->lock); } - dav1d_free_aligned(c->pfc); + dav1d_free_aligned(c->tc); } for (unsigned n = 0; c->fc && n < c->n_fc; n++) { Dav1dFrameContext *const f = &c->fc[n]; // clean-up threading stuff - if (c->n_fc > 1 && f->frame_thread.td.inited) { - pthread_mutex_lock(&f->frame_thread.td.lock); - f->frame_thread.die = 1; - pthread_cond_signal(&f->frame_thread.td.cond); - pthread_mutex_unlock(&f->frame_thread.td.lock); - pthread_join(f->frame_thread.td.thread, NULL); + if (c->n_fc > 1) { + freep(&f->tile_thread.lowest_pixel_mem); freep(&f->frame_thread.b); dav1d_freep_aligned(&f->frame_thread.pal_idx); dav1d_freep_aligned(&f->frame_thread.cf); freep(&f->frame_thread.tile_start_off); dav1d_freep_aligned(&f->frame_thread.pal); freep(&f->frame_thread.cbi); - pthread_mutex_destroy(&f->frame_thread.td.lock); - pthread_cond_destroy(&f->frame_thread.td.cond); + pthread_cond_destroy(&f->task_thread.cond); } - if (f->n_tc > 1 && f->tc && f->tile_thread.inited) { - pthread_mutex_lock(&f->tile_thread.lock); - for (int m = 0; m < f->n_tc; m++) { - Dav1dTileContext *const t = &f->tc[m]; - t->tile_thread.die = 1; - // mark not created tile threads as available - if (!t->tile_thread.td.inited) - f->tile_thread.available |= 1ULL<tile_thread.cond); - while (f->tile_thread.available != ~0ULL >> (64 - f->n_tc)) - pthread_cond_wait(&f->tile_thread.icond, - &f->tile_thread.lock); - pthread_mutex_unlock(&f->tile_thread.lock); - for (int m = 0; m < f->n_tc; m++) { - Dav1dTileContext *const t = &f->tc[m]; - if (f->n_tc > 1 && t->tile_thread.td.inited) { - pthread_join(t->tile_thread.td.thread, NULL); - pthread_mutex_destroy(&t->tile_thread.td.lock); - pthread_cond_destroy(&t->tile_thread.td.cond); - } - } - pthread_mutex_destroy(&f->tile_thread.lock); - pthread_cond_destroy(&f->tile_thread.cond); - pthread_cond_destroy(&f->tile_thread.icond); - freep(&f->tile_thread.task_idx_to_sby_and_tile_idx); - } - for (int m = 0; f->ts && m < f->n_ts; m++) { - Dav1dTileState *const ts = &f->ts[m]; - pthread_cond_destroy(&ts->tile_thread.cond); - pthread_mutex_destroy(&ts->tile_thread.lock); - } - if (f->lf.thread.inited) { - freep(&f->lf.thread.tasks); - pthread_cond_destroy(&f->lf.thread.cond); - } + freep(&f->task_thread.tasks); + freep(&f->task_thread.tile_tasks[0]); dav1d_free_aligned(f->ts); - dav1d_free_aligned(f->tc); dav1d_free_aligned(f->ipred_edge[0]); free(f->a); free(f->tile); free(f->lf.mask); free(f->lf.lr_mask); free(f->lf.level); free(f->lf.tx_lpf_right_edge[0]); + free(f->lf.start_of_tile_row); dav1d_refmvs_clear(&f->rf); dav1d_free_aligned(f->lf.cdef_line_buf); dav1d_free_aligned(f->lf.lr_lpf_line[0]); } dav1d_free_aligned(c->fc); diff --git a/third_party/dav1d/src/looprestoration_tmpl.c b/third_party/dav1d/src/looprestoration_tmpl.c --- a/third_party/dav1d/src/looprestoration_tmpl.c +++ b/third_party/dav1d/src/looprestoration_tmpl.c @@ -380,15 +380,15 @@ const int b = (BB[i] + ((1 << bitdepth_min_8) >> 1)) >> bitdepth_min_8; const unsigned p = imax(a * n - b * b, 0); const unsigned z = (p * s + (1 << 19)) >> 20; - const unsigned x = dav1d_sgr_x_by_x[imin(z, 255)]; + const unsigned x = dav1d_sgr_x_by_x[umin(z, 255)]; // This is where we invert A and B, so that B is of size coef. AA[i] = (x * BB[i] * sgr_one_by_x + (1 << 11)) >> 12; - BB[i] = 256 - x; + BB[i] = x; } AA += step * REST_UNIT_STRIDE; BB += step * REST_UNIT_STRIDE; } @@ -401,31 +401,31 @@ P[i + 1 - REST_UNIT_STRIDE] + P[i + 1 + REST_UNIT_STRIDE]) * 5) for (; j < h - 1; j+=2) { for (int i = 0; i < w; i++) { const int a = SIX_NEIGHBORS(B, i); const int b = SIX_NEIGHBORS(A, i); - dst[i] = (a * src[i] + b + (1 << 8)) >> 9; + dst[i] = (b - a * src[i] + (1 << 8)) >> 9; } dst += 384 /* Maximum restoration width is 384 (256 * 1.5) */; src += REST_UNIT_STRIDE; B += REST_UNIT_STRIDE; A += REST_UNIT_STRIDE; for (int i = 0; i < w; i++) { const int a = B[i] * 6 + (B[i - 1] + B[i + 1]) * 5; const int b = A[i] * 6 + (A[i - 1] + A[i + 1]) * 5; - dst[i] = (a * src[i] + b + (1 << 7)) >> 8; + dst[i] = (b - a * src[i] + (1 << 7)) >> 8; } dst += 384 /* Maximum restoration width is 384 (256 * 1.5) */; src += REST_UNIT_STRIDE; B += REST_UNIT_STRIDE; A += REST_UNIT_STRIDE; } if (j + 1 == h) { // Last row, when number of rows is odd for (int i = 0; i < w; i++) { const int a = SIX_NEIGHBORS(B, i); const int b = SIX_NEIGHBORS(A, i); - dst[i] = (a * src[i] + b + (1 << 8)) >> 9; + dst[i] = (b - a * src[i] + (1 << 8)) >> 9; } } #undef SIX_NEIGHBORS } else { #define EIGHT_NEIGHBORS(P, i)\ @@ -434,11 +434,11 @@ P[i + 1 - REST_UNIT_STRIDE] + P[i + 1 + REST_UNIT_STRIDE]) * 3) for (int j = 0; j < h; j++) { for (int i = 0; i < w; i++) { const int a = EIGHT_NEIGHBORS(B, i); const int b = EIGHT_NEIGHBORS(A, i); - dst[i] = (a * src[i] + b + (1 << 8)) >> 9; + dst[i] = (b - a * src[i] + (1 << 8)) >> 9; } dst += 384; src += REST_UNIT_STRIDE; B += REST_UNIT_STRIDE; A += REST_UNIT_STRIDE; @@ -466,13 +466,12 @@ params->sgr.s0 HIGHBD_TAIL_SUFFIX); const int w0 = params->sgr.w0; for (int j = 0; j < h; j++) { for (int i = 0; i < w; i++) { - const int u = (p[i] << 4); - const int v = (u << 7) + w0 * (dst[j * 384 + i] - u); - p[i] = iclip_pixel((v + (1 << 10)) >> 11); + const int v = w0 * dst[j * 384 + i]; + p[i] = iclip_pixel(p[i] + ((v + (1 << 10)) >> 11)); } p += PXSTRIDE(p_stride); } } @@ -490,13 +489,12 @@ params->sgr.s1 HIGHBD_TAIL_SUFFIX); const int w1 = params->sgr.w1; for (int j = 0; j < h; j++) { for (int i = 0; i < w; i++) { - const int u = (p[i] << 4); - const int v = (u << 7) + w1 * (dst[j * 384 + i] - u); - p[i] = iclip_pixel((v + (1 << 10)) >> 11); + const int v = w1 * dst[j * 384 + i]; + p[i] = iclip_pixel(p[i] + ((v + (1 << 10)) >> 11)); } p += PXSTRIDE(p_stride); } } @@ -518,14 +516,12 @@ const int w0 = params->sgr.w0; const int w1 = params->sgr.w1; for (int j = 0; j < h; j++) { for (int i = 0; i < w; i++) { - const int u = (p[i] << 4); - const int v = (u << 7) + w0 * (dst0[j * 384 + i] - u) + - w1 * (dst1[j * 384 + i] - u); - p[i] = iclip_pixel((v + (1 << 10)) >> 11); + const int v = w0 * dst0[j * 384 + i] + w1 * dst1[j * 384 + i]; + p[i] = iclip_pixel(p[i] + ((v + (1 << 10)) >> 11)); } p += PXSTRIDE(p_stride); } } diff --git a/third_party/dav1d/src/lr_apply_tmpl.c b/third_party/dav1d/src/lr_apply_tmpl.c --- a/third_party/dav1d/src/lr_apply_tmpl.c +++ b/third_party/dav1d/src/lr_apply_tmpl.c @@ -46,20 +46,20 @@ static void backup_lpf(const Dav1dFrameContext *const f, pixel *dst, const ptrdiff_t dst_stride, const pixel *src, const ptrdiff_t src_stride, const int ss_ver, const int sb128, int row, const int row_h, const int src_w, - const int h, const int ss_hor, const int pft) + const int h, const int ss_hor) { const int dst_w = f->frame_hdr->super_res.enabled ? (f->frame_hdr->width[1] + ss_hor) >> ss_hor : src_w; // The first stripe of the frame is shorter by 8 luma pixel rows. int stripe_h = (64 - 8 * !row) >> ss_ver; src += (stripe_h - 2) * PXSTRIDE(src_stride); - if (!pft) { + if (f->c->n_tc == 1) { if (row) { const int top = 4 << sb128; // Copy the top part of the stored loop filtered pixels from the // previous sb row needed above the first stripe of this sb row. pixel_copy(&dst[PXSTRIDE(dst_stride) * 0], @@ -106,18 +106,19 @@ } void bytefn(dav1d_lr_copy_lpf)(Dav1dFrameContext *const f, /*const*/ pixel *const src[3], const int sby) { - const int pft = f->c->n_pfc > 1; + const int have_tt = f->c->n_tc > 1; const int offset = 8 * !!sby; const ptrdiff_t *const src_stride = f->cur.stride; const ptrdiff_t lr_stride = ((f->sr_cur.p.p.w + 31) & ~31) * sizeof(pixel); + const ptrdiff_t tt_off = have_tt * sby * (4 << f->seq_hdr->sb128) * PXSTRIDE(lr_stride); pixel *const dst[3] = { - f->lf.lr_lpf_line[0] + pft * sby * (4 << f->seq_hdr->sb128) * PXSTRIDE(lr_stride), - f->lf.lr_lpf_line[1] + pft * sby * (4 << f->seq_hdr->sb128) * PXSTRIDE(lr_stride), - f->lf.lr_lpf_line[2] + pft * sby * (4 << f->seq_hdr->sb128) * PXSTRIDE(lr_stride) + f->lf.lr_lpf_line[0] + tt_off, + f->lf.lr_lpf_line[1] + tt_off, + f->lf.lr_lpf_line[2] + tt_off }; // TODO Also check block level restore type to reduce copying. const int restore_planes = f->lf.restore_planes; @@ -126,11 +127,11 @@ const int w = f->bw << 2; const int row_h = imin((sby + 1) << (6 + f->seq_hdr->sb128), h - 1); const int y_stripe = (sby << (6 + f->seq_hdr->sb128)) - offset; backup_lpf(f, dst[0], lr_stride, src[0] - offset * PXSTRIDE(src_stride[0]), src_stride[0], - 0, f->seq_hdr->sb128, y_stripe, row_h, w, h, 0, pft); + 0, f->seq_hdr->sb128, y_stripe, row_h, w, h, 0); } if (restore_planes & (LR_RESTORE_U | LR_RESTORE_V)) { const int ss_ver = f->sr_cur.p.p.layout == DAV1D_PIXEL_LAYOUT_I420; const int ss_hor = f->sr_cur.p.p.layout != DAV1D_PIXEL_LAYOUT_I444; const int h = (f->cur.p.h + ss_ver) >> ss_ver; @@ -139,16 +140,16 @@ const int offset_uv = offset >> ss_ver; const int y_stripe = (sby << ((6 - ss_ver) + f->seq_hdr->sb128)) - offset_uv; if (restore_planes & LR_RESTORE_U) { backup_lpf(f, dst[1], lr_stride, src[1] - offset_uv * PXSTRIDE(src_stride[1]), src_stride[1], - ss_ver, f->seq_hdr->sb128, y_stripe, row_h, w, h, ss_hor, pft); + ss_ver, f->seq_hdr->sb128, y_stripe, row_h, w, h, ss_hor); } if (restore_planes & LR_RESTORE_V) { backup_lpf(f, dst[2], lr_stride, src[2] - offset_uv * PXSTRIDE(src_stride[1]), src_stride[1], - ss_ver, f->seq_hdr->sb128, y_stripe, row_h, w, h, ss_hor, pft); + ss_ver, f->seq_hdr->sb128, y_stripe, row_h, w, h, ss_hor); } } } static void lr_stripe(const Dav1dFrameContext *const f, pixel *p, @@ -160,11 +161,13 @@ const int chroma = !!plane; const int ss_ver = chroma & (f->sr_cur.p.p.layout == DAV1D_PIXEL_LAYOUT_I420); const ptrdiff_t p_stride = f->sr_cur.p.stride[chroma]; const ptrdiff_t lpf_stride = sizeof(pixel) * ((f->sr_cur.p.p.w + 31) & ~31); const int sby = (y + (y ? 8 << ss_ver : 0)) >> (6 - ss_ver + f->seq_hdr->sb128); - const pixel *lpf = f->lf.lr_lpf_line[plane] + (f->c->n_pfc > 1) * (sby * (4 << f->seq_hdr->sb128) - 4) * PXSTRIDE(lpf_stride) + x; + const int have_tt = f->c->n_tc > 1; + const pixel *lpf = f->lf.lr_lpf_line[plane] + + have_tt * (sby * (4 << f->seq_hdr->sb128) - 4) * PXSTRIDE(lpf_stride) + x; // The first stripe of the frame is shorter by 8 luma pixel rows. int stripe_h = imin((64 - 8 * !y) >> ss_ver, row_h - y); looprestorationfilter_fn lr_fn; diff --git a/third_party/dav1d/src/meson.build b/third_party/dav1d/src/meson.build --- a/third_party/dav1d/src/meson.build +++ b/third_party/dav1d/src/meson.build @@ -90,10 +90,11 @@ if (host_machine.cpu_family() == 'aarch64' or host_machine.cpu_family().startswith('arm')) libdav1d_sources += files( 'arm/cpu.c', + 'arm/refmvs_init.c', ) libdav1d_tmpl_sources += files( 'arm/cdef_init_tmpl.c', 'arm/film_grain_init_tmpl.c', 'arm/ipred_init_tmpl.c', @@ -107,10 +108,11 @@ libdav1d_sources_asm = files( # itx.S is used for both 8 and 16 bpc. 'arm/64/itx.S', 'arm/64/looprestoration_common.S', 'arm/64/msac.S', + 'arm/64/refmvs.S', ) if dav1d_bitdepths.contains('8') libdav1d_sources_asm += files( 'arm/64/cdef.S', @@ -137,10 +139,11 @@ libdav1d_sources_asm = files( # itx.S is used for both 8 and 16 bpc. 'arm/32/itx.S', 'arm/32/looprestoration_common.S', 'arm/32/msac.S', + 'arm/32/refmvs.S', ) if dav1d_bitdepths.contains('8') libdav1d_sources_asm += files( 'arm/32/cdef.S', @@ -173,10 +176,11 @@ elif host_machine.cpu_family().startswith('x86') libdav1d_sources += files( 'x86/cpu.c', 'x86/msac_init.c', + 'x86/refmvs_init.c', ) libdav1d_tmpl_sources += files( 'x86/cdef_init_tmpl.c', 'x86/film_grain_init_tmpl.c', @@ -189,14 +193,16 @@ # NASM source files libdav1d_sources_asm = files( 'x86/cpuid.asm', 'x86/msac.asm', + 'x86/refmvs.asm', 'x86/cdef_avx2.asm', 'x86/itx_avx2.asm', 'x86/looprestoration_avx2.asm', 'x86/cdef_sse.asm', + 'x86/itx_sse.asm', ) if dav1d_bitdepths.contains('8') libdav1d_sources_asm += files( 'x86/cdef_avx512.asm', @@ -205,11 +211,10 @@ 'x86/film_grain_avx2.asm', 'x86/ipred_avx2.asm', 'x86/loopfilter_avx2.asm', 'x86/film_grain_sse.asm', 'x86/ipred_sse.asm', - 'x86/itx_sse.asm', 'x86/loopfilter_sse.asm', 'x86/looprestoration_sse.asm', 'x86/mc_sse.asm', ) endif @@ -222,10 +227,12 @@ 'x86/itx16_avx2.asm', 'x86/loopfilter16_avx2.asm', 'x86/looprestoration16_avx2.asm', 'x86/mc16_avx2.asm', 'x86/cdef16_sse.asm', + 'x86/film_grain16_sse.asm', + 'x86/ipred16_sse.asm', 'x86/itx16_sse.asm', 'x86/loopfilter16_sse.asm', 'x86/looprestoration16_sse.asm', 'x86/mc16_sse.asm', ) @@ -278,41 +285,41 @@ libdav1d_entrypoints_objs = static_library('dav1d_entrypoint', libdav1d_entrypoints_sources, rev_target, config_h_target, include_directories : dav1d_inc_dirs, - dependencies: [stdatomic_dependency], + dependencies: [stdatomic_dependencies], c_args : [stackalign_flag, stackrealign_flag, api_export_flags], install : false, build_by_default : false, -).extract_all_objects() +).extract_all_objects(recursive: true) # Helper library for each bitdepth libdav1d_bitdepth_objs = [] foreach bitdepth : dav1d_bitdepths libdav1d_bitdepth_objs += static_library( 'dav1d_bitdepth_@0@'.format(bitdepth), libdav1d_tmpl_sources, config_h_target, include_directories: dav1d_inc_dirs, - dependencies : [stdatomic_dependency], + dependencies : [stdatomic_dependencies], c_args : ['-DBITDEPTH=@0@'.format(bitdepth)] + stackalign_flag, install : false, build_by_default : false, - ).extract_all_objects() + ).extract_all_objects(recursive: true) endforeach # Helper library for each bitdepth and architecture-specific flags foreach bitdepth : dav1d_bitdepths libdav1d_bitdepth_objs += static_library( 'dav1d_arch_bitdepth_@0@'.format(bitdepth), libdav1d_arch_tmpl_sources, config_h_target, include_directories: dav1d_inc_dirs, - dependencies : [stdatomic_dependency], + dependencies : [stdatomic_dependencies], c_args : ['-DBITDEPTH=@0@'.format(bitdepth)] + stackalign_flag + arch_flags, install : false, build_by_default : false, - ).extract_all_objects() + ).extract_all_objects(recursive: true) endforeach # The final dav1d library if host_machine.system() == 'windows' dav1d_soversion = '' @@ -330,11 +337,11 @@ libdav1d_entrypoints_objs ], include_directories : dav1d_inc_dirs, dependencies : [ - stdatomic_dependency, + stdatomic_dependencies, thread_dependency, thread_compat_dep, libdl_dependency, ], c_args : [stackalign_flag, api_export_flags], diff --git a/third_party/dav1d/src/obu.c b/third_party/dav1d/src/obu.c --- a/third_party/dav1d/src/obu.c +++ b/third_party/dav1d/src/obu.c @@ -1545,22 +1545,30 @@ dav1d_picture_ref(&c->out, &c->refs[c->frame_hdr->existing_frame_idx].p.p); dav1d_data_props_copy(&c->out.m, &in->m); c->event_flags |= dav1d_picture_get_event_flags(&c->refs[c->frame_hdr->existing_frame_idx].p); } else { + pthread_mutex_lock(&c->task_thread.lock); // need to append this to the frame output queue const unsigned next = c->frame_thread.next++; if (c->frame_thread.next == c->n_fc) c->frame_thread.next = 0; Dav1dFrameContext *const f = &c->fc[next]; - pthread_mutex_lock(&f->frame_thread.td.lock); while (f->n_tile_data > 0) - pthread_cond_wait(&f->frame_thread.td.cond, - &f->frame_thread.td.lock); + pthread_cond_wait(&f->task_thread.cond, + &f->task_thread.ttd->lock); Dav1dThreadPicture *const out_delayed = &c->frame_thread.out_delayed[next]; + if (out_delayed->p.data[0] || atomic_load(&f->task_thread.error)) { + if (atomic_load(&c->task_thread.first) + 1U < c->n_fc) + atomic_fetch_add(&c->task_thread.first, 1U); + else + atomic_store(&c->task_thread.first, 0); + if (c->task_thread.cur < c->n_fc) + c->task_thread.cur--; + } if (out_delayed->p.data[0]) { const unsigned progress = atomic_load_explicit(&out_delayed->progress[1], memory_order_relaxed); if (out_delayed->visible && progress != FRAME_ERROR) { dav1d_picture_ref(&c->out, &out_delayed->p); @@ -1570,11 +1578,11 @@ } dav1d_thread_picture_ref(out_delayed, &c->refs[c->frame_hdr->existing_frame_idx].p); out_delayed->visible = 1; dav1d_data_props_copy(&out_delayed->p.m, &in->m); - pthread_mutex_unlock(&f->frame_thread.td.lock); + pthread_mutex_unlock(&c->task_thread.lock); } if (c->refs[c->frame_hdr->existing_frame_idx].p.p.frame_hdr->frame_type == DAV1D_FRAME_TYPE_KEY) { const int r = c->frame_hdr->existing_frame_idx; for (int i = 0; i < 8; i++) { if (i == r) continue; diff --git a/third_party/dav1d/src/picture.h b/third_party/dav1d/src/picture.h --- a/third_party/dav1d/src/picture.h +++ b/third_party/dav1d/src/picture.h @@ -50,11 +50,10 @@ typedef struct Dav1dThreadPicture { Dav1dPicture p; int visible; enum PictureFlags flags; - struct thread_data *t; // [0] block data (including segmentation map and motion vectors) // [1] pixel data atomic_uint *progress; } Dav1dThreadPicture; @@ -89,35 +88,10 @@ /** * Move a picture reference. */ void dav1d_picture_move_ref(Dav1dPicture *dst, Dav1dPicture *src); -/** - * Wait for picture to reach a certain stage. - * - * y is in full-pixel units. If pt is not UV, this is in luma - * units, else it is in chroma units. - * plane_type is used to determine how many pixels delay are - * introduced by loopfilter processes. - * - * Returns 0 on success, and 1 if there was an error while decoding p - */ -int dav1d_thread_picture_wait(const Dav1dThreadPicture *p, int y, - enum PlaneType plane_type); - -/** - * Signal decoding progress. - * - * y is in full-pixel luma units. FRAME_ERROR is used to signal a decoding - * error to frames using this frame as reference frame. - * plane_type denotes whether we have completed block data (pass 1; - * PLANE_TYPE_BLOCK), pixel data (pass 2, PLANE_TYPE_Y) or both (no - * 2-pass decoding; PLANE_TYPE_ALL). - */ -void dav1d_thread_picture_signal(const Dav1dThreadPicture *p, int y, - enum PlaneType plane_type); - int dav1d_default_picture_alloc(Dav1dPicture *p, void *cookie); void dav1d_default_picture_release(Dav1dPicture *p, void *cookie); void dav1d_picture_unref_internal(Dav1dPicture *p); /** diff --git a/third_party/dav1d/src/picture.c b/third_party/dav1d/src/picture.c --- a/third_party/dav1d/src/picture.c +++ b/third_party/dav1d/src/picture.c @@ -174,21 +174,21 @@ int dav1d_thread_picture_alloc(Dav1dContext *const c, Dav1dFrameContext *const f, const int bpc) { Dav1dThreadPicture *const p = &f->sr_cur; - p->t = c->n_fc > 1 ? &f->frame_thread.td : NULL; + const int have_frame_mt = c->n_fc > 1; const int res = picture_alloc_with_edges(c, &p->p, f->frame_hdr->width[1], f->frame_hdr->height, f->seq_hdr, f->seq_hdr_ref, f->frame_hdr, f->frame_hdr_ref, c->content_light, c->content_light_ref, c->mastering_display, c->mastering_display_ref, c->itut_t35, c->itut_t35_ref, bpc, &f->tile[0].data.m, &c->allocator, - p->t != NULL ? sizeof(atomic_int) * 2 : 0, + have_frame_mt ? sizeof(atomic_int) * 2 : 0, (void **) &p->progress); if (res) return res; // Must be removed from the context after being attached to the frame dav1d_ref_dec(&c->itut_t35_ref); @@ -196,11 +196,11 @@ p->flags = c->frame_flags; c->frame_flags = 0; p->visible = f->frame_hdr->show_frame; - if (p->t) { + if (have_frame_mt) { atomic_init(&p->progress[0], 0); atomic_init(&p->progress[1], 0); } return res; } @@ -252,11 +252,10 @@ void dav1d_thread_picture_ref(Dav1dThreadPicture *const dst, const Dav1dThreadPicture *const src) { dav1d_picture_ref(&dst->p, &src->p); - dst->t = src->t; dst->visible = src->visible; dst->progress = src->progress; dst->flags = src->flags; } @@ -277,58 +276,13 @@ } void dav1d_thread_picture_unref(Dav1dThreadPicture *const p) { dav1d_picture_unref_internal(&p->p); - p->t = NULL; p->progress = NULL; } -int dav1d_thread_picture_wait(const Dav1dThreadPicture *const p, - int y_unclipped, const enum PlaneType plane_type) -{ - assert(plane_type != PLANE_TYPE_ALL); - - if (!p->t) - return 0; - - // convert to luma units; include plane delay from loopfilters; clip - const int ss_ver = p->p.p.layout == DAV1D_PIXEL_LAYOUT_I420; - y_unclipped *= 1 << (plane_type & ss_ver); // we rely here on PLANE_TYPE_UV being 1 - y_unclipped += (plane_type != PLANE_TYPE_BLOCK) * 8; // delay imposed by loopfilter - const unsigned y = iclip(y_unclipped, 1, p->p.p.h); - atomic_uint *const progress = &p->progress[plane_type != PLANE_TYPE_BLOCK]; - unsigned state; - - if ((state = atomic_load_explicit(progress, memory_order_acquire)) >= y) - return state == FRAME_ERROR; - - pthread_mutex_lock(&p->t->lock); - while ((state = atomic_load_explicit(progress, memory_order_relaxed)) < y) - pthread_cond_wait(&p->t->cond, &p->t->lock); - pthread_mutex_unlock(&p->t->lock); - return state == FRAME_ERROR; -} - -void dav1d_thread_picture_signal(const Dav1dThreadPicture *const p, - const int y, // in pixel units - const enum PlaneType plane_type) -{ - assert(plane_type != PLANE_TYPE_UV); - - if (!p->t) - return; - - pthread_mutex_lock(&p->t->lock); - if (plane_type != PLANE_TYPE_Y) - atomic_store(&p->progress[0], y); - if (plane_type != PLANE_TYPE_BLOCK) - atomic_store(&p->progress[1], y); - pthread_cond_broadcast(&p->t->cond); - pthread_mutex_unlock(&p->t->lock); -} - enum Dav1dEventFlags dav1d_picture_get_event_flags(const Dav1dThreadPicture *const p) { if (!p->flags) return 0; enum Dav1dEventFlags flags = 0; diff --git a/third_party/dav1d/src/recon.h b/third_party/dav1d/src/recon.h --- a/third_party/dav1d/src/recon.h +++ b/third_party/dav1d/src/recon.h @@ -35,40 +35,42 @@ f->frame_hdr->frame_offset == 2 && t->by >= 0 && t->by < 4 && \ t->bx >= 8 && t->bx < 12 #define DEBUG_B_PIXELS 0 #define decl_recon_b_intra_fn(name) \ -void (name)(Dav1dTileContext *t, enum BlockSize bs, \ +void (name)(Dav1dTaskContext *t, enum BlockSize bs, \ enum EdgeFlags intra_edge_flags, const Av1Block *b) typedef decl_recon_b_intra_fn(*recon_b_intra_fn); #define decl_recon_b_inter_fn(name) \ -int (name)(Dav1dTileContext *t, enum BlockSize bs, const Av1Block *b) +int (name)(Dav1dTaskContext *t, enum BlockSize bs, const Av1Block *b) typedef decl_recon_b_inter_fn(*recon_b_inter_fn); #define decl_filter_sbrow_fn(name) \ void (name)(Dav1dFrameContext *f, int sby) typedef decl_filter_sbrow_fn(*filter_sbrow_fn); #define decl_backup_ipred_edge_fn(name) \ -void (name)(Dav1dTileContext *t) +void (name)(Dav1dTaskContext *t) typedef decl_backup_ipred_edge_fn(*backup_ipred_edge_fn); #define decl_read_coef_blocks_fn(name) \ -void (name)(Dav1dTileContext *t, enum BlockSize bs, const Av1Block *b) +void (name)(Dav1dTaskContext *t, enum BlockSize bs, const Av1Block *b) typedef decl_read_coef_blocks_fn(*read_coef_blocks_fn); decl_recon_b_intra_fn(dav1d_recon_b_intra_8bpc); decl_recon_b_intra_fn(dav1d_recon_b_intra_16bpc); decl_recon_b_inter_fn(dav1d_recon_b_inter_8bpc); decl_recon_b_inter_fn(dav1d_recon_b_inter_16bpc); decl_filter_sbrow_fn(dav1d_filter_sbrow_8bpc); decl_filter_sbrow_fn(dav1d_filter_sbrow_16bpc); -decl_filter_sbrow_fn(dav1d_filter_sbrow_deblock_8bpc); -decl_filter_sbrow_fn(dav1d_filter_sbrow_deblock_16bpc); +decl_filter_sbrow_fn(dav1d_filter_sbrow_deblock_cols_8bpc); +decl_filter_sbrow_fn(dav1d_filter_sbrow_deblock_cols_16bpc); +decl_filter_sbrow_fn(dav1d_filter_sbrow_deblock_rows_8bpc); +decl_filter_sbrow_fn(dav1d_filter_sbrow_deblock_rows_16bpc); decl_filter_sbrow_fn(dav1d_filter_sbrow_cdef_8bpc); decl_filter_sbrow_fn(dav1d_filter_sbrow_cdef_16bpc); decl_filter_sbrow_fn(dav1d_filter_sbrow_resize_8bpc); decl_filter_sbrow_fn(dav1d_filter_sbrow_resize_16bpc); decl_filter_sbrow_fn(dav1d_filter_sbrow_lr_8bpc); diff --git a/third_party/dav1d/src/recon_tmpl.c b/third_party/dav1d/src/recon_tmpl.c --- a/third_party/dav1d/src/recon_tmpl.c +++ b/third_party/dav1d/src/recon_tmpl.c @@ -316,11 +316,11 @@ offset = 26 + (y > 1 ? 10 : y * 5); } return offset + (mag > 512 ? 4 : (mag + 64) >> 7); } -static int decode_coefs(Dav1dTileContext *const t, +static int decode_coefs(Dav1dTaskContext *const t, uint8_t *const a, uint8_t *const l, const enum RectTxfmSize tx, const enum BlockSize bs, const Av1Block *const b, const int intra, const int plane, coef *cf, enum TxfmType *const txtp, uint8_t *res_ctx) @@ -717,11 +717,11 @@ *res_ctx = umin(cul_level, 63) | dc_sign_level; return eob; } -static void read_coef_tree(Dav1dTileContext *const t, +static void read_coef_tree(Dav1dTaskContext *const t, const enum BlockSize bs, const Av1Block *const b, const enum RectTxfmSize ytx, const int depth, const uint16_t *const tx_split, const int x_off, const int y_off, pixel *dst) { @@ -766,19 +766,20 @@ uint8_t cf_ctx; int eob; coef *cf; struct CodedBlockInfo *cbi; - if (f->frame_thread.pass) { - assert(ts->frame_thread.cf); - cf = ts->frame_thread.cf; - ts->frame_thread.cf += imin(t_dim->w, 8) * imin(t_dim->h, 8) * 16; + if (t->frame_thread.pass) { + const int p = t->frame_thread.pass & 1; + assert(ts->frame_thread[p].cf); + cf = ts->frame_thread[p].cf; + ts->frame_thread[p].cf += imin(t_dim->w, 8) * imin(t_dim->h, 8) * 16; cbi = &f->frame_thread.cbi[t->by * f->b4_stride + t->bx]; } else { cf = bitfn(t->cf); } - if (f->frame_thread.pass != 2) { + if (t->frame_thread.pass != 2) { eob = decode_coefs(t, &t->a->lcoef[bx4], &t->l.lcoef[by4], ytx, bs, b, 0, 0, cf, &txtp, &cf_ctx); if (DEBUG_BLOCK_INFO) printf("Post-y-cf-blk[tx=%d,txtp=%d,eob=%d]: r=%d\n", ytx, txtp, eob, ts->msac.rng); @@ -796,19 +797,19 @@ txtp_map += 32; \ } uint8_t *txtp_map = &t->txtp_map[by4 * 32 + bx4]; case_set_upto16(txw,,,); #undef set_ctx - if (f->frame_thread.pass == 1) { + if (t->frame_thread.pass == 1) { cbi->eob[0] = eob; cbi->txtp[0] = txtp; } } else { eob = cbi->eob[0]; txtp = cbi->txtp[0]; } - if (!(f->frame_thread.pass & 1)) { + if (!(t->frame_thread.pass & 1)) { assert(dst); if (eob >= 0) { if (DEBUG_BLOCK_INFO && DEBUG_B_PIXELS) coef_dump(cf, imin(t_dim->h, 8) * 4, imin(t_dim->w, 8) * 4, 3, "dq"); dsp->itx.itxfm_add[ytx][txtp](dst, f->cur.stride[0], cf, eob @@ -818,11 +819,11 @@ } } } } -void bytefn(dav1d_read_coef_blocks)(Dav1dTileContext *const t, +void bytefn(dav1d_read_coef_blocks)(Dav1dTaskContext *const t, const enum BlockSize bs, const Av1Block *const b) { const Dav1dFrameContext *const f = t->f; const int ss_ver = f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; const int ss_hor = f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I444; @@ -853,11 +854,11 @@ } Dav1dTileState *const ts = t->ts; const int w4 = imin(bw4, f->bw - t->bx), h4 = imin(bh4, f->bh - t->by); const int cw4 = (w4 + ss_hor) >> ss_hor, ch4 = (h4 + ss_ver) >> ss_ver; - assert(f->frame_thread.pass == 1); + assert(t->frame_thread.pass == 1); assert(!b->skip); const TxfmInfo *const uv_t_dim = &dav1d_txfm_dimensions[b->uvtx]; const TxfmInfo *const t_dim = &dav1d_txfm_dimensions[b->intra ? b->tx : b->max_ytx]; const uint16_t tx_split[2] = { b->tx_split0, b->tx_split1 }; @@ -882,16 +883,16 @@ uint8_t cf_ctx = 0x40; enum TxfmType txtp; const int eob = cbi[t->bx].eob[0] = decode_coefs(t, &t->a->lcoef[bx4 + x], &t->l.lcoef[by4 + y], b->tx, bs, b, 1, - 0, ts->frame_thread.cf, &txtp, &cf_ctx); + 0, ts->frame_thread[1].cf, &txtp, &cf_ctx); if (DEBUG_BLOCK_INFO) printf("Post-y-cf-blk[tx=%d,txtp=%d,eob=%d]: r=%d\n", b->tx, txtp, eob, ts->msac.rng); cbi[t->bx].txtp[0] = txtp; - ts->frame_thread.cf += imin(t_dim->w, 8) * imin(t_dim->h, 8) * 16; + ts->frame_thread[1].cf += imin(t_dim->w, 8) * imin(t_dim->h, 8) * 16; #define set_ctx(type, dir, diridx, off, mul, rep_macro) \ rep_macro(type, t->dir lcoef, off, mul * cf_ctx) #define default_memset(dir, diridx, off, sz) \ memset(&t->dir lcoef[off], cf_ctx, sz) case_set_upto16_with_default(imin(t_dim->h, f->bh - t->by), @@ -925,18 +926,18 @@ txtp = t->txtp_map[(by4 + (y << ss_ver)) * 32 + bx4 + (x << ss_hor)]; const int eob = cbi[t->bx].eob[1 + pl] = decode_coefs(t, &t->a->ccoef[pl][cbx4 + x], &t->l.ccoef[pl][cby4 + y], b->uvtx, bs, - b, b->intra, 1 + pl, ts->frame_thread.cf, + b, b->intra, 1 + pl, ts->frame_thread[1].cf, &txtp, &cf_ctx); if (DEBUG_BLOCK_INFO) printf("Post-uv-cf-blk[pl=%d,tx=%d," "txtp=%d,eob=%d]: r=%d\n", pl, b->uvtx, txtp, eob, ts->msac.rng); cbi[t->bx].txtp[1 + pl] = txtp; - ts->frame_thread.cf += uv_t_dim->w * uv_t_dim->h * 16; + ts->frame_thread[1].cf += uv_t_dim->w * uv_t_dim->h * 16; #define set_ctx(type, dir, diridx, off, mul, rep_macro) \ rep_macro(type, t->dir ccoef[pl], off, mul * cf_ctx) #define default_memset(dir, diridx, off, sz) \ memset(&t->dir ccoef[pl][off], cf_ctx, sz) case_set_upto16_with_default( \ @@ -954,11 +955,11 @@ } } } } -static int mc(Dav1dTileContext *const t, +static int mc(Dav1dTaskContext *const t, pixel *const dst8, int16_t *const dst16, const ptrdiff_t dst_stride, const int bw4, const int bh4, const int bx, const int by, const int pl, const mv mv, const Dav1dThreadPicture *const refp, const int refidx, const enum Filter2d filter_2d) @@ -977,15 +978,10 @@ const int dx = bx * h_mul + (mvx >> (3 + ss_hor)); const int dy = by * v_mul + (mvy >> (3 + ss_ver)); int w, h; if (refp->p.data[0] != f->cur.data[0]) { // i.e. not for intrabc - if (dav1d_thread_picture_wait(refp, dy + bh4 * v_mul + !!my * 4, - PLANE_TYPE_Y + !!pl)) - { - return -1; - } w = (f->cur.p.w + ss_hor) >> ss_hor; h = (f->cur.p.h + ss_ver) >> ss_ver; } else { w = f->bw * 4 >> ss_hor; h = f->bh * 4 >> ss_ver; @@ -1032,12 +1028,10 @@ const int right = ((pos_x + (bw4 * h_mul - 1) * f->svc[refidx][0].step) >> 10) + 1; const int bottom = ((pos_y + (bh4 * v_mul - 1) * f->svc[refidx][1].step) >> 10) + 1; - if (dav1d_thread_picture_wait(refp, bottom + 4, PLANE_TYPE_Y + !!pl)) - return -1; if (DEBUG_BLOCK_INFO) printf("Off %dx%d [%d,%d,%d], size %dx%d [%d,%d]\n", left, top, orig_pos_x, f->svc[refidx][0].scale, refidx, right-left, bottom-top, f->svc[refidx][0].step, f->svc[refidx][1].step); @@ -1075,11 +1069,11 @@ } return 0; } -static int obmc(Dav1dTileContext *const t, +static int obmc(Dav1dTaskContext *const t, pixel *const dst, const ptrdiff_t dst_stride, const uint8_t *const b_dim, const int pl, const int bx4, const int by4, const int w4, const int h4) { assert(!(t->bx & 1) && !(t->by & 1)); @@ -1136,11 +1130,11 @@ y += imax(l_b_dim[1], 2); } return 0; } -static int warp_affine(Dav1dTileContext *const t, +static int warp_affine(Dav1dTaskContext *const t, pixel *dst8, int16_t *dst16, const ptrdiff_t dstride, const uint8_t *const b_dim, const int pl, const Dav1dThreadPicture *const refp, const Dav1dWarpedMotionParams *const wmp) { @@ -1174,15 +1168,10 @@ wmp->u.p.delta * 4) & ~0x3f; const pixel *ref_ptr; ptrdiff_t ref_stride = refp->p.stride[!!pl]; - if (dav1d_thread_picture_wait(refp, dy + 4 + 8, - PLANE_TYPE_Y + !!pl)) - { - return -1; - } if (dx < 3 || dx + 8 + 4 > width || dy < 3 || dy + 8 + 4 > height) { pixel *const emu_edge_buf = bitfn(t->scratch.emu_edge); f->dsp->mc.emu_edge(15, 15, width, height, dx - 3, dy - 3, emu_edge_buf, 32 * sizeof(pixel), refp->p.data[pl], ref_stride); @@ -1202,11 +1191,11 @@ else dst16 += 8 * dstride; } return 0; } -void bytefn(dav1d_recon_b_intra)(Dav1dTileContext *const t, const enum BlockSize bs, +void bytefn(dav1d_recon_b_intra)(Dav1dTaskContext *const t, const enum BlockSize bs, const enum EdgeFlags intra_edge_flags, const Av1Block *const b) { Dav1dTileState *const ts = t->ts; const Dav1dFrameContext *const f = t->f; @@ -1237,18 +1226,19 @@ for (int init_x = 0; init_x < w4; init_x += 16) { if (b->pal_sz[0]) { pixel *dst = ((pixel *) f->cur.data[0]) + 4 * (t->by * PXSTRIDE(f->cur.stride[0]) + t->bx); const uint8_t *pal_idx; - if (f->frame_thread.pass) { - assert(ts->frame_thread.pal_idx); - pal_idx = ts->frame_thread.pal_idx; - ts->frame_thread.pal_idx += bw4 * bh4 * 16; + if (t->frame_thread.pass) { + const int p = t->frame_thread.pass & 1; + assert(ts->frame_thread[p].pal_idx); + pal_idx = ts->frame_thread[p].pal_idx; + ts->frame_thread[p].pal_idx += bw4 * bh4 * 16; } else { pal_idx = t->scratch.pal_idx; } - const uint16_t *const pal = f->frame_thread.pass ? + const uint16_t *const pal = t->frame_thread.pass ? f->frame_thread.pal[((t->by >> 1) + (t->bx & 1)) * (f->b4_stride >> 1) + ((t->bx >> 1) + (t->by & 1))][0] : t->scratch.pal[0]; f->dsp->ipred.pal_pred(dst, f->cur.stride[0], pal, pal_idx, bw4 * 4, bh4 * 4); if (DEBUG_BLOCK_INFO && DEBUG_B_PIXELS) @@ -1321,13 +1311,14 @@ skip_y_pred: {} if (!b->skip) { coef *cf; int eob; enum TxfmType txtp; - if (f->frame_thread.pass) { - cf = ts->frame_thread.cf; - ts->frame_thread.cf += imin(t_dim->w, 8) * imin(t_dim->h, 8) * 16; + if (t->frame_thread.pass) { + const int p = t->frame_thread.pass & 1; + cf = ts->frame_thread[p].cf; + ts->frame_thread[p].cf += imin(t_dim->w, 8) * imin(t_dim->h, 8) * 16; const struct CodedBlockInfo *const cbi = &f->frame_thread.cbi[t->by * f->b4_stride + t->bx]; eob = cbi->eob[0]; txtp = cbi->txtp[0]; } else { @@ -1360,11 +1351,11 @@ cf, eob HIGHBD_CALL_SUFFIX); if (DEBUG_BLOCK_INFO && DEBUG_B_PIXELS) hex_dump(dst, f->cur.stride[0], t_dim->w * 4, t_dim->h * 4, "recon"); } - } else if (!f->frame_thread.pass) { + } else if (!t->frame_thread.pass) { #define set_ctx(type, dir, diridx, off, mul, rep_macro) \ rep_macro(type, t->dir lcoef, off, mul * 0x40) case_set_upto16(t_dim->h, l., 1, by4 + y); case_set_upto16(t_dim->w, a->, 0, bx4 + x); #undef set_ctx @@ -1433,16 +1424,17 @@ } else if (b->pal_sz[1]) { const ptrdiff_t uv_dstoff = 4 * ((t->bx >> ss_hor) + (t->by >> ss_ver) * PXSTRIDE(f->cur.stride[1])); const uint16_t (*pal)[8]; const uint8_t *pal_idx; - if (f->frame_thread.pass) { - assert(ts->frame_thread.pal_idx); + if (t->frame_thread.pass) { + const int p = t->frame_thread.pass & 1; + assert(ts->frame_thread[p].pal_idx); pal = f->frame_thread.pal[((t->by >> 1) + (t->bx & 1)) * (f->b4_stride >> 1) + ((t->bx >> 1) + (t->by & 1))]; - pal_idx = ts->frame_thread.pal_idx; - ts->frame_thread.pal_idx += cbw4 * cbh4 * 16; + pal_idx = ts->frame_thread[p].pal_idx; + ts->frame_thread[p].pal_idx += cbw4 * cbh4 * 16; } else { pal = t->scratch.pal; pal_idx = &t->scratch.pal_idx[bw4 * bh4 * 16]; } @@ -1543,13 +1535,14 @@ skip_uv_pred: {} if (!b->skip) { enum TxfmType txtp; int eob; coef *cf; - if (f->frame_thread.pass) { - cf = ts->frame_thread.cf; - ts->frame_thread.cf += uv_t_dim->w * uv_t_dim->h * 16; + if (t->frame_thread.pass) { + const int p = t->frame_thread.pass & 1; + cf = ts->frame_thread[p].cf; + ts->frame_thread[p].cf += uv_t_dim->w * uv_t_dim->h * 16; const struct CodedBlockInfo *const cbi = &f->frame_thread.cbi[t->by * f->b4_stride + t->bx]; eob = cbi->eob[pl + 1]; txtp = cbi->txtp[pl + 1]; } else { @@ -1585,11 +1578,11 @@ cf, eob HIGHBD_CALL_SUFFIX); if (DEBUG_BLOCK_INFO && DEBUG_B_PIXELS) hex_dump(dst, stride, uv_t_dim->w * 4, uv_t_dim->h * 4, "recon"); } - } else if (!f->frame_thread.pass) { + } else if (!t->frame_thread.pass) { #define set_ctx(type, dir, diridx, off, mul, rep_macro) \ rep_macro(type, t->dir ccoef[pl], off, mul * 0x40) case_set_upto16(uv_t_dim->h, l., 1, cby4 + y); case_set_upto16(uv_t_dim->w, a->, 0, cbx4 + x); #undef set_ctx @@ -1602,11 +1595,11 @@ } } } } -int bytefn(dav1d_recon_b_inter)(Dav1dTileContext *const t, const enum BlockSize bs, +int bytefn(dav1d_recon_b_inter)(Dav1dTaskContext *const t, const enum BlockSize bs, const Av1Block *const b) { Dav1dTileState *const ts = t->ts; const Dav1dFrameContext *const f = t->f; const Dav1dDSPContext *const dsp = f->dsp; @@ -1717,11 +1710,11 @@ NULL, f->cur.stride[1], bw4, bh4, t->bx - 1, t->by - 1, 1 + pl, r[-1][t->bx - 1].mv.mv[0], &f->refp[r[-1][t->bx - 1].ref.ref[0] - 1], r[-1][t->bx - 1].ref.ref[0] - 1, - f->frame_thread.pass != 2 ? t->tl_4x4_filter : + t->frame_thread.pass != 2 ? t->tl_4x4_filter : f->frame_thread.b[((t->by - 1) * f->b4_stride) + t->bx - 1].filter2d); if (res) return res; } v_off = 2 * PXSTRIDE(f->cur.stride[1]); h_off = 2; @@ -1733,11 +1726,11 @@ res = mc(t, ((pixel *) f->cur.data[1 + pl]) + uvdstoff + v_off, NULL, f->cur.stride[1], bw4, bh4, t->bx - 1, t->by, 1 + pl, r[0][t->bx - 1].mv.mv[0], &f->refp[r[0][t->bx - 1].ref.ref[0] - 1], r[0][t->bx - 1].ref.ref[0] - 1, - f->frame_thread.pass != 2 ? left_filter_2d : + t->frame_thread.pass != 2 ? left_filter_2d : f->frame_thread.b[(t->by * f->b4_stride) + t->bx - 1].filter2d); if (res) return res; } h_off = 2; } @@ -1748,11 +1741,11 @@ res = mc(t, ((pixel *) f->cur.data[1 + pl]) + uvdstoff + h_off, NULL, f->cur.stride[1], bw4, bh4, t->bx, t->by - 1, 1 + pl, r[-1][t->bx].mv.mv[0], &f->refp[r[-1][t->bx].ref.ref[0] - 1], r[-1][t->bx].ref.ref[0] - 1, - f->frame_thread.pass != 2 ? top_filter_2d : + t->frame_thread.pass != 2 ? top_filter_2d : f->frame_thread.b[((t->by - 1) * f->b4_stride) + t->bx].filter2d); if (res) return res; } v_off = 2 * PXSTRIDE(f->cur.stride[1]); } @@ -1992,13 +1985,14 @@ x < imin(cw4, (init_x + 16) >> ss_hor); x += uvtx->w) { coef *cf; int eob; enum TxfmType txtp; - if (f->frame_thread.pass) { - cf = ts->frame_thread.cf; - ts->frame_thread.cf += uvtx->w * uvtx->h * 16; + if (t->frame_thread.pass) { + const int p = t->frame_thread.pass & 1; + cf = ts->frame_thread[p].cf; + ts->frame_thread[p].cf += uvtx->w * uvtx->h * 16; const struct CodedBlockInfo *const cbi = &f->frame_thread.cbi[t->by * f->b4_stride + t->bx]; eob = cbi->eob[1 + pl]; txtp = cbi->txtp[1 + pl]; } else { @@ -2049,24 +2043,35 @@ } } return 0; } -void bytefn(dav1d_filter_sbrow_deblock)(Dav1dFrameContext*const f, const int sby) { +void bytefn(dav1d_filter_sbrow_deblock_cols)(Dav1dFrameContext*const f, const int sby) { + const int y = sby * f->sb_step * 4; + const int ss_ver = f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; + pixel *const p[3] = { + f->lf.p[0] + y * PXSTRIDE(f->cur.stride[0]), + f->lf.p[1] + (y * PXSTRIDE(f->cur.stride[1]) >> ss_ver), + f->lf.p[2] + (y * PXSTRIDE(f->cur.stride[1]) >> ss_ver) + }; + Av1Filter *mask = f->lf.mask + (sby >> !f->seq_hdr->sb128) * f->sb128w; + if (f->frame_hdr->loopfilter.level_y[0] || f->frame_hdr->loopfilter.level_y[1]) + bytefn(dav1d_loopfilter_sbrow_cols)(f, p, mask, sby, + f->lf.start_of_tile_row[sby]); +} + +void bytefn(dav1d_filter_sbrow_deblock_rows)(Dav1dFrameContext*const f, const int sby) { const int y = sby * f->sb_step * 4; const int ss_ver = f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; pixel *const p[3] = { f->lf.p[0] + y * PXSTRIDE(f->cur.stride[0]), f->lf.p[1] + (y * PXSTRIDE(f->cur.stride[1]) >> ss_ver), f->lf.p[2] + (y * PXSTRIDE(f->cur.stride[1]) >> ss_ver) }; Av1Filter *mask = f->lf.mask + (sby >> !f->seq_hdr->sb128) * f->sb128w; if (f->frame_hdr->loopfilter.level_y[0] || f->frame_hdr->loopfilter.level_y[1]) { - int start_of_tile_row = 0; - if (f->frame_hdr->tiling.row_start_sb[f->lf.tile_row] == sby) - start_of_tile_row = f->lf.tile_row++; - bytefn(dav1d_loopfilter_sbrow)(f, p, mask, sby, start_of_tile_row); + bytefn(dav1d_loopfilter_sbrow_rows)(f, p, mask, sby); } if (f->lf.restore_planes) { // Store loop filtered pixels required by loop restoration bytefn(dav1d_lr_copy_lpf)(f, p, sby); } @@ -2143,20 +2148,21 @@ }; bytefn(dav1d_lr_sbrow)(f, sr_p, sby); } void bytefn(dav1d_filter_sbrow)(Dav1dFrameContext *const f, const int sby) { - bytefn(dav1d_filter_sbrow_deblock)(f, sby); + bytefn(dav1d_filter_sbrow_deblock_cols)(f, sby); + bytefn(dav1d_filter_sbrow_deblock_rows)(f, sby); if (f->seq_hdr->cdef) bytefn(dav1d_filter_sbrow_cdef)(f, sby); if (f->frame_hdr->width[0] != f->frame_hdr->width[1]) bytefn(dav1d_filter_sbrow_resize)(f, sby); if (f->lf.restore_planes) bytefn(dav1d_filter_sbrow_lr)(f, sby); } -void bytefn(dav1d_backup_ipred_edge)(Dav1dTileContext *const t) { +void bytefn(dav1d_backup_ipred_edge)(Dav1dTaskContext *const t) { const Dav1dFrameContext *const f = t->f; Dav1dTileState *const ts = t->ts; const int sby = t->by >> f->sb_shift; const int sby_off = f->sb128w * 128 * sby; const int x_off = ts->tiling.col_start; diff --git a/third_party/dav1d/src/refmvs.h b/third_party/dav1d/src/refmvs.h --- a/third_party/dav1d/src/refmvs.h +++ b/third_party/dav1d/src/refmvs.h @@ -33,11 +33,10 @@ #include "dav1d/headers.h" #include "common/intops.h" #include "src/intra_edge.h" -#include "src/levels.h" #include "src/tables.h" #define INVALID_MV 0x80008000 typedef struct refmvs_temporal_block { @@ -53,15 +52,15 @@ typedef union refmvs_mvpair { mv mv[2]; uint64_t n; } refmvs_mvpair; -typedef struct refmvs_block { +PACKED(typedef struct refmvs_block { refmvs_mvpair mv; refmvs_refpair ref; uint8_t bs, mf; // 1 = globalmv+affine, 2 = newmv -} refmvs_block; +}) ALIGN(refmvs_block, 4); typedef struct refmvs_frame { const Dav1dFrameHeader *frm_hdr; int iw4, ih4, iw8, ih8; int sbsz; @@ -78,11 +77,11 @@ refmvs_temporal_block *rp_proj; ptrdiff_t rp_stride; refmvs_block *r; // 35 x r_stride memory ptrdiff_t r_stride; - int n_tile_rows, n_tile_threads; + int n_tile_rows, n_tile_threads, n_frame_threads; } refmvs_frame; typedef struct refmvs_tile { const refmvs_frame *rf; refmvs_block *r[32 + 5]; @@ -95,10 +94,18 @@ typedef struct refmvs_candidate { refmvs_mvpair mv; int weight; } refmvs_candidate; +#define decl_splat_mv_fn(name) \ +void (name)(refmvs_block **rr, const refmvs_block *rmv, int bx4, int bw4, int bh4) +typedef decl_splat_mv_fn(*splat_mv_fn); + +typedef struct Dav1dRefmvsDSPContext { + splat_mv_fn splat_mv; +} Dav1dRefmvsDSPContext; + // call once per frame thread void dav1d_refmvs_init(refmvs_frame *rf); void dav1d_refmvs_clear(refmvs_frame *rf); // call once per frame @@ -107,11 +114,11 @@ const Dav1dFrameHeader *frm_hdr, const unsigned ref_poc[7], refmvs_temporal_block *rp, const unsigned ref_ref_poc[7][7], /*const*/ refmvs_temporal_block *const rp_ref[7], - int n_tile_threads); + int n_tile_threads, int n_frame_threads); // initialize temporal MVs; this can be done in any configuration, e.g. one // tile/sbrow at a time, where col_{start,end}8 are the tile boundaries; or // it can just be for the whole frame's sbrow, where col_{start,end}8 are the // frame boundaries. row_{start,end}8 are the superblock row boundaries. @@ -127,107 +134,18 @@ // initialize tile boundaries and refmvs_block pointers for one tile/sbrow void dav1d_refmvs_tile_sbrow_init(refmvs_tile *rt, const refmvs_frame *rf, int tile_col_start4, int tile_col_end4, int tile_row_start4, int tile_row_end4, - int sby, int tile_row_idx); + int sby, int tile_row_idx, int pass); // call for each block void dav1d_refmvs_find(const refmvs_tile *rt, refmvs_candidate mvstack[8], int *cnt, int *ctx, const refmvs_refpair ref, enum BlockSize bs, enum EdgeFlags edge_flags, int by4, int bx4); -static inline void splat_oneref_mv(refmvs_tile *const rt, - const int by4, const int bx4, - const enum BlockSize bs, - const enum InterPredMode mode, - const int ref, const mv mv, - const int is_interintra) -{ - const int bw4 = dav1d_block_dimensions[bs][0]; - int bh4 = dav1d_block_dimensions[bs][1]; - refmvs_block **rr = &rt->r[(by4 & 31) + 5]; - - const refmvs_block tmpl = (refmvs_block) { - .ref.ref = { ref + 1, is_interintra ? 0 : -1 }, - .mv.mv[0] = mv, - .bs = bs, - .mf = (mode == GLOBALMV && imin(bw4, bh4) >= 2) | ((mode == NEWMV) * 2), - }; - do { - refmvs_block *r = *rr++ + bx4; - for (int x = 0; x < bw4; x++) - r[x] = tmpl; - } while (--bh4); -} - -static inline void splat_intrabc_mv(refmvs_tile *const rt, - const int by4, const int bx4, - const enum BlockSize bs, const mv mv) -{ - const int bw4 = dav1d_block_dimensions[bs][0]; - int bh4 = dav1d_block_dimensions[bs][1]; - refmvs_block **rr = &rt->r[(by4 & 31) + 5]; - - const refmvs_block tmpl = (refmvs_block) { - .ref.ref = { 0, -1 }, - .mv.mv[0] = mv, - .bs = bs, - .mf = 0, - }; - do { - refmvs_block *r = *rr++ + bx4; - for (int x = 0; x < bw4; x++) { - r[x] = tmpl; - } - } while (--bh4); -} - -static inline void splat_tworef_mv(refmvs_tile *const rt, - const int by4, const int bx4, - const enum BlockSize bs, - const enum CompInterPredMode mode, - const refmvs_refpair ref, - const refmvs_mvpair mv) -{ - const int bw4 = dav1d_block_dimensions[bs][0]; - int bh4 = dav1d_block_dimensions[bs][1]; - refmvs_block **rr = &rt->r[(by4 & 31) + 5]; - - assert(bw4 >= 2 && bh4 >= 2); - const refmvs_block tmpl = (refmvs_block) { - .ref.pair = ref.pair + 0x0101, - .mv = mv, - .bs = bs, - .mf = (mode == GLOBALMV_GLOBALMV) | !!((1 << mode) & (0xbc)) * 2, - }; - do { - refmvs_block *r = *rr++ + bx4; - for (int x = 0; x < bw4; x++) - r[x] = tmpl; - } while (--bh4); -} - -static inline void splat_intraref(refmvs_tile *const rt, - const int by4, const int bx4, - const enum BlockSize bs) -{ - const int bw4 = dav1d_block_dimensions[bs][0]; - int bh4 = dav1d_block_dimensions[bs][1]; - refmvs_block **rr = &rt->r[(by4 & 31) + 5]; - - const refmvs_block tmpl = (refmvs_block) { - .ref.ref = { 0, -1 }, - .mv.mv[0].n = INVALID_MV, - .bs = bs, - .mf = 0, - }; - do { - refmvs_block *r = *rr++ + bx4; - for (int x = 0; x < bw4; x++) { - r[x] = tmpl; - } - } while (--bh4); -} +void dav1d_refmvs_dsp_init(Dav1dRefmvsDSPContext *dsp); +void dav1d_refmvs_dsp_init_arm(Dav1dRefmvsDSPContext *dsp); +void dav1d_refmvs_dsp_init_x86(Dav1dRefmvsDSPContext *dsp); #endif /* DAV1D_SRC_REF_MVS_H */ diff --git a/third_party/dav1d/src/refmvs.c b/third_party/dav1d/src/refmvs.c --- a/third_party/dav1d/src/refmvs.c +++ b/third_party/dav1d/src/refmvs.c @@ -33,10 +33,11 @@ #include "dav1d/common.h" #include "common/intops.h" #include "src/env.h" +#include "src/mem.h" #include "src/refmvs.h" static void add_spatial_candidate(refmvs_candidate *const mvstack, int *const cnt, const int weight, const refmvs_block *const b, const union refmvs_refpair ref, const mv gmv[2], @@ -650,15 +651,18 @@ } void dav1d_refmvs_tile_sbrow_init(refmvs_tile *const rt, const refmvs_frame *const rf, const int tile_col_start4, const int tile_col_end4, const int tile_row_start4, const int tile_row_end4, - const int sby, int tile_row_idx) + const int sby, int tile_row_idx, const int pass) { if (rf->n_tile_threads == 1) tile_row_idx = 0; rt->rp_proj = &rf->rp_proj[16 * rf->rp_stride * tile_row_idx]; - refmvs_block *r = &rf->r[35 * rf->r_stride * tile_row_idx]; + const int uses_2pass = rf->n_tile_threads > 1 && rf->n_frame_threads > 1; + const ptrdiff_t pass_off = (uses_2pass && pass == 2) ? + 35 * rf->r_stride * rf->n_tile_rows : 0; + refmvs_block *r = &rf->r[35 * rf->r_stride * tile_row_idx + pass_off]; const int sbsz = rf->sbsz; const int off = (sbsz * sby) & 16; for (int i = 0; i < sbsz; i++, r += rf->r_stride) rt->r[off + 5 + i] = r; rt->r[off + 0] = r; @@ -803,11 +807,11 @@ const Dav1dFrameHeader *const frm_hdr, const unsigned ref_poc[7], refmvs_temporal_block *const rp, const unsigned ref_ref_poc[7][7], /*const*/ refmvs_temporal_block *const rp_ref[7], - const int n_tile_threads) + const int n_tile_threads, const int n_frame_threads) { rf->sbsz = 16 << seq_hdr->sb128; rf->frm_hdr = frm_hdr; rf->iw8 = (frm_hdr->width[0] + 7) >> 3; rf->ih8 = (frm_hdr->height + 7) >> 3; @@ -815,25 +819,27 @@ rf->ih4 = rf->ih8 << 1; const ptrdiff_t r_stride = ((frm_hdr->width[0] + 127) & ~127) >> 2; const int n_tile_rows = n_tile_threads > 1 ? frm_hdr->tiling.rows : 1; if (r_stride != rf->r_stride || n_tile_rows != rf->n_tile_rows) { - if (rf->r) free(rf->r); - rf->r = malloc(sizeof(*rf->r) * 35 * r_stride * n_tile_rows); + if (rf->r) dav1d_freep_aligned(&rf->r); + const int uses_2pass = n_tile_threads > 1 && n_frame_threads > 1; + rf->r = dav1d_alloc_aligned(sizeof(*rf->r) * 35 * r_stride * n_tile_rows * (1 + uses_2pass), 64); if (!rf->r) return DAV1D_ERR(ENOMEM); rf->r_stride = r_stride; } const ptrdiff_t rp_stride = r_stride >> 1; if (rp_stride != rf->rp_stride || n_tile_rows != rf->n_tile_rows) { - if (rf->rp_proj) free(rf->rp_proj); - rf->rp_proj = malloc(sizeof(*rf->rp_proj) * 16 * rp_stride * n_tile_rows); + if (rf->rp_proj) dav1d_freep_aligned(&rf->rp_proj); + rf->rp_proj = dav1d_alloc_aligned(sizeof(*rf->rp_proj) * 16 * rp_stride * n_tile_rows, 64); if (!rf->rp_proj) return DAV1D_ERR(ENOMEM); rf->rp_stride = rp_stride; } rf->n_tile_rows = n_tile_rows; rf->n_tile_threads = n_tile_threads; + rf->n_frame_threads = n_frame_threads; rf->rp = rp; rf->rp_ref = rp_ref; const unsigned poc = frm_hdr->frame_offset; for (int i = 0; i < 7; i++) { const int poc_diff = get_poc_diff(seq_hdr->order_hint_n_bits, @@ -900,8 +906,31 @@ rf->rp_proj = NULL; rf->rp_stride = 0; } void dav1d_refmvs_clear(refmvs_frame *const rf) { - if (rf->r) free(rf->r); - if (rf->rp_proj) free(rf->rp_proj); + if (rf->r) dav1d_freep_aligned(&rf->r); + if (rf->rp_proj) dav1d_freep_aligned(&rf->rp_proj); +} + +static void splat_mv_c(refmvs_block **rr, const refmvs_block *const rmv, + const int bx4, const int bw4, int bh4) +{ + do { + refmvs_block *const r = *rr++ + bx4; + for (int x = 0; x < bw4; x++) + r[x] = *rmv; + } while (--bh4); } + +COLD void dav1d_refmvs_dsp_init(Dav1dRefmvsDSPContext *const c) +{ + c->splat_mv = splat_mv_c; + +#if HAVE_ASM +#if ARCH_AARCH64 || ARCH_ARM + dav1d_refmvs_dsp_init_arm(c); +#elif ARCH_X86 + dav1d_refmvs_dsp_init_x86(c); +#endif +#endif +} diff --git a/third_party/dav1d/src/thread_task.h b/third_party/dav1d/src/thread_task.h --- a/third_party/dav1d/src/thread_task.h +++ b/third_party/dav1d/src/thread_task.h @@ -33,35 +33,18 @@ #include "src/internal.h" #define FRAME_ERROR (UINT_MAX - 1) #define TILE_ERROR (INT_MAX - 1) -enum TaskStatus { - DAV1D_TASK_DEFAULT, - DAV1D_TASK_READY, - DAV1D_TASK_RUNNING, - DAV1D_TASK_DONE, -}; +// these functions assume the task scheduling lock is already taken +int dav1d_task_create_tile_sbrow(Dav1dFrameContext *f, int pass, int cond_signal); +void dav1d_task_frame_init(Dav1dFrameContext *f); + +void *dav1d_worker_task(void *data); -struct Dav1dTask { - enum TaskStatus status; // task status - int start; // frame thread start flag - unsigned frame_idx; // frame thread id - int frame_id; // frame ordering - int sby; // sbrow - filter_sbrow_fn fn; // task work - Dav1dTask *last_deps[2]; // dependencies - Dav1dTask *next_deps[2]; // dependant tasks - Dav1dTask *next_exec; // tasks scheduling -}; - -int dav1d_task_create_filter_sbrow(Dav1dFrameContext *f); -void dav1d_task_schedule(struct PostFilterThreadData *pftd, Dav1dTask *t); - -void *dav1d_frame_task(void *data); -void *dav1d_tile_task(void *data); -void *dav1d_postfilter_task(void *data); - +int dav1d_decode_frame_init(Dav1dFrameContext *f); +int dav1d_decode_frame_main(Dav1dFrameContext *f); +void dav1d_decode_frame_exit(Dav1dFrameContext *f, int retval); int dav1d_decode_frame(Dav1dFrameContext *f); -int dav1d_decode_tile_sbrow(Dav1dTileContext *t); +int dav1d_decode_tile_sbrow(Dav1dTaskContext *t); #endif /* DAV1D_SRC_THREAD_TASK_H */ diff --git a/third_party/dav1d/src/thread_task.c b/third_party/dav1d/src/thread_task.c --- a/third_party/dav1d/src/thread_task.c +++ b/third_party/dav1d/src/thread_task.c @@ -25,347 +25,663 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "config.h" +#include "common/frame.h" + #include "src/thread_task.h" -int dav1d_task_create_filter_sbrow(Dav1dFrameContext *const f) { - struct PostFilterThreadData *const pftd = f->lf.thread.pftd; - const int frame_idx = (int)(f - f->c->fc); +// This function resets the cur pointer to the first frame theoretically +// executable after a task completed (ie. each time we update some progress or +// insert some tasks in the queue). +// When frame_idx is set, it can be either from a completed task, or from tasks +// inserted in the queue, in which case we have to make sure the cur pointer +// isn't past this insert. +// The special case where frame_idx is UINT_MAX is to handle the reset after +// completing a task and locklessly signaling progress. In this case we don't +// enter a critical section, which is needed for this function, so we set an +// atomic for a delayed handling, happening here. Meaning we can call this +// function without any actual update other than what's in the atomic, hence +// this special case. +static inline int reset_task_cur(const Dav1dContext *const c, + struct TaskThreadData *const ttd, + unsigned frame_idx) +{ + const unsigned first = atomic_load(&ttd->first); + if (!ttd->cur && c->fc[first].task_thread.task_cur_prev == NULL) + return 0; + unsigned reset_frame_idx = atomic_exchange(&ttd->reset_task_cur, UINT_MAX); + if (reset_frame_idx != UINT_MAX) { + if (frame_idx == UINT_MAX) { + if (reset_frame_idx > first + ttd->cur) + return 0; + ttd->cur = reset_frame_idx - first; + goto cur_found; + } + } else if (frame_idx == UINT_MAX) + return 0; + if (frame_idx < first) frame_idx += c->n_fc; + const unsigned min_frame_idx = umin(reset_frame_idx, frame_idx); + const unsigned cur_frame_idx = first + ttd->cur; + if (ttd->cur < c->n_fc && cur_frame_idx < min_frame_idx) + return 0; + for (ttd->cur = min_frame_idx - first; ttd->cur < c->n_fc; ttd->cur++) + if (c->fc[(first + ttd->cur) % c->n_fc].task_thread.task_head) + break; +cur_found: + for (unsigned i = ttd->cur; i < c->n_fc; i++) + c->fc[(first + i) % c->n_fc].task_thread.task_cur_prev = NULL; + return 1; +} + +static inline void reset_task_cur_async(struct TaskThreadData *const ttd, + unsigned frame_idx, unsigned n_frames) +{ + if (frame_idx < (unsigned)atomic_load(&ttd->first)) frame_idx += n_frames; + unsigned last_idx = frame_idx; + do { + frame_idx = last_idx; + last_idx = atomic_exchange(&ttd->reset_task_cur, frame_idx); + } while (last_idx < frame_idx); +} +static void insert_tasks_between(Dav1dFrameContext *const f, + Dav1dTask *const first, Dav1dTask *const last, + Dav1dTask *const a, Dav1dTask *const b, + const int cond_signal) +{ + struct TaskThreadData *const ttd = f->task_thread.ttd; + if (atomic_load(f->c->flush)) return; + assert(!a || a->next == b); + if (!a) f->task_thread.task_head = first; + else a->next = first; + if (!b) f->task_thread.task_tail = last; + last->next = b; + reset_task_cur(f->c, ttd, first->frame_idx); + if (cond_signal && !atomic_fetch_or(&ttd->cond_signaled, 1)) + pthread_cond_signal(&ttd->cond); +} + +static void insert_tasks(Dav1dFrameContext *const f, + Dav1dTask *const first, Dav1dTask *const last, + const int cond_signal) +{ + // insert task back into task queue + Dav1dTask *t_ptr, *prev_t = NULL; + for (t_ptr = f->task_thread.task_head; + t_ptr; prev_t = t_ptr, t_ptr = t_ptr->next) + { + // entropy coding precedes other steps + if (t_ptr->type == DAV1D_TASK_TYPE_TILE_ENTROPY) { + if (first->type > DAV1D_TASK_TYPE_TILE_ENTROPY) continue; + // both are entropy + if (first->sby > t_ptr->sby) continue; + if (first->sby < t_ptr->sby) { + insert_tasks_between(f, first, last, prev_t, t_ptr, cond_signal); + return; + } + // same sby + } else { + if (first->type == DAV1D_TASK_TYPE_TILE_ENTROPY) { + insert_tasks_between(f, first, last, prev_t, t_ptr, cond_signal); + return; + } + if (first->sby > t_ptr->sby) continue; + if (first->sby < t_ptr->sby) { + insert_tasks_between(f, first, last, prev_t, t_ptr, cond_signal); + return; + } + // same sby + if (first->type > t_ptr->type) continue; + if (first->type < t_ptr->type) { + insert_tasks_between(f, first, last, prev_t, t_ptr, cond_signal); + return; + } + // same task type + } + + // sort by tile-id + assert(first->type <= DAV1D_TASK_TYPE_TILE_RECONSTRUCTION); + assert(first->type == t_ptr->type); + assert(t_ptr->sby == first->sby); + const int p = first->type == DAV1D_TASK_TYPE_TILE_ENTROPY; + const int t_tile_idx = (int) (first - f->task_thread.tile_tasks[p]); + const int p_tile_idx = (int) (t_ptr - f->task_thread.tile_tasks[p]); + assert(t_tile_idx != p_tile_idx); + if (t_tile_idx > p_tile_idx) continue; + insert_tasks_between(f, first, last, prev_t, t_ptr, cond_signal); + return; + } + // append at the end + insert_tasks_between(f, first, last, prev_t, NULL, cond_signal); +} + +static inline void insert_task(Dav1dFrameContext *const f, + Dav1dTask *const t, const int cond_signal) +{ + insert_tasks(f, t, t, cond_signal); +} + +static int create_filter_sbrow(Dav1dFrameContext *const f, + const int pass, Dav1dTask **res_t) +{ const int has_deblock = f->frame_hdr->loopfilter.level_y[0] || - f->frame_hdr->loopfilter.level_y[1] || - f->lf.restore_planes; + f->frame_hdr->loopfilter.level_y[1]; const int has_cdef = f->seq_hdr->cdef; const int has_resize = f->frame_hdr->width[0] != f->frame_hdr->width[1]; - const int has_lr = !!f->lf.restore_planes; - f->lf.thread.npf = has_deblock + has_cdef + has_resize + has_lr; - if (f->lf.thread.npf == 0) return 0; + const int has_lr = f->lf.restore_planes; - pthread_mutex_lock(&pftd->lock); - - Dav1dTask *tasks = f->lf.thread.tasks; - int num_tasks = f->sbh * f->lf.thread.npf; - if (num_tasks > f->lf.thread.num_tasks) { + Dav1dTask *tasks = f->task_thread.tasks; + const int uses_2pass = f->c->n_fc > 1; + int num_tasks = f->sbh * (1 + uses_2pass); + if (num_tasks > f->task_thread.num_tasks) { const size_t size = sizeof(Dav1dTask) * num_tasks; - tasks = realloc(f->lf.thread.tasks, size); - if (!tasks) { - pthread_mutex_unlock(&pftd->lock); - return -1; - } + tasks = realloc(f->task_thread.tasks, size); + if (!tasks) return -1; memset(tasks, 0, size); - f->lf.thread.tasks = tasks; - f->lf.thread.num_tasks = num_tasks; + f->task_thread.tasks = tasks; + f->task_thread.num_tasks = num_tasks; } + tasks += f->sbh * (pass & 1); -#define create_task(task, ready_cond, start_cond) \ - do { \ - t = &tasks[num_tasks++]; \ - t->status = ready_cond ? DAV1D_TASK_READY : DAV1D_TASK_DEFAULT; \ - t->start = start_cond; \ - t->frame_id = frame_cnt; \ - t->frame_idx = frame_idx; \ - t->sby = sby; \ - t->fn = f->bd_fn.filter_sbrow_##task; \ - t->last_deps[0] = NULL; \ - t->last_deps[1] = NULL; \ - t->next_deps[0] = NULL; \ - t->next_deps[1] = NULL; \ - t->next_exec = NULL; \ - } while (0) + if (pass & 1) { + f->frame_thread.entropy_progress = 0; + } else { + atomic_store(&f->frame_thread.deblock_progress, 0); + atomic_store(&f->frame_thread.cdef_progress, 0); + atomic_store(&f->frame_thread.lr_progress, 0); + } + f->frame_thread.next_tile_row[pass & 1] = 0; - Dav1dTask *last_sbrow_deblock = NULL; - Dav1dTask *last_sbrow_cdef = NULL; - Dav1dTask *last_sbrow_resize = NULL; - Dav1dTask *last_sbrow_lr = NULL; - num_tasks = 0; - const int frame_cnt = pftd->frame_cnt++; + Dav1dTask *t = &tasks[0]; + t->sby = 0; + t->recon_progress = 1; + t->deblock_progress = 0; + t->cdef_progress = 0; + t->lr_progress = 0; + t->type = pass == 1 ? DAV1D_TASK_TYPE_ENTROPY_PROGRESS : + has_deblock ? DAV1D_TASK_TYPE_DEBLOCK_COLS : + has_lr /* i.e. LR backup */ ? DAV1D_TASK_TYPE_DEBLOCK_ROWS : + has_cdef ? DAV1D_TASK_TYPE_CDEF : + has_resize ? DAV1D_TASK_TYPE_SUPER_RESOLUTION : + DAV1D_TASK_TYPE_LOOP_RESTORATION; + t->frame_idx = (int)(f - f->c->fc); + + *res_t = t; + return 0; +} - for (int sby = 0; sby < f->sbh; ++sby) { - Dav1dTask *t; - Dav1dTask *last = NULL; - if (has_deblock) { - create_task(deblock, sby == 0, 0); - if (sby) { - t->last_deps[1] = last_sbrow_deblock; - last_sbrow_deblock->next_deps[1] = t; - } - last = t; - last_sbrow_deblock = t; +int dav1d_task_create_tile_sbrow(Dav1dFrameContext *const f, const int pass, + const int cond_signal) +{ + Dav1dTask *tasks = f->task_thread.tile_tasks[0]; + const int uses_2pass = f->c->n_fc > 1; + const int num_tasks = f->frame_hdr->tiling.cols * f->frame_hdr->tiling.rows; + int alloc_num_tasks = num_tasks * (1 + uses_2pass); + if (alloc_num_tasks > f->task_thread.num_tile_tasks) { + const size_t size = sizeof(Dav1dTask) * alloc_num_tasks; + tasks = realloc(f->task_thread.tile_tasks[0], size); + if (!tasks) return -1; + memset(tasks, 0, size); + f->task_thread.tile_tasks[0] = tasks; + f->task_thread.num_tile_tasks = alloc_num_tasks; + } + f->task_thread.tile_tasks[1] = tasks + num_tasks; + tasks += num_tasks * (pass & 1); + + Dav1dTask *pf_t; + if (create_filter_sbrow(f, pass, &pf_t)) + return -1; + + Dav1dTask *prev_t = NULL; + for (int tile_idx = 0; tile_idx < num_tasks; tile_idx++) { + Dav1dTileState *const ts = &f->ts[tile_idx]; + Dav1dTask *t = &tasks[tile_idx]; + t->sby = ts->tiling.row_start >> f->sb_shift; + if (pf_t && t->sby) { + prev_t->next = pf_t; + prev_t = pf_t; + pf_t = NULL; } - if (has_cdef) { - create_task(cdef, sby == 0 && !has_deblock, has_deblock); - if (has_deblock) { - t->last_deps[0] = last; - last->next_deps[0] = t; - } - if (sby) { - t->last_deps[1] = last_sbrow_cdef; - last_sbrow_cdef->next_deps[1] = t; - } - last = t; - last_sbrow_cdef = t; - }; - if (has_resize) { - create_task(resize, sby == 0 && !last, !!last); - if (last) { - t->last_deps[0] = last; - last->next_deps[0] = t; - } - if (sby) { - t->last_deps[1] = last_sbrow_resize; - last_sbrow_resize->next_deps[1] = t; - } - last = t; - last_sbrow_resize = t; - } - if (has_lr) { - create_task(lr, sby == 0 && !last, !!last); - if (last) { - t->last_deps[0] = last; - last->next_deps[0] = t; - } - if (sby) { - t->last_deps[1] = last_sbrow_lr; - last_sbrow_lr->next_deps[1] = t; - } - last_sbrow_lr = t; - } + t->recon_progress = 0; + t->deblock_progress = 0; + t->cdef_progress = 0; + t->lr_progress = 0; + t->deps_skip = 0; + t->type = pass != 1 ? DAV1D_TASK_TYPE_TILE_RECONSTRUCTION : + DAV1D_TASK_TYPE_TILE_ENTROPY; + t->frame_idx = (int)(f - f->c->fc); + if (prev_t) prev_t->next = t; + prev_t = t; } - f->lf.thread.done = 0; - pthread_mutex_unlock(&pftd->lock); + if (pf_t) { + prev_t->next = pf_t; + prev_t = pf_t; + } + insert_tasks(f, &tasks[0], prev_t, cond_signal); + f->task_thread.done[pass & 1] = 0; return 0; } -void dav1d_task_schedule(struct PostFilterThreadData *const pftd, - Dav1dTask *const t) +void dav1d_task_frame_init(Dav1dFrameContext *const f) { + const Dav1dContext *const c = f->c; + + // schedule init task, which will schedule the remaining tasks + Dav1dTask *const t = &f->task_thread.init_task; + t->type = DAV1D_TASK_TYPE_INIT; + t->frame_idx = (int)(f - c->fc); + t->sby = 0; + t->recon_progress = t->deblock_progress = 0; + t->cdef_progress = t->lr_progress = 0; + insert_task(f, t, 1); +} + +static inline int ensure_progress(struct TaskThreadData *const ttd, + Dav1dFrameContext *const f, + Dav1dTask *const t, const enum TaskType type, + atomic_int *const state, int *const target) +{ + // deblock_rows (non-LR portion) depends on deblock of previous sbrow, + // so ensure that completed. if not, re-add to task-queue; else, fall-through + int p1 = atomic_load(state); + if (p1 < t->sby) { + pthread_mutex_lock(&ttd->lock); + p1 = atomic_load(state); + if (p1 < t->sby) { + t->type = type; + t->deblock_progress = t->recon_progress = 0; + t->cdef_progress = t->lr_progress = 0; + *target = t->sby; + insert_task(f, t, 0); + return 1; + } + pthread_mutex_unlock(&ttd->lock); + } + return 0; +} + +static inline int check_tile(Dav1dTask *const t, Dav1dFrameContext *const f, + const int frame_mt) { - Dav1dTask **pt = &pftd->tasks; - while (*pt && - ((*pt)->sby < t->sby || - ((*pt)->sby == t->sby && (*pt)->frame_id <= t->frame_id))) - pt = &(*pt)->next_exec; - t->next_exec = *pt; - *pt = t; - pthread_cond_signal(&pftd->cond); + const int tp = t->type == DAV1D_TASK_TYPE_TILE_ENTROPY; + const int tile_idx = (int)(t - f->task_thread.tile_tasks[tp]); + Dav1dTileState *const ts = &f->ts[tile_idx]; + const int p1 = atomic_load(&ts->progress[tp]); + if (p1 < t->sby) return 1; + int error = p1 == TILE_ERROR; + error |= atomic_fetch_or(&f->task_thread.error, error); + if (!error && frame_mt && !tp) { + const int p2 = atomic_load(&ts->progress[1]); + if (p2 <= t->sby) return 1; + error = p2 == TILE_ERROR; + error |= atomic_fetch_or(&f->task_thread.error, error); + } + if (!error && frame_mt && !IS_KEY_OR_INTRA(f->frame_hdr)) { + // check reference state + const Dav1dThreadPicture *p = &f->sr_cur; + const int ss_ver = p->p.p.layout == DAV1D_PIXEL_LAYOUT_I420; + const unsigned p_b = (t->sby + 1) << (f->sb_shift + 2); + const int tile_sby = t->sby - (ts->tiling.row_start >> f->sb_shift); + const int (*const lowest_px)[2] = ts->lowest_pixel[tile_sby]; + for (int n = t->deps_skip; n < 7; n++, t->deps_skip++) { + unsigned lowest; + if (tp) { + // if temporal mv refs are disabled, we only need this + // for the primary ref; if segmentation is disabled, we + // don't even need that + lowest = p_b; + } else { + // +8 is postfilter-induced delay + const int y = lowest_px[n][0] == INT_MIN ? INT_MIN : + lowest_px[n][0] + 8; + const int uv = lowest_px[n][1] == INT_MIN ? INT_MIN : + lowest_px[n][1] * (1 << ss_ver) + 8; + const int max = imax(y, uv); + if (max == INT_MIN) continue; + lowest = iclip(max, 1, f->refp[n].p.p.h); + } + const unsigned p3 = atomic_load(&f->refp[n].progress[!tp]); + if (p3 < lowest) return 1; + atomic_fetch_or(&f->task_thread.error, p3 == FRAME_ERROR); + } + } + return 0; } -static inline void update_task(Dav1dTask *const t, const int dep_type, - Dav1dFrameContext *const f) -{ - if (!t->last_deps[!dep_type] || - t->last_deps[!dep_type]->status == DAV1D_TASK_DONE) - { - t->status = DAV1D_TASK_READY; - if (t->start) - dav1d_task_schedule(f->lf.thread.pftd, t); - } -} +void *dav1d_worker_task(void *data) { + Dav1dTaskContext *const tc = data; + const Dav1dContext *const c = tc->c; + struct TaskThreadData *const ttd = tc->task_thread.ttd; + + dav1d_set_thread_name("dav1d-worker"); -void *dav1d_frame_task(void *const data) { - Dav1dFrameContext *const f = data; + pthread_mutex_lock(&ttd->lock); + for (;;) { + Dav1dFrameContext *f; + Dav1dTask *t, *prev_t; + if (tc->task_thread.die) break; + if (atomic_load(c->flush)) goto park; + while (ttd->cur < c->n_fc) { + const unsigned first = atomic_load(&ttd->first); + f = &c->fc[(first + ttd->cur) % c->n_fc]; + prev_t = f->task_thread.task_cur_prev; + t = prev_t ? prev_t->next : f->task_thread.task_head; + while (t) { + if (t->type == DAV1D_TASK_TYPE_INIT) { + const int p1 = f->in_cdf.progress ? + atomic_load(f->in_cdf.progress) : 1; + if (p1) { + atomic_fetch_or(&f->task_thread.error, p1 == TILE_ERROR); + goto found; + } + } else if (t->type <= DAV1D_TASK_TYPE_TILE_RECONSTRUCTION) { + // if not bottom sbrow of tile, this task will be re-added + // after it's finished + if (!check_tile(t, f, c->n_fc > 1)) + goto found; + } else if (t->recon_progress) { + const int p = t->type == DAV1D_TASK_TYPE_ENTROPY_PROGRESS; + int error = atomic_load(&f->task_thread.error); + assert(!f->task_thread.done[p] || error); + const int tile_row_base = f->frame_hdr->tiling.cols * + f->frame_thread.next_tile_row[p]; + if (p) { + const int p1 = f->frame_thread.entropy_progress; + if (p1 < t->sby) goto next; + atomic_fetch_or(&f->task_thread.error, p1 == TILE_ERROR); + } + for (int tc = 0; tc < f->frame_hdr->tiling.cols; tc++) { + Dav1dTileState *const ts = &f->ts[tile_row_base + tc]; + const int p2 = atomic_load(&ts->progress[p]); + if (p2 < t->recon_progress) goto next; + atomic_fetch_or(&f->task_thread.error, p2 == TILE_ERROR); + } + if (!p) { + atomic_int *state = NULL; + int needed; + if (t->cdef_progress) { + state = &f->frame_thread.cdef_progress; + needed = t->cdef_progress; + } else if (t->lr_progress) { + state = &f->frame_thread.lr_progress; + needed = t->lr_progress; + } + if (state) { + const int p3 = atomic_load(state); + if (p3 < needed) goto next; + atomic_fetch_or(&f->task_thread.error, p3 == TILE_ERROR); + } + } + if (t->sby + 1 < f->sbh) { + // add sby+1 to list to replace this one + Dav1dTask *next_t = &t[1]; + *next_t = *t; + next_t->sby++; + const int ntr = f->frame_thread.next_tile_row[p] + 1; + const int start = f->frame_hdr->tiling.row_start_sb[ntr]; + if (next_t->sby == start) + f->frame_thread.next_tile_row[p] = ntr; + next_t->recon_progress = next_t->sby + 1; + if (t->type == DAV1D_TASK_TYPE_CDEF) + next_t->cdef_progress = next_t->sby; + else if (t->type == DAV1D_TASK_TYPE_LOOP_RESTORATION) + next_t->lr_progress = next_t->sby; + insert_task(f, next_t, 0); + } + goto found; + } else { + assert(!!t->deblock_progress + !!t->cdef_progress + !!t->lr_progress == 1); + atomic_int *state; + int needed; + if (t->deblock_progress) { + needed = t->deblock_progress; + state = &f->frame_thread.deblock_progress; + } else if (t->cdef_progress) { + needed = t->cdef_progress; + state = &f->frame_thread.cdef_progress; + } else { + assert(t->lr_progress); + needed = t->lr_progress; + state = &f->frame_thread.lr_progress; + } + const int p1 = atomic_load(state); + if (p1 >= needed) { + atomic_fetch_or(&f->task_thread.error, p1 == TILE_ERROR); + goto found; + } + } + next: + prev_t = t; + t = t->next; + f->task_thread.task_cur_prev = prev_t; + } + ttd->cur++; + } + if (reset_task_cur(c, ttd, UINT_MAX)) continue; + park: + tc->task_thread.flushed = 1; + pthread_cond_signal(&tc->task_thread.td.cond); + // we want to be woken up next time progress is signaled + atomic_store(&ttd->cond_signaled, 0); + pthread_cond_wait(&ttd->cond, &ttd->lock); + tc->task_thread.flushed = 0; + reset_task_cur(c, ttd, UINT_MAX); + continue; + + found: + // remove t from list + if (prev_t) prev_t->next = t->next; + else f->task_thread.task_head = t->next; + if (!t->next) f->task_thread.task_tail = prev_t; + if (!f->task_thread.task_head) ttd->cur++; + // we don't need to check cond_signaled here, since we found a task + // after the last signal so we want to re-signal the next waiting thread + // and again won't need to signal after that + atomic_store(&ttd->cond_signaled, 1); + pthread_cond_signal(&ttd->cond); + pthread_mutex_unlock(&ttd->lock); + found_unlocked:; + const int flush = atomic_load(c->flush); + int error = atomic_fetch_or(&f->task_thread.error, flush) | flush; - dav1d_set_thread_name("dav1d-frame"); - pthread_mutex_lock(&f->frame_thread.td.lock); - for (;;) { - while (!f->n_tile_data && !f->frame_thread.die) { - pthread_cond_wait(&f->frame_thread.td.cond, - &f->frame_thread.td.lock); + // run it + tc->f = f; + int sby = t->sby; + switch (t->type) { + case DAV1D_TASK_TYPE_INIT: { + assert(c->n_fc > 1); + int res = -1; + if (!atomic_load(&f->task_thread.error)) + res = dav1d_decode_frame_init(f); + pthread_mutex_lock(&ttd->lock); + if (f->frame_hdr->refresh_context && !f->task_thread.update_set) { + atomic_store(f->out_cdf.progress, res < 0 ? TILE_ERROR : 1); + } + if (!res) { + assert(c->n_fc > 1); + for (int p = 1; p <= 2; p++) { + const int res = dav1d_task_create_tile_sbrow(f, p, 0); + if (res) { + // memory allocation failed + f->task_thread.done[2 - p] = 1; + atomic_store(&f->task_thread.error, 1); + f->task_thread.task_counter -= f->sbh + + f->frame_hdr->tiling.cols * f->frame_hdr->tiling.rows; + atomic_store(&f->sr_cur.progress[p - 1], FRAME_ERROR); + if (p == 2 && f->task_thread.done[1]) { + assert(!f->task_thread.task_counter); + dav1d_decode_frame_exit(f, -1); + f->n_tile_data = 0; + pthread_cond_signal(&f->task_thread.cond); + } + } + } + } else { + // init failed, signal completion + atomic_store(&f->task_thread.error, 1); + f->task_thread.task_counter = 0; + f->task_thread.done[0] = 1; + f->task_thread.done[1] = 1; + atomic_store(&f->sr_cur.progress[0], FRAME_ERROR); + atomic_store(&f->sr_cur.progress[1], FRAME_ERROR); + dav1d_decode_frame_exit(f, -1); + f->n_tile_data = 0; + pthread_cond_signal(&f->task_thread.cond); + } + reset_task_cur(c, ttd, t->frame_idx); + continue; } - if (f->frame_thread.die) break; - pthread_mutex_unlock(&f->frame_thread.td.lock); + case DAV1D_TASK_TYPE_TILE_ENTROPY: + case DAV1D_TASK_TYPE_TILE_RECONSTRUCTION: { + const int p = t->type == DAV1D_TASK_TYPE_TILE_ENTROPY; + const int tile_idx = (int)(t - f->task_thread.tile_tasks[p]); + Dav1dTileState *const ts = &f->ts[tile_idx]; + + tc->ts = ts; + tc->by = sby << f->sb_shift; + const int uses_2pass = c->n_fc > 1; + tc->frame_thread.pass = !uses_2pass ? 0 : + 1 + (t->type == DAV1D_TASK_TYPE_TILE_RECONSTRUCTION); + if (!error) error = dav1d_decode_tile_sbrow(tc); + const int progress = error ? TILE_ERROR : 1 + sby; - if (dav1d_decode_frame(f)) - memset(f->frame_thread.cf, 0, - (size_t)f->frame_thread.cf_sz * 128 * 128 / 2); - - pthread_mutex_lock(&f->frame_thread.td.lock); - f->n_tile_data = 0; - pthread_cond_signal(&f->frame_thread.td.cond); + // signal progress + atomic_fetch_or(&f->task_thread.error, error); + if (((sby + 1) << f->sb_shift) < ts->tiling.row_end) { + t->sby++; + t->deps_skip = 0; + if (!check_tile(t, f, uses_2pass)) { + atomic_store(&ts->progress[p], progress); + reset_task_cur_async(ttd, t->frame_idx, c->n_fc); + if (!atomic_fetch_or(&ttd->cond_signaled, 1)) + pthread_cond_signal(&ttd->cond); + goto found_unlocked; + } + pthread_mutex_lock(&ttd->lock); + atomic_store(&ts->progress[p], progress); + reset_task_cur(c, ttd, t->frame_idx); + insert_task(f, t, 0); + } else { + pthread_mutex_lock(&ttd->lock); + atomic_store(&ts->progress[p], progress); + reset_task_cur(c, ttd, t->frame_idx); + error = atomic_load(&f->task_thread.error); + if (f->frame_hdr->refresh_context && + tc->frame_thread.pass <= 1 && f->task_thread.update_set && + f->frame_hdr->tiling.update == tile_idx) + { + if (!error) + dav1d_cdf_thread_update(f->frame_hdr, f->out_cdf.data.cdf, + &f->ts[f->frame_hdr->tiling.update].cdf); + if (c->n_fc > 1) + atomic_store(f->out_cdf.progress, error ? TILE_ERROR : 1); + } + if (!--f->task_thread.task_counter && f->task_thread.done[0] && + (!uses_2pass || f->task_thread.done[1])) + { + dav1d_decode_frame_exit(f, error ? -1 : 0); + f->n_tile_data = 0; + pthread_cond_signal(&f->task_thread.cond); + } + assert(f->task_thread.task_counter >= 0); + if (!atomic_fetch_or(&ttd->cond_signaled, 1)) + pthread_cond_signal(&ttd->cond); + } + continue; + } + case DAV1D_TASK_TYPE_DEBLOCK_COLS: + if (!atomic_load(&f->task_thread.error)) + f->bd_fn.filter_sbrow_deblock_cols(f, sby); + if (ensure_progress(ttd, f, t, DAV1D_TASK_TYPE_DEBLOCK_ROWS, + &f->frame_thread.deblock_progress, + &t->deblock_progress)) continue; + // fall-through + case DAV1D_TASK_TYPE_DEBLOCK_ROWS: + if (!atomic_load(&f->task_thread.error)) + f->bd_fn.filter_sbrow_deblock_rows(f, sby); + // signal deblock progress + if (f->frame_hdr->loopfilter.level_y[0] || + f->frame_hdr->loopfilter.level_y[1]) + { + error = atomic_load(&f->task_thread.error); + atomic_store(&f->frame_thread.deblock_progress, + error ? TILE_ERROR : sby + 1); + reset_task_cur_async(ttd, t->frame_idx, c->n_fc); + if (!atomic_fetch_or(&ttd->cond_signaled, 1)) + pthread_cond_signal(&ttd->cond); + } + // fall-through + case DAV1D_TASK_TYPE_CDEF: + if (f->seq_hdr->cdef) { + // cdef caches top (pre-cdef) buffers internally and therefore + // needs to be vertically linear + if (ensure_progress(ttd, f, t, DAV1D_TASK_TYPE_CDEF, + &f->frame_thread.cdef_progress, + &t->cdef_progress)) continue; + if (!atomic_load(&f->task_thread.error)) + f->bd_fn.filter_sbrow_cdef(f, sby); + // signal cdef progress + error = atomic_load(&f->task_thread.error); + atomic_store(&f->frame_thread.cdef_progress, + error ? TILE_ERROR : sby + 1); + reset_task_cur_async(ttd, t->frame_idx, c->n_fc); + if (!atomic_fetch_or(&ttd->cond_signaled, 1)) + pthread_cond_signal(&ttd->cond); + } + // fall-through + case DAV1D_TASK_TYPE_SUPER_RESOLUTION: + if (f->frame_hdr->width[0] != f->frame_hdr->width[1]) + if (!atomic_load(&f->task_thread.error)) + f->bd_fn.filter_sbrow_resize(f, sby); + // fall-through + case DAV1D_TASK_TYPE_LOOP_RESTORATION: + // lr is the last step before signaling frame completion, and + // therefore needs to be done vertically linear + if (ensure_progress(ttd, f, t, DAV1D_TASK_TYPE_LOOP_RESTORATION, + &f->frame_thread.lr_progress, + &t->lr_progress)) continue; + if (!atomic_load(&f->task_thread.error) && f->lf.restore_planes) + f->bd_fn.filter_sbrow_lr(f, sby); + // fall-through + case DAV1D_TASK_TYPE_ENTROPY_PROGRESS: + // dummy to convert tile to frame + break; + default: abort(); + } + // if task completed [typically LR], signal picture progress as per below + const int uses_2pass = c->n_fc > 1; + const enum PlaneType progress_plane_type = + t->type == DAV1D_TASK_TYPE_ENTROPY_PROGRESS ? PLANE_TYPE_BLOCK : + c->n_fc > 1 ? PLANE_TYPE_Y : PLANE_TYPE_ALL; + const int sbh = f->sbh; + pthread_mutex_lock(&ttd->lock); + error = atomic_load(&f->task_thread.error); + const unsigned y = error ? FRAME_ERROR : + sby + 1 == sbh ? UINT_MAX : (unsigned)(sby + 1) * f->sb_step * 4; + if (c->n_fc > 1 && f->sr_cur.p.data[0] /* upon flush, this can be free'ed already */) { + if (!uses_2pass || t->type == DAV1D_TASK_TYPE_ENTROPY_PROGRESS) + atomic_store(&f->sr_cur.progress[0], y); + if (!uses_2pass || t->type != DAV1D_TASK_TYPE_ENTROPY_PROGRESS) + atomic_store(&f->sr_cur.progress[1], y); + } + const int progress = error ? TILE_ERROR : sby + 1; + if (progress_plane_type == PLANE_TYPE_BLOCK) + f->frame_thread.entropy_progress = progress; + else + atomic_store(&f->frame_thread.lr_progress, progress); + if (sby + 1 == sbh) + f->task_thread.done[progress_plane_type == PLANE_TYPE_BLOCK] = 1; + if (!--f->task_thread.task_counter && + f->task_thread.done[0] && (!uses_2pass || f->task_thread.done[1])) + { + dav1d_decode_frame_exit(f, error ? -1 : 0); + f->n_tile_data = 0; + pthread_cond_signal(&f->task_thread.cond); + } + reset_task_cur(c, ttd, t->frame_idx); } - pthread_mutex_unlock(&f->frame_thread.td.lock); + pthread_mutex_unlock(&ttd->lock); return NULL; } - -void *dav1d_tile_task(void *const data) { - Dav1dTileContext *const t = data; - struct FrameTileThreadData *const fttd = t->tile_thread.fttd; - const Dav1dFrameContext *const f = t->f; - const int tile_thread_idx = (int) (t - f->tc); - const uint64_t mask = 1ULL << tile_thread_idx; - - dav1d_set_thread_name("dav1d-tile"); - - for (;;) { - pthread_mutex_lock(&fttd->lock); - fttd->available |= mask; - int did_signal = 0; - while (!fttd->tasks_left && !t->tile_thread.die) { - if (!did_signal) { - did_signal = 1; - pthread_cond_signal(&fttd->icond); - } - pthread_cond_wait(&fttd->cond, &fttd->lock); - } - if (t->tile_thread.die) { - pthread_cond_signal(&fttd->icond); - pthread_mutex_unlock(&fttd->lock); - break; - } - fttd->available &= ~mask; - const int task_idx = fttd->num_tasks - fttd->tasks_left--; - pthread_mutex_unlock(&fttd->lock); - - if (f->frame_thread.pass == 1 || f->n_tc >= f->frame_hdr->tiling.cols) { - // we can (or in fact, if >, we need to) do full tile decoding. - // loopfilter happens in the main thread - Dav1dTileState *const ts = t->ts = &f->ts[task_idx]; - for (t->by = ts->tiling.row_start; t->by < ts->tiling.row_end; - t->by += f->sb_step) - { - const int error = dav1d_decode_tile_sbrow(t); - const int progress = error ? TILE_ERROR : 1 + (t->by >> f->sb_shift); - - // signal progress - pthread_mutex_lock(&ts->tile_thread.lock); - atomic_store(&ts->progress, progress); - pthread_cond_signal(&ts->tile_thread.cond); - pthread_mutex_unlock(&ts->tile_thread.lock); - if (error) break; - } - } else { - const int sby = f->tile_thread.task_idx_to_sby_and_tile_idx[task_idx][0]; - const int tile_idx = f->tile_thread.task_idx_to_sby_and_tile_idx[task_idx][1]; - Dav1dTileState *const ts = &f->ts[tile_idx]; - int progress; - - // the interleaved decoding can sometimes cause dependency issues - // if one part of the frame decodes signifcantly faster than others. - // Ideally, we'd "skip" tile_sbrows where dependencies are missing, - // and resume them later as dependencies are met. This also would - // solve the broadcast() below and allow us to use signal(). However, - // for now, we use linear dependency tracking because it's simpler. - if ((progress = atomic_load(&ts->progress)) < sby) { - pthread_mutex_lock(&ts->tile_thread.lock); - while ((progress = atomic_load(&ts->progress)) < sby) - pthread_cond_wait(&ts->tile_thread.cond, - &ts->tile_thread.lock); - pthread_mutex_unlock(&ts->tile_thread.lock); - } - if (progress == TILE_ERROR) continue; - - // we need to interleave sbrow decoding for all tile cols in a - // tile row, since otherwise subsequent threads will be blocked - // waiting for the post-filter to complete - t->ts = ts; - t->by = sby << f->sb_shift; - const int error = dav1d_decode_tile_sbrow(t); - progress = error ? TILE_ERROR : 1 + sby; - - // signal progress - pthread_mutex_lock(&ts->tile_thread.lock); - atomic_store(&ts->progress, progress); - pthread_cond_broadcast(&ts->tile_thread.cond); - pthread_mutex_unlock(&ts->tile_thread.lock); - } - } - - return NULL; -} - -static inline int handle_abortion(Dav1dPostFilterContext *const pf, - Dav1dContext *const c, - struct PostFilterThreadData *const pftd) -{ - const int flush = atomic_load_explicit(c->flush, memory_order_acquire); - if (flush) { - pthread_mutex_lock(&pf->td.lock); - pf->flushed = 0; - pthread_mutex_unlock(&pf->td.lock); - } - for (unsigned i = 0; i < c->n_fc; i++) { - Dav1dFrameContext *const f = &c->fc[i]; - int send_signal; - if (flush) // TODO before merge, see if this can be safely merged - send_signal = f->lf.thread.done != 1 && f->lf.thread.num_tasks != 0; - else - send_signal = f->lf.thread.done == -1; - for (int j = 0; send_signal && j < f->lf.thread.num_tasks; j++) { - Dav1dTask *const t = &f->lf.thread.tasks[j]; - if (t->status == DAV1D_TASK_RUNNING || - (t->status == DAV1D_TASK_DONE && t->start != -1)) - send_signal = 0; - } - if (send_signal) { - if (!flush) { - Dav1dTask **pt = &pftd->tasks; - while (*pt) { - if ((*pt)->frame_idx == i) - *pt = (*pt)->next_exec; - else - pt = &(*pt)->next_exec; - } - } - f->lf.thread.done = 1; - pthread_cond_signal(&f->lf.thread.cond); - } - } - if (flush) { - pthread_mutex_lock(&pf->td.lock); - pf->flushed = 1; - pthread_cond_signal(&pf->td.cond); - pthread_mutex_unlock(&pf->td.lock); - } - return !flush; -} - -void *dav1d_postfilter_task(void *data) { - Dav1dPostFilterContext *const pf = data; - Dav1dContext *const c = pf->c; - struct PostFilterThreadData *pftd = &c->postfilter_thread; - - dav1d_set_thread_name("dav1d-postfilter"); - - int exec = 1; - pthread_mutex_lock(&pftd->lock); - for (;;) { - if (!exec && !pf->die) - pthread_cond_wait(&pftd->cond, &pftd->lock); - if (!(exec = handle_abortion(pf, c, pftd))) continue; - if (pf->die) break; - - Dav1dTask *const t = pftd->tasks; - if (!t) { exec = 0; continue; } - pftd->tasks = t->next_exec; - t->status = DAV1D_TASK_RUNNING; - - pthread_mutex_unlock(&pftd->lock); - Dav1dFrameContext *const f = &c->fc[t->frame_idx]; - t->fn(f, t->sby); - exec = 1; - pthread_mutex_lock(&pftd->lock); - - if (t->next_deps[0]) - update_task(t->next_deps[0], 0, f); - if (t->next_deps[1]) - update_task(t->next_deps[1], 1, f); - t->status = DAV1D_TASK_DONE; - if (!t->next_deps[0]) { - const enum PlaneType progress_plane_type = - c->n_fc > 1 && f->frame_hdr->refresh_context ? - PLANE_TYPE_Y : PLANE_TYPE_ALL; - const int y = (t->sby + 1) * f->sb_step * 4; - dav1d_thread_picture_signal(&f->sr_cur, y, progress_plane_type); - if (t->sby + 1 == f->sbh) { - f->lf.thread.done = 1; - pthread_cond_signal(&f->lf.thread.cond); - } - } - t->start = -1; - } - pthread_mutex_unlock(&pftd->lock); - - return NULL; -} diff --git a/third_party/dav1d/src/x86/cdef16_avx2.asm b/third_party/dav1d/src/x86/cdef16_avx2.asm --- a/third_party/dav1d/src/x86/cdef16_avx2.asm +++ b/third_party/dav1d/src/x86/cdef16_avx2.asm @@ -28,31 +28,34 @@ %if ARCH_X86_64 SECTION_RODATA -tap_table: dw 4, 2, 3, 3, 2, 1 - db -1 * 16 + 1, -2 * 16 + 2 - db 0 * 16 + 1, -1 * 16 + 2 - db 0 * 16 + 1, 0 * 16 + 2 - db 0 * 16 + 1, 1 * 16 + 2 - db 1 * 16 + 1, 2 * 16 + 2 - db 1 * 16 + 0, 2 * 16 + 1 - db 1 * 16 + 0, 2 * 16 + 0 - db 1 * 16 + 0, 2 * 16 - 1 - ; the last 6 are repeats of the first 6 so we don't need to & 7 - db -1 * 16 + 1, -2 * 16 + 2 - db 0 * 16 + 1, -1 * 16 + 2 - db 0 * 16 + 1, 0 * 16 + 2 - db 0 * 16 + 1, 1 * 16 + 2 - db 1 * 16 + 1, 2 * 16 + 2 - db 1 * 16 + 0, 2 * 16 + 1 +%macro DIR_TABLE 1 ; stride + db 1 * %1 + 0, 2 * %1 + 0 + db 1 * %1 + 0, 2 * %1 - 2 + db -1 * %1 + 2, -2 * %1 + 4 + db 0 * %1 + 2, -1 * %1 + 4 + db 0 * %1 + 2, 0 * %1 + 4 + db 0 * %1 + 2, 1 * %1 + 4 + db 1 * %1 + 2, 2 * %1 + 4 + db 1 * %1 + 0, 2 * %1 + 2 + db 1 * %1 + 0, 2 * %1 + 0 + db 1 * %1 + 0, 2 * %1 - 2 + db -1 * %1 + 2, -2 * %1 + 4 + db 0 * %1 + 2, -1 * %1 + 4 +%endmacro -dir_shift: times 2 dw 0x4000 - times 2 dw 0x1000 +dir_table4: DIR_TABLE 16 +dir_table8: DIR_TABLE 32 +pri_taps: dw 4, 4, 3, 3, 2, 2, 3, 3 -pw_2048: times 2 dw 2048 +dir_shift: times 2 dw 0x4000 + times 2 dw 0x1000 + +pw_2048: times 2 dw 2048 +pw_m16384: times 2 dw -16384 cextern cdef_dir_8bpc_avx2.main SECTION .text @@ -62,411 +65,808 @@ %rotate 1 %%f(%1) %endrep %endmacro -%macro ACCUMULATE_TAP 6 ; tap_offset, shift, strength, mul_tap, w, stride - ; load p0/p1 - movsx offq, byte [dirq+kq+%1] ; off1 -%if %5 == 4 - movq xm5, [stkq+offq*2+%6*0] ; p0 - movq xm6, [stkq+offq*2+%6*2] - movhps xm5, [stkq+offq*2+%6*1] - movhps xm6, [stkq+offq*2+%6*3] - vinserti128 m5, xm6, 1 +%macro CDEF_FILTER 2 ; w, h + DEFINE_ARGS dst, stride, dir, pridmp, pri, sec, tmp + movifnidn prid, r4m + movifnidn secd, r5m + mov dird, r6m + vpbroadcastd m8, [base+pw_2048] + lea dirq, [base+dir_table%1+dirq*2] + test prid, prid + jz .sec_only +%if WIN64 + vpbroadcastw m6, prim + movaps [rsp+16*0], xmm9 + movaps [rsp+16*1], xmm10 +%else + movd xm6, prid + vpbroadcastw m6, xm6 +%endif + lzcnt pridmpd, prid + rorx tmpd, prid, 2 + cmp dword r9m, 0xfff ; if (bpc == 12) + cmove prid, tmpd ; pri >>= 2 + mov tmpd, r7m ; damping + and prid, 4 + sub tmpd, 31 + vpbroadcastd m9, [base+pri_taps+priq+8*0] + vpbroadcastd m10, [base+pri_taps+priq+8*1] + test secd, secd + jz .pri_only +%if WIN64 + movaps r8m, xmm13 + vpbroadcastw m13, secm + movaps r4m, xmm11 + movaps r6m, xmm12 +%else + movd xm0, secd + vpbroadcastw m13, xm0 +%endif + lzcnt secd, secd + xor prid, prid + add pridmpd, tmpd + cmovs pridmpd, prid + add secd, tmpd + lea tmpq, [px] + mov [pri_shift], pridmpq + mov [sec_shift], secq +%rep %1*%2/16 + call mangle(private_prefix %+ _cdef_filter_%1x%1_16bpc %+ SUFFIX).pri_sec +%endrep +%if WIN64 + movaps xmm11, r4m + movaps xmm12, r6m + movaps xmm13, r8m +%endif + jmp .pri_end +.pri_only: + add pridmpd, tmpd + cmovs pridmpd, secd + lea tmpq, [px] + mov [pri_shift], pridmpq +%rep %1*%2/16 + call mangle(private_prefix %+ _cdef_filter_%1x%1_16bpc %+ SUFFIX).pri +%endrep +.pri_end: +%if WIN64 + movaps xmm9, [rsp+16*0] + movaps xmm10, [rsp+16*1] +%endif +.end: + RET +.sec_only: + mov tmpd, r7m ; damping +%if WIN64 + vpbroadcastw m6, secm +%else + movd xm6, secd + vpbroadcastw m6, xm6 +%endif + tzcnt secd, secd + sub tmpd, secd + mov [sec_shift], tmpq + lea tmpq, [px] +%rep %1*%2/16 + call mangle(private_prefix %+ _cdef_filter_%1x%1_16bpc %+ SUFFIX).sec +%endrep + jmp .end +%if %1 == %2 +ALIGN function_align +.pri: + movsx offq, byte [dirq+4] ; off_k0 +%if %1 == 4 + mova m1, [tmpq+32*0] + punpcklqdq m1, [tmpq+32*1] ; 0 2 1 3 + movu m2, [tmpq+offq+32*0] + punpcklqdq m2, [tmpq+offq+32*1] ; k0p0 + neg offq + movu m3, [tmpq+offq+32*0] + punpcklqdq m3, [tmpq+offq+32*1] ; k0p1 %else - movu xm5, [stkq+offq*2+%6*0] ; p0 - vinserti128 m5, [stkq+offq*2+%6*1], 1 + mova xm1, [tmpq+32*0] + vinserti128 m1, [tmpq+32*1], 1 + movu xm2, [tmpq+offq+32*0] + vinserti128 m2, [tmpq+offq+32*1], 1 + neg offq + movu xm3, [tmpq+offq+32*0] + vinserti128 m3, [tmpq+offq+32*1], 1 +%endif + movsx offq, byte [dirq+5] ; off_k1 + psubw m2, m1 ; diff_k0p0 + psubw m3, m1 ; diff_k0p1 + pabsw m4, m2 ; adiff_k0p0 + psrlw m5, m4, [pri_shift+gprsize] + psubusw m0, m6, m5 + pabsw m5, m3 ; adiff_k0p1 + pminsw m0, m4 + psrlw m4, m5, [pri_shift+gprsize] + psignw m0, m2 ; constrain(diff_k0p0) + psubusw m2, m6, m4 + pminsw m2, m5 +%if %1 == 4 + movu m4, [tmpq+offq+32*0] + punpcklqdq m4, [tmpq+offq+32*1] ; k1p0 + neg offq + movu m5, [tmpq+offq+32*0] + punpcklqdq m5, [tmpq+offq+32*1] ; k1p1 +%else + movu xm4, [tmpq+offq+32*0] + vinserti128 m4, [tmpq+offq+32*1], 1 + neg offq + movu xm5, [tmpq+offq+32*0] + vinserti128 m5, [tmpq+offq+32*1], 1 +%endif + psubw m4, m1 ; diff_k1p0 + psubw m5, m1 ; diff_k1p1 + psignw m2, m3 ; constrain(diff_k0p1) + pabsw m3, m4 ; adiff_k1p0 + paddw m0, m2 ; constrain(diff_k0) + psrlw m2, m3, [pri_shift+gprsize] + psubusw m7, m6, m2 + pabsw m2, m5 ; adiff_k1p1 + pminsw m7, m3 + psrlw m3, m2, [pri_shift+gprsize] + psignw m7, m4 ; constrain(diff_k1p0) + psubusw m4, m6, m3 + pminsw m4, m2 + psignw m4, m5 ; constrain(diff_k1p1) + paddw m7, m4 ; constrain(diff_k1) + pmullw m0, m9 ; pri_tap_k0 + pmullw m7, m10 ; pri_tap_k1 + paddw m0, m7 ; sum + psraw m2, m0, 15 + paddw m0, m2 + pmulhrsw m0, m8 + add tmpq, 32*2 + paddw m0, m1 +%if %1 == 4 + vextracti128 xm1, m0, 1 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+r8 ], xm1 + lea dstq, [dstq+strideq*4] +%else + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] %endif - neg offq ; -off1 -%if %5 == 4 - movq xm6, [stkq+offq*2+%6*0] ; p1 - movq xm9, [stkq+offq*2+%6*2] - movhps xm6, [stkq+offq*2+%6*1] - movhps xm9, [stkq+offq*2+%6*3] - vinserti128 m6, xm9, 1 + ret +ALIGN function_align +.sec: + movsx offq, byte [dirq+8] ; off1_k0 +%if %1 == 4 + mova m1, [tmpq+32*0] + punpcklqdq m1, [tmpq+32*1] + movu m2, [tmpq+offq+32*0] + punpcklqdq m2, [tmpq+offq+32*1] ; k0s0 + neg offq + movu m3, [tmpq+offq+32*0] + punpcklqdq m3, [tmpq+offq+32*1] ; k0s1 %else - movu xm6, [stkq+offq*2+%6*0] ; p1 - vinserti128 m6, [stkq+offq*2+%6*1], 1 + mova xm1, [tmpq+32*0] + vinserti128 m1, [tmpq+32*1], 1 + movu xm2, [tmpq+offq+32*0] + vinserti128 m2, [tmpq+offq+32*1], 1 + neg offq + movu xm3, [tmpq+offq+32*0] + vinserti128 m3, [tmpq+offq+32*1], 1 +%endif + movsx offq, byte [dirq+0] ; off2_k0 + psubw m2, m1 ; diff_k0s0 + psubw m3, m1 ; diff_k0s1 + pabsw m4, m2 ; adiff_k0s0 + psrlw m5, m4, [sec_shift+gprsize] + psubusw m0, m6, m5 + pabsw m5, m3 ; adiff_k0s1 + pminsw m0, m4 + psrlw m4, m5, [sec_shift+gprsize] + psignw m0, m2 ; constrain(diff_k0s0) + psubusw m2, m6, m4 + pminsw m2, m5 +%if %1 == 4 + movu m4, [tmpq+offq+32*0] + punpcklqdq m4, [tmpq+offq+32*1] ; k0s2 + neg offq + movu m5, [tmpq+offq+32*0] + punpcklqdq m5, [tmpq+offq+32*1] ; k0s3 +%else + movu xm4, [tmpq+offq+32*0] + vinserti128 m4, [tmpq+offq+32*1], 1 + neg offq + movu xm5, [tmpq+offq+32*0] + vinserti128 m5, [tmpq+offq+32*1], 1 +%endif + movsx offq, byte [dirq+9] ; off1_k1 + psubw m4, m1 ; diff_k0s2 + psubw m5, m1 ; diff_k0s3 + psignw m2, m3 ; constrain(diff_k0s1) + pabsw m3, m4 ; adiff_k0s2 + paddw m0, m2 + psrlw m2, m3, [sec_shift+gprsize] + psubusw m7, m6, m2 + pabsw m2, m5 ; adiff_k0s3 + pminsw m7, m3 + psrlw m3, m2, [sec_shift+gprsize] + psignw m7, m4 ; constrain(diff_k0s2) + psubusw m4, m6, m3 + pminsw m4, m2 +%if %1 == 4 + movu m2, [tmpq+offq+32*0] + punpcklqdq m2, [tmpq+offq+32*1] ; k1s0 + neg offq + movu m3, [tmpq+offq+32*0] + punpcklqdq m3, [tmpq+offq+32*1] ; k1s1 +%else + movu xm2, [tmpq+offq+32*0] + vinserti128 m2, [tmpq+offq+32*1], 1 + neg offq + movu xm3, [tmpq+offq+32*0] + vinserti128 m3, [tmpq+offq+32*1], 1 %endif - ; out of bounds values are set to a value that is a both a large unsigned - ; value and a negative signed value. - ; use signed max and unsigned min to remove them - pmaxsw m7, m5 ; max after p0 - pminuw m8, m5 ; min after p0 - pmaxsw m7, m6 ; max after p1 - pminuw m8, m6 ; min after p1 - - ; accumulate sum[m15] over p0/p1 - psubw m5, m4 ; diff_p0(p0 - px) - psubw m6, m4 ; diff_p1(p1 - px) - pabsw m9, m5 - pabsw m10, m6 - psignw m11, %4, m5 - psignw m12, %4, m6 - psrlw m5, m9, %2 - psrlw m6, m10, %2 - psubusw m5, %3, m5 - psubusw m6, %3, m6 - pminuw m5, m9 ; constrain(diff_p0) - pminuw m6, m10 ; constrain(diff_p1) - pmullw m5, m11 ; constrain(diff_p0) * taps - pmullw m6, m12 ; constrain(diff_p1) * taps - paddw m15, m5 - paddw m15, m6 + movsx offq, byte [dirq+1] ; off2_k1 + paddw m0, m7 + psignw m4, m5 ; constrain(diff_k0s3) + paddw m0, m4 ; constrain(diff_k0) + psubw m2, m1 ; diff_k1s0 + psubw m3, m1 ; diff_k1s1 + paddw m0, m0 ; sec_tap_k0 + pabsw m4, m2 ; adiff_k1s0 + psrlw m5, m4, [sec_shift+gprsize] + psubusw m7, m6, m5 + pabsw m5, m3 ; adiff_k1s1 + pminsw m7, m4 + psrlw m4, m5, [sec_shift+gprsize] + psignw m7, m2 ; constrain(diff_k1s0) + psubusw m2, m6, m4 + pminsw m2, m5 +%if %1 == 4 + movu m4, [tmpq+offq+32*0] + punpcklqdq m4, [tmpq+offq+32*1] ; k1s2 + neg offq + movu m5, [tmpq+offq+32*0] + punpcklqdq m5, [tmpq+offq+32*1] ; k1s3 +%else + movu xm4, [tmpq+offq+32*0] + vinserti128 m4, [tmpq+offq+32*1], 1 + neg offq + movu xm5, [tmpq+offq+32*0] + vinserti128 m5, [tmpq+offq+32*1], 1 +%endif + paddw m0, m7 + psubw m4, m1 ; diff_k1s2 + psubw m5, m1 ; diff_k1s3 + psignw m2, m3 ; constrain(diff_k1s1) + pabsw m3, m4 ; adiff_k1s2 + paddw m0, m2 + psrlw m2, m3, [sec_shift+gprsize] + psubusw m7, m6, m2 + pabsw m2, m5 ; adiff_k1s3 + pminsw m7, m3 + psrlw m3, m2, [sec_shift+gprsize] + psignw m7, m4 ; constrain(diff_k1s2) + psubusw m4, m6, m3 + pminsw m4, m2 + paddw m0, m7 + psignw m4, m5 ; constrain(diff_k1s3) + paddw m0, m4 ; sum + psraw m2, m0, 15 + paddw m0, m2 + pmulhrsw m0, m8 + add tmpq, 32*2 + paddw m0, m1 +%if %1 == 4 + vextracti128 xm1, m0, 1 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+r8 ], xm1 + lea dstq, [dstq+strideq*4] +%else + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] +%endif + ret +ALIGN function_align +.pri_sec: + movsx offq, byte [dirq+8] ; off2_k0 +%if %1 == 4 + mova m1, [tmpq+32*0] + punpcklqdq m1, [tmpq+32*1] + movu m2, [tmpq+offq+32*0] + punpcklqdq m2, [tmpq+offq+32*1] ; k0s0 + neg offq + movu m3, [tmpq+offq+32*0] + punpcklqdq m3, [tmpq+offq+32*1] ; k0s1 +%else + mova xm1, [dstq+strideq*0] + vinserti128 m1, [dstq+strideq*1], 1 + movu xm2, [tmpq+offq+32*0] + vinserti128 m2, [tmpq+offq+32*1], 1 + neg offq + movu xm3, [tmpq+offq+32*0] + vinserti128 m3, [tmpq+offq+32*1], 1 +%endif + movsx offq, byte [dirq+0] ; off3_k0 + pmaxsw m11, m2, m3 + pminuw m12, m2, m3 + psubw m2, m1 ; diff_k0s0 + psubw m3, m1 ; diff_k0s1 + pabsw m4, m2 ; adiff_k0s0 + psrlw m5, m4, [sec_shift+gprsize] + psubusw m0, m13, m5 + pabsw m5, m3 ; adiff_k0s1 + pminsw m0, m4 + psrlw m4, m5, [sec_shift+gprsize] + psignw m0, m2 ; constrain(diff_k0s0) + psubusw m2, m13, m4 + pminsw m2, m5 +%if %1 == 4 + movu m4, [tmpq+offq+32*0] + punpcklqdq m4, [tmpq+offq+32*1] ; k0s2 + neg offq + movu m5, [tmpq+offq+32*0] + punpcklqdq m5, [tmpq+offq+32*1] ; k0s3 +%else + movu xm4, [tmpq+offq+32*0] + vinserti128 m4, [tmpq+offq+32*1], 1 + neg offq + movu xm5, [tmpq+offq+32*0] + vinserti128 m5, [tmpq+offq+32*1], 1 +%endif + movsx offq, byte [dirq+9] ; off2_k1 + psignw m2, m3 ; constrain(diff_k0s1) + pmaxsw m11, m4 + pminuw m12, m4 + pmaxsw m11, m5 + pminuw m12, m5 + psubw m4, m1 ; diff_k0s2 + psubw m5, m1 ; diff_k0s3 + paddw m0, m2 + pabsw m3, m4 ; adiff_k0s2 + psrlw m2, m3, [sec_shift+gprsize] + psubusw m7, m13, m2 + pabsw m2, m5 ; adiff_k0s3 + pminsw m7, m3 + psrlw m3, m2, [sec_shift+gprsize] + psignw m7, m4 ; constrain(diff_k0s2) + psubusw m4, m13, m3 + pminsw m4, m2 +%if %1 == 4 + movu m2, [tmpq+offq+32*0] + punpcklqdq m2, [tmpq+offq+32*1] ; k1s0 + neg offq + movu m3, [tmpq+offq+32*0] + punpcklqdq m3, [tmpq+offq+32*1] ; k1s1 +%else + movu xm2, [tmpq+offq+32*0] + vinserti128 m2, [tmpq+offq+32*1], 1 + neg offq + movu xm3, [tmpq+offq+32*0] + vinserti128 m3, [tmpq+offq+32*1], 1 +%endif + movsx offq, byte [dirq+1] ; off3_k1 + paddw m0, m7 + psignw m4, m5 ; constrain(diff_k0s3) + pmaxsw m11, m2 + pminuw m12, m2 + pmaxsw m11, m3 + pminuw m12, m3 + paddw m0, m4 ; constrain(diff_k0) + psubw m2, m1 ; diff_k1s0 + psubw m3, m1 ; diff_k1s1 + paddw m0, m0 ; sec_tap_k0 + pabsw m4, m2 ; adiff_k1s0 + psrlw m5, m4, [sec_shift+gprsize] + psubusw m7, m13, m5 + pabsw m5, m3 ; adiff_k1s1 + pminsw m7, m4 + psrlw m4, m5, [sec_shift+gprsize] + psignw m7, m2 ; constrain(diff_k1s0) + psubusw m2, m13, m4 + pminsw m2, m5 +%if %1 == 4 + movu m4, [tmpq+offq+32*0] + punpcklqdq m4, [tmpq+offq+32*1] ; k1s2 + neg offq + movu m5, [tmpq+offq+32*0] + punpcklqdq m5, [tmpq+offq+32*1] ; k1s3 +%else + movu xm4, [tmpq+offq+32*0] + vinserti128 m4, [tmpq+offq+32*1], 1 + neg offq + movu xm5, [tmpq+offq+32*0] + vinserti128 m5, [tmpq+offq+32*1], 1 +%endif + movsx offq, byte [dirq+4] ; off1_k0 + paddw m0, m7 + psignw m2, m3 ; constrain(diff_k1s1) + pmaxsw m11, m4 + pminuw m12, m4 + pmaxsw m11, m5 + pminuw m12, m5 + psubw m4, m1 ; diff_k1s2 + psubw m5, m1 ; diff_k1s3 + pabsw m3, m4 ; adiff_k1s2 + paddw m0, m2 + psrlw m2, m3, [sec_shift+gprsize] + psubusw m7, m13, m2 + pabsw m2, m5 ; adiff_k1s3 + pminsw m7, m3 + psrlw m3, m2, [sec_shift+gprsize] + psignw m7, m4 ; constrain(diff_k1s2) + psubusw m4, m13, m3 + pminsw m4, m2 + paddw m0, m7 +%if %1 == 4 + movu m2, [tmpq+offq+32*0] + punpcklqdq m2, [tmpq+offq+32*1] ; k0p0 + neg offq + movu m3, [tmpq+offq+32*0] + punpcklqdq m3, [tmpq+offq+32*1] ; k0p1 +%else + movu xm2, [tmpq+offq+32*0] + vinserti128 m2, [tmpq+offq+32*1], 1 + neg offq + movu xm3, [tmpq+offq+32*0] + vinserti128 m3, [tmpq+offq+32*1], 1 +%endif + movsx offq, byte [dirq+5] ; off1_k1 + psignw m4, m5 ; constrain(diff_k1s3) + pmaxsw m11, m2 + pminuw m12, m2 + pmaxsw m11, m3 + pminuw m12, m3 + psubw m2, m1 ; diff_k0p0 + psubw m3, m1 ; diff_k0p1 + paddw m0, m4 + pabsw m4, m2 ; adiff_k0p0 + psrlw m5, m4, [pri_shift+gprsize] + psubusw m7, m6, m5 + pabsw m5, m3 ; adiff_k0p1 + pminsw m7, m4 + psrlw m4, m5, [pri_shift+gprsize] + psignw m7, m2 ; constrain(diff_k0p0) + psubusw m2, m6, m4 + pminsw m2, m5 +%if %1 == 4 + movu m4, [tmpq+offq+32*0] + punpcklqdq m4, [tmpq+offq+32*1] ; k1p0 + neg offq + movu m5, [tmpq+offq+32*0] + punpcklqdq m5, [tmpq+offq+32*1] ; k1p1 +%else + movu xm4, [tmpq+offq+32*0] + vinserti128 m4, [tmpq+offq+32*1], 1 + neg offq + movu xm5, [tmpq+offq+32*0] + vinserti128 m5, [tmpq+offq+32*1], 1 +%endif + psignw m2, m3 ; constrain(diff_k0p1) + paddw m7, m2 ; constrain(diff_k0) + pmaxsw m11, m4 + pminuw m12, m4 + pmaxsw m11, m5 + pminuw m12, m5 + psubw m4, m1 ; diff_k1p0 + psubw m5, m1 ; diff_k1p1 + pabsw m3, m4 ; adiff_k1p0 + pmullw m7, m9 ; pri_tap_k0 + paddw m0, m7 + psrlw m2, m3, [pri_shift+gprsize] + psubusw m7, m6, m2 + pabsw m2, m5 ; adiff_k1p1 + pminsw m7, m3 + psrlw m3, m2, [pri_shift+gprsize] + psignw m7, m4 ; constrain(diff_k1p0) + psubusw m4, m6, m3 + pminsw m4, m2 + psignw m4, m5 ; constrain(diff_k1p1) + paddw m7, m4 ; constrain(diff_k1) + pmullw m7, m10 ; pri_tap_k1 + paddw m0, m7 ; sum + psraw m2, m0, 15 + paddw m0, m2 + pmulhrsw m0, m8 + add tmpq, 32*2 + pmaxsw m11, m1 + pminuw m12, m1 + paddw m0, m1 + pminsw m0, m11 + pmaxsw m0, m12 +%if %1 == 4 + vextracti128 xm1, m0, 1 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+r8 ], xm1 + lea dstq, [dstq+strideq*4] +%else + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] +%endif + ret +%endif %endmacro -%macro cdef_filter_fn 3 ; w, h, stride INIT_YMM avx2 -%if %1 != 4 || %2 != 8 -cglobal cdef_filter_%1x%2_16bpc, 4, 9, 16, 2 * 16 + (%2+4)*%3, \ - dst, stride, left, top, pri, sec, \ - stride3, dst4, edge +cglobal cdef_filter_4x4_16bpc, 4, 9, 9, 16*10, dst, stride, left, top, pri, sec, edge +%if WIN64 + %define px rsp+16*6 + %define offq r7 + %define pri_shift rsp+16*2 + %define sec_shift rsp+16*3 %else -cglobal cdef_filter_%1x%2_16bpc, 4, 10, 16, 2 * 16 + (%2+4)*%3, \ - dst, stride, left, top, pri, sec, \ - stride3, dst4, edge -%endif -%define px rsp+2*16+2*%3 - pcmpeqw m14, m14 - psllw m14, 15 ; 0x8000 - mov edged, r8m - - ; prepare pixel buffers - body/right -%if %1 == 4 - INIT_XMM avx2 -%endif -%if %2 == 8 - lea dst4q, [dstq+strideq*4] -%endif - lea stride3q, [strideq*3] - test edgeb, 2 ; have_right - jz .no_right - movu m1, [dstq+strideq*0] - movu m2, [dstq+strideq*1] - movu m3, [dstq+strideq*2] - movu m4, [dstq+stride3q] - mova [px+0*%3], m1 - mova [px+1*%3], m2 - mova [px+2*%3], m3 - mova [px+3*%3], m4 -%if %2 == 8 - movu m1, [dst4q+strideq*0] - movu m2, [dst4q+strideq*1] - movu m3, [dst4q+strideq*2] - movu m4, [dst4q+stride3q] - mova [px+4*%3], m1 - mova [px+5*%3], m2 - mova [px+6*%3], m3 - mova [px+7*%3], m4 -%endif - jmp .body_done -.no_right: -%if %1 == 4 - movq xm1, [dstq+strideq*0] - movq xm2, [dstq+strideq*1] - movq xm3, [dstq+strideq*2] - movq xm4, [dstq+stride3q] - movq [px+0*%3], xm1 - movq [px+1*%3], xm2 - movq [px+2*%3], xm3 - movq [px+3*%3], xm4 -%else - mova xm1, [dstq+strideq*0] - mova xm2, [dstq+strideq*1] - mova xm3, [dstq+strideq*2] - mova xm4, [dstq+stride3q] - mova [px+0*%3], xm1 - mova [px+1*%3], xm2 - mova [px+2*%3], xm3 - mova [px+3*%3], xm4 + %define px rsp+16*4 + %define offq r3 + %define pri_shift rsp+16*0 + %define sec_shift rsp+16*1 %endif - movd [px+0*%3+%1*2], xm14 - movd [px+1*%3+%1*2], xm14 - movd [px+2*%3+%1*2], xm14 - movd [px+3*%3+%1*2], xm14 -%if %2 == 8 - %if %1 == 4 - movq xm1, [dst4q+strideq*0] - movq xm2, [dst4q+strideq*1] - movq xm3, [dst4q+strideq*2] - movq xm4, [dst4q+stride3q] - movq [px+4*%3], xm1 - movq [px+5*%3], xm2 - movq [px+6*%3], xm3 - movq [px+7*%3], xm4 - %else - mova xm1, [dst4q+strideq*0] - mova xm2, [dst4q+strideq*1] - mova xm3, [dst4q+strideq*2] - mova xm4, [dst4q+stride3q] - mova [px+4*%3], xm1 - mova [px+5*%3], xm2 - mova [px+6*%3], xm3 - mova [px+7*%3], xm4 - %endif - movd [px+4*%3+%1*2], xm14 - movd [px+5*%3+%1*2], xm14 - movd [px+6*%3+%1*2], xm14 - movd [px+7*%3+%1*2], xm14 -%endif -.body_done: - - ; top - test edgeb, 4 ; have_top + %define base r7-dir_table4 + mov edged, r8m + lea r7, [dir_table4] + movu xm0, [dstq+strideq*0] + movu xm1, [dstq+strideq*1] + lea r8, [strideq*3] + movu xm2, [dstq+strideq*2] + movu xm3, [dstq+r8 ] + vpbroadcastd m7, [base+pw_m16384] + mova [px+16*0+0], xm0 + mova [px+16*1+0], xm1 + mova [px+16*2+0], xm2 + mova [px+16*3+0], xm3 + test edgeb, 4 ; HAVE_TOP jz .no_top - test edgeb, 1 ; have_left + movu xm0, [topq+strideq*0] + movu xm1, [topq+strideq*1] + mova [px-16*2+0], xm0 + mova [px-16*1+0], xm1 + test edgeb, 1 ; HAVE_LEFT jz .top_no_left - test edgeb, 2 ; have_right - jz .top_no_right - movu m1, [topq+strideq*0-%1] - movu m2, [topq+strideq*1-%1] - movu [px-2*%3-%1], m1 - movu [px-1*%3-%1], m2 - jmp .top_done -.top_no_right: - movu m1, [topq+strideq*0-%1*2] - movu m2, [topq+strideq*1-%1*2] - movu [px-2*%3-%1*2], m1 - movu [px-1*%3-%1*2], m2 - movd [px-2*%3+%1*2], xm14 - movd [px-1*%3+%1*2], xm14 - jmp .top_done -.top_no_left: - test edgeb, 2 ; have_right - jz .top_no_left_right - movu m1, [topq+strideq*0] - movu m2, [topq+strideq*1] - mova [px-2*%3+0], m1 - mova [px-1*%3+0], m2 - movd [px-2*%3-4], xm14 - movd [px-1*%3-4], xm14 - jmp .top_done -.top_no_left_right: -%if %1 == 4 - movq xm1, [topq+strideq*0] - movq xm2, [topq+strideq*1] - movq [px-2*%3+0], xm1 - movq [px-1*%3+0], xm2 -%else - mova xm1, [topq+strideq*0] - mova xm2, [topq+strideq*1] - mova [px-2*%3+0], xm1 - mova [px-1*%3+0], xm2 -%endif - movd [px-2*%3-4], xm14 - movd [px-1*%3-4], xm14 - movd [px-2*%3+%1*2], xm14 - movd [px-1*%3+%1*2], xm14 + movd xm0, [topq+strideq*0-4] + movd xm1, [topq+strideq*1-4] + movd [px-16*2-4], xm0 + movd [px-16*1-4], xm1 jmp .top_done .no_top: - movu [px-2*%3-%1], m14 - movu [px-1*%3-%1], m14 + mova [px-16*2+0], m7 +.top_no_left: + movd [px-16*2-4], xm7 + movd [px-16*1-4], xm7 .top_done: - - ; left - test edgeb, 1 ; have_left + test edgeb, 8 ; HAVE_BOTTOM + jz .no_bottom + lea r3, [dstq+strideq*4] + movu xm0, [r3+strideq*0] + movu xm1, [r3+strideq*1] + mova [px+16*4+0], xm0 + mova [px+16*5+0], xm1 + test edgeb, 1 ; HAVE_LEFT + jz .bottom_no_left + movd xm0, [r3+strideq*0-4] + movd xm1, [r3+strideq*1-4] + movd [px+16*4-4], xm0 + movd [px+16*5-4], xm1 + jmp .bottom_done +.no_bottom: + mova [px+16*4+0], m7 +.bottom_no_left: + movd [px+16*4-4], xm7 + movd [px+16*5-4], xm7 +.bottom_done: + test edgeb, 1 ; HAVE_LEFT jz .no_left - mova xm1, [leftq+ 0] -%if %2 == 8 - mova xm2, [leftq+16] -%endif - movd [px+0*%3-4], xm1 - pextrd [px+1*%3-4], xm1, 1 - pextrd [px+2*%3-4], xm1, 2 - pextrd [px+3*%3-4], xm1, 3 -%if %2 == 8 - movd [px+4*%3-4], xm2 - pextrd [px+5*%3-4], xm2, 1 - pextrd [px+6*%3-4], xm2, 2 - pextrd [px+7*%3-4], xm2, 3 -%endif + movd xm0, [leftq+4*0] + movd xm1, [leftq+4*1] + movd xm2, [leftq+4*2] + movd xm3, [leftq+4*3] + movd [px+16*0-4], xm0 + movd [px+16*1-4], xm1 + movd [px+16*2-4], xm2 + movd [px+16*3-4], xm3 jmp .left_done .no_left: - movd [px+0*%3-4], xm14 - movd [px+1*%3-4], xm14 - movd [px+2*%3-4], xm14 - movd [px+3*%3-4], xm14 -%if %2 == 8 - movd [px+4*%3-4], xm14 - movd [px+5*%3-4], xm14 - movd [px+6*%3-4], xm14 - movd [px+7*%3-4], xm14 -%endif + REPX {movd [px+16*x-4], xm7}, 0, 1, 2, 3 .left_done: + test edgeb, 2 ; HAVE_RIGHT + jnz .padding_done + REPX {movd [px+16*x+8], xm7}, -2, -1, 0, 1, 2, 3, 4, 5 +.padding_done: + CDEF_FILTER 4, 4 - ; bottom - DEFINE_ARGS dst, stride, dst8, dummy1, pri, sec, stride3, dummy3, edge - test edgeb, 8 ; have_bottom +cglobal cdef_filter_4x8_16bpc, 4, 9, 9, 16*14, dst, stride, left, top, pri, sec, edge + mov edged, r8m + movu xm0, [dstq+strideq*0] + movu xm1, [dstq+strideq*1] + lea r8, [strideq*3] + movu xm2, [dstq+strideq*2] + movu xm3, [dstq+r8 ] + lea r7, [dstq+strideq*4] + movu xm4, [r7 +strideq*0] + movu xm5, [r7 +strideq*1] + movu xm6, [r7 +strideq*2] + movu xm7, [r7 +r8 ] + lea r7, [dir_table4] + mova [px+16*0+0], xm0 + mova [px+16*1+0], xm1 + mova [px+16*2+0], xm2 + mova [px+16*3+0], xm3 + mova [px+16*4+0], xm4 + mova [px+16*5+0], xm5 + mova [px+16*6+0], xm6 + mova [px+16*7+0], xm7 + vpbroadcastd m7, [base+pw_m16384] + test edgeb, 4 ; HAVE_TOP + jz .no_top + movu xm0, [topq+strideq*0] + movu xm1, [topq+strideq*1] + mova [px-16*2+0], xm0 + mova [px-16*1+0], xm1 + test edgeb, 1 ; HAVE_LEFT + jz .top_no_left + movd xm0, [topq+strideq*0-4] + movd xm1, [topq+strideq*1-4] + movd [px-16*2-4], xm0 + movd [px-16*1-4], xm1 + jmp .top_done +.no_top: + mova [px-16*2+0], m7 +.top_no_left: + movd [px-16*2-4], xm7 + movd [px-16*1-4], xm7 +.top_done: + test edgeb, 8 ; HAVE_BOTTOM jz .no_bottom - lea dst8q, [dstq+%2*strideq] - test edgeb, 1 ; have_left + lea r3, [dstq+strideq*8] + movu xm0, [r3+strideq*0] + movu xm1, [r3+strideq*1] + mova [px+16*8+0], xm0 + mova [px+16*9+0], xm1 + test edgeb, 1 ; HAVE_LEFT jz .bottom_no_left - test edgeb, 2 ; have_right - jz .bottom_no_right - movu m1, [dst8q-%1] - movu m2, [dst8q+strideq-%1] - movu [px+(%2+0)*%3-%1], m1 - movu [px+(%2+1)*%3-%1], m2 - jmp .bottom_done -.bottom_no_right: - movu m1, [dst8q-%1*2] - movu m2, [dst8q+strideq-%1*2] - movu [px+(%2+0)*%3-%1*2], m1 - movu [px+(%2+1)*%3-%1*2], m2 -%if %1 == 8 - movd [px+(%2-1)*%3+%1*2], xm14 ; overwritten by previous movu -%endif - movd [px+(%2+0)*%3+%1*2], xm14 - movd [px+(%2+1)*%3+%1*2], xm14 - jmp .bottom_done -.bottom_no_left: - test edgeb, 2 ; have_right - jz .bottom_no_left_right - movu m1, [dst8q] - movu m2, [dst8q+strideq] - mova [px+(%2+0)*%3+0], m1 - mova [px+(%2+1)*%3+0], m2 - movd [px+(%2+0)*%3-4], xm14 - movd [px+(%2+1)*%3-4], xm14 - jmp .bottom_done -.bottom_no_left_right: -%if %1 == 4 - movq xm1, [dst8q] - movq xm2, [dst8q+strideq] - movq [px+(%2+0)*%3+0], xm1 - movq [px+(%2+1)*%3+0], xm2 -%else - mova xm1, [dst8q] - mova xm2, [dst8q+strideq] - mova [px+(%2+0)*%3+0], xm1 - mova [px+(%2+1)*%3+0], xm2 -%endif - movd [px+(%2+0)*%3-4], xm14 - movd [px+(%2+1)*%3-4], xm14 - movd [px+(%2+0)*%3+%1*2], xm14 - movd [px+(%2+1)*%3+%1*2], xm14 + movd xm0, [r3+strideq*0-4] + movd xm1, [r3+strideq*1-4] + movd [px+16*8-4], xm0 + movd [px+16*9-4], xm1 jmp .bottom_done .no_bottom: - movu [px+(%2+0)*%3-%1], m14 - movu [px+(%2+1)*%3-%1], m14 + mova [px+16*8+0], m7 +.bottom_no_left: + movd [px+16*8-4], xm7 + movd [px+16*9-4], xm7 .bottom_done: - - ; actual filter - INIT_YMM avx2 - DEFINE_ARGS dst, stride, pridmp, damping, pri, secdmp, stride3, zero - %undef edged - movifnidn prid, prim - mov dampingd, r7m - lzcnt pridmpd, prid -%if UNIX64 - movd xm0, prid - movd xm1, secdmpd -%endif - lzcnt secdmpd, secdmpm - sub dampingd, 31 - xor zerod, zerod - add pridmpd, dampingd - cmovs pridmpd, zerod - add secdmpd, dampingd - cmovs secdmpd, zerod - mov [rsp+0], pridmpq ; pri_shift - mov [rsp+8], secdmpq ; sec_shift + test edgeb, 1 ; HAVE_LEFT + jz .no_left + movd xm0, [leftq+4*0] + movd xm1, [leftq+4*1] + movd xm2, [leftq+4*2] + movd xm3, [leftq+4*3] + movd [px+16*0-4], xm0 + movd [px+16*1-4], xm1 + movd [px+16*2-4], xm2 + movd [px+16*3-4], xm3 + movd xm0, [leftq+4*4] + movd xm1, [leftq+4*5] + movd xm2, [leftq+4*6] + movd xm3, [leftq+4*7] + movd [px+16*4-4], xm0 + movd [px+16*5-4], xm1 + movd [px+16*6-4], xm2 + movd [px+16*7-4], xm3 + jmp .left_done +.no_left: + REPX {movd [px+16*x-4], xm7}, 0, 1, 2, 3, 4, 5, 6, 7 +.left_done: + test edgeb, 2 ; HAVE_RIGHT + jnz .padding_done + REPX {movd [px+16*x+8], xm7}, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 +.padding_done: + CDEF_FILTER 4, 8 - ; pri/sec_taps[k] [4 total] - DEFINE_ARGS dst, stride, table, dir, pri, sec, stride3 -%if UNIX64 - vpbroadcastw m0, xm0 ; pri_strength - vpbroadcastw m1, xm1 ; sec_strength +cglobal cdef_filter_8x8_16bpc, 4, 8, 9, 32*13, dst, stride, left, top, pri, sec, edge +%if WIN64 + %define px rsp+32*4 %else - vpbroadcastw m0, prim ; pri_strength - vpbroadcastw m1, secm ; sec_strength -%endif - rorx r2d, prid, 2 - cmp dword r9m, 0xfff - cmove prid, r2d - and prid, 4 - lea tableq, [tap_table] - lea priq, [tableq+priq] ; pri_taps - lea secq, [tableq+8] ; sec_taps - - ; off1/2/3[k] [6 total] from [tableq+12+(dir+0/2/6)*2+k] - mov dird, r6m - lea tableq, [tableq+dirq*2+12] -%if %1*%2*2/mmsize > 1 - %if %1 == 4 - DEFINE_ARGS dst, stride, dir, stk, pri, sec, stride3, h, off, k - %else - DEFINE_ARGS dst, stride, dir, stk, pri, sec, h, off, k - %endif - mov hd, %1*%2*2/mmsize -%else - DEFINE_ARGS dst, stride, dir, stk, pri, sec, stride3, off, k + %define px rsp+32*3 %endif - lea stkq, [px] - pxor m13, m13 -%if %1*%2*2/mmsize > 1 -.v_loop: -%endif - mov kd, 1 -%if %1 == 4 - movq xm4, [stkq+%3*0] - movhps xm4, [stkq+%3*1] - movq xm5, [stkq+%3*2] - movhps xm5, [stkq+%3*3] - vinserti128 m4, xm5, 1 -%else - mova xm4, [stkq+%3*0] ; px - vinserti128 m4, [stkq+%3*1], 1 -%endif - pxor m15, m15 ; sum - mova m7, m4 ; max - mova m8, m4 ; min -.k_loop: - vpbroadcastw m2, [priq+kq*2] ; pri_taps - vpbroadcastw m3, [secq+kq*2] ; sec_taps - - ACCUMULATE_TAP 0*2, [rsp+0], m0, m2, %1, %3 - ACCUMULATE_TAP 2*2, [rsp+8], m1, m3, %1, %3 - ACCUMULATE_TAP 6*2, [rsp+8], m1, m3, %1, %3 - - dec kq - jge .k_loop + %define base r7-dir_table8 + mov edged, r8m + movu m0, [dstq+strideq*0] + movu m1, [dstq+strideq*1] + lea r7, [dstq+strideq*2] + movu m2, [r7 +strideq*0] + movu m3, [r7 +strideq*1] + lea r7, [r7 +strideq*2] + movu m4, [r7 +strideq*0] + movu m5, [r7 +strideq*1] + lea r7, [r7 +strideq*2] + movu m6, [r7 +strideq*0] + movu m7, [r7 +strideq*1] + lea r7, [dir_table8] + mova [px+32*0+0], m0 + mova [px+32*1+0], m1 + mova [px+32*2+0], m2 + mova [px+32*3+0], m3 + mova [px+32*4+0], m4 + mova [px+32*5+0], m5 + mova [px+32*6+0], m6 + mova [px+32*7+0], m7 + vpbroadcastd m7, [base+pw_m16384] + test edgeb, 4 ; HAVE_TOP + jz .no_top + movu m0, [topq+strideq*0] + movu m1, [topq+strideq*1] + mova [px-32*2+0], m0 + mova [px-32*1+0], m1 + test edgeb, 1 ; HAVE_LEFT + jz .top_no_left + movd xm0, [topq+strideq*0-4] + movd xm1, [topq+strideq*1-4] + movd [px-32*2-4], xm0 + movd [px-32*1-4], xm1 + jmp .top_done +.no_top: + mova [px-32*2+0], m7 + mova [px-32*1+0], m7 +.top_no_left: + movd [px-32*2-4], xm7 + movd [px-32*1-4], xm7 +.top_done: + test edgeb, 8 ; HAVE_BOTTOM + jz .no_bottom + lea r3, [dstq+strideq*8] + movu m0, [r3+strideq*0] + movu m1, [r3+strideq*1] + mova [px+32*8+0], m0 + mova [px+32*9+0], m1 + test edgeb, 1 ; HAVE_LEFT + jz .bottom_no_left + movd xm0, [r3+strideq*0-4] + movd xm1, [r3+strideq*1-4] + movd [px+32*8-4], xm0 + movd [px+32*9-4], xm1 + jmp .bottom_done +.no_bottom: + mova [px+32*8+0], m7 + mova [px+32*9+0], m7 +.bottom_no_left: + movd [px+32*8-4], xm7 + movd [px+32*9-4], xm7 +.bottom_done: + test edgeb, 1 ; HAVE_LEFT + jz .no_left + movd xm0, [leftq+4*0] + movd xm1, [leftq+4*1] + movd xm2, [leftq+4*2] + movd xm3, [leftq+4*3] + movd [px+32*0-4], xm0 + movd [px+32*1-4], xm1 + movd [px+32*2-4], xm2 + movd [px+32*3-4], xm3 + movd xm0, [leftq+4*4] + movd xm1, [leftq+4*5] + movd xm2, [leftq+4*6] + movd xm3, [leftq+4*7] + movd [px+32*4-4], xm0 + movd [px+32*5-4], xm1 + movd [px+32*6-4], xm2 + movd [px+32*7-4], xm3 + jmp .left_done +.no_left: + REPX {movd [px+32*x-4], xm7}, 0, 1, 2, 3, 4, 5, 6, 7 +.left_done: + test edgeb, 2 ; HAVE_RIGHT + jnz .padding_done + REPX {movd [px+32*x+16], xm7}, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 +.padding_done: + CDEF_FILTER 8, 8 - vpbroadcastd m12, [pw_2048] - pcmpgtw m11, m13, m15 - paddw m15, m11 - pmulhrsw m15, m12 - paddw m4, m15 - pminsw m4, m7 - pmaxsw m4, m8 -%if %1 == 4 - vextracti128 xm5, m4, 1 - movq [dstq+strideq*0], xm4 - movhps [dstq+strideq*1], xm4 - movq [dstq+strideq*2], xm5 - movhps [dstq+stride3q], xm5 -%else - mova [dstq+strideq*0], xm4 - vextracti128 [dstq+strideq*1], m4, 1 -%endif - -%if %1*%2*2/mmsize > 1 - %define vloop_lines (mmsize/(%1*2)) - lea dstq, [dstq+strideq*vloop_lines] - add stkq, %3*vloop_lines - dec hd - jg .v_loop -%endif - - RET -%endmacro - -cdef_filter_fn 8, 8, 32 -cdef_filter_fn 4, 4, 32 - -INIT_YMM avx2 cglobal cdef_dir_16bpc, 4, 7, 6, src, stride, var, bdmax lea r6, [dir_shift] shr bdmaxd, 11 ; 0 for 10bpc, 1 for 12bpc vpbroadcastd m4, [r6+bdmaxq*4] lea r6, [strideq*3] diff --git a/third_party/dav1d/src/x86/cdef16_sse.asm b/third_party/dav1d/src/x86/cdef16_sse.asm --- a/third_party/dav1d/src/x86/cdef16_sse.asm +++ b/third_party/dav1d/src/x86/cdef16_sse.asm @@ -1,5 +1,7 @@ +; Copyright © 2021, VideoLAN and dav1d authors +; Copyright © 2021, Two Orioles, LLC ; Copyright (c) 2017-2021, The rav1e contributors ; Copyright (c) 2021, Nathan Egge ; All rights reserved. ; ; Redistribution and use in source and binary forms, with or without @@ -26,14 +28,37 @@ %include "config.asm" %include "ext/x86/x86inc.asm" SECTION_RODATA +%macro DUP8 1-* + %rep %0 + times 8 dw %1 + %rotate 1 + %endrep +%endmacro + +pri_taps: DUP8 4, 2, 3, 3 +dir_table: db 1 * 32 + 0, 2 * 32 + 0 + db 1 * 32 + 0, 2 * 32 - 2 + db -1 * 32 + 2, -2 * 32 + 4 + db 0 * 32 + 2, -1 * 32 + 4 + db 0 * 32 + 2, 0 * 32 + 4 + db 0 * 32 + 2, 1 * 32 + 4 + db 1 * 32 + 2, 2 * 32 + 4 + db 1 * 32 + 0, 2 * 32 + 2 + db 1 * 32 + 0, 2 * 32 + 0 + db 1 * 32 + 0, 2 * 32 - 2 + db -1 * 32 + 2, -2 * 32 + 4 + db 0 * 32 + 2, -1 * 32 + 4 + dir_shift: times 4 dw 0x4000 times 4 dw 0x1000 pw_128: times 4 dw 128 +pw_2048: times 8 dw 2048 +pw_m16384: times 8 dw -16384 cextern cdef_dir_8bpc_ssse3.main cextern cdef_dir_8bpc_sse4.main cextern shufw_6543210x @@ -45,10 +70,895 @@ %rotate 1 %%f(%1) %endrep %endmacro +%if ARCH_X86_32 +DECLARE_REG_TMP 5, 3 +%elif WIN64 +DECLARE_REG_TMP 7, 4 +%else +DECLARE_REG_TMP 7, 8 +%endif + +%macro CDEF_FILTER 2 ; w, h +%if ARCH_X86_64 + DEFINE_ARGS dst, stride, tmp, pridmp, pri, sec, dir + mova m8, [base+pw_2048] +%else + DEFINE_ARGS dst, pridmp, tmp, sec, pri, _, dir + %define m8 [base+pw_2048] + %define m9 [rsp+16*1+gprsize] + %define m10 [rsp+16*2+gprsize] +%endif + movifnidn prid, r4m + movifnidn secd, r5m + test prid, prid + jz .sec_only + movd m6, r4m +%if ARCH_X86_32 + mov [rsp+24], pridmpd +%endif + bsr pridmpd, prid + lea tmpd, [priq*4] + cmp dword r9m, 0x3ff ; if (bpc == 10) + cmove prid, tmpd ; pri <<= 2 + mov tmpd, r7m ; damping + mov dird, r6m + and prid, 16 + pshufb m6, m7 ; splat + lea dirq, [base+dir_table+dirq*2] + lea priq, [base+pri_taps+priq*2] + test secd, secd + jz .pri_only + mova [rsp], m6 + movd m6, secd + tzcnt secd, secd + sub pridmpd, tmpd + sub tmpd, secd + pshufb m6, m7 + xor secd, secd + neg pridmpd + cmovs pridmpd, secd +%if ARCH_X86_32 + mov [pri_shift+4], secd + mov [sec_shift+4], secd +%endif + mov [pri_shift+0], pridmpq + mov [sec_shift+0], tmpq + lea tmpq, [px] +%if WIN64 + movaps r4m, m9 + movaps r6m, m10 +%elif ARCH_X86_32 + mov pridmpd, [rsp+24] +%endif +%rep %1*%2/8 + call mangle(private_prefix %+ _cdef_filter_%1x%1_16bpc %+ SUFFIX).pri_sec +%endrep +%if WIN64 + movaps m9, r4m + movaps m10, r6m +%endif + jmp .end +.pri_only: + sub tmpd, pridmpd + cmovs tmpd, secd +%if ARCH_X86_32 + mov pridmpd, [rsp+24] + mov [pri_shift+4], secd +%endif + mov [pri_shift+0], tmpq + lea tmpq, [px] +%rep %1*%2/8 + call mangle(private_prefix %+ _cdef_filter_%1x%1_16bpc %+ SUFFIX).pri +%endrep +.end: + RET +.sec_only: + mov tmpd, r7m ; damping + movd m6, r5m + tzcnt secd, secd + mov dird, r6m + pshufb m6, m7 + sub tmpd, secd + lea dirq, [base+dir_table+dirq*2] +%if ARCH_X86_32 + mov [sec_shift+4], prid +%endif + mov [sec_shift+0], tmpq + lea tmpq, [px] +%rep %1*%2/8 + call mangle(private_prefix %+ _cdef_filter_%1x%1_16bpc %+ SUFFIX).sec +%endrep + jmp .end +%if %1 == %2 +DEFINE_ARGS dst, stride, tmp, off, pri, _, dir +ALIGN function_align +.pri: + movsx offq, byte [dirq+4] ; off_k0 +%if %1 == 4 + movq m1, [dstq+strideq*0] + movhps m1, [dstq+strideq*1] + movq m2, [tmpq+offq+32*0] ; k0p0 + movhps m2, [tmpq+offq+32*1] + neg offq + movq m3, [tmpq+offq+32*0] ; k0p1 + movhps m3, [tmpq+offq+32*1] +%else + mova m1, [dstq] + movu m2, [tmpq+offq] + neg offq + movu m3, [tmpq+offq] +%endif + movsx offq, byte [dirq+5] ; off_k1 + psubw m2, m1 ; diff_k0p0 + psubw m3, m1 ; diff_k0p1 + pabsw m4, m2 ; adiff_k0p0 + psrlw m5, m4, [pri_shift+gprsize] + psubusw m0, m6, m5 + pabsw m5, m3 ; adiff_k0p1 + pminsw m0, m4 + psrlw m4, m5, [pri_shift+gprsize] + psignw m0, m2 ; constrain(diff_k0p0) + psubusw m2, m6, m4 + pminsw m2, m5 +%if %1 == 4 + movq m4, [tmpq+offq+32*0] ; k1p0 + movhps m4, [tmpq+offq+32*1] + neg offq + movq m5, [tmpq+offq+32*0] ; k1p1 + movhps m5, [tmpq+offq+32*1] +%else + movu m4, [tmpq+offq] + neg offq + movu m5, [tmpq+offq] +%endif + psubw m4, m1 ; diff_k1p0 + psubw m5, m1 ; diff_k1p1 + psignw m2, m3 ; constrain(diff_k0p1) + pabsw m3, m4 ; adiff_k1p0 + paddw m0, m2 ; constrain(diff_k0) + psrlw m2, m3, [pri_shift+gprsize] + psubusw m7, m6, m2 + pabsw m2, m5 ; adiff_k1p1 + pminsw m7, m3 + psrlw m3, m2, [pri_shift+gprsize] + psignw m7, m4 ; constrain(diff_k1p0) + psubusw m4, m6, m3 + pminsw m4, m2 + psignw m4, m5 ; constrain(diff_k1p1) + paddw m7, m4 ; constrain(diff_k1) + pmullw m0, [priq+16*0] ; pri_tap_k0 + pmullw m7, [priq+16*1] ; pri_tap_k1 + paddw m0, m7 ; sum + psraw m2, m0, 15 + paddw m0, m2 + pmulhrsw m0, m8 + paddw m0, m1 +%if %1 == 4 + add tmpq, 32*2 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + lea dstq, [dstq+strideq*2] +%else + add tmpq, 32 + mova [dstq], m0 + add dstq, strideq +%endif + ret +ALIGN function_align +.sec: + movsx offq, byte [dirq+8] ; off1_k0 +%if %1 == 4 + movq m1, [dstq+strideq*0] + movhps m1, [dstq+strideq*1] + movq m2, [tmpq+offq+32*0] ; k0s0 + movhps m2, [tmpq+offq+32*1] + neg offq + movq m3, [tmpq+offq+32*0] ; k0s1 + movhps m3, [tmpq+offq+32*1] +%else + mova m1, [dstq] + movu m2, [tmpq+offq] + neg offq + movu m3, [tmpq+offq] +%endif + movsx offq, byte [dirq+0] ; off2_k0 + psubw m2, m1 ; diff_k0s0 + psubw m3, m1 ; diff_k0s1 + pabsw m4, m2 ; adiff_k0s0 + psrlw m5, m4, [sec_shift+gprsize] + psubusw m0, m6, m5 + pabsw m5, m3 ; adiff_k0s1 + pminsw m0, m4 + psrlw m4, m5, [sec_shift+gprsize] + psignw m0, m2 ; constrain(diff_k0s0) + psubusw m2, m6, m4 + pminsw m2, m5 +%if %1 == 4 + movq m4, [tmpq+offq+32*0] ; k0s2 + movhps m4, [tmpq+offq+32*1] + neg offq + movq m5, [tmpq+offq+32*0] ; k0s3 + movhps m5, [tmpq+offq+32*1] +%else + movu m4, [tmpq+offq] + neg offq + movu m5, [tmpq+offq] +%endif + movsx offq, byte [dirq+9] ; off1_k1 + psubw m4, m1 ; diff_k0s2 + psubw m5, m1 ; diff_k0s3 + psignw m2, m3 ; constrain(diff_k0s1) + pabsw m3, m4 ; adiff_k0s2 + paddw m0, m2 + psrlw m2, m3, [sec_shift+gprsize] + psubusw m7, m6, m2 + pabsw m2, m5 ; adiff_k0s3 + pminsw m7, m3 + psrlw m3, m2, [sec_shift+gprsize] + psignw m7, m4 ; constrain(diff_k0s2) + psubusw m4, m6, m3 + pminsw m4, m2 +%if %1 == 4 + movq m2, [tmpq+offq+32*0] ; k1s0 + movhps m2, [tmpq+offq+32*1] + neg offq + movq m3, [tmpq+offq+32*0] ; k1s1 + movhps m3, [tmpq+offq+32*1] +%else + movu m2, [tmpq+offq] + neg offq + movu m3, [tmpq+offq] +%endif + movsx offq, byte [dirq+1] ; off2_k1 + paddw m0, m7 + psignw m4, m5 ; constrain(diff_k0s3) + paddw m0, m4 ; constrain(diff_k0) + psubw m2, m1 ; diff_k1s0 + psubw m3, m1 ; diff_k1s1 + paddw m0, m0 ; sec_tap_k0 + pabsw m4, m2 ; adiff_k1s0 + psrlw m5, m4, [sec_shift+gprsize] + psubusw m7, m6, m5 + pabsw m5, m3 ; adiff_k1s1 + pminsw m7, m4 + psrlw m4, m5, [sec_shift+gprsize] + psignw m7, m2 ; constrain(diff_k1s0) + psubusw m2, m6, m4 + pminsw m2, m5 +%if %1 == 4 + movq m4, [tmpq+offq+32*0] ; k1s2 + movhps m4, [tmpq+offq+32*1] + neg offq + movq m5, [tmpq+offq+32*0] ; k1s3 + movhps m5, [tmpq+offq+32*1] +%else + movu m4, [tmpq+offq] + neg offq + movu m5, [tmpq+offq] +%endif + paddw m0, m7 + psubw m4, m1 ; diff_k1s2 + psubw m5, m1 ; diff_k1s3 + psignw m2, m3 ; constrain(diff_k1s1) + pabsw m3, m4 ; adiff_k1s2 + paddw m0, m2 + psrlw m2, m3, [sec_shift+gprsize] + psubusw m7, m6, m2 + pabsw m2, m5 ; adiff_k1s3 + pminsw m7, m3 + psrlw m3, m2, [sec_shift+gprsize] + psignw m7, m4 ; constrain(diff_k1s2) + psubusw m4, m6, m3 + pminsw m4, m2 + paddw m0, m7 + psignw m4, m5 ; constrain(diff_k1s3) + paddw m0, m4 ; sum + psraw m2, m0, 15 + paddw m0, m2 + pmulhrsw m0, m8 + paddw m0, m1 +%if %1 == 4 + add tmpq, 32*2 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + lea dstq, [dstq+strideq*2] +%else + add tmpq, 32 + mova [dstq], m0 + add dstq, strideq +%endif + ret +ALIGN function_align +.pri_sec: + movsx offq, byte [dirq+8] ; off2_k0 +%if %1 == 4 + movq m1, [dstq+strideq*0] + movhps m1, [dstq+strideq*1] + movq m2, [tmpq+offq+32*0] ; k0s0 + movhps m2, [tmpq+offq+32*1] + neg offq + movq m3, [tmpq+offq+32*0] ; k0s1 + movhps m3, [tmpq+offq+32*1] +%else + mova m1, [dstq] + movu m2, [tmpq+offq] + neg offq + movu m3, [tmpq+offq] +%endif + movsx offq, byte [dirq+0] ; off3_k0 + pabsw m4, m2 +%if ARCH_X86_64 + pabsw m10, m3 + pmaxsw m9, m2, m3 + pminsw m10, m4 +%else + pabsw m7, m3 + pmaxsw m5, m2, m3 + pminsw m4, m7 + mova m9, m5 + mova m10, m4 +%endif + psubw m2, m1 ; diff_k0s0 + psubw m3, m1 ; diff_k0s1 + pabsw m4, m2 ; adiff_k0s0 + psrlw m5, m4, [sec_shift+gprsize] + psubusw m0, m6, m5 + pabsw m5, m3 ; adiff_k0s1 + pminsw m0, m4 + psrlw m4, m5, [sec_shift+gprsize] + psignw m0, m2 ; constrain(diff_k0s0) + psubusw m2, m6, m4 + pminsw m2, m5 +%if %1 == 4 + movq m4, [tmpq+offq+32*0] ; k0s2 + movhps m4, [tmpq+offq+32*1] + neg offq + movq m5, [tmpq+offq+32*0] ; k0s3 + movhps m5, [tmpq+offq+32*1] +%else + movu m4, [tmpq+offq] + neg offq + movu m5, [tmpq+offq] +%endif + movsx offq, byte [dirq+9] ; off2_k1 + pabsw m7, m4 + psignw m2, m3 + pabsw m3, m5 ; constrain(diff_k0s1) +%if ARCH_X86_64 + pmaxsw m9, m4 + pminsw m10, m7 + pmaxsw m9, m5 + pminsw m10, m3 +%else + pminsw m7, m10 + pminsw m7, m3 + pmaxsw m3, m9, m4 + pmaxsw m3, m5 + mova m10, m7 + mova m9, m3 +%endif + psubw m4, m1 ; diff_k0s2 + psubw m5, m1 ; diff_k0s3 + paddw m0, m2 + pabsw m3, m4 ; adiff_k0s2 + psrlw m2, m3, [sec_shift+gprsize] + psubusw m7, m6, m2 + pabsw m2, m5 ; adiff_k0s3 + pminsw m7, m3 + psrlw m3, m2, [sec_shift+gprsize] + psignw m7, m4 ; constrain(diff_k0s2) + psubusw m4, m6, m3 + pminsw m4, m2 +%if %1 == 4 + movq m2, [tmpq+offq+32*0] ; k1s0 + movhps m2, [tmpq+offq+32*1] + neg offq + movq m3, [tmpq+offq+32*0] ; k1s1 + movhps m3, [tmpq+offq+32*1] +%else + movu m2, [tmpq+offq] + neg offq + movu m3, [tmpq+offq] +%endif + movsx offq, byte [dirq+1] ; off3_k1 + paddw m0, m7 + pabsw m7, m2 + psignw m4, m5 ; constrain(diff_k0s3) + pabsw m5, m3 +%if ARCH_X86_64 + pmaxsw m9, m2 + pminsw m10, m7 + pmaxsw m9, m3 + pminsw m10, m5 +%else + pminsw m7, m10 + pminsw m7, m5 + pmaxsw m5, m9, m2 + pmaxsw m5, m3 + mova m10, m7 + mova m9, m5 +%endif + paddw m0, m4 ; constrain(diff_k0) + psubw m2, m1 ; diff_k1s0 + psubw m3, m1 ; diff_k1s1 + paddw m0, m0 ; sec_tap_k0 + pabsw m4, m2 ; adiff_k1s0 + psrlw m5, m4, [sec_shift+gprsize] + psubusw m7, m6, m5 + pabsw m5, m3 ; adiff_k1s1 + pminsw m7, m4 + psrlw m4, m5, [sec_shift+gprsize] + psignw m7, m2 ; constrain(diff_k1s0) + psubusw m2, m6, m4 + pminsw m2, m5 +%if %1 == 4 + movq m4, [tmpq+offq+32*0] ; k1s2 + movhps m4, [tmpq+offq+32*1] + neg offq + movq m5, [tmpq+offq+32*0] ; k1s3 + movhps m5, [tmpq+offq+32*1] +%else + movu m4, [tmpq+offq] + neg offq + movu m5, [tmpq+offq] +%endif + movsx offq, byte [dirq+4] ; off1_k0 + paddw m0, m7 + pabsw m7, m4 + psignw m2, m3 ; constrain(diff_k1s1) + pabsw m3, m5 +%if ARCH_X86_64 + pmaxsw m9, m4 + pminsw m10, m7 + pmaxsw m9, m5 + pminsw m10, m3 +%else + pminsw m7, m10 + pminsw m7, m3 + pmaxsw m3, m9, m4 + pmaxsw m3, m5 + mova m10, m7 + mova m9, m3 +%endif + psubw m4, m1 ; diff_k1s2 + psubw m5, m1 ; diff_k1s3 + pabsw m3, m4 ; adiff_k1s2 + paddw m0, m2 + psrlw m2, m3, [sec_shift+gprsize] + psubusw m7, m6, m2 + pabsw m2, m5 ; adiff_k1s3 + pminsw m7, m3 + psrlw m3, m2, [sec_shift+gprsize] + psignw m7, m4 ; constrain(diff_k1s2) + psubusw m4, m6, m3 + pminsw m4, m2 + paddw m0, m7 +%if %1 == 4 + movq m2, [tmpq+offq+32*0] ; k0p0 + movhps m2, [tmpq+offq+32*1] + neg offq + movq m3, [tmpq+offq+32*0] ; k0p1 + movhps m3, [tmpq+offq+32*1] +%else + movu m2, [tmpq+offq] + neg offq + movu m3, [tmpq+offq] +%endif + movsx offq, byte [dirq+5] ; off1_k1 + pabsw m7, m2 + psignw m4, m5 ; constrain(diff_k1s3) + pabsw m5, m3 +%if ARCH_X86_64 + pmaxsw m9, m2 + pminsw m10, m7 + pmaxsw m9, m3 + pminsw m10, m5 +%else + pminsw m7, m10 + pminsw m7, m5 + pmaxsw m5, m9, m2 + pmaxsw m5, m3 + mova m10, m7 + mova m9, m5 +%endif + psubw m2, m1 ; diff_k0p0 + psubw m3, m1 ; diff_k0p1 + paddw m0, m4 + pabsw m4, m2 ; adiff_k0p0 + psrlw m5, m4, [pri_shift+gprsize] + psubusw m7, [rsp+gprsize], m5 + pabsw m5, m3 ; adiff_k0p1 + pminsw m7, m4 + psrlw m4, m5, [pri_shift+gprsize] + psignw m7, m2 ; constrain(diff_k0p0) + psubusw m2, [rsp+gprsize], m4 + pminsw m2, m5 +%if %1 == 4 + movq m4, [tmpq+offq+32*0] ; k1p0 + movhps m4, [tmpq+offq+32*1] + neg offq + movq m5, [tmpq+offq+32*0] ; k1p1 + movhps m5, [tmpq+offq+32*1] +%else + movu m4, [tmpq+offq] + neg offq + movu m5, [tmpq+offq] +%endif + psignw m2, m3 ; constrain(diff_k0p1) + pabsw m3, m4 + paddw m7, m2 ; constrain(diff_k0) + pabsw m2, m5 +%if ARCH_X86_64 + pmaxsw m9, m4 + pminsw m10, m3 + pmaxsw m9, m5 + pminsw m10, m2 +%else + pminsw m3, m10 + pminsw m3, m2 + pmaxsw m2, m9, m4 + pmaxsw m2, m5 + mova m10, m3 + mova m9, m2 +%endif + psubw m4, m1 ; diff_k1p0 + psubw m5, m1 ; diff_k1p1 + pabsw m3, m4 ; adiff_k1p0 + pmullw m7, [priq+16*0] ; pri_tap_k0 + paddw m0, m7 + psrlw m2, m3, [pri_shift+gprsize] + psubusw m7, [rsp+16*0+gprsize], m2 + pabsw m2, m5 ; adiff_k1p1 + pminsw m7, m3 + psrlw m3, m2, [pri_shift+gprsize] + psignw m7, m4 ; constrain(diff_k1p0) + psubusw m4, [rsp+16*0+gprsize], m3 + pminsw m4, m2 + psignw m4, m5 ; constrain(diff_k1p1) + paddw m7, m4 ; constrain(diff_k1) + pmullw m7, [priq+16*1] ; pri_tap_k1 + paddw m0, m7 ; sum + psraw m2, m0, 15 + paddw m0, m2 + pmulhrsw m0, m8 + paddw m0, m1 +%if ARCH_X86_64 + pmaxsw m9, m1 + pminsw m0, m9 +%else + pmaxsw m2, m9, m1 + pminsw m0, m2 +%endif + pminsw m1, m10 + pmaxsw m0, m1 +%if %1 == 4 + add tmpq, 32*2 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + lea dstq, [dstq+strideq*2] +%else + add tmpq, 32 + mova [dstq], m0 + add dstq, strideq +%endif + ret +%endif +%endmacro + +INIT_XMM ssse3 +%if ARCH_X86_64 +cglobal cdef_filter_4x4_16bpc, 4, 8, 9, 32*10, dst, stride, left, top, pri, sec, edge + %define px rsp+32*4 +%else +cglobal cdef_filter_4x4_16bpc, 2, 7, 8, -32*11, dst, stride, edge, top, left + %define px rsp+32*5 +%endif + %define base t0-dir_table + %define pri_shift px-16*6 + %define sec_shift px-16*5 + mov edged, r8m + LEA t0, dir_table + movu m0, [dstq+strideq*0] + movu m1, [dstq+strideq*1] + lea t1, [dstq+strideq*2] + movu m2, [t1 +strideq*0] + movu m3, [t1 +strideq*1] + movddup m7, [base+pw_m16384] + mova [px+32*0+0], m0 + mova [px+32*1+0], m1 + mova [px+32*2+0], m2 + mova [px+32*3+0], m3 + test edgeb, 4 ; HAVE_TOP + jz .no_top + movifnidn topq, topmp + movu m0, [topq+strideq*0] + movu m1, [topq+strideq*1] + mova [px-32*2+0], m0 + mova [px-32*1+0], m1 + test edgeb, 1 ; HAVE_LEFT + jz .top_no_left + movd m0, [topq+strideq*0-4] + movd m1, [topq+strideq*1-4] + movd [px-32*2-4], m0 + movd [px-32*1-4], m1 + jmp .top_done +.no_top: + mova [px-32*2+0], m7 + mova [px-32*1+0], m7 +.top_no_left: + movd [px-32*2-4], m7 + movd [px-32*1-4], m7 +.top_done: + test edgeb, 8 ; HAVE_BOTTOM + jz .no_bottom + lea r3, [dstq+strideq*4] + movu m0, [r3+strideq*0] + movu m1, [r3+strideq*1] + mova [px+32*4+0], m0 + mova [px+32*5+0], m1 + test edgeb, 1 ; HAVE_LEFT + jz .bottom_no_left + movd m0, [r3+strideq*0-4] + movd m1, [r3+strideq*1-4] + movd [px+32*4-4], m0 + movd [px+32*5-4], m1 + jmp .bottom_done +.no_bottom: + mova [px+32*4+0], m7 + mova [px+32*5+0], m7 +.bottom_no_left: + movd [px+32*4-4], m7 + movd [px+32*5-4], m7 +.bottom_done: + test edgeb, 1 ; HAVE_LEFT + jz .no_left + movifnidn leftq, r2mp + movd m0, [leftq+4*0] + movd m1, [leftq+4*1] + movd m2, [leftq+4*2] + movd m3, [leftq+4*3] + movd [px+32*0-4], m0 + movd [px+32*1-4], m1 + movd [px+32*2-4], m2 + movd [px+32*3-4], m3 + jmp .left_done +.no_left: + REPX {movd [px+32*x-4], m7}, 0, 1, 2, 3 +.left_done: + test edgeb, 2 ; HAVE_RIGHT + jnz .padding_done + REPX {movd [px+32*x+8], m7}, -2, -1, 0, 1, 2, 3, 4, 5 +.padding_done: + CDEF_FILTER 4, 4 + +%if ARCH_X86_64 +cglobal cdef_filter_4x8_16bpc, 4, 8, 9, 32*14, dst, stride, left, top, pri, sec, edge +%else +cglobal cdef_filter_4x8_16bpc, 2, 7, 8, -32*15, dst, stride, edge, top, left +%endif + mov edged, r8m + LEA t0, dir_table + movu m0, [dstq+strideq*0] + movu m1, [dstq+strideq*1] + lea t1, [dstq+strideq*2] + movu m2, [t1 +strideq*0] + movu m3, [t1 +strideq*1] + lea t1, [t1 +strideq*2] + movu m4, [t1 +strideq*0] + movu m5, [t1 +strideq*1] + lea t1, [t1 +strideq*2] + movu m6, [t1 +strideq*0] + movu m7, [t1 +strideq*1] + mova [px+32*0+0], m0 + mova [px+32*1+0], m1 + mova [px+32*2+0], m2 + mova [px+32*3+0], m3 + mova [px+32*4+0], m4 + mova [px+32*5+0], m5 + mova [px+32*6+0], m6 + mova [px+32*7+0], m7 + movddup m7, [base+pw_m16384] + test edgeb, 4 ; HAVE_TOP + jz .no_top + movifnidn topq, topmp + movu m0, [topq+strideq*0] + movu m1, [topq+strideq*1] + mova [px-32*2+0], m0 + mova [px-32*1+0], m1 + test edgeb, 1 ; HAVE_LEFT + jz .top_no_left + movd m0, [topq+strideq*0-4] + movd m1, [topq+strideq*1-4] + movd [px-32*2-4], m0 + movd [px-32*1-4], m1 + jmp .top_done +.no_top: + mova [px-32*2+0], m7 + mova [px-32*1+0], m7 +.top_no_left: + movd [px-32*2-4], m7 + movd [px-32*1-4], m7 +.top_done: + test edgeb, 8 ; HAVE_BOTTOM + jz .no_bottom + lea r3, [dstq+strideq*8] + movu m0, [r3+strideq*0] + movu m1, [r3+strideq*1] + mova [px+32*8+0], m0 + mova [px+32*9+0], m1 + test edgeb, 1 ; HAVE_LEFT + jz .bottom_no_left + movd m0, [r3+strideq*0-4] + movd m1, [r3+strideq*1-4] + movd [px+32*8-4], m0 + movd [px+32*9-4], m1 + jmp .bottom_done +.no_bottom: + mova [px+32*8+0], m7 + mova [px+32*9+0], m7 +.bottom_no_left: + movd [px+32*8-4], m7 + movd [px+32*9-4], m7 +.bottom_done: + test edgeb, 1 ; HAVE_LEFT + jz .no_left + movifnidn leftq, r2mp + movd m0, [leftq+4*0] + movd m1, [leftq+4*1] + movd m2, [leftq+4*2] + movd m3, [leftq+4*3] + movd [px+32*0-4], m0 + movd [px+32*1-4], m1 + movd [px+32*2-4], m2 + movd [px+32*3-4], m3 + movd m0, [leftq+4*4] + movd m1, [leftq+4*5] + movd m2, [leftq+4*6] + movd m3, [leftq+4*7] + movd [px+32*4-4], m0 + movd [px+32*5-4], m1 + movd [px+32*6-4], m2 + movd [px+32*7-4], m3 + jmp .left_done +.no_left: + REPX {movd [px+32*x-4], m7}, 0, 1, 2, 3, 4, 5, 6, 7 +.left_done: + test edgeb, 2 ; HAVE_RIGHT + jnz .padding_done + REPX {movd [px+32*x+8], m7}, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 +.padding_done: + CDEF_FILTER 4, 8 + +%if ARCH_X86_64 +cglobal cdef_filter_8x8_16bpc, 4, 8, 9, 32*14, dst, stride, left, top, pri, sec, edge +%else +cglobal cdef_filter_8x8_16bpc, 2, 7, 8, -32*15, dst, stride, edge, top, left +%endif + mov edged, r8m + LEA t0, dir_table + mova m0, [dstq+strideq*0+ 0] + movd m1, [dstq+strideq*0+16] + mova m2, [dstq+strideq*1+ 0] + movd m3, [dstq+strideq*1+16] + lea t1, [dstq+strideq*2] + mova m4, [t1 +strideq*0+ 0] + movd m5, [t1 +strideq*0+16] + mova m6, [t1 +strideq*1+ 0] + movd m7, [t1 +strideq*1+16] + lea t1, [t1 +strideq*2] + mova [px+32*0+ 0], m0 + movd [px+32*0+16], m1 + mova [px+32*1+ 0], m2 + movd [px+32*1+16], m3 + mova [px+32*2+ 0], m4 + movd [px+32*2+16], m5 + mova [px+32*3+ 0], m6 + movd [px+32*3+16], m7 + mova m0, [t1 +strideq*0+ 0] + movd m1, [t1 +strideq*0+16] + mova m2, [t1 +strideq*1+ 0] + movd m3, [t1 +strideq*1+16] + lea t1, [t1 +strideq*2] + mova m4, [t1 +strideq*0+ 0] + movd m5, [t1 +strideq*0+16] + mova m6, [t1 +strideq*1+ 0] + movd m7, [t1 +strideq*1+16] + mova [px+32*4+ 0], m0 + movd [px+32*4+16], m1 + mova [px+32*5+ 0], m2 + movd [px+32*5+16], m3 + mova [px+32*6+ 0], m4 + movd [px+32*6+16], m5 + mova [px+32*7+ 0], m6 + movd [px+32*7+16], m7 + movddup m7, [base+pw_m16384] + test edgeb, 4 ; HAVE_TOP + jz .no_top + movifnidn topq, topmp + mova m0, [topq+strideq*0+ 0] + mova m1, [topq+strideq*0+16] + mova m2, [topq+strideq*1+ 0] + mova m3, [topq+strideq*1+16] + mova [px-32*2+ 0], m0 + movd [px-32*2+16], m1 + mova [px-32*1+ 0], m2 + movd [px-32*1+16], m3 + test edgeb, 1 ; HAVE_LEFT + jz .top_no_left + movd m0, [topq+strideq*0-4] + movd m1, [topq+strideq*1-4] + movd [px-32*2-4], m0 + movd [px-32*1-4], m1 + jmp .top_done +.no_top: + mova [px-32*2+ 0], m7 + movd [px-32*2+16], m7 + mova [px-32*1+ 0], m7 + movd [px-32*1+16], m7 +.top_no_left: + movd [px-32*2- 4], m7 + movd [px-32*1- 4], m7 +.top_done: + test edgeb, 8 ; HAVE_BOTTOM + jz .no_bottom + lea r3, [dstq+strideq*8] + mova m0, [r3+strideq*0+ 0] + movd m1, [r3+strideq*0+16] + mova m2, [r3+strideq*1+ 0] + movd m3, [r3+strideq*1+16] + mova [px+32*8+ 0], m0 + movd [px+32*8+16], m1 + mova [px+32*9+ 0], m2 + movd [px+32*9+16], m3 + test edgeb, 1 ; HAVE_LEFT + jz .bottom_no_left + movd m0, [r3+strideq*0-4] + movd m1, [r3+strideq*1-4] + movd [px+32*8- 4], m0 + movd [px+32*9- 4], m1 + jmp .bottom_done +.no_bottom: + mova [px+32*8+ 0], m7 + movd [px+32*8+16], m7 + mova [px+32*9+ 0], m7 + movd [px+32*9+16], m7 +.bottom_no_left: + movd [px+32*8- 4], m7 + movd [px+32*9- 4], m7 +.bottom_done: + test edgeb, 1 ; HAVE_LEFT + jz .no_left + movifnidn leftq, r2mp + movd m0, [leftq+4*0] + movd m1, [leftq+4*1] + movd m2, [leftq+4*2] + movd m3, [leftq+4*3] + movd [px+32*0- 4], m0 + movd [px+32*1- 4], m1 + movd [px+32*2- 4], m2 + movd [px+32*3- 4], m3 + movd m0, [leftq+4*4] + movd m1, [leftq+4*5] + movd m2, [leftq+4*6] + movd m3, [leftq+4*7] + movd [px+32*4- 4], m0 + movd [px+32*5- 4], m1 + movd [px+32*6- 4], m2 + movd [px+32*7- 4], m3 + jmp .left_done +.no_left: + REPX {movd [px+32*x- 4], m7}, 0, 1, 2, 3, 4, 5, 6, 7 +.left_done: + test edgeb, 2 ; HAVE_RIGHT + jnz .padding_done + REPX {movd [px+32*x+16], m7}, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 +.padding_done: + CDEF_FILTER 8, 8 + %macro CDEF_DIR 0 %if ARCH_X86_64 cglobal cdef_dir_16bpc, 4, 7, 16, src, stride, var, bdmax lea r6, [dir_shift] shr bdmaxd, 11 ; 0 for 10bpc, 1 for 12bpc diff --git a/third_party/dav1d/src/x86/cdef_avx2.asm b/third_party/dav1d/src/x86/cdef_avx2.asm --- a/third_party/dav1d/src/x86/cdef_avx2.asm +++ b/third_party/dav1d/src/x86/cdef_avx2.asm @@ -470,11 +470,10 @@ test secdmpd, secdmpd jz .pri_only movd xm1, secdmpd lzcnt secdmpd, secdmpd add secdmpd, dampingd - cmovs secdmpd, zerod mov [rsp+8], secdmpq ; sec_shift DEFINE_ARGS dst, stride, left, top, pri, secdmp, table, pridmp lea tableq, [tap_table] vpbroadcastb m13, [tableq+pridmpq] ; pri_shift_mask @@ -550,11 +549,10 @@ .sec_only: DEFINE_ARGS dst, stride, left, top, _, secdmp, zero, _, damping movd xm1, secdmpd lzcnt secdmpd, secdmpd add secdmpd, dampingd - cmovs secdmpd, zerod mov [rsp+8], secdmpq ; sec_shift DEFINE_ARGS dst, stride, left, top, _, secdmp, table lea tableq, [tap_table] vpbroadcastb m14, [tableq+secdmpq] ; sec_shift_mask ; pri/sec_taps[k] [4 total] @@ -1479,11 +1477,10 @@ test secdmpd, secdmpd jz .border_pri_only movd xm1, secdmpd lzcnt secdmpd, secdmpd add secdmpd, dampingd - cmovs secdmpd, zerod mov [rsp+8], secdmpq ; sec_shift DEFINE_ARGS dst, stride, pridmp, table, pri, secdmp, stride3 lea tableq, [tap_table] vpbroadcastb m13, [tableq+pridmpq] ; pri_shift_mask @@ -1554,11 +1551,10 @@ .border_sec_only: DEFINE_ARGS dst, stride, _, damping, _, secdmp, stride3, zero movd xm1, secdmpd lzcnt secdmpd, secdmpd add secdmpd, dampingd - cmovs secdmpd, zerod mov [rsp+8], secdmpq ; sec_shift DEFINE_ARGS dst, stride, _, table, _, secdmp, stride3 lea tableq, [tap_table] vpbroadcastb m14, [tableq+secdmpq] ; sec_shift_mask DEFINE_ARGS dst, stride, dir, table, _, sec, stride3 diff --git a/third_party/dav1d/src/x86/cdef_init_tmpl.c b/third_party/dav1d/src/x86/cdef_init_tmpl.c --- a/third_party/dav1d/src/x86/cdef_init_tmpl.c +++ b/third_party/dav1d/src/x86/cdef_init_tmpl.c @@ -44,26 +44,24 @@ decl_cdef_dir_fn(BF(dav1d_cdef_dir, ssse3)); COLD void bitfn(dav1d_cdef_dsp_init_x86)(Dav1dCdefDSPContext *const c) { const unsigned flags = dav1d_get_cpu_flags(); +#if BITDEPTH == 8 if (!(flags & DAV1D_X86_CPU_FLAG_SSE2)) return; -#if BITDEPTH == 8 c->fb[0] = BF(dav1d_cdef_filter_8x8, sse2); c->fb[1] = BF(dav1d_cdef_filter_4x8, sse2); c->fb[2] = BF(dav1d_cdef_filter_4x4, sse2); #endif if (!(flags & DAV1D_X86_CPU_FLAG_SSSE3)) return; c->dir = BF(dav1d_cdef_dir, ssse3); -#if BITDEPTH == 8 c->fb[0] = BF(dav1d_cdef_filter_8x8, ssse3); c->fb[1] = BF(dav1d_cdef_filter_4x8, ssse3); c->fb[2] = BF(dav1d_cdef_filter_4x4, ssse3); -#endif if (!(flags & DAV1D_X86_CPU_FLAG_SSE41)) return; c->dir = BF(dav1d_cdef_dir, sse4); #if BITDEPTH == 8 @@ -75,13 +73,11 @@ #if ARCH_X86_64 if (!(flags & DAV1D_X86_CPU_FLAG_AVX2)) return; c->dir = BF(dav1d_cdef_dir, avx2); c->fb[0] = BF(dav1d_cdef_filter_8x8, avx2); -#if BITDEPTH == 8 c->fb[1] = BF(dav1d_cdef_filter_4x8, avx2); -#endif c->fb[2] = BF(dav1d_cdef_filter_4x4, avx2); if (!(flags & DAV1D_X86_CPU_FLAG_AVX512ICL)) return; #if HAVE_AVX512ICL && BITDEPTH == 8 diff --git a/third_party/dav1d/src/x86/cdef_sse.asm b/third_party/dav1d/src/x86/cdef_sse.asm --- a/third_party/dav1d/src/x86/cdef_sse.asm +++ b/third_party/dav1d/src/x86/cdef_sse.asm @@ -564,20 +564,19 @@ movd m1, prim bsr pridmpd, prid test secd, secd jz .pri_only movd m10, r5m - bsr secd, secd + tzcnt secd, secd and prid, 1 sub pridmpd, dampingd sub secd, dampingd xor dampingd, dampingd add prid, prid neg pridmpd cmovs pridmpd, dampingd neg secd - cmovs secd, dampingd PSHUFB_0 m1, m7 PSHUFB_0 m10, m7 %if ARCH_X86_64 DEFINE_ARGS dst, stride, pridmp, tap, pri, sec lea tapq, [tap_table] @@ -695,11 +694,11 @@ DEFINE_ARGS dst, stride, dir, damping, tap, sec, zero %else DEFINE_ARGS dst, stride, sec, damping, dir, tap, zero %endif movd m1, r5m - bsr secd, secd + tzcnt secd, secd mov dird, r6m xor zerod, zerod sub dampingd, secd cmovs dampingd, zerod PSHUFB_0 m1, m7 diff --git a/third_party/dav1d/src/x86/film_grain16_avx2.asm b/third_party/dav1d/src/x86/film_grain16_avx2.asm --- a/third_party/dav1d/src/x86/film_grain16_avx2.asm +++ b/third_party/dav1d/src/x86/film_grain16_avx2.asm @@ -27,12 +27,10 @@ %include "ext/x86/x86inc.asm" %if ARCH_X86_64 SECTION_RODATA 32 -pw_1024: times 16 dw 1024 -pw_23_22: times 8 dw 23, 22 pb_mask: db 0, 0x80, 0x80, 0, 0x80, 0, 0, 0x80, 0x80, 0, 0, 0x80, 0, 0x80, 0x80, 0 rnd_next_upperbit_mask: dw 0x100B, 0x2016, 0x402C, 0x8058 pw_seed_xor: times 2 dw 0xb524 times 2 dw 0x49d8 pd_16: dd 16 @@ -46,10 +44,11 @@ min: dw 0, 16*4, 16*16 pw_27_17_17_27: dw 27, 17, 17, 27 ; these two should be next to each other pw_4: times 2 dw 4 pw_16: times 2 dw 16 +pw_23_22: dw 23, 22, 0, 32 %macro JMP_TABLE 1-* %xdefine %1_table %%table %xdefine %%base %1_table %xdefine %%prefix mangle(private_prefix %+ _%1) @@ -60,10 +59,12 @@ %endrep %endmacro JMP_TABLE generate_grain_y_16bpc_avx2, 0, 1, 2, 3 JMP_TABLE generate_grain_uv_420_16bpc_avx2, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_422_16bpc_avx2, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_444_16bpc_avx2, 0, 1, 2, 3 struc FGData .seed: resd 1 .num_y_points: resd 1 .y_points: resb 14 * 2 @@ -401,12 +402,13 @@ %if WIN64 mov rsp, r6 %endif RET +%macro generate_grain_uv_fn 3 ; ss_name, ss_x, ss_y INIT_XMM avx2 -cglobal generate_grain_uv_420_16bpc, 4, 10, 16, buf, bufy, fg_data, uv, bdmax +cglobal generate_grain_uv_%1_16bpc, 4, 10, 16, buf, bufy, fg_data, uv, bdmax %define base r8-pb_mask lea r8, [pb_mask] movifnidn bdmaxd, bdmaxm movq xm1, [base+rnd_next_upperbit_mask] movq xm4, [base+mul_bits] @@ -420,14 +422,19 @@ vpbroadcastw xm0, [fg_dataq+FGData.seed] vpbroadcastw xm9, [base+pw_seed_xor+uvq*4] pxor xm0, xm9 vpbroadcastd xm9, [base+pd_m65536] lea r6, [gaussian_sequence] - mov r7d, 38 +%if %2 + mov r7d, 73-35*%3 add bufq, 44*2 .loop_y: mov r5, -44 +%else + mov r5, -82*73 + add bufq, 2*82*73 +%endif .loop_x: pand xm2, xm0, xm1 psrlw xm3, xm2, 10 por xm2, xm3 ; bits 0xf, 0x1e, 0x3c and 0x78 are set pmullw xm2, xm4 ; bits 0x0f00 are set @@ -449,18 +456,20 @@ ; shifts by 0, which pmulhrsw does not support pmulhrsw xm2, xm8 movq [bufq+r5*2], xm2 add r5, 4 jl .loop_x +%if %2 add bufq, 82*2 dec r7d jg .loop_y +%endif ; auto-regression code movsxd r5, [fg_dataq+FGData.ar_coeff_lag] - movsxd r5, [base+generate_grain_uv_420_16bpc_avx2_table+r5*4] - lea r5, [r5+base+generate_grain_uv_420_16bpc_avx2_table] + movsxd r5, [base+generate_grain_uv_%1_16bpc_avx2_table+r5*4] + lea r5, [r5+base+generate_grain_uv_%1_16bpc_avx2_table] jmp r5 .ar0: INIT_YMM avx2 DEFINE_ARGS buf, bufy, fg_data, uv, bdmax, shift @@ -471,67 +480,96 @@ sar bdmaxd, 1 movd xm14, bdmaxd pcmpeqw m7, m7 vpbroadcastw m14, xm14 ; max_gain pxor m7, m14 ; min_grain - DEFINE_ARGS buf, bufy, h + DEFINE_ARGS buf, bufy, h, x pmovsxbw xm4, xm4 - vpbroadcastw m6, [hmul_bits+4] +%if %2 + vpbroadcastw m6, [hmul_bits+2+%3*2] +%endif vpbroadcastw m4, xm4 pxor m5, m5 - sub bufq, 2*(82*38+82-(82*3+41)) +%if %2 + sub bufq, 2*(82*(73-35*%3)+82-(82*3+41)) +%else + sub bufq, 2*(82*70-3) +%endif add bufyq, 2*(3+82*3) - mov hd, 35 + mov hd, 70-35*%3 .y_loop_ar0: +%if %2 ; first 32 pixels movu xm8, [bufyq] + movu xm10, [bufyq+ 16] +%if %3 movu xm9, [bufyq+82*2] - movu xm10, [bufyq+ 16] movu xm11, [bufyq+82*2+16] +%endif vinserti128 m8, [bufyq+ 32], 1 + vinserti128 m10, [bufyq+ 48], 1 +%if %3 vinserti128 m9, [bufyq+82*2+32], 1 - vinserti128 m10, [bufyq+ 48], 1 vinserti128 m11, [bufyq+82*2+48], 1 paddw m8, m9 paddw m10, m11 +%endif phaddw m8, m10 movu xm10, [bufyq+ 64] + movu xm12, [bufyq+ 80] +%if %3 movu xm11, [bufyq+82*2+64] - movu xm12, [bufyq+ 80] movu xm13, [bufyq+82*2+80] +%endif vinserti128 m10, [bufyq+ 96], 1 + vinserti128 m12, [bufyq+ 112], 1 +%if %3 vinserti128 m11, [bufyq+82*2+96], 1 - vinserti128 m12, [bufyq+ 112], 1 vinserti128 m13, [bufyq+82*2+112], 1 paddw m10, m11 paddw m12, m13 +%endif phaddw m10, m12 pmulhrsw m8, m6 pmulhrsw m10, m6 +%else + xor xd, xd +.x_loop_ar0: + movu m8, [bufyq+xq*2] + movu m10, [bufyq+xq*2+32] +%endif punpckhwd m9, m8, m5 punpcklwd m8, m5 punpckhwd m11, m10, m5 punpcklwd m10, m5 REPX {pmaddwd x, m4}, m8, m9, m10, m11 REPX {psrad x, 5}, m8, m9, m10, m11 packssdw m8, m9 packssdw m10, m11 REPX {pmulhrsw x, m3}, m8, m10 +%if %2 paddw m8, [bufq+ 0] paddw m10, [bufq+32] +%else + paddw m8, [bufq+xq*2+ 0] + paddw m10, [bufq+xq*2+32] +%endif pminsw m8, m14 pminsw m10, m14 pmaxsw m8, m7 pmaxsw m10, m7 +%if %2 movu [bufq+ 0], m8 movu [bufq+32], m10 ; last 6 pixels movu xm8, [bufyq+32*4] movu xm10, [bufyq+32*4+16] +%if %3 paddw xm8, [bufyq+32*4+82*2] paddw xm10, [bufyq+32*4+82*2+16] +%endif phaddw xm8, xm10 pmulhrsw xm8, xm6 punpckhwd xm9, xm8, xm5 punpcklwd xm8, xm5 REPX {pmaddwd x, xm4}, xm8, xm9 @@ -542,13 +580,35 @@ paddw xm8, xm0 pminsw xm8, xm14 pmaxsw xm8, xm7 vpblendw xm0, xm8, xm0, 11000000b movu [bufq+32*2], xm0 +%else + movu [bufq+xq*2+ 0], m8 + movu [bufq+xq*2+32], m10 + add xd, 32 + cmp xd, 64 + jl .x_loop_ar0 + + ; last 12 pixels + movu m8, [bufyq+64*2] + punpckhwd m9, m8, m5 + punpcklwd m8, m5 + REPX {pmaddwd x, m4}, m8, m9 + REPX {psrad x, 5}, m8, m9 + packssdw m8, m9 + pmulhrsw m8, m3 + movu m0, [bufq+64*2] + paddw m8, m0 + pminsw m8, m14 + pmaxsw m8, m7 + vpblendd m0, m8, m0, 11000000b + movu [bufq+64*2], m0 +%endif add bufq, 82*2 - add bufyq, 82*4 + add bufyq, 82*2<<%3 dec hd jg .y_loop_ar0 RET .ar1: @@ -562,30 +622,44 @@ DEFINE_ARGS buf, bufy, h, val0, max, cf3, min, val3, x, shift pmovsxbw xm4, xm4 pshufd xm5, xm4, q1111 pshufd xm4, xm4, q0000 pmovsxwd xm3, [base+round_vals+shiftq*2-12] ; rnd - vpbroadcastw xm6, [hmul_bits+4] + vpbroadcastw xm6, [hmul_bits+2+%3*2] vpbroadcastd xm3, xm3 - sub bufq, 2*(82*38+44-(82*3+41)) +%if %2 + sub bufq, 2*(82*(73-35*%3)+44-(82*3+41)) +%else + sub bufq, 2*(82*69+3) +%endif add bufyq, 2*(79+82*3) - mov hd, 35 + mov hd, 70-35*%3 sar maxd, 1 mov mind, maxd xor mind, -1 .y_loop_ar1: - mov xq, -38 + mov xq, -(76>>%2) movsx val3d, word [bufq+xq*2-2] .x_loop_ar1: movu xm0, [bufq+xq*2-82*2-2] ; top/left +%if %2 movu xm8, [bufyq+xq*4] +%else + movq xm8, [bufyq+xq*2] +%endif psrldq xm2, xm0, 2 ; top psrldq xm1, xm0, 4 ; top/right +%if %2 +%if %3 phaddw xm8, [bufyq+xq*4+82*2] pshufd xm9, xm8, q3232 paddw xm8, xm9 +%else + phaddw xm8, xm8 +%endif pmulhrsw xm8, xm6 +%endif punpcklwd xm0, xm2 punpcklwd xm1, xm8 pmaddwd xm0, xm4 pmaddwd xm1, xm5 paddd xm0, xm1 @@ -610,11 +684,11 @@ jnz .x_loop_ar1_inner jmp .x_loop_ar1 .x_loop_ar1_end: add bufq, 82*2 - add bufyq, 82*4 + add bufyq, 82*2<<%3 dec hd jg .y_loop_ar1 RET INIT_YMM avx2 @@ -625,11 +699,13 @@ sar bdmaxd, 1 movd xm6, bdmaxd pcmpeqd xm5, xm5 vpbroadcastd xm6, xm6 ; max_grain pxor xm5, xm6 ; min_grain - vpbroadcastw xm7, [base+hmul_bits+4] +%if %2 + vpbroadcastw xm7, [base+hmul_bits+2+%3*2] +%endif vpbroadcastw xm15, [base+round_vals-12+shiftq*2] movd xm0, [fg_dataq+FGData.ar_coeffs_uv+uvq+5] pinsrb xm0, [fg_dataq+FGData.ar_coeffs_uv+uvq+12], 4 pinsrb xm0, [pb_1], 5 @@ -642,34 +718,48 @@ pshufd m12, m0, q0000 pshufd m11, m0, q1111 pshufd m10, m0, q2222 DEFINE_ARGS buf, bufy, fg_data, h, x - sub bufq, 2*(82*38+44-(82*3+41)) +%if %2 + sub bufq, 2*(82*(73-35*%3)+44-(82*3+41)) +%else + sub bufq, 2*(82*69+3) +%endif add bufyq, 2*(79+82*3) - mov hd, 35 + mov hd, 70-35*%3 .y_loop_ar2: - mov xq, -38 + mov xq, -(76>>%2) .x_loop_ar2: movu xm0, [bufq+xq*2-82*2-4] ; y=-1,x=[-2,+5] vinserti128 m0, [bufq+xq*2-82*4-4], 1 ; y=-2,x=[-2,+5] psrldq m1, m0, 2 ; y=-1/-2,x=[-1,+5] psrldq m2, m0, 4 ; y=-1/-2,x=[-0,+5] psrldq m3, m0, 6 ; y=-1/-2,x=[+1,+5] +%if %2 movu xm8, [bufyq+xq*4] +%if %3 paddw xm8, [bufyq+xq*4+82*2] +%endif phaddw xm8, xm8 +%else + movq xm8, [bufyq+xq*2] +%endif vinserti128 m4, xm0, 1 ; y=-1,x=[-2,+5] punpcklwd m2, m3 ; y=-1/-2,x=[+0/+1,+1/+2,+2/+3,+3/+4] punpckhwd m4, m0, m4 ; y=-2/-1 interleaved, x=[+2,+5] punpcklwd m0, m1 ; y=-1/-2,x=[-2/-1,-1/+0,+0/+1,+1/+2] +%if %2 pmulhrsw xm1, xm8, xm7 punpcklwd xm1, xm15 ; luma, round interleaved +%else + punpcklwd xm1, xm8, xm15 +%endif vpblendd m1, m1, m4, 11110000b pmaddwd m2, m11 pmaddwd m0, m12 pmaddwd m1, m10 @@ -701,11 +791,11 @@ jnz .x_loop_ar2_inner jmp .x_loop_ar2 .x_loop_ar2_end: add bufq, 82*2 - add bufyq, 82*4 + add bufyq, 82*2<<%3 dec hd jg .y_loop_ar2 RET .ar3: @@ -728,11 +818,13 @@ sar bdmaxd, 1 movd xm15, bdmaxd pcmpeqd xm13, xm13 vpbroadcastd xm15, xm15 ; max_grain pxor xm13, xm15 ; min_grain - vpbroadcastw xm12, [base+hmul_bits+4] +%if %2 + vpbroadcastw xm12, [base+hmul_bits+2+%3*2] +%endif movq xm0, [fg_dataq+FGData.ar_coeffs_uv+uvq+ 0] pinsrb xm0, [fg_dataq+FGData.ar_coeffs_uv+uvq+24], 7 ; luma movhps xm0, [fg_dataq+FGData.ar_coeffs_uv+uvq+ 7] pmovsxbw m0, xm0 @@ -755,34 +847,46 @@ mova [tmp+32*0], m1 mova [tmp+32*1], m2 pinsrw xm11, [base+round_vals-10+shiftq*2], 3 DEFINE_ARGS buf, bufy, fg_data, h, unused, x - sub bufq, 2*(82*38+44-(82*3+41)) +%if %2 + sub bufq, 2*(82*(73-35*%3)+44-(82*3+41)) +%else + sub bufq, 2*(82*69+3) +%endif add bufyq, 2*(79+82*3) - mov hd, 35 + mov hd, 70-35*%3 .y_loop_ar3: - mov xq, -38 + mov xq, -(76>>%2) .x_loop_ar3: movu xm0, [bufq+xq*2-82*6-6+ 0] ; y=-3,x=[-3,+4] movq xm1, [bufq+xq*2-82*6-6+16] ; y=-3,x=[+5,+8] movu xm2, [bufq+xq*2-82*2-6+ 0] ; y=-1,x=[-3,+4] vinserti128 m0, [bufq+xq*2-82*4-6+ 0], 1 ; y=-3/-2,x=[-3,+4] vinserti128 m1, [bufq+xq*2-82*4-6+16], 1 ; y=-3/-2,x=[+5,+12] vinserti128 m2, [bufq+xq*2-82*2-6+ 6], 1 ; y=-1,x=[+1,+8] +%if %2 movu xm7, [bufyq+xq*4] +%if %3 paddw xm7, [bufyq+xq*4+82*2] +%endif phaddw xm7, xm7 +%else + movq xm7, [bufyq+xq*2] +%endif palignr m4, m1, m0, 2 ; y=-3/-2,x=[-2,+5] palignr m1, m0, 12 ; y=-3/-2,x=[+3,+6] punpckhwd m5, m0, m4 ; y=-3/-2,x=[+1/+2,+2/+3,+3/+4,+4/+5] punpcklwd m0, m4 ; y=-3/-2,x=[-3/-2,-2/-1,-1/+0,+0/+1] palignr m6, m5, m0, 8 ; y=-3/-2,x=[-1/+0,+0/+1,+1/+2,+2/+3] +%if %2 pmulhrsw xm7, xm12 +%endif punpcklwd m1, m7 psrldq m3, m2, 2 psrldq m4, m2, 4 psrldq m7, m2, 6 @@ -828,17 +932,22 @@ jnz .x_loop_ar3_inner jmp .x_loop_ar3 .x_loop_ar3_end: add bufq, 82*2 - add bufyq, 82*4 + add bufyq, 82*2<<%3 dec hd jg .y_loop_ar3 %if WIN64 mov rsp, r6 %endif RET +%endmacro + +generate_grain_uv_fn 420, 1, 1 +generate_grain_uv_fn 422, 1, 0 +generate_grain_uv_fn 444, 0, 0 INIT_YMM avx2 cglobal fgy_32x32xn_16bpc, 6, 14, 16, dst, src, stride, fg_data, w, scaling, grain_lut mov r7d, [fg_dataq+FGData.scaling_shift] lea r8, [pb_mask] @@ -1334,11 +1443,12 @@ jl .loop_x_hv_overlap .end_hv: RET -cglobal fguv_32x32xn_i420_16bpc, 6, 15, 16, dst, src, stride, fg_data, w, scaling, \ +%macro FGUV_FN 3 ; name, ss_hor, ss_ver +cglobal fguv_32x32xn_i%1_16bpc, 6, 15, 16, dst, src, stride, fg_data, w, scaling, \ grain_lut, h, sby, luma, lstride, uv_pl, is_id %define base r8-pb_mask lea r8, [pb_mask] mov r7d, [fg_dataq+FGData.scaling_shift] vpbroadcastw m11, [base+mul_bits+r7*2-14] @@ -1356,11 +1466,11 @@ pxor m2, m2 cmp byte [fg_dataq+FGData.chroma_scaling_from_luma], 0 jne .csfl -%macro FGUV_32x32xN_LOOP 1 ; not-csfl +%macro %%FGUV_32x32xN_LOOP 3 ; not-csfl, ss_hor, ss_ver DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused, sby, see, overlap %if %1 mov r7d, r11m vpbroadcastw m0, [fg_dataq+FGData.uv_mult+r7*4] @@ -1368,12 +1478,16 @@ punpcklwd m14, m1, m0 vpbroadcastw m15, [fg_dataq+FGData.uv_offset+r7*4] vpbroadcastd m9, [base+pw_4+r9*4] pmullw m15, m9 %else - vpbroadcastd m14, [pw_1024] - vpbroadcastd m15, [pw_23_22] + vpbroadcastd m14, [pd_16] +%if %2 + vpbroadcastq m15, [pw_23_22] +%else + vpbroadcastq m15, [pw_27_17_17_27] +%endif %endif movifnidn sbyd, sbym test sbyd, sbyd setnz r7b @@ -1391,11 +1505,11 @@ mov lumaq, r9mp mov lstrideq, r10mp lea r10, [srcq+wq*2] lea r11, [dstq+wq*2] - lea r12, [lumaq+wq*4] + lea r12, [lumaq+wq*(2<<%2)] mov r10mp, r10 mov r11mp, r11 mov r12mp, r12 neg wq @@ -1412,36 +1526,45 @@ mov offxd, seed rorx offyd, seed, 8 shr offxd, 12 and offyd, 0xf - imul offyd, 82 - lea offyq, [offyq+offxq+498] ; offy*stride+offx + imul offyd, 164>>%3 + lea offyq, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+(3+(6>>%2))] ; offy*stride+offx DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ h, offxy, see, unused1, unused2, unused3, luma, lstride mov hd, hm mov grain_lutq, grain_lutmp %%loop_y: ; src mova m0, [srcq] +%if %2 mova m1, [srcq+strideq] ; m0-1: src as word +%else + mova m1, [srcq+32] +%endif ; luma_src +%if %2 mova xm4, [lumaq+lstrideq*0+ 0] mova xm7, [lumaq+lstrideq*0+16] vinserti128 m4, [lumaq+lstrideq*0+32], 1 vinserti128 m7, [lumaq+lstrideq*0+48], 1 - mova xm6, [lumaq+lstrideq*2+ 0] - mova xm8, [lumaq+lstrideq*2+16] - vinserti128 m6, [lumaq+lstrideq*2+32], 1 - vinserti128 m8, [lumaq+lstrideq*2+48], 1 + mova xm6, [lumaq+lstrideq*(1<<%3)+ 0] + mova xm8, [lumaq+lstrideq*(1<<%3)+16] + vinserti128 m6, [lumaq+lstrideq*(1<<%3)+32], 1 + vinserti128 m8, [lumaq+lstrideq*(1<<%3)+48], 1 phaddw m4, m7 phaddw m6, m8 pavgw m4, m2 pavgw m6, m2 +%else + mova m4, [lumaq] + mova m6, [lumaq+32] +%endif %if %1 punpckhwd m3, m4, m0 punpcklwd m4, m0 punpckhwd m5, m6, m1 @@ -1475,11 +1598,15 @@ packssdw m8, m4 packssdw m5, m6 ; grain = grain_lut[offy+y][offx+x] movu m9, [grain_lutq+offxyq*2] +%if %2 movu m3, [grain_lutq+offxyq*2+82*2] +%else + movu m3, [grain_lutq+offxyq*2+32] +%endif ; noise = round2(scaling[luma_src] * grain, scaling_shift) REPX {pmullw x, m11}, m8, m5 pmulhrsw m9, m8 pmulhrsw m3, m5 @@ -1490,27 +1617,38 @@ pmaxsw m0, m13 pmaxsw m1, m13 pminsw m0, m12 pminsw m1, m12 mova [dstq], m0 +%if %2 mova [dstq+strideq], m1 lea srcq, [srcq+strideq*2] lea dstq, [dstq+strideq*2] - lea lumaq, [lumaq+lstrideq*4] - add grain_lutq, 82*4 + lea lumaq, [lumaq+lstrideq*(2<<%3)] +%else + mova [dstq+32], m1 + add srcq, strideq + add dstq, strideq + add lumaq, lstrideq +%endif + add grain_lutq, 82*(2<<%2) +%if %2 sub hb, 2 +%else + dec hb +%endif jg %%loop_y - add wq, 16 + add wq, 32>>%2 jge %%end mov srcq, r10mp mov dstq, r11mp mov lumaq, r12mp lea srcq, [srcq+wq*2] lea dstq, [dstq+wq*2] - lea lumaq, [lumaq+wq*4] + lea lumaq, [lumaq+wq*(2<<%2)] cmp byte [fg_dataq+FGData.overlap_flag], 0 je %%loop_x ; r8m = sbym cmp dword r8m, 0 @@ -1526,40 +1664,48 @@ cmovp seed, r6d ; updated seed DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ offx, offy, see, left_offxy, unused1, unused2, luma, lstride - lea left_offxyd, [offyd+16] ; previous column's offy*stride+offx + lea left_offxyd, [offyd+(32>>%2)] ; previous column's offy*stride+offx mov offxd, seed rorx offyd, seed, 8 shr offxd, 12 and offyd, 0xf - imul offyd, 82 - lea offyq, [offyq+offxq+498] ; offy*stride+offx + imul offyd, 164>>%3 + lea offyq, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ h, offxy, see, left_offxy, unused1, unused2, luma, lstride mov hd, hm mov grain_lutq, grain_lutmp %%loop_y_h_overlap: mova m0, [srcq] +%if %2 mova m1, [srcq+strideq] ; luma_src mova xm4, [lumaq+lstrideq*0+ 0] mova xm7, [lumaq+lstrideq*0+16] vinserti128 m4, [lumaq+lstrideq*0+32], 1 vinserti128 m7, [lumaq+lstrideq*0+48], 1 - mova xm6, [lumaq+lstrideq*2+ 0] - mova xm8, [lumaq+lstrideq*2+16] - vinserti128 m6, [lumaq+lstrideq*2+32], 1 - vinserti128 m8, [lumaq+lstrideq*2+48], 1 + mova xm6, [lumaq+lstrideq*(1<<%3)+ 0] + mova xm8, [lumaq+lstrideq*(1<<%3)+16] + vinserti128 m6, [lumaq+lstrideq*(1<<%3)+32], 1 + vinserti128 m8, [lumaq+lstrideq*(1<<%3)+48], 1 phaddw m4, m7 phaddw m6, m8 pavgw m4, m2 pavgw m6, m2 +%else + mova m1, [srcq+32] + + ; luma_src + mova m4, [lumaq] + mova m6, [lumaq+32] +%endif %if %1 punpckhwd m3, m4, m0 punpcklwd m4, m0 punpckhwd m5, m6, m1 @@ -1575,34 +1721,48 @@ REPX {pminuw x, m10}, m4, m6 %endif ; grain = grain_lut[offy+y][offx+x] movu m9, [grain_lutq+offxyq*2] +%if %2 movu m3, [grain_lutq+offxyq*2+82*2] +%else + movu m3, [grain_lutq+offxyq*2+32] +%endif movd xm5, [grain_lutq+left_offxyq*2+ 0] - pinsrw xm5, [grain_lutq+left_offxyq*2+82*2], 1 ; {left0, left1} - punpcklwd xm7, xm9, xm3 ; {cur0, cur1} +%if %2 + pinsrw xm5, [grain_lutq+left_offxyq*2+82*2], 2 ; {left0, left1} + punpckldq xm7, xm9, xm3 ; {cur0, cur1} punpcklwd xm5, xm7 ; {left0, cur0, left1, cur1} +%else + punpcklwd xm5, xm9 +%endif %if %1 - pmaddwd xm5, [pw_23_22] +%if %2 + vpbroadcastq xm8, [pw_23_22] +%else + movq xm8, [pw_27_17_17_27] +%endif + pmaddwd xm5, xm8 + vpbroadcastd xm8, [pd_16] + paddd xm5, xm8 %else pmaddwd xm5, xm15 + paddd xm5, xm14 %endif - vpbroadcastd xm8, [pd_16] - paddd xm5, xm8 psrad xm5, 5 packssdw xm5, xm5 pcmpeqw xm8, xm8 psraw xm7, xm10, 1 pxor xm8, xm7 pmaxsw xm5, xm8 pminsw xm5, xm7 - vpblendw xm7, xm5, xm9, 11111110b - psrldq xm5, 2 - vpblendw xm5, xm3, 11111110b - vpblendd m9, m7, 00001111b - vpblendd m3, m5, 00001111b + vpblendd m9, m9, m5, 00000001b +%if %2 + psrldq xm5, 4 + vpblendd m3, m3, m5, 00000001b +%endif ; scaling[luma_src] punpckhwd m5, m4, m2 punpcklwd m4, m2 pcmpeqw m7, m7 @@ -1636,27 +1796,40 @@ pmaxsw m0, m13 pmaxsw m1, m13 pminsw m0, m12 pminsw m1, m12 mova [dstq], m0 +%if %2 mova [dstq+strideq], m1 lea srcq, [srcq+strideq*2] lea dstq, [dstq+strideq*2] - lea lumaq, [lumaq+lstrideq*4] - add grain_lutq, 82*4 + lea lumaq, [lumaq+lstrideq*(2<<%3)] +%else + mova [dstq+32], m1 + + add srcq, strideq + add dstq, strideq + add lumaq, lstrideq +%endif + + add grain_lutq, 82*(2<<%2) +%if %2 sub hb, 2 +%else + dec hb +%endif jg %%loop_y_h_overlap - add wq, 16 + add wq, 32>>%2 jge %%end mov srcq, r10mp mov dstq, r11mp mov lumaq, r12mp lea srcq, [srcq+wq*2] lea dstq, [dstq+wq*2] - lea lumaq, [lumaq+wq*4] + lea lumaq, [lumaq+wq*(2<<%2)] ; r8m = sbym cmp dword r8m, 0 jne %%loop_x_hv_overlap jmp %%loop_x_h_overlap @@ -1684,11 +1857,11 @@ mov lumaq, r9mp mov lstrideq, r10mp lea r10, [srcq+wq*2] lea r11, [dstq+wq*2] - lea r12, [lumaq+wq*4] + lea r12, [lumaq+wq*(2<<%2)] mov r10mp, r10 mov r11mp, r11 mov r12mp, r12 neg wq @@ -1711,40 +1884,51 @@ rorx offyd, seed, 8 rorx offxd, seed, 12 and offyd, 0xf000f and offxd, 0xf000f - imul offyd, 82 + imul offyd, 164>>%3 ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy - lea offyq, [offyq+offxq+0x10001*498+16*82] + lea offyq, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ h, offxy, see, unused1, top_offxy, unused2, luma, lstride movzx top_offxyd, offxyw shr offxyd, 16 +%if %2 == 0 + lea r10, [pw_27_17_17_27] +%endif mov hd, hm mov grain_lutq, grain_lutmp %%loop_y_v_overlap: ; src mova m0, [srcq] +%if %2 mova m1, [srcq+strideq] ; luma_src mova xm4, [lumaq+lstrideq*0+ 0] mova xm7, [lumaq+lstrideq*0+16] vinserti128 m4, [lumaq+lstrideq*0+32], 1 vinserti128 m7, [lumaq+lstrideq*0+48], 1 - mova xm6, [lumaq+lstrideq*2+ 0] - mova xm8, [lumaq+lstrideq*2+16] - vinserti128 m6, [lumaq+lstrideq*2+32], 1 - vinserti128 m8, [lumaq+lstrideq*2+48], 1 + mova xm6, [lumaq+lstrideq*(1<<%3)+ 0] + mova xm8, [lumaq+lstrideq*(1<<%3)+16] + vinserti128 m6, [lumaq+lstrideq*(1<<%3)+32], 1 + vinserti128 m8, [lumaq+lstrideq*(1<<%3)+48], 1 phaddw m4, m7 phaddw m6, m8 pavgw m4, m2 pavgw m6, m2 +%else + mova m1, [srcq+32] + + ; luma_src + mova m4, [lumaq] + mova m6, [lumaq+32] +%endif %if %1 punpckhwd m3, m4, m0 punpcklwd m4, m0 punpckhwd m5, m6, m1 @@ -1763,24 +1947,63 @@ ; grain = grain_lut[offy+y][offx+x] movu m9, [grain_lutq+offxyq*2] movu m5, [grain_lutq+top_offxyq*2] punpckhwd m7, m5, m9 punpcklwd m5, m9 ; {top/cur interleaved} -%if %1 - REPX {pmaddwd x, [pw_23_22]}, m7, m5 +%if %3 + vpbroadcastd m3, [pw_23_22] +%elif %2 + vpbroadcastd m3, [pw_27_17_17_27] %else - REPX {pmaddwd x, m15}, m7, m5 + vpbroadcastd m3, [r10] %endif - vpbroadcastd m3, [pd_16] - REPX {paddd x, m3}, m7, m5 + REPX {pmaddwd x, m3}, m7, m5 +%if %1 + vpbroadcastd m8, [pd_16] + REPX {paddd x, m8}, m7, m5 +%else + REPX {paddd x, m14}, m7, m5 +%endif REPX {psrad x, 5}, m7, m5 packssdw m9, m5, m7 +%if %2 + movu m3, [grain_lutq+offxyq*2+82*2] +%else + movu m3, [grain_lutq+offxyq*2+32] +%endif +%if %3 == 0 +%if %2 + movu m5, [grain_lutq+top_offxyq*2+82*2] +%else + movu m5, [grain_lutq+top_offxyq*2+32] +%endif + punpckhwd m7, m5, m3 + punpcklwd m5, m3 ; {top/cur interleaved} +%if %2 + vpbroadcastd m3, [pw_27_17_17_27+4] +%else + vpbroadcastd m3, [r10] +%endif + REPX {pmaddwd x, m3}, m7, m5 +%if %1 + REPX {paddd x, m8}, m7, m5 +%else + REPX {paddd x, m14}, m7, m5 +%endif + REPX {psrad x, 5}, m7, m5 + packssdw m3, m5, m7 +%endif ; %3 == 0 pcmpeqw m7, m7 psraw m5, m10, 1 pxor m7, m5 +%if %3 pmaxsw m9, m7 pminsw m9, m5 +%else + REPX {pmaxsw x, m7}, m9, m3 + REPX {pminsw x, m5}, m9, m3 +%endif ; scaling[luma_src] punpckhwd m5, m4, m2 punpcklwd m4, m2 pcmpeqw m7, m7 @@ -1792,22 +2015,21 @@ ; noise = round2(scaling[luma_src] * grain, scaling_shift) pmullw m8, m11 pmulhrsw m9, m8 - ; same for the other half + ; scaling for the other half punpckhwd m7, m6, m2 punpcklwd m6, m2 ; m4-7: luma_src as dword pcmpeqw m8, m8 mova m4, m8 vpgatherdd m5, [scalingq+m6-3], m8 vpgatherdd m6, [scalingq+m7-3], m4 REPX {psrld x, 24}, m5, m6 packssdw m5, m6 ; noise = round2(scaling[luma_src] * grain, scaling_shift) - movu m3, [grain_lutq+offxyq*2+82*2] pmullw m5, m11 pmulhrsw m3, m5 ; dst = clip_pixel(src, noise) paddw m0, m9 @@ -1815,29 +2037,47 @@ pmaxsw m0, m13 pmaxsw m1, m13 pminsw m0, m12 pminsw m1, m12 mova [dstq], m0 +%if %2 mova [dstq+strideq], m1 sub hb, 2 +%else + mova [dstq+32], m1 + dec hb +%endif jle %%end_y_v_overlap +%if %2 lea srcq, [srcq+strideq*2] lea dstq, [dstq+strideq*2] - lea lumaq, [lumaq+lstrideq*4] - add grain_lutq, 82*4 + lea lumaq, [lumaq+lstrideq*(2<<%3)] +%else + add srcq, strideq + add dstq, strideq + add lumaq, lstrideq +%endif + add grain_lutq, 82*(2<<%2) +%if %2 jmp %%loop_y +%else + btc hd, 16 + jc %%loop_y + add r10, 4 + jmp %%loop_y_v_overlap +%endif %%end_y_v_overlap: - add wq, 16 + add wq, 32>>%2 jge %%end_hv mov srcq, r10mp mov dstq, r11mp mov lumaq, r12mp lea srcq, [srcq+wq*2] lea dstq, [dstq+wq*2] - lea lumaq, [lumaq+wq*4] + lea lumaq, [lumaq+wq*(2<<%2)] ; since fg_dataq.overlap is guaranteed to be set, we never jump ; back to .loop_x_v_overlap, and instead always fall-through to ; h+v overlap @@ -1856,19 +2096,23 @@ rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ offx, offy, see, left_offxy, top_offxy, topleft_offxy, luma, lstride - lea topleft_offxyq, [top_offxyq+16] - lea left_offxyq, [offyq+16] +%if %2 == 0 + lea r12, [pw_27_17_17_27] + mov r13mp, r12 +%endif + lea topleft_offxyq, [top_offxyq+(32>>%2)] + lea left_offxyq, [offyq+(32>>%2)] rorx offyd, seed, 8 rorx offxd, seed, 12 and offyd, 0xf000f and offxd, 0xf000f - imul offyd, 82 + imul offyd, 164>>%3 ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy - lea offyq, [offyq+offxq+0x10001*498+16*82] + lea offyq, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ h, offxy, see, left_offxy, top_offxy, topleft_offxy, luma, lstride movzx top_offxyd, offxyw @@ -1877,70 +2121,157 @@ mov hd, hm mov grain_lutq, grain_lutmp %%loop_y_hv_overlap: ; grain = grain_lut[offy+y][offx+x] movd xm5, [grain_lutq+left_offxyq*2] - pinsrw xm5, [grain_lutq+left_offxyq*2+82*2], 1 - pinsrw xm5, [grain_lutq+topleft_offxyq*2], 2 ; { left0, left1, top/left } +%if %2 + pinsrw xm5, [grain_lutq+left_offxyq*2+82*2], 2 +%if %3 + vinserti128 m5, [grain_lutq+topleft_offxyq*2], 1 ; { left0, left1, top/left } +%else + ; insert both top/left lines + movd xm9, [grain_lutq+topleft_offxyq*2+82*2] + pinsrw xm9, [grain_lutq+topleft_offxyq*2], 2 + vinserti128 m5, xm9, 1 +%endif +%else + pinsrd xm5, [grain_lutq+topleft_offxyq*2], 1 +%endif movu m9, [grain_lutq+offxyq*2] +%if %2 movu m3, [grain_lutq+offxyq*2+82*2] +%else + movu m3, [grain_lutq+offxyq*2+32] +%endif movu m8, [grain_lutq+top_offxyq*2] - punpcklwd xm7, xm9, xm3 ; { cur0, cur1 } - punpckldq xm7, xm8 ; { cur0, cur1, top0 } - punpcklwd xm5, xm7 ; { cur/left } interleaved - pmaddwd xm5, [pw_23_22] - vpbroadcastd xm0, [pd_16] +%if %2 + punpckldq xm7, xm9, xm3 ; { cur0, cur1 } +%if %3 + vinserti128 m7, xm8, 1 ; { cur0, cur1, top0 } +%else + ; insert both top lines + movu m1, [grain_lutq+top_offxyq*2+82*2] + punpckldq xm0, xm1, xm8 + vinserti128 m7, xm0, 1 +%endif +%else + movu m1, [grain_lutq+top_offxyq*2+32] + punpckldq xm7, xm9, xm8 +%endif + punpcklwd m5, m7 ; { cur/left } interleaved +%if %2 +%if %1 + vpbroadcastq m0, [pw_23_22] + pmaddwd m5, m0 + vpbroadcastd m0, [pd_16] + paddd m5, m0 +%else + pmaddwd m5, m15 + paddd m5, m14 +%endif + psrad m5, 5 + vextracti128 xm0, m5, 1 + packssdw xm5, xm0 +%else +%if %1 + movddup xm0, [pw_27_17_17_27] + pmaddwd xm5, xm0 + vpbroadcastd m0, [pd_16] paddd xm5, xm0 +%else + pmaddwd xm5, xm15 + paddd xm5, xm14 +%endif psrad xm5, 5 packssdw xm5, xm5 - pcmpeqw xm0, xm0 - psraw xm7, xm10, 1 - pxor xm0, xm7 +%endif + pcmpeqw m0, m0 + psraw m7, m10, 1 + pxor m0, m7 pminsw xm5, xm7 pmaxsw xm5, xm0 - pcmpeqw xm7, xm7 - psrldq xm7, 14 ; 0xffff, 0..... - vpblendvb m9, m5, m7 ; line 0 - psrldq xm5, 2 - vpblendvb m3, m5, m7 ; line 1 - psrldq xm5, 2 - vpblendvb m5, m8, m5, m7 ; top line + vpblendd m9, m9, m5, 00000001b +%if %2 + psrldq xm5, 4 + vpblendd m3, m3, m5, 00000001b +%if %3 == 0 + psrldq xm5, 4 + vpblendd m1, m1, m5, 00000001b +%endif +%endif + psrldq xm5, 4 + vpblendd m5, m8, m5, 00000001b - punpckhwd m7, m5, m9 + punpckhwd m8, m5, m9 punpcklwd m5, m9 ; {top/cur interleaved} +%if %3 + vpbroadcastd m9, [pw_23_22] +%elif %2 + vpbroadcastd m9, [pw_27_17_17_27] +%else + xchg r12, r13mp + vpbroadcastd m9, [r12] +%endif + REPX {pmaddwd x, m9}, m8, m5 %if %1 - REPX {pmaddwd x, [pw_23_22]}, m7, m5 + vpbroadcastd m4, [pd_16] + REPX {paddd x, m4}, m8, m5 %else - REPX {pmaddwd x, m15}, m7, m5 + REPX {paddd x, m14}, m8, m5 %endif - vpbroadcastd m9, [pd_16] - REPX {paddd x, m9}, m5, m7 - REPX {psrad x, 5}, m5, m7 - packssdw m9, m5, m7 - pcmpeqw m5, m5 - psraw m7, m10, 1 - pxor m5, m7 - pmaxsw m9, m5 + REPX {psrad x, 5}, m8, m5 + packssdw m9, m5, m8 +%if %3 pminsw m9, m7 + pmaxsw m9, m0 +%else + punpckhwd m8, m1, m3 + punpcklwd m1, m3 ; {top/cur interleaved} +%if %2 + vpbroadcastd m3, [pw_27_17_17_27+4] +%else + vpbroadcastd m3, [r12] + xchg r12, r13mp +%endif + REPX {pmaddwd x, m3}, m8, m1 +%if %1 + REPX {paddd x, m4}, m8, m1 +%else + REPX {paddd x, m14}, m8, m1 +%endif + REPX {psrad x, 5}, m8, m1 + packssdw m3, m1, m8 + REPX {pminsw x, m7}, m9, m3 + REPX {pmaxsw x, m0}, m9, m3 +%endif ; src mova m0, [srcq] +%if %2 mova m1, [srcq+strideq] +%else + mova m1, [srcq+32] +%endif ; luma_src +%if %2 mova xm4, [lumaq+lstrideq*0+ 0] mova xm7, [lumaq+lstrideq*0+16] vinserti128 m4, [lumaq+lstrideq*0+32], 1 vinserti128 m7, [lumaq+lstrideq*0+48], 1 - mova xm6, [lumaq+lstrideq*2+ 0] - mova xm8, [lumaq+lstrideq*2+16] - vinserti128 m6, [lumaq+lstrideq*2+32], 1 - vinserti128 m8, [lumaq+lstrideq*2+48], 1 + mova xm6, [lumaq+lstrideq*(1<<%3)+ 0] + mova xm8, [lumaq+lstrideq*(1<<%3)+16] + vinserti128 m6, [lumaq+lstrideq*(1<<%3)+32], 1 + vinserti128 m8, [lumaq+lstrideq*(1<<%3)+48], 1 phaddw m4, m7 phaddw m6, m8 pavgw m4, m2 pavgw m6, m2 +%else + mova m4, [lumaq] + mova m6, [lumaq+32] +%endif %if %1 punpckhwd m8, m4, m0 punpcklwd m4, m0 punpckhwd m5, m6, m1 @@ -1990,34 +2321,55 @@ pmaxsw m0, m13 pmaxsw m1, m13 pminsw m0, m12 pminsw m1, m12 mova [dstq], m0 +%if %2 mova [dstq+strideq], m1 lea srcq, [srcq+strideq*2] lea dstq, [dstq+strideq*2] - lea lumaq, [lumaq+lstrideq*4] - add grain_lutq, 82*4 + lea lumaq, [lumaq+lstrideq*(2<<%3)] +%else + mova [dstq+32], m1 + + add srcq, strideq + add dstq, strideq + add lumaq, lstrideq +%endif + add grain_lutq, 82*(2<<%2) +%if %2 sub hb, 2 jg %%loop_y_h_overlap +%else + dec hb + jle %%end_y_hv_overlap + btc hd, 16 + jc %%loop_y_h_overlap + add r13mp, 4 + jmp %%loop_y_hv_overlap +%endif %%end_y_hv_overlap: - add wq, 16 + add wq, 32>>%2 jge %%end_hv mov srcq, r10mp mov dstq, r11mp mov lumaq, r12mp lea srcq, [srcq+wq*2] lea dstq, [dstq+wq*2] - lea lumaq, [lumaq+wq*4] + lea lumaq, [lumaq+wq*(2<<%2)] jmp %%loop_x_hv_overlap %%end_hv: RET %endmacro - FGUV_32x32xN_LOOP 1 + %%FGUV_32x32xN_LOOP 1, %2, %3 .csfl: - FGUV_32x32xN_LOOP 0 + %%FGUV_32x32xN_LOOP 0, %2, %3 +%endmacro +FGUV_FN 420, 1, 1 +FGUV_FN 422, 1, 0 +FGUV_FN 444, 0, 0 %endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/film_grain16_sse.asm b/third_party/dav1d/src/x86/film_grain16_sse.asm new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/x86/film_grain16_sse.asm @@ -0,0 +1,3450 @@ +; Copyright © 2021, VideoLAN and dav1d authors +; Copyright © 2021, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +SECTION_RODATA 16 +pd_16: times 4 dd 16 +pw_1: times 8 dw 1 +pw_16384: times 8 dw 16384 +pw_8192: times 8 dw 8192 +pw_23_22: dw 23, 22 + times 3 dw 0, 32 +pb_mask: db 0, 0x80, 0x80, 0, 0x80, 0, 0, 0x80, 0x80, 0, 0, 0x80, 0, 0x80, 0x80, 0 +pw_27_17_17_27: dw 27, 17, 17, 27 + times 2 dw 0, 32 +rnd_next_upperbit_mask: dw 0x100B, 0x2016, 0x402C, 0x8058 +pw_seed_xor: times 2 dw 0xb524 + times 2 dw 0x49d8 +pb_1: times 4 db 1 +hmul_bits: dw 32768, 16384, 8192, 4096 +round: dw 2048, 1024, 512 +mul_bits: dw 256, 128, 64, 32, 16 +round_vals: dw 32, 64, 128, 256, 512, 1024 +max: dw 256*4-1, 240*4, 235*4, 256*16-1, 240*16, 235*16 +min: dw 0, 16*4, 16*16 +; these two should be next to each other +pw_4: times 2 dw 4 +pw_16: times 2 dw 16 + +%macro JMP_TABLE 1-* + %xdefine %1_table %%table + %xdefine %%base %1_table + %xdefine %%prefix mangle(private_prefix %+ _%1) + %%table: + %rep %0 - 1 + dd %%prefix %+ .ar%2 - %%base + %rotate 1 + %endrep +%endmacro + +JMP_TABLE generate_grain_y_16bpc_ssse3, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_420_16bpc_ssse3, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_422_16bpc_ssse3, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_444_16bpc_ssse3, 0, 1, 2, 3 + +struc FGData + .seed: resd 1 + .num_y_points: resd 1 + .y_points: resb 14 * 2 + .chroma_scaling_from_luma: resd 1 + .num_uv_points: resd 2 + .uv_points: resb 2 * 10 * 2 + .scaling_shift: resd 1 + .ar_coeff_lag: resd 1 + .ar_coeffs_y: resb 24 + .ar_coeffs_uv: resb 2 * 28 ; includes padding + .ar_coeff_shift: resq 1 + .grain_scale_shift: resd 1 + .uv_mult: resd 2 + .uv_luma_mult: resd 2 + .uv_offset: resd 2 + .overlap_flag: resd 1 + .clip_to_restricted_range: resd 1 +endstruc + +cextern gaussian_sequence + +SECTION .text + +%macro REPX 2-* + %xdefine %%f(x) %1 +%rep %0 - 1 + %rotate 1 + %%f(%1) +%endrep +%endmacro + +%if ARCH_X86_32 +%undef base +%define PIC_ptr(a) base+a +%else +%define PIC_ptr(a) a +%endif + +%define m(x) mangle(private_prefix %+ _ %+ x %+ SUFFIX) + +%macro vpgatherdw 5-8 8, 1 ; dst, src, base, tmp_gpr[x2], cnt, stride, tmp_xmm_reg +%assign %%idx 0 +%define %%tmp %2 +%if %0 == 8 +%define %%tmp %8 +%endif +%rep (%6/2) +%if %%idx == 0 + movd %5 %+ d, %2 + pshuflw %%tmp, %2, q3232 +%else + movd %5 %+ d, %%tmp +%if %6 == 8 +%if %%idx == 2 + punpckhqdq %%tmp, %%tmp +%elif %%idx == 4 + psrlq %%tmp, 32 +%endif +%endif +%endif + movzx %4 %+ d, %5 %+ w + shr %5 %+ d, 16 + +%if %%idx == 0 + movd %1, [%3+%4*%7] +%else + pinsrw %1, [%3+%4*%7], %%idx + 0 +%endif + pinsrw %1, [%3+%5*%7], %%idx + 1 +%assign %%idx %%idx+2 +%endrep +%endmacro + +%macro SPLATD 2 ; dst, src +%ifnidn %1, %2 + movd %1, %2 +%endif + pshufd %1, %1, q0000 +%endmacro + +%macro SPLATW 2 ; dst, src +%ifnidn %1, %2 + movd %1, %2 +%endif + pshuflw %1, %1, q0000 + punpcklqdq %1, %1 +%endmacro + + +INIT_XMM ssse3 +%if ARCH_X86_64 +cglobal generate_grain_y_16bpc, 3, 8, 16, buf, fg_data, bdmax + lea r4, [pb_mask] +%define base r4-pb_mask +%else +cglobal generate_grain_y_16bpc, 3, 6, 8, buf, fg_data, bdmax + LEA r4, $$ +%define base r4-$$ +%endif + movq m1, [base+rnd_next_upperbit_mask] + movq m4, [base+mul_bits] + movq m7, [base+hmul_bits] + mov r3d, [fg_dataq+FGData.grain_scale_shift] + lea r5d, [bdmaxq+1] + shr r5d, 11 ; 0 for 10bpc, 2 for 12bpc + sub r3, r5 + SPLATW m6, [base+round+r3*2-2] + mova m5, [base+pb_mask] + SPLATW m0, [fg_dataq+FGData.seed] + mov r3, -73*82*2 + sub bufq, r3 +%if ARCH_X86_64 + lea r6, [gaussian_sequence] +%endif +.loop: + pand m2, m0, m1 + psrlw m3, m2, 10 + por m2, m3 ; bits 0xf, 0x1e, 0x3c and 0x78 are set + pmullw m2, m4 ; bits 0x0f00 are set + pshufb m3, m5, m2 ; set 15th bit for next 4 seeds + psllq m2, m3, 30 + por m2, m3 + psllq m3, m2, 15 + por m2, m3 ; aggregate each bit into next seed's high bit + pmulhuw m3, m0, m7 + por m2, m3 ; 4 next output seeds + pshuflw m0, m2, q3333 + psrlw m2, 5 +%if ARCH_X86_64 + vpgatherdw m3, m2, r6, r5, r7, 4, 2 +%else + vpgatherdw m3, m2, base+gaussian_sequence, r5, r2, 4, 2 +%endif + paddw m3, m3 ; otherwise bpc=12 w/ grain_scale_shift=0 + ; shifts by 0, which pmulhrsw does not support + pmulhrsw m3, m6 + movq [bufq+r3], m3 + add r3, 4*2 + jl .loop + + ; auto-regression code + movsxd r3, [fg_dataq+FGData.ar_coeff_lag] + movsxd r3, [base+generate_grain_y_16bpc_ssse3_table+r3*4] + lea r3, [r3+base+generate_grain_y_16bpc_ssse3_table] + jmp r3 + +.ar1: +%if WIN64 + DEFINE_ARGS shift, fg_data, max, buf, val3, min, cf3, x, val0 + lea bufq, [r0-2*(82*73-(82*3+79))] + PUSH r8 +%else +%if ARCH_X86_64 + DEFINE_ARGS buf, fg_data, max, shift, val3, min, cf3, x, val0 +%else ; x86-32 + DEFINE_ARGS buf, fg_data, min, val3, x, cf3, val0 + PUSH r6 +%define shiftd r1d +%endif + sub bufq, 2*(82*73-(82*3+79)) +%endif + movsx cf3d, byte [fg_dataq+FGData.ar_coeffs_y+3] + movd m4, [fg_dataq+FGData.ar_coeffs_y] + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] +%if WIN64 + DEFINE_ARGS shift, h, max, buf, val3, min, cf3, x, val0 +%elif ARCH_X86_64 + DEFINE_ARGS buf, h, max, shift, val3, min, cf3, x, val0 +%else ; x86-32 +%undef shiftd + DEFINE_ARGS buf, shift, min, val3, x, cf3, val0 +%define hd dword r0m +%define maxd dword minm +%endif +%if cpuflag(sse4) + pmovsxbw m4, m4 +%else + pxor m3, m3 + pcmpgtb m3, m4 + punpcklbw m4, m3 +%endif + pinsrw m4, [base+pw_1], 3 + pshufd m5, m4, q1111 + pshufd m4, m4, q0000 + SPLATW m3, [base+round_vals+shiftq*2-12] ; rnd + mov hd, 70 + sar maxd, 1 + mov mind, maxd + xor mind, -1 +.y_loop_ar1: + mov xq, -76 + movsx val3d, word [bufq+xq*2-2] +.x_loop_ar1: + movu m0, [bufq+xq*2-82*2-2] ; top/left + psrldq m2, m0, 2 ; top + psrldq m1, m0, 4 ; top/right + punpcklwd m0, m2 + punpcklwd m1, m3 + pmaddwd m0, m4 + pmaddwd m1, m5 + paddd m0, m1 +.x_loop_ar1_inner: + movd val0d, m0 + psrldq m0, 4 + imul val3d, cf3d + add val3d, val0d + sar val3d, shiftb + movsx val0d, word [bufq+xq*2] + add val3d, val0d + cmp val3d, maxd + cmovg val3d, maxd + cmp val3d, mind + cmovl val3d, mind + mov word [bufq+xq*2], val3w + ; keep val3d in-place as left for next x iteration + inc xq + jz .x_loop_ar1_end + test xq, 3 + jnz .x_loop_ar1_inner + jmp .x_loop_ar1 + +.x_loop_ar1_end: + add bufq, 82*2 + dec hd + jg .y_loop_ar1 +%if WIN64 + POP r8 +%elif ARCH_X86_32 + POP r6 +%undef maxd +%undef hd +%endif +.ar0: + RET + +.ar2: +%if ARCH_X86_32 +%assign stack_offset_old stack_offset + ALLOC_STACK -16*8 +%endif + DEFINE_ARGS buf, fg_data, bdmax, shift + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + movd m0, [base+round_vals-12+shiftq*2] + pshuflw m0, m0, q0000 + movu m6, [fg_dataq+FGData.ar_coeffs_y+0] ; cf0-11 + pxor m2, m2 + punpcklwd m0, m2 + pcmpgtb m2, m6 + punpckhbw m3, m6, m2 + punpcklbw m6, m2 + pshufd m2, m6, q3333 + pshufd m1, m6, q2222 + pshufd m7, m6, q1111 + pshufd m6, m6, q0000 + pshufd m4, m3, q1111 + pshufd m3, m3, q0000 +%if ARCH_X86_64 + SWAP 0, 12 + SWAP 1, 8 + SWAP 2, 9 + SWAP 3, 10 + SWAP 4, 11 +%else +%define m12 [rsp+0*16] +%define m8 [rsp+1*16] +%define m9 [rsp+2*16] +%define m10 [rsp+3*16] +%define m11 [rsp+4*16] + mova m12, m0 + mova m8, m1 + mova m9, m2 + mova m10, m3 + mova m11, m4 + mov bdmaxd, bdmaxm +%endif + sar bdmaxd, 1 + SPLATW m0, bdmaxd ; max_grain + pcmpeqw m1, m1 +%if !cpuflag(sse4) + pcmpeqw m2, m2 + psrldq m2, 14 + pslldq m2, 2 + pxor m2, m1 +%endif + pxor m1, m0 ; min_grain +%if ARCH_X86_64 + SWAP 0, 13 + SWAP 1, 14 + SWAP 2, 15 +%else +%define m13 [rsp+5*16] +%define m14 [rsp+6*16] + mova m13, m0 + mova m14, m1 +%if !cpuflag(sse4) +%define m15 [rsp+7*16] + mova m15, m2 +%endif +%endif + sub bufq, 2*(82*73-(82*3+79)) + DEFINE_ARGS buf, fg_data, h, x + mov hd, 70 +.y_loop_ar2: + mov xq, -76 + +.x_loop_ar2: + movu m0, [bufq+xq*2-82*4-4] ; y=-2,x=[-2,+5] + movu m1, [bufq+xq*2-82*2-4] ; y=-1,x=[-2,+5] + psrldq m2, m0, 2 + psrldq m3, m0, 4 + psrldq m4, m0, 6 + psrldq m5, m0, 8 + punpcklwd m0, m2 + punpcklwd m3, m4 + punpcklwd m5, m1 + psrldq m2, m1, 2 + psrldq m4, m1, 4 + punpcklwd m2, m4 + psrldq m4, m1, 6 + psrldq m1, 8 + punpcklwd m4, m1 + pmaddwd m0, m6 + pmaddwd m3, m7 + pmaddwd m5, m8 + pmaddwd m2, m9 + pmaddwd m4, m10 + paddd m0, m3 + paddd m5, m2 + paddd m0, m4 + paddd m0, m5 ; accumulated top 2 rows + paddd m0, m12 + + movu m1, [bufq+xq*2-4] ; y=0,x=[-2,+5] + pshufd m4, m1, q3321 + pxor m2, m2 + pcmpgtw m2, m4 + punpcklwd m4, m2 ; in dwords, y=0,x=[0,3] +.x_loop_ar2_inner: + pmaddwd m2, m1, m11 + paddd m2, m0 + psrldq m0, 4 ; shift top to next pixel + psrad m2, [fg_dataq+FGData.ar_coeff_shift] + paddd m2, m4 + packssdw m2, m2 + pminsw m2, m13 + pmaxsw m2, m14 + psrldq m4, 4 + pslldq m2, 2 + psrldq m1, 2 +%if cpuflag(sse4) + pblendw m1, m2, 00000010b +%else + pand m1, m15 + pandn m3, m15, m2 + por m1, m3 +%endif + ; overwrite previous pixel, this should be ok + movd [bufq+xq*2-2], m1 + inc xq + jz .x_loop_ar2_end + test xq, 3 + jnz .x_loop_ar2_inner + jmp .x_loop_ar2 + +.x_loop_ar2_end: + add bufq, 82*2 + dec hd + jg .y_loop_ar2 +%if ARCH_X86_32 +%undef m8 +%undef m9 +%undef m10 +%undef m11 +%undef m12 +%undef m13 +%undef m14 +%undef m15 +%endif + RET + +.ar3: + DEFINE_ARGS buf, fg_data, bdmax, shift +%if WIN64 + mov r6, rsp + and rsp, ~15 + sub rsp, 64 + %define tmp rsp +%elif ARCH_X86_64 + %define tmp rsp+stack_offset-72 +%else +%assign stack_offset stack_offset_old + ALLOC_STACK -16*12 + %define tmp rsp + mov bdmaxd, bdmaxm +%endif + sar bdmaxd, 1 + SPLATW m7, bdmaxd ; max_grain + pcmpeqw m6, m6 +%if !cpuflag(sse4) + pcmpeqw m4, m4 + psrldq m4, 14 + pslldq m4, 4 + pxor m4, m6 +%endif + pxor m6, m7 ; min_grain + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + +%if ARCH_X86_64 + SWAP 6, 14 + SWAP 7, 15 +%else +%define m14 [rsp+10*16] +%define m15 [esp+11*16] + mova m14, m6 + mova m15, m7 +%endif + + ; build cf0-1 until 18-19 in m5-12 and r0/1 + pxor m1, m1 + movu m0, [fg_dataq+FGData.ar_coeffs_y+ 0] ; cf0-15 + pcmpgtb m1, m0 + punpckhbw m2, m0, m1 + punpcklbw m0, m1 + +%if cpuflag(sse4) + pshufd m4, m2, q3333 +%else + pshufd m5, m2, q3333 + mova [tmp+48], m5 +%endif + pshufd m3, m2, q2222 + pshufd m1, m2, q0000 + pshufd m2, m2, q1111 + pshufd m7, m0, q2222 + pshufd m6, m0, q1111 + pshufd m5, m0, q0000 + pshufd m0, m0, q3333 + +%if ARCH_X86_64 + SWAP 0, 8 + SWAP 1, 9 + SWAP 2, 10 + SWAP 3, 11 + SWAP 4, 12 +%else +%define m8 [rsp+4*16] +%define m9 [esp+5*16] +%define m10 [rsp+6*16] +%define m11 [esp+7*16] +%define m12 [rsp+8*16] + mova m8, m0 + mova m9, m1 + mova m10, m2 + mova m11, m3 + mova m12, m4 +%endif + + ; build cf20,round in r2 + ; build cf21-23,round*2 in m13 + pxor m1, m1 + movq m0, [fg_dataq+FGData.ar_coeffs_y+16] ; cf16-23 + pcmpgtb m1, m0 + punpcklbw m0, m1 + pshufd m1, m0, q0000 + pshufd m2, m0, q1111 + mova [tmp+ 0], m1 + mova [tmp+16], m2 + psrldq m3, m0, 10 + pinsrw m3, [base+round_vals+shiftq*2-10], 3 + +%if ARCH_X86_64 + SWAP 3, 13 +%else +%define m13 [esp+9*16] + mova m13, m3 +%endif + + pinsrw m0, [base+round_vals+shiftq*2-12], 5 + pshufd m3, m0, q2222 + mova [tmp+32], m3 + + DEFINE_ARGS buf, fg_data, h, x + sub bufq, 2*(82*73-(82*3+79)) + mov hd, 70 +.y_loop_ar3: + mov xq, -76 + +.x_loop_ar3: + movu m0, [bufq+xq*2-82*6-6+ 0] ; y=-3,x=[-3,+4] + movd m1, [bufq+xq*2-82*6-6+16] ; y=-3,x=[+5,+6] + palignr m2, m1, m0, 2 ; y=-3,x=[-2,+5] + palignr m1, m1, m0, 12 ; y=-3,x=[+3,+6] + punpckhwd m3, m0, m2 ; y=-3,x=[+1/+2,+2/+3,+3/+4,+4/+5] + punpcklwd m0, m2 ; y=-3,x=[-3/-2,-2/-1,-1/+0,+0/+1] + shufps m2, m0, m3, q1032 ; y=-3,x=[-1/+0,+0/+1,+1/+2,+2/+3] + + pmaddwd m0, m5 + pmaddwd m2, m6 + pmaddwd m3, m7 + paddd m0, m2 + paddd m0, m3 + ; m0 = top line first 6 multiplied by cf, m1 = top line last entry + + movu m2, [bufq+xq*2-82*4-6+ 0] ; y=-2,x=[-3,+4] + movd m3, [bufq+xq*2-82*4-6+16] ; y=-2,x=[+5,+6] + punpcklwd m1, m2 ; y=-3/-2,x=[+3/-3,+4/-2,+5/-1,+6/+0] + palignr m4, m3, m2, 2 ; y=-3,x=[-2,+5] + palignr m3, m3, m2, 4 ; y=-3,x=[-1,+6] + punpckhwd m2, m4, m3 ; y=-2,x=[+2/+3,+3/+4,+4/+5,+5/+6] + punpcklwd m4, m3 ; y=-2,x=[-2/-1,-1/+0,+0/+1,+1/+2] + shufps m3, m4, m2, q1032 ; y=-2,x=[+0/+1,+1/+2,+2/+3,+3/+4] + + pmaddwd m1, m8 + pmaddwd m4, m9 + pmaddwd m3, m10 + pmaddwd m2, m11 + paddd m1, m4 + paddd m3, m2 + paddd m0, m1 + paddd m0, m3 + ; m0 = top 2 lines multiplied by cf + + movu m1, [bufq+xq*2-82*2-6+ 0] ; y=-1,x=[-3,+4] + movd m2, [bufq+xq*2-82*2-6+16] ; y=-1,x=[+5,+6] + palignr m3, m2, m1, 2 ; y=-1,x=[-2,+5] + palignr m2, m2, m1, 12 ; y=-1,x=[+3,+6] + punpckhwd m4, m1, m3 ; y=-1,x=[+1/+2,+2/+3,+3/+4,+4/+5] + punpcklwd m1, m3 ; y=-1,x=[-3/-2,-2/-1,-1/+0,+0/+1] + shufps m3, m1, m4, q1032 ; y=-1,x=[-1/+0,+0/+1,+1/+2,+2/+3] + punpcklwd m2, [base+pw_1] + +%if cpuflag(sse4) + pmaddwd m1, m12 +%else + pmaddwd m1, [tmp+48] +%endif + pmaddwd m3, [tmp+ 0] + pmaddwd m4, [tmp+16] + pmaddwd m2, [tmp+32] + paddd m1, m3 + paddd m4, m2 + paddd m0, m1 + paddd m0, m4 + ; m0 = top 3 lines multiplied by cf plus rounding for downshift + + movu m1, [bufq+xq*2-6] ; y=0,x=[-3,+4] +.x_loop_ar3_inner: + pmaddwd m2, m1, m13 + pshufd m3, m2, q1111 + paddd m2, m3 ; left+cur + paddd m2, m0 ; add top + psrldq m0, 4 + psrad m2, [fg_dataq+FGData.ar_coeff_shift] + packssdw m2, m2 + pminsw m2, m15 + pmaxsw m2, m14 + pslldq m2, 4 + psrldq m1, 2 +%if cpuflag(sse4) + pblendw m1, m2, 00000100b +%else + pand m1, m12 + pandn m3, m12, m2 + por m1, m3 +%endif + ; overwrite a couple of pixels, should be ok + movq [bufq+xq*2-4], m1 + inc xq + jz .x_loop_ar3_end + test xq, 3 + jnz .x_loop_ar3_inner + jmp .x_loop_ar3 + +.x_loop_ar3_end: + add bufq, 82*2 + dec hd + jg .y_loop_ar3 +%if WIN64 + mov rsp, r6 +%elif ARCH_X86_32 +%undef m8 +%undef m9 +%undef m10 +%undef m11 +%undef m12 +%undef m13 +%undef m14 +%undef m15 +%endif + RET + +%macro generate_grain_uv_fn 3 ; ss_name, ss_x, ss_y +INIT_XMM ssse3 +%if ARCH_X86_64 +cglobal generate_grain_uv_%1_16bpc, 4, 11, 16, buf, bufy, fg_data, uv, bdmax, x, gaussian_reg, h, pic_reg +%define base r8-pb_mask + lea r8, [pb_mask] + movifnidn bdmaxd, bdmaxm + lea r6d, [bdmaxq+1] +%else +cglobal generate_grain_uv_%1_16bpc, 1, 7, 8, buf, x, pic_reg, fg_data, h +%define base r2-$$ + LEA r2, $$ + mov fg_dataq, r2m + mov r6d, r4m + inc r6d +%endif + movq m1, [base+rnd_next_upperbit_mask] + movq m4, [base+mul_bits] + movq m7, [base+hmul_bits] + mov r5d, [fg_dataq+FGData.grain_scale_shift] + shr r6d, 11 ; 0 for 10bpc, 2 for 12bpc + sub r5, r6 + SPLATW m6, [base+round+r5*2-2] + mova m5, [base+pb_mask] + SPLATW m0, [fg_dataq+FGData.seed] +%if ARCH_X86_64 + SPLATW m2, [base+pw_seed_xor+uvq*4] +%else + mov r5d, r3m + SPLATW m2, [base+pw_seed_xor+r5*4] +%endif + pxor m0, m2 +%if ARCH_X86_64 + lea r6, [gaussian_sequence] +%endif +%if %2 + mov hd, 73-35*%3 + add bufq, 44*2 +.loop_y: + mov xq, -44 +%else + mov xq, -82*73 + add bufq, 82*73*2 +%endif +.loop_x: + pand m2, m0, m1 + psrlw m3, m2, 10 + por m2, m3 ; bits 0xf, 0x1e, 0x3c and 0x78 are set + pmullw m2, m4 ; bits 0x0f00 are set + pshufb m3, m5, m2 ; set 15th bit for next 4 seeds + psllq m2, m3, 30 + por m2, m3 + psllq m3, m2, 15 + por m2, m3 ; aggregate each bit into next seed's high bit + pmulhuw m3, m0, m7 + por m2, m3 ; 4 next output seeds + pshuflw m0, m2, q3333 + psrlw m2, 5 +%if ARCH_X86_64 + vpgatherdw m3, m2, r6, r9, r10, 4, 2 +%else + vpgatherdw m3, m2, base+gaussian_sequence, r5, r6, 4, 2 +%endif + paddw m3, m3 ; otherwise bpc=12 w/ grain_scale_shift=0 + ; shifts by 0, which pmulhrsw does not support + pmulhrsw m3, m6 + movq [bufq+xq*2], m3 + add xq, 4 + jl .loop_x +%if %2 + add bufq, 82*2 + dec hd + jg .loop_y +%endif + + ; auto-regression code + movsxd r5, [fg_dataq+FGData.ar_coeff_lag] + movsxd r5, [base+generate_grain_uv_%1_16bpc_ssse3_table+r5*4] + lea r5, [r5+base+generate_grain_uv_%1_16bpc_ssse3_table] + jmp r5 + +.ar0: +%if ARCH_X86_64 + DEFINE_ARGS buf, bufy, fg_data, uv, bdmax, shift +%else + DEFINE_ARGS buf, bufy, pic_reg, fg_data, uv, shift +%assign stack_offset_old stack_offset + ALLOC_STACK -16*2 + mov bufyq, r1m + mov uvd, r3m +%endif + imul uvd, 28 + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + movd m4, [fg_dataq+FGData.ar_coeffs_uv+uvq] + SPLATW m3, [base+hmul_bits+shiftq*2-10] +%if ARCH_X86_64 + sar bdmaxd, 1 + SPLATW m1, bdmaxd ; max_gain +%else + SPLATW m1, r4m + psraw m1, 1 +%endif + pcmpeqw m7, m7 + pxor m7, m1 ; min_grain +%if ARCH_X86_64 + SWAP 1, 14 + DEFINE_ARGS buf, bufy, h, x +%else +%define m14 [rsp+0*16] + mova m14, m1 + DEFINE_ARGS buf, bufy, pic_reg, h, x +%endif + pxor m5, m5 + pcmpgtb m5, m4 + punpcklbw m4, m5 +%if %2 + SPLATW m6, [base+hmul_bits+2+%3*2] +%endif + SPLATW m4, m4 + pxor m5, m5 +%if %2 +%if !cpuflag(sse4) + pcmpeqw m2, m2 + pslldq m2, 12 +%if ARCH_X86_64 + SWAP 2, 12 +%else +%define m12 [rsp+1*16] + mova m12, m2 +%endif +%endif +%endif +%if %2 + sub bufq, 2*(82*(73-35*%3)+82-(82*3+41)) +%else + sub bufq, 2*(82*70-3) +%endif + add bufyq, 2*(3+82*3) + mov hd, 70-35*%3 +.y_loop_ar0: + ; first 32 pixels + xor xd, xd +.x_loop_ar0: + movu m0, [bufyq+xq*(2<<%2)] +%if %2 +%if %3 + movu m2, [bufyq+xq*4+82*2] + paddw m0, m2 +%endif + movu m1, [bufyq+xq*4 +16] +%if %3 + movu m2, [bufyq+xq*4+82*2+16] + paddw m1, m2 +%endif + phaddw m0, m1 + pmulhrsw m0, m6 +%endif + punpckhwd m1, m0, m5 + punpcklwd m0, m5 + REPX {pmaddwd x, m4}, m0, m1 + REPX {psrad x, 5}, m0, m1 + packssdw m0, m1 + pmulhrsw m0, m3 + movu m1, [bufq+xq*2] + paddw m0, m1 + pminsw m0, m14 + pmaxsw m0, m7 + cmp xd, 72-40*%2 + je .end + movu [bufq+xq*2], m0 + add xd, 8 + jmp .x_loop_ar0 + + ; last 6/4 pixels +.end: +%if %2 +%if cpuflag(sse4) + pblendw m0, m1, 11000000b +%else + pand m1, m12 + pandn m2, m12, m0 + por m0, m1, m2 +%endif + movu [bufq+xq*2], m0 +%else + movq [bufq+xq*2], m0 +%endif + + add bufq, 82*2 + add bufyq, 82*(2<<%3) + dec hd + jg .y_loop_ar0 +%if ARCH_X86_32 +%undef m12 +%undef m14 +%endif + RET + +.ar1: +%if ARCH_X86_64 + DEFINE_ARGS buf, bufy, fg_data, uv, max, cf3, min, val3, x +%else +%assign stack_offset stack_offset_old +%xdefine rstk rsp +%assign stack_size_padded 0 + DEFINE_ARGS buf, shift, pic_reg, fg_data, uv, bufy, cf3 + mov bufyq, r1m + mov uvd, r3m +%endif + imul uvd, 28 + movsx cf3d, byte [fg_dataq+FGData.ar_coeffs_uv+uvq+3] + movq m4, [fg_dataq+FGData.ar_coeffs_uv+uvq] +%if WIN64 + DEFINE_ARGS shift, bufy, h, buf, max, cf3, min, val3, x, val0 +%if %2 + lea bufq, [r0-2*(82*(73-35*%3)+44-(82*3+41))] +%else + lea bufq, [r0-2*(82*69+3)] +%endif +%else +%if ARCH_X86_64 + DEFINE_ARGS buf, bufy, h, shift, max, cf3, min, val3, x, val0 +%else + DEFINE_ARGS buf, shift, pic_reg, fg_data, val0, bufy, cf3 +%define hd dword r1m +%define mind dword r3m +%define maxd dword r4m +%endif +%if %2 + sub bufq, 2*(82*(73-35*%3)+44-(82*3+41)) +%else + sub bufq, 2*(82*69+3) +%endif +%endif +%if ARCH_X86_64 + mov shiftd, [r2+FGData.ar_coeff_shift] +%else + mov shiftd, [r3+FGData.ar_coeff_shift] +%endif + pxor m5, m5 + pcmpgtb m5, m4 + punpcklbw m4, m5 ; cf0-4 in words + pshuflw m4, m4, q2100 + psrldq m4, 2 ; cf0-3,4 in words + pshufd m5, m4, q1111 + pshufd m4, m4, q0000 + movd m3, [base+round_vals+shiftq*2-12] ; rnd + pxor m6, m6 + punpcklwd m3, m6 +%if %2 + SPLATW m6, [base+hmul_bits+2+%3*2] +%endif + SPLATD m3, m3 + add bufyq, 2*(79+82*3) + mov hd, 70-35*%3 + sar maxd, 1 +%if ARCH_X86_64 + mov mind, maxd + xor mind, -1 +%else + DEFINE_ARGS buf, shift, val3, x, val0, bufy, cf3 + mov r2, maxd + xor r2, -1 + mov mind, r2 +%endif +.y_loop_ar1: + mov xq, -(76>>%2) + movsx val3d, word [bufq+xq*2-2] +.x_loop_ar1: + movu m0, [bufq+xq*2-82*2-2] ; top/left +%if %2 + movu m7, [bufyq+xq*4] +%if %3 + movu m1, [bufyq+xq*4+82*2] + phaddw m7, m1 +%else + phaddw m7, m7 +%endif +%else + movq m7, [bufyq+xq*2] +%endif + psrldq m2, m0, 2 ; top + psrldq m1, m0, 4 ; top/right + punpcklwd m0, m2 +%if %2 +%if %3 + pshufd m2, m7, q3232 + paddw m7, m2 +%endif + pmulhrsw m7, m6 +%endif + punpcklwd m1, m7 + pmaddwd m0, m4 + pmaddwd m1, m5 + paddd m0, m1 + paddd m0, m3 +.x_loop_ar1_inner: + movd val0d, m0 + psrldq m0, 4 + imul val3d, cf3d + add val3d, val0d + sar val3d, shiftb + movsx val0d, word [bufq+xq*2] + add val3d, val0d + cmp val3d, maxd + cmovg val3d, maxd + cmp val3d, mind + cmovl val3d, mind + mov word [bufq+xq*2], val3w + ; keep val3d in-place as left for next x iteration + inc xq + jz .x_loop_ar1_end + test xq, 3 + jnz .x_loop_ar1_inner + jmp .x_loop_ar1 + +.x_loop_ar1_end: + add bufq, 82*2 + add bufyq, 82*2<<%3 + dec hd + jg .y_loop_ar1 +%if ARCH_X86_32 +%undef maxd +%undef mind +%undef hd +%endif + RET + +.ar2: +%if ARCH_X86_64 + DEFINE_ARGS buf, bufy, fg_data, uv, bdmax, shift +%else + DEFINE_ARGS buf, bufy, pic_reg, fg_data, uv, shift + ALLOC_STACK -16*8 + mov bufyq, r1m + mov uvd, r3m +%endif + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + imul uvd, 28 +%if ARCH_X86_64 + sar bdmaxd, 1 + SPLATW m5, bdmaxd ; max_grain +%else + SPLATW m5, r4m + psraw m5, 1 +%endif + pcmpeqw m6, m6 +%if !cpuflag(sse4) + pcmpeqw m7, m7 + psrldq m7, 14 + pslldq m7, 2 + pxor m7, m6 +%endif + pxor m6, m5 ; min_grain +%if %2 && cpuflag(sse4) + SPLATW m7, [base+hmul_bits+2+%3*2] +%endif + +%if ARCH_X86_64 + SWAP 5, 13 + SWAP 6, 14 + SWAP 7, 15 +%else +%define m13 [rsp+5*16] +%define m14 [rsp+6*16] +%define m15 [rsp+7*16] + mova m13, m5 + mova m14, m6 + mova m15, m7 +%endif + + ; coef values + movu m0, [fg_dataq+FGData.ar_coeffs_uv+uvq+0] + pxor m1, m1 + pcmpgtb m1, m0 + punpckhbw m2, m0, m1 + punpcklbw m0, m1 + pinsrw m2, [base+round_vals-12+shiftq*2], 5 + + pshufd m6, m0, q0000 + pshufd m7, m0, q1111 + pshufd m1, m0, q3333 + pshufd m0, m0, q2222 + pshufd m3, m2, q1111 + pshufd m4, m2, q2222 + pshufd m2, m2, q0000 + +%if ARCH_X86_64 + SWAP 0, 8 + SWAP 1, 9 + SWAP 2, 10 + SWAP 3, 11 + SWAP 4, 12 +%else +%define m8 [rsp+0*16] +%define m9 [rsp+1*16] +%define m10 [rsp+2*16] +%define m11 [rsp+3*16] +%define m12 [rsp+4*16] + mova m8, m0 + mova m9, m1 + mova m10, m2 + mova m11, m3 + mova m12, m4 +%endif + +%if ARCH_X86_64 + DEFINE_ARGS buf, bufy, fg_data, h, x +%else + DEFINE_ARGS buf, bufy, pic_reg, fg_data, h, x +%endif +%if %2 + sub bufq, 2*(82*(73-35*%3)+44-(82*3+41)) +%else + sub bufq, 2*(82*69+3) +%endif + add bufyq, 2*(79+82*3) + mov hd, 70-35*%3 +.y_loop_ar2: + mov xq, -(76>>%2) + +.x_loop_ar2: + movu m0, [bufq+xq*2-82*4-4] ; y=-2,x=[-2,+5] + movu m5, [bufq+xq*2-82*2-4] ; y=-1,x=[-2,+5] + psrldq m4, m0, 2 ; y=-2,x=[-1,+5] + psrldq m1, m0, 4 ; y=-2,x=[-0,+5] + psrldq m3, m0, 6 ; y=-2,x=[+1,+5] + psrldq m2, m0, 8 ; y=-2,x=[+2,+5] + punpcklwd m0, m4 ; y=-2,x=[-2/-1,-1/+0,+0/+1,+1/+2] + punpcklwd m1, m3 ; y=-2,x=[+0/+1,+1/+2,+2/+3,+3/+4] + punpcklwd m2, m5 ; y=-2/-1,x=[+2/-2,+3/-1,+4/+0,+5/+1] + pmaddwd m0, m6 + pmaddwd m1, m7 + pmaddwd m2, m8 + paddd m0, m1 + paddd m0, m2 + psrldq m3, m5, 2 ; y=-1,x=[-1,+5] + psrldq m1, m5, 4 ; y=-1,x=[-0,+5] + psrldq m4, m5, 6 ; y=-1,x=[+1,+5] + psrldq m2, m5, 8 ; y=-1,x=[+2,+5] + punpcklwd m3, m1 + punpcklwd m4, m2 + pmaddwd m3, m9 + pmaddwd m4, m10 + paddd m3, m4 + paddd m0, m3 + + ; luma component & rounding +%if %2 + movu m1, [bufyq+xq*4] +%if %3 + movu m2, [bufyq+xq*4+82*2] + phaddw m1, m2 + pshufd m2, m1, q3232 + paddw m1, m2 +%else + phaddw m1, m1 +%endif +%if cpuflag(sse4) + pmulhrsw m1, m15 +%elif %3 + pmulhrsw m1, [base+pw_8192] +%else + pmulhrsw m1, [base+pw_16384] +%endif +%else + movq m1, [bufyq+xq*2] +%endif + punpcklwd m1, [base+pw_1] + pmaddwd m1, m12 + paddd m0, m1 + + movu m1, [bufq+xq*2-4] ; y=0,x=[-2,+5] + pshufd m2, m1, q3321 + pxor m3, m3 + pcmpgtw m3, m2 + punpcklwd m2, m3 ; y=0,x=[0,3] in dword +.x_loop_ar2_inner: + pmaddwd m3, m1, m11 + paddd m3, m0 + psrldq m0, 4 ; shift top to next pixel + psrad m3, [fg_dataq+FGData.ar_coeff_shift] + ; we do not need to packssdw since we only care about one value + paddd m3, m2 + packssdw m3, m3 + pminsw m3, m13 + pmaxsw m3, m14 + psrldq m1, 2 + pslldq m3, 2 + psrldq m2, 4 +%if cpuflag(sse4) + pblendw m1, m3, 00000010b +%else + pand m1, m15 + pandn m4, m15, m3 + por m1, m4 +%endif + ; overwrite previous pixel, should be ok + movd [bufq+xq*2-2], m1 + inc xq + jz .x_loop_ar2_end + test xq, 3 + jnz .x_loop_ar2_inner + jmp .x_loop_ar2 + +.x_loop_ar2_end: + add bufq, 82*2 + add bufyq, 82*2<<%3 + dec hd + jg .y_loop_ar2 +%if ARCH_X86_32 +%undef m13 +%undef m14 +%undef m15 +%endif + RET + +.ar3: +%if ARCH_X86_64 + DEFINE_ARGS buf, bufy, fg_data, uv, bdmax, shift +%if WIN64 + mov r6, rsp + and rsp, ~15 + sub rsp, 96 + %define tmp rsp +%else + %define tmp rsp+stack_offset-120 +%endif +%else + DEFINE_ARGS buf, bufy, pic_reg, fg_data, uv, shift +%assign stack_offset stack_offset_old + ALLOC_STACK -16*14 + mov bufyq, r1m + mov uvd, r3m + %define tmp rsp +%endif + mov shiftd, [fg_dataq+FGData.ar_coeff_shift] + imul uvd, 28 + SPLATW m4, [base+round_vals-12+shiftq*2] + pxor m5, m5 + pcmpgtw m5, m4 + punpcklwd m4, m5 +%if ARCH_X86_64 + sar bdmaxd, 1 + SPLATW m6, bdmaxd ; max_grain +%else + SPLATW m6, r4m + psraw m6, 1 +%endif + pcmpeqw m7, m7 +%if !cpuflag(sse4) + pcmpeqw m3, m3 + psrldq m3, 14 + pslldq m3, 4 + pxor m3, m7 +%endif + pxor m7, m6 ; min_grain +%if %2 && cpuflag(sse4) + SPLATW m3, [base+hmul_bits+2+%3*2] +%endif + +%if ARCH_X86_64 + SWAP 3, 11 + SWAP 4, 12 + SWAP 6, 14 + SWAP 7, 15 +%else +%define m11 [rsp+ 9*16] +%define m12 [rsp+10*16] +%define m14 [rsp+12*16] +%define m15 [rsp+13*16] + mova m11, m3 + mova m12, m4 + mova m14, m6 + mova m15, m7 +%endif + + ; cf from y=-3,x=-3 until y=-3,x=-2 + movu m0, [fg_dataq+FGData.ar_coeffs_uv+uvq+ 0] + pxor m1, m1 + pcmpgtb m1, m0 + punpckhbw m2, m0, m1 + punpcklbw m0, m1 + pshufd m1, m0, q0000 + pshufd m3, m0, q1111 + pshufd m4, m0, q2222 + pshufd m0, m0, q3333 + pshufd m5, m2, q0000 + pshufd m6, m2, q1111 + mova [tmp+16*0], m1 + mova [tmp+16*1], m3 + mova [tmp+16*2], m4 + mova [tmp+16*3], m0 + mova [tmp+16*4], m5 + mova [tmp+16*5], m6 + pshufd m6, m2, q2222 + pshufd m7, m2, q3333 + + ; cf from y=-1,x=-1 to y=0,x=-1 + luma component + movu m0, [fg_dataq+FGData.ar_coeffs_uv+uvq+16] + pxor m1, m1 + pcmpgtb m1, m0 + punpckhbw m2, m0, m1 ; luma + punpcklbw m0, m1 + pshufd m3, m0, q3232 + psrldq m5, m0, 10 + ; y=0,x=[-3 to -1] + "1.0" for current pixel + pinsrw m5, [base+round_vals-10+shiftq*2], 3 + ; y=-1,x=[-1 to +2] + pshufd m1, m0, q0000 + pshufd m0, m0, q1111 + ; y=-1,x=+3 + luma + punpcklwd m3, m2 + pshufd m3, m3, q0000 + +%if ARCH_X86_64 + SWAP 1, 8 + SWAP 0, 9 + SWAP 3, 10 + SWAP 5, 13 + DEFINE_ARGS buf, bufy, fg_data, h, x +%else +%define m8 [rsp+ 6*16] +%define m9 [rsp+ 7*16] +%define m10 [rsp+ 8*16] +%define m13 [rsp+11*16] + mova m8, m1 + mova m9, m0 + mova m10, m3 + mova m13, m5 + DEFINE_ARGS buf, bufy, pic_reg, fg_data, h, x +%endif +%if %2 + sub bufq, 2*(82*(73-35*%3)+44-(82*3+41)) +%else + sub bufq, 2*(82*69+3) +%endif + add bufyq, 2*(79+82*3) + mov hd, 70-35*%3 +.y_loop_ar3: + mov xq, -(76>>%2) + +.x_loop_ar3: + ; first line + movu m0, [bufq+xq*2-82*6-6+ 0] ; y=-3,x=[-3,+4] + movd m1, [bufq+xq*2-82*6-6+16] ; y=-3,x=[+5,+6] + palignr m2, m1, m0, 2 ; y=-3,x=[-2,+5] + palignr m1, m1, m0, 12 ; y=-3,x=[+3,+6] + punpckhwd m3, m0, m2 ; y=-3,x=[+1/+2,+2/+3,+3/+4,+4/+5] + punpcklwd m0, m2 ; y=-3,x=[-3/-2,-2/-1,-1/+0,+0/+1] + shufps m2, m0, m3, q1032 ; y=-3,x=[-1/+0,+0/+1,+1/+2,+2/+3] + + pmaddwd m0, [tmp+0*16] + pmaddwd m2, [tmp+1*16] + pmaddwd m3, [tmp+2*16] + paddd m0, m2 + paddd m0, m3 ; first 6 x of top y + + ; second line [m0/1 are busy] + movu m2, [bufq+xq*2-82*4-6+ 0] ; y=-2,x=[-3,+4] + movd m3, [bufq+xq*2-82*4-6+16] ; y=-2,x=[+5,+6] + punpcklwd m1, m2 ; y=-3/-2,x=[+3/-3,+4/-2,+5/-1,+6/+0] + palignr m4, m3, m2, 2 ; y=-2,x=[-2,+5] + palignr m3, m3, m2, 4 ; y=-2,x=[-2,+5] + punpckhwd m5, m4, m3 ; y=-2,x=[+2/+3,+3/+4,+4/+5,+5/+6] + punpcklwd m4, m3 ; y=-2,x=[-2/-1,-1/+0,+0/+1,+1/+2] + shufps m3, m4, m5, q1032 ; t=-2,x=[+0/+1,+1/+2,+2/+3,+3/+4] + pmaddwd m1, [tmp+3*16] + pmaddwd m4, [tmp+4*16] + pmaddwd m3, [tmp+5*16] + pmaddwd m5, m6 + paddd m1, m4 + paddd m3, m5 + paddd m0, m1 + paddd m0, m3 ; top 2 lines + + ; third line [m0 is busy] & luma + round + movu m1, [bufq+xq*2-82*2-6+ 0] ; y=-1,x=[-3,+4] + movd m2, [bufq+xq*2-82*2-6+16] ; y=-1,x=[+5,+6] +%if %2 + movu m5, [bufyq+xq*4] +%if %3 + movu m4, [bufyq+xq*4+82*2] + phaddw m5, m4 +%else + phaddw m5, m5 +%endif +%else + movq m5, [bufyq+xq*2] +%endif + palignr m3, m2, m1, 2 ; y=-1,x=[-2,+5] + palignr m2, m2, m1, 12 ; y=-1,x=[+3,+6] +%if %3 + pshufd m4, m5, q3232 + paddw m5, m4 +%endif +%if %2 +%if cpuflag(sse4) + pmulhrsw m5, m11 +%elif %3 + pmulhrsw m5, [base+pw_8192] +%else + pmulhrsw m5, [base+pw_16384] +%endif +%endif + punpckhwd m4, m1, m3 ; y=-1,x=[+1/+2,+2/+3,+3/+4,+4/+5] + punpcklwd m1, m3 ; y=-1,x=[-3/-2,-2/-1,-1/+0,+0/+1] + shufps m3, m1, m4, q1032 ; y=-1,x=[-1/+0,+0/+1,+1/+2,+2/+3] + punpcklwd m2, m5 + pmaddwd m1, m7 + pmaddwd m3, m8 + pmaddwd m4, m9 + pmaddwd m2, m10 + paddd m1, m3 + paddd m4, m2 + paddd m0, m12 ; += round + paddd m1, m4 + paddd m0, m1 + + movu m1, [bufq+xq*2-6] ; y=0,x=[-3,+4] +.x_loop_ar3_inner: + pmaddwd m2, m1, m13 + pshufd m3, m2, q1111 + paddd m2, m3 ; left+cur + paddd m2, m0 ; add top + psrldq m0, 4 + psrad m2, [fg_dataq+FGData.ar_coeff_shift] + packssdw m2, m2 + pminsw m2, m14 + pmaxsw m2, m15 + pslldq m2, 4 + psrldq m1, 2 +%if cpuflag(sse4) + pblendw m1, m2, 00000100b +%else + pand m1, m11 + pandn m3, m11, m2 + por m1, m3 +%endif + ; overwrite previous pixels, should be ok + movq [bufq+xq*2-4], m1 + inc xq + jz .x_loop_ar3_end + test xq, 3 + jnz .x_loop_ar3_inner + jmp .x_loop_ar3 + +.x_loop_ar3_end: + add bufq, 82*2 + add bufyq, 82*2<<%3 + dec hd + jg .y_loop_ar3 +%if WIN64 + mov rsp, r6 +%elif ARCH_X86_32 +%undef m8 +%undef m9 +%undef m10 +%undef m11 +%undef m12 +%undef m13 +%undef m14 +%undef m15 +%endif + RET +%endmacro + +generate_grain_uv_fn 420, 1, 1 +generate_grain_uv_fn 422, 1, 0 +generate_grain_uv_fn 444, 0, 0 + +%macro SCRATCH 3 +%if ARCH_X86_32 + mova [rsp+%3*mmsize], m%1 +%define m%2 [rsp+%3*mmsize] +%else + SWAP %1, %2 +%endif +%endmacro + +INIT_XMM ssse3 +%if ARCH_X86_32 +%if STACK_ALIGNMENT < mmsize +cglobal fgy_32x32xn_16bpc, 0, 7, 8, 0-(8 * mmsize + 12 * gprsize), \ + dst, src, scaling, unused1, fg_data, picptr, unused2 + ; copy stack arguments to new position post-alignment, so that we + ; don't have to keep the old stack location in a separate register + mov r0, r0m + mov r1, r2m + mov r2, r4m + mov r3, r6m + mov r4, r7m + mov r5, r8m + +%define r0m [rsp+8*mmsize+ 3*gprsize] +%define r2m [rsp+8*mmsize+ 5*gprsize] +%define r4m [rsp+8*mmsize+ 7*gprsize] +%define r6m [rsp+8*mmsize+ 9*gprsize] +%define r7m [rsp+8*mmsize+10*gprsize] +%define r8m [rsp+8*mmsize+11*gprsize] + + mov r0m, r0 + mov r2m, r1 + mov r4m, r2 + mov r6m, r3 + mov r7m, r4 + mov r8m, r5 +%else +cglobal fgy_32x32xn_16bpc, 0, 7, 8, 8 * mmsize + 4 * gprsize, \ + dst, src, scaling, unused1, fg_data, picptr, unused2 +%endif + mov srcq, srcm + mov scalingq, r5m + mov fg_dataq, r3m +%if STACK_ALIGNMENT < mmsize + mov r6, r9m + +%define r9m [rsp+8*mmsize+ 4*gprsize] +%define r3m [rsp+8*mmsize+ 6*gprsize] +%define r5m [rsp+8*mmsize+ 8*gprsize] + + mov r9m, r6 +%endif + LEA r5, $$ +%define base r5-$$ + mov r5m, picptrq +%else +cglobal fgy_32x32xn_16bpc, 6, 15, 16, dst, src, stride, fg_data, w, scaling, grain_lut + lea r8, [pb_mask] +%define base r8-pb_mask +%endif + mov r6d, [fg_dataq+FGData.scaling_shift] + SPLATW m3, [base+mul_bits+r6*2-14] + mov r6d, [fg_dataq+FGData.clip_to_restricted_range] +%if ARCH_X86_32 + DECLARE_REG_TMP 0, 3 +%else + DECLARE_REG_TMP 9, 10 +%endif + mov t0d, r9m ; bdmax + sar t0d, 11 ; is_12bpc + inc t0d + mov t1d, r6d + imul t1d, t0d + dec t0d + SPLATW m5, [base+min+t1*2] + lea t0d, [t0d*3] + lea t0d, [r6d*2+t0d] + SPLATW m4, [base+max+t0*2] + SPLATW m2, r9m + + pcmpeqw m1, m1 + psraw m7, m2, 1 ; max_grain + pxor m1, m7 ; min_grain + SPLATD m6, [base+pd_16] + + SCRATCH 1, 9, 0 + SCRATCH 2, 10, 1 + SCRATCH 3, 11, 2 + SCRATCH 4, 12, 3 + SCRATCH 5, 13, 4 + SCRATCH 6, 14, 5 + SCRATCH 7, 15, 6 + + mova m6, [base+pw_27_17_17_27] ; for horizontal filter + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, sby, fg_data, picptr, unused2 + DECLARE_REG_TMP 0 +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused1, \ + sby, see + DECLARE_REG_TMP 7 +%endif + + mov sbyd, r8m + movzx t0d, byte [fg_dataq+FGData.overlap_flag] + test t0d, t0d + jz .no_vertical_overlap + test sbyd, sbyd + jnz .vertical_overlap +.no_vertical_overlap: + mov dword r8m, t0d + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, see, fg_data, picptr, unused + imul seed, (173 << 24) | 37 +%else + imul seed, sbyd, (173 << 24) | 37 +%endif + add seed, (105 << 24) | 178 + rol seed, 8 + movzx seed, seew + xor seed, [fg_dataq+FGData.seed] + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, see, w, picptr, src_bak + + mov r3m, seed + mov wq, r4m +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + unused1, unused2, see, src_bak +%endif + + lea src_bakq, [srcq+wq*2] + mov r9mp, src_bakq + neg wq + sub dstmp, srcq +%if ARCH_X86_32 + mov r4m, wq +%endif + +.loop_x: +%if ARCH_X86_32 + mov seed, r3m +%endif + mov r6d, seed + or seed, 0xEFF4 + shr r6d, 1 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS dst, src, scaling, offy, unused1, unused2, offx + + mov offxd, offyd +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, src_bak + + mov offyd, seed + mov offxd, seed +%endif + ror offyd, 8 + shr offxd, 12 + and offyd, 0xf + imul offyd, 164 + lea offyq, [offyq+offxq*2+747] ; offy*stride+offx + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, src_bak +%endif + +.loop_x_odd: + movzx hd, word r7m + mov grain_lutq, grain_lutmp +.loop_y: + ; src + pand m0, m10, [srcq+ 0] + pand m1, m10, [srcq+16] ; m0-1: src as word + + ; scaling[src] +%if ARCH_X86_32 + vpgatherdw m2, m0, scalingq-1, r0, r5, 8, 1, m4 + vpgatherdw m3, m1, scalingq-1, r0, r5, 8, 1, m4 +%else + vpgatherdw m2, m0, scalingq-1, r11, r13, 8, 1, m4 + vpgatherdw m3, m1, scalingq-1, r11, r13, 8, 1, m4 +%endif + REPX {psrlw x, 8}, m2, m3 + + ; grain = grain_lut[offy+y][offx+x] + movu m4, [grain_lutq+offxyq*2] + movu m5, [grain_lutq+offxyq*2+16] + + ; noise = round2(scaling[src] * grain, scaling_shift) + REPX {pmullw x, m11}, m2, m3 + pmulhrsw m4, m2 + pmulhrsw m5, m3 + + ; dst = clip_pixel(src, noise) + paddw m0, m4 + paddw m1, m5 + pmaxsw m0, m13 + pmaxsw m1, m13 + pminsw m0, m12 + pminsw m1, m12 + movifnidn dstq, dstmp + mova [dstq+srcq+ 0], m0 + mova [dstq+srcq+16], m1 + + add srcq, r2mp ; src += stride + add grain_lutq, 82*2 + dec hd + jg .loop_y + +%if ARCH_X86_32 + add r4mp, 16 +%else + add wq, 16 +%endif + jge .end +%if ARCH_X86_32 + mov srcq, r9mp + add srcq, r4mp + add srcq, r4mp +%else + mov src_bakq, r9mp + lea srcq, [src_bakq+wq*2] +%endif + btc dword r8m, 2 + jc .next_blk + add offxyd, 16 + test dword r8m, 2 + jz .loop_x_odd +%if ARCH_X86_32 + add dword [rsp+8*mmsize+1*gprsize], 16 +%else + add r12d, 16 ; top_offxy += 16 +%endif + jmp .loop_x_odd_v_overlap + +.next_blk: + test dword r8m, 1 + jz .loop_x + + ; r8m = sbym + test dword r8m, 2 + jnz .loop_x_hv_overlap + + ; horizontal overlap (without vertical overlap) +.loop_x_h_overlap: +%if ARCH_X86_32 + add offxyd, 16 + mov [rsp+8*mmsize+0*gprsize], offxyd + DEFINE_ARGS dst, src, scaling, see, w, picptr, src_bak + mov seed, r3m +%endif + + mov r6d, seed + or seed, 0xEFF4 + shr r6d, 1 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS dst, src, scaling, offy, h, picptr, offx + + mov offxd, offyd +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, src_bak, left_offxy + + lea left_offxyd, [offyd+16] ; previous column's offy*stride+offx + + mov offyd, seed + mov offxd, seed +%endif + ror offyd, 8 + shr offxd, 12 + and offyd, 0xf + imul offyd, 164 + lea offyq, [offyq+offxq*2+747] ; offy*stride+offx + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, src_bak, left_offxy +%endif + + mov hd, dword r7m + mov grain_lutq, grain_lutmp +.loop_y_h_overlap: + ; grain = grain_lut[offy+y][offx+x] + movu m5, [grain_lutq+offxyq*2] +%if ARCH_X86_32 + mov r5, [rsp+8*mmsize+0*gprsize] + movd m4, [grain_lutq+r5*2] +%else + movd m4, [grain_lutq+left_offxyq*2] +%endif + punpcklwd m4, m5 + pmaddwd m4, m6 + paddd m4, m14 + psrad m4, 5 + packssdw m4, m4 + pminsw m4, m15 + pmaxsw m4, m9 + shufps m4, m5, q3210 + + ; src + pand m0, m10, [srcq+ 0] + pand m1, m10, [srcq+16] ; m0-1: src as word + + ; scaling[src] +%if ARCH_X86_32 + vpgatherdw m2, m0, scalingq-1, r0, r5, 8, 1, m5 + vpgatherdw m3, m1, scalingq-1, r0, r5, 8, 1, m5 +%else + vpgatherdw m2, m0, scalingq-1, r13, r14, 8, 1, m5 + vpgatherdw m3, m1, scalingq-1, r13, r14, 8, 1, m5 +%endif + REPX {psrlw x, 8}, m2, m3 + + ; noise = round2(scaling[src] * grain, scaling_shift) + movu m5, [grain_lutq+offxyq*2+16] + REPX {pmullw x, m11}, m2, m3 + pmulhrsw m4, m2 + pmulhrsw m5, m3 + + ; dst = clip_pixel(src, noise) + paddw m0, m4 + paddw m1, m5 + pmaxsw m0, m13 + pmaxsw m1, m13 + pminsw m0, m12 + pminsw m1, m12 + movifnidn dstq, dstmp + mova [dstq+srcq+ 0], m0 + mova [dstq+srcq+16], m1 + + add srcq, r2mp + add grain_lutq, 82*2 + dec hd + jg .loop_y_h_overlap + +%if ARCH_X86_32 + add r4mp, 16 +%else + add wq, 16 +%endif + jge .end +%if ARCH_X86_32 + mov srcq, r9mp + add srcq, r4mp + add srcq, r4mp +%else + mov src_bakq, r9mp + lea srcq, [src_bakq+wq*2] +%endif + or dword r8m, 4 + add offxyd, 16 + + ; r8m = sbym + test dword r8m, 2 + jz .loop_x_odd +%if ARCH_X86_32 + add dword [rsp+8*mmsize+1*gprsize], 16 +%else + add r12d, 16 ; top_offxy += 16 +%endif + jmp .loop_x_odd_v_overlap + +.end: + RET + +.vertical_overlap: + or t0d, 2 + mov r8m, t0d + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, sby, fg_data, picptr, unused +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused1, \ + sby, see +%endif + + movzx sbyd, sbyb +%if ARCH_X86_32 + imul r4, [fg_dataq+FGData.seed], 0x00010001 + DEFINE_ARGS dst, src, scaling, sby, see, picptr, unused +%else + imul seed, [fg_dataq+FGData.seed], 0x00010001 +%endif + imul t0d, sbyd, 173 * 0x00010001 + imul sbyd, 37 * 0x01000100 + add t0d, (105 << 16) | 188 + add sbyd, (178 << 24) | (141 << 8) + and t0d, 0x00ff00ff + and sbyd, 0xff00ff00 + xor seed, t0d +%if ARCH_X86_32 + xor sbyd, seed + + DEFINE_ARGS dst, src, scaling, see, w, picptr, src_bak + + mov r3m, seed + mov wq, r4m +%else + xor seed, sbyd ; (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + unused1, unused2, see, src_bak +%endif + + lea src_bakq, [srcq+wq*2] + mov r9mp, src_bakq + neg wq + sub dstmp, srcq +%if ARCH_X86_32 + mov r4m, wq +%endif + +.loop_x_v_overlap: +%if ARCH_X86_32 + mov r5, r5m + SPLATD m7, [base+pw_27_17_17_27] + mov seed, r3m +%else + SPLATD m7, [pw_27_17_17_27] +%endif + + ; we assume from the block above that bits 8-15 of r7d are zero'ed + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp t0b ; parity of top_seed + shr seed, 16 + shl t0d, 16 + test seeb, seeh + setp t0b ; parity of cur_seed + or r6d, 0x00010001 + xor t0d, r6d + mov seed, t0d + ror seed, 1 ; updated (cur_seed << 16) | top_seed + +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS dst, src, scaling, offy, unused1, unused2, offx + + mov offxd, offyd +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, src_bak, unused, top_offxy + + mov offyd, seed + mov offxd, seed +%endif + ror offyd, 8 + ror offxd, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyq, [offyq+offxq*2+0x10001*747+32*82] + +%if ARCH_X86_32 + DEFINE_ARGS top_offxy, src, scaling, offxy, h, picptr, grain_lut +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, src_bak, unused, top_offxy +%endif + + movzx top_offxyd, offxyw +%if ARCH_X86_32 + mov [rsp+8*mmsize+1*gprsize], top_offxyd + + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut +%endif + shr offxyd, 16 + +.loop_x_odd_v_overlap: +%if ARCH_X86_32 + mov r5, r5m +%endif + SPLATD m7, [PIC_ptr(pw_27_17_17_27)] + mov hd, dword r7m + mov grain_lutq, grain_lutmp +.loop_y_v_overlap: + ; grain = grain_lut[offy+y][offx+x] + movu m3, [grain_lutq+offxyq*2] +%if ARCH_X86_32 + mov r5, [rsp+8*mmsize+1*gprsize] + movu m2, [grain_lutq+r5*2] +%else + movu m2, [grain_lutq+top_offxyq*2] +%endif + punpckhwd m4, m2, m3 + punpcklwd m2, m3 + REPX {pmaddwd x, m7}, m4, m2 + REPX {paddd x, m14}, m4, m2 + REPX {psrad x, 5}, m4, m2 + packssdw m2, m4 + pminsw m2, m15 + pmaxsw m2, m9 + movu m4, [grain_lutq+offxyq*2+16] +%if ARCH_X86_32 + movu m3, [grain_lutq+r5*2+16] +%else + movu m3, [grain_lutq+top_offxyq*2+16] +%endif + punpckhwd m5, m3, m4 + punpcklwd m3, m4 + REPX {pmaddwd x, m7}, m5, m3 + REPX {paddd x, m14}, m5, m3 + REPX {psrad x, 5}, m5, m3 + packssdw m3, m5 + pminsw m3, m15 + pmaxsw m3, m9 + + ; src + pand m0, m10, [srcq+ 0] ; m0-1: src as word + pand m1, m10, [srcq+16] ; m0-1: src as word + + ; scaling[src] + ; noise = round2(scaling[src] * grain, scaling_shift) +%if ARCH_X86_32 + vpgatherdw m4, m0, scalingq-1, r0, r5, 8, 1, m5 +%else + vpgatherdw m4, m0, scalingq-1, r11, r13, 8, 1, m5 +%endif + psrlw m4, 8 + pmullw m4, m11 + pmulhrsw m4, m2 +%if ARCH_X86_32 + vpgatherdw m5, m1, scalingq-1, r0, r5, 8, 1, m2 +%else + vpgatherdw m5, m1, scalingq-1, r11, r13, 8, 1, m2 +%endif + psrlw m5, 8 + pmullw m5, m11 + pmulhrsw m5, m3 + + ; dst = clip_pixel(src, noise) + paddw m0, m4 + paddw m1, m5 + pmaxsw m0, m13 + pmaxsw m1, m13 + pminsw m0, m12 + pminsw m1, m12 + movifnidn dstq, dstmp + mova [dstq+srcq+ 0], m0 + mova [dstq+srcq+16], m1 + + add srcq, r2mp + add grain_lutq, 82*2 + dec hw + jz .end_y_v_overlap + ; 2 lines get vertical overlap, then fall back to non-overlap code for + ; remaining (up to) 30 lines +%if ARCH_X86_32 + mov r5, r5m +%endif + SPLATD m7, [PIC_ptr(pw_27_17_17_27)+4] + xor hd, 0x10000 + test hd, 0x10000 + jnz .loop_y_v_overlap + jmp .loop_y + +.end_y_v_overlap: +%if ARCH_X86_32 + add r4mp, 16 +%else + add wq, 16 +%endif + jge .end_hv +%if ARCH_X86_32 + mov srcq, r9mp + add srcq, r4mp + add srcq, r4mp +%else + mov src_bakq, r9mp + lea srcq, [src_bakq+wq*2] +%endif + btc dword r8m, 2 + jc .next_blk_v +%if ARCH_X86_32 + add dword [rsp+8*mmsize+1*gprsize], 16 +%else + add top_offxyd, 16 +%endif + add offxyd, 16 + jmp .loop_x_odd_v_overlap + +.next_blk_v: + ; since fg_dataq.overlap is guaranteed to be set, we never jump + ; back to .loop_x_v_overlap, and instead always fall-through to + ; h+v overlap + +.loop_x_hv_overlap: +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, see, w, picptr, src_bak + + mov r0, [rsp+8*mmsize+1*gprsize] + add r3, 16 + add r0, 16 + mov [rsp+8*mmsize+0*gprsize], r3 ; left_offxy + mov [rsp+8*mmsize+2*gprsize], r0 ; topleft_offxy + + mov seed, r3m + xor r0, r0 +%else + ; we assume from the block above that bits 8-15 of r7d are zero'ed +%endif + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp t0b ; parity of top_seed + shr seed, 16 + shl t0d, 16 + test seeb, seeh + setp t0b ; parity of cur_seed + or r6d, 0x00010001 + xor t0d, r6d + mov seed, t0d + ror seed, 1 ; updated (cur_seed << 16) | top_seed + +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS dst, src, scaling, offy, w, picptr, offx + + mov offxd, offyd +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, src_bak, left_offxy, top_offxy, topleft_offxy + + lea topleft_offxyq, [top_offxyq+16] + lea left_offxyq, [offyq+16] + mov offyd, seed + mov offxd, seed +%endif + ror offyd, 8 + ror offxd, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyq, [offyq+offxq*2+0x10001*747+32*82] + +%if ARCH_X86_32 + DEFINE_ARGS top_offxy, src, scaling, offxy, w, picptr, grain_lut +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, src_bak, left_offxy, top_offxy, topleft_offxy +%endif + + movzx top_offxyd, offxyw +%if ARCH_X86_32 + mov [rsp+8*mmsize+1*gprsize], top_offxyd + + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut +%endif + shr offxyd, 16 + +%if ARCH_X86_32 + mov r5, r5m +%endif + SPLATD m7, [PIC_ptr(pw_27_17_17_27)] + + movzx hd, word r7m + mov grain_lutq, grain_lutmp +.loop_y_hv_overlap: + ; grain = grain_lut[offy+y][offx+x] + movu m2, [grain_lutq+offxyq*2] +%if ARCH_X86_32 + mov r0, [rsp+8*mmsize+1*gprsize] ; top_offxy + mov r5, [rsp+8*mmsize+0*gprsize] ; left_offxy + movu m4, [grain_lutq+r0*2] + movd m5, [grain_lutq+r5*2] + mov r5, [rsp+8*mmsize+2*gprsize] ; topleft_offxy + movd m3, [grain_lutq+r5*2] +%else + movu m4, [grain_lutq+top_offxyq*2] + movd m5, [grain_lutq+left_offxyq*2] + movd m3, [grain_lutq+topleft_offxyq*2] +%endif + ; do h interpolation first (so top | top/left -> top, left | cur -> cur) + punpcklwd m5, m2 + punpcklwd m3, m4 + REPX {pmaddwd x, m6}, m5, m3 + REPX {paddd x, m14}, m5, m3 + REPX {psrad x, 5}, m5, m3 + packssdw m5, m3 + pminsw m5, m15 + pmaxsw m5, m9 + shufps m3, m5, m2, q3210 + shufps m5, m4, q3232 + ; followed by v interpolation (top | cur -> cur) + movu m0, [grain_lutq+offxyq*2+16] +%if ARCH_X86_32 + movu m1, [grain_lutq+r0*2+16] +%else + movu m1, [grain_lutq+top_offxyq*2+16] +%endif + punpcklwd m2, m5, m3 + punpckhwd m5, m3 + punpcklwd m3, m1, m0 + punpckhwd m1, m0 + REPX {pmaddwd x, m7}, m2, m5, m3, m1 + REPX {paddd x, m14}, m2, m5, m3, m1 + REPX {psrad x, 5}, m2, m5, m3, m1 + packssdw m2, m5 + packssdw m3, m1 + REPX {pminsw x, m15}, m2, m3 + REPX {pmaxsw x, m9}, m2, m3 + + ; src + pand m0, m10, [srcq+ 0] + pand m1, m10, [srcq+16] ; m0-1: src as word + + ; scaling[src] + ; noise = round2(scaling[src] * grain, scaling_shift) +%if ARCH_X86_32 + vpgatherdw m4, m0, scalingq-1, r0, r5, 8, 1, m5 +%else + vpgatherdw m4, m0, scalingq-1, r14, r10, 8, 1, m5 +%endif + psrlw m4, 8 + pmullw m4, m11 + pmulhrsw m2, m4 +%if ARCH_X86_32 + vpgatherdw m5, m1, scalingq-1, r0, r5, 8, 1, m4 +%else + vpgatherdw m5, m1, scalingq-1, r14, r10, 8, 1, m4 +%endif + psrlw m5, 8 + pmullw m5, m11 + pmulhrsw m3, m5 + + ; dst = clip_pixel(src, noise) + paddw m0, m2 + paddw m1, m3 + pmaxsw m0, m13 + pmaxsw m1, m13 + pminsw m0, m12 + pminsw m1, m12 + movifnidn dstq, dstmp + mova [dstq+srcq+ 0], m0 + mova [dstq+srcq+16], m1 + + add srcq, r2mp + add grain_lutq, 82*2 + dec hw + jz .end_y_hv_overlap + ; 2 lines get vertical overlap, then fall back to non-overlap code for + ; remaining (up to) 30 lines +%if ARCH_X86_32 + mov r5, r5m +%endif + SPLATD m7, [PIC_ptr(pw_27_17_17_27)+4] + xor hd, 0x10000 + test hd, 0x10000 + jnz .loop_y_hv_overlap + jmp .loop_y_h_overlap + +.end_y_hv_overlap: + or dword r8m, 4 +%if ARCH_X86_32 + add r4mp, 16 +%else + add wq, 16 +%endif + jge .end_hv +%if ARCH_X86_32 + mov r5, r5m + add offxyd, 16 + add dword [rsp+8*mmsize+1*gprsize], 16 ; top_offxy += 16 + mov srcq, r9mp + add srcq, r4mp + add srcq, r4mp +%else + add offxyd, 16 + add top_offxyd, 16 + mov src_bakq, r9mp + lea srcq, [src_bakq+wq*2] +%endif + jmp .loop_x_odd_v_overlap + +.end_hv: + RET +%if ARCH_X86_32 + DECLARE_ARG 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 +%endif + +%macro FGUV_FN 3 ; name, ss_hor, ss_ver +INIT_XMM ssse3 +%if ARCH_X86_32 +%if STACK_ALIGNMENT < mmsize +cglobal fguv_32x32xn_i%1_16bpc, 0, 7, 8, 0-(8 * mmsize + 16 * gprsize), \ + tmp, src, scaling, h, fg_data, picptr, unused + mov r0, r0m + mov r1, r1m + mov r2, r2m + mov r4, r3m + mov r3, r4m + mov r5, r5m +%define r0m [rsp+8*mmsize+ 3*gprsize] +%define r1m [rsp+8*mmsize+ 4*gprsize] +%define r2m [rsp+8*mmsize+ 5*gprsize] +%define r3m [rsp+8*mmsize+ 6*gprsize] +%define r4m [rsp+8*mmsize+ 7*gprsize] +%define r5m [rsp+8*mmsize+ 8*gprsize] + mov r0m, r0 + mov r2m, r2 + mov r4m, r3 + mov r5m, r5 + + mov r0, r6m + mov r2, r7m + mov r3, r8m + mov r5, r9m +%define r6m [rsp+8*mmsize+ 9*gprsize] +%define r7m [rsp+8*mmsize+10*gprsize] +%define r8m [rsp+8*mmsize+11*gprsize] +%define r9m [rsp+8*mmsize+12*gprsize] + mov r6m, r0 + mov r7m, r2 + mov r8m, r3 + mov r9m, r5 + + mov r2, r10m + mov r3, r11m + mov r5, r12m + mov r0, r13m +%define r10m [rsp+8*mmsize+13*gprsize] +%define r11m [rsp+8*mmsize+14*gprsize] +%define r12m [rsp+8*mmsize+15*gprsize] + mov r10m, r2 + mov r11m, r3 + mov r12m, r5 + + SPLATW m2, r13m +%else +cglobal fguv_32x32xn_i%1_16bpc, 0, 7, 8, 8 * mmsize + (4) * gprsize, \ + tmp, src, scaling, h, fg_data, picptr, unused + mov srcq, srcm + mov fg_dataq, r3m +%endif + LEA r5, $$ +%define base r5-$$ + + DECLARE_REG_TMP 0, 2, 3 +%else +cglobal fguv_32x32xn_i%1_16bpc, 6, 15, 16, dst, src, stride, fg_data, w, scaling, \ + grain_lut, h, sby, luma, lstride, uv_pl, is_id +%define base r8-pb_mask + lea r8, [pb_mask] + + DECLARE_REG_TMP 9, 10, 11 +%endif + mov r6d, [fg_dataq+FGData.scaling_shift] + SPLATW m3, [base+mul_bits+r6*2-14] + mov r6d, [fg_dataq+FGData.clip_to_restricted_range] +%if STACK_ALIGNMENT >= mmsize + mov t0d, r13m ; bdmax +%endif + sar t0d, 11 ; is_12bpc + inc t0d + mov t1d, r6d + imul t1d, t0d + dec t0d + SPLATW m5, [base+min+t1*2] + lea t1d, [t0d*3] + mov t2d, r12m + inc t2d + imul r6d, t2d + add t1d, r6d + SPLATW m4, [base+max+t1*2] +%if STACK_ALIGNMENT >= mmsize + SPLATW m2, r13m +%endif + + SCRATCH 2, 10, 2 + SCRATCH 3, 11, 3 + SCRATCH 4, 12, 4 + SCRATCH 5, 13, 5 + +%define mzero m7 + +%if %3 + SPLATD m2, [base+pw_23_22] +%endif + +%if ARCH_X86_32 + mov scalingq, r5m + mov r5m, r5 +%else + mov r13mp, strideq +%endif + + pcmpeqw m0, m0 + psraw m1, m10, 1 + pxor m0, m1 + + SCRATCH 0, 8, 0 + SCRATCH 1, 9, 1 + + cmp byte [fg_dataq+FGData.chroma_scaling_from_luma], 0 + jne .csfl + +%macro %%FGUV_32x32xN_LOOP 3 ; not-csfl, ss_h, ss_v +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, sby, fg_data, picptr, overlap + + DECLARE_REG_TMP 0 +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused, sby, see, overlap + + DECLARE_REG_TMP 9 +%endif + +%if %1 + mov r6d, r11m + SPLATW m0, [fg_dataq+FGData.uv_mult+r6*4] + SPLATW m1, [fg_dataq+FGData.uv_luma_mult+r6*4] + punpcklwd m6, m1, m0 + SPLATW m5, [fg_dataq+FGData.uv_offset+r6*4] + SPLATD m7, [base+pw_4+t0*4] + pmullw m5, m7 +%else + SPLATD m6, [base+pd_16] +%if %2 + mova m5, [base+pw_23_22] +%else + mova m5, [base+pw_27_17_17_27] +%endif +%endif + + SCRATCH 6, 14, 6 + SCRATCH 5, 15, 7 + +%if ARCH_X86_32 + DECLARE_REG_TMP 0 +%else + DECLARE_REG_TMP 7 +%endif + + mov sbyd, r8m + mov t0d, [fg_dataq+FGData.overlap_flag] + test t0d, t0d + jz %%no_vertical_overlap + test sbyd, sbyd + jnz %%vertical_overlap + +%%no_vertical_overlap: + mov r8m, t0d +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, see, fg_data, picptr, overlap + imul seed, (173 << 24) | 37 +%else + imul seed, sbyd, (173 << 24) | 37 +%endif + add seed, (105 << 24) | 178 + rol seed, 8 + movzx seed, seew + xor seed, [fg_dataq+FGData.seed] +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS dst, src, scaling, see, w, picptr, luma + + mov dstq, r0mp + mov lumaq, r9mp + mov wq, r4m + lea r3, [srcq+wq*2] + mov r1mp, r3 + lea r3, [dstq+wq*2] + mov r11mp, r3 + lea r3, [lumaq+wq*(2<<%2)] + mov r12mp, r3 +%if %3 + shl r10mp, 1 +%endif +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + unused2, unused3, see, unused4, unused5, unused6, luma, lstride + + mov lstrideq, r10mp +%if %3 + add lstrideq, lstrideq +%endif + mov lumaq, r9mp + lea r10, [srcq+wq*2] + lea r11, [dstq+wq*2] + lea r12, [lumaq+wq*(2<<%2)] + mov r10mp, r10 + mov r11mp, r11 + mov r12mp, r12 +%endif + neg wq +%if ARCH_X86_32 + mov r4mp, wq +%endif + +%%loop_x: +%if ARCH_X86_32 + mov seed, r3m +%endif + + mov r6d, seed + or seed, 0xEFF4 + shr r6d, 1 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS dst, src, scaling, offy, w, picptr, offx + + mov offxd, offyd +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, unused1, unused2, unused3, luma, lstride + + mov offxd, seed + mov offyd, seed +%endif + ror offyd, 8 + shr offxd, 12 + and offyd, 0xf + imul offyd, 164>>%3 + lea offyq, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, unused1, unused2, unused3, luma, lstride +%endif + +%if %2 == 0 +%%loop_x_odd: +%endif + mov hd, r7m + mov grain_lutq, grain_lutmp +%%loop_y: + ; src + mova m0, [srcq] + mova m1, [srcq+16] ; m0-1: src as word + + ; luma_src + pxor mzero, mzero +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, h, luma, grain_lut + + mov lumaq, r9m +%endif + mova m4, [lumaq+ 0] + mova m6, [lumaq+(16<<%2)] +%if %2 + phaddw m4, [lumaq+16] + phaddw m6, [lumaq+48] +%endif +%if ARCH_X86_32 + add lumaq, r10mp + mov r9m, lumaq +%endif +%if %2 + pavgw m4, mzero + pavgw m6, mzero +%endif + +%if %1 + punpckhwd m3, m4, m0 + punpcklwd m4, m0 + punpckhwd m5, m6, m1 + punpcklwd m6, m1 ; { luma, chroma } + REPX {pmaddwd x, m14}, m3, m4, m5, m6 + REPX {psrad x, 6}, m3, m4, m5, m6 + packssdw m4, m3 + packssdw m6, m5 + REPX {paddw x, m15}, m4, m6 + REPX {pmaxsw x, mzero}, m4, m6 + REPX {pminsw x, m10}, m4, m6 ; clip_pixel() +%else + REPX {pand x, m10}, m4, m6 +%endif + + ; scaling[luma_src] +%if ARCH_X86_32 + vpgatherdw m3, m4, scalingq-1, r0, r5, 8, 1 + vpgatherdw m5, m6, scalingq-1, r0, r5, 8, 1 +%else + vpgatherdw m3, m4, scalingq-1, r10, r12, 8, 1 + vpgatherdw m5, m6, scalingq-1, r10, r12, 8, 1 +%endif + REPX {psrlw x, 8}, m3, m5 + + ; grain = grain_lut[offy+y][offx+x] + movu m4, [grain_lutq+offxyq*2] + movu m6, [grain_lutq+offxyq*2+16] + + ; noise = round2(scaling[luma_src] * grain, scaling_shift) + REPX {pmullw x, m11}, m3, m5 + pmulhrsw m4, m3 + pmulhrsw m6, m5 + + ; dst = clip_pixel(src, noise) + paddw m0, m4 + paddw m1, m6 + pmaxsw m0, m13 + pmaxsw m1, m13 + pminsw m0, m12 + pminsw m1, m12 + movifnidn dstq, dstmp + mova [dstq+ 0], m0 + mova [dstq+16], m1 + +%if ARCH_X86_32 + add srcq, r2mp + add dstq, r2mp + mov dstmp, dstq +%else + add srcq, r13mp + add dstq, r13mp + add lumaq, lstrideq +%endif + add grain_lutq, 82*2 + dec hd + jg %%loop_y + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, w, picptr, luma + + mov wq, r4mp +%endif + add wq, 16 + jge %%end +%if ARCH_X86_32 + mov srcq, r1mp +%else + mov srcq, r10mp +%endif + mov dstq, r11mp + mov lumaq, r12mp + lea srcq, [srcq+wq*2] + lea dstq, [dstq+wq*2] + lea lumaq, [lumaq+wq*(2<<%2)] +%if ARCH_X86_32 + mov r0m, dstq + mov r9m, lumaq + mov r4m, wq +%endif +%if %2 == 0 + btc dword r8m, 2 + jc %%next_blk + add offxyd, 16 + test dword r8m, 2 + jz %%loop_x_odd +%if ARCH_X86_32 + add dword [rsp+8*mmsize+1*gprsize], 16 +%else + add r11d, 16 +%endif + jmp %%loop_x_odd_v_overlap +%%next_blk: +%endif + test dword r8m, 1 + je %%loop_x + + ; r8m = sbym + test dword r8m, 2 + jnz %%loop_x_hv_overlap + + ; horizontal overlap (without vertical overlap) +%%loop_x_h_overlap: +%if ARCH_X86_32 + add offxyd, 16 + mov [rsp+8*mmsize+0*gprsize], offxyd + + DEFINE_ARGS dst, src, scaling, see, w, picptr, grain_lut + + mov seed, r3m +%endif + mov r6d, seed + or seed, 0xEFF4 + shr r6d, 1 + test seeb, seeh + lea seed, [r6+0x8000] + cmovp seed, r6d ; updated seed + +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS dst, src, scaling, offy, w, picptr, offx + + mov offxd, offyd +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, left_offxy, unused1, unused2, luma, lstride + + lea left_offxyd, [offyd+16] ; previous column's offy*stride+offx + mov offxd, seed + mov offyd, seed +%endif + ror offyd, 8 + shr offxd, 12 + and offyd, 0xf + imul offyd, 164>>%3 + lea offyq, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, left_offxy, unused1, unused2, luma, lstride +%endif + + mov hd, r7m + mov grain_lutq, grain_lutmp +%%loop_y_h_overlap: + mova m0, [srcq] + mova m1, [srcq+16] + + ; luma_src + pxor mzero, mzero +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, h, luma, grain_lut + mov lumaq, r9m +%endif + mova m4, [lumaq+ 0] + mova m6, [lumaq+(16<<%2)] +%if %2 + phaddw m4, [lumaq+16] + phaddw m6, [lumaq+48] +%endif +%if ARCH_X86_32 + add lumaq, r10mp + mov r9m, lumaq +%endif +%if %2 + pavgw m4, mzero + pavgw m6, mzero +%endif + +%if %1 + punpckhwd m3, m4, m0 + punpcklwd m4, m0 + punpckhwd m5, m6, m1 + punpcklwd m6, m1 ; { luma, chroma } + REPX {pmaddwd x, m14}, m3, m4, m5, m6 + REPX {psrad x, 6}, m3, m4, m5, m6 + packssdw m4, m3 + packssdw m6, m5 + REPX {paddw x, m15}, m4, m6 + REPX {pmaxsw x, mzero}, m4, m6 + REPX {pminsw x, m10}, m4, m6 ; clip_pixel() +%else + REPX {pand x, m10}, m4, m6 +%endif + + ; grain = grain_lut[offy+y][offx+x] + movu m7, [grain_lutq+offxyq*2] +%if ARCH_X86_32 + mov r5, [rsp+8*mmsize+0*gprsize] + movd m5, [grain_lutq+r5*2] +%else + movd m5, [grain_lutq+left_offxyq*2+ 0] +%endif + punpcklwd m5, m7 ; {left0, cur0} +%if %1 +%if ARCH_X86_32 + mov r5, r5m +%endif +%if %2 + pmaddwd m5, [PIC_ptr(pw_23_22)] +%else + pmaddwd m5, [PIC_ptr(pw_27_17_17_27)] +%endif + paddd m5, [PIC_ptr(pd_16)] +%else + pmaddwd m5, m15 + paddd m5, m14 +%endif + psrad m5, 5 + packssdw m5, m5 + pmaxsw m5, m8 + pminsw m5, m9 + shufps m5, m7, q3210 + movu m3, [grain_lutq+offxyq*2+16] + + ; scaling[luma_src] +%if ARCH_X86_32 + vpgatherdw m7, m4, scalingq-1, r0, r5, 8, 1 + vpgatherdw m4, m6, scalingq-1, r0, r5, 8, 1 +%else + vpgatherdw m7, m4, scalingq-1, r2, r12, 8, 1 + vpgatherdw m4, m6, scalingq-1, r2, r12, 8, 1 +%endif + REPX {psrlw x, 8}, m7, m4 + + ; noise = round2(scaling[luma_src] * grain, scaling_shift) + REPX {pmullw x, m11}, m7, m4 + pmulhrsw m5, m7 + pmulhrsw m3, m4 + + ; dst = clip_pixel(src, noise) + paddw m0, m5 + paddw m1, m3 + pmaxsw m0, m13 + pmaxsw m1, m13 + pminsw m0, m12 + pminsw m1, m12 + movifnidn dstq, dstmp + mova [dstq+ 0], m0 + mova [dstq+16], m1 + +%if ARCH_X86_32 + add srcq, r2mp + add dstq, r2mp + mov dstmp, dstq +%else + add srcq, r13mp + add dstq, r13mp + add lumaq, lstrideq +%endif + add grain_lutq, 82*2 + dec hd + jg %%loop_y_h_overlap + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, w, luma, grain_lut + mov wq, r4mp +%endif + add wq, 16 + jge %%end +%if ARCH_X86_32 + mov srcq, r1mp +%else + mov srcq, r10mp +%endif + mov dstq, r11mp + mov lumaq, r12mp + lea srcq, [srcq+wq*2] + lea dstq, [dstq+wq*2] + lea lumaq, [lumaq+wq*(2<<%2)] +%if ARCH_X86_32 + mov r0mp, dstq + mov r9mp, lumaq + mov r4m, wq +%endif + +%if %2 + ; r8m = sbym + test dword r8m, 2 + jne %%loop_x_hv_overlap + jmp %%loop_x_h_overlap +%else + or dword r8m, 4 + add offxyd, 16 + + ; r8m = sbym + test dword r8m, 2 + jz %%loop_x_odd +%if ARCH_X86_32 + add dword [rsp+8*mmsize+1*gprsize], 16 +%else + add r11d, 16 ; top_offxy += 16 +%endif + jmp %%loop_x_odd_v_overlap +%endif + +%%end: + RET + +%%vertical_overlap: + or t0d, 2 + mov r8m, t0d + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, sby, fg_data, picptr, overlap +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused, \ + sby, see, unused1, unused2, unused3, lstride +%endif + + movzx sbyd, sbyb +%if ARCH_X86_32 + imul r4, [fg_dataq+FGData.seed], 0x00010001 + + DEFINE_ARGS tmp, src, scaling, sby, see, picptr, unused +%else + imul seed, [fg_dataq+FGData.seed], 0x00010001 +%endif + imul t0d, sbyd, 173 * 0x00010001 + imul sbyd, 37 * 0x01000100 + add t0d, (105 << 16) | 188 + add sbyd, (178 << 24) | (141 << 8) + and t0d, 0x00ff00ff + and sbyd, 0xff00ff00 + xor seed, t0d +%if ARCH_X86_32 + xor sbyd, seed + + DEFINE_ARGS dst, src, scaling, see, w, picptr, luma + + mov r3m, seed + mov dstq, r0mp + mov lumaq, r9mp + mov wq, r4m + lea r3, [srcq+wq*2] + mov r1mp, r3 + lea r3, [dstq+wq*2] + mov r11mp, r3 + lea r3, [lumaq+wq*(2<<%2)] + mov r12mp, r3 +%if %3 + shl r10mp, 1 +%endif +%else + xor seed, sbyd ; (cur_seed << 16) | top_seed + + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + unused1, unused2, see, unused3, unused4, unused5, luma, lstride + + mov lstrideq, r10mp +%if %3 + add lstrideq, lstrideq +%endif + mov lumaq, r9mp + lea r10, [srcq+wq*2] + lea r11, [dstq+wq*2] + lea r12, [lumaq+wq*(2<<%2)] + mov r10mp, r10 + mov r11mp, r11 + mov r12mp, r12 +%endif + neg wq +%if ARCH_X86_32 + mov r4m, wq +%endif + +%%loop_x_v_overlap: +%if ARCH_X86_32 + mov seed, r3m + xor t0d, t0d +%else + ; we assume from the block above that bits 8-15 of r7d are zero'ed +%endif + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp t0b ; parity of top_seed + shr seed, 16 + shl t0d, 16 + test seeb, seeh + setp t0b ; parity of cur_seed + or r6d, 0x00010001 + xor t0d, r6d + mov seed, t0d + ror seed, 1 ; updated (cur_seed << 16) | top_seed +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS dst, src, scaling, offy, w, picptr, offx + + mov offxd, offyd +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, unused1, top_offxy, unused2, luma, lstride + + mov offyd, seed + mov offxd, seed +%endif + ror offyd, 8 + ror offxd, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164>>%3 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyq, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] + +%if ARCH_X86_32 + DEFINE_ARGS top_offxy, src, scaling, offxy, h, picptr, grain_lut +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, unused1, top_offxy, unused2, luma, lstride +%endif + movzx top_offxyd, offxyw +%if ARCH_X86_32 + mov [rsp+8*mmsize+1*gprsize], top_offxyd + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut +%endif + shr offxyd, 16 + +%if %2 == 0 +%%loop_x_odd_v_overlap: +%endif +%if %3 == 0 +%if ARCH_X86_32 + mov r5, r5m +%endif + SPLATD m2, [PIC_ptr(pw_27_17_17_27)] +%endif + + mov hd, r7m + mov grain_lutq, grain_lutmp +%%loop_y_v_overlap: + ; grain = grain_lut[offy+y][offx+x] + movu m3, [grain_lutq+offxyq*2] +%if ARCH_X86_32 + mov r0, [rsp+mmsize*8+gprsize*1] ; top_offxy + movu m5, [grain_lutq+r0*2] +%else + movu m5, [grain_lutq+top_offxyq*2] +%endif + punpckhwd m7, m5, m3 + punpcklwd m5, m3 ; {top/cur interleaved} + REPX {pmaddwd x, m2}, m7, m5 +%if %1 +%if ARCH_X86_32 + mov r5, r5m +%endif + REPX {paddd x, [PIC_ptr(pd_16)]}, m7, m5 +%else + REPX {paddd x, m14}, m7, m5 +%endif + REPX {psrad x, 5}, m7, m5 + packssdw m3, m5, m7 + pmaxsw m3, m8 + pminsw m3, m9 + + ; grain = grain_lut[offy+y][offx+x] + movu m4, [grain_lutq+offxyq*2+16] +%if ARCH_X86_32 + movu m5, [grain_lutq+r0*2+16] +%else + movu m5, [grain_lutq+top_offxyq*2+16] +%endif + punpckhwd m7, m5, m4 + punpcklwd m5, m4 ; {top/cur interleaved} + REPX {pmaddwd x, m2}, m7, m5 +%if %1 + REPX {paddd x, [PIC_ptr(pd_16)]}, m7, m5 +%else + REPX {paddd x, m14}, m7, m5 +%endif + REPX {psrad x, 5}, m7, m5 + packssdw m4, m5, m7 + pmaxsw m4, m8 + pminsw m4, m9 + + ; src + mova m0, [srcq] + mova m1, [srcq+16] + + ; luma_src + pxor mzero, mzero +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, h, luma, grain_lut + + mov lumaq, r9mp +%endif + mova m5, [lumaq+ 0] + mova m6, [lumaq+(16<<%2)] +%if %2 + phaddw m5, [lumaq+16] + phaddw m6, [lumaq+48] +%endif +%if ARCH_X86_32 + add lumaq, r10mp + mov r9mp, lumaq +%endif +%if %2 + pavgw m5, mzero + pavgw m6, mzero +%endif + +%if %1 + punpckhwd m7, m5, m0 + punpcklwd m5, m0 + REPX {pmaddwd x, m14}, m7, m5 + REPX {psrad x, 6}, m7, m5 + packssdw m5, m7 + punpckhwd m7, m6, m1 + punpcklwd m6, m1 ; { luma, chroma } + REPX {pmaddwd x, m14}, m7, m6 + REPX {psrad x, 6}, m7, m6 + packssdw m6, m7 + pxor mzero, mzero + REPX {paddw x, m15}, m5, m6 + REPX {pmaxsw x, mzero}, m5, m6 + REPX {pminsw x, m10}, m5, m6 ; clip_pixel() +%else + REPX {pand x, m10}, m5, m6 +%endif + + ; scaling[luma_src] +%if ARCH_X86_32 + vpgatherdw m7, m5, scalingq-1, r0, r5, 8, 1 + vpgatherdw m5, m6, scalingq-1, r0, r5, 8, 1 +%else + vpgatherdw m7, m5, scalingq-1, r10, r12, 8, 1 + vpgatherdw m5, m6, scalingq-1, r10, r12, 8, 1 +%endif + REPX {psrlw x, 8}, m7, m5 + + ; noise = round2(scaling[luma_src] * grain, scaling_shift) + REPX {pmullw x, m11}, m7, m5 + pmulhrsw m3, m7 + pmulhrsw m4, m5 + + ; dst = clip_pixel(src, noise) + paddw m0, m3 + paddw m1, m4 + pmaxsw m0, m13 + pmaxsw m1, m13 + pminsw m0, m12 + pminsw m1, m12 + movifnidn dstq, dstmp + mova [dstq+ 0], m0 + mova [dstq+16], m1 + + dec hw + jle %%end_y_v_overlap +%if ARCH_X86_32 + add srcq, r2mp + add dstq, r2mp + mov dstmp, dstq +%else + add srcq, r13mp + add dstq, r13mp + add lumaq, lstrideq +%endif + add grain_lutq, 82*2 +%if %3 + jmp %%loop_y +%else + btc hd, 16 + jc %%loop_y +%if ARCH_X86_32 + mov r5, r5m +%endif + SPLATD m2, [PIC_ptr(pw_27_17_17_27)+4] + jmp %%loop_y_v_overlap +%endif + +%%end_y_v_overlap: +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, w, luma, grain_lut + + mov wq, r4m +%endif + add wq, 16 + jge %%end_hv +%if ARCH_X86_32 + mov srcq, r1mp +%else + mov srcq, r10mp +%endif + mov dstq, r11mp + mov lumaq, r12mp + lea srcq, [srcq+wq*2] + lea dstq, [dstq+wq*2] + lea lumaq, [lumaq+wq*(2<<%2)] +%if ARCH_X86_32 + mov r0mp, dstq + mov r9mp, lumaq + mov r4m, wq +%endif + +%if %2 + ; since fg_dataq.overlap is guaranteed to be set, we never jump + ; back to .loop_x_v_overlap, and instead always fall-through to + ; h+v overlap +%else + btc dword r8m, 2 + jc %%loop_x_hv_overlap + add offxyd, 16 +%if ARCH_X86_32 + add dword [rsp+8*mmsize+1*gprsize], 16 +%else + add r11d, 16 +%endif + jmp %%loop_x_odd_v_overlap +%endif + +%%loop_x_hv_overlap: +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, w, picptr, grain_lut + + mov t0d, [rsp+mmsize*8+gprsize*1] ; top_offxy + add offxyd, 16 + add t0d, 16 + mov [rsp+mmsize*8+gprsize*0], offxyd ; left_offxyd + mov [rsp+mmsize*8+gprsize*2], t0d ; topleft_offxyd + + DEFINE_ARGS dst, src, scaling, see, w, picptr, grain_lut + + mov seed, r3m + xor t0d, t0d +%else + ; we assume from the block above that bits 8-15 of r7d are zero'ed +%endif + mov r6d, seed + or seed, 0xeff4eff4 + test seeb, seeh + setp t0b ; parity of top_seed + shr seed, 16 + shl t0d, 16 + test seeb, seeh + setp t0b ; parity of cur_seed + or r6d, 0x00010001 + xor t0d, r6d + mov seed, t0d + ror seed, 1 ; updated (cur_seed << 16) | top_seed +%if ARCH_X86_32 + mov r3m, seed + + DEFINE_ARGS dst, src, scaling, offy, w, picptr, offx + + mov offxd, offyd +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + offx, offy, see, left_offxy, top_offxy, topleft_offxy, luma, lstride + + lea topleft_offxyq, [top_offxyq+16] + lea left_offxyq, [offyq+16] + mov offyd, seed + mov offxd, seed +%endif + ror offyd, 8 + ror offxd, 12 + and offyd, 0xf000f + and offxd, 0xf000f + imul offyd, 164>>%3 + ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy + lea offyq, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] + +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, top_offxy +%else + DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, \ + h, offxy, see, left_offxy, top_offxy, topleft_offxy, luma, lstride +%endif + movzx top_offxyd, offxyw +%if ARCH_X86_32 + mov [rsp+8*mmsize+1*gprsize], top_offxyd + + DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut +%endif + shr offxyd, 16 + +%if %3 == 0 +%if ARCH_X86_32 + mov r5, r5m +%endif + SPLATD m2, [PIC_ptr(pw_27_17_17_27)] +%endif + + mov hd, r7m + mov grain_lutq, grain_lutmp +%%loop_y_hv_overlap: + ; grain = grain_lut[offy+y][offx+x] +%if ARCH_X86_32 + mov r5, [rsp+8*mmsize+0*gprsize] ; left_offxy + mov r0, [rsp+8*mmsize+1*gprsize] ; top_offxy + movd m5, [grain_lutq+r5*2] +%else + movd m5, [grain_lutq+left_offxyq*2] +%endif + movu m7, [grain_lutq+offxyq*2] +%if ARCH_X86_32 + mov r5, [rsp+8*mmsize+2*gprsize] + movu m4, [grain_lutq+r0*2] +%if %2 + pinsrw m5, [grain_lutq+r5*2], 2 +%else + movd m3, [grain_lutq+r5*2] +%endif +%else + movu m4, [grain_lutq+top_offxyq*2] +%if %2 + pinsrw m5, [grain_lutq+topleft_offxyq*2], 2 ; { left, _, top/left } +%else + movd m3, [grain_lutq+topleft_offxyq*2] +%endif +%endif +%if %2 == 0 + punpckldq m5, m3 +%endif + punpckldq m3, m7, m4 ; { cur0/1,top0/1,cur2/3,top2/3 } + punpcklwd m5, m3 ; { left/cur0,_/cur1,topleft/top0,_/top1 } +%if %1 +%if ARCH_X86_32 + mov r5, r5m +%endif +%if %2 + movddup m0, [PIC_ptr(pw_23_22)] +%else + movddup m0, [PIC_ptr(pw_27_17_17_27)] +%endif +%else + pshufd m0, m15, q1010 +%endif + pmaddwd m5, m0 +%if %1 + paddd m5, [PIC_ptr(pd_16)] +%else + paddd m5, m14 +%endif + psrad m5, 5 + packssdw m5, m5 + pmaxsw m5, m8 + pminsw m5, m9 + shufps m5, m3, q3210 ; cur0/1,top0/1,cur2/3,top2/3 + shufps m3, m5, m7, q3220 ; cur0-7 post-h_filter + shufps m5, m4, q3231 ; top0-7 post-h_filter + + punpckhwd m7, m5, m3 + punpcklwd m5, m3 ; {top/cur interleaved} + REPX {pmaddwd x, m2}, m7, m5 +%if %1 + REPX {paddd x, [PIC_ptr(pd_16)]}, m5, m7 +%else + REPX {paddd x, m14}, m5, m7 +%endif + REPX {psrad x, 5}, m5, m7 + packssdw m3, m5, m7 + pmaxsw m3, m8 + pminsw m3, m9 + + ; right half + movu m4, [grain_lutq+offxyq*2+16] +%if ARCH_X86_32 + movu m0, [grain_lutq+r0*2+16] +%else + movu m0, [grain_lutq+top_offxyq*2+16] +%endif + punpckhwd m1, m0, m4 + punpcklwd m0, m4 ; {top/cur interleaved} + REPX {pmaddwd x, m2}, m1, m0 +%if %1 + REPX {paddd x, [PIC_ptr(pd_16)]}, m1, m0 +%else + REPX {paddd x, m14}, m1, m0 +%endif + REPX {psrad x, 5}, m1, m0 + packssdw m4, m0, m1 + pmaxsw m4, m8 + pminsw m4, m9 + + ; src + mova m0, [srcq] + mova m1, [srcq+16] + + ; luma_src + pxor mzero, mzero +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, h, luma, grain_lut + + mov lumaq, r9mp +%endif + mova m6, [lumaq+ 0] + mova m5, [lumaq+(16<<%2)] +%if %2 + phaddw m6, [lumaq+16] + phaddw m5, [lumaq+48] +%endif +%if ARCH_X86_32 + add lumaq, r10mp + mov r9mp, lumaq +%endif +%if %2 + pavgw m6, mzero + pavgw m5, mzero +%endif + +%if %1 + punpckhwd m7, m6, m0 + punpcklwd m6, m0 + REPX {pmaddwd x, m14}, m7, m6 + REPX {psrad x, 6}, m7, m6 + packssdw m6, m7 + punpckhwd m7, m5, m1 + punpcklwd m5, m1 ; { luma, chroma } + REPX {pmaddwd x, m14}, m7, m5 + REPX {psrad x, 6}, m7, m5 + packssdw m5, m7 + pxor mzero, mzero + REPX {paddw x, m15}, m6, m5 + REPX {pmaxsw x, mzero}, m6, m5 + REPX {pminsw x, m10}, m6, m5 ; clip_pixel() +%else + REPX {pand x, m10}, m6, m5 +%endif + + ; scaling[luma_src] +%if ARCH_X86_32 + vpgatherdw m7, m6, scalingq-1, r0, r5, 8, 1 + vpgatherdw m6, m5, scalingq-1, r0, r5, 8, 1 +%else +%if %3 == 0 + ; register shortage :) + push r12 +%endif + vpgatherdw m7, m6, scalingq-1, r2, r12, 8, 1 + vpgatherdw m6, m5, scalingq-1, r2, r12, 8, 1 +%if %3 == 0 + pop r12 +%endif +%endif + REPX {psrlw x, 8}, m7, m6 + + ; noise = round2(scaling[luma_src] * grain, scaling_shift) + REPX {pmullw x, m11}, m7, m6 + pmulhrsw m3, m7 + pmulhrsw m4, m6 + + ; dst = clip_pixel(src, noise) + paddw m0, m3 + paddw m1, m4 + pmaxsw m0, m13 + pmaxsw m1, m13 + pminsw m0, m12 + pminsw m1, m12 + movifnidn dstq, dstmp + mova [dstq+ 0], m0 + mova [dstq+16], m1 + +%if ARCH_X86_32 + add srcq, r2mp + add dstq, r2mp + mov dstmp, dstq +%else + add srcq, r13mp + add dstq, r13mp + add lumaq, lstrideq +%endif + add grain_lutq, 82*2 + dec hw +%if %3 + jg %%loop_y_h_overlap +%else + jle %%end_y_hv_overlap + btc hd, 16 + jc %%loop_y_h_overlap +%if ARCH_X86_32 + mov r5, r5m +%endif + SPLATD m2, [PIC_ptr(pw_27_17_17_27)+4] + jmp %%loop_y_hv_overlap +%%end_y_hv_overlap: +%endif +%if ARCH_X86_32 + DEFINE_ARGS dst, src, scaling, offxy, w, luma, grain_lut + + mov wq, r4m +%endif + add wq, 16 + jge %%end_hv +%if ARCH_X86_32 + mov srcq, r1mp +%else + mov srcq, r10mp +%endif + mov dstq, r11mp + mov lumaq, r12mp + lea srcq, [srcq+wq*2] + lea dstq, [dstq+wq*2] + lea lumaq, [lumaq+wq*(2<<%2)] +%if ARCH_X86_32 + mov dstmp, dstq + mov r9mp, lumaq + mov r4m, wq +%endif +%if %2 + jmp %%loop_x_hv_overlap +%else + or dword r8m, 4 + add offxyd, 16 +%if ARCH_X86_32 + add dword [rsp+8*mmsize+1*gprsize], 16 +%else + add r11d, 16 ; top_offxy += 16 +%endif + jmp %%loop_x_odd_v_overlap +%endif + +%%end_hv: + RET +%endmacro + + %%FGUV_32x32xN_LOOP 1, %2, %3 +.csfl: + %%FGUV_32x32xN_LOOP 0, %2, %3 + +%if STACK_ALIGNMENT < mmsize +DECLARE_ARG 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 +%endif +%endmacro + +FGUV_FN 420, 1, 1 +FGUV_FN 422, 1, 0 +FGUV_FN 444, 0, 0 diff --git a/third_party/dav1d/src/x86/film_grain_avx2.asm b/third_party/dav1d/src/x86/film_grain_avx2.asm --- a/third_party/dav1d/src/x86/film_grain_avx2.asm +++ b/third_party/dav1d/src/x86/film_grain_avx2.asm @@ -1,6 +1,6 @@ -; Copyright © 2019, VideoLAN and dav1d authors +; Copyright © 2019-2021, VideoLAN and dav1d authors ; Copyright © 2019, Two Orioles, LLC ; All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions are met: @@ -36,37 +36,39 @@ rnd_next_upperbit_mask: dw 0x100B, 0x2016, 0x402C, 0x8058 byte_blend: db 0, 0, 0, 0xff, 0, 0, 0, 0 pw_seed_xor: times 2 dw 0xb524 times 2 dw 0x49d8 pd_m65536: dd ~0xffff -pb_23_22: times 2 db 23, 22 +pb_23_22: db 23, 22 + times 3 db 0, 32 pb_1: times 4 db 1 hmul_bits: dw 32768, 16384, 8192, 4096 round: dw 2048, 1024, 512 mul_bits: dw 256, 128, 64, 32, 16 round_vals: dw 32, 64, 128, 256, 512 max: dw 255, 240, 235 min: dw 0, 16 pb_27_17_17_27: db 27, 17, 17, 27 + times 2 db 0, 32 pw_1: dw 1 -%macro JMP_TABLE 1-* - %xdefine %1_table %%table - %xdefine %%base %1_table - %xdefine %%prefix mangle(private_prefix %+ _%1) +%macro JMP_TABLE 2-* + %xdefine %1_8bpc_%2_table %%table + %xdefine %%base %1_8bpc_%2_table + %xdefine %%prefix mangle(private_prefix %+ _%1_8bpc_%2) %%table: - %rep %0 - 1 - dd %%prefix %+ .ar%2 - %%base + %rep %0 - 2 + dd %%prefix %+ .ar%3 - %%base %rotate 1 %endrep %endmacro ALIGN 4 -JMP_TABLE generate_grain_y_avx2, 0, 1, 2, 3 -JMP_TABLE generate_grain_uv_420_avx2, 0, 1, 2, 3 -JMP_TABLE generate_grain_uv_422_avx2, 0, 1, 2, 3 -JMP_TABLE generate_grain_uv_444_avx2, 0, 1, 2, 3 +JMP_TABLE generate_grain_y, avx2, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_420, avx2, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_422, avx2, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_444, avx2, 0, 1, 2, 3 struc FGData .seed: resd 1 .num_y_points: resd 1 .y_points: resb 14 * 2 @@ -88,12 +90,20 @@ cextern gaussian_sequence SECTION .text +%macro REPX 2-* + %xdefine %%f(x) %1 +%rep %0 - 1 + %rotate 1 + %%f(%1) +%endrep +%endmacro + INIT_XMM avx2 -cglobal generate_grain_y, 2, 9, 16, buf, fg_data +cglobal generate_grain_y_8bpc, 2, 9, 16, buf, fg_data lea r4, [pb_mask] %define base r4-pb_mask movq xm1, [base+rnd_next_upperbit_mask] movq xm4, [base+mul_bits] movq xm7, [base+hmul_bits] @@ -130,12 +140,12 @@ add r2, 4 jl .loop ; auto-regression code movsxd r2, [fg_dataq+FGData.ar_coeff_lag] - movsxd r2, [base+generate_grain_y_avx2_table+r2*4] - lea r2, [r2+base+generate_grain_y_avx2_table] + movsxd r2, [base+generate_grain_y_8bpc_avx2_table+r2*4] + lea r2, [r2+base+generate_grain_y_8bpc_avx2_table] jmp r2 .ar1: DEFINE_ARGS buf, fg_data, cf3, shift, val3, min, max, x, val0 mov shiftd, [fg_dataq+FGData.ar_coeff_shift] @@ -418,11 +428,11 @@ jg .y_loop_ar3 RET %macro generate_grain_uv_fn 3 ; ss_name, ss_x, ss_y INIT_XMM avx2 -cglobal generate_grain_uv_%1, 4, 10, 16, buf, bufy, fg_data, uv +cglobal generate_grain_uv_%1_8bpc, 4, 10, 16, buf, bufy, fg_data, uv lea r4, [pb_mask] %define base r4-pb_mask movq xm1, [base+rnd_next_upperbit_mask] movq xm4, [base+mul_bits] movq xm7, [base+hmul_bits] @@ -476,12 +486,12 @@ jl .loop %endif ; auto-regression code movsxd r5, [fg_dataq+FGData.ar_coeff_lag] - movsxd r5, [base+generate_grain_uv_%1_avx2_table+r5*4] - lea r5, [r5+base+generate_grain_uv_%1_avx2_table] + movsxd r5, [base+generate_grain_uv_%1_8bpc_avx2_table+r5*4] + lea r5, [r5+base+generate_grain_uv_%1_8bpc_avx2_table] jmp r5 .ar0: INIT_YMM avx2 DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift @@ -973,11 +983,11 @@ generate_grain_uv_fn 420, 1, 1 generate_grain_uv_fn 422, 1, 0 generate_grain_uv_fn 444, 0, 0 INIT_YMM avx2 -cglobal fgy_32x32xn, 6, 13, 16, dst, src, stride, fg_data, w, scaling, grain_lut +cglobal fgy_32x32xn_8bpc, 6, 13, 16, dst, src, stride, fg_data, w, scaling, grain_lut pcmpeqw m10, m10 psrld m10, 24 mov r7d, [fg_dataq+FGData.scaling_shift] lea r8, [pb_mask] %define base r8-pb_mask @@ -1090,16 +1100,16 @@ lea srcq, [src_bakq+wq] test overlapd, overlapd jz .loop_x ; r8m = sbym - movd xm15, [pb_27_17_17_27] + movq xm15, [pb_27_17_17_27] cmp dword r8m, 0 jne .loop_x_hv_overlap ; horizontal overlap (without vertical overlap) - movd xm14, [pw_1024] + movq xm14, [pw_1024] .loop_x_h_overlap: mov r6d, seed or seed, 0xEFF4 shr r6d, 1 test seeb, seeh @@ -1154,12 +1164,11 @@ movd xm4, [grain_lutq+left_offxyq] punpcklbw xm4, xm3 pmaddubsw xm4, xm15, xm4 pmulhrsw xm4, xm14 packsswb xm4, xm4 - vpblendw xm4, xm3, 11111110b - vpblendd m3, m4, 00001111b + vpblendd m3, m3, m4, 00000001b pcmpgtb m7, m2, m3 punpcklbw m2, m3, m7 punpckhbw m3, m7 ; noise = round2(scaling[src] * grain, scaling_shift) @@ -1327,11 +1336,11 @@ ; since fg_dataq.overlap is guaranteed to be set, we never jump ; back to .loop_x_v_overlap, and instead always fall-through to ; h+v overlap - movd xm15, [pb_27_17_17_27] + movq xm15, [pb_27_17_17_27] .loop_x_hv_overlap: vpbroadcastw m8, [pb_27_17_17_27] ; we assume from the block above that bits 8-15 of r7d are zero'ed mov r6d, seed @@ -1407,14 +1416,12 @@ pmaddubsw xm7, xm15, xm7 pmulhrsw xm4, xm14 pmulhrsw xm7, xm14 packsswb xm4, xm4 packsswb xm7, xm7 - vpblendw xm4, xm3, 11111110b - vpblendw xm7, xm6, 11111110b - vpblendd m3, m4, 00001111b - vpblendd m6, m7, 00001111b + vpblendd m3, m4, 00000001b + vpblendd m6, m7, 00000001b ; followed by v interpolation (top | cur -> cur) punpckhbw m7, m6, m3 punpcklbw m6, m3 pmaddubsw m7, m8, m7 pmaddubsw m6, m8, m6 @@ -1459,14 +1466,12 @@ .end_hv: RET %macro FGUV_FN 3 ; name, ss_hor, ss_ver -cglobal fguv_32x32xn_i%1, 6, 15, 16, dst, src, stride, fg_data, w, scaling, \ +cglobal fguv_32x32xn_i%1_8bpc, 6, 15, 16, dst, src, stride, fg_data, w, scaling, \ grain_lut, h, sby, luma, lstride, uv_pl, is_id - pcmpeqw m10, m10 - psrld m10, 24 mov r7d, [fg_dataq+FGData.scaling_shift] lea r8, [pb_mask] %define base r8-pb_mask vpbroadcastw m11, [base+mul_bits+r7*2-14] mov r7d, [fg_dataq+FGData.clip_to_restricted_range] @@ -1488,14 +1493,19 @@ punpcklbw m14, m1, m0 vpbroadcastw m15, [fg_dataq+FGData.uv_offset+r7*4] %else vpbroadcastd m14, [pw_1024] %if %2 - vpbroadcastd m15, [pb_23_22] + vpbroadcastq m15, [pb_23_22] %else - vpbroadcastd xm15, [pb_27_17_17_27] + vpbroadcastq xm15, [pb_27_17_17_27] +%endif %endif +%if %3 + vpbroadcastw m10, [pb_23_22] +%elif %2 + mova m10, [pb_8x_27_17_8x_17_27] %endif mov overlapd, [fg_dataq+FGData.overlap_flag] movifnidn sbyd, sbym test sbyd, sbyd @@ -1591,20 +1601,17 @@ punpcklwd m6, m2 ; m4-7: luma_src as dword ; scaling[luma_src] pcmpeqw m3, m3 pcmpeqw m9, m9 - vpgatherdd m8, [scalingq+m4], m3 - vpgatherdd m4, [scalingq+m5], m9 + vpgatherdd m8, [scalingq-3+m4], m3 + vpgatherdd m4, [scalingq-3+m5], m9 pcmpeqw m3, m3 pcmpeqw m9, m9 - vpgatherdd m5, [scalingq+m6], m3 - vpgatherdd m6, [scalingq+m7], m9 - pand m8, m10 - pand m4, m10 - pand m5, m10 - pand m6, m10 + vpgatherdd m5, [scalingq-3+m6], m3 + vpgatherdd m6, [scalingq-3+m7], m9 + REPX {psrld x, 24}, m8, m4, m5, m6 packusdw m8, m4 packusdw m5, m6 ; unpack chroma_source punpckhbw m1, m0, m2 @@ -1741,31 +1748,28 @@ punpcklwd m6, m2 ; m4-7: luma_src as dword ; scaling[luma_src] pcmpeqw m3, m3 pcmpeqw m9, m9 - vpgatherdd m8, [scalingq+m4], m3 - vpgatherdd m4, [scalingq+m5], m9 + vpgatherdd m8, [scalingq-3+m4], m3 + vpgatherdd m4, [scalingq-3+m5], m9 pcmpeqw m3, m3 pcmpeqw m9, m9 - vpgatherdd m5, [scalingq+m6], m3 - vpgatherdd m6, [scalingq+m7], m9 - pand m8, m10 - pand m4, m10 - pand m5, m10 - pand m6, m10 + vpgatherdd m5, [scalingq-3+m6], m3 + vpgatherdd m6, [scalingq-3+m7], m9 + REPX {psrld x, 24}, m8, m4, m5, m6 packusdw m8, m4 packusdw m5, m6 ; unpack chroma_source punpckhbw m1, m0, m2 punpcklbw m0, m2 ; m0-1: src as word ; grain = grain_lut[offy+y][offx+x] %if %2 %if %1 - vpbroadcastd m6, [pb_23_22] ; FIXME + vpbroadcastq m6, [pb_23_22] %endif movu xm3, [grain_lutq+offxyq+ 0] movd xm4, [grain_lutq+left_offxyq+ 0] vinserti128 m3, [grain_lutq+offxyq+82], 1 vinserti128 m4, [grain_lutq+left_offxyq+82], 1 @@ -1776,16 +1780,14 @@ %else pmaddubsw m4, m15, m4 pmulhrsw m4, m14 %endif packsswb m4, m4 - pcmpeqw m6, m6 ; FIXME - psrldq m6, 15 ; FIXME - vpblendvb m3, m3, m4, m6 + vpblendd m3, m3, m4, 00010001b %else %if %1 - vpbroadcastd xm6, [pb_27_17_17_27] + movq xm6, [pb_27_17_17_27] %endif movu m3, [grain_lutq+offxyq] movd xm4, [grain_lutq+left_offxyq] punpcklbw xm4, xm3 %if %1 @@ -1794,13 +1796,11 @@ %else pmaddubsw xm4, xm15, xm4 pmulhrsw xm4, xm14 %endif packsswb xm4, xm4 - pcmpeqw xm6, xm6 - psrldq xm6, 14 - vpblendvb m3, m3, m4, m6 + vpblendd m3, m3, m4, 00000001b %endif pcmpgtb m7, m2, m3 punpcklbw m2, m3, m7 punpckhbw m3, m7 @@ -1913,11 +1913,11 @@ shr offxyd, 16 mov hd, hm mov grain_lutq, grain_lutmp %if %2 == 0 - vbroadcasti128 m1, [pb_8x_27_17_8x_17_27] + vbroadcasti128 m10, [pb_8x_27_17_8x_17_27] %endif %%loop_y_v_overlap: ; src %if %2 mova xm4, [lumaq+lstrideq*0+ 0] @@ -1964,20 +1964,17 @@ punpcklwd m6, m2 ; m4-7: luma_src as dword ; scaling[luma_src] pcmpeqw m3, m3 pcmpeqw m9, m9 - vpgatherdd m8, [scalingq+m4], m3 - vpgatherdd m4, [scalingq+m5], m9 + vpgatherdd m8, [scalingq-3+m4], m3 + vpgatherdd m4, [scalingq-3+m5], m9 pcmpeqw m3, m3 pcmpeqw m9, m9 - vpgatherdd m5, [scalingq+m6], m3 - vpgatherdd m6, [scalingq+m7], m9 - pand m8, m10 - pand m4, m10 - pand m5, m10 - pand m6, m10 + vpgatherdd m5, [scalingq-3+m6], m3 + vpgatherdd m6, [scalingq-3+m7], m9 + REPX {psrld x, 24}, m8, m4, m5, m6 packusdw m8, m4 packusdw m5, m6 %if %2 ; unpack chroma_source @@ -1986,50 +1983,40 @@ %endif ; grain = grain_lut[offy+y][offx+x] %if %3 == 0 %if %2 - mova m6, [pb_8x_27_17_8x_17_27] movu xm3, [grain_lutq+offxyq] movu xm4, [grain_lutq+top_offxyq] vinserti128 m3, [grain_lutq+offxyq+82], 1 vinserti128 m4, [grain_lutq+top_offxyq+82], 1 %else movu m3, [grain_lutq+offxyq] movu m4, [grain_lutq+top_offxyq] %endif punpckhbw m9, m4, m3 punpcklbw m4, m3 -%if %2 - pmaddubsw m9, m6, m9 - pmaddubsw m4, m6, m4 -%else - pmaddubsw m9, m1, m9 - pmaddubsw m4, m1, m4 -%endif + pmaddubsw m9, m10, m9 + pmaddubsw m4, m10, m4 %if %1 pmulhrsw m9, [pw_1024] pmulhrsw m4, [pw_1024] %else pmulhrsw m9, m14 pmulhrsw m4, m14 %endif packsswb m3, m4, m9 %else -%if %1 - vpbroadcastd m6, [pb_23_22] -%endif movq xm3, [grain_lutq+offxyq] movq xm4, [grain_lutq+top_offxyq] vinserti128 m3, [grain_lutq+offxyq+8], 1 vinserti128 m4, [grain_lutq+top_offxyq+8], 1 punpcklbw m4, m3 + pmaddubsw m4, m10, m4 %if %1 - pmaddubsw m4, m6, m4 pmulhrsw m4, [pw_1024] %else - pmaddubsw m4, m15, m4 pmulhrsw m4, m14 %endif packsswb m4, m4 vpermq m4, m4, q3120 ; only interpolate first line, insert second line unmodified @@ -2082,11 +2069,11 @@ add dstq, strideq add lumaq, lstrideq %endif add grain_lutq, 82<<%2 %if %2 == 0 - vbroadcasti128 m1, [pb_8x_27_17_8x_17_27+16] + vbroadcasti128 m10, [pb_8x_27_17_8x_17_27+16] btc hd, 16 jnc %%loop_y_v_overlap %endif jmp %%loop_y @@ -2137,11 +2124,11 @@ shr offxyd, 16 mov hd, hm mov grain_lutq, grain_lutmp %if %2 == 0 - vbroadcasti128 m1, [pb_8x_27_17_8x_17_27] + vbroadcasti128 m10, [pb_8x_27_17_8x_17_27] %endif %%loop_y_hv_overlap: ; src %if %2 mova xm4, [lumaq+lstrideq*0+ 0] @@ -2188,20 +2175,17 @@ punpcklwd m6, m2 ; m4-7: src as dword ; scaling[src] pcmpeqw m9, m9 pcmpeqw m3, m3 - vpgatherdd m8, [scalingq+m4], m9 - vpgatherdd m4, [scalingq+m5], m3 + vpgatherdd m8, [scalingq-3+m4], m9 + vpgatherdd m4, [scalingq-3+m5], m3 pcmpeqw m9, m9 pcmpeqw m3, m3 - vpgatherdd m5, [scalingq+m6], m9 - vpgatherdd m6, [scalingq+m7], m3 - pand m8, m10 - pand m4, m10 - pand m5, m10 - pand m6, m10 + vpgatherdd m5, [scalingq-3+m6], m9 + vpgatherdd m6, [scalingq-3+m7], m3 + REPX {psrld x, 24}, m8, m4, m5, m6 packusdw m8, m4 packusdw m5, m6 %if %2 ; unpack chroma source @@ -2210,13 +2194,13 @@ %endif ; grain = grain_lut[offy+y][offx+x] %if %1 %if %2 - vpbroadcastd m9, [pb_23_22] + vpbroadcastq m9, [pb_23_22] %else - vpbroadcastd xm9, [pb_27_17_17_27] + vpbroadcastq xm9, [pb_27_17_17_27] %endif %endif %if %2 movu xm3, [grain_lutq+offxyq] @@ -2250,72 +2234,61 @@ %if %3 punpcklbw xm7, xm6 %else punpcklbw m7, m6 %endif - punpcklwd m4, m7 + punpcklqdq m4, m7 %if %1 pmaddubsw m4, m9, m4 pmulhrsw m4, [pw_1024] %else pmaddubsw m4, m15, m4 pmulhrsw m4, m14 %endif packsswb m4, m4 - pcmpeqw m9, m9 ; this is kind of ugly - psrldq m9, 15 - vpblendvb m3, m3, m4, m9 - psrldq m4, 1 + vpblendd m3, m4, 00010001b + psrldq m4, 4 %if %3 - shufpd m9, m9, m9, 1110b ; clear upper lane + vpblendd m6, m6, m4, 00000001b +%else + vpblendd m6, m6, m4, 00010001b %endif - vpblendvb m6, m6, m4, m9 %else punpcklbw xm4, xm3 punpcklbw xm7, xm6 - punpckldq xm4, xm7 + punpcklqdq xm4, xm7 %if %1 pmaddubsw xm4, xm9, xm4 pmulhrsw xm4, [pw_1024] %else pmaddubsw xm4, xm15, xm4 pmulhrsw xm4, xm14 %endif packsswb xm4, xm4 - pcmpeqw xm9, xm9 ; this is kind of ugly - psrldq xm9, 14 - vpblendvb m3, m3, m4, m9 - psrldq xm4, 2 - vpblendvb m6, m6, m4, m9 + vpblendd m3, m3, m4, 00000001b + psrldq xm4, 4 + vpblendd m6, m6, m4, 00000001b %endif ; followed by v interpolation (top | cur -> cur) %if %3 vpermq m9, m3, q3120 punpcklbw m6, m9 + pmaddubsw m6, m10, m6 %if %1 - vpbroadcastd m9, [pb_23_22] - pmaddubsw m6, m9, m6 pmulhrsw m6, [pw_1024] %else - pmaddubsw m6, m15, m6 pmulhrsw m6, m14 %endif packsswb m6, m6 vpermq m6, m6, q3120 vpblendd m3, m3, m6, 00001111b %else punpckhbw m9, m6, m3 punpcklbw m6, m3 -%if %2 - mova m3, [pb_8x_27_17_8x_17_27] - pmaddubsw m9, m3, m9 - pmaddubsw m6, m3, m6 -%else - pmaddubsw m9, m1, m9 - pmaddubsw m6, m1, m6 -%endif + pmaddubsw m9, m10, m9 + pmaddubsw m6, m10, m6 %if %1 pmulhrsw m9, [pw_1024] pmulhrsw m6, [pw_1024] %else pmulhrsw m9, m14 @@ -2371,11 +2344,11 @@ sub hb, 1+%2 %if %2 jg %%loop_y_h_overlap %else je %%end_y_hv_overlap - vbroadcasti128 m1, [pb_8x_27_17_8x_17_27+16] + vbroadcasti128 m10, [pb_8x_27_17_8x_17_27+16] btc hd, 16 jnc %%loop_y_hv_overlap jmp %%loop_y_h_overlap %endif diff --git a/third_party/dav1d/src/x86/film_grain_init_tmpl.c b/third_party/dav1d/src/x86/film_grain_init_tmpl.c --- a/third_party/dav1d/src/x86/film_grain_init_tmpl.c +++ b/third_party/dav1d/src/x86/film_grain_init_tmpl.c @@ -1,7 +1,7 @@ /* - * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018-2021, VideoLAN and dav1d authors * Copyright © 2018, Two Orioles, LLC * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -26,66 +26,50 @@ */ #include "src/cpu.h" #include "src/film_grain.h" -decl_generate_grain_y_fn(dav1d_generate_grain_y_ssse3); -decl_generate_grain_uv_fn(dav1d_generate_grain_uv_420_ssse3); -decl_generate_grain_uv_fn(dav1d_generate_grain_uv_422_ssse3); -decl_generate_grain_uv_fn(dav1d_generate_grain_uv_444_ssse3); -decl_fgy_32x32xn_fn(dav1d_fgy_32x32xn_ssse3); -decl_fguv_32x32xn_fn(dav1d_fguv_32x32xn_i420_ssse3); -decl_fguv_32x32xn_fn(dav1d_fguv_32x32xn_i422_ssse3); -decl_fguv_32x32xn_fn(dav1d_fguv_32x32xn_i444_ssse3); +decl_generate_grain_y_fn(BF(dav1d_generate_grain_y, ssse3)); +decl_generate_grain_uv_fn(BF(dav1d_generate_grain_uv_420, ssse3)); +decl_generate_grain_uv_fn(BF(dav1d_generate_grain_uv_422, ssse3)); +decl_generate_grain_uv_fn(BF(dav1d_generate_grain_uv_444, ssse3)); +decl_fgy_32x32xn_fn(BF(dav1d_fgy_32x32xn, ssse3)); +decl_fguv_32x32xn_fn(BF(dav1d_fguv_32x32xn_i420, ssse3)); +decl_fguv_32x32xn_fn(BF(dav1d_fguv_32x32xn_i422, ssse3)); +decl_fguv_32x32xn_fn(BF(dav1d_fguv_32x32xn_i444, ssse3)); -decl_generate_grain_y_fn(dav1d_generate_grain_y_avx2); -decl_generate_grain_uv_fn(dav1d_generate_grain_uv_420_avx2); -decl_generate_grain_uv_fn(dav1d_generate_grain_uv_422_avx2); -decl_generate_grain_uv_fn(dav1d_generate_grain_uv_444_avx2); -decl_fgy_32x32xn_fn(dav1d_fgy_32x32xn_avx2); -decl_fguv_32x32xn_fn(dav1d_fguv_32x32xn_i420_avx2); -decl_fguv_32x32xn_fn(dav1d_fguv_32x32xn_i422_avx2); -decl_fguv_32x32xn_fn(dav1d_fguv_32x32xn_i444_avx2); - -decl_generate_grain_y_fn(dav1d_generate_grain_y_16bpc_avx2); -decl_generate_grain_uv_fn(dav1d_generate_grain_uv_420_16bpc_avx2); -decl_fgy_32x32xn_fn(dav1d_fgy_32x32xn_16bpc_avx2); -decl_fguv_32x32xn_fn(dav1d_fguv_32x32xn_i420_16bpc_avx2); +decl_generate_grain_y_fn(BF(dav1d_generate_grain_y, avx2)); +decl_generate_grain_uv_fn(BF(dav1d_generate_grain_uv_420, avx2)); +decl_generate_grain_uv_fn(BF(dav1d_generate_grain_uv_422, avx2)); +decl_generate_grain_uv_fn(BF(dav1d_generate_grain_uv_444, avx2)); +decl_fgy_32x32xn_fn(BF(dav1d_fgy_32x32xn, avx2)); +decl_fguv_32x32xn_fn(BF(dav1d_fguv_32x32xn_i420, avx2)); +decl_fguv_32x32xn_fn(BF(dav1d_fguv_32x32xn_i422, avx2)); +decl_fguv_32x32xn_fn(BF(dav1d_fguv_32x32xn_i444, avx2)); COLD void bitfn(dav1d_film_grain_dsp_init_x86)(Dav1dFilmGrainDSPContext *const c) { const unsigned flags = dav1d_get_cpu_flags(); if (!(flags & DAV1D_X86_CPU_FLAG_SSSE3)) return; -#if BITDEPTH == 8 - c->generate_grain_y = dav1d_generate_grain_y_ssse3; - c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I420 - 1] = dav1d_generate_grain_uv_420_ssse3; - c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I422 - 1] = dav1d_generate_grain_uv_422_ssse3; - c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I444 - 1] = dav1d_generate_grain_uv_444_ssse3; - c->fgy_32x32xn = dav1d_fgy_32x32xn_ssse3; - c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I420 - 1] = dav1d_fguv_32x32xn_i420_ssse3; - c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I422 - 1] = dav1d_fguv_32x32xn_i422_ssse3; - c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I444 - 1] = dav1d_fguv_32x32xn_i444_ssse3; -#endif + c->generate_grain_y = BF(dav1d_generate_grain_y, ssse3); + c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I420 - 1] = BF(dav1d_generate_grain_uv_420, ssse3); + c->fgy_32x32xn = BF(dav1d_fgy_32x32xn, ssse3); + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I420 - 1] = BF(dav1d_fguv_32x32xn_i420, ssse3); + c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I422 - 1] = BF(dav1d_generate_grain_uv_422, ssse3); + c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I444 - 1] = BF(dav1d_generate_grain_uv_444, ssse3); + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I422 - 1] = BF(dav1d_fguv_32x32xn_i422, ssse3); + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I444 - 1] = BF(dav1d_fguv_32x32xn_i444, ssse3); #if ARCH_X86_64 if (!(flags & DAV1D_X86_CPU_FLAG_AVX2)) return; -#if BITDEPTH == 8 - c->generate_grain_y = dav1d_generate_grain_y_avx2; - c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I420 - 1] = dav1d_generate_grain_uv_420_avx2; - c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I422 - 1] = dav1d_generate_grain_uv_422_avx2; - c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I444 - 1] = dav1d_generate_grain_uv_444_avx2; - c->fgy_32x32xn = dav1d_fgy_32x32xn_avx2; - c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I420 - 1] = dav1d_fguv_32x32xn_i420_avx2; - c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I422 - 1] = dav1d_fguv_32x32xn_i422_avx2; - c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I444 - 1] = dav1d_fguv_32x32xn_i444_avx2; -#else - c->generate_grain_y = dav1d_generate_grain_y_16bpc_avx2; - c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I420 - 1] = - dav1d_generate_grain_uv_420_16bpc_avx2; - c->fgy_32x32xn = dav1d_fgy_32x32xn_16bpc_avx2; - c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I420 - 1] = - dav1d_fguv_32x32xn_i420_16bpc_avx2; -#endif + c->generate_grain_y = BF(dav1d_generate_grain_y, avx2); + c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I420 - 1] = BF(dav1d_generate_grain_uv_420, avx2); + c->fgy_32x32xn = BF(dav1d_fgy_32x32xn, avx2); + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I420 - 1] = BF(dav1d_fguv_32x32xn_i420, avx2); + c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I422 - 1] = BF(dav1d_generate_grain_uv_422, avx2); + c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I444 - 1] = BF(dav1d_generate_grain_uv_444, avx2); + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I422 - 1] = BF(dav1d_fguv_32x32xn_i422, avx2); + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I444 - 1] = BF(dav1d_fguv_32x32xn_i444, avx2); #endif } diff --git a/third_party/dav1d/src/x86/film_grain_sse.asm b/third_party/dav1d/src/x86/film_grain_sse.asm --- a/third_party/dav1d/src/x86/film_grain_sse.asm +++ b/third_party/dav1d/src/x86/film_grain_sse.asm @@ -1,6 +1,6 @@ -; Copyright © 2019, VideoLAN and dav1d authors +; Copyright © 2019-2021, VideoLAN and dav1d authors ; Copyright © 2019, Two Orioles, LLC ; All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions are met: @@ -27,44 +27,46 @@ %include "ext/x86/x86inc.asm" SECTION_RODATA pw_1024: times 8 dw 1024 +pb_27_17_17_27: db 27, 17, 17, 27 + times 6 db 0, 32 +pb_23_22_h: db 23, 22 + times 7 db 0, 32 pb_27_17: times 8 db 27, 17 pb_17_27: times 8 db 17, 27 +pb_23_22: times 8 db 23, 22 pb_mask: db 0, 0x80, 0x80, 0, 0x80, 0, 0, 0x80, 0x80, 0, 0, 0x80, 0, 0x80, 0x80, 0 rnd_next_upperbit_mask: dw 0x100B, 0x2016, 0x402C, 0x8058 byte_blend: db 0, 0, 0, 0xff, 0, 0, 0, 0 pw_seed_xor: times 2 dw 0xb524 times 2 dw 0x49d8 -pb_23_22: times 2 db 23, 22 pb_1: times 4 db 1 hmul_bits: dw 32768, 16384, 8192, 4096 round: dw 2048, 1024, 512 mul_bits: dw 256, 128, 64, 32, 16 round_vals: dw 32, 64, 128, 256, 512 max: dw 255, 240, 235 min: dw 0, 16 pw_1: dw 1 -%define pb_27_17_17_27 pb_17_27 - 2 - -%macro JMP_TABLE 1-* - %xdefine %1_table %%table - %xdefine %%base %1_table - %xdefine %%prefix mangle(private_prefix %+ _%1) +%macro JMP_TABLE 2-* + %xdefine %1_8bpc_%2_table %%table + %xdefine %%base %1_8bpc_%2_table + %xdefine %%prefix mangle(private_prefix %+ _%1_8bpc_%2) %%table: - %rep %0 - 1 - dd %%prefix %+ .ar%2 - %%base + %rep %0 - 2 + dd %%prefix %+ .ar%3 - %%base %rotate 1 %endrep %endmacro -JMP_TABLE generate_grain_y_ssse3, 0, 1, 2, 3 -JMP_TABLE generate_grain_uv_420_ssse3, 0, 1, 2, 3 -JMP_TABLE generate_grain_uv_422_ssse3, 0, 1, 2, 3 -JMP_TABLE generate_grain_uv_444_ssse3, 0, 1, 2, 3 +JMP_TABLE generate_grain_y, ssse3, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_420, ssse3, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_422, ssse3, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_444, ssse3, 0, 1, 2, 3 struc FGData .seed: resd 1 .num_y_points: resd 1 .y_points: resb 14 * 2 @@ -86,21 +88,35 @@ cextern gaussian_sequence SECTION .text +%macro REPX 2-* + %xdefine %%f(x) %1 +%rep %0 - 1 + %rotate 1 + %%f(%1) +%endrep +%endmacro + +%if ARCH_X86_32 +%define PIC_ptr(a) base+a +%else +%define PIC_ptr(a) a +%endif + %macro SCRATCH 3 %if ARCH_X86_32 mova [rsp+%3*mmsize], m%1 %define m%2 [rsp+%3*mmsize] %else SWAP %1, %2 %endif %endmacro INIT_XMM ssse3 -cglobal generate_grain_y, 2, 7 + 2 * ARCH_X86_64, 16, buf, fg_data +cglobal generate_grain_y_8bpc, 2, 7 + 2 * ARCH_X86_64, 16, buf, fg_data LEA r4, $$ %define base r4-$$ movq m1, [base+rnd_next_upperbit_mask] movq m4, [base+mul_bits] movq m7, [base+hmul_bits] @@ -162,12 +178,12 @@ add r2, 4 jl .loop ; auto-regression code movsxd r2, [fg_dataq+FGData.ar_coeff_lag] - movsxd r2, [base+generate_grain_y_ssse3_table+r2*4] - lea r2, [r2+base+generate_grain_y_ssse3_table] + movsxd r2, [base+generate_grain_y_8bpc_ssse3_table+r2*4] + lea r2, [r2+base+generate_grain_y_8bpc_ssse3_table] jmp r2 .ar1: %if ARCH_X86_32 DEFINE_ARGS buf, fg_data, cf3, unused, val3, min, max @@ -505,11 +521,11 @@ jg .y_loop_ar3 RET %macro generate_grain_uv_fn 3 ; ss_name, ss_x, ss_y INIT_XMM ssse3 -cglobal generate_grain_uv_%1, 1, 7 + 3 * ARCH_X86_64, 16, buf, bufy, fg_data, uv +cglobal generate_grain_uv_%1_8bpc, 1, 7 + 3 * ARCH_X86_64, 16, buf, bufy, fg_data, uv movifnidn r2, r2mp movifnidn r3, r3mp LEA r4, $$ %define base r4-$$ movq m1, [base+rnd_next_upperbit_mask] @@ -604,12 +620,12 @@ mov r2, r2mp %endif ; auto-regression code movsxd r5, [fg_dataq+FGData.ar_coeff_lag] - movsxd r5, [base+generate_grain_uv_%1_ssse3_table+r5*4] - lea r5, [r5+base+generate_grain_uv_%1_ssse3_table] + movsxd r5, [base+generate_grain_uv_%1_8bpc_ssse3_table+r5*4] + lea r5, [r5+base+generate_grain_uv_%1_8bpc_ssse3_table] jmp r5 .ar0: DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift movifnidn bufyq, bufymp @@ -1282,70 +1298,67 @@ INIT_XMM ssse3 ; fgy_32x32xn(dst, src, stride, fg_data, w, scaling, grain_lut, h, sby) %if ARCH_X86_32 %if STACK_ALIGNMENT < mmsize -cglobal fgy_32x32xn, 0, 7, 16, 0 - (6 * mmsize + (9 + 3) * gprsize), \ +cglobal fgy_32x32xn_8bpc, 0, 7, 16, 0 - (5 * mmsize + 16 * gprsize), \ dst, src, scaling, unused1, fg_data, picptr, unused2 ; copy stack arguments to new position post-alignment, so that we ; don't have to keep the old stack location in a separate register mov r0, r0m mov r1, r2m mov r2, r4m mov r3, r6m mov r4, r7m mov r5, r8m - mov [rsp+6*mmsize+ 3*gprsize], r0 - mov [rsp+6*mmsize+ 5*gprsize], r1 - mov [rsp+6*mmsize+ 7*gprsize], r2 - mov [rsp+6*mmsize+ 9*gprsize], r3 - mov [rsp+6*mmsize+10*gprsize], r4 - mov [rsp+6*mmsize+11*gprsize], r5 + mov [rsp+5*mmsize+ 4*gprsize], r0 + mov [rsp+5*mmsize+ 6*gprsize], r1 + mov [rsp+5*mmsize+ 8*gprsize], r2 + mov [rsp+5*mmsize+10*gprsize], r3 + mov [rsp+5*mmsize+11*gprsize], r4 + mov [rsp+5*mmsize+12*gprsize], r5 %else -cglobal fgy_32x32xn, 0, 7, 16, 6 * mmsize + (3 + 1) * gprsize, \ +cglobal fgy_32x32xn_8bpc, 0, 7, 16, 5 * mmsize + 4 * gprsize, \ dst, src, scaling, unused1, fg_data, picptr, unused2 %endif mov srcq, srcm mov fg_dataq, r3m mov scalingq, r5m %if STACK_ALIGNMENT < mmsize -%define r0m [rsp+6*mmsize+ 3*gprsize] -%define r1m [rsp+6*mmsize+ 4*gprsize] -%define r2m [rsp+6*mmsize+ 5*gprsize] -%define r3m [rsp+6*mmsize+ 6*gprsize] -%define r4m [rsp+6*mmsize+ 7*gprsize] -%define r5m [rsp+6*mmsize+ 8*gprsize] -%define r6m [rsp+6*mmsize+ 9*gprsize] -%define r7m [rsp+6*mmsize+10*gprsize] -%define r8m [rsp+6*mmsize+11*gprsize] +%define r0m [rsp+5*mmsize+ 4*gprsize] +%define r1m [rsp+5*mmsize+ 5*gprsize] +%define r2m [rsp+5*mmsize+ 6*gprsize] +%define r3m [rsp+5*mmsize+ 7*gprsize] +%define r4m [rsp+5*mmsize+ 8*gprsize] +%define r5m [rsp+5*mmsize+ 9*gprsize] +%define r6m [rsp+5*mmsize+10*gprsize] +%define r7m [rsp+5*mmsize+11*gprsize] +%define r8m [rsp+5*mmsize+12*gprsize] %endif LEA r5, pb_mask %define base r5-pb_mask mov r5m, picptrq %else -cglobal fgy_32x32xn, 6, 15, 16, dst, src, stride, fg_data, w, scaling, grain_lut +cglobal fgy_32x32xn_8bpc, 6, 15, 16, dst, src, stride, fg_data, w, scaling, grain_lut lea r7, [pb_mask] %define base r7-pb_mask %endif mov r6d, [fg_dataq+FGData.scaling_shift] movd m3, [base+mul_bits+r6*2-14] mov r6d, [fg_dataq+FGData.clip_to_restricted_range] - pcmpeqw m2, m2 - psrldq m2, 14 movd m4, [base+max+r6*4] movd m5, [base+min+r6*2] punpcklwd m3, m3 punpcklwd m4, m4 punpcklwd m5, m5 pshufd m3, m3, q0000 pshufd m4, m4, q0000 pshufd m5, m5, q0000 - SCRATCH 2, 10, 0 - SCRATCH 3, 11, 1 - SCRATCH 4, 12, 2 - SCRATCH 5, 13, 3 + SCRATCH 3, 11, 0 + SCRATCH 4, 12, 1 + SCRATCH 5, 13, 2 %if ARCH_X86_32 DEFINE_ARGS dst, src, scaling, sby, fg_data, picptr, overlap %else DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused, sby, see, overlap @@ -1354,13 +1367,13 @@ mov sbyd, r8m mov overlapd, [fg_dataq+FGData.overlap_flag] ; left_overlap: overlap & 1 test overlapd, overlapd jz .no_vertical_overlap mova m6, [base+pw_1024] - movd m7, [base+pb_27_17_17_27] - SCRATCH 6, 14, 4 - SCRATCH 7, 15, 5 + mova m7, [base+pb_27_17_17_27] + SCRATCH 6, 14, 3 + SCRATCH 7, 15, 4 test sbyd, sbyd jnz .vertical_overlap ; fall-through .no_vertical_overlap: @@ -1443,20 +1456,17 @@ punpckhbw m1, m0, m2 punpcklbw m0, m2 ; m0-1: src as word ; scaling[src] %if ARCH_X86_32 - vpgatherdw m4, m0, scalingq, r0, r5, m3 - vpgatherdw m5, m1, scalingq, r0, r5, m3 + vpgatherdw m4, m0, scalingq-1, r0, r5, m3 + vpgatherdw m5, m1, scalingq-1, r0, r5, m3 %else - vpgatherdw m4, m0, scalingq, r12, r13, m3 - vpgatherdw m5, m1, scalingq, r12, r13, m3 + vpgatherdw m4, m0, scalingq-1, r12, r13, m3 + vpgatherdw m5, m1, scalingq-1, r12, r13, m3 %endif - pcmpeqw m3, m3 - psrlw m3, 8 - pand m4, m3 - pand m5, m3 + REPX {psrlw x, 8}, m4, m5 ; grain = grain_lut[offy+y][offx+x] movu m3, [grain_lutq+offxyq] pcmpgtb m7, m2, m3 punpcklbw m2, m3, m7 @@ -1502,11 +1512,11 @@ add offxyd, 16 test dword r8m, 2 ; r8m & 2 = have_top_overlap jz .loop_x_odd %if ARCH_X86_32 - add dword [rsp+6*mmsize+1*gprsize], 16 + add dword [rsp+5*mmsize+1*gprsize], 16 %else add r11d, 16 ; top_offxyd %endif jnz .loop_x_odd_v_overlap @@ -1523,11 +1533,11 @@ ; r0m=dst, r1m=src_bak, r2m=stride, r3m=see, r4m=w, r5m=picptr, ; r6m=grain_lut, r7m=h, r8m=overlap_v|h DEFINE_ARGS dst, src, scaling, offxy, unused1, unused2, unused3 add offxyd, 16 ; left_offxyd - mov [rsp+6*mmsize+0*gprsize], offxyd + mov [rsp+5*mmsize+0*gprsize], offxyd DEFINE_ARGS dst, src, scaling, see, unused1, unused2, unused3 mov seed, r3m %else @@ -1576,36 +1586,31 @@ punpckhbw m1, m0, m2 punpcklbw m0, m2 ; m0-1: src as word ; scaling[src] %if ARCH_X86_32 - vpgatherdw m4, m0, scalingq, r0, r5, m3 - vpgatherdw m5, m1, scalingq, r0, r5, m3 + vpgatherdw m4, m0, scalingq-1, r0, r5, m3 + vpgatherdw m5, m1, scalingq-1, r0, r5, m3 %else - vpgatherdw m4, m0, scalingq, r12, r13, m3 - vpgatherdw m5, m1, scalingq, r12, r13, m3 + vpgatherdw m4, m0, scalingq-1, r12, r13, m3 + vpgatherdw m5, m1, scalingq-1, r12, r13, m3 %endif - pcmpeqw m3, m3 - psrlw m3, 8 - pand m4, m3 - pand m5, m3 + REPX {psrlw x, 8}, m4, m5 ; grain = grain_lut[offy+y][offx+x] movu m3, [grain_lutq+offxyq] %if ARCH_X86_32 - mov r5, [rsp+6*mmsize+0*gprsize] + mov r5, [rsp+5*mmsize+0*gprsize] movd m7, [grain_lutq+r5] %else movd m7, [grain_lutq+left_offxyq] %endif punpcklbw m7, m3 pmaddubsw m6, m15, m7 pmulhrsw m6, m14 packsswb m6, m6 - pand m6, m10 - pandn m7, m10, m3 - por m6, m7 + shufps m6, m3, q3210 pcmpgtb m2, m6 punpcklbw m7, m6, m2 punpckhbw m6, m2 ; noise = round2(scaling[src] * grain, scaling_shift) @@ -1647,11 +1652,11 @@ ; since this half-block had left-overlap, the next does not test dword r8m, 2 ; have_top_overlap jz .loop_x_odd %if ARCH_X86_32 - add dword [rsp+6*mmsize+1*gprsize], 16 + add dword [rsp+5*mmsize+1*gprsize], 16 %else add r11d, 16 ; top_offxyd %endif jmp .loop_x_odd_v_overlap @@ -1752,21 +1757,21 @@ h, offxy, see, unused, top_offxy %endif movzx top_offxyd, offxyw %if ARCH_X86_32 - mov [rsp+6*mmsize+1*gprsize], top_offxyd + mov [rsp+5*mmsize+1*gprsize], top_offxyd DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut %endif shr offxyd, 16 .loop_x_odd_v_overlap: %if ARCH_X86_32 mov r5, r5m lea r5, [base+pb_27_17] - mov [rsp+5*mmsize+8], r5 + mov [rsp+5*mmsize+12], r5 %else mova m8, [pb_27_17] %endif mov hd, r7m mov grain_lutq, grain_lutmp @@ -1777,33 +1782,30 @@ punpckhbw m1, m0, m2 punpcklbw m0, m2 ; m0-1: src as word ; scaling[src] %if ARCH_X86_32 - vpgatherdw m4, m0, scalingq, r0, r5, m3 - vpgatherdw m5, m1, scalingq, r0, r5, m3 + vpgatherdw m4, m0, scalingq-1, r0, r5, m3 + vpgatherdw m5, m1, scalingq-1, r0, r5, m3 %else - vpgatherdw m4, m0, scalingq, r12, r13, m3 - vpgatherdw m5, m1, scalingq, r12, r13, m3 + vpgatherdw m4, m0, scalingq-1, r12, r13, m3 + vpgatherdw m5, m1, scalingq-1, r12, r13, m3 %endif - pcmpeqw m3, m3 - psrlw m3, 8 - pand m4, m3 - pand m5, m3 + REPX {psrlw x, 8}, m4, m5 ; grain = grain_lut[offy+y][offx+x] movu m3, [grain_lutq+offxyq] %if ARCH_X86_32 - mov r5, [rsp+6*mmsize+1*gprsize] + mov r5, [rsp+5*mmsize+1*gprsize] movu m7, [grain_lutq+r5] %else movu m7, [grain_lutq+top_offxyq] %endif punpckhbw m6, m7, m3 punpcklbw m7, m3 %if ARCH_X86_32 - mov r5, [rsp+5*mmsize+8] + mov r5, [rsp+5*mmsize+12] pmaddubsw m3, [r5], m6 pmaddubsw m6, [r5], m7 %else pmaddubsw m3, m8, m6 pmaddubsw m6, m8, m7 @@ -1831,11 +1833,11 @@ packuswb m0, m1 movifnidn dstq, dstmp mova [dstq+srcq], m0 %if ARCH_X86_32 - add dword [rsp+5*mmsize+8], mmsize + add dword [rsp+5*mmsize+12], mmsize %else mova m8, [pb_17_27] %endif add srcq, r2mp add grain_lutq, 82 @@ -1862,30 +1864,30 @@ %endif btc dword r8m, 2 jc .loop_x_hv_overlap add offxyd, 16 %if ARCH_X86_32 - add dword [rsp+6*mmsize+1*gprsize], 16 + add dword [rsp+5*mmsize+1*gprsize], 16 %else add top_offxyd, 16 %endif jmp .loop_x_odd_v_overlap .loop_x_hv_overlap: %if ARCH_X86_32 mov r5, r5m lea r5, [base+pb_27_17] - mov [rsp+5*mmsize+8], r5 + mov [rsp+5*mmsize+12], r5 DEFINE_ARGS tmp, src, scaling, offxy, w, picptr, src_bak - mov r5, [rsp+6*mmsize+1*gprsize] + mov r5, [rsp+5*mmsize+1*gprsize] mov r4, offxyd add r5, 16 add r4, 16 - mov [rsp+6*mmsize+2*gprsize], r5 ; topleft_offxy - mov [rsp+6*mmsize+0*gprsize], r4 ; left_offxy + mov [rsp+5*mmsize+2*gprsize], r5 ; topleft_offxy + mov [rsp+5*mmsize+0*gprsize], r4 ; left_offxy DEFINE_ARGS tmp, src, scaling, see, w, picptr, src_bak xor tmpd, tmpd mov seed, r3m @@ -1935,11 +1937,11 @@ %if ARCH_X86_32 DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut movzx r5, offxyw ; top_offxy - mov [rsp+6*mmsize+1*gprsize], r5 + mov [rsp+5*mmsize+1*gprsize], r5 %else DEFINE_ARGS dst, src, stride, src_bak, w, scaling, grain_lut, \ h, offxy, see, left_offxy, top_offxy, topleft_offxy movzx top_offxyd, offxyw @@ -1950,14 +1952,14 @@ mov grain_lutq, grain_lutmp .loop_y_hv_overlap: ; grain = grain_lut[offy+y][offx+x] movu m3, [grain_lutq+offxyq] %if ARCH_X86_32 - mov r5, [rsp+6*mmsize+1*gprsize] ; top_offxy - mov r0, [rsp+6*mmsize+0*gprsize] ; left_offxy + mov r5, [rsp+5*mmsize+1*gprsize] ; top_offxy + mov r0, [rsp+5*mmsize+0*gprsize] ; left_offxy movu m6, [grain_lutq+r5] - mov r5, [rsp+6*mmsize+2*gprsize] ; topleft_offxy + mov r5, [rsp+5*mmsize+2*gprsize] ; topleft_offxy movd m4, [grain_lutq+r0] movd m7, [grain_lutq+r5] %else movu m6, [grain_lutq+top_offxyq] movd m4, [grain_lutq+left_offxyq] @@ -1970,21 +1972,17 @@ pmaddubsw m4, m15, m7 pmulhrsw m2, m14 pmulhrsw m4, m14 packsswb m2, m2 packsswb m4, m4 - pand m2, m10 - pand m4, m10 - pandn m7, m10, m3 - pandn m3, m10, m6 - por m7, m2 - por m3, m4 + shufps m2, m3, q3210 + shufps m4, m6, q3210 ; followed by v interpolation (top | cur -> cur) - punpckhbw m4, m3, m7 - punpcklbw m3, m7 + punpcklbw m3, m4, m2 + punpckhbw m4, m2 %if ARCH_X86_32 - mov r5, [rsp+5*mmsize+8] + mov r5, [rsp+5*mmsize+12] pmaddubsw m7, [r5], m4 pmaddubsw m4, [r5], m3 %else pmaddubsw m7, m8, m4 pmaddubsw m4, m8, m3 @@ -2002,20 +2000,17 @@ punpckhbw m1, m0, m2 punpcklbw m0, m2 ; m0-1: src as word ; scaling[src] %if ARCH_X86_32 - vpgatherdw m5, m0, scalingq, r0, r5, m7 - vpgatherdw m6, m1, scalingq, r0, r5, m7 + vpgatherdw m5, m0, scalingq-1, r0, r5, m7 + vpgatherdw m6, m1, scalingq-1, r0, r5, m7 %else - vpgatherdw m5, m0, scalingq, r13, r14, m7 - vpgatherdw m6, m1, scalingq, r13, r14, m7 + vpgatherdw m5, m0, scalingq-1, r13, r14, m7 + vpgatherdw m6, m1, scalingq-1, r13, r14, m7 %endif - pcmpeqw m7, m7 - psrlw m7, 8 - pand m5, m7 - pand m6, m7 + REPX {psrlw x, 8}, m5, m6 ; noise = round2(scaling[src] * grain, scaling_shift) pmullw m3, m5 pmullw m4, m6 pmulhrsw m3, m11 @@ -2031,11 +2026,11 @@ packuswb m0, m1 movifnidn dstq, dstmp mova [dstq+srcq], m0 %if ARCH_X86_32 - add dword [rsp+5*mmsize+8], mmsize + add dword [rsp+5*mmsize+12], mmsize %else mova m8, [pb_17_27] %endif add srcq, r2mp add grain_lutq, 82 @@ -2061,11 +2056,11 @@ lea srcq, [src_bakq+wq] %endif xor dword r8m, 4 add offxyd, 16 %if ARCH_X86_32 - add dword [rsp+6*mmsize+1*gprsize], 16 + add dword [rsp+5*mmsize+1*gprsize], 16 %else add top_offxyd, 16 %endif jmp .loop_x_odd_v_overlap @@ -2077,65 +2072,64 @@ %if ARCH_X86_32 ; fguv_32x32xn_i420_ssse3(dst, src, stride, fg_data, w, scaling, grain_lut, h, ; sby, luma, lstride, uv_pl, is_id) %if STACK_ALIGNMENT < mmsize DECLARE_ARG 0, 1, 2, 3, 4, 5, 6, 7, 8 -cglobal fguv_32x32xn_i%1, 0, 7, 8, 0 - (8 * mmsize + (13 + 3) * gprsize), \ +cglobal fguv_32x32xn_i%1_8bpc, 0, 7, 8, 0 - (7 * mmsize + (13 + 3) * gprsize), \ tmp, src, scaling, h, fg_data, picptr, unused mov r0, r0m mov r1, r2m mov r2, r4m mov r3, r6m mov r4, r7m - mov [rsp+8*mmsize+3*gprsize], r0 - mov [rsp+8*mmsize+5*gprsize], r1 - mov [rsp+8*mmsize+7*gprsize], r2 - mov [rsp+8*mmsize+9*gprsize], r3 - mov [rsp+8*mmsize+10*gprsize], r4 + mov [rsp+7*mmsize+3*gprsize], r0 + mov [rsp+7*mmsize+5*gprsize], r1 + mov [rsp+7*mmsize+7*gprsize], r2 + mov [rsp+7*mmsize+9*gprsize], r3 + mov [rsp+7*mmsize+10*gprsize], r4 mov r0, r8m mov r1, r9m mov r2, r10m mov r4, r11m mov r3, r12m - mov [rsp+8*mmsize+11*gprsize], r0 - mov [rsp+8*mmsize+12*gprsize], r1 - mov [rsp+8*mmsize+13*gprsize], r2 - mov [rsp+8*mmsize+14*gprsize], r4 + mov [rsp+7*mmsize+11*gprsize], r0 + mov [rsp+7*mmsize+12*gprsize], r1 + mov [rsp+7*mmsize+13*gprsize], r2 + mov [rsp+7*mmsize+14*gprsize], r4 %else -cglobal fguv_32x32xn_i%1, 0, 7, 8, 8 * mmsize + (4) * gprsize, \ +cglobal fguv_32x32xn_i%1_8bpc, 0, 7, 8, 7 * mmsize + (4) * gprsize, \ tmp, src, scaling, h, fg_data, picptr, unused %endif mov srcq, srcm mov fg_dataq, r3m mov scalingq, r5m %if STACK_ALIGNMENT < mmsize -%define r0m [rsp+8*mmsize+ 3*gprsize] -%define r1m [rsp+8*mmsize+ 4*gprsize] -%define r2m [rsp+8*mmsize+ 5*gprsize] -%define r3m [rsp+8*mmsize+ 6*gprsize] -%define r4m [rsp+8*mmsize+ 7*gprsize] -%define r5m [rsp+8*mmsize+ 8*gprsize] -%define r6m [rsp+8*mmsize+ 9*gprsize] -%define r7m [rsp+8*mmsize+10*gprsize] -%define r8m [rsp+8*mmsize+11*gprsize] -%define r9m [rsp+8*mmsize+12*gprsize] -%define r10m [rsp+8*mmsize+13*gprsize] -%define r11m [rsp+8*mmsize+14*gprsize] -%define r12m [rsp+8*mmsize+15*gprsize] +%define r0m [rsp+7*mmsize+ 3*gprsize] +%define r1m [rsp+7*mmsize+ 4*gprsize] +%define r2m [rsp+7*mmsize+ 5*gprsize] +%define r3m [rsp+7*mmsize+ 6*gprsize] +%define r4m [rsp+7*mmsize+ 7*gprsize] +%define r5m [rsp+7*mmsize+ 8*gprsize] +%define r6m [rsp+7*mmsize+ 9*gprsize] +%define r7m [rsp+7*mmsize+10*gprsize] +%define r8m [rsp+7*mmsize+11*gprsize] +%define r9m [rsp+7*mmsize+12*gprsize] +%define r10m [rsp+7*mmsize+13*gprsize] +%define r11m [rsp+7*mmsize+14*gprsize] +%define r12m [rsp+7*mmsize+15*gprsize] %endif LEA r5, pb_mask %define base r5-pb_mask mov r5m, r5 %else -cglobal fguv_32x32xn_i%1, 6, 15, 16, dst, src, stride, fg_data, w, scaling, \ +cglobal fguv_32x32xn_i%1_8bpc, 6, 15, 16, dst, src, stride, fg_data, w, scaling, \ grain_lut, tmp, sby, luma, lstride, uv_pl, is_id lea r8, [pb_mask] %define base r8-pb_mask %endif mov r6d, [fg_dataq+FGData.scaling_shift] - pcmpeqw m2, m2 movd m3, [base+mul_bits+r6*2-14] mov r6d, [fg_dataq+FGData.clip_to_restricted_range] lea tmpd, [r6d*2] %if ARCH_X86_32 && STACK_ALIGNMENT < mmsize test r3, r3 @@ -2143,21 +2137,19 @@ cmp dword r12m, 0 ; is_idm %endif movd m5, [base+min+r6*2] cmovne r6d, tmpd movd m4, [base+max+r6*2] - psrldq m2, 14+%2 punpcklwd m3, m3 punpcklwd m5, m5 punpcklwd m4, m4 pshufd m3, m3, q0000 pshufd m5, m5, q0000 pshufd m4, m4, q0000 - SCRATCH 2, 10, 0 - SCRATCH 3, 11, 1 - SCRATCH 4, 12, 2 - SCRATCH 5, 13, 3 + SCRATCH 3, 11, 0 + SCRATCH 4, 12, 1 + SCRATCH 5, 13, 2 cmp byte [fg_dataq+FGData.chroma_scaling_from_luma], 0 jne .csfl %macro %%FGUV_32x32xN_LOOP 3 ; not-csfl, ss_hor, ss_ver @@ -2175,36 +2167,35 @@ movd m7, [fg_dataq+FGData.uv_offset+r6*4] punpcklwd m6, m6 punpcklwd m7, m7 pshufd m6, m6, q0000 pshufd m7, m7, q0000 - SCRATCH 6, 14, 4 - SCRATCH 7, 15, 5 + SCRATCH 6, 14, 3 + SCRATCH 7, 15, 4 %endif mov sbyd, r8m mov overlapd, [fg_dataq+FGData.overlap_flag] ; left_overlap: overlap & 1 test overlapd, overlapd jz %%no_vertical_overlap %if ARCH_X86_32 %if %2 - movd m1, [base+pb_23_22] + mova m1, [base+pb_23_22_h] %else - movd m1, [base+pb_27_17_17_27] + mova m1, [base+pb_27_17_17_27] %endif mova m0, [base+pw_1024] %else %if %2 - movd m1, [pb_23_22] + mova m1, [pb_23_22_h] %else - movd m1, [pb_27_17_17_27] + mova m1, [pb_27_17_17_27] %endif mova m0, [pw_1024] %endif - pshufd m1, m1, q0000 - SCRATCH 0, 8, 6 - SCRATCH 1, 9, 7 + SCRATCH 0, 8, 5 + SCRATCH 1, 9, 6 test sbyd, sbyd jnz %%vertical_overlap ; fall-through %%no_vertical_overlap: @@ -2345,20 +2336,17 @@ punpcklbw m4, m2 %endif ; scaling[luma_src] %if ARCH_X86_32 - vpgatherdw m7, m4, scalingq, r0, r5 - vpgatherdw m5, m6, scalingq, r0, r5 + vpgatherdw m7, m4, scalingq-1, r0, r5 + vpgatherdw m5, m6, scalingq-1, r0, r5 %else - vpgatherdw m7, m4, scalingq, r12, r2 - vpgatherdw m5, m6, scalingq, r12, r2 + vpgatherdw m7, m4, scalingq-1, r12, r2 + vpgatherdw m5, m6, scalingq-1, r12, r2 %endif - pcmpeqw m1, m1 - psrlw m1, 8 - pand m7, m1 - pand m5, m1 + REPX {psrlw x, 8}, m7, m5 ; unpack chroma_source punpckhbw m1, m0, m2 punpcklbw m0, m2 ; m0-1: src as word @@ -2424,11 +2412,11 @@ mov r9m, lumaq %endif %if %2 == 0 ; adjust top_offxy %if ARCH_X86_32 - add dword [rsp+8*mmsize+1*gprsize], 16 + add dword [rsp+7*mmsize+1*gprsize], 16 %else add r11d, 16 %endif add offxyd, 16 btc dword r8m, 2 @@ -2448,13 +2436,13 @@ ; horizontal overlap (without vertical overlap) %%loop_x_h_overlap: %if ARCH_X86_32 %if %2 lea r6, [offxyd+16] - mov [rsp+8*mmsize+0*gprsize], r6 + mov [rsp+7*mmsize+0*gprsize], r6 %else - mov [rsp+8*mmsize+0*gprsize], offxyd + mov [rsp+7*mmsize+0*gprsize], offxyd %endif DEFINE_ARGS luma, src, scaling, see, w, picptr, grain_lut mov seed, r3m @@ -2556,40 +2544,35 @@ punpcklbw m4, m2 %endif ; scaling[luma_src] %if ARCH_X86_32 - vpgatherdw m7, m4, scalingq, r0, r5 - vpgatherdw m5, m6, scalingq, r0, r5 + vpgatherdw m7, m4, scalingq-1, r0, r5 + vpgatherdw m5, m6, scalingq-1, r0, r5 %else - vpgatherdw m7, m4, scalingq, r12, r2 - vpgatherdw m5, m6, scalingq, r12, r2 + vpgatherdw m7, m4, scalingq-1, r12, r2 + vpgatherdw m5, m6, scalingq-1, r12, r2 %endif - pcmpeqw m1, m1 - psrlw m1, 8 - pand m7, m1 - pand m5, m1 + REPX {psrlw x, 8}, m7, m5 ; unpack chroma_source punpckhbw m1, m0, m2 punpcklbw m0, m2 ; m0-1: src as word ; grain = grain_lut[offy+y][offx+x] - movu m3, [grain_lutq+offxyq+ 0] + movu m4, [grain_lutq+offxyq+ 0] %if ARCH_X86_32 - mov r0, [rsp+8*mmsize+0*gprsize] - movd m4, [grain_lutq+r0+ 0] + mov r0, [rsp+7*mmsize+0*gprsize] + movd m2, [grain_lutq+r0+ 0] %else - movd m4, [grain_lutq+left_offxyq+ 0] + movd m2, [grain_lutq+left_offxyq+ 0] %endif - punpcklbw m2, m4, m3 - pmaddubsw m4, m9, m2 - pmulhrsw m4, m8 - packsswb m4, m4 - pand m4, m10 - pandn m2, m10, m3 - por m3, m4, m2 + punpcklbw m2, m4 + pmaddubsw m3, m9, m2 + pmulhrsw m3, m8 + packsswb m3, m3 + shufps m3, m4, q3210 pxor m4, m4 pcmpgtb m4, m3 punpcklbw m2, m3, m4 punpckhbw m3, m4 @@ -2650,11 +2633,11 @@ %endif %if %2 == 0 xor dword r8m, 4 ; adjust top_offxyd %if ARCH_X86_32 - add dword [rsp+8*mmsize+1*gprsize], 16 + add dword [rsp+7*mmsize+1*gprsize], 16 %else add r11d, 16 %endif add offxyd, 16 %endif @@ -2778,23 +2761,25 @@ %endif movzx top_offxyd, offxyw shr offxyd, 16 %if ARCH_X86_32 - mov [rsp+8*mmsize+1*gprsize], top_offxyd + mov [rsp+7*mmsize+1*gprsize], top_offxyd DEFINE_ARGS luma, src, scaling, offxy, h, picptr, grain_lut %endif %%loop_x_odd_v_overlap: mov hd, r7m mov grain_lutq, grain_lutmp %if ARCH_X86_32 mov r5, r5m - mova m1, [base+pb_27_17] +%endif +%if %3 + mova m1, [PIC_ptr(pb_23_22)] %else - mova m1, [pb_27_17] + mova m1, [PIC_ptr(pb_27_17)] %endif %%loop_y_v_overlap: %if ARCH_X86_32 mov lumaq, r9mp %endif @@ -2846,38 +2831,30 @@ punpcklbw m4, m2 %endif ; scaling[luma_src] %if ARCH_X86_32 - vpgatherdw m7, m4, scalingq, r0, r5 - vpgatherdw m5, m6, scalingq, r0, r5 + vpgatherdw m7, m4, scalingq-1, r0, r5 + vpgatherdw m5, m6, scalingq-1, r0, r5 %else - vpgatherdw m7, m4, scalingq, r12, r2 - vpgatherdw m5, m6, scalingq, r12, r2 + vpgatherdw m7, m4, scalingq-1, r12, r2 + vpgatherdw m5, m6, scalingq-1, r12, r2 %endif - pcmpeqw m4, m4 - psrlw m4, 8 - pand m7, m4 - pand m5, m4 + REPX {psrlw x, 8}, m7, m5 ; grain = grain_lut[offy+y][offx+x] movu m3, [grain_lutq+offxyq] %if ARCH_X86_32 - mov r0, [rsp+8*mmsize+1*gprsize] + mov r0, [rsp+7*mmsize+1*gprsize] movu m4, [grain_lutq+r0] %else movu m4, [grain_lutq+top_offxyq] %endif punpckhbw m6, m4, m3 punpcklbw m4, m3 -%if %3 - pmaddubsw m2, m9, m6 - pmaddubsw m3, m9, m4 -%else pmaddubsw m2, m1, m6 pmaddubsw m3, m1, m4 -%endif pmulhrsw m2, m8 pmulhrsw m3, m8 packsswb m3, m2 pxor m6, m6 pcmpgtb m6, m3 @@ -2926,14 +2903,12 @@ add grain_lutq, 82 %if %3 == 0 btc hd, 16 %if ARCH_X86_32 mov r5, r5m - mova m1, [base+pb_17_27] -%else - mova m1, [pb_17_27] %endif + mova m1, [PIC_ptr(pb_17_27)] jnc %%loop_y_v_overlap %endif jmp %%loop_y %%end_y_v_overlap: @@ -2961,11 +2936,11 @@ ; since fg_dataq.overlap is guaranteed to be set, we never jump ; back to .loop_x_v_overlap, and instead always fall-through to ; h+v overlap %else %if ARCH_X86_32 - add dword [rsp+8*mmsize+1*gprsize], 16 + add dword [rsp+7*mmsize+1*gprsize], 16 %else add top_offxyd, 16 %endif add offxyd, 16 btc dword r8m, 2 @@ -2974,19 +2949,19 @@ %%loop_x_hv_overlap: %if ARCH_X86_32 DEFINE_ARGS tmp, src, scaling, offxy, w, picptr, unused - mov r6, [rsp+8*mmsize+1*gprsize] + mov r6, [rsp+7*mmsize+1*gprsize] %if %2 lea r0, [r3d+16] add r6, 16 - mov [rsp+8*mmsize+0*gprsize], r0 ; left_offxy + mov [rsp+7*mmsize+0*gprsize], r0 ; left_offxy %else - mov [rsp+8*mmsize+0*gprsize], r3 ; left_offxy + mov [rsp+7*mmsize+0*gprsize], r3 ; left_offxy %endif - mov [rsp+8*mmsize+2*gprsize], r6 ; topleft_offxy + mov [rsp+7*mmsize+2*gprsize], r6 ; topleft_offxy DEFINE_ARGS tmp, src, scaling, see, w, picptr, unused mov seed, r3m xor tmpd, tmpd @@ -3046,22 +3021,59 @@ %endif movzx top_offxyd, offxyw shr offxyd, 16 %if ARCH_X86_32 - mov [rsp+8*mmsize+1*gprsize], top_offxyd + mov [rsp+7*mmsize+1*gprsize], top_offxyd %endif mov hd, r7m mov grain_lutq, grain_lutmp %if ARCH_X86_32 mov r5, r5m - mova m3, [base+pb_27_17] +%endif +%if %3 + mova m3, [PIC_ptr(pb_23_22)] %else - mova m3, [pb_27_17] + mova m3, [PIC_ptr(pb_27_17)] %endif %%loop_y_hv_overlap: + ; grain = grain_lut[offy+y][offx+x] +%if ARCH_X86_32 + mov r0, [rsp+7*mmsize+2*gprsize] ; topleft_offxy + mov r5, [rsp+7*mmsize+1*gprsize] ; top_offxy + movd m1, [grain_lutq+r0] + mov r0, [rsp+7*mmsize+0*gprsize] ; left_offxy +%else + movd m1, [grain_lutq+topleft_offxyq] +%endif + movu m2, [grain_lutq+offxyq] +%if ARCH_X86_32 + movu m6, [grain_lutq+r5] + movd m4, [grain_lutq+r0] +%else + movu m6, [grain_lutq+top_offxyq] + movd m4, [grain_lutq+left_offxyq] +%endif + ; do h interpolation first (so top | top/left -> top, left | cur -> cur) + punpcklbw m1, m6 + punpcklbw m4, m2 + pmaddubsw m0, m9, m1 + pmaddubsw m1, m9, m4 + REPX {pmulhrsw x, m8}, m0, m1 + packsswb m0, m1 + shufps m4, m0, m2, q3232 + shufps m0, m6, q3210 + ; followed by v interpolation (top | cur -> cur) + punpcklbw m2, m0, m4 + punpckhbw m0, m4 + pmaddubsw m4, m3, m0 + pmaddubsw m1, m3, m2 + pmulhrsw m4, m8 + pmulhrsw m1, m8 + packsswb m1, m4 + ; src %if ARCH_X86_32 DEFINE_ARGS luma, src, scaling, offxy, w, picptr, grain_lut mov lumaq, r9mp @@ -3114,73 +3126,24 @@ punpcklbw m4, m2 %endif ; scaling[src] %if ARCH_X86_32 - vpgatherdw m7, m4, scalingq, r0, r5 - vpgatherdw m5, m6, scalingq, r0, r5 + vpgatherdw m7, m4, scalingq-1, r0, r5 + vpgatherdw m5, m6, scalingq-1, r0, r5 %else - movd m1, [grain_lutq+topleft_offxyq] %if %3 - vpgatherdw m7, m4, scalingq, r2, r12 - vpgatherdw m5, m6, scalingq, r2, r12 + vpgatherdw m7, m4, scalingq-1, r2, r12 + vpgatherdw m5, m6, scalingq-1, r2, r12 %else - vpgatherdw m7, m4, scalingq, r2, r13 - vpgatherdw m5, m6, scalingq, r2, r13 + vpgatherdw m7, m4, scalingq-1, r2, r13 + vpgatherdw m5, m6, scalingq-1, r2, r13 %endif %endif - pcmpeqw m2, m2 - psrlw m2, 8 - pand m7, m2 - pand m5, m2 + REPX {psrlw x, 8}, m7, m5 - ; grain = grain_lut[offy+y][offx+x] -%if ARCH_X86_32 - mov r0, [rsp+8*mmsize+2*gprsize] ; topleft_offxy - mov r5, [rsp+8*mmsize+1*gprsize] ; top_offxy - movd m1, [grain_lutq+r0] - mov r0, [rsp+8*mmsize+0*gprsize] ; left_offxy -%endif - movu m2, [grain_lutq+offxyq] -%if ARCH_X86_32 - movu m6, [grain_lutq+r5] - movd m4, [grain_lutq+r0] -%else - movu m6, [grain_lutq+top_offxyq] - movd m4, [grain_lutq+left_offxyq] -%endif - ; do h interpolation first (so top | top/left -> top, left | cur -> cur) - punpcklbw m1, m6 - punpcklbw m4, m2 -%if %2 - punpcklwd m4, m1 -%else - punpckldq m4, m1 -%endif - pmaddubsw m1, m9, m4 - pmulhrsw m1, m8 - packsswb m1, m1 - pandn m4, m10, m2 - pandn m2, m10, m6 - psrldq m6, m1, 2-%2 - pand m1, m10 - pand m6, m10 - por m4, m1 - por m2, m6 - ; followed by v interpolation (top | cur -> cur) - punpckhbw m1, m2, m4 - punpcklbw m2, m4 -%if %3 - pmaddubsw m4, m9, m1 - pmaddubsw m1, m9, m2 -%else - pmaddubsw m4, m3, m1 - pmaddubsw m1, m3, m2 -%endif - pmulhrsw m4, m8 - pmulhrsw m1, m8 - packsswb m1, m4 + ; unpack grain pxor m4, m4 pcmpgtb m4, m1 punpcklbw m2, m1, m4 punpckhbw m1, m4 @@ -3227,14 +3190,12 @@ jg %%loop_y_h_overlap %else jle %%end_y_hv_overlap %if ARCH_X86_32 mov r5, r5m - mova m3, [base+pb_17_27] -%else - mova m3, [pb_17_27] %endif + mova m3, [PIC_ptr(pb_17_27)] btc hd, 16 jnc %%loop_y_hv_overlap %if ARCH_X86_64 mov lstrideq, r10mp %endif @@ -3266,11 +3227,11 @@ %endif %if %2 jmp %%loop_x_hv_overlap %else %if ARCH_X86_32 - add dword [rsp+8*mmsize+1*gprsize], 16 + add dword [rsp+7*mmsize+1*gprsize], 16 %else add top_offxyd, 16 %endif add offxyd, 16 xor dword r8m, 4 diff --git a/third_party/dav1d/src/x86/ipred16_avx2.asm b/third_party/dav1d/src/x86/ipred16_avx2.asm --- a/third_party/dav1d/src/x86/ipred16_avx2.asm +++ b/third_party/dav1d/src/x86/ipred16_avx2.asm @@ -24,39 +24,43 @@ ; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. %include "config.asm" %include "ext/x86/x86inc.asm" -%if ARCH_X86_64 - SECTION_RODATA 32 -%macro SMOOTH_WEIGHT_TABLE 1-* +%macro SMOOTH_WEIGHTS 1-* +const smooth_weights_1d_16bpc ; sm_weights[] << 7 + %rep %0 + dw %1*128 + %rotate 1 + %endrep +const smooth_weights_2d_16bpc ; sm_weights[], 256 - sm_weights[] %rep %0 dw %1, 256-%1 %rotate 1 %endrep %endmacro -; sm_weights[], but modified to precalculate x and 256-x -smooth_weights: SMOOTH_WEIGHT_TABLE \ - 0, 0, 255, 128, 255, 149, 85, 64, \ - 255, 197, 146, 105, 73, 50, 37, 32, \ - 255, 225, 196, 170, 145, 123, 102, 84, \ - 68, 54, 43, 33, 26, 20, 17, 16, \ - 255, 240, 225, 210, 196, 182, 169, 157, \ - 145, 133, 122, 111, 101, 92, 83, 74, \ - 66, 59, 52, 45, 39, 34, 29, 25, \ - 21, 17, 14, 12, 10, 9, 8, 8, \ - 255, 248, 240, 233, 225, 218, 210, 203, \ - 196, 189, 182, 176, 169, 163, 156, 150, \ - 144, 138, 133, 127, 121, 116, 111, 106, \ - 101, 96, 91, 86, 82, 77, 73, 69, \ - 65, 61, 57, 54, 50, 47, 44, 41, \ - 38, 35, 32, 29, 27, 25, 22, 20, \ - 18, 16, 15, 13, 12, 10, 9, 8, \ - 7, 6, 6, 5, 5, 4, 4, 4 +SMOOTH_WEIGHTS 0, 0, 255, 128, 255, 149, 85, 64, \ + 255, 197, 146, 105, 73, 50, 37, 32, \ + 255, 225, 196, 170, 145, 123, 102, 84, \ + 68, 54, 43, 33, 26, 20, 17, 16, \ + 255, 240, 225, 210, 196, 182, 169, 157, \ + 145, 133, 122, 111, 101, 92, 83, 74, \ + 66, 59, 52, 45, 39, 34, 29, 25, \ + 21, 17, 14, 12, 10, 9, 8, 8, \ + 255, 248, 240, 233, 225, 218, 210, 203, \ + 196, 189, 182, 176, 169, 163, 156, 150, \ + 144, 138, 133, 127, 121, 116, 111, 106, \ + 101, 96, 91, 86, 82, 77, 73, 69, \ + 65, 61, 57, 54, 50, 47, 44, 41, \ + 38, 35, 32, 29, 27, 25, 22, 20, \ + 18, 16, 15, 13, 12, 10, 9, 8, \ + 7, 6, 6, 5, 5, 4, 4, 4 + +%if ARCH_X86_64 ipred_hv_shuf: db 6, 7, 6, 7, 0, 1, 2, 3, 2, 3, 2, 3, 8, 9, 10, 11 db 4, 5, 4, 5, 4, 5, 6, 7, 0, 1, 0, 1, 12, 13, 14, 15 filter_shuf1: db 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 14, 15, 12, 13, -1, -1 filter_shuf2: db 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 5, 2, 3, -1, -1 @@ -85,17 +89,16 @@ %define pw_2 (z_filter_k+32) %define pw_4 (z_filter_k+ 0) %define pw_16 (z2_ymul8 +20) +pw_1: times 2 dw 1 pw_3: times 2 dw 3 pw_62: times 2 dw 62 pw_512: times 2 dw 512 pw_2048: times 2 dw 2048 pd_8: dd 8 -pd_128: dd 128 -pd_256: dd 256 %macro JMP_TABLE 3-* %xdefine %1_%2_table (%%table - 2*4) %xdefine %%base mangle(private_prefix %+ _%1_%2) %%table: @@ -121,21 +124,27 @@ JMP_TABLE ipred_z3_16bpc, avx2, h4, h8, h16, h32, h64 JMP_TABLE ipred_filter_16bpc, avx2, w4, w8, w16, w32 JMP_TABLE ipred_cfl_16bpc, avx2, h4, h8, h16, h32, w4, w8, w16, w32, \ s4-8*4, s8-8*4, s16-8*4, s32-8*4 JMP_TABLE ipred_cfl_left_16bpc, avx2, h4, h8, h16, h32 -JMP_TABLE ipred_cfl_ac_420_16bpc, avx2, w16_wpad_pad1, w16_wpad_pad2, w16_wpad_pad3 -JMP_TABLE ipred_cfl_ac_422_16bpc, avx2, w16_wpad_pad1, w16_wpad_pad2, w16_wpad_pad3 +JMP_TABLE ipred_cfl_ac_444_16bpc, avx2, w4, w8, w16, w32 JMP_TABLE pal_pred_16bpc, avx2, w4, w8, w16, w32, w64 cextern dr_intra_derivative cextern filter_intra_taps SECTION .text +%macro REPX 2-* + %xdefine %%f(x) %1 +%rep %0 - 1 + %rotate 1 + %%f(%1) +%endrep +%endmacro + INIT_YMM avx2 - cglobal ipred_dc_top_16bpc, 3, 7, 6, dst, stride, tl, w, h movifnidn hd, hm add tlq, 2 movd xm4, wd pxor xm3, xm3 @@ -559,11 +568,10 @@ RET ALIGN function_align .w8: vbroadcasti128 m2, [tlq+2] movsldup m6, [base+ipred_hv_shuf] - lea r3, [strideq*3] psubw m4, m2, m3 pabsw m5, m4 .w8_loop: sub tlq, 4 vpbroadcastd m1, [tlq] @@ -645,285 +653,260 @@ add dstq, strideq dec hd jg .w64_loop RET -%macro SMOOTH 4 ; src[1-2], mul[1-2] - pmaddwd m0, m%3, m%1 - pmaddwd m1, m%4, m%2 - paddd m0, m2 - paddd m1, m2 - psrld m0, 8 - psrld m1, 8 - packssdw m0, m1 -%endmacro - cglobal ipred_smooth_v_16bpc, 3, 7, 6, dst, stride, tl, w, h, weights %define base r6-ipred_smooth_v_16bpc_avx2_table - lea r6, [ipred_smooth_v_16bpc_avx2_table] - tzcnt wd, wm - mov hd, hm - movsxd wq, [r6+wq*4] - vpbroadcastd m2, [base+pd_128] - lea weightsq, [base+smooth_weights+hq*8] - neg hq - vpbroadcastw m5, [tlq+hq*2] ; bottom - add wq, r6 - jmp wq + lea r6, [ipred_smooth_v_16bpc_avx2_table] + tzcnt wd, wm + mov hd, hm + movsxd wq, [r6+wq*4] + lea weightsq, [base+smooth_weights_1d_16bpc+hq*4] + neg hq + vpbroadcastw m5, [tlq+hq*2] ; bottom + add wq, r6 + jmp wq .w4: - vpbroadcastq m3, [tlq+2] - punpcklwd m3, m5 ; top, bottom - movshdup m5, [base+ipred_hv_shuf] - lea r3, [strideq*3] - punpcklqdq m4, m5, m5 - punpckhqdq m5, m5 + vpbroadcastq m4, [tlq+2] ; top + movsldup m3, [base+ipred_hv_shuf] + lea r6, [strideq*3] + psubw m4, m5 ; top - bottom .w4_loop: - vbroadcasti128 m1, [weightsq+hq*4] - pshufb m0, m1, m4 - pshufb m1, m5 - SMOOTH 3, 3, 0, 1 - vextracti128 xm1, m0, 1 - movq [dstq+strideq*0], xm0 - movq [dstq+strideq*1], xm1 - movhps [dstq+strideq*2], xm0 - movhps [dstq+r3 ], xm1 - lea dstq, [dstq+strideq*4] - add hq, 4 + vpbroadcastq m0, [weightsq+hq*2] + pshufb m0, m3 + pmulhrsw m0, m4 + paddw m0, m5 + vextracti128 xm1, m0, 1 + movhps [dstq+strideq*0], xm1 + movhps [dstq+strideq*1], xm0 + movq [dstq+strideq*2], xm1 + movq [dstq+r6 ], xm0 + lea dstq, [dstq+strideq*4] + add hq, 4 jl .w4_loop .ret: RET -ALIGN function_align .w8: - vbroadcasti128 m4, [tlq+2] - punpcklwd m3, m4, m5 - punpckhwd m4, m5 - movshdup m5, [base+ipred_hv_shuf] + vbroadcasti128 m4, [tlq+2] + movsldup m3, [base+ipred_hv_shuf] + lea r6, [strideq*3] + psubw m4, m5 .w8_loop: - vpbroadcastq m1, [weightsq+hq*4] - pshufb m1, m5 - SMOOTH 3, 4, 1, 1 - mova [dstq+strideq*0], xm0 - vextracti128 [dstq+strideq*1], m0, 1 - lea dstq, [dstq+strideq*2] - add hq, 2 + vpbroadcastd m0, [weightsq+hq*2+0] + vpbroadcastd m1, [weightsq+hq*2+4] + pshufb m0, m3 + pshufb m1, m3 + pmulhrsw m0, m4 + pmulhrsw m1, m4 + paddw m0, m5 + paddw m1, m5 + vextracti128 [dstq+strideq*0], m0, 1 + mova [dstq+strideq*1], xm0 + vextracti128 [dstq+strideq*2], m1, 1 + mova [dstq+r6 ], xm1 + lea dstq, [dstq+strideq*4] + add hq, 4 jl .w8_loop RET -ALIGN function_align .w16: - movu m4, [tlq+2] - punpcklwd m3, m4, m5 - punpckhwd m4, m5 + movu m4, [tlq+2] + lea r6, [strideq*3] + psubw m4, m5 .w16_loop: - vpbroadcastd m1, [weightsq+hq*4] - vpbroadcastd m5, [weightsq+hq*4+4] - SMOOTH 3, 4, 1, 1 - mova [dstq+strideq*0], m0 - SMOOTH 3, 4, 5, 5 - mova [dstq+strideq*1], m0 - lea dstq, [dstq+strideq*2] - add hq, 2 + vpbroadcastw m0, [weightsq+hq*2+0] + vpbroadcastw m1, [weightsq+hq*2+2] + vpbroadcastw m2, [weightsq+hq*2+4] + vpbroadcastw m3, [weightsq+hq*2+6] + REPX {pmulhrsw x, m4}, m0, m1, m2, m3 + REPX {paddw x, m5}, m0, m1, m2, m3 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + mova [dstq+strideq*2], m2 + mova [dstq+r6 ], m3 + lea dstq, [dstq+strideq*4] + add hq, 4 jl .w16_loop RET -ALIGN function_align .w32: - WIN64_SPILL_XMM 8 - movu m4, [tlq+2] - movu m7, [tlq+34] - punpcklwd m3, m4, m5 - punpckhwd m4, m5 - punpcklwd m6, m7, m5 - punpckhwd m7, m5 + WIN64_SPILL_XMM 7 + movu m4, [tlq+ 2] + movu m6, [tlq+34] + psubw m4, m5 + psubw m6, m5 .w32_loop: - vpbroadcastd m5, [weightsq+hq*4] - SMOOTH 3, 4, 5, 5 - mova [dstq+32*0], m0 - SMOOTH 6, 7, 5, 5 - mova [dstq+32*1], m0 - add dstq, strideq - inc hq + vpbroadcastw m1, [weightsq+hq*2+0] + vpbroadcastw m3, [weightsq+hq*2+2] + pmulhrsw m0, m4, m1 + pmulhrsw m1, m6 + pmulhrsw m2, m4, m3 + pmulhrsw m3, m6 + REPX {paddw x, m5}, m0, m1, m2, m3 + mova [dstq+strideq*0+32*0], m0 + mova [dstq+strideq*0+32*1], m1 + mova [dstq+strideq*1+32*0], m2 + mova [dstq+strideq*1+32*1], m3 + lea dstq, [dstq+strideq*2] + add hq, 2 jl .w32_loop RET -ALIGN function_align .w64: - WIN64_SPILL_XMM 12 - movu m4, [tlq+ 2] - movu m7, [tlq+34] - movu m9, [tlq+66] - movu m11, [tlq+98] - punpcklwd m3, m4, m5 - punpckhwd m4, m5 - punpcklwd m6, m7, m5 - punpckhwd m7, m5 - punpcklwd m8, m9, m5 - punpckhwd m9, m5 - punpcklwd m10, m11, m5 - punpckhwd m11, m5 + WIN64_SPILL_XMM 8 + movu m3, [tlq+ 2] + movu m4, [tlq+34] + movu m6, [tlq+66] + movu m7, [tlq+98] + REPX {psubw x, m5}, m3, m4, m6, m7 .w64_loop: - vpbroadcastd m5, [weightsq+hq*4] - SMOOTH 3, 4, 5, 5 - mova [dstq+32*0], m0 - SMOOTH 6, 7, 5, 5 - mova [dstq+32*1], m0 - SMOOTH 8, 9, 5, 5 - mova [dstq+32*2], m0 - SMOOTH 10, 11, 5, 5 - mova [dstq+32*3], m0 - add dstq, strideq - inc hq + vpbroadcastw m2, [weightsq+hq*2] + pmulhrsw m0, m3, m2 + pmulhrsw m1, m4, m2 + paddw m0, m5 + paddw m1, m5 + mova [dstq+32*0], m0 + pmulhrsw m0, m6, m2 + mova [dstq+32*1], m1 + pmulhrsw m1, m7, m2 + paddw m0, m5 + paddw m1, m5 + mova [dstq+32*2], m0 + mova [dstq+32*3], m1 + add dstq, strideq + inc hq jl .w64_loop RET -cglobal ipred_smooth_h_16bpc, 3, 7, 6, dst, stride, tl, w, h +cglobal ipred_smooth_h_16bpc, 3, 7, 6, dst, stride, tl, w, h, stride3 %define base r6-ipred_smooth_h_16bpc_avx2_table - lea r6, [ipred_smooth_h_16bpc_avx2_table] - mov wd, wm - mov hd, hm - vpbroadcastw m3, [tlq+wq*2] ; right - tzcnt wd, wd - movsxd wq, [r6+wq*4] - vpbroadcastd m2, [base+pd_128] - add wq, r6 - jmp wq + lea r6, [ipred_smooth_h_16bpc_avx2_table] + mov wd, wm + movifnidn hd, hm + vpbroadcastw m5, [tlq+wq*2] ; right + tzcnt wd, wd + add hd, hd + movsxd wq, [r6+wq*4] + sub tlq, hq + lea stride3q, [strideq*3] + add wq, r6 + jmp wq .w4: - vbroadcasti128 m4, [base+smooth_weights+4*4] - movsldup m5, [base+ipred_hv_shuf] - sub tlq, 8 - sub tlq, hq - sub tlq, hq - lea r3, [strideq*3] + vpbroadcastq m4, [base+smooth_weights_1d_16bpc+4*2] + movsldup m3, [base+ipred_hv_shuf] .w4_loop: - vpbroadcastq m1, [tlq+hq*2] - pshufb m1, m5 - punpcklwd m0, m1, m3 ; left, right - punpckhwd m1, m3 - SMOOTH 0, 1, 4, 4 - vextracti128 xm1, m0, 1 + vpbroadcastq m0, [tlq+hq-8] ; left + pshufb m0, m3 + psubw m0, m5 ; left - right + pmulhrsw m0, m4 + paddw m0, m5 + vextracti128 xm1, m0, 1 movq [dstq+strideq*0], xm0 movq [dstq+strideq*1], xm1 movhps [dstq+strideq*2], xm0 - movhps [dstq+r3 ], xm1 - lea dstq, [dstq+strideq*4] - sub hd, 4 + movhps [dstq+stride3q ], xm1 + lea dstq, [dstq+strideq*4] + sub hd, 4*2 jg .w4_loop RET -ALIGN function_align .w8: - WIN64_SPILL_XMM 7 - vbroadcasti128 m4, [base+smooth_weights+8*4+16*0] - vbroadcasti128 m5, [base+smooth_weights+8*4+16*1] - movsldup m6, [base+ipred_hv_shuf] - sub tlq, 4 - sub tlq, hq - sub tlq, hq + vbroadcasti128 m4, [base+smooth_weights_1d_16bpc+8*2] + movsldup m3, [base+ipred_hv_shuf] .w8_loop: - vpbroadcastd m1, [tlq+hq*2] - pshufb m1, m6 - punpcklwd m0, m1, m3 - punpckhwd m1, m3 - SMOOTH 0, 1, 4, 5 + vpbroadcastd m0, [tlq+hq-4] + vpbroadcastd m1, [tlq+hq-8] + pshufb m0, m3 + pshufb m1, m3 + psubw m0, m5 + psubw m1, m5 + pmulhrsw m0, m4 + pmulhrsw m1, m4 + paddw m0, m5 + paddw m1, m5 mova [dstq+strideq*0], xm0 vextracti128 [dstq+strideq*1], m0, 1 - lea dstq, [dstq+strideq*2] - sub hq, 2 + mova [dstq+strideq*2], xm1 + vextracti128 [dstq+stride3q ], m1, 1 + lea dstq, [dstq+strideq*4] + sub hq, 4*2 jg .w8_loop RET -ALIGN function_align .w16: - WIN64_SPILL_XMM 6 - mova xm4, [base+smooth_weights+16*4+16*0] - mova xm5, [base+smooth_weights+16*4+16*1] - vinserti128 m4, [base+smooth_weights+16*4+16*2], 1 - vinserti128 m5, [base+smooth_weights+16*4+16*3], 1 - sub tlq, 2 - sub tlq, hq - sub tlq, hq + movu m4, [base+smooth_weights_1d_16bpc+16*2] .w16_loop: - vpbroadcastw m1, [tlq+hq*2] - punpcklwd m0, m1, m3 - punpckhwd m1, m3 - SMOOTH 0, 1, 4, 5 - mova [dstq], m0 - add dstq, strideq - dec hq + vpbroadcastq m3, [tlq+hq-8] + punpcklwd m3, m3 + psubw m3, m5 + pshufd m0, m3, q3333 + pshufd m1, m3, q2222 + pshufd m2, m3, q1111 + pshufd m3, m3, q0000 + REPX {pmulhrsw x, m4}, m0, m1, m2, m3 + REPX {paddw x, m5}, m0, m1, m2, m3 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + mova [dstq+strideq*2], m2 + mova [dstq+stride3q ], m3 + lea dstq, [dstq+strideq*4] + sub hq, 4*2 jg .w16_loop RET -ALIGN function_align .w32: - WIN64_SPILL_XMM 10 - mova xm6, [base+smooth_weights+32*4+16*0] - mova xm7, [base+smooth_weights+32*4+16*1] - vinserti128 m6, [base+smooth_weights+32*4+16*2], 1 - vinserti128 m7, [base+smooth_weights+32*4+16*3], 1 - mova xm8, [base+smooth_weights+32*4+16*4] - mova xm9, [base+smooth_weights+32*4+16*5] - vinserti128 m8, [base+smooth_weights+32*4+16*6], 1 - vinserti128 m9, [base+smooth_weights+32*4+16*7], 1 - sub tlq, 2 - sub tlq, hq - sub tlq, hq + WIN64_SPILL_XMM 7 + movu m4, [base+smooth_weights_1d_16bpc+32*2] + movu m6, [base+smooth_weights_1d_16bpc+32*3] .w32_loop: - vpbroadcastw m5, [tlq+hq*2] - punpcklwd m4, m5, m3 - punpckhwd m5, m3 - SMOOTH 4, 5, 6, 7 - mova [dstq+32*0], m0 - SMOOTH 4, 5, 8, 9 - mova [dstq+32*1], m0 - add dstq, strideq - dec hq + vpbroadcastw m1, [tlq+hq-2] + vpbroadcastw m3, [tlq+hq-4] + psubw m1, m5 + psubw m3, m5 + pmulhrsw m0, m4, m1 + pmulhrsw m1, m6 + pmulhrsw m2, m4, m3 + pmulhrsw m3, m6 + REPX {paddw x, m5}, m0, m1, m2, m3 + mova [dstq+strideq*0+32*0], m0 + mova [dstq+strideq*0+32*1], m1 + mova [dstq+strideq*1+32*0], m2 + mova [dstq+strideq*1+32*1], m3 + lea dstq, [dstq+strideq*2] + sub hq, 2*2 jg .w32_loop RET -ALIGN function_align .w64: -%assign stack_offset stack_offset - stack_size_padded - WIN64_SPILL_XMM 14 - mova xm6, [base+smooth_weights+64*4+16* 0] - mova xm7, [base+smooth_weights+64*4+16* 1] - vinserti128 m6, [base+smooth_weights+64*4+16* 2], 1 - vinserti128 m7, [base+smooth_weights+64*4+16* 3], 1 - mova xm8, [base+smooth_weights+64*4+16* 4] - mova xm9, [base+smooth_weights+64*4+16* 5] - vinserti128 m8, [base+smooth_weights+64*4+16* 6], 1 - vinserti128 m9, [base+smooth_weights+64*4+16* 7], 1 - mova xm10, [base+smooth_weights+64*4+16* 8] - mova xm11, [base+smooth_weights+64*4+16* 9] - vinserti128 m10, [base+smooth_weights+64*4+16*10], 1 - vinserti128 m11, [base+smooth_weights+64*4+16*11], 1 - mova xm12, [base+smooth_weights+64*4+16*12] - mova xm13, [base+smooth_weights+64*4+16*13] - vinserti128 m12, [base+smooth_weights+64*4+16*14], 1 - vinserti128 m13, [base+smooth_weights+64*4+16*15], 1 - sub tlq, 2 - sub tlq, hq - sub tlq, hq + WIN64_SPILL_XMM 8 + movu m3, [base+smooth_weights_1d_16bpc+32*4] + movu m4, [base+smooth_weights_1d_16bpc+32*5] + movu m6, [base+smooth_weights_1d_16bpc+32*6] + movu m7, [base+smooth_weights_1d_16bpc+32*7] .w64_loop: - vpbroadcastw m5, [tlq+hq*2] - punpcklwd m4, m5, m3 - punpckhwd m5, m3 - SMOOTH 4, 5, 6, 7 - mova [dstq+32*0], m0 - SMOOTH 4, 5, 8, 9 - mova [dstq+32*1], m0 - SMOOTH 4, 5, 10, 11 - mova [dstq+32*2], m0 - SMOOTH 4, 5, 12, 13 - mova [dstq+32*3], m0 - add dstq, strideq - dec hq + vpbroadcastw m2, [tlq+hq-2] + psubw m2, m5 + pmulhrsw m0, m3, m2 + pmulhrsw m1, m4, m2 + paddw m0, m5 + paddw m1, m5 + mova [dstq+32*0], m0 + pmulhrsw m0, m6, m2 + mova [dstq+32*1], m1 + pmulhrsw m1, m7, m2 + paddw m0, m5 + paddw m1, m5 + mova [dstq+32*2], m0 + mova [dstq+32*3], m1 + add dstq, strideq + sub hq, 1*2 jg .w64_loop RET %macro SMOOTH_2D_END 6 ; src[1-2], mul[1-2], add[1-2] pmaddwd m0, m%1, m%3 pmaddwd m1, m%2, m%4 paddd m0, m%5 paddd m1, m%6 - paddd m0, m5 - paddd m1, m5 - psrld m0, 9 - psrld m1, 9 + psrld m0, 8 + psrld m1, 8 packssdw m0, m1 + pavgw m0, m5 %endmacro cglobal ipred_smooth_16bpc, 3, 7, 6, dst, stride, tl, w, h, v_weights %define base r6-ipred_smooth_16bpc_avx2_table lea r6, [ipred_smooth_16bpc_avx2_table] @@ -932,159 +915,147 @@ tzcnt wd, wd mov hd, hm sub tlq, hq sub tlq, hq movsxd wq, [r6+wq*4] - vpbroadcastd m5, [base+pd_256] + pxor m5, m5 add wq, r6 - lea v_weightsq, [base+smooth_weights+hq*4] + lea v_weightsq, [base+smooth_weights_2d_16bpc+hq*4] jmp wq .w4: WIN64_SPILL_XMM 11 vpbroadcastw m0, [tlq] ; bottom vpbroadcastq m6, [tlq+hq*2+2] movsldup m7, [base+ipred_hv_shuf] movshdup m9, [base+ipred_hv_shuf] - vbroadcasti128 m10, [base+smooth_weights+4*4] + vbroadcasti128 m10, [base+smooth_weights_2d_16bpc+4*4] punpcklwd m6, m0 ; top, bottom punpcklqdq m8, m9, m9 punpckhqdq m9, m9 lea r3, [strideq*3] - sub tlq, 8 .w4_loop: + vpbroadcastq m3, [tlq+hq*2-8] vbroadcasti128 m1, [v_weightsq] - vpbroadcastq m3, [tlq+hq*2] pshufb m3, m7 punpcklwd m2, m3, m4 ; left, right punpckhwd m3, m4 pmaddwd m2, m10 pmaddwd m3, m10 pshufb m0, m1, m8 pshufb m1, m9 - SMOOTH_2D_END 6, 6, 0, 1, 2, 3 + SMOOTH_2D_END 0, 1, 6, 6, 2, 3 vextracti128 xm1, m0, 1 movq [dstq+strideq*0], xm0 movq [dstq+strideq*1], xm1 movhps [dstq+strideq*2], xm0 movhps [dstq+r3 ], xm1 lea dstq, [dstq+strideq*4] add v_weightsq, 16 sub hd, 4 jg .w4_loop RET -ALIGN function_align .w8: %assign stack_offset stack_offset - stack_size_padded WIN64_SPILL_XMM 12 vpbroadcastw m0, [tlq] ; bottom vbroadcasti128 m7, [tlq+hq*2+2] movsldup m8, [base+ipred_hv_shuf] movshdup m9, [base+ipred_hv_shuf] - vbroadcasti128 m10, [base+smooth_weights+8*4+16*0] - vbroadcasti128 m11, [base+smooth_weights+8*4+16*1] + vbroadcasti128 m10, [base+smooth_weights_2d_16bpc+8*4+16*0] + vbroadcasti128 m11, [base+smooth_weights_2d_16bpc+8*4+16*1] punpcklwd m6, m7, m0 ; top, bottom punpckhwd m7, m0 - sub tlq, 4 .w8_loop: + vpbroadcastd m3, [tlq+hq*2-4] vpbroadcastq m1, [v_weightsq] - vpbroadcastd m3, [tlq+hq*2] pshufb m3, m8 punpcklwd m2, m3, m4 ; left, right punpckhwd m3, m4 pmaddwd m2, m10 pmaddwd m3, m11 pshufb m1, m9 - SMOOTH_2D_END 6, 7, 1, 1, 2, 3 + SMOOTH_2D_END 1, 1, 6, 7, 2, 3 mova [dstq+strideq*0], xm0 vextracti128 [dstq+strideq*1], m0, 1 lea dstq, [dstq+strideq*2] add v_weightsq, 8 sub hd, 2 jg .w8_loop RET -ALIGN function_align .w16: %assign stack_offset stack_offset - stack_size_padded - WIN64_SPILL_XMM 14 + WIN64_SPILL_XMM 11 vpbroadcastw m0, [tlq] ; bottom movu m7, [tlq+hq*2+2] - mova xm8, [base+smooth_weights+16*4+16*0] - mova xm9, [base+smooth_weights+16*4+16*1] - vinserti128 m8, [base+smooth_weights+16*4+16*2], 1 - vinserti128 m9, [base+smooth_weights+16*4+16*3], 1 + mova xm8, [base+smooth_weights_2d_16bpc+16*4+16*0] + mova xm9, [base+smooth_weights_2d_16bpc+16*4+16*1] + vinserti128 m8, [base+smooth_weights_2d_16bpc+16*4+16*2], 1 + vinserti128 m9, [base+smooth_weights_2d_16bpc+16*4+16*3], 1 punpcklwd m6, m7, m0 ; top, bottom punpckhwd m7, m0 - sub tlq, 2 .w16_loop: - vpbroadcastd m10, [v_weightsq+0] - vpbroadcastd m11, [v_weightsq+4] - vpbroadcastw m3, [tlq+hq*2-0] - vpbroadcastw m13, [tlq+hq*2-2] - punpcklwd m2, m3, m4 ; left, right - punpckhwd m3, m4 - punpcklwd m12, m13, m4 - punpckhwd m13, m4 - pmaddwd m2, m8 + vpbroadcastd m3, [tlq+hq*2-4] + vpbroadcastd m1, [v_weightsq+0] + punpcklwd m3, m4 ; left, right + pshufd m2, m3, q1111 + pmaddwd m10, m8, m2 + pmaddwd m2, m9 + pshufd m3, m3, q0000 + SMOOTH_2D_END 1, 1, 6, 7, 10, 2 + vpbroadcastd m1, [v_weightsq+4] + pmaddwd m2, m8, m3 pmaddwd m3, m9 - pmaddwd m12, m8 - pmaddwd m13, m9 - SMOOTH_2D_END 6, 7, 10, 10, 2, 3 mova [dstq+strideq*0], m0 - SMOOTH_2D_END 6, 7, 11, 11, 12, 13 + SMOOTH_2D_END 1, 1, 6, 7, 2, 3 mova [dstq+strideq*1], m0 lea dstq, [dstq+strideq*2] add v_weightsq, 8 sub hq, 2 jg .w16_loop RET -ALIGN function_align .w32: %assign stack_offset stack_offset - stack_size_padded - WIN64_SPILL_XMM 16 + WIN64_SPILL_XMM 15 vpbroadcastw m0, [tlq] ; bottom movu m7, [tlq+hq*2+ 2] movu m9, [tlq+hq*2+34] - mova xm10, [base+smooth_weights+32*4+16*0] - mova xm11, [base+smooth_weights+32*4+16*1] - vinserti128 m10, [base+smooth_weights+32*4+16*2], 1 - vinserti128 m11, [base+smooth_weights+32*4+16*3], 1 - mova xm12, [base+smooth_weights+32*4+16*4] - mova xm13, [base+smooth_weights+32*4+16*5] - vinserti128 m12, [base+smooth_weights+32*4+16*6], 1 - vinserti128 m13, [base+smooth_weights+32*4+16*7], 1 + mova xm10, [base+smooth_weights_2d_16bpc+32*4+16*0] + mova xm11, [base+smooth_weights_2d_16bpc+32*4+16*1] + vinserti128 m10, [base+smooth_weights_2d_16bpc+32*4+16*2], 1 + vinserti128 m11, [base+smooth_weights_2d_16bpc+32*4+16*3], 1 + mova xm12, [base+smooth_weights_2d_16bpc+32*4+16*4] + mova xm13, [base+smooth_weights_2d_16bpc+32*4+16*5] + vinserti128 m12, [base+smooth_weights_2d_16bpc+32*4+16*6], 1 + vinserti128 m13, [base+smooth_weights_2d_16bpc+32*4+16*7], 1 punpcklwd m6, m7, m0 punpckhwd m7, m0 punpcklwd m8, m9, m0 punpckhwd m9, m0 - sub tlq, 2 .w32_loop: - vpbroadcastw m3, [tlq+hq*2] - punpcklwd m2, m3, m4 - punpckhwd m3, m4 - pmaddwd m14, m2, m10 - pmaddwd m15, m3, m11 - pmaddwd m2, m12 + vpbroadcastw m3, [tlq+hq*2-2] + vpbroadcastd m14, [v_weightsq] + punpcklwd m3, m4 + pmaddwd m1, m10, m3 + pmaddwd m2, m11, m3 + pmaddwd m0, m6, m14 + paddd m0, m1 + pmaddwd m1, m7, m14 + paddd m1, m2 + pmaddwd m2, m12, m3 pmaddwd m3, m13 - vpbroadcastd m1, [v_weightsq] - pmaddwd m0, m6, m1 - paddd m0, m14 - paddd m0, m5 - psrld m0, 9 - pmaddwd m14, m7, m1 - paddd m14, m15 - paddd m14, m5 - psrld m14, 9 - packssdw m0, m14 + psrld m0, 8 + psrld m1, 8 + packssdw m0, m1 + pavgw m0, m5 mova [dstq+32*0], m0 - SMOOTH_2D_END 8, 9, 1, 1, 2, 3 + SMOOTH_2D_END 14, 14, 8, 9, 2, 3 mova [dstq+32*1], m0 add dstq, strideq add v_weightsq, 4 dec hd jg .w32_loop RET -ALIGN function_align .w64: %assign stack_offset stack_offset - stack_size_padded PROLOGUE 0, 11, 16, dst, stride, tl, tl_base, h, v_weights, dummy, v_weights_base, x, y, dst_base mov dst_baseq, dstq mov tl_baseq, tlq @@ -1094,41 +1065,39 @@ mov yq, hq lea tlq, [tl_baseq+hq*2] vpbroadcastw m0, [tl_baseq] ; bottom movu m7, [tlq+xq*2+ 2] movu m9, [tlq+xq*2+34] - mova xm10, [base+smooth_weights+64*4+16*0] - mova xm11, [base+smooth_weights+64*4+16*1] - vinserti128 m10, [base+smooth_weights+64*4+16*2], 1 - vinserti128 m11, [base+smooth_weights+64*4+16*3], 1 - mova xm12, [base+smooth_weights+64*4+16*4] - mova xm13, [base+smooth_weights+64*4+16*5] - vinserti128 m12, [base+smooth_weights+64*4+16*6], 1 - vinserti128 m13, [base+smooth_weights+64*4+16*7], 1 + mova xm10, [base+smooth_weights_2d_16bpc+64*4+16*0] + mova xm11, [base+smooth_weights_2d_16bpc+64*4+16*1] + vinserti128 m10, [base+smooth_weights_2d_16bpc+64*4+16*2], 1 + vinserti128 m11, [base+smooth_weights_2d_16bpc+64*4+16*3], 1 + mova xm12, [base+smooth_weights_2d_16bpc+64*4+16*4] + mova xm13, [base+smooth_weights_2d_16bpc+64*4+16*5] + vinserti128 m12, [base+smooth_weights_2d_16bpc+64*4+16*6], 1 + vinserti128 m13, [base+smooth_weights_2d_16bpc+64*4+16*7], 1 punpcklwd m6, m7, m0 punpckhwd m7, m0 punpcklwd m8, m9, m0 punpckhwd m9, m0 lea tlq, [tl_baseq-2] .w64_loop_y: + vpbroadcastw m3, [tlq+yq*2] vpbroadcastd m1, [v_weightsq] - vpbroadcastw m3, [tlq+yq*2] - punpcklwd m2, m3, m4 - punpckhwd m3, m4 - pmaddwd m14, m2, m10 - pmaddwd m15, m3, m11 - pmaddwd m2, m12 + punpcklwd m3, m4 + pmaddwd m14, m10, m3 + pmaddwd m15, m11, m3 + pmaddwd m2, m12, m3 pmaddwd m3, m13 pmaddwd m0, m6, m1 paddd m0, m14 - paddd m0, m5 - psrld m0, 9 pmaddwd m14, m7, m1 paddd m14, m15 - paddd m14, m5 - psrld m14, 9 + psrld m0, 8 + psrld m14, 8 packssdw m0, m14 + pavgw m0, m5 mova [dstq+32*0], m0 SMOOTH_2D_END 8, 9, 1, 1, 2, 3 mova [dstq+32*1], m0 add dstq, strideq add v_weightsq, 4 @@ -4462,378 +4431,473 @@ vpbroadcastw m7, r7m add wq, t0 movifnidn acq, acmp jmp wq -cglobal ipred_cfl_ac_420_16bpc, 4, 9, 6, ac, ypx, stride, wpad, hpad, w, h, sz, ac_bak +cglobal ipred_cfl_ac_420_16bpc, 4, 7, 6, ac, ypx, stride, wpad, hpad, w, h movifnidn hpadd, hpadm - movifnidn wd, wm + vpbroadcastd m5, [pw_2] mov hd, hm - mov szd, wd - mov ac_bakq, acq - imul szd, hd shl hpadd, 2 + pxor m4, m4 sub hd, hpadd - vpbroadcastd m2, [pw_2] - pxor m4, m4 - cmp wd, 8 + cmp dword wm, 8 jg .w16 je .w8 -DEFINE_ARGS ac, ypx, stride, wpad, hpad, stride3, h, sz, ac_bak .w4: - lea stride3q, [strideq*3] + lea r3, [strideq*3] + mov r5, acq .w4_loop: mova xm0, [ypxq+strideq*2] - mova xm1, [ypxq+stride3q ] + mova xm1, [ypxq+r3 ] vinserti128 m0, [ypxq+strideq*0], 1 vinserti128 m1, [ypxq+strideq*1], 1 - pmaddwd m0, m2 - pmaddwd m1, m2 + lea ypxq, [ypxq+strideq*4] + pmaddwd m0, m5 + pmaddwd m1, m5 paddd m0, m1 vextracti128 xm1, m0, 1 paddd m4, m0 packssdw xm1, xm0 mova [acq], xm1 - lea ypxq, [ypxq+strideq*4] add acq, 16 sub hd, 2 jg .w4_loop test hpadd, hpadd - jz .calc_avg + jz .dc vpermq m1, m1, q1111 pslld xm0, 2 .w4_hpad_loop: mova [acq], m1 paddd m4, m0 add acq, 32 sub hpadd, 4 jg .w4_hpad_loop - jmp .calc_avg + jmp .dc .w8: + mov r5, acq test wpadd, wpadd - jnz .w8_wpad + jnz .w8_wpad1 .w8_loop: - pmaddwd m0, m2, [ypxq+strideq*0] - pmaddwd m1, m2, [ypxq+strideq*1] + pmaddwd m0, m5, [ypxq+strideq*0] + pmaddwd m1, m5, [ypxq+strideq*1] + lea ypxq, [ypxq+strideq*2] paddd m0, m1 vextracti128 xm1, m0, 1 paddd m4, m0 packssdw xm1, xm0, xm1 mova [acq], xm1 - lea ypxq, [ypxq+strideq*2] add acq, 16 dec hd jg .w8_loop - jmp .w8_hpad -.w8_wpad: - pmaddwd xm0, xm2, [ypxq+strideq*0] - pmaddwd xm3, xm2, [ypxq+strideq*1] +.w8_hpad: + test hpadd, hpadd + jz .dc + vinserti128 m1, xm1, 1 + pslld m0, 2 + jmp .hpad +.w8_wpad1: + pmaddwd xm0, xm5, [ypxq+strideq*0] + pmaddwd xm3, xm5, [ypxq+strideq*1] + lea ypxq, [ypxq+strideq*2] paddd xm0, xm3 pshufd xm3, xm0, q3333 packssdw xm1, xm0, xm3 paddd xm0, xm3 paddd xm4, xm0 mova [acq], xm1 - lea ypxq, [ypxq+strideq*2] add acq, 16 dec hd - jg .w8_wpad -.w8_hpad: - test hpadd, hpadd - jz .calc_avg - vinserti128 m1, xm1, 1 - paddd m0, m0 -.w8_hpad_loop: + jg .w8_wpad1 + jmp .w8_hpad +.w16_wpad: + mova m0, [ypxq+strideq*0+ 0] + mova m1, [ypxq+strideq*1+ 0] + cmp wpadd, 2 + jl .w16_wpad1 + je .w16_wpad2 + vpbroadcastd m2, [ypxq+strideq*0+12] + vpbroadcastd m3, [ypxq+strideq*1+12] + vpblendd m0, m2, 0xf0 + vpblendd m1, m3, 0xf0 + jmp .w16_wpad_end +.w16_wpad2: + vpbroadcastd m2, [ypxq+strideq*0+28] + vpbroadcastd m3, [ypxq+strideq*1+28] + jmp .w16_wpad_end +.w16_wpad1: + vpbroadcastd m2, [ypxq+strideq*0+44] + vpbroadcastd m3, [ypxq+strideq*1+44] + vinserti128 m2, [ypxq+strideq*0+32], 0 + vinserti128 m3, [ypxq+strideq*1+32], 0 +.w16_wpad_end: + lea ypxq, [ypxq+strideq*2] + REPX {pmaddwd x, m5}, m0, m1, m2, m3 + paddd m0, m1 + paddd m2, m3 + packssdw m1, m0, m2 + paddd m0, m2 + vpermq m1, m1, q3120 paddd m4, m0 mova [acq], m1 add acq, 32 - sub hpadd, 2 - jg .w8_hpad_loop - jmp .calc_avg + dec hd + jg .w16_wpad + jmp .w16_hpad .w16: + mov r5, acq test wpadd, wpadd jnz .w16_wpad .w16_loop: - pmaddwd m0, m2, [ypxq+strideq*0+ 0] - pmaddwd m1, m2, [ypxq+strideq*1+ 0] - pmaddwd m3, m2, [ypxq+strideq*0+32] - pmaddwd m5, m2, [ypxq+strideq*1+32] + pmaddwd m0, m5, [ypxq+strideq*0+ 0] + pmaddwd m2, m5, [ypxq+strideq*0+32] + pmaddwd m1, m5, [ypxq+strideq*1+ 0] + pmaddwd m3, m5, [ypxq+strideq*1+32] + lea ypxq, [ypxq+strideq*2] paddd m0, m1 - paddd m3, m5 - packssdw m1, m0, m3 - paddd m0, m3 + paddd m2, m3 + packssdw m1, m0, m2 + paddd m0, m2 vpermq m1, m1, q3120 paddd m4, m0 mova [acq], m1 - lea ypxq, [ypxq+strideq*2] add acq, 32 dec hd jg .w16_loop - jmp .w16_hpad -.w16_wpad: -DEFINE_ARGS ac, ypx, stride, wpad, hpad, iptr, h, sz, ac_bak - lea iptrq, [ipred_cfl_ac_420_16bpc_avx2_table] - mov wpadd, wpadd - movsxd wpadq, [iptrq+wpadq*4+4] - add iptrq, wpadq - jmp iptrq -.w16_wpad_pad3: - vpbroadcastd m3, [ypxq+strideq*0+12] - vpbroadcastd m5, [ypxq+strideq*1+12] - vinserti128 m0, m3, [ypxq+strideq*0], 0 - vinserti128 m1, m5, [ypxq+strideq*1], 0 - jmp .w16_wpad_end -.w16_wpad_pad2: - mova m0, [ypxq+strideq*0+ 0] - mova m1, [ypxq+strideq*1+ 0] - vpbroadcastd m3, [ypxq+strideq*0+28] - vpbroadcastd m5, [ypxq+strideq*1+28] - jmp .w16_wpad_end -.w16_wpad_pad1: - mova m0, [ypxq+strideq*0+ 0] - mova m1, [ypxq+strideq*1+ 0] - vpbroadcastd m3, [ypxq+strideq*0+44] - vpbroadcastd m5, [ypxq+strideq*1+44] - vinserti128 m3, [ypxq+strideq*0+32], 0 - vinserti128 m5, [ypxq+strideq*1+32], 0 -.w16_wpad_end: - pmaddwd m0, m2 - pmaddwd m1, m2 - pmaddwd m3, m2 - pmaddwd m5, m2 - paddd m0, m1 - paddd m3, m5 - packssdw m1, m0, m3 - paddd m0, m3 - vpermq m1, m1, q3120 +.w16_hpad: + add hpadd, hpadd + jz .dc + paddd m0, m0 +.hpad: + mova [acq+32*0], m1 paddd m4, m0 - mova [acq], m1 - lea ypxq, [ypxq+strideq*2] - add acq, 32 - dec hd - jz .w16_hpad - jmp iptrq -.w16_hpad: - test hpadd, hpadd - jz .calc_avg -.w16_hpad_loop: - mova [acq], m1 - paddd m4, m0 - add acq, 32 - dec hpadd - jg .w16_hpad_loop -.calc_avg: - vextracti128 xm0, m4, 1 - tzcnt r1d, szd - movd xm3, szd - paddd xm0, xm4 - movd xm2, r1d - punpckhqdq xm1, xm0, xm0 - psrld xm3, 1 - paddd xm0, xm1 - pshuflw xm1, xm0, q1032 - paddd xm0, xm3 - paddd xm0, xm1 - psrld xm0, xm2 - vpbroadcastw m0, xm0 -.sub_loop: - mova m1, [ac_bakq] - psubw m1, m0 - mova [ac_bakq], m1 - add ac_bakq, 32 - sub szd, 16 - jg .sub_loop + mova [acq+32*1], m1 + add acq, 32*2 + sub hpadd, 4 + jg .hpad +.dc: + vextracti128 xm1, m4, 1 + sub r5, acq ; -w*h*2 + tzcnt r1d, r5d + paddd xm4, xm1 + sub r1d, 2 + punpckhqdq xm1, xm4, xm4 + movd xm0, r1d + paddd xm1, xm4 + pshuflw xm4, xm1, q1032 + paddd xm1, xm4 + psrld xm1, xm0 + pxor xm0, xm0 + pavgw xm1, xm0 + vpbroadcastw m1, xm1 +.dc_loop: + mova m0, [acq+r5] + psubw m0, m1 + mova [acq+r5], m0 + add r5, 32 + jl .dc_loop RET -cglobal ipred_cfl_ac_422_16bpc, 4, 9, 6, ac, ypx, stride, wpad, hpad, w, h, sz, ac_bak +cglobal ipred_cfl_ac_422_16bpc, 4, 7, 6, ac, ypx, stride, wpad, hpad, w, h movifnidn hpadd, hpadm - movifnidn wd, wm + vpbroadcastd m5, [pw_4] mov hd, hm - mov szd, wd - mov ac_bakq, acq - imul szd, hd shl hpadd, 2 + pxor m4, m4 sub hd, hpadd - vpbroadcastd m2, [pw_4] - pxor m4, m4 - cmp wd, 8 + cmp dword wm, 8 jg .w16 je .w8 -DEFINE_ARGS ac, ypx, stride, wpad, hpad, stride3, h, sz, ac_bak .w4: - lea stride3q, [strideq*3] + lea r3, [strideq*3] + mov r5, acq .w4_loop: mova xm0, [ypxq+strideq*0] mova xm1, [ypxq+strideq*1] vinserti128 m0, [ypxq+strideq*2], 1 - vinserti128 m1, [ypxq+stride3q ], 1 - pmaddwd m0, m2 - pmaddwd m1, m2 + vinserti128 m1, [ypxq+r3 ], 1 + lea ypxq, [ypxq+strideq*4] + pmaddwd m0, m5 + pmaddwd m1, m5 paddd m4, m0 packssdw m0, m1 paddd m4, m1 mova [acq], m0 - lea ypxq, [ypxq+strideq*4] add acq, 32 sub hd, 4 jg .w4_loop test hpadd, hpadd - jz .calc_avg + jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_avx2).dc + vextracti128 xm1, m1, 1 vpermq m0, m0, q3333 - vextracti128 xm1, m1, 1 pslld xm1, 2 .w4_hpad_loop: mova [acq], m0 paddd m4, m1 add acq, 32 sub hpadd, 4 jg .w4_hpad_loop - jmp .calc_avg + jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_avx2).dc .w8: + mov r5, acq test wpadd, wpadd - jnz .w8_wpad + jnz .w8_wpad1 .w8_loop: - pmaddwd m0, m2, [ypxq+strideq*0] - pmaddwd m1, m2, [ypxq+strideq*1] + pmaddwd m1, m5, [ypxq+strideq*0] + pmaddwd m0, m5, [ypxq+strideq*1] + lea ypxq, [ypxq+strideq*2] + paddd m4, m1 + packssdw m1, m0 paddd m4, m0 - packssdw m0, m1 - paddd m4, m1 - vpermq m0, m0, q3120 - mova [acq], m0 - lea ypxq, [ypxq+strideq*2] + vpermq m2, m1, q3120 + mova [acq], m2 add acq, 32 sub hd, 2 jg .w8_loop - jmp .w8_hpad -.w8_wpad: - vpbroadcastd m0, [ypxq+strideq*0+12] - vpbroadcastd m1, [ypxq+strideq*1+12] - vinserti128 m0, [ypxq+strideq*0+ 0], 0 - vinserti128 m1, [ypxq+strideq*1+ 0], 0 - pmaddwd m0, m2 - pmaddwd m1, m2 +.w8_hpad: + test hpadd, hpadd + jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_avx2).dc + vpermq m1, m1, q3131 + pslld m0, 2 + jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_avx2).hpad +.w8_wpad1: + vpbroadcastd m1, [ypxq+strideq*0+12] + vpbroadcastd m0, [ypxq+strideq*1+12] + vinserti128 m1, [ypxq+strideq*0+ 0], 0 + vinserti128 m0, [ypxq+strideq*1+ 0], 0 + lea ypxq, [ypxq+strideq*2] + pmaddwd m1, m5 + pmaddwd m0, m5 + paddd m4, m1 + packssdw m1, m0 paddd m4, m0 - packssdw m0, m1 - paddd m4, m1 - vpermq m0, m0, q3120 - mova [acq], m0 - lea ypxq, [ypxq+strideq*2] + vpermq m2, m1, q3120 + mova [acq], m2 add acq, 32 sub hd, 2 - jg .w8_wpad -.w8_hpad: - test hpadd, hpadd - jz .calc_avg - vpermq m0, m0, q3232 - paddd m1, m1 -.w8_hpad_loop: - mova [acq], m0 - paddd m4, m1 - add acq, 32 - sub hpadd, 2 - jg .w8_hpad_loop - jmp .calc_avg + jg .w8_wpad1 + jmp .w8_hpad .w16: + mov r5, acq test wpadd, wpadd jnz .w16_wpad .w16_loop: - pmaddwd m3, m2, [ypxq+strideq*0+ 0] - pmaddwd m0, m2, [ypxq+strideq*0+32] - pmaddwd m1, m2, [ypxq+strideq*1+ 0] - pmaddwd m5, m2, [ypxq+strideq*1+32] - paddd m4, m3 - packssdw m3, m0 + pmaddwd m2, m5, [ypxq+strideq*0+ 0] + pmaddwd m1, m5, [ypxq+strideq*0+32] + pmaddwd m0, m5, [ypxq+strideq*1+ 0] + pmaddwd m3, m5, [ypxq+strideq*1+32] + lea ypxq, [ypxq+strideq*2] + paddd m4, m2 + packssdw m2, m1 + paddd m4, m1 + packssdw m1, m0, m3 + paddd m0, m3 + vpermq m2, m2, q3120 paddd m4, m0 - packssdw m0, m1, m5 - paddd m1, m5 - paddd m4, m1 - vpermq m3, m3, q3120 - vpermq m0, m0, q3120 - mova [acq+ 0], m3 - mova [acq+32], m0 - lea ypxq, [ypxq+strideq*2] - add acq, 64 + vpermq m1, m1, q3120 + mova [acq+32*0], m2 + mova [acq+32*1], m1 + add acq, 32*2 sub hd, 2 jg .w16_loop - jmp .w16_hpad + jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_avx2).w16_hpad .w16_wpad: -DEFINE_ARGS ac, ypx, stride, wpad, hpad, iptr, h, sz, ac_bak - lea iptrq, [ipred_cfl_ac_422_16bpc_avx2_table] - mov wpadd, wpadd - movsxd wpadq, [iptrq+wpadq*4+4] - add iptrq, wpadq - jmp iptrq -.w16_wpad_pad3: - vpbroadcastd m0, [ypxq+strideq*0+12] + mova m2, [ypxq+strideq*0+ 0] + mova m0, [ypxq+strideq*1+ 0] + cmp wpadd, 2 + jl .w16_wpad1 + je .w16_wpad2 + vpbroadcastd m1, [ypxq+strideq*0+12] vpbroadcastd m3, [ypxq+strideq*1+12] - vinserti128 m5, m0, [ypxq+strideq*0], 0 - vinserti128 m1, m3, [ypxq+strideq*1], 0 + vpblendd m2, m1, 0xf0 + vpblendd m0, m3, 0xf0 jmp .w16_wpad_end -.w16_wpad_pad2: - mova m5, [ypxq+strideq*0+ 0] - mova m1, [ypxq+strideq*1+ 0] - vpbroadcastd m0, [ypxq+strideq*0+28] +.w16_wpad2: + vpbroadcastd m1, [ypxq+strideq*0+28] vpbroadcastd m3, [ypxq+strideq*1+28] jmp .w16_wpad_end -.w16_wpad_pad1: - mova m5, [ypxq+strideq*0+ 0] - mova m1, [ypxq+strideq*1+ 0] - vpbroadcastd m0, [ypxq+strideq*0+44] +.w16_wpad1: + vpbroadcastd m1, [ypxq+strideq*0+44] vpbroadcastd m3, [ypxq+strideq*1+44] - vinserti128 m0, [ypxq+strideq*0+32], 0 + vinserti128 m1, [ypxq+strideq*0+32], 0 vinserti128 m3, [ypxq+strideq*1+32], 0 .w16_wpad_end: - pmaddwd m5, m2 - pmaddwd m1, m2 - pmaddwd m0, m2 - pmaddwd m3, m2 - paddd m4, m5 - packssdw m5, m0 - paddd m4, m0 - packssdw m0, m1, m3 - paddd m1, m3 + lea ypxq, [ypxq+strideq*2] + REPX {pmaddwd x, m5}, m2, m0, m1, m3 + paddd m4, m2 + packssdw m2, m1 paddd m4, m1 - vpermq m5, m5, q3120 - vpermq m0, m0, q3120 - mova [acq+ 0], m5 - mova [acq+32], m0 - lea ypxq, [ypxq+strideq*2] - add acq, 64 + packssdw m1, m0, m3 + paddd m0, m3 + vpermq m2, m2, q3120 + paddd m4, m0 + vpermq m1, m1, q3120 + mova [acq+32*0], m2 + mova [acq+32*1], m1 + add acq, 32*2 sub hd, 2 - jz .w16_hpad - jmp iptrq -.w16_hpad: + jg .w16_wpad + jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_avx2).w16_hpad + +cglobal ipred_cfl_ac_444_16bpc, 4, 7, 6, ac, ypx, stride, wpad, hpad, w, h + lea r6, [ipred_cfl_ac_444_16bpc_avx2_table] + tzcnt wd, wm + movifnidn hpadd, hpadm + vpbroadcastd m5, [pw_1] + movsxd wq, [r6+wq*4] + shl hpadd, 2 + add wq, r6 + mov hd, hm + pxor m4, m4 + sub hd, hpadd + jmp wq +.w4: + lea r3, [strideq*3] + mov r5, acq +.w4_loop: + movq xm0, [ypxq+strideq*0] + movhps xm0, [ypxq+strideq*1] + vpbroadcastq m1, [ypxq+strideq*2] + vpbroadcastq m2, [ypxq+r3 ] + lea ypxq, [ypxq+strideq*4] + vpblendd m0, m1, 0x30 + vpblendd m0, m2, 0xc0 + psllw m0, 3 + pmaddwd m1, m0, m5 + mova [acq], m0 + add acq, 32 + paddd m4, m1 + sub hd, 4 + jg .w4_loop test hpadd, hpadd - jz .calc_avg -.w16_hpad_loop: - mova [acq], m0 + jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_avx2).dc + vpermq m0, m0, q3333 + paddd m1, m1 + mova [acq+32*0], m0 + vpermq m1, m1, q3333 + mova [acq+32*1], m0 + add acq, 32*2 paddd m4, m1 - add acq, 32 - dec hpadd - jg .w16_hpad_loop -.calc_avg: - vextracti128 xm0, m4, 1 - tzcnt r1d, szd - movd xm2, r1d - paddd xm0, xm4 - movd xm3, szd - punpckhqdq xm1, xm0, xm0 - paddd xm0, xm1 - psrld xm3, 1 - psrlq xm1, xm0, 32 - paddd xm0, xm3 - paddd xm0, xm1 - psrld xm0, xm2 - vpbroadcastw m0, xm0 -.sub_loop: - mova m1, [ac_bakq] - psubw m1, m0 - mova [ac_bakq], m1 - add ac_bakq, 32 - sub szd, 16 - jg .sub_loop - RET + jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_avx2).dc +.w8: + lea r3, [strideq*3] + mov r5, acq +.w8_loop: + mova xm2, [ypxq+strideq*0] + vinserti128 m2, [ypxq+strideq*1], 1 + mova xm1, [ypxq+strideq*2] + vinserti128 m1, [ypxq+r3 ], 1 + lea ypxq, [ypxq+strideq*4] + psllw m2, 3 + psllw m1, 3 + mova [acq+32*0], m2 + pmaddwd m2, m5 + mova [acq+32*1], m1 + pmaddwd m0, m1, m5 + add acq, 32*2 + paddd m4, m2 + paddd m4, m0 + sub hd, 4 + jg .w8_loop + test hpadd, hpadd + jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_avx2).dc + vperm2i128 m1, m1, 0x11 + pslld m0, 2 + pxor m2, m2 + vpblendd m0, m2, 0x0f + jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_avx2).hpad +.w16_wpad2: + vpbroadcastw m3, [ypxq+strideq*0+14] + vpbroadcastw m0, [ypxq+strideq*1+14] + vpblendd m2, m3, 0xf0 + vpblendd m1, m0, 0xf0 + jmp .w16_wpad_end +.w16: + mov r5, acq +.w16_loop: + mova m2, [ypxq+strideq*0] + mova m1, [ypxq+strideq*1] + test wpadd, wpadd + jnz .w16_wpad2 +.w16_wpad_end: + lea ypxq, [ypxq+strideq*2] + psllw m2, 3 + psllw m1, 3 + mova [acq+32*0], m2 + pmaddwd m2, m5 + mova [acq+32*1], m1 + pmaddwd m0, m1, m5 + add acq, 32*2 + paddd m4, m2 + paddd m4, m0 + sub hd, 2 + jg .w16_loop + add hpadd, hpadd + jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_avx2).dc + paddd m0, m0 + jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_avx2).hpad +.w32: + mov r5, acq + test wpadd, wpadd + jnz .w32_wpad +.w32_loop: + mova m0, [ypxq+ 0] + mova m1, [ypxq+32] + add ypxq, strideq + psllw m0, 3 + psllw m1, 3 + pmaddwd m2, m0, m5 + mova [acq+32*0], m0 + pmaddwd m3, m1, m5 + mova [acq+32*1], m1 + add acq, 32*2 + paddd m2, m3 + paddd m4, m2 + dec hd + jg .w32_loop +.w32_hpad: + test hpadd, hpadd + jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_avx2).dc + paddd m2, m2 +.w32_hpad_loop: + mova [acq+32*0], m0 + mova [acq+32*1], m1 + paddd m4, m2 + mova [acq+32*2], m0 + mova [acq+32*3], m1 + add acq, 32*4 + sub hpadd, 2 + jg .w32_hpad_loop + jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_avx2).dc +.w32_wpad: + mova m0, [ypxq+ 0] + cmp wpadd, 4 + jl .w32_wpad2 + je .w32_wpad4 + vpbroadcastw m1, [ypxq+14] + vpblendd m0, m1, 0xf0 + jmp .w32_wpad_end +.w32_wpad4: + vpbroadcastw m1, [ypxq+30] + jmp .w32_wpad_end +.w32_wpad2: + vpbroadcastw m1, [ypxq+46] + vinserti128 m1, [ypxq+32], 0 +.w32_wpad_end: + add ypxq, strideq + psllw m0, 3 + psllw m1, 3 + pmaddwd m2, m0, m5 + mova [acq+32*0], m0 + pmaddwd m3, m1, m5 + mova [acq+32*1], m1 + add acq, 32*2 + paddd m2, m3 + paddd m4, m2 + dec hd + jg .w32_wpad + jmp .w32_hpad cglobal pal_pred_16bpc, 4, 6, 5, dst, stride, pal, idx, w, h vbroadcasti128 m3, [palq] lea r2, [pal_pred_16bpc_avx2_table] tzcnt wd, wm diff --git a/third_party/dav1d/src/x86/ipred16_sse.asm b/third_party/dav1d/src/x86/ipred16_sse.asm new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/x86/ipred16_sse.asm @@ -0,0 +1,1931 @@ +; Copyright © 2021, VideoLAN and dav1d authors +; Copyright © 2021, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +SECTION_RODATA + +filter_shuf: db 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 4, 5, 2, 3, -1, -1 +pal_pred_shuf: db 0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15 + +pb_0_1: times 4 db 0, 1 +pb_2_3: times 4 db 2, 3 +pw_1: times 4 dw 1 +pw_2: times 4 dw 2 +pw_4: times 4 dw 4 +pw_512: times 4 dw 512 +pw_2048: times 4 dw 2048 + +%macro JMP_TABLE 3-* + %xdefine %1_%2_table (%%table - 2*4) + %xdefine %%base mangle(private_prefix %+ _%1_%2) + %%table: + %rep %0 - 2 + dd %%base %+ .%3 - (%%table - 2*4) + %rotate 1 + %endrep +%endmacro + +%define ipred_dc_splat_16bpc_ssse3_table (ipred_dc_16bpc_ssse3_table + 10*4) +%define ipred_dc_128_16bpc_ssse3_table (ipred_dc_16bpc_ssse3_table + 15*4) +%define ipred_cfl_splat_16bpc_ssse3_table (ipred_cfl_16bpc_ssse3_table + 8*4) + +JMP_TABLE ipred_dc_left_16bpc, ssse3, h4, h8, h16, h32, h64 +JMP_TABLE ipred_dc_16bpc, ssse3, h4, h8, h16, h32, h64, w4, w8, w16, w32, w64, \ + s4-10*4, s8-10*4, s16-10*4, s32-10*4, s64-10*4, \ + s4-15*4, s8-15*4, s16c-15*4, s32c-15*4, s64-15*4 +JMP_TABLE ipred_h_16bpc, ssse3, w4, w8, w16, w32, w64 +JMP_TABLE ipred_cfl_16bpc, ssse3, h4, h8, h16, h32, w4, w8, w16, w32, \ + s4-8*4, s8-8*4, s16-8*4, s32-8*4 +JMP_TABLE ipred_cfl_left_16bpc, ssse3, h4, h8, h16, h32 +JMP_TABLE ipred_cfl_ac_444_16bpc, ssse3, w4, w8, w16, w32 +JMP_TABLE pal_pred_16bpc, ssse3, w4, w8, w16, w32, w64 + +cextern smooth_weights_1d_16bpc +cextern smooth_weights_2d_16bpc +cextern filter_intra_taps + +SECTION .text + +%macro REPX 2-* + %xdefine %%f(x) %1 +%rep %0 - 1 + %rotate 1 + %%f(%1) +%endrep +%endmacro + +INIT_XMM ssse3 +cglobal ipred_dc_top_16bpc, 3, 7, 6, dst, stride, tl, w, h + LEA r5, ipred_dc_left_16bpc_ssse3_table + movd m4, wm + tzcnt wd, wm + add tlq, 2 + movifnidn hd, hm + pxor m3, m3 + pavgw m4, m3 + movd m5, wd + movu m0, [tlq] + movsxd r6, [r5+wq*4] + add r6, r5 + add r5, ipred_dc_128_16bpc_ssse3_table-ipred_dc_left_16bpc_ssse3_table + movsxd wq, [r5+wq*4] + add wq, r5 + jmp r6 + +cglobal ipred_dc_left_16bpc, 3, 7, 6, dst, stride, tl, w, h, stride3 + LEA r5, ipred_dc_left_16bpc_ssse3_table + mov hd, hm + movd m4, hm + tzcnt r6d, hd + sub tlq, hq + tzcnt wd, wm + pxor m3, m3 + sub tlq, hq + pavgw m4, m3 + movd m5, r6d + movu m0, [tlq] + movsxd r6, [r5+r6*4] + add r6, r5 + add r5, ipred_dc_128_16bpc_ssse3_table-ipred_dc_left_16bpc_ssse3_table + movsxd wq, [r5+wq*4] + add wq, r5 + jmp r6 +.h64: + movu m2, [tlq+112] + movu m1, [tlq+ 96] + paddw m0, m2 + movu m2, [tlq+ 80] + paddw m1, m2 + movu m2, [tlq+ 64] + paddw m0, m2 + paddw m0, m1 +.h32: + movu m1, [tlq+ 48] + movu m2, [tlq+ 32] + paddw m1, m2 + paddw m0, m1 +.h16: + movu m1, [tlq+ 16] + paddw m0, m1 +.h8: + movhlps m1, m0 + paddw m0, m1 +.h4: + punpcklwd m0, m3 + paddd m4, m0 + punpckhqdq m0, m0 + paddd m0, m4 + pshuflw m4, m0, q1032 + paddd m0, m4 + psrld m0, m5 + lea stride3q, [strideq*3] + pshuflw m0, m0, q0000 + punpcklqdq m0, m0 + jmp wq + +cglobal ipred_dc_16bpc, 4, 7, 6, dst, stride, tl, w, h, stride3 + movifnidn hd, hm + tzcnt r6d, hd + lea r5d, [wq+hq] + movd m4, r5d + tzcnt r5d, r5d + movd m5, r5d + LEA r5, ipred_dc_16bpc_ssse3_table + tzcnt wd, wd + movsxd r6, [r5+r6*4] + movsxd wq, [r5+wq*4+5*4] + pxor m3, m3 + psrlw m4, 1 + add r6, r5 + add wq, r5 + lea stride3q, [strideq*3] + jmp r6 +.h4: + movq m0, [tlq-8] + jmp wq +.w4: + movq m1, [tlq+2] + paddw m1, m0 + punpckhwd m0, m3 + punpcklwd m1, m3 + paddd m0, m1 + paddd m4, m0 + punpckhqdq m0, m0 + paddd m0, m4 + pshuflw m1, m0, q1032 + paddd m0, m1 + cmp hd, 4 + jg .w4_mul + psrlw m0, 3 + jmp .w4_end +.w4_mul: + mov r2d, 0xAAAB + mov r3d, 0x6667 + cmp hd, 16 + cmove r2d, r3d + psrld m0, 2 + movd m1, r2d + pmulhuw m0, m1 + psrlw m0, 1 +.w4_end: + pshuflw m0, m0, q0000 +.s4: + movq [dstq+strideq*0], m0 + movq [dstq+strideq*1], m0 + movq [dstq+strideq*2], m0 + movq [dstq+stride3q ], m0 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .s4 + RET +.h8: + mova m0, [tlq-16] + jmp wq +.w8: + movu m1, [tlq+2] + paddw m0, m1 + punpcklwd m1, m0, m3 + punpckhwd m0, m3 + paddd m0, m1 + paddd m4, m0 + punpckhqdq m0, m0 + paddd m0, m4 + pshuflw m1, m0, q1032 + paddd m0, m1 + psrld m0, m5 + cmp hd, 8 + je .w8_end + mov r2d, 0xAAAB + mov r3d, 0x6667 + cmp hd, 32 + cmove r2d, r3d + movd m1, r2d + pmulhuw m0, m1 + psrlw m0, 1 +.w8_end: + pshuflw m0, m0, q0000 + punpcklqdq m0, m0 +.s8: + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m0 + mova [dstq+strideq*2], m0 + mova [dstq+stride3q ], m0 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .s8 + RET +.h16: + mova m0, [tlq-32] + paddw m0, [tlq-16] + jmp wq +.w16: + movu m1, [tlq+ 2] + movu m2, [tlq+18] + paddw m1, m2 + paddw m0, m1 + punpckhwd m1, m0, m3 + punpcklwd m0, m3 + paddd m0, m1 + paddd m4, m0 + punpckhqdq m0, m0 + paddd m0, m4 + pshuflw m1, m0, q1032 + paddd m0, m1 + psrld m0, m5 + cmp hd, 16 + je .w16_end + mov r2d, 0xAAAB + mov r3d, 0x6667 + test hd, 8|32 + cmovz r2d, r3d + movd m1, r2d + pmulhuw m0, m1 + psrlw m0, 1 +.w16_end: + pshuflw m0, m0, q0000 + punpcklqdq m0, m0 +.s16c: + mova m1, m0 +.s16: + mova [dstq+strideq*0+16*0], m0 + mova [dstq+strideq*0+16*1], m1 + mova [dstq+strideq*1+16*0], m0 + mova [dstq+strideq*1+16*1], m1 + mova [dstq+strideq*2+16*0], m0 + mova [dstq+strideq*2+16*1], m1 + mova [dstq+stride3q +16*0], m0 + mova [dstq+stride3q +16*1], m1 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .s16 + RET +.h32: + mova m0, [tlq-64] + paddw m0, [tlq-48] + paddw m0, [tlq-32] + paddw m0, [tlq-16] + jmp wq +.w32: + movu m1, [tlq+ 2] + movu m2, [tlq+18] + paddw m1, m2 + movu m2, [tlq+34] + paddw m0, m2 + movu m2, [tlq+50] + paddw m1, m2 + paddw m0, m1 + punpcklwd m1, m0, m3 + punpckhwd m0, m3 + paddd m0, m1 + paddd m4, m0 + punpckhqdq m0, m0 + paddd m0, m4 + pshuflw m1, m0, q1032 + paddd m0, m1 + psrld m0, m5 + cmp hd, 32 + je .w32_end + mov r2d, 0xAAAB + mov r3d, 0x6667 + cmp hd, 8 + cmove r2d, r3d + movd m1, r2d + pmulhuw m0, m1 + psrlw m0, 1 +.w32_end: + pshuflw m0, m0, q0000 + punpcklqdq m0, m0 +.s32c: + mova m1, m0 + mova m2, m0 + mova m3, m0 +.s32: + mova [dstq+strideq*0+16*0], m0 + mova [dstq+strideq*0+16*1], m1 + mova [dstq+strideq*0+16*2], m2 + mova [dstq+strideq*0+16*3], m3 + mova [dstq+strideq*1+16*0], m0 + mova [dstq+strideq*1+16*1], m1 + mova [dstq+strideq*1+16*2], m2 + mova [dstq+strideq*1+16*3], m3 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .s32 + RET +.h64: + mova m0, [tlq-128] + mova m1, [tlq-112] + paddw m0, [tlq- 96] + paddw m1, [tlq- 80] + paddw m0, [tlq- 64] + paddw m1, [tlq- 48] + paddw m0, [tlq- 32] + paddw m1, [tlq- 16] + paddw m0, m1 + jmp wq +.w64: + movu m1, [tlq+ 2] + movu m2, [tlq+ 18] + paddw m1, m2 + movu m2, [tlq+ 34] + paddw m0, m2 + movu m2, [tlq+ 50] + paddw m1, m2 + movu m2, [tlq+ 66] + paddw m0, m2 + movu m2, [tlq+ 82] + paddw m1, m2 + movu m2, [tlq+ 98] + paddw m0, m2 + movu m2, [tlq+114] + paddw m1, m2 + paddw m0, m1 + punpcklwd m1, m0, m3 + punpckhwd m0, m3 + paddd m0, m1 + paddd m4, m0 + punpckhqdq m0, m0 + paddd m0, m4 + pshuflw m1, m0, q1032 + paddd m0, m1 + psrld m0, m5 + cmp hd, 64 + je .w64_end + mov r2d, 0xAAAB + mov r3d, 0x6667 + cmp hd, 16 + cmove r2d, r3d + movd m1, r2d + pmulhuw m0, m1 + psrlw m0, 1 +.w64_end: + pshuflw m0, m0, q0000 + punpcklqdq m0, m0 +.s64: + mova [dstq+16*0], m0 + mova [dstq+16*1], m0 + mova [dstq+16*2], m0 + mova [dstq+16*3], m0 + mova [dstq+16*4], m0 + mova [dstq+16*5], m0 + mova [dstq+16*6], m0 + mova [dstq+16*7], m0 + add dstq, strideq + dec hd + jg .s64 + RET + +cglobal ipred_dc_128_16bpc, 2, 7, 6, dst, stride, tl, w, h, stride3 + mov r6d, r8m + LEA r5, ipred_dc_128_16bpc_ssse3_table + tzcnt wd, wm + shr r6d, 11 + movifnidn hd, hm + movsxd wq, [r5+wq*4] + movddup m0, [r5-ipred_dc_128_16bpc_ssse3_table+pw_512+r6*8] + add wq, r5 + lea stride3q, [strideq*3] + jmp wq + +cglobal ipred_v_16bpc, 4, 7, 6, dst, stride, tl, w, h, stride3 + LEA r5, ipred_dc_splat_16bpc_ssse3_table + movifnidn hd, hm + movu m0, [tlq+ 2] + movu m1, [tlq+ 18] + movu m2, [tlq+ 34] + movu m3, [tlq+ 50] + cmp wd, 64 + je .w64 + tzcnt wd, wd + movsxd wq, [r5+wq*4] + add wq, r5 + lea stride3q, [strideq*3] + jmp wq +.w64: + WIN64_SPILL_XMM 8 + movu m4, [tlq+ 66] + movu m5, [tlq+ 82] + movu m6, [tlq+ 98] + movu m7, [tlq+114] +.w64_loop: + mova [dstq+16*0], m0 + mova [dstq+16*1], m1 + mova [dstq+16*2], m2 + mova [dstq+16*3], m3 + mova [dstq+16*4], m4 + mova [dstq+16*5], m5 + mova [dstq+16*6], m6 + mova [dstq+16*7], m7 + add dstq, strideq + dec hd + jg .w64_loop + RET + +cglobal ipred_h_16bpc, 3, 6, 4, dst, stride, tl, w, h, stride3 +%define base r5-ipred_h_16bpc_ssse3_table + tzcnt wd, wm + LEA r5, ipred_h_16bpc_ssse3_table + movifnidn hd, hm + movsxd wq, [r5+wq*4] + movddup m2, [base+pb_0_1] + movddup m3, [base+pb_2_3] + add wq, r5 + lea stride3q, [strideq*3] + jmp wq +.w4: + sub tlq, 8 + movq m3, [tlq] + pshuflw m0, m3, q3333 + pshuflw m1, m3, q2222 + pshuflw m2, m3, q1111 + pshuflw m3, m3, q0000 + movq [dstq+strideq*0], m0 + movq [dstq+strideq*1], m1 + movq [dstq+strideq*2], m2 + movq [dstq+stride3q ], m3 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w4 + RET +.w8: + sub tlq, 8 + movq m3, [tlq] + punpcklwd m3, m3 + pshufd m0, m3, q3333 + pshufd m1, m3, q2222 + pshufd m2, m3, q1111 + pshufd m3, m3, q0000 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + mova [dstq+strideq*2], m2 + mova [dstq+stride3q ], m3 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w8 + RET +.w16: + sub tlq, 4 + movd m1, [tlq] + pshufb m0, m1, m3 + pshufb m1, m2 + mova [dstq+strideq*0+16*0], m0 + mova [dstq+strideq*0+16*1], m0 + mova [dstq+strideq*1+16*0], m1 + mova [dstq+strideq*1+16*1], m1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w16 + RET +.w32: + sub tlq, 4 + movd m1, [tlq] + pshufb m0, m1, m3 + pshufb m1, m2 + mova [dstq+strideq*0+16*0], m0 + mova [dstq+strideq*0+16*1], m0 + mova [dstq+strideq*0+16*2], m0 + mova [dstq+strideq*0+16*3], m0 + mova [dstq+strideq*1+16*0], m1 + mova [dstq+strideq*1+16*1], m1 + mova [dstq+strideq*1+16*2], m1 + mova [dstq+strideq*1+16*3], m1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w32 + RET +.w64: + sub tlq, 2 + movd m0, [tlq] + pshufb m0, m2 + mova [dstq+16*0], m0 + mova [dstq+16*1], m0 + mova [dstq+16*2], m0 + mova [dstq+16*3], m0 + mova [dstq+16*4], m0 + mova [dstq+16*5], m0 + mova [dstq+16*6], m0 + mova [dstq+16*7], m0 + add dstq, strideq + dec hd + jg .w64 + RET + +cglobal ipred_paeth_16bpc, 4, 6, 8, dst, stride, tl, w, h, left +%define base r5-ipred_paeth_16bpc_ssse3_table + movifnidn hd, hm + pshuflw m4, [tlq], q0000 + mov leftq, tlq + add hd, hd + punpcklqdq m4, m4 ; topleft + sub leftq, hq + and wd, ~7 + jnz .w8 + movddup m5, [tlq+2] ; top + psubw m6, m5, m4 + pabsw m7, m6 +.w4_loop: + movd m1, [leftq+hq-4] + punpcklwd m1, m1 + punpckldq m1, m1 ; left +%macro PAETH 0 + paddw m0, m6, m1 + psubw m2, m4, m0 ; tldiff + psubw m0, m5 ; tdiff + pabsw m2, m2 + pabsw m0, m0 + pminsw m2, m0 + pcmpeqw m0, m2 + pand m3, m5, m0 + pandn m0, m4 + por m0, m3 + pcmpgtw m3, m7, m2 + pand m0, m3 + pandn m3, m1 + por m0, m3 +%endmacro + PAETH + movhps [dstq+strideq*0], m0 + movq [dstq+strideq*1], m0 + lea dstq, [dstq+strideq*2] + sub hd, 2*2 + jg .w4_loop + RET +.w8: +%if ARCH_X86_32 + PUSH r6 + %define r7d hm + %assign regs_used 7 +%elif WIN64 + movaps r4m, m8 + PUSH r7 + %assign regs_used 8 +%endif +%if ARCH_X86_64 + movddup m8, [pb_0_1] +%endif + lea tlq, [tlq+wq*2+2] + neg wq + mov r7d, hd +.w8_loop0: + movu m5, [tlq+wq*2] + mov r6, dstq + add dstq, 16 + psubw m6, m5, m4 + pabsw m7, m6 +.w8_loop: + movd m1, [leftq+hq-2] +%if ARCH_X86_64 + pshufb m1, m8 +%else + pshuflw m1, m1, q0000 + punpcklqdq m1, m1 +%endif + PAETH + mova [r6], m0 + add r6, strideq + sub hd, 1*2 + jg .w8_loop + mov hd, r7d + add wq, 8 + jl .w8_loop0 +%if WIN64 + movaps m8, r4m +%endif + RET + +%if ARCH_X86_64 +DECLARE_REG_TMP 7 +%else +DECLARE_REG_TMP 4 +%endif + +cglobal ipred_smooth_v_16bpc, 4, 6, 6, dst, stride, tl, w, h, weights + LEA weightsq, smooth_weights_1d_16bpc + mov hd, hm + lea weightsq, [weightsq+hq*4] + neg hq + movd m5, [tlq+hq*2] ; bottom + pshuflw m5, m5, q0000 + punpcklqdq m5, m5 + cmp wd, 4 + jne .w8 + movddup m4, [tlq+2] ; top + lea r3, [strideq*3] + psubw m4, m5 ; top - bottom +.w4_loop: + movq m1, [weightsq+hq*2] + punpcklwd m1, m1 + pshufd m0, m1, q1100 + punpckhdq m1, m1 + pmulhrsw m0, m4 + pmulhrsw m1, m4 + paddw m0, m5 + paddw m1, m5 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + movq [dstq+strideq*2], m1 + movhps [dstq+r3 ], m1 + lea dstq, [dstq+strideq*4] + add hq, 4 + jl .w4_loop + RET +.w8: +%if ARCH_X86_32 + PUSH r6 + %assign regs_used 7 + mov hm, hq + %define hq hm +%elif WIN64 + PUSH r7 + %assign regs_used 8 +%endif +.w8_loop0: + mov t0, hq + movu m4, [tlq+2] + add tlq, 16 + mov r6, dstq + add dstq, 16 + psubw m4, m5 +.w8_loop: + movq m3, [weightsq+t0*2] + punpcklwd m3, m3 + pshufd m0, m3, q0000 + pshufd m1, m3, q1111 + pshufd m2, m3, q2222 + pshufd m3, m3, q3333 + REPX {pmulhrsw x, m4}, m0, m1, m2, m3 + REPX {paddw x, m5}, m0, m1, m2, m3 + mova [r6+strideq*0], m0 + mova [r6+strideq*1], m1 + lea r6, [r6+strideq*2] + mova [r6+strideq*0], m2 + mova [r6+strideq*1], m3 + lea r6, [r6+strideq*2] + add t0, 4 + jl .w8_loop + sub wd, 8 + jg .w8_loop0 + RET + +cglobal ipred_smooth_h_16bpc, 3, 6, 6, dst, stride, tl, w, h, weights + LEA weightsq, smooth_weights_1d_16bpc + mov wd, wm + movifnidn hd, hm + movd m5, [tlq+wq*2] ; right + sub tlq, 8 + add hd, hd + pshuflw m5, m5, q0000 + sub tlq, hq + punpcklqdq m5, m5 + cmp wd, 4 + jne .w8 + movddup m4, [weightsq+4*2] + lea r3, [strideq*3] +.w4_loop: + movq m1, [tlq+hq] ; left + punpcklwd m1, m1 + psubw m1, m5 ; left - right + pshufd m0, m1, q3322 + punpckldq m1, m1 + pmulhrsw m0, m4 + pmulhrsw m1, m4 + paddw m0, m5 + paddw m1, m5 + movhps [dstq+strideq*0], m0 + movq [dstq+strideq*1], m0 + movhps [dstq+strideq*2], m1 + movq [dstq+r3 ], m1 + lea dstq, [dstq+strideq*4] + sub hd, 4*2 + jg .w4_loop + RET +.w8: + lea weightsq, [weightsq+wq*4] + neg wq +%if ARCH_X86_32 + PUSH r6 + %assign regs_used 7 + %define hd hm +%elif WIN64 + PUSH r7 + %assign regs_used 8 +%endif +.w8_loop0: + mov t0d, hd + mova m4, [weightsq+wq*2] + mov r6, dstq + add dstq, 16 +.w8_loop: + movq m3, [tlq+t0*(1+ARCH_X86_32)] + punpcklwd m3, m3 + psubw m3, m5 + pshufd m0, m3, q3333 + pshufd m1, m3, q2222 + pshufd m2, m3, q1111 + pshufd m3, m3, q0000 + REPX {pmulhrsw x, m4}, m0, m1, m2, m3 + REPX {paddw x, m5}, m0, m1, m2, m3 + mova [r6+strideq*0], m0 + mova [r6+strideq*1], m1 + lea r6, [r6+strideq*2] + mova [r6+strideq*0], m2 + mova [r6+strideq*1], m3 + lea r6, [r6+strideq*2] + sub t0d, 4*(1+ARCH_X86_64) + jg .w8_loop + add wq, 8 + jl .w8_loop0 + RET + +%if ARCH_X86_64 +DECLARE_REG_TMP 10 +%else +DECLARE_REG_TMP 3 +%endif + +cglobal ipred_smooth_16bpc, 3, 7, 8, dst, stride, tl, w, h, \ + h_weights, v_weights, top + LEA h_weightsq, smooth_weights_2d_16bpc + mov wd, wm + mov hd, hm + movd m7, [tlq+wq*2] ; right + lea v_weightsq, [h_weightsq+hq*8] + neg hq + movd m6, [tlq+hq*2] ; bottom + pshuflw m7, m7, q0000 + pshuflw m6, m6, q0000 + cmp wd, 4 + jne .w8 + movq m4, [tlq+2] ; top + mova m5, [h_weightsq+4*4] + punpcklwd m4, m6 ; top, bottom + pxor m6, m6 +.w4_loop: + movq m1, [v_weightsq+hq*4] + sub tlq, 4 + movd m3, [tlq] ; left + pshufd m0, m1, q0000 + pshufd m1, m1, q1111 + pmaddwd m0, m4 + punpcklwd m3, m7 ; left, right + pmaddwd m1, m4 + pshufd m2, m3, q1111 + pshufd m3, m3, q0000 + pmaddwd m2, m5 + pmaddwd m3, m5 + paddd m0, m2 + paddd m1, m3 + psrld m0, 8 + psrld m1, 8 + packssdw m0, m1 + pavgw m0, m6 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + lea dstq, [dstq+strideq*2] + add hq, 2 + jl .w4_loop + RET +.w8: +%if ARCH_X86_32 + lea h_weightsq, [h_weightsq+wq*4] + mov t0, tlq + mov r1m, tlq + mov r2m, hq + %define m8 [h_weightsq+16*0] + %define m9 [h_weightsq+16*1] +%else +%if WIN64 + movaps r4m, m8 + movaps r6m, m9 + PUSH r7 + PUSH r8 +%endif + PUSH r9 + PUSH r10 + %assign regs_used 11 + lea h_weightsq, [h_weightsq+wq*8] + lea topq, [tlq+wq*2] + neg wq + mov r8, tlq + mov r9, hq +%endif + punpcklqdq m6, m6 +.w8_loop0: +%if ARCH_X86_32 + movu m5, [t0+2] + add t0, 16 + mov r0m, t0 +%else + movu m5, [topq+wq*2+2] + mova m8, [h_weightsq+wq*4+16*0] + mova m9, [h_weightsq+wq*4+16*1] +%endif + mov t0, dstq + add dstq, 16 + punpcklwd m4, m5, m6 + punpckhwd m5, m6 +.w8_loop: + movd m1, [v_weightsq+hq*4] + sub tlq, 2 + movd m3, [tlq] ; left + pshufd m1, m1, q0000 + pmaddwd m0, m4, m1 + pshuflw m3, m3, q0000 + pmaddwd m1, m5 + punpcklwd m3, m7 ; left, right + pmaddwd m2, m8, m3 + pmaddwd m3, m9 + paddd m0, m2 + paddd m1, m3 + psrld m0, 8 + psrld m1, 8 + packssdw m0, m1 + pxor m1, m1 + pavgw m0, m1 + mova [t0], m0 + add t0, strideq + inc hq + jl .w8_loop +%if ARCH_X86_32 + mov t0, r0m + mov tlq, r1m + add h_weightsq, 16*2 + mov hq, r2m + sub dword wm, 8 + jg .w8_loop0 +%else + mov tlq, r8 + mov hq, r9 + add wq, 8 + jl .w8_loop0 +%endif +%if WIN64 + movaps m8, r4m + movaps m9, r6m +%endif + RET + +%if ARCH_X86_64 +cglobal ipred_filter_16bpc, 4, 7, 16, dst, stride, tl, w, h, filter +%else +cglobal ipred_filter_16bpc, 4, 7, 8, -16*8, dst, stride, tl, w, h, filter +%define m8 [esp+16*0] +%define m9 [esp+16*1] +%define m10 [esp+16*2] +%define m11 [esp+16*3] +%define m12 [esp+16*4] +%define m13 [esp+16*5] +%define m14 [esp+16*6] +%define m15 [esp+16*7] +%endif +%define base r6-$$ + movifnidn hd, hm + movd m6, r8m ; bitdepth_max +%ifidn filterd, filterm + movzx filterd, filterb +%else + movzx filterd, byte filterm +%endif + LEA r6, $$ + shl filterd, 6 + movu m0, [tlq-6] ; __ l1 l0 tl t0 t1 t2 t3 + mova m1, [base+filter_intra_taps+filterq+16*0] + mova m2, [base+filter_intra_taps+filterq+16*1] + mova m3, [base+filter_intra_taps+filterq+16*2] + mova m4, [base+filter_intra_taps+filterq+16*3] + pxor m5, m5 +%if ARCH_X86_64 + punpcklbw m8, m5, m1 ; place 8-bit coefficients in the upper + punpckhbw m9, m5, m1 ; half of each 16-bit word to avoid + punpcklbw m10, m5, m2 ; having to perform sign-extension. + punpckhbw m11, m5, m2 + punpcklbw m12, m5, m3 + punpckhbw m13, m5, m3 + punpcklbw m14, m5, m4 + punpckhbw m15, m5, m4 +%else + punpcklbw m7, m5, m1 + mova m8, m7 + punpckhbw m7, m5, m1 + mova m9, m7 + punpcklbw m7, m5, m2 + mova m10, m7 + punpckhbw m7, m5, m2 + mova m11, m7 + punpcklbw m7, m5, m3 + mova m12, m7 + punpckhbw m7, m5, m3 + mova m13, m7 + punpcklbw m7, m5, m4 + mova m14, m7 + punpckhbw m7, m5, m4 + mova m15, m7 +%endif + mova m7, [base+filter_shuf] + add hd, hd + mov r5, dstq + pshuflw m6, m6, q0000 + mov r6, tlq + punpcklqdq m6, m6 + sub tlq, hq +.left_loop: + pshufb m0, m7 ; tl t0 t1 t2 t3 l0 l1 __ + pshufd m1, m0, q0000 + pmaddwd m2, m8, m1 + pmaddwd m1, m9 + pshufd m4, m0, q1111 + pmaddwd m3, m10, m4 + pmaddwd m4, m11 + paddd m2, m3 + paddd m1, m4 + pshufd m4, m0, q2222 + pmaddwd m3, m12, m4 + pmaddwd m4, m13 + paddd m2, m3 + paddd m1, m4 + pshufd m3, m0, q3333 + pmaddwd m0, m14, m3 + pmaddwd m3, m15 + paddd m0, m2 + paddd m1, m3 + psrad m0, 11 ; x >> 3 + psrad m1, 11 + packssdw m0, m1 + pmaxsw m0, m5 + pavgw m0, m5 ; (x + 8) >> 4 + pminsw m0, m6 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + movlps m0, [tlq+hq-10] + lea dstq, [dstq+strideq*2] + sub hd, 2*2 + jg .left_loop + sub wd, 4 + jz .end + sub tld, r6d ; -h*2 + sub r6, r5 ; tl-dst +.right_loop0: + add r5, 8 + mov hd, tld + movu m0, [r5+r6] ; tl t0 t1 t2 t3 __ __ __ + mov dstq, r5 +.right_loop: + pshufd m2, m0, q0000 + pmaddwd m1, m8, m2 + pmaddwd m2, m9 + pshufd m4, m0, q1111 + pmaddwd m3, m10, m4 + pmaddwd m4, m11 + pinsrw m0, [dstq+strideq*0-2], 5 + paddd m1, m3 + paddd m2, m4 + pshufd m0, m0, q2222 + movddup m4, [dstq+strideq*1-8] + pmaddwd m3, m12, m0 + pmaddwd m0, m13 + paddd m1, m3 + paddd m0, m2 + pshuflw m2, m4, q3333 + punpcklwd m2, m5 + pmaddwd m3, m14, m2 + pmaddwd m2, m15 + paddd m1, m3 + paddd m0, m2 + psrad m1, 11 + psrad m0, 11 + packssdw m0, m1 + pmaxsw m0, m5 + pavgw m0, m5 + pminsw m0, m6 + movhps [dstq+strideq*0], m0 + movq [dstq+strideq*1], m0 + palignr m0, m4, 14 + lea dstq, [dstq+strideq*2] + add hd, 2*2 + jl .right_loop + sub wd, 4 + jg .right_loop0 +.end: + RET + +%if UNIX64 +DECLARE_REG_TMP 7 +%else +DECLARE_REG_TMP 5 +%endif + +cglobal ipred_cfl_top_16bpc, 4, 7, 8, dst, stride, tl, w, h, ac + LEA t0, ipred_cfl_left_16bpc_ssse3_table + movd m4, wd + tzcnt wd, wd + movifnidn hd, hm + add tlq, 2 + movsxd r6, [t0+wq*4] + movd m5, wd + jmp mangle(private_prefix %+ _ipred_cfl_left_16bpc_ssse3.start) + +cglobal ipred_cfl_left_16bpc, 3, 7, 8, dst, stride, tl, w, h, ac, alpha + movifnidn hd, hm + LEA t0, ipred_cfl_left_16bpc_ssse3_table + tzcnt wd, wm + lea r6d, [hq*2] + movd m4, hd + sub tlq, r6 + tzcnt r6d, hd + movd m5, r6d + movsxd r6, [t0+r6*4] +.start: + movd m7, r7m + movu m0, [tlq] + add r6, t0 + add t0, ipred_cfl_splat_16bpc_ssse3_table-ipred_cfl_left_16bpc_ssse3_table + movsxd wq, [t0+wq*4] + pxor m6, m6 + pshuflw m7, m7, q0000 + pcmpeqw m3, m3 + add wq, t0 + movifnidn acq, acmp + pavgw m4, m6 + punpcklqdq m7, m7 + jmp r6 +.h32: + movu m1, [tlq+48] + movu m2, [tlq+32] + paddw m0, m1 + paddw m0, m2 +.h16: + movu m1, [tlq+16] + paddw m0, m1 +.h8: + pshufd m1, m0, q1032 + paddw m0, m1 +.h4: + pmaddwd m0, m3 + psubd m4, m0 + pshuflw m0, m4, q1032 + paddd m0, m4 + psrld m0, m5 + pshuflw m0, m0, q0000 + punpcklqdq m0, m0 + jmp wq + +%macro IPRED_CFL 2 ; dst, src + pabsw m%1, m%2 + pmulhrsw m%1, m2 + psignw m%2, m1 + psignw m%1, m%2 + paddw m%1, m0 + pmaxsw m%1, m6 + pminsw m%1, m7 +%endmacro + +cglobal ipred_cfl_16bpc, 4, 7, 8, dst, stride, tl, w, h, ac, alpha + movifnidn hd, hm + tzcnt r6d, hd + lea t0d, [wq+hq] + movd m4, t0d + tzcnt t0d, t0d + movd m5, t0d + LEA t0, ipred_cfl_16bpc_ssse3_table + tzcnt wd, wd + movd m7, r7m + movsxd r6, [t0+r6*4] + movsxd wq, [t0+wq*4+4*4] + psrlw m4, 1 + pxor m6, m6 + pshuflw m7, m7, q0000 + add r6, t0 + add wq, t0 + movifnidn acq, acmp + pcmpeqw m3, m3 + punpcklqdq m7, m7 + jmp r6 +.h4: + movq m0, [tlq-8] + jmp wq +.w4: + movq m1, [tlq+2] + paddw m0, m1 + pmaddwd m0, m3 + psubd m4, m0 + pshufd m0, m4, q1032 + paddd m0, m4 + pshuflw m4, m0, q1032 + paddd m0, m4 + cmp hd, 4 + jg .w4_mul + psrld m0, 3 + jmp .w4_end +.w4_mul: + mov r6d, 0xAAAB + mov r2d, 0x6667 + cmp hd, 16 + cmove r6d, r2d + movd m1, r6d + psrld m0, 2 + pmulhuw m0, m1 + psrlw m0, 1 +.w4_end: + pshuflw m0, m0, q0000 + punpcklqdq m0, m0 +.s4: + movd m1, alpham + lea r6, [strideq*3] + pshuflw m1, m1, q0000 + punpcklqdq m1, m1 + pabsw m2, m1 + psllw m2, 9 +.s4_loop: + mova m4, [acq+16*0] + mova m5, [acq+16*1] + add acq, 16*2 + IPRED_CFL 3, 4 + IPRED_CFL 4, 5 + movq [dstq+strideq*0], m3 + movhps [dstq+strideq*1], m3 + movq [dstq+strideq*2], m4 + movhps [dstq+r6 ], m4 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .s4_loop + RET +.h8: + mova m0, [tlq-16] + jmp wq +.w8: + movu m1, [tlq+2] + paddw m0, m1 + pmaddwd m0, m3 + psubd m4, m0 + pshufd m0, m4, q1032 + paddd m0, m4 + pshuflw m4, m0, q1032 + paddd m0, m4 + psrld m0, m5 + cmp hd, 8 + je .w8_end + mov r6d, 0xAAAB + mov r2d, 0x6667 + cmp hd, 32 + cmove r6d, r2d + movd m1, r6d + pmulhuw m0, m1 + psrlw m0, 1 +.w8_end: + pshuflw m0, m0, q0000 + punpcklqdq m0, m0 +.s8: + movd m1, alpham + pshuflw m1, m1, q0000 + punpcklqdq m1, m1 + pabsw m2, m1 + psllw m2, 9 +.s8_loop: + mova m4, [acq+16*0] + mova m5, [acq+16*1] + add acq, 16*2 + IPRED_CFL 3, 4 + IPRED_CFL 4, 5 + mova [dstq+strideq*0], m3 + mova [dstq+strideq*1], m4 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .s8_loop + RET +.h16: + mova m0, [tlq-32] + paddw m0, [tlq-16] + jmp wq +.w16: + movu m1, [tlq+ 2] + movu m2, [tlq+18] + paddw m1, m2 + paddw m0, m1 + pmaddwd m0, m3 + psubd m4, m0 + pshufd m0, m4, q1032 + paddd m0, m4 + pshuflw m4, m0, q1032 + paddd m0, m4 + psrld m0, m5 + cmp hd, 16 + je .w16_end + mov r6d, 0xAAAB + mov r2d, 0x6667 + test hd, 8|32 + cmovz r6d, r2d + movd m1, r6d + pmulhuw m0, m1 + psrlw m0, 1 +.w16_end: + pshuflw m0, m0, q0000 + punpcklqdq m0, m0 +.s16: + movd m1, alpham + pshuflw m1, m1, q0000 + punpcklqdq m1, m1 + pabsw m2, m1 + psllw m2, 9 +.s16_loop: + mova m4, [acq+16*0] + mova m5, [acq+16*1] + add acq, 16*2 + IPRED_CFL 3, 4 + IPRED_CFL 4, 5 + mova [dstq+16*0], m3 + mova [dstq+16*1], m4 + add dstq, strideq + dec hd + jg .s16_loop + RET +.h32: + mova m0, [tlq-64] + paddw m0, [tlq-48] + paddw m0, [tlq-32] + paddw m0, [tlq-16] + jmp wq +.w32: + movu m1, [tlq+ 2] + movu m2, [tlq+18] + paddw m1, m2 + movu m2, [tlq+34] + paddw m1, m2 + movu m2, [tlq+50] + paddw m1, m2 + paddw m0, m1 + pmaddwd m0, m3 + psubd m4, m0 + pshufd m0, m4, q1032 + paddd m0, m4 + pshuflw m4, m0, q1032 + paddd m0, m4 + psrld m0, m5 + cmp hd, 32 + je .w32_end + mov r6d, 0xAAAB + mov r2d, 0x6667 + cmp hd, 8 + cmove r6d, r2d + movd m1, r6d + pmulhuw m0, m1 + psrlw m0, 1 +.w32_end: + pshuflw m0, m0, q0000 + punpcklqdq m0, m0 +.s32: + movd m1, alpham + pshuflw m1, m1, q0000 + punpcklqdq m1, m1 + pabsw m2, m1 + psllw m2, 9 +.s32_loop: + mova m4, [acq+16*0] + mova m5, [acq+16*1] + IPRED_CFL 3, 4 + IPRED_CFL 4, 5 + mova [dstq+16*0], m3 + mova [dstq+16*1], m4 + mova m4, [acq+16*2] + mova m5, [acq+16*3] + add acq, 16*4 + IPRED_CFL 3, 4 + IPRED_CFL 4, 5 + mova [dstq+16*2], m3 + mova [dstq+16*3], m4 + add dstq, strideq + dec hd + jg .s32_loop + RET + +cglobal ipred_cfl_128_16bpc, 3, 7, 8, dst, stride, tl, w, h, ac + tzcnt wd, wm + LEA t0, ipred_cfl_splat_16bpc_ssse3_table + mov r6d, r7m + movifnidn hd, hm + shr r6d, 11 + movd m7, r7m + movsxd wq, [t0+wq*4] + movddup m0, [t0-ipred_cfl_splat_16bpc_ssse3_table+pw_512+r6*8] + pshuflw m7, m7, q0000 + pxor m6, m6 + add wq, t0 + movifnidn acq, acmp + punpcklqdq m7, m7 + jmp wq + +cglobal ipred_cfl_ac_420_16bpc, 3, 7, 6, ac, ypx, stride, wpad, hpad, w, h + movifnidn hpadd, hpadm +%if ARCH_X86_32 && PIC + pcmpeqw m5, m5 + pabsw m5, m5 + paddw m5, m5 +%else + movddup m5, [pw_2] +%endif + mov hd, hm + shl hpadd, 2 + pxor m4, m4 + sub hd, hpadd + cmp dword wm, 8 + mov r5, acq + jg .w16 + je .w8 + lea r3, [strideq*3] +.w4_loop: + pmaddwd m0, m5, [ypxq+strideq*0] + pmaddwd m1, m5, [ypxq+strideq*1] + pmaddwd m2, m5, [ypxq+strideq*2] + pmaddwd m3, m5, [ypxq+r3 ] + lea ypxq, [ypxq+strideq*4] + paddd m0, m1 + paddd m2, m3 + paddd m4, m0 + packssdw m0, m2 + paddd m4, m2 + mova [acq], m0 + add acq, 16 + sub hd, 2 + jg .w4_loop + test hpadd, hpadd + jz .dc + punpckhqdq m0, m0 + pslld m2, 2 +.w4_hpad: + mova [acq+16*0], m0 + paddd m4, m2 + mova [acq+16*1], m0 + add acq, 16*2 + sub hpadd, 4 + jg .w4_hpad + jmp .dc +.w8: +%if ARCH_X86_32 + cmp dword wpadm, 0 +%else + test wpadd, wpadd +%endif + jnz .w8_wpad1 +.w8_loop: + pmaddwd m0, m5, [ypxq+strideq*0+16*0] + pmaddwd m2, m5, [ypxq+strideq*1+16*0] + pmaddwd m1, m5, [ypxq+strideq*0+16*1] + pmaddwd m3, m5, [ypxq+strideq*1+16*1] + lea ypxq, [ypxq+strideq*2] + paddd m0, m2 + paddd m1, m3 + paddd m2, m0, m1 + packssdw m0, m1 + paddd m4, m2 + mova [acq], m0 + add acq, 16 + dec hd + jg .w8_loop +.w8_hpad: + test hpadd, hpadd + jz .dc + pslld m2, 2 + mova m1, m0 + jmp .hpad +.w8_wpad1: + pmaddwd m0, m5, [ypxq+strideq*0] + pmaddwd m1, m5, [ypxq+strideq*1] + lea ypxq, [ypxq+strideq*2] + paddd m0, m1 + pshufd m1, m0, q3333 + paddd m2, m0, m1 + packssdw m0, m1 + paddd m4, m2 + mova [acq], m0 + add acq, 16 + dec hd + jg .w8_wpad1 + jmp .w8_hpad +.w16_wpad3: + pshufd m3, m0, q3333 + mova m1, m3 + mova m2, m3 + jmp .w16_wpad_end +.w16_wpad2: + pshufd m1, m3, q3333 + mova m2, m1 + jmp .w16_wpad_end +.w16_wpad1: + pshufd m2, m1, q3333 + jmp .w16_wpad_end +.w16: + movifnidn wpadd, wpadm + WIN64_SPILL_XMM 7 +.w16_loop: + pmaddwd m0, m5, [ypxq+strideq*0+16*0] + pmaddwd m6, m5, [ypxq+strideq*1+16*0] + paddd m0, m6 + cmp wpadd, 2 + jg .w16_wpad3 + pmaddwd m3, m5, [ypxq+strideq*0+16*1] + pmaddwd m6, m5, [ypxq+strideq*1+16*1] + paddd m3, m6 + je .w16_wpad2 + pmaddwd m1, m5, [ypxq+strideq*0+16*2] + pmaddwd m6, m5, [ypxq+strideq*1+16*2] + paddd m1, m6 + jp .w16_wpad1 + pmaddwd m2, m5, [ypxq+strideq*0+16*3] + pmaddwd m6, m5, [ypxq+strideq*1+16*3] + paddd m2, m6 +.w16_wpad_end: + lea ypxq, [ypxq+strideq*2] + paddd m6, m0, m3 + packssdw m0, m3 + paddd m6, m1 + mova [acq+16*0], m0 + packssdw m1, m2 + paddd m2, m6 + mova [acq+16*1], m1 + add acq, 16*2 + paddd m4, m2 + dec hd + jg .w16_loop + WIN64_RESTORE_XMM + add hpadd, hpadd + jz .dc + paddd m2, m2 +.hpad: + mova [acq+16*0], m0 + mova [acq+16*1], m1 + paddd m4, m2 + mova [acq+16*2], m0 + mova [acq+16*3], m1 + add acq, 16*4 + sub hpadd, 4 + jg .hpad +.dc: + sub r5, acq ; -w*h*2 + pshufd m2, m4, q1032 + tzcnt r1d, r5d + paddd m2, m4 + sub r1d, 2 + pshufd m4, m2, q2301 + movd m0, r1d + paddd m2, m4 + psrld m2, m0 + pxor m0, m0 + pavgw m2, m0 + packssdw m2, m2 +.dc_loop: + mova m0, [acq+r5+16*0] + mova m1, [acq+r5+16*1] + psubw m0, m2 + psubw m1, m2 + mova [acq+r5+16*0], m0 + mova [acq+r5+16*1], m1 + add r5, 16*2 + jl .dc_loop + RET + +cglobal ipred_cfl_ac_422_16bpc, 3, 7, 6, ac, ypx, stride, wpad, hpad, w, h + movifnidn hpadd, hpadm +%if ARCH_X86_32 && PIC + pcmpeqw m5, m5 + pabsw m5, m5 + psllw m5, 2 +%else + movddup m5, [pw_4] +%endif + mov hd, hm + shl hpadd, 2 + pxor m4, m4 + sub hd, hpadd + cmp dword wm, 8 + mov r5, acq + jg .w16 + je .w8 + lea r3, [strideq*3] +.w4_loop: + pmaddwd m0, m5, [ypxq+strideq*0] + pmaddwd m3, m5, [ypxq+strideq*1] + pmaddwd m1, m5, [ypxq+strideq*2] + pmaddwd m2, m5, [ypxq+r3 ] + lea ypxq, [ypxq+strideq*4] + paddd m4, m0 + packssdw m0, m3 + paddd m3, m1 + packssdw m1, m2 + paddd m4, m2 + paddd m4, m3 + mova [acq+16*0], m0 + mova [acq+16*1], m1 + add acq, 16*2 + sub hd, 4 + jg .w4_loop + test hpadd, hpadd + jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).dc + punpckhqdq m1, m1 + pslld m2, 3 + mova [acq+16*0], m1 + mova [acq+16*1], m1 + paddd m4, m2 + mova [acq+16*2], m1 + mova [acq+16*3], m1 + add acq, 16*4 + jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).dc +.w8: +%if ARCH_X86_32 + cmp dword wpadm, 0 +%else + test wpadd, wpadd +%endif + jnz .w8_wpad1 +.w8_loop: + pmaddwd m0, m5, [ypxq+strideq*0+16*0] + pmaddwd m2, m5, [ypxq+strideq*0+16*1] + pmaddwd m1, m5, [ypxq+strideq*1+16*0] + pmaddwd m3, m5, [ypxq+strideq*1+16*1] + lea ypxq, [ypxq+strideq*2] + paddd m4, m0 + packssdw m0, m2 + paddd m4, m2 + mova [acq+16*0], m0 + paddd m2, m1, m3 + packssdw m1, m3 + paddd m4, m2 + mova [acq+16*1], m1 + add acq, 16*2 + sub hd, 2 + jg .w8_loop +.w8_hpad: + test hpadd, hpadd + jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).dc + pslld m2, 2 + mova m0, m1 + jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).hpad +.w8_wpad1: + pmaddwd m0, m5, [ypxq+strideq*0] + pmaddwd m1, m5, [ypxq+strideq*1] + lea ypxq, [ypxq+strideq*2] + pshufd m2, m0, q3333 + pshufd m3, m1, q3333 + paddd m4, m0 + packssdw m0, m2 + paddd m4, m2 + paddd m2, m1, m3 + packssdw m1, m3 + paddd m4, m2 + mova [acq+16*0], m0 + mova [acq+16*1], m1 + add acq, 16*2 + sub hd, 2 + jg .w8_wpad1 + jmp .w8_hpad +.w16_wpad3: + pshufd m3, m0, q3333 + mova m1, m3 + mova m2, m3 + jmp .w16_wpad_end +.w16_wpad2: + pshufd m1, m3, q3333 + mova m2, m1 + jmp .w16_wpad_end +.w16_wpad1: + pshufd m2, m1, q3333 + jmp .w16_wpad_end +.w16: + movifnidn wpadd, wpadm + WIN64_SPILL_XMM 7 +.w16_loop: + pmaddwd m0, m5, [ypxq+16*0] + cmp wpadd, 2 + jg .w16_wpad3 + pmaddwd m3, m5, [ypxq+16*1] + je .w16_wpad2 + pmaddwd m1, m5, [ypxq+16*2] + jp .w16_wpad1 + pmaddwd m2, m5, [ypxq+16*3] +.w16_wpad_end: + add ypxq, strideq + paddd m6, m0, m3 + packssdw m0, m3 + mova [acq+16*0], m0 + paddd m6, m1 + packssdw m1, m2 + paddd m2, m6 + mova [acq+16*1], m1 + add acq, 16*2 + paddd m4, m2 + dec hd + jg .w16_loop + WIN64_RESTORE_XMM + add hpadd, hpadd + jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).dc + paddd m2, m2 + jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).hpad + +cglobal ipred_cfl_ac_444_16bpc, 3, 7, 6, ac, ypx, stride, wpad, hpad, w, h +%define base r6-ipred_cfl_ac_444_16bpc_ssse3_table + LEA r6, ipred_cfl_ac_444_16bpc_ssse3_table + tzcnt wd, wm + movifnidn hpadd, hpadm + pxor m4, m4 + movsxd wq, [r6+wq*4] + movddup m5, [base+pw_1] + add wq, r6 + mov hd, hm + shl hpadd, 2 + sub hd, hpadd + jmp wq +.w4: + lea r3, [strideq*3] + mov r5, acq +.w4_loop: + movq m0, [ypxq+strideq*0] + movhps m0, [ypxq+strideq*1] + movq m1, [ypxq+strideq*2] + movhps m1, [ypxq+r3 ] + lea ypxq, [ypxq+strideq*4] + psllw m0, 3 + psllw m1, 3 + mova [acq+16*0], m0 + pmaddwd m0, m5 + mova [acq+16*1], m1 + pmaddwd m2, m5, m1 + add acq, 16*2 + paddd m4, m0 + paddd m4, m2 + sub hd, 4 + jg .w4_loop + test hpadd, hpadd + jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).dc + punpckhqdq m1, m1 + mova [acq+16*0], m1 + pslld m2, 2 + mova [acq+16*1], m1 + punpckhqdq m2, m2 + mova [acq+16*2], m1 + paddd m4, m2 + mova [acq+16*3], m1 + add acq, 16*4 + jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).dc +.w8: + mov r5, acq +.w8_loop: + mova m0, [ypxq+strideq*0] + mova m1, [ypxq+strideq*1] + lea ypxq, [ypxq+strideq*2] + psllw m0, 3 + psllw m1, 3 + mova [acq+16*0], m0 + pmaddwd m0, m5 + mova [acq+16*1], m1 + pmaddwd m2, m5, m1 + add acq, 16*2 + paddd m4, m0 + paddd m4, m2 + sub hd, 2 + jg .w8_loop +.w8_hpad: + test hpadd, hpadd + jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).dc + pslld m2, 2 + mova m0, m1 + jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).hpad +.w16_wpad2: + pshufhw m3, m2, q3333 + pshufhw m1, m0, q3333 + punpckhqdq m3, m3 + punpckhqdq m1, m1 + jmp .w16_wpad_end +.w16: + movifnidn wpadd, wpadm + mov r5, acq +.w16_loop: + mova m2, [ypxq+strideq*0+16*0] + mova m0, [ypxq+strideq*1+16*0] + psllw m2, 3 + psllw m0, 3 + test wpadd, wpadd + jnz .w16_wpad2 + mova m3, [ypxq+strideq*0+16*1] + mova m1, [ypxq+strideq*1+16*1] + psllw m3, 3 + psllw m1, 3 +.w16_wpad_end: + lea ypxq, [ypxq+strideq*2] + mova [acq+16*0], m2 + pmaddwd m2, m5 + mova [acq+16*1], m3 + pmaddwd m3, m5 + paddd m4, m2 + pmaddwd m2, m5, m0 + mova [acq+16*2], m0 + paddd m4, m3 + pmaddwd m3, m5, m1 + mova [acq+16*3], m1 + add acq, 16*4 + paddd m2, m3 + paddd m4, m2 + sub hd, 2 + jg .w16_loop + add hpadd, hpadd + jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).dc + paddd m2, m2 + jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).hpad +.w32_wpad6: + pshufhw m1, m0, q3333 + punpckhqdq m1, m1 + mova m2, m1 + mova m3, m1 + jmp .w32_wpad_end +.w32_wpad4: + pshufhw m2, m1, q3333 + punpckhqdq m2, m2 + mova m3, m2 + jmp .w32_wpad_end +.w32_wpad2: + pshufhw m3, m2, q3333 + punpckhqdq m3, m3 + jmp .w32_wpad_end +.w32: + movifnidn wpadd, wpadm + mov r5, acq + WIN64_SPILL_XMM 8 +.w32_loop: + mova m0, [ypxq+16*0] + psllw m0, 3 + cmp wpadd, 4 + jg .w32_wpad6 + mova m1, [ypxq+16*1] + psllw m1, 3 + je .w32_wpad4 + mova m2, [ypxq+16*2] + psllw m2, 3 + jnp .w32_wpad2 + mova m3, [ypxq+16*3] + psllw m3, 3 +.w32_wpad_end: + add ypxq, strideq + pmaddwd m6, m5, m0 + mova [acq+16*0], m0 + pmaddwd m7, m5, m1 + mova [acq+16*1], m1 + paddd m6, m7 + pmaddwd m7, m5, m2 + mova [acq+16*2], m2 + paddd m6, m7 + pmaddwd m7, m5, m3 + mova [acq+16*3], m3 + add acq, 16*4 + paddd m6, m7 + paddd m4, m6 + dec hd + jg .w32_loop +%if WIN64 + mova m5, m6 + WIN64_RESTORE_XMM + SWAP 5, 6 +%endif + test hpadd, hpadd + jz mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).dc +.w32_hpad_loop: + mova [acq+16*0], m0 + mova [acq+16*1], m1 + paddd m4, m6 + mova [acq+16*2], m2 + mova [acq+16*3], m3 + add acq, 16*4 + dec hpadd + jg .w32_hpad_loop + jmp mangle(private_prefix %+ _ipred_cfl_ac_420_16bpc_ssse3).dc + +cglobal pal_pred_16bpc, 4, 5, 5, dst, stride, pal, idx, w, h +%define base r2-pal_pred_16bpc_ssse3_table +%if ARCH_X86_32 + %define hd r2d +%endif + mova m3, [palq] + LEA r2, pal_pred_16bpc_ssse3_table + tzcnt wd, wm + pshufb m3, [base+pal_pred_shuf] + movsxd wq, [r2+wq*4] + pshufd m4, m3, q1032 + add wq, r2 + movifnidn hd, hm + jmp wq +.w4: + mova m0, [idxq] + add idxq, 16 + pshufb m1, m3, m0 + pshufb m2, m4, m0 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + lea dstq, [dstq+strideq*2] + movq [dstq+strideq*0], m1 + movhps [dstq+strideq*1], m1 + lea dstq, [dstq+strideq*2] + sub hd, 4 + jg .w4 + RET +.w8: + mova m0, [idxq] + add idxq, 16 + pshufb m1, m3, m0 + pshufb m2, m4, m0 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w8 + RET +.w16: + mova m0, [idxq] + add idxq, 16 + pshufb m1, m3, m0 + pshufb m2, m4, m0 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + mova [dstq+16*0], m0 + mova [dstq+16*1], m1 + add dstq, strideq + dec hd + jg .w16 + RET +.w32: + mova m0, [idxq+16*0] + pshufb m1, m3, m0 + pshufb m2, m4, m0 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + mova m2, [idxq+16*1] + add idxq, 16*2 + mova [dstq+16*0], m0 + pshufb m0, m3, m2 + mova [dstq+16*1], m1 + pshufb m1, m4, m2 + punpcklbw m2, m0, m1 + punpckhbw m0, m1 + mova [dstq+16*2], m2 + mova [dstq+16*3], m0 + add dstq, strideq + dec hd + jg .w32 + RET +.w64: + mova m0, [idxq+16*0] + pshufb m1, m3, m0 + pshufb m2, m4, m0 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + mova m2, [idxq+16*1] + mova [dstq+16*0], m0 + pshufb m0, m3, m2 + mova [dstq+16*1], m1 + pshufb m1, m4, m2 + punpcklbw m2, m0, m1 + punpckhbw m0, m1 + mova m1, [idxq+16*2] + mova [dstq+16*2], m2 + pshufb m2, m3, m1 + mova [dstq+16*3], m0 + pshufb m0, m4, m1 + punpcklbw m1, m2, m0 + punpckhbw m2, m0 + mova m0, [idxq+16*3] + add idxq, 16*4 + mova [dstq+16*4], m1 + pshufb m1, m3, m0 + mova [dstq+16*5], m2 + pshufb m2, m4, m0 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + mova [dstq+16*6], m0 + mova [dstq+16*7], m1 + add dstq, strideq + dec hd + jg .w64 + RET diff --git a/third_party/dav1d/src/x86/ipred_avx2.asm b/third_party/dav1d/src/x86/ipred_avx2.asm --- a/third_party/dav1d/src/x86/ipred_avx2.asm +++ b/third_party/dav1d/src/x86/ipred_avx2.asm @@ -1,6 +1,6 @@ -; Copyright © 2018, VideoLAN and dav1d authors +; Copyright © 2018-2021, VideoLAN and dav1d authors ; Copyright © 2018, Two Orioles, LLC ; All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions are met: @@ -139,11 +139,11 @@ pw_255: times 2 dw 255 pw_512: times 2 dw 512 %macro JMP_TABLE 3-* %xdefine %1_%2_table (%%table - 2*4) - %xdefine %%base mangle(private_prefix %+ _%1_%2) + %xdefine %%base mangle(private_prefix %+ _%1_8bpc_%2) %%table: %rep %0 - 2 dd %%base %+ .%3 - (%%table - 2*4) %rotate 1 %endrep @@ -176,11 +176,11 @@ cextern filter_intra_taps SECTION .text INIT_YMM avx2 -cglobal ipred_dc_top, 3, 7, 6, dst, stride, tl, w, h +cglobal ipred_dc_top_8bpc, 3, 7, 6, dst, stride, tl, w, h lea r5, [ipred_dc_left_avx2_table] tzcnt wd, wm inc tlq movu m0, [tlq] movifnidn hd, hm @@ -194,11 +194,11 @@ add r5, ipred_dc_splat_avx2_table-ipred_dc_left_avx2_table movsxd wq, [r5+wq*4] add wq, r5 jmp r6 -cglobal ipred_dc_left, 3, 7, 6, dst, stride, tl, w, h, stride3 +cglobal ipred_dc_left_8bpc, 3, 7, 6, dst, stride, tl, w, h, stride3 mov hd, hm ; zero upper half tzcnt r6d, hd sub tlq, hq tzcnt wd, wm movu m0, [tlq] @@ -233,11 +233,11 @@ lea stride3q, [strideq*3] vpbroadcastb m0, xm0 mova m1, m0 jmp wq -cglobal ipred_dc, 3, 7, 6, dst, stride, tl, w, h, stride3 +cglobal ipred_dc_8bpc, 3, 7, 6, dst, stride, tl, w, h, stride3 movifnidn hd, hm movifnidn wd, wm tzcnt r6d, hd lea r5d, [wq+hq] movd xm4, r5d @@ -444,22 +444,22 @@ lea dstq, [dstq+strideq*4] sub hd, 4 jg .s64 RET -cglobal ipred_dc_128, 2, 7, 6, dst, stride, tl, w, h, stride3 +cglobal ipred_dc_128_8bpc, 2, 7, 6, dst, stride, tl, w, h, stride3 lea r5, [ipred_dc_splat_avx2_table] tzcnt wd, wm movifnidn hd, hm movsxd wq, [r5+wq*4] vpbroadcastd m0, [r5-ipred_dc_splat_avx2_table+pb_128] mova m1, m0 add wq, r5 lea stride3q, [strideq*3] jmp wq -cglobal ipred_v, 3, 7, 6, dst, stride, tl, w, h, stride3 +cglobal ipred_v_8bpc, 3, 7, 6, dst, stride, tl, w, h, stride3 lea r5, [ipred_dc_splat_avx2_table] tzcnt wd, wm movu m0, [tlq+ 1] movu m1, [tlq+33] movifnidn hd, hm @@ -484,11 +484,11 @@ RET ALIGN function_align %endmacro INIT_XMM avx2 -cglobal ipred_h, 3, 6, 4, dst, stride, tl, w, h, stride3 +cglobal ipred_h_8bpc, 3, 6, 4, dst, stride, tl, w, h, stride3 lea r5, [ipred_h_avx2_table] tzcnt wd, wm movifnidn hd, hm movsxd wq, [r5+wq*4] add wq, r5 @@ -541,11 +541,11 @@ pminub m1, m2 pcmpeqb m1, m2 ; ldiff <= tldiff || tdiff <= tldiff vpblendvb m0, m5, m0, m1 %endmacro -cglobal ipred_paeth, 3, 6, 9, dst, stride, tl, w, h +cglobal ipred_paeth_8bpc, 3, 6, 9, dst, stride, tl, w, h %define base r5-ipred_paeth_avx2_table lea r5, [ipred_paeth_avx2_table] tzcnt wd, wm vpbroadcastb m5, [tlq] ; topleft movifnidn hd, hm @@ -675,11 +675,11 @@ psrlw m0, 8 psrlw m1, 8 packuswb m0, m1 %endmacro -cglobal ipred_smooth_v, 3, 7, 0, dst, stride, tl, w, h, weights +cglobal ipred_smooth_v_8bpc, 3, 7, 0, dst, stride, tl, w, h, weights %define base r6-ipred_smooth_v_avx2_table lea r6, [ipred_smooth_v_avx2_table] tzcnt wd, wm mov hd, hm movsxd wq, [r6+wq*4] @@ -833,11 +833,11 @@ PUSH r%2 %endif ALLOC_STACK %1, %3 %endmacro -cglobal ipred_smooth_h, 3, 7, 0, dst, stride, tl, w, h +cglobal ipred_smooth_h_8bpc, 3, 7, 0, dst, stride, tl, w, h %define base r6-ipred_smooth_h_avx2_table lea r6, [ipred_smooth_h_avx2_table] mov wd, wm vpbroadcastb m3, [tlq+wq] ; right tzcnt wd, wd @@ -1043,11 +1043,11 @@ psrlw m0, 8 psrlw m1, 8 packuswb m0, m1 %endmacro -cglobal ipred_smooth, 3, 7, 0, dst, stride, tl, w, h, v_weights +cglobal ipred_smooth_8bpc, 3, 7, 0, dst, stride, tl, w, h, v_weights %define base r6-ipred_smooth_avx2_table lea r6, [ipred_smooth_avx2_table] mov wd, wm vpbroadcastb m4, [tlq+wq] ; right tzcnt wd, wd @@ -1313,11 +1313,11 @@ sub r3, hq sub tlq, hq sub r3, hq ret -cglobal ipred_z1, 3, 8, 0, dst, stride, tl, w, h, angle, dx, maxbase +cglobal ipred_z1_8bpc, 3, 8, 0, dst, stride, tl, w, h, angle, dx, maxbase %assign org_stack_offset stack_offset lea r6, [ipred_z1_avx2_table] tzcnt wd, wm movifnidn angled, anglem movifnidn hd, hm @@ -2142,11 +2142,11 @@ dec hd jg .w64_end_loop .w64_end: RET -cglobal ipred_z2, 3, 10, 16, 224, dst, stride, tl, w, h, angle, dx, dy +cglobal ipred_z2_8bpc, 3, 10, 16, 224, dst, stride, tl, w, h, angle, dx, dy %define base r9-z_filter_t0 lea r9, [ipred_z2_avx2_table] tzcnt wd, wm movifnidn angled, anglem movifnidn hd, hm @@ -2998,11 +2998,11 @@ movu xm3, [tlq+ 6] vinserti128 m3, [tlq+19], 1 movu [rsp+97], m0 jmp .w32_filter_above -cglobal ipred_z3, 4, 9, 0, dst, stride, tl, w, h, angle, dy, org_w, maxbase +cglobal ipred_z3_8bpc, 4, 9, 0, dst, stride, tl, w, h, angle, dy, org_w, maxbase %assign org_stack_offset stack_offset lea r6, [ipred_z3_avx2_table] tzcnt hd, hm movifnidn angled, anglem lea r7, [dr_intra_derivative+45*2-1] @@ -4209,11 +4209,11 @@ ; 2 2 3 2 4 5 7 2 4 5 7 c e f h ; 3 3 4 4 6 7 9 4 6 7 9 e g h j ; ___ 4 ___ 4 5 ___ 6 8 9 a ___ 6 8 9 a g i j k ___ ; 5 8 8 i -cglobal ipred_filter, 3, 7, 0, dst, stride, tl, w, h, filter +cglobal ipred_filter_8bpc, 3, 7, 0, dst, stride, tl, w, h, filter %define base r6-ipred_filter_avx2_table lea r6, [filter_intra_taps] tzcnt wd, wm %ifidn filterd, filterm movzx filterd, filterb @@ -4433,11 +4433,11 @@ pmulhrsw m%1, m2 psignw m%1, m3 paddw m%1, m0 %endmacro -cglobal ipred_cfl_top, 3, 7, 6, dst, stride, tl, w, h, ac, alpha +cglobal ipred_cfl_top_8bpc, 3, 7, 6, dst, stride, tl, w, h, ac, alpha lea t0, [ipred_cfl_left_avx2_table] tzcnt wd, wm inc tlq movu m0, [tlq] movifnidn hd, hm @@ -4452,11 +4452,11 @@ movsxd wq, [t0+wq*4] add wq, t0 movifnidn acq, acmp jmp r6 -cglobal ipred_cfl_left, 3, 7, 6, dst, stride, tl, w, h, ac, alpha +cglobal ipred_cfl_left_8bpc, 3, 7, 6, dst, stride, tl, w, h, ac, alpha mov hd, hm ; zero upper half tzcnt r6d, hd sub tlq, hq tzcnt wd, wm movu m0, [tlq] @@ -4486,11 +4486,11 @@ pmaddwd xm0, xm2 pmulhrsw xm0, xm3 vpbroadcastw m0, xm0 jmp wq -cglobal ipred_cfl, 3, 7, 6, dst, stride, tl, w, h, ac, alpha +cglobal ipred_cfl_8bpc, 3, 7, 6, dst, stride, tl, w, h, ac, alpha movifnidn hd, hm movifnidn wd, wm tzcnt r6d, hd lea t0d, [wq+hq] movd xm4, t0d @@ -4690,21 +4690,21 @@ add acq, 64 dec hd jg .s32_loop RET -cglobal ipred_cfl_128, 3, 7, 6, dst, stride, tl, w, h, ac, alpha +cglobal ipred_cfl_128_8bpc, 3, 7, 6, dst, stride, tl, w, h, ac, alpha lea t0, [ipred_cfl_splat_avx2_table] tzcnt wd, wm movifnidn hd, hm movsxd wq, [t0+wq*4] vpbroadcastd m0, [t0-ipred_cfl_splat_avx2_table+pw_128] add wq, t0 movifnidn acq, acmp jmp wq -cglobal ipred_cfl_ac_420, 4, 9, 5, ac, y, stride, wpad, hpad, w, h, sz, ac_bak +cglobal ipred_cfl_ac_420_8bpc, 4, 9, 5, ac, y, stride, wpad, hpad, w, h, sz, ac_bak movifnidn hpadd, hpadm movifnidn wd, wm mov hd, hm mov szd, wd mov ac_bakq, acq @@ -4881,11 +4881,11 @@ add ac_bakq, 32 sub szd, 16 jg .sub_loop RET -cglobal ipred_cfl_ac_422, 4, 9, 6, ac, y, stride, wpad, hpad, w, h, sz, ac_bak +cglobal ipred_cfl_ac_422_8bpc, 4, 9, 6, ac, y, stride, wpad, hpad, w, h, sz, ac_bak movifnidn hpadd, hpadm movifnidn wd, wm mov hd, hm mov szd, wd mov ac_bakq, acq @@ -5074,11 +5074,11 @@ add ac_bakq, 32 sub szd, 16 jg .sub_loop RET -cglobal ipred_cfl_ac_444, 4, 9, 6, ac, y, stride, wpad, hpad, w, h, sz, ac_bak +cglobal ipred_cfl_ac_444_8bpc, 4, 9, 6, ac, y, stride, wpad, hpad, w, h, sz, ac_bak movifnidn hpadd, hpadm movifnidn wd, wm mov hd, hm mov szd, wd imul szd, hd @@ -5304,11 +5304,11 @@ add ac_bakq, 32 sub szd, 16 jg .sub_loop RET -cglobal pal_pred, 4, 6, 5, dst, stride, pal, idx, w, h +cglobal pal_pred_8bpc, 4, 6, 5, dst, stride, pal, idx, w, h vbroadcasti128 m4, [palq] lea r2, [pal_pred_avx2_table] tzcnt wd, wm movifnidn hd, hm movsxd wq, [r2+wq*4] diff --git a/third_party/dav1d/src/x86/ipred_init_tmpl.c b/third_party/dav1d/src/x86/ipred_init_tmpl.c --- a/third_party/dav1d/src/x86/ipred_init_tmpl.c +++ b/third_party/dav1d/src/x86/ipred_init_tmpl.c @@ -1,7 +1,7 @@ /* - * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018-2021, VideoLAN and dav1d authors * Copyright © 2018, Two Orioles, LLC * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -26,23 +26,15 @@ */ #include "src/cpu.h" #include "src/ipred.h" -#if BITDEPTH == 8 #define decl_fn(type, name) \ - decl_##type##_fn(dav1d_##name##_ssse3); \ - decl_##type##_fn(dav1d_##name##_avx2) + decl_##type##_fn(BF(dav1d_##name, ssse3)); \ + decl_##type##_fn(BF(dav1d_##name, avx2)) #define init_fn(type0, type1, name, suffix) \ - c->type0[type1] = dav1d_##name##_##suffix -#else -#define decl_fn(type, name) \ - decl_##type##_fn(dav1d_##name##_16bpc_ssse3); \ - decl_##type##_fn(dav1d_##name##_16bpc_avx2) -#define init_fn(type0, type1, name, suffix) \ - c->type0[type1] = dav1d_##name##_16bpc_##suffix -#endif + c->type0[type1] = BF(dav1d_##name, suffix) #define init_angular_ipred_fn(type, name, suffix) \ init_fn(intra_pred, type, name, suffix) #define init_cfl_pred_fn(type, name, suffix) \ init_fn(cfl_pred, type, name, suffix) @@ -78,11 +70,10 @@ COLD void bitfn(dav1d_intra_pred_dsp_init_x86)(Dav1dIntraPredDSPContext *const c) { const unsigned flags = dav1d_get_cpu_flags(); if (!(flags & DAV1D_X86_CPU_FLAG_SSSE3)) return; -#if BITDEPTH == 8 init_angular_ipred_fn(DC_PRED, ipred_dc, ssse3); init_angular_ipred_fn(DC_128_PRED, ipred_dc_128, ssse3); init_angular_ipred_fn(TOP_DC_PRED, ipred_dc_top, ssse3); init_angular_ipred_fn(LEFT_DC_PRED, ipred_dc_left, ssse3); init_angular_ipred_fn(HOR_PRED, ipred_h, ssse3); @@ -100,12 +91,11 @@ init_cfl_ac_fn(DAV1D_PIXEL_LAYOUT_I420 - 1, ipred_cfl_ac_420, ssse3); init_cfl_ac_fn(DAV1D_PIXEL_LAYOUT_I422 - 1, ipred_cfl_ac_422, ssse3); init_cfl_ac_fn(DAV1D_PIXEL_LAYOUT_I444 - 1, ipred_cfl_ac_444, ssse3); - c->pal_pred = dav1d_pal_pred_ssse3; -#endif + c->pal_pred = BF(dav1d_pal_pred, ssse3); #if ARCH_X86_64 if (!(flags & DAV1D_X86_CPU_FLAG_AVX2)) return; init_angular_ipred_fn(DC_PRED, ipred_dc, avx2); @@ -128,14 +118,10 @@ init_cfl_pred_fn(TOP_DC_PRED, ipred_cfl_top, avx2); init_cfl_pred_fn(LEFT_DC_PRED, ipred_cfl_left, avx2); init_cfl_ac_fn(DAV1D_PIXEL_LAYOUT_I420 - 1, ipred_cfl_ac_420, avx2); init_cfl_ac_fn(DAV1D_PIXEL_LAYOUT_I422 - 1, ipred_cfl_ac_422, avx2); -#if BITDEPTH == 8 init_cfl_ac_fn(DAV1D_PIXEL_LAYOUT_I444 - 1, ipred_cfl_ac_444, avx2); - c->pal_pred = dav1d_pal_pred_avx2; -#else - c->pal_pred = dav1d_pal_pred_16bpc_avx2; -#endif + c->pal_pred = BF(dav1d_pal_pred, avx2); #endif } diff --git a/third_party/dav1d/src/x86/ipred_sse.asm b/third_party/dav1d/src/x86/ipred_sse.asm --- a/third_party/dav1d/src/x86/ipred_sse.asm +++ b/third_party/dav1d/src/x86/ipred_sse.asm @@ -1,6 +1,6 @@ -; Copyright © 2018, VideoLAN and dav1d authors +; Copyright © 2018-2021, VideoLAN and dav1d authors ; Copyright © 2018, Two Orioles, LLC ; All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions are met: @@ -72,11 +72,11 @@ pd_32768 : times 1 dd 32768 %macro JMP_TABLE 3-* %xdefine %1_%2_table (%%table - 2*4) - %xdefine %%base mangle(private_prefix %+ _%1_%2) + %xdefine %%base mangle(private_prefix %+ _%1_8bpc_%2) %%table: %rep %0 - 2 dd %%base %+ .%3 - (%%table - 2*4) %rotate 1 %endrep @@ -154,11 +154,11 @@ jg .w%1 RET %endmacro INIT_XMM ssse3 -cglobal ipred_h, 3, 6, 2, dst, stride, tl, w, h, stride3 +cglobal ipred_h_8bpc, 3, 6, 2, dst, stride, tl, w, h, stride3 LEA r5, ipred_h_ssse3_table tzcnt wd, wm movifnidn hd, hm movsxd wq, [r5+wq*4] add wq, r5 @@ -177,11 +177,11 @@ ;--------------------------------------------------------------------------------------- ;int dav1d_ipred_v_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft, ; const int width, const int height, const int a); ;--------------------------------------------------------------------------------------- -cglobal ipred_v, 3, 7, 6, dst, stride, tl, w, h, stride3 +cglobal ipred_v_8bpc, 3, 7, 6, dst, stride, tl, w, h, stride3 LEA r5, ipred_dc_splat_ssse3_table tzcnt wd, wm movu m0, [tlq+ 1] movu m1, [tlq+17] movu m2, [tlq+33] @@ -194,11 +194,11 @@ ;--------------------------------------------------------------------------------------- ;int dav1d_ipred_dc_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft, ; const int width, const int height, const int a); ;--------------------------------------------------------------------------------------- -cglobal ipred_dc, 3, 7, 6, dst, stride, tl, w, h, stride3 +cglobal ipred_dc_8bpc, 3, 7, 6, dst, stride, tl, w, h, stride3 movifnidn hd, hm movifnidn wd, wm tzcnt r6d, hd lea r5d, [wq+hq] movd m4, r5d @@ -436,11 +436,11 @@ ;--------------------------------------------------------------------------------------- ;int dav1d_ipred_dc_left_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft, ; const int width, const int height, const int a); ;--------------------------------------------------------------------------------------- -cglobal ipred_dc_left, 3, 7, 6, dst, stride, tl, w, h, stride3 +cglobal ipred_dc_left_8bpc, 3, 7, 6, dst, stride, tl, w, h, stride3 LEA r5, ipred_dc_left_ssse3_table mov hd, hm ; zero upper half tzcnt r6d, hd sub tlq, hq tzcnt wd, wm @@ -486,11 +486,11 @@ ;--------------------------------------------------------------------------------------- ;int dav1d_ipred_dc_128_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft, ; const int width, const int height, const int a); ;--------------------------------------------------------------------------------------- -cglobal ipred_dc_128, 2, 7, 6, dst, stride, tl, w, h, stride3 +cglobal ipred_dc_128_8bpc, 2, 7, 6, dst, stride, tl, w, h, stride3 LEA r5, ipred_dc_splat_ssse3_table tzcnt wd, wm movifnidn hd, hm movsxd wq, [r5+wq*4] movddup m0, [r5-ipred_dc_splat_ssse3_table+pb_128] @@ -503,11 +503,11 @@ ;--------------------------------------------------------------------------------------- ;int dav1d_ipred_dc_top_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft, ; const int width, const int height, const int a); ;--------------------------------------------------------------------------------------- -cglobal ipred_dc_top, 3, 7, 6, dst, stride, tl, w, h +cglobal ipred_dc_top_8bpc, 3, 7, 6, dst, stride, tl, w, h LEA r5, ipred_dc_left_ssse3_table tzcnt wd, wm inc tlq movu m0, [tlq] movifnidn hd, hm @@ -538,11 +538,11 @@ psrlw m6, 8 psrlw m0, 8 packuswb m6, m0 %endmacro -cglobal ipred_smooth_v, 3, 7, 7, dst, stride, tl, w, h, weights +cglobal ipred_smooth_v_8bpc, 3, 7, 7, dst, stride, tl, w, h, weights %define base r6-ipred_smooth_v_ssse3_table LEA r6, ipred_smooth_v_ssse3_table tzcnt wd, wm mov hd, hm movsxd wq, [r6+wq*4] @@ -699,11 +699,11 @@ ;--------------------------------------------------------------------------------------- ;int dav1d_ipred_smooth_h_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft, ; const int width, const int height, const int a); ;--------------------------------------------------------------------------------------- -cglobal ipred_smooth_h, 3, 7, 8, dst, stride, tl, w, h +cglobal ipred_smooth_h_8bpc, 3, 7, 8, dst, stride, tl, w, h %define base r6-ipred_smooth_h_ssse3_table LEA r6, ipred_smooth_h_ssse3_table mov wd, wm movd m3, [tlq+wq] pxor m1, m1 @@ -956,11 +956,11 @@ mova m0, [rsp+16*%10] ; recovery mova m4, [rsp+16*%11] ; recovery mova m5, [rsp+16*%12] ; recovery %endmacro -cglobal ipred_smooth, 3, 7, 8, -13*16, dst, stride, tl, w, h, v_weights +cglobal ipred_smooth_8bpc, 3, 7, 8, -13*16, dst, stride, tl, w, h, v_weights %define base r6-ipred_smooth_ssse3_table mov wd, wm mov hd, hm LEA r6, ipred_smooth_ssse3_table movd m4, [tlq+wq] ; right @@ -1192,11 +1192,11 @@ ;--------------------------------------------------------------------------------------- ;int dav1d_pal_pred_ssse3(pixel *dst, const ptrdiff_t stride, const uint16_t *const pal, ; const uint8_t *idx, const int w, const int h); ;--------------------------------------------------------------------------------------- -cglobal pal_pred, 4, 6, 5, dst, stride, pal, idx, w, h +cglobal pal_pred_8bpc, 4, 6, 5, dst, stride, pal, idx, w, h mova m4, [palq] LEA r2, pal_pred_ssse3_table tzcnt wd, wm movifnidn hd, hm movsxd wq, [r2+wq*4] @@ -1293,11 +1293,11 @@ DECLARE_REG_TMP 7 %else DECLARE_REG_TMP 5 %endif -cglobal ipred_cfl, 3, 7, 6, dst, stride, tl, w, h, ac, alpha +cglobal ipred_cfl_8bpc, 3, 7, 6, dst, stride, tl, w, h, ac, alpha movifnidn wd, wm movifnidn hd, hm tzcnt r6d, hd lea t0d, [wq+hq] movd m4, t0d @@ -1533,11 +1533,11 @@ ;--------------------------------------------------------------------------------------- ;void dav1d_ipred_cfl_left_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft, ; const int width, const int height, const int16_t *ac, const int alpha); ;--------------------------------------------------------------------------------------- -cglobal ipred_cfl_left, 3, 7, 6, dst, stride, tl, w, h, ac, alpha +cglobal ipred_cfl_left_8bpc, 3, 7, 6, dst, stride, tl, w, h, ac, alpha mov hd, hm ; zero upper half tzcnt r6d, hd sub tlq, hq tzcnt wd, wm movu m0, [tlq] @@ -1574,11 +1574,11 @@ ;--------------------------------------------------------------------------------------- ;void dav1d_ipred_cfl_top_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft, ; const int width, const int height, const int16_t *ac, const int alpha); ;--------------------------------------------------------------------------------------- -cglobal ipred_cfl_top, 3, 7, 6, dst, stride, tl, w, h, ac, alpha +cglobal ipred_cfl_top_8bpc, 3, 7, 6, dst, stride, tl, w, h, ac, alpha LEA t0, ipred_cfl_left_ssse3_table tzcnt wd, wm inc tlq movu m0, [tlq] movifnidn hd, hm @@ -1598,11 +1598,11 @@ ;--------------------------------------------------------------------------------------- ;void dav1d_ipred_cfl_128_ssse3(pixel *dst, const ptrdiff_t stride, const pixel *const topleft, ; const int width, const int height, const int16_t *ac, const int alpha); ;--------------------------------------------------------------------------------------- -cglobal ipred_cfl_128, 3, 7, 6, dst, stride, tl, w, h, ac, alpha +cglobal ipred_cfl_128_8bpc, 3, 7, 6, dst, stride, tl, w, h, ac, alpha tzcnt wd, wm movifnidn hd, hm LEA r6, ipred_cfl_splat_ssse3_table movsxd wq, [r6+wq*4] movddup m0, [r6-ipred_cfl_splat_ssse3_table+pw_128] @@ -1613,15 +1613,15 @@ %macro RELOAD_ACQ_32 1 mov acq, ac_bakq ; restore acq %endmacro %if ARCH_X86_64 -cglobal ipred_cfl_ac_420, 4, 8, 7, ac, y, stride, wpad, hpad, w, h, ac_bak +cglobal ipred_cfl_ac_420_8bpc, 4, 8, 7, ac, y, stride, wpad, hpad, w, h, ac_bak DECLARE_REG_TMP 7 movddup m2, [pb_2] %else -cglobal ipred_cfl_ac_420, 4, 7, 7, ac, y, stride, wpad, hpad, w, h +cglobal ipred_cfl_ac_420_8bpc, 4, 7, 7, ac, y, stride, wpad, hpad, w, h DECLARE_REG_TMP 4 %define ac_bakq acmp mov t0d, 0x02020202 movd m2, t0d pshufd m2, m2, q0000 @@ -1853,14 +1853,14 @@ sub szd, 8 jg .sub_loop RET %if ARCH_X86_64 -cglobal ipred_cfl_ac_422, 4, 8, 7, ac, y, stride, wpad, hpad, w, h, ac_bak +cglobal ipred_cfl_ac_422_8bpc, 4, 8, 7, ac, y, stride, wpad, hpad, w, h, ac_bak movddup m2, [pb_4] %else -cglobal ipred_cfl_ac_422, 4, 7, 7, ac, y, stride, wpad, hpad, w, h +cglobal ipred_cfl_ac_422_8bpc, 4, 7, 7, ac, y, stride, wpad, hpad, w, h mov t0d, 0x04040404 movd m2, t0d pshufd m2, m2, q0000 %endif movifnidn wd, wm @@ -2126,14 +2126,14 @@ sub szd, 8 jg .sub_loop RET %if ARCH_X86_64 -cglobal ipred_cfl_ac_444, 4, 8, 7, -4*16, ac, y, stride, wpad, hpad, w, h, ac_bak +cglobal ipred_cfl_ac_444_8bpc, 4, 8, 7, -4*16, ac, y, stride, wpad, hpad, w, h, ac_bak movddup m2, [pb_4] %else -cglobal ipred_cfl_ac_444, 4, 7, 7, -5*16, ac, y, stride, wpad, hpad, w, h +cglobal ipred_cfl_ac_444_8bpc, 4, 7, 7, -5*16, ac, y, stride, wpad, hpad, w, h %define ac_bakq [rsp+16*4] mov t0d, 0x04040404 movd m2, t0d pshufd m2, m2, q0000 %endif @@ -2767,11 +2767,11 @@ mova m2, m3 BLEND m0, m2, m%1 BLEND m1, m0, m5 %endmacro -cglobal ipred_paeth, 3, 6, 8, -7*16, dst, stride, tl, w, h +cglobal ipred_paeth_8bpc, 3, 6, 8, -7*16, dst, stride, tl, w, h %define base r5-ipred_paeth_ssse3_table tzcnt wd, wm movifnidn hd, hm pxor m0, m0 movd m5, [tlq] @@ -2935,11 +2935,11 @@ paddw m%1, m%3 psraw m%1, 4 packuswb m%1, m%1 %endmacro -cglobal ipred_filter, 3, 7, 8, dst, stride, tl, w, h, filter +cglobal ipred_filter_8bpc, 3, 7, 8, dst, stride, tl, w, h, filter %define base r6-$$ LEA r6, $$ tzcnt wd, wm %ifidn filterd, filterm movzx filterd, filterb diff --git a/third_party/dav1d/src/x86/itx16_avx2.asm b/third_party/dav1d/src/x86/itx16_avx2.asm --- a/third_party/dav1d/src/x86/itx16_avx2.asm +++ b/third_party/dav1d/src/x86/itx16_avx2.asm @@ -103,36 +103,36 @@ cextern pw_8192 cextern pw_16384 cextern pw_2896x8 cextern pd_2048 -cextern idct_4x8_internal_avx2.main -cextern idct_4x16_internal_avx2.main -cextern idct_8x8_internal_avx2.main -cextern idct_8x16_internal_avx2.main -cextern idct_16x4_internal_avx2.main -cextern idct_16x8_internal_avx2.main -cextern idct_16x16_internal_avx2.main -cextern inv_txfm_add_dct_dct_8x32_avx2.main -cextern inv_txfm_add_dct_dct_8x32_avx2.main_fast -cextern inv_txfm_add_dct_dct_16x32_avx2.main_oddhalf -cextern inv_txfm_add_dct_dct_16x32_avx2.main_oddhalf_fast -cextern inv_txfm_add_dct_dct_16x64_avx2.main_part1 -cextern inv_txfm_add_dct_dct_16x64_avx2.main_part2_internal +cextern idct_4x8_internal_8bpc_avx2.main +cextern idct_4x16_internal_8bpc_avx2.main +cextern idct_8x8_internal_8bpc_avx2.main +cextern idct_8x16_internal_8bpc_avx2.main +cextern idct_16x4_internal_8bpc_avx2.main +cextern idct_16x8_internal_8bpc_avx2.main +cextern idct_16x16_internal_8bpc_avx2.main +cextern inv_txfm_add_dct_dct_8x32_8bpc_avx2.main +cextern inv_txfm_add_dct_dct_8x32_8bpc_avx2.main_fast +cextern inv_txfm_add_dct_dct_16x32_8bpc_avx2.main_oddhalf +cextern inv_txfm_add_dct_dct_16x32_8bpc_avx2.main_oddhalf_fast +cextern inv_txfm_add_dct_dct_16x64_8bpc_avx2.main_part1 +cextern inv_txfm_add_dct_dct_16x64_8bpc_avx2.main_part2_internal -cextern iadst_4x4_internal_avx2.main -cextern iadst_4x8_internal_avx2.main_pass2 -cextern iadst_4x16_internal_avx2.main2 -cextern iadst_8x4_internal_avx2.main -cextern iadst_8x8_internal_avx2.main_pass2 -cextern iadst_8x16_internal_avx2.main -cextern iadst_8x16_internal_avx2.main_pass2_end -cextern iadst_16x4_internal_avx2.main -cextern iadst_16x8_internal_avx2.main -cextern iadst_16x8_internal_avx2.main_pass2_end -cextern iadst_16x16_internal_avx2.main -cextern iadst_16x16_internal_avx2.main_pass2_end +cextern iadst_4x4_internal_8bpc_avx2.main +cextern iadst_4x8_internal_8bpc_avx2.main_pass2 +cextern iadst_4x16_internal_8bpc_avx2.main2 +cextern iadst_8x4_internal_8bpc_avx2.main +cextern iadst_8x8_internal_8bpc_avx2.main_pass2 +cextern iadst_8x16_internal_8bpc_avx2.main +cextern iadst_8x16_internal_8bpc_avx2.main_pass2_end +cextern iadst_16x4_internal_8bpc_avx2.main +cextern iadst_16x8_internal_8bpc_avx2.main +cextern iadst_16x8_internal_8bpc_avx2.main_pass2_end +cextern iadst_16x16_internal_8bpc_avx2.main +cextern iadst_16x16_internal_8bpc_avx2.main_pass2_end SECTION .text %macro REPX 2-* %xdefine %%f(x) %1 @@ -382,11 +382,11 @@ pshufb m0, m4 jmp tx2q .pass2: lea rax, [deint_shuf+128] vextracti128 xm1, m0, 1 - call m(iadst_4x4_internal).main + call m(iadst_4x4_internal_8bpc).main .end: vpbroadcastd xm4, [pw_2048] movq xm2, [dstq+strideq*0] movhps xm2, [dstq+strideq*1] lea r6, [dstq+strideq*2] @@ -455,11 +455,11 @@ pshufb m0, m4 jmp tx2q .pass2: lea rax, [deint_shuf+128] vextracti128 xm1, m0, 1 - call m(iadst_4x4_internal).main + call m(iadst_4x4_internal_8bpc).main vpbroadcastd xm4, [pw_2048] movq xm3, [dstq+strideq*1] movhps xm3, [dstq+strideq*0] lea r6, [dstq+strideq*2] movq xm2, [r6 +strideq*1] @@ -605,11 +605,11 @@ punpcklwd m0, m1 punpckhdq m1, m0, m2 ; 2 3 punpckldq m0, m2 ; 0 1 vextracti128 xm2, m0, 1 ; 4 5 vextracti128 xm3, m1, 1 ; 6 7 - call m(idct_4x8_internal).main + call m(idct_4x8_internal_8bpc).main vpbroadcastd xm4, [pw_2048] REPX {pmulhrsw x, xm4}, xm0, xm1, xm2, xm3 lea r3, [strideq*3] lea r6, [dstq+strideq*4] movq xm4, [dstq+strideq*0] @@ -695,11 +695,11 @@ punpckldq m4, m0 vextracti128 xm2, m4, 1 ; 4 5 vextracti128 xm3, m5, 1 ; 6 7 pshufd xm4, xm4, q1032 ; 1 0 pshufd xm5, xm5, q1032 ; 3 2 - jmp m(iadst_4x8_internal).main_pass2 + jmp m(iadst_4x8_internal_8bpc).main_pass2 ALIGN function_align .main: vbroadcasti128 m0, [cq+16*0] vbroadcasti128 m2, [cq+16*2] vbroadcasti128 m3, [cq+16*5] @@ -932,11 +932,11 @@ punpckhdq m5, m2 ; a b vextracti128 xm2, m0, 1 ; 4 5 vextracti128 xm3, m1, 1 ; 6 7 vextracti128 xm6, m4, 1 ; c d vextracti128 xm7, m5, 1 ; e f - call m(idct_4x16_internal).main + call m(idct_4x16_internal_8bpc).main vpbroadcastd m9, [pw_2048] vinserti128 m0, m0, xm1, 1 ; 0 1 3 2 vinserti128 m1, m2, xm3, 1 ; 4 5 7 6 vinserti128 m2, m4, xm5, 1 ; 8 9 b a vinserti128 m3, m6, xm7, 1 ; c d f e @@ -1052,11 +1052,11 @@ shufpd m4, m5, 0x05 vperm2i128 m1, m0, m3, 0x31 ; 4 7 6 5 vinserti128 m0, xm3, 1 ; 0 3 2 1 vperm2i128 m3, m2, m4, 0x31 ; c f e d ; ???? vinserti128 m2, xm4, 1 ; b 8 9 a - call m(iadst_4x16_internal).main2 + call m(iadst_4x16_internal_8bpc).main2 vpbroadcastd m5, [pw_2896x8] paddsw m1, m2, m4 psubsw m2, m4 pmulhrsw m1, m5 ; -out7 out4 out6 -out5 pmulhrsw m2, m5 ; out8 -out11 -out9 out10 @@ -1432,11 +1432,11 @@ lea rax, [deint_shuf+128] vperm2i128 m1, m0, m2, 0x31 vinserti128 m0, xm2, 1 pshufb m0, m4 pshufb m1, m4 - jmp m(iadst_8x4_internal).main + jmp m(iadst_8x4_internal_8bpc).main ALIGN function_align .main: vpbroadcastd m1, [pd_2896] pmulld m0, m1, [cq+32*0] pmulld m3, m1, [cq+32*3] @@ -1634,11 +1634,11 @@ call .main call .round_shift1 jmp tx2q .pass2: call .transpose_8x8_packed - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main vpbroadcastd m12, [pw_2048] vpermq m0, m0, q3120 vpermq m1, m1, q2031 vpermq m2, m2, q3120 vpermq m3, m3, q2031 @@ -1752,11 +1752,11 @@ jmp tx2q .pass2: call m(idct_8x8_internal_16bpc).transpose_8x8_packed pshufd m4, m0, q1032 pshufd m5, m1, q1032 - call m(iadst_8x8_internal).main_pass2 + call m(iadst_8x8_internal_8bpc).main_pass2 vpbroadcastd m5, [pw_2048] vpbroadcastd xm12, [pw_4096] psubw m12, m5 REPX {vpermq x, x, q3120}, m0, m1, m2, m3 pmulhrsw m0, m12 @@ -1812,11 +1812,11 @@ jmp tx2q .pass2: call m(idct_8x8_internal_16bpc).transpose_8x8_packed pshufd m4, m0, q1032 pshufd m5, m1, q1032 - call m(iadst_8x8_internal).main_pass2 + call m(iadst_8x8_internal_8bpc).main_pass2 vpbroadcastd m12, [pw_2048] vpbroadcastd xm5, [pw_4096] psubw m12, m5 vpermq m8, m3, q2031 vpermq m9, m2, q2031 @@ -1969,11 +1969,11 @@ pxor m8, m8 REPX {mova x, m8}, m9, m10, m11, m12, m13, m14, m15 jmp tx2q .pass2: call .transpose - call m(idct_8x16_internal).main + call m(idct_8x16_internal_8bpc).main vpbroadcastd m12, [pw_2048] REPX {vpermq x, x, q3120}, m0, m2, m4, m6 REPX {vpermq x, x, q2031}, m1, m3, m5, m7 .end: pmulhrsw m0, m12 @@ -2165,12 +2165,12 @@ pxor m8, m8 REPX {mova x, m8}, m9, m10, m11, m12, m13, m14, m15 jmp tx2q .pass2: call m(idct_8x16_internal_16bpc).transpose - call m(iadst_8x16_internal).main - call m(iadst_8x16_internal).main_pass2_end + call m(iadst_8x16_internal_8bpc).main + call m(iadst_8x16_internal_8bpc).main_pass2_end vpbroadcastd m8, [pw_2048] vpbroadcastd xm12, [pw_4096] REPX {vpermq x, x, q2031}, m0, m1, m2, m3 REPX {vpermq x, x, q3120}, m4, m5, m6, m7 psubw m12, m8 @@ -2230,12 +2230,12 @@ pxor m8, m8 REPX {mova x, m8}, m9, m10, m11, m12, m13, m14, m15 jmp tx2q .pass2: call m(idct_8x16_internal_16bpc).transpose - call m(iadst_8x16_internal).main - call m(iadst_8x16_internal).main_pass2_end + call m(iadst_8x16_internal_8bpc).main + call m(iadst_8x16_internal_8bpc).main_pass2_end vpbroadcastd m12, [pw_2048] vpbroadcastd xm13, [pw_4096] mova m11, m0 vpermq m0, m7, q2031 mova m10, m1 @@ -2456,11 +2456,11 @@ REPX {psrad x, 1 }, m0, m1, m2, m3, m4, m5, m6, m7 jmp tx2q .pass2: call .transpose_4x16_packed lea rax, [deint_shuf+128] - call m(idct_16x4_internal).main + call m(idct_16x4_internal_8bpc).main .end: vpbroadcastd m4, [pw_2048] REPX {pmulhrsw x, m4}, m0, m1, m2, m3 .end2: paddw m0, [dstq+strideq*0] @@ -2515,11 +2515,11 @@ REPX {psrad x, 1}, m0, m1, m2, m3, m4, m5, m6, m7 jmp tx2q .pass2: call m(idct_16x4_internal_16bpc).transpose_4x16_packed lea rax, [deint_shuf+128] - call m(iadst_16x4_internal).main + call m(iadst_16x4_internal_8bpc).main jmp m(idct_16x4_internal_16bpc).end ALIGN function_align .main: vbroadcasti128 m6, [pd_1321] mova m0, [cq+32*0] @@ -2594,11 +2594,11 @@ paddd m0, m8, m11 jmp m(iadst_16x4_internal_16bpc).pass1_end .pass2: call m(idct_16x4_internal_16bpc).transpose_4x16_packed lea rax, [deint_shuf+128] - call m(iadst_16x4_internal).main + call m(iadst_16x4_internal_8bpc).main vpbroadcastd m4, [pw_2048] pmulhrsw m5, m3, m4 pmulhrsw m6, m2, m4 pmulhrsw m2, m1, m4 pmulhrsw m3, m0, m4 @@ -2710,11 +2710,11 @@ REPX {psrad x, 1}, m0, m1, m2, m3, m4, m5, m6, m7, \ m8, m9, m10, m11, m12, m13, m14, m15 jmp tx2q .pass2: call .transpose - call m(idct_16x8_internal).main + call m(idct_16x8_internal_8bpc).main vpbroadcastd m10, [pw_2048] .end: pmulhrsw m0, m10 pmulhrsw m1, m10 pmulhrsw m2, m10 @@ -2825,12 +2825,12 @@ REPX {psrad x, 1 }, m0, m1, m2, m3, m12, m13, m14, m15 REPX {psrad x, 13}, m4, m5, m6, m7, m8, m9, m10, m11 jmp tx2q .pass2: call m(idct_16x8_internal_16bpc).transpose - call m(iadst_16x8_internal).main - call m(iadst_16x8_internal).main_pass2_end + call m(iadst_16x8_internal_8bpc).main + call m(iadst_16x8_internal_8bpc).main_pass2_end vpbroadcastd m10, [pw_2048] pxor m11, m11 psubw m11, m10 pmulhrsw m0, m10 pmulhrsw m1, m11 @@ -3037,12 +3037,12 @@ psubd m0, m1, [r6-32*4] paddd m1, [r6-32*3] jmp m(iadst_16x8_internal_16bpc).pass1_end .pass2: call m(idct_16x8_internal_16bpc).transpose - call m(iadst_16x8_internal).main - call m(iadst_16x8_internal).main_pass2_end + call m(iadst_16x8_internal_8bpc).main + call m(iadst_16x8_internal_8bpc).main_pass2_end vpbroadcastd m10, [pw_2048] pxor m11, m11 psubw m11, m10 mova m12, m0 pmulhrsw m0, m7, m11 @@ -3214,11 +3214,11 @@ jmp tx2q .pass2: call .transpose lea rax, [pw_5+128] mova [rsp], m15 - call m(idct_16x16_internal).main + call m(idct_16x16_internal_8bpc).main mova m1, [rsp+32*1] .end: call .write_16x16 RET ALIGN function_align @@ -3448,12 +3448,12 @@ jmp tx2q .pass2: call m(idct_16x16_internal_16bpc).transpose lea rax, [pw_5+128] mova [rsp], m15 - call m(iadst_16x16_internal).main - call m(iadst_16x16_internal).main_pass2_end + call m(iadst_16x16_internal_8bpc).main + call m(iadst_16x16_internal_8bpc).main_pass2_end mova [rsp+32*0], m8 mova [rsp+32*2], m12 mova [rsp+32*3], m13 vpbroadcastd m12, [pw_2048] pxor m13, m13 @@ -3580,12 +3580,12 @@ jmp m(iadst_16x16_internal_16bpc).pass1_end .pass2: call m(idct_16x16_internal_16bpc).transpose lea rax, [pw_5+128] mova [rsp], m15 - call m(iadst_16x16_internal).main - call m(iadst_16x16_internal).main_pass2_end + call m(iadst_16x16_internal_8bpc).main + call m(iadst_16x16_internal_8bpc).main_pass2_end mova [rsp+32*3], m3 mova [rsp+32*2], m2 mova [rsp+32*0], m0 mova m2, m13 mova m3, m12 @@ -3738,11 +3738,11 @@ pxor m4, m4 .pass1_end_fast: vpbroadcastd m10, [pw_2048] lea rax, [deint_shuf+128] REPX {mova x, m4}, m5, m6, m7 - call m(inv_txfm_add_dct_dct_8x32).main_fast + call m(inv_txfm_add_dct_dct_8x32_8bpc).main_fast jmp .end .eob107: mova [rsp+32*3], m3 mova [r6-32*2], m0 mova [r6-32*1], m1 @@ -3776,11 +3776,11 @@ mova m2, [r6-32*2] ; out8 out10 mova m3, [r6-32*1] ; out12 out14 lea rax, [deint_shuf+128] mova m11, [rsp+32*3] ; out13 out15 vpbroadcastd m10, [pw_2048] - call m(inv_txfm_add_dct_dct_8x32).main + call m(inv_txfm_add_dct_dct_8x32_8bpc).main .end: ; [rsp+0*32] = m12 vpbroadcastd m12, [pw_2048] mov cq, r4 mova [rsp+32*1], m8 mova [rsp+32*2], m9 @@ -4292,11 +4292,11 @@ lea dstq, [r4+32] call .pass2 RET ALIGN function_align .pass2: - call m(idct_16x8_internal).main + call m(idct_16x8_internal_8bpc).main REPX {pmulhrsw x, m11}, m0, m1, m2, m3 call m(idct_16x8_internal_16bpc).write_16x4_start pmulhrsw m0, m11, m4 pmulhrsw m1, m11, m5 pmulhrsw m2, m11, m6 @@ -4402,11 +4402,11 @@ mova m1, [r4+32*1] mova m2, [r4+32*2] mova m3, [r4+32*3] .fast: lea rax, [pw_5+128] - call m(inv_txfm_add_dct_dct_16x32).main_oddhalf_fast + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast pxor m8, m8 REPX {mova x, m8}, m9, m10, m11, m12, m13, m14, m15 jmp .idct16 .eob151: mova [r4-16*8], xm0 @@ -4454,11 +4454,11 @@ mova m4, [r4-32*4] mova m5, [r4-32*3] mova m6, [r4-32*2] mova m7, [r4-32*1] lea rax, [pw_5 + 128] - call m(inv_txfm_add_dct_dct_16x32).main_oddhalf + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf lea r3, [rsp+32*8] mova m8, [r3+32*0] mova m9, [r3+32*1] mova m10, [r3+32*2] mova m11, [r3+32*3] @@ -4475,11 +4475,11 @@ mova m4, [r3-32*4] mova m5, [r3-32*3] mova m6, [r3-32*2] mova m7, [r3-32*1] mova [rsp], m15 - call m(idct_16x16_internal).main + call m(idct_16x16_internal_8bpc).main imul r2, strideq, 19 lea r3, [strideq*3] add r2, dstq call .pass2_end RET @@ -4709,11 +4709,11 @@ call m(inv_txfm_add_dct_dct_8x32_16bpc).transpose pxor m8, m8 REPX {mova x, m8}, m9, m10, m11, m12, m13, m14, [rsp] lea rax, [pw_5+128] mov r7, dstq - call m(idct_16x16_internal).main + call m(idct_16x16_internal_8bpc).main call .write_16x16 mova m0, [r5+32*3] mova m1, [r5+32*2] mova m2, [r5+32*1] mova m3, [r5+32*0] @@ -4748,11 +4748,11 @@ call .main sub r4, 32*16 ; topleft 16x8 call .transpose_16x16 lea rax, [pw_5+128] mov r7, dstq - call m(idct_16x16_internal).main + call m(idct_16x16_internal_8bpc).main call .write_16x16 mova m0, [r5+32*3] mova m1, [r5+32*2] mova m2, [r5+32*1] mova m3, [r5+32*0] @@ -4762,11 +4762,11 @@ mova m7, [r5-32*4] add r4, 32*8 ; bottomleft 16x8 call .transpose_16x16 .end: lea dstq, [r7+32] - call m(idct_16x16_internal).main + call m(idct_16x16_internal_8bpc).main call .write_16x16 RET ALIGN function_align .transpose_16x16: punpckhdq m8, m3, m1 @@ -5122,11 +5122,11 @@ mova m11, [r3+32*39] ; 23 mova m12, [r3+32*49] ; 25 mova m13, [r3+32*51] ; 27 mova m14, [r3+32*53] ; 29 mova m15, [r3+32*55] ; 31 - jmp m(inv_txfm_add_dct_dct_16x32).main_oddhalf + jmp m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf ALIGN function_align .pass2_evenhalf: mova m0, [r3+32* 0] ; 0 mova m1, [r3+32* 2] ; 2 mova m2, [r3+32* 4] ; 4 @@ -5142,11 +5142,11 @@ mova m12, [r3+32*48] ; 24 mova m13, [r3+32*50] ; 26 mova m14, [r3+32*52] ; 28 mova m15, [r3+32*54] ; 30 mova [rsp+gprsize], m15 - jmp m(idct_16x16_internal).main + jmp m(idct_16x16_internal_8bpc).main cglobal inv_txfm_add_identity_identity_32x32_16bpc, 4, 8, 8, dst, stride, c, eob %undef cmp vpbroadcastd m5, [pw_8192] vpbroadcastd m7, [pixel_max] @@ -5298,11 +5298,11 @@ mova m6, [rsp+32*26] ; in24 mova m7, [rsp+32*30] ; in28 pxor m8, m8 REPX {mova x, m8}, m9, m10, m11, m12, m13, m14 mova [rsp], m8 - call m(idct_16x16_internal).main + call m(idct_16x16_internal_8bpc).main mova m1, [rsp+32*1] lea r4, [rsp+32*38] mova [r4-32*4], m0 mova [r4-32*3], m1 mova [r4-32*2], m2 @@ -5328,11 +5328,11 @@ mova m5, [rsp+32*24] ; in22 mova m6, [rsp+32*28] ; in26 mova m7, [rsp+32*32] ; in30 lea r5, [r4+32*16] add r4, 32*8 - call m(inv_txfm_add_dct_dct_16x32).main_oddhalf_fast + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast mova m0, [rsp+32* 3] ; in1 mova m1, [rsp+32*33] ; in31 mova m2, [rsp+32*19] ; in17 mova m3, [rsp+32*17] ; in15 mova m4, [rsp+32*11] ; in9 @@ -5340,11 +5340,11 @@ mova m6, [rsp+32*27] ; in25 mova m7, [rsp+32* 9] ; in7 lea rax, [idct64_mul - 8] add r4, 32*16 add r5, 32*32 - call m(inv_txfm_add_dct_dct_16x64).main_part1 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 mova m0, [rsp+32* 7] ; in5 mova m1, [rsp+32*29] ; in27 mova m2, [rsp+32*23] ; in21 mova m3, [rsp+32*13] ; in11 mova m4, [rsp+32*15] ; in13 @@ -5352,11 +5352,11 @@ mova m6, [rsp+32*31] ; in29 mova m7, [rsp+32* 5] ; in3 add rax, 8 add r4, 32*8 sub r5, 32*8 - call m(inv_txfm_add_dct_dct_16x64).main_part1 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 lea r8, [strideq*4] lea r9, [strideq*5] lea r3, [r9+strideq*1] ; stride*6 lea r7, [r9+strideq*2] ; stride*7 call .main_part2_pass2 @@ -5447,11 +5447,11 @@ vpbroadcastd m13, [pw_2896_2896] lea rax, [pw_5+128] lea r2, [dstq+r7] .main_part2_pass2_loop: vpbroadcastd m14, [pw_m2896_2896] - call m(inv_txfm_add_dct_dct_16x64).main_part2_internal + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part2_internal vpbroadcastd m14, [pw_2048] IDCT64_PART2_END 0, 7, 0, 6, 9, 10, strideq*0, r3*4, r8*8, r7*8 IDCT64_PART2_END 7, 8, 5, 0, 6, 7, strideq*0, r3*4, r8*8, r7*8 IDCT64_PART2_END 8, 2, 1, 0, 6, 7, strideq*8, r8*4, r9*8, r3*8 IDCT64_PART2_END 15, 3, 4, 0, 6, 7, strideq*8, r8*4, r9*8, r3*8 @@ -5646,11 +5646,11 @@ mova m6, [r10+32*50] ; in24 mova m7, [r10+32*54] ; in28 pxor m8, m8 REPX {mova x, m8}, m9, m10, m11, m12, m13, m14 mova [rsp], m8 - call m(idct_16x16_internal).main + call m(idct_16x16_internal_8bpc).main mova m1, [rsp+32*1] lea r4, [rsp+32*70] mova [r4-32*4], m0 mova [r4-32*3], m1 mova [r4-32*2], m2 @@ -5676,11 +5676,11 @@ mova m5, [r10+32*40] ; in22 mova m6, [r10+32*52] ; in26 mova m7, [r10+32*56] ; in30 lea r5, [r4+32*16] add r4, 32*8 - call m(inv_txfm_add_dct_dct_16x32).main_oddhalf_fast + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast mova m0, [r10+32* 3] ; in1 mova m1, [r10+32*57] ; in31 mova m2, [r10+32*35] ; in17 mova m3, [r10+32*25] ; in15 mova m4, [r10+32*19] ; in9 @@ -5688,11 +5688,11 @@ mova m6, [r10+32*51] ; in25 mova m7, [r10+32* 9] ; in7 lea rax, [idct64_mul - 8] add r4, 32*16 add r5, 32*32 - call m(inv_txfm_add_dct_dct_16x64).main_part1 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 mova m0, [r10+32* 7] ; in5 mova m1, [r10+32*53] ; in27 mova m2, [r10+32*39] ; in21 mova m3, [r10+32*21] ; in11 mova m4, [r10+32*23] ; in13 @@ -5700,11 +5700,11 @@ mova m6, [r10+32*55] ; in29 mova m7, [r10+32* 5] ; in3 add rax, 8 add r4, 32*8 sub r5, 32*8 - call m(inv_txfm_add_dct_dct_16x64).main_part1 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 call m(inv_txfm_add_dct_dct_16x64_16bpc).main_part2_pass2 add r10, 32*8 sub r4, 32*98 ; rsp+32*16 sub dstq, r8 add dstq, 32 @@ -5875,11 +5875,11 @@ mova m13, [r7+32*1] mova m14, [r7+32*2] mova m15, [r7+32*3] sub r7, 32*24 mova [rsp], m15 - call m(idct_16x16_internal).main + call m(idct_16x16_internal_8bpc).main mova m1, [rsp+32*1] call m(inv_txfm_add_dct_dct_32x16_16bpc).write_16x16 add r5, 32 mov dstq, r5 cmp r7, r4 @@ -6107,11 +6107,11 @@ mova m11, [r7-32*29] mova m12, [r7-32* 3] mova m13, [r7-32* 1] mova m14, [r7+32* 1] mova m15, [r7+32* 3] - call m(inv_txfm_add_dct_dct_16x32).main_oddhalf + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf mova m0, [r7-32*100] mova m1, [r7-32*98] mova m2, [r7-32*96] mova m3, [r7-32*94] mova m4, [r7-32*68] @@ -6126,11 +6126,11 @@ mova m13, [r7-32* 2] mova m14, [r7+32* 0] mova m15, [r7+32* 2] add r7, 32*8 mova [rsp], m15 - call m(idct_16x16_internal).main + call m(idct_16x16_internal_8bpc).main call m(inv_txfm_add_dct_dct_16x32_16bpc).pass2_end sub dstq, r3 lea r2, [r2+r3+32] add dstq, 32 cmp r7, r4 @@ -6246,11 +6246,11 @@ mova m6, [r10-32* 4] ; in24 mova m7, [r10+32* 0] ; in28 pxor m8, m8 REPX {mova x, m8}, m9, m10, m11, m12, m13, m14 mova [rsp], m8 - call m(idct_16x16_internal).main + call m(idct_16x16_internal_8bpc).main mova m1, [rsp+32*1] mova [r4-32*4], m0 mova [r4-32*3], m1 mova [r4-32*2], m2 mova [r4-32*1], m3 @@ -6275,11 +6275,11 @@ mova m5, [r10-32*30] ; in22 mova m6, [r10-32* 2] ; in26 mova m7, [r10+32* 2] ; in30 lea r5, [r4+32*16] add r4, 32*8 - call m(inv_txfm_add_dct_dct_16x32).main_oddhalf_fast + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast mova m0, [r10-32*99] ; in1 mova m1, [r10+32* 3] ; in31 mova m2, [r10-32*35] ; in17 mova m3, [r10-32*61] ; in15 mova m4, [r10-32*67] ; in9 @@ -6287,11 +6287,11 @@ mova m6, [r10-32* 3] ; in25 mova m7, [r10-32*93] ; in7 lea rax, [idct64_mul - 8] add r4, 32*16 add r5, 32*32 - call m(inv_txfm_add_dct_dct_16x64).main_part1 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 mova m0, [r10-32*95] ; in5 mova m1, [r10-32* 1] ; in27 mova m2, [r10-32*31] ; in21 mova m3, [r10-32*65] ; in11 mova m4, [r10-32*63] ; in13 @@ -6299,11 +6299,11 @@ mova m6, [r10+32* 1] ; in29 mova m7, [r10-32*97] ; in3 add rax, 8 add r4, 32*8 sub r5, 32*8 - call m(inv_txfm_add_dct_dct_16x64).main_part1 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 call m(inv_txfm_add_dct_dct_16x64_16bpc).main_part2_pass2 add r10, 32*8 sub dstq, r8 sub r4, 32*44 add dstq, 32 diff --git a/third_party/dav1d/src/x86/itx16_sse.asm b/third_party/dav1d/src/x86/itx16_sse.asm --- a/third_party/dav1d/src/x86/itx16_sse.asm +++ b/third_party/dav1d/src/x86/itx16_sse.asm @@ -1,9 +1,10 @@ ; Copyright © 2021, VideoLAN and dav1d authors ; Copyright © 2021, Two Orioles, LLC ; Copyright © 2017-2021, The rav1e contributors ; Copyright © 2020, Nathan Egge +; Copyright © 2021, Matthias Dressel ; All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions are met: ; @@ -26,47 +27,206 @@ ; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. %include "config.asm" %include "ext/x86/x86inc.asm" +SECTION_RODATA +%macro COEF 1-2 +pd_%1: times 4 dd %1 +%if %0 == 2 +pd_m%1: times 4 dd -%1 +%endif +%endmacro + +COEF 201 +COEF 401 +COEF 601, 1 +COEF 799 +COEF 995 +COEF 1189, 1 +COEF 1380, 1 +COEF 1567 +COEF 1751 +COEF 1931 +COEF 2106, 1 +COEF 2276, 1 +COEF 2440 +COEF 2598, 1 +COEF 2751, 1 +COEF 2896 +COEF 3035 +COEF 3166 +COEF 3290 +COEF 3406 +COEF 3513 +COEF 3612 +COEF 3703 +COEF 3784 +COEF 3857 +COEF 3920 +COEF 3973 +COEF 4017 +COEF 4052 +COEF 4076 +COEF 4091 + +deint_shuf: db 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15 + +%if ARCH_X86_32 +pd_1: times 4 dd 1 +%endif +pd_2: times 4 dd 2 +pw_5: times 8 dw 5 +pd_1321: times 4 dd 1321 +pd_2482: times 4 dd 2482 +pd_m3344: times 4 dd -3344 +pd_2048: times 4 dd 2048 +pw_4x2048_4xm2048: times 4 dw 2048 + times 4 dw -2048 +pw_4xm2048_4x2048: times 4 dw -2048 + times 4 dw 2048 +pw_2048: times 8 dw 2048 +pw_m2048: times 8 dw -2048 +pd_3803: times 4 dd 3803 +pw_4096: times 8 dw 4096 +pd_5793: times 4 dd 5793 +pd_6144: times 4 dd 6144 +pw_8192: times 8 dw 8192 +pd_10240: times 4 dd 10240 +pd_11586: times 4 dd 11586 +pw_1697x8: times 8 dw 1697*8 +pw_2896x8: times 8 dw 2896*8 +pw_1697x16: times 8 dw 1697*16 +pw_16384: times 8 dw 16384 +pixel_10bpc_max: times 8 dw 0x03ff + +pw_1567_3784: times 4 dw 1567, 3784 +pw_m3784_1567: times 4 dw -3784, 1567 + +clip_min: times 4 dd -0x20000 +clip_max: times 4 dd 0x1ffff + +idct64_mul_16bpc: +dd 4095, 101, 2967, -2824, 3745, 1660, 3822, -1474, 401, 4076, 799, 4017 +dd -700, 4036, 2359, 3349, -2191, 3461, 897, 3996, -2598, -3166, -4017, -799 +dd 4065, 501, 3229, -2520, 3564, 2019, 3948, -1092, 1931, 3612, 3406, 2276 +dd -301, 4085, 2675, 3102, -1842, 3659, 1285, 3889, -1189, -3920, -2276, -3406 + +cextern inv_txfm_add_dct_dct_4x4_8bpc_ssse3 +cextern iadst_4x4_internal_8bpc_ssse3.main +cextern idct_4x8_internal_8bpc_ssse3.main +cextern iadst_4x8_internal_8bpc_ssse3.main +cextern idct_16x4_internal_8bpc_ssse3.main +cextern iadst_16x4_internal_8bpc_ssse3.main +cextern iadst_16x4_internal_8bpc_ssse3.main_pass2_end +cextern idct_8x4_internal_8bpc_ssse3.main +cextern iadst_8x4_internal_8bpc_ssse3.main +cextern idct_8x8_internal_8bpc_ssse3.main +cextern idct_8x8_internal_8bpc_ssse3.pass1_end3 +cextern iadst_8x8_internal_8bpc_ssse3.main +cextern iadst_8x8_internal_8bpc_ssse3.main_pass2_end +cextern idct_16x8_internal_8bpc_ssse3.main +cextern iadst_16x8_internal_8bpc_ssse3.main +cextern iadst_16x8_internal_8bpc_ssse3.main_pass2_end +cextern idct_8x32_internal_8bpc_ssse3.main +cextern idct_8x32_internal_8bpc_ssse3.main_fast +cextern idct_8x32_internal_8bpc_ssse3.main_veryfast +cextern idct_16x64_internal_8bpc_ssse3.main +cextern idct_16x64_internal_8bpc_ssse3.main_fast + +tbl_4x16_2d: db 0, 13, 29, 45 +tbl_4x16_h: db 0, 16, 32, 48 +tbl_4x16_v: db 0, 4, 8, 12 + +tbl_8x16_2d: db 0, 14, 30, 46 +tbl_8x16_v: db 0, 4, 8, 12 +tbl_8x16_h: db 0, 32, 64, 96 + +tbl_16x16_2d: db 0, 10, 36, 78 +tbl_16x16_v: db 0, 4, 8, 12 +tbl_16x16_h: db 0, 64, 128, 192 + +tbl_8x32_2d: dw 0, 14, 43, 75, 107, 139, 171, 203 + +tbl_16x32_2d: dw 0, 14, 44, 90, 151, 215, 279, 343 + +tbl_32x16_2d: ; first 4 entries of 32x32 are identical to this one +tbl_32x32_2d: dw 0, 10, 36, 78, 136, 210, 300, 406 + +tbl_Nx32_odd_offset: db 2*16, 2*23 + db 2*20, 2*19 + db 2*18, 2*21 + db 2*22, 2*17 + db 2*30, 2*25 + db 2*26, 2*29 + db 2*28, 2*27 + db 2*24, 2*31 + +tbl_Nx64_offset: db 2* 0, 2*32, 2*16, 2*46 + db 2* 8, 2*40, 2*23, 2*38 + db 2* 1, 2*36, 2*20, 2*42 + db 2* 9, 2*44, 2*19, 2*34 + db 2* 2, 2*60, 2*18, 2*50 + db 2*10, 2*52, 2*21, 2*58 + db 2* 3, 2*56, 2*22, 2*54 + db 2*11, 2*48, 2*17, 2*62 + SECTION .text -%macro IWHT4_1D_PACKED 0 +%macro REPX 2-* + %xdefine %%f(x) %1 +%rep %0 - 1 + %rotate 1 + %%f(%1) +%endrep +%endmacro + +%define m_suffix(x, sfx) mangle(private_prefix %+ _ %+ x %+ sfx) +%define m(x) m_suffix(x, SUFFIX) + +; This refers to the first function in itx_sse i.e. the start of the text section +; which is needed as a base pointer for constants. +%define itx8_start m_suffix(inv_txfm_add_dct_dct_4x4_8bpc, _ssse3) + +%if ARCH_X86_64 +%define o(x) x +%else +%define o(x) r6-$$+x ; PIC +%endif + +%macro IWHT4_1D 0 ; m0 = in0, m1 = in1, m2 = in2, m3 = in3 - paddd m0, m1 ; in0 += in1 - psubd m4, m2, m3 ; tmp0 = in2 - in3 - psubd m5, m0, m4 ; tmp1 = (in0 - tmp0) >> 1 - psrad m5, 1 - psubd m2, m5, m1 ; in2 = tmp1 - in1 - psubd m5, m3 ; in1 = tmp1 - in3 - psubd m0, m5 ; in0 -= in1 - paddd m4, m2 ; in3 = tmp0 + in2 + paddd m0, m1 ; in0 += in1 + psubd m4, m2, m3 ; tmp0 = in2 - in3 + psubd m5, m0, m4 ; tmp1 = (in0 - tmp0) >> 1 + psrad m5, 1 + psubd m2, m5, m1 ; in2 = tmp1 - in1 + psubd m5, m3 ; in1 = tmp1 - in3 + psubd m0, m5 ; in0 -= in1 + paddd m4, m2 ; in3 = tmp0 + in2 ; m0 = out0, m1 = in1, m2 = out2, m3 = in3 ; m4 = out3, m5 = out1 %endmacro INIT_XMM sse2 cglobal inv_txfm_add_wht_wht_4x4_16bpc, 3, 3, 6, dst, stride, c, eob, bdmax mova m0, [cq+16*0] mova m1, [cq+16*1] mova m2, [cq+16*2] mova m3, [cq+16*3] - psrad m0, 2 - psrad m1, 2 - psrad m2, 2 - psrad m3, 2 - IWHT4_1D_PACKED + REPX {psrad x, 2}, m0, m1, m2, m3 + IWHT4_1D punpckldq m1, m0, m5 punpckhdq m3, m0, m5 punpckldq m5, m2, m4 punpckhdq m2, m4 punpcklqdq m0, m1, m5 punpckhqdq m1, m5 punpcklqdq m4, m3, m2 punpckhqdq m3, m2 mova m2, m4 - IWHT4_1D_PACKED + IWHT4_1D packssdw m0, m4 ; low: out3, high: out0 packssdw m2, m5 ; low: out2, high: out1 pxor m4, m4 mova [cq+16*0], m4 mova [cq+16*1], m4 @@ -89,5 +249,7881 @@ movhps [r2 +strideq*1], m0 ; write out0 movhps [dstq+strideq*1], m2 ; write out1 movq [r2 +strideq*0], m2 ; write out2 movq [dstq+strideq*0], m0 ; write out3 RET + +; dst1 = (src1 * coef1 - src2 * coef2 + rnd) >> 12 +; dst2 = (src1 * coef2 + src2 * coef1 + rnd) >> 12 +; flags: 2 = inv_dst1, 4 = inv_dst2 +; skip round/shift if rnd is not a number +%macro ITX_MULSUB_2D 8-9 0 ; dst/src[1-2], tmp[1-3], rnd, coef[1-2], flags +; %1 dst/src[1] +; %2 dst/src[2] +; %3 tmp[1] +; %4 tmp[2] +; %5 tmp[3] +; %6 rnd +; %7 coef[1] +; %8 coef[2] +; %9 flags +%ifnidn %7,%8 ; optimize when coef1 == coef2 +%if %8 < 32 + pmulld m%4, m%1, m%8 + pmulld m%3, m%2, m%8 +%else + mova m%3, [o(pd_%8)] + pmulld m%4, m%1, m%3 + pmulld m%3, m%2 +%endif +%endif +%if %7 < 32 + pmulld m%1, m%7 + pmulld m%2, m%7 +%else + mova m%5, [o(pd_%7)] + pmulld m%1, m%5 + pmulld m%2, m%5 +%endif +%if %9 & 4 ; invert dst2 + paddd m%4, m%2 + psubd m%2, m%6, m%4 +%else +%ifnum %6 +%ifnidn %7,%8 + paddd m%4, m%6 +%else + paddd m%1, m%6 +%endif +%endif +%ifnidn %7,%8 + paddd m%2, m%4 +%else + mova m%3, m%2 + paddd m%2, m%1 +%endif +%endif +%if %9 & 2 ; invert dst1 + psubd m%3, m%1 + paddd m%1, m%3, m%6 +%else +%ifnum %6 +%ifnidn %7,%8 + paddd m%1, m%6 +%endif +%endif + psubd m%1, m%3 +%endif +%ifnum %6 + psrad m%2, 12 + psrad m%1, 12 +%endif +%endmacro + +%macro INV_TXFM_FN 4-5+ 8 ; type1, type2, eob_offset, size, mmsize/stack +cglobal inv_txfm_add_%1_%2_%4_16bpc, 4, 7, %5, dst, stride, c, eob, tx2 + %define %%p1 m(i%1_%4_internal_16bpc) +%if ARCH_X86_32 + LEA r6, $$ +%endif +%if has_epilogue +%ifidn %1_%2, dct_dct + test eobd, eobd + jz %%end +%endif + lea tx2q, [o(m(i%2_%4_internal_16bpc).pass2)] +%ifnum %3 +%if %3 + add eobd, %3 +%endif +%else + lea r5, [o(%3)] +%endif + call %%p1 + RET +%%end: +%else + ; Jump to the 1st txfm function if we're not taking the fast path, which + ; in turn performs an indirect jump to the 2nd txfm function. + lea tx2q, [o(m(i%2_%4_internal_16bpc).pass2)] +%ifnum %3 +%if %3 + add eobd, %3 +%endif +%else + lea r5, [o(%3)] +%endif +%ifidn %1_%2, dct_dct + test eobd, eobd + jnz %%p1 +%else + ; jump to the 1st txfm function unless it's located directly after this + times ((%%end - %%p1) >> 31) & 1 jmp %%p1 +ALIGN function_align +%%end: +%endif +%endif +%endmacro + +%macro INV_TXFM_4X4_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 0, 4x4 +%ifidn %1_%2, dct_dct + imul r5d, [cq], 2896 + movd m1, [o(pw_2896x8)] + mov [cq], eobd ; 0 + add r5d, 2048 + sar r5d, 12 + movd m0, r5d + packssdw m0, m0 + pmulhrsw m0, m1 + pshuflw m0, m0, q0000 + punpcklqdq m0, m0 + mova m1, m0 + TAIL_CALL m(iadst_4x4_internal_16bpc).end +%endif +%endmacro + +%macro IDCT4_1D 8 ; src[1-4], tmp[1-3], rnd + ; butterfly rotation + ITX_MULSUB_2D %1, %3, %5, %6, %7, %8, 2896, 2896 ; %1 out1 %3 out0 + ITX_MULSUB_2D %2, %4, %5, %6, %7, %8, 1567, 3784 ; %2 out2 %4 out3 + ; Hadamard rotation + psubd m%5, m%1, m%2 + paddd m%2, m%1 + paddd m%1, m%3, m%4 + psubd m%3, m%4 + ; %1 (src1) = out0 + ; %2 (src2) = out1 + ; %3 (src3) = out3 + ; $5 (tmp1) = out2 +%endmacro + +INIT_XMM sse4 + +INV_TXFM_4X4_FN dct, dct +INV_TXFM_4X4_FN dct, identity +INV_TXFM_4X4_FN dct, adst +INV_TXFM_4X4_FN dct, flipadst + +cglobal idct_4x4_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 + mova m0, [cq+16*0] + mova m1, [cq+16*1] + mova m2, [cq+16*2] + mova m3, [cq+16*3] + mova m5, [o(pd_2048)] + call .pass1_main + packssdw m0, m1 ; out0 out1 + packssdw m4, m2 ; out2 out3 + ; transpose + punpckhwd m2, m0, m4 + punpcklwd m0, m4 + punpckhwd m1, m0, m2 + punpcklwd m0, m2 + ; m0 = out0 out1 + ; m1 = out2 out3 + ; m5 = pd_2048 + jmp tx2q +.pass1_main: + IDCT4_1D 0, 1, 2, 3, 4, 6, 7, 5 + ret +.pass2: + ; m0 = in0 in1 + ; m1 = in2 in3 + ; m5 = pd_2048 + mova m4, [o(pw_m3784_1567)] + punpckhwd m2, m1, m0 + psubw m3, m0, m1 + paddw m0, m1 + punpcklqdq m0, m3 + pmaddwd m4, m2 + pmaddwd m2, [o(pw_1567_3784)] + pmulhrsw m0, [o(pw_2896x8)] ; t0 t1 + paddd m4, m5 + paddd m2, m5 + psrad m4, 12 + psrad m2, 12 + packssdw m2, m4 ; t3 t2 + psubsw m1, m0, m2 ; tmp3 tmp2 + paddsw m0, m2 ; tmp0 tmp1 + packssdw m5, m5 ; pw_2048 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + movq m2, [dstq+strideq*0] + movhps m2, [dstq+strideq*1] + lea r5, [dstq+strideq*2] + movq m3, [r5 +strideq*1] + movhps m3, [r5 +strideq*0] + mova m5, [o(pixel_10bpc_max)] + pxor m4, m4 + mova [cq+16*0], m4 + mova [cq+16*1], m4 + mova [cq+16*2], m4 + mova [cq+16*3], m4 + paddw m0, m2 + paddw m1, m3 + pmaxsw m0, m4 + pmaxsw m1, m4 + pminsw m0, m5 + pminsw m1, m5 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + movhps [r5 +strideq*0], m1 + movq [r5 +strideq*1], m1 + RET + +INV_TXFM_4X4_FN adst, dct +INV_TXFM_4X4_FN adst, adst +INV_TXFM_4X4_FN adst, flipadst +INV_TXFM_4X4_FN adst, identity + +cglobal iadst_4x4_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 + call .main + packssdw m0, m2 ; out0 out1 + packssdw m1, m4 ; out2 out3 + ; transpose + punpckhwd m2, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m0, m2 + punpcklwd m0, m2 + ; m0 = out0 out1 + ; m1 = out2 out3 + ; m5 = pd_2048 + jmp tx2q +.pass2: + ; m0 = in0 in1 + ; m1 = in2 in3 +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call m_suffix(iadst_4x4_internal_8bpc, _ssse3).main +.end: + mova m4, [o(pw_2048)] + movq m2, [dstq+strideq*0] + movhps m2, [dstq+strideq*1] + lea r5, [dstq+strideq*2] + movq m3, [r5 +strideq*0] + movhps m3, [r5 +strideq*1] + mova m5, [o(pixel_10bpc_max)] + pmulhrsw m0, m4 + pmulhrsw m1, m4 + pxor m4, m4 + mova [cq+16*0], m4 + mova [cq+16*1], m4 + mova [cq+16*2], m4 + mova [cq+16*3], m4 + paddw m0, m2 + paddw m1, m3 + pmaxsw m0, m4 + pmaxsw m1, m4 + pminsw m0, m5 + pminsw m1, m5 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + movq [r5 +strideq*0], m1 + movhps [r5 +strideq*1], m1 + RET +ALIGN function_align +.main: + mova m1, [cq+16*2] + mova m3, [cq+16*3] + mova m5, [cq+16*0] + lea r3, [cq+16*1] +.main2: + mova m0, [o(pd_1321)] ; SINPI_1_9 + mova m2, [o(pd_2482)] ; SINPI_2_9 + mova m6, [o(pd_3803)] ; SINPI_4_9 + pmulld m4, m0, m1 ; s[4] = SINPI_1_9 * T[2] + pmulld m7, m3, m6 ; s[6] = SINPI_4_9 * T[3] + pmulld m6, m1 ; s[3] = SINPI_4_9 * T[2] + pmulld m0, m5 ; s[0] = SINPI_1_9 * T[0] + psubd m1, m3 ; T[2] - T[3] + pmulld m3, m2 ; s[5] = SINPI_2_9 * T[3] + pmulld m2, m5 ; s[1] = SINPI_2_9 * T[0] + paddd m0, m6 ; s[0] += s[3] + paddd m0, m3 ; s[0] += s[5] + mova m3, [o(pd_m3344)] ; -SINPI_3_9 + psubd m2, m4 ; s[1] -= s[4] + psubd m2, m7 ; s[1] -= s[6] + psubd m1, m5 ; -b7 = (T[2] -T[3]) - T[0] + pmulld m1, m3 ; s[2] = -SINPI_3_9 * -b7 + pmulld m3, [r3] ; -s[3] = -SINPI_3_9 * T[1] + mova m5, [o(pd_2048)] + REPX {paddd x, m5}, m0, m1 ; {s[0], s[2]} + 2048 + paddd m4, m0, m2 ; x[3] = s[0] + s[1] + psubd m2, m3 ; x[1] = s[1] + s[3] + psubd m0, m3 ; x[0] = s[0] + s[3] + paddd m4, m3 ; x[3] -= s[3] + paddd m2, m5 ; x[1] + 2048 + REPX {psrad x, 12}, m0, m2, m1, m4 + ret + + +INV_TXFM_4X4_FN flipadst, dct +INV_TXFM_4X4_FN flipadst, adst +INV_TXFM_4X4_FN flipadst, flipadst +INV_TXFM_4X4_FN flipadst, identity + +cglobal iflipadst_4x4_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 + call m(iadst_4x4_internal_16bpc).main + packssdw m0, m2 ; out0 out1 + packssdw m1, m4 ; out2 out3 + ; transpose + punpcklwd m2, m1, m0 + punpckhwd m1, m0 + punpcklwd m0, m1, m2 + punpckhwd m1, m2 + ; m0 = out0 out1 + ; m1 = out2 out3 + ; m5 = pd_2048 + jmp tx2q +.pass2: + ; m0 = in0 in1 + ; m1 = in2 in3 +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call m_suffix(iadst_4x4_internal_8bpc, _ssse3).main + mova m4, [o(pw_2048)] + movq m3, [dstq+strideq*1] + movhps m3, [dstq+strideq*0] + lea r5, [dstq+strideq*2] + movq m2, [r5 +strideq*1] + movhps m2, [r5 +strideq*0] + mova m5, [o(pixel_10bpc_max)] + pmulhrsw m0, m4 + pmulhrsw m1, m4 + pxor m4, m4 + mova [cq+16*0], m4 + mova [cq+16*1], m4 + mova [cq+16*2], m4 + mova [cq+16*3], m4 + paddw m0, m2 + paddw m1, m3 + pmaxsw m0, m4 + pmaxsw m1, m4 + pminsw m0, m5 + pminsw m1, m5 + movhps [dstq+strideq*0], m1 + movq [dstq+strideq*1], m1 + movhps [r5 +strideq*0], m0 + movq [r5 +strideq*1], m0 + RET + +INV_TXFM_4X4_FN identity, dct +INV_TXFM_4X4_FN identity, adst +INV_TXFM_4X4_FN identity, flipadst +INV_TXFM_4X4_FN identity, identity + +cglobal iidentity_4x4_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 + mova m3, [o(pd_5793)] + pmulld m0, m3, [cq+16*0] + pmulld m1, m3, [cq+16*1] + pmulld m2, m3, [cq+16*2] + pmulld m3, [cq+16*3] + mova m5, [o(pd_2048)] + REPX {paddd x, m5}, m0, m1, m2, m3 + REPX {psrad x, 12}, m0, m1, m2, m3 + packssdw m0, m1 + packssdw m2, m3 + ; transpose + punpckhwd m3, m0, m2 + punpcklwd m0, m2 + punpckhwd m1, m0, m3 + punpcklwd m0, m3 + ; m0 = out0 out1 + ; m1 = out2 out3 + ; m5 = pd_2048 + jmp tx2q +.pass2: + ; m0 = in0 in1 + ; m1 = in2 in3 + ; m5 = pd_2048 + mova m4, [o(pw_1697x8)] + movq m2, [dstq+strideq*0] + movhps m2, [dstq+strideq*1] + lea r5, [dstq+strideq*2] + pmulhrsw m3, m4, m0 + pmulhrsw m4, m1 + paddsw m0, m3 + paddsw m1, m4 + movq m3, [r5 +strideq*0] + movhps m3, [r5 +strideq*1] + mova m4, [o(pixel_10bpc_max)] + packssdw m5, m5 ; pw_2048 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + pxor m5, m5 + mova [cq+16*0], m5 + mova [cq+16*1], m5 + mova [cq+16*2], m5 + mova [cq+16*3], m5 + paddw m0, m2 + paddw m1, m3 + pmaxsw m0, m5 + pmaxsw m1, m5 + pminsw m0, m4 + pminsw m1, m4 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + movq [r5 +strideq*0], m1 + movhps [r5 +strideq*1], m1 + RET + +%macro INV_TXFM_4X8_FN 2-3 0 ; type1, type2, eob_offset + INV_TXFM_FN %1, %2, %3, 4x8 +%ifidn %1_%2, dct_dct + imul r5d, [cq], 2896 + mov [cq], eobd ; 0 + mov r3d, 2 + add r5d, 2048 + sar r5d, 12 + imul r5d, 2896 + add r5d, 2048 + sar r5d, 12 +.end: + imul r5d, 2896 + add r5d, 34816 + movd m0, r5d + pshuflw m0, m0, q1111 + punpcklqdq m0, m0 + pxor m4, m4 + mova m3, [o(pixel_10bpc_max)] + lea r2, [strideq*3] +.loop: + movq m1, [dstq+strideq*0] + movq m2, [dstq+strideq*2] + movhps m1, [dstq+strideq*1] + movhps m2, [dstq+r2] + paddw m1, m0 + paddw m2, m0 + REPX {pminsw x, m3}, m1, m2 + REPX {pmaxsw x, m4}, m1, m2 + movq [dstq+strideq*0], m1 + movhps [dstq+strideq*1], m1 + movq [dstq+strideq*2], m2 + movhps [dstq+r2 ], m2 + lea dstq, [dstq+strideq*4] + dec r3d + jg .loop + RET +%endif +%endmacro + +INV_TXFM_4X8_FN dct, dct +INV_TXFM_4X8_FN dct, identity, 9 +INV_TXFM_4X8_FN dct, adst +INV_TXFM_4X8_FN dct, flipadst + +cglobal idct_4x8_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%undef cmp + mova m5, [o(pd_2048)] +%if ARCH_X86_64 + xor r5d, r5d + cmp eobd, 13 + setge r5b +%else + mov r5d, 1 + cmp eobd, 13 + sbb r5d, 0 +%endif + shl r5d, 4 +.loop_pass1: + mova m3, [o(pd_2896)] + pmulld m0, m3, [cq+32*0+r5] + pmulld m1, m3, [cq+32*1+r5] + pmulld m2, m3, [cq+32*2+r5] + pmulld m3, [cq+32*3+r5] + REPX {paddd x, m5}, m0, m1, m2, m3 + REPX {psrad x, 12}, m0, m1, m2, m3 + call m(idct_4x4_internal_16bpc).pass1_main + packssdw m0, m1 ; out0 out1 + packssdw m4, m2 ; out2 out3 + test r5d, r5d + jz .end_pass1 + mova [cq+32*0+16], m0 + mova [cq+32*1+16], m4 + xor r5d, r5d + jmp .loop_pass1 +.end_pass1: + punpckhwd m2, m0, m4 + punpcklwd m0, m4 + punpckhwd m1, m0, m2 + punpcklwd m0, m2 + mova m2, [cq+32*0+16] + mova m6, [cq+32*1+16] + punpckhwd m4, m2, m6 + punpcklwd m2, m6 + punpckhwd m3, m2, m4 + punpcklwd m2, m4 + ; m0-3 = packed & transposed output + jmp tx2q +.pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call m_suffix(idct_4x8_internal_8bpc, _ssse3).main + ; m0-3 is now out0/1,3/2,4/5,7/6 + mova m4, [o(pw_2048)] + shufps m1, m1, q1032 + shufps m3, m3, q1032 +.end: + REPX {pmulhrsw x, m4}, m0, m1, m2, m3 + pxor m4, m4 + REPX {mova [cq+16*x], m4}, 0, 1, 2, 3, 4, 5, 6, 7 + mova m7, [o(pixel_10bpc_max)] + lea r2, [strideq*3] + movq m5, [dstq+strideq*0] + movq m6, [dstq+strideq*2] + movhps m5, [dstq+strideq*1] + movhps m6, [dstq+r2] + lea r4, [dstq+strideq*4] + paddw m0, m5 + paddw m1, m6 + movq m5, [r4+strideq*0] + movq m6, [r4+strideq*2] + movhps m5, [r4+strideq*1] + movhps m6, [r4+r2] + paddw m2, m5 + paddw m3, m6 + REPX {pminsw x, m7}, m0, m1, m2, m3 + REPX {pmaxsw x, m4}, m0, m1, m2, m3 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + movq [dstq+strideq*2], m1 + movhps [dstq+r2 ], m1 + movq [r4 +strideq*0], m2 + movhps [r4 +strideq*1], m2 + movq [r4 +strideq*2], m3 + movhps [r4 +r2 ], m3 + RET + +INV_TXFM_4X8_FN adst, dct +INV_TXFM_4X8_FN adst, adst +INV_TXFM_4X8_FN adst, flipadst +INV_TXFM_4X8_FN adst, identity, 9 + +cglobal iadst_4x8_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 + call .pass1_main + punpckhwd m2, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m0, m2 + punpcklwd m0, m2 + mova m2, [cq+32*2+16] + mova m6, [cq+32*3+16] + punpckhwd m4, m2, m6 + punpcklwd m2, m6 + punpckhwd m3, m2, m4 + punpcklwd m2, m4 + ; m0-3 = packed & transposed output + jmp tx2q +.pass1_main: +%undef cmp +%if ARCH_X86_64 + xor r5d, r5d + cmp eobd, 13 + setge r5b +%else + mov r5d, 1 + cmp eobd, 13 + sbb r5d, 0 +%endif + shl r5d, 4 + lea r3, [cq+32*1+16] +.loop_pass1: + mova m0, [o(pd_2048)] + mova m3, [o(pd_2896)] + pmulld m5, m3, [cq+32*0+r5] + pmulld m2, m3, [cq+32*1+r5] + pmulld m1, m3, [cq+32*2+r5] + pmulld m3, [cq+32*3+r5] + REPX {paddd x, m0}, m5, m2, m1, m3 + REPX {psrad x, 12}, m5, m2, m1, m3 + mova [r3], m2 + call m(iadst_4x4_internal_16bpc).main2 + packssdw m0, m2 ; out0 out1 + packssdw m1, m4 ; out2 out3 + test r5d, r5d + jz .end_pass1 + mova [cq+32*2+16], m0 + mova [cq+32*3+16], m1 + xor r5d, r5d + jmp .loop_pass1 +.end_pass1: + ret +.pass2: + shufps m0, m0, q1032 + shufps m1, m1, q1032 +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call m_suffix(iadst_4x8_internal_8bpc, _ssse3).main + mova m4, [o(pw_4x2048_4xm2048)] + jmp m(idct_4x8_internal_16bpc).end + +INV_TXFM_4X8_FN flipadst, dct +INV_TXFM_4X8_FN flipadst, adst +INV_TXFM_4X8_FN flipadst, flipadst +INV_TXFM_4X8_FN flipadst, identity, 9 + +cglobal iflipadst_4x8_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 + call m(iadst_4x8_internal_16bpc).pass1_main + punpcklwd m2, m1, m0 + punpckhwd m1, m0 + punpcklwd m0, m1, m2 + punpckhwd m1, m2 + mova m6, [cq+32*2+16] + mova m2, [cq+32*3+16] + punpcklwd m4, m2, m6 + punpckhwd m2, m6 + punpckhwd m3, m2, m4 + punpcklwd m2, m4 + ; m0-3 = packed & transposed output + jmp tx2q +.pass2: + shufps m0, m0, q1032 + shufps m1, m1, q1032 +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call m_suffix(iadst_4x8_internal_8bpc, _ssse3).main + mova m4, m0 + mova m5, m1 + pshufd m0, m3, q1032 + pshufd m1, m2, q1032 + pshufd m2, m5, q1032 + pshufd m3, m4, q1032 + mova m4, [o(pw_4xm2048_4x2048)] + jmp m(idct_4x8_internal_16bpc).end + +INV_TXFM_4X8_FN identity, dct +INV_TXFM_4X8_FN identity, adst +INV_TXFM_4X8_FN identity, flipadst +INV_TXFM_4X8_FN identity, identity, 3 + +cglobal iidentity_4x8_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%undef cmp + mova m5, [o(pd_2048)] + mova m4, [o(pd_2896)] + mova m6, [o(pd_5793)] + ; clear m7 in case we skip the bottom square + pxor m7, m7 +%if ARCH_X86_64 + xor r5d, r5d + cmp eobd, 16 + setge r5b +%else + mov r5d, 1 + cmp eobd, 16 + sbb r5d, 0 +%endif + shl r5d, 4 +.loop_pass1: + pmulld m0, m4, [cq+32*0+r5] + pmulld m1, m4, [cq+32*1+r5] + pmulld m2, m4, [cq+32*2+r5] + pmulld m3, m4, [cq+32*3+r5] + REPX {paddd x, m5}, m0, m1, m2, m3 + REPX {psrad x, 12}, m0, m1, m2, m3 + REPX {pmulld x, m6}, m0, m1, m2, m3 + REPX {paddd x, m5}, m0, m1, m2, m3 + REPX {psrad x, 12}, m0, m1, m2, m3 + packssdw m0, m1 + packssdw m2, m3 + test r5d, r5d + jz .end_pass1 + mova [cq+32*0+16], m0 + mova m7, m2 + xor r5d, r5d + jmp .loop_pass1 +.end_pass1: + punpckhwd m4, m0, m2 + punpcklwd m0, m2 + punpckhwd m1, m0, m4 + punpcklwd m0, m4 + mova m2, [cq+32*0+16] + punpckhwd m4, m2, m7 + punpcklwd m2, m7 + punpckhwd m3, m2, m4 + punpcklwd m2, m4 + ; m0-3 = packed & transposed output + jmp tx2q +.pass2: + mova m4, [o(pw_4096)] + jmp m(idct_4x8_internal_16bpc).end + +%macro INV_TXFM_4X16_FN 2-3 2d ; type1, type2, eob_tbl_suffix + INV_TXFM_FN %1, %2, tbl_4x16_%3, 4x16 +%ifidn %1_%2, dct_dct + imul r5d, [cq], 2896 + mov [cq], eobd ; 0 + mov r3d, 4 + add r5d, 6144 + sar r5d, 13 + jmp m(inv_txfm_add_dct_dct_4x8_16bpc).end +%endif +%endmacro + +INV_TXFM_4X16_FN dct, dct +INV_TXFM_4X16_FN dct, identity, v +INV_TXFM_4X16_FN dct, adst +INV_TXFM_4X16_FN dct, flipadst + +cglobal idct_4x16_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%undef cmp +%if ARCH_X86_32 + mov r5m, r6d +%endif + mov r6d, 4 +.zero_loop: + dec r6d + cmp eobb, byte [r5+r6] + jl .zero_loop + mov r5d, r6d + shl r5d, 4 +%if ARCH_X86_32 + ; restore pic-ptr + mov r6, r5m +%endif + mova m5, [o(pd_2048)] +.loop_pass1: + mova m0, [cq+64*0+r5] + mova m1, [cq+64*1+r5] + mova m2, [cq+64*2+r5] + mova m3, [cq+64*3+r5] + call m(idct_4x4_internal_16bpc).pass1_main + pcmpeqd m3, m3 + REPX {psubd x, m3}, m0, m1, m4, m2 + REPX {psrad x, 1}, m0, m1, m4, m2 + packssdw m0, m1 ; out0 out1 + packssdw m4, m2 ; out2 out3 + punpckhwd m2, m0, m4 + punpcklwd m0, m4 + punpckhwd m1, m0, m2 + punpcklwd m0, m2 + test r5d, r5d + jz .end_pass1 + mova [cq+64*0+r5], m0 + mova [cq+64*1+r5], m1 + sub r5d, 16 + jmp .loop_pass1 +.end_pass1: + mova m2, [cq+64*0+16] + mova m3, [cq+64*1+16] + mova m4, [cq+64*0+32] + mova m5, [cq+64*1+32] + mova m6, [cq+64*0+48] + mova m7, [cq+64*1+48] + ; m0-7 = packed & transposed output + jmp tx2q +.pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call m_suffix(idct_16x4_internal_8bpc, _ssse3).main + ; m0-6 is out0-13 [with odd registers having inversed output] + ; [coeffq+16*7] has out15/14 + mova m7, [o(pw_2048)] + REPX {pmulhrsw x, m7}, m0, m1, m2, m3, m4, m5, m6 + pmulhrsw m7, [cq+16*7] + REPX {shufps x, x, q1032}, m1, m3, m5, m7 + mova [cq+16*0], m4 + mova [cq+16*1], m5 + mova [cq+16*2], m6 + mova [cq+16*3], m7 +.end: + pxor m4, m4 + REPX {mova [cq+16*x], m4}, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + mova m7, [o(pixel_10bpc_max)] + mov r5d, 2 + lea r3, [strideq*3] +.loop: + movq m5, [dstq+strideq*0] + movq m6, [dstq+strideq*2] + movhps m5, [dstq+strideq*1] + movhps m6, [dstq+r3] + lea r4, [dstq+strideq*4] + paddw m0, m5 + paddw m1, m6 + movq m5, [r4+strideq*0] + movq m6, [r4+strideq*2] + movhps m5, [r4+strideq*1] + movhps m6, [r4+r3] + paddw m2, m5 + paddw m3, m6 + REPX {pminsw x, m7}, m0, m1, m2, m3 + REPX {pmaxsw x, m4}, m0, m1, m2, m3 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + movq [dstq+strideq*2], m1 + movhps [dstq+r3 ], m1 + movq [r4 +strideq*0], m2 + movhps [r4 +strideq*1], m2 + movq [r4 +strideq*2], m3 + movhps [r4 +r3 ], m3 + dec r5d + jz .end2 + lea dstq, [dstq+strideq*8] + mova m0, [cq+0*16] + mova m1, [cq+1*16] + mova m2, [cq+2*16] + mova m3, [cq+3*16] + REPX {mova [cq+x*16], m4}, 0, 1, 2, 3 + jmp .loop +.end2: + RET + +INV_TXFM_4X16_FN adst, dct +INV_TXFM_4X16_FN adst, adst +INV_TXFM_4X16_FN adst, flipadst +INV_TXFM_4X16_FN adst, identity, v + +cglobal iadst_4x16_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%undef cmp +%if ARCH_X86_32 + mov r5m, r6d +%endif + mov r6d, 4 +.zero_loop: + dec r6d + cmp eobb, byte [r6+r5] + jl .zero_loop + mov r5d, r6d + shl r5d, 4 +%if ARCH_X86_32 + ; restore pic-ptr + mov r6, r5m +%endif +.loop_pass1: + mova m5, [cq+64*0+r5] + lea r3, [cq+64*1+r5] + mova m1, [cq+64*2+r5] + mova m3, [cq+64*3+r5] + call m(iadst_4x4_internal_16bpc).main2 + pcmpeqd m3, m3 + REPX {psubd x, m3}, m0, m2, m1, m4 + REPX {psrad x, 1}, m0, m2, m1, m4 + packssdw m0, m2 ; out0 out1 + packssdw m1, m4 ; out2 out3 + punpckhwd m2, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m0, m2 + punpcklwd m0, m2 + test r5d, r5d + jz m(idct_4x16_internal_16bpc).end_pass1 + mova [cq+64*0+r5], m0 + mova [cq+64*1+r5], m1 + sub r5d, 16 + jmp .loop_pass1 +.pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call m_suffix(iadst_16x4_internal_8bpc, _ssse3).main + call m_suffix(iadst_16x4_internal_8bpc, _ssse3).main_pass2_end + ; m7/5/2/4 = out4/-11,-5/10,6/-9,-7/8 + ; m0/3 & cq6/7 = out0/-15,-3/12,-1/14,2/-13 + mova m1, [o(pw_4x2048_4xm2048)] + REPX {pmulhrsw x, m1}, m7, m2, m0 + pshufd m6, m1, q1032 ; 4x-2048,4x2048 + pmulhrsw m1, [cq+16*7] + REPX {pmulhrsw x, m6}, m5, m4, m3 + pmulhrsw m6, [cq+16*6] + ; m7/5/2/4 = out4/11,5/10,6/9,7/8 + ; m0/3/6/1 = out0/15,3/12,1/14,2/13 + ; output should be as 0-3 for out0-7, and cq+0-3*16 for out8-15 + movhps [cq+0*8], m4 + movhps [cq+1*8], m2 + movhps [cq+2*8], m5 + movhps [cq+3*8], m7 + movhps [cq+4*8], m3 + movhps [cq+5*8], m1 + movhps [cq+6*8], m6 + movhps [cq+7*8], m0 + punpcklqdq m0, m6 + punpcklqdq m1, m3 + punpcklqdq m3, m2, m4 + punpcklqdq m2, m7, m5 + jmp m(idct_4x16_internal_16bpc).end + +INV_TXFM_4X16_FN flipadst, dct +INV_TXFM_4X16_FN flipadst, adst +INV_TXFM_4X16_FN flipadst, flipadst +INV_TXFM_4X16_FN flipadst, identity, v + +cglobal iflipadst_4x16_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%undef cmp +%if ARCH_X86_32 + mov r5m, r6d +%endif + mov r6d, 4 +.zero_loop: + dec r6d + cmp eobb, byte [r5+r6] + jl .zero_loop + mov r5d, r6d + shl r5d, 4 +%if ARCH_X86_32 + ; restore pic-ptr + mov r6, r5m +%endif +.loop_pass1: + mova m5, [cq+64*0+r5] + lea r3, [cq+64*1+r5] + mova m1, [cq+64*2+r5] + mova m3, [cq+64*3+r5] + call m(iadst_4x4_internal_16bpc).main2 + pcmpeqd m3, m3 + REPX {psubd x, m3}, m0, m2, m1, m4 + REPX {psrad x, 1}, m0, m2, m1, m4 + packssdw m0, m2 ; out3 out2 + packssdw m1, m4 ; out1 out0 + punpcklwd m2, m1, m0 + punpckhwd m1, m0 + punpcklwd m0, m1, m2 + punpckhwd m1, m2 + test r5d, r5d + jz m(idct_4x16_internal_16bpc).end_pass1 + mova [cq+64*0+r5], m0 + mova [cq+64*1+r5], m1 + sub r5d, 16 + jmp .loop_pass1 +.pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call m_suffix(iadst_16x4_internal_8bpc, _ssse3).main + call m_suffix(iadst_16x4_internal_8bpc, _ssse3).main_pass2_end + ; m7/5/2/4 = out11/-4,-10/5,9/-6,-8/7 + ; m0/3 & cq6/7 = out15/-0,-12/3,-14/1,13/-2 + mova m1, [o(pw_4x2048_4xm2048)] + REPX {pmulhrsw x, m1}, m7, m2, m0 + pshufd m6, m1, q1032 ; 4x-2048,4x2048 + pmulhrsw m1, [cq+16*7] + REPX {pmulhrsw x, m6}, m5, m4, m3 + pmulhrsw m6, [cq+16*6] + ; m7/5/2/4 = out11/4,10/5,9/6,8/7 + ; m0/3/6/1 = out15/0,12/3,14/1,13/2 + ; output should be as 0-3 for out0-7, and cq+0-3*16 for out8-15 + movq [cq+0*8], m4 + movq [cq+1*8], m2 + movq [cq+2*8], m5 + movq [cq+3*8], m7 + movq [cq+4*8], m3 + movq [cq+5*8], m1 + movq [cq+6*8], m6 + movq [cq+7*8], m0 + punpckhqdq m0, m6 + punpckhqdq m1, m3 + punpckhqdq m3, m2, m4 + punpckhqdq m2, m7, m5 + jmp m(idct_4x16_internal_16bpc).end + +INV_TXFM_4X16_FN identity, dct, h +INV_TXFM_4X16_FN identity, adst, h +INV_TXFM_4X16_FN identity, flipadst, h +INV_TXFM_4X16_FN identity, identity + +cglobal iidentity_4x16_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%undef cmp +%if ARCH_X86_32 + mov r5m, r6d +%endif + mov r6d, 4 +.zero_loop: + dec r6d + cmp eobb, byte [r5+r6] + jl .zero_loop + mov r5d, r6d + shl r5d, 4 +%if ARCH_X86_32 + ; restore pic-ptr + mov r6, r5m +%endif + mova m5, [o(pd_6144)] + mova m4, [o(pd_5793)] +.loop_pass1: + pmulld m0, m4, [cq+64*0+r5] + pmulld m1, m4, [cq+64*1+r5] + pmulld m2, m4, [cq+64*2+r5] + pmulld m3, m4, [cq+64*3+r5] + REPX {paddd x, m5}, m0, m1, m2, m3 + REPX {psrad x, 13}, m0, m1, m2, m3 + packssdw m0, m1 + packssdw m2, m3 + punpckhwd m3, m0, m2 + punpcklwd m0, m2 + punpckhwd m1, m0, m3 + punpcklwd m0, m3 + test r5d, r5d + jz m(idct_4x16_internal_16bpc).end_pass1 + mova [cq+64*0+r5], m0 + mova [cq+64*1+r5], m1 + sub r5d, 16 + jmp .loop_pass1 +.pass2: + mova [cq+16*4], m0 + mova [cq+16*5], m1 + mova [cq+16*6], m2 + mova [cq+16*7], m7 + mova m0, [o(pw_1697x16)] + mova m7, [o(pw_2048)] + pmulhrsw m1, m0, m4 + pmulhrsw m2, m0, m5 + REPX {paddsw x, x}, m4, m5 + paddsw m4, m1 + paddsw m5, m2 + REPX {pmulhrsw x, m7}, m4, m5 + mova [cq+16*0], m4 + mova [cq+16*1], m5 + mova m4, [cq+16*7] + pmulhrsw m1, m0, m6 + pmulhrsw m2, m0, m4 + REPX {paddsw x, x}, m6, m4 + paddsw m6, m1 + paddsw m4, m2 + REPX {pmulhrsw x, m7}, m6, m4 + mova [cq+16*2], m6 + mova [cq+16*3], m4 + mova m4, [cq+16*4] + mova m1, [cq+16*5] + mova m2, [cq+16*6] + pmulhrsw m5, m0, m2 + pmulhrsw m6, m0, m3 + REPX {paddsw x, x}, m2, m3 + paddsw m2, m5 + paddsw m3, m6 + pmulhrsw m6, m0, m1 + pmulhrsw m0, m4 + REPX {paddsw x, x}, m1, m4 + paddsw m1, m6 + paddsw m0, m4 + REPX {pmulhrsw x, m7}, m2, m3, m1, m0 + jmp m(idct_4x16_internal_16bpc).end + +%macro INV_TXFM_8X4_FN 2 ; type1, type2 +%if ARCH_X86_64 + INV_TXFM_FN %1, %2, 0, 8x4, 15 +%else + INV_TXFM_FN %1, %2, 0, 8x4, 8, 0-4*16 +%endif +%ifidn %1_%2, dct_dct + imul r5d, [cq], 2896 + mov [cq], eobd ; 0 + add r5d, 2048 + sar r5d, 12 + imul r5d, 2896 + add r5d, 2048 + sar r5d, 12 + imul r5d, 2896 + add r5d, 34816 + movd m0, r5d + pshuflw m0, m0, q1111 + punpcklqdq m0, m0 + mova m6, [o(pixel_10bpc_max)] + pxor m5, m5 + lea r2, [strideq*3] + mova m1, [dstq+strideq*0] + mova m2, [dstq+strideq*1] + mova m3, [dstq+strideq*2] + mova m4, [dstq+r2] + REPX {paddw x, m0}, m1, m2, m3, m4 + REPX {pmaxsw x, m5}, m1, m2, m3, m4 + REPX {pminsw x, m6}, m1, m2, m3, m4 + mova [dstq+strideq*0], m1 + mova [dstq+strideq*1], m2 + mova [dstq+strideq*2], m3 + mova [dstq+r2 ], m4 + RET +%endif +%endmacro + +INV_TXFM_8X4_FN dct, dct +INV_TXFM_8X4_FN dct, identity +INV_TXFM_8X4_FN dct, adst +INV_TXFM_8X4_FN dct, flipadst + +cglobal idct_8x4_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 + lea r5, [o(.main)] +.pass1_entry: +%if ARCH_X86_32 + lea r3, [rsp+gprsize] +%else + mova m11, [o(pd_2048)] + mova m12, [o(clip_min)] + mova m13, [o(clip_max)] + mova m14, [o(pd_2896)] +%endif + mova m0, [cq+0*16] + mova m1, [cq+1*16] + mova m2, [cq+2*16] + mova m3, [cq+3*16] + mova m4, [cq+4*16] + mova m5, [cq+5*16] + mova m6, [cq+6*16] + mova m7, [cq+7*16] + call .rect2_mul + call r5 + call .transpose4x8packed + ; m0-3 = packed & transposed output + jmp tx2q +.transpose4x8packed: + ; transpose + punpcklwd m1, m2, m6 + punpckhwd m2, m6 + punpckhwd m6, m0, m4 + punpcklwd m0, m4 + + punpckhwd m3, m0, m1 + punpcklwd m0, m1 + punpckhwd m4, m6, m2 + punpcklwd m6, m2 + + punpcklwd m2, m3, m4 + punpckhwd m3, m4 + punpckhwd m1, m0, m6 + punpcklwd m0, m6 + ret +.main: + call .main_pass1 + call .round + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + ret +.rect2_mul: +%if ARCH_X86_64 + REPX {pmulld x, m14}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {paddd x, m11}, m0, m1, m2, m3, m4, m5, m6, m7 +%else + mova [r3], m7 + mova m7, [o(pd_2896)] + REPX {pmulld x, m7}, m0, m1, m2, m3, m4, m5, m6 + pmulld m7, [r3] + mova [r3], m7 + mova m7, [o(pd_2048)] + REPX {paddd x, m7}, m0, m1, m2, m3, m4, m5, m6 + paddd m7, [r3] +%endif + REPX {psrad x, 12}, m0, m1, m2, m3, m4, m5, m6, m7 + ret +%if ARCH_X86_64 +.main_pass1_fast: + pmulld m5, m3, [o(pd_m2276)] + pmulld m3, [o(pd_3406)] + pmulld m7, m1, [o(pd_4017)] + pmulld m1, [o(pd_799)] + pmulld m6, m2, [o(pd_3784)] + pmulld m2, [o(pd_1567)] + pmulld m0, m14 + pxor m4, m4 + jmp .main_pass1_fast2 +.main_pass1: + ITX_MULSUB_2D 5, 3, 8, 9, 10, _, 3406, 2276 ; t5a t6a + ITX_MULSUB_2D 1, 7, 8, 9, 10, _, 799, 4017 ; t4a t7a + ITX_MULSUB_2D 2, 6, 8, 9, 10, _, 1567, 3784 ; t2 t3 + REPX {pmulld x, m14}, m0, m4 +.main_pass1_fast2: + REPX {paddd x, m11}, m1, m2, m3, m5, m6, m7 + REPX {psrad x, 12 }, m1, m2, m3, m5, m6, m7 + paddd m8, m1, m5 ; t4 + psubd m1, m5 ; t5a + paddd m9, m7, m3 ; t7 + psubd m7, m3 ; t6a + REPX {pmaxsd x, m12}, m1, m8, m7, m9 + REPX {pminsd x, m13}, m1, m8, m7, m9 + REPX {pmulld x, m14}, m7, m1 + paddd m0, m11 + paddd m7, m11 + psubd m5, m0, m4 + paddd m0, m4 + psubd m4, m7, m1 + paddd m7, m1 + REPX {psrad x, 12 }, m5, m0, m4, m7 + psubd m3, m0, m6 ; dct4 out3 + paddd m0, m6 ; dct4 out0 + paddd m6, m5, m2 ; dct4 out1 + psubd m5, m2 ; dct4 out2 + REPX {pmaxsd x, m12}, m0, m6, m5, m3 + REPX {pminsd x, m13}, m0, m6, m5, m3 + ret +.round: + paddd m1, m6, m7 ; out1 + psubd m6, m7 ; out6 + psubd m7, m0, m9 ; out7 + paddd m0, m9 ; out0 + paddd m2, m5, m4 ; out2 + psubd m5, m4 ; out5 + psubd m4, m3, m8 ; out4 + paddd m3, m8 ; out3 +%else +.main_pass1_fast: + pmulld m5, m3, [o(pd_m2276)] + pmulld m3, [o(pd_3406)] + pmulld m7, m1, [o(pd_4017)] + pmulld m1, [o(pd_799)] + pmulld m6, m2, [o(pd_3784)] + pmulld m2, [o(pd_1567)] + mova m4, [o(pd_2048)] + mova [r3+0*16], m2 + REPX {paddd x, m4}, m5, m3, m7, m1 + REPX {psrad x, 12}, m5, m3, m7, m1 + paddd m2, m1, m5 ; t4 + psubd m1, m5 ; t5a + pmulld m5, m0, [o(pd_2896)] + mova m0, m4 + paddd m4, m7, m3 ; t7 + psubd m7, m3 ; t6a + mova m3, [o(clip_min)] + REPX {pmaxsd x, m3 }, m1, m2, m7, m4 + mova m3, [o(clip_max)] + REPX {pminsd x, m3 }, m1, m2, m7, m4 + mova [r3+3*16], m2 + mova [r3+1*16], m4 + pxor m4, m4 + mova m2, [r3+0*16] + mova m3, [o(pd_2896)] + jmp .main_pass1_fast2 +.main_pass1: + mova [r3+0*16], m0 + mova [r3+1*16], m2 + mova [r3+2*16], m4 + mova [r3+3*16], m6 + mova m0, [o(pd_2048)] + ITX_MULSUB_2D 5, 3, 2, 4, 6, 0, 3406, 2276 ; t5a t6a + ITX_MULSUB_2D 1, 7, 2, 4, 6, 0, 799, 4017 ; t4a t7a + paddd m2, m1, m5 ; t4 + psubd m1, m5 ; t5a + paddd m4, m7, m3 ; t7 + psubd m7, m3 ; t6a + mova m6, [o(clip_min)] + REPX {pmaxsd x, m6 }, m1, m2, m7, m4 + mova m6, [o(clip_max)] + REPX {pminsd x, m6 }, m1, m2, m7, m4 + mova m6, [r3+3*16] + mova [r3+3*16], m2 + mova m2, [r3+1*16] + mova [r3+1*16], m4 + + ITX_MULSUB_2D 2, 6, 4, 3, 5, _, 1567, 3784 ; t2 t3 + mova m3, [o(pd_2896)] + mova m5, [r3+0*16] + mova m4, [r3+2*16] + REPX {pmulld x, m3 }, m5, m4 +.main_pass1_fast2: + REPX {paddd x, m0 }, m2, m6 + REPX {psrad x, 12 }, m2, m6 + REPX {pmulld x, m3 }, m7, m1 + paddd m7, m0 + paddd m0, m5 + + psubd m5, m0, m4 + paddd m0, m4 + psubd m4, m7, m1 + paddd m7, m1 + REPX {psrad x, 12 }, m5, m0, m4, m7 + psubd m3, m0, m6 ; dct4 out3 + paddd m0, m6 ; dct4 out0 + paddd m6, m5, m2 ; dct4 out1 + psubd m5, m2 ; dct4 out2 + + mova m1, [o(clip_min)] + REPX {pmaxsd x, m1 }, m0, m6, m5, m3 + mova m1, [o(clip_max)] + REPX {pminsd x, m1 }, m0, m6, m5, m3 + ret +.round: + paddd m1, m6, m7 ; out1 + psubd m6, m7 ; out6 + mova [r3+0*16], m6 + mova m6, [r3+1*16] + psubd m7, m0, m6 ; out7 + paddd m0, m6 ; out0 + paddd m2, m5, m4 ; out2 + psubd m5, m4 ; out5 + mova m6, [r3+3*16] + psubd m4, m3, m6 ; out4 + paddd m3, m6 ; out3 + mova m6, [r3+0*16] +%endif + ret + +.pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call m_suffix(idct_8x4_internal_8bpc, _ssse3).main +.end: + lea r3, [strideq*3] + call .round2_and_write_8x4 + REPX {mova [cq+16*x], m6}, 0, 1, 2, 3, 4, 5, 6, 7 + RET +.round2_and_write_8x4: + pxor m6, m6 + mova m5, [o(pixel_10bpc_max)] + mova m4, [o(pw_2048)] +.round1_and_write_8x4: + REPX {pmulhrsw x, m4}, m0, m1, m2, m3 +.write_8x4: + paddw m0, [dstq+strideq*0] + paddw m1, [dstq+strideq*1] + paddw m2, [dstq+strideq*2] + paddw m3, [dstq+r3] + REPX {pminsw x, m5}, m0, m1, m2, m3 + REPX {pmaxsw x, m6}, m0, m1, m2, m3 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + mova [dstq+strideq*2], m2 + mova [dstq+r3 ], m3 + ret + +INV_TXFM_8X4_FN adst, dct +INV_TXFM_8X4_FN adst, adst +INV_TXFM_8X4_FN adst, flipadst +INV_TXFM_8X4_FN adst, identity + +cglobal iadst_8x4_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 + lea r5, [o(.main)] + jmp m(idct_8x4_internal_16bpc).pass1_entry +.main: + call .main_pass1 + call .round + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + ret +.main_pass1: +%if ARCH_X86_64 + ITX_MULSUB_2D 7, 0, 8, 9, 10, 11, 401, 4076 ; t1a, t0a + ITX_MULSUB_2D 1, 6, 8, 9, 10, 11, 3920, 1189 ; t7a, t6a + ITX_MULSUB_2D 5, 2, 8, 9, 10, 11, 1931, 3612 ; t3a, t2a + ITX_MULSUB_2D 3, 4, 8, 9, 10, 11, 3166, 2598 ; t5a, t4a + psubd m8, m2, m6 ; t6 + paddd m2, m6 ; t2 + psubd m6, m0, m4 ; t4 + paddd m0, m4 ; t0 + psubd m4, m5, m1 ; t7 + paddd m5, m1 ; t3 + psubd m1, m7, m3 ; t5 + paddd m7, m3 ; t1 + REPX {pmaxsd x, m12}, m6, m1, m8, m4, m2, m0, m5, m7 + REPX {pminsd x, m13}, m6, m1, m8, m4, m2, m0, m5, m7 + ITX_MULSUB_2D 6, 1, 3, 9, 10, 11, 1567, 3784 ; t5a, t4a + ITX_MULSUB_2D 4, 8, 3, 9, 10, 11, 3784, 10 ; t6a, t7a + psubd m9, m6, m8 ; t7 + paddd m6, m8 ; out6 + mova m8, [o(pd_2896)] + psubd m3, m7, m5 ; t3 + paddd m7, m5 ; -out7 + psubd m5, m0, m2 ; t2 + paddd m0, m2 ; out0 + psubd m2, m1, m4 ; t6 + paddd m1, m4 ; -out1 + REPX {pmaxsd x, m12}, m5, m3, m2, m9 + REPX {pminsd x, m13}, m5, m3, m2, m9 + REPX {pmulld x, m14}, m5, m3, m2, m9 + psubd m4, m5, m3 ; (t2 - t3) * 2896 + paddd m3, m5 ; (t2 + t3) * 2896 + psubd m5, m2, m9 ; (t6 - t7) * 2896 + paddd m2, m9 ; (t6 + t7) * 2896 + ret +.round: + + ; m0=out0,m1=-out1,m6=out6,m7=-out7 + + pcmpeqd m8, m8 + REPX {pxor x, m8 }, m1, m7, m3, m5 + REPX {psubd x, m8 }, m1, m7 + REPX {paddd x, m11}, m2, m3, m4, m5 + REPX {psrad x, 12 }, m2, m3, m4, m5 +%else + mova [r3+0*16], m2 + mova [r3+1*16], m3 + mova [r3+2*16], m4 + mova [r3+3*16], m5 + mova m5, [o(pd_2048)] + + ITX_MULSUB_2D 7, 0, 2, 3, 4, 5, 401, 4076 ; t1a, t0a + ITX_MULSUB_2D 1, 6, 2, 3, 4, 5, 3920, 1189 ; t7a, t6a + mova m2, [r3+0*16] + mova m3, [r3+1*16] + mova m4, [r3+2*16] + mova [r3+0*16], m0 + mova [r3+1*16], m1 + mova [r3+2*16], m6 + mova m1, [r3+3*16] + mova [r3+3*16], m7 + ITX_MULSUB_2D 1, 2, 0, 6, 7, 5, 1931, 3612 ; t3a, t2a + ITX_MULSUB_2D 3, 4, 0, 6, 7, 5, 3166, 2598 ; t5a, t4a + mova m0, [r3+0*16] + mova m6, [r3+2*16] + psubd m7, m2, m6 ; t6 + paddd m2, m6 ; t2 + psubd m6, m0, m4 ; t4 + paddd m0, m4 ; t0 + mova [r3+0*16], m7 + mova m5, [r3+1*16] + mova m7, [r3+3*16] + psubd m4, m1, m5 ; t7 + paddd m5, m1 ; t3 + psubd m1, m7, m3 ; t5 + paddd m7, m3 ; t1 + mova m3, [o(clip_min)] + REPX {pmaxsd x, m3 }, m6, m1, m4, m2, m0, m5, m7 + mova [r3+1*16], m7 + mova m7, [o(clip_max)] + pmaxsd m3, [r3+0*16] + REPX {pminsd x, m7 }, m6, m1, m3, m4, m2, m0, m5 + pminsd m7, [r3+1*16] + mova [r3+0*16], m0 + mova [r3+1*16], m2 + mova [r3+2*16], m5 + mova [r3+3*16], m7 + mova m0, [o(pd_2048)] + ITX_MULSUB_2D 6, 1, 2, 5, 7, 0, 1567, 3784 ; t5a, t4a + ITX_MULSUB_2D 4, 3, 2, 5, 7, 0, 3784, 7 ; t6a, t7a + mova m5, [r3+2*16] + mova m7, [r3+3*16] + psubd m2, m6, m3 ; t7 + paddd m6, m3 ; out6 + mova [r3+3*16], m6 + mova m0, [r3+0*16] + mova m6, [r3+1*16] + psubd m3, m7, m5 ; t3 + paddd m7, m5 ; -out7 + psubd m5, m0, m6 ; t2 + paddd m0, m6 ; out0 + psubd m6, m1, m4 ; t6 + paddd m1, m4 ; -out1 + mova m4, [o(clip_min)] + REPX {pmaxsd x, m4 }, m5, m3, m6, m2 + mova m4, [o(clip_max)] + REPX {pminsd x, m4 }, m5, m3, m6, m2 + mova m4, [o(pd_2896)] + REPX {pmulld x, m4 }, m5, m3, m6, m2 + psubd m4, m5, m3 ; (t2 - t3) * 2896 + paddd m3, m5 ; (t2 + t3) * 2896 + psubd m5, m6, m2 ; (t6 - t7) * 2896 + paddd m2, m6 ; (t6 + t7) * 2896 + ret +.round: + mova [r3+2*16], m0 + + pcmpeqd m0, m0 + mova m6, [o(pd_2048)] + REPX {pxor x, m0 }, m1, m7, m3, m5 + REPX {psubd x, m0 }, m1, m7 + REPX {paddd x, m6 }, m2, m3, m4, m5 + REPX {psrad x, 12 }, m2, m3, m4, m5 + + mova m6, [r3+3*16] + mova m0, [r3+2*16] +%endif + ret + +.pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call m_suffix(iadst_8x4_internal_8bpc, _ssse3).main + jmp m(idct_8x4_internal_16bpc).end + +INV_TXFM_8X4_FN flipadst, dct +INV_TXFM_8X4_FN flipadst, adst +INV_TXFM_8X4_FN flipadst, flipadst +INV_TXFM_8X4_FN flipadst, identity + +cglobal iflipadst_8x4_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 + lea r5, [o(.main)] + jmp m(idct_8x4_internal_16bpc).pass1_entry +.main: + call m(iadst_8x4_internal_16bpc).main_pass1 + call m(iadst_8x4_internal_16bpc).round + packssdw m7, m6 + packssdw m5, m4 + packssdw m3, m2 + packssdw m1, m0 + mova m0, m7 + mova m2, m5 + mova m4, m3 + mova m6, m1 + ret +.pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call m_suffix(iadst_8x4_internal_8bpc, _ssse3).main + lea r3, [strideq*3] + add dstq, r3 + neg strideq + jmp m(idct_8x4_internal_16bpc).end + +INV_TXFM_8X4_FN identity, dct +INV_TXFM_8X4_FN identity, adst +INV_TXFM_8X4_FN identity, flipadst +INV_TXFM_8X4_FN identity, identity + +cglobal iidentity_8x4_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 + lea r5, [o(.main)] + jmp m(idct_8x4_internal_16bpc).pass1_entry +.main: + REPX {paddd x, x}, m0, m1, m2, m3, m4, m5, m6, m7 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + ret +.pass2: + mova m7, [o(pw_1697x8)] + pmulhrsw m4, m7, m0 + pmulhrsw m5, m7, m1 + pmulhrsw m6, m7, m2 + pmulhrsw m7, m3 + paddsw m0, m4 + paddsw m1, m5 + paddsw m2, m6 + paddsw m3, m7 + jmp m(idct_8x4_internal_16bpc).end + +%macro INV_TXFM_8X8_FN 2-3 0 ; type1, type2, eob_offset +%if ARCH_X86_64 + INV_TXFM_FN %1, %2, %3, 8x8, 15, 0-3*16 +%else + INV_TXFM_FN %1, %2, %3, 8x8, 8, 0-5*16 +%endif +%ifidn %1_%2, dct_dct + imul r5d, [cq], 2896 + mov [cq], eobd ; 0 + mov r3d, 2 +.end: + add r5d, 6144 + sar r5d, 13 +.end2: + imul r5d, 2896 + add r5d, 34816 + movd m0, r5d + pshuflw m0, m0, q1111 + punpcklqdq m0, m0 + mova m6, [o(pixel_10bpc_max)] + pxor m5, m5 + lea r2, [strideq*3] +.loop: + mova m1, [dstq+strideq*0] + mova m2, [dstq+strideq*1] + mova m3, [dstq+strideq*2] + mova m4, [dstq+r2] + REPX {paddw x, m0}, m1, m2, m3, m4 + REPX {pmaxsw x, m5}, m1, m2, m3, m4 + REPX {pminsw x, m6}, m1, m2, m3, m4 + mova [dstq+strideq*0], m1 + mova [dstq+strideq*1], m2 + mova [dstq+strideq*2], m3 + mova [dstq+r2 ], m4 + lea dstq, [dstq+strideq*4] + dec r3d + jg .loop + RET +%endif +%endmacro + +INV_TXFM_8X8_FN dct, dct +INV_TXFM_8X8_FN dct, identity, 6 +INV_TXFM_8X8_FN dct, adst +INV_TXFM_8X8_FN dct, flipadst + +cglobal idct_8x8_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if ARCH_X86_32 + DECLARE_REG_TMP 1 + mov [rsp+4*16+1*gprsize], r1 +%else + DECLARE_REG_TMP 6 +%endif + lea t0, [o(.pass1_main)] + +.pass1_full: +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_min)] + mova m13, [o(clip_max)] + mova m14, [o(pd_2896)] +%endif +%undef cmp +%if ARCH_X86_64 + xor r5d, r5d + cmp eobd, 10 + setge r5b +%else + mov r5d, 1 + cmp eobd, 10 + sbb r5d, 0 +%endif + shl r5d, 4 +%if ARCH_X86_32 + lea r3, [rsp+gprsize] +%endif +.loop_pass1: + mova m0, [cq+0*32+r5] + mova m1, [cq+1*32+r5] + mova m2, [cq+2*32+r5] + mova m3, [cq+3*32+r5] + mova m4, [cq+4*32+r5] + mova m5, [cq+5*32+r5] + mova m6, [cq+6*32+r5] + mova m7, [cq+7*32+r5] + call t0 + + test r5d, r5d + jz .end_pass1 + + mova [cq+0*32+16], m0 + mova [cq+1*32+16], m1 + mova [cq+2*32+16], m2 + mova [cq+3*32+16], m3 + + sub r5d, 16 + jmp .loop_pass1 +.end_pass1: + mova m4, [cq+0*32+16] + mova m5, [cq+1*32+16] + mova m6, [cq+2*32+16] + mova m7, [cq+3*32+16] +%if ARCH_X86_32 + mov r1, [rsp+4*16+1*gprsize] +%endif + jmp tx2q +.pass1_main: + call m(idct_8x4_internal_16bpc).main_pass1 + pcmpeqd m1, m1 + REPX {psubd x, m1}, m0, m6, m5, m3 + call m(idct_8x4_internal_16bpc).round + REPX {psrad x, 1 }, m0, m1, m2, m3, m4, m5, m6, m7 +.pack_and_transpose: + packssdw m2, m3 + packssdw m6, m7 + packssdw m0, m1 + packssdw m4, m5 + jmp m(idct_8x4_internal_16bpc).transpose4x8packed + +.pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call m_suffix(idct_8x8_internal_8bpc, _ssse3).main + lea r3, [strideq*3] +%if ARCH_X86_64 + mova m10, [o(pixel_10bpc_max)] + pxor m9, m9 +%endif + call .round3_and_write_8x8 +.zero: +%if ARCH_X86_64 +%define mzero m9 +%else +%define mzero m7 + pxor m7, m7 +%endif + REPX {mova [cq+16*x], mzero}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +%undef mzero + RET + + ; round (rounded right-shift by 5) before writing + ; data in m0-7 + ; on x86-64, pw_2048 is in m8 + ; .round1 is for m0-7 + ; .round2 is for m0-6 & [rsp+gprsize*2] + ; .round3 is same, but without using m8 on x86-64 (.round2/3 are identical on x86-32) + ; .round4 is x86-32-only, it is similar to .round2 but with constant already in m7 +%if ARCH_X86_32 +.round1_and_write_8x8: + mova [rsp+gprsize*2], m7 +.round2_and_write_8x8: +%endif +.round3_and_write_8x8: + mova m7, [o(pw_2048)] +%if ARCH_X86_32 +.round4_and_write_8x8: +%endif + REPX {pmulhrsw x, m7}, m0, m1, m2, m3, m4, m5, m6 + pmulhrsw m7, [rsp+gprsize*2] +%if ARCH_X86_64 + jmp .write_8x8 +.round2_and_write_8x8: + mova m7, [rsp+gprsize*2] +.round1_and_write_8x8: + REPX {pmulhrsw x, m8}, m0, m1, m2, m3, m4, m5, m6, m7 +%endif + + ; m0-7 have to-be-written data [pre-rounded] + ; on x86-64, m9-10 contain a zero/pixel_max + ; on x86-32, these are runtime-generated, and [rsp+gprsize*2] is scratch + ; r0,1,3 contain dstq/strideq/stride3q + ; r5 is a scratch register +.write_8x8: + lea r5, [dstq+strideq*4] + paddw m0, [dstq+strideq*0] + paddw m1, [dstq+strideq*1] + paddw m2, [dstq+strideq*2] + paddw m3, [dstq+r3] + paddw m4, [r5 +strideq*0] + paddw m5, [r5 +strideq*1] + paddw m6, [r5 +strideq*2] + paddw m7, [r5 +r3] +%if ARCH_X86_64 + REPX {pmaxsw x, m9 }, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {pminsw x, m10}, m0, m1, m2, m3, m4, m5, m6, m7 +%else + mova [rsp+gprsize*2], m7 + pxor m7, m7 + REPX {pmaxsw x, m7}, m0, m1, m2, m3, m4, m5, m6 + pmaxsw m7, [rsp+gprsize*2] + mova [rsp+gprsize*2], m7 + mova m7, [o(pixel_10bpc_max)] + REPX {pminsw x, m7}, m0, m1, m2, m3, m4, m5, m6 + pminsw m7, [rsp+gprsize*2] +%endif + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + mova [dstq+strideq*2], m2 + mova [dstq+r3 ], m3 + mova [r5 +strideq*0], m4 + mova [r5 +strideq*1], m5 + mova [r5 +strideq*2], m6 + mova [r5 +r3 ], m7 + ret + +INV_TXFM_8X8_FN adst, dct +INV_TXFM_8X8_FN adst, adst +INV_TXFM_8X8_FN adst, flipadst +INV_TXFM_8X8_FN adst, identity, 6 + +cglobal iadst_8x8_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if ARCH_X86_32 + mov [rsp+4*16+1*gprsize], r1 +%endif + lea t0, [o(.pass1_main)] + jmp m(idct_8x8_internal_16bpc).pass1_full +.pass1_main: + call m(iadst_8x4_internal_16bpc).main_pass1 + call .round + jmp m(idct_8x8_internal_16bpc).pack_and_transpose +.round: +%if ARCH_X86_64 + pcmpeqd m8, m8 ; -1 + REPX {psubd x, m8 }, m0, m6 + REPX {pxor x, m8 }, m1, m7, m3, m5 + REPX {psrad x, 1 }, m0, m1, m6, m7 + REPX {psubd x, m8 }, m1, m7 + mova m8, [o(pd_6144)] + REPX {paddd x, m8 }, m2, m3, m4, m5 + REPX {psrad x, 13 }, m2, m3, m4, m5 +%else + mova [r3+2*16], m0 + + pcmpeqd m0, m0 ; -1 + mova m6, [o(pd_6144)] + REPX {pxor x, m0 }, m1, m7, m3, m5 + REPX {psrad x, 1 }, m1, m7 + REPX {psubd x, m0 }, m1, m7 + REPX {paddd x, m6 }, m2, m3, m4, m5 + REPX {psrad x, 13 }, m2, m3, m4, m5 + + mova m0, [r3+2*16] + psrld m6, 12 ; +1 + paddd m0, m6 + paddd m6, [r3+3*16] + REPX {psrad x, 1 }, m0, m6 +%endif + ret + +.pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call m_suffix(iadst_8x8_internal_8bpc, _ssse3).main + call m_suffix(iadst_8x8_internal_8bpc, _ssse3).main_pass2_end + lea r3, [strideq*3] +%if ARCH_X86_64 + mova m10, [o(pixel_10bpc_max)] + pxor m9, m9 +%endif + call .round3_and_write_8x8 + jmp m(idct_8x8_internal_16bpc).zero + + ; round (rounded right-shift by 5) before writing; odd registers are negated + ; data in m0-7 + ; on x86-64, pw_2048 is in m8 and pw_m2048 is in m11 + ; .round1 is for m0-7 + ; .round2 is for m0-6 & [rsp+gprsize*2] + ; .round3 is same, but without using m8 on x86-64 (.round2/3 are identical on x86-32) +%if ARCH_X86_64 +.round2_and_write_8x8: + mova m7, [rsp+gprsize*2] +.round1_and_write_8x8: + REPX {pmulhrsw x, m8 }, m0, m2, m4, m6 + REPX {pmulhrsw x, m11}, m1, m3, m5, m7 + jmp m(idct_8x8_internal_16bpc).write_8x8 +%else +.round1_and_write_8x8: + mova [rsp+gprsize*2], m7 +.round2_and_write_8x8: +%endif +.round3_and_write_8x8: + mova m7, [o(pw_2048)] + REPX {pmulhrsw x, m7}, m0, m2, m4, m6 + mova m7, [o(pw_m2048)] + REPX {pmulhrsw x, m7}, m1, m3, m5 + pmulhrsw m7, [rsp+gprsize*2] + jmp m(idct_8x8_internal_16bpc).write_8x8 + +INV_TXFM_8X8_FN flipadst, dct +INV_TXFM_8X8_FN flipadst, adst +INV_TXFM_8X8_FN flipadst, flipadst +INV_TXFM_8X8_FN flipadst, identity, 6 + +cglobal iflipadst_8x8_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if ARCH_X86_32 + mov [rsp+4*16+1*gprsize], r1 +%endif + lea t0, [o(.pass1_main)] + jmp m(idct_8x8_internal_16bpc).pass1_full +.pass1_main: + call m(iadst_8x4_internal_16bpc).main_pass1 + call m(iadst_8x8_internal_16bpc).round + ; invert registers + packssdw m7, m6 + packssdw m5, m4 + packssdw m3, m2 + packssdw m1, m0 + mova m0, m7 + mova m2, m5 + mova m4, m3 + mova m6, m1 + jmp m(idct_8x4_internal_16bpc).transpose4x8packed + +.pass2: + lea dstq, [dstq+strideq*8] + sub dstq, strideq + neg strideq + jmp m(iadst_8x8_internal_16bpc).pass2 + +INV_TXFM_8X8_FN identity, dct +INV_TXFM_8X8_FN identity, adst +INV_TXFM_8X8_FN identity, flipadst +INV_TXFM_8X8_FN identity, identity + +cglobal iidentity_8x8_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 + mova m0, [cq+0*32] + mova m1, [cq+1*32] + mova m2, [cq+2*32] + mova m3, [cq+3*32] + mova m4, [cq+4*32] + mova m5, [cq+5*32] + mova m6, [cq+6*32] + mova m7, [cq+7*32] + packssdw m0, [cq+0*32+16] + packssdw m1, [cq+1*32+16] + packssdw m2, [cq+2*32+16] + packssdw m3, [cq+3*32+16] + packssdw m4, [cq+4*32+16] + packssdw m5, [cq+5*32+16] + packssdw m6, [cq+6*32+16] + packssdw m7, [cq+7*32+16] + mova [rsp+gprsize+16*1], m6 + jmp m_suffix(idct_8x8_internal_8bpc, _ssse3).pass1_end3 + +.pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + lea r3, [strideq*3] +%if ARCH_X86_64 + mova m10, [o(pixel_10bpc_max)] + pxor m9, m9 + mova m8, [o(pw_4096)] + call m(idct_8x8_internal_16bpc).round1_and_write_8x8 +%else + mova [rsp+gprsize], m7 + mova m7, [o(pw_4096)] + call m(idct_8x8_internal_16bpc).round4_and_write_8x8 +%endif + jmp m(idct_8x8_internal_16bpc).zero + +%macro INV_TXFM_8X16_FN 2-3 2d ; type1, type2, eob_tbl_suffix +%if ARCH_X86_64 + INV_TXFM_FN %1, %2, tbl_8x16_%3, 8x16, 15, 0-16*16 +%else + INV_TXFM_FN %1, %2, tbl_8x16_%3, 8x16, 8, 0-17*16 +%endif +%ifidn %1_%2, dct_dct + imul r5d, [cq], 2896 + mov [cq], eobd ; 0 + add r5d, 2048 + sar r5d, 12 + imul r5d, 2896 + mov r3d, 4 +%if stack_size_padded > 0 + ; adjust to caller's stack allocation + add rsp, (12+ARCH_X86_64)*16 +%endif + jmp m(inv_txfm_add_dct_dct_8x8_16bpc).end +%endif +%endmacro + +INV_TXFM_8X16_FN dct, dct +INV_TXFM_8X16_FN dct, identity, v +INV_TXFM_8X16_FN dct, adst +INV_TXFM_8X16_FN dct, flipadst + +%if ARCH_X86_64 +DECLARE_REG_TMP 7 +%endif + +cglobal idct_8x16_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if WIN64 + PUSH r7 +%elif ARCH_X86_32 + mov [rsp+16*16+gprsize*1], r1 + mov [rsp+16*16+gprsize*2], r6 +%endif + lea t0, [o(m(idct_8x8_internal_16bpc).pass1_main)] +.pass1_full: +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_min)] + mova m13, [o(clip_max)] + mova m14, [o(pd_2896)] +%endif +%undef cmp + mov r6d, 4 +.zero_loop: + dec r6d + cmp eobb, byte [r5+r6] + jl .zero_loop + mov r5d, r6d + shl r5d, 4 +%if ARCH_X86_32 + ; restore pic-ptr + mov r6, [rsp+16*16+2*gprsize] + ; setup stack pointer + lea r3, [rsp+gprsize] +%endif +.loop_pass1: + mova m0, [cq+0*64+r5] + mova m1, [cq+1*64+r5] + mova m2, [cq+2*64+r5] + mova m3, [cq+3*64+r5] + mova m4, [cq+4*64+r5] + mova m5, [cq+5*64+r5] + mova m6, [cq+6*64+r5] + mova m7, [cq+7*64+r5] + call m(idct_8x4_internal_16bpc).rect2_mul + call t0 + + mova [cq+0*64+r5], m0 + mova [cq+1*64+r5], m1 + mova [cq+2*64+r5], m2 + mova [cq+3*64+r5], m3 + sub r5d, 16 + jge .loop_pass1 +%if WIN64 + POP r7 +%elif ARCH_X86_32 + mov r1, [rsp+16*16+1*gprsize] +%endif + jmp tx2q + +.pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + + ; input is in cqN*16, where N=0/4/8/12/1/5/9/13/2/6/10/14/3/7/11/15 + ; some are still pre-loaded from the final loop iteration in pass=1 + + mova m1, m2 + mova m2, [cq+ 1*16] + mova m3, [cq+ 9*16] + mova m4, [cq+ 2*16] + mova m5, [cq+10*16] + mova m6, [cq+ 3*16] + mova m7, [cq+11*16] + call m_suffix(idct_8x8_internal_8bpc, _ssse3).main + mova [rsp+gprsize+3*16], m0 + mova [rsp+gprsize+4*16], m1 + mova [rsp+gprsize+5*16], m2 + mova [rsp+gprsize+6*16], m3 + mova [rsp+gprsize+7*16], m4 + mova [rsp+gprsize+8*16], m5 + mova [rsp+gprsize+9*16], m6 + ; m7 is already stored in [rsp+gprsize+0*16] + mova m0, [cq+ 4*16] + mova m1, [cq+12*16] + mova m2, [cq+ 5*16] + mova m3, [cq+13*16] + mova m4, [cq+ 6*16] + mova m5, [cq+14*16] + mova m6, [cq+ 7*16] + mova m7, [cq+15*16] + call m_suffix(idct_16x8_internal_8bpc, _ssse3).main + + ; out0-7 is in rsp+gprsize+3-10*mmsize + ; out8-14 is in m0-6, and out15 is in m7 as well as rsp+gprsize+0*mmsize + +%if ARCH_X86_64 + mova m8, [o(pw_2048)] + mova m10, [o(pixel_10bpc_max)] + pxor m9, m9 + mov r6, dstq +%else + mov [rsp+16*16+gprsize*1], dstq +%endif + lea r3, [strideq*3] + lea dstq, [dstq+strideq*8] + call m(idct_8x8_internal_16bpc).round2_and_write_8x8 +%if ARCH_X86_64 +%define mzero m9 +%else +%define mzero m7 + pxor m7, m7 +%endif + REPX {mova [cq+x*16], mzero}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, \ + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 +%undef mzero + mova m0, [rsp+gprsize+ 3*16] + mova m1, [rsp+gprsize+ 4*16] + mova m2, [rsp+gprsize+ 5*16] + mova m3, [rsp+gprsize+ 6*16] + mova m4, [rsp+gprsize+ 7*16] + mova m5, [rsp+gprsize+ 8*16] + mova m6, [rsp+gprsize+ 9*16] + mova m7, [rsp+gprsize+10*16] +%if ARCH_X86_64 + mov dstq, r6 +%else + mov dstq, [rsp+16*16+gprsize*1] +%endif + call m(idct_8x8_internal_16bpc).round1_and_write_8x8 + RET + +INV_TXFM_8X16_FN adst, dct +INV_TXFM_8X16_FN adst, adst +INV_TXFM_8X16_FN adst, flipadst +INV_TXFM_8X16_FN adst, identity, v + +cglobal iadst_8x16_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if WIN64 + PUSH r7 +%elif ARCH_X86_32 + mov [rsp+16*16+gprsize*1], r1 + mov [rsp+16*16+gprsize*2], r6 +%endif + lea t0, [o(m(iadst_8x8_internal_16bpc).pass1_main)] + jmp m(idct_8x16_internal_16bpc).pass1_full + +.pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + mova m4, [cq+ 9*16] + mova m5, [cq+13*16] + mova [rsp+gprsize+7*16], m0 + mova [rsp+gprsize+8*16], m1 + mova [rsp+gprsize+5*16], m4 + mova [rsp+gprsize+6*16], m5 + mova m0, m2 + mova m1, m3 + mova m2, [cq+ 1*16] + mova m3, [cq+ 5*16] + mova m4, [cq+ 2*16] + mova m5, [cq+ 6*16] + mova m6, [cq+11*16] + mova m7, [cq+15*16] + mova [rsp+gprsize+ 3*16], m4 + mova [rsp+gprsize+ 4*16], m5 + mova [rsp+gprsize+ 9*16], m6 + mova [rsp+gprsize+10*16], m7 + mova m4, [cq+10*16] + mova m5, [cq+14*16] + mova m6, [cq+ 3*16] + mova m7, [cq+ 7*16] + call m_suffix(iadst_16x8_internal_8bpc, _ssse3).main + call m_suffix(iadst_16x8_internal_8bpc, _ssse3).main_pass2_end + +%if ARCH_X86_64 + mova m11, [o(pw_m2048)] + mova m8, [o(pw_2048)] + mova m10, [o(pixel_10bpc_max)] + pxor m9, m9 + mov r6, dstq +%else + mov [rsp+16*16+gprsize*1], dstq +%endif + lea r3, [strideq*3] + lea dstq, [dstq+strideq*8] + call m(iadst_8x8_internal_16bpc).round2_and_write_8x8 +%if ARCH_X86_64 +%define mzero m9 +%else +%define mzero m7 + pxor m7, m7 +%endif + REPX {mova [cq+x*16], mzero}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, \ + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 +%undef mzero + mova m0, [rsp+gprsize+ 3*16] + mova m1, [rsp+gprsize+ 4*16] + mova m2, [rsp+gprsize+ 5*16] + mova m3, [rsp+gprsize+ 6*16] + mova m4, [rsp+gprsize+ 7*16] + mova m5, [rsp+gprsize+ 8*16] + mova m6, [rsp+gprsize+ 9*16] + mova m7, [rsp+gprsize+10*16] +%if ARCH_X86_64 + mov dstq, r6 +%else + mov dstq, [rsp+16*16+gprsize*1] +%endif + call m(iadst_8x8_internal_16bpc).round1_and_write_8x8 + RET + +INV_TXFM_8X16_FN flipadst, dct +INV_TXFM_8X16_FN flipadst, adst +INV_TXFM_8X16_FN flipadst, flipadst +INV_TXFM_8X16_FN flipadst, identity, v + +cglobal iflipadst_8x16_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if WIN64 + PUSH r7 +%elif ARCH_X86_32 + mov [rsp+16*16+gprsize*1], r1 + mov [rsp+16*16+gprsize*2], r6 +%endif + lea t0, [o(m(iflipadst_8x8_internal_16bpc).pass1_main)] + jmp m(idct_8x16_internal_16bpc).pass1_full + +.pass2: + lea r3, [strideq*3] + lea r3, [r3*5] + add dstq, r3 + neg strideq + jmp m(iadst_8x16_internal_16bpc).pass2 + +INV_TXFM_8X16_FN identity, dct, h +INV_TXFM_8X16_FN identity, adst, h +INV_TXFM_8X16_FN identity, flipadst, h +INV_TXFM_8X16_FN identity, identity + +cglobal iidentity_8x16_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if WIN64 + PUSH r7 +%elif ARCH_X86_32 + mov [rsp+16*16+gprsize*1], r1 + mov [rsp+16*16+gprsize*2], r6 +%endif + lea t0, [o(m(idct_8x8_internal_16bpc).pack_and_transpose)] + jmp m(idct_8x16_internal_16bpc).pass1_full + +.pass2: +%if ARCH_X86_64 + mova m4, [o(pw_2048)] + mova m5, [o(pixel_10bpc_max)] + pxor m6, m6 + mova m7, [o(pw_1697x16)] +%endif + mov r5d, 4 + lea r3, [strideq*3] +.pass2_loop: + call .main +%if ARCH_X86_64 + call m(idct_8x4_internal_16bpc).round1_and_write_8x4 +%else + call m(idct_8x4_internal_16bpc).round2_and_write_8x4 +%endif + REPX {mova [cq+x*16], m6}, 0, 4, 8, 12, 16, 20, 24, 28 + dec r5d + jle .end + add cq, 16 + lea dstq, [dstq+strideq*4] + mova m0, [cq+ 0*16] + mova m1, [cq+ 4*16] + mova m2, [cq+ 8*16] + mova m3, [cq+12*16] + jmp .pass2_loop +.end: + RET +.main: + ; y = pmulhrsw(x, pw_1697x16); x = paddsw(x, x); x = paddsw(x, y) +%if ARCH_X86_32 + mova m7, [o(pw_1697x16)] + pmulhrsw m4, m7, m0 + pmulhrsw m5, m7, m1 + pmulhrsw m6, m7, m2 + pmulhrsw m7, m3 +%else + pmulhrsw m8, m7, m0 + pmulhrsw m9, m7, m1 + pmulhrsw m10, m7, m2 + pmulhrsw m11, m7, m3 +%endif + REPX {paddsw x, x}, m0, m1, m2, m3 +%if ARCH_X86_64 + paddsw m0, m8 + paddsw m1, m9 + paddsw m2, m10 + paddsw m3, m11 +%else + paddsw m0, m4 + paddsw m1, m5 + paddsw m2, m6 + paddsw m3, m7 +%endif + ret + +%macro INV_TXFM_16X4_FN 2 ; type1, type2 +%if ARCH_X86_64 + INV_TXFM_FN %1, %2, 0, 16x4, 16, 0-8*16 +%else + INV_TXFM_FN %1, %2, 0, 16x4, 8, 0-12*16 +%endif +%ifidn %1_%2, dct_dct + imul r5d, [cq], 2896 + mov [cq], eobd ; 0 + mov r3d, 4 +.dconly: + add r5d, 6144 + sar r5d, 13 +.dconly2: + imul r5d, 2896 + add r5d, 34816 + movd m0, r5d + pshuflw m0, m0, q1111 + punpcklqdq m0, m0 + mova m3, [o(pixel_10bpc_max)] + pxor m4, m4 +.loop: + mova m1, [dstq+ 0] + mova m2, [dstq+16] + REPX {paddw x, m0}, m1, m2 + REPX {pminsw x, m3}, m1, m2 + REPX {pmaxsw x, m4}, m1, m2 + mova [dstq+ 0], m1 + mova [dstq+16], m2 + add dstq, strideq + dec r3d + jg .loop + RET +%endif +%endmacro + +INV_TXFM_16X4_FN dct, dct +INV_TXFM_16X4_FN dct, identity +INV_TXFM_16X4_FN dct, adst +INV_TXFM_16X4_FN dct, flipadst + +cglobal idct_16x4_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_min)] + mova m13, [o(clip_max)] + mova m14, [o(pd_2896)] +%endif + ; setup stack pointer + lea r3, [rsp+gprsize] + + mova m0, [cq+ 1*16] + mova m1, [cq+ 3*16] + mova m2, [cq+ 5*16] + mova m3, [cq+ 7*16] + mova m4, [cq+ 9*16] + mova m5, [cq+11*16] + mova m6, [cq+13*16] + mova m7, [cq+15*16] + call .main_oddhalf + mova m0, [cq+ 0*16] + mova m1, [cq+ 2*16] + mova m2, [cq+ 4*16] + mova m3, [cq+ 6*16] + mova m4, [cq+ 8*16] + mova m5, [cq+10*16] + mova m6, [cq+12*16] + mova m7, [cq+14*16] + call m(idct_8x4_internal_16bpc).main_pass1 + call m(idct_8x4_internal_16bpc).round + ; t0-7 is in m0-7 + + call .round + +%if ARCH_X86_64 +.pack_transpose: + ; transpose in two parts + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + packssdw m8, m9 + packssdw m10, m11 + packssdw m12, m13 + packssdw m14, m15 +.transpose: + call m(idct_8x4_internal_16bpc).transpose4x8packed + call .transpose4x8packed_hi +%else + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [r3+0*16], m0 + mova [r3+1*16], m1 + mova [r3+2*16], m2 + mova [r3+3*16], m3 + mova m0, [r3+ 8*16] + mova m2, [r3+ 9*16] + mova m4, [r3+10*16] + mova m6, [r3+11*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed +%endif + jmp tx2q +%if ARCH_X86_64 +.transpose4x8packed_hi: + punpcklwd m9, m10, m14 + punpckhwd m10, m14 + punpckhwd m14, m8, m12 + punpcklwd m8, m12 + + punpckhwd m11, m8, m9 + punpcklwd m8, m9 + punpckhwd m12, m14, m10 + punpcklwd m14, m10 + + punpcklwd m10, m11, m12 + punpckhwd m11, m12 + punpckhwd m9, m8, m14 + punpcklwd m8, m14 + ret +%endif +.main_oddhalf_fast: ; lower half zero + pmulld m7, m0, [o(pd_4076)] + pmulld m0, [o(pd_401)] + pmulld m6, m1, [o(pd_m1189)] + pmulld m1, [o(pd_3920)] +%if ARCH_X86_32 + mova m4, [o(pd_2048)] + REPX {paddd x, m4}, m1, m6 + REPX {psrad x, 12}, m1, m6 + mova [r3+1*16], m1 +%endif + pmulld m5, m2, [o(pd_3612)] + pmulld m2, [o(pd_1931)] +%if ARCH_X86_32 + pmulld m1, m3, [o(pd_m2598)] +%else + pmulld m4, m3, [o(pd_m2598)] +%endif + pmulld m3, [o(pd_3166)] + jmp .main_oddhalf_fast2 +.main_oddhalf: +%if ARCH_X86_64 + ITX_MULSUB_2D 0, 7, 8, 9, 10, _, 401, 4076 ; t8a, t15a + ITX_MULSUB_2D 6, 1, 8, 9, 10, _, 3920, 1189 ; t11a, t12a + ITX_MULSUB_2D 2, 5, 8, 9, 10, _, 1931, 3612 ; t10a, t13a + ITX_MULSUB_2D 4, 3, 8, 9, 10, _, 3166, 2598 ; t9a, t14a +.main_oddhalf_fast2: + REPX {paddd x, m11}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {psrad x, 12 }, m0, m1, m2, m3, m4, m5, m6, m7 + psubd m8, m0, m4 ; t9 + paddd m0, m4 ; t8 + psubd m4, m6, m2 ; t10 + paddd m2, m6 ; t11 + psubd m6, m1, m5 ; t13 + paddd m5, m1 ; t12 + psubd m1, m7, m3 ; t14 + paddd m7, m3 ; t15 + REPX {pmaxsd x, m12}, m8, m1, m4, m6, m0, m2, m5, m7 + REPX {pminsd x, m13}, m8, m1, m4, m6, m0, m2, m5, m7 + mova m15, [o(pd_3784)] + mova m10, [o(pd_1567)] + ITX_MULSUB_2D 1, 8, 3, 9, _, 11, 10, 15 + ITX_MULSUB_2D 6, 4, 3, 9, _, 11, 10, 15, 4 + psubd m3, m1, m4 ; t10 + paddd m1, m4 ; t9 + psubd m4, m0, m2 ; t11a + paddd m0, m2 ; t8a + psubd m2, m8, m6 ; t13 + paddd m6, m8 ; t14 + psubd m8, m7, m5 ; t12a + paddd m7, m5 ; t15a + REPX {pmaxsd x, m12}, m2, m8, m3, m4, m0, m1, m6, m7 + REPX {pminsd x, m13}, m2, m8, m3, m4, m0, m1, m6, m7 + REPX {pmulld x, m14}, m2, m8, m3, m4 + paddd m2, m11 + paddd m8, m11 + paddd m5, m2, m3 ; t13a + psubd m2, m3 ; t10a + psubd m3, m8, m4 ; t11 + paddd m4, m8 ; t12 + REPX {psrad x, 12}, m5, m2, m3, m4 + mova [r3+0*16], m0 + mova [r3+1*16], m1 + mova [r3+2*16], m2 + mova [r3+3*16], m3 + mova [r3+4*16], m4 + mova [r3+5*16], m5 + mova [r3+6*16], m6 + mova [r3+7*16], m7 +%else + mova [r3+0*16], m2 + mova [r3+1*16], m3 + mova [r3+2*16], m4 + mova [r3+3*16], m5 + mova m4, [o(pd_2048)] + + ITX_MULSUB_2D 0, 7, 2, 3, 5, _, 401, 4076 ; t8a, t15a + ITX_MULSUB_2D 6, 1, 2, 3, 5, 4, 3920, 1189 ; t11a, t12a + + mova m2, [r3+0*16] + mova m3, [r3+1*16] + mova [r3+0*16], m0 + mova [r3+1*16], m1 + mova m1, [r3+2*16] + mova m5, [r3+3*16] + mova [r3+2*16], m6 + mova [r3+3*16], m7 + + ITX_MULSUB_2D 2, 5, 0, 6, 7, _, 1931, 3612 ; t10a, t13a + ITX_MULSUB_2D 1, 3, 0, 6, 7, _, 3166, 2598 ; t9a, t14a + + mova m0, [r3+0*16] + mova m6, [r3+2*16] + mova m7, [r3+3*16] +.main_oddhalf_fast2: + REPX {paddd x, m4}, m0, m7, m2, m5, m1, m3 + REPX {psrad x, 12}, m0, m7, m2, m5, m1, m3 + psubd m4, m0, m1 ; t9 + paddd m0, m1 ; t8 + mova m1, [r3+1*16] + mova [r3+0*16], m4 + psubd m4, m6, m2 ; t10 + paddd m2, m6 ; t11 + psubd m6, m1, m5 ; t13 + paddd m5, m1 ; t12 + psubd m1, m7, m3 ; t14 + paddd m7, m3 ; t15 + mova m3, [o(clip_min)] + REPX {pmaxsd x, m3}, m1, m4, m6, m0, m2, m5, m7 + pmaxsd m3, [r3+0*16] + mova [r3+0*16], m3 + mova m3, [o(clip_max)] + REPX {pminsd x, m3}, m1, m4, m6, m0, m2, m5, m7 + pminsd m3, [r3+0*16] + mova [r3+0*16], m0 + mova [r3+1*16], m2 + mova [r3+2*16], m5 + mova [r3+3*16], m7 + mova m7, [o(pd_2048)] + ITX_MULSUB_2D 1, 3, 0, 2, 5, 7, 1567, 3784 + ITX_MULSUB_2D 6, 4, 0, 2, _, 7, 5, 3784, 4 + mova m0, [r3+0*16] + mova m2, [r3+1*16] + psubd m5, m1, m4 ; t10 + mova [r3+1*16], m5 + paddd m1, m4 ; t9 + psubd m4, m0, m2 ; t11a + paddd m0, m2 ; t8a + mova m5, [r3+2*16] + mova m7, [r3+3*16] + psubd m2, m3, m6 ; t13 + paddd m6, m3 ; t14 + paddd m3, m7, m5 ; t15a + psubd m7, m5 ; t12a + mova [r3+0*16], m3 + mova m3, [r3+1*16] + mova m5, [o(clip_min)] + REPX {pmaxsd x, m5}, m2, m7, m3, m4, m0, m1, m6 + pmaxsd m5, [r3+0*16] + mova [r3+0*16], m5 + mova m5, [o(clip_max)] + REPX {pminsd x, m5}, m2, m7, m3, m4, m0, m1, m6 + pminsd m5, [r3+0*16] + mova [r3+0*16], m5 + mova m5, [o(pd_2896)] + REPX {pmulld x, m5}, m2, m7, m3, m4 + mova m5, [o(pd_2048)] + REPX {paddd x, m5}, m2, m7 + paddd m5, m2, m3 ; t13a + psubd m2, m3 ; t10a + psubd m3, m7, m4 ; t11 + paddd m4, m7 ; t12 + REPX {psrad x, 12}, m5, m2, m3, m4 + mova m7, [r3+0*16] + mova [r3+11*16], m0 + mova [r3+10*16], m1 + mova [r3+9*16], m2 + mova [r3+8*16], m3 + mova [r3+7*16], m4 + mova [r3+6*16], m5 + mova [r3+5*16], m6 + mova [r3+4*16], m7 +%endif + ret +.round: +%if ARCH_X86_64 + pcmpeqd m8, m8 + REPX {psubd x, m8}, m0, m1, m2, m3, m4, m5, m6, m7 + mova m8, [r3+1*16] + mova m9, [r3+2*16] + mova m10, [r3+3*16] + mova m11, [r3+4*16] + mova m12, [r3+5*16] + mova m13, [r3+6*16] + mova m14, [r3+7*16] + psubd m15, m0, m14 ; out15 + paddd m0, m14 ; out0 + psubd m14, m1, m13 ; out14 + paddd m1, m13 ; out1 + psubd m13, m2, m12 ; out13 + paddd m2, m12 ; out2 + psubd m12, m3, m11 ; out12 + paddd m3, m11 ; out3 + psubd m11, m4, m10 ; out11 + paddd m4, m10 ; out4 + psubd m10, m5, m9 ; out10 + paddd m5, m9 ; out5 + psubd m9, m6, m8 ; out9 + paddd m6, m8 ; out6 + psubd m8, m7, [r3+0*16] ; out8 + paddd m7, [r3+0*16] ; out7 + REPX {psrad x, 1}, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14, m15 + ; and out0-15 is now in m0-15 +%else + mova [r3+ 0*16], m0 + pcmpeqd m0, m0 + REPX {psubd x, m0}, m1, m2, m3, m4, m5, m6, m7 + mova [r3+ 1*16], m1 + mova [r3+ 2*16], m2 + mova m1, [r3+ 0*16] + psubd m1, m0 + mova [r3+ 0*16], m1 + mova m1, [r3+11*16] + mova m2, [r3+10*16] + psubd m0, m7, m1 + paddd m7, m1 + psubd m1, m6, m2 + paddd m6, m2 + REPX {psrad x, 1}, m0, m1, m6, m7 + packssdw m0, m1 ; out8-9 + packssdw m6, m7 ; out6-7 + mova [r3+11*16], m6 + mova m1, [r3+9*16] + mova m7, [r3+8*16] + psubd m2, m5, m1 + paddd m5, m1 + psubd m1, m4, m7 + paddd m4, m7 + REPX {psrad x, 1}, m2, m1, m4, m5 + packssdw m2, m1 ; out10-11 + packssdw m4, m5 ; out4-5 + mova m1, [r3+2*16] + mova [r3+10*16], m4 + mova m6, [r3+7*16] + mova m7, [r3+6*16] + psubd m4, m3, m6 + paddd m3, m6 + psubd m6, m1, m7 + paddd m1, m7 + REPX {psrad x, 1}, m4, m6, m1, m3 + packssdw m4, m6 ; out12-13 + packssdw m1, m3 ; out2-3 + mova m3, [r3+1*16] + mova [r3+9*16], m1 + mova m1, [r3+0*16] + mova m5, [r3+5*16] + mova m7, [r3+4*16] + psubd m6, m3, m5 + paddd m3, m5 + psubd m5, m1, m7 + paddd m1, m7 + REPX {psrad x, 1}, m6, m5, m1, m3 + packssdw m6, m5 ; out14-15 + packssdw m1, m3 ; out0-1 + mova [r3+8*16], m1 +%endif + ret + +.pass2: + lea r4, [o(m_suffix(idct_8x4_internal_8bpc, _ssse3).main)] +.pass2_loop: + lea r3, [strideq*3] +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call r4 + call m(idct_8x4_internal_16bpc).round2_and_write_8x4 + REPX {mova [cq+x*16], m6}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +%if ARCH_X86_64 + mova m0, m8 + mova m1, m9 + mova m2, m10 + mova m3, m11 +%else + mova m0, [rsp+gprsize+0*16] + mova m1, [rsp+gprsize+1*16] + mova m2, [rsp+gprsize+2*16] + mova m3, [rsp+gprsize+3*16] +%endif + add dstq, 16 +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call r4 + call m(idct_8x4_internal_16bpc).round2_and_write_8x4 + RET + +INV_TXFM_16X4_FN adst, dct +INV_TXFM_16X4_FN adst, adst +INV_TXFM_16X4_FN adst, flipadst +INV_TXFM_16X4_FN adst, identity + +cglobal iadst_16x4_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 + ; setup stack pointer + lea r3, [rsp+gprsize] + call .main +%if ARCH_X86_64 + jmp m(idct_16x4_internal_16bpc).pack_transpose +%else + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [rsp+gprsize+0*16], m0 + mova [rsp+gprsize+1*16], m1 + mova [rsp+gprsize+2*16], m2 + mova [rsp+gprsize+3*16], m3 + mova m0, [rsp+gprsize+ 8*16] + mova m2, [rsp+gprsize+ 9*16] + mova m4, [rsp+gprsize+10*16] + mova m6, [rsp+gprsize+11*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed + jmp tx2q +%endif + +.main: +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_min)] + mova m13, [o(clip_max)] + mova m14, [o(pd_2896)] +%endif + mova m0, [cq+ 2*16] + mova m1, [cq+13*16] + mova m2, [cq+ 6*16] + mova m3, [cq+ 9*16] + mova m4, [cq+10*16] + mova m5, [cq+ 5*16] + mova m6, [cq+14*16] + mova m7, [cq+ 1*16] + call .main_part1 + mova m0, [cq+ 0*16] + mova m1, [cq+15*16] + mova m2, [cq+ 4*16] + mova m3, [cq+11*16] + mova m4, [cq+ 8*16] + mova m5, [cq+ 7*16] + mova m6, [cq+12*16] + mova m7, [cq+ 3*16] + call .main_part2 +.round: +%if ARCH_X86_64 + mova m15, [o(pd_6144)] + psrld m14, 11 ; pd_1 + pcmpeqd m8, m8 ; -1 + psubd m13, m15, m14 ; pd_6143 + REPX {paddd x, m14}, m0, m2 + REPX {paddd x, m15}, m4, m6 + REPX {pxor x, m8 }, m1, m3, m5, m7 + REPX {psrad x, 1 }, m1, m3 + REPX {paddd x, m15}, m5, m7 + REPX {psubd x, m8 }, m1, m3 + paddd m8, m15, m9 + psubd m9, m13, m10 + paddd m10, m15, m11 + psubd m11, m13, m12 + paddd m12, m14, [r3+3*16] + psubd m13, m14, [r3+2*16] + psubd m15, m14, [r3+0*16] + paddd m14, [r3+1*16] + REPX {psrad x, 1 }, m0, m2, m12, m13, m14, m15 + REPX {psrad x, 13}, m4, m5, m6, m7, m8, m9, m10, m11 +%else + mova [r3+8*16], m1 + mova [r3+9*16], m3 + mova m3, [o(pd_6144)] + pcmpeqd m1, m1 + REPX {pxor x, m1}, m5, m7 + REPX {paddd x, m3}, m4, m5, m6, m7 + REPX {psrad x, 13}, m4, m5, m6, m7 + packssdw m4, m5 + packssdw m6, m7 + mova [r3+10*16], m4 + mova [r3+11*16], m6 + mova m4, [r3+4*16] + mova m5, [r3+5*16] + mova m6, [r3+6*16] + mova m7, [r3+7*16] + REPX {pxor x, m1}, m5, m7 + REPX {psubd x, m1}, m4, m6 + REPX {psrad x, 1 }, m4, m5, m6, m7 + REPX {psubd x, m1}, m5, m7 + packssdw m4, m5 + packssdw m6, m7 + mova m5, [r3+8*16] + mova m7, [r3+9*16] + mova [r3+8*16], m4 + mova [r3+9*16], m6 + REPX {pxor x, m1}, m5, m7 + REPX {paddd x, m3}, m0, m5, m2, m7 + REPX {psrad x, 13}, m0, m5, m2, m7 + packssdw m0, m5 + packssdw m2, m7 + mova m4, [r3+0*16] + mova m5, [r3+1*16] + mova m6, [r3+2*16] + mova m7, [r3+3*16] + REPX {psubd x, m1}, m4, m6 + REPX {pxor x, m1}, m5, m7 + REPX {psrad x, 1 }, m4, m5, m6, m7 + REPX {psubd x, m1}, m5, m7 + packssdw m4, m5 + packssdw m6, m7 +%endif + ret + +.main_part2: +%if ARCH_X86_64 + ITX_MULSUB_2D 1, 0, 8, 9, 10, 11, 201, 4091 + ITX_MULSUB_2D 3, 2, 8, 9, 10, 11, 1751, 3703 + ITX_MULSUB_2D 5, 4, 8, 9, 10, 11, 3035, 2751 + ITX_MULSUB_2D 7, 6, 8, 9, 10, 11, 3857, 1380 + psubd m8, m0, m4 ; t8a + paddd m0, m4 ; t0a + psubd m4, m1, m5 ; t9a + paddd m1, m5 ; t1a + psubd m5, m2, m6 ; t12a + paddd m2, m6 ; t4a + psubd m6, m3, m7 ; t13a + paddd m7, m3 ; t5a + REPX {pmaxsd x, m12}, m8, m4, m5, m6, m0, m1, m2, m7 + REPX {pminsd x, m13}, m8, m4, m5, m6, m0, m1, m2, m7 + mova m15, [o(pd_4017)] + mova m10, [o(pd_799)] + ITX_MULSUB_2D 8, 4, 3, 9, _, 11, 10, 15 + ITX_MULSUB_2D 6, 5, 3, 9, _, 11, 15, 10 + psubd m3, m0, m2 ; t4 + paddd m0, m2 ; t0 + psubd m2, m1, m7 ; t5 + paddd m1, m7 ; t1 + psubd m7, m4, m6 ; t12a + paddd m4, m6 ; t8a + psubd m6, m8, m5 ; t13a + paddd m5, m8 ; t9a + REPX {pmaxsd x, m12}, m3, m2, m7, m6, m0, m1, m4, m5 + REPX {pminsd x, m13}, m3, m2, m7, m6, m0, m1, m4, m5 + mova m15, [o(pd_3784)] + mova m10, [o(pd_1567)] + ITX_MULSUB_2D 3, 2, 8, 9, _, 11, 10, 15 + ITX_MULSUB_2D 7, 6, 8, 9, _, 11, 10, 15 + mova m10, [r3+0*16] ; t2 + mova m8, [r3+1*16] ; t3 + psubd m9, m0, m10 ; t2a + paddd m0, m10 ; out0 + psubd m10, m1, m8 ; t3a + paddd m1, m8 ; -out15 + mova [r3+0*16], m1 + mova m15, [r3+3*16] ; t7a + mova m1, [r3+2*16] ; t6a + psubd m8, m3, m15 ; t7 + paddd m15, m3 ; out12 + paddd m3, m2, m1 ; -out3 + psubd m2, m1 ; t6 + mova [r3+3*16], m15 + mova [r3+1*16], m2 + mova m1, [r3+7*16] ; t15 + mova m2, [r3+6*16] ; t14 + paddd m15, m7, m1 ; -out13 + psubd m7, m1 ; t15a + psubd m11, m6, m2 ; t14a + paddd m2, m6 ; out2 + mova [r3+2*16], m15 + mova m1, [r3+4*16] ; t10a + mova m15, [r3+5*16] ; t11a + psubd m6, m4, m1 ; t10 + paddd m1, m4 ; -out1 + psubd m4, m5, m15 ; t11 + paddd m5, m15 ; out14 + REPX {pmaxsd x, m12}, m11, m7, m9, m10, m6, m4, m8 + pmaxsd m12, [r3+1*16] ; t6 + mova [r3+1*16], m5 + REPX {pminsd x, m13}, m11, m7, m9, m10, m6, m4, m12, m8 + REPX {pmulld x, m14}, m11, m7, m9, m10, m6, m4, m12, m8 + paddd m5, m11, m7 ; -out5 (unshifted) + psubd m11, m7 ; out10 (unshifted) + paddd m7, m9, m10 ; -out7 (unshifted) + psubd m9, m10 ; out8 (unshifted) + psubd m10, m6, m4 ; -out9 (unshifted) + paddd m6, m4 ; out6 (unshifted) + paddd m4, m12, m8 ; out4 (unshifted) + psubd m12, m8 ; -out11 (unshifted) +%else + mova [r3+8*16], m0 + mova [r3+9*16], m1 + mova [r3+10*16], m2 + mova [r3+11*16], m3 + mova m3, [o(pd_2048)] + ITX_MULSUB_2D 5, 4, 0, 1, 2, 3, 3035, 2751 + ITX_MULSUB_2D 7, 6, 0, 1, 2, 3, 3857, 1380 + mova m0, [r3+8*16] + mova m1, [r3+9*16] + mova [r3+8*16], m4 + mova m4, [r3+10*16] + mova [r3+9*16], m5 + mova [r3+10*16], m6 + mova m5, [r3+11*16] + mova [r3+11*16], m7 + ITX_MULSUB_2D 1, 0, 2, 6, 7, 3, 201, 4091 + ITX_MULSUB_2D 5, 4, 2, 6, 7, 3, 1751, 3703 + mova m2, [r3+8*16] + mova m6, [r3+9*16] + psubd m3, m0, m2 ; t8a + paddd m0, m2 ; t0a + mova [r3+8*16], m3 + psubd m2, m1, m6 ; t9a + paddd m1, m6 ; t1a + mova m3, [r3+10*16] + psubd m6, m4, m3 ; t12a + paddd m4, m3 ; t4a + mova m3, [r3+11*16] + psubd m7, m5, m3 ; t13a + paddd m5, m3 ; t5a + mova m3, [o(clip_min)] + REPX {pmaxsd x, m3}, m2, m6, m7, m0, m1, m4, m5 + pmaxsd m3, [r3+8*16] + mova [r3+8*16], m3 + mova m3, [o(clip_max)] + REPX {pminsd x, m3}, m2, m6, m7, m0, m1, m4, m5 + pminsd m3, [r3+8*16] + mova [r3+8*16], m3 + psubd m3, m0, m4 ; t4 + paddd m0, m4 ; t0 + psubd m4, m1, m5 ; t5 + paddd m1, m5 ; t1 + mova m5, [o(pd_2048)] + mova [r3+9*16], m1 + mova [r3+10*16], m4 + mova [r3+11*16], m3 + mova m3, [r3+8*16] + mova [r3+8*16], m0 + ITX_MULSUB_2D 3, 2, 0, 1, 4, 5, 799, 4017 + ITX_MULSUB_2D 7, 6, 0, 1, 4, 5, 4017, 4 + psubd m5, m2, m7 ; t12a + paddd m2, m7 ; t8a + psubd m7, m3, m6 ; t13a + paddd m6, m3 ; t9a + mova m0, [r3+8*16] + mova m1, [r3+9*16] + mova m4, [r3+10*16] + mova m3, [o(clip_min)] + REPX {pmaxsd x, m3}, m4, m5, m7, m0, m1, m2, m6 + pmaxsd m3, [r3+11*16] + mova [r3+8*16], m3 + mova m3, [o(clip_max)] + REPX {pminsd x, m3}, m4, m5, m7, m0, m1, m2, m6 + pminsd m3, [r3+8*16] + mova [r3+8*16], m0 + mova [r3+9*16], m1 + mova [r3+10*16], m2 + mova [r3+11*16], m6 + mova m0, [o(pd_2048)] + ITX_MULSUB_2D 3, 4, 1, 2, 6, 0, 1567, 3784 + ITX_MULSUB_2D 5, 7, 1, 2, 6, 0, 6, 3784 + mova m0, [r3+7*16] ; t7a + mova m2, [r3+6*16] ; t6a + psubd m1, m3, m0 ; t7 + paddd m0, m3 ; out12 + paddd m3, m4, m2 ; -out3 + psubd m4, m2 ; t6 + mova [r3+7*16], m3 + mova m3, [r3+3*16] ; t15 + mova m2, [r3+2*16] ; t14 + paddd m6, m5, m3 ; -out13 + psubd m5, m3 ; t15a + psubd m3, m7, m2 ; t14a + paddd m2, m7 ; out2 + mova [r3+6*16], m2 + mova m7, [r3+0*16] ; t10a + mova m2, [r3+1*16] ; t11a + mova [r3+0*16], m0 + mova [r3+1*16], m6 + mova m6, [r3+11*16] + psubd m0, m6, m2 ; t11 + paddd m6, m2 ; out14 + mova [r3+2*16], m6 + mova m2, [r3+10*16] + psubd m6, m2, m7 ; t10 + paddd m2, m7 ; -out1 + mova m7, [r3+5*16] ; t3 + mova [r3+5*16], m2 + mova [r3+10*16], m1 + mova m1, [r3+9*16] + psubd m2, m1, m7 ; t3a + paddd m1, m7 ; -out15 + mova [r3+3*16], m1 + mova m1, [r3+4*16] ; t2 + mova m7, [r3+8*16] + psubd m7, m1 ; t2a + paddd m1, [r3+8*16] ; out0 + mova [r3+4*16], m1 + mova m1, [o(clip_min)] + REPX {pmaxsd x, m1}, m0, m2, m3, m4, m5, m6, m7 + pmaxsd m1, [r3+10*16] + mova [r3+10*16], m1 + mova m1, [o(clip_max)] + REPX {pminsd x, m1}, m0, m2, m3, m4, m5, m6, m7 + pminsd m1, [r3+10*16] + mova [r3+10*16], m1 + mova m1, [o(pd_2896)] + REPX {pmulld x, m1}, m0, m2, m3, m4, m5, m6, m7 + pmulld m1, [r3+10*16] + mova [r3+11*16], m3 + psubd m3, m4, m1 ; -out11 (unshifted) + paddd m4, m1 ; out4 (unshifted) + psubd m1, m6, m0 ; -out9 (unshifted) + paddd m6, m0 ; out6 (unshifted) + psubd m0, m7, m2 ; out8 (unshifted) + paddd m7, m2 ; -out7 (unshifted) + mova m2, [r3+11*16] + mova [r3+11*16], m5 + paddd m5, m2 ; -out5 (unshifted) + psubd m2, [r3+11*16] ; out10 (unshifted) + ; m0-3 contain out8-11 (unshifted), m4-7 contain out4-7 (unshifted) + ; r[-4,3] contain out0-3 and out12-15 +%endif + ret +.main_part1: +%if ARCH_X86_64 + ITX_MULSUB_2D 1, 0, 8, 9, 10, 11, 995, 3973 + ITX_MULSUB_2D 3, 2, 8, 9, 10, 11, 2440, 3290 + ITX_MULSUB_2D 5, 4, 8, 9, 10, 11, 3513, 2106 + ITX_MULSUB_2D 7, 6, 8, 9, 10, 11, 4052, 601 + psubd m8, m0, m4 ; t10a + paddd m0, m4 ; t2a + psubd m4, m1, m5 ; t11a + paddd m1, m5 ; t3a + psubd m5, m2, m6 ; t14a + paddd m2, m6 ; t6a + psubd m6, m3, m7 ; t15a + paddd m7, m3 ; t7a + REPX {pmaxsd x, m12}, m8, m4, m5, m6, m0, m1, m2, m7 + REPX {pminsd x, m13}, m8, m4, m5, m6, m0, m1, m2, m7 + mova m15, [o(pd_2276)] + mova m10, [o(pd_3406)] + ITX_MULSUB_2D 8, 4, 3, 9, _, 11, 10, 15 + ITX_MULSUB_2D 6, 5, 3, 9, _, 11, 15, 10 + psubd m3, m0, m2 ; t6 + paddd m0, m2 ; t2 + psubd m2, m1, m7 ; t7 + paddd m1, m7 ; t3 + psubd m7, m4, m6 ; t14a + paddd m4, m6 ; t10a + psubd m6, m8, m5 ; t15a + paddd m5, m8 ; t11a + REPX {pmaxsd x, m12}, m3, m2, m7, m6, m0, m1, m4, m5 + REPX {pminsd x, m13}, m3, m2, m7, m6, m0, m1, m4, m5 + mova m15, [o(pd_1567)] + mova m10, [o(pd_3784)] + ITX_MULSUB_2D 2, 3, 8, 9, _, 11, 10, 15 + ITX_MULSUB_2D 6, 7, 8, 9, _, 11, 10, 15 + mova [r3+0*16], m0 + mova [r3+1*16], m1 + mova [r3+4*16], m4 + mova [r3+5*16], m5 + mova [r3+2*16], m2 + mova [r3+3*16], m3 + mova [r3+6*16], m6 + mova [r3+7*16], m7 +%else + mova [r3+4*16], m0 + mova [r3+5*16], m1 + mova [r3+6*16], m2 + mova [r3+7*16], m3 + mova m3, [o(pd_2048)] + ITX_MULSUB_2D 5, 4, 0, 1, 2, 3, 3513, 2106 + ITX_MULSUB_2D 7, 6, 0, 1, 2, 3, 4052, 601 + mova [r3+0*16], m4 + mova [r3+1*16], m5 + mova [r3+2*16], m6 + mova [r3+3*16], m7 + mova m0, [r3+4*16] + mova m1, [r3+5*16] + mova m2, [r3+6*16] + mova m7, [r3+7*16] + ITX_MULSUB_2D 1, 0, 4, 5, 6, 3, 995, 3973 + ITX_MULSUB_2D 7, 2, 4, 5, 6, 3, 2440, 3290 + mova m4, [r3+0*16] + mova m5, [r3+1*16] + psubd m6, m0, m4 ; t10a + paddd m0, m4 ; t2a + mova [r3+4*16], m6 + mova m6, [r3+2*16] + mova m3, [r3+3*16] + psubd m4, m1, m5 ; t11a + paddd m1, m5 ; t3a + psubd m5, m2, m6 ; t14a + paddd m2, m6 ; t6a + psubd m6, m7, m3 ; t15a + paddd m7, m3 ; t7a + mova m3, [o(clip_min)] + REPX {pmaxsd x, m3}, m4, m5, m6, m0, m1, m2, m7 + pmaxsd m3, [r3+4*16] + mova [r3+4*16], m3 + mova m3, [o(clip_max)] + REPX {pminsd x, m3}, m4, m5, m6, m0, m1, m2, m7 + pminsd m3, [r3+4*16] + mova [r3+4*16], m3 + psubd m3, m0, m2 ; t6 + paddd m0, m2 ; t2 + psubd m2, m1, m7 ; t7 + paddd m1, m7 ; t3 + mova [r3+5*16], m1 + mova [r3+6*16], m3 + mova [r3+7*16], m2 + mova m1, [r3+4*16] + mova [r3+4*16], m0 + mova m3, [o(pd_2048)] + ITX_MULSUB_2D 1, 4, 0, 7, 2, 3, 3406, 2276 + ITX_MULSUB_2D 6, 5, 0, 7, 2, 3, 2276, 2 + psubd m7, m4, m6 ; t14a + paddd m4, m6 ; t10a + psubd m6, m1, m5 ; t15a + paddd m5, m1 ; t11a + mova m1, [r3+5*16] + mova m3, [r3+6*16] + mova m2, [r3+7*16] + mova m0, [o(clip_min)] + REPX {pmaxsd x, m0}, m3, m2, m7, m6, m1, m4, m5 + pmaxsd m0, [r3+4*16] + mova [r3+4*16], m0 + mova m0, [o(clip_max)] + REPX {pminsd x, m0}, m3, m2, m7, m6, m1, m4, m5 + pminsd m0, [r3+4*16] + mova [r3+4*16], m0 + mova [r3+5*16], m1 + mova [r3+0*16], m4 + mova [r3+1*16], m5 + mova m0, [o(pd_2048)] + ITX_MULSUB_2D 2, 3, 1, 4, 5, 0, 3784, 1567 + ITX_MULSUB_2D 6, 7, 1, 4, 5, 0, 5, 1567 + mova [r3+6*16], m2 + mova [r3+7*16], m3 + mova [r3+2*16], m6 + mova [r3+3*16], m7 +%endif + ret + +.pass2: + lea r4, [o(m_suffix(iadst_8x4_internal_8bpc, _ssse3).main)] + jmp m(idct_16x4_internal_16bpc).pass2_loop + +INV_TXFM_16X4_FN flipadst, dct +INV_TXFM_16X4_FN flipadst, adst +INV_TXFM_16X4_FN flipadst, flipadst +INV_TXFM_16X4_FN flipadst, identity + +cglobal iflipadst_16x4_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 + lea r3, [rsp+gprsize] + call m(iadst_16x4_internal_16bpc).main +%if ARCH_X86_64 + packssdw m1, m0 + packssdw m3, m2 + packssdw m5, m4 + packssdw m7, m6 + packssdw m9, m8 + packssdw m11, m10 + packssdw m13, m12 + packssdw m15, m14 + mova m0, m15 + mova m2, m13 + mova m4, m11 + mova m6, m9 + mova m8, m7 + mova m10, m5 + mova m12, m3 + mova m14, m1 + jmp m(idct_16x4_internal_16bpc).transpose +%else + mova [rsp+gprsize+4*16], m0 + mova [rsp+gprsize+5*16], m2 + mova [rsp+gprsize+6*16], m4 + mova [rsp+gprsize+7*16], m6 + pshufd m6, [rsp+gprsize+ 8*16], q1032 + pshufd m4, [rsp+gprsize+ 9*16], q1032 + pshufd m2, [rsp+gprsize+10*16], q1032 + pshufd m0, [rsp+gprsize+11*16], q1032 + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [rsp+gprsize+0*16], m0 + mova [rsp+gprsize+1*16], m1 + mova [rsp+gprsize+2*16], m2 + mova [rsp+gprsize+3*16], m3 + pshufd m6, [rsp+gprsize+ 4*16], q1032 + pshufd m4, [rsp+gprsize+ 5*16], q1032 + pshufd m2, [rsp+gprsize+ 6*16], q1032 + pshufd m0, [rsp+gprsize+ 7*16], q1032 + call m(idct_8x4_internal_16bpc).transpose4x8packed + jmp tx2q +%endif + +.pass2: + lea r3, [strideq*3] + lea dstq, [dstq+r3] + neg strideq + lea r4, [o(m_suffix(iadst_8x4_internal_8bpc, _ssse3).main)] + jmp m(idct_16x4_internal_16bpc).pass2_loop + +INV_TXFM_16X4_FN identity, dct +INV_TXFM_16X4_FN identity, adst +INV_TXFM_16X4_FN identity, flipadst +INV_TXFM_16X4_FN identity, identity + +cglobal iidentity_16x4_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if ARCH_X86_64 + mova m15, [o(pd_11586)] + pmulld m0, m15, [cq+ 0*16] + pmulld m1, m15, [cq+ 1*16] + pmulld m2, m15, [cq+ 2*16] + pmulld m3, m15, [cq+ 3*16] + pmulld m4, m15, [cq+ 4*16] + pmulld m5, m15, [cq+ 5*16] + pmulld m6, m15, [cq+ 6*16] + pmulld m7, m15, [cq+ 7*16] + pmulld m8, m15, [cq+ 8*16] + pmulld m9, m15, [cq+ 9*16] + pmulld m10, m15, [cq+10*16] + pmulld m11, m15, [cq+11*16] + pmulld m12, m15, [cq+12*16] + pmulld m13, m15, [cq+13*16] + pmulld m14, m15, [cq+14*16] + pmulld m15, [cq+15*16] + mova [cq+ 0*16], m15 + mova m15, [o(pd_6144)] + REPX {paddd x, m15}, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14 + paddd m15, [cq+ 0*16] + REPX {psrad x, 13 }, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14, m15 + jmp m(idct_16x4_internal_16bpc).pack_transpose +%else + add cq, 8*16 + mov r5d, 2 +.loop_pass1: + mova m7, [o(pd_11586)] + pmulld m0, m7, [cq+0*16] + pmulld m1, m7, [cq+1*16] + pmulld m2, m7, [cq+2*16] + pmulld m3, m7, [cq+3*16] + pmulld m4, m7, [cq+4*16] + pmulld m5, m7, [cq+5*16] + pmulld m6, m7, [cq+6*16] + pmulld m7, [cq+7*16] + mova [cq+7*16], m7 + mova m7, [o(pd_6144)] + REPX {paddd x, m7}, m0, m1, m2, m3, m4, m5, m6 + paddd m7, [cq+7*16] + REPX {psrad x, 13}, m0, m1, m2, m3, m4, m5, m6, m7 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + call m(idct_8x4_internal_16bpc).transpose4x8packed + dec r5d + jz .end_pass1 + mova [rsp+gprsize+0*16], m0 + mova [rsp+gprsize+1*16], m1 + mova [rsp+gprsize+2*16], m2 + mova [rsp+gprsize+3*16], m3 + sub cq, 8*16 + jmp .loop_pass1 +.end_pass1: + jmp tx2q +%endif + +.pass2: +%if ARCH_X86_64 + mova m12, [o(pw_1697x8)] +%endif + lea r4, [o(.main)] + jmp m(idct_16x4_internal_16bpc).pass2_loop +.main: +%if ARCH_X86_64 + pmulhrsw m4, m0, m12 + pmulhrsw m5, m1, m12 + pmulhrsw m6, m2, m12 + pmulhrsw m7, m3, m12 +%else + mova m7, [o(pw_1697x8)] + pmulhrsw m4, m0, m7 + pmulhrsw m5, m1, m7 + pmulhrsw m6, m2, m7 + pmulhrsw m7, m3 +%endif + paddsw m0, m4 + paddsw m1, m5 + paddsw m2, m6 + paddsw m3, m7 + ret + +%macro INV_TXFM_16X8_FN 2-3 0 ; type1, type2, eob_offset +%if ARCH_X86_64 + INV_TXFM_FN %1, %2, %3, 16x8, 16, 0-8*16 +%else + INV_TXFM_FN %1, %2, %3, 16x8, 8, 0-13*16 +%endif +%ifidn %1_%2, dct_dct + imul r5d, [cq], 2896 + mov [cq], eobd ; 0 + mov r3d, 8 + add r5d, 2048 + sar r5d, 12 + imul r5d, 2896 +%if ARCH_X86_32 + add rsp, 1*16 +%endif + jmp m(inv_txfm_add_dct_dct_16x4_16bpc).dconly +%endif +%endmacro + +INV_TXFM_16X8_FN dct, dct +INV_TXFM_16X8_FN dct, identity, 6 +INV_TXFM_16X8_FN dct, adst +INV_TXFM_16X8_FN dct, flipadst + +cglobal idct_16x8_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if ARCH_X86_64 + DECLARE_REG_TMP 6, 4, 6 +%else + mov [rsp+gprsize+12*16], r1 + DECLARE_REG_TMP 1, 4, 3 +%endif + lea t0, [o(.main)] +.loop_main: +%undef cmp +%if ARCH_X86_64 + xor r5d, r5d + cmp eobd, 10 + setge r5b +%else + mov r5d, 1 + cmp eobd, 10 + sbb r5d, 0 +%endif + shl r5d, 4 + + lea r3, [rsp+gprsize] +.loop_pass1: + call t0 +%if ARCH_X86_64 + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [cq+4*32+r5], m8 + mova [cq+5*32+r5], m9 + mova [cq+6*32+r5], m10 + mova [cq+7*32+r5], m11 +%else + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [cq+4*32+r5], m0 + mova [cq+5*32+r5], m1 + mova [cq+6*32+r5], m2 + mova [cq+7*32+r5], m3 + mova m0, [rsp+gprsize+ 8*16] + mova m2, [rsp+gprsize+ 9*16] + mova m4, [rsp+gprsize+10*16] + mova m6, [rsp+gprsize+11*16] +%endif + call m(idct_8x4_internal_16bpc).transpose4x8packed + pxor m7, m7 + REPX {mova [cq+x*32+r5], m7}, 8, 9, 10, 11, 12, 13, 14, 15 + test r5d, r5d + jz .end + mova [cq+0*32+r5], m0 + mova [cq+1*32+r5], m1 + mova [cq+2*32+r5], m2 + mova [cq+3*32+r5], m3 + xor r5d, r5d + jmp .loop_pass1 +.end: + + jmp tx2q +.main: +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_min)] + mova m13, [o(clip_max)] + mova m14, [o(pd_2896)] +%endif + mova m0, [cq+ 1*32+r5] + mova m1, [cq+ 3*32+r5] + mova m2, [cq+ 5*32+r5] + mova m3, [cq+ 7*32+r5] + mova m4, [cq+ 9*32+r5] + mova m5, [cq+11*32+r5] + mova m6, [cq+13*32+r5] + mova m7, [cq+15*32+r5] + call m(idct_8x4_internal_16bpc).rect2_mul + call m(idct_16x4_internal_16bpc).main_oddhalf + + mova m0, [cq+ 0*32+r5] + mova m1, [cq+ 2*32+r5] + mova m2, [cq+ 4*32+r5] + mova m3, [cq+ 6*32+r5] + mova m4, [cq+ 8*32+r5] + mova m5, [cq+10*32+r5] + mova m6, [cq+12*32+r5] + mova m7, [cq+14*32+r5] + call m(idct_8x4_internal_16bpc).rect2_mul + call m(idct_8x4_internal_16bpc).main_pass1 + call m(idct_8x4_internal_16bpc).round + call m(idct_16x4_internal_16bpc).round +%if ARCH_X86_64 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + packssdw m8, m9 + packssdw m10, m11 + packssdw m12, m13 + packssdw m14, m15 +%endif + ret + +.pass2: +%if ARCH_X86_32 + mov strideq, [rsp+gprsize+12*16] +%endif + mov r4d, 2 +.pass2_main: +%if ARCH_X86_64 + mova m8, [o(pw_2048)] + pxor m9, m9 + mova m10, [o(pixel_10bpc_max)] +%endif + lea r3, [strideq*3] + jmp .loop_pass2_entry +.loop_pass2: + mova m0, [cq+0*32+ 0] + mova m1, [cq+1*32+ 0] + mova m2, [cq+2*32+ 0] + mova m3, [cq+3*32+ 0] +.loop_pass2_entry: + mova m4, [cq+0*32+16] + mova m5, [cq+1*32+16] + mova m6, [cq+2*32+16] + mova m7, [cq+3*32+16] +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call m_suffix(idct_8x8_internal_8bpc, _ssse3).main + call m(idct_8x8_internal_16bpc).round2_and_write_8x8 +%if ARCH_X86_64 +%define mzero m9 +%else +%define mzero m7 + pxor m7, m7 +%endif + REPX {mova [cq+x*16], mzero}, 0, 1, 2, 3, 4, 5, 6, 7 + add dstq, 16 + add cq, 4*32 + dec r4d + jg .loop_pass2 + RET + +INV_TXFM_16X8_FN adst, dct +INV_TXFM_16X8_FN adst, adst +INV_TXFM_16X8_FN adst, flipadst +INV_TXFM_16X8_FN adst, identity, 6 + +cglobal iadst_16x8_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if ARCH_X86_32 + mov [rsp+gprsize+12*16], r1 +%endif + lea t0, [o(.main)] + jmp m(idct_16x8_internal_16bpc).loop_main + +.main: +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_min)] + mova m13, [o(clip_max)] + mova m14, [o(pd_2896)] +%endif + mova m0, [cq+ 2*32+r5] + mova m1, [cq+13*32+r5] + mova m2, [cq+ 6*32+r5] + mova m3, [cq+ 9*32+r5] + mova m4, [cq+10*32+r5] + mova m5, [cq+ 5*32+r5] + mova m6, [cq+14*32+r5] + mova m7, [cq+ 1*32+r5] + call m(idct_8x4_internal_16bpc).rect2_mul + call m(iadst_16x4_internal_16bpc).main_part1 + mova m0, [cq+ 0*32+r5] + mova m1, [cq+15*32+r5] + mova m2, [cq+ 4*32+r5] + mova m3, [cq+11*32+r5] + mova m4, [cq+ 8*32+r5] + mova m5, [cq+ 7*32+r5] + mova m6, [cq+12*32+r5] + mova m7, [cq+ 3*32+r5] +%if ARCH_X86_32 + add r3, 8*16 +%endif + call m(idct_8x4_internal_16bpc).rect2_mul +%if ARCH_X86_32 + sub r3, 8*16 +%endif + call m(iadst_16x4_internal_16bpc).main_part2 + call m(iadst_16x4_internal_16bpc).round +%if ARCH_X86_64 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + packssdw m8, m9 + packssdw m10, m11 + packssdw m12, m13 + packssdw m14, m15 +%endif + ret + +.pass2: +%if ARCH_X86_32 + mov strideq, [rsp+gprsize+12*16] +%endif + mov r4d, 2 +%if ARCH_X86_64 + mova m8, [o(pw_2048)] + pxor m9, m9 + mova m10, [o(pixel_10bpc_max)] + mova m11, [o(pw_m2048)] +%endif + lea r3, [strideq*3] + jmp .loop_pass2_entry +.loop_pass2: + mova m0, [cq+0*32+ 0] + mova m1, [cq+1*32+ 0] + mova m2, [cq+2*32+ 0] + mova m3, [cq+3*32+ 0] +.loop_pass2_entry: + mova m4, [cq+0*32+16] + mova m5, [cq+1*32+16] + mova m6, [cq+2*32+16] + mova m7, [cq+3*32+16] +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + call m_suffix(iadst_8x8_internal_8bpc, _ssse3).main + call m_suffix(iadst_8x8_internal_8bpc, _ssse3).main_pass2_end + call m(iadst_8x8_internal_16bpc).round2_and_write_8x8 +%if ARCH_X86_64 +%define mzero m9 +%else +%define mzero m7 + pxor m7, m7 +%endif + REPX {mova [cq+x*16], mzero}, 0, 1, 2, 3, 4, 5, 6, 7 + add dstq, 16 + add cq, 4*32 + dec r4d + jg .loop_pass2 + RET + +INV_TXFM_16X8_FN flipadst, dct +INV_TXFM_16X8_FN flipadst, adst +INV_TXFM_16X8_FN flipadst, flipadst +INV_TXFM_16X8_FN flipadst, identity, 6 + +cglobal iflipadst_16x8_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if ARCH_X86_32 + mov [rsp+gprsize+12*16], r1 +%endif + lea t0, [o(.main)] + jmp m(idct_16x8_internal_16bpc).loop_main +.main: + call m(iadst_16x8_internal_16bpc).main +%if ARCH_X86_64 + pshufd m1, m0, q1032 + pshufd m3, m2, q1032 + pshufd m5, m4, q1032 + pshufd m7, m6, q1032 + pshufd m0, m14, q1032 + pshufd m2, m12, q1032 + pshufd m4, m10, q1032 + pshufd m6, m8, q1032 + mova m14, m1 + mova m12, m3 + mova m10, m5 + mova m8, m7 +%else + pshufd m1, m0, q1032 + pshufd m3, m2, q1032 + pshufd m5, m4, q1032 + pshufd m7, m6, q1032 + pshufd m0, [r3+11*16], q1032 + pshufd m2, [r3+10*16], q1032 + pshufd m4, [r3+9*16], q1032 + pshufd m6, [r3+8*16], q1032 + mova [r3+8*16], m7 + mova [r3+9*16], m5 + mova [r3+10*16], m3 + mova [r3+11*16], m1 +%endif + ret + +.pass2: +%if ARCH_X86_32 + mov strideq, [rsp+gprsize+12*16] +%endif + lea dstq, [dstq+strideq*8] + neg strideq + add dstq, strideq +%if ARCH_X86_32 + mov [rsp+gprsize+12*16], strideq +%endif + jmp m(iadst_16x8_internal_16bpc).pass2 + +INV_TXFM_16X8_FN identity, dct, -54 +INV_TXFM_16X8_FN identity, adst, -54 +INV_TXFM_16X8_FN identity, flipadst, -54 +INV_TXFM_16X8_FN identity, identity + +cglobal iidentity_16x8_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if ARCH_X86_32 + mov [rsp+gprsize+12*16], r1 +%endif + lea t0, [o(.main)] + jmp m(idct_16x8_internal_16bpc).loop_main +.main: +%if ARCH_X86_64 + mova m15, [o(pd_2896)] + pmulld m0, m15, [cq+ 0*32+r5] + pmulld m1, m15, [cq+ 1*32+r5] + pmulld m2, m15, [cq+ 2*32+r5] + pmulld m3, m15, [cq+ 3*32+r5] + pmulld m4, m15, [cq+ 4*32+r5] + pmulld m5, m15, [cq+ 5*32+r5] + pmulld m6, m15, [cq+ 6*32+r5] + pmulld m7, m15, [cq+ 7*32+r5] + pmulld m8, m15, [cq+ 8*32+r5] + pmulld m9, m15, [cq+ 9*32+r5] + pmulld m10, m15, [cq+10*32+r5] + pmulld m11, m15, [cq+11*32+r5] + pmulld m12, m15, [cq+12*32+r5] + pmulld m13, m15, [cq+13*32+r5] + pmulld m14, m15, [cq+14*32+r5] + pmulld m15, [cq+15*32+r5] + mova [r3], m15 + mova m15, [o(pd_2048)] + REPX {paddd x, m15}, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14 + paddd m15, [r3] + REPX {psrad x, 12 }, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14, m15 + mova [r3], m15 + mova m15, [o(pd_11586)] + REPX {pmulld x, m15}, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14 + pmulld m15, [r3] + mova [r3], m15 + mova m15, [o(pd_6144)] + REPX {paddd x, m15}, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14 + paddd m15, [r3] + REPX {psrad x, 13 }, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14, m15 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + packssdw m8, m9 + packssdw m10, m11 + packssdw m12, m13 + packssdw m14, m15 +%else + mova m0, [cq+ 0*32+r5] + mova m1, [cq+ 1*32+r5] + mova m2, [cq+ 2*32+r5] + mova m3, [cq+ 3*32+r5] + mova m4, [cq+ 4*32+r5] + mova m5, [cq+ 5*32+r5] + mova m6, [cq+ 6*32+r5] + mova m7, [cq+ 7*32+r5] + call m(idct_8x4_internal_16bpc).rect2_mul + mova [r3], m7 + mova m7, [o(pd_11586)] + REPX {pmulld x, m7}, m0, m1, m2, m3, m4, m5, m6 + pmulld m7, [r3] + mova [r3], m7 + mova m7, [o(pd_6144)] + REPX {paddd x, m7}, m0, m1, m2, m3, m4, m5, m6 + paddd m7, [r3] + REPX {psrad x, 13}, m0, m1, m2, m3, m4, m5, m6, m7 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + mova [r3+ 8*16], m0 + mova [r3+ 9*16], m2 + mova [r3+10*16], m4 + mova [r3+11*16], m6 + mova m0, [cq+ 8*32+r5] + mova m1, [cq+ 9*32+r5] + mova m2, [cq+10*32+r5] + mova m3, [cq+11*32+r5] + mova m4, [cq+12*32+r5] + mova m5, [cq+13*32+r5] + mova m6, [cq+14*32+r5] + mova m7, [cq+15*32+r5] + call m(idct_8x4_internal_16bpc).rect2_mul + mova [r3], m7 + mova m7, [o(pd_11586)] + REPX {pmulld x, m7}, m0, m1, m2, m3, m4, m5, m6 + pmulld m7, [r3] + mova [r3], m7 + mova m7, [o(pd_6144)] + REPX {paddd x, m7}, m0, m1, m2, m3, m4, m5, m6 + paddd m7, [r3] + REPX {psrad x, 13}, m0, m1, m2, m3, m4, m5, m6, m7 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 +%endif + ret +.pass2: +%if ARCH_X86_32 + mov strideq, [rsp+gprsize+12*16] +%endif + mov r4d, 2 +%if ARCH_X86_64 + mova m8, [o(pw_4096)] + pxor m9, m9 + mova m10, [o(pixel_10bpc_max)] +%endif + lea r3, [strideq*3] + jmp .loop_pass2_entry +.loop_pass2: + mova m0, [cq+0*32+ 0] + mova m1, [cq+1*32+ 0] + mova m2, [cq+2*32+ 0] + mova m3, [cq+3*32+ 0] +.loop_pass2_entry: + mova m4, [cq+0*32+16] + mova m5, [cq+1*32+16] + mova m6, [cq+2*32+16] + mova m7, [cq+3*32+16] +%if ARCH_X86_64 + call m(idct_8x8_internal_16bpc).round1_and_write_8x8 +%else + mova [rsp+gprsize], m7 + mova m7, [o(pw_4096)] + call m(idct_8x8_internal_16bpc).round4_and_write_8x8 +%endif +%if ARCH_X86_64 +%define mzero m9 +%else +%define mzero m7 + pxor m7, m7 +%endif + REPX {mova [cq+x*16], mzero}, 0, 1, 2, 3, 4, 5, 6, 7 + add dstq, 16 + add cq, 4*32 + dec r4d + jg .loop_pass2 + RET + +%macro INV_TXFM_16X16_FN 2-3 2d ; type1, type2, eob_tbl_suffix +%if ARCH_X86_64 + INV_TXFM_FN %1, %2, tbl_16x16_%3, 16x16, 16, 0-(16+WIN64)*16 +%else + INV_TXFM_FN %1, %2, tbl_16x16_%3, 16x16, 8, 0-17*16 +%endif +%ifidn %1_%2, dct_dct + imul r5d, [cq], 2896 + mov [cq], eobd ; 0 + mov r3d, 16 + add r5d, 10240 + sar r5d, 14 + add rsp, (5+ARCH_X86_64*3+WIN64)*16 + jmp m(inv_txfm_add_dct_dct_16x4_16bpc).dconly2 +%endif +%endmacro + +INV_TXFM_16X16_FN dct, dct +INV_TXFM_16X16_FN dct, identity, v +INV_TXFM_16X16_FN dct, adst +INV_TXFM_16X16_FN dct, flipadst + +cglobal idct_16x16_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if ARCH_X86_64 + DECLARE_REG_TMP 6, 7 +%if WIN64 + mov [rsp+16*16+gprsize], r7 +%endif +%elif ARCH_X86_32 + DECLARE_REG_TMP 1, 6 + mov [rsp+16*16+gprsize*1], r1 + mov [rsp+16*16+gprsize*2], r6 +%endif + lea t0, [o(.main)] +.pass1_full: +%undef cmp + mov t1d, 4 +.zero_loop: + dec t1d + cmp eobb, byte [r5+t1] + jb .zero_loop + mov r5d, t1d + shl r5d, 4 +%if ARCH_X86_32 + ; restore pic-ptr + mov r6, [rsp+16*16+2*gprsize] +%endif + ; setup stack pointer + lea r3, [rsp+gprsize] +.loop_pass1: + call t0 +%if ARCH_X86_64 + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [cq+4*64+r5], m8 + mova [cq+5*64+r5], m9 + mova [cq+6*64+r5], m10 + mova [cq+7*64+r5], m11 +%else + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [cq+4*64+r5], m0 + mova [cq+5*64+r5], m1 + mova [cq+6*64+r5], m2 + mova [cq+7*64+r5], m3 + mova m0, [rsp+gprsize+ 8*16] + mova m2, [rsp+gprsize+ 9*16] + mova m4, [rsp+gprsize+10*16] + mova m6, [rsp+gprsize+11*16] +%endif + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [cq+0*64+r5], m0 + mova [cq+1*64+r5], m1 + mova [cq+2*64+r5], m2 + mova [cq+3*64+r5], m3 + pxor m0, m0 + REPX {mova [cq+x*64+r5], m0}, 8, 9, 10, 11, 12, 13, 14, 15 + sub r5d, 16 + jge .loop_pass1 + +%if ARCH_X86_32 + ; restore pic-ptr + mov r1, [rsp+16*16+1*gprsize] +%endif + jmp tx2q +.main: +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_min)] + mova m13, [o(clip_max)] + mova m14, [o(pd_2896)] +%endif + + mova m0, [cq+ 1*64+r5] + mova m1, [cq+ 3*64+r5] + mova m2, [cq+ 5*64+r5] + mova m3, [cq+ 7*64+r5] + mova m4, [cq+ 9*64+r5] + mova m5, [cq+11*64+r5] + mova m6, [cq+13*64+r5] + mova m7, [cq+15*64+r5] + call m(idct_16x4_internal_16bpc).main_oddhalf + + mova m0, [cq+ 0*64+r5] + mova m1, [cq+ 2*64+r5] + mova m2, [cq+ 4*64+r5] + mova m3, [cq+ 6*64+r5] + mova m4, [cq+ 8*64+r5] + mova m5, [cq+10*64+r5] + mova m6, [cq+12*64+r5] + mova m7, [cq+14*64+r5] + call m(idct_8x4_internal_16bpc).main_pass1 + call m(idct_8x4_internal_16bpc).round + call .round +%if ARCH_X86_64 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + packssdw m8, m9 + packssdw m10, m11 + packssdw m12, m13 + packssdw m14, m15 +%endif + ret +.round: +%if ARCH_X86_64 + psrld m8, m11, 10 ; 2 + REPX {paddd x, m8}, m0, m1, m2, m3, m4, m5, m6, m7 + mova m8, [r3+1*16] + mova m9, [r3+2*16] + mova m10, [r3+3*16] + mova m11, [r3+4*16] + mova m12, [r3+5*16] + mova m13, [r3+6*16] + mova m14, [r3+7*16] + psubd m15, m0, m14 ; out15 + paddd m0, m14 ; out0 + psubd m14, m1, m13 ; out14 + paddd m1, m13 ; out1 + psubd m13, m2, m12 ; out13 + paddd m2, m12 ; out2 + psubd m12, m3, m11 ; out12 + paddd m3, m11 ; out3 + psubd m11, m4, m10 ; out11 + paddd m4, m10 ; out4 + psubd m10, m5, m9 ; out10 + paddd m5, m9 ; out5 + psubd m9, m6, m8 ; out9 + paddd m6, m8 ; out6 + psubd m8, m7, [r3+0*16] ; out8 + paddd m7, [r3+0*16] ; out7 + REPX {psrad x, 2}, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14, m15 + ; and out0-15 is now in m0-15 +%else + mova [r3+ 0*16], m0 + mova m0, [o(pd_2)] + REPX {paddd x, m0}, m1, m2, m3, m4, m5, m6, m7 + paddd m0, [r3+ 0*16] + mova [r3+ 0*16], m0 + mova [r3+ 1*16], m1 + mova [r3+ 2*16], m2 + mova m1, [r3+11*16] + mova m2, [r3+10*16] + psubd m0, m7, m1 + paddd m7, m1 + psubd m1, m6, m2 + paddd m6, m2 + REPX {psrad x, 2}, m0, m1, m6, m7 + packssdw m0, m1 ; out8-9 + packssdw m6, m7 ; out6-7 + mova [r3+11*16], m6 + mova m1, [r3+9*16] + mova m7, [r3+8*16] + psubd m2, m5, m1 + paddd m5, m1 + psubd m1, m4, m7 + paddd m4, m7 + REPX {psrad x, 2}, m2, m1, m4, m5 + packssdw m2, m1 ; out10-11 + packssdw m4, m5 ; out4-5 + mova m1, [r3+2*16] + mova [r3+10*16], m4 + mova m6, [r3+7*16] + mova m7, [r3+6*16] + psubd m4, m3, m6 + paddd m3, m6 + psubd m6, m1, m7 + paddd m1, m7 + REPX {psrad x, 2}, m4, m6, m1, m3 + packssdw m4, m6 ; out12-13 + packssdw m1, m3 ; out2-3 + mova m3, [r3+1*16] + mova [r3+9*16], m1 + mova m1, [r3+0*16] + mova m5, [r3+5*16] + mova m7, [r3+4*16] + psubd m6, m3, m5 + paddd m3, m5 + psubd m5, m1, m7 + paddd m1, m7 + REPX {psrad x, 2}, m6, m5, m1, m3 + packssdw m6, m5 ; out14-15 + packssdw m1, m3 ; out0-1 + mova [r3+8*16], m1 +%endif + ret + +.pass2: +%if ARCH_X86_64 + mova m8, [o(pw_2048)] + pxor m9, m9 + mova m10, [o(pixel_10bpc_max)] + mov r7, dstq +%else + mov [rsp+2*gprsize+16*16], dstq +%endif + lea r3, [strideq*3] + mov r4d, 2 +.loop_pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + mova m0, [cq+0*64+ 0] + mova m1, [cq+2*64+ 0] + mova m2, [cq+0*64+16] + mova m3, [cq+2*64+16] + mova m4, [cq+0*64+32] + mova m5, [cq+2*64+32] + mova m6, [cq+0*64+48] + mova m7, [cq+2*64+48] + call m_suffix(idct_8x8_internal_8bpc, _ssse3).main + mova [rsp+gprsize+3*16], m0 + mova [rsp+gprsize+4*16], m1 + mova [rsp+gprsize+5*16], m2 + mova [rsp+gprsize+6*16], m3 + mova [rsp+gprsize+7*16], m4 + mova [rsp+gprsize+8*16], m5 + mova [rsp+gprsize+9*16], m6 + ; m7 is already stored in [rsp+gprsize+0*16] + mova m0, [cq+1*64+ 0] + mova m1, [cq+3*64+ 0] + mova m2, [cq+1*64+16] + mova m3, [cq+3*64+16] + mova m4, [cq+1*64+32] + mova m5, [cq+3*64+32] + mova m6, [cq+1*64+48] + mova m7, [cq+3*64+48] + call m_suffix(idct_16x8_internal_8bpc, _ssse3).main + + ; out0-7 is in rsp+gprsize+3-10*mmsize + ; out8-14 is in m0-6, and out15 is in m7 as well as rsp+gprsize+0*mmsize + +%if ARCH_X86_64 + lea dstq, [r7+strideq*8] +%else + mov dstq, [rsp+2*gprsize+16*16] + lea dstq, [dstq+strideq*8] +%endif + call m(idct_8x8_internal_16bpc).round2_and_write_8x8 +%if ARCH_X86_64 + mov dstq, r7 +%else + mov dstq, [rsp+2*gprsize+16*16] +%endif + mova m0, [rsp+gprsize+ 3*16] + mova m1, [rsp+gprsize+ 4*16] + mova m2, [rsp+gprsize+ 5*16] + mova m3, [rsp+gprsize+ 6*16] + mova m4, [rsp+gprsize+ 7*16] + mova m5, [rsp+gprsize+ 8*16] + mova m6, [rsp+gprsize+ 9*16] + mova m7, [rsp+gprsize+10*16] + call m(idct_8x8_internal_16bpc).round1_and_write_8x8 +%if ARCH_X86_64 + add r7, 16 +%define mzero m9 +%else + add dword [rsp+2*gprsize+16*16], 16 +%define mzero m7 + pxor m7, m7 +%endif + REPX {mova [cq+x*16], mzero}, 0, 1, 2, 3, 4, 5, 6, 7 + add cq, 64*4 + REPX {mova [cq+x*16], mzero}, -8, -7, -6, -5, -4, -3, -2, -1 +%undef mzero + dec r4d + jg .loop_pass2 +%if WIN64 + mov r7, [rsp+16*16+gprsize] +%endif + RET + +INV_TXFM_16X16_FN adst, dct +INV_TXFM_16X16_FN adst, adst +INV_TXFM_16X16_FN adst, flipadst + +cglobal iadst_16x16_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if WIN64 + mov [rsp+16*16+gprsize], r7 +%elif ARCH_X86_32 + mov [rsp+16*16+gprsize*1], r1 + mov [rsp+16*16+gprsize*2], r6 +%endif + lea t0, [o(.main)] + jmp m(idct_16x16_internal_16bpc).pass1_full + +.main: +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_min)] + mova m13, [o(clip_max)] + mova m14, [o(pd_2896)] +%endif + mova m0, [cq+ 2*64+r5] + mova m1, [cq+13*64+r5] + mova m2, [cq+ 6*64+r5] + mova m3, [cq+ 9*64+r5] + mova m4, [cq+10*64+r5] + mova m5, [cq+ 5*64+r5] + mova m6, [cq+14*64+r5] + mova m7, [cq+ 1*64+r5] + call m(iadst_16x4_internal_16bpc).main_part1 + mova m0, [cq+ 0*64+r5] + mova m1, [cq+15*64+r5] + mova m2, [cq+ 4*64+r5] + mova m3, [cq+11*64+r5] + mova m4, [cq+ 8*64+r5] + mova m5, [cq+ 7*64+r5] + mova m6, [cq+12*64+r5] + mova m7, [cq+ 3*64+r5] + call m(iadst_16x4_internal_16bpc).main_part2 + call .round +%if ARCH_X86_64 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + packssdw m8, m9 + packssdw m10, m11 + packssdw m12, m13 + packssdw m14, m15 +%endif + ret +.round: +%if ARCH_X86_64 + pcmpeqd m8, m8 ; -1 + mova m15, [o(pd_10240)] + psrld m14, 10 ; +2 + psubd m13, m14, m8 ; +3 + REPX {pxor x, m8 }, m1, m3, m5, m7 + REPX {paddd x, m14}, m0, m2 + REPX {paddd x, m13}, m1, m3 + REPX {paddd x, m15}, m4, m5, m6, m7 + paddd m13, m15, m8 ; +10239 + paddd m8, m15, m9 + psubd m9, m13, m10 + paddd m10, m15, m11 + psubd m11, m13, m12 + paddd m12, m14, [r3+3*16] + psubd m13, m14, [r3+2*16] + psubd m15, m14, [r3+0*16] + paddd m14, [r3+1*16] + REPX {psrad x, 2 }, m0, m1, m2, m3, m12, m13, m14, m15 + REPX {psrad x, 14}, m4, m5, m6, m7, m8, m9, m10, m11 +%else + mova [r3+8*16], m1 + mova [r3+9*16], m3 + mova m3, [o(pd_10240)] + pcmpeqd m1, m1 + REPX {pxor x, m1}, m5, m7 + REPX {paddd x, m3}, m4, m5, m6, m7 + REPX {psrad x, 14}, m4, m5, m6, m7 + packssdw m4, m5 + packssdw m6, m7 + mova [r3+10*16], m4 + mova [r3+11*16], m6 + mova m4, [r3+4*16] + mova m5, [r3+5*16] + mova m6, [r3+6*16] + mova m7, [r3+7*16] + mova m3, [o(pd_2)] + REPX {pxor x, m1}, m5, m7 + REPX {paddd x, m3}, m4, m6 + psubd m3, m1 + REPX {paddd x, m3}, m5, m7 + REPX {psrad x, 2 }, m4, m5, m6, m7 + packssdw m4, m5 + packssdw m6, m7 + mova m5, [r3+8*16] + mova m7, [r3+9*16] + mova [r3+8*16], m4 + mova [r3+9*16], m6 + mova m3, [o(pd_10240)] + REPX {pxor x, m1}, m5, m7 + REPX {paddd x, m3}, m0, m5, m2, m7 + REPX {psrad x, 14}, m0, m5, m2, m7 + packssdw m0, m5 + packssdw m2, m7 + mova m4, [r3+0*16] + mova m5, [r3+1*16] + mova m6, [r3+2*16] + mova m7, [r3+3*16] + mova m3, [o(pd_2)] + REPX {pxor x, m1}, m5, m7 + REPX {paddd x, m3}, m4, m6 + psubd m3, m1 + REPX {paddd x, m3}, m5, m7 + REPX {psrad x, 2 }, m4, m5, m6, m7 + packssdw m4, m5 + packssdw m6, m7 +%endif + ret +.pass2: +%if ARCH_X86_64 + mova m8, [o(pw_2048)] + mova m11, [o(pw_m2048)] + pxor m9, m9 + mova m10, [o(pixel_10bpc_max)] + mov r7, dstq +%else + mov [rsp+2*gprsize+16*16], dstq +%endif + lea r3, [strideq*3] + mov r4d, 2 +.loop_pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + mova m0, [cq+0*64+32] + mova m1, [cq+1*64+32] + mova m2, [cq+2*64+16] + mova m3, [cq+3*64+16] + mova m4, [cq+0*64+ 0] + mova m5, [cq+1*64+ 0] + mova m6, [cq+2*64+48] + mova m7, [cq+3*64+48] + mova [rsp+gprsize+3*16], m0 + mova [rsp+gprsize+4*16], m1 + mova [rsp+gprsize+5*16], m2 + mova [rsp+gprsize+6*16], m3 + mova [rsp+gprsize+7*16], m4 + mova [rsp+gprsize+8*16], m5 + mova [rsp+gprsize+9*16], m6 + mova [rsp+gprsize+10*16], m7 + mova m0, [cq+2*64+ 0] + mova m1, [cq+3*64+ 0] + mova m2, [cq+0*64+16] + mova m3, [cq+1*64+16] + mova m4, [cq+2*64+32] + mova m5, [cq+3*64+32] + mova m6, [cq+0*64+48] + mova m7, [cq+1*64+48] + call m_suffix(iadst_16x8_internal_8bpc, _ssse3).main + call m_suffix(iadst_16x8_internal_8bpc, _ssse3).main_pass2_end + + ; out0-7 is in rsp+gprsize+3-10*mmsize + ; out8-14 is in m0-6, and out15 is in m7 as well as rsp+gprsize+0*mmsize + +%if ARCH_X86_64 + lea dstq, [r7+strideq*8] +%else + mov dstq, [rsp+2*gprsize+16*16] + lea dstq, [dstq+strideq*8] +%endif + call m(iadst_8x8_internal_16bpc).round2_and_write_8x8 +%if ARCH_X86_64 + mov dstq, r7 +%else + mov dstq, [rsp+2*gprsize+16*16] +%endif + mova m0, [rsp+gprsize+ 3*16] + mova m1, [rsp+gprsize+ 4*16] + mova m2, [rsp+gprsize+ 5*16] + mova m3, [rsp+gprsize+ 6*16] + mova m4, [rsp+gprsize+ 7*16] + mova m5, [rsp+gprsize+ 8*16] + mova m6, [rsp+gprsize+ 9*16] + mova m7, [rsp+gprsize+10*16] + call m(iadst_8x8_internal_16bpc).round1_and_write_8x8 +%if ARCH_X86_64 + add r7, 16 +%define mzero m9 +%else + add dword [rsp+2*gprsize+16*16], 16 +%define mzero m7 + pxor m7, m7 +%endif + REPX {mova [cq+x*16], mzero}, 0, 1, 2, 3, 4, 5, 6, 7 + add cq, 64*4 + REPX {mova [cq+x*16], mzero}, -8, -7, -6, -5, -4, -3, -2, -1 +%undef mzero + dec r4d + jg .loop_pass2 +%if WIN64 + mov r7, [rsp+16*16+gprsize] +%endif + RET + +INV_TXFM_16X16_FN flipadst, dct +INV_TXFM_16X16_FN flipadst, adst +INV_TXFM_16X16_FN flipadst, flipadst + +cglobal iflipadst_16x16_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if WIN64 + mov [rsp+16*16+gprsize], r7 +%elif ARCH_X86_32 + mov [rsp+16*16+gprsize*1], r1 + mov [rsp+16*16+gprsize*2], r6 +%endif + lea t0, [o(.main)] + jmp m(idct_16x16_internal_16bpc).pass1_full + +.main: + call m(iadst_16x16_internal_16bpc).main +%if ARCH_X86_64 + mova m1, m0 + mova m3, m2 + mova m5, m4 + mova m7, m6 + pshufd m0, m14, q1032 + pshufd m2, m12, q1032 + pshufd m4, m10, q1032 + pshufd m6, m8, q1032 + pshufd m8, m7, q1032 + pshufd m10, m5, q1032 + pshufd m12, m3, q1032 + pshufd m14, m1, q1032 +%else + pshufd m1, m0, q1032 + pshufd m3, m2, q1032 + pshufd m5, m4, q1032 + pshufd m7, m6, q1032 + pshufd m0, [r3+11*16], q1032 + pshufd m2, [r3+10*16], q1032 + pshufd m4, [r3+9*16], q1032 + pshufd m6, [r3+8*16], q1032 + mova [r3+11*16], m1 + mova [r3+10*16], m3 + mova [r3+ 9*16], m5 + mova [r3+ 8*16], m7 +%endif + ret + +.pass2: + lea r3, [strideq*3] + lea r3, [r3*5] + add dstq, r3 + neg strideq + jmp m(iadst_16x16_internal_16bpc).pass2 + +INV_TXFM_16X16_FN identity, dct, h +INV_TXFM_16X16_FN identity, identity + +cglobal iidentity_16x16_internal_16bpc, 0, 0, 0, dst, stride, c, eob, tx2 +%if WIN64 + mov [rsp+16*16+gprsize], r7 +%elif ARCH_X86_32 + mov [rsp+16*16+gprsize*1], r1 + mov [rsp+16*16+gprsize*2], r6 +%endif + lea t0, [o(.main)] + jmp m(idct_16x16_internal_16bpc).pass1_full + +.main: +%if ARCH_X86_64 + mova m15, [o(pd_11586)] + pmulld m0, m15, [cq+ 0*64+r5] + pmulld m1, m15, [cq+ 1*64+r5] + pmulld m2, m15, [cq+ 2*64+r5] + pmulld m3, m15, [cq+ 3*64+r5] + pmulld m4, m15, [cq+ 4*64+r5] + pmulld m5, m15, [cq+ 5*64+r5] + pmulld m6, m15, [cq+ 6*64+r5] + pmulld m7, m15, [cq+ 7*64+r5] + pmulld m8, m15, [cq+ 8*64+r5] + pmulld m9, m15, [cq+ 9*64+r5] + pmulld m10, m15, [cq+10*64+r5] + pmulld m11, m15, [cq+11*64+r5] + pmulld m12, m15, [cq+12*64+r5] + pmulld m13, m15, [cq+13*64+r5] + pmulld m14, m15, [cq+14*64+r5] + pmulld m15, [cq+15*64+r5] + mova [r3], m15 + mova m15, [o(pd_10240)] + REPX {paddd x, m15}, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14 + paddd m15, [r3] + REPX {psrad x, 14 }, m0, m1, m2, m3, m4, m5, m6, m7, \ + m8, m9, m10, m11, m12, m13, m14, m15 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + packssdw m8, m9 + packssdw m10, m11 + packssdw m12, m13 + packssdw m14, m15 +%else + mova m7, [o(pd_11586)] + pmulld m0, m7, [cq+ 0*64+r5] + pmulld m1, m7, [cq+ 1*64+r5] + pmulld m2, m7, [cq+ 2*64+r5] + pmulld m3, m7, [cq+ 3*64+r5] + pmulld m4, m7, [cq+ 4*64+r5] + pmulld m5, m7, [cq+ 5*64+r5] + pmulld m6, m7, [cq+ 6*64+r5] + pmulld m7, [cq+ 7*64+r5] + mova [r3], m7 + mova m7, [o(pd_10240)] + REPX {paddd x, m7}, m0, m1, m2, m3, m4, m5, m6 + paddd m7, [r3] + REPX {psrad x, 14}, m0, m1, m2, m3, m4, m5, m6, m7 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + mova [r3+8*16], m0 + mova [r3+9*16], m2 + mova [r3+10*16], m4 + mova [r3+11*16], m6 + mova m7, [o(pd_11586)] + pmulld m0, m7, [cq+ 8*64+r5] + pmulld m1, m7, [cq+ 9*64+r5] + pmulld m2, m7, [cq+10*64+r5] + pmulld m3, m7, [cq+11*64+r5] + pmulld m4, m7, [cq+12*64+r5] + pmulld m5, m7, [cq+13*64+r5] + pmulld m6, m7, [cq+14*64+r5] + pmulld m7, [cq+15*64+r5] + mova [r3], m7 + mova m7, [o(pd_10240)] + REPX {paddd x, m7}, m0, m1, m2, m3, m4, m5, m6 + paddd m7, [r3] + REPX {psrad x, 14}, m0, m1, m2, m3, m4, m5, m6, m7 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 +%endif + ret + +.pass2: +%if ARCH_X86_64 + mova m4, [o(pw_2048)] + mova m5, [o(pixel_10bpc_max)] + pxor m6, m6 + mova m7, [o(pw_1697x16)] + mov r7, dstq +%else + mov [rsp+2*gprsize+16*16], dstq +%endif + mov r5d, 4 + lea r3, [strideq*3] +.pass2_loop: + mova m0, [cq+0*64+0] + mova m1, [cq+1*64+0] + mova m2, [cq+2*64+0] + mova m3, [cq+3*64+0] + call m(iidentity_8x16_internal_16bpc).main +%if ARCH_X86_64 + call m(idct_8x4_internal_16bpc).round1_and_write_8x4 +%else + call m(idct_8x4_internal_16bpc).round2_and_write_8x4 +%endif + REPX {mova [cq+x*16], m6}, 0, 4, 8, 12 + add cq, 16 + lea dstq, [dstq+strideq*4] + dec r5w + jg .pass2_loop + add cq, 64*3 + btc r5d, 16 + jc .end +%if ARCH_X86_64 + lea dstq, [r7+16] +%else + mov dstq, [rsp+2*gprsize+16*16] + add dstq, 16 +%endif + add r5d, 4 + jmp .pass2_loop +.end: +%if WIN64 + mov r7, [rsp+16*16+gprsize] +%endif + RET + +cglobal inv_txfm_add_identity_identity_8x32_16bpc, 4, 7, 8, dst, stride, c, eob +%if ARCH_X86_32 + LEA r6, $$ +%endif + mova m5, [o(pw_5)] + mova m7, [o(pixel_10bpc_max)] + pxor m6, m6 + mov r5d, eobd + add eobb, 21 + cmovc eobd, r5d ; 43, 107, 171 -> 64, 128, 192 + lea r4, [strideq*3] +.loop: + mova m0, [cq+128*0] + packssdw m0, [cq+128*1] + mova m1, [cq+128*2] + packssdw m1, [cq+128*3] + mova m2, [cq+128*4] + packssdw m2, [cq+128*5] + mova m3, [cq+128*6] + packssdw m3, [cq+128*7] + REPX {paddsw x, m5}, m0, m1, m2, m3 + REPX {psraw x, 3 }, m0, m1, m2, m3 + call .main_zero + add cq, 16 + lea dstq, [dstq+strideq*4] + btc eobd, 16 + jnc .loop + sub eobd, 64 + jge .loop + RET +ALIGN function_align +.main_zero: + REPX {mova [cq+128*x], m6}, 0, 1, 2, 3, 4, 5, 6, 7 +.main: + punpckhwd m4, m0, m1 + punpcklwd m0, m1 + punpckhwd m1, m2, m3 + punpcklwd m2, m3 + punpckhwd m3, m0, m4 + punpcklwd m0, m4 + punpckhwd m4, m2, m1 + punpcklwd m2, m1 + punpckhqdq m1, m0, m2 + punpcklqdq m0, m2 + punpcklqdq m2, m3, m4 + punpckhqdq m3, m4 + paddw m0, [dstq+strideq*0] + paddw m1, [dstq+strideq*1] + paddw m2, [dstq+strideq*2] + paddw m3, [dstq+r4 ] + REPX {pmaxsw x, m6}, m0, m1, m2, m3 + REPX {pminsw x, m7}, m0, m1, m2, m3 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + mova [dstq+strideq*2], m2 + mova [dstq+r4 ], m3 + ret + +cglobal inv_txfm_add_identity_identity_32x8_16bpc, 4, 7, 8, dst, stride, c, eob +%if ARCH_X86_32 + LEA r6, $$ +%endif + mova m5, [o(pw_4096)] + mova m7, [o(pixel_10bpc_max)] + pxor m6, m6 + mov r4d, eobd + add eobb, 21 + cmovc eobd, r4d + lea r4, [strideq*3] + mov r5, dstq +.loop: + mova m0, [cq+32*0] + packssdw m0, [cq+32*1] + mova m1, [cq+32*2] + packssdw m1, [cq+32*3] + mova m2, [cq+32*4] + packssdw m2, [cq+32*5] + mova m3, [cq+32*6] + packssdw m3, [cq+32*7] + REPX {mova [cq+32*x], m6}, 0, 1, 2, 3, 4, 5, 6, 7 + REPX {pmulhrsw x, m5}, m0, m1, m2, m3 + call m(inv_txfm_add_identity_identity_8x32_16bpc).main + lea dstq, [dstq+strideq*4] + add cq, 16 + btc eobd, 16 + jnc .loop + add cq, 32*8-32 + add r5, 16 + mov dstq, r5 + sub eobd, 64 + jge .loop + RET + +cglobal inv_txfm_add_identity_identity_16x32_16bpc, 4, 7, 12, dst, stride, c, eob +%if ARCH_X86_32 + LEA r6, $$ +%else + mova m8, [o(pw_2896x8)] + mova m9, [o(pw_1697x16)] + mova m11, [o(pw_8192)] +%endif + mova m7, [o(pixel_10bpc_max)] + lea r4, [strideq*3] + pxor m6, m6 +%if ARCH_X86_64 + paddw m10, m11, m11 ; pw_16384 +%endif + mov r5, dstq + call .main + sub eobd, 36 + jl .ret + add cq, 128*8-32 + lea dstq, [r5+16] + call .main + sub cq, 128*8 + lea dstq, [r5+strideq*8] + mov r5, dstq + call .main + sub eobd, 107 ; eob < 143 + jl .ret + add cq, 128*8-32 + lea dstq, [r5+16] + call .main + sub cq, 128*8 + lea dstq, [r5+strideq*8] + mov r5, dstq + call .main + sub eobd, 128 ; eob < 271 + jl .ret + add cq, 128*8-32 + lea dstq, [r5+16] + call .main + sub cq, 128*8 + lea dstq, [r5+strideq*8] + mov r5, dstq + call .main + sub eobd, 128 ; eob < 399 + jl .ret + add cq, 128*8-32 + lea dstq, [r5+16] + call .main +.ret: + RET +ALIGN function_align +.main: + mova m0, [cq+128*0] + packssdw m0, [cq+128*1] + mova m1, [cq+128*2] + packssdw m1, [cq+128*3] + mova m2, [cq+128*4] + packssdw m2, [cq+128*5] + mova m3, [cq+128*6] + packssdw m3, [cq+128*7] +%if ARCH_X86_64 + REPX {pmulhrsw x, m8 }, m0, m1, m2, m3 + pmulhrsw m4, m9, m0 + pmulhrsw m5, m9, m1 + REPX {pmulhrsw x, m10}, m4, m5 +%else + mova m6, [o(pw_2896x8)] + REPX {pmulhrsw x, m6 }, m0, m1, m2, m3 + mova m5, [o(pw_1697x16)] + pmulhrsw m4, m5, m0 + pmulhrsw m5, m1 + mova m6, [o(pw_16384)] + REPX {pmulhrsw x, m6 }, m4, m5 +%endif + paddsw m0, m4 + paddsw m1, m5 +%if ARCH_X86_64 + pmulhrsw m4, m9, m2 + pmulhrsw m5, m9, m3 + REPX {pmulhrsw x, m10}, m4, m5 +%else + mova m5, [o(pw_1697x16)] + pmulhrsw m4, m5, m2 + pmulhrsw m5, m3 + REPX {pmulhrsw x, m6 }, m4, m5 +%endif + paddsw m2, m4 + paddsw m3, m5 +%if ARCH_X86_64 + REPX {pmulhrsw x, m11}, m0, m1, m2, m3 +%else + psrlw m6, 1 ; pw_8192 + REPX {pmulhrsw x, m6 }, m0, m1, m2, m3 + pxor m6, m6 +%endif + call m(inv_txfm_add_identity_identity_8x32_16bpc).main_zero + lea dstq, [dstq+strideq*4] + add cq, 16 + btc eobd, 16 + jnc .main + ret + +cglobal inv_txfm_add_identity_identity_32x16_16bpc, 4, 7, 11, dst, stride, c, eob +%if ARCH_X86_32 + LEA r6, $$ +%else + mova m8, [o(pw_2896x8)] + mova m9, [o(pw_1697x16)] + mova m10, [o(pw_2048)] +%endif + mova m7, [o(pixel_10bpc_max)] + lea r4, [strideq*3] + pxor m6, m6 + mov r5, dstq + call .main + sub eobd, 36 + jl .ret + call .main + add cq, 64*8-64 + lea dstq, [r5+16*1] + call .main + sub eobd, 107 ; eob < 143 + jl .ret + call .main + add cq, 64*8-64 + lea dstq, [r5+16*2] + call .main + sub eobd, 128 ; eob < 271 + jl .ret + call .main + add cq, 64*8-64 + lea dstq, [r5+16*3] + call .main + sub eobd, 128 ; eob < 399 + jl .ret + call .main +.ret: + RET +ALIGN function_align +.main: + mova m0, [cq+64*0] + packssdw m0, [cq+64*1] + mova m1, [cq+64*2] + packssdw m1, [cq+64*3] + mova m2, [cq+64*4] + packssdw m2, [cq+64*5] + mova m3, [cq+64*6] + packssdw m3, [cq+64*7] +%if ARCH_X86_64 + REPX {pmulhrsw x, m8 }, m0, m1, m2, m3 +%else + mova m6, [o(pw_2896x8)] + REPX {pmulhrsw x, m6 }, m0, m1, m2, m3 +%endif + REPX {paddsw x, x }, m0, m1, m2, m3 +%if ARCH_X86_64 + pmulhrsw m4, m9, m0 + pmulhrsw m5, m9, m1 +%else + mova m6, [o(pw_1697x16)] + pmulhrsw m4, m6, m0 + pmulhrsw m5, m6, m1 +%endif + REPX {paddsw x, x }, m0, m1 + paddsw m0, m4 + paddsw m1, m5 +%if ARCH_X86_64 + pmulhrsw m4, m9, m2 + pmulhrsw m5, m9, m3 +%else + pmulhrsw m4, m6, m2 + pmulhrsw m6, m3 +%endif + REPX {paddsw x, x }, m2, m3 + paddsw m2, m4 +%if ARCH_X86_64 + paddsw m3, m5 + REPX {pmulhrsw x, m10}, m0, m1, m2, m3 +%else + paddsw m3, m6 + mova m6, [o(pw_2048)] + REPX {pmulhrsw x, m6 }, m0, m1, m2, m3 + pxor m6, m6 +%endif + REPX {mova [cq+64*x], m6}, 0, 1, 2, 3, 4, 5, 6, 7 + call m(inv_txfm_add_identity_identity_8x32_16bpc).main + lea dstq, [dstq+strideq*4] + add cq, 16 + btc eobd, 16 + jnc .main + ret + +cglobal inv_txfm_add_identity_identity_32x32_16bpc, 4, 7, 8, dst, stride, c, eob +%undef cmp +%if ARCH_X86_32 + LEA r6, $$ +%endif + mova m5, [o(pw_8192)] + mova m7, [o(pixel_10bpc_max)] + pxor m6, m6 + lea r4, [strideq*3] + mov r5, dstq + call .main ; 0 + cmp eobd, 36 + jl .ret + add cq, 128*8-32 ; 0 1 + lea dstq, [r5+16] ; 1 + call .main + call .main2 + cmp eobd, 136 + jl .ret + add cq, 128*16-64 ; 0 1 2 + lea dstq, [r5+16*2] ; 1 2 + call .main ; 2 + call .main2 + call .main2 + cmp eobd, 300 + jl .ret + add cq, 128*24-96 ; 0 1 2 3 + add r5, 16*3 ; 1 2 3 + mov dstq, r5 ; 2 3 + call .main ; 3 + call .main2 + call .main2 + call .main2 + cmp eobd, 535 + jl .ret + add cq, 128*24-96 ; 0 1 2 3 + lea dstq, [r5+strideq*8] ; 1 2 3 4 + mov r5, dstq ; 2 3 4 + call .main ; 3 4 + call .main2 + call .main2 + cmp eobd, 755 + jl .ret + add cq, 128*16-64 ; 0 1 2 3 + lea dstq, [r5+strideq*8] ; 1 2 3 4 + mov r5, dstq ; 2 3 4 5 + call .main ; 3 4 5 + call .main2 + cmp eobd, 911 + jl .ret + add cq, 128*8-32 ; 0 1 2 3 + lea dstq, [r5+strideq*8] ; 1 2 3 4 + call .main ; 2 3 4 5 +.ret: ; 3 4 5 6 + RET +ALIGN function_align +.main2: + sub cq, 128*8 + sub dstq, 16 +.main: + mova m0, [cq+128*0] + packssdw m0, [cq+128*1] + mova m1, [cq+128*2] + packssdw m1, [cq+128*3] + mova m2, [cq+128*4] + packssdw m2, [cq+128*5] + mova m3, [cq+128*6] + packssdw m3, [cq+128*7] + REPX {pmulhrsw x, m5}, m0, m1, m2, m3 + call m(inv_txfm_add_identity_identity_8x32_16bpc).main_zero + lea dstq, [dstq+strideq*4] + add cq, 16 + btc eobd, 16 + jnc .main + ret + +cglobal inv_txfm_add_dct_dct_8x32_16bpc, 4, 7, 15, 0-36*16, \ + dst, stride, c, eob +%if ARCH_X86_32 + LEA r6, $$ +%define base $$ + DECLARE_REG_TMP 0, 4 +%else + lea r6, [tbl_Nx32_odd_offset] +%define base tbl_Nx32_odd_offset + DECLARE_REG_TMP 4, 7 +%if WIN64 + mov [rsp+gprsize*1+35*16], r7 +%endif +%endif +%define o2(x) r6-base+x + test eobd, eobd + jz .dconly + +%if ARCH_X86_32 + mov [rsp+gprsize*1+35*16], r0 +%endif +%undef cmp + ; remove entirely-zero iterations + mov r5d, 7*2 + cmp eobw, word [o2(tbl_8x32_2d)+r5] + jge .end_zero_loop + pxor m0, m0 +.zero_loop: + movzx t0d, word [o2(tbl_Nx32_odd_offset)+r5] + movzx t1d, t0b + shr t0d, 8 + mova [rsp+ 3*16+r5*8], m0 + mova [rsp+11*16+r5*8], m0 + mova [rsp+ 3*16+t0*8], m0 + mova [rsp+ 3*16+t1*8], m0 + sub r5d, 2 + cmp eobw, word [o2(tbl_8x32_2d)+r5] + jl .zero_loop +.end_zero_loop: + ; actual first pass after skipping all-zero data + mov [rsp+gprsize*0+35*16], eobd + mov r3, rsp +.loop_pass1: +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_min)] + mova m13, [o(clip_max)] + mova m14, [o(pd_2896)] +%endif + mova m0, [cq+0*128+r5*8] + mova m1, [cq+1*128+r5*8] + mova m2, [cq+2*128+r5*8] + mova m3, [cq+3*128+r5*8] + mova m4, [cq+4*128+r5*8] + mova m5, [cq+5*128+r5*8] + mova m6, [cq+6*128+r5*8] + mova m7, [cq+7*128+r5*8] + call m(idct_8x4_internal_16bpc).main_pass1 + mova m1, [o(pd_2)] + REPX {paddd x, m1}, m0, m6, m5, m3 + call m(idct_8x4_internal_16bpc).round + REPX {psrad x, 2 }, m0, m1, m2, m3, m4, m5, m6, m7 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + call m(idct_8x4_internal_16bpc).transpose4x8packed + + movzx t0d, word [o2(tbl_Nx32_odd_offset)+r5] + movzx t1d, t0b + shr t0d, 8 + mova [r3+ 3*16+r5*8], m0 + mova [r3+11*16+r5*8], m2 + mova [r3+ 3*16+t1*8], m1 + mova [r3+ 3*16+t0*8], m3 + pxor m7, m7 + REPX {mova [cq+x*128+r5*8], m7}, 0, 1, 2, 3, 4, 5, 6, 7 + sub r5d, 2 + jge .loop_pass1 + + ; pass 2 code starts here + ; m0 is already loaded from last iteration of first pass +%if ARCH_X86_32 + mov r0, [rsp+gprsize*1+35*16] +%endif + mov eobd, [rsp+gprsize*0+35*16] + cmp eobd, 43 + jl .load_veryfast + cmp eobd, 107 + jl .load_fast + ; load normal + lea r4, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main)] + jmp .run +.load_fast: + lea r4, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main_fast)] + jmp .run +.load_veryfast: + lea r4, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main_veryfast)] + ; fall-through +.run: + call .pass2 +%if WIN64 + mov r7, [rsp+gprsize*1+35*16] +%endif + RET + +.pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + mova m1, [rsp+gprsize+16* 4] + mova m2, [rsp+gprsize+16* 5] + mova m3, [rsp+gprsize+16* 6] + mova m4, [rsp+gprsize+16* 7] + mova m5, [rsp+gprsize+16* 8] + mova m6, [rsp+gprsize+16* 9] + mova m7, [rsp+gprsize+16*10] + call m_suffix(idct_8x8_internal_8bpc, _ssse3).main + mova [rsp+gprsize+ 3*16], m0 + mova [rsp+gprsize+ 4*16], m1 + mova [rsp+gprsize+ 5*16], m2 + mova [rsp+gprsize+ 6*16], m3 + mova [rsp+gprsize+ 7*16], m4 + mova [rsp+gprsize+ 8*16], m5 + mova [rsp+gprsize+ 9*16], m6 + mova m0, [rsp+gprsize+11*16] + mova m1, [rsp+gprsize+12*16] + mova m2, [rsp+gprsize+13*16] + mova m3, [rsp+gprsize+14*16] + mova m4, [rsp+gprsize+15*16] + mova m5, [rsp+gprsize+16*16] + mova m6, [rsp+gprsize+17*16] + mova m7, [rsp+gprsize+18*16] + call m_suffix(idct_16x8_internal_8bpc, _ssse3).main + mova m7, [rsp+gprsize+ 0*16] + mova [rsp+gprsize+11*16], m0 + mova [rsp+gprsize+12*16], m1 + mova [rsp+gprsize+13*16], m2 + mova [rsp+gprsize+14*16], m3 + mova [rsp+gprsize+15*16], m4 + mova [rsp+gprsize+16*16], m5 + mova [rsp+gprsize+17*16], m6 + mova [rsp+gprsize+18*16], m7 + call r4 +%if ARCH_X86_64 + mova m8, [o(pw_2048)] + pxor m9, m9 + mova m10, [o(pixel_10bpc_max)] +%endif + lea r3, [strideq*3] + call m(idct_8x8_internal_16bpc).round1_and_write_8x8 + lea dstq, [dstq+strideq*8] + mova m0, [rsp+gprsize+11*16] + mova m1, [rsp+gprsize+12*16] + mova m2, [rsp+gprsize+13*16] + mova m3, [rsp+gprsize+14*16] + mova m4, [rsp+gprsize+15*16] + mova m5, [rsp+gprsize+16*16] + mova m6, [rsp+gprsize+17*16] + mova m7, [rsp+gprsize+18*16] + call m(idct_8x8_internal_16bpc).round1_and_write_8x8 + lea dstq, [dstq+strideq*8] + mova m0, [rsp+gprsize+19*16] + mova m1, [rsp+gprsize+20*16] + mova m2, [rsp+gprsize+21*16] + mova m3, [rsp+gprsize+22*16] + mova m4, [rsp+gprsize+23*16] + mova m5, [rsp+gprsize+24*16] + mova m6, [rsp+gprsize+25*16] + mova m7, [rsp+gprsize+26*16] + call m(idct_8x8_internal_16bpc).round1_and_write_8x8 + lea dstq, [dstq+strideq*8] + mova m0, [rsp+gprsize+27*16] + mova m1, [rsp+gprsize+28*16] + mova m2, [rsp+gprsize+29*16] + mova m3, [rsp+gprsize+30*16] + mova m4, [rsp+gprsize+31*16] + mova m5, [rsp+gprsize+32*16] + mova m6, [rsp+gprsize+33*16] + mova m7, [rsp+gprsize+34*16] + call m(idct_8x8_internal_16bpc).round1_and_write_8x8 + ret +.dconly: + imul r5d, [cq], 2896 + mov [cq], eobd ; 0 + mov r3d, 8 + add r5d, 10240 + sar r5d, 14 + add rsp, (31+2*ARCH_X86_64)*16 + jmp m(inv_txfm_add_dct_dct_8x8_16bpc).end2 + +cglobal inv_txfm_add_dct_dct_16x32_16bpc, 4, 7, 16, 0-77*16, \ + dst, stride, c, eob + LEA r6, base + test eobd, eobd + jz .dconly + +%if ARCH_X86_32 + mov [rsp+gprsize*1+76*16], r0 +%elif WIN64 + mov [rsp+gprsize*1+76*16], r7 +%endif +%undef cmp + ; remove entirely-zero iterations + mov r5d, 7*2 + cmp eobw, word [o2(tbl_16x32_2d)+r5] + jge .end_zero_loop + pxor m0, m0 +.zero_loop: + movzx t0d, word [o2(tbl_Nx32_odd_offset)+r5] + movzx t1d, t0b + shr t0d, 8 + mova [rsp+12*16+r5*8], m0 + mova [rsp+20*16+r5*8], m0 + mova [rsp+12*16+t0*8], m0 + mova [rsp+12*16+t1*8], m0 + mova [rsp+44*16+r5*8], m0 + mova [rsp+52*16+r5*8], m0 + mova [rsp+44*16+t0*8], m0 + mova [rsp+44*16+t1*8], m0 + sub r5d, 2 + cmp eobw, word [o2(tbl_16x32_2d)+r5] + jl .zero_loop +.end_zero_loop: + ; actual first pass after skipping all-zero data + mov [rsp+gprsize*0+76*16], eobd + mov r3, rsp +.loop_pass1: +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_min)] + mova m13, [o(clip_max)] + mova m14, [o(pd_2896)] +%endif + mova m0, [cq+ 1*128+r5*8] + mova m1, [cq+ 3*128+r5*8] + mova m2, [cq+ 5*128+r5*8] + mova m3, [cq+ 7*128+r5*8] + mova m4, [cq+ 9*128+r5*8] + mova m5, [cq+11*128+r5*8] + mova m6, [cq+13*128+r5*8] + mova m7, [cq+15*128+r5*8] + call m(idct_8x4_internal_16bpc).rect2_mul + call m(idct_16x4_internal_16bpc).main_oddhalf + + mova m0, [cq+ 0*128+r5*8] + mova m1, [cq+ 2*128+r5*8] + mova m2, [cq+ 4*128+r5*8] + mova m3, [cq+ 6*128+r5*8] + mova m4, [cq+ 8*128+r5*8] + mova m5, [cq+10*128+r5*8] + mova m6, [cq+12*128+r5*8] + mova m7, [cq+14*128+r5*8] + call m(idct_8x4_internal_16bpc).rect2_mul + call m(idct_8x4_internal_16bpc).main_pass1 + call m(idct_8x4_internal_16bpc).round + call m(idct_16x4_internal_16bpc).round +%if ARCH_X86_64 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + packssdw m8, m9 + packssdw m10, m11 + packssdw m12, m13 + packssdw m14, m15 +%endif + call m(idct_8x4_internal_16bpc).transpose4x8packed + movzx t0d, word [o2(tbl_Nx32_odd_offset)+r5] + movzx t1d, t0b + shr t0d, 8 +%if ARCH_X86_64 + mova [rsp+12*16+r5*8], m0 + mova [rsp+20*16+r5*8], m2 + mova [rsp+12*16+t1*8], m1 + mova [rsp+12*16+t0*8], m3 + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [rsp+44*16+r5*8], m8 + mova [rsp+52*16+r5*8], m10 + mova [rsp+44*16+t1*8], m9 + mova [rsp+44*16+t0*8], m11 +%else + mova [rsp+44*16+r5*8], m0 + mova [rsp+52*16+r5*8], m2 + mova [rsp+44*16+t1*8], m1 + mova [rsp+44*16+t0*8], m3 + mova m0, [r3+ 8*16] + mova m2, [r3+ 9*16] + mova m4, [r3+10*16] + mova m6, [r3+11*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [rsp+12*16+r5*8], m0 + mova [rsp+20*16+r5*8], m2 + mova [rsp+12*16+t1*8], m1 + mova [rsp+12*16+t0*8], m3 +%endif + pxor m7, m7 + REPX {mova [cq+x*128+r5*8], m7}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + sub r5d, 2 + jge .loop_pass1 + + ; pass=2 + add rsp, 9*16 +%if ARCH_X86_64 + mov r6, dstq +%else + mov dstq, [rsp+gprsize*1+67*16] +%endif + mov eobd, [rsp+gprsize*0+67*16] + cmp eobd, 44 + jl .load_veryfast + cmp eobd, 151 + jl .load_fast + ; load normal + lea r4, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main)] + jmp .run +.load_fast: + lea r4, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main_fast)] + jmp .run +.load_veryfast: + lea r4, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main_veryfast)] + ; fall-through +.run: +%if ARCH_X86_64 + lea r2, [dstq+32] + mov r7, -4 +%else + lea r2, [rsp+67*16] + mov dword [r2+0*gprsize], 2 +%endif + jmp .loop_pass2_entry +.loop_pass2: + mova m0, [rsp+16* 3] +.loop_pass2_entry: +%if ARCH_X86_32 + mov dstq, [r2+1*gprsize] +%endif + call m(inv_txfm_add_dct_dct_8x32_16bpc).pass2 + add rsp, 32*16 +%if ARCH_X86_64 + add r7, 2 + lea dstq, [r2+r7*8] + jl .loop_pass2 +%if WIN64 + mov r7, [rsp+gprsize*1+3*16] +%endif +%else + add dword [r2+1*gprsize], 16 + dec dword [r2+0*gprsize] + jg .loop_pass2 +%endif +%assign stack_size (stack_size-73*16) +%if STACK_ALIGNMENT >= 16 +%assign stack_size_padded (stack_size_padded-73*16) +%assign stack_offset (stack_offset-73*16) +%else +%xdefine rstkm [rsp + stack_size] +%endif + RET +.dconly: + imul r5d, [cq], 2896 + mov [cq], eobd ; 0 + mov r3d, 32 + add r5d, 2048 + sar r5d, 12 + imul r5d, 2896 + add rsp, (65+4*ARCH_X86_64)*16 + jmp m(inv_txfm_add_dct_dct_16x4_16bpc).dconly + +cglobal inv_txfm_add_dct_dct_32x8_16bpc, 4, 7, 16, 0-(24+8*ARCH_X86_32)*16, \ + dst, stride, c, eob +%if ARCH_X86_32 + LEA r6, $$ +%endif + test eobd, eobd + jz .dconly + + ; remove entirely-zero iterations +%undef cmp +%if ARCH_X86_64 + xor r5d, r5d + cmp eobd, 10 + setge r5b +%else + mov r5d, 1 + cmp eobd, 10 + sbb r5d, 0 +%endif + add r5d, r5d + + ; actual first pass after skipping all-zero data +.loop_pass1: + mova m0, [cq+32* 1+r5*8] + mova m1, [cq+32* 7+r5*8] + mova m2, [cq+32* 9+r5*8] + mova m3, [cq+32*15+r5*8] + mova m4, [cq+32*17+r5*8] + mova m5, [cq+32*23+r5*8] + mova m6, [cq+32*25+r5*8] + mova m7, [cq+32*31+r5*8] +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_min)] + mova m13, [o(clip_max)] + mova m14, [o(pd_2896)] +%endif + mov r3, rsp + call .main_oddhalf_part1 + mova m0, [cq+32* 3+r5*8] + mova m1, [cq+32* 5+r5*8] + mova m2, [cq+32*11+r5*8] + mova m3, [cq+32*13+r5*8] + mova m4, [cq+32*19+r5*8] + mova m5, [cq+32*21+r5*8] + mova m6, [cq+32*27+r5*8] + mova m7, [cq+32*29+r5*8] + call .main_oddhalf_part2 + mova m0, [cq+32* 2+r5*8] + mova m1, [cq+32* 6+r5*8] + mova m2, [cq+32*10+r5*8] + mova m3, [cq+32*14+r5*8] + mova m4, [cq+32*18+r5*8] + mova m5, [cq+32*22+r5*8] + mova m6, [cq+32*26+r5*8] + mova m7, [cq+32*30+r5*8] + add r3, 16*(16+4*ARCH_X86_32) + call m(idct_16x4_internal_16bpc).main_oddhalf + mova m0, [cq+32* 0+r5*8] + mova m1, [cq+32* 4+r5*8] + mova m2, [cq+32* 8+r5*8] + mova m3, [cq+32*12+r5*8] + mova m4, [cq+32*16+r5*8] + mova m5, [cq+32*20+r5*8] + mova m6, [cq+32*24+r5*8] + mova m7, [cq+32*28+r5*8] + call m(idct_8x4_internal_16bpc).main_pass1 + call m(idct_8x4_internal_16bpc).round + sub r3, 16*(16+4*ARCH_X86_32) + call .round_dct32 +%if ARCH_X86_64 + call m(idct_8x4_internal_16bpc).transpose4x8packed + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [cq+32* 8+r5*8], m8 + mova [cq+32* 9+r5*8], m9 + mova [cq+32*10+r5*8], m10 + mova [cq+32*11+r5*8], m11 + mova m8, [r3+16* 9] ; 8 9 + mova m10, [r3+16*11] ; 10 11 + mova m12, [r3+16*13] ; 12 13 + mova m14, [r3+16*15] ; 14 15 + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [cq+32* 4+r5*8], m8 + mova [cq+32* 5+r5*8], m9 + mova [cq+32* 6+r5*8], m10 + mova [cq+32* 7+r5*8], m11 + mova m8, [r3+16* 8] ; 24 25 + mova m10, [r3+16*10] ; 26 27 + mova m12, [r3+16*12] ; 28 29 + mova m14, [r3+16*14] ; 30 31 + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [cq+32*12+r5*8], m8 + mova [cq+32*13+r5*8], m9 + mova [cq+32*14+r5*8], m10 + mova [cq+32*15+r5*8], m11 +%else + sub r3, 8*16 + mova m0, [r3+ 8*16] + mova m2, [r3+10*16] + mova m4, [r3+12*16] + mova m6, [r3+14*16] + packssdw m0, [r3+ 9*16] + packssdw m2, [r3+11*16] + packssdw m4, [r3+13*16] + packssdw m6, [r3+15*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [cq+32* 4+r5*8], m0 + mova [cq+32* 5+r5*8], m1 + mova [cq+32* 6+r5*8], m2 + mova [cq+32* 7+r5*8], m3 + mova m0, [r3+16*16] + mova m2, [r3+18*16] + mova m4, [r3+20*16] + mova m6, [r3+22*16] + packssdw m0, [r3+17*16] + packssdw m2, [r3+19*16] + packssdw m4, [r3+21*16] + packssdw m6, [r3+23*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [cq+32* 8+r5*8], m0 + mova [cq+32* 9+r5*8], m1 + mova [cq+32*10+r5*8], m2 + mova [cq+32*11+r5*8], m3 + mova m0, [r3+31*16] + mova m2, [r3+29*16] + mova m4, [r3+27*16] + mova m6, [r3+25*16] + packssdw m0, [r3+30*16] + packssdw m2, [r3+28*16] + packssdw m4, [r3+26*16] + packssdw m6, [r3+24*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [cq+32*12+r5*8], m0 + mova [cq+32*13+r5*8], m1 + mova [cq+32*14+r5*8], m2 + mova [cq+32*15+r5*8], m3 + mova m0, [r3+ 0*16] + mova m2, [r3+ 2*16] + mova m4, [r3+ 4*16] + mova m6, [r3+ 6*16] + packssdw m0, [r3+ 1*16] + packssdw m2, [r3+ 3*16] + packssdw m4, [r3+ 5*16] + packssdw m6, [r3+ 7*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed +%endif + pxor m7, m7 + ; clear lower half of [cq] + REPX {mova [cq+x*32+r5*8], m7}, 16, 17, 18, 19, 20, 21, 22, 23, \ + 24, 25, 26, 27, 28, 29, 30, 31 + test r5d, r5d + jz .end_pass1 + mova [cq+32* 0+r5*8], m0 + mova [cq+32* 1+r5*8], m1 + mova [cq+32* 2+r5*8], m2 + mova [cq+32* 3+r5*8], m3 + sub r5d, 2 + jmp .loop_pass1 +.end_pass1: + + ; pass=2, we need to call this otherwise the stack pointer has + ; the wrong offset in the 8-bit code + mov r4d, 4 + call m(idct_16x8_internal_16bpc).pass2_main + RET + +.main_oddhalf_part1_fast: ; lower half zero + pmulld m7, m0, [o(pd_4091)] + pmulld m0, [o(pd_201)] + pmulld m4, m3, [o(pd_m2751)] +%if ARCH_X86_32 + pmulld m3, [o(pd_3035)] + mova m5, [o(pd_2048)] + REPX {paddd x, m5}, m0, m7 + REPX {psrad x, 12}, m0, m7 + mova [r3+3*16], m7 + mova m7, m3 + mova m3, m5 +%else + pmulld m3, [o(pd_3035)] +%endif + pmulld m6, m1, [o(pd_m1380)] + pmulld m1, [o(pd_3857)] + pmulld m5, m2, [o(pd_3703)] + pmulld m2, [o(pd_1751)] + jmp .main_oddhalf_part1_fast2 +.main_oddhalf_part1: ; in1, in7, in9, in15, in17, in23, in25, in31 +%if ARCH_X86_64 + ITX_MULSUB_2D 0, 7, 8, 9, 10, _, 201, 4091 ; t16a, t31a + ITX_MULSUB_2D 6, 1, 8, 9, 10, _, 3857, 1380 ; t19a, t28a + ITX_MULSUB_2D 2, 5, 8, 9, 10, _, 1751, 3703 ; t18a, t29a + ITX_MULSUB_2D 4, 3, 8, 9, 10, _, 3035, 2751 ; t17a, t30a +.main_oddhalf_part1_fast2: + REPX {paddd x, m11}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {psrad x, 12 }, m0, m1, m2, m3, m4, m5, m6, m7 + psubd m8, m0, m4 ; t17 + paddd m0, m4 ; t16 + psubd m4, m6, m2 ; t18 + paddd m6, m2 ; t19 + psubd m2, m1, m5 ; t29 + paddd m1, m5 ; t28 + psubd m5, m7, m3 ; t30 + paddd m7, m3 ; t31 + REPX {pmaxsd x, m12}, m8, m5, m4, m2, m0, m6, m1, m7 + REPX {pminsd x, m13}, m8, m5, m4, m2, m0, m6, m1, m7 + mova m15, [o(pd_4017)] + mova m10, [o(pd_799)] + ITX_MULSUB_2D 5, 8, 3, 9, _, 11, 10, 15 ; t17a, t30a + ITX_MULSUB_2D 2, 4, 3, 9, _, 11, 10, 15, 4 ; t29a, t18a + psubd m3, m0, m6 ; t19a + paddd m0, m6 ; t16a + psubd m6, m7, m1 ; t28a + paddd m7, m1 ; t31a + psubd m1, m5, m4 ; t18 + paddd m5, m4 ; t17 + psubd m4, m8, m2 ; t29 + paddd m8, m2 ; t30 + REPX {pmaxsd x, m12}, m3, m6, m1, m4, m0, m7, m5, m8 + REPX {pminsd x, m13}, m3, m6, m1, m4, m0, m7, m5, m8 + mova m15, [o(pd_3784)] + mova m10, [o(pd_1567)] + ITX_MULSUB_2D 4, 1, 2, 9, _, 11, 10, 15 ; t18a, t29a + ITX_MULSUB_2D 6, 3, 2, 9, _, 11, 10, 15 ; t19, t28 + mova [r3+16*0], m0 + mova [r3+16*1], m5 + mova [r3+16*2], m4 + mova [r3+16*3], m6 + mova [r3+16*4], m3 + mova [r3+16*5], m1 + mova [r3+16*6], m8 + mova [r3+16*7], m7 +%else + mova [r3+0*16], m2 + mova [r3+1*16], m3 + mova [r3+2*16], m4 + mova [r3+3*16], m5 + mova m3, [o(pd_2048)] + ITX_MULSUB_2D 0, 7, 2, 4, 5, 3, 201, 4091 ; t16a, t31a + ITX_MULSUB_2D 6, 1, 2, 4, 5, _, 3857, 1380 ; t19a, t28a + mova m4, [r3+2*16] + mova m5, [r3+3*16] + mova [r3+2*16], m6 + mova [r3+3*16], m7 + mova m2, [r3+0*16] + mova m7, [r3+1*16] + mova [r3+0*16], m0 + mova [r3+1*16], m1 + ITX_MULSUB_2D 2, 5, 0, 1, 6, _, 1751, 3703 ; t18a, t29a + ITX_MULSUB_2D 4, 7, 0, 1, 6, _, 3035, 2751 ; t17a, t30a + mova m0, [r3+0*16] + mova m1, [r3+1*16] + mova m6, [r3+2*16] +.main_oddhalf_part1_fast2: + REPX {paddd x, m3}, m1, m2, m4, m5, m6, m7 + REPX {psrad x, 12}, m1, m2, m4, m5, m6, m7 + psubd m3, m0, m4 ; t17 + mova [r3+0*16], m3 + mova m3, [r3+3*16] + paddd m0, m4 ; t16 + psubd m4, m6, m2 ; t18 + paddd m6, m2 ; t19 + psubd m2, m1, m5 ; t29 + paddd m1, m5 ; t28 + psubd m5, m3, m7 ; t30 + paddd m7, m3 ; t31 + mova m3, [o(clip_min)] + REPX {pmaxsd x, m3}, m5, m4, m2, m0, m6, m1, m7 + pmaxsd m3, [r3+0*16] + mova [r3+0*16], m3 + mova m3, [o(clip_max)] + REPX {pminsd x, m3}, m5, m4, m2, m0, m6, m1, m7 + pminsd m3, [r3+0*16] + mova [r3+0*16], m0 + mova [r3+1*16], m1 + mova [r3+2*16], m6 + mova [r3+3*16], m7 + mova m0, [o(pd_2048)] + ITX_MULSUB_2D 5, 3, 1, 6, 7, 0, 799, 4017 ; t17a, t30a + ITX_MULSUB_2D 2, 4, 1, 6, _, 0, 7, 4017, 4 ; t29a, t18a + psubd m1, m5, m4 ; t18 + paddd m5, m4 ; t17 + psubd m4, m3, m2 ; t29 + paddd m3, m2 ; t30 + mova m0, [r3+0*16] + mova m2, [r3+1*16] + mova m6, [r3+2*16] + mova m7, [r3+3*16] + mova [r3+0*16], m3 + psubd m3, m0, m6 ; t19a + paddd m0, m6 ; t16a + psubd m6, m7, m2 ; t28a + paddd m7, m2 ; t31a + mova m2, [o(clip_min)] + REPX {pmaxsd x, m2}, m3, m6, m1, m4, m0, m7, m5 + pmaxsd m2, [r3+0*16] + mova [r3+0*16], m2 + mova m2, [o(clip_max)] + REPX {pminsd x, m2}, m3, m6, m1, m4, m0, m7, m5 + pminsd m2, [r3+0*16] + mova [r3+16*0], m0 + mova [r3+16*1], m5 + mova [r3+16*6], m2 + mova [r3+16*7], m7 + mova m7, [o(pd_2048)] + ITX_MULSUB_2D 4, 1, 0, 5, 2, 7, 1567, 3784 ; t18a, t29a + ITX_MULSUB_2D 6, 3, 0, 5, 2, 7, 2, 3784 ; t19, t28 + mova [r3+16*2], m4 + mova [r3+16*3], m6 + mova [r3+16*4], m3 + mova [r3+16*5], m1 +%endif + ret +.main_oddhalf_part2_fast: ; lower half zero + pmulld m7, m0, [o(pd_m601)] + pmulld m0, [o(pd_4052)] + pmulld m4, m3, [o(pd_3290)] +%if ARCH_X86_32 + pmulld m3, [o(pd_2440)] + mova m5, [o(pd_2048)] + REPX {paddd x, m5}, m0, m7 + REPX {psrad x, 12}, m0, m7 + mova [r3+11*16], m7 + mova m7, m3 + mova m3, m5 +%else + pmulld m3, [o(pd_2440)] +%endif + pmulld m6, m1, [o(pd_3973)] + pmulld m1, [o(pd_995)] + pmulld m5, m2, [o(pd_m2106)] + pmulld m2, [o(pd_3513)] + jmp .main_oddhalf_part2_fast2 +.main_oddhalf_part2: ; in3, in5, in11, in13, in19, in21, in27, in29 +%if ARCH_X86_64 + ITX_MULSUB_2D 7, 0, 8, 9, 10, _, 4052, 601 ; t23a, t24a + ITX_MULSUB_2D 1, 6, 8, 9, 10, _, 995, 3973 ; t20a, t27a + ITX_MULSUB_2D 5, 2, 8, 9, 10, _, 3513, 2106 ; t21a, t26a + ITX_MULSUB_2D 3, 4, 8, 9, 10, _, 2440, 3290 ; t22a, t25a +.main_oddhalf_part2_fast2: + REPX {paddd x, m11}, m0, m1, m2, m3, m4, m5, m6, m7 + REPX {psrad x, 12 }, m0, m1, m2, m3, m4, m5, m6, m7 + psubd m8, m0, m4 ; t25 + paddd m0, m4 ; t24 + psubd m4, m6, m2 ; t26 + paddd m6, m2 ; t27 + psubd m2, m1, m5 ; t21 + paddd m1, m5 ; t20 + psubd m5, m7, m3 ; t22 + paddd m7, m3 ; t23 + REPX {pmaxsd x, m12}, m8, m5, m4, m2, m0, m6, m1, m7 + REPX {pminsd x, m13}, m8, m5, m4, m2, m0, m6, m1, m7 + mova m15, [o(pd_2276)] + mova m10, [o(pd_3406)] + ITX_MULSUB_2D 4, 2, 3, 9, _, 11, 10, 15 ; t21a, t26a + ITX_MULSUB_2D 8, 5, 3, 9, _, 11, 10, 15, 4 ; t25a, t22a + psubd m3, m0, m6 ; t27a + paddd m0, m6 ; t24a + psubd m6, m7, m1 ; t20a + paddd m7, m1 ; t23a + psubd m1, m5, m4 ; t21 + paddd m5, m4 ; t22 + psubd m4, m8, m2 ; t26 + paddd m8, m2 ; t25 + REPX {pmaxsd x, m12}, m3, m6, m1, m4, m0, m7, m5, m8 + REPX {pminsd x, m13}, m3, m6, m1, m4, m0, m7, m5, m8 + mova m15, [o(pd_3784)] + mova m10, [o(pd_1567)] + ITX_MULSUB_2D 4, 1, 2, 9, _, 11, 10, 15, 4 ; t26a, t21a + ITX_MULSUB_2D 3, 6, 2, 9, _, 11, 10, 15, 4 ; t27, t20 + mova m9, [r3+16*0] ; t16a + mova m10, [r3+16*1] ; t17 + psubd m2, m9, m7 ; t23 + paddd m9, m7 ; t16 + psubd m7, m10, m5 ; t22a + paddd m10, m5 ; t17a + REPX {pmaxsd x, m12}, m9, m10, m2, m7 + REPX {pminsd x, m13}, m9, m10, m2, m7 + mova [r3+16*0], m9 + mova [r3+16*1], m10 + mova m9, [r3+16*2] ; t18a + mova m10, [r3+16*3] ; t19 + psubd m5, m9, m1 ; t21 + paddd m9, m1 ; t18 + psubd m1, m10, m6 ; t20a + paddd m10, m6 ; t19a + REPX {pmaxsd x, m12}, m9, m10, m5, m1 + REPX {pminsd x, m13}, m9, m10, m5, m1 + mova [r3+16*2], m9 + mova [r3+16*3], m10 + mova m9, [r3+16*4] ; t28 + mova m10, [r3+16*5] ; t29a + psubd m6, m9, m3 ; t27a + paddd m9, m3 ; t28a + psubd m3, m10, m4 ; t26 + paddd m10, m4 ; t29 + REPX {pmaxsd x, m12}, m9, m10, m6, m3 + REPX {pminsd x, m13}, m9, m10, m6, m3 + REPX {pmulld x, m14}, m6, m3, m1, m5 + paddd m6, m11 + paddd m3, m11 + psubd m4, m6, m1 ; t20 + paddd m6, m1 ; t27 + psubd m1, m3, m5 ; t21a + paddd m3, m5 ; t26a + REPX {psrad x, 12 }, m4, m1, m3, m6 + mova [r3+16*4], m4 + mova [r3+16*5], m1 + mova m4, [r3+16*6] ; t30 + mova m1, [r3+16*7] ; t31a + psubd m5, m4, m8 ; t25a + paddd m4, m8 ; t30a + psubd m8, m1, m0 ; t24 + paddd m1, m0 ; t31 + REPX {pmaxsd x, m12}, m8, m5, m4, m1 + REPX {pminsd x, m13}, m8, m5, m4, m1 + REPX {pmulld x, m14}, m5, m8, m7, m2 + paddd m5, m11 + paddd m8, m11 + psubd m0, m5, m7 ; t22 + paddd m5, m7 ; t25 + psubd m7, m8, m2 ; t23a + paddd m2, m8 ; t24a + REPX {psrad x, 12 }, m0, m7, m2, m5 + mova [r3+16*6], m0 + mova [r3+16*7], m7 + mova [r3+16*8], m2 + mova [r3+16*9], m5 + mova [r3+16*10], m3 + mova [r3+16*11], m6 + mova [r3+16*12], m9 + mova [r3+16*13], m10 + mova [r3+16*14], m4 + mova [r3+16*15], m1 +%else + mova [r3+ 8*16], m2 + mova [r3+ 9*16], m3 + mova [r3+10*16], m4 + mova [r3+11*16], m5 + mova m3, [o(pd_2048)] + ITX_MULSUB_2D 7, 0, 2, 4, 5, 3, 4052, 601 ; t23a, t24a + ITX_MULSUB_2D 1, 6, 2, 4, 5, _, 995, 3973 ; t20a, t27a + mova m2, [r3+ 8*16] + mova m4, [r3+10*16] + mova m5, [r3+11*16] + mova [r3+ 8*16], m0 + mova [r3+10*16], m6 + mova [r3+11*16], m7 + mova m7, [r3+ 9*16] + mova [r3+ 9*16], m1 + ITX_MULSUB_2D 5, 2, 0, 6, 1, _, 3513, 2106 ; t21a, t26a + ITX_MULSUB_2D 7, 4, 0, 6, 1, _, 2440, 3290 ; t22a, t25a + mova m0, [r3+ 8*16] + mova m1, [r3+ 9*16] + mova m6, [r3+10*16] +.main_oddhalf_part2_fast2: + REPX {paddd x, m3}, m1, m2, m7, m4, m5, m6 + REPX {psrad x, 12}, m1, m2, m7, m4, m5, m6 + psubd m3, m0, m4 ; t25 + mova [r3+ 8*16], m3 + mova m3, [r3+11*16] + paddd m0, m4 ; t24 + psubd m4, m6, m2 ; t26 + paddd m6, m2 ; t27 + psubd m2, m1, m5 ; t21 + paddd m1, m5 ; t20 + psubd m5, m3, m7 ; t22 + paddd m7, m3 ; t23 + mova m3, [o(clip_min)] + REPX {pmaxsd x, m3}, m5, m4, m2, m0, m6, m1, m7 + pmaxsd m3, [r3+ 8*16] + mova [r3+ 8*16], m3 + mova m3, [o(clip_max)] + REPX {pminsd x, m3}, m5, m4, m2, m0, m6, m1, m7 + pminsd m3, [r3+ 8*16] + mova [r3+ 8*16], m0 + mova [r3+ 9*16], m1 + mova [r3+10*16], m6 + mova [r3+11*16], m7 + mova m7, [o(pd_2048)] + ITX_MULSUB_2D 4, 2, 0, 1, 6, 7, 3406, 2276 ; t21a, t26a + ITX_MULSUB_2D 3, 5, 0, 1, _, 7, 6, 2276, 4 ; t25a, t22a + psubd m1, m5, m4 ; t21 + paddd m5, m4 ; t22 + psubd m4, m3, m2 ; t26 + paddd m3, m2 ; t25 + mova m0, [r3+ 8*16] + mova m2, [r3+ 9*16] + mova m6, [r3+10*16] + mova m7, [r3+11*16] + mova [r3+ 8*16], m3 + psubd m3, m0, m6 ; t27a + paddd m0, m6 ; t24a + psubd m6, m7, m2 ; t20a + paddd m7, m2 ; t23a + mova m2, [o(clip_min)] + REPX {pmaxsd x, m2}, m3, m6, m1, m4, m0, m7, m5 + pmaxsd m2, [r3+ 8*16] + mova [r3+ 8*16], m2 + mova m2, [o(clip_max)] + REPX {pminsd x, m2}, m3, m6, m1, m4, m0, m7, m5 + pminsd m2, [r3+ 8*16] + mova [r3+ 8*16], m0 + mova [r3+ 9*16], m2 + mova [r3+14*16], m5 + mova [r3+15*16], m7 + mova m0, [o(pd_2048)] + ITX_MULSUB_2D 4, 1, 2, 5, 7, 0, 1567, 3784, 4 ; t26a, t21a + ITX_MULSUB_2D 3, 6, 2, 5, _, 0, 7, 3784, 4 ; t27, t20 + mova [r3+10*16], m3 + mova m0, [o(clip_min)] + mova m2, [o(clip_max)] + mova m5, [r3+16*2] ; t18a + mova m7, [r3+16*3] ; t19 + psubd m3, m5, m1 ; t21 + paddd m5, m1 ; t18 + psubd m1, m7, m6 ; t20a + paddd m7, m6 ; t19a + REPX {pmaxsd x, m0}, m5, m7, m3, m1 + REPX {pminsd x, m2}, m5, m7, m3, m1 + mova [r3+16*2], m5 + mova [r3+16*3], m7 + mova [r3+11*16], m3 + mova m3, [r3+10*16] + mova m5, [r3+16*4] ; t28 + mova m7, [r3+16*5] ; t29a + psubd m6, m5, m3 ; t27a + paddd m5, m3 ; t28a + psubd m3, m7, m4 ; t26 + paddd m7, m4 ; t29 + REPX {pmaxsd x, m0}, m5, m7, m6, m3 + REPX {pminsd x, m2}, m5, m7, m6, m3 + mova [r3+16*12], m5 + mova [r3+16*13], m7 + mova m5, [o(pd_2048)] + mova m7, [o(pd_2896)] + mova m4, [r3+11*16] + REPX {pmulld x, m7}, m6, m3, m1, m4 + paddd m6, m5 + paddd m3, m5 + psubd m5, m6, m1 ; t20 + paddd m6, m1 ; t27 + psubd m1, m3, m4 ; t21a + paddd m3, m4 ; t26a + REPX {psrad x, 12}, m5, m1, m3, m6 + mova [r3+16*4], m5 + mova [r3+16*5], m1 + mova [r3+16*10], m3 + mova [r3+16*11], m6 + + mova m5, [r3+14*16] + mova m6, [r3+15*16] + mova m3, [r3+16*0] ; t16a + mova m4, [r3+16*1] ; t17 + psubd m1, m3, m6 ; t23 + paddd m3, m6 ; t16 + psubd m6, m4, m5 ; t22a + paddd m4, m5 ; t17a + REPX {pmaxsd x, m0}, m3, m4, m1, m6 + REPX {pminsd x, m2}, m3, m4, m1, m6 + mova [r3+16*0], m3 + mova [r3+16*1], m4 + mova m5, [r3+ 8*16] + mova m3, [r3+ 9*16] + mova [r3+ 8*16], m1 + mova [r3+ 9*16], m6 + mova m4, [r3+16*6] ; t30 + mova m1, [r3+16*7] ; t31a + psubd m6, m1, m5 ; t24 + paddd m1, m5 ; t31 + psubd m5, m4, m3 ; t25a + paddd m4, m3 ; t30a + REPX {pmaxsd x, m0}, m6, m5, m4, m1 + REPX {pminsd x, m2}, m6, m5, m4, m1 + mova [r3+16*14], m4 + mova [r3+16*15], m1 + mova m4, [o(pd_2048)] + mova m1, [r3+ 9*16] + mova m2, [r3+ 8*16] + REPX {pmulld x, m7}, m5, m6, m1, m2 + paddd m5, m4 + paddd m6, m4 + psubd m0, m5, m1 ; t22 + paddd m5, m1 ; t25 + psubd m1, m6, m2 ; t23a + paddd m2, m6 ; t24a + REPX {psrad x, 12}, m0, m1, m2, m5 + mova [r3+16*6], m0 + mova [r3+16*7], m1 + mova [r3+16*8], m2 + mova [r3+16*9], m5 +%endif + ret + + ; final sumsub for idct16 as well as idct32, plus final downshift +%macro IDCT32_END 6 ; in/out1, out2-4, tmp, shift, idx + mova m%4, [r3+16*(23-%1)] + psubd m%3, m%1, m%4 ; idct16 out15 - n + paddd m%1, m%4 ; idct16 out0 + n + pmaxsd m%1, m12 + pmaxsd m%3, m12 + pminsd m%1, m13 + pminsd m%3, m13 + paddd m%1, m11 + paddd m%3, m11 + mova m%5, [r3+16*( 0+%1)] + mova m%2, [r3+16*(15-%1)] + psubd m%4, m%1, m%2 ; out31 - n + paddd m%1, m%2 ; out0 + n + paddd m%2, m%3, m%5 ; out15 - n + psubd m%3, m%5 ; out16 + n + REPX {psrad x, %6}, m%1, m%3, m%2, m%4 +%endmacro + +.round_dct32: +%if ARCH_X86_64 + psrld m11, 10 ; pd_2 + IDCT32_END 0, 15, 8, 9, 10, 2 ; 0 15 16 31 + mova [r3+ 0*16], m6 + mova [r3+23*16], m7 + IDCT32_END 1, 14, 6, 7, 10, 2 ; 1 14 17 30 + packssdw m0, m1 ; 0 1 + packssdw m14, m15 ; 14 15 + packssdw m8, m6 ; 16 17 + packssdw m7, m9 ; 30 31 + mova [r3+16*15], m14 + mova [r3+16*14], m7 + IDCT32_END 2, 15, 10, 7, 6, 2 ; 2 13 18 29 + IDCT32_END 3, 14, 1, 9, 6, 2 ; 3 12 19 28 + packssdw m2, m3 ; 2 3 + packssdw m14, m15 ; 12 13 + packssdw m10, m1 ; 18 19 + packssdw m9, m7 ; 28 29 + mova [r3+16*13], m14 + mova [r3+16*12], m9 + IDCT32_END 4, 15, 1, 7, 6, 2 ; 4 11 20 27 + IDCT32_END 5, 14, 3, 9, 6, 2 ; 5 10 21 26 + packssdw m4, m5 ; 4 5 + packssdw m14, m15 ; 10 11 + packssdw m1, m3 ; 20 21 + packssdw m9, m7 ; 26 27 + mova [r3+16*11], m14 + mova [r3+16*10], m9 + mova m6, [r3+ 0*16] + mova m7, [r3+23*16] + IDCT32_END 6, 15, 14, 5, 3, 2 ; 6 9 22 25 + IDCT32_END 7, 11, 3, 9, 13, 2 ; 7 8 23 24 + packssdw m6, m7 ; 6 7 + packssdw m11, m15 ; 8 9 + packssdw m14, m3 ; 22 23 + packssdw m9, m5 ; 24 25 + mova [r3+16*9], m11 + mova [r3+16*8], m9 + mova m12, m1 + ret +%else + mova [r3+16*16], m0 + mova [r3+17*16], m1 + mova [r3+18*16], m2 + mova [r3+19*16], m3 + mova [r3+20*16], m4 + mova [r3+21*16], m5 + mova [r3+22*16], m6 + mova [r3+23*16], m7 + mova m1, [o(pd_2)] + mova m2, [o(clip_min)] + mova m3, [o(clip_max)] + + mov r4, 15*16 +.loop_dct32_end: + mova m0, [r3+16*16] + mova m6, [r3+16*24] + psubd m5, m0, m6 ; idct16 out15 - n + paddd m0, m6 ; idct16 out0 + n + pmaxsd m0, m2 + pmaxsd m5, m2 + pminsd m0, m3 + pminsd m5, m3 + paddd m0, m1 + paddd m5, m1 + mova m7, [r3] + mova m4, [r3+r4] + psubd m6, m0, m4 ; out31 - n + paddd m0, m4 ; out0 + n + paddd m4, m5, m7 ; out15 - n + psubd m5, m7 ; out16 + n + REPX {psrad x, 2}, m0, m5, m4, m6 + mova [r3], m0 + mova [r3+r4], m4 + mova [r3+16*16], m5 + mova [r3+24*16], m6 + add r3, 16 + sub r4, 32 + jg .loop_dct32_end + ret +%endif + +.dconly: + imul r5d, [cq], 2896 + mov [cq], eobd ; 0 + mov r3d, 8 +.dconly1: + add r5d, 10240 + sar r5d, 14 +.dconly2: + imul r5d, 2896 + add r5d, 34816 + movd m0, r5d + pshuflw m0, m0, q1111 + punpcklqdq m0, m0 + mova m6, [o(pixel_10bpc_max)] + pxor m5, m5 +.dconly_loop: + mova m1, [dstq+16*0] + mova m2, [dstq+16*1] + mova m3, [dstq+16*2] + mova m4, [dstq+16*3] + REPX {paddw x, m0}, m1, m2, m3, m4 + REPX {pminsw x, m6}, m1, m2, m3, m4 + REPX {pmaxsw x, m5}, m1, m2, m3, m4 + mova [dstq+16*0], m1 + mova [dstq+16*1], m2 + mova [dstq+16*2], m3 + mova [dstq+16*3], m4 + add dstq, strideq + dec r3d + jg .dconly_loop + RET + +cglobal inv_txfm_add_dct_dct_32x16_16bpc, 4, 7, 16, 0-(24+8*ARCH_X86_32)*16, \ + dst, stride, c, eob + LEA r6, base + test eobd, eobd + jz .dconly + + ; remove entirely-zero iterations +%undef cmp + mov r5d, 8 +.zero_loop: + sub r5d, 2 + cmp eobw, word [o2(tbl_32x16_2d)+r5] + jl .zero_loop + + ; actual first pass after skipping all-zero data +.loop_pass1: +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_min)] + mova m13, [o(clip_max)] + mova m14, [o(pd_2896)] +%endif + mova m0, [cq+64* 1+r5*8] + mova m1, [cq+64* 7+r5*8] + mova m2, [cq+64* 9+r5*8] + mova m3, [cq+64*15+r5*8] + mova m4, [cq+64*17+r5*8] + mova m5, [cq+64*23+r5*8] + mova m6, [cq+64*25+r5*8] + mova m7, [cq+64*31+r5*8] + mov r3, rsp + call m(idct_8x4_internal_16bpc).rect2_mul + call m(inv_txfm_add_dct_dct_32x8_16bpc).main_oddhalf_part1 + + mova m0, [cq+64* 3+r5*8] + mova m1, [cq+64* 5+r5*8] + mova m2, [cq+64*11+r5*8] + mova m3, [cq+64*13+r5*8] + mova m4, [cq+64*19+r5*8] + mova m5, [cq+64*21+r5*8] + mova m6, [cq+64*27+r5*8] + mova m7, [cq+64*29+r5*8] +%if ARCH_X86_32 + add r3, 16*8 +%endif + call m(idct_8x4_internal_16bpc).rect2_mul +%if ARCH_X86_32 + sub r3, 16*8 +%endif + call m(inv_txfm_add_dct_dct_32x8_16bpc).main_oddhalf_part2 + add r3, 16*(16+4*ARCH_X86_32) + + mova m0, [cq+64* 2+r5*8] + mova m1, [cq+64* 6+r5*8] + mova m2, [cq+64*10+r5*8] + mova m3, [cq+64*14+r5*8] + mova m4, [cq+64*18+r5*8] + mova m5, [cq+64*22+r5*8] + mova m6, [cq+64*26+r5*8] + mova m7, [cq+64*30+r5*8] + call m(idct_8x4_internal_16bpc).rect2_mul + call m(idct_16x4_internal_16bpc).main_oddhalf + + mova m0, [cq+64* 0+r5*8] + mova m1, [cq+64* 4+r5*8] + mova m2, [cq+64* 8+r5*8] + mova m3, [cq+64*12+r5*8] + mova m4, [cq+64*16+r5*8] + mova m5, [cq+64*20+r5*8] + mova m6, [cq+64*24+r5*8] + mova m7, [cq+64*28+r5*8] + call m(idct_8x4_internal_16bpc).rect2_mul + call m(idct_8x4_internal_16bpc).main_pass1 + call m(idct_8x4_internal_16bpc).round + sub r3, 16*(16+4*ARCH_X86_32) + call .round_dct32 + +%if ARCH_X86_64 + call m(idct_8x4_internal_16bpc).transpose4x8packed + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [cq+64* 8+r5*8], m8 + mova [cq+64* 9+r5*8], m9 + mova [cq+64*10+r5*8], m10 + mova [cq+64*11+r5*8], m11 + mova m8, [r3+16* 9] ; 8 9 + mova m10, [r3+16*11] ; 10 11 + mova m12, [r3+16*13] ; 12 13 + mova m14, [r3+16*15] ; 14 15 + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [cq+64* 4+r5*8], m8 + mova [cq+64* 5+r5*8], m9 + mova [cq+64* 6+r5*8], m10 + mova [cq+64* 7+r5*8], m11 + mova m8, [r3+16* 8] ; 24 25 + mova m10, [r3+16*10] ; 26 27 + mova m12, [r3+16*12] ; 28 29 + mova m14, [r3+16*14] ; 30 31 + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [cq+64*12+r5*8], m8 + mova [cq+64*13+r5*8], m9 + mova [cq+64*14+r5*8], m10 + mova [cq+64*15+r5*8], m11 +%else + sub r3, 8*16 + mova m0, [r3+ 8*16] + mova m2, [r3+10*16] + mova m4, [r3+12*16] + mova m6, [r3+14*16] + packssdw m0, [r3+ 9*16] + packssdw m2, [r3+11*16] + packssdw m4, [r3+13*16] + packssdw m6, [r3+15*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [cq+64* 4+r5*8], m0 + mova [cq+64* 5+r5*8], m1 + mova [cq+64* 6+r5*8], m2 + mova [cq+64* 7+r5*8], m3 + mova m0, [r3+16*16] + mova m2, [r3+18*16] + mova m4, [r3+20*16] + mova m6, [r3+22*16] + packssdw m0, [r3+17*16] + packssdw m2, [r3+19*16] + packssdw m4, [r3+21*16] + packssdw m6, [r3+23*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [cq+64* 8+r5*8], m0 + mova [cq+64* 9+r5*8], m1 + mova [cq+64*10+r5*8], m2 + mova [cq+64*11+r5*8], m3 + mova m0, [r3+31*16] + mova m2, [r3+29*16] + mova m4, [r3+27*16] + mova m6, [r3+25*16] + packssdw m0, [r3+30*16] + packssdw m2, [r3+28*16] + packssdw m4, [r3+26*16] + packssdw m6, [r3+24*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [cq+64*12+r5*8], m0 + mova [cq+64*13+r5*8], m1 + mova [cq+64*14+r5*8], m2 + mova [cq+64*15+r5*8], m3 + mova m0, [r3+ 0*16] + mova m2, [r3+ 2*16] + mova m4, [r3+ 4*16] + mova m6, [r3+ 6*16] + packssdw m0, [r3+ 1*16] + packssdw m2, [r3+ 3*16] + packssdw m4, [r3+ 5*16] + packssdw m6, [r3+ 7*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed +%endif + mova [cq+64* 0+r5*8], m0 + mova [cq+64* 1+r5*8], m1 + mova [cq+64* 2+r5*8], m2 + mova [cq+64* 3+r5*8], m3 + pxor m0, m0 + REPX {mova [cq+x*64+r5*8], m0}, 16, 17, 18, 19, 20, 21, 22, 23, \ + 24, 25, 26, 27, 28, 29, 30, 31 + sub r5d, 2 + jge .loop_pass1 + + ; pass=2, we need to call this otherwise the stack pointer has + ; the wrong offset in the 8-bit code + call .pass2 + RET + +.pass2: +%if ARCH_X86_64 + mova m8, [o(pw_2048)] + pxor m9, m9 + mova m10, [o(pixel_10bpc_max)] +%if WIN64 + mov [rsp+16*16+gprsize], r7 +%endif + mov r7, dstq +%else + mov [rsp+2*gprsize+16*16], dstq +%endif + lea r3, [strideq*3] + mov r4d, 4 + jmp m(idct_16x16_internal_16bpc).loop_pass2 + +.round_dct32: +%if ARCH_X86_64 + psrld m11, 11 ; pd_1 + IDCT32_END 0, 15, 8, 9, 10, 1 ; 0 15 16 31 + mova [r3+ 0*16], m6 + mova [r3+23*16], m7 + IDCT32_END 1, 14, 6, 7, 10, 1 ; 1 14 17 30 + packssdw m0, m1 ; 0 1 + packssdw m14, m15 ; 14 15 + packssdw m8, m6 ; 16 17 + packssdw m7, m9 ; 30 31 + mova [r3+16*15], m14 + mova [r3+16*14], m7 + IDCT32_END 2, 15, 10, 7, 6, 1 ; 2 13 18 29 + IDCT32_END 3, 14, 1, 9, 6, 1 ; 3 12 19 28 + packssdw m2, m3 ; 2 3 + packssdw m14, m15 ; 12 13 + packssdw m10, m1 ; 18 19 + packssdw m9, m7 ; 28 29 + mova [r3+16*13], m14 + mova [r3+16*12], m9 + IDCT32_END 4, 15, 1, 7, 6, 1 ; 4 11 20 27 + IDCT32_END 5, 14, 3, 9, 6, 1 ; 5 10 21 26 + packssdw m4, m5 ; 4 5 + packssdw m14, m15 ; 10 11 + packssdw m1, m3 ; 20 21 + packssdw m9, m7 ; 26 27 + mova [r3+16*11], m14 + mova [r3+16*10], m9 + mova m6, [r3+ 0*16] + mova m7, [r3+23*16] + IDCT32_END 6, 15, 14, 5, 3, 1 ; 6 9 22 25 + IDCT32_END 7, 11, 3, 9, 13, 1 ; 7 8 23 24 + packssdw m6, m7 ; 6 7 + packssdw m11, m15 ; 8 9 + packssdw m14, m3 ; 22 23 + packssdw m9, m5 ; 24 25 + mova [r3+16*9], m11 + mova [r3+16*8], m9 + mova m12, m1 + ret +%else + mova [r3+16*16], m0 + mova [r3+17*16], m1 + mova [r3+18*16], m2 + mova [r3+19*16], m3 + mova [r3+20*16], m4 + mova [r3+21*16], m5 + mova [r3+22*16], m6 + mova [r3+23*16], m7 + pcmpeqd m1, m1 ; -1 + mova m2, [o(clip_min)] + mova m3, [o(clip_max)] + + mov r4, 15*16 +.loop_dct32_end: + mova m0, [r3+16*16] + mova m6, [r3+16*24] + psubd m5, m0, m6 ; idct16 out15 - n + paddd m0, m6 ; idct16 out0 + n + pmaxsd m0, m2 + pmaxsd m5, m2 + pminsd m0, m3 + pminsd m5, m3 + psubd m0, m1 + psubd m5, m1 + mova m7, [r3] + mova m4, [r3+r4] + psubd m6, m0, m4 ; out31 - n + paddd m0, m4 ; out0 + n + paddd m4, m5, m7 ; out15 - n + psubd m5, m7 ; out16 + n + REPX {psrad x, 1}, m0, m5, m4, m6 + mova [r3], m0 + mova [r3+r4], m4 + mova [r3+16*16], m5 + mova [r3+24*16], m6 + add r3, 16 + sub r4, 32 + jg .loop_dct32_end + ret +%endif + +.dconly: + imul r5d, [cq], 2896 + mov [cq], eobd ; 0 + mov r3d, 16 + add r5d, 2048 + sar r5d, 12 + imul r5d, 2896 + add r5d, 6144 + sar r5d, 13 + jmp m(inv_txfm_add_dct_dct_32x8_16bpc).dconly2 + +cglobal inv_txfm_add_dct_dct_32x32_16bpc, 4, 7, 16, 0-(5*32+1)*16, \ + dst, stride, c, eob + LEA r6, base + test eobd, eobd + jz .dconly + + ; remove entirely-zero iterations +%if ARCH_X86_32 + mov [rsp+5*32*16+1*gprsize], dstq +%elif WIN64 + mov [rsp+5*32*16+1*gprsize], r7 +%endif +%undef cmp + mov r5d, 14 + cmp eobw, word [o2(tbl_32x32_2d)+r5] + jge .end_zero_loop + pxor m0, m0 +.zero_loop: + movzx t0d, word [o2(tbl_Nx32_odd_offset)+r5] + movzx t1d, t0b + shr t0d, 8 + mova [rsp+32*16+r5*8+0*32*16], m0 + mova [rsp+40*16+r5*8+0*32*16], m0 + mova [rsp+32*16+t0*8+0*32*16], m0 + mova [rsp+32*16+t1*8+0*32*16], m0 + mova [rsp+32*16+r5*8+1*32*16], m0 + mova [rsp+40*16+r5*8+1*32*16], m0 + mova [rsp+32*16+t0*8+1*32*16], m0 + mova [rsp+32*16+t1*8+1*32*16], m0 + mova [rsp+32*16+r5*8+2*32*16], m0 + mova [rsp+40*16+r5*8+2*32*16], m0 + mova [rsp+32*16+t0*8+2*32*16], m0 + mova [rsp+32*16+t1*8+2*32*16], m0 + mova [rsp+32*16+r5*8+3*32*16], m0 + mova [rsp+40*16+r5*8+3*32*16], m0 + mova [rsp+32*16+t0*8+3*32*16], m0 + mova [rsp+32*16+t1*8+3*32*16], m0 + sub r5d, 2 + cmp eobw, word [o2(tbl_32x32_2d)+r5] + jl .zero_loop +.end_zero_loop: + + ; actual first pass after skipping all-zero data + mov [rsp+gprsize*0+5*32*16], eobd +.loop_pass1: + mova m0, [cq+128* 1+r5*8] + mova m1, [cq+128* 7+r5*8] + mova m2, [cq+128* 9+r5*8] + mova m3, [cq+128*15+r5*8] + mova m4, [cq+128*17+r5*8] + mova m5, [cq+128*23+r5*8] + mova m6, [cq+128*25+r5*8] + mova m7, [cq+128*31+r5*8] +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_min)] + mova m13, [o(clip_max)] + mova m14, [o(pd_2896)] +%endif + mov r3, rsp + call m(inv_txfm_add_dct_dct_32x8_16bpc).main_oddhalf_part1 + mova m0, [cq+128* 3+r5*8] + mova m1, [cq+128* 5+r5*8] + mova m2, [cq+128*11+r5*8] + mova m3, [cq+128*13+r5*8] + mova m4, [cq+128*19+r5*8] + mova m5, [cq+128*21+r5*8] + mova m6, [cq+128*27+r5*8] + mova m7, [cq+128*29+r5*8] + call m(inv_txfm_add_dct_dct_32x8_16bpc).main_oddhalf_part2 + mova m0, [cq+128* 2+r5*8] + mova m1, [cq+128* 6+r5*8] + mova m2, [cq+128*10+r5*8] + mova m3, [cq+128*14+r5*8] + mova m4, [cq+128*18+r5*8] + mova m5, [cq+128*22+r5*8] + mova m6, [cq+128*26+r5*8] + mova m7, [cq+128*30+r5*8] + add r3, 16*(16+4*ARCH_X86_32) + call m(idct_16x4_internal_16bpc).main_oddhalf + mova m0, [cq+128* 0+r5*8] + mova m1, [cq+128* 4+r5*8] + mova m2, [cq+128* 8+r5*8] + mova m3, [cq+128*12+r5*8] + mova m4, [cq+128*16+r5*8] + mova m5, [cq+128*20+r5*8] + mova m6, [cq+128*24+r5*8] + mova m7, [cq+128*28+r5*8] + call m(idct_8x4_internal_16bpc).main_pass1 + call m(idct_8x4_internal_16bpc).round + sub r3, 16*(16+4*ARCH_X86_32) + call m(inv_txfm_add_dct_dct_32x8_16bpc).round_dct32 + movzx t0d, word [o2(tbl_Nx32_odd_offset)+r5] + movzx t1d, t0b + shr t0d, 8 +%if ARCH_X86_64 + call m(idct_8x4_internal_16bpc).transpose4x8packed + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [rsp+32*16+r5*8+2*32*16], m8 + mova [rsp+40*16+r5*8+2*32*16], m10 + mova [rsp+32*16+t1*8+2*32*16], m9 + mova [rsp+32*16+t0*8+2*32*16], m11 + mova m8, [r3+16* 9] ; 8 9 + mova m10, [r3+16*11] ; 10 11 + mova m12, [r3+16*13] ; 12 13 + mova m14, [r3+16*15] ; 14 15 + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [rsp+32*16+r5*8+1*32*16], m8 + mova [rsp+40*16+r5*8+1*32*16], m10 + mova [rsp+32*16+t1*8+1*32*16], m9 + mova [rsp+32*16+t0*8+1*32*16], m11 + mova m8, [r3+16* 8] ; 24 25 + mova m10, [r3+16*10] ; 26 27 + mova m12, [r3+16*12] ; 28 29 + mova m14, [r3+16*14] ; 30 31 + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [rsp+32*16+r5*8+3*32*16], m8 + mova [rsp+40*16+r5*8+3*32*16], m10 + mova [rsp+32*16+t1*8+3*32*16], m9 + mova [rsp+32*16+t0*8+3*32*16], m11 +%else + sub r3, 8*16 + mova m0, [r3+ 8*16] + mova m2, [r3+10*16] + mova m4, [r3+12*16] + mova m6, [r3+14*16] + packssdw m0, [r3+ 9*16] + packssdw m2, [r3+11*16] + packssdw m4, [r3+13*16] + packssdw m6, [r3+15*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [rsp+32*16+r5*8+1*32*16], m0 + mova [rsp+40*16+r5*8+1*32*16], m2 + mova [rsp+32*16+t1*8+1*32*16], m1 + mova [rsp+32*16+t0*8+1*32*16], m3 + mova m0, [r3+16*16] + mova m2, [r3+18*16] + mova m4, [r3+20*16] + mova m6, [r3+22*16] + packssdw m0, [r3+17*16] + packssdw m2, [r3+19*16] + packssdw m4, [r3+21*16] + packssdw m6, [r3+23*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [rsp+32*16+r5*8+2*32*16], m0 + mova [rsp+40*16+r5*8+2*32*16], m2 + mova [rsp+32*16+t1*8+2*32*16], m1 + mova [rsp+32*16+t0*8+2*32*16], m3 + mova m0, [r3+31*16] + mova m2, [r3+29*16] + mova m4, [r3+27*16] + mova m6, [r3+25*16] + packssdw m0, [r3+30*16] + packssdw m2, [r3+28*16] + packssdw m4, [r3+26*16] + packssdw m6, [r3+24*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [rsp+32*16+r5*8+3*32*16], m0 + mova [rsp+40*16+r5*8+3*32*16], m2 + mova [rsp+32*16+t1*8+3*32*16], m1 + mova [rsp+32*16+t0*8+3*32*16], m3 + mova m0, [r3+ 0*16] + mova m2, [r3+ 2*16] + mova m4, [r3+ 4*16] + mova m6, [r3+ 6*16] + packssdw m0, [r3+ 1*16] + packssdw m2, [r3+ 3*16] + packssdw m4, [r3+ 5*16] + packssdw m6, [r3+ 7*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed +%endif + pxor m7, m7 + ; clear lower half of [cq] + REPX {mova [cq+x*128+r5*8], m7}, 0, 1, 2, 3, 4, 5, 6, 7, \ + 8, 9, 10, 11, 12, 13, 14, 15, \ + 16, 17, 18, 19, 20, 21, 22, 23, \ + 24, 25, 26, 27, 28, 29, 30, 31 + mova [rsp+32*16+r5*8+0*32*16], m0 + mova [rsp+40*16+r5*8+0*32*16], m2 + mova [rsp+32*16+t1*8+0*32*16], m1 + mova [rsp+32*16+t0*8+0*32*16], m3 + sub r5d, 2 + jge .loop_pass1 + + ; pass=2 code starts here + mov eobd, [rsp+gprsize*0+5*32*16] + add rsp, 29*16 + cmp eobd, 36 + jl .load_veryfast + cmp eobd, 136 + jl .load_fast + ; load normal + lea r4, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main)] + jmp .run +.load_fast: + lea r4, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main_fast)] + jmp .run +.load_veryfast: + lea r4, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main_veryfast)] + ; fall-through +.run: +%if ARCH_X86_64 + lea r2, [dstq+64] + mov r7, -8 +%else + lea r2, [rsp+(4*32+3)*16] + mov dword [r2+0*gprsize], 4 +%endif + jmp m(inv_txfm_add_dct_dct_16x32_16bpc).loop_pass2_entry + +.dconly: + imul r5d, [cq], 2896 + mov [cq], eobd ; 0 + mov r3d, 32 + add rsp, (5*32+1-(24+8*ARCH_X86_32))*16 + jmp m(inv_txfm_add_dct_dct_32x8_16bpc).dconly1 + +cglobal inv_txfm_add_dct_dct_16x64_16bpc, 4, 7, 16, \ + 0-(12+2*64)*16-(4+4*ARCH_X86_32)*gprsize, \ + dst, stride, c, eob + LEA r6, base + test eobd, eobd + jz .dconly + +%if ARCH_X86_32 + DECLARE_REG_TMP 4, 1, 2, 0 + mov [rsp+gprsize*1+(64*2+12)*16], r0 + mov [rsp+gprsize*2+(64*2+12)*16], r1 + mov [rsp+gprsize*3+(64*2+12)*16], r2 +%else + DECLARE_REG_TMP 8, 9, 4, 7 + mov [rsp+gprsize*1+(64*2+12)*16], r9 +%if WIN64 + mov [rsp+gprsize*2+(64*2+12)*16], r7 + mov [rsp+gprsize*3+(64*2+12)*16], r8 +%endif +%endif +%undef cmp + ; remove entirely-zero iterations + mov r5d, 7*2 + cmp eobw, word [o2(tbl_16x32_2d)+r5] + jge .end_zero_loop + pxor m0, m0 +.zero_loop: + movzx t1d, word [o2(tbl_Nx64_offset)+r5*2+0] + movzx t3d, word [o2(tbl_Nx64_offset)+r5*2+2] + movzx t0d, t1b + movzx t2d, t3b + shr t1d, 8 + shr t3d, 8 + mova [rsp+12*16+t0*8], m0 + mova [rsp+12*16+t1*8], m0 + mova [rsp+12*16+t2*8], m0 + mova [rsp+12*16+t3*8], m0 + mova [rsp+76*16+t0*8], m0 + mova [rsp+76*16+t1*8], m0 + mova [rsp+76*16+t2*8], m0 + mova [rsp+76*16+t3*8], m0 + sub r5d, 2 + cmp eobw, word [o2(tbl_16x32_2d)+r5] + jl .zero_loop +.end_zero_loop: + ; actual first pass after skipping all-zero data + mov [rsp+gprsize*0+(64*2+12)*16], eobd + mov r3, rsp +%if ARCH_X86_32 + DECLARE_REG_TMP 4, 1, 6, 0 + mov r2, [rsp+gprsize*3+(64*2+12)*16] + mov [rsp+gprsize*3+(64*2+12)*16], r6 +%endif +.loop_pass1: +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_min)] + mova m13, [o(clip_max)] + mova m14, [o(pd_2896)] +%endif + mova m0, [cq+ 1*128+r5*8] + mova m1, [cq+ 3*128+r5*8] + mova m2, [cq+ 5*128+r5*8] + mova m3, [cq+ 7*128+r5*8] + mova m4, [cq+ 9*128+r5*8] + mova m5, [cq+11*128+r5*8] + mova m6, [cq+13*128+r5*8] + mova m7, [cq+15*128+r5*8] + call m(idct_16x4_internal_16bpc).main_oddhalf + + mova m0, [cq+ 0*128+r5*8] + mova m1, [cq+ 2*128+r5*8] + mova m2, [cq+ 4*128+r5*8] + mova m3, [cq+ 6*128+r5*8] + mova m4, [cq+ 8*128+r5*8] + mova m5, [cq+10*128+r5*8] + mova m6, [cq+12*128+r5*8] + mova m7, [cq+14*128+r5*8] + call m(idct_8x4_internal_16bpc).main_pass1 + call m(idct_8x4_internal_16bpc).round + call m(idct_16x16_internal_16bpc).round +%if ARCH_X86_64 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + packssdw m8, m9 + packssdw m10, m11 + packssdw m12, m13 + packssdw m14, m15 +%endif + call m(idct_8x4_internal_16bpc).transpose4x8packed + movzx t1d, word [o2(tbl_Nx64_offset)+r5*2+0] + movzx t3d, word [o2(tbl_Nx64_offset)+r5*2+2] + movzx t0d, t1b + movzx t2d, t3b + shr t1d, 8 + shr t3d, 8 +%if ARCH_X86_64 + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [rsp+76*16+t0*8], m8 + mova [rsp+76*16+t1*8], m9 + mova [rsp+76*16+t2*8], m10 + mova [rsp+76*16+t3*8], m11 +%else + mova [rsp+76*16+t0*8], m0 + mova [rsp+76*16+t1*8], m1 + mova [rsp+76*16+t2*8], m2 + mova [rsp+76*16+t3*8], m3 + mova m0, [rsp+ 8*16] + mova m2, [rsp+ 9*16] + mova m4, [rsp+10*16] + mova m6, [rsp+11*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed +%endif + mova [rsp+12*16+t0*8], m0 + mova [rsp+12*16+t1*8], m1 + mova [rsp+12*16+t2*8], m2 + mova [rsp+12*16+t3*8], m3 +%if ARCH_X86_32 + mov r6, [rsp+gprsize*3+(64*2+12)*16] +%endif + pxor m7, m7 + REPX {mova [cq+x*128+r5*8], m7}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 + sub r5d, 2 + jge .loop_pass1 + + ; pass=2 + mov eobd, [rsp+gprsize*0+(64*2+12)*16] + cmp eobd, 151 + jl .fast + ; fall-through +%if ARCH_X86_64 + DECLARE_REG_TMP 8, 9 +%else + DECLARE_REG_TMP 1, 5 +%endif + lea t0, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main_fast)] + lea t1, [o(m_suffix(idct_16x64_internal_8bpc, _ssse3).main)] + jmp .run +.fast: + lea t0, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main_veryfast)] + lea t1, [o(m_suffix(idct_16x64_internal_8bpc, _ssse3).main_fast)] +.run: + add rsp, 9*16 + +%if ARCH_X86_64 + lea r2, [dstq+32] + mov r7, -4 +%else + lea r2, [rsp+(64*2+3)*16] + mov [r2+4*gprsize], t0 + mov [r2+5*gprsize], t1 + mov r1, [r2+2*gprsize] + mov dword [r2+0*gprsize], 2 +%endif +.loop_pass2: +%if ARCH_X86_32 + mov dstq, [r2+1*gprsize] +%endif + call .pass2 + add rsp, 64*16 +%if ARCH_X86_64 + add r7, 2 + lea dstq, [r2+r7*8] + jl .loop_pass2 +%else + add dword [r2+1*gprsize], 16 + dec dword [r2+0*gprsize] + jg .loop_pass2 +%endif +%assign stack_size (stack_size-(64*2+9)*16) +%if STACK_ALIGNMENT >= 16 +%assign stack_size_padded (stack_size_padded-(64*2+9)*16) +%assign stack_offset (stack_offset-(64*2+9)*16) +%else +%xdefine rstkm [rsp + stack_size] +%endif +%if ARCH_X86_64 + mov r9, [rsp+gprsize*1+3*16] +%if WIN64 + mov r7, [rsp+gprsize*2+3*16] + mov r8, [rsp+gprsize*3+3*16] +%endif +%endif + RET + +.pass2: +%if ARCH_X86_32 + lea r5, [o(itx8_start)] +%endif + mova m0, [rsp+gprsize+16* 3] + mova m1, [rsp+gprsize+16* 4] + mova m2, [rsp+gprsize+16* 5] + mova m3, [rsp+gprsize+16* 6] + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + call m_suffix(idct_8x8_internal_8bpc, _ssse3).main + mova [rsp+gprsize+ 3*16], m0 + mova [rsp+gprsize+ 4*16], m1 + mova [rsp+gprsize+ 5*16], m2 + mova [rsp+gprsize+ 6*16], m3 + mova [rsp+gprsize+ 7*16], m4 + mova [rsp+gprsize+ 8*16], m5 + mova [rsp+gprsize+ 9*16], m6 + mova [rsp+gprsize+10*16], m7 + mova m0, [rsp+gprsize+16*11] + mova m1, [rsp+gprsize+16*12] + mova m2, [rsp+gprsize+16*13] + mova m3, [rsp+gprsize+16*14] + pxor m4, m4 + REPX {mova x, m4}, m5, m6, m7 + call m_suffix(idct_16x8_internal_8bpc, _ssse3).main + mova m7, [rsp+gprsize+ 0*16] + mova [rsp+gprsize+11*16], m0 + mova [rsp+gprsize+12*16], m1 + mova [rsp+gprsize+13*16], m2 + mova [rsp+gprsize+14*16], m3 + mova [rsp+gprsize+15*16], m4 + mova [rsp+gprsize+16*16], m5 + mova [rsp+gprsize+17*16], m6 + mova [rsp+gprsize+18*16], m7 +%if ARCH_X86_64 + call r8 +%else + call [r2+4*gprsize] +%endif + mova [rsp+gprsize+ 3*16], m0 + mova [rsp+gprsize+ 5*16], m2 + mova [rsp+gprsize+ 8*16], m5 + mova [rsp+gprsize+10*16], m7 +%if ARCH_X86_64 + call r9 + mova m8, [o(pw_2048)] + pxor m9, m9 + mova m10, [o(pixel_10bpc_max)] +%else + call [r2+5*gprsize] +%endif + lea r3, [strideq*3] + lea r4, [rsp+gprsize+ 3*16] +%if ARCH_X86_64 + mov r6d, 8 +%else + mov dword [r2+2*gprsize], 8 +%endif +.loop_write: + mova m0, [r4+0*16] + mova m1, [r4+1*16] + mova m2, [r4+2*16] + mova m3, [r4+3*16] + mova m4, [r4+4*16] + mova m5, [r4+5*16] + mova m6, [r4+6*16] + mova m7, [r4+7*16] + call m(idct_8x8_internal_16bpc).round1_and_write_8x8 + lea dstq, [dstq+strideq*8] + add r4, 8*16 +%if ARCH_X86_64 + dec r6d +%else + dec dword [r2+2*gprsize] +%endif + jg .loop_write + ret + +.dconly: + imul r5d, [cq], 2896 + mov [cq], eobd ; 0 + mov r3d, 64 + add r5d, 10240 + sar r5d, 14 + add rsp, (12+2*64)*16+(4+4*ARCH_X86_32)*gprsize-(8+4*ARCH_X86_32)*16 + jmp m(inv_txfm_add_dct_dct_16x4_16bpc).dconly2 + +cglobal inv_txfm_add_dct_dct_32x64_16bpc, 4, 7, 16, \ + 0-(32+4*64)*16-(4+4*ARCH_X86_32)*gprsize, \ + dst, stride, c, eob + LEA r6, base + test eobd, eobd + jz .dconly + +%if ARCH_X86_32 + DECLARE_REG_TMP 4, 1, 2, 0 + mov [rsp+gprsize*1+(64*4+32)*16], r0 + mov [rsp+gprsize*2+(64*4+32)*16], r1 + mov [rsp+gprsize*3+(64*4+32)*16], r2 +%else + DECLARE_REG_TMP 8, 9, 4, 7 + mov [rsp+gprsize*1+(64*4+32)*16], r9 +%if WIN64 + mov [rsp+gprsize*2+(64*4+32)*16], r7 + mov [rsp+gprsize*3+(64*4+32)*16], r8 +%endif +%endif +%undef cmp + ; remove entirely-zero iterations + mov r5d, 7*2 + cmp eobw, word [o2(tbl_32x32_2d)+r5] + jge .end_zero_loop + pxor m0, m0 +.zero_loop: + movzx t1d, word [o2(tbl_Nx64_offset)+r5*2+0] + movzx t3d, word [o2(tbl_Nx64_offset)+r5*2+2] + movzx t0d, t1b + movzx t2d, t3b + shr t1d, 8 + shr t3d, 8 + mova [rsp+ 32*16+t0*8], m0 + mova [rsp+ 32*16+t1*8], m0 + mova [rsp+ 32*16+t2*8], m0 + mova [rsp+ 32*16+t3*8], m0 + mova [rsp+ 96*16+t0*8], m0 + mova [rsp+ 96*16+t1*8], m0 + mova [rsp+ 96*16+t2*8], m0 + mova [rsp+ 96*16+t3*8], m0 + mova [rsp+160*16+t0*8], m0 + mova [rsp+160*16+t1*8], m0 + mova [rsp+160*16+t2*8], m0 + mova [rsp+160*16+t3*8], m0 + mova [rsp+224*16+t0*8], m0 + mova [rsp+224*16+t1*8], m0 + mova [rsp+224*16+t2*8], m0 + mova [rsp+224*16+t3*8], m0 + sub r5d, 2 + cmp eobw, word [o2(tbl_32x32_2d)+r5] + jl .zero_loop +.end_zero_loop: + ; actual first pass after skipping all-zero data + mov [rsp+gprsize*0+(64*4+32)*16], eobd + mov r3, rsp +%if ARCH_X86_32 + DECLARE_REG_TMP 4, 1, 6, 0 + mov r2, [rsp+gprsize*3+(64*4+32)*16] + mov [rsp+gprsize*3+(64*4+32)*16], r6 +%endif +.loop_pass1: +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_min)] + mova m13, [o(clip_max)] + mova m14, [o(pd_2896)] +%endif + mova m0, [cq+128* 1+r5*8] + mova m1, [cq+128* 7+r5*8] + mova m2, [cq+128* 9+r5*8] + mova m3, [cq+128*15+r5*8] + mova m4, [cq+128*17+r5*8] + mova m5, [cq+128*23+r5*8] + mova m6, [cq+128*25+r5*8] + mova m7, [cq+128*31+r5*8] + mov r3, rsp + call m(idct_8x4_internal_16bpc).rect2_mul + call m(inv_txfm_add_dct_dct_32x8_16bpc).main_oddhalf_part1 + + mova m0, [cq+128* 3+r5*8] + mova m1, [cq+128* 5+r5*8] + mova m2, [cq+128*11+r5*8] + mova m3, [cq+128*13+r5*8] + mova m4, [cq+128*19+r5*8] + mova m5, [cq+128*21+r5*8] + mova m6, [cq+128*27+r5*8] + mova m7, [cq+128*29+r5*8] +%if ARCH_X86_32 + add r3, 16*8 +%endif + call m(idct_8x4_internal_16bpc).rect2_mul +%if ARCH_X86_32 + sub r3, 16*8 +%endif + call m(inv_txfm_add_dct_dct_32x8_16bpc).main_oddhalf_part2 + add r3, 16*(16+4*ARCH_X86_32) + + mova m0, [cq+128* 2+r5*8] + mova m1, [cq+128* 6+r5*8] + mova m2, [cq+128*10+r5*8] + mova m3, [cq+128*14+r5*8] + mova m4, [cq+128*18+r5*8] + mova m5, [cq+128*22+r5*8] + mova m6, [cq+128*26+r5*8] + mova m7, [cq+128*30+r5*8] + call m(idct_8x4_internal_16bpc).rect2_mul + call m(idct_16x4_internal_16bpc).main_oddhalf + + mova m0, [cq+128* 0+r5*8] + mova m1, [cq+128* 4+r5*8] + mova m2, [cq+128* 8+r5*8] + mova m3, [cq+128*12+r5*8] + mova m4, [cq+128*16+r5*8] + mova m5, [cq+128*20+r5*8] + mova m6, [cq+128*24+r5*8] + mova m7, [cq+128*28+r5*8] + call m(idct_8x4_internal_16bpc).rect2_mul + call m(idct_8x4_internal_16bpc).main_pass1 + call m(idct_8x4_internal_16bpc).round + sub r3, 16*(16+4*ARCH_X86_32) + call m(inv_txfm_add_dct_dct_32x16_16bpc).round_dct32 + + movzx t1d, word [o2(tbl_Nx64_offset)+r5*2+0] + movzx t3d, word [o2(tbl_Nx64_offset)+r5*2+2] + movzx t0d, t1b + movzx t2d, t3b + shr t1d, 8 + shr t3d, 8 +%if ARCH_X86_64 + call m(idct_8x4_internal_16bpc).transpose4x8packed + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [rsp+160*16+t0*8], m8 + mova [rsp+160*16+t1*8], m9 + mova [rsp+160*16+t2*8], m10 + mova [rsp+160*16+t3*8], m11 + mova m8, [r3+16* 9] ; 8 9 + mova m10, [r3+16*11] ; 10 11 + mova m12, [r3+16*13] ; 12 13 + mova m14, [r3+16*15] ; 14 15 + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [rsp+ 96*16+t0*8], m8 + mova [rsp+ 96*16+t1*8], m9 + mova [rsp+ 96*16+t2*8], m10 + mova [rsp+ 96*16+t3*8], m11 + mova m8, [r3+16* 8] ; 24 25 + mova m10, [r3+16*10] ; 26 27 + mova m12, [r3+16*12] ; 28 29 + mova m14, [r3+16*14] ; 30 31 + call m(idct_16x4_internal_16bpc).transpose4x8packed_hi + mova [rsp+224*16+t0*8], m8 + mova [rsp+224*16+t1*8], m9 + mova [rsp+224*16+t2*8], m10 + mova [rsp+224*16+t3*8], m11 +%else + sub r3, 8*16 + mova m0, [r3+ 8*16] + mova m2, [r3+10*16] + mova m4, [r3+12*16] + mova m6, [r3+14*16] + packssdw m0, [r3+ 9*16] + packssdw m2, [r3+11*16] + packssdw m4, [r3+13*16] + packssdw m6, [r3+15*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [rsp+ 96*16+t0*8], m0 + mova [rsp+ 96*16+t1*8], m1 + mova [rsp+ 96*16+t2*8], m2 + mova [rsp+ 96*16+t3*8], m3 + mova m0, [r3+16*16] + mova m2, [r3+18*16] + mova m4, [r3+20*16] + mova m6, [r3+22*16] + packssdw m0, [r3+17*16] + packssdw m2, [r3+19*16] + packssdw m4, [r3+21*16] + packssdw m6, [r3+23*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [rsp+160*16+t0*8], m0 + mova [rsp+160*16+t1*8], m1 + mova [rsp+160*16+t2*8], m2 + mova [rsp+160*16+t3*8], m3 + mova m0, [r3+31*16] + mova m2, [r3+29*16] + mova m4, [r3+27*16] + mova m6, [r3+25*16] + packssdw m0, [r3+30*16] + packssdw m2, [r3+28*16] + packssdw m4, [r3+26*16] + packssdw m6, [r3+24*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [rsp+224*16+t0*8], m0 + mova [rsp+224*16+t1*8], m1 + mova [rsp+224*16+t2*8], m2 + mova [rsp+224*16+t3*8], m3 + mova m0, [r3+ 0*16] + mova m2, [r3+ 2*16] + mova m4, [r3+ 4*16] + mova m6, [r3+ 6*16] + packssdw m0, [r3+ 1*16] + packssdw m2, [r3+ 3*16] + packssdw m4, [r3+ 5*16] + packssdw m6, [r3+ 7*16] + call m(idct_8x4_internal_16bpc).transpose4x8packed +%endif + mova [rsp+ 32*16+t0*8], m0 + mova [rsp+ 32*16+t1*8], m1 + mova [rsp+ 32*16+t2*8], m2 + mova [rsp+ 32*16+t3*8], m3 + pxor m0, m0 + REPX {mova [cq+x*128+r5*8], m0}, 0, 1, 2, 3, 4, 5, 6, 7, \ + 8, 9, 10, 11, 12, 13, 14, 15, \ + 16, 17, 18, 19, 20, 21, 22, 23, \ + 24, 25, 26, 27, 28, 29, 30, 31 +%if ARCH_X86_32 + mov r6, [rsp+gprsize*3+(64*4+32)*16] +%endif + sub r5d, 2 + jge .loop_pass1 + + ; pass=2 + mov eobd, [rsp+gprsize*0+(64*4+32)*16] + cmp eobd, 136 + jl .fast + ; fall-through +%if ARCH_X86_64 + DECLARE_REG_TMP 8, 9 +%else + DECLARE_REG_TMP 1, 5 +%endif + lea t0, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main_fast)] + lea t1, [o(m_suffix(idct_16x64_internal_8bpc, _ssse3).main)] + jmp .run +.fast: + lea t0, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main_veryfast)] + lea t1, [o(m_suffix(idct_16x64_internal_8bpc, _ssse3).main_fast)] +.run: + add rsp, 29*16 + +%if ARCH_X86_64 + lea r2, [dstq+64] + mov r7, -8 +%else + lea r2, [rsp+(64*4+3)*16] + mov [r2+4*gprsize], t0 + mov [r2+5*gprsize], t1 + mov r1, [r2+2*gprsize] + mov dword [r2+0*gprsize], 4 +%endif + jmp m(inv_txfm_add_dct_dct_16x64_16bpc).loop_pass2 + +.dconly: + imul r5d, [cq], 2896 + mov [cq], eobd ; 0 + mov r3d, 64 + add r5d, 2048 + sar r5d, 12 + imul r5d, 2896 + add r5d, 6144 + sar r5d, 13 + add rsp, (32+4*64)*16+(4+4*ARCH_X86_32)*gprsize-(24+8*ARCH_X86_32)*16 + jmp m(inv_txfm_add_dct_dct_32x8_16bpc).dconly2 + +cglobal inv_txfm_add_dct_dct_64x16_16bpc, 4, 7, 16, 0-(64+8*ARCH_X86_32)*16, \ + dst, stride, c, eob + LEA r6, base + test eobd, eobd + jz .dconly + + ; remove entirely-zero iterations +%undef cmp + mov r5d, 8 +.zero_loop: + sub r5d, 2 + cmp eobw, word [o2(tbl_32x16_2d)+r5] + jl .zero_loop + + ; actual first pass after skipping all-zero data +.loop_pass1: +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_min)] + mova m13, [o(clip_max)] + mova m14, [o(pd_2896)] +%endif + + mov r3, rsp + lea r4, [o(idct64_mul_16bpc)] + mova m0, [cq+64* 1+r5*8] + mova m1, [cq+64*31+r5*8] + mova m2, [cq+64*17+r5*8] + mova m3, [cq+64*15+r5*8] + call .main_part1 + mova m0, [cq+64* 7+r5*8] + mova m1, [cq+64*25+r5*8] + mova m2, [cq+64*23+r5*8] + mova m3, [cq+64* 9+r5*8] + call .main_part1 + mova m0, [cq+64* 5+r5*8] + mova m1, [cq+64*27+r5*8] + mova m2, [cq+64*21+r5*8] + mova m3, [cq+64*11+r5*8] + call .main_part1 + mova m0, [cq+64* 3+r5*8] + mova m1, [cq+64*29+r5*8] + mova m2, [cq+64*19+r5*8] + mova m3, [cq+64*13+r5*8] + call .main_part1 + call .main_part2 + + mova m0, [cq+64* 2+r5*8] + mova m1, [cq+64*14+r5*8] + mova m2, [cq+64*18+r5*8] + mova m3, [cq+64*30+r5*8] + call m(inv_txfm_add_dct_dct_32x8_16bpc).main_oddhalf_part1_fast + + mova m0, [cq+64* 6+r5*8] + mova m1, [cq+64*10+r5*8] + mova m2, [cq+64*22+r5*8] + mova m3, [cq+64*26+r5*8] + call m(inv_txfm_add_dct_dct_32x8_16bpc).main_oddhalf_part2_fast + add r3, 16*(24+4*ARCH_X86_32) + + mova m0, [cq+64* 4+r5*8] + mova m1, [cq+64*12+r5*8] + mova m2, [cq+64*20+r5*8] + mova m3, [cq+64*28+r5*8] + call m(idct_16x4_internal_16bpc).main_oddhalf_fast + + mova m0, [cq+64* 0+r5*8] + mova m1, [cq+64* 8+r5*8] + mova m2, [cq+64*16+r5*8] + mova m3, [cq+64*24+r5*8] + call m(idct_8x4_internal_16bpc).main_pass1_fast + call m(idct_8x4_internal_16bpc).round + mova [r3-(7+4*ARCH_X86_32)*16], m1 + mova [r3-(6+4*ARCH_X86_32)*16], m2 + mova [r3-(5+4*ARCH_X86_32)*16], m3 + mova [r3-(4+4*ARCH_X86_32)*16], m4 + mova [r3-(3+4*ARCH_X86_32)*16], m5 + mova [r3-(2+4*ARCH_X86_32)*16], m6 + mova [r3-(1+4*ARCH_X86_32)*16], m7 + sub r3, 16*(40+4*ARCH_X86_32-4) + +%if ARCH_X86_64 + psrld m15, m11, 10 ; pd_2 +%else + mova m7, [o(pd_2)] +%endif + call .main_end_loop_start + + lea r3, [rsp+56*16] + lea r4, [cq+r5*8+64*28] + call .shift_transpose + sub r5d, 2 + jge .loop_pass1 + + ; pass=2, we need to call this otherwise the stack pointer has + ; the wrong offset in the 8-bit code + call .pass2 + RET + +.pass2: +%if ARCH_X86_64 + mova m8, [o(pw_2048)] + pxor m9, m9 + mova m10, [o(pixel_10bpc_max)] +%if WIN64 + mov [rsp+16*16+gprsize], r7 +%endif + mov r7, dstq +%else + mov [rsp+2*gprsize+16*16], dstq +%endif + lea r3, [strideq*3] + mov r4d, 8 + jmp m(idct_16x16_internal_16bpc).loop_pass2 + +.main_part1: ; idct64 steps 1-5 + ; in1/31/17/15 -> t32a/33/34a/35/60/61a/62/63a + ; in7/25/23/ 9 -> t56a/57/58a/59/36/37a/38/39a + ; in5/27/21/11 -> t40a/41/42a/43/52/53a/54/55a + ; in3/29/19/13 -> t48a/49/50a/51/44/45a/46/47a +%if ARCH_X86_64 + movd m7, [r4+4*0] + movd m8, [r4+4*1] + movd m6, [r4+4*2] + movd m9, [r4+4*3] + movd m5, [r4+4*4] + movd m10, [r4+4*5] + movd m4, [r4+4*6] + movd m15, [r4+4*7] + REPX {pshufd x, x, q0000}, m7, m8, m6, m9, m5, m10, m4, m15 + pmulld m7, m0 ; t63a + pmulld m0, m8 ; t32a + pmulld m6, m1 ; t62a + pmulld m1, m9 ; t33a + pmulld m5, m2 ; t61a + pmulld m2, m10 ; t34a + pmulld m4, m3 ; t60a + pmulld m3, m15 ; t35a + movd m10, [r4+4*8] + movd m15, [r4+4*9] + REPX {pshufd x, x, q0000}, m10, m15 + REPX {paddd x, m11}, m7, m0, m6, m1, m5, m2, m4, m3 + REPX {psrad x, 12 }, m0, m1, m7, m6, m2, m3, m5, m4 + psubd m8, m0, m1 ; t33 + paddd m0, m1 ; t32 + psubd m1, m7, m6 ; t62 + paddd m7, m6 ; t63 + psubd m6, m3, m2 ; t34 + paddd m3, m2 ; t35 + psubd m2, m4, m5 ; t61 + paddd m4, m5 ; t60 + REPX {pmaxsd x, m12}, m8, m1, m6, m2 + REPX {pminsd x, m13}, m8, m1, m6, m2 + ITX_MULSUB_2D 1, 8, 5, 9, _, 11, 10, 15 ; t33a, t62a + ITX_MULSUB_2D 2, 6, 5, 9, _, 11, 10, 15, 4 ; t61a, t34a + REPX {pmaxsd x, m12}, m0, m3, m7, m4 + REPX {pminsd x, m13}, m0, m3, m7, m4 + movd m10, [r4+4*10] + movd m15, [r4+4*11] + REPX {pshufd x, x, q0000}, m10, m15 + psubd m5, m0, m3 ; t35a + paddd m0, m3 ; t32a + psubd m3, m7, m4 ; t60a + paddd m7, m4 ; t63a + psubd m4, m1, m6 ; t34 + paddd m1, m6 ; t33 + psubd m6, m8, m2 ; t61 + paddd m8, m2 ; t62 + REPX {pmaxsd x, m12}, m5, m3, m4, m6 + REPX {pminsd x, m13}, m5, m3, m4, m6 + ITX_MULSUB_2D 3, 5, 2, 9, _, 11, 10, 15 ; t35, t60 + ITX_MULSUB_2D 6, 4, 2, 9, _, 11, 10, 15 ; t34a, t61a + REPX {pmaxsd x, m12}, m0, m7, m1, m8 + REPX {pminsd x, m13}, m0, m7, m1, m8 + add r4, 4*12 + mova [r3+16*0], m0 + mova [r3+16*7], m7 + mova [r3+16*1], m1 + mova [r3+16*6], m8 + mova [r3+16*2], m6 + mova [r3+16*5], m4 + mova [r3+16*3], m3 + mova [r3+16*4], m5 +%else + movd m7, [r4+4*0] + movd m6, [r4+4*2] + movd m5, [r4+4*4] + movd m4, [r4+4*6] + REPX {pshufd x, x, q0000}, m7, m6, m5, m4 + pmulld m7, m0 ; t63a + pmulld m6, m1 ; t62a + pmulld m5, m2 ; t61a + pmulld m4, m3 ; t60a + mova [r3+0*16], m6 + mova [r3+1*16], m7 + movd m6, [r4+4*1] + movd m7, [r4+4*3] + REPX {pshufd x, x, q0000}, m7, m6 + pmulld m0, m6 ; t32a + pmulld m1, m7 ; t33a + movd m6, [r4+4*5] + movd m7, [r4+4*7] + REPX {pshufd x, x, q0000}, m7, m6 + pmulld m2, m6 ; t34a + pmulld m3, m7 ; t35a + mova m6, [r3+0*16] + mova m7, [o(pd_2048)] + REPX {paddd x, m7}, m0, m1, m2, m3, m4, m5, m6 + paddd m7, [r3+1*16] + REPX {psrad x, 12}, m0, m1, m7, m6, m2, m3, m5, m4 + mova [r3+0*16], m5 + psubd m5, m0, m1 ; t33 + paddd m0, m1 ; t32 + mova [r3+1*16], m0 + mova m0, [r3+0*16] + psubd m1, m7, m6 ; t62 + paddd m7, m6 ; t63 + psubd m6, m3, m2 ; t34 + paddd m3, m2 ; t35 + psubd m2, m4, m0 ; t61 + paddd m4, m0 ; t60 + mova m0, [o(clip_min)] + REPX {pmaxsd x, m0}, m5, m1, m7, m6, m3, m2, m4 + pmaxsd m0, [r3+1*16] + mova [r3+0*16], m0 + mova m0, [o(clip_max)] + REPX {pminsd x, m0}, m5, m1, m7, m6, m3, m2, m4 + pminsd m0, [r3+0*16] + mova [r3+0*16], m0 + mova [r3+1*16], m3 + mova [r3+2*16], m4 + mova [r3+3*16], m7 + mova m0, [o(pd_2048)] + movd m3, [r4+4*8] + movd m4, [r4+4*9] + REPX {pshufd x, x, q0000}, m3, m4 + mova [r3+4*16], m2 + ITX_MULSUB_2D 1, 5, 2, 7, _, 0, 3, 4 ; t33a, t62a + mova m2, [r3+4*16] + mova [r3+4*16], m5 + ITX_MULSUB_2D 2, 6, 5, 7, _, 0, 3, 4, 4 ; t61a, t34a + mova m0, [r3+0*16] + mova m3, [r3+1*16] + mova m4, [r3+2*16] + mova m7, [r3+3*16] + psubd m5, m0, m3 ; t35a + paddd m0, m3 ; t32a + mova [r3+0*16], m5 + mova m5, [r3+4*16] + psubd m3, m7, m4 ; t60a + paddd m7, m4 ; t63a + psubd m4, m1, m6 ; t34 + paddd m1, m6 ; t33 + psubd m6, m5, m2 ; t61 + paddd m2, m5 ; t62 + mova m5, [o(clip_min)] + REPX {pmaxsd x, m5}, m0, m3, m7, m4, m1, m6, m2 + pmaxsd m5, [r3+0*16] + mova [r3+0*16], m5 + mova m5, [o(clip_max)] + REPX {pminsd x, m5}, m0, m3, m7, m4, m1, m6, m2 + pminsd m5, [r3+0*16] + mova [r3+16*0], m0 + mova [r3+16*7], m7 + mova [r3+16*1], m1 + mova [r3+16*6], m2 + mova [r3+16*2], m4 + mova m7, [o(pd_2048)] + movd m0, [r4+4*10] + movd m1, [r4+4*11] + REPX {pshufd x, x, q0000}, m0, m1 + ITX_MULSUB_2D 3, 5, 2, 4, _, 7, 0, 1 ; t35, t60 + mova [r3+16*3], m3 + mova [r3+16*4], m5 + mova m4, [r3+2*16] + ITX_MULSUB_2D 6, 4, 2, 3, _, 7, 0, 1 ; t34a, t61a + add r4, 4*12 + mova [r3+16*2], m6 + mova [r3+16*5], m4 +%endif + add r3, 16*8 + ret + +.main_part2: ; idct64 steps 6-9 + lea r4, [r3+16*7] +%if ARCH_X86_64 + mova m10, [o(pd_1567)] + mova m15, [o(pd_3784)] +.main_part2_loop: + mova m0, [r3-16*32] ; t32a + mova m1, [r4-16*24] ; t39a + mova m2, [r4-16*32] ; t63a + mova m3, [r3-16*24] ; t56a + mova m4, [r3-16*16] ; t40a + mova m5, [r4-16* 8] ; t47a + mova m6, [r4-16*16] ; t55a + mova m7, [r3-16* 8] ; t48a + psubd m8, m0, m1 ; t39 + paddd m0, m1 ; t32 + psubd m1, m2, m3 ; t56 + paddd m2, m3 ; t63 + psubd m3, m5, m4 ; t40 + paddd m5, m4 ; t47 + psubd m4, m7, m6 ; t55 + paddd m7, m6 ; t48 + REPX {pmaxsd x, m12}, m8, m1, m3, m4 + REPX {pminsd x, m13}, m8, m1, m3, m4 + ITX_MULSUB_2D 1, 8, 6, 9, _, 11, 10, 15 ; t39a, t56a + ITX_MULSUB_2D 4, 3, 6, 9, _, 11, 10, 15, 4 ; t55a, t40a + REPX {pmaxsd x, m12}, m0, m2, m5, m7 + REPX {pminsd x, m13}, m0, m5, m2, m7 + psubd m6, m2, m7 ; t48a + paddd m2, m7 ; t63a + psubd m7, m0, m5 ; t47a + paddd m0, m5 ; t32a + psubd m5, m8, m4 ; t55 + paddd m8, m4 ; t56 + psubd m4, m1, m3 ; t40 + paddd m1, m3 ; t39 + REPX {pmaxsd x, m12}, m6, m7, m5, m4 + REPX {pminsd x, m13}, m6, m7, m5, m4 + REPX {pmulld x, m14}, m6, m7, m5, m4 + REPX {pmaxsd x, m12}, m2, m0, m8, m1 + REPX {pminsd x, m13}, m2, m0, m8, m1 + paddd m6, m11 + paddd m5, m11 + psubd m3, m6, m7 ; t47 + paddd m6, m7 ; t48 + psubd m7, m5, m4 ; t40a + paddd m5, m4 ; t55a + REPX {psrad x, 12}, m3, m6, m7, m5 + mova [r4-16* 8], m2 + mova [r3-16*32], m0 + mova [r3-16* 8], m8 + mova [r4-16*32], m1 + mova [r4-16*24], m3 + mova [r3-16*16], m6 + mova [r3-16*24], m7 + mova [r4-16*16], m5 +%else +.main_part2_loop: + mova m0, [r3-16*32] ; t32a + mova m1, [r4-16*24] ; t39a + mova m2, [r4-16*32] ; t63a + mova m3, [r3-16*24] ; t56a + mova m4, [r3-16*16] ; t40a + mova m5, [r4-16* 8] ; t47a + mova m6, [r4-16*16] ; t55a + psubd m7, m0, m1 ; t39 + paddd m0, m1 ; t32 + mova [r3+0*16], m7 + mova m7, [r3-16* 8] ; t48a + psubd m1, m2, m3 ; t56 + paddd m2, m3 ; t63 + psubd m3, m5, m4 ; t40 + paddd m5, m4 ; t47 + psubd m4, m7, m6 ; t55 + paddd m7, m6 ; t48 + mova m6, [o(clip_min)] + REPX {pmaxsd x, m6}, m0, m1, m2, m3, m5, m4, m7 + pmaxsd m6, [r3+0*16] + mova [r3+0*16], m6 + mova m6, [o(clip_max)] + REPX {pminsd x, m6}, m0, m1, m2, m3, m5, m4, m7 + pminsd m6, [r3+0*16] + mova [r3+0*16], m0 + mova [r3+1*16], m2 + mova [r3+2*16], m5 + mova [r3+3*16], m7 + mova m0, [o(pd_2048)] + ITX_MULSUB_2D 1, 6, 2, 5, 7, 0, 1567, 3784 ; t39a, t56a + ITX_MULSUB_2D 4, 3, 2, 5, _, 0, 7, 3784, 4 ; t55a, t40a + mova m2, [r3+1*16] + mova m7, [r3+3*16] + psubd m5, m2, m7 ; t48a + paddd m2, m7 ; t63a + mova [r3+1*16], m5 + mova m0, [r3+0*16] + mova m5, [r3+2*16] + psubd m7, m0, m5 ; t47a + paddd m0, m5 ; t32a + psubd m5, m6, m4 ; t55 + paddd m6, m4 ; t56 + psubd m4, m1, m3 ; t40 + paddd m1, m3 ; t39 + mova m3, [o(clip_min)] + REPX {pmaxsd x, m3}, m2, m7, m0, m5, m6, m4, m1 + pmaxsd m3, [r3+1*16] + mova [r3+0*16], m3 + mova m3, [o(clip_max)] + REPX {pminsd x, m3}, m2, m7, m0, m5, m6, m4, m1 + pminsd m3, [r3+0*16] + mova [r4-16* 8], m2 + mova [r3-16*32], m0 + mova [r3-16* 8], m6 + mova [r4-16*32], m1 + mova m0, [o(pd_2896)] + mova m1, [o(pd_2048)] + REPX {pmulld x, m0}, m3, m7, m5, m4 + REPX {paddd x, m1}, m3, m5 + psubd m6, m3, m7 ; t47 + paddd m3, m7 ; t48 + psubd m7, m5, m4 ; t40a + paddd m5, m4 ; t55a + REPX {psrad x, 12}, m6, m3, m7, m5 + mova [r4-16*24], m6 + mova [r3-16*16], m3 + mova [r3-16*24], m7 + mova [r4-16*16], m5 +%endif + add r3, 16 + sub r4, 16 + cmp r3, r4 + jl .main_part2_loop + sub r3, 4*16 + ret + +.main_end_loop: + mova m0, [r3+16*28] ; idct8 0 + n +.main_end_loop_start: + mova m2, [r3+16*12] ; idct32 16 + n + mova m3, [r4+16*12] ; idct32 31 - n +%if ARCH_X86_64 + mova m1, [r4+16*28] ; idct16 15 - n + mova m4, [r4-16* 4] ; idct64 63 - n + mova m5, [r3-16* 4] ; idct64 48 + n + mova m6, [r4-16*20] ; idct64 47 - n + mova m7, [r3-16*20] ; idct64 32 + n + paddd m8, m0, m1 ; idct16 out0 + n + psubd m0, m1 ; idct16 out15 - n + REPX {pmaxsd x, m12}, m8, m0 + REPX {pminsd x, m13}, m8, m0 + paddd m1, m8, m3 ; idct32 out0 + n + psubd m8, m3 ; idct32 out31 - n + paddd m3, m0, m2 ; idct32 out15 - n + psubd m0, m2 ; idct32 out16 + n + REPX {pmaxsd x, m12}, m1, m8, m3, m0 + REPX {pminsd x, m13}, m1, m3, m8, m0 + REPX {paddd x, m15}, m1, m3, m0, m8 + paddd m2, m1, m4 ; idct64 out0 + n (unshifted) + psubd m1, m4 ; idct64 out63 - n (unshifted) + paddd m4, m3, m5 ; idct64 out15 - n (unshifted) + psubd m3, m5 ; idct64 out48 + n (unshifted) + paddd m5, m0, m6 ; idct64 out16 + n (unshifted) + psubd m0, m6 ; idct64 out47 - n (unshifted) + paddd m6, m8, m7 ; idct64 out31 - n (unshifted) + psubd m8, m7 ; idct64 out32 + n (unshifted) + mova [r3-16*20], m2 + mova [r4+16*28], m1 + mova [r4-16*20], m4 + mova [r3+16*28], m3 + mova [r3-16* 4], m5 + mova [r4+16*12], m0 + mova [r4-16* 4], m6 + mova [r3+16*12], m8 +%else + mova m1, [r3+16*44] ; idct16 15 - n + paddd m4, m0, m1 ; idct16 out0 + n + psubd m0, m1 ; idct16 out15 - n + mova m5, [o(clip_min)] + mova m6, [o(clip_max)] + REPX {pmaxsd x, m5}, m4, m0 + REPX {pminsd x, m6}, m4, m0 + paddd m1, m4, m3 ; idct32 out0 + n + psubd m4, m3 ; idct32 out31 - n + paddd m3, m0, m2 ; idct32 out15 - n + psubd m0, m2 ; idct32 out16 + n + REPX {pmaxsd x, m5}, m1, m4, m3, m0 + REPX {pminsd x, m6}, m1, m3, m4, m0 + REPX {paddd x, m7}, m1, m3, m0, m4 + mova m5, [r4-16* 4] ; idct64 63 - n + mova m6, [r3-16* 4] ; idct64 48 + n + paddd m2, m1, m5 ; idct64 out0 + n (unshifted) + psubd m1, m5 ; idct64 out63 - n (unshifted) + paddd m5, m3, m6 ; idct64 out15 - n (unshifted) + psubd m3, m6 ; idct64 out48 + n (unshifted) + mova [r4+16*28], m1 + mova [r3+16*28], m3 + mova m6, [r4-16*20] ; idct64 47 - n + mova m1, [r3-16*20] ; idct64 32 + n + mova [r3-16*20], m2 + mova [r4-16*20], m5 + paddd m5, m0, m6 ; idct64 out16 + n (unshifted) + psubd m0, m6 ; idct64 out47 - n (unshifted) + paddd m6, m4, m1 ; idct64 out31 - n (unshifted) + psubd m4, m1 ; idct64 out32 + n (unshifted) + mova [r3-16* 4], m5 + mova [r4+16*12], m0 + mova [r4-16* 4], m6 + mova [r3+16*12], m4 +%endif + sub r4, 16 + add r3, 16 + cmp r3, r4 + jl .main_end_loop + ret + +.shift_transpose: + mova m0, [r3+0*16] + mova m1, [r3+1*16] + mova m2, [r3+2*16] + mova m3, [r3+3*16] + mova m4, [r3+4*16] + mova m5, [r3+5*16] + mova m6, [r3+6*16] + mova m7, [r3+7*16] + REPX {psrad x, 2}, m0, m1, m2, m3, m4, m5, m6, m7 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [r4+0*64], m0 + mova [r4+1*64], m1 + mova [r4+2*64], m2 + mova [r4+3*64], m3 + sub r4, 4*64 + sub r3, 8*16 + cmp r3, rsp + jg .shift_transpose + ret + +.dconly: + imul r5d, [cq], 2896 + mov [cq], eobd ; 0 + mov r3d, 16 +.dconly1: + add r5d, 10240 + sar r5d, 14 +.dconly2: + imul r5d, 2896 + add r5d, 34816 + movd m0, r5d + pshuflw m0, m0, q1111 + punpcklqdq m0, m0 + mova m6, [o(pixel_10bpc_max)] + pxor m5, m5 +.dconly_loop: + paddw m1, m0, [dstq+16*0] + paddw m2, m0, [dstq+16*1] + paddw m3, m0, [dstq+16*2] + paddw m4, m0, [dstq+16*3] + REPX {pmaxsw x, m5}, m1, m2, m3, m4 + REPX {pminsw x, m6}, m1, m2, m3, m4 + mova [dstq+16*0], m1 + mova [dstq+16*1], m2 + mova [dstq+16*2], m3 + mova [dstq+16*3], m4 + add dstq, 64 + btc r3d, 16 + jnc .dconly_loop + lea dstq, [dstq+strideq-128] + dec r3d + jg .dconly_loop + RET + +cglobal inv_txfm_add_dct_dct_64x32_16bpc, 4, 7, 16, \ + 0-(1+64+8*ARCH_X86_32+8*32+1*WIN64)*16, \ + dst, stride, c, eob + LEA r6, base + test eobd, eobd + jz .dconly + +%if ARCH_X86_32 + DECLARE_REG_TMP 0, 4, 1 + mov [rsp+(8*32+64+8)*16+1*gprsize], dstq + mov [rsp+(8*32+64+8)*16+2*gprsize], strideq +%else + DECLARE_REG_TMP 4, 7, 8 +%if WIN64 + mov [rsp+(8*32+64+1)*16+1*gprsize], r7 + mov [rsp+64*16+0*gprsize], r8 +%endif +%endif +%undef cmp + ; remove entirely-zero iterations + mov r5d, 14 + cmp eobw, word [o2(tbl_32x32_2d)+r5] + jge .end_zero_loop + pxor m0, m0 +.zero_loop: + movzx t0d, word [o2(tbl_Nx32_odd_offset)+r5] + movzx t1d, t0b + shr t0d, 8 + lea t2, [rsp+7*32*16] +.zero_loop_inner: + mova [t2+(64+8*ARCH_X86_32+1*WIN64)*16+r5*8], m0 + mova [t2+(72+8*ARCH_X86_32+1*WIN64)*16+r5*8], m0 + mova [t2+(64+8*ARCH_X86_32+1*WIN64)*16+t0*8], m0 + mova [t2+(64+8*ARCH_X86_32+1*WIN64)*16+t1*8], m0 + sub t2, 32*16 + cmp t2, rsp + jge .zero_loop_inner + sub r5d, 2 + cmp eobw, word [o2(tbl_32x32_2d)+r5] + jl .zero_loop +.end_zero_loop: + mov [rsp+(8*32+64+8*ARCH_X86_32+1*WIN64)*16+0*gprsize], eobd + ; actual first pass after skipping all-zero data +.loop_pass1: +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_min)] + mova m13, [o(clip_max)] + mova m14, [o(pd_2896)] +%endif + + mov r3, rsp + lea r4, [o(idct64_mul_16bpc)] + mova m0, [cq+128* 1+r5*8] + mova m1, [cq+128*31+r5*8] + mova m2, [cq+128*17+r5*8] + mova m3, [cq+128*15+r5*8] + call .rect2_mul_fast + call m(inv_txfm_add_dct_dct_64x16_16bpc).main_part1 + mova m0, [cq+128* 7+r5*8] + mova m1, [cq+128*25+r5*8] + mova m2, [cq+128*23+r5*8] + mova m3, [cq+128* 9+r5*8] + call .rect2_mul_fast + call m(inv_txfm_add_dct_dct_64x16_16bpc).main_part1 + mova m0, [cq+128* 5+r5*8] + mova m1, [cq+128*27+r5*8] + mova m2, [cq+128*21+r5*8] + mova m3, [cq+128*11+r5*8] + call .rect2_mul_fast + call m(inv_txfm_add_dct_dct_64x16_16bpc).main_part1 + mova m0, [cq+128* 3+r5*8] + mova m1, [cq+128*29+r5*8] + mova m2, [cq+128*19+r5*8] + mova m3, [cq+128*13+r5*8] + call .rect2_mul_fast + call m(inv_txfm_add_dct_dct_64x16_16bpc).main_part1 + call m(inv_txfm_add_dct_dct_64x16_16bpc).main_part2 + + mova m0, [cq+128* 2+r5*8] + mova m1, [cq+128*14+r5*8] + mova m2, [cq+128*18+r5*8] + mova m3, [cq+128*30+r5*8] + call .rect2_mul_fast + call m(inv_txfm_add_dct_dct_32x8_16bpc).main_oddhalf_part1_fast + + mova m0, [cq+128* 6+r5*8] + mova m1, [cq+128*10+r5*8] + mova m2, [cq+128*22+r5*8] + mova m3, [cq+128*26+r5*8] + call .rect2_mul_fast + call m(inv_txfm_add_dct_dct_32x8_16bpc).main_oddhalf_part2_fast + add r3, 16*(24+4*ARCH_X86_32) + + mova m0, [cq+128* 4+r5*8] + mova m1, [cq+128*12+r5*8] + mova m2, [cq+128*20+r5*8] + mova m3, [cq+128*28+r5*8] + call .rect2_mul_fast + call m(idct_16x4_internal_16bpc).main_oddhalf_fast + + mova m0, [cq+128* 0+r5*8] + mova m1, [cq+128* 8+r5*8] + mova m2, [cq+128*16+r5*8] + mova m3, [cq+128*24+r5*8] + call .rect2_mul_fast + call m(idct_8x4_internal_16bpc).main_pass1_fast + call m(idct_8x4_internal_16bpc).round + mova [r3-(7+4*ARCH_X86_32)*16], m1 + mova [r3-(6+4*ARCH_X86_32)*16], m2 + mova [r3-(5+4*ARCH_X86_32)*16], m3 + mova [r3-(4+4*ARCH_X86_32)*16], m4 + mova [r3-(3+4*ARCH_X86_32)*16], m5 + mova [r3-(2+4*ARCH_X86_32)*16], m6 + mova [r3-(1+4*ARCH_X86_32)*16], m7 + sub r3, 16*(40+4*ARCH_X86_32-4) + +%if ARCH_X86_64 + psrld m15, m11, 11 ; pd_1 +%else + mova m7, [o(pd_1)] +%endif + call m(inv_txfm_add_dct_dct_64x16_16bpc).main_end_loop_start + + lea r3, [rsp+56*16] + lea t2, [rsp+7*32*16+(64+8*ARCH_X86_32+1*WIN64)*16] + movzx t0d, word [o2(tbl_Nx32_odd_offset)+r5] + movzx t1d, t0b + shr t0d, 8 + call .shift_transpose + ; zero cq + pxor m7, m7 + lea r4, [cq+30*128+r5*8] +.zero_cq_loop: + REPX {mova [r4+x*128], m7}, -2, -1, 0, 1 + sub r4, 4*128 + cmp r4, cq + jg .zero_cq_loop + sub r5d, 2 + jge .loop_pass1 + + ; pass=2 code starts here + mov eobd, [rsp+gprsize*0+(8*32+64+8*ARCH_X86_32+1*WIN64)*16] +%if ARCH_X86_32 + mov strideq, [rsp+gprsize*2+(8*32+64+8)*16] +%elif WIN64 + mov r8, [rsp+gprsize*0+64*16] +%endif + add rsp, (64+8*ARCH_X86_32+1*WIN64-3)*16 + cmp eobd, 36 + jl .load_veryfast + cmp eobd, 136 + jl .load_fast + ; load normal + lea r4, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main)] + jmp .run +.load_fast: + lea r4, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main_fast)] + jmp .run +.load_veryfast: + lea r4, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main_veryfast)] + ; fall-through +.run: +%if ARCH_X86_64 + lea r2, [dstq+128] + mov r7, -16 +%else + lea r2, [rsp+(8*32+3)*16] + mov dword [r2+0*gprsize], 8 +%endif + jmp m(inv_txfm_add_dct_dct_16x32_16bpc).loop_pass2_entry + +.rect2_mul_fast: +%if ARCH_X86_64 + REPX {pmulld x, m14}, m0, m1, m2, m3 + REPX {paddd x, m11}, m0, m1, m2, m3 +%else + mova m4, [o(pd_2896)] + mova m5, [o(pd_2048)] + REPX {pmulld x, m4 }, m0, m1, m2, m3 + REPX {paddd x, m5 }, m0, m1, m2, m3 +%endif + REPX {psrad x, 12 }, m0, m1, m2, m3 + ret + +.shift_transpose: + mova m0, [r3+0*16] + mova m1, [r3+1*16] + mova m2, [r3+2*16] + mova m3, [r3+3*16] + mova m4, [r3+4*16] + mova m5, [r3+5*16] + mova m6, [r3+6*16] + mova m7, [r3+7*16] + REPX {psrad x, 1}, m0, m1, m2, m3, m4, m5, m6, m7 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [t2+0*16+r5*8], m0 + mova [t2+8*16+r5*8], m2 + mova [t2+0*16+t0*8], m3 + mova [t2+0*16+t1*8], m1 + sub t2, 16*32 + sub r3, 8*16 + cmp r3, rsp + jg .shift_transpose + ret + +.dconly: + imul r5d, [cq], 2896 + mov [cq], eobd ; 0 + mov r3d, 32 + add r5d, 2048 + sar r5d, 12 + imul r5d, 2896 + add r5d, 6144 + sar r5d, 13 + add rsp, (1+8*32+1*WIN64)*16 + jmp m(inv_txfm_add_dct_dct_64x16_16bpc).dconly2 + +cglobal inv_txfm_add_dct_dct_64x64_16bpc, 4, 7, 16, \ + 0-(64+8*ARCH_X86_32+8*64+1*ARCH_X86_64)*16-(4+4*ARCH_X86_32)*gprsize, \ + dst, stride, c, eob + LEA r6, base + test eobd, eobd + jz .dconly + +%if ARCH_X86_32 + DECLARE_REG_TMP 4, 1, 2, 0, 6 + mov [rsp+gprsize*1+(64*9+8)*16], r0 + mov [rsp+gprsize*2+(64*9+8)*16], r1 + mov [rsp+gprsize*3+(64*9+8)*16], r2 + mov [rsp+gprsize*4+(64*9+8)*16], r6 +%else + DECLARE_REG_TMP 8, 9, 4, 7, 0 + mov [rsp+gprsize*1+(64*9+1)*16], r9 + mov [rsp+gprsize*0+64*16], r0 +%if WIN64 + mov [rsp+gprsize*2+(64*9+1)*16], r7 + mov [rsp+gprsize*3+(64*9+1)*16], r8 +%endif +%endif +%undef cmp + + ; remove entirely-zero iterations + mov r5d, 14 + cmp eobw, word [o2(tbl_32x32_2d)+r5] + jge .end_zero_loop + pxor m0, m0 +.zero_loop: + movzx t1d, word [o2(tbl_Nx64_offset)+r5*2+0] + movzx t3d, word [o2(tbl_Nx64_offset)+r5*2+2] + movzx t0d, t1b + movzx t2d, t3b + shr t1d, 8 + shr t3d, 8 + lea t4, [rsp+7*64*16] +.zero_loop_inner: + mova [t4+(64+8*ARCH_X86_32+1*ARCH_X86_64)*16+t0*8], m0 + mova [t4+(64+8*ARCH_X86_32+1*ARCH_X86_64)*16+t1*8], m0 + mova [t4+(64+8*ARCH_X86_32+1*ARCH_X86_64)*16+t2*8], m0 + mova [t4+(64+8*ARCH_X86_32+1*ARCH_X86_64)*16+t3*8], m0 + sub t4, 64*16 + cmp t4, rsp + jge .zero_loop_inner +%if ARCH_X86_32 + mov r6, [rsp+gprsize*4+(64*9+8)*16] +%endif + sub r5d, 2 + cmp eobw, word [o2(tbl_32x32_2d)+r5] + jl .zero_loop +.end_zero_loop: + mov [rsp+gprsize*0+(9*64+8*ARCH_X86_32+1*ARCH_X86_64)*16], eobd +%if ARCH_X86_32 + mov cq, [rsp+gprsize*3+(64*9+8)*16] +%endif + ; actual first pass after skipping all-zero data +.loop_pass1: +%if ARCH_X86_64 + mova m11, [o(pd_2048)] + mova m12, [o(clip_min)] + mova m13, [o(clip_max)] + mova m14, [o(pd_2896)] +%endif + + mov r3, rsp + lea r4, [o(idct64_mul_16bpc)] + mova m0, [cq+128* 1+r5*8] + mova m1, [cq+128*31+r5*8] + mova m2, [cq+128*17+r5*8] + mova m3, [cq+128*15+r5*8] + call m(inv_txfm_add_dct_dct_64x16_16bpc).main_part1 + mova m0, [cq+128* 7+r5*8] + mova m1, [cq+128*25+r5*8] + mova m2, [cq+128*23+r5*8] + mova m3, [cq+128* 9+r5*8] + call m(inv_txfm_add_dct_dct_64x16_16bpc).main_part1 + mova m0, [cq+128* 5+r5*8] + mova m1, [cq+128*27+r5*8] + mova m2, [cq+128*21+r5*8] + mova m3, [cq+128*11+r5*8] + call m(inv_txfm_add_dct_dct_64x16_16bpc).main_part1 + mova m0, [cq+128* 3+r5*8] + mova m1, [cq+128*29+r5*8] + mova m2, [cq+128*19+r5*8] + mova m3, [cq+128*13+r5*8] + call m(inv_txfm_add_dct_dct_64x16_16bpc).main_part1 + call m(inv_txfm_add_dct_dct_64x16_16bpc).main_part2 + + mova m0, [cq+128* 2+r5*8] + mova m1, [cq+128*14+r5*8] + mova m2, [cq+128*18+r5*8] + mova m3, [cq+128*30+r5*8] + call m(inv_txfm_add_dct_dct_32x8_16bpc).main_oddhalf_part1_fast + + mova m0, [cq+128* 6+r5*8] + mova m1, [cq+128*10+r5*8] + mova m2, [cq+128*22+r5*8] + mova m3, [cq+128*26+r5*8] + call m(inv_txfm_add_dct_dct_32x8_16bpc).main_oddhalf_part2_fast + add r3, 16*(24+4*ARCH_X86_32) + + mova m0, [cq+128* 4+r5*8] + mova m1, [cq+128*12+r5*8] + mova m2, [cq+128*20+r5*8] + mova m3, [cq+128*28+r5*8] + call m(idct_16x4_internal_16bpc).main_oddhalf_fast + + mova m0, [cq+128* 0+r5*8] + mova m1, [cq+128* 8+r5*8] + mova m2, [cq+128*16+r5*8] + mova m3, [cq+128*24+r5*8] + call m(idct_8x4_internal_16bpc).main_pass1_fast + call m(idct_8x4_internal_16bpc).round + mova [r3-(7+4*ARCH_X86_32)*16], m1 + mova [r3-(6+4*ARCH_X86_32)*16], m2 + mova [r3-(5+4*ARCH_X86_32)*16], m3 + mova [r3-(4+4*ARCH_X86_32)*16], m4 + mova [r3-(3+4*ARCH_X86_32)*16], m5 + mova [r3-(2+4*ARCH_X86_32)*16], m6 + mova [r3-(1+4*ARCH_X86_32)*16], m7 + sub r3, 16*(40+4*ARCH_X86_32-4) + +%if ARCH_X86_64 + psrld m15, m11, 10 ; pd_2 +%else + mova m7, [o(pd_2)] +%endif + call m(inv_txfm_add_dct_dct_64x16_16bpc).main_end_loop_start + + lea r3, [rsp+56*16] + movzx t1d, word [o2(tbl_Nx64_offset)+r5*2+0] + movzx t3d, word [o2(tbl_Nx64_offset)+r5*2+2] + movzx t0d, t1b + movzx t2d, t3b + shr t1d, 8 + shr t3d, 8 + lea t4, [rsp+7*64*16+(64+8*ARCH_X86_32+1*ARCH_X86_64)*16] + call .shift_transpose + ; zero cq + pxor m7, m7 +%if ARCH_X86_32 + mov cq, [rsp+gprsize*3+(64*9+8)*16] +%endif + lea r4, [cq+30*128+r5*8] +.zero_cq_loop: + REPX {mova [r4+x*128], m7}, -2, -1, 0, 1 + sub r4, 4*128 + cmp r4, cq + jg .zero_cq_loop +%if ARCH_X86_32 + mov r6, [rsp+gprsize*4+(64*9+8)*16] +%endif + sub r5d, 2 + jge .loop_pass1 + + ; pass=2 code starts here + mov eobd, [rsp+gprsize*0+(9*64+8*ARCH_X86_32+1*ARCH_X86_64)*16] +%if ARCH_X86_32 + mov strideq, [rsp+gprsize*2+(9*64+8)*16] +%else + mov r0, [rsp+gprsize*0+64*16] +%endif + add rsp, (64+8*ARCH_X86_32+1*ARCH_X86_64-3)*16 + cmp eobd, 151 + jl .fast + ; fall-through +%if ARCH_X86_64 + DECLARE_REG_TMP 8, 9 +%else + DECLARE_REG_TMP 1, 5 +%endif + lea t0, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main_fast)] + lea t1, [o(m_suffix(idct_16x64_internal_8bpc, _ssse3).main)] + jmp .run +.fast: + lea t0, [o(m_suffix(idct_8x32_internal_8bpc, _ssse3).main_veryfast)] + lea t1, [o(m_suffix(idct_16x64_internal_8bpc, _ssse3).main_fast)] +.run: + +%if ARCH_X86_64 + lea r2, [dstq+128] + mov r7, -16 +%else + lea r2, [rsp+(64*8+3)*16] + mov [r2+4*gprsize], t0 + mov [r2+5*gprsize], t1 + mov r1, [r2+2*gprsize] + mov dword [r2+0*gprsize], 8 +%endif + jmp m(inv_txfm_add_dct_dct_16x64_16bpc).loop_pass2 + + ; copy of pass=1 tmp-regs +%if ARCH_X86_32 + DECLARE_REG_TMP 4, 1, 2, 0, 6 +%else + DECLARE_REG_TMP 8, 9, 4, 7, 0 +%endif + +.shift_transpose: + mova m0, [r3+0*16] + mova m1, [r3+1*16] + mova m2, [r3+2*16] + mova m3, [r3+3*16] + mova m4, [r3+4*16] + mova m5, [r3+5*16] + mova m6, [r3+6*16] + mova m7, [r3+7*16] + REPX {psrad x, 2}, m0, m1, m2, m3, m4, m5, m6, m7 + packssdw m0, m1 + packssdw m2, m3 + packssdw m4, m5 + packssdw m6, m7 + call m(idct_8x4_internal_16bpc).transpose4x8packed + mova [t4+t0*8], m0 + mova [t4+t1*8], m1 + mova [t4+t2*8], m2 + mova [t4+t3*8], m3 + sub t4, 16*64 + sub r3, 8*16 + cmp r3, rsp + jg .shift_transpose + ret + +.dconly: + imul r5d, [cq], 2896 + mov [cq], eobd ; 0 + mov r3d, 64 + add rsp, (64+8*ARCH_X86_32+8*64+1*ARCH_X86_64)*16 + \ + (4+4*ARCH_X86_32)*gprsize - (64+8*ARCH_X86_32)*16 + jmp m(inv_txfm_add_dct_dct_64x16_16bpc).dconly1 diff --git a/third_party/dav1d/src/x86/itx_avx2.asm b/third_party/dav1d/src/x86/itx_avx2.asm --- a/third_party/dav1d/src/x86/itx_avx2.asm +++ b/third_party/dav1d/src/x86/itx_avx2.asm @@ -1,6 +1,6 @@ -; Copyright © 2018, VideoLAN and dav1d authors +; Copyright © 2018-2021, VideoLAN and dav1d authors ; Copyright © 2018, Two Orioles, LLC ; All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions are met: @@ -342,11 +342,11 @@ psubw m0, m1 ; ____ out0 paddw m2, m1 ; out3 ____ %endmacro INIT_XMM avx2 -cglobal inv_txfm_add_wht_wht_4x4, 3, 3, 4, dst, stride, c +cglobal inv_txfm_add_wht_wht_4x4_8bpc, 3, 3, 4, dst, stride, c mova m0, [cq+16*0] mova m1, [cq+16*1] pxor m2, m2 mova [cq+16*0], m2 mova [cq+16*1], m2 @@ -360,16 +360,16 @@ IWHT4_1D_PACKED vpblendd m0, m2, 0x03 ITX4_END 3, 0, 2, 1, 0 %macro INV_TXFM_FN 3 ; type1, type2, size -cglobal inv_txfm_add_%1_%2_%3, 4, 5, 0, dst, stride, c, eob, tx2 - %define %%p1 m(i%1_%3_internal) +cglobal inv_txfm_add_%1_%2_%3_8bpc, 4, 5, 0, dst, stride, c, eob, tx2 + %define %%p1 m(i%1_%3_internal_8bpc) lea rax, [o_base] ; Jump to the 1st txfm function if we're not taking the fast path, which ; in turn performs an indirect jump to the 2nd txfm function. - lea tx2q, [m(i%2_%3_internal).pass2] + lea tx2q, [m(i%2_%3_internal_8bpc).pass2] %ifidn %1_%2, dct_dct test eobd, eobd jnz %%p1 %else ; jump to the 1st txfm function unless it's located directly after this @@ -386,11 +386,11 @@ vpbroadcastd m1, [o(pw_2896x8)] pmulhrsw m0, m1 mov [cq], eobd ; 0 pmulhrsw m0, m1 mova m1, m0 - jmp m(iadst_4x4_internal).end2 + jmp m(iadst_4x4_internal_8bpc).end2 %endif %endmacro %macro IDCT4_1D_PACKED 0 vpbroadcastd m4, [o(pd_2048)] @@ -436,11 +436,11 @@ INV_TXFM_4X4_FN dct, dct INV_TXFM_4X4_FN dct, adst INV_TXFM_4X4_FN dct, flipadst INV_TXFM_4X4_FN dct, identity -cglobal idct_4x4_internal, 0, 5, 6, dst, stride, c, eob, tx2 +cglobal idct_4x4_internal_8bpc, 0, 5, 6, dst, stride, c, eob, tx2 mova m0, [cq+16*0] mova m1, [cq+16*1] IDCT4_1D_PACKED mova m2, [o(deint_shuf)] shufps m3, m0, m1, q1331 @@ -458,11 +458,11 @@ INV_TXFM_4X4_FN adst, dct INV_TXFM_4X4_FN adst, adst INV_TXFM_4X4_FN adst, flipadst INV_TXFM_4X4_FN adst, identity -cglobal iadst_4x4_internal, 0, 5, 6, dst, stride, c, eob, tx2 +cglobal iadst_4x4_internal_8bpc, 0, 5, 6, dst, stride, c, eob, tx2 mova m0, [cq+16*0] mova m1, [cq+16*1] call .main punpckhwd m3, m0, m1 punpcklwd m0, m1 @@ -485,21 +485,21 @@ INV_TXFM_4X4_FN flipadst, dct INV_TXFM_4X4_FN flipadst, adst INV_TXFM_4X4_FN flipadst, flipadst INV_TXFM_4X4_FN flipadst, identity -cglobal iflipadst_4x4_internal, 0, 5, 6, dst, stride, c, eob, tx2 +cglobal iflipadst_4x4_internal_8bpc, 0, 5, 6, dst, stride, c, eob, tx2 mova m0, [cq+16*0] mova m1, [cq+16*1] - call m(iadst_4x4_internal).main + call m(iadst_4x4_internal_8bpc).main punpcklwd m2, m1, m0 punpckhwd m1, m0 punpcklwd m0, m1, m2 punpckhwd m1, m2 jmp tx2q .pass2: - call m(iadst_4x4_internal).main + call m(iadst_4x4_internal_8bpc).main .end: pxor m2, m2 mova [cq+16*0], m2 mova [cq+16*1], m2 .end2: @@ -508,11 +508,11 @@ INV_TXFM_4X4_FN identity, dct INV_TXFM_4X4_FN identity, adst INV_TXFM_4X4_FN identity, flipadst INV_TXFM_4X4_FN identity, identity -cglobal iidentity_4x4_internal, 0, 5, 6, dst, stride, c, eob, tx2 +cglobal iidentity_4x4_internal_8bpc, 0, 5, 6, dst, stride, c, eob, tx2 mova m0, [cq+16*0] mova m1, [cq+16*1] vpbroadcastd m3, [o(pw_1697x8)] pmulhrsw m2, m3, m0 pmulhrsw m3, m1 @@ -527,11 +527,11 @@ vpbroadcastd m3, [o(pw_1697x8)] pmulhrsw m2, m3, m0 pmulhrsw m3, m1 paddsw m0, m2 paddsw m1, m3 - jmp m(iadst_4x4_internal).end + jmp m(iadst_4x4_internal_8bpc).end %macro WRITE_4X8 2 ; coefs[1-2] movd xm4, [dstq+strideq*0] pinsrd xm4, [dstq+strideq*1], 1 movd xm5, [dstq+strideq*2] @@ -566,11 +566,11 @@ pmulhrsw xm0, xm1 pmulhrsw xm0, xm1 pmulhrsw xm0, xm2 vpbroadcastw m0, xm0 mova m1, m0 - jmp m(iadst_4x8_internal).end3 + jmp m(iadst_4x8_internal_8bpc).end3 %endif %endmacro %macro IDCT8_1D_PACKED 0 vpbroadcastd m6, [o(pd_2048)] @@ -685,11 +685,11 @@ INV_TXFM_4X8_FN dct, dct INV_TXFM_4X8_FN dct, adst INV_TXFM_4X8_FN dct, flipadst INV_TXFM_4X8_FN dct, identity -cglobal idct_4x8_internal, 0, 5, 7, dst, stride, c, eob, tx2 +cglobal idct_4x8_internal_8bpc, 0, 5, 7, dst, stride, c, eob, tx2 vpermq m0, [cq+32*0], q3120 vpermq m1, [cq+32*1], q3120 vpbroadcastd m2, [o(pw_2896x8)] pmulhrsw m0, m2 pmulhrsw m1, m2 @@ -706,28 +706,28 @@ call .main vpbroadcastd m4, [o(pw_2048)] vinserti128 m0, xm2, 1 vinserti128 m1, xm3, 1 pshufd m1, m1, q1032 - jmp m(iadst_4x8_internal).end2 + jmp m(iadst_4x8_internal_8bpc).end2 ALIGN function_align cglobal_label .main WRAP_XMM IDCT8_1D_PACKED ret INV_TXFM_4X8_FN adst, dct INV_TXFM_4X8_FN adst, adst INV_TXFM_4X8_FN adst, flipadst INV_TXFM_4X8_FN adst, identity -cglobal iadst_4x8_internal, 0, 5, 7, dst, stride, c, eob, tx2 +cglobal iadst_4x8_internal_8bpc, 0, 5, 7, dst, stride, c, eob, tx2 vpermq m0, [cq+32*0], q3120 vpermq m1, [cq+32*1], q3120 vpbroadcastd m2, [o(pw_2896x8)] pmulhrsw m0, m2 pmulhrsw m1, m2 - call m(iadst_8x4_internal).main + call m(iadst_8x4_internal_8bpc).main punpckhwd m3, m0, m1 punpcklwd m0, m1 punpckhwd m1, m0, m3 punpcklwd m0, m3 jmp tx2q @@ -768,43 +768,43 @@ INV_TXFM_4X8_FN flipadst, dct INV_TXFM_4X8_FN flipadst, adst INV_TXFM_4X8_FN flipadst, flipadst INV_TXFM_4X8_FN flipadst, identity -cglobal iflipadst_4x8_internal, 0, 5, 7, dst, stride, c, eob, tx2 +cglobal iflipadst_4x8_internal_8bpc, 0, 5, 7, dst, stride, c, eob, tx2 vpermq m0, [cq+32*0], q3120 vpermq m1, [cq+32*1], q3120 vpbroadcastd m2, [o(pw_2896x8)] pmulhrsw m0, m2 pmulhrsw m1, m2 - call m(iadst_8x4_internal).main + call m(iadst_8x4_internal_8bpc).main punpcklwd m3, m1, m0 punpckhwd m1, m0 punpcklwd m0, m1, m3 punpckhwd m1, m3 jmp tx2q .pass2: vextracti128 xm2, m0, 1 vextracti128 xm3, m1, 1 pshufd xm4, xm0, q1032 pshufd xm5, xm1, q1032 - call m(iadst_4x8_internal).main_pass2 + call m(iadst_4x8_internal_8bpc).main_pass2 vpbroadcastd m5, [o(pw_2048)] vinserti128 m3, xm1, 1 vinserti128 m2, xm0, 1 pxor m4, m4 psubw m4, m5 pshufd m0, m3, q1032 pshufd m1, m2, q1032 - jmp m(iadst_4x8_internal).end + jmp m(iadst_4x8_internal_8bpc).end INV_TXFM_4X8_FN identity, dct INV_TXFM_4X8_FN identity, adst INV_TXFM_4X8_FN identity, flipadst INV_TXFM_4X8_FN identity, identity -cglobal iidentity_4x8_internal, 0, 5, 7, dst, stride, c, eob, tx2 +cglobal iidentity_4x8_internal_8bpc, 0, 5, 7, dst, stride, c, eob, tx2 vpermq m2, [cq+32*0], q3120 vpermq m0, [cq+32*1], q3120 vpbroadcastd m3, [o(pw_2896x8)] vpbroadcastd m4, [o(pw_1697x8)] punpcklwd m1, m2, m0 @@ -818,11 +818,11 @@ paddsw m0, m2 paddsw m1, m4 jmp tx2q .pass2: vpbroadcastd m4, [o(pw_4096)] - jmp m(iadst_4x8_internal).end2 + jmp m(iadst_4x8_internal_8bpc).end2 %macro INV_TXFM_4X16_FN 2 ; type1, type2 INV_TXFM_FN %1, %2, 4x16 %ifidn %1_%2, dct_dct movd xm1, [o(pw_2896x8)] @@ -835,11 +835,11 @@ pmulhrsw xm0, xm3 vpbroadcastw m0, xm0 mova m1, m0 mova m2, m0 mova m3, m0 - jmp m(iadst_4x16_internal).end3 + jmp m(iadst_4x16_internal_8bpc).end3 %endif %endmacro %macro IDCT16_1D_PACKED 0 vpbroadcastd m10, [o(pd_2048)] @@ -913,16 +913,16 @@ INV_TXFM_4X16_FN dct, dct INV_TXFM_4X16_FN dct, adst INV_TXFM_4X16_FN dct, flipadst INV_TXFM_4X16_FN dct, identity -cglobal idct_4x16_internal, 0, 5, 11, dst, stride, c, eob, tx2 +cglobal idct_4x16_internal_8bpc, 0, 5, 11, dst, stride, c, eob, tx2 mova m0, [cq+32*0] mova m1, [cq+32*1] mova m2, [cq+32*2] mova m3, [cq+32*3] - call m(idct_16x4_internal).main + call m(idct_16x4_internal_8bpc).main vpbroadcastd m5, [o(pw_16384)] punpckhwd m4, m2, m3 punpcklwd m2, m3 punpckhwd m3, m0, m1 punpcklwd m0, m1 @@ -943,27 +943,27 @@ vpbroadcastd m5, [o(pw_2048)] vinserti128 m2, xm6, 1 vinserti128 m3, xm7, 1 pshufd m1, m1, q1032 pshufd m3, m3, q1032 - jmp m(iadst_4x16_internal).end2 + jmp m(iadst_4x16_internal_8bpc).end2 ALIGN function_align cglobal_label .main WRAP_XMM IDCT16_1D_PACKED ret INV_TXFM_4X16_FN adst, dct INV_TXFM_4X16_FN adst, adst INV_TXFM_4X16_FN adst, flipadst INV_TXFM_4X16_FN adst, identity -cglobal iadst_4x16_internal, 0, 5, 11, dst, stride, c, eob, tx2 +cglobal iadst_4x16_internal_8bpc, 0, 5, 11, dst, stride, c, eob, tx2 mova m0, [cq+32*0] mova m1, [cq+32*1] mova m2, [cq+32*2] mova m3, [cq+32*3] - call m(iadst_16x4_internal).main + call m(iadst_16x4_internal_8bpc).main vpbroadcastd m5, [o(pw_16384)] punpckhwd m4, m2, m3 punpcklwd m2, m3 punpckhwd m3, m0, m1 punpcklwd m0, m1 @@ -1083,16 +1083,16 @@ INV_TXFM_4X16_FN flipadst, dct INV_TXFM_4X16_FN flipadst, adst INV_TXFM_4X16_FN flipadst, flipadst INV_TXFM_4X16_FN flipadst, identity -cglobal iflipadst_4x16_internal, 0, 5, 11, dst, stride, c, eob, tx2 +cglobal iflipadst_4x16_internal_8bpc, 0, 5, 11, dst, stride, c, eob, tx2 mova m0, [cq+32*0] mova m1, [cq+32*1] mova m2, [cq+32*2] mova m3, [cq+32*3] - call m(iadst_16x4_internal).main + call m(iadst_16x4_internal_8bpc).main vpbroadcastd m5, [o(pw_16384)] punpcklwd m4, m1, m0 punpckhwd m1, m0 punpcklwd m0, m3, m2 punpckhwd m3, m2 @@ -1101,11 +1101,11 @@ punpckhdq m3, m1 punpckhdq m1, m0, m4 punpckldq m0, m4 jmp tx2q .pass2: - call m(iadst_4x16_internal).main + call m(iadst_4x16_internal_8bpc).main vpbroadcastd m5, [o(pw_2896x8)] paddsw m1, m2, m4 psubsw m2, m4 pmulhrsw m1, m5 ; -out7 out4 out6 -out5 pmulhrsw m2, m5 ; out8 -out11 -out9 out10 @@ -1118,18 +1118,18 @@ vpermq m0, m0, q3120 vpermq m1, m1, q0213 vpermq m2, m2, q2031 vpermq m3, m4, q1302 psubw m5, m7, m6 - jmp m(iadst_4x16_internal).end + jmp m(iadst_4x16_internal_8bpc).end INV_TXFM_4X16_FN identity, dct INV_TXFM_4X16_FN identity, adst INV_TXFM_4X16_FN identity, flipadst INV_TXFM_4X16_FN identity, identity -cglobal iidentity_4x16_internal, 0, 5, 11, dst, stride, c, eob, tx2 +cglobal iidentity_4x16_internal_8bpc, 0, 5, 11, dst, stride, c, eob, tx2 mova m3, [cq+32*0] mova m2, [cq+32*1] mova m4, [cq+32*2] mova m5, [cq+32*3] vpbroadcastd m8, [o(pw_1697x8)] @@ -1169,11 +1169,11 @@ REPX {paddsw x, x}, m0, m1, m2, m3 paddsw m0, m4 paddsw m1, m6 paddsw m2, m7 paddsw m3, m8 - jmp m(iadst_4x16_internal).end2 + jmp m(iadst_4x16_internal_8bpc).end2 %macro WRITE_8X4 4-7 strideq*1, strideq*2, r3 ; coefs[1-2], tmp[1-2], off[1-3] movq xm%3, [dstq ] movhps xm%3, [dstq+%5] movq xm%4, [dstq+%6] @@ -1207,26 +1207,26 @@ movd xm2, [o(pw_2048)] pmulhrsw xm0, xm1 pmulhrsw xm0, xm2 vpbroadcastw m0, xm0 mova m1, m0 - jmp m(iadst_8x4_internal).end3 + jmp m(iadst_8x4_internal_8bpc).end3 %endif %endmacro INV_TXFM_8X4_FN dct, dct INV_TXFM_8X4_FN dct, adst INV_TXFM_8X4_FN dct, flipadst INV_TXFM_8X4_FN dct, identity -cglobal idct_8x4_internal, 0, 5, 7, dst, stride, c, eob, tx2 +cglobal idct_8x4_internal_8bpc, 0, 5, 7, dst, stride, c, eob, tx2 vpbroadcastd xm3, [o(pw_2896x8)] pmulhrsw xm0, xm3, [cq+16*0] pmulhrsw xm1, xm3, [cq+16*1] pmulhrsw xm2, xm3, [cq+16*2] pmulhrsw xm3, [cq+16*3] - call m(idct_4x8_internal).main + call m(idct_4x8_internal_8bpc).main vbroadcasti128 m4, [o(deint_shuf)] vinserti128 m3, m1, xm3, 1 vinserti128 m1, m0, xm2, 1 shufps m0, m1, m3, q0220 shufps m1, m3, q1331 @@ -1235,26 +1235,26 @@ jmp tx2q .pass2: IDCT4_1D_PACKED vpermq m0, m0, q3120 vpermq m1, m1, q2031 - jmp m(iadst_8x4_internal).end2 + jmp m(iadst_8x4_internal_8bpc).end2 INV_TXFM_8X4_FN adst, dct INV_TXFM_8X4_FN adst, adst INV_TXFM_8X4_FN adst, flipadst INV_TXFM_8X4_FN adst, identity -cglobal iadst_8x4_internal, 0, 5, 7, dst, stride, c, eob, tx2 +cglobal iadst_8x4_internal_8bpc, 0, 5, 7, dst, stride, c, eob, tx2 vpbroadcastd xm0, [o(pw_2896x8)] pshufd xm4, [cq+16*0], q1032 pmulhrsw xm3, xm0, [cq+16*3] pshufd xm5, [cq+16*1], q1032 pmulhrsw xm2, xm0, [cq+16*2] pmulhrsw xm4, xm0 pmulhrsw xm5, xm0 - call m(iadst_4x8_internal).main_pass1 + call m(iadst_4x8_internal_8bpc).main_pass1 vinserti128 m0, xm2, 1 vinserti128 m1, xm3, 1 punpckhwd m2, m0, m1 punpcklwd m0, m1 pxor m3, m3 @@ -1287,41 +1287,41 @@ INV_TXFM_8X4_FN flipadst, dct INV_TXFM_8X4_FN flipadst, adst INV_TXFM_8X4_FN flipadst, flipadst INV_TXFM_8X4_FN flipadst, identity -cglobal iflipadst_8x4_internal, 0, 5, 7, dst, stride, c, eob, tx2 +cglobal iflipadst_8x4_internal_8bpc, 0, 5, 7, dst, stride, c, eob, tx2 vpbroadcastd xm0, [o(pw_2896x8)] pshufd xm4, [cq+16*0], q1032 pmulhrsw xm3, xm0, [cq+16*3] pshufd xm5, [cq+16*1], q1032 pmulhrsw xm2, xm0, [cq+16*2] pmulhrsw xm4, xm0 pmulhrsw xm5, xm0 - call m(iadst_4x8_internal).main_pass1 + call m(iadst_4x8_internal_8bpc).main_pass1 vinserti128 m3, xm1, 1 vinserti128 m2, xm0, 1 punpckhwd m1, m3, m2 punpcklwd m3, m2 pxor m0, m0 psubsw m0, m1 punpckhwd m1, m0, m3 punpcklwd m0, m3 jmp tx2q .pass2: - call m(iadst_8x4_internal).main + call m(iadst_8x4_internal_8bpc).main mova m2, m1 vpermq m1, m0, q2031 vpermq m0, m2, q2031 - jmp m(iadst_8x4_internal).end2 + jmp m(iadst_8x4_internal_8bpc).end2 INV_TXFM_8X4_FN identity, dct INV_TXFM_8X4_FN identity, adst INV_TXFM_8X4_FN identity, flipadst INV_TXFM_8X4_FN identity, identity -cglobal iidentity_8x4_internal, 0, 5, 7, dst, stride, c, eob, tx2 +cglobal iidentity_8x4_internal_8bpc, 0, 5, 7, dst, stride, c, eob, tx2 mova xm2, [cq+16*0] mova xm0, [cq+16*1] vinserti128 m2, [cq+16*2], 1 vinserti128 m0, [cq+16*3], 1 vpbroadcastd m3, [o(pw_2896x8)] @@ -1338,11 +1338,11 @@ vpbroadcastd m3, [o(pw_1697x8)] pmulhrsw m2, m3, m0 pmulhrsw m3, m1 paddsw m0, m2 paddsw m1, m3 - jmp m(iadst_8x4_internal).end + jmp m(iadst_8x4_internal_8bpc).end %macro INV_TXFM_8X8_FN 2 ; type1, type2 INV_TXFM_FN %1, %2, 8x8 %ifidn %1_%2, dct_dct movd xm1, [o(pw_2896x8)] @@ -1370,11 +1370,11 @@ INV_TXFM_8X8_FN dct, dct INV_TXFM_8X8_FN dct, adst INV_TXFM_8X8_FN dct, flipadst INV_TXFM_8X8_FN dct, identity -cglobal idct_8x8_internal, 0, 5, 7, dst, stride, c, eob, tx2 +cglobal idct_8x8_internal_8bpc, 0, 5, 7, dst, stride, c, eob, tx2 vpermq m0, [cq+32*0], q3120 ; 0 1 vpermq m3, [cq+32*3], q3120 ; 6 7 vpermq m2, [cq+32*2], q3120 ; 4 5 vpermq m1, [cq+32*1], q3120 ; 2 3 call .main @@ -1396,22 +1396,22 @@ vpbroadcastd m4, [o(pw_2048)] vpermq m0, m0, q3120 vpermq m1, m1, q2031 vpermq m2, m2, q3120 vpermq m3, m3, q2031 - jmp m(iadst_8x8_internal).end2 + jmp m(iadst_8x8_internal_8bpc).end2 ALIGN function_align cglobal_label .main IDCT8_1D_PACKED ret INV_TXFM_8X8_FN adst, dct INV_TXFM_8X8_FN adst, adst INV_TXFM_8X8_FN adst, flipadst INV_TXFM_8X8_FN adst, identity -cglobal iadst_8x8_internal, 0, 5, 7, dst, stride, c, eob, tx2 +cglobal iadst_8x8_internal_8bpc, 0, 5, 7, dst, stride, c, eob, tx2 vpermq m4, [cq+32*0], q1302 ; 1 0 vpermq m3, [cq+32*3], q3120 ; 6 7 vpermq m5, [cq+32*1], q1302 ; 3 2 vpermq m2, [cq+32*2], q3120 ; 4 5 call .main_pass1 @@ -1474,16 +1474,16 @@ INV_TXFM_8X8_FN flipadst, dct INV_TXFM_8X8_FN flipadst, adst INV_TXFM_8X8_FN flipadst, flipadst INV_TXFM_8X8_FN flipadst, identity -cglobal iflipadst_8x8_internal, 0, 5, 7, dst, stride, c, eob, tx2 +cglobal iflipadst_8x8_internal_8bpc, 0, 5, 7, dst, stride, c, eob, tx2 vpermq m4, [cq+32*0], q1302 ; 1 0 vpermq m3, [cq+32*3], q3120 ; 6 7 vpermq m5, [cq+32*1], q1302 ; 3 2 vpermq m2, [cq+32*2], q3120 ; 4 5 - call m(iadst_8x8_internal).main_pass1 + call m(iadst_8x8_internal_8bpc).main_pass1 vpbroadcastd m5, [o(pw_16384)] punpckhwd m4, m3, m2 punpcklwd m3, m2 punpckhwd m2, m1, m0 punpcklwd m1, m0 @@ -1503,28 +1503,28 @@ vperm2i128 m2, m4, m2, 0x31 jmp tx2q .pass2: pshufd m4, m0, q1032 pshufd m5, m1, q1032 - call m(iadst_8x8_internal).main_pass2 + call m(iadst_8x8_internal_8bpc).main_pass2 vpbroadcastd m4, [o(pw_2048)] vpbroadcastd xm5, [o(pw_4096)] psubw m4, m5 ; lower half = -2048, upper half = 2048 vpermq m5, m3, q2031 vpermq m3, m0, q2031 vpermq m0, m2, q2031 vpermq m2, m1, q2031 pmulhrsw m1, m0, m4 pmulhrsw m0, m5, m4 - jmp m(iadst_8x8_internal).end3 + jmp m(iadst_8x8_internal_8bpc).end3 INV_TXFM_8X8_FN identity, dct INV_TXFM_8X8_FN identity, adst INV_TXFM_8X8_FN identity, flipadst INV_TXFM_8X8_FN identity, identity -cglobal iidentity_8x8_internal, 0, 5, 7, dst, stride, c, eob, tx2 +cglobal iidentity_8x8_internal_8bpc, 0, 5, 7, dst, stride, c, eob, tx2 mova xm3, [cq+16*0] mova xm2, [cq+16*1] vinserti128 m3, [cq+16*4], 1 vinserti128 m2, [cq+16*5], 1 mova xm4, [cq+16*2] @@ -1540,11 +1540,11 @@ punpckldq m2, m3, m4 punpckhdq m3, m4 jmp tx2q .pass2: vpbroadcastd m4, [o(pw_4096)] - jmp m(iadst_8x8_internal).end + jmp m(iadst_8x8_internal_8bpc).end %macro INV_TXFM_8X16_FN 2 ; type1, type2 INV_TXFM_FN %1, %2, 8x16 %ifidn %1_%2, dct_dct movd xm1, [o(pw_2896x8)] @@ -1556,11 +1556,11 @@ psrlw xm2, 3 ; pw_2048 pmulhrsw xm0, xm1 pmulhrsw xm0, xm2 vpbroadcastw m0, xm0 mov r2d, 4 - jmp m(inv_txfm_add_dct_dct_8x8).end2 + jmp m(inv_txfm_add_dct_dct_8x8_8bpc).end2 %endif %endmacro %macro ITX_8X16_LOAD_COEFS 0 vpbroadcastd m4, [o(pw_2896x8)] @@ -1578,13 +1578,13 @@ INV_TXFM_8X16_FN dct, dct INV_TXFM_8X16_FN dct, adst INV_TXFM_8X16_FN dct, flipadst INV_TXFM_8X16_FN dct, identity -cglobal idct_8x16_internal, 0, 5, 13, dst, stride, c, eob, tx2 +cglobal idct_8x16_internal_8bpc, 0, 5, 13, dst, stride, c, eob, tx2 ITX_8X16_LOAD_COEFS - call m(idct_16x8_internal).main + call m(idct_16x8_internal_8bpc).main vpbroadcastd m10, [o(pw_16384)] .pass1_end: vperm2i128 m9, m3, m7, 0x31 vinserti128 m3, xm7, 1 vperm2i128 m8, m2, m6, 0x31 @@ -1640,28 +1640,28 @@ INV_TXFM_8X16_FN adst, dct INV_TXFM_8X16_FN adst, adst INV_TXFM_8X16_FN adst, flipadst INV_TXFM_8X16_FN adst, identity -cglobal iadst_8x16_internal, 0, 5, 13, dst, stride, c, eob, tx2 +cglobal iadst_8x16_internal_8bpc, 0, 5, 13, dst, stride, c, eob, tx2 ITX_8X16_LOAD_COEFS - call m(iadst_16x8_internal).main - call m(iadst_16x8_internal).main_pass1_end + call m(iadst_16x8_internal_8bpc).main + call m(iadst_16x8_internal_8bpc).main_pass1_end vpbroadcastd m10, [o(pw_16384)] pslld m9, m10, 17 psubw m10, m9 ; 16384, -16384 - jmp m(idct_8x16_internal).pass1_end + jmp m(idct_8x16_internal_8bpc).pass1_end ALIGN function_align .pass2: call .main call .main_pass2_end vpbroadcastd m9, [o(pw_2048)] vpbroadcastd xm8, [o(pw_4096)] psubw m8, m9 REPX {vpermq x, x, q2031}, m0, m1, m2, m3 REPX {vpermq x, x, q3120}, m4, m5, m6, m7 - jmp m(idct_8x16_internal).end2 + jmp m(idct_8x16_internal_8bpc).end2 ALIGN function_align cglobal_label .main REPX {pshufd x, x, q1032}, m7, m1, m5, m3 .main2: vpbroadcastd m10, [o(pd_2048)] @@ -1781,14 +1781,14 @@ INV_TXFM_8X16_FN flipadst, dct INV_TXFM_8X16_FN flipadst, adst INV_TXFM_8X16_FN flipadst, flipadst INV_TXFM_8X16_FN flipadst, identity -cglobal iflipadst_8x16_internal, 0, 5, 13, dst, stride, c, eob, tx2 +cglobal iflipadst_8x16_internal_8bpc, 0, 5, 13, dst, stride, c, eob, tx2 ITX_8X16_LOAD_COEFS - call m(iadst_16x8_internal).main - call m(iadst_16x8_internal).main_pass1_end + call m(iadst_16x8_internal_8bpc).main + call m(iadst_16x8_internal_8bpc).main_pass1_end vpbroadcastd m9, [o(pw_16384)] pslld m10, m9, 17 psubw m10, m9 ; -16384, 16384 vperm2i128 m9, m4, m0, 0x31 vinserti128 m0, m4, xm0, 1 @@ -1800,14 +1800,14 @@ vperm2i128 m6, m6, m2, 0x31 punpcklwd m2, m4, m0 punpckhwd m4, m0 punpcklwd m0, m3, m1 punpckhwd m3, m1 - jmp m(idct_8x16_internal).pass1_end2 + jmp m(idct_8x16_internal_8bpc).pass1_end2 .pass2: - call m(iadst_8x16_internal).main - call m(iadst_8x16_internal).main_pass2_end + call m(iadst_8x16_internal_8bpc).main + call m(iadst_8x16_internal_8bpc).main_pass2_end vpbroadcastd m8, [o(pw_2048)] vpbroadcastd xm9, [o(pw_4096)] psubw m8, m9 vpermq m9, m0, q3120 vpermq m0, m7, q2031 @@ -1823,11 +1823,11 @@ pmulhrsw m3, m8 pmulhrsw m4, m5, m8 pmulhrsw m5, m6, m8 pmulhrsw m6, m7, m8 pmulhrsw m7, m9, m8 - jmp m(idct_8x16_internal).end3 + jmp m(idct_8x16_internal_8bpc).end3 INV_TXFM_8X16_FN identity, dct INV_TXFM_8X16_FN identity, adst INV_TXFM_8X16_FN identity, flipadst INV_TXFM_8X16_FN identity, identity @@ -1840,11 +1840,11 @@ paddsw m%1, m%1 %endif paddsw m%1, m%2 %endmacro -cglobal iidentity_8x16_internal, 0, 5, 13, dst, stride, c, eob, tx2 +cglobal iidentity_8x16_internal_8bpc, 0, 5, 13, dst, stride, c, eob, tx2 mova xm3, [cq+16*0] mova xm2, [cq+16*2] add cq, 16*8 vinserti128 m3, [cq+16*0], 1 vinserti128 m2, [cq+16*2], 1 @@ -1881,11 +1881,11 @@ jmp tx2q .pass2: vpbroadcastd m8, [o(pw_1697x16)] REPX {vpermq x, x, q3120}, m0, m1, m2, m3, m4, m5, m6, m7 REPX {IDTX16 x, 9, 8}, 0, 1, 2, 3, 4, 5, 6, 7 - jmp m(idct_8x16_internal).end + jmp m(idct_8x16_internal_8bpc).end %macro WRITE_16X2 6 ; coefs[1-2], tmp[1-2], offset[1-2] pmovzxbw m%3, [dstq+%5] %ifnum %1 paddw m%3, m%1 @@ -1939,34 +1939,34 @@ INV_TXFM_16X4_FN dct, dct INV_TXFM_16X4_FN dct, adst INV_TXFM_16X4_FN dct, flipadst INV_TXFM_16X4_FN dct, identity -cglobal idct_16x4_internal, 0, 5, 11, dst, stride, c, eob, tx2 +cglobal idct_16x4_internal_8bpc, 0, 5, 11, dst, stride, c, eob, tx2 mova xm0, [cq+16*0] mova xm1, [cq+16*1] mova xm2, [cq+16*2] mova xm3, [cq+16*3] mova xm4, [cq+16*4] mova xm5, [cq+16*5] mova xm6, [cq+16*6] mova xm7, [cq+16*7] - call m(idct_4x16_internal).main + call m(idct_4x16_internal_8bpc).main vinserti128 m6, m2, xm6, 1 vinserti128 m2, m0, xm4, 1 vinserti128 m0, m1, xm5, 1 vinserti128 m1, m3, xm7, 1 punpcklwd m3, m2, m6 punpckhwd m2, m6 vpbroadcastd m6, [o(pw_16384)] punpckhwd m4, m0, m1 punpcklwd m0, m1 mova m1, m6 - jmp m(iadst_16x4_internal).pass1_end + jmp m(iadst_16x4_internal_8bpc).pass1_end .pass2: call .main - jmp m(iadst_16x4_internal).end + jmp m(iadst_16x4_internal_8bpc).end ALIGN function_align cglobal_label .main vpbroadcastd m6, [o(pd_2048)] IDCT4_1D 0, 1, 2, 3, 4, 5, 6 ret @@ -1974,17 +1974,17 @@ INV_TXFM_16X4_FN adst, dct INV_TXFM_16X4_FN adst, adst INV_TXFM_16X4_FN adst, flipadst INV_TXFM_16X4_FN adst, identity -cglobal iadst_16x4_internal, 0, 5, 11, dst, stride, c, eob, tx2 +cglobal iadst_16x4_internal_8bpc, 0, 5, 11, dst, stride, c, eob, tx2 vpermq m0, [cq+32*0], q1230 vpermq m3, [cq+32*3], q2103 vpermq m1, [cq+32*1], q1230 vpermq m2, [cq+32*2], q2103 - call m(iadst_4x16_internal).main2 - call m(iadst_4x16_internal).main_pass1_end + call m(iadst_4x16_internal_8bpc).main2 + call m(iadst_4x16_internal_8bpc).main_pass1_end punpcklwd m4, m3, m1 punpcklwd m5, m2, m0 punpckhwd m0, m1 punpckhwd m2, m3 vpbroadcastd m1, [o(pw_16384)] @@ -2078,31 +2078,31 @@ INV_TXFM_16X4_FN flipadst, dct INV_TXFM_16X4_FN flipadst, adst INV_TXFM_16X4_FN flipadst, flipadst INV_TXFM_16X4_FN flipadst, identity -cglobal iflipadst_16x4_internal, 0, 5, 11, dst, stride, c, eob, tx2 +cglobal iflipadst_16x4_internal_8bpc, 0, 5, 11, dst, stride, c, eob, tx2 vpermq m0, [cq+32*0], q1230 vpermq m3, [cq+32*3], q2103 vpermq m1, [cq+32*1], q1230 vpermq m2, [cq+32*2], q2103 - call m(iadst_4x16_internal).main2 - call m(iadst_4x16_internal).main_pass1_end + call m(iadst_4x16_internal_8bpc).main2 + call m(iadst_4x16_internal_8bpc).main_pass1_end punpckhwd m4, m3, m2 punpckhwd m5, m1, m0 punpcklwd m0, m2 punpcklwd m1, m3 vpbroadcastd m6, [o(pw_16384)] vinserti128 m3, m0, xm1, 1 vperm2i128 m2, m0, m1, 0x31 vinserti128 m0, m4, xm5, 1 vperm2i128 m4, m4, m5, 0x31 psubw m1, m7, m6 - jmp m(iadst_16x4_internal).pass1_end + jmp m(iadst_16x4_internal_8bpc).pass1_end ALIGN function_align .pass2: - call m(iadst_16x4_internal).main + call m(iadst_16x4_internal_8bpc).main vpbroadcastd m4, [o(pw_2048)] REPX {pmulhrsw x, m4}, m3, m2, m1, m0 pxor m4, m4 mova [cq+32*0], m4 mova [cq+32*1], m4 @@ -2116,11 +2116,11 @@ INV_TXFM_16X4_FN identity, dct INV_TXFM_16X4_FN identity, adst INV_TXFM_16X4_FN identity, flipadst INV_TXFM_16X4_FN identity, identity -cglobal iidentity_16x4_internal, 0, 5, 11, dst, stride, c, eob, tx2 +cglobal iidentity_16x4_internal_8bpc, 0, 5, 11, dst, stride, c, eob, tx2 mova xm2, [cq+16*0] mova xm4, [cq+16*1] vinserti128 m2, [cq+16*4], 1 vinserti128 m4, [cq+16*5], 1 mova xm0, [cq+16*2] @@ -2159,22 +2159,22 @@ pmulhrsw m7, m3 paddsw m0, m4 paddsw m1, m5 paddsw m2, m6 paddsw m3, m7 - jmp m(iadst_16x4_internal).end + jmp m(iadst_16x4_internal_8bpc).end %macro INV_TXFM_16X8_FN 2 ; type1, type2 INV_TXFM_FN %1, %2, 16x8 %ifidn %1_%2, dct_dct movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] movd xm2, [o(pw_16384)] mov [cq], eobd pmulhrsw xm0, xm1 mov r2d, 4 - jmp m(inv_txfm_add_dct_dct_16x4).dconly + jmp m(inv_txfm_add_dct_dct_16x4_8bpc).dconly %endif %endmacro %macro ITX_16X8_LOAD_COEFS 1 ; shuf_odd vpbroadcastd m8, [o(pw_2896x8)] @@ -2193,13 +2193,13 @@ INV_TXFM_16X8_FN dct, dct INV_TXFM_16X8_FN dct, adst INV_TXFM_16X8_FN dct, flipadst INV_TXFM_16X8_FN dct, identity -cglobal idct_16x8_internal, 0, 5, 13, dst, stride, c, eob, tx2 +cglobal idct_16x8_internal_8bpc, 0, 5, 13, dst, stride, c, eob, tx2 ITX_16X8_LOAD_COEFS 3120 - call m(idct_8x16_internal).main + call m(idct_8x16_internal_8bpc).main vpbroadcastd m10, [o(pw_16384)] punpckhwd m8, m0, m2 punpcklwd m0, m2 punpckhwd m2, m1, m3 punpcklwd m1, m3 @@ -2263,33 +2263,33 @@ INV_TXFM_16X8_FN adst, dct INV_TXFM_16X8_FN adst, adst INV_TXFM_16X8_FN adst, flipadst INV_TXFM_16X8_FN adst, identity -cglobal iadst_16x8_internal, 0, 5, 13, dst, stride, c, eob, tx2 +cglobal iadst_16x8_internal_8bpc, 0, 5, 13, dst, stride, c, eob, tx2 ITX_16X8_LOAD_COEFS 1302 - call m(iadst_8x16_internal).main2 - call m(iadst_8x16_internal).main_pass1_end + call m(iadst_8x16_internal_8bpc).main2 + call m(iadst_8x16_internal_8bpc).main_pass1_end psubw m11, m9, m10 punpcklwd m8, m0, m2 punpckhwd m0, m2 punpckhwd m2, m1, m3 punpcklwd m1, m3 punpcklwd m9, m4, m6 punpckhwd m4, m6 punpckhwd m6, m5, m7 punpcklwd m5, m7 REPX {pmulhrsw x, m11}, m8, m1, m4, m6 - jmp m(idct_16x8_internal).pass1_end + jmp m(idct_16x8_internal_8bpc).pass1_end ALIGN function_align .pass2: call .main call .main_pass2_end pxor m8, m8 psubw m8, m9 REPX {pmulhrsw x, m9}, m0, m2, m4, m6 - jmp m(idct_16x8_internal).end2 + jmp m(idct_16x8_internal_8bpc).end2 ALIGN function_align cglobal_label .main vpbroadcastd m10, [o(pd_2048)] ITX_MULSUB_2W 7, 0, 8, 9, 10, 401, 4076 ; t1a, t0a ITX_MULSUB_2W 3, 4, 8, 9, 10, 3166, 2598 ; t5a, t4a @@ -2356,14 +2356,14 @@ INV_TXFM_16X8_FN flipadst, dct INV_TXFM_16X8_FN flipadst, adst INV_TXFM_16X8_FN flipadst, flipadst INV_TXFM_16X8_FN flipadst, identity -cglobal iflipadst_16x8_internal, 0, 5, 13, dst, stride, c, eob, tx2 +cglobal iflipadst_16x8_internal_8bpc, 0, 5, 13, dst, stride, c, eob, tx2 ITX_16X8_LOAD_COEFS 1302 - call m(iadst_8x16_internal).main2 - call m(iadst_8x16_internal).main_pass1_end + call m(iadst_8x16_internal_8bpc).main2 + call m(iadst_8x16_internal_8bpc).main_pass1_end psubw m9, m10 punpcklwd m8, m6, m4 punpckhwd m6, m4 punpcklwd m4, m7, m5 punpckhwd m7, m5 @@ -2397,12 +2397,12 @@ vinserti128 m0, xm3, 1 vinserti128 m3, m7, xm8, 1 vperm2i128 m7, m8, 0x31 jmp tx2q .pass2: - call m(iadst_16x8_internal).main - call m(iadst_16x8_internal).main_pass2_end + call m(iadst_16x8_internal_8bpc).main + call m(iadst_16x8_internal_8bpc).main_pass2_end pxor m8, m8 psubw m8, m9 pmulhrsw m10, m7, m8 pmulhrsw m7, m0, m9 pmulhrsw m0, m6, m9 @@ -2412,18 +2412,18 @@ pmulhrsw m2, m4, m9 pmulhrsw m4, m3, m8 lea r3, [strideq*3] WRITE_16X2 10, 0, 8, 9, strideq*0, strideq*1 WRITE_16X2 1, 2, 0, 1, strideq*2, r3 - jmp m(idct_16x8_internal).end3 + jmp m(idct_16x8_internal_8bpc).end3 INV_TXFM_16X8_FN identity, dct INV_TXFM_16X8_FN identity, adst INV_TXFM_16X8_FN identity, flipadst INV_TXFM_16X8_FN identity, identity -cglobal iidentity_16x8_internal, 0, 5, 13, dst, stride, c, eob, tx2 +cglobal iidentity_16x8_internal_8bpc, 0, 5, 13, dst, stride, c, eob, tx2 mova xm7, [cq+16*0] mova xm2, [cq+16*1] add cq, 16*8 vpbroadcastd m3, [o(pw_2896x8)] vinserti128 m7, [cq+16*0], 1 @@ -2469,11 +2469,11 @@ punpcklqdq m6, m7, m8 punpckhqdq m7, m8 jmp tx2q .pass2: vpbroadcastd m8, [o(pw_4096)] - jmp m(idct_16x8_internal).end + jmp m(idct_16x8_internal_8bpc).end %define o_base pw_5 + 128 %macro INV_TXFM_16X16_FN 2 ; type1, type2 INV_TXFM_FN %1, %2, 16x16 @@ -2481,11 +2481,11 @@ movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] movd xm2, [o(pw_8192)] mov [cq], eobd mov r2d, 8 - jmp m(inv_txfm_add_dct_dct_16x4).dconly + jmp m(inv_txfm_add_dct_dct_16x4_8bpc).dconly %endif %endmacro %macro ITX_16X16_LOAD_COEFS 0 mova m0, [cq+32*0] @@ -2511,11 +2511,11 @@ INV_TXFM_16X16_FN dct, dct INV_TXFM_16X16_FN dct, adst INV_TXFM_16X16_FN dct, flipadst INV_TXFM_16X16_FN dct, identity -cglobal idct_16x16_internal, 0, 5, 16, 32*3, dst, stride, c, eob, tx2 +cglobal idct_16x16_internal_8bpc, 0, 5, 16, 32*3, dst, stride, c, eob, tx2 ITX_16X16_LOAD_COEFS call .main .pass1_end: vpbroadcastd m1, [o(pw_8192)] REPX {pmulhrsw x, m1}, m0, m2, m4, m6, m8, m10, m12, m14 @@ -2656,11 +2656,11 @@ INV_TXFM_16X16_FN adst, dct INV_TXFM_16X16_FN adst, adst INV_TXFM_16X16_FN adst, flipadst -cglobal iadst_16x16_internal, 0, 5, 16, 32*3, dst, stride, c, eob, tx2 +cglobal iadst_16x16_internal_8bpc, 0, 5, 16, 32*3, dst, stride, c, eob, tx2 ITX_16X16_LOAD_COEFS call .main call .main_pass1_end pmulhrsw m0, m1, [cq+32*0] pmulhrsw m2, m1, [cq+32*1] @@ -2669,20 +2669,20 @@ pmulhrsw m14, m1, [cq+32*3] vextracti128 [rsp+16*5], m8, 1 mova [rsp+16*1], xm8 pxor m8, m8 psubw m1, m8, m1 - jmp m(idct_16x16_internal).pass1_end2 + jmp m(idct_16x16_internal_8bpc).pass1_end2 ALIGN function_align .pass2: call .main call .main_pass2_end REPX {pmulhrsw x, m1}, m0, m2, m4, m6, m8, m10, m12, m14 mova [rsp+32*0], m6 pxor m6, m6 psubw m1, m6, m1 - jmp m(idct_16x16_internal).end2 + jmp m(idct_16x16_internal_8bpc).end2 ALIGN function_align cglobal_label .main vpbroadcastd m15, [o(pd_2048)] mova [rsp+gprsize+32*1], m0 mova [rsp+gprsize+32*2], m4 @@ -2831,14 +2831,14 @@ INV_TXFM_16X16_FN flipadst, dct INV_TXFM_16X16_FN flipadst, adst INV_TXFM_16X16_FN flipadst, flipadst -cglobal iflipadst_16x16_internal, 0, 5, 16, 32*3, dst, stride, c, eob, tx2 +cglobal iflipadst_16x16_internal_8bpc, 0, 5, 16, 32*3, dst, stride, c, eob, tx2 ITX_16X16_LOAD_COEFS - call m(iadst_16x16_internal).main - call m(iadst_16x16_internal).main_pass1_end + call m(iadst_16x16_internal_8bpc).main + call m(iadst_16x16_internal_8bpc).main_pass1_end pmulhrsw m6, m1 pmulhrsw m2, m1, m8 mova [rsp+32*2], m6 pmulhrsw m6, m1, m4 pmulhrsw m4, m1, m10 @@ -2867,14 +2867,14 @@ vperm2i128 m10, m10, m6, 0x31 vinserti128 m6, m1, xm0, 1 vperm2i128 m13, m1, m0, 0x31 vinserti128 m1, m8, [rsp+32*2], 1 vperm2i128 m8, m8, [rsp+32*2], 0x31 - jmp m(idct_16x16_internal).pass1_end3 + jmp m(idct_16x16_internal_8bpc).pass1_end3 .pass2: - call m(iadst_16x16_internal).main - call m(iadst_16x16_internal).main_pass2_end + call m(iadst_16x16_internal_8bpc).main + call m(iadst_16x16_internal_8bpc).main_pass2_end pmulhrsw m0, m1 pmulhrsw m8, m1 mova [rsp+32*0], m0 mova [rsp+32*2], m8 pxor m0, m0 @@ -2898,22 +2898,22 @@ mova m15, [rsp+32*0] WRITE_16X2 3, 4, 0, 1, strideq*2, r3 lea dstq, [dstq+strideq*4] WRITE_16X2 5, 6, 0, 1, strideq*0, strideq*1 WRITE_16X2 7, [rsp+32*2], 0, 1, strideq*2, r3 - jmp m(idct_16x16_internal).end3 + jmp m(idct_16x16_internal_8bpc).end3 %macro IDTX16B 3 ; src/dst, tmp, pw_1697x16 pmulhrsw m%2, m%3, m%1 psraw m%2, 1 pavgw m%1, m%2 ; signs are guaranteed to be equal %endmacro INV_TXFM_16X16_FN identity, dct INV_TXFM_16X16_FN identity, identity -cglobal iidentity_16x16_internal, 0, 5, 16, 32*3, dst, stride, c, eob, tx2 +cglobal iidentity_16x16_internal_8bpc, 0, 5, 16, 32*3, dst, stride, c, eob, tx2 vpbroadcastd m7, [o(pw_1697x16)] mova xm0, [cq+16* 0] vinserti128 m0, [cq+16*16], 1 mova xm15, [cq+16* 1] vinserti128 m15, [cq+16*17], 1 @@ -2951,11 +2951,11 @@ mova xm0, [cq-16* 2] vinserti128 m0, [cq+16*14], 1 pmulhrsw m7, m0 psraw m7, 1 pavgw m7, m0 - jmp m(idct_16x16_internal).pass1_end3 + jmp m(idct_16x16_internal_8bpc).pass1_end3 ALIGN function_align .pass2: vpbroadcastd m15, [o(pw_1697x16)] mova [rsp+32*1], m0 REPX {IDTX16 x, 0, 15}, 1, 2, 3, 4, 5, 6, 7, \ @@ -2965,11 +2965,11 @@ IDTX16 0, 1, 15 mova m1, [rsp+32*0] pmulhrsw m15, m1 paddsw m1, m1 paddsw m15, m1 - jmp m(idct_16x16_internal).end + jmp m(idct_16x16_internal_8bpc).end %define o_base deint_shuf + 128 %macro LOAD_8ROWS 2-3 0 ; src, stride, is_rect2 %if %3 @@ -3026,20 +3026,20 @@ vpbroadcastd m%3, [r5-pw_201_4091x8+pw_%6_%7x8] punpckhwd m%2, m%2 pmulhrsw m%2, m%3 %endmacro -cglobal inv_txfm_add_dct_dct_8x32, 4, 4, 0, dst, stride, c, eob +cglobal inv_txfm_add_dct_dct_8x32_8bpc, 4, 4, 0, dst, stride, c, eob lea rax, [o_base] test eobd, eobd jz .dconly PROLOGUE 0, 4, 16, 32*3, dst, stride, c, eob %undef cmp cmp eobd, 106 jle .fast LOAD_8ROWS cq+32*1, 32*2 - call m(idct_16x8_internal).main + call m(idct_16x8_internal_8bpc).main vperm2i128 m11, m0, m4, 0x31 vinserti128 m0, xm4, 1 vperm2i128 m4, m1, m5, 0x31 vinserti128 m1, xm5, 1 vperm2i128 m5, m2, m6, 0x31 @@ -3075,11 +3075,11 @@ mova [rsp+32*0], m0 mova [rsp+32*1], m4 mova [rsp+32*2], m2 .fast: LOAD_8ROWS cq+32*0, 32*2 - call m(idct_16x8_internal).main + call m(idct_16x8_internal_8bpc).main vperm2i128 m8, m0, m4, 0x31 vinserti128 m0, xm4, 1 vperm2i128 m4, m1, m5, 0x31 vinserti128 m1, xm5, 1 vperm2i128 m5, m2, m6, 0x31 @@ -3133,11 +3133,11 @@ psrlw xm2, 2 ; pw_2048 pmulhrsw xm0, xm1 pmulhrsw xm0, xm2 vpbroadcastw m0, xm0 mov r2d, 8 - jmp m(inv_txfm_add_dct_dct_8x8).end2 + jmp m(inv_txfm_add_dct_dct_8x8_8bpc).end2 .full: REPX {pmulhrsw x, m9}, m12, m13, m14, m15 pmulhrsw m6, m9, [rsp+32*2] mova [rsp+32*2], m4 pmulhrsw m4, m9, [rsp+32*0] @@ -3173,11 +3173,11 @@ lea dstq, [dstq+strideq*4] WRITE_8X4 14, 15, 4, 6 RET ALIGN function_align cglobal_label .main_fast ; bottom half is zero - call m(idct_8x16_internal).main + call m(idct_8x16_internal_8bpc).main mova m8, [rsp+gprsize+0*32] mova [rsp+gprsize+0*32], m0 mova m9, [rsp+gprsize+1*32] mova [rsp+gprsize+1*32], m1 mova m0, [rsp+gprsize+2*32] @@ -3188,11 +3188,11 @@ ITX_UNPACK_MULHRSW 14, 0, 6, 1751, 3703, m2106, 3513 ; t18a, t29a, t21a, t26a ITX_UNPACK_MULHRSW 13, 11, 6, 2440, 3290, m2751, 3035 ; t22a, t25a, t17a, t30a jmp .main2 ALIGN function_align cglobal_label .main - call m(idct_8x16_internal).main + call m(idct_8x16_internal_8bpc).main mova m8, [rsp+gprsize+0*32] mova [rsp+gprsize+0*32], m0 mova m9, [rsp+gprsize+1*32] mova [rsp+gprsize+1*32], m1 mova m0, [rsp+gprsize+2*32] @@ -3289,11 +3289,11 @@ vbroadcasti128 m%1, [cq+16*%3] vbroadcasti128 m%2, [cq+16*%4] shufpd m%1, m%2, 0x0c %endmacro -cglobal inv_txfm_add_dct_dct_32x8, 4, 4, 0, dst, stride, c, eob +cglobal inv_txfm_add_dct_dct_32x8_8bpc, 4, 4, 0, dst, stride, c, eob lea rax, [o_base] test eobd, eobd jnz .normal movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] @@ -3339,11 +3339,11 @@ mova [rsp+32*2], m6 cmp eobd, 106 jg .full pxor m4, m4 REPX {mova x, m4}, m5, m6, m7 - call m(inv_txfm_add_dct_dct_8x32).main_fast + call m(inv_txfm_add_dct_dct_8x32_8bpc).main_fast jmp .pass2 .full: LOAD_PACKED_16X2 4, 7, 0, 2 ; in16 in18 LOAD_PACKED_16X2 12, 7, 3, 1 ; in19 in17 LOAD_PACKED_16X2 5, 7, 4, 6 ; in20 in22 @@ -3354,11 +3354,11 @@ LOAD_PACKED_16X2 14, 7, 3, 1 ; in27 in25 LOAD_PACKED_16X2 7, 8, 4, 6 ; in28 in30 LOAD_PACKED_16X2 15, 8, 7, 5 ; in31 in29 pxor m8, m8 REPX {mova [cq+32*x], m8}, 0, 1, 2, 3 - call m(inv_txfm_add_dct_dct_8x32).main + call m(inv_txfm_add_dct_dct_8x32_8bpc).main .pass2: vpbroadcastd m12, [o(pw_8192)] REPX {pmulhrsw x, m12}, m8, m9, m10, m11, m13, m14, m15 mova [rsp+32*1], m9 mova [rsp+32*2], m10 @@ -3395,11 +3395,11 @@ vinserti128 m1, xm7, 1 vperm2i128 m6, m2, m9, 0x31 vinserti128 m2, xm9, 1 vperm2i128 m7, m3, m10, 0x31 vinserti128 m3, xm10, 1 - call m(idct_16x8_internal).main + call m(idct_16x8_internal_8bpc).main vpbroadcastd m8, [o(pw_2048)] REPX {pmulhrsw x, m8}, m0, m1, m2, m3, m4, m5, m6, m7 lea r2, [strideq*3] WRITE_16X2 0, 1, 8, 0, strideq*0, strideq*1 WRITE_16X2 2, 3, 0, 1, strideq*2, r2 @@ -3440,11 +3440,11 @@ vinserti128 m1, xm7, 1 vperm2i128 m6, m2, m9, 0x31 vinserti128 m2, xm9, 1 vperm2i128 m7, m3, m12, 0x31 vinserti128 m3, xm12, 1 - call m(idct_16x8_internal).main2 + call m(idct_16x8_internal_8bpc).main2 vpbroadcastd m8, [o(pw_2048)] REPX {pmulhrsw x, m8}, m0, m1, m2, m3, m4, m5, m6, m7 add r0, 16 add r3, 16 %define dstq r0 @@ -3453,11 +3453,11 @@ %define dstq r3 WRITE_16X2 4, 5, 0, 1, strideq*0, strideq*1 WRITE_16X2 6, 7, 0, 1, strideq*2, r2 RET -cglobal inv_txfm_add_identity_identity_8x32, 4, 5, 11, dst, stride, c, eob +cglobal inv_txfm_add_identity_identity_8x32_8bpc, 4, 5, 11, dst, stride, c, eob vpbroadcastd m9, [pw_5] lea r4, [strideq*3] sub eobd, 107 ; loop_iterations = 1 + (eobd >= 107) .loop: mova xm0,[cq+16* 0] @@ -3523,11 +3523,11 @@ punpckhqdq m5, m7 punpckhqdq m7, m6, m8 punpcklqdq m6, m8 ret -cglobal inv_txfm_add_identity_identity_32x8, 4, 6, 10, dst, stride, c, eob +cglobal inv_txfm_add_identity_identity_32x8_8bpc, 4, 6, 10, dst, stride, c, eob add cq, 16*8 vpbroadcastd m9, [pw_4096] lea r4, [strideq*3] lea r5, [dstq+strideq*4] sub eobd, 107 @@ -3548,11 +3548,11 @@ mova xm7, [cq-16*1] vinserti128 m6, [cq+16*6], 1 vinserti128 m7, [cq+16*7], 1 pxor m8, m8 REPX {mova [cq+32*x], m8}, -4, -3, -2, -1, 0, 1, 2, 3 - call m(inv_txfm_add_identity_identity_8x32).transpose8x8 + call m(inv_txfm_add_identity_identity_8x32_8bpc).transpose8x8 REPX {pmulhrsw x, m9}, m0, m1, m2, m3, m4, m5, m6, m7 WRITE_16X2 0, 1, 8, 0, strideq*0, strideq*1 WRITE_16X2 2, 3, 0, 1, strideq*2, r4 %define dstq r5 WRITE_16X2 4, 5, 0, 1, strideq*0, strideq*1 @@ -3625,19 +3625,19 @@ vpermq m%3, m%3, q3120 mova [dstq+%6], xm%3 vextracti128 [r2+%7], m%3, 1 %endmacro -cglobal inv_txfm_add_dct_dct_16x32, 4, 4, 0, dst, stride, c, eob +cglobal inv_txfm_add_dct_dct_16x32_8bpc, 4, 4, 0, dst, stride, c, eob lea rax, [o_base] test eobd, eobd jz .dconly PROLOGUE 0, 8, 16, 32*35, dst, stride, c, eob, tmp1, tmp2, \ base, tmp3 %undef cmp LOAD_16ROWS cq, 64, 1 - call m(idct_16x16_internal).main + call m(idct_16x16_internal_8bpc).main lea tmp1q, [rsp+32*7] lea tmp2q, [tmp1q+32*8] lea tmp3q, [tmp1q+32*16] mova m1, [rsp+32*1] mova [rsp+32*0], m6 @@ -3680,22 +3680,22 @@ pmulhrsw xm0, xm1, [cq] movd xm2, [o(pw_16384)] mov [cq], eobd pmulhrsw xm0, xm1 mov r2d, 16 - jmp m(inv_txfm_add_dct_dct_16x4).dconly + jmp m(inv_txfm_add_dct_dct_16x4_8bpc).dconly .full: mova [tmp1q-32*4], m1 mova [tmp1q-32*3], m3 mova [tmp1q-32*2], m5 mova [tmp1q-32*1], m7 mova [tmp1q+32*0], m9 mova [tmp1q+32*1], m11 mova [tmp1q+32*2], m13 mova [tmp1q+32*3], m15 LOAD_16ROWS cq+32, 64, 1 - call m(idct_16x16_internal).main + call m(idct_16x16_internal_8bpc).main lea r2, [tmp3q+32*8] mova m1, [rsp+32*1] mova [rsp+32*0], m6 mova [rsp+32*1], m7 vpbroadcastd m7, [o(pw_16384)] @@ -3744,11 +3744,11 @@ call .main_oddhalf LOAD_8ROWS_H r2-32*4, 32 .idct16: LOAD_8ROWS tmp3q-32*4, 32 mova [rsp], m15 - call m(idct_16x16_internal).main + call m(idct_16x16_internal_8bpc).main imul r2, strideq, 19 lea r3, [strideq*3] add r2, dstq call .pass2_end RET @@ -3989,21 +3989,21 @@ vextracti128 [tmp2q+32*( 3-%1)], m%2, 1 vperm2i128 m%2, m%1, m%4, 0x31 vinserti128 m%1, xm%4, 1 %endmacro -cglobal inv_txfm_add_dct_dct_32x16, 4, 4, 0, dst, stride, c, eob +cglobal inv_txfm_add_dct_dct_32x16_8bpc, 4, 4, 0, dst, stride, c, eob lea rax, [o_base] test eobd, eobd jnz .normal movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] movd xm2, [o(pw_16384)] mov [cq], eobd pmulhrsw xm0, xm1 mov r2d, 16 - jmp m(inv_txfm_add_dct_dct_32x8).dconly + jmp m(inv_txfm_add_dct_dct_32x8_8bpc).dconly .normal: PROLOGUE 0, 6, 16, 32*19, dst, stride, c, eob, tmp1, tmp2 vpbroadcastd m15, [o(pw_2896x8)] pmulhrsw m0, m15, [cq+32* 1] pmulhrsw m1, m15, [cq+32* 3] @@ -4021,11 +4021,11 @@ pmulhrsw m13, m15, [cq+32*27] pmulhrsw m14, m15, [cq+32*29] pmulhrsw m15, [cq+32*31] lea tmp1q, [rsp+32*7] lea tmp2q, [tmp1q+32*8] - call m(inv_txfm_add_dct_dct_16x32).main_oddhalf + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf LOAD_16ROWS cq+32*0, 32*2, 1, 0 pxor m15, m15 mov r3d, 8 .zero_loop: mova [cq+32*0], m15 @@ -4033,18 +4033,18 @@ mova [cq+32*2], m15 mova [cq+32*3], m15 add cq, 32*4 dec r3d jg .zero_loop - call m(idct_16x16_internal).main + call m(idct_16x16_internal_8bpc).main call .pass1_end lea r2, [strideq*3] mov r3, dstq .pass2: vpbroadcastd m7, [o(pw_16384)] - call m(inv_txfm_add_dct_dct_16x32).transpose_2x8x8_round - call m(idct_16x16_internal).main + call m(inv_txfm_add_dct_dct_16x32_8bpc).transpose_2x8x8_round + call m(idct_16x16_internal_8bpc).main mova [rsp+32*2], m15 vpbroadcastd m15, [o(pw_2048)] REPX {pmulhrsw x, m15}, m2, m3, m0 WRITE_16X2 2, 3, 1, 2, strideq*2, r2 pmulhrsw m1, m15, [rsp+32*1] @@ -4088,11 +4088,11 @@ mova [rsp+gprsize+32*0], m6 mova [rsp+gprsize+32*1], m7 IDCT32_PASS1_END 1, 9, 6, 7 ret -cglobal inv_txfm_add_identity_identity_16x32, 4, 5, 13, dst, stride, c, eob +cglobal inv_txfm_add_identity_identity_16x32_8bpc, 4, 5, 13, dst, stride, c, eob %undef cmp lea rax, [o_base] vpbroadcastd m9, [o(pw_2896x8)] vpbroadcastd m10, [o(pw_1697x16)] vpbroadcastd m12, [o(pw_8192)] @@ -4122,11 +4122,11 @@ mova xm7, [cq+64* 7] vinserti128 m6, [cq+64*14], 1 vinserti128 m7, [cq+64*15], 1 REPX {pmulhrsw x, m9 }, m0, m1, m2, m3, m4, m5, m6, m7 REPX {IDTX16 x, 8, 10, 11}, 0, 1, 2, 3, 4, 5, 6, 7 - call m(inv_txfm_add_identity_identity_8x32).transpose8x8 + call m(inv_txfm_add_identity_identity_8x32_8bpc).transpose8x8 REPX {pmulhrsw x, m12}, m0, m1, m2, m3, m4, m5, m6, m7 WRITE_16X2 0, 1, 8, 0, strideq*0, strideq*1 WRITE_16X2 2, 3, 0, 1, strideq*2, r3 lea dstq, [dstq+strideq*4] WRITE_16X2 4, 5, 0, 1, strideq*0, strideq*1 @@ -4157,11 +4157,11 @@ add rax, 32*4 dec r0d jg .zero_loop RET -cglobal inv_txfm_add_identity_identity_32x16, 4, 6, 12, dst, stride, c, eob +cglobal inv_txfm_add_identity_identity_32x16_8bpc, 4, 6, 12, dst, stride, c, eob %undef cmp lea rax, [o_base] vpbroadcastd m9, [o(pw_2896x8)] vpbroadcastd m10, [o(pw_1697x16)] vpbroadcastd m11, [o(pw_2048)] @@ -4190,11 +4190,11 @@ mova xm7, [cq+32* 7] vinserti128 m6, [cq+32*14], 1 vinserti128 m7, [cq+32*15], 1 REPX {pmulhrsw x, m9 }, m0, m1, m2, m3, m4, m5, m6, m7 REPX {paddsw x, x }, m0, m1, m2, m3, m4, m5, m6, m7 - call m(inv_txfm_add_identity_identity_8x32).transpose8x8 + call m(inv_txfm_add_identity_identity_8x32_8bpc).transpose8x8 REPX {IDTX16 x, 8, 10}, 0, 1, 2, 3, 4, 5, 6, 7 REPX {pmulhrsw x, m11}, m0, m1, m2, m3, m4, m5, m6, m7 WRITE_16X2 0, 1, 8, 0, strideq*0, strideq*1 WRITE_16X2 2, 3, 0, 1, strideq*2, r3 lea dstq, [dstq+strideq*4] @@ -4221,20 +4221,20 @@ add rax, 32*4 sub cd, 128 jge .zero_loop RET -cglobal inv_txfm_add_dct_dct_32x32, 4, 4, 0, dst, stride, c, eob +cglobal inv_txfm_add_dct_dct_32x32_8bpc, 4, 4, 0, dst, stride, c, eob lea rax, [o_base] test eobd, eobd jnz .normal movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] movd xm2, [o(pw_8192)] mov [cq], eobd mov r2d, 32 - jmp m(inv_txfm_add_dct_dct_32x8).dconly + jmp m(inv_txfm_add_dct_dct_32x8_8bpc).dconly .normal: PROLOGUE 0, 9, 16, 32*67, dst, stride, c, eob, tmp1, tmp2, \ base, tmp3, tmp4 %undef cmp lea tmp1q, [rsp+32*7] @@ -4246,30 +4246,30 @@ pxor m8, m8 REPX {mova [cq+64*x], m8}, 1, 3, 5, 7, 9, 11, 13, 15 test tmp4d, tmp4d jl .fast LOAD_8ROWS_H cq+64*17, 64*2 - call m(inv_txfm_add_dct_dct_16x32).main_oddhalf + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf LOAD_8ROWS_H cq+64*16, 64*2 pxor m0, m0 REPX {mova [cq+64*x], m0}, 16, 17, 18, 19, 20, 21, 22, 23, \ 24, 25, 26, 27, 28, 29, 30, 31 mova [rsp], m15 jmp .idct16 .fast: - call m(inv_txfm_add_dct_dct_16x32).main_oddhalf_fast + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast pxor m8, m8 REPX {mova x, m8}, m9, m10, m11, m12, m13, m14 mova [rsp], m8 .idct16: LOAD_8ROWS cq+64*0, 64*2 pxor m15, m15 REPX {mova [cq+64*x], m15}, 0, 2, 4, 6, 8, 10, 12, 14 - call m(idct_16x16_internal).main - call m(inv_txfm_add_dct_dct_32x16).pass1_end + call m(idct_16x16_internal_8bpc).main + call m(inv_txfm_add_dct_dct_32x16_8bpc).pass1_end vpbroadcastd m7, [o(pw_8192)] - call m(inv_txfm_add_dct_dct_16x32).transpose_2x8x8_round + call m(inv_txfm_add_dct_dct_16x32_8bpc).transpose_2x8x8_round lea tmp3q, [tmp1q+32*32] mova m15, [rsp] mova [tmp3q-32*4], m0 mova [tmp3q-32*3], m2 mova [tmp3q-32*2], m4 @@ -4294,11 +4294,11 @@ pmulhrsw m3, m9, [tmp1q-32*1] pmulhrsw m4, m9, [tmp1q+32*0] pmulhrsw m5, m9, [tmp1q+32*1] pmulhrsw m6, m9, [tmp1q+32*2] pmulhrsw m7, m9, [tmp1q+32*3] - call m(inv_txfm_add_identity_identity_8x32).transpose8x8 + call m(inv_txfm_add_identity_identity_8x32_8bpc).transpose8x8 mova [tmp1q-32*4], m0 pmulhrsw m0, m9, [tmp2q-32*4] mova [tmp2q-32*4], m1 pmulhrsw m1, m9, [tmp2q-32*3] mova [tmp1q-32*3], m2 @@ -4311,11 +4311,11 @@ pmulhrsw m5, m9, [tmp2q+32*1] mova [tmp1q-32*1], m6 pmulhrsw m6, m9, [tmp2q+32*2] mova [tmp2q-32*1], m7 pmulhrsw m7, m9, [tmp2q+32*3] - call m(inv_txfm_add_identity_identity_8x32).transpose8x8 + call m(inv_txfm_add_identity_identity_8x32_8bpc).transpose8x8 mova [tmp1q+32*0], m0 mova [tmp2q+32*0], m1 mova [tmp1q+32*1], m2 mova [tmp2q+32*1], m3 mova [tmp1q+32*2], m4 @@ -4339,25 +4339,25 @@ .pass2_loop: LOAD_8ROWS tmp2q-32*4, 32 test tmp4d, tmp4d jl .fast2 LOAD_8ROWS_H tmp3q-32*4, 32 - call m(inv_txfm_add_dct_dct_16x32).main_oddhalf + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf sub tmp3q, 32*8 LOAD_8ROWS_H tmp3q-32*4, 32 sub tmp3q, 32*16 jmp .pass2_loop_end .fast2: - call m(inv_txfm_add_dct_dct_16x32).main_oddhalf_fast + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast sub tmp3q, 32*24 pxor m8, m8 REPX {mova x, m8}, m9, m10, m11, m12, m13, m14, m15 .pass2_loop_end: LOAD_8ROWS tmp3q-32*4, 32 mova [rsp], m15 - call m(idct_16x16_internal).main - call m(inv_txfm_add_dct_dct_16x32).pass2_end + call m(idct_16x16_internal_8bpc).main + call m(inv_txfm_add_dct_dct_16x32_8bpc).pass2_end lea tmp3q, [tmp1q-32*32] cmp tmp2q, tmp3q jb .ret sub tmp2q, 32*32 sub dstq, r3 @@ -4365,11 +4365,11 @@ add dstq, 16 jmp .pass2_loop .ret: RET -cglobal inv_txfm_add_identity_identity_32x32, 4, 6, 10, dst, stride, c, eob +cglobal inv_txfm_add_identity_identity_32x32_8bpc, 4, 6, 10, dst, stride, c, eob %undef cmp vpbroadcastd m9, [pw_8192] sub eobd, 136 ; if (eob < 136) shr eobd, 30 ; topleft 16x16 only lea eobd, [eobq*2-8] @@ -4391,11 +4391,11 @@ vinserti128 m5, [cq+64*13], 1 mova xm6, [cq+64* 6] mova xm7, [cq+64* 7] vinserti128 m6, [cq+64*14], 1 vinserti128 m7, [cq+64*15], 1 - call m(inv_txfm_add_identity_identity_8x32).transpose8x8 + call m(inv_txfm_add_identity_identity_8x32_8bpc).transpose8x8 REPX {pmulhrsw x, m9}, m0, m1, m2, m3, m4, m5, m6, m7 WRITE_16X2 0, 1, 8, 0, strideq*0, strideq*1 WRITE_16X2 2, 3, 0, 1, strideq*2, r4 lea dstq, [dstq+strideq*4] WRITE_16X2 4, 5, 0, 1, strideq*0, strideq*1 @@ -4485,35 +4485,35 @@ mova [%%d1+%10], xm%3 vextracti128 [%%d0+%7 ], m%3, 1 %endif %endmacro -cglobal inv_txfm_add_dct_dct_16x64, 4, 4, 0, dst, stride, c, eob +cglobal inv_txfm_add_dct_dct_16x64_8bpc, 4, 4, 0, dst, stride, c, eob lea rax, [o_base] test eobd, eobd jnz .normal movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] movd xm2, [o(pw_8192)] mov [cq], eobd mov r2d, 32 - jmp m(inv_txfm_add_dct_dct_16x4).dconly + jmp m(inv_txfm_add_dct_dct_16x4_8bpc).dconly .normal: PROLOGUE 0, 10, 16, 32*67, dst, stride, c, eob, tmp1, tmp2 %undef cmp lea tmp1q, [rsp+32*23] lea tmp2q, [tmp1q+32*24] sub eobd, 151 mov r7d, eobd .pass1_loop: LOAD_16ROWS cq, 64 - call m(idct_16x16_internal).main + call m(idct_16x16_internal_8bpc).main mova m1, [rsp+32*1] mova [rsp+32*0], m6 mova [rsp+32*1], m7 vpbroadcastd m7, [o(pw_8192)] - call m(inv_txfm_add_dct_dct_16x32).transpose_2x8x8_round + call m(inv_txfm_add_dct_dct_16x32_8bpc).transpose_2x8x8_round mova m15, [rsp+32*0] mova [tmp1q-32*4], m0 mova [tmp1q-32*3], m2 mova [tmp1q-32*2], m4 mova [tmp1q-32*1], m6 @@ -4557,11 +4557,11 @@ vinserti128 m6, [r3+32*0+16], 1 vinserti128 m7, [r3+32*2+16], 1 .fast: mova [rsp], m8 lea tmp1q, [rsp+32*7] - call m(idct_16x16_internal).main + call m(idct_16x16_internal_8bpc).main mova m1, [rsp+32*1] mova [tmp1q-32*4], m0 mova [tmp1q-32*3], m1 mova [tmp1q-32*2], m2 mova [tmp1q-32*1], m3 @@ -4599,11 +4599,11 @@ vinserti128 m6, [r3+32*1+16], 1 vinserti128 m7, [r3+32*3+16], 1 .fast2: add tmp1q, 32*8 lea tmp2q, [tmp1q+32*8] - call m(inv_txfm_add_dct_dct_16x32).main_oddhalf_fast + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast add r2, 32*24 vpbroadcastd m15, [o(pd_2048)] add tmp1q, 32*16 add tmp2q, 32*32 mova xm0, [r2-32*4+ 0] @@ -4627,11 +4627,11 @@ mova xm6, [r3-32*4+16] vinserti128 m5, [r3+32*3+ 0], 1 vinserti128 m6, [r3+32*0+16], 1 .fast3: add rax, o_idct64_offset - call m(inv_txfm_add_dct_dct_16x64).main_part1 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 add rax, 8 add tmp1q, 32*8 sub tmp2q, 32*8 mova xm0, [r2-32*2+ 0] mova xm3, [r2-32*3+16] @@ -4652,12 +4652,12 @@ mova xm5, [r3-32*3+ 0] mova xm6, [r3-32*2+16] vinserti128 m5, [r3+32*1+ 0], 1 vinserti128 m6, [r3+32*2+16], 1 .fast4: - call m(inv_txfm_add_dct_dct_16x64).main_part1 - call m(inv_txfm_add_dct_dct_16x64).main_part2_pass2 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part2_pass2 RET ALIGN function_align %define o_base idct64_mul - 8 cglobal_label .main_part1 ; idct64 steps 1-5: @@ -4831,11 +4831,11 @@ sub r2, strideq cmp tmp1q, tmp2q jne .main_part2_pass2_loop ret -cglobal inv_txfm_add_dct_dct_64x16, 4, 4, 0, dst, stride, c, eob +cglobal inv_txfm_add_dct_dct_64x16_8bpc, 4, 4, 0, dst, stride, c, eob lea rax, [o_base] test eobd, eobd jnz .normal movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] @@ -4874,11 +4874,11 @@ pxor m8, m8 REPX {mova [cq+32*x], m8}, 0, 4, 8, 12, 16, 20, 24, 28 REPX {mova x, m8}, m9, m10, m11, m12, m13, m14 mova [rsp], m8 lea tmp1q, [rsp+32*7] - call m(idct_16x16_internal).main + call m(idct_16x16_internal_8bpc).main mova m1, [rsp+32*1] mova [tmp1q-32*4], m0 mova [tmp1q-32*3], m1 mova [tmp1q-32*2], m2 mova [tmp1q-32*1], m3 @@ -4898,11 +4898,11 @@ LOAD_8ROWS cq+32*2, 32*4 pxor m8, m8 REPX {mova [cq+32*x], m8}, 2, 6, 10, 14, 18, 22, 26, 30 add tmp1q, 32*8 lea tmp2q, [tmp1q+32*8] - call m(inv_txfm_add_dct_dct_16x32).main_oddhalf_fast + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast vpbroadcastd m15, [o(pd_2048)] add tmp1q, 32*16 add tmp2q, 32*32 mova m0, [cq+32* 1] mova m1, [cq+32*31] @@ -4913,11 +4913,11 @@ mova m6, [cq+32*25] mova m7, [cq+32* 7] pxor m8, m8 REPX {mova [cq+32*x], m8}, 1, 31, 17, 15, 9, 23, 25, 7 add rax, o_idct64_offset - call m(inv_txfm_add_dct_dct_16x64).main_part1 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 add rax, 8 add tmp1q, 32*8 sub tmp2q, 32*8 mova m0, [cq+32* 5] mova m1, [cq+32*27] @@ -4927,12 +4927,12 @@ mova m5, [cq+32*19] mova m6, [cq+32*29] mova m7, [cq+32* 3] pxor m8, m8 REPX {mova [cq+32*x], m8}, 5, 27, 21, 11, 13, 19, 29, 3 - call m(inv_txfm_add_dct_dct_16x64).main_part1 - call m(inv_txfm_add_dct_dct_16x64).main_part2_pass1 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part2_pass1 sub tmp1q, 32*36 lea r2, [strideq*3] mov tmp2d, 4 .pass2_loop: lea r3, [tmp1q-32*8] @@ -4969,12 +4969,12 @@ vinserti128 m14, [tmp1q+32*2+16], 1 vinserti128 m15, [tmp1q+32*3+16], 1 mova [rsp+32*0], m6 mova [rsp+32*1], m7 vpbroadcastd m7, [o(pw_8192)] - call m(inv_txfm_add_dct_dct_16x32).transpose_2x8x8_round - call m(idct_16x16_internal).main + call m(inv_txfm_add_dct_dct_16x32_8bpc).transpose_2x8x8_round + call m(idct_16x16_internal_8bpc).main mova [rsp+32*0], m15 vpbroadcastd m15, [o(pw_2048)] REPX {pmulhrsw x, m15}, m0, m2, m3, m4, m5, m6, m7 WRITE_16X2 2, 3, 1, 2, strideq*2, r2 pmulhrsw m1, m15, [rsp+32*1] @@ -4995,21 +4995,21 @@ add r0, 16 dec tmp2d jg .pass2_loop RET -cglobal inv_txfm_add_dct_dct_32x64, 4, 4, 0, dst, stride, c, eob +cglobal inv_txfm_add_dct_dct_32x64_8bpc, 4, 4, 0, dst, stride, c, eob lea rax, [o_base] test eobd, eobd jnz .normal movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] movd xm2, [o(pw_16384)] mov [cq], eobd pmulhrsw xm0, xm1 mov r2d, 64 - jmp m(inv_txfm_add_dct_dct_32x8).dconly + jmp m(inv_txfm_add_dct_dct_32x8_8bpc).dconly .normal: PROLOGUE 0, 11, 16, 32*99, dst, stride, c, eob, tmp1, tmp2 lea tmp1q, [rsp+32*7] lea r10d, [eobq-136] sar r10d, 31 @@ -5019,30 +5019,30 @@ pxor m8, m8 REPX {mova [cq+64*x], m8}, 1, 3, 5, 7, 9, 11, 13, 15 test r10b, r10b jnz .fast LOAD_8ROWS_H cq+64*17, 64*2, 2 - call m(inv_txfm_add_dct_dct_16x32).main_oddhalf + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf LOAD_8ROWS_H cq+64*16, 64*2, 1 mova [rsp], m15 pxor m15, m15 REPX {mova [cq+64*x], m15}, 16, 17, 18, 19, 20, 21, 22, 23, \ 24, 25, 26, 27, 28, 29, 30, 31 jmp .idct16 .fast: - call m(inv_txfm_add_dct_dct_16x32).main_oddhalf_fast + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast pxor m8, m8 REPX {mova x, m8}, m9, m10, m11, m12, m13, m14 mova [rsp], m8 .idct16: LOAD_8ROWS cq+64*0, 64*2, 1 pxor m15, m15 REPX {mova [cq+64*x], m15}, 0, 2, 4, 6, 8, 10, 12, 14 - call m(idct_16x16_internal).main - call m(inv_txfm_add_dct_dct_32x16).pass1_end + call m(idct_16x16_internal_8bpc).main + call m(inv_txfm_add_dct_dct_32x16_8bpc).pass1_end vpbroadcastd m7, [o(pw_16384)] - call m(inv_txfm_add_dct_dct_16x32).transpose_2x8x8_round + call m(inv_txfm_add_dct_dct_16x32_8bpc).transpose_2x8x8_round lea r3, [tmp1q+32*48] mova m15, [rsp] mova [r3-32*4], m0 mova [r3-32*3], m2 mova [r3-32*2], m4 @@ -5067,11 +5067,11 @@ pmulhrsw m3, m9, [tmp1q-32*1] pmulhrsw m4, m9, [tmp1q+32*0] pmulhrsw m5, m9, [tmp1q+32*1] pmulhrsw m6, m9, [tmp1q+32*2] pmulhrsw m7, m9, [tmp1q+32*3] - call m(inv_txfm_add_identity_identity_8x32).transpose8x8 + call m(inv_txfm_add_identity_identity_8x32_8bpc).transpose8x8 mova [tmp1q-32*4], m0 pmulhrsw m0, m9, [tmp2q-32*4] mova [tmp2q-32*4], m1 pmulhrsw m1, m9, [tmp2q-32*3] mova [tmp1q-32*3], m2 @@ -5084,11 +5084,11 @@ pmulhrsw m5, m9, [tmp2q+32*1] mova [tmp1q-32*1], m6 pmulhrsw m6, m9, [tmp2q+32*2] mova [tmp2q-32*1], m7 pmulhrsw m7, m9, [tmp2q+32*3] - call m(inv_txfm_add_identity_identity_8x32).transpose8x8 + call m(inv_txfm_add_identity_identity_8x32_8bpc).transpose8x8 mova [tmp1q+32*0], m0 mova [tmp2q+32*0], m1 mova [tmp1q+32*1], m2 mova [tmp2q+32*1], m3 mova [tmp1q+32*2], m4 @@ -5117,11 +5117,11 @@ mova m6, [r3+32*0] mova m7, [r3+32*2] .fast2: mova [rsp], m8 lea tmp1q, [rsp+32*39] - call m(idct_16x16_internal).main + call m(idct_16x16_internal_8bpc).main mova m1, [rsp+32*1] mova [tmp1q-32*4], m0 mova [tmp1q-32*3], m1 mova [tmp1q-32*2], m2 mova [tmp1q-32*1], m3 @@ -5151,11 +5151,11 @@ mova m6, [r3+32*1] mova m7, [r3+32*3] .fast3: add tmp1q, 32*8 lea tmp2q, [tmp1q+32*8] - call m(inv_txfm_add_dct_dct_16x32).main_oddhalf_fast + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast vpbroadcastd m15, [o(pd_2048)] add tmp1q, 32*16 add tmp2q, 32*32 mova m0, [r7-32*4] mova m3, [r7+32*3] @@ -5169,11 +5169,11 @@ mova m2, [r8-32*4] mova m5, [r8-32*1] mova m6, [r8+32*0] .fast4: add rax, o_idct64_offset - call m(inv_txfm_add_dct_dct_16x64).main_part1 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 add rax, 8 add tmp1q, 32*8 sub tmp2q, 32*8 mova m0, [r7-32*2] mova m3, [r7+32*1] @@ -5186,33 +5186,33 @@ mova m1, [r8+32*1] mova m2, [r8-32*2] mova m5, [r8-32*3] mova m6, [r8+32*2] .fast5: - call m(inv_txfm_add_dct_dct_16x64).main_part1 - call m(inv_txfm_add_dct_dct_16x64).main_part2_pass2 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part2_pass2 add r10d, 0x80000000 jc .ret lea r2, [rsp+32*7] lea r7, [r2+32*16] sub dstq, r8 lea dstq, [dstq+strideq*4+16] jmp .pass2_loop .ret: RET -cglobal inv_txfm_add_dct_dct_64x32, 4, 4, 0, dst, stride, c, eob +cglobal inv_txfm_add_dct_dct_64x32_8bpc, 4, 4, 0, dst, stride, c, eob lea rax, [o_base] test eobd, eobd jnz .normal movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] movd xm2, [o(pw_16384)] mov [cq], eobd pmulhrsw xm0, xm1 mov r2d, 32 - jmp m(inv_txfm_add_dct_dct_64x16).dconly + jmp m(inv_txfm_add_dct_dct_64x16_8bpc).dconly .normal: PROLOGUE 0, 9, 16, 32*131, dst, stride, c, eob, tmp1, tmp2, \ base, tmp3, tmp4 lea tmp1q, [rsp+32*7] lea tmp4d, [eobq-136] @@ -5220,11 +5220,11 @@ LOAD_8ROWS cq+64*0, 64*4, 1 pxor m8, m8 REPX {mova [cq+64*x], m8}, 0, 4, 8, 12, 16, 20, 24, 28 REPX {mova x, m8}, m9, m10, m11, m12, m13, m14 mova [rsp], m8 - call m(idct_16x16_internal).main + call m(idct_16x16_internal_8bpc).main mova m1, [rsp+32*1] mova [tmp1q-32*4], m0 mova [tmp1q-32*3], m1 mova [tmp1q-32*2], m2 mova [tmp1q-32*1], m3 @@ -5244,11 +5244,11 @@ LOAD_8ROWS cq+64*2, 64*4, 1 pxor m8, m8 REPX {mova [cq+64*x], m8}, 2, 6, 10, 14, 18, 22, 26, 30 add tmp1q, 32*8 lea tmp2q, [tmp1q+32*8] - call m(inv_txfm_add_dct_dct_16x32).main_oddhalf_fast + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast vpbroadcastd m15, [o(pd_2048)] add tmp1q, 32*16 add tmp2q, 32*32 vpbroadcastd m7, [o(pw_2896x8)] pmulhrsw m0, m7, [cq+64* 1] @@ -5260,11 +5260,11 @@ pmulhrsw m6, m7, [cq+64*25] pmulhrsw m7, [cq+64* 7] pxor m8, m8 REPX {mova [cq+64*x], m8}, 1, 31, 17, 15, 9, 23, 25, 7 add rax, o_idct64_offset - call m(inv_txfm_add_dct_dct_16x64).main_part1 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 vpbroadcastd m7, [o(pw_2896x8-(o_idct64_offset))] add rax, 8 add tmp1q, 32*8 sub tmp2q, 32*8 pmulhrsw m0, m7, [cq+64* 5] @@ -5275,15 +5275,15 @@ pmulhrsw m5, m7, [cq+64*19] pmulhrsw m6, m7, [cq+64*29] pmulhrsw m7, [cq+64* 3] pxor m8, m8 REPX {mova [cq+64*x], m8}, 5, 27, 21, 11, 13, 19, 29, 3 - call m(inv_txfm_add_dct_dct_16x64).main_part1 - call m(inv_txfm_add_dct_dct_16x64).main_part2_pass1 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part2_pass1 sub tmp1q, 32*44 vpbroadcastd m10, [o(pw_16384)] - call m(inv_txfm_add_dct_dct_64x32).transpose_round_interleave + call m(inv_txfm_add_dct_dct_64x32_8bpc).transpose_round_interleave add cq, 32 add tmp4d, 0x80000000 jnc .pass1_loop lea tmp1q, [rsp+32*15] imul r2, strideq, 19 @@ -5294,25 +5294,25 @@ lea tmp2q, [tmp1q+32*64] LOAD_8ROWS tmp1q-32*4, 32 test tmp4d, 0x40000000 jnz .fast LOAD_8ROWS_H tmp2q-32*4, 32 - call m(inv_txfm_add_dct_dct_16x32).main_oddhalf + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf lea tmp3q, [tmp2q-32*8] LOAD_8ROWS_H tmp3q-32*4, 32 mova [rsp], m15 jmp .idct16 .fast: - call m(inv_txfm_add_dct_dct_16x32).main_oddhalf_fast + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast pxor m8, m8 REPX {mova x, m8}, m9, m10, m11, m12, m13, m14 mova [rsp], m8 .idct16: lea tmp3q, [tmp1q-32*8] LOAD_8ROWS tmp3q-32*4, 32 - call m(idct_16x16_internal).main - call m(inv_txfm_add_dct_dct_16x32).pass2_end + call m(idct_16x16_internal_8bpc).main + call m(inv_txfm_add_dct_dct_16x32_8bpc).pass2_end add tmp1q, 32*16 sub dstq, r3 lea r2, [r2+r3+16] add dstq, 16 dec tmp4b @@ -5338,11 +5338,11 @@ mova xm6, [tmp1q+32*2] mova xm7, [tmp1q+32*3] vinserti128 m6, [tmp2q+32*2], 1 vinserti128 m7, [tmp2q+32*3], 1 REPX {pmulhrsw x, m10}, m0, m1, m2, m3, m4, m5, m6, m7 - call m(inv_txfm_add_identity_identity_8x32).transpose8x8 + call m(inv_txfm_add_identity_identity_8x32_8bpc).transpose8x8 mova xm8, [tmp1q-32*4+16] mova xm9, [tmp1q-32*3+16] vinserti128 m8, [tmp2q-32*4+16], 1 vinserti128 m9, [tmp2q-32*3+16], 1 mova [tmp1q-32*4], m0 @@ -5366,11 +5366,11 @@ vinserti128 m6, [tmp2q+32*2+16], 1 vinserti128 m7, [tmp2q+32*3+16], 1 pmulhrsw m0, m8, m10 pmulhrsw m1, m9, m10 REPX {pmulhrsw x, m10}, m2, m3, m4, m5, m6, m7 - call m(inv_txfm_add_identity_identity_8x32).transpose8x8 + call m(inv_txfm_add_identity_identity_8x32_8bpc).transpose8x8 mova [tmp1q+32*0], m0 mova [tmp2q+32*0], m1 mova [tmp1q+32*1], m2 mova [tmp2q+32*1], m3 mova [tmp1q+32*2], m4 @@ -5380,31 +5380,31 @@ add tmp1q, 32*16 dec tmp3d jg .loop ret -cglobal inv_txfm_add_dct_dct_64x64, 4, 4, 0, dst, stride, c, eob +cglobal inv_txfm_add_dct_dct_64x64_8bpc, 4, 4, 0, dst, stride, c, eob lea rax, [o_base] test eobd, eobd jnz .normal movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] movd xm2, [o(pw_8192)] mov [cq], eobd mov r2d, 64 - jmp m(inv_txfm_add_dct_dct_64x16).dconly + jmp m(inv_txfm_add_dct_dct_64x16_8bpc).dconly .normal: PROLOGUE 0, 11, 16, 32*199, dst, stride, c, eob, tmp1, tmp2 lea tmp1q, [rsp+32*71] lea r10d, [eobq-136] .pass1_loop: LOAD_8ROWS cq+64*0, 64*4 pxor m8, m8 REPX {mova [cq+64*x], m8}, 0, 4, 8, 12, 16, 20, 24, 28 REPX {mova x, m8}, m9, m10, m11, m12, m13, m14 mova [rsp], m8 - call m(idct_16x16_internal).main + call m(idct_16x16_internal_8bpc).main mova m1, [rsp+32*1] mova [tmp1q-32*4], m0 mova [tmp1q-32*3], m1 mova [tmp1q-32*2], m2 mova [tmp1q-32*1], m3 @@ -5424,11 +5424,11 @@ LOAD_8ROWS cq+64*2, 64*4 pxor m8, m8 REPX {mova [cq+64*x], m8}, 2, 6, 10, 14, 18, 22, 26, 30 add tmp1q, 32*8 lea tmp2q, [tmp1q+32*8] - call m(inv_txfm_add_dct_dct_16x32).main_oddhalf_fast + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast vpbroadcastd m15, [o(pd_2048)] add tmp1q, 32*16 add tmp2q, 32*32 mova m0, [cq+64* 1] mova m1, [cq+64*31] @@ -5439,11 +5439,11 @@ mova m6, [cq+64*25] mova m7, [cq+64* 7] pxor m8, m8 REPX {mova [cq+64*x], m8}, 1, 31, 17, 15, 9, 23, 25, 7 add rax, o_idct64_offset - call m(inv_txfm_add_dct_dct_16x64).main_part1 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 add rax, 8 add tmp1q, 32*8 sub tmp2q, 32*8 mova m0, [cq+64* 5] mova m1, [cq+64*27] @@ -5453,15 +5453,15 @@ mova m5, [cq+64*19] mova m6, [cq+64*29] mova m7, [cq+64* 3] pxor m8, m8 REPX {mova [cq+64*x], m8}, 5, 27, 21, 11, 13, 19, 29, 3 - call m(inv_txfm_add_dct_dct_16x64).main_part1 - call m(inv_txfm_add_dct_dct_16x64).main_part2_pass1 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part2_pass1 sub tmp1q, 32*44 vpbroadcastd m10, [o(pw_8192)] - call m(inv_txfm_add_dct_dct_64x32).transpose_round_interleave + call m(inv_txfm_add_dct_dct_64x32_8bpc).transpose_round_interleave add cq, 32 add r10d, 0x80000000 jnc .pass1_loop lea tmp1q, [rsp+32*7] mov r10b, 4 @@ -5480,11 +5480,11 @@ mova m4, [r3-32*4] mova m5, [r3-32*2] mova m6, [r3+32*0] mova m7, [r3+32*2] .fast: - call m(idct_16x16_internal).main + call m(idct_16x16_internal_8bpc).main mova m1, [rsp+32*1] mova [tmp1q-32*4], m0 mova [tmp1q-32*3], m1 mova [tmp1q-32*2], m2 mova [tmp1q-32*1], m3 @@ -5514,11 +5514,11 @@ mova m6, [r3+32*1] mova m7, [r3+32*3] .fast2: add tmp1q, 32*8 lea tmp2q, [tmp1q+32*8] - call m(inv_txfm_add_dct_dct_16x32).main_oddhalf_fast + call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast vpbroadcastd m15, [o(pd_2048)] add r2, 32*8 add r3, 32*8 add tmp1q, 32*16 add tmp2q, 32*32 @@ -5534,11 +5534,11 @@ mova m2, [r3-32*4] ; 17 mova m5, [r3-32*1] ; 23 mova m6, [r3+32*0] ; 25 .fast3: add rax, o_idct64_offset - call m(inv_txfm_add_dct_dct_16x64).main_part1 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 add rax, 8 add tmp1q, 32*8 sub tmp2q, 32*8 mova m0, [r2-32*2] ; 5 mova m3, [r2+32*1] ; 11 @@ -5551,12 +5551,12 @@ mova m1, [r3+32*1] ; 27 mova m2, [r3-32*2] ; 21 mova m5, [r3-32*3] ; 19 mova m6, [r3+32*2] ; 29 .fast4: - call m(inv_txfm_add_dct_dct_16x64).main_part1 - call m(inv_txfm_add_dct_dct_16x64).main_part2_pass2 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part1 + call m(inv_txfm_add_dct_dct_16x64_8bpc).main_part2_pass2 sub tmp1q, 32*28 sub dstq, r8 lea dstq, [dstq+strideq*4+16] dec r10b jg .pass2_loop diff --git a/third_party/dav1d/src/x86/itx_init_tmpl.c b/third_party/dav1d/src/x86/itx_init_tmpl.c --- a/third_party/dav1d/src/x86/itx_init_tmpl.c +++ b/third_party/dav1d/src/x86/itx_init_tmpl.c @@ -1,7 +1,7 @@ /* - * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018-2021, VideoLAN and dav1d authors * Copyright © 2018, Two Orioles, LLC * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -27,90 +27,69 @@ #include "src/cpu.h" #include "src/itx.h" #define decl_itx2_fns(w, h, opt) \ -decl_itx_fn(dav1d_inv_txfm_add_dct_dct_##w##x##h##_##opt); \ -decl_itx_fn(dav1d_inv_txfm_add_identity_identity_##w##x##h##_##opt) +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_identity_identity_##w##x##h, opt)) #define decl_itx12_fns(w, h, opt) \ decl_itx2_fns(w, h, opt); \ -decl_itx_fn(dav1d_inv_txfm_add_dct_adst_##w##x##h##_##opt); \ -decl_itx_fn(dav1d_inv_txfm_add_dct_flipadst_##w##x##h##_##opt); \ -decl_itx_fn(dav1d_inv_txfm_add_dct_identity_##w##x##h##_##opt); \ -decl_itx_fn(dav1d_inv_txfm_add_adst_dct_##w##x##h##_##opt); \ -decl_itx_fn(dav1d_inv_txfm_add_adst_adst_##w##x##h##_##opt); \ -decl_itx_fn(dav1d_inv_txfm_add_adst_flipadst_##w##x##h##_##opt); \ -decl_itx_fn(dav1d_inv_txfm_add_flipadst_dct_##w##x##h##_##opt); \ -decl_itx_fn(dav1d_inv_txfm_add_flipadst_adst_##w##x##h##_##opt); \ -decl_itx_fn(dav1d_inv_txfm_add_flipadst_flipadst_##w##x##h##_##opt); \ -decl_itx_fn(dav1d_inv_txfm_add_identity_dct_##w##x##h##_##opt) +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_adst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_flipadst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_identity_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_adst_dct_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_adst_adst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_adst_flipadst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_dct_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_adst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_flipadst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_identity_dct_##w##x##h, opt)) #define decl_itx16_fns(w, h, opt) \ decl_itx12_fns(w, h, opt); \ -decl_itx_fn(dav1d_inv_txfm_add_adst_identity_##w##x##h##_##opt); \ -decl_itx_fn(dav1d_inv_txfm_add_flipadst_identity_##w##x##h##_##opt); \ -decl_itx_fn(dav1d_inv_txfm_add_identity_adst_##w##x##h##_##opt); \ -decl_itx_fn(dav1d_inv_txfm_add_identity_flipadst_##w##x##h##_##opt) +decl_itx_fn(BF(dav1d_inv_txfm_add_adst_identity_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_identity_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_identity_adst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_identity_flipadst_##w##x##h, opt)) #define decl_itx17_fns(w, h, opt) \ decl_itx16_fns(w, h, opt); \ -decl_itx_fn(dav1d_inv_txfm_add_wht_wht_##w##x##h##_##opt) - -#define avx2_fns(avx2) \ -decl_itx17_fns( 4, 4, avx2); \ -decl_itx16_fns( 4, 8, avx2); \ -decl_itx16_fns( 4, 16, avx2); \ -decl_itx16_fns( 8, 4, avx2); \ -decl_itx16_fns( 8, 8, avx2); \ -decl_itx16_fns( 8, 16, avx2); \ -decl_itx2_fns ( 8, 32, avx2); \ -decl_itx16_fns(16, 4, avx2); \ -decl_itx16_fns(16, 8, avx2); \ -decl_itx12_fns(16, 16, avx2); \ -decl_itx2_fns (16, 32, avx2); \ -decl_itx2_fns (32, 8, avx2); \ -decl_itx2_fns (32, 16, avx2); \ -decl_itx2_fns (32, 32, avx2); \ -decl_itx_fn(dav1d_inv_txfm_add_dct_dct_16x64_##avx2); \ -decl_itx_fn(dav1d_inv_txfm_add_dct_dct_32x64_##avx2); \ -decl_itx_fn(dav1d_inv_txfm_add_dct_dct_64x16_##avx2); \ -decl_itx_fn(dav1d_inv_txfm_add_dct_dct_64x32_##avx2); \ -decl_itx_fn(dav1d_inv_txfm_add_dct_dct_64x64_##avx2) +decl_itx_fn(BF(dav1d_inv_txfm_add_wht_wht_##w##x##h, opt)) -avx2_fns(avx2); -avx2_fns(16bpc_avx2); +#define decl_itx_fns(ext) \ +decl_itx17_fns( 4, 4, ext); \ +decl_itx16_fns( 4, 8, ext); \ +decl_itx16_fns( 4, 16, ext); \ +decl_itx16_fns( 8, 4, ext); \ +decl_itx16_fns( 8, 8, ext); \ +decl_itx16_fns( 8, 16, ext); \ +decl_itx2_fns ( 8, 32, ext); \ +decl_itx16_fns(16, 4, ext); \ +decl_itx16_fns(16, 8, ext); \ +decl_itx12_fns(16, 16, ext); \ +decl_itx2_fns (16, 32, ext); \ +decl_itx2_fns (32, 8, ext); \ +decl_itx2_fns (32, 16, ext); \ +decl_itx2_fns (32, 32, ext); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_16x64, ext)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_32x64, ext)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_64x16, ext)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_64x32, ext)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_64x64, ext)) -decl_itx17_fns( 4, 4, ssse3); -decl_itx16_fns( 4, 8, ssse3); -decl_itx16_fns( 8, 4, ssse3); -decl_itx16_fns( 8, 8, ssse3); -decl_itx16_fns( 4, 16, ssse3); -decl_itx16_fns(16, 4, ssse3); -decl_itx16_fns( 8, 16, ssse3); -decl_itx16_fns(16, 8, ssse3); -decl_itx12_fns(16, 16, ssse3); -decl_itx2_fns ( 8, 32, ssse3); -decl_itx2_fns (32, 8, ssse3); -decl_itx2_fns (16, 32, ssse3); -decl_itx2_fns (32, 16, ssse3); -decl_itx2_fns (32, 32, ssse3); - -decl_itx_fn(dav1d_inv_txfm_add_dct_dct_16x64_ssse3); -decl_itx_fn(dav1d_inv_txfm_add_dct_dct_32x64_ssse3); -decl_itx_fn(dav1d_inv_txfm_add_dct_dct_64x16_ssse3); -decl_itx_fn(dav1d_inv_txfm_add_dct_dct_64x32_ssse3); -decl_itx_fn(dav1d_inv_txfm_add_dct_dct_64x64_ssse3); - +decl_itx_fns(avx2); +decl_itx_fns(sse4); +decl_itx_fns(ssse3); decl_itx_fn(dav1d_inv_txfm_add_wht_wht_4x4_16bpc_sse2); COLD void bitfn(dav1d_itx_dsp_init_x86)(Dav1dInvTxfmDSPContext *const c, const int bpc) { #define assign_itx_fn(pfx, w, h, type, type_enum, ext) \ c->itxfm_add[pfx##TX_##w##X##h][type_enum] = \ - dav1d_inv_txfm_add_##type##_##w##x##h##_##ext + BF(dav1d_inv_txfm_add_##type##_##w##x##h, ext) #define assign_itx1_fn(pfx, w, h, ext) \ assign_itx_fn(pfx, w, h, dct_dct, DCT_DCT, ext) #define assign_itx2_fn(pfx, w, h, ext) \ @@ -144,11 +123,11 @@ const unsigned flags = dav1d_get_cpu_flags(); if (!(flags & DAV1D_X86_CPU_FLAG_SSE2)) return; #if BITDEPTH == 16 - assign_itx_fn(, 4, 4, wht_wht, WHT_WHT, 16bpc_sse2); + assign_itx_fn(, 4, 4, wht_wht, WHT_WHT, sse2); #endif if (!(flags & DAV1D_X86_CPU_FLAG_SSSE3)) return; #if BITDEPTH == 8 @@ -171,40 +150,61 @@ assign_itx1_fn (R, 64, 16, ssse3); assign_itx1_fn (R, 64, 32, ssse3); assign_itx1_fn ( , 64, 64, ssse3); #endif + if (!(flags & DAV1D_X86_CPU_FLAG_SSE41)) return; + +#if BITDEPTH == 16 + if (bpc <= 10) { + assign_itx16_fn(, 4, 4, sse4); + assign_itx16_fn(R, 4, 8, sse4); + assign_itx16_fn(R, 4, 16, sse4); + assign_itx16_fn(R, 8, 4, sse4); + assign_itx16_fn(, 8, 8, sse4); + assign_itx16_fn(R, 8, 16, sse4); + assign_itx16_fn(R, 16, 4, sse4); + assign_itx16_fn(R, 16, 8, sse4); + assign_itx12_fn(, 16, 16, sse4); + assign_itx2_fn (R, 8, 32, sse4); + assign_itx2_fn (R, 32, 8, sse4); + assign_itx2_fn (R, 16, 32, sse4); + assign_itx2_fn (R, 32, 16, sse4); + assign_itx2_fn (, 32, 32, sse4); + assign_itx1_fn (R, 16, 64, sse4); + assign_itx1_fn (R, 32, 64, sse4); + assign_itx1_fn (R, 64, 16, sse4); + assign_itx1_fn (R, 64, 32, sse4); + assign_itx1_fn (, 64, 64, sse4); + } +#endif + if (!(flags & DAV1D_X86_CPU_FLAG_AVX2)) return; -#if ARCH_X86_64 && BITDEPTH == 16 - assign_itx_fn(, 4, 4, wht_wht, WHT_WHT, 16bpc_avx2); +#if ARCH_X86_64 + assign_itx_fn(, 4, 4, wht_wht, WHT_WHT, avx2); #endif if (bpc > 10) return; #if ARCH_X86_64 -#if BITDEPTH == 8 -#define SUFFIX avx2 -#else -#define SUFFIX 16bpc_avx2 -#endif - assign_itx17_fn( , 4, 4, SUFFIX); - assign_itx16_fn(R, 4, 8, SUFFIX); - assign_itx16_fn(R, 4, 16, SUFFIX); - assign_itx16_fn(R, 8, 4, SUFFIX); - assign_itx16_fn( , 8, 8, SUFFIX); - assign_itx16_fn(R, 8, 16, SUFFIX); - assign_itx2_fn (R, 8, 32, SUFFIX); - assign_itx16_fn(R, 16, 4, SUFFIX); - assign_itx16_fn(R, 16, 8, SUFFIX); - assign_itx12_fn( , 16, 16, SUFFIX); - assign_itx2_fn (R, 16, 32, SUFFIX); - assign_itx1_fn (R, 16, 64, SUFFIX); - assign_itx2_fn (R, 32, 8, SUFFIX); - assign_itx2_fn (R, 32, 16, SUFFIX); - assign_itx2_fn ( , 32, 32, SUFFIX); - assign_itx1_fn (R, 32, 64, SUFFIX); - assign_itx1_fn (R, 64, 16, SUFFIX); - assign_itx1_fn (R, 64, 32, SUFFIX); - assign_itx1_fn ( , 64, 64, SUFFIX); + assign_itx17_fn( , 4, 4, avx2); + assign_itx16_fn(R, 4, 8, avx2); + assign_itx16_fn(R, 4, 16, avx2); + assign_itx16_fn(R, 8, 4, avx2); + assign_itx16_fn( , 8, 8, avx2); + assign_itx16_fn(R, 8, 16, avx2); + assign_itx2_fn (R, 8, 32, avx2); + assign_itx16_fn(R, 16, 4, avx2); + assign_itx16_fn(R, 16, 8, avx2); + assign_itx12_fn( , 16, 16, avx2); + assign_itx2_fn (R, 16, 32, avx2); + assign_itx1_fn (R, 16, 64, avx2); + assign_itx2_fn (R, 32, 8, avx2); + assign_itx2_fn (R, 32, 16, avx2); + assign_itx2_fn ( , 32, 32, avx2); + assign_itx1_fn (R, 32, 64, avx2); + assign_itx1_fn (R, 64, 16, avx2); + assign_itx1_fn (R, 64, 32, avx2); + assign_itx1_fn ( , 64, 64, avx2); #endif } diff --git a/third_party/dav1d/src/x86/itx_sse.asm b/third_party/dav1d/src/x86/itx_sse.asm --- a/third_party/dav1d/src/x86/itx_sse.asm +++ b/third_party/dav1d/src/x86/itx_sse.asm @@ -1,6 +1,6 @@ -; Copyright © 2018, VideoLAN and dav1d authors +; Copyright © 2018-2021, VideoLAN and dav1d authors ; Copyright © 2018, Two Orioles, LLC ; All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions are met: @@ -238,26 +238,26 @@ psubsw m1, m0, m2 ;high: out2 ;low: out3 paddsw m0, m2 ;high: out1 ;low: out0 %endmacro %macro INV_TXFM_FN 4+ ; type1, type2, size, xmm/stack -cglobal inv_txfm_add_%1_%2_%3, 4, 6, %4, dst, stride, coeff, eob, tx2 - %define %%p1 m(i%1_%3_internal) +cglobal inv_txfm_add_%1_%2_%3_8bpc, 4, 6, %4, dst, stride, coeff, eob, tx2 + %define %%p1 m(i%1_%3_internal_8bpc) %if ARCH_X86_32 LEA r5, $$ %endif %if has_epilogue %ifidn %1_%2, dct_dct test eobd, eobd jz %%end %endif - lea tx2q, [o(m(i%2_%3_internal).pass2)] + lea tx2q, [o(m(i%2_%3_internal_8bpc).pass2)] call %%p1 RET %%end: %else - lea tx2q, [o(m(i%2_%3_internal).pass2)] + lea tx2q, [o(m(i%2_%3_internal_8bpc).pass2)] %ifidn %1_%2, dct_dct test eobd, eobd jnz %%p1 %else times ((%%end - %%p1) >> 31) & 1 jmp %%p1 @@ -275,22 +275,23 @@ mova m1, [o(pw_2896x8)] pmulhrsw m0, m1 mov [coeffq], eobd ;0 pmulhrsw m0, m1 mova m1, m0 - TAIL_CALL m(iadst_4x4_internal).end2 + TAIL_CALL m(iadst_4x4_internal_8bpc).end2 %endif %endmacro INIT_XMM ssse3 +; itx16 relies on dct_dct being the first function. If you change the order, adjust `itx8_start` in itx16. INV_TXFM_4X4_FN dct, dct INV_TXFM_4X4_FN dct, adst INV_TXFM_4X4_FN dct, flipadst INV_TXFM_4X4_FN dct, identity -cglobal idct_4x4_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal idct_4x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m0, [coeffq+16*0] ;high: in1 ;low: in0 mova m1, [coeffq+16*1] ;high: in3 ;low in2 IDCT4_1D_PACKED @@ -313,11 +314,11 @@ INV_TXFM_4X4_FN adst, dct INV_TXFM_4X4_FN adst, adst INV_TXFM_4X4_FN adst, flipadst INV_TXFM_4X4_FN adst, identity -cglobal iadst_4x4_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal iadst_4x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m0, [coeffq+16*0] mova m1, [coeffq+16*1] call .main punpckhwd m2, m0, m1 punpcklwd m0, m1 @@ -335,11 +336,11 @@ .end2: ITX4_END 0, 1, 2, 3 ALIGN function_align -.main: +cglobal_label .main punpcklwd m2, m0, m1 ;unpacked in0 in2 punpckhwd m0, m1 ;unpacked in1 in3 mova m3, m0 pmaddwd m1, m2, [o(pw_3344_m3344)];3344 * in0 - 3344 * in2 pmaddwd m0, [o(pw_0_3344)] ;3344 * in3 @@ -365,22 +366,22 @@ INV_TXFM_4X4_FN flipadst, dct INV_TXFM_4X4_FN flipadst, adst INV_TXFM_4X4_FN flipadst, flipadst INV_TXFM_4X4_FN flipadst, identity -cglobal iflipadst_4x4_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal iflipadst_4x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m0, [coeffq+16*0] mova m1, [coeffq+16*1] - call m(iadst_4x4_internal).main + call m(iadst_4x4_internal_8bpc).main punpcklwd m2, m1, m0 punpckhwd m1, m0 punpcklwd m0, m1, m2 ;high: in3 ;low :in2 punpckhwd m1, m2 ;high: in1 ;low: in0 jmp tx2q .pass2: - call m(iadst_4x4_internal).main + call m(iadst_4x4_internal_8bpc).main .end: pxor m2, m2 mova [coeffq+16*0], m2 mova [coeffq+16*1], m2 @@ -391,11 +392,11 @@ INV_TXFM_4X4_FN identity, dct INV_TXFM_4X4_FN identity, adst INV_TXFM_4X4_FN identity, flipadst INV_TXFM_4X4_FN identity, identity -cglobal iidentity_4x4_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal iidentity_4x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m0, [coeffq+16*0] mova m1, [coeffq+16*1] mova m3, [o(pw_1697x8)] pmulhrsw m2, m0, m3 pmulhrsw m3, m1 @@ -411,11 +412,11 @@ mova m3, [o(pw_1697x8)] pmulhrsw m2, m3, m0 pmulhrsw m3, m1 paddsw m0, m2 paddsw m1, m3 - jmp m(iadst_4x4_internal).end + jmp m(iadst_4x4_internal_8bpc).end %macro IWHT4_1D_PACKED 0 punpckhqdq m3, m0, m1 ;low: in1 high: in3 punpcklqdq m0, m1 ;low: in0 high: in2 psubw m2, m0, m3 ;low: in0 - in1 high: in2 - in3 @@ -427,11 +428,11 @@ psubw m1, m3 ;low: t1/out2 high: t3/out1 psubw m0, m1 ;high: out0 paddw m2, m1 ;low: out3 %endmacro -cglobal inv_txfm_add_wht_wht_4x4, 3, 3, 4, dst, stride, coeff +cglobal inv_txfm_add_wht_wht_4x4_8bpc, 3, 3, 4, dst, stride, coeff mova m0, [coeffq+16*0] mova m1, [coeffq+16*1] pxor m2, m2 mova [coeffq+16*0], m2 mova [coeffq+16*1], m2 @@ -559,57 +560,57 @@ pmulhrsw m0, m1 pmulhrsw m0, [o(pw_2048)] mova m1, m0 mova m2, m0 mova m3, m0 - TAIL_CALL m(iadst_4x8_internal).end3 + TAIL_CALL m(iadst_4x8_internal_8bpc).end3 %endif %endmacro INV_TXFM_4X8_FN dct, dct INV_TXFM_4X8_FN dct, adst INV_TXFM_4X8_FN dct, flipadst INV_TXFM_4X8_FN dct, identity -cglobal idct_4x8_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal idct_4x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m3, [o(pw_2896x8)] pmulhrsw m0, m3, [coeffq+16*0] pmulhrsw m1, m3, [coeffq+16*1] pmulhrsw m2, m3, [coeffq+16*2] pmulhrsw m3, [coeffq+16*3] .pass1: - call m(idct_8x4_internal).main - jmp m(iadst_4x8_internal).pass1_end + call m(idct_8x4_internal_8bpc).main + jmp m(iadst_4x8_internal_8bpc).pass1_end .pass2: call .main shufps m1, m1, q1032 shufps m3, m3, q1032 mova m4, [o(pw_2048)] - jmp m(iadst_4x8_internal).end2 + jmp m(iadst_4x8_internal_8bpc).end2 ALIGN function_align -.main: +cglobal_label .main IDCT8_1D_PACKED ret INV_TXFM_4X8_FN adst, dct INV_TXFM_4X8_FN adst, adst INV_TXFM_4X8_FN adst, flipadst INV_TXFM_4X8_FN adst, identity -cglobal iadst_4x8_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal iadst_4x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m3, [o(pw_2896x8)] pmulhrsw m0, m3, [coeffq+16*0] pmulhrsw m1, m3, [coeffq+16*1] pmulhrsw m2, m3, [coeffq+16*2] pmulhrsw m3, [coeffq+16*3] .pass1: - call m(iadst_8x4_internal).main + call m(iadst_8x4_internal_8bpc).main .pass1_end: INV_4X8 jmp tx2q @@ -638,11 +639,11 @@ .end3: WRITE_4X8 0, 1, 2, 3 RET ALIGN function_align -.main: +cglobal_label .main mova m6, [o(pd_2048)] punpckhwd m4, m3, m0 ;unpacked in7 in0 punpckhwd m5, m2, m1 ;unpacked in5 in2 punpcklwd m1, m2 ;unpacked in3 in4 punpcklwd m0, m3 ;unpacked in1 in6 @@ -688,19 +689,19 @@ INV_TXFM_4X8_FN flipadst, dct INV_TXFM_4X8_FN flipadst, adst INV_TXFM_4X8_FN flipadst, flipadst INV_TXFM_4X8_FN flipadst, identity -cglobal iflipadst_4x8_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal iflipadst_4x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m3, [o(pw_2896x8)] pmulhrsw m0, m3, [coeffq+16*0] pmulhrsw m1, m3, [coeffq+16*1] pmulhrsw m2, m3, [coeffq+16*2] pmulhrsw m3, [coeffq+16*3] .pass1: - call m(iadst_8x4_internal).main + call m(iadst_8x4_internal_8bpc).main punpcklwd m4, m3, m2 punpckhwd m3, m2 punpcklwd m5, m1, m0 punpckhwd m1, m0 @@ -711,29 +712,29 @@ jmp tx2q .pass2: shufps m0, m0, q1032 shufps m1, m1, q1032 - call m(iadst_4x8_internal).main + call m(iadst_4x8_internal_8bpc).main mova m4, m0 mova m5, m1 pshufd m0, m3, q1032 pshufd m1, m2, q1032 pshufd m2, m5, q1032 pshufd m3, m4, q1032 mova m5, [o(pw_2048)] pxor m4, m4 psubw m4, m5 - jmp m(iadst_4x8_internal).end + jmp m(iadst_4x8_internal_8bpc).end INV_TXFM_4X8_FN identity, dct INV_TXFM_4X8_FN identity, adst INV_TXFM_4X8_FN identity, flipadst INV_TXFM_4X8_FN identity, identity -cglobal iidentity_4x8_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal iidentity_4x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m3, [o(pw_2896x8)] pmulhrsw m0, m3, [coeffq+16*0] pmulhrsw m1, m3, [coeffq+16*1] pmulhrsw m2, m3, [coeffq+16*2] pmulhrsw m3, [coeffq+16*3] @@ -746,15 +747,15 @@ pmulhrsw m7, m3 paddsw m0, m4 paddsw m1, m5 paddsw m2, m6 paddsw m3, m7 - jmp m(iadst_4x8_internal).pass1_end + jmp m(iadst_4x8_internal_8bpc).pass1_end .pass2: mova m4, [o(pw_4096)] - jmp m(iadst_4x8_internal).end2 + jmp m(iadst_4x8_internal_8bpc).end2 %macro WRITE_8X2 5 ;coefs[1-2], tmp[1-3] movq m%3, [dstq ] movq m%4, [dstq+strideq] @@ -795,27 +796,27 @@ pmulhrsw m0, m1 pmulhrsw m0, m2 mova m1, m0 mova m2, m0 mova m3, m0 - TAIL_CALL m(iadst_8x4_internal).end2 + TAIL_CALL m(iadst_8x4_internal_8bpc).end2 %endif %endmacro INV_TXFM_8X4_FN dct, dct INV_TXFM_8X4_FN dct, adst INV_TXFM_8X4_FN dct, flipadst INV_TXFM_8X4_FN dct, identity -cglobal idct_8x4_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal idct_8x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m3, [o(pw_2896x8)] pmulhrsw m0, m3, [coeffq+16*0] pmulhrsw m1, m3, [coeffq+16*1] pmulhrsw m2, m3, [coeffq+16*2] pmulhrsw m3, [coeffq+16*3] - call m(idct_4x8_internal).main + call m(idct_4x8_internal_8bpc).main mova m4, [o(deint_shuf1)] mova m5, [o(deint_shuf2)] pshufb m0, m4 pshufb m1, m5 @@ -831,33 +832,33 @@ punpcklqdq m2 ,m4, m5 ;in2 jmp tx2q .pass2: call .main - jmp m(iadst_8x4_internal).end + jmp m(iadst_8x4_internal_8bpc).end ALIGN function_align -.main: +cglobal_label .main mova m6, [o(pd_2048)] IDCT4_1D 0, 1, 2, 3, 4, 5, 6 ret INV_TXFM_8X4_FN adst, dct INV_TXFM_8X4_FN adst, adst INV_TXFM_8X4_FN adst, flipadst INV_TXFM_8X4_FN adst, identity -cglobal iadst_8x4_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal iadst_8x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m3, [o(pw_2896x8)] pmulhrsw m0, m3, [coeffq+16*0] pmulhrsw m1, m3, [coeffq+16*1] pmulhrsw m2, m3, [coeffq+16*2] pmulhrsw m3, [coeffq+16*3] shufps m0, m0, q1032 shufps m1, m1, q1032 - call m(iadst_4x8_internal).main + call m(iadst_4x8_internal_8bpc).main punpckhwd m4, m0, m1 punpcklwd m0, m1 punpckhwd m1, m2, m3 punpcklwd m2, m3 @@ -893,11 +894,11 @@ .end3: WRITE_8X4 0, 1, 2, 3, 4, 5, 6 RET ALIGN function_align -.main: +cglobal_label .main punpckhwd m6, m0, m2 ;unpacked in0 in2 punpcklwd m0, m2 ;unpacked in0 in2 punpckhwd m7, m1, m3 ;unpacked in1 in3 punpcklwd m1, m3 ;unpacked in1 in3 @@ -962,20 +963,20 @@ INV_TXFM_8X4_FN flipadst, dct INV_TXFM_8X4_FN flipadst, adst INV_TXFM_8X4_FN flipadst, flipadst INV_TXFM_8X4_FN flipadst, identity -cglobal iflipadst_8x4_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal iflipadst_8x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m3, [o(pw_2896x8)] pmulhrsw m0, m3, [coeffq+16*0] pmulhrsw m1, m3, [coeffq+16*1] pmulhrsw m2, m3, [coeffq+16*2] pmulhrsw m3, [coeffq+16*3] shufps m0, m0, q1032 shufps m1, m1, q1032 - call m(iadst_4x8_internal).main + call m(iadst_4x8_internal_8bpc).main punpckhwd m5, m3, m2 punpcklwd m3, m2 punpckhwd m2, m1, m0 punpcklwd m1, m0 @@ -992,25 +993,25 @@ punpckhwd m3, m2, m4 ;in3 punpcklwd m2, m4 ;in2 jmp tx2q .pass2: - call m(iadst_8x4_internal).main + call m(iadst_8x4_internal_8bpc).main mova m4, m0 mova m5, m1 mova m0, m3 mova m1, m2 mova m2, m5 mova m3, m4 - jmp m(iadst_8x4_internal).end + jmp m(iadst_8x4_internal_8bpc).end INV_TXFM_8X4_FN identity, dct INV_TXFM_8X4_FN identity, adst INV_TXFM_8X4_FN identity, flipadst INV_TXFM_8X4_FN identity, identity -cglobal iidentity_8x4_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal iidentity_8x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m3, [o(pw_2896x8)] pmulhrsw m0, m3, [coeffq+16*0] pmulhrsw m1, m3, [coeffq+16*1] pmulhrsw m2, m3, [coeffq+16*2] pmulhrsw m3, [coeffq+16*3] @@ -1041,11 +1042,11 @@ pmulhrsw m7, m3 paddsw m0, m4 paddsw m1, m5 paddsw m2, m6 paddsw m3, m7 - jmp m(iadst_8x4_internal).end + jmp m(iadst_8x4_internal_8bpc).end %macro INV_TXFM_8X8_FN 2 ; type1, type2 INV_TXFM_FN %1, %2, 8x8, 8, 16*4 %ifidn %1_%2, dct_dct pshuflw m0, [coeffq], q0000 @@ -1058,11 +1059,11 @@ psrlw m2, 3 pmulhrsw m0, m1 pmulhrsw m0, m2 .end: mov r3d, 2 - lea tx2q, [o(m(inv_txfm_add_dct_dct_8x8).end3)] + lea tx2q, [o(m(inv_txfm_add_dct_dct_8x8_8bpc).end3)] .loop: WRITE_8X4 0, 0, 0, 0, 1, 2, 3 lea dstq, [dstq+strideq*2] dec r3d jg .loop @@ -1108,11 +1109,11 @@ INV_TXFM_8X8_FN dct, dct INV_TXFM_8X8_FN dct, adst INV_TXFM_8X8_FN dct, flipadst INV_TXFM_8X8_FN dct, identity -cglobal idct_8x8_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal idct_8x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 LOAD_8ROWS coeffq, 16 .pass1: call .main @@ -1125,11 +1126,11 @@ .pass1_end2: REPX {pmulhrsw x, m7}, m1, m3, m5 pmulhrsw m7, [rsp+gprsize+16*0] -.pass1_end3: +cglobal_label .pass1_end3 punpcklwd m6, m1, m5 ;10 50 11 51 12 52 13 53 punpckhwd m1, m5 ;14 54 15 55 16 56 17 57 punpckhwd m5, m0, m4 ;04 44 05 45 06 46 07 47 punpcklwd m0, m4 ;00 40 01 41 02 42 03 43 punpckhwd m4, m3, m7 ;34 74 35 75 36 76 37 77 @@ -1159,11 +1160,11 @@ punpcklwd m0, m7 ;00 10 20 30 40 50 60 70 mova m7, [rsp+gprsize+16*0] jmp tx2q .pass2: - lea tx2q, [o(m(idct_8x8_internal).end4)] + lea tx2q, [o(m(idct_8x8_internal_8bpc).end4)] .pass2_main: call .main .end: @@ -1187,11 +1188,11 @@ pxor m7, m7 REPX {mova [coeffq+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7 ret ALIGN function_align -.main: +cglobal_label .main mova [rsp+gprsize*2+16*0], m7 mova [rsp+gprsize*2+16*1], m3 mova [rsp+gprsize*2+16*2], m1 mova m7, [o(pd_2048)] IDCT4_1D 0, 2, 4, 6, 1, 3, 7 @@ -1220,11 +1221,11 @@ INV_TXFM_8X8_FN adst, dct INV_TXFM_8X8_FN adst, adst INV_TXFM_8X8_FN adst, flipadst INV_TXFM_8X8_FN adst, identity -cglobal iadst_8x8_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal iadst_8x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 LOAD_8ROWS coeffq, 16 .pass1: call .main call .main_pass1_end @@ -1236,15 +1237,15 @@ REPX {pmulhrsw x, m7}, m0, m2, m4, m6 mova [rsp+gprsize+16*1], m6 pxor m6, m6 psubw m6, m7 mova m7, m6 - jmp m(idct_8x8_internal).pass1_end2 + jmp m(idct_8x8_internal_8bpc).pass1_end2 ALIGN function_align .pass2: - lea tx2q, [o(m(idct_8x8_internal).end4)] + lea tx2q, [o(m(idct_8x8_internal_8bpc).end4)] .pass2_main: call .main call .main_pass2_end @@ -1253,14 +1254,14 @@ REPX {pmulhrsw x, m7}, m0, m2, m4, m6 mova [rsp+gprsize+16*1], m6 pxor m6, m6 psubw m6, m7 mova m7, m6 - jmp m(idct_8x8_internal).end2 + jmp m(idct_8x8_internal_8bpc).end2 ALIGN function_align -.main: +cglobal_label .main mova [rsp+gprsize*2+16*0], m7 mova [rsp+gprsize*2+16*1], m3 mova [rsp+gprsize*2+16*2], m4 mova m7, [o(pd_2048)] ITX_MULSUB_2W 5, 2, 3, 4, 7, 1931, 3612 ;t3a, t2a @@ -1341,11 +1342,11 @@ packssdw m4, m1 ;-out5 mova m1, [rsp+gprsize*2+16*1] mova m6, [rsp+gprsize*2+16*2] ret ALIGN function_align -.main_pass2_end: +cglobal_label .main_pass2_end paddsw m7, m4, m3 ;t2 + t3 psubsw m4, m3 ;t2 - t3 paddsw m3, m5, m2 ;t6 + t7 psubsw m5, m2 ;t6 - t7 mova m2, [o(pw_2896x8)] @@ -1359,16 +1360,16 @@ INV_TXFM_8X8_FN flipadst, dct INV_TXFM_8X8_FN flipadst, adst INV_TXFM_8X8_FN flipadst, flipadst INV_TXFM_8X8_FN flipadst, identity -cglobal iflipadst_8x8_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal iflipadst_8x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 LOAD_8ROWS coeffq, 16 .pass1: - call m(iadst_8x8_internal).main - call m(iadst_8x8_internal).main_pass1_end + call m(iadst_8x8_internal_8bpc).main + call m(iadst_8x8_internal_8bpc).main_pass1_end .pass1_end: mova m7, [o(pw_m16384)] .pass1_end1: @@ -1386,19 +1387,19 @@ pxor m7, m7 psubw m7, m0 pmulhrsw m0, [rsp+gprsize+16*0] REPX {pmulhrsw x, m7}, m1, m3, m5 pmulhrsw m7, m6 - jmp m(idct_8x8_internal).pass1_end3 + jmp m(idct_8x8_internal_8bpc).pass1_end3 ALIGN function_align .pass2: - lea tx2q, [o(m(idct_8x8_internal).end4)] + lea tx2q, [o(m(idct_8x8_internal_8bpc).end4)] .pass2_main: - call m(iadst_8x8_internal).main - call m(iadst_8x8_internal).main_pass2_end + call m(iadst_8x8_internal_8bpc).main + call m(iadst_8x8_internal_8bpc).main_pass2_end .end: mova m7, [o(pw_2048)] REPX {pmulhrsw x, m7}, m0, m2, m4, m6 mova [rsp+gprsize+16*2], m2 @@ -1413,34 +1414,34 @@ mova m1, m6 pmulhrsw m4, m3, m0 pmulhrsw m0, [rsp+gprsize+16*0] mova m3, m5 mova [rsp+gprsize+16*0], m7 - jmp m(idct_8x8_internal).end3 + jmp m(idct_8x8_internal_8bpc).end3 INV_TXFM_8X8_FN identity, dct INV_TXFM_8X8_FN identity, adst INV_TXFM_8X8_FN identity, flipadst INV_TXFM_8X8_FN identity, identity -cglobal iidentity_8x8_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal iidentity_8x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 LOAD_8ROWS coeffq, 16 mova [rsp+gprsize+16*1], m6 - jmp m(idct_8x8_internal).pass1_end3 + jmp m(idct_8x8_internal_8bpc).pass1_end3 ALIGN function_align .pass2: - lea tx2q, [o(m(idct_8x8_internal).end4)] + lea tx2q, [o(m(idct_8x8_internal_8bpc).end4)] .end: pmulhrsw m7, [o(pw_4096)] mova [rsp+gprsize+16*0], m7 mova m7, [o(pw_4096)] REPX {pmulhrsw x, m7}, m0, m1, m2, m3, m4, m5, m6 mova [rsp+gprsize+16*2], m5 mova [rsp+gprsize+16*1], m6 - jmp m(idct_8x8_internal).end3 + jmp m(idct_8x8_internal_8bpc).end3 %macro INV_TXFM_4X16_FN 2 ; type1, type2 INV_TXFM_FN %1, %2, 4x16, 8 %ifidn %1_%2, dct_dct @@ -1467,20 +1468,20 @@ INV_TXFM_4X16_FN dct, dct INV_TXFM_4X16_FN dct, adst INV_TXFM_4X16_FN dct, flipadst INV_TXFM_4X16_FN dct, identity -cglobal idct_4x16_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 - lea r3, [o(m(idct_4x8_internal).pass1)] +cglobal idct_4x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + lea r3, [o(m(idct_4x8_internal_8bpc).pass1)] .pass1: mova m0, [coeffq+16*1] mova m1, [coeffq+16*3] mova m2, [coeffq+16*5] mova m3, [coeffq+16*7] push tx2q - lea tx2q, [o(m(idct_4x16_internal).pass1_2)] + lea tx2q, [o(m(idct_4x16_internal_8bpc).pass1_2)] jmp r3 .pass1_2: mova [coeffq+16*1], m0 mova [coeffq+16*3], m1 @@ -1488,11 +1489,11 @@ mova [coeffq+16*7], m3 mova m0, [coeffq+16*0] mova m1, [coeffq+16*2] mova m2, [coeffq+16*4] mova m3, [coeffq+16*6] - lea tx2q, [o(m(idct_4x16_internal).pass1_end)] + lea tx2q, [o(m(idct_4x16_internal_8bpc).pass1_end)] jmp r3 .pass1_end: pop tx2q @@ -1505,11 +1506,11 @@ pmulhrsw m7, [coeffq+16*7] mova [coeffq+16*7], m7 jmp tx2q .pass2: - call m(idct_16x4_internal).main + call m(idct_16x4_internal_8bpc).main .end: mova m7, [o(pw_2048)] REPX {pmulhrsw x, m7}, m0, m1, m2, m3, m4, m5, m6 pmulhrsw m7, [coeffq+16*7] @@ -1536,17 +1537,17 @@ INV_TXFM_4X16_FN adst, dct INV_TXFM_4X16_FN adst, adst INV_TXFM_4X16_FN adst, flipadst INV_TXFM_4X16_FN adst, identity -cglobal iadst_4x16_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 - lea r3, [o(m(iadst_4x8_internal).pass1)] - jmp m(idct_4x16_internal).pass1 +cglobal iadst_4x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + lea r3, [o(m(iadst_4x8_internal_8bpc).pass1)] + jmp m(idct_4x16_internal_8bpc).pass1 .pass2: - call m(iadst_16x4_internal).main - call m(iadst_16x4_internal).main_pass2_end + call m(iadst_16x4_internal_8bpc).main + call m(iadst_16x4_internal_8bpc).main_pass2_end punpcklqdq m6, m5, m4 ;low: -out5 high: -out7 punpckhqdq m4, m5 ;low: out8 high: out10 punpcklqdq m5, m7, m2 ;low: out4 high: out6 punpckhqdq m2, m7 ;low: -out9 high: -out11 @@ -1604,17 +1605,17 @@ INV_TXFM_4X16_FN flipadst, dct INV_TXFM_4X16_FN flipadst, adst INV_TXFM_4X16_FN flipadst, flipadst INV_TXFM_4X16_FN flipadst, identity -cglobal iflipadst_4x16_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 - lea r3, [o(m(iflipadst_4x8_internal).pass1)] - jmp m(idct_4x16_internal).pass1 +cglobal iflipadst_4x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + lea r3, [o(m(iflipadst_4x8_internal_8bpc).pass1)] + jmp m(idct_4x16_internal_8bpc).pass1 .pass2: - call m(iadst_16x4_internal).main - call m(iadst_16x4_internal).main_pass2_end + call m(iadst_16x4_internal_8bpc).main + call m(iadst_16x4_internal_8bpc).main_pass2_end punpckhqdq m6, m5, m4 ;low: out5 high: out7 punpcklqdq m4, m5 ;low: -out8 high: -out10 punpckhqdq m5, m7, m2 ;low: -out4 high: -out6 punpcklqdq m2, m7 ;low: out9 high: out11 @@ -1626,11 +1627,11 @@ punpckhqdq m0, m6 ;low: -out0 high: -out2 punpcklqdq m6, m3, m2 ;low: -out12 high: -out14 punpckhqdq m2, m3 ;low: out1 high: out3 mova m7, [o(pw_m2048)] - jmp m(iadst_4x16_internal).end1 + jmp m(iadst_4x16_internal_8bpc).end1 INV_TXFM_4X16_FN identity, dct INV_TXFM_4X16_FN identity, adst INV_TXFM_4X16_FN identity, flipadst @@ -1644,11 +1645,11 @@ paddsw m%1, m%1 %endif paddsw m%1, m%2 %endmacro -cglobal iidentity_4x16_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal iidentity_4x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m0, [coeffq+16*1] mova m6, [o(pw_1697x8)] mova m1, [coeffq+16*3] mova m2, [coeffq+16*5] mova m3, [coeffq+16*7] @@ -1670,11 +1671,11 @@ pcmpeqw m2, m7 pavgw m5, m3 pcmpeqw m3, m7 pandn m2, m4 pandn m3, m5 - jmp m(iadst_4x8_internal).pass1_end + jmp m(iadst_4x8_internal_8bpc).pass1_end .pass1_2: mova [coeffq+16*1], m0 mova [coeffq+16*3], m1 mova [coeffq+16*5], m2 mova [coeffq+16*7], m3 @@ -1702,22 +1703,22 @@ paddsw m6, m7 mova m7, [o(pw_2048)] REPX {pmulhrsw x, m7}, m0, m1, m2, m3, m4, m5, m6 pmulhrsw m7, [coeffq+16*7] mova [coeffq+16*4], m4 - jmp m(iadst_4x16_internal).end2 + jmp m(iadst_4x16_internal_8bpc).end2 %macro INV_TXFM_16X4_FN 2 ; type1, type2 INV_TXFM_FN %1, %2, 16x4, 8 %ifidn %1_%2, dct_dct movd m1, [o(pw_2896x8)] pmulhrsw m0, m1, [coeffq] movd m2, [o(pw_16384)] mov [coeffq], eobd mov r2d, 2 - lea tx2q, [o(m(inv_txfm_add_dct_dct_16x4).end)] + lea tx2q, [o(m(inv_txfm_add_dct_dct_16x4_8bpc).end)] .dconly: pmulhrsw m0, m2 movd m2, [o(pw_2048)] ;intentionally rip-relative pmulhrsw m0, m1 pmulhrsw m0, m2 @@ -1804,11 +1805,11 @@ INV_TXFM_16X4_FN dct, dct INV_TXFM_16X4_FN dct, adst INV_TXFM_16X4_FN dct, flipadst INV_TXFM_16X4_FN dct, identity -cglobal idct_16x4_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal idct_16x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 LOAD_7ROWS coeffq, 16 call .main .pass1_end: punpckhwd m7, m0, m2 ;packed out1, out5 @@ -1848,11 +1849,11 @@ punpckhwd m3, m2, m7 ;0, 1, 2, 3, 4, 5, 6, 7(3) punpcklwd m2, m7 ;0, 1, 2, 3, 4, 5, 6, 7(2) jmp tx2q .pass2: - lea tx2q, [o(m(idct_8x4_internal).pass2)] + lea tx2q, [o(m(idct_8x4_internal_8bpc).pass2)] .pass2_end: mova [coeffq+16*4], m4 mova [coeffq+16*5], m5 mova [coeffq+16*6], m6 @@ -1866,11 +1867,11 @@ mova m3, [coeffq+16*3] mov dstq, r3 jmp tx2q ALIGN function_align -.main: +cglobal_label .main punpckhqdq m7, m0, m1 ;low:in1 high:in3 punpcklqdq m0, m1 punpcklqdq m1, m2, m3 punpckhqdq m3, m2 ;low:in7 high:in5 mova [coeffq+16*4], m7 @@ -1909,11 +1910,11 @@ INV_TXFM_16X4_FN adst, dct INV_TXFM_16X4_FN adst, adst INV_TXFM_16X4_FN adst, flipadst INV_TXFM_16X4_FN adst, identity -cglobal iadst_16x4_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal iadst_16x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 LOAD_7ROWS coeffq, 16 call .main call .main_pass1_end punpckhwd m6, m7, m0 ;packed -out11, -out15 @@ -1937,18 +1938,18 @@ psubw m2, m7 mova m7, [coeffq+16*6] REPX {pmulhrsw x, m2}, m7, m3, m6 pmulhrsw m2, [coeffq+16*7] mova [coeffq+16*6], m7 - jmp m(idct_16x4_internal).pass1_end3 + jmp m(idct_16x4_internal_8bpc).pass1_end3 .pass2: - lea tx2q, [o(m(iadst_8x4_internal).pass2)] - jmp m(idct_16x4_internal).pass2_end + lea tx2q, [o(m(iadst_8x4_internal_8bpc).pass2)] + jmp m(idct_16x4_internal_8bpc).pass2_end ALIGN function_align -.main: +cglobal_label .main mova [coeffq+16*6], m0 pshufd m0, m1, q1032 pshufd m2, m2, q1032 punpckhwd m1, m6, m0 ;packed in13, in2 punpcklwd m0, m6 ;packed in3, in12 @@ -2067,11 +2068,11 @@ packssdw m5, m1 ;low:-out5 high:out10 mova m0, [coeffq+16*4] mova m3, [coeffq+16*5] ret ALIGN function_align -.main_pass2_end: +cglobal_label .main_pass2_end mova m7, [o(pw_2896x8)] punpckhqdq m6, m2, m1 ;low:t11 high:t15a punpcklqdq m2, m1 ;low:t10 high:t14a psubsw m1, m2, m6 paddsw m2, m6 @@ -2093,14 +2094,14 @@ INV_TXFM_16X4_FN flipadst, dct INV_TXFM_16X4_FN flipadst, adst INV_TXFM_16X4_FN flipadst, flipadst INV_TXFM_16X4_FN flipadst, identity -cglobal iflipadst_16x4_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal iflipadst_16x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 LOAD_7ROWS coeffq, 16 - call m(iadst_16x4_internal).main - call m(iadst_16x4_internal).main_pass1_end + call m(iadst_16x4_internal_8bpc).main + call m(iadst_16x4_internal_8bpc).main_pass1_end punpcklwd m6, m7, m0 ;packed out11, out15 punpckhwd m0, m7 ;packed -out0, -out4 punpckhwd m7, m3, m4 ;packed out3, out7 punpcklwd m4, m3 ;packed -out8, -out12 @@ -2112,23 +2113,23 @@ mova [coeffq+16*7], m7 punpcklwd m3, m2, m1 ;packed out9, out13 punpckhwd m1, m2 ;packed -out2, -out6 mova m7, [o(pw_m16384)] - jmp m(iadst_16x4_internal).pass1_end + jmp m(iadst_16x4_internal_8bpc).pass1_end .pass2: - lea tx2q, [o(m(iflipadst_8x4_internal).pass2)] - jmp m(idct_16x4_internal).pass2_end + lea tx2q, [o(m(iflipadst_8x4_internal_8bpc).pass2)] + jmp m(idct_16x4_internal_8bpc).pass2_end INV_TXFM_16X4_FN identity, dct INV_TXFM_16X4_FN identity, adst INV_TXFM_16X4_FN identity, flipadst INV_TXFM_16X4_FN identity, identity -cglobal iidentity_16x4_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal iidentity_16x4_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m1, [coeffq+16*6] mova m0, [coeffq+16*5] mova m2, [coeffq+16*7] mova m6, [o(pw_1697x16)] mova m7, [o(pw_16384)] @@ -2174,15 +2175,15 @@ mova m7, [coeffq+16*7] punpckhwd m3, m4, m6 ;packed out9, out13 punpcklwd m4, m6 ;packed out8, out12 punpckhwd m6, m5, m7 ;packed out11, out15 punpcklwd m5, m7 ;packed out10, out14 - jmp m(idct_16x4_internal).pass1_end3 + jmp m(idct_16x4_internal_8bpc).pass1_end3 .pass2: - lea tx2q, [o(m(iidentity_8x4_internal).pass2)] - jmp m(idct_16x4_internal).pass2_end + lea tx2q, [o(m(iidentity_8x4_internal_8bpc).pass2)] + jmp m(idct_16x4_internal_8bpc).pass2_end %macro SAVE_8ROWS 2 ;src, stride mova [%1+%2*0], m0 mova [%1+%2*1], m1 @@ -2207,39 +2208,39 @@ pmulhrsw m0, m2 psrlw m2, 3 ; pw_2048 pmulhrsw m0, m1 pmulhrsw m0, m2 mov r3d, 4 - lea tx2q, [o(m(inv_txfm_add_dct_dct_8x16).end)] - jmp m(inv_txfm_add_dct_dct_8x8).loop + lea tx2q, [o(m(inv_txfm_add_dct_dct_8x16_8bpc).end)] + jmp m(inv_txfm_add_dct_dct_8x8_8bpc).loop .end: RET %endif %endmacro INV_TXFM_8X16_FN dct, dct INV_TXFM_8X16_FN dct, adst INV_TXFM_8X16_FN dct, flipadst INV_TXFM_8X16_FN dct, identity -cglobal idct_8x16_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 - lea r3, [o(m(idct_8x8_internal).pass1)] +cglobal idct_8x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + lea r3, [o(m(idct_8x8_internal_8bpc).pass1)] .pass1: LOAD_8ROWS coeffq+16*1, 32, 1 mov [rsp+gprsize+16*11], tx2q - lea tx2q, [o(m(idct_8x16_internal).pass1_end)] + lea tx2q, [o(m(idct_8x16_internal_8bpc).pass1_end)] jmp r3 .pass1_end: SAVE_8ROWS coeffq+16*1, 32 LOAD_8ROWS coeffq+16*0, 32, 1 mov tx2q, [rsp+gprsize+16*11] jmp r3 .pass2: - lea tx2q, [o(m(idct_8x16_internal).end)] + lea tx2q, [o(m(idct_8x16_internal_8bpc).end)] .pass2_pre: mova [coeffq+16*2 ], m1 mova [coeffq+16*6 ], m3 mova [coeffq+16*10], m5 @@ -2251,33 +2252,33 @@ mova m5, [coeffq+16*5 ] mova m6, [coeffq+16*9 ] mova m7, [coeffq+16*13] .pass2_main: - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main SAVE_7ROWS rsp+gprsize+16*3, 16 mova m0, [coeffq+16*2 ] mova m1, [coeffq+16*6 ] mova m2, [coeffq+16*10] mova m3, [coeffq+16*14] mova m4, [coeffq+16*3 ] mova m5, [coeffq+16*7 ] mova m6, [coeffq+16*11] mova m7, [coeffq+16*15] - call m(idct_16x8_internal).main + call m(idct_16x8_internal_8bpc).main mov r3, dstq lea dstq, [dstq+strideq*8] - jmp m(idct_8x8_internal).end + jmp m(idct_8x8_internal_8bpc).end .end: LOAD_8ROWS rsp+gprsize+16*3, 16 mova [rsp+gprsize+16*0], m7 - lea tx2q, [o(m(idct_8x16_internal).end1)] + lea tx2q, [o(m(idct_8x16_internal_8bpc).end1)] mov dstq, r3 - jmp m(idct_8x8_internal).end + jmp m(idct_8x8_internal_8bpc).end .end1: pxor m7, m7 REPX {mova [coeffq+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 ret @@ -2285,16 +2286,16 @@ INV_TXFM_8X16_FN adst, dct INV_TXFM_8X16_FN adst, adst INV_TXFM_8X16_FN adst, flipadst INV_TXFM_8X16_FN adst, identity -cglobal iadst_8x16_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 - lea r3, [o(m(iadst_8x8_internal).pass1)] - jmp m(idct_8x16_internal).pass1 +cglobal iadst_8x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + lea r3, [o(m(iadst_8x8_internal_8bpc).pass1)] + jmp m(idct_8x16_internal_8bpc).pass1 .pass2: - lea tx2q, [o(m(iadst_8x16_internal).end)] + lea tx2q, [o(m(iadst_8x16_internal_8bpc).end)] .pass2_pre: mova [rsp+gprsize+16*7], m0 mova [rsp+gprsize+16*8], m1 mova [rsp+gprsize+16*5], m6 @@ -2316,36 +2317,36 @@ mova m4, [coeffq+16*5 ] mova m5, [coeffq+16*7 ] mova m6, [coeffq+16*9 ] mova m7, [coeffq+16*11] - call m(iadst_16x8_internal).main - call m(iadst_16x8_internal).main_pass2_end + call m(iadst_16x8_internal_8bpc).main + call m(iadst_16x8_internal_8bpc).main_pass2_end mov r3, dstq lea dstq, [dstq+strideq*8] - jmp m(iadst_8x8_internal).end + jmp m(iadst_8x8_internal_8bpc).end .end: LOAD_8ROWS rsp+gprsize+16*3, 16 mova [rsp+gprsize+16*0], m7 - lea tx2q, [o(m(idct_8x16_internal).end1)] + lea tx2q, [o(m(idct_8x16_internal_8bpc).end1)] mov dstq, r3 - jmp m(iadst_8x8_internal).end + jmp m(iadst_8x8_internal_8bpc).end INV_TXFM_8X16_FN flipadst, dct INV_TXFM_8X16_FN flipadst, adst INV_TXFM_8X16_FN flipadst, flipadst INV_TXFM_8X16_FN flipadst, identity -cglobal iflipadst_8x16_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 - lea r3, [o(m(iflipadst_8x8_internal).pass1)] - jmp m(idct_8x16_internal).pass1 +cglobal iflipadst_8x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 + lea r3, [o(m(iflipadst_8x8_internal_8bpc).pass1)] + jmp m(idct_8x16_internal_8bpc).pass1 .pass2: - lea tx2q, [o(m(iflipadst_8x16_internal).end)] + lea tx2q, [o(m(iflipadst_8x16_internal_8bpc).end)] lea r3, [dstq+strideq*8] .pass2_pre: mova [rsp+gprsize+16*7], m0 mova [rsp+gprsize+16*8], m1 @@ -2368,43 +2369,43 @@ mova m4, [coeffq+16*5 ] mova m5, [coeffq+16*7 ] mova m6, [coeffq+16*9 ] mova m7, [coeffq+16*11] - call m(iadst_16x8_internal).main - call m(iadst_16x8_internal).main_pass2_end - jmp m(iflipadst_8x8_internal).end + call m(iadst_16x8_internal_8bpc).main + call m(iadst_16x8_internal_8bpc).main_pass2_end + jmp m(iflipadst_8x8_internal_8bpc).end .end: LOAD_8ROWS rsp+gprsize+16*3, 16 mova [rsp+gprsize+16*0], m7 - lea tx2q, [o(m(idct_8x16_internal).end1)] + lea tx2q, [o(m(idct_8x16_internal_8bpc).end1)] mov dstq, r3 - jmp m(iflipadst_8x8_internal).end + jmp m(iflipadst_8x8_internal_8bpc).end INV_TXFM_8X16_FN identity, dct INV_TXFM_8X16_FN identity, adst INV_TXFM_8X16_FN identity, flipadst INV_TXFM_8X16_FN identity, identity -cglobal iidentity_8x16_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal iidentity_8x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 LOAD_8ROWS coeffq+16*1, 32, 1 mov r3, tx2q - lea tx2q, [o(m(iidentity_8x16_internal).pass1_end)] + lea tx2q, [o(m(iidentity_8x16_internal_8bpc).pass1_end)] mova [rsp+gprsize+16*1], m6 - jmp m(idct_8x8_internal).pass1_end3 + jmp m(idct_8x8_internal_8bpc).pass1_end3 .pass1_end: SAVE_8ROWS coeffq+16*1, 32 LOAD_8ROWS coeffq+16*0, 32, 1 mov tx2q, r3 mova [rsp+gprsize+16*1], m6 - jmp m(idct_8x8_internal).pass1_end3 + jmp m(idct_8x8_internal_8bpc).pass1_end3 .pass2: - lea tx2q, [o(m(iidentity_8x16_internal).end1)] + lea tx2q, [o(m(iidentity_8x16_internal_8bpc).end1)] .end: mova [rsp+gprsize+16*0], m7 mova [rsp+gprsize+16*1], m6 mova m7, [o(pw_1697x16)] @@ -2418,15 +2419,15 @@ REPX {pmulhrsw x, m7}, m0, m1, m2, m3, m4, m5, m6 pmulhrsw m7, [rsp+gprsize+16*2] mova [rsp+gprsize+16*0], m5 mova [rsp+gprsize+16*1], m6 mova [rsp+gprsize+16*2], m7 - jmp m(idct_8x8_internal).end3 + jmp m(idct_8x8_internal_8bpc).end3 .end1: LOAD_8ROWS coeffq+16*1, 32 - lea tx2q, [o(m(idct_8x16_internal).end1)] + lea tx2q, [o(m(idct_8x16_internal_8bpc).end1)] lea dstq, [dstq+strideq*2] jmp .end %macro INV_TXFM_16X8_FN 2 ; type1, type2 @@ -2436,54 +2437,54 @@ pmulhrsw m0, m1, [coeffq] movd m2, [o(pw_16384)] mov [coeffq], eobd pmulhrsw m0, m1 mov r2d, 4 - lea tx2q, [o(m(inv_txfm_add_dct_dct_16x8).end)] - jmp m(inv_txfm_add_dct_dct_16x4).dconly + lea tx2q, [o(m(inv_txfm_add_dct_dct_16x8_8bpc).end)] + jmp m(inv_txfm_add_dct_dct_16x4_8bpc).dconly .end: RET %endif %endmacro INV_TXFM_16X8_FN dct, dct INV_TXFM_16X8_FN dct, adst INV_TXFM_16X8_FN dct, flipadst INV_TXFM_16X8_FN dct, identity -cglobal idct_16x8_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal idct_16x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 LOAD_8ROWS coeffq+16*0, 32, 1 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main SAVE_7ROWS rsp+gprsize+16*3, 16 LOAD_8ROWS coeffq+16*1, 32, 1 call .main mov r3, tx2q - lea tx2q, [o(m(idct_16x8_internal).pass1_end)] - jmp m(idct_8x8_internal).pass1_end + lea tx2q, [o(m(idct_16x8_internal_8bpc).pass1_end)] + jmp m(idct_8x8_internal_8bpc).pass1_end .pass1_end: SAVE_8ROWS coeffq+16*1, 32 LOAD_8ROWS rsp+gprsize+16*3, 16 mova [rsp+gprsize+16*0], m7 mov tx2q, r3 - jmp m(idct_8x8_internal).pass1_end + jmp m(idct_8x8_internal_8bpc).pass1_end .pass2: - lea tx2q, [o(m(idct_16x8_internal).end)] + lea tx2q, [o(m(idct_16x8_internal_8bpc).end)] lea r3, [dstq+8] - jmp m(idct_8x8_internal).pass2_main + jmp m(idct_8x8_internal_8bpc).pass2_main .end: LOAD_8ROWS coeffq+16*1, 32 - lea tx2q, [o(m(idct_8x16_internal).end1)] + lea tx2q, [o(m(idct_8x16_internal_8bpc).end1)] mov dstq, r3 - jmp m(idct_8x8_internal).pass2_main + jmp m(idct_8x8_internal_8bpc).pass2_main ALIGN function_align -.main: +cglobal_label .main mova [rsp+gprsize*2+16*1], m2 mova [rsp+gprsize*2+16*2], m6 mova [rsp+gprsize*2+32*5], m5 mova m6, [o(pd_2048)] @@ -2565,11 +2566,11 @@ INV_TXFM_16X8_FN adst, dct INV_TXFM_16X8_FN adst, adst INV_TXFM_16X8_FN adst, flipadst INV_TXFM_16X8_FN adst, identity -cglobal iadst_16x8_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal iadst_16x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m7, [o(pw_2896x8)] pmulhrsw m0, m7, [coeffq+16*0 ] pmulhrsw m1, m7, [coeffq+16*1 ] pmulhrsw m2, m7, [coeffq+16*14] pmulhrsw m3, m7, [coeffq+16*15] @@ -2595,33 +2596,33 @@ pmulhrsw m7, [coeffq+16*13] call .main call .main_pass1_end mov r3, tx2q - lea tx2q, [o(m(iadst_16x8_internal).pass1_end)] - jmp m(iadst_8x8_internal).pass1_end + lea tx2q, [o(m(iadst_16x8_internal_8bpc).pass1_end)] + jmp m(iadst_8x8_internal_8bpc).pass1_end .pass1_end: SAVE_8ROWS coeffq+16*1, 32 LOAD_8ROWS rsp+gprsize+16*3, 16 mova [rsp+gprsize+16*0], m7 mov tx2q, r3 - jmp m(iadst_8x8_internal).pass1_end + jmp m(iadst_8x8_internal_8bpc).pass1_end .pass2: - lea tx2q, [o(m(iadst_16x8_internal).end)] + lea tx2q, [o(m(iadst_16x8_internal_8bpc).end)] lea r3, [dstq+8] - jmp m(iadst_8x8_internal).pass2_main + jmp m(iadst_8x8_internal_8bpc).pass2_main .end: LOAD_8ROWS coeffq+16*1, 32 - lea tx2q, [o(m(idct_8x16_internal).end1)] + lea tx2q, [o(m(idct_8x16_internal_8bpc).end1)] mov dstq, r3 - jmp m(iadst_8x8_internal).pass2_main + jmp m(iadst_8x8_internal_8bpc).pass2_main ALIGN function_align -.main: +cglobal_label .main mova [rsp+gprsize*2+16*0], m1 mova [rsp+gprsize*2+16*1], m2 mova [rsp+gprsize*2+16*2], m6 mova m6, [o(pd_2048)] @@ -2810,11 +2811,11 @@ mova m4, [rsp+gprsize*2+16* 2] ;out12 mova m5, [rsp+gprsize*2+16*14] ;-out13 mova m6, [rsp+gprsize*2+16*15] ;out14 ret ALIGN function_align -.main_pass2_end: +cglobal_label .main_pass2_end mova m7, [o(pw_2896x8)] mova m1, [rsp+gprsize*2+16* 9] mova m2, [rsp+gprsize*2+16*14] paddsw m0, m1, m2 psubsw m1, m2 @@ -2845,11 +2846,11 @@ INV_TXFM_16X8_FN flipadst, dct INV_TXFM_16X8_FN flipadst, adst INV_TXFM_16X8_FN flipadst, flipadst INV_TXFM_16X8_FN flipadst, identity -cglobal iflipadst_16x8_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal iflipadst_16x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m7, [o(pw_2896x8)] pmulhrsw m0, m7, [coeffq+16*0 ] pmulhrsw m1, m7, [coeffq+16*1 ] pmulhrsw m2, m7, [coeffq+16*14] pmulhrsw m3, m7, [coeffq+16*15] @@ -2872,53 +2873,53 @@ pmulhrsw m4, m7, [coeffq+16*10] pmulhrsw m5, m7, [coeffq+16*11] pmulhrsw m6, m7, [coeffq+16*12] pmulhrsw m7, [coeffq+16*13] - call m(iadst_16x8_internal).main - call m(iadst_16x8_internal).main_pass1_end + call m(iadst_16x8_internal_8bpc).main + call m(iadst_16x8_internal_8bpc).main_pass1_end mova m7, [rsp+gprsize+16*0] SAVE_8ROWS coeffq+16*0, 32 LOAD_8ROWS rsp+gprsize+16*3, 16 mova [rsp+gprsize+16*0], m7 mov r3, tx2q - lea tx2q, [o(m(iflipadst_16x8_internal).pass1_end)] - jmp m(iflipadst_8x8_internal).pass1_end + lea tx2q, [o(m(iflipadst_16x8_internal_8bpc).pass1_end)] + jmp m(iflipadst_8x8_internal_8bpc).pass1_end .pass1_end: SAVE_8ROWS coeffq+16*1, 32 LOAD_8ROWS coeffq+16*0, 32 mova [rsp+gprsize+16*0], m7 mov tx2q, r3 - jmp m(iflipadst_8x8_internal).pass1_end + jmp m(iflipadst_8x8_internal_8bpc).pass1_end .pass2: - lea tx2q, [o(m(iflipadst_16x8_internal).end)] + lea tx2q, [o(m(iflipadst_16x8_internal_8bpc).end)] lea r3, [dstq+8] - jmp m(iflipadst_8x8_internal).pass2_main + jmp m(iflipadst_8x8_internal_8bpc).pass2_main .end: LOAD_8ROWS coeffq+16*1, 32 - lea tx2q, [o(m(idct_8x16_internal).end1)] + lea tx2q, [o(m(idct_8x16_internal_8bpc).end1)] mov dstq, r3 - jmp m(iflipadst_8x8_internal).pass2_main + jmp m(iflipadst_8x8_internal_8bpc).pass2_main INV_TXFM_16X8_FN identity, dct INV_TXFM_16X8_FN identity, adst INV_TXFM_16X8_FN identity, flipadst INV_TXFM_16X8_FN identity, identity -cglobal iidentity_16x8_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal iidentity_16x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 add coeffq, 16*16 mova m4, [coeffq-16*7] mova m5, [coeffq-16*5] mova m6, [coeffq-16*3] mova m7, [coeffq-16*1] mov r3, tx2q - lea tx2q, [o(m(iidentity_16x8_internal).pass1_end)] + lea tx2q, [o(m(iidentity_16x8_internal_8bpc).pass1_end)] .pass1: mova m0, [o(pw_2896x8)] mova m2, [o(pw_1697x16)] mova m3, [o(pw_16384)] @@ -2953,11 +2954,11 @@ paddsw m2, m6 ; 2 pmulhrsw m6, m0, [o(pw_1697x16)] pmulhrsw m6, m3 mova m3, [rsp+gprsize+16*0] paddsw m0, m6 - jmp m(idct_8x8_internal).pass1_end3 + jmp m(idct_8x8_internal_8bpc).pass1_end3 .pass1_end: mova [coeffq+16*1], m4 mova [coeffq+16*3], m5 mova [coeffq+16*5], m6 @@ -2972,90 +2973,90 @@ mova [coeffq-16*1], m3 mov tx2q, r3 jmp .pass1 .pass2: - lea tx2q, [o(m(iidentity_16x8_internal).end)] + lea tx2q, [o(m(iidentity_16x8_internal_8bpc).end)] lea r3, [dstq+8] - jmp m(iidentity_8x8_internal).end + jmp m(iidentity_8x8_internal_8bpc).end .end: LOAD_8ROWS coeffq+16*1, 32 - lea tx2q, [o(m(idct_8x16_internal).end1)] + lea tx2q, [o(m(idct_8x16_internal_8bpc).end1)] mov dstq, r3 - jmp m(iidentity_8x8_internal).end + jmp m(iidentity_8x8_internal_8bpc).end %macro INV_TXFM_16X16_FN 2 ; type1, type2 INV_TXFM_FN %1, %2, 16x16, 8, 16*16 %ifidn %1_%2, dct_dct movd m1, [o(pw_2896x8)] pmulhrsw m0, m1, [coeffq] movd m2, [o(pw_8192)] mov [coeffq], eobd mov r2d, 8 - lea tx2q, [o(m(inv_txfm_add_dct_dct_16x16).end)] - jmp m(inv_txfm_add_dct_dct_16x4).dconly + lea tx2q, [o(m(inv_txfm_add_dct_dct_16x16_8bpc).end)] + jmp m(inv_txfm_add_dct_dct_16x4_8bpc).dconly .end: RET %endif %endmacro INV_TXFM_16X16_FN dct, dct INV_TXFM_16X16_FN dct, adst INV_TXFM_16X16_FN dct, flipadst INV_TXFM_16X16_FN dct, identity -cglobal idct_16x16_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal idct_16x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 LOAD_8ROWS coeffq+16*1, 64 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main SAVE_7ROWS rsp+gprsize+16*3, 16 LOAD_8ROWS coeffq+16*3, 64 - call m(idct_16x8_internal).main + call m(idct_16x8_internal_8bpc).main mov r3, tx2q - lea tx2q, [o(m(idct_16x16_internal).pass1_end)] + lea tx2q, [o(m(idct_16x16_internal_8bpc).pass1_end)] mova m7, [o(pw_8192)] - jmp m(idct_8x8_internal).pass1_end1 + jmp m(idct_8x8_internal_8bpc).pass1_end1 .pass1_end: SAVE_8ROWS coeffq+16*17, 32 LOAD_8ROWS rsp+gprsize+16*3, 16 mova [rsp+gprsize+16*0], m7 - lea tx2q, [o(m(idct_16x16_internal).pass1_end1)] + lea tx2q, [o(m(idct_16x16_internal_8bpc).pass1_end1)] mova m7, [o(pw_8192)] - jmp m(idct_8x8_internal).pass1_end1 + jmp m(idct_8x8_internal_8bpc).pass1_end1 .pass1_end1: SAVE_8ROWS coeffq+16*1, 32 LOAD_8ROWS coeffq+16*0, 64 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main SAVE_7ROWS rsp+gprsize+16*3, 16 LOAD_8ROWS coeffq+16*2, 64 - call m(idct_16x8_internal).main - lea tx2q, [o(m(idct_16x16_internal).pass1_end2)] + call m(idct_16x8_internal_8bpc).main + lea tx2q, [o(m(idct_16x16_internal_8bpc).pass1_end2)] mova m7, [o(pw_8192)] - jmp m(idct_8x8_internal).pass1_end1 + jmp m(idct_8x8_internal_8bpc).pass1_end1 .pass1_end2: SAVE_8ROWS coeffq+16*16, 32 LOAD_8ROWS rsp+gprsize+16*3, 16 mova [rsp+gprsize+16*0], m7 mov tx2q, r3 mova m7, [o(pw_8192)] - jmp m(idct_8x8_internal).pass1_end1 + jmp m(idct_8x8_internal_8bpc).pass1_end1 .pass2: - lea tx2q, [o(m(idct_16x16_internal).end)] - jmp m(idct_8x16_internal).pass2_pre + lea tx2q, [o(m(idct_16x16_internal_8bpc).end)] + jmp m(idct_8x16_internal_8bpc).pass2_pre .end: LOAD_8ROWS rsp+gprsize+16*3, 16 mova [rsp+gprsize+16*0], m7 - lea tx2q, [o(m(idct_16x16_internal).end1)] + lea tx2q, [o(m(idct_16x16_internal_8bpc).end1)] mov dstq, r3 lea r3, [dstq+8] - jmp m(idct_8x8_internal).end + jmp m(idct_8x8_internal_8bpc).end .end1: pxor m7, m7 REPX {mova [coeffq+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 @@ -3068,12 +3069,12 @@ mova m3, [coeffq+16*12] mova m4, [coeffq+16*1 ] mova m5, [coeffq+16*5 ] mova m6, [coeffq+16*9 ] mova m7, [coeffq+16*13] - lea tx2q, [o(m(idct_8x16_internal).end)] - jmp m(idct_8x16_internal).pass2_main + lea tx2q, [o(m(idct_8x16_internal_8bpc).end)] + jmp m(idct_8x16_internal_8bpc).pass2_main %macro ITX_16X16_ADST_LOAD_ODD_COEFS 0 mova m0, [coeffq+16*1 ] mova m1, [coeffq+16*3 ] @@ -3130,57 +3131,57 @@ INV_TXFM_16X16_FN adst, dct INV_TXFM_16X16_FN adst, adst INV_TXFM_16X16_FN adst, flipadst -cglobal iadst_16x16_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal iadst_16x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 ITX_16X16_ADST_LOAD_ODD_COEFS - call m(iadst_16x8_internal).main - call m(iadst_16x8_internal).main_pass1_end + call m(iadst_16x8_internal_8bpc).main + call m(iadst_16x8_internal_8bpc).main_pass1_end mov r3, tx2q - lea tx2q, [o(m(iadst_16x16_internal).pass1_end)] + lea tx2q, [o(m(iadst_16x16_internal_8bpc).pass1_end)] mova m7, [o(pw_8192)] - jmp m(iadst_8x8_internal).pass1_end1 + jmp m(iadst_8x8_internal_8bpc).pass1_end1 .pass1_end: SAVE_8ROWS coeffq+16*17, 32 LOAD_8ROWS rsp+gprsize+16*3, 16 mova [rsp+gprsize+16*0], m7 - lea tx2q, [o(m(iadst_16x16_internal).pass1_end1)] + lea tx2q, [o(m(iadst_16x16_internal_8bpc).pass1_end1)] mova m7, [o(pw_8192)] - jmp m(iadst_8x8_internal).pass1_end1 + jmp m(iadst_8x8_internal_8bpc).pass1_end1 .pass1_end1: SAVE_8ROWS coeffq+16*1, 32 ITX_16X16_ADST_LOAD_EVEN_COEFS - call m(iadst_16x8_internal).main - call m(iadst_16x8_internal).main_pass1_end - - lea tx2q, [o(m(iadst_16x16_internal).pass1_end2)] + call m(iadst_16x8_internal_8bpc).main + call m(iadst_16x8_internal_8bpc).main_pass1_end + + lea tx2q, [o(m(iadst_16x16_internal_8bpc).pass1_end2)] mova m7, [o(pw_8192)] - jmp m(iadst_8x8_internal).pass1_end1 + jmp m(iadst_8x8_internal_8bpc).pass1_end1 .pass1_end2: SAVE_8ROWS coeffq+16*16, 32 LOAD_8ROWS rsp+gprsize+16*3, 16 mova [rsp+gprsize+16*0], m7 mov tx2q, r3 mova m7, [o(pw_8192)] - jmp m(iadst_8x8_internal).pass1_end1 + jmp m(iadst_8x8_internal_8bpc).pass1_end1 .pass2: - lea tx2q, [o(m(iadst_16x16_internal).end)] - jmp m(iadst_8x16_internal).pass2_pre + lea tx2q, [o(m(iadst_16x16_internal_8bpc).end)] + jmp m(iadst_8x16_internal_8bpc).pass2_pre .end: LOAD_8ROWS rsp+gprsize+16*3, 16 mova [rsp+gprsize+16*0], m7 - lea tx2q, [o(m(iadst_16x16_internal).end1)] + lea tx2q, [o(m(iadst_16x16_internal_8bpc).end1)] mov dstq, r3 lea r3, [dstq+8] - jmp m(iadst_8x8_internal).end + jmp m(iadst_8x8_internal_8bpc).end .end1: pxor m7, m7 REPX {mova [coeffq+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 @@ -3197,69 +3198,69 @@ mova m7, [coeffq+16*14] mova [rsp+gprsize+16*7], m4 mova [rsp+gprsize+16*8], m5 mova [rsp+gprsize+16*5], m6 mova [rsp+gprsize+16*6], m7 - lea tx2q, [o(m(iadst_8x16_internal).end)] - jmp m(iadst_8x16_internal).pass2_main + lea tx2q, [o(m(iadst_8x16_internal_8bpc).end)] + jmp m(iadst_8x16_internal_8bpc).pass2_main INV_TXFM_16X16_FN flipadst, dct INV_TXFM_16X16_FN flipadst, adst INV_TXFM_16X16_FN flipadst, flipadst -cglobal iflipadst_16x16_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal iflipadst_16x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 ITX_16X16_ADST_LOAD_ODD_COEFS - call m(iadst_16x8_internal).main - call m(iadst_16x8_internal).main_pass1_end + call m(iadst_16x8_internal_8bpc).main + call m(iadst_16x8_internal_8bpc).main_pass1_end mov r3, tx2q - lea tx2q, [o(m(iflipadst_16x16_internal).pass1_end)] + lea tx2q, [o(m(iflipadst_16x16_internal_8bpc).pass1_end)] mova m7, [o(pw_m8192)] - jmp m(iflipadst_8x8_internal).pass1_end1 + jmp m(iflipadst_8x8_internal_8bpc).pass1_end1 .pass1_end: SAVE_8ROWS coeffq+16*1, 32 LOAD_8ROWS rsp+gprsize+16*3, 16 mova [rsp+gprsize+16*0], m7 - lea tx2q, [o(m(iflipadst_16x16_internal).pass1_end1)] + lea tx2q, [o(m(iflipadst_16x16_internal_8bpc).pass1_end1)] mova m7, [o(pw_m8192)] - jmp m(iflipadst_8x8_internal).pass1_end1 + jmp m(iflipadst_8x8_internal_8bpc).pass1_end1 .pass1_end1: SAVE_8ROWS coeffq+16*17, 32 ITX_16X16_ADST_LOAD_EVEN_COEFS - call m(iadst_16x8_internal).main - call m(iadst_16x8_internal).main_pass1_end + call m(iadst_16x8_internal_8bpc).main + call m(iadst_16x8_internal_8bpc).main_pass1_end mova m7, [rsp+gprsize+16*0] SAVE_8ROWS coeffq+16*0, 32 LOAD_8ROWS rsp+gprsize+16*3, 16 mova [rsp+gprsize+16*0], m7 - lea tx2q, [o(m(iflipadst_16x16_internal).pass1_end2)] + lea tx2q, [o(m(iflipadst_16x16_internal_8bpc).pass1_end2)] mova m7, [o(pw_m8192)] - jmp m(iflipadst_8x8_internal).pass1_end1 + jmp m(iflipadst_8x8_internal_8bpc).pass1_end1 .pass1_end2: SAVE_8ROWS coeffq+16*16, 32 LOAD_8ROWS coeffq+16* 0, 32 mova [rsp+gprsize+16*0], m7 mov tx2q, r3 mova m7, [o(pw_m8192)] - jmp m(iflipadst_8x8_internal).pass1_end1 + jmp m(iflipadst_8x8_internal_8bpc).pass1_end1 .pass2: - lea tx2q, [o(m(iflipadst_16x16_internal).end)] + lea tx2q, [o(m(iflipadst_16x16_internal_8bpc).end)] lea r3, [dstq+8] - jmp m(iflipadst_8x16_internal).pass2_pre + jmp m(iflipadst_8x16_internal_8bpc).pass2_pre .end: LOAD_8ROWS rsp+gprsize+16*3, 16 mova [rsp+gprsize+16*0], m7 - lea tx2q, [o(m(iflipadst_16x16_internal).end1)] + lea tx2q, [o(m(iflipadst_16x16_internal_8bpc).end1)] lea dstq, [dstq+strideq*2] - jmp m(iflipadst_8x8_internal).end + jmp m(iflipadst_8x8_internal_8bpc).end .end1: pxor m7, m7 REPX {mova [coeffq+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 @@ -3276,20 +3277,20 @@ mova [rsp+gprsize+16*7], m4 mova [rsp+gprsize+16*8], m5 mova [rsp+gprsize+16*5], m6 mova [rsp+gprsize+16*6], m7 - lea tx2q, [o(m(iflipadst_16x16_internal).end2)] + lea tx2q, [o(m(iflipadst_16x16_internal_8bpc).end2)] mov dstq, r3 - jmp m(iflipadst_8x16_internal).pass2_main + jmp m(iflipadst_8x16_internal_8bpc).pass2_main .end2: LOAD_8ROWS rsp+gprsize+16*3, 16 mova [rsp+gprsize+16*0], m7 - lea tx2q, [o(m(idct_8x16_internal).end1)] + lea tx2q, [o(m(idct_8x16_internal_8bpc).end1)] lea dstq, [dstq+strideq*2] - jmp m(iflipadst_8x8_internal).end + jmp m(iflipadst_8x8_internal_8bpc).end %macro IDTX16B 3 ; src/dst, tmp, pw_1697x16 pmulhrsw m%2, m%3, m%1 psraw m%2, 1 @@ -3297,14 +3298,14 @@ %endmacro INV_TXFM_16X16_FN identity, dct INV_TXFM_16X16_FN identity, identity -cglobal iidentity_16x16_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal iidentity_16x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 add coeffq, 16*17 mov r3, tx2q - lea tx2q, [o(m(iidentity_16x16_internal).pass1_end)] + lea tx2q, [o(m(iidentity_16x16_internal_8bpc).pass1_end)] .pass1: mova m6, [o(pw_1697x16)] mova m7, [coeffq+32*6] mova m0, [coeffq+32*0] @@ -3316,33 +3317,33 @@ mova m5, [coeffq+32*5] mova [rsp+gprsize+16*1], m7 IDTX16B 5, 7, 6 mova m7, [coeffq+32*7] IDTX16B 7, 6, 6 - jmp m(idct_8x8_internal).pass1_end3 + jmp m(idct_8x8_internal_8bpc).pass1_end3 .pass1_end: SAVE_8ROWS coeffq, 32 sub coeffq, 16 - lea tx2q, [o(m(iidentity_16x16_internal).pass1_end1)] + lea tx2q, [o(m(iidentity_16x16_internal_8bpc).pass1_end1)] jmp .pass1 .pass1_end1: SAVE_8ROWS coeffq, 32 sub coeffq, 15*16 - lea tx2q, [o(m(iidentity_16x16_internal).pass1_end2)] + lea tx2q, [o(m(iidentity_16x16_internal_8bpc).pass1_end2)] jmp .pass1 .pass1_end2: SAVE_8ROWS coeffq, 32 sub coeffq, 16 mov tx2q, r3 jmp .pass1 .pass2: lea r3, [dstq+8] - lea tx2q, [o(m(iidentity_16x16_internal).end1)] + lea tx2q, [o(m(iidentity_16x16_internal_8bpc).end1)] .end: mova [rsp+gprsize+16*0], m7 mova [rsp+gprsize+16*1], m4 mova m7, [o(pw_1697x16)] @@ -3357,42 +3358,42 @@ mova m6, [rsp+gprsize+16*0] IDTX16 6, 7, 7 REPX {pmulhrsw x, m4}, m0, m1, m2, m3, m6 pmulhrsw m4, m5 mova [rsp+gprsize+16*0], m6 - jmp m(idct_8x8_internal).end3 + jmp m(idct_8x8_internal_8bpc).end3 .end1: LOAD_8ROWS coeffq+16*1, 32 - lea tx2q, [o(m(iidentity_16x16_internal).end2)] + lea tx2q, [o(m(iidentity_16x16_internal_8bpc).end2)] lea dstq, [dstq+strideq*2] jmp .end .end2: pxor m7, m7 REPX {mova [coeffq+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 add coeffq, 32*8 LOAD_8ROWS coeffq, 32 - lea tx2q, [o(m(iidentity_16x16_internal).end3)] + lea tx2q, [o(m(iidentity_16x16_internal_8bpc).end3)] mov dstq, r3 jmp .end .end3: LOAD_8ROWS coeffq+16*1, 32 - lea tx2q, [o(m(idct_8x16_internal).end1)] + lea tx2q, [o(m(idct_8x16_internal_8bpc).end1)] lea dstq, [dstq+strideq*2] jmp .end -cglobal inv_txfm_add_dct_dct_8x32, 4, 6, 8, 16*36, dst, stride, coeff, eob, tx2 +cglobal inv_txfm_add_dct_dct_8x32_8bpc, 4, 6, 8, 16*36, dst, stride, coeff, eob, tx2 %if ARCH_X86_32 LEA r5, $$ %endif test eobd, eobd jz .dconly - call m(idct_8x32_internal) + call m(idct_8x32_internal_8bpc) RET .dconly: movd m1, [o(pw_2896x8)] pmulhrsw m0, m1, [coeffq] @@ -3403,28 +3404,28 @@ pmulhrsw m0, m1 pmulhrsw m0, m2 pshuflw m0, m0, q0000 punpcklwd m0, m0 mov r3d, 8 - lea tx2q, [o(m(inv_txfm_add_dct_dct_8x32).end)] - jmp m(inv_txfm_add_dct_dct_8x8).loop + lea tx2q, [o(m(inv_txfm_add_dct_dct_8x32_8bpc).end)] + jmp m(inv_txfm_add_dct_dct_8x8_8bpc).loop .end: RET -cglobal idct_8x32_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal idct_8x32_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 %undef cmp cmp eobd, 106 jle .fast LOAD_8ROWS coeffq+16*3, 64 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main mova m7, [o(pw_8192)] - lea tx2q, [o(m(idct_8x32_internal).pass1)] - jmp m(idct_8x8_internal).pass1_end1 + lea tx2q, [o(m(idct_8x32_internal_8bpc).pass1)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 .pass1: mova [rsp+gprsize+16*9 ], m0 ;in24 mova [rsp+gprsize+16*10], m4 ;in28 mova [rsp+gprsize+16*17], m2 ;in26 @@ -3432,14 +3433,14 @@ mova [rsp+gprsize+16*31], m1 ;in25 mova [rsp+gprsize+16*30], m3 ;in27 mova [rsp+gprsize+16*27], m5 ;in29 mova [rsp+gprsize+16*34], m7 ;in31 LOAD_8ROWS coeffq+16*2, 64 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main mova m7, [o(pw_8192)] - lea tx2q, [o(m(idct_8x32_internal).pass1_1)] - jmp m(idct_8x8_internal).pass1_end1 + lea tx2q, [o(m(idct_8x32_internal_8bpc).pass1_1)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 .pass1_1: mova [rsp+gprsize+16*7 ], m0 ;in16 mova [rsp+gprsize+16*8 ], m4 ;in20 mova [rsp+gprsize+16*15], m2 ;in18 @@ -3449,14 +3450,14 @@ mova [rsp+gprsize+16*29], m5 ;in21 mova [rsp+gprsize+16*32], m7 ;in23 .fast: LOAD_8ROWS coeffq+16*1, 64 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main mova m7, [o(pw_8192)] - lea tx2q, [o(m(idct_8x32_internal).pass1_end)] - jmp m(idct_8x8_internal).pass1_end1 + lea tx2q, [o(m(idct_8x32_internal_8bpc).pass1_end)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 .pass1_end: mova [rsp+gprsize+16*5 ], m0 ;in8 mova [rsp+gprsize+16*6 ], m4 ;in12 mova [rsp+gprsize+16*13], m2 ;in10 @@ -3464,14 +3465,14 @@ mova [rsp+gprsize+16*21], m1 ;in9 mova [rsp+gprsize+16*24], m3 ;in11 mova [rsp+gprsize+16*25], m5 ;in13 mova [rsp+gprsize+16*20], m7 ;in15 LOAD_8ROWS coeffq+16*0, 64 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main mova m7, [o(pw_8192)] - lea tx2q, [o(m(idct_8x32_internal).pass1_end1)] - jmp m(idct_8x8_internal).pass1_end1 + lea tx2q, [o(m(idct_8x32_internal_8bpc).pass1_end1)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 .pass1_end1: mova [rsp+gprsize+16*11], m2 ;in2 mova [rsp+gprsize+16*12], m6 ;in6 mova [rsp+gprsize+16*19], m1 ;in1 @@ -3485,19 +3486,19 @@ cmp eobd, 106 jg .full pxor m4, m4 REPX {mova x, m4}, m5, m6, m7 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main SAVE_7ROWS rsp+gprsize+16*3 , 16 mova m0, [rsp+gprsize+16*11] mova m1, [rsp+gprsize+16*12] mova m2, [rsp+gprsize+16*13] mova m3, [rsp+gprsize+16*14] pxor m4, m4 REPX {mova x, m4}, m5, m6, m7 - call m(idct_16x8_internal).main + call m(idct_16x8_internal_8bpc).main mova m7, [rsp+gprsize+16*0] SAVE_8ROWS rsp+gprsize+16*11, 16 call .main_fast jmp .pass2 @@ -3505,24 +3506,24 @@ .full: mova m4, [rsp+gprsize+16*7 ] ;in16 mova m5, [rsp+gprsize+16*8 ] ;in20 mova m6, [rsp+gprsize+16*9 ] ;in24 mova m7, [rsp+gprsize+16*10] ;in28 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main SAVE_7ROWS rsp+gprsize+16*3 , 16 LOAD_8ROWS rsp+gprsize+16*11, 16 - call m(idct_16x8_internal).main + call m(idct_16x8_internal_8bpc).main mova m7, [rsp+gprsize+16*0] SAVE_8ROWS rsp+gprsize+16*11, 16 call .main .pass2: - lea r3, [o(m(idct_8x32_internal).end6)] + lea r3, [o(m(idct_8x32_internal_8bpc).end6)] .end: mova [rsp+gprsize+16*0 ], m7 - lea tx2q, [o(m(idct_8x32_internal).end2)] + lea tx2q, [o(m(idct_8x32_internal_8bpc).end2)] .end1: pxor m7, m7 REPX {mova [coeffq+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7, \ 8, 9, 10, 11, 12, 13, 14, 15, \ @@ -3530,39 +3531,39 @@ 24, 25, 26, 27, 28, 29, 30, 31 jmp tx2q .end2: - lea tx2q, [o(m(idct_8x32_internal).end3)] - jmp m(idct_8x8_internal).end + lea tx2q, [o(m(idct_8x32_internal_8bpc).end3)] + jmp m(idct_8x8_internal_8bpc).end .end3: LOAD_8ROWS rsp+gprsize+16*11, 16 mova [rsp+gprsize+16*0 ], m7 lea dstq, [dstq+strideq*2] - lea tx2q, [o(m(idct_8x32_internal).end4)] - jmp m(idct_8x8_internal).end + lea tx2q, [o(m(idct_8x32_internal_8bpc).end4)] + jmp m(idct_8x8_internal_8bpc).end .end4: LOAD_8ROWS rsp+gprsize+16*19, 16 mova [rsp+gprsize+16*0 ], m7 lea dstq, [dstq+strideq*2] - lea tx2q, [o(m(idct_8x32_internal).end5)] - jmp m(idct_8x8_internal).end + lea tx2q, [o(m(idct_8x32_internal_8bpc).end5)] + jmp m(idct_8x8_internal_8bpc).end .end5: LOAD_8ROWS rsp+gprsize+16*27, 16 mova [rsp+gprsize+16*0 ], m7 lea dstq, [dstq+strideq*2] mov tx2q, r3 - jmp m(idct_8x8_internal).end + jmp m(idct_8x8_internal_8bpc).end .end6: ret ALIGN function_align -.main_veryfast: +cglobal_label .main_veryfast mova m0, [rsp+gprsize*2+16*19] ;in1 pmulhrsw m3, m0, [o(pw_4091x8)] ;t30,t31 pmulhrsw m0, [o(pw_201x8)] ;t16,t17 mova m7, [o(pd_2048)] mova [rsp+gprsize*2+16*19], m0 ;t16 @@ -3592,11 +3593,11 @@ pmulhrsw m1, m2, [o(pw_4052x8)] pmulhrsw m2, [o(pw_m601x8)] jmp .main2 ALIGN function_align -.main_fast: ;bottom half is zero +cglobal_label .main_fast ;bottom half is zero mova m0, [rsp+gprsize*2+16*19] ;in1 mova m1, [rsp+gprsize*2+16*20] ;in15 pmulhrsw m3, m0, [o(pw_4091x8)] ;t31a pmulhrsw m0, [o(pw_201x8)] ;t16a pmulhrsw m2, m1, [o(pw_3035x8)] ;t30a @@ -3648,11 +3649,11 @@ pmulhrsw m1, m2, [o(pw_4052x8)] pmulhrsw m2, [o(pw_m601x8)] jmp .main2 ALIGN function_align -.main: +cglobal_label .main mova m7, [o(pd_2048)] mova m0, [rsp+gprsize*2+16*19] ;in1 mova m1, [rsp+gprsize*2+16*20] ;in15 mova m2, [rsp+gprsize*2+16*33] ;in17 mova m3, [rsp+gprsize*2+16*34] ;in31 @@ -3868,26 +3869,26 @@ mova m6, [rsp+gprsize*2+16*9 ] ;out6 mova m1, [rsp+gprsize*2+16*4 ] ;out1 ret -cglobal inv_txfm_add_dct_dct_32x8, 4, 6, 8, 16*36, dst, stride, coeff, eob, tx2 +cglobal inv_txfm_add_dct_dct_32x8_8bpc, 4, 6, 8, 16*36, dst, stride, coeff, eob, tx2 %if ARCH_X86_32 LEA r5, $$ %endif test eobd, eobd jz .dconly - call m(idct_32x8_internal) + call m(idct_32x8_internal_8bpc) RET .dconly: movd m1, [o(pw_2896x8)] pmulhrsw m0, m1, [coeffq] movd m2, [o(pw_8192)] mov [coeffq], eobd mov r3d, 8 - lea tx2q, [o(m(inv_txfm_add_dct_dct_32x8).end)] + lea tx2q, [o(m(inv_txfm_add_dct_dct_32x8_8bpc).end)] .body: pmulhrsw m0, m2 movd m2, [o(pw_2048)] ;intentionally rip-relative pmulhrsw m0, m1 @@ -3918,18 +3919,18 @@ .end: RET -cglobal idct_32x8_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal idct_32x8_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 %undef cmp LOAD_8ROWS coeffq+16*0, 64 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main SAVE_7ROWS rsp+gprsize+16*3, 16 LOAD_8ROWS coeffq+16*2, 64 - call m(idct_16x8_internal).main + call m(idct_16x8_internal_8bpc).main mova m7, [rsp+gprsize+16*0] SAVE_8ROWS rsp+gprsize+16*11, 16 LOAD_8ROWS coeffq+16*1, 32 mova [rsp+gprsize+16*19], m0 ;in1 @@ -3941,11 +3942,11 @@ mova [rsp+gprsize+16*25], m6 ;in13 mova [rsp+gprsize+16*20], m7 ;in15 cmp eobd, 106 jg .full - call m(idct_8x32_internal).main_fast + call m(idct_8x32_internal_8bpc).main_fast jmp .pass2 .full: LOAD_8ROWS coeffq+16*17, 32 mova [rsp+gprsize+16*33], m0 ;in17 @@ -3954,100 +3955,100 @@ mova [rsp+gprsize+16*32], m3 ;in23 mova [rsp+gprsize+16*31], m4 ;in25 mova [rsp+gprsize+16*30], m5 ;in27 mova [rsp+gprsize+16*27], m6 ;in29 mova [rsp+gprsize+16*34], m7 ;in31 - call m(idct_8x32_internal).main + call m(idct_8x32_internal_8bpc).main .pass2: mova [rsp+gprsize+16*0 ], m7 - lea tx2q, [o(m(idct_32x8_internal).end)] - jmp m(idct_8x32_internal).end1 + lea tx2q, [o(m(idct_32x8_internal_8bpc).end)] + jmp m(idct_8x32_internal_8bpc).end1 .end: mova m7, [o(pw_8192)] - lea tx2q, [o(m(idct_32x8_internal).end1)] - jmp m(idct_8x8_internal).pass1_end1 + lea tx2q, [o(m(idct_32x8_internal_8bpc).end1)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 .end1: lea r3, [dstq+8] - lea tx2q, [o(m(idct_32x8_internal).end2)] - jmp m(idct_8x8_internal).pass2_main + lea tx2q, [o(m(idct_32x8_internal_8bpc).end2)] + jmp m(idct_8x8_internal_8bpc).pass2_main .end2: LOAD_8ROWS rsp+gprsize+16*11, 16 mova [rsp+gprsize+16*0 ], m7 mova m7, [o(pw_8192)] - lea tx2q, [o(m(idct_32x8_internal).end3)] - jmp m(idct_8x8_internal).pass1_end1 + lea tx2q, [o(m(idct_32x8_internal_8bpc).end3)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 .end3: mov dstq, r3 add r3, 8 - lea tx2q, [o(m(idct_32x8_internal).end4)] - jmp m(idct_8x8_internal).pass2_main + lea tx2q, [o(m(idct_32x8_internal_8bpc).end4)] + jmp m(idct_8x8_internal_8bpc).pass2_main .end4: LOAD_8ROWS rsp+gprsize+16*19, 16 mova [rsp+gprsize+16*0 ], m7 mova m7, [o(pw_8192)] - lea tx2q, [o(m(idct_32x8_internal).end5)] - jmp m(idct_8x8_internal).pass1_end1 + lea tx2q, [o(m(idct_32x8_internal_8bpc).end5)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 .end5: mov dstq, r3 add r3, 8 - lea tx2q, [o(m(idct_32x8_internal).end6)] - jmp m(idct_8x8_internal).pass2_main + lea tx2q, [o(m(idct_32x8_internal_8bpc).end6)] + jmp m(idct_8x8_internal_8bpc).pass2_main .end6: LOAD_8ROWS rsp+gprsize+16*27, 16 mova [rsp+gprsize+16*0 ], m7 mova m7, [o(pw_8192)] - lea tx2q, [o(m(idct_32x8_internal).end7)] - jmp m(idct_8x8_internal).pass1_end1 + lea tx2q, [o(m(idct_32x8_internal_8bpc).end7)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 .end7: mov dstq, r3 - lea tx2q, [o(m(idct_32x8_internal).end8)] - jmp m(idct_8x8_internal).pass2_main + lea tx2q, [o(m(idct_32x8_internal_8bpc).end8)] + jmp m(idct_8x8_internal_8bpc).pass2_main .end8: ret -cglobal inv_txfm_add_identity_identity_8x32, 4, 6, 8, 16*4, dst, stride, coeff, eob, tx2 +cglobal inv_txfm_add_identity_identity_8x32_8bpc, 4, 6, 8, 16*4, dst, stride, coeff, eob, tx2 mov r5d, 4 mov tx2d, 2 cmp eobd, 107 cmovns tx2d, r5d mov r3d, tx2d %if ARCH_X86_32 LEA r5, $$ %endif - lea tx2q, [o(m(idct_32x8_internal).end8)] + lea tx2q, [o(m(idct_32x8_internal_8bpc).end8)] .loop: LOAD_8ROWS coeffq+16*0, 64 paddsw m6, [o(pw_5)] mova [rsp+16*1], m6 mova m6, [o(pw_5)] REPX {paddsw x, m6}, m0, m1, m2, m3, m4, m5, m7 - call m(idct_8x8_internal).pass1_end3 + call m(idct_8x8_internal_8bpc).pass1_end3 REPX {psraw x, 3 }, m0, m1, m2, m3, m4, m5, m6, m7 mova [rsp+16*2], m5 mova [rsp+16*1], m6 mova [rsp+16*0], m7 - call m(idct_8x8_internal).end3 + call m(idct_8x8_internal_8bpc).end3 lea dstq, [dstq+strideq*2] pxor m7, m7 REPX {mova [coeffq+64*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7 add coeffq, 16 dec r3d jg .loop RET -cglobal inv_txfm_add_identity_identity_32x8, 4, 6, 8, 16*4, dst, stride, coeff, eob, tx2 +cglobal inv_txfm_add_identity_identity_32x8_8bpc, 4, 6, 8, 16*4, dst, stride, coeff, eob, tx2 mov r5d, 4 mov tx2d, 2 cmp eobd, 107 cmovns tx2d, r5d mov r3d, tx2d @@ -4059,68 +4060,68 @@ LOAD_8ROWS coeffq+16*0, 16 pmulhrsw m6, [o(pw_4096)] mova [rsp+16*1], m6 mova m6, [o(pw_4096)] REPX {pmulhrsw x, m6}, m0, m1, m2, m3, m4, m5, m7 - lea tx2q, [o(m(idct_32x8_internal).end8)] - call m(idct_8x8_internal).pass1_end3 + lea tx2q, [o(m(idct_32x8_internal_8bpc).end8)] + call m(idct_8x8_internal_8bpc).pass1_end3 mov [rsp+16*3], dstq mova [rsp+16*2], m5 mova [rsp+16*1], m6 mova [rsp+16*0], m7 - lea tx2q, [o(m(idct_8x8_internal).end4)] - call m(idct_8x8_internal).end3 + lea tx2q, [o(m(idct_8x8_internal_8bpc).end4)] + call m(idct_8x8_internal_8bpc).end3 add coeffq, 16*8 mov dstq, [rsp+16*3] lea dstq, [dstq+8] dec r3d jg .loop jnc .loop RET -cglobal inv_txfm_add_dct_dct_16x32, 4, 6, 8, 16*36, dst, stride, coeff, eob, tx2 +cglobal inv_txfm_add_dct_dct_16x32_8bpc, 4, 6, 8, 16*36, dst, stride, coeff, eob, tx2 %if ARCH_X86_32 LEA r5, $$ %endif test eobd, eobd jz .dconly - call m(idct_16x32_internal) + call m(idct_16x32_internal_8bpc) RET .dconly: movd m1, [o(pw_2896x8)] pmulhrsw m0, m1, [coeffq] movd m2, [o(pw_16384)] mov [coeffq], eobd pmulhrsw m0, m1 mov r2d, 16 - lea tx2q, [o(m(inv_txfm_add_dct_dct_16x32).end)] - jmp m(inv_txfm_add_dct_dct_16x4).dconly + lea tx2q, [o(m(inv_txfm_add_dct_dct_16x32_8bpc).end)] + jmp m(inv_txfm_add_dct_dct_16x4_8bpc).dconly .end: RET -cglobal idct_16x32_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal idct_16x32_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 %undef cmp LOAD_8ROWS coeffq+16*1, 128, 1 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main SAVE_7ROWS rsp+gprsize+16*3, 16 LOAD_8ROWS coeffq+16*5, 128, 1 - call m(idct_16x8_internal).main - lea tx2q, [o(m(idct_16x32_internal).pass1_end)] - jmp m(idct_8x8_internal).pass1_end + call m(idct_16x8_internal_8bpc).main + lea tx2q, [o(m(idct_16x32_internal_8bpc).pass1_end)] + jmp m(idct_8x8_internal_8bpc).pass1_end .pass1_end: SAVE_8ROWS coeffq+16*33, 64 ;in8~in15 LOAD_8ROWS rsp+gprsize+16*3, 16 mova [rsp+gprsize+16*0], m7 - lea tx2q, [o(m(idct_16x32_internal).pass1_end1)] - jmp m(idct_8x8_internal).pass1_end + lea tx2q, [o(m(idct_16x32_internal_8bpc).pass1_end1)] + jmp m(idct_8x8_internal_8bpc).pass1_end .pass1_end1: mova [coeffq+16*1 ], m0 ;in8 mova [coeffq+16*5 ], m4 ;in12 mova [rsp+gprsize+16*13], m2 ;in10 @@ -4128,23 +4129,23 @@ mova [rsp+gprsize+16*21], m1 ;in9 mova [rsp+gprsize+16*24], m3 ;in11 mova [rsp+gprsize+16*25], m5 ;in13 mova [rsp+gprsize+16*20], m7 ;in15 LOAD_8ROWS coeffq+16*0, 128, 1 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main SAVE_7ROWS rsp+gprsize+16*3, 16 LOAD_8ROWS coeffq+16*4, 128, 1 - call m(idct_16x8_internal).main - lea tx2q, [o(m(idct_16x32_internal).pass1_end2)] - jmp m(idct_8x8_internal).pass1_end + call m(idct_16x8_internal_8bpc).main + lea tx2q, [o(m(idct_16x32_internal_8bpc).pass1_end2)] + jmp m(idct_8x8_internal_8bpc).pass1_end .pass1_end2: SAVE_8ROWS coeffq+16*32, 64 ;in0~in7 LOAD_8ROWS rsp+gprsize+16*3, 16 mova [rsp+gprsize+16*0], m7 - lea tx2q, [o(m(idct_16x32_internal).pass1_end3)] - jmp m(idct_8x8_internal).pass1_end + lea tx2q, [o(m(idct_16x32_internal_8bpc).pass1_end3)] + jmp m(idct_8x8_internal_8bpc).pass1_end .pass1_end3: mova [rsp+gprsize+16*11], m2 ;in2 mova [rsp+gprsize+16*12], m6 ;in6 mova [rsp+gprsize+16*19], m1 ;in1 @@ -4158,43 +4159,43 @@ mova m1, m4 ;in4 mova m2, [coeffq+16*1 ] ;in8 mova m3, [coeffq+16*5 ] ;in12 pxor m4, m4 REPX {mova x, m4}, m5, m6, m7 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main SAVE_7ROWS rsp+gprsize+16*3, 16 mova m0, [rsp+gprsize+16*11] ;in2 mova m1, [rsp+gprsize+16*12] ;in6 mova m2, [rsp+gprsize+16*13] ;in10 mova m3, [rsp+gprsize+16*14] ;in14 pxor m4, m4 REPX {mova x, m4}, m5, m6, m7 - call m(idct_16x8_internal).main + call m(idct_16x8_internal_8bpc).main mova m7, [rsp+gprsize+16*0] SAVE_8ROWS rsp+gprsize+16*11, 16 - call m(idct_8x32_internal).main_fast + call m(idct_8x32_internal_8bpc).main_fast jmp .pass2 .full: mova [coeffq+16*0 ], m0 ;in0 mova [coeffq+16*4 ], m4 ;in4 LOAD_8ROWS coeffq+16*2, 128, 1 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main SAVE_7ROWS rsp+gprsize+16*3, 16 LOAD_8ROWS coeffq+16*6, 128, 1 - call m(idct_16x8_internal).main - lea tx2q, [o(m(idct_16x32_internal).pass1_end4)] - jmp m(idct_8x8_internal).pass1_end + call m(idct_16x8_internal_8bpc).main + lea tx2q, [o(m(idct_16x32_internal_8bpc).pass1_end4)] + jmp m(idct_8x8_internal_8bpc).pass1_end .pass1_end4: SAVE_8ROWS coeffq+16*34, 64 ;in16~in23 LOAD_8ROWS rsp+gprsize+16*3, 16 mova [rsp+gprsize+16*0], m7 - lea tx2q, [o(m(idct_16x32_internal).pass1_end5)] - jmp m(idct_8x8_internal).pass1_end + lea tx2q, [o(m(idct_16x32_internal_8bpc).pass1_end5)] + jmp m(idct_8x8_internal_8bpc).pass1_end .pass1_end5: mova [coeffq+16*2 ], m0 ;in16 mova [coeffq+16*6 ], m4 ;in20 mova [rsp+gprsize+16*15], m2 ;in18 @@ -4203,23 +4204,23 @@ mova [rsp+gprsize+16*28], m3 ;in19 mova [rsp+gprsize+16*29], m5 ;in21 mova [rsp+gprsize+16*32], m7 ;in23 LOAD_8ROWS coeffq+16*3, 128, 1 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main SAVE_7ROWS rsp+gprsize+16*3, 16 LOAD_8ROWS coeffq+16*7, 128, 1 - call m(idct_16x8_internal).main - lea tx2q, [o(m(idct_16x32_internal).pass1_end6)] - jmp m(idct_8x8_internal).pass1_end + call m(idct_16x8_internal_8bpc).main + lea tx2q, [o(m(idct_16x32_internal_8bpc).pass1_end6)] + jmp m(idct_8x8_internal_8bpc).pass1_end .pass1_end6: SAVE_8ROWS coeffq+16*35, 64 ;in24~in31 LOAD_8ROWS rsp+gprsize+16*3, 16 mova [rsp+gprsize+16*0], m7 - lea tx2q, [o(m(idct_16x32_internal).pass1_end7)] - jmp m(idct_8x8_internal).pass1_end + lea tx2q, [o(m(idct_16x32_internal_8bpc).pass1_end7)] + jmp m(idct_8x8_internal_8bpc).pass1_end .pass1_end7: mova [rsp+gprsize+16*17], m2 ;in26 mova [rsp+gprsize+16*18], m6 ;in30 mova [rsp+gprsize+16*31], m1 ;in25 @@ -4233,25 +4234,25 @@ mova m1, [coeffq+16*4 ] ;in4 mova m2, [coeffq+16*1 ] ;in8 mova m3, [coeffq+16*5 ] ;in12 mova m4, [coeffq+16*2 ] ;in16 mova m5, [coeffq+16*6 ] ;in20 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main SAVE_7ROWS rsp+gprsize+16*3 , 16 LOAD_8ROWS rsp+gprsize+16*11, 16 - call m(idct_16x8_internal).main + call m(idct_16x8_internal_8bpc).main mova m7, [rsp+gprsize+16*0] SAVE_8ROWS rsp+gprsize+16*11, 16 - call m(idct_8x32_internal).main + call m(idct_8x32_internal_8bpc).main .pass2: mov [rsp+gprsize*1+16*35], eobd lea r3, [dstq+8] mov [rsp+gprsize*2+16*35], r3 - lea r3, [o(m(idct_16x32_internal).end)] - jmp m(idct_8x32_internal).end + lea r3, [o(m(idct_16x32_internal_8bpc).end)] + jmp m(idct_8x32_internal_8bpc).end .end: mov dstq, [rsp+gprsize*2+16*35] mov eobd, [rsp+gprsize*1+16*35] add coeffq, 16*32 @@ -4282,43 +4283,43 @@ cmp eobd, 150 jg .full1 pxor m4, m4 REPX {mova x, m4}, m5, m6, m7 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main SAVE_7ROWS rsp+gprsize+16*3, 16 mova m0, [coeffq+16*8 ] ;in2 mova m1, [coeffq+16*24] ;in6 mova m2, [coeffq+16*9 ] ;in10 mova m3, [coeffq+16*25] ;in14 pxor m4, m4 REPX {mova x, m4}, m5, m6, m7 - call m(idct_16x8_internal).main + call m(idct_16x8_internal_8bpc).main mova m7, [rsp+gprsize+16*0] SAVE_8ROWS rsp+gprsize+16*11, 16 - call m(idct_8x32_internal).main_fast + call m(idct_8x32_internal_8bpc).main_fast jmp .end1 .full1: mova m4, [coeffq+16*2 ] ;in16 mova m5, [coeffq+16*18] ;in20 mova m6, [coeffq+16*3 ] ;in24 mova m7, [coeffq+16*19] ;in26 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main SAVE_7ROWS rsp+gprsize+16*3, 16 mova m0, [coeffq+16*8 ] ;in2 mova m1, [coeffq+16*24] ;in6 mova m2, [coeffq+16*9 ] ;in10 mova m3, [coeffq+16*25] ;in14 mova m4, [coeffq+16*10] ;in18 mova m5, [coeffq+16*26] ;in22 mova m6, [coeffq+16*11] ;in26 mova m7, [coeffq+16*27] ;in30 - call m(idct_16x8_internal).main + call m(idct_16x8_internal_8bpc).main mova m7, [rsp+gprsize+16*0] SAVE_8ROWS rsp+gprsize+16*11, 16 mova m0, [coeffq+16*6 ] ;in17 mova m1, [coeffq+16*14] ;in19 @@ -4336,75 +4337,75 @@ mova [rsp+gprsize+16*31], m4 ;in25 mova [rsp+gprsize+16*30], m5 ;in27 mova [rsp+gprsize+16*27], m6 ;in29 mova [rsp+gprsize+16*34], m7 ;in31 - call m(idct_8x32_internal).main + call m(idct_8x32_internal_8bpc).main .end1: - jmp m(idct_8x32_internal).pass2 - - - -cglobal inv_txfm_add_dct_dct_32x16, 4, 6, 8, 16*36, dst, stride, coeff, eob, tx2 + jmp m(idct_8x32_internal_8bpc).pass2 + + + +cglobal inv_txfm_add_dct_dct_32x16_8bpc, 4, 6, 8, 16*36, dst, stride, coeff, eob, tx2 %if ARCH_X86_32 LEA r5, $$ %endif test eobd, eobd jz .dconly - call m(idct_32x16_internal) - call m(idct_8x16_internal).pass2 + call m(idct_32x16_internal_8bpc) + call m(idct_8x16_internal_8bpc).pass2 add coeffq, 16*16 lea dstq, [r3+8] LOAD_8ROWS rsp+16*11, 16 mova [rsp+16*0], m7 - lea tx2q, [o(m(idct_32x16_internal).end)] - call m(idct_8x8_internal).pass1_end - call m(idct_8x16_internal).pass2 + lea tx2q, [o(m(idct_32x16_internal_8bpc).end)] + call m(idct_8x8_internal_8bpc).pass1_end + call m(idct_8x16_internal_8bpc).pass2 add coeffq, 16*16 lea dstq, [r3+8] LOAD_8ROWS rsp+16*19, 16 mova [rsp+16*0], m7 - lea tx2q, [o(m(idct_32x16_internal).end)] - call m(idct_8x8_internal).pass1_end - call m(idct_8x16_internal).pass2 + lea tx2q, [o(m(idct_32x16_internal_8bpc).end)] + call m(idct_8x8_internal_8bpc).pass1_end + call m(idct_8x16_internal_8bpc).pass2 add coeffq, 16*16 lea dstq, [r3+8] LOAD_8ROWS rsp+16*27, 16 mova [rsp+16*0], m7 - lea tx2q, [o(m(idct_32x16_internal).end)] - call m(idct_8x8_internal).pass1_end - call m(idct_8x16_internal).pass2 + lea tx2q, [o(m(idct_32x16_internal_8bpc).end)] + call m(idct_8x8_internal_8bpc).pass1_end + call m(idct_8x16_internal_8bpc).pass2 RET .dconly: movd m1, [o(pw_2896x8)] pmulhrsw m0, m1, [coeffq] movd m2, [o(pw_16384)] mov [coeffq], eobd pmulhrsw m0, m1 mov r3d, 16 - lea tx2q, [o(m(inv_txfm_add_dct_dct_32x8).end)] - jmp m(inv_txfm_add_dct_dct_32x8).body - - -cglobal idct_32x16_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 + lea tx2q, [o(m(inv_txfm_add_dct_dct_32x8_8bpc).end)] + jmp m(inv_txfm_add_dct_dct_32x8_8bpc).body + + +cglobal idct_32x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 %undef cmp add coeffq, 16 - lea r3, [o(m(idct_32x16_internal).pass1_end1)] + lea r3, [o(m(idct_32x16_internal_8bpc).pass1_end1)] .pass1: LOAD_8ROWS coeffq+16*0, 128, 1 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main SAVE_7ROWS rsp+gprsize+16*3, 16 LOAD_8ROWS coeffq+16*4, 128, 1 - call m(idct_16x8_internal).main + call m(idct_16x8_internal_8bpc).main mova m7, [rsp+gprsize+16*0] SAVE_8ROWS rsp+gprsize+16*11, 16 LOAD_8ROWS coeffq+16*2, 64, 1 mova [rsp+gprsize+16*19], m0 ;in1 @@ -4423,50 +4424,50 @@ mova [rsp+gprsize+16*32], m3 ;in23 mova [rsp+gprsize+16*31], m4 ;in25 mova [rsp+gprsize+16*30], m5 ;in27 mova [rsp+gprsize+16*27], m6 ;in29 mova [rsp+gprsize+16*34], m7 ;in31 - call m(idct_8x32_internal).main + call m(idct_8x32_internal_8bpc).main .pass1_end: mova [rsp+gprsize+16*0 ], m7 mov tx2q, r3 - jmp m(idct_8x8_internal).pass1_end + jmp m(idct_8x8_internal_8bpc).pass1_end .pass1_end1: SAVE_8ROWS coeffq+16*0, 32 LOAD_8ROWS rsp+gprsize+16*11, 16 mova [rsp+gprsize+16*0 ], m7 - lea tx2q, [o(m(idct_32x16_internal).pass1_end2)] - jmp m(idct_8x8_internal).pass1_end + lea tx2q, [o(m(idct_32x16_internal_8bpc).pass1_end2)] + jmp m(idct_8x8_internal_8bpc).pass1_end .pass1_end2: SAVE_8ROWS coeffq+16*16, 32 LOAD_8ROWS rsp+gprsize+16*19, 16 mova [rsp+gprsize+16*0 ], m7 - lea tx2q, [o(m(idct_32x16_internal).pass1_end3)] - jmp m(idct_8x8_internal).pass1_end + lea tx2q, [o(m(idct_32x16_internal_8bpc).pass1_end3)] + jmp m(idct_8x8_internal_8bpc).pass1_end .pass1_end3: SAVE_8ROWS coeffq+16*32, 32 LOAD_8ROWS rsp+gprsize+16*27, 16 mova [rsp+gprsize+16*0 ], m7 - lea tx2q, [o(m(idct_32x16_internal).pass1_end4)] - jmp m(idct_8x8_internal).pass1_end + lea tx2q, [o(m(idct_32x16_internal_8bpc).pass1_end4)] + jmp m(idct_8x8_internal_8bpc).pass1_end .pass1_end4: SAVE_8ROWS coeffq+16*48, 32 sub coeffq, 16 - lea r3, [o(m(idct_32x16_internal).end)] + lea r3, [o(m(idct_32x16_internal_8bpc).end)] jmp .pass1 .end: ret -cglobal inv_txfm_add_identity_identity_16x32, 4, 6, 8, 16*4, dst, stride, coeff, eob, tx2 +cglobal inv_txfm_add_identity_identity_16x32_8bpc, 4, 6, 8, 16*4, dst, stride, coeff, eob, tx2 %undef cmp mov r4d, eobd cmp eobd, 43 ;if (eob > 43) sbb r3d, r3d ; iteration_count++ @@ -4486,12 +4487,12 @@ .loop: LOAD_8ROWS coeffq, 64, 1 mova [rsp+16*1], m6 pxor m6, m6 REPX {mova [coeffq+64*x], m6}, 0, 1, 2, 3, 4, 5, 6, 7 - lea tx2q, [o(m(idct_32x16_internal).end)] - call m(idct_8x8_internal).pass1_end3 + lea tx2q, [o(m(idct_32x16_internal_8bpc).end)] + call m(idct_8x8_internal_8bpc).pass1_end3 mova [rsp+16*0], m2 mova [rsp+16*1], m3 mova [rsp+16*2], m4 mova m3, [o(pw_1697x16)] mova m4, [o(pw_16384)] @@ -4511,11 +4512,11 @@ psrlw m4, 1 ; pw_8192 paddsw m3, m5 pmulhrsw m2, m4 pmulhrsw m3, m4 pmulhrsw m4, m7 - call m(idct_8x8_internal).end3 + call m(idct_8x8_internal_8bpc).end3 lea dstq, [dstq+strideq*2] add coeffq, 16 dec r3d jg .loop mov coeffq, [rsp+gprsize*2+16*3] @@ -4527,11 +4528,11 @@ test r3d, r3d jnz .loop RET -cglobal inv_txfm_add_identity_identity_32x16, 4, 6, 8, 16*4, dst, stride, coeff, eob, tx2 +cglobal inv_txfm_add_identity_identity_32x16_8bpc, 4, 6, 8, 16*4, dst, stride, coeff, eob, tx2 %undef cmp mov r4d, 12 ;0100b mov r5d, 136 ;1000 1000b cmp eobd, 44 ;if (eob > 43) @@ -4548,12 +4549,12 @@ .loop: LOAD_8ROWS coeffq, 32, 1 REPX {paddsw x, x}, m0, m1, m2, m3, m4, m5, m6, m7 mova [rsp+16*1], m6 - lea tx2q, [o(m(idct_32x16_internal).end)] - call m(idct_8x8_internal).pass1_end3 + lea tx2q, [o(m(idct_32x16_internal_8bpc).end)] + call m(idct_8x8_internal_8bpc).pass1_end3 mova [rsp+16*1], m5 mova [rsp+16*2], m6 mova m6, [o(pw_1697x16)] REPX {IDTX16 x, 5, 6}, 7, 0, 1, 2, 3, 4 pmulhrsw m7, [o(pw_2048)] @@ -4564,11 +4565,11 @@ IDTX16 7, 6, 6 mova m6, [o(pw_2048)] REPX {pmulhrsw x, m6}, m0, m1, m2, m3, m4, m5, m7 mova [rsp+16*2], m5 mova [rsp+16*1], m7 - call m(idct_8x8_internal).end3 + call m(idct_8x8_internal_8bpc).end3 lea dstq, [dstq+strideq*2] pxor m7, m7 REPX {mova [coeffq+32*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7 .loop_end: @@ -4587,31 +4588,31 @@ .ret: RET -cglobal inv_txfm_add_dct_dct_32x32, 4, 6, 8, 16*36, dst, stride, coeff, eob, tx2 +cglobal inv_txfm_add_dct_dct_32x32_8bpc, 4, 6, 8, 16*36, dst, stride, coeff, eob, tx2 %if ARCH_X86_32 LEA r5, $$ %endif test eobd, eobd jz .dconly - call m(idct_32x32_internal) + call m(idct_32x32_internal_8bpc) RET .dconly: movd m1, [o(pw_2896x8)] pmulhrsw m0, m1, [coeffq] movd m2, [o(pw_8192)] mov [coeffq], eobd mov r3d, 32 - lea tx2q, [o(m(inv_txfm_add_dct_dct_32x8).end)] - jmp m(inv_txfm_add_dct_dct_32x8).body - - -cglobal idct_32x32_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 + lea tx2q, [o(m(inv_txfm_add_dct_dct_32x8_8bpc).end)] + jmp m(inv_txfm_add_dct_dct_32x8_8bpc).body + + +cglobal idct_32x32_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 %undef cmp mov r4d, 2 sub eobd, 136 mov [rsp+gprsize*1+16*35], eobd @@ -4639,14 +4640,14 @@ test tx2d, tx2d jl .fast .full: LOAD_8ROWS coeffq+64*0, 64*4 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main SAVE_7ROWS rsp+gprsize+16*3, 16 LOAD_8ROWS coeffq+64*2, 64*4 - call m(idct_16x8_internal).main + call m(idct_16x8_internal_8bpc).main mova m7, [rsp+gprsize+16*0] SAVE_8ROWS rsp+gprsize+16*11, 16 LOAD_8ROWS coeffq+64*17, 64*2 mova [rsp+gprsize+16*33], m0 ;in17 @@ -4656,64 +4657,64 @@ mova [rsp+gprsize+16*31], m4 ;in25 mova [rsp+gprsize+16*30], m5 ;in27 mova [rsp+gprsize+16*27], m6 ;in29 mova [rsp+gprsize+16*34], m7 ;in31 - call m(idct_8x32_internal).main + call m(idct_8x32_internal_8bpc).main jmp .pass1_end .fast: mova m0, [coeffq+256*0] mova m1, [coeffq+256*1] mova m2, [coeffq+256*2] mova m3, [coeffq+256*3] pxor m4, m4 REPX {mova x, m4}, m5, m6, m7 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main SAVE_7ROWS rsp+gprsize+16*3, 16 mova m0, [coeffq+128*1] mova m1, [coeffq+128*3] mova m2, [coeffq+128*5] mova m3, [coeffq+128*7] pxor m4, m4 REPX {mova x, m4}, m5, m6, m7 - call m(idct_16x8_internal).main + call m(idct_16x8_internal_8bpc).main mova m7, [rsp+gprsize+16*0] SAVE_8ROWS rsp+gprsize+16*11, 16 - call m(idct_8x32_internal).main_fast + call m(idct_8x32_internal_8bpc).main_fast .pass1_end: mova [rsp+gprsize+16*0], m7 mova m7, [o(pw_8192)] - lea tx2q, [o(m(idct_32x32_internal).pass1_end1)] - jmp m(idct_8x8_internal).pass1_end1 + lea tx2q, [o(m(idct_32x32_internal_8bpc).pass1_end1)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 .pass1_end1: SAVE_8ROWS coeffq+64*0, 64 LOAD_8ROWS rsp+gprsize+16*11, 16 mova [rsp+gprsize+16*0], m7 mova m7, [o(pw_8192)] - lea tx2q, [o(m(idct_32x32_internal).pass1_end2)] - jmp m(idct_8x8_internal).pass1_end1 + lea tx2q, [o(m(idct_32x32_internal_8bpc).pass1_end2)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 .pass1_end2: SAVE_8ROWS coeffq+64*8, 64 LOAD_8ROWS rsp+gprsize+16*19, 16 mova [rsp+gprsize+16*0], m7 mova m7, [o(pw_8192)] - lea tx2q, [o(m(idct_32x32_internal).pass1_end3)] - jmp m(idct_8x8_internal).pass1_end1 + lea tx2q, [o(m(idct_32x32_internal_8bpc).pass1_end3)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 .pass1_end3: SAVE_8ROWS coeffq+64*16, 64 LOAD_8ROWS rsp+gprsize+16*27, 16 mova [rsp+gprsize+16*0], m7 mova m7, [o(pw_8192)] - lea tx2q, [o(m(idct_32x32_internal).pass1_end4)] - jmp m(idct_8x8_internal).pass1_end1 + lea tx2q, [o(m(idct_32x32_internal_8bpc).pass1_end4)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 .pass1_end4: SAVE_8ROWS coeffq+64*24, 64 add coeffq, 16 @@ -4722,11 +4723,11 @@ .pass2: mov coeffq, [rsp+gprsize*2+16*35] mov r3d, 4 - lea tx2q, [o(m(idct_32x32_internal).pass2_end)] + lea tx2q, [o(m(idct_32x32_internal_8bpc).pass2_end)] .pass2_loop: mov [rsp+gprsize*3+16*35], r3d lea r3, [dstq+8] mov [rsp+gprsize*2+16*35], r3 @@ -4759,22 +4760,22 @@ mova m3, [coeffq+16*17] mova m4, [coeffq+16*2 ] mova m5, [coeffq+16*18] mova m6, [coeffq+16*3 ] mova m7, [coeffq+16*19] - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main SAVE_7ROWS rsp+gprsize+16*3, 16 mova m0, [coeffq+16*8 ] mova m1, [coeffq+16*24] mova m2, [coeffq+16*9 ] mova m3, [coeffq+16*25] mova m4, [coeffq+16*10] mova m5, [coeffq+16*26] mova m6, [coeffq+16*11] mova m7, [coeffq+16*27] - call m(idct_16x8_internal).main + call m(idct_16x8_internal_8bpc).main mova m7, [rsp+gprsize+16*0] SAVE_8ROWS rsp+gprsize+16*11, 16 mova m0, [coeffq+16*6 ] mova m1, [coeffq+16*14] @@ -4791,52 +4792,52 @@ mova [rsp+gprsize+16*31], m4 ;in25 mova [rsp+gprsize+16*30], m5 ;in27 mova [rsp+gprsize+16*27], m6 ;in29 mova [rsp+gprsize+16*34], m7 ;in31 - call m(idct_8x32_internal).main + call m(idct_8x32_internal_8bpc).main jmp tx2q .fast1: mova m0, [coeffq+16*0 ] mova m1, [coeffq+16*16] mova m2, [coeffq+16*1 ] mova m3, [coeffq+16*17] pxor m4, m4 REPX {mova x, m4}, m5, m6, m7 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main SAVE_7ROWS rsp+gprsize+16*3, 16 mova m0, [coeffq+16*8 ] mova m1, [coeffq+16*24] mova m2, [coeffq+16*9 ] mova m3, [coeffq+16*25] pxor m4, m4 REPX {mova x, m4}, m5, m6, m7 - call m(idct_16x8_internal).main + call m(idct_16x8_internal_8bpc).main mova m7, [rsp+gprsize+16*0] SAVE_8ROWS rsp+gprsize+16*11, 16 - call m(idct_8x32_internal).main_fast + call m(idct_8x32_internal_8bpc).main_fast jmp tx2q .pass2_end: - lea r3, [o(m(idct_32x32_internal).pass2_end1)] - jmp m(idct_8x32_internal).end + lea r3, [o(m(idct_32x32_internal_8bpc).pass2_end1)] + jmp m(idct_8x32_internal_8bpc).end .pass2_end1: - lea tx2q, [o(m(idct_32x32_internal).pass2_end)] + lea tx2q, [o(m(idct_32x32_internal_8bpc).pass2_end)] add coeffq, 16*32 mov dstq, [rsp+gprsize*2+16*35] mov r3d, [rsp+gprsize*3+16*35] dec r3d jg .pass2_loop ret -cglobal inv_txfm_add_identity_identity_32x32, 4, 6, 8, 16*5, dst, stride, coeff, eob, tx2 +cglobal inv_txfm_add_identity_identity_32x32_8bpc, 4, 6, 8, 16*5, dst, stride, coeff, eob, tx2 %undef cmp mov r4d, 2 cmp eobd, 136 mov r3d, 4 @@ -4853,19 +4854,19 @@ mov [rsp+gprsize*3+16*3], coeffq .loop: LOAD_8ROWS coeffq, 64 mova [rsp+16*1], m6 - lea tx2q, [o(m(idct_32x16_internal).end)] - call m(idct_8x8_internal).pass1_end3 + lea tx2q, [o(m(idct_32x16_internal_8bpc).end)] + call m(idct_8x8_internal_8bpc).pass1_end3 pmulhrsw m7, [o(pw_8192)] mova [rsp+16*0], m7 mova m7, [o(pw_8192)] REPX {pmulhrsw x, m7}, m0, m1, m2, m3, m4, m5, m6 mova [rsp+16*1], m6 mova [rsp+16*2], m5 - call m(idct_8x8_internal).end3 + call m(idct_8x8_internal_8bpc).end3 lea dstq, [dstq+strideq*2] pxor m7, m7 REPX {mova [coeffq+64*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7 @@ -4889,34 +4890,34 @@ .ret: RET -cglobal inv_txfm_add_dct_dct_16x64, 4, 6, 8, 16*68, dst, stride, coeff, eob, tx2 +cglobal inv_txfm_add_dct_dct_16x64_8bpc, 4, 6, 8, 16*68, dst, stride, coeff, eob, tx2 %if ARCH_X86_32 LEA r5, $$ %endif test eobd, eobd jz .dconly - call m(idct_16x64_internal) + call m(idct_16x64_internal_8bpc) RET .dconly: movd m1, [o(pw_2896x8)] pmulhrsw m0, m1, [coeffq] movd m2, [o(pw_8192)] mov [coeffq], eobd mov r2d, 32 - lea tx2q, [o(m(inv_txfm_add_dct_dct_16x64).end)] - jmp m(inv_txfm_add_dct_dct_16x4).dconly + lea tx2q, [o(m(inv_txfm_add_dct_dct_16x64_8bpc).end)] + jmp m(inv_txfm_add_dct_dct_16x4_8bpc).dconly .end: RET -cglobal idct_16x64_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal idct_16x64_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 %undef cmp mov r4d, 2 sub eobd, 151 mov [rsp+gprsize*1+16*67], eobd @@ -4929,25 +4930,25 @@ mov [rsp+gprsize*2+16*67], coeffq .pass1_loop: LOAD_8ROWS coeffq+64*0, 64*2 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main SAVE_7ROWS rsp+gprsize+16*3, 16 LOAD_8ROWS coeffq+64*1, 64*2 - call m(idct_16x8_internal).main + call m(idct_16x8_internal_8bpc).main mova m7, [o(pw_8192)] - lea tx2q, [o(m(idct_16x64_internal).pass1_end)] - jmp m(idct_8x8_internal).pass1_end1 + lea tx2q, [o(m(idct_16x64_internal_8bpc).pass1_end)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 .pass1_end: SAVE_8ROWS coeffq+64*8, 64 LOAD_8ROWS rsp+gprsize+16*3, 16 mova [rsp+gprsize+16*0], m7 mova m7, [o(pw_8192)] - lea tx2q, [o(m(idct_16x64_internal).pass1_end1)] - jmp m(idct_8x8_internal).pass1_end1 + lea tx2q, [o(m(idct_16x64_internal_8bpc).pass1_end1)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 .pass1_end1: SAVE_8ROWS coeffq+64*0, 64 add coeffq, 16 @@ -4956,11 +4957,11 @@ mov coeffq, [rsp+gprsize*2+16*67] mov r3d, 2 lea r4, [dstq+8] mov [rsp+gprsize*2+16*67], r4 - lea r4, [o(m(idct_16x64_internal).end1)] + lea r4, [o(m(idct_16x64_internal_8bpc).end1)] .pass2_loop: mov [rsp+gprsize*3+16*67], r3d mov eobd, [rsp+gprsize*1+16*67] @@ -4991,21 +4992,21 @@ .full: mova m2, [coeffq+16*2] mova m3, [coeffq+16*3] REPX {mova x, m4}, m5, m6, m7 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main SAVE_7ROWS rsp+gprsize+16*3, 16 pxor m4, m4 mova m0, [coeffq+16*16] mova m1, [coeffq+16*17] mova m2, [coeffq+16*18] mova m3, [coeffq+16*19] REPX {mova x, m4}, m5, m6, m7 - call m(idct_16x8_internal).main + call m(idct_16x8_internal_8bpc).main mova m7, [rsp+gprsize+16*0] SAVE_8ROWS rsp+gprsize+16*11, 16 mova m0, [coeffq+16*8 ] mova m1, [coeffq+16*24] @@ -5022,11 +5023,11 @@ mova [rsp+gprsize+16*21], m4 mova [rsp+gprsize+16*24], m5 mova [rsp+gprsize+16*25], m6 mova [rsp+gprsize+16*20], m7 - call m(idct_8x32_internal).main_fast + call m(idct_8x32_internal_8bpc).main_fast SAVE_8ROWS rsp+gprsize+16*3, 16 mova m0, [coeffq+16*6 ] ;in17 mova m1, [coeffq+16*14] ;in19 mova m2, [coeffq+16*22] ;in21 @@ -5047,19 +5048,19 @@ call .main jmp .end .fast: REPX {mova x, m4}, m2, m3, m5, m6, m7 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main SAVE_7ROWS rsp+gprsize+16*3, 16 pxor m4, m4 mova m0, [coeffq+16*16] mova m1, [coeffq+16*17] REPX {mova x, m4}, m2, m3, m5, m6, m7 - call m(idct_16x8_internal).main + call m(idct_16x8_internal_8bpc).main mova m7, [rsp+gprsize+16*0] SAVE_8ROWS rsp+gprsize+16*11, 16 mova m0, [coeffq+16*8 ] mova m1, [coeffq+16*24] @@ -5068,45 +5069,45 @@ mova [rsp+gprsize+16*19], m0 ;in1 mova [rsp+gprsize+16*26], m1 ;in3 mova [rsp+gprsize+16*23], m2 ;in5 mova [rsp+gprsize+16*22], m3 ;in7 - call m(idct_8x32_internal).main_veryfast + call m(idct_8x32_internal_8bpc).main_veryfast SAVE_8ROWS rsp+gprsize+16*3, 16 call .main_fast .end: LOAD_8ROWS rsp+gprsize+16*3, 16 mova [rsp+gprsize+16*0], m7 mov r3, r4 - jmp m(idct_8x32_internal).end2 + jmp m(idct_8x32_internal_8bpc).end2 .end1: LOAD_8ROWS rsp+gprsize+16*35, 16 lea dstq, [dstq+strideq*2] add rsp, 16*32 - lea r3, [o(m(idct_16x64_internal).end2)] - jmp m(idct_8x32_internal).end + lea r3, [o(m(idct_16x64_internal_8bpc).end2)] + jmp m(idct_8x32_internal_8bpc).end .end2: add coeffq, 16*32 sub rsp, 16*32 mov dstq, [rsp+gprsize*2+16*67] mov r3d, [rsp+gprsize*3+16*67] lea r4, [dstq+8] mov [rsp+gprsize*2+16*67], r4 - lea r4, [o(m(idct_16x64_internal).end1)] + lea r4, [o(m(idct_16x64_internal_8bpc).end1)] dec r3d jg .pass2_loop ret ALIGN function_align -.main_fast: +cglobal_label .main_fast mova m0, [rsp+gprsize*2+16*35] ;in1 pmulhrsw m3, m0, [o(pw_4095x8)] ;t62,t63 pmulhrsw m0, [o(pw_101x8)] ;t32,t33 mova m7, [o(pd_2048)] mova [rsp+gprsize*2+16*35], m0 ;t32 @@ -5176,11 +5177,11 @@ mova m5, m0 jmp .main2 ALIGN function_align -.main: +cglobal_label .main mova m0, [rsp+gprsize*2+16*35] ;in1 mova m1, [rsp+gprsize*2+16*65] ;in31 pmulhrsw m3, m0, [o(pw_4095x8)] ;t63a pmulhrsw m0, [o(pw_101x8)] ;t32a pmulhrsw m2, m1, [o(pw_2967x8)] ;t62a @@ -5749,27 +5750,27 @@ mova [rsp+gprsize*2+16*51], m3 ;out48 mova [rsp+gprsize*2+16*18], m1 ;out15 ret -cglobal inv_txfm_add_dct_dct_64x16, 4, 6, 8, 16*132, dst, stride, coeff, eob, tx2 +cglobal inv_txfm_add_dct_dct_64x16_8bpc, 4, 6, 8, 16*132, dst, stride, coeff, eob, tx2 %if ARCH_X86_32 LEA r5, $$ %endif test eobd, eobd jz .dconly - call m(idct_64x16_internal) + call m(idct_64x16_internal_8bpc) RET .dconly: movd m1, [o(pw_2896x8)] pmulhrsw m0, m1, [coeffq] movd m2, [o(pw_8192)] mov [coeffq], eobd mov r3d, 16 - lea tx2q, [o(m(inv_txfm_add_dct_dct_64x16).end)] + lea tx2q, [o(m(inv_txfm_add_dct_dct_64x16_8bpc).end)] .body: pmulhrsw m0, m2 movd m2, [o(pw_2048)] ;intentionally rip-relative pmulhrsw m0, m1 @@ -5837,27 +5838,27 @@ mova m5, [%1+%2*1] mova m6, [%1+%2*2] mova m7, [%1+%2*3] %endmacro -cglobal idct_64x16_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal idct_64x16_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 mov r3d, 2 mov [rsp+gprsize*2+16*67], dstq lea dstq, [rsp+gprsize+16*68] .pass1_loop: LOAD_4ROWS coeffq+32*0, 32*8 pxor m4, m4 REPX {mova x, m4}, m5, m6, m7 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main SAVE_7ROWS rsp+gprsize+16*3, 16 pxor m4, m4 LOAD_4ROWS coeffq+32*4, 32*8 REPX {mova x, m4}, m5, m6, m7 - call m(idct_16x8_internal).main + call m(idct_16x8_internal_8bpc).main mova m7, [rsp+gprsize+16*0] SAVE_8ROWS rsp+gprsize+16*11, 16 LOAD_8ROWS coeffq+32*2, 32*4 mova [rsp+gprsize+16*19], m0 @@ -5867,11 +5868,11 @@ mova [rsp+gprsize+16*21], m4 mova [rsp+gprsize+16*24], m5 mova [rsp+gprsize+16*25], m6 mova [rsp+gprsize+16*20], m7 - call m(idct_8x32_internal).main_fast + call m(idct_8x32_internal_8bpc).main_fast SAVE_8ROWS rsp+gprsize+16*3, 16 LOAD_8ROWS coeffq+32*1, 32*2 mova [rsp+gprsize+16*35], m0 ;in1 mova [rsp+gprsize+16*49], m1 ;in3 @@ -5890,73 +5891,73 @@ mova [rsp+gprsize+16*59], m4 ;in25 mova [rsp+gprsize+16*57], m5 ;in27 mova [rsp+gprsize+16*51], m6 ;in29 mova [rsp+gprsize+16*65], m7 ;in31 - call m(idct_16x64_internal).main + call m(idct_16x64_internal_8bpc).main LOAD_8ROWS rsp+gprsize+16*3, 16 mova [rsp+gprsize+16*0], m7 mova m7, [o(pw_8192)] - lea tx2q, [o(m(idct_64x16_internal).pass1_end)] - jmp m(idct_8x8_internal).pass1_end1 + lea tx2q, [o(m(idct_64x16_internal_8bpc).pass1_end)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 .pass1_end: SAVE_8ROWS coeffq+32*0, 32 LOAD_8ROWS rsp+gprsize+16*11, 16 mova [rsp+gprsize+16*0], m7 mova m7, [o(pw_8192)] - lea tx2q, [o(m(idct_64x16_internal).pass1_end1)] - jmp m(idct_8x8_internal).pass1_end1 + lea tx2q, [o(m(idct_64x16_internal_8bpc).pass1_end1)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 .pass1_end1: SAVE_8ROWS coeffq+32*8, 32 LOAD_8ROWS rsp+gprsize+16*19, 16 mova [rsp+gprsize+16*0], m7 mova m7, [o(pw_8192)] - lea tx2q, [o(m(idct_64x16_internal).pass1_end2)] - jmp m(idct_8x8_internal).pass1_end1 + lea tx2q, [o(m(idct_64x16_internal_8bpc).pass1_end2)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 .pass1_end2: SAVE_8ROWS coeffq+32*16, 32 LOAD_8ROWS rsp+gprsize+16*27, 16 mova [rsp+gprsize+16*0], m7 mova m7, [o(pw_8192)] - lea tx2q, [o(m(idct_64x16_internal).pass1_end3)] - jmp m(idct_8x8_internal).pass1_end1 + lea tx2q, [o(m(idct_64x16_internal_8bpc).pass1_end3)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 .pass1_end3: SAVE_8ROWS coeffq+32*24, 32 LOAD_8ROWS rsp+gprsize+16*35, 16 mova [rsp+gprsize+16*0], m7 mova m7, [o(pw_8192)] - lea tx2q, [o(m(idct_64x16_internal).pass1_end4)] - jmp m(idct_8x8_internal).pass1_end1 + lea tx2q, [o(m(idct_64x16_internal_8bpc).pass1_end4)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 .pass1_end4: SAVE_8ROWS dstq+32*0, 32 LOAD_8ROWS rsp+gprsize+16*43, 16 mova [rsp+gprsize+16*0], m7 mova m7, [o(pw_8192)] - lea tx2q, [o(m(idct_64x16_internal).pass1_end5)] - jmp m(idct_8x8_internal).pass1_end1 + lea tx2q, [o(m(idct_64x16_internal_8bpc).pass1_end5)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 .pass1_end5: SAVE_8ROWS dstq+32*8, 32 LOAD_8ROWS rsp+gprsize+16*51, 16 mova [rsp+gprsize+16*0], m7 mova m7, [o(pw_8192)] - lea tx2q, [o(m(idct_64x16_internal).pass1_end6)] - jmp m(idct_8x8_internal).pass1_end1 + lea tx2q, [o(m(idct_64x16_internal_8bpc).pass1_end6)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 .pass1_end6: SAVE_8ROWS dstq+32*16, 32 LOAD_8ROWS rsp+gprsize+16*59, 16 mova [rsp+gprsize+16*0], m7 mova m7, [o(pw_8192)] - lea tx2q, [o(m(idct_64x16_internal).pass1_end7)] - jmp m(idct_8x8_internal).pass1_end1 + lea tx2q, [o(m(idct_64x16_internal_8bpc).pass1_end7)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 .pass1_end7: SAVE_8ROWS dstq+32*24, 32 add coeffq, 16 @@ -5972,27 +5973,27 @@ .pass2_loop: mov [rsp+gprsize*1+16*67], r3d LOAD_4ROWS coeffq+16*0, 32*2 LOAD_4ROWS_H coeffq+16*1, 32*2 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main SAVE_7ROWS rsp+gprsize+16*3, 16 LOAD_4ROWS coeffq+16*2, 32*2 LOAD_4ROWS_H coeffq+16*3, 32*2 - call m(idct_16x8_internal).main + call m(idct_16x8_internal_8bpc).main mov r3, dstq - lea tx2q, [o(m(idct_64x16_internal).end)] + lea tx2q, [o(m(idct_64x16_internal_8bpc).end)] lea dstq, [dstq+strideq*8] - jmp m(idct_8x8_internal).end + jmp m(idct_8x8_internal_8bpc).end .end: LOAD_8ROWS rsp+gprsize+16*3, 16 mova [rsp+gprsize+16*0], m7 - lea tx2q, [o(m(idct_64x16_internal).end1)] + lea tx2q, [o(m(idct_64x16_internal_8bpc).end1)] mov dstq, r3 - jmp m(idct_8x8_internal).end + jmp m(idct_8x8_internal_8bpc).end .end1: pxor m7, m7 REPX {mova [coeffq+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 @@ -6009,27 +6010,27 @@ .pass2_loop2: mov [rsp+gprsize*1+16*67], r3d LOAD_4ROWS coeffq+16*0, 32*2 LOAD_4ROWS_H coeffq+16*1, 32*2 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main SAVE_7ROWS rsp+gprsize+16*3, 16 LOAD_4ROWS coeffq+16*2, 32*2 LOAD_4ROWS_H coeffq+16*3, 32*2 - call m(idct_16x8_internal).main + call m(idct_16x8_internal_8bpc).main mov r3, dstq - lea tx2q, [o(m(idct_64x16_internal).end2)] + lea tx2q, [o(m(idct_64x16_internal_8bpc).end2)] lea dstq, [dstq+strideq*8] - jmp m(idct_8x8_internal).end + jmp m(idct_8x8_internal_8bpc).end .end2: LOAD_8ROWS rsp+gprsize+16*3, 16 mova [rsp+gprsize+16*0], m7 - lea tx2q, [o(m(idct_64x16_internal).end3)] + lea tx2q, [o(m(idct_64x16_internal_8bpc).end3)] mov dstq, r3 - jmp m(idct_8x8_internal).end + jmp m(idct_8x8_internal_8bpc).end .end3: add coeffq, 16*16 mov r3d, [rsp+gprsize*1+16*67] @@ -6039,35 +6040,35 @@ dec r3d jg .pass2_loop2 ret -cglobal inv_txfm_add_dct_dct_32x64, 4, 6, 8, 16*68, dst, stride, coeff, eob, tx2 +cglobal inv_txfm_add_dct_dct_32x64_8bpc, 4, 6, 8, 16*68, dst, stride, coeff, eob, tx2 %if ARCH_X86_32 LEA r5, $$ %endif test eobd, eobd jz .dconly - call m(idct_32x64_internal) + call m(idct_32x64_internal_8bpc) RET .dconly: movd m1, [o(pw_2896x8)] pmulhrsw m0, m1, [coeffq] movd m2, [o(pw_16384)] mov [coeffq], eobd pmulhrsw m0, m1 mov r3d, 64 - lea tx2q, [o(m(inv_txfm_add_dct_dct_32x64).end)] - jmp m(inv_txfm_add_dct_dct_32x8).body + lea tx2q, [o(m(inv_txfm_add_dct_dct_32x64_8bpc).end)] + jmp m(inv_txfm_add_dct_dct_32x8_8bpc).body .end: RET -cglobal idct_32x64_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal idct_32x64_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 %undef cmp mov r4d, 2 sub eobd, 136 mov [rsp+gprsize*1+16*67], eobd @@ -6095,14 +6096,14 @@ test tx2d, tx2d jl .fast .full: LOAD_8ROWS coeffq+64*0, 64*4, 1 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main SAVE_7ROWS rsp+gprsize+16*3, 16 LOAD_8ROWS coeffq+64*2, 64*4, 1 - call m(idct_16x8_internal).main + call m(idct_16x8_internal_8bpc).main mova m7, [rsp+gprsize+16*0] SAVE_8ROWS rsp+gprsize+16*11, 16 LOAD_8ROWS coeffq+64*17, 64*2, 1 mova [rsp+gprsize+16*33], m0 ;in17 @@ -6112,54 +6113,54 @@ mova [rsp+gprsize+16*31], m4 ;in25 mova [rsp+gprsize+16*30], m5 ;in27 mova [rsp+gprsize+16*27], m6 ;in29 mova [rsp+gprsize+16*34], m7 ;in31 - call m(idct_8x32_internal).main + call m(idct_8x32_internal_8bpc).main jmp .pass1_end .fast: LOAD_4ROWS coeffq, 256, 1 pxor m4, m4 REPX {mova x, m4}, m5, m6, m7 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main SAVE_7ROWS rsp+gprsize+16*3, 16 LOAD_4ROWS coeffq+128*1, 256, 1 pxor m4, m4 REPX {mova x, m4}, m5, m6, m7 - call m(idct_16x8_internal).main + call m(idct_16x8_internal_8bpc).main mova m7, [rsp+gprsize+16*0] SAVE_8ROWS rsp+gprsize+16*11, 16 - call m(idct_8x32_internal).main_fast + call m(idct_8x32_internal_8bpc).main_fast .pass1_end: mova [rsp+gprsize+16*0], m7 - lea tx2q, [o(m(idct_32x64_internal).pass1_end1)] - jmp m(idct_8x8_internal).pass1_end + lea tx2q, [o(m(idct_32x64_internal_8bpc).pass1_end1)] + jmp m(idct_8x8_internal_8bpc).pass1_end .pass1_end1: SAVE_8ROWS coeffq+64*0, 64 LOAD_8ROWS rsp+gprsize+16*11, 16 mova [rsp+gprsize+16*0], m7 - lea tx2q, [o(m(idct_32x64_internal).pass1_end2)] - jmp m(idct_8x8_internal).pass1_end + lea tx2q, [o(m(idct_32x64_internal_8bpc).pass1_end2)] + jmp m(idct_8x8_internal_8bpc).pass1_end .pass1_end2: SAVE_8ROWS coeffq+64*8, 64 LOAD_8ROWS rsp+gprsize+16*19, 16 mova [rsp+gprsize+16*0], m7 - lea tx2q, [o(m(idct_32x64_internal).pass1_end3)] - jmp m(idct_8x8_internal).pass1_end + lea tx2q, [o(m(idct_32x64_internal_8bpc).pass1_end3)] + jmp m(idct_8x8_internal_8bpc).pass1_end .pass1_end3: SAVE_8ROWS coeffq+64*16, 64 LOAD_8ROWS rsp+gprsize+16*27, 16 mova [rsp+gprsize+16*0], m7 - lea tx2q, [o(m(idct_32x64_internal).pass1_end4)] - jmp m(idct_8x8_internal).pass1_end + lea tx2q, [o(m(idct_32x64_internal_8bpc).pass1_end4)] + jmp m(idct_8x8_internal_8bpc).pass1_end .pass1_end4: SAVE_8ROWS coeffq+64*24, 64 add coeffq, 16 @@ -6169,38 +6170,38 @@ .pass2: mov coeffq, [rsp+gprsize*2+16*67] mov r3d, 4 lea r4, [dstq+8] mov [rsp+gprsize*2+16*67], r4 - lea r4, [o(m(idct_16x64_internal).end1)] - jmp m(idct_16x64_internal).pass2_loop - - -cglobal inv_txfm_add_dct_dct_64x32, 4, 6, 8, 16*197, dst, stride, coeff, eob, tx2 + lea r4, [o(m(idct_16x64_internal_8bpc).end1)] + jmp m(idct_16x64_internal_8bpc).pass2_loop + + +cglobal inv_txfm_add_dct_dct_64x32_8bpc, 4, 6, 8, 16*197, dst, stride, coeff, eob, tx2 %if ARCH_X86_32 LEA r5, $$ %endif test eobd, eobd jz .dconly - call m(idct_64x32_internal) + call m(idct_64x32_internal_8bpc) RET .dconly: movd m1, [o(pw_2896x8)] pmulhrsw m0, m1, [coeffq] movd m2, [o(pw_16384)] pmulhrsw m0, m1 mov [coeffq], eobd mov r3d, 32 - lea tx2q, [o(m(inv_txfm_add_dct_dct_64x32).end)] - jmp m(inv_txfm_add_dct_dct_64x16).body + lea tx2q, [o(m(inv_txfm_add_dct_dct_64x32_8bpc).end)] + jmp m(inv_txfm_add_dct_dct_64x16_8bpc).body .end: RET -cglobal idct_64x32_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 +cglobal idct_64x32_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 %undef cmp mov r4d, 2 sub eobd, 136 mov [rsp+gprsize*1+16*67], eobd @@ -6218,18 +6219,18 @@ .pass1_loop: LOAD_4ROWS coeffq+64*0, 64*8, 1 pxor m4, m4 REPX {mova x, m4}, m5, m6, m7 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main SAVE_7ROWS rsp+gprsize+16*3, 16 pxor m4, m4 LOAD_4ROWS coeffq+64*4, 64*8, 1 REPX {mova x, m4}, m5, m6, m7 - call m(idct_16x8_internal).main + call m(idct_16x8_internal_8bpc).main mova m7, [rsp+gprsize+16*0] SAVE_8ROWS rsp+gprsize+16*11, 16 LOAD_8ROWS coeffq+64*2, 64*4, 1 mova [rsp+gprsize+16*19], m0 @@ -6239,11 +6240,11 @@ mova [rsp+gprsize+16*21], m4 mova [rsp+gprsize+16*24], m5 mova [rsp+gprsize+16*25], m6 mova [rsp+gprsize+16*20], m7 - call m(idct_8x32_internal).main_fast + call m(idct_8x32_internal_8bpc).main_fast SAVE_8ROWS rsp+gprsize+16*3, 16 LOAD_8ROWS coeffq+64*1, 64*2, 1 mova [rsp+gprsize+16*35], m0 ;in1 mova [rsp+gprsize+16*49], m1 ;in3 @@ -6262,65 +6263,65 @@ mova [rsp+gprsize+16*59], m4 ;in25 mova [rsp+gprsize+16*57], m5 ;in27 mova [rsp+gprsize+16*51], m6 ;in29 mova [rsp+gprsize+16*65], m7 ;in31 - call m(idct_16x64_internal).main + call m(idct_16x64_internal_8bpc).main LOAD_8ROWS rsp+gprsize+16*3, 16 mova [rsp+gprsize+16*0], m7 - lea tx2q, [o(m(idct_64x32_internal).pass1_end)] - jmp m(idct_8x8_internal).pass1_end + lea tx2q, [o(m(idct_64x32_internal_8bpc).pass1_end)] + jmp m(idct_8x8_internal_8bpc).pass1_end .pass1_end: SAVE_8ROWS coeffq+64*0, 64 LOAD_8ROWS rsp+gprsize+16*11, 16 mova [rsp+gprsize+16*0], m7 - lea tx2q, [o(m(idct_64x32_internal).pass1_end1)] - jmp m(idct_8x8_internal).pass1_end + lea tx2q, [o(m(idct_64x32_internal_8bpc).pass1_end1)] + jmp m(idct_8x8_internal_8bpc).pass1_end .pass1_end1: SAVE_8ROWS coeffq+64*8, 64 LOAD_8ROWS rsp+gprsize+16*19, 16 mova [rsp+gprsize+16*0], m7 - lea tx2q, [o(m(idct_64x32_internal).pass1_end2)] - jmp m(idct_8x8_internal).pass1_end + lea tx2q, [o(m(idct_64x32_internal_8bpc).pass1_end2)] + jmp m(idct_8x8_internal_8bpc).pass1_end .pass1_end2: SAVE_8ROWS coeffq+64*16, 64 LOAD_8ROWS rsp+gprsize+16*27, 16 mova [rsp+gprsize+16*0], m7 - lea tx2q, [o(m(idct_64x32_internal).pass1_end3)] - jmp m(idct_8x8_internal).pass1_end + lea tx2q, [o(m(idct_64x32_internal_8bpc).pass1_end3)] + jmp m(idct_8x8_internal_8bpc).pass1_end .pass1_end3: SAVE_8ROWS coeffq+64*24, 64 LOAD_8ROWS rsp+gprsize+16*35, 16 mova [rsp+gprsize+16*0], m7 - lea tx2q, [o(m(idct_64x32_internal).pass1_end4)] - jmp m(idct_8x8_internal).pass1_end + lea tx2q, [o(m(idct_64x32_internal_8bpc).pass1_end4)] + jmp m(idct_8x8_internal_8bpc).pass1_end .pass1_end4: SAVE_8ROWS dstq+64*0, 64 LOAD_8ROWS rsp+gprsize+16*43, 16 mova [rsp+gprsize+16*0], m7 - lea tx2q, [o(m(idct_64x32_internal).pass1_end5)] - jmp m(idct_8x8_internal).pass1_end + lea tx2q, [o(m(idct_64x32_internal_8bpc).pass1_end5)] + jmp m(idct_8x8_internal_8bpc).pass1_end .pass1_end5: SAVE_8ROWS dstq+64*8, 64 LOAD_8ROWS rsp+gprsize+16*51, 16 mova [rsp+gprsize+16*0], m7 - lea tx2q, [o(m(idct_64x32_internal).pass1_end6)] - jmp m(idct_8x8_internal).pass1_end + lea tx2q, [o(m(idct_64x32_internal_8bpc).pass1_end6)] + jmp m(idct_8x8_internal_8bpc).pass1_end .pass1_end6: SAVE_8ROWS dstq+64*16, 64 LOAD_8ROWS rsp+gprsize+16*59, 16 mova [rsp+gprsize+16*0], m7 - lea tx2q, [o(m(idct_64x32_internal).pass1_end7)] - jmp m(idct_8x8_internal).pass1_end + lea tx2q, [o(m(idct_64x32_internal_8bpc).pass1_end7)] + jmp m(idct_8x8_internal_8bpc).pass1_end .pass1_end7: SAVE_8ROWS dstq+64*24, 64 add coeffq, 16 @@ -6332,55 +6333,55 @@ mov coeffq, [rsp+gprsize*4+16*67] mov dstq, [rsp+gprsize*3+16*67] mov eobd, [rsp+gprsize*1+16*67] lea dstq, [dstq+32] mov [rsp+gprsize*1+16*35], eobd - lea tx2q, [o(m(idct_64x32_internal).pass2_end)] + lea tx2q, [o(m(idct_64x32_internal_8bpc).pass2_end)] mov r3d, 4 - jmp m(idct_32x32_internal).pass2_loop + jmp m(idct_32x32_internal_8bpc).pass2_loop .pass2_end: mova [rsp+gprsize+16*0], m7 - lea r3, [o(m(idct_64x32_internal).pass2_end1)] - jmp m(idct_8x32_internal).end2 + lea r3, [o(m(idct_64x32_internal_8bpc).pass2_end1)] + jmp m(idct_8x32_internal_8bpc).end2 .pass2_end1: - lea tx2q, [o(m(idct_64x32_internal).pass2_end)] + lea tx2q, [o(m(idct_64x32_internal_8bpc).pass2_end)] add coeffq, 16*32 mov dstq, [rsp+gprsize*2+16*35] mov r3d, [rsp+gprsize*3+16*35] dec r3d - jg m(idct_32x32_internal).pass2_loop + jg m(idct_32x32_internal_8bpc).pass2_loop .pass2_end2: mov dstq, [rsp+gprsize*3+16*67] mov coeffq, [rsp+gprsize*2+16*67] - lea tx2q, [o(m(idct_32x32_internal).pass2_end)] + lea tx2q, [o(m(idct_32x32_internal_8bpc).pass2_end)] mov r3d, 4 - jmp m(idct_32x32_internal).pass2_loop - - -cglobal inv_txfm_add_dct_dct_64x64, 4, 6, 8, 16*197, dst, stride, coeff, eob, tx2 + jmp m(idct_32x32_internal_8bpc).pass2_loop + + +cglobal inv_txfm_add_dct_dct_64x64_8bpc, 4, 6, 8, 16*197, dst, stride, coeff, eob, tx2 %if ARCH_X86_32 LEA r5, $$ %endif test eobd, eobd jz .dconly - call m(idct_64x64_internal) + call m(idct_64x64_internal_8bpc) RET .dconly: movd m1, [o(pw_2896x8)] pmulhrsw m0, m1, [coeffq] movd m2, [o(pw_8192)] mov [coeffq], eobd mov r3d, 64 - lea tx2q, [o(m(inv_txfm_add_dct_dct_64x32).end)] - jmp m(inv_txfm_add_dct_dct_64x16).body - -cglobal idct_64x64_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 + lea tx2q, [o(m(inv_txfm_add_dct_dct_64x32_8bpc).end)] + jmp m(inv_txfm_add_dct_dct_64x16_8bpc).body + +cglobal idct_64x64_internal_8bpc, 0, 0, 0, dst, stride, coeff, eob, tx2 %undef cmp mov r5d, 4 mov r4d, 2 sub eobd, 136 @@ -6399,18 +6400,18 @@ .pass1_loop: LOAD_4ROWS coeffq+64*0, 64*8 pxor m4, m4 REPX {mova x, m4}, m5, m6, m7 - call m(idct_8x8_internal).main + call m(idct_8x8_internal_8bpc).main SAVE_7ROWS rsp+gprsize+16*3, 16 pxor m4, m4 LOAD_4ROWS coeffq+64*4, 64*8 REPX {mova x, m4}, m5, m6, m7 - call m(idct_16x8_internal).main + call m(idct_16x8_internal_8bpc).main mova m7, [rsp+gprsize+16*0] SAVE_8ROWS rsp+gprsize+16*11, 16 LOAD_8ROWS coeffq+64*2, 64*4 mova [rsp+gprsize+16*19], m0 @@ -6420,11 +6421,11 @@ mova [rsp+gprsize+16*21], m4 mova [rsp+gprsize+16*24], m5 mova [rsp+gprsize+16*25], m6 mova [rsp+gprsize+16*20], m7 - call m(idct_8x32_internal).main_fast + call m(idct_8x32_internal_8bpc).main_fast SAVE_8ROWS rsp+gprsize+16*3, 16 LOAD_8ROWS coeffq+64*1, 64*2 mova [rsp+gprsize+16*35], m0 ;in1 mova [rsp+gprsize+16*49], m1 ;in3 @@ -6443,73 +6444,73 @@ mova [rsp+gprsize+16*59], m4 ;in25 mova [rsp+gprsize+16*57], m5 ;in27 mova [rsp+gprsize+16*51], m6 ;in29 mova [rsp+gprsize+16*65], m7 ;in31 - call m(idct_16x64_internal).main + call m(idct_16x64_internal_8bpc).main LOAD_8ROWS rsp+gprsize+16*3, 16 mova [rsp+gprsize+16*0], m7 mova m7, [o(pw_8192)] - lea tx2q, [o(m(idct_64x64_internal).pass1_end)] - jmp m(idct_8x8_internal).pass1_end1 + lea tx2q, [o(m(idct_64x64_internal_8bpc).pass1_end)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 .pass1_end: SAVE_8ROWS coeffq+64*0, 64 LOAD_8ROWS rsp+gprsize+16*11, 16 mova [rsp+gprsize+16*0], m7 mova m7, [o(pw_8192)] - lea tx2q, [o(m(idct_64x64_internal).pass1_end1)] - jmp m(idct_8x8_internal).pass1_end1 + lea tx2q, [o(m(idct_64x64_internal_8bpc).pass1_end1)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 .pass1_end1: SAVE_8ROWS coeffq+64*8, 64 LOAD_8ROWS rsp+gprsize+16*19, 16 mova [rsp+gprsize+16*0], m7 mova m7, [o(pw_8192)] - lea tx2q, [o(m(idct_64x64_internal).pass1_end2)] - jmp m(idct_8x8_internal).pass1_end1 + lea tx2q, [o(m(idct_64x64_internal_8bpc).pass1_end2)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 .pass1_end2: SAVE_8ROWS coeffq+64*16, 64 LOAD_8ROWS rsp+gprsize+16*27, 16 mova [rsp+gprsize+16*0], m7 mova m7, [o(pw_8192)] - lea tx2q, [o(m(idct_64x64_internal).pass1_end3)] - jmp m(idct_8x8_internal).pass1_end1 + lea tx2q, [o(m(idct_64x64_internal_8bpc).pass1_end3)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 .pass1_end3: SAVE_8ROWS coeffq+64*24, 64 LOAD_8ROWS rsp+gprsize+16*35, 16 mova [rsp+gprsize+16*0], m7 mova m7, [o(pw_8192)] - lea tx2q, [o(m(idct_64x64_internal).pass1_end4)] - jmp m(idct_8x8_internal).pass1_end1 + lea tx2q, [o(m(idct_64x64_internal_8bpc).pass1_end4)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 .pass1_end4: SAVE_8ROWS dstq+64*0, 64 LOAD_8ROWS rsp+gprsize+16*43, 16 mova [rsp+gprsize+16*0], m7 mova m7, [o(pw_8192)] - lea tx2q, [o(m(idct_64x64_internal).pass1_end5)] - jmp m(idct_8x8_internal).pass1_end1 + lea tx2q, [o(m(idct_64x64_internal_8bpc).pass1_end5)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 .pass1_end5: SAVE_8ROWS dstq+64*8, 64 LOAD_8ROWS rsp+gprsize+16*51, 16 mova [rsp+gprsize+16*0], m7 mova m7, [o(pw_8192)] - lea tx2q, [o(m(idct_64x64_internal).pass1_end6)] - jmp m(idct_8x8_internal).pass1_end1 + lea tx2q, [o(m(idct_64x64_internal_8bpc).pass1_end6)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 .pass1_end6: SAVE_8ROWS dstq+64*16, 64 LOAD_8ROWS rsp+gprsize+16*59, 16 mova [rsp+gprsize+16*0], m7 mova m7, [o(pw_8192)] - lea tx2q, [o(m(idct_64x64_internal).pass1_end7)] - jmp m(idct_8x8_internal).pass1_end1 + lea tx2q, [o(m(idct_64x64_internal_8bpc).pass1_end7)] + jmp m(idct_8x8_internal_8bpc).pass1_end1 .pass1_end7: SAVE_8ROWS dstq+64*24, 64 add coeffq, 16 @@ -6522,38 +6523,38 @@ mov coeffq, [rsp+gprsize*2+16*67] lea dstq, [dstq+32] mov r3d, 4 lea r4, [dstq+8] mov [rsp+gprsize*2+16*67], r4 - lea r4, [o(m(idct_64x64_internal).pass2_end)] - jmp m(idct_16x64_internal).pass2_loop + lea r4, [o(m(idct_64x64_internal_8bpc).pass2_end)] + jmp m(idct_16x64_internal_8bpc).pass2_loop .pass2_end: LOAD_8ROWS rsp+gprsize+16*35, 16 lea dstq, [dstq+strideq*2] add rsp, 16*32 mova [rsp+gprsize+16*0], m7 - lea r3, [o(m(idct_64x64_internal).pass2_end1)] - jmp m(idct_8x32_internal).end2 + lea r3, [o(m(idct_64x64_internal_8bpc).pass2_end1)] + jmp m(idct_8x32_internal_8bpc).end2 .pass2_end1: add coeffq, 16*32 sub rsp, 16*32 mov dstq, [rsp+gprsize*2+16*67] mov r3d, [rsp+gprsize*3+16*67] lea r4, [dstq+8] mov [rsp+gprsize*2+16*67], r4 - lea r4, [o(m(idct_64x64_internal).pass2_end)] + lea r4, [o(m(idct_64x64_internal_8bpc).pass2_end)] dec r3d - jg m(idct_16x64_internal).pass2_loop + jg m(idct_16x64_internal_8bpc).pass2_loop .pass2_end2: mov coeffq, [rsp+gprsize*4+16*67] mov dstq, [rsp+gprsize*2+16*67] mov r3d, 4 sub dstq, 72 lea r4, [dstq+8] mov [rsp+gprsize*2+16*67], r4 - lea r4, [o(m(idct_16x64_internal).end1)] - jmp m(idct_16x64_internal).pass2_loop + lea r4, [o(m(idct_16x64_internal_8bpc).end1)] + jmp m(idct_16x64_internal_8bpc).pass2_loop diff --git a/third_party/dav1d/src/x86/loopfilter16_avx2.asm b/third_party/dav1d/src/x86/loopfilter16_avx2.asm --- a/third_party/dav1d/src/x86/loopfilter16_avx2.asm +++ b/third_party/dav1d/src/x86/loopfilter16_avx2.asm @@ -621,13 +621,11 @@ paddw m8, m3 ; p6*7+p3+p1 paddw m10, m4 ; (p5+p4)*2+p2+p0 paddw m8, m5 ; p6*7+p3+p1+q0 paddw m8, m10 ; p6*7+p5*2+p4*2+p3+p2+p1+p0+q0 psrlw m10, m8, 4 - pand m10, m1 - pandn m9, m1, m2 - por m10, m9 + vpblendvb m10, m2, m10, m1 %ifidn %2, v mova [tmpq+strideq*2], m10 ; p5 %else mova [rsp+8*32], m10 %endif @@ -636,13 +634,11 @@ paddw m8, m11 paddw m10, m0, m0 paddw m8, m6 psubw m8, m10 psrlw m10, m8, 4 - pand m10, m1 - pandn m9, m1, m7 - por m10, m9 + vpblendvb m10, m7, m10, m1 %ifidn %2, v mova [tmpq+stride3q], m10 ; p4 %else mova [rsp+9*32], m10 %endif @@ -651,13 +647,11 @@ psubw m8, m0 paddw m10, m13, m14 psubw m8, m2 paddw m8, m10 psrlw m10, m8, 4 - pand m10, m1 - pandn m9, m1, m11 - por m10, m9 + vpblendvb m10, m11, m10, m1 %ifidn %2, v mova [tmpq+strideq*4], m10 ; p3 lea tmpq, [dstq+strideq*4] %else mova [rsp+5*32], m10 @@ -667,13 +661,11 @@ paddw m8, m3 paddw m10, m0, m7 paddw m8, m15 psubw m8, m10 psrlw m10, m8, 4 - pand m10, m1 - pandn m9, m1, m13 - por m10, m9 + vpblendvb m10, m13, m10, m1 mova [rsp+1*32], m10 ; don't clobber p2/m13 ; sub p6/p3, add p0/q4 paddw m8, m4 paddw m10, m0, m11 @@ -682,13 +674,11 @@ %else paddw m8, [rsp+10*32] %endif psubw m8, m10 psrlw m10, m8, 4 - pand m10, m1 - pandn m9, m1, m3 - por m10, m9 + vpblendvb m10, m3, m10, m1 mova [rsp+2*32], m10 ; don't clobber p1/m3 ; sub p6/p2, add q0/q5 paddw m8, m5 paddw m10, m0, m13 @@ -697,13 +687,11 @@ %else paddw m8, [rsp+11*32] %endif psubw m8, m10 psrlw m10, m8, 4 - pand m10, m1 - pandn m9, m1, m4 - por m10, m9 + vpblendvb m10, m4, m10, m1 mova [rsp+3*32], m10 ; don't clobber p0/m4 ; sub p6/p1, add q1/q6 paddw m8, m6 paddw m10, m0, m3 @@ -713,34 +701,28 @@ mova m0, [rsp+12*32] ; q6 %endif paddw m8, m0 psubw m8, m10 psrlw m10, m8, 4 - pand m10, m1 - pandn m9, m1, m5 - por m10, m9 + vpblendvb m10, m5, m10, m1 mova [rsp+4*32], m10 ; don't clobber q0/m5 ; sub p5/p0, add q2/q6 paddw m8, m14 paddw m10, m2, m4 paddw m8, m0 psubw m8, m10 psrlw m10, m8, 4 - pand m10, m1 - pandn m9, m1, m6 - por m2, m10, m9 ; don't clobber q1/m6 + vpblendvb m2, m6, m10, m1 ; don't clobber q1/m6 ; sub p4/q0, add q3/q6 paddw m8, m15 paddw m10, m7, m5 paddw m8, m0 psubw m8, m10 psrlw m10, m8, 4 - pand m10, m1 - pandn m9, m1, m14 - por m7, m10, m9 ; don't clobber q2/m14 + vpblendvb m7, m14, m10, m1 ; don't clobber q2/m14 ; sub p3/q1, add q4/q6 %ifidn %2, v paddw m8, [tmpq+strideq*0] %else @@ -748,13 +730,11 @@ %endif paddw m10, m11, m6 paddw m8, m0 psubw m8, m10 psrlw m10, m8, 4 - pand m10, m1 - pandn m9, m1, m15 - por m10, m9 + vpblendvb m10, m15, m10, m1 %ifidn %2, v mova [tmpq+mstrideq], m10 ; q3 %else mova [rsp+14*32], m10 %endif @@ -767,17 +747,16 @@ %endif paddw m10, m13, m14 paddw m8, m0 psubw m8, m10 psrlw m10, m8, 4 - pand m10, m1 %ifidn %2, v - pandn m9, m1, [tmpq+strideq*0] + mova m9, [tmpq+strideq*0] %else - pandn m9, m1, [rsp+10*32] + mova m9, [rsp+10*32] %endif - por m10, m9 + vpblendvb m10, m9, m10, m1 %ifidn %2, v mova [tmpq+strideq*0], m10 ; q4 %else mova [rsp+10*32], m10 %endif @@ -788,15 +767,15 @@ psubw m8, m15 paddw m8, m0 psrlw m10, m8, 4 pand m10, m1 %ifidn %2, v - pandn m9, m1, [tmpq+strideq*1] + mova m9, [tmpq+strideq*1] %else - pandn m9, m1, [rsp+11*32] + mova m9, [rsp+11*32] %endif - por m10, m9 + vpblendvb m10, m9, m10, m1 %ifidn %2, v mova [tmpq+strideq*1], m10 ; q5 %else mova [rsp+11*32], m10 %endif @@ -857,18 +836,16 @@ psubw m2, m6 paddw m0, m15, m14 paddw m2, m0 pmulhrsw m2, [pw_4096] - REPX {pand x, m9}, m7, m8, m10, m11, m1, m2 - REPX {pandn x, m9, x}, m13, m3, m4, m5, m6, m14 - por m13, m7 - por m3, m8 - por m4, m10 - por m5, m11 - por m6, m1 - por m14, m2 + vpblendvb m13, m13, m7, m9 + vpblendvb m3, m3, m8, m9 + vpblendvb m4, m4, m10, m9 + vpblendvb m5, m5, m11, m9 + vpblendvb m6, m6, m1, m9 + vpblendvb m14, m14, m2, m9 %ifidn %2, v mova [tmpq+strideq*1], m13 ; p2 mova [tmpq+strideq*2], m3 ; p1 mova [tmpq+stride3q ], m4 ; p0 @@ -982,16 +959,14 @@ paddw m14, m14 psubw m8, m4 paddw m8, m14 pmulhrsw m8, [pw_4096] - REPX {pand x, m9}, m2, m10, m11, m8 - REPX {pandn x, m9, x}, m3, m4, m5, m6 - por m3, m2 - por m4, m10 - por m5, m11 - por m6, m8 + vpblendvb m3, m3, m2, m9 + vpblendvb m4, m4, m10, m9 + vpblendvb m5, m5, m11, m9 + vpblendvb m6, m6, m8, m9 %ifidn %2, v mova [tmpq+strideq*2], m3 ; p1 mova [tmpq+stride3q ], m4 ; p0 mova [dstq+strideq*0], m5 ; q0 diff --git a/third_party/dav1d/src/x86/loopfilter16_sse.asm b/third_party/dav1d/src/x86/loopfilter16_sse.asm --- a/third_party/dav1d/src/x86/loopfilter16_sse.asm +++ b/third_party/dav1d/src/x86/loopfilter16_sse.asm @@ -24,13 +24,18 @@ ; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. %include "config.asm" %include "ext/x86/x86inc.asm" -%if ARCH_X86_64 +SECTION_RODATA 16 -SECTION_RODATA 16 +%if ARCH_X86_64 +%define PIC_sym(a) a +%else +%define PIC_base $$ +%define PIC_sym(a) pic_regq+a-PIC_base +%endif pb_4x1_4x5_4x9_4x13: times 4 db 0, 1 times 4 db 8, 9 pw_1: times 8 dw 1 @@ -45,10 +50,64 @@ pb_mask: dd 1, 1, 2, 2 SECTION .text +%if ARCH_X86_32 +%if STACK_ALIGNMENT < 16 +%define extra_stack 2 +%else +%define extra_stack 0 +%endif +%endif + +%macro RELOC_ARGS 2 ; h/v, off +ASSERT ARCH_X86_32 +%if STACK_ALIGNMENT < 16 + mov r5d, [rstk + stack_offset + 4*4 + 4] +%define lstridem [esp+%2+0*gprsize] + mov lstridem, r5d + mov r5d, [rstk + stack_offset + 4*5 + 4] +%define lutm [esp+%2+1*gprsize] + mov lutm, r5d + mov r5d, [rstk + stack_offset + 4*6 + 4] +%ifidn %1, v +%define wm [esp+%2+2*gprsize] + mov wm, r5d + mov r5d, [rstk + stack_offset + 4*3 + 4] +%define lm [esp+%2+3*gprsize] + mov lm, r5d +%else ; %1 == h +%define hm [esp+%2+2*gprsize] + mov hm, r5d +%endif ; %1==v + mov r5d, r7m +%define bdmulm [esp+%2+4*gprsize] + mov bdmulm, r5d +%else +%define lstridem r4m +%define lutm r5m +%ifidn %1, v +%define wm r6m +%define lm r3m +%else +%define hm r6m +%endif +%define bdmulm r7m +%endif ; STACK_ALIGNMENT +%endmacro + +%macro UNRELOC_ARGS 0 +%if ARCH_X86_32 +%undef lm +%undef lstridem +%undef wm +%undef hm +%undef lutm +%endif +%endmacro + %macro REPX 2-* %xdefine %%f(x) %1 %rep %0 - 1 %rotate 1 %%f(%1) @@ -84,39 +143,40 @@ SWAP %1, %4 SWAP %2, %5, %3 %endmacro ; in: out: -; xmm%1 a b c d e f g h a i q y 6 E M U -; xmm%2 i j k l m n o p b j r z 7 F N V -; xmm%3 q r s t u v w x c k s 0 8 G O W -; xmm%4 y z 0 1 2 3 4 5 d l t 1 9 H P X -; xmm%5 6 7 8 9 A B C D -> e m u 2 A I Q Y -; xmm%6 E F G H I J K L f n v 3 B J R Z -; xmm%7 M N O P Q R S T g o w 4 C K S + -; xmm%8 U V W X Y Z + = h p x 5 D L T = +; m%1 a b c d e f g h a i q y 6 E M U +; m%2 i j k l m n o p b j r z 7 F N V +; m%3 q r s t u v w x c k s 0 8 G O W +; m%4 y z 0 1 2 3 4 5 d l t 1 9 H P X +; m%5 6 7 8 9 A B C D -> e m u 2 A I Q Y +; m%6 E F G H I J K L f n v 3 B J R Z +; m%7 M N O P Q R S T g o w 4 C K S + +; m%8 U V W X Y Z + = h p x 5 D L T = +%if ARCH_X86_64 %macro TRANSPOSE8X8W 9 - ; xmm%1 a b c d e f g h a i q y b j r z - ; xmm%2 i j k l m n o p c k s 0 d l t 1 - ; xmm%3 q r s t u v w x -> e m u 2 f n v 3 - ; xmm%4 y z 0 1 2 3 4 5 g o w 4 h p x 5 + ; m%1 a b c d e f g h a i q y b j r z + ; m%2 i j k l m n o p c k s 0 d l t 1 + ; m%3 q r s t u v w x -> e m u 2 f n v 3 + ; m%4 y z 0 1 2 3 4 5 g o w 4 h p x 5 TRANSPOSE4X4W %1, %2, %3, %4, %9 - ; xmm%5 6 7 8 9 A B C D 6 E M U 7 F N V - ; xmm%6 E F G H I J K L 8 G O W 9 H P X - ; xmm%7 M N O P Q R S T -> A I Q Y B J R Z - ; xmm%8 U V W X Y Z + = C K S + D L T = + ; m%5 6 7 8 9 A B C D 6 E M U 7 F N V + ; m%6 E F G H I J K L 8 G O W 9 H P X + ; m%7 M N O P Q R S T -> A I Q Y B J R Z + ; m%8 U V W X Y Z + = C K S + D L T = TRANSPOSE4X4W %5, %6, %7, %8, %9 - ; xmm%1 a i q y b j r z a i q y 6 E M U - ; xmm%2 c k s 0 d l t 1 b j r z 7 F N V - ; xmm%3 e m u 2 f n v 3 c k s 0 8 G O W - ; xmm%4 g o w 4 h p x 5 d l t 1 9 H P X - ; xmm%5 6 E M U 7 F N V -> e m u 2 A I Q Y - ; xmm%6 8 G O W 9 H P X f n v 3 B J R Z - ; xmm%7 A I Q Y B J R Z g o w 4 C K S + - ; xmm%8 C K S + D L T = h p x 5 D L T = + ; m%1 a i q y b j r z a i q y 6 E M U + ; m%2 c k s 0 d l t 1 b j r z 7 F N V + ; m%3 e m u 2 f n v 3 c k s 0 8 G O W + ; m%4 g o w 4 h p x 5 d l t 1 9 H P X + ; m%5 6 E M U 7 F N V -> e m u 2 A I Q Y + ; m%6 8 G O W 9 H P X f n v 3 B J R Z + ; m%7 A I Q Y B J R Z g o w 4 C K S + + ; m%8 C K S + D L T = h p x 5 D L T = punpckhqdq m%9, m%1, m%5 punpcklqdq m%1, m%5 punpckhqdq m%5, m%2, m%6 punpcklqdq m%2, m%6 punpckhqdq m%6, m%3, m%7 @@ -124,1012 +184,1618 @@ punpckhqdq m%7, m%4, m%8 punpcklqdq m%4, m%8 SWAP %8, %7, %4, %5, %3, %2, %9 %endmacro +%else ; x86-32 +; input: 1-7 in registers, 8 in first memory [read-only] +; second memory is scratch, and may overlap with first or third memory +; output: 1-5,7-8 in registers, 6 in third memory [write-only] +%macro TRANSPOSE8X8W 13 ; regs [8x], mem [3x], a/u [in/out alignment [2x] + TRANSPOSE4X4W %1, %2, %3, %4, %8 +%ifnidn %9, "" + mov%12 m%8, %9 +%else + mova m%8, %10 +%endif + mova %10, m%4 + TRANSPOSE4X4W %5, %6, %7, %8, %4 + punpckhqdq m%4, m%1, m%5 + punpcklqdq m%1, m%5 + punpckhqdq m%5, m%2, m%6 + punpcklqdq m%2, m%6 + punpckhqdq m%6, m%3, m%7 + punpcklqdq m%3, m%7 + mova m%7, %10 +%ifnidn %11, "" + mov%13 %11, m%6 +%else + mova %10, m%6 +%endif + punpckhqdq m%6, m%7, m%8 + punpcklqdq m%7, m%8 -; transpose and write m3-6, everything else is scratch -%macro TRANSPOSE_8x4_AND_WRITE_4x8 0 + ; 1,4,2,5,3,8,7,6 -> 1,2,3,4,5,6,7,8 + SWAP %2, %4, %5, %3 + SWAP %6, %8 +%endmacro +%endif ; x86-32/64 + +; transpose and write m8-11, everything else is scratch +%macro TRANSPOSE_8x4_AND_WRITE_4x8 5 ; p1, p0, q0, q1, tmp ; transpose 8x4 - punpcklwd m0, m3, m4 - punpckhwd m3, m4 - punpcklwd m4, m5, m6 - punpckhwd m5, m6 - punpckldq m6, m0, m4 - punpckhdq m0, m4 - punpckldq m4, m3, m5 - punpckhdq m3, m5 + punpcklwd %5, %1, %2 + punpckhwd %1, %2 + punpcklwd %2, %3, %4 + punpckhwd %3, %4 + punpckldq %4, %5, %2 + punpckhdq %5, %2 + punpckldq %2, %1, %3 + punpckhdq %1, %3 ; write out - movq [dstq+strideq*0-4], xm6 - movhps [dstq+strideq*1-4], xm6 - movq [dstq+strideq*2-4], xm0 - movhps [dstq+stride3q -4], xm0 + movq [dstq+strideq*0-4], %4 + movhps [dstq+strideq*1-4], %4 + movq [dstq+strideq*2-4], %5 + movhps [dstq+stride3q -4], %5 lea dstq, [dstq+strideq*4] - movq [dstq+strideq*0-4], xm4 - movhps [dstq+strideq*1-4], xm4 - movq [dstq+strideq*2-4], xm3 - movhps [dstq+stride3q -4], xm3 + movq [dstq+strideq*0-4], %2 + movhps [dstq+strideq*1-4], %2 + movq [dstq+strideq*2-4], %1 + movhps [dstq+stride3q -4], %1 lea dstq, [dstq+strideq*4] %endmacro %macro FILTER 2 ; width [4/6/8/16], dir [h/v] ; load data %ifidn %2, v %if %1 == 4 - lea tmpq, [dstq+mstrideq*2] - mova m3, [tmpq+strideq*0] ; p1 - mova m4, [tmpq+strideq*1] ; p0 - mova m5, [tmpq+strideq*2] ; q0 - mova m6, [tmpq+stride3q] ; q1 -%else +%if ARCH_X86_64 +%define P1 m8 +%define P0 m9 +%define Q0 m10 +%define Q1 m11 + mova P1, [dstq+mstrideq*2] ; p1 + mova P0, [dstq+mstrideq*1] ; p0 + mova Q0, [dstq+strideq*0] ; q0 + mova Q1, [dstq+strideq*1] ; q1 +%else ; x86-32 +%define P1 [dstq+mstrideq*2] +%define P0 [dstq+mstrideq*1] +%define Q0 [dstq+strideq*0] +%define Q1 [dstq+strideq*1] +%endif ; x86-32/64 +%else ; %1 != 4 ; load 6-8 pixels, remainder (for wd=16) will be read inline lea tmpq, [dstq+mstrideq*4] +%if ARCH_X86_64 ; we load p3 later - mova m13, [tmpq+strideq*1] - mova m3, [tmpq+strideq*2] - mova m4, [tmpq+stride3q] - mova m5, [dstq+strideq*0] - mova m6, [dstq+strideq*1] - mova m14, [dstq+strideq*2] +%define P2 m13 +%define P1 m8 +%define P0 m9 +%define Q0 m10 +%define Q1 m11 +%define Q2 m14 + mova P2, [tmpq+strideq*1] + mova P1, [tmpq+strideq*2] + mova P0, [tmpq+stride3q] + mova Q0, [dstq+strideq*0] + mova Q1, [dstq+strideq*1] + mova Q2, [dstq+strideq*2] %if %1 != 6 - mova m15, [dstq+stride3q] -%endif -%endif -%else +%define P3 [tmpq+strideq*0] +%define Q3 m15 + mova Q3, [dstq+stride3q] +%endif ; %1 != 6 +%else ; x86-32 +%define P2 [tmpq+strideq*1] +%define P1 [dstq+mstrideq*2] +%define P0 [dstq+mstrideq*1] +%define Q0 [dstq+strideq*0] +%define Q1 [dstq+strideq*1] +%define Q2 [dstq+strideq*2] +%if %1 != 6 +%define P3 [dstq+mstrideq*4] +%define Q3 [dstq+stride3q] +%endif ; %1 != 6 +%endif ; x86-32/64 +%endif ; %1 ==/!= 4 +%else ; %2 != v ; load lines %if %1 == 4 - movq xm3, [dstq+strideq*0-4] - movq xm4, [dstq+strideq*1-4] - movq xm5, [dstq+strideq*2-4] - movq xm6, [dstq+stride3q -4] + movq m0, [dstq+strideq*0-4] + movq m2, [dstq+strideq*1-4] + movq m4, [dstq+strideq*2-4] + movq m5, [dstq+stride3q -4] lea tmpq, [dstq+strideq*4] - movq xm11, [tmpq+strideq*0-4] - movq xm13, [tmpq+strideq*1-4] - movq xm14, [tmpq+strideq*2-4] - movq xm15, [tmpq+stride3q -4] + movq m3, [tmpq+strideq*0-4] + movq m6, [tmpq+strideq*1-4] + movq m1, [tmpq+strideq*2-4] + movq m7, [tmpq+stride3q -4] ; transpose 4x8 - ; xm3: A-D0,A-D4 - ; xm4: A-D1,A-D5 - ; xm5: A-D2,A-D6 - ; xm6: A-D3,A-D7 - punpcklwd m7, m3, m4 - punpcklwd m3, m11, m13 - punpcklwd m4, m5, m6 - punpcklwd m5, m14, m15 - ; xm7: A0-1,B0-1,C0-1,D0-1 - ; xm3: A4-5,B4-5,C4-5,D4-5 - ; xm4: A2-3,B2-3,C2-3,D2-3 - ; xm5: A6-7,B6-7,C6-7,D6-7 - punpckldq m6, m7, m4 - punpckhdq m7, m4 - punpckldq m8, m3, m5 - punpckhdq m3, m5 - SWAP 3, 5 - ; xm6: A0-3,B0-3 - ; xm7: C0-3,D0-3 - ; xm8: A4-7,B4-7 - ; xm5: C4-7,D4-7 - punpcklqdq m3, m6, m8 - punpckhqdq m4, m6, m8 - punpckhqdq m6, m7, m5 - punpcklqdq m7, m5 - SWAP 7, 5 - ; xm3: A0-7 - ; xm4: B0-7 - ; xm5: C0-7 - ; xm6: D0-7 + ; m0: A-D0 + ; m2: A-D1 + ; m4: A-D2 + ; m5: A-D3 + ; m3: A-D4 + ; m6: A-D5 + ; m1: A-D6 + ; m7: A-D7 + punpcklwd m0, m2 + punpcklwd m4, m5 + punpcklwd m3, m6 + punpcklwd m1, m7 + ; m0: A0-1,B0-1,C0-1,D0-1 + ; m4: A2-3,B2-3,C2-3,D2-3 + ; m3: A4-5,B4-5,C4-5,D4-5 + ; m1: A6-7,B6-7,C6-7,D6-7 + punpckhdq m2, m0, m4 + punpckldq m0, m4 + punpckhdq m4, m3, m1 + punpckldq m3, m1 + ; m0: A0-3,B0-3 + ; m2: C0-3,D0-3 + ; m3: A4-7,B4-7 + ; m4: C4-7,D4-7 + punpckhqdq m1, m0, m3 + punpcklqdq m0, m3 + punpckhqdq m3, m2, m4 + punpcklqdq m2, m4 + ; m0: A0-7 + ; m1: B0-7 + ; m2: C0-7 + ; m3: D0-7 +%if ARCH_X86_64 + SWAP 0, 8 + SWAP 1, 9 + SWAP 2, 10 + SWAP 3, 11 +%define P1 m8 +%define P0 m9 +%define Q0 m10 +%define Q1 m11 +%else +%define P1 [esp+3*mmsize] +%define P0 [esp+4*mmsize] +%define Q0 [esp+5*mmsize] +%define Q1 [esp+6*mmsize] + mova P1, m0 + mova P0, m1 + mova Q0, m2 + mova Q1, m3 +%endif %elif %1 == 6 || %1 == 8 - movu xm3, [dstq+strideq*0-8] - movu xm4, [dstq+strideq*1-8] - movu xm5, [dstq+strideq*2-8] - movu xm6, [dstq+stride3q -8] + movu m0, [dstq+strideq*0-8] + movu m1, [dstq+strideq*1-8] + movu m2, [dstq+strideq*2-8] + movu m3, [dstq+stride3q -8] lea tmpq, [dstq+strideq*4] - movu xm11, [tmpq+strideq*0-8] - movu xm13, [tmpq+strideq*1-8] - movu xm14, [tmpq+strideq*2-8] - movu xm15, [tmpq+stride3q -8] + movu m4, [tmpq+strideq*0-8] + movu m5, [tmpq+strideq*1-8] + movu m6, [tmpq+strideq*2-8] +%if ARCH_X86_64 + movu m7, [tmpq+stride3q -8] +%endif ; transpose 8x16 - ; xm3: A-H0,A-H8 - ; xm4: A-H1,A-H9 - ; xm5: A-H2,A-H10 - ; xm6: A-H3,A-H11 - ; xm11: A-H4,A-H12 - ; xm13: A-H5,A-H13 - ; xm14: A-H6,A-H14 - ; xm15: A-H7,A-H15 - punpcklwd m7, m3, m4 - punpckhwd m3, m4 - punpcklwd m4, m5, m6 - punpckhwd m5, m6 - punpcklwd m6, m11, m13 - punpckhwd m11, m13 - punpcklwd m13, m14, m15 - punpckhwd m14, m15 - ; xm7: A0-1,B0-1,C0-1,D0-1 - ; xm3: E0-1,F0-1,G0-1,H0-1 - ; xm4: A2-3,B2-3,C2-3,D2-3 - ; xm5: E2-3,F2-3,G2-3,H2-3 - ; xm6: A4-5,B4-5,C4-5,D4-5 - ; xm11: E4-5,F4-5,G4-5,H4-5 - ; xm13: A6-7,B6-7,C6-7,D6-7 - ; xm14: E6-7,F6-7,G6-7,H6-7 - punpckldq m15, m7, m4 - punpckhdq m7, m4 - punpckldq m9, m3, m5 - punpckhdq m8, m3, m5 - punpckldq m3, m6, m13 - punpckhdq m6, m13 - punpckldq m10, m11, m14 - punpckhdq m11, m14 - ; xm15: A0-3,B0-3 - ; xm7: C0-3,D0-3 - ; xm9: E0-3,F0-3 - ; xm8: G0-3,H0-3 - ; xm3: A4-7,B4-7 - ; xm6: C4-7,D4-7 - ; xm10: E4-7,F4-7 - ; xm11: G4-7,H4-7 + ; m0: A-H0,A-H8 + ; m1: A-H1,A-H9 + ; m2: A-H2,A-H10 + ; m3: A-H3,A-H11 + ; m4: A-H4,A-H12 + ; m5: A-H5,A-H13 + ; m6: A-H6,A-H14 + ; m7: A-H7,A-H15 +%if ARCH_X86_64 + punpcklwd m8, m0, m1 +%else + punpcklwd m7, m0, m1 +%endif + punpckhwd m0, m1 + punpcklwd m1, m2, m3 + punpckhwd m2, m3 + punpcklwd m3, m4, m5 + punpckhwd m4, m5 +%if ARCH_X86_64 + punpcklwd m5, m6, m7 + punpckhwd m6, m7 +%else + mova [rsp+3*16], m4 + movu m4, [tmpq+stride3q -8] + punpcklwd m5, m6, m4 + punpckhwd m6, m4 +%endif + ; m8: A0-1,B0-1,C0-1,D0-1 [m7 on x86-32] + ; m0: E0-1,F0-1,G0-1,H0-1 + ; m1: A2-3,B2-3,C2-3,D2-3 + ; m2: E2-3,F2-3,G2-3,H2-3 + ; m3: A4-5,B4-5,C4-5,D4-5 + ; m4: E4-5,F4-5,G4-5,H4-5 [r3 on x86-32] + ; m5: A6-7,B6-7,C6-7,D6-7 + ; m6: E6-7,F6-7,G6-7,H6-7 +%if ARCH_X86_64 + punpckldq m7, m8, m1 + punpckhdq m8, m1 +%else + punpckldq m4, m7, m1 + punpckhdq m7, m1 +%endif + punpckldq m1, m0, m2 + punpckhdq m0, m2 + punpckldq m2, m3, m5 + punpckhdq m3, m5 +%if ARCH_X86_64 + punpckldq m5, m4, m6 + punpckhdq m4, m6 +%else + mova [rsp+4*16], m3 + mova m3, [rsp+3*16] + punpckldq m5, m3, m6 + punpckhdq m3, m6 +%endif + ; m7: A0-3,B0-3 [m4 on x86-32] + ; m8: C0-3,D0-3 [m7 on x86-32] + ; m1: E0-3,F0-3 + ; m0: G0-3,H0-3 + ; m2: A4-7,B4-7 + ; m3: C4-7,D4-7 [r4 on x86-32] + ; m5: E4-7,F4-7 + ; m4: G4-7,H4-7 [m3 on x86-32] +%if ARCH_X86_64 %if %1 != 6 - punpcklqdq m0, m15, m3 + punpcklqdq m6, m7, m2 +%endif + punpckhqdq m7, m2 + punpcklqdq m2, m8, m3 + punpckhqdq m8, m3 + punpcklqdq m3, m1, m5 + punpckhqdq m1, m5 +%if %1 != 6 + punpckhqdq m5, m0, m4 +%endif + punpcklqdq m0, m4 +%if %1 == 8 + mova [rsp+1*16], m6 +%define P3 [rsp+1*16] +%endif + ; 7,2,8,3,1,0,5 -> 13,8,9,10,11,14,15 + SWAP 7, 13 + SWAP 8, 2, 9 + SWAP 3, 10 + SWAP 1, 11 + SWAP 0, 14 + SWAP 5, 15 +%define P2 m13 +%define P1 m8 +%define P0 m9 +%define Q0 m10 +%define Q1 m11 +%define Q2 m14 +%if %1 == 8 +%define Q3 m15 %endif - punpckhqdq m13, m15, m3 - punpcklqdq m3, m7, m6 - punpckhqdq m4, m7, m6 - punpcklqdq m5, m9, m10 - punpckhqdq m6, m9, m10 - punpcklqdq m14, m8, m11 -%if %1 != 6 - punpckhqdq m15, m8, m11 - mova [rsp+5*32], m0 +%else ; x86-32 +%if %1 == 8 +%define P3 [rsp+ 6*16] + punpcklqdq m6, m4, m2 + mova P3, m6 +%endif + mova m6, [rsp+4*16] + punpckhqdq m4, m2 + punpcklqdq m2, m7, m6 + punpckhqdq m7, m6 + punpcklqdq m6, m1, m5 + punpckhqdq m1, m5 +%if %1 == 8 +%define Q3 [rsp+24*16] + punpckhqdq m5, m0, m3 + mova Q3, m5 %endif + punpcklqdq m0, m3 +%if %1 == 8 +%define P2 [rsp+18*16] +%define P1 [rsp+19*16] +%define P0 [rsp+20*16] +%define Q0 [rsp+21*16] +%define Q1 [rsp+22*16] +%define Q2 [rsp+23*16] %else +%define P2 [rsp+3*16] +%define P1 [rsp+4*16] +%define P0 [rsp+5*16] +%define Q0 [rsp+6*16] +%define Q1 [rsp+7*16] +%define Q2 [rsp+8*16] +%endif + mova P2, m4 + mova P1, m2 + mova P0, m7 + mova Q0, m6 + mova Q1, m1 + mova Q2, m0 +%endif ; x86-32/64 +%else ; %1 == 16 ; We only use 14 pixels but we'll need the remainder at the end for ; the second transpose - mova xm0, [dstq+strideq*0-16] - mova xm1, [dstq+strideq*1-16] - mova xm2, [dstq+strideq*2-16] - mova xm3, [dstq+stride3q -16] + mova m0, [dstq+strideq*0-16] + mova m1, [dstq+strideq*1-16] + mova m2, [dstq+strideq*2-16] + mova m3, [dstq+stride3q -16] lea tmpq, [dstq+strideq*4] - mova xm4, [tmpq+strideq*0-16] - mova xm5, [tmpq+strideq*1-16] - mova xm6, [tmpq+strideq*2-16] - mova xm7, [tmpq+stride3q -16] + mova m4, [tmpq+strideq*0-16] + mova m5, [tmpq+strideq*1-16] + mova m6, [tmpq+strideq*2-16] +%if ARCH_X86_64 + mova m7, [tmpq+stride3q -16] TRANSPOSE8X8W 0, 1, 2, 3, 4, 5, 6, 7, 8 + SWAP 5, 13 + SWAP 6, 8 + SWAP 7, 9 +%define P2 m13 +%define P1 m8 +%define P0 m9 +%else ; x86-32 +%define P2 [esp+18*16] +%define P1 [esp+19*16] +%define P0 [esp+20*16] + TRANSPOSE8X8W 0, 1, 2, 3, 4, 5, 6, 7, \ + [tmpq+stride3q -16], P2, "", a, a + mova P1, m6 + mova P0, m7 +%endif ; x86-32/64 + mova [rsp+ 7*16], m0 + mova [rsp+ 8*16], m1 + mova [rsp+ 9*16], m2 + mova [rsp+10*16], m3 +%define P3 [rsp+6*16] + mova P3, m4 - mova [rsp+6*32], m0 - mova [rsp+7*32], m1 - mova [rsp+8*32], m2 - mova [rsp+9*32], m3 - mova [rsp+5*32], m4 - - mova xm0, [dstq+strideq*0] - mova xm1, [dstq+strideq*1] - mova xm2, [dstq+strideq*2] - mova xm3, [dstq+stride3q ] + mova m0, [dstq+strideq*0] + mova m1, [dstq+strideq*1] + mova m2, [dstq+strideq*2] + mova m3, [dstq+stride3q ] lea tmpq, [dstq+strideq*4] - mova xm8, [tmpq+strideq*0] - mova xm9, [tmpq+strideq*1] - mova xm10, [tmpq+strideq*2] - mova xm11, [tmpq+stride3q ] - - TRANSPOSE8X8W 0, 1, 2, 3, 8, 9, 10, 11, 4 + mova m4, [tmpq+strideq*0] + mova m5, [tmpq+strideq*1] + mova m6, [tmpq+strideq*2] +%if ARCH_X86_64 + mova m7, [tmpq+stride3q ] - mova [rsp+10*32], m8 - mova [rsp+11*32], m9 - mova [rsp+12*32], m10 - mova [rsp+13*32], m11 + TRANSPOSE8X8W 0, 1, 2, 3, 4, 5, 6, 7, 10 + SWAP 0, 10 + SWAP 1, 11 + SWAP 2, 14 + SWAP 3, 15 +%define Q0 m10 +%define Q1 m11 +%define Q2 m14 +%define Q3 m15 +%else ; x86-32 + TRANSPOSE8X8W 0, 1, 2, 3, 4, 5, 6, 7, \ + [tmpq+stride3q ], [rsp+12*16], "", a, a +%define Q0 [esp+21*16] +%define Q1 [esp+22*16] +%define Q2 [esp+23*16] +%define Q3 [esp+24*16] + mova Q0, m0 + mova Q1, m1 + mova Q2, m2 + mova Q3, m3 +%endif ; x86-32/64 - ; 5,6,7,0,1,2,3 -> 13,3,4,5,6,14,15 - SWAP 13, 5, 0 - SWAP 3, 6, 1, 15 - SWAP 4, 7 - SWAP 2, 14 + mova [rsp+11*16], m4 +%if ARCH_X86_64 + mova [rsp+12*16], m5 +%endif + mova [rsp+13*16], m6 + mova [rsp+14*16], m7 +%endif ; %1 == 4/6/8/16 +%endif ; %2 ==/!= v + + ; load L/E/I/H +%if ARCH_X86_32 +%define l_strideq r5 + mov l_strideq, dword lstridem +%ifidn %2, v +%define lq r3 + mov lq, dword lm %endif %endif - - ; load L/E/I/H %ifidn %2, v %if cpuflag(sse4) pmovzxbw m1, [lq] pmovzxbw m0, [lq+l_strideq] pxor m2, m2 -%else +%else ; ssse3 movq m1, [lq] movq m0, [lq+l_strideq] pxor m2, m2 REPX {punpcklbw x, m2}, m1, m0 -%endif -%else +%endif ; ssse3/sse4 +%else ; %2 != v movq m0, [lq] ; l0, l1 movq m1, [lq+l_strideq] ; l2, l3 punpckldq m0, m1 ; l0, l2, l1, l3 pxor m2, m2 punpcklbw m1, m0, m2 ; l0, l2 punpckhbw m0, m2 ; l1, l3 +%endif ; %2==/!=v +%if ARCH_X86_32 +%ifidn %2, v +%undef lq + mov mstrideq, mstridem %endif - pcmpeqw m10, m2, m0 - pand m1, m10 +%endif + pcmpeqw m5, m2, m0 + pand m1, m5 por m0, m1 ; l[x][] ? l[x][] : l[x-stride][] - pshufb m0, [pb_4x1_4x5_4x9_4x13] ; l[x][1] - pcmpeqw m10, m2, m0 ; !L - psrlw m10, 1 + pshufb m0, [PIC_sym(pb_4x1_4x5_4x9_4x13)] ; l[x][1] + pcmpeqw m5, m2, m0 ; !L + psrlw m5, 1 +%if ARCH_X86_64 psrlw m2, m0, [lutq+128] SPLATW m1, [lutq+136] +%else ; x86-32 + mov r5, lutm + psrlw m2, m0, [r5+128] + SPLATW m1, [r5+136] +%endif ; x86-32/64 pminsw m2, m1 - pmaxsw m2, [pw_1] ; I + pmaxsw m2, [PIC_sym(pw_1)] ; I psrlw m1, m0, 4 ; H - paddw m0, [pw_2] + paddw m0, [PIC_sym(pw_2)] paddw m0, m0 paddw m0, m2 ; E - REPX {pmullw x, [r11]}, m0, m1, m2 - - psubw m8, m3, m4 ; p1-p0 - psubw m9, m5, m6 ; q1-q0 - REPX {pabsw x, x}, m8, m9 - pmaxsw m8, m10 - pmaxsw m8, m9 - pcmpgtw m7, m8, m1 ; hev -%if %1 != 4 - psubw m9, m13, m4 ; p2-p0 - pabsw m9, m9 - pmaxsw m9, m8 -%if %1 != 6 -%ifidn %2, v - mova m11, [tmpq+strideq*0] ; p3 -%else - mova m11, [rsp+5*32] ; p3 -%endif - psubw m10, m11, m4 ; p3-p0 - pabsw m10, m10 - pmaxsw m9, m10 + REPX {pmullw x, [bdmulq]}, m0, m1, m2 +%if ARCH_X86_32 +%undef l_strideq + lea stride3q, [strideq*3] %endif - psubw m10, m5, m14 ; q2-q0 - pabsw m10, m10 - pmaxsw m9, m10 -%if %1 != 6 - psubw m10, m5, m15 ; q3-q0 - pabsw m10, m10 - pmaxsw m9, m10 -%endif - pcmpgtw m9, [r11] ; !flat8in - psubw m10, m13, m3 ; p2-p1 - pabsw m10, m10 + psubw m3, P1, P0 ; p1-p0 + psubw m4, Q0, Q1 ; q0-q1 + REPX {pabsw x, x}, m3, m4 + pmaxsw m3, m5 + pmaxsw m3, m4 + pcmpgtw m7, m3, m1 ; hev +%if %1 != 4 + psubw m4, P2, P0 ; p2-p0 + pabsw m4, m4 + pmaxsw m4, m3 +%if %1 != 6 + mova m6, P3 ; p3 + psubw m5, m6, P0 ; p3-p0 + pabsw m5, m5 + pmaxsw m4, m5 +%endif ; %1 != 6 + psubw m5, Q0, Q2 ; q0-q2 + pabsw m5, m5 + pmaxsw m4, m5 %if %1 != 6 - psubw m11, m13 ; p3-p2 - pabsw m11, m11 - pmaxsw m10, m11 - psubw m11, m14, m15 ; q3-q2 - pabsw m11, m11 - pmaxsw m10, m11 -%endif - psubw m11, m14, m6 ; q2-q1 - pabsw m11, m11 - pmaxsw m10, m11 + psubw m5, Q0, Q3 ; q0-q3 + pabsw m5, m5 + pmaxsw m4, m5 +%endif ; %1 != 6 + pcmpgtw m4, [bdmulq] ; !flat8in + + psubw m5, P2, P1 ; p2-p1 + pabsw m5, m5 +%if %1 != 6 + psubw m6, P2 ; p3-p2 + pabsw m6, m6 + pmaxsw m5, m6 + psubw m6, Q2, Q3 ; q2-q3 + pabsw m6, m6 + pmaxsw m5, m6 +%endif ; %1 != 6 + psubw m6, Q2, Q1 ; q2-q1 + pabsw m6, m6 + pmaxsw m5, m6 %if %1 == 16 - SPLATD m11, [maskq+8] + SPLATD m6, [maskq+8] SPLATD m1, [maskq+4] - por m11, m1 - pand m11, m12 - pcmpeqd m11, m12 - pand m10, m11 -%else - SPLATD m11, [maskq+4] - pand m11, m12 - pcmpeqd m11, m12 - pand m10, m11 ; only apply fm-wide to wd>4 blocks -%endif - pmaxsw m8, m10 -%endif - pcmpgtw m8, m2 + por m6, m1 + pand m6, m12 + pcmpeqd m6, m12 + pand m5, m6 +%else ; %1 != 16 + SPLATD m6, [maskq+4] + pand m6, m12 + pcmpeqd m6, m12 + pand m5, m6 ; only apply fm-wide to wd>4 blocks +%endif ; %1==/!=16 + pmaxsw m3, m5 +%endif ; %1 != 4 + pcmpgtw m3, m2 - psubw m10, m3, m6 ; p1-q1 - psubw m11, m4, m5 ; p0-q0 - REPX {pabsw x, x}, m10, m11 - paddw m11, m11 - psrlw m10, 1 - paddw m10, m11 ; abs(p0-q0)*2+(abs(p1-q1)>>1) - pcmpgtw m10, m0 ; abs(p0-q0)*2+(abs(p1-q1)>>1) > E - por m8, m10 + psubw m5, P1, Q1 ; p1-q1 + psubw m6, P0, Q0 ; p0-q0 + REPX {pabsw x, x}, m5, m6 + paddw m6, m6 + psrlw m5, 1 + paddw m5, m6 ; abs(p0-q0)*2+(abs(p1-q1)>>1) + pcmpgtw m5, m0 ; abs(p0-q0)*2+(abs(p1-q1)>>1) > E + por m3, m5 %if %1 == 16 %ifidn %2, v lea tmpq, [dstq+mstrideq*8] mova m0, [tmpq+strideq*1] mova m1, [tmpq+strideq*2] mova m2, [tmpq+stride3q] -%else - mova m0, [rsp+7*32] - mova m1, [rsp+8*32] - mova m2, [rsp+9*32] -%endif - REPX {psubw x, m4}, m0, m1, m2 +%else ; %2 != v + mova m0, [rsp+ 8*16] + mova m1, [rsp+ 9*16] + mova m2, [rsp+10*16] +%endif ; %2==/!=v + REPX {psubw x, P0}, m0, m1, m2 REPX {pabsw x, x}, m0, m1, m2 pmaxsw m1, m0 pmaxsw m1, m2 %ifidn %2, v lea tmpq, [dstq+strideq*4] mova m0, [tmpq+strideq*0] mova m2, [tmpq+strideq*1] - mova m10, [tmpq+strideq*2] -%else - mova m0, [rsp+10*32] - mova m2, [rsp+11*32] - mova m10, [rsp+12*32] -%endif - REPX {psubw x, m5}, m0, m2, m10 - REPX {pabsw x, x}, m0, m2, m10 + mova m5, [tmpq+strideq*2] +%else ; %2 != v + mova m0, [rsp+11*16] + mova m2, [rsp+12*16] + mova m5, [rsp+13*16] +%endif ; %2==/!=v + REPX {psubw x, Q0}, m0, m2, m5 + REPX {pabsw x, x}, m0, m2, m5 pmaxsw m0, m2 - pmaxsw m1, m10 + pmaxsw m1, m5 pmaxsw m1, m0 - pcmpgtw m1, [r11] ; !flat8out - por m1, m9 ; !flat8in | !flat8out + pcmpgtw m1, [bdmulq] ; !flat8out + por m1, m4 ; !flat8in | !flat8out SPLATD m2, [maskq+8] - pand m10, m2, m12 - pcmpeqd m10, m12 - pandn m1, m10 ; flat16 - pandn m10, m8, m1 ; flat16 & fm - SWAP 1, 10 + pand m5, m2, m12 + pcmpeqd m5, m12 + pandn m1, m5 ; flat16 + pandn m5, m3, m1 ; flat16 & fm + SWAP 1, 5 - SPLATD m10, [maskq+4] - por m10, m2 - pand m2, m10, m12 + SPLATD m5, [maskq+4] + por m5, m2 + pand m2, m5, m12 pcmpeqd m2, m12 - pandn m9, m2 ; flat8in - pandn m2, m8, m9 - SWAP 2, 9 + pandn m4, m2 ; flat8in + pandn m2, m3, m4 + SWAP 2, 4 SPLATD m2, [maskq+0] - por m2, m10 + por m2, m5 pand m2, m12 pcmpeqd m2, m12 - pandn m8, m2 - pandn m0, m9, m8 ; fm & !flat8 & !flat16 - SWAP 0, 8 - pandn m0, m1, m9 ; flat8 & !flat16 - SWAP 0, 9 + pandn m3, m2 + pandn m0, m4, m3 ; fm & !flat8 & !flat16 + SWAP 0, 3 + pandn m0, m1, m4 ; flat8 & !flat16 + SWAP 0, 4 %elif %1 != 4 SPLATD m0, [maskq+4] pand m2, m0, m12 pcmpeqd m2, m12 - pandn m9, m2 - pandn m2, m8, m9 ; flat8 & fm - SWAP 2, 9 + pandn m4, m2 + pandn m2, m3, m4 ; flat8 & fm + SWAP 2, 4 SPLATD m2, [maskq+0] por m0, m2 pand m0, m12 pcmpeqd m0, m12 - pandn m8, m0 - pandn m0, m9, m8 ; fm & !flat8 - SWAP 0, 8 -%else + pandn m3, m0 + pandn m0, m4, m3 ; fm & !flat8 + SWAP 0, 3 +%else ; %1 == 4 SPLATD m0, [maskq+0] pand m0, m12 pcmpeqd m0, m12 - pandn m8, m0 ; fm -%endif + pandn m3, m0 ; fm +%endif ; %1==/!=4 ; short filter - +%if ARCH_X86_64 SPLATW m0, r7m +%else + SPLATW m0, bdmulm +%endif pcmpeqw m2, m2 psrlw m0, 1 ; 511 or 2047 pxor m2, m0 ; -512 or -2048 - psubw m10, m5, m4 - paddw m11, m10, m10 - paddw m11, m10 - psubw m10, m3, m6 ; iclip_diff(p1-q1) - pminsw m10, m0 - pmaxsw m10, m2 - pand m10, m7 ; f=iclip_diff(p1-q1)&hev - paddw m10, m11 ; f=iclip_diff(3*(q0-p0)+f) - pminsw m10, m0 - pmaxsw m10, m2 - pand m8, m10 ; f&=fm - paddw m10, m8, [pw_3] - paddw m8, [pw_4] - REPX {pminsw x, m0}, m10, m8 - psraw m10, 3 ; f2 - psraw m8, 3 ; f1 - paddw m4, m10 - psubw m5, m8 + psubw m5, Q0, P0 ; q0-p0 + paddw m6, m5, m5 + paddw m6, m5 ; 3*(q0-p0) + psubw m5, P1, Q1 ; iclip_diff(p1-q1) + pminsw m5, m0 + pmaxsw m5, m2 + pand m5, m7 ; f=iclip_diff(p1-q1)&hev + paddw m5, m6 ; f=iclip_diff(3*(q0-p0)+f) + pminsw m5, m0 + pmaxsw m5, m2 + pand m3, m5 ; f&=fm + paddw m5, m3, [PIC_sym(pw_3)] + paddw m3, [PIC_sym(pw_4)] + REPX {pminsw x, m0}, m5, m3 + psraw m5, 3 ; f2 + psraw m3, 3 ; f1 + psubw m0, m2 ; 1023 or 4095 + pxor m2, m2 +%if ARCH_X86_64 + paddw P0, m5 + psubw Q0, m3 +%else + paddw m5, P0 + psubw m6, Q0, m3 + REPX {pminsw x, m0}, m5, m6 + REPX {pmaxsw x, m2}, m5, m6 +%endif - paddw m8, [pw_1] - psraw m8, 1 ; f=(f1+1)>>1 - pandn m7, m8 ; f&=!hev - SWAP 7, 8 - paddw m3, m8 - psubw m6, m8 - pxor m8, m8 - psubw m0, m2 ; 1023 or 4095 - REPX {pminsw x, m0}, m3, m4, m5, m6 - REPX {pmaxsw x, m8}, m3, m4, m5, m6 + paddw m3, [PIC_sym(pw_1)] + psraw m3, 1 ; f=(f1+1)>>1 + pandn m7, m3 ; f&=!hev + SWAP 7, 3 +%if ARCH_X86_64 + paddw P1, m3 + psubw Q1, m3 + REPX {pminsw x, m0}, P1, P0, Q0, Q1 + REPX {pmaxsw x, m2}, P1, P0, Q0, Q1 +%else + psubw m7, Q1, m3 + paddw m3, P1 + REPX {pminsw x, m0}, m7, m3 + REPX {pmaxsw x, m2}, m7, m3 +%if %1 > 4 + mova P1, m3 + mova P0, m5 + mova Q0, m6 + mova Q1, m7 +%endif +%endif %if %1 == 16 -; m3-6 = p1/p0/q0/q1, m9=flat8, m1=flat16 +; m8-11 = p1/p0/q0/q1, m4=flat8, m1=flat16 ; m12=filter bits mask ; m13-15=p2/q2/q3 -; m0,2,7-8,10-11 = free +; m0,2-3,5-7 = free ; flat16 filter %ifidn %2, v lea tmpq, [dstq+mstrideq*8] mova m0, [tmpq+strideq*1] ; p6 mova m2, [tmpq+strideq*2] ; p5 mova m7, [tmpq+stride3q] ; p4 - mova m11, [tmpq+strideq*4] ; p3 -%else - mova m0, [rsp+7*32] - mova m2, [rsp+8*32] - mova m7, [rsp+9*32] - mova m11, [rsp+5*32] -%endif + mova m6, [tmpq+strideq*4] ; p3 + lea tmpq, [dstq+mstrideq*4] +%else ; %2 != v + mova m0, [rsp+ 8*16] + mova m2, [rsp+ 9*16] + mova m7, [rsp+10*16] + mova m6, [rsp+ 6*16] +%endif ; %2==/!=v - mova [rsp+ 0*32], m9 + mova [rsp+ 0*16], m4 ; p6*7+p5*2+p4*2+p3+p2+p1+p0+q0 - psllw m8, m0, 3 ; p6*8 - paddw m8, [pw_8] - paddw m10, m2, m7 ; p5+p4 - psubw m8, m0 - paddw m10, m10 ; (p5+p4)*2 - paddw m8, m11 ; p6*7+p3 - paddw m10, m13 ; (p5+p4)*2+p2 - paddw m8, m3 ; p6*7+p3+p1 - paddw m10, m4 ; (p5+p4)*2+p2+p0 - paddw m8, m5 ; p6*7+p3+p1+q0 - paddw m8, m10 ; p6*7+p5*2+p4*2+p3+p2+p1+p0+q0 - psrlw m10, m8, 4 - pand m10, m1 - pandn m9, m1, m2 - por m10, m9 + psllw m3, m0, 3 ; p6*8 + paddw m3, [PIC_sym(pw_8)] + paddw m5, m2, m7 ; p5+p4 + psubw m3, m0 + paddw m5, m5 ; (p5+p4)*2 + paddw m3, m6 ; p6*7+p3 + paddw m5, P2 ; (p5+p4)*2+p2 + paddw m3, P1 ; p6*7+p3+p1 + paddw m5, P0 ; (p5+p4)*2+p2+p0 + paddw m3, Q0 ; p6*7+p3+p1+q0 + paddw m3, m5 ; p6*7+p5*2+p4*2+p3+p2+p1+p0+q0 + psrlw m5, m3, 4 + pand m5, m1 + pandn m4, m1, m2 + por m5, m4 %ifidn %2, v - mova [tmpq+strideq*2], m10 ; p5 -%else - mova [rsp+8*32], m10 -%endif + mova [tmpq+mstrideq*2], m5 ; p5 +%else ; %2 != v + mova [rsp+9*16], m5 +%endif ; %2==/!=v ; sub p6*2, add p3/q1 - paddw m8, m11 - paddw m10, m0, m0 - paddw m8, m6 - psubw m8, m10 - psrlw m10, m8, 4 - pand m10, m1 - pandn m9, m1, m7 - por m10, m9 + paddw m3, m6 + paddw m5, m0, m0 + paddw m3, Q1 + psubw m3, m5 + psrlw m5, m3, 4 + pand m5, m1 + pandn m4, m1, m7 + por m5, m4 %ifidn %2, v - mova [tmpq+stride3q], m10 ; p4 -%else - mova [rsp+9*32], m10 -%endif + mova [tmpq+mstrideq*1], m5 ; p4 +%else ; %2 != v + mova [rsp+10*16], m5 +%endif ; %2==/!=v ; sub p6/p5, add p2/q2 - psubw m8, m0 - paddw m10, m13, m14 - psubw m8, m2 - paddw m8, m10 - psrlw m10, m8, 4 - pand m10, m1 - pandn m9, m1, m11 - por m10, m9 + psubw m3, m0 + paddw m5, P2, Q2 + psubw m3, m2 + paddw m3, m5 + psrlw m5, m3, 4 + pand m5, m1 + pandn m4, m1, m6 + por m5, m4 %ifidn %2, v - mova [tmpq+strideq*4], m10 ; p3 - lea tmpq, [dstq+strideq*4] -%else - mova [rsp+5*32], m10 + mova [tmpq+strideq*0], m5 ; p3 +%else ; %2 != v + mova [rsp+6*16], m5 +%endif ; %2==/!=v + +%define WRITE_IN_PLACE 0 +%ifidn %2, v +%if ARCH_X86_64 +%define WRITE_IN_PLACE 1 +%endif %endif ; sub p6/p4, add p1/q3 - paddw m8, m3 - paddw m10, m0, m7 - paddw m8, m15 - psubw m8, m10 - psrlw m10, m8, 4 - pand m10, m1 - pandn m9, m1, m13 - por m10, m9 - mova [rsp+1*32], m10 ; don't clobber p2/m13 + paddw m3, P1 + paddw m5, m0, m7 + paddw m3, Q3 + psubw m3, m5 + psrlw m5, m3, 4 + pand m5, m1 + pandn m4, m1, P2 + por m5, m4 +%if WRITE_IN_PLACE + mova [tmpq+strideq*1], m5 +%else + mova [rsp+1*16], m5 ; don't clobber p2/m13 +%endif ; sub p6/p3, add p0/q4 - paddw m8, m4 - paddw m10, m0, m11 + paddw m3, P0 + paddw m5, m0, m6 %ifidn %2, v - paddw m8, [tmpq+strideq*0] + paddw m3, [dstq+strideq*4] +%else ; %2 != v + paddw m3, [rsp+11*16] +%endif ; %2==/!=v + psubw m3, m5 + psrlw m5, m3, 4 + pand m5, m1 + pandn m4, m1, P1 + por m5, m4 +%if WRITE_IN_PLACE + mova [dstq+mstrideq*2], m5 %else - paddw m8, [rsp+10*32] + mova [rsp+2*16], m5 ; don't clobber p1/m3 %endif - psubw m8, m10 - psrlw m10, m8, 4 - pand m10, m1 - pandn m9, m1, m3 - por m10, m9 - mova [rsp+2*32], m10 ; don't clobber p1/m3 ; sub p6/p2, add q0/q5 - paddw m8, m5 - paddw m10, m0, m13 + paddw m3, Q0 + paddw m5, m0, P2 %ifidn %2, v - paddw m8, [tmpq+strideq*1] -%else - paddw m8, [rsp+11*32] +%if ARCH_X86_32 + lea r4, P2 %endif - psubw m8, m10 - psrlw m10, m8, 4 - pand m10, m1 - pandn m9, m1, m4 - por m10, m9 - mova [rsp+3*32], m10 ; don't clobber p0/m4 + lea tmpq, [dstq+strideq*4] + paddw m3, [tmpq+strideq*1] +%else ; %2 != v + paddw m3, [rsp+12*16] +%endif ; %2==/!=v + psubw m3, m5 + psrlw m5, m3, 4 + pand m5, m1 + pandn m4, m1, P0 + por m5, m4 +%if WRITE_IN_PLACE + mova [dstq+mstrideq*1], m5 +%else + mova [rsp+3*16], m5 ; don't clobber p0/m4 +%endif ; sub p6/p1, add q1/q6 - paddw m8, m6 - paddw m10, m0, m3 + paddw m3, Q1 + paddw m5, m0, P1 %ifidn %2, v mova m0, [tmpq+strideq*2] ; q6 +%else ; %2 != v + mova m0, [rsp+13*16] ; q6 +%endif ; %2==/!=v + paddw m3, m0 + psubw m3, m5 + psrlw m5, m3, 4 + pand m5, m1 + pandn m4, m1, Q0 + por m5, m4 +%if WRITE_IN_PLACE + mova [dstq], m5 %else - mova m0, [rsp+12*32] ; q6 + mova [rsp+4*16], m5 ; don't clobber q0/m5 %endif - paddw m8, m0 - psubw m8, m10 - psrlw m10, m8, 4 - pand m10, m1 - pandn m9, m1, m5 - por m10, m9 - mova [rsp+4*32], m10 ; don't clobber q0/m5 ; sub p5/p0, add q2/q6 - paddw m8, m14 - paddw m10, m2, m4 - paddw m8, m0 - psubw m8, m10 - psrlw m10, m8, 4 - pand m10, m1 - pandn m9, m1, m6 - por m2, m10, m9 ; don't clobber q1/m6 + paddw m3, Q2 + paddw m5, m2, P0 + paddw m3, m0 + psubw m3, m5 + psrlw m5, m3, 4 + pand m5, m1 + pandn m4, m1, Q1 + por m2, m5, m4 ; don't clobber q1/m6 ; sub p4/q0, add q3/q6 - paddw m8, m15 - paddw m10, m7, m5 - paddw m8, m0 - psubw m8, m10 - psrlw m10, m8, 4 - pand m10, m1 - pandn m9, m1, m14 - por m7, m10, m9 ; don't clobber q2/m14 + paddw m3, Q3 + paddw m7, Q0 + paddw m3, m0 + psubw m3, m7 + psrlw m7, m3, 4 + pand m7, m1 + pandn m4, m1, Q2 + por m7, m4 ; don't clobber q2/m14 ; sub p3/q1, add q4/q6 %ifidn %2, v - paddw m8, [tmpq+strideq*0] -%else - paddw m8, [rsp+10*32] -%endif - paddw m10, m11, m6 - paddw m8, m0 - psubw m8, m10 - psrlw m10, m8, 4 - pand m10, m1 - pandn m9, m1, m15 - por m10, m9 -%ifidn %2, v - mova [tmpq+mstrideq], m10 ; q3 -%else - mova [rsp+14*32], m10 -%endif + paddw m3, [tmpq+strideq*0] +%else ; %2 != v + paddw m3, [rsp+11*16] +%endif ; %2==/!=v + paddw m6, Q1 + paddw m3, m0 + psubw m3, m6 + psrlw m6, m3, 4 + pand m6, m1 + pandn m4, m1, Q3 + por m6, m4 +%if WRITE_IN_PLACE + mova [tmpq+mstrideq], m6 ; q3 +%else ; %2 != v + mova [rsp+5*16], m6 +%endif ; %2==/!=v ; sub p2/q2, add q5/q6 %ifidn %2, v - paddw m8, [tmpq+strideq*1] + paddw m3, [tmpq+strideq*1] +%if ARCH_X86_64 + paddw m5, P2, Q2 %else - paddw m8, [rsp+11*32] + ; because tmpq is clobbered, so we use a backup pointer for P2 instead + paddw m5, [r4], Q2 + mov pic_regq, pic_regm %endif - paddw m10, m13, m14 - paddw m8, m0 - psubw m8, m10 - psrlw m10, m8, 4 - pand m10, m1 +%else ; %2 != v + paddw m3, [rsp+12*16] + paddw m5, P2, Q2 +%endif ; %2==/!=v + paddw m3, m0 + psubw m3, m5 + psrlw m5, m3, 4 + pand m5, m1 %ifidn %2, v - pandn m9, m1, [tmpq+strideq*0] -%else - pandn m9, m1, [rsp+10*32] -%endif - por m10, m9 + pandn m4, m1, [tmpq+strideq*0] +%else ; %2 != v + pandn m4, m1, [rsp+11*16] +%endif ; %2==/!=v + por m5, m4 %ifidn %2, v - mova [tmpq+strideq*0], m10 ; q4 -%else - mova [rsp+10*32], m10 -%endif + mova [tmpq+strideq*0], m5 ; q4 +%else ; %2 != v + mova [rsp+11*16], m5 +%endif ; %2==/!=v ; sub p1/q3, add q6*2 - psubw m8, m3 + psubw m3, P1 paddw m0, m0 - psubw m8, m15 - paddw m8, m0 - psrlw m10, m8, 4 - pand m10, m1 -%ifidn %2, v - pandn m9, m1, [tmpq+strideq*1] -%else - pandn m9, m1, [rsp+11*32] -%endif - por m10, m9 + psubw m3, Q3 + paddw m3, m0 + psrlw m5, m3, 4 + pand m5, m1 %ifidn %2, v - mova [tmpq+strideq*1], m10 ; q5 -%else - mova [rsp+11*32], m10 -%endif + pandn m4, m1, [tmpq+strideq*1] +%else ; %2 != v + pandn m4, m1, [rsp+12*16] +%endif ; %2==/!=v + por m5, m4 +%ifidn %2, v + mova [tmpq+strideq*1], m5 ; q5 +%else ; %2 != v + mova [rsp+12*16], m5 +%endif ; %2==/!=v - mova m9, [rsp+0*32] - mova m13, [rsp+1*32] - mova m3, [rsp+2*32] - mova m4, [rsp+3*32] - mova m5, [rsp+4*32] - SWAP 2, 6 - SWAP 7, 14 + mova m4, [rsp+0*16] %ifidn %2, v lea tmpq, [dstq+mstrideq*4] -%else - mova m15, [rsp+14*32] %endif -%endif +%if ARCH_X86_64 + SWAP 2, 11 + SWAP 7, 14 + SWAP 6, 15 +%else ; x86-32 + mova Q1, m2 + mova Q2, m7 +%endif ; x86-32/64 +%if WRITE_IN_PLACE + mova P2, [tmpq+strideq*1] + mova P1, [tmpq+strideq*2] + mova P0, [tmpq+stride3q] + mova Q0, [dstq] +%elif ARCH_X86_64 + mova P2, [rsp+1*16] + mova P1, [rsp+2*16] + mova P0, [rsp+3*16] + mova Q0, [rsp+4*16] +%else ; !WRITE_IN_PLACE & x86-32 + mova m0, [rsp+1*16] + mova m1, [rsp+2*16] + mova m2, [rsp+3*16] + mova m3, [rsp+4*16] + mova m7, [rsp+5*16] + mova P2, m0 + mova P1, m1 + mova P0, m2 + mova Q0, m3 + mova Q3, m7 +%endif ; WRITE_IN_PLACE / x86-32/64 +%undef WRITE_IN_PLACE +%endif ; %1 == 16 %if %1 >= 8 + ; flat8 filter -%ifidn %2, v - mova m0, [tmpq+strideq*0] ; p3 -%else - mova m0, [rsp+5*32] ; p3 -%endif - paddw m1, m0, m13 ; p3+p2 - paddw m2, m3, m4 ; p1+p0 - paddw m8, m1, m1 ; 2*(p3+p2) + mova m0, P3 ; p3 + paddw m1, m0, P2 ; p3+p2 + paddw m2, P1, P0 ; p1+p0 + paddw m3, m1, m1 ; 2*(p3+p2) paddw m2, m0 ; p1+p0+p3 - paddw m8, m5 ; 2*(p3+p2)+q0 - paddw m2, m8 ; 3*p3+2*p2+p1+p0+q0 - pmulhrsw m7, m2, [pw_4096] + paddw m3, Q0 ; 2*(p3+p2)+q0 + paddw m2, m3 ; 3*p3+2*p2+p1+p0+q0 + pmulhrsw m7, m2, [PIC_sym(pw_4096)] + psubw m7, P2 + pand m7, m4 - paddw m8, m3, m6 - psubw m2, m1 - paddw m2, m8 - pmulhrsw m8, m2, [pw_4096] + paddw m3, P1, Q1 ; p1+q1 + psubw m2, m1 ; 2*p3+p2+p1+p0+q0 + paddw m2, m3 ; 2*p3+p2+2*p1+p0+q0+q1 + pmulhrsw m3, m2, [PIC_sym(pw_4096)] + psubw m3, P1 + pand m3, m4 - paddw m10, m0, m3 - paddw m11, m4, m14 - psubw m2, m10 - paddw m2, m11 - pmulhrsw m10, m2, [pw_4096] + paddw m5, m0, P1 ; p3+p1 + paddw m6, P0, Q2 ; p0+q2 + psubw m2, m5 ; p3+p2+p1+p0+q0+q1 + paddw m2, m6 ; p3+p2+p1+2*p0+q0+q1+q2 + pmulhrsw m5, m2, [PIC_sym(pw_4096)] + psubw m5, P0 + pand m5, m4 - paddw m11, m0, m4 - paddw m1, m5, m15 - psubw m2, m11 - paddw m2, m1 - pmulhrsw m11, m2, [pw_4096] - - paddw m2, m6 - paddw m2, m15 - paddw m1, m13, m5 - psubw m2, m1 - pmulhrsw m1, m2, [pw_4096] + paddw m6, m0, P0 ; p3+p0 + paddw m1, Q0, Q3 ; q0+q3 + psubw m2, m6 ; p2+p1+p0+q0+q1+q2 + paddw m2, m1 ; p2+p1+p0+2*q0+q1+q2+q3 + pmulhrsw m6, m2, [PIC_sym(pw_4096)] + psubw m6, Q0 + pand m6, m4 - psubw m2, m3 - psubw m2, m6 - paddw m0, m15, m14 - paddw m2, m0 - pmulhrsw m2, [pw_4096] + paddw m2, Q1 ; p2+p1+p0+2*q0+2*q1+q2+q3 + paddw m2, Q3 ; p2+p1+p0+2*q0+2*q1+q2+2*q3 + paddw m1, P2, Q0 ; p2+q0 + psubw m2, m1 ; p1+p0+q0+2*q1+q2+2*q3 + pmulhrsw m1, m2, [PIC_sym(pw_4096)] + psubw m1, Q1 + pand m1, m4 - REPX {pand x, m9}, m7, m8, m10, m11, m1, m2 -%if avx_enabled - REPX {pandn x, m9}, m13, m3, m4, m5, m6, m14 -%else - pcmpeqw m0, m0 - pxor m0, m9 - REPX {pand x, m0}, m13, m3, m4, m5, m6, m14 -%endif - por m13, m7 - por m3, m8 - por m4, m10 - por m5, m11 - por m6, m1 - por m14, m2 + psubw m2, P1 ; p0+q0+2*q1+q2+2*q3 + psubw m2, Q1 ; p0+q0+q1+q2+2*q3 + paddw m0, Q3, Q2 ; q3+q2 + paddw m2, m0 ; p0+q0+q1+2*q2+3*q3 + pmulhrsw m2, [PIC_sym(pw_4096)] + psubw m2, Q2 + pand m2, m4 + + paddw m7, P2 + paddw m3, P1 + paddw m5, P0 + paddw m6, Q0 + paddw m1, Q1 + paddw m2, Q2 %ifidn %2, v - mova [tmpq+strideq*1], m13 ; p2 + mova [tmpq+strideq*1], m7 ; p2 mova [tmpq+strideq*2], m3 ; p1 - mova [tmpq+stride3q ], m4 ; p0 - mova [dstq+strideq*0], m5 ; q0 - mova [dstq+strideq*1], m6 ; q1 - mova [dstq+strideq*2], m14 ; q2 + mova [tmpq+stride3q ], m5 ; p0 + mova [dstq+strideq*0], m6 ; q0 + mova [dstq+strideq*1], m1 ; q1 + mova [dstq+strideq*2], m2 ; q2 +%else ; %2 != v + mova m0, P3 + +%if %1 == 8 + lea tmpq, [dstq+strideq*4] +%if ARCH_X86_64 + SWAP 4, 15 + TRANSPOSE8X8W 0, 7, 3, 5, 6, 1, 2, 4, 8 %else - mova m0, [rsp+5*32] -%if %1 == 8 - TRANSPOSE8X8W 0, 13, 3, 4, 5, 6, 14, 15, 1 + TRANSPOSE8X8W 0, 7, 3, 5, 6, 1, 2, 4, "", \ + Q3, [tmpq+strideq*1-8], a, u +%endif ; write 8x8 - movu [dstq+strideq*0-8], xm0 - movu [dstq+strideq*1-8], xm13 - movu [dstq+strideq*2-8], xm3 - movu [dstq+stride3q -8], xm4 - lea dstq, [dstq+strideq*4] - movu [dstq+strideq*0-8], xm5 - movu [dstq+strideq*1-8], xm6 - movu [dstq+strideq*2-8], xm14 - movu [dstq+stride3q -8], xm15 - lea dstq, [dstq+strideq*4] + movu [dstq+strideq*0-8], m0 + movu [dstq+strideq*1-8], m7 + movu [dstq+strideq*2-8], m3 + movu [dstq+stride3q -8], m5 + movu [tmpq+strideq*0-8], m6 +%if ARCH_X86_64 + movu [tmpq+strideq*1-8], m1 +%endif + movu [tmpq+strideq*2-8], m2 + movu [tmpq+stride3q -8], m4 + lea dstq, [dstq+strideq*8] +%else ; %1 != 8 +%if ARCH_X86_64 + SWAP 6, 8 + SWAP 1, 9 + SWAP 2, 10 %else - mova m0, [rsp+6*32] - mova m1, [rsp+7*32] - mova m2, [rsp+8*32] - mova m7, [rsp+9*32] - mova m8, [rsp+5*32] - TRANSPOSE8X8W 0, 1, 2, 7, 8, 13, 3, 4, 9 + mova [rsp+1*16], m6 + mova [rsp+2*16], m1 + mova [rsp+3*16], m2 +%endif - mova [dstq+strideq*0-16], xm0 - mova [dstq+strideq*1-16], xm1 - mova [dstq+strideq*2-16], xm2 - mova [dstq+stride3q -16], xm7 + mova m1, [rsp+ 7*16] + mova m2, [rsp+ 8*16] + mova m4, [rsp+ 9*16] + mova m6, [rsp+10*16] lea tmpq, [dstq+strideq*4] - mova [tmpq+strideq*0-16], xm8 - mova [tmpq+strideq*1-16], xm13 - mova [tmpq+strideq*2-16], xm3 - mova [tmpq+stride3q -16], xm4 +%if ARCH_X86_64 + TRANSPOSE8X8W 1, 2, 4, 6, 0, 7, 3, 5, 11 +%else + mova [rsp+7*16], m5 + TRANSPOSE8X8W 1, 2, 4, 6, 0, 7, 3, 5, "", \ + [rsp+7*16], [tmpq+strideq*1-16], a, a +%endif + + mova [dstq+strideq*0-16], m1 + mova [dstq+strideq*1-16], m2 + mova [dstq+strideq*2-16], m4 + mova [dstq+stride3q -16], m6 + mova [tmpq+strideq*0-16], m0 +%if ARCH_X86_64 + mova [tmpq+strideq*1-16], m7 +%endif + mova [tmpq+strideq*2-16], m3 + mova [tmpq+stride3q -16], m5 - mova m0, [rsp+10*32] - mova m1, [rsp+11*32] - mova m2, [rsp+12*32] - mova m3, [rsp+13*32] - TRANSPOSE8X8W 5, 6, 14, 15, 0, 1, 2, 3, 4 - mova [dstq+strideq*0], xm5 - mova [dstq+strideq*1], xm6 - mova [dstq+strideq*2], xm14 - mova [dstq+stride3q ], xm15 - lea dstq, [dstq+strideq*4] - mova [dstq+strideq*0], xm0 - mova [dstq+strideq*1], xm1 - mova [dstq+strideq*2], xm2 - mova [dstq+stride3q ], xm3 - lea dstq, [dstq+strideq*4] +%if ARCH_X86_64 + SWAP 6, 8 + SWAP 1, 9 + SWAP 2, 10 + SWAP 4, 15 +%else + mova m6, [rsp+1*16] + mova m1, [rsp+2*16] + mova m2, [rsp+3*16] + mova m4, Q3 %endif + mova m0, [rsp+11*16] + mova m3, [rsp+12*16] + mova m5, [rsp+13*16] +%if ARCH_X86_64 + mova m7, [rsp+14*16] + TRANSPOSE8X8W 6, 1, 2, 4, 0, 3, 5, 7, 8 +%else + TRANSPOSE8X8W 6, 1, 2, 4, 0, 3, 5, 7, "", \ + [rsp+14*16], [tmpq+strideq*1], a, a %endif + mova [dstq+strideq*0], m6 + mova [dstq+strideq*1], m1 + mova [dstq+strideq*2], m2 + mova [dstq+stride3q ], m4 + mova [tmpq+strideq*0], m0 +%if ARCH_X86_64 + mova [tmpq+strideq*1], m3 +%endif + mova [tmpq+strideq*2], m5 + mova [tmpq+stride3q ], m7 + lea dstq, [dstq+strideq*8] +%endif ; %1==/!=8 +%endif ; %2==/!=v %elif %1 == 6 ; flat6 filter + paddw m3, P1, P0 ; p1+p0 + paddw m3, P2 ; p2+p1+p0 + paddw m6, P2, Q0 ; p2+q0 + paddw m3, m3 ; 2*(p2+p1+p0) + paddw m3, m6 ; p2+2*(p2+p1+p0)+q0 + pmulhrsw m2, m3, [PIC_sym(pw_4096)] + psubw m2, P1 + pand m2, m4 - paddw m8, m3, m4 - paddw m8, m13 ; p2+p1+p0 - paddw m11, m13, m5 - paddw m8, m8 - paddw m8, m11 ; p2+2*(p2+p1+p0)+q0 - pmulhrsw m2, m8, [pw_4096] - - paddw m8, m5 - paddw m11, m13, m13 - paddw m8, m6 - psubw m8, m11 - pmulhrsw m10, m8, [pw_4096] + paddw m3, Q0 ; p2+2*(p2+p1+p0+q0) + paddw m6, P2, P2 ; 2*p2 + paddw m3, Q1 ; p2+2*(p2+p1+p0+q0)+q1 + psubw m3, m6 ; p2+2*(p1+p0+q0)+q1 + pmulhrsw m5, m3, [PIC_sym(pw_4096)] + psubw m5, P0 + pand m5, m4 - paddw m8, m6 - paddw m11, m13, m3 - paddw m8, m14 - psubw m8, m11 - pmulhrsw m11, m8, [pw_4096] - - psubw m8, m3 - paddw m14, m14 - psubw m8, m4 - paddw m8, m14 - pmulhrsw m8, [pw_4096] + paddw m3, Q1 ; p2+2*(p1+p0+q0+q1) + paddw m6, P2, P1 ; p2+p1 + paddw m3, Q2 ; p2+2*(p1+p0+q0+q1)+q2 + psubw m3, m6 ; p1+2*(p0+q0+q1)+q2 + pmulhrsw m6, m3, [PIC_sym(pw_4096)] + psubw m6, Q0 + pand m6, m4 - REPX {pand x, m9}, m2, m10, m11, m8 -%if avx_enabled - REPX {pandn x, m9, x}, m3, m4, m5, m6 + psubw m3, P1 ; 2*(p0+q0+q1)+q2 +%if ARCH_X86_64 + paddw Q2, Q2 ; q2*2 +%else + mova m0, Q2 + paddw m0, m0 +%endif + psubw m3, P0 ; p0+2*(q0+q1)+q2 +%if ARCH_X86_64 + paddw m3, Q2 ; p0+q*(q0+q1+q2)+q2 %else - pcmpeqw m0, m0 - pxor m0, m9 - REPX {pand x, m0}, m3, m4, m5, m6 + paddw m3, m0 %endif - por m3, m2 - por m4, m10 - por m5, m11 - por m6, m8 + pmulhrsw m3, [PIC_sym(pw_4096)] + psubw m3, Q1 + pand m3, m4 + + paddw m2, P1 + paddw m5, P0 + paddw m6, Q0 + paddw m3, Q1 %ifidn %2, v - mova [tmpq+strideq*2], m3 ; p1 - mova [tmpq+stride3q ], m4 ; p0 - mova [dstq+strideq*0], m5 ; q0 - mova [dstq+strideq*1], m6 ; q1 -%else - TRANSPOSE_8x4_AND_WRITE_4x8 -%endif -%else + mova [dstq+mstrideq*2], m2 ; p1 + mova [dstq+mstrideq*1], m5 ; p0 + mova [dstq+strideq*0], m6 ; q0 + mova [dstq+strideq*1], m3 ; q1 +%else ; %2 != v + TRANSPOSE_8x4_AND_WRITE_4x8 m2, m5, m6, m3, m0 +%endif ; %2==/!=v +%else ; %1 == 4 +%if ARCH_X86_64 %ifidn %2, v - mova [tmpq+strideq*0], m3 ; p1 - mova [tmpq+strideq*1], m4 ; p0 - mova [tmpq+strideq*2], m5 ; q0 - mova [tmpq+stride3q ], m6 ; q1 -%else - TRANSPOSE_8x4_AND_WRITE_4x8 -%endif -%endif + mova [dstq+mstrideq*2], P1 ; p1 + mova [dstq+mstrideq*1], P0 ; p0 + mova [dstq+strideq*0], Q0 ; q0 + mova [dstq+strideq*1], Q1 ; q1 +%else ; %2 != v + TRANSPOSE_8x4_AND_WRITE_4x8 P1, P0, Q0, Q1, m0 +%endif ; %2==/!=v +%else ; x86-32 +%ifidn %2, v + mova [dstq+mstrideq*2], m3 + mova [dstq+mstrideq*1], m5 + mova [dstq+strideq*0], m6 + mova [dstq+strideq*1], m7 +%else ; %2 != v + TRANSPOSE_8x4_AND_WRITE_4x8 m3, m5, m6, m7, m0 +%endif ; %2==/!=v +%endif ; x86-32/64 +%endif ; %1 +%undef P3 +%undef P2 +%undef P1 +%undef P0 +%undef Q0 +%undef Q1 +%undef Q2 +%undef Q3 %endmacro INIT_XMM ssse3 -cglobal lpf_v_sb_y_16bpc, 6, 12, 16, 32 * 5, \ +; stack layout: +; r0 - flat8 backup inside flat16 code +%if ARCH_X86_64 +cglobal lpf_v_sb_y_16bpc, 6, 12, 16, -16 * 1, \ dst, stride, mask, l, l_stride, lut, \ - w, stride3, mstride, tmp, mask_bits + w, stride3, mstride, tmp, mask_bits, bdmul mov r6d, r7m sar r6d, 7 and r6d, 16 ; 0 for 10bpc, 16 for 12bpc - lea r11, [pw_4] - add r11, r6 + lea bdmulq, [pw_4] + add bdmulq, r6 mov wd, wm shl l_strideq, 2 sub lq, l_strideq +%else +; stack layout [32bit only]: +; r1-4 - p2-q0 post-filter16 +; r5 - p3 +; r6 - q3 post-filter16 +; r7 - GPRs [mask_bitsm, mstridem] +; r8 - m12/pb_mask +; r9 - bdmulq +cglobal lpf_v_sb_y_16bpc, 4, 7, 8, -16 * (10 + extra_stack), \ + dst, stride, mask, mstride, pic_reg, stride3, tmp + RELOC_ARGS v, 10*16 +%if STACK_ALIGNMENT >= 16 + mov r5d, r7m +%endif + sar r5d, 7 + and r5d, 16 ; 0 for 10bpc, 16 for 12bpc + LEA pic_regq, PIC_base +%define pic_regm dword [esp+7*16+2*gprsize] + mov pic_regm, pic_regq + mova m0, [PIC_sym(pw_4)+r5] +%define bdmulq esp+9*16 + mova [bdmulq], m0 + shl dword lstridem, 2 + sub r3, dword lstridem + mov dword lm, r3 +%endif mov mstrideq, strideq neg mstrideq lea stride3q, [strideq*3] +%if ARCH_X86_64 mov mask_bitsd, 0x3 mova m12, [pb_mask] +%else +%define mstridem dword [esp+7*16+1*gprsize] + mov mstridem, mstrideq +%define mask_bitsm dword [esp+7*16+0*gprsize] + mov mask_bitsm, 0x3 + mova m0, [PIC_sym(pb_mask)] +%define m12 [esp+8*16] + mova m12, m0 +%endif .loop: +%if ARCH_X86_64 test [maskq+8], mask_bitsd ; vmask[2] +%else + mov r6d, mask_bitsm + test [maskq+8], r6d +%endif jz .no_flat16 FILTER 16, v jmp .end .no_flat16: +%if ARCH_X86_64 test [maskq+4], mask_bitsd ; vmask[1] +%else + test [maskq+4], r6d +%endif jz .no_flat FILTER 8, v jmp .end .no_flat: +%if ARCH_X86_64 test [maskq+0], mask_bitsd ; vmask[0] +%else + test [maskq+0], r6d +%endif jz .end FILTER 4, v .end: +%if ARCH_X86_64 pslld m12, 2 add lq, 8 +%else + mova m0, m12 + pslld m0, 2 + mova m12, m0 + add dword lm, 8 +%endif add dstq, 16 +%if ARCH_X86_64 shl mask_bitsd, 2 sub wd, 2 +%else + shl mask_bitsm, 2 + sub dword wm, 2 +%endif jg .loop +%undef mask_bitsm +%undef bdmulq + UNRELOC_ARGS RET INIT_XMM ssse3 -cglobal lpf_h_sb_y_16bpc, 6, 12, 16, 32 * 15, \ +; stack layout: +; r0 - flat8 backup inside flat16 +; r1-4 - p2-q0 post-filter16 backup +; r5 - q3 post-filter16 backup +; r6 - p3 +; r7-10 - p7-4 +; r11-14 - q4-7 +%if ARCH_X86_64 +cglobal lpf_h_sb_y_16bpc, 6, 11, 16, -16 * 15, \ dst, stride, mask, l, l_stride, lut, \ - h, stride3, l_stride3, tmp, mask_bits + h, stride3, tmp, mask_bits, bdmul mov r6d, r7m sar r6d, 7 and r6d, 16 ; 0 for 10bpc, 16 for 12bpc - lea r11, [pw_4] - add r11, r6 + lea bdmulq, [pw_4] + add bdmulq, r6 mov hd, hm shl l_strideq, 2 +%else +; stack layout [32bit only]: +; r15 - GPRs [mask_bitsm] +; r16 - m12/pb_mask +; r17 - bdmulq +; r18-24 - p2-q3 +cglobal lpf_h_sb_y_16bpc, 4, 7, 8, -16 * (25 + extra_stack), \ + dst, stride, mask, l, pic_reg, stride3, tmp + RELOC_ARGS h, 25*16 +%if STACK_ALIGNMENT >= 16 + mov r5d, r7m +%endif + sar r5d, 7 + and r5d, 16 ; 0 for 10bpc, 16 for 12bpc + LEA pic_regq, PIC_base + mova m0, [PIC_sym(pw_4)+r5] +%define bdmulq esp+17*16 + mova [bdmulq], m0 + shl dword lstridem, 2 +%endif sub lq, 4 lea stride3q, [strideq*3] - lea l_stride3q, [l_strideq*3] +%if ARCH_X86_64 mov mask_bitsd, 0x3 mova m12, [pb_mask] +%else +%define mask_bitsm dword [esp+15*16+0*gprsize] + mov mask_bitsm, 0x3 + mova m0, [PIC_sym(pb_mask)] +%define m12 [esp+16*16] + mova m12, m0 +%endif .loop: +%if ARCH_X86_64 test [maskq+8], mask_bitsd ; vmask[2] +%else + mov r6d, mask_bitsm + test [maskq+8], r6d +%endif jz .no_flat16 FILTER 16, h jmp .end .no_flat16: +%if ARCH_X86_64 test [maskq+4], mask_bitsd ; vmask[1] +%else + test [maskq+4], r6d +%endif jz .no_flat FILTER 8, h jmp .end .no_flat: +%if ARCH_X86_64 test [maskq+0], mask_bitsd ; vmask[0] +%else + test [maskq+0], r6d +%endif jz .no_filter FILTER 4, h jmp .end .no_filter: lea dstq, [dstq+strideq*8] .end: +%if ARCH_X86_64 pslld m12, 2 lea lq, [lq+l_strideq*2] shl mask_bitsd, 2 sub hd, 2 +%else + mova m0, m12 + pslld m0, 2 + mova m12, m0 + add lq, dword lstridem + add lq, dword lstridem + shl mask_bitsm, 2 + sub dword hm, 2 +%endif jg .loop +%undef mask_bitsm +%undef bdmulq + UNRELOC_ARGS RET INIT_XMM ssse3 +%if ARCH_X86_64 cglobal lpf_v_sb_uv_16bpc, 6, 12, 16, \ dst, stride, mask, l, l_stride, lut, \ - w, stride3, mstride, tmp, mask_bits + w, stride3, mstride, tmp, mask_bits, bdmul mov r6d, r7m sar r6d, 7 and r6d, 16 ; 0 for 10bpc, 16 for 12bpc - lea r11, [pw_4] - add r11, r6 + lea bdmulq, [pw_4] + add bdmulq, r6 mov wd, wm shl l_strideq, 2 sub lq, l_strideq +%else +; stack layout [32bit only]: +; r0 - GPRs [mask_bitsm, mstridem] +; r1 - m12/pb_mask +; r2 - bdmulq +cglobal lpf_v_sb_uv_16bpc, 4, 7, 8, -16 * (3 + extra_stack), \ + dst, stride, mask, mstride, pic_reg, stride3, tmp + RELOC_ARGS v, 3*16 +%if STACK_ALIGNMENT >= 16 + mov r5d, r7m +%endif + sar r5d, 7 + and r5d, 16 ; 0 for 10bpc, 16 for 12bpc + LEA pic_regq, PIC_base + mova m0, [PIC_sym(pw_4)+r5] +%define bdmulq esp+2*16 + mova [bdmulq], m0 + shl dword lstridem, 2 + sub r3, dword lstridem + mov dword lm, r3 +%endif mov mstrideq, strideq neg mstrideq lea stride3q, [strideq*3] +%if ARCH_X86_64 mov mask_bitsd, 0x3 mova m12, [pb_mask] +%else +%define mask_bitsm dword [esp+0*gprsize] +%define mstridem dword [esp+1*gprsize] + mov mask_bitsm, 0x3 + mov mstridem, mstrideq + mova m0, [PIC_sym(pb_mask)] +%define m12 [esp+1*16] + mova m12, m0 +%endif .loop: +%if ARCH_X86_64 test [maskq+4], mask_bitsd ; vmask[1] +%else + mov r6d, mask_bitsm + test [maskq+4], r6d +%endif jz .no_flat FILTER 6, v jmp .end .no_flat: +%if ARCH_X86_64 test [maskq+0], mask_bitsd ; vmask[0] +%else + test [maskq+0], r6d +%endif jz .end FILTER 4, v .end: +%if ARCH_X86_64 pslld m12, 2 add lq, 8 +%else + mova m0, m12 + pslld m0, 2 + mova m12, m0 + add dword lm, 8 +%endif add dstq, 16 +%if ARCH_X86_64 shl mask_bitsd, 2 sub wd, 2 +%else + shl mask_bitsm, 2 + sub dword wm, 2 +%endif jg .loop +%undef mask_bitsm +%undef bdmulq + UNRELOC_ARGS RET INIT_XMM ssse3 -cglobal lpf_h_sb_uv_16bpc, 6, 12, 16, \ +%if ARCH_X86_64 +cglobal lpf_h_sb_uv_16bpc, 6, 11, 16, \ dst, stride, mask, l, l_stride, lut, \ - h, stride3, l_stride3, tmp, mask_bits + h, stride3, tmp, mask_bits, bdmul mov r6d, r7m sar r6d, 7 and r6d, 16 ; 0 for 10bpc, 16 for 12bpc - lea r11, [pw_4] - add r11, r6 + lea bdmulq, [pw_4] + add bdmulq, r6 mov hd, hm shl l_strideq, 2 +%else +; stack layout [32bit only]: +; r0 - GPRs [mask_bitsm] +; r1 - m12/pb_mask +; r2 - bdmulq +; r3-8 - p2-q2 +cglobal lpf_h_sb_uv_16bpc, 4, 7, 8, -16 * (9 + extra_stack), \ + dst, stride, mask, l, pic_reg, stride3, tmp + RELOC_ARGS h, 9*16 +%if STACK_ALIGNMENT >= 16 + mov r5d, r7m +%endif + sar r5d, 7 + and r5d, 16 ; 0 for 10bpc, 16 for 12bpc + LEA pic_regq, PIC_base + mova m0, [PIC_sym(pw_4)+r5] +%define bdmulq esp+2*16 + mova [bdmulq], m0 + shl dword lstridem, 2 +%endif sub lq, 4 lea stride3q, [strideq*3] - lea l_stride3q, [l_strideq*3] +%if ARCH_X86_64 mov mask_bitsd, 0x3 mova m12, [pb_mask] +%else +%define mask_bitsm dword [esp+0*gprsize] + mov mask_bitsm, 0x3 + mova m0, [PIC_sym(pb_mask)] +%define m12 [esp+1*16] + mova m12, m0 +%endif .loop: +%if ARCH_X86_64 test [maskq+4], mask_bitsd ; vmask[1] +%else + mov r6d, mask_bitsm + test [maskq+4], r6d +%endif jz .no_flat FILTER 6, h jmp .end .no_flat: +%if ARCH_X86_64 test [maskq+0], mask_bitsd ; vmask[0] +%else + test [maskq+0], r6d +%endif jz .no_filter FILTER 4, h jmp .end .no_filter: lea dstq, [dstq+strideq*8] .end: +%if ARCH_X86_64 pslld m12, 2 lea lq, [lq+l_strideq*2] shl mask_bitsd, 2 sub hd, 2 +%else + mova m0, m12 + pslld m0, 2 + mova m12, m0 + add lq, dword lstridem + add lq, dword lstridem + shl mask_bitsm, 2 + sub dword hm, 2 +%endif jg .loop +%undef mask_bitsm +%undef bdmulq + UNRELOC_ARGS RET - -%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/loopfilter_avx2.asm b/third_party/dav1d/src/x86/loopfilter_avx2.asm --- a/third_party/dav1d/src/x86/loopfilter_avx2.asm +++ b/third_party/dav1d/src/x86/loopfilter_avx2.asm @@ -1,6 +1,6 @@ -; Copyright © 2018, VideoLAN and dav1d authors +; Copyright © 2018-2021, VideoLAN and dav1d authors ; Copyright © 2018, Two Orioles, LLC ; All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions are met: @@ -820,13 +820,11 @@ paddw m10, m8 paddw m11, m9 ; p6*5+p5*2+p4*2+p3*2+p2+p1+p0+q0+q1 pmulhrsw m8, m10, [pw_2048] pmulhrsw m9, m11, [pw_2048] packuswb m8, m9 - pand m8, m1 - pandn m9, m1, m7 - por m8, m9 + vpblendvb m8, m7, m8, m1 %ifidn %2, v mova [tmpq+stride3q], m8 ; p4 %else mova [rsp+14*32], m8 %endif @@ -848,13 +846,11 @@ paddw m10, m8 paddw m11, m2 ; p6*4+p5+p4*2+p3*2+p2*2+p1+p0+q0+q1+q2 pmulhrsw m8, m10, [pw_2048] pmulhrsw m9, m11, [pw_2048] packuswb m8, m9 - pand m8, m1 - pandn m9, m1, m12 - por m8, m9 + vpblendvb m8, m12, m8, m1 %ifidn %2, v mova [tmpq+strideq*4], m8 ; p3 %else mova [rsp+19*32], m8 %endif @@ -876,13 +872,11 @@ paddw m10, m8 paddw m11, m7 ; p6*3+p5+p4+p3*2+p2*2+p1*2+p0+q0+q1+q2+q3 pmulhrsw m8, m10, [pw_2048] pmulhrsw m9, m11, [pw_2048] packuswb m8, m9 - pand m8, m1 - pandn m9, m1, m13 - por m8, m9 + vpblendvb m8, m13, m8, m1 mova [rsp+6*32], m8 ; don't clobber p2/m13 since we need it in F ; sub p6/p3, add p0/q4 [-p6,+p0][-p3,+q4|save] E ; write -2 %ifidn %2, v @@ -908,13 +902,11 @@ paddw m10, m8 paddw m11, m9 ; p6*2+p5+p4+p3+p2*2+p1*2+p0*2+q0+q1+q2+q3+q4 pmulhrsw m8, m10, [pw_2048] pmulhrsw m9, m11, [pw_2048] packuswb m8, m9 - pand m8, m1 - pandn m9, m1, m3 - por m8, m9 + vpblendvb m8, m3, m8, m1 mova [rsp+8*32], m8 ; don't clobber p1/m3 since we need it in G ; sub p6/p2, add q0/q5 [-p6,+q0][-p2,+q5|save] F ; write -1 %ifidn %2, v @@ -938,13 +930,11 @@ paddw m10, m0 paddw m11, m9 ; p6+p5+p4+p3+p2+p1*2+p0*2+q0*2+q1+q2+q3+q4+q5 pmulhrsw m0, m10, [pw_2048] pmulhrsw m8, m11, [pw_2048] packuswb m0, m8 - pand m0, m1 - pandn m8, m1, m4 - por m0, m8 + vpblendvb m0, m4, m0, m1 mova [rsp+6*32], m0 ; don't clobber p0/m4 since we need it in H ; sub p6/p1, add q1/q6 [reuse -p6,+q1 from B][-p1,+q6|save] G ; write +0 %ifidn %2, v @@ -964,13 +954,11 @@ paddw m10, m8 paddw m11, m9 ; p5+p4+p3+p2+p1+p0*2+q0*2+q1*2+q2+q3+q4+q5+q6 pmulhrsw m8, m10, [pw_2048] pmulhrsw m9, m11, [pw_2048] packuswb m8, m9 - pand m8, m1 - pandn m9, m1, m5 - por m8, m9 + vpblendvb m8, m5, m8, m1 mova [rsp+8*32], m8 ; don't clobber q0/m5 since we need it in I ; sub p5/p0, add q2/q6 [reuse -p5,+q2 from C][-p0,+q6] H ; write +1 paddw m10, [rsp+1*32] @@ -983,13 +971,11 @@ paddw m10, m8 paddw m11, m2 ; p4+p3+p2+p1+p0+q0*2+q1*2+q2*2+q3+q4+q5+q6*2 pmulhrsw m2, m10, [pw_2048] pmulhrsw m9, m11, [pw_2048] packuswb m2, m9 - pand m2, m1 - pandn m9, m1, m6 - por m2, m9 ; don't clobber q1/m6 since we need it in K + vpblendvb m2, m6, m2, m1 ; don't clobber q1/m6 since we need it in K ; sub p4/q0, add q3/q6 [reuse -p4,+q3 from D][-q0,+q6] I ; write +2 paddw m10, [rsp+2*32] paddw m11, m7 ; p3+p2+p1+p0+q0*2+q1*2+q2*2+q3*2+q4+q5+q6*2 @@ -1001,13 +987,11 @@ paddw m10, m8 paddw m11, m9 ; p3+p2+p1+p0+q0+q1*2+q2*2+q3*2+q4+q5+q6*3 pmulhrsw m7, m10, [pw_2048] pmulhrsw m9, m11, [pw_2048] packuswb m7, m9 - pand m7, m1 - pandn m9, m1, m14 - por m7, m9 ; don't clobber q2/m14 since we need it in K + vpblendvb m7, m14, m7, m1 ; don't clobber q2/m14 since we need it in K ; sub p3/q1, add q4/q6 [reuse -p3,+q4 from E][-q1,+q6] J ; write +3 paddw m10, [rsp+7*32] paddw m11, [rsp+5*32] ; p2+p1+p0+q0+q1*2+q2*2+q3*2+q4*2+q5+q6*3 @@ -1019,13 +1003,11 @@ paddw m10, m8 paddw m11, m9 ; p2+p1+p0+q0+q1+q2*2+q3*2+q4*2+q5+q6*4 pmulhrsw m8, m10, [pw_2048] pmulhrsw m9, m11, [pw_2048] packuswb m8, m9 - pand m8, m1 - pandn m9, m1, m15 - por m8, m9 + vpblendvb m8, m15, m8, m1 %ifidn %2, v mova [tmpq+mstrideq], m8 ; q3 %else mova [rsp+20*32], m8 %endif @@ -1042,17 +1024,16 @@ paddw m10, m8 paddw m11, m9 ; p1+p0+q0+q1+q2+q3*2+q4*2+q5*2+q6*5 pmulhrsw m8, m10, [pw_2048] pmulhrsw m9, m11, [pw_2048] packuswb m8, m9 - pand m8, m1 %ifidn %2, v - pandn m9, m1, [tmpq+strideq*0] + mova m9, [tmpq+strideq*0] %else - pandn m9, m1, [rsp+15*32] + mova m9, [rsp+15*32] %endif - por m8, m9 + vpblendvb m8, m9, m8, m1 %ifidn %2, v mova [tmpq+strideq*0], m8 ; q4 %else mova [rsp+15*32], m8 %endif @@ -1068,17 +1049,16 @@ paddw m10, m8 paddw m11, m9 ; p1+p0+q0+q1+q2+q3*2+q4*2+q5*2+q6*5 pmulhrsw m10, [pw_2048] pmulhrsw m11, [pw_2048] packuswb m10, m11 - pand m10, m1 %ifidn %2, v - pandn m11, m1, [tmpq+strideq*1] + mova m11, [tmpq+strideq*1] %else - pandn m11, m1, [rsp+16*32] + mova m11, [rsp+16*32] %endif - por m10, m11 + vpblendvb m10, m11, m10, m1 %ifidn %2, v mova [tmpq+strideq*1], m10 ; q5 %else mova [rsp+16*32], m10 %endif @@ -1107,13 +1087,11 @@ paddw m2, m8 paddw m7, m11 ; 3 * p3 + 2 * p2 + p1 + p0 + q0 + 4 psrlw m8, m2, 3 psrlw m11, m7, 3 packuswb m8, m11 - pand m8, m9 - pandn m11, m9, m13 - por m10, m8, m11 ; p2 + vpblendvb m10, m13, m8, m9 ; p2 %ifidn %2, v mova [tmpq+strideq*1], m10 ; p2 %endif pmaddubsw m8, m0, [pb_m1_1] @@ -1127,13 +1105,11 @@ paddw m2, m8 paddw m7, m11 ; 2 * p3 + p2 + 2 * p1 + p0 + q0 + q1 + 4 psrlw m8, m2, 3 psrlw m11, m7, 3 packuswb m8, m11 - pand m8, m9 - pandn m11, m9, m3 - por m8, m11 ; p1 + vpblendvb m8, m3, m8, m9 ; p1 %ifidn %2, v mova [tmpq+strideq*2], m8 ; p1 %else mova [rsp+0*32], m8 %endif @@ -1149,13 +1125,11 @@ paddw m2, m8 paddw m7, m11 ; p3 + p2 + p1 + 2 * p0 + q0 + q1 + q2 + 4 psrlw m8, m2, 3 psrlw m11, m7, 3 packuswb m8, m11 - pand m8, m9 - pandn m11, m9, m4 - por m8, m11 ; p0 + vpblendvb m8, m4, m8, m9 ; p0 %ifidn %2, v mova [tmpq+stride3q ], m8 ; p0 %else mova [rsp+1*32], m8 %endif @@ -1173,13 +1147,11 @@ psubw m2, m8 psubw m7, m11 ; p2 + p1 + p0 + 2 * q0 + q1 + q2 + q3 + 4 psrlw m8, m2, 3 psrlw m11, m7, 3 packuswb m8, m11 - pand m8, m9 - pandn m11, m9, m5 - por m11, m8, m11 ; q0 + vpblendvb m11, m5, m8, m9 ; q0 %ifidn %2, v mova [dstq+strideq*0], m11 ; q0 %endif pmaddubsw m0, [pb_m1_1] @@ -1193,13 +1165,11 @@ paddw m2, m8 paddw m7, m13 ; p1 + p0 + q0 + 2 * q1 + q2 + 2 * q3 + 4 psrlw m8, m2, 3 psrlw m13, m7, 3 packuswb m8, m13 - pand m8, m9 - pandn m13, m9, m6 - por m13, m8, m13 ; q1 + vpblendvb m13, m6, m8, m9 ; q1 %ifidn %2, v mova [dstq+strideq*1], m13 ; q1 %endif punpcklbw m0, m3, m6 @@ -1215,13 +1185,11 @@ paddw m2, m0 paddw m7, m1 ; p0 + q0 + q1 + q2 + 2 * q2 + 3 * q3 + 4 psrlw m2, 3 psrlw m7, 3 packuswb m2, m7 - pand m2, m9 - pandn m7, m9, m14 - por m2, m7 ; q2 + vpblendvb m2, m14, m2, m9 ; q2 %ifidn %2, v mova [dstq+strideq*2], m2 ; q2 %else mova m0, [rsp+0*32] mova m1, [rsp+1*32] @@ -1378,13 +1346,11 @@ paddw m0, m2 paddw m1, m12 pmulhrsw m2, m0, [pw_4096] pmulhrsw m12, m1, [pw_4096] packuswb m2, m12 - pand m2, m9 - pandn m12, m9, m3 - por m2, m12 + vpblendvb m2, m3, m2, m9 %ifidn %2, v mova [tmpq+strideq*2], m2 ; p1 %endif pmaddubsw m8, [pb_m1_1] @@ -1398,13 +1364,11 @@ paddw m0, m8 paddw m1, m11 pmulhrsw m12, m0, [pw_4096] pmulhrsw m13, m1, [pw_4096] packuswb m12, m13 - pand m12, m9 - pandn m13, m9, m4 - por m12, m13 + vpblendvb m12, m4, m12, m9 %ifidn %2, v mova [tmpq+stride3q], m12 ; p0 %endif paddw m0, m8 @@ -1416,13 +1380,11 @@ paddw m0, m14 paddw m1, m13 pmulhrsw m14, m0, [pw_4096] pmulhrsw m13, m1, [pw_4096] packuswb m14, m13 - pand m14, m9 - pandn m13, m9, m5 - por m14, m13 + vpblendvb m14, m5, m14, m9 %ifidn %2, v mova [dstq+strideq*0], m14 ; q0 %endif pmaddubsw m8, [pb_m1_2] @@ -1434,13 +1396,11 @@ paddw m0, m7 paddw m1, m10 pmulhrsw m0, [pw_4096] pmulhrsw m1, [pw_4096] packuswb m0, m1 - pand m0, m9 - pandn m9, m6 - por m0, m9 + vpblendvb m0, m6, m0, m9 %ifidn %2, v mova [dstq+strideq*1], m0 ; q1 %else TRANSPOSE_16x4_AND_WRITE_4x32 2, 12, 14, 0, 1 %endif @@ -1455,11 +1415,11 @@ %endif %endif %endmacro INIT_YMM avx2 -cglobal lpf_v_sb_y, 7, 10, 16, 32 * 11, \ +cglobal lpf_v_sb_y_8bpc, 7, 10, 16, 32 * 11, \ dst, stride, mask, l, l_stride, lut, \ w, stride3, mstride, tmp shl l_strideq, 2 sub lq, l_strideq mov mstrideq, strideq @@ -1493,11 +1453,11 @@ sub wd, 8 jg .loop RET INIT_YMM avx2 -cglobal lpf_h_sb_y, 7, 10, 16, 32 * 21, \ +cglobal lpf_h_sb_y_8bpc, 7, 10, 16, 32 * 21, \ dst, stride, mask, l, l_stride, lut, \ h, stride3, l_stride3, tmp shl l_strideq, 2 sub lq, 4 lea stride3q, [strideq*3] @@ -1533,11 +1493,11 @@ sub hd, 8 jg .loop RET INIT_YMM avx2 -cglobal lpf_v_sb_uv, 7, 10, 16, \ +cglobal lpf_v_sb_uv_8bpc, 7, 10, 16, \ dst, stride, mask, l, l_stride, lut, \ w, stride3, mstride, tmp shl l_strideq, 2 sub lq, l_strideq mov mstrideq, strideq @@ -1564,11 +1524,11 @@ sub wd, 8 jg .loop RET INIT_YMM avx2 -cglobal lpf_h_sb_uv, 7, 10, 16, \ +cglobal lpf_h_sb_uv_8bpc, 7, 10, 16, \ dst, stride, mask, l, l_stride, lut, \ h, stride3, l_stride3, tmp shl l_strideq, 2 sub lq, 4 lea stride3q, [strideq*3] diff --git a/third_party/dav1d/src/x86/loopfilter_init_tmpl.c b/third_party/dav1d/src/x86/loopfilter_init_tmpl.c --- a/third_party/dav1d/src/x86/loopfilter_init_tmpl.c +++ b/third_party/dav1d/src/x86/loopfilter_init_tmpl.c @@ -1,7 +1,7 @@ /* - * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2018-2021, VideoLAN and dav1d authors * Copyright © 2018, Two Orioles, LLC * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -27,50 +27,32 @@ #include "src/cpu.h" #include "src/loopfilter.h" #define decl_loopfilter_sb_fns(ext) \ -decl_loopfilter_sb_fn(dav1d_lpf_h_sb_y_##ext); \ -decl_loopfilter_sb_fn(dav1d_lpf_v_sb_y_##ext); \ -decl_loopfilter_sb_fn(dav1d_lpf_h_sb_uv_##ext); \ -decl_loopfilter_sb_fn(dav1d_lpf_v_sb_uv_##ext) +decl_loopfilter_sb_fn(BF(dav1d_lpf_h_sb_y, ext)); \ +decl_loopfilter_sb_fn(BF(dav1d_lpf_v_sb_y, ext)); \ +decl_loopfilter_sb_fn(BF(dav1d_lpf_h_sb_uv, ext)); \ +decl_loopfilter_sb_fn(BF(dav1d_lpf_v_sb_uv, ext)) decl_loopfilter_sb_fns(ssse3); decl_loopfilter_sb_fns(avx2); -decl_loopfilter_sb_fns(16bpc_ssse3); -decl_loopfilter_sb_fns(16bpc_avx2); COLD void bitfn(dav1d_loop_filter_dsp_init_x86)(Dav1dLoopFilterDSPContext *const c) { const unsigned flags = dav1d_get_cpu_flags(); if (!(flags & DAV1D_X86_CPU_FLAG_SSSE3)) return; -#if BITDEPTH == 8 - c->loop_filter_sb[0][0] = dav1d_lpf_h_sb_y_ssse3; - c->loop_filter_sb[0][1] = dav1d_lpf_v_sb_y_ssse3; - c->loop_filter_sb[1][0] = dav1d_lpf_h_sb_uv_ssse3; - c->loop_filter_sb[1][1] = dav1d_lpf_v_sb_uv_ssse3; -#else -#if ARCH_X86_64 - c->loop_filter_sb[0][0] = dav1d_lpf_h_sb_y_16bpc_ssse3; - c->loop_filter_sb[0][1] = dav1d_lpf_v_sb_y_16bpc_ssse3; - c->loop_filter_sb[1][0] = dav1d_lpf_h_sb_uv_16bpc_ssse3; - c->loop_filter_sb[1][1] = dav1d_lpf_v_sb_uv_16bpc_ssse3; -#endif -#endif + c->loop_filter_sb[0][0] = BF(dav1d_lpf_h_sb_y, ssse3); + c->loop_filter_sb[0][1] = BF(dav1d_lpf_v_sb_y, ssse3); + c->loop_filter_sb[1][0] = BF(dav1d_lpf_h_sb_uv, ssse3); + c->loop_filter_sb[1][1] = BF(dav1d_lpf_v_sb_uv, ssse3); if (!(flags & DAV1D_X86_CPU_FLAG_AVX2)) return; #if ARCH_X86_64 -#if BITDEPTH == 8 - c->loop_filter_sb[0][0] = dav1d_lpf_h_sb_y_avx2; - c->loop_filter_sb[0][1] = dav1d_lpf_v_sb_y_avx2; - c->loop_filter_sb[1][0] = dav1d_lpf_h_sb_uv_avx2; - c->loop_filter_sb[1][1] = dav1d_lpf_v_sb_uv_avx2; -#else - c->loop_filter_sb[0][0] = dav1d_lpf_h_sb_y_16bpc_avx2; - c->loop_filter_sb[0][1] = dav1d_lpf_v_sb_y_16bpc_avx2; - c->loop_filter_sb[1][0] = dav1d_lpf_h_sb_uv_16bpc_avx2; - c->loop_filter_sb[1][1] = dav1d_lpf_v_sb_uv_16bpc_avx2; -#endif + c->loop_filter_sb[0][0] = BF(dav1d_lpf_h_sb_y, avx2); + c->loop_filter_sb[0][1] = BF(dav1d_lpf_v_sb_y, avx2); + c->loop_filter_sb[1][0] = BF(dav1d_lpf_h_sb_uv, avx2); + c->loop_filter_sb[1][1] = BF(dav1d_lpf_v_sb_uv, avx2); #endif } diff --git a/third_party/dav1d/src/x86/loopfilter_sse.asm b/third_party/dav1d/src/x86/loopfilter_sse.asm --- a/third_party/dav1d/src/x86/loopfilter_sse.asm +++ b/third_party/dav1d/src/x86/loopfilter_sse.asm @@ -1,6 +1,6 @@ -; Copyright © 2018, VideoLAN and dav1d authors +; Copyright © 2018-2021, VideoLAN and dav1d authors ; Copyright © 2018, Two Orioles, LLC ; All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions are met: @@ -1975,15 +1975,15 @@ %define l_stride3q r6 %endif INIT_XMM ssse3 %if ARCH_X86_64 -cglobal lpf_v_sb_y, 7, 11, 16, 16 * 15, \ +cglobal lpf_v_sb_y_8bpc, 7, 11, 16, 16 * 15, \ dst, stride, mask, l, l_stride, lut, \ w, stride3, mstride, tmp, mask_bits %else -cglobal lpf_v_sb_y, 6, 7, 8, -16 * (26 + copy_args), \ +cglobal lpf_v_sb_y_8bpc, 6, 7, 8, -16 * (26 + copy_args), \ dst, stride, mask, l, l_stride, lut, mask_bits RELOC_ARGS w SETUP_PIC %define m12 m5 %endif @@ -2073,15 +2073,15 @@ jg .loop RET INIT_XMM ssse3 %if ARCH_X86_64 -cglobal lpf_h_sb_y, 7, 11, 16, 16 * 26, \ +cglobal lpf_h_sb_y_8bpc, 7, 11, 16, 16 * 26, \ dst, stride, mask, l, l_stride, lut, \ h, stride3, l_stride3, tmp, mask_bits %else -cglobal lpf_h_sb_y, 6, 7, 8, -16 * (39 + copy_args), \ +cglobal lpf_h_sb_y_8bpc, 6, 7, 8, -16 * (39 + copy_args), \ dst, stride, mask, l, l_stride, lut, mask_bits RELOC_ARGS h SETUP_PIC %define m12 m5 %endif @@ -2177,15 +2177,15 @@ jg .loop RET INIT_XMM ssse3 %if ARCH_X86_64 -cglobal lpf_v_sb_uv, 7, 11, 16, 3 * 16, \ +cglobal lpf_v_sb_uv_8bpc, 7, 11, 16, 3 * 16, \ dst, stride, mask, l, l_stride, lut, \ w, stride3, mstride, tmp, mask_bits %else -cglobal lpf_v_sb_uv, 6, 7, 8, -16 * (12 + copy_args), \ +cglobal lpf_v_sb_uv_8bpc, 6, 7, 8, -16 * (12 + copy_args), \ dst, stride, mask, l, l_stride, lut, mask_bits RELOC_ARGS w SETUP_PIC %define m12 m4 %endif @@ -2259,15 +2259,15 @@ jg .loop RET INIT_XMM ssse3 %if ARCH_X86_64 -cglobal lpf_h_sb_uv, 7, 11, 16, 16 * 3, \ +cglobal lpf_h_sb_uv_8bpc, 7, 11, 16, 16 * 3, \ dst, stride, mask, l, l_stride, lut, \ h, stride3, l_stride3, tmp, mask_bits %else -cglobal lpf_h_sb_uv, 6, 7, 8, -16 * (13 + copy_args), \ +cglobal lpf_h_sb_uv_8bpc, 6, 7, 8, -16 * (13 + copy_args), \ dst, stride, mask, l, l_stride, lut, mask_bits RELOC_ARGS h SETUP_PIC %define m12 m4 %endif diff --git a/third_party/dav1d/src/x86/looprestoration16_avx2.asm b/third_party/dav1d/src/x86/looprestoration16_avx2.asm --- a/third_party/dav1d/src/x86/looprestoration16_avx2.asm +++ b/third_party/dav1d/src/x86/looprestoration16_avx2.asm @@ -660,11 +660,11 @@ mova [dstq+r10], m0 add r10, 32 jl .v_loop ret -cglobal sgr_filter_5x5_16bpc, 5, 14, 16, 400*24+16, dst, dst_stride, left, lpf, \ +cglobal sgr_filter_5x5_16bpc, 5, 14, 15, 400*24+16, dst, dst_stride, left, lpf, \ lpf_stride, w, edge, params, h movifnidn wd, wm mov paramsq, paramsmp lea r13, [sgr_x_by_x_avx2+256*4] mov edged, r8m @@ -678,17 +678,16 @@ add dstq, wq vpbroadcastd m10, [paramsq+0] ; s0 lea t3, [rsp+wq*2+400*12+16] vpbroadcastd m11, [pd_0xf00800a4] lea t4, [rsp+wq+400*20+16] - vpbroadcastd m12, [pw_256] + mova xm12, [sgr_lshuf5] neg wq vpbroadcastd m13, [pd_34816] ; (1 << 11) + (1 << 15) pxor m6, m6 vpbroadcastd m14, [pw_1023] psllw m7, 4 - mova xm15, [sgr_lshuf5] test edgeb, 4 ; LR_HAVE_TOP jz .no_top call .h_top add lpfq, lpf_strideq mov t2, t1 @@ -784,11 +783,11 @@ add leftq, 8 palignr m4, m5, 10 jmp .h_main .h_extend_left: mova xm4, [lpfq+wq] - pshufb xm4, xm15 + pshufb xm4, xm12 vinserti128 m4, [lpfq+wq+10], 1 jmp .h_main .h_top: lea r10, [wq-4] test edgeb, 1 ; LR_HAVE_LEFT @@ -865,11 +864,11 @@ add leftq, 8 palignr m4, m5, 10 jmp .hv_main .hv_extend_left: mova xm4, [lpfq+wq] - pshufb xm4, xm15 + pshufb xm4, xm12 vinserti128 m4, [lpfq+wq+10], 1 jmp .hv_main .hv_bottom: lea r10, [wq-4] test edgeb, 1 ; LR_HAVE_LEFT @@ -943,17 +942,16 @@ pmaddwd m0, m11 ; b * 164 pmaddwd m1, m11 paddusw m4, m11 paddusw m5, m11 psrad m3, m4, 20 ; min(z, 255) - 256 - vpgatherdd m2, [r13+m3*4], m4 + vpgatherdd m2, [r13+m3*4], m4 ; x psrad m4, m5, 20 vpgatherdd m3, [r13+m4*4], m5 pmulld m0, m2 pmulld m1, m3 packssdw m2, m3 - psubw m2, m12, m2 ; a paddd m0, m13 ; x * b * 164 + (1 << 11) + (1 << 15) paddd m1, m13 mova [t4+r10+4], m2 psrld m0, 12 ; b psrld m1, 12 @@ -1013,17 +1011,16 @@ pmaddwd m0, m11 ; b * 164 pmaddwd m1, m11 paddusw m4, m11 paddusw m5, m11 psrad m3, m4, 20 ; min(z, 255) - 256 - vpgatherdd m2, [r13+m3*4], m4 + vpgatherdd m2, [r13+m3*4], m4 ; x psrad m4, m5, 20 vpgatherdd m3, [r13+m4*4], m5 pmulld m0, m2 pmulld m1, m3 packssdw m2, m3 - psubw m2, m12, m2 ; a paddd m0, m13 ; x * b * 164 + (1 << 11) + (1 << 15) paddd m1, m13 mova [t4+r10+4], m2 psrld m0, 12 ; b psrld m1, 12 @@ -1096,19 +1093,17 @@ punpckhwd m1, m0, m6 punpckhwd m3, m6 pmaddwd m3, m1 vinserti128 m1, m4, xm5, 1 vperm2i128 m4, m5, 0x31 - paddd m2, m1 ; a * src + b + (1 << 8) - paddd m3, m4 - psrld m2, 9 - psrld m3, 9 - packssdw m2, m3 - psllw m1, m0, 4 - psubw m2, m1 - pmulhrsw m2, m7 - paddw m0, m2 + psubd m1, m2 ; b - a * src + (1 << 8) + psubd m4, m3 + psrad m1, 9 + psrad m4, 9 + packssdw m1, m4 + pmulhrsw m1, m7 + paddw m0, m1 pmaxsw m0, m6 pminsw m0, m14 mova [dstq+r10], m0 add r10, 32 jl .n0_loop @@ -1128,28 +1123,26 @@ punpckhwd m1, m0, m6 punpckhwd m3, m6 pmaddwd m3, m1 vinserti128 m1, m4, xm5, 1 vperm2i128 m4, m5, 0x31 - paddd m2, m1 ; a * src + b + (1 <<7) - paddd m3, m4 - psrld m2, 8 - psrld m3, 8 - packssdw m2, m3 - psllw m1, m0, 4 - psubw m2, m1 - pmulhrsw m2, m7 - paddw m0, m2 + psubd m1, m2 ; b - a * src + (1 << 7) + psubd m4, m3 + psrad m1, 8 + psrad m4, 8 + packssdw m1, m4 + pmulhrsw m1, m7 + paddw m0, m1 pmaxsw m0, m6 pminsw m0, m14 mova [dstq+r10], m0 add r10, 32 jl .n1_loop add dstq, dst_strideq ret -cglobal sgr_filter_3x3_16bpc, 5, 14, 15, 400*42+8, dst, dst_stride, left, lpf, \ +cglobal sgr_filter_3x3_16bpc, 5, 14, 14, 400*42+8, dst, dst_stride, left, lpf, \ lpf_stride, w, edge, params, h movifnidn wd, wm mov paramsq, paramsmp lea r13, [sgr_x_by_x_avx2+256*4] mov edged, r8m @@ -1164,15 +1157,14 @@ lea t3, [rsp+wq*2+400*12+8] vpbroadcastd m10, [pd_0xf00801c7] lea t4, [rsp+wq+400*32+8] vpbroadcastd m11, [pd_34816] neg wq - vpbroadcastd m12, [pw_256] + mova xm12, [sgr_lshuf3] pxor m6, m6 vpbroadcastd m13, [pw_1023] psllw m7, 4 - mova xm14, [sgr_lshuf3] test edgeb, 4 ; LR_HAVE_TOP jz .no_top call .h_top add lpfq, lpf_strideq mov t2, t1 @@ -1266,11 +1258,11 @@ add leftq, 8 palignr m4, m5, 12 jmp .h_main .h_extend_left: mova xm4, [lpfq+wq] - pshufb xm4, xm14 + pshufb xm4, xm12 vinserti128 m4, [lpfq+wq+12], 1 jmp .h_main .h_top: lea r10, [wq-4] test edgeb, 1 ; LR_HAVE_LEFT @@ -1316,11 +1308,11 @@ add leftq, 8 palignr m4, m5, 12 jmp .hv0_main .hv0_extend_left: mova xm4, [lpfq+wq] - pshufb xm4, xm14 + pshufb xm4, xm12 vinserti128 m4, [lpfq+wq+12], 1 jmp .hv0_main .hv0_bottom: lea r10, [wq-4] test edgeb, 1 ; LR_HAVE_LEFT @@ -1386,19 +1378,18 @@ pmaddwd m0, m10 ; b * 455 pmaddwd m1, m10 paddusw m4, m10 paddusw m5, m10 psrad m3, m4, 20 ; min(z, 255) - 256 - vpgatherdd m2, [r13+m3*4], m4 + vpgatherdd m2, [r13+m3*4], m4 ; x psrad m4, m5, 20 vpgatherdd m3, [r13+m4*4], m5 pmulld m0, m2 pmulld m1, m3 packssdw m2, m3 paddd m0, m11 ; x * b * 455 + (1 << 11) + (1 << 15) paddd m1, m11 - psubw m2, m12, m2 psrld m0, 12 psrld m1, 12 mova [t4+r10*1+400*0+ 4], m2 mova [t3+r10*2+400*0+ 8], xm0 vextracti128 [t3+r10*2+400*0+40], m0, 1 @@ -1418,11 +1409,11 @@ add leftq, 8 palignr m4, m5, 12 jmp .hv1_main .hv1_extend_left: mova xm4, [lpfq+wq] - pshufb xm4, xm14 + pshufb xm4, xm12 vinserti128 m4, [lpfq+wq+12], 1 jmp .hv1_main .hv1_bottom: lea r10, [wq-4] test edgeb, 1 ; LR_HAVE_LEFT @@ -1482,19 +1473,18 @@ pmaddwd m0, m10 ; b * 455 pmaddwd m1, m10 paddusw m4, m10 paddusw m5, m10 psrad m3, m4, 20 ; min(z, 255) - 256 - vpgatherdd m2, [r13+m3*4], m4 + vpgatherdd m2, [r13+m3*4], m4 ; x psrad m4, m5, 20 vpgatherdd m3, [r13+m4*4], m5 pmulld m0, m2 pmulld m1, m3 packssdw m2, m3 paddd m0, m11 ; x * b * 455 + (1 << 11) + (1 << 15) paddd m1, m11 - psubw m2, m12, m2 psrld m0, 12 psrld m1, 12 mova [t4+r10*1+400*2 +4], m2 mova [t3+r10*2+400*4+ 8], xm0 vextracti128 [t3+r10*2+400*4+40], m0, 1 @@ -1546,19 +1536,18 @@ pmaddwd m0, m10 ; b * 455 pmaddwd m1, m10 paddusw m4, m10 paddusw m5, m10 psrad m3, m4, 20 ; min(z, 255) - 256 - vpgatherdd m2, [r13+m3*4], m4 + vpgatherdd m2, [r13+m3*4], m4 ; x psrad m4, m5, 20 vpgatherdd m3, [r13+m4*4], m5 pmulld m0, m2 pmulld m1, m3 packssdw m2, m3 paddd m0, m11 ; x * b * 455 + (1 << 11) + (1 << 15) paddd m1, m11 - psubw m2, m12, m2 psrld m0, 12 psrld m1, 12 mova [t4+r10*1+400*0+ 4], m2 mova [t3+r10*2+400*0+ 8], xm0 vextracti128 [t3+r10*2+400*0+40], m0, 1 @@ -1604,19 +1593,18 @@ pmaddwd m0, m10 ; b * 455 pmaddwd m1, m10 paddusw m4, m10 paddusw m5, m10 psrad m3, m4, 20 ; min(z, 255) - 256 - vpgatherdd m2, [r13+m3*4], m4 + vpgatherdd m2, [r13+m3*4], m4 ; x psrad m4, m5, 20 vpgatherdd m3, [r13+m4*4], m5 pmulld m0, m2 pmulld m1, m3 packssdw m2, m3 paddd m0, m11 ; x * b * 455 + (1 << 11) + (1 << 15) paddd m1, m11 - psubw m2, m12, m2 psrld m0, 12 psrld m1, 12 mova [t4+r10*1+400*2+ 4], m2 mova [t3+r10*2+400*4+ 8], xm0 vextracti128 [t3+r10*2+400*4+40], m0, 1 @@ -1698,19 +1686,17 @@ punpckhwd m1, m0, m6 punpckhwd m3, m6 pmaddwd m3, m1 vinserti128 m1, m4, xm5, 1 vperm2i128 m4, m5, 0x31 - paddd m2, m1 ; a * src + b + (1 << 8) - paddd m3, m4 - psrld m2, 9 - psrld m3, 9 - packssdw m2, m3 - psllw m1, m0, 4 - psubw m2, m1 - pmulhrsw m2, m7 - paddw m0, m2 + psubd m1, m2 ; b - a * src + (1 << 8) + psubd m4, m3 + psrad m1, 9 + psrad m4, 9 + packssdw m1, m4 + pmulhrsw m1, m7 + paddw m0, m1 pmaxsw m0, m6 pminsw m0, m13 mova [dstq+r10], m0 add r10, 32 jl .n0_loop @@ -1754,19 +1740,17 @@ punpckhwd m1, m0, m6 punpckhwd m3, m6 pmaddwd m3, m1 vinserti128 m1, m4, xm5, 1 vperm2i128 m4, m5, 0x31 - paddd m2, m1 ; a * src + b + (1 << 8) - paddd m3, m4 - psrld m2, 9 - psrld m3, 9 - packssdw m2, m3 - psllw m1, m0, 4 - psubw m2, m1 - pmulhrsw m2, m7 - paddw m0, m2 + psubd m1, m2 ; b - a * src + (1 << 8) + psubd m4, m3 + psrad m1, 9 + psrad m4, 9 + packssdw m1, m4 + pmulhrsw m1, m7 + paddw m0, m1 pmaxsw m0, m6 pminsw m0, m13 mova [dstq+r10], m0 add r10, 32 jl .n1_loop @@ -1784,11 +1768,11 @@ add lpfq, wq vpbroadcastd m9, [pd_8] lea t1, [rsp+wq+12] vpbroadcastd m10, [pd_34816] add dstq, wq - vpbroadcastd m11, [pw_256] + vpbroadcastd m11, [pd_4096] lea t3, [rsp+wq*2+400*24+8] vpbroadcastd m12, [pd_0xf00801c7] lea t4, [rsp+wq+400*52+8] vpbroadcastd m15, [paramsq+8] ; w0 w1 neg wq @@ -2046,19 +2030,18 @@ pmaddwd m0, m12 ; b3 * 455 pmaddwd m1, m12 paddusw m4, m12 paddusw m5, m12 psrad m3, m4, 20 ; min(z3, 255) - 256 - vpgatherdd m2, [r13+m3*4], m4 + vpgatherdd m2, [r13+m3*4], m4 ; x3 psrad m4, m5, 20 vpgatherdd m3, [r13+m4*4], m5 pmulld m0, m2 pmulld m1, m3 packssdw m2, m3 paddd m0, m10 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) paddd m1, m10 - psubw m2, m11, m2 psrld m0, 12 psrld m1, 12 mova [t4+r10*1+400*2+ 4], m2 mova [t3+r10*2+400*4+ 8], xm0 vextracti128 [t3+r10*2+400*4+40], m0, 1 @@ -2152,19 +2135,18 @@ pmaddwd m0, m12 ; b3 * 455 pmaddwd m1, m12 paddusw m2, m12 paddusw m3, m12 psrad m7, m2, 20 ; min(z3, 255) - 256 - vpgatherdd m6, [r13+m7*4], m2 + vpgatherdd m6, [r13+m7*4], m2 ; x3 psrad m2, m3, 20 vpgatherdd m7, [r13+m2*4], m3 pmulld m0, m6 packssdw m6, m7 pmulld m7, m1 paddd m0, m10 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) paddd m7, m10 - psubw m6, m11, m6 psrld m0, 12 psrld m7, 12 paddw m1, m8, [t2+r10+400*0] paddd m2, m4, [t2+r10+400*2] paddd m3, m5, [t2+r10+400*4] @@ -2205,19 +2187,18 @@ pmaddwd m0, m4 ; b5 * 164 pmaddwd m1, m4 paddusw m2, m4 paddusw m3, m4 psrad m5, m2, 20 ; min(z5, 255) - 256 - vpgatherdd m4, [r13+m5*4], m2 + vpgatherdd m4, [r13+m5*4], m2 ; x5 psrad m2, m3, 20 vpgatherdd m5, [r13+m2*4], m3 pmulld m0, m4 pmulld m1, m5 packssdw m4, m5 paddd m0, m10 ; x5 * b5 * 164 + (1 << 11) + (1 << 15) paddd m1, m10 - psubw m4, m11, m4 psrld m0, 12 psrld m1, 12 mova [t4+r10*1+400*0+ 4], m4 mova [t3+r10*2+400*0+ 8], xm0 vextracti128 [t3+r10*2+400*0+40], m0, 1 @@ -2269,19 +2250,18 @@ pmaddwd m0, m12 ; b3 * 455 pmaddwd m1, m12 paddusw m4, m12 paddusw m5, m12 psrad m3, m4, 20 ; min(z3, 255) - 256 - vpgatherdd m2, [r13+m3*4], m4 + vpgatherdd m2, [r13+m3*4], m4 ; x3 psrad m4, m5, 20 vpgatherdd m3, [r13+m4*4], m5 pmulld m0, m2 pmulld m1, m3 packssdw m2, m3 paddd m0, m10 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) paddd m1, m10 - psubw m2, m11, m2 psrld m0, 12 psrld m1, 12 mova m3, [t1+r10+400*0] mova m4, [t1+r10+400*2] mova m5, [t1+r10+400*4] @@ -2339,19 +2319,18 @@ pmaddwd m0, m12 ; b3 * 455 pmaddwd m1, m12 paddusw m4, m12 paddusw m5, m12 psrad m3, m4, 20 ; min(z3, 255) - 256 - vpgatherdd m2, [r13+m3*4], m4 + vpgatherdd m2, [r13+m3*4], m4 ; x3 psrad m4, m5, 20 vpgatherdd m3, [r13+m4*4], m5 pmulld m0, m2 pmulld m1, m3 packssdw m2, m3 paddd m0, m10 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) paddd m1, m10 - psubw m2, m11, m2 psrld m0, 12 psrld m8, m1, 12 mova [t4+r10*1+400*4+4], m2 mova m4, [t3+r10*2+400*8+ 8] mova m5, [t3+r10*2+400*0+ 8] @@ -2394,19 +2373,18 @@ pmaddwd m0, m4 ; b5 * 164 pmaddwd m1, m4 paddusw m2, m4 paddusw m3, m4 psrad m5, m2, 20 ; min(z5, 255) - 256 - vpgatherdd m4, [r13+m5*4], m2 + vpgatherdd m4, [r13+m5*4], m2 ; x5 psrad m2, m3, 20 vpgatherdd m5, [r13+m2*4], m3 pmulld m0, m4 pmulld m1, m5 packssdw m4, m5 paddd m0, m10 ; x5 * b5 * 164 + (1 << 11) + (1 << 15) paddd m1, m10 - psubw m4, m11, m4 psrld m0, 12 psrld m1, 12 mova [t4+r10*1+400*0+ 4], m4 mova [t3+r10*2+400*0+ 8], xm0 vextracti128 [t3+r10*2+400*0+40], m0, 1 @@ -2506,20 +2484,17 @@ pmovzxwd m2, xm2 ; a5 pmovzxwd m3, xm3 ; a3 pmaddwd m2, m4 ; a5 * src pmaddwd m3, m4 ; a3 * src pslld m4, 13 - psubd m0, m4 - psubd m1, m4 - paddd m0, m2 ; a5 * src + b5 + (1 << 8) - (src << 13) - paddd m1, m3 ; a3 * src + b3 + (1 << 8) - (src << 13) + psubd m0, m2 ; b5 - a5 * src + (1 << 8) + psubd m1, m3 ; b3 - a3 * src + (1 << 8) psrld m0, 9 pslld m1, 7 pblendw m0, m1, 0xaa pmaddwd m0, m15 - vpbroadcastd m1, [pd_4096] - paddd m4, m1 + paddd m4, m11 paddd m0, m4 psrad m0, 7 vextracti128 xm1, m0, 1 packusdw xm0, xm1 ; clip psrlw xm0, 6 @@ -2549,26 +2524,23 @@ paddd m1, m4, [t3+r10*2+400*24] paddd m1, [t3+r10*2+400*20] mova [t3+r10*2+400*20], m5 mova [t3+r10*2+400*24], m4 pmovzxwd m4, [dstq+r10] - pmovzxwd m0, [t4+r10*1+400* 6] + pmovzxwd m2, [t4+r10*1+400* 6] pmovzxwd m3, xm3 - pmaddwd m0, m4 ; a5 * src + mova m0, [t3+r10*2+400*12] + pmaddwd m2, m4 ; a5 * src pmaddwd m3, m4 ; a3 * src - pslld m4, 12 - psubd m2, m4, [t3+r10*2+400*12] - paddd m4, m4 - psubd m1, m4 - psubd m0, m2 ; a5 * src + b5 + (1 << 8) - (src << 13) - paddd m1, m3 ; a3 * src + b3 + (1 << 8) - (src << 13) + pslld m4, 13 + psubd m0, m2 ; b5 - a5 * src + (1 << 8) + psubd m1, m3 ; b3 - a3 * src + (1 << 8) psrld m0, 8 pslld m1, 7 pblendw m0, m1, 0xaa pmaddwd m0, m15 - vpbroadcastd m1, [pd_4096] - paddd m4, m1 + paddd m4, m11 paddd m0, m4 psrad m0, 7 vextracti128 xm1, m0, 1 packusdw xm0, xm1 ; clip psrlw xm0, 6 diff --git a/third_party/dav1d/src/x86/looprestoration16_sse.asm b/third_party/dav1d/src/x86/looprestoration16_sse.asm --- a/third_party/dav1d/src/x86/looprestoration16_sse.asm +++ b/third_party/dav1d/src/x86/looprestoration16_sse.asm @@ -33,40 +33,67 @@ wiener_shufC: db 6, 7, 8, 9, 8, 9, 10, 11, 10, 11, 12, 13, 12, 13, 14, 15 wiener_shufD: db 2, 3, -1, -1, 4, 5, -1, -1, 6, 7, -1, -1, 8, 9, -1, -1 wiener_shufE: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15 wiener_lshuf5: db 0, 1, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 wiener_lshuf7: db 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7 +sgr_lshuf3: db 0, 1, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 +sgr_lshuf5: db 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 pb_0to15: db 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 +pb_m14_m13: times 8 db -14,-13 pb_m10_m9: times 8 db -10, -9 pb_m6_m5: times 8 db -6, -5 pb_m2_m1: times 8 db -2, -1 pb_2_3: times 8 db 2, 3 pb_6_7: times 8 db 6, 7 +pw_256: times 8 dw 256 +pw_1023: times 8 dw 1023 +pd_8: times 4 dd 8 +pd_4096: times 4 dd 4096 +pd_34816: times 4 dd 34816 pd_m262128: times 4 dd -262128 +pd_0xffff: times 4 dd 0xffff +pd_0xf00800a4: times 4 dd 0xf00800a4 +pd_0xf00801c7: times 4 dd 0xf00801c7 +pd_0xfffffff0: times 4 dd 0xfffffff0 wiener_shifts: dw 4, 4, 2048, 2048, 1, 1, 8192, 8192 wiener_round: dd 1049600, 1048832 +cextern sgr_x_by_x + SECTION .text +%macro movif64 2 ; dst, src + %if ARCH_X86_64 + mov %1, %2 + %endif +%endmacro + +%macro movif32 2 ; dst, src + %if ARCH_X86_32 + mov %1, %2 + %endif +%endmacro + INIT_XMM ssse3 %if ARCH_X86_32 DECLARE_REG_TMP 4, 6 %if STACK_ALIGNMENT < 16 - %assign stack_size 13*16+384*12 + %assign extra_stack 14*16 %else - %assign stack_size 11*16+384*12 + %assign extra_stack 12*16 %endif -cglobal wiener_filter7_16bpc, 5, 7, 8, -stack_size, dst, dst_stride, left, \ - lpf, lpf_stride, w, flt +cglobal wiener_filter7_16bpc, 5, 7, 8, -384*12-16-extra_stack, \ + dst, dst_stride, left, lpf, lpf_stride, w, flt %if STACK_ALIGNMENT < 16 - %define lpfm dword [esp+calloff+16*10+0] - %define lpf_stridem dword [esp+calloff+16*10+4] - %define wm dword [esp+calloff+16*10+8] - %define hd dword [esp+calloff+16*10+12] - %define edgeb byte [esp+calloff+16*10+16] + %define lpfm dword [esp+calloff+16*12+ 0] + %define lpf_stridem dword [esp+calloff+16*12+ 4] + %define wm dword [esp+calloff+16*12+ 8] + %define hd dword [esp+calloff+16*12+12] + %define edgeb byte [esp+calloff+16*12+16] + %define edged dword [esp+calloff+16*12+16] %else %define hd dword r6m %define edgeb byte r8m %endif %define PICmem dword [esp+calloff+4*0] @@ -88,20 +115,21 @@ %define m11 [esp+calloff+16*5] %define m12 [esp+calloff+16*6] %define m13 [esp+calloff+16*7] %define m14 [esp+calloff+16*8] %define m15 [esp+calloff+16*9] + %define r10 r5 %define base t0-wiener_shifts %assign calloff 0 %if STACK_ALIGNMENT < 16 mov wd, [rstk+stack_offset+24] mov lpf_stridem, lpf_strideq mov wm, wd mov r4, [rstk+stack_offset+28] mov hd, r4 mov r4, [rstk+stack_offset+36] - mov [esp+16*11], r4 ; edge + mov edged, r4 ; edge %endif %else DECLARE_REG_TMP 4, 9, 7, 11, 12, 13, 14 ; wiener ring buffer pointers cglobal wiener_filter7_16bpc, 5, 15, 16, -384*12-16, dst, dst_stride, left, lpf, \ lpf_stride, w, edge, flt, h @@ -122,19 +150,18 @@ mov t0, [rstk+stack_offset+32] mov t1, [rstk+stack_offset+40] ; pixel_max movq m1, [t0] ; fx movq m3, [t0+16] ; fy LEA t0, wiener_shifts - mov PICmem, t0 %else LEA t0, wiener_shifts mov fltq, r7m movq m1, [fltq] movq m3, [fltq+16] mov t1, r9m ; pixel_max + %endif mov PICmem, t0 - %endif %endif mova m6, [base+wiener_shufA] mova m7, [base+wiener_shufB] %if ARCH_X86_64 lea t4, [wiener_shifts] @@ -160,10 +187,12 @@ pmullw m13, m0 ; horizontal downshift constant DEFINE_ARGS dst, dst_stride, left, lpf, lpf_stride, _, edge, _, h, _, w %define lpfm [rsp+0] %define lpf_stridem [rsp+8] %define base + %define wiener_lshuf7_mem [wiener_lshuf7] + %define pd_m262128_mem [pd_m262128] %else add wd, wd mova m4, [base+wiener_shufC] mova m5, [base+wiener_shufD] pshufd m0, m1, q0000 @@ -174,29 +203,31 @@ mova m9, m5 mova m14, m2 mova m15, m3 shr t1, 11 add lpfq, wq + mova m3, [base+pd_m262128] movd m4, [base+wiener_round+t1*4] movq m5, [base+wiener_shifts+t1*8] - %if STACK_ALIGNMENT < 16 - lea t1, [esp+16*12+wq+16] - %else - lea t1, [esp+16*10+wq+16] - %endif + lea t1, [esp+extra_stack+wq+16] add dstq, wq neg wq pshufd m4, m4, q0000 pshufd m2, m5, q0000 pshufd m5, m5, q1111 mov wm, wq pmullw m0, m2 pmullw m1, m2 + mova m2, [base+wiener_lshuf7] + %define pd_m262128_mem [esp+calloff+16*10] + mova pd_m262128_mem, m3 mova m10, m4 mova m11, m5 mova m12, m0 mova m13, m1 + %define wiener_lshuf7_mem [esp+calloff+16*11] + mova wiener_lshuf7_mem, m2 %endif test edgeb, 4 ; LR_HAVE_TOP jz .no_top call .h_top %if ARCH_X86_64 @@ -206,29 +237,18 @@ %endif mov t6, t1 mov t5, t1 add t1, 384*2 call .h_top -%if ARCH_X86_64 - lea r7, [lpfq+lpf_strideq*4] + movif32 lpf_strideq, lpf_stridem + lea r10, [lpfq+lpf_strideq*4] mov lpfq, dstq mov t4, t1 add t1, 384*2 - mov lpf_stridem, lpf_strideq - add r7, lpf_strideq - mov lpfm, r7 ; below -%else - mov t4m, t1 - mov t0, lpf_stridem - lea t1, [lpfq+t0*4] - mov lpfq, dstq - add t1, t0 - mov lpfm, t1 ; below - mov t1, t4m - mov t0, PICmem - add t1, 384*2 -%endif + movif64 lpf_stridem, lpf_strideq + add r10, lpf_strideq + mov lpfm, r10 ; below call .h mov t3, t1 mov t2, t1 dec hd jz .v1 @@ -257,28 +277,17 @@ call .hv_bottom .v1: call .v RET .no_top: -%if ARCH_X86_64 - lea r7, [lpfq+lpf_strideq*4] + movif32 lpf_strideq, lpf_stridem + lea r10, [lpfq+lpf_strideq*4] mov lpfq, dstq - mov lpf_stridem, lpf_strideq - lea r7, [r7+lpf_strideq*2] - mov lpfm, r7 + movif64 lpf_stridem, lpf_strideq + lea r10, [r10+lpf_strideq*2] + mov lpfm, r10 call .h -%else - mov t1m, t1 - mov t0, lpf_stridem - lea t1, [lpfq+t0*4] - mov lpfq, dstq - lea t1, [t1+t0*2] - mov lpfm, t1 - mov t0, PICmem - mov t1, t1m - call .h -%endif mov t6, t1 mov t5, t1 mov t4, t1 mov t3, t1 mov t2, t1 @@ -303,23 +312,19 @@ call .hv dec hd jnz .main .v3: call .v -%if ARCH_X86_32 - mov wq, wm -%endif + movif32 wq, wm .v2: call .v -%if ARCH_X86_32 - mov wq, wm -%endif + movif32 wq, wm jmp .v1 .extend_right: -%assign stack_offset_tmp stack_offset %assign stack_offset stack_offset+8 %assign calloff 8 + movif32 t0, PICmem pxor m0, m0 movd m1, wd mova m2, [base+pb_0to15] pshufb m1, m0 mova m0, [base+pb_6_7] @@ -332,33 +337,29 @@ pshufb m4, m0 mova m0, [base+pb_m10_m9] psubb m0, m1 pminub m0, m2 pshufb m5, m0 + movif32 t0, t0m ret %assign stack_offset stack_offset-4 %assign calloff 4 .h: -%if ARCH_X86_64 - mov wq, r5 -%else - mov wq, wm -%endif + movif64 wq, r5 + movif32 wq, wm test edgeb, 1 ; LR_HAVE_LEFT jz .h_extend_left movq m3, [leftq] movhps m3, [lpfq+wq] add leftq, 8 jmp .h_main .h_extend_left: - mova m3, [lpfq+wq] ; avoid accessing memory located - pshufb m3, [base+wiener_lshuf7] ; before the start of the buffer + mova m3, [lpfq+wq] ; avoid accessing memory located + pshufb m3, wiener_lshuf7_mem ; before the start of the buffer jmp .h_main .h_top: -%if ARCH_X86_64 - mov wq, r5 -%endif + movif64 wq, r5 test edgeb, 1 ; LR_HAVE_LEFT jz .h_extend_left .h_loop: movu m3, [lpfq+wq-8] .h_main: @@ -379,11 +380,11 @@ paddw m3, m1 pshufb m1, m4, m6 pmaddwd m3, m13 pshufb m2, m5, m7 paddw m1, m2 - mova m2, [base+pd_m262128] ; (1 << 4) - (1 << 18) + mova m2, pd_m262128_mem ; (1 << 4) - (1 << 18) pshufb m4, m8 pmaddwd m1, m12 pshufb m5, m9 paddw m4, m5 pmaddwd m4, m13 @@ -396,42 +397,32 @@ packssdw m0, m1 psraw m0, 1 mova [t1+wq], m0 add wq, 16 jl .h_loop -%if ARCH_X86_32 - mov wq, wm -%endif + movif32 wq, wm ret ALIGN function_align .hv: add lpfq, dst_strideq -%if ARCH_X86_64 - mov wq, r5 -%else - mov t0m, t0 - mov t1m, t1 - mov t0, PICmem -%endif + movif64 wq, r5 + movif32 t0m, t0 + movif32 t1m, t1 test edgeb, 1 ; LR_HAVE_LEFT jz .hv_extend_left movq m3, [leftq] movhps m3, [lpfq+wq] add leftq, 8 jmp .hv_main .hv_extend_left: mova m3, [lpfq+wq] - pshufb m3, [base+wiener_lshuf7] + pshufb m3, wiener_lshuf7_mem jmp .hv_main .hv_bottom: -%if ARCH_X86_64 - mov wq, r5 -%else - mov t0m, t0 - mov t1m, t1 - mov t0, PICmem -%endif + movif64 wq, r5 + movif32 t0m, t0 + movif32 t1m, t1 test edgeb, 1 ; LR_HAVE_LEFT jz .hv_extend_left .hv_loop: movu m3, [lpfq+wq-8] .hv_main: @@ -441,13 +432,12 @@ jnz .hv_have_right cmp wd, -18 jl .hv_have_right call .extend_right .hv_have_right: -%if ARCH_X86_32 - mov t1, t4m -%endif + movif32 t1, t4m + movif32 t0, t2m pshufb m0, m3, m6 pshufb m1, m4, m7 paddw m0, m1 pshufb m3, m8 pmaddwd m0, m12 @@ -455,11 +445,11 @@ paddw m3, m1 pshufb m1, m4, m6 pmaddwd m3, m13 pshufb m2, m5, m7 paddw m1, m2 - mova m2, [base+pd_m262128] + mova m2, pd_m262128_mem pshufb m4, m8 pmaddwd m1, m12 pshufb m5, m9 paddw m4, m5 pmaddwd m4, m13 @@ -468,17 +458,16 @@ %if ARCH_X86_64 mova m2, [t4+wq] paddw m2, [t2+wq] mova m5, [t3+wq] %else - mov t0, t0m mova m2, [t1+wq] - mov t1, t2m - paddw m2, [t1+wq] + paddw m2, [t0+wq] mov t1, t3m + mov t0, t5m mova m5, [t1+wq] - mov t1, t5m + mov t1, t1m %endif paddd m0, m3 paddd m1, m4 psrad m0, 4 psrad m1, 4 @@ -487,15 +476,15 @@ mova m4, [t5+wq] paddw m4, [t1+wq] psraw m0, 1 paddw m3, m0, [t6+wq] %else - mova m4, [t1+wq] - mov t1, t1m + mova m4, [t0+wq] paddw m4, [t1+wq] + mov t0, t0m + mov t1, t6m psraw m0, 1 - mov t1, t6m paddw m3, m0, [t1+wq] %endif mova [t0+wq], m0 punpcklwd m0, m2, m5 pmaddwd m0, m15 @@ -515,24 +504,20 @@ pmulhw m0, m11 pxor m1, m1 pmaxsw m0, m1 mova [dstq+wq], m0 add wq, 16 + jl .hv_loop %if ARCH_X86_64 - jl .hv_loop mov t6, t5 mov t5, t4 mov t4, t3 mov t3, t2 mov t2, t1 mov t1, t0 mov t0, t6 %else - jge .hv_end - mov t0, PICmem - jmp .hv_loop -.hv_end: mov r5, t5m mov t1, t4m mov t6m, r5 mov t5m, t1 mov r5, t3m @@ -546,33 +531,33 @@ mov wq, wm %endif add dstq, dst_strideq ret .v: + movif64 wq, r5 + movif32 t0m, t0 + movif32 t1m, t1 +.v_loop: %if ARCH_X86_64 - mov wq, r5 -.v_loop: mova m1, [t4+wq] paddw m1, [t2+wq] mova m2, [t3+wq] mova m4, [t1+wq] paddw m3, m4, [t6+wq] paddw m4, [t5+wq] %else - mov t1m, t1 -.v_loop: - mov t1, t4m - mova m1, [t1+wq] + mov t0, t4m mov t1, t2m + mova m1, [t0+wq] paddw m1, [t1+wq] - mov t1, t3m - mova m2, [t1+wq] + mov t0, t3m mov t1, t1m + mova m2, [t0+wq] mova m4, [t1+wq] - mov t1, t6m - paddw m3, m4, [t1+wq] + mov t0, t6m mov t1, t5m + paddw m3, m4, [t0+wq] paddw m4, [t1+wq] %endif punpcklwd m0, m1, m2 pmaddwd m0, m15 punpckhwd m1, m2 @@ -599,19 +584,20 @@ mov t5, t4 mov t4, t3 mov t3, t2 mov t2, t1 %else - mov t1, t5m - mov r5, t4m - mov t6m, t1 - mov t5m, r5 - mov t1, t3m + mov t0, t5m + mov t1, t4m + mov r5, t3m + mov t6m, t0 + mov t5m, t1 + mov t4m, r5 mov r5, t2m - mov t4m, t1 + mov t1, t1m + mov t0, t0m mov t3m, r5 - mov t1, t1m mov t2m, t1 %endif add dstq, dst_strideq ret @@ -627,10 +613,11 @@ %define lpfm dword [esp+calloff+4*6] %define lpf_stridem dword [esp+calloff+4*7] %define wm dword [esp+calloff+16*10+0] %define hd dword [esp+calloff+16*10+4] %define edgeb byte [esp+calloff+16*10+8] + %define edged dword [esp+calloff+16*10+8] %else %define hd dword r6m %define edgeb byte r8m %endif %define PICmem dword [esp+calloff+4*0] @@ -657,14 +644,14 @@ mov lpf_stridem, lpf_strideq mov wm, wd mov r4, [rstk+stack_offset+28] mov hd, r4 mov r4, [rstk+stack_offset+36] - mov [esp+16*10+8], r4 ; edge + mov edged, r4 ; edge %endif %else -cglobal wiener_filter5_16bpc, 5, 15, 16, 384*8+16, dst, dst_stride, left, lpf, \ +cglobal wiener_filter5_16bpc, 5, 14, 16, 384*8+16, dst, dst_stride, left, lpf, \ lpf_stride, w, edge, flt, h %define base %endif %if ARCH_X86_64 || STACK_ALIGNMENT >= 16 movifnidn wd, wm @@ -681,19 +668,18 @@ mov t0, [rstk+stack_offset+32] mov t1, [rstk+stack_offset+40] ; pixel_max movq m1, [t0] ; fx movq m3, [t0+16] ; fy LEA t0, wiener_shifts - mov PICmem, t0 %else LEA t0, wiener_shifts mov fltq, r7m movq m1, [fltq] movq m3, [fltq+16] mov t1, r9m ; pixel_max + %endif mov PICmem, t0 - %endif %endif mova m5, [base+wiener_shufE] mova m6, [base+wiener_shufB] mova m7, [base+wiener_shufD] %if ARCH_X86_64 @@ -769,28 +755,18 @@ add lpfq, lpf_stridem %endif mov t4, t1 add t1, 384*2 call .h_top -%if ARCH_X86_64 - lea r7, [lpfq+lpf_strideq*4] + movif32 lpf_strideq, lpf_stridem + lea r10, [lpfq+lpf_strideq*4] mov lpfq, dstq mov t3, t1 add t1, 384*2 - mov lpf_stridem, lpf_strideq - add r7, lpf_strideq - mov lpfm, r7 ; below -%else - mov t3m, t1 - mov t0, lpf_stridem - lea t1, [lpfq+t0*4] - mov lpfq, dstq - add t1, t0 - mov lpfm, t1 ; below - mov t1, t3m - add t1, 384*2 -%endif + movif64 lpf_stridem, lpf_strideq + add r10, lpf_strideq + mov lpfm, r10 ; below call .h mov t2, t1 dec hd jz .v1 add lpfq, dst_strideq @@ -811,27 +787,17 @@ add lpfq, lpf_stridem call .hv_bottom .end: RET .no_top: -%if ARCH_X86_64 - lea r7, [lpfq+lpf_strideq*4] + movif32 lpf_strideq, lpf_stridem + lea r10, [lpfq+lpf_strideq*4] mov lpfq, dstq - mov lpf_stridem, lpf_strideq - lea r7, [r7+lpf_strideq*2] - mov lpfm, r7 + movif64 lpf_stridem, lpf_strideq + lea r10, [r10+lpf_strideq*2] + mov lpfm, r10 call .h -%else - mov t1m, t1 - mov t0, lpf_stridem - lea t1, [lpfq+t0*4] - mov lpfq, dstq - lea t1, [t1+t0*2] - mov lpfm, t1 - mov t1, t1m - call .h -%endif mov t4, t1 mov t3, t1 mov t2, t1 dec hd jz .v1 @@ -866,16 +832,13 @@ add dstq, dst_strideq .v1: call .v jmp .end .extend_right: -%assign stack_offset_tmp stack_offset %assign stack_offset stack_offset+8 %assign calloff 8 -%if ARCH_X86_32 - mov t0, PICmem -%endif + movif32 t0, PICmem pxor m1, m1 movd m2, wd mova m0, [base+pb_2_3] pshufb m2, m1 mova m1, [base+pb_m6_m5] @@ -888,15 +851,12 @@ pshufb m4, m1 ret %assign stack_offset stack_offset-4 %assign calloff 4 .h: -%if ARCH_X86_64 - mov wq, r5 -%else - mov wq, wm -%endif + movif64 wq, r5 + movif32 wq, wm test edgeb, 1 ; LR_HAVE_LEFT jz .h_extend_left mova m4, [lpfq+wq] movd m3, [leftq+4] pslldq m4, 4 @@ -906,15 +866,12 @@ .h_extend_left: mova m3, [lpfq+wq] ; avoid accessing memory located pshufb m3, m15 ; before the start of the buffer jmp .h_main .h_top: -%if ARCH_X86_64 - mov wq, r5 -%else - mov wq, wm -%endif + movif64 wq, r5 + movif32 wq, wm test edgeb, 1 ; LR_HAVE_LEFT jz .h_extend_left .h_loop: movu m3, [lpfq+wq-4] .h_main: @@ -946,23 +903,18 @@ packssdw m0, m1 psraw m0, 1 mova [t1+wq], m0 add wq, 16 jl .h_loop -%if ARCH_X86_32 - mov wq, wm -%endif + movif32 wq, wm ret ALIGN function_align .hv: add lpfq, dst_strideq -%if ARCH_X86_64 - mov wq, r5 -%else - mov t0m, t0 - mov t1m, t1 -%endif + movif64 wq, r5 + movif32 t0m, t0 + movif32 t1m, t1 test edgeb, 1 ; LR_HAVE_LEFT jz .hv_extend_left mova m4, [lpfq+wq] movd m3, [leftq+4] pslldq m4, 4 @@ -972,16 +924,13 @@ .hv_extend_left: mova m3, [lpfq+wq] pshufb m3, m15 jmp .hv_main .hv_bottom: -%if ARCH_X86_64 - mov wq, r5 -%else - mov t0m, t0 - mov t1m, t1 -%endif + movif64 wq, r5 + movif32 t0m, t0 + movif32 t1m, t1 test edgeb, 1 ; LR_HAVE_LEFT jz .hv_extend_left .hv_loop: movu m3, [lpfq+wq-4] .hv_main: @@ -990,14 +939,12 @@ jnz .hv_have_right cmp wd, -18 jl .hv_have_right call .extend_right .hv_have_right: -%if ARCH_X86_32 - mov t1, t1m - mov t0, t3m -%endif + movif32 t1, t1m + movif32 t0, t3m pshufb m0, m3, m5 pmaddwd m0, m11 pshufb m1, m4, m5 pmaddwd m1, m11 pshufb m2, m3, m6 @@ -1074,20 +1021,19 @@ mov wq, wm %endif add dstq, dst_strideq ret .v: + movif64 wq, r5 + movif32 t1m, t1 +.v_loop: %if ARCH_X86_64 - mov wq, r5 -.v_loop: mova m0, [t1+wq] paddw m2, m0, [t3+wq] mova m1, [t2+wq] mova m4, [t4+wq] %else - mov t1m, t1 -.v_loop: mov t0, t3m mova m0, [t1+wq] mov t1, t2m paddw m2, m0, [t0+wq] mov t0, t4m @@ -1121,5 +1067,2694 @@ mov t1, t1m jmp .v_loop .v_end: %endif ret + +%macro GATHERDD 3 ; dst, src, tmp + movd %3d, %2 + %if ARCH_X86_64 + movd %1, [r13+%3] + pextrw %3d, %2, 2 + pinsrw %1, [r13+%3+2], 3 + pextrw %3d, %2, 4 + pinsrw %1, [r13+%3+2], 5 + pextrw %3d, %2, 6 + pinsrw %1, [r13+%3+2], 7 + %else + movd %1, [base+sgr_x_by_x-0xf03+%3] + pextrw %3, %2, 2 + pinsrw %1, [base+sgr_x_by_x-0xf03+%3+2], 3 + pextrw %3, %2, 4 + pinsrw %1, [base+sgr_x_by_x-0xf03+%3+2], 5 + pextrw %3, %2, 6 + pinsrw %1, [base+sgr_x_by_x-0xf03+%3+2], 7 + %endif +%endmacro + +%macro GATHER_X_BY_X 5 ; dst, src0, src1, tmp32, tmp32_restore + %if ARCH_X86_64 + %define tmp r14 + %else + %define tmp %4 + %endif + GATHERDD %1, %2, tmp + GATHERDD %2, %3, tmp + movif32 %4, %5 + psrld %1, 24 + psrld %2, 24 + packssdw %1, %2 +%endmacro + +%macro MAXSD 3-4 0 ; dst, src, restore_tmp + pcmpgtd %3, %1, %2 + pand %1, %3 + pandn %3, %2 + por %1, %3 + %if %4 == 1 + pxor %3, %3 + %endif +%endmacro + +%macro MULLD 3 ; dst, src, tmp + pmulhuw %3, %1, %2 + pmullw %1, %2 + pslld %3, 16 + paddd %1, %3 +%endmacro + +%if ARCH_X86_32 +DECLARE_REG_TMP 0, 1, 2, 3, 4 + %if STACK_ALIGNMENT < 16 + %assign extra_stack 5*16 + %else + %assign extra_stack 3*16 + %endif +cglobal sgr_filter_5x5_16bpc, 1, 7, 8, -400*24-16-extra_stack, \ + dst, dst_stride, left, lpf, lpf_stride, w, params, h + %if STACK_ALIGNMENT < 16 + %define dstm dword [esp+calloff+16*0+4*6] + %define dst_stridemp dword [esp+calloff+16*0+4*7] + %define leftm dword [esp+calloff+16*3+4*0] + %define lpfm dword [esp+calloff+16*3+4*1] + %define lpf_stridem dword [esp+calloff+16*3+4*2] + %define w0m dword [esp+calloff+16*3+4*3] + %define hd dword [esp+calloff+16*3+4*4] + %define edgeb byte [esp+calloff+16*3+4*5] + %define edged dword [esp+calloff+16*3+4*5] + %define leftmp leftm + %else + %define w0m wm + %define hd dword r6m + %define edgeb byte r8m + %define edged dword r8m + %endif + %define hvsrcm dword [esp+calloff+4*0] + %define w1m dword [esp+calloff+4*1] + %define t0m dword [esp+calloff+4*2] + %define t2m dword [esp+calloff+4*3] + %define t3m dword [esp+calloff+4*4] + %define t4m dword [esp+calloff+4*5] + %define m8 [base+pd_8] + %define m9 [base+pd_0xfffffff0] + %define m10 [esp+calloff+16*2] + %define m11 [base+pd_0xf00800a4] + %define m12 [base+sgr_lshuf5] + %define m13 [base+pd_34816] + %define m14 [base+pw_1023] + %define r10 r5 + %define base r6-$$ + %assign calloff 0 + %if STACK_ALIGNMENT < 16 + mov dst_strideq, [rstk+stack_offset+ 8] + mov leftq, [rstk+stack_offset+12] + mov lpfq, [rstk+stack_offset+16] + mov lpf_strideq, [rstk+stack_offset+20] + mov wd, [rstk+stack_offset+24] + mov dstm, dstq + mov dst_stridemp, dst_strideq + mov leftm, leftq + mov r1, [rstk+stack_offset+28] + mov r2, [rstk+stack_offset+36] + mov lpfm, lpfq + mov lpf_stridem, lpf_strideq + mov hd, r1 + mov edged, r2 + %endif +%else +cglobal sgr_filter_5x5_16bpc, 5, 15, 15, -400*24-16, dst, dst_stride, left, lpf, \ + lpf_stride, w, edge, params, h +%endif +%if ARCH_X86_64 || STACK_ALIGNMENT >= 16 + movifnidn wd, wm +%endif +%if ARCH_X86_64 + mov paramsq, paramsmp + lea r13, [sgr_x_by_x-0xf03] + mov edged, r8m + add wd, wd + mov hd, r6m + movu m10, [paramsq] + mova m12, [sgr_lshuf5] + add lpfq, wq + mova m8, [pd_8] + lea t1, [rsp+wq+20] + mova m9, [pd_0xfffffff0] + add dstq, wq + lea t3, [rsp+wq*2+400*12+16] + mova m11, [pd_0xf00800a4] + lea t4, [rsp+wq+400*20+16] + pshufhw m7, m10, q0000 + pshufb m10, [pw_256] ; s0 + punpckhqdq m7, m7 ; w0 + neg wq + mova m13, [pd_34816] ; (1 << 11) + (1 << 15) + pxor m6, m6 + mova m14, [pw_1023] + psllw m7, 4 + DEFINE_ARGS dst, dst_stride, left, lpf, lpf_stride, _, edge, _, h, _, w + %define lpfm [rsp+0] + %define lpf_stridem [rsp+8] +%else + mov r1, [rstk+stack_offset+32] ; params + LEA r6, $$ + add wd, wd + movu m1, [r1] + add lpfm, wq + lea t1, [rsp+extra_stack+wq+20] + add dstq, wq + lea t3, [rsp+extra_stack+wq*2+400*12+16] + mov dstm, dstq + lea t4, [rsp+extra_stack+wq+400*20+16] + mov t3m, t3 + pshufhw m7, m1, q0000 + mov t4m, t4 + pshufb m1, [base+pw_256] ; s0 + punpckhqdq m7, m7 ; w0 + psllw m7, 4 + neg wq + mova m10, m1 + pxor m6, m6 + mov w1m, wd + sub wd, 4 + mov lpfq, lpfm + mov lpf_strideq, lpf_stridem + mov w0m, wd +%endif + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, lpf_strideq + movif32 t2m, t1 + mov t2, t1 + call .top_fixup + add t1, 400*6 + call .h_top + lea r10, [lpfq+lpf_strideq*4] + mov lpfq, dstq + movif64 lpf_stridem, lpf_strideq + add r10, lpf_strideq + mov lpfm, r10 ; below + movif32 t0m, t2 + mov t0, t2 + dec hd + jz .height1 + or edged, 16 + call .h +.main: + add lpfq, dst_stridemp + movif32 t4, t4m + call .hv + call .prep_n + sub hd, 2 + jl .extend_bottom +.main_loop: + movif32 lpfq, hvsrcm + add lpfq, dst_stridemp +%if ARCH_X86_64 + test hb, hb +%else + mov r5, hd + test r5, r5 +%endif + jz .odd_height + call .h + add lpfq, dst_stridemp + call .hv + movif32 dstq, dstm + call .n0 + call .n1 + sub hd, 2 + movif32 t0, t0m + jge .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .extend_bottom + mov lpfq, lpfm + call .h_top + add lpfq, lpf_stridem + call .hv_bottom +.end: + movif32 dstq, dstm + call .n0 + call .n1 +.end2: + RET +.height1: + movif32 t4, t4m + call .hv + call .prep_n + jmp .odd_height_end +.odd_height: + call .hv + movif32 dstq, dstm + call .n0 + call .n1 +.odd_height_end: + call .v + movif32 dstq, dstm + call .n0 + jmp .end2 +.extend_bottom: + call .v + jmp .end +.no_top: + lea r10, [lpfq+lpf_strideq*4] + mov lpfq, dstq + movif64 lpf_stridem, lpf_strideq + lea r10, [r10+lpf_strideq*2] + mov lpfm, r10 + call .h + lea t2, [t1+400*6] + movif32 t2m, t2 + call .top_fixup + dec hd + jz .no_top_height1 + or edged, 16 + mov t0, t1 + mov t1, t2 + movif32 t0m, t0 + jmp .main +.no_top_height1: + movif32 t3, t3m + movif32 t4, t4m + call .v + call .prep_n + jmp .odd_height_end +.extend_right: + movd m0, wd + movd m1, [lpfq-2] + mova m2, [base+pw_256] + mova m3, [base+pb_m14_m13] + pshufb m0, m6 + pshufb m1, m2 + psubb m2, m0 + psubb m3, m0 + mova m0, [base+pb_0to15] + pcmpgtb m2, m0 + pcmpgtb m3, m0 + pand m4, m2 + pand m5, m3 + pandn m2, m1 + pandn m3, m1 + por m4, m2 + por m5, m3 + ret +%assign stack_offset stack_offset+4 +%assign calloff 4 +.h: ; horizontal boxsum +%if ARCH_X86_64 + lea wq, [r5-4] +%else + %define leftq r5 +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movif32 leftq, leftm + movddup m5, [leftq] + movif32 wq, w0m + mova m4, [lpfq+wq+4] + add leftmp, 8 + palignr m4, m5, 10 + jmp .h_main +.h_extend_left: + movif32 wq, w0m + mova m4, [lpfq+wq+4] + pshufb m4, m12 + jmp .h_main +.h_top: +%if ARCH_X86_64 + lea wq, [r5-4] +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movif32 wq, w0m +.h_loop: + movu m4, [lpfq+wq- 2] +.h_main: + movu m5, [lpfq+wq+14] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp wd, -20 + jl .h_have_right + call .extend_right +.h_have_right: + palignr m2, m5, m4, 2 + paddw m0, m4, m2 + palignr m3, m5, m4, 6 + paddw m0, m3 + punpcklwd m1, m2, m3 + pmaddwd m1, m1 + punpckhwd m2, m3 + pmaddwd m2, m2 + palignr m5, m4, 8 + paddw m0, m5 + punpcklwd m3, m4, m5 + pmaddwd m3, m3 + paddd m1, m3 + punpckhwd m3, m4, m5 + pmaddwd m3, m3 + shufps m4, m5, q2121 + paddw m0, m4 ; sum + punpcklwd m5, m4, m6 + pmaddwd m5, m5 + punpckhwd m4, m6 + pmaddwd m4, m4 + paddd m2, m3 + test edgeb, 16 ; y > 0 + jz .h_loop_end + paddw m0, [t1+wq+400*0] + paddd m1, [t1+wq+400*2] + paddd m2, [t1+wq+400*4] +.h_loop_end: + paddd m1, m5 ; sumsq + paddd m2, m4 + mova [t1+wq+400*0], m0 + mova [t1+wq+400*2], m1 + mova [t1+wq+400*4], m2 + add wq, 16 + jl .h_loop + ret +.top_fixup: +%if ARCH_X86_64 + lea wq, [r5-4] +%else + mov wd, w0m +%endif +.top_fixup_loop: ; the sums of the first row needs to be doubled + mova m0, [t1+wq+400*0] + mova m1, [t1+wq+400*2] + mova m2, [t1+wq+400*4] + paddw m0, m0 + paddd m1, m1 + paddd m2, m2 + mova [t2+wq+400*0], m0 + mova [t2+wq+400*2], m1 + mova [t2+wq+400*4], m2 + add wq, 16 + jl .top_fixup_loop + ret +ALIGN function_align +.hv: ; horizontal boxsum + vertical boxsum + ab +%if ARCH_X86_64 + lea wq, [r5-4] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left + movif32 leftq, leftm + movddup m5, [leftq] + movif32 wq, w0m + mova m4, [lpfq+wq+4] + add leftmp, 8 + palignr m4, m5, 10 + jmp .hv_main +.hv_extend_left: + movif32 wq, w0m + mova m4, [lpfq+wq+4] + pshufb m4, m12 + jmp .hv_main +.hv_bottom: +%if ARCH_X86_64 + lea wq, [r5-4] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left + movif32 wq, w0m +%if ARCH_X86_32 + jmp .hv_loop_start +%endif +.hv_loop: + movif32 lpfq, hvsrcm +.hv_loop_start: + movu m4, [lpfq+wq- 2] +.hv_main: + movu m5, [lpfq+wq+14] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv_have_right + cmp wd, -20 + jl .hv_have_right + call .extend_right +.hv_have_right: + movif32 t3, hd + palignr m3, m5, m4, 2 + paddw m0, m4, m3 + palignr m1, m5, m4, 6 + paddw m0, m1 + punpcklwd m2, m3, m1 + pmaddwd m2, m2 + punpckhwd m3, m1 + pmaddwd m3, m3 + palignr m5, m4, 8 + paddw m0, m5 + punpcklwd m1, m4, m5 + pmaddwd m1, m1 + paddd m2, m1 + punpckhwd m1, m4, m5 + pmaddwd m1, m1 + shufps m4, m5, q2121 + paddw m0, m4 ; h sum + punpcklwd m5, m4, m6 + pmaddwd m5, m5 + punpckhwd m4, m6 + pmaddwd m4, m4 + paddd m3, m1 + paddd m2, m5 ; h sumsq + paddd m3, m4 + paddw m1, m0, [t1+wq+400*0] + paddd m4, m2, [t1+wq+400*2] + paddd m5, m3, [t1+wq+400*4] +%if ARCH_X86_64 + test hd, hd +%else + test t3, t3 +%endif + jz .hv_last_row +.hv_main2: + paddw m1, [t2+wq+400*0] ; hv sum + paddd m4, [t2+wq+400*2] ; hv sumsq + paddd m5, [t2+wq+400*4] + mova [t0+wq+400*0], m0 + mova [t0+wq+400*2], m2 + mova [t0+wq+400*4], m3 + psrlw m3, m1, 1 + paddd m4, m8 + pavgw m3, m6 ; (b + 2) >> 2 + paddd m5, m8 + pand m4, m9 ; ((a + 8) >> 4) << 4 + pand m5, m9 + psrld m2, m4, 4 + psrld m0, m5, 4 + paddd m2, m4 + psrld m4, 1 + paddd m0, m5 + psrld m5, 1 + paddd m4, m2 ; a * 25 + paddd m5, m0 + punpcklwd m2, m3, m6 + punpckhwd m3, m6 + pmaddwd m2, m2 ; b * b + pmaddwd m3, m3 + punpcklwd m0, m1, m6 ; b + punpckhwd m1, m6 + MAXSD m4, m2, m6 + MAXSD m5, m3, m6, 1 + psubd m4, m2 ; p + psubd m5, m3 + MULLD m4, m10, m2 ; p * s + MULLD m5, m10, m2 + pmaddwd m0, m11 ; b * 164 + pmaddwd m1, m11 + paddusw m4, m11 + paddusw m5, m11 + psrld m4, 20 ; min(z, 255) + movif32 t3, t3m + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, t2, t2m + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m2 + MULLD m1, m5, m2 + paddd m0, m13 ; x * b * 164 + (1 << 11) + (1 << 15) + paddd m1, m13 + mova [t4+wq+4], m3 + psrld m0, 12 ; b + psrld m1, 12 + mova [t3+wq*2+ 8], m0 + mova [t3+wq*2+24], m1 + add wq, 16 + jl .hv_loop + mov t2, t1 + mov t1, t0 + mov t0, t2 + movif32 t2m, t2 + movif32 t0m, t0 + ret +.hv_last_row: ; esoteric edge case for odd heights + mova [t1+wq+400*0], m1 + paddw m1, m0 + mova [t1+wq+400*2], m4 + paddd m4, m2 + mova [t1+wq+400*4], m5 + paddd m5, m3 + jmp .hv_main2 +.v: ; vertical boxsum + ab +%if ARCH_X86_64 + lea wq, [r5-4] +%else + mov wd, w0m +%endif +.v_loop: + mova m0, [t1+wq+400*0] + mova m2, [t1+wq+400*2] + mova m3, [t1+wq+400*4] + paddw m1, m0, [t2+wq+400*0] + paddd m4, m2, [t2+wq+400*2] + paddd m5, m3, [t2+wq+400*4] + paddw m0, m0 + paddd m2, m2 + paddd m3, m3 + paddw m1, m0 ; hv sum + paddd m4, m2 ; hv sumsq + paddd m5, m3 + psrlw m3, m1, 1 + paddd m4, m8 + pavgw m3, m6 ; (b + 2) >> 2 + paddd m5, m8 + pand m4, m9 ; ((a + 8) >> 4) << 4 + pand m5, m9 + psrld m2, m4, 4 + psrld m0, m5, 4 + paddd m2, m4 + psrld m4, 1 + paddd m0, m5 + psrld m5, 1 + paddd m4, m2 ; a * 25 + paddd m5, m0 + punpcklwd m2, m3, m6 + punpckhwd m3, m6 + pmaddwd m2, m2 ; b * b + pmaddwd m3, m3 + punpcklwd m0, m1, m6 ; b + punpckhwd m1, m6 + MAXSD m4, m2, m6 + MAXSD m5, m3, m6, 1 + psubd m4, m2 ; p + psubd m5, m3 + MULLD m4, m10, m2 ; p * s + MULLD m5, m10, m2 + pmaddwd m0, m11 ; b * 164 + pmaddwd m1, m11 + paddusw m4, m11 + paddusw m5, m11 + psrld m4, 20 ; min(z, 255) + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, t2, t2m + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m2 + MULLD m1, m5, m2 + paddd m0, m13 ; x * b * 164 + (1 << 11) + (1 << 15) + paddd m1, m13 + mova [t4+wq+4], m3 + psrld m0, 12 ; b + psrld m1, 12 + mova [t3+wq*2+ 8], m0 + mova [t3+wq*2+24], m1 + add wq, 16 + jl .v_loop + ret +.prep_n: ; initial neighbor setup + movif64 wq, r5 + movif32 wd, w1m +.prep_n_loop: + movu m0, [t4+wq*1+ 2] + movu m3, [t4+wq*1+ 4] + movu m1, [t3+wq*2+ 4] + movu m4, [t3+wq*2+ 8] + movu m2, [t3+wq*2+20] + movu m5, [t3+wq*2+24] + paddw m3, m0 + paddd m4, m1 + paddd m5, m2 + paddw m3, [t4+wq*1+ 0] + paddd m4, [t3+wq*2+ 0] + paddd m5, [t3+wq*2+16] + paddw m0, m3 + psllw m3, 2 + paddd m1, m4 + pslld m4, 2 + paddd m2, m5 + pslld m5, 2 + paddw m0, m3 ; a 565 + paddd m1, m4 ; b 565 + paddd m2, m5 + mova [t4+wq*1+400*2+ 0], m0 + mova [t3+wq*2+400*4+ 0], m1 + mova [t3+wq*2+400*4+16], m2 + add wq, 16 + jl .prep_n_loop + ret +ALIGN function_align +.n0: ; neighbor + output (even rows) + movif64 wq, r5 + movif32 wd, w1m +.n0_loop: + movu m0, [t4+wq*1+ 2] + movu m3, [t4+wq*1+ 4] + movu m1, [t3+wq*2+ 4] + movu m4, [t3+wq*2+ 8] + movu m2, [t3+wq*2+20] + movu m5, [t3+wq*2+24] + paddw m3, m0 + paddd m4, m1 + paddd m5, m2 + paddw m3, [t4+wq*1+ 0] + paddd m4, [t3+wq*2+ 0] + paddd m5, [t3+wq*2+16] + paddw m0, m3 + psllw m3, 2 + paddd m1, m4 + pslld m4, 2 + paddd m2, m5 + pslld m5, 2 + paddw m0, m3 ; a 565 + paddd m1, m4 ; b 565 + paddd m2, m5 + paddw m3, m0, [t4+wq*1+400*2+ 0] + paddd m4, m1, [t3+wq*2+400*4+ 0] + paddd m5, m2, [t3+wq*2+400*4+16] + mova [t4+wq*1+400*2+ 0], m0 + mova [t3+wq*2+400*4+ 0], m1 + mova [t3+wq*2+400*4+16], m2 + mova m0, [dstq+wq] + punpcklwd m1, m0, m6 ; src + punpcklwd m2, m3, m6 ; a + pmaddwd m2, m1 ; a * src + punpckhwd m1, m0, m6 + punpckhwd m3, m6 + pmaddwd m3, m1 + psubd m4, m2 ; b - a * src + (1 << 8) + psubd m5, m3 + psrad m4, 9 + psrad m5, 9 + packssdw m4, m5 + pmulhrsw m4, m7 + paddw m0, m4 + pmaxsw m0, m6 + pminsw m0, m14 + mova [dstq+wq], m0 + add wq, 16 + jl .n0_loop + add dstq, dst_stridemp + ret +ALIGN function_align +.n1: ; neighbor + output (odd rows) + movif64 wq, r5 + movif32 wd, w1m +.n1_loop: + mova m0, [dstq+wq] + mova m3, [t4+wq*1+400*2+ 0] + mova m4, [t3+wq*2+400*4+ 0] + mova m5, [t3+wq*2+400*4+16] + punpcklwd m1, m0, m6 ; src + punpcklwd m2, m3, m6 ; a + pmaddwd m2, m1 + punpckhwd m1, m0, m6 + punpckhwd m3, m6 + pmaddwd m3, m1 + psubd m4, m2 ; b - a * src + (1 << 7) + psubd m5, m3 + psrad m4, 8 + psrad m5, 8 + packssdw m4, m5 + pmulhrsw m4, m7 + paddw m0, m4 + pmaxsw m0, m6 + pminsw m0, m14 + mova [dstq+wq], m0 + add wq, 16 + jl .n1_loop + add dstq, dst_stridemp + movif32 dstm, dstq + ret + +%if ARCH_X86_32 + %if STACK_ALIGNMENT < 16 + %assign extra_stack 4*16 + %else + %assign extra_stack 2*16 + %endif +cglobal sgr_filter_3x3_16bpc, 1, 7, 8, -400*42-16-extra_stack, \ + dst, dst_stride, left, lpf, lpf_stride, w, params, h + %if STACK_ALIGNMENT < 16 + %define dstm dword [esp+calloff+16*2+4*0] + %define dst_stridemp dword [esp+calloff+16*2+4*1] + %define leftm dword [esp+calloff+16*2+4*2] + %define lpfm dword [esp+calloff+16*2+4*3] + %define lpf_stridem dword [esp+calloff+16*2+4*4] + %define w0m dword [esp+calloff+16*2+4*5] + %define hd dword [esp+calloff+16*2+4*6] + %define edgeb byte [esp+calloff+16*2+4*7] + %define edged dword [esp+calloff+16*2+4*7] + %define leftmp leftm + %else + %define w0m wm + %define hd dword r6m + %define edgeb byte r8m + %define edged dword r8m + %endif + %define hvsrcm dword [esp+calloff+4*0] + %define w1m dword [esp+calloff+4*1] + %define t3m dword [esp+calloff+4*2] + %define t4m dword [esp+calloff+4*3] + %define m8 [base+pd_8] + %define m9 [esp+calloff+16*1] + %define m10 [base+pd_0xf00801c7] + %define m11 [base+pd_34816] + %define m12 [base+sgr_lshuf3] + %define m13 [base+pw_1023] + %define m14 m6 + %define base r6-$$ + %assign calloff 0 + %if STACK_ALIGNMENT < 16 + mov dst_strideq, [rstk+stack_offset+ 8] + mov leftq, [rstk+stack_offset+12] + mov lpfq, [rstk+stack_offset+16] + mov lpf_strideq, [rstk+stack_offset+20] + mov wd, [rstk+stack_offset+24] + mov dstm, dstq + mov dst_stridemp, dst_strideq + mov leftm, leftq + mov r1, [rstk+stack_offset+28] + mov r2, [rstk+stack_offset+36] + mov lpfm, lpfq + mov lpf_stridem, lpf_strideq + mov hd, r1 + mov edged, r2 + %endif +%else +cglobal sgr_filter_3x3_16bpc, 5, 15, 15, -400*42-8, dst, dst_stride, left, lpf, \ + lpf_stride, w, edge, params, h +%endif +%if ARCH_X86_64 || STACK_ALIGNMENT >= 16 + movifnidn wd, wm +%endif +%if ARCH_X86_64 + mov paramsq, paramsmp + lea r13, [sgr_x_by_x-0xf03] + mov edged, r8m + add wd, wd + mov hd, r6m + movq m9, [paramsq+4] + add lpfq, wq + lea t1, [rsp+wq+12] + mova m8, [pd_8] + add dstq, wq + lea t3, [rsp+wq*2+400*12+8] + mova m10, [pd_0xf00801c7] + lea t4, [rsp+wq+400*32+8] + mova m11, [pd_34816] + pshuflw m7, m9, q3333 + pshufb m9, [pw_256] ; s1 + punpcklqdq m7, m7 ; w1 + neg wq + pxor m6, m6 + mova m13, [pw_1023] + psllw m7, 4 + mova m12, [sgr_lshuf3] + DEFINE_ARGS dst, dst_stride, left, lpf, lpf_stride, _, edge, _, h, _, w + %define lpfm [rsp] +%else + mov r1, [rstk+stack_offset+32] ; params + LEA r6, $$ + add wd, wd + movq m1, [r1+4] + add lpfm, wq + lea t1, [rsp+extra_stack+wq+20] + add dstq, wq + lea t3, [rsp+extra_stack+wq*2+400*12+16] + mov dstm, dstq + lea t4, [rsp+extra_stack+wq+400*32+16] + mov t3m, t3 + pshuflw m7, m1, q3333 + mov t4m, t4 + pshufb m1, [base+pw_256] ; s1 + punpcklqdq m7, m7 ; w1 + psllw m7, 4 + neg wq + mova m9, m1 + pxor m6, m6 + mov w1m, wd + sub wd, 4 + mov lpfq, lpfm + mov lpf_strideq, lpf_stridem + mov w0m, wd +%endif + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, lpf_strideq + mov t2, t1 + add t1, 400*6 + call .h_top + lea r10, [lpfq+lpf_strideq*4] + mov lpfq, dstq + add r10, lpf_strideq + mov lpfm, r10 ; below + movif32 t4, t4m + call .hv0 +.main: + dec hd + jz .height1 + movif32 lpfq, hvsrcm + add lpfq, dst_stridemp + call .hv1 + call .prep_n + sub hd, 2 + jl .extend_bottom +.main_loop: + movif32 lpfq, hvsrcm + add lpfq, dst_stridemp + call .hv0 +%if ARCH_X86_64 + test hb, hb +%else + mov r5, hd + test r5, r5 +%endif + jz .odd_height + movif32 lpfq, hvsrcm + add lpfq, dst_stridemp + call .hv1 + call .n0 + call .n1 + sub hd, 2 + jge .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .extend_bottom + mov lpfq, lpfm + call .hv0_bottom +%if ARCH_X86_64 + add lpfq, lpf_strideq +%else + mov lpfq, hvsrcm + add lpfq, lpf_stridem +%endif + call .hv1_bottom +.end: + call .n0 + call .n1 +.end2: + RET +.height1: + call .v1 + call .prep_n + jmp .odd_height_end +.odd_height: + call .v1 + call .n0 + call .n1 +.odd_height_end: + call .v0 + call .v1 + call .n0 + jmp .end2 +.extend_bottom: + call .v0 + call .v1 + jmp .end +.no_top: + lea r10, [lpfq+lpf_strideq*4] + mov lpfq, dstq + lea r10, [r10+lpf_strideq*2] + mov lpfm, r10 + call .h +%if ARCH_X86_64 + lea wq, [r5-4] +%else + mov wq, w0m + mov hvsrcm, lpfq +%endif + lea t2, [t1+400*6] +.top_fixup_loop: + mova m0, [t1+wq+400*0] + mova m1, [t1+wq+400*2] + mova m2, [t1+wq+400*4] + mova [t2+wq+400*0], m0 + mova [t2+wq+400*2], m1 + mova [t2+wq+400*4], m2 + add wq, 16 + jl .top_fixup_loop + movif32 t3, t3m + movif32 t4, t4m + call .v0 + jmp .main +.extend_right: + movd m1, wd + movd m5, [lpfq-2] + mova m2, [base+pw_256] + mova m3, [base+pb_0to15] + pshufb m1, m6 + pshufb m5, m2 + psubb m2, m1 + pcmpgtb m2, m3 + pand m4, m2 + pandn m2, m5 + por m4, m2 + ret +%assign stack_offset stack_offset+4 +%assign calloff 4 +.h: ; horizontal boxsum +%if ARCH_X86_64 + lea wq, [r5-4] +%else + %define leftq r5 +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movif32 leftq, leftm + movddup m5, [leftq] + movif32 wq, w0m + mova m4, [lpfq+wq+4] + add leftmp, 8 + palignr m4, m5, 12 + jmp .h_main +.h_extend_left: + movif32 wq, w0m + mova m4, [lpfq+wq+4] + pshufb m4, m12 + jmp .h_main +.h_top: +%if ARCH_X86_64 + lea wq, [r5-4] +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movif32 wq, w0m +.h_loop: + movu m4, [lpfq+wq+ 0] +.h_main: + movu m5, [lpfq+wq+16] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp wd, -18 + jl .h_have_right + call .extend_right +.h_have_right: + palignr m0, m5, m4, 2 + paddw m1, m4, m0 + punpcklwd m2, m4, m0 + pmaddwd m2, m2 + punpckhwd m3, m4, m0 + pmaddwd m3, m3 + palignr m5, m4, 4 + paddw m1, m5 ; sum + punpcklwd m4, m5, m6 + pmaddwd m4, m4 + punpckhwd m5, m6 + pmaddwd m5, m5 + paddd m2, m4 ; sumsq + paddd m3, m5 + mova [t1+wq+400*0], m1 + mova [t1+wq+400*2], m2 + mova [t1+wq+400*4], m3 + add wq, 16 + jl .h_loop + ret +ALIGN function_align +.hv0: ; horizontal boxsum + vertical boxsum + ab (even rows) +%if ARCH_X86_64 + lea wq, [r5-4] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv0_extend_left + movif32 leftq, leftm + movddup m5, [leftq] + movif32 wq, w0m + mova m4, [lpfq+wq+4] + add leftmp, 8 + palignr m4, m5, 12 + jmp .hv0_main +.hv0_extend_left: + movif32 wq, w0m + mova m4, [lpfq+wq+4] + pshufb m4, m12 + jmp .hv0_main +.hv0_bottom: +%if ARCH_X86_64 + lea wq, [r5-4] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv0_extend_left + movif32 wq, w0m +%if ARCH_X86_32 + jmp .hv0_loop_start +%endif +.hv0_loop: + movif32 lpfq, hvsrcm +.hv0_loop_start: + movu m4, [lpfq+wq+ 0] +.hv0_main: + movu m5, [lpfq+wq+16] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv0_have_right + cmp wd, -18 + jl .hv0_have_right + call .extend_right +.hv0_have_right: + palignr m0, m5, m4, 2 + paddw m1, m4, m0 + punpcklwd m2, m4, m0 + pmaddwd m2, m2 + punpckhwd m3, m4, m0 + pmaddwd m3, m3 + palignr m5, m4, 4 + paddw m1, m5 ; sum + punpcklwd m4, m5, m6 + pmaddwd m4, m4 + punpckhwd m5, m6 + pmaddwd m5, m5 + paddd m2, m4 ; sumsq + paddd m3, m5 + paddw m0, m1, [t1+wq+400*0] + paddd m4, m2, [t1+wq+400*2] + paddd m5, m3, [t1+wq+400*4] + mova [t1+wq+400*0], m1 + mova [t1+wq+400*2], m2 + mova [t1+wq+400*4], m3 + paddw m1, m0, [t2+wq+400*0] + paddd m2, m4, [t2+wq+400*2] + paddd m3, m5, [t2+wq+400*4] + mova [t2+wq+400*0], m0 + mova [t2+wq+400*2], m4 + mova [t2+wq+400*4], m5 + paddd m2, m8 + paddd m3, m8 + psrld m2, 4 ; (a + 8) >> 4 + psrld m3, 4 + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; ((a + 8) >> 4) * 9 + paddd m5, m3 + psrlw m3, m1, 1 + pavgw m3, m6 ; (b + 2) >> 2 + punpcklwd m2, m3, m6 + pmaddwd m2, m2 + punpckhwd m3, m6 + pmaddwd m3, m3 + punpcklwd m0, m1, m6 ; b + punpckhwd m1, m6 + MAXSD m4, m2, m14 + MAXSD m5, m3, m14 + psubd m4, m2 ; p + psubd m5, m3 + MULLD m4, m9, m14 ; p * s + MULLD m5, m9, m14 + pmaddwd m0, m10 ; b * 455 + pmaddwd m1, m10 + paddusw m4, m10 + paddusw m5, m10 + psrld m4, 20 ; min(z, 255) + movif32 t3, t3m + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, r0, dstm + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m14 + MULLD m1, m5, m14 +%if ARCH_X86_32 + pxor m6, m6 +%endif + paddd m0, m11 ; x * b * 455 + (1 << 11) + (1 << 15) + paddd m1, m11 + mova [t4+wq+4], m3 + psrld m0, 12 + psrld m1, 12 + mova [t3+wq*2+ 8], m0 + mova [t3+wq*2+24], m1 + add wq, 16 + jl .hv0_loop + ret +ALIGN function_align +.hv1: ; horizontal boxsums + vertical boxsums + ab (odd rows) +%if ARCH_X86_64 + lea wq, [r5-4] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv1_extend_left + movif32 leftq, leftm + movddup m5, [leftq] + movif32 wq, w0m + mova m4, [lpfq+wq+4] + add leftmp, 8 + palignr m4, m5, 12 + jmp .hv1_main +.hv1_extend_left: + movif32 wq, w0m + mova m4, [lpfq+wq+4] + pshufb m4, m12 + jmp .hv1_main +.hv1_bottom: +%if ARCH_X86_64 + lea wq, [r5-4] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv1_extend_left + movif32 wq, w0m +%if ARCH_X86_32 + jmp .hv1_loop_start +%endif +.hv1_loop: + movif32 lpfq, hvsrcm +.hv1_loop_start: + movu m4, [lpfq+wq+ 0] +.hv1_main: + movu m5, [lpfq+wq+16] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv1_have_right + cmp wd, -18 + jl .hv1_have_right + call .extend_right +.hv1_have_right: + palignr m1, m5, m4, 2 + paddw m0, m4, m1 + punpcklwd m2, m4, m1 + pmaddwd m2, m2 + punpckhwd m3, m4, m1 + pmaddwd m3, m3 + palignr m5, m4, 4 + paddw m0, m5 ; h sum + punpcklwd m1, m5, m6 + pmaddwd m1, m1 + punpckhwd m5, m6 + pmaddwd m5, m5 + paddd m2, m1 ; h sumsq + paddd m3, m5 + paddw m1, m0, [t2+wq+400*0] + paddd m4, m2, [t2+wq+400*2] + paddd m5, m3, [t2+wq+400*4] + mova [t2+wq+400*0], m0 + mova [t2+wq+400*2], m2 + mova [t2+wq+400*4], m3 + paddd m4, m8 + paddd m5, m8 + psrld m4, 4 ; (a + 8) >> 4 + psrld m5, 4 + pslld m2, m4, 3 + pslld m3, m5, 3 + paddd m4, m2 ; ((a + 8) >> 4) * 9 + paddd m5, m3 + psrlw m3, m1, 1 + pavgw m3, m6 ; (b + 2) >> 2 + punpcklwd m2, m3, m6 + pmaddwd m2, m2 + punpckhwd m3, m6 + pmaddwd m3, m3 + punpcklwd m0, m1, m6 ; b + punpckhwd m1, m6 + MAXSD m4, m2, m14 + MAXSD m5, m3, m14 + psubd m4, m2 ; p + psubd m5, m3 + MULLD m4, m9, m14 ; p * s + MULLD m5, m9, m14 + pmaddwd m0, m10 ; b * 455 + pmaddwd m1, m10 + paddusw m4, m10 + paddusw m5, m10 + psrld m4, 20 ; min(z, 255) + movif32 t3, t3m + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, r0, dstm + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m14 + MULLD m1, m5, m14 +%if ARCH_X86_32 + pxor m6, m6 +%endif + paddd m0, m11 ; x * b * 455 + (1 << 11) + (1 << 15) + paddd m1, m11 + mova [t4+wq*1+400*2 +4], m3 + psrld m0, 12 + psrld m1, 12 + mova [t3+wq*2+400*4+ 8], m0 + mova [t3+wq*2+400*4+24], m1 + add wq, 16 + jl .hv1_loop + mov r10, t2 + mov t2, t1 + mov t1, r10 + ret +.v0: ; vertical boxsums + ab (even rows) +%if ARCH_X86_64 + lea wq, [r5-4] +%else + mov wd, w0m +%endif +.v0_loop: + mova m0, [t1+wq+400*0] + mova m4, [t1+wq+400*2] + mova m5, [t1+wq+400*4] + paddw m0, m0 + paddd m4, m4 + paddd m5, m5 + paddw m1, m0, [t2+wq+400*0] + paddd m2, m4, [t2+wq+400*2] + paddd m3, m5, [t2+wq+400*4] + mova [t2+wq+400*0], m0 + mova [t2+wq+400*2], m4 + mova [t2+wq+400*4], m5 + paddd m2, m8 + paddd m3, m8 + psrld m2, 4 ; (a + 8) >> 4 + psrld m3, 4 + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; ((a + 8) >> 4) * 9 + paddd m5, m3 + psrlw m3, m1, 1 + pavgw m3, m6 ; (b + 2) >> 2 + punpcklwd m2, m3, m6 + pmaddwd m2, m2 + punpckhwd m3, m6 + pmaddwd m3, m3 + punpcklwd m0, m1, m6 ; b + punpckhwd m1, m6 + MAXSD m4, m2, m14 + MAXSD m5, m3, m14 + psubd m4, m2 ; p + psubd m5, m3 + MULLD m4, m9, m14 ; p * s + MULLD m5, m9, m14 + pmaddwd m0, m10 ; b * 455 + pmaddwd m1, m10 + paddusw m4, m10 + paddusw m5, m10 + psrld m4, 20 ; min(z, 255) + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, r0, dstm + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m14 + MULLD m1, m5, m14 +%if ARCH_X86_32 + pxor m6, m6 +%endif + paddd m0, m11 ; x * b * 455 + (1 << 11) + (1 << 15) + paddd m1, m11 + mova [t4+wq*1+400*0+ 4], m3 + psrld m0, 12 + psrld m1, 12 + mova [t3+wq*2+400*0+ 8], m0 + mova [t3+wq*2+400*0+24], m1 + add wq, 16 + jl .v0_loop + ret +.v1: ; vertical boxsums + ab (odd rows) +%if ARCH_X86_64 + lea wq, [r5-4] +%else + mov wd, w0m +%endif +.v1_loop: + mova m0, [t1+wq+400*0] + mova m4, [t1+wq+400*2] + mova m5, [t1+wq+400*4] + paddw m1, m0, [t2+wq+400*0] + paddd m2, m4, [t2+wq+400*2] + paddd m3, m5, [t2+wq+400*4] + mova [t2+wq+400*0], m0 + mova [t2+wq+400*2], m4 + mova [t2+wq+400*4], m5 + paddd m2, m8 + paddd m3, m8 + psrld m2, 4 ; (a + 8) >> 4 + psrld m3, 4 + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; ((a + 8) >> 4) * 9 + paddd m5, m3 + psrlw m3, m1, 1 + pavgw m3, m6 ; (b + 2) >> 2 + punpcklwd m2, m3, m6 + pmaddwd m2, m2 + punpckhwd m3, m6 + pmaddwd m3, m3 + punpcklwd m0, m1, m6 ; b + punpckhwd m1, m6 + MAXSD m4, m2, m14 + MAXSD m5, m3, m14 + psubd m4, m2 ; p + psubd m5, m3 + MULLD m4, m9, m14 ; p * s + MULLD m5, m9, m14 + pmaddwd m0, m10 ; b * 455 + pmaddwd m1, m10 + paddusw m4, m10 + paddusw m5, m10 + psrld m4, 20 ; min(z, 255) + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, r0, dstm + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m14 + MULLD m1, m5, m14 +%if ARCH_X86_32 + pxor m6, m6 +%endif + paddd m0, m11 ; x * b * 455 + (1 << 11) + (1 << 15) + paddd m1, m11 + mova [t4+wq*1+400*2+ 4], m3 + psrld m0, 12 + psrld m1, 12 + mova [t3+wq*2+400*4+ 8], m0 + mova [t3+wq*2+400*4+24], m1 + add wq, 16 + jl .v1_loop + mov r10, t2 + mov t2, t1 + mov t1, r10 + ret +.prep_n: ; initial neighbor setup + movif64 wq, r5 + movif32 wd, w1m +.prep_n_loop: + movu m0, [t4+wq*1+400*0+ 4] + movu m1, [t3+wq*2+400*0+ 8] + movu m2, [t3+wq*2+400*0+24] + movu m3, [t4+wq*1+400*0+ 2] + movu m4, [t3+wq*2+400*0+ 4] + movu m5, [t3+wq*2+400*0+20] + paddw m0, [t4+wq*1+400*0+ 0] + paddd m1, [t3+wq*2+400*0+ 0] + paddd m2, [t3+wq*2+400*0+16] + paddw m3, m0 + paddd m4, m1 + paddd m5, m2 + psllw m3, 2 ; a[-1] 444 + pslld m4, 2 ; b[-1] 444 + pslld m5, 2 + psubw m3, m0 ; a[-1] 343 + psubd m4, m1 ; b[-1] 343 + psubd m5, m2 + mova [t4+wq*1+400*4], m3 + mova [t3+wq*2+400*8+ 0], m4 + mova [t3+wq*2+400*8+16], m5 + movu m0, [t4+wq*1+400*2+ 4] + movu m1, [t3+wq*2+400*4+ 8] + movu m2, [t3+wq*2+400*4+24] + movu m3, [t4+wq*1+400*2+ 2] + movu m4, [t3+wq*2+400*4+ 4] + movu m5, [t3+wq*2+400*4+20] + paddw m0, [t4+wq*1+400*2+ 0] + paddd m1, [t3+wq*2+400*4+ 0] + paddd m2, [t3+wq*2+400*4+16] + paddw m3, m0 + paddd m4, m1 + paddd m5, m2 + psllw m3, 2 ; a[ 0] 444 + pslld m4, 2 ; b[ 0] 444 + pslld m5, 2 + mova [t4+wq*1+400* 6], m3 + mova [t3+wq*2+400*12+ 0], m4 + mova [t3+wq*2+400*12+16], m5 + psubw m3, m0 ; a[ 0] 343 + psubd m4, m1 ; b[ 0] 343 + psubd m5, m2 + mova [t4+wq*1+400* 8], m3 + mova [t3+wq*2+400*16+ 0], m4 + mova [t3+wq*2+400*16+16], m5 + add wq, 16 + jl .prep_n_loop + ret +ALIGN function_align +.n0: ; neighbor + output (even rows) + movif64 wq, r5 + movif32 wd, w1m +.n0_loop: + movu m3, [t4+wq*1+400*0+4] + movu m1, [t4+wq*1+400*0+2] + paddw m3, [t4+wq*1+400*0+0] + paddw m1, m3 + psllw m1, 2 ; a[ 1] 444 + psubw m2, m1, m3 ; a[ 1] 343 + paddw m3, m2, [t4+wq*1+400*4] + paddw m3, [t4+wq*1+400*6] + mova [t4+wq*1+400*4], m2 + mova [t4+wq*1+400*6], m1 + movu m4, [t3+wq*2+400*0+8] + movu m1, [t3+wq*2+400*0+4] + paddd m4, [t3+wq*2+400*0+0] + paddd m1, m4 + pslld m1, 2 ; b[ 1] 444 + psubd m2, m1, m4 ; b[ 1] 343 + paddd m4, m2, [t3+wq*2+400* 8+ 0] + paddd m4, [t3+wq*2+400*12+ 0] + mova [t3+wq*2+400* 8+ 0], m2 + mova [t3+wq*2+400*12+ 0], m1 + movu m5, [t3+wq*2+400*0+24] + movu m1, [t3+wq*2+400*0+20] + paddd m5, [t3+wq*2+400*0+16] + paddd m1, m5 + pslld m1, 2 + psubd m2, m1, m5 + paddd m5, m2, [t3+wq*2+400* 8+16] + paddd m5, [t3+wq*2+400*12+16] + mova [t3+wq*2+400* 8+16], m2 + mova [t3+wq*2+400*12+16], m1 + mova m0, [dstq+wq] + punpcklwd m1, m0, m6 + punpcklwd m2, m3, m6 + pmaddwd m2, m1 ; a * src + punpckhwd m1, m0, m6 + punpckhwd m3, m6 + pmaddwd m3, m1 + psubd m4, m2 ; b - a * src + (1 << 8) + psubd m5, m3 + psrad m4, 9 + psrad m5, 9 + packssdw m4, m5 + pmulhrsw m4, m7 + paddw m0, m4 + pmaxsw m0, m6 + pminsw m0, m13 + mova [dstq+wq], m0 + add wq, 16 + jl .n0_loop + add dstq, dst_stridemp + ret +ALIGN function_align +.n1: ; neighbor + output (odd rows) + movif64 wq, r5 + movif32 wd, w1m +.n1_loop: + movu m3, [t4+wq*1+400*2+4] + movu m1, [t4+wq*1+400*2+2] + paddw m3, [t4+wq*1+400*2+0] + paddw m1, m3 + psllw m1, 2 ; a[ 1] 444 + psubw m2, m1, m3 ; a[ 1] 343 + paddw m3, m2, [t4+wq*1+400*6] + paddw m3, [t4+wq*1+400*8] + mova [t4+wq*1+400*6], m1 + mova [t4+wq*1+400*8], m2 + movu m4, [t3+wq*2+400*4+8] + movu m1, [t3+wq*2+400*4+4] + paddd m4, [t3+wq*2+400*4+0] + paddd m1, m4 + pslld m1, 2 ; b[ 1] 444 + psubd m2, m1, m4 ; b[ 1] 343 + paddd m4, m2, [t3+wq*2+400*12+ 0] + paddd m4, [t3+wq*2+400*16+ 0] + mova [t3+wq*2+400*12+ 0], m1 + mova [t3+wq*2+400*16+ 0], m2 + movu m5, [t3+wq*2+400*4+24] + movu m1, [t3+wq*2+400*4+20] + paddd m5, [t3+wq*2+400*4+16] + paddd m1, m5 + pslld m1, 2 + psubd m2, m1, m5 + paddd m5, m2, [t3+wq*2+400*12+16] + paddd m5, [t3+wq*2+400*16+16] + mova [t3+wq*2+400*12+16], m1 + mova [t3+wq*2+400*16+16], m2 + mova m0, [dstq+wq] + punpcklwd m1, m0, m6 + punpcklwd m2, m3, m6 + pmaddwd m2, m1 ; a * src + punpckhwd m1, m0, m6 + punpckhwd m3, m6 + pmaddwd m3, m1 + psubd m4, m2 ; b - a * src + (1 << 8) + psubd m5, m3 + psrad m4, 9 + psrad m5, 9 + packssdw m4, m5 + pmulhrsw m4, m7 + paddw m0, m4 + pmaxsw m0, m6 + pminsw m0, m13 + mova [dstq+wq], m0 + add wq, 16 + jl .n1_loop + add dstq, dst_stridemp + movif32 dstm, dstq + ret + +%if ARCH_X86_32 + %if STACK_ALIGNMENT < 16 + %assign extra_stack 10*16 + %else + %assign extra_stack 8*16 + %endif +cglobal sgr_filter_mix_16bpc, 1, 7, 8, -400*66-48-extra_stack, \ + dst, dst_stride, left, lpf, lpf_stride, w, params, h + %if STACK_ALIGNMENT < 16 + %define dstm dword [esp+calloff+16*8+4*0] + %define dst_stridemp dword [esp+calloff+16*8+4*1] + %define leftm dword [esp+calloff+16*8+4*2] + %define lpfm dword [esp+calloff+16*8+4*3] + %define lpf_stridem dword [esp+calloff+16*8+4*4] + %define w0m dword [esp+calloff+16*8+4*5] + %define hd dword [esp+calloff+16*8+4*6] + %define edgeb byte [esp+calloff+16*8+4*7] + %define edged dword [esp+calloff+16*8+4*7] + %define leftmp leftm + %else + %define w0m wm + %define hd dword r6m + %define edgeb byte r8m + %define edged dword r8m + %endif + %define hvsrcm dword [esp+calloff+4*0] + %define w1m dword [esp+calloff+4*1] + %define t3m dword [esp+calloff+4*2] + %define t4m dword [esp+calloff+4*3] + %xdefine m8 m6 + %define m9 [base+pd_8] + %define m10 [base+pd_34816] + %define m11 [base+pd_0xf00801c7] + %define m12 [base+pd_0xf00800a4] + %define m13 [esp+calloff+16*4] + %define m14 [esp+calloff+16*5] + %define m15 [esp+calloff+16*6] + %define m6 [esp+calloff+16*7] + %define base r6-$$ + %assign calloff 0 + %if STACK_ALIGNMENT < 16 + mov dst_strideq, [rstk+stack_offset+ 8] + mov leftq, [rstk+stack_offset+12] + mov lpfq, [rstk+stack_offset+16] + mov lpf_strideq, [rstk+stack_offset+20] + mov wd, [rstk+stack_offset+24] + mov dstm, dstq + mov dst_stridemp, dst_strideq + mov leftm, leftq + mov r1, [rstk+stack_offset+28] + mov r2, [rstk+stack_offset+36] + mov lpfm, lpfq + mov lpf_stridem, lpf_strideq + mov hd, r1 + mov edged, r2 + %endif +%else +cglobal sgr_filter_mix_16bpc, 5, 15, 16, -400*66-40, dst, dst_stride, left, \ + lpf, lpf_stride, w, edge, \ + params, h +%endif +%if ARCH_X86_64 || STACK_ALIGNMENT >= 16 + movifnidn wd, wm +%endif +%if ARCH_X86_64 + mov paramsq, paramsmp + lea r13, [sgr_x_by_x-0xf03] + mov edged, r8m + add wd, wd + mov hd, r6m + mova m14, [paramsq] + add lpfq, wq + mova m9, [pd_8] + lea t1, [rsp+wq+44] + mova m10, [pd_34816] + add dstq, wq + mova m11, [pd_0xf00801c7] + lea t3, [rsp+wq*2+400*24+40] + mova m12, [pd_0xf00800a4] + lea t4, [rsp+wq+400*52+40] + neg wq + pshufd m15, m14, q2222 ; w0 w1 + punpcklwd m14, m14 + pshufd m13, m14, q0000 ; s0 + pshufd m14, m14, q2222 ; s1 + pxor m6, m6 + psllw m15, 2 + DEFINE_ARGS dst, dst_stride, left, lpf, lpf_stride, _, edge, _, h, _, w + %define lpfm [rsp] +%else + mov r1, [rstk+stack_offset+32] ; params + LEA r6, $$ + add wd, wd + mova m2, [r1] + add lpfm, wq + lea t1, [rsp+extra_stack+wq+52] + add dstq, wq + lea t3, [rsp+extra_stack+wq*2+400*24+48] + mov dstm, dstq + lea t4, [rsp+extra_stack+wq+400*52+48] + mov t3m, t3 + mov t4m, t4 + neg wq + pshuflw m0, m2, q0000 + pshuflw m1, m2, q2222 + pshufhw m2, m2, q1010 + punpcklqdq m0, m0 ; s0 + punpcklqdq m1, m1 ; s1 + punpckhqdq m2, m2 ; w0 w1 + mov w1m, wd + pxor m3, m3 + psllw m2, 2 + mova m13, m0 + mova m14, m1 + sub wd, 4 + mova m15, m2 + mova m6, m3 + mov lpfq, lpfm + mov lpf_strideq, lpf_stridem + mov w0m, wd +%endif + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, lpf_strideq + mov t2, t1 +%if ARCH_X86_64 + call mangle(private_prefix %+ _sgr_filter_5x5_16bpc_ssse3).top_fixup +%else + mov wq, w0m + call mangle(private_prefix %+ _sgr_filter_5x5_16bpc_ssse3).top_fixup_loop +%endif + add t1, 400*12 + call .h_top + lea r10, [lpfq+lpf_strideq*4] + mov lpfq, dstq + add r10, lpf_strideq + mov lpfm, r10 ; below + movif32 t4, t4m + call .hv0 +.main: + dec hd + jz .height1 + movif32 lpfq, hvsrcm + add lpfq, dst_stridemp + call .hv1 + call .prep_n + sub hd, 2 + jl .extend_bottom +.main_loop: + movif32 lpfq, hvsrcm + add lpfq, dst_stridemp + call .hv0 +%if ARCH_X86_64 + test hd, hd +%else + mov r5, hd + test r5, r5 +%endif + jz .odd_height + movif32 lpfq, hvsrcm + add lpfq, dst_stridemp + call .hv1 + call .n0 + call .n1 + sub hd, 2 + jge .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .extend_bottom + mov lpfq, lpfm + call .hv0_bottom +%if ARCH_X86_64 + add lpfq, lpf_strideq +%else + mov lpfq, hvsrcm + add lpfq, lpf_stridem +%endif + call .hv1_bottom +.end: + call .n0 + call .n1 +.end2: + RET +.height1: + call .v1 + call .prep_n + jmp .odd_height_end +.odd_height: + call .v1 + call .n0 + call .n1 +.odd_height_end: + call .v0 + call .v1 + call .n0 + jmp .end2 +.extend_bottom: + call .v0 + call .v1 + jmp .end +.no_top: + lea r10, [lpfq+lpf_strideq*4] + mov lpfq, dstq + lea r10, [r10+lpf_strideq*2] + mov lpfm, r10 + call .h +%if ARCH_X86_64 + lea wq, [r5-4] +%else + mov wq, w0m + mov hvsrcm, lpfq +%endif + lea t2, [t1+400*12] +.top_fixup_loop: + mova m0, [t1+wq+400* 0] + mova m1, [t1+wq+400* 2] + mova m2, [t1+wq+400* 4] + paddw m0, m0 + mova m3, [t1+wq+400* 6] + paddd m1, m1 + mova m4, [t1+wq+400* 8] + paddd m2, m2 + mova m5, [t1+wq+400*10] + mova [t2+wq+400* 0], m0 + mova [t2+wq+400* 2], m1 + mova [t2+wq+400* 4], m2 + mova [t2+wq+400* 6], m3 + mova [t2+wq+400* 8], m4 + mova [t2+wq+400*10], m5 + add wq, 16 + jl .top_fixup_loop + movif32 t3, t3m + movif32 t4, t4m + call .v0 + jmp .main +.h: ; horizontal boxsum +%assign stack_offset stack_offset+4 +%assign calloff 4 +%if ARCH_X86_64 + lea wq, [r5-4] +%else + %define leftq r5 +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movif32 leftq, leftm + movddup m5, [leftq] + movif32 wq, w0m + mova m4, [lpfq+wq+4] + add leftmp, 8 + palignr m4, m5, 10 + jmp .h_main +.h_extend_left: + movif32 wq, w0m + mova m4, [lpfq+wq+4] + pshufb m4, [base+sgr_lshuf5] + jmp .h_main +.h_top: +%if ARCH_X86_64 + lea wq, [r5-4] +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movif32 wq, w0m +.h_loop: + movu m4, [lpfq+wq- 2] +.h_main: + movu m5, [lpfq+wq+14] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp wd, -20 + jl .h_have_right +%if ARCH_X86_32 + pxor m8, m8 +%endif + call mangle(private_prefix %+ _sgr_filter_5x5_16bpc_ssse3).extend_right +.h_have_right: + palignr m3, m5, m4, 2 + palignr m0, m5, m4, 4 + paddw m1, m3, m0 + punpcklwd m2, m3, m0 + pmaddwd m2, m2 + punpckhwd m3, m0 + pmaddwd m3, m3 + palignr m0, m5, m4, 6 + paddw m1, m0 ; sum3 + punpcklwd m7, m0, m6 + pmaddwd m7, m7 + punpckhwd m0, m6 + pmaddwd m0, m0 + paddd m2, m7 ; sumsq3 + palignr m5, m4, 8 + punpcklwd m7, m5, m4 + paddw m8, m4, m5 + pmaddwd m7, m7 + punpckhwd m5, m4 + pmaddwd m5, m5 + paddd m3, m0 + mova [t1+wq+400* 6], m1 + mova [t1+wq+400* 8], m2 + mova [t1+wq+400*10], m3 + paddw m8, m1 ; sum5 + paddd m7, m2 ; sumsq5 + paddd m5, m3 + mova [t1+wq+400* 0], m8 + mova [t1+wq+400* 2], m7 + mova [t1+wq+400* 4], m5 + add wq, 16 + jl .h_loop + ret +ALIGN function_align +.hv0: ; horizontal boxsum + vertical boxsum + ab3 (even rows) +%if ARCH_X86_64 + lea wq, [r5-4] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv0_extend_left + movif32 leftq, leftm + movddup m5, [leftq] + movif32 wq, w0m + mova m4, [lpfq+wq+4] + add leftmp, 8 + palignr m4, m5, 10 + jmp .hv0_main +.hv0_extend_left: + movif32 wq, w0m + mova m4, [lpfq+wq+4] + pshufb m4, [base+sgr_lshuf5] + jmp .hv0_main +.hv0_bottom: +%if ARCH_X86_64 + lea wq, [r5-4] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv0_extend_left + movif32 wq, w0m +%if ARCH_X86_32 + jmp .hv0_loop_start +%endif +.hv0_loop: + movif32 lpfq, hvsrcm +.hv0_loop_start: + movu m4, [lpfq+wq- 2] +.hv0_main: + movu m5, [lpfq+wq+14] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv0_have_right + cmp wd, -20 + jl .hv0_have_right +%if ARCH_X86_32 + pxor m8, m8 +%endif + call mangle(private_prefix %+ _sgr_filter_5x5_16bpc_ssse3).extend_right +.hv0_have_right: + palignr m3, m5, m4, 2 + palignr m0, m5, m4, 4 + movif32 t3, t3m + paddw m1, m3, m0 + punpcklwd m2, m3, m0 + pmaddwd m2, m2 + punpckhwd m3, m0 + pmaddwd m3, m3 + palignr m0, m5, m4, 6 + paddw m1, m0 ; h sum3 + punpcklwd m7, m0, m6 + pmaddwd m7, m7 + punpckhwd m0, m6 + pmaddwd m0, m0 + paddd m2, m7 ; h sumsq3 + palignr m5, m4, 8 + punpcklwd m7, m5, m4 + paddw m8, m4, m5 + pmaddwd m7, m7 + punpckhwd m5, m4 + pmaddwd m5, m5 + paddd m3, m0 + paddw m8, m1 ; h sum5 + paddd m7, m2 ; h sumsq5 + paddd m5, m3 + mova [t3+wq*2+400*8+ 8], m8 + mova [t3+wq*2+400*0+ 8], m7 + mova [t3+wq*2+400*0+24], m5 + paddw m8, [t1+wq+400* 0] + paddd m7, [t1+wq+400* 2] + paddd m5, [t1+wq+400* 4] + mova [t1+wq+400* 0], m8 + mova [t1+wq+400* 2], m7 + mova [t1+wq+400* 4], m5 + paddw m0, m1, [t1+wq+400* 6] + paddd m4, m2, [t1+wq+400* 8] + paddd m5, m3, [t1+wq+400*10] + mova [t1+wq+400* 6], m1 + mova [t1+wq+400* 8], m2 + mova [t1+wq+400*10], m3 + paddw m1, m0, [t2+wq+400* 6] + paddd m2, m4, [t2+wq+400* 8] + paddd m3, m5, [t2+wq+400*10] + mova [t2+wq+400* 6], m0 + mova [t2+wq+400* 8], m4 + mova [t2+wq+400*10], m5 + paddd m2, m9 + paddd m3, m9 + psrld m2, 4 ; (a3 + 8) >> 4 + psrld m3, 4 +%if ARCH_X86_32 + pxor m7, m7 +%else + SWAP m7, m6 +%endif + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; ((a3 + 8) >> 4) * 9 + paddd m5, m3 + psrlw m3, m1, 1 + pavgw m3, m7 ; (b3 + 2) >> 2 + punpcklwd m2, m3, m7 + pmaddwd m2, m2 + punpckhwd m3, m7 + pmaddwd m3, m3 + punpcklwd m0, m1, m7 ; b3 + punpckhwd m1, m7 +%if ARCH_X86_64 + SWAP m7, m6 +%endif + MAXSD m4, m2, m7 + MAXSD m5, m3, m7 + psubd m4, m2 ; p3 + psubd m5, m3 + MULLD m4, m14, m7 ; p3 * s1 + MULLD m5, m14, m7 + pmaddwd m0, m11 ; b3 * 455 + pmaddwd m1, m11 + paddusw m4, m11 + paddusw m5, m11 + psrld m4, 20 ; min(z3, 255) + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, r0, dstm + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m7 + MULLD m1, m5, m7 + paddd m0, m10 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + paddd m1, m10 + mova [t4+wq*1+400*2+ 4], m3 + psrld m0, 12 + psrld m1, 12 + mova [t3+wq*2+400*4+ 8], m0 + mova [t3+wq*2+400*4+24], m1 + add wq, 16 + jl .hv0_loop + ret +ALIGN function_align +.hv1: ; horizontal boxsums + vertical boxsums + ab (odd rows) +%if ARCH_X86_64 + lea wq, [r5-4] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv1_extend_left + movif32 leftq, leftm + movddup m5, [leftq] + movif32 wq, w0m + mova m4, [lpfq+wq+4] + add leftmp, 8 + palignr m4, m5, 10 + jmp .hv1_main +.hv1_extend_left: + movif32 wq, w0m + mova m4, [lpfq+wq+4] + pshufb m4, [base+sgr_lshuf5] + jmp .hv1_main +.hv1_bottom: +%if ARCH_X86_64 + lea wq, [r5-4] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv1_extend_left + movif32 wq, w0m +%if ARCH_X86_32 + jmp .hv1_loop_start +%endif +.hv1_loop: + movif32 lpfq, hvsrcm +.hv1_loop_start: + movu m4, [lpfq+wq- 2] +.hv1_main: + movu m5, [lpfq+wq+14] + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv1_have_right + cmp wd, -20 + jl .hv1_have_right +%if ARCH_X86_32 + pxor m8, m8 +%endif + call mangle(private_prefix %+ _sgr_filter_5x5_16bpc_ssse3).extend_right +.hv1_have_right: + palignr m7, m5, m4, 2 + palignr m3, m5, m4, 4 + paddw m2, m7, m3 + punpcklwd m0, m7, m3 + pmaddwd m0, m0 + punpckhwd m7, m3 + pmaddwd m7, m7 + palignr m3, m5, m4, 6 + paddw m2, m3 ; h sum3 + punpcklwd m1, m3, m6 + pmaddwd m1, m1 + punpckhwd m3, m6 + pmaddwd m3, m3 + paddd m0, m1 ; h sumsq3 + palignr m5, m4, 8 + punpckhwd m1, m4, m5 + paddw m8, m4, m5 + pmaddwd m1, m1 + punpcklwd m4, m5 + pmaddwd m4, m4 + paddd m7, m3 + paddw m5, m2, [t2+wq+400* 6] + mova [t2+wq+400* 6], m2 + paddw m8, m2 ; h sum5 + paddd m2, m0, [t2+wq+400* 8] + paddd m3, m7, [t2+wq+400*10] + mova [t2+wq+400* 8], m0 + mova [t2+wq+400*10], m7 + paddd m4, m0 ; h sumsq5 + paddd m1, m7 + paddd m2, m9 + paddd m3, m9 + psrld m2, 4 ; (a3 + 8) >> 4 + psrld m3, 4 + pslld m0, m2, 3 + pslld m7, m3, 3 + paddd m2, m0 ; ((a3 + 8) >> 4) * 9 + paddd m3, m7 + psrlw m7, m5, 1 + pavgw m7, m6 ; (b3 + 2) >> 2 + punpcklwd m0, m7, m6 + pmaddwd m0, m0 + punpckhwd m7, m6 + pmaddwd m7, m7 +%if ARCH_X86_32 + mova [esp+20], m8 +%else + SWAP m8, m6 +%endif + MAXSD m2, m0, m8 + MAXSD m3, m7, m8 + pxor m8, m8 + psubd m2, m0 ; p3 + psubd m3, m7 + punpcklwd m0, m5, m8 ; b3 + punpckhwd m5, m8 + MULLD m2, m14, m8 ; p3 * s1 + MULLD m3, m14, m8 + pmaddwd m0, m11 ; b3 * 455 + pmaddwd m5, m11 + paddusw m2, m11 + paddusw m3, m11 + psrld m2, 20 ; min(z3, 255) + movif32 t3, t3m + psrld m3, 20 + GATHER_X_BY_X m8, m2, m3, r0, dstm + punpcklwd m2, m8, m8 + punpckhwd m3, m8, m8 + MULLD m0, m2, m7 + MULLD m5, m3, m7 + paddd m0, m10 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + paddd m5, m10 + psrld m0, 12 + psrld m5, 12 + mova [t4+wq*1+400*4+4], m8 + mova [t3+wq*2+400*8+ 8], m0 + mova [t3+wq*2+400*8+24], m5 +%if ARCH_X86_32 + mova m8, [esp+20] +%else + SWAP m6, m8 + pxor m6, m6 +%endif + paddw m5, m8, [t2+wq+400*0] + paddd m2, m4, [t2+wq+400*2] + paddd m3, m1, [t2+wq+400*4] + paddw m5, [t1+wq+400*0] + paddd m2, [t1+wq+400*2] + paddd m3, [t1+wq+400*4] + mova [t2+wq+400*0], m8 + paddd m2, m9 + paddd m3, m9 + psrld m2, 4 ; (a5 + 8) >> 4 + psrld m3, 4 + mova [t2+wq+400*2], m4 + pslld m8, m2, 4 + mova [t2+wq+400*4], m1 + pslld m4, m3, 4 + paddd m8, m2 + pslld m2, 3 + paddd m4, m3 + pslld m3, 3 + paddd m2, m8 ; ((a5 + 8) >> 4) * 25 + paddd m3, m4 +%if ARCH_X86_32 + pxor m7, m7 +%else + SWAP m7, m6 +%endif + psrlw m1, m5, 1 + pavgw m1, m7 ; (b5 + 2) >> 2 + punpcklwd m4, m1, m7 + pmaddwd m4, m4 + punpckhwd m1, m7 + pmaddwd m1, m1 + punpcklwd m0, m5, m7 ; b5 + punpckhwd m5, m7 +%if ARCH_X86_64 + SWAP m7, m6 +%endif + MAXSD m2, m4, m7 + psubd m2, m4 ; p5 + MAXSD m3, m1, m7 + psubd m3, m1 + MULLD m2, m13, m7 ; p5 * s0 + MULLD m3, m13, m7 + pmaddwd m0, m12 ; b5 * 164 + pmaddwd m5, m12 + paddusw m2, m12 + paddusw m3, m12 + psrld m2, 20 ; min(z5, 255) + psrld m3, 20 + GATHER_X_BY_X m1, m2, m3, r0, dstm + punpcklwd m2, m1, m1 + punpckhwd m3, m1, m1 + MULLD m0, m2, m7 + MULLD m5, m3, m7 + paddd m0, m10 ; x5 * b5 * 164 + (1 << 11) + (1 << 15) + paddd m5, m10 + mova [t4+wq*1+400*0+ 4], m1 + psrld m0, 12 + psrld m5, 12 + mova [t3+wq*2+400*0+ 8], m0 + mova [t3+wq*2+400*0+24], m5 + add wq, 16 + jl .hv1_loop + mov r10, t2 + mov t2, t1 + mov t1, r10 + ret +.v0: ; vertical boxsums + ab3 (even rows) +%if ARCH_X86_64 + lea wq, [r5-4] +%else + mov wd, w0m +%endif +.v0_loop: + mova m0, [t1+wq+400* 6] + mova m4, [t1+wq+400* 8] + mova m5, [t1+wq+400*10] + paddw m0, m0 + paddd m4, m4 + paddd m5, m5 + paddw m1, m0, [t2+wq+400* 6] + paddd m2, m4, [t2+wq+400* 8] + paddd m3, m5, [t2+wq+400*10] + mova [t2+wq+400* 6], m0 + mova [t2+wq+400* 8], m4 + mova [t2+wq+400*10], m5 + paddd m2, m9 + paddd m3, m9 + psrld m2, 4 ; (a3 + 8) >> 4 + psrld m3, 4 +%if ARCH_X86_32 + pxor m7, m7 +%else + SWAP m7, m6 +%endif + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; ((a3 + 8) >> 4) * 9 + paddd m5, m3 + psrlw m3, m1, 1 + pavgw m3, m7 ; (b3 + 2) >> 2 + punpcklwd m2, m3, m7 + pmaddwd m2, m2 + punpckhwd m3, m7 + pmaddwd m3, m3 + punpcklwd m0, m1, m7 ; b3 + punpckhwd m1, m7 +%if ARCH_X86_64 + SWAP m7, m6 +%endif + MAXSD m4, m2, m7 + MAXSD m5, m3, m7 + psubd m4, m2 ; p3 + psubd m5, m3 + MULLD m4, m14, m7 ; p3 * s1 + MULLD m5, m14, m7 + pmaddwd m0, m11 ; b3 * 455 + pmaddwd m1, m11 + paddusw m4, m11 + paddusw m5, m11 + psrld m4, 20 ; min(z3, 255) + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, r0, dstm + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m7 + MULLD m1, m5, m7 + paddd m0, m10 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + paddd m1, m10 + mova [t4+wq*1+400*2+4], m3 + psrld m0, 12 + psrld m1, 12 + mova m3, [t1+wq+400*0] + mova m4, [t1+wq+400*2] + mova m5, [t1+wq+400*4] + mova [t3+wq*2+400*8+ 8], m3 + mova [t3+wq*2+400*0+ 8], m4 + mova [t3+wq*2+400*0+24], m5 + paddw m3, m3 ; cc5 + paddd m4, m4 + paddd m5, m5 + mova [t1+wq+400*0], m3 + mova [t1+wq+400*2], m4 + mova [t1+wq+400*4], m5 + mova [t3+wq*2+400*4+ 8], m0 + mova [t3+wq*2+400*4+24], m1 + add wq, 16 + jl .v0_loop + ret +.v1: ; vertical boxsums + ab (odd rows) +%if ARCH_X86_64 + lea wq, [r5-4] +%else + mov wd, w0m +%endif +.v1_loop: + mova m4, [t1+wq+400* 6] + mova m5, [t1+wq+400* 8] + mova m7, [t1+wq+400*10] + paddw m1, m4, [t2+wq+400* 6] + paddd m2, m5, [t2+wq+400* 8] + paddd m3, m7, [t2+wq+400*10] + mova [t2+wq+400* 6], m4 + mova [t2+wq+400* 8], m5 + mova [t2+wq+400*10], m7 + paddd m2, m9 + paddd m3, m9 + psrld m2, 4 ; (a3 + 8) >> 4 + psrld m3, 4 +%if ARCH_X86_32 + pxor m7, m7 +%else + SWAP m7, m6 +%endif + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; ((a3 + 8) >> 4) * 9 + paddd m5, m3 + psrlw m3, m1, 1 + pavgw m3, m7 ; (b3 + 2) >> 2 + punpcklwd m2, m3, m7 + pmaddwd m2, m2 + punpckhwd m3, m7 + pmaddwd m3, m3 + punpcklwd m0, m1, m7 ; b3 + punpckhwd m1, m7 +%if ARCH_X86_64 + SWAP m7, m6 +%endif + MAXSD m4, m2, m7 + MAXSD m5, m3, m7 + psubd m4, m2 ; p3 + psubd m5, m3 + MULLD m4, m14, m7 ; p3 * s1 + MULLD m5, m14, m7 + pmaddwd m0, m11 ; b3 * 455 + pmaddwd m1, m11 + paddusw m4, m11 + paddusw m5, m11 + psrld m4, 20 ; min(z3, 255) + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, r0, dstm + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m7 + MULLD m1, m5, m7 + paddd m0, m10 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + paddd m1, m10 + mova [t4+wq*1+400*4+4], m3 + psrld m0, 12 + psrld m8, m1, 12 + mova m4, [t3+wq*2+400*8+ 8] + mova m5, [t3+wq*2+400*0+ 8] + mova m7, [t3+wq*2+400*0+24] + paddw m1, m4, [t2+wq+400*0] + paddd m2, m5, [t2+wq+400*2] + paddd m3, m7, [t2+wq+400*4] + paddw m1, [t1+wq+400*0] + paddd m2, [t1+wq+400*2] + paddd m3, [t1+wq+400*4] + mova [t2+wq+400*0], m4 + mova [t2+wq+400*2], m5 + mova [t2+wq+400*4], m7 + paddd m2, m9 + paddd m3, m9 + psrld m2, 4 ; (a5 + 8) >> 4 + psrld m3, 4 + mova [t3+wq*2+400*8+ 8], m0 + pslld m4, m2, 4 + mova [t3+wq*2+400*8+24], m8 + pslld m5, m3, 4 + paddd m4, m2 + pslld m2, 3 + paddd m5, m3 + pslld m3, 3 + paddd m2, m4 + paddd m3, m5 +%if ARCH_X86_32 + pxor m7, m7 +%else + SWAP m7, m6 +%endif + psrlw m5, m1, 1 + pavgw m5, m7 ; (b5 + 2) >> 2 + punpcklwd m4, m5, m7 + pmaddwd m4, m4 + punpckhwd m5, m7 + pmaddwd m5, m5 + punpcklwd m0, m1, m7 ; b5 + punpckhwd m1, m7 +%if ARCH_X86_64 + SWAP m7, m6 +%endif + MAXSD m2, m4, m7 + psubd m2, m4 ; p5 + MAXSD m3, m5, m7 + psubd m3, m5 + MULLD m2, m13, m7 ; p5 * s0 + MULLD m3, m13, m7 + pmaddwd m0, m12 ; b5 * 164 + pmaddwd m1, m12 + paddusw m2, m12 + paddusw m3, m12 + psrld m2, 20 ; min(z5, 255) + psrld m3, 20 + GATHER_X_BY_X m4, m2, m3, r0, dstm + punpcklwd m2, m4, m4 + punpckhwd m3, m4, m4 + MULLD m0, m2, m7 + MULLD m1, m3, m7 + paddd m0, m10 ; x5 * b5 * 164 + (1 << 11) + (1 << 15) + paddd m1, m10 + mova [t4+wq*1+400*0+ 4], m4 + psrld m0, 12 + psrld m1, 12 + mova [t3+wq*2+400*0+ 8], m0 + mova [t3+wq*2+400*0+24], m1 + add wq, 16 + jl .v1_loop + mov r10, t2 + mov t2, t1 + mov t1, r10 + ret +.prep_n: ; initial neighbor setup + movif64 wq, r5 + movif32 wd, w1m +.prep_n_loop: + movu m0, [t4+wq*1+400*0+ 2] + movu m1, [t3+wq*2+400*0+ 4] + movu m2, [t3+wq*2+400*0+20] + movu m7, [t4+wq*1+400*0+ 4] + movu m8, [t3+wq*2+400*0+ 8] + paddw m3, m0, [t4+wq*1+400*0+ 0] + paddd m4, m1, [t3+wq*2+400*0+ 0] + paddd m5, m2, [t3+wq*2+400*0+16] + paddw m3, m7 + paddd m4, m8 + movu m7, [t3+wq*2+400*0+24] + paddw m0, m3 + paddd m1, m4 + psllw m3, 2 + pslld m4, 2 + paddd m5, m7 + paddd m2, m5 + pslld m5, 2 + paddw m0, m3 ; a5 565 + paddd m1, m4 ; b5 565 + paddd m2, m5 + mova [t4+wq*1+400* 6+ 0], m0 + mova [t3+wq*2+400*12+ 0], m1 + mova [t3+wq*2+400*12+16], m2 + movu m0, [t4+wq*1+400*2+ 4] + movu m1, [t3+wq*2+400*4+ 8] + movu m2, [t3+wq*2+400*4+24] + movu m3, [t4+wq*1+400*2+ 2] + movu m4, [t3+wq*2+400*4+ 4] + movu m5, [t3+wq*2+400*4+20] + paddw m0, [t4+wq*1+400*2+ 0] + paddd m1, [t3+wq*2+400*4+ 0] + paddd m2, [t3+wq*2+400*4+16] + paddw m3, m0 + paddd m4, m1 + paddd m5, m2 + psllw m3, 2 ; a3[-1] 444 + pslld m4, 2 ; b3[-1] 444 + pslld m5, 2 + psubw m3, m0 ; a3[-1] 343 + psubd m4, m1 ; b3[-1] 343 + psubd m5, m2 + mova [t4+wq*1+400* 8+ 0], m3 + mova [t3+wq*2+400*16+ 0], m4 + mova [t3+wq*2+400*16+16], m5 + movu m0, [t4+wq*1+400*4+ 4] + movu m1, [t3+wq*2+400*8+ 8] + movu m2, [t3+wq*2+400*8+24] + movu m3, [t4+wq*1+400*4+ 2] + movu m4, [t3+wq*2+400*8+ 4] + movu m5, [t3+wq*2+400*8+20] + paddw m0, [t4+wq*1+400*4+ 0] + paddd m1, [t3+wq*2+400*8+ 0] + paddd m2, [t3+wq*2+400*8+16] + paddw m3, m0 + paddd m4, m1 + paddd m5, m2 + psllw m3, 2 ; a3[ 0] 444 + pslld m4, 2 ; b3[ 0] 444 + pslld m5, 2 + mova [t4+wq*1+400*10+ 0], m3 + mova [t3+wq*2+400*20+ 0], m4 + mova [t3+wq*2+400*20+16], m5 + psubw m3, m0 ; a3[ 0] 343 + psubd m4, m1 ; b3[ 0] 343 + psubd m5, m2 + mova [t4+wq*1+400*12+ 0], m3 + mova [t3+wq*2+400*24+ 0], m4 + mova [t3+wq*2+400*24+16], m5 + add wq, 16 + jl .prep_n_loop + ret +ALIGN function_align +.n0: ; neighbor + output (even rows) + movif64 wq, r5 + movif32 wd, w1m +.n0_loop: + movu m0, [t4+wq*1+ 4] + movu m2, [t4+wq*1+ 2] + paddw m0, [t4+wq*1+ 0] + paddw m0, m2 + paddw m2, m0 + psllw m0, 2 + paddw m0, m2 ; a5 + movu m4, [t3+wq*2+ 8] + movu m5, [t3+wq*2+24] + movu m1, [t3+wq*2+ 4] + movu m3, [t3+wq*2+20] + paddd m4, [t3+wq*2+ 0] + paddd m5, [t3+wq*2+16] + paddd m4, m1 + paddd m5, m3 + paddd m1, m4 + paddd m3, m5 + pslld m4, 2 + pslld m5, 2 + paddd m4, m1 ; b5 + paddd m5, m3 + movu m2, [t4+wq*1+400* 6] + paddw m2, m0 + mova [t4+wq*1+400* 6], m0 + paddd m0, m4, [t3+wq*2+400*12+ 0] + paddd m1, m5, [t3+wq*2+400*12+16] + mova [t3+wq*2+400*12+ 0], m4 + mova [t3+wq*2+400*12+16], m5 + mova [rsp+16+ARCH_X86_32*4], m1 + movu m3, [t4+wq*1+400*2+4] + movu m5, [t4+wq*1+400*2+2] + paddw m3, [t4+wq*1+400*2+0] + paddw m5, m3 + psllw m5, 2 ; a3[ 1] 444 + psubw m4, m5, m3 ; a3[ 1] 343 + movu m3, [t4+wq*1+400* 8] + paddw m3, [t4+wq*1+400*10] + paddw m3, m4 + mova [t4+wq*1+400* 8], m4 + mova [t4+wq*1+400*10], m5 + movu m1, [t3+wq*2+400*4+ 8] + movu m5, [t3+wq*2+400*4+ 4] + movu m7, [t3+wq*2+400*4+24] + movu m8, [t3+wq*2+400*4+20] + paddd m1, [t3+wq*2+400*4+ 0] + paddd m7, [t3+wq*2+400*4+16] + paddd m5, m1 + paddd m8, m7 + pslld m5, 2 ; b3[ 1] 444 + pslld m8, 2 + psubd m4, m5, m1 ; b3[ 1] 343 +%if ARCH_X86_32 + mova [esp+52], m8 + psubd m8, m7 +%else + psubd m6, m8, m7 + SWAP m8, m6 +%endif + paddd m1, m4, [t3+wq*2+400*16+ 0] + paddd m7, m8, [t3+wq*2+400*16+16] + paddd m1, [t3+wq*2+400*20+ 0] + paddd m7, [t3+wq*2+400*20+16] + mova [t3+wq*2+400*16+ 0], m4 + mova [t3+wq*2+400*16+16], m8 + mova [t3+wq*2+400*20+ 0], m5 +%if ARCH_X86_32 + mova m8, [esp+52] +%else + SWAP m8, m6 + pxor m6, m6 +%endif + mova [t3+wq*2+400*20+16], m8 + mova [rsp+32+ARCH_X86_32*4], m7 + movu m5, [dstq+wq] + punpcklwd m4, m5, m6 + punpcklwd m7, m2, m6 + pmaddwd m7, m4 ; a5 * src + punpcklwd m8, m3, m6 + pmaddwd m8, m4 ; a3 * src + punpckhwd m5, m6 + punpckhwd m2, m6 + pmaddwd m2, m5 + punpckhwd m3, m6 + pmaddwd m3, m5 + pslld m4, 13 + pslld m5, 13 + psubd m0, m7 ; b5 - a5 * src + (1 << 8) + psubd m1, m8 ; b3 - a3 * src + (1 << 8) + mova m7, [base+pd_0xffff] + psrld m0, 9 + pslld m1, 7 + pand m0, m7 + pandn m8, m7, m1 + por m0, m8 + mova m1, [rsp+16+ARCH_X86_32*4] + mova m8, [rsp+32+ARCH_X86_32*4] + psubd m1, m2 + psubd m8, m3 + mova m2, [base+pd_4096] + psrld m1, 9 + pslld m8, 7 + pand m1, m7 + pandn m7, m8 + por m1, m7 + pmaddwd m0, m15 + pmaddwd m1, m15 +%if ARCH_X86_32 + pxor m7, m7 +%else + SWAP m7, m6 +%endif + paddd m4, m2 + paddd m5, m2 + paddd m0, m4 + paddd m1, m5 + psrad m0, 8 + psrad m1, 8 + packssdw m0, m1 ; clip + pmaxsw m0, m7 + psrlw m0, 5 + mova [dstq+wq], m0 + add wq, 16 + jl .n0_loop + add dstq, dst_stridemp + ret +%if ARCH_X86_64 + SWAP m6, m7 +%endif +ALIGN function_align +.n1: ; neighbor + output (odd rows) + movif64 wq, r5 + movif32 wd, w1m +.n1_loop: + movu m3, [t4+wq*1+400*4+4] + movu m5, [t4+wq*1+400*4+2] + paddw m3, [t4+wq*1+400*4+0] + paddw m5, m3 + psllw m5, 2 ; a3[ 1] 444 + psubw m4, m5, m3 ; a3[ 1] 343 + paddw m3, m4, [t4+wq*1+400*12] + paddw m3, [t4+wq*1+400*10] + mova [t4+wq*1+400*10], m5 + mova [t4+wq*1+400*12], m4 + movu m1, [t3+wq*2+400*8+ 8] + movu m5, [t3+wq*2+400*8+ 4] + movu m7, [t3+wq*2+400*8+24] + movu m8, [t3+wq*2+400*8+20] + paddd m1, [t3+wq*2+400*8+ 0] + paddd m7, [t3+wq*2+400*8+16] + paddd m5, m1 + paddd m8, m7 + pslld m5, 2 ; b3[ 1] 444 + pslld m8, 2 + psubd m4, m5, m1 ; b3[ 1] 343 + psubd m0, m8, m7 + paddd m1, m4, [t3+wq*2+400*24+ 0] + paddd m7, m0, [t3+wq*2+400*24+16] + paddd m1, [t3+wq*2+400*20+ 0] + paddd m7, [t3+wq*2+400*20+16] + mova [t3+wq*2+400*20+ 0], m5 + mova [t3+wq*2+400*20+16], m8 + mova [t3+wq*2+400*24+ 0], m4 + mova [t3+wq*2+400*24+16], m0 + mova m5, [dstq+wq] + mova m2, [t4+wq*1+400* 6] + punpcklwd m4, m5, m6 + punpcklwd m8, m2, m6 + pmaddwd m8, m4 ; a5 * src + punpcklwd m0, m3, m6 + pmaddwd m0, m4 ; a3 * src + punpckhwd m5, m6 + punpckhwd m2, m6 + pmaddwd m2, m5 + punpckhwd m3, m6 + pmaddwd m3, m5 + psubd m1, m0 ; b3 - a3 * src + (1 << 8) + pslld m4, 13 + pslld m5, 13 + mova m0, [t3+wq*2+400*12+ 0] + psubd m0, m8 ; b5 - a5 * src + (1 << 8) + mova m8, [t3+wq*2+400*12+16] + psubd m8, m2 + psubd m7, m3 + mova m2, [base+pd_0xffff] + pslld m1, 7 + psrld m0, 8 + psrld m8, 8 + pslld m7, 7 + pand m0, m2 + pandn m3, m2, m1 + por m0, m3 + pand m8, m2 + pandn m2, m7 + por m2, m8 + mova m1, [base+pd_4096] + pmaddwd m0, m15 + pmaddwd m2, m15 +%if ARCH_X86_64 + SWAP m7, m6 +%endif + pxor m7, m7 + paddd m4, m1 + paddd m5, m1 + paddd m0, m4 + paddd m2, m5 + psrad m0, 8 + psrad m2, 8 + packssdw m0, m2 ; clip + pmaxsw m0, m7 + psrlw m0, 5 + mova [dstq+wq], m0 + add wq, 16 + jl .n1_loop + add dstq, dst_stridemp + movif32 dstm, dstq + ret diff --git a/third_party/dav1d/src/x86/looprestoration_avx2.asm b/third_party/dav1d/src/x86/looprestoration_avx2.asm --- a/third_party/dav1d/src/x86/looprestoration_avx2.asm +++ b/third_party/dav1d/src/x86/looprestoration_avx2.asm @@ -77,18 +77,10 @@ pd_0xf00801c7: dd 0xf00801c7 pd_0xf00800a4: dd 0xf00800a4 SECTION .text -%macro REPX 2-* - %xdefine %%f(x) %1 -%rep %0 - 1 - %rotate 1 - %%f(%1) -%endrep -%endmacro - DECLARE_REG_TMP 4, 9, 7, 11, 12, 13, 14 ; ring buffer pointers INIT_YMM avx2 cglobal wiener_filter7_8bpc, 5, 15, 16, -384*12-16, dst, dst_stride, left, lpf, \ lpf_stride, w, edge, flt, h @@ -109,10 +101,12 @@ lea t1, [rsp+wq*2+16] vpbroadcastd m14, [fltq+16] ; y0 y1 add dstq, wq vpbroadcastd m15, [fltq+20] ; y2 y3 neg wq + psllw m14, 5 + psllw m15, 5 test edgeb, 4 ; LR_HAVE_TOP jz .no_top call .h_top add lpfq, lpf_strideq mov t6, t1 @@ -355,13 +349,11 @@ mova m2, [t4+r10*2+32] paddw m2, [t2+r10*2+32] mova m3, [t3+r10*2+32] mova m5, [t5+r10*2+32] paddw m5, [t1+r10*2+32] - psrad m0, 11 - psrad m4, 11 - packssdw m0, m4 + packuswb m0, m4 paddw m4, m1, [t6+r10*2+32] mova [t0+r10*2+32], m1 punpcklwd m1, m2, m3 pmaddwd m1, m15 punpckhwd m2, m3 @@ -370,13 +362,13 @@ pmaddwd m3, m14 punpckhwd m4, m5 pmaddwd m4, m14 paddd m1, m3 paddd m2, m4 - psrad m1, 11 - psrad m2, 11 - packssdw m1, m2 + packuswb m1, m2 + psrlw m0, 8 + psrlw m1, 8 packuswb m0, m1 mova [dstq+r10], m0 add r10, 32 jl .hv_loop mov t6, t5 @@ -421,13 +413,14 @@ pmaddwd m7, m14 paddd m0, m4 paddd m2, m6 paddd m1, m5 paddd m3, m7 - REPX {psrad x, 11}, m0, m2, m1, m3 - packssdw m0, m2 - packssdw m1, m3 + packuswb m0, m2 + packuswb m1, m3 + psrlw m0, 8 + psrlw m1, 8 packuswb m0, m1 mova [dstq+r10], m0 add r10, 32 jl .v_loop mov t6, t5 @@ -457,10 +450,12 @@ mova m11, [wiener_l_shuf] vpbroadcastd m14, [fltq+16] ; __ y1 add dstq, wq vpbroadcastd m15, [fltq+20] ; y2 y3 neg wq + psllw m14, 5 + psllw m15, 5 test edgeb, 4 ; LR_HAVE_TOP jz .no_top call .h_top add lpfq, lpf_strideq mov t4, t1 @@ -659,13 +654,11 @@ paddd m0, m3 paddd m4, m2 mova m2, [t3+r10*2+32] paddw m2, [t1+r10*2+32] mova m3, [t2+r10*2+32] - psrad m0, 11 - psrad m4, 11 - packssdw m0, m4 + packuswb m0, m4 paddw m4, m1, [t4+r10*2+32] mova [t0+r10*2+32], m1 punpcklwd m1, m2, m3 pmaddwd m1, m15 punpckhwd m2, m3 @@ -674,13 +667,13 @@ pmaddwd m3, m14 punpckhwd m4, m4 pmaddwd m4, m14 paddd m1, m3 paddd m2, m4 - psrad m1, 11 - psrad m2, 11 - packssdw m1, m2 + packuswb m1, m2 + psrlw m0, 8 + psrlw m1, 8 packuswb m0, m1 mova [dstq+r10], m0 add r10, 32 jl .hv_loop mov t4, t3 @@ -718,13 +711,14 @@ pmaddwd m7, m13 paddd m0, m4 paddd m2, m6 paddd m1, m5 paddd m3, m7 - REPX {psrad x, 11}, m0, m2, m1, m3 - packssdw m0, m2 - packssdw m1, m3 + packuswb m0, m2 + packuswb m1, m3 + psrlw m0, 8 + psrlw m1, 8 packuswb m0, m1 mova [dstq+r10], m0 add r10, 32 jl .v_loop ret @@ -1001,11 +995,11 @@ pmaddwd m0, m13 ; b * 164 pmaddwd m1, m13 paddusw m4, m13 paddusw m5, m13 psrad m3, m4, 20 ; min(z, 255) - 256 - vpgatherdd m2, [r12+m3*4], m4 + vpgatherdd m2, [r12+m3*4], m4 ; x psrad m4, m5, 20 vpgatherdd m3, [r12+m4*4], m5 pmulld m0, m2 pmulld m1, m3 paddd m0, m14 ; x * b * 164 + (1 << 11) + (1 << 15) @@ -1061,11 +1055,11 @@ pmaddwd m0, m13 ; b * 164 pmaddwd m1, m13 paddusw m4, m13 paddusw m5, m13 psrad m3, m4, 20 ; min(z, 255) - 256 - vpgatherdd m2, [r12+m3*4], m4 + vpgatherdd m2, [r12+m3*4], m4 ; x psrad m4, m5, 20 vpgatherdd m3, [r12+m4*4], m5 pmulld m0, m2 pmulld m1, m3 paddd m0, m14 ; x * b * 164 + (1 << 11) + (1 << 15) @@ -1094,16 +1088,13 @@ pslld m2, 2 paddd m1, m3 pslld m3, 2 paddd m2, m0 ; ab 565 paddd m3, m1 - ; a = 4096 - (ab & 4095) = -(ab | ~4095), so by - ; using OR instead of AND for the masking we get - ; the subtraction for free (with a negated result) - por m0, m15, m2 ; -a - psrld m2, 12 ; b - por m1, m15, m3 + pandn m0, m15, m2 ; a + psrld m2, 12 ; b + pandn m1, m15, m3 psrld m3, 12 mova [t3+r10*4+400*4+ 0], m0 mova [t3+r10*4+400*8+ 0], m2 mova [t3+r10*4+400*4+32], m1 mova [t3+r10*4+400*8+32], m3 @@ -1124,34 +1115,32 @@ pslld m2, 2 paddd m1, m3 pslld m3, 2 paddd m2, m0 paddd m3, m1 - por m0, m15, m2 + pandn m0, m15, m2 psrld m2, 12 - por m1, m15, m3 + pandn m1, m15, m3 psrld m3, 12 - paddd m4, m0, [t3+r10*4+400*4+ 0] ; -a + paddd m4, m0, [t3+r10*4+400*4+ 0] ; a paddd m5, m1, [t3+r10*4+400*4+32] mova [t3+r10*4+400*4+ 0], m0 mova [t3+r10*4+400*4+32], m1 paddd m0, m2, [t3+r10*4+400*8+ 0] ; b paddd m1, m3, [t3+r10*4+400*8+32] mova [t3+r10*4+400*8+ 0], m2 mova [t3+r10*4+400*8+32], m3 pmovzxbd m2, [dstq+r10+0] pmovzxbd m3, [dstq+r10+8] - pmaddwd m4, m2 ; -a * src + pmaddwd m4, m2 ; a * src pmaddwd m5, m3 packssdw m2, m3 - psubd m0, m4 ; a * src + b + (1 << 8) + psubd m0, m4 ; b - a * src + (1 << 8) psubd m1, m5 - psrld m0, 9 - psrld m1, 9 + psrad m0, 9 + psrad m1, 9 packssdw m0, m1 - psllw m1, m2, 4 - psubw m0, m1 pmulhrsw m0, m7 paddw m0, m2 vextracti128 xm1, m0, 1 packuswb xm0, xm1 pshufd xm0, xm0, q3120 @@ -1164,22 +1153,20 @@ .n1: ; neighbor + output (odd rows) mov r10, wq .n1_loop: pmovzxbd m2, [dstq+r10+0] pmovzxbd m3, [dstq+r10+8] - pmaddwd m4, m2, [t3+r10*4+400*4+ 0] ; -a * src + pmaddwd m4, m2, [t3+r10*4+400*4+ 0] ; a * src pmaddwd m5, m3, [t3+r10*4+400*4+32] mova m0, [t3+r10*4+400*8+ 0] ; b mova m1, [t3+r10*4+400*8+32] packssdw m2, m3 - psubd m0, m4 ; a * src + b + (1 << 7) + psubd m0, m4 ; b - a * src + (1 << 7) psubd m1, m5 - psrld m0, 8 - psrld m1, 8 + psrad m0, 8 + psrad m1, 8 packssdw m0, m1 - psllw m1, m2, 4 - psubw m0, m1 pmulhrsw m0, m7 paddw m0, m2 vextracti128 xm1, m0, 1 packuswb xm0, xm1 pshufd xm0, xm0, q3120 @@ -1507,35 +1494,33 @@ paddd m5, m5 mova [t3+r10*4+400*4+32], m5 paddd m5, m5 psubd m5, m4 mova [t5+r10*4+32], m5 - por m4, m14, m0 + pandn m4, m14, m0 psrld m0, 12 paddd m3, m5 - por m5, m14, m2 + pandn m5, m14, m2 psrld m2, 12 - paddd m4, m5 ; -a - por m5, m14, m1 + paddd m4, m5 ; a + pandn m5, m14, m1 psrld m1, 12 - paddd m0, m2 ; b + (1 << 8) - por m2, m14, m3 + paddd m0, m2 ; b + (1 << 8) + pandn m2, m14, m3 psrld m3, 12 paddd m5, m2 pmovzxbd m2, [dstq+r10+0] paddd m1, m3 pmovzxbd m3, [dstq+r10+8] - pmaddwd m4, m2 ; -a * src + pmaddwd m4, m2 ; a * src pmaddwd m5, m3 packssdw m2, m3 - psubd m0, m4 ; a * src + b + (1 << 8) + psubd m0, m4 ; b - a * src + (1 << 8) psubd m1, m5 - psrld m0, 9 - psrld m1, 9 + psrad m0, 9 + psrad m1, 9 packssdw m0, m1 - psllw m1, m2, 4 - psubw m0, m1 pmulhrsw m0, m7 paddw m0, m2 vextracti128 xm1, m0, 1 packuswb xm0, xm1 pshufd xm0, xm0, q3120 @@ -1906,26 +1891,26 @@ paddusw m7, m2 psrad m3, m6, 20 ; min(z3, 255) - 256 vpgatherdd m2, [r12+m3*4], m6 psrad m6, m7, 20 vpgatherdd m3, [r12+m6*4], m7 - vpbroadcastd m6, [base+pd_34816] + vpbroadcastd m6, [base+pd_34816] ; x3 pmulld m0, m2 vpbroadcastd m7, [base+pd_m4096] pmulld m1, m3 paddd m0, m6 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) paddd m1, m6 pand m0, m7 pand m7, m1 por m0, m2 ; a3 | (b3 << 12) por m7, m3 - paddw m1, m8, [t2+r10*2+400*0] - paddd m2, m4, [t2+r10*2+400*2] - paddd m3, m5, [t2+r10*2+400*4] - paddw m1, [t1+r10*2+400*0] - paddd m2, [t1+r10*2+400*2] - paddd m3, [t1+r10*2+400*4] + paddw m1, m8, [t2+r10*2+400*0] + paddd m2, m4, [t2+r10*2+400*2] + paddd m3, m5, [t2+r10*2+400*4] + paddw m1, [t1+r10*2+400*0] + paddd m2, [t1+r10*2+400*2] + paddd m3, [t1+r10*2+400*4] mova [t2+r10*2+400*0], m8 mova [t2+r10*2+400*2], m4 mova [t2+r10*2+400*4], m5 mova [t3+r10*4+400*8+ 8], xm0 vextracti128 [t3+r10*4+400*8+40], m0, 1 @@ -1947,11 +1932,11 @@ pmaddwd m0, m4 ; b5 * 164 pmaddwd m1, m4 paddusw m2, m4 paddusw m3, m4 psrad m5, m2, 20 ; min(z5, 255) - 256 - vpgatherdd m4, [r12+m5*4], m2 + vpgatherdd m4, [r12+m5*4], m2 ; x5 psrad m2, m3, 20 vpgatherdd m5, [r12+m2*4], m3 pmulld m0, m4 pmulld m1, m5 paddd m0, m6 ; x5 * b5 * 164 + (1 << 11) + (1 << 15) @@ -2004,11 +1989,11 @@ pmaddwd m0, m2 ; b3 * 455 pmaddwd m1, m2 paddusw m4, m2 paddusw m5, m2 psrad m3, m4, 20 ; min(z3, 255) - 256 - vpgatherdd m2, [r12+m3*4], m4 + vpgatherdd m2, [r12+m3*4], m4 ; x3 psrad m4, m5, 20 vpgatherdd m3, [r12+m4*4], m5 pmulld m0, m2 pmulld m1, m3 paddd m0, m6 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) @@ -2021,11 +2006,11 @@ mova m3, [t1+r10*2+400*2] mova m4, [t1+r10*2+400*4] mova [t3+r10*4+400*8+ 8], m2 mova [t3+r10*4+400*0+ 8], m3 mova [t3+r10*4+400*0+40], m4 - paddw m2, m2 ; cc5 + paddw m2, m2 ; cc5 paddd m3, m3 paddd m4, m4 mova [t1+r10*2+400*0], m2 mova [t1+r10*2+400*2], m3 mova [t1+r10*2+400*4], m4 @@ -2064,11 +2049,11 @@ pmaddwd m0, m2 ; b3 * 455 pmaddwd m1, m2 paddusw m4, m2 paddusw m5, m2 psrad m3, m4, 20 ; min(z3, 255) - 256 - vpgatherdd m2, [r12+m3*4], m4 + vpgatherdd m2, [r12+m3*4], m4 ; x3 psrad m4, m5, 20 vpgatherdd m3, [r12+m4*4], m5 vpbroadcastd m4, [base+pd_34816] pmulld m0, m2 vpbroadcastd m8, [base+pd_m4096] @@ -2110,11 +2095,11 @@ pmaddwd m0, m4 ; b5 * 164 pmaddwd m1, m4 paddusw m2, m4 paddusw m3, m4 psrad m5, m2, 20 ; min(z5, 255) - 256 - vpgatherdd m4, [r12+m5*4], m2 + vpgatherdd m4, [r12+m5*4], m2 ; x5 psrad m2, m3, 20 vpgatherdd m5, [r12+m2*4], m3 pmulld m0, m4 vpbroadcastd m6, [base+pd_34816] pmulld m1, m5 @@ -2152,11 +2137,11 @@ pslld m2, 2 paddd m1, m0 ; ab5 565 paddd m3, m3 ; ab3[ 0] 222 psubd m2, m4 ; ab3[-1] 343 mova [t3+r10*4+400*20], m3 - por m0, m6, m1 ; a5 565 + pandn m0, m6, m1 ; a5 565 mova [t3+r10*4+400*24], m2 psrld m1, 12 ; b5 565 mova [t3+r10*4+400*12], m0 paddd m3, m3 mova [t3+r10*4+400*16], m1 @@ -2173,15 +2158,15 @@ paddd m4, m0, [t3+r10*4+0] paddd m4, [t3+r10*4+8] paddd m0, m4 pslld m4, 2 paddd m4, m0 - por m0, m6, m4 + pandn m0, m6, m4 psrld m4, 12 - paddd m2, m0, [t3+r10*4+400*12] ; -a5 + paddd m2, m0, [t3+r10*4+400*12] ; a5 mova [t3+r10*4+400*12], m0 - paddd m0, m4, [t3+r10*4+400*16] ; b5 + (1 << 8) + paddd m0, m4, [t3+r10*4+400*16] ; b5 + (1 << 8) mova [t3+r10*4+400*16], m4 mova m3, [t3+r10*4+400*4+0] paddd m3, [t3+r10*4+400*4+8] paddd m5, m3, [t3+r10*4+400*4+4] paddd m5, m5 ; ab3[ 1] 222 @@ -2190,31 +2175,28 @@ mova [t3+r10*4+400*20], m5 paddd m5, m5 psubd m5, m3 ; ab3[ 1] 343 mova [t3+r10*4+400*24], m5 paddd m4, m5 ; ab3[ 0] 222 + ab3[ 1] 343 - por m3, m6, m1 + pandn m3, m6, m1 psrld m1, 12 - por m5, m6, m4 + pandn m5, m6, m4 psrld m4, 12 - paddd m3, m5 ; -a3 - paddd m1, m4 ; b3 + (1 << 8) + paddd m3, m5 ; a3 + paddd m1, m4 ; b3 + (1 << 8) pmovzxbd m4, [dstq+r10] - pmaddwd m2, m4 ; -a5 * src - pmaddwd m3, m4 ; -a3 * src - pslld m4, 13 - psubd m0, m4 - psubd m1, m4 - psubd m0, m2 ; a5 * src + b5 + (1 << 8) - psubd m1, m3 ; a3 * src + b3 + (1 << 8) + pmaddwd m2, m4 ; a5 * src + pmaddwd m3, m4 ; a3 * src + psubd m0, m2 ; b5 - a5 * src + (1 << 8) + psubd m1, m3 ; b3 - a3 * src + (1 << 8) psrld m0, 9 pslld m1, 7 pblendw m0, m1, 0xaa pmaddwd m0, m15 - psubd m4, m6 + psubd m0, m6 + psrad m0, 13 paddd m0, m4 - psrad m0, 13 vextracti128 xm1, m0, 1 packssdw xm0, xm1 packuswb xm0, xm0 movq [dstq+r10], xm0 add r10, 8 @@ -2234,33 +2216,29 @@ mova [t3+r10*4+400*20], m5 paddd m5, m5 psubd m5, m3 ; ab3[ 1] 343 mova [t3+r10*4+400*28], m5 paddd m4, m5 ; ab3[ 0] 222 + ab3[ 1] 343 - por m3, m6, m1 + pandn m3, m6, m1 psrld m1, 12 - por m5, m6, m4 + pandn m5, m6, m4 psrld m4, 12 paddd m3, m5 ; -a3 paddd m1, m4 ; b3 + (1 << 8) pmovzxbd m4, [dstq+r10] pmaddwd m2, m4, [t3+r10*4+400*12] ; -a5 * src mova m0, [t3+r10*4+400*16] ; b5 + (1 << 7) pmaddwd m3, m4 ; -a3 * src - pslld m4, 12 - psubd m0, m4 - paddd m4, m4 - psubd m1, m4 psubd m0, m2 ; a5 * src + b5 + (1 << 7) psubd m1, m3 ; a3 * src + b3 + (1 << 8) psrld m0, 8 pslld m1, 7 pblendw m0, m1, 0xaa pmaddwd m0, m15 - psubd m4, m6 + psubd m0, m6 + psrad m0, 13 paddd m0, m4 - psrad m0, 13 vextracti128 xm1, m0, 1 packssdw xm0, xm1 packuswb xm0, xm0 movq [dstq+r10], xm0 add r10, 8 diff --git a/third_party/dav1d/src/x86/looprestoration_init_tmpl.c b/third_party/dav1d/src/x86/looprestoration_init_tmpl.c --- a/third_party/dav1d/src/x86/looprestoration_init_tmpl.c +++ b/third_party/dav1d/src/x86/looprestoration_init_tmpl.c @@ -37,156 +37,16 @@ #define decl_sgr_filter_fns(ext) \ decl_lr_filter_fn(BF(dav1d_sgr_filter_5x5, ext)); \ decl_lr_filter_fn(BF(dav1d_sgr_filter_3x3, ext)); \ decl_lr_filter_fn(BF(dav1d_sgr_filter_mix, ext)) -/* FIXME: Replace with a port of the AVX2 code */ -#define SGR_FILTER_OLD(ext) \ -void BF(dav1d_sgr_box3_h, ext)(int32_t *sumsq, int16_t *sum, \ - const pixel (*left)[4], \ - const pixel *src, const ptrdiff_t stride, \ - const int w, const int h, \ - const enum LrEdgeFlags edges); \ -void BF(dav1d_sgr_box3_v, ext)(int32_t *sumsq, int16_t *sum, \ - const int w, const int h, \ - const enum LrEdgeFlags edges); \ -void BF(dav1d_sgr_calc_ab1, ext)(int32_t *a, int16_t *b, \ - const int w, const int h, const unsigned s); \ -void BF(dav1d_sgr_finish_filter1, ext)(int16_t *tmp, \ - const pixel *src, const ptrdiff_t stride, \ - const int32_t *a, const int16_t *b, \ - const int w, const int h); \ -\ -/* filter with a 3x3 box (radius=1) */ \ -static void BF(dav1d_sgr_filter1, ext)(int16_t *tmp, \ - const pixel *src, const ptrdiff_t stride, \ - const pixel (*left)[4], \ - const pixel *lpf, const ptrdiff_t lpf_stride, \ - const int w, const int h, const int strength, \ - const enum LrEdgeFlags edges) \ -{ \ - ALIGN_STK_32(int32_t, sumsq_mem, (384 + 16) * 68 + 8,); \ - int32_t *const sumsq = &sumsq_mem[(384 + 16) * 2 + 8], *const a = sumsq; \ - ALIGN_STK_32(int16_t, sum_mem, (384 + 16) * 68 + 16,); \ - int16_t *const sum = &sum_mem[(384 + 16) * 2 + 16], *const b = sum; \ -\ - BF(dav1d_sgr_box3_h, ext)(sumsq, sum, left, src, stride, w, h, edges); \ - if (edges & LR_HAVE_TOP) \ - BF(dav1d_sgr_box3_h, ext)(&sumsq[-2 * (384 + 16)], &sum[-2 * (384 + 16)], \ - NULL, lpf, lpf_stride, w, 2, edges); \ -\ - if (edges & LR_HAVE_BOTTOM) \ - BF(dav1d_sgr_box3_h, ext)(&sumsq[h * (384 + 16)], &sum[h * (384 + 16)], \ - NULL, lpf + 6 * PXSTRIDE(lpf_stride), \ - lpf_stride, w, 2, edges); \ -\ - BF(dav1d_sgr_box3_v, ext)(sumsq, sum, w, h, edges); \ - BF(dav1d_sgr_calc_ab1, ext)(a, b, w, h, strength); \ - BF(dav1d_sgr_finish_filter1, ext)(tmp, src, stride, a, b, w, h); \ -} \ -\ -void BF(dav1d_sgr_box5_h, ext)(int32_t *sumsq, int16_t *sum, \ - const pixel (*left)[4], \ - const pixel *src, const ptrdiff_t stride, \ - const int w, const int h, \ - const enum LrEdgeFlags edges); \ -void BF(dav1d_sgr_box5_v, ext)(int32_t *sumsq, int16_t *sum, \ - const int w, const int h, \ - const enum LrEdgeFlags edges); \ -void BF(dav1d_sgr_calc_ab2, ext)(int32_t *a, int16_t *b, \ - const int w, const int h, const int strength); \ -void BF(dav1d_sgr_finish_filter2, ext)(int16_t *tmp, \ - const pixel *src, const ptrdiff_t stride, \ - const int32_t *a, const int16_t *b, \ - const int w, const int h); \ -\ -/* filter with a 5x5 box (radius=2) */ \ -static void BF(dav1d_sgr_filter2, ext)(int16_t *tmp, \ - const pixel *src, const ptrdiff_t stride, \ - const pixel (*left)[4], \ - const pixel *lpf, const ptrdiff_t lpf_stride, \ - const int w, const int h, const int strength, \ - const enum LrEdgeFlags edges) \ -{ \ - ALIGN_STK_32(int32_t, sumsq_mem, (384 + 16) * 68 + 8,); \ - int32_t *const sumsq = &sumsq_mem[(384 + 16) * 2 + 8], *const a = sumsq; \ - ALIGN_STK_32(int16_t, sum_mem, (384 + 16) * 68 + 16,); \ - int16_t *const sum = &sum_mem[(384 + 16) * 2 + 16], *const b = sum; \ -\ - BF(dav1d_sgr_box5_h, ext)(sumsq, sum, left, src, stride, w, h, edges); \ - if (edges & LR_HAVE_TOP) \ - BF(dav1d_sgr_box5_h, ext)(&sumsq[-2 * (384 + 16)], &sum[-2 * (384 + 16)], \ - NULL, lpf, lpf_stride, w, 2, edges); \ -\ - if (edges & LR_HAVE_BOTTOM) \ - BF(dav1d_sgr_box5_h, ext)(&sumsq[h * (384 + 16)], &sum[h * (384 + 16)], \ - NULL, lpf + 6 * PXSTRIDE(lpf_stride), \ - lpf_stride, w, 2, edges); \ -\ - BF(dav1d_sgr_box5_v, ext)(sumsq, sum, w, h, edges); \ - BF(dav1d_sgr_calc_ab2, ext)(a, b, w, h, strength); \ - BF(dav1d_sgr_finish_filter2, ext)(tmp, src, stride, a, b, w, h); \ -} \ -\ -void BF(dav1d_sgr_weighted1, ext)(pixel *dst, const ptrdiff_t stride, \ - const int16_t *t1, const int w, const int h, \ - const int wt); \ -void BF(dav1d_sgr_weighted2, ext)(pixel *dst, const ptrdiff_t stride, \ - const int16_t *t1, const int16_t *t2, \ - const int w, const int h, \ - const uint32_t wt); \ -\ -static void BF(sgr_filter_5x5, ext)(pixel *const dst, const ptrdiff_t dst_stride, \ - const pixel (*const left)[4], \ - const pixel *lpf, const ptrdiff_t lpf_stride, \ - const int w, const int h, \ - const LooprestorationParams *const params, \ - const enum LrEdgeFlags edges HIGHBD_DECL_SUFFIX) \ -{ \ - ALIGN_STK_32(int16_t, tmp, 64 * 384,); \ - BF(dav1d_sgr_filter2, ext)(tmp, dst, dst_stride, left, lpf, lpf_stride, \ - w, h, params->sgr.s0, edges); \ - BF(dav1d_sgr_weighted1, ext)(dst, dst_stride, tmp, w, h, params->sgr.w0); \ -} \ -static void BF(sgr_filter_3x3, ext)(pixel *const dst, const ptrdiff_t dst_stride, \ - const pixel (*const left)[4], \ - const pixel *lpf, const ptrdiff_t lpf_stride, \ - const int w, const int h, \ - const LooprestorationParams *const params, \ - const enum LrEdgeFlags edges HIGHBD_DECL_SUFFIX) \ -{ \ - ALIGN_STK_32(int16_t, tmp, 64 * 384,); \ - BF(dav1d_sgr_filter1, ext)(tmp, dst, dst_stride, left, lpf, lpf_stride, \ - w, h, params->sgr.s1, edges); \ - BF(dav1d_sgr_weighted1, ext)(dst, dst_stride, tmp, w, h, params->sgr.w1); \ -} \ -static void BF(sgr_filter_mix, ext)(pixel *const dst, const ptrdiff_t dst_stride, \ - const pixel (*const left)[4], \ - const pixel *lpf, const ptrdiff_t lpf_stride, \ - const int w, const int h, \ - const LooprestorationParams *const params, \ - const enum LrEdgeFlags edges HIGHBD_DECL_SUFFIX) \ -{ \ - ALIGN_STK_32(int16_t, tmp1, 64 * 384,); \ - ALIGN_STK_32(int16_t, tmp2, 64 * 384,); \ - BF(dav1d_sgr_filter2, ext)(tmp1, dst, dst_stride, left, lpf, lpf_stride, \ - w, h, params->sgr.s0, edges); \ - BF(dav1d_sgr_filter1, ext)(tmp2, dst, dst_stride, left, lpf, lpf_stride, \ - w, h, params->sgr.s1, edges); \ - const uint32_t wt = (params->sgr.w1 << 16) | (uint16_t) params->sgr.w0; \ - BF(dav1d_sgr_weighted2, ext)(dst, dst_stride, tmp1, tmp2, w, h, wt); \ -} - decl_wiener_filter_fns(sse2); decl_wiener_filter_fns(ssse3); decl_wiener_filter_fns(avx2); +decl_sgr_filter_fns(ssse3); decl_sgr_filter_fns(avx2); -#if BITDEPTH == 8 -SGR_FILTER_OLD(ssse3) -#endif - COLD void bitfn(dav1d_loop_restoration_dsp_init_x86)(Dav1dLoopRestorationDSPContext *const c, const int bpc) { const unsigned flags = dav1d_get_cpu_flags(); @@ -197,15 +57,15 @@ #endif if (!(flags & DAV1D_X86_CPU_FLAG_SSSE3)) return; c->wiener[0] = BF(dav1d_wiener_filter7, ssse3); c->wiener[1] = BF(dav1d_wiener_filter5, ssse3); -#if BITDEPTH == 8 - c->sgr[0] = BF(sgr_filter_5x5, ssse3); - c->sgr[1] = BF(sgr_filter_3x3, ssse3); - c->sgr[2] = BF(sgr_filter_mix, ssse3); -#endif + if (bpc <= 10) { + c->sgr[0] = BF(dav1d_sgr_filter_5x5, ssse3); + c->sgr[1] = BF(dav1d_sgr_filter_3x3, ssse3); + c->sgr[2] = BF(dav1d_sgr_filter_mix, ssse3); + } #if ARCH_X86_64 if (!(flags & DAV1D_X86_CPU_FLAG_AVX2)) return; c->wiener[0] = BF(dav1d_wiener_filter7, avx2); diff --git a/third_party/dav1d/src/x86/looprestoration_sse.asm b/third_party/dav1d/src/x86/looprestoration_sse.asm --- a/third_party/dav1d/src/x86/looprestoration_sse.asm +++ b/third_party/dav1d/src/x86/looprestoration_sse.asm @@ -33,39 +33,43 @@ wiener_shufA: db 1, 7, 2, 8, 3, 9, 4, 10, 5, 11, 6, 12, 7, 13, 8, 14 wiener_shufB: db 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10 wiener_shufC: db 6, 5, 7, 6, 8, 7, 9, 8, 10, 9, 11, 10, 12, 11, 13, 12 wiener_shufD: db 4, -1, 5, -1, 6, -1, 7, -1, 8, -1, 9, -1, 10, -1, 11, -1 wiener_l_shuf: db 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 -pb_unpcklwdw: db 0, 1, 0, 1, 4, 5, 4, 5, 8, 9, 8, 9, 12, 13, 12, 13 +sgr_lshuf3: db 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 +sgr_lshuf5: db 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 +pb_0to15: db 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 pb_right_ext_mask: times 24 db 0xff times 8 db 0 -pb_0: times 16 db 0 +pb_1: times 16 db 1 pb_3: times 16 db 3 -pb_15: times 16 db 15 -pb_0_1: times 8 db 0, 1 -pb_14_15: times 8 db 14, 15 -pw_1: times 8 dw 1 -pw_16: times 8 dw 16 -pw_128: times 8 dw 128 pw_256: times 8 dw 256 -pw_2048: times 8 dw 2048 pw_2056: times 8 dw 2056 pw_m16380: times 8 dw -16380 -pw_5_6: times 4 dw 5, 6 -pd_1024: times 4 dd 1024 -%if ARCH_X86_32 -pd_512: times 4 dd 512 -pd_2048: times 4 dd 2048 -%endif -pd_0xF0080029: times 4 dd 0xF0080029 -pd_0xF00801C7: times 4 dd 0XF00801C7 +pd_4096: times 4 dd 4096 +pd_34816: times 4 dd 34816 +pd_0xffff: times 4 dd 0xffff +pd_0xf00800a4: times 4 dd 0xf00800a4 +pd_0xf00801c7: times 4 dd 0xf00801c7 cextern sgr_x_by_x SECTION .text +%macro movif64 2 ; dst, src + %if ARCH_X86_64 + mov %1, %2 + %endif +%endmacro + +%macro movif32 2 ; dst, src + %if ARCH_X86_32 + mov %1, %2 + %endif +%endmacro + %if ARCH_X86_32 %define PIC_base_offset $$ %macro SETUP_PIC 1-3 1,0 ; PIC_reg, save_PIC_reg, restore_PIC_reg %assign pic_reg_stk_off 4 @@ -197,10 +201,11 @@ %endif mova m12, m1 mova m13, m2 mova m14, m3 %endif + psllw m7, 5 pshufd m6, m7, q0000 ; y0 y1 pshufd m7, m7, q1111 ; y2 y3 test edgeb, 4 ; LR_HAVE_TOP jz .no_top call .h_top @@ -541,13 +546,11 @@ mov r2, t5 mova m5, [r2+xq*2+16] mov r2, t6 %endif paddw m5, [t1+xq*2+16] - psrad m0, 11 - psrad m4, 11 - packssdw m0, m4 + packuswb m0, m4 %if ARCH_X86_64 paddw m4, m1, [t6+xq*2+16] %else paddw m4, m1, [r2+xq*2+16] mov dstq, dstmp @@ -561,13 +564,13 @@ pmaddwd m3, m6 punpckhwd m4, m5 pmaddwd m4, m6 paddd m1, m3 paddd m2, m4 - psrad m1, 11 - psrad m2, 11 - packssdw m1, m2 + packuswb m1, m2 + psrlw m0, 8 + psrlw m1, 8 packuswb m0, m1 mova [dstq+xq], m0 add xq, 16 jl .hv_loop add dstq, dst_strideq @@ -645,26 +648,24 @@ paddw m4, m5, [r2+xq*2+16] mov r2, t5 paddw m5, [r2+xq*2+16] movifnidn dstq, dstmp %endif - psrad m0, 11 - psrad m1, 11 - packssdw m0, m1 + packuswb m0, m1 punpcklwd m1, m2, m3 pmaddwd m1, m7 punpckhwd m2, m3 pmaddwd m2, m7 punpcklwd m3, m4, m5 pmaddwd m3, m6 punpckhwd m4, m5 pmaddwd m4, m6 paddd m1, m3 paddd m2, m4 - psrad m1, 11 - psrad m2, 11 - packssdw m1, m2 + packuswb m1, m2 + psrlw m0, 8 + psrlw m1, 8 packuswb m0, m1 mova [dstq+xq], m0 add xq, 16 jl .v_loop add dstq, dst_strideq @@ -760,10 +761,11 @@ mova m11, m0 %endif mova m13, m1 mova m14, m2 %endif + psllw m7, 5 pshufd m6, m7, q0000 ; __ y1 pshufd m7, m7, q1111 ; y2 y3 test edgeb, 4 ; LR_HAVE_TOP jz .no_top call .h_top @@ -1020,13 +1022,11 @@ pmaddwd m4, m6 paddd m0, m3 paddd m4, m2 mova m2, [t3+xq*2+16] paddw m2, [t1+xq*2+16] - psrad m0, 11 - psrad m4, 11 - packssdw m0, m4 + packuswb m0, m4 %if ARCH_X86_64 mova m3, [t2+xq*2+16] paddw m4, m1, [t4+xq*2+16] %else paddw m4, m1, [r2+xq*2+16] @@ -1043,13 +1043,13 @@ pmaddwd m3, m6 punpckhwd m4, m4 pmaddwd m4, m6 paddd m1, m3 paddd m2, m4 - psrad m1, 11 - psrad m2, 11 - packssdw m1, m2 + packuswb m1, m2 + psrlw m0, 8 + psrlw m1, 8 packuswb m0, m1 mova [dstq+xq], m0 add xq, 16 jl .hv_loop add dstq, dst_strideq @@ -1094,26 +1094,24 @@ paddw m4, [r2+xq*2+16] mov r2, t2 mova m3, [r2+xq*2+16] mov dstq, dstmp %endif - psrad m0, 11 - psrad m1, 11 - packssdw m0, m1 + packuswb m0, m1 punpcklwd m1, m2, m3 pmaddwd m1, m7 punpckhwd m2, m3 pmaddwd m2, m7 punpcklwd m3, m4 pmaddwd m3, m6 punpckhwd m4, m4 pmaddwd m4, m6 paddd m1, m3 paddd m2, m4 - psrad m1, 11 - psrad m2, 11 - packssdw m1, m2 + packuswb m1, m2 + psrlw m0, 8 + psrlw m1, 8 packuswb m0, m1 mova [dstq+xq], m0 add xq, 16 jl .v_loop ret @@ -1128,1321 +1126,2575 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;; ;; self-guided ;; ;;;;;;;;;;;;;;;;;;;;;;;;;; -%macro MULLD 2 - pmulhuw m5, %1, %2 - pmullw %1, %2 - pslld m5, 16 - paddd %1, m5 +%macro GATHERDD 3 ; dst, src, tmp + movd %3d, %2 + %if ARCH_X86_64 + movd %1, [r13+%3] + pextrw %3d, %2, 2 + pinsrw %1, [r13+%3+2], 3 + pextrw %3d, %2, 4 + pinsrw %1, [r13+%3+2], 5 + pextrw %3d, %2, 6 + pinsrw %1, [r13+%3+2], 7 + %else + movd %1, [base+sgr_x_by_x-0xf03+%3] + pextrw %3, %2, 2 + pinsrw %1, [base+sgr_x_by_x-0xf03+%3+2], 3 + pextrw %3, %2, 4 + pinsrw %1, [base+sgr_x_by_x-0xf03+%3+2], 5 + pextrw %3, %2, 6 + pinsrw %1, [base+sgr_x_by_x-0xf03+%3+2], 7 + %endif %endmacro -%macro GATHERDD 2 - mova m5, m7 - movd r6d, %2 +%macro GATHER_X_BY_X 5 ; dst, src0, src1, tmp32, tmp32_restore %if ARCH_X86_64 - movd %1, [r5+r6] - pextrw r6d, %2, 2 - pinsrw m5, [r5+r6+2], 3 - pextrw r6d, %2, 4 - pinsrw %1, [r5+r6+2], 5 - pextrw r6d, %2, 6 - pinsrw m5, [r5+r6+2], 7 + %define tmp r14 %else - movd %1, [PIC_sym(sgr_x_by_x-0xF03)+r6] - pextrw r6d, %2, 2 - pinsrw m5, [PIC_sym(sgr_x_by_x-0xF03)+r6+2], 3 - pextrw r6d, %2, 4 - pinsrw %1, [PIC_sym(sgr_x_by_x-0xF03)+r6+2], 5 - pextrw r6d, %2, 6 - pinsrw m5, [PIC_sym(sgr_x_by_x-0xF03)+r6+2], 7 + %define tmp %4 %endif - por %1, m5 + GATHERDD %1, %2, tmp + GATHERDD %2, %3, tmp + movif32 %4, %5 + psrld %1, 24 + psrld %2, 24 + packssdw %1, %2 +%endmacro + +%macro MULLD 3 ; dst, src, tmp + pmulhuw %3, %1, %2 + pmullw %1, %2 + pslld %3, 16 + paddd %1, %3 %endmacro -%if ARCH_X86_64 -cglobal sgr_box3_h_8bpc, 5, 11, 8, sumsq, sum, left, src, stride, x, h, edge, w, xlim - mov xlimd, edgem - movifnidn xd, xm - mov hd, hm - mov edged, xlimd - and xlimd, 2 ; have_right - add xd, xlimd - xor xlimd, 2 ; 2*!have_right -%else -cglobal sgr_box3_h_8bpc, 6, 7, 8, sumsq, sum, left, src, stride, x, h, edge, w, xlim - %define wq r0m - %define xlimd r1m - %define hd hmp - %define edgeb byte edgem - - mov r6, edgem - and r6, 2 ; have_right - add xd, r6 - xor r6, 2 ; 2*!have_right - mov xlimd, r6 - SETUP_PIC r6, 0 -%endif - - jnz .no_right - add xd, 7 - and xd, ~7 -.no_right: - pxor m1, m1 - lea srcq, [srcq+xq] - lea sumq, [sumq+xq*2-2] - lea sumsqq, [sumsqq+xq*4-4] - neg xq - mov wq, xq -%if ARCH_X86_64 - lea r10, [pb_right_ext_mask+24] -%endif -.loop_y: - mov xq, wq - - ; load left - test edgeb, 1 ; have_left - jz .no_left - test leftq, leftq - jz .load_left_from_main - movd m0, [leftq] - pslldq m0, 12 - add leftq, 4 - jmp .expand_x -.no_left: - movd m0, [srcq+xq] - pshufb m0, [PIC_sym(pb_0)] - jmp .expand_x -.load_left_from_main: - movd m0, [srcq+xq-2] - pslldq m0, 14 -.expand_x: - punpckhbw xm0, xm1 - - ; when we reach this, m0 contains left two px in highest words - cmp xd, -8 - jle .loop_x -.partial_load_and_extend: - movd m3, [srcq-4] - pshufb m3, [PIC_sym(pb_3)] - movq m2, [srcq+xq] - punpcklbw m2, m1 - punpcklbw m3, m1 -%if ARCH_X86_64 - movu m4, [r10+xq*2] -%else - movu m4, [PIC_sym(pb_right_ext_mask)+xd*2+24] -%endif - pand m2, m4 - pandn m4, m3 - por m2, m4 - jmp .loop_x_noload -.right_extend: - pshufb m2, m0, [PIC_sym(pb_14_15)] - jmp .loop_x_noload - -.loop_x: - movq m2, [srcq+xq] - punpcklbw m2, m1 -.loop_x_noload: - palignr m3, m2, m0, 12 - palignr m4, m2, m0, 14 - - punpcklwd m5, m3, m2 - punpckhwd m6, m3, m2 - paddw m3, m4 - punpcklwd m7, m4, m1 - punpckhwd m4, m1 - pmaddwd m5, m5 - pmaddwd m6, m6 - pmaddwd m7, m7 - pmaddwd m4, m4 - paddd m5, m7 - paddd m6, m4 - paddw m3, m2 - movu [sumq+xq*2], m3 - movu [sumsqq+xq*4+ 0], m5 - movu [sumsqq+xq*4+16], m6 - - mova m0, m2 - add xq, 8 - - ; if x <= -8 we can reload more pixels - ; else if x < 0 we reload and extend (this implies have_right=0) - ; else if x < xlimd we extend from previous load (this implies have_right=0) - ; else we are done - - cmp xd, -8 - jle .loop_x - test xd, xd - jl .partial_load_and_extend - cmp xd, xlimd - jl .right_extend - - add sumsqq, (384+16)*4 - add sumq, (384+16)*2 - add srcq, strideq - dec hd - jg .loop_y - RET - -%if ARCH_X86_64 -cglobal sgr_box3_v_8bpc, 4, 10, 9, sumsq, sum, w, h, edge, x, y, sumsq_base, sum_base, ylim - movifnidn edged, edgem -%else -cglobal sgr_box3_v_8bpc, 3, 7, 8, -28, sumsq, sum, w, edge, h, x, y - %define sumsq_baseq dword [esp+0] - %define sum_baseq dword [esp+4] - %define ylimd dword [esp+8] - %define m8 [esp+12] - mov edged, r4m - mov hd, r3m -%endif - mov xq, -2 -%if ARCH_X86_64 - mov ylimd, edged - and ylimd, 8 ; have_bottom - shr ylimd, 2 - sub ylimd, 2 ; -2 if have_bottom=0, else 0 - mov sumsq_baseq, sumsqq - mov sum_baseq, sumq -.loop_x: - mov sumsqq, sumsq_baseq - mov sumq, sum_baseq - lea yd, [hq+ylimq+2] -%else - mov yd, edged - and yd, 8 ; have_bottom - shr yd, 2 - sub yd, 2 ; -2 if have_bottom=0, else 0 - mov sumsq_baseq, sumsqq - mov sum_baseq, sumq - mov ylimd, yd -.loop_x: - mov sumsqd, sumsq_baseq - mov sumd, sum_baseq - lea yd, [hq+2] - add yd, ylimd -%endif - lea sumsqq, [sumsqq+xq*4+4-(384+16)*4] - lea sumq, [sumq+xq*2+2-(384+16)*2] - test edgeb, 4 ; have_top - jnz .load_top - movu m0, [sumsqq+(384+16)*4*1] - movu m1, [sumsqq+(384+16)*4*1+16] - mova m2, m0 - mova m3, m1 - mova m4, m0 - mova m5, m1 - movu m6, [sumq+(384+16)*2*1] - mova m7, m6 - mova m8, m6 - jmp .loop_y_noload -.load_top: - movu m0, [sumsqq-(384+16)*4*1] ; l2sq [left] - movu m1, [sumsqq-(384+16)*4*1+16] ; l2sq [right] - movu m2, [sumsqq-(384+16)*4*0] ; l1sq [left] - movu m3, [sumsqq-(384+16)*4*0+16] ; l1sq [right] - movu m6, [sumq-(384+16)*2*1] ; l2 - movu m7, [sumq-(384+16)*2*0] ; l1 -.loop_y: -%if ARCH_X86_64 - movu m8, [sumq+(384+16)*2*1] ; l0 -%else - movu m4, [sumq+(384+16)*2*1] ; l0 - mova m8, m4 -%endif - movu m4, [sumsqq+(384+16)*4*1] ; l0sq [left] - movu m5, [sumsqq+(384+16)*4*1+16] ; l0sq [right] -.loop_y_noload: - paddd m0, m2 - paddd m1, m3 - paddw m6, m7 - paddd m0, m4 - paddd m1, m5 - paddw m6, m8 - movu [sumsqq+ 0], m0 - movu [sumsqq+16], m1 - movu [sumq], m6 - - ; shift position down by one - mova m0, m2 - mova m1, m3 - mova m2, m4 - mova m3, m5 - mova m6, m7 - mova m7, m8 - add sumsqq, (384+16)*4 - add sumq, (384+16)*2 - dec yd - jg .loop_y - cmp yd, ylimd - jg .loop_y_noload - add xd, 8 - cmp xd, wd - jl .loop_x - RET - -cglobal sgr_calc_ab1_8bpc, 4, 7, 12, a, b, w, h, s - movifnidn sd, sm - sub aq, (384+16-1)*4 - sub bq, (384+16-1)*2 - add hd, 2 -%if ARCH_X86_64 - LEA r5, sgr_x_by_x-0xF03 -%else - SETUP_PIC r5, 0 -%endif - movd m6, sd - pshuflw m6, m6, q0000 - punpcklqdq m6, m6 - pxor m7, m7 - DEFINE_ARGS a, b, w, h, x -%if ARCH_X86_64 - mova m8, [pd_0xF00801C7] - mova m9, [pw_256] - psrld m10, m9, 13 ; pd_2048 - mova m11, [pb_unpcklwdw] -%else - %define m8 [PIC_sym(pd_0xF00801C7)] - %define m9 [PIC_sym(pw_256)] - %define m10 [PIC_sym(pd_2048)] - %define m11 [PIC_sym(pb_unpcklwdw)] -%endif -.loop_y: - mov xq, -2 -.loop_x: - movq m0, [bq+xq*2] - movq m1, [bq+xq*2+(384+16)*2] - punpcklwd m0, m7 - punpcklwd m1, m7 - movu m2, [aq+xq*4] - movu m3, [aq+xq*4+(384+16)*4] - pslld m4, m2, 3 - pslld m5, m3, 3 - paddd m2, m4 ; aa * 9 - paddd m3, m5 - pmaddwd m4, m0, m0 - pmaddwd m5, m1, m1 - pmaddwd m0, m8 - pmaddwd m1, m8 - psubd m2, m4 ; p = aa * 9 - bb * bb - psubd m3, m5 - MULLD m2, m6 - MULLD m3, m6 - paddusw m2, m8 - paddusw m3, m8 - psrld m2, 20 ; z - psrld m3, 20 - GATHERDD m4, m2 ; xx - GATHERDD m2, m3 - psrld m4, 24 - psrld m2, 24 - packssdw m3, m4, m2 - pshufb m4, m11 - MULLD m0, m4 - pshufb m2, m11 - MULLD m1, m2 - psubw m5, m9, m3 - paddd m0, m10 - paddd m1, m10 - psrld m0, 12 - psrld m1, 12 - movq [bq+xq*2], m5 - psrldq m5, 8 - movq [bq+xq*2+(384+16)*2], m5 - movu [aq+xq*4], m0 - movu [aq+xq*4+(384+16)*4], m1 - add xd, 4 - cmp xd, wd - jl .loop_x - add aq, (384+16)*4*2 - add bq, (384+16)*2*2 - sub hd, 2 - jg .loop_y - RET - -%if ARCH_X86_64 -cglobal sgr_finish_filter1_8bpc, 5, 13, 16, t, src, stride, a, b, w, h, \ - tmp_base, src_base, a_base, b_base, x, y - movifnidn wd, wm - mov hd, hm - mova m15, [pw_16] - mov tmp_baseq, tq - mov src_baseq, srcq - mov a_baseq, aq - mov b_baseq, bq - xor xd, xd -%else -cglobal sgr_finish_filter1_8bpc, 7, 7, 8, -144, t, src, stride, a, b, x, y - %define tmp_baseq [esp+8] - %define src_baseq [esp+12] - %define a_baseq [esp+16] - %define b_baseq [esp+20] - %define wd [esp+24] - %define hd [esp+28] - mov tmp_baseq, tq - mov src_baseq, srcq - mov a_baseq, aq - mov b_baseq, bq - mov wd, xd - mov hd, yd - xor xd, xd - SETUP_PIC yd, 1, 1 - jmp .loop_start -%endif - -.loop_x: - mov tq, tmp_baseq - mov srcq, src_baseq - mov aq, a_baseq - mov bq, b_baseq %if ARCH_X86_32 -.loop_start: - movu m0, [bq+xq*2-(384+16)*2-2] - movu m2, [bq+xq*2-(384+16)*2+2] - mova m1, [bq+xq*2-(384+16)*2] ; b:top - paddw m0, m2 ; b:tl+tr - movu m2, [bq+xq*2-2] - movu m3, [bq+xq*2+2] - paddw m1, [bq+xq*2] ; b:top+ctr - paddw m2, m3 ; b:l+r - mova [esp+0x80], m0 - mova [esp+0x70], m1 - mova [esp+0x60], m2 -%endif - movu m0, [aq+xq*4-(384+16)*4-4] - movu m2, [aq+xq*4-(384+16)*4+4] - mova m1, [aq+xq*4-(384+16)*4] ; a:top [first half] - paddd m0, m2 ; a:tl+tr [first half] - movu m2, [aq+xq*4-(384+16)*4-4+16] - movu m4, [aq+xq*4-(384+16)*4+4+16] - mova m3, [aq+xq*4-(384+16)*4+16] ; a:top [second half] - paddd m2, m4 ; a:tl+tr [second half] - movu m4, [aq+xq*4-4] - movu m5, [aq+xq*4+4] - paddd m1, [aq+xq*4] ; a:top+ctr [first half] - paddd m4, m5 ; a:l+r [first half] - movu m5, [aq+xq*4+16-4] - movu m6, [aq+xq*4+16+4] - paddd m3, [aq+xq*4+16] ; a:top+ctr [second half] - paddd m5, m6 ; a:l+r [second half] -%if ARCH_X86_64 - movu m6, [bq+xq*2-(384+16)*2-2] - movu m8, [bq+xq*2-(384+16)*2+2] - mova m7, [bq+xq*2-(384+16)*2] ; b:top - paddw m6, m8 ; b:tl+tr - movu m8, [bq+xq*2-2] - movu m9, [bq+xq*2+2] - paddw m7, [bq+xq*2] ; b:top+ctr - paddw m8, m9 ; b:l+r -%endif - - lea tq, [tq+xq*2] - lea srcq, [srcq+xq*1] - lea aq, [aq+xq*4+(384+16)*4] - lea bq, [bq+xq*2+(384+16)*2] - mov yd, hd -.loop_y: -%if ARCH_X86_64 - movu m9, [bq-2] - movu m10, [bq+2] - paddw m7, [bq] ; b:top+ctr+bottom - paddw m9, m10 ; b:bl+br - paddw m10, m7, m8 ; b:top+ctr+bottom+l+r - paddw m6, m9 ; b:tl+tr+bl+br - psubw m7, [bq-(384+16)*2*2] ; b:ctr+bottom - paddw m10, m6 - psllw m10, 2 - psubw m10, m6 ; aa - pxor m14, m14 - movq m12, [srcq] - punpcklbw m12, m14 - punpcklwd m6, m10, m15 - punpckhwd m10, m15 - punpcklwd m13, m12, m15 - punpckhwd m12, m15 - pmaddwd m6, m13 ; aa*src[x]+256 [first half] - pmaddwd m10, m12 ; aa*src[x]+256 [second half] -%else - paddd m1, [aq] ; a:top+ctr+bottom [first half] - paddd m3, [aq+16] ; a:top+ctr+bottom [second half] - mova [esp+0x50], m1 - mova [esp+0x40], m3 - mova [esp+0x30], m4 - movu m6, [aq-4] - movu m7, [aq+4] - paddd m1, m4 ; a:top+ctr+bottom+l+r [first half] - paddd m3, m5 ; a:top+ctr+bottom+l+r [second half] - paddd m6, m7 ; a:bl+br [first half] - movu m7, [aq+16-4] - movu m4, [aq+16+4] - paddd m7, m4 ; a:bl+br [second half] - paddd m0, m6 ; a:tl+tr+bl+br [first half] - paddd m2, m7 ; a:tl+tr+bl+br [second half] - paddd m1, m0 - paddd m3, m2 - pslld m1, 2 - pslld m3, 2 - psubd m1, m0 ; bb [first half] - psubd m3, m2 ; bb [second half] -%endif - -%if ARCH_X86_64 - movu m11, [aq-4] - movu m12, [aq+4] - paddd m1, [aq] ; a:top+ctr+bottom [first half] - paddd m11, m12 ; a:bl+br [first half] - movu m12, [aq+16-4] - movu m13, [aq+16+4] - paddd m3, [aq+16] ; a:top+ctr+bottom [second half] - paddd m12, m13 ; a:bl+br [second half] - paddd m13, m1, m4 ; a:top+ctr+bottom+l+r [first half] - paddd m14, m3, m5 ; a:top+ctr+bottom+l+r [second half] - paddd m0, m11 ; a:tl+tr+bl+br [first half] - paddd m2, m12 ; a:tl+tr+bl+br [second half] - paddd m13, m0 - paddd m14, m2 - pslld m13, 2 - pslld m14, 2 - psubd m13, m0 ; bb [first half] - psubd m14, m2 ; bb [second half] - psubd m1, [aq-(384+16)*4*2] ; a:ctr+bottom [first half] - psubd m3, [aq-(384+16)*4*2+16] ; a:ctr+bottom [second half] -%else - mova m4, [esp+0x80] - mova [esp+0x80], m5 - mova m5, [esp+0x70] - mova [esp+0x70], m6 - mova m6, [esp+0x60] - mova [esp+0x60], m7 - mova [esp+0x20], m1 - movu m7, [bq-2] - movu m1, [bq+2] - paddw m5, [bq] ; b:top+ctr+bottom - paddw m7, m1 - paddw m1, m5, m6 ; b:top+ctr+bottom+l+r - paddw m4, m7 ; b:tl+tr+bl+br - psubw m5, [bq-(384+16)*2*2] ; b:ctr+bottom - paddw m1, m4 - psllw m1, 2 - psubw m1, m4 ; aa - movq m0, [srcq] - XCHG_PIC_REG - punpcklbw m0, [PIC_sym(pb_0)] - punpcklwd m4, m1, [PIC_sym(pw_16)] - punpckhwd m1, [PIC_sym(pw_16)] - punpcklwd m2, m0, [PIC_sym(pw_16)] - punpckhwd m0, [PIC_sym(pw_16)] - XCHG_PIC_REG - pmaddwd m4, m2 ; aa*src[x]+256 [first half] - pmaddwd m1, m0 ; aa*src[x]+256 [second half] -%endif - -%if ARCH_X86_64 - paddd m6, m13 - paddd m10, m14 - psrad m6, 9 - psrad m10, 9 - packssdw m6, m10 - mova [tq], m6 -%else - paddd m4, [esp+0x20] - paddd m1, m3 - psrad m4, 9 - psrad m1, 9 - packssdw m4, m1 - mova [tq], m4 -%endif - - ; shift to next row -%if ARCH_X86_64 - mova m0, m4 - mova m2, m5 - mova m4, m11 - mova m5, m12 - mova m6, m8 - mova m8, m9 -%else - mova m1, [esp+0x50] - mova m3, [esp+0x40] - mova m0, [esp+0x30] - mova m2, [esp+0x80] - mova m4, [esp+0x70] - mova [esp+0x70], m5 - mova m5, [esp+0x60] - mova [esp+0x80], m6 - mova [esp+0x60], m7 - psubd m1, [aq-(384+16)*4*2] ; a:ctr+bottom [first half] - psubd m3, [aq-(384+16)*4*2+16] ; a:ctr+bottom [second half] -%endif - - add srcq, strideq - add aq, (384+16)*4 - add bq, (384+16)*2 - add tq, 384*2 - dec yd - jg .loop_y - add xd, 8 - cmp xd, wd - jl .loop_x - RET - -cglobal sgr_weighted1_8bpc, 4, 7, 8, dst, stride, t, w, h, wt - movifnidn hd, hm -%if ARCH_X86_32 - SETUP_PIC r6, 0 -%endif - movd m0, wtm - pshufb m0, [PIC_sym(pb_0_1)] - psllw m0, 4 - pxor m7, m7 - DEFINE_ARGS dst, stride, t, w, h, idx -.loop_y: - xor idxd, idxd -.loop_x: - mova m1, [tq+idxq*2+ 0] - mova m4, [tq+idxq*2+16] - mova m5, [dstq+idxq] - punpcklbw m2, m5, m7 - punpckhbw m5, m7 - psllw m3, m2, 4 - psllw m6, m5, 4 - psubw m1, m3 - psubw m4, m6 - pmulhrsw m1, m0 - pmulhrsw m4, m0 - paddw m1, m2 - paddw m4, m5 - packuswb m1, m4 - mova [dstq+idxq], m1 - add idxd, 16 - cmp idxd, wd - jl .loop_x - add dstq, strideq - add tq, 384 * 2 - dec hd - jg .loop_y - RET - -%if ARCH_X86_64 -cglobal sgr_box5_h_8bpc, 5, 11, 12, sumsq, sum, left, src, stride, w, h, edge, x, xlim - mov edged, edgem - movifnidn wd, wm - mov hd, hm - mova m10, [pb_0] - mova m11, [pb_0_1] -%else -cglobal sgr_box5_h_8bpc, 7, 7, 8, sumsq, sum, left, src, xlim, x, h, edge - %define edgeb byte edgem - %define wd xd - %define wq wd - %define wm r5m - %define strideq r4m - SUB esp, 8 - SETUP_PIC sumsqd, 1, 1 +DECLARE_REG_TMP 0, 1, 2, 3, 4 + %if STACK_ALIGNMENT < 16 + %assign extra_stack 5*16 + %else + %assign extra_stack 3*16 + %endif +cglobal sgr_filter_5x5_8bpc, 1, 7, 8, -400*24-16-extra_stack, \ + dst, dst_stride, left, lpf, lpf_stride, w, params, h + %if STACK_ALIGNMENT < 16 + %define dstm dword [esp+calloff+16*0+4*6] + %define dst_stridemp dword [esp+calloff+16*0+4*7] + %define leftm dword [esp+calloff+16*3+4*0] + %define lpfm dword [esp+calloff+16*3+4*1] + %define lpf_stridem dword [esp+calloff+16*3+4*2] + %define w0m dword [esp+calloff+16*3+4*3] + %define hd dword [esp+calloff+16*3+4*4] + %define edgeb byte [esp+calloff+16*3+4*5] + %define edged dword [esp+calloff+16*3+4*5] + %define leftmp leftm + %else + %define w0m wm + %define hd dword r6m + %define edgeb byte r8m + %define edged dword r8m + %endif + %define hvsrcm dword [esp+calloff+4*0] + %define w1m dword [esp+calloff+4*1] + %define t0m dword [esp+calloff+4*2] + %define t2m dword [esp+calloff+4*3] + %define t3m dword [esp+calloff+4*4] + %define t4m dword [esp+calloff+4*5] + %define m8 [base+pb_1] + %define m9 [esp+calloff+16*2] + %define m10 [base+pd_0xf00800a4] + %define m11 [base+sgr_lshuf5] + %define m12 [base+pd_34816] + %define m13 [base+pb_0to15] - %define m10 [PIC_sym(pb_0)] - %define m11 [PIC_sym(pb_0_1)] + %define r10 r5 + %define base r6-$$ + %assign calloff 0 + %if STACK_ALIGNMENT < 16 + mov dst_strideq, [rstk+stack_offset+ 8] + mov leftq, [rstk+stack_offset+12] + mov lpfq, [rstk+stack_offset+16] + mov lpf_strideq, [rstk+stack_offset+20] + mov wd, [rstk+stack_offset+24] + mov dstm, dstq + mov dst_stridemp, dst_strideq + mov leftm, leftq + mov r1, [rstk+stack_offset+28] + mov r2, [rstk+stack_offset+36] + mov lpfm, lpfq + mov lpf_stridem, lpf_strideq + mov hd, r1 + mov edged, r2 + %endif +%else +DECLARE_REG_TMP 4, 9, 7, 11, 12 +cglobal sgr_filter_5x5_8bpc, 5, 15, 14, -400*24-16, dst, dst_stride, left, lpf, \ + lpf_stride, w, edge, params, h %endif - - test edgeb, 2 ; have_right - jz .no_right - xor xlimd, xlimd - add wd, 2 - add wd, 15 - and wd, ~15 - jmp .right_done -.no_right: - mov xlimd, 3 - dec wd -.right_done: - pxor m1, m1 - lea srcq, [srcq+wq+1] - lea sumq, [sumq+wq*2-2] - lea sumsqq, [sumsqq+wq*4-4] - neg wq -%if ARCH_X86_64 - lea r10, [pb_right_ext_mask+24] -%else - mov wm, xd - %define wq wm +%if ARCH_X86_64 || STACK_ALIGNMENT >= 16 + mov wd, wm %endif - -.loop_y: - mov xq, wq - ; load left - test edgeb, 1 ; have_left - jz .no_left - test leftq, leftq - jz .load_left_from_main - movd m0, [leftq] - movd m2, [srcq+xq-1] - pslldq m2, 4 - por m0, m2 - pslldq m0, 11 - add leftq, 4 - jmp .expand_x -.no_left: - movd m0, [srcq+xq-1] - XCHG_PIC_REG - pshufb m0, m10 - XCHG_PIC_REG - jmp .expand_x -.load_left_from_main: - movd m0, [srcq+xq-4] - pslldq m0, 12 -.expand_x: - punpckhbw m0, m1 - - ; when we reach this, m0 contains left two px in highest words - cmp xd, -8 - jle .loop_x - test xd, xd - jge .right_extend -.partial_load_and_extend: - XCHG_PIC_REG - movd m3, [srcq-1] - movq m2, [srcq+xq] - pshufb m3, m10 - punpcklbw m3, m1 - punpcklbw m2, m1 %if ARCH_X86_64 - movu m4, [r10+xq*2] + mov paramsq, paramsmp + lea r13, [sgr_x_by_x-0xf03] + mov edged, r8m + mov hd, r6m + movu m9, [paramsq] + add lpfq, wq + mova m8, [pb_1] + lea t1, [rsp+wq*2+20] + mova m10, [pd_0xf00800a4] + add dstq, wq + lea t3, [rsp+wq*4+400*12+16] + mova m12, [pd_34816] ; (1 << 11) + (1 << 15) + lea t4, [rsp+wq*2+400*20+16] + pshufhw m7, m9, q0000 + pshufb m9, [pw_256] ; s0 + punpckhqdq m7, m7 ; w0 + neg wq + mova m13, [pb_0to15] + pxor m6, m6 + mova m11, [sgr_lshuf5] + psllw m7, 4 + DEFINE_ARGS dst, dst_stride, left, lpf, lpf_stride, _, edge, _, h, _, w + %define lpfm [rsp+0] + %define lpf_stridem [rsp+8] %else - movu m4, [PIC_sym(pb_right_ext_mask)+xd*2+24] - XCHG_PIC_REG + mov r1, [rstk+stack_offset+32] ; params + LEA r6, $$ + movu m1, [r1] + add lpfm, wq + lea t1, [rsp+extra_stack+wq*2+20] + add dstq, wq + lea t3, [rsp+extra_stack+wq*4+400*12+16] + mov dstm, dstq + lea t4, [rsp+extra_stack+wq*2+400*20+16] + mov t3m, t3 + pshufhw m7, m1, q0000 + mov t4m, t4 + pshufb m1, [base+pw_256] ; s0 + punpckhqdq m7, m7 ; w0 + psllw m7, 4 + neg wq + mova m9, m1 + pxor m6, m6 + mov w1m, wd + sub wd, 2 + mov lpfq, lpfm + mov lpf_strideq, lpf_stridem + mov w0m, wd %endif - pand m2, m4 - pandn m4, m3 - por m2, m4 - jmp .loop_x_noload -.right_extend: - psrldq m2, m0, 14 - XCHG_PIC_REG - pshufb m2, m11 - XCHG_PIC_REG - jmp .loop_x_noload - -.loop_x: - movq m2, [srcq+xq] - punpcklbw m2, m1 -.loop_x_noload: - palignr m3, m2, m0, 8 - palignr m4, m2, m0, 10 - palignr m5, m2, m0, 12 - palignr m6, m2, m0, 14 - + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, lpf_strideq + movif32 t2m, t1 + mov t2, t1 + call .top_fixup + add t1, 400*6 + call .h_top + lea r10, [lpfq+lpf_strideq*4] + mov lpfq, dstq + movif64 lpf_stridem, lpf_strideq + add r10, lpf_strideq + mov lpfm, r10 ; below + movif32 t0m, t2 + mov t0, t2 + dec hd + jz .height1 + or edged, 16 + call .h +.main: + add lpfq, dst_stridemp + movif32 t4, t4m + call .hv + call .prep_n + sub hd, 2 + jl .extend_bottom +.main_loop: + movif32 lpfq, hvsrcm + add lpfq, dst_stridemp %if ARCH_X86_64 - paddw m0, m3, m2 - punpcklwd m7, m3, m2 - punpckhwd m3, m2 - paddw m0, m4 - punpcklwd m8, m4, m5 - punpckhwd m4, m5 - paddw m0, m5 - punpcklwd m9, m6, m1 - punpckhwd m5, m6, m1 - paddw m0, m6 - pmaddwd m7, m7 - pmaddwd m3, m3 - pmaddwd m8, m8 - pmaddwd m4, m4 - pmaddwd m9, m9 - pmaddwd m5, m5 - paddd m7, m8 - paddd m3, m4 - paddd m7, m9 - paddd m3, m5 - movu [sumq+xq*2], m0 - movu [sumsqq+xq*4+ 0], m7 - movu [sumsqq+xq*4+16], m3 + test hb, hb %else - paddw m0, m3, m2 - paddw m0, m4 - paddw m0, m5 - paddw m0, m6 - movu [sumq+xq*2], m0 - punpcklwd m7, m3, m2 - punpckhwd m3, m2 - punpcklwd m0, m4, m5 - punpckhwd m4, m5 - punpckhwd m5, m6, m1 - pmaddwd m7, m7 - pmaddwd m3, m3 - pmaddwd m0, m0 - pmaddwd m4, m4 - pmaddwd m5, m5 - paddd m7, m0 - paddd m3, m4 - paddd m3, m5 - punpcklwd m0, m6, m1 - pmaddwd m0, m0 - paddd m7, m0 - movu [sumsqq+xq*4+ 0], m7 - movu [sumsqq+xq*4+16], m3 + mov r5, hd + test r5, r5 %endif - - mova m0, m2 - add xq, 8 - - ; if x <= -8 we can reload more pixels - ; else if x < 0 we reload and extend (this implies have_right=0) - ; else if x < xlimd we extend from previous load (this implies have_right=0) - ; else we are done - - cmp xd, -8 - jle .loop_x - test xd, xd - jl .partial_load_and_extend - cmp xd, xlimd - jl .right_extend - - add srcq, strideq - add sumsqq, (384+16)*4 - add sumq, (384+16)*2 - dec hd - jg .loop_y -%if ARCH_X86_32 - ADD esp, 8 -%endif + jz .odd_height + call .h + add lpfq, dst_stridemp + call .hv + movif32 dstq, dstm + call .n0 + call .n1 + sub hd, 2 + movif32 t0, t0m + jge .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .extend_bottom + mov lpfq, lpfm + call .h_top + add lpfq, lpf_stridem + call .hv_bottom +.end: + movif32 dstq, dstm + call .n0 + call .n1 +.end2: RET - -%if ARCH_X86_64 -cglobal sgr_box5_v_8bpc, 4, 10, 15, sumsq, sum, w, h, edge, x, y, sumsq_ptr, sum_ptr, ylim - movifnidn edged, edgem - mov ylimd, edged -%else -cglobal sgr_box5_v_8bpc, 5, 7, 8, -44, sumsq, sum, x, y, ylim, sumsq_ptr, sum_ptr - %define wm [esp+0] - %define hm [esp+4] - %define edgem [esp+8] - mov wm, xd - mov hm, yd - mov edgem, ylimd -%endif - - and ylimd, 8 ; have_bottom - shr ylimd, 2 - sub ylimd, 3 ; -3 if have_bottom=0, else -1 - mov xq, -2 +.height1: + movif32 t4, t4m + call .hv + call .prep_n + jmp .odd_height_end +.odd_height: + call .hv + movif32 dstq, dstm + call .n0 + call .n1 +.odd_height_end: + call .v + movif32 dstq, dstm + call .n0 + jmp .end2 +.extend_bottom: + call .v + jmp .end +.no_top: + lea r10, [lpfq+lpf_strideq*4] + mov lpfq, dstq + movif64 lpf_stridem, lpf_strideq + lea r10, [r10+lpf_strideq*2] + mov lpfm, r10 + call .h + lea t2, [t1+400*6] + movif32 t2m, t2 + call .top_fixup + dec hd + jz .no_top_height1 + or edged, 16 + mov t0, t1 + mov t1, t2 + movif32 t0m, t0 + jmp .main +.no_top_height1: + movif32 t3, t3m + movif32 t4, t4m + call .v + call .prep_n + jmp .odd_height_end +.extend_right: +%assign stack_offset stack_offset+8 +%assign calloff 8 + movd m1, wd + movd m3, [lpfq-1] + pshufb m1, m6 + pshufb m3, m6 + psubb m2, m8, m1 + pcmpgtb m2, m13 + pand m5, m2 + pandn m2, m3 + por m5, m2 + ret +%assign stack_offset stack_offset-4 +%assign calloff 4 +.h: ; horizontal boxsum %if ARCH_X86_64 -.loop_x: - lea yd, [hd+ylimd+2] - lea sumsq_ptrq, [sumsqq+xq*4+4-(384+16)*4] - lea sum_ptrq, [ sumq+xq*2+2-(384+16)*2] - test edgeb, 4 ; have_top - jnz .load_top - movu m0, [sumsq_ptrq+(384+16)*4*1] - movu m1, [sumsq_ptrq+(384+16)*4*1+16] - mova m2, m0 - mova m3, m1 - mova m4, m0 - mova m5, m1 - mova m6, m0 - mova m7, m1 - movu m10, [sum_ptrq+(384+16)*2*1] - mova m11, m10 - mova m12, m10 - mova m13, m10 - jmp .loop_y_second_load -.load_top: - movu m0, [sumsq_ptrq-(384+16)*4*1] ; l3/4sq [left] - movu m1, [sumsq_ptrq-(384+16)*4*1+16] ; l3/4sq [right] - movu m4, [sumsq_ptrq-(384+16)*4*0] ; l2sq [left] - movu m5, [sumsq_ptrq-(384+16)*4*0+16] ; l2sq [right] - mova m2, m0 - mova m3, m1 - movu m10, [sum_ptrq-(384+16)*2*1] ; l3/4 - movu m12, [sum_ptrq-(384+16)*2*0] ; l2 - mova m11, m10 -.loop_y: - movu m6, [sumsq_ptrq+(384+16)*4*1] ; l1sq [left] - movu m7, [sumsq_ptrq+(384+16)*4*1+16] ; l1sq [right] - movu m13, [sum_ptrq+(384+16)*2*1] ; l1 -.loop_y_second_load: - test yd, yd - jle .emulate_second_load - movu m8, [sumsq_ptrq+(384+16)*4*2] ; l0sq [left] - movu m9, [sumsq_ptrq+(384+16)*4*2+16] ; l0sq [right] - movu m14, [sum_ptrq+(384+16)*2*2] ; l0 -.loop_y_noload: - paddd m0, m2 - paddd m1, m3 - paddw m10, m11 - paddd m0, m4 - paddd m1, m5 - paddw m10, m12 - paddd m0, m6 - paddd m1, m7 - paddw m10, m13 - paddd m0, m8 - paddd m1, m9 - paddw m10, m14 - movu [sumsq_ptrq+ 0], m0 - movu [sumsq_ptrq+16], m1 - movu [sum_ptrq], m10 - - ; shift position down by one - mova m0, m4 - mova m1, m5 - mova m2, m6 - mova m3, m7 - mova m4, m8 - mova m5, m9 - mova m10, m12 - mova m11, m13 - mova m12, m14 - add sumsq_ptrq, (384+16)*4*2 - add sum_ptrq, (384+16)*2*2 - sub yd, 2 - jge .loop_y - ; l1 = l0 - mova m6, m8 - mova m7, m9 - mova m13, m14 - cmp yd, ylimd - jg .loop_y_noload - add xd, 8 - cmp xd, wd - jl .loop_x - RET -.emulate_second_load: - mova m8, m6 - mova m9, m7 - mova m14, m13 - jmp .loop_y_noload + lea wq, [r5-2] +%else + %define leftq r5 +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movif32 leftq, leftm + movddup m4, [leftq-4] + movif32 wq, w0m + mova m5, [lpfq+wq+2] + add leftmp, 4 + palignr m5, m4, 13 + jmp .h_main +.h_extend_left: + movif32 wq, w0m + mova m5, [lpfq+wq+2] + pshufb m5, m11 + jmp .h_main +.h_top: +%if ARCH_X86_64 + lea wq, [r5-2] +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movif32 wq, w0m +.h_loop: + movu m5, [lpfq+wq-1] +.h_main: + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp wd, -10 + jl .h_have_right + call .extend_right +.h_have_right: + punpcklbw m4, m5, m6 + punpckhbw m5, m6 + palignr m2, m5, m4, 2 + paddw m0, m4, m2 + palignr m3, m5, m4, 6 + paddw m0, m3 + punpcklwd m1, m2, m3 + pmaddwd m1, m1 + punpckhwd m2, m3 + pmaddwd m2, m2 + palignr m5, m4, 8 + paddw m0, m5 + punpcklwd m3, m4, m5 + pmaddwd m3, m3 + paddd m1, m3 + punpckhwd m3, m4, m5 + pmaddwd m3, m3 + shufps m4, m5, q2121 + paddw m0, m4 ; sum + punpcklwd m5, m4, m6 + pmaddwd m5, m5 + punpckhwd m4, m6 + pmaddwd m4, m4 + paddd m2, m3 + test edgeb, 16 ; y > 0 + jz .h_loop_end + paddw m0, [t1+wq*2+400*0] + paddd m1, [t1+wq*2+400*2] + paddd m2, [t1+wq*2+400*4] +.h_loop_end: + paddd m1, m5 ; sumsq + paddd m2, m4 + mova [t1+wq*2+400*0], m0 + mova [t1+wq*2+400*2], m1 + mova [t1+wq*2+400*4], m2 + add wq, 8 + jl .h_loop + ret +.top_fixup: +%if ARCH_X86_64 + lea wq, [r5-2] +%else + mov wd, w0m +%endif +.top_fixup_loop: ; the sums of the first row needs to be doubled + mova m0, [t1+wq*2+400*0] + mova m1, [t1+wq*2+400*2] + mova m2, [t1+wq*2+400*4] + paddw m0, m0 + paddd m1, m1 + paddd m2, m2 + mova [t2+wq*2+400*0], m0 + mova [t2+wq*2+400*2], m1 + mova [t2+wq*2+400*4], m2 + add wq, 8 + jl .top_fixup_loop + ret +ALIGN function_align +.hv: ; horizontal boxsum + vertical boxsum + ab +%if ARCH_X86_64 + lea wq, [r5-2] %else -.sumsq_loop_x: - lea yd, [ylimd+2] - add yd, hm - lea sumsq_ptrq, [sumsqq+xq*4+4-(384+16)*4] - test byte edgem, 4 ; have_top - jnz .sumsq_load_top - movu m0, [sumsq_ptrq+(384+16)*4*1] - movu m1, [sumsq_ptrq+(384+16)*4*1+16] - mova m4, m0 - mova m5, m1 - mova m6, m0 - mova m7, m1 - mova [esp+0x1c], m0 - mova [esp+0x0c], m1 - jmp .sumsq_loop_y_second_load -.sumsq_load_top: - movu m0, [sumsq_ptrq-(384+16)*4*1] ; l3/4sq [left] - movu m1, [sumsq_ptrq-(384+16)*4*1+16] ; l3/4sq [right] - movu m4, [sumsq_ptrq-(384+16)*4*0] ; l2sq [left] - movu m5, [sumsq_ptrq-(384+16)*4*0+16] ; l2sq [right] - mova [esp+0x1c], m0 - mova [esp+0x0c], m1 -.sumsq_loop_y: - movu m6, [sumsq_ptrq+(384+16)*4*1] ; l1sq [left] - movu m7, [sumsq_ptrq+(384+16)*4*1+16] ; l1sq [right] -.sumsq_loop_y_second_load: - test yd, yd - jle .sumsq_emulate_second_load - movu m2, [sumsq_ptrq+(384+16)*4*2] ; l0sq [left] - movu m3, [sumsq_ptrq+(384+16)*4*2+16] ; l0sq [right] -.sumsq_loop_y_noload: - paddd m0, [esp+0x1c] - paddd m1, [esp+0x0c] - paddd m0, m4 - paddd m1, m5 - paddd m0, m6 - paddd m1, m7 - paddd m0, m2 - paddd m1, m3 - movu [sumsq_ptrq+ 0], m0 - movu [sumsq_ptrq+16], m1 - - ; shift position down by one - mova m0, m4 - mova m1, m5 - mova m4, m2 - mova m5, m3 - mova [esp+0x1c], m6 - mova [esp+0x0c], m7 - add sumsq_ptrq, (384+16)*4*2 - sub yd, 2 - jge .sumsq_loop_y - ; l1 = l0 - mova m6, m2 - mova m7, m3 - cmp yd, ylimd - jg .sumsq_loop_y_noload - add xd, 8 - cmp xd, wm - jl .sumsq_loop_x - - mov xd, -2 -.sum_loop_x: - lea yd, [ylimd+2] - add yd, hm - lea sum_ptrq, [sumq+xq*2+2-(384+16)*2] - test byte edgem, 4 ; have_top - jnz .sum_load_top - movu m0, [sum_ptrq+(384+16)*2*1] - mova m1, m0 - mova m2, m0 - mova m3, m0 - jmp .sum_loop_y_second_load -.sum_load_top: - movu m0, [sum_ptrq-(384+16)*2*1] ; l3/4 - movu m2, [sum_ptrq-(384+16)*2*0] ; l2 - mova m1, m0 -.sum_loop_y: - movu m3, [sum_ptrq+(384+16)*2*1] ; l1 -.sum_loop_y_second_load: - test yd, yd - jle .sum_emulate_second_load - movu m4, [sum_ptrq+(384+16)*2*2] ; l0 -.sum_loop_y_noload: - paddw m0, m1 - paddw m0, m2 - paddw m0, m3 - paddw m0, m4 - movu [sum_ptrq], m0 - - ; shift position down by one - mova m0, m2 - mova m1, m3 - mova m2, m4 - add sum_ptrq, (384+16)*2*2 - sub yd, 2 - jge .sum_loop_y - ; l1 = l0 - mova m3, m4 - cmp yd, ylimd - jg .sum_loop_y_noload - add xd, 8 - cmp xd, wm - jl .sum_loop_x - RET -.sumsq_emulate_second_load: - mova m2, m6 - mova m3, m7 - jmp .sumsq_loop_y_noload -.sum_emulate_second_load: - mova m4, m3 - jmp .sum_loop_y_noload + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left + movif32 leftq, leftm + movddup m4, [leftq-4] + movif32 wq, w0m + mova m5, [lpfq+wq+2] + add leftmp, 4 + palignr m5, m4, 13 + jmp .hv_main +.hv_extend_left: + movif32 wq, w0m + mova m5, [lpfq+wq+2] + pshufb m5, m11 + jmp .hv_main +.hv_bottom: +%if ARCH_X86_64 + lea wq, [r5-2] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv_extend_left + movif32 wq, w0m +%if ARCH_X86_32 + jmp .hv_loop_start %endif - -cglobal sgr_calc_ab2_8bpc, 4, 7, 11, a, b, w, h, s - movifnidn sd, sm - sub aq, (384+16-1)*4 - sub bq, (384+16-1)*2 - add hd, 2 +.hv_loop: + movif32 lpfq, hvsrcm +.hv_loop_start: + movu m5, [lpfq+wq-1] +.hv_main: + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv_have_right + cmp wd, -10 + jl .hv_have_right + call .extend_right +.hv_have_right: + movif32 t3, hd + punpcklbw m4, m5, m6 + punpckhbw m5, m6 + palignr m3, m5, m4, 2 + paddw m0, m4, m3 + palignr m1, m5, m4, 6 + paddw m0, m1 + punpcklwd m2, m3, m1 + pmaddwd m2, m2 + punpckhwd m3, m1 + pmaddwd m3, m3 + palignr m5, m4, 8 + paddw m0, m5 + punpcklwd m1, m4, m5 + pmaddwd m1, m1 + paddd m2, m1 + punpckhwd m1, m4, m5 + pmaddwd m1, m1 + shufps m4, m5, q2121 + paddw m0, m4 ; h sum + punpcklwd m5, m4, m6 + pmaddwd m5, m5 + punpckhwd m4, m6 + pmaddwd m4, m4 + paddd m3, m1 + paddd m2, m5 ; h sumsq + paddd m3, m4 + paddw m1, m0, [t1+wq*2+400*0] + paddd m4, m2, [t1+wq*2+400*2] + paddd m5, m3, [t1+wq*2+400*4] %if ARCH_X86_64 - LEA r5, sgr_x_by_x-0xF03 + test hd, hd %else - SETUP_PIC r5, 0 + test t3, t3 %endif - movd m6, sd - pshuflw m6, m6, q0000 - punpcklqdq m6, m6 - pxor m7, m7 - DEFINE_ARGS a, b, w, h, x + jz .hv_last_row +.hv_main2: + paddw m1, [t2+wq*2+400*0] ; hv sum + paddd m4, [t2+wq*2+400*2] ; hv sumsq + paddd m5, [t2+wq*2+400*4] + mova [t0+wq*2+400*0], m0 + pslld m0, m4, 4 + mova [t0+wq*2+400*2], m2 + mova [t0+wq*2+400*4], m3 + pslld m2, m4, 3 + paddd m4, m0 + pslld m0, m5, 4 + paddd m4, m2 ; a * 25 + pslld m2, m5, 3 + paddd m5, m0 + paddd m5, m2 + punpcklwd m0, m1, m6 ; b + punpckhwd m1, m6 + pmaddwd m2, m0, m0 ; b * b + pmaddwd m3, m1, m1 + psubd m4, m2 ; p + psubd m5, m3 + MULLD m4, m9, m2 ; p * s + MULLD m5, m9, m2 + pmaddwd m0, m10 ; b * 164 + pmaddwd m1, m10 + paddusw m4, m10 + paddusw m5, m10 + psrld m4, 20 ; min(z, 255) + movif32 t3, t3m + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, t2, t2m + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m2 + MULLD m1, m5, m2 + paddd m0, m12 ; x * b * 164 + (1 << 11) + (1 << 15) + paddd m1, m12 + mova [t4+wq*2+4], m3 + psrld m0, 12 ; b + psrld m1, 12 + mova [t3+wq*4+ 8], m0 + mova [t3+wq*4+24], m1 + add wq, 8 + jl .hv_loop + mov t2, t1 + mov t1, t0 + mov t0, t2 + movif32 t2m, t2 + movif32 t0m, t0 + ret +.hv_last_row: ; esoteric edge case for odd heights + mova [t1+wq*2+400*0], m1 + paddw m1, m0 + mova [t1+wq*2+400*2], m4 + paddd m4, m2 + mova [t1+wq*2+400*4], m5 + paddd m5, m3 + jmp .hv_main2 +.v: ; vertical boxsum + ab %if ARCH_X86_64 - mova m8, [pd_0xF0080029] - mova m9, [pw_256] - psrld m10, m9, 15 ; pd_512 + lea wq, [r5-2] %else - %define m8 [PIC_sym(pd_0xF0080029)] - %define m9 [PIC_sym(pw_256)] - %define m10 [PIC_sym(pd_512)] + mov wd, w0m %endif -.loop_y: - mov xq, -2 -.loop_x: - movq m0, [bq+xq*2+0] - movq m1, [bq+xq*2+8] - punpcklwd m0, m7 - punpcklwd m1, m7 - movu m2, [aq+xq*4+ 0] - movu m3, [aq+xq*4+16] - pslld m4, m2, 3 ; aa * 8 - pslld m5, m3, 3 - paddd m2, m4 ; aa * 9 - paddd m3, m5 - paddd m4, m4 ; aa * 16 - paddd m5, m5 - paddd m2, m4 ; aa * 25 - paddd m3, m5 - pmaddwd m4, m0, m0 - pmaddwd m5, m1, m1 - psubd m2, m4 ; p = aa * 25 - bb * bb - psubd m3, m5 - MULLD m2, m6 - MULLD m3, m6 - paddusw m2, m8 - paddusw m3, m8 - psrld m2, 20 ; z - psrld m3, 20 - GATHERDD m4, m2 ; xx - GATHERDD m2, m3 - psrld m4, 24 - psrld m2, 24 - packssdw m3, m4, m2 - pmullw m4, m8 - pmullw m2, m8 - psubw m5, m9, m3 - pmaddwd m0, m4 - pmaddwd m1, m2 - paddd m0, m10 - paddd m1, m10 - psrld m0, 10 - psrld m1, 10 - movu [bq+xq*2], m5 - movu [aq+xq*4+ 0], m0 - movu [aq+xq*4+16], m1 - add xd, 8 - cmp xd, wd - jl .loop_x - add aq, (384+16)*4*2 - add bq, (384+16)*2*2 - sub hd, 2 - jg .loop_y - RET - -%if ARCH_X86_64 -cglobal sgr_finish_filter2_8bpc, 5, 13, 14, t, src, stride, a, b, w, h, \ - tmp_base, src_base, a_base, b_base, x, y - movifnidn wd, wm - mov hd, hm - mov tmp_baseq, tq - mov src_baseq, srcq - mov a_baseq, aq - mov b_baseq, bq - mova m9, [pw_5_6] - mova m12, [pw_256] - psrlw m10, m12, 8 ; pw_1 - psrlw m11, m12, 1 ; pw_128 - pxor m13, m13 -%else -cglobal sgr_finish_filter2_8bpc, 6, 7, 8, t, src, stride, a, b, x, y - %define tmp_baseq r0m - %define src_baseq r1m - %define a_baseq r3m - %define b_baseq r4m - %define wd r5m - %define hd r6m - - SUB esp, 8 - SETUP_PIC yd - - %define m8 m5 - %define m9 [PIC_sym(pw_5_6)] - %define m10 [PIC_sym(pw_1)] - %define m11 [PIC_sym(pw_128)] - %define m12 [PIC_sym(pw_256)] - %define m13 m0 -%endif - xor xd, xd -.loop_x: - mov tq, tmp_baseq - mov srcq, src_baseq - mov aq, a_baseq - mov bq, b_baseq - movu m0, [aq+xq*4-(384+16)*4-4] - mova m1, [aq+xq*4-(384+16)*4] - movu m2, [aq+xq*4-(384+16)*4+4] - movu m3, [aq+xq*4-(384+16)*4-4+16] - mova m4, [aq+xq*4-(384+16)*4+16] - movu m5, [aq+xq*4-(384+16)*4+4+16] - paddd m0, m2 - paddd m3, m5 - paddd m0, m1 - paddd m3, m4 - pslld m2, m0, 2 - pslld m5, m3, 2 - paddd m2, m0 - paddd m5, m3 - paddd m0, m2, m1 ; prev_odd_b [first half] - paddd m1, m5, m4 ; prev_odd_b [second half] - movu m3, [bq+xq*2-(384+16)*2-2] - mova m4, [bq+xq*2-(384+16)*2] - movu m5, [bq+xq*2-(384+16)*2+2] - paddw m3, m5 - punpcklwd m5, m3, m4 - punpckhwd m3, m4 - pmaddwd m5, m9 - pmaddwd m3, m9 - mova m2, m5 - packssdw m2, m3 ; prev_odd_a - lea tq, [tq+xq*2] - lea srcq, [srcq+xq*1] - lea aq, [aq+xq*4+(384+16)*4] - lea bq, [bq+xq*2+(384+16)*2] -%if ARCH_X86_32 - mov [esp], PIC_reg -%endif - mov yd, hd - XCHG_PIC_REG -.loop_y: - movu m3, [aq-4] - mova m4, [aq] - movu m5, [aq+4] - paddd m3, m5 - paddd m3, m4 - pslld m5, m3, 2 - paddd m5, m3 - paddd m5, m4 ; cur_odd_b [first half] - movu m3, [aq+16-4] - mova m6, [aq+16] - movu m7, [aq+16+4] - paddd m3, m7 - paddd m3, m6 - pslld m7, m3, 2 - paddd m7, m3 - paddd m4, m7, m6 ; cur_odd_b [second half] - movu m3, [bq-2] - mova m6, [bq] - movu m7, [bq+2] - paddw m3, m7 - punpcklwd m7, m3, m6 - punpckhwd m3, m6 - pmaddwd m7, m9 - pmaddwd m3, m9 - packssdw m6, m7, m3 ; cur_odd_a - - paddd m0, m5 ; cur_even_b [first half] - paddd m1, m4 ; cur_even_b [second half] - paddw m2, m6 ; cur_even_a - - movq m3, [srcq] -%if ARCH_X86_64 - punpcklbw m3, m13 -%else - mova [td], m5 - pxor m7, m7 - punpcklbw m3, m7 -%endif - punpcklwd m7, m3, m10 - punpckhwd m3, m10 - punpcklwd m8, m2, m12 - punpckhwd m2, m12 - pmaddwd m7, m8 - pmaddwd m3, m2 - paddd m7, m0 - paddd m3, m1 - psrad m7, 9 - psrad m3, 9 +.v_loop: + mova m0, [t1+wq*2+400*0] + mova m2, [t1+wq*2+400*2] + mova m3, [t1+wq*2+400*4] + paddw m1, m0, [t2+wq*2+400*0] + paddd m4, m2, [t2+wq*2+400*2] + paddd m5, m3, [t2+wq*2+400*4] + paddw m0, m0 + paddd m2, m2 + paddd m3, m3 + paddw m1, m0 ; hv sum + paddd m4, m2 ; hv sumsq + pslld m0, m4, 4 + paddd m5, m3 + pslld m2, m4, 3 + paddd m4, m0 + pslld m0, m5, 4 + paddd m4, m2 ; a * 25 + pslld m2, m5, 3 + paddd m5, m0 + paddd m5, m2 + punpcklwd m0, m1, m6 + punpckhwd m1, m6 + pmaddwd m2, m0, m0 ; b * b + pmaddwd m3, m1, m1 + psubd m4, m2 ; p + psubd m5, m3 + MULLD m4, m9, m2 ; p * s + MULLD m5, m9, m2 + pmaddwd m0, m10 ; b * 164 + pmaddwd m1, m10 + paddusw m4, m10 + paddusw m5, m10 + psrld m4, 20 ; min(z, 255) + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, t2, t2m + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m2 + MULLD m1, m5, m2 + paddd m0, m12 ; x * b * 164 + (1 << 11) + (1 << 15) + paddd m1, m12 + mova [t4+wq*2+4], m3 + psrld m0, 12 ; b + psrld m1, 12 + mova [t3+wq*4+ 8], m0 + mova [t3+wq*4+24], m1 + add wq, 8 + jl .v_loop + ret +.prep_n: ; initial neighbor setup + movif64 wq, r5 + movif32 wd, w1m +.prep_n_loop: + movu m0, [t4+wq*2+ 2] + movu m3, [t4+wq*2+ 4] + movu m1, [t3+wq*4+ 4] + movu m4, [t3+wq*4+ 8] + movu m2, [t3+wq*4+20] + movu m5, [t3+wq*4+24] + paddw m3, m0 + paddd m4, m1 + paddd m5, m2 + paddw m3, [t4+wq*2+ 0] + paddd m4, [t3+wq*4+ 0] + paddd m5, [t3+wq*4+16] + paddw m0, m3 + psllw m3, 2 + paddd m1, m4 + pslld m4, 2 + paddd m2, m5 + pslld m5, 2 + paddw m0, m3 ; a 565 + paddd m1, m4 ; b 565 + paddd m2, m5 + mova [t4+wq*2+400*2+ 0], m0 + mova [t3+wq*4+400*4+ 0], m1 + mova [t3+wq*4+400*4+16], m2 + add wq, 8 + jl .prep_n_loop + ret +ALIGN function_align +.n0: ; neighbor + output (even rows) + movif64 wq, r5 + movif32 wd, w1m +.n0_loop: + movu m0, [t4+wq*2+ 2] + movu m3, [t4+wq*2+ 4] + movu m1, [t3+wq*4+ 4] + movu m4, [t3+wq*4+ 8] + movu m2, [t3+wq*4+20] + movu m5, [t3+wq*4+24] + paddw m3, m0 + paddd m4, m1 + paddd m5, m2 + paddw m3, [t4+wq*2+ 0] + paddd m4, [t3+wq*4+ 0] + paddd m5, [t3+wq*4+16] + paddw m0, m3 + psllw m3, 2 + paddd m1, m4 + pslld m4, 2 + paddd m2, m5 + pslld m5, 2 + paddw m0, m3 ; a 565 + paddd m1, m4 ; b 565 + paddd m2, m5 + paddw m3, m0, [t4+wq*2+400*2+ 0] + paddd m4, m1, [t3+wq*4+400*4+ 0] + paddd m5, m2, [t3+wq*4+400*4+16] + mova [t4+wq*2+400*2+ 0], m0 + mova [t3+wq*4+400*4+ 0], m1 + mova [t3+wq*4+400*4+16], m2 + movq m0, [dstq+wq] + punpcklbw m0, m6 + punpcklwd m1, m0, m6 ; src + punpcklwd m2, m3, m6 ; a + pmaddwd m2, m1 ; a * src + punpckhwd m1, m0, m6 + punpckhwd m3, m6 + pmaddwd m3, m1 + psubd m4, m2 ; b - a * src + (1 << 8) + psubd m5, m3 + psrad m4, 9 + psrad m5, 9 + packssdw m4, m5 + pmulhrsw m4, m7 + paddw m0, m4 + packuswb m0, m0 + movq [dstq+wq], m0 + add wq, 8 + jl .n0_loop + add dstq, dst_stridemp + ret +ALIGN function_align +.n1: ; neighbor + output (odd rows) + movif64 wq, r5 + movif32 wd, w1m +.n1_loop: + movq m0, [dstq+wq] + mova m3, [t4+wq*2+400*2+ 0] + mova m4, [t3+wq*4+400*4+ 0] + mova m5, [t3+wq*4+400*4+16] + punpcklbw m0, m6 + punpcklwd m1, m0, m6 ; src + punpcklwd m2, m3, m6 ; a + pmaddwd m2, m1 ; a * src + punpckhwd m1, m0, m6 + punpckhwd m3, m6 + pmaddwd m3, m1 + psubd m4, m2 ; b - a * src + (1 << 7) + psubd m5, m3 + psrad m4, 8 + psrad m5, 8 + packssdw m4, m5 + pmulhrsw m4, m7 + paddw m0, m4 + packuswb m0, m0 + movq [dstq+wq], m0 + add wq, 8 + jl .n1_loop + add dstq, dst_stridemp + movif32 dstm, dstq + ret %if ARCH_X86_32 - pxor m13, m13 + %if STACK_ALIGNMENT < 16 + %assign extra_stack 4*16 + %else + %assign extra_stack 2*16 + %endif +cglobal sgr_filter_3x3_8bpc, 1, 7, 8, -400*42-16-extra_stack, \ + dst, dst_stride, left, lpf, lpf_stride, w, params, h + %if STACK_ALIGNMENT < 16 + %define dstm dword [esp+calloff+16*2+4*0] + %define dst_stridemp dword [esp+calloff+16*2+4*1] + %define leftm dword [esp+calloff+16*2+4*2] + %define lpfm dword [esp+calloff+16*2+4*3] + %define lpf_stridem dword [esp+calloff+16*2+4*4] + %define w0m dword [esp+calloff+16*2+4*5] + %define hd dword [esp+calloff+16*2+4*6] + %define edgeb byte [esp+calloff+16*2+4*7] + %define edged dword [esp+calloff+16*2+4*7] + %define leftmp leftm + %else + %define w0m wm + %define hd dword r6m + %define edgeb byte r8m + %define edged dword r8m + %endif + %define hvsrcm dword [esp+calloff+4*0] + %define w1m dword [esp+calloff+4*1] + %define t3m dword [esp+calloff+4*2] + %define t4m dword [esp+calloff+4*3] + %define m8 [base+pb_0to15] + %define m9 [esp+calloff+16*1] + %define m10 [base+pd_0xf00801c7] + %define m11 [base+pd_34816] + %define m12 m6 + %define m13 [base+sgr_lshuf3] + %define base r6-$$ + %assign calloff 0 + %if STACK_ALIGNMENT < 16 + mov dst_strideq, [rstk+stack_offset+ 8] + mov leftq, [rstk+stack_offset+12] + mov lpfq, [rstk+stack_offset+16] + mov lpf_strideq, [rstk+stack_offset+20] + mov wd, [rstk+stack_offset+24] + mov dstm, dstq + mov dst_stridemp, dst_strideq + mov leftm, leftq + mov r1, [rstk+stack_offset+28] + mov r2, [rstk+stack_offset+36] + mov lpfm, lpfq + mov lpf_stridem, lpf_strideq + mov hd, r1 + mov edged, r2 + %endif +%else +cglobal sgr_filter_3x3_8bpc, 5, 15, 14, -400*42-8, dst, dst_stride, left, lpf, \ + lpf_stride, w, edge, params, h +%endif +%if ARCH_X86_64 || STACK_ALIGNMENT >= 16 + mov wd, wm +%endif +%if ARCH_X86_64 + mov paramsq, paramsmp + lea r13, [sgr_x_by_x-0xf03] + mov edged, r8m + mov hd, r6m + movq m9, [paramsq+4] + add lpfq, wq + lea t1, [rsp+wq*2+12] + mova m8, [pb_0to15] + add dstq, wq + lea t3, [rsp+wq*4+400*12+8] + mova m10, [pd_0xf00801c7] + lea t4, [rsp+wq*2+400*32+8] + mova m11, [pd_34816] + pshuflw m7, m9, q3333 + pshufb m9, [pw_256] ; s1 + punpcklqdq m7, m7 ; w1 + neg wq + pxor m6, m6 + mova m13, [sgr_lshuf3] + psllw m7, 4 + DEFINE_ARGS dst, dst_stride, left, lpf, lpf_stride, _, edge, _, h, _, w + %define lpfm [rsp] +%else + mov r1, [rstk+stack_offset+32] ; params + LEA r6, $$ + movq m1, [r1+4] + add lpfm, wq + lea t1, [rsp+extra_stack+wq*2+20] + add dstq, wq + lea t3, [rsp+extra_stack+wq*4+400*12+16] + mov dstm, dstq + lea t4, [rsp+extra_stack+wq*2+400*32+16] + mov t3m, t3 + pshuflw m7, m1, q3333 + mov t4m, t4 + pshufb m1, [base+pw_256] ; s1 + punpcklqdq m7, m7 ; w1 + psllw m7, 4 + neg wq + mova m9, m1 + pxor m6, m6 + mov w1m, wd + sub wd, 2 + mov lpfq, lpfm + mov lpf_strideq, lpf_stridem + mov w0m, wd +%endif + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, lpf_strideq + mov t2, t1 + add t1, 400*6 + call .h_top + lea r10, [lpfq+lpf_strideq*4] + mov lpfq, dstq + add r10, lpf_strideq + mov lpfm, r10 ; below + movif32 t4, t4m + call .hv0 +.main: + dec hd + jz .height1 + movif32 lpfq, hvsrcm + add lpfq, dst_stridemp + call .hv1 + call .prep_n + sub hd, 2 + jl .extend_bottom +.main_loop: + movif32 lpfq, hvsrcm + add lpfq, dst_stridemp + call .hv0 +%if ARCH_X86_64 + test hb, hb +%else + mov r5, hd + test r5, r5 %endif - movq m8, [srcq+strideq] - punpcklbw m8, m13 - punpcklwd m0, m8, m10 - punpckhwd m8, m10 - punpcklwd m1, m6, m11 - punpckhwd m2, m6, m11 - pmaddwd m0, m1 - pmaddwd m8, m2 + jz .odd_height + movif32 lpfq, hvsrcm + add lpfq, dst_stridemp + call .hv1 + call .n0 + call .n1 + sub hd, 2 + jge .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .extend_bottom + mov lpfq, lpfm + call .hv0_bottom +%if ARCH_X86_64 + add lpfq, lpf_strideq +%else + mov lpfq, hvsrcm + add lpfq, lpf_stridem +%endif + call .hv1_bottom +.end: + call .n0 + call .n1 +.end2: + RET +.height1: + call .v1 + call .prep_n + jmp .odd_height_end +.odd_height: + call .v1 + call .n0 + call .n1 +.odd_height_end: + call .v0 + call .v1 + call .n0 + jmp .end2 +.extend_bottom: + call .v0 + call .v1 + jmp .end +.no_top: + lea r10, [lpfq+lpf_strideq*4] + mov lpfq, dstq + lea r10, [r10+lpf_strideq*2] + mov lpfm, r10 + call .h %if ARCH_X86_64 - paddd m0, m5 + lea wq, [r5-2] +%else + mov wq, w0m + mov hvsrcm, lpfq +%endif + lea t2, [t1+400*6] +.top_fixup_loop: + mova m0, [t1+wq*2+400*0] + mova m1, [t1+wq*2+400*2] + mova m2, [t1+wq*2+400*4] + mova [t2+wq*2+400*0], m0 + mova [t2+wq*2+400*2], m1 + mova [t2+wq*2+400*4], m2 + add wq, 8 + jl .top_fixup_loop + movif32 t3, t3m + movif32 t4, t4m + call .v0 + jmp .main +.extend_right: +%assign stack_offset stack_offset+8 +%assign calloff 8 + movd m0, [lpfq-1] + movd m1, wd + mova m3, m8 + pshufb m0, m6 + pshufb m1, m6 + mova m2, m6 + psubb m2, m1 + pcmpgtb m2, m3 + pand m5, m2 + pandn m2, m0 + por m5, m2 + ret +%assign stack_offset stack_offset-4 +%assign calloff 4 +.h: ; horizontal boxsum +%if ARCH_X86_64 + lea wq, [r5-2] +%else + %define leftq r5 +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movif32 leftq, leftm + movddup m4, [leftq-4] + movif32 wq, w0m + mova m5, [lpfq+wq+2] + add leftmp, 4 + palignr m5, m4, 14 + jmp .h_main +.h_extend_left: + movif32 wq, w0m + mova m5, [lpfq+wq+2] + pshufb m5, m13 + jmp .h_main +.h_top: +%if ARCH_X86_64 + lea wq, [r5-2] +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movif32 wq, w0m +.h_loop: + movu m5, [lpfq+wq] +.h_main: + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .h_have_right + cmp wd, -9 + jl .h_have_right + call .extend_right +.h_have_right: + punpcklbw m4, m5, m6 + punpckhbw m5, m6 + palignr m0, m5, m4, 2 + paddw m1, m4, m0 + punpcklwd m2, m4, m0 + pmaddwd m2, m2 + punpckhwd m3, m4, m0 + pmaddwd m3, m3 + palignr m5, m4, 4 + paddw m1, m5 ; sum + punpcklwd m4, m5, m6 + pmaddwd m4, m4 + punpckhwd m5, m6 + pmaddwd m5, m5 + paddd m2, m4 ; sumsq + paddd m3, m5 + mova [t1+wq*2+400*0], m1 + mova [t1+wq*2+400*2], m2 + mova [t1+wq*2+400*4], m3 + add wq, 8 + jl .h_loop + ret +ALIGN function_align +.hv0: ; horizontal boxsum + vertical boxsum + ab (even rows) +%if ARCH_X86_64 + lea wq, [r5-2] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv0_extend_left + movif32 leftq, leftm + movddup m4, [leftq-4] + movif32 wq, w0m + mova m5, [lpfq+wq+2] + add leftmp, 4 + palignr m5, m4, 14 + jmp .hv0_main +.hv0_extend_left: + movif32 wq, w0m + mova m5, [lpfq+wq+2] + pshufb m5, m13 + jmp .hv0_main +.hv0_bottom: +%if ARCH_X86_64 + lea wq, [r5-2] %else - paddd m0, [td] + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv0_extend_left + movif32 wq, w0m +%if ARCH_X86_32 + jmp .hv0_loop_start +%endif +.hv0_loop: + movif32 lpfq, hvsrcm +.hv0_loop_start: + movu m5, [lpfq+wq] +.hv0_main: + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv0_have_right + cmp wd, -9 + jl .hv0_have_right + call .extend_right +.hv0_have_right: + punpcklbw m4, m5, m6 + punpckhbw m5, m6 + palignr m0, m5, m4, 2 + paddw m1, m4, m0 + punpcklwd m2, m4, m0 + pmaddwd m2, m2 + punpckhwd m3, m4, m0 + pmaddwd m3, m3 + palignr m5, m4, 4 + paddw m1, m5 ; sum + punpcklwd m4, m5, m6 + pmaddwd m4, m4 + punpckhwd m5, m6 + pmaddwd m5, m5 + paddd m2, m4 ; sumsq + paddd m3, m5 + paddw m0, m1, [t1+wq*2+400*0] + paddd m4, m2, [t1+wq*2+400*2] + paddd m5, m3, [t1+wq*2+400*4] + mova [t1+wq*2+400*0], m1 + mova [t1+wq*2+400*2], m2 + mova [t1+wq*2+400*4], m3 + paddw m1, m0, [t2+wq*2+400*0] + paddd m2, m4, [t2+wq*2+400*2] + paddd m3, m5, [t2+wq*2+400*4] + mova [t2+wq*2+400*0], m0 + mova [t2+wq*2+400*2], m4 + mova [t2+wq*2+400*4], m5 + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; a * 9 + paddd m5, m3 + punpcklwd m0, m1, m6 ; b + pmaddwd m2, m0, m0 ; b * b + punpckhwd m1, m6 + pmaddwd m3, m1, m1 + psubd m4, m2 ; p + psubd m5, m3 + MULLD m4, m9, m12 ; p * s + MULLD m5, m9, m12 + pmaddwd m0, m10 ; b * 455 + pmaddwd m1, m10 + paddusw m4, m10 + paddusw m5, m10 + psrld m4, 20 ; min(z, 255) + movif32 t3, t3m + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, r0, dstm + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m12 + MULLD m1, m5, m12 +%if ARCH_X86_32 + pxor m6, m6 +%endif + paddd m0, m11 ; x * b * 455 + (1 << 11) + (1 << 15) + paddd m1, m11 + mova [t4+wq*2+4], m3 + psrld m0, 12 + psrld m1, 12 + mova [t3+wq*4+ 8], m0 + mova [t3+wq*4+24], m1 + add wq, 8 + jl .hv0_loop + ret +ALIGN function_align +.hv1: ; horizontal boxsums + vertical boxsums + ab (odd rows) +%if ARCH_X86_64 + lea wq, [r5-2] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv1_extend_left + movif32 leftq, leftm + movddup m4, [leftq-4] + movif32 wq, w0m + mova m5, [lpfq+wq+2] + add leftmp, 4 + palignr m5, m4, 14 + jmp .hv1_main +.hv1_extend_left: + movif32 wq, w0m + mova m5, [lpfq+wq+2] + pshufb m5, m13 + jmp .hv1_main +.hv1_bottom: +%if ARCH_X86_64 + lea wq, [r5-2] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv1_extend_left + movif32 wq, w0m +%if ARCH_X86_32 + jmp .hv1_loop_start +%endif +.hv1_loop: + movif32 lpfq, hvsrcm +.hv1_loop_start: + movu m5, [lpfq+wq] +.hv1_main: + test edgeb, 2 ; LR_HAVE_RIGHT + jnz .hv1_have_right + cmp wd, -9 + jl .hv1_have_right + call .extend_right +.hv1_have_right: + punpcklbw m4, m5, m6 + punpckhbw m5, m6 + palignr m1, m5, m4, 2 + paddw m0, m4, m1 + punpcklwd m2, m4, m1 + pmaddwd m2, m2 + punpckhwd m3, m4, m1 + pmaddwd m3, m3 + palignr m5, m4, 4 + paddw m0, m5 ; h sum + punpcklwd m1, m5, m6 + pmaddwd m1, m1 + punpckhwd m5, m6 + pmaddwd m5, m5 + paddd m2, m1 ; h sumsq + paddd m3, m5 + paddw m1, m0, [t2+wq*2+400*0] + paddd m4, m2, [t2+wq*2+400*2] + paddd m5, m3, [t2+wq*2+400*4] + mova [t2+wq*2+400*0], m0 + mova [t2+wq*2+400*2], m2 + mova [t2+wq*2+400*4], m3 + pslld m2, m4, 3 + pslld m3, m5, 3 + paddd m4, m2 ; a * 9 + paddd m5, m3 + punpcklwd m0, m1, m6 ; b + pmaddwd m2, m0, m0 ; b * b + punpckhwd m1, m6 + pmaddwd m3, m1, m1 + psubd m4, m2 ; p + psubd m5, m3 + MULLD m4, m9, m12 ; p * s + MULLD m5, m9, m12 + pmaddwd m0, m10 ; b * 455 + pmaddwd m1, m10 + paddusw m4, m10 + paddusw m5, m10 + psrld m4, 20 ; min(z, 255) + movif32 t3, t3m + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, r0, dstm + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m12 + MULLD m1, m5, m12 +%if ARCH_X86_32 + pxor m6, m6 +%endif + paddd m0, m11 ; x * b * 455 + (1 << 11) + (1 << 15) + paddd m1, m11 + mova [t4+wq*2+400*2 +4], m3 + psrld m0, 12 + psrld m1, 12 + mova [t3+wq*4+400*4+ 8], m0 + mova [t3+wq*4+400*4+24], m1 + add wq, 8 + jl .hv1_loop + mov r10, t2 + mov t2, t1 + mov t1, r10 + ret +.v0: ; vertical boxsums + ab (even rows) +%if ARCH_X86_64 + lea wq, [r5-2] +%else + mov wd, w0m %endif - paddd m8, m4 - psrad m0, 8 - psrad m8, 8 - - packssdw m7, m3 - packssdw m0, m8 +.v0_loop: + mova m0, [t1+wq*2+400*0] + mova m4, [t1+wq*2+400*2] + mova m5, [t1+wq*2+400*4] + paddw m0, m0 + paddd m4, m4 + paddd m5, m5 + paddw m1, m0, [t2+wq*2+400*0] + paddd m2, m4, [t2+wq*2+400*2] + paddd m3, m5, [t2+wq*2+400*4] + mova [t2+wq*2+400*0], m0 + mova [t2+wq*2+400*2], m4 + mova [t2+wq*2+400*4], m5 + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; a * 9 + paddd m5, m3 + punpcklwd m0, m1, m6 ; b + pmaddwd m2, m0, m0 ; b * b + punpckhwd m1, m6 + pmaddwd m3, m1, m1 + psubd m4, m2 ; p + psubd m5, m3 + MULLD m4, m9, m12 ; p * s + MULLD m5, m9, m12 + pmaddwd m0, m10 ; b * 455 + pmaddwd m1, m10 + paddusw m4, m10 + paddusw m5, m10 + psrld m4, 20 ; min(z, 255) + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, r0, dstm + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m12 + MULLD m1, m5, m12 +%if ARCH_X86_32 + pxor m6, m6 +%endif + paddd m0, m11 ; x * b * 455 + (1 << 11) + (1 << 15) + paddd m1, m11 + mova [t4+wq*2+4], m3 + psrld m0, 12 + psrld m1, 12 + mova [t3+wq*4+ 8], m0 + mova [t3+wq*4+24], m1 + add wq, 8 + jl .v0_loop + ret +.v1: ; vertical boxsums + ab (odd rows) +%if ARCH_X86_64 + lea wq, [r5-2] +%else + mov wd, w0m +%endif +.v1_loop: + mova m0, [t1+wq*2+400*0] + mova m4, [t1+wq*2+400*2] + mova m5, [t1+wq*2+400*4] + paddw m1, m0, [t2+wq*2+400*0] + paddd m2, m4, [t2+wq*2+400*2] + paddd m3, m5, [t2+wq*2+400*4] + mova [t2+wq*2+400*0], m0 + mova [t2+wq*2+400*2], m4 + mova [t2+wq*2+400*4], m5 + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; a * 9 + paddd m5, m3 + punpcklwd m0, m1, m6 ; b + pmaddwd m2, m0, m0 ; b * b + punpckhwd m1, m6 + pmaddwd m3, m1, m1 + psubd m4, m2 ; p + psubd m5, m3 + MULLD m4, m9, m12 ; p * s + MULLD m5, m9, m12 + pmaddwd m0, m10 ; b * 455 + pmaddwd m1, m10 + paddusw m4, m10 + paddusw m5, m10 + psrld m4, 20 ; min(z, 255) + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, r0, dstm + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m12 + MULLD m1, m5, m12 %if ARCH_X86_32 - mova m5, [td] + pxor m6, m6 %endif - mova [tq+384*2*0], m7 - mova [tq+384*2*1], m0 + paddd m0, m11 ; x * b * 455 + (1 << 11) + (1 << 15) + paddd m1, m11 + mova [t4+wq*2+400*2+ 4], m3 + psrld m0, 12 + psrld m1, 12 + mova [t3+wq*4+400*4+ 8], m0 + mova [t3+wq*4+400*4+24], m1 + add wq, 8 + jl .v1_loop + mov r10, t2 + mov t2, t1 + mov t1, r10 + ret +.prep_n: ; initial neighbor setup + movif64 wq, r5 + movif32 wd, w1m +.prep_n_loop: + movu m0, [t4+wq*2+400*0+ 4] + movu m1, [t3+wq*4+400*0+ 8] + movu m2, [t3+wq*4+400*0+24] + movu m3, [t4+wq*2+400*0+ 2] + movu m4, [t3+wq*4+400*0+ 4] + movu m5, [t3+wq*4+400*0+20] + paddw m0, [t4+wq*2+400*0+ 0] + paddd m1, [t3+wq*4+400*0+ 0] + paddd m2, [t3+wq*4+400*0+16] + paddw m3, m0 + paddd m4, m1 + paddd m5, m2 + psllw m3, 2 ; a[-1] 444 + pslld m4, 2 ; b[-1] 444 + pslld m5, 2 + psubw m3, m0 ; a[-1] 343 + psubd m4, m1 ; b[-1] 343 + psubd m5, m2 + mova [t4+wq*2+400*4], m3 + mova [t3+wq*4+400*8+ 0], m4 + mova [t3+wq*4+400*8+16], m5 + movu m0, [t4+wq*2+400*2+ 4] + movu m1, [t3+wq*4+400*4+ 8] + movu m2, [t3+wq*4+400*4+24] + movu m3, [t4+wq*2+400*2+ 2] + movu m4, [t3+wq*4+400*4+ 4] + movu m5, [t3+wq*4+400*4+20] + paddw m0, [t4+wq*2+400*2+ 0] + paddd m1, [t3+wq*4+400*4+ 0] + paddd m2, [t3+wq*4+400*4+16] + paddw m3, m0 + paddd m4, m1 + paddd m5, m2 + psllw m3, 2 ; a[ 0] 444 + pslld m4, 2 ; b[ 0] 444 + pslld m5, 2 + mova [t4+wq*2+400* 6], m3 + mova [t3+wq*4+400*12+ 0], m4 + mova [t3+wq*4+400*12+16], m5 + psubw m3, m0 ; a[ 0] 343 + psubd m4, m1 ; b[ 0] 343 + psubd m5, m2 + mova [t4+wq*2+400* 8], m3 + mova [t3+wq*4+400*16+ 0], m4 + mova [t3+wq*4+400*16+16], m5 + add wq, 8 + jl .prep_n_loop + ret +ALIGN function_align +.n0: ; neighbor + output (even rows) + movif64 wq, r5 + movif32 wd, w1m +.n0_loop: + movu m3, [t4+wq*2+400*0+4] + movu m1, [t4+wq*2+400*0+2] + paddw m3, [t4+wq*2+400*0+0] + paddw m1, m3 + psllw m1, 2 ; a[ 1] 444 + psubw m2, m1, m3 ; a[ 1] 343 + paddw m3, m2, [t4+wq*2+400*4] + paddw m3, [t4+wq*2+400*6] + mova [t4+wq*2+400*4], m2 + mova [t4+wq*2+400*6], m1 + movu m4, [t3+wq*4+400*0+8] + movu m1, [t3+wq*4+400*0+4] + paddd m4, [t3+wq*4+400*0+0] + paddd m1, m4 + pslld m1, 2 ; b[ 1] 444 + psubd m2, m1, m4 ; b[ 1] 343 + paddd m4, m2, [t3+wq*4+400* 8+ 0] + paddd m4, [t3+wq*4+400*12+ 0] + mova [t3+wq*4+400* 8+ 0], m2 + mova [t3+wq*4+400*12+ 0], m1 + movu m5, [t3+wq*4+400*0+24] + movu m1, [t3+wq*4+400*0+20] + paddd m5, [t3+wq*4+400*0+16] + paddd m1, m5 + pslld m1, 2 + psubd m2, m1, m5 + paddd m5, m2, [t3+wq*4+400* 8+16] + paddd m5, [t3+wq*4+400*12+16] + mova [t3+wq*4+400* 8+16], m2 + mova [t3+wq*4+400*12+16], m1 + movq m0, [dstq+wq] + punpcklbw m0, m6 + punpcklwd m1, m0, m6 + punpcklwd m2, m3, m6 + pmaddwd m2, m1 ; a * src + punpckhwd m1, m0, m6 + punpckhwd m3, m6 + pmaddwd m3, m1 + psubd m4, m2 ; b - a * src + (1 << 8) + psubd m5, m3 + psrad m4, 9 + psrad m5, 9 + packssdw m4, m5 + pmulhrsw m4, m7 + paddw m0, m4 + packuswb m0, m0 + movq [dstq+wq], m0 + add wq, 8 + jl .n0_loop + add dstq, dst_stridemp + ret +ALIGN function_align +.n1: ; neighbor + output (odd rows) + movif64 wq, r5 + movif32 wd, w1m +.n1_loop: + movu m3, [t4+wq*2+400*2+4] + movu m1, [t4+wq*2+400*2+2] + paddw m3, [t4+wq*2+400*2+0] + paddw m1, m3 + psllw m1, 2 ; a[ 1] 444 + psubw m2, m1, m3 ; a[ 1] 343 + paddw m3, m2, [t4+wq*2+400*6] + paddw m3, [t4+wq*2+400*8] + mova [t4+wq*2+400*6], m1 + mova [t4+wq*2+400*8], m2 + movu m4, [t3+wq*4+400*4+8] + movu m1, [t3+wq*4+400*4+4] + paddd m4, [t3+wq*4+400*4+0] + paddd m1, m4 + pslld m1, 2 ; b[ 1] 444 + psubd m2, m1, m4 ; b[ 1] 343 + paddd m4, m2, [t3+wq*4+400*12+ 0] + paddd m4, [t3+wq*4+400*16+ 0] + mova [t3+wq*4+400*12+ 0], m1 + mova [t3+wq*4+400*16+ 0], m2 + movu m5, [t3+wq*4+400*4+24] + movu m1, [t3+wq*4+400*4+20] + paddd m5, [t3+wq*4+400*4+16] + paddd m1, m5 + pslld m1, 2 + psubd m2, m1, m5 + paddd m5, m2, [t3+wq*4+400*12+16] + paddd m5, [t3+wq*4+400*16+16] + mova [t3+wq*4+400*12+16], m1 + mova [t3+wq*4+400*16+16], m2 + movq m0, [dstq+wq] + punpcklbw m0, m6 + punpcklwd m1, m0, m6 + punpcklwd m2, m3, m6 + pmaddwd m2, m1 ; a * src + punpckhwd m1, m0, m6 + punpckhwd m3, m6 + pmaddwd m3, m1 + psubd m4, m2 ; b - a * src + (1 << 8) + psubd m5, m3 + psrad m4, 9 + psrad m5, 9 + packssdw m4, m5 + pmulhrsw m4, m7 + paddw m0, m4 + packuswb m0, m0 + movq [dstq+wq], m0 + add wq, 8 + jl .n1_loop + add dstq, dst_stridemp + movif32 dstm, dstq + ret - mova m0, m5 - mova m1, m4 - mova m2, m6 - add aq, (384+16)*4*2 - add bq, (384+16)*2*2 - add tq, 384*2*2 - lea srcq, [srcq+strideq*2] -%if ARCH_X86_64 - sub yd, 2 +%if ARCH_X86_32 + %if STACK_ALIGNMENT < 16 + %assign extra_stack 10*16 + %else + %assign extra_stack 8*16 + %endif +cglobal sgr_filter_mix_8bpc, 1, 7, 8, -400*66-48-extra_stack, \ + dst, dst_stride, left, lpf, lpf_stride, w, params, h + %if STACK_ALIGNMENT < 16 + %define dstm dword [esp+calloff+16*8+4*0] + %define dst_stridemp dword [esp+calloff+16*8+4*1] + %define leftm dword [esp+calloff+16*8+4*2] + %define lpfm dword [esp+calloff+16*8+4*3] + %define lpf_stridem dword [esp+calloff+16*8+4*4] + %define w0m dword [esp+calloff+16*8+4*5] + %define hd dword [esp+calloff+16*8+4*6] + %define edgeb byte [esp+calloff+16*8+4*7] + %define edged dword [esp+calloff+16*8+4*7] + %define leftmp leftm + %else + %define w0m wm + %define hd dword r6m + %define edgeb byte r8m + %define edged dword r8m + %endif + %define hvsrcm dword [esp+calloff+4*0] + %define w1m dword [esp+calloff+4*1] + %define t3m dword [esp+calloff+4*2] + %define t4m dword [esp+calloff+4*3] + %xdefine m8 m6 + %define m9 [base+pd_0xffff] + %define m10 [base+pd_34816] + %define m11 [base+pd_0xf00801c7] + %define m12 [base+pd_0xf00800a4] + %define m13 [esp+calloff+16*4] + %define m14 [esp+calloff+16*5] + %define m15 [esp+calloff+16*6] + %define m6 [esp+calloff+16*7] + %define base r6-$$ + %assign calloff 0 + %if STACK_ALIGNMENT < 16 + mov dst_strideq, [rstk+stack_offset+ 8] + mov leftq, [rstk+stack_offset+12] + mov lpfq, [rstk+stack_offset+16] + mov lpf_strideq, [rstk+stack_offset+20] + mov wd, [rstk+stack_offset+24] + mov dstm, dstq + mov dst_stridemp, dst_strideq + mov leftm, leftq + mov r1, [rstk+stack_offset+28] + mov r2, [rstk+stack_offset+36] + mov lpfm, lpfq + mov lpf_stridem, lpf_strideq + mov hd, r1 + mov edged, r2 + %endif %else - sub dword [esp+4], 2 +cglobal sgr_filter_mix_8bpc, 5, 15, 16, -400*66-40, dst, dst_stride, left, \ + lpf, lpf_stride, w, edge, \ + params, h +%endif +%if ARCH_X86_64 || STACK_ALIGNMENT >= 16 + mov wd, wm %endif - jg .loop_y - add xd, 8 - cmp xd, wd - jl .loop_x -%if ARCH_X86_32 - ADD esp, 8 +%if ARCH_X86_64 + mov paramsq, paramsmp + lea r13, [sgr_x_by_x-0xf03] + mov edged, r8m + mov hd, r6m + mova m15, [paramsq] + add lpfq, wq + mova m9, [pd_0xffff] + lea t1, [rsp+wq*2+44] + mova m10, [pd_34816] + add dstq, wq + lea t3, [rsp+wq*4+400*24+40] + mova m11, [pd_0xf00801c7] + lea t4, [rsp+wq*2+400*52+40] + mova m12, [base+pd_0xf00800a4] + neg wq + pshuflw m13, m15, q0000 + pshuflw m14, m15, q2222 + pshufhw m15, m15, q1010 + punpcklqdq m13, m13 ; s0 + punpcklqdq m14, m14 ; s1 + punpckhqdq m15, m15 ; w0 w1 + pxor m6, m6 + psllw m15, 2 + DEFINE_ARGS dst, dst_stride, left, lpf, lpf_stride, _, edge, _, h, _, w + %define lpfm [rsp] +%else + mov r1, [rstk+stack_offset+32] ; params + LEA r6, $$ + mova m2, [r1] + add lpfm, wq + lea t1, [rsp+extra_stack+wq*2+52] + add dstq, wq + lea t3, [rsp+extra_stack+wq*4+400*24+48] + mov dstm, dstq + lea t4, [rsp+extra_stack+wq*2+400*52+48] + mov t3m, t3 + mov t4m, t4 + neg wq + pshuflw m0, m2, q0000 + pshuflw m1, m2, q2222 + pshufhw m2, m2, q1010 + punpcklqdq m0, m0 ; s0 + punpcklqdq m1, m1 ; s1 + punpckhqdq m2, m2 ; w0 w1 + mov w1m, wd + pxor m3, m3 + psllw m2, 2 + mova m13, m0 + mova m14, m1 + sub wd, 2 + mova m15, m2 + mova m6, m3 + mov lpfq, lpfm + mov lpf_strideq, lpf_stridem + mov w0m, wd %endif - RET - -%undef t2 -cglobal sgr_weighted2_8bpc, 4, 7, 12, dst, stride, t1, t2, w, h, wt - movifnidn wd, wm - movd m0, wtm + test edgeb, 4 ; LR_HAVE_TOP + jz .no_top + call .h_top + add lpfq, lpf_strideq + mov t2, t1 %if ARCH_X86_64 - movifnidn hd, hm - mova m10, [pd_1024] - pxor m11, m11 + call mangle(private_prefix %+ _sgr_filter_5x5_8bpc_ssse3).top_fixup +%else + mov wq, w0m + call mangle(private_prefix %+ _sgr_filter_5x5_8bpc_ssse3).top_fixup_loop +%endif + add t1, 400*12 + call .h_top + lea r10, [lpfq+lpf_strideq*4] + mov lpfq, dstq + add r10, lpf_strideq + mov lpfm, r10 ; below + movif32 t4, t4m + call .hv0 +.main: + dec hd + jz .height1 + movif32 lpfq, hvsrcm + add lpfq, dst_stridemp + call .hv1 + call .prep_n + sub hd, 2 + jl .extend_bottom +.main_loop: + movif32 lpfq, hvsrcm + add lpfq, dst_stridemp + call .hv0 +%if ARCH_X86_64 + test hd, hd +%else + mov r5, hd + test r5, r5 +%endif + jz .odd_height + movif32 lpfq, hvsrcm + add lpfq, dst_stridemp + call .hv1 + call .n0 + call .n1 + sub hd, 2 + jge .main_loop + test edgeb, 8 ; LR_HAVE_BOTTOM + jz .extend_bottom + mov lpfq, lpfm + call .hv0_bottom +%if ARCH_X86_64 + add lpfq, lpf_strideq %else - SETUP_PIC hd, 0 - %define m10 [PIC_sym(pd_1024)] - %define m11 m7 + mov lpfq, hvsrcm + add lpfq, lpf_stridem +%endif + call .hv1_bottom +.end: + call .n0 + call .n1 +.end2: + RET +.height1: + call .v1 + call .prep_n + jmp .odd_height_end +.odd_height: + call .v1 + call .n0 + call .n1 +.odd_height_end: + call .v0 + call .v1 + call .n0 + jmp .end2 +.extend_bottom: + call .v0 + call .v1 + jmp .end +.no_top: + lea r10, [lpfq+lpf_strideq*4] + mov lpfq, dstq + lea r10, [r10+lpf_strideq*2] + mov lpfm, r10 + call .h +%if ARCH_X86_64 + lea wq, [r5-2] +%else + mov wq, w0m + mov hvsrcm, lpfq +%endif + lea t2, [t1+400*12] +.top_fixup_loop: + mova m0, [t1+wq*2+400* 0] + mova m1, [t1+wq*2+400* 2] + mova m2, [t1+wq*2+400* 4] + paddw m0, m0 + mova m3, [t1+wq*2+400* 6] + paddd m1, m1 + mova m4, [t1+wq*2+400* 8] + paddd m2, m2 + mova m5, [t1+wq*2+400*10] + mova [t2+wq*2+400* 0], m0 + mova [t2+wq*2+400* 2], m1 + mova [t2+wq*2+400* 4], m2 + mova [t2+wq*2+400* 6], m3 + mova [t2+wq*2+400* 8], m4 + mova [t2+wq*2+400*10], m5 + add wq, 8 + jl .top_fixup_loop + movif32 t3, t3m + movif32 t4, t4m + call .v0 + jmp .main +.extend_right: +%assign stack_offset stack_offset+8 +%assign calloff 8 +%if ARCH_X86_64 + SWAP m8, m6 +%endif + movd m1, wd + movd m3, [lpfq-1] + pshufb m1, m8 + pshufb m3, m8 + psubb m2, [base+pb_1], m1 + pcmpgtb m2, [base+pb_0to15] + pand m5, m2 + pandn m2, m3 + por m5, m2 +%if ARCH_X86_64 + SWAP m6, m8 +%endif + ret +%assign stack_offset stack_offset-4 +%assign calloff 4 +.h: ; horizontal boxsum +%if ARCH_X86_64 + lea wq, [r5-2] +%else + %define leftq r5 +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movif32 leftq, leftm + movddup m4, [leftq-4] + movif32 wq, w0m + mova m5, [lpfq+wq+2] + add leftmp, 4 + palignr m5, m4, 13 + jmp .h_main +.h_extend_left: + movif32 wq, w0m + mova m5, [lpfq+wq+2] + pshufb m5, [base+sgr_lshuf5] + jmp .h_main +.h_top: +%if ARCH_X86_64 + lea wq, [r5-2] %endif - pshufd m0, m0, 0 - DEFINE_ARGS dst, stride, t1, t2, w, h, idx + test edgeb, 1 ; LR_HAVE_LEFT + jz .h_extend_left + movif32 wq, w0m +.h_loop: + movu m5, [lpfq+wq-1] +.h_main: + test edgeb, 2 ; LR_HAVE_RIGHT %if ARCH_X86_32 - %define hd hmp + pxor m8, m8 +%else + SWAP m8, m6 +%endif + jnz .h_have_right + cmp wd, -10 + jl .h_have_right + call .extend_right +.h_have_right: + punpcklbw m4, m5, m8 + punpckhbw m5, m8 + palignr m3, m5, m4, 2 + palignr m0, m5, m4, 4 + paddw m1, m3, m0 + punpcklwd m2, m3, m0 + pmaddwd m2, m2 + punpckhwd m3, m0 + pmaddwd m3, m3 + palignr m0, m5, m4, 6 + paddw m1, m0 ; sum3 + punpcklwd m7, m0, m8 + pmaddwd m7, m7 + punpckhwd m0, m8 + pmaddwd m0, m0 +%if ARCH_X86_64 + SWAP m6, m8 +%endif + paddd m2, m7 ; sumsq3 + palignr m5, m4, 8 + punpcklwd m7, m5, m4 + paddw m8, m4, m5 + pmaddwd m7, m7 + punpckhwd m5, m4 + pmaddwd m5, m5 + paddd m3, m0 + mova [t1+wq*2+400* 6], m1 + mova [t1+wq*2+400* 8], m2 + mova [t1+wq*2+400*10], m3 + paddw m8, m1 ; sum5 + paddd m7, m2 ; sumsq5 + paddd m5, m3 + mova [t1+wq*2+400* 0], m8 + mova [t1+wq*2+400* 2], m7 + mova [t1+wq*2+400* 4], m5 + add wq, 8 + jl .h_loop + ret +ALIGN function_align +.hv0: ; horizontal boxsum + vertical boxsum + ab3 (even rows) +%if ARCH_X86_64 + lea wq, [r5-2] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv0_extend_left + movif32 leftq, leftm + movddup m4, [leftq-4] + movif32 wq, w0m + mova m5, [lpfq+wq+2] + add leftmp, 4 + palignr m5, m4, 13 + jmp .hv0_main +.hv0_extend_left: + movif32 wq, w0m + mova m5, [lpfq+wq+2] + pshufb m5, [base+sgr_lshuf5] + jmp .hv0_main +.hv0_bottom: +%if ARCH_X86_64 + lea wq, [r5-2] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv0_extend_left + movif32 wq, w0m +%if ARCH_X86_32 + jmp .hv0_loop_start +%endif +.hv0_loop: + movif32 lpfq, hvsrcm +.hv0_loop_start: + movu m5, [lpfq+wq-1] +.hv0_main: + test edgeb, 2 ; LR_HAVE_RIGHT +%if ARCH_X86_32 + pxor m8, m8 +%else + SWAP m8, m6 +%endif + jnz .hv0_have_right + cmp wd, -10 + jl .hv0_have_right + call .extend_right +.hv0_have_right: + punpcklbw m4, m5, m8 + punpckhbw m5, m8 + palignr m3, m5, m4, 2 + palignr m0, m5, m4, 4 + movif32 t3, t3m + paddw m1, m3, m0 + punpcklwd m2, m3, m0 + pmaddwd m2, m2 + punpckhwd m3, m0 + pmaddwd m3, m3 + palignr m0, m5, m4, 6 + paddw m1, m0 ; h sum3 + punpcklwd m7, m0, m8 + pmaddwd m7, m7 + punpckhwd m0, m8 +%if ARCH_X86_64 + SWAP m6, m8 %endif - -.loop_y: - xor idxd, idxd -.loop_x: - mova m1, [t1q+idxq*2+ 0] - mova m2, [t1q+idxq*2+16] - mova m3, [t2q+idxq*2+ 0] - mova m4, [t2q+idxq*2+16] - mova m6, [dstq+idxq] + pmaddwd m0, m0 + paddd m2, m7 ; h sumsq3 + palignr m5, m4, 8 + punpcklwd m7, m5, m4 + paddw m8, m4, m5 + pmaddwd m7, m7 + punpckhwd m5, m4 + pmaddwd m5, m5 + paddd m3, m0 + paddw m8, m1 ; h sum5 + paddd m7, m2 ; h sumsq5 + paddd m5, m3 + mova [t3+wq*4+400*8+ 8], m8 + mova [t3+wq*4+400*0+ 8], m7 + mova [t3+wq*4+400*0+24], m5 + paddw m8, [t1+wq*2+400* 0] + paddd m7, [t1+wq*2+400* 2] + paddd m5, [t1+wq*2+400* 4] + mova [t1+wq*2+400* 0], m8 + mova [t1+wq*2+400* 2], m7 + mova [t1+wq*2+400* 4], m5 + paddw m0, m1, [t1+wq*2+400* 6] + paddd m4, m2, [t1+wq*2+400* 8] + paddd m5, m3, [t1+wq*2+400*10] + mova [t1+wq*2+400* 6], m1 + mova [t1+wq*2+400* 8], m2 + mova [t1+wq*2+400*10], m3 + paddw m1, m0, [t2+wq*2+400* 6] + paddd m2, m4, [t2+wq*2+400* 8] + paddd m3, m5, [t2+wq*2+400*10] + mova [t2+wq*2+400* 6], m0 + mova [t2+wq*2+400* 8], m4 + mova [t2+wq*2+400*10], m5 +%if ARCH_X86_32 + pxor m7, m7 +%else + SWAP m7, m6 +%endif + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; a3 * 9 + paddd m5, m3 + punpcklwd m0, m1, m7 ; b3 + pmaddwd m2, m0, m0 + punpckhwd m1, m7 + pmaddwd m3, m1, m1 +%if ARCH_X86_64 + SWAP m7, m6 +%endif + psubd m4, m2 ; p3 + psubd m5, m3 + MULLD m4, m14, m7 ; p3 * s1 + MULLD m5, m14, m7 + pmaddwd m0, m11 ; b3 * 455 + pmaddwd m1, m11 + paddusw m4, m11 + paddusw m5, m11 + psrld m4, 20 ; min(z3, 255) + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, r0, dstm + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m7 + MULLD m1, m5, m7 + paddd m0, m10 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + paddd m1, m10 + mova [t4+wq*2+400*2+ 4], m3 + psrld m0, 12 + psrld m1, 12 + mova [t3+wq*4+400*4+ 8], m0 + mova [t3+wq*4+400*4+24], m1 + add wq, 8 + jl .hv0_loop + ret +ALIGN function_align +.hv1: ; horizontal boxsums + vertical boxsums + ab (odd rows) +%if ARCH_X86_64 + lea wq, [r5-2] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv1_extend_left + movif32 leftq, leftm + movddup m4, [leftq-4] + movif32 wq, w0m + mova m5, [lpfq+wq+2] + add leftmp, 4 + palignr m5, m4, 13 + jmp .hv1_main +.hv1_extend_left: + movif32 wq, w0m + mova m5, [lpfq+wq+2] + pshufb m5, [base+sgr_lshuf5] + jmp .hv1_main +.hv1_bottom: +%if ARCH_X86_64 + lea wq, [r5-2] +%else + mov hvsrcm, lpfq +%endif + test edgeb, 1 ; LR_HAVE_LEFT + jz .hv1_extend_left + movif32 wq, w0m +%if ARCH_X86_32 + jmp .hv1_loop_start +%endif +.hv1_loop: + movif32 lpfq, hvsrcm +.hv1_loop_start: + movu m5, [lpfq+wq-1] +.hv1_main: + test edgeb, 2 ; LR_HAVE_RIGHT %if ARCH_X86_32 - pxor m11, m11 + pxor m8, m8 +%else + SWAP m8, m6 +%endif + jnz .hv1_have_right + cmp wd, -10 + jl .hv1_have_right + call .extend_right +.hv1_have_right: + punpcklbw m4, m5, m8 + punpckhbw m5, m8 + palignr m7, m5, m4, 2 + palignr m3, m5, m4, 4 + paddw m2, m7, m3 + punpcklwd m0, m7, m3 + pmaddwd m0, m0 + punpckhwd m7, m3 + pmaddwd m7, m7 + palignr m3, m5, m4, 6 + paddw m2, m3 ; h sum3 + punpcklwd m1, m3, m8 + pmaddwd m1, m1 + punpckhwd m3, m8 +%if ARCH_X86_64 + SWAP m6, m8 +%endif + pmaddwd m3, m3 + paddd m0, m1 ; h sumsq3 + palignr m5, m4, 8 + punpckhwd m1, m4, m5 + paddw m8, m4, m5 + pmaddwd m1, m1 + punpcklwd m4, m5 + pmaddwd m4, m4 + paddd m7, m3 + paddw m5, m2, [t2+wq*2+400* 6] + mova [t2+wq*2+400* 6], m2 + paddw m8, m2 ; h sum5 + paddd m2, m0, [t2+wq*2+400* 8] + paddd m3, m7, [t2+wq*2+400*10] + mova [t2+wq*2+400* 8], m0 + mova [t2+wq*2+400*10], m7 + paddd m4, m0 ; h sumsq5 + paddd m1, m7 + pslld m0, m2, 3 + pslld m7, m3, 3 + paddd m2, m0 ; a3 * 9 + paddd m3, m7 +%if ARCH_X86_32 + mova [esp+20], m8 + pxor m8, m8 +%else + SWAP m8, m6 +%endif + punpcklwd m0, m5, m8 ; b3 + pmaddwd m7, m0, m0 + punpckhwd m5, m8 + pmaddwd m8, m5, m5 + psubd m2, m7 ; p3 + psubd m3, m8 + MULLD m2, m14, m8 ; p3 * s1 + MULLD m3, m14, m8 + pmaddwd m0, m11 ; b3 * 455 + pmaddwd m5, m11 + paddusw m2, m11 + paddusw m3, m11 + psrld m2, 20 ; min(z3, 255) + movif32 t3, t3m + psrld m3, 20 + GATHER_X_BY_X m8, m2, m3, r0, dstm + punpcklwd m2, m8, m8 + punpckhwd m3, m8, m8 + MULLD m0, m2, m7 + MULLD m5, m3, m7 + paddd m0, m10 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + paddd m5, m10 + psrld m0, 12 + psrld m5, 12 + mova [t4+wq*2+400*4+ 4], m8 + mova [t3+wq*4+400*8+ 8], m0 + mova [t3+wq*4+400*8+24], m5 +%if ARCH_X86_32 + mova m8, [esp+20] +%else + SWAP m6, m8 + pxor m6, m6 +%endif + paddw m5, m8, [t2+wq*2+400*0] + paddd m2, m4, [t2+wq*2+400*2] + paddd m3, m1, [t2+wq*2+400*4] + paddw m5, [t1+wq*2+400*0] + paddd m2, [t1+wq*2+400*2] + paddd m3, [t1+wq*2+400*4] + mova [t2+wq*2+400*0], m8 + pslld m0, m2, 4 + mova [t2+wq*2+400*2], m4 + pslld m8, m3, 4 + mova [t2+wq*2+400*4], m1 + pslld m4, m2, 3 + paddd m2, m0 + pslld m7, m3, 3 + paddd m3, m8 + paddd m2, m4 ; a5 * 25 + paddd m3, m7 +%if ARCH_X86_32 + pxor m7, m7 +%else + SWAP m7, m6 +%endif + punpcklwd m0, m5, m7 ; b5 + pmaddwd m4, m0, m0 + punpckhwd m5, m7 + pmaddwd m1, m5, m5 +%if ARCH_X86_64 + SWAP m7, m6 %endif - punpcklbw m5, m6, m11 - punpckhbw m6, m11 - psllw m7, m5, 4 - psubw m1, m7 - psubw m3, m7 - psllw m7, m6, 4 - psubw m2, m7 - psubw m4, m7 - punpcklwd m7, m1, m3 - punpckhwd m1, m3 - punpcklwd m3, m2, m4 - punpckhwd m2, m4 - pmaddwd m7, m0 - pmaddwd m1, m0 - pmaddwd m3, m0 - pmaddwd m2, m0 - paddd m7, m10 - paddd m1, m10 - paddd m3, m10 - paddd m2, m10 - psrad m7, 11 - psrad m1, 11 - psrad m3, 11 - psrad m2, 11 - packssdw m7, m1 - packssdw m3, m2 - paddw m7, m5 - paddw m3, m6 - packuswb m7, m3 - mova [dstq+idxq], m7 - add idxd, 16 - cmp idxd, wd - jl .loop_x - add dstq, strideq - add t1q, 384 * 2 - add t2q, 384 * 2 - dec hd - jg .loop_y - RET + psubd m2, m4 ; p5 + psubd m3, m1 + MULLD m2, m13, m7 ; p5 * s0 + MULLD m3, m13, m7 + pmaddwd m0, m12 ; b5 * 164 + pmaddwd m5, m12 + paddusw m2, m12 + paddusw m3, m12 + psrld m2, 20 ; min(z5, 255) + psrld m3, 20 + GATHER_X_BY_X m1, m2, m3, r0, dstm + punpcklwd m2, m1, m1 + punpckhwd m3, m1, m1 + MULLD m0, m2, m7 + MULLD m5, m3, m7 + paddd m0, m10 ; x5 * b5 * 164 + (1 << 11) + (1 << 15) + paddd m5, m10 + mova [t4+wq*2+4], m1 + psrld m0, 12 + psrld m5, 12 + mova [t3+wq*4+ 8], m0 + mova [t3+wq*4+24], m5 + add wq, 8 + jl .hv1_loop + mov r10, t2 + mov t2, t1 + mov t1, r10 + ret +.v0: ; vertical boxsums + ab3 (even rows) +%if ARCH_X86_64 + lea wq, [r5-2] +%else + mov wd, w0m +%endif +.v0_loop: + mova m0, [t1+wq*2+400* 6] + mova m4, [t1+wq*2+400* 8] + mova m5, [t1+wq*2+400*10] + paddw m0, m0 + paddd m4, m4 + paddd m5, m5 + paddw m1, m0, [t2+wq*2+400* 6] + paddd m2, m4, [t2+wq*2+400* 8] + paddd m3, m5, [t2+wq*2+400*10] + mova [t2+wq*2+400* 6], m0 + mova [t2+wq*2+400* 8], m4 + mova [t2+wq*2+400*10], m5 +%if ARCH_X86_32 + pxor m7, m7 +%else + SWAP m7, m6 +%endif + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; a3 * 9 + paddd m5, m3 + punpcklwd m0, m1, m7 ; b3 + pmaddwd m2, m0, m0 + punpckhwd m1, m7 + pmaddwd m3, m1, m1 + psubd m4, m2 ; p3 + psubd m5, m3 +%if ARCH_X86_64 + SWAP m7, m6 +%endif + MULLD m4, m14, m7 ; p3 * s1 + MULLD m5, m14, m7 + pmaddwd m0, m11 ; b3 * 455 + pmaddwd m1, m11 + paddusw m4, m11 + paddusw m5, m11 + psrld m4, 20 ; min(z3, 255) + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, r0, dstm + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m7 + MULLD m1, m5, m7 + paddd m0, m10 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + paddd m1, m10 + mova [t4+wq*2+400*2+4], m3 + psrld m0, 12 + psrld m1, 12 + mova m3, [t1+wq*2+400*0] + mova m4, [t1+wq*2+400*2] + mova m5, [t1+wq*2+400*4] + mova [t3+wq*4+400*8+ 8], m3 + mova [t3+wq*4+400*0+ 8], m4 + mova [t3+wq*4+400*0+24], m5 + paddw m3, m3 ; cc5 + paddd m4, m4 + paddd m5, m5 + mova [t1+wq*2+400*0], m3 + mova [t1+wq*2+400*2], m4 + mova [t1+wq*2+400*4], m5 + mova [t3+wq*4+400*4+ 8], m0 + mova [t3+wq*4+400*4+24], m1 + add wq, 8 + jl .v0_loop + ret +.v1: ; vertical boxsums + ab (odd rows) +%if ARCH_X86_64 + lea wq, [r5-2] +%else + mov wd, w0m +%endif +.v1_loop: + mova m4, [t1+wq*2+400* 6] + mova m5, [t1+wq*2+400* 8] + mova m7, [t1+wq*2+400*10] + paddw m1, m4, [t2+wq*2+400* 6] + paddd m2, m5, [t2+wq*2+400* 8] + paddd m3, m7, [t2+wq*2+400*10] + mova [t2+wq*2+400* 6], m4 + mova [t2+wq*2+400* 8], m5 + mova [t2+wq*2+400*10], m7 +%if ARCH_X86_32 + pxor m7, m7 +%else + SWAP m7, m6 +%endif + pslld m4, m2, 3 + pslld m5, m3, 3 + paddd m4, m2 ; ((a3 + 8) >> 4) * 9 + paddd m5, m3 + punpcklwd m0, m1, m7 ; b3 + pmaddwd m2, m0, m0 + punpckhwd m1, m7 + pmaddwd m3, m1, m1 + psubd m4, m2 ; p3 + psubd m5, m3 +%if ARCH_X86_64 + SWAP m7, m6 +%endif + MULLD m4, m14, m7 ; p3 * s1 + MULLD m5, m14, m7 + pmaddwd m0, m11 ; b3 * 455 + pmaddwd m1, m11 + paddusw m4, m11 + paddusw m5, m11 + psrld m4, 20 ; min(z3, 255) + psrld m5, 20 + GATHER_X_BY_X m3, m4, m5, r0, dstm + punpcklwd m4, m3, m3 + punpckhwd m5, m3, m3 + MULLD m0, m4, m7 + MULLD m1, m5, m7 + paddd m0, m10 ; x3 * b3 * 455 + (1 << 11) + (1 << 15) + paddd m1, m10 + mova [t4+wq*2+400*4+4], m3 + psrld m0, 12 + psrld m8, m1, 12 + mova m4, [t3+wq*4+400*8+ 8] + mova m5, [t3+wq*4+400*0+ 8] + mova m7, [t3+wq*4+400*0+24] + paddw m1, m4, [t2+wq*2+400*0] + paddd m2, m5, [t2+wq*2+400*2] + paddd m3, m7, [t2+wq*2+400*4] + paddw m1, [t1+wq*2+400*0] + paddd m2, [t1+wq*2+400*2] + paddd m3, [t1+wq*2+400*4] + mova [t2+wq*2+400*0], m4 + mova [t2+wq*2+400*2], m5 + mova [t2+wq*2+400*4], m7 + pslld m4, m2, 4 + mova [t3+wq*4+400*8+ 8], m0 + pslld m5, m3, 4 + mova [t3+wq*4+400*8+24], m8 + pslld m7, m2, 3 + paddd m2, m4 + pslld m8, m3, 3 + paddd m3, m5 + paddd m2, m7 ; a5 * 25 + paddd m3, m8 +%if ARCH_X86_32 + pxor m7, m7 +%else + SWAP m7, m6 +%endif + punpcklwd m0, m1, m7 ; b5 + pmaddwd m4, m0, m0 + punpckhwd m1, m7 + pmaddwd m5, m1, m1 + psubd m2, m4 ; p5 + psubd m3, m5 +%if ARCH_X86_64 + SWAP m7, m6 +%endif + MULLD m2, m13, m7 ; p5 * s0 + MULLD m3, m13, m7 + pmaddwd m0, m12 ; b5 * 164 + pmaddwd m1, m12 + paddusw m2, m12 + paddusw m3, m12 + psrld m2, 20 ; min(z5, 255) + psrld m3, 20 + GATHER_X_BY_X m4, m2, m3, r0, dstm + punpcklwd m2, m4, m4 + punpckhwd m3, m4, m4 + MULLD m0, m2, m7 + MULLD m1, m3, m7 + paddd m0, m10 ; x5 * b5 * 164 + (1 << 11) + (1 << 15) + paddd m1, m10 + mova [t4+wq*2+4], m4 + psrld m0, 12 + psrld m1, 12 + mova [t3+wq*4+ 8], m0 + mova [t3+wq*4+24], m1 + add wq, 8 + jl .v1_loop + mov r10, t2 + mov t2, t1 + mov t1, r10 + ret +.prep_n: ; initial neighbor setup + movif64 wq, r5 + movif32 wd, w1m +.prep_n_loop: + movu m0, [t4+wq*2+400*0+ 2] + movu m1, [t3+wq*4+400*0+ 4] + movu m2, [t3+wq*4+400*0+20] + movu m7, [t4+wq*2+400*0+ 4] + movu m8, [t3+wq*4+400*0+ 8] + paddw m3, m0, [t4+wq*2+400*0+ 0] + paddd m4, m1, [t3+wq*4+400*0+ 0] + paddd m5, m2, [t3+wq*4+400*0+16] + paddw m3, m7 + paddd m4, m8 + movu m7, [t3+wq*4+400*0+24] + paddw m0, m3 + paddd m1, m4 + psllw m3, 2 + pslld m4, 2 + paddd m5, m7 + paddd m2, m5 + pslld m5, 2 + paddw m0, m3 ; a5 565 + paddd m1, m4 ; b5 565 + paddd m2, m5 + mova [t4+wq*2+400* 6+ 0], m0 + mova [t3+wq*4+400*12+ 0], m1 + mova [t3+wq*4+400*12+16], m2 + movu m0, [t4+wq*2+400*2+ 4] + movu m1, [t3+wq*4+400*4+ 8] + movu m2, [t3+wq*4+400*4+24] + movu m3, [t4+wq*2+400*2+ 2] + movu m4, [t3+wq*4+400*4+ 4] + movu m5, [t3+wq*4+400*4+20] + paddw m0, [t4+wq*2+400*2+ 0] + paddd m1, [t3+wq*4+400*4+ 0] + paddd m2, [t3+wq*4+400*4+16] + paddw m3, m0 + paddd m4, m1 + paddd m5, m2 + psllw m3, 2 ; a3[-1] 444 + pslld m4, 2 ; b3[-1] 444 + pslld m5, 2 + psubw m3, m0 ; a3[-1] 343 + psubd m4, m1 ; b3[-1] 343 + psubd m5, m2 + mova [t4+wq*2+400* 8+ 0], m3 + mova [t3+wq*4+400*16+ 0], m4 + mova [t3+wq*4+400*16+16], m5 + movu m0, [t4+wq*2+400*4+ 4] + movu m1, [t3+wq*4+400*8+ 8] + movu m2, [t3+wq*4+400*8+24] + movu m3, [t4+wq*2+400*4+ 2] + movu m4, [t3+wq*4+400*8+ 4] + movu m5, [t3+wq*4+400*8+20] + paddw m0, [t4+wq*2+400*4+ 0] + paddd m1, [t3+wq*4+400*8+ 0] + paddd m2, [t3+wq*4+400*8+16] + paddw m3, m0 + paddd m4, m1 + paddd m5, m2 + psllw m3, 2 ; a3[ 0] 444 + pslld m4, 2 ; b3[ 0] 444 + pslld m5, 2 + mova [t4+wq*2+400*10+ 0], m3 + mova [t3+wq*4+400*20+ 0], m4 + mova [t3+wq*4+400*20+16], m5 + psubw m3, m0 ; a3[ 0] 343 + psubd m4, m1 ; b3[ 0] 343 + psubd m5, m2 + mova [t4+wq*2+400*12+ 0], m3 + mova [t3+wq*4+400*24+ 0], m4 + mova [t3+wq*4+400*24+16], m5 + add wq, 8 + jl .prep_n_loop + ret +ALIGN function_align +.n0: ; neighbor + output (even rows) + movif64 wq, r5 + movif32 wd, w1m +.n0_loop: + movu m0, [t4+wq*2+ 4] + movu m2, [t4+wq*2+ 2] + paddw m0, [t4+wq*2+ 0] + paddw m0, m2 + paddw m2, m0 + psllw m0, 2 + paddw m0, m2 ; a5 + movu m4, [t3+wq*4+ 8] + movu m5, [t3+wq*4+24] + movu m1, [t3+wq*4+ 4] + movu m3, [t3+wq*4+20] + paddd m4, [t3+wq*4+ 0] + paddd m5, [t3+wq*4+16] + paddd m4, m1 + paddd m5, m3 + paddd m1, m4 + paddd m3, m5 + pslld m4, 2 + pslld m5, 2 + paddd m4, m1 ; b5 + paddd m5, m3 + movu m2, [t4+wq*2+400* 6] + paddw m2, m0 + mova [t4+wq*2+400* 6], m0 + paddd m0, m4, [t3+wq*4+400*12+ 0] + paddd m1, m5, [t3+wq*4+400*12+16] + mova [t3+wq*4+400*12+ 0], m4 + mova [t3+wq*4+400*12+16], m5 + mova [rsp+16+ARCH_X86_32*4], m1 + movu m3, [t4+wq*2+400*2+4] + movu m5, [t4+wq*2+400*2+2] + paddw m3, [t4+wq*2+400*2+0] + paddw m5, m3 + psllw m5, 2 ; a3[ 1] 444 + psubw m4, m5, m3 ; a3[ 1] 343 + movu m3, [t4+wq*2+400* 8] + paddw m3, [t4+wq*2+400*10] + paddw m3, m4 + mova [t4+wq*2+400* 8], m4 + mova [t4+wq*2+400*10], m5 + movu m1, [t3+wq*4+400*4+ 8] + movu m5, [t3+wq*4+400*4+ 4] + movu m7, [t3+wq*4+400*4+24] + movu m8, [t3+wq*4+400*4+20] + paddd m1, [t3+wq*4+400*4+ 0] + paddd m7, [t3+wq*4+400*4+16] + paddd m5, m1 + paddd m8, m7 + pslld m5, 2 ; b3[ 1] 444 + pslld m8, 2 + psubd m4, m5, m1 ; b3[ 1] 343 +%if ARCH_X86_32 + mova [esp+52], m8 + psubd m8, m7 +%else + psubd m6, m8, m7 + SWAP m8, m6 +%endif + paddd m1, m4, [t3+wq*4+400*16+ 0] + paddd m7, m8, [t3+wq*4+400*16+16] + paddd m1, [t3+wq*4+400*20+ 0] + paddd m7, [t3+wq*4+400*20+16] + mova [t3+wq*4+400*16+ 0], m4 + mova [t3+wq*4+400*16+16], m8 + mova [t3+wq*4+400*20+ 0], m5 +%if ARCH_X86_32 + mova m8, [esp+52] +%else + SWAP m8, m6 + pxor m6, m6 +%endif + mova [t3+wq*4+400*20+16], m8 + mova [rsp+32+ARCH_X86_32*4], m7 + movq m4, [dstq+wq] + punpcklbw m4, m6 + punpcklwd m5, m4, m6 + punpcklwd m7, m2, m6 + pmaddwd m7, m5 ; a5 * src + punpcklwd m8, m3, m6 + pmaddwd m8, m5 ; a3 * src + punpckhwd m5, m4, m6 + punpckhwd m2, m6 + pmaddwd m2, m5 + punpckhwd m3, m6 + pmaddwd m3, m5 + psubd m0, m7 ; b5 - a5 * src + (1 << 8) - (src << 13) + psubd m1, m8 ; b3 - a3 * src + (1 << 8) - (src << 13) + psrld m0, 9 + pslld m1, 7 + pand m0, m9 + pandn m8, m9, m1 + por m0, m8 + mova m1, [rsp+16+ARCH_X86_32*4] + psubd m1, m2 + mova m2, [rsp+32+ARCH_X86_32*4] + psubd m2, m3 + mova m3, [base+pd_4096] + psrld m1, 9 + pslld m2, 7 + pand m1, m9 + pandn m5, m9, m2 + por m1, m5 + pmaddwd m0, m15 + pmaddwd m1, m15 + paddd m0, m3 + paddd m1, m3 + psrad m0, 13 + psrad m1, 13 + packssdw m0, m1 + paddw m0, m4 + packuswb m0, m0 + movq [dstq+wq], m0 + add wq, 8 + jl .n0_loop + add dstq, dst_stridemp + ret +ALIGN function_align +.n1: ; neighbor + output (odd rows) + movif64 wq, r5 + movif32 wd, w1m +.n1_loop: + movu m3, [t4+wq*2+400*4+4] + movu m5, [t4+wq*2+400*4+2] + paddw m3, [t4+wq*2+400*4+0] + paddw m5, m3 + psllw m5, 2 ; a3[ 1] 444 + psubw m4, m5, m3 ; a3[ 1] 343 + paddw m3, m4, [t4+wq*2+400*12] + paddw m3, [t4+wq*2+400*10] + mova [t4+wq*2+400*10], m5 + mova [t4+wq*2+400*12], m4 + movu m1, [t3+wq*4+400*8+ 8] + movu m5, [t3+wq*4+400*8+ 4] + movu m7, [t3+wq*4+400*8+24] + movu m8, [t3+wq*4+400*8+20] + paddd m1, [t3+wq*4+400*8+ 0] + paddd m7, [t3+wq*4+400*8+16] + paddd m5, m1 + paddd m8, m7 + pslld m5, 2 ; b3[ 1] 444 + pslld m8, 2 + psubd m4, m5, m1 ; b3[ 1] 343 + psubd m0, m8, m7 + paddd m1, m4, [t3+wq*4+400*24+ 0] + paddd m7, m0, [t3+wq*4+400*24+16] + paddd m1, [t3+wq*4+400*20+ 0] + paddd m7, [t3+wq*4+400*20+16] + mova [t3+wq*4+400*20+ 0], m5 + mova [t3+wq*4+400*20+16], m8 + mova [t3+wq*4+400*24+ 0], m4 + mova [t3+wq*4+400*24+16], m0 + movq m5, [dstq+wq] + mova m2, [t4+wq*2+400* 6] + punpcklbw m5, m6 + punpcklwd m4, m5, m6 + punpcklwd m8, m2, m6 + pmaddwd m8, m4 ; a5 * src + punpcklwd m0, m3, m6 + pmaddwd m0, m4 ; a3 * src + punpckhwd m4, m5, m6 + punpckhwd m2, m6 + pmaddwd m2, m4 + punpckhwd m3, m6 + pmaddwd m3, m4 + psubd m1, m0 ; b3 - a3 * src + (1 << 8) - (src << 13) + mova m0, [t3+wq*4+400*12+ 0] + psubd m0, m8 ; b5 - a5 * src + (1 << 8) - (src << 13) + mova m4, [t3+wq*4+400*12+16] + psubd m4, m2 + psubd m7, m3 + pslld m1, 7 + psrld m0, 8 + psrld m4, 8 + pslld m7, 7 + pandn m3, m9, m1 + pand m0, m9 + por m0, m3 + pand m4, m9 + pandn m2, m9, m7 + por m2, m4 + mova m1, [base+pd_4096] + pmaddwd m0, m15 + pmaddwd m2, m15 + paddd m0, m1 + paddd m2, m1 + psrad m0, 13 + psrad m2, 13 + packssdw m0, m2 + paddw m0, m5 + packuswb m0, m0 + movq [dstq+wq], m0 + add wq, 8 + jl .n1_loop + add dstq, dst_stridemp + movif32 dstm, dstq + ret diff --git a/third_party/dav1d/src/x86/mc16_avx2.asm b/third_party/dav1d/src/x86/mc16_avx2.asm --- a/third_party/dav1d/src/x86/mc16_avx2.asm +++ b/third_party/dav1d/src/x86/mc16_avx2.asm @@ -42,35 +42,56 @@ blend_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3 deint_shuf: dd 0, 4, 1, 5, 2, 6, 3, 7 subpel_h_shufA: db 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9 subpel_h_shufB: db 4, 5, 6, 7, 6, 7, 8, 9, 8, 9, 10, 11, 10, 11, 12, 13 subpel_h_shuf2: db 0, 1, 2, 3, 4, 5, 6, 7, 2, 3, 4, 5, 6, 7, 8, 9 +subpel_s_shuf2: db 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7 +subpel_s_shuf8: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15 +rescale_mul: dd 0, 1, 2, 3, 4, 5, 6, 7 +rescale_mul2: dd 0, 1, 4, 5, 2, 3, 6, 7 +resize_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7 + db 8, 9, 10, 11, 12, 13, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15 +wswap: db 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13 +bdct_lb_q: times 8 db 0 + times 8 db 4 + times 8 db 8 + times 8 db 12 -put_bilin_h_rnd: dw 8, 8, 10, 10 -prep_mul: dw 16, 16, 4, 4 +prep_mul: dw 16, 16, 4, 4 +put_bilin_h_rnd: dw 8, 8, 10, 10 put_8tap_h_rnd: dd 34, 40 +s_8tap_h_rnd: dd 2, 8 +s_8tap_h_sh: dd 2, 4 +put_s_8tap_v_rnd: dd 512, 128 +put_s_8tap_v_sh: dd 10, 8 prep_8tap_1d_rnd: dd 8 - (8192 << 4) prep_8tap_2d_rnd: dd 32 - (8192 << 5) warp8x8t_rnd: dd 16384 - (8192 << 15) warp8x8_shift: dd 5, 3 warp8x8_rnd: dw 4096, 4096, 16384, 16384 bidir_rnd: dw -16400, -16400, -16388, -16388 bidir_mul: dw 2048, 2048, 8192, 8192 %define pw_16 prep_mul +%define pd_512 put_s_8tap_v_rnd -pw_2: times 2 dw 2 -pw_64: times 2 dw 64 -pw_2048: times 2 dw 2048 -pw_8192: times 2 dw 8192 -pw_27615: times 2 dw 27615 -pw_32766: times 2 dw 32766 -pw_m512: times 2 dw -512 -pd_32: dd 32 -pd_512: dd 512 -pd_32768: dd 32768 -pd_65538: dd 65538 +pw_2: times 2 dw 2 +pw_64: times 2 dw 64 +pw_2048: times 2 dw 2048 +pw_8192: times 2 dw 8192 +pw_27615: times 2 dw 27615 +pw_32766: times 2 dw 32766 +pw_m512: times 2 dw -512 +pd_32: dd 32 +pd_63: dd 63 +pd_64: dd 64 +pd_32768: dd 32768 +pd_65538: dd 65538 +pd_m524256: dd -524256 ; -8192 << 6 + 32 +pd_0x3ff: dd 0x3ff +pq_0x40000000: dq 0x40000000 + dd 0 %macro BIDIR_JMP_TABLE 2-* %xdefine %1_%2_table (%%table - 2*%3) %xdefine %%base %1_%2_table %xdefine %%prefix mangle(private_prefix %+ _%1_16bpc_%2) @@ -140,19 +161,55 @@ %endmacro HV_JMP_TABLE put, bilin, avx2, 7, 2, 4, 8, 16, 32, 64, 128 HV_JMP_TABLE prep, bilin, avx2, 7, 4, 8, 16, 32, 64, 128 +%macro SCALED_JMP_TABLE 2-* + %xdefine %1_%2_table (%%table - %3) + %xdefine %%base mangle(private_prefix %+ _%1_16bpc_%2) +%%table: + %rep %0 - 2 + dw %%base %+ .w%3 - %%base + %rotate 1 + %endrep + %rotate 2 + %%dy_1024: + %xdefine %1_%2_dy1_table (%%dy_1024 - %3) + %rep %0 - 2 + dw %%base %+ .dy1_w%3 - %%base + %rotate 1 + %endrep + %rotate 2 + %%dy_2048: + %xdefine %1_%2_dy2_table (%%dy_2048 - %3) + %rep %0 - 2 + dw %%base %+ .dy2_w%3 - %%base + %rotate 1 + %endrep +%endmacro + +SCALED_JMP_TABLE put_8tap_scaled, avx2, 2, 4, 8, 16, 32, 64, 128 +SCALED_JMP_TABLE prep_8tap_scaled, avx2, 4, 8, 16, 32, 64, 128 + %define table_offset(type, fn) type %+ fn %+ SUFFIX %+ _table - type %+ SUFFIX cextern mc_subpel_filters %define subpel_filters (mangle(private_prefix %+ _mc_subpel_filters)-8) cextern mc_warp_filter +cextern resize_filter SECTION .text +%macro REPX 2-* + %xdefine %%f(x) %1 +%rep %0 - 1 + %rotate 1 + %%f({%1}) +%endrep +%endmacro + INIT_XMM avx2 cglobal put_bilin_16bpc, 4, 8, 0, dst, ds, src, ss, w, h, mxy mov mxyd, r6m ; mx lea r7, [put_avx2] %if UNIX64 @@ -1169,38 +1226,39 @@ ; int8_t subpel_filters[5][15][8] %assign FILTER_REGULAR (0*15 << 16) | 3*15 %assign FILTER_SMOOTH (1*15 << 16) | 4*15 %assign FILTER_SHARP (2*15 << 16) | 3*15 -%macro MC_8TAP_FN 4 ; prefix, type, type_h, type_v -cglobal %1_8tap_%2_16bpc +%macro FN 4 ; prefix, type, type_h, type_v +cglobal %1_%2_16bpc mov t0d, FILTER_%3 %ifidn %3, %4 mov t1d, t0d %else mov t1d, FILTER_%4 %endif %ifnidn %2, regular ; skip the jump in the last filter - jmp mangle(private_prefix %+ _%1_8tap_16bpc %+ SUFFIX) + jmp mangle(private_prefix %+ _%1_16bpc %+ SUFFIX) %endif %endmacro %if WIN64 DECLARE_REG_TMP 4, 5 %else DECLARE_REG_TMP 7, 8 %endif -MC_8TAP_FN put, sharp, SHARP, SHARP -MC_8TAP_FN put, sharp_smooth, SHARP, SMOOTH -MC_8TAP_FN put, smooth_sharp, SMOOTH, SHARP -MC_8TAP_FN put, smooth, SMOOTH, SMOOTH -MC_8TAP_FN put, sharp_regular, SHARP, REGULAR -MC_8TAP_FN put, regular_sharp, REGULAR, SHARP -MC_8TAP_FN put, smooth_regular, SMOOTH, REGULAR -MC_8TAP_FN put, regular_smooth, REGULAR, SMOOTH -MC_8TAP_FN put, regular, REGULAR, REGULAR +%define PUT_8TAP_FN FN put_8tap, +PUT_8TAP_FN sharp, SHARP, SHARP +PUT_8TAP_FN sharp_smooth, SHARP, SMOOTH +PUT_8TAP_FN smooth_sharp, SMOOTH, SHARP +PUT_8TAP_FN smooth, SMOOTH, SMOOTH +PUT_8TAP_FN sharp_regular, SHARP, REGULAR +PUT_8TAP_FN regular_sharp, REGULAR, SHARP +PUT_8TAP_FN smooth_regular, SMOOTH, REGULAR +PUT_8TAP_FN regular_smooth, REGULAR, SMOOTH +PUT_8TAP_FN regular, REGULAR, REGULAR cglobal put_8tap_16bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my %define base r8-put_avx2 imul mxd, mxm, 0x010101 add mxd, t0d ; 8tap_h, mx, 4tap_h @@ -1907,19 +1965,20 @@ DECLARE_REG_TMP 6, 4 %else DECLARE_REG_TMP 6, 7 %endif -MC_8TAP_FN prep, sharp, SHARP, SHARP -MC_8TAP_FN prep, sharp_smooth, SHARP, SMOOTH -MC_8TAP_FN prep, smooth_sharp, SMOOTH, SHARP -MC_8TAP_FN prep, smooth, SMOOTH, SMOOTH -MC_8TAP_FN prep, sharp_regular, SHARP, REGULAR -MC_8TAP_FN prep, regular_sharp, REGULAR, SHARP -MC_8TAP_FN prep, smooth_regular, SMOOTH, REGULAR -MC_8TAP_FN prep, regular_smooth, REGULAR, SMOOTH -MC_8TAP_FN prep, regular, REGULAR, REGULAR +%define PREP_8TAP_FN FN prep_8tap, +PREP_8TAP_FN sharp, SHARP, SHARP +PREP_8TAP_FN sharp_smooth, SHARP, SMOOTH +PREP_8TAP_FN smooth_sharp, SMOOTH, SHARP +PREP_8TAP_FN smooth, SMOOTH, SMOOTH +PREP_8TAP_FN sharp_regular, SHARP, REGULAR +PREP_8TAP_FN regular_sharp, REGULAR, SHARP +PREP_8TAP_FN smooth_regular, SMOOTH, REGULAR +PREP_8TAP_FN regular_smooth, REGULAR, SMOOTH +PREP_8TAP_FN regular, REGULAR, REGULAR cglobal prep_8tap_16bpc, 4, 8, 0, tmp, src, stride, w, h, mx, my %define base r7-prep_avx2 imul mxd, mxm, 0x010101 add mxd, t0d ; 8tap_h, mx, 4tap_h @@ -2503,10 +2562,1728 @@ %if WIN64 POP r8 %endif RET +%macro movifprep 2 + %if isprep + mov %1, %2 + %endif +%endmacro + +%macro REMAP_REG 2 + %xdefine r%1 r%2 + %xdefine r%1q r%2q + %xdefine r%1d r%2d +%endmacro + +%macro MCT_8TAP_SCALED_REMAP_REGS_TO_PREV 0 + %if isprep + %xdefine r14_save r14 + %assign %%i 14 + %rep 14 + %assign %%j %%i-1 + REMAP_REG %%i, %%j + %assign %%i %%i-1 + %endrep + %endif +%endmacro + +%macro MCT_8TAP_SCALED_REMAP_REGS_TO_DEFAULT 0 + %if isprep + %assign %%i 1 + %rep 13 + %assign %%j %%i+1 + REMAP_REG %%i, %%j + %assign %%i %%i+1 + %endrep + %xdefine r14 r14_save + %undef r14_save + %endif +%endmacro + +%macro MC_8TAP_SCALED_RET 0-1 1 ; leave_mapping_unchanged + MCT_8TAP_SCALED_REMAP_REGS_TO_DEFAULT + RET + %if %1 + MCT_8TAP_SCALED_REMAP_REGS_TO_PREV + %endif +%endmacro + +%macro MC_8TAP_SCALED_H 8-9 0 ; dst, tmp[0-6], load_hrnd + movu xm%1, [srcq+ r4*2] + movu xm%2, [srcq+ r6*2] + movu xm%3, [srcq+ r7*2] + movu xm%4, [srcq+ r9*2] + vinserti128 m%1, [srcq+r10*2], 1 + vinserti128 m%2, [srcq+r11*2], 1 + vinserti128 m%3, [srcq+r13*2], 1 + vinserti128 m%4, [srcq+ rX*2], 1 + add srcq, ssq + movu xm%5, [srcq+ r4*2] + movu xm%6, [srcq+ r6*2] + movu xm%7, [srcq+ r7*2] + movu xm%8, [srcq+ r9*2] + vinserti128 m%5, [srcq+r10*2], 1 + vinserti128 m%6, [srcq+r11*2], 1 + vinserti128 m%7, [srcq+r13*2], 1 + vinserti128 m%8, [srcq+ rX*2], 1 + add srcq, ssq + pmaddwd m%1, m12 + pmaddwd m%2, m13 + pmaddwd m%3, m14 + pmaddwd m%4, m15 + pmaddwd m%5, m12 + pmaddwd m%6, m13 + pmaddwd m%7, m14 + pmaddwd m%8, m15 + phaddd m%1, m%2 + %if %9 + mova m10, [rsp+0x00] + %endif + phaddd m%3, m%4 + phaddd m%5, m%6 + phaddd m%7, m%8 + phaddd m%1, m%3 + phaddd m%5, m%7 + paddd m%1, m10 + paddd m%5, m10 + psrad m%1, xm11 + psrad m%5, xm11 + packssdw m%1, m%5 +%endmacro + +%macro MC_8TAP_SCALED 1 +%ifidn %1, put + %assign isput 1 + %assign isprep 0 + %if required_stack_alignment <= STACK_ALIGNMENT +cglobal put_8tap_scaled_16bpc, 4, 15, 16, 0xe0, dst, ds, src, ss, w, h, mx, my, dx, dy, pxmax + %else +cglobal put_8tap_scaled_16bpc, 4, 14, 16, 0xe0, dst, ds, src, ss, w, h, mx, my, dx, dy, pxmax + %endif + %xdefine base_reg r12 + mov r7d, pxmaxm +%else + %assign isput 0 + %assign isprep 1 + %if required_stack_alignment <= STACK_ALIGNMENT +cglobal prep_8tap_scaled_16bpc, 4, 15, 16, 0xe0, tmp, src, ss, w, h, mx, my, dx, dy, pxmax + %xdefine tmp_stridem r14q + %else +cglobal prep_8tap_scaled_16bpc, 4, 14, 16, 0xe0, tmp, src, ss, w, h, mx, my, dx, dy, pxmax + %define tmp_stridem qword [rsp+0xd0] + %endif + %xdefine base_reg r11 +%endif + lea base_reg, [%1_8tap_scaled_16bpc_avx2] +%define base base_reg-%1_8tap_scaled_16bpc_avx2 + tzcnt wd, wm + vpbroadcastd m8, dxm +%if isprep && UNIX64 + movd xm10, mxd + vpbroadcastd m10, xm10 + mov r5d, t0d + DECLARE_REG_TMP 5, 7 + mov r6d, pxmaxm +%else + vpbroadcastd m10, mxm + %if isput + vpbroadcastw m11, pxmaxm + %else + mov r6d, pxmaxm + %endif +%endif + mov dyd, dym +%if isput + %if WIN64 + mov r8d, hm + DEFINE_ARGS dst, ds, src, ss, w, _, _, my, h, dy, ss3 + %define hm r5m + %define dxm r8m + %else + DEFINE_ARGS dst, ds, src, ss, w, h, _, my, dx, dy, ss3 + %define hm r6m + %endif + %if required_stack_alignment > STACK_ALIGNMENT + %define dsm [rsp+0x98] + %define rX r1 + %define rXd r1d + %else + %define dsm dsq + %define rX r14 + %define rXd r14d + %endif +%else ; prep + %if WIN64 + mov r7d, hm + DEFINE_ARGS tmp, src, ss, w, _, _, my, h, dy, ss3 + %define hm r4m + %define dxm r7m + %else + DEFINE_ARGS tmp, src, ss, w, h, _, my, dx, dy, ss3 + %define hm [rsp+0x98] + %endif + MCT_8TAP_SCALED_REMAP_REGS_TO_PREV + %define rX r14 + %define rXd r14d +%endif + shr r7d, 11 + vpbroadcastd m6, [base+pd_0x3ff] + vpbroadcastd m12, [base+s_8tap_h_rnd+r7*4] + movd xm7, [base+s_8tap_h_sh+r7*4] +%if isput + vpbroadcastd m13, [base+put_s_8tap_v_rnd+r7*4] + pinsrd xm7, [base+put_s_8tap_v_sh+r7*4], 2 +%else + vpbroadcastd m13, [base+pd_m524256] +%endif + pxor m9, m9 + lea ss3q, [ssq*3] + movzx r7d, t1b + shr t1d, 16 + cmp hd, 6 + cmovs t1d, r7d + sub srcq, ss3q + cmp dyd, 1024 + je .dy1 + cmp dyd, 2048 + je .dy2 + movzx wd, word [base+%1_8tap_scaled_avx2_table+wq*2] + add wq, base_reg + jmp wq +%if isput +.w2: + mov myd, mym + movzx t0d, t0b + sub srcq, 2 + movd xm15, t0d + punpckldq m8, m9, m8 + paddd m10, m8 ; mx+dx*[0,1] + vpbroadcastd xm14, [base+pq_0x40000000+2] + vpbroadcastd xm15, xm15 + pand xm8, xm10, xm6 + psrld xm8, 6 + paddd xm15, xm8 + movd r4d, xm15 + pextrd r6d, xm15, 1 + vbroadcasti128 m5, [base+bdct_lb_q] + vbroadcasti128 m6, [base+subpel_s_shuf2] + vpbroadcastd xm15, [base+subpel_filters+r4*8+2] + vpbroadcastd xm4, [base+subpel_filters+r6*8+2] + pcmpeqd xm8, xm9 + psrld m10, 10 + paddd m10, m10 + movu xm0, [srcq+ssq*0] + movu xm1, [srcq+ssq*1] + movu xm2, [srcq+ssq*2] + movu xm3, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + pshufb m10, m5 + paddb m10, m6 + vpblendd xm15, xm4, 0xa + pblendvb xm15, xm14, xm8 + pmovsxbw m15, xm15 + vinserti128 m0, [srcq+ssq*0], 1 ; 0 4 + vinserti128 m1, [srcq+ssq*1], 1 ; 1 5 + vinserti128 m2, [srcq+ssq*2], 1 ; 2 6 + vinserti128 m3, [srcq+ss3q ], 1 ; 3 7 + lea srcq, [srcq+ssq*4] + REPX {pshufb x, m10}, m0, m1, m2, m3 + REPX {pmaddwd x, m15}, m0, m1, m2, m3 + phaddd m0, m1 + phaddd m2, m3 + paddd m0, m12 + paddd m2, m12 + psrad m0, xm7 + psrad m2, xm7 + packssdw m0, m2 ; 0 1 2 3 4 5 6 7 + vextracti128 xm1, m0, 1 + palignr xm2, xm1, xm0, 4 ; 1 2 3 4 + punpcklwd xm3, xm0, xm2 ; 01 12 + punpckhwd xm0, xm2 ; 23 34 + pshufd xm4, xm1, q0321 ; 5 6 7 _ + punpcklwd xm2, xm1, xm4 ; 45 56 + punpckhwd xm4, xm1, xm4 ; 67 __ +.w2_loop: + and myd, 0x3ff + mov r6d, 64 << 24 + mov r4d, myd + shr r4d, 6 + lea r4d, [t1+r4] + cmovnz r6q, [base+subpel_filters+r4*8] + movq xm14, r6q + pmovsxbw xm14, xm14 + pshufd xm8, xm14, q0000 + pshufd xm9, xm14, q1111 + pmaddwd xm5, xm3, xm8 + pmaddwd xm6, xm0, xm9 + pshufd xm8, xm14, q2222 + pshufd xm14, xm14, q3333 + paddd xm5, xm6 + pmaddwd xm6, xm2, xm8 + pmaddwd xm8, xm4, xm14 + psrldq xm9, xm7, 8 + paddd xm5, xm6 + paddd xm5, xm13 + paddd xm5, xm8 + psrad xm5, xm9 + packusdw xm5, xm5 + pminsw xm5, xm11 + movd [dstq], xm5 + add dstq, dsq + dec hd + jz .ret + add myd, dyd + test myd, ~0x3ff + jz .w2_loop + movu xm5, [srcq] + test myd, 0x400 + jz .w2_skip_line + add srcq, ssq + shufps xm3, xm0, q1032 ; 01 12 + shufps xm0, xm2, q1032 ; 23 34 + shufps xm2, xm4, q1032 ; 45 56 + pshufb xm5, xm10 + pmaddwd xm5, xm15 + phaddd xm5, xm5 + paddd xm5, xm12 + psrad xm5, xm7 + packssdw xm5, xm5 + palignr xm1, xm5, xm1, 12 + punpcklqdq xm1, xm1 ; 6 7 6 7 + punpcklwd xm4, xm1, xm5 ; 67 __ + jmp .w2_loop +.w2_skip_line: + movu xm6, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mova xm3, xm0 ; 01 12 + mova xm0, xm2 ; 23 34 + pshufb xm5, xm10 + pshufb xm6, xm10 + pmaddwd xm5, xm15 + pmaddwd xm6, xm15 + phaddd xm5, xm6 + paddd xm5, xm12 + psrad xm5, xm7 + packssdw xm5, xm5 ; 6 7 6 7 + palignr xm1, xm5, xm1, 8 ; 4 5 6 7 + pshufd xm5, xm1, q0321 ; 5 6 7 _ + punpcklwd xm2, xm1, xm5 ; 45 56 + punpckhwd xm4, xm1, xm5 ; 67 __ + jmp .w2_loop +%endif +.w4: + mov myd, mym + mova [rsp+0x00], m12 +%if isput + mova [rsp+0x20], xm13 +%else + SWAP m11, m13 +%endif + mova [rsp+0x30], xm7 + vbroadcasti128 m7, [base+rescale_mul] + movzx t0d, t0b + sub srcq, 2 + movd xm15, t0d + pmaddwd m8, m7 + vpbroadcastq m2, [base+pq_0x40000000+1] + vpbroadcastd xm15, xm15 + SWAP m13, m10 + paddd m13, m8 ; mx+dx*[0-3] + pand m6, m13 + psrld m6, 6 + paddd xm15, xm6 + movd r4d, xm15 + pextrd r6d, xm15, 1 + pextrd r11d, xm15, 2 + pextrd r13d, xm15, 3 + vbroadcasti128 m5, [base+bdct_lb_q+ 0] + vbroadcasti128 m1, [base+bdct_lb_q+16] + vbroadcasti128 m0, [base+subpel_s_shuf2] + vpbroadcastd xm14, [base+subpel_filters+r4*8+2] + vpbroadcastd xm7, [base+subpel_filters+r6*8+2] + vpbroadcastd xm15, [base+subpel_filters+r11*8+2] + vpbroadcastd xm8, [base+subpel_filters+r13*8+2] + pcmpeqd m6, m9 + punpckldq m10, m6, m6 + punpckhdq m6, m6 + psrld m13, 10 + paddd m13, m13 + vpblendd xm14, xm7, 0xa + vpblendd xm15, xm8, 0xa + pmovsxbw m14, xm14 + pmovsxbw m15, xm15 + pblendvb m14, m2, m10 + pblendvb m15, m2, m6 + pextrd r4, xm13, 2 + pshufb m12, m13, m5 + pshufb m13, m1 + lea r6, [r4+ssq*1] + lea r11, [r4+ssq*2] + lea r13, [r4+ss3q ] + movu xm7, [srcq+ssq*0] + movu xm9, [srcq+ssq*1] + movu xm8, [srcq+ssq*2] + movu xm10, [srcq+ss3q ] + movu xm1, [srcq+r4 ] + movu xm3, [srcq+r6 ] + movu xm2, [srcq+r11 ] + movu xm4, [srcq+r13 ] + lea srcq, [srcq+ssq*4] + vinserti128 m7, [srcq+ssq*0], 1 + vinserti128 m9, [srcq+ssq*1], 1 + vinserti128 m8, [srcq+ssq*2], 1 + vinserti128 m10, [srcq+ss3q ], 1 + vinserti128 m1, [srcq+r4 ], 1 + vinserti128 m3, [srcq+r6 ], 1 + vinserti128 m2, [srcq+r11 ], 1 + vinserti128 m4, [srcq+r13 ], 1 + lea srcq, [srcq+ssq*4] + vpbroadcastb m5, xm13 + psubb m13, m5 + paddb m12, m0 + paddb m13, m0 + REPX {pshufb x, m12}, m7, m9, m8, m10 + REPX {pmaddwd x, m14}, m7, m9, m8, m10 + REPX {pshufb x, m13}, m1, m2, m3, m4 + REPX {pmaddwd x, m15}, m1, m2, m3, m4 + mova m5, [rsp+0x00] + movd xm6, [rsp+0x30] + phaddd m7, m1 + phaddd m9, m3 + phaddd m8, m2 + phaddd m10, m4 + REPX {paddd x, m5}, m7, m9, m8, m10 + REPX {psrad x, xm6}, m7, m9, m8, m10 + packssdw m7, m9 ; 0 1 4 5 + packssdw m8, m10 ; 2 3 6 7 + vextracti128 xm9, m7, 1 ; 4 5 + vextracti128 xm3, m8, 1 ; 6 7 + shufps xm4, xm7, xm8, q1032 ; 1 2 + shufps xm5, xm8, xm9, q1032 ; 3 4 + shufps xm6, xm9, xm3, q1032 ; 5 6 + psrldq xm10, xm3, 8 ; 7 _ + punpcklwd xm0, xm7, xm4 ; 01 + punpckhwd xm7, xm4 ; 12 + punpcklwd xm1, xm8, xm5 ; 23 + punpckhwd xm8, xm5 ; 34 + punpcklwd xm2, xm9, xm6 ; 45 + punpckhwd xm9, xm6 ; 56 + punpcklwd xm3, xm10 ; 67 + mova [rsp+0x40], xm7 + mova [rsp+0x50], xm8 + mova [rsp+0x60], xm9 +.w4_loop: + and myd, 0x3ff + mov r11d, 64 << 24 + mov r13d, myd + shr r13d, 6 + lea r13d, [t1+r13] + cmovnz r11q, [base+subpel_filters+r13*8] + movq xm9, r11q + pmovsxbw xm9, xm9 + pshufd xm7, xm9, q0000 + pshufd xm8, xm9, q1111 + pmaddwd xm4, xm0, xm7 + pmaddwd xm5, xm1, xm8 + pshufd xm7, xm9, q2222 + pshufd xm9, xm9, q3333 + pmaddwd xm6, xm2, xm7 + pmaddwd xm8, xm3, xm9 +%if isput + mova xm7, [rsp+0x20] + movd xm9, [rsp+0x38] +%else + SWAP m7, m11 +%endif + paddd xm4, xm5 + paddd xm6, xm8 + paddd xm4, xm6 + paddd xm4, xm7 +%if isput + psrad xm4, xm9 + packusdw xm4, xm4 + pminuw xm4, xm11 + movq [dstq], xm4 + add dstq, dsq +%else + SWAP m11, m7 + psrad xm4, 6 + packssdw xm4, xm4 + movq [tmpq], xm4 + add tmpq, 8 +%endif + dec hd + jz .ret + mova xm8, [rsp+0x00] + movd xm9, [rsp+0x30] + add myd, dyd + test myd, ~0x3ff + jz .w4_loop + movu xm4, [srcq] + movu xm5, [srcq+r4] + test myd, 0x400 + jz .w4_skip_line + mova xm0, [rsp+0x40] + mova [rsp+0x40], xm1 + mova xm1, [rsp+0x50] + mova [rsp+0x50], xm2 + mova xm2, [rsp+0x60] + mova [rsp+0x60], xm3 + pshufb xm4, xm12 + pshufb xm5, xm13 + pmaddwd xm4, xm14 + pmaddwd xm5, xm15 + phaddd xm4, xm5 + paddd xm4, xm8 + psrad xm4, xm9 + packssdw xm4, xm4 + punpcklwd xm3, xm10, xm4 + mova xm10, xm4 + add srcq, ssq + jmp .w4_loop +.w4_skip_line: + movu xm6, [srcq+ssq*1] + movu xm7, [srcq+r6] + movu m0, [rsp+0x50] + pshufb xm4, xm12 + pshufb xm6, xm12 + pshufb xm5, xm13 + pshufb xm7, xm13 + pmaddwd xm4, xm14 + pmaddwd xm6, xm14 + pmaddwd xm5, xm15 + pmaddwd xm7, xm15 + mova [rsp+0x40], m0 + phaddd xm4, xm5 + phaddd xm6, xm7 + paddd xm4, xm8 + paddd xm6, xm8 + psrad xm4, xm9 + psrad xm6, xm9 + packssdw xm4, xm6 + punpcklwd xm9, xm10, xm4 + mova [rsp+0x60], xm9 + psrldq xm10, xm4, 8 + mova xm0, xm1 + mova xm1, xm2 + mova xm2, xm3 + punpcklwd xm3, xm4, xm10 + lea srcq, [srcq+ssq*2] + jmp .w4_loop + SWAP m10, m13 +%if isprep + SWAP m13, m11 +%endif +.w8: + mov dword [rsp+0x80], 1 + movifprep tmp_stridem, 16 + jmp .w_start +.w16: + mov dword [rsp+0x80], 2 + movifprep tmp_stridem, 32 + jmp .w_start +.w32: + mov dword [rsp+0x80], 4 + movifprep tmp_stridem, 64 + jmp .w_start +.w64: + mov dword [rsp+0x80], 8 + movifprep tmp_stridem, 128 + jmp .w_start +.w128: + mov dword [rsp+0x80], 16 + movifprep tmp_stridem, 256 +.w_start: + SWAP m10, m12, m1 + SWAP m11, m7 + ; m1=mx, m7=pxmax, m10=h_rnd, m11=h_sh, m12=free +%if isput + movifnidn dsm, dsq + mova [rsp+0xb0], xm7 +%endif + mova [rsp+0x00], m10 + mova [rsp+0x20], m13 + shr t0d, 16 + sub srcq, 6 + pmaddwd m8, [base+rescale_mul2] + movd xm15, t0d + mov [rsp+0x84], t0d + mov [rsp+0x88], srcq + mov [rsp+0x90], r0q ; dstq / tmpq +%if UNIX64 + mov hm, hd +%endif + shl dword dxm, 3 ; dx*8 + vpbroadcastd m15, xm15 + paddd m1, m8 ; mx+dx*[0-7] + jmp .hloop +.hloop_prep: + dec dword [rsp+0x80] + jz .ret + add qword [rsp+0x90], 16 + mov hd, hm + vpbroadcastd m8, dxm + vpbroadcastd m6, [base+pd_0x3ff] + paddd m1, m8, [rsp+0x40] + vpbroadcastd m15, [rsp+0x84] + pxor m9, m9 + mov srcq, [rsp+0x88] + mov r0q, [rsp+0x90] ; dstq / tmpq +.hloop: + vpbroadcastq xm2, [base+pq_0x40000000] + pand m5, m1, m6 + psrld m5, 6 + paddd m15, m5 + pcmpeqd m5, m9 + vextracti128 xm7, m15, 1 + movq r6, xm15 + pextrq r9, xm15, 1 + movq r11, xm7 + pextrq rX, xm7, 1 + mov r4d, r6d + shr r6, 32 + mov r7d, r9d + shr r9, 32 + mov r10d, r11d + shr r11, 32 + mov r13d, rXd + shr rX, 32 + mova [rsp+0x40], m1 + movq xm12, [base+subpel_filters+ r4*8] + movq xm13, [base+subpel_filters+ r6*8] + movhps xm12, [base+subpel_filters+ r7*8] + movhps xm13, [base+subpel_filters+ r9*8] + movq xm14, [base+subpel_filters+r10*8] + movq xm15, [base+subpel_filters+r11*8] + movhps xm14, [base+subpel_filters+r13*8] + movhps xm15, [base+subpel_filters+ rX*8] + psrld m1, 10 + vextracti128 xm7, m1, 1 + vextracti128 xm6, m5, 1 + movq [rsp+0xa0], xm1 + movq [rsp+0xa8], xm7 + movq r6, xm1 + pextrq r11, xm1, 1 + movq r9, xm7 + pextrq rX, xm7, 1 + mov r4d, r6d + shr r6, 32 + mov r10d, r11d + shr r11, 32 + mov r7d, r9d + shr r9, 32 + mov r13d, rXd + shr rX, 32 + pshufd xm4, xm5, q2200 + pshufd xm5, xm5, q3311 + pshufd xm7, xm6, q2200 + pshufd xm6, xm6, q3311 + pblendvb xm12, xm2, xm4 + pblendvb xm13, xm2, xm5 + pblendvb xm14, xm2, xm7 + pblendvb xm15, xm2, xm6 + pmovsxbw m12, xm12 + pmovsxbw m13, xm13 + pmovsxbw m14, xm14 + pmovsxbw m15, xm15 + MC_8TAP_SCALED_H 0, 1, 2, 3, 4, 5, 6, 7 ; 0a 1a 0b 1b + mova [rsp+0x60], m0 + MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 7, 8 ; 2a 3a 2b 3b + MC_8TAP_SCALED_H 2, 3, 4, 5, 6, 7, 8, 9 ; 4a 5a 4b 5b + MC_8TAP_SCALED_H 3, 4, 5, 6, 7, 8, 9, 0 ; 6a 7a 6b 7b + mova m0, [rsp+0x60] + vbroadcasti128 m9, [base+subpel_s_shuf8] + mov myd, mym + mov dyd, dym + pshufb m0, m9 ; 01a 01b + pshufb m1, m9 ; 23a 23b + pshufb m2, m9 ; 45a 45b + pshufb m3, m9 ; 67a 67b +.vloop: + and myd, 0x3ff + mov r6d, 64 << 24 + mov r4d, myd + shr r4d, 6 + lea r4d, [t1+r4] + cmovnz r6q, [base+subpel_filters+r4*8] + movq xm9, r6q + punpcklqdq xm9, xm9 + pmovsxbw m9, xm9 + pshufd m8, m9, q0000 + pshufd m7, m9, q1111 + pmaddwd m4, m0, m8 + pmaddwd m5, m1, m7 + pshufd m8, m9, q2222 + pshufd m9, m9, q3333 + pmaddwd m6, m2, m8 + pmaddwd m7, m3, m9 +%if isput + psrldq xm8, xm11, 8 +%endif + paddd m4, [rsp+0x20] + paddd m6, m7 + paddd m4, m5 + paddd m4, m6 +%if isput + psrad m4, xm8 + vextracti128 xm5, m4, 1 + packusdw xm4, xm5 + pminsw xm4, [rsp+0xb0] + mova [dstq], xm4 + add dstq, dsm +%else + psrad m4, 6 + vextracti128 xm5, m4, 1 + packssdw xm4, xm5 + mova [tmpq], xm4 + add tmpq, tmp_stridem +%endif + dec hd + jz .hloop_prep + add myd, dyd + test myd, ~0x3ff + jz .vloop + test myd, 0x400 + mov [rsp+0x60], myd + mov r4d, [rsp+0xa0] + mov r6d, [rsp+0xa4] + mov r7d, [rsp+0xa8] + mov r9d, [rsp+0xac] + jz .skip_line + vbroadcasti128 m9, [base+wswap] + movu xm4, [srcq+ r4*2] + movu xm5, [srcq+ r6*2] + movu xm6, [srcq+ r7*2] + movu xm7, [srcq+ r9*2] + vinserti128 m4, [srcq+r10*2], 1 + vinserti128 m5, [srcq+r11*2], 1 + vinserti128 m6, [srcq+r13*2], 1 + vinserti128 m7, [srcq+ rX*2], 1 + add srcq, ssq + mov myd, [rsp+0x60] + mov dyd, dym + pshufb m0, m9 + pshufb m1, m9 + pshufb m2, m9 + pshufb m3, m9 + pmaddwd m4, m12 + pmaddwd m5, m13 + pmaddwd m6, m14 + pmaddwd m7, m15 + phaddd m4, m5 + phaddd m6, m7 + phaddd m4, m6 + paddd m4, m10 + psrad m4, xm11 + pslld m4, 16 + pblendw m0, m1, 0xaa + pblendw m1, m2, 0xaa + pblendw m2, m3, 0xaa + pblendw m3, m4, 0xaa + jmp .vloop +.skip_line: + mova m0, m1 + mova m1, m2 + mova m2, m3 + MC_8TAP_SCALED_H 3, 10, 4, 5, 6, 7, 8, 9, 1 + vbroadcasti128 m9, [base+subpel_s_shuf8] + mov myd, [rsp+0x60] + mov dyd, dym + pshufb m3, m9 + jmp .vloop + SWAP m1, m12, m10 + SWAP m7, m11 +.dy1: + movzx wd, word [base+%1_8tap_scaled_avx2_dy1_table+wq*2] + add wq, base_reg + jmp wq +%if isput +.dy1_w2: + mov myd, mym + movzx t0d, t0b + sub srcq, 2 + movd xm15, t0d + punpckldq m8, m9, m8 + paddd m10, m8 ; mx+dx*[0-1] + vpbroadcastd xm14, [base+pq_0x40000000+2] + vpbroadcastd xm15, xm15 + pand xm8, xm10, xm6 + psrld xm8, 6 + paddd xm15, xm8 + movd r4d, xm15 + pextrd r6d, xm15, 1 + vbroadcasti128 m5, [base+bdct_lb_q] + vbroadcasti128 m6, [base+subpel_s_shuf2] + vpbroadcastd m15, [base+subpel_filters+r4*8+2] + vpbroadcastd m4, [base+subpel_filters+r6*8+2] + pcmpeqd xm8, xm9 + psrld m10, 10 + paddd m10, m10 + movu xm0, [srcq+ssq*0] + movu xm1, [srcq+ssq*1] + movu xm2, [srcq+ssq*2] + movu xm3, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + shr myd, 6 + mov r4d, 64 << 24 + lea myd, [t1+myq] + cmovnz r4q, [base+subpel_filters+myq*8] + pshufb m10, m5 + paddb m10, m6 + vpblendd xm15, xm4, 0xa + pblendvb xm15, xm14, xm8 + pmovsxbw m15, xm15 + vinserti128 m0, [srcq+ssq*0], 1 + vinserti128 m1, [srcq+ssq*1], 1 + vinserti128 m2, [srcq+ssq*2], 1 + add srcq, ss3q + movq xm6, r4q + pmovsxbw xm6, xm6 + pshufd xm8, xm6, q0000 + pshufd xm9, xm6, q1111 + pshufd xm14, xm6, q2222 + pshufd xm6, xm6, q3333 + REPX {pshufb x, m10}, m0, m1, m2 + pshufb xm3, xm10 + REPX {pmaddwd x, m15}, m0, m1, m2 + pmaddwd xm3, xm15 + phaddd m0, m1 + phaddd m2, m3 + paddd m0, m12 + paddd m2, m12 + psrad m0, xm7 + psrad m2, xm7 + packssdw m0, m2 + vextracti128 xm1, m0, 1 + palignr xm2, xm1, xm0, 4 + pshufd xm4, xm1, q2121 + punpcklwd xm3, xm0, xm2 ; 01 12 + punpckhwd xm0, xm2 ; 23 34 + punpcklwd xm2, xm1, xm4 ; 45 56 +.dy1_w2_loop: + movu xm1, [srcq+ssq*0] + movu xm5, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pshufb xm1, xm10 + pshufb xm5, xm10 + pmaddwd xm1, xm15 + pmaddwd xm5, xm15 + phaddd xm1, xm5 + pmaddwd xm5, xm3, xm8 + mova xm3, xm0 + pmaddwd xm0, xm9 + paddd xm1, xm12 + psrad xm1, xm7 + packssdw xm1, xm1 + paddd xm5, xm0 + mova xm0, xm2 + pmaddwd xm2, xm14 + paddd xm5, xm2 + palignr xm2, xm1, xm4, 12 + punpcklwd xm2, xm1 ; 67 78 + pmaddwd xm4, xm2, xm6 + paddd xm5, xm13 + paddd xm5, xm4 + mova xm4, xm1 + psrldq xm1, xm7, 8 + psrad xm5, xm1 + packusdw xm5, xm5 + pminsw xm5, xm11 + movd [dstq+dsq*0], xm5 + pextrd [dstq+dsq*1], xm5, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .dy1_w2_loop + RET +%endif +.dy1_w4: + mov myd, mym +%if isput + mova [rsp+0x50], xm11 +%endif + mova [rsp+0x00], m12 + mova [rsp+0x20], m13 + mova [rsp+0x40], xm7 + vbroadcasti128 m7, [base+rescale_mul] + movzx t0d, t0b + sub srcq, 2 + movd xm15, t0d + pmaddwd m8, m7 + vpbroadcastq m2, [base+pq_0x40000000+1] + vpbroadcastd xm15, xm15 + SWAP m13, m10 + paddd m13, m8 ; mx+dx*[0-3] + pand m6, m13 + psrld m6, 6 + paddd xm15, xm6 + movd r4d, xm15 + pextrd r6d, xm15, 1 + pextrd r11d, xm15, 2 + pextrd r13d, xm15, 3 + vbroadcasti128 m5, [base+bdct_lb_q+ 0] + vbroadcasti128 m1, [base+bdct_lb_q+16] + vbroadcasti128 m4, [base+subpel_s_shuf2] + vpbroadcastd xm14, [base+subpel_filters+r4*8+2] + vpbroadcastd xm7, [base+subpel_filters+r6*8+2] + vpbroadcastd xm15, [base+subpel_filters+r11*8+2] + vpbroadcastd xm8, [base+subpel_filters+r13*8+2] + pcmpeqd m6, m9 + punpckldq m10, m6, m6 + punpckhdq m6, m6 + psrld m13, 10 + paddd m13, m13 + vpblendd xm14, xm7, 0xa + vpblendd xm15, xm8, 0xa + pmovsxbw m14, xm14 + pmovsxbw m15, xm15 + pblendvb m14, m2, m10 + pblendvb m15, m2, m6 + pextrd r4, xm13, 2 + pshufb m12, m13, m5 + pshufb m13, m1 + lea r6, [r4+ssq*2] + lea r11, [r4+ssq*1] + lea r13, [r4+ss3q ] + movu xm0, [srcq+ssq*0] + movu xm7, [srcq+r4 ] + movu xm1, [srcq+ssq*2] + movu xm8, [srcq+r6 ] + vinserti128 m0, [srcq+ssq*1], 1 ; 0 1 + vinserti128 m7, [srcq+r11 ], 1 + vinserti128 m1, [srcq+ss3q ], 1 ; 2 3 + vinserti128 m8, [srcq+r13 ], 1 + lea srcq, [srcq+ssq*4] + movu xm2, [srcq+ssq*0] + movu xm9, [srcq+r4 ] + movu xm3, [srcq+ssq*2] ; 6 _ + movu xm10, [srcq+r6 ] + vinserti128 m2, [srcq+ssq*1], 1 ; 4 5 + vinserti128 m9, [srcq+r11 ], 1 + lea srcq, [srcq+ss3q ] + vpbroadcastb m5, xm13 + psubb m13, m5 + paddb m12, m4 + paddb m13, m4 + mova m5, [rsp+0x00] + movd xm6, [rsp+0x40] + pshufb m0, m12 + pshufb m1, m12 + pmaddwd m0, m14 + pmaddwd m1, m14 + pshufb m7, m13 + pshufb m8, m13 + pmaddwd m7, m15 + pmaddwd m8, m15 + pshufb m2, m12 + pshufb xm3, xm12 + pmaddwd m2, m14 + pmaddwd xm3, xm14 + pshufb m9, m13 + pshufb xm10, xm13 + pmaddwd m9, m15 + pmaddwd xm10, xm15 + phaddd m0, m7 + phaddd m1, m8 + phaddd m2, m9 + phaddd xm3, xm10 + paddd m0, m5 + paddd m1, m5 + paddd m2, m5 + paddd xm3, xm5 + psrad m0, xm6 + psrad m1, xm6 + psrad m2, xm6 + psrad xm3, xm6 + vperm2i128 m4, m0, m1, 0x21 ; 1 2 + vperm2i128 m5, m1, m2, 0x21 ; 3 4 + vperm2i128 m6, m2, m3, 0x21 ; 5 6 + shr myd, 6 + mov r13d, 64 << 24 + lea myd, [t1+myq] + cmovnz r13q, [base+subpel_filters+myq*8] + pslld m4, 16 + pslld m5, 16 + pslld m6, 16 + pblendw m0, m4, 0xaa ; 01 12 + pblendw m1, m5, 0xaa ; 23 34 + pblendw m2, m6, 0xaa ; 45 56 + movq xm10, r13q + punpcklqdq xm10, xm10 + pmovsxbw m10, xm10 + pshufd m7, m10, q0000 + pshufd m8, m10, q1111 + pshufd m9, m10, q2222 + pshufd m10, m10, q3333 +.dy1_w4_loop: + movu xm11, [srcq+ssq*0] + movu xm6, [srcq+r4 ] + vinserti128 m11, [srcq+ssq*1], 1 + vinserti128 m6, [srcq+r11 ], 1 + lea srcq, [srcq+ssq*2] + pmaddwd m4, m0, m7 + pmaddwd m5, m1, m8 + pshufb m11, m12 + pshufb m6, m13 + pmaddwd m11, m14 + pmaddwd m6, m15 + paddd m4, [rsp+0x20] + phaddd m11, m6 + pmaddwd m6, m2, m9 + paddd m11, [rsp+0x00] + psrad m11, [rsp+0x40] + mova m0, m1 + mova m1, m2 + paddd m5, m6 + paddd m4, m5 + vinserti128 m2, m3, xm11, 1 + pslld m3, m11, 16 + pblendw m2, m3, 0xaa ; 67 78 + pmaddwd m5, m2, m10 + vextracti128 xm3, m11, 1 + paddd m4, m5 +%if isput + psrad m4, [rsp+0x48] + vextracti128 xm5, m4, 1 + packusdw xm4, xm5 + pminsw xm4, [rsp+0x50] + movq [dstq+dsq*0], xm4 + movhps [dstq+dsq*1], xm4 + lea dstq, [dstq+dsq*2] +%else + psrad m4, 6 + vextracti128 xm5, m4, 1 + packssdw xm4, xm5 + mova [tmpq], xm4 + add tmpq, 16 +%endif + sub hd, 2 + jg .dy1_w4_loop + MC_8TAP_SCALED_RET + SWAP m10, m13 +.dy1_w8: + mov dword [rsp+0xa0], 1 + movifprep tmp_stridem, 16 + jmp .dy1_w_start +.dy1_w16: + mov dword [rsp+0xa0], 2 + movifprep tmp_stridem, 32 + jmp .dy1_w_start +.dy1_w32: + mov dword [rsp+0xa0], 4 + movifprep tmp_stridem, 64 + jmp .dy1_w_start +.dy1_w64: + mov dword [rsp+0xa0], 8 + movifprep tmp_stridem, 128 + jmp .dy1_w_start +.dy1_w128: + mov dword [rsp+0xa0], 16 + movifprep tmp_stridem, 256 +.dy1_w_start: + SWAP m10, m12, m1 + SWAP m11, m7 + ; m1=mx, m7=pxmax, m10=h_rnd, m11=h_sh, m12=free + mov myd, mym +%if isput + %if required_stack_alignment > STACK_ALIGNMENT + %define dsm [rsp+0xb8] + %endif + movifnidn dsm, dsq + mova [rsp+0xc0], xm7 +%else + %if UNIX64 + %define hm [rsp+0xb8] + %endif +%endif + mova [rsp+0x00], m10 + mova [rsp+0x20], m13 + mova [rsp+0x40], xm11 + shr t0d, 16 + sub srcq, 6 + shr myd, 6 + mov r4d, 64 << 24 + lea myd, [t1+myq] + cmovnz r4q, [base+subpel_filters+myq*8] + pmaddwd m8, [base+rescale_mul2] + movd xm15, t0d + mov [rsp+0xa4], t0d + mov [rsp+0xa8], srcq + mov [rsp+0xb0], r0q ; dstq / tmpq +%if UNIX64 + mov hm, hd +%endif + shl dword dxm, 3 ; dx*8 + vpbroadcastd m15, xm15 + paddd m1, m8 ; mx+dx*[0-7] + movq xm0, r4q + pmovsxbw xm0, xm0 + mova [rsp+0x50], xm0 + jmp .dy1_hloop +.dy1_hloop_prep: + dec dword [rsp+0xa0] + jz .ret + add qword [rsp+0xb0], 16 + mov hd, hm + vpbroadcastd m8, dxm + vpbroadcastd m6, [base+pd_0x3ff] + paddd m1, m8, [rsp+0x60] + vpbroadcastd m15, [rsp+0xa4] + pxor m9, m9 + mov srcq, [rsp+0xa8] + mov r0q, [rsp+0xb0] ; dstq / tmpq + mova m10, [rsp+0x00] + mova xm11, [rsp+0x40] +.dy1_hloop: + vpbroadcastq xm2, [base+pq_0x40000000] + pand m5, m1, m6 + psrld m5, 6 + paddd m15, m5 + pcmpeqd m5, m9 + vextracti128 xm7, m15, 1 + movq r6, xm15 + pextrq r9, xm15, 1 + movq r11, xm7 + pextrq rX, xm7, 1 + mov r4d, r6d + shr r6, 32 + mov r7d, r9d + shr r9, 32 + mov r10d, r11d + shr r11, 32 + mov r13d, rXd + shr rX, 32 + mova [rsp+0x60], m1 + movq xm12, [base+subpel_filters+ r4*8] + movq xm13, [base+subpel_filters+ r6*8] + movhps xm12, [base+subpel_filters+ r7*8] + movhps xm13, [base+subpel_filters+ r9*8] + movq xm14, [base+subpel_filters+r10*8] + movq xm15, [base+subpel_filters+r11*8] + movhps xm14, [base+subpel_filters+r13*8] + movhps xm15, [base+subpel_filters+ rX*8] + psrld m1, 10 + vextracti128 xm7, m1, 1 + vextracti128 xm6, m5, 1 + movq r6, xm1 + pextrq r11, xm1, 1 + movq r9, xm7 + pextrq rX, xm7, 1 + mov r4d, r6d + shr r6, 32 + mov r10d, r11d + shr r11, 32 + mov r7d, r9d + shr r9, 32 + mov r13d, rXd + shr rX, 32 + pshufd xm4, xm5, q2200 + pshufd xm5, xm5, q3311 + pshufd xm7, xm6, q2200 + pshufd xm6, xm6, q3311 + pblendvb xm12, xm2, xm4 + pblendvb xm13, xm2, xm5 + pblendvb xm14, xm2, xm7 + pblendvb xm15, xm2, xm6 + pmovsxbw m12, xm12 + pmovsxbw m13, xm13 + pmovsxbw m14, xm14 + pmovsxbw m15, xm15 + MC_8TAP_SCALED_H 0, 1, 2, 3, 4, 5, 6, 7 ; 0a 1a 0b 1b + mova [rsp+0x80], m0 + MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 7, 8 ; 2a 3a 2b 3b + MC_8TAP_SCALED_H 2, 3, 4, 5, 6, 7, 8, 9 ; 4a 5a 4b 5b + MC_8TAP_SCALED_H 3, 4, 5, 6, 7, 8, 9, 0 ; 6a 7a 6b 7b + mova m0, [rsp+0x80] + vbroadcasti128 m7, [base+subpel_s_shuf8] + vpbroadcastd m8, [rsp+0x50] + vpbroadcastd m9, [rsp+0x54] + vpbroadcastd m10, [rsp+0x58] + vpbroadcastd m11, [rsp+0x5c] + pshufb m0, m7 ; 01a 01b + pshufb m1, m7 ; 23a 23b + pshufb m2, m7 ; 45a 45b + pshufb m3, m7 ; 67a 67b +.dy1_vloop: + pmaddwd m4, m0, m8 + pmaddwd m5, m1, m9 + pmaddwd m6, m2, m10 + pmaddwd m7, m3, m11 + paddd m4, [rsp+0x20] + paddd m6, m7 + paddd m4, m5 + paddd m4, m6 +%if isput + psrad m4, [rsp+0x48] + vextracti128 xm5, m4, 1 + packusdw xm4, xm5 + pminsw xm4, [rsp+0xc0] + mova [dstq], xm4 + add dstq, dsm +%else + psrad m4, 6 + vextracti128 xm5, m4, 1 + packssdw xm4, xm5 + mova [tmpq], xm4 + add tmpq, tmp_stridem +%endif + dec hd + jz .dy1_hloop_prep + vbroadcasti128 m7, [base+wswap] + pshufb m0, m7 + pshufb m1, m7 + pshufb m2, m7 + pshufb m3, m7 + movu xm4, [srcq+ r4*2] + movu xm5, [srcq+ r6*2] + movu xm6, [srcq+ r7*2] + movu xm7, [srcq+ r9*2] + vinserti128 m4, [srcq+r10*2], 1 + vinserti128 m5, [srcq+r11*2], 1 + vinserti128 m6, [srcq+r13*2], 1 + vinserti128 m7, [srcq+ rX*2], 1 + add srcq, ssq + pmaddwd m4, m12 + pmaddwd m5, m13 + pmaddwd m6, m14 + pmaddwd m7, m15 + phaddd m4, m5 + phaddd m6, m7 + phaddd m4, m6 + paddd m4, [rsp+0x00] + psrad m4, [rsp+0x40] + pslld m4, 16 + pblendw m0, m1, 0xaa + pblendw m1, m2, 0xaa + pblendw m2, m3, 0xaa + pblendw m3, m4, 0xaa + jmp .dy1_vloop + SWAP m1, m12, m10 + SWAP m7, m11 +.dy2: + movzx wd, word [base+%1_8tap_scaled_avx2_dy2_table+wq*2] + add wq, base_reg + jmp wq +%if isput +.dy2_w2: + mov myd, mym + movzx t0d, t0b + sub srcq, 2 + movd xm15, t0d + punpckldq m8, m9, m8 + paddd m10, m8 ; mx+dx*[0-1] + vpbroadcastd xm14, [base+pq_0x40000000+2] + vpbroadcastd xm15, xm15 + pand xm8, xm10, xm6 + psrld xm8, 6 + paddd xm15, xm8 + movd r4d, xm15 + pextrd r6d, xm15, 1 + vbroadcasti128 m5, [base+bdct_lb_q] + vbroadcasti128 m6, [base+subpel_s_shuf2] + vpbroadcastd xm15, [base+subpel_filters+r4*8+2] + vpbroadcastd xm4, [base+subpel_filters+r6*8+2] + pcmpeqd xm8, xm9 + psrld m10, 10 + paddd m10, m10 + movu xm0, [srcq+ssq*0] + movu xm1, [srcq+ssq*2] + movu xm2, [srcq+ssq*4] + pshufb m10, m5 + paddb m10, m6 + vpblendd xm15, xm4, 0xa + pblendvb xm15, xm14, xm8 + pmovsxbw m15, xm15 + vinserti128 m0, [srcq+ssq*1], 1 ; 0 1 + vinserti128 m1, [srcq+ss3q ], 1 ; 2 3 + lea srcq, [srcq+ssq*4] + vinserti128 m2, [srcq+ssq*1], 1 ; 4 5 + lea srcq, [srcq+ssq*2] + shr myd, 6 + mov r4d, 64 << 24 + lea myd, [t1+myq] + cmovnz r4q, [base+subpel_filters+myq*8] + pshufb m0, m10 + pshufb m1, m10 + pshufb m2, m10 + pmaddwd m0, m15 + pmaddwd m1, m15 + pmaddwd m2, m15 + movq xm6, r4q + pmovsxbw xm6, xm6 + phaddd m0, m1 + phaddd m1, m2 + paddd m0, m12 + paddd m1, m12 + psrad m0, xm7 + psrad m1, xm7 + packssdw m0, m1 ; 0 2 2 4 1 3 3 5 + vextracti128 xm1, m0, 1 + pshufd xm8, xm6, q0000 + pshufd xm9, xm6, q1111 + pshufd xm14, xm6, q2222 + pshufd xm6, xm6, q3333 + punpcklwd xm2, xm0, xm1 ; 01 23 + punpckhwd xm1, xm0, xm1 ; 23 45 +.dy2_w2_loop: + movu xm3, [srcq+ssq*0] + movu xm5, [srcq+ssq*2] + vinserti128 m3, [srcq+ssq*1], 1 ; 6 7 + vinserti128 m5, [srcq+ss3q ], 1 ; 8 9 + lea srcq, [srcq+ssq*4] + pmaddwd xm4, xm2, xm8 + pmaddwd xm1, xm9 + pshufb m3, m10 + pshufb m5, m10 + pmaddwd m3, m15 + pmaddwd m5, m15 + phaddd m3, m5 + paddd xm4, xm1 + paddd m3, m12 + psrad m3, xm7 + packssdw m3, m3 + pshufd m3, m3, q2100 + palignr m0, m3, m0, 12 ; 4 6 6 8 5 7 7 9 + vextracti128 xm1, m0, 1 + punpcklwd xm2, xm0, xm1 ; 45 67 + punpckhwd xm1, xm0, xm1 ; 67 89 + pmaddwd xm3, xm2, xm14 + pmaddwd xm5, xm1, xm6 + paddd xm4, xm13 + paddd xm4, xm3 + psrldq xm3, xm7, 8 + paddd xm4, xm5 + psrad xm4, xm3 + packusdw xm4, xm4 + pminsw xm4, xm11 + movd [dstq+dsq*0], xm4 + pextrd [dstq+dsq*1], xm4, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .dy2_w2_loop + RET +%endif +.dy2_w4: + mov myd, mym +%if isput + mova [rsp+0x50], xm11 +%endif + mova [rsp+0x00], m12 + mova [rsp+0x20], m13 + mova [rsp+0x40], xm7 + vbroadcasti128 m7, [base+rescale_mul] + movzx t0d, t0b + sub srcq, 2 + movd xm15, t0d + pmaddwd m8, m7 + vpbroadcastq m2, [base+pq_0x40000000+1] + vpbroadcastd xm15, xm15 + SWAP m13, m10 + paddd m13, m8 ; mx+dx*[0-3] + pand m6, m13 + psrld m6, 6 + paddd xm15, xm6 + movd r4d, xm15 + pextrd r6d, xm15, 1 + pextrd r11d, xm15, 2 + pextrd r13d, xm15, 3 + vbroadcasti128 m5, [base+bdct_lb_q+ 0] + vbroadcasti128 m1, [base+bdct_lb_q+16] + vbroadcasti128 m4, [base+subpel_s_shuf2] + vpbroadcastd xm14, [base+subpel_filters+r4*8+2] + vpbroadcastd xm7, [base+subpel_filters+r6*8+2] + vpbroadcastd xm15, [base+subpel_filters+r11*8+2] + vpbroadcastd xm8, [base+subpel_filters+r13*8+2] + shr myd, 6 + mov r13d, 64 << 24 + lea myd, [t1+myq] + cmovnz r13q, [base+subpel_filters+myq*8] + pcmpeqd m6, m9 + punpckldq m11, m6, m6 + punpckhdq m6, m6 + psrld m13, 10 + paddd m13, m13 + vpblendd xm14, xm7, 0xa + vpblendd xm15, xm8, 0xa + pmovsxbw m14, xm14 + pmovsxbw m15, xm15 + movq xm10, r13q + pblendvb m14, m2, m11 + pblendvb m15, m2, m6 + pextrd r4, xm13, 2 + pshufb m12, m13, m5 + pshufb m13, m1 + lea r6, [r4+ssq*1] + lea r11, [r4+ssq*2] + lea r13, [r4+ss3q ] + movu xm0, [srcq+ssq*0] + movu xm7, [srcq+r4 ] + movu xm1, [srcq+ssq*1] + movu xm8, [srcq+r6 ] + vinserti128 m0, [srcq+ssq*2], 1 ; 0 2 + vinserti128 m7, [srcq+r11 ], 1 + vinserti128 m1, [srcq+ss3q ], 1 ; 1 3 + vinserti128 m8, [srcq+r13 ], 1 + lea srcq, [srcq+ssq*4] + movu xm2, [srcq+ssq*0] + movu xm9, [srcq+r4 ] + vinserti128 m2, [srcq+ssq*1], 1 ; 4 5 + vinserti128 m9, [srcq+r6 ], 1 + lea srcq, [srcq+ssq*2] + vpbroadcastb m5, xm13 + psubb m13, m5 + paddb m12, m4 + paddb m13, m4 + mova m5, [rsp+0x00] + movd xm6, [rsp+0x40] + pshufb m0, m12 + pshufb m1, m12 + pshufb m2, m12 + pmaddwd m0, m14 + pmaddwd m1, m14 + pmaddwd m2, m14 + pshufb m7, m13 + pshufb m8, m13 + pshufb m9, m13 + pmaddwd m7, m15 + pmaddwd m8, m15 + pmaddwd m9, m15 + punpcklqdq xm10, xm10 + pmovsxbw m10, xm10 + phaddd m0, m7 + phaddd m1, m8 + phaddd m2, m9 + paddd m0, m5 + paddd m1, m5 + paddd m2, m5 + psrad m0, xm6 + psrad m1, xm6 + psrad m2, xm6 + vperm2i128 m3, m0, m2, 0x21 ; 2 4 + vperm2i128 m2, m1, 0x13 ; 3 5 + pshufd m7, m10, q0000 + pshufd m8, m10, q1111 + pshufd m9, m10, q2222 + pshufd m10, m10, q3333 + packssdw m0, m3 ; 0 2 2 4 + packssdw m1, m2 ; 1 3 3 5 + punpckhwd m2, m0, m1 ; 23 45 + punpcklwd m0, m1 ; 01 23 +.dy2_w4_loop: + movu xm1, [srcq+ssq*0] + movu xm6, [srcq+r4 ] + movu xm3, [srcq+ssq*1] + movu xm11, [srcq+r6 ] + vinserti128 m1, [srcq+ssq*2], 1 ; 6 8 + vinserti128 m6, [srcq+r11 ], 1 + vinserti128 m3, [srcq+ss3q ], 1 ; 7 9 + vinserti128 m11, [srcq+r13 ], 1 + lea srcq, [srcq+ssq*4] + pmaddwd m4, m0, m7 + pmaddwd m5, m2, m8 + pshufb m1, m12 + pshufb m3, m12 + pmaddwd m1, m14 + pmaddwd m3, m14 + mova m0, [rsp+0x00] + pshufb m6, m13 + pshufb m11, m13 + pmaddwd m6, m15 + pmaddwd m11, m15 + paddd m4, m5 + movd xm5, [rsp+0x40] + phaddd m1, m6 + phaddd m3, m11 + paddd m1, m0 + paddd m3, m0 + psrad m1, xm5 + psrad m3, xm5 + pslld m3, 16 + pblendw m1, m3, 0xaa ; 67 89 + vperm2i128 m0, m2, m1, 0x21 ; 45 67 + paddd m4, [rsp+0x20] + mova m2, m1 + pmaddwd m5, m0, m9 + pmaddwd m6, m2, m10 + paddd m4, m5 + paddd m4, m6 +%if isput + psrad m4, [rsp+0x48] + vextracti128 xm5, m4, 1 + packusdw xm4, xm5 + pminsw xm4, [rsp+0x50] + movq [dstq+dsq*0], xm4 + movhps [dstq+dsq*1], xm4 + lea dstq, [dstq+dsq*2] +%else + psrad m4, 6 + vextracti128 xm5, m4, 1 + packssdw xm4, xm5 + mova [tmpq], xm4 + add tmpq, 16 +%endif + sub hd, 2 + jg .dy2_w4_loop + MC_8TAP_SCALED_RET + SWAP m10, m13 +.dy2_w8: + mov dword [rsp+0xa0], 1 + movifprep tmp_stridem, 16 + jmp .dy2_w_start +.dy2_w16: + mov dword [rsp+0xa0], 2 + movifprep tmp_stridem, 32 + jmp .dy2_w_start +.dy2_w32: + mov dword [rsp+0xa0], 4 + movifprep tmp_stridem, 64 + jmp .dy2_w_start +.dy2_w64: + mov dword [rsp+0xa0], 8 + movifprep tmp_stridem, 128 + jmp .dy2_w_start +.dy2_w128: + mov dword [rsp+0xa0], 16 + movifprep tmp_stridem, 256 +.dy2_w_start: + SWAP m10, m12, m1 + SWAP m11, m7 + ; m1=mx, m7=pxmax, m10=h_rnd, m11=h_sh, m12=free + mov myd, mym +%if isput + movifnidn dsm, dsq + mova [rsp+0xc0], xm7 +%endif + mova [rsp+0x00], m10 + mova [rsp+0x20], m13 + mova [rsp+0x40], xm11 + shr t0d, 16 + sub srcq, 6 + shr myd, 6 + mov r4d, 64 << 24 + lea myd, [t1+myq] + cmovnz r4q, [base+subpel_filters+myq*8] + pmaddwd m8, [base+rescale_mul2] + movd xm15, t0d + mov [rsp+0xa4], t0d + mov [rsp+0xa8], srcq + mov [rsp+0xb0], r0q ; dstq / tmpq +%if UNIX64 + mov hm, hd +%endif + shl dword dxm, 3 ; dx*8 + vpbroadcastd m15, xm15 + paddd m1, m8 ; mx+dx*[0-7] + movq xm0, r4q + pmovsxbw xm0, xm0 + mova [rsp+0x50], xm0 + jmp .dy2_hloop +.dy2_hloop_prep: + dec dword [rsp+0xa0] + jz .ret + add qword [rsp+0xb0], 16 + mov hd, hm + vpbroadcastd m8, dxm + vpbroadcastd m6, [base+pd_0x3ff] + paddd m1, m8, [rsp+0x60] + vpbroadcastd m15, [rsp+0xa4] + pxor m9, m9 + mov srcq, [rsp+0xa8] + mov r0q, [rsp+0xb0] ; dstq / tmpq + mova m10, [rsp+0x00] + mova xm11, [rsp+0x40] +.dy2_hloop: + vpbroadcastq xm2, [base+pq_0x40000000] + pand m5, m1, m6 + psrld m5, 6 + paddd m15, m5 + pcmpeqd m5, m9 + vextracti128 xm7, m15, 1 + movq r6, xm15 + pextrq r9, xm15, 1 + movq r11, xm7 + pextrq rX, xm7, 1 + mov r4d, r6d + shr r6, 32 + mov r7d, r9d + shr r9, 32 + mov r10d, r11d + shr r11, 32 + mov r13d, rXd + shr rX, 32 + mova [rsp+0x60], m1 + movq xm12, [base+subpel_filters+ r4*8] + movq xm13, [base+subpel_filters+ r6*8] + movhps xm12, [base+subpel_filters+ r7*8] + movhps xm13, [base+subpel_filters+ r9*8] + movq xm14, [base+subpel_filters+r10*8] + movq xm15, [base+subpel_filters+r11*8] + movhps xm14, [base+subpel_filters+r13*8] + movhps xm15, [base+subpel_filters+ rX*8] + psrld m1, 10 + vextracti128 xm7, m1, 1 + vextracti128 xm6, m5, 1 + movq r6, xm1 + pextrq r11, xm1, 1 + movq r9, xm7 + pextrq rX, xm7, 1 + mov r4d, r6d + shr r6, 32 + mov r10d, r11d + shr r11, 32 + mov r7d, r9d + shr r9, 32 + mov r13d, rXd + shr rX, 32 + pshufd xm4, xm5, q2200 + pshufd xm5, xm5, q3311 + pshufd xm7, xm6, q2200 + pshufd xm6, xm6, q3311 + pblendvb xm12, xm2, xm4 + pblendvb xm13, xm2, xm5 + pblendvb xm14, xm2, xm7 + pblendvb xm15, xm2, xm6 + pmovsxbw m12, xm12 + pmovsxbw m13, xm13 + pmovsxbw m14, xm14 + pmovsxbw m15, xm15 + MC_8TAP_SCALED_H 0, 1, 2, 3, 4, 5, 6, 7 ; 0a 1a 0b 1b + mova [rsp+0x80], m0 + MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 7, 8 ; 2a 3a 2b 3b + MC_8TAP_SCALED_H 2, 3, 4, 5, 6, 7, 8, 9 ; 4a 5a 4b 5b + MC_8TAP_SCALED_H 3, 4, 5, 6, 7, 8, 9, 0 ; 6a 7a 6b 7b + mova m0, [rsp+0x80] + vbroadcasti128 m7, [base+subpel_s_shuf8] + vpbroadcastd m8, [rsp+0x50] + vpbroadcastd m9, [rsp+0x54] + vpbroadcastd m10, [rsp+0x58] + vpbroadcastd m11, [rsp+0x5c] + pshufb m0, m7 ; 01a 01b + pshufb m1, m7 ; 23a 23b + pshufb m2, m7 ; 45a 45b + pshufb m3, m7 ; 67a 67b +.dy2_vloop: + pmaddwd m4, m0, m8 + pmaddwd m5, m1, m9 + pmaddwd m6, m2, m10 + pmaddwd m7, m3, m11 + paddd m4, [rsp+0x20] + paddd m6, m7 + paddd m4, m5 + paddd m4, m6 +%if isput + psrad m4, [rsp+0x48] + vextracti128 xm5, m4, 1 + packusdw xm4, xm5 + pminsw xm4, [rsp+0xc0] + mova [dstq], xm4 + add dstq, dsm +%else + psrad m4, 6 + vextracti128 xm5, m4, 1 + packssdw xm4, xm5 + mova [tmpq], xm4 + add tmpq, tmp_stridem +%endif + dec hd + jz .dy2_hloop_prep + mova m0, m1 + mova m1, m2 + mova m2, m3 + movu xm3, [srcq+ r4*2] + movu xm4, [srcq+ r6*2] + movu xm5, [srcq+ r7*2] + movu xm6, [srcq+ r9*2] + vinserti128 m3, [srcq+r10*2], 1 + vinserti128 m4, [srcq+r11*2], 1 + vinserti128 m5, [srcq+r13*2], 1 + vinserti128 m6, [srcq+ rX*2], 1 + add srcq, ssq + pmaddwd m3, m12 + pmaddwd m4, m13 + pmaddwd m5, m14 + pmaddwd m6, m15 + phaddd m3, m4 + phaddd m5, m6 + phaddd m3, m5 + movu xm4, [srcq+ r4*2] + movu xm5, [srcq+ r6*2] + movu xm6, [srcq+ r7*2] + movu xm7, [srcq+ r9*2] + vinserti128 m4, [srcq+r10*2], 1 + vinserti128 m5, [srcq+r11*2], 1 + vinserti128 m6, [srcq+r13*2], 1 + vinserti128 m7, [srcq+ rX*2], 1 + add srcq, ssq + pmaddwd m4, m12 + pmaddwd m5, m13 + pmaddwd m6, m14 + pmaddwd m7, m15 + phaddd m4, m5 + phaddd m6, m7 + mova m5, [rsp+0x00] + movd xm7, [rsp+0x40] + phaddd m4, m6 + paddd m3, m5 + paddd m4, m5 + psrad m3, xm7 + psrad m4, xm7 + pslld m4, 16 + pblendw m3, m4, 0xaa + jmp .dy2_vloop +.ret: + MC_8TAP_SCALED_RET 0 +%undef isput +%undef isprep +%endmacro + +%macro BILIN_SCALED_FN 1 +cglobal %1_bilin_scaled_16bpc + mov t0d, (5*15 << 16) | 5*15 + mov t1d, t0d + jmp mangle(private_prefix %+ _%1_8tap_scaled_16bpc %+ SUFFIX) +%endmacro + +%if WIN64 +DECLARE_REG_TMP 6, 5 +%else +DECLARE_REG_TMP 6, 8 +%endif + +%define PUT_8TAP_SCALED_FN FN put_8tap_scaled, +BILIN_SCALED_FN put +PUT_8TAP_SCALED_FN sharp, SHARP, SHARP +PUT_8TAP_SCALED_FN sharp_smooth, SHARP, SMOOTH +PUT_8TAP_SCALED_FN smooth_sharp, SMOOTH, SHARP +PUT_8TAP_SCALED_FN smooth, SMOOTH, SMOOTH +PUT_8TAP_SCALED_FN sharp_regular, SHARP, REGULAR +PUT_8TAP_SCALED_FN regular_sharp, REGULAR, SHARP +PUT_8TAP_SCALED_FN smooth_regular, SMOOTH, REGULAR +PUT_8TAP_SCALED_FN regular_smooth, REGULAR, SMOOTH +PUT_8TAP_SCALED_FN regular, REGULAR, REGULAR +MC_8TAP_SCALED put + +%if WIN64 +DECLARE_REG_TMP 5, 4 +%else +DECLARE_REG_TMP 6, 7 +%endif + +%define PREP_8TAP_SCALED_FN FN prep_8tap_scaled, +BILIN_SCALED_FN prep +PREP_8TAP_SCALED_FN sharp, SHARP, SHARP +PREP_8TAP_SCALED_FN sharp_smooth, SHARP, SMOOTH +PREP_8TAP_SCALED_FN smooth_sharp, SMOOTH, SHARP +PREP_8TAP_SCALED_FN smooth, SMOOTH, SMOOTH +PREP_8TAP_SCALED_FN sharp_regular, SHARP, REGULAR +PREP_8TAP_SCALED_FN regular_sharp, REGULAR, SHARP +PREP_8TAP_SCALED_FN smooth_regular, SMOOTH, REGULAR +PREP_8TAP_SCALED_FN regular_smooth, REGULAR, SMOOTH +PREP_8TAP_SCALED_FN regular, REGULAR, REGULAR +MC_8TAP_SCALED prep + %macro WARP_V 5 ; dst, 01, 23, 45, 67 lea tmp1d, [myq+deltaq*4] lea tmp2d, [myq+deltaq*1] shr myd, 10 shr tmp1d, 10 @@ -4001,6 +5778,125 @@ jl .top_x_loop .end: RET +cglobal resize_16bpc, 6, 12, 16, dst, dst_stride, src, src_stride, \ + dst_w, h, src_w, dx, mx0, pxmax + sub dword mx0m, 4<<14 + sub dword src_wm, 8 + vpbroadcastd m5, dxm + vpbroadcastd m8, mx0m + vpbroadcastd m6, src_wm + DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, h, x, picptr, _, pxmax + LEA r7, $$ +%define base r7-$$ + vpbroadcastd m3, [base+pd_64] + vpbroadcastw xm7, pxmaxm + pmaddwd m2, m5, [base+rescale_mul] ; dx*[0,1,2,3,4,5,6,7] + pslld m5, 3 ; dx*8 + pslld m6, 14 + paddd m8, m2 ; mx+[0..7]*dx +.loop_y: + xor xd, xd + mova m4, m8 ; per-line working version of mx +.loop_x: + vpbroadcastd m10, [base+pd_63] + pxor m2, m2 + pmaxsd m0, m4, m2 + psrad m9, m4, 8 ; filter offset (unmasked) + pminsd m0, m6 ; iclip(mx, 0, src_w-8) + psubd m1, m4, m0 ; pshufb offset + psrad m0, 14 ; clipped src_x offset + psrad m1, 14 ; pshufb edge_emu offset + pand m9, m10 ; filter offset (masked) + ; load source pixels + movd r8d, xm0 + pextrd r9d, xm0, 1 + pextrd r10d, xm0, 2 + pextrd r11d, xm0, 3 + vextracti128 xm0, m0, 1 + movu xm10, [srcq+r8*2] + movu xm11, [srcq+r9*2] + movu xm12, [srcq+r10*2] + movu xm13, [srcq+r11*2] + movd r8d, xm0 + pextrd r9d, xm0, 1 + pextrd r10d, xm0, 2 + pextrd r11d, xm0, 3 + vinserti128 m10, [srcq+r8*2], 1 + vinserti128 m11, [srcq+r9*2], 1 + vinserti128 m12, [srcq+r10*2], 1 + vinserti128 m13, [srcq+r11*2], 1 + ptest m1, m1 + jz .filter + movq r9, xm1 + pextrq r11, xm1, 1 + movsxd r8, r9d + sar r9, 32 + movsxd r10, r11d + sar r11, 32 + vextracti128 xm1, m1, 1 + movu xm14, [base+resize_shuf+8+r8*2] + movu xm15, [base+resize_shuf+8+r9*2] + movu xm0, [base+resize_shuf+8+r10*2] + movu xm2, [base+resize_shuf+8+r11*2] + movq r9, xm1 + pextrq r11, xm1, 1 + movsxd r8, r9d + sar r9, 32 + movsxd r10, r11d + sar r11, 32 + vinserti128 m14, [base+resize_shuf+8+r8*2], 1 + vinserti128 m15, [base+resize_shuf+8+r9*2], 1 + vinserti128 m0, [base+resize_shuf+8+r10*2], 1 + vinserti128 m2, [base+resize_shuf+8+r11*2], 1 + pshufb m10, m14 + pshufb m11, m15 + pshufb m12, m0 + pshufb m13, m2 +.filter: + movd r8d, xm9 + pextrd r9d, xm9, 1 + pextrd r10d, xm9, 2 + pextrd r11d, xm9, 3 + vextracti128 xm9, m9, 1 + movq xm14, [base+resize_filter+r8*8] + movq xm15, [base+resize_filter+r9*8] + movq xm0, [base+resize_filter+r10*8] + movq xm2, [base+resize_filter+r11*8] + movd r8d, xm9 + pextrd r9d, xm9, 1 + pextrd r10d, xm9, 2 + pextrd r11d, xm9, 3 + movhps xm14, [base+resize_filter+r8*8] + movhps xm15, [base+resize_filter+r9*8] + movhps xm0, [base+resize_filter+r10*8] + movhps xm2, [base+resize_filter+r11*8] + pmovsxbw m14, xm14 + pmovsxbw m15, xm15 + pmovsxbw m0, xm0 + pmovsxbw m2, xm2 + pmaddwd m10, m14 + pmaddwd m11, m15 + pmaddwd m12, m0 + pmaddwd m13, m2 + phaddd m10, m11 + phaddd m12, m13 + phaddd m10, m12 + psubd m10, m3, m10 + psrad m10, 7 + vextracti128 xm0, m10, 1 + packusdw xm10, xm0 + pminsw xm10, xm7 + mova [dstq+xq*2], xm10 + paddd m4, m5 + add xd, 8 + cmp xd, dst_wd + jl .loop_x + add dstq, dst_strideq + add srcq, src_strideq + dec hd + jg .loop_y + RET + %endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/mc16_sse.asm b/third_party/dav1d/src/x86/mc16_sse.asm --- a/third_party/dav1d/src/x86/mc16_sse.asm +++ b/third_party/dav1d/src/x86/mc16_sse.asm @@ -39,10 +39,13 @@ blend_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3 spel_h_shufA: db 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9 spel_h_shufB: db 4, 5, 6, 7, 6, 7, 8, 9, 8, 9, 10, 11, 10, 11, 12, 13 spel_h_shuf2: db 0, 1, 2, 3, 4, 5, 6, 7, 2, 3, 4, 5, 6, 7, 8, 9 +rescale_mul: dd 0, 1, 2, 3 +resize_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 4, 5, 6, 7 + db 8, 9, 10, 11, 12, 13, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15 pw_2: times 8 dw 2 pw_16: times 4 dw 16 prep_mul: times 4 dw 16 times 8 dw 4 @@ -52,10 +55,12 @@ bidir_mul: times 4 dw 2048 pw_8192: times 8 dw 8192 pw_27615: times 8 dw 27615 pw_32766: times 8 dw 32766 pw_m512: times 8 dw -512 +pd_63: times 4 dd 63 +pd_64: times 4 dd 64 pd_512: times 4 dd 512 pd_65538: times 2 dd 65538 put_bilin_h_rnd: times 4 dw 8 times 4 dw 10 @@ -63,10 +68,16 @@ times 4 dw -16388 put_8tap_h_rnd: dd 34, 34, 40, 40 prep_8tap_1d_rnd: times 2 dd 8 - (8192 << 4) prep_8tap_2d_rnd: times 4 dd 32 - (8192 << 5) +warp8x8_shift: dd 11, 13 +warp8x8_rnd1: dd 1024, 1024, 4096, 4096 +warp8x8_rnd2: times 4 dw 4096 + times 4 dw 16384 +warp8x8t_rnd: times 2 dd 16384 - (8192 << 15) + %macro BIDIR_JMP_TABLE 2-* %xdefine %1_%2_table (%%table - 2*%3) %xdefine %%base %1_%2_table %xdefine %%prefix mangle(private_prefix %+ _%1_16bpc_%2) %%table: @@ -103,10 +114,13 @@ BASE_JMP_TABLE prep, ssse3, 4, 8, 16, 32, 64, 128 cextern mc_subpel_filters %define subpel_filters (mangle(private_prefix %+ _mc_subpel_filters)-8) +cextern mc_warp_filter +cextern resize_filter + SECTION .text %macro REPX 2-* %xdefine %%f(x) %1 %rep %0 - 1 @@ -2524,10 +2538,402 @@ sub wd, 1<<8 jg .hv_loop0 RET %undef tmp +%if ARCH_X86_64 +; warp8x8t spills one less xmm register than warp8x8 on WIN64, compensate that +; by allocating 16 bytes more stack space so that stack offsets match up. +%if WIN64 && STACK_ALIGNMENT == 16 +%assign stksz 16*14 +%else +%assign stksz 16*13 +%endif +cglobal warp_affine_8x8t_16bpc, 4, 13, 9, stksz, dst, ds, src, ss, delta, \ + mx, tmp, alpha, beta, \ + filter, my, gamma, cnt +%assign stack_size_padded_8x8t stack_size_padded +%else +cglobal warp_affine_8x8t_16bpc, 0, 7, 8, -16*17, alpha, gamma, src, tmp, \ + filter, mx, my +%define m8 [esp+16*13] +%define m9 [esp+16*14] +%define cntd dword [esp+4*63] +%define dstq tmpq +%define dsq 0 +%if STACK_ALIGNMENT < 16 +%define dstm [esp+4*65] +%define dsm [esp+4*66] +%else +%define dstm r0m +%define dsm r1m +%endif +%endif +%define base filterq-$$ + mov t0d, r7m + LEA filterq, $$ + shr t0d, 11 +%if ARCH_X86_64 + movddup m8, [base+warp8x8t_rnd] +%else + movddup m1, [base+warp8x8t_rnd] + mov r1, r1m + add r1, r1 + mova m8, m1 + mov r1m, r1 ; ds *= 2 +%endif + call mangle(private_prefix %+ _warp_affine_8x8_16bpc_ssse3).main + jmp .start +.loop: +%if ARCH_X86_64 + lea dstq, [dstq+dsq*4] +%else + add dstq, dsm + mov dstm, dstq +%endif + call mangle(private_prefix %+ _warp_affine_8x8_16bpc_ssse3).main2 +.start: +%if ARCH_X86_32 + mov dstq, dstm +%endif + paddd m1, m8 + paddd m2, m8 + psrad m1, 15 + psrad m2, 15 + packssdw m1, m2 + mova [dstq+dsq*0], m1 + call mangle(private_prefix %+ _warp_affine_8x8_16bpc_ssse3).main3 +%if ARCH_X86_32 + mov dstq, dstm + add dstq, dsm +%endif + paddd m1, m8 + paddd m2, m8 + psrad m1, 15 + psrad m2, 15 + packssdw m1, m2 + mova [dstq+dsq*2], m1 + dec cntd + jg .loop + RET + +%if ARCH_X86_64 +cglobal warp_affine_8x8_16bpc, 4, 13, 10, 16*13, dst, ds, src, ss, delta, \ + mx, tmp, alpha, beta, \ + filter, my, gamma, cnt +ASSERT stack_size_padded == stack_size_padded_8x8t +%else +cglobal warp_affine_8x8_16bpc, 0, 7, 8, -16*17, alpha, gamma, src, tmp, \ + filter, mx, my +%endif + mov t0d, r7m + LEA filterq, $$ + shr t0d, 11 +%if ARCH_X86_64 + movddup m8, [base+warp8x8_rnd2+t0*8] + movd m9, r7m ; pixel_max + pshufb m9, [base+pw_256] +%else + movddup m1, [base+warp8x8_rnd2+t0*8] + movd m2, r7m ; pixel_max + pshufb m2, [base+pw_256] + mova m8, m1 + mova m9, m2 +%endif + call .main + jmp .start +.loop: +%if ARCH_X86_64 + lea dstq, [dstq+dsq*2] +%else + add dstq, dsm + mov dstm, dstq +%endif + call .main2 +.start: +%if ARCH_X86_32 + mov dstq, dstm +%endif + psrad m1, 16 + psrad m2, 16 + packssdw m1, m2 + pmaxsw m1, m6 + pmulhrsw m1, m8 + pminsw m1, m9 + mova [dstq+dsq*0], m1 + call .main3 +%if ARCH_X86_32 + mov dstq, dstm + add dstq, dsm +%endif + psrad m1, 16 + psrad m2, 16 + packssdw m1, m2 + pmaxsw m1, m6 + pmulhrsw m1, m8 + pminsw m1, m9 + mova [dstq+dsq*1], m1 + dec cntd + jg .loop + RET +ALIGN function_align +.main: + ; Stack args offset by one (r4m -> r5m etc.) due to call +%if WIN64 + mov deltaq, r5m + mov mxd, r6m +%endif + movd m0, [base+warp8x8_shift+t0*4] + movddup m7, [base+warp8x8_rnd1+t0*8] + add filterq, mc_warp_filter-$$ +%if ARCH_X86_64 + movsx alphad, word [deltaq+2*0] + movsx betad, word [deltaq+2*1] + movsx gammad, word [deltaq+2*2] + movsx deltad, word [deltaq+2*3] + lea tmpq, [ssq*3] + add mxd, 512+(64<<10) + sub srcq, tmpq ; src -= ss*3 + imul tmpd, alphad, -7 + mov myd, r7m + add betad, tmpd ; beta -= alpha*7 + imul tmpd, gammad, -7 + add myd, 512+(64<<10) + mov cntd, 4 + add deltad, tmpd ; delta -= gamma*7 +%else +%if STACK_ALIGNMENT < 16 + %assign stack_offset stack_offset - gprsize +%endif + mov r3d, r5m ; abcd +%if STACK_ALIGNMENT < 16 + mov r0, r1m ; dst + mov r1, r2m ; ds + mov [esp+gprsize+4*65], r0 + mov [esp+gprsize+4*66], r1 +%endif + movsx alphad, word [r3+2*0] + movsx r2d, word [r3+2*1] + movsx gammad, word [r3+2*2] + movsx r3d, word [r3+2*3] + imul r5d, alphad, -7 + add r2d, r5d ; beta -= alpha*7 + imul r5d, gammad, -7 + mov [esp+gprsize+4*60], r2d + add r3d, r5d ; delta -= gamma*7 + mov [esp+gprsize+4*61], r3d + mov r3d, r4m ; ss + mov srcq, r3m + mov mxd, r6m + mov myd, r7m + mov dword [esp+gprsize+4*63], 4 ; cnt + mov [esp+gprsize+4*62], r3 + lea r3, [r3*3] + add mxd, 512+(64<<10) + add myd, 512+(64<<10) + sub srcq, r3 ; src -= ss*3 +%if STACK_ALIGNMENT < 16 + %assign stack_offset stack_offset + gprsize +%endif +%endif + mova [rsp+gprsize], m0 + pxor m6, m6 + call .h + mova m5, m0 + call .h + punpcklwd m1, m5, m0 ; 01 + punpckhwd m5, m0 + mova [rsp+gprsize+16* 1], m1 + mova [rsp+gprsize+16* 4], m5 + mova m5, m0 + call .h + punpcklwd m1, m5, m0 ; 12 + punpckhwd m5, m0 + mova [rsp+gprsize+16* 7], m1 + mova [rsp+gprsize+16*10], m5 + mova m5, m0 + call .h + punpcklwd m1, m5, m0 ; 23 + punpckhwd m5, m0 + mova [rsp+gprsize+16* 2], m1 + mova [rsp+gprsize+16* 5], m5 + mova m5, m0 + call .h + punpcklwd m1, m5, m0 ; 34 + punpckhwd m5, m0 + mova [rsp+gprsize+16* 8], m1 + mova [rsp+gprsize+16*11], m5 + mova m5, m0 + call .h + punpcklwd m1, m5, m0 ; 45 + punpckhwd m5, m0 + mova [rsp+gprsize+16* 3], m1 + mova [rsp+gprsize+16* 6], m5 + mova m5, m0 + call .h + punpcklwd m1, m5, m0 ; 56 + punpckhwd m5, m0 + mova [rsp+gprsize+16* 9], m1 + mova [rsp+gprsize+16*12], m5 + mova m5, m0 +.main2: + call .h +%macro WARP_V 6 ; 01l, 23l, 45l, 01h, 23h, 45h + lea tmpd, [myq+gammaq] + shr myd, 10 + movq m4, [filterq+myq*8] ; a + lea myd, [tmpq+gammaq] + shr tmpd, 10 + movq m2, [filterq+tmpq*8] ; b + lea tmpd, [myq+gammaq] + shr myd, 10 + movq m3, [filterq+myq*8] ; c + lea myd, [tmpq+gammaq] + shr tmpd, 10 + movq m1, [filterq+tmpq*8] ; d + lea tmpd, [myq+gammaq] + shr myd, 10 + punpcklwd m4, m2 + punpcklwd m3, m1 + punpckldq m2, m4, m3 + punpckhdq m4, m3 + punpcklbw m1, m6, m2 ; a0 a1 b0 b1 c0 c1 d0 d1 << 8 + pmaddwd m1, [rsp+gprsize+16*%1] + punpckhbw m3, m6, m2 ; a2 a3 b2 b3 c2 c3 d2 d3 << 8 + mova m2, [rsp+gprsize+16*%2] + pmaddwd m3, m2 + mova [rsp+gprsize+16*%1], m2 + paddd m1, m3 + punpcklbw m3, m6, m4 ; a4 a5 b4 b5 c4 c5 d4 d5 << 8 + mova m2, [rsp+gprsize+16*%3] + pmaddwd m3, m2 + mova [rsp+gprsize+16*%2], m2 + paddd m1, m3 + punpcklwd m3, m5, m0 ; 67 + punpckhbw m2, m6, m4 ; a6 a7 b6 b7 c6 c7 d6 d7 << 8 + pmaddwd m2, m3 + mova [rsp+gprsize+16*%3], m3 + paddd m1, m2 + movq m4, [filterq+myq*8] ; e + lea myd, [tmpq+gammaq] + shr tmpd, 10 + movq m3, [filterq+tmpq*8] ; f + lea tmpd, [myq+gammaq] + shr myd, 10 + movq m2, [filterq+myq*8] ; g +%if ARCH_X86_64 + lea myd, [tmpq+deltaq] ; my += delta +%else + mov myd, [esp+gprsize+4*61] + add myd, tmpd +%endif + shr tmpd, 10 + punpcklwd m4, m3 + movq m3, [filterq+tmpq*8] ; h + punpcklwd m2, m3 + punpckldq m3, m4, m2 + punpckhdq m4, m2 + punpcklbw m2, m6, m3 ; e0 e1 f0 f1 g0 g1 h0 h1 << 8 + pmaddwd m2, [rsp+gprsize+16*%4] + punpckhbw m6, m3 ; e2 e3 f2 f3 g2 g3 h2 h3 << 8 + mova m3, [rsp+gprsize+16*%5] + pmaddwd m6, m3 + mova [rsp+gprsize+16*%4], m3 + pxor m3, m3 + paddd m2, m6 + punpcklbw m3, m4 ; e4 e5 f4 f5 g4 g5 h4 h5 << 8 + mova m6, [rsp+gprsize+16*%6] + pmaddwd m3, m6 + mova [rsp+gprsize+16*%5], m6 + punpckhwd m5, m0 + pxor m6, m6 + paddd m2, m3 + punpckhbw m3, m6, m4 ; e6 e7 f6 f7 g6 g7 h6 h7 << 8 + pmaddwd m3, m5 + mova [rsp+gprsize+16*%6], m5 + mova m5, m0 + paddd m2, m3 +%endmacro + WARP_V 1, 2, 3, 4, 5, 6 + ret +.main3: + call .h + WARP_V 7, 8, 9, 10, 11, 12 + ret +ALIGN function_align +.h: + lea tmpd, [mxq+alphaq] + shr mxd, 10 + movq m3, [filterq+mxq*8] + punpcklbw m0, m6, m3 + movu m3, [srcq-6] + pmaddwd m0, m3 ; 0 + lea mxd, [tmpq+alphaq] + shr tmpd, 10 + movq m3, [filterq+tmpq*8] + punpcklbw m2, m6, m3 + movu m3, [srcq-4] + pmaddwd m2, m3 ; 1 + lea tmpd, [mxq+alphaq] + shr mxd, 10 + movq m3, [filterq+mxq*8] + phaddd m0, m2 ; 0 1 + punpcklbw m2, m6, m3 + movu m3, [srcq-2] + pmaddwd m2, m3 ; 2 + lea mxd, [tmpq+alphaq] + shr tmpd, 10 + movq m3, [filterq+tmpq*8] + punpcklbw m1, m6, m3 + movu m3, [srcq+0] + pmaddwd m1, m3 ; 3 + lea tmpd, [mxq+alphaq] + shr mxd, 10 + movq m3, [filterq+mxq*8] + phaddd m2, m1 ; 2 3 + punpcklbw m1, m6, m3 + movu m3, [srcq+2] + pmaddwd m1, m3 ; 4 + lea mxd, [tmpq+alphaq] + shr tmpd, 10 + movq m3, [filterq+tmpq*8] + phaddd m0, m2 ; 0 1 2 3 + punpcklbw m2, m6, m3 + movu m3, [srcq+4] + pmaddwd m2, m3 ; 5 + lea tmpd, [mxq+alphaq] + shr mxd, 10 + movq m3, [filterq+mxq*8] + phaddd m1, m2 ; 4 5 + punpcklbw m2, m6, m3 + movu m3, [srcq+6] + pmaddwd m2, m3 ; 6 +%if ARCH_X86_64 + lea mxd, [tmpq+betaq] ; mx += beta +%else + mov mxd, [esp+gprsize*2+4*60] + add mxd, tmpd +%endif + shr tmpd, 10 + movq m3, [filterq+tmpq*8] + punpcklbw m4, m6, m3 + movu m3, [srcq+8] +%if ARCH_X86_64 + add srcq, ssq +%else + add srcq, [esp+gprsize*2+4*62] +%endif + pmaddwd m3, m4 ; 7 + phaddd m2, m3 ; 6 7 + phaddd m1, m2 ; 4 5 6 7 + paddd m0, m7 + paddd m1, m7 + psrad m0, [rsp+gprsize*2] + psrad m1, [rsp+gprsize*2] + packssdw m0, m1 + ret + %macro BIDIR_FN 0 call .main jmp wq .w4_loop: call .main @@ -4140,5 +4546,235 @@ RET %undef reg_dstride %undef reg_blkm %undef reg_tmp + +%macro SCRATCH 3 +%if ARCH_X86_32 + mova [rsp+%3*mmsize], m%1 +%define m%2 [rsp+%3*mmsize] +%else + SWAP %1, %2 +%endif +%endmacro + +%if ARCH_X86_64 +cglobal resize_16bpc, 0, 12, 16, 1*16, dst, dst_stride, src, src_stride, \ + dst_w, h, src_w, dx, mx0, pxmax +%elif STACK_ALIGNMENT >= 16 +cglobal resize_16bpc, 0, 7, 8, 6*16, dst, dst_stride, src, src_stride, \ + dst_w, h, src_w, dx, mx0, pxmax +%else +cglobal resize_16bpc, 0, 6, 8, 6*16, dst, dst_stride, src, src_stride, \ + dst_w, h, src_w, dx, mx0, pxmax +%endif + movifnidn dstq, dstmp + movifnidn srcq, srcmp +%if STACK_ALIGNMENT >= 16 + movifnidn dst_wd, dst_wm +%endif +%if ARCH_X86_64 + movifnidn hd, hm +%endif + sub dword mx0m, 4<<14 + sub dword src_wm, 8 + movd m4, pxmaxm + movd m7, dxm + movd m6, mx0m + movd m5, src_wm + punpcklwd m4, m4 + pshufd m4, m4, q0000 + pshufd m7, m7, q0000 + pshufd m6, m6, q0000 + pshufd m5, m5, q0000 + mova [rsp+16*3*ARCH_X86_32], m4 +%if ARCH_X86_64 + DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, h, x, picptr + LEA r7, $$ + %define base r7-$$ +%else + DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, x + %define hd dword r5m + %if STACK_ALIGNMENT >= 16 + LEA r6, $$ + %define base r6-$$ + %else + LEA r4, $$ + %define base r4-$$ + %endif +%endif +%if ARCH_X86_64 + mova m12, [base+pd_64] + mova m11, [base+pd_63] +%else + %define m12 [base+pd_64] + %define m11 [base+pd_63] +%endif + pmaddwd m4, m7, [base+rescale_mul] ; dx*[0,1,2,3] + pslld m7, 2 ; dx*4 + pslld m5, 14 + paddd m6, m4 ; mx+[0..3]*dx + SCRATCH 7, 15, 0 + SCRATCH 6, 14, 1 + SCRATCH 5, 13, 2 + pxor m1, m1 +.loop_y: + xor xd, xd + mova m0, m14 ; per-line working version of mx +.loop_x: + pcmpgtd m1, m0 + pandn m1, m0 + psrad m2, m0, 8 ; filter offset (unmasked) + pcmpgtd m3, m13, m1 + pand m1, m3 + pandn m3, m13 + por m1, m3 + psubd m3, m0, m1 ; pshufb offset + psrad m1, 14 ; clipped src_x offset + psrad m3, 14 ; pshufb edge_emu offset + pand m2, m11 ; filter offset (masked) + ; load source pixels +%if ARCH_X86_64 + movd r8d, m1 + pshuflw m1, m1, q3232 + movd r9d, m1 + punpckhqdq m1, m1 + movd r10d, m1 + psrlq m1, 32 + movd r11d, m1 + movu m4, [srcq+r8*2] + movu m5, [srcq+r9*2] + movu m6, [srcq+r10*2] + movu m7, [srcq+r11*2] + ; if no emulation is required, we don't need to shuffle or emulate edges + packssdw m3, m3 + movq r11, m3 + test r11, r11 + jz .filter + movsx r8, r11w + sar r11, 16 + movsx r9, r11w + sar r11, 16 + movsx r10, r11w + sar r11, 16 + movu m1, [base+resize_shuf+8+r8*2] + movu m3, [base+resize_shuf+8+r9*2] + movu m8, [base+resize_shuf+8+r10*2] + movu m9, [base+resize_shuf+8+r11*2] + pshufb m4, m1 + pshufb m5, m3 + pshufb m6, m8 + pshufb m7, m9 +.filter: + movd r8d, m2 + pshuflw m2, m2, q3232 + movd r9d, m2 + punpckhqdq m2, m2 + movd r10d, m2 + psrlq m2, 32 + movd r11d, m2 + movq m8, [base+resize_filter+r8*8] + movq m2, [base+resize_filter+r9*8] + pxor m9, m9 + punpcklbw m1, m9, m8 + punpcklbw m3, m9, m2 + psraw m1, 8 + psraw m3, 8 + movq m10, [base+resize_filter+r10*8] + movq m2, [base+resize_filter+r11*8] + punpcklbw m8, m9, m10 + punpcklbw m9, m2 + psraw m8, 8 + psraw m9, 8 + pmaddwd m4, m1 + pmaddwd m5, m3 + pmaddwd m6, m8 + pmaddwd m7, m9 + phaddd m4, m5 +%else + movd r3, m1 + pshuflw m1, m1, q3232 + movd r1, m1 + punpckhqdq m1, m1 + movu m4, [srcq+r3*2] + movu m5, [srcq+r1*2] + movd r3, m1 + psrlq m1, 32 + movd r1, m1 + movu m6, [srcq+r3*2] + movu m7, [srcq+r1*2] + ; if no emulation is required, we don't need to shuffle or emulate edges + pxor m1, m1 + pcmpeqb m1, m3 + pmovmskb r3d, m1 + cmp r3d, 0xffff + je .filter + movd r3, m3 + movu m1, [base+resize_shuf+8+r3*2] + pshuflw m3, m3, q3232 + movd r1, m3 + pshufb m4, m1 + movu m1, [base+resize_shuf+8+r1*2] + punpckhqdq m3, m3 + movd r3, m3 + pshufb m5, m1 + movu m1, [base+resize_shuf+8+r3*2] + psrlq m3, 32 + movd r1, m3 + pshufb m6, m1 + movu m1, [base+resize_shuf+8+r1*2] + pshufb m7, m1 +.filter: + mova [esp+4*16], m6 + mova [esp+5*16], m7 + movd r3, m2 + pshuflw m2, m2, q3232 + movd r1, m2 + movq m6, [base+resize_filter+r3*8] + movq m7, [base+resize_filter+r1*8] + pxor m3, m3 + punpcklbw m1, m3, m6 + punpcklbw m3, m7 + psraw m1, 8 + psraw m3, 8 + pmaddwd m4, m1 + pmaddwd m5, m3 + punpckhqdq m2, m2 + movd r3, m2 + psrlq m2, 32 + movd r1, m2 + phaddd m4, m5 + movq m2, [base+resize_filter+r3*8] + movq m5, [base+resize_filter+r1*8] + mova m6, [esp+4*16] + mova m7, [esp+5*16] + pxor m3, m3 + punpcklbw m1, m3, m2 + punpcklbw m3, m5 + psraw m1, 8 + psraw m3, 8 + pmaddwd m6, m1 + pmaddwd m7, m3 +%endif + phaddd m6, m7 + phaddd m4, m6 + pxor m1, m1 + psubd m2, m12, m4 + psrad m2, 7 + packssdw m2, m2 + pmaxsw m2, m1 + pminsw m2, [rsp+16*3*ARCH_X86_32] + movq [dstq+xq*2], m2 + paddd m0, m15 + add xd, 4 +%if STACK_ALIGNMENT >= 16 + cmp xd, dst_wd +%else + cmp xd, dst_wm +%endif + jl .loop_x + add dstq, dst_stridemp + add srcq, src_stridemp + dec hd + jg .loop_y + RET diff --git a/third_party/dav1d/src/x86/mc_avx2.asm b/third_party/dav1d/src/x86/mc_avx2.asm --- a/third_party/dav1d/src/x86/mc_avx2.asm +++ b/third_party/dav1d/src/x86/mc_avx2.asm @@ -1,7 +1,7 @@ -; Copyright © 2018-2020, VideoLAN and dav1d authors -; Copyright © 2018-2020, Two Orioles, LLC +; Copyright © 2018-2021, VideoLAN and dav1d authors +; Copyright © 2018-2021, Two Orioles, LLC ; All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions are met: ; @@ -67,11 +67,10 @@ pb_8x0_8x8: db 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 8, 8, 8, 8 bdct_lb_dw: db 0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 8, 12, 12, 12, 12 wswap: db 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13 rescale_mul: dd 0, 1, 2, 3, 4, 5, 6, 7 resize_shuf: db 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 7, 7, 7, 7 - db 7, 7, 7, 7, 7, 7, 7, 7 wm_420_sign: dd 0x01020102, 0x01010101 wm_422_sign: dd 0x80808080, 0x7f7f7f7f pb_64: times 4 db 64 @@ -108,11 +107,11 @@ %rotate 1 %endrep %endmacro %macro HV_JMP_TABLE 5-* - %xdefine %%prefix mangle(private_prefix %+ _%1_%2_%3) + %xdefine %%prefix mangle(private_prefix %+ _%1_%2_8bpc_%3) %xdefine %%base %1_%3 %assign %%types %4 %if %%types & 1 %xdefine %1_%2_h_%3_table (%%h - %5) %%h: @@ -139,72 +138,72 @@ %rotate 1 %endrep %endif %endmacro -%macro BIDIR_JMP_TABLE 1-* - %xdefine %1_table (%%table - 2*%2) - %xdefine %%base %1_table - %xdefine %%prefix mangle(private_prefix %+ _%1) +%macro BIDIR_JMP_TABLE 2-* + %xdefine %1_%2_table (%%table - 2*%3) + %xdefine %%base %1_%2_table + %xdefine %%prefix mangle(private_prefix %+ _%1_8bpc_%2) %%table: - %rep %0 - 1 - dd %%prefix %+ .w%2 - %%base + %rep %0 - 2 + dd %%prefix %+ .w%3 - %%base %rotate 1 %endrep %endmacro -%macro SCALED_JMP_TABLE 1-* - %xdefine %1_table (%%table - %2) - %xdefine %%base mangle(private_prefix %+ _%1) +%macro SCALED_JMP_TABLE 2-* + %xdefine %1_%2_table (%%table - %3) + %xdefine %%base mangle(private_prefix %+ _%1_8bpc_%2) %%table: - %rep %0 - 1 - dw %%base %+ .w%2 - %%base + %rep %0 - 2 + dw %%base %+ .w%3 - %%base %rotate 1 %endrep - %rotate 1 + %rotate 2 %%dy_1024: - %xdefine %1_dy1_table (%%dy_1024 - %2) - %rep %0 - 1 - dw %%base %+ .dy1_w%2 - %%base + %xdefine %1_%2_dy1_table (%%dy_1024 - %3) + %rep %0 - 2 + dw %%base %+ .dy1_w%3 - %%base %rotate 1 %endrep - %rotate 1 + %rotate 2 %%dy_2048: - %xdefine %1_dy2_table (%%dy_2048 - %2) - %rep %0 - 1 - dw %%base %+ .dy2_w%2 - %%base + %xdefine %1_%2_dy2_table (%%dy_2048 - %3) + %rep %0 - 2 + dw %%base %+ .dy2_w%3 - %%base %rotate 1 %endrep %endmacro -%xdefine put_avx2 mangle(private_prefix %+ _put_bilin_avx2.put) -%xdefine prep_avx2 mangle(private_prefix %+ _prep_bilin_avx2.prep) +%xdefine put_avx2 mangle(private_prefix %+ _put_bilin_8bpc_avx2.put) +%xdefine prep_avx2 mangle(private_prefix %+ _prep_bilin_8bpc_avx2.prep) %define table_offset(type, fn) type %+ fn %+ SUFFIX %+ _table - type %+ SUFFIX -BASE_JMP_TABLE put, avx2, 2, 4, 8, 16, 32, 64, 128 -BASE_JMP_TABLE prep, avx2, 4, 8, 16, 32, 64, 128 -HV_JMP_TABLE put, bilin, avx2, 7, 2, 4, 8, 16, 32, 64, 128 -HV_JMP_TABLE prep, bilin, avx2, 7, 4, 8, 16, 32, 64, 128 -HV_JMP_TABLE put, 8tap, avx2, 3, 2, 4, 8, 16, 32, 64, 128 -HV_JMP_TABLE prep, 8tap, avx2, 1, 4, 8, 16, 32, 64, 128 -SCALED_JMP_TABLE put_8tap_scaled_avx2, 2, 4, 8, 16, 32, 64, 128 -SCALED_JMP_TABLE prep_8tap_scaled_avx2, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE avg_avx2, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE w_avg_avx2, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE mask_avx2, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE w_mask_420_avx2, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE w_mask_422_avx2, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE w_mask_444_avx2, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE blend_avx2, 4, 8, 16, 32 -BIDIR_JMP_TABLE blend_v_avx2, 2, 4, 8, 16, 32 -BIDIR_JMP_TABLE blend_h_avx2, 2, 4, 8, 16, 32, 32, 32 +BASE_JMP_TABLE put, avx2, 2, 4, 8, 16, 32, 64, 128 +BASE_JMP_TABLE prep, avx2, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE put, bilin, avx2, 7, 2, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE prep, bilin, avx2, 7, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE put, 8tap, avx2, 3, 2, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE prep, 8tap, avx2, 1, 4, 8, 16, 32, 64, 128 +SCALED_JMP_TABLE put_8tap_scaled, avx2, 2, 4, 8, 16, 32, 64, 128 +SCALED_JMP_TABLE prep_8tap_scaled, avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE avg, avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_avg, avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE mask, avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_420, avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_422, avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_444, avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE blend, avx2, 4, 8, 16, 32 +BIDIR_JMP_TABLE blend_v, avx2, 2, 4, 8, 16, 32 +BIDIR_JMP_TABLE blend_h, avx2, 2, 4, 8, 16, 32, 32, 32 SECTION .text INIT_XMM avx2 -cglobal put_bilin, 4, 8, 0, dst, ds, src, ss, w, h, mxy +cglobal put_bilin_8bpc, 4, 8, 0, dst, ds, src, ss, w, h, mxy movifnidn mxyd, r6m ; mx lea r7, [put_avx2] tzcnt wd, wm movifnidn hd, hm test mxyd, mxyd @@ -767,11 +766,11 @@ %if WIN64 movaps xmm8, r4m %endif RET -cglobal prep_bilin, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3 +cglobal prep_bilin_8bpc, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3 movifnidn mxyd, r5m ; mx lea r6, [prep%+SUFFIX] tzcnt wd, wm movifnidn hd, hm test mxyd, mxyd @@ -1437,41 +1436,40 @@ %assign FILTER_REGULAR (0*15 << 16) | 3*15 %assign FILTER_SMOOTH (1*15 << 16) | 4*15 %assign FILTER_SHARP (2*15 << 16) | 3*15 %macro FN 4 ; fn, type, type_h, type_v -cglobal %1_%2 +cglobal %1_%2_8bpc mov t0d, FILTER_%3 %ifidn %3, %4 mov t1d, t0d %else mov t1d, FILTER_%4 %endif %ifnidn %2, regular ; skip the jump in the last filter - jmp mangle(private_prefix %+ _%1 %+ SUFFIX) + jmp mangle(private_prefix %+ _%1_8bpc %+ SUFFIX) %endif %endmacro %if WIN64 DECLARE_REG_TMP 4, 5 %else DECLARE_REG_TMP 7, 8 %endif %define PUT_8TAP_FN FN put_8tap, - PUT_8TAP_FN sharp, SHARP, SHARP PUT_8TAP_FN sharp_smooth, SHARP, SMOOTH PUT_8TAP_FN smooth_sharp, SMOOTH, SHARP PUT_8TAP_FN smooth, SMOOTH, SMOOTH PUT_8TAP_FN sharp_regular, SHARP, REGULAR PUT_8TAP_FN regular_sharp, REGULAR, SHARP PUT_8TAP_FN smooth_regular, SMOOTH, REGULAR PUT_8TAP_FN regular_smooth, REGULAR, SMOOTH PUT_8TAP_FN regular, REGULAR, REGULAR -cglobal put_8tap, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 +cglobal put_8tap_8bpc, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 imul mxd, mxm, 0x010101 add mxd, t0d ; 8tap_h, mx, 4tap_h imul myd, mym, 0x010101 add myd, t1d ; 8tap_v, my, 4tap_v lea r8, [put_avx2] @@ -2122,22 +2120,21 @@ %else DECLARE_REG_TMP 6, 7 %endif %define PREP_8TAP_FN FN prep_8tap, - PREP_8TAP_FN sharp, SHARP, SHARP PREP_8TAP_FN sharp_smooth, SHARP, SMOOTH PREP_8TAP_FN smooth_sharp, SMOOTH, SHARP PREP_8TAP_FN smooth, SMOOTH, SMOOTH PREP_8TAP_FN sharp_regular, SHARP, REGULAR PREP_8TAP_FN regular_sharp, REGULAR, SHARP PREP_8TAP_FN smooth_regular, SMOOTH, REGULAR PREP_8TAP_FN regular_smooth, REGULAR, SMOOTH PREP_8TAP_FN regular, REGULAR, REGULAR -cglobal prep_8tap, 3, 8, 0, tmp, src, stride, w, h, mx, my, stride3 +cglobal prep_8tap_8bpc, 3, 8, 0, tmp, src, stride, w, h, mx, my, stride3 imul mxd, mxm, 0x010101 add mxd, t0d ; 8tap_h, mx, 4tap_h imul myd, mym, 0x010101 add myd, t1d ; 8tap_v, my, 4tap_v lea r7, [prep%+SUFFIX] @@ -2723,30 +2720,30 @@ %macro MC_8TAP_SCALED 1 %ifidn %1, put %assign isprep 0 %if required_stack_alignment <= STACK_ALIGNMENT -cglobal put_8tap_scaled, 4, 15, 16, 112, dst, ds, src, ss, w, h, mx, my, dx, dy +cglobal put_8tap_scaled_8bpc, 4, 15, 16, 112, dst, ds, src, ss, w, h, mx, my, dx, dy %else -cglobal put_8tap_scaled, 4, 14, 16, 128, dst, ds, src, ss, w, h, mx, my, dx, dy +cglobal put_8tap_scaled_8bpc, 4, 14, 16, 128, dst, ds, src, ss, w, h, mx, my, dx, dy %endif %xdefine base_reg r12 %define rndshift 10 %else %assign isprep 1 %if required_stack_alignment <= STACK_ALIGNMENT -cglobal prep_8tap_scaled, 4, 15, 16, 128, tmp, src, ss, w, h, mx, my, dx, dy +cglobal prep_8tap_scaled_8bpc, 4, 15, 16, 128, tmp, src, ss, w, h, mx, my, dx, dy %xdefine tmp_stridem r14q %else -cglobal prep_8tap_scaled, 4, 14, 16, 128, tmp, src, ss, w, h, mx, my, dx, dy +cglobal prep_8tap_scaled_8bpc, 4, 14, 16, 128, tmp, src, ss, w, h, mx, my, dx, dy %define tmp_stridem qword [rsp+120] %endif %xdefine base_reg r11 %define rndshift 6 %endif - lea base_reg, [%1_8tap_scaled_avx2] -%define base base_reg-%1_8tap_scaled_avx2 + lea base_reg, [%1_8tap_scaled_8bpc_avx2] +%define base base_reg-%1_8tap_scaled_8bpc_avx2 tzcnt wd, wm vpbroadcastd m8, dxm %if isprep && UNIX64 movd xm14, mxd vpbroadcastd m14, xm14 @@ -2815,11 +2812,11 @@ mov myd, mym movzx t0d, t0b dec srcq movd xm15, t0d punpckldq m8, m9, m8 - paddd m14, m8 ; mx+dx*[0-1] + paddd m14, m8 ; mx+dx*[0,1] vpbroadcastd m11, [base+pd_0x4000] vpbroadcastd xm15, xm15 pand m8, m14, m10 psrld m8, 6 paddd xm15, xm8 @@ -2866,12 +2863,11 @@ mov r4d, myd shr r4d, 6 lea r4d, [t1+r4] cmovnz r6q, [base+subpel_filters+r4*8] movq xm11, r6q - punpcklbw xm11, xm11 - psraw xm11, 8 + pmovsxbw xm11, xm11 pshufd xm8, xm11, q0000 pshufd xm9, xm11, q1111 pshufd xm10, xm11, q2222 pshufd xm11, xm11, q3333 pmaddwd xm5, xm3, xm8 @@ -2995,12 +2991,11 @@ mov r4d, myd shr r4d, 6 lea r4d, [t1+r4] cmovnz r6q, [base+subpel_filters+r4*8] movq xm10, r6q - punpcklbw xm10, xm10 - psraw xm10, 8 + pmovsxbw xm10, xm10 pshufd xm7, xm10, q0000 pshufd xm8, xm10, q1111 pshufd xm9, xm10, q2222 pshufd xm10, xm10, q3333 pmaddwd xm4, xm0, xm7 @@ -3170,13 +3165,12 @@ mov r4d, myd shr r4d, 6 lea r4d, [t1+r4] cmovnz r6q, [base+subpel_filters+r4*8] movq xm11, r6q - punpcklbw xm11, xm11 - psraw xm11, 8 - vinserti128 m11, xm11, 1 + punpcklqdq xm11, xm11 + pmovsxbw m11, xm11 pshufd m8, m11, q0000 pshufd m9, m11, q1111 pmaddwd m4, m0, m8 pmaddwd m5, m1, m9 pshufd m8, m11, q2222 @@ -3318,12 +3312,11 @@ vinserti128 m0, [srcq+ssq*0], 1 vinserti128 m1, [srcq+ssq*2], 1 vpbroadcastq m2, [srcq+ssq*1] add srcq, ss3q movq xm10, r4q - punpcklbw xm10, xm10 - psraw xm10, 8 + pmovsxbw xm10, xm10 vpblendd m15, m7, 0xaa pblendvb m15, m11, m8 pshufd xm8, xm10, q0000 pshufd xm9, xm10, q1111 pshufd xm11, xm10, q3333 @@ -3415,13 +3408,12 @@ add srcq, ss3q vpblendd m15, m7, 0x30 punpcklqdq m15, m15 pblendvb m15, m11, m8 movq xm10, r4q - punpcklbw xm10, xm10 - psraw xm10, 8 - vinserti128 m10, xm10, 1 + punpcklqdq xm10, xm10 + pmovsxbw m10, xm10 pshufb m2, m14 pshufb m3, m14 pshufb m4, m14 pshufb xm5, xm14 vpermq m2, m2, q3120 @@ -3524,12 +3516,11 @@ %endif shl dword dxm, 3 ; dx*8 vpbroadcastd m15, xm15 paddd m14, m8 ; mx+dx*[0-7] movq xm0, r4q - punpcklbw xm0, xm0 - psraw xm0, 8 + pmovsxbw xm0, xm0 mova [rsp+96], xm0 jmp .dy1_hloop .dy1_hloop_prep: dec dword [rsp+72] jz .ret @@ -3693,12 +3684,11 @@ pshufb m0, m14 pshufb m1, m14 pmaddubsw m0, m15 pmaddubsw m1, m15 movq xm11, r4q - punpcklbw xm11, xm11 - psraw xm11, 8 + pmovsxbw xm11, xm11 phaddw m0, m1 pmulhrsw m0, m12 ; 0 2 _ 4 1 3 _ 5 pshufd xm8, xm11, q0000 pshufd xm9, xm11, q1111 pshufd xm10, xm11, q2222 @@ -3790,13 +3780,12 @@ pmaddubsw xm0, xm15 pmaddubsw m2, m15 pmaddubsw xm1, xm15 pmaddubsw m3, m15 movq xm11, r4q - punpcklbw xm11, xm11 - psraw xm11, 8 - vinserti128 m11, xm11, 1 + punpcklqdq xm11, xm11 + pmovsxbw m11, xm11 phaddw m0, m2 phaddw m1, m3 pmulhrsw m0, m12 ; 0 2 _ 4 pmulhrsw m1, m12 ; 1 3 _ 5 pshufd m8, m11, q0000 @@ -3887,12 +3876,11 @@ %endif shl dword dxm, 3 ; dx*8 vpbroadcastd m15, xm15 paddd m14, m8 ; mx+dx*[0-7] movq xm0, r4q - punpcklbw xm0, xm0 - psraw xm0, 8 + pmovsxbw xm0, xm0 mova [rsp+0x50], xm0 jmp .dy2_hloop .dy2_hloop_prep: dec dword [rsp+40] jz .ret @@ -4023,14 +4011,14 @@ MC_8TAP_SCALED_RET 0 %undef isprep %endmacro %macro BILIN_SCALED_FN 1 -cglobal %1_bilin_scaled +cglobal %1_bilin_scaled_8bpc mov t0d, (5*15 << 16) | 5*15 mov t1d, t0d - jmp mangle(private_prefix %+ _%1_8tap_scaled %+ SUFFIX) + jmp mangle(private_prefix %+ _%1_8tap_scaled_8bpc %+ SUFFIX) %endmacro %if WIN64 DECLARE_REG_TMP 6, 5 %else @@ -4111,31 +4099,31 @@ paddd m%2, m9 paddd m0, m8 paddd m%1, m0, m%2 %endmacro -cglobal warp_affine_8x8t, 0, 14, 0, tmp, ts +cglobal warp_affine_8x8t_8bpc, 0, 14, 0, tmp, ts %if WIN64 sub rsp, 0xa0 %endif - call mangle(private_prefix %+ _warp_affine_8x8_avx2).main + call mangle(private_prefix %+ _warp_affine_8x8_8bpc_avx2).main .loop: psrad m7, 13 psrad m0, 13 packssdw m7, m0 pmulhrsw m7, m14 ; (x + (1 << 6)) >> 7 vpermq m7, m7, q3120 mova [tmpq+tsq*0], xm7 vextracti128 [tmpq+tsq*2], m7, 1 dec r4d - jz mangle(private_prefix %+ _warp_affine_8x8_avx2).end - call mangle(private_prefix %+ _warp_affine_8x8_avx2).main2 + jz mangle(private_prefix %+ _warp_affine_8x8_8bpc_avx2).end + call mangle(private_prefix %+ _warp_affine_8x8_8bpc_avx2).main2 lea tmpq, [tmpq+tsq*4] jmp .loop -cglobal warp_affine_8x8, 0, 14, 0, dst, ds, src, ss, abcd, mx, tmp2, alpha, \ - beta, filter, tmp1, delta, my, gamma +cglobal warp_affine_8x8_8bpc, 0, 14, 0, dst, ds, src, ss, abcd, mx, tmp2, alpha, \ + beta, filter, tmp1, delta, my, gamma %if WIN64 sub rsp, 0xa0 %assign xmm_regs_used 16 %assign stack_size_padded 0xa0 %assign stack_offset stack_offset+stack_size_padded @@ -4387,11 +4375,11 @@ %macro AVG_INC_PTR 1 add tmp1q, %1*32 add tmp2q, %1*32 %endmacro -cglobal avg, 4, 7, 3, dst, stride, tmp1, tmp2, w, h, stride3 +cglobal avg_8bpc, 4, 7, 3, dst, stride, tmp1, tmp2, w, h, stride3 %define base r6-avg %+ SUFFIX %+ _table lea r6, [avg %+ SUFFIX %+ _table] tzcnt wd, wm movifnidn hd, hm movsxd wq, dword [r6+wq*4] @@ -4417,11 +4405,11 @@ packuswb m0, m1 %endmacro %define W_AVG_INC_PTR AVG_INC_PTR -cglobal w_avg, 4, 7, 6, dst, stride, tmp1, tmp2, w, h, stride3 +cglobal w_avg_8bpc, 4, 7, 6, dst, stride, tmp1, tmp2, w, h, stride3 %define base r6-w_avg %+ SUFFIX %+ _table lea r6, [w_avg %+ SUFFIX %+ _table] tzcnt wd, wm movifnidn hd, hm vpbroadcastw m4, r6m ; weight @@ -4467,11 +4455,11 @@ add maskq, %1*16 add tmp2q, %1*32 add tmp1q, %1*32 %endmacro -cglobal mask, 4, 8, 6, dst, stride, tmp1, tmp2, w, h, mask, stride3 +cglobal mask_8bpc, 4, 8, 6, dst, stride, tmp1, tmp2, w, h, mask, stride3 %define base r7-mask %+ SUFFIX %+ _table lea r7, [mask %+ SUFFIX %+ _table] tzcnt wd, wm movifnidn hd, hm mov maskq, maskmp @@ -4510,11 +4498,11 @@ pmulhrsw m%1, m7 pmulhrsw m1, m7 packuswb m%1, m1 %endmacro -cglobal blend, 3, 7, 7, dst, ds, tmp, w, h, mask +cglobal blend_8bpc, 3, 7, 7, dst, ds, tmp, w, h, mask %define base r6-blend_avx2_table lea r6, [blend_avx2_table] tzcnt wd, wm movifnidn hd, hm movifnidn maskq, maskmp @@ -4627,11 +4615,11 @@ add dstq, dsq dec hd jg .w32 RET -cglobal blend_v, 3, 6, 6, dst, ds, tmp, w, h, mask +cglobal blend_v_8bpc, 3, 6, 6, dst, ds, tmp, w, h, mask %define base r5-blend_v_avx2_table lea r5, [blend_v_avx2_table] tzcnt wd, wm movifnidn hd, hm movsxd wq, dword [r5+wq*4] @@ -4738,11 +4726,11 @@ add dstq, dsq dec hd jg .w32_loop RET -cglobal blend_h, 4, 7, 6, dst, ds, tmp, w, h, mask +cglobal blend_h_8bpc, 4, 7, 6, dst, ds, tmp, w, h, mask %define base r5-blend_h_avx2_table lea r5, [blend_h_avx2_table] mov r6d, wd tzcnt wd, wd mov hd, hm @@ -4864,11 +4852,11 @@ add dstq, dsq inc hq jl .w32_loop0 RET -cglobal emu_edge, 10, 13, 1, bw, bh, iw, ih, x, y, dst, dstride, src, sstride, \ +cglobal emu_edge_8bpc, 10, 13, 1, bw, bh, iw, ih, x, y, dst, dstride, src, sstride, \ bottomext, rightext ; we assume that the buffer (stride) is larger than width, so we can ; safely overwrite by a few bytes ; ref += iclip(y, 0, ih - 1) * PXSTRIDE(ref_stride) @@ -5051,12 +5039,12 @@ jl .top_x_loop .end: RET -cglobal resize, 6, 14, 16, dst, dst_stride, src, src_stride, \ - dst_w, h, src_w, dx, mx0 +cglobal resize_8bpc, 6, 12, 16, dst, dst_stride, src, src_stride, \ + dst_w, h, src_w, dx, mx0 sub dword mx0m, 4<<14 sub dword src_wm, 8 vpbroadcastd m5, dxm vpbroadcastd m8, mx0m vpbroadcastd m6, src_wm @@ -5115,31 +5103,27 @@ ; if no emulation is required, we don't need to shuffle or emulate edges ; this also saves 2 quasi-vpgatherdqs vptest m1, m1 jz .filter - movd r8d, xm1 - pextrd r9d, xm1, 1 - pextrd r10d, xm1, 2 - pextrd r11d, xm1, 3 - movsxd r8, r8d - movsxd r9, r9d - movsxd r10, r10d - movsxd r11, r11d + movq r9, xm1 + pextrq r11, xm1, 1 + movsxd r8, r9d + sar r9, 32 + movsxd r10, r11d + sar r11, 32 vextracti128 xm1, m1, 1 movq xm14, [base+resize_shuf+4+r8] movq xm0, [base+resize_shuf+4+r10] movhps xm14, [base+resize_shuf+4+r9] movhps xm0, [base+resize_shuf+4+r11] - movd r8d, xm1 - pextrd r9d, xm1, 1 - pextrd r10d, xm1, 2 - pextrd r11d, xm1, 3 - movsxd r8, r8d - movsxd r9, r9d - movsxd r10, r10d - movsxd r11, r11d + movq r9, xm1 + pextrq r11, xm1, 1 + movsxd r8, r9d + sar r9, 32 + movsxd r10, r11d + sar r11, 32 vinserti128 m14, [base+resize_shuf+4+r8], 1 vinserti128 m0, [base+resize_shuf+4+r10], 1 vpbroadcastq m10, [base+resize_shuf+4+r9] vpbroadcastq m11, [base+resize_shuf+4+r11] vpblendd m14, m10, 11000000b @@ -5189,11 +5173,11 @@ add srcq, src_strideq dec hd jg .loop_y RET -cglobal w_mask_420, 4, 8, 14, dst, stride, tmp1, tmp2, w, h, mask, stride3 +cglobal w_mask_420_8bpc, 4, 8, 14, dst, stride, tmp1, tmp2, w, h, mask, stride3 %define base r7-w_mask_420_avx2_table lea r7, [w_mask_420_avx2_table] tzcnt wd, wm mov r6d, r7m ; sign movifnidn hd, hm @@ -5395,11 +5379,11 @@ add maskq, 64 dec hd jg .w128_loop RET -cglobal w_mask_422, 4, 8, 11, dst, stride, tmp1, tmp2, w, h, mask, stride3 +cglobal w_mask_422_8bpc, 4, 8, 11, dst, stride, tmp1, tmp2, w, h, mask, stride3 %define base r7-w_mask_422_avx2_table lea r7, [w_mask_422_avx2_table] tzcnt wd, wm mov r6d, r7m ; sign movifnidn hd, hm @@ -5568,11 +5552,11 @@ mova [maskq+32*1], m5 dec hd jg .w128_loop RET -cglobal w_mask_444, 4, 8, 8, dst, stride, tmp1, tmp2, w, h, mask, stride3 +cglobal w_mask_444_8bpc, 4, 8, 8, dst, stride, tmp1, tmp2, w, h, mask, stride3 %define base r7-w_mask_444_avx2_table lea r7, [w_mask_444_avx2_table] tzcnt wd, wm movifnidn hd, hm mov maskq, maskmp diff --git a/third_party/dav1d/src/x86/mc_avx512.asm b/third_party/dav1d/src/x86/mc_avx512.asm --- a/third_party/dav1d/src/x86/mc_avx512.asm +++ b/third_party/dav1d/src/x86/mc_avx512.asm @@ -144,11 +144,11 @@ %rotate 1 %endrep %endmacro %macro HV_JMP_TABLE 5-* - %xdefine %%prefix mangle(private_prefix %+ _%1_%2_%3) + %xdefine %%prefix mangle(private_prefix %+ _%1_%2_8bpc_%3) %xdefine %%base %1_%3 %assign %%types %4 %if %%types & 1 %xdefine %1_%2_h_%3_table (%%h - %5) %%h: @@ -175,34 +175,34 @@ %rotate 1 %endrep %endif %endmacro -%macro BIDIR_JMP_TABLE 1-* - %xdefine %1_table (%%table - 2*%2) - %xdefine %%base %1_table - %xdefine %%prefix mangle(private_prefix %+ _%1) +%macro BIDIR_JMP_TABLE 2-* + %xdefine %1_%2_table (%%table - 2*%3) + %xdefine %%base %1_%2_table + %xdefine %%prefix mangle(private_prefix %+ _%1_8bpc_%2) %%table: - %rep %0 - 1 - dd %%prefix %+ .w%2 - %%base + %rep %0 - 2 + dd %%prefix %+ .w%3 - %%base %rotate 1 %endrep %endmacro -%xdefine prep_avx512icl mangle(private_prefix %+ _prep_bilin_avx512icl.prep) +%xdefine prep_avx512icl mangle(private_prefix %+ _prep_bilin_8bpc_avx512icl.prep) %define table_offset(type, fn) type %+ fn %+ SUFFIX %+ _table - type %+ SUFFIX BASE_JMP_TABLE prep, avx512icl, 4, 8, 16, 32, 64, 128 HV_JMP_TABLE prep, bilin, avx512icl, 7, 4, 8, 16, 32, 64, 128 HV_JMP_TABLE prep, 8tap, avx512icl, 7, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE avg_avx512icl, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE w_avg_avx512icl, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE mask_avx512icl, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE w_mask_420_avx512icl, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE w_mask_422_avx512icl, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE w_mask_444_avx512icl, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE avg, avx512icl, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_avg, avx512icl, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE mask, avx512icl, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_420, avx512icl, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_422, avx512icl, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_444, avx512icl, 4, 8, 16, 32, 64, 128 SECTION .text %macro REPX 2-* %xdefine %%f(x) %1 @@ -219,11 +219,11 @@ %endmacro DECLARE_REG_TMP 3, 5, 6 INIT_ZMM avx512icl -cglobal prep_bilin, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3 +cglobal prep_bilin_8bpc, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3 movifnidn mxyd, r5m ; mx lea t2, [prep_avx512icl] tzcnt wd, wm movifnidn hd, hm test mxyd, mxyd @@ -770,19 +770,19 @@ %assign FILTER_REGULAR (0*15 << 16) | 3*15 %assign FILTER_SMOOTH (1*15 << 16) | 4*15 %assign FILTER_SHARP (2*15 << 16) | 3*15 %macro FN 4 ; fn, type, type_h, type_v -cglobal %1_%2 +cglobal %1_%2_8bpc mov t0d, FILTER_%3 %ifidn %3, %4 mov t1d, t0d %else mov t1d, FILTER_%4 %endif %ifnidn %2, regular ; skip the jump in the last filter - jmp mangle(private_prefix %+ _%1 %+ SUFFIX) + jmp mangle(private_prefix %+ _%1_8bpc %+ SUFFIX) %endif %endmacro %macro PREP_8TAP_H 0 vpermb m10, m5, m0 @@ -827,11 +827,11 @@ PREP_8TAP_FN regular_sharp, REGULAR, SHARP PREP_8TAP_FN smooth_regular, SMOOTH, REGULAR PREP_8TAP_FN regular_smooth, REGULAR, SMOOTH PREP_8TAP_FN regular, REGULAR, REGULAR -cglobal prep_8tap, 3, 8, 0, tmp, src, stride, w, h, mx, my, stride3 +cglobal prep_8tap_8bpc, 3, 8, 0, tmp, src, stride, w, h, mx, my, stride3 imul mxd, mxm, 0x010101 add mxd, t0d ; 8tap_h, mx, 4tap_h imul myd, mym, 0x010101 add myd, t1d ; 8tap_v, my, 4tap_v lea r7, [prep_avx512icl] @@ -1751,11 +1751,11 @@ %macro AVG_INC_PTR 1 add tmp1q, %1*mmsize add tmp2q, %1*mmsize %endmacro -cglobal avg, 4, 7, 3, dst, stride, tmp1, tmp2, w, h, stride3 +cglobal avg_8bpc, 4, 7, 3, dst, stride, tmp1, tmp2, w, h, stride3 %define base r6-avg_avx512icl_table lea r6, [avg_avx512icl_table] tzcnt wd, wm movifnidn hd, hm movsxd wq, dword [r6+wq*4] @@ -1781,11 +1781,11 @@ packuswb m0, m1 %endmacro %define W_AVG_INC_PTR AVG_INC_PTR -cglobal w_avg, 4, 7, 6, dst, stride, tmp1, tmp2, w, h, stride3 +cglobal w_avg_8bpc, 4, 7, 6, dst, stride, tmp1, tmp2, w, h, stride3 %define base r6-w_avg_avx512icl_table lea r6, [w_avg_avx512icl_table] tzcnt wd, wm movifnidn hd, hm vpbroadcastw m4, r6m ; weight @@ -1835,11 +1835,11 @@ add maskq, %1*32 add tmp2q, %1*64 add tmp1q, %1*64 %endmacro -cglobal mask, 4, 8, 6, dst, stride, tmp1, tmp2, w, h, mask, stride3 +cglobal mask_8bpc, 4, 8, 6, dst, stride, tmp1, tmp2, w, h, mask, stride3 %define base r7-mask_avx512icl_table lea r7, [mask_avx512icl_table] tzcnt wd, wm movifnidn hd, hm mov maskq, maskmp @@ -1875,11 +1875,11 @@ pmulhrsw m%1, m7 pmulhrsw m1, m7 packuswb m%1, m1 %endmacro -cglobal w_mask_420, 4, 8, 16, dst, stride, tmp1, tmp2, w, h, mask, stride3 +cglobal w_mask_420_8bpc, 4, 8, 16, dst, stride, tmp1, tmp2, w, h, mask, stride3 %define base r7-w_mask_420_avx512icl_table lea r7, [w_mask_420_avx512icl_table] tzcnt wd, wm mov r6d, r7m ; sign movifnidn hd, hm @@ -2068,11 +2068,11 @@ lea dstq, [dstq+strideq*2] sub hd, 2 jg .w128_loop RET -cglobal w_mask_422, 4, 8, 14, dst, stride, tmp1, tmp2, w, h, mask, stride3 +cglobal w_mask_422_8bpc, 4, 8, 14, dst, stride, tmp1, tmp2, w, h, mask, stride3 %define base r7-w_mask_422_avx512icl_table lea r7, [w_mask_422_avx512icl_table] tzcnt wd, wm mov r6d, r7m ; sign movifnidn hd, hm @@ -2241,11 +2241,11 @@ add dstq, strideq dec hd jg .w128_loop RET -cglobal w_mask_444, 4, 8, 12, dst, stride, tmp1, tmp2, w, h, mask, stride3 +cglobal w_mask_444_8bpc, 4, 8, 12, dst, stride, tmp1, tmp2, w, h, mask, stride3 %define base r7-w_mask_444_avx512icl_table lea r7, [w_mask_444_avx512icl_table] tzcnt wd, wm movifnidn hd, hm movsxd wq, dword [r7+wq*4] diff --git a/third_party/dav1d/src/x86/mc_init_tmpl.c b/third_party/dav1d/src/x86/mc_init_tmpl.c --- a/third_party/dav1d/src/x86/mc_init_tmpl.c +++ b/third_party/dav1d/src/x86/mc_init_tmpl.c @@ -1,8 +1,8 @@ /* - * Copyright © 2018, VideoLAN and dav1d authors - * Copyright © 2018, Two Orioles, LLC + * Copyright © 2018-2021, VideoLAN and dav1d authors + * Copyright © 2018-2021, Two Orioles, LLC * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * @@ -26,39 +26,23 @@ */ #include "src/cpu.h" #include "src/mc.h" -#if BITDEPTH == 8 #define decl_fn(type, name) \ - decl_##type##_fn(name##_sse2); \ - decl_##type##_fn(name##_ssse3); \ - decl_##type##_fn(name##_avx2); \ - decl_##type##_fn(name##_avx512icl); + decl_##type##_fn(BF(name, sse2)); \ + decl_##type##_fn(BF(name, ssse3)); \ + decl_##type##_fn(BF(name, avx2)); \ + decl_##type##_fn(BF(name, avx512icl)); #define init_mc_fn(type, name, suffix) \ - c->mc[type] = dav1d_put_##name##_##suffix + c->mc[type] = BF(dav1d_put_##name, suffix) #define init_mct_fn(type, name, suffix) \ - c->mct[type] = dav1d_prep_##name##_##suffix + c->mct[type] = BF(dav1d_prep_##name, suffix) #define init_mc_scaled_fn(type, name, suffix) \ - c->mc_scaled[type] = dav1d_put_##name##_##suffix + c->mc_scaled[type] = BF(dav1d_put_##name, suffix) #define init_mct_scaled_fn(type, name, suffix) \ - c->mct_scaled[type] = dav1d_prep_##name##_##suffix -#else -#define decl_fn(type, name) \ - decl_##type##_fn(name##_16bpc_sse2); \ - decl_##type##_fn(name##_16bpc_ssse3); \ - decl_##type##_fn(name##_16bpc_avx2); \ - decl_##type##_fn(name##_16bpc_avx512icl); -#define init_mc_fn(type, name, suffix) \ - c->mc[type] = dav1d_put_##name##_16bpc_##suffix -#define init_mct_fn(type, name, suffix) \ - c->mct[type] = dav1d_prep_##name##_16bpc_##suffix -#define init_mc_scaled_fn(type, name, suffix) \ - c->mc_scaled[type] = dav1d_put_##name##_16bpc_##suffix -#define init_mct_scaled_fn(type, name, suffix) \ - c->mct_scaled[type] = dav1d_prep_##name##_16bpc_##suffix -#endif + c->mct_scaled[type] = BF(dav1d_prep_##name, suffix) decl_fn(mc, dav1d_put_8tap_regular); decl_fn(mc, dav1d_put_8tap_regular_smooth); decl_fn(mc, dav1d_put_8tap_regular_sharp); decl_fn(mc, dav1d_put_8tap_smooth); @@ -111,18 +95,17 @@ decl_fn(blend, dav1d_blend); decl_fn(blend_dir, dav1d_blend_v); decl_fn(blend_dir, dav1d_blend_h); decl_fn(warp8x8, dav1d_warp_affine_8x8); -decl_warp8x8_fn(dav1d_warp_affine_8x8_sse4); +decl_warp8x8_fn(BF(dav1d_warp_affine_8x8, sse4)); decl_fn(warp8x8t, dav1d_warp_affine_8x8t); -decl_warp8x8t_fn(dav1d_warp_affine_8x8t_sse4); +decl_warp8x8t_fn(BF(dav1d_warp_affine_8x8t, sse4)); decl_fn(emu_edge, dav1d_emu_edge); -decl_resize_fn(dav1d_resize_avx2); -decl_resize_fn(dav1d_resize_ssse3); +decl_fn(resize, dav1d_resize); COLD void bitfn(dav1d_mc_dsp_init_x86)(Dav1dMCDSPContext *const c) { const unsigned flags = dav1d_get_cpu_flags(); if(!(flags & DAV1D_X86_CPU_FLAG_SSE2)) @@ -138,12 +121,12 @@ init_mct_fn(FILTER_2D_8TAP_SMOOTH_SHARP, 8tap_smooth_sharp, sse2); init_mct_fn(FILTER_2D_8TAP_SHARP_REGULAR, 8tap_sharp_regular, sse2); init_mct_fn(FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_sharp_smooth, sse2); init_mct_fn(FILTER_2D_8TAP_SHARP, 8tap_sharp, sse2); - c->warp8x8 = dav1d_warp_affine_8x8_sse2; - c->warp8x8t = dav1d_warp_affine_8x8t_sse2; + c->warp8x8 = BF(dav1d_warp_affine_8x8, sse2); + c->warp8x8t = BF(dav1d_warp_affine_8x8t, sse2); #endif if(!(flags & DAV1D_X86_CPU_FLAG_SSSE3)) return; @@ -191,44 +174,30 @@ init_mct_scaled_fn(FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_scaled_sharp_smooth, ssse3); init_mct_scaled_fn(FILTER_2D_8TAP_SHARP, 8tap_scaled_sharp, ssse3); init_mct_scaled_fn(FILTER_2D_BILINEAR, bilin_scaled, ssse3); #endif -#if BITDEPTH == 8 - c->avg = dav1d_avg_ssse3; - c->w_avg = dav1d_w_avg_ssse3; - c->mask = dav1d_mask_ssse3; - c->w_mask[2] = dav1d_w_mask_420_ssse3; - c->blend = dav1d_blend_ssse3; - c->blend_v = dav1d_blend_v_ssse3; - c->blend_h = dav1d_blend_h_ssse3; - - c->warp8x8 = dav1d_warp_affine_8x8_ssse3; - c->warp8x8t = dav1d_warp_affine_8x8t_ssse3; - - c->emu_edge = dav1d_emu_edge_ssse3; - c->resize = dav1d_resize_ssse3; -#else - c->avg = dav1d_avg_16bpc_ssse3; - c->w_avg = dav1d_w_avg_16bpc_ssse3; - c->mask = dav1d_mask_16bpc_ssse3; - c->w_mask[0] = dav1d_w_mask_444_16bpc_ssse3; - c->w_mask[1] = dav1d_w_mask_422_16bpc_ssse3; - c->w_mask[2] = dav1d_w_mask_420_16bpc_ssse3; - c->blend = dav1d_blend_16bpc_ssse3; - c->blend_v = dav1d_blend_v_16bpc_ssse3; - c->blend_h = dav1d_blend_h_16bpc_ssse3; - - c->emu_edge = dav1d_emu_edge_16bpc_ssse3; -#endif + c->avg = BF(dav1d_avg, ssse3); + c->w_avg = BF(dav1d_w_avg, ssse3); + c->mask = BF(dav1d_mask, ssse3); + c->w_mask[0] = BF(dav1d_w_mask_444, ssse3); + c->w_mask[1] = BF(dav1d_w_mask_422, ssse3); + c->w_mask[2] = BF(dav1d_w_mask_420, ssse3); + c->blend = BF(dav1d_blend, ssse3); + c->blend_v = BF(dav1d_blend_v, ssse3); + c->blend_h = BF(dav1d_blend_h, ssse3); + c->warp8x8 = BF(dav1d_warp_affine_8x8, ssse3); + c->warp8x8t = BF(dav1d_warp_affine_8x8t, ssse3); + c->emu_edge = BF(dav1d_emu_edge, ssse3); + c->resize = BF(dav1d_resize, ssse3); if(!(flags & DAV1D_X86_CPU_FLAG_SSE41)) return; #if BITDEPTH == 8 - c->warp8x8 = dav1d_warp_affine_8x8_sse4; - c->warp8x8t = dav1d_warp_affine_8x8t_sse4; + c->warp8x8 = BF(dav1d_warp_affine_8x8, sse4); + c->warp8x8t = BF(dav1d_warp_affine_8x8t, sse4); #endif #if ARCH_X86_64 if (!(flags & DAV1D_X86_CPU_FLAG_AVX2)) return; @@ -253,11 +222,10 @@ init_mct_fn(FILTER_2D_8TAP_SHARP_REGULAR, 8tap_sharp_regular, avx2); init_mct_fn(FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_sharp_smooth, avx2); init_mct_fn(FILTER_2D_8TAP_SHARP, 8tap_sharp, avx2); init_mct_fn(FILTER_2D_BILINEAR, bilin, avx2); -#if BITDEPTH == 8 init_mc_scaled_fn(FILTER_2D_8TAP_REGULAR, 8tap_scaled_regular, avx2); init_mc_scaled_fn(FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_scaled_regular_smooth, avx2); init_mc_scaled_fn(FILTER_2D_8TAP_REGULAR_SHARP, 8tap_scaled_regular_sharp, avx2); init_mc_scaled_fn(FILTER_2D_8TAP_SMOOTH_REGULAR, 8tap_scaled_smooth_regular, avx2); init_mc_scaled_fn(FILTER_2D_8TAP_SMOOTH, 8tap_scaled_smooth, avx2); @@ -276,39 +244,23 @@ init_mct_scaled_fn(FILTER_2D_8TAP_SHARP_REGULAR, 8tap_scaled_sharp_regular, avx2); init_mct_scaled_fn(FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_scaled_sharp_smooth, avx2); init_mct_scaled_fn(FILTER_2D_8TAP_SHARP, 8tap_scaled_sharp, avx2); init_mct_scaled_fn(FILTER_2D_BILINEAR, bilin_scaled, avx2); - c->avg = dav1d_avg_avx2; - c->w_avg = dav1d_w_avg_avx2; - c->mask = dav1d_mask_avx2; - c->w_mask[0] = dav1d_w_mask_444_avx2; - c->w_mask[1] = dav1d_w_mask_422_avx2; - c->w_mask[2] = dav1d_w_mask_420_avx2; - c->blend = dav1d_blend_avx2; - c->blend_v = dav1d_blend_v_avx2; - c->blend_h = dav1d_blend_h_avx2; - - c->warp8x8 = dav1d_warp_affine_8x8_avx2; - c->warp8x8t = dav1d_warp_affine_8x8t_avx2; - - c->emu_edge = dav1d_emu_edge_avx2; - c->resize = dav1d_resize_avx2; -#else - c->avg = dav1d_avg_16bpc_avx2; - c->w_avg = dav1d_w_avg_16bpc_avx2; - c->mask = dav1d_mask_16bpc_avx2; - c->w_mask[0] = dav1d_w_mask_444_16bpc_avx2; - c->w_mask[1] = dav1d_w_mask_422_16bpc_avx2; - c->w_mask[2] = dav1d_w_mask_420_16bpc_avx2; - c->blend = dav1d_blend_16bpc_avx2; - c->blend_v = dav1d_blend_v_16bpc_avx2; - c->blend_h = dav1d_blend_h_16bpc_avx2; - c->warp8x8 = dav1d_warp_affine_8x8_16bpc_avx2; - c->warp8x8t = dav1d_warp_affine_8x8t_16bpc_avx2; - c->emu_edge = dav1d_emu_edge_16bpc_avx2; -#endif + c->avg = BF(dav1d_avg, avx2); + c->w_avg = BF(dav1d_w_avg, avx2); + c->mask = BF(dav1d_mask, avx2); + c->w_mask[0] = BF(dav1d_w_mask_444, avx2); + c->w_mask[1] = BF(dav1d_w_mask_422, avx2); + c->w_mask[2] = BF(dav1d_w_mask_420, avx2); + c->blend = BF(dav1d_blend, avx2); + c->blend_v = BF(dav1d_blend_v, avx2); + c->blend_h = BF(dav1d_blend_h, avx2); + c->warp8x8 = BF(dav1d_warp_affine_8x8, avx2); + c->warp8x8t = BF(dav1d_warp_affine_8x8t, avx2); + c->emu_edge = BF(dav1d_emu_edge, avx2); + c->resize = BF(dav1d_resize, avx2); if (!(flags & DAV1D_X86_CPU_FLAG_AVX512ICL)) return; #if HAVE_AVX512ICL && BITDEPTH == 8 @@ -321,14 +273,14 @@ init_mct_fn(FILTER_2D_8TAP_SHARP_REGULAR, 8tap_sharp_regular, avx512icl); init_mct_fn(FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_sharp_smooth, avx512icl); init_mct_fn(FILTER_2D_8TAP_SHARP, 8tap_sharp, avx512icl); init_mct_fn(FILTER_2D_BILINEAR, bilin, avx512icl); - c->avg = dav1d_avg_avx512icl; - c->w_avg = dav1d_w_avg_avx512icl; - c->mask = dav1d_mask_avx512icl; - c->w_mask[0] = dav1d_w_mask_444_avx512icl; - c->w_mask[1] = dav1d_w_mask_422_avx512icl; - c->w_mask[2] = dav1d_w_mask_420_avx512icl; + c->avg = BF(dav1d_avg, avx512icl); + c->w_avg = BF(dav1d_w_avg, avx512icl); + c->mask = BF(dav1d_mask, avx512icl); + c->w_mask[0] = BF(dav1d_w_mask_444, avx512icl); + c->w_mask[1] = BF(dav1d_w_mask_422, avx512icl); + c->w_mask[2] = BF(dav1d_w_mask_420, avx512icl); #endif #endif } diff --git a/third_party/dav1d/src/x86/mc_sse.asm b/third_party/dav1d/src/x86/mc_sse.asm --- a/third_party/dav1d/src/x86/mc_sse.asm +++ b/third_party/dav1d/src/x86/mc_sse.asm @@ -58,21 +58,24 @@ subpel_s_shuf2: db 0, 1, 2, 3, 0, 1, 2, 3, 8, 9, 10, 11, 8, 9, 10, 11 subpel_s_shuf8: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15 bilin_h_shuf4: db 0, 1, 1, 2, 2, 3, 3, 4, 8, 9, 9, 10, 10, 11, 11, 12 bilin_h_shuf8: db 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 unpckw: db 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15 +rescale_mul: dd 0, 1, 2, 3 +resize_shuf: db 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 7, 7, 7, 7 + +wm_420_sign: times 4 dw 258 + times 4 dw 257 +wm_422_sign: times 8 db 128 + times 8 db 127 pb_8x0_8x8: times 8 db 0 times 8 db 8 bdct_lb_dw: times 4 db 0 times 4 db 4 times 4 db 8 times 4 db 12 -rescale_mul: dd 0, 1, 2, 3 -resize_shuf: times 5 db 0 - db 1, 2, 3, 4, 5, 6 - times 5+16 db 7 pb_64: times 16 db 64 pw_m256: times 8 dw -256 pw_1: times 8 dw 1 pw_2: times 8 dw 2 @@ -200,30 +203,32 @@ pw_258: times 2 dw 258 cextern mc_subpel_filters %define subpel_filters (mangle(private_prefix %+ _mc_subpel_filters)-8) -%macro BIDIR_JMP_TABLE 1-* +%macro BIDIR_JMP_TABLE 2-* ;evaluated at definition time (in loop below) - %xdefine %1_table (%%table - 2*%2) - %xdefine %%base %1_table - %xdefine %%prefix mangle(private_prefix %+ _%1) + %xdefine %1_%2_table (%%table - 2*%3) + %xdefine %%base %1_%2_table + %xdefine %%prefix mangle(private_prefix %+ _%1_8bpc_%2) ; dynamically generated label %%table: - %rep %0 - 1 ; repeat for num args - dd %%prefix %+ .w%2 - %%base + %rep %0 - 2 ; repeat for num args + dd %%prefix %+ .w%3 - %%base %rotate 1 %endrep %endmacro -BIDIR_JMP_TABLE avg_ssse3, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE w_avg_ssse3, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE mask_ssse3, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE w_mask_420_ssse3, 4, 8, 16, 16, 16, 16 -BIDIR_JMP_TABLE blend_ssse3, 4, 8, 16, 32 -BIDIR_JMP_TABLE blend_v_ssse3, 2, 4, 8, 16, 32 -BIDIR_JMP_TABLE blend_h_ssse3, 2, 4, 8, 16, 16, 16, 16 +BIDIR_JMP_TABLE avg, ssse3, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_avg, ssse3, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE mask, ssse3, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_420, ssse3, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_422, ssse3, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_444, ssse3, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE blend, ssse3, 4, 8, 16, 32 +BIDIR_JMP_TABLE blend_v, ssse3, 2, 4, 8, 16, 32 +BIDIR_JMP_TABLE blend_h, ssse3, 2, 4, 8, 16, 16, 16, 16 %macro BASE_JMP_TABLE 3-* %xdefine %1_%2_table (%%table - %3) %xdefine %%base %1_%2 %%table: @@ -231,19 +236,19 @@ dw %%base %+ _w%3 - %%base %rotate 1 %endrep %endmacro -%xdefine prep_sse2 mangle(private_prefix %+ _prep_bilin_sse2.prep) -%xdefine put_ssse3 mangle(private_prefix %+ _put_bilin_ssse3.put) -%xdefine prep_ssse3 mangle(private_prefix %+ _prep_bilin_ssse3.prep) +%xdefine prep_sse2 mangle(private_prefix %+ _prep_bilin_8bpc_sse2.prep) +%xdefine put_ssse3 mangle(private_prefix %+ _put_bilin_8bpc_ssse3.put) +%xdefine prep_ssse3 mangle(private_prefix %+ _prep_bilin_8bpc_ssse3.prep) BASE_JMP_TABLE put, ssse3, 2, 4, 8, 16, 32, 64, 128 BASE_JMP_TABLE prep, ssse3, 4, 8, 16, 32, 64, 128 %macro HV_JMP_TABLE 5-* - %xdefine %%prefix mangle(private_prefix %+ _%1_%2_%3) + %xdefine %%prefix mangle(private_prefix %+ _%1_%2_8bpc_%3) %xdefine %%base %1_%3 %assign %%types %4 %if %%types & 1 %xdefine %1_%2_h_%3_table (%%h - %5) %%h: @@ -277,37 +282,37 @@ HV_JMP_TABLE put, 8tap, ssse3, 3, 2, 4, 8, 16, 32, 64, 128 HV_JMP_TABLE prep, 8tap, ssse3, 1, 4, 8, 16, 32, 64, 128 HV_JMP_TABLE put, bilin, ssse3, 7, 2, 4, 8, 16, 32, 64, 128 HV_JMP_TABLE prep, bilin, ssse3, 7, 4, 8, 16, 32, 64, 128 -%macro SCALED_JMP_TABLE 1-* - %xdefine %1_table (%%table - %2) - %xdefine %%base mangle(private_prefix %+ _%1) +%macro SCALED_JMP_TABLE 2-* + %xdefine %1_%2_table (%%table - %3) + %xdefine %%base mangle(private_prefix %+ _%1_8bpc_%2) %%table: - %rep %0 - 1 - dw %%base %+ .w%2 - %%base + %rep %0 - 2 + dw %%base %+ .w%3 - %%base %rotate 1 %endrep - %rotate 1 + %rotate 2 %%dy_1024: - %xdefine %1_dy1_table (%%dy_1024 - %2) - %rep %0 - 1 - dw %%base %+ .dy1_w%2 - %%base + %xdefine %1_%2_dy1_table (%%dy_1024 - %3) + %rep %0 - 2 + dw %%base %+ .dy1_w%3 - %%base %rotate 1 %endrep - %rotate 1 + %rotate 2 %%dy_2048: - %xdefine %1_dy2_table (%%dy_2048 - %2) - %rep %0 - 1 - dw %%base %+ .dy2_w%2 - %%base + %xdefine %1_%2_dy2_table (%%dy_2048 - %3) + %rep %0 - 2 + dw %%base %+ .dy2_w%3 - %%base %rotate 1 %endrep %endmacro %if ARCH_X86_64 -SCALED_JMP_TABLE put_8tap_scaled_ssse3, 2, 4, 8, 16, 32, 64, 128 -SCALED_JMP_TABLE prep_8tap_scaled_ssse3, 4, 8, 16, 32, 64, 128 +SCALED_JMP_TABLE put_8tap_scaled, ssse3, 2, 4, 8, 16, 32, 64, 128 +SCALED_JMP_TABLE prep_8tap_scaled, ssse3, 4, 8, 16, 32, 64, 128 %endif %define table_offset(type, fn) type %+ fn %+ SUFFIX %+ _table - type %+ SUFFIX SECTION .text @@ -326,11 +331,11 @@ %if ARCH_X86_32 mov %1, dsm ; restore dsq %endif %endmacro -cglobal put_bilin, 1, 8, 0, dst, ds, src, ss, w, h, mxy +cglobal put_bilin_8bpc, 1, 8, 0, dst, ds, src, ss, w, h, mxy movifnidn mxyd, r6m ; mx LEA t0, put_ssse3 movifnidn srcq, srcmp movifnidn ssq, ssmp tzcnt wd, wm @@ -765,11 +770,11 @@ sub hd, 2 jg .hv_w2_loop RET .hv_w4: mova m4, [base+bilin_h_shuf4] - movddup xm0, [srcq+ssq*0] + movddup m0, [srcq+ssq*0] movifnidn dsq, dsmp pshufb m0, m4 pmaddubsw m0, m5 .hv_w4_loop: movq m1, [srcq+ssq*1] @@ -951,11 +956,11 @@ %define base r6-prep%+SUFFIX %else %define base 0 %endif -cglobal prep_bilin, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3 +cglobal prep_bilin_8bpc, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3 movifnidn mxyd, r5m ; mx LEA r6, prep%+SUFFIX tzcnt wd, wm movifnidn hd, hm test mxyd, mxyd @@ -1548,19 +1553,19 @@ %assign FILTER_REGULAR (0*15 << 16) | 3*15 %assign FILTER_SMOOTH (1*15 << 16) | 4*15 %assign FILTER_SHARP (2*15 << 16) | 3*15 %macro FN 4 ; prefix, type, type_h, type_v -cglobal %1_%2 +cglobal %1_%2_8bpc mov t0d, FILTER_%3 %ifidn %3, %4 mov t1d, t0d %else mov t1d, FILTER_%4 %endif %ifnidn %2, regular ; skip the jump in the last filter - jmp mangle(private_prefix %+ _%1 %+ SUFFIX) + jmp mangle(private_prefix %+ _%1_8bpc %+ SUFFIX) %endif %endmacro %if ARCH_X86_32 DECLARE_REG_TMP 1, 2 @@ -1586,11 +1591,11 @@ %else %define base_reg r8 %define base 0 %endif -cglobal put_8tap, 1, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 +cglobal put_8tap_8bpc, 1, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 %assign org_stack_offset stack_offset imul mxd, mxm, 0x010101 add mxd, t0d ; 8tap_h, mx, 4tap_h %if ARCH_X86_64 imul myd, mym, 0x010101 @@ -2024,12 +2029,12 @@ paddw m14, m12 paddw m15, m13 pmulhrsw m14, m7 pmulhrsw m15, m7 packuswb m14, m15 - movq [dstq+dsq*0], xm14 - movhps [dstq+dsq*1], xm14 + movq [dstq+dsq*0], m14 + movhps [dstq+dsq*1], m14 lea dstq, [dstq+dsq*2] sub hd, 2 jg .v_w8_loop add r4, 8 add r7, 8 @@ -2837,11 +2842,11 @@ %define base base_reg-prep%+SUFFIX %else %define base_reg r7 %define base 0 %endif -cglobal prep_8tap, 1, 9, 0, tmp, src, stride, w, h, mx, my, stride3 +cglobal prep_8tap_8bpc, 1, 9, 0, tmp, src, stride, w, h, mx, my, stride3 %assign org_stack_offset stack_offset imul mxd, mxm, 0x010101 add mxd, t0d ; 8tap_h, mx, 4tap_h imul myd, mym, 0x010101 add myd, t1d ; 8tap_v, my, 4tap_v @@ -4018,30 +4023,30 @@ %macro MC_8TAP_SCALED 1 %ifidn %1, put %assign isprep 0 %if required_stack_alignment <= STACK_ALIGNMENT -cglobal put_8tap_scaled, 4, 15, 16, 0x180, dst, ds, src, ss, w, h, mx, my, dx, dy +cglobal put_8tap_scaled_8bpc, 4, 15, 16, 0x180, dst, ds, src, ss, w, h, mx, my, dx, dy %else -cglobal put_8tap_scaled, 4, 14, 16, 0x180, dst, ds, src, ss, w, h, mx, my, dx, dy +cglobal put_8tap_scaled_8bpc, 4, 14, 16, 0x180, dst, ds, src, ss, w, h, mx, my, dx, dy %endif %xdefine base_reg r12 %define rndshift 10 %else %assign isprep 1 %if required_stack_alignment <= STACK_ALIGNMENT -cglobal prep_8tap_scaled, 4, 15, 16, 0x180, tmp, src, ss, w, h, mx, my, dx, dy +cglobal prep_8tap_scaled_8bpc, 4, 15, 16, 0x180, tmp, src, ss, w, h, mx, my, dx, dy %xdefine tmp_stridem r14q %else -cglobal prep_8tap_scaled, 4, 14, 16, 0x180, tmp, src, ss, w, h, mx, my, dx, dy +cglobal prep_8tap_scaled_8bpc, 4, 14, 16, 0x180, tmp, src, ss, w, h, mx, my, dx, dy %define tmp_stridem qword [rsp+0x138] %endif %xdefine base_reg r11 %define rndshift 6 %endif - LEA base_reg, %1_8tap_scaled_ssse3 -%define base base_reg-%1_8tap_scaled_ssse3 + LEA base_reg, %1_8tap_scaled_8bpc_ssse3 +%define base base_reg-%1_8tap_scaled_8bpc_ssse3 tzcnt wd, wm movd m8, dxm movd m14, mxm pshufd m8, m8, q0000 pshufd m14, m14, q0000 @@ -4697,13 +4702,13 @@ paddb m14, m6 movq m1, [srcq+ssq*0] movq m3, [srcq+ssq*2] movhps m1, [srcq+ssq*1] add srcq, ss3q - movq xm10, r4q - punpcklbw xm10, xm10 - psraw xm10, 8 + movq m10, r4q + punpcklbw m10, m10 + psraw m10, 8 punpckldq m15, m7 punpcklqdq m15, m15 pand m11, m8 pandn m8, m15 SWAP m15, m8 @@ -5620,14 +5625,14 @@ MC_8TAP_SCALED_RET 0 %undef isprep %endmacro %macro BILIN_SCALED_FN 1 -cglobal %1_bilin_scaled +cglobal %1_bilin_scaled_8bpc mov t0d, (5*15 << 16) | 5*15 mov t1d, (5*15 << 16) | 5*15 - jmp mangle(private_prefix %+ _%1_8tap_scaled %+ SUFFIX) + jmp mangle(private_prefix %+ _%1_8tap_scaled_8bpc %+ SUFFIX) %endmacro %if ARCH_X86_64 %if WIN64 DECLARE_REG_TMP 6, 5 @@ -5817,19 +5822,19 @@ %endif %endif %macro WARP_AFFINE_8X8T 0 %if ARCH_X86_64 -cglobal warp_affine_8x8t, 6, 14, 16, 0x90, tmp, ts +cglobal warp_affine_8x8t_8bpc, 6, 14, 16, 0x90, tmp, ts %else -cglobal warp_affine_8x8t, 0, 7, 16, -0x130-copy_args, tmp, ts +cglobal warp_affine_8x8t_8bpc, 0, 7, 16, -0x130-copy_args, tmp, ts %if copy_args %define tmpm [esp+stack_size-4*1] %define tsm [esp+stack_size-4*2] %endif %endif - call mangle(private_prefix %+ _warp_affine_8x8_%+cpuname).main + call mangle(private_prefix %+ _warp_affine_8x8_8bpc_%+cpuname).main .loop: %if ARCH_X86_32 %define m12 m4 %define m13 m5 %define m14 m6 @@ -5866,30 +5871,30 @@ packssdw m14, m15 %endif mova [tmpq+tsq*0], m12 mova [tmpq+tsq*2], m14 dec counterd - jz mangle(private_prefix %+ _warp_affine_8x8_%+cpuname).end + jz mangle(private_prefix %+ _warp_affine_8x8_8bpc_%+cpuname).end %if ARCH_X86_32 mov tmpm, tmpd mov r0, [esp+0x100] mov r1, [esp+0x104] %endif - call mangle(private_prefix %+ _warp_affine_8x8_%+cpuname).main2 + call mangle(private_prefix %+ _warp_affine_8x8_8bpc_%+cpuname).main2 lea tmpq, [tmpq+tsq*4] jmp .loop %endmacro %macro WARP_AFFINE_8X8 0 %if ARCH_X86_64 -cglobal warp_affine_8x8, 6, 14, 16, 0x90, \ - dst, ds, src, ss, abcd, mx, tmp2, alpha, beta, \ - filter, tmp1, delta, my, gamma +cglobal warp_affine_8x8_8bpc, 6, 14, 16, 0x90, \ + dst, ds, src, ss, abcd, mx, tmp2, alpha, beta, \ + filter, tmp1, delta, my, gamma %else -cglobal warp_affine_8x8, 0, 7, 16, -0x130-copy_args, \ - dst, ds, src, ss, abcd, mx, tmp2, alpha, beta, \ - filter, tmp1, delta, my, gamma +cglobal warp_affine_8x8_8bpc, 0, 7, 16, -0x130-copy_args, \ + dst, ds, src, ss, abcd, mx, tmp2, alpha, beta, \ + filter, tmp1, delta, my, gamma %define alphaq r0 %define alphad r0 %define alpham [esp+gprsize+0x100] %define betaq r1 %define betad r1 @@ -6473,11 +6478,11 @@ %macro AVG_INC_PTR 1 add tmp1q, %1*mmsize add tmp2q, %1*mmsize %endmacro -cglobal avg, 4, 7, 3, dst, stride, tmp1, tmp2, w, h, stride3 +cglobal avg_8bpc, 4, 7, 3, dst, stride, tmp1, tmp2, w, h, stride3 LEA r6, avg_ssse3_table tzcnt wd, wm ; leading zeros movifnidn hd, hm ; move h(stack) to h(register) if not already that register movsxd wq, dword [r6+wq*4] ; push table entry matching the tile width (tzcnt) in widen reg mova m2, [pw_1024+r6-avg_ssse3_table] ; fill m2 with shift/align @@ -6504,11 +6509,11 @@ packuswb m0, m1 %endmacro %define W_AVG_INC_PTR AVG_INC_PTR -cglobal w_avg, 4, 7, 6, dst, stride, tmp1, tmp2, w, h, stride3 +cglobal w_avg_8bpc, 4, 7, 6, dst, stride, tmp1, tmp2, w, h, stride3 LEA r6, w_avg_ssse3_table tzcnt wd, wm movd m4, r6m movifnidn hd, hm pxor m0, m0 @@ -6558,14 +6563,14 @@ add tmp1q, %1*mmsize add tmp2q, %1*mmsize %endmacro %if ARCH_X86_64 -cglobal mask, 4, 8, 7, dst, stride, tmp1, tmp2, w, h, mask, stride3 +cglobal mask_8bpc, 4, 8, 7, dst, stride, tmp1, tmp2, w, h, mask, stride3 movifnidn hd, hm %else -cglobal mask, 4, 7, 7, dst, stride, tmp1, tmp2, w, mask, stride3 +cglobal mask_8bpc, 4, 7, 7, dst, stride, tmp1, tmp2, w, mask, stride3 %define hd dword r5m %endif %define base r6-mask_ssse3_table LEA r6, mask_ssse3_table tzcnt wd, wm @@ -6575,171 +6580,494 @@ add wq, r6 mov maskq, r6m BIDIR_FN MASK %undef hd -%macro W_MASK_420_B 2 ; src_offset in bytes, mask_out - ;**** do m0 = u16.dst[7..0], m%2 = u16.m[7..0] **** - mova m0, [tmp1q+(%1)] - mova m1, [tmp2q+(%1)] - mova m2, reg_pw_6903 - psubw m1, m0 - pabsw m%2, m1 ; abs(tmp1 - tmp2) - mova m3, m2 - psubusw m2, m%2 - psrlw m2, 8 ; 64 - m - mova m%2, m2 - psllw m2, 10 - pmulhw m1, m2 ; tmp2 * () - paddw m0, m1 ; tmp1 + () - ;**** do m1 = u16.dst[7..0], m%2 = u16.m[7..0] **** - mova m1, [tmp1q+(%1)+mmsize] - mova m2, [tmp2q+(%1)+mmsize] - psubw m2, m1 - pabsw m7, m2 ; abs(tmp1 - tmp2) - psubusw m3, m7 - psrlw m3, 8 ; 64 - m - phaddw m%2, m3 ; pack both u16.m[8..0]runs as u8.m [15..0] - psllw m3, 10 - pmulhw m2, m3 -%if ARCH_X86_32 - mova reg_pw_2048, [base+pw_2048] -%endif - paddw m1, m2 - pmulhrsw m0, reg_pw_2048 ; round/scale 2048 - pmulhrsw m1, reg_pw_2048 ; round/scale 2048 - packuswb m0, m1 ; concat m0 = u8.dst[15..0] +%macro W_MASK_420_END 1-* +%rep %0 + call .main + paddw m2, [maskq+16*%1] + mova [maskq+16*%1], m2 + mova [dstq+strideq*1+16*(2*%1+0)], m0 + call .main + psubw m3, m7, m2 + psubw m1, m7, [maskq+16*%1] + psubw m3, [dstq+strideq*1+16*(2*%1+1)] + psrlw m1, 2 + psrlw m3, 2 + packuswb m1, m3 + mova [maskq+16*%1], m1 + mova [dstq+strideq*1+16*(2*%1+1)], m0 + %rotate 1 +%endrep %endmacro -%macro W_MASK_420 2 - W_MASK_420_B (%1*16), %2 -%endmacro - -%define base r6-w_mask_420_ssse3_table +%if UNIX64 +DECLARE_REG_TMP 7 +%else +DECLARE_REG_TMP 5 +%endif + +cglobal w_mask_420_8bpc, 4, 7, 9, dst, stride, tmp1, tmp2, w, h, mask +%define base t0-w_mask_420_ssse3_table + LEA t0, w_mask_420_ssse3_table + tzcnt wd, wm + mov r6d, r7m ; sign + sub tmp2q, tmp1q + movsxd wq, [t0+wq*4] + mova m6, [base+pw_2048] + movddup m7, [base+wm_420_sign+r6*8] ; 258 - sign + add wq, t0 %if ARCH_X86_64 -%define reg_pw_6903 m8 -%define reg_pw_2048 m9 -; args: dst, stride, tmp1, tmp2, w, h, mask, sign -cglobal w_mask_420, 4, 8, 10, dst, stride, tmp1, tmp2, w, h, mask - lea r6, [w_mask_420_ssse3_table] - mov wd, wm - tzcnt r7d, wd - movd m0, r7m ; sign + mova m8, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 movifnidn hd, hm - movsxd r7, [r6+r7*4] - mova reg_pw_6903, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 - mova reg_pw_2048, [base+pw_2048] - movd m6, [base+pw_258] ; 64 * 4 + 2 - add r7, r6 +%else + %define m8 [base+pw_6903] + %define hd dword hm +%endif mov maskq, maskmp - psubw m6, m0 - pshuflw m6, m6, q0000 - punpcklqdq m6, m6 - W_MASK_420 0, 4 - jmp r7 - %define loop_w r7d -%else -%define reg_pw_6903 [base+pw_6903] -%define reg_pw_2048 m3 -cglobal w_mask_420, 4, 7, 8, dst, stride, tmp1, tmp2, w, mask - tzcnt wd, wm - LEA r6, w_mask_420_ssse3_table - movd m0, r7m ; sign - mov maskq, r6mp - mov wd, [r6+wq*4] - movd m6, [base+pw_258] - add wq, r6 - psubw m6, m0 - pshuflw m6, m6, q0000 - punpcklqdq m6, m6 - W_MASK_420 0, 4 - jmp wd - %define loop_w dword r0m - %define hd dword r5m -%endif + call .main + jmp wq .w4_loop: - add tmp1q, 2*16 - add tmp2q, 2*16 - W_MASK_420 0, 4 + call .main + add maskq, 4 lea dstq, [dstq+strideq*2] - add maskq, 4 .w4: - movd [dstq ], m0 ; copy m0[0] - pshuflw m1, m0, q1032 - movd [dstq+strideq*1], m1 ; copy m0[1] - lea dstq, [dstq+strideq*2] - punpckhqdq m0, m0 - movd [dstq+strideq*0], m0 ; copy m0[2] - psrlq m0, 32 - movd [dstq+strideq*1], m0 ; copy m0[3] - psubw m1, m6, m4 ; a _ c _ - psrlq m4, 32 ; b _ d _ - psubw m1, m4 + pshufd m3, m2, q2020 + pshufd m2, m2, q3131 + psubw m1, m7, m3 + psubw m1, m2 psrlw m1, 2 packuswb m1, m1 - pshuflw m1, m1, q2020 movd [maskq], m1 + movd [dstq+strideq*0], m0 + pshuflw m1, m0, q1032 + movd [dstq+strideq*1], m1 + punpckhqdq m0, m0 + lea dstq, [dstq+strideq*2] + movd [dstq+strideq*0], m0 + pshuflw m1, m0, q1032 + movd [dstq+strideq*1], m1 sub hd, 4 jg .w4_loop RET .w8_loop: - add tmp1q, 2*16 - add tmp2q, 2*16 - W_MASK_420 0, 4 + call .main + add maskq, 4 lea dstq, [dstq+strideq*2] - add maskq, 4 .w8: - movq [dstq ], m0 + movhlps m3, m2 + psubw m1, m7, m2 + psubw m1, m3 + psrlw m1, 2 + packuswb m1, m1 + movd [maskq], m1 + movq [dstq+strideq*0], m0 movhps [dstq+strideq*1], m0 - psubw m0, m6, m4 - punpckhqdq m4, m4 - psubw m0, m4 - psrlw m0, 2 - packuswb m0, m0 - movd [maskq], m0 sub hd, 2 jg .w8_loop RET -.w16: ; w32/64/128 -%if ARCH_X86_32 - mov wd, wm ; because we altered it in 32bit setup -%endif - mov loop_w, wd ; use width as counter - jmp .w16ge_inner_loop_first -.w16ge_loop: - lea tmp1q, [tmp1q+wq*2] ; skip even line pixels - lea tmp2q, [tmp2q+wq*2] ; skip even line pixels - sub dstq, wq - mov loop_w, wd +.w16_loop: + call .main + add maskq, 8 lea dstq, [dstq+strideq*2] -.w16ge_inner_loop: - W_MASK_420_B 0, 4 -.w16ge_inner_loop_first: - mova [dstq ], m0 - W_MASK_420_B wq*2, 5 ; load matching even line (offset = widthpx * (16+16)) - mova [dstq+strideq*1], m0 - psubw m1, m6, m4 ; m9 == 64 * 4 + 2 - psubw m1, m5 ; - odd line mask - psrlw m1, 2 ; >> 2 +.w16: + mova [dstq+strideq*1], m2 + mova [dstq+strideq*0], m0 + call .main + psubw m1, m7, [dstq+strideq*1] + psubw m1, m2 + psrlw m1, 2 packuswb m1, m1 movq [maskq], m1 - add tmp1q, 2*16 - add tmp2q, 2*16 + mova [dstq+strideq*1], m0 + sub hd, 2 + jg .w16_loop + RET +.w32_loop: + call .main + add maskq, 16 + lea dstq, [dstq+strideq*2] +.w32: + mova [maskq], m2 + mova [dstq+strideq*0+16*0], m0 + call .main + mova [dstq+strideq*1+16*1], m2 + mova [dstq+strideq*0+16*1], m0 + W_MASK_420_END 0 + sub hd, 2 + jg .w32_loop + RET +.w64_loop: + call .main + add maskq, 16*2 + lea dstq, [dstq+strideq*2] +.w64: + mova [maskq+16*0], m2 + mova [dstq+strideq*0+16*0], m0 + call .main + mova [dstq+strideq*1+16*1], m2 + mova [dstq+strideq*0+16*1], m0 + call .main + mova [maskq+16*1], m2 + mova [dstq+strideq*0+16*2], m0 + call .main + mova [dstq+strideq*1+16*3], m2 + mova [dstq+strideq*0+16*3], m0 + W_MASK_420_END 0, 1 + sub hd, 2 + jg .w64_loop + RET +.w128_loop: + call .main + add maskq, 16*4 + lea dstq, [dstq+strideq*2] +.w128: + mova [maskq+16*0], m2 + mova [dstq+strideq*0+16*0], m0 + call .main + mova [dstq+strideq*1+16*1], m2 + mova [dstq+strideq*0+16*1], m0 + call .main + mova [maskq+16*1], m2 + mova [dstq+strideq*0+16*2], m0 + call .main + mova [dstq+strideq*1+16*3], m2 + mova [dstq+strideq*0+16*3], m0 + call .main + mova [maskq+16*2], m2 + mova [dstq+strideq*0+16*4], m0 + call .main + mova [dstq+strideq*1+16*5], m2 + mova [dstq+strideq*0+16*5], m0 + call .main + mova [maskq+16*3], m2 + mova [dstq+strideq*0+16*6], m0 + call .main + mova [dstq+strideq*1+16*7], m2 + mova [dstq+strideq*0+16*7], m0 + W_MASK_420_END 0, 1, 2, 3 + sub hd, 2 + jg .w128_loop + RET +ALIGN function_align +.main: + mova m0, [tmp1q +16*0] + mova m3, [tmp1q+tmp2q+16*0] + mova m1, [tmp1q +16*1] + mova m4, [tmp1q+tmp2q+16*1] + add tmp1q, 16*2 + psubw m3, m0 + psubw m4, m1 + pabsw m5, m3 + psubusw m2, m8, m5 + psrlw m2, 8 ; 64 - m + psllw m5, m2, 10 + pmulhw m3, m5 + pabsw m5, m4 + paddw m0, m3 + psubusw m3, m8, m5 + psrlw m3, 8 + phaddw m2, m3 + psllw m3, 10 + pmulhw m4, m3 + paddw m1, m4 + pmulhrsw m0, m6 + pmulhrsw m1, m6 + packuswb m0, m1 + ret + +%macro W_MASK_422_BACKUP 1 ; mask_offset +%if ARCH_X86_64 + mova m10, m2 +%else + mova [maskq+16*%1], m2 +%endif +%endmacro + +%macro W_MASK_422_END 1 ; mask_offset +%if ARCH_X86_64 + packuswb m10, m2 + psubb m1, m7, m10 + pavgb m1, m9 +%else + mova m3, [maskq+16*%1] + packuswb m3, m2 + pxor m2, m2 + psubb m1, m7, m3 + pavgb m1, m2 +%endif + mova [maskq+16*%1], m1 +%endmacro + +cglobal w_mask_422_8bpc, 4, 7, 11, dst, stride, tmp1, tmp2, w, h, mask +%define base t0-w_mask_422_ssse3_table + LEA t0, w_mask_422_ssse3_table + tzcnt wd, wm + mov r6d, r7m ; sign + sub tmp2q, tmp1q + movsxd wq, [t0+wq*4] + mova m6, [base+pw_2048] + movddup m7, [base+wm_422_sign+r6*8] ; 128 - sign + add wq, t0 +%if ARCH_X86_64 + mova m8, [base+pw_6903] + pxor m9, m9 + movifnidn hd, hm +%else + add t0, w_mask_420_ssse3_table-w_mask_422_ssse3_table + %define hd dword hm +%endif + mov maskq, maskmp + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + jmp wq +.w4_loop: + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main add maskq, 8 - add dstq, 16 - sub loop_w, 16 - jg .w16ge_inner_loop + lea dstq, [dstq+strideq*2] +.w4: + packuswb m2, m2 + psubb m1, m7, m2 +%if ARCH_X86_64 + pavgb m1, m9 +%else + pxor m2, m2 + pavgb m1, m2 +%endif + movq [maskq], m1 + movd [dstq+strideq*0], m0 + pshuflw m1, m0, q1032 + movd [dstq+strideq*1], m1 + punpckhqdq m0, m0 + lea dstq, [dstq+strideq*2] + movd [dstq+strideq*0], m0 + pshuflw m1, m0, q1032 + movd [dstq+strideq*1], m1 + sub hd, 4 + jg .w4_loop + RET +.w8_loop: + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + add maskq, 16 + lea dstq, [dstq+strideq*2] +.w8: + W_MASK_422_BACKUP 0 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + lea dstq, [dstq+strideq*2] + W_MASK_422_END 0 + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + sub hd, 4 + jg .w8_loop + RET +.w16_loop: + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + add maskq, 16 + lea dstq, [dstq+strideq*2] +.w16: + W_MASK_422_BACKUP 0 + mova [dstq+strideq*0], m0 + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + W_MASK_422_END 0 + mova [dstq+strideq*1], m0 sub hd, 2 - jg .w16ge_loop + jg .w16_loop + RET +.w32_loop: + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + add maskq, 16 + add dstq, strideq +.w32: + W_MASK_422_BACKUP 0 + mova [dstq+16*0], m0 + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + W_MASK_422_END 0 + mova [dstq+16*1], m0 + dec hd + jg .w32_loop + RET +.w64_loop: + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + add maskq, 16*2 + add dstq, strideq +.w64: + W_MASK_422_BACKUP 0 + mova [dstq+16*0], m0 + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + W_MASK_422_END 0 + mova [dstq+16*1], m0 + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + W_MASK_422_BACKUP 1 + mova [dstq+16*2], m0 + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + W_MASK_422_END 1 + mova [dstq+16*3], m0 + dec hd + jg .w64_loop + RET +.w128_loop: + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + add maskq, 16*4 + add dstq, strideq +.w128: + W_MASK_422_BACKUP 0 + mova [dstq+16*0], m0 + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + W_MASK_422_END 0 + mova [dstq+16*1], m0 + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + W_MASK_422_BACKUP 1 + mova [dstq+16*2], m0 + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + W_MASK_422_END 1 + mova [dstq+16*3], m0 + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + W_MASK_422_BACKUP 2 + mova [dstq+16*4], m0 + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + W_MASK_422_END 2 + mova [dstq+16*5], m0 + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + W_MASK_422_BACKUP 3 + mova [dstq+16*6], m0 + call mangle(private_prefix %+ _w_mask_420_8bpc_ssse3).main + W_MASK_422_END 3 + mova [dstq+16*7], m0 + dec hd + jg .w128_loop RET -%undef reg_pw_6903 -%undef reg_pw_2048 -%undef dst_bak -%undef loop_w -%undef orig_w -%undef hd +cglobal w_mask_444_8bpc, 4, 7, 9, dst, stride, tmp1, tmp2, w, h, mask +%define base t0-w_mask_444_ssse3_table + LEA t0, w_mask_444_ssse3_table + tzcnt wd, wm + mov maskq, maskmp + sub tmp2q, tmp1q + movsxd wq, [t0+wq*4] + mova m6, [base+pw_6903] + mova m7, [base+pw_2048] + add wq, t0 +%if ARCH_X86_64 + mova m8, [base+pb_64] + movifnidn hd, hm +%else + %define m8 [base+pb_64] + %define hd dword hm +%endif + call .main + jmp wq +.w4_loop: + call .main + lea dstq, [dstq+strideq*2] +.w4: + movd [dstq+strideq*0], m0 + pshuflw m1, m0, q1032 + movd [dstq+strideq*1], m1 + punpckhqdq m0, m0 + lea dstq, [dstq+strideq*2] + movd [dstq+strideq*0], m0 + pshuflw m1, m0, q1032 + movd [dstq+strideq*1], m1 + sub hd, 4 + jg .w4_loop + RET +.w8_loop: + call .main + lea dstq, [dstq+strideq*2] +.w8: + movq [dstq+strideq*0], m0 + movhps [dstq+strideq*1], m0 + sub hd, 2 + jg .w8_loop + RET +.w16_loop: + call .main + lea dstq, [dstq+strideq*2] +.w16: + mova [dstq+strideq*0], m0 + call .main + mova [dstq+strideq*1], m0 + sub hd, 2 + jg .w16_loop + RET +.w32_loop: + call .main + add dstq, strideq +.w32: + mova [dstq+16*0], m0 + call .main + mova [dstq+16*1], m0 + dec hd + jg .w32_loop + RET +.w64_loop: + call .main + add dstq, strideq +.w64: + mova [dstq+16*0], m0 + call .main + mova [dstq+16*1], m0 + call .main + mova [dstq+16*2], m0 + call .main + mova [dstq+16*3], m0 + dec hd + jg .w64_loop + RET +.w128_loop: + call .main + add dstq, strideq +.w128: + mova [dstq+16*0], m0 + call .main + mova [dstq+16*1], m0 + call .main + mova [dstq+16*2], m0 + call .main + mova [dstq+16*3], m0 + call .main + mova [dstq+16*4], m0 + call .main + mova [dstq+16*5], m0 + call .main + mova [dstq+16*6], m0 + call .main + mova [dstq+16*7], m0 + dec hd + jg .w128_loop + RET +ALIGN function_align +.main: + mova m0, [tmp1q +16*0] + mova m3, [tmp1q+tmp2q+16*0] + mova m1, [tmp1q +16*1] + mova m4, [tmp1q+tmp2q+16*1] + add tmp1q, 16*2 + psubw m3, m0 + psubw m4, m1 + pabsw m5, m3 + psubusw m2, m6, m5 + psrlw m2, 8 ; 64 - m + psllw m5, m2, 10 + pmulhw m3, m5 + pabsw m5, m4 + paddw m0, m3 + psubusw m3, m6, m5 + psrlw m3, 8 + packuswb m2, m3 + psllw m3, 10 + pmulhw m4, m3 + psubb m3, m8, m2 + paddw m1, m4 + pmulhrsw m0, m7 + pmulhrsw m1, m7 + mova [maskq], m3 + add maskq, 16 + packuswb m0, m1 + ret %macro BLEND_64M 4; a, b, mask1, mask2 punpcklbw m0, %1, %2; {b;a}[7..0] punpckhbw %1, %2 ; {b;a}[15..8] pmaddubsw m0, %3 ; {b*m[0] + (64-m[0])*a}[7..0] u16 @@ -6754,11 +7082,11 @@ punpcklbw m2, m3, m0 ; {m;(64-m)}[7..0] punpckhbw m3, m0 ; {m;(64-m)}[15..8] BLEND_64M %1, %2, m2, m3 %endmacro -cglobal blend, 3, 7, 7, dst, ds, tmp, w, h, mask +cglobal blend_8bpc, 3, 7, 7, dst, ds, tmp, w, h, mask %define base r6-blend_ssse3_table LEA r6, blend_ssse3_table tzcnt wd, wm movifnidn hd, hm movifnidn maskq, maskmp @@ -6830,11 +7158,11 @@ add dstq, dsq ; dst_stride dec hd jg .w32 RET -cglobal blend_v, 3, 6, 6, dst, ds, tmp, w, h, mask +cglobal blend_v_8bpc, 3, 6, 6, dst, ds, tmp, w, h, mask %define base r5-blend_v_ssse3_table LEA r5, blend_v_ssse3_table tzcnt wd, wm movifnidn hd, hm movsxd wq, dword [r5+wq*4] @@ -6938,11 +7266,11 @@ %if WIN64 mova xmm6, [rsp+8] %endif RET -cglobal blend_h, 3, 7, 6, dst, ds, tmp, w, h, mask +cglobal blend_h_8bpc, 3, 7, 6, dst, ds, tmp, w, h, mask %define base t0-blend_h_ssse3_table %if ARCH_X86_32 ; We need to keep the PIC pointer for w4, reload wd from stack instead DECLARE_REG_TMP 6 %else @@ -7052,13 +7380,13 @@ ; const pixel *ref, const ptrdiff_t ref_stride ; ; bw, bh total filled size ; iw, ih, copied block -> fill bottom, right ; x, y, offset in bw/bh -> fill top, left -cglobal emu_edge, 10, 13, 2, bw, bh, iw, ih, x, \ - y, dst, dstride, src, sstride, \ - bottomext, rightext, blk +cglobal emu_edge_8bpc, 10, 13, 2, bw, bh, iw, ih, x, \ + y, dst, dstride, src, sstride, \ + bottomext, rightext, blk ; we assume that the buffer (stride) is larger than width, so we can ; safely overwrite by a few bytes pxor m1, m1 %if ARCH_X86_64 @@ -7415,18 +7743,18 @@ SWAP %1, %2 %endif %endmacro %if ARCH_X86_64 -cglobal resize, 0, 14, 16, dst, dst_stride, src, src_stride, \ - dst_w, h, src_w, dx, mx0 +cglobal resize_8bpc, 0, 12, 14, dst, dst_stride, src, src_stride, \ + dst_w, h, src_w, dx, mx0 %elif STACK_ALIGNMENT >= 16 -cglobal resize, 0, 7, 8, 3 * 16, dst, dst_stride, src, src_stride, \ - dst_w, h, src_w, dx, mx0 +cglobal resize_8bpc, 0, 7, 8, 3 * 16, dst, dst_stride, src, src_stride, \ + dst_w, h, src_w, dx, mx0 %else -cglobal resize, 0, 6, 8, 3 * 16, dst, dst_stride, src, src_stride, \ - dst_w, h, src_w, dx, mx0 +cglobal resize_8bpc, 0, 6, 8, 3 * 16, dst, dst_stride, src, src_stride, \ + dst_w, h, src_w, dx, mx0 %endif movifnidn dstq, dstmp movifnidn srcq, srcmp %if STACK_ALIGNMENT >= 16 movifnidn dst_wd, dst_wm @@ -7447,82 +7775,83 @@ DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, h, x, picptr LEA r7, $$ %define base r7-$$ %else DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, x +%define hd dword r5m %if STACK_ALIGNMENT >= 16 LEA r6, $$ %define base r6-$$ %else LEA r4, $$ %define base r4-$$ %endif %endif %if ARCH_X86_64 - mova m12, [base+pw_m256] - mova m11, [base+pd_63] - mova m10, [base+pb_8x0_8x8] + mova m10, [base+pw_m256] + mova m9, [base+pd_63] + mova m8, [base+pb_8x0_8x8] %else -%define m12 [base+pw_m256] -%define m11 [base+pd_63] -%define m10 [base+pb_8x0_8x8] +%define m10 [base+pw_m256] +%define m9 [base+pd_63] +%define m8 [base+pb_8x0_8x8] %endif pmaddwd m4, m7, [base+rescale_mul] ; dx*[0,1,2,3] pslld m7, 2 ; dx*4 pslld m5, 14 paddd m6, m4 ; mx+[0..3]*dx - SCRATCH 7, 15, 0 - SCRATCH 6, 14, 1 - SCRATCH 5, 13, 2 - - ; m2 = 0, m3 = pmulhrsw constant for x=(x+64)>>7 - ; m8 = mx+[0..3]*dx, m5 = dx*4, m6 = src_w, m7 = 0x3f, m15=0,8 + SCRATCH 7, 13, 0 + SCRATCH 6, 12, 1 + SCRATCH 5, 11, 2 + + ; m10 = pmulhrsw constant for x=(x+64)>>7 + ; m12 = mx+[0..3]*dx, m13 = dx*4, m11 = src_w, m9 = 0x3f, m8=0,8 .loop_y: xor xd, xd - mova m0, m14 ; per-line working version of mx + mova m0, m12 ; per-line working version of mx .loop_x: pxor m1, m1 pcmpgtd m1, m0 pandn m1, m0 psrad m2, m0, 8 ; filter offset (unmasked) - pcmpgtd m3, m13, m1 + pcmpgtd m3, m11, m1 pand m1, m3 - pandn m3, m13 + pandn m3, m11 por m1, m3 psubd m3, m0, m1 ; pshufb offset psrad m1, 14 ; clipped src_x offset psrad m3, 14 ; pshufb edge_emu offset - pand m2, m11 ; filter offset (masked) + pand m2, m9 ; filter offset (masked) ; load source pixels %if ARCH_X86_64 - movd r8d, xm1 - pshuflw xm1, xm1, q3232 - movd r9d, xm1 - punpckhqdq xm1, xm1 - movd r10d, xm1 - psrlq xm1, 32 - movd r11d, xm1 - movq xm4, [srcq+r8] - movq xm5, [srcq+r10] - movhps xm4, [srcq+r9] - movhps xm5, [srcq+r11] + movd r8d, m1 + pshuflw m1, m1, q3232 + movd r9d, m1 + punpckhqdq m1, m1 + movd r10d, m1 + psrlq m1, 32 + movd r11d, m1 + movq m4, [srcq+r8] + movq m5, [srcq+r10] + movhps m4, [srcq+r9] + movhps m5, [srcq+r11] %else - movd r3d, xm1 - pshufd xm1, xm1, q3312 - movd r1d, xm1 - pshuflw xm1, xm1, q3232 - movq xm4, [srcq+r3] - movq xm5, [srcq+r1] - movd r3d, xm1 - punpckhqdq xm1, xm1 - movd r1d, xm1 - movhps xm4, [srcq+r3] - movhps xm5, [srcq+r1] + movd r3d, m1 + pshufd m1, m1, q3312 + movd r1d, m1 + pshuflw m1, m1, q3232 + movq m4, [srcq+r3] + movq m5, [srcq+r1] + movd r3d, m1 + punpckhqdq m1, m1 + movd r1d, m1 + movhps m4, [srcq+r3] + movhps m5, [srcq+r1] %endif ; if no emulation is required, we don't need to shuffle or emulate edges ; this also saves 2 quasi-vpgatherdqs pxor m6, m6 @@ -7535,97 +7864,91 @@ cmp r3d, 0xffff %endif je .filter %if ARCH_X86_64 - movd r8d, xm3 - pshuflw xm3, xm3, q3232 - movd r9d, xm3 - punpckhqdq xm3, xm3 - movd r10d, xm3 - psrlq xm3, 32 - movd r11d, xm3 + movd r8d, m3 + pshuflw m3, m3, q3232 + movd r9d, m3 + punpckhqdq m3, m3 + movd r10d, m3 + psrlq m3, 32 + movd r11d, m3 movsxd r8, r8d movsxd r9, r9d movsxd r10, r10d movsxd r11, r11d - movq xm6, [base+resize_shuf+4+r8] - movq xm7, [base+resize_shuf+4+r10] - movhps xm6, [base+resize_shuf+4+r9] - movhps xm7, [base+resize_shuf+4+r11] + movq m6, [base+resize_shuf+4+r8] + movq m7, [base+resize_shuf+4+r10] + movhps m6, [base+resize_shuf+4+r9] + movhps m7, [base+resize_shuf+4+r11] %else - movd r3d, xm3 - pshufd xm3, xm3, q3312 - movd r1d, xm3 - pshuflw xm3, xm3, q3232 - movq xm6, [base+resize_shuf+4+r3] - movq xm7, [base+resize_shuf+4+r1] - movd r3d, xm3 - punpckhqdq xm3, xm3 - movd r1d, xm3 - movhps xm6, [base+resize_shuf+4+r3] - movhps xm7, [base+resize_shuf+4+r1] -%endif - - paddb m6, m10 - paddb m7, m10 + movd r3d, m3 + pshufd m3, m3, q3312 + movd r1d, m3 + pshuflw m3, m3, q3232 + movq m6, [base+resize_shuf+4+r3] + movq m7, [base+resize_shuf+4+r1] + movd r3d, m3 + punpckhqdq m3, m3 + movd r1d, m3 + movhps m6, [base+resize_shuf+4+r3] + movhps m7, [base+resize_shuf+4+r1] +%endif + + paddb m6, m8 + paddb m7, m8 pshufb m4, m6 pshufb m5, m7 .filter: %if ARCH_X86_64 - movd r8d, xm2 - pshuflw xm2, xm2, q3232 - movd r9d, xm2 - punpckhqdq xm2, xm2 - movd r10d, xm2 - psrlq xm2, 32 - movd r11d, xm2 - movq xm6, [base+resize_filter+r8*8] - movq xm7, [base+resize_filter+r10*8] - movhps xm6, [base+resize_filter+r9*8] - movhps xm7, [base+resize_filter+r11*8] + movd r8d, m2 + pshuflw m2, m2, q3232 + movd r9d, m2 + punpckhqdq m2, m2 + movd r10d, m2 + psrlq m2, 32 + movd r11d, m2 + movq m6, [base+resize_filter+r8*8] + movq m7, [base+resize_filter+r10*8] + movhps m6, [base+resize_filter+r9*8] + movhps m7, [base+resize_filter+r11*8] %else - movd r3d, xm2 - pshufd xm2, xm2, q3312 - movd r1d, xm2 - pshuflw xm2, xm2, q3232 - movq xm6, [base+resize_filter+r3*8] - movq xm7, [base+resize_filter+r1*8] - movd r3d, xm2 - punpckhqdq xm2, xm2 - movd r1d, xm2 - movhps xm6, [base+resize_filter+r3*8] - movhps xm7, [base+resize_filter+r1*8] + movd r3d, m2 + pshufd m2, m2, q3312 + movd r1d, m2 + pshuflw m2, m2, q3232 + movq m6, [base+resize_filter+r3*8] + movq m7, [base+resize_filter+r1*8] + movd r3d, m2 + punpckhqdq m2, m2 + movd r1d, m2 + movhps m6, [base+resize_filter+r3*8] + movhps m7, [base+resize_filter+r1*8] %endif pmaddubsw m4, m6 pmaddubsw m5, m7 phaddw m4, m5 phaddsw m4, m4 - pmulhrsw m4, m12 ; x=(x+64)>>7 + pmulhrsw m4, m10 ; x=(x+64)>>7 packuswb m4, m4 movd [dstq+xq], m4 - paddd m0, m15 + paddd m0, m13 add xd, 4 %if STACK_ALIGNMENT >= 16 cmp xd, dst_wd %else cmp xd, dst_wm %endif jl .loop_x -%if ARCH_X86_64 - add dstq, dst_strideq - add srcq, src_strideq + add dstq, dst_stridemp + add srcq, src_stridemp dec hd -%else - add dstq, dst_stridem - add srcq, src_stridem - dec dword r5m -%endif jg .loop_y RET INIT_XMM ssse3 PREP_BILIN diff --git a/third_party/dav1d/src/x86/refmvs.asm b/third_party/dav1d/src/x86/refmvs.asm new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/x86/refmvs.asm @@ -0,0 +1,169 @@ +; Copyright © 2021, VideoLAN and dav1d authors +; Copyright © 2021, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "config.asm" +%include "ext/x86/x86inc.asm" + +SECTION_RODATA 32 + +%macro JMP_TABLE 2-* + %xdefine %%prefix mangle(private_prefix %+ _%1) + %1_table: + %xdefine %%base %1_table + %rep %0 - 1 + dd %%prefix %+ .w%2 - %%base + %rotate 1 + %endrep +%endmacro + +%if ARCH_X86_64 +splat_mv_shuf: db 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3 + db 4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3, 4, 5, 6, 7 + +JMP_TABLE splat_mv_avx2, 1, 2, 4, 8, 16, 32 +%endif +JMP_TABLE splat_mv_sse2, 1, 2, 4, 8, 16, 32 + +SECTION .text + +INIT_XMM sse2 +; refmvs_block **rr, refmvs_block *a, int bx4, int bw4, int bh4 +cglobal splat_mv, 4, 5, 3, rr, a, bx4, bw4, bh4 + add bx4d, bw4d + tzcnt bw4d, bw4d + mova m2, [aq] + LEA aq, splat_mv_sse2_table + lea bx4q, [bx4q*3-32] + movsxd bw4q, [aq+bw4q*4] + movifnidn bh4d, bh4m + pshufd m0, m2, q0210 + pshufd m1, m2, q1021 + pshufd m2, m2, q2102 + add bw4q, aq +.loop: + mov aq, [rrq] + add rrq, gprsize + lea aq, [aq+bx4q*4] + jmp bw4q +.w32: + mova [aq-16*16], m0 + mova [aq-16*15], m1 + mova [aq-16*14], m2 + mova [aq-16*13], m0 + mova [aq-16*12], m1 + mova [aq-16*11], m2 + mova [aq-16*10], m0 + mova [aq-16* 9], m1 + mova [aq-16* 8], m2 + mova [aq-16* 7], m0 + mova [aq-16* 6], m1 + mova [aq-16* 5], m2 +.w16: + mova [aq-16* 4], m0 + mova [aq-16* 3], m1 + mova [aq-16* 2], m2 + mova [aq-16* 1], m0 + mova [aq+16* 0], m1 + mova [aq+16* 1], m2 +.w8: + mova [aq+16* 2], m0 + mova [aq+16* 3], m1 + mova [aq+16* 4], m2 +.w4: + mova [aq+16* 5], m0 + mova [aq+16* 6], m1 + mova [aq+16* 7], m2 + dec bh4d + jg .loop + RET +.w2: + movu [aq+104], m0 + movq [aq+120], m1 + dec bh4d + jg .loop + RET +.w1: + movq [aq+116], m0 + movd [aq+124], m2 + dec bh4d + jg .loop + RET + +%if ARCH_X86_64 +INIT_YMM avx2 +cglobal splat_mv, 4, 5, 3, rr, a, bx4, bw4, bh4 + add bx4d, bw4d + tzcnt bw4d, bw4d + vbroadcasti128 m0, [aq] + lea aq, [splat_mv_avx2_table] + lea bx4q, [bx4q*3-32] + movsxd bw4q, [aq+bw4q*4] + pshufb m0, [splat_mv_shuf] + movifnidn bh4d, bh4m + pshufd m1, m0, q2102 + pshufd m2, m0, q1021 + add bw4q, aq +.loop: + mov aq, [rrq] + add rrq, gprsize + lea aq, [aq+bx4q*4] + jmp bw4q +.w32: + mova [aq-32*8], m0 + mova [aq-32*7], m1 + mova [aq-32*6], m2 + mova [aq-32*5], m0 + mova [aq-32*4], m1 + mova [aq-32*3], m2 +.w16: + mova [aq-32*2], m0 + mova [aq-32*1], m1 + mova [aq+32*0], m2 +.w8: + mova [aq+32*1], m0 + mova [aq+32*2], m1 + mova [aq+32*3], m2 + dec bh4d + jg .loop + RET +.w4: + movu [aq+ 80], m0 + mova [aq+112], xm1 + dec bh4d + jg .loop + RET +.w2: + movu [aq+104], xm0 + movq [aq+120], xm2 + dec bh4d + jg .loop + RET +.w1: + movq [aq+116], xm0 + movd [aq+124], xm1 + dec bh4d + jg .loop + RET +%endif diff --git a/third_party/dav1d/src/x86/refmvs_init.c b/third_party/dav1d/src/x86/refmvs_init.c new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/x86/refmvs_init.c @@ -0,0 +1,46 @@ +/* + * Copyright © 2021, VideoLAN and dav1d authors + * Copyright © 2021, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/cpu.h" +#include "src/refmvs.h" + +decl_splat_mv_fn(dav1d_splat_mv_sse2); +decl_splat_mv_fn(dav1d_splat_mv_avx2); + +COLD void dav1d_refmvs_dsp_init_x86(Dav1dRefmvsDSPContext *const c) { + const unsigned flags = dav1d_get_cpu_flags(); + + if (!(flags & DAV1D_X86_CPU_FLAG_SSE2)) return; + + c->splat_mv = dav1d_splat_mv_sse2; + +#if ARCH_X86_64 + if (!(flags & DAV1D_X86_CPU_FLAG_AVX2)) return; + + c->splat_mv = dav1d_splat_mv_avx2; +#endif +}