diff --git a/.arcconfig b/.arcconfig --- a/.arcconfig +++ b/.arcconfig @@ -1,5 +1,5 @@ { - "phabricator.uri" : "https://phabricator.services.mozilla.com/", + "phabricator.uri" : "https://phabricator-dev.allizom.org/", "repository.callsign": "MOZILLACENTRAL", "history.immutable": false } diff --git a/media/libdav1d/moz.yaml b/media/libdav1d/moz.yaml --- a/media/libdav1d/moz.yaml +++ b/media/libdav1d/moz.yaml @@ -18,15 +18,15 @@ # Usually different from repository url url: https://code.videolan.org/videolan/dav1d # Human-readable identifier for this version/release # Generally "version NNN", "tag SSS", "bookmark SSS" - release: commit efd9e5518e0ed5114f8b4579debd7ee6dbede21f (2020-03-06T00:16:53.000+01:00). + release: commit 8dd9c651857dc1dcb9dd2b0ffca4177b47779905 (2020-07-02T22:49:09.000+03:00). # Revision to pull in # Must be a long or short commit SHA (long preferred) - revision: efd9e5518e0ed5114f8b4579debd7ee6dbede21f + revision: 8dd9c651857dc1dcb9dd2b0ffca4177b47779905 # The package's license, where possible using the mnemonic from # https://spdx.org/licenses/ # Multiple licenses can be specified (as a YAML list) # A "LICENSE" file must exist containing the full license text diff --git a/media/libdav1d/vcs_version.h b/media/libdav1d/vcs_version.h --- a/media/libdav1d/vcs_version.h +++ b/media/libdav1d/vcs_version.h @@ -1,2 +1,2 @@ /* auto-generated, do not edit */ -#define DAV1D_VERSION "0.6.0-0-gefd9e55" +#define DAV1D_VERSION "8dd9c651857dc1dcb9dd2b0ffca4177b47779905" diff --git a/third_party/dav1d/.gitlab-ci.yml b/third_party/dav1d/.gitlab-ci.yml deleted file mode 100644 --- a/third_party/dav1d/.gitlab-ci.yml +++ /dev/null @@ -1,472 +0,0 @@ -stages: - - style - - build - - test - -style-check: - image: registry.videolan.org:5000/dav1d-debian-unstable:20190215130514 - stage: style - tags: - - debian - - amd64 - script: - - git grep -I -n -P "\t|\r| $" -- . ':(exclude)*/compat/*' && exit 1 - - git grep -I -n -i "david" -- . ':(exclude)THANKS.md' ':(exclude).gitlab-ci.yml' && exit 1 - - git grep -I -l -z "" -- . ':(exclude)*/compat/*' | while IFS= read -r -d '' i; do - if [ -n "$(tail -c 1 "$i")" ]; then - echo "No newline at end of $i"; - exit 1; - fi; - done - - git remote rm upstream 2> /dev/null || true - - git remote add upstream https://code.videolan.org/videolan/dav1d.git - - git fetch -q upstream master - - for i in $(git rev-list HEAD ^upstream/master); do - echo "Checking commit message of $i"; - msg="$(git log --format=%B -n 1 $i)"; - if [ -n "$(echo "$msg" | awk "NR==2")" ]; then - echo "Malformed commit message in $i, second line must be empty"; - exit 1; - fi; - if echo "$msg" | head -1 | grep -q '\.$'; then - echo "Malformed commit message in $i, trailing period in subject line"; - exit 1; - fi; - done - -build-debian: - image: registry.videolan.org:5000/dav1d-debian-unstable:20190215130514 - stage: build - tags: - - avx2 - - amd64 - script: - - meson build --buildtype release --werror - - ninja -C build - - cd build && meson test -v - -build-debian-static: - image: registry.videolan.org:5000/dav1d-debian-unstable:20190215130514 - stage: build - tags: - - debian - - amd64 - script: - - meson build --buildtype release --default-library static --werror - - ninja -C build - - cd build && meson test -v - - nm -A -g src/libdav1d.a | grep " [ABCDGRST] " | (! grep -v " _*dav1d_") - -build-debian32: - image: registry.videolan.org:5000/dav1d-debian-unstable:20181218135732 - stage: build - tags: - - debian - - amd64 - script: - - meson build --buildtype release - --werror - --cross-file /opt/crossfiles/linux32.meson - - ninja -C build - - cd build && meson test -v - -build-win32: - image: registry.videolan.org:5000/dav1d-debian-unstable:20190215130514 - stage: build - tags: - - debian - - amd64 - script: - - wineserver -p && wine wineboot - - meson build --buildtype release - --werror - --libdir lib - --prefix "$(pwd)/build/dav1d_install" - --cross-file /opt/crossfiles/i686-w64-mingw32.meson - -Ddefault_library=both - - ninja -C build - - ninja -C build install - - cd build && meson test -v - - i686-w64-mingw32-nm -A -g src/libdav1d.a | grep " [ABCDGRST] " | (! grep -E -v " \.| _*dav1d_") - artifacts: - name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" - paths: - - build/dav1d_install/ - expire_in: 1 week - -build-win32-unaligned-stack: - image: registry.videolan.org:5000/vlc-debian-llvm-mingw:20190218133533 - stage: build - tags: - - debian - - amd64 - script: - - wineserver -p && wine wineboot - - meson build --buildtype release - --werror - --cross-file /opt/crossfiles/i686-w64-mingw32.meson - -Dstack_alignment=4 - - ninja -C build - - cd build && meson test -v - -build-win64: - image: registry.videolan.org:5000/dav1d-debian-unstable:20190215130514 - stage: build - tags: - - debian - - amd64 - script: - - wineserver -p && wine wineboot - - meson build --buildtype release - --werror - --libdir lib - --prefix "$(pwd)/build/dav1d_install" - --cross-file /opt/crossfiles/x86_64-w64-mingw32.meson - -Ddefault_library=both - - ninja -C build - - ninja -C build install - - cd build && meson test -v - - x86_64-w64-mingw32-nm -A -g src/libdav1d.a | grep " [ABCDGRST] " | (! grep -E -v " \.| _*dav1d_") - artifacts: - name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" - paths: - - build/dav1d_install/ - expire_in: 1 week - -build-win-arm32: - image: registry.videolan.org:5000/vlc-debian-llvm-mingw:20190218133533 - stage: build - tags: - - debian - - amd64 - script: - - meson build --buildtype release - --werror - --libdir lib - --prefix "$(pwd)/build/dav1d_install" - --cross-file /opt/crossfiles/armv7-w64-mingw32.meson - -Ddefault_library=both - - ninja -C build - - armv7-w64-mingw32-nm -A -g build/src/libdav1d.a | grep " [ABCDGRST] " | (! grep -E -v " \.| _*dav1d_") - -build-win-arm64: - image: registry.videolan.org:5000/vlc-debian-llvm-mingw:20190218133533 - stage: build - tags: - - debian - - amd64 - script: - - meson build --buildtype release - --werror - --libdir lib - --prefix "$(pwd)/build/dav1d_install" - --cross-file /opt/crossfiles/aarch64-w64-mingw32.meson - -Ddefault_library=both - - ninja -C build - - ninja -C build install - - aarch64-w64-mingw32-nm -A -g build/src/libdav1d.a | grep " [ABCDGRST] " | (! grep -E -v " \.| _*dav1d_") - artifacts: - name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" - paths: - - build/dav1d_install/ - expire_in: 1 week - -build-debian-aarch64: - stage: build - image: registry.videolan.org/dav1d-debian-buster-aarch64:20200218203017 - tags: - - aarch64 - - debian - script: - - meson build --buildtype debugoptimized --werror - - ninja -C build - - cd build && meson test -v - -build-debian-aarch64-clang-5: - stage: build - image: registry.videolan.org/dav1d-debian-buster-aarch64:20200218203017 - tags: - - aarch64 - - debian - script: - - env CC=clang-5.0 CFLAGS='-integrated-as' meson build --buildtype release - - ninja -C build - - cd build && meson test -v - -build-macos: - stage: build - tags: - - macos - script: - - meson build --buildtype release -Ddefault_library=both --werror - - ninja -C build - - cd build && meson test -v - -build-debian-werror: - image: registry.videolan.org/dav1d-debian-buster-aarch64:20200218203017 - stage: build - tags: - - aarch64 - - debian - script: - - env CC='clang-7' meson build --buildtype debug --werror - - ninja -C build - -build-debian-armv7: - stage: build - image: registry.videolan.org:5000/dav1d-debian-unstable-armv7:20190202101732 - tags: - - armv7 - - debian - script: - - linux32 meson build --buildtype debugoptimized --werror - - ninja -C build - - cd build && meson test -v - -build-debian-armv7-clang-5: - stage: build - image: registry.videolan.org:5000/dav1d-debian-unstable-armv7:20190202101732 - tags: - - armv7 - - debian - script: - - env CC=clang-5.0 CFLAGS='-integrated-as' linux32 meson build --buildtype release - - ninja -C build - - cd build && meson test -v - -build-ubuntu-snap: - stage: build - image: registry.videolan.org/dav1d-ubuntu-bionic:20200121182340 - tags: - - debian - - amd64 - script: - - cd package/snap && snapcraft snap - - | - if [ "$CI_PROJECT_NAMESPACE" = "videolan" ]; then - echo $SNAP_LOGIN | base64 --decode | snapcraft login --with - - snapcraft push dav1d_*.snap --release edge - snapcraft logout - fi - artifacts: - name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" - paths: - - package/snap/dav1d_*.snap - expire_in: 1 week - allow_failure: true - -build-debian-ppc64le: - stage: build - image: registry.videolan.org/dav1d-debian-unstable-ppc64le:20190606105121 - tags: - - ppc64le - - docker - script: - - meson build --buildtype release --werror - - ninja -C build - - cd build && meson test -v - -test-debian: - image: registry.videolan.org:5000/dav1d-debian-unstable:20190215130514 - stage: test - needs: ["build-debian"] - tags: - - debian - - amd64 - cache: - key: testdata.git-20190215 - paths: - - cache/dav1d-test-data.git/ - script: - - test -d cache || mkdir cache - - test -d cache/dav1d-test-data.git && GIT_DIR=cache/dav1d-test-data.git git fetch --refmap=refs/heads/master:refs/heads/master origin master - - test -d cache/dav1d-test-data.git || git clone --bare https://code.videolan.org/videolan/dav1d-test-data.git cache/dav1d-test-data.git - - git clone cache/dav1d-test-data.git tests/dav1d-test-data - - meson build --buildtype release -Dtestdata_tests=true -Dlogging=false - - ninja -C build - - cd build && time meson test -v - dependencies: [] - -test-debian-unaligned-stack: - image: registry.videolan.org:5000/dav1d-debian-unstable:20190215130514 - stage: test - needs: ["build-debian"] - tags: - - avx2 - - amd64 - cache: - key: testdata.git-20190215 - paths: - - cache/dav1d-test-data.git/ - script: - - test -d cache || mkdir cache - - test -d cache/dav1d-test-data.git && GIT_DIR=cache/dav1d-test-data.git git fetch --refmap=refs/heads/master:refs/heads/master origin master - - test -d cache/dav1d-test-data.git || git clone --bare https://code.videolan.org/videolan/dav1d-test-data.git cache/dav1d-test-data.git - - git clone cache/dav1d-test-data.git tests/dav1d-test-data - - meson build --buildtype release -Dtestdata_tests=true -Dlogging=false -Dstack_alignment=16 - - ninja -C build - - cd build && time meson test -v - dependencies: [] - -test-debian-asan: - image: registry.videolan.org:5000/dav1d-debian-unstable:20190215130514 - stage: test - needs: ["build-debian"] - tags: - - debian - - amd64 - cache: - key: testdata.git-20190215 - paths: - - cache/dav1d-test-data.git/ - variables: - ASAN_OPTIONS: 'detect_leaks=0' - script: - - test -d cache || mkdir cache - - test -d cache/dav1d-test-data.git && GIT_DIR=cache/dav1d-test-data.git git fetch --refmap=refs/heads/master:refs/heads/master origin master - - test -d cache/dav1d-test-data.git || git clone --bare https://code.videolan.org/videolan/dav1d-test-data.git cache/dav1d-test-data.git - - git clone cache/dav1d-test-data.git tests/dav1d-test-data - - meson build --buildtype debugoptimized -Dtestdata_tests=true -Dlogging=false -Db_sanitize=address -Denable_asm=false - - ninja -C build - - cd build && time meson test -v --setup=sanitizer - dependencies: [] - -test-debian-msan: - image: registry.videolan.org:5000/dav1d-debian-unstable:20190215130514 - stage: test - needs: ["build-debian"] - tags: - - debian - - amd64 - cache: - key: testdata.git-20190215 - paths: - - cache/dav1d-test-data.git/ - variables: - MSAN_OPTIONS: 'exitcode=1' - script: - - test -d cache || mkdir cache - - test -d cache/dav1d-test-data.git && GIT_DIR=cache/dav1d-test-data.git git fetch --refmap=refs/heads/master:refs/heads/master origin master - - test -d cache/dav1d-test-data.git || git clone --bare https://code.videolan.org/videolan/dav1d-test-data.git cache/dav1d-test-data.git - - git clone cache/dav1d-test-data.git tests/dav1d-test-data - - env CC=clang meson build --buildtype debugoptimized -Dtestdata_tests=true -Dlogging=false -Db_sanitize=memory -Db_lundef=false -Denable_asm=false - - ninja -C build - - cd build && time meson test -v --setup=sanitizer - dependencies: [] - -test-debian-ubsan: - image: registry.videolan.org:5000/dav1d-debian-unstable:20190215130514 - stage: test - needs: ["build-debian"] - tags: - - debian - - amd64 - cache: - key: testdata.git-20190215 - paths: - - cache/dav1d-test-data.git/ - variables: - UBSAN_OPTIONS: 'print_stacktrace=1:halt_on_error=1' - script: - - test -d cache || mkdir cache - - test -d cache/dav1d-test-data.git && GIT_DIR=cache/dav1d-test-data.git git fetch --refmap=refs/heads/master:refs/heads/master origin master - - test -d cache/dav1d-test-data.git || git clone --bare https://code.videolan.org/videolan/dav1d-test-data.git cache/dav1d-test-data.git - - git clone cache/dav1d-test-data.git tests/dav1d-test-data - - env CC=clang meson build --buildtype debugoptimized -Dtestdata_tests=true -Dlogging=false -Db_sanitize=undefined -Db_lundef=false -Denable_asm=false - - ninja -C build - - cd build && time meson test -v --setup=sanitizer - dependencies: [] - -test-win64: - image: registry.videolan.org:5000/dav1d-debian-unstable:20190215130514 - stage: test - needs: ["build-win64"] - tags: - - avx2 - - amd64 - cache: - key: testdata.git-20190215 - paths: - - cache/dav1d-test-data.git/ - script: - - test -d cache || mkdir cache - - test -d cache/dav1d-test-data.git && GIT_DIR=cache/dav1d-test-data.git git fetch --refmap=refs/heads/master:refs/heads/master origin master - - test -d cache/dav1d-test-data.git || git clone --bare https://code.videolan.org/videolan/dav1d-test-data.git cache/dav1d-test-data.git - - git clone cache/dav1d-test-data.git tests/dav1d-test-data - - wineserver -p && wine wineboot - - meson build --buildtype release - -Dtestdata_tests=true - -Dlogging=false - --cross-file /opt/crossfiles/x86_64-w64-mingw32.meson - - ninja -C build - - cd build && time meson test -v - dependencies: [] - -test-debian-aarch64: - image: registry.videolan.org/dav1d-debian-buster-aarch64:20200218203017 - stage: test - needs: ["build-debian-aarch64"] - tags: - - aarch64 - - debian - cache: - key: testdata.git-20190215 - paths: - - cache/dav1d-test-data.git/ - script: - - test -d cache || mkdir cache - - test -d cache/dav1d-test-data.git && GIT_DIR=cache/dav1d-test-data.git git fetch --refmap=refs/heads/master:refs/heads/master origin master - - test -d cache/dav1d-test-data.git || git clone --bare https://code.videolan.org/videolan/dav1d-test-data.git cache/dav1d-test-data.git - - git clone cache/dav1d-test-data.git tests/dav1d-test-data - - meson build --buildtype release - -Dtestdata_tests=true - -Dlogging=false - - ninja -C build - - cd build && time meson test -v - dependencies: [] - -test-debian-ppc64le: - image: registry.videolan.org/dav1d-debian-unstable-ppc64le:20190606105121 - stage: test - needs: ["build-debian-ppc64le"] - tags: - - ppc64le - - docker - cache: - key: testdata.git-20190215 - paths: - - cache/dav1d-test-data.git/ - script: - - test -d cache || mkdir cache - - test -d cache/dav1d-test-data.git && GIT_DIR=cache/dav1d-test-data.git git fetch --refmap=refs/heads/master:refs/heads/master origin master - - test -d cache/dav1d-test-data.git || git clone --bare https://code.videolan.org/videolan/dav1d-test-data.git cache/dav1d-test-data.git - - git clone cache/dav1d-test-data.git tests/dav1d-test-data - - meson build --buildtype release - -Dtestdata_tests=true - -Dlogging=false - - ninja -C build - - cd build && time meson test -v - dependencies: [] - -test-debian-armv7-clang-5: - stage: test - image: registry.videolan.org:5000/dav1d-debian-unstable-armv7:20190202101732 - needs: ["build-debian-armv7-clang-5"] - tags: - - armv7 - - debian - cache: - key: testdata.git-20190215 - paths: - - cache/dav1d-test-data.git/ - script: - - test -d cache || mkdir cache - - test -d cache/dav1d-test-data.git && GIT_DIR=cache/dav1d-test-data.git git fetch --refmap=refs/heads/master:refs/heads/master origin master - - test -d cache/dav1d-test-data.git || git clone --bare https://code.videolan.org/videolan/dav1d-test-data.git cache/dav1d-test-data.git - - git clone cache/dav1d-test-data.git tests/dav1d-test-data - - env CC=clang-5.0 CFLAGS='-integrated-as' linux32 meson build --buildtype release - -Dtestdata_tests=true - -Dlogging=false - - ninja -C build - - cd build && time meson test -v - dependencies: [] diff --git a/third_party/dav1d/NEWS b/third_party/dav1d/NEWS --- a/third_party/dav1d/NEWS +++ b/third_party/dav1d/NEWS @@ -1,5 +1,35 @@ +Changes for 0.7.1 'Frigatebird': +------------------------------ + +0.7.1 is a minor update on 0.7.0: + - ARM32 NEON optimizations for itxfm, which can give up to 28% speedup, and MSAC + - SSE2 optimizations for prep_bilin and prep_8tap + - AVX2 optimizations for MC scaled + - Fix a clamping issue in motion vector projection + - Fix an issue on some specific Haswell CPU on ipred_z AVX2 functions + - Improvements on the dav1dplay utility player to support resizing + + +Changes for 0.7.0 'Frigatebird': +------------------------------ + +0.7.0 is a major release for dav1d: + - Faster refmv implementation gaining up to 12% speed while -25% of RAM (Single Thread) + - 10b/12b ARM64 optimizations are mostly complete: + - ipred (paeth, smooth, dc, pal, filter, cfl) + - itxfm (only 10b) + - AVX2/SSSE3 for non-4:2:0 film grain and for mc.resize + - AVX2 for cfl4:4:4 + - AVX-512 CDEF filter + - ARM64 8b improvements for cfl_ac and itxfm + - ARM64 implementation for emu_edge in 8b/10b/12b + - ARM32 implementation for emu_edge in 8b + - Improvements on the dav1dplay utility player to support 10 bit, + non-4:2:0 pixel formats and film grain on the GPU + + Changes for 0.6.0 'Gyrfalcon': ------------------------------ 0.6.0 is a major release for dav1d: - New ARM64 optimizations for the 10/12bit depth: diff --git a/third_party/dav1d/README.md b/third_party/dav1d/README.md --- a/third_party/dav1d/README.md +++ b/third_party/dav1d/README.md @@ -22,11 +22,11 @@ The reasoning behind this decision is the same as for libvorbis, see [RMS on vorbis](https://lwn.net/2001/0301/a/rms-ov-license.php3). # Roadmap -The plan is the folllowing: +The plan is the following: ### Reached 1. Complete C implementation of the decoder, 2. Provide a usable API, 3. Port to most platforms, @@ -76,10 +76,32 @@ 1. Install [Meson](https://mesonbuild.com/) (0.47 or higher), [Ninja](https://ninja-build.org/), and, for x86\* targets, [nasm](https://nasm.us/) (2.14 or higher) 2. Run `mkdir build && cd build` to create a build directory and enter it 3. Run `meson ..` to configure meson, add `--default-library=static` if static linking is desired 4. Run `ninja` to compile +## Cross-Compilation for 32- or 64-bit Windows, 32-bit Linux + +If you're on a linux build machine trying to compile .exe for a Windows target/host machine, run + +``` +meson build --cross-file=package/crossfiles/x86_64-w64-mingw32.meson +``` + +or, for 32-bit: + +``` +meson build --cross-file=package/crossfiles/i686-w64-mingw32.meson +``` + +`mingw-w64` is a pre-requisite and should be installed on your linux machine via your preferred method or package manager. Note the binary name formats may differ between distributions. Verify the names, and use `alias` if certain binaries cannot be found. + +For 32-bit linux, run + +``` +meson build --cross-file=package/crossfiles/i686-linux32.meson +``` + # Run tests 1. In the root directory, run `git clone https://code.videolan.org/videolan/dav1d-test-data.git tests/dav1d-test-data` to fetch the test data repository 2. During meson configuration, specify `-Dtestdata_tests=true` 3. Run `meson test -v` after compiling diff --git a/third_party/dav1d/doc/meson.build b/third_party/dav1d/doc/meson.build --- a/third_party/dav1d/doc/meson.build +++ b/third_party/dav1d/doc/meson.build @@ -21,15 +21,16 @@ # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. doxygen = find_program('doxygen', required: false) +dot = find_program('dot', required: false) -if doxygen.found() +if doxygen.found() and dot.found() conf_data = configuration_data() - conf_data.set('DOXYGEN_INPUT', join_paths(meson.source_root(), 'include/dav1d')) - conf_data.set('DOXYGEN_STRIP', join_paths(meson.source_root(), 'include')) + conf_data.set('DOXYGEN_INPUT', join_paths(dav1d_src_root, 'include/dav1d')) + conf_data.set('DOXYGEN_STRIP', join_paths(dav1d_src_root, 'include')) conf_data.set('DOXYGEN_OUTPUT', meson.current_build_dir()) doxyfile = configure_file(input: 'Doxyfile.in', output: 'Doxyfile', configuration: conf_data) diff --git a/third_party/dav1d/examples/dav1dplay.c b/third_party/dav1d/examples/dav1dplay.c --- a/third_party/dav1d/examples/dav1dplay.c +++ b/third_party/dav1d/examples/dav1dplay.c @@ -27,691 +27,22 @@ #include "config.h" #include "vcs_version.h" #include #include -#include -#include -#include #include -#include "common/attributes.h" - #include "dav1d/dav1d.h" +#include "common/attributes.h" #include "tools/input/input.h" - -/** - * Settings structure - * Hold all settings available for the player, - * this is usually filled by parsing arguments - * from the console. - */ -typedef struct { - const char *inputfile; - int highquality; - int untimed; - int zerocopy; -} Dav1dPlaySettings; - -#define WINDOW_WIDTH 910 -#define WINDOW_HEIGHT 512 - -#define DAV1D_EVENT_NEW_FRAME 1 -#define DAV1D_EVENT_DEC_QUIT 2 - -/* - * Fifo helper functions - */ -typedef struct dp_fifo -{ - SDL_mutex *lock; - SDL_cond *cond_change; - size_t capacity; - size_t count; - void **entries; -} Dav1dPlayPtrFifo; - -static void dp_fifo_destroy(Dav1dPlayPtrFifo *fifo) -{ - assert(fifo->count == 0); - SDL_DestroyMutex(fifo->lock); - SDL_DestroyCond(fifo->cond_change); - free(fifo->entries); - free(fifo); -} - -static Dav1dPlayPtrFifo *dp_fifo_create(size_t capacity) -{ - Dav1dPlayPtrFifo *fifo; - - assert(capacity > 0); - if (capacity <= 0) - return NULL; - - fifo = malloc(sizeof(*fifo)); - if (fifo == NULL) - return NULL; - - fifo->capacity = capacity; - fifo->count = 0; - - fifo->lock = SDL_CreateMutex(); - if (fifo->lock == NULL) { - free(fifo); - return NULL; - } - fifo->cond_change = SDL_CreateCond(); - if (fifo->cond_change == NULL) { - SDL_DestroyMutex(fifo->lock); - free(fifo); - return NULL; - } - - fifo->entries = calloc(capacity, sizeof(void*)); - if (fifo->entries == NULL) { - dp_fifo_destroy(fifo); - return NULL; - } - - return fifo; -} - -static void dp_fifo_push(Dav1dPlayPtrFifo *fifo, void *element) -{ - SDL_LockMutex(fifo->lock); - while (fifo->count == fifo->capacity) - SDL_CondWait(fifo->cond_change, fifo->lock); - fifo->entries[fifo->count++] = element; - if (fifo->count == 1) - SDL_CondSignal(fifo->cond_change); - SDL_UnlockMutex(fifo->lock); -} - -static void *dp_fifo_array_shift(void **arr, size_t len) -{ - void *shifted_element = arr[0]; - for (size_t i = 1; i < len; ++i) - arr[i-1] = arr[i]; - return shifted_element; -} - -static void *dp_fifo_shift(Dav1dPlayPtrFifo *fifo) -{ - SDL_LockMutex(fifo->lock); - while (fifo->count == 0) - SDL_CondWait(fifo->cond_change, fifo->lock); - void *res = dp_fifo_array_shift(fifo->entries, fifo->count--); - if (fifo->count == fifo->capacity - 1) - SDL_CondSignal(fifo->cond_change); - SDL_UnlockMutex(fifo->lock); - return res; -} - -/** - * Renderer info - */ -typedef struct rdr_info -{ - // Cookie passed to the renderer implementation callbacks - void *cookie; - // Callback to create the renderer - void* (*create_renderer)(void *data); - // Callback to destroy the renderer - void (*destroy_renderer)(void *cookie); - // Callback to the render function that renders a prevously sent frame - void (*render)(void *cookie, const Dav1dPlaySettings *settings); - // Callback to the send frame function - int (*update_frame)(void *cookie, Dav1dPicture *dav1d_pic, - const Dav1dPlaySettings *settings); - // Callback for alloc/release pictures (optional) - int (*alloc_pic)(Dav1dPicture *pic, void *cookie); - void (*release_pic)(Dav1dPicture *pic, void *cookie); -} Dav1dPlayRenderInfo; - -#ifdef HAVE_PLACEBO_VULKAN - -#include -#include -#include -#include - - -/** - * Renderer context for libplacebo - */ -typedef struct renderer_priv_ctx -{ - // Placebo context - struct pl_context *ctx; - // Placebo renderer - struct pl_renderer *renderer; - // Placebo Vulkan handle - const struct pl_vulkan *vk; - // Placebo Vulkan instance - const struct pl_vk_inst *vk_inst; - // Vulkan surface - VkSurfaceKHR surf; - // Placebo swapchain - const struct pl_swapchain *swapchain; - // Lock protecting access to the texture - SDL_mutex *lock; - // Planes to render - struct pl_plane y_plane; - struct pl_plane u_plane; - struct pl_plane v_plane; - // Textures to render - const struct pl_tex *y_tex; - const struct pl_tex *u_tex; - const struct pl_tex *v_tex; -} Dav1dPlayRendererPrivateContext; - -static void *placebo_renderer_create(void *data) -{ - // Alloc - Dav1dPlayRendererPrivateContext *rd_priv_ctx = malloc(sizeof(Dav1dPlayRendererPrivateContext)); - if (rd_priv_ctx == NULL) { - return NULL; - } - - // Init libplacebo - rd_priv_ctx->ctx = pl_context_create(PL_API_VER, &(struct pl_context_params) { - .log_cb = pl_log_color, -#ifndef NDEBUG - .log_level = PL_LOG_DEBUG, -#else - .log_level = PL_LOG_WARN, -#endif - }); - if (rd_priv_ctx->ctx == NULL) { - free(rd_priv_ctx); - return NULL; - } - - // Create Mutex - rd_priv_ctx->lock = SDL_CreateMutex(); - if (rd_priv_ctx->lock == NULL) { - fprintf(stderr, "SDL_CreateMutex failed: %s\n", SDL_GetError()); - pl_context_destroy(&(rd_priv_ctx->ctx)); - free(rd_priv_ctx); - return NULL; - } - - // Init Vulkan - struct pl_vk_inst_params iparams = pl_vk_inst_default_params; - - SDL_Window *sdlwin = data; - - unsigned num = 0; - if (!SDL_Vulkan_GetInstanceExtensions(sdlwin, &num, NULL)) { - fprintf(stderr, "Failed enumerating Vulkan extensions: %s\n", SDL_GetError()); - exit(1); - } - - iparams.extensions = malloc(num * sizeof(const char *)); - iparams.num_extensions = num; - assert(iparams.extensions); - - SDL_bool ok = SDL_Vulkan_GetInstanceExtensions(sdlwin, &num, iparams.extensions); - if (!ok) { - fprintf(stderr, "Failed getting Vk instance extensions\n"); - exit(1); - } - - if (num > 0) { - printf("Requesting %d additional Vulkan extensions:\n", num); - for (unsigned i = 0; i < num; i++) - printf(" %s\n", iparams.extensions[i]); - } - - rd_priv_ctx->vk_inst = pl_vk_inst_create(rd_priv_ctx->ctx, &iparams); - if (!rd_priv_ctx->vk_inst) { - fprintf(stderr, "Failed creating Vulkan instance!\n"); - exit(1); - } - free(iparams.extensions); - - if (!SDL_Vulkan_CreateSurface(sdlwin, rd_priv_ctx->vk_inst->instance, &rd_priv_ctx->surf)) { - fprintf(stderr, "Failed creating vulkan surface: %s\n", SDL_GetError()); - exit(1); - } - - struct pl_vulkan_params params = pl_vulkan_default_params; - params.instance = rd_priv_ctx->vk_inst->instance; - params.surface = rd_priv_ctx->surf; - params.allow_software = true; - - rd_priv_ctx->vk = pl_vulkan_create(rd_priv_ctx->ctx, ¶ms); - if (!rd_priv_ctx->vk) { - fprintf(stderr, "Failed creating vulkan device!\n"); - exit(2); - } - - // Create swapchain - rd_priv_ctx->swapchain = pl_vulkan_create_swapchain(rd_priv_ctx->vk, - &(struct pl_vulkan_swapchain_params) { - .surface = rd_priv_ctx->surf, - .present_mode = VK_PRESENT_MODE_IMMEDIATE_KHR, - }); - - if (!rd_priv_ctx->swapchain) { - fprintf(stderr, "Failed creating vulkan swapchain!\n"); - exit(2); - } - - int w = WINDOW_WIDTH, h = WINDOW_HEIGHT; - if (!pl_swapchain_resize(rd_priv_ctx->swapchain, &w, &h)) { - fprintf(stderr, "Failed resizing vulkan swapchain!\n"); - exit(2); - } - - if (w != WINDOW_WIDTH || h != WINDOW_HEIGHT) - printf("Note: window dimensions differ (got %dx%d)\n", w, h); - - rd_priv_ctx->y_tex = NULL; - rd_priv_ctx->u_tex = NULL; - rd_priv_ctx->v_tex = NULL; - - rd_priv_ctx->renderer = NULL; - - return rd_priv_ctx; -} - -static void placebo_renderer_destroy(void *cookie) -{ - Dav1dPlayRendererPrivateContext *rd_priv_ctx = cookie; - assert(rd_priv_ctx != NULL); - - pl_renderer_destroy(&(rd_priv_ctx->renderer)); - pl_tex_destroy(rd_priv_ctx->vk->gpu, &(rd_priv_ctx->y_tex)); - pl_tex_destroy(rd_priv_ctx->vk->gpu, &(rd_priv_ctx->u_tex)); - pl_tex_destroy(rd_priv_ctx->vk->gpu, &(rd_priv_ctx->v_tex)); - pl_swapchain_destroy(&(rd_priv_ctx->swapchain)); - pl_vulkan_destroy(&(rd_priv_ctx->vk)); - vkDestroySurfaceKHR(rd_priv_ctx->vk_inst->instance, rd_priv_ctx->surf, NULL); - pl_vk_inst_destroy(&(rd_priv_ctx->vk_inst)); - pl_context_destroy(&(rd_priv_ctx->ctx)); -} - -static void placebo_render(void *cookie, const Dav1dPlaySettings *settings) -{ - Dav1dPlayRendererPrivateContext *rd_priv_ctx = cookie; - assert(rd_priv_ctx != NULL); - - SDL_LockMutex(rd_priv_ctx->lock); - if (rd_priv_ctx->y_tex == NULL) { - SDL_UnlockMutex(rd_priv_ctx->lock); - return; - } - - // Prepare rendering - if (rd_priv_ctx->renderer == NULL) { - rd_priv_ctx->renderer = pl_renderer_create(rd_priv_ctx->ctx, rd_priv_ctx->vk->gpu); - } - - struct pl_swapchain_frame frame; - bool ok = pl_swapchain_start_frame(rd_priv_ctx->swapchain, &frame); - if (!ok) { - SDL_UnlockMutex(rd_priv_ctx->lock); - return; - } - - const struct pl_tex *img = rd_priv_ctx->y_plane.texture; - struct pl_image image = { - .num_planes = 3, - .planes = { rd_priv_ctx->y_plane, rd_priv_ctx->u_plane, rd_priv_ctx->v_plane }, - .repr = pl_color_repr_hdtv, - .color = pl_color_space_unknown, - .width = img->params.w, - .height = img->params.h, - }; - - struct pl_render_params render_params = {0}; - if (settings->highquality) - render_params = pl_render_default_params; +#include "dp_fifo.h" +#include "dp_renderer.h" - struct pl_render_target target; - pl_render_target_from_swapchain(&target, &frame); - target.profile = (struct pl_icc_profile) { - .data = NULL, - .len = 0, - }; - - if (!pl_render_image(rd_priv_ctx->renderer, &image, &target, &render_params)) { - fprintf(stderr, "Failed rendering frame!\n"); - SDL_UnlockMutex(rd_priv_ctx->lock); - return; - } - - ok = pl_swapchain_submit_frame(rd_priv_ctx->swapchain); - if (!ok) { - fprintf(stderr, "Failed submitting frame!\n"); - SDL_UnlockMutex(rd_priv_ctx->lock); - return; - } - - pl_swapchain_swap_buffers(rd_priv_ctx->swapchain); - SDL_UnlockMutex(rd_priv_ctx->lock); -} - -static int placebo_upload_planes(void *cookie, Dav1dPicture *dav1d_pic, - const Dav1dPlaySettings *settings) -{ - Dav1dPlayRendererPrivateContext *rd_priv_ctx = cookie; - assert(rd_priv_ctx != NULL); - - SDL_LockMutex(rd_priv_ctx->lock); - - if (dav1d_pic == NULL) { - SDL_UnlockMutex(rd_priv_ctx->lock); - return 0; - } - - int width = dav1d_pic->p.w; - int height = dav1d_pic->p.h; - - enum Dav1dPixelLayout dav1d_layout = dav1d_pic->p.layout; - - if (DAV1D_PIXEL_LAYOUT_I420 != dav1d_layout || dav1d_pic->p.bpc != 8) { - fprintf(stderr, "Unsupported pixel format, only 8bit 420 supported so far.\n"); - exit(50); - } - - struct pl_plane_data data_y = { - .type = PL_FMT_UNORM, - .width = width, - .height = height, - .pixel_stride = 1, - .row_stride = dav1d_pic->stride[0], - .component_size = {8}, - .component_map = {0}, - }; - - struct pl_plane_data data_u = { - .type = PL_FMT_UNORM, - .width = width/2, - .height = height/2, - .pixel_stride = 1, - .row_stride = dav1d_pic->stride[1], - .component_size = {8}, - .component_map = {1}, - }; - - struct pl_plane_data data_v = { - .type = PL_FMT_UNORM, - .width = width/2, - .height = height/2, - .pixel_stride = 1, - .row_stride = dav1d_pic->stride[1], - .component_size = {8}, - .component_map = {2}, - }; - - if (settings->zerocopy) { - const struct pl_buf *buf = dav1d_pic->allocator_data; - assert(buf); - data_y.buf = data_u.buf = data_v.buf = buf; - data_y.buf_offset = (uintptr_t) dav1d_pic->data[0] - (uintptr_t) buf->data; - data_u.buf_offset = (uintptr_t) dav1d_pic->data[1] - (uintptr_t) buf->data; - data_v.buf_offset = (uintptr_t) dav1d_pic->data[2] - (uintptr_t) buf->data; - } else { - data_y.pixels = dav1d_pic->data[0]; - data_u.pixels = dav1d_pic->data[1]; - data_v.pixels = dav1d_pic->data[2]; - } - - bool ok = true; - ok &= pl_upload_plane(rd_priv_ctx->vk->gpu, &(rd_priv_ctx->y_plane), &(rd_priv_ctx->y_tex), &data_y); - ok &= pl_upload_plane(rd_priv_ctx->vk->gpu, &(rd_priv_ctx->u_plane), &(rd_priv_ctx->u_tex), &data_u); - ok &= pl_upload_plane(rd_priv_ctx->vk->gpu, &(rd_priv_ctx->v_plane), &(rd_priv_ctx->v_tex), &data_v); - - pl_chroma_location_offset(PL_CHROMA_LEFT, &rd_priv_ctx->u_plane.shift_x, &rd_priv_ctx->u_plane.shift_y); - pl_chroma_location_offset(PL_CHROMA_LEFT, &rd_priv_ctx->v_plane.shift_x, &rd_priv_ctx->v_plane.shift_y); - - if (!ok) { - fprintf(stderr, "Failed uploading planes!\n"); - } - - SDL_UnlockMutex(rd_priv_ctx->lock); - return !ok; -} - -// Align to power of 2 -#define ALIGN2(x, align) (((x) + (align) - 1) & ~((align) - 1)) - -static int placebo_alloc_pic(Dav1dPicture *const p, void *cookie) -{ - Dav1dPlayRendererPrivateContext *rd_priv_ctx = cookie; - assert(rd_priv_ctx != NULL); - SDL_LockMutex(rd_priv_ctx->lock); - - const struct pl_gpu *gpu = rd_priv_ctx->vk->gpu; - int ret = DAV1D_ERR(ENOMEM); - - // Copied from dav1d_default_picture_alloc - const int hbd = p->p.bpc > 8; - const int aligned_w = ALIGN2(p->p.w, 128); - const int aligned_h = ALIGN2(p->p.h, 128); - const int has_chroma = p->p.layout != DAV1D_PIXEL_LAYOUT_I400; - const int ss_ver = p->p.layout == DAV1D_PIXEL_LAYOUT_I420; - const int ss_hor = p->p.layout != DAV1D_PIXEL_LAYOUT_I444; - p->stride[0] = aligned_w << hbd; - p->stride[1] = has_chroma ? (aligned_w >> ss_hor) << hbd : 0; - - // Align strides up to multiples of the GPU performance hints - p->stride[0] = ALIGN2(p->stride[0], gpu->limits.align_tex_xfer_stride); - p->stride[1] = ALIGN2(p->stride[1], gpu->limits.align_tex_xfer_stride); - - // Aligning offsets to 4 also implicity aligns to the texel size (1 or 2) - size_t off_align = ALIGN2(gpu->limits.align_tex_xfer_offset, 4); - const size_t y_sz = ALIGN2(p->stride[0] * aligned_h, off_align); - const size_t uv_sz = ALIGN2(p->stride[1] * (aligned_h >> ss_ver), off_align); - - // The extra DAV1D_PICTURE_ALIGNMENTs are to brute force plane alignment, - // even in the case that the driver gives us insane alignments - const size_t pic_size = y_sz + 2 * uv_sz; - const size_t total_size = pic_size + DAV1D_PICTURE_ALIGNMENT * 4; - - // Validate size limitations - if (total_size > gpu->limits.max_xfer_size) { - printf("alloc of %zu bytes exceeds limits\n", total_size); - goto err; - } - - const struct pl_buf *buf = pl_buf_create(gpu, &(struct pl_buf_params) { - .type = PL_BUF_TEX_TRANSFER, - .host_mapped = true, - .size = total_size, - .memory_type = PL_BUF_MEM_HOST, - .user_data = p, - }); - - if (!buf) { - printf("alloc of GPU mapped buffer failed\n"); - goto err; - } - - assert(buf->data); - uintptr_t base = (uintptr_t) buf->data, data[3]; - data[0] = ALIGN2(base, DAV1D_PICTURE_ALIGNMENT); - data[1] = ALIGN2(data[0] + y_sz, DAV1D_PICTURE_ALIGNMENT); - data[2] = ALIGN2(data[1] + uv_sz, DAV1D_PICTURE_ALIGNMENT); - - // Sanity check offset alignment for the sake of debugging - if (data[0] - base != ALIGN2(data[0] - base, off_align) || - data[1] - base != ALIGN2(data[1] - base, off_align) || - data[2] - base != ALIGN2(data[2] - base, off_align)) - { - printf("GPU buffer horribly misaligned, expect slowdown!\n"); - } - - p->allocator_data = (void *) buf; - p->data[0] = (void *) data[0]; - p->data[1] = (void *) data[1]; - p->data[2] = (void *) data[2]; - ret = 0; - - // fall through -err: - SDL_UnlockMutex(rd_priv_ctx->lock); - return ret; -} - -static void placebo_release_pic(Dav1dPicture *pic, void *cookie) -{ - Dav1dPlayRendererPrivateContext *rd_priv_ctx = cookie; - assert(rd_priv_ctx != NULL); - assert(pic->allocator_data); - - SDL_LockMutex(rd_priv_ctx->lock); - const struct pl_gpu *gpu = rd_priv_ctx->vk->gpu; - pl_buf_destroy(gpu, (const struct pl_buf **) &pic->allocator_data); - SDL_UnlockMutex(rd_priv_ctx->lock); -} - -static const Dav1dPlayRenderInfo renderer_info = { - .create_renderer = placebo_renderer_create, - .destroy_renderer = placebo_renderer_destroy, - .render = placebo_render, - .update_frame = placebo_upload_planes, - .alloc_pic = placebo_alloc_pic, - .release_pic = placebo_release_pic, -}; - -#else - -/** - * Renderer context for SDL - */ -typedef struct renderer_priv_ctx -{ - // SDL renderer - SDL_Renderer *renderer; - // Lock protecting access to the texture - SDL_mutex *lock; - // Texture to render - SDL_Texture *tex; -} Dav1dPlayRendererPrivateContext; - -static void *sdl_renderer_create(void *data) -{ - SDL_Window *win = data; - - // Alloc - Dav1dPlayRendererPrivateContext *rd_priv_ctx = malloc(sizeof(Dav1dPlayRendererPrivateContext)); - if (rd_priv_ctx == NULL) { - return NULL; - } - - // Create renderer - rd_priv_ctx->renderer = SDL_CreateRenderer(win, -1, SDL_RENDERER_ACCELERATED); - // Set scale quality - SDL_SetHint(SDL_HINT_RENDER_SCALE_QUALITY, "linear"); - - // Create Mutex - rd_priv_ctx->lock = SDL_CreateMutex(); - if (rd_priv_ctx->lock == NULL) { - fprintf(stderr, "SDL_CreateMutex failed: %s\n", SDL_GetError()); - free(rd_priv_ctx); - return NULL; - } - - rd_priv_ctx->tex = NULL; - - return rd_priv_ctx; -} - -static void sdl_renderer_destroy(void *cookie) -{ - Dav1dPlayRendererPrivateContext *rd_priv_ctx = cookie; - assert(rd_priv_ctx != NULL); - - SDL_DestroyRenderer(rd_priv_ctx->renderer); - SDL_DestroyMutex(rd_priv_ctx->lock); - free(rd_priv_ctx); -} - -static void sdl_render(void *cookie, const Dav1dPlaySettings *settings) -{ - Dav1dPlayRendererPrivateContext *rd_priv_ctx = cookie; - assert(rd_priv_ctx != NULL); - - SDL_LockMutex(rd_priv_ctx->lock); - - if (rd_priv_ctx->tex == NULL) { - SDL_UnlockMutex(rd_priv_ctx->lock); - return; - } - - // Display the frame - SDL_RenderClear(rd_priv_ctx->renderer); - SDL_RenderCopy(rd_priv_ctx->renderer, rd_priv_ctx->tex, NULL, NULL); - SDL_RenderPresent(rd_priv_ctx->renderer); - - SDL_UnlockMutex(rd_priv_ctx->lock); -} - -static int sdl_update_texture(void *cookie, Dav1dPicture *dav1d_pic, - const Dav1dPlaySettings *settings) -{ - Dav1dPlayRendererPrivateContext *rd_priv_ctx = cookie; - assert(rd_priv_ctx != NULL); - - SDL_LockMutex(rd_priv_ctx->lock); - - if (dav1d_pic == NULL) { - rd_priv_ctx->tex = NULL; - SDL_UnlockMutex(rd_priv_ctx->lock); - return 0; - } - - int width = dav1d_pic->p.w; - int height = dav1d_pic->p.h; - int tex_w = width; - int tex_h = height; - - enum Dav1dPixelLayout dav1d_layout = dav1d_pic->p.layout; - - if (DAV1D_PIXEL_LAYOUT_I420 != dav1d_layout || dav1d_pic->p.bpc != 8) { - fprintf(stderr, "Unsupported pixel format, only 8bit 420 supported so far.\n"); - exit(50); - } - - SDL_Texture *texture = rd_priv_ctx->tex; - if (texture != NULL) { - SDL_QueryTexture(texture, NULL, NULL, &tex_w, &tex_h); - if (tex_w != width || tex_h != height) { - SDL_DestroyTexture(texture); - texture = NULL; - } - } - - if (texture == NULL) { - texture = SDL_CreateTexture(rd_priv_ctx->renderer, SDL_PIXELFORMAT_IYUV, - SDL_TEXTUREACCESS_STREAMING, width, height); - } - - SDL_UpdateYUVTexture(texture, NULL, - dav1d_pic->data[0], (int)dav1d_pic->stride[0], // Y - dav1d_pic->data[1], (int)dav1d_pic->stride[1], // U - dav1d_pic->data[2], (int)dav1d_pic->stride[1] // V - ); - - rd_priv_ctx->tex = texture; - SDL_UnlockMutex(rd_priv_ctx->lock); - return 0; -} - -static const Dav1dPlayRenderInfo renderer_info = { - .create_renderer = sdl_renderer_create, - .destroy_renderer = sdl_renderer_destroy, - .render = sdl_render, - .update_frame = sdl_update_texture -}; - -#endif +// Selected renderer callbacks and cookie +static const Dav1dPlayRenderInfo *renderer_info = { NULL }; /** * Render context structure * This structure contains informations necessary * to be shared between the decoder and the renderer @@ -720,12 +51,10 @@ typedef struct render_context { Dav1dPlaySettings settings; Dav1dSettings lib_settings; - // Renderer callbacks - Dav1dPlayRenderInfo *renderer_info; // Renderer private data (passed to callbacks) void *rd_priv; // Lock to protect access to the context structure SDL_mutex *lock; @@ -766,11 +95,13 @@ " --untimed/-u: ignore PTS, render as fast as possible\n" " --framethreads $num: number of frame threads (default: 1)\n" " --tilethreads $num: number of tile threads (default: 1)\n" " --highquality: enable high quality rendering\n" " --zerocopy/-z: enable zero copy upload path\n" - " --version/-v: print version and exit\n"); + " --gpugrain/-g: enable GPU grain synthesis\n" + " --version/-v: print version and exit\n" + " --renderer/-r: select renderer backend (default: auto)\n"); exit(1); } static unsigned parse_unsigned(const char *const optarg, const int option, const char *const app) @@ -789,11 +120,11 @@ int o; Dav1dPlaySettings *settings = &rd_ctx->settings; Dav1dSettings *lib_settings = &rd_ctx->lib_settings; // Short options - static const char short_opts[] = "i:vuz"; + static const char short_opts[] = "i:vuzgr:"; enum { ARG_FRAME_THREADS = 256, ARG_TILE_THREADS, ARG_HIGH_QUALITY, @@ -806,10 +137,12 @@ { "untimed", 0, NULL, 'u' }, { "framethreads", 1, NULL, ARG_FRAME_THREADS }, { "tilethreads", 1, NULL, ARG_TILE_THREADS }, { "highquality", 0, NULL, ARG_HIGH_QUALITY }, { "zerocopy", 0, NULL, 'z' }, + { "gpugrain", 0, NULL, 'g' }, + { "renderer", 0, NULL, 'r'}, { NULL, 0, NULL, 0 }, }; while ((o = getopt_long(argc, argv, short_opts, long_opts, NULL)) != -1) { switch (o) { @@ -822,19 +155,19 @@ case 'u': settings->untimed = true; break; case ARG_HIGH_QUALITY: settings->highquality = true; -#ifndef HAVE_PLACEBO_VULKAN - fprintf(stderr, "warning: --highquality requires libplacebo\n"); -#endif break; case 'z': settings->zerocopy = true; -#ifndef HAVE_PLACEBO_VULKAN - fprintf(stderr, "warning: --zerocopy requires libplacebo\n"); -#endif + break; + case 'g': + settings->gpugrain = true; + break; + case 'r': + settings->renderer_name = optarg; break; case ARG_FRAME_THREADS: lib_settings->n_frame_threads = parse_unsigned(optarg, ARG_FRAME_THREADS, argv[0]); break; @@ -850,20 +183,22 @@ if (optind < argc) dp_settings_print_usage(argv[0], "Extra/unused arguments found, e.g. '%s'\n", argv[optind]); if (!settings->inputfile) dp_settings_print_usage(argv[0], "Input file (-i/--input) is required"); + if (settings->renderer_name && strcmp(settings->renderer_name, "auto") == 0) + settings->renderer_name = NULL; } /** * Destroy a Dav1dPlayRenderContext */ static void dp_rd_ctx_destroy(Dav1dPlayRenderContext *rd_ctx) { assert(rd_ctx != NULL); - renderer_info.destroy_renderer(rd_ctx->rd_priv); + renderer_info->destroy_renderer(rd_ctx->rd_priv); dp_fifo_destroy(rd_ctx->fifo); SDL_DestroyMutex(rd_ctx->lock); free(rd_ctx); } @@ -871,11 +206,11 @@ * Create a Dav1dPlayRenderContext * * \note The Dav1dPlayRenderContext must be destroyed * again by using dp_rd_ctx_destroy. */ -static Dav1dPlayRenderContext *dp_rd_ctx_create(void *rd_data) +static Dav1dPlayRenderContext *dp_rd_ctx_create(int argc, char **argv) { Dav1dPlayRenderContext *rd_ctx; // Alloc rd_ctx = malloc(sizeof(Dav1dPlayRenderContext)); @@ -905,21 +240,33 @@ dp_fifo_destroy(rd_ctx->fifo); free(rd_ctx); return NULL; } - rd_ctx->rd_priv = renderer_info.create_renderer(rd_data); + // Parse and validate arguments + dav1d_default_settings(&rd_ctx->lib_settings); + memset(&rd_ctx->settings, 0, sizeof(rd_ctx->settings)); + dp_rd_ctx_parse_args(rd_ctx, argc, argv); + + // Select renderer + renderer_info = dp_get_renderer(rd_ctx->settings.renderer_name); + + if (renderer_info == NULL) { + printf("No suitable rendered matching %s found.\n", + (rd_ctx->settings.renderer_name) ? rd_ctx->settings.renderer_name : "auto"); + } else { + printf("Using %s renderer\n", renderer_info->name); + } + + rd_ctx->rd_priv = (renderer_info) ? renderer_info->create_renderer() : NULL; if (rd_ctx->rd_priv == NULL) { SDL_DestroyMutex(rd_ctx->lock); dp_fifo_destroy(rd_ctx->fifo); free(rd_ctx); return NULL; } - dav1d_default_settings(&rd_ctx->lib_settings); - memset(&rd_ctx->settings, 0, sizeof(rd_ctx->settings)); - rd_ctx->last_pts = 0; rd_ctx->last_ticks = 0; rd_ctx->current_pts = 0; rd_ctx->timebase = 0; rd_ctx->dec_should_terminate = 0; @@ -947,11 +294,11 @@ * new picture. */ static void dp_rd_ctx_update_with_dav1d_picture(Dav1dPlayRenderContext *rd_ctx, Dav1dPicture *dav1d_pic) { - renderer_info.update_frame(rd_ctx->rd_priv, dav1d_pic, &rd_ctx->settings); + renderer_info->update_frame(rd_ctx->rd_priv, dav1d_pic, &rd_ctx->settings); rd_ctx->current_pts = dav1d_pic->m.timestamp; } /** * Terminate decoder thread (async) @@ -1002,11 +349,11 @@ SDL_Delay(wait_time); } else if (wait_time < -10) { // Do not warn for minor time drifts fprintf(stderr, "Frame displayed %f seconds too late\n", wait_time/(float)1000); } - renderer_info.render(rd_ctx->rd_priv, &rd_ctx->settings); + renderer_info->render(rd_ctx->rd_priv, &rd_ctx->settings); rd_ctx->last_ticks = SDL_GetTicks(); } /* Decoder thread "main" function */ @@ -1024,13 +371,10 @@ // Store current ticks for stats calculation uint32_t decoder_start = SDL_GetTicks(); Dav1dPlaySettings settings = rd_ctx->settings; - // Init dav1d tools demuxers - init_demuxers(); - if ((res = input_open(&in_ctx, "ivf", settings.inputfile, fps, &total, timebase)) < 0) { fprintf(stderr, "Failed to open demuxer\n"); @@ -1153,11 +497,10 @@ } int main(int argc, char **argv) { SDL_Thread *decoder_thread; - SDL_Window *win = NULL; // Check for version mismatch between library and tool const char *version = dav1d_version(); if (strcmp(version, DAV1D_VERSION)) { fprintf(stderr, "Version mismatch (library: %s, executable: %s)\n", @@ -1167,38 +510,34 @@ // Init SDL2 library if (SDL_Init(SDL_INIT_VIDEO | SDL_INIT_TIMER) < 0) return 10; - // Create Window and Renderer - int window_flags = SDL_WINDOW_SHOWN | SDL_WINDOW_ALLOW_HIGHDPI; -#ifdef HAVE_PLACEBO_VULKAN - window_flags |= SDL_WINDOW_VULKAN; -#endif - win = SDL_CreateWindow("Dav1dPlay", SDL_WINDOWPOS_CENTERED, SDL_WINDOWPOS_CENTERED, - WINDOW_WIDTH, WINDOW_HEIGHT, window_flags); - SDL_SetWindowResizable(win, SDL_TRUE); - // Create render context - Dav1dPlayRenderContext *rd_ctx = dp_rd_ctx_create(win); + Dav1dPlayRenderContext *rd_ctx = dp_rd_ctx_create(argc, argv); if (rd_ctx == NULL) { fprintf(stderr, "Failed creating render context\n"); return 5; } - // Parse and validate arguments - dp_rd_ctx_parse_args(rd_ctx, argc, argv); - if (rd_ctx->settings.zerocopy) { - if (renderer_info.alloc_pic) { + if (renderer_info->alloc_pic) { rd_ctx->lib_settings.allocator = (Dav1dPicAllocator) { .cookie = rd_ctx->rd_priv, - .alloc_picture_callback = renderer_info.alloc_pic, - .release_picture_callback = renderer_info.release_pic, + .alloc_picture_callback = renderer_info->alloc_pic, + .release_picture_callback = renderer_info->release_pic, }; } else { - fprintf(stderr, "--zerocopy unsupported by compiled renderer\n"); + fprintf(stderr, "--zerocopy unsupported by selected renderer\n"); + } + } + + if (rd_ctx->settings.gpugrain) { + if (renderer_info->supports_gpu_grain) { + rd_ctx->lib_settings.apply_grain = 0; + } else { + fprintf(stderr, "--gpugrain unsupported by selected renderer\n"); } } // Start decoder thread decoder_thread = SDL_CreateThread(decoder_thread_main, "Decoder thread", rd_ctx); @@ -1208,10 +547,14 @@ SDL_Event e; if (SDL_WaitEvent(&e)) { if (e.type == SDL_QUIT) { dp_rd_ctx_request_shutdown(rd_ctx); + } else if (e.type == SDL_WINDOWEVENT) { + if (e.window.event == SDL_WINDOWEVENT_SIZE_CHANGED) { + // TODO: Handle window resizes + } } else if (e.type == rd_ctx->renderer_event_type) { if (e.user.code == DAV1D_EVENT_NEW_FRAME) { // Dequeue frame and update the render context with it Dav1dPicture *p = dp_fifo_shift(rd_ctx->fifo); @@ -1233,9 +576,8 @@ int decoder_ret = 0; SDL_WaitThread(decoder_thread, &decoder_ret); dp_rd_ctx_destroy(rd_ctx); - SDL_DestroyWindow(win); return decoder_ret; } diff --git a/third_party/dav1d/examples/dp_fifo.h b/third_party/dav1d/examples/dp_fifo.h new file mode 100644 --- /dev/null +++ b/third_party/dav1d/examples/dp_fifo.h @@ -0,0 +1,61 @@ +/* + * Copyright © 2019, VideoLAN and dav1d authors + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* + * Dav1dPlay FIFO helper + */ + +typedef struct dp_fifo Dav1dPlayPtrFifo; + +/* Create a FIFO + * + * Creates a FIFO with the given capacity. + * If the capacity is reached, new inserts into the FIFO + * will block until enough space is available again. + */ +Dav1dPlayPtrFifo *dp_fifo_create(size_t capacity); + +/* Destroy a FIFO + * + * The FIFO must be empty before it is destroyed! + */ +void dp_fifo_destroy(Dav1dPlayPtrFifo *fifo); + +/* Shift FIFO + * + * Return the first item from the FIFO, thereby removing it from + * the FIFO and making room for new entries. + */ +void *dp_fifo_shift(Dav1dPlayPtrFifo *fifo); + +/* Push to FIFO + * + * Add an item to the end of the FIFO. + * If the FIFO is full, this call will block until there is again enough + * space in the FIFO, so calling this from the "consumer" thread if no + * other thread will call dp_fifo_shift will lead to a deadlock. + */ +void dp_fifo_push(Dav1dPlayPtrFifo *fifo, void *element); diff --git a/third_party/dav1d/examples/dp_fifo.c b/third_party/dav1d/examples/dp_fifo.c new file mode 100644 --- /dev/null +++ b/third_party/dav1d/examples/dp_fifo.c @@ -0,0 +1,123 @@ +/* + * Copyright © 2019, VideoLAN and dav1d authors + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include "dp_fifo.h" + +// FIFO structure +struct dp_fifo +{ + SDL_mutex *lock; + SDL_cond *cond_change; + size_t capacity; + size_t count; + void **entries; +}; + + +Dav1dPlayPtrFifo *dp_fifo_create(size_t capacity) +{ + Dav1dPlayPtrFifo *fifo; + + assert(capacity > 0); + if (capacity <= 0) + return NULL; + + fifo = malloc(sizeof(*fifo)); + if (fifo == NULL) + return NULL; + + fifo->capacity = capacity; + fifo->count = 0; + + fifo->lock = SDL_CreateMutex(); + if (fifo->lock == NULL) { + free(fifo); + return NULL; + } + fifo->cond_change = SDL_CreateCond(); + if (fifo->cond_change == NULL) { + SDL_DestroyMutex(fifo->lock); + free(fifo); + return NULL; + } + + fifo->entries = calloc(capacity, sizeof(void*)); + if (fifo->entries == NULL) { + dp_fifo_destroy(fifo); + return NULL; + } + + return fifo; +} + +// Destroy FIFO +void dp_fifo_destroy(Dav1dPlayPtrFifo *fifo) +{ + assert(fifo->count == 0); + SDL_DestroyMutex(fifo->lock); + SDL_DestroyCond(fifo->cond_change); + free(fifo->entries); + free(fifo); +} + +// Push to FIFO +void dp_fifo_push(Dav1dPlayPtrFifo *fifo, void *element) +{ + SDL_LockMutex(fifo->lock); + while (fifo->count == fifo->capacity) + SDL_CondWait(fifo->cond_change, fifo->lock); + fifo->entries[fifo->count++] = element; + if (fifo->count == 1) + SDL_CondSignal(fifo->cond_change); + SDL_UnlockMutex(fifo->lock); +} + +// Helper that shifts the FIFO array +static void *dp_fifo_array_shift(void **arr, size_t len) +{ + void *shifted_element = arr[0]; + for (size_t i = 1; i < len; ++i) + arr[i-1] = arr[i]; + return shifted_element; +} + +// Get item from FIFO +void *dp_fifo_shift(Dav1dPlayPtrFifo *fifo) +{ + SDL_LockMutex(fifo->lock); + while (fifo->count == 0) + SDL_CondWait(fifo->cond_change, fifo->lock); + void *res = dp_fifo_array_shift(fifo->entries, fifo->count--); + if (fifo->count == fifo->capacity - 1) + SDL_CondSignal(fifo->cond_change); + SDL_UnlockMutex(fifo->lock); + return res; +} + + diff --git a/third_party/dav1d/examples/dp_renderer.h b/third_party/dav1d/examples/dp_renderer.h new file mode 100644 --- /dev/null +++ b/third_party/dav1d/examples/dp_renderer.h @@ -0,0 +1,132 @@ +/* + * Copyright © 2020, VideoLAN and dav1d authors + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include "dav1d/dav1d.h" + +#include +#ifdef HAVE_PLACEBO +# include +#endif + +// Check libplacebo Vulkan rendering +#if defined(HAVE_VULKAN) && defined(SDL_VIDEO_VULKAN) +# if defined(PL_HAVE_VULKAN) && PL_HAVE_VULKAN +# define HAVE_RENDERER_PLACEBO +# define HAVE_PLACEBO_VULKAN +# endif +#endif + +// Check libplacebo OpenGL rendering +#if defined(PL_HAVE_OPENGL) && PL_HAVE_OPENGL +# define HAVE_RENDERER_PLACEBO +# define HAVE_PLACEBO_OPENGL +#endif + +/** + * Settings structure + * Hold all settings available for the player, + * this is usually filled by parsing arguments + * from the console. + */ +typedef struct { + const char *inputfile; + const char *renderer_name; + int highquality; + int untimed; + int zerocopy; + int gpugrain; +} Dav1dPlaySettings; + +#define WINDOW_WIDTH 910 +#define WINDOW_HEIGHT 512 + +#define DAV1D_EVENT_NEW_FRAME 1 +#define DAV1D_EVENT_DEC_QUIT 2 + +/** + * Renderer info + */ +typedef struct rdr_info +{ + // Renderer name + const char *name; + // Cookie passed to the renderer implementation callbacks + void *cookie; + // Callback to create the renderer + void* (*create_renderer)(); + // Callback to destroy the renderer + void (*destroy_renderer)(void *cookie); + // Callback to the render function that renders a prevously sent frame + void (*render)(void *cookie, const Dav1dPlaySettings *settings); + // Callback to the send frame function + int (*update_frame)(void *cookie, Dav1dPicture *dav1d_pic, + const Dav1dPlaySettings *settings); + // Callback for alloc/release pictures (optional) + int (*alloc_pic)(Dav1dPicture *pic, void *cookie); + void (*release_pic)(Dav1dPicture *pic, void *cookie); + // Whether or not this renderer can apply on-GPU film grain synthesis + int supports_gpu_grain; +} Dav1dPlayRenderInfo; + +extern const Dav1dPlayRenderInfo rdr_placebo_vk; +extern const Dav1dPlayRenderInfo rdr_placebo_gl; +extern const Dav1dPlayRenderInfo rdr_sdl; + +// Available renderes ordered by priority +static const Dav1dPlayRenderInfo* const dp_renderers[] = { + &rdr_placebo_vk, + &rdr_placebo_gl, + &rdr_sdl, +}; + +static inline const Dav1dPlayRenderInfo *dp_get_renderer(const char *name) +{ + for (size_t i = 0; i < (sizeof(dp_renderers)/sizeof(*dp_renderers)); ++i) + { + if (dp_renderers[i]->name == NULL) + continue; + + if (name == NULL || strcmp(name, dp_renderers[i]->name) == 0) { + return dp_renderers[i]; + } + } + return NULL; +} + +static inline SDL_Window *dp_create_sdl_window(int window_flags) +{ + SDL_Window *win; + window_flags |= SDL_WINDOW_SHOWN | SDL_WINDOW_ALLOW_HIGHDPI; + + win = SDL_CreateWindow("Dav1dPlay", SDL_WINDOWPOS_CENTERED, SDL_WINDOWPOS_CENTERED, + WINDOW_WIDTH, WINDOW_HEIGHT, window_flags); + SDL_SetWindowResizable(win, SDL_TRUE); + + return win; +} diff --git a/third_party/dav1d/examples/dp_renderer_placebo.c b/third_party/dav1d/examples/dp_renderer_placebo.c new file mode 100644 --- /dev/null +++ b/third_party/dav1d/examples/dp_renderer_placebo.c @@ -0,0 +1,723 @@ +/* + * Copyright © 2020, VideoLAN and dav1d authors + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "dp_renderer.h" + +#ifdef HAVE_RENDERER_PLACEBO +#include + +#include +#include + +#ifdef HAVE_PLACEBO_VULKAN +# include +# include +#endif +#ifdef HAVE_PLACEBO_OPENGL +# include +# include +#endif + + +/** + * Renderer context for libplacebo + */ +typedef struct renderer_priv_ctx +{ + // SDL window + SDL_Window *win; + // Placebo context + struct pl_context *ctx; + // Placebo renderer + struct pl_renderer *renderer; +#ifdef HAVE_PLACEBO_VULKAN + // Placebo Vulkan handle + const struct pl_vulkan *vk; + // Placebo Vulkan instance + const struct pl_vk_inst *vk_inst; + // Vulkan surface + VkSurfaceKHR surf; +#endif +#ifdef HAVE_PLACEBO_OPENGL + // Placebo OpenGL handle + const struct pl_opengl *gl; +#endif + // Placebo GPU + const struct pl_gpu *gpu; + // Placebo swapchain + const struct pl_swapchain *swapchain; + // Lock protecting access to the texture + SDL_mutex *lock; + // Image to render, and planes backing them + struct pl_image image; + const struct pl_tex *plane_tex[3]; +} Dav1dPlayRendererPrivateContext; + +static Dav1dPlayRendererPrivateContext* + placebo_renderer_create_common(int window_flags) +{ + // Create Window + SDL_Window *sdlwin = dp_create_sdl_window(window_flags | SDL_WINDOW_RESIZABLE); + if (sdlwin == NULL) + return NULL; + + // Alloc + Dav1dPlayRendererPrivateContext *rd_priv_ctx = malloc(sizeof(Dav1dPlayRendererPrivateContext)); + if (rd_priv_ctx == NULL) { + return NULL; + } + + *rd_priv_ctx = (Dav1dPlayRendererPrivateContext) {0}; + rd_priv_ctx->win = sdlwin; + + // Init libplacebo + rd_priv_ctx->ctx = pl_context_create(PL_API_VER, &(struct pl_context_params) { + .log_cb = pl_log_color, +#ifndef NDEBUG + .log_level = PL_LOG_DEBUG, +#else + .log_level = PL_LOG_WARN, +#endif + }); + if (rd_priv_ctx->ctx == NULL) { + free(rd_priv_ctx); + return NULL; + } + + // Create Mutex + rd_priv_ctx->lock = SDL_CreateMutex(); + if (rd_priv_ctx->lock == NULL) { + fprintf(stderr, "SDL_CreateMutex failed: %s\n", SDL_GetError()); + pl_context_destroy(&(rd_priv_ctx->ctx)); + free(rd_priv_ctx); + return NULL; + } + + return rd_priv_ctx; +} + +#ifdef HAVE_PLACEBO_OPENGL +static void *placebo_renderer_create_gl() +{ + SDL_Window *sdlwin = NULL; + SDL_GL_SetAttribute(SDL_GL_CONTEXT_FLAGS, SDL_GL_CONTEXT_DEBUG_FLAG); + + // Common init + Dav1dPlayRendererPrivateContext *rd_priv_ctx = + placebo_renderer_create_common(SDL_WINDOW_OPENGL); + + if (rd_priv_ctx == NULL) + return NULL; + sdlwin = rd_priv_ctx->win; + + // Init OpenGL + struct pl_opengl_params params = pl_opengl_default_params; +# ifndef NDEBUG + params.debug = true; +# endif + + SDL_GLContext glcontext = SDL_GL_CreateContext(sdlwin); + SDL_GL_MakeCurrent(sdlwin, glcontext); + + rd_priv_ctx->gl = pl_opengl_create(rd_priv_ctx->ctx, ¶ms); + if (!rd_priv_ctx->gl) { + fprintf(stderr, "Failed creating opengl device!\n"); + exit(2); + } + + rd_priv_ctx->swapchain = pl_opengl_create_swapchain(rd_priv_ctx->gl, + &(struct pl_opengl_swapchain_params) { + .swap_buffers = (void (*)(void *)) SDL_GL_SwapWindow, + .priv = sdlwin, + }); + + if (!rd_priv_ctx->swapchain) { + fprintf(stderr, "Failed creating opengl swapchain!\n"); + exit(2); + } + + int w = WINDOW_WIDTH, h = WINDOW_HEIGHT; + SDL_GL_GetDrawableSize(sdlwin, &w, &h); + + if (!pl_swapchain_resize(rd_priv_ctx->swapchain, &w, &h)) { + fprintf(stderr, "Failed resizing vulkan swapchain!\n"); + exit(2); + } + + rd_priv_ctx->gpu = rd_priv_ctx->gl->gpu; + + if (w != WINDOW_WIDTH || h != WINDOW_HEIGHT) + printf("Note: window dimensions differ (got %dx%d)\n", w, h); + + return rd_priv_ctx; +} +#endif + +#ifdef HAVE_PLACEBO_VULKAN +static void *placebo_renderer_create_vk() +{ + SDL_Window *sdlwin = NULL; + + // Common init + Dav1dPlayRendererPrivateContext *rd_priv_ctx = + placebo_renderer_create_common(SDL_WINDOW_VULKAN); + + if (rd_priv_ctx == NULL) + return NULL; + sdlwin = rd_priv_ctx->win; + + // Init Vulkan + unsigned num = 0; + if (!SDL_Vulkan_GetInstanceExtensions(sdlwin, &num, NULL)) { + fprintf(stderr, "Failed enumerating Vulkan extensions: %s\n", SDL_GetError()); + exit(1); + } + + const char **extensions = malloc(num * sizeof(const char *)); + assert(extensions); + + SDL_bool ok = SDL_Vulkan_GetInstanceExtensions(sdlwin, &num, extensions); + if (!ok) { + fprintf(stderr, "Failed getting Vk instance extensions\n"); + exit(1); + } + + if (num > 0) { + printf("Requesting %d additional Vulkan extensions:\n", num); + for (unsigned i = 0; i < num; i++) + printf(" %s\n", extensions[i]); + } + + struct pl_vk_inst_params iparams = pl_vk_inst_default_params; + iparams.extensions = extensions; + iparams.num_extensions = num; + + rd_priv_ctx->vk_inst = pl_vk_inst_create(rd_priv_ctx->ctx, &iparams); + if (!rd_priv_ctx->vk_inst) { + fprintf(stderr, "Failed creating Vulkan instance!\n"); + exit(1); + } + free(extensions); + + if (!SDL_Vulkan_CreateSurface(sdlwin, rd_priv_ctx->vk_inst->instance, &rd_priv_ctx->surf)) { + fprintf(stderr, "Failed creating vulkan surface: %s\n", SDL_GetError()); + exit(1); + } + + struct pl_vulkan_params params = pl_vulkan_default_params; + params.instance = rd_priv_ctx->vk_inst->instance; + params.surface = rd_priv_ctx->surf; + params.allow_software = true; + + rd_priv_ctx->vk = pl_vulkan_create(rd_priv_ctx->ctx, ¶ms); + if (!rd_priv_ctx->vk) { + fprintf(stderr, "Failed creating vulkan device!\n"); + exit(2); + } + + // Create swapchain + rd_priv_ctx->swapchain = pl_vulkan_create_swapchain(rd_priv_ctx->vk, + &(struct pl_vulkan_swapchain_params) { + .surface = rd_priv_ctx->surf, + .present_mode = VK_PRESENT_MODE_IMMEDIATE_KHR, + }); + + if (!rd_priv_ctx->swapchain) { + fprintf(stderr, "Failed creating vulkan swapchain!\n"); + exit(2); + } + + int w = WINDOW_WIDTH, h = WINDOW_HEIGHT; + if (!pl_swapchain_resize(rd_priv_ctx->swapchain, &w, &h)) { + fprintf(stderr, "Failed resizing vulkan swapchain!\n"); + exit(2); + } + + rd_priv_ctx->gpu = rd_priv_ctx->vk->gpu; + + if (w != WINDOW_WIDTH || h != WINDOW_HEIGHT) + printf("Note: window dimensions differ (got %dx%d)\n", w, h); + + return rd_priv_ctx; +} +#endif + +static void placebo_renderer_destroy(void *cookie) +{ + Dav1dPlayRendererPrivateContext *rd_priv_ctx = cookie; + assert(rd_priv_ctx != NULL); + + pl_renderer_destroy(&(rd_priv_ctx->renderer)); + pl_swapchain_destroy(&(rd_priv_ctx->swapchain)); + for (int i = 0; i < 3; i++) + pl_tex_destroy(rd_priv_ctx->gpu, &(rd_priv_ctx->plane_tex[i])); + +#ifdef HAVE_PLACEBO_VULKAN + if (rd_priv_ctx->vk) { + pl_vulkan_destroy(&(rd_priv_ctx->vk)); + vkDestroySurfaceKHR(rd_priv_ctx->vk_inst->instance, rd_priv_ctx->surf, NULL); + pl_vk_inst_destroy(&(rd_priv_ctx->vk_inst)); + } +#endif +#ifdef HAVE_PLACEBO_OPENGL + if (rd_priv_ctx->gl) + pl_opengl_destroy(&(rd_priv_ctx->gl)); +#endif + + SDL_DestroyWindow(rd_priv_ctx->win); + + pl_context_destroy(&(rd_priv_ctx->ctx)); +} + +static void placebo_render(void *cookie, const Dav1dPlaySettings *settings) +{ + Dav1dPlayRendererPrivateContext *rd_priv_ctx = cookie; + assert(rd_priv_ctx != NULL); + + SDL_LockMutex(rd_priv_ctx->lock); + if (!rd_priv_ctx->image.num_planes) { + SDL_UnlockMutex(rd_priv_ctx->lock); + return; + } + + // Prepare rendering + if (rd_priv_ctx->renderer == NULL) { + rd_priv_ctx->renderer = pl_renderer_create(rd_priv_ctx->ctx, rd_priv_ctx->gpu); + } + + struct pl_swapchain_frame frame; + bool ok = pl_swapchain_start_frame(rd_priv_ctx->swapchain, &frame); + if (!ok) { + SDL_UnlockMutex(rd_priv_ctx->lock); + return; + } + + struct pl_render_params render_params = {0}; + if (settings->highquality) + render_params = pl_render_default_params; + + struct pl_render_target target; + pl_render_target_from_swapchain(&target, &frame); + target.profile = (struct pl_icc_profile) { + .data = NULL, + .len = 0, + }; + +#if PL_API_VER >= 66 + pl_rect2df_aspect_copy(&target.dst_rect, &rd_priv_ctx->image.src_rect, 0.0); + if (pl_render_target_partial(&target)) + pl_tex_clear(rd_priv_ctx->gpu, target.fbo, (float[4]){ 0.0 }); +#endif + + if (!pl_render_image(rd_priv_ctx->renderer, &rd_priv_ctx->image, &target, &render_params)) { + fprintf(stderr, "Failed rendering frame!\n"); + pl_tex_clear(rd_priv_ctx->gpu, target.fbo, (float[4]){ 1.0 }); + } + + ok = pl_swapchain_submit_frame(rd_priv_ctx->swapchain); + if (!ok) { + fprintf(stderr, "Failed submitting frame!\n"); + SDL_UnlockMutex(rd_priv_ctx->lock); + return; + } + + pl_swapchain_swap_buffers(rd_priv_ctx->swapchain); + SDL_UnlockMutex(rd_priv_ctx->lock); +} + +static int placebo_upload_image(void *cookie, Dav1dPicture *dav1d_pic, + const Dav1dPlaySettings *settings) +{ + Dav1dPlayRendererPrivateContext *rd_priv_ctx = cookie; + assert(rd_priv_ctx != NULL); + + SDL_LockMutex(rd_priv_ctx->lock); + + if (dav1d_pic == NULL) { + SDL_UnlockMutex(rd_priv_ctx->lock); + return 0; + } + + int width = dav1d_pic->p.w; + int height = dav1d_pic->p.h; + int sub_x = 0, sub_y = 0; + int bytes = (dav1d_pic->p.bpc + 7) / 8; // rounded up + enum pl_chroma_location chroma_loc = PL_CHROMA_UNKNOWN; + + struct pl_image *image = &rd_priv_ctx->image; + *image = (struct pl_image) { + .num_planes = 3, + .width = width, + .height = height, + .src_rect = {0, 0, width, height}, + + .repr = { + .bits = { + .sample_depth = bytes * 8, + .color_depth = dav1d_pic->p.bpc, + }, + }, + }; + + // Figure out the correct plane dimensions/count + switch (dav1d_pic->p.layout) { + case DAV1D_PIXEL_LAYOUT_I400: + image->num_planes = 1; + break; + case DAV1D_PIXEL_LAYOUT_I420: + sub_x = sub_y = 1; + break; + case DAV1D_PIXEL_LAYOUT_I422: + sub_x = 1; + break; + case DAV1D_PIXEL_LAYOUT_I444: + break; + } + + // Set the right colorspace metadata etc. + switch (dav1d_pic->seq_hdr->pri) { + case DAV1D_COLOR_PRI_UNKNOWN: image->color.primaries = PL_COLOR_PRIM_UNKNOWN; break; + case DAV1D_COLOR_PRI_BT709: image->color.primaries = PL_COLOR_PRIM_BT_709; break; + case DAV1D_COLOR_PRI_BT470M: image->color.primaries = PL_COLOR_PRIM_BT_470M; break; + case DAV1D_COLOR_PRI_BT470BG: image->color.primaries = PL_COLOR_PRIM_BT_601_625; break; + case DAV1D_COLOR_PRI_BT601: image->color.primaries = PL_COLOR_PRIM_BT_601_625; break; + case DAV1D_COLOR_PRI_BT2020: image->color.primaries = PL_COLOR_PRIM_BT_2020; break; + + case DAV1D_COLOR_PRI_XYZ: + // Handled below + assert(dav1d_pic->seq_hdr->mtrx == DAV1D_MC_IDENTITY); + break; + + default: + printf("warning: unknown dav1d color primaries %d.. ignoring, picture " + "may be very incorrect\n", dav1d_pic->seq_hdr->pri); + break; + } + + switch (dav1d_pic->seq_hdr->trc) { + case DAV1D_TRC_BT709: + case DAV1D_TRC_BT470M: + case DAV1D_TRC_BT470BG: + case DAV1D_TRC_BT601: + case DAV1D_TRC_SMPTE240: + case DAV1D_TRC_BT2020_10BIT: + case DAV1D_TRC_BT2020_12BIT: + // These all map to the effective "SDR" CRT-based EOTF, BT.1886 + image->color.transfer = PL_COLOR_TRC_BT_1886; + break; + + case DAV1D_TRC_UNKNOWN: image->color.transfer = PL_COLOR_TRC_UNKNOWN; break; + case DAV1D_TRC_LINEAR: image->color.transfer = PL_COLOR_TRC_LINEAR; break; + case DAV1D_TRC_SRGB: image->color.transfer = PL_COLOR_TRC_SRGB; break; + case DAV1D_TRC_SMPTE2084: image->color.transfer = PL_COLOR_TRC_PQ; break; + case DAV1D_TRC_HLG: image->color.transfer = PL_COLOR_TRC_HLG; break; + + default: + printf("warning: unknown dav1d color transfer %d.. ignoring, picture " + "may be very incorrect\n", dav1d_pic->seq_hdr->trc); + break; + } + + switch (dav1d_pic->seq_hdr->mtrx) { + case DAV1D_MC_IDENTITY: + // This is going to be either RGB or XYZ + if (dav1d_pic->seq_hdr->pri == DAV1D_COLOR_PRI_XYZ) { + image->repr.sys = PL_COLOR_SYSTEM_XYZ; + } else { + image->repr.sys = PL_COLOR_SYSTEM_RGB; + } + break; + + case DAV1D_MC_UNKNOWN: + // PL_COLOR_SYSTEM_UNKNOWN maps to RGB, so hard-code this one + image->repr.sys = pl_color_system_guess_ycbcr(width, height); + break; + + case DAV1D_MC_BT709: image->repr.sys = PL_COLOR_SYSTEM_BT_709; break; + case DAV1D_MC_BT601: image->repr.sys = PL_COLOR_SYSTEM_BT_601; break; + case DAV1D_MC_SMPTE240: image->repr.sys = PL_COLOR_SYSTEM_SMPTE_240M; break; + case DAV1D_MC_SMPTE_YCGCO: image->repr.sys = PL_COLOR_SYSTEM_YCGCO; break; + case DAV1D_MC_BT2020_NCL: image->repr.sys = PL_COLOR_SYSTEM_BT_2020_NC; break; + case DAV1D_MC_BT2020_CL: image->repr.sys = PL_COLOR_SYSTEM_BT_2020_C; break; + + case DAV1D_MC_ICTCP: + // This one is split up based on the actual HDR curve in use + if (dav1d_pic->seq_hdr->trc == DAV1D_TRC_HLG) { + image->repr.sys = PL_COLOR_SYSTEM_BT_2100_HLG; + } else { + image->repr.sys = PL_COLOR_SYSTEM_BT_2100_PQ; + } + break; + + default: + printf("warning: unknown dav1d color matrix %d.. ignoring, picture " + "may be very incorrect\n", dav1d_pic->seq_hdr->mtrx); + break; + } + + if (dav1d_pic->seq_hdr->color_range) { + image->repr.levels = PL_COLOR_LEVELS_PC; + } else { + image->repr.levels = PL_COLOR_LEVELS_TV; + } + + switch (dav1d_pic->seq_hdr->chr) { + case DAV1D_CHR_UNKNOWN: chroma_loc = PL_CHROMA_UNKNOWN; break; + case DAV1D_CHR_VERTICAL: chroma_loc = PL_CHROMA_LEFT; break; + case DAV1D_CHR_COLOCATED: chroma_loc = PL_CHROMA_TOP_LEFT; break; + } + +#if PL_API_VER >= 63 + if (settings->gpugrain && dav1d_pic->frame_hdr->film_grain.present) { + Dav1dFilmGrainData *src = &dav1d_pic->frame_hdr->film_grain.data; + struct pl_av1_grain_data *dst = &image->av1_grain; + *dst = (struct pl_av1_grain_data) { + .grain_seed = src->seed, + .num_points_y = src->num_y_points, + .chroma_scaling_from_luma = src->chroma_scaling_from_luma, + .num_points_uv = { src->num_uv_points[0], src->num_uv_points[1] }, + .scaling_shift = src->scaling_shift, + .ar_coeff_lag = src->ar_coeff_lag, + .ar_coeff_shift = src->ar_coeff_shift, + .grain_scale_shift = src->grain_scale_shift, + .uv_mult = { src->uv_mult[0], src->uv_mult[1] }, + .uv_mult_luma = { src->uv_luma_mult[0], src->uv_luma_mult[1] }, + .uv_offset = { src->uv_offset[0], src->uv_offset[1] }, + .overlap = src->overlap_flag, + }; + + assert(sizeof(dst->points_y) == sizeof(src->y_points)); + assert(sizeof(dst->points_uv) == sizeof(src->uv_points)); + assert(sizeof(dst->ar_coeffs_y) == sizeof(src->ar_coeffs_y)); + memcpy(dst->points_y, src->y_points, sizeof(src->y_points)); + memcpy(dst->points_uv, src->uv_points, sizeof(src->uv_points)); + memcpy(dst->ar_coeffs_y, src->ar_coeffs_y, sizeof(src->ar_coeffs_y)); + + // this one has different row sizes for alignment + for (int c = 0; c < 2; c++) { + for (int i = 0; i < 25; i++) + dst->ar_coeffs_uv[c][i] = src->ar_coeffs_uv[c][i]; + } + } +#endif + + // Upload the actual planes + struct pl_plane_data data[3] = { + { + // Y plane + .type = PL_FMT_UNORM, + .width = width, + .height = height, + .pixel_stride = bytes, + .row_stride = dav1d_pic->stride[0], + .component_size = {bytes * 8}, + .component_map = {0}, + }, { + // U plane + .type = PL_FMT_UNORM, + .width = width >> sub_x, + .height = height >> sub_y, + .pixel_stride = bytes, + .row_stride = dav1d_pic->stride[1], + .component_size = {bytes * 8}, + .component_map = {1}, + }, { + // V plane + .type = PL_FMT_UNORM, + .width = width >> sub_x, + .height = height >> sub_y, + .pixel_stride = bytes, + .row_stride = dav1d_pic->stride[1], + .component_size = {bytes * 8}, + .component_map = {2}, + }, + }; + + bool ok = true; + + for (int i = 0; i < image->num_planes; i++) { + if (settings->zerocopy) { + const struct pl_buf *buf = dav1d_pic->allocator_data; + assert(buf); + data[i].buf = buf; + data[i].buf_offset = (uintptr_t) dav1d_pic->data[i] - (uintptr_t) buf->data; + } else { + data[i].pixels = dav1d_pic->data[i]; + } + + ok &= pl_upload_plane(rd_priv_ctx->gpu, &image->planes[i], &rd_priv_ctx->plane_tex[i], &data[i]); + } + + // Apply the correct chroma plane shift. This has to be done after pl_upload_plane +#if PL_API_VER >= 67 + pl_image_set_chroma_location(image, chroma_loc); +#else + pl_chroma_location_offset(chroma_loc, &image->planes[1].shift_x, &image->planes[1].shift_y); + pl_chroma_location_offset(chroma_loc, &image->planes[2].shift_x, &image->planes[2].shift_y); +#endif + + if (!ok) { + fprintf(stderr, "Failed uploading planes!\n"); + *image = (struct pl_image) {0}; + } + + SDL_UnlockMutex(rd_priv_ctx->lock); + return !ok; +} + +// Align to power of 2 +#define ALIGN2(x, align) (((x) + (align) - 1) & ~((align) - 1)) + +static int placebo_alloc_pic(Dav1dPicture *const p, void *cookie) +{ + Dav1dPlayRendererPrivateContext *rd_priv_ctx = cookie; + assert(rd_priv_ctx != NULL); + SDL_LockMutex(rd_priv_ctx->lock); + + const struct pl_gpu *gpu = rd_priv_ctx->gpu; + int ret = DAV1D_ERR(ENOMEM); + + // Copied from dav1d_default_picture_alloc + const int hbd = p->p.bpc > 8; + const int aligned_w = ALIGN2(p->p.w, 128); + const int aligned_h = ALIGN2(p->p.h, 128); + const int has_chroma = p->p.layout != DAV1D_PIXEL_LAYOUT_I400; + const int ss_ver = p->p.layout == DAV1D_PIXEL_LAYOUT_I420; + const int ss_hor = p->p.layout != DAV1D_PIXEL_LAYOUT_I444; + p->stride[0] = aligned_w << hbd; + p->stride[1] = has_chroma ? (aligned_w >> ss_hor) << hbd : 0; + + // Align strides up to multiples of the GPU performance hints + p->stride[0] = ALIGN2(p->stride[0], gpu->limits.align_tex_xfer_stride); + p->stride[1] = ALIGN2(p->stride[1], gpu->limits.align_tex_xfer_stride); + + // Aligning offsets to 4 also implicity aligns to the texel size (1 or 2) + size_t off_align = ALIGN2(gpu->limits.align_tex_xfer_offset, 4); + const size_t y_sz = ALIGN2(p->stride[0] * aligned_h, off_align); + const size_t uv_sz = ALIGN2(p->stride[1] * (aligned_h >> ss_ver), off_align); + + // The extra DAV1D_PICTURE_ALIGNMENTs are to brute force plane alignment, + // even in the case that the driver gives us insane alignments + const size_t pic_size = y_sz + 2 * uv_sz; + const size_t total_size = pic_size + DAV1D_PICTURE_ALIGNMENT * 4; + + // Validate size limitations + if (total_size > gpu->limits.max_xfer_size) { + printf("alloc of %zu bytes exceeds limits\n", total_size); + goto err; + } + + const struct pl_buf *buf = pl_buf_create(gpu, &(struct pl_buf_params) { + .type = PL_BUF_TEX_TRANSFER, + .host_mapped = true, + .size = total_size, + .memory_type = PL_BUF_MEM_HOST, + .user_data = p, + }); + + if (!buf) { + printf("alloc of GPU mapped buffer failed\n"); + goto err; + } + + assert(buf->data); + uintptr_t base = (uintptr_t) buf->data, data[3]; + data[0] = ALIGN2(base, DAV1D_PICTURE_ALIGNMENT); + data[1] = ALIGN2(data[0] + y_sz, DAV1D_PICTURE_ALIGNMENT); + data[2] = ALIGN2(data[1] + uv_sz, DAV1D_PICTURE_ALIGNMENT); + + // Sanity check offset alignment for the sake of debugging + if (data[0] - base != ALIGN2(data[0] - base, off_align) || + data[1] - base != ALIGN2(data[1] - base, off_align) || + data[2] - base != ALIGN2(data[2] - base, off_align)) + { + printf("GPU buffer horribly misaligned, expect slowdown!\n"); + } + + p->allocator_data = (void *) buf; + p->data[0] = (void *) data[0]; + p->data[1] = (void *) data[1]; + p->data[2] = (void *) data[2]; + ret = 0; + + // fall through +err: + SDL_UnlockMutex(rd_priv_ctx->lock); + return ret; +} + +static void placebo_release_pic(Dav1dPicture *pic, void *cookie) +{ + Dav1dPlayRendererPrivateContext *rd_priv_ctx = cookie; + assert(rd_priv_ctx != NULL); + assert(pic->allocator_data); + + SDL_LockMutex(rd_priv_ctx->lock); + const struct pl_gpu *gpu = rd_priv_ctx->gpu; + pl_buf_destroy(gpu, (const struct pl_buf **) &pic->allocator_data); + SDL_UnlockMutex(rd_priv_ctx->lock); +} + +#ifdef HAVE_PLACEBO_VULKAN +const Dav1dPlayRenderInfo rdr_placebo_vk = { + .name = "placebo-vk", + .create_renderer = placebo_renderer_create_vk, + .destroy_renderer = placebo_renderer_destroy, + .render = placebo_render, + .update_frame = placebo_upload_image, + .alloc_pic = placebo_alloc_pic, + .release_pic = placebo_release_pic, + +# if PL_API_VER >= 63 + .supports_gpu_grain = 1, +# endif +}; +#else +const Dav1dPlayRenderInfo rdr_placebo_vk = { NULL }; +#endif + +#ifdef HAVE_PLACEBO_OPENGL +const Dav1dPlayRenderInfo rdr_placebo_gl = { + .name = "placebo-gl", + .create_renderer = placebo_renderer_create_gl, + .destroy_renderer = placebo_renderer_destroy, + .render = placebo_render, + .update_frame = placebo_upload_image, + .alloc_pic = placebo_alloc_pic, + .release_pic = placebo_release_pic, + +# if PL_API_VER >= 63 + .supports_gpu_grain = 1, +# endif +}; +#else +const Dav1dPlayRenderInfo rdr_placebo_gl = { NULL }; +#endif + +#else +const Dav1dPlayRenderInfo rdr_placebo_vk = { NULL }; +const Dav1dPlayRenderInfo rdr_placebo_gl = { NULL }; +#endif diff --git a/third_party/dav1d/examples/dp_renderer_sdl.c b/third_party/dav1d/examples/dp_renderer_sdl.c new file mode 100644 --- /dev/null +++ b/third_party/dav1d/examples/dp_renderer_sdl.c @@ -0,0 +1,164 @@ +/* + * Copyright © 2020, VideoLAN and dav1d authors + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "dp_renderer.h" + +#include + +/** + * Renderer context for SDL + */ +typedef struct renderer_priv_ctx +{ + // SDL window + SDL_Window *win; + // SDL renderer + SDL_Renderer *renderer; + // Lock protecting access to the texture + SDL_mutex *lock; + // Texture to render + SDL_Texture *tex; +} Dav1dPlayRendererPrivateContext; + +static void *sdl_renderer_create() +{ + SDL_Window *win = dp_create_sdl_window(0); + if (win == NULL) + return NULL; + + // Alloc + Dav1dPlayRendererPrivateContext *rd_priv_ctx = malloc(sizeof(Dav1dPlayRendererPrivateContext)); + if (rd_priv_ctx == NULL) { + return NULL; + } + rd_priv_ctx->win = win; + + // Create renderer + rd_priv_ctx->renderer = SDL_CreateRenderer(win, -1, SDL_RENDERER_ACCELERATED); + // Set scale quality + SDL_SetHint(SDL_HINT_RENDER_SCALE_QUALITY, "linear"); + + // Create Mutex + rd_priv_ctx->lock = SDL_CreateMutex(); + if (rd_priv_ctx->lock == NULL) { + fprintf(stderr, "SDL_CreateMutex failed: %s\n", SDL_GetError()); + free(rd_priv_ctx); + return NULL; + } + + rd_priv_ctx->tex = NULL; + + return rd_priv_ctx; +} + +static void sdl_renderer_destroy(void *cookie) +{ + Dav1dPlayRendererPrivateContext *rd_priv_ctx = cookie; + assert(rd_priv_ctx != NULL); + + SDL_DestroyRenderer(rd_priv_ctx->renderer); + SDL_DestroyMutex(rd_priv_ctx->lock); + free(rd_priv_ctx); +} + +static void sdl_render(void *cookie, const Dav1dPlaySettings *settings) +{ + Dav1dPlayRendererPrivateContext *rd_priv_ctx = cookie; + assert(rd_priv_ctx != NULL); + + SDL_LockMutex(rd_priv_ctx->lock); + + if (rd_priv_ctx->tex == NULL) { + SDL_UnlockMutex(rd_priv_ctx->lock); + return; + } + + // Display the frame + SDL_RenderClear(rd_priv_ctx->renderer); + SDL_RenderCopy(rd_priv_ctx->renderer, rd_priv_ctx->tex, NULL, NULL); + SDL_RenderPresent(rd_priv_ctx->renderer); + + SDL_UnlockMutex(rd_priv_ctx->lock); +} + +static int sdl_update_texture(void *cookie, Dav1dPicture *dav1d_pic, + const Dav1dPlaySettings *settings) +{ + Dav1dPlayRendererPrivateContext *rd_priv_ctx = cookie; + assert(rd_priv_ctx != NULL); + + SDL_LockMutex(rd_priv_ctx->lock); + + if (dav1d_pic == NULL) { + rd_priv_ctx->tex = NULL; + SDL_UnlockMutex(rd_priv_ctx->lock); + return 0; + } + + int width = dav1d_pic->p.w; + int height = dav1d_pic->p.h; + int tex_w = width; + int tex_h = height; + + enum Dav1dPixelLayout dav1d_layout = dav1d_pic->p.layout; + + if (DAV1D_PIXEL_LAYOUT_I420 != dav1d_layout || dav1d_pic->p.bpc != 8) { + fprintf(stderr, "Unsupported pixel format, only 8bit 420 supported so far.\n"); + exit(50); + } + + SDL_Texture *texture = rd_priv_ctx->tex; + if (texture != NULL) { + SDL_QueryTexture(texture, NULL, NULL, &tex_w, &tex_h); + if (tex_w != width || tex_h != height) { + SDL_DestroyTexture(texture); + texture = NULL; + } + } + + if (texture == NULL) { + texture = SDL_CreateTexture(rd_priv_ctx->renderer, SDL_PIXELFORMAT_IYUV, + SDL_TEXTUREACCESS_STREAMING, width, height); + } + + SDL_UpdateYUVTexture(texture, NULL, + dav1d_pic->data[0], (int)dav1d_pic->stride[0], // Y + dav1d_pic->data[1], (int)dav1d_pic->stride[1], // U + dav1d_pic->data[2], (int)dav1d_pic->stride[1] // V + ); + + rd_priv_ctx->tex = texture; + SDL_UnlockMutex(rd_priv_ctx->lock); + return 0; +} + +const Dav1dPlayRenderInfo rdr_sdl = { + .name = "sdl", + .create_renderer = sdl_renderer_create, + .destroy_renderer = sdl_renderer_destroy, + .render = sdl_render, + .update_frame = sdl_update_texture +}; diff --git a/third_party/dav1d/examples/meson.build b/third_party/dav1d/examples/meson.build --- a/third_party/dav1d/examples/meson.build +++ b/third_party/dav1d/examples/meson.build @@ -33,30 +33,42 @@ # dav1d player sources dav1dplay_sources = files( 'dav1dplay.c', + 'dp_fifo.c', + 'dp_renderer_placebo.c', + 'dp_renderer_sdl.c', ) -sdl2_dependency = dependency('sdl2', version: '>= 2.0.1', required: false) +sdl2_dependency = dependency('sdl2', version: '>= 2.0.1', required: true) if sdl2_dependency.found() + dav1dplay_deps = [sdl2_dependency] + dav1dplay_cflags = [] + placebo_dependency = dependency('libplacebo', version: '>= 1.18.0', required: false) - vulkan_dependency = dependency('vulkan', required: false) - sdl_has_vulkan = cc.has_header('SDL_vulkan.h', dependencies: [sdl2_dependency]) - cflag_placebo = [] - deps_placebo = [] - if placebo_dependency.found() and vulkan_dependency.found() and sdl_has_vulkan - cflag_placebo += '-DHAVE_PLACEBO_VULKAN=1' - deps_placebo = [vulkan_dependency, placebo_dependency] + + if placebo_dependency.found() + dav1dplay_deps += placebo_dependency + dav1dplay_cflags += '-DHAVE_PLACEBO' + + # If libplacebo is found, we might be able to use Vulkan + # with it, in which case we need the Vulkan library too. + vulkan_dependency = dependency('vulkan', required: false) + if vulkan_dependency.found() + dav1dplay_deps += vulkan_dependency + dav1dplay_cflags += '-DHAVE_VULKAN' + endif endif + dav1dplay = executable('dav1dplay', dav1dplay_sources, rev_target, link_with : [libdav1d, dav1d_input_objs], include_directories : [dav1d_inc_dirs], - dependencies : [getopt_dependency, sdl2_dependency, deps_placebo], + dependencies : [getopt_dependency, dav1dplay_deps], install : true, - c_args : cflag_placebo, + c_args : dav1dplay_cflags, ) endif diff --git a/third_party/dav1d/gcovr.cfg b/third_party/dav1d/gcovr.cfg new file mode 100644 --- /dev/null +++ b/third_party/dav1d/gcovr.cfg @@ -0,0 +1,3 @@ +exclude = .*/tests/.* +exclude = .*/tools/.* +exclude = .*/include/common/dump.h diff --git a/third_party/dav1d/include/common/attributes.h b/third_party/dav1d/include/common/attributes.h --- a/third_party/dav1d/include/common/attributes.h +++ b/third_party/dav1d/include/common/attributes.h @@ -157,6 +157,10 @@ static inline int clzll(const unsigned long long mask) { return __builtin_clzll(mask); } #endif /* !_MSC_VER */ +#ifndef __has_feature +#define __has_feature(x) 0 +#endif + #endif /* DAV1D_COMMON_ATTRIBUTES_H */ diff --git a/third_party/dav1d/include/common/mem.h b/third_party/dav1d/include/common/mem.h --- a/third_party/dav1d/include/common/mem.h +++ b/third_party/dav1d/include/common/mem.h @@ -35,17 +35,17 @@ #endif #include "common/attributes.h" /* - * Allocate 32-byte aligned memory. The return value can be released - * by calling the standard free() function. + * Allocate align-byte aligned memory. The return value can be released + * by calling the dav1d_free_aligned() function. */ static inline void *dav1d_alloc_aligned(size_t sz, size_t align) { + assert(!(align & (align - 1))); #ifdef HAVE_POSIX_MEMALIGN void *ptr; - assert(!(align & (align - 1))); if (posix_memalign(&ptr, align, sz)) return NULL; return ptr; #elif defined(HAVE_ALIGNED_MALLOC) return _aligned_malloc(sz, align); #elif defined(HAVE_MEMALIGN) diff --git a/third_party/dav1d/include/dav1d/dav1d.h b/third_party/dav1d/include/dav1d/dav1d.h --- a/third_party/dav1d/include/dav1d/dav1d.h +++ b/third_party/dav1d/include/dav1d/dav1d.h @@ -47,11 +47,11 @@ #define DAV1D_MAX_TILE_THREADS 64 typedef struct Dav1dLogger { void *cookie; ///< Custom data to pass to the callback. /** - * Logger callback. Default prints to stderr. May be NULL to disable logging. + * Logger callback. May be NULL to disable logging. * * @param cookie Custom pointer passed to all calls. * @param format The vprintf compatible format string. * @param ap List of arguments referenced by the format string. */ @@ -64,12 +64,12 @@ int apply_grain; int operating_point; ///< select an operating point for scalable AV1 bitstreams (0 - 31) int all_layers; ///< output all spatial layers of a scalable AV1 biststream unsigned frame_size_limit; ///< maximum frame size, in pixels (0 = unlimited) uint8_t reserved[32]; ///< reserved for future use - Dav1dPicAllocator allocator; - Dav1dLogger logger; + Dav1dPicAllocator allocator; ///< Picture allocator callback. + Dav1dLogger logger; ///< Logger callback. } Dav1dSettings; /** * Get library version. */ diff --git a/third_party/dav1d/include/dav1d/headers.h b/third_party/dav1d/include/dav1d/headers.h --- a/third_party/dav1d/include/dav1d/headers.h +++ b/third_party/dav1d/include/dav1d/headers.h @@ -334,11 +334,12 @@ int present, update; } film_grain; ///< film grain parameters enum Dav1dFrameType frame_type; ///< type of the picture int width[2 /* { coded_width, superresolution_upscaled_width } */], height; int frame_offset; ///< frame number - int temporal_id, spatial_id; ///< spatial and temporal id of the frame for SVC + int temporal_id; ///< temporal id of the frame for SVC + int spatial_id; ///< spatial id of the frame for SVC int show_existing_frame; int existing_frame_idx; int frame_id; int frame_presentation_delay; diff --git a/third_party/dav1d/include/dav1d/picture.h b/third_party/dav1d/include/dav1d/picture.h --- a/third_party/dav1d/include/dav1d/picture.h +++ b/third_party/dav1d/include/dav1d/picture.h @@ -82,12 +82,15 @@ */ Dav1dITUTT35 *itut_t35; uintptr_t reserved[4]; ///< reserved for future use - struct Dav1dRef *frame_hdr_ref, *seq_hdr_ref; ///< Frame parameter allocation origins - struct Dav1dRef *content_light_ref, *mastering_display_ref, *itut_t35_ref; ///< Metadata allocation origins + struct Dav1dRef *frame_hdr_ref; ///< Dav1dFrameHeader allocation origin + struct Dav1dRef *seq_hdr_ref; ///< Dav1dSequenceHeader allocation origin + struct Dav1dRef *content_light_ref; ///< Dav1dContentLightLevel allocation origin + struct Dav1dRef *mastering_display_ref; ///< Dav1dMasteringDisplay allocation origin + struct Dav1dRef *itut_t35_ref; ///< Dav1dITUTT35 allocation origin uintptr_t reserved_ref[4]; ///< reserved for future use struct Dav1dRef *ref; ///< Frame data allocation origin void *allocator_data; ///< pointer managed by the allocator } Dav1dPicture; diff --git a/third_party/dav1d/meson.build b/third_party/dav1d/meson.build --- a/third_party/dav1d/meson.build +++ b/third_party/dav1d/meson.build @@ -1,6 +1,6 @@ -# Copyright © 2018-2019, VideoLAN and dav1d authors +# Copyright © 2018-2020, VideoLAN and dav1d authors # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # @@ -21,18 +21,18 @@ # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. project('dav1d', ['c'], - version: '0.6.0', + version: '0.7.1', default_options: ['c_std=c99', 'warning_level=2', 'buildtype=release', 'b_ndebug=if-release'], meson_version: '>= 0.47.0') -dav1d_soname_version = '4.0.0' +dav1d_soname_version = '4.0.2' dav1d_api_version_array = dav1d_soname_version.split('.') dav1d_api_version_major = dav1d_api_version_array[0] dav1d_api_version_minor = dav1d_api_version_array[1] dav1d_api_version_revision = dav1d_api_version_array[2] @@ -131,10 +131,25 @@ endif cdata.set('HAVE_CLOCK_GETTIME', 1) endif endif +# check for fseeko on android. It is not always available if _FILE_OFFSET_BITS is defined to 64 +have_fseeko = true +if host_machine.system() == 'android' + if not cc.has_function('fseeko', prefix : '#include ', args : test_args) + if cc.has_function('fseeko', prefix : '#include ', args : test_args + ['-U_FILE_OFFSET_BITS']) + warning('Files larger than 2 gigabytes might not be supported in the dav1d CLI tool.') + add_project_arguments('-U_FILE_OFFSET_BITS', language: 'c') + elif get_option('enable_tools') + error('dav1d CLI tool needs fseeko()') + else + have_fseeko = false + endif + endif +endif + libdl_dependency = [] if host_machine.system() == 'linux' libdl_dependency = cc.find_library('dl', required : false) if cc.has_function('dlsym', prefix : '#include ', args : test_args, dependencies : libdl_dependency) cdata.set('HAVE_DLSYM', 1) @@ -179,14 +194,14 @@ ) else getopt_dependency = [] endif -if cc.has_function('posix_memalign', prefix : '#include ', args : test_args) +if cc.has_function('_aligned_malloc', prefix : '#include ', args : test_args) + cdata.set('HAVE_ALIGNED_MALLOC', 1) +elif cc.has_function('posix_memalign', prefix : '#include ', args : test_args) cdata.set('HAVE_POSIX_MEMALIGN', 1) -elif cc.has_function('_aligned_malloc', prefix : '#include ', args : test_args) - cdata.set('HAVE_ALIGNED_MALLOC', 1) elif cc.has_function('memalign', prefix : '#include ', args : test_args) cdata.set('HAVE_MEMALIGN', 1) endif if (host_machine.cpu_family() == 'aarch64' or @@ -296,12 +311,12 @@ endif cdata_asm.set('STACK_ALIGNMENT', stack_alignment) cdata.set('STACK_ALIGNMENT', stack_alignment) endif -cdata.set10('ARCH_AARCH64', host_machine.cpu_family() == 'aarch64') -cdata.set10('ARCH_ARM', host_machine.cpu_family().startswith('arm')) +cdata.set10('ARCH_AARCH64', host_machine.cpu_family() == 'aarch64' or host_machine.cpu() == 'arm64') +cdata.set10('ARCH_ARM', host_machine.cpu_family().startswith('arm') and host_machine.cpu() != 'arm64') if (is_asm_enabled and (host_machine.cpu_family() == 'aarch64' or host_machine.cpu_family().startswith('arm'))) as_func_code = '''__asm__ ( @@ -345,24 +360,15 @@ if cc.symbols_have_underscore_prefix() cdata.set10('PREFIX', true) cdata_asm.set10('PREFIX', true) endif -# Generate config.h -config_h_target = configure_file(output: 'config.h', configuration: cdata) - - - # # ASM specific stuff # if is_asm_enabled and host_machine.cpu_family().startswith('x86') - # Generate config.asm - config_asm_target = configure_file(output: 'config.asm', output_format: 'nasm', configuration: cdata_asm) - - # NASM compiler support nasm = find_program('nasm') # check NASM version @@ -373,18 +379,26 @@ error('failed running nasm to obtain its version') endif out = nasm_r.stdout().strip().split() if out[1].to_lower() == 'version' - if out[2].version_compare('<2.14') - error('nasm 2.14 or later is required, found nasm @0@'.format(out[2])) + if out[2].version_compare('<2.13.02') + error('nasm 2.13.02 or later is required, found nasm @0@'.format(out[2])) + elif out[2].version_compare('<2.14') and get_option('enable_avx512') + error('nasm 2.14 or later is required for AVX-512 asm.\n' + + 'AVX-512 asm can be disabled with \'-Denable_avx512=false\'') endif + cdata.set10('HAVE_AVX512ICL', get_option('enable_avx512')) + cdata_asm.set10('HAVE_AVX512ICL', get_option('enable_avx512')) else error('unexpected nasm version string: @0@'.format(nasm_r.stdout())) endif endif + # Generate config.asm + config_asm_target = configure_file(output: 'config.asm', output_format: 'nasm', configuration: cdata_asm) + if host_machine.system() == 'windows' nasm_format = 'win' elif host_machine.system() == 'darwin' nasm_format = 'macho' else @@ -399,20 +413,24 @@ nasm_gen = generator(nasm, output: '@BASENAME@.obj', depfile: '@BASENAME@.obj.ndep', arguments: [ '-f', nasm_format, - '-I', '@0@/src/'.format(meson.current_source_dir()), + '-I', '@0@/src/'.format(dav1d_src_root), '-I', '@0@/'.format(meson.current_build_dir()), '-MQ', '@OUTPUT@', '-MF', '@DEPFILE@', '@EXTRA_ARGS@', '@INPUT@', '-o', '@OUTPUT@' ]) endif +# Generate config.h +config_h_target = configure_file(output: 'config.h', configuration: cdata) + + # # Include subdir meson.build files # The order is important! diff --git a/third_party/dav1d/meson_options.txt b/third_party/dav1d/meson_options.txt --- a/third_party/dav1d/meson_options.txt +++ b/third_party/dav1d/meson_options.txt @@ -8,10 +8,15 @@ option('enable_asm', type: 'boolean', value: true, description: 'Build asm files, if available') +option('enable_avx512', + type: 'boolean', + value: true, + description: 'Build AVX-512 asm files, requires nasm 2.14') + option('enable_tools', type: 'boolean', value: true, description: 'Build dav1d cli tools') diff --git a/third_party/dav1d/package/crossfiles/aarch64-android.meson b/third_party/dav1d/package/crossfiles/aarch64-android.meson new file mode 100644 --- /dev/null +++ b/third_party/dav1d/package/crossfiles/aarch64-android.meson @@ -0,0 +1,16 @@ +[binaries] +c = 'aarch64-linux-android21-clang' +cpp = 'aarch64-linux-android21-clang++' +ar = 'aarch64-linux-android-ar' +strip = 'aarch64-linux-android-strip' +pkgconfig = 'pkg-config' +windres = 'aarch64-linux-android-windres' + +[properties] +needs_exe_wrapper = true + +[host_machine] +system = 'android' +cpu_family = 'aarch64' +endian = 'little' +cpu = 'aarch64' diff --git a/third_party/dav1d/package/crossfiles/arm-android.meson b/third_party/dav1d/package/crossfiles/arm-android.meson new file mode 100644 --- /dev/null +++ b/third_party/dav1d/package/crossfiles/arm-android.meson @@ -0,0 +1,16 @@ +[binaries] +c = 'armv7a-linux-androideabi16-clang' +cpp = 'armv7a-linux-androideabi16-clang++' +ar = 'arm-linux-androideabi-ar' +strip = 'arm-linux-androideabi-strip' +pkgconfig = 'pkg-config' +windres = 'arm-linux-androideabi-windres' + +[properties] +needs_exe_wrapper = true + +[host_machine] +system = 'android' +cpu_family = 'arm' +endian = 'little' +cpu = 'arm' diff --git a/third_party/dav1d/package/crossfiles/i686-linux32.meson b/third_party/dav1d/package/crossfiles/i686-linux32.meson new file mode 100644 --- /dev/null +++ b/third_party/dav1d/package/crossfiles/i686-linux32.meson @@ -0,0 +1,15 @@ +[binaries] +c = 'gcc' +cpp = 'g++' +ar = 'ar' +strip = 'strip' + +[properties] +c_link_args = ['-m32', '-Wl,-z,text'] +c_args = ['-m32'] + +[host_machine] +system = 'linux' +cpu_family = 'x86' +cpu = 'i686' +endian = 'little' diff --git a/third_party/dav1d/package/crossfiles/i686-w64-mingw32.meson b/third_party/dav1d/package/crossfiles/i686-w64-mingw32.meson new file mode 100644 --- /dev/null +++ b/third_party/dav1d/package/crossfiles/i686-w64-mingw32.meson @@ -0,0 +1,17 @@ +[binaries] +c = 'i686-w64-mingw32-gcc' +cpp = 'i686-w64-mingw32-g++' +ar = 'i686-w64-mingw32-ar' +strip = 'i686-w64-mingw32-strip' +windres = 'i686-w64-mingw32-windres' +exe_wrapper = ['wine'] + +[properties] +c_link_args = ['-static-libgcc'] +needs_exe_wrapper = true + +[host_machine] +system = 'windows' +cpu_family = 'x86' +cpu = 'i686' +endian = 'little' diff --git a/third_party/dav1d/package/crossfiles/x86_64-w64-mingw32.meson b/third_party/dav1d/package/crossfiles/x86_64-w64-mingw32.meson new file mode 100644 --- /dev/null +++ b/third_party/dav1d/package/crossfiles/x86_64-w64-mingw32.meson @@ -0,0 +1,17 @@ +[binaries] +c = 'x86_64-w64-mingw32-gcc' +cpp = 'x86_64-w64-mingw32-g++' +ar = 'x86_64-w64-mingw32-ar' +strip = 'x86_64-w64-mingw32-strip' +windres = 'x86_64-w64-mingw32-windres' +exe_wrapper = ['wine'] + +[properties] +c_link_args = ['-static-libgcc'] +needs_exe_wrapper = true + +[host_machine] +system = 'windows' +cpu_family = 'x86_64' +cpu = 'x86_64' +endian = 'little' diff --git a/third_party/dav1d/src/arm/32/ipred.S b/third_party/dav1d/src/arm/32/ipred.S --- a/third_party/dav1d/src/arm/32/ipred.S +++ b/third_party/dav1d/src/arm/32/ipred.S @@ -1,8 +1,8 @@ /* * Copyright © 2018, VideoLAN and dav1d authors - * Copyright © 2019, Martin Storsjo + * Copyright © 2020, Martin Storsjo * Copyright © 2019, B Krishnan Iyer * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -27,15 +27,15 @@ */ #include "src/arm/asm.S" #include "util.S" -// void ipred_dc_128_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_dc_128_neon, export=1 +// void ipred_dc_128_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_128_8bpc_neon, export=1 push {r4, lr} ldr r4, [sp, #8] clz r3, r3 adr r2, L(ipred_dc_128_tbl) sub r3, r3, #25 @@ -105,15 +105,15 @@ vst1.8 {d4, d5, d6, d7}, [r12, :128], r1 bgt 64b pop {r4, pc} endfunc -// void ipred_v_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_v_neon, export=1 +// void ipred_v_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_v_8bpc_neon, export=1 push {r4, lr} ldr lr, [sp, #8] clz r3, r3 adr r4, L(ipred_v_tbl) sub r3, r3, #25 @@ -130,11 +130,11 @@ .word 320f - L(ipred_v_tbl) + CONFIG_THUMB .word 160f - L(ipred_v_tbl) + CONFIG_THUMB .word 80f - L(ipred_v_tbl) + CONFIG_THUMB .word 40f - L(ipred_v_tbl) + CONFIG_THUMB 40: - vld1.32 {d0[0]}, [r2] + vld1.32 {d0[]}, [r2] 4: vst1.32 {d0[0]}, [r0, :32], r1 vst1.32 {d0[0]}, [r12, :32], r1 subs lr, lr, #4 vst1.32 {d0[0]}, [r0, :32], r1 @@ -187,15 +187,15 @@ vst1.8 {d4, d5, d6, d7}, [r12, :128], r1 bgt 64b pop {r4, pc} endfunc -// void ipred_h_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_h_neon, export=1 +// void ipred_h_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_h_8bpc_neon, export=1 push {r4-r5, lr} ldr r4, [sp, #12] clz r3, r3 adr r5, L(ipred_h_tbl) sub r3, r3, #25 @@ -213,20 +213,20 @@ .word 320f - L(ipred_h_tbl) + CONFIG_THUMB .word 160f - L(ipred_h_tbl) + CONFIG_THUMB .word 8f - L(ipred_h_tbl) + CONFIG_THUMB .word 4f - L(ipred_h_tbl) + CONFIG_THUMB 4: - vld4.8 {d0[], d1[], d2[], d3[]}, [r2], lr + vld4.8 {d0[], d1[], d2[], d3[]}, [r2, :32], lr vst1.32 {d3[0]}, [r0, :32], r1 vst1.32 {d2[0]}, [r12, :32], r1 subs r4, r4, #4 vst1.32 {d1[0]}, [r0, :32], r1 vst1.32 {d0[0]}, [r12, :32], r1 bgt 4b pop {r4-r5, pc} 8: - vld4.8 {d0[], d1[], d2[], d3[]}, [r2], lr + vld4.8 {d0[], d1[], d2[], d3[]}, [r2, :32], lr vst1.8 {d3}, [r0, :64], r1 vst1.8 {d2}, [r12, :64], r1 subs r4, r4, #4 vst1.8 {d1}, [r0, :64], r1 vst1.8 {d0}, [r12, :64], r1 @@ -295,15 +295,15 @@ vst1.8 {q3}, [r12, :128], r1 bgt 64b pop {r4-r5, pc} endfunc -// void ipred_dc_top_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_dc_top_neon, export=1 +// void ipred_dc_top_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_top_8bpc_neon, export=1 push {r4-r5, lr} ldr r4, [sp, #12] clz r3, r3 adr r5, L(ipred_dc_top_tbl) sub r3, r3, #25 @@ -416,15 +416,15 @@ vst1.8 {d4, d5, d6, d7}, [r12, :128], r1 bgt 64b pop {r4-r5, pc} endfunc -// void ipred_dc_left_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_dc_left_neon, export=1 +// void ipred_dc_left_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_left_8bpc_neon, export=1 push {r4-r5, lr} ldr r4, [sp, #12] sub r2, r2, r4 clz r3, r3 clz lr, r4 @@ -451,11 +451,11 @@ .word L(ipred_dc_left_w16) - L(ipred_dc_left_tbl) + CONFIG_THUMB .word L(ipred_dc_left_w8) - L(ipred_dc_left_tbl) + CONFIG_THUMB .word L(ipred_dc_left_w4) - L(ipred_dc_left_tbl) + CONFIG_THUMB L(ipred_dc_left_h4): - vld1.32 {d0[]}, [r2] + vld1.32 {d0[]}, [r2, :32] vpaddl.u8 d0, d0 vpadd.u16 d0, d0 vrshrn.u16 d0, q0, #2 vdup.8 q0, d0[0] bx r3 @@ -466,11 +466,11 @@ vst1.32 {d0[0]}, [r0, :32], r1 vst1.32 {d0[0]}, [r12, :32], r1 bgt L(ipred_dc_left_w4) pop {r4-r5, pc} L(ipred_dc_left_h8): - vld1.8 {d0}, [r2] + vld1.8 {d0}, [r2, :64] vpaddl.u8 d0, d0 vpadd.u16 d0, d0 vpadd.u16 d0, d0 vrshrn.u16 d0, q0, #3 vdup.8 q0, d0[0] @@ -482,11 +482,11 @@ vst1.8 {d0}, [r0, :64], r1 vst1.8 {d0}, [r12, :64], r1 bgt L(ipred_dc_left_w8) pop {r4-r5, pc} L(ipred_dc_left_h16): - vld1.8 {d0, d1}, [r2] + vld1.8 {d0, d1}, [r2, :128] vaddl.u8 q0, d0, d1 vadd.u16 d0, d0, d1 vpadd.u16 d0, d0 vpadd.u16 d0, d0 vrshrn.u16 d0, q0, #4 @@ -499,11 +499,11 @@ vst1.8 {d0, d1}, [r0, :128], r1 vst1.8 {d0, d1}, [r12, :128], r1 bgt L(ipred_dc_left_w16) pop {r4-r5, pc} L(ipred_dc_left_h32): - vld1.8 {d0, d1, d2, d3}, [r2] + vld1.8 {d0, d1, d2, d3}, [r2, :128] vaddl.u8 q0, d0, d1 vaddl.u8 q1, d2, d3 vadd.u16 q0, q0, q1 vadd.u16 d0, d0, d1 vpadd.u16 d0, d0 @@ -520,12 +520,12 @@ vst1.8 {d0, d1, d2, d3}, [r0, :128], r1 vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 bgt 1b pop {r4-r5, pc} L(ipred_dc_left_h64): - vld1.8 {d0, d1, d2, d3}, [r2]! - vld1.8 {d4, d5, d6, d7}, [r2] + vld1.8 {d0, d1, d2, d3}, [r2, :128]! + vld1.8 {d4, d5, d6, d7}, [r2, :128] vaddl.u8 q0, d0, d1 vaddl.u8 q1, d2, d3 vaddl.u8 q2, d4, d5 vaddl.u8 q3, d6, d7 vadd.u16 q0, q0, q1 @@ -554,23 +554,22 @@ vst1.8 {d4, d5, d6, d7}, [r12, :128], r1 bgt 1b pop {r4-r5, pc} endfunc -// void ipred_dc_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_dc_neon, export=1 +// void ipred_dc_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_8bpc_neon, export=1 push {r4-r6, lr} ldr r4, [sp, #16] sub r2, r2, r4 add lr, r3, r4 // width + height clz r3, r3 clz r12, r4 vdup.16 q15, lr // width + height - mov r6, #0 adr r5, L(ipred_dc_tbl) rbit lr, lr // rbit(width + height) sub r3, r3, #20 // 25 leading bits, minus table offset 5 sub r12, r12, #25 clz lr, lr // ctz(width + height) @@ -597,26 +596,25 @@ .word L(ipred_dc_w16) - L(ipred_dc_tbl) + CONFIG_THUMB .word L(ipred_dc_w8) - L(ipred_dc_tbl) + CONFIG_THUMB .word L(ipred_dc_w4) - L(ipred_dc_tbl) + CONFIG_THUMB L(ipred_dc_h4): - vld1.32 {d0[0]}, [r2]! + vld1.32 {d0[]}, [r2, :32]! vpaddl.u8 d0, d0 vpadd.u16 d0, d0 bx r3 L(ipred_dc_w4): add r2, r2, #1 - vld1.32 {d1[0]}, [r2] - vmov.32 d1[1], r6 + vld1.32 {d1[]}, [r2] vadd.s16 d0, d0, d30 vpaddl.u8 d1, d1 vpadd.u16 d1, d1 - vpadd.u16 d1, d1 cmp r4, #4 vadd.s16 d0, d0, d1 vshl.u16 d0, d0, d28 - beq 1f // h = 8/16 + beq 1f + // h = 8/16 movw lr, #(0x3334/2) movw r5, #(0x5556/2) cmp r4, #16 it ne movne lr, r5 @@ -632,11 +630,11 @@ vst1.32 {d0[0]}, [r12, :32], r1 bgt 2b pop {r4-r6, pc} L(ipred_dc_h8): - vld1.8 {d0}, [r2]! + vld1.8 {d0}, [r2, :64]! vpaddl.u8 d0, d0 vpadd.u16 d0, d0 vpadd.u16 d0, d0 bx r3 L(ipred_dc_w8): @@ -647,17 +645,18 @@ vpadd.u16 d2, d2 vpadd.u16 d2, d2 cmp r4, #8 vadd.s16 d0, d0, d2 vshl.u16 d0, d0, d28 - beq 1f // h = 4/16/32 + beq 1f + // h = 4/16/32 cmp r4, #32 movw lr, #(0x3334/2) movw r5, #(0x5556/2) it ne movne lr, r5 - vdup.16 q12, lr + vdup.16 d24, lr vqdmulh.s16 d0, d0, d24 1: vdup.8 d0, d0[0] 2: vst1.8 {d0}, [r0, :64], r1 @@ -667,11 +666,11 @@ vst1.8 {d0}, [r12, :64], r1 bgt 2b pop {r4-r6, pc} L(ipred_dc_h16): - vld1.8 {d0, d1}, [r2]! + vld1.8 {d0, d1}, [r2, :128]! vaddl.u8 q0, d0, d1 vadd.u16 d0, d0, d1 vpadd.u16 d0, d0 vpadd.u16 d0, d0 bx r3 @@ -684,17 +683,18 @@ vpadd.u16 d2, d2 vpadd.u16 d2, d2 cmp r4, #16 vadd.s16 d0, d0, d2 vshl.u16 d0, d0, d28 - beq 1f // h = 4/8/32/64 + beq 1f + // h = 4/8/32/64 tst r4, #(32+16+8) // 16 added to make a consecutive bitmask movw lr, #(0x3334/2) movw r5, #(0x5556/2) it ne movne lr, r5 - vdup.16 q12, lr + vdup.16 d24, lr vqdmulh.s16 d0, d0, d24 1: vdup.8 q0, d0[0] 2: vst1.8 {d0, d1}, [r0, :128], r1 @@ -704,11 +704,11 @@ vst1.8 {d0, d1}, [r12, :128], r1 bgt 2b pop {r4-r6, pc} L(ipred_dc_h32): - vld1.8 {d0, d1, d2, d3}, [r2]! + vld1.8 {d0, d1, d2, d3}, [r2, :128]! vaddl.u8 q0, d0, d1 vaddl.u8 q1, d2, d3 vadd.u16 q0, q0, q1 vadd.u16 d0, d0, d1 vpadd.u16 d0, d0 @@ -716,29 +716,27 @@ bx r3 L(ipred_dc_w32): add r2, r2, #1 vld1.8 {d2, d3, d4, d5}, [r2] vadd.s16 d0, d0, d30 + vaddl.u8 q1, d2, d3 vaddl.u8 q2, d4, d5 - vadd.u16 d4, d4, d5 - vaddl.u8 q1, d2, d3 + vadd.u16 q1, q1, q2 vadd.u16 d2, d2, d3 - vpadd.u16 d4, d4 vpadd.u16 d2, d2 - vpadd.u16 d4, d4 vpadd.u16 d2, d2 cmp r4, #32 - vadd.s16 d0, d0, d4 vadd.s16 d0, d0, d2 vshl.u16 d4, d0, d28 - beq 1f // h = 8/16/64 + beq 1f + // h = 8/16/64 cmp r4, #8 movw lr, #(0x3334/2) movw r5, #(0x5556/2) it ne movne lr, r5 - vdup.16 q12, lr + vdup.16 d24, lr vqdmulh.s16 d4, d4, d24 1: vdup.8 q0, d4[0] vdup.8 q1, d4[0] 2: @@ -749,13 +747,13 @@ vst1.8 {d0, d1, d2, d3}, [r12, :128], r1 bgt 2b pop {r4-r6, pc} L(ipred_dc_h64): - vld1.8 {d0, d1, d2, d3}, [r2]! + vld1.8 {d0, d1, d2, d3}, [r2, :128]! vaddl.u8 q0, d0, d1 - vld1.8 {d4, d5, d6, d7}, [r2]! + vld1.8 {d4, d5, d6, d7}, [r2, :128]! vaddl.u8 q1, d2, d3 vaddl.u8 q2, d4, d5 vaddl.u8 q3, d6, d7 vadd.u16 q0, q0, q1 vadd.u16 q1, q2, q3 @@ -763,14 +761,10 @@ vadd.u16 d0, d0, d1 vpadd.u16 d0, d0 vpadd.u16 d0, d0 bx r3 L(ipred_dc_w64): - vmov.8 q1, q0 - vmov.8 q2, q0 - vmov.8 q3, q0 -2: add r2, r2, #1 vld1.8 {d2, d3, d4, d5}, [r2]! vadd.s16 d0, d0, d30 vaddl.u8 q2, d4, d5 vaddl.u8 q1, d2, d3 @@ -821,5 +815,2145 @@ vst1.8 {d4, d5, d6, d7}, [r12, :128], r1 bgt 2b pop {r4-r6, pc} endfunc +// void ipred_paeth_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_paeth_8bpc_neon, export=1 + push {r4-r8, lr} + ldr r4, [sp, #24] + clz lr, r3 + adr r5, L(ipred_paeth_tbl) + sub lr, lr, #25 + ldr lr, [r5, lr, lsl #2] + vld1.8 {d4[], d5[]}, [r2] + add r8, r2, #1 + sub r2, r2, #4 + add r5, r5, lr + mov r7, #-4 + add r6, r0, r1 + lsl r1, r1, #1 + bx r5 + + .align 2 +L(ipred_paeth_tbl): + .word 640f - L(ipred_paeth_tbl) + CONFIG_THUMB + .word 320f - L(ipred_paeth_tbl) + CONFIG_THUMB + .word 160f - L(ipred_paeth_tbl) + CONFIG_THUMB + .word 80f - L(ipred_paeth_tbl) + CONFIG_THUMB + .word 40f - L(ipred_paeth_tbl) + CONFIG_THUMB + +40: + vld1.32 {d6[], d7[]}, [r8] + vsubl.u8 q8, d6, d4 // top - topleft +4: + vld4.8 {d0[], d1[], d2[], d3[]}, [r2, :32], r7 + vzip.32 d0, d1 + vzip.32 d2, d3 + vaddw.u8 q9, q8, d0 + vaddw.u8 q10, q8, d2 + vqmovun.s16 d18, q9 // base + vqmovun.s16 d19, q10 + vmov d1, d2 + vabd.u8 q10, q3, q9 // tdiff + vabd.u8 q11, q2, q9 // tldiff + vabd.u8 q9, q0, q9 // ldiff + vmin.u8 q12, q10, q11 // min(tdiff, tldiff) + vcge.u8 q10, q11, q10 // tldiff >= tdiff + vcge.u8 q9, q12, q9 // min(tdiff, tldiff) >= ldiff + vbsl q10, q3, q2 // tdiff <= tldiff ? top : topleft + vbit q10, q0, q9 // ldiff <= min ? left : ... + vst1.32 {d21[1]}, [r0, :32], r1 + vst1.32 {d21[0]}, [r6, :32], r1 + subs r4, r4, #4 + vst1.32 {d20[1]}, [r0, :32], r1 + vst1.32 {d20[0]}, [r6, :32], r1 + bgt 4b + pop {r4-r8, pc} +80: + vld1.8 {d6}, [r8] + vsubl.u8 q8, d6, d4 // top - topleft + vmov d7, d6 +8: + vld4.8 {d0[], d1[], d2[], d3[]}, [r2, :32], r7 + vaddw.u8 q9, q8, d0 + vaddw.u8 q10, q8, d1 + vaddw.u8 q11, q8, d2 + vaddw.u8 q12, q8, d3 + vqmovun.s16 d18, q9 // base + vqmovun.s16 d19, q10 + vqmovun.s16 d20, q11 + vqmovun.s16 d21, q12 + vabd.u8 q11, q3, q9 // tdiff + vabd.u8 q12, q3, q10 + vabd.u8 q13, q2, q9 // tldiff + vabd.u8 q14, q2, q10 + vabd.u8 q10, q1, q10 // ldiff + vabd.u8 q9, q0, q9 + vmin.u8 q15, q12, q14 // min(tdiff, tldiff) + vcge.u8 q12, q14, q12 // tldiff >= tdiff + vmin.u8 q14, q11, q13 // min(tdiff, tldiff) + vcge.u8 q11, q13, q11 // tldiff >= tdiff + vcge.u8 q10, q15, q10 // min(tdiff, tldiff) >= ldiff + vcge.u8 q9, q14, q9 + vbsl q12, q3, q2 // tdiff <= tldiff ? top : topleft + vbsl q11, q3, q2 + vbit q12, q1, q10 // ldiff <= min ? left : ... + vbit q11, q0, q9 + vst1.8 {d25}, [r0, :64], r1 + vst1.8 {d24}, [r6, :64], r1 + subs r4, r4, #4 + vst1.8 {d23}, [r0, :64], r1 + vst1.8 {d22}, [r6, :64], r1 + bgt 8b + pop {r4-r8, pc} +160: +320: +640: + vld1.8 {d6}, [r8]! + mov r12, r3 + // Set up pointers for four rows in parallel; r0, r6, r5, lr + add r5, r0, r1 + add lr, r6, r1 + lsl r1, r1, #1 + sub r1, r1, r3 +1: + vld4.8 {d0[], d1[], d2[], d3[]}, [r2, :32], r7 +2: + vsubl.u8 q8, d6, d4 // top - topleft + vmov d7, d6 + vaddw.u8 q9, q8, d0 + vaddw.u8 q10, q8, d1 + vaddw.u8 q11, q8, d2 + vaddw.u8 q12, q8, d3 + vqmovun.s16 d18, q9 // base + vqmovun.s16 d19, q10 + vqmovun.s16 d20, q11 + vqmovun.s16 d21, q12 + vabd.u8 q11, q3, q9 // tdiff + vabd.u8 q12, q3, q10 + vabd.u8 q13, q2, q9 // tldiff + vabd.u8 q14, q2, q10 + vabd.u8 q10, q1, q10 // ldiff + vabd.u8 q9, q0, q9 + vmin.u8 q15, q12, q14 // min(tdiff, tldiff) + vcge.u8 q12, q14, q12 // tldiff >= tdiff + vmin.u8 q14, q11, q13 // min(tdiff, tldiff) + vcge.u8 q11, q13, q11 // tldiff >= tdiff + vcge.u8 q10, q15, q10 // min(tdiff, tldiff) >= ldiff + vcge.u8 q9, q14, q9 + vbsl q12, q3, q2 // tdiff <= tldiff ? top : topleft + vbsl q11, q3, q2 + vbit q12, q1, q10 // ldiff <= min ? left : ... + vbit q11, q0, q9 + subs r3, r3, #8 + vst1.8 {d25}, [r0, :64]! + vst1.8 {d24}, [r6, :64]! + vst1.8 {d23}, [r5, :64]! + vst1.8 {d22}, [lr, :64]! + ble 8f + vld1.8 {d6}, [r8]! + b 2b +8: + subs r4, r4, #4 + ble 9f + // End of horizontal loop, move pointers to next four rows + sub r8, r8, r12 + add r0, r0, r1 + add r6, r6, r1 + vld1.8 {d6}, [r8]! + add r5, r5, r1 + add lr, lr, r1 + mov r3, r12 + b 1b +9: + pop {r4-r8, pc} +endfunc + +// void ipred_smooth_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_smooth_8bpc_neon, export=1 + push {r4-r10, lr} + ldr r4, [sp, #32] + movrel r10, X(sm_weights) + add r12, r10, r4 + add r10, r10, r3 + clz r9, r3 + adr r5, L(ipred_smooth_tbl) + sub lr, r2, r4 + sub r9, r9, #25 + ldr r9, [r5, r9, lsl #2] + vld1.8 {d4[]}, [lr] // bottom + add r8, r2, #1 + add r5, r5, r9 + add r6, r0, r1 + lsl r1, r1, #1 + bx r5 + + .align 2 +L(ipred_smooth_tbl): + .word 640f - L(ipred_smooth_tbl) + CONFIG_THUMB + .word 320f - L(ipred_smooth_tbl) + CONFIG_THUMB + .word 160f - L(ipred_smooth_tbl) + CONFIG_THUMB + .word 80f - L(ipred_smooth_tbl) + CONFIG_THUMB + .word 40f - L(ipred_smooth_tbl) + CONFIG_THUMB + +40: + vld1.32 {d16[]}, [r8] // top + vld1.32 {d18[]}, [r10, :32] // weights_hor + sub r2, r2, #4 + mov r7, #-4 + vdup.8 q3, d16[3] // right + vsubl.u8 q8, d16, d4 // top-bottom + vmovl.u8 q9, d18 // weights_hor +4: + vld4.8 {d0[], d1[], d2[], d3[]}, [r2, :32], r7 // left + vld4.8 {d20[], d21[], d22[], d23[]}, [r12, :32]! // weights_ver + vshll.i8 q12, d6, #8 // right*256 + vshll.i8 q13, d6, #8 + vzip.32 d1, d0 // left, flipped + vzip.32 d3, d2 + vzip.32 d20, d21 // weights_ver + vzip.32 d22, d23 + vshll.i8 q14, d4, #8 // bottom*256 + vshll.i8 q15, d4, #8 + vsubl.u8 q0, d1, d6 // left-right + vsubl.u8 q1, d3, d6 + vmovl.u8 q10, d20 // weights_ver + vmovl.u8 q11, d22 + vmla.i16 q12, q1, q9 // right*256 + (left-right)*weights_hor + vmla.i16 q13, q0, q9 // (left flipped) + vmla.i16 q14, q8, q10 // bottom*256 + (top-bottom)*weights_ver + vmla.i16 q15, q8, q11 + vhadd.u16 q12, q12, q14 + vhadd.u16 q13, q13, q15 + vrshrn.i16 d24, q12, #8 + vrshrn.i16 d25, q13, #8 + vst1.32 {d24[0]}, [r0, :32], r1 + vst1.32 {d24[1]}, [r6, :32], r1 + subs r4, r4, #4 + vst1.32 {d25[0]}, [r0, :32], r1 + vst1.32 {d25[1]}, [r6, :32], r1 + bgt 4b + pop {r4-r10, pc} +80: + vld1.8 {d16}, [r8] // top + vld1.8 {d18}, [r10, :64] // weights_hor + sub r2, r2, #2 + mov r7, #-2 + vdup.8 q3, d16[7] // right + vsubl.u8 q8, d16, d4 // top-bottom + vmovl.u8 q9, d18 // weights_hor +8: + vld2.8 {d0[], d1[]}, [r2, :16], r7 // left + vld2.8 {d20[], d22[]}, [r12, :16]! // weights_ver + vshll.i8 q12, d6, #8 // right*256 + vshll.i8 q13, d6, #8 + vshll.i8 q14, d4, #8 // bottom*256 + vshll.i8 q15, d4, #8 + vsubl.u8 q1, d0, d6 // left-right (left flipped) + vsubl.u8 q0, d1, d6 + vmovl.u8 q10, d20 // weights_ver + vmovl.u8 q11, d22 + vmla.i16 q12, q0, q9 // right*256 + (left-right)*weights_hor + vmla.i16 q13, q1, q9 + vmla.i16 q14, q8, q10 // bottom*256 + (top-bottom)*weights_ver + vmla.i16 q15, q8, q11 + vhadd.u16 q12, q12, q14 + vhadd.u16 q13, q13, q15 + vrshrn.i16 d24, q12, #8 + vrshrn.i16 d25, q13, #8 + subs r4, r4, #2 + vst1.8 {d24}, [r0, :64], r1 + vst1.8 {d25}, [r6, :64], r1 + bgt 8b + pop {r4-r10, pc} +160: +320: +640: + add lr, r2, r3 + sub r2, r2, #2 + mov r7, #-2 + vld1.8 {d6[], d7[]}, [lr] // right + sub r1, r1, r3 + mov r9, r3 + +1: + vld2.8 {d0[], d1[]}, [r2, :16], r7 // left + vld2.8 {d20[], d22[]}, [r12, :16]! // weights_ver + vsubl.u8 q1, d0, d6 // left-right (left flipped) + vsubl.u8 q0, d1, d6 + vmovl.u8 q10, d20 // weights_ver + vmovl.u8 q11, d22 +2: + vld1.8 {d16}, [r8]! // top + vld1.8 {d18}, [r10, :64]! // weights_hor + vshll.i8 q12, d6, #8 // right*256 + vshll.i8 q13, d6, #8 + vmovl.u8 q9, d18 // weights_hor + vshll.i8 q14, d4, #8 // bottom*256 + vshll.i8 q15, d4, #8 + vsubl.u8 q8, d16, d4 // top-bottom + vmla.i16 q12, q0, q9 // right*256 + (left-right)*weights_hor + vmla.i16 q13, q1, q9 + vmla.i16 q14, q8, q10 // bottom*256 + (top-bottom)*weights_ver + vmla.i16 q15, q8, q11 + vhadd.u16 q12, q12, q14 + vhadd.u16 q13, q13, q15 + vrshrn.i16 d24, q12, #8 + vrshrn.i16 d25, q13, #8 + subs r3, r3, #8 + vst1.8 {d24}, [r0, :64]! + vst1.8 {d25}, [r6, :64]! + bgt 2b + subs r4, r4, #2 + ble 9f + sub r8, r8, r9 + sub r10, r10, r9 + add r0, r0, r1 + add r6, r6, r1 + mov r3, r9 + b 1b +9: + pop {r4-r10, pc} +endfunc + +// void ipred_smooth_v_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_smooth_v_8bpc_neon, export=1 + push {r4-r7, lr} + ldr r4, [sp, #20] + movrel r7, X(sm_weights) + add r7, r7, r4 + clz lr, r3 + adr r5, L(ipred_smooth_v_tbl) + sub r12, r2, r4 + sub lr, lr, #25 + ldr lr, [r5, lr, lsl #2] + vld1.8 {d4[]}, [r12] // bottom + add r2, r2, #1 + add r5, r5, lr + add r6, r0, r1 + lsl r1, r1, #1 + bx r5 + + .align 2 +L(ipred_smooth_v_tbl): + .word 640f - L(ipred_smooth_v_tbl) + CONFIG_THUMB + .word 320f - L(ipred_smooth_v_tbl) + CONFIG_THUMB + .word 160f - L(ipred_smooth_v_tbl) + CONFIG_THUMB + .word 80f - L(ipred_smooth_v_tbl) + CONFIG_THUMB + .word 40f - L(ipred_smooth_v_tbl) + CONFIG_THUMB + +40: + vld1.32 {d6[]}, [r2] // top + vsubl.u8 q3, d6, d4 // top-bottom +4: + vld4.8 {d16[], d17[], d18[], d19[]}, [r7, :32]! // weights_ver + vshll.i8 q10, d4, #8 // bottom*256 + vshll.i8 q11, d4, #8 + vzip.32 d16, d17 // weights_ver + vzip.32 d18, d19 + vmovl.u8 q8, d16 // weights_ver + vmovl.u8 q9, d18 + subs r4, r4, #4 + vmla.i16 q10, q3, q8 // bottom*256 + (top-bottom)*weights_ver + vmla.i16 q11, q3, q9 + vrshrn.i16 d20, q10, #8 + vrshrn.i16 d21, q11, #8 + vst1.32 {d20[0]}, [r0, :32], r1 + vst1.32 {d20[1]}, [r6, :32], r1 + vst1.32 {d21[0]}, [r0, :32], r1 + vst1.32 {d21[1]}, [r6, :32], r1 + bgt 4b + pop {r4-r7, pc} +80: + vld1.8 {d6}, [r2] // top + vsubl.u8 q3, d6, d4 // top-bottom +8: + vld4.8 {d16[], d18[], d20[], d22[]}, [r7, :32]! // weights_ver + vshll.i8 q12, d4, #8 // bottom*256 + vshll.i8 q13, d4, #8 + vshll.i8 q14, d4, #8 + vshll.i8 q15, d4, #8 + vmovl.u8 q8, d16 // weights_ver + vmovl.u8 q9, d18 + vmovl.u8 q10, d20 + vmovl.u8 q11, d22 + vmla.i16 q12, q3, q8 // bottom*256 + (top-bottom)*weights_ver + vmla.i16 q13, q3, q9 + vmla.i16 q14, q3, q10 + vmla.i16 q15, q3, q11 + vrshrn.i16 d24, q12, #8 + vrshrn.i16 d25, q13, #8 + vrshrn.i16 d26, q14, #8 + vrshrn.i16 d27, q15, #8 + vst1.8 {d24}, [r0, :64], r1 + vst1.8 {d25}, [r6, :64], r1 + subs r4, r4, #4 + vst1.8 {d26}, [r0, :64], r1 + vst1.8 {d27}, [r6, :64], r1 + bgt 8b + pop {r4-r7, pc} +160: +320: +640: + vpush {q4-q7} + // Set up pointers for four rows in parallel; r0, r6, r5, lr + add r5, r0, r1 + add lr, r6, r1 + lsl r1, r1, #1 + sub r1, r1, r3 + mov r12, r3 + +1: + vld4.8 {d8[], d10[], d12[], d14[]}, [r7, :32]! // weights_ver + vmovl.u8 q4, d8 // weights_ver + vmovl.u8 q5, d10 + vmovl.u8 q6, d12 + vmovl.u8 q7, d14 +2: + vld1.8 {q3}, [r2]! // top + vshll.i8 q8, d4, #8 // bottom*256 + vshll.i8 q9, d4, #8 + vshll.i8 q10, d4, #8 + vshll.i8 q11, d4, #8 + vsubl.u8 q0, d6, d4 // top-bottom + vsubl.u8 q1, d7, d4 + vshll.i8 q12, d4, #8 + vshll.i8 q13, d4, #8 + vshll.i8 q14, d4, #8 + vshll.i8 q15, d4, #8 + vmla.i16 q8, q0, q4 // bottom*256 + (top-bottom)*weights_ver + vmla.i16 q9, q1, q4 + vmla.i16 q10, q0, q5 + vmla.i16 q11, q1, q5 + vmla.i16 q12, q0, q6 // bottom*256 + (top-bottom)*weights_ver + vmla.i16 q13, q1, q6 + vmla.i16 q14, q0, q7 + vmla.i16 q15, q1, q7 + vrshrn.i16 d16, q8, #8 + vrshrn.i16 d17, q9, #8 + vrshrn.i16 d18, q10, #8 + vrshrn.i16 d19, q11, #8 + vrshrn.i16 d20, q12, #8 + vrshrn.i16 d21, q13, #8 + vrshrn.i16 d22, q14, #8 + vrshrn.i16 d23, q15, #8 + subs r3, r3, #16 + vst1.8 {q8}, [r0, :128]! + vst1.8 {q9}, [r6, :128]! + vst1.8 {q10}, [r5, :128]! + vst1.8 {q11}, [lr, :128]! + bgt 2b + subs r4, r4, #4 + ble 9f + sub r2, r2, r12 + add r0, r0, r1 + add r6, r6, r1 + add r5, r5, r1 + add lr, lr, r1 + mov r3, r12 + b 1b +9: + vpop {q4-q7} + pop {r4-r7, pc} +endfunc + +// void ipred_smooth_h_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_smooth_h_8bpc_neon, export=1 + push {r4-r8, lr} + ldr r4, [sp, #24] + movrel r8, X(sm_weights) + add r8, r8, r3 + clz lr, r3 + adr r5, L(ipred_smooth_h_tbl) + add r12, r2, r3 + sub lr, lr, #25 + ldr lr, [r5, lr, lsl #2] + vld1.8 {d4[]}, [r12] // right + add r5, r5, lr + add r6, r0, r1 + lsl r1, r1, #1 + bx r5 + + .align 2 +L(ipred_smooth_h_tbl): + .word 640f - L(ipred_smooth_h_tbl) + CONFIG_THUMB + .word 320f - L(ipred_smooth_h_tbl) + CONFIG_THUMB + .word 160f - L(ipred_smooth_h_tbl) + CONFIG_THUMB + .word 80f - L(ipred_smooth_h_tbl) + CONFIG_THUMB + .word 40f - L(ipred_smooth_h_tbl) + CONFIG_THUMB + +40: + vld1.32 {d6[]}, [r8, :32] // weights_hor + sub r2, r2, #4 + mov r7, #-4 + vmovl.u8 q3, d6 // weights_hor +4: + vld4.8 {d0[], d1[], d2[], d3[]}, [r2, :32], r7 // left + vshll.i8 q8, d4, #8 // right*256 + vshll.i8 q9, d4, #8 + vzip.32 d3, d2 // left, flipped + vzip.32 d1, d0 + vsubl.u8 q1, d3, d4 // left-right + vsubl.u8 q0, d1, d4 + subs r4, r4, #4 + vmla.i16 q8, q1, q3 // right*256 + (left-right)*weights_hor + vmla.i16 q9, q0, q3 + vrshrn.i16 d16, q8, #8 + vrshrn.i16 d17, q9, #8 + vst1.32 {d16[0]}, [r0, :32], r1 + vst1.32 {d16[1]}, [r6, :32], r1 + vst1.32 {d17[0]}, [r0, :32], r1 + vst1.32 {d17[1]}, [r6, :32], r1 + bgt 4b + pop {r4-r8, pc} +80: + vld1.8 {d6}, [r8, :64] // weights_hor + sub r2, r2, #4 + mov r7, #-4 + vmovl.u8 q3, d6 // weights_hor +8: + vld4.8 {d16[], d18[], d20[], d22[]}, [r2, :32], r7 // left + vshll.i8 q12, d4, #8 // right*256 + vshll.i8 q13, d4, #8 + vshll.i8 q14, d4, #8 + vshll.i8 q15, d4, #8 + vsubl.u8 q11, d22, d4 // left-right + vsubl.u8 q10, d20, d4 + vsubl.u8 q9, d18, d4 + vsubl.u8 q8, d16, d4 + vmla.i16 q12, q11, q3 // right*256 + (left-right)*weights_hor + vmla.i16 q13, q10, q3 // (left flipped) + vmla.i16 q14, q9, q3 + vmla.i16 q15, q8, q3 + vrshrn.i16 d24, q12, #8 + vrshrn.i16 d25, q13, #8 + vrshrn.i16 d26, q14, #8 + vrshrn.i16 d27, q15, #8 + vst1.8 {d24}, [r0, :64], r1 + vst1.8 {d25}, [r6, :64], r1 + subs r4, r4, #4 + vst1.8 {d26}, [r0, :64], r1 + vst1.8 {d27}, [r6, :64], r1 + bgt 8b + pop {r4-r8, pc} +160: +320: +640: + vpush {q4-q7} + sub r2, r2, #4 + mov r7, #-4 + // Set up pointers for four rows in parallel; r0, r6, r5, lr + add r5, r0, r1 + add lr, r6, r1 + lsl r1, r1, #1 + sub r1, r1, r3 + mov r12, r3 + +1: + vld4.8 {d8[], d10[], d12[], d14[]}, [r2, :32], r7 // left + vsubl.u8 q4, d8, d4 // left-right + vsubl.u8 q5, d10, d4 + vsubl.u8 q6, d12, d4 + vsubl.u8 q7, d14, d4 +2: + vld1.8 {q1}, [r8, :128]! // weights_hor + vshll.i8 q8, d4, #8 // right*256 + vshll.i8 q9, d4, #8 + vshll.i8 q10, d4, #8 + vshll.i8 q11, d4, #8 + vmovl.u8 q0, d2 // weights_hor + vmovl.u8 q1, d3 + vshll.i8 q12, d4, #8 + vshll.i8 q13, d4, #8 + vshll.i8 q14, d4, #8 + vshll.i8 q15, d4, #8 + vmla.i16 q8, q7, q0 // right*256 + (left-right)*weights_hor + vmla.i16 q9, q7, q1 // (left flipped) + vmla.i16 q10, q6, q0 + vmla.i16 q11, q6, q1 + vmla.i16 q12, q5, q0 + vmla.i16 q13, q5, q1 + vmla.i16 q14, q4, q0 + vmla.i16 q15, q4, q1 + vrshrn.i16 d16, q8, #8 + vrshrn.i16 d17, q9, #8 + vrshrn.i16 d18, q10, #8 + vrshrn.i16 d19, q11, #8 + vrshrn.i16 d20, q12, #8 + vrshrn.i16 d21, q13, #8 + vrshrn.i16 d22, q14, #8 + vrshrn.i16 d23, q15, #8 + subs r3, r3, #16 + vst1.8 {q8}, [r0, :128]! + vst1.8 {q9}, [r6, :128]! + vst1.8 {q10}, [r5, :128]! + vst1.8 {q11}, [lr, :128]! + bgt 2b + subs r4, r4, #4 + ble 9f + sub r8, r8, r12 + add r0, r0, r1 + add r6, r6, r1 + add r5, r5, r1 + add lr, lr, r1 + mov r3, r12 + b 1b +9: + vpop {q4-q7} + pop {r4-r8, pc} +endfunc + +// void ipred_filter_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int filt_idx, +// const int max_width, const int max_height); +function ipred_filter_8bpc_neon, export=1 + push {r4-r8, lr} + movw r12, #511 + ldr r5, [sp, #28] + ldr r4, [sp, #24] + and r5, r5, r12 // 511 + movrel r6, X(filter_intra_taps) + lsl r5, r5, #6 + add r6, r6, r5 + vld1.8 {d20, d21, d22, d23}, [r6, :128]! + clz lr, r3 + adr r5, L(ipred_filter_tbl) + vld1.8 {d27, d28, d29}, [r6, :64] + sub lr, lr, #26 + ldr lr, [r5, lr, lsl #2] + vmovl.s8 q8, d20 + vmovl.s8 q9, d21 + add r5, r5, lr + vmovl.s8 q10, d22 + vmovl.s8 q11, d23 + add r6, r0, r1 + lsl r1, r1, #1 + vmovl.s8 q12, d27 + vmovl.s8 q13, d28 + vmovl.s8 q14, d29 + add r8, r2, #1 + bx r5 + + .align 2 +L(ipred_filter_tbl): + .word 320f - L(ipred_filter_tbl) + CONFIG_THUMB + .word 160f - L(ipred_filter_tbl) + CONFIG_THUMB + .word 80f - L(ipred_filter_tbl) + CONFIG_THUMB + .word 40f - L(ipred_filter_tbl) + CONFIG_THUMB + +40: + vld1.32 {d0[]}, [r8] // top (0-3) + sub r2, r2, #2 + mov r7, #-2 + vmovl.u8 q0, d0 // top (0-3) +4: + vld1.32 {d2[]}, [r2], r7 // left (0-1) + topleft (2) + vmul.i16 q2, q9, d0[0] // p1(top[0]) * filter(1) + vmla.i16 q2, q10, d0[1] // p2(top[1]) * filter(2) + vmla.i16 q2, q11, d0[2] // p3(top[2]) * filter(3) + vmovl.u8 q1, d2 // left (0-1) + topleft (2) + vmla.i16 q2, q12, d0[3] // p4(top[3]) * filter(4) + vmla.i16 q2, q8, d2[2] // p0(topleft) * filter(0) + vmla.i16 q2, q13, d2[1] // p5(left[0]) * filter(5) + vmla.i16 q2, q14, d2[0] // p6(left[1]) * filter(6) + vqrshrun.s16 d4, q2, #4 + subs r4, r4, #2 + vst1.32 {d4[0]}, [r0, :32], r1 + vmovl.u8 q0, d4 + vst1.32 {d4[1]}, [r6, :32], r1 + vext.8 q0, q0, q0, #8 // move top from [4-7] to [0-3] + bgt 4b + pop {r4-r8, pc} +80: + vld1.8 {d0}, [r8] // top (0-7) + sub r2, r2, #2 + mov r7, #-2 + vmovl.u8 q0, d0 // top (0-7) +8: + vld1.32 {d2[]}, [r2], r7 // left (0-1) + topleft (2) + vmul.i16 q2, q9, d0[0] // p1(top[0]) * filter(1) + vmla.i16 q2, q10, d0[1] // p2(top[1]) * filter(2) + vmla.i16 q2, q11, d0[2] // p3(top[2]) * filter(3) + vmovl.u8 q1, d2 // left (0-1) + topleft (2) + vmla.i16 q2, q12, d0[3] // p4(top[3]) * filter(4) + vmla.i16 q2, q8, d2[2] // p0(topleft) * filter(0) + vmla.i16 q2, q13, d2[1] // p5(left[0]) * filter(5) + vmla.i16 q2, q14, d2[0] // p6(left[1]) * filter(6) + vmul.i16 q3, q9, d1[0] // p1(top[0]) * filter(1) + vmla.i16 q3, q10, d1[1] // p2(top[1]) * filter(2) + vmla.i16 q3, q11, d1[2] // p3(top[2]) * filter(3) + vqrshrun.s16 d4, q2, #4 + vmovl.u8 q1, d4 // first block, in 16 bit + vmla.i16 q3, q12, d1[3] // p4(top[3]) * filter(4) + vmla.i16 q3, q8, d0[3] // p0(topleft) * filter(0) + vmla.i16 q3, q13, d2[3] // p5(left[0]) * filter(5) + vmla.i16 q3, q14, d3[3] // p6(left[1]) * filter(6) + vqrshrun.s16 d5, q3, #4 + vzip.32 d4, d5 + subs r4, r4, #2 + vst1.64 {d4}, [r0, :64], r1 + vmovl.u8 q0, d5 + vst1.64 {d5}, [r6, :64], r1 + bgt 8b + pop {r4-r8, pc} +160: +320: + vpush {q4-q5} + sub r2, r2, #2 + mov r7, #-2 + sub r1, r1, r3 + mov lr, r3 + +1: + vld1.32 {d0[]}, [r2], r7 // left (0-1) + topleft (2) + vmovl.u8 q0, d0 // left (0-1) + topleft (2) +2: + vld1.8 {q2}, [r8]! // top(0-15) + vmul.i16 q3, q8, d0[2] // p0(topleft) * filter(0) + vmla.i16 q3, q13, d0[1] // p5(left[0]) * filter(5) + vmovl.u8 q1, d4 // top(0-7) + vmovl.u8 q2, d5 // top(8-15) + vmla.i16 q3, q14, d0[0] // p6(left[1]) * filter(6) + vmla.i16 q3, q9, d2[0] // p1(top[0]) * filter(1) + vmla.i16 q3, q10, d2[1] // p2(top[1]) * filter(2) + vmla.i16 q3, q11, d2[2] // p3(top[2]) * filter(3) + vmla.i16 q3, q12, d2[3] // p4(top[3]) * filter(4) + + vmul.i16 q4, q9, d3[0] // p1(top[0]) * filter(1) + vmla.i16 q4, q10, d3[1] // p2(top[1]) * filter(2) + vmla.i16 q4, q11, d3[2] // p3(top[2]) * filter(3) + vqrshrun.s16 d6, q3, #4 + vmovl.u8 q0, d6 // first block, in 16 bit + vmla.i16 q4, q12, d3[3] // p4(top[3]) * filter(4) + vmla.i16 q4, q8, d2[3] // p0(topleft) * filter(0) + vmla.i16 q4, q13, d0[3] // p5(left[0]) * filter(5) + vmla.i16 q4, q14, d1[3] // p6(left[1]) * filter(6) + + vmul.i16 q5, q9, d4[0] // p1(top[0]) * filter(1) + vmla.i16 q5, q10, d4[1] // p2(top[1]) * filter(2) + vmla.i16 q5, q11, d4[2] // p3(top[2]) * filter(3) + vqrshrun.s16 d7, q4, #4 + vmovl.u8 q0, d7 // second block, in 16 bit + vmla.i16 q5, q12, d4[3] // p4(top[3]) * filter(4) + vmla.i16 q5, q8, d3[3] // p0(topleft) * filter(0) + vmla.i16 q5, q13, d0[3] // p5(left[0]) * filter(5) + vmla.i16 q5, q14, d1[3] // p6(left[1]) * filter(6) + + vmul.i16 q15, q9, d5[0] // p1(top[0]) * filter(1) + vmla.i16 q15, q10, d5[1] // p2(top[1]) * filter(2) + vmla.i16 q15, q11, d5[2] // p3(top[2]) * filter(3) + vqrshrun.s16 d8, q5, #4 + vmovl.u8 q0, d8 // third block, in 16 bit + vmov.u8 r12, d5[6] + vmla.i16 q15, q12, d5[3] // p4(top[3]) * filter(4) + vmla.i16 q15, q8, d4[3] // p0(topleft) * filter(0) + vmla.i16 q15, q13, d0[3] // p5(left[0]) * filter(5) + vmla.i16 q15, q14, d1[3] // p6(left[1]) * filter(6) + vmov.8 d0[4], r12 + + subs r3, r3, #16 + vqrshrun.s16 d9, q15, #4 + + vst4.32 {d6[0], d7[0], d8[0], d9[0]}, [r0, :128]! + vst4.32 {d6[1], d7[1], d8[1], d9[1]}, [r6, :128]! + ble 8f + vmov.u8 r12, d9[7] + vmov.8 d0[0], r12 + vmov.u8 r12, d9[3] + vmov.8 d0[2], r12 + b 2b +8: + subs r4, r4, #2 + + ble 9f + sub r8, r6, lr + add r0, r0, r1 + add r6, r6, r1 + mov r3, lr + b 1b +9: + vpop {q4-q5} + pop {r4-r8, pc} +endfunc + +// void pal_pred_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const uint16_t *const pal, const uint8_t *idx, +// const int w, const int h); +function pal_pred_8bpc_neon, export=1 + push {r4-r5, lr} + ldr r4, [sp, #12] + ldr r5, [sp, #16] + vld1.16 {q0}, [r2, :128] + clz lr, r4 + adr r12, L(pal_pred_tbl) + sub lr, lr, #25 + ldr lr, [r12, lr, lsl #2] + vmovn.i16 d0, q0 + add r12, r12, lr + add r2, r0, r1 + bx r12 + + .align 2 +L(pal_pred_tbl): + .word 640f - L(pal_pred_tbl) + CONFIG_THUMB + .word 320f - L(pal_pred_tbl) + CONFIG_THUMB + .word 160f - L(pal_pred_tbl) + CONFIG_THUMB + .word 80f - L(pal_pred_tbl) + CONFIG_THUMB + .word 40f - L(pal_pred_tbl) + CONFIG_THUMB + +40: + lsl r1, r1, #1 +4: + vld1.8 {q1}, [r3, :128]! + subs r5, r5, #4 + vtbl.8 d2, {d0}, d2 + vtbl.8 d3, {d0}, d3 + vst1.32 {d2[0]}, [r0, :32], r1 + vst1.32 {d2[1]}, [r2, :32], r1 + vst1.32 {d3[0]}, [r0, :32], r1 + vst1.32 {d3[1]}, [r2, :32], r1 + bgt 4b + pop {r4-r5, pc} +80: + lsl r1, r1, #1 +8: + vld1.8 {q1, q2}, [r3, :128]! + subs r5, r5, #4 + vtbl.8 d2, {d0}, d2 + vtbl.8 d3, {d0}, d3 + vst1.8 {d2}, [r0, :64], r1 + vtbl.8 d4, {d0}, d4 + vst1.8 {d3}, [r2, :64], r1 + vtbl.8 d5, {d0}, d5 + vst1.8 {d4}, [r0, :64], r1 + vst1.8 {d5}, [r2, :64], r1 + bgt 8b + pop {r4-r5, pc} +160: + lsl r1, r1, #1 +16: + vld1.8 {q8, q9}, [r3, :128]! + subs r5, r5, #4 + vld1.8 {q10, q11}, [r3, :128]! + vtbl.8 d16, {d0}, d16 + vtbl.8 d17, {d0}, d17 + vtbl.8 d18, {d0}, d18 + vtbl.8 d19, {d0}, d19 + vtbl.8 d20, {d0}, d20 + vtbl.8 d21, {d0}, d21 + vst1.8 {q8}, [r0, :128], r1 + vtbl.8 d22, {d0}, d22 + vst1.8 {q9}, [r2, :128], r1 + vtbl.8 d23, {d0}, d23 + vst1.8 {q10}, [r0, :128], r1 + vst1.8 {q11}, [r2, :128], r1 + bgt 16b + pop {r4-r5, pc} +320: + lsl r1, r1, #1 +32: + vld1.8 {q8, q9}, [r3, :128]! + subs r5, r5, #2 + vld1.8 {q10, q11}, [r3, :128]! + vtbl.8 d16, {d0}, d16 + vtbl.8 d17, {d0}, d17 + vtbl.8 d18, {d0}, d18 + vtbl.8 d19, {d0}, d19 + vtbl.8 d20, {d0}, d20 + vtbl.8 d21, {d0}, d21 + vst1.8 {q8, q9}, [r0, :128], r1 + vtbl.8 d22, {d0}, d22 + vtbl.8 d23, {d0}, d23 + vst1.8 {q10, q11}, [r2, :128], r1 + bgt 32b + pop {r4-r5, pc} +640: + sub r1, r1, #32 +64: + vld1.8 {q8, q9}, [r3, :128]! + subs r5, r5, #1 + vld1.8 {q10, q11}, [r3, :128]! + vtbl.8 d16, {d0}, d16 + vtbl.8 d17, {d0}, d17 + vtbl.8 d18, {d0}, d18 + vtbl.8 d19, {d0}, d19 + vtbl.8 d20, {d0}, d20 + vtbl.8 d21, {d0}, d21 + vst1.8 {q8, q9}, [r0, :128]! + vtbl.8 d22, {d0}, d22 + vtbl.8 d23, {d0}, d23 + vst1.8 {q10, q11}, [r0, :128], r1 + bgt 64b + pop {r4-r5, pc} +endfunc + +// void ipred_cfl_128_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha); +function ipred_cfl_128_8bpc_neon, export=1 + push {r4-r8, lr} + ldr r4, [sp, #24] + ldr r5, [sp, #28] + ldr r6, [sp, #32] + clz lr, r3 + adr r12, L(ipred_cfl_128_tbl) + sub lr, lr, #26 + ldr lr, [r12, lr, lsl #2] + vmov.i16 q0, #128 // dc + vdup.i16 q1, r6 // alpha + add r12, r12, lr + add r6, r0, r1 + lsl r1, r1, #1 + bx r12 + + .align 2 +L(ipred_cfl_128_tbl): +L(ipred_cfl_splat_tbl): + .word L(ipred_cfl_splat_w16) - L(ipred_cfl_128_tbl) + CONFIG_THUMB + .word L(ipred_cfl_splat_w16) - L(ipred_cfl_128_tbl) + CONFIG_THUMB + .word L(ipred_cfl_splat_w8) - L(ipred_cfl_128_tbl) + CONFIG_THUMB + .word L(ipred_cfl_splat_w4) - L(ipred_cfl_128_tbl) + CONFIG_THUMB + +L(ipred_cfl_splat_w4): + vld1.16 {q2, q3}, [r5, :128]! + vmul.i16 q2, q2, q1 // diff = ac * alpha + vmul.i16 q3, q3, q1 + vshr.s16 q8, q2, #15 // sign = diff >> 15 + vshr.s16 q9, q3, #15 + vadd.i16 q2, q2, q8 // diff + sign + vadd.i16 q3, q3, q9 + vrshr.s16 q2, q2, #6 // (diff + sign + 32) >> 6 = apply_sign() + vrshr.s16 q3, q3, #6 + vadd.i16 q2, q2, q0 // dc + apply_sign() + vadd.i16 q3, q3, q0 + vqmovun.s16 d4, q2 // iclip_pixel(dc + apply_sign()) + vqmovun.s16 d5, q3 + vst1.32 {d4[0]}, [r0, :32], r1 + vst1.32 {d4[1]}, [r6, :32], r1 + subs r4, r4, #4 + vst1.32 {d5[0]}, [r0, :32], r1 + vst1.32 {d5[1]}, [r6, :32], r1 + bgt L(ipred_cfl_splat_w4) + pop {r4-r8, pc} +L(ipred_cfl_splat_w8): + vld1.16 {q8, q9}, [r5, :128]! + vld1.16 {q10, q11}, [r5, :128]! + vmul.i16 q8, q8, q1 // diff = ac * alpha + vmul.i16 q9, q9, q1 + vmul.i16 q10, q10, q1 + vmul.i16 q11, q11, q1 + vshr.s16 q12, q8, #15 // sign = diff >> 15 + vshr.s16 q13, q9, #15 + vshr.s16 q14, q10, #15 + vshr.s16 q15, q11, #15 + vadd.i16 q8, q8, q12 // diff + sign + vadd.i16 q9, q9, q13 + vadd.i16 q10, q10, q14 + vadd.i16 q11, q11, q15 + vrshr.s16 q8, q8, #6 // (diff + sign + 32) >> 6 = apply_sign() + vrshr.s16 q9, q9, #6 + vrshr.s16 q10, q10, #6 + vrshr.s16 q11, q11, #6 + vadd.i16 q8, q8, q0 // dc + apply_sign() + vadd.i16 q9, q9, q0 + vadd.i16 q10, q10, q0 + vadd.i16 q11, q11, q0 + vqmovun.s16 d16, q8 // iclip_pixel(dc + apply_sign()) + vqmovun.s16 d17, q9 + vqmovun.s16 d18, q10 + vqmovun.s16 d19, q11 + vst1.8 {d16}, [r0, :64], r1 + vst1.8 {d17}, [r6, :64], r1 + subs r4, r4, #4 + vst1.8 {d18}, [r0, :64], r1 + vst1.8 {d19}, [r6, :64], r1 + bgt L(ipred_cfl_splat_w8) + pop {r4-r8, pc} +L(ipred_cfl_splat_w16): + add r12, r5, r3, lsl #1 + sub r1, r1, r3 + mov lr, r3 +1: + vld1.16 {q8, q9}, [r5, :128]! + vmul.i16 q8, q8, q1 // diff = ac * alpha + vld1.16 {q10, q11}, [r12, :128]! + vmul.i16 q9, q9, q1 + vmul.i16 q10, q10, q1 + vmul.i16 q11, q11, q1 + vshr.s16 q12, q8, #15 // sign = diff >> 15 + vshr.s16 q13, q9, #15 + vshr.s16 q14, q10, #15 + vshr.s16 q15, q11, #15 + vadd.i16 q8, q8, q12 // diff + sign + vadd.i16 q9, q9, q13 + vadd.i16 q10, q10, q14 + vadd.i16 q11, q11, q15 + vrshr.s16 q8, q8, #6 // (diff + sign + 32) >> 6 = apply_sign() + vrshr.s16 q9, q9, #6 + vrshr.s16 q10, q10, #6 + vrshr.s16 q11, q11, #6 + vadd.i16 q8, q8, q0 // dc + apply_sign() + vadd.i16 q9, q9, q0 + vadd.i16 q10, q10, q0 + vadd.i16 q11, q11, q0 + vqmovun.s16 d16, q8 // iclip_pixel(dc + apply_sign()) + vqmovun.s16 d17, q9 + vqmovun.s16 d18, q10 + vqmovun.s16 d19, q11 + subs r3, r3, #16 + vst1.16 {q8}, [r0, :128]! + vst1.16 {q9}, [r6, :128]! + bgt 1b + subs r4, r4, #2 + add r5, r5, lr, lsl #1 + add r12, r12, lr, lsl #1 + add r0, r0, r1 + add r6, r6, r1 + mov r3, lr + bgt 1b + pop {r4-r8, pc} +endfunc + +// void ipred_cfl_top_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha); +function ipred_cfl_top_8bpc_neon, export=1 + push {r4-r8, lr} + ldr r4, [sp, #24] + ldr r5, [sp, #28] + ldr r6, [sp, #32] + clz lr, r3 + adr r12, L(ipred_cfl_top_tbl) + sub lr, lr, #26 + ldr lr, [r12, lr, lsl #2] + vdup.16 q1, r6 // alpha + add r2, r2, #1 + add r12, r12, lr + add r6, r0, r1 + lsl r1, r1, #1 + bx r12 + + .align 2 +L(ipred_cfl_top_tbl): + .word 32f - L(ipred_cfl_top_tbl) + CONFIG_THUMB + .word 16f - L(ipred_cfl_top_tbl) + CONFIG_THUMB + .word 8f - L(ipred_cfl_top_tbl) + CONFIG_THUMB + .word 4f - L(ipred_cfl_top_tbl) + CONFIG_THUMB + +4: + vld1.32 {d0[]}, [r2] + vpaddl.u8 d0, d0 + vpadd.u16 d0, d0 + vrshr.u16 d0, d0, #2 + vdup.16 q0, d0[0] + b L(ipred_cfl_splat_w4) +8: + vld1.8 {d0}, [r2] + vpaddl.u8 d0, d0 + vpadd.u16 d0, d0 + vpadd.u16 d0, d0 + vrshr.u16 d0, d0, #3 + vdup.16 q0, d0[0] + b L(ipred_cfl_splat_w8) +16: + vld1.8 {q0}, [r2] + vaddl.u8 q0, d0, d1 + vadd.u16 d0, d0, d1 + vpadd.u16 d0, d0 + vpadd.u16 d0, d0 + vrshr.u16 d0, d0, #4 + vdup.16 q0, d0[0] + b L(ipred_cfl_splat_w16) +32: + vld1.8 {q2, q3}, [r2] + vaddl.u8 q2, d4, d5 + vaddl.u8 q3, d6, d7 + vadd.u16 q0, q2, q3 + vadd.u16 d0, d0, d1 + vpadd.u16 d0, d0 + vpadd.u16 d0, d0 + vrshr.u16 d0, d0, #5 + vdup.16 q0, d0[0] + b L(ipred_cfl_splat_w16) +endfunc + +// void ipred_cfl_left_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha); +function ipred_cfl_left_8bpc_neon, export=1 + push {r4-r8, lr} + ldr r4, [sp, #24] + ldr r5, [sp, #28] + ldr r6, [sp, #32] + sub r2, r2, r4 + clz lr, r3 + clz r8, r4 + adr r12, L(ipred_cfl_splat_tbl) + adr r7, L(ipred_cfl_left_tbl) + sub lr, lr, #26 + sub r8, r8, #26 + ldr lr, [r12, lr, lsl #2] + ldr r8, [r7, r8, lsl #2] + vdup.16 q1, r6 // alpha + add r12, r12, lr + add r7, r7, r8 + add r6, r0, r1 + lsl r1, r1, #1 + bx r7 + + .align 2 +L(ipred_cfl_left_tbl): + .word L(ipred_cfl_left_h32) - L(ipred_cfl_left_tbl) + CONFIG_THUMB + .word L(ipred_cfl_left_h16) - L(ipred_cfl_left_tbl) + CONFIG_THUMB + .word L(ipred_cfl_left_h8) - L(ipred_cfl_left_tbl) + CONFIG_THUMB + .word L(ipred_cfl_left_h4) - L(ipred_cfl_left_tbl) + CONFIG_THUMB + +L(ipred_cfl_left_h4): + vld1.32 {d0[]}, [r2, :32] + vpaddl.u8 d0, d0 + vpadd.u16 d0, d0 + vrshr.u16 d0, d0, #2 + vdup.16 q0, d0[0] + bx r12 + +L(ipred_cfl_left_h8): + vld1.8 {d0}, [r2, :64] + vpaddl.u8 d0, d0 + vpadd.u16 d0, d0 + vpadd.u16 d0, d0 + vrshr.u16 d0, d0, #3 + vdup.16 q0, d0[0] + bx r12 + +L(ipred_cfl_left_h16): + vld1.8 {q0}, [r2, :128] + vaddl.u8 q0, d0, d1 + vadd.u16 d0, d0, d1 + vpadd.u16 d0, d0 + vpadd.u16 d0, d0 + vrshr.u16 d0, d0, #4 + vdup.16 q0, d0[0] + bx r12 + +L(ipred_cfl_left_h32): + vld1.8 {q2, q3}, [r2, :128] + vaddl.u8 q2, d4, d5 + vaddl.u8 q3, d6, d7 + vadd.u16 q0, q2, q3 + vadd.u16 d0, d0, d1 + vpadd.u16 d0, d0 + vpadd.u16 d0, d0 + vrshr.u16 d0, d0, #5 + vdup.16 q0, d0[0] + bx r12 +endfunc + +// void ipred_cfl_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha); +function ipred_cfl_8bpc_neon, export=1 + push {r4-r8, lr} + ldr r4, [sp, #24] + ldr r5, [sp, #28] + ldr r6, [sp, #32] + sub r2, r2, r4 + add r8, r3, r4 // width + height + vdup.16 q1, r6 // alpha + clz lr, r3 + clz r6, r4 + vdup.16 d16, r8 // width + height + adr r7, L(ipred_cfl_tbl) + rbit r8, r8 // rbit(width + height) + sub lr, lr, #22 // 26 leading bits, minus table offset 4 + sub r6, r6, #26 + clz r8, r8 // ctz(width + height) + ldr lr, [r7, lr, lsl #2] + ldr r6, [r7, r6, lsl #2] + neg r8, r8 // -ctz(width + height) + add r12, r7, lr + add r7, r7, r6 + vshr.u16 d16, d16, #1 // (width + height) >> 1 + vdup.16 d17, r8 // -ctz(width + height) + add r6, r0, r1 + lsl r1, r1, #1 + bx r7 + + .align 2 +L(ipred_cfl_tbl): + .word L(ipred_cfl_h32) - L(ipred_cfl_tbl) + CONFIG_THUMB + .word L(ipred_cfl_h16) - L(ipred_cfl_tbl) + CONFIG_THUMB + .word L(ipred_cfl_h8) - L(ipred_cfl_tbl) + CONFIG_THUMB + .word L(ipred_cfl_h4) - L(ipred_cfl_tbl) + CONFIG_THUMB + .word L(ipred_cfl_w32) - L(ipred_cfl_tbl) + CONFIG_THUMB + .word L(ipred_cfl_w16) - L(ipred_cfl_tbl) + CONFIG_THUMB + .word L(ipred_cfl_w8) - L(ipred_cfl_tbl) + CONFIG_THUMB + .word L(ipred_cfl_w4) - L(ipred_cfl_tbl) + CONFIG_THUMB + +L(ipred_cfl_h4): + vld1.32 {d0[]}, [r2, :32]! + vpaddl.u8 d0, d0 + vpadd.i16 d0, d0 + bx r12 +L(ipred_cfl_w4): + add r2, r2, #1 + vld1.32 {d1[]}, [r2] + vadd.i16 d0, d0, d16 + vpaddl.u8 d1, d1 + vpadd.u16 d1, d1 + cmp r4, #4 + vadd.i16 d0, d0, d1 + vshl.u16 d0, d0, d17 + beq 1f + // h = 8/16 + movw lr, #(0x3334/2) + movw r8, #(0x5556/2) + cmp r4, #16 + it ne + movne lr, r8 + vdup.16 d18, lr + vqdmulh.s16 d0, d0, d18 +1: + vdup.16 q0, d0[0] + b L(ipred_cfl_splat_w4) + +L(ipred_cfl_h8): + vld1.8 {d0}, [r2, :64]! + vpaddl.u8 d0, d0 + vpadd.i16 d0, d0 + vpadd.i16 d0, d0 + bx r12 +L(ipred_cfl_w8): + add r2, r2, #1 + vld1.8 {d1}, [r2] + vadd.i16 d0, d0, d16 + vpaddl.u8 d1, d1 + vpadd.i16 d1, d1 + vpadd.i16 d1, d1 + cmp r4, #8 + vadd.i16 d0, d0, d1 + vshl.u16 d0, d0, d17 + beq 1f + // h = 4/16/32 + cmp r4, #32 + movw lr, #(0x3334/2) + movw r8, #(0x5556/2) + it ne + movne lr, r8 + vdup.16 d18, lr + vqdmulh.s16 d0, d0, d18 +1: + vdup.16 q0, d0[0] + b L(ipred_cfl_splat_w8) + +L(ipred_cfl_h16): + vld1.8 {q0}, [r2, :128]! + vaddl.u8 q0, d0, d1 + vadd.i16 d0, d0, d1 + vpadd.i16 d0, d0 + vpadd.i16 d0, d0 + bx r12 +L(ipred_cfl_w16): + add r2, r2, #1 + vld1.8 {q2}, [r2] + vadd.i16 d0, d0, d16 + vaddl.u8 q2, d4, d5 + vadd.i16 d4, d4, d5 + vpadd.i16 d4, d4 + vpadd.i16 d4, d4 + cmp r4, #16 + vadd.i16 d0, d0, d4 + vshl.u16 d0, d0, d17 + beq 1f + // h = 4/8/32/64 + tst r4, #(32+16+8) // 16 added to make a consecutive bitmask + movw lr, #(0x3334/2) + movw r8, #(0x5556/2) + it ne + movne lr, r8 + vdup.16 d18, lr + vqdmulh.s16 d0, d0, d18 +1: + vdup.16 q0, d0[0] + b L(ipred_cfl_splat_w16) + +L(ipred_cfl_h32): + vld1.8 {q2, q3}, [r2, :128]! + vaddl.u8 q2, d4, d5 + vaddl.u8 q3, d6, d7 + vadd.i16 q0, q2, q3 + vadd.i16 d0, d0, d1 + vpadd.i16 d0, d0 + vpadd.i16 d0, d0 + bx r12 +L(ipred_cfl_w32): + add r2, r2, #1 + vld1.8 {q2, q3}, [r2] + vadd.i16 d0, d0, d16 + vaddl.u8 q2, d4, d5 + vaddl.u8 q3, d6, d7 + vadd.i16 q2, q2, q3 + vadd.i16 d4, d4, d5 + vpadd.i16 d4, d4 + vpadd.i16 d4, d4 + cmp r4, #32 + vadd.i16 d0, d0, d4 + vshl.u16 d0, d0, d17 + beq 1f + // h = 8/16/64 + cmp r4, #8 + movw lr, #(0x3334/2) + movw r8, #(0x5556/2) + it ne + movne lr, r8 + vdup.16 d18, lr + vqdmulh.s16 d0, d0, d18 +1: + vdup.16 q0, d0[0] + b L(ipred_cfl_splat_w16) +endfunc + +// void cfl_ac_420_8bpc_neon(int16_t *const ac, const pixel *const ypx, +// const ptrdiff_t stride, const int w_pad, +// const int h_pad, const int cw, const int ch); +function ipred_cfl_ac_420_8bpc_neon, export=1 + push {r4-r8,lr} + ldr r4, [sp, #24] + ldr r5, [sp, #28] + ldr r6, [sp, #32] + clz r8, r5 + lsl r4, r4, #2 + adr r7, L(ipred_cfl_ac_420_tbl) + sub r8, r8, #27 + ldr r8, [r7, r8, lsl #2] + vmov.i16 q8, #0 + vmov.i16 q9, #0 + vmov.i16 q10, #0 + vmov.i16 q11, #0 + add r7, r7, r8 + sub r8, r6, r4 // height - h_pad + rbit lr, r5 // rbit(width) + rbit r12, r6 // rbit(height) + clz lr, lr // ctz(width) + clz r12, r12 // ctz(height) + add lr, lr, r12 // log2sz + add r12, r1, r2 + vdup.32 d31, lr + lsl r2, r2, #1 + vneg.s32 d31, d31 // -log2sz + bx r7 + + .align 2 +L(ipred_cfl_ac_420_tbl): + .word L(ipred_cfl_ac_420_w16) - L(ipred_cfl_ac_420_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_420_w8) - L(ipred_cfl_ac_420_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_420_w4) - L(ipred_cfl_ac_420_tbl) + CONFIG_THUMB + +L(ipred_cfl_ac_420_w4): +1: // Copy and subsample input + vld1.8 {d0}, [r1, :64], r2 + vld1.8 {d2}, [r12, :64], r2 + vld1.8 {d1}, [r1, :64], r2 + vld1.8 {d3}, [r12, :64], r2 + vpaddl.u8 q0, q0 + vpaddl.u8 q1, q1 + vadd.i16 q0, q0, q1 + vshl.i16 q0, q0, #1 + subs r8, r8, #2 + vst1.16 {q0}, [r0, :128]! + vadd.i16 q8, q8, q0 + bgt 1b + cmp r4, #0 + vmov d0, d1 + vmov d2, d1 + vmov d3, d1 +L(ipred_cfl_ac_420_w4_hpad): + beq 3f // This assumes that all callers already did "cmp r4, #0" +2: // Vertical padding (h_pad > 0) + subs r4, r4, #4 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q8, q8, q1 + bgt 2b +3: +L(ipred_cfl_ac_420_w4_calc_subtract_dc): + // Aggregate the sums + vadd.i16 q0, q8, q9 + vadd.i16 q1, q10, q11 + vpaddl.u16 q0, q0 + vpaddl.u16 q1, q1 + vadd.i32 q0, q1 + vadd.i32 d0, d0, d1 + vpadd.i32 d0, d0, d0 // sum + sub r0, r0, r6, lsl #3 + vrshl.u32 d16, d0, d31 // (sum + (1 << (log2sz - 1))) >>= log2sz + vdup.16 q8, d16[0] +L(ipred_cfl_ac_420_w4_subtract_dc): +6: // Subtract dc from ac + vld1.16 {q0, q1}, [r0, :128] + subs r6, r6, #4 + vsub.i16 q0, q0, q8 + vsub.i16 q1, q1, q8 + vst1.16 {q0, q1}, [r0, :128]! + bgt 6b + pop {r4-r8, pc} + +L(ipred_cfl_ac_420_w8): + cmp r3, #0 + bne L(ipred_cfl_ac_420_w8_wpad) +1: // Copy and subsample input, without padding + vld1.8 {q0}, [r1, :128], r2 + vld1.8 {q1}, [r12, :128], r2 + vld1.8 {q2}, [r1, :128], r2 + vpaddl.u8 q0, q0 + vld1.8 {q3}, [r12, :128], r2 + vpaddl.u8 q1, q1 + vpaddl.u8 q2, q2 + vpaddl.u8 q3, q3 + vadd.i16 q0, q0, q1 + vadd.i16 q2, q2, q3 + vshl.i16 q0, q0, #1 + vshl.i16 q1, q2, #1 + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + bgt 1b + cmp r4, #0 + vmov q0, q1 + b L(ipred_cfl_ac_420_w8_hpad) + +L(ipred_cfl_ac_420_w8_wpad): +1: // Copy and subsample input, padding 4 + vld1.16 {d0}, [r1, :64], r2 + vld1.16 {d2}, [r12, :64], r2 + vld1.16 {d1}, [r1, :64], r2 + vld1.16 {d3}, [r12, :64], r2 + vpaddl.u8 q0, q0 + vpaddl.u8 q1, q1 + vadd.i16 q0, q0, q1 + vshl.i16 q0, q0, #1 + vdup.16 d3, d1[3] + vmov d2, d1 + vdup.16 d1, d0[3] + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + bgt 1b + cmp r4, #0 + vmov q0, q1 + +L(ipred_cfl_ac_420_w8_hpad): + beq 3f // This assumes that all callers already did "cmp r4, #0" +2: // Vertical padding (h_pad > 0) + subs r4, r4, #4 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q10, q10, q0 + vadd.i16 q11, q11, q1 + bgt 2b +3: + + // Double the height and reuse the w4 summing/subtracting + lsl r6, r6, #1 + b L(ipred_cfl_ac_420_w4_calc_subtract_dc) + +L(ipred_cfl_ac_420_w16): + adr r7, L(ipred_cfl_ac_420_w16_tbl) + ldr r3, [r7, r3, lsl #2] + add r7, r7, r3 + bx r7 + + .align 2 +L(ipred_cfl_ac_420_w16_tbl): + .word L(ipred_cfl_ac_420_w16_wpad0) - L(ipred_cfl_ac_420_w16_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_420_w16_wpad1) - L(ipred_cfl_ac_420_w16_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_420_w16_wpad2) - L(ipred_cfl_ac_420_w16_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_420_w16_wpad3) - L(ipred_cfl_ac_420_w16_tbl) + CONFIG_THUMB + +L(ipred_cfl_ac_420_w16_wpad0): +1: // Copy and subsample input, without padding + vld1.8 {q0, q1}, [r1, :128], r2 + vld1.8 {q2, q3}, [r12, :128], r2 + vpaddl.u8 q0, q0 + vld1.8 {q12, q13}, [r1, :128], r2 + vpaddl.u8 q1, q1 + vpaddl.u8 q2, q2 + vpaddl.u8 q3, q3 + vadd.i16 q0, q0, q2 + vadd.i16 q1, q1, q3 + vld1.8 {q2, q3}, [r12, :128], r2 + vpaddl.u8 q12, q12 + vpaddl.u8 q13, q13 + vpaddl.u8 q2, q2 + vpaddl.u8 q3, q3 + vadd.i16 q12, q12, q2 + vadd.i16 q13, q13, q3 + vshl.i16 q0, q0, #1 + vshl.i16 q1, q1, #1 + vshl.i16 q2, q12, #1 + vshl.i16 q3, q13, #1 + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + bgt 1b + cmp r4, #0 + vmov q0, q2 + vmov q1, q3 + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_420_w16_wpad1): +1: // Copy and subsample input, padding 4 + vldr d2, [r1, #16] + vld1.8 {q0}, [r1, :128], r2 + vldr d6, [r12, #16] + vld1.8 {q2}, [r12, :128], r2 + vpaddl.u8 d2, d2 + vldr d26, [r1, #16] + vpaddl.u8 q0, q0 + vld1.8 {q12}, [r1, :128], r2 + vpaddl.u8 d6, d6 + vldr d30, [r12, #16] + vpaddl.u8 q2, q2 + vld1.8 {q14}, [r12, :128], r2 + vpaddl.u8 d26, d26 + vpaddl.u8 q12, q12 + vpaddl.u8 d30, d30 + vpaddl.u8 q14, q14 + vadd.i16 d2, d2, d6 + vadd.i16 q0, q0, q2 + vadd.i16 d26, d26, d30 + vadd.i16 q12, q12, q14 + vshl.i16 d2, d2, #1 + vshl.i16 q0, q0, #1 + vshl.i16 d6, d26, #1 + vshl.i16 q2, q12, #1 + vdup.16 d3, d2[3] + vdup.16 d7, d6[3] + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + bgt 1b + cmp r4, #0 + vmov q0, q2 + vmov q1, q3 + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_420_w16_wpad2): +1: // Copy and subsample input, padding 8 + vld1.8 {q0}, [r1, :128], r2 + vld1.8 {q1}, [r12, :128], r2 + vld1.8 {q2}, [r1, :128], r2 + vpaddl.u8 q0, q0 + vld1.8 {q3}, [r12, :128], r2 + vpaddl.u8 q1, q1 + vpaddl.u8 q2, q2 + vpaddl.u8 q3, q3 + vadd.i16 q0, q0, q1 + vadd.i16 q2, q2, q3 + vshl.i16 q0, q0, #1 + vshl.i16 q2, q2, #1 + vdup.16 q1, d1[3] + vdup.16 q3, d5[3] + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + bgt 1b + cmp r4, #0 + vmov q0, q2 + vmov q1, q3 + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_420_w16_wpad3): +1: // Copy and subsample input, padding 12 + vld1.8 {d0}, [r1, :64], r2 + vld1.8 {d1}, [r12, :64], r2 + vld1.8 {d4}, [r1, :64], r2 + vpaddl.u8 q0, q0 + vld1.8 {d5}, [r12, :64], r2 + vpaddl.u8 q2, q2 + vadd.i16 d0, d0, d1 + vadd.i16 d4, d4, d5 + vshl.i16 d0, d0, #1 + vshl.i16 d4, d4, #1 + vdup.16 q1, d0[3] + vdup.16 q3, d4[3] + vdup.16 d1, d0[3] + vdup.16 d5, d4[3] + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + bgt 1b + cmp r4, #0 + vmov q0, q2 + vmov q1, q3 + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_420_w16_hpad): + beq 3f // This assumes that all callers already did "cmp r4, #0" +2: // Vertical padding (h_pad > 0) + subs r4, r4, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + bgt 2b +3: + + // Quadruple the height and reuse the w4 summing/subtracting + lsl r6, r6, #2 + b L(ipred_cfl_ac_420_w4_calc_subtract_dc) +endfunc + +// void cfl_ac_422_8bpc_neon(int16_t *const ac, const pixel *const ypx, +// const ptrdiff_t stride, const int w_pad, +// const int h_pad, const int cw, const int ch); +function ipred_cfl_ac_422_8bpc_neon, export=1 + push {r4-r8,lr} + ldr r4, [sp, #24] + ldr r5, [sp, #28] + ldr r6, [sp, #32] + clz r8, r5 + lsl r4, r4, #2 + adr r7, L(ipred_cfl_ac_422_tbl) + sub r8, r8, #27 + ldr r8, [r7, r8, lsl #2] + vmov.i16 q8, #0 + vmov.i16 q9, #0 + vmov.i16 q10, #0 + vmov.i16 q11, #0 + add r7, r7, r8 + sub r8, r6, r4 // height - h_pad + rbit lr, r5 // rbit(width) + rbit r12, r6 // rbit(height) + clz lr, lr // ctz(width) + clz r12, r12 // ctz(height) + add lr, lr, r12 // log2sz + add r12, r1, r2 + vdup.32 d31, lr + lsl r2, r2, #1 + vneg.s32 d31, d31 // -log2sz + bx r7 + + .align 2 +L(ipred_cfl_ac_422_tbl): + .word L(ipred_cfl_ac_422_w16) - L(ipred_cfl_ac_422_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_422_w8) - L(ipred_cfl_ac_422_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_422_w4) - L(ipred_cfl_ac_422_tbl) + CONFIG_THUMB + +L(ipred_cfl_ac_422_w4): +1: // Copy and subsample input + vld1.8 {d0}, [r1, :64], r2 + vld1.8 {d1}, [r12, :64], r2 + vld1.8 {d2}, [r1, :64], r2 + vld1.8 {d3}, [r12, :64], r2 + vpaddl.u8 q0, q0 + vpaddl.u8 q1, q1 + vshl.i16 q0, q0, #2 + vshl.i16 q1, q1, #2 + subs r8, r8, #4 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + bgt 1b + cmp r4, #0 + vmov d0, d3 + vmov d1, d3 + vmov d2, d3 + b L(ipred_cfl_ac_420_w4_hpad) + +L(ipred_cfl_ac_422_w8): + cmp r3, #0 + bne L(ipred_cfl_ac_422_w8_wpad) +1: // Copy and subsample input, without padding + vld1.8 {q0}, [r1, :128], r2 + vld1.8 {q1}, [r12, :128], r2 + vld1.8 {q2}, [r1, :128], r2 + vpaddl.u8 q0, q0 + vld1.8 {q3}, [r12, :128], r2 + vpaddl.u8 q1, q1 + vpaddl.u8 q2, q2 + vpaddl.u8 q3, q3 + vshl.i16 q0, q0, #2 + vshl.i16 q1, q1, #2 + vshl.i16 q2, q2, #2 + vshl.i16 q3, q3, #2 + subs r8, r8, #4 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + bgt 1b + cmp r4, #0 + vmov q0, q3 + vmov q1, q3 + b L(ipred_cfl_ac_420_w8_hpad) + +L(ipred_cfl_ac_422_w8_wpad): +1: // Copy and subsample input, padding 4 + vld1.8 {d0}, [r1, :64], r2 + vld1.8 {d1}, [r12, :64], r2 + vld1.8 {d2}, [r1, :64], r2 + vld1.8 {d3}, [r12, :64], r2 + vpaddl.u8 q0, q0 + vpaddl.u8 q1, q1 + vshl.i16 q0, q0, #2 + vshl.i16 q1, q1, #2 + vdup.16 d7, d3[3] + vmov d6, d3 + vdup.16 d5, d2[3] + vmov d4, d2 + vdup.16 d3, d1[3] + vmov d2, d1 + vdup.16 d1, d0[3] + subs r8, r8, #4 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + bgt 1b + cmp r4, #0 + vmov q0, q3 + vmov q1, q3 + b L(ipred_cfl_ac_420_w8_hpad) + +L(ipred_cfl_ac_422_w16): + adr r7, L(ipred_cfl_ac_422_w16_tbl) + ldr r3, [r7, r3, lsl #2] + add r7, r7, r3 + bx r7 + + .align 2 +L(ipred_cfl_ac_422_w16_tbl): + .word L(ipred_cfl_ac_422_w16_wpad0) - L(ipred_cfl_ac_422_w16_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_422_w16_wpad1) - L(ipred_cfl_ac_422_w16_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_422_w16_wpad2) - L(ipred_cfl_ac_422_w16_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_422_w16_wpad3) - L(ipred_cfl_ac_422_w16_tbl) + CONFIG_THUMB + +L(ipred_cfl_ac_422_w16_wpad0): +1: // Copy and subsample input, without padding + vld1.8 {q0, q1}, [r1, :128], r2 + vld1.8 {q2, q3}, [r12, :128], r2 + vpaddl.u8 q0, q0 + vpaddl.u8 q1, q1 + vpaddl.u8 q2, q2 + vpaddl.u8 q3, q3 + vshl.i16 q0, q0, #2 + vshl.i16 q1, q1, #2 + vshl.i16 q2, q2, #2 + vshl.i16 q3, q3, #2 + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + bgt 1b + cmp r4, #0 + vmov q0, q2 + vmov q1, q3 + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_422_w16_wpad1): +1: // Copy and subsample input, padding 4 + vldr d2, [r1, #16] + vld1.8 {q0}, [r1, :128], r2 + vldr d6, [r12, #16] + vld1.8 {q2}, [r12, :128], r2 + vpaddl.u8 d2, d2 + vpaddl.u8 q0, q0 + vpaddl.u8 d6, d6 + vpaddl.u8 q2, q2 + vshl.i16 d2, d2, #2 + vshl.i16 q0, q0, #2 + vshl.i16 d6, d6, #2 + vshl.i16 q2, q2, #2 + vdup.16 d3, d2[3] + vdup.16 d7, d6[3] + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + bgt 1b + cmp r4, #0 + vmov q0, q2 + vmov q1, q3 + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_422_w16_wpad2): +1: // Copy and subsample input, padding 8 + vld1.8 {q0}, [r1, :128], r2 + vld1.8 {q2}, [r12, :128], r2 + vpaddl.u8 q0, q0 + vpaddl.u8 q2, q2 + vshl.i16 q0, q0, #2 + vshl.i16 q2, q2, #2 + vdup.16 q1, d1[3] + vdup.16 q3, d5[3] + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + bgt 1b + cmp r4, #0 + vmov q0, q2 + vmov q1, q3 + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_422_w16_wpad3): +1: // Copy and subsample input, padding 12 + vld1.8 {d0}, [r1, :64], r2 + vld1.8 {d1}, [r12, :64], r2 + vpaddl.u8 q0, q0 + vshl.i16 q0, q0, #2 + vdup.16 q3, d1[3] + vdup.16 q1, d0[3] + vdup.16 d5, d1[3] + vmov d4, d1 + vdup.16 d1, d0[3] + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + bgt 1b + cmp r4, #0 + vmov q0, q2 + vmov q1, q3 + b L(ipred_cfl_ac_420_w16_hpad) +endfunc + +// void cfl_ac_444_8bpc_neon(int16_t *const ac, const pixel *const ypx, +// const ptrdiff_t stride, const int w_pad, +// const int h_pad, const int cw, const int ch); +function ipred_cfl_ac_444_8bpc_neon, export=1 + push {r4-r8,lr} + ldr r4, [sp, #24] + ldr r5, [sp, #28] + ldr r6, [sp, #32] + clz r8, r5 + lsl r4, r4, #2 + adr r7, L(ipred_cfl_ac_444_tbl) + sub r8, r8, #26 + ldr r8, [r7, r8, lsl #2] + vmov.i16 q8, #0 + vmov.i16 q9, #0 + vmov.i16 q10, #0 + vmov.i16 q11, #0 + add r7, r7, r8 + sub r8, r6, r4 // height - h_pad + rbit lr, r5 // rbit(width) + rbit r12, r6 // rbit(height) + clz lr, lr // ctz(width) + clz r12, r12 // ctz(height) + add lr, lr, r12 // log2sz + add r12, r1, r2 + vdup.32 d31, lr + lsl r2, r2, #1 + vneg.s32 d31, d31 // -log2sz + bx r7 + + .align 2 +L(ipred_cfl_ac_444_tbl): + .word L(ipred_cfl_ac_444_w32) - L(ipred_cfl_ac_444_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_444_w16) - L(ipred_cfl_ac_444_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_444_w8) - L(ipred_cfl_ac_444_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_444_w4) - L(ipred_cfl_ac_444_tbl) + CONFIG_THUMB + +L(ipred_cfl_ac_444_w4): +1: // Copy and expand input + vld1.32 {d0[]}, [r1, :32], r2 + vld1.32 {d0[1]}, [r12, :32], r2 + vld1.32 {d2[]}, [r1, :32], r2 + vld1.32 {d2[1]}, [r12, :32], r2 + vshll.u8 q0, d0, #3 + vshll.u8 q1, d2, #3 + subs r8, r8, #4 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + bgt 1b + cmp r4, #0 + vmov d0, d3 + vmov d1, d3 + vmov d2, d3 + b L(ipred_cfl_ac_420_w4_hpad) + +L(ipred_cfl_ac_444_w8): +1: // Copy and expand input + vld1.16 {d0}, [r1, :64], r2 + vld1.16 {d2}, [r12, :64], r2 + vld1.16 {d4}, [r1, :64], r2 + vshll.u8 q0, d0, #3 + vld1.16 {d6}, [r12, :64], r2 + vshll.u8 q1, d2, #3 + vshll.u8 q2, d4, #3 + vshll.u8 q3, d6, #3 + subs r8, r8, #4 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + bgt 1b + cmp r4, #0 + vmov q0, q3 + vmov q1, q3 + b L(ipred_cfl_ac_420_w8_hpad) + +L(ipred_cfl_ac_444_w16): + cmp r3, #0 + bne L(ipred_cfl_ac_444_w16_wpad) +1: // Copy and expand input, without padding + vld1.8 {q1}, [r1, :128], r2 + vld1.8 {q3}, [r12, :128], r2 + vshll.u8 q0, d2, #3 + vshll.u8 q1, d3, #3 + vshll.u8 q2, d6, #3 + vshll.u8 q3, d7, #3 + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + bgt 1b + cmp r4, #0 + vmov q0, q2 + vmov q1, q3 + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_444_w16_wpad): +1: // Copy and expand input, padding 8 + vld1.8 {d0}, [r1, :64], r2 + vld1.8 {d4}, [r12, :64], r2 + vshll.u8 q0, d0, #3 + vshll.u8 q2, d4, #3 + vdup.16 q1, d1[3] + vdup.16 q3, d5[3] + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + bgt 1b + cmp r4, #0 + vmov q0, q2 + vmov q1, q3 + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_444_w32): + adr r7, L(ipred_cfl_ac_444_w32_tbl) + ldr r3, [r7, r3, lsl #1] // (w3>>1) << 2 + add r7, r7, r3 + bx r7 + + .align 2 +L(ipred_cfl_ac_444_w32_tbl): + .word L(ipred_cfl_ac_444_w32_wpad0) - L(ipred_cfl_ac_444_w32_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_444_w32_wpad2) - L(ipred_cfl_ac_444_w32_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_444_w32_wpad4) - L(ipred_cfl_ac_444_w32_tbl) + CONFIG_THUMB + .word L(ipred_cfl_ac_444_w32_wpad6) - L(ipred_cfl_ac_444_w32_tbl) + CONFIG_THUMB + +L(ipred_cfl_ac_444_w32_wpad0): +1: // Copy and expand input, without padding + vld1.8 {q2, q3}, [r1, :128], r2 + vld1.8 {q13, q14}, [r12, :128], r2 + vshll.u8 q0, d4, #3 + vshll.u8 q1, d5, #3 + vshll.u8 q2, d6, #3 + vshll.u8 q3, d7, #3 + vshll.u8 q12, d26, #3 + vshll.u8 q13, d27, #3 + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vshll.u8 q0, d28, #3 + vshll.u8 q1, d29, #3 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + vst1.16 {q12, q13}, [r0, :128]! + vadd.i16 q8, q8, q12 + vadd.i16 q9, q9, q13 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q10, q10, q0 + vadd.i16 q11, q11, q1 + bgt 1b + cmp r4, #0 + b L(ipred_cfl_ac_444_w32_hpad) + +L(ipred_cfl_ac_444_w32_wpad2): +1: // Copy and expand input, padding 8 + vldr d4, [r1, #16] + vld1.8 {q1}, [r1, :128], r2 + vldr d28, [r12, #16] + vld1.8 {q13}, [r12, :128], r2 + vshll.u8 q2, d4, #3 + vshll.u8 q0, d2, #3 + vshll.u8 q1, d3, #3 + vshll.u8 q12, d26, #3 + vshll.u8 q13, d27, #3 + vdup.16 q3, d5[3] + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vshll.u8 q0, d28, #3 + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + vdup.16 q1, d1[3] + vst1.16 {q12, q13}, [r0, :128]! + vadd.i16 q8, q8, q12 + vadd.i16 q9, q9, q13 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q10, q10, q0 + vadd.i16 q11, q11, q1 + bgt 1b + cmp r4, #0 + b L(ipred_cfl_ac_444_w32_hpad) + +L(ipred_cfl_ac_444_w32_wpad4): +1: // Copy and expand input, padding 16 + vld1.8 {q1}, [r1, :128], r2 + vld1.8 {q13}, [r12, :128], r2 + vshll.u8 q0, d2, #3 + vshll.u8 q1, d3, #3 + vshll.u8 q12, d26, #3 + vshll.u8 q13, d27, #3 + vdup.16 q2, d3[3] + vdup.16 q3, d3[3] + subs r8, r8, #2 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vdup.16 q0, d27[3] + vdup.16 q1, d27[3] + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + vst1.16 {q12, q13}, [r0, :128]! + vadd.i16 q8, q8, q12 + vadd.i16 q9, q9, q13 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q10, q10, q0 + vadd.i16 q11, q11, q1 + bgt 1b + cmp r4, #0 + b L(ipred_cfl_ac_444_w32_hpad) + +L(ipred_cfl_ac_444_w32_wpad6): +1: // Copy and expand input, padding 24 + vld1.8 {d0}, [r1, :64], r2 + vld1.8 {d24}, [r12, :64], r2 + vshll.u8 q0, d0, #3 + vshll.u8 q12, d24, #3 + subs r8, r8, #2 + vdup.16 q1, d1[3] + vdup.16 q2, d1[3] + vdup.16 q3, d1[3] + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q8, q8, q0 + vadd.i16 q9, q9, q1 + vdup.16 q13, d25[3] + vdup.16 q0, d25[3] + vdup.16 q1, d25[3] + vst1.16 {q2, q3}, [r0, :128]! + vadd.i16 q10, q10, q2 + vadd.i16 q11, q11, q3 + vst1.16 {q12, q13}, [r0, :128]! + vadd.i16 q8, q8, q12 + vadd.i16 q9, q9, q13 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q10, q10, q0 + vadd.i16 q11, q11, q1 + bgt 1b + cmp r4, #0 + +L(ipred_cfl_ac_444_w32_hpad): + beq 3f // This assumes that all callers already did "cmp r4, #0" +2: // Vertical padding (h_pad > 0) + subs r4, r4, #1 + vst1.16 {q12, q13}, [r0, :128]! + vadd.i16 q8, q8, q12 + vadd.i16 q9, q9, q13 + vst1.16 {q0, q1}, [r0, :128]! + vadd.i16 q10, q10, q0 + vadd.i16 q11, q11, q1 + bgt 2b +3: + + // Multiply the height by eight and reuse the w4 subtracting + lsl r6, r6, #3 + // Aggregate the sums, with wider intermediates earlier than in + // ipred_cfl_ac_420_w8_calc_subtract_dc. + vpaddl.u16 q0, q8 + vpaddl.u16 q1, q9 + vpaddl.u16 q2, q10 + vpaddl.u16 q3, q11 + vadd.i32 q0, q0, q1 + vadd.i32 q2, q2, q3 + vadd.i32 q0, q0, q2 + vadd.i32 d0, d0, d1 + vpadd.i32 d0, d0, d0 // sum + sub r0, r0, r6, lsl #3 + vrshl.u32 d16, d0, d31 // (sum + (1 << (log2sz - 1))) >>= log2sz + vdup.16 q8, d16[0] + b L(ipred_cfl_ac_420_w4_subtract_dc) +endfunc diff --git a/third_party/dav1d/src/arm/32/itx.S b/third_party/dav1d/src/arm/32/itx.S new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/arm/32/itx.S @@ -0,0 +1,3386 @@ +/****************************************************************************** + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2020, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + *****************************************************************************/ + +#include "src/arm/asm.S" +#include "util.S" + +// The exported functions in this file have got the following signature: +// void itxfm_add(pixel *dst, ptrdiff_t dst_stride, coef *coeff, int eob); + +// Most of the functions use the following register layout: +// r0-r3 external parameters +// r4 function pointer to first transform +// r5 function pointer to second transform +// r6 output parameter for helper function +// r7 input parameter for helper function +// r8 input stride for helper function +// r9 scratch variable for helper functions +// r10-r11 pointer to list of eob thresholds, eob threshold value, +// scratch variables within helper functions (backed up) + +// The SIMD registers most often use the following layout: +// d0-d3 multiplication coefficients +// d4-d7 scratch registers +// d8-d15 unused in some transforms, used for scratch registers in others +// d16-v31 inputs/outputs of transforms + +// Potential further optimizations, that are left unimplemented for now: +// - Trying to keep multiplication coefficients in registers across multiple +// transform functions. (The register layout is designed to potentially +// allow this.) +// - Use a simplified version of the transforms themselves for cases where +// we know a significant number of inputs are zero. E.g. if the eob value +// indicates only a quarter of input values are set, for idct16 and up, +// a significant amount of calculation can be skipped, at the cost of more +// code duplication and special casing. + +const idct_coeffs, align=4 + // idct4 + .short 2896, 2896*8, 1567, 3784 + // idct8 + .short 799, 4017, 3406, 2276 + // idct16 + .short 401, 4076, 3166, 2598 + .short 1931, 3612, 3920, 1189 + // idct32 + .short 201, 4091, 3035, 2751 + .short 1751, 3703, 3857, 1380 + .short 995, 3973, 3513, 2106 + .short 2440, 3290, 4052, 601 +endconst + +const idct64_coeffs, align=4 + .short 101*8, 4095*8, 2967*8, -2824*8 + .short 1660*8, 3745*8, 3822*8, -1474*8 + .short 4076, 401, 4017, 799 + + .short 4036*8, -700*8, 2359*8, 3349*8 + .short 3461*8, -2191*8, 897*8, 3996*8 + .short -3166, -2598, -799, -4017 + + .short 501*8, 4065*8, 3229*8, -2520*8 + .short 2019*8, 3564*8, 3948*8, -1092*8 + .short 3612, 1931, 2276, 3406 + + .short 4085*8, -301*8, 2675*8, 3102*8 + .short 3659*8, -1842*8, 1285*8, 3889*8 + .short -3920, -1189, -3406, -2276 +endconst + +const iadst4_coeffs, align=4 + // .h[4-5] can be interpreted as .s[2] + .short 1321, 3803, 2482, 3344, 3344, 0 +endconst + +const iadst8_coeffs, align=4 + .short 4076, 401, 3612, 1931 + .short 2598, 3166, 1189, 3920 + // idct_coeffs + .short 2896, 0, 1567, 3784, 0, 0, 0, 0 +endconst + +const iadst16_coeffs, align=4 + .short 4091, 201, 3973, 995 + .short 3703, 1751, 3290, 2440 + .short 2751, 3035, 2106, 3513 + .short 1380, 3857, 601, 4052 +endconst + +.macro vmull_vmlal d0, s0, s1, c0, c1 + vmull.s16 \d0, \s0, \c0 + vmlal.s16 \d0, \s1, \c1 +.endm + +.macro vmull_vmlal_8h d0, d1, s0, s1, s2, s3, c0, c1 + vmull.s16 \d0, \s0, \c0 + vmlal.s16 \d0, \s2, \c1 + vmull.s16 \d1, \s1, \c0 + vmlal.s16 \d1, \s3, \c1 +.endm + +.macro vmull_vmlsl d0, s0, s1, c0, c1 + vmull.s16 \d0, \s0, \c0 + vmlsl.s16 \d0, \s1, \c1 +.endm + +.macro vmull_vmlsl_8h d0, d1, s0, s1, s2, s3, c0, c1 + vmull.s16 \d0, \s0, \c0 + vmlsl.s16 \d0, \s2, \c1 + vmull.s16 \d1, \s1, \c0 + vmlsl.s16 \d1, \s3, \c1 +.endm + +.macro vrshrn_8h d0, d1, s0, s1, shift + vrshrn.i32 \d0, \s0, \shift + vrshrn.i32 \d1, \s1, \shift +.endm + +.macro scale_input c, r0, r1, r2 r3, r4, r5, r6, r7 + vqrdmulh.s16 \r0, \r0, \c + vqrdmulh.s16 \r1, \r1, \c +.ifnb \r2 + vqrdmulh.s16 \r2, \r2, \c + vqrdmulh.s16 \r3, \r3, \c +.endif +.ifnb \r4 + vqrdmulh.s16 \r4, \r4, \c + vqrdmulh.s16 \r5, \r5, \c + vqrdmulh.s16 \r6, \r6, \c + vqrdmulh.s16 \r7, \r7, \c +.endif +.endm + +.macro load_add_store load, shift, addsrc, adddst, narrowsrc, narrowdst, store, dst, src, shiftbits=4 +.ifnb \load + vld1.8 {\load}, [\src, :64], r1 +.endif +.ifnb \shift + vrshr.s16 \shift, \shift, #\shiftbits +.endif +.ifnb \addsrc + vaddw.u8 \adddst, \adddst, \addsrc +.endif +.ifnb \narrowsrc + vqmovun.s16 \narrowdst, \narrowsrc +.endif +.ifnb \store + vst1.8 {\store}, [\dst, :64], r1 +.endif +.endm +.macro load_add_store_8x8 dst, src, shiftbits=4 + mov \src, \dst + load_add_store d2, q8, , , , , , \dst, \src, \shiftbits + load_add_store d3, q9, , , , , , \dst, \src, \shiftbits + load_add_store d4, q10, d2, q8, , , , \dst, \src, \shiftbits + load_add_store d5, q11, d3, q9, q8, d2, , \dst, \src, \shiftbits + load_add_store d6, q12, d4, q10, q9, d3, d2, \dst, \src, \shiftbits + load_add_store d7, q13, d5, q11, q10, d4, d3, \dst, \src, \shiftbits + load_add_store d2, q14, d6, q12, q11, d5, d4, \dst, \src, \shiftbits + load_add_store d3, q15, d7, q13, q12, d6, d5, \dst, \src, \shiftbits + load_add_store , , d2, q14, q13, d7, d6, \dst, \src, \shiftbits + load_add_store , , d3, q15, q14, d2, d7, \dst, \src, \shiftbits + load_add_store , , , , q15, d3, d2, \dst, \src, \shiftbits + load_add_store , , , , , , d3, \dst, \src, \shiftbits +.endm +.macro load_add_store_8x4 dst, src + mov \src, \dst + load_add_store d2, q8, , , , , , \dst, \src + load_add_store d3, q9, , , , , , \dst, \src + load_add_store d4, q10, d2, q8, , , , \dst, \src + load_add_store d5, q11, d3, q9, q8, d2, , \dst, \src + load_add_store , , d4, q10, q9, d3, d2, \dst, \src + load_add_store , , d5, q11, q10, d4, d3, \dst, \src + load_add_store , , , , q11, d5, d4, \dst, \src + load_add_store , , , , , , d5, \dst, \src +.endm +.macro load_add_store4 load, shift, addsrc, adddst, narrowsrc, narrowdst, store, dst, src +.ifnb \load + vld1.32 {\load[0]}, [\src, :32], r1 +.endif +.ifnb \shift + vrshr.s16 \shift, \shift, #4 +.endif +.ifnb \load + vld1.32 {\load[1]}, [\src, :32], r1 +.endif +.ifnb \addsrc + vaddw.u8 \adddst, \adddst, \addsrc +.endif +.ifnb \store + vst1.32 {\store[0]}, [\dst, :32], r1 +.endif +.ifnb \narrowsrc + vqmovun.s16 \narrowdst, \narrowsrc +.endif +.ifnb \store + vst1.32 {\store[1]}, [\dst, :32], r1 +.endif +.endm +.macro load_add_store_4x16 dst, src + mov \src, \dst + load_add_store4 d0, , , , , , , \dst, \src + load_add_store4 d1, q8, , , , , , \dst, \src + load_add_store4 d2, q9, d0, q8, , , , \dst, \src + load_add_store4 d3, q10, d1, q9, q8, d0, , \dst, \src + load_add_store4 d4, q11, d2, q10, q9, d1, d0, \dst, \src + load_add_store4 d5, q12, d3, q11, q10, d2, d1, \dst, \src + load_add_store4 d6, q13, d4, q12, q11, d3, d2, \dst, \src + load_add_store4 d7, q14, d5, q13, q12, d4, d3, \dst, \src + load_add_store4 , q15, d6, q14, q13, d5, d4, \dst, \src + load_add_store4 , , d7, q15, q14, d6, d5, \dst, \src + load_add_store4 , , , , q15, d7, d6, \dst, \src + load_add_store4 , , , , , , d7, \dst, \src +.endm +.macro load_add_store_4x8 dst, src + mov \src, \dst + load_add_store4 d0, , , , , , , \dst, \src + load_add_store4 d1, q8, , , , , , \dst, \src + load_add_store4 d2, q9, d0, q8, , , , \dst, \src + load_add_store4 d3, q10, d1, q9, q8, d0, , \dst, \src + load_add_store4 , q11, d2, q10, q9, d1, d0, \dst, \src + load_add_store4 , , d3, q11, q10, d2, d1, \dst, \src + load_add_store4 , , , , q11, d3, d2, \dst, \src + load_add_store4 , , , , , , d3, \dst, \src +.endm + +.macro idct_dc w, h, shift + cmp r3, #0 + bne 1f + vmov.i16 d30, #0 + movw r12, #2896*8 + vld1.16 {d16[]}, [r2, :16] + vdup.16 d0, r12 + vqrdmulh.s16 d16, d16, d0[0] + vst1.16 {d30[0]}, [r2, :16] +.if (\w == 2*\h) || (2*\w == \h) + vqrdmulh.s16 d16, d16, d0[0] +.endif +.if \shift > 0 + vrshr.s16 d16, d16, #\shift +.endif + vqrdmulh.s16 d20, d16, d0[0] + mov r3, #\h + vrshr.s16 d16, d20, #4 + vrshr.s16 d17, d20, #4 + b idct_dc_w\w\()_neon +1: +.endm + +function idct_dc_w4_neon +1: + vld1.32 {d0[0]}, [r0, :32], r1 + vld1.32 {d0[1]}, [r0, :32], r1 + vld1.32 {d1[0]}, [r0, :32], r1 + vld1.32 {d1[1]}, [r0, :32], r1 + subs r3, r3, #4 + sub r0, r0, r1, lsl #2 + vaddw.u8 q10, q8, d0 + vqmovun.s16 d0, q10 + vaddw.u8 q11, q8, d1 + vst1.32 {d0[0]}, [r0, :32], r1 + vqmovun.s16 d1, q11 + vst1.32 {d0[1]}, [r0, :32], r1 + vst1.32 {d1[0]}, [r0, :32], r1 + vst1.32 {d1[1]}, [r0, :32], r1 + bgt 1b + bx lr +endfunc + +function idct_dc_w8_neon +1: + vld1.8 {d0}, [r0, :64], r1 + vld1.8 {d1}, [r0, :64], r1 + vld1.8 {d2}, [r0, :64], r1 + vaddw.u8 q10, q8, d0 + vld1.8 {d3}, [r0, :64], r1 + sub r0, r0, r1, lsl #2 + subs r3, r3, #4 + vaddw.u8 q11, q8, d1 + vqmovun.s16 d0, q10 + vaddw.u8 q12, q8, d2 + vqmovun.s16 d1, q11 + vaddw.u8 q13, q8, d3 + vst1.8 {d0}, [r0, :64], r1 + vqmovun.s16 d2, q12 + vst1.8 {d1}, [r0, :64], r1 + vqmovun.s16 d3, q13 + vst1.8 {d2}, [r0, :64], r1 + vst1.8 {d3}, [r0, :64], r1 + bgt 1b + bx lr +endfunc + +function idct_dc_w16_neon +1: + vld1.8 {q0}, [r0, :128], r1 + vld1.8 {q1}, [r0, :128], r1 + vld1.8 {q2}, [r0, :128], r1 + subs r3, r3, #4 + vaddw.u8 q10, q8, d0 + vaddw.u8 q11, q8, d1 + vld1.8 {q3}, [r0, :128], r1 + vaddw.u8 q12, q8, d2 + vaddw.u8 q13, q8, d3 + sub r0, r0, r1, lsl #2 + vaddw.u8 q14, q8, d4 + vaddw.u8 q15, q8, d5 + vqmovun.s16 d0, q10 + vqmovun.s16 d1, q11 + vaddw.u8 q10, q8, d6 + vaddw.u8 q11, q8, d7 + vqmovun.s16 d2, q12 + vqmovun.s16 d3, q13 + vqmovun.s16 d4, q14 + vqmovun.s16 d5, q15 + vst1.8 {q0}, [r0, :128], r1 + vqmovun.s16 d6, q10 + vqmovun.s16 d7, q11 + vst1.8 {q1}, [r0, :128], r1 + vst1.8 {q2}, [r0, :128], r1 + vst1.8 {q3}, [r0, :128], r1 + bgt 1b + bx lr +endfunc + +function idct_dc_w32_neon +1: + vld1.8 {q0, q1}, [r0, :128], r1 + subs r3, r3, #2 + vld1.8 {q2, q3}, [r0, :128], r1 + vaddw.u8 q10, q8, d0 + vaddw.u8 q11, q8, d1 + vaddw.u8 q12, q8, d2 + vaddw.u8 q13, q8, d3 + sub r0, r0, r1, lsl #1 + vaddw.u8 q14, q8, d4 + vaddw.u8 q15, q8, d5 + vqmovun.s16 d0, q10 + vqmovun.s16 d1, q11 + vaddw.u8 q10, q8, d6 + vaddw.u8 q11, q8, d7 + vqmovun.s16 d2, q12 + vqmovun.s16 d3, q13 + vqmovun.s16 d4, q14 + vqmovun.s16 d5, q15 + vst1.8 {q0, q1}, [r0, :128], r1 + vqmovun.s16 d6, q10 + vqmovun.s16 d7, q11 + vst1.8 {q2, q3}, [r0, :128], r1 + bgt 1b + bx lr +endfunc + +function idct_dc_w64_neon + sub r1, r1, #32 +1: + vld1.8 {q0, q1}, [r0, :128]! + subs r3, r3, #1 + vld1.8 {q2, q3}, [r0, :128] + vaddw.u8 q10, q8, d0 + vaddw.u8 q11, q8, d1 + vaddw.u8 q12, q8, d2 + vaddw.u8 q13, q8, d3 + sub r0, r0, #32 + vaddw.u8 q14, q8, d4 + vaddw.u8 q15, q8, d5 + vqmovun.s16 d0, q10 + vqmovun.s16 d1, q11 + vaddw.u8 q10, q8, d6 + vaddw.u8 q11, q8, d7 + vqmovun.s16 d2, q12 + vqmovun.s16 d3, q13 + vqmovun.s16 d4, q14 + vqmovun.s16 d5, q15 + vst1.8 {q0, q1}, [r0, :128]! + vqmovun.s16 d6, q10 + vqmovun.s16 d7, q11 + vst1.8 {q2, q3}, [r0, :128], r1 + bgt 1b + bx lr +endfunc + +.macro iwht4 + vadd.i16 d16, d16, d17 + vsub.i16 d21, d18, d19 + vsub.i16 d20, d16, d21 + vshr.s16 d20, d20, #1 + vsub.i16 d18, d20, d17 + vsub.i16 d17, d20, d19 + vadd.i16 d19, d21, d18 + vsub.i16 d16, d16, d17 +.endm + +.macro idct_4h_x4 r0, r1, r2, r3 + vmull_vmlal q3, \r1, \r3, d0[3], d0[2] + vmull_vmlsl q2, \r1, \r3, d0[2], d0[3] + vmull_vmlal q1, \r0, \r2, d0[0], d0[0] + vrshrn.i32 d6, q3, #12 + vrshrn.i32 d7, q2, #12 + vmull_vmlsl q2, \r0, \r2, d0[0], d0[0] + vrshrn.i32 d2, q1, #12 + vrshrn.i32 d3, q2, #12 + vqadd.s16 \r0, d2, d6 + vqsub.s16 \r3, d2, d6 + vqadd.s16 \r1, d3, d7 + vqsub.s16 \r2, d3, d7 +.endm + +.macro idct_8h_x4 q0, q1, q2, q3, r0, r1, r2, r3, r4, r5, r6, r7 + vmull_vmlal_8h q6, q7, \r2, \r3, \r6, \r7, d0[3], d0[2] + vmull_vmlsl_8h q4, q5, \r2, \r3, \r6, \r7, d0[2], d0[3] + vmull_vmlal_8h q2, q3, \r0, \r1, \r4, \r5, d0[0], d0[0] + vrshrn_8h d12, d13, q6, q7, #12 + vrshrn_8h d14, d15, q4, q5, #12 + vmull_vmlsl_8h q4, q5, \r0, \r1, \r4, \r5, d0[0], d0[0] + vrshrn_8h d4, d5, q2, q3, #12 + vrshrn_8h d6, d7, q4, q5, #12 + vqadd.s16 \q0, q2, q6 + vqsub.s16 \q3, q2, q6 + vqadd.s16 \q1, q3, q7 + vqsub.s16 \q2, q3, q7 +.endm + +function inv_dct_4h_x4_neon, export=1 + movrel_local r12, idct_coeffs + vld1.16 {d0}, [r12, :64] + idct_4h_x4 d16, d17, d18, d19 + bx lr +endfunc + +function inv_dct_8h_x4_neon, export=1 + movrel_local r12, idct_coeffs + vld1.16 {d0}, [r12, :64] + idct_8h_x4 q8, q9, q10, q11, d16, d17, d18, d19, d20, d21, d22, d23 + bx lr +endfunc + +.macro iadst_4x4 o0, o1, o2, o3 + movrel_local r12, iadst4_coeffs + vld1.16 {d0, d1}, [r12, :128] + + vsubl.s16 q1, d16, d18 + vmull.s16 q2, d16, d0[0] + vmlal.s16 q2, d18, d0[1] + vmlal.s16 q2, d19, d0[2] + vmull.s16 q10, d17, d0[3] + vaddw.s16 q1, q1, d19 + vmull.s16 q3, d16, d0[2] + vmlsl.s16 q3, d18, d0[0] + vmlsl.s16 q3, d19, d0[1] + + vadd.s32 q11, q2, q3 + vmul.s32 q1, q1, d1[0] + vadd.s32 q2, q2, q10 + vadd.s32 q3, q3, q10 + vsub.s32 q11, q11, q10 + + vrshrn.i32 \o0, q2, #12 + vrshrn.i32 \o2, q1, #12 + vrshrn.i32 \o1, q3, #12 + vrshrn.i32 \o3, q11, #12 +.endm + +function inv_adst_4h_x4_neon, export=1 + iadst_4x4 d16, d17, d18, d19 + bx lr +endfunc + +function inv_flipadst_4h_x4_neon, export=1 + iadst_4x4 d19, d18, d17, d16 + bx lr +endfunc + +.macro iadst_8x4 o0, o1, o2, o3, o4, o5, o6, o7 + movrel_local r12, iadst4_coeffs + vld1.16 {d0, d1}, [r12, :128] + + vsubl.s16 q2, d16, d20 + vsubl.s16 q3, d17, d21 + vmull.s16 q4, d16, d0[0] + vmlal.s16 q4, d20, d0[1] + vmlal.s16 q4, d22, d0[2] + vmull.s16 q5, d17, d0[0] + vmlal.s16 q5, d21, d0[1] + vmlal.s16 q5, d23, d0[2] + vaddw.s16 q2, q2, d22 + vaddw.s16 q3, q3, d23 + vmull.s16 q6, d16, d0[2] + vmlsl.s16 q6, d20, d0[0] + vmlsl.s16 q6, d22, d0[1] + vmull.s16 q7, d17, d0[2] + vmlsl.s16 q7, d21, d0[0] + vmlsl.s16 q7, d23, d0[1] + + vmul.s32 q10, q2, d1[0] + vmul.s32 q11, q3, d1[0] + + vmull.s16 q2, d18, d0[3] + vmull.s16 q3, d19, d0[3] + + vadd.s32 q8, q4, q2 // out0 + vadd.s32 q9, q5, q3 + + vadd.s32 q4, q4, q6 // out3 + vadd.s32 q5, q5, q7 + + vadd.s32 q6, q6, q2 // out1 + vadd.s32 q7, q7, q3 + + vsub.s32 q4, q4, q2 // out3 + vsub.s32 q5, q5, q3 + + vrshrn.i32 d20, q10, #12 + vrshrn.i32 d21, q11, #12 + + vrshrn.i32 \o0, q8, #12 + vrshrn.i32 \o1, q9, #12 + +.ifc \o4, d18 + vmov q9, q10 +.endif + + vrshrn.i32 \o2, q6, #12 + vrshrn.i32 \o3, q7, #12 + + vrshrn.i32 \o6, q4, #12 + vrshrn.i32 \o7, q5, #12 +.endm + +function inv_adst_8h_x4_neon, export=1 + iadst_8x4 d16, d17, d18, d19, d20, d21, d22, d23 + bx lr +endfunc + +function inv_flipadst_8h_x4_neon, export=1 + iadst_8x4 d22, d23, d20, d21, d18, d19, d16, d17 + bx lr +endfunc + +function inv_identity_4h_x4_neon, export=1 + movw r12, #(5793-4096)*8 + vdup.16 d0, r12 + vqrdmulh.s16 q2, q8, d0[0] + vqrdmulh.s16 q3, q9, d0[0] + vqadd.s16 q8, q8, q2 + vqadd.s16 q9, q9, q3 + bx lr +endfunc + +function inv_identity_8h_x4_neon, export=1 + movw r12, #(5793-4096)*8 + vdup.16 d0, r12 + vqrdmulh.s16 q1, q8, d0[0] + vqrdmulh.s16 q2, q9, d0[0] + vqrdmulh.s16 q3, q10, d0[0] + vqadd.s16 q8, q8, q1 + vqrdmulh.s16 q1, q11, d0[0] + vqadd.s16 q9, q9, q2 + vqadd.s16 q10, q10, q3 + vqadd.s16 q11, q11, q1 + bx lr +endfunc + +.macro identity_8x4_shift1 r0, r1, r2, r3, c +.irp i, \r0, \r1, \r2, \r3 + vqrdmulh.s16 q1, \i, \c + vrhadd.s16 \i, \i, q1 +.endr +.endm + +function inv_txfm_add_wht_wht_4x4_8bpc_neon, export=1 + push {r4-r5,lr} + vmov.i16 q15, #0 + vld1.16 {d16, d17, d18, d19}, [r2, :128] + vst1.16 {q15}, [r2, :128]! + + vshr.s16 q8, q8, #2 + vshr.s16 q9, q9, #2 + + iwht4 + + vst1.16 {q15}, [r2, :128]! + transpose_4x4h q8, q9, d16, d17, d18, d19 + + iwht4 + + vld1.32 {d0[]}, [r0, :32], r1 + vld1.32 {d0[1]}, [r0, :32], r1 + vld1.32 {d1[]}, [r0, :32], r1 + vld1.32 {d1[1]}, [r0, :32], r1 + + b L(itx_4x4_end) +endfunc + +function inv_txfm_add_4x4_neon + vmov.i16 q15, #0 + vld1.16 {d16, d17, d18, d19}, [r2, :128] + vst1.16 {q15}, [r2, :128]! + + blx r4 + + vst1.16 {q15}, [r2, :128]! + transpose_4x4h q8, q9, d16, d17, d18, d19 + + blx r5 + + vld1.32 {d0[]}, [r0, :32], r1 + vld1.32 {d0[1]}, [r0, :32], r1 + vld1.32 {d1[]}, [r0, :32], r1 + vld1.32 {d1[1]}, [r0, :32], r1 + vrshr.s16 q8, q8, #4 + vrshr.s16 q9, q9, #4 + +L(itx_4x4_end): + sub r0, r0, r1, lsl #2 + vaddw.u8 q8, q8, d0 + vqmovun.s16 d0, q8 + vaddw.u8 q9, q9, d1 + vst1.32 {d0[0]}, [r0, :32], r1 + vqmovun.s16 d1, q9 + vst1.32 {d0[1]}, [r0, :32], r1 + vst1.32 {d1[0]}, [r0, :32], r1 + vst1.32 {d1[1]}, [r0, :32], r1 + + pop {r4-r5,pc} +endfunc + +.macro def_fn_4x4 txfm1, txfm2 +function inv_txfm_add_\txfm1\()_\txfm2\()_4x4_8bpc_neon, export=1 + push {r4-r5,lr} + +.ifc \txfm1\()_\txfm2, dct_dct + cmp r3, #0 + bne 1f + vmov.i16 d30, #0 + movw r12, #2896*8 + vld1.16 {d16[]}, [r2, :16] + vdup.16 d4, r12 + vst1.16 {d30[0]}, [r2, :16] + vqrdmulh.s16 d16, d16, d4[0] + vld1.32 {d0[0]}, [r0, :32], r1 + vqrdmulh.s16 d20, d16, d4[0] + vld1.32 {d0[1]}, [r0, :32], r1 + vrshr.s16 d16, d20, #4 + vrshr.s16 d17, d20, #4 + vld1.32 {d1[0]}, [r0, :32], r1 + vmov q9, q8 + vld1.32 {d1[1]}, [r0, :32], r1 + b L(itx_4x4_end) +1: +.endif + movrel_local r4, inv_\txfm1\()_4h_x4_neon + movrel_local r5, inv_\txfm2\()_4h_x4_neon + b inv_txfm_add_4x4_neon +endfunc +.endm + +def_fn_4x4 dct, dct +def_fn_4x4 identity, identity +def_fn_4x4 dct, adst +def_fn_4x4 dct, flipadst +def_fn_4x4 dct, identity +def_fn_4x4 adst, dct +def_fn_4x4 adst, adst +def_fn_4x4 adst, flipadst +def_fn_4x4 flipadst, dct +def_fn_4x4 flipadst, adst +def_fn_4x4 flipadst, flipadst +def_fn_4x4 identity, dct + +def_fn_4x4 adst, identity +def_fn_4x4 flipadst, identity +def_fn_4x4 identity, adst +def_fn_4x4 identity, flipadst + +.macro idct_8h_x8 q0, q1, q2, q3, q4, q5, q6, q7, r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, r14, r15 + idct_8h_x4 \q0, \q2, \q4, \q6, \r0, \r1, \r4, \r5, \r8, \r9, \r12, \r13 + + vmull_vmlsl_8h q2, q3, \r2, \r3, \r14, \r15, d1[0], d1[1] // -> t4a + vmull_vmlal_8h q4, q5, \r2, \r3, \r14, \r15, d1[1], d1[0] // -> t7a + vmull_vmlsl_8h q6, q7, \r10, \r11, \r6, \r7, d1[2], d1[3] // -> t5a + vrshrn_8h \r2, \r3, q2, q3, #12 // t4a + vrshrn_8h \r14, \r15, q4, q5, #12 // t7a + vmull_vmlal_8h q2, q3, \r10, \r11, \r6, \r7, d1[3], d1[2] // -> t6a + vrshrn_8h \r6, \r7, q6, q7, #12 // t5a + vrshrn_8h \r10, \r11, q2, q3, #12 // taa + + vqadd.s16 q2, \q1, \q3 // t4 + vqsub.s16 \q1, \q1, \q3 // t5a + vqadd.s16 q3, \q7, \q5 // t7 + vqsub.s16 \q3, \q7, \q5 // t6a + + vmull_vmlsl_8h q4, q5, \r6, \r7, \r2, \r3, d0[0], d0[0] // -> t5 + vmull_vmlal_8h q6, q7, \r6, \r7, \r2, \r3, d0[0], d0[0] // -> t6 + vrshrn_8h d8, d9, q4, q5, #12 // t5 + vrshrn_8h d10, d11, q6, q7, #12 // t6 + + vqsub.s16 \q7, \q0, q3 // out7 + vqadd.s16 \q0, \q0, q3 // out0 + vqadd.s16 \q1, \q2, q5 // out1 + vqsub.s16 q6, \q2, q5 // out6 + vqadd.s16 \q2, \q4, q4 // out2 + vqsub.s16 \q5, \q4, q4 // out5 + vqadd.s16 \q3, \q6, q2 // out3 + vqsub.s16 \q4, \q6, q2 // out4 + vmov \q6, q6 // out6 +.endm + +.macro idct_4h_x8 r0, r1, r2, r3, r4, r5, r6, r7 + idct_4h_x4 \r0, \r2, \r4, \r6 + + vmull_vmlsl q1, \r1, \r7, d1[0], d1[1] // -> t4a + vmull_vmlal q2, \r1, \r7, d1[1], d1[0] // -> t7a + vmull_vmlsl q3, \r5, \r3, d1[2], d1[3] // -> t5a + vrshrn.i32 \r1, q1, #12 // t4a + vmull_vmlal q1, \r5, \r3, d1[3], d1[2] // -> t6a + vrshrn.i32 \r7, q2, #12 // t7a + vrshrn.i32 \r3, q3, #12 // t5a + vrshrn.i32 \r5, q1, #12 // taa + + vqadd.s16 d2, \r1, \r3 // t4 + vqsub.s16 \r1, \r1, \r3 // t5a + vqadd.s16 d3, \r7, \r5 // t7 + vqsub.s16 \r3, \r7, \r5 // t6a + + vmull_vmlsl q2, \r3, \r1, d0[0], d0[0] // -> t5 + vmull_vmlal q3, \r3, \r1, d0[0], d0[0] // -> t6 + vrshrn.i32 d4, q2, #12 // t5 + vrshrn.i32 d5, q3, #12 // t6 + + vqsub.s16 \r7, \r0, d3 // out7 + vqadd.s16 \r0, \r0, d3 // out0 + vqadd.s16 \r1, \r2, d5 // out1 + vqsub.s16 d6, \r2, d5 // out6 + vqadd.s16 \r2, \r4, d4 // out2 + vqsub.s16 \r5, \r4, d4 // out5 + vqadd.s16 \r3, \r6, d2 // out3 + vqsub.s16 \r4, \r6, d2 // out4 + vmov \r6, d6 // out6 +.endm + +function inv_dct_8h_x8_neon, export=1 + movrel_local r12, idct_coeffs + vld1.16 {q0}, [r12, :128] + idct_8h_x8 q8, q9, q10, q11, q12, q13, q14, q15, d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, d28, d29, d30, d31 + bx lr +endfunc + +function inv_dct_4h_x8_neon, export=1 + movrel_local r12, idct_coeffs + vld1.16 {q0}, [r12, :128] + idct_4h_x8 d16, d17, d18, d19, d20, d21, d22, d23 + bx lr +endfunc + +.macro iadst_8h_x8 q0, q1, q2, q3, q4, q5, q6, q7, r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, r10, r11, r12, r13, r14, r15 + movrel_local r12, iadst8_coeffs + vld1.16 {d0, d1, d2}, [r12, :64] + + vmull_vmlal_8h q2, q3, d30, d31, d16, d17, d0[0], d0[1] + vmull_vmlsl_8h q4, q5, d30, d31, d16, d17, d0[1], d0[0] + vmull_vmlal_8h q6, q7, d26, d27, d20, d21, d0[2], d0[3] + vrshrn_8h d16, d17, q2, q3, #12 // t0a + vrshrn_8h d30, d31, q4, q5, #12 // t1a + vmull_vmlsl_8h q2, q3, d26, d27, d20, d21, d0[3], d0[2] + vmull_vmlal_8h q4, q5, d22, d23, d24, d25, d1[0], d1[1] + vrshrn_8h d20, d21, q6, q7, #12 // t2a + vrshrn_8h d26, d27, q2, q3, #12 // t3a + vmull_vmlsl_8h q6, q7, d22, d23, d24, d25, d1[1], d1[0] + vmull_vmlal_8h q2, q3, d18, d19, d28, d29, d1[2], d1[3] + vrshrn_8h d24, d25, q4, q5, #12 // t4a + vrshrn_8h d22, d23, q6, q7, #12 // t5a + vmull_vmlsl_8h q4, q5, d18, d19, d28, d29, d1[3], d1[2] + vrshrn_8h d28, d29, q2, q3, #12 // t6a + vrshrn_8h d18, d19, q4, q5, #12 // t7a + + vqadd.s16 q2, q8, q12 // t0 + vqsub.s16 q3, q8, q12 // t4 + vqadd.s16 q4, q15, q11 // t1 + vqsub.s16 q5, q15, q11 // t5 + vqadd.s16 q6, q10, q14 // t2 + vqsub.s16 q7, q10, q14 // t6 + vqadd.s16 q10, q13, q9 // t3 + vqsub.s16 q11, q13, q9 // t7 + + vmull_vmlal_8h q8, q9, d6, d7, d10, d11, d2[3], d2[2] + vmull_vmlsl_8h q12, q13, d6, d7, d10, d11, d2[2], d2[3] + vmull_vmlsl_8h q14, q15, d22, d23, d14, d15, d2[3], d2[2] + + vrshrn_8h d6, d7, q8, q9, #12 // t4a + vrshrn_8h d10, d11, q12, q13, #12 // t5a + + vmull_vmlal_8h q8, q9, d22, d23, d14, d15, d2[2], d2[3] + + vrshrn_8h d14, d15, q14, q15, #12 // t6a + vrshrn_8h d22, d23, q8, q9, #12 // t7a + + vqadd.s16 \q0, q2, q6 // out0 + vqsub.s16 q2, q2, q6 // t2 + vqadd.s16 \q7, q4, q10 // out7 + vqsub.s16 q4, q4, q10 // t3 + vqneg.s16 \q7, \q7 // out7 + + vqadd.s16 \q1, q3, q7 // out1 + vqsub.s16 q3, q3, q7 // t6 + vqadd.s16 \q6, q5, q11 // out6 + vqsub.s16 q5, q5, q11 // t7 + vqneg.s16 \q1, \q1 // out1 + + vmull_vmlal_8h q10, q11, d4, d5, d8, d9, d2[0], d2[0] // -> out3 (q11 or q12) + vmull_vmlsl_8h q6, q7, d4, d5, d8, d9, d2[0], d2[0] // -> out4 (q12 or q11) + vmull_vmlsl_8h q12, q13, d6, d7, d10, d11, d2[0], d2[0] // -> out5 (q13 or q10) + vrshrn_8h d4, d5, q10, q11, #12 // out3 + vmull_vmlal_8h q10, q11, d6, d7, d10, d11, d2[0], d2[0] // -> out2 (q10 or q13) + vrshrn_8h d6, d7, q12, q13, #12 // out5 + vrshrn_8h \r4, \r5, q10, q11, #12 // out2 (q10 or q13) + vrshrn_8h \r8, \r9, q6, q7, #12 // out4 (q12 or q11) + + vqneg.s16 \q3, q2 // out3 + vqneg.s16 \q5, q3 // out5 +.endm + +.macro iadst_4h_x8 r0, r1, r2, r3, r4, r5, r6, r7 + movrel_local r12, iadst8_coeffs + vld1.16 {d0, d1, d2}, [r12, :64] + + vmull_vmlal q2, d23, d16, d0[0], d0[1] + vmull_vmlsl q3, d23, d16, d0[1], d0[0] + vmull_vmlal q4, d21, d18, d0[2], d0[3] + vrshrn.i32 d16, q2, #12 // t0a + vrshrn.i32 d23, q3, #12 // t1a + vmull_vmlsl q5, d21, d18, d0[3], d0[2] + vmull_vmlal q6, d19, d20, d1[0], d1[1] + vrshrn.i32 d18, q4, #12 // t2a + vrshrn.i32 d21, q5, #12 // t3a + vmull_vmlsl q7, d19, d20, d1[1], d1[0] + vmull_vmlal q2, d17, d22, d1[2], d1[3] + vrshrn.i32 d20, q6, #12 // t4a + vrshrn.i32 d19, q7, #12 // t5a + vmull_vmlsl q3, d17, d22, d1[3], d1[2] + vrshrn.i32 d22, q2, #12 // t6a + vrshrn.i32 d17, q3, #12 // t7a + + vqadd.s16 d4, d16, d20 // t0 + vqsub.s16 d5, d16, d20 // t4 + vqadd.s16 d6, d23, d19 // t1 + vqsub.s16 d7, d23, d19 // t5 + vqadd.s16 d8, d18, d22 // t2 + vqsub.s16 d9, d18, d22 // t6 + vqadd.s16 d18, d21, d17 // t3 + vqsub.s16 d19, d21, d17 // t7 + + vmull_vmlal q8, d5, d7, d2[3], d2[2] + vmull_vmlsl q10, d5, d7, d2[2], d2[3] + vmull_vmlsl q11, d19, d9, d2[3], d2[2] + + vrshrn.i32 d5, q8, #12 // t4a + vrshrn.i32 d7, q10, #12 // t5a + + vmull_vmlal q8, d19, d9, d2[2], d2[3] + + vrshrn.i32 d9, q11, #12 // t6a + vrshrn.i32 d19, q8, #12 // t7a + + vqadd.s16 \r0, d4, d8 // out0 + vqsub.s16 d4, d4, d8 // t2 + vqadd.s16 \r7, d6, d18 // out7 + vqsub.s16 d6, d6, d18 // t3 + vqneg.s16 \r7, \r7 // out7 + + vqadd.s16 \r1, d5, d9 // out1 + vqsub.s16 d5, d5, d9 // t6 + vqadd.s16 \r6, d7, d19 // out6 + vqsub.s16 d7, d7, d19 // t7 + vqneg.s16 \r1, \r1 // out1 + + vmull_vmlal q9, d4, d6, d2[0], d2[0] // -> out3 (d19 or d20) + vmull_vmlsl q4, d4, d6, d2[0], d2[0] // -> out4 (d20 or d19) + vmull_vmlsl q10, d5, d7, d2[0], d2[0] // -> out5 (d21 or d18) + vrshrn.i32 d4, q9, #12 // out3 + vmull_vmlal q9, d5, d7, d2[0], d2[0] // -> out2 (d18 or d21) + vrshrn.i32 d5, q10, #12 // out5 + vrshrn.i32 \r2, q9, #12 // out2 (d18 or d21) + vrshrn.i32 \r4, q4, #12 // out4 (d20 or d19) + + vqneg.s16 \r3, d4 // out3 + vqneg.s16 \r5, d5 // out5 +.endm + +function inv_adst_8h_x8_neon, export=1 + iadst_8h_x8 q8, q9, q10, q11, q12, q13, q14, q15, d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, d28, d29, d30, d31 + bx lr +endfunc + +function inv_flipadst_8h_x8_neon, export=1 + iadst_8h_x8 q15, q14, q13, q12, q11, q10, q9, q8, d30, d31, d28, d29, d26, d27, d24, d25, d22, d23, d20, d21, d18, d19, d16, d17 + bx lr +endfunc + +function inv_adst_4h_x8_neon, export=1 + iadst_4h_x8 d16, d17, d18, d19, d20, d21, d22, d23 + bx lr +endfunc + +function inv_flipadst_4h_x8_neon, export=1 + iadst_4h_x8 d23, d22, d21, d20, d19, d18, d17, d16 + bx lr +endfunc + +function inv_identity_8h_x8_neon, export=1 + vqshl.s16 q8, q8, #1 + vqshl.s16 q9, q9, #1 + vqshl.s16 q10, q10, #1 + vqshl.s16 q11, q11, #1 + vqshl.s16 q12, q12, #1 + vqshl.s16 q13, q13, #1 + vqshl.s16 q14, q14, #1 + vqshl.s16 q15, q15, #1 + bx lr +endfunc + +function inv_identity_4h_x8_neon, export=1 + vqshl.s16 q8, q8, #1 + vqshl.s16 q9, q9, #1 + vqshl.s16 q10, q10, #1 + vqshl.s16 q11, q11, #1 + bx lr +endfunc + +.macro def_fn_8x8_base variant +function inv_txfm_\variant\()add_8x8_neon + vmov.i16 q0, #0 + vmov.i16 q1, #0 + vld1.16 {q8, q9}, [r2, :128] + vst1.16 {q0, q1}, [r2, :128]! + vld1.16 {q10, q11}, [r2, :128] + vst1.16 {q0, q1}, [r2, :128]! + vld1.16 {q12, q13}, [r2, :128] + vst1.16 {q0, q1}, [r2, :128]! + vld1.16 {q14, q15}, [r2, :128] + vst1.16 {q0, q1}, [r2, :128] + +.ifc \variant, identity_ + // The identity shl #1 and downshift srshr #1 cancel out +.else + blx r4 + + vrshr.s16 q8, q8, #1 + vrshr.s16 q9, q9, #1 + vrshr.s16 q10, q10, #1 + vrshr.s16 q11, q11, #1 + vrshr.s16 q12, q12, #1 + vrshr.s16 q13, q13, #1 + vrshr.s16 q14, q14, #1 + vrshr.s16 q15, q15, #1 +.endif + + transpose_8x8h q8, q9, q10, q11, q12, q13, q14, q15, d17, d19, d21, d23, d24, d26, d28, d30 + + blx r5 + + load_add_store_8x8 r0, r7 + vpop {q4-q7} + pop {r4-r5,r7,pc} +endfunc +.endm + +def_fn_8x8_base +def_fn_8x8_base identity_ + +.macro def_fn_8x8 txfm1, txfm2 +function inv_txfm_add_\txfm1\()_\txfm2\()_8x8_8bpc_neon, export=1 +.ifc \txfm1\()_\txfm2, dct_dct + idct_dc 8, 8, 1 +.endif + push {r4-r5,r7,lr} + vpush {q4-q7} + movrel_local r5, inv_\txfm2\()_8h_x8_neon +.ifc \txfm1, identity + b inv_txfm_identity_add_8x8_neon +.else + movrel_local r4, inv_\txfm1\()_8h_x8_neon + b inv_txfm_add_8x8_neon +.endif +endfunc +.endm + +def_fn_8x8 dct, dct +def_fn_8x8 identity, identity +def_fn_8x8 dct, adst +def_fn_8x8 dct, flipadst +def_fn_8x8 dct, identity +def_fn_8x8 adst, dct +def_fn_8x8 adst, adst +def_fn_8x8 adst, flipadst +def_fn_8x8 flipadst, dct +def_fn_8x8 flipadst, adst +def_fn_8x8 flipadst, flipadst +def_fn_8x8 identity, dct +def_fn_8x8 adst, identity +def_fn_8x8 flipadst, identity +def_fn_8x8 identity, adst +def_fn_8x8 identity, flipadst + +function inv_txfm_add_8x4_neon + vmov.i16 q14, #0 + vmov.i16 q15, #0 + movw r12, #2896*8 + vdup.16 d0, r12 + vld1.16 {d16, d17, d18, d19}, [r2, :128] + vst1.16 {q14, q15}, [r2, :128]! + vld1.16 {d20, d21, d22, d23}, [r2, :128] + vst1.16 {q14, q15}, [r2, :128] + + scale_input d0[0], q8, q9, q10, q11 + + blx r4 + + transpose_4x4h q8, q9, d16, d17, d18, d19 + transpose_4x4h q10, q11, d20, d21, d22, d23 + vswp d17, d20 + vswp d19, d21 + vswp d18, d20 + vswp d21, d22 + + blx r5 + + load_add_store_8x4 r0, r7 + vpop {q4-q7} + pop {r4-r5,r7,pc} +endfunc + +function inv_txfm_add_4x8_neon + vmov.i16 q14, #0 + vmov.i16 q15, #0 + movw r12, #2896*8 + vdup.16 d0, r12 + vld1.16 {q8, q9}, [r2, :128] + vst1.16 {q14, q15}, [r2, :128]! + vld1.16 {q10, q11}, [r2, :128] + vst1.16 {q14, q15}, [r2, :128] + + scale_input d0[0], q8, q9, q10, q11 + + blx r4 + + transpose_4x8h q8, q9, q10, q11 + vswp d17, d20 + vswp d19, d21 + vswp d17, d18 + vswp d19, d22 + + blx r5 + + load_add_store_4x8 r0, r7 + vpop {q4-q7} + pop {r4-r5,r7,pc} +endfunc + +.macro def_fn_48 w, h, txfm1, txfm2 +function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_8bpc_neon, export=1 +.ifc \txfm1\()_\txfm2, dct_dct + idct_dc \w, \h, 0 +.endif + push {r4-r5,r7,lr} + vpush {q4-q7} + movrel_local r4, inv_\txfm1\()_\h\()h_x\w\()_neon + movrel_local r5, inv_\txfm2\()_\w\()h_x\h\()_neon + b inv_txfm_add_\w\()x\h\()_neon +endfunc +.endm + +.macro def_fns_48 w, h +def_fn_48 \w, \h, dct, dct +def_fn_48 \w, \h, identity, identity +def_fn_48 \w, \h, dct, adst +def_fn_48 \w, \h, dct, flipadst +def_fn_48 \w, \h, dct, identity +def_fn_48 \w, \h, adst, dct +def_fn_48 \w, \h, adst, adst +def_fn_48 \w, \h, adst, flipadst +def_fn_48 \w, \h, flipadst, dct +def_fn_48 \w, \h, flipadst, adst +def_fn_48 \w, \h, flipadst, flipadst +def_fn_48 \w, \h, identity, dct +def_fn_48 \w, \h, adst, identity +def_fn_48 \w, \h, flipadst, identity +def_fn_48 \w, \h, identity, adst +def_fn_48 \w, \h, identity, flipadst +.endm + +def_fns_48 4, 8 +def_fns_48 8, 4 + +function inv_dct_4h_x16_neon, export=1 + movrel_local r12, idct_coeffs + vld1.16 {q0, q1}, [r12, :128] + + vmull_vmlsl q2, d17, d31, d2[0], d2[1] // -> t8a + vmull_vmlal q3, d17, d31, d2[1], d2[0] // -> t15a + vmull_vmlsl q4, d25, d23, d2[2], d2[3] // -> t9a + vrshrn.i32 d17, q2, #12 // t8a + vrshrn.i32 d31, q3, #12 // t15a + vmull_vmlal q2, d25, d23, d2[3], d2[2] // -> t14a + vmull_vmlsl q3, d21, d27, d3[0], d3[1] // -> t10a + vrshrn.i32 d23, q4, #12 // t9a + vrshrn.i32 d25, q2, #12 // t14a + vmull_vmlal q4, d21, d27, d3[1], d3[0] // -> t13a + vmull_vmlsl q2, d29, d19, d3[2], d3[3] // -> t11a + vrshrn.i32 d21, q3, #12 // t10a + vrshrn.i32 d27, q4, #12 // t13a + vmull_vmlal q3, d29, d19, d3[3], d3[2] // -> t12a + vrshrn.i32 d19, q2, #12 // t11a + vrshrn.i32 d29, q3, #12 // t12a + + idct_4h_x8 d16, d18, d20, d22, d24, d26, d28, d30 + + vqsub.s16 d4, d17, d23 // t9 + vqadd.s16 d17, d17, d23 // t8 + vqsub.s16 d5, d31, d25 // t14 + vqadd.s16 d31, d31, d25 // t15 + vqsub.s16 d23, d19, d21 // t10 + vqadd.s16 d19, d19, d21 // t11 + vqadd.s16 d25, d29, d27 // t12 + vqsub.s16 d29, d29, d27 // t13 + + vmull_vmlsl q3, d5, d4, d0[2], d0[3] // -> t9a + vmull_vmlal q4, d5, d4, d0[3], d0[2] // -> t14a + vrshrn.i32 d21, q3, #12 // t9a + vrshrn.i32 d27, q4, #12 // t14a + + vmull_vmlsl q3, d29, d23, d0[2], d0[3] // -> t13a + vmull_vmlal q4, d29, d23, d0[3], d0[2] // -> t10a + vrshrn.i32 d29, q3, #12 // t13a + vneg.s32 q4, q4 + vrshrn.i32 d23, q4, #12 // t10a + + vqsub.s16 d4, d17, d19 // t11a + vqadd.s16 d17, d17, d19 // t8a + vqsub.s16 d5, d31, d25 // t12a + vqadd.s16 d31, d31, d25 // t15a + vqadd.s16 d19, d21, d23 // t9 + vqsub.s16 d21, d21, d23 // t10 + vqsub.s16 d25, d27, d29 // t13 + vqadd.s16 d27, d27, d29 // t14 + + vmull_vmlsl q3, d5, d4, d0[0], d0[0] // -> t11 + vmull_vmlal q4, d5, d4, d0[0], d0[0] // -> t12 + vmull_vmlsl q2, d25, d21, d0[0], d0[0] // -> t10a + + vrshrn.i32 d6, q3, #12 // t11 + vrshrn.i32 d7, q4, #12 // t12 + vmull_vmlal q4, d25, d21, d0[0], d0[0] // -> t10a + vrshrn.i32 d4, q2, #12 // t10a + vrshrn.i32 d5, q4, #12 // t13a + + vqadd.s16 d8, d16, d31 // out0 + vqsub.s16 d31, d16, d31 // out15 + vmov d16, d8 + vqadd.s16 d23, d30, d17 // out7 + vqsub.s16 d9, d30, d17 // out8 + vqadd.s16 d17, d18, d27 // out1 + vqsub.s16 d30, d18, d27 // out14 + vqadd.s16 d18, d20, d5 // out2 + vqsub.s16 d29, d20, d5 // out13 + vqadd.s16 d5, d28, d19 // out6 + vqsub.s16 d25, d28, d19 // out9 + vqadd.s16 d19, d22, d7 // out3 + vqsub.s16 d28, d22, d7 // out12 + vqadd.s16 d20, d24, d6 // out4 + vqsub.s16 d27, d24, d6 // out11 + vqadd.s16 d21, d26, d4 // out5 + vqsub.s16 d26, d26, d4 // out10 + vmov d24, d9 + vmov d22, d5 + + bx lr +endfunc + +.macro iadst_16 o0, o1, o2, o3, o4, o5, o6, o7, o8, o9, o10, o11, o12, o13, o14, o15 + movrel_local r12, iadst16_coeffs + vld1.16 {q0, q1}, [r12, :128] + movrel_local r12, idct_coeffs + + vmull_vmlal q2, d31, d16, d0[0], d0[1] // -> t0 + vmull_vmlsl q3, d31, d16, d0[1], d0[0] // -> t1 + vmull_vmlal q4, d29, d18, d0[2], d0[3] // -> t2 + vrshrn.i32 d16, q2, #12 // t0 + vrshrn.i32 d31, q3, #12 // t1 + vmull_vmlsl q2, d29, d18, d0[3], d0[2] // -> t3 + vmull_vmlal q3, d27, d20, d1[0], d1[1] // -> t4 + vrshrn.i32 d18, q4, #12 // t2 + vrshrn.i32 d29, q2, #12 // t3 + vmull_vmlsl q4, d27, d20, d1[1], d1[0] // -> t5 + vmull_vmlal q2, d25, d22, d1[2], d1[3] // -> t6 + vrshrn.i32 d20, q3, #12 // t4 + vrshrn.i32 d27, q4, #12 // t5 + vmull_vmlsl q3, d25, d22, d1[3], d1[2] // -> t7 + vmull_vmlal q4, d23, d24, d2[0], d2[1] // -> t8 + vrshrn.i32 d22, q2, #12 // t6 + vrshrn.i32 d25, q3, #12 // t7 + vmull_vmlsl q2, d23, d24, d2[1], d2[0] // -> t9 + vmull_vmlal q3, d21, d26, d2[2], d2[3] // -> t10 + vrshrn.i32 d23, q4, #12 // t8 + vrshrn.i32 d24, q2, #12 // t9 + vmull_vmlsl q4, d21, d26, d2[3], d2[2] // -> t11 + vmull_vmlal q2, d19, d28, d3[0], d3[1] // -> t12 + vrshrn.i32 d21, q3, #12 // t10 + vrshrn.i32 d26, q4, #12 // t11 + vmull_vmlsl q3, d19, d28, d3[1], d3[0] // -> t13 + vmull_vmlal q4, d17, d30, d3[2], d3[3] // -> t14 + vrshrn.i32 d19, q2, #12 // t12 + vrshrn.i32 d28, q3, #12 // t13 + vmull_vmlsl q2, d17, d30, d3[3], d3[2] // -> t15 + vrshrn.i32 d17, q4, #12 // t14 + vrshrn.i32 d30, q2, #12 // t15 + + vld1.16 {q0}, [r12, :128] + + vqsub.s16 d2, d16, d23 // t8a + vqadd.s16 d16, d16, d23 // t0a + vqsub.s16 d3, d31, d24 // t9a + vqadd.s16 d31, d31, d24 // t1a + vqadd.s16 d23, d18, d21 // t2a + vqsub.s16 d18, d18, d21 // t10a + vqadd.s16 d24, d29, d26 // t3a + vqsub.s16 d29, d29, d26 // t11a + vqadd.s16 d21, d20, d19 // t4a + vqsub.s16 d20, d20, d19 // t12a + vqadd.s16 d26, d27, d28 // t5a + vqsub.s16 d27, d27, d28 // t13a + vqadd.s16 d19, d22, d17 // t6a + vqsub.s16 d22, d22, d17 // t14a + vqadd.s16 d28, d25, d30 // t7a + vqsub.s16 d25, d25, d30 // t15a + + vmull_vmlal q2, d2, d3, d1[1], d1[0] // -> t8 + vmull_vmlsl q3, d2, d3, d1[0], d1[1] // -> t9 + vmull_vmlal q4, d18, d29, d1[3], d1[2] // -> t10 + vrshrn.i32 d17, q2, #12 // t8 + vrshrn.i32 d30, q3, #12 // t9 + vmull_vmlsl q2, d18, d29, d1[2], d1[3] // -> t11 + vmull_vmlsl q3, d27, d20, d1[1], d1[0] // -> t12 + vrshrn.i32 d18, q4, #12 // t10 + vrshrn.i32 d29, q2, #12 // t11 + vmull_vmlal q4, d27, d20, d1[0], d1[1] // -> t13 + vmull_vmlsl q2, d25, d22, d1[3], d1[2] // -> t14 + vrshrn.i32 d27, q3, #12 // t12 + vrshrn.i32 d20, q4, #12 // t13 + vmull_vmlal q3, d25, d22, d1[2], d1[3] // -> t15 + vrshrn.i32 d25, q2, #12 // t14 + vrshrn.i32 d22, q3, #12 // t15 + + vqsub.s16 d2, d16, d21 // t4 + vqadd.s16 d16, d16, d21 // t0 + vqsub.s16 d3, d31, d26 // t5 + vqadd.s16 d31, d31, d26 // t1 + vqadd.s16 d21, d23, d19 // t2 + vqsub.s16 d23, d23, d19 // t6 + vqadd.s16 d26, d24, d28 // t3 + vqsub.s16 d24, d24, d28 // t7 + vqadd.s16 d19, d17, d27 // t8a + vqsub.s16 d17, d17, d27 // t12a + vqadd.s16 d28, d30, d20 // t9a + vqsub.s16 d30, d30, d20 // t13a + vqadd.s16 d27, d18, d25 // t10a + vqsub.s16 d18, d18, d25 // t14a + vqadd.s16 d20, d29, d22 // t11a + vqsub.s16 d29, d29, d22 // t15a + + vmull_vmlal q2, d2, d3, d0[3], d0[2] // -> t4a + vmull_vmlsl q3, d2, d3, d0[2], d0[3] // -> t5a + vmull_vmlsl q4, d24, d23, d0[3], d0[2] // -> t6a + vrshrn.i32 d22, q2, #12 // t4a + vrshrn.i32 d25, q3, #12 // t5a + vmull_vmlal q2, d24, d23, d0[2], d0[3] // -> t7a + vmull_vmlal q3, d17, d30, d0[3], d0[2] // -> t12 + vrshrn.i32 d24, q4, #12 // t6a + vrshrn.i32 d23, q2, #12 // t7a + vmull_vmlsl q4, d17, d30, d0[2], d0[3] // -> t13 + vmull_vmlsl q2, d29, d18, d0[3], d0[2] // -> t14 + vrshrn.i32 d17, q3, #12 // t12 + vmull_vmlal q3, d29, d18, d0[2], d0[3] // -> t15 + vrshrn.i32 d29, q4, #12 // t13 + vrshrn.i32 d30, q2, #12 // t14 + vrshrn.i32 d18, q3, #12 // t15 + + vqsub.s16 d2, d16, d21 // t2a +.ifc \o0, d16 + vqadd.s16 \o0, d16, d21 // out0 + vqsub.s16 d21, d31, d26 // t3a + vqadd.s16 \o15,d31, d26 // out15 +.else + vqadd.s16 d4, d16, d21 // out0 + vqsub.s16 d21, d31, d26 // t3a + vqadd.s16 \o15,d31, d26 // out15 + vmov \o0, d4 +.endif + vqneg.s16 \o15, \o15 // out15 + + vqsub.s16 d3, d29, d18 // t15a + vqadd.s16 \o13,d29, d18 // out13 + vqadd.s16 \o2, d17, d30 // out2 + vqsub.s16 d26, d17, d30 // t14a + vqneg.s16 \o13,\o13 // out13 + + vqadd.s16 \o1, d19, d27 // out1 + vqsub.s16 d27, d19, d27 // t10 + vqadd.s16 \o14,d28, d20 // out14 + vqsub.s16 d20, d28, d20 // t11 + vqneg.s16 \o1, \o1 // out1 + + vqadd.s16 \o3, d22, d24 // out3 + vqsub.s16 d22, d22, d24 // t6 + vqadd.s16 \o12,d25, d23 // out12 + vqsub.s16 d23, d25, d23 // t7 + vqneg.s16 \o3, \o3 // out3 + + vmull_vmlsl q12, d2, d21, d0[0], d0[0] // -> out8 (d24 or d23) + vmull_vmlal q2, d2, d21, d0[0], d0[0] // -> out7 (d23 or d24) + vmull_vmlal q3, d26, d3, d0[0], d0[0] // -> out5 (d21 or d26) + + vrshrn.i32 d24, q12, #12 // out8 + vrshrn.i32 d4, q2, #12 // out7 + vrshrn.i32 d5, q3, #12 // out5 + vmull_vmlsl q4, d26, d3, d0[0], d0[0] // -> out10 (d26 or d21) + vmull_vmlal q1, d22, d23, d0[0], d0[0] // -> out4 (d20 or d27) + vrshrn.i32 d26, q4, #12 // out10 + + vmull_vmlsl q4, d22, d23, d0[0], d0[0] // -> out11 (d27 or d20) + vmull_vmlal q11, d27, d20, d0[0], d0[0] // -> out6 (d22 or d25) + vmull_vmlsl q3, d27, d20, d0[0], d0[0] // -> out9 (d25 or d22) + + vrshrn.i32 \o4, q1, #12 // out4 + vrshrn.i32 d7, q3, #12 // out9 + vrshrn.i32 d6, q4, #12 // out11 + vrshrn.i32 \o6, q11, #12 // out6 + +.ifc \o8, d23 + vmov \o8, d24 + vmov \o10,d26 +.endif + + vqneg.s16 \o7, d4 // out7 + vqneg.s16 \o5, d5 // out5 + vqneg.s16 \o11,d6 // out11 + vqneg.s16 \o9, d7 // out9 +.endm + +function inv_adst_4h_x16_neon, export=1 + iadst_16 d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, d28, d29, d30, d31 + bx lr +endfunc + +function inv_flipadst_4h_x16_neon, export=1 + iadst_16 d31, d30, d29, d28, d27, d26, d25, d24, d23, d22, d21, d20, d19, d18, d17, d16 + bx lr +endfunc + +function inv_identity_4h_x16_neon, export=1 + movw r12, #2*(5793-4096)*8 + vdup.16 d0, r12 +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vqrdmulh.s16 q1, \i, d0[0] + vqadd.s16 \i, \i, \i + vqadd.s16 \i, \i, q1 +.endr + bx lr +endfunc + +.macro identity_4x16_shift2 c +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vqrdmulh.s16 q2, \i, \c + vshr.s16 q2, q2, #1 + vrhadd.s16 \i, \i, q2 +.endr +.endm + +.macro identity_4x16_shift1 c +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vqrdmulh.s16 q2, \i, \c + vrshr.s16 q2, q2, #1 + vqadd.s16 \i, \i, q2 +.endr +.endm + +.macro identity_8x8_shift1 c + identity_4x16_shift1 \c +.endm + +.macro identity_8x8 c +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vqrdmulh.s16 q2, \i, \c + vqadd.s16 \i, \i, \i + vqadd.s16 \i, \i, q2 +.endr +.endm + +.macro def_horz_16 scale=0, identity=0, shift=2, suffix +function inv_txfm_horz\suffix\()_16x4_neon + push {lr} + vmov.i16 d7, #0 +.if \identity + movw r12, #2*(5793-4096)*8 + vdup.16 d0, r12 +.endif +.if \scale + movw r12, #2896*8 + vdup.16 d1, r12 +.endif +.irp i, d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, d28, d29, d30, d31 + vld1.16 {\i}, [r7, :64] + vst1.16 {d7}, [r7, :64], r8 +.endr +.if \scale + scale_input d1[0], q8, q9, q10, q11, q12, q13, q14, q15 +.endif +.if \identity +.if \shift == -2 + identity_4x16_shift2 d0[0] +.else + identity_4x16_shift1 d0[0] +.endif +.else + blx r4 +.endif +.if \shift > 0 +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vrshr.s16 \i, \i, #\shift +.endr +.endif + transpose_4x4h q8, q9, d16, d17, d18, d19 + transpose_4x4h q10, q11, d20, d21, d22, d23 + transpose_4x4h q12, q13, d24, d25, d26, d27 + transpose_4x4h q14, q15, d28, d29, d30, d31 + +.irp i, d16, d20, d24, d28, d17, d21, d25, d29, d18, d22, d26, d30, d19, d23, d27, d31 + vst1.16 {\i}, [r6, :64]! +.endr + + pop {pc} +endfunc +.endm + +def_horz_16 scale=0, identity=0, shift=2 +def_horz_16 scale=1, identity=0, shift=1, suffix=_scale +def_horz_16 scale=0, identity=1, shift=-2, suffix=_identity +def_horz_16 scale=1, identity=1, shift=-1, suffix=_scale_identity + +function inv_txfm_add_vert_4x16_neon + push {lr} +.irp i, d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, d28, d29, d30, d31 + vld1.16 {\i}, [r7, :64], r8 +.endr + blx r5 + load_add_store_4x16 r6, r7 + pop {pc} +endfunc + +.macro sub_sp_align space +#if CONFIG_THUMB + mov r7, sp + and r7, r7, #15 +#else + and r7, sp, #15 +#endif + sub sp, sp, r7 + // Now the stack is aligned, store the amount of adjustment back + // on the stack, as we don't want to waste a register as frame + // pointer. + str r7, [sp, #-16]! +#ifdef _WIN32 +.if \space > 8192 + // Here, we'd need to touch two (or more) pages while decrementing + // the stack pointer. + .error "sub_sp_align doesn't support values over 8K at the moment" +.elseif \space > 4096 + sub r7, sp, #4096 + ldr r12, [r7] + sub r7, r7, #(\space - 4096) + mov sp, r7 +.else + sub sp, sp, #\space +.endif +#else +.if \space >= 4096 + sub sp, sp, #(\space)/4096*4096 +.endif +.if (\space % 4096) != 0 + sub sp, sp, #(\space)%4096 +.endif +#endif +.endm + +.macro add_sp_align space +.if \space >= 4096 + add sp, sp, #(\space)/4096*4096 +.endif +.if (\space % 4096) != 0 + add sp, sp, #(\space)%4096 +.endif + ldr r7, [sp], #16 + // Add back the original stack adjustment + add sp, sp, r7 +.endm + +function inv_txfm_add_16x16_neon + sub_sp_align 512 + ldrh r11, [r10], #2 +.irp i, 0, 4, 8, 12 + add r6, sp, #(\i*16*2) +.if \i > 0 + mov r8, #(16 - \i) + cmp r3, r11 + blt 1f +.if \i < 12 + ldrh r11, [r10], #2 +.endif +.endif + add r7, r2, #(\i*2) + mov r8, #16*2 + blx r9 +.endr + b 3f +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r8, r8, #4 +.rept 4 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b +3: +.irp i, 0, 4, 8, 12 + add r6, r0, #(\i) + add r7, sp, #(\i*2) + mov r8, #32 + bl inv_txfm_add_vert_4x16_neon +.endr + + add_sp_align 512 + vpop {q4} + pop {r4-r11,pc} +endfunc + +const eob_16x16 + .short 10, 36, 78, 256 +endconst + +const eob_16x16_identity + .short 4, 8, 12, 256 +endconst + +.macro def_fn_16x16 txfm1, txfm2 +function inv_txfm_add_\txfm1\()_\txfm2\()_16x16_8bpc_neon, export=1 +.ifc \txfm1\()_\txfm2, dct_dct + idct_dc 16, 16, 2 +.endif + push {r4-r11,lr} + vpush {q4} +.ifc \txfm1, identity + movrel_local r9, inv_txfm_horz_identity_16x4_neon +.else + movrel_local r9, inv_txfm_horz_16x4_neon + movrel_local r4, inv_\txfm1\()_4h_x16_neon +.endif + movrel_local r5, inv_\txfm2\()_4h_x16_neon +.ifc \txfm1, identity +.ifc \txfm2, identity + movrel_local r10, eob_16x16 +.else + movrel_local r10, eob_16x16_identity +.endif +.else +.ifc \txfm2, identity + movrel_local r10, eob_16x16_identity +.else + movrel_local r10, eob_16x16 +.endif +.endif + b inv_txfm_add_16x16_neon +endfunc +.endm + +def_fn_16x16 dct, dct +def_fn_16x16 identity, identity +def_fn_16x16 dct, adst +def_fn_16x16 dct, flipadst +def_fn_16x16 dct, identity +def_fn_16x16 adst, dct +def_fn_16x16 adst, adst +def_fn_16x16 adst, flipadst +def_fn_16x16 flipadst, dct +def_fn_16x16 flipadst, adst +def_fn_16x16 flipadst, flipadst +def_fn_16x16 identity, dct + +.macro def_fn_416_base variant +function inv_txfm_\variant\()add_16x4_neon + +.ifc \variant, identity_ + vmov.i16 d4, #0 +.irp i, d16, d18, d20, d22 + vld1.16 {\i}, [r2, :64] + vst1.16 {d4}, [r2, :64]! +.endr +.irp i, d17, d19, d21, d23 + vld1.16 {\i}, [r2, :64] + vst1.16 {d4}, [r2, :64]! +.endr + movw r12, #2*(5793-4096)*8 + vdup.16 d0, r12 +.irp i, d24, d26, d28, d30 + vld1.16 {\i}, [r2, :64] + vst1.16 {d4}, [r2, :64]! +.endr +.irp i, d25, d27, d29, d31 + vld1.16 {\i}, [r2, :64] + vst1.16 {d4}, [r2, :64]! +.endr + + identity_4x16_shift1 d0[0] +.else + vmov.i16 q2, #0 + vmov.i16 q3, #0 + vld1.16 {d16, d17, d18, d19}, [r2, :128] + vst1.16 {q2, q3}, [r2, :128]! + vld1.16 {d20, d21, d22, d23}, [r2, :128] + vst1.16 {q2, q3}, [r2, :128]! + vld1.16 {d24, d25, d26, d27}, [r2, :128] + vst1.16 {q2, q3}, [r2, :128]! + vld1.16 {d28, d29, d30, d31}, [r2, :128] + vst1.16 {q2, q3}, [r2, :128]! + + blx r4 + + vswp d17, d20 + vswp d19, d22 + vswp d18, d20 + vswp d19, d21 +.irp i, q8, q9, q10, q11 + vrshr.s16 \i, \i, #1 +.endr +.endif + transpose_4x8h q8, q9, q10, q11 + blx r5 + mov r6, r0 + load_add_store_8x4 r6, r7 + +.ifc \variant, identity_ + vmov q8, q12 + vmov q9, q13 + vmov q10, q14 + vmov q11, q15 +.else + vswp d25, d28 + vswp d27, d30 + vswp d26, d28 + vswp d27, d29 + vrshr.s16 q8, q12, #1 + vrshr.s16 q9, q13, #1 + vrshr.s16 q10, q14, #1 + vrshr.s16 q11, q15, #1 +.endif + transpose_4x8h q8, q9, q10, q11 + blx r5 + add r6, r0, #8 + load_add_store_8x4 r6, r7 + + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +function inv_txfm_\variant\()add_4x16_neon + vmov.i16 q2, #0 + + mov r11, #32 + cmp r3, r10 + blt 1f + + add r6, r2, #16 +.ifc \variant, identity_ +.irp i, q12, q13, q14, q15 + vld1.16 {\i}, [r6, :128] + vst1.16 {q2}, [r6, :128], r11 +.endr + movw r12, #(5793-4096)*8 + vdup.16 d0, r12 + identity_8x4_shift1 q12, q13, q14, q15, d0[0] +.else +.irp i, q8, q9, q10, q11 + vld1.16 {\i}, [r6, :128] + vst1.16 {q2}, [r6, :128], r11 +.endr + blx r4 + vrshr.s16 q12, q8, #1 + vrshr.s16 q13, q9, #1 + vrshr.s16 q14, q10, #1 + vrshr.s16 q15, q11, #1 +.endif + transpose_4x8h q12, q13, q14, q15 + vswp d27, d29 + vswp d26, d28 + vswp d27, d30 + vswp d25, d28 + + b 2f +1: +.irp i, q12, q13, q14, q15 + vmov.i16 \i, #0 +.endr +2: + vmov.i16 q2, #0 +.irp i, q8, q9, q10, q11 + vld1.16 {\i}, [r2, :128] + vst1.16 {q2}, [r2, :128], r11 +.endr +.ifc \variant, identity_ + movw r12, #(5793-4096)*8 + vdup.16 d0, r12 + identity_8x4_shift1 q8, q9, q10, q11, d0[0] +.else + blx r4 +.irp i, q8, q9, q10, q11 + vrshr.s16 \i, \i, #1 +.endr +.endif + transpose_4x8h q8, q9, q10, q11 + vswp d19, d21 + vswp d18, d20 + vswp d19, d22 + vswp d17, d20 + + blx r5 + + load_add_store_4x16 r0, r6 + + vpop {q4-q7} + pop {r4-r11,pc} +endfunc +.endm + +def_fn_416_base +def_fn_416_base identity_ + +.macro def_fn_416 w, h, txfm1, txfm2, eob_half +function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_8bpc_neon, export=1 +.ifc \txfm1\()_\txfm2, dct_dct + idct_dc \w, \h, 1 +.endif + push {r4-r11,lr} + vpush {q4-q7} +.if \w == 4 + movrel_local r4, inv_\txfm1\()_8h_x\w\()_neon + movrel_local r5, inv_\txfm2\()_4h_x\h\()_neon + mov r10, #\eob_half +.else + movrel_local r4, inv_\txfm1\()_4h_x\w\()_neon + movrel_local r5, inv_\txfm2\()_8h_x\h\()_neon +.endif +.ifc \txfm1, identity + b inv_txfm_identity_add_\w\()x\h\()_neon +.else + b inv_txfm_add_\w\()x\h\()_neon +.endif +endfunc +.endm + +.macro def_fns_416 w, h +def_fn_416 \w, \h, dct, dct, 29 +def_fn_416 \w, \h, identity, identity, 29 +def_fn_416 \w, \h, dct, adst, 29 +def_fn_416 \w, \h, dct, flipadst, 29 +def_fn_416 \w, \h, dct, identity, 8 +def_fn_416 \w, \h, adst, dct, 29 +def_fn_416 \w, \h, adst, adst, 29 +def_fn_416 \w, \h, adst, flipadst, 29 +def_fn_416 \w, \h, flipadst, dct, 29 +def_fn_416 \w, \h, flipadst, adst, 29 +def_fn_416 \w, \h, flipadst, flipadst, 29 +def_fn_416 \w, \h, identity, dct, 32 +def_fn_416 \w, \h, adst, identity, 8 +def_fn_416 \w, \h, flipadst, identity, 8 +def_fn_416 \w, \h, identity, adst, 32 +def_fn_416 \w, \h, identity, flipadst, 32 +.endm + +def_fns_416 4, 16 +def_fns_416 16, 4 + +.macro def_fn_816_base variant +function inv_txfm_\variant\()add_16x8_neon + sub_sp_align 256 + +.irp i, 0, 4 + add r6, sp, #(\i*16*2) +.if \i > 0 + cmp r3, r10 + blt 1f +.endif + add r7, r2, #(\i*2) + mov r8, #8*2 + blx r9 +.endr + b 2f +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +.rept 4 + vst1.16 {q2, q3}, [r6, :128]! +.endr +2: + +.irp i, 0, 8 + add r7, sp, #(\i*2) + mov r8, #32 +.irp j, q8, q9, q10, q11, q12, q13, q14, q15 + vld1.16 {\j}, [r7, :128], r8 +.endr + blx r5 + + add r6, r0, #(\i) + load_add_store_8x8 r6, r7 +.endr + + add_sp_align 256 + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +function inv_txfm_\variant\()add_8x16_neon + sub_sp_align 256 + +.irp i, 0, 8 + add r6, sp, #(\i*8*2) +.if \i > 0 + cmp r3, r10 + blt 1f +.endif + add r7, r2, #(\i*2) + mov r8, #16*2 + + vmov.i16 q2, #0 + movw r12, #2896*8 + vdup.16 d0, r12 + +.irp j, q8, q9, q10, q11, q12, q13, q14, q15 + vld1.16 {\j}, [r7, :128] + vst1.16 {q2}, [r7, :128], r8 +.endr + scale_input d0[0], q8, q9, q10, q11, q12, q13, q14, q15 +.ifc \variant, identity_ + // The identity shl #1 and downshift vrshr #1 cancel out +.else + blx r4 +.irp j, q8, q9, q10, q11, q12, q13, q14, q15 + vrshr.s16 \j, \j, #1 +.endr +.endif + transpose_8x8h q8, q9, q10, q11, q12, q13, q14, q15, d17, d19, d21, d23, d24, d26, d28, d30 + vst1.16 {q8, q9}, [r6, :128]! + vst1.16 {q10, q11}, [r6, :128]! + vst1.16 {q12, q13}, [r6, :128]! + vst1.16 {q14, q15}, [r6, :128]! +.endr + b 2f +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +.rept 4 + vst1.16 {q2, q3}, [r6, :128]! +.endr +2: + +.irp i, 0, 4 + add r6, r0, #(\i) + add r7, sp, #(\i*2) + mov r8, #16 + bl inv_txfm_add_vert_4x16_neon +.endr + + add_sp_align 256 + vpop {q4-q7} + pop {r4-r11,pc} +endfunc +.endm + +def_fn_816_base +def_fn_816_base identity_ + +.macro def_fn_816 w, h, txfm1, txfm2, eob_8x8, eob_4x4 +function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_8bpc_neon, export=1 +.ifc \txfm1\()_\txfm2, dct_dct + idct_dc \w, \h, 1 +.endif + push {r4-r11,lr} + vpush {q4-q7} +.if \w == 8 + movrel_local r4, inv_\txfm1\()_8h_x8_neon + movrel_local r5, inv_\txfm2\()_4h_x16_neon +.else +.ifc \txfm1, identity + movrel_local r9, inv_txfm_horz_scale_identity_16x4_neon +.else + movrel_local r4, inv_\txfm1\()_4h_x16_neon + movrel_local r9, inv_txfm_horz_scale_16x4_neon +.endif + movrel_local r5, inv_\txfm2\()_8h_x8_neon +.endif +.if \w == 8 + mov r10, #\eob_8x8 +.else + mov r10, #\eob_4x4 +.endif +.ifc \txfm1, identity + b inv_txfm_identity_add_\w\()x\h\()_neon +.else + b inv_txfm_add_\w\()x\h\()_neon +.endif +endfunc +.endm + +.macro def_fns_816 w, h +def_fn_816 \w, \h, dct, dct, 43, 10 +def_fn_816 \w, \h, identity, identity, 43, 10 +def_fn_816 \w, \h, dct, adst, 43, 10 +def_fn_816 \w, \h, dct, flipadst, 43, 10 +def_fn_816 \w, \h, dct, identity, 8, 4 +def_fn_816 \w, \h, adst, dct, 43, 10 +def_fn_816 \w, \h, adst, adst, 43, 10 +def_fn_816 \w, \h, adst, flipadst, 43, 10 +def_fn_816 \w, \h, flipadst, dct, 43, 10 +def_fn_816 \w, \h, flipadst, adst, 43, 10 +def_fn_816 \w, \h, flipadst, flipadst, 43, 10 +def_fn_816 \w, \h, identity, dct, 64, 4 +def_fn_816 \w, \h, adst, identity, 8, 4 +def_fn_816 \w, \h, flipadst, identity, 8, 4 +def_fn_816 \w, \h, identity, adst, 64, 4 +def_fn_816 \w, \h, identity, flipadst, 64, 4 +.endm + +def_fns_816 8, 16 +def_fns_816 16, 8 + +function inv_dct32_odd_4h_x16_neon, export=1 + movrel_local r12, idct_coeffs, 2*16 + vld1.16 {q0, q1}, [r12, :128] + sub r12, r12, #2*16 + + vmull_vmlsl q2, d16, d31, d0[0], d0[1] // -> t16a + vmull_vmlal q3, d16, d31, d0[1], d0[0] // -> t31a + vmull_vmlsl q4, d24, d23, d0[2], d0[3] // -> t17a + vrshrn.i32 d16, q2, #12 // t16a + vrshrn.i32 d31, q3, #12 // t31a + vmull_vmlal q2, d24, d23, d0[3], d0[2] // -> t30a + vmull_vmlsl q3, d20, d27, d1[0], d1[1] // -> t18a + vrshrn.i32 d24, q4, #12 // t17a + vrshrn.i32 d23, q2, #12 // t30a + vmull_vmlal q4, d20, d27, d1[1], d1[0] // -> t29a + vmull_vmlsl q2, d28, d19, d1[2], d1[3] // -> t19a + vrshrn.i32 d20, q3, #12 // t18a + vrshrn.i32 d27, q4, #12 // t29a + vmull_vmlal q3, d28, d19, d1[3], d1[2] // -> t28a + vmull_vmlsl q4, d18, d29, d2[0], d2[1] // -> t20a + vrshrn.i32 d28, q2, #12 // t19a + vrshrn.i32 d19, q3, #12 // t28a + vmull_vmlal q2, d18, d29, d2[1], d2[0] // -> t27a + vmull_vmlsl q3, d26, d21, d2[2], d2[3] // -> t21a + vrshrn.i32 d18, q4, #12 // t20a + vrshrn.i32 d29, q2, #12 // t27a + vmull_vmlal q4, d26, d21, d2[3], d2[2] // -> t26a + vmull_vmlsl q2, d22, d25, d3[0], d3[1] // -> t22a + vrshrn.i32 d26, q3, #12 // t21a + vrshrn.i32 d21, q4, #12 // t26a + vmull_vmlal q3, d22, d25, d3[1], d3[0] // -> t25a + vmull_vmlsl q4, d30, d17, d3[2], d3[3] // -> t23a + vrshrn.i32 d22, q2, #12 // t22a + vrshrn.i32 d25, q3, #12 // t25a + vmull_vmlal q2, d30, d17, d3[3], d3[2] // -> t24a + vrshrn.i32 d30, q4, #12 // t23a + vrshrn.i32 d17, q2, #12 // t24a + + vld1.16 {q0}, [r12, :128] + + vqsub.s16 d2, d16, d24 // t17 + vqadd.s16 d16, d16, d24 // t16 + vqsub.s16 d3, d31, d23 // t30 + vqadd.s16 d31, d31, d23 // t31 + vqsub.s16 d24, d28, d20 // t18 + vqadd.s16 d28, d28, d20 // t19 + vqadd.s16 d23, d18, d26 // t20 + vqsub.s16 d18, d18, d26 // t21 + vqsub.s16 d20, d30, d22 // t22 + vqadd.s16 d30, d30, d22 // t23 + vqadd.s16 d26, d17, d25 // t24 + vqsub.s16 d17, d17, d25 // t25 + vqsub.s16 d22, d29, d21 // t26 + vqadd.s16 d29, d29, d21 // t27 + vqadd.s16 d25, d19, d27 // t28 + vqsub.s16 d19, d19, d27 // t29 + + vmull_vmlsl q2, d3, d2, d1[0], d1[1] // -> t17a + vmull_vmlal q3, d3, d2, d1[1], d1[0] // -> t30a + vmull_vmlal q4, d19, d24, d1[1], d1[0] // -> t18a + vrshrn.i32 d21, q2, #12 // t17a + vrshrn.i32 d27, q3, #12 // t30a + vneg.s32 q4, q4 // -> t18a + vmull_vmlsl q1, d19, d24, d1[0], d1[1] // -> t29a + vmull_vmlsl q2, d22, d18, d1[2], d1[3] // -> t21a + vrshrn.i32 d19, q4, #12 // t18a + vrshrn.i32 d24, q1, #12 // t29a + vmull_vmlal q3, d22, d18, d1[3], d1[2] // -> t26a + vmull_vmlal q4, d17, d20, d1[3], d1[2] // -> t22a + vrshrn.i32 d22, q2, #12 // t21a + vrshrn.i32 d18, q3, #12 // t26a + vneg.s32 q4, q4 // -> t22a + vmull_vmlsl q1, d17, d20, d1[2], d1[3] // -> t25a + vrshrn.i32 d17, q4, #12 // t22a + vrshrn.i32 d20, q1, #12 // t25a + + vqsub.s16 d2, d27, d24 // t29 + vqadd.s16 d27, d27, d24 // t30 + vqsub.s16 d3, d21, d19 // t18 + vqadd.s16 d21, d21, d19 // t17 + vqsub.s16 d24, d16, d28 // t19a + vqadd.s16 d16, d16, d28 // t16a + vqsub.s16 d19, d30, d23 // t20a + vqadd.s16 d30, d30, d23 // t23a + vqsub.s16 d28, d17, d22 // t21 + vqadd.s16 d17, d17, d22 // t22 + vqadd.s16 d23, d26, d29 // t24a + vqsub.s16 d26, d26, d29 // t27a + vqadd.s16 d22, d20, d18 // t25 + vqsub.s16 d20, d20, d18 // t26 + vqsub.s16 d29, d31, d25 // t28a + vqadd.s16 d31, d31, d25 // t31a + + vmull_vmlsl q2, d2, d3, d0[2], d0[3] // -> t18a + vmull_vmlal q3, d2, d3, d0[3], d0[2] // -> t29a + vmull_vmlsl q4, d29, d24, d0[2], d0[3] // -> t19 + vrshrn.i32 d18, q2, #12 // t18a + vrshrn.i32 d25, q3, #12 // t29a + vmull_vmlal q1, d29, d24, d0[3], d0[2] // -> t28 + vmull_vmlal q2, d26, d19, d0[3], d0[2] // -> t20 + vrshrn.i32 d29, q4, #12 // t19 + vrshrn.i32 d24, q1, #12 // t28 + vneg.s32 q2, q2 // -> t20 + vmull_vmlsl q3, d26, d19, d0[2], d0[3] // -> t27 + vmull_vmlal q4, d20, d28, d0[3], d0[2] // -> t21a + vrshrn.i32 d26, q2, #12 // t20 + vrshrn.i32 d19, q3, #12 // t27 + vneg.s32 q4, q4 // -> t21a + vmull_vmlsl q1, d20, d28, d0[2], d0[3] // -> t26a + vrshrn.i32 d20, q4, #12 // t21a + vrshrn.i32 d28, q1, #12 // t26a + + vqsub.s16 d2, d16, d30 // t23 + vqadd.s16 d16, d16, d30 // t16 = out16 + vqsub.s16 d3, d31, d23 // t24 + vqadd.s16 d31, d31, d23 // t31 = out31 + vqsub.s16 d23, d21, d17 // t22a + vqadd.s16 d17, d21, d17 // t17a = out17 + vqadd.s16 d30, d27, d22 // t30a = out30 + vqsub.s16 d21, d27, d22 // t25a + vqsub.s16 d27, d18, d20 // t21 + vqadd.s16 d18, d18, d20 // t18 = out18 + vqadd.s16 d4, d29, d26 // t19a = out19 + vqsub.s16 d26, d29, d26 // t20a + vqadd.s16 d29, d25, d28 // t29 = out29 + vqsub.s16 d25, d25, d28 // t26 + vqadd.s16 d28, d24, d19 // t28a = out28 + vqsub.s16 d24, d24, d19 // t27a + vmov d19, d4 // out19 + + vmull_vmlsl q2, d24, d26, d0[0], d0[0] // -> t20 + vmull_vmlal q3, d24, d26, d0[0], d0[0] // -> t27 + vrshrn.i32 d20, q2, #12 // t20 + vrshrn.i32 d22, q3, #12 // t27 + + vmull_vmlal q2, d25, d27, d0[0], d0[0] // -> t26a + vmull_vmlsl q3, d25, d27, d0[0], d0[0] // -> t21a + vmov d27, d22 // t27 + vrshrn.i32 d26, q2, #12 // t26a + + vmull_vmlsl q12, d21, d23, d0[0], d0[0] // -> t22 + vmull_vmlal q2, d21, d23, d0[0], d0[0] // -> t25 + vrshrn.i32 d21, q3, #12 // t21a + vrshrn.i32 d22, q12, #12 // t22 + vrshrn.i32 d25, q2, #12 // t25 + + vmull_vmlsl q2, d3, d2, d0[0], d0[0] // -> t23a + vmull_vmlal q3, d3, d2, d0[0], d0[0] // -> t24a + vrshrn.i32 d23, q2, #12 // t23a + vrshrn.i32 d24, q3, #12 // t24a + + bx lr +endfunc + +.macro def_horz_32 scale=0, shift=2, suffix +function inv_txfm_horz\suffix\()_dct_32x4_neon + push {lr} + vmov.i16 d7, #0 + lsl r8, r8, #1 +.if \scale + movw r12, #2896*8 + vdup.16 d0, r12 +.endif + +.irp i, d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, d28, d29, d30, d31 + vld1.16 {\i}, [r7, :64] + vst1.16 {d7}, [r7, :64], r8 +.endr + sub r7, r7, r8, lsl #4 + add r7, r7, r8, lsr #1 +.if \scale + scale_input d0[0], q8, q9, q10, q11, q12, q13, q14, q15 +.endif + bl inv_dct_4h_x16_neon + transpose_4x4h q8, q9, d16, d17, d18, d19 + transpose_4x4h q10, q11, d20, d21, d22, d23 + transpose_4x4h q12, q13, d24, d25, d26, d27 + transpose_4x4h q14, q15, d28, d29, d30, d31 + +.macro store1 r0, r1, r2, r3 + vst1.16 {\r0}, [r6, :64]! + vst1.16 {\r1}, [r6, :64]! + vst1.16 {\r2}, [r6, :64]! + vst1.16 {\r3}, [r6, :64]! + add r6, r6, #32 +.endm + store1 d16, d20, d24, d28 + store1 d17, d21, d25, d29 + store1 d18, d22, d26, d30 + store1 d19, d23, d27, d31 +.purgem store1 + sub r6, r6, #64*4 + + vmov.i16 d7, #0 +.irp i, d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, d28, d29, d30, d31 + vld1.16 {\i}, [r7, :64] + vst1.16 {d7}, [r7, :64], r8 +.endr +.if \scale + // This relies on the fact that the idct also leaves the right coeff in d0[1] + scale_input d0[1], q8, q9, q10, q11, q12, q13, q14, q15 +.endif + bl inv_dct32_odd_4h_x16_neon + transpose_4x4h q15, q14, d31, d30, d29, d28 + transpose_4x4h q13, q12, d27, d26, d25, d24 + transpose_4x4h q11, q10, d23, d22, d21, d20 + transpose_4x4h q9, q8, d19, d18, d17, d16 +.macro store2 r0, r1, r2, r3, shift + vld1.16 {q0, q1}, [r6, :128] + vqsub.s16 d7, d0, \r0 + vqadd.s16 d0, d0, \r0 + vqsub.s16 d6, d1, \r1 + vqadd.s16 d1, d1, \r1 + vqsub.s16 d5, d2, \r2 + vqadd.s16 d2, d2, \r2 + vqsub.s16 d4, d3, \r3 + vqadd.s16 d3, d3, \r3 + vrev64.16 q2, q2 + vrev64.16 q3, q3 + vrshr.s16 q0, q0, #\shift + vrshr.s16 q1, q1, #\shift + vrshr.s16 q2, q2, #\shift + vrshr.s16 q3, q3, #\shift + vst1.16 {q0, q1}, [r6, :128]! + vst1.16 {q2, q3}, [r6, :128]! +.endm + + store2 d31, d27, d23, d19, \shift + store2 d30, d26, d22, d18, \shift + store2 d29, d25, d21, d17, \shift + store2 d28, d24, d20, d16, \shift +.purgem store2 + pop {pc} +endfunc +.endm + +def_horz_32 scale=0, shift=2 +def_horz_32 scale=1, shift=1, suffix=_scale + +function inv_txfm_add_vert_dct_4x32_neon + push {r10-r11,lr} + lsl r8, r8, #1 + +.irp i, d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, d28, d29, d30, d31 + vld1.16 {\i}, [r7, :64], r8 +.endr + sub r7, r7, r8, lsl #4 + + bl inv_dct_4h_x16_neon + +.irp i, d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, d28, d29, d30, d31 + vst1.16 {\i}, [r7, :64], r8 +.endr + sub r7, r7, r8, lsl #4 + add r7, r7, r8, lsr #1 + +.irp i, d16, d17, d18, d19, d20, d21, d22, d23, d24, d25, d26, d27, d28, d29, d30, d31 + vld1.16 {\i}, [r7, :64], r8 +.endr + sub r7, r7, r8, lsl #4 + sub r7, r7, r8, lsr #1 + bl inv_dct32_odd_4h_x16_neon + + neg r9, r8 + mov r10, r6 +.macro combine r0, r1, r2, r3, op, stride + vld1.16 {d4}, [r7, :64], \stride + vld1.32 {d2[0]}, [r10, :32], r1 + vld1.16 {d5}, [r7, :64], \stride + vld1.32 {d2[1]}, [r10, :32], r1 + \op\().s16 d4, d4, \r0 + vld1.16 {d6}, [r7, :64], \stride + vld1.32 {d3[0]}, [r10, :32], r1 + \op\().s16 d5, d5, \r1 + vld1.32 {d3[1]}, [r10, :32], r1 + vrshr.s16 q2, q2, #4 + \op\().s16 d6, d6, \r2 + vld1.16 {d7}, [r7, :64], \stride + vaddw.u8 q2, q2, d2 + \op\().s16 d7, d7, \r3 + vqmovun.s16 d2, q2 + vrshr.s16 q3, q3, #4 + vst1.32 {d2[0]}, [r6, :32], r1 + vaddw.u8 q3, q3, d3 + vst1.32 {d2[1]}, [r6, :32], r1 + vqmovun.s16 d3, q3 + vst1.32 {d3[0]}, [r6, :32], r1 + vst1.32 {d3[1]}, [r6, :32], r1 +.endm + combine d31, d30, d29, d28, vqadd, r8 + combine d27, d26, d25, d24, vqadd, r8 + combine d23, d22, d21, d20, vqadd, r8 + combine d19, d18, d17, d16, vqadd, r8 + sub r7, r7, r8 + combine d16, d17, d18, d19, vqsub, r9 + combine d20, d21, d22, d23, vqsub, r9 + combine d24, d25, d26, d27, vqsub, r9 + combine d28, d29, d30, d31, vqsub, r9 +.purgem combine + + pop {r10-r11,pc} +endfunc + +const eob_32x32 + .short 10, 36, 78, 136, 210, 300, 406, 1024 +endconst + +const eob_16x32 + .short 10, 36, 78, 151, 215, 279, 343, 512 +endconst + +const eob_16x32_shortside + .short 10, 36, 78, 512 +endconst + +const eob_8x32 + // Contrary to the others, this one is only ever used in increments of 8x8 + .short 43, 107, 171, 256 +endconst + +function inv_txfm_add_identity_identity_32x32_8bpc_neon, export=1 + push {r4-r7,lr} + vmov.i16 q0, #0 + movrel_local r5, eob_32x32, 2 + + mov r6, #2*32 +1: + mov r12, #0 + movrel_local r4, eob_32x32, 2 +2: + add r12, r12, #8 +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vld1.16 {\i}, [r2, :128] + vst1.16 {q0}, [r2, :128], r6 +.endr + transpose_8x8h q8, q9, q10, q11, q12, q13, q14, q15, d17, d19, d21, d23, d24, d26, d28, d30 + + load_add_store_8x8 r0, r7, shiftbits=2 + ldrh lr, [r4], #4 + sub r0, r0, r1, lsl #3 + cmp r3, lr + add r0, r0, #8 + bge 2b + + ldrh lr, [r5], #4 + cmp r3, lr + blt 9f + + sub r0, r0, r12 + add r0, r0, r1, lsl #3 + mls r2, r6, r12, r2 + add r2, r2, #2*8 + b 1b +9: + pop {r4-r7,pc} +endfunc + +.macro shift_8_regs op, shift +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + \op \i, \i, #\shift +.endr +.endm + +.macro def_identity_1632 w, h, wshort, hshort +function inv_txfm_add_identity_identity_\w\()x\h\()_8bpc_neon, export=1 + push {r4-r7,lr} + movw r6, #2896*8 + movw r7, #2*(5793-4096)*8 + vdup.i16 d0, r6 + movrel_local r5, eob_16x32\hshort, 2 + vmov.16 d0[1], r7 + + mov r6, #2*\h +1: + mov r12, #0 + movrel_local r4, eob_16x32\wshort, 2 +2: + vmov.i16 q1, #0 + add r12, r12, #8 +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vld1.16 {\i}, [r2, :128] + vst1.16 {q1}, [r2, :128], r6 +.endr + scale_input d0[0], q8, q9, q10, q11, q12, q13, q14, q15 + +.if \w == 16 + // 16x32 + identity_8x8_shift1 d0[1] +.else + // 32x16 + shift_8_regs vqshl.s16, 1 + identity_8x8 d0[1] +.endif + + transpose_8x8h q8, q9, q10, q11, q12, q13, q14, q15, d17, d19, d21, d23, d24, d26, d28, d30 + +.if \w == 16 + load_add_store_8x8 r0, r7, shiftbits=2 +.else + load_add_store_8x8 r0, r7, shiftbits=4 +.endif + ldrh lr, [r4], #4 + sub r0, r0, r1, lsl #3 + cmp r3, lr + add r0, r0, #8 + bge 2b + + ldrh lr, [r5], #4 + cmp r3, lr + blt 9f + + sub r0, r0, r12 + add r0, r0, r1, lsl #3 + mls r2, r6, r12, r2 + add r2, r2, #2*8 + b 1b +9: + pop {r4-r7,pc} +endfunc +.endm + +def_identity_1632 16, 32, _shortside, +def_identity_1632 32, 16, , _shortside + +.macro def_identity_832 w, h +function inv_txfm_add_identity_identity_\w\()x\h\()_8bpc_neon, export=1 + push {r4-r5,lr} + vmov.i16 q0, #0 + movrel_local r4, eob_8x32 + + mov r12, #2*\h +1: + ldrh lr, [r4], #2 +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vld1.16 {\i}, [r2, :128] + vst1.16 {q0}, [r2, :128], r12 +.endr + +.if \w == 8 + // 8x32 + shift_8_regs vrshr.s16, 1 +.endif + + transpose_8x8h q8, q9, q10, q11, q12, q13, q14, q15, d17, d19, d21, d23, d24, d26, d28, d30 + + cmp r3, lr +.if \w == 8 + load_add_store_8x8 r0, r5, shiftbits=2 +.else + load_add_store_8x8 r0, r5, shiftbits=3 +.endif + + blt 9f +.if \w == 8 + sub r2, r2, r12, lsl #3 + add r2, r2, #2*8 +.else + sub r0, r0, r1, lsl #3 + add r0, r0, #8 +.endif + b 1b + +9: + pop {r4-r5,pc} +endfunc +.endm + +def_identity_832 8, 32 +def_identity_832 32, 8 + +function inv_txfm_add_dct_dct_32x32_8bpc_neon, export=1 + idct_dc 32, 32, 2 + + push {r4-r11,lr} + vpush {q4} + sub_sp_align 2048 + movrel_local r10, eob_32x32 + ldrh r11, [r10], #2 + +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add r6, sp, #(\i*32*2) +.if \i > 0 + mov r8, #(32 - \i) + cmp r3, r11 + blt 1f +.if \i < 28 + ldrh r11, [r10], #2 +.endif +.endif + add r7, r2, #(\i*2) + mov r8, #32*2 + bl inv_txfm_horz_dct_32x4_neon +.endr + b 3f + +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r8, r8, #2 +.rept 4 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b + +3: +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add r6, r0, #(\i) + add r7, sp, #(\i*2) + mov r8, #32*2 + bl inv_txfm_add_vert_dct_4x32_neon +.endr + + add_sp_align 2048 + vpop {q4} + pop {r4-r11,pc} +endfunc + +function inv_txfm_add_dct_dct_16x32_8bpc_neon, export=1 + idct_dc 16, 32, 1 + + push {r4-r11,lr} + vpush {q4} + sub_sp_align 1024 + movrel_local r10, eob_16x32 + ldrh r11, [r10], #2 + movrel_local r4, inv_dct_4h_x16_neon + +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add r6, sp, #(\i*16*2) + add r7, r2, #(\i*2) +.if \i > 0 + mov r8, #(32 - \i) + cmp r3, r11 + blt 1f +.if \i < 28 + ldrh r11, [r10], #2 +.endif +.endif + mov r8, #2*32 + bl inv_txfm_horz_scale_16x4_neon +.endr + b 3f + +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r8, r8, #4 +.rept 4 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b + +3: +.irp i, 0, 4, 8, 12 + add r6, r0, #(\i) + add r7, sp, #(\i*2) + mov r8, #16*2 + bl inv_txfm_add_vert_dct_4x32_neon +.endr + + add_sp_align 1024 + vpop {q4} + pop {r4-r11,pc} +endfunc + +function inv_txfm_add_dct_dct_32x16_8bpc_neon, export=1 + idct_dc 32, 16, 1 + + push {r4-r11,lr} + vpush {q4} + sub_sp_align 1024 + movrel_local r10, eob_16x32 + ldrh r11, [r10], #2 + movrel_local r5, inv_dct_4h_x16_neon + +.irp i, 0, 4, 8, 12 + add r6, sp, #(\i*32*2) + add r7, r2, #(\i*2) +.if \i > 0 + mov r8, #(16 - \i) + cmp r3, r11 + blt 1f +.if \i < 12 + ldrh r11, [r10], #2 +.endif +.endif + mov r8, #2*16 + bl inv_txfm_horz_scale_dct_32x4_neon +.endr + b 3f + +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r8, r8, #2 +.rept 4 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b + +3: +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add r6, r0, #(\i) + add r7, sp, #(\i*2) + mov r8, #32*2 + bl inv_txfm_add_vert_4x16_neon +.endr + + add_sp_align 1024 + vpop {q4} + pop {r4-r11,pc} +endfunc + +function inv_txfm_add_dct_dct_8x32_8bpc_neon, export=1 + idct_dc 8, 32, 2 + + push {r4-r11,lr} + vpush {q4-q7} + sub_sp_align 512 + + movrel_local r10, eob_8x32 + + mov r8, #2*32 + mov r9, #32 + mov r6, sp +1: + vmov.i16 q0, #0 +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vld1.16 {\i}, [r2, :128] + vst1.16 {q0}, [r2, :128], r8 +.endr + ldrh r11, [r10], #2 + sub r2, r2, r8, lsl #3 + sub r9, r9, #8 + add r2, r2, #2*8 + + bl inv_dct_8h_x8_neon + +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vrshr.s16 \i, \i, #2 +.endr + + transpose_8x8h q8, q9, q10, q11, q12, q13, q14, q15, d17, d19, d21, d23, d24, d26, d28, d30 + + vst1.16 {q8, q9}, [r6, :128]! + cmp r3, r11 + vst1.16 {q10, q11}, [r6, :128]! + vst1.16 {q12, q13}, [r6, :128]! + vst1.16 {q14, q15}, [r6, :128]! + + bge 1b + cmp r9, #0 + beq 3f + + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r9, r9, #8 +.rept 4 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b + +3: +.irp i, 0, 4 + add r6, r0, #(\i) + add r7, sp, #(\i*2) + mov r8, #8*2 + bl inv_txfm_add_vert_dct_4x32_neon +.endr + + add_sp_align 512 + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +function inv_txfm_add_dct_dct_32x8_8bpc_neon, export=1 + idct_dc 32, 8, 2 + + push {r4-r11,lr} + vpush {q4-q7} + sub_sp_align 512 + +.irp i, 0, 4 + add r6, sp, #(\i*32*2) + add r7, r2, #(\i*2) +.if \i > 0 + cmp r3, #10 + blt 1f +.endif + mov r8, #8*2 + bl inv_txfm_horz_dct_32x4_neon +.endr + b 2f + +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +.rept 8 + vst1.16 {q2, q3}, [r6, :128]! +.endr + +2: + mov r8, #2*32 + mov r9, #0 +1: + add r6, r0, r9 + add r7, sp, r9, lsl #1 // #(\i*2) + +.irp i, q8, q9, q10, q11, q12, q13, q14, q15 + vld1.16 {\i}, [r7, :128], r8 +.endr + add r9, r9, #8 + + bl inv_dct_8h_x8_neon + + cmp r9, #32 + + load_add_store_8x8 r6, r7 + + blt 1b + + add_sp_align 512 + vpop {q4-q7} + pop {r4-r11,pc} +endfunc + +function inv_dct64_step1_neon + // in1/31/17/15 -> t32a/33/34a/35/60/61a/62/63a + // in7/25/23/ 9 -> t56a/57/58a/59/36/37a/38/39a + // in5/27/21/11 -> t40a/41/42a/43/52/53a/54/55a + // in3/29/19/13 -> t48a/49/50a/51/44/45a/46/47a + + vld1.16 {d0, d1, d2}, [r12, :64]! + + vqrdmulh.s16 d23, d16, d0[1] // t63a + vqrdmulh.s16 d16, d16, d0[0] // t32a + vqrdmulh.s16 d22, d17, d0[2] // t62a + vqrdmulh.s16 d17, d17, d0[3] // t33a + vqrdmulh.s16 d21, d18, d1[1] // t61a + vqrdmulh.s16 d18, d18, d1[0] // t34a + vqrdmulh.s16 d20, d19, d1[2] // t60a + vqrdmulh.s16 d19, d19, d1[3] // t35a + + vqadd.s16 d24, d16, d17 // t32 + vqsub.s16 d25, d16, d17 // t33 + vqsub.s16 d26, d19, d18 // t34 + vqadd.s16 d27, d19, d18 // t35 + vqadd.s16 d28, d20, d21 // t60 + vqsub.s16 d29, d20, d21 // t61 + vqsub.s16 d30, d23, d22 // t62 + vqadd.s16 d31, d23, d22 // t63 + + vmull_vmlal q2, d29, d26, d2[0], d2[1] // -> t34a + vmull_vmlsl q3, d29, d26, d2[1], d2[0] // -> t61a + vneg.s32 q2, q2 // t34a + vmull_vmlsl q4, d30, d25, d2[1], d2[0] // -> t33a + vrshrn.i32 d26, q2, #12 // t34a + vmull_vmlal q2, d30, d25, d2[0], d2[1] // -> t62a + vrshrn.i32 d29, q3, #12 // t61a + vrshrn.i32 d25, q4, #12 // t33a + vrshrn.i32 d30, q2, #12 // t62a + + vqadd.s16 d16, d24, d27 // t32a + vqsub.s16 d19, d24, d27 // t35a + vqadd.s16 d17, d25, d26 // t33 + vqsub.s16 d18, d25, d26 // t34 + vqsub.s16 d20, d31, d28 // t60a + vqadd.s16 d23, d31, d28 // t63a + vqsub.s16 d21, d30, d29 // t61 + vqadd.s16 d22, d30, d29 // t62 + + vmull_vmlal q2, d21, d18, d2[2], d2[3] // -> t61a + vmull_vmlsl q3, d21, d18, d2[3], d2[2] // -> t34a + vmull_vmlal q4, d20, d19, d2[2], d2[3] // -> t60 + vrshrn.i32 d21, q2, #12 // t61a + vrshrn.i32 d18, q3, #12 // t34a + vmull_vmlsl q2, d20, d19, d2[3], d2[2] // -> t35 + vrshrn.i32 d20, q4, #12 // t60 + vrshrn.i32 d19, q2, #12 // t35 + + vst1.16 {d16, d17, d18, d19}, [r6, :128]! + vst1.16 {d20, d21, d22, d23}, [r6, :128]! + + bx lr +endfunc + +function inv_dct64_step2_neon + movrel_local r12, idct_coeffs + vld1.16 {d0}, [r12, :64] +1: + // t32a/33/34a/35/60/61a/62/63a + // t56a/57/58a/59/36/37a/38/39a + // t40a/41/42a/43/52/53a/54/55a + // t48a/49/50a/51/44/45a/46/47a + vldr d16, [r6, #2*4*0] // t32a + vldr d17, [r9, #2*4*8] // t39a + vldr d18, [r9, #2*4*0] // t63a + vldr d19, [r6, #2*4*8] // t56a + vldr d20, [r6, #2*4*16] // t40a + vldr d21, [r9, #2*4*24] // t47a + vldr d22, [r9, #2*4*16] // t55a + vldr d23, [r6, #2*4*24] // t48a + + vqadd.s16 d24, d16, d17 // t32 + vqsub.s16 d25, d16, d17 // t39 + vqadd.s16 d26, d18, d19 // t63 + vqsub.s16 d27, d18, d19 // t56 + vqsub.s16 d28, d21, d20 // t40 + vqadd.s16 d29, d21, d20 // t47 + vqadd.s16 d30, d23, d22 // t48 + vqsub.s16 d31, d23, d22 // t55 + + vmull_vmlal q2, d27, d25, d0[3], d0[2] // -> t56a + vmull_vmlsl q3, d27, d25, d0[2], d0[3] // -> t39a + vmull_vmlal q4, d31, d28, d0[3], d0[2] // -> t40a + vrshrn.i32 d25, q2, #12 // t56a + vrshrn.i32 d27, q3, #12 // t39a + vneg.s32 q4, q4 // t40a + vmull_vmlsl q2, d31, d28, d0[2], d0[3] // -> t55a + vrshrn.i32 d31, q4, #12 // t40a + vrshrn.i32 d28, q2, #12 // t55a + + vqadd.s16 d16, d24, d29 // t32a + vqsub.s16 d19, d24, d29 // t47a + vqadd.s16 d17, d27, d31 // t39 + vqsub.s16 d18, d27, d31 // t40 + vqsub.s16 d20, d26, d30 // t48a + vqadd.s16 d23, d26, d30 // t63a + vqsub.s16 d21, d25, d28 // t55 + vqadd.s16 d22, d25, d28 // t56 + + vmull_vmlsl q2, d21, d18, d0[0], d0[0] // -> t40a + vmull_vmlal q3, d21, d18, d0[0], d0[0] // -> t55a + vmull_vmlsl q4, d20, d19, d0[0], d0[0] // -> t47 + vrshrn.i32 d18, q2, #12 // t40a + vrshrn.i32 d21, q3, #12 // t55a + vmull_vmlal q2, d20, d19, d0[0], d0[0] // -> t48 + vrshrn.i32 d19, q4, #12 // t47 + vrshrn.i32 d20, q2, #12 // t48 + + vstr d16, [r6, #2*4*0] // t32a + vstr d17, [r9, #2*4*0] // t39 + vstr d18, [r6, #2*4*8] // t40a + vstr d19, [r9, #2*4*8] // t47 + vstr d20, [r6, #2*4*16] // t48 + vstr d21, [r9, #2*4*16] // t55a + vstr d22, [r6, #2*4*24] // t56 + vstr d23, [r9, #2*4*24] // t63a + + add r6, r6, #2*4 + sub r9, r9, #2*4 + cmp r6, r9 + blt 1b + bx lr +endfunc + +.macro load8 src, strd, zero, clear +.irp i, d16, d17, d18, d19, d20, d21, d22, d23 +.if \clear + vld1.16 {\i}, [\src, :64] + vst1.16 {\zero}, [\src, :64], \strd +.else + vld1.16 {\i}, [\src, :64], \strd +.endif +.endr +.endm + +.macro store16 dst + vst1.16 {q8, q9}, [\dst, :128]! + vst1.16 {q10, q11}, [\dst, :128]! + vst1.16 {q12, q13}, [\dst, :128]! + vst1.16 {q14, q15}, [\dst, :128]! +.endm + +.macro clear_upper8 +.irp i, q12, q13, q14, q15 + vmov.i16 \i, #0 +.endr +.endm + +.macro vmov_if reg, val, cond +.if \cond + vmov.i16 \reg, \val +.endif +.endm + +.macro movdup_if reg, gpr, val, cond +.if \cond + movw \gpr, \val + vdup.16 \reg, \gpr +.endif +.endm + +.macro vst1_if regs, dst, dstalign, cond +.if \cond + vst1.16 \regs, \dst, \dstalign +.endif +.endm + +.macro scale_if cond, c, r0, r1, r2, r3, r4, r5, r6, r7 +.if \cond + scale_input \c, \r0, \r1, \r2, \r3, \r4, \r5, \r6, \r7 +.endif +.endm + +.macro def_dct64_func suffix, clear=0, scale=0 +function inv_txfm_dct\suffix\()_4h_x64_neon, export=1 + mov r6, sp + + push {r10-r11,lr} + + lsl r8, r8, #2 + + movdup_if d0, r12, #2896*8, \scale + vmov_if d7, #0, \clear + load8 r7, r8, d7, \clear + clear_upper8 + sub r7, r7, r8, lsl #3 + add r7, r7, r8, lsr #1 + scale_if \scale, d0[0], q8, q9, q10, q11 + + bl inv_dct_4h_x16_neon + + store16 r6 + + movdup_if d0, r12, #2896*8, \scale + vmov_if d7, #0, \clear + load8 r7, r8, d7, \clear + clear_upper8 + sub r7, r7, r8, lsl #3 + lsr r8, r8, #1 + sub r7, r7, r8, lsr #1 + scale_if \scale, d0[0], q8, q9, q10, q11 + + bl inv_dct32_odd_4h_x16_neon + + add r10, r6, #8*15 + sub r6, r6, #8*16 + + mov r9, #-8 + +.macro store_addsub r0, r1, r2, r3 + vld1.16 {d2}, [r6, :64]! + vld1.16 {d3}, [r6, :64]! + vqadd.s16 d6, d2, \r0 + vqsub.s16 \r0, d2, \r0 + vld1.16 {d4}, [r6, :64]! + vqadd.s16 d7, d3, \r1 + vqsub.s16 \r1, d3, \r1 + vld1.16 {d5}, [r6, :64]! + vqadd.s16 d2, d4, \r2 + sub r6, r6, #8*4 + vqsub.s16 \r2, d4, \r2 + vst1.16 {d6}, [r6, :64]! + vst1.16 {\r0}, [r10, :64], r9 + vqadd.s16 d3, d5, \r3 + vqsub.s16 \r3, d5, \r3 + vst1.16 {d7}, [r6, :64]! + vst1.16 {\r1}, [r10, :64], r9 + vst1.16 {d2}, [r6, :64]! + vst1.16 {\r2}, [r10, :64], r9 + vst1.16 {d3}, [r6, :64]! + vst1.16 {\r3}, [r10, :64], r9 +.endm + store_addsub d31, d30, d29, d28 + store_addsub d27, d26, d25, d24 + store_addsub d23, d22, d21, d20 + store_addsub d19, d18, d17, d16 +.purgem store_addsub + + add r6, r6, #2*4*16 + + movrel_local r12, idct64_coeffs + movdup_if d0, lr, #2896*8, \scale + vmov_if d7, #0, \clear + add r9, r7, r8, lsl #4 // offset 16 + add r10, r7, r8, lsl #3 // offset 8 + sub r9, r9, r8 // offset 15 + sub r11, r10, r8 // offset 7 + vld1.16 {d16}, [r7, :64] // in1 (offset 0) + vld1.16 {d17}, [r9, :64] // in31 (offset 15) + vld1.16 {d18}, [r10, :64] // in17 (offset 8) + vld1.16 {d19}, [r11, :64] // in15 (offset 7) + vst1_if {d7}, [r7, :64], \clear + vst1_if {d7}, [r9, :64], \clear + vst1_if {d7}, [r10, :64], \clear + vst1_if {d7}, [r11, :64], \clear + scale_if \scale, d0[0], q8, q9 + bl inv_dct64_step1_neon + movdup_if d0, lr, #2896*8, \scale + vmov_if d7, #0, \clear + add r7, r7, r8, lsl #2 // offset 4 + sub r9, r9, r8, lsl #2 // offset 11 + sub r10, r7, r8 // offset 3 + add r11, r9, r8 // offset 12 + vld1.16 {d16}, [r10, :64] // in7 (offset 3) + vld1.16 {d17}, [r11, :64] // in25 (offset 12) + vld1.16 {d18}, [r9, :64] // in23 (offset 11) + vld1.16 {d19}, [r7, :64] // in9 (offset 4) + vst1_if {d7}, [r7, :64], \clear + vst1_if {d7}, [r9, :64], \clear + vst1_if {d7}, [r10, :64], \clear + vst1_if {d7}, [r11, :64], \clear + scale_if \scale, d0[0], q8, q9 + bl inv_dct64_step1_neon + movdup_if d0, lr, #2896*8, \scale + vmov_if d7, #0, \clear + sub r10, r10, r8, lsl #1 // offset 1 + sub r9, r9, r8, lsl #1 // offset 9 + add r10, r10, r8 // offset 2 + add r9, r9, r8 // offset 10 + add r7, r7, r8 // offset 5 + add r11, r11, r8 // offset 13 + vld1.16 d16, [r10, :64] // in5 (offset 2) + vld1.16 d17, [r11, :64] // in27 (offset 13) + vld1.16 d18, [r9, :64] // in21 (offset 10) + vld1.16 d19, [r7, :64] // in11 (offset 5) + vst1_if d7, [r10, :64], \clear + vst1_if d7, [r11, :64], \clear + vst1_if d7, [r9, :64], \clear + vst1_if d7, [r7, :64], \clear + scale_if \scale, d0[0], q8, q9 + bl inv_dct64_step1_neon + movdup_if d0, lr, #2896*8, \scale + vmov_if d7, #0, \clear + sub r10, r10, r8 // offset 1 + sub r9, r9, r8 // offset 9 + add r11, r11, r8 // offset 14 + add r7, r7, r8 // offset 6 + vld1.16 d16, [r10, :64] // in3 (offset 1) + vld1.16 d17, [r11, :64] // in29 (offset 14) + vld1.16 d18, [r9, :64] // in19 (offset 9) + vld1.16 d19, [r7, :64] // in13 (offset 6) + vst1_if d7, [r10, :64], \clear + vst1_if d7, [r11, :64], \clear + vst1_if d7, [r9, :64], \clear + vst1_if d7, [r7, :64], \clear + scale_if \scale, d0[0], q8, q9 + bl inv_dct64_step1_neon + + sub r6, r6, #2*4*32 + add r9, r6, #2*4*7 + + bl inv_dct64_step2_neon + + pop {r10-r11,pc} +endfunc +.endm + +def_dct64_func +def_dct64_func _clear, clear=1 +def_dct64_func _clear_scale, clear=1, scale=1 + +function inv_txfm_horz_dct_64x4_neon + vdup.16 q3, r9 + + mov r7, sp + add r8, sp, #2*4*(64 - 4) + add r9, r6, #2*56 + + push {r10-r11,lr} + + mov r10, #2*64 + mov r11, #-2*4*4 + +1: + vld1.16 {d16, d17, d18, d19}, [r7, :128]! + vld1.16 {d28, d29, d30, d31}, [r8, :128], r11 + vld1.16 {d20, d21, d22, d23}, [r7, :128]! + vld1.16 {d24, d25, d26, d27}, [r8, :128], r11 + transpose_4x4h q8, q9, d16, d17, d18, d19 + transpose_4x4h q15, q14, d31, d30, d29, d28 + transpose_4x4h q10, q11, d20, d21, d22, d23 + transpose_4x4h q13, q12, d27, d26, d25, d24 + +.macro store_addsub src0, src1, src2, src3 + vqsub.s16 d3, \src0, \src1 + vqsub.s16 d2, \src2, \src3 + vqadd.s16 d0, \src0, \src1 + vqadd.s16 d1, \src2, \src3 + vrshl.s16 q1, q1, q3 + vrshl.s16 q0, q0, q3 + vrev64.16 q1, q1 + vst1.16 {q0}, [r6, :128], r10 + vst1.16 {q1}, [r9, :128], r10 +.endm + store_addsub d16, d31, d20, d27 + store_addsub d17, d30, d21, d26 + store_addsub d18, d29, d22, d25 + store_addsub d19, d28, d23, d24 +.purgem store_addsub + sub r6, r6, r10, lsl #2 + sub r9, r9, r10, lsl #2 + add r6, r6, #16 + sub r9, r9, #16 + + cmp r7, r8 + blt 1b + pop {r10-r11,pc} +endfunc + +function inv_txfm_add_vert_dct_4x64_neon + lsl r8, r8, #1 + + mov r7, sp + add r8, sp, #2*4*(64 - 4) + add r9, r6, r1, lsl #6 + sub r9, r9, r1 + + push {r10-r11,lr} + + neg r10, r1 + mov r11, #-2*4*4 + +1: + vld1.16 {d16, d17, d18, d19}, [r7, :128]! + vld1.16 {d28, d29, d30, d31}, [r8, :128], r11 + vld1.16 {d20, d21, d22, d23}, [r7, :128]! + vld1.16 {d24, d25, d26, d27}, [r8, :128], r11 + +.macro add_dest_addsub src0, src1, src2, src3 + vld1.32 {d0[0]}, [r6, :32], r1 + vld1.32 {d1[0]}, [r9, :32], r10 + vqadd.s16 d4, \src0, \src1 + vld1.32 {d0[1]}, [r6, :32] + vqadd.s16 d5, \src2, \src3 + vld1.32 {d1[1]}, [r9, :32] + vqsub.s16 d6, \src0, \src1 + vqsub.s16 d7, \src2, \src3 + sub r6, r6, r1 + sub r9, r9, r10 + vrshr.s16 q2, q2, #4 + vrshr.s16 q3, q3, #4 + vaddw.u8 q2, q2, d0 + vaddw.u8 q3, q3, d1 + vqmovun.s16 d0, q2 + vqmovun.s16 d1, q3 + vst1.32 {d0[0]}, [r6, :32], r1 + vst1.32 {d1[0]}, [r9, :32], r10 + vst1.32 {d0[1]}, [r6, :32], r1 + vst1.32 {d1[1]}, [r9, :32], r10 +.endm + add_dest_addsub d16, d31, d17, d30 + add_dest_addsub d18, d29, d19, d28 + add_dest_addsub d20, d27, d21, d26 + add_dest_addsub d22, d25, d23, d24 +.purgem add_dest_addsub + cmp r7, r8 + blt 1b + + pop {r10-r11,pc} +endfunc + +function inv_txfm_add_dct_dct_64x64_8bpc_neon, export=1 + idct_dc 64, 64, 2 + + push {r4-r11,lr} + vpush {q4} + + sub_sp_align 64*32*2+64*4*2 + add r5, sp, #64*4*2 + + movrel_local r10, eob_32x32 + +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add r6, r5, #(\i*64*2) +.if \i > 0 + mov r8, #(32 - \i) + cmp r3, r11 + blt 1f +.endif + add r7, r2, #(\i*2) + mov r8, #32*2 + bl inv_txfm_dct_clear_4h_x64_neon + add r6, r5, #(\i*64*2) + mov r9, #-2 // shift + bl inv_txfm_horz_dct_64x4_neon +.if \i < 28 + ldrh r11, [r10], #2 +.endif +.endr + b 3f + +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r8, r8, #2 +.rept 8 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b + +3: +.irp i, 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60 + add r7, r5, #(\i*2) + mov r8, #64*2 + bl inv_txfm_dct_4h_x64_neon + add r6, r0, #(\i) + bl inv_txfm_add_vert_dct_4x64_neon +.endr + + add_sp_align 64*32*2+64*4*2 + vpop {q4} + pop {r4-r11,pc} +endfunc + +function inv_txfm_add_dct_dct_64x32_8bpc_neon, export=1 + idct_dc 64, 32, 1 + + push {r4-r11,lr} + vpush {q4} + + sub_sp_align 64*32*2+64*4*2 + add r5, sp, #64*4*2 + + movrel_local r10, eob_32x32 + +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add r6, r5, #(\i*64*2) +.if \i > 0 + mov r8, #(32 - \i) + cmp r3, r11 + blt 1f +.endif + add r7, r2, #(\i*2) + mov r8, #32*2 + bl inv_txfm_dct_clear_scale_4h_x64_neon + add r6, r5, #(\i*64*2) + mov r9, #-1 // shift + bl inv_txfm_horz_dct_64x4_neon +.if \i < 28 + ldrh r11, [r10], #2 +.endif +.endr + b 3f + +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r8, r8, #2 +.rept 8 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b + +3: +.irp i, 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60 + add r6, r0, #(\i) + add r7, r5, #(\i*2) + mov r8, #64*2 + bl inv_txfm_add_vert_dct_4x32_neon +.endr + + add_sp_align 64*32*2+64*4*2 + vpop {q4} + pop {r4-r11,pc} +endfunc + +function inv_txfm_add_dct_dct_32x64_8bpc_neon, export=1 + idct_dc 32, 64, 1 + + push {r4-r11,lr} + vpush {q4} + + sub_sp_align 32*32*2+64*4*2 + add r5, sp, #64*4*2 + + movrel_local r10, eob_32x32 + ldrh r11, [r10], #2 + +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add r6, r5, #(\i*32*2) +.if \i > 0 + mov r8, #(32 - \i) + cmp r3, r11 + blt 1f + ldrh r11, [r10], #2 +.endif + add r7, r2, #(\i*2) + mov r8, #32*2 + bl inv_txfm_horz_scale_dct_32x4_neon +.endr + b 3f + +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r8, r8, #2 +.rept 4 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b + +3: +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add r7, r5, #(\i*2) + mov r8, #32*2 + bl inv_txfm_dct_4h_x64_neon + add r6, r0, #(\i) + bl inv_txfm_add_vert_dct_4x64_neon +.endr + + add_sp_align 32*32*2+64*4*2 + vpop {q4} + pop {r4-r11,pc} +endfunc + +function inv_txfm_add_dct_dct_64x16_8bpc_neon, export=1 + idct_dc 64, 16, 2 + + push {r4-r11,lr} + vpush {q4} + + sub_sp_align 64*16*2+64*4*2 + add r4, sp, #64*4*2 + + movrel_local r10, eob_16x32 + +.irp i, 0, 4, 8, 12 + add r6, r4, #(\i*64*2) +.if \i > 0 + mov r8, #(16 - \i) + cmp r3, r11 + blt 1f +.endif + add r7, r2, #(\i*2) + mov r8, #16*2 + bl inv_txfm_dct_clear_4h_x64_neon + add r6, r4, #(\i*64*2) + mov r9, #-2 // shift + bl inv_txfm_horz_dct_64x4_neon +.if \i < 8 + ldrh r11, [r10], #2 +.endif +.endr + b 3f + +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r8, r8, #2 +.rept 8 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b + +3: + movrel_local r5, inv_dct_4h_x16_neon +.irp i, 0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60 + add r6, r0, #(\i) + add r7, r4, #(\i*2) + mov r8, #64*2 + bl inv_txfm_add_vert_4x16_neon +.endr + + add_sp_align 64*16*2+64*4*2 + vpop {q4} + pop {r4-r11,pc} +endfunc + +function inv_txfm_add_dct_dct_16x64_8bpc_neon, export=1 + idct_dc 16, 64, 2 + + push {r4-r11,lr} + vpush {q4} + + sub_sp_align 16*32*2+64*4*2 + add r5, sp, #64*4*2 + + movrel_local r10, eob_16x32 + ldrh r11, [r10], #2 + + movrel_local r4, inv_dct_4h_x16_neon +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add r6, r5, #(\i*16*2) +.if \i > 0 + mov r8, #(32 - \i) + cmp r3, r11 + blt 1f + ldrh r11, [r10], #2 +.endif + add r7, r2, #(\i*2) + mov r8, #32*2 + bl inv_txfm_horz_16x4_neon +.endr + b 3f + +1: + vmov.i16 q2, #0 + vmov.i16 q3, #0 +2: + subs r8, r8, #4 +.rept 4 + vst1.16 {q2, q3}, [r6, :128]! +.endr + bgt 2b + +3: +.irp i, 0, 4, 8, 12 + add r7, r5, #(\i*2) + mov r8, #16*2 + bl inv_txfm_dct_4h_x64_neon + add r6, r0, #(\i) + bl inv_txfm_add_vert_dct_4x64_neon +.endr + + add_sp_align 16*32*2+64*4*2 + vpop {q4} + pop {r4-r11,pc} +endfunc diff --git a/third_party/dav1d/src/arm/32/mc.S b/third_party/dav1d/src/arm/32/mc.S --- a/third_party/dav1d/src/arm/32/mc.S +++ b/third_party/dav1d/src/arm/32/mc.S @@ -3166,5 +3166,186 @@ endfunc .endm warp , 11 warp t, 7 + +// void dav1d_emu_edge_8bpc_neon( +// const intptr_t bw, const intptr_t bh, +// const intptr_t iw, const intptr_t ih, +// const intptr_t x, const intptr_t y, +// pixel *dst, const ptrdiff_t dst_stride, +// const pixel *ref, const ptrdiff_t ref_stride) +function emu_edge_8bpc_neon, export=1 + push {r4-r11,lr} + ldrd r4, r5, [sp, #36] + ldrd r6, r7, [sp, #44] + ldrd r8, r9, [sp, #52] + + // ref += iclip(y, 0, ih - 1) * PXSTRIDE(ref_stride) + // ref += iclip(x, 0, iw - 1) + sub r12, r3, #1 // ih - 1 + cmp r5, r3 + sub lr, r2, #1 // iw - 1 + it lt + movlt r12, r5 // min(y, ih - 1) + cmp r4, r2 + bic r12, r12, r12, asr #31 // max(min(y, ih - 1), 0) + it lt + movlt lr, r4 // min(x, iw - 1) + bic lr, lr, lr, asr #31 // max(min(x, iw - 1), 0) + mla r8, r12, r9, r8 // ref += iclip() * stride + add r8, r8, lr // ref += iclip() + + // bottom_ext = iclip(y + bh - ih, 0, bh - 1) + // top_ext = iclip(-y, 0, bh - 1) + add r10, r5, r1 // y + bh + neg r5, r5 // -y + sub r10, r10, r3 // y + bh - ih + sub r12, r1, #1 // bh - 1 + cmp r10, r1 + bic r5, r5, r5, asr #31 // max(-y, 0) + it ge + movge r10, r12 // min(y + bh - ih, bh-1) + cmp r5, r1 + bic r10, r10, r10, asr #31 // max(min(y + bh - ih, bh-1), 0) + it ge + movge r5, r12 // min(max(-y, 0), bh-1) + + // right_ext = iclip(x + bw - iw, 0, bw - 1) + // left_ext = iclip(-x, 0, bw - 1) + add r11, r4, r0 // x + bw + neg r4, r4 // -x + sub r11, r11, r2 // x + bw - iw + sub lr, r0, #1 // bw - 1 + cmp r11, r0 + bic r4, r4, r4, asr #31 // max(-x, 0) + it ge + movge r11, lr // min(x + bw - iw, bw-1) + cmp r4, r0 + bic r11, r11, r11, asr #31 // max(min(x + bw - iw, bw-1), 0) + it ge + movge r4, lr // min(max(-x, 0), bw - 1) + + // center_h = bh - top_ext - bottom_ext + // dst += top_ext * PXSTRIDE(dst_stride) + // center_w = bw - left_ext - right_ext + sub r1, r1, r5 // bh - top_ext + mla r6, r5, r7, r6 + sub r2, r0, r4 // bw - left_ext + sub r1, r1, r10 // center_h = bh - top_ext - bottom_ext + sub r2, r2, r11 // center_w = bw - left_ext - right_ext + + mov r0, r6 // backup of dst + +.macro v_loop need_left, need_right +0: +.if \need_left + vld1.8 {d0[]}, [r8] + mov r12, r6 // out = dst + mov r3, r4 + vmov d1, d0 +1: + subs r3, r3, #16 + vst1.8 {q0}, [r12, :128]! + bgt 1b +.endif + mov lr, r8 + add r12, r6, r4 // out = dst + left_ext + mov r3, r2 +1: + vld1.8 {q0, q1}, [lr]! + subs r3, r3, #32 +.if \need_left + vst1.8 {q0, q1}, [r12]! +.else + vst1.8 {q0, q1}, [r12, :128]! +.endif + bgt 1b +.if \need_right + add r3, r8, r2 // in + center_w + sub r3, r3, #1 // in + center_w - 1 + add r12, r6, r4 // dst + left_ext + vld1.8 {d0[]}, [r3] + add r12, r12, r2 // out = dst + left_ext + center_w + mov r3, r11 + vmov d1, d0 +1: + subs r3, r3, #16 + vst1.8 {q0}, [r12]! + bgt 1b +.endif + + subs r1, r1, #1 // center_h-- + add r6, r6, r7 + add r8, r8, r9 + bgt 0b +.endm + + cmp r4, #0 + beq 2f + // need_left + cmp r11, #0 + beq 3f + // need_left + need_right + v_loop 1, 1 + b 5f + +2: + // !need_left + cmp r11, #0 + beq 4f + // !need_left + need_right + v_loop 0, 1 + b 5f + +3: + // need_left + !need_right + v_loop 1, 0 + b 5f + +4: + // !need_left + !need_right + v_loop 0, 0 + +5: + cmp r10, #0 + // Storing the original dst in r0 overwrote bw, recalculate it here + add r2, r2, r4 // center_w + left_ext + add r2, r2, r11 // bw = center_w + left_ext + right_ext + + beq 3f + // need_bottom + sub r8, r6, r7 // ref = dst - stride + mov r4, r2 +1: + vld1.8 {q0, q1}, [r8, :128]! + mov r3, r10 +2: + subs r3, r3, #1 + vst1.8 {q0, q1}, [r6, :128], r7 + bgt 2b + mls r6, r7, r10, r6 // dst -= bottom_ext * stride + subs r4, r4, #32 // bw -= 32 + add r6, r6, #32 // dst += 32 + bgt 1b + +3: + cmp r5, #0 + beq 3f + // need_top + mls r6, r7, r5, r0 // dst = stored_dst - top_ext * stride +1: + vld1.8 {q0, q1}, [r0, :128]! + mov r3, r5 +2: + subs r3, r3, #1 + vst1.8 {q0, q1}, [r6, :128], r7 + bgt 2b + mls r6, r7, r5, r6 // dst -= top_ext * stride + subs r2, r2, #32 // bw -= 32 + add r6, r6, #32 // dst += 32 + bgt 1b + +3: + pop {r4-r11,pc} +endfunc diff --git a/third_party/dav1d/src/arm/32/msac.S b/third_party/dav1d/src/arm/32/msac.S new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/arm/32/msac.S @@ -0,0 +1,575 @@ +/* + * Copyright © 2019, VideoLAN and dav1d authors + * Copyright © 2020, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +#define BUF_POS 0 +#define BUF_END 4 +#define DIF 8 +#define RNG 12 +#define CNT 16 +#define ALLOW_UPDATE_CDF 20 + +const coeffs + .short 60, 56, 52, 48, 44, 40, 36, 32, 28, 24, 20, 16, 12, 8, 4, 0 + .short 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 +endconst + +const bits, align=4 + .short 0x1, 0x2, 0x4, 0x8, 0x10, 0x20, 0x40, 0x80 + .short 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000 +endconst + +.macro vld1_align_n d0, q0, q1, src, n +.if \n == 4 + vld1.16 {\d0}, [\src, :64] +.elseif \n == 8 + vld1.16 {\q0}, [\src, :128] +.else + vld1.16 {\q0, \q1}, [\src, :128] +.endif +.endm + +.macro vld1_n d0, q0, q1, src, n +.if \n == 4 + vld1.16 {\d0}, [\src] +.elseif \n == 8 + vld1.16 {\q0}, [\src] +.else + vld1.16 {\q0, \q1}, [\src] +.endif +.endm + +.macro vst1_align_n d0, q0, q1, src, n +.if \n == 4 + vst1.16 {\d0}, [\src, :64] +.elseif \n == 8 + vst1.16 {\q0}, [\src, :128] +.else + vst1.16 {\q0, \q1}, [\src, :128] +.endif +.endm + +.macro vst1_n d0, q0, q1, src, n +.if \n == 4 + vst1.16 {\d0}, [\src] +.elseif \n == 8 + vst1.16 {\q0}, [\src] +.else + vst1.16 {\q0, \q1}, [\src] +.endif +.endm + +.macro vshr_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n +.if \n == 4 + vshr.u16 \d0, \s0, \s3 +.else + vshr.u16 \d1, \s1, \s4 +.if \n == 16 + vshr.u16 \d2, \s2, \s5 +.endif +.endif +.endm + +.macro vadd_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n +.if \n == 4 + vadd.i16 \d0, \s0, \s3 +.else + vadd.i16 \d1, \s1, \s4 +.if \n == 16 + vadd.i16 \d2, \s2, \s5 +.endif +.endif +.endm + +.macro vsub_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n +.if \n == 4 + vsub.i16 \d0, \s0, \s3 +.else + vsub.i16 \d1, \s1, \s4 +.if \n == 16 + vsub.i16 \d2, \s2, \s5 +.endif +.endif +.endm + +.macro vand_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n +.if \n == 4 + vand \d0, \s0, \s3 +.else + vand \d1, \s1, \s4 +.if \n == 16 + vand \d2, \s2, \s5 +.endif +.endif +.endm + +.macro vcge_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n +.if \n == 4 + vcge.u16 \d0, \s0, \s3 +.else + vcge.u16 \d1, \s1, \s4 +.if \n == 16 + vcge.u16 \d2, \s2, \s5 +.endif +.endif +.endm + +.macro vrhadd_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n +.if \n == 4 + vrhadd.u16 \d0, \s0, \s3 +.else + vrhadd.u16 \d1, \s1, \s4 +.if \n == 16 + vrhadd.u16 \d2, \s2, \s5 +.endif +.endif +.endm + +.macro vshl_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n +.if \n == 4 + vshl.s16 \d0, \s0, \s3 +.else + vshl.s16 \d1, \s1, \s4 +.if \n == 16 + vshl.s16 \d2, \s2, \s5 +.endif +.endif +.endm + +.macro vqdmulh_n d0, d1, d2, s0, s1, s2, s3, s4, s5, n +.if \n == 4 + vqdmulh.s16 \d0, \s0, \s3 +.else + vqdmulh.s16 \d1, \s1, \s4 +.if \n == 16 + vqdmulh.s16 \d2, \s2, \s5 +.endif +.endif +.endm + +// unsigned dav1d_msac_decode_symbol_adapt4_neon(MsacContext *s, uint16_t *cdf, +// size_t n_symbols); + +function msac_decode_symbol_adapt4_neon, export=1 +.macro decode_update n + push {r4-r10,lr} + sub sp, sp, #48 + add r8, r0, #RNG + + vld1_align_n d0, q0, q1, r1, \n // cdf + vld1.16 {d16[]}, [r8, :16] // rng + movrel_local r9, coeffs, 30 + vmov.i16 d30, #0x7f00 // 0x7f00 + sub r9, r9, r2, lsl #1 + vmvn.i16 q14, #0x3f // 0xffc0 + add r8, sp, #14 + vand d22, d16, d30 // rng & 0x7f00 + vst1.16 {d16[0]}, [r8, :16] // store original u = s->rng + vand_n d4, q2, q3, d0, q0, q1, d28, q14, q14, \n // cdf & 0xffc0 +.if \n > 4 + vmov d23, d22 +.endif + + vld1_n d16, q8, q9, r9, \n // EC_MIN_PROB * (n_symbols - ret) + vqdmulh_n d20, q10, q11, d4, q2, q3, d22, q11, q11, \n // ((cdf >> EC_PROB_SHIFT) * (r - 128)) >> 1 + add r8, r0, #DIF + 2 + + vadd_n d16, q8, q9, d4, q2, q3, d16, q8, q9, \n // v = cdf + EC_MIN_PROB * (n_symbols - ret) +.if \n == 4 + vmov.i16 d17, #0 +.endif + vadd_n d16, q8, q9, d20, q10, q11, d16, q8, q9, \n // v = ((cdf >> EC_PROB_SHIFT) * r) >> 1 + EC_MIN_PROB * (n_symbols - ret) + + add r9, sp, #16 + vld1.16 {d20[]}, [r8, :16] // dif >> (EC_WIN_SIZE - 16) + movrel_local r8, bits + vst1_n q8, q8, q9, r9, \n // store v values to allow indexed access + + vmov d21, d20 + vld1_align_n q12, q12, q13, r8, \n +.if \n == 16 + vmov q11, q10 +.endif + + vcge_n q2, q2, q3, q10, q10, q11, q8, q8, q9, \n // c >= v + + vand_n q10, q10, q11, q2, q2, q3, q12, q12, q13, \n // One bit per halfword set in the mask +.if \n == 16 + vadd.i16 q10, q10, q11 +.endif + vadd.i16 d20, d20, d21 // Aggregate mask bits + ldr r4, [r0, #ALLOW_UPDATE_CDF] + vpadd.i16 d20, d20, d20 + lsl r10, r2, #1 + vpadd.i16 d20, d20, d20 + vmov.u16 r3, d20[0] + cmp r4, #0 + rbit r3, r3 + clz lr, r3 // ret + + beq L(renorm) + // update_cdf + ldrh r3, [r1, r10] // count = cdf[n_symbols] + vmov.i8 q10, #0xff +.if \n == 16 + mov r4, #-5 +.else + mvn r12, r2 + mov r4, #-4 + cmn r12, #3 // set C if n_symbols <= 2 +.endif + vrhadd_n d16, q8, q9, d20, q10, q10, d4, q2, q3, \n // i >= val ? -1 : 32768 +.if \n == 16 + sub r4, r4, r3, lsr #4 // -((count >> 4) + 5) +.else + lsr r12, r3, #4 // count >> 4 + sbc r4, r4, r12 // -((count >> 4) + (n_symbols > 2) + 4) +.endif + vsub_n d16, q8, q9, d16, q8, q9, d0, q0, q1, \n // (32768 - cdf[i]) or (-1 - cdf[i]) +.if \n == 4 + vdup.16 d20, r4 // -rate +.else + vdup.16 q10, r4 // -rate +.endif + + sub r3, r3, r3, lsr #5 // count - (count == 32) + vsub_n d0, q0, q1, d0, q0, q1, d4, q2, q3, \n // cdf + (i >= val ? 1 : 0) + vshl_n d16, q8, q9, d16, q8, q9, d20, q10, q10, \n // ({32768,-1} - cdf[i]) >> rate + add r3, r3, #1 // count + (count < 32) + vadd_n d0, q0, q1, d0, q0, q1, d16, q8, q9, \n // cdf + (32768 - cdf[i]) >> rate + vst1_align_n d0, q0, q1, r1, \n + strh r3, [r1, r10] +.endm + + decode_update 4 + +L(renorm): + add r8, sp, #16 + add r8, r8, lr, lsl #1 + ldrh r3, [r8] // v + ldrh r4, [r8, #-2] // u + ldr r6, [r0, #CNT] + ldr r7, [r0, #DIF] + sub r4, r4, r3 // rng = u - v + clz r5, r4 // clz(rng) + eor r5, r5, #16 // d = clz(rng) ^ 16 + mvn r7, r7 // ~dif + add r7, r7, r3, lsl #16 // ~dif + (v << 16) +L(renorm2): + lsl r4, r4, r5 // rng << d + subs r6, r6, r5 // cnt -= d + lsl r7, r7, r5 // (~dif + (v << 16)) << d + str r4, [r0, #RNG] + mvn r7, r7 // ~dif + bhs 9f + + // refill + ldr r3, [r0, #BUF_POS] // BUF_POS + ldr r4, [r0, #BUF_END] // BUF_END + add r5, r3, #4 + cmp r5, r4 + bgt 2f + + ldr r3, [r3] // next_bits + add r8, r6, #23 // shift_bits = cnt + 23 + add r6, r6, #16 // cnt += 16 + rev r3, r3 // next_bits = bswap(next_bits) + sub r5, r5, r8, lsr #3 // buf_pos -= shift_bits >> 3 + and r8, r8, #24 // shift_bits &= 24 + lsr r3, r3, r8 // next_bits >>= shift_bits + sub r8, r8, r6 // shift_bits -= 16 + cnt + str r5, [r0, #BUF_POS] + lsl r3, r3, r8 // next_bits <<= shift_bits + rsb r6, r8, #16 // cnt = cnt + 32 - shift_bits + eor r7, r7, r3 // dif ^= next_bits + b 9f + +2: // refill_eob + rsb r5, r6, #8 // c = 8 - cnt +3: + cmp r3, r4 + bge 4f + ldrb r8, [r3], #1 + lsl r8, r8, r5 + eor r7, r7, r8 + subs r5, r5, #8 + bge 3b + +4: // refill_eob_end + str r3, [r0, #BUF_POS] + rsb r6, r5, #8 // cnt = 8 - c + +9: + str r6, [r0, #CNT] + str r7, [r0, #DIF] + + mov r0, lr + add sp, sp, #48 + + pop {r4-r10,pc} +endfunc + +function msac_decode_symbol_adapt8_neon, export=1 + decode_update 8 + b L(renorm) +endfunc + +function msac_decode_symbol_adapt16_neon, export=1 + decode_update 16 + b L(renorm) +endfunc + +function msac_decode_hi_tok_neon, export=1 + push {r4-r10,lr} + vld1.16 {d0}, [r1, :64] // cdf + add r4, r0, #RNG + vmov.i16 d31, #0x7f00 // 0x7f00 + movrel_local r5, coeffs, 30-2*3 + vmvn.i16 d30, #0x3f // 0xffc0 + ldrh r9, [r1, #6] // count = cdf[n_symbols] + vld1.16 {d1[]}, [r4, :16] // rng + movrel_local r4, bits + vld1.16 {d29}, [r5] // EC_MIN_PROB * (n_symbols - ret) + add r5, r0, #DIF + 2 + vld1.16 {q8}, [r4, :128] + mov r2, #-24 + vand d20, d0, d30 // cdf & 0xffc0 + ldr r10, [r0, #ALLOW_UPDATE_CDF] + vld1.16 {d2[]}, [r5, :16] // dif >> (EC_WIN_SIZE - 16) + sub sp, sp, #48 + ldr r6, [r0, #CNT] + ldr r7, [r0, #DIF] + vmov d3, d2 +1: + vand d23, d1, d31 // rng & 0x7f00 + vqdmulh.s16 d18, d20, d23 // ((cdf >> EC_PROB_SHIFT) * (r - 128)) >> 1 + add r12, sp, #14 + vadd.i16 d6, d20, d29 // v = cdf + EC_MIN_PROB * (n_symbols - ret) + vadd.i16 d6, d18, d6 // v = ((cdf >> EC_PROB_SHIFT) * r) >> 1 + EC_MIN_PROB * (n_symbols - ret) + vmov.i16 d7, #0 + vst1.16 {d1[0]}, [r12, :16] // store original u = s->rng + add r12, sp, #16 + vcge.u16 q2, q1, q3 // c >= v + vst1.16 {q3}, [r12] // store v values to allow indexed access + vand q9, q2, q8 // One bit per halfword set in the mask + + vadd.i16 d18, d18, d19 // Aggregate mask bits + vpadd.i16 d18, d18, d18 + vpadd.i16 d18, d18, d18 + vmov.u16 r3, d18[0] + cmp r10, #0 + add r2, r2, #5 + rbit r3, r3 + add r8, sp, #16 + clz lr, r3 // ret + + beq 2f + // update_cdf + vmov.i8 d22, #0xff + mov r4, #-5 + vrhadd.u16 d6, d22, d4 // i >= val ? -1 : 32768 + sub r4, r4, r9, lsr #4 // -((count >> 4) + 5) + vsub.i16 d6, d6, d0 // (32768 - cdf[i]) or (-1 - cdf[i]) + vdup.16 d18, r4 // -rate + + sub r9, r9, r9, lsr #5 // count - (count == 32) + vsub.i16 d0, d0, d4 // cdf + (i >= val ? 1 : 0) + vshl.s16 d6, d6, d18 // ({32768,-1} - cdf[i]) >> rate + add r9, r9, #1 // count + (count < 32) + vadd.i16 d0, d0, d6 // cdf + (32768 - cdf[i]) >> rate + vst1.16 {d0}, [r1, :64] + vand d20, d0, d30 // cdf & 0xffc0 + strh r9, [r1, #6] + +2: + add r8, r8, lr, lsl #1 + ldrh r3, [r8] // v + ldrh r4, [r8, #-2] // u + sub r4, r4, r3 // rng = u - v + clz r5, r4 // clz(rng) + eor r5, r5, #16 // d = clz(rng) ^ 16 + mvn r7, r7 // ~dif + add r7, r7, r3, lsl #16 // ~dif + (v << 16) + lsl r4, r4, r5 // rng << d + subs r6, r6, r5 // cnt -= d + lsl r7, r7, r5 // (~dif + (v << 16)) << d + str r4, [r0, #RNG] + vdup.16 d1, r4 + mvn r7, r7 // ~dif + bhs 9f + + // refill + ldr r3, [r0, #BUF_POS] // BUF_POS + ldr r4, [r0, #BUF_END] // BUF_END + add r5, r3, #4 + cmp r5, r4 + bgt 2f + + ldr r3, [r3] // next_bits + add r8, r6, #23 // shift_bits = cnt + 23 + add r6, r6, #16 // cnt += 16 + rev r3, r3 // next_bits = bswap(next_bits) + sub r5, r5, r8, lsr #3 // buf_pos -= shift_bits >> 3 + and r8, r8, #24 // shift_bits &= 24 + lsr r3, r3, r8 // next_bits >>= shift_bits + sub r8, r8, r6 // shift_bits -= 16 + cnt + str r5, [r0, #BUF_POS] + lsl r3, r3, r8 // next_bits <<= shift_bits + rsb r6, r8, #16 // cnt = cnt + 32 - shift_bits + eor r7, r7, r3 // dif ^= next_bits + b 9f + +2: // refill_eob + rsb r5, r6, #8 // c = 40 - cnt +3: + cmp r3, r4 + bge 4f + ldrb r8, [r3], #1 + lsl r8, r8, r5 + eor r7, r7, r8 + subs r5, r5, #8 + bge 3b + +4: // refill_eob_end + str r3, [r0, #BUF_POS] + rsb r6, r5, #8 // cnt = 40 - c + +9: + lsl lr, lr, #1 + sub lr, lr, #5 + lsr r12, r7, #16 + adds r2, r2, lr // carry = tok_br < 3 || tok == 15 + vdup.16 q1, r12 + bcc 1b // loop if !carry + add r2, r2, #30 + str r6, [r0, #CNT] + add sp, sp, #48 + str r7, [r0, #DIF] + lsr r0, r2, #1 + pop {r4-r10,pc} +endfunc + +function msac_decode_bool_equi_neon, export=1 + push {r4-r10,lr} + ldr r5, [r0, #RNG] + ldr r6, [r0, #CNT] + sub sp, sp, #48 + ldr r7, [r0, #DIF] + bic r4, r5, #0xff // r &= 0xff00 + add r4, r4, #8 + mov r2, #0 + subs r8, r7, r4, lsl #15 // dif - vw + lsr r4, r4, #1 // v + sub r5, r5, r4 // r - v + itee lo + movlo r2, #1 + movhs r4, r5 // if (ret) v = r - v; + movhs r7, r8 // if (ret) dif = dif - vw; + + clz r5, r4 // clz(rng) + mvn r7, r7 // ~dif + eor r5, r5, #16 // d = clz(rng) ^ 16 + mov lr, r2 + b L(renorm2) +endfunc + +function msac_decode_bool_neon, export=1 + push {r4-r10,lr} + ldr r5, [r0, #RNG] + ldr r6, [r0, #CNT] + sub sp, sp, #48 + ldr r7, [r0, #DIF] + lsr r4, r5, #8 // r >> 8 + bic r1, r1, #0x3f // f &= ~63 + mul r4, r4, r1 + mov r2, #0 + lsr r4, r4, #7 + add r4, r4, #4 // v + subs r8, r7, r4, lsl #16 // dif - vw + sub r5, r5, r4 // r - v + itee lo + movlo r2, #1 + movhs r4, r5 // if (ret) v = r - v; + movhs r7, r8 // if (ret) dif = dif - vw; + + clz r5, r4 // clz(rng) + mvn r7, r7 // ~dif + eor r5, r5, #16 // d = clz(rng) ^ 16 + mov lr, r2 + b L(renorm2) +endfunc + +function msac_decode_bool_adapt_neon, export=1 + push {r4-r10,lr} + ldr r9, [r1] // cdf[0-1] + ldr r5, [r0, #RNG] + movw lr, #0xffc0 + ldr r6, [r0, #CNT] + sub sp, sp, #48 + ldr r7, [r0, #DIF] + lsr r4, r5, #8 // r >> 8 + and r2, r9, lr // f &= ~63 + mul r4, r4, r2 + mov r2, #0 + lsr r4, r4, #7 + add r4, r4, #4 // v + subs r8, r7, r4, lsl #16 // dif - vw + sub r5, r5, r4 // r - v + ldr r10, [r0, #ALLOW_UPDATE_CDF] + itee lo + movlo r2, #1 + movhs r4, r5 // if (ret) v = r - v; + movhs r7, r8 // if (ret) dif = dif - vw; + + cmp r10, #0 + clz r5, r4 // clz(rng) + mvn r7, r7 // ~dif + eor r5, r5, #16 // d = clz(rng) ^ 16 + mov lr, r2 + + beq L(renorm2) + + lsr r2, r9, #16 // count = cdf[1] + uxth r9, r9 // cdf[0] + + sub r3, r2, r2, lsr #5 // count - (count >= 32) + lsr r2, r2, #4 // count >> 4 + add r10, r3, #1 // count + (count < 32) + add r2, r2, #4 // rate = (count >> 4) | 4 + + sub r9, r9, lr // cdf[0] -= bit + sub r3, r9, lr, lsl #15 // {cdf[0], cdf[0] - 32769} + asr r3, r3, r2 // {cdf[0], cdf[0] - 32769} >> rate + sub r9, r9, r3 // cdf[0] + + strh r9, [r1] + strh r10, [r1, #2] + + b L(renorm2) +endfunc diff --git a/third_party/dav1d/src/arm/32/util.S b/third_party/dav1d/src/arm/32/util.S --- a/third_party/dav1d/src/arm/32/util.S +++ b/third_party/dav1d/src/arm/32/util.S @@ -82,13 +82,45 @@ vtrn.8 \r2, \r3 vtrn.8 \r4, \r5 vtrn.8 \r6, \r7 .endm +.macro transpose_8x8h r0, r1, r2, r3, r4, r5, r6, r7, d0, d1, d2, d3, d4, d5, d6, d7 + vswp \d0, \d4 + vswp \d1, \d5 + vswp \d2, \d6 + vswp \d3, \d7 + + vtrn.32 \r0, \r2 + vtrn.32 \r1, \r3 + vtrn.32 \r4, \r6 + vtrn.32 \r5, \r7 + + vtrn.16 \r0, \r1 + vtrn.16 \r2, \r3 + vtrn.16 \r4, \r5 + vtrn.16 \r6, \r7 +.endm + .macro transpose_4x8b q0, q1, r0, r1, r2, r3 vtrn.16 \q0, \q1 vtrn.8 \r0, \r1 vtrn.8 \r2, \r3 .endm +.macro transpose_4x4h q0, q1, r0, r1, r2, r3 + vtrn.32 \q0, \q1 + + vtrn.16 \r0, \r1 + vtrn.16 \r2, \r3 +.endm + +.macro transpose_4x8h r0, r1, r2, r3 + vtrn.32 \r0, \r2 + vtrn.32 \r1, \r3 + + vtrn.16 \r0, \r1 + vtrn.16 \r2, \r3 +.endm + #endif /* DAV1D_SRC_ARM_32_UTIL_S */ diff --git a/third_party/dav1d/src/arm/64/ipred.S b/third_party/dav1d/src/arm/64/ipred.S --- a/third_party/dav1d/src/arm/64/ipred.S +++ b/third_party/dav1d/src/arm/64/ipred.S @@ -26,15 +26,15 @@ */ #include "src/arm/asm.S" #include "util.S" -// void ipred_dc_128_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_dc_128_neon, export=1 +// void ipred_dc_128_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_128_8bpc_neon, export=1 clz w3, w3 adr x5, L(ipred_dc_128_tbl) sub w3, w3, #25 ldrh w3, [x5, w3, uxtw #1] movi v0.16b, #128 @@ -95,15 +95,15 @@ .hword L(ipred_dc_128_tbl) - 16b .hword L(ipred_dc_128_tbl) - 8b .hword L(ipred_dc_128_tbl) - 4b endfunc -// void ipred_v_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_v_neon, export=1 +// void ipred_v_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_v_8bpc_neon, export=1 clz w3, w3 adr x5, L(ipred_v_tbl) sub w3, w3, #25 ldrh w3, [x5, w3, uxtw #1] add x2, x2, #1 @@ -130,11 +130,11 @@ st1 {v0.8b}, [x0], x1 st1 {v0.8b}, [x6], x1 b.gt 8b ret 160: - ld1 {v0.16b}, [x2], #16 + ld1 {v0.16b}, [x2] 16: st1 {v0.16b}, [x0], x1 st1 {v0.16b}, [x6], x1 subs w4, w4, #4 st1 {v0.16b}, [x0], x1 @@ -168,15 +168,15 @@ .hword L(ipred_v_tbl) - 160b .hword L(ipred_v_tbl) - 80b .hword L(ipred_v_tbl) - 40b endfunc -// void ipred_h_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_h_neon, export=1 +// void ipred_h_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_h_8bpc_neon, export=1 clz w3, w3 adr x5, L(ipred_h_tbl) sub w3, w3, #25 ldrh w3, [x5, w3, uxtw #1] sub x2, x2, #4 @@ -249,15 +249,15 @@ .hword L(ipred_h_tbl) - 16b .hword L(ipred_h_tbl) - 8b .hword L(ipred_h_tbl) - 4b endfunc -// void ipred_dc_top_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_dc_top_neon, export=1 +// void ipred_dc_top_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_top_8bpc_neon, export=1 clz w3, w3 adr x5, L(ipred_dc_top_tbl) sub w3, w3, #25 ldrh w3, [x5, w3, uxtw #1] add x2, x2, #1 @@ -349,15 +349,15 @@ .hword L(ipred_dc_top_tbl) - 160b .hword L(ipred_dc_top_tbl) - 80b .hword L(ipred_dc_top_tbl) - 40b endfunc -// void ipred_dc_left_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_dc_left_neon, export=1 +// void ipred_dc_left_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_left_8bpc_neon, export=1 sub x2, x2, w4, uxtw clz w3, w3 clz w7, w4 adr x5, L(ipred_dc_left_tbl) sub w3, w3, #20 // 25 leading bits, minus table offset 5 @@ -470,15 +470,15 @@ .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w16) .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w8) .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w4) endfunc -// void ipred_dc_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_dc_neon, export=1 +// void ipred_dc_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_8bpc_neon, export=1 sub x2, x2, w4, uxtw add w7, w3, w4 // width + height clz w3, w3 clz w6, w4 dup v16.8h, w7 // width + height @@ -606,22 +606,22 @@ uaddlv h1, v1.16b uaddlv h2, v2.16b cmp w4, #32 add v0.4h, v0.4h, v1.4h add v0.4h, v0.4h, v2.4h - ushl v0.4h, v0.4h, v17.4h + ushl v4.4h, v0.4h, v17.4h b.eq 1f // h = 8/16/64 cmp w4, #8 mov w16, #(0x3334/2) mov w17, #(0x5556/2) csel w16, w16, w17, eq dup v16.4h, w16 - sqdmulh v0.4h, v0.4h, v16.4h + sqdmulh v4.4h, v4.4h, v16.4h 1: - dup v0.16b, v0.b[0] - dup v1.16b, v0.b[0] + dup v0.16b, v4.b[0] + dup v1.16b, v4.b[0] 2: st1 {v0.16b, v1.16b}, [x0], x1 st1 {v0.16b, v1.16b}, [x6], x1 subs w4, w4, #4 st1 {v0.16b, v1.16b}, [x0], x1 @@ -638,14 +638,10 @@ add v0.4h, v0.4h, v1.4h add v2.4h, v2.4h, v3.4h add v0.4h, v0.4h, v2.4h br x3 L(ipred_dc_w64): - mov v1.16b, v0.16b - mov v2.16b, v0.16b - mov v3.16b, v0.16b -2: add x2, x2, #1 ld1 {v1.16b, v2.16b, v3.16b, v4.16b}, [x2] add v0.4h, v0.4h, v16.4h uaddlv h1, v1.16b uaddlv h2, v2.16b @@ -654,23 +650,23 @@ add v1.4h, v1.4h, v2.4h add v3.4h, v3.4h, v4.4h cmp w4, #64 add v0.4h, v0.4h, v1.4h add v0.4h, v0.4h, v3.4h - ushl v0.4h, v0.4h, v17.4h + ushl v4.4h, v0.4h, v17.4h b.eq 1f // h = 16/32 mov w16, #(0x5556/2) movk w16, #(0x3334/2), lsl #16 lsr w16, w16, w4 dup v16.4h, w16 - sqdmulh v0.4h, v0.4h, v16.4h + sqdmulh v4.4h, v4.4h, v16.4h 1: - dup v0.16b, v0.b[0] - dup v1.16b, v0.b[0] - dup v2.16b, v0.b[0] - dup v3.16b, v0.b[0] + dup v0.16b, v4.b[0] + dup v1.16b, v4.b[0] + dup v2.16b, v4.b[0] + dup v3.16b, v4.b[0] 2: st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], x1 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x6], x1 subs w4, w4, #4 st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], x1 @@ -689,15 +685,15 @@ .hword L(ipred_dc_tbl) - L(ipred_dc_w16) .hword L(ipred_dc_tbl) - L(ipred_dc_w8) .hword L(ipred_dc_tbl) - L(ipred_dc_w4) endfunc -// void ipred_paeth_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_paeth_neon, export=1 +// void ipred_paeth_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_paeth_8bpc_neon, export=1 clz w9, w3 adr x5, L(ipred_paeth_tbl) sub w9, w9, #25 ldrh w9, [x5, w9, uxtw #1] ld1r {v4.16b}, [x2] @@ -866,15 +862,15 @@ .hword L(ipred_paeth_tbl) - 160b .hword L(ipred_paeth_tbl) - 80b .hword L(ipred_paeth_tbl) - 40b endfunc -// void ipred_smooth_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_smooth_neon, export=1 +// void ipred_smooth_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_smooth_8bpc_neon, export=1 movrel x10, X(sm_weights) add x11, x10, w4, uxtw add x10, x10, w3, uxtw clz w9, w3 adr x5, L(ipred_smooth_tbl) @@ -886,14 +882,14 @@ sub x5, x5, w9, uxtw add x6, x0, x1 lsl x1, x1, #1 br x5 40: + ld1r {v6.2s}, [x8] // top + ld1r {v7.2s}, [x10] // weights_hor sub x2, x2, #4 mov x7, #-4 - ld1r {v6.2s}, [x8] // top - ld1r {v7.2s}, [x10] // weights_hor dup v5.16b, v6.b[3] // right usubl v6.8h, v6.8b, v4.8b // top-bottom uxtl v7.8h, v7.8b // weights_hor 4: ld4r {v0.8b, v1.8b, v2.8b, v3.8b}, [x2], x7 // left @@ -924,14 +920,14 @@ st1 {v21.s}[0], [x0], x1 st1 {v21.s}[1], [x6], x1 b.gt 4b ret 80: + ld1 {v6.8b}, [x8] // top + ld1 {v7.8b}, [x10] // weights_hor sub x2, x2, #4 mov x7, #-4 - ld1 {v6.8b}, [x8] // top - ld1 {v7.8b}, [x10] // weights_hor dup v5.16b, v6.b[7] // right usubl v6.8h, v6.8b, v4.8b // top-bottom uxtl v7.8h, v7.8b // weights_hor 8: ld4r {v0.8b, v1.8b, v2.8b, v3.8b}, [x2], x7 // left @@ -1044,15 +1040,15 @@ .hword L(ipred_smooth_tbl) - 160b .hword L(ipred_smooth_tbl) - 80b .hword L(ipred_smooth_tbl) - 40b endfunc -// void ipred_smooth_v_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_smooth_v_neon, export=1 +// void ipred_smooth_v_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_smooth_v_8bpc_neon, export=1 movrel x7, X(sm_weights) add x7, x7, w4, uxtw clz w9, w3 adr x5, L(ipred_smooth_v_tbl) sub x8, x2, w4, uxtw @@ -1182,15 +1178,15 @@ .hword L(ipred_smooth_v_tbl) - 160b .hword L(ipred_smooth_v_tbl) - 80b .hword L(ipred_smooth_v_tbl) - 40b endfunc -// void ipred_smooth_h_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int a, -// const int max_width, const int max_height); -function ipred_smooth_h_neon, export=1 +// void ipred_smooth_h_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_smooth_h_8bpc_neon, export=1 movrel x8, X(sm_weights) add x8, x8, w3, uxtw clz w9, w3 adr x5, L(ipred_smooth_h_tbl) add x12, x2, w3, uxtw @@ -1325,15 +1321,15 @@ .hword L(ipred_smooth_h_tbl) - 160b .hword L(ipred_smooth_h_tbl) - 80b .hword L(ipred_smooth_h_tbl) - 40b endfunc -// void ipred_filter_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, const int filt_idx, -// const int max_width, const int max_height); -function ipred_filter_neon, export=1 +// void ipred_filter_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int filt_idx, +// const int max_width, const int max_height); +function ipred_filter_8bpc_neon, export=1 and w5, w5, #511 movrel x6, X(filter_intra_taps) lsl w5, w5, #6 add x6, x6, w5, uxtw ld1 {v16.8b, v17.8b, v18.8b, v19.8b}, [x6], #32 @@ -1462,16 +1458,18 @@ mla v6.8h, v22.8h, v0.h[7] // p6(left[1]) * filter(6) subs w3, w3, #16 sqrshrun v6.8b, v6.8h, #4 + st4 {v3.s, v4.s, v5.s, v6.s}[0], [x0], #16 + st4 {v3.s, v4.s, v5.s, v6.s}[1], [x6], #16 + b.le 8f ins v0.h[2], v2.h[7] - st4 {v3.s, v4.s, v5.s, v6.s}[0], [x0], #16 ins v0.b[0], v6.b[7] - st4 {v3.s, v4.s, v5.s, v6.s}[1], [x6], #16 ins v0.b[2], v6.b[3] - b.gt 2b + b 2b +8: subs w4, w4, #2 b.le 9f sub x8, x6, w9, uxtw add x0, x0, x1 add x6, x6, x1 @@ -1485,14 +1483,14 @@ .hword L(ipred_filter_tbl) - 160b .hword L(ipred_filter_tbl) - 80b .hword L(ipred_filter_tbl) - 40b endfunc -// void pal_pred_neon(pixel *dst, const ptrdiff_t stride, -// const uint16_t *const pal, const uint8_t *idx, -// const int w, const int h); -function pal_pred_neon, export=1 +// void pal_pred_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const uint16_t *const pal, const uint8_t *idx, +// const int w, const int h); +function pal_pred_8bpc_neon, export=1 ld1 {v0.8h}, [x2] clz w9, w4 adr x6, L(pal_pred_tbl) sub w9, w9, #25 ldrh w9, [x6, w9, uxtw #1] @@ -1576,15 +1574,15 @@ .hword L(pal_pred_tbl) - 16b .hword L(pal_pred_tbl) - 8b .hword L(pal_pred_tbl) - 4b endfunc -// void ipred_cfl_128_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, -// const int16_t *ac, const int alpha); -function ipred_cfl_128_neon, export=1 +// void ipred_cfl_128_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha); +function ipred_cfl_128_8bpc_neon, export=1 clz w9, w3 adr x7, L(ipred_cfl_128_tbl) sub w9, w9, #26 ldrh w9, [x7, w9, uxtw #1] movi v0.8h, #128 // dc @@ -1697,15 +1695,15 @@ .hword L(ipred_cfl_128_tbl) - L(ipred_cfl_splat_w16) .hword L(ipred_cfl_128_tbl) - L(ipred_cfl_splat_w8) .hword L(ipred_cfl_128_tbl) - L(ipred_cfl_splat_w4) endfunc -// void ipred_cfl_top_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, -// const int16_t *ac, const int alpha); -function ipred_cfl_top_neon, export=1 +// void ipred_cfl_top_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha); +function ipred_cfl_top_8bpc_neon, export=1 clz w9, w3 adr x7, L(ipred_cfl_top_tbl) sub w9, w9, #26 ldrh w9, [x7, w9, uxtw #1] dup v1.8h, w6 // alpha @@ -1715,46 +1713,46 @@ lsl x1, x1, #1 br x7 4: ld1r {v0.2s}, [x2] uaddlv h0, v0.8b - urshr v0.8h, v0.8h, #3 + urshr v0.4h, v0.4h, #3 dup v0.8h, v0.h[0] b L(ipred_cfl_splat_w4) 8: ld1 {v0.8b}, [x2] uaddlv h0, v0.8b - urshr v0.8h, v0.8h, #3 + urshr v0.4h, v0.4h, #3 dup v0.8h, v0.h[0] b L(ipred_cfl_splat_w8) 16: ld1 {v0.16b}, [x2] uaddlv h0, v0.16b - urshr v0.8h, v0.8h, #4 + urshr v0.4h, v0.4h, #4 dup v0.8h, v0.h[0] b L(ipred_cfl_splat_w16) 32: ld1 {v2.16b, v3.16b}, [x2] uaddlv h2, v2.16b uaddlv h3, v3.16b add v2.4h, v2.4h, v3.4h - urshr v2.8h, v2.8h, #5 + urshr v2.4h, v2.4h, #5 dup v0.8h, v2.h[0] b L(ipred_cfl_splat_w16) L(ipred_cfl_top_tbl): .hword L(ipred_cfl_top_tbl) - 32b .hword L(ipred_cfl_top_tbl) - 16b .hword L(ipred_cfl_top_tbl) - 8b .hword L(ipred_cfl_top_tbl) - 4b endfunc -// void ipred_cfl_left_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, -// const int16_t *ac, const int alpha); -function ipred_cfl_left_neon, export=1 +// void ipred_cfl_left_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha); +function ipred_cfl_left_8bpc_neon, export=1 sub x2, x2, w4, uxtw clz w9, w3 clz w8, w4 adr x10, L(ipred_cfl_splat_tbl) adr x7, L(ipred_cfl_left_tbl) @@ -1770,58 +1768,58 @@ br x7 L(ipred_cfl_left_h4): ld1r {v0.2s}, [x2] uaddlv h0, v0.8b - urshr v0.8h, v0.8h, #3 + urshr v0.4h, v0.4h, #3 dup v0.8h, v0.h[0] br x9 L(ipred_cfl_left_h8): ld1 {v0.8b}, [x2] uaddlv h0, v0.8b - urshr v0.8h, v0.8h, #3 + urshr v0.4h, v0.4h, #3 dup v0.8h, v0.h[0] br x9 L(ipred_cfl_left_h16): ld1 {v0.16b}, [x2] uaddlv h0, v0.16b - urshr v0.8h, v0.8h, #4 + urshr v0.4h, v0.4h, #4 dup v0.8h, v0.h[0] br x9 L(ipred_cfl_left_h32): ld1 {v2.16b, v3.16b}, [x2] uaddlv h2, v2.16b uaddlv h3, v3.16b add v2.4h, v2.4h, v3.4h - urshr v2.8h, v2.8h, #5 + urshr v2.4h, v2.4h, #5 dup v0.8h, v2.h[0] br x9 L(ipred_cfl_left_tbl): .hword L(ipred_cfl_left_tbl) - L(ipred_cfl_left_h32) .hword L(ipred_cfl_left_tbl) - L(ipred_cfl_left_h16) .hword L(ipred_cfl_left_tbl) - L(ipred_cfl_left_h8) .hword L(ipred_cfl_left_tbl) - L(ipred_cfl_left_h4) endfunc -// void ipred_cfl_neon(pixel *dst, const ptrdiff_t stride, -// const pixel *const topleft, -// const int width, const int height, -// const int16_t *ac, const int alpha); -function ipred_cfl_neon, export=1 +// void ipred_cfl_8bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha); +function ipred_cfl_8bpc_neon, export=1 sub x2, x2, w4, uxtw add w8, w3, w4 // width + height dup v1.8h, w6 // alpha clz w9, w3 clz w6, w4 dup v16.8h, w8 // width + height adr x7, L(ipred_cfl_tbl) rbit w8, w8 // rbit(width + height) - sub w9, w9, #22 // 22 leading bits, minus table offset 4 + sub w9, w9, #22 // 26 leading bits, minus table offset 4 sub w6, w6, #26 clz w8, w8 // ctz(width + height) ldrh w9, [x7, w9, uxtw #1] ldrh w6, [x7, w6, uxtw #1] neg w8, w8 // -ctz(width + height) @@ -1944,34 +1942,34 @@ .hword L(ipred_cfl_tbl) - L(ipred_cfl_w16) .hword L(ipred_cfl_tbl) - L(ipred_cfl_w8) .hword L(ipred_cfl_tbl) - L(ipred_cfl_w4) endfunc -// void cfl_ac_420_neon(int16_t *const ac, const pixel *const ypx, -// const ptrdiff_t stride, const int w_pad, -// const int h_pad, const int cw, const int ch); -function ipred_cfl_ac_420_neon, export=1 +// void cfl_ac_420_8bpc_neon(int16_t *const ac, const pixel *const ypx, +// const ptrdiff_t stride, const int w_pad, +// const int h_pad, const int cw, const int ch); +function ipred_cfl_ac_420_8bpc_neon, export=1 clz w8, w5 lsl w4, w4, #2 adr x7, L(ipred_cfl_ac_420_tbl) sub w8, w8, #27 ldrh w8, [x7, w8, uxtw #1] + movi v16.8h, #0 + movi v17.8h, #0 + movi v18.8h, #0 + movi v19.8h, #0 sub x7, x7, w8, uxtw sub w8, w6, w4 // height - h_pad rbit w9, w5 // rbit(width) rbit w10, w6 // rbit(height) clz w9, w9 // ctz(width) clz w10, w10 // ctz(height) add w9, w9, w10 // log2sz - movi v16.4s, #1 add x10, x1, x2 + dup v31.4s, w9 lsl x2, x2, #1 - dup v17.4s, w9 - sshl v16.4s, v16.4s, v17.4s // 1 << log2sz - neg v17.4s, v17.4s // -log2sz - ushr v16.4s, v16.4s, #1 // 1 << (log2sz - 1) - mov w9, w6 + neg v31.4s, v31.4s // -log2sz br x7 L(ipred_cfl_ac_420_w4): 1: // Copy and subsample input ld1 {v0.8b}, [x1], x2 @@ -1982,41 +1980,32 @@ uaddlp v1.8h, v1.16b add v0.8h, v0.8h, v1.8h shl v0.8h, v0.8h, #1 subs w8, w8, #2 st1 {v0.8h}, [x0], #16 + add v16.8h, v16.8h, v0.8h b.gt 1b trn2 v1.2d, v0.2d, v0.2d trn2 v0.2d, v0.2d, v0.2d L(ipred_cfl_ac_420_w4_hpad): cbz w4, 3f 2: // Vertical padding (h_pad > 0) subs w4, w4, #4 st1 {v0.8h, v1.8h}, [x0], #32 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h b.gt 2b 3: + // Aggregate the sums + add v0.8h, v16.8h, v17.8h + uaddlv s0, v0.8h // sum sub x0, x0, w6, uxtw #3 - // Sum the produced ac values - subs w6, w6, #4 - ld1 {v0.8h, v1.8h}, [x0], #32 - b.le 5f -4: - ld1 {v2.8h, v3.8h}, [x0], #32 - subs w6, w6, #4 - add v0.8h, v0.8h, v2.8h - add v1.8h, v1.8h, v3.8h - b.gt 4b -5: - add v0.8h, v0.8h, v1.8h - uaddlv s0, v0.8h // sum - sub x0, x0, w9, uxtw #3 - add v0.2s, v0.2s, v16.2s // sum += 1 << (log2sz - 1) - ushl v4.2s, v0.2s, v17.2s // sum >>= log2sz + urshl v4.2s, v0.2s, v31.2s // (sum + (1 << (log2sz - 1))) >>= log2sz dup v4.8h, v4.h[0] 6: // Subtract dc from ac ld1 {v0.8h, v1.8h}, [x0] - subs w9, w9, #4 + subs w6, w6, #4 sub v0.8h, v0.8h, v4.8h sub v1.8h, v1.8h, v4.8h st1 {v0.8h, v1.8h}, [x0], #32 b.gt 6b ret @@ -2036,10 +2025,12 @@ add v2.8h, v2.8h, v3.8h shl v0.8h, v0.8h, #1 shl v1.8h, v2.8h, #1 subs w8, w8, #2 st1 {v0.8h, v1.8h}, [x0], #32 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h b.gt 1b mov v0.16b, v1.16b b L(ipred_cfl_ac_420_w8_hpad) L(ipred_cfl_ac_420_w8_wpad): @@ -2055,51 +2046,46 @@ dup v1.4h, v0.h[3] dup v3.4h, v0.h[7] trn2 v2.2d, v0.2d, v0.2d subs w8, w8, #2 st1 {v0.4h, v1.4h, v2.4h, v3.4h}, [x0], #32 + add v16.4h, v16.4h, v0.4h + add v17.4h, v17.4h, v1.4h + add v18.4h, v18.4h, v2.4h + add v19.4h, v19.4h, v3.4h b.gt 1b trn1 v0.2d, v2.2d, v3.2d trn1 v1.2d, v2.2d, v3.2d L(ipred_cfl_ac_420_w8_hpad): cbz w4, 3f 2: // Vertical padding (h_pad > 0) subs w4, w4, #4 st1 {v0.8h, v1.8h}, [x0], #32 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h st1 {v0.8h, v1.8h}, [x0], #32 + add v18.8h, v18.8h, v0.8h + add v19.8h, v19.8h, v1.8h b.gt 2b 3: L(ipred_cfl_ac_420_w8_calc_subtract_dc): - sub x0, x0, w6, uxtw #4 - // Sum the produced ac values - subs w6, w6, #4 - ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 - b.le 5f -4: - ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #64 - subs w6, w6, #4 - add v0.8h, v0.8h, v4.8h - add v1.8h, v1.8h, v5.8h - add v2.8h, v2.8h, v6.8h - add v3.8h, v3.8h, v7.8h - b.gt 4b -5: - add v0.8h, v0.8h, v1.8h - add v2.8h, v2.8h, v3.8h + // Aggregate the sums + add v0.8h, v16.8h, v17.8h + add v2.8h, v18.8h, v19.8h uaddlp v0.4s, v0.8h uaddlp v2.4s, v2.8h add v0.4s, v0.4s, v2.4s addv s0, v0.4s // sum - sub x0, x0, w9, uxtw #4 - add v0.2s, v0.2s, v16.2s // sum += 1 << (log2sz - 1) - ushl v4.2s, v0.2s, v17.2s // sum >>= log2sz + sub x0, x0, w6, uxtw #4 + urshl v4.2s, v0.2s, v31.2s // (sum + (1 << (log2sz - 1))) >>= log2sz dup v4.8h, v4.h[0] +L(ipred_cfl_ac_420_w8_subtract_dc): 6: // Subtract dc from ac ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0] - subs w9, w9, #4 + subs w6, w6, #4 sub v0.8h, v0.8h, v4.8h sub v1.8h, v1.8h, v4.8h sub v2.8h, v2.8h, v4.8h sub v3.8h, v3.8h, v4.8h st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 @@ -2134,10 +2120,14 @@ shl v1.8h, v1.8h, #1 shl v2.8h, v4.8h, #1 shl v3.8h, v5.8h, #1 subs w8, w8, #2 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h b.gt 1b mov v0.16b, v2.16b mov v1.16b, v3.16b b L(ipred_cfl_ac_420_w16_hpad) @@ -2171,10 +2161,14 @@ dup v5.4h, v3.h[3] trn1 v1.2d, v1.2d, v4.2d trn1 v3.2d, v3.2d, v5.2d subs w8, w8, #2 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h b.gt 1b mov v0.16b, v2.16b mov v1.16b, v3.16b b L(ipred_cfl_ac_420_w16_hpad) @@ -2194,10 +2188,14 @@ shl v2.8h, v4.8h, #1 dup v1.8h, v0.h[7] dup v3.8h, v2.h[7] subs w8, w8, #2 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h b.gt 1b mov v0.16b, v2.16b mov v1.16b, v3.16b b L(ipred_cfl_ac_420_w16_hpad) @@ -2219,27 +2217,37 @@ dup v3.8h, v2.h[3] trn1 v0.2d, v0.2d, v1.2d trn1 v2.2d, v2.2d, v3.2d subs w8, w8, #2 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h b.gt 1b mov v0.16b, v2.16b mov v1.16b, v3.16b - b L(ipred_cfl_ac_420_w16_hpad) L(ipred_cfl_ac_420_w16_hpad): cbz w4, 3f 2: // Vertical padding (h_pad > 0) subs w4, w4, #4 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h b.gt 2b 3: // Double the height and reuse the w8 summing/subtracting lsl w6, w6, #1 - lsl w9, w9, #1 b L(ipred_cfl_ac_420_w8_calc_subtract_dc) L(ipred_cfl_ac_420_tbl): .hword L(ipred_cfl_ac_420_tbl) - L(ipred_cfl_ac_420_w16) .hword L(ipred_cfl_ac_420_tbl) - L(ipred_cfl_ac_420_w8) @@ -2251,34 +2259,34 @@ .hword L(ipred_cfl_ac_420_w16_tbl) - L(ipred_cfl_ac_420_w16_wpad1) .hword L(ipred_cfl_ac_420_w16_tbl) - L(ipred_cfl_ac_420_w16_wpad2) .hword L(ipred_cfl_ac_420_w16_tbl) - L(ipred_cfl_ac_420_w16_wpad3) endfunc -// void cfl_ac_422_neon(int16_t *const ac, const pixel *const ypx, -// const ptrdiff_t stride, const int w_pad, -// const int h_pad, const int cw, const int ch); -function ipred_cfl_ac_422_neon, export=1 +// void cfl_ac_422_8bpc_neon(int16_t *const ac, const pixel *const ypx, +// const ptrdiff_t stride, const int w_pad, +// const int h_pad, const int cw, const int ch); +function ipred_cfl_ac_422_8bpc_neon, export=1 clz w8, w5 lsl w4, w4, #2 adr x7, L(ipred_cfl_ac_422_tbl) sub w8, w8, #27 ldrh w8, [x7, w8, uxtw #1] + movi v16.8h, #0 + movi v17.8h, #0 + movi v18.8h, #0 + movi v19.8h, #0 sub x7, x7, w8, uxtw sub w8, w6, w4 // height - h_pad rbit w9, w5 // rbit(width) rbit w10, w6 // rbit(height) clz w9, w9 // ctz(width) clz w10, w10 // ctz(height) add w9, w9, w10 // log2sz - movi v16.4s, #1 add x10, x1, x2 + dup v31.4s, w9 lsl x2, x2, #1 - dup v17.4s, w9 - sshl v16.4s, v16.4s, v17.4s // 1 << log2sz - neg v17.4s, v17.4s // -log2sz - ushr v16.4s, v16.4s, #1 // 1 << (log2sz - 1) - mov w9, w6 + neg v31.4s, v31.4s // -log2sz br x7 L(ipred_cfl_ac_422_w4): 1: // Copy and subsample input ld1 {v0.8b}, [x1], x2 @@ -2288,10 +2296,12 @@ uaddlp v0.8h, v0.16b uaddlp v1.8h, v1.16b shl v0.8h, v0.8h, #2 shl v1.8h, v1.8h, #2 subs w8, w8, #4 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h st1 {v0.8h, v1.8h}, [x0], #32 b.gt 1b trn2 v0.2d, v1.2d, v1.2d trn2 v1.2d, v1.2d, v1.2d b L(ipred_cfl_ac_420_w4_hpad) @@ -2311,10 +2321,14 @@ shl v1.8h, v1.8h, #2 shl v2.8h, v2.8h, #2 shl v3.8h, v3.8h, #2 subs w8, w8, #4 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h b.gt 1b mov v0.16b, v3.16b mov v1.16b, v3.16b b L(ipred_cfl_ac_420_w8_hpad) @@ -2336,10 +2350,14 @@ trn1 v0.2d, v0.2d, v4.2d trn2 v3.2d, v2.2d, v7.2d trn1 v2.2d, v2.2d, v6.2d subs w8, w8, #4 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h b.gt 1b mov v0.16b, v3.16b mov v1.16b, v3.16b b L(ipred_cfl_ac_420_w8_hpad) @@ -2361,10 +2379,14 @@ shl v1.8h, v1.8h, #2 shl v2.8h, v2.8h, #2 shl v3.8h, v3.8h, #2 subs w8, w8, #2 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h b.gt 1b mov v0.16b, v2.16b mov v1.16b, v3.16b b L(ipred_cfl_ac_420_w16_hpad) @@ -2386,10 +2408,14 @@ dup v5.4h, v3.h[3] trn1 v1.2d, v1.2d, v4.2d trn1 v3.2d, v3.2d, v5.2d subs w8, w8, #2 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h b.gt 1b mov v0.16b, v2.16b mov v1.16b, v3.16b b L(ipred_cfl_ac_420_w16_hpad) @@ -2403,10 +2429,14 @@ shl v2.8h, v2.8h, #2 dup v1.8h, v0.h[7] dup v3.8h, v2.h[7] subs w8, w8, #2 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h b.gt 1b mov v0.16b, v2.16b mov v1.16b, v3.16b b L(ipred_cfl_ac_420_w16_hpad) @@ -2422,10 +2452,14 @@ dup v3.8h, v2.h[3] trn1 v0.2d, v0.2d, v1.2d trn1 v2.2d, v2.2d, v3.2d subs w8, w8, #2 st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h b.gt 1b mov v0.16b, v2.16b mov v1.16b, v3.16b b L(ipred_cfl_ac_420_w16_hpad) @@ -2439,5 +2473,292 @@ .hword L(ipred_cfl_ac_422_w16_tbl) - L(ipred_cfl_ac_422_w16_wpad0) .hword L(ipred_cfl_ac_422_w16_tbl) - L(ipred_cfl_ac_422_w16_wpad1) .hword L(ipred_cfl_ac_422_w16_tbl) - L(ipred_cfl_ac_422_w16_wpad2) .hword L(ipred_cfl_ac_422_w16_tbl) - L(ipred_cfl_ac_422_w16_wpad3) endfunc + +// void cfl_ac_444_8bpc_neon(int16_t *const ac, const pixel *const ypx, +// const ptrdiff_t stride, const int w_pad, +// const int h_pad, const int cw, const int ch); +function ipred_cfl_ac_444_8bpc_neon, export=1 + clz w8, w5 + lsl w4, w4, #2 + adr x7, L(ipred_cfl_ac_444_tbl) + sub w8, w8, #26 + ldrh w8, [x7, w8, uxtw #1] + movi v16.8h, #0 + movi v17.8h, #0 + movi v18.8h, #0 + movi v19.8h, #0 + sub x7, x7, w8, uxtw + sub w8, w6, w4 // height - h_pad + rbit w9, w5 // rbit(width) + rbit w10, w6 // rbit(height) + clz w9, w9 // ctz(width) + clz w10, w10 // ctz(height) + add w9, w9, w10 // log2sz + add x10, x1, x2 + dup v31.4s, w9 + lsl x2, x2, #1 + neg v31.4s, v31.4s // -log2sz + br x7 + +L(ipred_cfl_ac_444_w4): +1: // Copy and expand input + ld1 {v0.s}[0], [x1], x2 + ld1 {v0.s}[1], [x10], x2 + ld1 {v1.s}[0], [x1], x2 + ld1 {v1.s}[1], [x10], x2 + ushll v0.8h, v0.8b, #3 + ushll v1.8h, v1.8b, #3 + subs w8, w8, #4 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + st1 {v0.8h, v1.8h}, [x0], #32 + b.gt 1b + trn2 v0.2d, v1.2d, v1.2d + trn2 v1.2d, v1.2d, v1.2d + b L(ipred_cfl_ac_420_w4_hpad) + +L(ipred_cfl_ac_444_w8): +1: // Copy and expand input + ld1 {v0.8b}, [x1], x2 + ld1 {v1.8b}, [x10], x2 + ld1 {v2.8b}, [x1], x2 + ushll v0.8h, v0.8b, #3 + ld1 {v3.8b}, [x10], x2 + ushll v1.8h, v1.8b, #3 + ushll v2.8h, v2.8b, #3 + ushll v3.8h, v3.8b, #3 + subs w8, w8, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h + b.gt 1b + mov v0.16b, v3.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w8_hpad) + +L(ipred_cfl_ac_444_w16): + cbnz w3, L(ipred_cfl_ac_444_w16_wpad) +1: // Copy and expand input, without padding + ld1 {v0.16b}, [x1], x2 + ld1 {v2.16b}, [x10], x2 + ld1 {v4.16b}, [x1], x2 + ushll2 v1.8h, v0.16b, #3 + ushll v0.8h, v0.8b, #3 + ld1 {v6.16b}, [x10], x2 + ushll2 v3.8h, v2.16b, #3 + ushll v2.8h, v2.8b, #3 + ushll2 v5.8h, v4.16b, #3 + ushll v4.8h, v4.8b, #3 + ushll2 v7.8h, v6.16b, #3 + ushll v6.8h, v6.8b, #3 + subs w8, w8, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #64 + add v16.8h, v16.8h, v4.8h + add v17.8h, v17.8h, v5.8h + add v18.8h, v18.8h, v6.8h + add v19.8h, v19.8h, v7.8h + b.gt 1b + mov v0.16b, v6.16b + mov v1.16b, v7.16b + mov v2.16b, v6.16b + mov v3.16b, v7.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_444_w16_wpad): +1: // Copy and expand input, padding 8 + ld1 {v0.8b}, [x1], x2 + ld1 {v2.8b}, [x10], x2 + ld1 {v4.8b}, [x1], x2 + ld1 {v6.8b}, [x10], x2 + ushll v0.8h, v0.8b, #3 + ushll v2.8h, v2.8b, #3 + ushll v4.8h, v4.8b, #3 + ushll v6.8h, v6.8b, #3 + dup v1.8h, v0.h[7] + dup v3.8h, v2.h[7] + dup v5.8h, v4.h[7] + dup v7.8h, v6.h[7] + subs w8, w8, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #64 + add v16.8h, v16.8h, v4.8h + add v17.8h, v17.8h, v5.8h + add v18.8h, v18.8h, v6.8h + add v19.8h, v19.8h, v7.8h + b.gt 1b + mov v0.16b, v6.16b + mov v1.16b, v7.16b + mov v2.16b, v6.16b + mov v3.16b, v7.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_444_w32): + adr x7, L(ipred_cfl_ac_444_w32_tbl) + ldrh w3, [x7, w3, uxtw] // (w3>>1) << 1 + sub x7, x7, w3, uxtw + br x7 + +L(ipred_cfl_ac_444_w32_wpad0): +1: // Copy and expand input, without padding + ld1 {v2.16b, v3.16b}, [x1], x2 + ld1 {v6.16b, v7.16b}, [x10], x2 + ushll v0.8h, v2.8b, #3 + ushll2 v1.8h, v2.16b, #3 + ushll v2.8h, v3.8b, #3 + ushll2 v3.8h, v3.16b, #3 + ushll v4.8h, v6.8b, #3 + ushll2 v5.8h, v6.16b, #3 + ushll v6.8h, v7.8b, #3 + ushll2 v7.8h, v7.16b, #3 + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #64 + add v16.8h, v16.8h, v4.8h + add v17.8h, v17.8h, v5.8h + add v18.8h, v18.8h, v6.8h + add v19.8h, v19.8h, v7.8h + b.gt 1b + b L(ipred_cfl_ac_444_w32_hpad) + +L(ipred_cfl_ac_444_w32_wpad2): +1: // Copy and expand input, padding 8 + ldr d2, [x1, #16] + ld1 {v1.16b}, [x1], x2 + ldr d6, [x10, #16] + ld1 {v5.16b}, [x10], x2 + ushll v2.8h, v2.8b, #3 + ushll v0.8h, v1.8b, #3 + ushll2 v1.8h, v1.16b, #3 + ushll v6.8h, v6.8b, #3 + ushll v4.8h, v5.8b, #3 + ushll2 v5.8h, v5.16b, #3 + dup v3.8h, v2.h[7] + dup v7.8h, v6.h[7] + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #64 + add v16.8h, v16.8h, v4.8h + add v17.8h, v17.8h, v5.8h + add v18.8h, v18.8h, v6.8h + add v19.8h, v19.8h, v7.8h + b.gt 1b + b L(ipred_cfl_ac_444_w32_hpad) + +L(ipred_cfl_ac_444_w32_wpad4): +1: // Copy and expand input, padding 16 + ld1 {v1.16b}, [x1], x2 + ld1 {v5.16b}, [x10], x2 + ushll v0.8h, v1.8b, #3 + ushll2 v1.8h, v1.16b, #3 + ushll v4.8h, v5.8b, #3 + ushll2 v5.8h, v5.16b, #3 + dup v2.8h, v1.h[7] + dup v3.8h, v1.h[7] + dup v6.8h, v5.h[7] + dup v7.8h, v5.h[7] + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #64 + add v16.8h, v16.8h, v4.8h + add v17.8h, v17.8h, v5.8h + add v18.8h, v18.8h, v6.8h + add v19.8h, v19.8h, v7.8h + b.gt 1b + b L(ipred_cfl_ac_444_w32_hpad) + +L(ipred_cfl_ac_444_w32_wpad6): +1: // Copy and expand input, padding 24 + ld1 {v0.8b}, [x1], x2 + ld1 {v4.8b}, [x10], x2 + ushll v0.8h, v0.8b, #3 + ushll v4.8h, v4.8b, #3 + dup v1.8h, v0.h[7] + dup v2.8h, v0.h[7] + dup v3.8h, v0.h[7] + dup v5.8h, v4.h[7] + dup v6.8h, v4.h[7] + dup v7.8h, v4.h[7] + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + add v16.8h, v16.8h, v0.8h + add v17.8h, v17.8h, v1.8h + add v18.8h, v18.8h, v2.8h + add v19.8h, v19.8h, v3.8h + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #64 + add v16.8h, v16.8h, v4.8h + add v17.8h, v17.8h, v5.8h + add v18.8h, v18.8h, v6.8h + add v19.8h, v19.8h, v7.8h + b.gt 1b + +L(ipred_cfl_ac_444_w32_hpad): + cbz w4, 3f +2: // Vertical padding (h_pad > 0) + subs w4, w4, #2 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #64 + add v16.8h, v16.8h, v4.8h + add v17.8h, v17.8h, v5.8h + add v18.8h, v18.8h, v6.8h + add v19.8h, v19.8h, v7.8h + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #64 + add v16.8h, v16.8h, v4.8h + add v17.8h, v17.8h, v5.8h + add v18.8h, v18.8h, v6.8h + add v19.8h, v19.8h, v7.8h + b.gt 2b +3: + + // Quadruple the height and reuse the w8 subtracting + lsl w6, w6, #2 + // Aggregate the sums, with wider intermediates earlier than in + // ipred_cfl_ac_420_w8_calc_subtract_dc. + uaddlp v0.4s, v16.8h + uaddlp v1.4s, v17.8h + uaddlp v2.4s, v18.8h + uaddlp v3.4s, v19.8h + add v0.4s, v0.4s, v1.4s + add v2.4s, v2.4s, v3.4s + add v0.4s, v0.4s, v2.4s + addv s0, v0.4s // sum + sub x0, x0, w6, uxtw #4 + urshl v4.2s, v0.2s, v31.2s // (sum + (1 << (log2sz - 1))) >>= log2sz + dup v4.8h, v4.h[0] + b L(ipred_cfl_ac_420_w8_subtract_dc) + +L(ipred_cfl_ac_444_tbl): + .hword L(ipred_cfl_ac_444_tbl) - L(ipred_cfl_ac_444_w32) + .hword L(ipred_cfl_ac_444_tbl) - L(ipred_cfl_ac_444_w16) + .hword L(ipred_cfl_ac_444_tbl) - L(ipred_cfl_ac_444_w8) + .hword L(ipred_cfl_ac_444_tbl) - L(ipred_cfl_ac_444_w4) + +L(ipred_cfl_ac_444_w32_tbl): + .hword L(ipred_cfl_ac_444_w32_tbl) - L(ipred_cfl_ac_444_w32_wpad0) + .hword L(ipred_cfl_ac_444_w32_tbl) - L(ipred_cfl_ac_444_w32_wpad2) + .hword L(ipred_cfl_ac_444_w32_tbl) - L(ipred_cfl_ac_444_w32_wpad4) + .hword L(ipred_cfl_ac_444_w32_tbl) - L(ipred_cfl_ac_444_w32_wpad6) +endfunc diff --git a/third_party/dav1d/src/arm/64/ipred16.S b/third_party/dav1d/src/arm/64/ipred16.S new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/arm/64/ipred16.S @@ -0,0 +1,3076 @@ +/* + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2019, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "src/arm/asm.S" +#include "util.S" + +// void ipred_dc_128_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height, +// const int bitdepth_max); +function ipred_dc_128_16bpc_neon, export=1 + ldr w8, [sp] + clz w3, w3 + adr x5, L(ipred_dc_128_tbl) + sub w3, w3, #25 + ldrh w3, [x5, w3, uxtw #1] + dup v0.8h, w8 + sub x5, x5, w3, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + urshr v0.8h, v0.8h, #1 + br x5 +4: + st1 {v0.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + b.gt 4b + ret +8: + st1 {v0.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + b.gt 8b + ret +160: + mov v1.16b, v0.16b +16: + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v0.8h, v1.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v0.8h, v1.8h}, [x6], x1 + b.gt 16b + ret +320: + mov v1.16b, v0.16b + mov v2.16b, v0.16b + mov v3.16b, v0.16b +32: + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + b.gt 32b + ret +640: + mov v1.16b, v0.16b + mov v2.16b, v0.16b + mov v3.16b, v0.16b + sub x1, x1, #64 +64: + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + b.gt 64b + ret + +L(ipred_dc_128_tbl): + .hword L(ipred_dc_128_tbl) - 640b + .hword L(ipred_dc_128_tbl) - 320b + .hword L(ipred_dc_128_tbl) - 160b + .hword L(ipred_dc_128_tbl) - 8b + .hword L(ipred_dc_128_tbl) - 4b +endfunc + +// void ipred_v_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_v_16bpc_neon, export=1 + clz w3, w3 + adr x5, L(ipred_v_tbl) + sub w3, w3, #25 + ldrh w3, [x5, w3, uxtw #1] + add x2, x2, #2 + sub x5, x5, w3, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + br x5 +40: + ld1 {v0.4h}, [x2] +4: + st1 {v0.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + b.gt 4b + ret +80: + ld1 {v0.8h}, [x2] +8: + st1 {v0.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + b.gt 8b + ret +160: + ld1 {v0.8h, v1.8h}, [x2] +16: + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v0.8h, v1.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v0.8h, v1.8h}, [x6], x1 + b.gt 16b + ret +320: + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x2] +32: + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + b.gt 32b + ret +640: + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], #64 + sub x1, x1, #64 + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2] +64: + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], #64 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], x1 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], #64 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], x1 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], x1 + b.gt 64b + ret + +L(ipred_v_tbl): + .hword L(ipred_v_tbl) - 640b + .hword L(ipred_v_tbl) - 320b + .hword L(ipred_v_tbl) - 160b + .hword L(ipred_v_tbl) - 80b + .hword L(ipred_v_tbl) - 40b +endfunc + +// void ipred_h_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_h_16bpc_neon, export=1 + clz w3, w3 + adr x5, L(ipred_h_tbl) + sub w3, w3, #25 + ldrh w3, [x5, w3, uxtw #1] + sub x2, x2, #8 + sub x5, x5, w3, uxtw + mov x7, #-8 + add x6, x0, x1 + lsl x1, x1, #1 + br x5 +4: + ld4r {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], x7 + st1 {v3.4h}, [x0], x1 + st1 {v2.4h}, [x6], x1 + subs w4, w4, #4 + st1 {v1.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + b.gt 4b + ret +8: + ld4r {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], x7 + st1 {v3.8h}, [x0], x1 + st1 {v2.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v1.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + b.gt 8b + ret +16: + ld4r {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], x7 + str q3, [x0, #16] + str q2, [x6, #16] + st1 {v3.8h}, [x0], x1 + st1 {v2.8h}, [x6], x1 + subs w4, w4, #4 + str q1, [x0, #16] + str q0, [x6, #16] + st1 {v1.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + b.gt 16b + ret +32: + ld4r {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], x7 + str q3, [x0, #16] + str q2, [x6, #16] + stp q3, q3, [x0, #32] + stp q2, q2, [x6, #32] + st1 {v3.8h}, [x0], x1 + st1 {v2.8h}, [x6], x1 + subs w4, w4, #4 + str q1, [x0, #16] + str q0, [x6, #16] + stp q1, q1, [x0, #32] + stp q0, q0, [x6, #32] + st1 {v1.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + b.gt 32b + ret +64: + ld4r {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], x7 + str q3, [x0, #16] + str q2, [x6, #16] + stp q3, q3, [x0, #32] + stp q2, q2, [x6, #32] + stp q3, q3, [x0, #64] + stp q2, q2, [x6, #64] + stp q3, q3, [x0, #96] + stp q2, q2, [x6, #96] + st1 {v3.8h}, [x0], x1 + st1 {v2.8h}, [x6], x1 + subs w4, w4, #4 + str q1, [x0, #16] + str q0, [x6, #16] + stp q1, q1, [x0, #32] + stp q0, q0, [x6, #32] + stp q1, q1, [x0, #64] + stp q0, q0, [x6, #64] + stp q1, q1, [x0, #96] + stp q0, q0, [x6, #96] + st1 {v1.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + b.gt 64b + ret + +L(ipred_h_tbl): + .hword L(ipred_h_tbl) - 64b + .hword L(ipred_h_tbl) - 32b + .hword L(ipred_h_tbl) - 16b + .hword L(ipred_h_tbl) - 8b + .hword L(ipred_h_tbl) - 4b +endfunc + +// void ipred_dc_top_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_top_16bpc_neon, export=1 + clz w3, w3 + adr x5, L(ipred_dc_top_tbl) + sub w3, w3, #25 + ldrh w3, [x5, w3, uxtw #1] + add x2, x2, #2 + sub x5, x5, w3, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + br x5 +40: + ld1 {v0.4h}, [x2] + addv h0, v0.4h + urshr v0.4h, v0.4h, #2 + dup v0.4h, v0.h[0] +4: + st1 {v0.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + b.gt 4b + ret +80: + ld1 {v0.8h}, [x2] + addv h0, v0.8h + urshr v0.4h, v0.4h, #3 + dup v0.8h, v0.h[0] +8: + st1 {v0.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + b.gt 8b + ret +160: + ld1 {v0.8h, v1.8h}, [x2] + addp v0.8h, v0.8h, v1.8h + addv h0, v0.8h + urshr v2.4h, v0.4h, #4 + dup v0.8h, v2.h[0] + dup v1.8h, v2.h[0] +16: + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v0.8h, v1.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v0.8h, v1.8h}, [x6], x1 + b.gt 16b + ret +320: + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x2] + addp v0.8h, v0.8h, v1.8h + addp v2.8h, v2.8h, v3.8h + addp v0.8h, v0.8h, v2.8h + uaddlv s0, v0.8h + rshrn v4.4h, v0.4s, #5 + dup v0.8h, v4.h[0] + dup v1.8h, v4.h[0] + dup v2.8h, v4.h[0] + dup v3.8h, v4.h[0] +32: + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + b.gt 32b + ret +640: + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], #64 + addp v0.8h, v0.8h, v1.8h + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2] + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v6.8h, v6.8h, v7.8h + addp v0.8h, v0.8h, v2.8h + addp v4.8h, v4.8h, v6.8h + addp v0.8h, v0.8h, v4.8h + uaddlv s0, v0.8h + rshrn v4.4h, v0.4s, #6 + sub x1, x1, #64 + dup v0.8h, v4.h[0] + dup v1.8h, v4.h[0] + dup v2.8h, v4.h[0] + dup v3.8h, v4.h[0] +64: + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + b.gt 64b + ret + +L(ipred_dc_top_tbl): + .hword L(ipred_dc_top_tbl) - 640b + .hword L(ipred_dc_top_tbl) - 320b + .hword L(ipred_dc_top_tbl) - 160b + .hword L(ipred_dc_top_tbl) - 80b + .hword L(ipred_dc_top_tbl) - 40b +endfunc + +// void ipred_dc_left_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_left_16bpc_neon, export=1 + sub x2, x2, w4, uxtw #1 + clz w3, w3 + clz w7, w4 + adr x5, L(ipred_dc_left_tbl) + sub w3, w3, #20 // 25 leading bits, minus table offset 5 + sub w7, w7, #25 + ldrh w3, [x5, w3, uxtw #1] + ldrh w7, [x5, w7, uxtw #1] + sub x3, x5, w3, uxtw + sub x5, x5, w7, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + br x5 + +L(ipred_dc_left_h4): + ld1 {v0.4h}, [x2] + addv h0, v0.4h + urshr v0.4h, v0.4h, #2 + dup v0.8h, v0.h[0] + br x3 +L(ipred_dc_left_w4): + st1 {v0.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + b.gt L(ipred_dc_left_w4) + ret + +L(ipred_dc_left_h8): + ld1 {v0.8h}, [x2] + addv h0, v0.8h + urshr v0.4h, v0.4h, #3 + dup v0.8h, v0.h[0] + br x3 +L(ipred_dc_left_w8): + st1 {v0.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + b.gt L(ipred_dc_left_w8) + ret + +L(ipred_dc_left_h16): + ld1 {v0.8h, v1.8h}, [x2] + addp v0.8h, v0.8h, v1.8h + addv h0, v0.8h + urshr v2.4h, v0.4h, #4 + dup v0.8h, v2.h[0] + dup v1.8h, v2.h[0] + br x3 +L(ipred_dc_left_w16): + mov v1.16b, v0.16b +1: + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v0.8h, v1.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v0.8h, v1.8h}, [x6], x1 + b.gt 1b + ret + +L(ipred_dc_left_h32): + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x2] + addp v0.8h, v0.8h, v1.8h + addp v2.8h, v2.8h, v3.8h + addp v0.8h, v0.8h, v2.8h + uaddlp v0.4s, v0.8h + addv s0, v0.4s + rshrn v4.4h, v0.4s, #5 + dup v0.8h, v4.h[0] + br x3 +L(ipred_dc_left_w32): + mov v1.16b, v0.16b + mov v2.16b, v0.16b + mov v3.16b, v0.16b +1: + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + b.gt 1b + ret + +L(ipred_dc_left_h64): + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], #64 + addp v0.8h, v0.8h, v1.8h + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2] + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v6.8h, v6.8h, v7.8h + addp v0.8h, v0.8h, v2.8h + addp v4.8h, v4.8h, v6.8h + addp v0.8h, v0.8h, v4.8h + uaddlv s0, v0.8h + rshrn v4.4h, v0.4s, #6 + dup v0.8h, v4.h[0] + br x3 +L(ipred_dc_left_w64): + mov v1.16b, v0.16b + mov v2.16b, v0.16b + mov v3.16b, v0.16b + sub x1, x1, #64 +1: + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + b.gt 1b + ret + +L(ipred_dc_left_tbl): + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_h64) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_h32) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_h16) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_h8) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_h4) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w64) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w32) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w16) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w8) + .hword L(ipred_dc_left_tbl) - L(ipred_dc_left_w4) +endfunc + +// void ipred_dc_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_dc_16bpc_neon, export=1 + sub x2, x2, w4, uxtw #1 + add w7, w3, w4 // width + height + clz w3, w3 + clz w6, w4 + dup v16.4s, w7 // width + height + adr x5, L(ipred_dc_tbl) + rbit w7, w7 // rbit(width + height) + sub w3, w3, #20 // 25 leading bits, minus table offset 5 + sub w6, w6, #25 + clz w7, w7 // ctz(width + height) + ldrh w3, [x5, w3, uxtw #1] + ldrh w6, [x5, w6, uxtw #1] + neg w7, w7 // -ctz(width + height) + sub x3, x5, w3, uxtw + sub x5, x5, w6, uxtw + ushr v16.4s, v16.4s, #1 // (width + height) >> 1 + dup v17.4s, w7 // -ctz(width + height) + add x6, x0, x1 + lsl x1, x1, #1 + br x5 + +L(ipred_dc_h4): + ld1 {v0.4h}, [x2], #8 + uaddlv s0, v0.4h + br x3 +L(ipred_dc_w4): + add x2, x2, #2 + ld1 {v1.4h}, [x2] + add v0.2s, v0.2s, v16.2s + uaddlv s1, v1.4h + cmp w4, #4 + add v0.2s, v0.2s, v1.2s + ushl v0.2s, v0.2s, v17.2s + b.eq 1f + // h = 8/16 + cmp w4, #16 + mov w16, #0x6667 + mov w17, #0xAAAB + csel w16, w16, w17, eq + dup v16.2s, w16 + mul v0.2s, v0.2s, v16.2s + ushr v0.2s, v0.2s, #17 +1: + dup v0.4h, v0.h[0] +2: + st1 {v0.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.4h}, [x0], x1 + st1 {v0.4h}, [x6], x1 + b.gt 2b + ret + +L(ipred_dc_h8): + ld1 {v0.8h}, [x2], #16 + uaddlv s0, v0.8h + br x3 +L(ipred_dc_w8): + add x2, x2, #2 + ld1 {v1.8h}, [x2] + add v0.2s, v0.2s, v16.2s + uaddlv s1, v1.8h + cmp w4, #8 + add v0.2s, v0.2s, v1.2s + ushl v0.2s, v0.2s, v17.2s + b.eq 1f + // h = 4/16/32 + cmp w4, #32 + mov w16, #0x6667 + mov w17, #0xAAAB + csel w16, w16, w17, eq + dup v16.2s, w16 + mul v0.2s, v0.2s, v16.2s + ushr v0.2s, v0.2s, #17 +1: + dup v0.8h, v0.h[0] +2: + st1 {v0.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h}, [x0], x1 + st1 {v0.8h}, [x6], x1 + b.gt 2b + ret + +L(ipred_dc_h16): + ld1 {v0.8h, v1.8h}, [x2], #32 + addp v0.8h, v0.8h, v1.8h + uaddlv s0, v0.8h + br x3 +L(ipred_dc_w16): + add x2, x2, #2 + ld1 {v1.8h, v2.8h}, [x2] + add v0.2s, v0.2s, v16.2s + addp v1.8h, v1.8h, v2.8h + uaddlv s1, v1.8h + cmp w4, #16 + add v0.2s, v0.2s, v1.2s + ushl v4.2s, v0.2s, v17.2s + b.eq 1f + // h = 4/8/32/64 + tst w4, #(32+16+8) // 16 added to make a consecutive bitmask + mov w16, #0x6667 + mov w17, #0xAAAB + csel w16, w16, w17, eq + dup v16.2s, w16 + mul v4.2s, v4.2s, v16.2s + ushr v4.2s, v4.2s, #17 +1: + dup v0.8h, v4.h[0] + dup v1.8h, v4.h[0] +2: + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v0.8h, v1.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h}, [x0], x1 + st1 {v0.8h, v1.8h}, [x6], x1 + b.gt 2b + ret + +L(ipred_dc_h32): + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], #64 + addp v0.8h, v0.8h, v1.8h + addp v2.8h, v2.8h, v3.8h + addp v0.8h, v0.8h, v2.8h + uaddlv s0, v0.8h + br x3 +L(ipred_dc_w32): + add x2, x2, #2 + ld1 {v1.8h, v2.8h, v3.8h, v4.8h}, [x2] + add v0.2s, v0.2s, v16.2s + addp v1.8h, v1.8h, v2.8h + addp v3.8h, v3.8h, v4.8h + addp v1.8h, v1.8h, v3.8h + uaddlv s1, v1.8h + cmp w4, #32 + add v0.2s, v0.2s, v1.2s + ushl v4.2s, v0.2s, v17.2s + b.eq 1f + // h = 8/16/64 + cmp w4, #8 + mov w16, #0x6667 + mov w17, #0xAAAB + csel w16, w16, w17, eq + dup v16.2s, w16 + mul v4.2s, v4.2s, v16.2s + ushr v4.2s, v4.2s, #17 +1: + dup v0.8h, v4.h[0] + dup v1.8h, v4.h[0] + dup v2.8h, v4.h[0] + dup v3.8h, v4.h[0] +2: + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + b.gt 2b + ret + +L(ipred_dc_h64): + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], #64 + addp v0.8h, v0.8h, v1.8h + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2], #64 + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v6.8h, v6.8h, v7.8h + addp v0.8h, v0.8h, v2.8h + addp v4.8h, v4.8h, v6.8h + addp v0.8h, v0.8h, v4.8h + uaddlv s0, v0.8h + br x3 +L(ipred_dc_w64): + add x2, x2, #2 + ld1 {v1.8h, v2.8h, v3.8h, v4.8h}, [x2], #64 + add v0.2s, v0.2s, v16.2s + addp v1.8h, v1.8h, v2.8h + ld1 {v20.8h, v21.8h, v22.8h, v23.8h}, [x2] + addp v3.8h, v3.8h, v4.8h + addp v20.8h, v20.8h, v21.8h + addp v22.8h, v22.8h, v23.8h + addp v1.8h, v1.8h, v3.8h + addp v20.8h, v20.8h, v22.8h + addp v1.8h, v1.8h, v20.8h + uaddlv s1, v1.8h + cmp w4, #64 + add v0.2s, v0.2s, v1.2s + ushl v4.2s, v0.2s, v17.2s + b.eq 1f + // h = 16/32 + cmp w4, #16 + mov w16, #0x6667 + mov w17, #0xAAAB + csel w16, w16, w17, eq + dup v16.2s, w16 + mul v4.2s, v4.2s, v16.2s + ushr v4.2s, v4.2s, #17 +1: + sub x1, x1, #64 + dup v0.8h, v4.h[0] + dup v1.8h, v4.h[0] + dup v2.8h, v4.h[0] + dup v3.8h, v4.h[0] +2: + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], #64 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x1 + b.gt 2b + ret + +L(ipred_dc_tbl): + .hword L(ipred_dc_tbl) - L(ipred_dc_h64) + .hword L(ipred_dc_tbl) - L(ipred_dc_h32) + .hword L(ipred_dc_tbl) - L(ipred_dc_h16) + .hword L(ipred_dc_tbl) - L(ipred_dc_h8) + .hword L(ipred_dc_tbl) - L(ipred_dc_h4) + .hword L(ipred_dc_tbl) - L(ipred_dc_w64) + .hword L(ipred_dc_tbl) - L(ipred_dc_w32) + .hword L(ipred_dc_tbl) - L(ipred_dc_w16) + .hword L(ipred_dc_tbl) - L(ipred_dc_w8) + .hword L(ipred_dc_tbl) - L(ipred_dc_w4) +endfunc + +// void ipred_paeth_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_paeth_16bpc_neon, export=1 + clz w9, w3 + adr x5, L(ipred_paeth_tbl) + sub w9, w9, #25 + ldrh w9, [x5, w9, uxtw #1] + ld1r {v4.8h}, [x2] + add x8, x2, #2 + sub x2, x2, #8 + sub x5, x5, w9, uxtw + mov x7, #-8 + add x6, x0, x1 + lsl x1, x1, #1 + br x5 +40: + ld1r {v5.2d}, [x8] + sub v6.8h, v5.8h, v4.8h // top - topleft +4: + ld4r {v0.4h, v1.4h, v2.4h, v3.4h}, [x2], x7 + zip1 v0.2d, v0.2d, v1.2d + zip1 v2.2d, v2.2d, v3.2d + add v16.8h, v6.8h, v0.8h // base + add v17.8h, v6.8h, v2.8h + sabd v20.8h, v5.8h, v16.8h // tdiff + sabd v21.8h, v5.8h, v17.8h + sabd v22.8h, v4.8h, v16.8h // tldiff + sabd v23.8h, v4.8h, v17.8h + sabd v16.8h, v0.8h, v16.8h // ldiff + sabd v17.8h, v2.8h, v17.8h + umin v18.8h, v20.8h, v22.8h // min(tdiff, tldiff) + umin v19.8h, v21.8h, v23.8h + cmge v20.8h, v22.8h, v20.8h // tldiff >= tdiff + cmge v21.8h, v23.8h, v21.8h + cmge v16.8h, v18.8h, v16.8h // min(tdiff, tldiff) >= ldiff + cmge v17.8h, v19.8h, v17.8h + bsl v21.16b, v5.16b, v4.16b // tdiff <= tldiff ? top : topleft + bsl v20.16b, v5.16b, v4.16b + bit v21.16b, v2.16b, v17.16b // ldiff <= min ? left : ... + bit v20.16b, v0.16b, v16.16b + st1 {v21.d}[1], [x0], x1 + st1 {v21.d}[0], [x6], x1 + subs w4, w4, #4 + st1 {v20.d}[1], [x0], x1 + st1 {v20.d}[0], [x6], x1 + b.gt 4b + ret +80: +160: +320: +640: + ld1 {v5.8h}, [x8], #16 + mov w9, w3 + // Set up pointers for four rows in parallel; x0, x6, x5, x10 + add x5, x0, x1 + add x10, x6, x1 + lsl x1, x1, #1 + sub x1, x1, w3, uxtw #1 +1: + ld4r {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], x7 +2: + sub v6.8h, v5.8h, v4.8h // top - topleft + add v16.8h, v6.8h, v0.8h // base + add v17.8h, v6.8h, v1.8h + add v18.8h, v6.8h, v2.8h + add v19.8h, v6.8h, v3.8h + sabd v20.8h, v5.8h, v16.8h // tdiff + sabd v21.8h, v5.8h, v17.8h + sabd v22.8h, v5.8h, v18.8h + sabd v23.8h, v5.8h, v19.8h + sabd v24.8h, v4.8h, v16.8h // tldiff + sabd v25.8h, v4.8h, v17.8h + sabd v26.8h, v4.8h, v18.8h + sabd v27.8h, v4.8h, v19.8h + sabd v16.8h, v0.8h, v16.8h // ldiff + sabd v17.8h, v1.8h, v17.8h + sabd v18.8h, v2.8h, v18.8h + sabd v19.8h, v3.8h, v19.8h + umin v28.8h, v20.8h, v24.8h // min(tdiff, tldiff) + umin v29.8h, v21.8h, v25.8h + umin v30.8h, v22.8h, v26.8h + umin v31.8h, v23.8h, v27.8h + cmge v20.8h, v24.8h, v20.8h // tldiff >= tdiff + cmge v21.8h, v25.8h, v21.8h + cmge v22.8h, v26.8h, v22.8h + cmge v23.8h, v27.8h, v23.8h + cmge v16.8h, v28.8h, v16.8h // min(tdiff, tldiff) >= ldiff + cmge v17.8h, v29.8h, v17.8h + cmge v18.8h, v30.8h, v18.8h + cmge v19.8h, v31.8h, v19.8h + bsl v23.16b, v5.16b, v4.16b // tdiff <= tldiff ? top : topleft + bsl v22.16b, v5.16b, v4.16b + bsl v21.16b, v5.16b, v4.16b + bsl v20.16b, v5.16b, v4.16b + bit v23.16b, v3.16b, v19.16b // ldiff <= min ? left : ... + bit v22.16b, v2.16b, v18.16b + bit v21.16b, v1.16b, v17.16b + bit v20.16b, v0.16b, v16.16b + st1 {v23.8h}, [x0], #16 + st1 {v22.8h}, [x6], #16 + subs w3, w3, #8 + st1 {v21.8h}, [x5], #16 + st1 {v20.8h}, [x10], #16 + b.le 8f + ld1 {v5.8h}, [x8], #16 + b 2b +8: + subs w4, w4, #4 + b.le 9f + // End of horizontal loop, move pointers to next four rows + sub x8, x8, w9, uxtw #1 + add x0, x0, x1 + add x6, x6, x1 + // Load the top row as early as possible + ld1 {v5.8h}, [x8], #16 + add x5, x5, x1 + add x10, x10, x1 + mov w3, w9 + b 1b +9: + ret + +L(ipred_paeth_tbl): + .hword L(ipred_paeth_tbl) - 640b + .hword L(ipred_paeth_tbl) - 320b + .hword L(ipred_paeth_tbl) - 160b + .hword L(ipred_paeth_tbl) - 80b + .hword L(ipred_paeth_tbl) - 40b +endfunc + +// void ipred_smooth_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_smooth_16bpc_neon, export=1 + movrel x10, X(sm_weights) + add x11, x10, w4, uxtw + add x10, x10, w3, uxtw + clz w9, w3 + adr x5, L(ipred_smooth_tbl) + sub x12, x2, w4, uxtw #1 + sub w9, w9, #25 + ldrh w9, [x5, w9, uxtw #1] + ld1r {v4.8h}, [x12] // bottom + add x8, x2, #2 + sub x5, x5, w9, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + br x5 +40: + ld1r {v6.2d}, [x8] // top + ld1r {v7.2s}, [x10] // weights_hor + sub x2, x2, #8 + mov x7, #-8 + dup v5.8h, v6.h[3] // right + sub v6.8h, v6.8h, v4.8h // top-bottom + uxtl v7.8h, v7.8b // weights_hor + add v31.4h, v4.4h, v5.4h // bottom+right +4: + ld4r {v0.4h, v1.4h, v2.4h, v3.4h}, [x2], x7 // left + ld4r {v16.8b, v17.8b, v18.8b, v19.8b}, [x11], #4 // weights_ver + ushll v20.4s, v31.4h, #8 // (bottom+right)*256 + ushll v21.4s, v31.4h, #8 + ushll v22.4s, v31.4h, #8 + ushll v23.4s, v31.4h, #8 + zip1 v1.2d, v1.2d, v0.2d // left, flipped + zip1 v0.2d, v3.2d, v2.2d + zip1 v16.2s, v16.2s, v17.2s // weights_ver + zip1 v18.2s, v18.2s, v19.2s + sub v0.8h, v0.8h, v5.8h // left-right + sub v1.8h, v1.8h, v5.8h + uxtl v16.8h, v16.8b // weights_ver + uxtl v18.8h, v18.8b + smlal v20.4s, v0.4h, v7.4h // += (left-right)*weights_hor + smlal2 v21.4s, v0.8h, v7.8h + smlal v22.4s, v1.4h, v7.4h + smlal2 v23.4s, v1.8h, v7.8h + smlal v20.4s, v6.4h, v16.4h // += (top-bottom)*weights_ver + smlal2 v21.4s, v6.8h, v16.8h + smlal v22.4s, v6.4h, v18.4h + smlal2 v23.4s, v6.8h, v18.8h + rshrn v20.4h, v20.4s, #9 + rshrn v21.4h, v21.4s, #9 + rshrn v22.4h, v22.4s, #9 + rshrn v23.4h, v23.4s, #9 + st1 {v20.4h}, [x0], x1 + st1 {v21.4h}, [x6], x1 + subs w4, w4, #4 + st1 {v22.4h}, [x0], x1 + st1 {v23.4h}, [x6], x1 + b.gt 4b + ret +80: + ld1 {v6.8h}, [x8] // top + ld1 {v7.8b}, [x10] // weights_hor + sub x2, x2, #8 + mov x7, #-8 + dup v5.8h, v6.h[7] // right + sub v6.8h, v6.8h, v4.8h // top-bottom + uxtl v7.8h, v7.8b // weights_hor + add v31.4h, v4.4h, v5.4h // bottom+right +8: + ld4r {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], x7 // left + ld4r {v16.8b, v17.8b, v18.8b, v19.8b}, [x11], #4 // weights_ver + ushll v20.4s, v31.4h, #8 // (bottom+right)*256 + ushll v21.4s, v31.4h, #8 + ushll v22.4s, v31.4h, #8 + ushll v23.4s, v31.4h, #8 + ushll v24.4s, v31.4h, #8 + ushll v25.4s, v31.4h, #8 + ushll v26.4s, v31.4h, #8 + ushll v27.4s, v31.4h, #8 + sub v0.8h, v0.8h, v5.8h // left-right + sub v1.8h, v1.8h, v5.8h + sub v2.8h, v2.8h, v5.8h + sub v3.8h, v3.8h, v5.8h + uxtl v16.8h, v16.8b // weights_ver + uxtl v17.8h, v17.8b + uxtl v18.8h, v18.8b + uxtl v19.8h, v19.8b + smlal v20.4s, v3.4h, v7.4h // += (left-right)*weights_hor + smlal2 v21.4s, v3.8h, v7.8h // (left flipped) + smlal v22.4s, v2.4h, v7.4h + smlal2 v23.4s, v2.8h, v7.8h + smlal v24.4s, v1.4h, v7.4h + smlal2 v25.4s, v1.8h, v7.8h + smlal v26.4s, v0.4h, v7.4h + smlal2 v27.4s, v0.8h, v7.8h + smlal v20.4s, v6.4h, v16.4h // += (top-bottom)*weights_ver + smlal2 v21.4s, v6.8h, v16.8h + smlal v22.4s, v6.4h, v17.4h + smlal2 v23.4s, v6.8h, v17.8h + smlal v24.4s, v6.4h, v18.4h + smlal2 v25.4s, v6.8h, v18.8h + smlal v26.4s, v6.4h, v19.4h + smlal2 v27.4s, v6.8h, v19.8h + rshrn v20.4h, v20.4s, #9 + rshrn2 v20.8h, v21.4s, #9 + rshrn v21.4h, v22.4s, #9 + rshrn2 v21.8h, v23.4s, #9 + rshrn v22.4h, v24.4s, #9 + rshrn2 v22.8h, v25.4s, #9 + rshrn v23.4h, v26.4s, #9 + rshrn2 v23.8h, v27.4s, #9 + st1 {v20.8h}, [x0], x1 + st1 {v21.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v22.8h}, [x0], x1 + st1 {v23.8h}, [x6], x1 + b.gt 8b + ret +160: +320: +640: + add x12, x2, w3, uxtw #1 + sub x1, x1, w3, uxtw #1 + ld1r {v5.8h}, [x12] // right + sub x2, x2, #4 + mov x7, #-4 + mov w9, w3 + add v31.4h, v4.4h, v5.4h // bottom+right + +1: + ld2r {v0.8h, v1.8h}, [x2], x7 // left + ld2r {v16.8b, v17.8b}, [x11], #2 // weights_ver + sub v0.8h, v0.8h, v5.8h // left-right + sub v1.8h, v1.8h, v5.8h + uxtl v16.8h, v16.8b // weights_ver + uxtl v17.8h, v17.8b +2: + ld1 {v7.16b}, [x10], #16 // weights_hor + ld1 {v2.8h, v3.8h}, [x8], #32 // top + ushll v20.4s, v31.4h, #8 // (bottom+right)*256 + ushll v21.4s, v31.4h, #8 + ushll v22.4s, v31.4h, #8 + ushll v23.4s, v31.4h, #8 + ushll v24.4s, v31.4h, #8 + ushll v25.4s, v31.4h, #8 + ushll v26.4s, v31.4h, #8 + ushll v27.4s, v31.4h, #8 + uxtl v6.8h, v7.8b // weights_hor + uxtl2 v7.8h, v7.16b + sub v2.8h, v2.8h, v4.8h // top-bottom + sub v3.8h, v3.8h, v4.8h + smlal v20.4s, v1.4h, v6.4h // += (left-right)*weights_hor + smlal2 v21.4s, v1.8h, v6.8h // (left flipped) + smlal v22.4s, v1.4h, v7.4h + smlal2 v23.4s, v1.8h, v7.8h + smlal v24.4s, v0.4h, v6.4h + smlal2 v25.4s, v0.8h, v6.8h + smlal v26.4s, v0.4h, v7.4h + smlal2 v27.4s, v0.8h, v7.8h + smlal v20.4s, v2.4h, v16.4h // += (top-bottom)*weights_ver + smlal2 v21.4s, v2.8h, v16.8h + smlal v22.4s, v3.4h, v16.4h + smlal2 v23.4s, v3.8h, v16.8h + smlal v24.4s, v2.4h, v17.4h + smlal2 v25.4s, v2.8h, v17.8h + smlal v26.4s, v3.4h, v17.4h + smlal2 v27.4s, v3.8h, v17.8h + rshrn v20.4h, v20.4s, #9 + rshrn2 v20.8h, v21.4s, #9 + rshrn v21.4h, v22.4s, #9 + rshrn2 v21.8h, v23.4s, #9 + rshrn v22.4h, v24.4s, #9 + rshrn2 v22.8h, v25.4s, #9 + rshrn v23.4h, v26.4s, #9 + rshrn2 v23.8h, v27.4s, #9 + subs w3, w3, #16 + st1 {v20.8h, v21.8h}, [x0], #32 + st1 {v22.8h, v23.8h}, [x6], #32 + b.gt 2b + subs w4, w4, #2 + b.le 9f + sub x8, x8, w9, uxtw #1 + sub x10, x10, w9, uxtw + add x0, x0, x1 + add x6, x6, x1 + mov w3, w9 + b 1b +9: + ret + +L(ipred_smooth_tbl): + .hword L(ipred_smooth_tbl) - 640b + .hword L(ipred_smooth_tbl) - 320b + .hword L(ipred_smooth_tbl) - 160b + .hword L(ipred_smooth_tbl) - 80b + .hword L(ipred_smooth_tbl) - 40b +endfunc + +// void ipred_smooth_v_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_smooth_v_16bpc_neon, export=1 + movrel x7, X(sm_weights) + add x7, x7, w4, uxtw + clz w9, w3 + adr x5, L(ipred_smooth_v_tbl) + sub x8, x2, w4, uxtw #1 + sub w9, w9, #25 + ldrh w9, [x5, w9, uxtw #1] + ld1r {v4.8h}, [x8] // bottom + add x2, x2, #2 + sub x5, x5, w9, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + br x5 +40: + ld1r {v6.2d}, [x2] // top + sub v6.8h, v6.8h, v4.8h // top-bottom +4: + ld4r {v16.8b, v17.8b, v18.8b, v19.8b}, [x7], #4 // weights_ver + zip1 v16.2s, v16.2s, v17.2s // weights_ver + zip1 v18.2s, v18.2s, v19.2s + ushll v16.8h, v16.8b, #7 // weights_ver << 7 + ushll v18.8h, v18.8b, #7 + sqrdmulh v20.8h, v6.8h, v16.8h // ((top-bottom)*weights_ver + 128) >> 8 + sqrdmulh v21.8h, v6.8h, v18.8h + add v20.8h, v20.8h, v4.8h + add v21.8h, v21.8h, v4.8h + st1 {v20.d}[0], [x0], x1 + st1 {v20.d}[1], [x6], x1 + subs w4, w4, #4 + st1 {v21.d}[0], [x0], x1 + st1 {v21.d}[1], [x6], x1 + b.gt 4b + ret +80: + ld1 {v6.8h}, [x2] // top + sub v6.8h, v6.8h, v4.8h // top-bottom +8: + ld4r {v16.8b, v17.8b, v18.8b, v19.8b}, [x7], #4 // weights_ver + ushll v16.8h, v16.8b, #7 // weights_ver << 7 + ushll v17.8h, v17.8b, #7 + ushll v18.8h, v18.8b, #7 + ushll v19.8h, v19.8b, #7 + sqrdmulh v20.8h, v6.8h, v16.8h // ((top-bottom)*weights_ver + 128) >> 8 + sqrdmulh v21.8h, v6.8h, v17.8h + sqrdmulh v22.8h, v6.8h, v18.8h + sqrdmulh v23.8h, v6.8h, v19.8h + add v20.8h, v20.8h, v4.8h + add v21.8h, v21.8h, v4.8h + add v22.8h, v22.8h, v4.8h + add v23.8h, v23.8h, v4.8h + st1 {v20.8h}, [x0], x1 + st1 {v21.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v22.8h}, [x0], x1 + st1 {v23.8h}, [x6], x1 + b.gt 8b + ret +160: +320: +640: + // Set up pointers for four rows in parallel; x0, x6, x5, x8 + add x5, x0, x1 + add x8, x6, x1 + lsl x1, x1, #1 + sub x1, x1, w3, uxtw #1 + mov w9, w3 + +1: + ld4r {v16.8b, v17.8b, v18.8b, v19.8b}, [x7], #4 // weights_ver + ushll v16.8h, v16.8b, #7 // weights_ver << 7 + ushll v17.8h, v17.8b, #7 + ushll v18.8h, v18.8b, #7 + ushll v19.8h, v19.8b, #7 +2: + ld1 {v2.8h, v3.8h}, [x2], #32 // top + sub v2.8h, v2.8h, v4.8h // top-bottom + sub v3.8h, v3.8h, v4.8h + sqrdmulh v20.8h, v2.8h, v16.8h // ((top-bottom)*weights_ver + 128) >> 8 + sqrdmulh v21.8h, v3.8h, v16.8h + sqrdmulh v22.8h, v2.8h, v17.8h + sqrdmulh v23.8h, v3.8h, v17.8h + sqrdmulh v24.8h, v2.8h, v18.8h + sqrdmulh v25.8h, v3.8h, v18.8h + sqrdmulh v26.8h, v2.8h, v19.8h + sqrdmulh v27.8h, v3.8h, v19.8h + add v20.8h, v20.8h, v4.8h + add v21.8h, v21.8h, v4.8h + add v22.8h, v22.8h, v4.8h + add v23.8h, v23.8h, v4.8h + add v24.8h, v24.8h, v4.8h + add v25.8h, v25.8h, v4.8h + add v26.8h, v26.8h, v4.8h + add v27.8h, v27.8h, v4.8h + subs w3, w3, #16 + st1 {v20.8h, v21.8h}, [x0], #32 + st1 {v22.8h, v23.8h}, [x6], #32 + st1 {v24.8h, v25.8h}, [x5], #32 + st1 {v26.8h, v27.8h}, [x8], #32 + b.gt 2b + subs w4, w4, #4 + b.le 9f + sub x2, x2, w9, uxtw #1 + add x0, x0, x1 + add x6, x6, x1 + add x5, x5, x1 + add x8, x8, x1 + mov w3, w9 + b 1b +9: + ret + +L(ipred_smooth_v_tbl): + .hword L(ipred_smooth_v_tbl) - 640b + .hword L(ipred_smooth_v_tbl) - 320b + .hword L(ipred_smooth_v_tbl) - 160b + .hword L(ipred_smooth_v_tbl) - 80b + .hword L(ipred_smooth_v_tbl) - 40b +endfunc + +// void ipred_smooth_h_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int a, +// const int max_width, const int max_height); +function ipred_smooth_h_16bpc_neon, export=1 + movrel x8, X(sm_weights) + add x8, x8, w3, uxtw + clz w9, w3 + adr x5, L(ipred_smooth_h_tbl) + add x12, x2, w3, uxtw #1 + sub w9, w9, #25 + ldrh w9, [x5, w9, uxtw #1] + ld1r {v5.8h}, [x12] // right + sub x5, x5, w9, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + br x5 +40: + ld1r {v7.2s}, [x8] // weights_hor + sub x2, x2, #8 + mov x7, #-8 + ushll v7.8h, v7.8b, #7 // weights_hor << 7 +4: + ld4r {v0.4h, v1.4h, v2.4h, v3.4h}, [x2], x7 // left + zip1 v1.2d, v1.2d, v0.2d // left, flipped + zip1 v0.2d, v3.2d, v2.2d + sub v0.8h, v0.8h, v5.8h // left-right + sub v1.8h, v1.8h, v5.8h + sqrdmulh v20.8h, v0.8h, v7.8h // ((left-right)*weights_hor + 128) >> 8 + sqrdmulh v21.8h, v1.8h, v7.8h + add v20.8h, v20.8h, v5.8h + add v21.8h, v21.8h, v5.8h + st1 {v20.d}[0], [x0], x1 + st1 {v20.d}[1], [x6], x1 + subs w4, w4, #4 + st1 {v21.d}[0], [x0], x1 + st1 {v21.d}[1], [x6], x1 + b.gt 4b + ret +80: + ld1 {v7.8b}, [x8] // weights_hor + sub x2, x2, #8 + mov x7, #-8 + ushll v7.8h, v7.8b, #7 // weights_hor << 7 +8: + ld4r {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], x7 // left + sub v3.8h, v3.8h, v5.8h // left-right + sub v2.8h, v2.8h, v5.8h + sub v1.8h, v1.8h, v5.8h + sub v0.8h, v0.8h, v5.8h + sqrdmulh v20.8h, v3.8h, v7.8h // ((left-right)*weights_hor + 128) >> 8 + sqrdmulh v21.8h, v2.8h, v7.8h // (left flipped) + sqrdmulh v22.8h, v1.8h, v7.8h + sqrdmulh v23.8h, v0.8h, v7.8h + add v20.8h, v20.8h, v5.8h + add v21.8h, v21.8h, v5.8h + add v22.8h, v22.8h, v5.8h + add v23.8h, v23.8h, v5.8h + st1 {v20.8h}, [x0], x1 + st1 {v21.8h}, [x6], x1 + subs w4, w4, #4 + st1 {v22.8h}, [x0], x1 + st1 {v23.8h}, [x6], x1 + b.gt 8b + ret +160: +320: +640: + sub x2, x2, #8 + mov x7, #-8 + // Set up pointers for four rows in parallel; x0, x6, x5, x10 + add x5, x0, x1 + add x10, x6, x1 + lsl x1, x1, #1 + sub x1, x1, w3, uxtw #1 + mov w9, w3 + +1: + ld4r {v0.8h, v1.8h, v2.8h, v3.8h}, [x2], x7 // left + sub v0.8h, v0.8h, v5.8h // left-right + sub v1.8h, v1.8h, v5.8h + sub v2.8h, v2.8h, v5.8h + sub v3.8h, v3.8h, v5.8h +2: + ld1 {v7.16b}, [x8], #16 // weights_hor + ushll v6.8h, v7.8b, #7 // weights_hor << 7 + ushll2 v7.8h, v7.16b, #7 + sqrdmulh v20.8h, v3.8h, v6.8h // ((left-right)*weights_hor + 128) >> 8 + sqrdmulh v21.8h, v3.8h, v7.8h // (left flipped) + sqrdmulh v22.8h, v2.8h, v6.8h + sqrdmulh v23.8h, v2.8h, v7.8h + sqrdmulh v24.8h, v1.8h, v6.8h + sqrdmulh v25.8h, v1.8h, v7.8h + sqrdmulh v26.8h, v0.8h, v6.8h + sqrdmulh v27.8h, v0.8h, v7.8h + add v20.8h, v20.8h, v5.8h + add v21.8h, v21.8h, v5.8h + add v22.8h, v22.8h, v5.8h + add v23.8h, v23.8h, v5.8h + add v24.8h, v24.8h, v5.8h + add v25.8h, v25.8h, v5.8h + add v26.8h, v26.8h, v5.8h + add v27.8h, v27.8h, v5.8h + subs w3, w3, #16 + st1 {v20.8h, v21.8h}, [x0], #32 + st1 {v22.8h, v23.8h}, [x6], #32 + st1 {v24.8h, v25.8h}, [x5], #32 + st1 {v26.8h, v27.8h}, [x10], #32 + b.gt 2b + subs w4, w4, #4 + b.le 9f + sub x8, x8, w9, uxtw + add x0, x0, x1 + add x6, x6, x1 + add x5, x5, x1 + add x10, x10, x1 + mov w3, w9 + b 1b +9: + ret + +L(ipred_smooth_h_tbl): + .hword L(ipred_smooth_h_tbl) - 640b + .hword L(ipred_smooth_h_tbl) - 320b + .hword L(ipred_smooth_h_tbl) - 160b + .hword L(ipred_smooth_h_tbl) - 80b + .hword L(ipred_smooth_h_tbl) - 40b +endfunc + +// void ipred_filter_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, const int filt_idx, +// const int max_width, const int max_height, +// const int bitdepth_max); +.macro filter_fn bpc +function ipred_filter_\bpc\()bpc_neon + and w5, w5, #511 + movrel x6, X(filter_intra_taps) + lsl w5, w5, #6 + add x6, x6, w5, uxtw + ld1 {v16.8b, v17.8b, v18.8b, v19.8b}, [x6], #32 + clz w9, w3 + adr x5, L(ipred_filter\bpc\()_tbl) + ld1 {v20.8b, v21.8b, v22.8b}, [x6] + sub w9, w9, #26 + ldrh w9, [x5, w9, uxtw #1] + sxtl v16.8h, v16.8b + sxtl v17.8h, v17.8b + sub x5, x5, w9, uxtw + sxtl v18.8h, v18.8b + sxtl v19.8h, v19.8b + add x6, x0, x1 + lsl x1, x1, #1 + sxtl v20.8h, v20.8b + sxtl v21.8h, v21.8b + sxtl v22.8h, v22.8b + dup v31.8h, w8 + movi v30.8h, #0 + br x5 +40: + ldur d0, [x2, #2] // top (0-3) + sub x2, x2, #4 + mov x7, #-4 +4: + ld1 {v1.4h}, [x2], x7 // left (0-1) + topleft (2) +.if \bpc == 10 + mul v2.8h, v17.8h, v0.h[0] // p1(top[0]) * filter(1) + mla v2.8h, v18.8h, v0.h[1] // p2(top[1]) * filter(2) + mla v2.8h, v19.8h, v0.h[2] // p3(top[2]) * filter(3) + mla v2.8h, v20.8h, v0.h[3] // p4(top[3]) * filter(4) + mla v2.8h, v16.8h, v1.h[2] // p0(topleft) * filter(0) + mla v2.8h, v21.8h, v1.h[1] // p5(left[0]) * filter(5) + mla v2.8h, v22.8h, v1.h[0] // p6(left[1]) * filter(6) + srshr v2.8h, v2.8h, #4 + smax v2.8h, v2.8h, v30.8h +.else + smull v2.4s, v17.4h, v0.h[0] // p1(top[0]) * filter(1) + smlal v2.4s, v18.4h, v0.h[1] // p2(top[1]) * filter(2) + smlal v2.4s, v19.4h, v0.h[2] // p3(top[2]) * filter(3) + smlal v2.4s, v20.4h, v0.h[3] // p4(top[3]) * filter(4) + smlal v2.4s, v16.4h, v1.h[2] // p0(topleft) * filter(0) + smlal v2.4s, v21.4h, v1.h[1] // p5(left[0]) * filter(5) + smlal v2.4s, v22.4h, v1.h[0] // p6(left[1]) * filter(6) + smull2 v3.4s, v17.8h, v0.h[0] // p1(top[0]) * filter(1) + smlal2 v3.4s, v18.8h, v0.h[1] // p2(top[1]) * filter(2) + smlal2 v3.4s, v19.8h, v0.h[2] // p3(top[2]) * filter(3) + smlal2 v3.4s, v20.8h, v0.h[3] // p4(top[3]) * filter(4) + smlal2 v3.4s, v16.8h, v1.h[2] // p0(topleft) * filter(0) + smlal2 v3.4s, v21.8h, v1.h[1] // p5(left[0]) * filter(5) + smlal2 v3.4s, v22.8h, v1.h[0] // p6(left[1]) * filter(6) + sqrshrun v2.4h, v2.4s, #4 + sqrshrun2 v2.8h, v3.4s, #4 +.endif + smin v2.8h, v2.8h, v31.8h + subs w4, w4, #2 + st1 {v2.d}[0], [x0], x1 + uxtl v0.8h, v2.8b + ext v0.16b, v2.16b, v2.16b, #8 // move top from [4-7] to [0-3] + st1 {v2.d}[1], [x6], x1 + b.gt 4b + ret +80: + ldur q0, [x2, #2] // top (0-7) + sub x2, x2, #4 + mov x7, #-4 +8: + ld1 {v1.4h}, [x2], x7 // left (0-1) + topleft (2) +.if \bpc == 10 + mul v2.8h, v17.8h, v0.h[0] // p1(top[0]) * filter(1) + mla v2.8h, v18.8h, v0.h[1] // p2(top[1]) * filter(2) + mla v2.8h, v19.8h, v0.h[2] // p3(top[2]) * filter(3) + mla v2.8h, v20.8h, v0.h[3] // p4(top[3]) * filter(4) + mla v2.8h, v16.8h, v1.h[2] // p0(topleft) * filter(0) + mla v2.8h, v21.8h, v1.h[1] // p5(left[0]) * filter(5) + mla v2.8h, v22.8h, v1.h[0] // p6(left[1]) * filter(6) + mul v3.8h, v17.8h, v0.h[4] // p1(top[0]) * filter(1) + mla v3.8h, v18.8h, v0.h[5] // p2(top[1]) * filter(2) + mla v3.8h, v19.8h, v0.h[6] // p3(top[2]) * filter(3) + srshr v2.8h, v2.8h, #4 + smax v2.8h, v2.8h, v30.8h + smin v2.8h, v2.8h, v31.8h + mla v3.8h, v20.8h, v0.h[7] // p4(top[3]) * filter(4) + mla v3.8h, v16.8h, v0.h[3] // p0(topleft) * filter(0) + mla v3.8h, v21.8h, v2.h[3] // p5(left[0]) * filter(5) + mla v3.8h, v22.8h, v2.h[7] // p6(left[1]) * filter(6) + srshr v3.8h, v3.8h, #4 + smax v3.8h, v3.8h, v30.8h +.else + smull v2.4s, v17.4h, v0.h[0] // p1(top[0]) * filter(1) + smlal v2.4s, v18.4h, v0.h[1] // p2(top[1]) * filter(2) + smlal v2.4s, v19.4h, v0.h[2] // p3(top[2]) * filter(3) + smlal v2.4s, v20.4h, v0.h[3] // p4(top[3]) * filter(4) + smlal v2.4s, v16.4h, v1.h[2] // p0(topleft) * filter(0) + smlal v2.4s, v21.4h, v1.h[1] // p5(left[0]) * filter(5) + smlal v2.4s, v22.4h, v1.h[0] // p6(left[1]) * filter(6) + smull2 v3.4s, v17.8h, v0.h[0] // p1(top[0]) * filter(1) + smlal2 v3.4s, v18.8h, v0.h[1] // p2(top[1]) * filter(2) + smlal2 v3.4s, v19.8h, v0.h[2] // p3(top[2]) * filter(3) + smlal2 v3.4s, v20.8h, v0.h[3] // p4(top[3]) * filter(4) + smlal2 v3.4s, v16.8h, v1.h[2] // p0(topleft) * filter(0) + smlal2 v3.4s, v21.8h, v1.h[1] // p5(left[0]) * filter(5) + smlal2 v3.4s, v22.8h, v1.h[0] // p6(left[1]) * filter(6) + smull v4.4s, v17.4h, v0.h[4] // p1(top[0]) * filter(1) + smlal v4.4s, v18.4h, v0.h[5] // p2(top[1]) * filter(2) + smlal v4.4s, v19.4h, v0.h[6] // p3(top[2]) * filter(3) + sqrshrun v2.4h, v2.4s, #4 + sqrshrun2 v2.8h, v3.4s, #4 + smin v2.8h, v2.8h, v31.8h + smlal v4.4s, v20.4h, v0.h[7] // p4(top[3]) * filter(4) + smlal v4.4s, v16.4h, v0.h[3] // p0(topleft) * filter(0) + smlal v4.4s, v21.4h, v2.h[3] // p5(left[0]) * filter(5) + smlal v4.4s, v22.4h, v2.h[7] // p6(left[1]) * filter(6) + smull2 v5.4s, v17.8h, v0.h[4] // p1(top[0]) * filter(1) + smlal2 v5.4s, v18.8h, v0.h[5] // p2(top[1]) * filter(2) + smlal2 v5.4s, v19.8h, v0.h[6] // p3(top[2]) * filter(3) + smlal2 v5.4s, v20.8h, v0.h[7] // p4(top[3]) * filter(4) + smlal2 v5.4s, v16.8h, v0.h[3] // p0(topleft) * filter(0) + smlal2 v5.4s, v21.8h, v2.h[3] // p5(left[0]) * filter(5) + smlal2 v5.4s, v22.8h, v2.h[7] // p6(left[1]) * filter(6) + sqrshrun v3.4h, v4.4s, #4 + sqrshrun2 v3.8h, v5.4s, #4 +.endif + smin v3.8h, v3.8h, v31.8h + subs w4, w4, #2 + st2 {v2.d, v3.d}[0], [x0], x1 + zip2 v0.2d, v2.2d, v3.2d + st2 {v2.d, v3.d}[1], [x6], x1 + b.gt 8b + ret +160: +320: + add x8, x2, #2 + sub x2, x2, #4 + mov x7, #-4 + sub x1, x1, w3, uxtw #1 + mov w9, w3 + +1: + ld1 {v0.4h}, [x2], x7 // left (0-1) + topleft (2) +2: + ld1 {v1.8h, v2.8h}, [x8], #32 // top(0-15) +.if \bpc == 10 + mul v3.8h, v16.8h, v0.h[2] // p0(topleft) * filter(0) + mla v3.8h, v21.8h, v0.h[1] // p5(left[0]) * filter(5) + mla v3.8h, v22.8h, v0.h[0] // p6(left[1]) * filter(6) + mla v3.8h, v17.8h, v1.h[0] // p1(top[0]) * filter(1) + mla v3.8h, v18.8h, v1.h[1] // p2(top[1]) * filter(2) + mla v3.8h, v19.8h, v1.h[2] // p3(top[2]) * filter(3) + mla v3.8h, v20.8h, v1.h[3] // p4(top[3]) * filter(4) + + mul v4.8h, v17.8h, v1.h[4] // p1(top[0]) * filter(1) + mla v4.8h, v18.8h, v1.h[5] // p2(top[1]) * filter(2) + mla v4.8h, v19.8h, v1.h[6] // p3(top[2]) * filter(3) + srshr v3.8h, v3.8h, #4 + smax v3.8h, v3.8h, v30.8h + smin v3.8h, v3.8h, v31.8h + mla v4.8h, v20.8h, v1.h[7] // p4(top[3]) * filter(4) + mla v4.8h, v16.8h, v1.h[3] // p0(topleft) * filter(0) + mla v4.8h, v21.8h, v3.h[3] // p5(left[0]) * filter(5) + mla v4.8h, v22.8h, v3.h[7] // p6(left[1]) * filter(6) + + mul v5.8h, v17.8h, v2.h[0] // p1(top[0]) * filter(1) + mla v5.8h, v18.8h, v2.h[1] // p2(top[1]) * filter(2) + mla v5.8h, v19.8h, v2.h[2] // p3(top[2]) * filter(3) + srshr v4.8h, v4.8h, #4 + smax v4.8h, v4.8h, v30.8h + smin v4.8h, v4.8h, v31.8h + mla v5.8h, v20.8h, v2.h[3] // p4(top[3]) * filter(4) + mla v5.8h, v16.8h, v1.h[7] // p0(topleft) * filter(0) + mla v5.8h, v21.8h, v4.h[3] // p5(left[0]) * filter(5) + mla v5.8h, v22.8h, v4.h[7] // p6(left[1]) * filter(6) + + mul v6.8h, v17.8h, v2.h[4] // p1(top[0]) * filter(1) + mla v6.8h, v18.8h, v2.h[5] // p2(top[1]) * filter(2) + mla v6.8h, v19.8h, v2.h[6] // p3(top[2]) * filter(3) + srshr v5.8h, v5.8h, #4 + smax v5.8h, v5.8h, v30.8h + smin v5.8h, v5.8h, v31.8h + mla v6.8h, v20.8h, v2.h[7] // p4(top[3]) * filter(4) + mla v6.8h, v16.8h, v2.h[3] // p0(topleft) * filter(0) + mla v6.8h, v21.8h, v5.h[3] // p5(left[0]) * filter(5) + mla v6.8h, v22.8h, v5.h[7] // p6(left[1]) * filter(6) + + subs w3, w3, #16 + srshr v6.8h, v6.8h, #4 + smax v6.8h, v6.8h, v30.8h +.else + smull v3.4s, v16.4h, v0.h[2] // p0(topleft) * filter(0) + smlal v3.4s, v21.4h, v0.h[1] // p5(left[0]) * filter(5) + smlal v3.4s, v22.4h, v0.h[0] // p6(left[1]) * filter(6) + smlal v3.4s, v17.4h, v1.h[0] // p1(top[0]) * filter(1) + smlal v3.4s, v18.4h, v1.h[1] // p2(top[1]) * filter(2) + smlal v3.4s, v19.4h, v1.h[2] // p3(top[2]) * filter(3) + smlal v3.4s, v20.4h, v1.h[3] // p4(top[3]) * filter(4) + smull2 v4.4s, v16.8h, v0.h[2] // p0(topleft) * filter(0) + smlal2 v4.4s, v21.8h, v0.h[1] // p5(left[0]) * filter(5) + smlal2 v4.4s, v22.8h, v0.h[0] // p6(left[1]) * filter(6) + smlal2 v4.4s, v17.8h, v1.h[0] // p1(top[0]) * filter(1) + smlal2 v4.4s, v18.8h, v1.h[1] // p2(top[1]) * filter(2) + smlal2 v4.4s, v19.8h, v1.h[2] // p3(top[2]) * filter(3) + smlal2 v4.4s, v20.8h, v1.h[3] // p4(top[3]) * filter(4) + + smull v5.4s, v17.4h, v1.h[4] // p1(top[0]) * filter(1) + smlal v5.4s, v18.4h, v1.h[5] // p2(top[1]) * filter(2) + smlal v5.4s, v19.4h, v1.h[6] // p3(top[2]) * filter(3) + sqrshrun v3.4h, v3.4s, #4 + sqrshrun2 v3.8h, v4.4s, #4 + smin v3.8h, v3.8h, v31.8h + smlal v5.4s, v20.4h, v1.h[7] // p4(top[3]) * filter(4) + smlal v5.4s, v16.4h, v1.h[3] // p0(topleft) * filter(0) + smlal v5.4s, v21.4h, v3.h[3] // p5(left[0]) * filter(5) + smlal v5.4s, v22.4h, v3.h[7] // p6(left[1]) * filter(6) + smull2 v6.4s, v17.8h, v1.h[4] // p1(top[0]) * filter(1) + smlal2 v6.4s, v18.8h, v1.h[5] // p2(top[1]) * filter(2) + smlal2 v6.4s, v19.8h, v1.h[6] // p3(top[2]) * filter(3) + smlal2 v6.4s, v20.8h, v1.h[7] // p4(top[3]) * filter(4) + smlal2 v6.4s, v16.8h, v1.h[3] // p0(topleft) * filter(0) + smlal2 v6.4s, v21.8h, v3.h[3] // p5(left[0]) * filter(5) + smlal2 v6.4s, v22.8h, v3.h[7] // p6(left[1]) * filter(6) + + smull v24.4s, v17.4h, v2.h[0] // p1(top[0]) * filter(1) + smlal v24.4s, v18.4h, v2.h[1] // p2(top[1]) * filter(2) + smlal v24.4s, v19.4h, v2.h[2] // p3(top[2]) * filter(3) + sqrshrun v4.4h, v5.4s, #4 + sqrshrun2 v4.8h, v6.4s, #4 + smin v4.8h, v4.8h, v31.8h + smlal v24.4s, v20.4h, v2.h[3] // p4(top[3]) * filter(4) + smlal v24.4s, v16.4h, v1.h[7] // p0(topleft) * filter(0) + smlal v24.4s, v21.4h, v4.h[3] // p5(left[0]) * filter(5) + smlal v24.4s, v22.4h, v4.h[7] // p6(left[1]) * filter(6) + smull2 v25.4s, v17.8h, v2.h[0] // p1(top[0]) * filter(1) + smlal2 v25.4s, v18.8h, v2.h[1] // p2(top[1]) * filter(2) + smlal2 v25.4s, v19.8h, v2.h[2] // p3(top[2]) * filter(3) + smlal2 v25.4s, v20.8h, v2.h[3] // p4(top[3]) * filter(4) + smlal2 v25.4s, v16.8h, v1.h[7] // p0(topleft) * filter(0) + smlal2 v25.4s, v21.8h, v4.h[3] // p5(left[0]) * filter(5) + smlal2 v25.4s, v22.8h, v4.h[7] // p6(left[1]) * filter(6) + + smull v26.4s, v17.4h, v2.h[4] // p1(top[0]) * filter(1) + smlal v26.4s, v18.4h, v2.h[5] // p2(top[1]) * filter(2) + smlal v26.4s, v19.4h, v2.h[6] // p3(top[2]) * filter(3) + sqrshrun v5.4h, v24.4s, #4 + sqrshrun2 v5.8h, v25.4s, #4 + smin v5.8h, v5.8h, v31.8h + smlal v26.4s, v20.4h, v2.h[7] // p4(top[3]) * filter(4) + smlal v26.4s, v16.4h, v2.h[3] // p0(topleft) * filter(0) + smlal v26.4s, v21.4h, v5.h[3] // p5(left[0]) * filter(5) + smlal v26.4s, v22.4h, v5.h[7] // p6(left[1]) * filter(6) + smull2 v27.4s, v17.8h, v2.h[4] // p1(top[0]) * filter(1) + smlal2 v27.4s, v18.8h, v2.h[5] // p2(top[1]) * filter(2) + smlal2 v27.4s, v19.8h, v2.h[6] // p3(top[2]) * filter(3) + smlal2 v27.4s, v20.8h, v2.h[7] // p4(top[3]) * filter(4) + smlal2 v27.4s, v16.8h, v2.h[3] // p0(topleft) * filter(0) + smlal2 v27.4s, v21.8h, v5.h[3] // p5(left[0]) * filter(5) + smlal2 v27.4s, v22.8h, v5.h[7] // p6(left[1]) * filter(6) + + subs w3, w3, #16 + sqrshrun v6.4h, v26.4s, #4 + sqrshrun2 v6.8h, v27.4s, #4 +.endif + smin v6.8h, v6.8h, v31.8h + + ins v0.h[2], v2.h[7] + st4 {v3.d, v4.d, v5.d, v6.d}[0], [x0], #32 + ins v0.h[0], v6.h[7] + st4 {v3.d, v4.d, v5.d, v6.d}[1], [x6], #32 + ins v0.h[1], v6.h[3] + b.gt 2b + subs w4, w4, #2 + b.le 9f + sub x8, x6, w9, uxtw #1 + add x0, x0, x1 + add x6, x6, x1 + mov w3, w9 + b 1b +9: + ret + +L(ipred_filter\bpc\()_tbl): + .hword L(ipred_filter\bpc\()_tbl) - 320b + .hword L(ipred_filter\bpc\()_tbl) - 160b + .hword L(ipred_filter\bpc\()_tbl) - 80b + .hword L(ipred_filter\bpc\()_tbl) - 40b +endfunc +.endm + +filter_fn 10 +filter_fn 12 + +function ipred_filter_16bpc_neon, export=1 + ldr w8, [sp] + cmp w8, 0x3ff + b.le ipred_filter_10bpc_neon + b ipred_filter_12bpc_neon +endfunc + +// void pal_pred_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const uint16_t *const pal, const uint8_t *idx, +// const int w, const int h); +function pal_pred_16bpc_neon, export=1 + ld1 {v30.8h}, [x2] + clz w9, w4 + adr x6, L(pal_pred_tbl) + sub w9, w9, #25 + ldrh w9, [x6, w9, uxtw #1] + movi v31.8h, #1, lsl #8 + sub x6, x6, w9, uxtw + br x6 +40: + add x2, x0, x1 + lsl x1, x1, #1 +4: + ld1 {v1.16b}, [x3], #16 + subs w5, w5, #4 + // Restructure v1 from a, b, c, ... into 2*a, 2*a+1, 2*b, 2*b+1, 2*c, 2*c+1, ... + add v1.16b, v1.16b, v1.16b + zip1 v0.16b, v1.16b, v1.16b + zip2 v1.16b, v1.16b, v1.16b + add v0.8h, v0.8h, v31.8h + add v1.8h, v1.8h, v31.8h + tbl v0.16b, {v30.16b}, v0.16b + st1 {v0.d}[0], [x0], x1 + tbl v1.16b, {v30.16b}, v1.16b + st1 {v0.d}[1], [x2], x1 + st1 {v1.d}[0], [x0], x1 + st1 {v1.d}[1], [x2], x1 + b.gt 4b + ret +80: + add x2, x0, x1 + lsl x1, x1, #1 +8: + ld1 {v2.16b, v3.16b}, [x3], #32 + subs w5, w5, #4 + add v2.16b, v2.16b, v2.16b + add v3.16b, v3.16b, v3.16b + zip1 v0.16b, v2.16b, v2.16b + zip2 v1.16b, v2.16b, v2.16b + zip1 v2.16b, v3.16b, v3.16b + zip2 v3.16b, v3.16b, v3.16b + add v0.8h, v0.8h, v31.8h + add v1.8h, v1.8h, v31.8h + add v2.8h, v2.8h, v31.8h + add v3.8h, v3.8h, v31.8h + tbl v0.16b, {v30.16b}, v0.16b + tbl v1.16b, {v30.16b}, v1.16b + st1 {v0.8h}, [x0], x1 + tbl v2.16b, {v30.16b}, v2.16b + st1 {v1.8h}, [x2], x1 + tbl v3.16b, {v30.16b}, v3.16b + st1 {v2.8h}, [x0], x1 + st1 {v3.8h}, [x2], x1 + b.gt 8b + ret +160: + add x2, x0, x1 + lsl x1, x1, #1 +16: + ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x3], #64 + subs w5, w5, #4 + add v4.16b, v4.16b, v4.16b + add v5.16b, v5.16b, v5.16b + add v6.16b, v6.16b, v6.16b + add v7.16b, v7.16b, v7.16b + zip1 v0.16b, v4.16b, v4.16b + zip2 v1.16b, v4.16b, v4.16b + zip1 v2.16b, v5.16b, v5.16b + zip2 v3.16b, v5.16b, v5.16b + zip1 v4.16b, v6.16b, v6.16b + zip2 v5.16b, v6.16b, v6.16b + zip1 v6.16b, v7.16b, v7.16b + zip2 v7.16b, v7.16b, v7.16b + add v0.8h, v0.8h, v31.8h + add v1.8h, v1.8h, v31.8h + add v2.8h, v2.8h, v31.8h + add v3.8h, v3.8h, v31.8h + add v4.8h, v4.8h, v31.8h + tbl v0.16b, {v30.16b}, v0.16b + add v5.8h, v5.8h, v31.8h + tbl v1.16b, {v30.16b}, v1.16b + add v6.8h, v6.8h, v31.8h + tbl v2.16b, {v30.16b}, v2.16b + add v7.8h, v7.8h, v31.8h + tbl v3.16b, {v30.16b}, v3.16b + tbl v4.16b, {v30.16b}, v4.16b + tbl v5.16b, {v30.16b}, v5.16b + st1 {v0.8h, v1.8h}, [x0], x1 + tbl v6.16b, {v30.16b}, v6.16b + st1 {v2.8h, v3.8h}, [x2], x1 + tbl v7.16b, {v30.16b}, v7.16b + st1 {v4.8h, v5.8h}, [x0], x1 + st1 {v6.8h, v7.8h}, [x2], x1 + b.gt 16b + ret +320: + add x2, x0, x1 + lsl x1, x1, #1 +32: + ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x3], #64 + subs w5, w5, #2 + add v4.16b, v4.16b, v4.16b + add v5.16b, v5.16b, v5.16b + add v6.16b, v6.16b, v6.16b + add v7.16b, v7.16b, v7.16b + zip1 v0.16b, v4.16b, v4.16b + zip2 v1.16b, v4.16b, v4.16b + zip1 v2.16b, v5.16b, v5.16b + zip2 v3.16b, v5.16b, v5.16b + zip1 v4.16b, v6.16b, v6.16b + zip2 v5.16b, v6.16b, v6.16b + zip1 v6.16b, v7.16b, v7.16b + zip2 v7.16b, v7.16b, v7.16b + add v0.8h, v0.8h, v31.8h + add v1.8h, v1.8h, v31.8h + add v2.8h, v2.8h, v31.8h + add v3.8h, v3.8h, v31.8h + add v4.8h, v4.8h, v31.8h + tbl v0.16b, {v30.16b}, v0.16b + add v5.8h, v5.8h, v31.8h + tbl v1.16b, {v30.16b}, v1.16b + add v6.8h, v6.8h, v31.8h + tbl v2.16b, {v30.16b}, v2.16b + add v7.8h, v7.8h, v31.8h + tbl v3.16b, {v30.16b}, v3.16b + tbl v4.16b, {v30.16b}, v4.16b + tbl v5.16b, {v30.16b}, v5.16b + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + tbl v6.16b, {v30.16b}, v6.16b + tbl v7.16b, {v30.16b}, v7.16b + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2], x1 + b.gt 32b + ret +640: + add x2, x0, #64 +64: + ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x3], #64 + subs w5, w5, #1 + add v4.16b, v4.16b, v4.16b + add v5.16b, v5.16b, v5.16b + add v6.16b, v6.16b, v6.16b + add v7.16b, v7.16b, v7.16b + zip1 v0.16b, v4.16b, v4.16b + zip2 v1.16b, v4.16b, v4.16b + zip1 v2.16b, v5.16b, v5.16b + zip2 v3.16b, v5.16b, v5.16b + zip1 v4.16b, v6.16b, v6.16b + zip2 v5.16b, v6.16b, v6.16b + zip1 v6.16b, v7.16b, v7.16b + zip2 v7.16b, v7.16b, v7.16b + add v0.8h, v0.8h, v31.8h + add v1.8h, v1.8h, v31.8h + add v2.8h, v2.8h, v31.8h + add v3.8h, v3.8h, v31.8h + add v4.8h, v4.8h, v31.8h + tbl v0.16b, {v30.16b}, v0.16b + add v5.8h, v5.8h, v31.8h + tbl v1.16b, {v30.16b}, v1.16b + add v6.8h, v6.8h, v31.8h + tbl v2.16b, {v30.16b}, v2.16b + add v7.8h, v7.8h, v31.8h + tbl v3.16b, {v30.16b}, v3.16b + tbl v4.16b, {v30.16b}, v4.16b + tbl v5.16b, {v30.16b}, v5.16b + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + tbl v6.16b, {v30.16b}, v6.16b + tbl v7.16b, {v30.16b}, v7.16b + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x2], x1 + b.gt 64b + ret + +L(pal_pred_tbl): + .hword L(pal_pred_tbl) - 640b + .hword L(pal_pred_tbl) - 320b + .hword L(pal_pred_tbl) - 160b + .hword L(pal_pred_tbl) - 80b + .hword L(pal_pred_tbl) - 40b +endfunc + +// void ipred_cfl_128_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha, +// const int bitdepth_max); +function ipred_cfl_128_16bpc_neon, export=1 + dup v31.8h, w7 // bitdepth_max + clz w9, w3 + adr x7, L(ipred_cfl_128_tbl) + sub w9, w9, #26 + ldrh w9, [x7, w9, uxtw #1] + urshr v0.8h, v31.8h, #1 + dup v1.8h, w6 // alpha + sub x7, x7, w9, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + movi v30.8h, #0 + br x7 +L(ipred_cfl_splat_w4): + ld1 {v4.8h, v5.8h}, [x5], #32 + subs w4, w4, #4 + smull v2.4s, v4.4h, v1.4h // diff = ac * alpha + smull2 v3.4s, v4.8h, v1.8h + smull v4.4s, v5.4h, v1.4h + smull2 v5.4s, v5.8h, v1.8h + sshr v16.4s, v2.4s, #31 // sign = diff >> 31 + sshr v17.4s, v3.4s, #31 + sshr v18.4s, v4.4s, #31 + sshr v19.4s, v5.4s, #31 + add v2.4s, v2.4s, v16.4s // diff + sign + add v3.4s, v3.4s, v17.4s + add v4.4s, v4.4s, v18.4s + add v5.4s, v5.4s, v19.4s + rshrn v2.4h, v2.4s, #6 // (diff + sign + 32) >> 6 = apply_sign() + rshrn2 v2.8h, v3.4s, #6 + rshrn v3.4h, v4.4s, #6 + rshrn2 v3.8h, v5.4s, #6 + add v2.8h, v2.8h, v0.8h // dc + apply_sign() + add v3.8h, v3.8h, v0.8h + smax v2.8h, v2.8h, v30.8h + smax v3.8h, v3.8h, v30.8h + smin v2.8h, v2.8h, v31.8h + smin v3.8h, v3.8h, v31.8h + st1 {v2.d}[0], [x0], x1 + st1 {v2.d}[1], [x6], x1 + st1 {v3.d}[0], [x0], x1 + st1 {v3.d}[1], [x6], x1 + b.gt L(ipred_cfl_splat_w4) + ret +L(ipred_cfl_splat_w8): + ld1 {v4.8h, v5.8h}, [x5], #32 + subs w4, w4, #2 + smull v2.4s, v4.4h, v1.4h // diff = ac * alpha + smull2 v3.4s, v4.8h, v1.8h + smull v4.4s, v5.4h, v1.4h + smull2 v5.4s, v5.8h, v1.8h + sshr v16.4s, v2.4s, #31 // sign = diff >> 31 + sshr v17.4s, v3.4s, #31 + sshr v18.4s, v4.4s, #31 + sshr v19.4s, v5.4s, #31 + add v2.4s, v2.4s, v16.4s // diff + sign + add v3.4s, v3.4s, v17.4s + add v4.4s, v4.4s, v18.4s + add v5.4s, v5.4s, v19.4s + rshrn v2.4h, v2.4s, #6 // (diff + sign + 32) >> 6 = apply_sign() + rshrn2 v2.8h, v3.4s, #6 + rshrn v3.4h, v4.4s, #6 + rshrn2 v3.8h, v5.4s, #6 + add v2.8h, v2.8h, v0.8h // dc + apply_sign() + add v3.8h, v3.8h, v0.8h + smax v2.8h, v2.8h, v30.8h + smax v3.8h, v3.8h, v30.8h + smin v2.8h, v2.8h, v31.8h + smin v3.8h, v3.8h, v31.8h + st1 {v2.8h}, [x0], x1 + st1 {v3.8h}, [x6], x1 + b.gt L(ipred_cfl_splat_w8) + ret +L(ipred_cfl_splat_w16): + add x7, x5, w3, uxtw #1 + sub x1, x1, w3, uxtw #1 + mov w9, w3 +1: + ld1 {v2.8h, v3.8h}, [x5], #32 + ld1 {v4.8h, v5.8h}, [x7], #32 + subs w3, w3, #16 + smull v16.4s, v2.4h, v1.4h // diff = ac * alpha + smull2 v17.4s, v2.8h, v1.8h + smull v18.4s, v3.4h, v1.4h + smull2 v19.4s, v3.8h, v1.8h + smull v2.4s, v4.4h, v1.4h + smull2 v3.4s, v4.8h, v1.8h + smull v4.4s, v5.4h, v1.4h + smull2 v5.4s, v5.8h, v1.8h + sshr v20.4s, v16.4s, #31 // sign = diff >> 31 + sshr v21.4s, v17.4s, #31 + sshr v22.4s, v18.4s, #31 + sshr v23.4s, v19.4s, #31 + sshr v24.4s, v2.4s, #31 + sshr v25.4s, v3.4s, #31 + sshr v26.4s, v4.4s, #31 + sshr v27.4s, v5.4s, #31 + add v16.4s, v16.4s, v20.4s // diff + sign + add v17.4s, v17.4s, v21.4s + add v18.4s, v18.4s, v22.4s + add v19.4s, v19.4s, v23.4s + add v2.4s, v2.4s, v24.4s + add v3.4s, v3.4s, v25.4s + add v4.4s, v4.4s, v26.4s + add v5.4s, v5.4s, v27.4s + rshrn v16.4h, v16.4s, #6 // (diff + sign + 32) >> 6 = apply_sign() + rshrn2 v16.8h, v17.4s, #6 + rshrn v17.4h, v18.4s, #6 + rshrn2 v17.8h, v19.4s, #6 + rshrn v6.4h, v2.4s, #6 + rshrn2 v6.8h, v3.4s, #6 + rshrn v7.4h, v4.4s, #6 + rshrn2 v7.8h, v5.4s, #6 + add v2.8h, v16.8h, v0.8h // dc + apply_sign() + add v3.8h, v17.8h, v0.8h + add v4.8h, v6.8h, v0.8h + add v5.8h, v7.8h, v0.8h + smax v2.8h, v2.8h, v30.8h + smax v3.8h, v3.8h, v30.8h + smax v4.8h, v4.8h, v30.8h + smax v5.8h, v5.8h, v30.8h + smin v2.8h, v2.8h, v31.8h + smin v3.8h, v3.8h, v31.8h + smin v4.8h, v4.8h, v31.8h + smin v5.8h, v5.8h, v31.8h + st1 {v2.8h, v3.8h}, [x0], #32 + st1 {v4.8h, v5.8h}, [x6], #32 + b.gt 1b + subs w4, w4, #2 + add x5, x5, w9, uxtw #1 + add x7, x7, w9, uxtw #1 + add x0, x0, x1 + add x6, x6, x1 + mov w3, w9 + b.gt 1b + ret + +L(ipred_cfl_128_tbl): +L(ipred_cfl_splat_tbl): + .hword L(ipred_cfl_128_tbl) - L(ipred_cfl_splat_w16) + .hword L(ipred_cfl_128_tbl) - L(ipred_cfl_splat_w16) + .hword L(ipred_cfl_128_tbl) - L(ipred_cfl_splat_w8) + .hword L(ipred_cfl_128_tbl) - L(ipred_cfl_splat_w4) +endfunc + +// void ipred_cfl_top_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha, +// const int bitdepth_max); +function ipred_cfl_top_16bpc_neon, export=1 + dup v31.8h, w7 // bitdepth_max + clz w9, w3 + adr x7, L(ipred_cfl_top_tbl) + sub w9, w9, #26 + ldrh w9, [x7, w9, uxtw #1] + dup v1.8h, w6 // alpha + add x2, x2, #2 + sub x7, x7, w9, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + movi v30.8h, #0 + br x7 +4: + ld1 {v0.4h}, [x2] + addv h0, v0.4h + urshr v0.4h, v0.4h, #2 + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w4) +8: + ld1 {v0.8h}, [x2] + addv h0, v0.8h + urshr v0.4h, v0.4h, #3 + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w8) +16: + ld1 {v2.8h, v3.8h}, [x2] + addp v0.8h, v2.8h, v3.8h + addv h0, v0.8h + urshr v0.4h, v0.4h, #4 + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w16) +32: + ld1 {v2.8h, v3.8h, v4.8h, v5.8h}, [x2] + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v0.8h, v2.8h, v4.8h + uaddlv s0, v0.8h + rshrn v0.4h, v0.4s, #5 + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w16) + +L(ipred_cfl_top_tbl): + .hword L(ipred_cfl_top_tbl) - 32b + .hword L(ipred_cfl_top_tbl) - 16b + .hword L(ipred_cfl_top_tbl) - 8b + .hword L(ipred_cfl_top_tbl) - 4b +endfunc + +// void ipred_cfl_left_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha, +// const int bitdepth_max); +function ipred_cfl_left_16bpc_neon, export=1 + dup v31.8h, w7 // bitdepth_max + sub x2, x2, w4, uxtw #1 + clz w9, w3 + clz w8, w4 + adr x10, L(ipred_cfl_splat_tbl) + adr x7, L(ipred_cfl_left_tbl) + sub w9, w9, #26 + sub w8, w8, #26 + ldrh w9, [x10, w9, uxtw #1] + ldrh w8, [x7, w8, uxtw #1] + dup v1.8h, w6 // alpha + sub x9, x10, w9, uxtw + sub x7, x7, w8, uxtw + add x6, x0, x1 + lsl x1, x1, #1 + movi v30.8h, #0 + br x7 + +L(ipred_cfl_left_h4): + ld1 {v0.4h}, [x2] + addv h0, v0.4h + urshr v0.4h, v0.4h, #2 + dup v0.8h, v0.h[0] + br x9 + +L(ipred_cfl_left_h8): + ld1 {v0.8h}, [x2] + addv h0, v0.8h + urshr v0.4h, v0.4h, #3 + dup v0.8h, v0.h[0] + br x9 + +L(ipred_cfl_left_h16): + ld1 {v2.8h, v3.8h}, [x2] + addp v0.8h, v2.8h, v3.8h + addv h0, v0.8h + urshr v0.4h, v0.4h, #4 + dup v0.8h, v0.h[0] + br x9 + +L(ipred_cfl_left_h32): + ld1 {v2.8h, v3.8h, v4.8h, v5.8h}, [x2] + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v0.8h, v2.8h, v4.8h + uaddlv s0, v0.8h + rshrn v0.4h, v0.4s, #5 + dup v0.8h, v0.h[0] + br x9 + +L(ipred_cfl_left_tbl): + .hword L(ipred_cfl_left_tbl) - L(ipred_cfl_left_h32) + .hword L(ipred_cfl_left_tbl) - L(ipred_cfl_left_h16) + .hword L(ipred_cfl_left_tbl) - L(ipred_cfl_left_h8) + .hword L(ipred_cfl_left_tbl) - L(ipred_cfl_left_h4) +endfunc + +// void ipred_cfl_16bpc_neon(pixel *dst, const ptrdiff_t stride, +// const pixel *const topleft, +// const int width, const int height, +// const int16_t *ac, const int alpha, +// const int bitdepth_max); +function ipred_cfl_16bpc_neon, export=1 + dup v31.8h, w7 // bitdepth_max + sub x2, x2, w4, uxtw #1 + add w8, w3, w4 // width + height + dup v1.8h, w6 // alpha + clz w9, w3 + clz w6, w4 + dup v16.4s, w8 // width + height + adr x7, L(ipred_cfl_tbl) + rbit w8, w8 // rbit(width + height) + sub w9, w9, #22 // 26 leading bits, minus table offset 4 + sub w6, w6, #26 + clz w8, w8 // ctz(width + height) + ldrh w9, [x7, w9, uxtw #1] + ldrh w6, [x7, w6, uxtw #1] + neg w8, w8 // -ctz(width + height) + sub x9, x7, w9, uxtw + sub x7, x7, w6, uxtw + ushr v16.4s, v16.4s, #1 // (width + height) >> 1 + dup v17.4s, w8 // -ctz(width + height) + add x6, x0, x1 + lsl x1, x1, #1 + movi v30.8h, #0 + br x7 + +L(ipred_cfl_h4): + ld1 {v0.4h}, [x2], #8 + uaddlv s0, v0.4h + br x9 +L(ipred_cfl_w4): + add x2, x2, #2 + ld1 {v2.4h}, [x2] + add v0.2s, v0.2s, v16.2s + uaddlv s2, v2.4h + cmp w4, #4 + add v0.2s, v0.2s, v2.2s + ushl v0.2s, v0.2s, v17.2s + b.eq 1f + // h = 8/16 + cmp w4, #16 + mov w16, #0x6667 + mov w17, #0xAAAB + csel w16, w16, w17, eq + dup v16.2s, w16 + mul v0.2s, v0.2s, v16.2s + ushr v0.2s, v0.2s, #17 +1: + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w4) + +L(ipred_cfl_h8): + ld1 {v0.8h}, [x2], #16 + uaddlv s0, v0.8h + br x9 +L(ipred_cfl_w8): + add x2, x2, #2 + ld1 {v2.8h}, [x2] + add v0.2s, v0.2s, v16.2s + uaddlv s2, v2.8h + cmp w4, #8 + add v0.2s, v0.2s, v2.2s + ushl v0.2s, v0.2s, v17.2s + b.eq 1f + // h = 4/16/32 + cmp w4, #32 + mov w16, #0x6667 + mov w17, #0xAAAB + csel w16, w16, w17, eq + dup v16.2s, w16 + mul v0.2s, v0.2s, v16.2s + ushr v0.2s, v0.2s, #17 +1: + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w8) + +L(ipred_cfl_h16): + ld1 {v2.8h, v3.8h}, [x2], #32 + addp v0.8h, v2.8h, v3.8h + uaddlv s0, v0.8h + br x9 +L(ipred_cfl_w16): + add x2, x2, #2 + ld1 {v2.8h, v3.8h}, [x2] + add v0.2s, v0.2s, v16.2s + addp v2.8h, v2.8h, v3.8h + uaddlv s2, v2.8h + cmp w4, #16 + add v0.2s, v0.2s, v2.2s + ushl v0.2s, v0.2s, v17.2s + b.eq 1f + // h = 4/8/32 + tst w4, #(32+16+8) // 16 added to make a consecutive bitmask + mov w16, #0x6667 + mov w17, #0xAAAB + csel w16, w16, w17, eq + dup v16.2s, w16 + mul v0.2s, v0.2s, v16.2s + ushr v0.2s, v0.2s, #17 +1: + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w16) + +L(ipred_cfl_h32): + ld1 {v2.8h, v3.8h, v4.8h, v5.8h}, [x2], #64 + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v0.8h, v2.8h, v4.8h + uaddlv s0, v0.8h + br x9 +L(ipred_cfl_w32): + add x2, x2, #2 + ld1 {v2.8h, v3.8h, v4.8h, v5.8h}, [x2] + add v0.4s, v0.4s, v16.4s + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v2.8h, v2.8h, v4.8h + cmp w4, #32 + uaddlv s2, v2.8h + add v0.2s, v0.2s, v2.2s + ushl v0.2s, v0.2s, v17.2s + b.eq 1f + // h = 8/16 + cmp w4, #8 + mov w16, #0x6667 + mov w17, #0xAAAB + csel w16, w16, w17, eq + dup v16.2s, w16 + mul v0.2s, v0.2s, v16.2s + ushr v0.2s, v0.2s, #17 +1: + dup v0.8h, v0.h[0] + b L(ipred_cfl_splat_w16) + +L(ipred_cfl_tbl): + .hword L(ipred_cfl_tbl) - L(ipred_cfl_h32) + .hword L(ipred_cfl_tbl) - L(ipred_cfl_h16) + .hword L(ipred_cfl_tbl) - L(ipred_cfl_h8) + .hword L(ipred_cfl_tbl) - L(ipred_cfl_h4) + .hword L(ipred_cfl_tbl) - L(ipred_cfl_w32) + .hword L(ipred_cfl_tbl) - L(ipred_cfl_w16) + .hword L(ipred_cfl_tbl) - L(ipred_cfl_w8) + .hword L(ipred_cfl_tbl) - L(ipred_cfl_w4) +endfunc + +// void cfl_ac_420_16bpc_neon(int16_t *const ac, const pixel *const ypx, +// const ptrdiff_t stride, const int w_pad, +// const int h_pad, const int cw, const int ch); +function ipred_cfl_ac_420_16bpc_neon, export=1 + clz w8, w5 + lsl w4, w4, #2 + adr x7, L(ipred_cfl_ac_420_tbl) + sub w8, w8, #27 + ldrh w8, [x7, w8, uxtw #1] + movi v24.4s, #0 + movi v25.4s, #0 + movi v26.4s, #0 + movi v27.4s, #0 + sub x7, x7, w8, uxtw + sub w8, w6, w4 // height - h_pad + rbit w9, w5 // rbit(width) + rbit w10, w6 // rbit(height) + clz w9, w9 // ctz(width) + clz w10, w10 // ctz(height) + add w9, w9, w10 // log2sz + add x10, x1, x2 + dup v31.4s, w9 + lsl x2, x2, #1 + neg v31.4s, v31.4s // -log2sz + br x7 + +L(ipred_cfl_ac_420_w4): +1: // Copy and subsample input + ld1 {v0.8h}, [x1], x2 + ld1 {v1.8h}, [x10], x2 + ld1 {v2.8h}, [x1], x2 + ld1 {v3.8h}, [x10], x2 + addp v0.8h, v0.8h, v2.8h + addp v1.8h, v1.8h, v3.8h + add v0.8h, v0.8h, v1.8h + shl v0.8h, v0.8h, #1 + subs w8, w8, #2 + st1 {v0.8h}, [x0], #16 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + b.gt 1b + trn2 v1.2d, v0.2d, v0.2d + trn2 v0.2d, v0.2d, v0.2d +L(ipred_cfl_ac_420_w4_hpad): + cbz w4, 3f +2: // Vertical padding (h_pad > 0) + subs w4, w4, #4 + st1 {v0.8h, v1.8h}, [x0], #32 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + b.gt 2b +3: +L(ipred_cfl_ac_420_w4_calc_subtract_dc): + // Aggregate the sums + add v24.4s, v24.4s, v25.4s + add v26.4s, v26.4s, v27.4s + add v0.4s, v24.4s, v26.4s + addv s0, v0.4s // sum + sub x0, x0, w6, uxtw #3 + urshl v4.2s, v0.2s, v31.2s // (sum + (1 << (log2sz - 1))) >>= log2sz + dup v4.8h, v4.h[0] +6: // Subtract dc from ac + ld1 {v0.8h, v1.8h}, [x0] + subs w6, w6, #4 + sub v0.8h, v0.8h, v4.8h + sub v1.8h, v1.8h, v4.8h + st1 {v0.8h, v1.8h}, [x0], #32 + b.gt 6b + ret + +L(ipred_cfl_ac_420_w8): + cbnz w3, L(ipred_cfl_ac_420_w8_wpad) +1: // Copy and subsample input, without padding + ld1 {v0.8h, v1.8h}, [x1], x2 + ld1 {v2.8h, v3.8h}, [x10], x2 + ld1 {v4.8h, v5.8h}, [x1], x2 + addp v0.8h, v0.8h, v1.8h + ld1 {v6.8h, v7.8h}, [x10], x2 + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v6.8h, v6.8h, v7.8h + add v0.8h, v0.8h, v2.8h + add v4.8h, v4.8h, v6.8h + shl v0.8h, v0.8h, #1 + shl v1.8h, v4.8h, #1 + subs w8, w8, #2 + st1 {v0.8h, v1.8h}, [x0], #32 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + b.gt 1b + mov v0.16b, v1.16b + b L(ipred_cfl_ac_420_w8_hpad) + +L(ipred_cfl_ac_420_w8_wpad): +1: // Copy and subsample input, padding 4 + ld1 {v0.8h}, [x1], x2 + ld1 {v1.8h}, [x10], x2 + ld1 {v2.8h}, [x1], x2 + ld1 {v3.8h}, [x10], x2 + addp v0.8h, v0.8h, v2.8h + addp v1.8h, v1.8h, v3.8h + add v0.8h, v0.8h, v1.8h + shl v0.8h, v0.8h, #1 + dup v1.4h, v0.h[3] + dup v3.4h, v0.h[7] + trn2 v2.2d, v0.2d, v0.2d + subs w8, w8, #2 + st1 {v0.4h, v1.4h, v2.4h, v3.4h}, [x0], #32 + uaddw v24.4s, v24.4s, v0.4h + uaddw v25.4s, v25.4s, v1.4h + uaddw v26.4s, v26.4s, v2.4h + uaddw v27.4s, v27.4s, v3.4h + b.gt 1b + trn1 v0.2d, v2.2d, v3.2d + trn1 v1.2d, v2.2d, v3.2d + +L(ipred_cfl_ac_420_w8_hpad): + cbz w4, 3f +2: // Vertical padding (h_pad > 0) + subs w4, w4, #4 + st1 {v0.8h, v1.8h}, [x0], #32 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + st1 {v0.8h, v1.8h}, [x0], #32 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + b.gt 2b +3: + + // Double the height and reuse the w4 summing/subtracting + lsl w6, w6, #1 + b L(ipred_cfl_ac_420_w4_calc_subtract_dc) + +L(ipred_cfl_ac_420_w16): + adr x7, L(ipred_cfl_ac_420_w16_tbl) + ldrh w3, [x7, w3, uxtw #1] + sub x7, x7, w3, uxtw + br x7 + +L(ipred_cfl_ac_420_w16_wpad0): +1: // Copy and subsample input, without padding + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x1], x2 + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x10], x2 + addp v0.8h, v0.8h, v1.8h + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v6.8h, v6.8h, v7.8h + ld1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x1], x2 + add v0.8h, v0.8h, v4.8h + ld1 {v20.8h, v21.8h, v22.8h, v23.8h}, [x10], x2 + add v2.8h, v2.8h, v6.8h + addp v16.8h, v16.8h, v17.8h + addp v18.8h, v18.8h, v19.8h + addp v20.8h, v20.8h, v21.8h + addp v22.8h, v22.8h, v23.8h + add v16.8h, v16.8h, v20.8h + add v18.8h, v18.8h, v22.8h + shl v0.8h, v0.8h, #1 + shl v1.8h, v2.8h, #1 + shl v2.8h, v16.8h, #1 + shl v3.8h, v18.8h, #1 + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_420_w16_wpad1): +1: // Copy and subsample input, padding 4 + ldr q2, [x1, #32] + ld1 {v0.8h, v1.8h}, [x1], x2 + ldr q5, [x10, #32] + ld1 {v3.8h, v4.8h}, [x10], x2 + addp v2.8h, v2.8h, v2.8h + addp v0.8h, v0.8h, v1.8h + addp v5.8h, v5.8h, v5.8h + addp v3.8h, v3.8h, v4.8h + ldr q18, [x1, #32] + add v2.4h, v2.4h, v5.4h + ld1 {v16.8h, v17.8h}, [x1], x2 + add v0.8h, v0.8h, v3.8h + ldr q21, [x10, #32] + ld1 {v19.8h, v20.8h}, [x10], x2 + addp v18.8h, v18.8h, v18.8h + addp v16.8h, v16.8h, v17.8h + addp v21.8h, v21.8h, v21.8h + addp v19.8h, v19.8h, v20.8h + add v18.4h, v18.4h, v21.4h + add v16.8h, v16.8h, v19.8h + shl v1.4h, v2.4h, #1 + shl v0.8h, v0.8h, #1 + shl v3.4h, v18.4h, #1 + shl v2.8h, v16.8h, #1 + dup v4.4h, v1.h[3] + dup v5.4h, v3.h[3] + trn1 v1.2d, v1.2d, v4.2d + trn1 v3.2d, v3.2d, v5.2d + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_420_w16_wpad2): +1: // Copy and subsample input, padding 8 + ld1 {v0.8h, v1.8h}, [x1], x2 + ld1 {v2.8h, v3.8h}, [x10], x2 + ld1 {v4.8h, v5.8h}, [x1], x2 + addp v0.8h, v0.8h, v1.8h + ld1 {v6.8h, v7.8h}, [x10], x2 + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v6.8h, v6.8h, v7.8h + add v0.8h, v0.8h, v2.8h + add v4.8h, v4.8h, v6.8h + shl v0.8h, v0.8h, #1 + shl v2.8h, v4.8h, #1 + dup v1.8h, v0.h[7] + dup v3.8h, v2.h[7] + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_420_w16_wpad3): +1: // Copy and subsample input, padding 12 + ld1 {v0.8h}, [x1], x2 + ld1 {v2.8h}, [x10], x2 + ld1 {v4.8h}, [x1], x2 + ld1 {v6.8h}, [x10], x2 + addp v0.8h, v0.8h, v4.8h + addp v2.8h, v2.8h, v6.8h + add v0.8h, v0.8h, v2.8h + shl v0.8h, v0.8h, #1 + dup v1.8h, v0.h[3] + dup v3.8h, v0.h[7] + trn2 v2.2d, v0.2d, v3.2d + trn1 v0.2d, v0.2d, v1.2d + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + +L(ipred_cfl_ac_420_w16_hpad): + cbz w4, 3f +2: // Vertical padding (h_pad > 0) + subs w4, w4, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 2b +3: + + // Quadruple the height and reuse the w4 summing/subtracting + lsl w6, w6, #2 + b L(ipred_cfl_ac_420_w4_calc_subtract_dc) + +L(ipred_cfl_ac_420_tbl): + .hword L(ipred_cfl_ac_420_tbl) - L(ipred_cfl_ac_420_w16) + .hword L(ipred_cfl_ac_420_tbl) - L(ipred_cfl_ac_420_w8) + .hword L(ipred_cfl_ac_420_tbl) - L(ipred_cfl_ac_420_w4) + .hword 0 + +L(ipred_cfl_ac_420_w16_tbl): + .hword L(ipred_cfl_ac_420_w16_tbl) - L(ipred_cfl_ac_420_w16_wpad0) + .hword L(ipred_cfl_ac_420_w16_tbl) - L(ipred_cfl_ac_420_w16_wpad1) + .hword L(ipred_cfl_ac_420_w16_tbl) - L(ipred_cfl_ac_420_w16_wpad2) + .hword L(ipred_cfl_ac_420_w16_tbl) - L(ipred_cfl_ac_420_w16_wpad3) +endfunc + +// void cfl_ac_422_16bpc_neon(int16_t *const ac, const pixel *const ypx, +// const ptrdiff_t stride, const int w_pad, +// const int h_pad, const int cw, const int ch); +function ipred_cfl_ac_422_16bpc_neon, export=1 + clz w8, w5 + lsl w4, w4, #2 + adr x7, L(ipred_cfl_ac_422_tbl) + sub w8, w8, #27 + ldrh w8, [x7, w8, uxtw #1] + movi v24.4s, #0 + movi v25.4s, #0 + movi v26.4s, #0 + movi v27.4s, #0 + sub x7, x7, w8, uxtw + sub w8, w6, w4 // height - h_pad + rbit w9, w5 // rbit(width) + rbit w10, w6 // rbit(height) + clz w9, w9 // ctz(width) + clz w10, w10 // ctz(height) + add w9, w9, w10 // log2sz + add x10, x1, x2 + dup v31.4s, w9 + lsl x2, x2, #1 + neg v31.4s, v31.4s // -log2sz + br x7 + +L(ipred_cfl_ac_422_w4): +1: // Copy and subsample input + ld1 {v0.8h}, [x1], x2 + ld1 {v1.8h}, [x10], x2 + ld1 {v2.8h}, [x1], x2 + ld1 {v3.8h}, [x10], x2 + addp v0.8h, v0.8h, v1.8h + addp v2.8h, v2.8h, v3.8h + shl v0.8h, v0.8h, #2 + shl v1.8h, v2.8h, #2 + subs w8, w8, #4 + st1 {v0.8h, v1.8h}, [x0], #32 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + b.gt 1b + trn2 v0.2d, v1.2d, v1.2d + trn2 v1.2d, v1.2d, v1.2d + b L(ipred_cfl_ac_420_w4_hpad) + +L(ipred_cfl_ac_422_w8): + cbnz w3, L(ipred_cfl_ac_422_w8_wpad) +1: // Copy and subsample input, without padding + ld1 {v0.8h, v1.8h}, [x1], x2 + ld1 {v2.8h, v3.8h}, [x10], x2 + ld1 {v4.8h, v5.8h}, [x1], x2 + addp v0.8h, v0.8h, v1.8h + ld1 {v6.8h, v7.8h}, [x10], x2 + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v6.8h, v6.8h, v7.8h + shl v0.8h, v0.8h, #2 + shl v1.8h, v2.8h, #2 + shl v2.8h, v4.8h, #2 + shl v3.8h, v6.8h, #2 + subs w8, w8, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v3.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w8_hpad) + +L(ipred_cfl_ac_422_w8_wpad): +1: // Copy and subsample input, padding 4 + ld1 {v0.8h}, [x1], x2 + ld1 {v1.8h}, [x10], x2 + ld1 {v2.8h}, [x1], x2 + ld1 {v3.8h}, [x10], x2 + addp v0.8h, v0.8h, v1.8h + addp v2.8h, v2.8h, v3.8h + shl v0.8h, v0.8h, #2 + shl v2.8h, v2.8h, #2 + dup v4.4h, v0.h[3] + dup v5.8h, v0.h[7] + dup v6.4h, v2.h[3] + dup v7.8h, v2.h[7] + trn2 v1.2d, v0.2d, v5.2d + trn1 v0.2d, v0.2d, v4.2d + trn2 v3.2d, v2.2d, v7.2d + trn1 v2.2d, v2.2d, v6.2d + subs w8, w8, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v3.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w8_hpad) + +L(ipred_cfl_ac_422_w16): + adr x7, L(ipred_cfl_ac_422_w16_tbl) + ldrh w3, [x7, w3, uxtw #1] + sub x7, x7, w3, uxtw + br x7 + +L(ipred_cfl_ac_422_w16_wpad0): +1: // Copy and subsample input, without padding + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x1], x2 + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x10], x2 + addp v0.8h, v0.8h, v1.8h + addp v2.8h, v2.8h, v3.8h + addp v4.8h, v4.8h, v5.8h + addp v6.8h, v6.8h, v7.8h + shl v0.8h, v0.8h, #2 + shl v1.8h, v2.8h, #2 + shl v2.8h, v4.8h, #2 + shl v3.8h, v6.8h, #2 + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_422_w16_wpad1): +1: // Copy and subsample input, padding 4 + ldr q2, [x1, #32] + ld1 {v0.8h, v1.8h}, [x1], x2 + ldr q6, [x10, #32] + ld1 {v4.8h, v5.8h}, [x10], x2 + addp v2.8h, v2.8h, v2.8h + addp v0.8h, v0.8h, v1.8h + addp v6.8h, v6.8h, v6.8h + addp v4.8h, v4.8h, v5.8h + shl v1.4h, v2.4h, #2 + shl v0.8h, v0.8h, #2 + shl v3.4h, v6.4h, #2 + shl v2.8h, v4.8h, #2 + dup v4.4h, v1.h[3] + dup v5.4h, v3.h[3] + trn1 v1.2d, v1.2d, v4.2d + trn1 v3.2d, v3.2d, v5.2d + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_422_w16_wpad2): +1: // Copy and subsample input, padding 8 + ld1 {v0.8h, v1.8h}, [x1], x2 + ld1 {v2.8h, v3.8h}, [x10], x2 + addp v0.8h, v0.8h, v1.8h + addp v2.8h, v2.8h, v3.8h + shl v0.8h, v0.8h, #2 + shl v2.8h, v2.8h, #2 + dup v1.8h, v0.h[7] + dup v3.8h, v2.h[7] + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_422_w16_wpad3): +1: // Copy and subsample input, padding 12 + ld1 {v0.8h}, [x1], x2 + ld1 {v2.8h}, [x10], x2 + addp v0.8h, v0.8h, v0.8h + addp v2.8h, v2.8h, v2.8h + shl v0.4h, v0.4h, #2 + shl v2.4h, v2.4h, #2 + dup v1.8h, v0.h[3] + dup v3.8h, v2.h[3] + trn1 v0.2d, v0.2d, v1.2d + trn1 v2.2d, v2.2d, v3.2d + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_422_tbl): + .hword L(ipred_cfl_ac_422_tbl) - L(ipred_cfl_ac_422_w16) + .hword L(ipred_cfl_ac_422_tbl) - L(ipred_cfl_ac_422_w8) + .hword L(ipred_cfl_ac_422_tbl) - L(ipred_cfl_ac_422_w4) + .hword 0 + +L(ipred_cfl_ac_422_w16_tbl): + .hword L(ipred_cfl_ac_422_w16_tbl) - L(ipred_cfl_ac_422_w16_wpad0) + .hword L(ipred_cfl_ac_422_w16_tbl) - L(ipred_cfl_ac_422_w16_wpad1) + .hword L(ipred_cfl_ac_422_w16_tbl) - L(ipred_cfl_ac_422_w16_wpad2) + .hword L(ipred_cfl_ac_422_w16_tbl) - L(ipred_cfl_ac_422_w16_wpad3) +endfunc + +// void cfl_ac_444_16bpc_neon(int16_t *const ac, const pixel *const ypx, +// const ptrdiff_t stride, const int w_pad, +// const int h_pad, const int cw, const int ch); +function ipred_cfl_ac_444_16bpc_neon, export=1 + clz w8, w5 + lsl w4, w4, #2 + adr x7, L(ipred_cfl_ac_444_tbl) + sub w8, w8, #26 + ldrh w8, [x7, w8, uxtw #1] + movi v24.4s, #0 + movi v25.4s, #0 + movi v26.4s, #0 + movi v27.4s, #0 + sub x7, x7, w8, uxtw + sub w8, w6, w4 // height - h_pad + rbit w9, w5 // rbit(width) + rbit w10, w6 // rbit(height) + clz w9, w9 // ctz(width) + clz w10, w10 // ctz(height) + add w9, w9, w10 // log2sz + add x10, x1, x2 + dup v31.4s, w9 + lsl x2, x2, #1 + neg v31.4s, v31.4s // -log2sz + br x7 + +L(ipred_cfl_ac_444_w4): +1: // Copy and expand input + ld1 {v0.4h}, [x1], x2 + ld1 {v0.d}[1], [x10], x2 + ld1 {v1.4h}, [x1], x2 + ld1 {v1.d}[1], [x10], x2 + shl v0.8h, v0.8h, #3 + shl v1.8h, v1.8h, #3 + subs w8, w8, #4 + st1 {v0.8h, v1.8h}, [x0], #32 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + b.gt 1b + trn2 v0.2d, v1.2d, v1.2d + trn2 v1.2d, v1.2d, v1.2d + b L(ipred_cfl_ac_420_w4_hpad) + +L(ipred_cfl_ac_444_w8): +1: // Copy and expand input + ld1 {v0.8h}, [x1], x2 + ld1 {v1.8h}, [x10], x2 + ld1 {v2.8h}, [x1], x2 + shl v0.8h, v0.8h, #3 + ld1 {v3.8h}, [x10], x2 + shl v1.8h, v1.8h, #3 + shl v2.8h, v2.8h, #3 + shl v3.8h, v3.8h, #3 + subs w8, w8, #4 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v3.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w8_hpad) + +L(ipred_cfl_ac_444_w16): + cbnz w3, L(ipred_cfl_ac_444_w16_wpad) +1: // Copy and expand input, without padding + ld1 {v0.8h, v1.8h}, [x1], x2 + ld1 {v2.8h, v3.8h}, [x10], x2 + shl v0.8h, v0.8h, #3 + shl v1.8h, v1.8h, #3 + shl v2.8h, v2.8h, #3 + shl v3.8h, v3.8h, #3 + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_444_w16_wpad): +1: // Copy and expand input, padding 8 + ld1 {v0.8h}, [x1], x2 + ld1 {v2.8h}, [x10], x2 + shl v0.8h, v0.8h, #3 + shl v2.8h, v2.8h, #3 + dup v1.8h, v0.h[7] + dup v3.8h, v2.h[7] + subs w8, w8, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + mov v0.16b, v2.16b + mov v1.16b, v3.16b + b L(ipred_cfl_ac_420_w16_hpad) + +L(ipred_cfl_ac_444_w32): + adr x7, L(ipred_cfl_ac_444_w32_tbl) + ldrh w3, [x7, w3, uxtw] // (w3>>1) << 1 + lsr x2, x2, #1 // Restore the stride to one line increments + sub x7, x7, w3, uxtw + br x7 + +L(ipred_cfl_ac_444_w32_wpad0): +1: // Copy and expand input, without padding + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x1], x2 + shl v0.8h, v0.8h, #3 + shl v1.8h, v1.8h, #3 + shl v2.8h, v2.8h, #3 + shl v3.8h, v3.8h, #3 + subs w8, w8, #1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + b L(ipred_cfl_ac_444_w32_hpad) + +L(ipred_cfl_ac_444_w32_wpad2): +1: // Copy and expand input, padding 8 + ld1 {v0.8h, v1.8h, v2.8h}, [x1], x2 + shl v2.8h, v2.8h, #3 + shl v0.8h, v0.8h, #3 + shl v1.8h, v1.8h, #3 + dup v3.8h, v2.h[7] + subs w8, w8, #1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + b L(ipred_cfl_ac_444_w32_hpad) + +L(ipred_cfl_ac_444_w32_wpad4): +1: // Copy and expand input, padding 16 + ld1 {v0.8h, v1.8h}, [x1], x2 + shl v1.8h, v1.8h, #3 + shl v0.8h, v0.8h, #3 + dup v2.8h, v1.h[7] + dup v3.8h, v1.h[7] + subs w8, w8, #1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + b L(ipred_cfl_ac_444_w32_hpad) + +L(ipred_cfl_ac_444_w32_wpad6): +1: // Copy and expand input, padding 24 + ld1 {v0.8h}, [x1], x2 + shl v0.8h, v0.8h, #3 + dup v1.8h, v0.h[7] + dup v2.8h, v0.h[7] + dup v3.8h, v0.h[7] + subs w8, w8, #1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 1b + +L(ipred_cfl_ac_444_w32_hpad): + cbz w4, 3f +2: // Vertical padding (h_pad > 0) + subs w4, w4, #2 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + uaddw v24.4s, v24.4s, v0.4h + uaddw2 v25.4s, v25.4s, v0.8h + uaddw v26.4s, v26.4s, v1.4h + uaddw2 v27.4s, v27.4s, v1.8h + uaddw v24.4s, v24.4s, v2.4h + uaddw2 v25.4s, v25.4s, v2.8h + uaddw v26.4s, v26.4s, v3.4h + uaddw2 v27.4s, v27.4s, v3.8h + b.gt 2b +3: + + // Multiply the height by eight and reuse the w4 subtracting + lsl w6, w6, #3 + b L(ipred_cfl_ac_420_w4_calc_subtract_dc) + +L(ipred_cfl_ac_444_tbl): + .hword L(ipred_cfl_ac_444_tbl) - L(ipred_cfl_ac_444_w32) + .hword L(ipred_cfl_ac_444_tbl) - L(ipred_cfl_ac_444_w16) + .hword L(ipred_cfl_ac_444_tbl) - L(ipred_cfl_ac_444_w8) + .hword L(ipred_cfl_ac_444_tbl) - L(ipred_cfl_ac_444_w4) + +L(ipred_cfl_ac_444_w32_tbl): + .hword L(ipred_cfl_ac_444_w32_tbl) - L(ipred_cfl_ac_444_w32_wpad0) + .hword L(ipred_cfl_ac_444_w32_tbl) - L(ipred_cfl_ac_444_w32_wpad2) + .hword L(ipred_cfl_ac_444_w32_tbl) - L(ipred_cfl_ac_444_w32_wpad4) + .hword L(ipred_cfl_ac_444_w32_tbl) - L(ipred_cfl_ac_444_w32_wpad6) +endfunc diff --git a/third_party/dav1d/src/arm/64/itx.S b/third_party/dav1d/src/arm/64/itx.S --- a/third_party/dav1d/src/arm/64/itx.S +++ b/third_party/dav1d/src/arm/64/itx.S @@ -56,11 +56,10 @@ // - Use a simplified version of the transforms themselves for cases where // we know a significant number of inputs are zero. E.g. if the eob value // indicates only a quarter of input values are set, for idct16 and up, // a significant amount of calculation can be skipped, at the cost of more // code duplication and special casing. -// - Special case functions for e.g. more combinations with identity. const idct_coeffs, align=4 // idct4 .short 2896, 2896*8, 1567, 3784 // idct8 @@ -104,11 +103,11 @@ const iadst8_coeffs, align=4 .short 4076, 401, 3612, 1931 .short 2598, 3166, 1189, 3920 // idct_coeffs - .short 2896, 2896*8, 1567, 3784, 0, 0, 0, 0 + .short 2896, 0, 1567, 3784, 0, 0, 0, 0 endconst const iadst16_coeffs, align=4 .short 4091, 201, 3973, 995 .short 3703, 1751, 3290, 2440 @@ -132,17 +131,10 @@ smull2 \d1\().4s, \s0\().8h, \c0 smlsl2 \d1\().4s, \s1\().8h, \c1 .endif .endm -.macro smull_sz d0, d1, s0, c, sz - smull \d0\().4s, \s0\().4h, \c -.ifc \sz, .8h - smull2 \d1\().4s, \s0\().8h, \c -.endif -.endm - .macro rshrn_sz d0, s0, s1, shift, sz rshrn \d0\().4h, \s0\().4s, \shift .ifc \sz, .8h rshrn2 \d0\().8h, \s1\().4s, \shift .endif @@ -455,18 +447,18 @@ sqsub \r3\sz, v2\sz, v6\sz sqadd \r1\sz, v3\sz, v7\sz sqsub \r2\sz, v3\sz, v7\sz .endm -function inv_dct_4x4_neon +function inv_dct_4h_x4_neon, export=1 movrel x16, idct_coeffs ld1 {v0.4h}, [x16] idct_4 v16, v17, v18, v19, .4h ret endfunc -function inv_dct_8x4_neon +function inv_dct_8h_x4_neon, export=1 movrel x16, idct_coeffs ld1 {v0.4h}, [x16] idct_4 v16, v17, v18, v19, .8h ret endfunc @@ -495,16 +487,16 @@ rshrn \o2\().4h, \o2\().4s, #12 rshrn \o1\().4h, \o1\().4s, #12 rshrn \o3\().4h, \o3\().4s, #12 .endm -function inv_adst_4x4_neon +function inv_adst_4h_x4_neon, export=1 iadst_4x4 v16, v17, v18, v19 ret endfunc -function inv_flipadst_4x4_neon +function inv_flipadst_4h_x4_neon, export=1 iadst_4x4 v19, v18, v17, v16 ret endfunc .macro iadst_8x4 o0, o1, o2, o3 @@ -561,21 +553,21 @@ rshrn \o3\().4h, v4.4s, #12 rshrn2 \o3\().8h, v5.4s, #12 .endm -function inv_adst_8x4_neon +function inv_adst_8h_x4_neon, export=1 iadst_8x4 v16, v17, v18, v19 ret endfunc -function inv_flipadst_8x4_neon +function inv_flipadst_8h_x4_neon, export=1 iadst_8x4 v19, v18, v17, v16 ret endfunc -function inv_identity_4x4_neon +function inv_identity_4h_x4_neon, export=1 mov w16, #(5793-4096)*8 dup v0.4h, w16 sqrdmulh v4.4h, v16.4h, v0.h[0] sqrdmulh v5.4h, v17.4h, v0.h[0] sqrdmulh v6.4h, v18.4h, v0.h[0] @@ -585,11 +577,11 @@ sqadd v18.4h, v18.4h, v6.4h sqadd v19.4h, v19.4h, v7.4h ret endfunc -function inv_identity_8x4_neon +function inv_identity_8h_x4_neon, export=1 mov w16, #(5793-4096)*8 dup v0.4h, w16 sqrdmulh v4.8h, v16.8h, v0.h[0] sqrdmulh v5.8h, v17.8h, v0.h[0] sqrdmulh v6.8h, v18.8h, v0.h[0] @@ -606,11 +598,11 @@ sqrdmulh v2.8h, \i, \c srhadd \i, \i, v2.8h .endr .endm -function inv_txfm_add_wht_wht_4x4_neon, export=1 +function inv_txfm_add_wht_wht_4x4_8bpc_neon, export=1 mov x15, x30 movi v31.8h, #0 ld1 {v16.4h,v17.4h,v18.4h,v19.4h}, [x2] st1 {v31.8h}, [x2], #16 @@ -670,11 +662,11 @@ br x15 endfunc .macro def_fn_4x4 txfm1, txfm2 -function inv_txfm_add_\txfm1\()_\txfm2\()_4x4_neon, export=1 +function inv_txfm_add_\txfm1\()_\txfm2\()_4x4_8bpc_neon, export=1 mov x15, x30 .ifc \txfm1\()_\txfm2, dct_dct cbnz w3, 1f mov w16, #2896*8 @@ -690,12 +682,12 @@ srshr v18.8h, v20.8h, #4 ld1 {v1.s}[1], [x0], x1 b L(itx_4x4_end) 1: .endif - adr x4, inv_\txfm1\()_4x4_neon - adr x5, inv_\txfm2\()_4x4_neon + adr x4, inv_\txfm1\()_4h_x4_neon + adr x5, inv_\txfm2\()_4h_x4_neon b inv_txfm_add_4x4_neon endfunc .endm def_fn_4x4 dct, dct @@ -747,18 +739,18 @@ sqadd \r3\sz, \r6\sz, v2\sz // out3 sqsub \r4\sz, \r6\sz, v2\sz // out4 mov \r6\szb, v6\szb // out6 .endm -function inv_dct_8x8_neon +function inv_dct_8h_x8_neon, export=1 movrel x16, idct_coeffs ld1 {v0.8h}, [x16] idct_8 v16, v17, v18, v19, v20, v21, v22, v23, .8h, .16b ret endfunc -function inv_dct_4x8_neon +function inv_dct_4h_x8_neon, export=1 movrel x16, idct_coeffs ld1 {v0.8h}, [x16] idct_8 v16, v17, v18, v19, v20, v21, v22, v23, .4h, .8b ret endfunc @@ -828,31 +820,31 @@ sqneg \o3\()\sz, v2\sz // out3 sqneg \o5\()\sz, v3\sz // out5 .endm -function inv_adst_8x8_neon +function inv_adst_8h_x8_neon, export=1 iadst_8 v16, v17, v18, v19, v20, v21, v22, v23, .8h ret endfunc -function inv_flipadst_8x8_neon +function inv_flipadst_8h_x8_neon, export=1 iadst_8 v23, v22, v21, v20, v19, v18, v17, v16, .8h ret endfunc -function inv_adst_4x8_neon +function inv_adst_4h_x8_neon, export=1 iadst_8 v16, v17, v18, v19, v20, v21, v22, v23, .4h ret endfunc -function inv_flipadst_4x8_neon +function inv_flipadst_4h_x8_neon, export=1 iadst_8 v23, v22, v21, v20, v19, v18, v17, v16, .4h ret endfunc -function inv_identity_8x8_neon +function inv_identity_8h_x8_neon, export=1 sqshl v16.8h, v16.8h, #1 sqshl v17.8h, v17.8h, #1 sqshl v18.8h, v18.8h, #1 sqshl v19.8h, v19.8h, #1 sqshl v20.8h, v20.8h, #1 @@ -860,11 +852,11 @@ sqshl v22.8h, v22.8h, #1 sqshl v23.8h, v23.8h, #1 ret endfunc -function inv_identity_4x8_neon +function inv_identity_4h_x8_neon, export=1 sqshl v16.4h, v16.4h, #1 sqshl v17.4h, v17.4h, #1 sqshl v18.4h, v18.4h, #1 sqshl v19.4h, v19.4h, #1 sqshl v20.4h, v20.4h, #1 @@ -911,21 +903,21 @@ def_fn_8x8_base def_fn_8x8_base identity_ .macro def_fn_8x8 txfm1, txfm2 -function inv_txfm_add_\txfm1\()_\txfm2\()_8x8_neon, export=1 +function inv_txfm_add_\txfm1\()_\txfm2\()_8x8_8bpc_neon, export=1 mov x15, x30 .ifc \txfm1\()_\txfm2, dct_dct idct_dc 8, 8, 1 .endif - adr x5, inv_\txfm2\()_8x8_neon + adr x5, inv_\txfm2\()_8h_x8_neon .ifc \txfm1, identity b inv_txfm_identity_add_8x8_neon .else - adr x4, inv_\txfm1\()_8x8_neon + adr x4, inv_\txfm1\()_8h_x8_neon b inv_txfm_add_8x8_neon .endif endfunc .endm @@ -998,18 +990,18 @@ load_add_store_4x8 x0, x7 br x15 endfunc .macro def_fn_48 w, h, txfm1, txfm2 -function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_neon, export=1 +function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_8bpc_neon, export=1 mov x15, x30 .ifc \txfm1\()_\txfm2, dct_dct idct_dc \w, \h, 0 .endif - adr x4, inv_\txfm1\()_\h\()x\w\()_neon - adr x5, inv_\txfm2\()_\w\()x\h\()_neon + adr x4, inv_\txfm1\()_\h\()h_x\w\()_neon + adr x5, inv_\txfm2\()_\w\()h_x\h\()_neon b inv_txfm_add_\w\()x\h\()_neon endfunc .endm .macro def_fns_48 w, h @@ -1116,18 +1108,18 @@ sqsub v26\sz, v26\sz, v2\sz // out10 mov v24\szb, v7\szb mov v22\szb, v3\szb .endm -function inv_dct_8x16_neon +function inv_dct_8h_x16_neon, export=1 movrel x16, idct_coeffs ld1 {v0.8h, v1.8h}, [x16] idct_16 .8h, .16b ret endfunc -function inv_dct_4x16_neon +function inv_dct_4h_x16_neon, export=1 movrel x16, idct_coeffs ld1 {v0.8h, v1.8h}, [x16] idct_16 .4h, .8b ret endfunc @@ -1300,42 +1292,42 @@ sqneg \o5\sz, v5\sz // out5 sqneg \o11\sz, v6\sz // out11 sqneg \o9\sz, v7\sz // out9 .endm -function inv_adst_8x16_neon +function inv_adst_8h_x16_neon, export=1 iadst_16 v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, .8h, .16b ret endfunc -function inv_flipadst_8x16_neon +function inv_flipadst_8h_x16_neon, export=1 iadst_16 v31, v30, v29, v28, v27, v26, v25, v24, v23, v22, v21, v20, v19, v18, v17, v16, .8h, .16b ret endfunc -function inv_adst_4x16_neon +function inv_adst_4h_x16_neon, export=1 iadst_16 v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31, .4h, .8b ret endfunc -function inv_flipadst_4x16_neon +function inv_flipadst_4h_x16_neon, export=1 iadst_16 v31, v30, v29, v28, v27, v26, v25, v24, v23, v22, v21, v20, v19, v18, v17, v16, .4h, .8b ret endfunc -function inv_identity_8x16_neon +function inv_identity_8h_x16_neon, export=1 mov w16, #2*(5793-4096)*8 dup v0.4h, w16 .irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 sqrdmulh v2.8h, v\i\().8h, v0.h[0] sqadd v\i\().8h, v\i\().8h, v\i\().8h sqadd v\i\().8h, v\i\().8h, v2.8h .endr ret endfunc -function inv_identity_4x16_neon +function inv_identity_4h_x16_neon, export=1 mov w16, #2*(5793-4096)*8 dup v0.4h, w16 .irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 sqrdmulh v2.4h, v\i\().4h, v0.h[0] sqadd v\i\().4h, v\i\().4h, v\i\().4h @@ -1374,75 +1366,53 @@ sqadd \i, \i, \i sqadd \i, \i, v2.8h .endr .endm -function inv_txfm_horz_16x8_neon +.macro def_horz_16 scale=0, identity=0, shift=2, suffix +function inv_txfm_horz\suffix\()_16x8_neon mov x14, x30 movi v7.8h, #0 -.irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 - ld1 {v\i\().8h}, [x7] +.if \identity + mov w16, #2*(5793-4096)*8 + dup v0.4h, w16 +.elseif \scale + mov w16, #2896*8 + dup v0.4h, w16 +.endif +.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h, v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h + ld1 {\i}, [x7] st1 {v7.8h}, [x7], x8 .endr +.if \scale + scale_input .8h, v0.h[0], v16, v17, v18, v19, v20, v21, v22, v23 + scale_input .8h, v0.h[0], v24, v25, v26, v27, v28, v29, v30, v31 +.endif +.if \identity + identity_8x16_shift2 v0.h[0] +.else blr x4 -.irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 - srshr v\i\().8h, v\i\().8h, #2 +.endif +.if \shift > 0 +.irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h, v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h + srshr \i, \i, #\shift .endr +.endif transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v4, v5 transpose_8x8h v24, v25, v26, v27, v28, v29, v30, v31, v4, v5 -.irp i, 16, 24, 17, 25, 18, 26, 19, 27, 20, 28, 21, 29, 22, 30, 23, 31 - st1 {v\i\().8h}, [x6], #16 +.irp i, v16.8h, v24.8h, v17.8h, v25.8h, v18.8h, v26.8h, v19.8h, v27.8h, v20.8h, v28.8h, v21.8h, v29.8h, v22.8h, v30.8h, v23.8h, v31.8h + st1 {\i}, [x6], #16 .endr br x14 endfunc - -function inv_txfm_horz_identity_16x8_neon - mov x14, x30 - movi v7.8h, #0 -.irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 - ld1 {v\i\().8h}, [x7] - st1 {v7.8h}, [x7], x8 -.endr - mov w16, #2*(5793-4096)*8 - dup v0.4h, w16 - identity_8x16_shift2 v0.h[0] - transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v4, v5 - transpose_8x8h v24, v25, v26, v27, v28, v29, v30, v31, v4, v5 - -.irp i, 16, 24, 17, 25, 18, 26, 19, 27, 20, 28, 21, 29, 22, 30, 23, 31 - st1 {v\i\().8h}, [x6], #16 -.endr - - br x14 -endfunc - -function inv_txfm_horz_scale_16x8_neon - mov x14, x30 - movi v7.8h, #0 - mov w16, #2896*8 - dup v0.4h, w16 -.irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 - ld1 {v\i\().8h}, [x7] - st1 {v7.8h}, [x7], x8 -.endr - scale_input .8h, v0.h[0], v16, v17, v18, v19, v20, v21, v22, v23 - scale_input .8h, v0.h[0], v24, v25, v26, v27, v28, v29, v30, v31 - blr x4 -.irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 - srshr v\i\().8h, v\i\().8h, #1 -.endr - transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v4, v5 - transpose_8x8h v24, v25, v26, v27, v28, v29, v30, v31, v4, v5 - -.irp i, 16, 24, 17, 25, 18, 26, 19, 27, 20, 28, 21, 29, 22, 30, 23, 31 - st1 {v\i\().8h}, [x6], #16 -.endr - - br x14 -endfunc +.endm + +def_horz_16 scale=0, identity=0, shift=2 +def_horz_16 scale=1, identity=0, shift=1, suffix=_scale +def_horz_16 scale=0, identity=1, shift=0, suffix=_identity function inv_txfm_add_vert_8x16_neon mov x14, x30 .irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 ld1 {v\i\().8h}, [x7], x8 @@ -1485,21 +1455,21 @@ add sp, sp, #512 br x15 endfunc .macro def_fn_16x16 txfm1, txfm2, eob_half -function inv_txfm_add_\txfm1\()_\txfm2\()_16x16_neon, export=1 +function inv_txfm_add_\txfm1\()_\txfm2\()_16x16_8bpc_neon, export=1 .ifc \txfm1\()_\txfm2, dct_dct idct_dc 16, 16, 2 .endif .ifc \txfm1, identity adr x9, inv_txfm_horz_identity_16x8_neon .else adr x9, inv_txfm_horz_16x8_neon - adr x4, inv_\txfm1\()_8x16_neon + adr x4, inv_\txfm1\()_8h_x16_neon .endif - adr x5, inv_\txfm2\()_8x16_neon + adr x5, inv_\txfm2\()_8h_x16_neon mov x13, #\eob_half b inv_txfm_add_16x16_neon endfunc .endm @@ -1657,21 +1627,21 @@ def_fn_416_base def_fn_416_base identity_ .macro def_fn_416 w, h, txfm1, txfm2, eob_half -function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_neon, export=1 +function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_8bpc_neon, export=1 .ifc \txfm1\()_\txfm2, dct_dct idct_dc \w, \h, 1 .endif .if \w == 4 - adr x4, inv_\txfm1\()_8x\w\()_neon - adr x5, inv_\txfm2\()_4x\h\()_neon + adr x4, inv_\txfm1\()_8h_x\w\()_neon + adr x5, inv_\txfm2\()_4h_x\h\()_neon mov w13, #\eob_half .else - adr x4, inv_\txfm1\()_4x\w\()_neon - adr x5, inv_\txfm2\()_8x\h\()_neon + adr x4, inv_\txfm1\()_4h_x\w\()_neon + adr x5, inv_\txfm2\()_8h_x\h\()_neon .endif .ifc \txfm1, identity b inv_txfm_identity_add_\w\()x\h\()_neon .else b inv_txfm_add_\w\()x\h\()_neon @@ -1840,16 +1810,16 @@ def_fn_816_base def_fn_816_base identity_ .macro def_fn_816 w, h, txfm1, txfm2, eob_half -function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_neon, export=1 +function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_8bpc_neon, export=1 .ifc \txfm1\()_\txfm2, dct_dct idct_dc \w, \h, 1 .endif - adr x4, inv_\txfm1\()_8x\w\()_neon - adr x5, inv_\txfm2\()_8x\h\()_neon + adr x4, inv_\txfm1\()_8h_x\w\()_neon + adr x5, inv_\txfm2\()_8h_x\h\()_neon .if \w == 8 mov x13, #\eob_half .endif .ifc \txfm1, identity b inv_txfm_identity_add_\w\()x\h\()_neon @@ -1879,11 +1849,11 @@ .endm def_fns_816 8, 16 def_fns_816 16, 8 -function inv_dct32_odd_8x16_neon +function inv_dct32_odd_8h_x16_neon, export=1 movrel x16, idct_coeffs, 2*16 ld1 {v0.8h, v1.8h}, [x16] sub x16, x16, #2*16 smull_smlsl v2, v3, v16, v31, v0.h[0], v0.h[1], .8h // -> t16a @@ -2057,11 +2027,11 @@ add x7, x7, x8, lsr #1 .if \scale scale_input .8h, v0.h[0], v16, v17, v18, v19, v20, v21, v22, v23 scale_input .8h, v0.h[0], v24, v25, v26, v27, v28, v29, v30, v31 .endif - bl inv_dct_8x16_neon + bl inv_dct_8h_x16_neon transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v4, v5 transpose_8x8h v24, v25, v26, v27, v28, v29, v30, v31, v4, v5 .macro store1 r0, r1 st1 {\r0}, [x6], #16 @@ -2087,33 +2057,29 @@ .if \scale // This relies on the fact that the idct also leaves the right coeff in v0.h[1] scale_input .8h, v0.h[1], v16, v17, v18, v19, v20, v21, v22, v23 scale_input .8h, v0.h[1], v24, v25, v26, v27, v28, v29, v30, v31 .endif - bl inv_dct32_odd_8x16_neon + bl inv_dct32_odd_8h_x16_neon transpose_8x8h v31, v30, v29, v28, v27, v26, v25, v24, v4, v5 transpose_8x8h v23, v22, v21, v20, v19, v18, v17, v16, v4, v5 .macro store2 r0, r1, shift - ld1 {v4.8h}, [x6], #16 - ld1 {v5.8h}, [x6] + ld1 {v4.8h, v5.8h}, [x6] sqsub v7.8h, v4.8h, \r0 sqsub v6.8h, v5.8h, \r1 - sub x6, x6, #16 sqadd v4.8h, v4.8h, \r0 sqadd v5.8h, v5.8h, \r1 rev64 v6.8h, v6.8h rev64 v7.8h, v7.8h srshr v4.8h, v4.8h, #\shift srshr v5.8h, v5.8h, #\shift srshr v6.8h, v6.8h, #\shift srshr v7.8h, v7.8h, #\shift - st1 {v4.8h}, [x6], #16 ext v6.16b, v6.16b, v6.16b, #8 - st1 {v5.8h}, [x6], #16 + st1 {v4.8h, v5.8h}, [x6], #32 ext v7.16b, v7.16b, v7.16b, #8 - st1 {v6.8h}, [x6], #16 - st1 {v7.8h}, [x6], #16 + st1 {v6.8h, v7.8h}, [x6], #32 .endm store2 v31.8h, v23.8h, \shift store2 v30.8h, v22.8h, \shift store2 v29.8h, v21.8h, \shift @@ -2137,11 +2103,11 @@ .irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 ld1 {v\i\().8h}, [x7], x8 .endr sub x7, x7, x8, lsl #4 - bl inv_dct_8x16_neon + bl inv_dct_8h_x16_neon .irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 st1 {v\i\().8h}, [x7], x8 .endr sub x7, x7, x8, lsl #4 @@ -2150,11 +2116,11 @@ .irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 ld1 {v\i\().8h}, [x7], x8 .endr sub x7, x7, x8, lsl #4 sub x7, x7, x8, lsr #1 - bl inv_dct32_odd_8x16_neon + bl inv_dct32_odd_8h_x16_neon neg x9, x8 mov x10, x6 .macro combine r0, r1, r2, r3, op, stride ld1 {v5.8h}, [x7], \stride @@ -2214,11 +2180,11 @@ const eob_8x32 .short 43, 107, 171, 256 endconst -function inv_txfm_add_identity_identity_32x32_neon, export=1 +function inv_txfm_add_identity_identity_32x32_8bpc_neon, export=1 movi v0.8h, #0 movrel x13, eob_32x32 mov x8, #2*32 1: @@ -2257,11 +2223,11 @@ \op \i, \i, #\shift .endr .endm .macro def_identity_1632 w, h, wshort, hshort -function inv_txfm_add_identity_identity_\w\()x\h\()_neon, export=1 +function inv_txfm_add_identity_identity_\w\()x\h\()_8bpc_neon, export=1 mov w16, #2896*8 mov w17, #2*(5793-4096)*8 dup v1.4h, w16 movi v0.8h, #0 mov v1.h[1], w17 @@ -2283,11 +2249,11 @@ // 16x32 identity_8x8_shift1 v1.h[1] .else // 32x16 shift_8_regs sqshl, 1 - identity_8x8 v1.h[1] + identity_8x8 v1.h[1] .endif transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v4, v5 .if \w == 16 @@ -2317,16 +2283,17 @@ def_identity_1632 16, 32, _shortside, def_identity_1632 32, 16, , _shortside .macro def_identity_832 w, h -function inv_txfm_add_identity_identity_\w\()x\h\()_neon, export=1 +function inv_txfm_add_identity_identity_\w\()x\h\()_8bpc_neon, export=1 movi v0.8h, #0 movrel x13, eob_8x32 mov w8, #2*\h 1: + ldrh w12, [x13], #2 .irp i, v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h ld1 {\i}, [x2] st1 {v0.8h}, [x2], x8 .endr @@ -2335,18 +2302,17 @@ shift_8_regs srshr, 1 .endif transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v4, v5 + cmp w3, w12 .if \w == 8 load_add_store_8x8 x0, x7, shiftbits=2 .else load_add_store_8x8 x0, x7, shiftbits=3 .endif - ldrh w12, [x13], #2 - cmp w3, w12 b.lt 9f .if \w == 8 sub x2, x2, x8, lsl #3 add x2, x2, #2*8 .else @@ -2361,11 +2327,11 @@ .endm def_identity_832 8, 32 def_identity_832 32, 8 -function inv_txfm_add_dct_dct_32x32_neon, export=1 +function inv_txfm_add_dct_dct_32x32_8bpc_neon, export=1 idct_dc 32, 32, 2 mov x15, x30 sub sp, sp, #2048 movrel x13, eob_32x32 @@ -2409,18 +2375,18 @@ add sp, sp, #2048 br x15 endfunc -function inv_txfm_add_dct_dct_16x32_neon, export=1 +function inv_txfm_add_dct_dct_16x32_8bpc_neon, export=1 idct_dc 16, 32, 1 mov x15, x30 sub sp, sp, #1024 movrel x13, eob_16x32 ldrh w12, [x13], #2 - adr x4, inv_dct_8x16_neon + adr x4, inv_dct_8h_x16_neon .irp i, 0, 8, 16, 24 add x6, sp, #(\i*16*2) add x7, x2, #(\i*2) .if \i > 0 @@ -2458,17 +2424,17 @@ add sp, sp, #1024 br x15 endfunc -function inv_txfm_add_dct_dct_32x16_neon, export=1 +function inv_txfm_add_dct_dct_32x16_8bpc_neon, export=1 idct_dc 32, 16, 1 mov x15, x30 sub sp, sp, #1024 - adr x5, inv_dct_8x16_neon + adr x5, inv_dct_8h_x16_neon .irp i, 0, 8 add x6, sp, #(\i*32*2) add x7, x2, #(\i*2) .if \i > 0 @@ -2503,11 +2469,11 @@ add sp, sp, #1024 br x15 endfunc -function inv_txfm_add_dct_dct_8x32_neon, export=1 +function inv_txfm_add_dct_dct_8x32_8bpc_neon, export=1 idct_dc 8, 32, 2 mov x15, x30 sub sp, sp, #512 @@ -2515,33 +2481,31 @@ movi v28.8h, #0 mov x8, #2*32 mov w9, #32 mov x6, sp - mov x7, x2 1: .irp i, 16, 17, 18, 19, 20, 21, 22, 23 - ld1 {v\i\().8h}, [x7] - st1 {v28.8h}, [x7], x8 + ld1 {v\i\().8h}, [x2] + st1 {v28.8h}, [x2], x8 .endr ldrh w12, [x13], #2 + sub x2, x2, x8, lsl #3 sub w9, w9, #8 - sub x7, x7, x8, lsl #3 - add x7, x7, #2*8 - - bl inv_dct_8x8_neon + add x2, x2, #2*8 + + bl inv_dct_8h_x8_neon .irp i, 16, 17, 18, 19, 20, 21, 22, 23 srshr v\i\().8h, v\i\().8h, #2 .endr transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v24, v25 + st1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x6], #64 cmp w3, w12 -.irp i, 16, 17, 18, 19, 20, 21, 22, 23 - st1 {v\i\().8h}, [x6], #16 -.endr + st1 {v20.8h, v21.8h, v22.8h, v23.8h}, [x6], #64 b.ge 1b cbz w9, 3f movi v29.8h, #0 @@ -2562,11 +2526,11 @@ add sp, sp, #512 br x15 endfunc -function inv_txfm_add_dct_dct_32x8_neon, export=1 +function inv_txfm_add_dct_dct_32x8_8bpc_neon, export=1 idct_dc 32, 8, 2 mov x15, x30 sub sp, sp, #512 @@ -2584,11 +2548,11 @@ .irp i, 16, 17, 18, 19, 20, 21, 22, 23 ld1 {v\i\().8h}, [x7], x8 .endr add w9, w9, #8 - bl inv_dct_8x8_neon + bl inv_dct_8h_x8_neon cmp w9, #32 load_add_store_8x8 x6, x7 @@ -2789,11 +2753,11 @@ scale_input .8h, \c, \r0, \r1, \r2, \r3, \r4, \r5, \r6, \r7 .endif .endm .macro def_dct64_func suffix, clear=0, scale=0 -function inv_txfm_dct\suffix\()_8x64_neon +function inv_txfm_dct\suffix\()_8h_x64_neon, export=1 mov x14, x30 mov x6, sp lsl x8, x8, #2 movdup_if v0.4h, w16, #2896*8, \scale @@ -2802,11 +2766,11 @@ clear_upper8 sub x7, x7, x8, lsl #3 add x7, x7, x8, lsr #1 scale_if \scale, v0.h[0], v16, v17, v18, v19, v20, v21, v22, v23 - bl inv_dct_8x16_neon + bl inv_dct_8h_x16_neon store16 x6 movdup_if v0.4h, w16, #2896*8, \scale movi_if v7.8h, #0, \clear @@ -2815,11 +2779,11 @@ sub x7, x7, x8, lsl #3 lsr x8, x8, #1 sub x7, x7, x8, lsr #1 scale_if \scale, v0.h[0], v16, v17, v18, v19, v20, v21, v22, v23 - bl inv_dct32_odd_8x16_neon + bl inv_dct32_odd_8h_x16_neon add x10, x6, #16*15 sub x6, x6, #16*16 mov x9, #-16 @@ -3038,30 +3002,32 @@ br x14 endfunc .macro sub_sp space #ifdef _WIN32 -.if \space > 4096 +.if \space > 8192 + // Here, we'd need to touch two (or more) pages while decrementing + // the stack pointer. + .error "sub_sp_align doesn't support values over 8K at the moment" +.elseif \space > 4096 sub x16, sp, #4096 ldr xzr, [x16] sub sp, x16, #(\space - 4096) .else sub sp, sp, #\space .endif #else .if \space >= 4096 sub sp, sp, #(\space)/4096*4096 +.endif .if (\space % 4096) != 0 sub sp, sp, #(\space)%4096 .endif -.else - sub sp, sp, #\space -.endif #endif .endm -function inv_txfm_add_dct_dct_64x64_neon, export=1 +function inv_txfm_add_dct_dct_64x64_8bpc_neon, export=1 idct_dc 64, 64, 2 mov x15, x30 sub_sp 64*32*2+64*8*2 @@ -3077,11 +3043,11 @@ b.lt 1f .endif add x7, x2, #(\i*2) mov x8, #32*2 mov x12, #-2 // shift - bl inv_txfm_dct_clear_8x64_neon + bl inv_txfm_dct_clear_8h_x64_neon add x6, x5, #(\i*64*2) bl inv_txfm_horz_dct_64x8_neon .if \i < 24 ldrh w12, [x13], #2 .endif @@ -3102,20 +3068,20 @@ 3: .irp i, 0, 8, 16, 24, 32, 40, 48, 56 add x7, x5, #(\i*2) mov x8, #64*2 - bl inv_txfm_dct_8x64_neon + bl inv_txfm_dct_8h_x64_neon add x6, x0, #(\i) bl inv_txfm_add_vert_dct_8x64_neon .endr add sp, x5, #64*32*2 br x15 endfunc -function inv_txfm_add_dct_dct_64x32_neon, export=1 +function inv_txfm_add_dct_dct_64x32_8bpc_neon, export=1 idct_dc 64, 32, 1 mov x15, x30 sub_sp 64*32*2+64*8*2 @@ -3131,11 +3097,11 @@ b.lt 1f .endif add x7, x2, #(\i*2) mov x8, #32*2 mov x12, #-1 // shift - bl inv_txfm_dct_clear_scale_8x64_neon + bl inv_txfm_dct_clear_scale_8h_x64_neon add x6, x5, #(\i*64*2) bl inv_txfm_horz_dct_64x8_neon .if \i < 24 ldrh w12, [x13], #2 .endif @@ -3164,11 +3130,11 @@ add sp, x5, #64*32*2 br x15 endfunc -function inv_txfm_add_dct_dct_32x64_neon, export=1 +function inv_txfm_add_dct_dct_32x64_8bpc_neon, export=1 idct_dc 32, 64, 1 mov x15, x30 sub_sp 32*32*2+64*8*2 @@ -3205,20 +3171,20 @@ 3: .irp i, 0, 8, 16, 24 add x7, x5, #(\i*2) mov x8, #32*2 - bl inv_txfm_dct_8x64_neon + bl inv_txfm_dct_8h_x64_neon add x6, x0, #(\i) bl inv_txfm_add_vert_dct_8x64_neon .endr add sp, x5, #32*32*2 br x15 endfunc -function inv_txfm_add_dct_dct_64x16_neon, export=1 +function inv_txfm_add_dct_dct_64x16_8bpc_neon, export=1 idct_dc 64, 16, 2 mov x15, x30 sub_sp 64*16*2+64*8*2 @@ -3230,18 +3196,20 @@ add x6, x4, #(\i*64*2) .if \i > 0 mov w8, #(16 - \i) cmp w3, w12 b.lt 1f - ldrh w12, [x13], #2 .endif add x7, x2, #(\i*2) mov x8, #16*2 mov x12, #-2 // shift - bl inv_txfm_dct_clear_8x64_neon + bl inv_txfm_dct_clear_8h_x64_neon add x6, x4, #(\i*64*2) bl inv_txfm_horz_dct_64x8_neon +.if \i < 8 + ldrh w12, [x13], #2 +.endif .endr b 3f 1: movi v4.8h, #0 @@ -3254,11 +3222,11 @@ st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64 .endr b.gt 2b 3: - adr x5, inv_dct_8x16_neon + adr x5, inv_dct_8h_x16_neon .irp i, 0, 8, 16, 24, 32, 40, 48, 56 add x6, x0, #(\i) add x7, x4, #(\i*2) mov x8, #64*2 bl inv_txfm_add_vert_8x16_neon @@ -3266,22 +3234,22 @@ add sp, x4, #64*16*2 br x15 endfunc -function inv_txfm_add_dct_dct_16x64_neon, export=1 +function inv_txfm_add_dct_dct_16x64_8bpc_neon, export=1 idct_dc 16, 64, 2 mov x15, x30 sub_sp 16*32*2+64*8*2 add x5, sp, #64*8*2 movrel x13, eob_16x32 ldrh w12, [x13], #2 - adr x4, inv_dct_8x16_neon + adr x4, inv_dct_8h_x16_neon .irp i, 0, 8, 16, 24 add x6, x5, #(\i*16*2) .if \i > 0 mov w8, #(32 - \i) cmp w3, w12 @@ -3308,11 +3276,11 @@ 3: .irp i, 0, 8 add x7, x5, #(\i*2) mov x8, #16*2 - bl inv_txfm_dct_8x64_neon + bl inv_txfm_dct_8h_x64_neon add x6, x0, #(\i) bl inv_txfm_add_vert_dct_8x64_neon .endr add sp, x5, #16*32*2 diff --git a/third_party/dav1d/src/arm/64/itx16.S b/third_party/dav1d/src/arm/64/itx16.S new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/arm/64/itx16.S @@ -0,0 +1,3526 @@ +/****************************************************************************** + * Copyright © 2018, VideoLAN and dav1d authors + * Copyright © 2020, Martin Storsjo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + *****************************************************************************/ + +#include "src/arm/asm.S" +#include "util.S" + +// The exported functions in this file have got the following signature: +// void itxfm_add(pixel *dst, ptrdiff_t dst_stride, coef *coeff, int eob, +// int bitdepth_max); + +// Most of the functions use the following register layout: +// x0-x3 external parameters +// x4 function pointer to first transform +// x5 function pointer to second transform +// x6 output parameter for helper function +// x7 input parameter for helper function +// x8 input stride for helper function +// x9-x12 scratch variables for helper functions +// x13 pointer to list of eob thresholds +// x14 return pointer for helper function +// x15 return pointer for main function + +// The SIMD registers most often use the following layout: +// v0-v1 multiplication coefficients +// v2-v7 scratch registers +// v8-v15 unused +// v16-v31 inputs/outputs of transforms + +const idct_coeffs, align=4 + // idct4 + .int 2896, 2896*8*(1<<16), 1567, 3784 + // idct8 + .int 799, 4017, 3406, 2276 + // idct16 + .int 401, 4076, 3166, 2598 + .int 1931, 3612, 3920, 1189 + // idct32 + .int 201, 4091, 3035, 2751 + .int 1751, 3703, 3857, 1380 + .int 995, 3973, 3513, 2106 + .int 2440, 3290, 4052, 601 +endconst + +const idct64_coeffs, align=4 + .int 101*8*(1<<16), 4095*8*(1<<16), 2967*8*(1<<16), -2824*8*(1<<16) + .int 1660*8*(1<<16), 3745*8*(1<<16), 3822*8*(1<<16), -1474*8*(1<<16) + .int 4076, 401, 4017, 799 + + .int 4036*8*(1<<16), -700*8*(1<<16), 2359*8*(1<<16), 3349*8*(1<<16) + .int 3461*8*(1<<16), -2191*8*(1<<16), 897*8*(1<<16), 3996*8*(1<<16) + .int -3166, -2598, -799, -4017 + + .int 501*8*(1<<16), 4065*8*(1<<16), 3229*8*(1<<16), -2520*8*(1<<16) + .int 2019*8*(1<<16), 3564*8*(1<<16), 3948*8*(1<<16), -1092*8*(1<<16) + .int 3612, 1931, 2276, 3406 + + .int 4085*8*(1<<16), -301*8*(1<<16), 2675*8*(1<<16), 3102*8*(1<<16) + .int 3659*8*(1<<16), -1842*8*(1<<16), 1285*8*(1<<16), 3889*8*(1<<16) + .int -3920, -1189, -3406, -2276 +endconst + +const iadst4_coeffs, align=4 + .int 1321, 3803, 2482, 3344 +endconst + +const iadst8_coeffs, align=4 + .int 4076, 401, 3612, 1931 + .int 2598, 3166, 1189, 3920 + // idct_coeffs + .int 2896, 0, 1567, 3784 +endconst + +const iadst16_coeffs, align=4 + .int 4091, 201, 3973, 995 + .int 3703, 1751, 3290, 2440 + .int 2751, 3035, 2106, 3513 + .int 1380, 3857, 601, 4052 +endconst + +.macro mul_mla d, s0, s1, c0, c1 + mul \d\().4s, \s0\().4s, \c0 + mla \d\().4s, \s1\().4s, \c1 +.endm + +.macro mul_mls d, s0, s1, c0, c1 + mul \d\().4s, \s0\().4s, \c0 + mls \d\().4s, \s1\().4s, \c1 +.endm + +.macro scale_input sz, c, r0, r1, r2 r3, r4, r5, r6, r7 + sqrdmulh \r0\sz, \r0\sz, \c + sqrdmulh \r1\sz, \r1\sz, \c + sqrdmulh \r2\sz, \r2\sz, \c + sqrdmulh \r3\sz, \r3\sz, \c +.ifnb \r4 + sqrdmulh \r4\sz, \r4\sz, \c + sqrdmulh \r5\sz, \r5\sz, \c + sqrdmulh \r6\sz, \r6\sz, \c + sqrdmulh \r7\sz, \r7\sz, \c +.endif +.endm + +.macro load_add_store load, shift, addsrc, adddst, max, min, store, dst, src, shiftbits=4 +.ifnb \load + ld1 {\load}, [\src], x1 +.endif +.ifnb \shift + srshr \shift, \shift, #\shiftbits +.endif +.ifnb \addsrc + sqadd \adddst, \adddst, \addsrc +.endif +.ifnb \max + smax \max, \max, v6.8h +.endif +.ifnb \min + smin \min, \min, v7.8h +.endif +.ifnb \store + st1 {\store}, [\dst], x1 +.endif +.endm +.macro load_add_store_8x16 dst, src + mov \src, \dst + movi v6.8h, #0 + mvni v7.8h, #0xfc, lsl #8 // 0x3ff + load_add_store v2.8h, v16.8h, , , , , , \dst, \src + load_add_store v3.8h, v17.8h, , , , , , \dst, \src + load_add_store v4.8h, v18.8h, v2.8h, v16.8h, , , , \dst, \src + load_add_store v5.8h, v19.8h, v3.8h, v17.8h, v16.8h, , , \dst, \src + load_add_store v2.8h, v20.8h, v4.8h, v18.8h, v17.8h, v16.8h, , \dst, \src + load_add_store v3.8h, v21.8h, v5.8h, v19.8h, v18.8h, v17.8h, v16.8h, \dst, \src + load_add_store v4.8h, v22.8h, v2.8h, v20.8h, v19.8h, v18.8h, v17.8h, \dst, \src + load_add_store v5.8h, v23.8h, v3.8h, v21.8h, v20.8h, v19.8h, v18.8h, \dst, \src + load_add_store v2.8h, v24.8h, v4.8h, v22.8h, v21.8h, v20.8h, v19.8h, \dst, \src + load_add_store v3.8h, v25.8h, v5.8h, v23.8h, v22.8h, v21.8h, v20.8h, \dst, \src + load_add_store v4.8h, v26.8h, v2.8h, v24.8h, v23.8h, v22.8h, v21.8h, \dst, \src + load_add_store v5.8h, v27.8h, v3.8h, v25.8h, v24.8h, v23.8h, v22.8h, \dst, \src + load_add_store v2.8h, v28.8h, v4.8h, v26.8h, v25.8h, v24.8h, v23.8h, \dst, \src + load_add_store v3.8h, v29.8h, v5.8h, v27.8h, v26.8h, v25.8h, v24.8h, \dst, \src + load_add_store v4.8h, v30.8h, v2.8h, v28.8h, v27.8h, v26.8h, v25.8h, \dst, \src + load_add_store v5.8h, v31.8h, v3.8h, v29.8h, v28.8h, v27.8h, v26.8h, \dst, \src + load_add_store , , v4.8h, v30.8h, v29.8h, v28.8h, v27.8h, \dst, \src + load_add_store , , v5.8h, v31.8h, v30.8h, v29.8h, v28.8h, \dst, \src + load_add_store , , , , v31.8h, v30.8h, v29.8h, \dst, \src + load_add_store , , , , , v31.8h, v30.8h, \dst, \src + load_add_store , , , , , , v31.8h, \dst, \src +.endm +.macro load_add_store_8x8 dst, src, shiftbits=4 + mov \src, \dst + movi v6.8h, #0 + mvni v7.8h, #0xfc, lsl #8 // 0x3ff + load_add_store v2.8h, v16.8h, , , , , , \dst, \src, \shiftbits + load_add_store v3.8h, v17.8h, , , , , , \dst, \src, \shiftbits + load_add_store v4.8h, v18.8h, v2.8h, v16.8h, , , , \dst, \src, \shiftbits + load_add_store v5.8h, v19.8h, v3.8h, v17.8h, v16.8h, , , \dst, \src, \shiftbits + load_add_store v2.8h, v20.8h, v4.8h, v18.8h, v17.8h, v16.8h, , \dst, \src, \shiftbits + load_add_store v3.8h, v21.8h, v5.8h, v19.8h, v18.8h, v17.8h, v16.8h, \dst, \src, \shiftbits + load_add_store v4.8h, v22.8h, v2.8h, v20.8h, v19.8h, v18.8h, v17.8h, \dst, \src, \shiftbits + load_add_store v5.8h, v23.8h, v3.8h, v21.8h, v20.8h, v19.8h, v18.8h, \dst, \src, \shiftbits + load_add_store , , v4.8h, v22.8h, v21.8h, v20.8h, v19.8h, \dst, \src, \shiftbits + load_add_store , , v5.8h, v23.8h, v22.8h, v21.8h, v20.8h, \dst, \src, \shiftbits + load_add_store , , , , v23.8h, v22.8h, v21.8h, \dst, \src, \shiftbits + load_add_store , , , , , v23.8h, v22.8h, \dst, \src, \shiftbits + load_add_store , , , , , , v23.8h, \dst, \src, \shiftbits +.endm +.macro load_add_store_8x4 dst, src, shiftbits=4 + mov \src, \dst + movi v6.8h, #0 + mvni v7.8h, #0xfc, lsl #8 // 0x3ff + load_add_store v2.8h, v16.8h, , , , , , \dst, \src, \shiftbits + load_add_store v3.8h, v17.8h, , , , , , \dst, \src, \shiftbits + load_add_store v4.8h, v18.8h, v2.8h, v16.8h, , , , \dst, \src, \shiftbits + load_add_store v5.8h, v19.8h, v3.8h, v17.8h, v16.8h, , , \dst, \src, \shiftbits + load_add_store , , v4.8h, v18.8h, v17.8h, v16.8h, , \dst, \src, \shiftbits + load_add_store , , v5.8h, v19.8h, v18.8h, v17.8h, v16.8h, \dst, \src, \shiftbits + load_add_store , , , , v19.8h, v18.8h, v17.8h, \dst, \src, \shiftbits + load_add_store , , , , , v19.8h, v18.8h, \dst, \src, \shiftbits + load_add_store , , , , , , v19.8h, \dst, \src, \shiftbits +.endm +.macro load_add_store4 load, inssrc, insdst, shift, addsrc, adddst, max, min, store, dst, src +.ifnb \load + ld1 {\load}[0], [\src], x1 +.endif +.ifnb \inssrc + ins \insdst\().d[1], \inssrc\().d[0] +.endif +.ifnb \shift + srshr \shift, \shift, #4 +.endif +.ifnb \load + ld1 {\load}[1], [\src], x1 +.endif +.ifnb \addsrc + sqadd \adddst, \adddst, \addsrc +.endif +.ifnb \store + st1 {\store}[0], [\dst], x1 +.endif +.ifnb \max + smax \max, \max, v6.8h +.endif +.ifnb \min + smin \min, \min, v7.8h +.endif +.ifnb \store + st1 {\store}[1], [\dst], x1 +.endif +.endm +.macro load_add_store_4x16 dst, src + mov \src, \dst + movi v6.8h, #0 + mvni v7.8h, #0xfc, lsl #8 // 0x3ff + load_add_store4 v0.d, v17, v16, , , , , , , \dst, \src + load_add_store4 v1.d, v19, v18, , , , , , , \dst, \src + load_add_store4 v2.d, v21, v20, v16.8h, , , , , , \dst, \src + load_add_store4 v3.d, v23, v22, v18.8h, v0.8h, v16.8h, , , , \dst, \src + load_add_store4 v0.d, v25, v24, v20.8h, v1.8h, v18.8h, v16.8h, , , \dst, \src + load_add_store4 v1.d, v27, v26, v22.8h, v2.8h, v20.8h, v18.8h, v16.8h, , \dst, \src + load_add_store4 v2.d, v29, v28, v24.8h, v3.8h, v22.8h, v20.8h, v18.8h, v16.d, \dst, \src + load_add_store4 v3.d, v31, v30, v26.8h, v0.8h, v24.8h, v22.8h, v20.8h, v18.d, \dst, \src + load_add_store4 , , , v28.8h, v1.8h, v26.8h, v24.8h, v22.8h, v20.d, \dst, \src + load_add_store4 , , , v30.8h, v2.8h, v28.8h, v26.8h, v24.8h, v22.d, \dst, \src + load_add_store4 , , , , v3.8h, v30.8h, v28.8h, v26.8h, v24.d, \dst, \src + load_add_store4 , , , , , , v30.8h, v28.8h, v26.d, \dst, \src + load_add_store4 , , , , , , , v30.8h, v28.d, \dst, \src + load_add_store4 , , , , , , , , v30.d, \dst, \src +.endm +.macro load_add_store_4x8 dst, src + mov \src, \dst + movi v6.8h, #0 + mvni v7.8h, #0xfc, lsl #8 // 0x3ff + load_add_store4 v0.d, v17, v16, , , , , , , \dst, \src + load_add_store4 v1.d, v19, v18, , , , , , , \dst, \src + load_add_store4 v2.d, v21, v20, v16.8h, , , , , , \dst, \src + load_add_store4 v3.d, v23, v22, v18.8h, v0.8h, v16.8h, , , , \dst, \src + load_add_store4 , , , v20.8h, v1.8h, v18.8h, v16.8h, , , \dst, \src + load_add_store4 , , , v22.8h, v2.8h, v20.8h, v18.8h, v16.8h, , \dst, \src + load_add_store4 , , , , v3.8h, v22.8h, v20.8h, v18.8h, v16.d, \dst, \src + load_add_store4 , , , , , , v22.8h, v20.8h, v18.d, \dst, \src + load_add_store4 , , , , , , , v22.8h, v20.d, \dst, \src + load_add_store4 , , , , , , , , v22.d, \dst, \src +.endm + +.macro idct_dc w, h, shift + cbnz w3, 1f + movz w16, #2896*8, lsl #16 + ld1r {v16.4s}, [x2] + dup v0.2s, w16 + sqrdmulh v20.4s, v16.4s, v0.s[0] + str wzr, [x2] +.if (\w == 2*\h) || (2*\w == \h) + sqrdmulh v20.4s, v20.4s, v0.s[0] +.endif +.if \shift > 0 + sqrshrn v16.4h, v20.4s, #\shift + sqrshrn2 v16.8h, v20.4s, #\shift +.else + sqxtn v16.4h, v20.4s + sqxtn2 v16.8h, v20.4s +.endif + sqrdmulh v16.8h, v16.8h, v0.h[1] + srshr v16.8h, v16.8h, #4 + mov w4, #\h + b idct_dc_w\w\()_neon +1: +.endm + +function idct_dc_w4_neon + movi v30.8h, #0 + mvni v31.8h, #0xfc, lsl #8 // 0x3ff +1: + ld1 {v0.d}[0], [x0], x1 + ld1 {v0.d}[1], [x0], x1 + ld1 {v1.d}[0], [x0], x1 + subs w4, w4, #4 + ld1 {v1.d}[1], [x0], x1 + sqadd v0.8h, v0.8h, v16.8h + sub x0, x0, x1, lsl #2 + sqadd v1.8h, v1.8h, v16.8h + smax v0.8h, v0.8h, v30.8h + smax v1.8h, v1.8h, v30.8h + smin v0.8h, v0.8h, v31.8h + st1 {v0.d}[0], [x0], x1 + smin v1.8h, v1.8h, v31.8h + st1 {v0.d}[1], [x0], x1 + st1 {v1.d}[0], [x0], x1 + st1 {v1.d}[1], [x0], x1 + b.gt 1b + ret +endfunc + +function idct_dc_w8_neon + movi v30.8h, #0 + mvni v31.8h, #0xfc, lsl #8 // 0x3ff +1: + ld1 {v0.8h}, [x0], x1 + subs w4, w4, #4 + ld1 {v1.8h}, [x0], x1 + sqadd v0.8h, v0.8h, v16.8h + ld1 {v2.8h}, [x0], x1 + sqadd v1.8h, v1.8h, v16.8h + ld1 {v3.8h}, [x0], x1 + sqadd v2.8h, v2.8h, v16.8h + sqadd v3.8h, v3.8h, v16.8h + sub x0, x0, x1, lsl #2 + smax v0.8h, v0.8h, v30.8h + smax v1.8h, v1.8h, v30.8h + smax v2.8h, v2.8h, v30.8h + smax v3.8h, v3.8h, v30.8h + smin v0.8h, v0.8h, v31.8h + smin v1.8h, v1.8h, v31.8h + st1 {v0.8h}, [x0], x1 + smin v2.8h, v2.8h, v31.8h + st1 {v1.8h}, [x0], x1 + smin v3.8h, v3.8h, v31.8h + st1 {v2.8h}, [x0], x1 + st1 {v3.8h}, [x0], x1 + b.gt 1b + ret +endfunc + +function idct_dc_w16_neon + movi v30.8h, #0 + mvni v31.8h, #0xfc, lsl #8 // 0x3ff +1: + ld1 {v0.8h, v1.8h}, [x0], x1 + subs w4, w4, #2 + ld1 {v2.8h, v3.8h}, [x0], x1 + sqadd v0.8h, v0.8h, v16.8h + sqadd v1.8h, v1.8h, v16.8h + sub x0, x0, x1, lsl #1 + sqadd v2.8h, v2.8h, v16.8h + sqadd v3.8h, v3.8h, v16.8h + smax v0.8h, v0.8h, v30.8h + smax v1.8h, v1.8h, v30.8h + smax v2.8h, v2.8h, v30.8h + smax v3.8h, v3.8h, v30.8h + smin v0.8h, v0.8h, v31.8h + smin v1.8h, v1.8h, v31.8h + smin v2.8h, v2.8h, v31.8h + st1 {v0.8h, v1.8h}, [x0], x1 + smin v3.8h, v3.8h, v31.8h + st1 {v2.8h, v3.8h}, [x0], x1 + b.gt 1b + ret +endfunc + +function idct_dc_w32_neon + movi v30.8h, #0 + mvni v31.8h, #0xfc, lsl #8 // 0x3ff +1: + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0] + subs w4, w4, #1 + sqadd v0.8h, v0.8h, v16.8h + sqadd v1.8h, v1.8h, v16.8h + sqadd v2.8h, v2.8h, v16.8h + sqadd v3.8h, v3.8h, v16.8h + smax v0.8h, v0.8h, v30.8h + smax v1.8h, v1.8h, v30.8h + smax v2.8h, v2.8h, v30.8h + smax v3.8h, v3.8h, v30.8h + smin v0.8h, v0.8h, v31.8h + smin v1.8h, v1.8h, v31.8h + smin v2.8h, v2.8h, v31.8h + smin v3.8h, v3.8h, v31.8h + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], x1 + b.gt 1b + ret +endfunc + +function idct_dc_w64_neon + movi v30.8h, #0 + mvni v31.8h, #0xfc, lsl #8 // 0x3ff + sub x1, x1, #64 +1: + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + subs w4, w4, #1 + sqadd v0.8h, v0.8h, v16.8h + ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0] + sqadd v1.8h, v1.8h, v16.8h + sub x0, x0, #64 + sqadd v2.8h, v2.8h, v16.8h + sqadd v3.8h, v3.8h, v16.8h + sqadd v4.8h, v4.8h, v16.8h + sqadd v5.8h, v5.8h, v16.8h + sqadd v6.8h, v6.8h, v16.8h + sqadd v7.8h, v7.8h, v16.8h + smax v0.8h, v0.8h, v30.8h + smax v1.8h, v1.8h, v30.8h + smax v2.8h, v2.8h, v30.8h + smax v3.8h, v3.8h, v30.8h + smax v4.8h, v4.8h, v30.8h + smax v5.8h, v5.8h, v30.8h + smax v6.8h, v6.8h, v30.8h + smax v7.8h, v7.8h, v30.8h + smin v0.8h, v0.8h, v31.8h + smin v1.8h, v1.8h, v31.8h + smin v2.8h, v2.8h, v31.8h + smin v3.8h, v3.8h, v31.8h + smin v4.8h, v4.8h, v31.8h + smin v5.8h, v5.8h, v31.8h + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64 + smin v6.8h, v6.8h, v31.8h + smin v7.8h, v7.8h, v31.8h + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], x1 + b.gt 1b + ret +endfunc + +.macro iwht4 + add v16.4s, v16.4s, v17.4s + sub v21.4s, v18.4s, v19.4s + sub v20.4s, v16.4s, v21.4s + sshr v20.4s, v20.4s, #1 + sub v18.4s, v20.4s, v17.4s + sub v17.4s, v20.4s, v19.4s + add v19.4s, v21.4s, v18.4s + sub v16.4s, v16.4s, v17.4s +.endm + +.macro idct_4 r0, r1, r2, r3 + mul_mla v6, \r1, \r3, v0.s[3], v0.s[2] + mul_mls v4, \r1, \r3, v0.s[2], v0.s[3] + mul_mla v2, \r0, \r2, v0.s[0], v0.s[0] + mul_mls v3, \r0, \r2, v0.s[0], v0.s[0] + srshr v6.4s, v6.4s, #12 + srshr v7.4s, v4.4s, #12 + srshr v2.4s, v2.4s, #12 + srshr v3.4s, v3.4s, #12 + sqadd \r0\().4s, v2.4s, v6.4s + sqsub \r3\().4s, v2.4s, v6.4s + sqadd \r1\().4s, v3.4s, v7.4s + sqsub \r2\().4s, v3.4s, v7.4s +.endm + +function inv_dct_4s_x4_neon + movrel x16, idct_coeffs + ld1 {v0.4s}, [x16] + idct_4 v16, v17, v18, v19 + ret +endfunc + +.macro iadst_4x4 o0, o1, o2, o3 + movrel x16, iadst4_coeffs + ld1 {v0.4s}, [x16] + + sub v3.4s, v16.4s, v18.4s + mul v4.4s, v16.4s, v0.s[0] + mla v4.4s, v18.4s, v0.s[1] + mla v4.4s, v19.4s, v0.s[2] + mul v7.4s, v17.4s, v0.s[3] + add v3.4s, v3.4s, v19.4s + mul v5.4s, v16.4s, v0.s[2] + mls v5.4s, v18.4s, v0.s[0] + mls v5.4s, v19.4s, v0.s[1] + + add \o3\().4s, v4.4s, v5.4s + mul \o2\().4s, v3.4s, v0.s[3] + add \o0\().4s, v4.4s, v7.4s + add \o1\().4s, v5.4s, v7.4s + sub \o3\().4s, \o3\().4s, v7.4s + + srshr \o0\().4s, \o0\().4s, #12 + srshr \o2\().4s, \o2\().4s, #12 + srshr \o1\().4s, \o1\().4s, #12 + srshr \o3\().4s, \o3\().4s, #12 +.endm + +function inv_adst_4s_x4_neon + iadst_4x4 v16, v17, v18, v19 + ret +endfunc + +function inv_flipadst_4s_x4_neon + iadst_4x4 v19, v18, v17, v16 + ret +endfunc + +function inv_identity_4s_x4_neon + movz w16, #(5793-4096)*8, lsl #16 + dup v0.2s, w16 + sqrdmulh v4.4s, v16.4s, v0.s[0] + sqrdmulh v5.4s, v17.4s, v0.s[0] + sqrdmulh v6.4s, v18.4s, v0.s[0] + sqrdmulh v7.4s, v19.4s, v0.s[0] + sqadd v16.4s, v16.4s, v4.4s + sqadd v17.4s, v17.4s, v5.4s + sqadd v18.4s, v18.4s, v6.4s + sqadd v19.4s, v19.4s, v7.4s + ret +endfunc + +function inv_txfm_add_wht_wht_4x4_16bpc_neon, export=1 + mov x15, x30 + movi v30.4s, #0 + movi v31.4s, #0 + ld1 {v16.4s,v17.4s,v18.4s,v19.4s}, [x2] + st1 {v30.4s, v31.4s}, [x2], #32 + + sshr v16.4s, v16.4s, #2 + sshr v17.4s, v17.4s, #2 + sshr v18.4s, v18.4s, #2 + sshr v19.4s, v19.4s, #2 + + iwht4 + + st1 {v30.4s, v31.4s}, [x2], #32 + transpose_4x4s v16, v17, v18, v19, v20, v21, v22, v23 + + iwht4 + + ld1 {v0.d}[0], [x0], x1 + sqxtn v16.4h, v16.4s + ld1 {v0.d}[1], [x0], x1 + sqxtn2 v16.8h, v17.4s + ld1 {v1.d}[0], [x0], x1 + sqxtn v18.4h, v18.4s + ld1 {v1.d}[1], [x0], x1 + sqxtn2 v18.8h, v19.4s + + b L(itx_4x4_end) +endfunc + +function inv_txfm_add_4x4_neon + movi v30.4s, #0 + movi v31.4s, #0 + ld1 {v16.4s,v17.4s,v18.4s,v19.4s}, [x2] + st1 {v30.4s, v31.4s}, [x2], #32 + + blr x4 + + st1 {v30.4s, v31.4s}, [x2], #32 + sqxtn v16.4h, v16.4s + sqxtn v17.4h, v17.4s + sqxtn v18.4h, v18.4s + sqxtn v19.4h, v19.4s + transpose_4x4h v16, v17, v18, v19, v20, v21, v22, v23 + + blr x5 + + ld1 {v0.d}[0], [x0], x1 + ld1 {v0.d}[1], [x0], x1 + ins v16.d[1], v17.d[0] + ins v18.d[1], v19.d[0] + ld1 {v1.d}[0], [x0], x1 + ld1 {v1.d}[1], [x0], x1 + srshr v16.8h, v16.8h, #4 + srshr v18.8h, v18.8h, #4 + +L(itx_4x4_end): + mvni v31.8h, #0xfc, lsl #8 // 0x3ff + sub x0, x0, x1, lsl #2 + sqadd v16.8h, v16.8h, v0.8h + sqadd v18.8h, v18.8h, v1.8h + smax v16.8h, v16.8h, v30.8h + smax v18.8h, v18.8h, v30.8h + smin v16.8h, v16.8h, v31.8h + st1 {v16.d}[0], [x0], x1 + smin v18.8h, v18.8h, v31.8h + st1 {v16.d}[1], [x0], x1 + st1 {v18.d}[0], [x0], x1 + st1 {v18.d}[1], [x0], x1 + + br x15 +endfunc + +.macro def_fn_4x4 txfm1, txfm2 +function inv_txfm_add_\txfm1\()_\txfm2\()_4x4_16bpc_neon, export=1 + mov x15, x30 + +.ifc \txfm1\()_\txfm2, dct_dct + cbnz w3, 1f + movz w16, #2896*8, lsl #16 + ld1r {v16.4s}, [x2] + dup v4.2s, w16 + str wzr, [x2] + sqrdmulh v16.4s, v16.4s, v4.s[0] + ld1 {v0.d}[0], [x0], x1 + sqxtn v20.4h, v16.4s + sqxtn2 v20.8h, v16.4s + ld1 {v0.d}[1], [x0], x1 + sqrdmulh v20.8h, v20.8h, v4.h[1] + ld1 {v1.d}[0], [x0], x1 + srshr v16.8h, v20.8h, #4 + ld1 {v1.d}[1], [x0], x1 + srshr v18.8h, v20.8h, #4 + movi v30.8h, #0 + b L(itx_4x4_end) +1: +.endif + adr x4, inv_\txfm1\()_4s_x4_neon + movrel x5, X(inv_\txfm2\()_4h_x4_neon) + b inv_txfm_add_4x4_neon +endfunc +.endm + +def_fn_4x4 dct, dct +def_fn_4x4 identity, identity +def_fn_4x4 dct, adst +def_fn_4x4 dct, flipadst +def_fn_4x4 dct, identity +def_fn_4x4 adst, dct +def_fn_4x4 adst, adst +def_fn_4x4 adst, flipadst +def_fn_4x4 flipadst, dct +def_fn_4x4 flipadst, adst +def_fn_4x4 flipadst, flipadst +def_fn_4x4 identity, dct + +def_fn_4x4 adst, identity +def_fn_4x4 flipadst, identity +def_fn_4x4 identity, adst +def_fn_4x4 identity, flipadst + +.macro idct_8 r0, r1, r2, r3, r4, r5, r6, r7 + idct_4 \r0, \r2, \r4, \r6 + + mul_mls v2, \r1, \r7, v1.s[0], v1.s[1] // -> t4a + mul_mla v4, \r1, \r7, v1.s[1], v1.s[0] // -> t7a + mul_mls v6, \r5, \r3, v1.s[2], v1.s[3] // -> t5a + mul_mla v7, \r5, \r3, v1.s[3], v1.s[2] // -> t6a + srshr \r1\().4s, v2.4s, #12 // t4a + srshr \r7\().4s, v4.4s, #12 // t7a + srshr \r3\().4s, v6.4s, #12 // t5a + srshr \r5\().4s, v7.4s, #12 // taa + + sqadd v2.4s, \r1\().4s, \r3\().4s // t4 + sqsub \r1\().4s, \r1\().4s, \r3\().4s // t5a + sqadd v3.4s, \r7\().4s, \r5\().4s // t7 + sqsub \r3\().4s, \r7\().4s, \r5\().4s // t6a + + mul_mls v4, \r3, \r1, v0.s[0], v0.s[0] // -> t5 + mul_mla v6, \r3, \r1, v0.s[0], v0.s[0] // -> t6 + srshr v4.4s, v4.4s, #12 // t5 + srshr v5.4s, v6.4s, #12 // t6 + + sqsub \r7\().4s, \r0\().4s, v3.4s // out7 + sqadd \r0\().4s, \r0\().4s, v3.4s // out0 + sqadd \r1\().4s, \r2\().4s, v5.4s // out1 + sqsub v6.4s, \r2\().4s, v5.4s // out6 + sqadd \r2\().4s, \r4\().4s, v4.4s // out2 + sqsub \r5\().4s, \r4\().4s, v4.4s // out5 + sqadd \r3\().4s, \r6\().4s, v2.4s // out3 + sqsub \r4\().4s, \r6\().4s, v2.4s // out4 + mov \r6\().16b, v6.16b // out6 +.endm + +function inv_dct_4s_x8_neon + movrel x16, idct_coeffs + ld1 {v0.4s, v1.4s}, [x16] + idct_8 v16, v17, v18, v19, v20, v21, v22, v23 + ret +endfunc + +.macro iadst_8 o0, o1, o2, o3, o4, o5, o6, o7 + movrel x16, iadst8_coeffs + ld1 {v0.4s, v1.4s}, [x16], #32 + + mul_mla v2, v23, v16, v0.s[0], v0.s[1] + mul_mls v4, v23, v16, v0.s[1], v0.s[0] + mul_mla v6, v21, v18, v0.s[2], v0.s[3] + srshr v16.4s, v2.4s, #12 // t0a + srshr v23.4s, v4.4s, #12 // t1a + mul_mls v2, v21, v18, v0.s[3], v0.s[2] + mul_mla v4, v19, v20, v1.s[0], v1.s[1] + srshr v18.4s, v6.4s, #12 // t2a + srshr v21.4s, v2.4s, #12 // t3a + mul_mls v6, v19, v20, v1.s[1], v1.s[0] + mul_mla v2, v17, v22, v1.s[2], v1.s[3] + srshr v20.4s, v4.4s, #12 // t4a + srshr v19.4s, v6.4s, #12 // t5a + mul_mls v4, v17, v22, v1.s[3], v1.s[2] + srshr v22.4s, v2.4s, #12 // t6a + srshr v17.4s, v4.4s, #12 // t7a + + ld1 {v0.4s}, [x16] + + sqadd v2.4s, v16.4s, v20.4s // t0 + sqsub v3.4s, v16.4s, v20.4s // t4 + sqadd v4.4s, v23.4s, v19.4s // t1 + sqsub v5.4s, v23.4s, v19.4s // t5 + sqadd v6.4s, v18.4s, v22.4s // t2 + sqsub v7.4s, v18.4s, v22.4s // t6 + sqadd v18.4s, v21.4s, v17.4s // t3 + sqsub v19.4s, v21.4s, v17.4s // t7 + + mul_mla v16, v3, v5, v0.s[3], v0.s[2] + mul_mls v20, v3, v5, v0.s[2], v0.s[3] + mul_mls v22, v19, v7, v0.s[3], v0.s[2] + + srshr v3.4s, v16.4s, #12 // t4a + srshr v5.4s, v20.4s, #12 // t5a + + mul_mla v16, v19, v7, v0.s[2], v0.s[3] + + srshr v7.4s, v22.4s, #12 // t6a + srshr v19.4s, v16.4s, #12 // t7a + + sqadd \o0\().4s, v2.4s, v6.4s // out0 + sqsub v2.4s, v2.4s, v6.4s // t2 + sqadd \o7\().4s, v4.4s, v18.4s // out7 + sqsub v4.4s, v4.4s, v18.4s // t3 + sqneg \o7\().4s, \o7\().4s // out7 + + sqadd \o1\().4s, v3.4s, v7.4s // out1 + sqsub v3.4s, v3.4s, v7.4s // t6 + sqadd \o6\().4s, v5.4s, v19.4s // out6 + sqsub v5.4s, v5.4s, v19.4s // t7 + sqneg \o1\().4s, \o1\().4s // out1 + + mul_mla v18, v2, v4, v0.s[0], v0.s[0] // -> out3 (v19 or v20) + mul_mls v6, v2, v4, v0.s[0], v0.s[0] // -> out4 (v20 or v19) + mul_mls v20, v3, v5, v0.s[0], v0.s[0] // -> out5 (v21 or v18) + srshr v2.4s, v18.4s, #12 // out3 + mul_mla v18, v3, v5, v0.s[0], v0.s[0] // -> out2 (v18 or v21) + srshr v3.4s, v20.4s, #12 // out5 + srshr \o2\().4s, v18.4s, #12 // out2 (v18 or v21) + srshr \o4\().4s, v6.4s, #12 // out4 (v20 or v19) + + sqneg \o3\().4s, v2.4s // out3 + sqneg \o5\().4s, v3.4s // out5 +.endm + +function inv_adst_4s_x8_neon + iadst_8 v16, v17, v18, v19, v20, v21, v22, v23 + ret +endfunc + +function inv_flipadst_4s_x8_neon + iadst_8 v23, v22, v21, v20, v19, v18, v17, v16 + ret +endfunc + +function inv_identity_4s_x8_neon + sqshl v16.4s, v16.4s, #1 + sqshl v17.4s, v17.4s, #1 + sqshl v18.4s, v18.4s, #1 + sqshl v19.4s, v19.4s, #1 + sqshl v20.4s, v20.4s, #1 + sqshl v21.4s, v21.4s, #1 + sqshl v22.4s, v22.4s, #1 + sqshl v23.4s, v23.4s, #1 + ret +endfunc + +function inv_txfm_add_8x8_neon + movi v31.4s, #0 + + cmp w3, w13 + mov x11, #32 + b.lt 1f + + add x6, x2, #16 +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s + ld1 {\i}, [x6] + st1 {v31.4s}, [x6], x11 +.endr + + blr x4 + + sqrshrn v24.4h, v16.4s, #1 + sqrshrn v25.4h, v17.4s, #1 + sqrshrn v26.4h, v18.4s, #1 + sqrshrn v27.4h, v19.4s, #1 + sqrshrn2 v24.8h, v20.4s, #1 + sqrshrn2 v25.8h, v21.4s, #1 + sqrshrn2 v26.8h, v22.4s, #1 + sqrshrn2 v27.8h, v23.4s, #1 + + transpose_4x8h v24, v25, v26, v27, v2, v3, v4, v5 + + b 2f + +1: +.irp i, v24.8h, v25.8h, v26.8h, v27.8h + movi \i, #0 +.endr + +2: + +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s + ld1 {\i}, [x2] + st1 {v31.4s}, [x2], x11 +.endr + + blr x4 + + sqrshrn v16.4h, v16.4s, #1 + sqrshrn v17.4h, v17.4s, #1 + sqrshrn v18.4h, v18.4s, #1 + sqrshrn v19.4h, v19.4s, #1 + sqrshrn2 v16.8h, v20.4s, #1 + sqrshrn2 v17.8h, v21.4s, #1 + sqrshrn2 v18.8h, v22.4s, #1 + sqrshrn2 v19.8h, v23.4s, #1 + + transpose_4x8h v16, v17, v18, v19, v20, v21, v22, v23 + + mov v20.16b, v24.16b + mov v21.16b, v25.16b + mov v22.16b, v26.16b + mov v23.16b, v27.16b + + blr x5 + + load_add_store_8x8 x0, x7 + br x15 +endfunc + +.macro def_fn_8x8 txfm1, txfm2, eob_half +function inv_txfm_add_\txfm1\()_\txfm2\()_8x8_16bpc_neon, export=1 + mov x15, x30 + +.ifc \txfm1\()_\txfm2, dct_dct + idct_dc 8, 8, 1 +.endif + movrel x5, X(inv_\txfm2\()_8h_x8_neon) + mov w13, #\eob_half + adr x4, inv_\txfm1\()_4s_x8_neon + b inv_txfm_add_8x8_neon +endfunc +.endm + +def_fn_8x8 dct, dct, 10 +def_fn_8x8 identity, identity, 10 +def_fn_8x8 dct, adst, 10 +def_fn_8x8 dct, flipadst, 10 +def_fn_8x8 dct, identity, 4 +def_fn_8x8 adst, dct, 10 +def_fn_8x8 adst, adst, 10 +def_fn_8x8 adst, flipadst, 10 +def_fn_8x8 flipadst, dct, 10 +def_fn_8x8 flipadst, adst, 10 +def_fn_8x8 flipadst, flipadst, 10 +def_fn_8x8 identity, dct, 4 +def_fn_8x8 adst, identity, 4 +def_fn_8x8 flipadst, identity, 4 +def_fn_8x8 identity, adst, 4 +def_fn_8x8 identity, flipadst, 4 + +function inv_txfm_add_8x4_neon + movi v28.4s, #0 + movi v29.4s, #0 + movi v30.4s, #0 + movi v31.4s, #0 + ld1 {v16.4s,v17.4s,v18.4s,v19.4s}, [x2] + st1 {v28.4s,v29.4s,v30.4s,v31.4s}, [x2], #64 + movz w16, #2896*8, lsl #16 + dup v0.2s, w16 + ld1 {v20.4s,v21.4s,v22.4s,v23.4s}, [x2] + st1 {v28.4s,v29.4s,v30.4s,v31.4s}, [x2] + + scale_input .4s, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23 + + blr x4 + + sqxtn v16.4h, v16.4s + sqxtn v17.4h, v17.4s + sqxtn v18.4h, v18.4s + sqxtn v19.4h, v19.4s + sqxtn v20.4h, v20.4s + sqxtn v21.4h, v21.4s + sqxtn v22.4h, v22.4s + sqxtn v23.4h, v23.4s + + transpose_4x4h v16, v17, v18, v19, v4, v5, v6, v7 + transpose_4x4h v20, v21, v22, v23, v4, v5, v6, v7 + ins v16.d[1], v20.d[0] + ins v17.d[1], v21.d[0] + ins v18.d[1], v22.d[0] + ins v19.d[1], v23.d[0] + + blr x5 + + load_add_store_8x4 x0, x7 + br x15 +endfunc + +function inv_txfm_add_4x8_neon + movz w16, #2896*8, lsl #16 + movi v31.4s, #0 + dup v30.2s, w16 + + cmp w3, w13 + mov x11, #32 + b.lt 1f + + add x6, x2, #16 +.irp i, v16.4s, v17.4s, v18.4s, v19.4s + ld1 {\i}, [x6] + st1 {v31.4s}, [x6], x11 +.endr + scale_input .4s, v30.s[0], v16, v17, v18, v19 + blr x4 + sqxtn v20.4h, v16.4s + sqxtn v21.4h, v17.4s + sqxtn v22.4h, v18.4s + sqxtn v23.4h, v19.4s + transpose_4x4h v20, v21, v22, v23, v4, v5, v6, v7 + + b 2f + +1: +.irp i, v20, v21, v22, v23 + movi \i\().4h, #0 +.endr + +2: + +.irp i, v16.4s, v17.4s, v18.4s, v19.4s + ld1 {\i}, [x2] + st1 {v31.4s}, [x2], x11 +.endr + scale_input .4s, v30.s[0], v16, v17, v18, v19 + blr x4 + sqxtn v16.4h, v16.4s + sqxtn v17.4h, v17.4s + sqxtn v18.4h, v18.4s + sqxtn v19.4h, v19.4s + transpose_4x4h v16, v17, v18, v19, v4, v5, v6, v7 + + blr x5 + + load_add_store_4x8 x0, x7 + br x15 +endfunc + +.macro def_fn_48 w, h, txfm1, txfm2, eob_half +function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_16bpc_neon, export=1 + mov x15, x30 + +.ifc \txfm1\()_\txfm2, dct_dct + idct_dc \w, \h, 0 +.endif + adr x4, inv_\txfm1\()_4s_x\w\()_neon +.if \w == 4 + mov w13, #\eob_half +.endif + movrel x5, X(inv_\txfm2\()_\w\()h_x\h\()_neon) + b inv_txfm_add_\w\()x\h\()_neon +endfunc +.endm + +.macro def_fns_48 w, h +def_fn_48 \w, \h, dct, dct, 13 +def_fn_48 \w, \h, identity, identity, 13 +def_fn_48 \w, \h, dct, adst, 13 +def_fn_48 \w, \h, dct, flipadst, 13 +def_fn_48 \w, \h, dct, identity, 4 +def_fn_48 \w, \h, adst, dct, 13 +def_fn_48 \w, \h, adst, adst, 13 +def_fn_48 \w, \h, adst, flipadst, 13 +def_fn_48 \w, \h, flipadst, dct, 13 +def_fn_48 \w, \h, flipadst, adst, 13 +def_fn_48 \w, \h, flipadst, flipadst, 13 +def_fn_48 \w, \h, identity, dct, 16 +def_fn_48 \w, \h, adst, identity, 4 +def_fn_48 \w, \h, flipadst, identity, 4 +def_fn_48 \w, \h, identity, adst, 16 +def_fn_48 \w, \h, identity, flipadst, 16 +.endm + +def_fns_48 4, 8 +def_fns_48 8, 4 + + +function inv_dct_4s_x16_neon + movrel x16, idct_coeffs + ld1 {v0.4s, v1.4s}, [x16], #32 + + idct_8 v16, v18, v20, v22, v24, v26, v28, v30 + + ld1 {v0.4s, v1.4s}, [x16] + sub x16, x16, #32 + + mul_mls v2, v17, v31, v0.s[0], v0.s[1] // -> t8a + mul_mla v4, v17, v31, v0.s[1], v0.s[0] // -> t15a + mul_mls v6, v25, v23, v0.s[2], v0.s[3] // -> t9a + srshr v17.4s, v2.4s, #12 // t8a + srshr v31.4s, v4.4s, #12 // t15a + mul_mla v2, v25, v23, v0.s[3], v0.s[2] // -> t14a + mul_mls v4, v21, v27, v1.s[0], v1.s[1] // -> t10a + srshr v23.4s, v6.4s, #12 // t9a + srshr v25.4s, v2.4s, #12 // t14a + mul_mla v6, v21, v27, v1.s[1], v1.s[0] // -> t13a + mul_mls v2, v29, v19, v1.s[2], v1.s[3] // -> t11a + srshr v21.4s, v4.4s, #12 // t10a + srshr v27.4s, v6.4s, #12 // t13a + mul_mla v4, v29, v19, v1.s[3], v1.s[2] // -> t12a + srshr v19.4s, v2.4s, #12 // t11a + srshr v29.4s, v4.4s, #12 // t12a + + ld1 {v0.4s}, [x16] + + sqsub v2.4s, v17.4s, v23.4s // t9 + sqadd v17.4s, v17.4s, v23.4s // t8 + sqsub v3.4s, v31.4s, v25.4s // t14 + sqadd v31.4s, v31.4s, v25.4s // t15 + sqsub v23.4s, v19.4s, v21.4s // t10 + sqadd v19.4s, v19.4s, v21.4s // t11 + sqadd v25.4s, v29.4s, v27.4s // t12 + sqsub v29.4s, v29.4s, v27.4s // t13 + + mul_mls v4, v3, v2, v0.s[2], v0.s[3] // -> t9a + mul_mla v6, v3, v2, v0.s[3], v0.s[2] // -> t14a + srshr v21.4s, v4.4s, #12 // t9a + srshr v27.4s, v6.4s, #12 // t14a + + mul_mls v4, v29, v23, v0.s[2], v0.s[3] // -> t13a + mul_mla v6, v29, v23, v0.s[3], v0.s[2] // -> t10a + srshr v29.4s, v4.4s, #12 // t13a + neg v6.4s, v6.4s + srshr v23.4s, v6.4s, #12 // t10a + + sqsub v2.4s, v17.4s, v19.4s // t11a + sqadd v17.4s, v17.4s, v19.4s // t8a + sqsub v3.4s, v31.4s, v25.4s // t12a + sqadd v31.4s, v31.4s, v25.4s // t15a + sqadd v19.4s, v21.4s, v23.4s // t9 + sqsub v21.4s, v21.4s, v23.4s // t10 + sqsub v25.4s, v27.4s, v29.4s // t13 + sqadd v27.4s, v27.4s, v29.4s // t14 + + mul_mls v4, v3, v2, v0.s[0], v0.s[0] // -> t11 + mul_mla v6, v3, v2, v0.s[0], v0.s[0] // -> t12 + mul_mls v2, v25, v21, v0.s[0], v0.s[0] // -> t10a + + srshr v4.4s, v4.4s, #12 // t11 + srshr v5.4s, v6.4s, #12 // t12 + mul_mla v6, v25, v21, v0.s[0], v0.s[0] // -> t10a + srshr v2.4s, v2.4s, #12 // t10a + srshr v3.4s, v6.4s, #12 // t13a + + sqadd v6.4s, v16.4s, v31.4s // out0 + sqsub v31.4s, v16.4s, v31.4s // out15 + mov v16.16b, v6.16b + sqadd v23.4s, v30.4s, v17.4s // out7 + sqsub v7.4s, v30.4s, v17.4s // out8 + sqadd v17.4s, v18.4s, v27.4s // out1 + sqsub v30.4s, v18.4s, v27.4s // out14 + sqadd v18.4s, v20.4s, v3.4s // out2 + sqsub v29.4s, v20.4s, v3.4s // out13 + sqadd v3.4s, v28.4s, v19.4s // out6 + sqsub v25.4s, v28.4s, v19.4s // out9 + sqadd v19.4s, v22.4s, v5.4s // out3 + sqsub v28.4s, v22.4s, v5.4s // out12 + sqadd v20.4s, v24.4s, v4.4s // out4 + sqsub v27.4s, v24.4s, v4.4s // out11 + sqadd v21.4s, v26.4s, v2.4s // out5 + sqsub v26.4s, v26.4s, v2.4s // out10 + mov v24.16b, v7.16b + mov v22.16b, v3.16b + + ret +endfunc + +.macro iadst_16 o0, o1, o2, o3, o4, o5, o6, o7, o8, o9, o10, o11, o12, o13, o14, o15 + movrel x16, iadst16_coeffs + ld1 {v0.4s, v1.4s}, [x16], #32 + + mul_mla v2, v31, v16, v0.s[0], v0.s[1] // -> t0 + mul_mls v4, v31, v16, v0.s[1], v0.s[0] // -> t1 + mul_mla v6, v29, v18, v0.s[2], v0.s[3] // -> t2 + srshr v16.4s, v2.4s, #12 // t0 + srshr v31.4s, v4.4s, #12 // t1 + mul_mls v2, v29, v18, v0.s[3], v0.s[2] // -> t3 + mul_mla v4, v27, v20, v1.s[0], v1.s[1] // -> t4 + srshr v18.4s, v6.4s, #12 // t2 + srshr v29.4s, v2.4s, #12 // t3 + mul_mls v6, v27, v20, v1.s[1], v1.s[0] // -> t5 + mul_mla v2, v25, v22, v1.s[2], v1.s[3] // -> t6 + srshr v20.4s, v4.4s, #12 // t4 + srshr v27.4s, v6.4s, #12 // t5 + mul_mls v4, v25, v22, v1.s[3], v1.s[2] // -> t7 + ld1 {v0.4s, v1.4s}, [x16] + movrel x16, idct_coeffs + mul_mla v6, v23, v24, v0.s[0], v0.s[1] // -> t8 + srshr v22.4s, v2.4s, #12 // t6 + srshr v25.4s, v4.4s, #12 // t7 + mul_mls v2, v23, v24, v0.s[1], v0.s[0] // -> t9 + mul_mla v4, v21, v26, v0.s[2], v0.s[3] // -> t10 + srshr v23.4s, v6.4s, #12 // t8 + srshr v24.4s, v2.4s, #12 // t9 + mul_mls v6, v21, v26, v0.s[3], v0.s[2] // -> t11 + mul_mla v2, v19, v28, v1.s[0], v1.s[1] // -> t12 + srshr v21.4s, v4.4s, #12 // t10 + srshr v26.4s, v6.4s, #12 // t11 + mul_mls v4, v19, v28, v1.s[1], v1.s[0] // -> t13 + mul_mla v6, v17, v30, v1.s[2], v1.s[3] // -> t14 + srshr v19.4s, v2.4s, #12 // t12 + srshr v28.4s, v4.4s, #12 // t13 + mul_mls v2, v17, v30, v1.s[3], v1.s[2] // -> t15 + srshr v17.4s, v6.4s, #12 // t14 + srshr v30.4s, v2.4s, #12 // t15 + + ld1 {v0.4s, v1.4s}, [x16] + + sqsub v2.4s, v16.4s, v23.4s // t8a + sqadd v16.4s, v16.4s, v23.4s // t0a + sqsub v3.4s, v31.4s, v24.4s // t9a + sqadd v31.4s, v31.4s, v24.4s // t1a + sqadd v23.4s, v18.4s, v21.4s // t2a + sqsub v18.4s, v18.4s, v21.4s // t10a + sqadd v24.4s, v29.4s, v26.4s // t3a + sqsub v29.4s, v29.4s, v26.4s // t11a + sqadd v21.4s, v20.4s, v19.4s // t4a + sqsub v20.4s, v20.4s, v19.4s // t12a + sqadd v26.4s, v27.4s, v28.4s // t5a + sqsub v27.4s, v27.4s, v28.4s // t13a + sqadd v19.4s, v22.4s, v17.4s // t6a + sqsub v22.4s, v22.4s, v17.4s // t14a + sqadd v28.4s, v25.4s, v30.4s // t7a + sqsub v25.4s, v25.4s, v30.4s // t15a + + mul_mla v4, v2, v3, v1.s[1], v1.s[0] // -> t8 + mul_mls v6, v2, v3, v1.s[0], v1.s[1] // -> t9 + mul_mla v2, v18, v29, v1.s[3], v1.s[2] // -> t10 + srshr v17.4s, v4.4s, #12 // t8 + srshr v30.4s, v6.4s, #12 // t9 + mul_mls v4, v18, v29, v1.s[2], v1.s[3] // -> t11 + mul_mls v6, v27, v20, v1.s[1], v1.s[0] // -> t12 + srshr v18.4s, v2.4s, #12 // t10 + srshr v29.4s, v4.4s, #12 // t11 + mul_mla v2, v27, v20, v1.s[0], v1.s[1] // -> t13 + mul_mls v4, v25, v22, v1.s[3], v1.s[2] // -> t14 + srshr v27.4s, v6.4s, #12 // t12 + srshr v20.4s, v2.4s, #12 // t13 + mul_mla v6, v25, v22, v1.s[2], v1.s[3] // -> t15 + srshr v25.4s, v4.4s, #12 // t14 + srshr v22.4s, v6.4s, #12 // t15 + + sqsub v2.4s, v16.4s, v21.4s // t4 + sqadd v16.4s, v16.4s, v21.4s // t0 + sqsub v3.4s, v31.4s, v26.4s // t5 + sqadd v31.4s, v31.4s, v26.4s // t1 + sqadd v21.4s, v23.4s, v19.4s // t2 + sqsub v23.4s, v23.4s, v19.4s // t6 + sqadd v26.4s, v24.4s, v28.4s // t3 + sqsub v24.4s, v24.4s, v28.4s // t7 + sqadd v19.4s, v17.4s, v27.4s // t8a + sqsub v17.4s, v17.4s, v27.4s // t12a + sqadd v28.4s, v30.4s, v20.4s // t9a + sqsub v30.4s, v30.4s, v20.4s // t13a + sqadd v27.4s, v18.4s, v25.4s // t10a + sqsub v18.4s, v18.4s, v25.4s // t14a + sqadd v20.4s, v29.4s, v22.4s // t11a + sqsub v29.4s, v29.4s, v22.4s // t15a + + mul_mla v4, v2, v3, v0.s[3], v0.s[2] // -> t4a + mul_mls v6, v2, v3, v0.s[2], v0.s[3] // -> t5a + mul_mls v2, v24, v23, v0.s[3], v0.s[2] // -> t6a + srshr v22.4s, v4.4s, #12 // t4a + srshr v25.4s, v6.4s, #12 // t5a + mul_mla v4, v24, v23, v0.s[2], v0.s[3] // -> t7a + mul_mla v6, v17, v30, v0.s[3], v0.s[2] // -> t12 + srshr v24.4s, v2.4s, #12 // t6a + srshr v23.4s, v4.4s, #12 // t7a + mul_mls v2, v17, v30, v0.s[2], v0.s[3] // -> t13 + mul_mls v4, v29, v18, v0.s[3], v0.s[2] // -> t14 + srshr v17.4s, v6.4s, #12 // t12 + mul_mla v6, v29, v18, v0.s[2], v0.s[3] // -> t15 + srshr v29.4s, v2.4s, #12 // t13 + srshr v30.4s, v4.4s, #12 // t14 + srshr v18.4s, v6.4s, #12 // t15 + + sqsub v2.4s, v16.4s, v21.4s // t2a +.ifc \o0, v16 + sqadd \o0\().4s, v16.4s, v21.4s // out0 + sqsub v21.4s, v31.4s, v26.4s // t3a + sqadd \o15\().4s, v31.4s, v26.4s // out15 +.else + sqadd v4.4s, v16.4s, v21.4s // out0 + sqsub v21.4s, v31.4s, v26.4s // t3a + sqadd \o15\().4s, v31.4s, v26.4s // out15 + mov \o0\().16b, v4.16b +.endif + sqneg \o15\().4s, \o15\().4s // out15 + + sqsub v3.4s, v29.4s, v18.4s // t15a + sqadd \o13\().4s, v29.4s, v18.4s // out13 + sqadd \o2\().4s, v17.4s, v30.4s // out2 + sqsub v26.4s, v17.4s, v30.4s // t14a + sqneg \o13\().4s, \o13\().4s // out13 + + sqadd \o1\().4s, v19.4s, v27.4s // out1 + sqsub v27.4s, v19.4s, v27.4s // t10 + sqadd \o14\().4s, v28.4s, v20.4s // out14 + sqsub v20.4s, v28.4s, v20.4s // t11 + sqneg \o1\().4s, \o1\().4s // out1 + + sqadd \o3\().4s, v22.4s, v24.4s // out3 + sqsub v22.4s, v22.4s, v24.4s // t6 + sqadd \o12\().4s, v25.4s, v23.4s // out12 + sqsub v23.4s, v25.4s, v23.4s // t7 + sqneg \o3\().4s, \o3\().4s // out3 + + mul_mls v24, v2, v21, v0.s[0], v0.s[0] // -> out8 (v24 or v23) + mul_mla v4, v2, v21, v0.s[0], v0.s[0] // -> out7 (v23 or v24) + mul_mla v6, v26, v3, v0.s[0], v0.s[0] // -> out5 (v21 or v26) + + srshr v24.4s, v24.4s, #12 // out8 + srshr v4.4s, v4.4s, #12 // out7 + srshr v5.4s, v6.4s, #12 // out5 + mul_mls v6, v26, v3, v0.s[0], v0.s[0] // -> out10 (v26 or v21) + mul_mla v2, v22, v23, v0.s[0], v0.s[0] // -> out4 (v20 or v27) + srshr v26.4s, v6.4s, #12 // out10 + + mul_mls v6, v22, v23, v0.s[0], v0.s[0] // -> out11 (v27 or v20) + mul_mla v22, v27, v20, v0.s[0], v0.s[0] // -> out6 (v22 or v25) + mul_mls v21, v27, v20, v0.s[0], v0.s[0] // -> out9 (v25 or v22) + + srshr \o4\().4s, v2.4s, #12 // out4 + srshr v6.4s, v6.4s, #12 // out11 + srshr v7.4s, v21.4s, #12 // out9 + srshr \o6\().4s, v22.4s, #12 // out6 + +.ifc \o8, v23 + mov \o8\().16b, v24.16b + mov \o10\().16b, v26.16b +.endif + + sqneg \o7\().4s, v4.4s // out7 + sqneg \o5\().4s, v5.4s // out5 + sqneg \o11\().4s, v6.4s // out11 + sqneg \o9\().4s, v7.4s // out9 +.endm + +function inv_adst_4s_x16_neon + iadst_16 v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27, v28, v29, v30, v31 + ret +endfunc + +function inv_flipadst_4s_x16_neon + iadst_16 v31, v30, v29, v28, v27, v26, v25, v24, v23, v22, v21, v20, v19, v18, v17, v16 + ret +endfunc + +function inv_identity_4s_x16_neon + movz w16, #2*(5793-4096)*8, lsl #16 + dup v0.2s, w16 +.irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + sqrdmulh v2.4s, v\i\().4s, v0.s[0] + sqadd v\i\().4s, v\i\().4s, v\i\().4s + sqadd v\i\().4s, v\i\().4s, v2.4s +.endr + ret +endfunc + +.macro identity_4x16_shift1 c +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s + sqrdmulh v3.4s, \i, \c + srshr v3.4s, v3.4s, #1 + sqadd \i, \i, v3.4s +.endr +.endm + +.macro identity_4x16 c +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s + sqrdmulh v3.4s, \i, \c + sqadd \i, \i, \i + sqadd \i, \i, v3.4s +.endr +.endm + +.macro def_horz_16 scale=0, shift=2, suffix +function inv_txfm_horz\suffix\()_16x4_neon + mov x14, x30 + movi v7.4s, #0 +.if \scale + movz w16, #2896*8, lsl #16 + dup v0.2s, w16 +.endif +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s + ld1 {\i}, [x7] + st1 {v7.4s}, [x7], x8 +.endr +.if \scale + scale_input .4s, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23 + scale_input .4s, v0.s[0], v24, v25, v26, v27, v28, v29, v30, v31 +.endif + blr x4 + sqrshrn v16.4h, v16.4s, #\shift + sqrshrn v17.4h, v17.4s, #\shift + sqrshrn v18.4h, v18.4s, #\shift + sqrshrn v19.4h, v19.4s, #\shift + sqrshrn2 v16.8h, v20.4s, #\shift + sqrshrn2 v17.8h, v21.4s, #\shift + sqrshrn2 v18.8h, v22.4s, #\shift + sqrshrn2 v19.8h, v23.4s, #\shift + sqrshrn v20.4h, v24.4s, #\shift + sqrshrn v21.4h, v25.4s, #\shift + sqrshrn v22.4h, v26.4s, #\shift + sqrshrn v23.4h, v27.4s, #\shift + sqrshrn2 v20.8h, v28.4s, #\shift + sqrshrn2 v21.8h, v29.4s, #\shift + sqrshrn2 v22.8h, v30.4s, #\shift + sqrshrn2 v23.8h, v31.4s, #\shift + transpose_4x8h v16, v17, v18, v19, v4, v5, v6, v7 + transpose_4x8h v20, v21, v22, v23, v4, v5, v6, v7 + +.irp i, v16.8h, v20.8h, v17.8h, v21.8h, v18.8h, v22.8h, v19.8h, v23.8h + st1 {\i}, [x6], #16 +.endr + + br x14 +endfunc +.endm + +def_horz_16 scale=0, shift=2 +def_horz_16 scale=1, shift=1, suffix=_scale + +function inv_txfm_add_vert_8x16_neon + mov x14, x30 +.irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + ld1 {v\i\().8h}, [x7], x8 +.endr + blr x5 + load_add_store_8x16 x6, x7 + br x14 +endfunc + +function inv_txfm_add_16x16_neon + mov x15, x30 + sub sp, sp, #512 + ldrh w12, [x13], #2 +.irp i, 0, 4, 8, 12 + add x6, sp, #(\i*16*2) +.if \i > 0 + mov w8, #(16 - \i) + cmp w3, w12 + b.lt 1f +.if \i < 12 + ldrh w12, [x13], #2 +.endif +.endif + add x7, x2, #(\i*4) + mov x8, #16*4 + bl inv_txfm_horz_16x4_neon +.endr + b 3f +1: + movi v4.8h, #0 + movi v5.8h, #0 + movi v6.8h, #0 + movi v7.8h, #0 +2: + subs w8, w8, #4 +.rept 2 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64 +.endr + b.gt 2b +3: +.irp i, 0, 8 + add x6, x0, #(\i*2) + add x7, sp, #(\i*2) + mov x8, #32 + bl inv_txfm_add_vert_8x16_neon +.endr + + add sp, sp, #512 + br x15 +endfunc + +const eob_16x16 + .short 10, 36, 78, 256 +endconst + +const eob_16x16_identity + .short 4, 8, 12, 256 +endconst + +.macro def_fn_16x16 txfm1, txfm2 +function inv_txfm_add_\txfm1\()_\txfm2\()_16x16_16bpc_neon, export=1 +.ifc \txfm1\()_\txfm2, dct_dct + idct_dc 16, 16, 2 +.endif + adr x4, inv_\txfm1\()_4s_x16_neon + movrel x5, X(inv_\txfm2\()_8h_x16_neon) +.ifc \txfm1, identity +.ifc \txfm2, identity + movrel x13, eob_16x16 +.else + movrel x13, eob_16x16_identity +.endif +.else +.ifc \txfm2, identity + movrel x13, eob_16x16_identity +.else + movrel x13, eob_16x16 +.endif +.endif + b inv_txfm_add_16x16_neon +endfunc +.endm + +def_fn_16x16 dct, dct +def_fn_16x16 identity, identity +def_fn_16x16 dct, adst +def_fn_16x16 dct, flipadst +def_fn_16x16 dct, identity +def_fn_16x16 adst, dct +def_fn_16x16 adst, adst +def_fn_16x16 adst, flipadst +def_fn_16x16 flipadst, dct +def_fn_16x16 flipadst, adst +def_fn_16x16 flipadst, flipadst +def_fn_16x16 identity, dct + +function inv_txfm_add_16x4_neon + mov x15, x30 + movi v4.4s, #0 + +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s + ld1 {\i}, [x2] + st1 {v4.4s}, [x2], #16 +.endr + + blr x4 + + sqrshrn v16.4h, v16.4s, #1 + sqrshrn v17.4h, v17.4s, #1 + sqrshrn v18.4h, v18.4s, #1 + sqrshrn v19.4h, v19.4s, #1 + sqrshrn2 v16.8h, v20.4s, #1 + sqrshrn2 v17.8h, v21.4s, #1 + sqrshrn2 v18.8h, v22.4s, #1 + sqrshrn2 v19.8h, v23.4s, #1 + transpose_4x8h v16, v17, v18, v19, v2, v3, v4, v5 + blr x5 + mov x6, x0 + load_add_store_8x4 x6, x7 + + sqrshrn v16.4h, v24.4s, #1 + sqrshrn v17.4h, v25.4s, #1 + sqrshrn v18.4h, v26.4s, #1 + sqrshrn v19.4h, v27.4s, #1 + sqrshrn2 v16.8h, v28.4s, #1 + sqrshrn2 v17.8h, v29.4s, #1 + sqrshrn2 v18.8h, v30.4s, #1 + sqrshrn2 v19.8h, v31.4s, #1 + transpose_4x8h v16, v17, v18, v19, v2, v3, v4, v5 + blr x5 + add x6, x0, #16 + load_add_store_8x4 x6, x7 + + br x15 +endfunc + +function inv_txfm_add_4x16_neon + ldrh w12, [x13, #4] + mov x15, x30 + + mov x11, #64 + + cmp w3, w12 + ldrh w12, [x13, #2] + b.lt 1f + + add x6, x2, #48 + movi v2.4s, #0 +.irp i, v16.4s, v17.4s, v18.4s, v19.4s + ld1 {\i}, [x6] + st1 {v2.4s}, [x6], x11 +.endr + blr x4 + rshrn v28.4h, v16.4s, #1 + rshrn v29.4h, v17.4s, #1 + rshrn v30.4h, v18.4s, #1 + rshrn v31.4h, v19.4s, #1 + transpose_4x4h v28, v29, v30, v31, v4, v5, v6, v7 + + b 2f +1: +.irp i, v28.4h, v29.4h, v30.4h, v31.4h + movi \i, #0 +.endr +2: + cmp w3, w12 + ldrh w12, [x13, #0] + b.lt 1f + + add x6, x2, #32 + movi v2.4s, #0 +.irp i, v16.4s, v17.4s, v18.4s, v19.4s + ld1 {\i}, [x6] + st1 {v2.4s}, [x6], x11 +.endr + blr x4 + rshrn v24.4h, v16.4s, #1 + rshrn v25.4h, v17.4s, #1 + rshrn v26.4h, v18.4s, #1 + rshrn v27.4h, v19.4s, #1 + transpose_4x4h v24, v25, v26, v27, v4, v5, v6, v7 + + b 2f +1: +.irp i, v24.4h, v25.4h, v26.4h, v27.4h + movi \i, #0 +.endr +2: + cmp w3, w12 + b.lt 1f + + add x6, x2, #16 + movi v2.4s, #0 +.irp i, v16.4s, v17.4s, v18.4s, v19.4s + ld1 {\i}, [x6] + st1 {v2.4s}, [x6], x11 +.endr + blr x4 + rshrn v20.4h, v16.4s, #1 + rshrn v21.4h, v17.4s, #1 + rshrn v22.4h, v18.4s, #1 + rshrn v23.4h, v19.4s, #1 + transpose_4x4h v20, v21, v22, v23, v4, v5, v6, v7 + + b 2f +1: +.irp i, v20.4h, v21.4h, v22.4h, v23.4h + movi \i, #0 +.endr +2: + + movi v2.4s, #0 +.irp i, v16.4s, v17.4s, v18.4s, v19.4s + ld1 {\i}, [x2] + st1 {v2.4s}, [x2], x11 +.endr + blr x4 + rshrn v16.4h, v16.4s, #1 + rshrn v17.4h, v17.4s, #1 + rshrn v18.4h, v18.4s, #1 + rshrn v19.4h, v19.4s, #1 + transpose_4x8h v16, v17, v18, v19, v4, v5, v6, v7 + + blr x5 + + load_add_store_4x16 x0, x6 + + br x15 +endfunc + +const eob_4x16 + .short 13, 29, 45, 64 +endconst + +const eob_4x16_identity1 + .short 16, 32, 48, 64 +endconst + +const eob_4x16_identity2 + .short 4, 8, 12, 64 +endconst + +.macro def_fn_416 w, h, txfm1, txfm2 +function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_16bpc_neon, export=1 +.ifc \txfm1\()_\txfm2, dct_dct + idct_dc \w, \h, 1 +.endif +.if \w == 4 + adr x4, inv_\txfm1\()_4s_x\w\()_neon + movrel x5, X(inv_\txfm2\()_4h_x\h\()_neon) +.ifc \txfm1, identity +.ifc \txfm2, identity + movrel x13, eob_4x16 +.else + movrel x13, eob_4x16_identity1 +.endif +.else +.ifc \txfm2, identity + movrel x13, eob_4x16_identity2 +.else + movrel x13, eob_4x16 +.endif +.endif +.else + adr x4, inv_\txfm1\()_4s_x\w\()_neon + movrel x5, X(inv_\txfm2\()_8h_x\h\()_neon) +.endif + b inv_txfm_add_\w\()x\h\()_neon +endfunc +.endm + +.macro def_fns_416 w, h +def_fn_416 \w, \h, dct, dct +def_fn_416 \w, \h, identity, identity +def_fn_416 \w, \h, dct, adst +def_fn_416 \w, \h, dct, flipadst +def_fn_416 \w, \h, dct, identity +def_fn_416 \w, \h, adst, dct +def_fn_416 \w, \h, adst, adst +def_fn_416 \w, \h, adst, flipadst +def_fn_416 \w, \h, flipadst, dct +def_fn_416 \w, \h, flipadst, adst +def_fn_416 \w, \h, flipadst, flipadst +def_fn_416 \w, \h, identity, dct +def_fn_416 \w, \h, adst, identity +def_fn_416 \w, \h, flipadst, identity +def_fn_416 \w, \h, identity, adst +def_fn_416 \w, \h, identity, flipadst +.endm + +def_fns_416 4, 16 +def_fns_416 16, 4 + + +function inv_txfm_add_16x8_neon + mov x15, x30 + stp d8, d9, [sp, #-0x40]! + stp d10, d11, [sp, #0x10] + stp d12, d13, [sp, #0x20] + stp d14, d15, [sp, #0x30] + + cmp w3, w13 + mov x11, #32 + b.lt 1f + + movi v4.4s, #0 + movz w16, #2896*8, lsl #16 + dup v0.2s, w16 + + add x6, x2, #16 +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s + ld1 {\i}, [x6] + st1 {v4.4s}, [x6], x11 +.endr + + scale_input .4s, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23 + scale_input .4s, v0.s[0], v24, v25, v26, v27, v28, v29, v30, v31 + blr x4 + + sqrshrn v8.4h, v16.4s, #1 + sqrshrn v9.4h, v17.4s, #1 + sqrshrn v10.4h, v18.4s, #1 + sqrshrn v11.4h, v19.4s, #1 + sqrshrn2 v8.8h, v20.4s, #1 + sqrshrn2 v9.8h, v21.4s, #1 + sqrshrn2 v10.8h, v22.4s, #1 + sqrshrn2 v11.8h, v23.4s, #1 + sqrshrn v12.4h, v24.4s, #1 + sqrshrn v13.4h, v25.4s, #1 + sqrshrn v14.4h, v26.4s, #1 + sqrshrn v15.4h, v27.4s, #1 + sqrshrn2 v12.8h, v28.4s, #1 + sqrshrn2 v13.8h, v29.4s, #1 + sqrshrn2 v14.8h, v30.4s, #1 + sqrshrn2 v15.8h, v31.4s, #1 + + transpose_4x8h v8, v9, v10, v11, v2, v3, v4, v5 + transpose_4x8h v12, v13, v14, v15, v2, v3, v4, v5 + + b 2f +1: +.irp i, v8.8h, v9.8h, v10.8h, v11.8h, v12.8h, v13.8h, v14.8h, v15.8h + movi \i, #0 +.endr +2: + movz w16, #2896*8, lsl #16 + dup v0.2s, w16 + + movi v4.4s, #0 +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s + ld1 {\i}, [x2] + st1 {v4.4s}, [x2], x11 +.endr + + scale_input .4s, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23 + scale_input .4s, v0.s[0], v24, v25, v26, v27, v28, v29, v30, v31 + blr x4 + + sqrshrn v16.4h, v16.4s, #1 + sqrshrn v17.4h, v17.4s, #1 + sqrshrn v18.4h, v18.4s, #1 + sqrshrn v19.4h, v19.4s, #1 + sqrshrn2 v16.8h, v20.4s, #1 + sqrshrn2 v17.8h, v21.4s, #1 + sqrshrn2 v18.8h, v22.4s, #1 + sqrshrn2 v19.8h, v23.4s, #1 + + mov v20.16b, v8.16b + mov v21.16b, v9.16b + mov v22.16b, v10.16b + mov v23.16b, v11.16b + + transpose_4x8h v16, v17, v18, v19, v2, v3, v4, v5 + + sqrshrn v8.4h, v24.4s, #1 + sqrshrn v9.4h, v25.4s, #1 + sqrshrn v10.4h, v26.4s, #1 + sqrshrn v11.4h, v27.4s, #1 + sqrshrn2 v8.8h, v28.4s, #1 + sqrshrn2 v9.8h, v29.4s, #1 + sqrshrn2 v10.8h, v30.4s, #1 + sqrshrn2 v11.8h, v31.4s, #1 + + transpose_4x8h v8, v9, v10, v11, v2, v3, v4, v5 + + blr x5 + + mov x6, x0 + load_add_store_8x8 x6, x7 + + mov v16.16b, v8.16b + mov v17.16b, v9.16b + mov v18.16b, v10.16b + mov v19.16b, v11.16b + mov v20.16b, v12.16b + mov v21.16b, v13.16b + mov v22.16b, v14.16b + mov v23.16b, v15.16b + + blr x5 + + add x0, x0, #16 + load_add_store_8x8 x0, x7 + + ldp d14, d15, [sp, #0x30] + ldp d12, d13, [sp, #0x20] + ldp d10, d11, [sp, #0x10] + ldp d8, d9, [sp], 0x40 + br x15 +endfunc + +function inv_txfm_add_8x16_neon + mov x15, x30 + stp d8, d9, [sp, #-0x20]! + stp d10, d11, [sp, #0x10] + ldrh w12, [x13, #4] + + mov x11, #64 + + cmp w3, w12 + ldrh w12, [x13, #2] + b.lt 1f + + add x6, x2, #48 + movi v4.4s, #0 + movz w16, #2896*8, lsl #16 + dup v0.2s, w16 +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s + ld1 {\i}, [x6] + st1 {v4.4s}, [x6], x11 +.endr + scale_input .4s, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23 + blr x4 + + sqrshrn v28.4h, v16.4s, #1 + sqrshrn v29.4h, v17.4s, #1 + sqrshrn v30.4h, v18.4s, #1 + sqrshrn v31.4h, v19.4s, #1 + sqrshrn2 v28.8h, v20.4s, #1 + sqrshrn2 v29.8h, v21.4s, #1 + sqrshrn2 v30.8h, v22.4s, #1 + sqrshrn2 v31.8h, v23.4s, #1 + transpose_4x8h v28, v29, v30, v31, v2, v3, v4, v5 + + b 2f + +1: +.irp i, v28.8h, v29.8h, v30.8h, v31.8h + movi \i, #0 +.endr + +2: + cmp w3, w12 + ldrh w12, [x13, #0] + b.lt 1f + + add x6, x2, #32 + movi v4.4s, #0 + movz w16, #2896*8, lsl #16 + dup v0.2s, w16 +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s + ld1 {\i}, [x6] + st1 {v4.4s}, [x6], x11 +.endr + scale_input .4s, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23 + blr x4 + + sqrshrn v24.4h, v16.4s, #1 + sqrshrn v25.4h, v17.4s, #1 + sqrshrn v26.4h, v18.4s, #1 + sqrshrn v27.4h, v19.4s, #1 + sqrshrn2 v24.8h, v20.4s, #1 + sqrshrn2 v25.8h, v21.4s, #1 + sqrshrn2 v26.8h, v22.4s, #1 + sqrshrn2 v27.8h, v23.4s, #1 + transpose_4x8h v24, v25, v26, v27, v2, v3, v4, v5 + + b 2f + +1: +.irp i, v24.8h, v25.8h, v26.8h, v27.8h + movi \i, #0 +.endr + +2: + cmp w3, w12 + b.lt 1f + + add x6, x2, #16 + movi v4.4s, #0 + movz w16, #2896*8, lsl #16 + dup v0.2s, w16 +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s + ld1 {\i}, [x6] + st1 {v4.4s}, [x6], x11 +.endr + scale_input .4s, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23 + blr x4 + + sqrshrn v8.4h, v16.4s, #1 + sqrshrn v9.4h, v17.4s, #1 + sqrshrn v10.4h, v18.4s, #1 + sqrshrn v11.4h, v19.4s, #1 + sqrshrn2 v8.8h, v20.4s, #1 + sqrshrn2 v9.8h, v21.4s, #1 + sqrshrn2 v10.8h, v22.4s, #1 + sqrshrn2 v11.8h, v23.4s, #1 + transpose_4x8h v8, v9, v10, v11, v2, v3, v4, v5 + + b 2f + +1: +.irp i, v8.8h, v9.8h, v10.8h, v11.8h + movi \i, #0 +.endr + +2: + movi v4.4s, #0 + movz w16, #2896*8, lsl #16 + dup v0.2s, w16 +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s + ld1 {\i}, [x2] + st1 {v4.4s}, [x2], x11 +.endr + scale_input .4s, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23 + blr x4 + + sqrshrn v16.4h, v16.4s, #1 + sqrshrn v17.4h, v17.4s, #1 + sqrshrn v18.4h, v18.4s, #1 + sqrshrn v19.4h, v19.4s, #1 + sqrshrn2 v16.8h, v20.4s, #1 + sqrshrn2 v17.8h, v21.4s, #1 + sqrshrn2 v18.8h, v22.4s, #1 + sqrshrn2 v19.8h, v23.4s, #1 + transpose_4x8h v16, v17, v18, v19, v2, v3, v4, v5 + + mov v20.16b, v8.16b + mov v21.16b, v9.16b + mov v22.16b, v10.16b + mov v23.16b, v11.16b + + blr x5 + + load_add_store_8x16 x0, x6 + + ldp d10, d11, [sp, #0x10] + ldp d8, d9, [sp], 0x20 + + br x15 +endfunc + +const eob_8x16 + .short 10, 43, 75, 128 +endconst + +const eob_8x16_identity1 + .short 4, 64, 96, 128 +endconst + +const eob_8x16_identity2 + .short 4, 8, 12, 128 +endconst + +.macro def_fn_816 w, h, txfm1, txfm2 +function inv_txfm_add_\txfm1\()_\txfm2\()_\w\()x\h\()_16bpc_neon, export=1 +.ifc \txfm1\()_\txfm2, dct_dct + idct_dc \w, \h, 1 +.endif + adr x4, inv_\txfm1\()_4s_x\w\()_neon + movrel x5, X(inv_\txfm2\()_8h_x\h\()_neon) +.ifc \txfm1, identity +.ifc \txfm2, identity + movrel x13, eob_8x16 +.else + movrel x13, eob_8x16_identity1 +.endif +.else +.ifc \txfm2, identity + movrel x13, eob_8x16_identity2 +.else + movrel x13, eob_8x16 +.endif +.endif +.if \h == 8 + ldrh w13, [x13] +.endif + b inv_txfm_add_\w\()x\h\()_neon +endfunc +.endm + +.macro def_fns_816 w, h +def_fn_816 \w, \h, dct, dct +def_fn_816 \w, \h, identity, identity +def_fn_816 \w, \h, dct, adst +def_fn_816 \w, \h, dct, flipadst +def_fn_816 \w, \h, dct, identity +def_fn_816 \w, \h, adst, dct +def_fn_816 \w, \h, adst, adst +def_fn_816 \w, \h, adst, flipadst +def_fn_816 \w, \h, flipadst, dct +def_fn_816 \w, \h, flipadst, adst +def_fn_816 \w, \h, flipadst, flipadst +def_fn_816 \w, \h, identity, dct +def_fn_816 \w, \h, adst, identity +def_fn_816 \w, \h, flipadst, identity +def_fn_816 \w, \h, identity, adst +def_fn_816 \w, \h, identity, flipadst +.endm + +def_fns_816 8, 16 +def_fns_816 16, 8 + +function inv_dct32_odd_4s_x16_neon + movrel x16, idct_coeffs, 4*16 + ld1 {v0.4s, v1.4s}, [x16], #32 + + mul_mls v2, v16, v31, v0.s[0], v0.s[1] // -> t16a + mul_mla v4, v16, v31, v0.s[1], v0.s[0] // -> t31a + mul_mls v6, v24, v23, v0.s[2], v0.s[3] // -> t17a + srshr v16.4s, v2.4s, #12 // t16a + srshr v31.4s, v4.4s, #12 // t31a + mul_mla v2, v24, v23, v0.s[3], v0.s[2] // -> t30a + mul_mls v4, v20, v27, v1.s[0], v1.s[1] // -> t18a + srshr v24.4s, v6.4s, #12 // t17a + srshr v23.4s, v2.4s, #12 // t30a + mul_mla v6, v20, v27, v1.s[1], v1.s[0] // -> t29a + mul_mls v2, v28, v19, v1.s[2], v1.s[3] // -> t19a + srshr v20.4s, v4.4s, #12 // t18a + srshr v27.4s, v6.4s, #12 // t29a + mul_mla v4, v28, v19, v1.s[3], v1.s[2] // -> t28a + ld1 {v0.4s, v1.4s}, [x16] + sub x16, x16, #4*24 + mul_mls v6, v18, v29, v0.s[0], v0.s[1] // -> t20a + srshr v28.4s, v2.4s, #12 // t19a + srshr v19.4s, v4.4s, #12 // t28a + mul_mla v2, v18, v29, v0.s[1], v0.s[0] // -> t27a + mul_mls v4, v26, v21, v0.s[2], v0.s[3] // -> t21a + srshr v18.4s, v6.4s, #12 // t20a + srshr v29.4s, v2.4s, #12 // t27a + mul_mla v6, v26, v21, v0.s[3], v0.s[2] // -> t26a + mul_mls v2, v22, v25, v1.s[0], v1.s[1] // -> t22a + srshr v26.4s, v4.4s, #12 // t21a + srshr v21.4s, v6.4s, #12 // t26a + mul_mla v4, v22, v25, v1.s[1], v1.s[0] // -> t25a + mul_mls v6, v30, v17, v1.s[2], v1.s[3] // -> t23a + srshr v22.4s, v2.4s, #12 // t22a + srshr v25.4s, v4.4s, #12 // t25a + mul_mla v2, v30, v17, v1.s[3], v1.s[2] // -> t24a + srshr v30.4s, v6.4s, #12 // t23a + srshr v17.4s, v2.4s, #12 // t24a + + ld1 {v0.4s, v1.4s}, [x16] + + sqsub v2.4s, v16.4s, v24.4s // t17 + sqadd v16.4s, v16.4s, v24.4s // t16 + sqsub v3.4s, v31.4s, v23.4s // t30 + sqadd v31.4s, v31.4s, v23.4s // t31 + sqsub v24.4s, v28.4s, v20.4s // t18 + sqadd v28.4s, v28.4s, v20.4s // t19 + sqadd v23.4s, v18.4s, v26.4s // t20 + sqsub v18.4s, v18.4s, v26.4s // t21 + sqsub v20.4s, v30.4s, v22.4s // t22 + sqadd v30.4s, v30.4s, v22.4s // t23 + sqadd v26.4s, v17.4s, v25.4s // t24 + sqsub v17.4s, v17.4s, v25.4s // t25 + sqsub v22.4s, v29.4s, v21.4s // t26 + sqadd v29.4s, v29.4s, v21.4s // t27 + sqadd v25.4s, v19.4s, v27.4s // t28 + sqsub v19.4s, v19.4s, v27.4s // t29 + + mul_mls v4, v3, v2, v1.s[0], v1.s[1] // -> t17a + mul_mla v6, v3, v2, v1.s[1], v1.s[0] // -> t30a + mul_mla v2, v19, v24, v1.s[1], v1.s[0] // -> t18a + srshr v21.4s, v4.4s, #12 // t17a + srshr v27.4s, v6.4s, #12 // t30a + neg v2.4s, v2.4s // -> t18a + mul_mls v4, v19, v24, v1.s[0], v1.s[1] // -> t29a + mul_mls v6, v22, v18, v1.s[2], v1.s[3] // -> t21a + srshr v19.4s, v2.4s, #12 // t18a + srshr v24.4s, v4.4s, #12 // t29a + mul_mla v2, v22, v18, v1.s[3], v1.s[2] // -> t26a + mul_mla v4, v17, v20, v1.s[3], v1.s[2] // -> t22a + srshr v22.4s, v6.4s, #12 // t21a + srshr v18.4s, v2.4s, #12 // t26a + neg v4.4s, v4.4s // -> t22a + mul_mls v6, v17, v20, v1.s[2], v1.s[3] // -> t25a + srshr v17.4s, v4.4s, #12 // t22a + srshr v20.4s, v6.4s, #12 // t25a + + sqsub v2.4s, v27.4s, v24.4s // t29 + sqadd v27.4s, v27.4s, v24.4s // t30 + sqsub v3.4s, v21.4s, v19.4s // t18 + sqadd v21.4s, v21.4s, v19.4s // t17 + sqsub v24.4s, v16.4s, v28.4s // t19a + sqadd v16.4s, v16.4s, v28.4s // t16a + sqsub v19.4s, v30.4s, v23.4s // t20a + sqadd v30.4s, v30.4s, v23.4s // t23a + sqsub v28.4s, v17.4s, v22.4s // t21 + sqadd v17.4s, v17.4s, v22.4s // t22 + sqadd v23.4s, v26.4s, v29.4s // t24a + sqsub v26.4s, v26.4s, v29.4s // t27a + sqadd v22.4s, v20.4s, v18.4s // t25 + sqsub v20.4s, v20.4s, v18.4s // t26 + sqsub v29.4s, v31.4s, v25.4s // t28a + sqadd v31.4s, v31.4s, v25.4s // t31a + + mul_mls v4, v2, v3, v0.s[2], v0.s[3] // -> t18a + mul_mla v6, v2, v3, v0.s[3], v0.s[2] // -> t29a + mul_mls v2, v29, v24, v0.s[2], v0.s[3] // -> t19 + srshr v18.4s, v4.4s, #12 // t18a + srshr v25.4s, v6.4s, #12 // t29a + mul_mla v4, v29, v24, v0.s[3], v0.s[2] // -> t28 + mul_mla v6, v26, v19, v0.s[3], v0.s[2] // -> t20 + srshr v29.4s, v2.4s, #12 // t19 + srshr v24.4s, v4.4s, #12 // t28 + neg v6.4s, v6.4s // -> t20 + mul_mls v2, v26, v19, v0.s[2], v0.s[3] // -> t27 + mul_mla v4, v20, v28, v0.s[3], v0.s[2] // -> t21a + srshr v26.4s, v6.4s, #12 // t20 + srshr v19.4s, v2.4s, #12 // t27 + neg v4.4s, v4.4s // -> t21a + mul_mls v6, v20, v28, v0.s[2], v0.s[3] // -> t26a + srshr v20.4s, v4.4s, #12 // t21a + srshr v28.4s, v6.4s, #12 // t26a + + sqsub v2.4s, v16.4s, v30.4s // t23 + sqadd v16.4s, v16.4s, v30.4s // t16 = out16 + sqsub v3.4s, v31.4s, v23.4s // t24 + sqadd v31.4s, v31.4s, v23.4s // t31 = out31 + sqsub v23.4s, v21.4s, v17.4s // t22a + sqadd v17.4s, v21.4s, v17.4s // t17a = out17 + sqadd v30.4s, v27.4s, v22.4s // t30a = out30 + sqsub v21.4s, v27.4s, v22.4s // t25a + sqsub v27.4s, v18.4s, v20.4s // t21 + sqadd v18.4s, v18.4s, v20.4s // t18 = out18 + sqadd v4.4s, v29.4s, v26.4s // t19a = out19 + sqsub v26.4s, v29.4s, v26.4s // t20a + sqadd v29.4s, v25.4s, v28.4s // t29 = out29 + sqsub v25.4s, v25.4s, v28.4s // t26 + sqadd v28.4s, v24.4s, v19.4s // t28a = out28 + sqsub v24.4s, v24.4s, v19.4s // t27a + mov v19.16b, v4.16b // out19 + + mul_mls v4, v24, v26, v0.s[0], v0.s[0] // -> t20 + mul_mla v6, v24, v26, v0.s[0], v0.s[0] // -> t27 + srshr v20.4s, v4.4s, #12 // t20 + srshr v22.4s, v6.4s, #12 // t27 + + mul_mla v4, v25, v27, v0.s[0], v0.s[0] // -> t26a + mul_mls v6, v25, v27, v0.s[0], v0.s[0] // -> t21a + mov v27.16b, v22.16b // t27 + srshr v26.4s, v4.4s, #12 // t26a + + mul_mls v24, v21, v23, v0.s[0], v0.s[0] // -> t22 + mul_mla v4, v21, v23, v0.s[0], v0.s[0] // -> t25 + srshr v21.4s, v6.4s, #12 // t21a + srshr v22.4s, v24.4s, #12 // t22 + srshr v25.4s, v4.4s, #12 // t25 + + mul_mls v4, v3, v2, v0.s[0], v0.s[0] // -> t23a + mul_mla v6, v3, v2, v0.s[0], v0.s[0] // -> t24a + srshr v23.4s, v4.4s, #12 // t23a + srshr v24.4s, v6.4s, #12 // t24a + + ret +endfunc + +.macro def_horz_32 scale=0, shift=2, suffix +function inv_txfm_horz\suffix\()_dct_32x4_neon + mov x14, x30 + movi v7.4s, #0 + lsl x8, x8, #1 +.if \scale + movz w16, #2896*8, lsl #16 + dup v0.2s, w16 +.endif + +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s + ld1 {\i}, [x7] + st1 {v7.4s}, [x7], x8 +.endr + sub x7, x7, x8, lsl #4 + add x7, x7, x8, lsr #1 +.if \scale + scale_input .4s, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23 + scale_input .4s, v0.s[0], v24, v25, v26, v27, v28, v29, v30, v31 +.endif + bl inv_dct_4s_x16_neon + transpose_4x4s v16, v17, v18, v19, v2, v3, v4, v5 + transpose_4x4s v20, v21, v22, v23, v2, v3, v4, v5 + transpose_4x4s v24, v25, v26, v27, v2, v3, v4, v5 + transpose_4x4s v28, v29, v30, v31, v2, v3, v4, v5 + +.macro store1 r0, r1, r2, r3 + st1 {\r0}, [x6], #16 + st1 {\r1}, [x6], #16 + st1 {\r2}, [x6], #16 + st1 {\r3}, [x6], #16 +.endm + store1 v16.4s, v20.4s, v24.4s, v28.4s + store1 v17.4s, v21.4s, v25.4s, v29.4s + store1 v18.4s, v22.4s, v26.4s, v30.4s + store1 v19.4s, v23.4s, v27.4s, v31.4s +.purgem store1 + sub x6, x6, #64*4 + + movi v7.4s, #0 +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s + ld1 {\i}, [x7] + st1 {v7.4s}, [x7], x8 +.endr +.if \scale + // This relies on the fact that the idct also leaves the right coeff in v0.s[1] + scale_input .4s, v0.s[1], v16, v17, v18, v19, v20, v21, v22, v23 + scale_input .4s, v0.s[1], v24, v25, v26, v27, v28, v29, v30, v31 +.endif + bl inv_dct32_odd_4s_x16_neon + transpose_4x4s v31, v30, v29, v28, v2, v3, v4, v5 + transpose_4x4s v27, v26, v25, v24, v2, v3, v4, v5 + transpose_4x4s v23, v22, v21, v20, v2, v3, v4, v5 + transpose_4x4s v19, v18, v17, v16, v2, v3, v4, v5 +.macro store2 r0, r1, r2, r3, shift + ld1 {v0.4s, v1.4s, v2.4s, v3.4s}, [x6] + sqsub v4.4s, v0.4s, \r0 + sqadd v0.4s, v0.4s, \r0 + sqsub v5.4s, v1.4s, \r1 + sqadd v1.4s, v1.4s, \r1 + sqsub v6.4s, v2.4s, \r2 + sqadd v2.4s, v2.4s, \r2 + sqsub v7.4s, v3.4s, \r3 + sqadd v3.4s, v3.4s, \r3 + sqrshrn v0.4h, v0.4s, #\shift + sqrshrn2 v0.8h, v1.4s, #\shift + sqrshrn v1.4h, v2.4s, #\shift + sqrshrn2 v1.8h, v3.4s, #\shift + sqrshrn v2.4h, v7.4s, #\shift + sqrshrn2 v2.8h, v6.4s, #\shift + sqrshrn v3.4h, v5.4s, #\shift + sqrshrn2 v3.8h, v4.4s, #\shift + st1 {v0.8h, v1.8h}, [x6], #32 + rev64 v2.8h, v2.8h + rev64 v3.8h, v3.8h + st1 {v2.8h, v3.8h}, [x6], #32 +.endm + + store2 v31.4s, v27.4s, v23.4s, v19.4s, \shift + store2 v30.4s, v26.4s, v22.4s, v18.4s, \shift + store2 v29.4s, v25.4s, v21.4s, v17.4s, \shift + store2 v28.4s, v24.4s, v20.4s, v16.4s, \shift +.purgem store2 + br x14 +endfunc +.endm + +def_horz_32 scale=0, shift=2 +def_horz_32 scale=1, shift=1, suffix=_scale + +function inv_txfm_add_vert_dct_8x32_neon + mov x14, x30 + lsl x8, x8, #1 + +.irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + ld1 {v\i\().8h}, [x7], x8 +.endr + sub x7, x7, x8, lsl #4 + + bl X(inv_dct_8h_x16_neon) + +.irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + st1 {v\i\().8h}, [x7], x8 +.endr + sub x7, x7, x8, lsl #4 + add x7, x7, x8, lsr #1 + +.irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + ld1 {v\i\().8h}, [x7], x8 +.endr + sub x7, x7, x8, lsl #4 + sub x7, x7, x8, lsr #1 + bl X(inv_dct32_odd_8h_x16_neon) + + neg x9, x8 + mov x10, x6 + movi v0.8h, #0 + mvni v1.8h, #0xfc, lsl #8 // 0x3ff +.macro combine r0, r1, r2, r3, op, stride + ld1 {v5.8h}, [x7], \stride + ld1 {v2.8h}, [x10], x1 + ld1 {v6.8h}, [x7], \stride + ld1 {v3.8h}, [x10], x1 + \op v5.8h, v5.8h, \r0 + ld1 {v7.8h}, [x7], \stride + ld1 {v4.8h}, [x10], x1 + srshr v5.8h, v5.8h, #4 + \op v6.8h, v6.8h, \r1 + sqadd v5.8h, v5.8h, v2.8h + srshr v6.8h, v6.8h, #4 + \op v7.8h, v7.8h, \r2 + smax v2.8h, v5.8h, v0.8h + ld1 {v5.8h}, [x7], \stride + sqadd v6.8h, v6.8h, v3.8h + smin v2.8h, v2.8h, v1.8h + srshr v7.8h, v7.8h, #4 + \op v5.8h, v5.8h, \r3 + st1 {v2.8h}, [x6], x1 + ld1 {v2.8h}, [x10], x1 + smax v3.8h, v6.8h, v0.8h + sqadd v7.8h, v7.8h, v4.8h + smin v3.8h, v3.8h, v1.8h + srshr v5.8h, v5.8h, #4 + st1 {v3.8h}, [x6], x1 + smax v4.8h, v7.8h, v0.8h + sqadd v5.8h, v5.8h, v2.8h + smin v4.8h, v4.8h, v1.8h + st1 {v4.8h}, [x6], x1 + smax v2.8h, v5.8h, v0.8h + smin v2.8h, v2.8h, v1.8h + st1 {v2.8h}, [x6], x1 +.endm + combine v31.8h, v30.8h, v29.8h, v28.8h, sqadd, x8 + combine v27.8h, v26.8h, v25.8h, v24.8h, sqadd, x8 + combine v23.8h, v22.8h, v21.8h, v20.8h, sqadd, x8 + combine v19.8h, v18.8h, v17.8h, v16.8h, sqadd, x8 + sub x7, x7, x8 + combine v16.8h, v17.8h, v18.8h, v19.8h, sqsub, x9 + combine v20.8h, v21.8h, v22.8h, v23.8h, sqsub, x9 + combine v24.8h, v25.8h, v26.8h, v27.8h, sqsub, x9 + combine v28.8h, v29.8h, v30.8h, v31.8h, sqsub, x9 +.purgem combine + + br x14 +endfunc + +const eob_32x32 + .short 10, 36, 78, 136, 210, 300, 406, 1024 +endconst + +const eob_16x32 + .short 10, 36, 78, 151, 215, 279, 343, 512 +endconst + +const eob_16x32_shortside + .short 10, 36, 78, 512 +endconst + +const eob_8x32 + .short 10, 43, 75, 107, 139, 171, 203, 256 +endconst + +function inv_txfm_add_identity_identity_32x32_16bpc_neon, export=1 + movi v0.8h, #0 + movi v1.8h, #0 + movrel x13, eob_32x32, 2 + + mov x8, #4*32 +1: + mov w9, #0 + movrel x12, eob_32x32, 2 +2: + add w9, w9, #8 + ld1 {v16.4s, v17.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v18.4s, v19.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v20.4s, v21.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v22.4s, v23.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v24.4s, v25.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v26.4s, v27.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v28.4s, v29.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v30.4s, v31.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + sqxtn v16.4h, v16.4s + sqxtn2 v16.8h, v17.4s + sqxtn v17.4h, v18.4s + sqxtn2 v17.8h, v19.4s + sqxtn v18.4h, v20.4s + sqxtn2 v18.8h, v21.4s + sqxtn v19.4h, v22.4s + sqxtn2 v19.8h, v23.4s + sqxtn v20.4h, v24.4s + sqxtn2 v20.8h, v25.4s + sqxtn v21.4h, v26.4s + sqxtn2 v21.8h, v27.4s + sqxtn v22.4h, v28.4s + sqxtn2 v22.8h, v29.4s + sqxtn v23.4h, v30.4s + sqxtn2 v23.8h, v31.4s + transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v4, v5 + + load_add_store_8x8 x0, x7, shiftbits=2 + ldrh w11, [x12], #4 + sub x0, x0, x1, lsl #3 + add x0, x0, #2*8 + cmp w3, w11 + b.ge 2b + + ldrh w11, [x13], #4 + cmp w3, w11 + b.lt 9f + + sub x0, x0, w9, uxtw #1 + add x0, x0, x1, lsl #3 + msub x2, x8, x9, x2 + add x2, x2, #4*8 + b 1b +9: + ret +endfunc + +.macro shift_16_regs op, shift +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s + \op \i, \i, #\shift +.endr +.endm + +.macro def_identity_1632 w, h, wshort, hshort +function inv_txfm_add_identity_identity_\w\()x\h\()_16bpc_neon, export=1 + movz w16, #2896*8, lsl #16 + movz w17, #2*(5793-4096)*8, lsl #16 + movi v0.4s, #0 + movi v1.4s, #0 + movrel x13, eob_16x32\hshort, 2 + + mov x8, #4*\h +1: + mov w9, #0 + movrel x12, eob_16x32\wshort, 2 +2: + add w9, w9, #8 + ld1 {v16.4s, v17.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + dup v2.2s, w16 + ld1 {v18.4s, v19.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + mov v2.s[1], w17 + ld1 {v20.4s, v21.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v22.4s, v23.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v24.4s, v25.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v26.4s, v27.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v28.4s, v29.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v30.4s, v31.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + scale_input .4s, v2.s[0], v16, v17, v18, v19, v20, v21, v22, v23 + scale_input .4s, v2.s[0], v24, v25, v26, v27, v28, v29, v30, v31 + +.if \w == 16 + // 16x32 + identity_4x16_shift1 v2.s[1] +.else + // 32x16 + shift_16_regs sqshl, 1 + identity_4x16 v2.s[1] +.endif + sqxtn v16.4h, v16.4s + sqxtn2 v16.8h, v17.4s + sqxtn v17.4h, v18.4s + sqxtn2 v17.8h, v19.4s + sqxtn v18.4h, v20.4s + sqxtn2 v18.8h, v21.4s + sqxtn v19.4h, v22.4s + sqxtn2 v19.8h, v23.4s + sqxtn v20.4h, v24.4s + sqxtn2 v20.8h, v25.4s + sqxtn v21.4h, v26.4s + sqxtn2 v21.8h, v27.4s + sqxtn v22.4h, v28.4s + sqxtn2 v22.8h, v29.4s + sqxtn v23.4h, v30.4s + sqxtn2 v23.8h, v31.4s + + transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v4, v5 + +.if \w == 16 + load_add_store_8x8 x0, x7, shiftbits=2 +.else + load_add_store_8x8 x0, x7, shiftbits=4 +.endif + ldrh w11, [x12], #4 + sub x0, x0, x1, lsl #3 + add x0, x0, #16 + cmp w3, w11 + b.ge 2b + + ldrh w11, [x13], #4 + cmp w3, w11 + b.lt 9f + + sub x0, x0, w9, uxtw #1 + add x0, x0, x1, lsl #3 + msub x2, x8, x9, x2 + add x2, x2, #4*8 + b 1b +9: + ret +endfunc +.endm + +def_identity_1632 16, 32, _shortside, +def_identity_1632 32, 16, , _shortside + +.macro def_identity_832 w, h +function inv_txfm_add_identity_identity_\w\()x\h\()_16bpc_neon, export=1 + movi v0.4s, #0 + movi v1.4s, #0 + // Working on 8x8 blocks, read every other entry from eob_8x32 + movrel x13, eob_8x32, 2 + + mov w8, #4*\h +1: + // Working on 8x8 blocks, read every other entry from eob_8x32 + ldrh w12, [x13], #4 + ld1 {v16.4s, v17.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v18.4s, v19.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v20.4s, v21.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v22.4s, v23.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v24.4s, v25.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v26.4s, v27.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v28.4s, v29.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + ld1 {v30.4s, v31.4s}, [x2] + st1 {v0.4s, v1.4s}, [x2], x8 + +.if \w == 8 + sqrshrn v16.4h, v16.4s, #1 + sqrshrn2 v16.8h, v17.4s, #1 + sqrshrn v17.4h, v18.4s, #1 + sqrshrn2 v17.8h, v19.4s, #1 + sqrshrn v18.4h, v20.4s, #1 + sqrshrn2 v18.8h, v21.4s, #1 + sqrshrn v19.4h, v22.4s, #1 + sqrshrn2 v19.8h, v23.4s, #1 + sqrshrn v20.4h, v24.4s, #1 + sqrshrn2 v20.8h, v25.4s, #1 + sqrshrn v21.4h, v26.4s, #1 + sqrshrn2 v21.8h, v27.4s, #1 + sqrshrn v22.4h, v28.4s, #1 + sqrshrn2 v22.8h, v29.4s, #1 + sqrshrn v23.4h, v30.4s, #1 + sqrshrn2 v23.8h, v31.4s, #1 +.else + sqxtn v16.4h, v16.4s + sqxtn2 v16.8h, v17.4s + sqxtn v17.4h, v18.4s + sqxtn2 v17.8h, v19.4s + sqxtn v18.4h, v20.4s + sqxtn2 v18.8h, v21.4s + sqxtn v19.4h, v22.4s + sqxtn2 v19.8h, v23.4s + sqxtn v20.4h, v24.4s + sqxtn2 v20.8h, v25.4s + sqxtn v21.4h, v26.4s + sqxtn2 v21.8h, v27.4s + sqxtn v22.4h, v28.4s + sqxtn2 v22.8h, v29.4s + sqxtn v23.4h, v30.4s + sqxtn2 v23.8h, v31.4s +.endif + + transpose_8x8h v16, v17, v18, v19, v20, v21, v22, v23, v4, v5 + + + cmp w3, w12 +.if \w == 8 + load_add_store_8x8 x0, x7, shiftbits=2 +.else + load_add_store_8x8 x0, x7, shiftbits=3 +.endif + + b.lt 9f +.if \w == 8 + sub x2, x2, x8, lsl #3 + add x2, x2, #4*8 +.else + sub x0, x0, x1, lsl #3 + add x0, x0, #2*8 +.endif + b 1b + +9: + ret +endfunc +.endm + +def_identity_832 8, 32 +def_identity_832 32, 8 + +function inv_txfm_add_dct_dct_32x32_16bpc_neon, export=1 + idct_dc 32, 32, 2 + + mov x15, x30 + sub sp, sp, #2048 + movrel x13, eob_32x32 + ldrh w12, [x13], #2 + +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add x6, sp, #(\i*32*2) +.if \i > 0 + mov w8, #(32 - \i) + cmp w3, w12 + b.lt 1f +.if \i < 28 + ldrh w12, [x13], #2 +.endif +.endif + add x7, x2, #(\i*4) + mov x8, #32*4 + bl inv_txfm_horz_dct_32x4_neon +.endr + b 3f + +1: + movi v4.8h, #0 + movi v5.8h, #0 + movi v6.8h, #0 + movi v7.8h, #0 +2: + subs w8, w8, #4 +.rept 4 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64 +.endr + b.gt 2b + +3: +.irp i, 0, 8, 16, 24 + add x6, x0, #(\i*2) + add x7, sp, #(\i*2) + mov x8, #32*2 + bl inv_txfm_add_vert_dct_8x32_neon +.endr + + add sp, sp, #2048 + br x15 +endfunc + +function inv_txfm_add_dct_dct_16x32_16bpc_neon, export=1 + idct_dc 16, 32, 1 + + mov x15, x30 + sub sp, sp, #1024 + movrel x13, eob_16x32 + ldrh w12, [x13], #2 + adr x4, inv_dct_4s_x16_neon + +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add x6, sp, #(\i*16*2) + add x7, x2, #(\i*4) +.if \i > 0 + mov w8, #(32 - \i) + cmp w3, w12 + b.lt 1f +.if \i < 28 + ldrh w12, [x13], #2 +.endif +.endif + mov x8, #4*32 + bl inv_txfm_horz_scale_16x4_neon +.endr + b 3f + +1: + movi v4.8h, #0 + movi v5.8h, #0 + movi v6.8h, #0 + movi v7.8h, #0 +2: + subs w8, w8, #4 +.rept 2 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64 +.endr + b.gt 2b + +3: +.irp i, 0, 8 + add x6, x0, #(\i*2) + add x7, sp, #(\i*2) + mov x8, #16*2 + bl inv_txfm_add_vert_dct_8x32_neon +.endr + + add sp, sp, #1024 + br x15 +endfunc + +function inv_txfm_add_dct_dct_32x16_16bpc_neon, export=1 + idct_dc 32, 16, 1 + + mov x15, x30 + sub sp, sp, #1024 + + movrel x13, eob_16x32 + movrel x5, X(inv_dct_8h_x16_neon) + ldrh w12, [x13], #2 + +.irp i, 0, 4, 8, 12 + add x6, sp, #(\i*32*2) + add x7, x2, #(\i*4) +.if \i > 0 + mov w8, #(16 - \i) + cmp w3, w12 + b.lt 1f + ldrh w12, [x13], #2 +.endif + mov x8, #4*16 + bl inv_txfm_horz_scale_dct_32x4_neon +.endr + b 3f + +1: + movi v4.8h, #0 + movi v5.8h, #0 + movi v6.8h, #0 + movi v7.8h, #0 +2: + subs w8, w8, #4 +.rept 4 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64 +.endr + b.gt 2b + +3: +.irp i, 0, 8, 16, 24 + add x6, x0, #(\i*2) + add x7, sp, #(\i*2) + mov x8, #32*2 + bl inv_txfm_add_vert_8x16_neon +.endr + + add sp, sp, #1024 + br x15 +endfunc + +function inv_txfm_add_dct_dct_8x32_16bpc_neon, export=1 + idct_dc 8, 32, 2 + + mov x15, x30 + sub sp, sp, #512 + + movrel x13, eob_8x32 + + movi v28.4s, #0 + mov x8, #4*32 + mov w9, #32 + mov x6, sp + mov x7, x2 +1: +.irp i, 16, 17, 18, 19, 20, 21, 22, 23 + ld1 {v\i\().4s}, [x7] + st1 {v28.4s}, [x7], x8 +.endr + ldrh w12, [x13], #2 + sub w9, w9, #4 + sub x7, x7, x8, lsl #3 + add x7, x7, #4*4 + + bl inv_dct_4s_x8_neon + + sqrshrn v16.4h, v16.4s, #2 + sqrshrn v17.4h, v17.4s, #2 + sqrshrn v18.4h, v18.4s, #2 + sqrshrn v19.4h, v19.4s, #2 + sqrshrn2 v16.8h, v20.4s, #2 + sqrshrn2 v17.8h, v21.4s, #2 + sqrshrn2 v18.8h, v22.4s, #2 + sqrshrn2 v19.8h, v23.4s, #2 + + transpose_4x8h v16, v17, v18, v19, v2, v3, v4, v5 + + cmp w3, w12 + st1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x6], #64 + + b.ge 1b + cbz w9, 3f + + movi v29.8h, #0 + movi v30.8h, #0 + movi v31.8h, #0 +2: + subs w9, w9, #4 + st1 {v28.8h,v29.8h,v30.8h,v31.8h}, [x6], #64 + b.gt 2b + +3: + mov x6, x0 + mov x7, sp + mov x8, #8*2 + bl inv_txfm_add_vert_dct_8x32_neon + + add sp, sp, #512 + br x15 +endfunc + +function inv_txfm_add_dct_dct_32x8_16bpc_neon, export=1 + idct_dc 32, 8, 2 + + mov x15, x30 + sub sp, sp, #512 + +.irp i, 0, 4 + add x6, sp, #(\i*32*2) + add x7, x2, #(\i*4) +.if \i > 0 + cmp w3, #10 + b.lt 1f +.endif + mov x8, #8*4 + bl inv_txfm_horz_dct_32x4_neon +.endr + b 2f + +1: + movi v4.8h, #0 + movi v5.8h, #0 + movi v6.8h, #0 + movi v7.8h, #0 +.rept 4 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64 +.endr + +2: + mov x8, #2*32 + mov w9, #0 +1: + add x6, x0, x9, lsl #1 + add x7, sp, x9, lsl #1 // #(\i*2) + +.irp i, 16, 17, 18, 19, 20, 21, 22, 23 + ld1 {v\i\().8h}, [x7], x8 +.endr + add w9, w9, #8 + + bl X(inv_dct_8h_x8_neon) + + cmp w9, #32 + + load_add_store_8x8 x6, x7 + + b.lt 1b + + add sp, sp, #512 + br x15 +endfunc + +function inv_dct64_step1_neon + // in1/31/17/15 -> t32a/33/34a/35/60/61a/62/63a + // in7/25/23/ 9 -> t56a/57/58a/59/36/37a/38/39a + // in5/27/21/11 -> t40a/41/42a/43/52/53a/54/55a + // in3/29/19/13 -> t48a/49/50a/51/44/45a/46/47a + + ld1 {v0.4s, v1.4s}, [x17], #32 + + sqrdmulh v23.4s, v16.4s, v0.s[1] // t63a + sqrdmulh v16.4s, v16.4s, v0.s[0] // t32a + sqrdmulh v22.4s, v17.4s, v0.s[2] // t62a + sqrdmulh v17.4s, v17.4s, v0.s[3] // t33a + sqrdmulh v21.4s, v18.4s, v1.s[1] // t61a + sqrdmulh v18.4s, v18.4s, v1.s[0] // t34a + sqrdmulh v20.4s, v19.4s, v1.s[2] // t60a + sqrdmulh v19.4s, v19.4s, v1.s[3] // t35a + + ld1 {v0.4s}, [x17], #16 + + sqadd v24.4s, v16.4s, v17.4s // t32 + sqsub v25.4s, v16.4s, v17.4s // t33 + sqsub v26.4s, v19.4s, v18.4s // t34 + sqadd v27.4s, v19.4s, v18.4s // t35 + sqadd v28.4s, v20.4s, v21.4s // t60 + sqsub v29.4s, v20.4s, v21.4s // t61 + sqsub v30.4s, v23.4s, v22.4s // t62 + sqadd v31.4s, v23.4s, v22.4s // t63 + + mul_mla v2, v29, v26, v0.s[0], v0.s[1] // -> t34a + mul_mls v4, v29, v26, v0.s[1], v0.s[0] // -> t61a + neg v2.4s, v2.4s // t34a + mul_mls v6, v30, v25, v0.s[1], v0.s[0] // -> t33a + srshr v26.4s, v2.4s, #12 // t34a + mul_mla v2, v30, v25, v0.s[0], v0.s[1] // -> t62a + srshr v29.4s, v4.4s, #12 // t61a + srshr v25.4s, v6.4s, #12 // t33a + srshr v30.4s, v2.4s, #12 // t62a + + sqadd v16.4s, v24.4s, v27.4s // t32a + sqsub v19.4s, v24.4s, v27.4s // t35a + sqadd v17.4s, v25.4s, v26.4s // t33 + sqsub v18.4s, v25.4s, v26.4s // t34 + sqsub v20.4s, v31.4s, v28.4s // t60a + sqadd v23.4s, v31.4s, v28.4s // t63a + sqsub v21.4s, v30.4s, v29.4s // t61 + sqadd v22.4s, v30.4s, v29.4s // t62 + + mul_mla v2, v21, v18, v0.s[2], v0.s[3] // -> t61a + mul_mls v4, v21, v18, v0.s[3], v0.s[2] // -> t34a + mul_mla v6, v20, v19, v0.s[2], v0.s[3] // -> t60 + srshr v21.4s, v2.4s, #12 // t61a + srshr v18.4s, v4.4s, #12 // t34a + mul_mls v2, v20, v19, v0.s[3], v0.s[2] // -> t35 + srshr v20.4s, v6.4s, #12 // t60 + srshr v19.4s, v2.4s, #12 // t35 + + st1 {v16.4s, v17.4s, v18.4s, v19.4s}, [x6], #64 + st1 {v20.4s, v21.4s, v22.4s, v23.4s}, [x6], #64 + + ret +endfunc + +function inv_dct64_step2_neon + movrel x16, idct_coeffs + ld1 {v0.4s}, [x16] +1: + // t32a/33/34a/35/60/61a/62/63a + // t56a/57/58a/59/36/37a/38/39a + // t40a/41/42a/43/52/53a/54/55a + // t48a/49/50a/51/44/45a/46/47a + ldr q16, [x6, #4*4*0] // t32a + ldr q17, [x9, #4*4*8] // t39a + ldr q18, [x9, #4*4*0] // t63a + ldr q19, [x6, #4*4*8] // t56a + ldr q20, [x6, #4*4*16] // t40a + ldr q21, [x9, #4*4*24] // t47a + ldr q22, [x9, #4*4*16] // t55a + ldr q23, [x6, #4*4*24] // t48a + + sqadd v24.4s, v16.4s, v17.4s // t32 + sqsub v25.4s, v16.4s, v17.4s // t39 + sqadd v26.4s, v18.4s, v19.4s // t63 + sqsub v27.4s, v18.4s, v19.4s // t56 + sqsub v28.4s, v21.4s, v20.4s // t40 + sqadd v29.4s, v21.4s, v20.4s // t47 + sqadd v30.4s, v23.4s, v22.4s // t48 + sqsub v31.4s, v23.4s, v22.4s // t55 + + mul_mla v2, v27, v25, v0.s[3], v0.s[2] // -> t56a + mul_mls v4, v27, v25, v0.s[2], v0.s[3] // -> t39a + mul_mla v6, v31, v28, v0.s[3], v0.s[2] // -> t40a + srshr v25.4s, v2.4s, #12 // t56a + srshr v27.4s, v4.4s, #12 // t39a + neg v6.4s, v6.4s // t40a + mul_mls v2, v31, v28, v0.s[2], v0.s[3] // -> t55a + srshr v31.4s, v6.4s, #12 // t40a + srshr v28.4s, v2.4s, #12 // t55a + + sqadd v16.4s, v24.4s, v29.4s // t32a + sqsub v19.4s, v24.4s, v29.4s // t47a + sqadd v17.4s, v27.4s, v31.4s // t39 + sqsub v18.4s, v27.4s, v31.4s // t40 + sqsub v20.4s, v26.4s, v30.4s // t48a + sqadd v23.4s, v26.4s, v30.4s // t63a + sqsub v21.4s, v25.4s, v28.4s // t55 + sqadd v22.4s, v25.4s, v28.4s // t56 + + mul_mls v2, v21, v18, v0.s[0], v0.s[0] // -> t40a + mul_mla v4, v21, v18, v0.s[0], v0.s[0] // -> t55a + mul_mls v6, v20, v19, v0.s[0], v0.s[0] // -> t47 + srshr v18.4s, v2.4s, #12 // t40a + srshr v21.4s, v4.4s, #12 // t55a + mul_mla v2, v20, v19, v0.s[0], v0.s[0] // -> t48 + srshr v19.4s, v6.4s, #12 // t47 + srshr v20.4s, v2.4s, #12 // t48 + + str q16, [x6, #4*4*0] // t32a + str q17, [x9, #4*4*0] // t39 + str q18, [x6, #4*4*8] // t40a + str q19, [x9, #4*4*8] // t47 + str q20, [x6, #4*4*16] // t48 + str q21, [x9, #4*4*16] // t55a + str q22, [x6, #4*4*24] // t56 + str q23, [x9, #4*4*24] // t63a + + add x6, x6, #4*4 + sub x9, x9, #4*4 + cmp x6, x9 + b.lt 1b + ret +endfunc + +.macro load8 src, strd, zero, clear +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s +.if \clear + ld1 {\i}, [\src] + st1 {\zero}, [\src], \strd +.else + ld1 {\i}, [\src], \strd +.endif +.endr +.endm + +.macro store16 dst +.irp i, v16.4s, v17.4s, v18.4s, v19.4s, v20.4s, v21.4s, v22.4s, v23.4s, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s + st1 {\i}, [\dst], #16 +.endr +.endm + +.macro clear_upper8 +.irp i, v24.4s, v25.4s, v26.4s, v27.4s, v28.4s, v29.4s, v30.4s, v31.4s + movi \i, #0 +.endr +.endm + +.macro movi_if reg, val, cond +.if \cond + movi \reg, \val +.endif +.endm + +.macro movz16dup_if reg, gpr, val, cond +.if \cond + movz \gpr, \val, lsl #16 + dup \reg, \gpr +.endif +.endm + +.macro st1_if regs, dst, cond +.if \cond + st1 \regs, \dst +.endif +.endm + +.macro str_if reg, dst, cond +.if \cond + str \reg, \dst +.endif +.endm + +.macro stroff_if reg, dst, dstoff, cond +.if \cond + str \reg, \dst, \dstoff +.endif +.endm + +.macro scale_if cond, c, r0, r1, r2, r3, r4, r5, r6, r7 +.if \cond + scale_input .4s, \c, \r0, \r1, \r2, \r3, \r4, \r5, \r6, \r7 +.endif +.endm + +.macro def_dct64_func suffix, clear=0, scale=0 +function inv_txfm_dct\suffix\()_4s_x64_neon + mov x14, x30 + mov x6, sp + lsl x8, x8, #2 + + movz16dup_if v0.2s, w16, #2896*8, \scale + movi_if v7.4s, #0, \clear + load8 x7, x8, v7.4s, \clear + clear_upper8 + sub x7, x7, x8, lsl #3 + add x7, x7, x8, lsr #1 + scale_if \scale, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23 + + bl inv_dct_4s_x16_neon + + store16 x6 + + movz16dup_if v0.2s, w16, #2896*8, \scale + movi_if v7.8h, #0, \clear + load8 x7, x8, v7.4s, \clear + clear_upper8 + sub x7, x7, x8, lsl #3 + lsr x8, x8, #1 + sub x7, x7, x8, lsr #1 + scale_if \scale, v0.s[0], v16, v17, v18, v19, v20, v21, v22, v23 + + bl inv_dct32_odd_4s_x16_neon + + add x10, x6, #16*15 + sub x6, x6, #16*16 + + mov x9, #-16 + +.macro store_addsub r0, r1, r2, r3 + ld1 {v2.4s}, [x6], #16 + ld1 {v3.4s}, [x6], #16 + sqadd v6.4s, v2.4s, \r0 + sqsub \r0, v2.4s, \r0 + ld1 {v4.4s}, [x6], #16 + sqadd v7.4s, v3.4s, \r1 + sqsub \r1, v3.4s, \r1 + ld1 {v5.4s}, [x6], #16 + sqadd v2.4s, v4.4s, \r2 + sub x6, x6, #16*4 + sqsub \r2, v4.4s, \r2 + st1 {v6.4s}, [x6], #16 + st1 {\r0}, [x10], x9 + sqadd v3.4s, v5.4s, \r3 + sqsub \r3, v5.4s, \r3 + st1 {v7.4s}, [x6], #16 + st1 {\r1}, [x10], x9 + st1 {v2.4s}, [x6], #16 + st1 {\r2}, [x10], x9 + st1 {v3.4s}, [x6], #16 + st1 {\r3}, [x10], x9 +.endm + store_addsub v31.4s, v30.4s, v29.4s, v28.4s + store_addsub v27.4s, v26.4s, v25.4s, v24.4s + store_addsub v23.4s, v22.4s, v21.4s, v20.4s + store_addsub v19.4s, v18.4s, v17.4s, v16.4s +.purgem store_addsub + + add x6, x6, #4*4*16 + + movrel x17, idct64_coeffs + movz16dup_if v0.2s, w16, #2896*8, \scale + movi_if v7.4s, #0, \clear + add x9, x7, x8, lsl #4 // offset 16 + add x10, x7, x8, lsl #3 // offset 8 + sub x9, x9, x8 // offset 15 + sub x11, x10, x8 // offset 7 + ld1 {v16.4s}, [x7] // in1 (offset 0) + ld1 {v17.4s}, [x9] // in31 (offset 15) + ld1 {v18.4s}, [x10] // in17 (offset 8) + ld1 {v19.4s}, [x11] // in15 (offset 7) + st1_if {v7.4s}, [x7], \clear + st1_if {v7.4s}, [x9], \clear + st1_if {v7.4s}, [x10], \clear + st1_if {v7.4s}, [x11], \clear + scale_if \scale, v0.s[0], v16, v17, v18, v19 + bl inv_dct64_step1_neon + movz16dup_if v0.2s, w16, #2896*8, \scale + movi_if v7.4s, #0, \clear + add x7, x7, x8, lsl #2 // offset 4 + sub x9, x9, x8, lsl #2 // offset 11 + sub x10, x7, x8 // offset 3 + add x11, x9, x8 // offset 12 + ld1 {v16.4s}, [x10] // in7 (offset 3) + ld1 {v17.4s}, [x11] // in25 (offset 12) + ld1 {v18.4s}, [x9] // in23 (offset 11) + ld1 {v19.4s}, [x7] // in9 (offset 4) + st1_if {v7.4s}, [x7], \clear + st1_if {v7.4s}, [x9], \clear + st1_if {v7.4s}, [x10], \clear + st1_if {v7.4s}, [x11], \clear + scale_if \scale, v0.s[0], v16, v17, v18, v19 + bl inv_dct64_step1_neon + movz16dup_if v0.2s, w16, #2896*8, \scale + movi_if v7.4s, #0, \clear + sub x10, x10, x8, lsl #1 // offset 1 + sub x9, x9, x8, lsl #1 // offset 9 + add x7, x7, x8 // offset 5 + add x11, x11, x8 // offset 13 + ldr q16, [x10, x8] // in5 (offset 2) + ldr q17, [x11] // in27 (offset 13) + ldr q18, [x9, x8] // in21 (offset 10) + ldr q19, [x7] // in11 (offset 5) + stroff_if q7, [x10, x8], \clear + str_if q7, [x11], \clear + stroff_if q7, [x9, x8], \clear + str_if q7, [x7], \clear + scale_if \scale, v0.s[0], v16, v17, v18, v19 + bl inv_dct64_step1_neon + movz16dup_if v0.2s, w16, #2896*8, \scale + movi_if v7.4s, #0, \clear + ldr q16, [x10] // in3 (offset 1) + ldr q17, [x11, x8] // in29 (offset 14) + ldr q18, [x9] // in19 (offset 9) + ldr q19, [x7, x8] // in13 (offset 6) + str_if q7, [x10], \clear + stroff_if q7, [x11, x8], \clear + str_if q7, [x9], \clear + stroff_if q7, [x7, x8], \clear + scale_if \scale, v0.s[0], v16, v17, v18, v19 + bl inv_dct64_step1_neon + + sub x6, x6, #4*4*32 + add x9, x6, #4*4*7 + + bl inv_dct64_step2_neon + + br x14 +endfunc +.endm + +def_dct64_func _clear, clear=1 +def_dct64_func _clear_scale, clear=1, scale=1 + + +function inv_txfm_horz_dct_64x4_neon + mov x14, x30 + + mov x7, sp + add x8, sp, #4*4*(64 - 4) + add x9, x6, #2*56 + mov x10, #2*64 + mov x11, #-4*4*4 + + dup v7.4s, w12 +1: + ld1 {v16.4s, v17.4s, v18.4s, v19.4s}, [x7], #64 + ld1 {v28.4s, v29.4s, v30.4s, v31.4s}, [x8], x11 + ld1 {v20.4s, v21.4s, v22.4s, v23.4s}, [x7], #64 + ld1 {v24.4s, v25.4s, v26.4s, v27.4s}, [x8], x11 + transpose_4x4s v16, v17, v18, v19, v2, v3, v4, v5 + transpose_4x4s v20, v21, v22, v23, v2, v3, v4, v5 + transpose_4x4s v31, v30, v29, v28, v2, v3, v4, v5 + transpose_4x4s v27, v26, v25, v24, v2, v3, v4, v5 + +.macro store_addsub src0, src1, src2, src3 + sqsub v1.4s, \src0, \src1 + sqadd v0.4s, \src0, \src1 + sqsub v3.4s, \src2, \src3 + srshl v1.4s, v1.4s, v7.4s + sqadd v2.4s, \src2, \src3 + srshl v3.4s, v3.4s, v7.4s + srshl v0.4s, v0.4s, v7.4s + srshl v2.4s, v2.4s, v7.4s + sqxtn v3.4h, v3.4s + sqxtn2 v3.8h, v1.4s + sqxtn v0.4h, v0.4s + sqxtn2 v0.8h, v2.4s + rev64 v3.8h, v3.8h + st1 {v0.8h}, [x6], x10 + st1 {v3.8h}, [x9], x10 +.endm + store_addsub v16.4s, v31.4s, v20.4s, v27.4s + store_addsub v17.4s, v30.4s, v21.4s, v26.4s + store_addsub v18.4s, v29.4s, v22.4s, v25.4s + store_addsub v19.4s, v28.4s, v23.4s, v24.4s +.purgem store_addsub + sub x6, x6, x10, lsl #2 + sub x9, x9, x10, lsl #2 + add x6, x6, #16 + sub x9, x9, #16 + + cmp x7, x8 + b.lt 1b + br x14 +endfunc + +function inv_txfm_add_vert_dct_8x64_neon + mov x14, x30 + lsl x8, x8, #1 + + mov x7, sp + add x8, sp, #2*8*(64 - 4) + add x9, x6, x1, lsl #6 + sub x9, x9, x1 + neg x10, x1 + mov x11, #-2*8*4 + +1: + ld1 {v16.8h, v17.8h, v18.8h, v19.8h}, [x7], #64 + ld1 {v28.8h, v29.8h, v30.8h, v31.8h}, [x8], x11 + ld1 {v20.8h, v21.8h, v22.8h, v23.8h}, [x7], #64 + ld1 {v24.8h, v25.8h, v26.8h, v27.8h}, [x8], x11 + + movi v6.8h, #0 + mvni v7.8h, #0xfc, lsl #8 // 0x3ff +.macro add_dest_addsub src0, src1, src2, src3 + ld1 {v0.8h}, [x6], x1 + ld1 {v1.8h}, [x9], x10 + sqadd v4.8h, \src0, \src1 + ld1 {v2.8h}, [x6] + sqsub \src0, \src0, \src1 + ld1 {v3.8h}, [x9] + sqadd v5.8h, \src2, \src3 + sqsub \src2, \src2, \src3 + sub x6, x6, x1 + sub x9, x9, x10 + srshr v4.8h, v4.8h, #4 + srshr v5.8h, v5.8h, #4 + srshr \src0, \src0, #4 + sqadd v0.8h, v0.8h, v4.8h + srshr \src2, \src2, #4 + sqadd v1.8h, v1.8h, \src0 + sqadd v2.8h, v2.8h, v5.8h + smax v0.8h, v0.8h, v6.8h + sqadd v3.8h, v3.8h, \src2 + smax v1.8h, v1.8h, v6.8h + smin v0.8h, v0.8h, v7.8h + smax v2.8h, v2.8h, v6.8h + smin v1.8h, v1.8h, v7.8h + st1 {v0.8h}, [x6], x1 + smax v3.8h, v3.8h, v6.8h + smin v2.8h, v2.8h, v7.8h + st1 {v1.8h}, [x9], x10 + smin v3.8h, v3.8h, v7.8h + st1 {v2.8h}, [x6], x1 + st1 {v3.8h}, [x9], x10 +.endm + add_dest_addsub v16.8h, v31.8h, v17.8h, v30.8h + add_dest_addsub v18.8h, v29.8h, v19.8h, v28.8h + add_dest_addsub v20.8h, v27.8h, v21.8h, v26.8h + add_dest_addsub v22.8h, v25.8h, v23.8h, v24.8h +.purgem add_dest_addsub + cmp x7, x8 + b.lt 1b + + br x14 +endfunc + +.macro sub_sp space +#ifdef _WIN32 +.if \space > 8192 + // Here, we'd need to touch two (or more) pages while decrementing + // the stack pointer. + .error "sub_sp_align doesn't support values over 8K at the moment" +.elseif \space > 4096 + sub x16, sp, #4096 + ldr xzr, [x16] + sub sp, x16, #(\space - 4096) +.else + sub sp, sp, #\space +.endif +#else +.if \space >= 4096 + sub sp, sp, #(\space)/4096*4096 +.endif +.if (\space % 4096) != 0 + sub sp, sp, #(\space)%4096 +.endif +#endif +.endm + +function inv_txfm_add_dct_dct_64x64_16bpc_neon, export=1 + idct_dc 64, 64, 2 + + mov x15, x30 + + sub_sp 64*32*2+64*4*4 + add x5, sp, #64*4*4 + + movrel x13, eob_32x32 + +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add x6, x5, #(\i*64*2) +.if \i > 0 + mov w8, #(32 - \i) + cmp w3, w12 + b.lt 1f +.endif + add x7, x2, #(\i*4) + mov x8, #32*4 + mov x12, #-2 // shift + bl inv_txfm_dct_clear_4s_x64_neon + add x6, x5, #(\i*64*2) + bl inv_txfm_horz_dct_64x4_neon +.if \i < 28 + ldrh w12, [x13], #2 +.endif +.endr + b 3f + +1: + movi v4.8h, #0 + movi v5.8h, #0 + movi v6.8h, #0 + movi v7.8h, #0 +2: + subs w8, w8, #2 +.rept 4 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64 +.endr + b.gt 2b + +3: +.irp i, 0, 8, 16, 24, 32, 40, 48, 56 + add x7, x5, #(\i*2) + mov x8, #64*2 + bl X(inv_txfm_dct_8h_x64_neon) + add x6, x0, #(\i*2) + bl inv_txfm_add_vert_dct_8x64_neon +.endr + + add sp, x5, #64*32*2 + br x15 +endfunc + +function inv_txfm_add_dct_dct_64x32_16bpc_neon, export=1 + idct_dc 64, 32, 1 + + mov x15, x30 + + sub_sp 64*32*2+64*4*4 + add x5, sp, #64*4*4 + + movrel x13, eob_32x32 + +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add x6, x5, #(\i*64*2) +.if \i > 0 + mov w8, #(32 - \i) + cmp w3, w12 + b.lt 1f +.endif + add x7, x2, #(\i*4) + mov x8, #32*4 + mov x12, #-1 // shift + bl inv_txfm_dct_clear_scale_4s_x64_neon + add x6, x5, #(\i*64*2) + bl inv_txfm_horz_dct_64x4_neon +.if \i < 28 + ldrh w12, [x13], #2 +.endif +.endr + b 3f + +1: + movi v4.8h, #0 + movi v5.8h, #0 + movi v6.8h, #0 + movi v7.8h, #0 +2: + subs w8, w8, #2 +.rept 4 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64 +.endr + b.gt 2b + +3: +.irp i, 0, 8, 16, 24, 32, 40, 48, 56 + add x6, x0, #(\i*2) + add x7, x5, #(\i*2) + mov x8, #64*2 + bl inv_txfm_add_vert_dct_8x32_neon +.endr + + add sp, x5, #64*32*2 + br x15 +endfunc + +function inv_txfm_add_dct_dct_32x64_16bpc_neon, export=1 + idct_dc 32, 64, 1 + + mov x15, x30 + + sub_sp 32*32*2+64*8*2 + add x5, sp, #64*8*2 + + movrel x13, eob_32x32 + ldrh w12, [x13], #2 + +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add x6, x5, #(\i*32*2) +.if \i > 0 + mov w8, #(32 - \i) + cmp w3, w12 + b.lt 1f + ldrh w12, [x13], #2 +.endif + add x7, x2, #(\i*4) + mov x8, #32*4 + bl inv_txfm_horz_scale_dct_32x4_neon +.endr + b 3f + +1: + movi v4.8h, #0 + movi v5.8h, #0 + movi v6.8h, #0 + movi v7.8h, #0 +2: + subs w8, w8, #4 +.rept 4 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64 +.endr + b.gt 2b + +3: +.irp i, 0, 8, 16, 24 + add x7, x5, #(\i*2) + mov x8, #32*2 + bl X(inv_txfm_dct_8h_x64_neon) + add x6, x0, #(\i*2) + bl inv_txfm_add_vert_dct_8x64_neon +.endr + + add sp, x5, #32*32*2 + br x15 +endfunc + +function inv_txfm_add_dct_dct_64x16_16bpc_neon, export=1 + idct_dc 64, 16, 2 + + mov x15, x30 + + sub_sp 64*16*2+64*4*4 + add x4, sp, #64*4*4 + + movrel x13, eob_16x32 + +.irp i, 0, 4, 8, 12 + add x6, x4, #(\i*64*2) +.if \i > 0 + mov w8, #(16 - \i) + cmp w3, w12 + b.lt 1f +.endif + add x7, x2, #(\i*4) + mov x8, #16*4 + mov x12, #-2 // shift + bl inv_txfm_dct_clear_4s_x64_neon + add x6, x4, #(\i*64*2) + bl inv_txfm_horz_dct_64x4_neon +.if \i < 12 + ldrh w12, [x13], #2 +.endif +.endr + b 3f + +1: + movi v4.8h, #0 + movi v5.8h, #0 + movi v6.8h, #0 + movi v7.8h, #0 +2: + subs w8, w8, #2 +.rept 4 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64 +.endr + b.gt 2b + +3: + movrel x5, X(inv_dct_8h_x16_neon) +.irp i, 0, 8, 16, 24, 32, 40, 48, 56 + add x6, x0, #(\i*2) + add x7, x4, #(\i*2) + mov x8, #64*2 + bl inv_txfm_add_vert_8x16_neon +.endr + + add sp, x4, #64*16*2 + br x15 +endfunc + +function inv_txfm_add_dct_dct_16x64_16bpc_neon, export=1 + idct_dc 16, 64, 2 + + mov x15, x30 + + sub_sp 16*32*2+64*8*2 + add x5, sp, #64*8*2 + + movrel x13, eob_16x32 + ldrh w12, [x13], #2 + + adr x4, inv_dct_4s_x16_neon +.irp i, 0, 4, 8, 12, 16, 20, 24, 28 + add x6, x5, #(\i*16*2) +.if \i > 0 + mov w8, #(32 - \i) + cmp w3, w12 + b.lt 1f + ldrh w12, [x13], #2 +.endif + add x7, x2, #(\i*4) + mov x8, #32*4 + bl inv_txfm_horz_16x4_neon +.endr + b 3f + +1: + movi v4.8h, #0 + movi v5.8h, #0 + movi v6.8h, #0 + movi v7.8h, #0 +2: + subs w8, w8, #4 +.rept 2 + st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x6], #64 +.endr + b.gt 2b + +3: +.irp i, 0, 8 + add x7, x5, #(\i*2) + mov x8, #16*2 + bl X(inv_txfm_dct_8h_x64_neon) + add x6, x0, #(\i*2) + bl inv_txfm_add_vert_dct_8x64_neon +.endr + + add sp, x5, #16*32*2 + br x15 +endfunc diff --git a/third_party/dav1d/src/arm/64/mc.S b/third_party/dav1d/src/arm/64/mc.S --- a/third_party/dav1d/src/arm/64/mc.S +++ b/third_party/dav1d/src/arm/64/mc.S @@ -3087,5 +3087,163 @@ endfunc .endm warp , 11 warp t, 7 + +// void dav1d_emu_edge_8bpc_neon( +// const intptr_t bw, const intptr_t bh, +// const intptr_t iw, const intptr_t ih, +// const intptr_t x, const intptr_t y, +// pixel *dst, const ptrdiff_t dst_stride, +// const pixel *ref, const ptrdiff_t ref_stride) +function emu_edge_8bpc_neon, export=1 + ldp x8, x9, [sp] + + // ref += iclip(y, 0, ih - 1) * PXSTRIDE(ref_stride) + // ref += iclip(x, 0, iw - 1) + sub x12, x3, #1 // ih - 1 + cmp x5, x3 + sub x13, x2, #1 // iw - 1 + csel x12, x12, x5, ge // min(y, ih - 1) + cmp x4, x2 + bic x12, x12, x12, asr #63 // max(min(y, ih - 1), 0) + csel x13, x13, x4, ge // min(x, iw - 1) + bic x13, x13, x13, asr #63 // max(min(x, iw - 1), 0) + madd x8, x12, x9, x8 // ref += iclip() * stride + add x8, x8, x13 // ref += iclip() + + // bottom_ext = iclip(y + bh - ih, 0, bh - 1) + // top_ext = iclip(-y, 0, bh - 1) + add x10, x5, x1 // y + bh + neg x5, x5 // -y + sub x10, x10, x3 // y + bh - ih + sub x12, x1, #1 // bh - 1 + cmp x10, x1 + bic x5, x5, x5, asr #63 // max(-y, 0) + csel x10, x10, x12, lt // min(y + bh - ih, bh-1) + cmp x5, x1 + bic x10, x10, x10, asr #63 // max(min(y + bh - ih, bh-1), 0) + csel x5, x5, x12, lt // min(max(-y, 0), bh-1) + + // right_ext = iclip(x + bw - iw, 0, bw - 1) + // left_ext = iclip(-x, 0, bw - 1) + add x11, x4, x0 // x + bw + neg x4, x4 // -x + sub x11, x11, x2 // x + bw - iw + sub x13, x0, #1 // bw - 1 + cmp x11, x0 + bic x4, x4, x4, asr #63 // max(-x, 0) + csel x11, x11, x13, lt // min(x + bw - iw, bw-1) + cmp x4, x0 + bic x11, x11, x11, asr #63 // max(min(x + bw - iw, bw-1), 0) + csel x4, x4, x13, lt // min(max(-x, 0), bw - 1) + + // center_h = bh - top_ext - bottom_ext + // dst += top_ext * PXSTRIDE(dst_stride) + // center_w = bw - left_ext - right_ext + sub x1, x1, x5 // bh - top_ext + madd x6, x5, x7, x6 + sub x2, x0, x4 // bw - left_ext + sub x1, x1, x10 // center_h = bh - top_ext - bottom_ext + sub x2, x2, x11 // center_w = bw - left_ext - right_ext + + mov x14, x6 // backup of dst + +.macro v_loop need_left, need_right +0: +.if \need_left + ld1r {v0.16b}, [x8] + mov x12, x6 // out = dst + mov x3, x4 +1: + subs x3, x3, #16 + st1 {v0.16b}, [x12], #16 + b.gt 1b +.endif + mov x13, x8 + add x12, x6, x4 // out = dst + left_ext + mov x3, x2 +1: + ld1 {v0.16b, v1.16b}, [x13], #32 + subs x3, x3, #32 + st1 {v0.16b, v1.16b}, [x12], #32 + b.gt 1b +.if \need_right + add x3, x8, x2 // in + center_w + sub x3, x3, #1 // in + center_w - 1 + add x12, x6, x4 // dst + left_ext + ld1r {v0.16b}, [x3] + add x12, x12, x2 // out = dst + left_ext + center_w + mov x3, x11 +1: + subs x3, x3, #16 + st1 {v0.16b}, [x12], #16 + b.gt 1b +.endif + + subs x1, x1, #1 // center_h-- + add x6, x6, x7 + add x8, x8, x9 + b.gt 0b +.endm + + cbz x4, 2f + // need_left + cbz x11, 3f + // need_left + need_right + v_loop 1, 1 + b 5f + +2: + // !need_left + cbz x11, 4f + // !need_left + need_right + v_loop 0, 1 + b 5f + +3: + // need_left + !need_right + v_loop 1, 0 + b 5f + +4: + // !need_left + !need_right + v_loop 0, 0 + +5: + + cbz x10, 3f + // need_bottom + sub x8, x6, x7 // ref = dst - stride + mov x4, x0 +1: + ld1 {v0.16b, v1.16b}, [x8], #32 + mov x3, x10 +2: + subs x3, x3, #1 + st1 {v0.16b, v1.16b}, [x6], x7 + b.gt 2b + msub x6, x7, x10, x6 // dst -= bottom_ext * stride + subs x4, x4, #32 // bw -= 32 + add x6, x6, #32 // dst += 32 + b.gt 1b + +3: + cbz x5, 3f + // need_top + msub x6, x7, x5, x14 // dst = stored_dst - top_ext * stride +1: + ld1 {v0.16b, v1.16b}, [x14], #32 + mov x3, x5 +2: + subs x3, x3, #1 + st1 {v0.16b, v1.16b}, [x6], x7 + b.gt 2b + msub x6, x7, x5, x6 // dst -= top_ext * stride + subs x0, x0, #32 // bw -= 32 + add x6, x6, #32 // dst += 32 + b.gt 1b + +3: + ret +endfunc diff --git a/third_party/dav1d/src/arm/64/mc16.S b/third_party/dav1d/src/arm/64/mc16.S --- a/third_party/dav1d/src/arm/64/mc16.S +++ b/third_party/dav1d/src/arm/64/mc16.S @@ -3405,5 +3405,165 @@ endfunc .endm warp warp t + +// void dav1d_emu_edge_16bpc_neon( +// const intptr_t bw, const intptr_t bh, +// const intptr_t iw, const intptr_t ih, +// const intptr_t x, const intptr_t y, +// pixel *dst, const ptrdiff_t dst_stride, +// const pixel *ref, const ptrdiff_t ref_stride) +function emu_edge_16bpc_neon, export=1 + ldp x8, x9, [sp] + + // ref += iclip(y, 0, ih - 1) * PXSTRIDE(ref_stride) + // ref += iclip(x, 0, iw - 1) + sub x12, x3, #1 // ih - 1 + cmp x5, x3 + sub x13, x2, #1 // iw - 1 + csel x12, x12, x5, ge // min(y, ih - 1) + cmp x4, x2 + bic x12, x12, x12, asr #63 // max(min(y, ih - 1), 0) + csel x13, x13, x4, ge // min(x, iw - 1) + bic x13, x13, x13, asr #63 // max(min(x, iw - 1), 0) + madd x8, x12, x9, x8 // ref += iclip() * stride + add x8, x8, x13, lsl #1 // ref += iclip() + + // bottom_ext = iclip(y + bh - ih, 0, bh - 1) + // top_ext = iclip(-y, 0, bh - 1) + add x10, x5, x1 // y + bh + neg x5, x5 // -y + sub x10, x10, x3 // y + bh - ih + sub x12, x1, #1 // bh - 1 + cmp x10, x1 + bic x5, x5, x5, asr #63 // max(-y, 0) + csel x10, x10, x12, lt // min(y + bh - ih, bh-1) + cmp x5, x1 + bic x10, x10, x10, asr #63 // max(min(y + bh - ih, bh-1), 0) + csel x5, x5, x12, lt // min(max(-y, 0), bh-1) + + // right_ext = iclip(x + bw - iw, 0, bw - 1) + // left_ext = iclip(-x, 0, bw - 1) + add x11, x4, x0 // x + bw + neg x4, x4 // -x + sub x11, x11, x2 // x + bw - iw + sub x13, x0, #1 // bw - 1 + cmp x11, x0 + bic x4, x4, x4, asr #63 // max(-x, 0) + csel x11, x11, x13, lt // min(x + bw - iw, bw-1) + cmp x4, x0 + bic x11, x11, x11, asr #63 // max(min(x + bw - iw, bw-1), 0) + csel x4, x4, x13, lt // min(max(-x, 0), bw - 1) + + // center_h = bh - top_ext - bottom_ext + // dst += top_ext * PXSTRIDE(dst_stride) + // center_w = bw - left_ext - right_ext + sub x1, x1, x5 // bh - top_ext + madd x6, x5, x7, x6 + sub x2, x0, x4 // bw - left_ext + sub x1, x1, x10 // center_h = bh - top_ext - bottom_ext + sub x2, x2, x11 // center_w = bw - left_ext - right_ext + + mov x14, x6 // backup of dst + +.macro v_loop need_left, need_right +0: +.if \need_left + ld1r {v0.8h}, [x8] + mov x12, x6 // out = dst + mov x3, x4 + mov v1.16b, v0.16b +1: + subs x3, x3, #16 + st1 {v0.8h, v1.8h}, [x12], #32 + b.gt 1b +.endif + mov x13, x8 + add x12, x6, x4, lsl #1 // out = dst + left_ext + mov x3, x2 +1: + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x13], #64 + subs x3, x3, #32 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x12], #64 + b.gt 1b +.if \need_right + add x3, x8, x2, lsl #1 // in + center_w + sub x3, x3, #2 // in + center_w - 1 + add x12, x6, x4, lsl #1 // dst + left_ext + ld1r {v0.8h}, [x3] + add x12, x12, x2, lsl #1 // out = dst + left_ext + center_w + mov x3, x11 + mov v1.16b, v0.16b +1: + subs x3, x3, #16 + st1 {v0.8h, v1.8h}, [x12], #32 + b.gt 1b +.endif + + subs x1, x1, #1 // center_h-- + add x6, x6, x7 + add x8, x8, x9 + b.gt 0b +.endm + + cbz x4, 2f + // need_left + cbz x11, 3f + // need_left + need_right + v_loop 1, 1 + b 5f + +2: + // !need_left + cbz x11, 4f + // !need_left + need_right + v_loop 0, 1 + b 5f + +3: + // need_left + !need_right + v_loop 1, 0 + b 5f + +4: + // !need_left + !need_right + v_loop 0, 0 + +5: + + cbz x10, 3f + // need_bottom + sub x8, x6, x7 // ref = dst - stride + mov x4, x0 +1: + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x8], #64 + mov x3, x10 +2: + subs x3, x3, #1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x7 + b.gt 2b + msub x6, x7, x10, x6 // dst -= bottom_ext * stride + subs x4, x4, #32 // bw -= 32 + add x6, x6, #64 // dst += 32 + b.gt 1b + +3: + cbz x5, 3f + // need_top + msub x6, x7, x5, x14 // dst = stored_dst - top_ext * stride +1: + ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x14], #64 + mov x3, x5 +2: + subs x3, x3, #1 + st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x6], x7 + b.gt 2b + msub x6, x7, x5, x6 // dst -= top_ext * stride + subs x0, x0, #32 // bw -= 32 + add x6, x6, #64 // dst += 32 + b.gt 1b + +3: + ret +endfunc diff --git a/third_party/dav1d/src/arm/64/msac.S b/third_party/dav1d/src/arm/64/msac.S --- a/third_party/dav1d/src/arm/64/msac.S +++ b/third_party/dav1d/src/arm/64/msac.S @@ -116,13 +116,13 @@ sqdmulh \d1\sz, \s1\sz, \s3\sz .endif .endm .macro str_n idx0, idx1, dstreg, dstoff, n - str q\idx0, [\dstreg, \dstoff] + str \idx0, [\dstreg, \dstoff] .if \n == 16 - str q\idx1, [\dstreg, \dstoff + 16] + str \idx1, [\dstreg, \dstoff + 16] .endif .endm // unsigned dav1d_msac_decode_symbol_adapt4_neon(MsacContext *s, uint16_t *cdf, // size_t n_symbols); @@ -148,11 +148,11 @@ add_n v4, v5, v2, v3, v4, v5, \sz, \n // v = cdf + EC_MIN_PROB * (n_symbols - ret) add_n v4, v5, v6, v7, v4, v5, \sz, \n // v = ((cdf >> EC_PROB_SHIFT) * r) >> 1 + EC_MIN_PROB * (n_symbols - ret) ld1r {v6.8h}, [x8] // dif >> (EC_WIN_SIZE - 16) movrel x8, bits - str_n 4, 5, sp, #16, \n // store v values to allow indexed access + str_n q4, q5, sp, #16, \n // store v values to allow indexed access ld1_n v16, v17, x8, .8h, \n cmhs_n v2, v3, v6, v6, v4, v5, .8h, \n // c >= v @@ -183,11 +183,11 @@ .else lsr w14, w3, #4 // count >> 4 sbc w4, w4, w14 // -((count >> 4) + (n_symbols > 2) + 4) .endif sub_n v4, v5, v4, v5, v0, v1, \sz, \n // (32768 - cdf[i]) or (-1 - cdf[i]) - dup v6.8h, w4 // -rate + dup v6\sz, w4 // -rate sub w3, w3, w3, lsr #5 // count - (count == 32) sub_n v0, v1, v0, v1, v2, v3, \sz, \n // cdf + (i >= val ? 1 : 0) sshl_n v4, v5, v4, v5, v6, v6, \sz, \n // ({32768,-1} - cdf[i]) >> rate add w3, w3, #1 // count + (count < 32) @@ -214,11 +214,11 @@ lsl w4, w4, w5 // rng << d subs w6, w6, w5 // cnt -= d lsl x7, x7, x5 // (~dif + (v << 48)) << d str w4, [x0, #RNG] mvn x7, x7 // ~dif - b.ge 9f + b.hs 9f // refill ldp x3, x4, [x0] // BUF_POS, BUF_END add x5, x3, #8 cmp x5, x4 @@ -272,10 +272,132 @@ function msac_decode_symbol_adapt16_neon, export=1 decode_update .8h, .16b, 16 b L(renorm) endfunc +function msac_decode_hi_tok_neon, export=1 + ld1 {v0.4h}, [x1] // cdf + add x16, x0, #RNG + movi v31.4h, #0x7f, lsl #8 // 0x7f00 + movrel x17, coeffs, 30-2*3 + mvni v30.4h, #0x3f // 0xffc0 + ldrh w9, [x1, #6] // count = cdf[n_symbols] + ld1r {v3.4h}, [x16] // rng + movrel x16, bits + ld1 {v29.4h}, [x17] // EC_MIN_PROB * (n_symbols - ret) + add x17, x0, #DIF + 6 + ld1 {v16.8h}, [x16] + mov w13, #-24 + and v17.8b, v0.8b, v30.8b // cdf & 0xffc0 + ldr w10, [x0, #ALLOW_UPDATE_CDF] + ld1r {v1.8h}, [x17] // dif >> (EC_WIN_SIZE - 16) + sub sp, sp, #48 + ldr w6, [x0, #CNT] + ldr x7, [x0, #DIF] +1: + and v7.8b, v3.8b, v31.8b // rng & 0x7f00 + sqdmulh v6.4h, v17.4h, v7.4h // ((cdf >> EC_PROB_SHIFT) * (r - 128)) >> 1 + add v4.4h, v17.4h, v29.4h // v = cdf + EC_MIN_PROB * (n_symbols - ret) + add v4.4h, v6.4h, v4.4h // v = ((cdf >> EC_PROB_SHIFT) * r) >> 1 + EC_MIN_PROB * (n_symbols - ret) + str h3, [sp, #14] // store original u = s->rng + cmhs v2.8h, v1.8h, v4.8h // c >= v + str q4, [sp, #16] // store v values to allow indexed access + and v6.16b, v2.16b, v16.16b // One bit per halfword set in the mask + addv h6, v6.8h // Aggregate mask bits + umov w3, v6.h[0] + add w13, w13, #5 + rbit w3, w3 + add x8, sp, #16 + clz w15, w3 // ret + + cbz w10, 2f + // update_cdf + movi v5.8b, #0xff + mov w4, #-5 + urhadd v4.4h, v5.4h, v2.4h // i >= val ? -1 : 32768 + sub w4, w4, w9, lsr #4 // -((count >> 4) + 5) + sub v4.4h, v4.4h, v0.4h // (32768 - cdf[i]) or (-1 - cdf[i]) + dup v6.4h, w4 // -rate + + sub w9, w9, w9, lsr #5 // count - (count == 32) + sub v0.4h, v0.4h, v2.4h // cdf + (i >= val ? 1 : 0) + sshl v4.4h, v4.4h, v6.4h // ({32768,-1} - cdf[i]) >> rate + add w9, w9, #1 // count + (count < 32) + add v0.4h, v0.4h, v4.4h // cdf + (32768 - cdf[i]) >> rate + st1 {v0.4h}, [x1] + and v17.8b, v0.8b, v30.8b // cdf & 0xffc0 + strh w9, [x1, #6] + +2: + add x8, x8, w15, uxtw #1 + ldrh w3, [x8] // v + ldurh w4, [x8, #-2] // u + sub w4, w4, w3 // rng = u - v + clz w5, w4 // clz(rng) + eor w5, w5, #16 // d = clz(rng) ^ 16 + mvn x7, x7 // ~dif + add x7, x7, x3, lsl #48 // ~dif + (v << 48) + lsl w4, w4, w5 // rng << d + subs w6, w6, w5 // cnt -= d + lsl x7, x7, x5 // (~dif + (v << 48)) << d + str w4, [x0, #RNG] + dup v3.4h, w4 + mvn x7, x7 // ~dif + b.hs 9f + + // refill + ldp x3, x4, [x0] // BUF_POS, BUF_END + add x5, x3, #8 + cmp x5, x4 + b.gt 2f + + ldr x3, [x3] // next_bits + add w8, w6, #23 // shift_bits = cnt + 23 + add w6, w6, #16 // cnt += 16 + rev x3, x3 // next_bits = bswap(next_bits) + sub x5, x5, x8, lsr #3 // buf_pos -= shift_bits >> 3 + and w8, w8, #24 // shift_bits &= 24 + lsr x3, x3, x8 // next_bits >>= shift_bits + sub w8, w8, w6 // shift_bits -= 16 + cnt + str x5, [x0, #BUF_POS] + lsl x3, x3, x8 // next_bits <<= shift_bits + mov w4, #48 + sub w6, w4, w8 // cnt = cnt + 64 - shift_bits + eor x7, x7, x3 // dif ^= next_bits + b 9f + +2: // refill_eob + mov w14, #40 + sub w5, w14, w6 // c = 40 - cnt +3: + cmp x3, x4 + b.ge 4f + ldrb w8, [x3], #1 + lsl x8, x8, x5 + eor x7, x7, x8 + subs w5, w5, #8 + b.ge 3b + +4: // refill_eob_end + str x3, [x0, #BUF_POS] + sub w6, w14, w5 // cnt = 40 - c + +9: + lsl w15, w15, #1 + sub w15, w15, #5 + lsr x12, x7, #48 + adds w13, w13, w15 // carry = tok_br < 3 || tok == 15 + dup v1.8h, w12 + b.cc 1b // loop if !carry + add w13, w13, #30 + str w6, [x0, #CNT] + add sp, sp, #48 + str x7, [x0, #DIF] + lsr w0, w13, #1 + ret +endfunc + function msac_decode_bool_equi_neon, export=1 ldp w5, w6, [x0, #RNG] // + CNT sub sp, sp, #48 ldr x7, [x0, #DIF] bic w4, w5, #0xff // r &= 0xff00 diff --git a/third_party/dav1d/src/arm/64/util.S b/third_party/dav1d/src/arm/64/util.S --- a/third_party/dav1d/src/arm/64/util.S +++ b/third_party/dav1d/src/arm/64/util.S @@ -168,10 +168,22 @@ trn2 \r2\().2s, \t4\().2s, \t6\().2s trn1 \r1\().2s, \t5\().2s, \t7\().2s trn2 \r3\().2s, \t5\().2s, \t7\().2s .endm +.macro transpose_4x4s r0, r1, r2, r3, t4, t5, t6, t7 + trn1 \t4\().4s, \r0\().4s, \r1\().4s + trn2 \t5\().4s, \r0\().4s, \r1\().4s + trn1 \t6\().4s, \r2\().4s, \r3\().4s + trn2 \t7\().4s, \r2\().4s, \r3\().4s + + trn1 \r0\().2d, \t4\().2d, \t6\().2d + trn2 \r2\().2d, \t4\().2d, \t6\().2d + trn1 \r1\().2d, \t5\().2d, \t7\().2d + trn2 \r3\().2d, \t5\().2d, \t7\().2d +.endm + .macro transpose_4x8h r0, r1, r2, r3, t4, t5, t6, t7 trn1 \t4\().8h, \r0\().8h, \r1\().8h trn2 \t5\().8h, \r0\().8h, \r1\().8h trn1 \t6\().8h, \r2\().8h, \r3\().8h trn2 \t7\().8h, \r2\().8h, \r3\().8h diff --git a/third_party/dav1d/src/arm/asm.S b/third_party/dav1d/src/arm/asm.S --- a/third_party/dav1d/src/arm/asm.S +++ b/third_party/dav1d/src/arm/asm.S @@ -91,10 +91,11 @@ .align \align .if \export .global EXTERN\name #ifdef __ELF__ .type EXTERN\name, %function + .hidden EXTERN\name #endif #if HAVE_AS_FUNC .func EXTERN\name #endif EXTERN\name: @@ -107,11 +108,11 @@ #endif .endif \name: .endm -.macro const name, align=2 +.macro const name, export=0, align=2 .macro endconst #ifdef __ELF__ .size \name, . - \name #endif .purgem endconst @@ -122,10 +123,17 @@ .section .rodata #else .const_data #endif .align \align + .if \export + .global EXTERN\name +#ifdef __ELF__ + .hidden EXTERN\name +#endif +EXTERN\name: + .endif \name: .endm #ifdef __APPLE__ #define L(x) L ## x @@ -133,6 +141,11 @@ #define L(x) .L ## x #endif #define X(x) CONCAT(EXTERN, x) +#if ARCH_AARCH64 +#define x18 do_not_use_x18 +#define w18 do_not_use_w18 +#endif + #endif /* DAV1D_SRC_ARM_ASM_S */ diff --git a/third_party/dav1d/src/arm/cpu.c b/third_party/dav1d/src/arm/cpu.c --- a/third_party/dav1d/src/arm/cpu.c +++ b/third_party/dav1d/src/arm/cpu.c @@ -64,11 +64,14 @@ return 1; } // if line is incomplete seek back to avoid splitting the search // string into two buffers if (!strchr(line, '\n') && strlen(line) > strlen(flag)) { - if (fseeko(file, -strlen(flag), SEEK_CUR)) + // use fseek since the 64 bit fseeko is only available since + // Android API level 24 and meson defines _FILE_OFFSET_BITS + // by default 64 + if (fseek(file, -strlen(flag), SEEK_CUR)) break; } } fclose(file); diff --git a/third_party/dav1d/src/arm/ipred_init_tmpl.c b/third_party/dav1d/src/arm/ipred_init_tmpl.c --- a/third_party/dav1d/src/arm/ipred_init_tmpl.c +++ b/third_party/dav1d/src/arm/ipred_init_tmpl.c @@ -25,58 +25,58 @@ */ #include "src/cpu.h" #include "src/ipred.h" -decl_angular_ipred_fn(dav1d_ipred_dc_neon); -decl_angular_ipred_fn(dav1d_ipred_dc_128_neon); -decl_angular_ipred_fn(dav1d_ipred_dc_top_neon); -decl_angular_ipred_fn(dav1d_ipred_dc_left_neon); -decl_angular_ipred_fn(dav1d_ipred_h_neon); -decl_angular_ipred_fn(dav1d_ipred_v_neon); -decl_angular_ipred_fn(dav1d_ipred_paeth_neon); -decl_angular_ipred_fn(dav1d_ipred_smooth_neon); -decl_angular_ipred_fn(dav1d_ipred_smooth_v_neon); -decl_angular_ipred_fn(dav1d_ipred_smooth_h_neon); -decl_angular_ipred_fn(dav1d_ipred_filter_neon); +decl_angular_ipred_fn(BF(dav1d_ipred_dc, neon)); +decl_angular_ipred_fn(BF(dav1d_ipred_dc_128, neon)); +decl_angular_ipred_fn(BF(dav1d_ipred_dc_top, neon)); +decl_angular_ipred_fn(BF(dav1d_ipred_dc_left, neon)); +decl_angular_ipred_fn(BF(dav1d_ipred_h, neon)); +decl_angular_ipred_fn(BF(dav1d_ipred_v, neon)); +decl_angular_ipred_fn(BF(dav1d_ipred_paeth, neon)); +decl_angular_ipred_fn(BF(dav1d_ipred_smooth, neon)); +decl_angular_ipred_fn(BF(dav1d_ipred_smooth_v, neon)); +decl_angular_ipred_fn(BF(dav1d_ipred_smooth_h, neon)); +decl_angular_ipred_fn(BF(dav1d_ipred_filter, neon)); -decl_cfl_pred_fn(dav1d_ipred_cfl_neon); -decl_cfl_pred_fn(dav1d_ipred_cfl_128_neon); -decl_cfl_pred_fn(dav1d_ipred_cfl_top_neon); -decl_cfl_pred_fn(dav1d_ipred_cfl_left_neon); +decl_cfl_pred_fn(BF(dav1d_ipred_cfl, neon)); +decl_cfl_pred_fn(BF(dav1d_ipred_cfl_128, neon)); +decl_cfl_pred_fn(BF(dav1d_ipred_cfl_top, neon)); +decl_cfl_pred_fn(BF(dav1d_ipred_cfl_left, neon)); -decl_cfl_ac_fn(dav1d_ipred_cfl_ac_420_neon); -decl_cfl_ac_fn(dav1d_ipred_cfl_ac_422_neon); +decl_cfl_ac_fn(BF(dav1d_ipred_cfl_ac_420, neon)); +decl_cfl_ac_fn(BF(dav1d_ipred_cfl_ac_422, neon)); +decl_cfl_ac_fn(BF(dav1d_ipred_cfl_ac_444, neon)); -decl_pal_pred_fn(dav1d_pal_pred_neon); +decl_pal_pred_fn(BF(dav1d_pal_pred, neon)); COLD void bitfn(dav1d_intra_pred_dsp_init_arm)(Dav1dIntraPredDSPContext *const c) { const unsigned flags = dav1d_get_cpu_flags(); if (!(flags & DAV1D_ARM_CPU_FLAG_NEON)) return; -#if BITDEPTH == 8 - c->intra_pred[DC_PRED] = dav1d_ipred_dc_neon; - c->intra_pred[DC_128_PRED] = dav1d_ipred_dc_128_neon; - c->intra_pred[TOP_DC_PRED] = dav1d_ipred_dc_top_neon; - c->intra_pred[LEFT_DC_PRED] = dav1d_ipred_dc_left_neon; - c->intra_pred[HOR_PRED] = dav1d_ipred_h_neon; - c->intra_pred[VERT_PRED] = dav1d_ipred_v_neon; -#if ARCH_AARCH64 - c->intra_pred[PAETH_PRED] = dav1d_ipred_paeth_neon; - c->intra_pred[SMOOTH_PRED] = dav1d_ipred_smooth_neon; - c->intra_pred[SMOOTH_V_PRED] = dav1d_ipred_smooth_v_neon; - c->intra_pred[SMOOTH_H_PRED] = dav1d_ipred_smooth_h_neon; - c->intra_pred[FILTER_PRED] = dav1d_ipred_filter_neon; +#if BITDEPTH == 8 || ARCH_AARCH64 + c->intra_pred[DC_PRED] = BF(dav1d_ipred_dc, neon); + c->intra_pred[DC_128_PRED] = BF(dav1d_ipred_dc_128, neon); + c->intra_pred[TOP_DC_PRED] = BF(dav1d_ipred_dc_top, neon); + c->intra_pred[LEFT_DC_PRED] = BF(dav1d_ipred_dc_left, neon); + c->intra_pred[HOR_PRED] = BF(dav1d_ipred_h, neon); + c->intra_pred[VERT_PRED] = BF(dav1d_ipred_v, neon); + c->intra_pred[PAETH_PRED] = BF(dav1d_ipred_paeth, neon); + c->intra_pred[SMOOTH_PRED] = BF(dav1d_ipred_smooth, neon); + c->intra_pred[SMOOTH_V_PRED] = BF(dav1d_ipred_smooth_v, neon); + c->intra_pred[SMOOTH_H_PRED] = BF(dav1d_ipred_smooth_h, neon); + c->intra_pred[FILTER_PRED] = BF(dav1d_ipred_filter, neon); - c->cfl_pred[DC_PRED] = dav1d_ipred_cfl_neon; - c->cfl_pred[DC_128_PRED] = dav1d_ipred_cfl_128_neon; - c->cfl_pred[TOP_DC_PRED] = dav1d_ipred_cfl_top_neon; - c->cfl_pred[LEFT_DC_PRED] = dav1d_ipred_cfl_left_neon; + c->cfl_pred[DC_PRED] = BF(dav1d_ipred_cfl, neon); + c->cfl_pred[DC_128_PRED] = BF(dav1d_ipred_cfl_128, neon); + c->cfl_pred[TOP_DC_PRED] = BF(dav1d_ipred_cfl_top, neon); + c->cfl_pred[LEFT_DC_PRED] = BF(dav1d_ipred_cfl_left, neon); - c->cfl_ac[DAV1D_PIXEL_LAYOUT_I420 - 1] = dav1d_ipred_cfl_ac_420_neon; - c->cfl_ac[DAV1D_PIXEL_LAYOUT_I422 - 1] = dav1d_ipred_cfl_ac_422_neon; + c->cfl_ac[DAV1D_PIXEL_LAYOUT_I420 - 1] = BF(dav1d_ipred_cfl_ac_420, neon); + c->cfl_ac[DAV1D_PIXEL_LAYOUT_I422 - 1] = BF(dav1d_ipred_cfl_ac_422, neon); + c->cfl_ac[DAV1D_PIXEL_LAYOUT_I444 - 1] = BF(dav1d_ipred_cfl_ac_444, neon); - c->pal_pred = dav1d_pal_pred_neon; -#endif + c->pal_pred = BF(dav1d_pal_pred, neon); #endif } diff --git a/third_party/dav1d/src/arm/itx_init_tmpl.c b/third_party/dav1d/src/arm/itx_init_tmpl.c --- a/third_party/dav1d/src/arm/itx_init_tmpl.c +++ b/third_party/dav1d/src/arm/itx_init_tmpl.c @@ -27,36 +27,36 @@ #include "src/cpu.h" #include "src/itx.h" #define decl_itx2_fns(w, h, opt) \ -decl_itx_fn(dav1d_inv_txfm_add_dct_dct_##w##x##h##_##opt); \ -decl_itx_fn(dav1d_inv_txfm_add_identity_identity_##w##x##h##_##opt) +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_identity_identity_##w##x##h, opt)) #define decl_itx12_fns(w, h, opt) \ decl_itx2_fns(w, h, opt); \ -decl_itx_fn(dav1d_inv_txfm_add_dct_adst_##w##x##h##_##opt); \ -decl_itx_fn(dav1d_inv_txfm_add_dct_flipadst_##w##x##h##_##opt); \ -decl_itx_fn(dav1d_inv_txfm_add_dct_identity_##w##x##h##_##opt); \ -decl_itx_fn(dav1d_inv_txfm_add_adst_dct_##w##x##h##_##opt); \ -decl_itx_fn(dav1d_inv_txfm_add_adst_adst_##w##x##h##_##opt); \ -decl_itx_fn(dav1d_inv_txfm_add_adst_flipadst_##w##x##h##_##opt); \ -decl_itx_fn(dav1d_inv_txfm_add_flipadst_dct_##w##x##h##_##opt); \ -decl_itx_fn(dav1d_inv_txfm_add_flipadst_adst_##w##x##h##_##opt); \ -decl_itx_fn(dav1d_inv_txfm_add_flipadst_flipadst_##w##x##h##_##opt); \ -decl_itx_fn(dav1d_inv_txfm_add_identity_dct_##w##x##h##_##opt) +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_adst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_flipadst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_identity_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_adst_dct_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_adst_adst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_adst_flipadst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_dct_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_adst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_flipadst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_identity_dct_##w##x##h, opt)) #define decl_itx16_fns(w, h, opt) \ decl_itx12_fns(w, h, opt); \ -decl_itx_fn(dav1d_inv_txfm_add_adst_identity_##w##x##h##_##opt); \ -decl_itx_fn(dav1d_inv_txfm_add_flipadst_identity_##w##x##h##_##opt); \ -decl_itx_fn(dav1d_inv_txfm_add_identity_adst_##w##x##h##_##opt); \ -decl_itx_fn(dav1d_inv_txfm_add_identity_flipadst_##w##x##h##_##opt) +decl_itx_fn(BF(dav1d_inv_txfm_add_adst_identity_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_flipadst_identity_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_identity_adst_##w##x##h, opt)); \ +decl_itx_fn(BF(dav1d_inv_txfm_add_identity_flipadst_##w##x##h, opt)) #define decl_itx17_fns(w, h, opt) \ decl_itx16_fns(w, h, opt); \ -decl_itx_fn(dav1d_inv_txfm_add_wht_wht_##w##x##h##_##opt) +decl_itx_fn(BF(dav1d_inv_txfm_add_wht_wht_##w##x##h, opt)) decl_itx17_fns( 4, 4, neon); decl_itx16_fns( 4, 8, neon); decl_itx16_fns( 4, 16, neon); decl_itx16_fns( 8, 4, neon); @@ -69,20 +69,20 @@ decl_itx2_fns (16, 32, neon); decl_itx2_fns (32, 8, neon); decl_itx2_fns (32, 16, neon); decl_itx2_fns (32, 32, neon); -decl_itx_fn(dav1d_inv_txfm_add_dct_dct_16x64_neon); -decl_itx_fn(dav1d_inv_txfm_add_dct_dct_32x64_neon); -decl_itx_fn(dav1d_inv_txfm_add_dct_dct_64x16_neon); -decl_itx_fn(dav1d_inv_txfm_add_dct_dct_64x32_neon); -decl_itx_fn(dav1d_inv_txfm_add_dct_dct_64x64_neon); +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_16x64, neon)); +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_32x64, neon)); +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_64x16, neon)); +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_64x32, neon)); +decl_itx_fn(BF(dav1d_inv_txfm_add_dct_dct_64x64, neon)); -COLD void bitfn(dav1d_itx_dsp_init_arm)(Dav1dInvTxfmDSPContext *const c) { +COLD void bitfn(dav1d_itx_dsp_init_arm)(Dav1dInvTxfmDSPContext *const c, int bpc) { #define assign_itx_fn(pfx, w, h, type, type_enum, ext) \ c->itxfm_add[pfx##TX_##w##X##h][type_enum] = \ - dav1d_inv_txfm_add_##type##_##w##x##h##_##ext + BF(dav1d_inv_txfm_add_##type##_##w##x##h, ext) #define assign_itx1_fn(pfx, w, h, ext) \ assign_itx_fn(pfx, w, h, dct_dct, DCT_DCT, ext) #define assign_itx2_fn(pfx, w, h, ext) \ @@ -115,11 +115,13 @@ const unsigned flags = dav1d_get_cpu_flags(); if (!(flags & DAV1D_ARM_CPU_FLAG_NEON)) return; -#if BITDEPTH == 8 && ARCH_AARCH64 + if (bpc > 10) return; + +#if ARCH_AARCH64 || BITDEPTH == 8 assign_itx17_fn( , 4, 4, neon); assign_itx16_fn(R, 4, 8, neon); assign_itx16_fn(R, 4, 16, neon); assign_itx16_fn(R, 8, 4, neon); assign_itx16_fn( , 8, 8, neon); diff --git a/third_party/dav1d/src/arm/mc_init_tmpl.c b/third_party/dav1d/src/arm/mc_init_tmpl.c --- a/third_party/dav1d/src/arm/mc_init_tmpl.c +++ b/third_party/dav1d/src/arm/mc_init_tmpl.c @@ -64,10 +64,12 @@ decl_w_mask_fn(BF(dav1d_w_mask_420, neon)); decl_warp8x8_fn(BF(dav1d_warp_affine_8x8, neon)); decl_warp8x8t_fn(BF(dav1d_warp_affine_8x8t, neon)); +decl_emu_edge_fn(BF(dav1d_emu_edge, neon)); + void bitfn(dav1d_mc_dsp_init_arm)(Dav1dMCDSPContext *const c) { #define init_mc_fn(type, name, suffix) \ c->mc[type] = BF(dav1d_put_##name, suffix) #define init_mct_fn(type, name, suffix) \ c->mct[type] = BF(dav1d_prep_##name, suffix) @@ -107,7 +109,8 @@ c->w_mask[0] = BF(dav1d_w_mask_444, neon); c->w_mask[1] = BF(dav1d_w_mask_422, neon); c->w_mask[2] = BF(dav1d_w_mask_420, neon); c->warp8x8 = BF(dav1d_warp_affine_8x8, neon); c->warp8x8t = BF(dav1d_warp_affine_8x8t, neon); + c->emu_edge = BF(dav1d_emu_edge, neon); #endif } diff --git a/third_party/dav1d/src/arm/msac.h b/third_party/dav1d/src/arm/msac.h --- a/third_party/dav1d/src/arm/msac.h +++ b/third_party/dav1d/src/arm/msac.h @@ -32,18 +32,20 @@ size_t n_symbols); unsigned dav1d_msac_decode_symbol_adapt8_neon(MsacContext *s, uint16_t *cdf, size_t n_symbols); unsigned dav1d_msac_decode_symbol_adapt16_neon(MsacContext *s, uint16_t *cdf, size_t n_symbols); +unsigned dav1d_msac_decode_hi_tok_neon(MsacContext *s, uint16_t *cdf); unsigned dav1d_msac_decode_bool_adapt_neon(MsacContext *s, uint16_t *cdf); unsigned dav1d_msac_decode_bool_equi_neon(MsacContext *s); unsigned dav1d_msac_decode_bool_neon(MsacContext *s, unsigned f); -#if ARCH_AARCH64 +#if ARCH_AARCH64 || defined(__ARM_NEON) #define dav1d_msac_decode_symbol_adapt4 dav1d_msac_decode_symbol_adapt4_neon #define dav1d_msac_decode_symbol_adapt8 dav1d_msac_decode_symbol_adapt8_neon #define dav1d_msac_decode_symbol_adapt16 dav1d_msac_decode_symbol_adapt16_neon +#define dav1d_msac_decode_hi_tok dav1d_msac_decode_hi_tok_neon #define dav1d_msac_decode_bool_adapt dav1d_msac_decode_bool_adapt_neon #define dav1d_msac_decode_bool_equi dav1d_msac_decode_bool_equi_neon #define dav1d_msac_decode_bool dav1d_msac_decode_bool_neon #endif diff --git a/third_party/dav1d/src/cdef_apply_tmpl.c b/third_party/dav1d/src/cdef_apply_tmpl.c --- a/third_party/dav1d/src/cdef_apply_tmpl.c +++ b/third_party/dav1d/src/cdef_apply_tmpl.c @@ -109,10 +109,13 @@ const int damping = f->frame_hdr->cdef.damping + bitdepth_min_8; const enum Dav1dPixelLayout layout = f->cur.p.layout; const int uv_idx = DAV1D_PIXEL_LAYOUT_I444 - layout; const int ss_ver = layout == DAV1D_PIXEL_LAYOUT_I420; const int ss_hor = layout != DAV1D_PIXEL_LAYOUT_I444; + static const uint8_t uv_dirs[2][8] = { { 0, 1, 2, 3, 4, 5, 6, 7 }, + { 7, 0, 2, 4, 5, 6, 6, 6 } }; + const uint8_t *uv_dir = uv_dirs[layout == DAV1D_PIXEL_LAYOUT_I422]; for (int bit = 0, by = by_start; by < by_end; by += 2, edges |= CDEF_HAVE_TOP) { const int tf = f->lf.top_pre_cdef_toggle; const int by_idx = by & 30; if (by + 2 >= f->bh) edges &= ~CDEF_HAVE_BOTTOM; @@ -197,12 +200,11 @@ &f->lf.cdef_line[tf][0][bx * 4], 0, y_sec_lvl, 0, damping, edges HIGHBD_CALL_SUFFIX); if (uv_lvl) { assert(layout != DAV1D_PIXEL_LAYOUT_I400); - const int uvdir = uv_pri_lvl ? layout == DAV1D_PIXEL_LAYOUT_I422 ? - ((const uint8_t[]) { 7, 0, 2, 4, 5, 6, 6, 6 })[dir] : dir : 0; + const int uvdir = uv_pri_lvl ? uv_dir[dir] : 0; for (int pl = 1; pl <= 2; pl++) { dsp->cdef.fb[uv_idx](bptrs[pl], f->cur.stride[1], lr_bak[bit][pl], &f->lf.cdef_line[tf][pl][bx * 4 >> ss_hor], uv_pri_lvl, uv_sec_lvl, uvdir, damping - 1, edges HIGHBD_CALL_SUFFIX); diff --git a/third_party/dav1d/src/cpu.c b/third_party/dav1d/src/cpu.c --- a/third_party/dav1d/src/cpu.c +++ b/third_party/dav1d/src/cpu.c @@ -29,11 +29,15 @@ #include #include "src/cpu.h" static unsigned flags = 0; -#if ARCH_X86 + +#if __has_feature(memory_sanitizer) +// memory sanitizer is inherently incompatible with asm +static unsigned flags_mask = 0; +#elif ARCH_X86 /* Disable AVX-512 by default for the time being */ static unsigned flags_mask = ~DAV1D_X86_CPU_FLAG_AVX512ICL; #else static unsigned flags_mask = -1; #endif diff --git a/third_party/dav1d/src/decode.c b/third_party/dav1d/src/decode.c --- a/third_party/dav1d/src/decode.c +++ b/third_party/dav1d/src/decode.c @@ -221,24 +221,22 @@ const int bw4, const int bh4, const int w4, const int h4, const int have_left, const int have_top, const int ref, uint64_t masks[2]) { - const Dav1dFrameContext *const f = t->f; - const ptrdiff_t b4_stride = f->b4_stride; - const refmvs *const r = &f->mvs[t->by * b4_stride + t->bx]; + /*const*/ refmvs_block *const *r = &t->rt.r[(t->by & 31) + 5]; int count = 0; int have_topleft = have_top && have_left; int have_topright = imax(bw4, bh4) < 32 && have_top && t->bx + bw4 < t->ts->tiling.col_end && (intra_edge_flags & EDGE_I444_TOP_HAS_RIGHT); -#define bs(rp) dav1d_block_dimensions[dav1d_sbtype_to_bs[(rp)->sb_type]] -#define matches(rp) ((rp)->ref[0] == ref + 1 && (rp)->ref[1] == -1) +#define bs(rp) dav1d_block_dimensions[(rp)->bs] +#define matches(rp) ((rp)->ref.ref[0] == ref + 1 && (rp)->ref.ref[1] == -1) if (have_top) { - const refmvs *r2 = &r[-b4_stride]; + const refmvs_block *r2 = &r[-1][t->bx]; if (matches(r2)) { masks[0] |= 1; count = 1; } int aw4 = bs(r2)[0]; @@ -258,84 +256,82 @@ mask <<= aw4; } } } if (have_left) { - const refmvs *r2 = &r[-1]; - if (matches(r2)) { + /*const*/ refmvs_block *const *r2 = r; + if (matches(&r2[0][t->bx - 1])) { masks[1] |= 1; if (++count >= 8) return; } - int lh4 = bs(r2)[1]; + int lh4 = bs(&r2[0][t->bx - 1])[1]; if (lh4 >= bh4) { if (t->by & (lh4 - 1)) have_topleft = 0; } else { unsigned mask = 1 << lh4; for (int y = lh4; y < h4; y += lh4) { - r2 += lh4 * b4_stride; - if (matches(r2)) { + r2 += lh4; + if (matches(&r2[0][t->bx - 1])) { masks[1] |= mask; if (++count >= 8) return; } - lh4 = bs(r2)[1]; + lh4 = bs(&r2[0][t->bx - 1])[1]; mask <<= lh4; } } } - if (have_topleft && matches(&r[-(1 + b4_stride)])) { + if (have_topleft && matches(&r[-1][t->bx - 1])) { masks[1] |= 1ULL << 32; if (++count >= 8) return; } - if (have_topright && matches(&r[bw4 - b4_stride])) { + if (have_topright && matches(&r[-1][t->bx + bw4])) { masks[0] |= 1ULL << 32; } #undef matches } static void derive_warpmv(const Dav1dTileContext *const t, const int bw4, const int bh4, - const uint64_t masks[2], const struct mv mv, + const uint64_t masks[2], const union mv mv, Dav1dWarpedMotionParams *const wmp) { int pts[8][2 /* in, out */][2 /* x, y */], np = 0; - const Dav1dFrameContext *const f = t->f; - const ptrdiff_t b4_stride = f->b4_stride; - const refmvs *const r = &f->mvs[t->by * b4_stride + t->bx]; + /*const*/ refmvs_block *const *r = &t->rt.r[(t->by & 31) + 5]; #define add_sample(dx, dy, sx, sy, rp) do { \ pts[np][0][0] = 16 * (2 * dx + sx * bs(rp)[0]) - 8; \ pts[np][0][1] = 16 * (2 * dy + sy * bs(rp)[1]) - 8; \ - pts[np][1][0] = pts[np][0][0] + (rp)->mv[0].x; \ - pts[np][1][1] = pts[np][0][1] + (rp)->mv[0].y; \ + pts[np][1][0] = pts[np][0][0] + (rp)->mv.mv[0].x; \ + pts[np][1][1] = pts[np][0][1] + (rp)->mv.mv[0].y; \ np++; \ } while (0) // use masks[] to find the projectable motion vectors in the edges if ((unsigned) masks[0] == 1 && !(masks[1] >> 32)) { - const int off = t->bx & (bs(&r[-b4_stride])[0] - 1); - add_sample(-off, 0, 1, -1, &r[-b4_stride]); + const int off = t->bx & (bs(&r[-1][t->bx])[0] - 1); + add_sample(-off, 0, 1, -1, &r[-1][t->bx]); } else for (unsigned off = 0, xmask = (uint32_t) masks[0]; np < 8 && xmask;) { // top const int tz = ctz(xmask); off += tz; xmask >>= tz; - add_sample(off, 0, 1, -1, &r[off - b4_stride]); + add_sample(off, 0, 1, -1, &r[-1][t->bx + off]); xmask &= ~1; } if (np < 8 && masks[1] == 1) { - const int off = t->by & (bs(&r[-1])[1] - 1); - add_sample(0, -off, -1, 1, &r[-1 - off * b4_stride]); + const int off = t->by & (bs(&r[0][t->bx - 1])[1] - 1); + add_sample(0, -off, -1, 1, &r[-off][t->bx - 1]); } else for (unsigned off = 0, ymask = (uint32_t) masks[1]; np < 8 && ymask;) { // left const int tz = ctz(ymask); off += tz; ymask >>= tz; - add_sample(0, off, -1, 1, &r[off * b4_stride - 1]); + add_sample(0, off, -1, 1, &r[off][t->bx - 1]); ymask &= ~1; } if (np < 8 && masks[1] >> 32) // top/left - add_sample(0, 0, -1, -1, &r[-(1 + b4_stride)]); + add_sample(0, 0, -1, -1, &r[-1][t->bx - 1]); if (np < 8 && masks[0] >> 32) // top/right - add_sample(bw4, 0, 1, -1, &r[bw4 - b4_stride]); + add_sample(bw4, 0, 1, -1, &r[-1][t->bx + bw4]); assert(np > 0 && np <= 8); #undef bs // select according to motion vector difference against a threshold int mvd[8], ret = 0; @@ -623,11 +619,11 @@ const Dav1dFrameContext *const f = t->f; const uint8_t *const b_dim = dav1d_block_dimensions[bs]; const int bw4 = b_dim[0], bh4 = b_dim[1]; // var-tx tree coding - b->tx_split[0] = b->tx_split[1] = 0; + uint16_t tx_split[2] = { 0 }; b->max_ytx = dav1d_max_txfm_size_for_bs[bs][0]; if (!b->skip && (f->frame_hdr->segmentation.lossless[b->seg_id] || b->max_ytx == TX_4X4)) { b->max_ytx = b->uvtx = TX_4X4; @@ -651,23 +647,26 @@ assert(bw4 <= 16 || bh4 <= 16 || b->max_ytx == TX_64X64); int y, x, y_off, x_off; const TxfmInfo *const ytx = &dav1d_txfm_dimensions[b->max_ytx]; for (y = 0, y_off = 0; y < bh4; y += ytx->h, y_off++) { for (x = 0, x_off = 0; x < bw4; x += ytx->w, x_off++) { - read_tx_tree(t, b->max_ytx, 0, b->tx_split, x_off, y_off); + read_tx_tree(t, b->max_ytx, 0, tx_split, x_off, y_off); // contexts are updated inside read_tx_tree() t->bx += ytx->w; } t->bx -= x; t->by += ytx->h; } t->by -= y; if (DEBUG_BLOCK_INFO) printf("Post-vartxtree[%x/%x]: r=%d\n", - b->tx_split[0], b->tx_split[1], t->ts->msac.rng); + tx_split[0], tx_split[1], t->ts->msac.rng); b->uvtx = dav1d_max_txfm_size_for_bs[bs][f->cur.p.layout]; } + assert(!(tx_split[0] & ~0x33)); + b->tx_split0 = (uint8_t)tx_split[0]; + b->tx_split1 = tx_split[1]; } static inline unsigned get_prev_frame_segid(const Dav1dFrameContext *const f, const int by, const int bx, const int w4, int h4, @@ -727,10 +726,22 @@ rep_macro(type, t->dir mode, off, mul * y_mode_nofilt); \ rep_macro(type, t->dir intra, off, mul) case_set(bh4, l., 1, by4); case_set(bw4, a->, 0, bx4); #undef set_ctx + if (f->frame_hdr->frame_type & 1) { + refmvs_block *const r = &t->rt.r[(t->by & 31) + 5 + bh4 - 1][t->bx]; + for (int x = 0; x < bw4; x++) { + r[x].ref.ref[0] = 0; + r[x].bs = bs; + } + refmvs_block *const *rr = &t->rt.r[(t->by & 31) + 5]; + for (int y = 0; y < bh4 - 1; y++) { + rr[y][t->bx + bw4 - 1].ref.ref[0] = 0; + rr[y][t->bx + bw4 - 1].bs = bs; + } + } if (has_chroma) { #define set_ctx(type, dir, diridx, off, mul, rep_macro) \ rep_macro(type, t->dir uvmode, off, mul * b->uv_mode) case_set(cbh4, l., 1, cby4); @@ -739,14 +750,38 @@ } } else { if (f->frame_hdr->frame_type & 1 /* not intrabc */ && b->comp_type == COMP_INTER_NONE && b->motion_mode == MM_WARP) { - uint64_t mask[2] = { 0, 0 }; - find_matching_ref(t, intra_edge_flags, bw4, bh4, w4, h4, - have_left, have_top, b->ref[0], mask); - derive_warpmv(t, bw4, bh4, mask, b->mv[0], &t->warpmv); + if (b->matrix[0] == SHRT_MIN) { + t->warpmv.type = DAV1D_WM_TYPE_IDENTITY; + } else { + t->warpmv.type = DAV1D_WM_TYPE_AFFINE; + t->warpmv.matrix[2] = b->matrix[0] + 0x10000; + t->warpmv.matrix[3] = b->matrix[1]; + t->warpmv.matrix[4] = b->matrix[2]; + t->warpmv.matrix[5] = b->matrix[3] + 0x10000; + dav1d_set_affine_mv2d(bw4, bh4, b->mv2d, &t->warpmv, + t->bx, t->by); + dav1d_get_shear_params(&t->warpmv); +#define signabs(v) v < 0 ? '-' : ' ', abs(v) + if (DEBUG_BLOCK_INFO) + printf("[ %c%x %c%x %c%x\n %c%x %c%x %c%x ]\n" + "alpha=%c%x, beta=%c%x, gamma=%c%x, delta=%c%x, mv=y:%d,x:%d\n", + signabs(t->warpmv.matrix[0]), + signabs(t->warpmv.matrix[1]), + signabs(t->warpmv.matrix[2]), + signabs(t->warpmv.matrix[3]), + signabs(t->warpmv.matrix[4]), + signabs(t->warpmv.matrix[5]), + signabs(t->warpmv.alpha), + signabs(t->warpmv.beta), + signabs(t->warpmv.gamma), + signabs(t->warpmv.delta), + b->mv2d.y, b->mv2d.x); +#undef signabs + } } if (f->bd_fn.recon_b_inter(t, bs, b)) return -1; const uint8_t *const filter = dav1d_filter_dir[b->filter2d]; #define set_ctx(type, dir, diridx, off, mul, rep_macro) \ @@ -755,10 +790,25 @@ rep_macro(type, t->dir intra, off, 0) case_set(bh4, l., 1, by4); case_set(bw4, a->, 0, bx4); #undef set_ctx + if (f->frame_hdr->frame_type & 1) { + refmvs_block *const r = &t->rt.r[(t->by & 31) + 5 + bh4 - 1][t->bx]; + for (int x = 0; x < bw4; x++) { + r[x].ref.ref[0] = b->ref[0] + 1; + r[x].mv.mv[0] = b->mv[0]; + r[x].bs = bs; + } + refmvs_block *const *rr = &t->rt.r[(t->by & 31) + 5]; + for (int y = 0; y < bh4 - 1; y++) { + rr[y][t->bx + bw4 - 1].ref.ref[0] = b->ref[0] + 1; + rr[y][t->bx + bw4 - 1].mv.mv[0] = b->mv[0]; + rr[y][t->bx + bw4 - 1].bs = bs; + } + } + if (has_chroma) { #define set_ctx(type, dir, diridx, off, mul, rep_macro) \ rep_macro(type, t->dir uvmode, off, mul * DC_PRED) case_set(cbh4, l., 1, cby4); case_set(cbw4, a->, 0, cbx4); @@ -1243,39 +1293,35 @@ memcpy(t->al_pal[1][by4 + y][pl], pal[pl], 16); } } } if ((f->frame_hdr->frame_type & 1) || f->frame_hdr->allow_intrabc) { - splat_intraref(f->mvs, f->b4_stride, t->by, t->bx, bs, - y_mode_nofilt); + splat_intraref(&t->rt, t->by, t->bx, bs); } } else if (!(f->frame_hdr->frame_type & 1)) { // intra block copy - candidate_mv mvstack[8]; - int n_mvs; - mv mvlist[2][2]; - dav1d_find_ref_mvs(mvstack, &n_mvs, mvlist, NULL, - (int[2]) { -1, -1 }, f->bw, f->bh, - bs, bp, t->by, t->bx, ts->tiling.col_start, - ts->tiling.col_end, ts->tiling.row_start, - ts->tiling.row_end, f->libaom_cm); + refmvs_candidate mvstack[8]; + int n_mvs, ctx; + dav1d_refmvs_find(&t->rt, mvstack, &n_mvs, &ctx, + (union refmvs_refpair) { .ref = { 0, -1 }}, + bs, intra_edge_flags, t->by, t->bx); - if (mvlist[0][0].y | mvlist[0][0].x) - b->mv[0] = mvlist[0][0]; - else if (mvlist[0][1].y | mvlist[0][1].x) - b->mv[0] = mvlist[0][1]; + if (mvstack[0].mv.mv[0].n) + b->mv[0] = mvstack[0].mv.mv[0]; + else if (mvstack[1].mv.mv[0].n) + b->mv[0] = mvstack[1].mv.mv[0]; else { if (t->by - (16 << f->seq_hdr->sb128) < ts->tiling.row_start) { b->mv[0].y = 0; b->mv[0].x = -(512 << f->seq_hdr->sb128) - 2048; } else { b->mv[0].y = -(512 << f->seq_hdr->sb128); b->mv[0].x = 0; } } - const struct mv ref = b->mv[0]; + const union mv ref = b->mv[0]; read_mv_residual(t, &b->mv[0], &ts->cdf.dmv, 0); // clip intrabc motion vector to decoded parts of current tile int border_left = ts->tiling.col_start * 4; int border_top = ts->tiling.row_start * 4; @@ -1333,22 +1379,22 @@ b->mv[0].y = (src_top - t->by * 4) * 8; if (DEBUG_BLOCK_INFO) printf("Post-dmv[%d/%d,ref=%d/%d|%d/%d]: r=%d\n", b->mv[0].y, b->mv[0].x, ref.y, ref.x, - mvlist[0][0].y, mvlist[0][0].x, ts->msac.rng); + mvstack[0].mv.mv[0].y, mvstack[0].mv.mv[0].x, ts->msac.rng); read_vartx_tree(t, b, bs, bx4, by4); // reconstruction if (f->frame_thread.pass == 1) { f->bd_fn.read_coef_blocks(t, bs, b); b->filter2d = FILTER_2D_BILINEAR; } else { if (f->bd_fn.recon_b_inter(t, bs, b)) return -1; } - splat_intrabc_mv(f->mvs, f->b4_stride, t->by, t->bx, bs, b->mv[0]); + splat_intrabc_mv(&t->rt, t->by, t->bx, bs, b->mv[0]); #define set_ctx(type, dir, diridx, off, mul, rep_macro) \ rep_macro(type, t->dir tx_intra, off, mul * b_dim[2 + diridx]); \ rep_macro(type, t->dir mode, off, mul * DC_PRED); \ rep_macro(type, t->dir pal_sz, off, 0); \ @@ -1393,21 +1439,19 @@ b->comp_type = COMP_INTER_AVG; b->inter_mode = NEARESTMV_NEARESTMV; b->drl_idx = NEAREST_DRL; has_subpel_filter = 0; - candidate_mv mvstack[8]; + refmvs_candidate mvstack[8]; int n_mvs, ctx; - mv mvlist[2][2]; - dav1d_find_ref_mvs(mvstack, &n_mvs, mvlist, &ctx, - (int[2]) { b->ref[0], b->ref[1] }, f->bw, f->bh, - bs, bp, t->by, t->bx, ts->tiling.col_start, - ts->tiling.col_end, ts->tiling.row_start, - ts->tiling.row_end, f->libaom_cm); + dav1d_refmvs_find(&t->rt, mvstack, &n_mvs, &ctx, + (union refmvs_refpair) { .ref = { + b->ref[0] + 1, b->ref[1] + 1 }}, + bs, intra_edge_flags, t->by, t->bx); - b->mv[0] = mvstack[0].this_mv; - b->mv[1] = mvstack[0].comp_mv; + b->mv[0] = mvstack[0].mv.mv[0]; + b->mv[1] = mvstack[0].mv.mv[1]; fix_mv_precision(f->frame_hdr, &b->mv[0]); fix_mv_precision(f->frame_hdr, &b->mv[1]); if (DEBUG_BLOCK_INFO) printf("Post-skipmodeblock[mv=1:y=%d,x=%d,2:y=%d,x=%d,refs=%d+%d\n", b->mv[0].y, b->mv[0].x, b->mv[1].y, b->mv[1].x, @@ -1473,18 +1517,16 @@ } if (DEBUG_BLOCK_INFO) printf("Post-refs[%d/%d]: r=%d\n", b->ref[0], b->ref[1], ts->msac.rng); - candidate_mv mvstack[8]; + refmvs_candidate mvstack[8]; int n_mvs, ctx; - mv mvlist[2][2]; - dav1d_find_ref_mvs(mvstack, &n_mvs, mvlist, &ctx, - (int[2]) { b->ref[0], b->ref[1] }, f->bw, f->bh, - bs, bp, t->by, t->bx, ts->tiling.col_start, - ts->tiling.col_end, ts->tiling.row_start, - ts->tiling.row_end, f->libaom_cm); + dav1d_refmvs_find(&t->rt, mvstack, &n_mvs, &ctx, + (union refmvs_refpair) { .ref = { + b->ref[0] + 1, b->ref[1] + 1 }}, + bs, intra_edge_flags, t->by, t->bx); b->inter_mode = dav1d_msac_decode_symbol_adapt8(&ts->msac, ts->cdf.m.comp_inter_mode[ctx], N_COMP_INTER_PRED_MODES - 1); if (DEBUG_BLOCK_INFO) @@ -1523,34 +1565,33 @@ b->drl_idx, n_mvs, ts->msac.rng); } } assert(b->drl_idx >= NEAREST_DRL && b->drl_idx <= NEARISH_DRL); -#define assign_comp_mv(idx, pfx) \ +#define assign_comp_mv(idx) \ switch (im[idx]) { \ case NEARMV: \ case NEARESTMV: \ - b->mv[idx] = mvstack[b->drl_idx].pfx##_mv; \ + b->mv[idx] = mvstack[b->drl_idx].mv.mv[idx]; \ fix_mv_precision(f->frame_hdr, &b->mv[idx]); \ break; \ case GLOBALMV: \ has_subpel_filter |= \ f->frame_hdr->gmv[b->ref[idx]].type == DAV1D_WM_TYPE_TRANSLATION; \ b->mv[idx] = get_gmv_2d(&f->frame_hdr->gmv[b->ref[idx]], \ t->bx, t->by, bw4, bh4, f->frame_hdr); \ - fix_mv_precision(f->frame_hdr, &b->mv[idx]); \ break; \ case NEWMV: \ - b->mv[idx] = mvstack[b->drl_idx].pfx##_mv; \ + b->mv[idx] = mvstack[b->drl_idx].mv.mv[idx]; \ read_mv_residual(t, &b->mv[idx], &ts->cdf.mv, \ !f->frame_hdr->force_integer_mv); \ break; \ } has_subpel_filter = imin(bw4, bh4) == 1 || b->inter_mode != GLOBALMV_GLOBALMV; - assign_comp_mv(0, this); - assign_comp_mv(1, comp); + assign_comp_mv(0); + assign_comp_mv(1); #undef assign_comp_mv if (DEBUG_BLOCK_INFO) printf("Post-residual_mv[1:y=%d,x=%d,2:y=%d,x=%d]: r=%d\n", b->mv[0].y, b->mv[0].x, b->mv[1].y, b->mv[1].x, ts->msac.rng); @@ -1651,18 +1692,15 @@ if (DEBUG_BLOCK_INFO) printf("Post-ref[%d]: r=%d\n", b->ref[0], ts->msac.rng); } b->ref[1] = -1; - candidate_mv mvstack[8]; + refmvs_candidate mvstack[8]; int n_mvs, ctx; - mv mvlist[2][2]; - dav1d_find_ref_mvs(mvstack, &n_mvs, mvlist, &ctx, - (int[2]) { b->ref[0], -1 }, f->bw, f->bh, bs, bp, - t->by, t->bx, ts->tiling.col_start, - ts->tiling.col_end, ts->tiling.row_start, - ts->tiling.row_end, f->libaom_cm); + dav1d_refmvs_find(&t->rt, mvstack, &n_mvs, &ctx, + (union refmvs_refpair) { .ref = { b->ref[0] + 1, -1 }}, + bs, intra_edge_flags, t->by, t->bx); // mode parsing and mv derivation from ref_mvs if ((seg && (seg->skip || seg->globalmv)) || dav1d_msac_decode_bool_adapt(&ts->msac, ts->cdf.m.newmv_mode[ctx & 7])) @@ -1672,11 +1710,10 @@ ts->cdf.m.globalmv_mode[(ctx >> 3) & 1])) { b->inter_mode = GLOBALMV; b->mv[0] = get_gmv_2d(&f->frame_hdr->gmv[b->ref[0]], t->bx, t->by, bw4, bh4, f->frame_hdr); - fix_mv_precision(f->frame_hdr, &b->mv[0]); has_subpel_filter = imin(bw4, bh4) == 1 || f->frame_hdr->gmv[b->ref[0]].type == DAV1D_WM_TYPE_TRANSLATION; } else { has_subpel_filter = 1; if (dav1d_msac_decode_bool_adapt(&ts->msac, @@ -1698,16 +1735,13 @@ } else { b->inter_mode = NEARESTMV; b->drl_idx = NEAREST_DRL; } assert(b->drl_idx >= NEAREST_DRL && b->drl_idx <= NEARISH_DRL); - if (b->drl_idx >= NEAR_DRL) { - b->mv[0] = mvstack[b->drl_idx].this_mv; - } else { - b->mv[0] = mvlist[0][b->drl_idx]; + b->mv[0] = mvstack[b->drl_idx].mv.mv[0]; + if (b->drl_idx < NEAR_DRL) fix_mv_precision(f->frame_hdr, &b->mv[0]); - } } if (DEBUG_BLOCK_INFO) printf("Post-intermode[%d,drl=%d,mv=y:%d,x:%d,n_mvs=%d]: r=%d\n", b->inter_mode, b->drl_idx, b->mv[0].y, b->mv[0].x, n_mvs, @@ -1726,14 +1760,14 @@ ts->cdf.m.drl_bit[drl_ctx_v2]); } } assert(b->drl_idx >= NEAREST_DRL && b->drl_idx <= NEARISH_DRL); if (n_mvs > 1) { - b->mv[0] = mvstack[b->drl_idx].this_mv; + b->mv[0] = mvstack[b->drl_idx].mv.mv[0]; } else { assert(!b->drl_idx); - b->mv[0] = mvlist[0][0]; + b->mv[0] = mvstack[0].mv.mv[0]; fix_mv_precision(f->frame_hdr, &b->mv[0]); } if (DEBUG_BLOCK_INFO) printf("Post-intermode[%d,drl=%d]: r=%d\n", b->inter_mode, b->drl_idx, ts->msac.rng); @@ -1799,22 +1833,34 @@ has_subpel_filter = 0; derive_warpmv(t, bw4, bh4, mask, b->mv[0], &t->warpmv); #define signabs(v) v < 0 ? '-' : ' ', abs(v) if (DEBUG_BLOCK_INFO) printf("[ %c%x %c%x %c%x\n %c%x %c%x %c%x ]\n" - "alpha=%c%x, beta=%c%x, gamma=%c%x, delta=%c%x\n", + "alpha=%c%x, beta=%c%x, gamma=%c%x, delta=%c%x, " + "mv=y:%d,x:%d\n", signabs(t->warpmv.matrix[0]), signabs(t->warpmv.matrix[1]), signabs(t->warpmv.matrix[2]), signabs(t->warpmv.matrix[3]), signabs(t->warpmv.matrix[4]), signabs(t->warpmv.matrix[5]), signabs(t->warpmv.alpha), signabs(t->warpmv.beta), signabs(t->warpmv.gamma), - signabs(t->warpmv.delta)); + signabs(t->warpmv.delta), + b->mv[0].y, b->mv[0].x); #undef signabs + if (f->frame_thread.pass) { + if (t->warpmv.type == DAV1D_WM_TYPE_AFFINE) { + b->matrix[0] = t->warpmv.matrix[2] - 0x10000; + b->matrix[1] = t->warpmv.matrix[3]; + b->matrix[2] = t->warpmv.matrix[4]; + b->matrix[3] = t->warpmv.matrix[5] - 0x10000; + } else { + b->matrix[0] = SHRT_MIN; + } + } } if (DEBUG_BLOCK_INFO) printf("Post-motionmode[%d]: r=%d [mask: 0x%" PRIu64 "x/0x%" PRIu64 "x]\n", b->motion_mode, ts->msac.rng, mask[0], @@ -1874,29 +1920,29 @@ { const int is_globalmv = b->inter_mode == (is_comp ? GLOBALMV_GLOBALMV : GLOBALMV); const uint8_t (*const lf_lvls)[8][2] = (const uint8_t (*)[8][2]) &ts->lflvl[b->seg_id][0][b->ref[0] + 1][!is_globalmv]; + const uint16_t tx_split[2] = { b->tx_split0, b->tx_split1 }; dav1d_create_lf_mask_inter(t->lf_mask, f->lf.level, f->b4_stride, lf_lvls, t->bx, t->by, f->w4, f->h4, b->skip, bs, f->frame_hdr->segmentation.lossless[b->seg_id] ? (enum RectTxfmSize) TX_4X4 : b->max_ytx, - b->tx_split, b->uvtx, f->cur.p.layout, + tx_split, b->uvtx, f->cur.p.layout, &t->a->tx_lpf_y[bx4], &t->l.tx_lpf_y[by4], has_chroma ? &t->a->tx_lpf_uv[cbx4] : NULL, has_chroma ? &t->l.tx_lpf_uv[cby4] : NULL); } // context updates if (is_comp) { - splat_tworef_mv(f->mvs, f->b4_stride, t->by, t->bx, bs, - b->inter_mode, b->ref[0], b->ref[1], - b->mv[0], b->mv[1]); + splat_tworef_mv(&t->rt, t->by, t->bx, bs, b->inter_mode, + (refmvs_refpair) { .ref = { b->ref[0], b->ref[1] }}, + (refmvs_mvpair) { .mv = { [0] = b->mv[0], [1] = b->mv[1] }}); } else { - splat_oneref_mv(f->mvs, f->b4_stride, t->by, t->bx, bs, - b->inter_mode, b->ref[0], b->mv[0], - b->interintra_type); + splat_oneref_mv(&t->rt, t->by, t->bx, bs, b->inter_mode, + b->ref[0], b->mv[0], b->interintra_type); } #define set_ctx(type, dir, diridx, off, mul, rep_macro) \ rep_macro(type, t->dir seg_pred, off, mul * seg_pred); \ rep_macro(type, t->dir skip_mode, off, mul * b->skip_mode); \ @@ -1950,11 +1996,10 @@ } return 0; } -#if defined(__has_feature) #if __has_feature(memory_sanitizer) #include static int checked_decode_b(Dav1dTileContext *const t, @@ -2003,11 +2048,10 @@ } #define decode_b checked_decode_b #endif /* defined(__has_feature) */ -#endif /* __has_feature(memory_sanitizer) */ static int decode_sb(Dav1dTileContext *const t, const enum BlockLevel bl, const EdgeNode *const node) { const Dav1dFrameContext *const f = t->f; @@ -2468,10 +2512,17 @@ const int sb_step = f->sb_step; const int tile_row = ts->tiling.row, tile_col = ts->tiling.col; const int col_sb_start = f->frame_hdr->tiling.col_start_sb[tile_col]; const int col_sb128_start = col_sb_start >> !f->seq_hdr->sb128; + if ((f->frame_hdr->frame_type & 1) || f->frame_hdr->allow_intrabc) { + dav1d_refmvs_tile_sbrow_init(&t->rt, &f->rf, ts->tiling.col_start, + ts->tiling.col_end, ts->tiling.row_start, + ts->tiling.row_end, t->by >> f->sb_shift, + ts->tiling.row); + } + reset_context(&t->l, !(f->frame_hdr->frame_type & 1), f->frame_thread.pass); if (f->frame_thread.pass == 2) { for (t->bx = ts->tiling.col_start, t->a = f->a + col_sb128_start + tile_row * f->sb128w; t->bx < ts->tiling.col_end; t->bx += sb_step) @@ -2488,20 +2539,20 @@ } // error out on symbol decoder overread if (ts->msac.cnt < -15) return 1; - if (c->n_fc > 1 && f->frame_hdr->use_ref_frame_mvs) { - for (int n = 0; n < 7; n++) + if (f->n_tc > 1 && f->frame_hdr->use_ref_frame_mvs) { + if (c->n_fc > 1) for (int n = 0; n < 7; n++) if (dav1d_thread_picture_wait(&f->refp[n], 4 * (t->by + sb_step), PLANE_TYPE_BLOCK)) { return 1; } - dav1d_init_ref_mv_tile_row(f->libaom_cm, - ts->tiling.col_start, ts->tiling.col_end, - t->by, imin(t->by + sb_step, f->bh)); + dav1d_refmvs_load_tmvs(&f->rf, ts->tiling.row, + ts->tiling.col_start >> 1, ts->tiling.col_end >> 1, + t->by >> 1, (t->by + sb_step) >> 1); } memset(t->pal_sz_uv[1], 0, sizeof(*t->pal_sz_uv)); const int sb128y = t->by >> 5; for (t->bx = ts->tiling.col_start, t->a = f->a + col_sb128_start + tile_row * f->sb128w, t->lf_mask = f->lf.mask + sb128y * f->sb128w + col_sb128_start; @@ -2579,10 +2630,16 @@ t->a++; t->lf_mask++; } } + if (f->n_tc > 1 && f->frame_hdr->frame_type & 1) { + dav1d_refmvs_save_tmvs(&t->rt, + ts->tiling.col_start >> 1, ts->tiling.col_end >> 1, + t->by >> 1, (t->by + sb_step) >> 1); + } + // backup pre-loopfilter pixels for intra prediction of the next sbrow if (f->frame_thread.pass != 1) f->bd_fn.backup_ipred_edge(t); // backup t->a/l.tx_lpf_y/uv at tile boundaries to use them to "fix" @@ -2892,24 +2949,14 @@ f->lf.re_sz = re_sz; } // init ref mvs if ((f->frame_hdr->frame_type & 1) || f->frame_hdr->allow_intrabc) { - f->mvs = f->mvs_ref->data; - const int order_hint_n_bits = f->seq_hdr->order_hint * f->seq_hdr->order_hint_n_bits; - const int ret = dav1d_init_ref_mv_common(f->libaom_cm, f->bw >> 1, f->bh >> 1, - f->b4_stride, f->seq_hdr->sb128, - f->mvs, f->ref_mvs, - f->cur.frame_hdr->frame_offset, - f->refpoc, - f->refrefpoc, f->frame_hdr->gmv, - f->frame_hdr->hp, f->frame_hdr->force_integer_mv, - f->frame_hdr->use_ref_frame_mvs, - order_hint_n_bits); + const int ret = + dav1d_refmvs_init_frame(&f->rf, f->seq_hdr, f->frame_hdr, + f->refpoc, f->mvs, f->refrefpoc, f->ref_mvs, f->n_tc); if (ret < 0) goto error; - if (c->n_fc == 1 && f->frame_hdr->use_ref_frame_mvs) - dav1d_init_ref_mv_tile_row(f->libaom_cm, 0, f->bw, 0, f->bh); } retval = DAV1D_ERR(EINVAL); // setup dequant tables init_quant_tables(f->seq_hdr, f->frame_hdr, f->frame_hdr->quant.yac, f->dq); @@ -3046,15 +3093,30 @@ imin(f->frame_hdr->tiling.row_start_sb[tile_row + 1], f->sbh); for (int sby = f->frame_hdr->tiling.row_start_sb[tile_row]; sby < sbh_end; sby++) { t->by = sby << (4 + f->seq_hdr->sb128); + const int by_end = (t->by + f->sb_step) >> 1; + if (f->frame_thread.pass <= 1 && f->frame_hdr->use_ref_frame_mvs) { + if (c->n_fc > 1) for (int n = 0; n < 7; n++) + if (dav1d_thread_picture_wait(&f->refp[n], + 4 * (t->by + f->sb_step), + PLANE_TYPE_BLOCK)) + { + return 1; + } + dav1d_refmvs_load_tmvs(&f->rf, tile_row, + 0, f->bw >> 1, t->by >> 1, by_end); + } for (int tile_col = 0; tile_col < f->frame_hdr->tiling.cols; tile_col++) { t->ts = &f->ts[tile_row * f->frame_hdr->tiling.cols + tile_col]; if (dav1d_decode_tile_sbrow(t)) goto error; } + if (f->frame_thread.pass <= 1 && f->frame_hdr->frame_type & 1) { + dav1d_refmvs_save_tmvs(&t->rt, 0, f->bw >> 1, t->by >> 1, by_end); + } // loopfilter + cdef + restoration if (f->frame_thread.pass != 1) f->bd_fn.filter_sbrow(f, sby); dav1d_thread_picture_signal(&f->sr_cur, (sby + 1) * f->sb_step * 4, @@ -3238,11 +3300,11 @@ switch (bpc) { #define assign_bitdepth_case(bd) \ dav1d_cdef_dsp_init_##bd##bpc(&dsp->cdef); \ dav1d_intra_pred_dsp_init_##bd##bpc(&dsp->ipred); \ - dav1d_itx_dsp_init_##bd##bpc(&dsp->itx); \ + dav1d_itx_dsp_init_##bd##bpc(&dsp->itx, bpc); \ dav1d_loop_filter_dsp_init_##bd##bpc(&dsp->lf); \ dav1d_loop_restoration_dsp_init_##bd##bpc(&dsp->lr, bpc); \ dav1d_mc_dsp_init_##bd##bpc(&dsp->mc); \ dav1d_film_grain_dsp_init_##bd##bpc(&dsp->fg); \ break @@ -3399,11 +3461,11 @@ f->b4_stride = (f->bw + 31) & ~31; f->bitdepth_max = (1 << f->cur.p.bpc) - 1; // ref_mvs if ((f->frame_hdr->frame_type & 1) || f->frame_hdr->allow_intrabc) { - f->mvs_ref = dav1d_ref_create(f->sb128h * 32 * f->b4_stride * + f->mvs_ref = dav1d_ref_create(f->sb128h * 16 * (f->b4_stride >> 1) * sizeof(*f->mvs)); if (!f->mvs_ref) { res = DAV1D_ERR(ENOMEM); goto error; } diff --git a/third_party/dav1d/src/env.h b/third_party/dav1d/src/env.h --- a/third_party/dav1d/src/env.h +++ b/third_party/dav1d/src/env.h @@ -31,11 +31,11 @@ #include #include #include #include "src/levels.h" -#include "src/ref_mvs.h" +#include "src/refmvs.h" #include "src/tables.h" typedef struct BlockContext { uint8_t ALIGN(mode[32], 8); uint8_t ALIGN(lcoef[32], 8); @@ -426,11 +426,11 @@ cnt[1] += cnt[2]; return cnt[0] == cnt[1] ? 1 : cnt[0] < cnt[1] ? 0 : 2; } -static inline int get_drl_context(const candidate_mv *const ref_mv_stack, +static inline int get_drl_context(const refmvs_candidate *const ref_mv_stack, const int ref_idx) { if (ref_mv_stack[ref_idx].weight >= 640) return ref_mv_stack[ref_idx + 1].weight < 640; @@ -458,10 +458,26 @@ *seg_ctx = 0; return have_left ? cur_seg_map[-1] : have_top ? cur_seg_map[-stride] : 0; } } +static inline void fix_int_mv_precision(mv *const mv) { + mv->x = (mv->x - (mv->x >> 15) + 3) & ~7U; + mv->y = (mv->y - (mv->y >> 15) + 3) & ~7U; +} + +static inline void fix_mv_precision(const Dav1dFrameHeader *const hdr, + mv *const mv) +{ + if (hdr->force_integer_mv) { + fix_int_mv_precision(mv); + } else if (!hdr->hp) { + mv->x = (mv->x - (mv->x >> 15)) & ~1U; + mv->y = (mv->y - (mv->y >> 15)) & ~1U; + } +} + static inline mv get_gmv_2d(const Dav1dWarpedMotionParams *const gmv, const int bx4, const int by4, const int bw4, const int bh4, const Dav1dFrameHeader *const hdr) { @@ -478,20 +494,27 @@ gmv->matrix[3] * y + gmv->matrix[0]; const int yc = (gmv->matrix[5] - (1 << 16)) * y + gmv->matrix[4] * x + gmv->matrix[1]; const int shift = 16 - (3 - !hdr->hp); const int round = (1 << shift) >> 1; - return (mv) { + mv res = (mv) { .y = apply_sign(((abs(yc) + round) >> shift) << !hdr->hp, yc), .x = apply_sign(((abs(xc) + round) >> shift) << !hdr->hp, xc), }; + if (hdr->force_integer_mv) + fix_int_mv_precision(&res); + return res; } - case DAV1D_WM_TYPE_TRANSLATION: - return (mv) { + case DAV1D_WM_TYPE_TRANSLATION: { + mv res = (mv) { .y = gmv->matrix[0] >> 13, .x = gmv->matrix[1] >> 13, }; + if (hdr->force_integer_mv) + fix_int_mv_precision(&res); + return res; + } case DAV1D_WM_TYPE_IDENTITY: return (mv) { .x = 0, .y = 0 }; } } diff --git a/third_party/dav1d/src/ext/x86/x86inc.asm b/third_party/dav1d/src/ext/x86/x86inc.asm --- a/third_party/dav1d/src/ext/x86/x86inc.asm +++ b/third_party/dav1d/src/ext/x86/x86inc.asm @@ -1,9 +1,9 @@ ;***************************************************************************** ;* x86inc.asm: x264asm abstraction layer ;***************************************************************************** -;* Copyright (C) 2005-2019 x264 project +;* Copyright (C) 2005-2020 x264 project ;* ;* Authors: Loren Merritt ;* Henrik Gramner ;* Anton Mitrofanov ;* Fiona Glaser @@ -356,11 +356,11 @@ %define required_stack_alignment ((mmsize + 15) & ~15) %define vzeroupper_required (mmsize > 16 && (ARCH_X86_64 == 0 || xmm_regs_used > 16 || notcpuflag(avx512))) %define high_mm_regs (16*cpuflag(avx512)) -%macro ALLOC_STACK 1-2 0 ; stack_size, n_xmm_regs (for win64 only) +%macro ALLOC_STACK 0-2 0, 0 ; stack_size, n_xmm_regs (for win64 only) %ifnum %1 %if %1 != 0 %assign %%pad 0 %assign stack_size %1 %if stack_size < 0 @@ -401,11 +401,11 @@ WIN64_PUSH_XMM %endif %endif %endmacro -%macro SETUP_STACK_POINTER 1 +%macro SETUP_STACK_POINTER 0-1 0 %ifnum %1 %if %1 != 0 && required_stack_alignment > STACK_ALIGNMENT %if %1 > 0 ; Reserve an additional register for storing the original stack pointer, but avoid using ; eax/rax for this purpose since it can potentially get overwritten as a return value. @@ -423,20 +423,10 @@ %endif %endif %endif %endmacro -%macro DEFINE_ARGS_INTERNAL 3+ - %ifnum %2 - DEFINE_ARGS %3 - %elif %1 == 4 - DEFINE_ARGS %2 - %elif %1 > 4 - DEFINE_ARGS %2, %3 - %endif -%endmacro - %if WIN64 ; Windows x64 ;================================================= DECLARE_REG 0, rcx DECLARE_REG 1, rdx DECLARE_REG 2, R8 @@ -451,11 +441,11 @@ DECLARE_REG 11, R14, 96 DECLARE_REG 12, R15, 104 DECLARE_REG 13, R12, 112 DECLARE_REG 14, R13, 120 -%macro PROLOGUE 2-5+ 0 ; #args, #regs, #xmm_regs, [stack_size,] arg_names... +%macro PROLOGUE 2-5+ 0, 0 ; #args, #regs, #xmm_regs, [stack_size,] arg_names... %assign num_args %1 %assign regs_used %2 ASSERT regs_used >= num_args SETUP_STACK_POINTER %4 ASSERT regs_used <= 15 @@ -463,11 +453,19 @@ ALLOC_STACK %4, %3 %if mmsize != 8 && stack_size == 0 WIN64_SPILL_XMM %3 %endif LOAD_IF_USED 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 - DEFINE_ARGS_INTERNAL %0, %4, %5 + %if %0 > 4 + %ifnum %4 + DEFINE_ARGS %5 + %else + DEFINE_ARGS %4, %5 + %endif + %elifnnum %4 + DEFINE_ARGS %4 + %endif %endmacro %macro WIN64_PUSH_XMM 0 ; Use the shadow space to store XMM6 and XMM7, the rest needs stack space allocated. %if xmm_regs_used > 6 + high_mm_regs @@ -559,21 +557,29 @@ DECLARE_REG 11, R14, 48 DECLARE_REG 12, R15, 56 DECLARE_REG 13, R12, 64 DECLARE_REG 14, R13, 72 -%macro PROLOGUE 2-5+ 0 ; #args, #regs, #xmm_regs, [stack_size,] arg_names... +%macro PROLOGUE 2-5+ 0, 0 ; #args, #regs, #xmm_regs, [stack_size,] arg_names... %assign num_args %1 %assign regs_used %2 %assign xmm_regs_used %3 ASSERT regs_used >= num_args SETUP_STACK_POINTER %4 ASSERT regs_used <= 15 PUSH_IF_USED 9, 10, 11, 12, 13, 14 ALLOC_STACK %4 LOAD_IF_USED 6, 7, 8, 9, 10, 11, 12, 13, 14 - DEFINE_ARGS_INTERNAL %0, %4, %5 + %if %0 > 4 + %ifnum %4 + DEFINE_ARGS %5 + %else + DEFINE_ARGS %4, %5 + %endif + %elifnnum %4 + DEFINE_ARGS %4 + %endif %endmacro %define has_epilogue regs_used > 9 || stack_size > 0 || vzeroupper_required %macro RET 0 @@ -610,11 +616,11 @@ %endrep %endmacro DECLARE_ARG 7, 8, 9, 10, 11, 12, 13, 14 -%macro PROLOGUE 2-5+ ; #args, #regs, #xmm_regs, [stack_size,] arg_names... +%macro PROLOGUE 2-5+ 0, 0 ; #args, #regs, #xmm_regs, [stack_size,] arg_names... %assign num_args %1 %assign regs_used %2 ASSERT regs_used >= num_args %if num_args > 7 %assign num_args 7 @@ -625,11 +631,19 @@ SETUP_STACK_POINTER %4 ASSERT regs_used <= 7 PUSH_IF_USED 3, 4, 5, 6 ALLOC_STACK %4 LOAD_IF_USED 0, 1, 2, 3, 4, 5, 6 - DEFINE_ARGS_INTERNAL %0, %4, %5 + %if %0 > 4 + %ifnum %4 + DEFINE_ARGS %5 + %else + DEFINE_ARGS %4, %5 + %endif + %elifnnum %4 + DEFINE_ARGS %4 + %endif %endmacro %define has_epilogue regs_used > 3 || stack_size > 0 || vzeroupper_required %macro RET 0 @@ -989,10 +1003,12 @@ INIT_CPUFLAGS %1 DEFINE_MMREGS xmm %if WIN64 AVX512_MM_PERMUTATION 6 ; Swap callee-saved registers with volatile registers %endif + %xdefine bcstd 1to4 + %xdefine bcstq 1to2 %endmacro %macro INIT_YMM 0-1+ %assign avx_enabled 1 %define RESET_MM_PERMUTATION INIT_YMM %1 @@ -1002,10 +1018,12 @@ %undef movh %define movnta movntdq INIT_CPUFLAGS %1 DEFINE_MMREGS ymm AVX512_MM_PERMUTATION + %xdefine bcstd 1to8 + %xdefine bcstq 1to4 %endmacro %macro INIT_ZMM 0-1+ %assign avx_enabled 1 %define RESET_MM_PERMUTATION INIT_ZMM %1 @@ -1015,10 +1033,12 @@ %undef movh %define movnta movntdq INIT_CPUFLAGS %1 DEFINE_MMREGS zmm AVX512_MM_PERMUTATION + %xdefine bcstd 1to16 + %xdefine bcstq 1to8 %endmacro INIT_XMM %macro DECLARE_MMCAST 1 @@ -1248,10 +1268,16 @@ %error use of ``%1'' %2 instruction in cpuname function: current_function %elif %3 == 0 && __sizeofreg == 16 && notcpuflag(sse2) %error use of ``%1'' sse2 instruction in cpuname function: current_function %elif %3 == 0 && __sizeofreg == 32 && notcpuflag(avx2) %error use of ``%1'' avx2 instruction in cpuname function: current_function + %elif __sizeofreg == 16 && notcpuflag(sse) + %error use of ``%1'' sse instruction in cpuname function: current_function + %elif __sizeofreg == 32 && notcpuflag(avx) + %error use of ``%1'' avx instruction in cpuname function: current_function + %elif __sizeofreg == 64 && notcpuflag(avx512) + %error use of ``%1'' avx512 instruction in cpuname function: current_function %elifidn %1, pextrw ; special case because the base instruction is mmx2, %ifnid %6 ; but sse4 is required for memory operands %if notcpuflag(sse4) %error use of ``%1'' sse4 instruction in cpuname function: current_function %endif @@ -1670,10 +1696,11 @@ GPR_INSTR andn, bmi1 GPR_INSTR bextr, bmi1 GPR_INSTR blsi, bmi1 GPR_INSTR blsmsk, bmi1 +GPR_INSTR blsr, bmi1 GPR_INSTR bzhi, bmi2 GPR_INSTR mulx, bmi2 GPR_INSTR pdep, bmi2 GPR_INSTR pext, bmi2 GPR_INSTR popcnt, sse42 diff --git a/third_party/dav1d/src/getbits.c b/third_party/dav1d/src/getbits.c --- a/third_party/dav1d/src/getbits.c +++ b/third_party/dav1d/src/getbits.c @@ -25,17 +25,21 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "config.h" +#include + #include "common/intops.h" #include "src/getbits.h" void dav1d_init_get_bits(GetBits *const c, const uint8_t *const data, const size_t sz) { + // If sz were 0, c->eof would need to be initialized to 1. + assert(sz); c->ptr = c->ptr_start = data; c->ptr_end = &c->ptr_start[sz]; c->bits_left = 0; c->state = 0; c->error = 0; @@ -75,29 +79,27 @@ const int shift = 31 - n; const int res = dav1d_get_bits(c, n + 1) << shift; return res >> shift; } -unsigned dav1d_get_uleb128(GetBits *c) { - unsigned val = 0, more, i = 0; +unsigned dav1d_get_uleb128(GetBits *const c) { + uint64_t val = 0; + unsigned i = 0, more; do { - more = dav1d_get_bits(c, 1); - unsigned bits = dav1d_get_bits(c, 7); - if (i <= 3 || (i == 4 && bits < (1 << 4))) - val |= bits << (i * 7); - else if (bits) { - c->error = 1; - return 0; - } - if (more && ++i == 8) { - c->error = 1; - return 0; - } - } while (more); + const int v = dav1d_get_bits(c, 8); + more = v & 0x80; + val |= ((uint64_t) (v & 0x7F)) << i; + i += 7; + } while (more && i < 56); - return val; + if (val > UINT_MAX || more) { + c->error = 1; + return 0; + } + + return (unsigned) val; } unsigned dav1d_get_uniform(GetBits *const c, const unsigned max) { // Output in range [0..max-1] // max must be > 1, or else nothing is read from the bitstream diff --git a/third_party/dav1d/src/internal.h b/third_party/dav1d/src/internal.h --- a/third_party/dav1d/src/internal.h +++ b/third_party/dav1d/src/internal.h @@ -52,11 +52,11 @@ #include "src/looprestoration.h" #include "src/mc.h" #include "src/msac.h" #include "src/picture.h" #include "src/recon.h" -#include "src/ref_mvs.h" +#include "src/refmvs.h" #include "src/thread.h" typedef struct Dav1dDSPContext { Dav1dFilmGrainDSPContext fg; Dav1dIntraPredDSPContext ipred; @@ -144,11 +144,11 @@ Dav1dFrameHeader *frame_hdr; Dav1dThreadPicture refp[7]; Dav1dPicture cur; // during block coding / reconstruction Dav1dThreadPicture sr_cur; // after super-resolution upscaling Dav1dRef *mvs_ref; - refmvs *mvs, *ref_mvs[7]; + refmvs_temporal_block *mvs, *ref_mvs[7]; Dav1dRef *ref_mvs_ref[7]; Dav1dRef *cur_segmap_ref, *prev_segmap_ref; uint8_t *cur_segmap; const uint8_t *prev_segmap; unsigned refpoc[7], refrefpoc[7][7]; @@ -185,11 +185,11 @@ int w4, h4, bw, bh, sb128w, sb128h, sbh, sb_shift, sb_step, sr_sb128w; uint16_t dq[DAV1D_MAX_SEGMENTS][3 /* plane */][2 /* dc/ac */]; const uint8_t *qm[2 /* is_1d */][N_RECT_TX_SIZES][3 /* plane */]; BlockContext *a; int a_sz /* w*tile_rows */; - AV1_COMMON *libaom_cm; // FIXME + refmvs_frame rf; uint8_t jnt_weights[7][7]; int bitdepth_max; struct { struct thread_data td; @@ -288,10 +288,11 @@ // FIXME types can be changed to pixel (and dynamically allocated) // which would make copy/assign operations slightly faster? uint16_t al_pal[2 /* a/l */][32 /* bx/y4 */][3 /* plane */][8 /* palette_idx */]; uint8_t pal_sz_uv[2 /* a/l */][32 /* bx4/by4 */]; uint8_t txtp_map[32 * 32]; // inter-only + refmvs_tile rt; ALIGN(union, 64) { struct { union { uint8_t lap_8bpc [128 * 32]; uint16_t lap_16bpc[128 * 32]; diff --git a/third_party/dav1d/src/ipred_tmpl.c b/third_party/dav1d/src/ipred_tmpl.c --- a/third_party/dav1d/src/ipred_tmpl.c +++ b/third_party/dav1d/src/ipred_tmpl.c @@ -131,11 +131,11 @@ const pixel *const topleft, const int width, const int height, const int16_t *ac, const int alpha HIGHBD_DECL_SUFFIX) { - unsigned dc = dc_gen_left(topleft, height); + const unsigned dc = dc_gen_left(topleft, height); cfl_pred(dst, stride, width, height, dc, ac, alpha HIGHBD_TAIL_SUFFIX); } #if BITDEPTH == 8 #define MULTIPLIER_1x2 0x5556 @@ -623,29 +623,25 @@ { filt_idx &= 511; assert(filt_idx < 5); const int8_t *const filter = dav1d_filter_intra_taps[filt_idx]; - int x, y; - ptrdiff_t left_stride; - const pixel *left, *topleft, *top; - - top = &topleft_in[1]; - for (y = 0; y < height; y += 2) { - topleft = &topleft_in[-y]; - left = &topleft[-1]; - left_stride = -1; - for (x = 0; x < width; x += 4) { + const pixel *top = &topleft_in[1]; + for (int y = 0; y < height; y += 2) { + const pixel *topleft = &topleft_in[-y]; + const pixel *left = &topleft[-1]; + ptrdiff_t left_stride = -1; + for (int x = 0; x < width; x += 4) { const int p0 = *topleft; const int p1 = top[0], p2 = top[1], p3 = top[2], p4 = top[3]; const int p5 = left[0 * left_stride], p6 = left[1 * left_stride]; pixel *ptr = &dst[x]; const int8_t *flt_ptr = filter; for (int yy = 0; yy < 2; yy++) { for (int xx = 0; xx < 4; xx++, flt_ptr += FLT_INCR) { - int acc = FILTER(flt_ptr, p0, p1, p2, p3, p4, p5, p6); + const int acc = FILTER(flt_ptr, p0, p1, p2, p3, p4, p5, p6); ptr[xx] = iclip_pixel((acc + 8) >> 4); } ptr += PXSTRIDE(stride); } left = &dst[x + 4 - 1]; diff --git a/third_party/dav1d/src/itx.h b/third_party/dav1d/src/itx.h --- a/third_party/dav1d/src/itx.h +++ b/third_party/dav1d/src/itx.h @@ -41,10 +41,10 @@ typedef struct Dav1dInvTxfmDSPContext { itxfm_fn itxfm_add[N_RECT_TX_SIZES][N_TX_TYPES_PLUS_LL]; } Dav1dInvTxfmDSPContext; -bitfn_decls(void dav1d_itx_dsp_init, Dav1dInvTxfmDSPContext *c); -bitfn_decls(void dav1d_itx_dsp_init_arm, Dav1dInvTxfmDSPContext *c); +bitfn_decls(void dav1d_itx_dsp_init, Dav1dInvTxfmDSPContext *c, int bpc); +bitfn_decls(void dav1d_itx_dsp_init_arm, Dav1dInvTxfmDSPContext *c, int bpc); bitfn_decls(void dav1d_itx_dsp_init_x86, Dav1dInvTxfmDSPContext *c); #endif /* DAV1D_SRC_ITX_H */ diff --git a/third_party/dav1d/src/itx_1d.c b/third_party/dav1d/src/itx_1d.c --- a/third_party/dav1d/src/itx_1d.c +++ b/third_party/dav1d/src/itx_1d.c @@ -117,17 +117,17 @@ t5a = (in5 * 1703 - in3 * 1138 + 1024) >> 11; t6a = (in5 * 1138 + in3 * 1703 + 1024) >> 11; t7a = ((in1 * (4017 - 4096) + in7 * 799 + 2048) >> 12) + in1; } - int t4 = CLIP(t4a + t5a); - t5a = CLIP(t4a - t5a); - int t7 = CLIP(t7a + t6a); - t6a = CLIP(t7a - t6a); + const int t4 = CLIP(t4a + t5a); + t5a = CLIP(t4a - t5a); + const int t7 = CLIP(t7a + t6a); + t6a = CLIP(t7a - t6a); - int t5 = ((t6a - t5a) * 181 + 128) >> 8; - int t6 = ((t6a + t5a) * 181 + 128) >> 8; + const int t5 = ((t6a - t5a) * 181 + 128) >> 8; + const int t6 = ((t6a + t5a) * 181 + 128) >> 8; const int t0 = c[0 * stride]; const int t1 = c[2 * stride]; const int t2 = c[4 * stride]; const int t3 = c[6 * stride]; @@ -810,27 +810,27 @@ const int in0 = in[0 * in_s], in1 = in[1 * in_s]; const int in2 = in[2 * in_s], in3 = in[3 * in_s]; const int in4 = in[4 * in_s], in5 = in[5 * in_s]; const int in6 = in[6 * in_s], in7 = in[7 * in_s]; - int t0a = (((4076 - 4096) * in7 + 401 * in0 + 2048) >> 12) + in7; - int t1a = (( 401 * in7 - (4076 - 4096) * in0 + 2048) >> 12) - in0; - int t2a = (((3612 - 4096) * in5 + 1931 * in2 + 2048) >> 12) + in5; - int t3a = (( 1931 * in5 - (3612 - 4096) * in2 + 2048) >> 12) - in2; - int t4a = ( 1299 * in3 + 1583 * in4 + 1024) >> 11; - int t5a = ( 1583 * in3 - 1299 * in4 + 1024) >> 11; - int t6a = (( 1189 * in1 + (3920 - 4096) * in6 + 2048) >> 12) + in6; - int t7a = (((3920 - 4096) * in1 - 1189 * in6 + 2048) >> 12) + in1; + const int t0a = (((4076 - 4096) * in7 + 401 * in0 + 2048) >> 12) + in7; + const int t1a = (( 401 * in7 - (4076 - 4096) * in0 + 2048) >> 12) - in0; + const int t2a = (((3612 - 4096) * in5 + 1931 * in2 + 2048) >> 12) + in5; + const int t3a = (( 1931 * in5 - (3612 - 4096) * in2 + 2048) >> 12) - in2; + int t4a = ( 1299 * in3 + 1583 * in4 + 1024) >> 11; + int t5a = ( 1583 * in3 - 1299 * in4 + 1024) >> 11; + int t6a = (( 1189 * in1 + (3920 - 4096) * in6 + 2048) >> 12) + in6; + int t7a = (((3920 - 4096) * in1 - 1189 * in6 + 2048) >> 12) + in1; - int t0 = CLIP(t0a + t4a); - int t1 = CLIP(t1a + t5a); - int t2 = CLIP(t2a + t6a); - int t3 = CLIP(t3a + t7a); - int t4 = CLIP(t0a - t4a); - int t5 = CLIP(t1a - t5a); - int t6 = CLIP(t2a - t6a); - int t7 = CLIP(t3a - t7a); + const int t0 = CLIP(t0a + t4a); + const int t1 = CLIP(t1a + t5a); + int t2 = CLIP(t2a + t6a); + int t3 = CLIP(t3a + t7a); + const int t4 = CLIP(t0a - t4a); + const int t5 = CLIP(t1a - t5a); + int t6 = CLIP(t2a - t6a); + int t7 = CLIP(t3a - t7a); t4a = (((3784 - 4096) * t4 + 1567 * t5 + 2048) >> 12) + t4; t5a = (( 1567 * t4 - (3784 - 4096) * t5 + 2048) >> 12) - t5; t6a = (((3784 - 4096) * t7 - 1567 * t6 + 2048) >> 12) + t7; t7a = (( 1567 * t7 + (3784 - 4096) * t6 + 2048) >> 12) + t6; diff --git a/third_party/dav1d/src/itx_tmpl.c b/third_party/dav1d/src/itx_tmpl.c --- a/third_party/dav1d/src/itx_tmpl.c +++ b/third_party/dav1d/src/itx_tmpl.c @@ -178,11 +178,11 @@ for (int y = 0; y < 4; y++, dst += PXSTRIDE(stride)) for (int x = 0; x < 4; x++) dst[x] = iclip_pixel(dst[x] + *c++); } -COLD void bitfn(dav1d_itx_dsp_init)(Dav1dInvTxfmDSPContext *const c) { +COLD void bitfn(dav1d_itx_dsp_init)(Dav1dInvTxfmDSPContext *const c, int bpc) { #define assign_itx_all_fn64(w, h, pfx) \ c->itxfm_add[pfx##TX_##w##X##h][DCT_DCT ] = \ inv_txfm_add_dct_dct_##w##x##h##_c #define assign_itx_all_fn32(w, h, pfx) \ @@ -222,12 +222,10 @@ c->itxfm_add[pfx##TX_##w##X##h][H_ADST] = \ inv_txfm_add_adst_identity_##w##x##h##_c; \ c->itxfm_add[pfx##TX_##w##X##h][V_ADST] = \ inv_txfm_add_identity_adst_##w##x##h##_c; \ - memset(c, 0, sizeof(*c)); /* Zero unused function pointer elements. */ - c->itxfm_add[TX_4X4][WHT_WHT] = inv_txfm_add_wht_wht_4x4_c; assign_itx_all_fn84( 4, 4, ); assign_itx_all_fn84( 4, 8, R); assign_itx_all_fn84( 4, 16, R); assign_itx_all_fn84( 8, 4, R); @@ -247,11 +245,11 @@ assign_itx_all_fn64(64, 32, R); assign_itx_all_fn64(64, 64, ); #if HAVE_ASM #if ARCH_AARCH64 || ARCH_ARM - bitfn(dav1d_itx_dsp_init_arm)(c); + bitfn(dav1d_itx_dsp_init_arm)(c, bpc); #endif #if ARCH_X86 bitfn(dav1d_itx_dsp_init_x86)(c); #endif #endif diff --git a/third_party/dav1d/src/levels.h b/third_party/dav1d/src/levels.h --- a/third_party/dav1d/src/levels.h +++ b/third_party/dav1d/src/levels.h @@ -241,12 +241,15 @@ INTER_INTRA_NONE, INTER_INTRA_BLEND, INTER_INTRA_WEDGE, }; -typedef struct mv { - int16_t y, x; +typedef union mv { + struct { + int16_t y, x; + }; + uint32_t n; } mv; enum MotionMode { MM_TRANSLATION, MM_OBMC, @@ -262,16 +265,24 @@ struct { uint8_t y_mode, uv_mode, tx, pal_sz[2]; int8_t y_angle, uv_angle, cfl_alpha[2]; }; // intra struct { + union { + struct { + union mv mv[2]; + uint8_t wedge_idx, mask_sign, interintra_mode; + }; + struct { + union mv mv2d; + int16_t matrix[4]; + }; + }; + uint8_t comp_type, inter_mode, motion_mode, drl_idx; int8_t ref[2]; - uint8_t comp_type, wedge_idx, mask_sign, inter_mode, drl_idx; - uint8_t interintra_type, interintra_mode, motion_mode; - uint8_t max_ytx, filter2d; - uint16_t tx_split[2]; - mv mv[2]; + uint8_t max_ytx, filter2d, interintra_type, tx_split0; + uint16_t tx_split1; }; // inter }; } Av1Block; #endif /* DAV1D_SRC_LEVELS_H */ diff --git a/third_party/dav1d/src/lib.c b/third_party/dav1d/src/lib.c --- a/third_party/dav1d/src/lib.c +++ b/third_party/dav1d/src/lib.c @@ -29,11 +29,11 @@ #include "vcs_version.h" #include #include -#ifdef __linux__ +#if defined(__linux__) && defined(HAVE_DLSYM) #include #endif #include "dav1d/dav1d.h" #include "dav1d/data.h" @@ -79,11 +79,11 @@ static void close_internal(Dav1dContext **const c_out, int flush); NO_SANITIZE("cfi-icall") // CFI is broken with dlsym() static COLD size_t get_stack_size_internal(const pthread_attr_t *const thread_attr) { -#if defined(__linux__) && defined(HAVE_DLSYM) +#if defined(__linux__) && defined(HAVE_DLSYM) && defined(__GLIBC__) /* glibc has an issue where the size of the TLS is subtracted from the stack * size instead of allocated separately. As a result the specified stack * size may be insufficient when used in an application with large amounts * of TLS data. The following is a workaround to compensate for that. * See https://sourceware.org/bugzilla/show_bug.cgi?id=11787 */ @@ -189,12 +189,11 @@ goto error; } t->tile_thread.td.inited = 1; } } - f->libaom_cm = dav1d_alloc_ref_mv_common(); - if (!f->libaom_cm) goto error; + dav1d_refmvs_init(&f->rf); if (c->n_fc > 1) { if (pthread_mutex_init(&f->frame_thread.td.lock, NULL)) goto error; if (pthread_cond_init(&f->frame_thread.td.cond, NULL)) { pthread_mutex_destroy(&f->frame_thread.td.lock); goto error; @@ -271,25 +270,10 @@ dav1d_close(&c); return res; } -int dav1d_send_data(Dav1dContext *const c, Dav1dData *const in) -{ - validate_input_or_ret(c != NULL, DAV1D_ERR(EINVAL)); - validate_input_or_ret(in != NULL, DAV1D_ERR(EINVAL)); - validate_input_or_ret(in->data == NULL || in->sz, DAV1D_ERR(EINVAL)); - - c->drain = 0; - - if (c->in.data) - return DAV1D_ERR(EAGAIN); - dav1d_data_move_ref(&c->in, in); - - return 0; -} - static int output_image(Dav1dContext *const c, Dav1dPicture *const out, Dav1dPicture *const in) { const Dav1dFilmGrainData *fgdata = &in->frame_hdr->film_grain.data; int has_grain = fgdata->num_y_points || fgdata->num_uv_points[0] || @@ -372,25 +356,17 @@ } while (++drain_count < c->n_fc); return DAV1D_ERR(EAGAIN); } -int dav1d_get_picture(Dav1dContext *const c, Dav1dPicture *const out) +static int gen_picture(Dav1dContext *const c) { int res; - - validate_input_or_ret(c != NULL, DAV1D_ERR(EINVAL)); - validate_input_or_ret(out != NULL, DAV1D_ERR(EINVAL)); + Dav1dData *const in = &c->in; - const int drain = c->drain; - c->drain = 1; - - Dav1dData *const in = &c->in; - if (!in->data) { - if (c->n_fc == 1) return DAV1D_ERR(EAGAIN); - return drain_picture(c, out); - } + if (output_picture_ready(c)) + return 0; while (in->sz > 0) { res = dav1d_parse_obus(c, in, 0); if (res < 0) { dav1d_data_unref_internal(in); @@ -404,10 +380,44 @@ break; if (res < 0) return res; } + return 0; +} + +int dav1d_send_data(Dav1dContext *const c, Dav1dData *const in) +{ + validate_input_or_ret(c != NULL, DAV1D_ERR(EINVAL)); + validate_input_or_ret(in != NULL, DAV1D_ERR(EINVAL)); + validate_input_or_ret(in->data == NULL || in->sz, DAV1D_ERR(EINVAL)); + + if (in->data) + c->drain = 0; + if (c->in.data) + return DAV1D_ERR(EAGAIN); + dav1d_data_ref(&c->in, in); + + int res = gen_picture(c); + if (!res) + dav1d_data_unref_internal(in); + + return res; +} + +int dav1d_get_picture(Dav1dContext *const c, Dav1dPicture *const out) +{ + validate_input_or_ret(c != NULL, DAV1D_ERR(EINVAL)); + validate_input_or_ret(out != NULL, DAV1D_ERR(EINVAL)); + + const int drain = c->drain; + c->drain = 1; + + int res = gen_picture(c); + if (res < 0) + return res; + if (output_picture_ready(c)) return output_image(c, out, &c->out); if (c->n_fc > 1 && drain) return drain_picture(c, out); @@ -531,11 +541,11 @@ free(f->tile); free(f->lf.mask); free(f->lf.lr_mask); free(f->lf.level); free(f->lf.tx_lpf_right_edge[0]); - if (f->libaom_cm) dav1d_free_ref_mv_common(f->libaom_cm); + dav1d_refmvs_clear(&f->rf); dav1d_free_aligned(f->lf.cdef_line_buf); dav1d_free_aligned(f->lf.lr_lpf_line[0]); } dav1d_free_aligned(c->fc); dav1d_data_unref_internal(&c->in); diff --git a/third_party/dav1d/src/log.h b/third_party/dav1d/src/log.h --- a/third_party/dav1d/src/log.h +++ b/third_party/dav1d/src/log.h @@ -33,15 +33,15 @@ #include "dav1d/dav1d.h" #include "common/attributes.h" -void dav1d_log_default_callback(void *cookie, const char *format, va_list ap); - #if CONFIG_LOG #define dav1d_log dav1d_log +void dav1d_log_default_callback(void *cookie, const char *format, va_list ap); void dav1d_log(Dav1dContext *c, const char *format, ...) ATTR_FORMAT_PRINTF(2, 3); #else +#define dav1d_log_default_callback NULL #define dav1d_log(...) do { } while(0) #endif #endif /* DAV1D_SRC_LOG_H */ diff --git a/third_party/dav1d/src/log.c b/third_party/dav1d/src/log.c --- a/third_party/dav1d/src/log.c +++ b/third_party/dav1d/src/log.c @@ -34,17 +34,17 @@ #include "common/validate.h" #include "src/internal.h" #include "src/log.h" +#if CONFIG_LOG COLD void dav1d_log_default_callback(void *const cookie, const char *const format, va_list ap) { vfprintf(stderr, format, ap); } -#if CONFIG_LOG COLD void dav1d_log(Dav1dContext *const c, const char *const format, ...) { validate_input(c != NULL); if (!c->logger.callback) return; diff --git a/third_party/dav1d/src/lr_apply_tmpl.c b/third_party/dav1d/src/lr_apply_tmpl.c --- a/third_party/dav1d/src/lr_apply_tmpl.c +++ b/third_party/dav1d/src/lr_apply_tmpl.c @@ -71,15 +71,15 @@ } dst += 4 * PXSTRIDE(dst_stride); src += (stripe_h - 2) * PXSTRIDE(src_stride); - if (f->frame_hdr->super_res.enabled) { + if (f->frame_hdr->width[0] != f->frame_hdr->width[1]) { while (row + stripe_h <= row_h) { const int n_lines = 4 - (row + stripe_h + 1 == h); f->dsp->mc.resize(dst, dst_stride, src, src_stride, - dst_w, src_w, n_lines, f->resize_step[ss_hor], + dst_w, n_lines, src_w, f->resize_step[ss_hor], f->resize_start[ss_hor] HIGHBD_CALL_SUFFIX); row += stripe_h; // unmodified stripe_h for the 1st stripe stripe_h = 64 >> ss_ver; src += stripe_h * PXSTRIDE(src_stride); dst += n_lines * PXSTRIDE(dst_stride); diff --git a/third_party/dav1d/src/mc.h b/third_party/dav1d/src/mc.h --- a/third_party/dav1d/src/mc.h +++ b/third_party/dav1d/src/mc.h @@ -108,11 +108,11 @@ typedef decl_emu_edge_fn(*emu_edge_fn); #define decl_resize_fn(name) \ void (name)(pixel *dst, ptrdiff_t dst_stride, \ const pixel *src, ptrdiff_t src_stride, \ - int dst_w, int src_w, int h, int dx, int mx HIGHBD_DECL_SUFFIX) + int dst_w, int h, int src_w, int dx, int mx HIGHBD_DECL_SUFFIX) typedef decl_resize_fn(*resize_fn); typedef struct Dav1dMCDSPContext { mc_fn mc[N_2D_FILTERS]; mc_scaled_fn mc_scaled[N_2D_FILTERS]; diff --git a/third_party/dav1d/src/mc_tmpl.c b/third_party/dav1d/src/mc_tmpl.c --- a/third_party/dav1d/src/mc_tmpl.c +++ b/third_party/dav1d/src/mc_tmpl.c @@ -883,25 +883,25 @@ } } static void resize_c(pixel *dst, const ptrdiff_t dst_stride, const pixel *src, const ptrdiff_t src_stride, - const int dst_w, const int src_w, int h, + const int dst_w, int h, const int src_w, const int dx, const int mx0 HIGHBD_DECL_SUFFIX) { do { int mx = mx0, src_x = -1; for (int x = 0; x < dst_w; x++) { - const int16_t *const F = dav1d_resize_filter[mx >> 8]; - dst[x] = iclip_pixel((F[0] * src[iclip(src_x - 3, 0, src_w - 1)] + - F[1] * src[iclip(src_x - 2, 0, src_w - 1)] + - F[2] * src[iclip(src_x - 1, 0, src_w - 1)] + - F[3] * src[iclip(src_x + 0, 0, src_w - 1)] + - F[4] * src[iclip(src_x + 1, 0, src_w - 1)] + - F[5] * src[iclip(src_x + 2, 0, src_w - 1)] + - F[6] * src[iclip(src_x + 3, 0, src_w - 1)] + - F[7] * src[iclip(src_x + 4, 0, src_w - 1)] + + const int8_t *const F = dav1d_resize_filter[mx >> 8]; + dst[x] = iclip_pixel((-(F[0] * src[iclip(src_x - 3, 0, src_w - 1)] + + F[1] * src[iclip(src_x - 2, 0, src_w - 1)] + + F[2] * src[iclip(src_x - 1, 0, src_w - 1)] + + F[3] * src[iclip(src_x + 0, 0, src_w - 1)] + + F[4] * src[iclip(src_x + 1, 0, src_w - 1)] + + F[5] * src[iclip(src_x + 2, 0, src_w - 1)] + + F[6] * src[iclip(src_x + 3, 0, src_w - 1)] + + F[7] * src[iclip(src_x + 4, 0, src_w - 1)]) + 64) >> 7); mx += dx; src_x += mx >> 14; mx &= 0x3fff; } diff --git a/third_party/dav1d/src/meson.build b/third_party/dav1d/src/meson.build --- a/third_party/dav1d/src/meson.build +++ b/third_party/dav1d/src/meson.build @@ -41,11 +41,11 @@ 'msac.c', 'obu.c', 'picture.c', 'qm.c', 'ref.c', - 'ref_mvs.c', + 'refmvs.c', 'scan.c', 'tables.c', 'warpmv.c', 'wedge.c', ) @@ -98,43 +98,49 @@ 'arm/itx_init_tmpl.c', 'arm/loopfilter_init_tmpl.c', 'arm/looprestoration_init_tmpl.c', 'arm/mc_init_tmpl.c', ) - if host_machine.cpu_family() == 'aarch64' + if (host_machine.cpu_family() == 'aarch64' or + host_machine.cpu() == 'arm64') libdav1d_sources += files( + # itx.S is used for both 8 and 16 bpc. + 'arm/64/itx.S', 'arm/64/looprestoration_common.S', 'arm/64/msac.S', ) if dav1d_bitdepths.contains('8') libdav1d_sources += files( 'arm/64/cdef.S', 'arm/64/ipred.S', - 'arm/64/itx.S', 'arm/64/loopfilter.S', 'arm/64/looprestoration.S', 'arm/64/mc.S', ) endif if dav1d_bitdepths.contains('16') libdav1d_sources += files( 'arm/64/cdef16.S', + 'arm/64/ipred16.S', + 'arm/64/itx16.S', 'arm/64/loopfilter16.S', 'arm/64/looprestoration16.S', 'arm/64/mc16.S', ) endif elif host_machine.cpu_family().startswith('arm') libdav1d_sources += files( + 'arm/32/msac.S', ) if dav1d_bitdepths.contains('8') libdav1d_sources += files( 'arm/32/cdef.S', 'arm/32/ipred.S', + 'arm/32/itx.S', 'arm/32/loopfilter.S', 'arm/32/looprestoration.S', 'arm/32/mc.S', ) endif @@ -146,18 +152,13 @@ endif elif host_machine.cpu_family().startswith('x86') libdav1d_sources += files( 'x86/cpu.c', + 'x86/msac_init.c', ) - if host_machine.cpu_family() == 'x86_64' - libdav1d_sources += files( - 'x86/msac_init.c', - ) - endif - libdav1d_tmpl_sources += files( 'x86/cdef_init_tmpl.c', 'x86/film_grain_init_tmpl.c', 'x86/ipred_init_tmpl.c', 'x86/itx_init_tmpl.c', @@ -172,24 +173,26 @@ 'x86/msac.asm', ) if dav1d_bitdepths.contains('8') libdav1d_sources_asm += files( - 'x86/cdef.asm', + 'x86/cdef_avx512.asm', + 'x86/mc_avx512.asm', + 'x86/cdef_avx2.asm', + 'x86/mc_avx2.asm', 'x86/film_grain.asm', 'x86/ipred.asm', 'x86/itx.asm', 'x86/loopfilter.asm', 'x86/looprestoration.asm', - 'x86/mc.asm', 'x86/cdef_sse.asm', 'x86/film_grain_ssse3.asm', 'x86/ipred_ssse3.asm', 'x86/itx_ssse3.asm', 'x86/loopfilter_ssse3.asm', 'x86/looprestoration_ssse3.asm', - 'x86/mc_ssse3.asm', + 'x86/mc_sse.asm', ) endif if dav1d_bitdepths.contains('16') libdav1d_sources_asm += files( diff --git a/third_party/dav1d/src/msac.c b/third_party/dav1d/src/msac.c --- a/third_party/dav1d/src/msac.c +++ b/third_party/dav1d/src/msac.c @@ -36,11 +36,11 @@ #define EC_PROB_SHIFT 6 #define EC_MIN_PROB 4 // must be <= (1<buf_pos; const uint8_t *buf_end = s->buf_end; int c = EC_WIN_SIZE - s->cnt - 24; ec_win dif = s->dif; while (c >= 0 && buf_pos < buf_end) { @@ -55,47 +55,49 @@ /* Takes updated dif and range values, renormalizes them so that * 32768 <= rng < 65536 (reading more bytes from the stream into dif if * necessary), and stores them back in the decoder context. * dif: The new value of dif. * rng: The new value of the range. */ -static inline void ctx_norm(MsacContext *s, ec_win dif, unsigned rng) { +static inline void ctx_norm(MsacContext *const s, const ec_win dif, + const unsigned rng) +{ const int d = 15 ^ (31 ^ clz(rng)); assert(rng <= 65535U); s->cnt -= d; s->dif = ((dif + 1) << d) - 1; /* Shift in 1s in the LSBs */ s->rng = rng << d; if (s->cnt < 0) ctx_refill(s); } unsigned dav1d_msac_decode_bool_equi_c(MsacContext *const s) { - ec_win vw, dif = s->dif; - unsigned ret, v, r = s->rng; + const unsigned r = s->rng; + ec_win dif = s->dif; assert((dif >> (EC_WIN_SIZE - 16)) < r); // When the probability is 1/2, f = 16384 >> EC_PROB_SHIFT = 256 and we can // replace the multiply with a simple shift. - v = ((r >> 8) << 7) + EC_MIN_PROB; - vw = (ec_win)v << (EC_WIN_SIZE - 16); - ret = dif >= vw; - dif -= ret*vw; - v += ret*(r - 2*v); + unsigned v = ((r >> 8) << 7) + EC_MIN_PROB; + const ec_win vw = (ec_win)v << (EC_WIN_SIZE - 16); + const unsigned ret = dif >= vw; + dif -= ret * vw; + v += ret * (r - 2 * v); ctx_norm(s, dif, v); return !ret; } /* Decode a single binary value. * f: The probability that the bit is one * Return: The value decoded (0 or 1). */ unsigned dav1d_msac_decode_bool_c(MsacContext *const s, const unsigned f) { - ec_win vw, dif = s->dif; - unsigned ret, v, r = s->rng; + const unsigned r = s->rng; + ec_win dif = s->dif; assert((dif >> (EC_WIN_SIZE - 16)) < r); - v = ((r >> 8) * (f >> EC_PROB_SHIFT) >> (7 - EC_PROB_SHIFT)) + EC_MIN_PROB; - vw = (ec_win)v << (EC_WIN_SIZE - 16); - ret = dif >= vw; - dif -= ret*vw; - v += ret*(r - 2*v); + unsigned v = ((r >> 8) * (f >> EC_PROB_SHIFT) >> (7 - EC_PROB_SHIFT)) + EC_MIN_PROB; + const ec_win vw = (ec_win)v << (EC_WIN_SIZE - 16); + const unsigned ret = dif >= vw; + dif -= ret * vw; + v += ret * (r - 2 * v); ctx_norm(s, dif, v); return !ret; } int dav1d_msac_decode_subexp(MsacContext *const s, const int ref, @@ -194,14 +196,13 @@ s->buf_end = data + sz; s->dif = ((ec_win)1 << (EC_WIN_SIZE - 1)) - 1; s->rng = 0x8000; s->cnt = -15; s->allow_update_cdf = !disable_cdf_update_flag; + ctx_refill(s); #if ARCH_X86_64 && HAVE_ASM s->symbol_adapt16 = dav1d_msac_decode_symbol_adapt_c; dav1d_msac_init_x86(s); #endif - - ctx_refill(s); } diff --git a/third_party/dav1d/src/obu.c b/third_party/dav1d/src/obu.c --- a/third_party/dav1d/src/obu.c +++ b/third_party/dav1d/src/obu.c @@ -83,11 +83,11 @@ if (hdr->timing_info_present) { hdr->num_units_in_tick = dav1d_get_bits(gb, 32); hdr->time_scale = dav1d_get_bits(gb, 32); hdr->equal_picture_interval = dav1d_get_bits(gb, 1); if (hdr->equal_picture_interval) { - unsigned num_ticks_per_picture = dav1d_get_vlc(gb); + const unsigned num_ticks_per_picture = dav1d_get_vlc(gb); if (num_ticks_per_picture == 0xFFFFFFFFU) goto error; hdr->num_ticks_per_picture = num_ticks_per_picture + 1; } @@ -109,19 +109,19 @@ hdr->display_model_info_present = dav1d_get_bits(gb, 1); hdr->num_operating_points = dav1d_get_bits(gb, 5) + 1; for (int i = 0; i < hdr->num_operating_points; i++) { struct Dav1dSequenceHeaderOperatingPoint *const op = &hdr->operating_points[i]; - struct Dav1dSequenceHeaderOperatingParameterInfo *const opi = - &hdr->operating_parameter_info[i]; op->idc = dav1d_get_bits(gb, 12); op->major_level = 2 + dav1d_get_bits(gb, 3); op->minor_level = dav1d_get_bits(gb, 2); op->tier = op->major_level > 3 ? dav1d_get_bits(gb, 1) : 0; op->decoder_model_param_present = hdr->decoder_model_info_present && dav1d_get_bits(gb, 1); if (op->decoder_model_param_present) { + struct Dav1dSequenceHeaderOperatingParameterInfo *const opi = + &hdr->operating_parameter_info[i]; opi->decoder_buffer_delay = dav1d_get_bits(gb, hdr->encoder_decoder_buffer_delay_length); opi->encoder_buffer_delay = dav1d_get_bits(gb, hdr->encoder_decoder_buffer_delay_length); opi->low_delay_mode = dav1d_get_bits(gb, 1); @@ -130,14 +130,13 @@ hdr->display_model_info_present && dav1d_get_bits(gb, 1); if (op->display_model_param_present) { op->initial_display_delay = dav1d_get_bits(gb, 4) + 1; } } - if (c->operating_point < hdr->num_operating_points) - c->operating_point_idc = hdr->operating_points[c->operating_point].idc; - else - c->operating_point_idc = hdr->operating_points[0].idc; + const int op_idx = + c->operating_point < hdr->num_operating_points ? c->operating_point : 0; + c->operating_point_idc = hdr->operating_points[op_idx].idc; #if DEBUG_SEQ_HDR printf("SEQHDR: post-operating-points: off=%ld\n", dav1d_get_bits_pos(gb) - init_bit_pos); #endif } @@ -293,11 +292,11 @@ Dav1dFrameHeader *const hdr = c->frame_hdr; if (use_ref) { for (int i = 0; i < 7; i++) { if (dav1d_get_bits(gb, 1)) { - Dav1dThreadPicture *const ref = + const Dav1dThreadPicture *const ref = &c->refs[c->frame_hdr->refidx[i]].p; if (!ref->p.data[0]) return -1; hdr->width[1] = ref->p.p.w; hdr->height = ref->p.p.h; hdr->render_width = ref->p.frame_hdr->render_width; @@ -341,11 +340,11 @@ hdr->render_height = hdr->height; } return 0; } -static inline int tile_log2(int sz, int tgt) { +static inline int tile_log2(const int sz, const int tgt) { int k; for (k = 0; (sz << k) < tgt; k++) ; return k; } @@ -360,11 +359,10 @@ #if DEBUG_FRAME_HDR const uint8_t *const init_ptr = gb->ptr; #endif const Dav1dSequenceHeader *const seqhdr = c->seq_hdr; Dav1dFrameHeader *const hdr = c->frame_hdr; - int res; hdr->show_existing_frame = !seqhdr->reduced_still_picture_header && dav1d_get_bits(gb, 1); #if DEBUG_FRAME_HDR printf("HDR: post-show_existing_frame: off=%ld\n", @@ -442,11 +440,11 @@ hdr->refresh_frame_flags = (hdr->frame_type == DAV1D_FRAME_TYPE_KEY && hdr->show_frame) ? 0xff : dav1d_get_bits(gb, 8); if (hdr->refresh_frame_flags != 0xff && hdr->error_resilient_mode && seqhdr->order_hint) for (int i = 0; i < 8; i++) dav1d_get_bits(gb, seqhdr->order_hint_n_bits); - if ((res = read_frame_size(c, gb, 0)) < 0) goto error; + if (read_frame_size(c, gb, 0) < 0) goto error; hdr->allow_intrabc = hdr->allow_screen_content_tools && !hdr->super_res.enabled && dav1d_get_bits(gb, 1); hdr->use_ref_frame_mvs = 0; } else { hdr->allow_intrabc = 0; @@ -477,11 +475,11 @@ used_frame[hdr->refidx[0]] = 1; used_frame[hdr->refidx[3]] = 1; int latest_frame_offset = -1; for (int i = 0; i < 8; i++) { - int hint = shifted_frame_offset[i]; + const int hint = shifted_frame_offset[i]; if (!used_frame[i] && hint >= current_frame_offset && hint >= latest_frame_offset) { hdr->refidx[6] = i; latest_frame_offset = hint; @@ -490,11 +488,11 @@ if (latest_frame_offset != -1) used_frame[hdr->refidx[6]] = 1; int earliest_frame_offset = INT_MAX; for (int i = 0; i < 8; i++) { - int hint = shifted_frame_offset[i]; + const int hint = shifted_frame_offset[i]; if (!used_frame[i] && hint >= current_frame_offset && hint < earliest_frame_offset) { hdr->refidx[4] = i; earliest_frame_offset = hint; @@ -503,11 +501,11 @@ if (earliest_frame_offset != INT_MAX) used_frame[hdr->refidx[4]] = 1; earliest_frame_offset = INT_MAX; for (int i = 0; i < 8; i++) { - int hint = shifted_frame_offset[i]; + const int hint = shifted_frame_offset[i]; if (!used_frame[i] && hint >= current_frame_offset && (hint < earliest_frame_offset)) { hdr->refidx[5] = i; earliest_frame_offset = hint; @@ -518,11 +516,11 @@ for (int i = 1; i < 7; i++) { if (hdr->refidx[i] < 0) { latest_frame_offset = -1; for (int j = 0; j < 8; j++) { - int hint = shifted_frame_offset[j]; + const int hint = shifted_frame_offset[j]; if (!used_frame[j] && hint < current_frame_offset && hint >= latest_frame_offset) { hdr->refidx[i] = j; latest_frame_offset = hint; @@ -534,11 +532,11 @@ } earliest_frame_offset = INT_MAX; int ref = -1; for (int i = 0; i < 8; i++) { - int hint = shifted_frame_offset[i]; + const int hint = shifted_frame_offset[i]; if (hint < earliest_frame_offset) { ref = i; earliest_frame_offset = hint; } } @@ -553,11 +551,11 @@ if (seqhdr->frame_id_numbers_present) dav1d_get_bits(gb, seqhdr->delta_frame_id_n_bits); } const int use_ref = !hdr->error_resilient_mode && hdr->frame_size_override; - if ((res = read_frame_size(c, gb, use_ref)) < 0) goto error; + if (read_frame_size(c, gb, use_ref) < 0) goto error; hdr->hp = !hdr->force_integer_mv && dav1d_get_bits(gb, 1); hdr->subpel_filter_mode = dav1d_get_bits(gb, 1) ? DAV1D_FILTER_SWITCHABLE : dav1d_get_bits(gb, 2); hdr->switchable_motion_mode = dav1d_get_bits(gb, 1); hdr->use_ref_frame_mvs = !hdr->error_resilient_mode && @@ -577,19 +575,19 @@ #endif // tile data hdr->tiling.uniform = dav1d_get_bits(gb, 1); const int sbsz_min1 = (64 << seqhdr->sb128) - 1; - int sbsz_log2 = 6 + seqhdr->sb128; - int sbw = (hdr->width[0] + sbsz_min1) >> sbsz_log2; - int sbh = (hdr->height + sbsz_min1) >> sbsz_log2; - int max_tile_width_sb = 4096 >> sbsz_log2; - int max_tile_area_sb = 4096 * 2304 >> (2 * sbsz_log2); + const int sbsz_log2 = 6 + seqhdr->sb128; + const int sbw = (hdr->width[0] + sbsz_min1) >> sbsz_log2; + const int sbh = (hdr->height + sbsz_min1) >> sbsz_log2; + const int max_tile_width_sb = 4096 >> sbsz_log2; + const int max_tile_area_sb = 4096 * 2304 >> (2 * sbsz_log2); hdr->tiling.min_log2_cols = tile_log2(max_tile_width_sb, sbw); hdr->tiling.max_log2_cols = tile_log2(1, imin(sbw, DAV1D_MAX_TILE_COLS)); hdr->tiling.max_log2_rows = tile_log2(1, imin(sbh, DAV1D_MAX_TILE_ROWS)); - int min_log2_tiles = imax(tile_log2(max_tile_area_sb, sbw * sbh), + const int min_log2_tiles = imax(tile_log2(max_tile_area_sb, sbw * sbh), hdr->tiling.min_log2_cols); if (hdr->tiling.uniform) { for (hdr->tiling.log2_cols = hdr->tiling.min_log2_cols; hdr->tiling.log2_cols < hdr->tiling.max_log2_cols && dav1d_get_bits(gb, 1); hdr->tiling.log2_cols++) ; @@ -619,11 +617,11 @@ sbx += tile_w; widest_tile = imax(widest_tile, tile_w); } hdr->tiling.log2_cols = tile_log2(1, hdr->tiling.cols); if (min_log2_tiles) max_tile_area_sb >>= min_log2_tiles + 1; - int max_tile_height_sb = imax(max_tile_area_sb / widest_tile, 1); + const int max_tile_height_sb = imax(max_tile_area_sb / widest_tile, 1); hdr->tiling.rows = 0; for (int sby = 0; sby < sbh && hdr->tiling.rows < DAV1D_MAX_TILE_ROWS; hdr->tiling.rows++) { const int tile_height_sb = imin(sbh - sby, max_tile_height_sb); const int tile_h = (tile_height_sb > 1) ? @@ -655,11 +653,11 @@ hdr->quant.ydc_delta = dav1d_get_bits(gb, 1) ? dav1d_get_sbits(gb, 6) : 0; if (!seqhdr->monochrome) { // If the sequence header says that delta_q might be different // for U, V, we must check whether it actually is for this // frame. - int diff_uv_delta = seqhdr->separate_uv_delta_q ? dav1d_get_bits(gb, 1) : 0; + const int diff_uv_delta = seqhdr->separate_uv_delta_q ? dav1d_get_bits(gb, 1) : 0; hdr->quant.udc_delta = dav1d_get_bits(gb, 1) ? dav1d_get_sbits(gb, 6) : 0; hdr->quant.uac_delta = dav1d_get_bits(gb, 1) ? dav1d_get_sbits(gb, 6) : 0; if (diff_uv_delta) { hdr->quant.vdc_delta = dav1d_get_bits(gb, 1) ? dav1d_get_sbits(gb, 6) : 0; hdr->quant.vac_delta = dav1d_get_bits(gb, 1) ? dav1d_get_sbits(gb, 6) : 0; @@ -1051,11 +1049,11 @@ const int refidx = dav1d_get_bits(gb, 3); int i; for (i = 0; i < 7; i++) if (hdr->refidx[i] == refidx) break; - if (i == 7 || !c->refs[refidx].p.p.frame_hdr) goto error; + if (i == 7 || !c->refs[refidx].p.p.frame_hdr) goto error; hdr->film_grain.data = c->refs[refidx].p.p.frame_hdr->film_grain.data; hdr->film_grain.data.seed = seed; } else { Dav1dFilmGrainData *const fgd = &hdr->film_grain.data; fgd->seed = seed; @@ -1131,14 +1129,12 @@ dav1d_log(c, "Error parsing frame header\n"); return DAV1D_ERR(EINVAL); } static void parse_tile_hdr(Dav1dContext *const c, GetBits *const gb) { - int have_tile_pos = 0; const int n_tiles = c->frame_hdr->tiling.cols * c->frame_hdr->tiling.rows; - if (n_tiles > 1) - have_tile_pos = dav1d_get_bits(gb, 1); + const int have_tile_pos = n_tiles > 1 ? dav1d_get_bits(gb, 1) : 0; if (have_tile_pos) { const int n_bits = c->frame_hdr->tiling.log2_cols + c->frame_hdr->tiling.log2_rows; c->tile[c->n_tile_data].start = dav1d_get_bits(gb, n_bits); @@ -1149,21 +1145,21 @@ } } // Check that we haven't read more than obu_len bytes from the buffer // since init_bit_pos. -static int -check_for_overrun(Dav1dContext *const c, GetBits *const gb, - unsigned init_bit_pos, unsigned obu_len) +static int check_for_overrun(Dav1dContext *const c, GetBits *const gb, + const unsigned init_bit_pos, + const unsigned obu_len) { // Make sure we haven't actually read past the end of the gb buffer if (gb->error) { dav1d_log(c, "Overrun in OBU bit buffer\n"); return 1; } - unsigned pos = dav1d_get_bits_pos(gb); + const unsigned pos = dav1d_get_bits_pos(gb); // We assume that init_bit_pos was the bit position of the buffer // at some point in the past, so cannot be smaller than pos. assert (init_bit_pos <= pos); @@ -1173,11 +1169,11 @@ } return 0; } -int dav1d_parse_obus(Dav1dContext *const c, Dav1dData *const in, int global) { +int dav1d_parse_obus(Dav1dContext *const c, Dav1dData *const in, const int global) { GetBits gb; int res; dav1d_init_get_bits(&gb, in->data, in->sz); @@ -1194,15 +1190,12 @@ spatial_id = dav1d_get_bits(&gb, 2); dav1d_get_bits(&gb, 3); // reserved } // obu length field - unsigned len = 0; - if (has_length_field) - len = dav1d_get_uleb128(&gb); - else - len = (int) in->sz - 1 - has_extension; + const unsigned len = has_length_field ? + dav1d_get_uleb128(&gb) : (unsigned) in->sz - 1 - has_extension; if (gb.error) goto error; const unsigned init_bit_pos = dav1d_get_bits_pos(&gb); const unsigned init_byte_pos = init_bit_pos >> 3; const unsigned pkt_bytelen = init_byte_pos + len; @@ -1440,11 +1433,11 @@ // Don't take into account meta_type bytes payload_size -= meta_type_len; int country_code_extension_byte = 0; - int country_code = dav1d_get_bits(&gb, 8); + const int country_code = dav1d_get_bits(&gb, 8); payload_size--; if (country_code == 0xFF) { country_code_extension_byte = dav1d_get_bits(&gb, 8); payload_size--; } diff --git a/third_party/dav1d/src/picture.c b/third_party/dav1d/src/picture.c --- a/third_party/dav1d/src/picture.c +++ b/third_party/dav1d/src/picture.c @@ -66,11 +66,11 @@ p->stride[0] = y_stride; p->stride[1] = uv_stride; const size_t y_sz = y_stride * aligned_h; const size_t uv_sz = uv_stride * (aligned_h >> ss_ver); const size_t pic_size = y_sz + 2 * uv_sz + DAV1D_PICTURE_ALIGNMENT; - uint8_t *data = dav1d_alloc_aligned(pic_size, DAV1D_PICTURE_ALIGNMENT); + uint8_t *const data = dav1d_alloc_aligned(pic_size, DAV1D_PICTURE_ALIGNMENT); if (!data) return DAV1D_ERR(ENOMEM); p->data[0] = data; p->data[1] = has_chroma ? data + y_sz : NULL; p->data[2] = has_chroma ? data + y_sz + uv_sz : NULL; @@ -102,31 +102,32 @@ pic_ctx->allocator.release_picture_callback(&pic_ctx->pic, pic_ctx->allocator.cookie); free(pic_ctx); } -static int picture_alloc_with_edges(Dav1dContext *const c, Dav1dPicture *const p, +static int picture_alloc_with_edges(Dav1dContext *const c, + Dav1dPicture *const p, const int w, const int h, - Dav1dSequenceHeader *seq_hdr, Dav1dRef *seq_hdr_ref, - Dav1dFrameHeader *frame_hdr, Dav1dRef *frame_hdr_ref, - Dav1dContentLightLevel *content_light, Dav1dRef *content_light_ref, - Dav1dMasteringDisplay *mastering_display, Dav1dRef *mastering_display_ref, - Dav1dITUTT35 *itut_t35, Dav1dRef *itut_t35_ref, - const int bpc, const Dav1dDataProps *props, + Dav1dSequenceHeader *const seq_hdr, Dav1dRef *const seq_hdr_ref, + Dav1dFrameHeader *const frame_hdr, Dav1dRef *const frame_hdr_ref, + Dav1dContentLightLevel *const content_light, Dav1dRef *const content_light_ref, + Dav1dMasteringDisplay *const mastering_display, Dav1dRef *const mastering_display_ref, + Dav1dITUTT35 *const itut_t35, Dav1dRef *const itut_t35_ref, + const int bpc, + const Dav1dDataProps *const props, Dav1dPicAllocator *const p_allocator, const size_t extra, void **const extra_ptr) { if (p->data[0]) { dav1d_log(c, "Picture already allocated!\n"); return -1; } assert(bpc > 0 && bpc <= 16); struct pic_ctx_context *pic_ctx = malloc(extra + sizeof(struct pic_ctx_context)); - if (pic_ctx == NULL) { + if (pic_ctx == NULL) return DAV1D_ERR(ENOMEM); - } p->p.w = w; p->p.h = h; p->seq_hdr = seq_hdr; p->frame_hdr = frame_hdr; @@ -134,11 +135,11 @@ p->mastering_display = mastering_display; p->itut_t35 = itut_t35; p->p.layout = seq_hdr->layout; p->p.bpc = bpc; dav1d_data_props_set_defaults(&p->m); - int res = p_allocator->alloc_picture_callback(p, p_allocator->cookie); + const int res = p_allocator->alloc_picture_callback(p, p_allocator->cookie); if (res < 0) { free(pic_ctx); return res; } @@ -248,12 +249,12 @@ *dst = *src; memset(src, 0, sizeof(*src)); } -void dav1d_thread_picture_ref(Dav1dThreadPicture *dst, - const Dav1dThreadPicture *src) +void dav1d_thread_picture_ref(Dav1dThreadPicture *const dst, + const Dav1dThreadPicture *const src) { dav1d_picture_ref(&dst->p, &src->p); dst->t = src->t; dst->visible = src->visible; dst->progress = src->progress; diff --git a/third_party/dav1d/src/recon_tmpl.c b/third_party/dav1d/src/recon_tmpl.c --- a/third_party/dav1d/src/recon_tmpl.c +++ b/third_party/dav1d/src/recon_tmpl.c @@ -772,14 +772,15 @@ const int cw4 = (w4 + ss_hor) >> ss_hor, ch4 = (h4 + ss_ver) >> ss_ver; assert(f->frame_thread.pass == 1); assert(!b->skip); const TxfmInfo *const uv_t_dim = &dav1d_txfm_dimensions[b->uvtx]; const TxfmInfo *const t_dim = &dav1d_txfm_dimensions[b->intra ? b->tx : b->max_ytx]; + const uint16_t tx_split[2] = { b->tx_split0, b->tx_split1 }; for (int init_y = 0; init_y < h4; init_y += 16) { + const int sub_h4 = imin(h4, 16 + init_y); for (int init_x = 0; init_x < w4; init_x += 16) { - const int sub_h4 = imin(h4, 16 + init_y); const int sub_w4 = imin(w4, init_x + 16); int y_off = !!init_y, y, x; for (y = init_y, t->by += init_y; y < sub_h4; y += t_dim->h, t->by += t_dim->h, y_off++) { @@ -788,11 +789,11 @@ int x_off = !!init_x; for (x = init_x, t->bx += init_x; x < sub_w4; x += t_dim->w, t->bx += t_dim->w, x_off++) { if (!b->intra) { - read_coef_tree(t, bs, b, b->max_ytx, 0, b->tx_split, + read_coef_tree(t, bs, b, b->max_ytx, 0, tx_split, x_off, y_off, NULL); } else { uint8_t cf_ctx = 0x40; enum TxfmType txtp; const int eob = cbi[t->bx].eob[0] = @@ -929,12 +930,12 @@ HIGHBD_CALL_SUFFIX); } } else { assert(refp != &f->sr_cur); - int orig_pos_y = (by * v_mul << 4) + mvy * (1 << !ss_ver); - int orig_pos_x = (bx * h_mul << 4) + mvx * (1 << !ss_hor); + const int orig_pos_y = (by * v_mul << 4) + mvy * (1 << !ss_ver); + const int orig_pos_x = (bx * h_mul << 4) + mvx * (1 << !ss_hor); #define scale_mv(res, val, scale) do { \ const int64_t tmp = (int64_t)(val) * scale + (scale - 0x4000) * 8; \ res = apply_sign64((int) ((llabs(tmp) + 128) >> 8), tmp) + 32; \ } while (0) int pos_y, pos_x; @@ -996,11 +997,11 @@ const uint8_t *const b_dim, const int pl, const int bx4, const int by4, const int w4, const int h4) { assert(!(t->bx & 1) && !(t->by & 1)); const Dav1dFrameContext *const f = t->f; - const refmvs *const r = &f->mvs[t->by * f->b4_stride + t->bx]; + /*const*/ refmvs_block **r = &t->rt.r[(t->by & 31) + 5]; pixel *const lap = bitfn(t->scratch.lap); const int ss_ver = !!pl && f->cur.p.layout == DAV1D_PIXEL_LAYOUT_I420; const int ss_hor = !!pl && f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I444; const int h_mul = 4 >> ss_hor, v_mul = 4 >> ss_ver; int res; @@ -1008,20 +1009,19 @@ if (t->by > t->ts->tiling.row_start && (!pl || b_dim[0] * h_mul + b_dim[1] * v_mul >= 16)) { for (int i = 0, x = 0; x < w4 && i < imin(b_dim[2], 4); ) { // only odd blocks are considered for overlap handling, hence +1 - const refmvs *const a_r = &r[x - f->b4_stride + 1]; - const uint8_t *const a_b_dim = - dav1d_block_dimensions[dav1d_sbtype_to_bs[a_r->sb_type]]; + const refmvs_block *const a_r = &r[-1][t->bx + x + 1]; + const uint8_t *const a_b_dim = dav1d_block_dimensions[a_r->bs]; - if (a_r->ref[0] > 0) { + if (a_r->ref.ref[0] > 0) { const int ow4 = iclip(a_b_dim[0], 2, b_dim[0]); const int oh4 = imin(b_dim[1], 16) >> 1; res = mc(t, lap, NULL, ow4 * h_mul * sizeof(pixel), ow4, (oh4 * 3 + 3) >> 2, - t->bx + x, t->by, pl, a_r->mv[0], - &f->refp[a_r->ref[0] - 1], a_r->ref[0] - 1, + t->bx + x, t->by, pl, a_r->mv.mv[0], + &f->refp[a_r->ref.ref[0] - 1], a_r->ref.ref[0] - 1, dav1d_filter_2d[t->a->filter[1][bx4 + x + 1]][t->a->filter[0][bx4 + x + 1]]); if (res) return res; f->dsp->mc.blend_h(&dst[x * h_mul], dst_stride, lap, h_mul * ow4, v_mul * oh4); i++; @@ -1031,20 +1031,19 @@ } if (t->bx > t->ts->tiling.col_start) for (int i = 0, y = 0; y < h4 && i < imin(b_dim[3], 4); ) { // only odd blocks are considered for overlap handling, hence +1 - const refmvs *const l_r = &r[(y + 1) * f->b4_stride - 1]; - const uint8_t *const l_b_dim = - dav1d_block_dimensions[dav1d_sbtype_to_bs[l_r->sb_type]]; + const refmvs_block *const l_r = &r[y + 1][t->bx - 1]; + const uint8_t *const l_b_dim = dav1d_block_dimensions[l_r->bs]; - if (l_r->ref[0] > 0) { + if (l_r->ref.ref[0] > 0) { const int ow4 = imin(b_dim[0], 16) >> 1; const int oh4 = iclip(l_b_dim[1], 2, b_dim[1]); res = mc(t, lap, NULL, h_mul * ow4 * sizeof(pixel), ow4, oh4, - t->bx, t->by + y, pl, l_r->mv[0], - &f->refp[l_r->ref[0] - 1], l_r->ref[0] - 1, + t->bx, t->by + y, pl, l_r->mv.mv[0], + &f->refp[l_r->ref.ref[0] - 1], l_r->ref.ref[0] - 1, dav1d_filter_2d[t->l.filter[1][by4 + y + 1]][t->l.filter[0][by4 + y + 1]]); if (res) return res; f->dsp->mc.blend_v(&dst[y * v_mul * PXSTRIDE(dst_stride)], dst_stride, lap, h_mul * ow4, v_mul * oh4); i++; @@ -1070,19 +1069,19 @@ const int32_t *const mat = wmp->matrix; const int width = (refp->p.p.w + ss_hor) >> ss_hor; const int height = (refp->p.p.h + ss_ver) >> ss_ver; for (int y = 0; y < b_dim[1] * v_mul; y += 8) { + const int src_y = t->by * 4 + ((y + 4) << ss_ver); + const int64_t mat3_y = (int64_t) mat[3] * src_y + mat[0]; + const int64_t mat5_y = (int64_t) mat[5] * src_y + mat[1]; for (int x = 0; x < b_dim[0] * h_mul; x += 8) { // calculate transformation relative to center of 8x8 block in // luma pixel units const int src_x = t->bx * 4 + ((x + 4) << ss_hor); - const int src_y = t->by * 4 + ((y + 4) << ss_ver); - const int64_t mvx = ((int64_t) mat[2] * src_x + - (int64_t) mat[3] * src_y + mat[0]) >> ss_hor; - const int64_t mvy = ((int64_t) mat[4] * src_x + - (int64_t) mat[5] * src_y + mat[1]) >> ss_ver; + const int64_t mvx = ((int64_t) mat[2] * src_x + mat3_y) >> ss_hor; + const int64_t mvy = ((int64_t) mat[4] * src_x + mat5_y) >> ss_ver; const int dx = (int) (mvx >> 16) - 4; const int mx = (((int) mvx & 0xffff) - wmp->alpha * 4 - wmp->beta * 7) & ~0x3f; const int dy = (int) (mvy >> 16) - 4; @@ -1146,10 +1145,12 @@ const int cbw4 = (bw4 + ss_hor) >> ss_hor, cbh4 = (bh4 + ss_ver) >> ss_ver; const int intra_edge_filter_flag = f->seq_hdr->intra_edge_filter << 10; for (int init_y = 0; init_y < h4; init_y += 16) { + const int sub_h4 = imin(h4, 16 + init_y); + const int sub_ch4 = imin(ch4, (init_y + 16) >> ss_ver); for (int init_x = 0; init_x < w4; init_x += 16) { if (b->pal_sz[0]) { pixel *dst = ((pixel *) f->cur.data[0]) + 4 * (t->by * PXSTRIDE(f->cur.stride[0]) + t->bx); const uint8_t *pal_idx; @@ -1176,11 +1177,10 @@ const int sb_has_tr = init_x + 16 < w4 ? 1 : init_y ? 0 : intra_edge_flags & EDGE_I444_TOP_HAS_RIGHT; const int sb_has_bl = init_x ? 0 : init_y + 16 < h4 ? 1 : intra_edge_flags & EDGE_I444_LEFT_HAS_BOTTOM; int y, x; - const int sub_h4 = imin(h4, 16 + init_y); const int sub_w4 = imin(w4, init_x + 16); for (y = init_y, t->by += init_y; y < sub_h4; y += t_dim->h, t->by += t_dim->h) { pixel *dst = ((pixel *) f->cur.data[0]) + @@ -1344,12 +1344,12 @@ ac_dump(ac, 4*cbw4, 4*cbh4, "ac"); hex_dump(uv_dst[0], stride, cbw4 * 4, cbh4 * 4, "u-cfl-pred"); hex_dump(uv_dst[1], stride, cbw4 * 4, cbh4 * 4, "v-cfl-pred"); } } else if (b->pal_sz[1]) { - ptrdiff_t uv_dstoff = 4 * ((t->bx >> ss_hor) + - (t->by >> ss_ver) * PXSTRIDE(f->cur.stride[1])); + const ptrdiff_t uv_dstoff = 4 * ((t->bx >> ss_hor) + + (t->by >> ss_ver) * PXSTRIDE(f->cur.stride[1])); const uint16_t (*pal)[8]; const uint8_t *pal_idx; if (f->frame_thread.pass) { assert(ts->frame_thread.pal_idx); pal = f->frame_thread.pal[((t->by >> 1) + (t->bx & 1)) * (f->b4_stride >> 1) + @@ -1383,11 +1383,10 @@ ((init_x + 16) >> ss_hor) < cw4 ? 1 : init_y ? 0 : intra_edge_flags & (EDGE_I420_TOP_HAS_RIGHT >> (f->cur.p.layout - 1)); const int uv_sb_has_bl = init_x ? 0 : ((init_y + 16) >> ss_ver) < ch4 ? 1 : intra_edge_flags & (EDGE_I420_LEFT_HAS_BOTTOM >> (f->cur.p.layout - 1)); - const int sub_ch4 = imin(ch4, (init_y + 16) >> ss_ver); const int sub_cw4 = imin(cw4, (init_x + 16) >> ss_hor); for (int pl = 0; pl < 2; pl++) { for (y = init_y >> ss_ver, t->by += init_y; y < sub_ch4; y += uv_t_dim->h, t->by += uv_t_dim->h << ss_ver) { @@ -1519,11 +1518,11 @@ } } } int bytefn(dav1d_recon_b_inter)(Dav1dTileContext *const t, const enum BlockSize bs, - const Av1Block *const b) + const Av1Block *const b) { Dav1dTileState *const ts = t->ts; const Dav1dFrameContext *const f = t->f; const Dav1dDSPContext *const dsp = f->dsp; const int bx4 = t->bx & 31, by4 = t->by & 31; @@ -1611,18 +1610,18 @@ if (!has_chroma) goto skip_inter_chroma_pred; // sub8x8 derivation int is_sub8x8 = bw4 == ss_hor || bh4 == ss_ver; - refmvs *r; + refmvs_block *const *r; if (is_sub8x8) { assert(ss_hor == 1); - r = &f->mvs[t->by * f->b4_stride + t->bx]; - if (bw4 == 1) is_sub8x8 &= r[-1].ref[0] > 0; - if (bh4 == ss_ver) is_sub8x8 &= r[-f->b4_stride].ref[0] > 0; + r = &t->rt.r[(t->by & 31) + 5]; + if (bw4 == 1) is_sub8x8 &= r[0][t->bx - 1].ref.ref[0] > 0; + if (bh4 == ss_ver) is_sub8x8 &= r[-1][t->bx].ref.ref[0] > 0; if (bw4 == 1 && bh4 == ss_ver) - is_sub8x8 &= r[-(1 + f->b4_stride)].ref[0] > 0; + is_sub8x8 &= r[-1][t->bx - 1].ref.ref[0] > 0; } // chroma prediction if (is_sub8x8) { assert(ss_hor == 1); @@ -1630,13 +1629,13 @@ if (bw4 == 1 && bh4 == ss_ver) { for (int pl = 0; pl < 2; pl++) { res = mc(t, ((pixel *) f->cur.data[1 + pl]) + uvdstoff, NULL, f->cur.stride[1], bw4, bh4, t->bx - 1, t->by - 1, 1 + pl, - r[-(f->b4_stride + 1)].mv[0], - &f->refp[r[-(f->b4_stride + 1)].ref[0] - 1], - r[-(f->b4_stride + 1)].ref[0] - 1, + r[-1][t->bx - 1].mv.mv[0], + &f->refp[r[-1][t->bx - 1].ref.ref[0] - 1], + r[-1][t->bx - 1].ref.ref[0] - 1, f->frame_thread.pass != 2 ? t->tl_4x4_filter : f->frame_thread.b[((t->by - 1) * f->b4_stride) + t->bx - 1].filter2d); if (res) return res; } v_off = 2 * PXSTRIDE(f->cur.stride[1]); @@ -1646,12 +1645,13 @@ const enum Filter2d left_filter_2d = dav1d_filter_2d[t->l.filter[1][by4]][t->l.filter[0][by4]]; for (int pl = 0; pl < 2; pl++) { res = mc(t, ((pixel *) f->cur.data[1 + pl]) + uvdstoff + v_off, NULL, f->cur.stride[1], bw4, bh4, t->bx - 1, - t->by, 1 + pl, r[-1].mv[0], &f->refp[r[-1].ref[0] - 1], - r[-1].ref[0] - 1, + t->by, 1 + pl, r[0][t->bx - 1].mv.mv[0], + &f->refp[r[0][t->bx - 1].ref.ref[0] - 1], + r[0][t->bx - 1].ref.ref[0] - 1, f->frame_thread.pass != 2 ? left_filter_2d : f->frame_thread.b[(t->by * f->b4_stride) + t->bx - 1].filter2d); if (res) return res; } h_off = 2; @@ -1660,13 +1660,13 @@ const enum Filter2d top_filter_2d = dav1d_filter_2d[t->a->filter[1][bx4]][t->a->filter[0][bx4]]; for (int pl = 0; pl < 2; pl++) { res = mc(t, ((pixel *) f->cur.data[1 + pl]) + uvdstoff + h_off, NULL, f->cur.stride[1], bw4, bh4, t->bx, t->by - 1, - 1 + pl, r[-f->b4_stride].mv[0], - &f->refp[r[-f->b4_stride].ref[0] - 1], - r[-f->b4_stride].ref[0] - 1, + 1 + pl, r[-1][t->bx].mv.mv[0], + &f->refp[r[-1][t->bx].ref.ref[0] - 1], + r[-1][t->bx].ref.ref[0] - 1, f->frame_thread.pass != 2 ? top_filter_2d : f->frame_thread.b[((t->by - 1) * f->b4_stride) + t->bx].filter2d); if (res) return res; } v_off = 2 * PXSTRIDE(f->cur.stride[1]); @@ -1868,10 +1868,11 @@ return 0; } const TxfmInfo *const uvtx = &dav1d_txfm_dimensions[b->uvtx]; const TxfmInfo *const ytx = &dav1d_txfm_dimensions[b->max_ytx]; + const uint16_t tx_split[2] = { b->tx_split0, b->tx_split1 }; for (int init_y = 0; init_y < bh4; init_y += 16) { for (int init_x = 0; init_x < bw4; init_x += 16) { // coefficient coding & inverse transforms int y_off = !!init_y, y; @@ -1881,11 +1882,11 @@ { int x, x_off = !!init_x; for (x = init_x, t->bx += init_x; x < imin(w4, init_x + 16); x += ytx->w, x_off++) { - read_coef_tree(t, bs, b, b->max_ytx, 0, b->tx_split, + read_coef_tree(t, bs, b, b->max_ytx, 0, tx_split, x_off, y_off, &dst[x * 4]); t->bx += ytx->w; } dst += PXSTRIDE(f->cur.stride[0]) * 4 * ytx->h; t->bx -= x; @@ -2010,13 +2011,14 @@ const int ss_hor = pl && f->cur.p.layout != DAV1D_PIXEL_LAYOUT_I444; const int dst_w = (f->sr_cur.p.p.w + ss_hor) >> ss_hor; const int src_w = (4 * f->bw + ss_hor) >> ss_hor; const int img_h = (f->cur.p.h - sbsz * 4 * sby + ss_ver) >> ss_ver; - f->dsp->mc.resize(dst, dst_stride, src, src_stride, dst_w, src_w, - imin(img_h, h_end) + h_start, f->resize_step[!!pl], - f->resize_start[!!pl] HIGHBD_CALL_SUFFIX); + f->dsp->mc.resize(dst, dst_stride, src, src_stride, dst_w, + imin(img_h, h_end) + h_start, src_w, + f->resize_step[!!pl], f->resize_start[!!pl] + HIGHBD_CALL_SUFFIX); } } if (f->lf.restore_planes) { bytefn(dav1d_lr_sbrow)(f, f->lf.sr_p, sby); } diff --git a/third_party/dav1d/src/ref.c b/third_party/dav1d/src/ref.c --- a/third_party/dav1d/src/ref.c +++ b/third_party/dav1d/src/ref.c @@ -35,29 +35,25 @@ assert(data == user_data); dav1d_free_aligned(user_data); } Dav1dRef *dav1d_ref_create(const size_t size) { - Dav1dRef *res; void *data = dav1d_alloc_aligned(size, 32); - if (!data) { - return NULL; - } + if (!data) return NULL; - res = dav1d_ref_wrap(data, default_free_callback, data); - if (!res) { + Dav1dRef *const res = dav1d_ref_wrap(data, default_free_callback, data); + if (res) + res->data = data; + else dav1d_free_aligned(data); - } else { - res->data = data; - } return res; } Dav1dRef *dav1d_ref_wrap(const uint8_t *const ptr, void (*free_callback)(const uint8_t *data, void *user_data), - void *user_data) + void *const user_data) { Dav1dRef *res = malloc(sizeof(Dav1dRef)); if (!res) return NULL; res->data = NULL; diff --git a/third_party/dav1d/src/ref_mvs.h b/third_party/dav1d/src/ref_mvs.h deleted file mode 100644 --- a/third_party/dav1d/src/ref_mvs.h +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#ifndef DAV1D_SRC_REF_MVS_H -#define DAV1D_SRC_REF_MVS_H - -#include - -#include "src/levels.h" -#include "src/tables.h" - -typedef struct refmvs { - mv mv[2]; - int8_t ref[2]; // [0] = 0: intra=1, [1] = -1: comp=0 - int8_t mode, sb_type; -} refmvs; - -typedef struct candidate_mv { - mv this_mv; - mv comp_mv; - int weight; -} candidate_mv; - -typedef struct AV1_COMMON AV1_COMMON; - -// call once per frame thread -AV1_COMMON *dav1d_alloc_ref_mv_common(void); -void dav1d_free_ref_mv_common(AV1_COMMON *cm); - -// call once per frame -int dav1d_init_ref_mv_common(AV1_COMMON *cm, int w8, int h8, - ptrdiff_t stride, int allow_sb128, - refmvs *cur, refmvs *ref_mvs[7], - unsigned cur_poc, - const unsigned ref_poc[7], - const unsigned ref_ref_poc[7][7], - const Dav1dWarpedMotionParams gmv[7], - int allow_hp, int force_int_mv, - int allow_ref_frame_mvs, int order_hint); - -// call for start of each sbrow per tile -void dav1d_init_ref_mv_tile_row(AV1_COMMON *cm, - int tile_col_start4, int tile_col_end4, - int row_start4, int row_end4); - -// call for each block -void dav1d_find_ref_mvs(candidate_mv *mvstack, int *cnt, mv (*mvlist)[2], - int *ctx, int refidx[2], int w4, int h4, - enum BlockSize bs, enum BlockPartition bp, - int by4, int bx4, int tile_col_start4, - int tile_col_end4, int tile_row_start4, - int tile_row_end4, AV1_COMMON *cm); - -extern const uint8_t dav1d_bs_to_sbtype[]; -extern const uint8_t dav1d_sbtype_to_bs[]; -static inline void splat_oneref_mv(refmvs *r, const ptrdiff_t stride, - const int by4, const int bx4, - const enum BlockSize bs, - const enum InterPredMode mode, - const int ref, const mv mv, - const int is_interintra) -{ - const int bw4 = dav1d_block_dimensions[bs][0]; - int bh4 = dav1d_block_dimensions[bs][1]; - - r += by4 * stride + bx4; - const refmvs tmpl = (refmvs) { - .ref = { ref + 1, is_interintra ? 0 : -1 }, - .mv = { mv }, - .sb_type = dav1d_bs_to_sbtype[bs], - .mode = N_INTRA_PRED_MODES + mode, - }; - do { - for (int x = 0; x < bw4; x++) - r[x] = tmpl; - r += stride; - } while (--bh4); -} - -static inline void splat_intrabc_mv(refmvs *r, const ptrdiff_t stride, - const int by4, const int bx4, - const enum BlockSize bs, const mv mv) -{ - const int bw4 = dav1d_block_dimensions[bs][0]; - int bh4 = dav1d_block_dimensions[bs][1]; - - r += by4 * stride + bx4; - const refmvs tmpl = (refmvs) { - .ref = { 0, -1 }, - .mv = { mv }, - .sb_type = dav1d_bs_to_sbtype[bs], - .mode = DC_PRED, - }; - do { - for (int x = 0; x < bw4; x++) - r[x] = tmpl; - r += stride; - } while (--bh4); -} - -static inline void splat_tworef_mv(refmvs *r, const ptrdiff_t stride, - const int by4, const int bx4, - const enum BlockSize bs, - const enum CompInterPredMode mode, - const int ref1, const int ref2, - const mv mv1, const mv mv2) -{ - const int bw4 = dav1d_block_dimensions[bs][0]; - int bh4 = dav1d_block_dimensions[bs][1]; - - r += by4 * stride + bx4; - const refmvs tmpl = (refmvs) { - .ref = { ref1 + 1, ref2 + 1 }, - .mv = { mv1, mv2 }, - .sb_type = dav1d_bs_to_sbtype[bs], - .mode = N_INTRA_PRED_MODES + N_INTER_PRED_MODES + mode, - }; - do { - for (int x = 0; x < bw4; x++) - r[x] = tmpl; - r += stride; - } while (--bh4); -} - -static inline void splat_intraref(refmvs *r, const ptrdiff_t stride, - const int by4, const int bx4, - const enum BlockSize bs, - const enum IntraPredMode mode) -{ - const int bw4 = dav1d_block_dimensions[bs][0]; - int bh4 = dav1d_block_dimensions[bs][1]; - - r += by4 * stride + bx4; - do { - int x; - - for (x = 0; x < bw4; x++) - r[x] = (refmvs) { - .ref = { 0, -1 }, - .mv = { [0] = { .y = -0x8000, .x = -0x8000 }, }, - .sb_type = dav1d_bs_to_sbtype[bs], - .mode = mode, - }; - r += stride; - } while (--bh4); -} - -static inline void fix_mv_precision(const Dav1dFrameHeader *const hdr, - mv *const mv) -{ - if (hdr->force_integer_mv) { - const int xmod = mv->x & 7; - mv->x &= ~7; - mv->x += (xmod > 4 - (mv->x < 0)) << 3; - const int ymod = mv->y & 7; - mv->y &= ~7; - mv->y += (ymod > 4 - (mv->y < 0)) << 3; - } else if (!hdr->hp) { - if (mv->x & 1) { - if (mv->x < 0) mv->x++; - else mv->x--; - } - if (mv->y & 1) { - if (mv->y < 0) mv->y++; - else mv->y--; - } - } -} - -#endif /* DAV1D_SRC_REF_MVS_H */ diff --git a/third_party/dav1d/src/ref_mvs.c b/third_party/dav1d/src/ref_mvs.c deleted file mode 100644 --- a/third_party/dav1d/src/ref_mvs.c +++ /dev/null @@ -1,2104 +0,0 @@ -/* - * Copyright (c) 2001-2016, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -/* - * Changes made compared to libaom version: - * - we disable TMV and enable MV_COMPRESS so that the - * input array for prev_frames can be at 4x4 instead of - * 8x8 resolution, and therefore shared between cur_frame - * and prev_frame. To make enc/dec behave consistent, we - * also make this change around line 2580: -#if 0 - AOMMIN(((mi_row >> 1) << 1) + 1 + (((xd->n8_h - 1) >> 1) << 1), - mi_row_end - 1) * - prev_frame_mvs_stride + - AOMMIN(((mi_col >> 1) << 1) + 1 + (((xd->n8_w - 1) >> 1) << 1), - mi_col_end - 1) -#else - (((mi_row >> 1) << 1) + 1) * prev_frame_mvs_stride + - (((mi_col >> 1) << 1) + 1) -#endif - * and the same change (swap mi_cols from prev_frame.mv_stride) on line 2407 - * - we disable rect-block overhanging edge inclusion (see - * line 2642): - if (num_8x8_blocks_wide == num_8x8_blocks_high || 1) { - mv_ref_search[5].row = -1; - mv_ref_search[5].col = 0; - mv_ref_search[6].row = 0; - mv_ref_search[6].col = -1; - } else { - mv_ref_search[5].row = -1; - mv_ref_search[5].col = num_8x8_blocks_wide; - mv_ref_search[6].row = num_8x8_blocks_high; - mv_ref_search[6].col = -1; - } - * Note that this is a bitstream change and needs the same - * change on the decoder side also. - * - we change xd->mi to be a pointer instead of a double ptr. - */ - -#include "config.h" - -#include -#include -#include -#include -#include -#include - -#include "dav1d/common.h" - -#include "common/intops.h" - -#define av1_zero(a) memset(a, 0, sizeof(a)) - -#define ATTRIBUTE_PACKED -#define INLINE inline -#define IMPLIES(a, b) (!(a) || (b)) // Logical 'a implies b' (or 'a -> b') - -#define ROUND_POWER_OF_TWO(value, n) (((value) + (((1 << (n)) >> 1))) >> (n)) -#define ROUND_POWER_OF_TWO_SIGNED(value, n) \ - (((value) < 0) ? -ROUND_POWER_OF_TWO(-(value), (n)) \ - : ROUND_POWER_OF_TWO((value), (n))) -#define NELEMENTS(x) (int)(sizeof(x) / sizeof(x[0])) - -#define MAX_MV_REF_CANDIDATES 2 - -#define MAX_REF_MV_STACK_SIZE 8 -#define REF_CAT_LEVEL 640 - -#define FRAME_OFFSET_BITS 5 -#define MAX_FRAME_DISTANCE ((1 << FRAME_OFFSET_BITS) - 1) -#define INVALID_MV 0x80008000 - -#define COMP_NEWMV_CTXS 5 -#define REFMV_OFFSET 4 -#define REFMV_CTX_MASK ((1 << (8 - REFMV_OFFSET)) - 1) - -#define MV_IN_USE_BITS 14 -#define MV_UPP (1 << MV_IN_USE_BITS) -#define MV_LOW (-(1 << MV_IN_USE_BITS)) - -typedef struct MV { - int16_t row; - int16_t col; -} MV; -typedef union int_mv { - uint32_t as_int; - MV as_mv; -} int_mv; -typedef int8_t MV_REFERENCE_FRAME; -#define MFMV_STACK_SIZE 3 -typedef struct { - int_mv mfmv0; - uint8_t ref_frame_offset; -} TPL_MV_REF; -typedef struct { - int_mv mv[2]; - MV_REFERENCE_FRAME ref_frame[2]; - int8_t mode, sb_type; -} MV_REF; -#define MB_MODE_INFO MV_REF - -#define AOMMAX(a,b) ((a)>(b)?(a):(b)) -#define AOMMIN(a,b) ((a)<(b)?(a):(b)) - -typedef struct candidate_mv { - int_mv this_mv; - int_mv comp_mv; - int weight; -} CANDIDATE_MV; -#define NONE_FRAME -1 -#define INTRA_FRAME 0 -#define LAST_FRAME 1 - -#define LAST2_FRAME 2 -#define LAST3_FRAME 3 -#define GOLDEN_FRAME 4 -#define BWDREF_FRAME 5 -#define ALTREF2_FRAME 6 -#define ALTREF_FRAME 7 -#define LAST_REF_FRAMES (LAST3_FRAME - LAST_FRAME + 1) - -#define INTER_REFS_PER_FRAME (ALTREF_FRAME - LAST_FRAME + 1) -#define TOTAL_REFS_PER_FRAME (ALTREF_FRAME - INTRA_FRAME + 1) - -#define FWD_REFS (GOLDEN_FRAME - LAST_FRAME + 1) -#define FWD_RF_OFFSET(ref) (ref - LAST_FRAME) -#define BWD_REFS (ALTREF_FRAME - BWDREF_FRAME + 1) -#define BWD_RF_OFFSET(ref) (ref - BWDREF_FRAME) -#define FWD_REFS (GOLDEN_FRAME - LAST_FRAME + 1) -#define SINGLE_REFS (FWD_REFS + BWD_REFS) -typedef enum ATTRIBUTE_PACKED { - LAST_LAST2_FRAMES, // { LAST_FRAME, LAST2_FRAME } - LAST_LAST3_FRAMES, // { LAST_FRAME, LAST3_FRAME } - LAST_GOLDEN_FRAMES, // { LAST_FRAME, GOLDEN_FRAME } - BWDREF_ALTREF_FRAMES, // { BWDREF_FRAME, ALTREF_FRAME } - LAST2_LAST3_FRAMES, // { LAST2_FRAME, LAST3_FRAME } - LAST2_GOLDEN_FRAMES, // { LAST2_FRAME, GOLDEN_FRAME } - LAST3_GOLDEN_FRAMES, // { LAST3_FRAME, GOLDEN_FRAME } - BWDREF_ALTREF2_FRAMES, // { BWDREF_FRAME, ALTREF2_FRAME } - ALTREF2_ALTREF_FRAMES, // { ALTREF2_FRAME, ALTREF_FRAME } - TOTAL_UNIDIR_COMP_REFS, - // NOTE: UNIDIR_COMP_REFS is the number of uni-directional reference pairs - // that are explicitly signaled. - UNIDIR_COMP_REFS = BWDREF_ALTREF_FRAMES + 1, -} UNIDIR_COMP_REF; -#define TOTAL_COMP_REFS (FWD_REFS * BWD_REFS + TOTAL_UNIDIR_COMP_REFS) -#define MODE_CTX_REF_FRAMES (TOTAL_REFS_PER_FRAME + TOTAL_COMP_REFS) - -#define GLOBALMV_OFFSET 3 -#define NEWMV_CTX_MASK ((1 << GLOBALMV_OFFSET) - 1) -#define GLOBALMV_CTX_MASK ((1 << (REFMV_OFFSET - GLOBALMV_OFFSET)) - 1) -#define MI_SIZE_LOG2 2 -#define MI_SIZE (1 << MI_SIZE_LOG2) -#define MAX_SB_SIZE_LOG2 7 -#define MAX_MIB_SIZE_LOG2 (MAX_SB_SIZE_LOG2 - MI_SIZE_LOG2) -#define MIN_MIB_SIZE_LOG2 (MIN_SB_SIZE_LOG2 - MI_SIZE_LOG2) -#define MAX_MIB_SIZE (1 << MAX_MIB_SIZE_LOG2) -#define MI_SIZE_64X64 (64 >> MI_SIZE_LOG2) -#define MI_SIZE_128X128 (128 >> MI_SIZE_LOG2) -#define REFMV_OFFSET 4 - -typedef enum ATTRIBUTE_PACKED { - BLOCK_4X4, - BLOCK_4X8, - BLOCK_8X4, - BLOCK_8X8, - BLOCK_8X16, - BLOCK_16X8, - BLOCK_16X16, - BLOCK_16X32, - BLOCK_32X16, - BLOCK_32X32, - BLOCK_32X64, - BLOCK_64X32, - BLOCK_64X64, - BLOCK_64X128, - BLOCK_128X64, - BLOCK_128X128, - BLOCK_4X16, - BLOCK_16X4, - BLOCK_8X32, - BLOCK_32X8, - BLOCK_16X64, - BLOCK_64X16, - BLOCK_32X128, - BLOCK_128X32, - BLOCK_SIZES_ALL, - BLOCK_SIZES = BLOCK_4X16, - BLOCK_INVALID = 255, - BLOCK_LARGEST = (BLOCK_SIZES - 1) -} BLOCK_SIZE; - -typedef enum ATTRIBUTE_PACKED { - PARTITION_NONE, - PARTITION_HORZ, - PARTITION_VERT, - PARTITION_SPLIT, - PARTITION_HORZ_A, // HORZ split and the top partition is split again - PARTITION_HORZ_B, // HORZ split and the bottom partition is split again - PARTITION_VERT_A, // VERT split and the left partition is split again - PARTITION_VERT_B, // VERT split and the right partition is split again - PARTITION_HORZ_4, // 4:1 horizontal partition - PARTITION_VERT_4, // 4:1 vertical partition - EXT_PARTITION_TYPES, - PARTITION_TYPES = PARTITION_SPLIT + 1, - PARTITION_INVALID = 255 -} PARTITION_TYPE; -typedef struct CUR_MODE_INFO { - PARTITION_TYPE partition; -} CUR_MODE_INFO ; - -typedef enum ATTRIBUTE_PACKED { - DC_PRED, // Average of above and left pixels - V_PRED, // Vertical - H_PRED, // Horizontal - D45_PRED, // Directional 45 deg = round(arctan(1/1) * 180/pi) - D135_PRED, // Directional 135 deg = 180 - 45 - D117_PRED, // Directional 117 deg = 180 - 63 - D153_PRED, // Directional 153 deg = 180 - 27 - D207_PRED, // Directional 207 deg = 180 + 27 - D63_PRED, // Directional 63 deg = round(arctan(2/1) * 180/pi) - SMOOTH_PRED, // Combination of horizontal and vertical interpolation - SMOOTH_V_PRED, // Vertical interpolation - SMOOTH_H_PRED, // Horizontal interpolation - PAETH_PRED, // Predict from the direction of smallest gradient - NEARESTMV, - NEARMV, - GLOBALMV, - NEWMV, - // Compound ref compound modes - NEAREST_NEARESTMV, - NEAR_NEARMV, - NEAREST_NEWMV, - NEW_NEARESTMV, - NEAR_NEWMV, - NEW_NEARMV, - GLOBAL_GLOBALMV, - NEW_NEWMV, - MB_MODE_COUNT, - INTRA_MODES = PAETH_PRED + 1, // PAETH_PRED has to be the last intra mode. - INTRA_INVALID = MB_MODE_COUNT // For uv_mode in inter blocks -} PREDICTION_MODE; -typedef enum { - IDENTITY = 0, // identity transformation, 0-parameter - TRANSLATION = 1, // translational motion 2-parameter - ROTZOOM = 2, // simplified affine with rotation + zoom only, 4-parameter - AFFINE = 3, // affine, 6-parameter - TRANS_TYPES, -} TransformationType; - -#define LEAST_SQUARES_SAMPLES_MAX_BITS 3 -#define LEAST_SQUARES_SAMPLES_MAX (1 << LEAST_SQUARES_SAMPLES_MAX_BITS) -#define SAMPLES_ARRAY_SIZE (LEAST_SQUARES_SAMPLES_MAX * 2) - -static const uint8_t mi_size_wide[BLOCK_SIZES_ALL] = { - 1, 1, 2, 2, 2, 4, 4, 4, 8, 8, 8, 16, 16, - 16, 32, 32, 1, 4, 2, 8, 4, 16, 8, 32 -}; -static const uint8_t mi_size_high[BLOCK_SIZES_ALL] = { - 1, 2, 1, 2, 4, 2, 4, 8, 4, 8, 16, 8, 16, - 32, 16, 32, 4, 1, 8, 2, 16, 4, 32, 8 -}; - -static const uint8_t block_size_wide[BLOCK_SIZES_ALL] = { - 4, 4, - 8, 8, - 8, 16, - 16, 16, - 32, 32, - 32, 64, - 64, 64, 128, 128, 4, - 16, 8, - 32, 16, - 64, 32, 128 -}; - -static const uint8_t block_size_high[BLOCK_SIZES_ALL] = { - 4, 8, - 4, 8, - 16, 8, - 16, 32, - 16, 32, - 64, 32, - 64, 128, 64, 128, 16, - 4, 32, - 8, 64, - 16, 128, 32 -}; - -static INLINE int is_global_mv_block(const MB_MODE_INFO *const mbmi, - TransformationType type) { - const PREDICTION_MODE mode = mbmi->mode; - const BLOCK_SIZE bsize = mbmi->sb_type; - const int block_size_allowed = - AOMMIN(block_size_wide[bsize], block_size_high[bsize]) >= 8; - return block_size_allowed && type > TRANSLATION && - (mode == GLOBALMV || mode == GLOBAL_GLOBALMV); -} - -typedef struct { - TransformationType wmtype; - int32_t wmmat[6]; - int16_t alpha, beta, gamma, delta; -} Dav1dWarpedMotionParams; - -#define REF_FRAMES_LOG2 3 -#define REF_FRAMES (1 << REF_FRAMES_LOG2) -#define FRAME_BUFFERS (REF_FRAMES + 7) -typedef struct { - - unsigned int cur_frame_offset; - unsigned int ref_frame_offset[INTER_REFS_PER_FRAME]; - - MV_REF *mvs; - ptrdiff_t mv_stride; - int mi_rows; - int mi_cols; - uint8_t intra_only; -} RefCntBuffer; - -#define INVALID_IDX -1 // Invalid buffer index. -typedef struct TileInfo { - int mi_row_start, mi_row_end; - int mi_col_start, mi_col_end; - int tg_horz_boundary; -} TileInfo; -typedef struct macroblockd { - TileInfo tile; - int mi_stride; - - CUR_MODE_INFO cur_mi; - MB_MODE_INFO *mi; - int up_available; - int left_available; - /* Distance of MB away from frame edges in subpixels (1/8th pixel) */ - int mb_to_left_edge; - int mb_to_right_edge; - int mb_to_top_edge; - int mb_to_bottom_edge; - // block dimension in the unit of mode_info. - uint8_t n8_w, n8_h; - uint8_t is_sec_rect; - -} MACROBLOCKD; -typedef struct RefBuffer { - int idx; // frame buf idx -} RefBuffer; -typedef struct BufferPool { - RefCntBuffer frame_bufs[FRAME_BUFFERS]; -} BufferPool; -typedef struct AV1Common { - - // TODO(hkuang): Combine this with cur_buf in macroblockd. - RefCntBuffer cur_frame; - - // Each Inter frame can reference INTER_REFS_PER_FRAME buffers - RefBuffer frame_refs[INTER_REFS_PER_FRAME]; - - int allow_high_precision_mv; - int cur_frame_force_integer_mv; // 0 the default in AOM, 1 only integer - int mi_rows; - int mi_cols; - int mi_stride; - - // Whether to use previous frame's motion vectors for prediction. - int allow_ref_frame_mvs; - - int ref_frame_sign_bias[TOTAL_REFS_PER_FRAME]; /* Two state 0, 1 */ - int frame_parallel_decode; // frame-based threading. - - unsigned int frame_offset; - - // External BufferPool passed from outside. - BufferPool buffer_pool; - - Dav1dWarpedMotionParams global_motion[TOTAL_REFS_PER_FRAME]; - struct { - BLOCK_SIZE sb_size; - int enable_order_hint; - int order_hint_bits_minus1; - } seq_params; - TPL_MV_REF *tpl_mvs; - // TODO(jingning): This can be combined with sign_bias later. - int8_t ref_frame_side[TOTAL_REFS_PER_FRAME]; - - int ref_buf_idx[INTER_REFS_PER_FRAME]; - int ref_order_hint[INTER_REFS_PER_FRAME]; -} AV1_COMMON; - -static INLINE void integer_mv_precision(MV *mv) { - int mod = (mv->row % 8); - if (mod != 0) { - mv->row -= mod; - if (abs(mod) > 4) { - if (mod > 0) { - mv->row += 8; - } else { - mv->row -= 8; - } - } - } - - mod = (mv->col % 8); - if (mod != 0) { - mv->col -= mod; - if (abs(mod) > 4) { - if (mod > 0) { - mv->col += 8; - } else { - mv->col -= 8; - } - } - } -} - -static INLINE int clamp(int value, int low, int high) { - return value < low ? low : (value > high ? high : value); -} - -static INLINE void clamp_mv(MV *mv, int min_col, int max_col, int min_row, - int max_row) { - mv->col = clamp(mv->col, min_col, max_col); - mv->row = clamp(mv->row, min_row, max_row); -} - -static INLINE int is_intrabc_block(const MB_MODE_INFO *mbmi) { - return mbmi->ref_frame[0] == INTRA_FRAME && mbmi->mv[0].as_mv.row != -0x8000; - //return mbmi->use_intrabc; -} - -static INLINE int is_inter_block(const MB_MODE_INFO *mbmi) { - if (is_intrabc_block(mbmi)) return 1; - return mbmi->ref_frame[0] > INTRA_FRAME; -} - -static INLINE MV_REFERENCE_FRAME comp_ref0(int ref_idx) { - static const MV_REFERENCE_FRAME lut[] = { - LAST_FRAME, // LAST_LAST2_FRAMES, - LAST_FRAME, // LAST_LAST3_FRAMES, - LAST_FRAME, // LAST_GOLDEN_FRAMES, - BWDREF_FRAME, // BWDREF_ALTREF_FRAMES, - LAST2_FRAME, // LAST2_LAST3_FRAMES - LAST2_FRAME, // LAST2_GOLDEN_FRAMES, - LAST3_FRAME, // LAST3_GOLDEN_FRAMES, - BWDREF_FRAME, // BWDREF_ALTREF2_FRAMES, - ALTREF2_FRAME, // ALTREF2_ALTREF_FRAMES, - }; - assert(NELEMENTS(lut) == TOTAL_UNIDIR_COMP_REFS); - return lut[ref_idx]; -} - -static INLINE MV_REFERENCE_FRAME comp_ref1(int ref_idx) { - static const MV_REFERENCE_FRAME lut[] = { - LAST2_FRAME, // LAST_LAST2_FRAMES, - LAST3_FRAME, // LAST_LAST3_FRAMES, - GOLDEN_FRAME, // LAST_GOLDEN_FRAMES, - ALTREF_FRAME, // BWDREF_ALTREF_FRAMES, - LAST3_FRAME, // LAST2_LAST3_FRAMES - GOLDEN_FRAME, // LAST2_GOLDEN_FRAMES, - GOLDEN_FRAME, // LAST3_GOLDEN_FRAMES, - ALTREF2_FRAME, // BWDREF_ALTREF2_FRAMES, - ALTREF_FRAME, // ALTREF2_ALTREF_FRAMES, - }; - assert(NELEMENTS(lut) == TOTAL_UNIDIR_COMP_REFS); - return lut[ref_idx]; -} - -#define WARPEDMODEL_PREC_BITS 16 -#define GM_TRANS_ONLY_PREC_DIFF (WARPEDMODEL_PREC_BITS - 3) -#define WARPEDMODEL_ROW3HOMO_PREC_BITS 16 - -static INLINE int convert_to_trans_prec(int allow_hp, int coor) { - if (allow_hp) - return ROUND_POWER_OF_TWO_SIGNED(coor, WARPEDMODEL_PREC_BITS - 3); - else - return ROUND_POWER_OF_TWO_SIGNED(coor, WARPEDMODEL_PREC_BITS - 2) * 2; -} - -static INLINE int block_center_x(int mi_col, BLOCK_SIZE bs) { - const int bw = block_size_wide[bs]; - return mi_col * MI_SIZE + bw / 2 - 1; -} - -static INLINE int block_center_y(int mi_row, BLOCK_SIZE bs) { - const int bh = block_size_high[bs]; - return mi_row * MI_SIZE + bh / 2 - 1; -} - -// Convert a global motion vector into a motion vector at the centre of the -// given block. -// -// The resulting motion vector will have three fractional bits of precision. If -// allow_hp is zero, the bottom bit will always be zero. If CONFIG_AMVR and -// is_integer is true, the bottom three bits will be zero (so the motion vector -// represents an integer) -static INLINE int_mv gm_get_motion_vector(const Dav1dWarpedMotionParams *gm, - int allow_hp, BLOCK_SIZE bsize, - int mi_col, int mi_row, - int is_integer) { - int_mv res; - const int32_t *mat = gm->wmmat; - int x, y, tx, ty; - - if (gm->wmtype == TRANSLATION) { - // All global motion vectors are stored with WARPEDMODEL_PREC_BITS (16) - // bits of fractional precision. The offset for a translation is stored in - // entries 0 and 1. For translations, all but the top three (two if - // cm->allow_high_precision_mv is false) fractional bits are always zero. - // - // After the right shifts, there are 3 fractional bits of precision. If - // allow_hp is false, the bottom bit is always zero (so we don't need a - // call to convert_to_trans_prec here) - res.as_mv.row = gm->wmmat[0] >> GM_TRANS_ONLY_PREC_DIFF; - res.as_mv.col = gm->wmmat[1] >> GM_TRANS_ONLY_PREC_DIFF; - assert(IMPLIES(1 & (res.as_mv.row | res.as_mv.col), allow_hp)); - if (is_integer) { - integer_mv_precision(&res.as_mv); - } - return res; - } - - x = block_center_x(mi_col, bsize); - y = block_center_y(mi_row, bsize); - - if (gm->wmtype == ROTZOOM) { - assert(gm->wmmat[5] == gm->wmmat[2]); - assert(gm->wmmat[4] == -gm->wmmat[3]); - } - if (gm->wmtype > AFFINE) { - int xc = (int)((int64_t)mat[2] * x + (int64_t)mat[3] * y + mat[0]); - int yc = (int)((int64_t)mat[4] * x + (int64_t)mat[5] * y + mat[1]); - const int Z = (int)((int64_t)mat[6] * x + (int64_t)mat[7] * y + - (1 << WARPEDMODEL_ROW3HOMO_PREC_BITS)); - xc *= 1 << (WARPEDMODEL_ROW3HOMO_PREC_BITS - WARPEDMODEL_PREC_BITS); - yc *= 1 << (WARPEDMODEL_ROW3HOMO_PREC_BITS - WARPEDMODEL_PREC_BITS); - xc = (int)(xc > 0 ? ((int64_t)xc + Z / 2) / Z : ((int64_t)xc - Z / 2) / Z); - yc = (int)(yc > 0 ? ((int64_t)yc + Z / 2) / Z : ((int64_t)yc - Z / 2) / Z); - tx = convert_to_trans_prec(allow_hp, xc) - (x << 3); - ty = convert_to_trans_prec(allow_hp, yc) - (y << 3); - } else { - const int xc = - (mat[2] - (1 << WARPEDMODEL_PREC_BITS)) * x + mat[3] * y + mat[0]; - const int yc = - mat[4] * x + (mat[5] - (1 << WARPEDMODEL_PREC_BITS)) * y + mat[1]; - tx = convert_to_trans_prec(allow_hp, xc); - ty = convert_to_trans_prec(allow_hp, yc); - } - - res.as_mv.row = ty; - res.as_mv.col = tx; - - if (is_integer) { - integer_mv_precision(&res.as_mv); - } - return res; -} - -static INLINE int have_newmv_in_inter_mode(PREDICTION_MODE mode) { - return (mode == NEWMV || mode == NEW_NEWMV || mode == NEAREST_NEWMV || - mode == NEW_NEARESTMV || mode == NEAR_NEWMV || mode == NEW_NEARMV); -} - -/* - * Copyright (c) 2016, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ -#ifndef AV1_COMMON_MVREF_COMMON_H_ -#define AV1_COMMON_MVREF_COMMON_H_ - -//#include "av1/common/onyxc_int.h" -//#include "av1/common/blockd.h" - -#ifdef __cplusplus -extern "C" { -#endif - -#define MVREF_ROW_COLS 3 - -// Set the upper limit of the motion vector component magnitude. -// This would make a motion vector fit in 26 bits. Plus 3 bits for the -// reference frame index. A tuple of motion vector can hence be stored within -// 32 bit range for efficient load/store operations. -#define REFMVS_LIMIT ((1 << 12) - 1) - -typedef struct position { - int row; - int col; -} POSITION; - -// clamp_mv_ref -#define MV_BORDER (16 << 3) // Allow 16 pels in 1/8th pel units - -static INLINE int get_relative_dist(const AV1_COMMON *cm, int a, int b) { - if (!cm->seq_params.enable_order_hint) return 0; - - const int bits = cm->seq_params.order_hint_bits_minus1 + 1; - - assert(bits >= 1); - assert(a >= 0 && a < (1 << bits)); - assert(b >= 0 && b < (1 << bits)); - - int diff = a - b; - int m = 1 << (bits - 1); - diff = (diff & (m - 1)) - (diff & m); - return diff; -} - -static INLINE void clamp_mv_ref(MV *mv, int bw, int bh, const MACROBLOCKD *xd) { - clamp_mv(mv, xd->mb_to_left_edge - bw * 8 - MV_BORDER, - xd->mb_to_right_edge + bw * 8 + MV_BORDER, - xd->mb_to_top_edge - bh * 8 - MV_BORDER, - xd->mb_to_bottom_edge + bh * 8 + MV_BORDER); -} - -// This function returns either the appropriate sub block or block's mv -// on whether the block_size < 8x8 and we have check_sub_blocks set. -static INLINE int_mv get_sub_block_mv(const MB_MODE_INFO *candidate, - int which_mv, int search_col) { - (void)search_col; - return candidate->mv[which_mv]; -} - -// Checks that the given mi_row, mi_col and search point -// are inside the borders of the tile. -static INLINE int is_inside(const TileInfo *const tile, int mi_col, int mi_row, - int mi_rows, const POSITION *mi_pos) { - const int dependent_horz_tile_flag = 0; - if (dependent_horz_tile_flag && !tile->tg_horz_boundary) { - return !(mi_row + mi_pos->row < 0 || - mi_col + mi_pos->col < tile->mi_col_start || - mi_row + mi_pos->row >= mi_rows || - mi_col + mi_pos->col >= tile->mi_col_end); - } else { - return !(mi_row + mi_pos->row < tile->mi_row_start || - mi_col + mi_pos->col < tile->mi_col_start || - mi_row + mi_pos->row >= tile->mi_row_end || - mi_col + mi_pos->col >= tile->mi_col_end); - } -} - -static INLINE int find_valid_row_offset(const TileInfo *const tile, int mi_row, - int mi_rows, int row_offset) { - const int dependent_horz_tile_flag = 0; - if (dependent_horz_tile_flag && !tile->tg_horz_boundary) - return clamp(row_offset, -mi_row, mi_rows - mi_row - 1); - else - return clamp(row_offset, tile->mi_row_start - mi_row, - tile->mi_row_end - mi_row - 1); -} - -static INLINE int find_valid_col_offset(const TileInfo *const tile, int mi_col, - int col_offset) { - return clamp(col_offset, tile->mi_col_start - mi_col, - tile->mi_col_end - mi_col - 1); -} - -static INLINE void lower_mv_precision(MV *mv, int allow_hp, - int is_integer) { - if (is_integer) { - integer_mv_precision(mv); - } else { - if (!allow_hp) { - if (mv->row & 1) mv->row += (mv->row > 0 ? -1 : 1); - if (mv->col & 1) mv->col += (mv->col > 0 ? -1 : 1); - } - } -} - -static INLINE int8_t get_uni_comp_ref_idx(const MV_REFERENCE_FRAME *const rf) { - // Single ref pred - if (rf[1] <= INTRA_FRAME) return -1; - - // Bi-directional comp ref pred - if ((rf[0] < BWDREF_FRAME) && (rf[1] >= BWDREF_FRAME)) return -1; - - for (int8_t ref_idx = 0; ref_idx < TOTAL_UNIDIR_COMP_REFS; ++ref_idx) { - if (rf[0] == comp_ref0(ref_idx) && rf[1] == comp_ref1(ref_idx)) - return ref_idx; - } - return -1; -} - -static INLINE int8_t av1_ref_frame_type(const MV_REFERENCE_FRAME *const rf) { - if (rf[1] > INTRA_FRAME) { - const int8_t uni_comp_ref_idx = get_uni_comp_ref_idx(rf); - if (uni_comp_ref_idx >= 0) { - assert((REF_FRAMES + FWD_REFS * BWD_REFS + uni_comp_ref_idx) < - MODE_CTX_REF_FRAMES); - return REF_FRAMES + FWD_REFS * BWD_REFS + uni_comp_ref_idx; - } else { - return REF_FRAMES + FWD_RF_OFFSET(rf[0]) + - BWD_RF_OFFSET(rf[1]) * FWD_REFS; - } - } - - return rf[0]; -} - -// clang-format off -static MV_REFERENCE_FRAME ref_frame_map[TOTAL_COMP_REFS][2] = { - { LAST_FRAME, BWDREF_FRAME }, { LAST2_FRAME, BWDREF_FRAME }, - { LAST3_FRAME, BWDREF_FRAME }, { GOLDEN_FRAME, BWDREF_FRAME }, - - { LAST_FRAME, ALTREF2_FRAME }, { LAST2_FRAME, ALTREF2_FRAME }, - { LAST3_FRAME, ALTREF2_FRAME }, { GOLDEN_FRAME, ALTREF2_FRAME }, - - { LAST_FRAME, ALTREF_FRAME }, { LAST2_FRAME, ALTREF_FRAME }, - { LAST3_FRAME, ALTREF_FRAME }, { GOLDEN_FRAME, ALTREF_FRAME }, - - { LAST_FRAME, LAST2_FRAME }, { LAST_FRAME, LAST3_FRAME }, - { LAST_FRAME, GOLDEN_FRAME }, { BWDREF_FRAME, ALTREF_FRAME }, - - // NOTE: Following reference frame pairs are not supported to be explicitly - // signalled, but they are possibly chosen by the use of skip_mode, - // which may use the most recent one-sided reference frame pair. - { LAST2_FRAME, LAST3_FRAME }, { LAST2_FRAME, GOLDEN_FRAME }, - { LAST3_FRAME, GOLDEN_FRAME }, {BWDREF_FRAME, ALTREF2_FRAME}, - { ALTREF2_FRAME, ALTREF_FRAME } -}; -// clang-format on - -static INLINE void av1_set_ref_frame(MV_REFERENCE_FRAME *rf, - int8_t ref_frame_type) { - if (ref_frame_type >= REF_FRAMES) { - rf[0] = ref_frame_map[ref_frame_type - REF_FRAMES][0]; - rf[1] = ref_frame_map[ref_frame_type - REF_FRAMES][1]; - } else { - rf[0] = ref_frame_type; - rf[1] = NONE_FRAME; - assert(ref_frame_type > NONE_FRAME); - } -} - -static uint16_t compound_mode_ctx_map[3][COMP_NEWMV_CTXS] = { - { 0, 1, 1, 1, 1 }, - { 1, 2, 3, 4, 4 }, - { 4, 4, 5, 6, 7 }, -}; - -static INLINE int16_t av1_mode_context_analyzer( - const int16_t *const mode_context, const MV_REFERENCE_FRAME *const rf) { - const int8_t ref_frame = av1_ref_frame_type(rf); - - if (rf[1] <= INTRA_FRAME) return mode_context[ref_frame]; - - const int16_t newmv_ctx = mode_context[ref_frame] & NEWMV_CTX_MASK; - const int16_t refmv_ctx = - (mode_context[ref_frame] >> REFMV_OFFSET) & REFMV_CTX_MASK; - - const int16_t comp_ctx = compound_mode_ctx_map[refmv_ctx >> 1][AOMMIN( - newmv_ctx, COMP_NEWMV_CTXS - 1)]; - return comp_ctx; -} - -#define INTRABC_DELAY_PIXELS 256 // Delay of 256 pixels -#define INTRABC_DELAY_SB64 (INTRABC_DELAY_PIXELS / 64) -#define USE_WAVE_FRONT 1 // Use only top left area of frame for reference. - -#ifdef __cplusplus -} // extern "C" -#endif - -#endif // AV1_COMMON_MVREF_COMMON_H_ - -/* - * Copyright (c) 2016, Alliance for Open Media. All rights reserved - * - * This source code is subject to the terms of the BSD 2 Clause License and - * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License - * was not distributed with this source code in the LICENSE file, you can - * obtain it at www.aomedia.org/license/software. If the Alliance for Open - * Media Patent License 1.0 was not distributed with this source code in the - * PATENTS file, you can obtain it at www.aomedia.org/license/patent. - */ - -#include - -//#include "av1/common/mvref_common.h" -//#include "av1/common/warped_motion.h" - -// Although we assign 32 bit integers, all the values are strictly under 14 -// bits. -static int div_mult[32] = { 0, 16384, 8192, 5461, 4096, 3276, 2730, 2340, - 2048, 1820, 1638, 1489, 1365, 1260, 1170, 1092, - 1024, 963, 910, 862, 819, 780, 744, 712, - 682, 655, 630, 606, 585, 564, 546, 528 }; - -// TODO(jingning): Consider the use of lookup table for (num / den) -// altogether. -static void get_mv_projection(MV *output, MV ref, int num, int den) { - den = AOMMIN(den, MAX_FRAME_DISTANCE); - num = num > 0 ? AOMMIN(num, MAX_FRAME_DISTANCE) - : AOMMAX(num, -MAX_FRAME_DISTANCE); - int mv_row = ROUND_POWER_OF_TWO_SIGNED(ref.row * num * div_mult[den], 14); - int mv_col = ROUND_POWER_OF_TWO_SIGNED(ref.col * num * div_mult[den], 14); - const int clamp_max = MV_UPP - 1; - const int clamp_min = MV_LOW + 1; - output->row = (int16_t)clamp(mv_row, clamp_min, clamp_max); - output->col = (int16_t)clamp(mv_col, clamp_min, clamp_max); -} - -static void add_ref_mv_candidate( - const MB_MODE_INFO *const candidate, const MV_REFERENCE_FRAME rf[2], - uint8_t *refmv_count, uint8_t *ref_match_count, uint8_t *newmv_count, - CANDIDATE_MV *ref_mv_stack, int_mv *gm_mv_candidates, - const Dav1dWarpedMotionParams *gm_params, int col, int weight) { - if (!is_inter_block(candidate)) return; // for intrabc - int index = 0, ref; - assert(weight % 2 == 0); - - if (rf[1] == NONE_FRAME) { - // single reference frame - for (ref = 0; ref < 2; ++ref) { - if (candidate->ref_frame[ref] == rf[0]) { - int_mv this_refmv; - if (is_global_mv_block(candidate, gm_params[rf[0]].wmtype)) - this_refmv = gm_mv_candidates[0]; - else - this_refmv = get_sub_block_mv(candidate, ref, col); - - for (index = 0; index < *refmv_count; ++index) - if (ref_mv_stack[index].this_mv.as_int == this_refmv.as_int) break; - - if (index < *refmv_count) ref_mv_stack[index].weight += weight; - - // Add a new item to the list. - if (index == *refmv_count && *refmv_count < MAX_REF_MV_STACK_SIZE) { - ref_mv_stack[index].this_mv = this_refmv; - ref_mv_stack[index].weight = weight; - ++(*refmv_count); - } - if (have_newmv_in_inter_mode(candidate->mode)) ++*newmv_count; - ++*ref_match_count; - } - } - } else { - // compound reference frame - if (candidate->ref_frame[0] == rf[0] && candidate->ref_frame[1] == rf[1]) { - int_mv this_refmv[2]; - - for (ref = 0; ref < 2; ++ref) { - if (is_global_mv_block(candidate, gm_params[rf[ref]].wmtype)) - this_refmv[ref] = gm_mv_candidates[ref]; - else - this_refmv[ref] = get_sub_block_mv(candidate, ref, col); - } - - for (index = 0; index < *refmv_count; ++index) - if ((ref_mv_stack[index].this_mv.as_int == this_refmv[0].as_int) && - (ref_mv_stack[index].comp_mv.as_int == this_refmv[1].as_int)) - break; - - if (index < *refmv_count) ref_mv_stack[index].weight += weight; - - // Add a new item to the list. - if (index == *refmv_count && *refmv_count < MAX_REF_MV_STACK_SIZE) { - ref_mv_stack[index].this_mv = this_refmv[0]; - ref_mv_stack[index].comp_mv = this_refmv[1]; - ref_mv_stack[index].weight = weight; - ++(*refmv_count); - } - if (have_newmv_in_inter_mode(candidate->mode)) ++*newmv_count; - ++*ref_match_count; - } - } -} - -static void scan_row_mbmi(const AV1_COMMON *cm, const MACROBLOCKD *xd, - int mi_row, int mi_col, - const MV_REFERENCE_FRAME rf[2], int row_offset, - CANDIDATE_MV *ref_mv_stack, uint8_t *refmv_count, - uint8_t *ref_match_count, uint8_t *newmv_count, - int_mv *gm_mv_candidates, int max_row_offset, - int *processed_rows) { - int end_mi = AOMMIN(xd->n8_w, cm->mi_cols - mi_col); - end_mi = AOMMIN(end_mi, mi_size_wide[BLOCK_64X64]); - const int n8_w_8 = mi_size_wide[BLOCK_8X8]; - const int n8_w_16 = mi_size_wide[BLOCK_16X16]; - int i; - int col_offset = 0; - const int shift = 0; - // TODO(jingning): Revisit this part after cb4x4 is stable. - if (abs(row_offset) > 1) { - col_offset = 1; - if ((mi_col & 0x01) && xd->n8_w < n8_w_8) --col_offset; - } - const int use_step_16 = (xd->n8_w >= 16); - MB_MODE_INFO *const candidate_mi0 = xd->mi + row_offset * xd->mi_stride; - (void)mi_row; - - for (i = 0; i < end_mi;) { - const MB_MODE_INFO *const candidate = &candidate_mi0[col_offset + i]; - const int candidate_bsize = candidate->sb_type; - const int n8_w = mi_size_wide[candidate_bsize]; - int len = AOMMIN(xd->n8_w, n8_w); - if (use_step_16) - len = AOMMAX(n8_w_16, len); - else if (abs(row_offset) > 1) - len = AOMMAX(len, n8_w_8); - - int weight = 2; - if (xd->n8_w >= n8_w_8 && xd->n8_w <= n8_w) { - int inc = AOMMIN(-max_row_offset + row_offset + 1, - mi_size_high[candidate_bsize]); - // Obtain range used in weight calculation. - weight = AOMMAX(weight, (inc << shift)); - // Update processed rows. - *processed_rows = inc - row_offset - 1; - } - - add_ref_mv_candidate(candidate, rf, refmv_count, ref_match_count, - newmv_count, ref_mv_stack, gm_mv_candidates, - cm->global_motion, col_offset + i, len * weight); - - i += len; - } -} - -static void scan_col_mbmi(const AV1_COMMON *cm, const MACROBLOCKD *xd, - int mi_row, int mi_col, - const MV_REFERENCE_FRAME rf[2], int col_offset, - CANDIDATE_MV *ref_mv_stack, uint8_t *refmv_count, - uint8_t *ref_match_count, uint8_t *newmv_count, - int_mv *gm_mv_candidates, int max_col_offset, - int *processed_cols) { - int end_mi = AOMMIN(xd->n8_h, cm->mi_rows - mi_row); - end_mi = AOMMIN(end_mi, mi_size_high[BLOCK_64X64]); - const int n8_h_8 = mi_size_high[BLOCK_8X8]; - const int n8_h_16 = mi_size_high[BLOCK_16X16]; - int i; - int row_offset = 0; - const int shift = 0; - if (abs(col_offset) > 1) { - row_offset = 1; - if ((mi_row & 0x01) && xd->n8_h < n8_h_8) --row_offset; - } - const int use_step_16 = (xd->n8_h >= 16); - (void)mi_col; - - for (i = 0; i < end_mi;) { - const MB_MODE_INFO *const candidate = - &xd->mi[(row_offset + i) * xd->mi_stride + col_offset]; - const int candidate_bsize = candidate->sb_type; - const int n8_h = mi_size_high[candidate_bsize]; - int len = AOMMIN(xd->n8_h, n8_h); - if (use_step_16) - len = AOMMAX(n8_h_16, len); - else if (abs(col_offset) > 1) - len = AOMMAX(len, n8_h_8); - - int weight = 2; - if (xd->n8_h >= n8_h_8 && xd->n8_h <= n8_h) { - int inc = AOMMIN(-max_col_offset + col_offset + 1, - mi_size_wide[candidate_bsize]); - // Obtain range used in weight calculation. - weight = AOMMAX(weight, (inc << shift)); - // Update processed cols. - *processed_cols = inc - col_offset - 1; - } - - add_ref_mv_candidate(candidate, rf, refmv_count, ref_match_count, - newmv_count, ref_mv_stack, gm_mv_candidates, - cm->global_motion, col_offset, len * weight); - - i += len; - } -} - -static void scan_blk_mbmi(const AV1_COMMON *cm, const MACROBLOCKD *xd, - const int mi_row, const int mi_col, - const MV_REFERENCE_FRAME rf[2], int row_offset, - int col_offset, CANDIDATE_MV *ref_mv_stack, - uint8_t *ref_match_count, uint8_t *newmv_count, - int_mv *gm_mv_candidates, - uint8_t refmv_count[MODE_CTX_REF_FRAMES]) { - const TileInfo *const tile = &xd->tile; - POSITION mi_pos; - - mi_pos.row = row_offset; - mi_pos.col = col_offset; - - if (is_inside(tile, mi_col, mi_row, cm->mi_rows, &mi_pos)) { - const MB_MODE_INFO *const candidate = - &xd->mi[mi_pos.row * xd->mi_stride + mi_pos.col]; - const int len = mi_size_wide[BLOCK_8X8]; - - add_ref_mv_candidate(candidate, rf, refmv_count, ref_match_count, - newmv_count, ref_mv_stack, gm_mv_candidates, - cm->global_motion, mi_pos.col, 2 * len); - } // Analyze a single 8x8 block motion information. -} - -static int has_top_right(const AV1_COMMON *cm, const MACROBLOCKD *xd, - int mi_row, int mi_col, int bs) { - const int sb_mi_size = mi_size_wide[cm->seq_params.sb_size]; - const int mask_row = mi_row & (sb_mi_size - 1); - const int mask_col = mi_col & (sb_mi_size - 1); - - if (bs > mi_size_wide[BLOCK_64X64]) return 0; - - // In a split partition all apart from the bottom right has a top right - int has_tr = !((mask_row & bs) && (mask_col & bs)); - - // bs > 0 and bs is a power of 2 - assert(bs > 0 && !(bs & (bs - 1))); - - // For each 4x4 group of blocks, when the bottom right is decoded the blocks - // to the right have not been decoded therefore the bottom right does - // not have a top right - while (bs < sb_mi_size) { - if (mask_col & bs) { - if ((mask_col & (2 * bs)) && (mask_row & (2 * bs))) { - has_tr = 0; - break; - } - } else { - break; - } - bs <<= 1; - } - - // The left hand of two vertical rectangles always has a top right (as the - // block above will have been decoded) - if (xd->n8_w < xd->n8_h) - if (!xd->is_sec_rect) has_tr = 1; - - // The bottom of two horizontal rectangles never has a top right (as the block - // to the right won't have been decoded) - if (xd->n8_w > xd->n8_h) - if (xd->is_sec_rect) has_tr = 0; - - // The bottom left square of a Vertical A (in the old format) does - // not have a top right as it is decoded before the right hand - // rectangle of the partition - if (xd->cur_mi.partition == PARTITION_VERT_A) { - if (xd->n8_w == xd->n8_h) - if (mask_row & bs) has_tr = 0; - } - - return has_tr; -} - -static int check_sb_border(const int mi_row, const int mi_col, - const int row_offset, const int col_offset) { - const int sb_mi_size = mi_size_wide[BLOCK_64X64]; - const int row = mi_row & (sb_mi_size - 1); - const int col = mi_col & (sb_mi_size - 1); - - if (row + row_offset < 0 || row + row_offset >= sb_mi_size || - col + col_offset < 0 || col + col_offset >= sb_mi_size) - return 0; - - return 1; -} - -static int add_tpl_ref_mv(const AV1_COMMON *cm, const MACROBLOCKD *xd, - int mi_row, int mi_col, MV_REFERENCE_FRAME ref_frame, - int blk_row, int blk_col, int_mv *gm_mv_candidates, - uint8_t refmv_count[MODE_CTX_REF_FRAMES], - CANDIDATE_MV ref_mv_stacks[][MAX_REF_MV_STACK_SIZE], - int16_t *mode_context) { - POSITION mi_pos; - int idx; - const int weight_unit = 1; // mi_size_wide[BLOCK_8X8]; - - mi_pos.row = (mi_row & 0x01) ? blk_row : blk_row + 1; - mi_pos.col = (mi_col & 0x01) ? blk_col : blk_col + 1; - - if (!is_inside(&xd->tile, mi_col, mi_row, cm->mi_rows, &mi_pos)) return 0; - - const TPL_MV_REF *prev_frame_mvs = - cm->tpl_mvs + ((mi_row + mi_pos.row) >> 1) * (cm->mi_stride >> 1) + - ((mi_col + mi_pos.col) >> 1); - - MV_REFERENCE_FRAME rf[2]; - av1_set_ref_frame(rf, ref_frame); - - if (rf[1] == NONE_FRAME) { - int cur_frame_index = cm->cur_frame.cur_frame_offset; - int buf_idx_0 = cm->frame_refs[FWD_RF_OFFSET(rf[0])].idx; - int frame0_index = cm->buffer_pool.frame_bufs[buf_idx_0].cur_frame_offset; - int cur_offset_0 = get_relative_dist(cm, cur_frame_index, frame0_index); - CANDIDATE_MV *ref_mv_stack = ref_mv_stacks[rf[0]]; - - if (prev_frame_mvs->mfmv0.as_int != INVALID_MV) { - int_mv this_refmv; - - get_mv_projection(&this_refmv.as_mv, prev_frame_mvs->mfmv0.as_mv, - cur_offset_0, prev_frame_mvs->ref_frame_offset); - lower_mv_precision(&this_refmv.as_mv, cm->allow_high_precision_mv, - cm->cur_frame_force_integer_mv); - - if (blk_row == 0 && blk_col == 0) - if (abs(this_refmv.as_mv.row - gm_mv_candidates[0].as_mv.row) >= 16 || - abs(this_refmv.as_mv.col - gm_mv_candidates[0].as_mv.col) >= 16) - mode_context[ref_frame] |= (1 << GLOBALMV_OFFSET); - - for (idx = 0; idx < refmv_count[rf[0]]; ++idx) - if (this_refmv.as_int == ref_mv_stack[idx].this_mv.as_int) break; - - if (idx < refmv_count[rf[0]]) ref_mv_stack[idx].weight += 2 * weight_unit; - - if (idx == refmv_count[rf[0]] && - refmv_count[rf[0]] < MAX_REF_MV_STACK_SIZE) { - ref_mv_stack[idx].this_mv.as_int = this_refmv.as_int; - ref_mv_stack[idx].weight = 2 * weight_unit; - ++(refmv_count[rf[0]]); - } - - return 1; - } - } else { - // Process compound inter mode - int cur_frame_index = cm->cur_frame.cur_frame_offset; - int buf_idx_0 = cm->frame_refs[FWD_RF_OFFSET(rf[0])].idx; - int frame0_index = cm->buffer_pool.frame_bufs[buf_idx_0].cur_frame_offset; - - int cur_offset_0 = get_relative_dist(cm, cur_frame_index, frame0_index); - int buf_idx_1 = cm->frame_refs[FWD_RF_OFFSET(rf[1])].idx; - int frame1_index = cm->buffer_pool.frame_bufs[buf_idx_1].cur_frame_offset; - int cur_offset_1 = get_relative_dist(cm, cur_frame_index, frame1_index); - CANDIDATE_MV *ref_mv_stack = ref_mv_stacks[ref_frame]; - - if (prev_frame_mvs->mfmv0.as_int != INVALID_MV) { - int_mv this_refmv; - int_mv comp_refmv; - get_mv_projection(&this_refmv.as_mv, prev_frame_mvs->mfmv0.as_mv, - cur_offset_0, prev_frame_mvs->ref_frame_offset); - get_mv_projection(&comp_refmv.as_mv, prev_frame_mvs->mfmv0.as_mv, - cur_offset_1, prev_frame_mvs->ref_frame_offset); - - lower_mv_precision(&this_refmv.as_mv, cm->allow_high_precision_mv, - cm->cur_frame_force_integer_mv); - lower_mv_precision(&comp_refmv.as_mv, cm->allow_high_precision_mv, - cm->cur_frame_force_integer_mv); - - if (blk_row == 0 && blk_col == 0) - if (abs(this_refmv.as_mv.row - gm_mv_candidates[0].as_mv.row) >= 16 || - abs(this_refmv.as_mv.col - gm_mv_candidates[0].as_mv.col) >= 16 || - abs(comp_refmv.as_mv.row - gm_mv_candidates[1].as_mv.row) >= 16 || - abs(comp_refmv.as_mv.col - gm_mv_candidates[1].as_mv.col) >= 16) - mode_context[ref_frame] |= (1 << GLOBALMV_OFFSET); - - for (idx = 0; idx < refmv_count[ref_frame]; ++idx) - if (this_refmv.as_int == ref_mv_stack[idx].this_mv.as_int && - comp_refmv.as_int == ref_mv_stack[idx].comp_mv.as_int) - break; - - if (idx < refmv_count[ref_frame]) - ref_mv_stack[idx].weight += 2 * weight_unit; - - if (idx == refmv_count[ref_frame] && - refmv_count[ref_frame] < MAX_REF_MV_STACK_SIZE) { - ref_mv_stack[idx].this_mv.as_int = this_refmv.as_int; - ref_mv_stack[idx].comp_mv.as_int = comp_refmv.as_int; - ref_mv_stack[idx].weight = 2 * weight_unit; - ++(refmv_count[ref_frame]); - } - return 1; - } - } - return 0; -} - -static void setup_ref_mv_list( - const AV1_COMMON *cm, const MACROBLOCKD *xd, MV_REFERENCE_FRAME ref_frame, - uint8_t refmv_count[MODE_CTX_REF_FRAMES], - CANDIDATE_MV ref_mv_stack[][MAX_REF_MV_STACK_SIZE], - int_mv mv_ref_list[][MAX_MV_REF_CANDIDATES], int_mv *gm_mv_candidates, - int mi_row, int mi_col, int16_t *mode_context) { - const int bs = AOMMAX(xd->n8_w, xd->n8_h); - const int has_tr = has_top_right(cm, xd, mi_row, mi_col, bs); - MV_REFERENCE_FRAME rf[2]; - - const TileInfo *const tile = &xd->tile; - int max_row_offset = 0, max_col_offset = 0; - const int row_adj = (xd->n8_h < mi_size_high[BLOCK_8X8]) && (mi_row & 0x01); - const int col_adj = (xd->n8_w < mi_size_wide[BLOCK_8X8]) && (mi_col & 0x01); - int processed_rows = 0; - int processed_cols = 0; - - av1_set_ref_frame(rf, ref_frame); - mode_context[ref_frame] = 0; - refmv_count[ref_frame] = 0; - - // Find valid maximum row/col offset. - if (xd->up_available) { - max_row_offset = -(MVREF_ROW_COLS << 1) + row_adj; - - if (xd->n8_h < mi_size_high[BLOCK_8X8]) - max_row_offset = -(2 << 1) + row_adj; - - max_row_offset = - find_valid_row_offset(tile, mi_row, cm->mi_rows, max_row_offset); - } - - if (xd->left_available) { - max_col_offset = -(MVREF_ROW_COLS << 1) + col_adj; - - if (xd->n8_w < mi_size_wide[BLOCK_8X8]) - max_col_offset = -(2 << 1) + col_adj; - - max_col_offset = find_valid_col_offset(tile, mi_col, max_col_offset); - } - - uint8_t col_match_count = 0; - uint8_t row_match_count = 0; - uint8_t newmv_count = 0; - - // Scan the first above row mode info. row_offset = -1; - if (abs(max_row_offset) >= 1) - scan_row_mbmi(cm, xd, mi_row, mi_col, rf, -1, ref_mv_stack[ref_frame], - &refmv_count[ref_frame], &row_match_count, &newmv_count, - gm_mv_candidates, max_row_offset, &processed_rows); - // Scan the first left column mode info. col_offset = -1; - if (abs(max_col_offset) >= 1) - scan_col_mbmi(cm, xd, mi_row, mi_col, rf, -1, ref_mv_stack[ref_frame], - &refmv_count[ref_frame], &col_match_count, &newmv_count, - gm_mv_candidates, max_col_offset, &processed_cols); - // Check top-right boundary - if (has_tr) - scan_blk_mbmi(cm, xd, mi_row, mi_col, rf, -1, xd->n8_w, - ref_mv_stack[ref_frame], &row_match_count, &newmv_count, - gm_mv_candidates, &refmv_count[ref_frame]); - - uint8_t nearest_match = (row_match_count > 0) + (col_match_count > 0); - uint8_t nearest_refmv_count = refmv_count[ref_frame]; - - // TODO(yunqing): for comp_search, do it for all 3 cases. - for (int idx = 0; idx < nearest_refmv_count; ++idx) - ref_mv_stack[ref_frame][idx].weight += REF_CAT_LEVEL; - - if (cm->allow_ref_frame_mvs) { - int is_available = 0; - const int voffset = AOMMAX(mi_size_high[BLOCK_8X8], xd->n8_h); - const int hoffset = AOMMAX(mi_size_wide[BLOCK_8X8], xd->n8_w); - const int blk_row_end = AOMMIN(xd->n8_h, mi_size_high[BLOCK_64X64]); - const int blk_col_end = AOMMIN(xd->n8_w, mi_size_wide[BLOCK_64X64]); - - const int tpl_sample_pos[3][2] = { - { voffset, -2 }, - { voffset, hoffset }, - { voffset - 2, hoffset }, - }; - const int allow_extension = (xd->n8_h >= mi_size_high[BLOCK_8X8]) && - (xd->n8_h < mi_size_high[BLOCK_64X64]) && - (xd->n8_w >= mi_size_wide[BLOCK_8X8]) && - (xd->n8_w < mi_size_wide[BLOCK_64X64]); - - int step_h = (xd->n8_h >= mi_size_high[BLOCK_64X64]) - ? mi_size_high[BLOCK_16X16] - : mi_size_high[BLOCK_8X8]; - int step_w = (xd->n8_w >= mi_size_wide[BLOCK_64X64]) - ? mi_size_wide[BLOCK_16X16] - : mi_size_wide[BLOCK_8X8]; - - for (int blk_row = 0; blk_row < blk_row_end; blk_row += step_h) { - for (int blk_col = 0; blk_col < blk_col_end; blk_col += step_w) { - int ret = add_tpl_ref_mv(cm, xd, mi_row, mi_col, ref_frame, blk_row, - blk_col, gm_mv_candidates, refmv_count, - ref_mv_stack, mode_context); - if (blk_row == 0 && blk_col == 0) is_available = ret; - } - } - - if (is_available == 0) mode_context[ref_frame] |= (1 << GLOBALMV_OFFSET); - - for (int i = 0; i < 3 && allow_extension; ++i) { - const int blk_row = tpl_sample_pos[i][0]; - const int blk_col = tpl_sample_pos[i][1]; - - if (!check_sb_border(mi_row, mi_col, blk_row, blk_col)) continue; - add_tpl_ref_mv(cm, xd, mi_row, mi_col, ref_frame, blk_row, blk_col, - gm_mv_candidates, refmv_count, ref_mv_stack, mode_context); - } - } - - uint8_t dummy_newmv_count = 0; - - // Scan the second outer area. - scan_blk_mbmi(cm, xd, mi_row, mi_col, rf, -1, -1, ref_mv_stack[ref_frame], - &row_match_count, &dummy_newmv_count, gm_mv_candidates, - &refmv_count[ref_frame]); - - for (int idx = 2; idx <= MVREF_ROW_COLS; ++idx) { - const int row_offset = -(idx << 1) + 1 + row_adj; - const int col_offset = -(idx << 1) + 1 + col_adj; - - if (abs(row_offset) <= abs(max_row_offset) && - abs(row_offset) > processed_rows) - scan_row_mbmi(cm, xd, mi_row, mi_col, rf, row_offset, - ref_mv_stack[ref_frame], &refmv_count[ref_frame], - &row_match_count, &dummy_newmv_count, gm_mv_candidates, - max_row_offset, &processed_rows); - - if (abs(col_offset) <= abs(max_col_offset) && - abs(col_offset) > processed_cols) - scan_col_mbmi(cm, xd, mi_row, mi_col, rf, col_offset, - ref_mv_stack[ref_frame], &refmv_count[ref_frame], - &col_match_count, &dummy_newmv_count, gm_mv_candidates, - max_col_offset, &processed_cols); - } - - uint8_t ref_match_count = (row_match_count > 0) + (col_match_count > 0); - - switch (nearest_match) { - case 0: - mode_context[ref_frame] |= 0; - if (ref_match_count >= 1) mode_context[ref_frame] |= 1; - if (ref_match_count == 1) - mode_context[ref_frame] |= (1 << REFMV_OFFSET); - else if (ref_match_count >= 2) - mode_context[ref_frame] |= (2 << REFMV_OFFSET); - break; - case 1: - mode_context[ref_frame] |= (newmv_count > 0) ? 2 : 3; - if (ref_match_count == 1) - mode_context[ref_frame] |= (3 << REFMV_OFFSET); - else if (ref_match_count >= 2) - mode_context[ref_frame] |= (4 << REFMV_OFFSET); - break; - case 2: - default: - if (newmv_count >= 1) - mode_context[ref_frame] |= 4; - else - mode_context[ref_frame] |= 5; - - mode_context[ref_frame] |= (5 << REFMV_OFFSET); - break; - } - - // Rank the likelihood and assign nearest and near mvs. - int len = nearest_refmv_count; - while (len > 0) { - int nr_len = 0; - for (int idx = 1; idx < len; ++idx) { - if (ref_mv_stack[ref_frame][idx - 1].weight < - ref_mv_stack[ref_frame][idx].weight) { - CANDIDATE_MV tmp_mv = ref_mv_stack[ref_frame][idx - 1]; - ref_mv_stack[ref_frame][idx - 1] = ref_mv_stack[ref_frame][idx]; - ref_mv_stack[ref_frame][idx] = tmp_mv; - nr_len = idx; - } - } - len = nr_len; - } - - len = refmv_count[ref_frame]; - while (len > nearest_refmv_count) { - int nr_len = nearest_refmv_count; - for (int idx = nearest_refmv_count + 1; idx < len; ++idx) { - if (ref_mv_stack[ref_frame][idx - 1].weight < - ref_mv_stack[ref_frame][idx].weight) { - CANDIDATE_MV tmp_mv = ref_mv_stack[ref_frame][idx - 1]; - ref_mv_stack[ref_frame][idx - 1] = ref_mv_stack[ref_frame][idx]; - ref_mv_stack[ref_frame][idx] = tmp_mv; - nr_len = idx; - } - } - len = nr_len; - } - - if (rf[1] > NONE_FRAME) { - // TODO(jingning, yunqing): Refactor and consolidate the compound and - // single reference frame modes. Reduce unnecessary redundancy. - if (refmv_count[ref_frame] < MAX_MV_REF_CANDIDATES) { - int_mv ref_id[2][2], ref_diff[2][2]; - int ref_id_count[2] = { 0 }, ref_diff_count[2] = { 0 }; - - int mi_width = AOMMIN(mi_size_wide[BLOCK_64X64], xd->n8_w); - mi_width = AOMMIN(mi_width, cm->mi_cols - mi_col); - int mi_height = AOMMIN(mi_size_high[BLOCK_64X64], xd->n8_h); - mi_height = AOMMIN(mi_height, cm->mi_rows - mi_row); - int mi_size = AOMMIN(mi_width, mi_height); - - for (int idx = 0; abs(max_row_offset) >= 1 && idx < mi_size;) { - const MB_MODE_INFO *const candidate = &xd->mi[-xd->mi_stride + idx]; - const int candidate_bsize = candidate->sb_type; - - for (int rf_idx = 0; rf_idx < 2; ++rf_idx) { - MV_REFERENCE_FRAME can_rf = candidate->ref_frame[rf_idx]; - - for (int cmp_idx = 0; cmp_idx < 2; ++cmp_idx) { - if (can_rf == rf[cmp_idx] && ref_id_count[cmp_idx] < 2) { - ref_id[cmp_idx][ref_id_count[cmp_idx]] = candidate->mv[rf_idx]; - ++ref_id_count[cmp_idx]; - } else if (can_rf > INTRA_FRAME && ref_diff_count[cmp_idx] < 2) { - int_mv this_mv = candidate->mv[rf_idx]; - if (cm->ref_frame_sign_bias[can_rf] != - cm->ref_frame_sign_bias[rf[cmp_idx]]) { - this_mv.as_mv.row = -this_mv.as_mv.row; - this_mv.as_mv.col = -this_mv.as_mv.col; - } - ref_diff[cmp_idx][ref_diff_count[cmp_idx]] = this_mv; - ++ref_diff_count[cmp_idx]; - } - } - } - idx += mi_size_wide[candidate_bsize]; - } - - for (int idx = 0; abs(max_col_offset) >= 1 && idx < mi_size;) { - const MB_MODE_INFO *const candidate = &xd->mi[idx * xd->mi_stride - 1]; - const int candidate_bsize = candidate->sb_type; - - for (int rf_idx = 0; rf_idx < 2; ++rf_idx) { - MV_REFERENCE_FRAME can_rf = candidate->ref_frame[rf_idx]; - - for (int cmp_idx = 0; cmp_idx < 2; ++cmp_idx) { - if (can_rf == rf[cmp_idx] && ref_id_count[cmp_idx] < 2) { - ref_id[cmp_idx][ref_id_count[cmp_idx]] = candidate->mv[rf_idx]; - ++ref_id_count[cmp_idx]; - } else if (can_rf > INTRA_FRAME && ref_diff_count[cmp_idx] < 2) { - int_mv this_mv = candidate->mv[rf_idx]; - if (cm->ref_frame_sign_bias[can_rf] != - cm->ref_frame_sign_bias[rf[cmp_idx]]) { - this_mv.as_mv.row = -this_mv.as_mv.row; - this_mv.as_mv.col = -this_mv.as_mv.col; - } - ref_diff[cmp_idx][ref_diff_count[cmp_idx]] = this_mv; - ++ref_diff_count[cmp_idx]; - } - } - } - idx += mi_size_high[candidate_bsize]; - } - - // Build up the compound mv predictor - int_mv comp_list[3][2]; - - for (int idx = 0; idx < 2; ++idx) { - int comp_idx = 0; - for (int list_idx = 0; list_idx < ref_id_count[idx] && comp_idx < 2; - ++list_idx, ++comp_idx) - comp_list[comp_idx][idx] = ref_id[idx][list_idx]; - for (int list_idx = 0; list_idx < ref_diff_count[idx] && comp_idx < 2; - ++list_idx, ++comp_idx) - comp_list[comp_idx][idx] = ref_diff[idx][list_idx]; - for (; comp_idx < 3; ++comp_idx) - comp_list[comp_idx][idx] = gm_mv_candidates[idx]; - } - - if (refmv_count[ref_frame]) { - assert(refmv_count[ref_frame] == 1); - if (comp_list[0][0].as_int == - ref_mv_stack[ref_frame][0].this_mv.as_int && - comp_list[0][1].as_int == - ref_mv_stack[ref_frame][0].comp_mv.as_int) { - ref_mv_stack[ref_frame][refmv_count[ref_frame]].this_mv = - comp_list[1][0]; - ref_mv_stack[ref_frame][refmv_count[ref_frame]].comp_mv = - comp_list[1][1]; - } else { - ref_mv_stack[ref_frame][refmv_count[ref_frame]].this_mv = - comp_list[0][0]; - ref_mv_stack[ref_frame][refmv_count[ref_frame]].comp_mv = - comp_list[0][1]; - } - ref_mv_stack[ref_frame][refmv_count[ref_frame]].weight = 2; - ++refmv_count[ref_frame]; - } else { - for (int idx = 0; idx < MAX_MV_REF_CANDIDATES; ++idx) { - ref_mv_stack[ref_frame][refmv_count[ref_frame]].this_mv = - comp_list[idx][0]; - ref_mv_stack[ref_frame][refmv_count[ref_frame]].comp_mv = - comp_list[idx][1]; - ref_mv_stack[ref_frame][refmv_count[ref_frame]].weight = 2; - ++refmv_count[ref_frame]; - } - } - } - - assert(refmv_count[ref_frame] >= 2); - - for (int idx = 0; idx < refmv_count[ref_frame]; ++idx) { - clamp_mv_ref(&ref_mv_stack[ref_frame][idx].this_mv.as_mv, - xd->n8_w << MI_SIZE_LOG2, xd->n8_h << MI_SIZE_LOG2, xd); - clamp_mv_ref(&ref_mv_stack[ref_frame][idx].comp_mv.as_mv, - xd->n8_w << MI_SIZE_LOG2, xd->n8_h << MI_SIZE_LOG2, xd); - } - } else { - // Handle single reference frame extension - int mi_width = AOMMIN(mi_size_wide[BLOCK_64X64], xd->n8_w); - mi_width = AOMMIN(mi_width, cm->mi_cols - mi_col); - int mi_height = AOMMIN(mi_size_high[BLOCK_64X64], xd->n8_h); - mi_height = AOMMIN(mi_height, cm->mi_rows - mi_row); - int mi_size = AOMMIN(mi_width, mi_height); - - for (int idx = 0; abs(max_row_offset) >= 1 && idx < mi_size && - refmv_count[ref_frame] < MAX_MV_REF_CANDIDATES;) { - const MB_MODE_INFO *const candidate = &xd->mi[-xd->mi_stride + idx]; - const int candidate_bsize = candidate->sb_type; - - // TODO(jingning): Refactor the following code. - for (int rf_idx = 0; rf_idx < 2; ++rf_idx) { - if (candidate->ref_frame[rf_idx] > INTRA_FRAME) { - int_mv this_mv = candidate->mv[rf_idx]; - if (cm->ref_frame_sign_bias[candidate->ref_frame[rf_idx]] != - cm->ref_frame_sign_bias[ref_frame]) { - this_mv.as_mv.row = -this_mv.as_mv.row; - this_mv.as_mv.col = -this_mv.as_mv.col; - } - int stack_idx; - for (stack_idx = 0; stack_idx < refmv_count[ref_frame]; ++stack_idx) { - int_mv stack_mv = ref_mv_stack[ref_frame][stack_idx].this_mv; - if (this_mv.as_int == stack_mv.as_int) break; - } - - if (stack_idx == refmv_count[ref_frame]) { - ref_mv_stack[ref_frame][stack_idx].this_mv = this_mv; - - // TODO(jingning): Set an arbitrary small number here. The weight - // doesn't matter as long as it is properly initialized. - ref_mv_stack[ref_frame][stack_idx].weight = 2; - ++refmv_count[ref_frame]; - } - } - } - idx += mi_size_wide[candidate_bsize]; - } - - for (int idx = 0; abs(max_col_offset) >= 1 && idx < mi_size && - refmv_count[ref_frame] < MAX_MV_REF_CANDIDATES;) { - const MB_MODE_INFO *const candidate = &xd->mi[idx * xd->mi_stride - 1]; - const int candidate_bsize = candidate->sb_type; - - // TODO(jingning): Refactor the following code. - for (int rf_idx = 0; rf_idx < 2; ++rf_idx) { - if (candidate->ref_frame[rf_idx] > INTRA_FRAME) { - int_mv this_mv = candidate->mv[rf_idx]; - if (cm->ref_frame_sign_bias[candidate->ref_frame[rf_idx]] != - cm->ref_frame_sign_bias[ref_frame]) { - this_mv.as_mv.row = -this_mv.as_mv.row; - this_mv.as_mv.col = -this_mv.as_mv.col; - } - int stack_idx; - for (stack_idx = 0; stack_idx < refmv_count[ref_frame]; ++stack_idx) { - int_mv stack_mv = ref_mv_stack[ref_frame][stack_idx].this_mv; - if (this_mv.as_int == stack_mv.as_int) break; - } - - if (stack_idx == refmv_count[ref_frame]) { - ref_mv_stack[ref_frame][stack_idx].this_mv = this_mv; - - // TODO(jingning): Set an arbitrary small number here. The weight - // doesn't matter as long as it is properly initialized. - ref_mv_stack[ref_frame][stack_idx].weight = 2; - ++refmv_count[ref_frame]; - } - } - } - idx += mi_size_high[candidate_bsize]; - } - - for (int idx = 0; idx < refmv_count[ref_frame]; ++idx) { - clamp_mv_ref(&ref_mv_stack[ref_frame][idx].this_mv.as_mv, - xd->n8_w << MI_SIZE_LOG2, xd->n8_h << MI_SIZE_LOG2, xd); - } - - if (mv_ref_list != NULL) { - for (int idx = refmv_count[ref_frame]; idx < MAX_MV_REF_CANDIDATES; ++idx) - mv_ref_list[rf[0]][idx].as_int = gm_mv_candidates[0].as_int; - - for (int idx = 0; - idx < AOMMIN(MAX_MV_REF_CANDIDATES, refmv_count[ref_frame]); ++idx) { - mv_ref_list[rf[0]][idx].as_int = - ref_mv_stack[ref_frame][idx].this_mv.as_int; - } - } - } -} - -static void av1_find_mv_refs(const AV1_COMMON *cm, const MACROBLOCKD *xd, - MB_MODE_INFO *mi, MV_REFERENCE_FRAME ref_frame, - uint8_t ref_mv_count[MODE_CTX_REF_FRAMES], - CANDIDATE_MV ref_mv_stack[][MAX_REF_MV_STACK_SIZE], - int_mv mv_ref_list[][MAX_MV_REF_CANDIDATES], - int_mv *global_mvs, int mi_row, int mi_col, - int16_t *mode_context) { - int_mv zeromv[2]; - BLOCK_SIZE bsize = mi->sb_type; - MV_REFERENCE_FRAME rf[2]; - av1_set_ref_frame(rf, ref_frame); - - if (ref_frame < REF_FRAMES) { - if (ref_frame != INTRA_FRAME) { - global_mvs[ref_frame] = gm_get_motion_vector( - &cm->global_motion[ref_frame], cm->allow_high_precision_mv, bsize, - mi_col, mi_row, cm->cur_frame_force_integer_mv); - } else { - global_mvs[ref_frame].as_int = INVALID_MV; - } - } - - if (ref_frame != INTRA_FRAME) { - zeromv[0].as_int = - gm_get_motion_vector(&cm->global_motion[rf[0]], - cm->allow_high_precision_mv, bsize, mi_col, mi_row, - cm->cur_frame_force_integer_mv) - .as_int; - zeromv[1].as_int = - (rf[1] != NONE_FRAME) - ? gm_get_motion_vector(&cm->global_motion[rf[1]], - cm->allow_high_precision_mv, bsize, mi_col, - mi_row, cm->cur_frame_force_integer_mv) - .as_int - : 0; - } else { - zeromv[0].as_int = zeromv[1].as_int = 0; - } - - setup_ref_mv_list(cm, xd, ref_frame, ref_mv_count, ref_mv_stack, mv_ref_list, - zeromv, mi_row, mi_col, mode_context); -} - -static void av1_setup_frame_buf_refs(AV1_COMMON *cm) { - cm->cur_frame.cur_frame_offset = cm->frame_offset; - - MV_REFERENCE_FRAME ref_frame; - for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) { - const int buf_idx = cm->frame_refs[ref_frame - LAST_FRAME].idx; - if (buf_idx >= 0) - cm->cur_frame.ref_frame_offset[ref_frame - LAST_FRAME] = - cm->buffer_pool.frame_bufs[buf_idx].cur_frame_offset; - } -} - -#define MAX_OFFSET_WIDTH 64 -#define MAX_OFFSET_HEIGHT 0 - -static int get_block_position(AV1_COMMON *cm, int *mi_r, int *mi_c, int blk_row, - int blk_col, MV mv, int sign_bias) { - const int base_blk_row = (blk_row >> 3) << 3; - const int base_blk_col = (blk_col >> 3) << 3; - - const int row_offset = (mv.row >= 0) ? (mv.row >> (4 + MI_SIZE_LOG2)) - : -((-mv.row) >> (4 + MI_SIZE_LOG2)); - - const int col_offset = (mv.col >= 0) ? (mv.col >> (4 + MI_SIZE_LOG2)) - : -((-mv.col) >> (4 + MI_SIZE_LOG2)); - - int row = (sign_bias == 1) ? blk_row - row_offset : blk_row + row_offset; - int col = (sign_bias == 1) ? blk_col - col_offset : blk_col + col_offset; - - if (row < 0 || row >= (cm->mi_rows >> 1) || col < 0 || - col >= (cm->mi_cols >> 1)) - return 0; - - if (row < base_blk_row - (MAX_OFFSET_HEIGHT >> 3) || - row >= base_blk_row + 8 + (MAX_OFFSET_HEIGHT >> 3) || - col < base_blk_col - (MAX_OFFSET_WIDTH >> 3) || - col >= base_blk_col + 8 + (MAX_OFFSET_WIDTH >> 3)) - return 0; - - *mi_r = row; - *mi_c = col; - - return 1; -} - -static int motion_field_projection(AV1_COMMON *cm, MV_REFERENCE_FRAME ref_frame, - int dir, - const int from_x4, const int to_x4, - const int from_y4, const int to_y4) { - TPL_MV_REF *tpl_mvs_base = cm->tpl_mvs; - int ref_offset[TOTAL_REFS_PER_FRAME] = { 0 }; - int ref_sign[TOTAL_REFS_PER_FRAME] = { 0 }; - - (void)dir; - - int ref_frame_idx = cm->frame_refs[FWD_RF_OFFSET(ref_frame)].idx; - if (ref_frame_idx < 0) return 0; - - if (cm->buffer_pool.frame_bufs[ref_frame_idx].intra_only) return 0; - - if (cm->buffer_pool.frame_bufs[ref_frame_idx].mi_rows != cm->mi_rows || - cm->buffer_pool.frame_bufs[ref_frame_idx].mi_cols != cm->mi_cols) - return 0; - - int ref_frame_index = - cm->buffer_pool.frame_bufs[ref_frame_idx].cur_frame_offset; - unsigned int *ref_rf_idx = - &cm->buffer_pool.frame_bufs[ref_frame_idx].ref_frame_offset[0]; - int cur_frame_index = cm->cur_frame.cur_frame_offset; - int ref_to_cur = get_relative_dist(cm, ref_frame_index, cur_frame_index); - - for (MV_REFERENCE_FRAME rf = LAST_FRAME; rf <= INTER_REFS_PER_FRAME; ++rf) { - ref_offset[rf] = - get_relative_dist(cm, ref_frame_index, ref_rf_idx[rf - LAST_FRAME]); - // note the inverted sign - ref_sign[rf] = - get_relative_dist(cm, ref_rf_idx[rf - LAST_FRAME], ref_frame_index) < 0; - } - - if (dir == 2) ref_to_cur = -ref_to_cur; - - MV_REF *mv_ref_base = cm->buffer_pool.frame_bufs[ref_frame_idx].mvs; - const ptrdiff_t mv_stride = - cm->buffer_pool.frame_bufs[ref_frame_idx].mv_stride; - const int mvs_rows = (cm->mi_rows + 1) >> 1; - const int mvs_cols = (cm->mi_cols + 1) >> 1; - - assert(from_y4 >= 0); - const int row_start8 = from_y4 >> 1; - const int row_end8 = imin(to_y4 >> 1, mvs_rows); - const int col_start8 = imax((from_x4 - (MAX_OFFSET_WIDTH >> 2)) >> 1, 0); - const int col_end8 = imin((to_x4 + (MAX_OFFSET_WIDTH >> 2)) >> 1, mvs_cols); - for (int blk_row = row_start8; blk_row < row_end8; ++blk_row) { - for (int blk_col = col_start8; blk_col < col_end8; ++blk_col) { - MV_REF *mv_ref = &mv_ref_base[((blk_row << 1) + 1) * mv_stride + - (blk_col << 1) + 1]; - int diridx; - const int ref0 = mv_ref->ref_frame[0], ref1 = mv_ref->ref_frame[1]; - if (ref1 > 0 && ref_sign[ref1] && - abs(mv_ref->mv[1].as_mv.row) < (1 << 12) && - abs(mv_ref->mv[1].as_mv.col) < (1 << 12)) - { - diridx = 1; - } else if (ref0 > 0 && ref_sign[ref0] && - abs(mv_ref->mv[0].as_mv.row) < (1 << 12) && - abs(mv_ref->mv[0].as_mv.col) < (1 << 12)) - { - diridx = 0; - } else { - continue; - } - MV fwd_mv = mv_ref->mv[diridx].as_mv; - - if (mv_ref->ref_frame[diridx] > INTRA_FRAME) { - int_mv this_mv; - int mi_r, mi_c; - const int ref_frame_offset = ref_offset[mv_ref->ref_frame[diridx]]; - - int pos_valid = abs(ref_frame_offset) <= MAX_FRAME_DISTANCE && - ref_frame_offset > 0 && - abs(ref_to_cur) <= MAX_FRAME_DISTANCE; - - if (pos_valid) { - get_mv_projection(&this_mv.as_mv, fwd_mv, ref_to_cur, - ref_frame_offset); - pos_valid = get_block_position(cm, &mi_r, &mi_c, blk_row, blk_col, - this_mv.as_mv, dir >> 1); - } - - if (pos_valid && mi_c >= (from_x4 >> 1) && mi_c < (to_x4 >> 1)) { - int mi_offset = mi_r * (cm->mi_stride >> 1) + mi_c; - - tpl_mvs_base[mi_offset].mfmv0.as_mv.row = fwd_mv.row; - tpl_mvs_base[mi_offset].mfmv0.as_mv.col = fwd_mv.col; - tpl_mvs_base[mi_offset].ref_frame_offset = ref_frame_offset; - } - } - } - } - - return 1; -} - -static void av1_setup_motion_field(AV1_COMMON *cm) { - if (!cm->seq_params.enable_order_hint) return; - - TPL_MV_REF *tpl_mvs_base = cm->tpl_mvs; - int size = (((cm->mi_rows + 31) & ~31) >> 1) * (cm->mi_stride >> 1); - for (int idx = 0; idx < size; ++idx) { - tpl_mvs_base[idx].mfmv0.as_int = INVALID_MV; - tpl_mvs_base[idx].ref_frame_offset = 0; - } - - memset(cm->ref_frame_side, 0, sizeof(cm->ref_frame_side)); - RefCntBuffer *const frame_bufs = cm->buffer_pool.frame_bufs; - - const int cur_order_hint = cm->cur_frame.cur_frame_offset; - int *const ref_buf_idx = cm->ref_buf_idx; - int *const ref_order_hint = cm->ref_order_hint; - - for (int ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ref_frame++) { - const int ref_idx = ref_frame - LAST_FRAME; - const int buf_idx = cm->frame_refs[ref_idx].idx; - int order_hint = 0; - - if (buf_idx >= 0) order_hint = frame_bufs[buf_idx].cur_frame_offset; - - ref_buf_idx[ref_idx] = buf_idx; - ref_order_hint[ref_idx] = order_hint; - - if (get_relative_dist(cm, order_hint, cur_order_hint) > 0) - cm->ref_frame_side[ref_frame] = 1; - else if (order_hint == cur_order_hint) - cm->ref_frame_side[ref_frame] = -1; - } -} - -enum BlockSize { - BS_128x128, - BS_128x64, - BS_64x128, - BS_64x64, - BS_64x32, - BS_64x16, - BS_32x64, - BS_32x32, - BS_32x16, - BS_32x8, - BS_16x64, - BS_16x32, - BS_16x16, - BS_16x8, - BS_16x4, - BS_8x32, - BS_8x16, - BS_8x8, - BS_8x4, - BS_4x16, - BS_4x8, - BS_4x4, - N_BS_SIZES, -}; -extern const uint8_t dav1d_block_dimensions[N_BS_SIZES][4]; -const uint8_t dav1d_bs_to_sbtype[N_BS_SIZES] = { - [BS_128x128] = BLOCK_128X128, - [BS_128x64] = BLOCK_128X64, - [BS_64x128] = BLOCK_64X128, - [BS_64x64] = BLOCK_64X64, - [BS_64x32] = BLOCK_64X32, - [BS_64x16] = BLOCK_64X16, - [BS_32x64] = BLOCK_32X64, - [BS_32x32] = BLOCK_32X32, - [BS_32x16] = BLOCK_32X16, - [BS_32x8] = BLOCK_32X8, - [BS_16x64] = BLOCK_16X64, - [BS_16x32] = BLOCK_16X32, - [BS_16x16] = BLOCK_16X16, - [BS_16x8] = BLOCK_16X8, - [BS_16x4] = BLOCK_16X4, - [BS_8x32] = BLOCK_8X32, - [BS_8x16] = BLOCK_8X16, - [BS_8x8] = BLOCK_8X8, - [BS_8x4] = BLOCK_8X4, - [BS_4x16] = BLOCK_4X16, - [BS_4x8] = BLOCK_4X8, - [BS_4x4] = BLOCK_4X4, -}; -const uint8_t dav1d_sbtype_to_bs[BLOCK_SIZES_ALL] = { - [BLOCK_128X128] = BS_128x128, - [BLOCK_128X64] = BS_128x64, - [BLOCK_64X128] = BS_64x128, - [BLOCK_64X64] = BS_64x64, - [BLOCK_64X32] = BS_64x32, - [BLOCK_64X16] = BS_64x16, - [BLOCK_32X64] = BS_32x64, - [BLOCK_32X32] = BS_32x32, - [BLOCK_32X16] = BS_32x16, - [BLOCK_32X8] = BS_32x8, - [BLOCK_16X64] = BS_16x64, - [BLOCK_16X32] = BS_16x32, - [BLOCK_16X16] = BS_16x16, - [BLOCK_16X8] = BS_16x8, - [BLOCK_16X4] = BS_16x4, - [BLOCK_8X32] = BS_8x32, - [BLOCK_8X16] = BS_8x16, - [BLOCK_8X8] = BS_8x8, - [BLOCK_8X4] = BS_8x4, - [BLOCK_4X16] = BS_4x16, - [BLOCK_4X8] = BS_4x8, - [BLOCK_4X4] = BS_4x4, -}; - -#include - -void dav1d_find_ref_mvs(CANDIDATE_MV *mvstack, int *cnt, int_mv (*mvlist)[2], - int *ctx, int refidx_dav1d[2], - int w4, int h4, int bs, int bp, int by4, int bx4, - int tile_col_start4, int tile_col_end4, - int tile_row_start4, int tile_row_end4, - AV1_COMMON *cm); -void dav1d_find_ref_mvs(CANDIDATE_MV *mvstack, int *cnt, int_mv (*mvlist)[2], - int *ctx, int refidx_dav1d[2], - int w4, int h4, int bs, int bp, int by4, int bx4, - int tile_col_start4, int tile_col_end4, - int tile_row_start4, int tile_row_end4, - AV1_COMMON *cm) -{ - const int bw4 = dav1d_block_dimensions[bs][0]; - const int bh4 = dav1d_block_dimensions[bs][1]; - int stride = (int) cm->cur_frame.mv_stride; - MACROBLOCKD xd = (MACROBLOCKD) { - .n8_w = bw4, - .n8_h = bh4, - .mi_stride = stride, - .up_available = by4 > tile_row_start4, - .left_available = bx4 > tile_col_start4, - .tile = { - .mi_col_end = AOMMIN(w4, tile_col_end4), - .mi_row_end = AOMMIN(h4, tile_row_end4), - .tg_horz_boundary = 0, - .mi_row_start = tile_row_start4, - .mi_col_start = tile_col_start4, - }, - .mi = (MB_MODE_INFO *) &cm->cur_frame.mvs[by4 * stride + bx4], - .mb_to_bottom_edge = (h4 - bh4 - by4) * 32, - .mb_to_left_edge = -bx4 * 32, - .mb_to_right_edge = (w4 - bw4 - bx4) * 32, - .mb_to_top_edge = -by4 * 32, - .is_sec_rect = 0, - .cur_mi = { - .partition = bp, - }, - }; - xd.mi->sb_type = dav1d_bs_to_sbtype[bs]; - if (xd.n8_w < xd.n8_h) { - // Only mark is_sec_rect as 1 for the last block. - // For PARTITION_VERT_4, it would be (0, 0, 0, 1); - // For other partitions, it would be (0, 1). - if (!((bx4 + xd.n8_w) & (xd.n8_h - 1))) xd.is_sec_rect = 1; - } - - if (xd.n8_w > xd.n8_h) - if (by4 & (xd.n8_w - 1)) xd.is_sec_rect = 1; - - MV_REFERENCE_FRAME rf[2] = { refidx_dav1d[0] + 1, refidx_dav1d[1] + 1 }; - const int refidx = av1_ref_frame_type(rf); - int16_t single_context[MODE_CTX_REF_FRAMES]; - uint8_t mv_cnt[MODE_CTX_REF_FRAMES]; - CANDIDATE_MV mv_stack[MODE_CTX_REF_FRAMES][MAX_REF_MV_STACK_SIZE]; - int_mv mv_list[MODE_CTX_REF_FRAMES][MAX_MV_REF_CANDIDATES]; - int_mv gmvs[MODE_CTX_REF_FRAMES]; - av1_find_mv_refs(cm, &xd, xd.mi, refidx, mv_cnt, - mv_stack, mv_list, gmvs, by4, bx4, - single_context); - for (int i = 0; i < mv_cnt[refidx]; i++) - mvstack[i] = mv_stack[refidx][i]; - *cnt = mv_cnt[refidx]; - - mvlist[0][0] = mv_list[refidx_dav1d[0] + 1][0]; - mvlist[0][1] = mv_list[refidx_dav1d[0] + 1][1]; - if (refidx_dav1d[1] != -1) { - mvlist[1][0] = mv_list[refidx_dav1d[1] + 1][0]; - mvlist[1][1] = mv_list[refidx_dav1d[1] + 1][1]; - } - - if (ctx) { - if (refidx_dav1d[1] == -1) - *ctx = single_context[refidx_dav1d[0] + 1]; - else - *ctx = av1_mode_context_analyzer(single_context, rf); - } -} - -int dav1d_init_ref_mv_common(AV1_COMMON *cm, const int w8, const int h8, - const ptrdiff_t stride, const int allow_sb128, - MV_REF *cur, MV_REF *ref_mvs[7], - const unsigned cur_poc, - const unsigned ref_poc[7], - const unsigned ref_ref_poc[7][7], - const Dav1dWarpedMotionParams gmv[7], - const int allow_hp, const int force_int_mv, - const int allow_ref_frame_mvs, - const int order_hint); -int dav1d_init_ref_mv_common(AV1_COMMON *cm, const int w8, const int h8, - const ptrdiff_t stride, const int allow_sb128, - MV_REF *cur, MV_REF *ref_mvs[7], - const unsigned cur_poc, - const unsigned ref_poc[7], - const unsigned ref_ref_poc[7][7], - const Dav1dWarpedMotionParams gmv[7], - const int allow_hp, const int force_int_mv, - const int allow_ref_frame_mvs, - const int order_hint) -{ - if (cm->mi_cols != (w8 << 1) || cm->mi_rows != (h8 << 1)) { - const int align_h = (h8 + 15) & ~15; - if (cm->tpl_mvs) free(cm->tpl_mvs); - cm->tpl_mvs = malloc(sizeof(*cm->tpl_mvs) * (stride >> 1) * align_h); - if (!cm->tpl_mvs) { - cm->mi_cols = cm->mi_rows = 0; - return DAV1D_ERR(ENOMEM); - } - for (int i = 0; i < 7; i++) - cm->frame_refs[i].idx = i; - cm->mi_cols = w8 << 1; - cm->mi_rows = h8 << 1; - cm->mi_stride = (int) stride; - for (int i = 0; i < 7; i++) { - cm->buffer_pool.frame_bufs[i].mi_rows = cm->mi_rows; - cm->buffer_pool.frame_bufs[i].mi_cols = cm->mi_cols; - cm->buffer_pool.frame_bufs[i].mv_stride = stride; - } - cm->cur_frame.mv_stride = stride; - } - - cm->allow_high_precision_mv = allow_hp; - cm->seq_params.sb_size = allow_sb128 ? BLOCK_128X128 : BLOCK_64X64; - - cm->seq_params.enable_order_hint = !!order_hint; - cm->seq_params.order_hint_bits_minus1 = order_hint - 1; - // FIXME get these from the sequence/frame headers instead of hardcoding - cm->frame_parallel_decode = 0; - cm->cur_frame_force_integer_mv = force_int_mv; - - memcpy(&cm->global_motion[1], gmv, sizeof(*gmv) * 7); - - cm->frame_offset = cur_poc; - cm->allow_ref_frame_mvs = allow_ref_frame_mvs; - cm->cur_frame.mvs = cur; - for (int i = 0; i < 7; i++) { - cm->buffer_pool.frame_bufs[i].mvs = ref_mvs[i]; - cm->buffer_pool.frame_bufs[i].intra_only = ref_mvs[i] == NULL; - cm->buffer_pool.frame_bufs[i].cur_frame_offset = ref_poc[i]; - for (int j = 0; j < 7; j++) - cm->buffer_pool.frame_bufs[i].ref_frame_offset[j] = - ref_ref_poc[i][j]; - } - av1_setup_frame_buf_refs(cm); - for (int i = 0; i < 7; i++) { - const int ref_poc = cm->buffer_pool.frame_bufs[i].cur_frame_offset; - cm->ref_frame_sign_bias[1 + i] = get_relative_dist(cm, ref_poc, cur_poc) > 0; - } - if (allow_ref_frame_mvs) { - av1_setup_motion_field(cm); - } - - return 0; -} - -void dav1d_init_ref_mv_tile_row(AV1_COMMON *cm, - int tile_col_start4, int tile_col_end4, - int row_start4, int row_end4); -void dav1d_init_ref_mv_tile_row(AV1_COMMON *cm, - int tile_col_start4, int tile_col_end4, - int row_start4, int row_end4) -{ - RefCntBuffer *const frame_bufs = cm->buffer_pool.frame_bufs; - const int cur_order_hint = cm->cur_frame.cur_frame_offset; - int *const ref_buf_idx = cm->ref_buf_idx; - int *const ref_order_hint = cm->ref_order_hint; - - int ref_stamp = MFMV_STACK_SIZE - 1; - - if (ref_buf_idx[LAST_FRAME - LAST_FRAME] >= 0) { - const int alt_of_lst_order_hint = - frame_bufs[ref_buf_idx[LAST_FRAME - LAST_FRAME]] - .ref_frame_offset[ALTREF_FRAME - LAST_FRAME]; - - const int is_lst_overlay = - (alt_of_lst_order_hint == ref_order_hint[GOLDEN_FRAME - LAST_FRAME]); - if (!is_lst_overlay) motion_field_projection(cm, LAST_FRAME, 2, - tile_col_start4, tile_col_end4, - row_start4, row_end4); - --ref_stamp; - } - - if (get_relative_dist(cm, ref_order_hint[BWDREF_FRAME - LAST_FRAME], - cur_order_hint) > 0) { - if (motion_field_projection(cm, BWDREF_FRAME, 0, - tile_col_start4, tile_col_end4, - row_start4, row_end4)) --ref_stamp; - } - - if (get_relative_dist(cm, ref_order_hint[ALTREF2_FRAME - LAST_FRAME], - cur_order_hint) > 0) { - if (motion_field_projection(cm, ALTREF2_FRAME, 0, - tile_col_start4, tile_col_end4, - row_start4, row_end4)) --ref_stamp; - } - - if (get_relative_dist(cm, ref_order_hint[ALTREF_FRAME - LAST_FRAME], - cur_order_hint) > 0 && - ref_stamp >= 0) - if (motion_field_projection(cm, ALTREF_FRAME, 0, - tile_col_start4, tile_col_end4, - row_start4, row_end4)) --ref_stamp; - - if (ref_stamp >= 0 && ref_buf_idx[LAST2_FRAME - LAST_FRAME] >= 0) - if (motion_field_projection(cm, LAST2_FRAME, 2, - tile_col_start4, tile_col_end4, - row_start4, row_end4)) --ref_stamp; -} - -AV1_COMMON *dav1d_alloc_ref_mv_common(void); -AV1_COMMON *dav1d_alloc_ref_mv_common(void) { - return calloc(1, sizeof(AV1_COMMON)); -} - -void dav1d_free_ref_mv_common(AV1_COMMON *cm); -void dav1d_free_ref_mv_common(AV1_COMMON *cm) { - if (cm->tpl_mvs) free(cm->tpl_mvs); - free(cm); -} diff --git a/third_party/dav1d/src/refmvs.h b/third_party/dav1d/src/refmvs.h new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/refmvs.h @@ -0,0 +1,233 @@ +/* + * Copyright © 2020, VideoLAN and dav1d authors + * Copyright © 2020, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef DAV1D_SRC_REF_MVS_H +#define DAV1D_SRC_REF_MVS_H + +#include + +#include "dav1d/headers.h" + +#include "common/intops.h" + +#include "src/intra_edge.h" +#include "src/levels.h" +#include "src/tables.h" + +#define INVALID_MV 0x80008000 + +typedef struct refmvs_temporal_block { + mv mv; + int8_t ref; +} refmvs_temporal_block; + +typedef union refmvs_refpair { + int8_t ref[2]; // [0] = 0: intra=1, [1] = -1: comp=0 + uint16_t pair; +} refmvs_refpair; + +typedef union refmvs_mvpair { + mv mv[2]; + uint64_t n; +} refmvs_mvpair; + +typedef struct refmvs_block { + refmvs_mvpair mv; + refmvs_refpair ref; + uint8_t bs, mf; // 1 = globalmv+affine, 2 = newmv +} refmvs_block; + +typedef struct refmvs_frame { + const Dav1dFrameHeader *frm_hdr; + int iw4, ih4, iw8, ih8; + int sbsz; + int use_ref_frame_mvs; + uint8_t sign_bias[7], mfmv_sign[7]; + int8_t pocdiff[7]; + uint8_t mfmv_ref[3]; + int mfmv_ref2cur[3]; + int mfmv_ref2ref[3][7]; + int n_mfmvs; + + refmvs_temporal_block *rp; + /*const*/ refmvs_temporal_block *const *rp_ref; + refmvs_temporal_block *rp_proj; + ptrdiff_t rp_stride; + + refmvs_block *r; // 35 x r_stride memory + ptrdiff_t r_stride; + int n_tile_rows, n_tile_threads; +} refmvs_frame; + +typedef struct refmvs_tile { + const refmvs_frame *rf; + refmvs_block *r[32 + 5]; + refmvs_temporal_block *rp_proj; + struct { + int start, end; + } tile_col, tile_row; +} refmvs_tile; + +typedef struct refmvs_candidate { + refmvs_mvpair mv; + int weight; +} refmvs_candidate; + +// call once per frame thread +void dav1d_refmvs_init(refmvs_frame *rf); +void dav1d_refmvs_clear(refmvs_frame *rf); + +// call once per frame +int dav1d_refmvs_init_frame(refmvs_frame *rf, + const Dav1dSequenceHeader *seq_hdr, + const Dav1dFrameHeader *frm_hdr, + const unsigned ref_poc[7], + refmvs_temporal_block *rp, + const unsigned ref_ref_poc[7][7], + /*const*/ refmvs_temporal_block *const rp_ref[7], + int n_tile_threads); + +// initialize temporal MVs; this can be done in any configuration, e.g. one +// tile/sbrow at a time, where col_{start,end}8 are the tile boundaries; or +// it can just be for the whole frame's sbrow, where col_{start,end}8 are the +// frame boundaries. row_{start,end}8 are the superblock row boundaries. +void dav1d_refmvs_load_tmvs(const refmvs_frame *rf, int tile_row_idx, + int col_start8, int col_end8, + int row_start8, int row_end8); + +// cache the current tile/sbrow (or frame/sbrow)'s projectable motion vectors +// into buffers for use in future frame's temporal MV prediction +void dav1d_refmvs_save_tmvs(const refmvs_tile *rt, + int col_start8, int col_end8, + int row_start8, int row_end8); + +// initialize tile boundaries and refmvs_block pointers for one tile/sbrow +void dav1d_refmvs_tile_sbrow_init(refmvs_tile *rt, const refmvs_frame *rf, + int tile_col_start4, int tile_col_end4, + int tile_row_start4, int tile_row_end4, + int sby, int tile_row_idx); + +// call for each block +void dav1d_refmvs_find(const refmvs_tile *rt, + refmvs_candidate mvstack[8], int *cnt, + int *ctx, const refmvs_refpair ref, enum BlockSize bs, + enum EdgeFlags edge_flags, int by4, int bx4); + +static inline void splat_oneref_mv(refmvs_tile *const rt, + const int by4, const int bx4, + const enum BlockSize bs, + const enum InterPredMode mode, + const int ref, const mv mv, + const int is_interintra) +{ + const int bw4 = dav1d_block_dimensions[bs][0]; + int bh4 = dav1d_block_dimensions[bs][1]; + refmvs_block **rr = &rt->r[(by4 & 31) + 5]; + + const refmvs_block tmpl = (refmvs_block) { + .ref.ref = { ref + 1, is_interintra ? 0 : -1 }, + .mv.mv[0] = mv, + .bs = bs, + .mf = (mode == GLOBALMV && imin(bw4, bh4) >= 2) | ((mode == NEWMV) * 2), + }; + do { + refmvs_block *r = *rr++ + bx4; + for (int x = 0; x < bw4; x++) + r[x] = tmpl; + } while (--bh4); +} + +static inline void splat_intrabc_mv(refmvs_tile *const rt, + const int by4, const int bx4, + const enum BlockSize bs, const mv mv) +{ + const int bw4 = dav1d_block_dimensions[bs][0]; + int bh4 = dav1d_block_dimensions[bs][1]; + refmvs_block **rr = &rt->r[(by4 & 31) + 5]; + + const refmvs_block tmpl = (refmvs_block) { + .ref.ref = { 0, -1 }, + .mv.mv[0] = mv, + .bs = bs, + .mf = 0, + }; + do { + refmvs_block *r = *rr++ + bx4; + for (int x = 0; x < bw4; x++) { + r[x] = tmpl; + } + } while (--bh4); +} + +static inline void splat_tworef_mv(refmvs_tile *const rt, + const int by4, const int bx4, + const enum BlockSize bs, + const enum CompInterPredMode mode, + const refmvs_refpair ref, + const refmvs_mvpair mv) +{ + const int bw4 = dav1d_block_dimensions[bs][0]; + int bh4 = dav1d_block_dimensions[bs][1]; + refmvs_block **rr = &rt->r[(by4 & 31) + 5]; + + assert(bw4 >= 2 && bh4 >= 2); + const refmvs_block tmpl = (refmvs_block) { + .ref.pair = ref.pair + 0x0101, + .mv = mv, + .bs = bs, + .mf = (mode == GLOBALMV_GLOBALMV) | !!((1 << mode) & (0xbc)) * 2, + }; + do { + refmvs_block *r = *rr++ + bx4; + for (int x = 0; x < bw4; x++) + r[x] = tmpl; + } while (--bh4); +} + +static inline void splat_intraref(refmvs_tile *const rt, + const int by4, const int bx4, + const enum BlockSize bs) +{ + const int bw4 = dav1d_block_dimensions[bs][0]; + int bh4 = dav1d_block_dimensions[bs][1]; + refmvs_block **rr = &rt->r[(by4 & 31) + 5]; + + const refmvs_block tmpl = (refmvs_block) { + .ref.ref = { 0, -1 }, + .mv.mv[0].n = INVALID_MV, + .bs = bs, + .mf = 0, + }; + do { + refmvs_block *r = *rr++ + bx4; + for (int x = 0; x < bw4; x++) { + r[x] = tmpl; + } + } while (--bh4); +} + +#endif /* DAV1D_SRC_REF_MVS_H */ diff --git a/third_party/dav1d/src/refmvs.c b/third_party/dav1d/src/refmvs.c new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/refmvs.c @@ -0,0 +1,909 @@ +/* + * Copyright © 2020, VideoLAN and dav1d authors + * Copyright © 2020, Two Orioles, LLC + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "config.h" + +#include +#include + +#include "dav1d/common.h" + +#include "common/intops.h" + +#include "src/env.h" +#include "src/refmvs.h" + +static void add_spatial_candidate(refmvs_candidate *const mvstack, int *const cnt, + const int weight, const refmvs_block *const b, + const union refmvs_refpair ref, const mv gmv[2], + int *const have_newmv_match, + int *const have_refmv_match) +{ + if (b->mv.mv[0].n == INVALID_MV) return; // intra block, no intrabc + + if (ref.ref[1] == -1) { + for (int n = 0; n < 2; n++) { + if (b->ref.ref[n] == ref.ref[0]) { + const mv cand_mv = ((b->mf & 1) && gmv[0].n != INVALID_MV) ? + gmv[0] : b->mv.mv[n]; + + const int last = *cnt; + for (int m = 0; m < last; m++) + if (mvstack[m].mv.mv[0].n == cand_mv.n) { + mvstack[m].weight += weight; + *have_refmv_match = 1; + *have_newmv_match |= b->mf >> 1; + return; + } + + if (last < 8) { + mvstack[last].mv.mv[0] = cand_mv; + mvstack[last].weight = weight; + *cnt = last + 1; + } + *have_refmv_match = 1; + *have_newmv_match |= b->mf >> 1; + return; + } + } + } else if (b->ref.pair == ref.pair) { + const refmvs_mvpair cand_mv = { .mv = { + [0] = ((b->mf & 1) && gmv[0].n != INVALID_MV) ? gmv[0] : b->mv.mv[0], + [1] = ((b->mf & 1) && gmv[1].n != INVALID_MV) ? gmv[1] : b->mv.mv[1], + }}; + + const int last = *cnt; + for (int n = 0; n < last; n++) + if (mvstack[n].mv.n == cand_mv.n) { + mvstack[n].weight += weight; + *have_refmv_match = 1; + *have_newmv_match |= b->mf >> 1; + return; + } + + if (last < 8) { + mvstack[last].mv = cand_mv; + mvstack[last].weight = weight; + *cnt = last + 1; + } + *have_refmv_match = 1; + *have_newmv_match |= b->mf >> 1; + } +} + +static int scan_row(refmvs_candidate *const mvstack, int *const cnt, + const union refmvs_refpair ref, const mv gmv[2], + const refmvs_block *b, const int bw4, const int w4, + const int max_rows, const int step, + int *const have_newmv_match, int *const have_refmv_match) +{ + const refmvs_block *cand_b = b; + const enum BlockSize first_cand_bs = cand_b->bs; + const uint8_t *const first_cand_b_dim = dav1d_block_dimensions[first_cand_bs]; + int cand_bw4 = first_cand_b_dim[0]; + int len = imax(step, imin(bw4, cand_bw4)); + + if (bw4 <= cand_bw4) { + // FIXME weight can be higher for odd blocks (bx4 & 1), but then the + // position of the first block has to be odd already, i.e. not just + // for row_offset=-3/-5 + // FIXME why can this not be cand_bw4? + const int weight = bw4 == 1 ? 2 : + imax(2, imin(2 * max_rows, first_cand_b_dim[1])); + add_spatial_candidate(mvstack, cnt, len * weight, cand_b, ref, gmv, + have_newmv_match, have_refmv_match); + return weight >> 1; + } + + for (int x = 0;;) { + // FIXME if we overhang above, we could fill a bitmask so we don't have + // to repeat the add_spatial_candidate() for the next row, but just increase + // the weight here + add_spatial_candidate(mvstack, cnt, len * 2, cand_b, ref, gmv, + have_newmv_match, have_refmv_match); + x += len; + if (x >= w4) return 1; + cand_b = &b[x]; + cand_bw4 = dav1d_block_dimensions[cand_b->bs][0]; + assert(cand_bw4 < bw4); + len = imax(step, cand_bw4); + } +} + +static int scan_col(refmvs_candidate *const mvstack, int *const cnt, + const union refmvs_refpair ref, const mv gmv[2], + /*const*/ refmvs_block *const *b, const int bh4, const int h4, + const int bx4, const int max_cols, const int step, + int *const have_newmv_match, int *const have_refmv_match) +{ + const refmvs_block *cand_b = &b[0][bx4]; + const enum BlockSize first_cand_bs = cand_b->bs; + const uint8_t *const first_cand_b_dim = dav1d_block_dimensions[first_cand_bs]; + int cand_bh4 = first_cand_b_dim[1]; + int len = imax(step, imin(bh4, cand_bh4)); + + if (bh4 <= cand_bh4) { + // FIXME weight can be higher for odd blocks (by4 & 1), but then the + // position of the first block has to be odd already, i.e. not just + // for col_offset=-3/-5 + // FIXME why can this not be cand_bh4? + const int weight = bh4 == 1 ? 2 : + imax(2, imin(2 * max_cols, first_cand_b_dim[0])); + add_spatial_candidate(mvstack, cnt, len * weight, cand_b, ref, gmv, + have_newmv_match, have_refmv_match); + return weight >> 1; + } + + for (int y = 0;;) { + // FIXME if we overhang above, we could fill a bitmask so we don't have + // to repeat the add_spatial_candidate() for the next row, but just increase + // the weight here + add_spatial_candidate(mvstack, cnt, len * 2, cand_b, ref, gmv, + have_newmv_match, have_refmv_match); + y += len; + if (y >= h4) return 1; + cand_b = &b[y][bx4]; + cand_bh4 = dav1d_block_dimensions[cand_b->bs][1]; + assert(cand_bh4 < bh4); + len = imax(step, cand_bh4); + } +} + +static inline union mv mv_projection(const union mv mv, const int num, const int den) { + static const uint16_t div_mult[32] = { + 0, 16384, 8192, 5461, 4096, 3276, 2730, 2340, + 2048, 1820, 1638, 1489, 1365, 1260, 1170, 1092, + 1024, 963, 910, 862, 819, 780, 744, 712, + 682, 655, 630, 606, 585, 564, 546, 528 + }; + assert(den > 0 && den < 32); + assert(num > -32 && num < 32); + const int frac = num * div_mult[den]; + const int y = mv.y * frac, x = mv.x * frac; + // Round and clip according to AV1 spec section 7.9.3 + return (union mv) { // 0x3fff == (1 << 14) - 1 + .y = iclip((y + 8192 + (y >> 31)) >> 14, -0x3fff, 0x3fff), + .x = iclip((x + 8192 + (x >> 31)) >> 14, -0x3fff, 0x3fff) + }; +} + +static void add_temporal_candidate(const refmvs_frame *const rf, + refmvs_candidate *const mvstack, int *const cnt, + const refmvs_temporal_block *const rb, + const union refmvs_refpair ref, int *const globalmv_ctx, + const union mv gmv[]) +{ + if (rb->mv.n == INVALID_MV) return; + + union mv mv = mv_projection(rb->mv, rf->pocdiff[ref.ref[0] - 1], rb->ref); + fix_mv_precision(rf->frm_hdr, &mv); + + const int last = *cnt; + if (ref.ref[1] == -1) { + if (globalmv_ctx) + *globalmv_ctx = (abs(mv.x - gmv[0].x) | abs(mv.y - gmv[0].y)) >= 16; + + for (int n = 0; n < last; n++) + if (mvstack[n].mv.mv[0].n == mv.n) { + mvstack[n].weight += 2; + return; + } + if (last < 8) { + mvstack[last].mv.mv[0] = mv; + mvstack[last].weight = 2; + *cnt = last + 1; + } + } else { + refmvs_mvpair mvp = { .mv = { + [0] = mv, + [1] = mv_projection(rb->mv, rf->pocdiff[ref.ref[1] - 1], rb->ref), + }}; + fix_mv_precision(rf->frm_hdr, &mvp.mv[1]); + + for (int n = 0; n < last; n++) + if (mvstack[n].mv.n == mvp.n) { + mvstack[n].weight += 2; + return; + } + if (last < 8) { + mvstack[last].mv = mvp; + mvstack[last].weight = 2; + *cnt = last + 1; + } + } +} + +static void add_compound_extended_candidate(refmvs_candidate *const same, + int *const same_count, + const refmvs_block *const cand_b, + const int sign0, const int sign1, + const union refmvs_refpair ref, + const uint8_t *const sign_bias) +{ + refmvs_candidate *const diff = &same[2]; + int *const diff_count = &same_count[2]; + + for (int n = 0; n < 2; n++) { + const int cand_ref = cand_b->ref.ref[n]; + + if (cand_ref <= 0) break; + + mv cand_mv = cand_b->mv.mv[n]; + if (cand_ref == ref.ref[0]) { + if (same_count[0] < 2) + same[same_count[0]++].mv.mv[0] = cand_mv; + if (diff_count[1] < 2) { + if (sign1 ^ sign_bias[cand_ref - 1]) { + cand_mv.y = -cand_mv.y; + cand_mv.x = -cand_mv.x; + } + diff[diff_count[1]++].mv.mv[1] = cand_mv; + } + } else if (cand_ref == ref.ref[1]) { + if (same_count[1] < 2) + same[same_count[1]++].mv.mv[1] = cand_mv; + if (diff_count[0] < 2) { + if (sign0 ^ sign_bias[cand_ref - 1]) { + cand_mv.y = -cand_mv.y; + cand_mv.x = -cand_mv.x; + } + diff[diff_count[0]++].mv.mv[0] = cand_mv; + } + } else { + mv i_cand_mv = (union mv) { + .x = -cand_mv.x, + .y = -cand_mv.y + }; + + if (diff_count[0] < 2) { + diff[diff_count[0]++].mv.mv[0] = + sign0 ^ sign_bias[cand_ref - 1] ? + i_cand_mv : cand_mv; + } + + if (diff_count[1] < 2) { + diff[diff_count[1]++].mv.mv[1] = + sign1 ^ sign_bias[cand_ref - 1] ? + i_cand_mv : cand_mv; + } + } + } +} + +static void add_single_extended_candidate(refmvs_candidate mvstack[8], int *const cnt, + const refmvs_block *const cand_b, + const int sign, const uint8_t *const sign_bias) +{ + for (int n = 0; n < 2; n++) { + const int cand_ref = cand_b->ref.ref[n]; + + if (cand_ref <= 0) break; + // we need to continue even if cand_ref == ref.ref[0], since + // the candidate could have been added as a globalmv variant, + // which changes the value + // FIXME if scan_{row,col}() returned a mask for the nearest + // edge, we could skip the appropriate ones here + + mv cand_mv = cand_b->mv.mv[n]; + if (sign ^ sign_bias[cand_ref - 1]) { + cand_mv.y = -cand_mv.y; + cand_mv.x = -cand_mv.x; + } + + int m; + const int last = *cnt; + for (m = 0; m < last; m++) + if (cand_mv.n == mvstack[m].mv.mv[0].n) + break; + if (m == last) { + mvstack[m].mv.mv[0] = cand_mv; + mvstack[m].weight = 2; // "minimal" + *cnt = last + 1; + } + } +} + +/* + * refmvs_frame allocates memory for one sbrow (32 blocks high, whole frame + * wide) of 4x4-resolution refmvs_block entries for spatial MV referencing. + * mvrefs_tile[] keeps a list of 35 (32 + 3 above) pointers into this memory, + * and each sbrow, the bottom entries (y=27/29/31) are exchanged with the top + * (-5/-3/-1) pointers by calling dav1d_refmvs_tile_sbrow_init() at the start + * of each tile/sbrow. + * + * For temporal MV referencing, we call dav1d_refmvs_save_tmvs() at the end of + * each tile/sbrow (when tile column threading is enabled), or at the start of + * each interleaved sbrow (i.e. once for all tile columns together, when tile + * column threading is disabled). This will copy the 4x4-resolution spatial MVs + * into 8x8-resolution refmvs_temporal_block structures. Then, for subsequent + * frames, at the start of each tile/sbrow (when tile column threading is + * enabled) or at the start of each interleaved sbrow (when tile column + * threading is disabled), we call load_tmvs(), which will project the MVs to + * their respective position in the current frame. + */ + +void dav1d_refmvs_find(const refmvs_tile *const rt, + refmvs_candidate mvstack[8], int *const cnt, + int *const ctx, + const union refmvs_refpair ref, const enum BlockSize bs, + const enum EdgeFlags edge_flags, + const int by4, const int bx4) +{ + const refmvs_frame *const rf = rt->rf; + const uint8_t *const b_dim = dav1d_block_dimensions[bs]; + const int bw4 = b_dim[0], w4 = imin(imin(bw4, 16), rt->tile_col.end - bx4); + const int bh4 = b_dim[1], h4 = imin(imin(bh4, 16), rt->tile_row.end - by4); + mv gmv[2], tgmv[2]; + + *cnt = 0; + assert(ref.ref[0] >= 0 && ref.ref[0] <= 8 && + ref.ref[1] >= -1 && ref.ref[1] <= 8); + if (ref.ref[0] > 0) { + tgmv[0] = get_gmv_2d(&rf->frm_hdr->gmv[ref.ref[0] - 1], + bx4, by4, bw4, bh4, rf->frm_hdr); + gmv[0] = rf->frm_hdr->gmv[ref.ref[0] - 1].type > DAV1D_WM_TYPE_TRANSLATION ? + tgmv[0] : (mv) { .n = INVALID_MV }; + } else { + tgmv[0] = (mv) { .n = 0 }; + gmv[0] = (mv) { .n = INVALID_MV }; + } + if (ref.ref[1] > 0) { + tgmv[1] = get_gmv_2d(&rf->frm_hdr->gmv[ref.ref[1] - 1], + bx4, by4, bw4, bh4, rf->frm_hdr); + gmv[1] = rf->frm_hdr->gmv[ref.ref[1] - 1].type > DAV1D_WM_TYPE_TRANSLATION ? + tgmv[1] : (mv) { .n = INVALID_MV }; + } + + // top + int have_newmv = 0, have_col_mvs = 0, have_row_mvs = 0; + unsigned max_rows = 0, n_rows = ~0; + const refmvs_block *b_top; + if (by4 > rt->tile_row.start) { + max_rows = imin((by4 - rt->tile_row.start + 1) >> 1, 2 + (bh4 > 1)); + b_top = &rt->r[(by4 & 31) - 1 + 5][bx4]; + n_rows = scan_row(mvstack, cnt, ref, gmv, b_top, + bw4, w4, max_rows, bw4 >= 16 ? 4 : 1, + &have_newmv, &have_row_mvs); + } + + // left + unsigned max_cols = 0, n_cols = ~0U; + refmvs_block *const *b_left; + if (bx4 > rt->tile_col.start) { + max_cols = imin((bx4 - rt->tile_col.start + 1) >> 1, 2 + (bw4 > 1)); + b_left = &rt->r[(by4 & 31) + 5]; + n_cols = scan_col(mvstack, cnt, ref, gmv, b_left, + bh4, h4, bx4 - 1, max_cols, bh4 >= 16 ? 4 : 1, + &have_newmv, &have_col_mvs); + } + + // top/right + if (n_rows != ~0U && edge_flags & EDGE_I444_TOP_HAS_RIGHT && + imax(bw4, bh4) <= 16 && bw4 + bx4 < rt->tile_col.end) + { + add_spatial_candidate(mvstack, cnt, 4, &b_top[bw4], ref, gmv, + &have_newmv, &have_row_mvs); + } + + const int nearest_match = have_col_mvs + have_row_mvs; + const int nearest_cnt = *cnt; + for (int n = 0; n < nearest_cnt; n++) + mvstack[n].weight += 640; + + // temporal + int globalmv_ctx = rf->frm_hdr->use_ref_frame_mvs; + if (rf->use_ref_frame_mvs) { + const ptrdiff_t stride = rf->rp_stride; + const int by8 = by4 >> 1, bx8 = bx4 >> 1; + const refmvs_temporal_block *const rbi = &rt->rp_proj[(by8 & 15) * stride + bx8]; + const refmvs_temporal_block *rb = rbi; + const int step_h = bw4 >= 16 ? 2 : 1, step_v = bh4 >= 16 ? 2 : 1; + const int w8 = imin((w4 + 1) >> 1, 8), h8 = imin((h4 + 1) >> 1, 8); + for (int y = 0; y < h8; y += step_v) { + for (int x = 0; x < w8; x+= step_h) { + add_temporal_candidate(rf, mvstack, cnt, &rb[x], ref, + !(x | y) ? &globalmv_ctx : NULL, tgmv); + } + rb += stride * step_v; + } + if (imin(bw4, bh4) >= 2 && imax(bw4, bh4) < 16) { + const int bh8 = bh4 >> 1, bw8 = bw4 >> 1; + rb = &rbi[bh8 * stride]; + const int has_bottom = by8 + bh8 < imin(rt->tile_row.end >> 1, + (by8 & ~7) + 8); + if (has_bottom && bx8 - 1 >= imax(rt->tile_col.start >> 1, bx8 & ~7)) { + add_temporal_candidate(rf, mvstack, cnt, &rb[-1], ref, + NULL, NULL); + } + if (bx8 + bw8 < imin(rt->tile_col.end >> 1, (bx8 & ~7) + 8)) { + if (has_bottom) { + add_temporal_candidate(rf, mvstack, cnt, &rb[bw8], ref, + NULL, NULL); + } + if (by8 + bh8 - 1 < imin(rt->tile_row.end >> 1, (by8 & ~7) + 8)) { + add_temporal_candidate(rf, mvstack, cnt, &rb[bw8 - stride], + ref, NULL, NULL); + } + } + } + } + assert(*cnt <= 8); + + // top/left (which, confusingly, is part of "secondary" references) + int have_dummy_newmv_match; + if ((n_rows | n_cols) != ~0U) { + add_spatial_candidate(mvstack, cnt, 4, &b_top[-1], ref, gmv, + &have_dummy_newmv_match, &have_row_mvs); + } + + // "secondary" (non-direct neighbour) top & left edges + // what is different about secondary is that everything is now in 8x8 resolution + for (int n = 2; n <= 3; n++) { + if ((unsigned) n > n_rows && (unsigned) n <= max_rows) { + n_rows += scan_row(mvstack, cnt, ref, gmv, + &rt->r[(((by4 & 31) - 2 * n + 1) | 1) + 5][bx4 | 1], + bw4, w4, 1 + max_rows - n, bw4 >= 16 ? 4 : 2, + &have_dummy_newmv_match, &have_row_mvs); + } + + if ((unsigned) n > n_cols && (unsigned) n <= max_cols) { + n_cols += scan_col(mvstack, cnt, ref, gmv, &rt->r[((by4 & 31) | 1) + 5], + bh4, h4, (bx4 - n * 2 + 1) | 1, + 1 + max_cols - n, bh4 >= 16 ? 4 : 2, + &have_dummy_newmv_match, &have_col_mvs); + } + } + assert(*cnt <= 8); + + const int ref_match_count = have_col_mvs + have_row_mvs; + + // context build-up + int refmv_ctx, newmv_ctx; + switch (nearest_match) { + case 0: + refmv_ctx = imin(2, ref_match_count); + newmv_ctx = ref_match_count > 0; + break; + case 1: + refmv_ctx = imin(ref_match_count * 3, 4); + newmv_ctx = 3 - have_newmv; + break; + case 2: + refmv_ctx = 5; + newmv_ctx = 5 - have_newmv; + break; + } + + // sorting (nearest, then "secondary") + int len = nearest_cnt; + while (len) { + int last = 0; + for (int n = 1; n < len; n++) { + if (mvstack[n - 1].weight < mvstack[n].weight) { +#define EXCHANGE(a, b) do { refmvs_candidate tmp = a; a = b; b = tmp; } while (0) + EXCHANGE(mvstack[n - 1], mvstack[n]); + last = n; + } + } + len = last; + } + len = *cnt; + while (len > nearest_cnt) { + int last = nearest_cnt; + for (int n = nearest_cnt + 1; n < len; n++) { + if (mvstack[n - 1].weight < mvstack[n].weight) { + EXCHANGE(mvstack[n - 1], mvstack[n]); +#undef EXCHANGE + last = n; + } + } + len = last; + } + + if (ref.ref[1] > 0) { + if (*cnt < 2) { + const int sign0 = rf->sign_bias[ref.ref[0] - 1]; + const int sign1 = rf->sign_bias[ref.ref[1] - 1]; + const int sz4 = imin(w4, h4); + refmvs_candidate *const same = &mvstack[*cnt]; + int same_count[4] = { 0 }; + + // non-self references in top + if (n_rows != ~0U) for (int x = 0; x < sz4;) { + const refmvs_block *const cand_b = &b_top[x]; + add_compound_extended_candidate(same, same_count, cand_b, + sign0, sign1, ref, rf->sign_bias); + x += dav1d_block_dimensions[cand_b->bs][0]; + } + + // non-self references in left + if (n_cols != ~0U) for (int y = 0; y < sz4;) { + const refmvs_block *const cand_b = &b_left[y][bx4 - 1]; + add_compound_extended_candidate(same, same_count, cand_b, + sign0, sign1, ref, rf->sign_bias); + y += dav1d_block_dimensions[cand_b->bs][1]; + } + + refmvs_candidate *const diff = &same[2]; + const int *const diff_count = &same_count[2]; + + // merge together + for (int n = 0; n < 2; n++) { + int m = same_count[n]; + + if (m >= 2) continue; + + const int l = diff_count[n]; + if (l) { + same[m].mv.mv[n] = diff[0].mv.mv[n]; + if (++m == 2) continue; + if (l == 2) { + same[1].mv.mv[n] = diff[1].mv.mv[n]; + continue; + } + } + do { + same[m].mv.mv[n] = tgmv[n]; + } while (++m < 2); + } + + // if the first extended was the same as the non-extended one, + // then replace it with the second extended one + int n = *cnt; + if (n == 1 && mvstack[0].mv.n == same[0].mv.n) + mvstack[1].mv = mvstack[2].mv; + do { + mvstack[n].weight = 2; + } while (++n < 2); + *cnt = 2; + } + + // clamping + const int left = -(bx4 + bw4 + 4) * 4 * 8; + const int right = (rf->iw4 - bx4 + 4) * 4 * 8; + const int top = -(by4 + bh4 + 4) * 4 * 8; + const int bottom = (rf->ih4 - by4 + 4) * 4 * 8; + + const int n_refmvs = *cnt; + int n = 0; + do { + mvstack[n].mv.mv[0].x = iclip(mvstack[n].mv.mv[0].x, left, right); + mvstack[n].mv.mv[0].y = iclip(mvstack[n].mv.mv[0].y, top, bottom); + mvstack[n].mv.mv[1].x = iclip(mvstack[n].mv.mv[1].x, left, right); + mvstack[n].mv.mv[1].y = iclip(mvstack[n].mv.mv[1].y, top, bottom); + } while (++n < n_refmvs); + + switch (refmv_ctx >> 1) { + case 0: + *ctx = imin(newmv_ctx, 1); + break; + case 1: + *ctx = 1 + imin(newmv_ctx, 3); + break; + case 2: + *ctx = iclip(3 + newmv_ctx, 4, 7); + break; + } + + return; + } else if (*cnt < 2 && ref.ref[0] > 0) { + const int sign = rf->sign_bias[ref.ref[0] - 1]; + const int sz4 = imin(w4, h4); + + // non-self references in top + if (n_rows != ~0U) for (int x = 0; x < sz4 && *cnt < 2;) { + const refmvs_block *const cand_b = &b_top[x]; + add_single_extended_candidate(mvstack, cnt, cand_b, sign, rf->sign_bias); + x += dav1d_block_dimensions[cand_b->bs][0]; + } + + // non-self references in left + if (n_cols != ~0U) for (int y = 0; y < sz4 && *cnt < 2;) { + const refmvs_block *const cand_b = &b_left[y][bx4 - 1]; + add_single_extended_candidate(mvstack, cnt, cand_b, sign, rf->sign_bias); + y += dav1d_block_dimensions[cand_b->bs][1]; + } + } + assert(*cnt <= 8); + + // clamping + int n_refmvs = *cnt; + if (n_refmvs) { + const int left = -(bx4 + bw4 + 4) * 4 * 8; + const int right = (rf->iw4 - bx4 + 4) * 4 * 8; + const int top = -(by4 + bh4 + 4) * 4 * 8; + const int bottom = (rf->ih4 - by4 + 4) * 4 * 8; + + int n = 0; + do { + mvstack[n].mv.mv[0].x = iclip(mvstack[n].mv.mv[0].x, left, right); + mvstack[n].mv.mv[0].y = iclip(mvstack[n].mv.mv[0].y, top, bottom); + } while (++n < n_refmvs); + } + + for (int n = *cnt; n < 2; n++) + mvstack[n].mv.mv[0] = tgmv[0]; + + *ctx = (refmv_ctx << 4) | (globalmv_ctx << 3) | newmv_ctx; +} + +void dav1d_refmvs_tile_sbrow_init(refmvs_tile *const rt, const refmvs_frame *const rf, + const int tile_col_start4, const int tile_col_end4, + const int tile_row_start4, const int tile_row_end4, + const int sby, int tile_row_idx) +{ + if (rf->n_tile_threads == 1) tile_row_idx = 0; + rt->rp_proj = &rf->rp_proj[16 * rf->rp_stride * tile_row_idx]; + refmvs_block *r = &rf->r[35 * rf->r_stride * tile_row_idx]; + const int sbsz = rf->sbsz; + const int off = (sbsz * sby) & 16; + for (int i = 0; i < sbsz; i++, r += rf->r_stride) + rt->r[off + 5 + i] = r; + rt->r[off + 0] = r; + r += rf->r_stride; + rt->r[off + 1] = NULL; + rt->r[off + 2] = r; + r += rf->r_stride; + rt->r[off + 3] = NULL; + rt->r[off + 4] = r; + if (sby & 1) { +#define EXCHANGE(a, b) do { void *const tmp = a; a = b; b = tmp; } while (0) + EXCHANGE(rt->r[off + 0], rt->r[off + sbsz + 0]); + EXCHANGE(rt->r[off + 2], rt->r[off + sbsz + 2]); + EXCHANGE(rt->r[off + 4], rt->r[off + sbsz + 4]); +#undef EXCHANGE + } + + rt->rf = rf; + rt->tile_row.start = tile_row_start4; + rt->tile_row.end = imin(tile_row_end4, rf->ih4); + rt->tile_col.start = tile_col_start4; + rt->tile_col.end = imin(tile_col_end4, rf->iw4); +} + +void dav1d_refmvs_load_tmvs(const refmvs_frame *const rf, int tile_row_idx, + const int col_start8, const int col_end8, + const int row_start8, int row_end8) +{ + if (rf->n_tile_threads == 1) tile_row_idx = 0; + assert(row_start8 >= 0); + assert((unsigned) (row_end8 - row_start8) <= 16U); + row_end8 = imin(row_end8, rf->ih8); + const int col_start8i = imax(col_start8 - 8, 0); + const int col_end8i = imin(col_end8 + 8, rf->iw8); + + const ptrdiff_t stride = rf->rp_stride; + refmvs_temporal_block *rp_proj = + &rf->rp_proj[16 * stride * tile_row_idx + (row_start8 & 15) * stride]; + for (int y = row_start8; y < row_end8; y++) { + for (int x = col_start8; x < col_end8; x++) + rp_proj[x].mv.n = INVALID_MV; + rp_proj += stride; + } + + rp_proj = &rf->rp_proj[16 * stride * tile_row_idx]; + for (int n = 0; n < rf->n_mfmvs; n++) { + const int ref2cur = rf->mfmv_ref2cur[n]; + if (ref2cur == INT_MIN) continue; + + const int ref = rf->mfmv_ref[n]; + const int ref_sign = ref - 4; + const refmvs_temporal_block *r = &rf->rp_ref[ref][row_start8 * stride]; + for (int y = row_start8; y < row_end8; y++) { + const int y_sb_align = y & ~7; + const int y_proj_start = imax(y_sb_align, row_start8); + const int y_proj_end = imin(y_sb_align + 8, row_end8); + for (int x = col_start8i; x < col_end8i; x++) { + const refmvs_temporal_block *rb = &r[x]; + const int b_ref = rb->ref; + if (!b_ref) continue; + const int ref2ref = rf->mfmv_ref2ref[n][b_ref - 1]; + if (!ref2ref) continue; + const mv b_mv = rb->mv; + const mv offset = mv_projection(b_mv, ref2cur, ref2ref); + int pos_x = x + apply_sign(abs(offset.x) >> 6, + offset.x ^ ref_sign); + const int pos_y = y + apply_sign(abs(offset.y) >> 6, + offset.y ^ ref_sign); + if (pos_y >= y_proj_start && pos_y < y_proj_end) { + const ptrdiff_t pos = (pos_y & 15) * stride; + for (;;) { + const int x_sb_align = x & ~7; + if (pos_x >= imax(x_sb_align - 8, col_start8) && + pos_x < imin(x_sb_align + 16, col_end8)) + { + rp_proj[pos + pos_x].mv = rb->mv; + rp_proj[pos + pos_x].ref = ref2ref; + } + if (++x >= col_end8i) break; + rb++; + if (rb->ref != b_ref || rb->mv.n != b_mv.n) break; + pos_x++; + } + } else { + for (;;) { + if (++x >= col_end8i) break; + rb++; + if (rb->ref != b_ref || rb->mv.n != b_mv.n) break; + } + } + x--; + } + r += stride; + } + } +} + +void dav1d_refmvs_save_tmvs(const refmvs_tile *const rt, + const int col_start8, int col_end8, + const int row_start8, int row_end8) +{ + const refmvs_frame *const rf = rt->rf; + + assert(row_start8 >= 0); + assert((unsigned) (row_end8 - row_start8) <= 16U); + row_end8 = imin(row_end8, rf->ih8); + col_end8 = imin(col_end8, rf->iw8); + + const ptrdiff_t stride = rf->rp_stride; + const uint8_t *const ref_sign = rf->mfmv_sign; + refmvs_temporal_block *rp = &rf->rp[row_start8 * stride]; + for (int y = row_start8; y < row_end8; y++) { + const refmvs_block *const b = rt->r[6 + (y & 15) * 2]; + + for (int x = col_start8; x < col_end8;) { + const refmvs_block *const cand_b = &b[x * 2 + 1]; + const int bw8 = (dav1d_block_dimensions[cand_b->bs][0] + 1) >> 1; + + if (cand_b->ref.ref[1] > 0 && ref_sign[cand_b->ref.ref[1] - 1] && + (abs(cand_b->mv.mv[1].y) | abs(cand_b->mv.mv[1].x)) < 4096) + { + for (int n = 0; n < bw8; n++, x++) + rp[x] = (refmvs_temporal_block) { .mv = cand_b->mv.mv[1], + .ref = cand_b->ref.ref[1] }; + } else if (cand_b->ref.ref[0] > 0 && ref_sign[cand_b->ref.ref[0] - 1] && + (abs(cand_b->mv.mv[0].y) | abs(cand_b->mv.mv[0].x)) < 4096) + { + for (int n = 0; n < bw8; n++, x++) + rp[x] = (refmvs_temporal_block) { .mv = cand_b->mv.mv[0], + .ref = cand_b->ref.ref[0] }; + } else { + for (int n = 0; n < bw8; n++, x++) + rp[x].ref = 0; // "invalid" + } + } + rp += stride; + } +} + +int dav1d_refmvs_init_frame(refmvs_frame *const rf, + const Dav1dSequenceHeader *const seq_hdr, + const Dav1dFrameHeader *const frm_hdr, + const unsigned ref_poc[7], + refmvs_temporal_block *const rp, + const unsigned ref_ref_poc[7][7], + /*const*/ refmvs_temporal_block *const rp_ref[7], + const int n_tile_threads) +{ + rf->sbsz = 16 << seq_hdr->sb128; + rf->frm_hdr = frm_hdr; + rf->iw8 = (frm_hdr->width[0] + 7) >> 3; + rf->ih8 = (frm_hdr->height + 7) >> 3; + rf->iw4 = rf->iw8 << 1; + rf->ih4 = rf->ih8 << 1; + + const ptrdiff_t r_stride = ((frm_hdr->width[0] + 127) & ~127) >> 2; + const int n_tile_rows = n_tile_threads > 1 ? frm_hdr->tiling.rows : 1; + if (r_stride != rf->r_stride || n_tile_rows != rf->n_tile_rows) { + if (rf->r) free(rf->r); + rf->r = malloc(sizeof(*rf->r) * 35 * r_stride * n_tile_rows); + if (!rf->r) return DAV1D_ERR(ENOMEM); + rf->r_stride = r_stride; + } + + const ptrdiff_t rp_stride = r_stride >> 1; + if (rp_stride != rf->rp_stride || n_tile_rows != rf->n_tile_rows) { + if (rf->rp_proj) free(rf->rp_proj); + rf->rp_proj = malloc(sizeof(*rf->rp_proj) * 16 * rp_stride * n_tile_rows); + if (!rf->rp_proj) return DAV1D_ERR(ENOMEM); + rf->rp_stride = rp_stride; + } + rf->n_tile_rows = n_tile_rows; + rf->n_tile_threads = n_tile_threads; + rf->rp = rp; + rf->rp_ref = rp_ref; + const unsigned poc = frm_hdr->frame_offset; + for (int i = 0; i < 7; i++) { + const int poc_diff = get_poc_diff(seq_hdr->order_hint_n_bits, + ref_poc[i], poc); + rf->sign_bias[i] = poc_diff > 0; + rf->mfmv_sign[i] = poc_diff < 0; + rf->pocdiff[i] = iclip(get_poc_diff(seq_hdr->order_hint_n_bits, + poc, ref_poc[i]), -31, 31); + } + + // temporal MV setup + rf->n_mfmvs = 0; + if (frm_hdr->use_ref_frame_mvs && seq_hdr->order_hint_n_bits) { + int total = 2; + if (rp_ref[0] && ref_ref_poc[0][6] != ref_poc[3] /* alt-of-last != gold */) { + rf->mfmv_ref[rf->n_mfmvs++] = 0; // last + total = 3; + } + if (rp_ref[4] && get_poc_diff(seq_hdr->order_hint_n_bits, ref_poc[4], + frm_hdr->frame_offset) > 0) + { + rf->mfmv_ref[rf->n_mfmvs++] = 4; // bwd + } + if (rp_ref[5] && get_poc_diff(seq_hdr->order_hint_n_bits, ref_poc[5], + frm_hdr->frame_offset) > 0) + { + rf->mfmv_ref[rf->n_mfmvs++] = 5; // altref2 + } + if (rf->n_mfmvs < total && rp_ref[6] && + get_poc_diff(seq_hdr->order_hint_n_bits, ref_poc[6], + frm_hdr->frame_offset) > 0) + { + rf->mfmv_ref[rf->n_mfmvs++] = 6; // altref + } + if (rf->n_mfmvs < total && rp_ref[1]) + rf->mfmv_ref[rf->n_mfmvs++] = 1; // last2 + + for (int n = 0; n < rf->n_mfmvs; n++) { + const unsigned rpoc = ref_poc[rf->mfmv_ref[n]]; + const int diff1 = get_poc_diff(seq_hdr->order_hint_n_bits, + rpoc, frm_hdr->frame_offset); + if (abs(diff1) > 31) { + rf->mfmv_ref2cur[n] = INT_MIN; + } else { + rf->mfmv_ref2cur[n] = rf->mfmv_ref[n] < 4 ? -diff1 : diff1; + for (int m = 0; m < 7; m++) { + const unsigned rrpoc = ref_ref_poc[rf->mfmv_ref[n]][m]; + const int diff2 = get_poc_diff(seq_hdr->order_hint_n_bits, + rpoc, rrpoc); + // unsigned comparison also catches the < 0 case + rf->mfmv_ref2ref[n][m] = (unsigned) diff2 > 31U ? 0 : diff2; + } + } + } + } + rf->use_ref_frame_mvs = rf->n_mfmvs > 0; + + return 0; +} + +void dav1d_refmvs_init(refmvs_frame *const rf) { + rf->r = NULL; + rf->r_stride = 0; + rf->rp_proj = NULL; + rf->rp_stride = 0; +} + +void dav1d_refmvs_clear(refmvs_frame *const rf) { + if (rf->r) free(rf->r); + if (rf->rp_proj) free(rf->rp_proj); +} diff --git a/third_party/dav1d/src/tables.h b/third_party/dav1d/src/tables.h --- a/third_party/dav1d/src/tables.h +++ b/third_party/dav1d/src/tables.h @@ -108,13 +108,13 @@ extern const int8_t dav1d_cdef_directions[12][2]; extern const int16_t dav1d_sgr_params[16][4]; extern const uint8_t dav1d_sgr_x_by_x[256]; -extern const int8_t dav1d_mc_subpel_filters[5][15][8]; +extern const int8_t dav1d_mc_subpel_filters[5+ARCH_X86_64][15][8]; extern const int8_t dav1d_mc_warp_filter[193][8]; -extern const int16_t dav1d_resize_filter[64][8]; +extern const int8_t dav1d_resize_filter[64][8]; extern const uint8_t dav1d_sm_weights[128]; extern const uint16_t dav1d_dr_intra_derivative[44]; extern const int8_t dav1d_filter_intra_taps[5][64]; diff --git a/third_party/dav1d/src/tables.c b/third_party/dav1d/src/tables.c --- a/third_party/dav1d/src/tables.c +++ b/third_party/dav1d/src/tables.c @@ -440,11 +440,11 @@ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0 }; -const int8_t ALIGN(dav1d_mc_subpel_filters[5][15][8], 8) = { +const int8_t ALIGN(dav1d_mc_subpel_filters[5+ARCH_X86_64][15][8], 8) = { [DAV1D_FILTER_8TAP_REGULAR] = { { 0, 1, -3, 63, 4, -1, 0, 0 }, { 0, 1, -5, 61, 9, -2, 0, 0 }, { 0, 1, -6, 58, 14, -4, 1, 0 }, { 0, 1, -7, 55, 19, -5, 1, 0 }, @@ -522,10 +522,31 @@ { 0, 0, 4, 22, 29, 9, 0, 0 }, { 0, 0, 3, 21, 30, 10, 0, 0 }, { 0, 0, 2, 20, 31, 11, 0, 0 }, { 0, 0, 2, 18, 31, 13, 0, 0 }, { 0, 0, 1, 17, 31, 15, 0, 0 } +#if ARCH_X86_64 + /* Bilin scaled being very rarely used, add a new table entry + * and use the put/prep_8tap_scaled code, thus acting as a + * scaled bilinear filter. */ + }, [5] = { + { 0, 0, 0, 60, 4, 0, 0, 0 }, + { 0, 0, 0, 56, 8, 0, 0, 0 }, + { 0, 0, 0, 52, 12, 0, 0, 0 }, + { 0, 0, 0, 48, 16, 0, 0, 0 }, + { 0, 0, 0, 44, 20, 0, 0, 0 }, + { 0, 0, 0, 40, 24, 0, 0, 0 }, + { 0, 0, 0, 36, 28, 0, 0, 0 }, + { 0, 0, 0, 32, 32, 0, 0, 0 }, + { 0, 0, 0, 28, 36, 0, 0, 0 }, + { 0, 0, 0, 24, 40, 0, 0, 0 }, + { 0, 0, 0, 20, 44, 0, 0, 0 }, + { 0, 0, 0, 16, 48, 0, 0, 0 }, + { 0, 0, 0, 12, 52, 0, 0, 0 }, + { 0, 0, 0, 8, 56, 0, 0, 0 }, + { 0, 0, 0, 4, 60, 0, 0, 0 } +#endif } }; #if ARCH_X86 #define W(v0, v1, v2, v3, v4, v5, v6, v7) { v0, v2, v4, v6, v1, v3, v5, v7 } @@ -634,46 +655,46 @@ W( 0, 0, 0, - 1, 4, 127, - 3, 1 ), W( 0, 0, 0, 0, 2, 127, - 1, 0 ), // dummy (replicate row index 191) W( 0, 0, 0, 0, 2, 127, - 1, 0 ), }; -const int16_t dav1d_resize_filter[64][8] = { - { 0, 0, 0, 128, 0, 0, 0, 0 }, { 0, 0, -1, 128, 2, -1, 0, 0 }, - { 0, 1, -3, 127, 4, -2, 1, 0 }, { 0, 1, -4, 127, 6, -3, 1, 0 }, - { 0, 2, -6, 126, 8, -3, 1, 0 }, { 0, 2, -7, 125, 11, -4, 1, 0 }, - { -1, 2, -8, 125, 13, -5, 2, 0 }, { -1, 3, -9, 124, 15, -6, 2, 0 }, - { -1, 3, -10, 123, 18, -6, 2, -1 }, { -1, 3, -11, 122, 20, -7, 3, -1 }, - { -1, 4, -12, 121, 22, -8, 3, -1 }, { -1, 4, -13, 120, 25, -9, 3, -1 }, - { -1, 4, -14, 118, 28, -9, 3, -1 }, { -1, 4, -15, 117, 30, -10, 4, -1 }, - { -1, 5, -16, 116, 32, -11, 4, -1 }, { -1, 5, -16, 114, 35, -12, 4, -1 }, - { -1, 5, -17, 112, 38, -12, 4, -1 }, { -1, 5, -18, 111, 40, -13, 5, -1 }, - { -1, 5, -18, 109, 43, -14, 5, -1 }, { -1, 6, -19, 107, 45, -14, 5, -1 }, - { -1, 6, -19, 105, 48, -15, 5, -1 }, { -1, 6, -19, 103, 51, -16, 5, -1 }, - { -1, 6, -20, 101, 53, -16, 6, -1 }, { -1, 6, -20, 99, 56, -17, 6, -1 }, - { -1, 6, -20, 97, 58, -17, 6, -1 }, { -1, 6, -20, 95, 61, -18, 6, -1 }, - { -2, 7, -20, 93, 64, -18, 6, -2 }, { -2, 7, -20, 91, 66, -19, 6, -1 }, - { -2, 7, -20, 88, 69, -19, 6, -1 }, { -2, 7, -20, 86, 71, -19, 6, -1 }, - { -2, 7, -20, 84, 74, -20, 7, -2 }, { -2, 7, -20, 81, 76, -20, 7, -1 }, - { -2, 7, -20, 79, 79, -20, 7, -2 }, { -1, 7, -20, 76, 81, -20, 7, -2 }, - { -2, 7, -20, 74, 84, -20, 7, -2 }, { -1, 6, -19, 71, 86, -20, 7, -2 }, - { -1, 6, -19, 69, 88, -20, 7, -2 }, { -1, 6, -19, 66, 91, -20, 7, -2 }, - { -2, 6, -18, 64, 93, -20, 7, -2 }, { -1, 6, -18, 61, 95, -20, 6, -1 }, - { -1, 6, -17, 58, 97, -20, 6, -1 }, { -1, 6, -17, 56, 99, -20, 6, -1 }, - { -1, 6, -16, 53, 101, -20, 6, -1 }, { -1, 5, -16, 51, 103, -19, 6, -1 }, - { -1, 5, -15, 48, 105, -19, 6, -1 }, { -1, 5, -14, 45, 107, -19, 6, -1 }, - { -1, 5, -14, 43, 109, -18, 5, -1 }, { -1, 5, -13, 40, 111, -18, 5, -1 }, - { -1, 4, -12, 38, 112, -17, 5, -1 }, { -1, 4, -12, 35, 114, -16, 5, -1 }, - { -1, 4, -11, 32, 116, -16, 5, -1 }, { -1, 4, -10, 30, 117, -15, 4, -1 }, - { -1, 3, -9, 28, 118, -14, 4, -1 }, { -1, 3, -9, 25, 120, -13, 4, -1 }, - { -1, 3, -8, 22, 121, -12, 4, -1 }, { -1, 3, -7, 20, 122, -11, 3, -1 }, - { -1, 2, -6, 18, 123, -10, 3, -1 }, { 0, 2, -6, 15, 124, -9, 3, -1 }, - { 0, 2, -5, 13, 125, -8, 2, -1 }, { 0, 1, -4, 11, 125, -7, 2, 0 }, - { 0, 1, -3, 8, 126, -6, 2, 0 }, { 0, 1, -3, 6, 127, -4, 1, 0 }, - { 0, 1, -2, 4, 127, -3, 1, 0 }, { 0, 0, -1, 2, 128, -1, 0, 0 }, +const int8_t ALIGN(dav1d_resize_filter[64][8], 8) = { + { 0, 0, 0, -128, 0, 0, 0, 0 }, { 0, 0, 1, -128, -2, 1, 0, 0 }, + { 0, -1, 3, -127, -4, 2, -1, 0 }, { 0, -1, 4, -127, -6, 3, -1, 0 }, + { 0, -2, 6, -126, -8, 3, -1, 0 }, { 0, -2, 7, -125, -11, 4, -1, 0 }, + { 1, -2, 8, -125, -13, 5, -2, 0 }, { 1, -3, 9, -124, -15, 6, -2, 0 }, + { 1, -3, 10, -123, -18, 6, -2, 1 }, { 1, -3, 11, -122, -20, 7, -3, 1 }, + { 1, -4, 12, -121, -22, 8, -3, 1 }, { 1, -4, 13, -120, -25, 9, -3, 1 }, + { 1, -4, 14, -118, -28, 9, -3, 1 }, { 1, -4, 15, -117, -30, 10, -4, 1 }, + { 1, -5, 16, -116, -32, 11, -4, 1 }, { 1, -5, 16, -114, -35, 12, -4, 1 }, + { 1, -5, 17, -112, -38, 12, -4, 1 }, { 1, -5, 18, -111, -40, 13, -5, 1 }, + { 1, -5, 18, -109, -43, 14, -5, 1 }, { 1, -6, 19, -107, -45, 14, -5, 1 }, + { 1, -6, 19, -105, -48, 15, -5, 1 }, { 1, -6, 19, -103, -51, 16, -5, 1 }, + { 1, -6, 20, -101, -53, 16, -6, 1 }, { 1, -6, 20, -99, -56, 17, -6, 1 }, + { 1, -6, 20, -97, -58, 17, -6, 1 }, { 1, -6, 20, -95, -61, 18, -6, 1 }, + { 2, -7, 20, -93, -64, 18, -6, 2 }, { 2, -7, 20, -91, -66, 19, -6, 1 }, + { 2, -7, 20, -88, -69, 19, -6, 1 }, { 2, -7, 20, -86, -71, 19, -6, 1 }, + { 2, -7, 20, -84, -74, 20, -7, 2 }, { 2, -7, 20, -81, -76, 20, -7, 1 }, + { 2, -7, 20, -79, -79, 20, -7, 2 }, { 1, -7, 20, -76, -81, 20, -7, 2 }, + { 2, -7, 20, -74, -84, 20, -7, 2 }, { 1, -6, 19, -71, -86, 20, -7, 2 }, + { 1, -6, 19, -69, -88, 20, -7, 2 }, { 1, -6, 19, -66, -91, 20, -7, 2 }, + { 2, -6, 18, -64, -93, 20, -7, 2 }, { 1, -6, 18, -61, -95, 20, -6, 1 }, + { 1, -6, 17, -58, -97, 20, -6, 1 }, { 1, -6, 17, -56, -99, 20, -6, 1 }, + { 1, -6, 16, -53, -101, 20, -6, 1 }, { 1, -5, 16, -51, -103, 19, -6, 1 }, + { 1, -5, 15, -48, -105, 19, -6, 1 }, { 1, -5, 14, -45, -107, 19, -6, 1 }, + { 1, -5, 14, -43, -109, 18, -5, 1 }, { 1, -5, 13, -40, -111, 18, -5, 1 }, + { 1, -4, 12, -38, -112, 17, -5, 1 }, { 1, -4, 12, -35, -114, 16, -5, 1 }, + { 1, -4, 11, -32, -116, 16, -5, 1 }, { 1, -4, 10, -30, -117, 15, -4, 1 }, + { 1, -3, 9, -28, -118, 14, -4, 1 }, { 1, -3, 9, -25, -120, 13, -4, 1 }, + { 1, -3, 8, -22, -121, 12, -4, 1 }, { 1, -3, 7, -20, -122, 11, -3, 1 }, + { 1, -2, 6, -18, -123, 10, -3, 1 }, { 0, -2, 6, -15, -124, 9, -3, 1 }, + { 0, -2, 5, -13, -125, 8, -2, 1 }, { 0, -1, 4, -11, -125, 7, -2, 0 }, + { 0, -1, 3, -8, -126, 6, -2, 0 }, { 0, -1, 3, -6, -127, 4, -1, 0 }, + { 0, -1, 2, -4, -127, 3, -1, 0 }, { 0, 0, 1, -2, -128, 1, 0, 0 }, }; -const uint8_t dav1d_sm_weights[128] = { +const uint8_t ALIGN(dav1d_sm_weights[128], 16) = { // Unused, because we always offset by bs, which is at least 2. 0, 0, // bs = 2 255, 128, // bs = 4 diff --git a/third_party/dav1d/src/thread_task.c b/third_party/dav1d/src/thread_task.c --- a/third_party/dav1d/src/thread_task.c +++ b/third_party/dav1d/src/thread_task.c @@ -40,12 +40,11 @@ &f->frame_thread.td.lock); } if (f->frame_thread.die) break; pthread_mutex_unlock(&f->frame_thread.td.lock); - const int res = dav1d_decode_frame(f); - if (res) + if (dav1d_decode_frame(f)) memset(f->frame_thread.cf, 0, (size_t)f->frame_thread.cf_sz * 128 * 128 / 2); pthread_mutex_lock(&f->frame_thread.td.lock); f->n_tile_data = 0; @@ -90,12 +89,12 @@ // loopfilter happens in the main thread Dav1dTileState *const ts = t->ts = &f->ts[task_idx]; for (t->by = ts->tiling.row_start; t->by < ts->tiling.row_end; t->by += f->sb_step) { - int error = dav1d_decode_tile_sbrow(t); - int progress = error ? TILE_ERROR : 1 + (t->by >> f->sb_shift); + const int error = dav1d_decode_tile_sbrow(t); + const int progress = error ? TILE_ERROR : 1 + (t->by >> f->sb_shift); // signal progress pthread_mutex_lock(&ts->tile_thread.lock); atomic_store(&ts->progress, progress); pthread_cond_signal(&ts->tile_thread.cond); @@ -126,11 +125,11 @@ // we need to interleave sbrow decoding for all tile cols in a // tile row, since otherwise subsequent threads will be blocked // waiting for the post-filter to complete t->ts = ts; t->by = sby << f->sb_shift; - int error = dav1d_decode_tile_sbrow(t); + const int error = dav1d_decode_tile_sbrow(t); progress = error ? TILE_ERROR : 1 + sby; // signal progress pthread_mutex_lock(&ts->tile_thread.lock); atomic_store(&ts->progress, progress); diff --git a/third_party/dav1d/src/warpmv.h b/third_party/dav1d/src/warpmv.h --- a/third_party/dav1d/src/warpmv.h +++ b/third_party/dav1d/src/warpmv.h @@ -30,8 +30,10 @@ #include "src/levels.h" int dav1d_get_shear_params(Dav1dWarpedMotionParams *wm); int dav1d_find_affine_int(const int (*pts)[2][2], int np, int bw4, int bh4, - mv mv, Dav1dWarpedMotionParams *wm, int by, int bx); + mv mv, Dav1dWarpedMotionParams *wm, int bx, int by); +void dav1d_set_affine_mv2d(int bw4, int bh4, + mv mv, Dav1dWarpedMotionParams *wm, int bx, int by); #endif /* DAV1D_SRC_WARPMV_H */ diff --git a/third_party/dav1d/src/warpmv.c b/third_party/dav1d/src/warpmv.c --- a/third_party/dav1d/src/warpmv.c +++ b/third_party/dav1d/src/warpmv.c @@ -128,10 +128,26 @@ ((1LL << shift) >> 1)) >> shift), v1); return iclip(v2, 0xe001, 0x11fff); } +void dav1d_set_affine_mv2d(const int bw4, const int bh4, + const mv mv, Dav1dWarpedMotionParams *const wm, + const int bx4, const int by4) +{ + int32_t *const mat = wm->matrix; + const int rsuy = 2 * bh4 - 1; + const int rsux = 2 * bw4 - 1; + const int isuy = by4 * 4 + rsuy; + const int isux = bx4 * 4 + rsux; + + mat[0] = iclip(mv.x * 0x2000 - (isux * (mat[2] - 0x10000) + isuy * mat[3]), + -0x800000, 0x7fffff); + mat[1] = iclip(mv.y * 0x2000 - (isux * mat[4] + isuy * (mat[5] - 0x10000)), + -0x800000, 0x7fffff); +} + int dav1d_find_affine_int(const int (*pts)[2][2], const int np, const int bw4, const int bh4, const mv mv, Dav1dWarpedMotionParams *const wm, const int bx4, const int by4) { diff --git a/third_party/dav1d/src/wedge.c b/third_party/dav1d/src/wedge.c --- a/third_party/dav1d/src/wedge.c +++ b/third_party/dav1d/src/wedge.c @@ -81,43 +81,43 @@ { WEDGE_OBLIQUE153, 4, 2 }, { WEDGE_OBLIQUE153, 4, 6 }, { WEDGE_OBLIQUE63, 2, 4 }, { WEDGE_OBLIQUE63, 6, 4 }, { WEDGE_OBLIQUE117, 2, 4 }, { WEDGE_OBLIQUE117, 6, 4 }, }; -static uint8_t ALIGN(wedge_masks_444_32x32[2 * 16 * 32 * 32], 32); -static uint8_t ALIGN(wedge_masks_444_32x16[2 * 16 * 32 * 16], 32); -static uint8_t ALIGN(wedge_masks_444_32x8 [2 * 16 * 32 * 8], 32); -static uint8_t ALIGN(wedge_masks_444_16x32[2 * 16 * 16 * 32], 32); -static uint8_t ALIGN(wedge_masks_444_16x16[2 * 16 * 16 * 16], 32); -static uint8_t ALIGN(wedge_masks_444_16x8 [2 * 16 * 16 * 8], 32); -static uint8_t ALIGN(wedge_masks_444_8x32 [2 * 16 * 8 * 32], 32); -static uint8_t ALIGN(wedge_masks_444_8x16 [2 * 16 * 8 * 16], 32); -static uint8_t ALIGN(wedge_masks_444_8x8 [2 * 16 * 8 * 8], 32); +static uint8_t ALIGN(wedge_masks_444_32x32[2 * 16 * 32 * 32], 64); +static uint8_t ALIGN(wedge_masks_444_32x16[2 * 16 * 32 * 16], 64); +static uint8_t ALIGN(wedge_masks_444_32x8 [2 * 16 * 32 * 8], 64); +static uint8_t ALIGN(wedge_masks_444_16x32[2 * 16 * 16 * 32], 64); +static uint8_t ALIGN(wedge_masks_444_16x16[2 * 16 * 16 * 16], 64); +static uint8_t ALIGN(wedge_masks_444_16x8 [2 * 16 * 16 * 8], 64); +static uint8_t ALIGN(wedge_masks_444_8x32 [2 * 16 * 8 * 32], 64); +static uint8_t ALIGN(wedge_masks_444_8x16 [2 * 16 * 8 * 16], 64); +static uint8_t ALIGN(wedge_masks_444_8x8 [2 * 16 * 8 * 8], 64); -static uint8_t ALIGN(wedge_masks_422_16x32[2 * 16 * 16 * 32], 32); -static uint8_t ALIGN(wedge_masks_422_16x16[2 * 16 * 16 * 16], 32); -static uint8_t ALIGN(wedge_masks_422_16x8 [2 * 16 * 16 * 8], 32); -static uint8_t ALIGN(wedge_masks_422_8x32 [2 * 16 * 8 * 32], 32); -static uint8_t ALIGN(wedge_masks_422_8x16 [2 * 16 * 8 * 16], 32); -static uint8_t ALIGN(wedge_masks_422_8x8 [2 * 16 * 8 * 8], 32); -static uint8_t ALIGN(wedge_masks_422_4x32 [2 * 16 * 4 * 32], 32); -static uint8_t ALIGN(wedge_masks_422_4x16 [2 * 16 * 4 * 16], 32); +static uint8_t ALIGN(wedge_masks_422_16x32[2 * 16 * 16 * 32], 64); +static uint8_t ALIGN(wedge_masks_422_16x16[2 * 16 * 16 * 16], 64); +static uint8_t ALIGN(wedge_masks_422_16x8 [2 * 16 * 16 * 8], 64); +static uint8_t ALIGN(wedge_masks_422_8x32 [2 * 16 * 8 * 32], 64); +static uint8_t ALIGN(wedge_masks_422_8x16 [2 * 16 * 8 * 16], 64); +static uint8_t ALIGN(wedge_masks_422_8x8 [2 * 16 * 8 * 8], 64); +static uint8_t ALIGN(wedge_masks_422_4x32 [2 * 16 * 4 * 32], 64); +static uint8_t ALIGN(wedge_masks_422_4x16 [2 * 16 * 4 * 16], 64); static uint8_t ALIGN(wedge_masks_422_4x8 [2 * 16 * 4 * 8], 32); -static uint8_t ALIGN(wedge_masks_420_16x16[2 * 16 * 16 * 16], 32); -static uint8_t ALIGN(wedge_masks_420_16x8 [2 * 16 * 16 * 8], 32); -static uint8_t ALIGN(wedge_masks_420_16x4 [2 * 16 * 16 * 4], 32); -static uint8_t ALIGN(wedge_masks_420_8x16 [2 * 16 * 8 * 16], 32); -static uint8_t ALIGN(wedge_masks_420_8x8 [2 * 16 * 8 * 8], 32); -static uint8_t ALIGN(wedge_masks_420_8x4 [2 * 16 * 8 * 4], 32); -static uint8_t ALIGN(wedge_masks_420_4x16 [2 * 16 * 4 * 16], 32); +static uint8_t ALIGN(wedge_masks_420_16x16[2 * 16 * 16 * 16], 64); +static uint8_t ALIGN(wedge_masks_420_16x8 [2 * 16 * 16 * 8], 64); +static uint8_t ALIGN(wedge_masks_420_16x4 [2 * 16 * 16 * 4], 64); +static uint8_t ALIGN(wedge_masks_420_8x16 [2 * 16 * 8 * 16], 64); +static uint8_t ALIGN(wedge_masks_420_8x8 [2 * 16 * 8 * 8], 64); +static uint8_t ALIGN(wedge_masks_420_8x4 [2 * 16 * 8 * 4], 64); +static uint8_t ALIGN(wedge_masks_420_4x16 [2 * 16 * 4 * 16], 64); static uint8_t ALIGN(wedge_masks_420_4x8 [2 * 16 * 4 * 8], 32); -static uint8_t ALIGN(wedge_masks_420_4x4 [2 * 16 * 4 * 4], 32); +static uint8_t ALIGN(wedge_masks_420_4x4 [2 * 16 * 4 * 4], 16); const uint8_t *dav1d_wedge_masks[N_BS_SIZES][3][2][16]; -static void insert_border(uint8_t *const dst, const uint8_t *src, +static void insert_border(uint8_t *const dst, const uint8_t *const src, const int ctr) { if (ctr > 4) memset(dst, 0, ctr - 4); memcpy(dst + imax(ctr, 4) - 4, src + imax(4 - ctr, 0), imin(64 - ctr, 8)); if (ctr < 64 - 4) @@ -154,11 +154,12 @@ dst += w; } } static COLD void init_chroma(uint8_t *chroma, const uint8_t *luma, - const int sign, const int w, const int h, const int ss_ver) + const int sign, const int w, const int h, + const int ss_ver) { for (int y = 0; y < h; y += 1 + ss_ver) { for (int x = 0; x < w; x += 2) { int sum = luma[x] + luma[x + 1] + 1; if (ss_ver) sum += luma[w + x] + luma[w + x + 1] + 1; @@ -271,20 +272,20 @@ fill( 8, 8, 4x8, 4x4, heqw, 0x7bfb); #undef fill } #define N_II_PRED_MODES (N_INTER_INTRA_PRED_MODES - 1) -static uint8_t ALIGN(ii_dc_mask[32 * 32], 32); -static uint8_t ALIGN(ii_nondc_mask_32x32[N_II_PRED_MODES][32 * 32], 32); -static uint8_t ALIGN(ii_nondc_mask_16x32[N_II_PRED_MODES][16 * 32], 32); -static uint8_t ALIGN(ii_nondc_mask_16x16[N_II_PRED_MODES][16 * 16], 32); -static uint8_t ALIGN(ii_nondc_mask_8x32 [N_II_PRED_MODES][ 8 * 32], 32); -static uint8_t ALIGN(ii_nondc_mask_8x16 [N_II_PRED_MODES][ 8 * 16], 32); -static uint8_t ALIGN(ii_nondc_mask_8x8 [N_II_PRED_MODES][ 8 * 8], 32); -static uint8_t ALIGN(ii_nondc_mask_4x16 [N_II_PRED_MODES][ 4 * 16], 32); +static uint8_t ALIGN(ii_dc_mask[32 * 32], 64); +static uint8_t ALIGN(ii_nondc_mask_32x32[N_II_PRED_MODES][32 * 32], 64); +static uint8_t ALIGN(ii_nondc_mask_16x32[N_II_PRED_MODES][16 * 32], 64); +static uint8_t ALIGN(ii_nondc_mask_16x16[N_II_PRED_MODES][16 * 16], 64); +static uint8_t ALIGN(ii_nondc_mask_8x32 [N_II_PRED_MODES][ 8 * 32], 64); +static uint8_t ALIGN(ii_nondc_mask_8x16 [N_II_PRED_MODES][ 8 * 16], 64); +static uint8_t ALIGN(ii_nondc_mask_8x8 [N_II_PRED_MODES][ 8 * 8], 64); +static uint8_t ALIGN(ii_nondc_mask_4x16 [N_II_PRED_MODES][ 4 * 16], 64); static uint8_t ALIGN(ii_nondc_mask_4x8 [N_II_PRED_MODES][ 4 * 8], 32); -static uint8_t ALIGN(ii_nondc_mask_4x4 [N_II_PRED_MODES][ 4 * 4], 32); +static uint8_t ALIGN(ii_nondc_mask_4x4 [N_II_PRED_MODES][ 4 * 4], 16); #undef N_II_PRED_MODES #define set1(sz) \ [II_DC_PRED] = ii_dc_mask, \ [II_VERT_PRED] = ii_nondc_mask_##sz[II_VERT_PRED - 1], \ diff --git a/third_party/dav1d/src/x86/cdef.asm b/third_party/dav1d/src/x86/cdef.asm deleted file mode 100644 --- a/third_party/dav1d/src/x86/cdef.asm +++ /dev/null @@ -1,2001 +0,0 @@ -; Copyright © 2018, VideoLAN and dav1d authors -; Copyright © 2018, Two Orioles, LLC -; All rights reserved. -; -; Redistribution and use in source and binary forms, with or without -; modification, are permitted provided that the following conditions are met: -; -; 1. Redistributions of source code must retain the above copyright notice, this -; list of conditions and the following disclaimer. -; -; 2. Redistributions in binary form must reproduce the above copyright notice, -; this list of conditions and the following disclaimer in the documentation -; and/or other materials provided with the distribution. -; -; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -%include "ext/x86/x86inc.asm" - -%if ARCH_X86_64 - -%macro DUP4 1-* - %rep %0 - times 4 db %1 - %rotate 1 - %endrep -%endmacro - -%macro DIRS 16 ; cdef_directions[] - %rep 4 + 16 + 4 ; 6 7 0 1 2 3 4 5 6 7 0 1 - ; masking away unused bits allows us to use a single vpaddd {1to16} - ; instruction instead of having to do vpbroadcastd + paddb - db %13 & 0x3f, -%13 & 0x3f - %rotate 1 - %endrep -%endmacro - -%macro JMP_TABLE 2-* - %xdefine %1_jmptable %%table - %xdefine %%base mangle(private_prefix %+ _%1_avx2) - %%table: - %rep %0 - 1 - dd %%base %+ .%2 - %%table - %rotate 1 - %endrep -%endmacro - -%macro CDEF_FILTER_JMP_TABLE 1 -JMP_TABLE cdef_filter_%1, \ - d6k0, d6k1, d7k0, d7k1, \ - d0k0, d0k1, d1k0, d1k1, d2k0, d2k1, d3k0, d3k1, \ - d4k0, d4k1, d5k0, d5k1, d6k0, d6k1, d7k0, d7k1, \ - d0k0, d0k1, d1k0, d1k1 -%endmacro - -SECTION_RODATA 64 - -lut_perm_4x4: db 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79 - db 16, 17, 0, 1, 2, 3, 4, 5, 18, 19, 8, 9, 10, 11, 12, 13 - db 20, 21, 80, 81, 82, 83, 84, 85, 22, 23, 32, 33, 34, 35, 36, 37 - db 98, 99,100,101,102,103,104,105, 50, 51, 52, 53, 54, 55, 56, 57 -edge_mask: dq 0x00003c3c3c3c0000, 0x00003f3f3f3f0000 ; 0000, 0001 - dq 0x0000fcfcfcfc0000, 0x0000ffffffff0000 ; 0010, 0011 - dq 0x00003c3c3c3c3c3c, 0x00003f3f3f3f3f3f ; 0100, 0101 - dq 0x0000fcfcfcfcfcfc, 0x0000ffffffffffff ; 0110, 0111 - dq 0x3c3c3c3c3c3c0000, 0x3f3f3f3f3f3f0000 ; 1000, 1001 - dq 0xfcfcfcfcfcfc0000, 0xffffffffffff0000 ; 1010, 1011 - dq 0x3c3c3c3c3c3c3c3c, 0x3f3f3f3f3f3f3f3f ; 1100, 1101 - dq 0xfcfcfcfcfcfcfcfc, 0xffffffffffffffff ; 1110, 1111 -px_idx: DUP4 18, 19, 20, 21, 26, 27, 28, 29, 34, 35, 36, 37, 42, 43, 44, 45 -cdef_dirs: DIRS -7,-14, 1, -6, 1, 2, 1, 10, 9, 18, 8, 17, 8, 16, 8, 15 -gf_shr: dq 0x0102040810204080, 0x0102040810204080 ; >> 0, >> 0 - dq 0x0204081020408000, 0x0408102040800000 ; >> 1, >> 2 - dq 0x0810204080000000, 0x1020408000000000 ; >> 3, >> 4 - dq 0x2040800000000000, 0x4080000000000000 ; >> 5, >> 6 -end_perm: db 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61 -pri_tap: db 64, 64, 32, 32, 48, 48, 48, 48 ; left-shifted by 4 -sec_tap: db 32, 32, 16, 16 -pd_268435568: dd 268435568 -blend_4x4: dd 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00 - dd 0x80, 0x00, 0x00 -blend_4x8_0: dd 0x00, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80 -blend_4x8_1: dd 0x00, 0x00, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80 - dd 0x00, 0x00 -blend_4x8_2: dd 0x0000, 0x8080, 0x8080, 0x8080, 0x8080, 0x8080, 0x8080, 0x8080 - dd 0x0000 -blend_4x8_3: dd 0x0000, 0x0000, 0x8080, 0x8080, 0x8080, 0x8080, 0x8080, 0x8080 - dd 0x0000, 0x0000 -blend_8x8_0: dq 0x00, 0x00, 0x80, 0x80, 0x80, 0x80 -blend_8x8_1: dq 0x0000, 0x0000, 0x8080, 0x8080, 0x8080, 0x8080, 0x0000, 0x0000 -pd_47130256: dd 4, 7, 1, 3, 0, 2, 5, 6 -div_table: dd 840, 420, 280, 210, 168, 140, 120, 105, 420, 210, 140, 105 -shufw_6543210x:db 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1, 14, 15 -shufb_lohi: db 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15 -pw_128: times 2 dw 128 -pw_2048: times 2 dw 2048 -tap_table: ; masks for 8 bit shifts - db 0xFF, 0x7F, 0x3F, 0x1F, 0x0F, 0x07, 0x03, 0x01 - ; weights - db 4, 2, 3, 3, 2, 1 - db -1 * 16 + 1, -2 * 16 + 2 - db 0 * 16 + 1, -1 * 16 + 2 - db 0 * 16 + 1, 0 * 16 + 2 - db 0 * 16 + 1, 1 * 16 + 2 - db 1 * 16 + 1, 2 * 16 + 2 - db 1 * 16 + 0, 2 * 16 + 1 - db 1 * 16 + 0, 2 * 16 + 0 - db 1 * 16 + 0, 2 * 16 - 1 - ; the last 6 are repeats of the first 6 so we don't need to & 7 - db -1 * 16 + 1, -2 * 16 + 2 - db 0 * 16 + 1, -1 * 16 + 2 - db 0 * 16 + 1, 0 * 16 + 2 - db 0 * 16 + 1, 1 * 16 + 2 - db 1 * 16 + 1, 2 * 16 + 2 - db 1 * 16 + 0, 2 * 16 + 1 - -CDEF_FILTER_JMP_TABLE 4x4 -CDEF_FILTER_JMP_TABLE 4x8 -CDEF_FILTER_JMP_TABLE 8x8 - -SECTION .text - -%macro PREP_REGS 2 ; w, h - ; off1/2/3[k] [6 total] from [tapq+12+(dir+0/2/6)*2+k] - mov dird, r6m - lea tableq, [cdef_filter_%1x%2_jmptable] - lea dirq, [tableq+dirq*2*4] -%if %1 == 4 - %if %2 == 4 - DEFINE_ARGS dst, stride, left, top, pri, sec, \ - table, dir, dirjmp, dst4, stride3, k - %else - DEFINE_ARGS dst, stride, left, top, pri, sec, \ - table, dir, dirjmp, dst4, dst8, stride3, k - lea dst8q, [dstq+strideq*8] - %endif -%else - DEFINE_ARGS dst, stride, h, top1, pri, sec, \ - table, dir, dirjmp, top2, dst4, stride3, k - mov hq, -8 - lea top1q, [top1q+strideq*0] - lea top2q, [top1q+strideq*1] -%endif - lea dst4q, [dstq+strideq*4] -%if %1 == 4 - lea stride3q, [strideq*3] -%endif -%endmacro - -%macro LOAD_BLOCK 2-3 0 ; w, h, init_min_max - mov kd, 1 - pxor m15, m15 ; sum -%if %2 == 8 - pxor m12, m12 - %if %1 == 4 - movd xm4, [dstq +strideq*0] - movd xm6, [dstq +strideq*1] - movd xm5, [dstq +strideq*2] - movd xm7, [dstq +stride3q ] - vinserti128 m4, [dst4q+strideq*0], 1 - vinserti128 m6, [dst4q+strideq*1], 1 - vinserti128 m5, [dst4q+strideq*2], 1 - vinserti128 m7, [dst4q+stride3q ], 1 - punpckldq m4, m6 - punpckldq m5, m7 - %else - movq xm4, [dstq+strideq*0] - movq xm5, [dstq+strideq*1] - vinserti128 m4, [dstq+strideq*2], 1 - vinserti128 m5, [dstq+stride3q ], 1 - %endif - punpcklqdq m4, m5 -%else - movd xm4, [dstq+strideq*0] - movd xm5, [dstq+strideq*1] - vinserti128 m4, [dstq+strideq*2], 1 - vinserti128 m5, [dstq+stride3q ], 1 - punpckldq m4, m5 -%endif -%if %3 == 1 - mova m7, m4 ; min - mova m8, m4 ; max -%endif -%endmacro - -%macro ACCUMULATE_TAP_BYTE 7-8 0 ; tap_offset, shift, mask, strength - ; mul_tap, w, h, clip - ; load p0/p1 - movsxd dirjmpq, [dirq+kq*4+%1*2*4] - add dirjmpq, tableq - call dirjmpq - -%if %8 == 1 - pmaxub m7, m5 - pminub m8, m5 - pmaxub m7, m6 - pminub m8, m6 -%endif - - ; accumulate sum[m15] over p0/p1 -%if %7 == 4 - punpcklbw m5, m6 - punpcklbw m6, m4, m4 - psubusb m9, m5, m6 - psubusb m5, m6, m5 - por m9, m5 ; abs_diff_p01(p01 - px) - pcmpeqb m5, m9 - por m5, %5 - psignb m6, %5, m5 - psrlw m5, m9, %2 ; emulate 8-bit shift - pand m5, %3 - psubusb m5, %4, m5 - pminub m5, m9 - pmaddubsw m5, m6 - paddw m15, m5 -%else - psubusb m9, m5, m4 - psubusb m5, m4, m5 - psubusb m11, m6, m4 - psubusb m6, m4, m6 - por m9, m5 ; abs_diff_p0(p0 - px) - por m11, m6 ; abs_diff_p1(p1 - px) - pcmpeqb m5, m9 - pcmpeqb m6, m11 - punpckhbw m10, m9, m11 - punpcklbw m9, m11 - por m5, %5 - por m11, m6, %5 - punpckhbw m6, m5, m11 - punpcklbw m5, m11 - psignb m11, %5, m6 - psrlw m6, m10, %2 ; emulate 8-bit shift - pand m6, %3 - psubusb m6, %4, m6 - pminub m6, m10 - pmaddubsw m6, m11 - paddw m12, m6 - psignb m11, %5, m5 - psrlw m5, m9, %2 ; emulate 8-bit shift - pand m5, %3 - psubusb m5, %4, m5 - pminub m5, m9 - pmaddubsw m5, m11 - paddw m15, m5 -%endif -%endmacro - -%macro ADJUST_PIXEL 4-5 0 ; w, h, zero, pw_2048, clip -%if %2 == 4 - %if %5 == 1 - punpcklbw m4, %3 - %endif - pcmpgtw %3, m15 - paddw m15, %3 - pmulhrsw m15, %4 - %if %5 == 0 - packsswb m15, m15 - paddb m4, m15 - %else - paddw m4, m15 - packuswb m4, m4 ; clip px in [0x0,0xff] - pminub m4, m7 - pmaxub m4, m8 - %endif - vextracti128 xm5, m4, 1 - movd [dstq+strideq*0], xm4 - movd [dstq+strideq*2], xm5 - pextrd [dstq+strideq*1], xm4, 1 - pextrd [dstq+stride3q ], xm5, 1 -%else - pcmpgtw m6, %3, m12 - pcmpgtw m5, %3, m15 - paddw m12, m6 - paddw m15, m5 - %if %5 == 1 - punpckhbw m5, m4, %3 - punpcklbw m4, %3 - %endif - pmulhrsw m12, %4 - pmulhrsw m15, %4 - %if %5 == 0 - packsswb m15, m12 - paddb m4, m15 - %else - paddw m5, m12 - paddw m4, m15 - packuswb m4, m5 ; clip px in [0x0,0xff] - pminub m4, m7 - pmaxub m4, m8 - %endif - vextracti128 xm5, m4, 1 - %if %1 == 4 - movd [dstq +strideq*0], xm4 - movd [dst4q+strideq*0], xm5 - pextrd [dstq +strideq*1], xm4, 1 - pextrd [dst4q+strideq*1], xm5, 1 - pextrd [dstq +strideq*2], xm4, 2 - pextrd [dst4q+strideq*2], xm5, 2 - pextrd [dstq +stride3q ], xm4, 3 - pextrd [dst4q+stride3q ], xm5, 3 - %else - movq [dstq+strideq*0], xm4 - movq [dstq+strideq*2], xm5 - movhps [dstq+strideq*1], xm4 - movhps [dstq+stride3q ], xm5 - %endif -%endif -%endmacro - -%macro BORDER_PREP_REGS 2 ; w, h - ; off1/2/3[k] [6 total] from [tapq+12+(dir+0/2/6)*2+k] - mov dird, r6m - lea dirq, [tableq+dirq*2+14] -%if %1*%2*2/mmsize > 1 - %if %1 == 4 - DEFINE_ARGS dst, stride, dir, stk, pri, sec, stride3, h, off, k - %else - DEFINE_ARGS dst, stride, dir, stk, pri, sec, h, off, k - %endif - mov hd, %1*%2*2/mmsize -%else - DEFINE_ARGS dst, stride, dir, stk, pri, sec, stride3, off, k -%endif - lea stkq, [px] - pxor m11, m11 -%endmacro - -%macro BORDER_LOAD_BLOCK 2-3 0 ; w, h, init_min_max - mov kd, 1 -%if %1 == 4 - movq xm4, [stkq+32*0] - movhps xm4, [stkq+32*1] - movq xm5, [stkq+32*2] - movhps xm5, [stkq+32*3] - vinserti128 m4, xm5, 1 -%else - mova xm4, [stkq+32*0] ; px - vinserti128 m4, [stkq+32*1], 1 -%endif - pxor m15, m15 ; sum -%if %3 == 1 - mova m7, m4 ; max - mova m8, m4 ; min -%endif -%endmacro - -%macro ACCUMULATE_TAP_WORD 6-7 0 ; tap_offset, shift, mask, strength - ; mul_tap, w, clip - ; load p0/p1 - movsx offq, byte [dirq+kq+%1] ; off1 -%if %6 == 4 - movq xm5, [stkq+offq*2+32*0] ; p0 - movq xm6, [stkq+offq*2+32*2] - movhps xm5, [stkq+offq*2+32*1] - movhps xm6, [stkq+offq*2+32*3] - vinserti128 m5, xm6, 1 -%else - movu xm5, [stkq+offq*2+32*0] ; p0 - vinserti128 m5, [stkq+offq*2+32*1], 1 -%endif - neg offq ; -off1 -%if %6 == 4 - movq xm6, [stkq+offq*2+32*0] ; p1 - movq xm9, [stkq+offq*2+32*2] - movhps xm6, [stkq+offq*2+32*1] - movhps xm9, [stkq+offq*2+32*3] - vinserti128 m6, xm9, 1 -%else - movu xm6, [stkq+offq*2+32*0] ; p1 - vinserti128 m6, [stkq+offq*2+32*1], 1 -%endif -%if %7 == 1 - ; out of bounds values are set to a value that is a both a large unsigned - ; value and a negative signed value. - ; use signed max and unsigned min to remove them - pmaxsw m7, m5 ; max after p0 - pminuw m8, m5 ; min after p0 - pmaxsw m7, m6 ; max after p1 - pminuw m8, m6 ; min after p1 -%endif - - ; accumulate sum[m15] over p0/p1 - ; calculate difference before converting - psubw m5, m4 ; diff_p0(p0 - px) - psubw m6, m4 ; diff_p1(p1 - px) - - ; convert to 8-bits with signed saturation - ; saturating to large diffs has no impact on the results - packsswb m5, m6 - - ; group into pairs so we can accumulate using maddubsw - pshufb m5, m12 - pabsb m9, m5 - psignb m10, %5, m5 - psrlw m5, m9, %2 ; emulate 8-bit shift - pand m5, %3 - psubusb m5, %4, m5 - - ; use unsigned min since abs diff can equal 0x80 - pminub m5, m9 - pmaddubsw m5, m10 - paddw m15, m5 -%endmacro - -%macro BORDER_ADJUST_PIXEL 2-3 0 ; w, pw_2048, clip - pcmpgtw m9, m11, m15 - paddw m15, m9 - pmulhrsw m15, %2 - paddw m4, m15 -%if %3 == 1 - pminsw m4, m7 - pmaxsw m4, m8 -%endif - packuswb m4, m4 - vextracti128 xm5, m4, 1 -%if %1 == 4 - movd [dstq+strideq*0], xm4 - pextrd [dstq+strideq*1], xm4, 1 - movd [dstq+strideq*2], xm5 - pextrd [dstq+stride3q], xm5, 1 -%else - movq [dstq+strideq*0], xm4 - movq [dstq+strideq*1], xm5 -%endif -%endmacro - -%macro CDEF_FILTER 2 ; w, h -INIT_YMM avx2 -cglobal cdef_filter_%1x%2, 4, 9, 0, dst, stride, left, top, \ - pri, sec, dir, damping, edge -%assign stack_offset_entry stack_offset - mov edged, edgem - cmp edged, 0xf - jne .border_block - - PUSH r9 - PUSH r10 - PUSH r11 -%if %2 == 4 - %assign regs_used 12 - %if STACK_ALIGNMENT < 32 - PUSH r%+regs_used - %assign regs_used regs_used+1 - %endif - ALLOC_STACK 0x60, 16 - pmovzxbw xm0, [leftq+1] - vpermq m0, m0, q0110 - psrldq m1, m0, 4 - vpalignr m2, m0, m0, 12 - movu [rsp+0x10], m0 - movu [rsp+0x28], m1 - movu [rsp+0x40], m2 -%elif %1 == 4 - PUSH r12 - %assign regs_used 13 - %if STACK_ALIGNMENT < 32 - PUSH r%+regs_used - %assign regs_used regs_used+1 - %endif - ALLOC_STACK 8*2+%1*%2*1, 16 - pmovzxwd m0, [leftq] - mova [rsp+0x10], m0 -%else - PUSH r12 - PUSH r13 - %assign regs_used 14 - %if STACK_ALIGNMENT < 32 - PUSH r%+regs_used - %assign regs_used regs_used+1 - %endif - ALLOC_STACK 8*2+%1*%2*2+32, 16 - lea r11, [strideq*3] - movu xm4, [dstq+strideq*2] - pmovzxwq m0, [leftq+0] - pmovzxwq m1, [leftq+8] - vinserti128 m4, [dstq+r11], 1 - pmovzxbd m2, [leftq+1] - pmovzxbd m3, [leftq+9] - mova [rsp+0x10], m0 - mova [rsp+0x30], m1 - mova [rsp+0x50], m2 - mova [rsp+0x70], m3 - mova [rsp+0x90], m4 -%endif - - DEFINE_ARGS dst, stride, left, top, pri, secdmp, zero, pridmp, damping - mov dampingd, r7m - xor zerod, zerod - movifnidn prid, prim - sub dampingd, 31 - movifnidn secdmpd, secdmpm - or prid, 0 - jz .sec_only - movd xm0, prid - lzcnt pridmpd, prid - add pridmpd, dampingd - cmovs pridmpd, zerod - mov [rsp+0], pridmpq ; pri_shift - or secdmpd, 0 - jz .pri_only - movd xm1, secdmpd - lzcnt secdmpd, secdmpd - add secdmpd, dampingd - cmovs secdmpd, zerod - mov [rsp+8], secdmpq ; sec_shift - - DEFINE_ARGS dst, stride, left, top, pri, secdmp, table, pridmp - lea tableq, [tap_table] - vpbroadcastb m13, [tableq+pridmpq] ; pri_shift_mask - vpbroadcastb m14, [tableq+secdmpq] ; sec_shift_mask - - ; pri/sec_taps[k] [4 total] - DEFINE_ARGS dst, stride, left, top, pri, sec, table, dir - vpbroadcastb m0, xm0 ; pri_strength - vpbroadcastb m1, xm1 ; sec_strength - and prid, 1 - lea priq, [tableq+priq*2+8] ; pri_taps - lea secq, [tableq+12] ; sec_taps - - PREP_REGS %1, %2 -%if %1*%2 > mmsize -.v_loop: -%endif - LOAD_BLOCK %1, %2, 1 -.k_loop: - vpbroadcastb m2, [priq+kq] ; pri_taps - vpbroadcastb m3, [secq+kq] ; sec_taps - ACCUMULATE_TAP_BYTE 2, [rsp+0], m13, m0, m2, %1, %2, 1 ; dir + 0 - ACCUMULATE_TAP_BYTE 4, [rsp+8], m14, m1, m3, %1, %2, 1 ; dir + 2 - ACCUMULATE_TAP_BYTE 0, [rsp+8], m14, m1, m3, %1, %2, 1 ; dir - 2 - dec kq - jge .k_loop - - vpbroadcastd m10, [pw_2048] - pxor m9, m9 - ADJUST_PIXEL %1, %2, m9, m10, 1 -%if %1*%2 > mmsize - mov dstq, dst4q - lea top1q, [rsp+0x90] - lea top2q, [rsp+0xA0] - lea dst4q, [dst4q+strideq*4] - add hq, 4 - jl .v_loop -%endif - RET - -.pri_only: - DEFINE_ARGS dst, stride, left, top, pri, _, table, pridmp - lea tableq, [tap_table] - vpbroadcastb m13, [tableq+pridmpq] ; pri_shift_mask - ; pri/sec_taps[k] [4 total] - DEFINE_ARGS dst, stride, left, top, pri, _, table, dir - vpbroadcastb m0, xm0 ; pri_strength - and prid, 1 - lea priq, [tableq+priq*2+8] ; pri_taps - PREP_REGS %1, %2 - vpbroadcastd m3, [pw_2048] - pxor m1, m1 -%if %1*%2 > mmsize -.pri_v_loop: -%endif - LOAD_BLOCK %1, %2 -.pri_k_loop: - vpbroadcastb m2, [priq+kq] ; pri_taps - ACCUMULATE_TAP_BYTE 2, [rsp+0], m13, m0, m2, %1, %2 ; dir + 0 - dec kq - jge .pri_k_loop - ADJUST_PIXEL %1, %2, m1, m3 -%if %1*%2 > mmsize - mov dstq, dst4q - lea top1q, [rsp+0x90] - lea top2q, [rsp+0xA0] - lea dst4q, [dst4q+strideq*4] - add hq, 4 - jl .pri_v_loop -%endif - RET - -.sec_only: - DEFINE_ARGS dst, stride, left, top, _, secdmp, zero, _, damping - movd xm1, secdmpd - lzcnt secdmpd, secdmpd - add secdmpd, dampingd - cmovs secdmpd, zerod - mov [rsp+8], secdmpq ; sec_shift - DEFINE_ARGS dst, stride, left, top, _, secdmp, table - lea tableq, [tap_table] - vpbroadcastb m14, [tableq+secdmpq] ; sec_shift_mask - ; pri/sec_taps[k] [4 total] - DEFINE_ARGS dst, stride, left, top, _, sec, table, dir - vpbroadcastb m1, xm1 ; sec_strength - lea secq, [tableq+12] ; sec_taps - PREP_REGS %1, %2 - vpbroadcastd m2, [pw_2048] - pxor m0, m0 -%if %1*%2 > mmsize -.sec_v_loop: -%endif - LOAD_BLOCK %1, %2 -.sec_k_loop: - vpbroadcastb m3, [secq+kq] ; sec_taps - ACCUMULATE_TAP_BYTE 4, [rsp+8], m14, m1, m3, %1, %2 ; dir + 2 - ACCUMULATE_TAP_BYTE 0, [rsp+8], m14, m1, m3, %1, %2 ; dir - 2 - dec kq - jge .sec_k_loop - ADJUST_PIXEL %1, %2, m0, m2 -%if %1*%2 > mmsize - mov dstq, dst4q - lea top1q, [rsp+0x90] - lea top2q, [rsp+0xA0] - lea dst4q, [dst4q+strideq*4] - add hq, 4 - jl .sec_v_loop -%endif - RET - -.d0k0: -%if %1 == 4 - %if %2 == 4 - vpbroadcastq m6, [dstq+strideq*1-1] - vpbroadcastq m10, [dstq+strideq*2-1] - movd xm5, [topq+strideq*1+1] - movd xm9, [dstq+strideq*0+1] - psrldq m11, m6, 2 - psrldq m12, m10, 2 - vinserti128 m6, [dstq+stride3q -1], 1 - vinserti128 m10, [dstq+strideq*4-1], 1 - vpblendd m5, m11, 0x10 - vpblendd m9, m12, 0x10 - movu m11, [blend_4x4+16] - punpckldq m6, m10 - punpckldq m5, m9 - vpblendvb m6, [rsp+gprsize+0x28], m11 - %else - movd xm5, [topq +strideq*1+1] - movq xm6, [dstq +strideq*1-1] - movq xm10, [dstq +stride3q -1] - movq xm11, [dst4q+strideq*1-1] - pinsrd xm5, [dstq +strideq*0+1], 1 - movhps xm6, [dstq +strideq*2-1] - movhps xm10, [dst4q+strideq*0-1] - movhps xm11, [dst4q+strideq*2-1] - psrldq xm9, xm6, 2 - shufps xm5, xm9, q2010 ; -1 +0 +1 +2 - shufps xm6, xm10, q2020 ; +1 +2 +3 +4 - psrldq xm9, xm11, 2 - psrldq xm10, 2 - shufps xm10, xm9, q2020 ; +3 +4 +5 +6 - movd xm9, [dst4q+stride3q -1] - pinsrd xm9, [dst4q+strideq*4-1], 1 - shufps xm11, xm9, q1020 ; +5 +6 +7 +8 - pmovzxbw m9, [leftq+3] - vinserti128 m6, xm11, 1 - movu m11, [blend_4x8_0+4] - vinserti128 m5, xm10, 1 - vpblendvb m6, m9, m11 - %endif -%else - lea r13, [blend_8x8_0+16] - movq xm5, [top2q +1] - vbroadcasti128 m10, [dstq+strideq*1-1] - vbroadcasti128 m11, [dstq+strideq*2-1] - movhps xm5, [dstq+strideq*0+1] - vinserti128 m6, m10, [dstq+stride3q -1], 1 - vinserti128 m9, m11, [dstq+strideq*4-1], 1 - psrldq m10, 2 - psrldq m11, 2 - punpcklqdq m6, m9 - movu m9, [r13+hq*2*1+16*1] - punpcklqdq m10, m11 - vpblendd m5, m10, 0xF0 - vpblendvb m6, [rsp+gprsize+80+hq*8+64+8*1], m9 -%endif - ret -.d1k0: -.d2k0: -.d3k0: -%if %1 == 4 - %if %2 == 4 - movq xm6, [dstq+strideq*0-1] - movq xm9, [dstq+strideq*1-1] - vinserti128 m6, [dstq+strideq*2-1], 1 - vinserti128 m9, [dstq+stride3q -1], 1 - movu m11, [rsp+gprsize+0x10] - pcmpeqd m12, m12 - psrldq m5, m6, 2 - psrldq m10, m9, 2 - psrld m12, 24 - punpckldq m6, m9 - punpckldq m5, m10 - vpblendvb m6, m11, m12 - %else - movq xm6, [dstq +strideq*0-1] - movq xm9, [dstq +strideq*2-1] - movhps xm6, [dstq +strideq*1-1] - movhps xm9, [dstq +stride3q -1] - movq xm10, [dst4q+strideq*0-1] - movhps xm10, [dst4q+strideq*1-1] - psrldq xm5, xm6, 2 - psrldq xm11, xm9, 2 - shufps xm5, xm11, q2020 - movq xm11, [dst4q+strideq*2-1] - movhps xm11, [dst4q+stride3q -1] - shufps xm6, xm9, q2020 - shufps xm9, xm10, xm11, q2020 - vinserti128 m6, xm9, 1 - pmovzxbw m9, [leftq+1] - psrldq xm10, 2 - psrldq xm11, 2 - shufps xm10, xm11, q2020 - vpbroadcastd m11, [blend_4x8_0+4] - vinserti128 m5, xm10, 1 - vpblendvb m6, m9, m11 - %endif -%else - movu xm5, [dstq+strideq*0-1] - movu xm9, [dstq+strideq*1-1] - vinserti128 m5, [dstq+strideq*2-1], 1 - vinserti128 m9, [dstq+stride3q -1], 1 - mova m10, [blend_8x8_0+16] - punpcklqdq m6, m5, m9 - vpblendvb m6, [rsp+gprsize+80+hq*8+64], m10 - psrldq m5, 2 - psrldq m9, 2 - punpcklqdq m5, m9 -%endif - ret -.d4k0: -%if %1 == 4 - %if %2 == 4 - vpbroadcastq m10, [dstq+strideq*1-1] - vpbroadcastq m11, [dstq+strideq*2-1] - movd xm6, [topq+strideq*1-1] - movd xm9, [dstq+strideq*0-1] - psrldq m5, m10, 2 - psrldq m12, m11, 2 - vpblendd m6, m10, 0x10 - vpblendd m9, m11, 0x10 - movu m10, [blend_4x4] - vinserti128 m5, [dstq+stride3q +1], 1 - vinserti128 m12, [dstq+strideq*4+1], 1 - punpckldq m6, m9 - punpckldq m5, m12 - vpblendvb m6, [rsp+gprsize+0x40], m10 - %else - movd xm6, [topq +strideq*1-1] - movq xm9, [dstq +strideq*1-1] - movq xm10, [dstq +stride3q -1] - movq xm11, [dst4q+strideq*1-1] - pinsrd xm6, [dstq +strideq*0-1], 1 - movhps xm9, [dstq +strideq*2-1] - movhps xm10, [dst4q+strideq*0-1] - movhps xm11, [dst4q+strideq*2-1] - psrldq xm5, xm9, 2 - shufps xm6, xm9, q2010 - psrldq xm9, xm10, 2 - shufps xm5, xm9, q2020 - shufps xm10, xm11, q2020 - movd xm9, [dst4q+stride3q +1] - vinserti128 m6, xm10, 1 - pinsrd xm9, [dst4q+strideq*4+1], 1 - psrldq xm11, 2 - pmovzxbw m10, [leftq-1] - shufps xm11, xm9, q1020 - movu m9, [blend_4x8_0] - vinserti128 m5, xm11, 1 - vpblendvb m6, m10, m9 - %endif -%else - lea r13, [blend_8x8_0+8] - movq xm6, [top2q -1] - vbroadcasti128 m5, [dstq+strideq*1-1] - vbroadcasti128 m9, [dstq+strideq*2-1] - movhps xm6, [dstq+strideq*0-1] - movu m11, [r13+hq*2*1+16*1] - punpcklqdq m10, m5, m9 - vinserti128 m5, [dstq+stride3q -1], 1 - vinserti128 m9, [dstq+strideq*4-1], 1 - vpblendd m6, m10, 0xF0 - vpblendvb m6, [rsp+gprsize+80+hq*8+64-8*1], m11 - psrldq m5, 2 - psrldq m9, 2 - punpcklqdq m5, m9 -%endif - ret -.d5k0: -.d6k0: -.d7k0: -%if %1 == 4 - %if %2 == 4 - movd xm6, [topq+strideq*1 ] - vpbroadcastd m5, [dstq+strideq*1 ] - vpbroadcastd m9, [dstq+strideq*2 ] - vpblendd xm6, [dstq+strideq*0-4], 0x2 - vpblendd m5, m9, 0x22 - vpblendd m6, m5, 0x30 - vinserti128 m5, [dstq+stride3q ], 1 - vpblendd m5, [dstq+strideq*4-20], 0x20 - %else - movd xm6, [topq +strideq*1] - movd xm5, [dstq +strideq*1] - movd xm9, [dstq +stride3q ] - movd xm10, [dst4q+strideq*1] - movd xm11, [dst4q+stride3q ] - pinsrd xm6, [dstq +strideq*0], 1 - pinsrd xm5, [dstq +strideq*2], 1 - pinsrd xm9, [dst4q+strideq*0], 1 - pinsrd xm10, [dst4q+strideq*2], 1 - pinsrd xm11, [dst4q+strideq*4], 1 - punpcklqdq xm6, xm5 - punpcklqdq xm5, xm9 - punpcklqdq xm9, xm10 - punpcklqdq xm10, xm11 - vinserti128 m6, xm9, 1 - vinserti128 m5, xm10, 1 - %endif -%else - movq xm6, [top2q ] - movq xm5, [dstq+strideq*1] - movq xm9, [dstq+stride3q ] - movhps xm6, [dstq+strideq*0] - movhps xm5, [dstq+strideq*2] - movhps xm9, [dstq+strideq*4] - vinserti128 m6, xm5, 1 - vinserti128 m5, xm9, 1 -%endif - ret -.d0k1: -%if %1 == 4 - %if %2 == 4 - movd xm6, [dstq +strideq*2-2] - movd xm9, [dstq +stride3q -2] - movd xm5, [topq +strideq*0+2] - movd xm10, [topq +strideq*1+2] - pinsrw xm6, [leftq+4], 0 - pinsrw xm9, [leftq+6], 0 - vinserti128 m5, [dstq +strideq*0+2], 1 - vinserti128 m10, [dstq +strideq*1+2], 1 - vinserti128 m6, [dst4q+strideq*0-2], 1 - vinserti128 m9, [dst4q+strideq*1-2], 1 - punpckldq m5, m10 - punpckldq m6, m9 - %else - movq xm6, [dstq +strideq*2-2] - movd xm10, [dst4q+strideq*2-2] - movd xm5, [topq +strideq*0+2] - movq xm9, [dst4q+strideq*0-2] - movhps xm6, [dstq +stride3q -2] - pinsrw xm10, [dst4q+stride3q ], 3 - pinsrd xm5, [topq +strideq*1+2], 1 - movhps xm9, [dst4q+strideq*1-2] - pinsrd xm10, [dst8q+strideq*0-2], 2 - pinsrd xm5, [dstq +strideq*0+2], 2 - pinsrd xm10, [dst8q+strideq*1-2], 3 - pinsrd xm5, [dstq +strideq*1+2], 3 - shufps xm11, xm6, xm9, q3131 - shufps xm6, xm9, q2020 - movu m9, [blend_4x8_3+8] - vinserti128 m6, xm10, 1 - vinserti128 m5, xm11, 1 - vpblendvb m6, [rsp+gprsize+16+8], m9 - %endif -%else - lea r13, [blend_8x8_1+16] - movq xm6, [dstq +strideq*2-2] - movq xm9, [dstq +stride3q -2] - movq xm5, [top1q +2] - movq xm10, [top2q +2] - movu m11, [r13+hq*2*2+16*2] - vinserti128 m6, [dst4q+strideq*0-2], 1 - vinserti128 m9, [dst4q+strideq*1-2], 1 - vinserti128 m5, [dstq +strideq*0+2], 1 - vinserti128 m10, [dstq +strideq*1+2], 1 - punpcklqdq m6, m9 - punpcklqdq m5, m10 - vpblendvb m6, [rsp+gprsize+16+hq*8+64+8*2], m11 -%endif - ret -.d1k1: -%if %1 == 4 - %if %2 == 4 - vpbroadcastq m6, [dstq+strideq*1-2] - vpbroadcastq m9, [dstq+strideq*2-2] - movd xm5, [topq+strideq*1+2] - movd xm10, [dstq+strideq*0+2] - psrldq m11, m6, 4 - psrldq m12, m9, 4 - vpblendd m5, m11, 0x10 - movq xm11, [leftq+2] - vinserti128 m6, [dstq+stride3q -2], 1 - punpckldq xm11, xm11 - vpblendd m10, m12, 0x10 - pcmpeqd m12, m12 - pmovzxwd m11, xm11 - psrld m12, 16 - punpckldq m6, m9 - vpbroadcastd m9, [dstq+strideq*4-2] - vpblendvb m6, m11, m12 - punpckldq m5, m10 - vpblendd m6, m9, 0x20 - %else - movd xm5, [topq +strideq*1+2] - movq xm6, [dstq +strideq*1-2] - movq xm9, [dstq +stride3q -2] - movq xm10, [dst4q+strideq*1-2] - movd xm11, [dst4q+stride3q -2] - pinsrd xm5, [dstq +strideq*0+2], 1 - movhps xm6, [dstq +strideq*2-2] - movhps xm9, [dst4q+strideq*0-2] - movhps xm10, [dst4q+strideq*2-2] - pinsrd xm11, [dst4q+strideq*4-2], 1 - shufps xm5, xm6, q3110 - shufps xm6, xm9, q2020 - shufps xm9, xm10, q3131 - shufps xm10, xm11, q1020 - movu m11, [blend_4x8_2+4] - vinserti128 m6, xm10, 1 - vinserti128 m5, xm9, 1 - vpblendvb m6, [rsp+gprsize+16+4], m11 - %endif -%else - lea r13, [blend_8x8_1+16] - movq xm5, [top2q +2] - vbroadcasti128 m6, [dstq+strideq*1-2] - vbroadcasti128 m9, [dstq+strideq*2-2] - movhps xm5, [dstq+strideq*0+2] - shufps m10, m6, m9, q2121 - vinserti128 m6, [dstq+stride3q -2], 1 - vinserti128 m9, [dstq+strideq*4-2], 1 - movu m11, [r13+hq*2*1+16*1] - vpblendd m5, m10, 0xF0 - punpcklqdq m6, m9 - vpblendvb m6, [rsp+gprsize+16+hq*8+64+8*1], m11 -%endif - ret -.d2k1: -%if %1 == 4 - %if %2 == 4 - movq xm11, [leftq] - movq xm6, [dstq+strideq*0-2] - movq xm9, [dstq+strideq*1-2] - vinserti128 m6, [dstq+strideq*2-2], 1 - vinserti128 m9, [dstq+stride3q -2], 1 - punpckldq xm11, xm11 - psrldq m5, m6, 4 - psrldq m10, m9, 4 - pmovzxwd m11, xm11 - punpckldq m6, m9 - punpckldq m5, m10 - pblendw m6, m11, 0x05 - %else - movq xm5, [dstq +strideq*0-2] - movq xm9, [dstq +strideq*2-2] - movq xm10, [dst4q+strideq*0-2] - movq xm11, [dst4q+strideq*2-2] - movhps xm5, [dstq +strideq*1-2] - movhps xm9, [dstq +stride3q -2] - movhps xm10, [dst4q+strideq*1-2] - movhps xm11, [dst4q+stride3q -2] - shufps xm6, xm5, xm9, q2020 - shufps xm5, xm9, q3131 - shufps xm9, xm10, xm11, q2020 - shufps xm10, xm11, q3131 - pmovzxwd m11, [leftq] - vinserti128 m6, xm9, 1 - vinserti128 m5, xm10, 1 - pblendw m6, m11, 0x55 - %endif -%else - mova m11, [rsp+gprsize+16+hq*8+64] - movu xm5, [dstq+strideq*0-2] - movu xm9, [dstq+strideq*1-2] - vinserti128 m5, [dstq+strideq*2-2], 1 - vinserti128 m9, [dstq+stride3q -2], 1 - shufps m6, m5, m9, q1010 - shufps m5, m9, q2121 - pblendw m6, m11, 0x11 -%endif - ret -.d3k1: -%if %1 == 4 - %if %2 == 4 - vpbroadcastq m11, [dstq+strideq*1-2] - vpbroadcastq m12, [dstq+strideq*2-2] - movd xm6, [topq+strideq*1-2] - movd xm9, [dstq+strideq*0-2] - pblendw m11, [leftq-16+2], 0x01 - pblendw m12, [leftq-16+4], 0x01 - pinsrw xm9, [leftq- 0+0], 0 - psrldq m5, m11, 4 - psrldq m10, m12, 4 - vinserti128 m5, [dstq+stride3q +2], 1 - vinserti128 m10, [dstq+strideq*4+2], 1 - vpblendd m6, m11, 0x10 - vpblendd m9, m12, 0x10 - punpckldq m6, m9 - punpckldq m5, m10 - %else - movd xm6, [topq +strideq*1-2] - movq xm5, [dstq +strideq*1-2] - movq xm9, [dstq +stride3q -2] - movq xm10, [dst4q+strideq*1-2] - movd xm11, [dst4q+stride3q +2] - pinsrw xm6, [dstq +strideq*0 ], 3 - movhps xm5, [dstq +strideq*2-2] - movhps xm9, [dst4q+strideq*0-2] - movhps xm10, [dst4q+strideq*2-2] - pinsrd xm11, [dst4q+strideq*4+2], 1 - shufps xm6, xm5, q2010 - shufps xm5, xm9, q3131 - shufps xm9, xm10, q2020 - shufps xm10, xm11, q1031 - movu m11, [blend_4x8_2] - vinserti128 m6, xm9, 1 - vinserti128 m5, xm10, 1 - vpblendvb m6, [rsp+gprsize+16-4], m11 - %endif -%else - lea r13, [blend_8x8_1+8] - movq xm6, [top2q -2] - vbroadcasti128 m5, [dstq+strideq*1-2] - vbroadcasti128 m10, [dstq+strideq*2-2] - movhps xm6, [dstq+strideq*0-2] - punpcklqdq m9, m5, m10 - vinserti128 m5, [dstq+stride3q -2], 1 - vinserti128 m10, [dstq+strideq*4-2], 1 - movu m11, [r13+hq*2*1+16*1] - vpblendd m6, m9, 0xF0 - shufps m5, m10, q2121 - vpblendvb m6, [rsp+gprsize+16+hq*8+64-8*1], m11 -%endif - ret -.d4k1: -%if %1 == 4 - %if %2 == 4 - vinserti128 m6, [dstq +strideq*0-2], 1 - vinserti128 m9, [dstq +strideq*1-2], 1 - movd xm5, [dstq +strideq*2+2] - movd xm10, [dstq +stride3q +2] - pblendw m6, [leftq-16+0], 0x01 - pblendw m9, [leftq-16+2], 0x01 - vinserti128 m5, [dst4q+strideq*0+2], 1 - vinserti128 m10, [dst4q+strideq*1+2], 1 - vpblendd m6, [topq +strideq*0-2], 0x01 - vpblendd m9, [topq +strideq*1-2], 0x01 - punpckldq m5, m10 - punpckldq m6, m9 - %else - movd xm6, [topq +strideq*0-2] - movq xm5, [dstq +strideq*2-2] - movq xm9, [dst4q+strideq*0-2] - movd xm10, [dst4q+strideq*2+2] - pinsrd xm6, [topq +strideq*1-2], 1 - movhps xm5, [dstq +stride3q -2] - movhps xm9, [dst4q+strideq*1-2] - pinsrd xm10, [dst4q+stride3q +2], 1 - pinsrd xm6, [dstq +strideq*0-2], 2 - pinsrd xm10, [dst8q+strideq*0+2], 2 - pinsrd xm6, [dstq +strideq*1-2], 3 - pinsrd xm10, [dst8q+strideq*1+2], 3 - shufps xm11, xm5, xm9, q2020 - shufps xm5, xm9, q3131 - movu m9, [blend_4x8_3] - vinserti128 m6, xm11, 1 - vinserti128 m5, xm10, 1 - vpblendvb m6, [rsp+gprsize+16-8], m9 - %endif -%else - lea r13, [blend_8x8_1] - movu m11, [r13+hq*2*2+16*2] - movq xm6, [top1q -2] - movq xm9, [top2q -2] - movq xm5, [dstq +strideq*2+2] - movq xm10, [dstq +stride3q +2] - vinserti128 m6, [dstq +strideq*0-2], 1 - vinserti128 m9, [dstq +strideq*1-2], 1 - vinserti128 m5, [dst4q+strideq*0+2], 1 - vinserti128 m10, [dst4q+strideq*1+2], 1 - punpcklqdq m6, m9 - vpblendvb m6, [rsp+gprsize+16+hq*8+64-8*2], m11 - punpcklqdq m5, m10 -%endif - ret -.d5k1: -%if %1 == 4 - %if %2 == 4 - movd xm6, [topq +strideq*0-1] - movd xm9, [topq +strideq*1-1] - movd xm5, [dstq +strideq*2+1] - movd xm10, [dstq +stride3q +1] - pcmpeqd m12, m12 - pmovzxbw m11, [leftq-8+1] - psrld m12, 24 - vinserti128 m6, [dstq +strideq*0-1], 1 - vinserti128 m9, [dstq +strideq*1-1], 1 - vinserti128 m5, [dst4q+strideq*0+1], 1 - vinserti128 m10, [dst4q+strideq*1+1], 1 - punpckldq m6, m9 - pxor m9, m9 - vpblendd m12, m9, 0x0F - punpckldq m5, m10 - vpblendvb m6, m11, m12 - %else - movd xm6, [topq +strideq*0-1] - movq xm5, [dstq +strideq*2-1] - movq xm9, [dst4q+strideq*0-1] - movd xm10, [dst4q+strideq*2+1] - pinsrd xm6, [topq +strideq*1-1], 1 - movhps xm5, [dstq +stride3q -1] - movhps xm9, [dst4q+strideq*1-1] - pinsrd xm10, [dst4q+stride3q +1], 1 - pinsrd xm6, [dstq +strideq*0-1], 2 - pinsrd xm10, [dst8q+strideq*0+1], 2 - pinsrd xm6, [dstq +strideq*1-1], 3 - pinsrd xm10, [dst8q+strideq*1+1], 3 - shufps xm11, xm5, xm9, q2020 - vinserti128 m6, xm11, 1 - pmovzxbw m11, [leftq-3] - psrldq xm5, 2 - psrldq xm9, 2 - shufps xm5, xm9, q2020 - movu m9, [blend_4x8_1] - vinserti128 m5, xm10, 1 - vpblendvb m6, m11, m9 - %endif -%else - lea r13, [blend_8x8_0] - movu m11, [r13+hq*2*2+16*2] - movq xm6, [top1q -1] - movq xm9, [top2q -1] - movq xm5, [dstq +strideq*2+1] - movq xm10, [dstq +stride3q +1] - vinserti128 m6, [dstq +strideq*0-1], 1 - vinserti128 m9, [dstq +strideq*1-1], 1 - vinserti128 m5, [dst4q+strideq*0+1], 1 - vinserti128 m10, [dst4q+strideq*1+1], 1 - punpcklqdq m6, m9 - punpcklqdq m5, m10 - vpblendvb m6, [rsp+gprsize+80+hq*8+64-8*2], m11 -%endif - ret -.d6k1: -%if %1 == 4 - %if %2 == 4 - movd xm6, [topq +strideq*0] - movd xm9, [topq +strideq*1] - movd xm5, [dstq +strideq*2] - movd xm10, [dstq +stride3q ] - vinserti128 m6, [dstq +strideq*0], 1 - vinserti128 m9, [dstq +strideq*1], 1 - vinserti128 m5, [dst4q+strideq*0], 1 - vinserti128 m10, [dst4q+strideq*1], 1 - punpckldq m6, m9 - punpckldq m5, m10 - %else - movd xm5, [dstq +strideq*2] - movd xm6, [topq +strideq*0] - movd xm9, [dst4q+strideq*2] - pinsrd xm5, [dstq +stride3q ], 1 - pinsrd xm6, [topq +strideq*1], 1 - pinsrd xm9, [dst4q+stride3q ], 1 - pinsrd xm5, [dst4q+strideq*0], 2 - pinsrd xm6, [dstq +strideq*0], 2 - pinsrd xm9, [dst8q+strideq*0], 2 - pinsrd xm5, [dst4q+strideq*1], 3 - pinsrd xm6, [dstq +strideq*1], 3 - pinsrd xm9, [dst8q+strideq*1], 3 - vinserti128 m6, xm5, 1 - vinserti128 m5, xm9, 1 - %endif -%else - movq xm5, [dstq +strideq*2] - movq xm9, [dst4q+strideq*0] - movq xm6, [top1q ] - movq xm10, [dstq +strideq*0] - movhps xm5, [dstq +stride3q ] - movhps xm9, [dst4q+strideq*1] - movhps xm6, [top2q ] - movhps xm10, [dstq +strideq*1] - vinserti128 m5, xm9, 1 - vinserti128 m6, xm10, 1 -%endif - ret -.d7k1: -%if %1 == 4 - %if %2 == 4 - movd xm5, [dstq +strideq*2-1] - movd xm9, [dstq +stride3q -1] - movd xm6, [topq +strideq*0+1] - movd xm10, [topq +strideq*1+1] - pinsrb xm5, [leftq+ 5], 0 - pinsrb xm9, [leftq+ 7], 0 - vinserti128 m6, [dstq +strideq*0+1], 1 - vinserti128 m10, [dstq +strideq*1+1], 1 - vinserti128 m5, [dst4q+strideq*0-1], 1 - vinserti128 m9, [dst4q+strideq*1-1], 1 - punpckldq m6, m10 - punpckldq m5, m9 - %else - movd xm6, [topq +strideq*0+1] - movq xm9, [dstq +strideq*2-1] - movq xm10, [dst4q+strideq*0-1] - movd xm11, [dst4q+strideq*2-1] - pinsrd xm6, [topq +strideq*1+1], 1 - movhps xm9, [dstq +stride3q -1] - movhps xm10, [dst4q+strideq*1-1] - pinsrd xm11, [dst4q+stride3q -1], 1 - pinsrd xm6, [dstq +strideq*0+1], 2 - pinsrd xm11, [dst8q+strideq*0-1], 2 - pinsrd xm6, [dstq +strideq*1+1], 3 - pinsrd xm11, [dst8q+strideq*1-1], 3 - shufps xm5, xm9, xm10, q2020 - vinserti128 m5, xm11, 1 - pmovzxbw m11, [leftq+5] - psrldq xm9, 2 - psrldq xm10, 2 - shufps xm9, xm10, q2020 - movu m10, [blend_4x8_1+8] - vinserti128 m6, xm9, 1 - vpblendvb m5, m11, m10 - %endif -%else - lea r13, [blend_8x8_0+16] - movq xm5, [dstq +strideq*2-1] - movq xm9, [dst4q+strideq*0-1] - movq xm6, [top1q +1] - movq xm10, [dstq +strideq*0+1] - movhps xm5, [dstq +stride3q -1] - movhps xm9, [dst4q+strideq*1-1] - movhps xm6, [top2q +1] - movhps xm10, [dstq +strideq*1+1] - movu m11, [r13+hq*2*2+16*2] - vinserti128 m5, xm9, 1 - vinserti128 m6, xm10, 1 - vpblendvb m5, [rsp+gprsize+80+hq*8+64+8*2], m11 -%endif - ret - -.border_block: - DEFINE_ARGS dst, stride, left, top, pri, sec, stride3, dst4, edge -%define rstk rsp -%assign stack_offset stack_offset_entry -%if %1 == 4 && %2 == 8 - PUSH r9 - %assign regs_used 10 -%else - %assign regs_used 9 -%endif -%if STACK_ALIGNMENT < 32 - PUSH r%+regs_used - %assign regs_used regs_used+1 -%endif - ALLOC_STACK 2*16+(%2+4)*32, 16 -%define px rsp+2*16+2*32 - - pcmpeqw m14, m14 - psllw m14, 15 ; 0x8000 - - ; prepare pixel buffers - body/right -%if %1 == 4 - INIT_XMM avx2 -%endif -%if %2 == 8 - lea dst4q, [dstq+strideq*4] -%endif - lea stride3q, [strideq*3] - test edgeb, 2 ; have_right - jz .no_right - pmovzxbw m1, [dstq+strideq*0] - pmovzxbw m2, [dstq+strideq*1] - pmovzxbw m3, [dstq+strideq*2] - pmovzxbw m4, [dstq+stride3q] - mova [px+0*32], m1 - mova [px+1*32], m2 - mova [px+2*32], m3 - mova [px+3*32], m4 -%if %2 == 8 - pmovzxbw m1, [dst4q+strideq*0] - pmovzxbw m2, [dst4q+strideq*1] - pmovzxbw m3, [dst4q+strideq*2] - pmovzxbw m4, [dst4q+stride3q] - mova [px+4*32], m1 - mova [px+5*32], m2 - mova [px+6*32], m3 - mova [px+7*32], m4 -%endif - jmp .body_done -.no_right: -%if %1 == 4 - movd xm1, [dstq+strideq*0] - movd xm2, [dstq+strideq*1] - movd xm3, [dstq+strideq*2] - movd xm4, [dstq+stride3q] - pmovzxbw xm1, xm1 - pmovzxbw xm2, xm2 - pmovzxbw xm3, xm3 - pmovzxbw xm4, xm4 - movq [px+0*32], xm1 - movq [px+1*32], xm2 - movq [px+2*32], xm3 - movq [px+3*32], xm4 -%else - pmovzxbw xm1, [dstq+strideq*0] - pmovzxbw xm2, [dstq+strideq*1] - pmovzxbw xm3, [dstq+strideq*2] - pmovzxbw xm4, [dstq+stride3q] - mova [px+0*32], xm1 - mova [px+1*32], xm2 - mova [px+2*32], xm3 - mova [px+3*32], xm4 -%endif - movd [px+0*32+%1*2], xm14 - movd [px+1*32+%1*2], xm14 - movd [px+2*32+%1*2], xm14 - movd [px+3*32+%1*2], xm14 -%if %2 == 8 - %if %1 == 4 - movd xm1, [dst4q+strideq*0] - movd xm2, [dst4q+strideq*1] - movd xm3, [dst4q+strideq*2] - movd xm4, [dst4q+stride3q] - pmovzxbw xm1, xm1 - pmovzxbw xm2, xm2 - pmovzxbw xm3, xm3 - pmovzxbw xm4, xm4 - movq [px+4*32], xm1 - movq [px+5*32], xm2 - movq [px+6*32], xm3 - movq [px+7*32], xm4 - %else - pmovzxbw xm1, [dst4q+strideq*0] - pmovzxbw xm2, [dst4q+strideq*1] - pmovzxbw xm3, [dst4q+strideq*2] - pmovzxbw xm4, [dst4q+stride3q] - mova [px+4*32], xm1 - mova [px+5*32], xm2 - mova [px+6*32], xm3 - mova [px+7*32], xm4 - %endif - movd [px+4*32+%1*2], xm14 - movd [px+5*32+%1*2], xm14 - movd [px+6*32+%1*2], xm14 - movd [px+7*32+%1*2], xm14 -%endif -.body_done: - - ; top - test edgeb, 4 ; have_top - jz .no_top - test edgeb, 1 ; have_left - jz .top_no_left - test edgeb, 2 ; have_right - jz .top_no_right - pmovzxbw m1, [topq+strideq*0-(%1/2)] - pmovzxbw m2, [topq+strideq*1-(%1/2)] - movu [px-2*32-%1], m1 - movu [px-1*32-%1], m2 - jmp .top_done -.top_no_right: - pmovzxbw m1, [topq+strideq*0-%1] - pmovzxbw m2, [topq+strideq*1-%1] - movu [px-2*32-%1*2], m1 - movu [px-1*32-%1*2], m2 - movd [px-2*32+%1*2], xm14 - movd [px-1*32+%1*2], xm14 - jmp .top_done -.top_no_left: - test edgeb, 2 ; have_right - jz .top_no_left_right - pmovzxbw m1, [topq+strideq*0] - pmovzxbw m2, [topq+strideq*1] - mova [px-2*32+0], m1 - mova [px-1*32+0], m2 - movd [px-2*32-4], xm14 - movd [px-1*32-4], xm14 - jmp .top_done -.top_no_left_right: -%if %1 == 4 - movd xm1, [topq+strideq*0] - pinsrd xm1, [topq+strideq*1], 1 - pmovzxbw xm1, xm1 - movq [px-2*32+0], xm1 - movhps [px-1*32+0], xm1 -%else - pmovzxbw xm1, [topq+strideq*0] - pmovzxbw xm2, [topq+strideq*1] - mova [px-2*32+0], xm1 - mova [px-1*32+0], xm2 -%endif - movd [px-2*32-4], xm14 - movd [px-1*32-4], xm14 - movd [px-2*32+%1*2], xm14 - movd [px-1*32+%1*2], xm14 - jmp .top_done -.no_top: - movu [px-2*32-%1], m14 - movu [px-1*32-%1], m14 -.top_done: - - ; left - test edgeb, 1 ; have_left - jz .no_left - pmovzxbw xm1, [leftq+ 0] -%if %2 == 8 - pmovzxbw xm2, [leftq+ 8] -%endif - movd [px+0*32-4], xm1 - pextrd [px+1*32-4], xm1, 1 - pextrd [px+2*32-4], xm1, 2 - pextrd [px+3*32-4], xm1, 3 -%if %2 == 8 - movd [px+4*32-4], xm2 - pextrd [px+5*32-4], xm2, 1 - pextrd [px+6*32-4], xm2, 2 - pextrd [px+7*32-4], xm2, 3 -%endif - jmp .left_done -.no_left: - movd [px+0*32-4], xm14 - movd [px+1*32-4], xm14 - movd [px+2*32-4], xm14 - movd [px+3*32-4], xm14 -%if %2 == 8 - movd [px+4*32-4], xm14 - movd [px+5*32-4], xm14 - movd [px+6*32-4], xm14 - movd [px+7*32-4], xm14 -%endif -.left_done: - - ; bottom - DEFINE_ARGS dst, stride, dst8, dummy1, pri, sec, stride3, dummy3, edge - test edgeb, 8 ; have_bottom - jz .no_bottom - lea dst8q, [dstq+%2*strideq] - test edgeb, 1 ; have_left - jz .bottom_no_left - test edgeb, 2 ; have_right - jz .bottom_no_right - pmovzxbw m1, [dst8q-(%1/2)] - pmovzxbw m2, [dst8q+strideq-(%1/2)] - movu [px+(%2+0)*32-%1], m1 - movu [px+(%2+1)*32-%1], m2 - jmp .bottom_done -.bottom_no_right: - pmovzxbw m1, [dst8q-%1] - pmovzxbw m2, [dst8q+strideq-%1] - movu [px+(%2+0)*32-%1*2], m1 - movu [px+(%2+1)*32-%1*2], m2 -%if %1 == 8 - movd [px+(%2-1)*32+%1*2], xm14 ; overwritten by previous movu -%endif - movd [px+(%2+0)*32+%1*2], xm14 - movd [px+(%2+1)*32+%1*2], xm14 - jmp .bottom_done -.bottom_no_left: - test edgeb, 2 ; have_right - jz .bottom_no_left_right - pmovzxbw m1, [dst8q] - pmovzxbw m2, [dst8q+strideq] - mova [px+(%2+0)*32+0], m1 - mova [px+(%2+1)*32+0], m2 - movd [px+(%2+0)*32-4], xm14 - movd [px+(%2+1)*32-4], xm14 - jmp .bottom_done -.bottom_no_left_right: -%if %1 == 4 - movd xm1, [dst8q] - pinsrd xm1, [dst8q+strideq], 1 - pmovzxbw xm1, xm1 - movq [px+(%2+0)*32+0], xm1 - movhps [px+(%2+1)*32+0], xm1 -%else - pmovzxbw xm1, [dst8q] - pmovzxbw xm2, [dst8q+strideq] - mova [px+(%2+0)*32+0], xm1 - mova [px+(%2+1)*32+0], xm2 -%endif - movd [px+(%2+0)*32-4], xm14 - movd [px+(%2+1)*32-4], xm14 - movd [px+(%2+0)*32+%1*2], xm14 - movd [px+(%2+1)*32+%1*2], xm14 - jmp .bottom_done -.no_bottom: - movu [px+(%2+0)*32-%1], m14 - movu [px+(%2+1)*32-%1], m14 -.bottom_done: - - ; actual filter - INIT_YMM avx2 - DEFINE_ARGS dst, stride, pridmp, damping, pri, secdmp, stride3, zero -%undef edged - ; register to shuffle values into after packing - vbroadcasti128 m12, [shufb_lohi] - - mov dampingd, r7m - xor zerod, zerod - movifnidn prid, prim - sub dampingd, 31 - movifnidn secdmpd, secdmpm - or prid, 0 - jz .border_sec_only - movd xm0, prid - lzcnt pridmpd, prid - add pridmpd, dampingd - cmovs pridmpd, zerod - mov [rsp+0], pridmpq ; pri_shift - or secdmpd, 0 - jz .border_pri_only - movd xm1, secdmpd - lzcnt secdmpd, secdmpd - add secdmpd, dampingd - cmovs secdmpd, zerod - mov [rsp+8], secdmpq ; sec_shift - - DEFINE_ARGS dst, stride, pridmp, table, pri, secdmp, stride3 - lea tableq, [tap_table] - vpbroadcastb m13, [tableq+pridmpq] ; pri_shift_mask - vpbroadcastb m14, [tableq+secdmpq] ; sec_shift_mask - - ; pri/sec_taps[k] [4 total] - DEFINE_ARGS dst, stride, dir, table, pri, sec, stride3 - vpbroadcastb m0, xm0 ; pri_strength - vpbroadcastb m1, xm1 ; sec_strength - and prid, 1 - lea priq, [tableq+priq*2+8] ; pri_taps - lea secq, [tableq+12] ; sec_taps - - BORDER_PREP_REGS %1, %2 -%if %1*%2*2/mmsize > 1 -.border_v_loop: -%endif - BORDER_LOAD_BLOCK %1, %2, 1 -.border_k_loop: - vpbroadcastb m2, [priq+kq] ; pri_taps - vpbroadcastb m3, [secq+kq] ; sec_taps - ACCUMULATE_TAP_WORD 0*2, [rsp+0], m13, m0, m2, %1, 1 - ACCUMULATE_TAP_WORD 2*2, [rsp+8], m14, m1, m3, %1, 1 - ACCUMULATE_TAP_WORD 6*2, [rsp+8], m14, m1, m3, %1, 1 - dec kq - jge .border_k_loop - - vpbroadcastd m10, [pw_2048] - BORDER_ADJUST_PIXEL %1, m10, 1 -%if %1*%2*2/mmsize > 1 - %define vloop_lines (mmsize/(%1*2)) - lea dstq, [dstq+strideq*vloop_lines] - add stkq, 32*vloop_lines - dec hd - jg .border_v_loop -%endif - RET - -.border_pri_only: - DEFINE_ARGS dst, stride, pridmp, table, pri, _, stride3 - lea tableq, [tap_table] - vpbroadcastb m13, [tableq+pridmpq] ; pri_shift_mask - DEFINE_ARGS dst, stride, dir, table, pri, _, stride3 - vpbroadcastb m0, xm0 ; pri_strength - and prid, 1 - lea priq, [tableq+priq*2+8] ; pri_taps - BORDER_PREP_REGS %1, %2 - vpbroadcastd m1, [pw_2048] -%if %1*%2*2/mmsize > 1 -.border_pri_v_loop: -%endif - BORDER_LOAD_BLOCK %1, %2 -.border_pri_k_loop: - vpbroadcastb m2, [priq+kq] ; pri_taps - ACCUMULATE_TAP_WORD 0*2, [rsp+0], m13, m0, m2, %1 - dec kq - jge .border_pri_k_loop - BORDER_ADJUST_PIXEL %1, m1 -%if %1*%2*2/mmsize > 1 - %define vloop_lines (mmsize/(%1*2)) - lea dstq, [dstq+strideq*vloop_lines] - add stkq, 32*vloop_lines - dec hd - jg .border_pri_v_loop -%endif - RET - -.border_sec_only: - DEFINE_ARGS dst, stride, _, damping, _, secdmp, stride3, zero - movd xm1, secdmpd - lzcnt secdmpd, secdmpd - add secdmpd, dampingd - cmovs secdmpd, zerod - mov [rsp+8], secdmpq ; sec_shift - DEFINE_ARGS dst, stride, _, table, _, secdmp, stride3 - lea tableq, [tap_table] - vpbroadcastb m14, [tableq+secdmpq] ; sec_shift_mask - DEFINE_ARGS dst, stride, dir, table, _, sec, stride3 - vpbroadcastb m1, xm1 ; sec_strength - lea secq, [tableq+12] ; sec_taps - BORDER_PREP_REGS %1, %2 - vpbroadcastd m0, [pw_2048] -%if %1*%2*2/mmsize > 1 -.border_sec_v_loop: -%endif - BORDER_LOAD_BLOCK %1, %2 -.border_sec_k_loop: - vpbroadcastb m3, [secq+kq] ; sec_taps - ACCUMULATE_TAP_WORD 2*2, [rsp+8], m14, m1, m3, %1 - ACCUMULATE_TAP_WORD 6*2, [rsp+8], m14, m1, m3, %1 - dec kq - jge .border_sec_k_loop - BORDER_ADJUST_PIXEL %1, m0 -%if %1*%2*2/mmsize > 1 - %define vloop_lines (mmsize/(%1*2)) - lea dstq, [dstq+strideq*vloop_lines] - add stkq, 32*vloop_lines - dec hd - jg .border_sec_v_loop -%endif - RET -%endmacro - -CDEF_FILTER 8, 8 -CDEF_FILTER 4, 8 -CDEF_FILTER 4, 4 - -INIT_YMM avx2 -cglobal cdef_dir, 3, 4, 15, src, stride, var, stride3 - lea stride3q, [strideq*3] - movq xm0, [srcq+strideq*0] - movq xm1, [srcq+strideq*1] - movq xm2, [srcq+strideq*2] - movq xm3, [srcq+stride3q] - lea srcq, [srcq+strideq*4] - vpbroadcastq m4, [srcq+strideq*0] - vpbroadcastq m5, [srcq+strideq*1] - vpbroadcastq m6, [srcq+strideq*2] - vpbroadcastq m7, [srcq+stride3q] - vpbroadcastd m8, [pw_128] - pxor m9, m9 - - vpblendd m0, m0, m7, 0xf0 - vpblendd m1, m1, m6, 0xf0 - vpblendd m2, m2, m5, 0xf0 - vpblendd m3, m3, m4, 0xf0 - - punpcklbw m0, m9 - punpcklbw m1, m9 - punpcklbw m2, m9 - punpcklbw m3, m9 - - psubw m0, m8 - psubw m1, m8 - psubw m2, m8 - psubw m3, m8 - - ; shuffle registers to generate partial_sum_diag[0-1] together - vpermq m7, m0, q1032 - vpermq m6, m1, q1032 - vpermq m5, m2, q1032 - vpermq m4, m3, q1032 - - ; start with partial_sum_hv[0-1] - paddw m8, m0, m1 - paddw m9, m2, m3 - phaddw m10, m0, m1 - phaddw m11, m2, m3 - paddw m8, m9 - phaddw m10, m11 - vextracti128 xm9, m8, 1 - vextracti128 xm11, m10, 1 - paddw xm8, xm9 ; partial_sum_hv[1] - phaddw xm10, xm11 ; partial_sum_hv[0] - vinserti128 m8, xm10, 1 - vpbroadcastd m9, [div_table+44] - pmaddwd m8, m8 - pmulld m8, m9 ; cost6[2a-d] | cost2[a-d] - - ; create aggregates [lower half]: - ; m9 = m0:01234567+m1:x0123456+m2:xx012345+m3:xxx01234+ - ; m4:xxxx0123+m5:xxxxx012+m6:xxxxxx01+m7:xxxxxxx0 - ; m10= m1:7xxxxxxx+m2:67xxxxxx+m3:567xxxxx+ - ; m4:4567xxxx+m5:34567xxx+m6:234567xx+m7:1234567x - ; and [upper half]: - ; m9 = m0:xxxxxxx0+m1:xxxxxx01+m2:xxxxx012+m3:xxxx0123+ - ; m4:xxx01234+m5:xx012345+m6:x0123456+m7:01234567 - ; m10= m0:1234567x+m1:234567xx+m2:34567xxx+m3:4567xxxx+ - ; m4:567xxxxx+m5:67xxxxxx+m6:7xxxxxxx - ; and then shuffle m11 [shufw_6543210x], unpcklwd, pmaddwd, pmulld, paddd - - pslldq m9, m1, 2 - psrldq m10, m1, 14 - pslldq m11, m2, 4 - psrldq m12, m2, 12 - pslldq m13, m3, 6 - psrldq m14, m3, 10 - paddw m9, m11 - paddw m10, m12 - paddw m9, m13 - paddw m10, m14 - pslldq m11, m4, 8 - psrldq m12, m4, 8 - pslldq m13, m5, 10 - psrldq m14, m5, 6 - paddw m9, m11 - paddw m10, m12 - paddw m9, m13 - paddw m10, m14 - pslldq m11, m6, 12 - psrldq m12, m6, 4 - pslldq m13, m7, 14 - psrldq m14, m7, 2 - paddw m9, m11 - paddw m10, m12 - paddw m9, m13 - paddw m10, m14 ; partial_sum_diag[0/1][8-14,zero] - vbroadcasti128 m14, [shufw_6543210x] - vbroadcasti128 m13, [div_table+16] - vbroadcasti128 m12, [div_table+0] - paddw m9, m0 ; partial_sum_diag[0/1][0-7] - pshufb m10, m14 - punpckhwd m11, m9, m10 - punpcklwd m9, m10 - pmaddwd m11, m11 - pmaddwd m9, m9 - pmulld m11, m13 - pmulld m9, m12 - paddd m9, m11 ; cost0[a-d] | cost4[a-d] - - ; merge horizontally and vertically for partial_sum_alt[0-3] - paddw m10, m0, m1 - paddw m11, m2, m3 - paddw m12, m4, m5 - paddw m13, m6, m7 - phaddw m0, m4 - phaddw m1, m5 - phaddw m2, m6 - phaddw m3, m7 - - ; create aggregates [lower half]: - ; m4 = m10:01234567+m11:x0123456+m12:xx012345+m13:xxx01234 - ; m11= m11:7xxxxxxx+m12:67xxxxxx+m13:567xxxxx - ; and [upper half]: - ; m4 = m10:xxx01234+m11:xx012345+m12:x0123456+m13:01234567 - ; m11= m10:567xxxxx+m11:67xxxxxx+m12:7xxxxxxx - ; and then pshuflw m11 3012, unpcklwd, pmaddwd, pmulld, paddd - - pslldq m4, m11, 2 - psrldq m11, 14 - pslldq m5, m12, 4 - psrldq m12, 12 - pslldq m6, m13, 6 - psrldq m13, 10 - paddw m4, m10 - paddw m11, m12 - vpbroadcastd m12, [div_table+44] - paddw m5, m6 - paddw m11, m13 ; partial_sum_alt[3/2] right - vbroadcasti128 m13, [div_table+32] - paddw m4, m5 ; partial_sum_alt[3/2] left - pshuflw m5, m11, q3012 - punpckhwd m6, m11, m4 - punpcklwd m4, m5 - pmaddwd m6, m6 - pmaddwd m4, m4 - pmulld m6, m12 - pmulld m4, m13 - paddd m4, m6 ; cost7[a-d] | cost5[a-d] - - ; create aggregates [lower half]: - ; m5 = m0:01234567+m1:x0123456+m2:xx012345+m3:xxx01234 - ; m1 = m1:7xxxxxxx+m2:67xxxxxx+m3:567xxxxx - ; and [upper half]: - ; m5 = m0:xxx01234+m1:xx012345+m2:x0123456+m3:01234567 - ; m1 = m0:567xxxxx+m1:67xxxxxx+m2:7xxxxxxx - ; and then pshuflw m1 3012, unpcklwd, pmaddwd, pmulld, paddd - - pslldq m5, m1, 2 - psrldq m1, 14 - pslldq m6, m2, 4 - psrldq m2, 12 - pslldq m7, m3, 6 - psrldq m3, 10 - paddw m5, m0 - paddw m1, m2 - paddw m6, m7 - paddw m1, m3 ; partial_sum_alt[0/1] right - paddw m5, m6 ; partial_sum_alt[0/1] left - pshuflw m0, m1, q3012 - punpckhwd m1, m5 - punpcklwd m5, m0 - pmaddwd m1, m1 - pmaddwd m5, m5 - pmulld m1, m12 - pmulld m5, m13 - paddd m5, m1 ; cost1[a-d] | cost3[a-d] - - mova xm0, [pd_47130256+ 16] - mova m1, [pd_47130256] - phaddd m9, m8 - phaddd m5, m4 - phaddd m9, m5 - vpermd m0, m9 ; cost[0-3] - vpermd m1, m9 ; cost[4-7] | cost[0-3] - - ; now find the best cost - pmaxsd xm2, xm0, xm1 - pshufd xm3, xm2, q1032 - pmaxsd xm2, xm3 - pshufd xm3, xm2, q2301 - pmaxsd xm2, xm3 ; best cost - - ; find the idx using minpos - ; make everything other than the best cost negative via subtraction - ; find the min of unsigned 16-bit ints to sort out the negative values - psubd xm4, xm1, xm2 - psubd xm3, xm0, xm2 - packssdw xm3, xm4 - phminposuw xm3, xm3 - - ; convert idx to 32-bits - psrld xm3, 16 - movd eax, xm3 - - ; get idx^4 complement - vpermd m3, m1 - psubd xm2, xm3 - psrld xm2, 10 - movd [varq], xm2 - RET - -%if WIN64 -DECLARE_REG_TMP 5, 6 -%else -DECLARE_REG_TMP 8, 5 -%endif - -; lut: -; t0 t1 t2 t3 t4 t5 t6 t7 -; T0 T1 T2 T3 T4 T5 T6 T7 -; L0 L1 00 01 02 03 04 05 -; L2 L3 10 11 12 13 14 15 -; L4 L5 20 21 22 23 24 25 -; L6 L7 30 31 32 33 34 35 -; 4e 4f 40 41 42 43 44 45 -; 5e 5f 50 51 52 53 54 55 - -INIT_ZMM avx512icl -cglobal cdef_filter_4x4, 4, 8, 13, dst, stride, left, top, pri, sec, dir, damping, edge -%define base r7-edge_mask - movq xmm0, [dstq+strideq*0] - movhps xmm0, [dstq+strideq*1] - lea r7, [edge_mask] - movq xmm1, [topq+strideq*0-2] - movhps xmm1, [topq+strideq*1-2] - mov r6d, edgem - vinserti32x4 ym0, ymm0, [leftq], 1 - lea r2, [strideq*3] - vinserti32x4 ym1, ymm1, [dstq+strideq*2], 1 - mova m5, [base+lut_perm_4x4] - vinserti32x4 m0, [dstq+r2], 2 - test r6b, 0x08 ; avoid buffer overread - jz .main - lea r3, [dstq+strideq*4-4] - vinserti32x4 m1, [r3+strideq*0], 2 - vinserti32x4 m0, [r3+strideq*1], 3 -.main: - movifnidn prid, prim - mov t0d, dirm - mova m3, [base+px_idx] - mov r3d, dampingm - vpermi2b m5, m0, m1 ; lut - vpbroadcastd m0, [base+pd_268435568] ; (1 << 28) + (7 << 4) - pxor m7, m7 - lea r3, [r7+r3*8] ; gf_shr + (damping - 30) * 8 - vpermb m6, m3, m5 ; px - cmp r6d, 0x0f - jne .mask_edges ; mask edges only if required - test prid, prid - jz .sec_only - vpaddd m1, m3, [base+cdef_dirs+(t0+2)*4] {1to16} ; dir - vpermb m1, m1, m5 ; k0p0 k0p1 k1p0 k1p1 -%macro CDEF_FILTER_4x4_PRI 0 - vpcmpub k1, m6, m1, 6 ; px > pN - psubb m2, m1, m6 - lzcnt r6d, prid - vpsubb m2{k1}, m6, m1 ; abs(diff) - vpbroadcastb m4, prim - and prid, 1 - vgf2p8affineqb m9, m2, [r3+r6*8] {1to8}, 0 ; abs(diff) >> shift - movifnidn t1d, secm - vpbroadcastd m10, [base+pri_tap+priq*4] - vpsubb m10{k1}, m7, m10 ; apply_sign(pri_tap) - psubusb m4, m9 ; imax(0, pri_strength - (abs(diff) >> shift))) - pminub m2, m4 - vpdpbusd m0, m2, m10 ; sum -%endmacro - CDEF_FILTER_4x4_PRI - test t1d, t1d ; sec - jz .end_no_clip - call .sec -.end_clip: - pminub m4, m6, m1 - pmaxub m1, m6 - pminub m5, m2, m3 - pmaxub m2, m3 - pminub m4, m5 - pmaxub m2, m1 - psrldq m1, m4, 2 - psrldq m3, m2, 2 - pminub m1, m4 - vpcmpw k1, m0, m7, 1 - vpshldd m6, m0, 8 - pmaxub m2, m3 - pslldq m3, m1, 1 - psubw m7, m0 - paddusw m0, m6 ; clip >0xff - vpsubusw m0{k1}, m6, m7 ; clip <0x00 - pslldq m4, m2, 1 - pminub m1, m3 - pmaxub m2, m4 - pmaxub m0, m1 - pminub m0, m2 - jmp .end -.sec_only: - movifnidn t1d, secm - call .sec -.end_no_clip: - vpshldd m6, m0, 8 ; (px << 8) + ((sum > -8) << 4) - paddw m0, m6 ; (px << 8) + ((sum + (sum > -8) + 7) << 4) -.end: - mova xm1, [base+end_perm] - vpermb m0, m1, m0 ; output in bits 8-15 of each dword - movd [dstq+strideq*0], xm0 - pextrd [dstq+strideq*1], xm0, 1 - pextrd [dstq+strideq*2], xm0, 2 - pextrd [dstq+r2 ], xm0, 3 - RET -.mask_edges_sec_only: - movifnidn t1d, secm - call .mask_edges_sec - jmp .end_no_clip -ALIGN function_align -.mask_edges: - vpbroadcastq m8, [base+edge_mask+r6*8] - test prid, prid - jz .mask_edges_sec_only - vpaddd m2, m3, [base+cdef_dirs+(t0+2)*4] {1to16} - vpshufbitqmb k1, m8, m2 ; index in-range - mova m1, m6 - vpermb m1{k1}, m2, m5 - CDEF_FILTER_4x4_PRI - test t1d, t1d - jz .end_no_clip - call .mask_edges_sec - jmp .end_clip -.mask_edges_sec: - vpaddd m4, m3, [base+cdef_dirs+(t0+4)*4] {1to16} - vpaddd m9, m3, [base+cdef_dirs+(t0+0)*4] {1to16} - vpshufbitqmb k1, m8, m4 - mova m2, m6 - vpermb m2{k1}, m4, m5 - vpshufbitqmb k1, m8, m9 - mova m3, m6 - vpermb m3{k1}, m9, m5 - jmp .sec_main -ALIGN function_align -.sec: - vpaddd m2, m3, [base+cdef_dirs+(t0+4)*4] {1to16} ; dir + 2 - vpaddd m3, [base+cdef_dirs+(t0+0)*4] {1to16} ; dir - 2 - vpermb m2, m2, m5 ; k0s0 k0s1 k1s0 k1s1 - vpermb m3, m3, m5 ; k0s2 k0s3 k1s2 k1s3 -.sec_main: - vpbroadcastd m8, [base+sec_tap] - vpcmpub k1, m6, m2, 6 - psubb m4, m2, m6 - vpbroadcastb m12, t1d - lzcnt t1d, t1d - vpsubb m4{k1}, m6, m2 - vpcmpub k2, m6, m3, 6 - vpbroadcastq m11, [r3+t1*8] - gf2p8affineqb m10, m4, m11, 0 - psubb m5, m3, m6 - mova m9, m8 - vpsubb m8{k1}, m7, m8 - psubusb m10, m12, m10 - vpsubb m5{k2}, m6, m3 - pminub m4, m10 - vpdpbusd m0, m4, m8 - gf2p8affineqb m11, m5, m11, 0 - vpsubb m9{k2}, m7, m9 - psubusb m12, m11 - pminub m5, m12 - vpdpbusd m0, m5, m9 - ret - -%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/cdef_avx2.asm b/third_party/dav1d/src/x86/cdef_avx2.asm new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/x86/cdef_avx2.asm @@ -0,0 +1,1798 @@ +; Copyright © 2018, VideoLAN and dav1d authors +; Copyright © 2018, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "ext/x86/x86inc.asm" + +%if ARCH_X86_64 + +%macro JMP_TABLE 2-* + %xdefine %1_jmptable %%table + %xdefine %%base mangle(private_prefix %+ _%1_avx2) + %%table: + %rep %0 - 1 + dd %%base %+ .%2 - %%table + %rotate 1 + %endrep +%endmacro + +%macro CDEF_FILTER_JMP_TABLE 1 +JMP_TABLE cdef_filter_%1, \ + d6k0, d6k1, d7k0, d7k1, \ + d0k0, d0k1, d1k0, d1k1, d2k0, d2k1, d3k0, d3k1, \ + d4k0, d4k1, d5k0, d5k1, d6k0, d6k1, d7k0, d7k1, \ + d0k0, d0k1, d1k0, d1k1 +%endmacro + +SECTION_RODATA 32 + +pd_47130256: dd 4, 7, 1, 3, 0, 2, 5, 6 +blend_4x4: dd 0x00, 0x80, 0x00, 0x00, 0x80, 0x80, 0x00, 0x00 + dd 0x80, 0x00, 0x00 +blend_4x8_0: dd 0x00, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80 +blend_4x8_1: dd 0x00, 0x00, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80 + dd 0x00, 0x00 +blend_4x8_2: dd 0x0000, 0x8080, 0x8080, 0x8080, 0x8080, 0x8080, 0x8080, 0x8080 + dd 0x0000 +blend_4x8_3: dd 0x0000, 0x0000, 0x8080, 0x8080, 0x8080, 0x8080, 0x8080, 0x8080 + dd 0x0000, 0x0000 +blend_8x8_0: dq 0x00, 0x00, 0x80, 0x80, 0x80, 0x80 +blend_8x8_1: dq 0x0000, 0x0000, 0x8080, 0x8080, 0x8080, 0x8080, 0x0000, 0x0000 +div_table: dd 840, 420, 280, 210, 168, 140, 120, 105, 420, 210, 140, 105 +shufw_6543210x:db 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1, 14, 15 +shufb_lohi: db 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15 +pw_128: times 2 dw 128 +pw_2048: times 2 dw 2048 +tap_table: ; masks for 8 bit shifts + db 0xFF, 0x7F, 0x3F, 0x1F, 0x0F, 0x07, 0x03, 0x01 + ; weights + db 4, 2, 3, 3, 2, 1 + db -1 * 16 + 1, -2 * 16 + 2 + db 0 * 16 + 1, -1 * 16 + 2 + db 0 * 16 + 1, 0 * 16 + 2 + db 0 * 16 + 1, 1 * 16 + 2 + db 1 * 16 + 1, 2 * 16 + 2 + db 1 * 16 + 0, 2 * 16 + 1 + db 1 * 16 + 0, 2 * 16 + 0 + db 1 * 16 + 0, 2 * 16 - 1 + ; the last 6 are repeats of the first 6 so we don't need to & 7 + db -1 * 16 + 1, -2 * 16 + 2 + db 0 * 16 + 1, -1 * 16 + 2 + db 0 * 16 + 1, 0 * 16 + 2 + db 0 * 16 + 1, 1 * 16 + 2 + db 1 * 16 + 1, 2 * 16 + 2 + db 1 * 16 + 0, 2 * 16 + 1 + +CDEF_FILTER_JMP_TABLE 4x4 +CDEF_FILTER_JMP_TABLE 4x8 +CDEF_FILTER_JMP_TABLE 8x8 + +SECTION .text + +%macro PREP_REGS 2 ; w, h + ; off1/2/3[k] [6 total] from [tapq+12+(dir+0/2/6)*2+k] + mov dird, r6m + lea tableq, [cdef_filter_%1x%2_jmptable] + lea dirq, [tableq+dirq*2*4] +%if %1 == 4 + %if %2 == 4 + DEFINE_ARGS dst, stride, left, top, pri, sec, \ + table, dir, dirjmp, dst4, stride3, k + %else + DEFINE_ARGS dst, stride, left, top, pri, sec, \ + table, dir, dirjmp, dst4, dst8, stride3, k + lea dst8q, [dstq+strideq*8] + %endif +%else + DEFINE_ARGS dst, stride, h, top1, pri, sec, \ + table, dir, dirjmp, top2, dst4, stride3, k + mov hq, -8 + lea top1q, [top1q+strideq*0] + lea top2q, [top1q+strideq*1] +%endif + lea dst4q, [dstq+strideq*4] +%if %1 == 4 + lea stride3q, [strideq*3] +%endif +%endmacro + +%macro LOAD_BLOCK 2-3 0 ; w, h, init_min_max + mov kd, 1 + pxor m15, m15 ; sum +%if %2 == 8 + pxor m12, m12 + %if %1 == 4 + movd xm4, [dstq +strideq*0] + movd xm6, [dstq +strideq*1] + movd xm5, [dstq +strideq*2] + movd xm7, [dstq +stride3q ] + vinserti128 m4, [dst4q+strideq*0], 1 + vinserti128 m6, [dst4q+strideq*1], 1 + vinserti128 m5, [dst4q+strideq*2], 1 + vinserti128 m7, [dst4q+stride3q ], 1 + punpckldq m4, m6 + punpckldq m5, m7 + %else + movq xm4, [dstq+strideq*0] + movq xm5, [dstq+strideq*1] + vinserti128 m4, [dstq+strideq*2], 1 + vinserti128 m5, [dstq+stride3q ], 1 + %endif + punpcklqdq m4, m5 +%else + movd xm4, [dstq+strideq*0] + movd xm5, [dstq+strideq*1] + vinserti128 m4, [dstq+strideq*2], 1 + vinserti128 m5, [dstq+stride3q ], 1 + punpckldq m4, m5 +%endif +%if %3 == 1 + mova m7, m4 ; min + mova m8, m4 ; max +%endif +%endmacro + +%macro ACCUMULATE_TAP_BYTE 7-8 0 ; tap_offset, shift, mask, strength + ; mul_tap, w, h, clip + ; load p0/p1 + movsxd dirjmpq, [dirq+kq*4+%1*2*4] + add dirjmpq, tableq + call dirjmpq + +%if %8 == 1 + pmaxub m7, m5 + pminub m8, m5 + pmaxub m7, m6 + pminub m8, m6 +%endif + + ; accumulate sum[m15] over p0/p1 +%if %7 == 4 + punpcklbw m5, m6 + punpcklbw m6, m4, m4 + psubusb m9, m5, m6 + psubusb m5, m6, m5 + por m9, m5 ; abs_diff_p01(p01 - px) + pcmpeqb m5, m9 + por m5, %5 + psignb m6, %5, m5 + psrlw m5, m9, %2 ; emulate 8-bit shift + pand m5, %3 + psubusb m5, %4, m5 + pminub m5, m9 + pmaddubsw m5, m6 + paddw m15, m5 +%else + psubusb m9, m5, m4 + psubusb m5, m4, m5 + psubusb m11, m6, m4 + psubusb m6, m4, m6 + por m9, m5 ; abs_diff_p0(p0 - px) + por m11, m6 ; abs_diff_p1(p1 - px) + pcmpeqb m5, m9 + pcmpeqb m6, m11 + punpckhbw m10, m9, m11 + punpcklbw m9, m11 + por m5, %5 + por m11, m6, %5 + punpckhbw m6, m5, m11 + punpcklbw m5, m11 + psignb m11, %5, m6 + psrlw m6, m10, %2 ; emulate 8-bit shift + pand m6, %3 + psubusb m6, %4, m6 + pminub m6, m10 + pmaddubsw m6, m11 + paddw m12, m6 + psignb m11, %5, m5 + psrlw m5, m9, %2 ; emulate 8-bit shift + pand m5, %3 + psubusb m5, %4, m5 + pminub m5, m9 + pmaddubsw m5, m11 + paddw m15, m5 +%endif +%endmacro + +%macro ADJUST_PIXEL 4-5 0 ; w, h, zero, pw_2048, clip +%if %2 == 4 + %if %5 == 1 + punpcklbw m4, %3 + %endif + pcmpgtw %3, m15 + paddw m15, %3 + pmulhrsw m15, %4 + %if %5 == 0 + packsswb m15, m15 + paddb m4, m15 + %else + paddw m4, m15 + packuswb m4, m4 ; clip px in [0x0,0xff] + pminub m4, m7 + pmaxub m4, m8 + %endif + vextracti128 xm5, m4, 1 + movd [dstq+strideq*0], xm4 + movd [dstq+strideq*2], xm5 + pextrd [dstq+strideq*1], xm4, 1 + pextrd [dstq+stride3q ], xm5, 1 +%else + pcmpgtw m6, %3, m12 + pcmpgtw m5, %3, m15 + paddw m12, m6 + paddw m15, m5 + %if %5 == 1 + punpckhbw m5, m4, %3 + punpcklbw m4, %3 + %endif + pmulhrsw m12, %4 + pmulhrsw m15, %4 + %if %5 == 0 + packsswb m15, m12 + paddb m4, m15 + %else + paddw m5, m12 + paddw m4, m15 + packuswb m4, m5 ; clip px in [0x0,0xff] + pminub m4, m7 + pmaxub m4, m8 + %endif + vextracti128 xm5, m4, 1 + %if %1 == 4 + movd [dstq +strideq*0], xm4 + movd [dst4q+strideq*0], xm5 + pextrd [dstq +strideq*1], xm4, 1 + pextrd [dst4q+strideq*1], xm5, 1 + pextrd [dstq +strideq*2], xm4, 2 + pextrd [dst4q+strideq*2], xm5, 2 + pextrd [dstq +stride3q ], xm4, 3 + pextrd [dst4q+stride3q ], xm5, 3 + %else + movq [dstq+strideq*0], xm4 + movq [dstq+strideq*2], xm5 + movhps [dstq+strideq*1], xm4 + movhps [dstq+stride3q ], xm5 + %endif +%endif +%endmacro + +%macro BORDER_PREP_REGS 2 ; w, h + ; off1/2/3[k] [6 total] from [tapq+12+(dir+0/2/6)*2+k] + mov dird, r6m + lea dirq, [tableq+dirq*2+14] +%if %1*%2*2/mmsize > 1 + %if %1 == 4 + DEFINE_ARGS dst, stride, dir, stk, pri, sec, stride3, h, off, k + %else + DEFINE_ARGS dst, stride, dir, stk, pri, sec, h, off, k + %endif + mov hd, %1*%2*2/mmsize +%else + DEFINE_ARGS dst, stride, dir, stk, pri, sec, stride3, off, k +%endif + lea stkq, [px] + pxor m11, m11 +%endmacro + +%macro BORDER_LOAD_BLOCK 2-3 0 ; w, h, init_min_max + mov kd, 1 +%if %1 == 4 + movq xm4, [stkq+32*0] + movhps xm4, [stkq+32*1] + movq xm5, [stkq+32*2] + movhps xm5, [stkq+32*3] + vinserti128 m4, xm5, 1 +%else + mova xm4, [stkq+32*0] ; px + vinserti128 m4, [stkq+32*1], 1 +%endif + pxor m15, m15 ; sum +%if %3 == 1 + mova m7, m4 ; max + mova m8, m4 ; min +%endif +%endmacro + +%macro ACCUMULATE_TAP_WORD 6-7 0 ; tap_offset, shift, mask, strength + ; mul_tap, w, clip + ; load p0/p1 + movsx offq, byte [dirq+kq+%1] ; off1 +%if %6 == 4 + movq xm5, [stkq+offq*2+32*0] ; p0 + movq xm6, [stkq+offq*2+32*2] + movhps xm5, [stkq+offq*2+32*1] + movhps xm6, [stkq+offq*2+32*3] + vinserti128 m5, xm6, 1 +%else + movu xm5, [stkq+offq*2+32*0] ; p0 + vinserti128 m5, [stkq+offq*2+32*1], 1 +%endif + neg offq ; -off1 +%if %6 == 4 + movq xm6, [stkq+offq*2+32*0] ; p1 + movq xm9, [stkq+offq*2+32*2] + movhps xm6, [stkq+offq*2+32*1] + movhps xm9, [stkq+offq*2+32*3] + vinserti128 m6, xm9, 1 +%else + movu xm6, [stkq+offq*2+32*0] ; p1 + vinserti128 m6, [stkq+offq*2+32*1], 1 +%endif +%if %7 == 1 + ; out of bounds values are set to a value that is a both a large unsigned + ; value and a negative signed value. + ; use signed max and unsigned min to remove them + pmaxsw m7, m5 ; max after p0 + pminuw m8, m5 ; min after p0 + pmaxsw m7, m6 ; max after p1 + pminuw m8, m6 ; min after p1 +%endif + + ; accumulate sum[m15] over p0/p1 + ; calculate difference before converting + psubw m5, m4 ; diff_p0(p0 - px) + psubw m6, m4 ; diff_p1(p1 - px) + + ; convert to 8-bits with signed saturation + ; saturating to large diffs has no impact on the results + packsswb m5, m6 + + ; group into pairs so we can accumulate using maddubsw + pshufb m5, m12 + pabsb m9, m5 + psignb m10, %5, m5 + psrlw m5, m9, %2 ; emulate 8-bit shift + pand m5, %3 + psubusb m5, %4, m5 + + ; use unsigned min since abs diff can equal 0x80 + pminub m5, m9 + pmaddubsw m5, m10 + paddw m15, m5 +%endmacro + +%macro BORDER_ADJUST_PIXEL 2-3 0 ; w, pw_2048, clip + pcmpgtw m9, m11, m15 + paddw m15, m9 + pmulhrsw m15, %2 + paddw m4, m15 +%if %3 == 1 + pminsw m4, m7 + pmaxsw m4, m8 +%endif + packuswb m4, m4 + vextracti128 xm5, m4, 1 +%if %1 == 4 + movd [dstq+strideq*0], xm4 + pextrd [dstq+strideq*1], xm4, 1 + movd [dstq+strideq*2], xm5 + pextrd [dstq+stride3q], xm5, 1 +%else + movq [dstq+strideq*0], xm4 + movq [dstq+strideq*1], xm5 +%endif +%endmacro + +%macro CDEF_FILTER 2 ; w, h +INIT_YMM avx2 +cglobal cdef_filter_%1x%2, 4, 9, 0, dst, stride, left, top, \ + pri, sec, dir, damping, edge +%assign stack_offset_entry stack_offset + mov edged, edgem + cmp edged, 0xf + jne .border_block + + PUSH r9 + PUSH r10 + PUSH r11 +%if %2 == 4 + %assign regs_used 12 + %if STACK_ALIGNMENT < 32 + PUSH r%+regs_used + %assign regs_used regs_used+1 + %endif + ALLOC_STACK 0x60, 16 + pmovzxbw xm0, [leftq+1] + vpermq m0, m0, q0110 + psrldq m1, m0, 4 + vpalignr m2, m0, m0, 12 + movu [rsp+0x10], m0 + movu [rsp+0x28], m1 + movu [rsp+0x40], m2 +%elif %1 == 4 + PUSH r12 + %assign regs_used 13 + %if STACK_ALIGNMENT < 32 + PUSH r%+regs_used + %assign regs_used regs_used+1 + %endif + ALLOC_STACK 8*2+%1*%2*1, 16 + pmovzxwd m0, [leftq] + mova [rsp+0x10], m0 +%else + PUSH r12 + PUSH r13 + %assign regs_used 14 + %if STACK_ALIGNMENT < 32 + PUSH r%+regs_used + %assign regs_used regs_used+1 + %endif + ALLOC_STACK 8*2+%1*%2*2+32, 16 + lea r11, [strideq*3] + movu xm4, [dstq+strideq*2] + pmovzxwq m0, [leftq+0] + pmovzxwq m1, [leftq+8] + vinserti128 m4, [dstq+r11], 1 + pmovzxbd m2, [leftq+1] + pmovzxbd m3, [leftq+9] + mova [rsp+0x10], m0 + mova [rsp+0x30], m1 + mova [rsp+0x50], m2 + mova [rsp+0x70], m3 + mova [rsp+0x90], m4 +%endif + + DEFINE_ARGS dst, stride, left, top, pri, secdmp, zero, pridmp, damping + mov dampingd, r7m + xor zerod, zerod + movifnidn prid, prim + sub dampingd, 31 + movifnidn secdmpd, secdmpm + test prid, prid + jz .sec_only + movd xm0, prid + lzcnt pridmpd, prid + add pridmpd, dampingd + cmovs pridmpd, zerod + mov [rsp+0], pridmpq ; pri_shift + test secdmpd, secdmpd + jz .pri_only + movd xm1, secdmpd + lzcnt secdmpd, secdmpd + add secdmpd, dampingd + cmovs secdmpd, zerod + mov [rsp+8], secdmpq ; sec_shift + + DEFINE_ARGS dst, stride, left, top, pri, secdmp, table, pridmp + lea tableq, [tap_table] + vpbroadcastb m13, [tableq+pridmpq] ; pri_shift_mask + vpbroadcastb m14, [tableq+secdmpq] ; sec_shift_mask + + ; pri/sec_taps[k] [4 total] + DEFINE_ARGS dst, stride, left, top, pri, sec, table, dir + vpbroadcastb m0, xm0 ; pri_strength + vpbroadcastb m1, xm1 ; sec_strength + and prid, 1 + lea priq, [tableq+priq*2+8] ; pri_taps + lea secq, [tableq+12] ; sec_taps + + PREP_REGS %1, %2 +%if %1*%2 > mmsize +.v_loop: +%endif + LOAD_BLOCK %1, %2, 1 +.k_loop: + vpbroadcastb m2, [priq+kq] ; pri_taps + vpbroadcastb m3, [secq+kq] ; sec_taps + ACCUMULATE_TAP_BYTE 2, [rsp+0], m13, m0, m2, %1, %2, 1 ; dir + 0 + ACCUMULATE_TAP_BYTE 4, [rsp+8], m14, m1, m3, %1, %2, 1 ; dir + 2 + ACCUMULATE_TAP_BYTE 0, [rsp+8], m14, m1, m3, %1, %2, 1 ; dir - 2 + dec kq + jge .k_loop + + vpbroadcastd m10, [pw_2048] + pxor m9, m9 + ADJUST_PIXEL %1, %2, m9, m10, 1 +%if %1*%2 > mmsize + mov dstq, dst4q + lea top1q, [rsp+0x90] + lea top2q, [rsp+0xA0] + lea dst4q, [dst4q+strideq*4] + add hq, 4 + jl .v_loop +%endif + RET + +.pri_only: + DEFINE_ARGS dst, stride, left, top, pri, _, table, pridmp + lea tableq, [tap_table] + vpbroadcastb m13, [tableq+pridmpq] ; pri_shift_mask + ; pri/sec_taps[k] [4 total] + DEFINE_ARGS dst, stride, left, top, pri, _, table, dir + vpbroadcastb m0, xm0 ; pri_strength + and prid, 1 + lea priq, [tableq+priq*2+8] ; pri_taps + PREP_REGS %1, %2 + vpbroadcastd m3, [pw_2048] + pxor m1, m1 +%if %1*%2 > mmsize +.pri_v_loop: +%endif + LOAD_BLOCK %1, %2 +.pri_k_loop: + vpbroadcastb m2, [priq+kq] ; pri_taps + ACCUMULATE_TAP_BYTE 2, [rsp+0], m13, m0, m2, %1, %2 ; dir + 0 + dec kq + jge .pri_k_loop + ADJUST_PIXEL %1, %2, m1, m3 +%if %1*%2 > mmsize + mov dstq, dst4q + lea top1q, [rsp+0x90] + lea top2q, [rsp+0xA0] + lea dst4q, [dst4q+strideq*4] + add hq, 4 + jl .pri_v_loop +%endif + RET + +.sec_only: + DEFINE_ARGS dst, stride, left, top, _, secdmp, zero, _, damping + movd xm1, secdmpd + lzcnt secdmpd, secdmpd + add secdmpd, dampingd + cmovs secdmpd, zerod + mov [rsp+8], secdmpq ; sec_shift + DEFINE_ARGS dst, stride, left, top, _, secdmp, table + lea tableq, [tap_table] + vpbroadcastb m14, [tableq+secdmpq] ; sec_shift_mask + ; pri/sec_taps[k] [4 total] + DEFINE_ARGS dst, stride, left, top, _, sec, table, dir + vpbroadcastb m1, xm1 ; sec_strength + lea secq, [tableq+12] ; sec_taps + PREP_REGS %1, %2 + vpbroadcastd m2, [pw_2048] + pxor m0, m0 +%if %1*%2 > mmsize +.sec_v_loop: +%endif + LOAD_BLOCK %1, %2 +.sec_k_loop: + vpbroadcastb m3, [secq+kq] ; sec_taps + ACCUMULATE_TAP_BYTE 4, [rsp+8], m14, m1, m3, %1, %2 ; dir + 2 + ACCUMULATE_TAP_BYTE 0, [rsp+8], m14, m1, m3, %1, %2 ; dir - 2 + dec kq + jge .sec_k_loop + ADJUST_PIXEL %1, %2, m0, m2 +%if %1*%2 > mmsize + mov dstq, dst4q + lea top1q, [rsp+0x90] + lea top2q, [rsp+0xA0] + lea dst4q, [dst4q+strideq*4] + add hq, 4 + jl .sec_v_loop +%endif + RET + +.d0k0: +%if %1 == 4 + %if %2 == 4 + vpbroadcastq m6, [dstq+strideq*1-1] + vpbroadcastq m10, [dstq+strideq*2-1] + movd xm5, [topq+strideq*1+1] + movd xm9, [dstq+strideq*0+1] + psrldq m11, m6, 2 + psrldq m12, m10, 2 + vinserti128 m6, [dstq+stride3q -1], 1 + vinserti128 m10, [dstq+strideq*4-1], 1 + vpblendd m5, m11, 0x10 + vpblendd m9, m12, 0x10 + movu m11, [blend_4x4+16] + punpckldq m6, m10 + punpckldq m5, m9 + vpblendvb m6, [rsp+gprsize+0x28], m11 + %else + movd xm5, [topq +strideq*1+1] + movq xm6, [dstq +strideq*1-1] + movq xm10, [dstq +stride3q -1] + movq xm11, [dst4q+strideq*1-1] + pinsrd xm5, [dstq +strideq*0+1], 1 + movhps xm6, [dstq +strideq*2-1] + movhps xm10, [dst4q+strideq*0-1] + movhps xm11, [dst4q+strideq*2-1] + psrldq xm9, xm6, 2 + shufps xm5, xm9, q2010 ; -1 +0 +1 +2 + shufps xm6, xm10, q2020 ; +1 +2 +3 +4 + psrldq xm9, xm11, 2 + psrldq xm10, 2 + shufps xm10, xm9, q2020 ; +3 +4 +5 +6 + movd xm9, [dst4q+stride3q -1] + pinsrd xm9, [dst4q+strideq*4-1], 1 + shufps xm11, xm9, q1020 ; +5 +6 +7 +8 + pmovzxbw m9, [leftq+3] + vinserti128 m6, xm11, 1 + movu m11, [blend_4x8_0+4] + vinserti128 m5, xm10, 1 + vpblendvb m6, m9, m11 + %endif +%else + lea r13, [blend_8x8_0+16] + movq xm5, [top2q +1] + vbroadcasti128 m10, [dstq+strideq*1-1] + vbroadcasti128 m11, [dstq+strideq*2-1] + movhps xm5, [dstq+strideq*0+1] + vinserti128 m6, m10, [dstq+stride3q -1], 1 + vinserti128 m9, m11, [dstq+strideq*4-1], 1 + psrldq m10, 2 + psrldq m11, 2 + punpcklqdq m6, m9 + movu m9, [r13+hq*2*1+16*1] + punpcklqdq m10, m11 + vpblendd m5, m10, 0xF0 + vpblendvb m6, [rsp+gprsize+80+hq*8+64+8*1], m9 +%endif + ret +.d1k0: +.d2k0: +.d3k0: +%if %1 == 4 + %if %2 == 4 + movq xm6, [dstq+strideq*0-1] + movq xm9, [dstq+strideq*1-1] + vinserti128 m6, [dstq+strideq*2-1], 1 + vinserti128 m9, [dstq+stride3q -1], 1 + movu m11, [rsp+gprsize+0x10] + pcmpeqd m12, m12 + psrldq m5, m6, 2 + psrldq m10, m9, 2 + psrld m12, 24 + punpckldq m6, m9 + punpckldq m5, m10 + vpblendvb m6, m11, m12 + %else + movq xm6, [dstq +strideq*0-1] + movq xm9, [dstq +strideq*2-1] + movhps xm6, [dstq +strideq*1-1] + movhps xm9, [dstq +stride3q -1] + movq xm10, [dst4q+strideq*0-1] + movhps xm10, [dst4q+strideq*1-1] + psrldq xm5, xm6, 2 + psrldq xm11, xm9, 2 + shufps xm5, xm11, q2020 + movq xm11, [dst4q+strideq*2-1] + movhps xm11, [dst4q+stride3q -1] + shufps xm6, xm9, q2020 + shufps xm9, xm10, xm11, q2020 + vinserti128 m6, xm9, 1 + pmovzxbw m9, [leftq+1] + psrldq xm10, 2 + psrldq xm11, 2 + shufps xm10, xm11, q2020 + vpbroadcastd m11, [blend_4x8_0+4] + vinserti128 m5, xm10, 1 + vpblendvb m6, m9, m11 + %endif +%else + movu xm5, [dstq+strideq*0-1] + movu xm9, [dstq+strideq*1-1] + vinserti128 m5, [dstq+strideq*2-1], 1 + vinserti128 m9, [dstq+stride3q -1], 1 + movu m10, [blend_8x8_0+16] + punpcklqdq m6, m5, m9 + vpblendvb m6, [rsp+gprsize+80+hq*8+64], m10 + psrldq m5, 2 + psrldq m9, 2 + punpcklqdq m5, m9 +%endif + ret +.d4k0: +%if %1 == 4 + %if %2 == 4 + vpbroadcastq m10, [dstq+strideq*1-1] + vpbroadcastq m11, [dstq+strideq*2-1] + movd xm6, [topq+strideq*1-1] + movd xm9, [dstq+strideq*0-1] + psrldq m5, m10, 2 + psrldq m12, m11, 2 + vpblendd m6, m10, 0x10 + vpblendd m9, m11, 0x10 + movu m10, [blend_4x4] + vinserti128 m5, [dstq+stride3q +1], 1 + vinserti128 m12, [dstq+strideq*4+1], 1 + punpckldq m6, m9 + punpckldq m5, m12 + vpblendvb m6, [rsp+gprsize+0x40], m10 + %else + movd xm6, [topq +strideq*1-1] + movq xm9, [dstq +strideq*1-1] + movq xm10, [dstq +stride3q -1] + movq xm11, [dst4q+strideq*1-1] + pinsrd xm6, [dstq +strideq*0-1], 1 + movhps xm9, [dstq +strideq*2-1] + movhps xm10, [dst4q+strideq*0-1] + movhps xm11, [dst4q+strideq*2-1] + psrldq xm5, xm9, 2 + shufps xm6, xm9, q2010 + psrldq xm9, xm10, 2 + shufps xm5, xm9, q2020 + shufps xm10, xm11, q2020 + movd xm9, [dst4q+stride3q +1] + vinserti128 m6, xm10, 1 + pinsrd xm9, [dst4q+strideq*4+1], 1 + psrldq xm11, 2 + pmovzxbw m10, [leftq-1] + shufps xm11, xm9, q1020 + movu m9, [blend_4x8_0] + vinserti128 m5, xm11, 1 + vpblendvb m6, m10, m9 + %endif +%else + lea r13, [blend_8x8_0+8] + movq xm6, [top2q -1] + vbroadcasti128 m5, [dstq+strideq*1-1] + vbroadcasti128 m9, [dstq+strideq*2-1] + movhps xm6, [dstq+strideq*0-1] + movu m11, [r13+hq*2*1+16*1] + punpcklqdq m10, m5, m9 + vinserti128 m5, [dstq+stride3q -1], 1 + vinserti128 m9, [dstq+strideq*4-1], 1 + vpblendd m6, m10, 0xF0 + vpblendvb m6, [rsp+gprsize+80+hq*8+64-8*1], m11 + psrldq m5, 2 + psrldq m9, 2 + punpcklqdq m5, m9 +%endif + ret +.d5k0: +.d6k0: +.d7k0: +%if %1 == 4 + %if %2 == 4 + movd xm6, [topq+strideq*1 ] + vpbroadcastd m5, [dstq+strideq*1 ] + vpbroadcastd m9, [dstq+strideq*2 ] + vpblendd xm6, [dstq+strideq*0-4], 0x2 + vpblendd m5, m9, 0x22 + vpblendd m6, m5, 0x30 + vinserti128 m5, [dstq+stride3q ], 1 + vpblendd m5, [dstq+strideq*4-20], 0x20 + %else + movd xm6, [topq +strideq*1] + movd xm5, [dstq +strideq*1] + movd xm9, [dstq +stride3q ] + movd xm10, [dst4q+strideq*1] + movd xm11, [dst4q+stride3q ] + pinsrd xm6, [dstq +strideq*0], 1 + pinsrd xm5, [dstq +strideq*2], 1 + pinsrd xm9, [dst4q+strideq*0], 1 + pinsrd xm10, [dst4q+strideq*2], 1 + pinsrd xm11, [dst4q+strideq*4], 1 + punpcklqdq xm6, xm5 + punpcklqdq xm5, xm9 + punpcklqdq xm9, xm10 + punpcklqdq xm10, xm11 + vinserti128 m6, xm9, 1 + vinserti128 m5, xm10, 1 + %endif +%else + movq xm6, [top2q ] + movq xm5, [dstq+strideq*1] + movq xm9, [dstq+stride3q ] + movhps xm6, [dstq+strideq*0] + movhps xm5, [dstq+strideq*2] + movhps xm9, [dstq+strideq*4] + vinserti128 m6, xm5, 1 + vinserti128 m5, xm9, 1 +%endif + ret +.d0k1: +%if %1 == 4 + %if %2 == 4 + movd xm6, [dstq +strideq*2-2] + movd xm9, [dstq +stride3q -2] + movd xm5, [topq +strideq*0+2] + movd xm10, [topq +strideq*1+2] + pinsrw xm6, [leftq+4], 0 + pinsrw xm9, [leftq+6], 0 + vinserti128 m5, [dstq +strideq*0+2], 1 + vinserti128 m10, [dstq +strideq*1+2], 1 + vinserti128 m6, [dst4q+strideq*0-2], 1 + vinserti128 m9, [dst4q+strideq*1-2], 1 + punpckldq m5, m10 + punpckldq m6, m9 + %else + movq xm6, [dstq +strideq*2-2] + movd xm10, [dst4q+strideq*2-2] + movd xm5, [topq +strideq*0+2] + movq xm9, [dst4q+strideq*0-2] + movhps xm6, [dstq +stride3q -2] + pinsrw xm10, [dst4q+stride3q ], 3 + pinsrd xm5, [topq +strideq*1+2], 1 + movhps xm9, [dst4q+strideq*1-2] + pinsrd xm10, [dst8q+strideq*0-2], 2 + pinsrd xm5, [dstq +strideq*0+2], 2 + pinsrd xm10, [dst8q+strideq*1-2], 3 + pinsrd xm5, [dstq +strideq*1+2], 3 + shufps xm11, xm6, xm9, q3131 + shufps xm6, xm9, q2020 + movu m9, [blend_4x8_3+8] + vinserti128 m6, xm10, 1 + vinserti128 m5, xm11, 1 + vpblendvb m6, [rsp+gprsize+16+8], m9 + %endif +%else + lea r13, [blend_8x8_1+16] + movq xm6, [dstq +strideq*2-2] + movq xm9, [dstq +stride3q -2] + movq xm5, [top1q +2] + movq xm10, [top2q +2] + movu m11, [r13+hq*2*2+16*2] + vinserti128 m6, [dst4q+strideq*0-2], 1 + vinserti128 m9, [dst4q+strideq*1-2], 1 + vinserti128 m5, [dstq +strideq*0+2], 1 + vinserti128 m10, [dstq +strideq*1+2], 1 + punpcklqdq m6, m9 + punpcklqdq m5, m10 + vpblendvb m6, [rsp+gprsize+16+hq*8+64+8*2], m11 +%endif + ret +.d1k1: +%if %1 == 4 + %if %2 == 4 + vpbroadcastq m6, [dstq+strideq*1-2] + vpbroadcastq m9, [dstq+strideq*2-2] + movd xm5, [topq+strideq*1+2] + movd xm10, [dstq+strideq*0+2] + psrldq m11, m6, 4 + psrldq m12, m9, 4 + vpblendd m5, m11, 0x10 + movq xm11, [leftq+2] + vinserti128 m6, [dstq+stride3q -2], 1 + punpckldq xm11, xm11 + vpblendd m10, m12, 0x10 + pcmpeqd m12, m12 + pmovzxwd m11, xm11 + psrld m12, 16 + punpckldq m6, m9 + vpbroadcastd m9, [dstq+strideq*4-2] + vpblendvb m6, m11, m12 + punpckldq m5, m10 + vpblendd m6, m9, 0x20 + %else + movd xm5, [topq +strideq*1+2] + movq xm6, [dstq +strideq*1-2] + movq xm9, [dstq +stride3q -2] + movq xm10, [dst4q+strideq*1-2] + movd xm11, [dst4q+stride3q -2] + pinsrd xm5, [dstq +strideq*0+2], 1 + movhps xm6, [dstq +strideq*2-2] + movhps xm9, [dst4q+strideq*0-2] + movhps xm10, [dst4q+strideq*2-2] + pinsrd xm11, [dst4q+strideq*4-2], 1 + shufps xm5, xm6, q3110 + shufps xm6, xm9, q2020 + shufps xm9, xm10, q3131 + shufps xm10, xm11, q1020 + movu m11, [blend_4x8_2+4] + vinserti128 m6, xm10, 1 + vinserti128 m5, xm9, 1 + vpblendvb m6, [rsp+gprsize+16+4], m11 + %endif +%else + lea r13, [blend_8x8_1+16] + movq xm5, [top2q +2] + vbroadcasti128 m6, [dstq+strideq*1-2] + vbroadcasti128 m9, [dstq+strideq*2-2] + movhps xm5, [dstq+strideq*0+2] + shufps m10, m6, m9, q2121 + vinserti128 m6, [dstq+stride3q -2], 1 + vinserti128 m9, [dstq+strideq*4-2], 1 + movu m11, [r13+hq*2*1+16*1] + vpblendd m5, m10, 0xF0 + punpcklqdq m6, m9 + vpblendvb m6, [rsp+gprsize+16+hq*8+64+8*1], m11 +%endif + ret +.d2k1: +%if %1 == 4 + %if %2 == 4 + movq xm11, [leftq] + movq xm6, [dstq+strideq*0-2] + movq xm9, [dstq+strideq*1-2] + vinserti128 m6, [dstq+strideq*2-2], 1 + vinserti128 m9, [dstq+stride3q -2], 1 + punpckldq xm11, xm11 + psrldq m5, m6, 4 + psrldq m10, m9, 4 + pmovzxwd m11, xm11 + punpckldq m6, m9 + punpckldq m5, m10 + pblendw m6, m11, 0x05 + %else + movq xm5, [dstq +strideq*0-2] + movq xm9, [dstq +strideq*2-2] + movq xm10, [dst4q+strideq*0-2] + movq xm11, [dst4q+strideq*2-2] + movhps xm5, [dstq +strideq*1-2] + movhps xm9, [dstq +stride3q -2] + movhps xm10, [dst4q+strideq*1-2] + movhps xm11, [dst4q+stride3q -2] + shufps xm6, xm5, xm9, q2020 + shufps xm5, xm9, q3131 + shufps xm9, xm10, xm11, q2020 + shufps xm10, xm11, q3131 + pmovzxwd m11, [leftq] + vinserti128 m6, xm9, 1 + vinserti128 m5, xm10, 1 + pblendw m6, m11, 0x55 + %endif +%else + mova m11, [rsp+gprsize+16+hq*8+64] + movu xm5, [dstq+strideq*0-2] + movu xm9, [dstq+strideq*1-2] + vinserti128 m5, [dstq+strideq*2-2], 1 + vinserti128 m9, [dstq+stride3q -2], 1 + shufps m6, m5, m9, q1010 + shufps m5, m9, q2121 + pblendw m6, m11, 0x11 +%endif + ret +.d3k1: +%if %1 == 4 + %if %2 == 4 + vpbroadcastq m11, [dstq+strideq*1-2] + vpbroadcastq m12, [dstq+strideq*2-2] + movd xm6, [topq+strideq*1-2] + movd xm9, [dstq+strideq*0-2] + pblendw m11, [leftq-16+2], 0x01 + pblendw m12, [leftq-16+4], 0x01 + pinsrw xm9, [leftq- 0+0], 0 + psrldq m5, m11, 4 + psrldq m10, m12, 4 + vinserti128 m5, [dstq+stride3q +2], 1 + vinserti128 m10, [dstq+strideq*4+2], 1 + vpblendd m6, m11, 0x10 + vpblendd m9, m12, 0x10 + punpckldq m6, m9 + punpckldq m5, m10 + %else + movd xm6, [topq +strideq*1-2] + movq xm5, [dstq +strideq*1-2] + movq xm9, [dstq +stride3q -2] + movq xm10, [dst4q+strideq*1-2] + movd xm11, [dst4q+stride3q +2] + pinsrw xm6, [dstq +strideq*0 ], 3 + movhps xm5, [dstq +strideq*2-2] + movhps xm9, [dst4q+strideq*0-2] + movhps xm10, [dst4q+strideq*2-2] + pinsrd xm11, [dst4q+strideq*4+2], 1 + shufps xm6, xm5, q2010 + shufps xm5, xm9, q3131 + shufps xm9, xm10, q2020 + shufps xm10, xm11, q1031 + movu m11, [blend_4x8_2] + vinserti128 m6, xm9, 1 + vinserti128 m5, xm10, 1 + vpblendvb m6, [rsp+gprsize+16-4], m11 + %endif +%else + lea r13, [blend_8x8_1+8] + movq xm6, [top2q -2] + vbroadcasti128 m5, [dstq+strideq*1-2] + vbroadcasti128 m10, [dstq+strideq*2-2] + movhps xm6, [dstq+strideq*0-2] + punpcklqdq m9, m5, m10 + vinserti128 m5, [dstq+stride3q -2], 1 + vinserti128 m10, [dstq+strideq*4-2], 1 + movu m11, [r13+hq*2*1+16*1] + vpblendd m6, m9, 0xF0 + shufps m5, m10, q2121 + vpblendvb m6, [rsp+gprsize+16+hq*8+64-8*1], m11 +%endif + ret +.d4k1: +%if %1 == 4 + %if %2 == 4 + vinserti128 m6, [dstq +strideq*0-2], 1 + vinserti128 m9, [dstq +strideq*1-2], 1 + movd xm5, [dstq +strideq*2+2] + movd xm10, [dstq +stride3q +2] + pblendw m6, [leftq-16+0], 0x01 + pblendw m9, [leftq-16+2], 0x01 + vinserti128 m5, [dst4q+strideq*0+2], 1 + vinserti128 m10, [dst4q+strideq*1+2], 1 + vpblendd m6, [topq +strideq*0-2], 0x01 + vpblendd m9, [topq +strideq*1-2], 0x01 + punpckldq m5, m10 + punpckldq m6, m9 + %else + movd xm6, [topq +strideq*0-2] + movq xm5, [dstq +strideq*2-2] + movq xm9, [dst4q+strideq*0-2] + movd xm10, [dst4q+strideq*2+2] + pinsrd xm6, [topq +strideq*1-2], 1 + movhps xm5, [dstq +stride3q -2] + movhps xm9, [dst4q+strideq*1-2] + pinsrd xm10, [dst4q+stride3q +2], 1 + pinsrd xm6, [dstq +strideq*0-2], 2 + pinsrd xm10, [dst8q+strideq*0+2], 2 + pinsrd xm6, [dstq +strideq*1-2], 3 + pinsrd xm10, [dst8q+strideq*1+2], 3 + shufps xm11, xm5, xm9, q2020 + shufps xm5, xm9, q3131 + movu m9, [blend_4x8_3] + vinserti128 m6, xm11, 1 + vinserti128 m5, xm10, 1 + vpblendvb m6, [rsp+gprsize+16-8], m9 + %endif +%else + lea r13, [blend_8x8_1] + movu m11, [r13+hq*2*2+16*2] + movq xm6, [top1q -2] + movq xm9, [top2q -2] + movq xm5, [dstq +strideq*2+2] + movq xm10, [dstq +stride3q +2] + vinserti128 m6, [dstq +strideq*0-2], 1 + vinserti128 m9, [dstq +strideq*1-2], 1 + vinserti128 m5, [dst4q+strideq*0+2], 1 + vinserti128 m10, [dst4q+strideq*1+2], 1 + punpcklqdq m6, m9 + vpblendvb m6, [rsp+gprsize+16+hq*8+64-8*2], m11 + punpcklqdq m5, m10 +%endif + ret +.d5k1: +%if %1 == 4 + %if %2 == 4 + movd xm6, [topq +strideq*0-1] + movd xm9, [topq +strideq*1-1] + movd xm5, [dstq +strideq*2+1] + movd xm10, [dstq +stride3q +1] + pcmpeqd m12, m12 + pmovzxbw m11, [leftq-8+1] + psrld m12, 24 + vinserti128 m6, [dstq +strideq*0-1], 1 + vinserti128 m9, [dstq +strideq*1-1], 1 + vinserti128 m5, [dst4q+strideq*0+1], 1 + vinserti128 m10, [dst4q+strideq*1+1], 1 + punpckldq m6, m9 + pxor m9, m9 + vpblendd m12, m9, 0x0F + punpckldq m5, m10 + vpblendvb m6, m11, m12 + %else + movd xm6, [topq +strideq*0-1] + movq xm5, [dstq +strideq*2-1] + movq xm9, [dst4q+strideq*0-1] + movd xm10, [dst4q+strideq*2+1] + pinsrd xm6, [topq +strideq*1-1], 1 + movhps xm5, [dstq +stride3q -1] + movhps xm9, [dst4q+strideq*1-1] + pinsrd xm10, [dst4q+stride3q +1], 1 + pinsrd xm6, [dstq +strideq*0-1], 2 + pinsrd xm10, [dst8q+strideq*0+1], 2 + pinsrd xm6, [dstq +strideq*1-1], 3 + pinsrd xm10, [dst8q+strideq*1+1], 3 + shufps xm11, xm5, xm9, q2020 + vinserti128 m6, xm11, 1 + pmovzxbw m11, [leftq-3] + psrldq xm5, 2 + psrldq xm9, 2 + shufps xm5, xm9, q2020 + movu m9, [blend_4x8_1] + vinserti128 m5, xm10, 1 + vpblendvb m6, m11, m9 + %endif +%else + lea r13, [blend_8x8_0] + movu m11, [r13+hq*2*2+16*2] + movq xm6, [top1q -1] + movq xm9, [top2q -1] + movq xm5, [dstq +strideq*2+1] + movq xm10, [dstq +stride3q +1] + vinserti128 m6, [dstq +strideq*0-1], 1 + vinserti128 m9, [dstq +strideq*1-1], 1 + vinserti128 m5, [dst4q+strideq*0+1], 1 + vinserti128 m10, [dst4q+strideq*1+1], 1 + punpcklqdq m6, m9 + punpcklqdq m5, m10 + vpblendvb m6, [rsp+gprsize+80+hq*8+64-8*2], m11 +%endif + ret +.d6k1: +%if %1 == 4 + %if %2 == 4 + movd xm6, [topq +strideq*0] + movd xm9, [topq +strideq*1] + movd xm5, [dstq +strideq*2] + movd xm10, [dstq +stride3q ] + vinserti128 m6, [dstq +strideq*0], 1 + vinserti128 m9, [dstq +strideq*1], 1 + vinserti128 m5, [dst4q+strideq*0], 1 + vinserti128 m10, [dst4q+strideq*1], 1 + punpckldq m6, m9 + punpckldq m5, m10 + %else + movd xm5, [dstq +strideq*2] + movd xm6, [topq +strideq*0] + movd xm9, [dst4q+strideq*2] + pinsrd xm5, [dstq +stride3q ], 1 + pinsrd xm6, [topq +strideq*1], 1 + pinsrd xm9, [dst4q+stride3q ], 1 + pinsrd xm5, [dst4q+strideq*0], 2 + pinsrd xm6, [dstq +strideq*0], 2 + pinsrd xm9, [dst8q+strideq*0], 2 + pinsrd xm5, [dst4q+strideq*1], 3 + pinsrd xm6, [dstq +strideq*1], 3 + pinsrd xm9, [dst8q+strideq*1], 3 + vinserti128 m6, xm5, 1 + vinserti128 m5, xm9, 1 + %endif +%else + movq xm5, [dstq +strideq*2] + movq xm9, [dst4q+strideq*0] + movq xm6, [top1q ] + movq xm10, [dstq +strideq*0] + movhps xm5, [dstq +stride3q ] + movhps xm9, [dst4q+strideq*1] + movhps xm6, [top2q ] + movhps xm10, [dstq +strideq*1] + vinserti128 m5, xm9, 1 + vinserti128 m6, xm10, 1 +%endif + ret +.d7k1: +%if %1 == 4 + %if %2 == 4 + movd xm5, [dstq +strideq*2-1] + movd xm9, [dstq +stride3q -1] + movd xm6, [topq +strideq*0+1] + movd xm10, [topq +strideq*1+1] + pinsrb xm5, [leftq+ 5], 0 + pinsrb xm9, [leftq+ 7], 0 + vinserti128 m6, [dstq +strideq*0+1], 1 + vinserti128 m10, [dstq +strideq*1+1], 1 + vinserti128 m5, [dst4q+strideq*0-1], 1 + vinserti128 m9, [dst4q+strideq*1-1], 1 + punpckldq m6, m10 + punpckldq m5, m9 + %else + movd xm6, [topq +strideq*0+1] + movq xm9, [dstq +strideq*2-1] + movq xm10, [dst4q+strideq*0-1] + movd xm11, [dst4q+strideq*2-1] + pinsrd xm6, [topq +strideq*1+1], 1 + movhps xm9, [dstq +stride3q -1] + movhps xm10, [dst4q+strideq*1-1] + pinsrd xm11, [dst4q+stride3q -1], 1 + pinsrd xm6, [dstq +strideq*0+1], 2 + pinsrd xm11, [dst8q+strideq*0-1], 2 + pinsrd xm6, [dstq +strideq*1+1], 3 + pinsrd xm11, [dst8q+strideq*1-1], 3 + shufps xm5, xm9, xm10, q2020 + vinserti128 m5, xm11, 1 + pmovzxbw m11, [leftq+5] + psrldq xm9, 2 + psrldq xm10, 2 + shufps xm9, xm10, q2020 + movu m10, [blend_4x8_1+8] + vinserti128 m6, xm9, 1 + vpblendvb m5, m11, m10 + %endif +%else + lea r13, [blend_8x8_0+16] + movq xm5, [dstq +strideq*2-1] + movq xm9, [dst4q+strideq*0-1] + movq xm6, [top1q +1] + movq xm10, [dstq +strideq*0+1] + movhps xm5, [dstq +stride3q -1] + movhps xm9, [dst4q+strideq*1-1] + movhps xm6, [top2q +1] + movhps xm10, [dstq +strideq*1+1] + movu m11, [r13+hq*2*2+16*2] + vinserti128 m5, xm9, 1 + vinserti128 m6, xm10, 1 + vpblendvb m5, [rsp+gprsize+80+hq*8+64+8*2], m11 +%endif + ret + +.border_block: + DEFINE_ARGS dst, stride, left, top, pri, sec, stride3, dst4, edge +%define rstk rsp +%assign stack_offset stack_offset_entry +%if %1 == 4 && %2 == 8 + PUSH r9 + %assign regs_used 10 +%else + %assign regs_used 9 +%endif +%if STACK_ALIGNMENT < 32 + PUSH r%+regs_used + %assign regs_used regs_used+1 +%endif + ALLOC_STACK 2*16+(%2+4)*32, 16 +%define px rsp+2*16+2*32 + + pcmpeqw m14, m14 + psllw m14, 15 ; 0x8000 + + ; prepare pixel buffers - body/right +%if %1 == 4 + INIT_XMM avx2 +%endif +%if %2 == 8 + lea dst4q, [dstq+strideq*4] +%endif + lea stride3q, [strideq*3] + test edgeb, 2 ; have_right + jz .no_right + pmovzxbw m1, [dstq+strideq*0] + pmovzxbw m2, [dstq+strideq*1] + pmovzxbw m3, [dstq+strideq*2] + pmovzxbw m4, [dstq+stride3q] + mova [px+0*32], m1 + mova [px+1*32], m2 + mova [px+2*32], m3 + mova [px+3*32], m4 +%if %2 == 8 + pmovzxbw m1, [dst4q+strideq*0] + pmovzxbw m2, [dst4q+strideq*1] + pmovzxbw m3, [dst4q+strideq*2] + pmovzxbw m4, [dst4q+stride3q] + mova [px+4*32], m1 + mova [px+5*32], m2 + mova [px+6*32], m3 + mova [px+7*32], m4 +%endif + jmp .body_done +.no_right: +%if %1 == 4 + movd xm1, [dstq+strideq*0] + movd xm2, [dstq+strideq*1] + movd xm3, [dstq+strideq*2] + movd xm4, [dstq+stride3q] + pmovzxbw xm1, xm1 + pmovzxbw xm2, xm2 + pmovzxbw xm3, xm3 + pmovzxbw xm4, xm4 + movq [px+0*32], xm1 + movq [px+1*32], xm2 + movq [px+2*32], xm3 + movq [px+3*32], xm4 +%else + pmovzxbw xm1, [dstq+strideq*0] + pmovzxbw xm2, [dstq+strideq*1] + pmovzxbw xm3, [dstq+strideq*2] + pmovzxbw xm4, [dstq+stride3q] + mova [px+0*32], xm1 + mova [px+1*32], xm2 + mova [px+2*32], xm3 + mova [px+3*32], xm4 +%endif + movd [px+0*32+%1*2], xm14 + movd [px+1*32+%1*2], xm14 + movd [px+2*32+%1*2], xm14 + movd [px+3*32+%1*2], xm14 +%if %2 == 8 + %if %1 == 4 + movd xm1, [dst4q+strideq*0] + movd xm2, [dst4q+strideq*1] + movd xm3, [dst4q+strideq*2] + movd xm4, [dst4q+stride3q] + pmovzxbw xm1, xm1 + pmovzxbw xm2, xm2 + pmovzxbw xm3, xm3 + pmovzxbw xm4, xm4 + movq [px+4*32], xm1 + movq [px+5*32], xm2 + movq [px+6*32], xm3 + movq [px+7*32], xm4 + %else + pmovzxbw xm1, [dst4q+strideq*0] + pmovzxbw xm2, [dst4q+strideq*1] + pmovzxbw xm3, [dst4q+strideq*2] + pmovzxbw xm4, [dst4q+stride3q] + mova [px+4*32], xm1 + mova [px+5*32], xm2 + mova [px+6*32], xm3 + mova [px+7*32], xm4 + %endif + movd [px+4*32+%1*2], xm14 + movd [px+5*32+%1*2], xm14 + movd [px+6*32+%1*2], xm14 + movd [px+7*32+%1*2], xm14 +%endif +.body_done: + + ; top + test edgeb, 4 ; have_top + jz .no_top + test edgeb, 1 ; have_left + jz .top_no_left + test edgeb, 2 ; have_right + jz .top_no_right + pmovzxbw m1, [topq+strideq*0-(%1/2)] + pmovzxbw m2, [topq+strideq*1-(%1/2)] + movu [px-2*32-%1], m1 + movu [px-1*32-%1], m2 + jmp .top_done +.top_no_right: + pmovzxbw m1, [topq+strideq*0-%1] + pmovzxbw m2, [topq+strideq*1-%1] + movu [px-2*32-%1*2], m1 + movu [px-1*32-%1*2], m2 + movd [px-2*32+%1*2], xm14 + movd [px-1*32+%1*2], xm14 + jmp .top_done +.top_no_left: + test edgeb, 2 ; have_right + jz .top_no_left_right + pmovzxbw m1, [topq+strideq*0] + pmovzxbw m2, [topq+strideq*1] + mova [px-2*32+0], m1 + mova [px-1*32+0], m2 + movd [px-2*32-4], xm14 + movd [px-1*32-4], xm14 + jmp .top_done +.top_no_left_right: +%if %1 == 4 + movd xm1, [topq+strideq*0] + pinsrd xm1, [topq+strideq*1], 1 + pmovzxbw xm1, xm1 + movq [px-2*32+0], xm1 + movhps [px-1*32+0], xm1 +%else + pmovzxbw xm1, [topq+strideq*0] + pmovzxbw xm2, [topq+strideq*1] + mova [px-2*32+0], xm1 + mova [px-1*32+0], xm2 +%endif + movd [px-2*32-4], xm14 + movd [px-1*32-4], xm14 + movd [px-2*32+%1*2], xm14 + movd [px-1*32+%1*2], xm14 + jmp .top_done +.no_top: + movu [px-2*32-%1], m14 + movu [px-1*32-%1], m14 +.top_done: + + ; left + test edgeb, 1 ; have_left + jz .no_left + pmovzxbw xm1, [leftq+ 0] +%if %2 == 8 + pmovzxbw xm2, [leftq+ 8] +%endif + movd [px+0*32-4], xm1 + pextrd [px+1*32-4], xm1, 1 + pextrd [px+2*32-4], xm1, 2 + pextrd [px+3*32-4], xm1, 3 +%if %2 == 8 + movd [px+4*32-4], xm2 + pextrd [px+5*32-4], xm2, 1 + pextrd [px+6*32-4], xm2, 2 + pextrd [px+7*32-4], xm2, 3 +%endif + jmp .left_done +.no_left: + movd [px+0*32-4], xm14 + movd [px+1*32-4], xm14 + movd [px+2*32-4], xm14 + movd [px+3*32-4], xm14 +%if %2 == 8 + movd [px+4*32-4], xm14 + movd [px+5*32-4], xm14 + movd [px+6*32-4], xm14 + movd [px+7*32-4], xm14 +%endif +.left_done: + + ; bottom + DEFINE_ARGS dst, stride, dst8, dummy1, pri, sec, stride3, dummy3, edge + test edgeb, 8 ; have_bottom + jz .no_bottom + lea dst8q, [dstq+%2*strideq] + test edgeb, 1 ; have_left + jz .bottom_no_left + test edgeb, 2 ; have_right + jz .bottom_no_right + pmovzxbw m1, [dst8q-(%1/2)] + pmovzxbw m2, [dst8q+strideq-(%1/2)] + movu [px+(%2+0)*32-%1], m1 + movu [px+(%2+1)*32-%1], m2 + jmp .bottom_done +.bottom_no_right: + pmovzxbw m1, [dst8q-%1] + pmovzxbw m2, [dst8q+strideq-%1] + movu [px+(%2+0)*32-%1*2], m1 + movu [px+(%2+1)*32-%1*2], m2 +%if %1 == 8 + movd [px+(%2-1)*32+%1*2], xm14 ; overwritten by previous movu +%endif + movd [px+(%2+0)*32+%1*2], xm14 + movd [px+(%2+1)*32+%1*2], xm14 + jmp .bottom_done +.bottom_no_left: + test edgeb, 2 ; have_right + jz .bottom_no_left_right + pmovzxbw m1, [dst8q] + pmovzxbw m2, [dst8q+strideq] + mova [px+(%2+0)*32+0], m1 + mova [px+(%2+1)*32+0], m2 + movd [px+(%2+0)*32-4], xm14 + movd [px+(%2+1)*32-4], xm14 + jmp .bottom_done +.bottom_no_left_right: +%if %1 == 4 + movd xm1, [dst8q] + pinsrd xm1, [dst8q+strideq], 1 + pmovzxbw xm1, xm1 + movq [px+(%2+0)*32+0], xm1 + movhps [px+(%2+1)*32+0], xm1 +%else + pmovzxbw xm1, [dst8q] + pmovzxbw xm2, [dst8q+strideq] + mova [px+(%2+0)*32+0], xm1 + mova [px+(%2+1)*32+0], xm2 +%endif + movd [px+(%2+0)*32-4], xm14 + movd [px+(%2+1)*32-4], xm14 + movd [px+(%2+0)*32+%1*2], xm14 + movd [px+(%2+1)*32+%1*2], xm14 + jmp .bottom_done +.no_bottom: + movu [px+(%2+0)*32-%1], m14 + movu [px+(%2+1)*32-%1], m14 +.bottom_done: + + ; actual filter + INIT_YMM avx2 + DEFINE_ARGS dst, stride, pridmp, damping, pri, secdmp, stride3, zero +%undef edged + ; register to shuffle values into after packing + vbroadcasti128 m12, [shufb_lohi] + + mov dampingd, r7m + xor zerod, zerod + movifnidn prid, prim + sub dampingd, 31 + movifnidn secdmpd, secdmpm + test prid, prid + jz .border_sec_only + movd xm0, prid + lzcnt pridmpd, prid + add pridmpd, dampingd + cmovs pridmpd, zerod + mov [rsp+0], pridmpq ; pri_shift + test secdmpd, secdmpd + jz .border_pri_only + movd xm1, secdmpd + lzcnt secdmpd, secdmpd + add secdmpd, dampingd + cmovs secdmpd, zerod + mov [rsp+8], secdmpq ; sec_shift + + DEFINE_ARGS dst, stride, pridmp, table, pri, secdmp, stride3 + lea tableq, [tap_table] + vpbroadcastb m13, [tableq+pridmpq] ; pri_shift_mask + vpbroadcastb m14, [tableq+secdmpq] ; sec_shift_mask + + ; pri/sec_taps[k] [4 total] + DEFINE_ARGS dst, stride, dir, table, pri, sec, stride3 + vpbroadcastb m0, xm0 ; pri_strength + vpbroadcastb m1, xm1 ; sec_strength + and prid, 1 + lea priq, [tableq+priq*2+8] ; pri_taps + lea secq, [tableq+12] ; sec_taps + + BORDER_PREP_REGS %1, %2 +%if %1*%2*2/mmsize > 1 +.border_v_loop: +%endif + BORDER_LOAD_BLOCK %1, %2, 1 +.border_k_loop: + vpbroadcastb m2, [priq+kq] ; pri_taps + vpbroadcastb m3, [secq+kq] ; sec_taps + ACCUMULATE_TAP_WORD 0*2, [rsp+0], m13, m0, m2, %1, 1 + ACCUMULATE_TAP_WORD 2*2, [rsp+8], m14, m1, m3, %1, 1 + ACCUMULATE_TAP_WORD 6*2, [rsp+8], m14, m1, m3, %1, 1 + dec kq + jge .border_k_loop + + vpbroadcastd m10, [pw_2048] + BORDER_ADJUST_PIXEL %1, m10, 1 +%if %1*%2*2/mmsize > 1 + %define vloop_lines (mmsize/(%1*2)) + lea dstq, [dstq+strideq*vloop_lines] + add stkq, 32*vloop_lines + dec hd + jg .border_v_loop +%endif + RET + +.border_pri_only: + DEFINE_ARGS dst, stride, pridmp, table, pri, _, stride3 + lea tableq, [tap_table] + vpbroadcastb m13, [tableq+pridmpq] ; pri_shift_mask + DEFINE_ARGS dst, stride, dir, table, pri, _, stride3 + vpbroadcastb m0, xm0 ; pri_strength + and prid, 1 + lea priq, [tableq+priq*2+8] ; pri_taps + BORDER_PREP_REGS %1, %2 + vpbroadcastd m1, [pw_2048] +%if %1*%2*2/mmsize > 1 +.border_pri_v_loop: +%endif + BORDER_LOAD_BLOCK %1, %2 +.border_pri_k_loop: + vpbroadcastb m2, [priq+kq] ; pri_taps + ACCUMULATE_TAP_WORD 0*2, [rsp+0], m13, m0, m2, %1 + dec kq + jge .border_pri_k_loop + BORDER_ADJUST_PIXEL %1, m1 +%if %1*%2*2/mmsize > 1 + %define vloop_lines (mmsize/(%1*2)) + lea dstq, [dstq+strideq*vloop_lines] + add stkq, 32*vloop_lines + dec hd + jg .border_pri_v_loop +%endif + RET + +.border_sec_only: + DEFINE_ARGS dst, stride, _, damping, _, secdmp, stride3, zero + movd xm1, secdmpd + lzcnt secdmpd, secdmpd + add secdmpd, dampingd + cmovs secdmpd, zerod + mov [rsp+8], secdmpq ; sec_shift + DEFINE_ARGS dst, stride, _, table, _, secdmp, stride3 + lea tableq, [tap_table] + vpbroadcastb m14, [tableq+secdmpq] ; sec_shift_mask + DEFINE_ARGS dst, stride, dir, table, _, sec, stride3 + vpbroadcastb m1, xm1 ; sec_strength + lea secq, [tableq+12] ; sec_taps + BORDER_PREP_REGS %1, %2 + vpbroadcastd m0, [pw_2048] +%if %1*%2*2/mmsize > 1 +.border_sec_v_loop: +%endif + BORDER_LOAD_BLOCK %1, %2 +.border_sec_k_loop: + vpbroadcastb m3, [secq+kq] ; sec_taps + ACCUMULATE_TAP_WORD 2*2, [rsp+8], m14, m1, m3, %1 + ACCUMULATE_TAP_WORD 6*2, [rsp+8], m14, m1, m3, %1 + dec kq + jge .border_sec_k_loop + BORDER_ADJUST_PIXEL %1, m0 +%if %1*%2*2/mmsize > 1 + %define vloop_lines (mmsize/(%1*2)) + lea dstq, [dstq+strideq*vloop_lines] + add stkq, 32*vloop_lines + dec hd + jg .border_sec_v_loop +%endif + RET +%endmacro + +CDEF_FILTER 8, 8 +CDEF_FILTER 4, 8 +CDEF_FILTER 4, 4 + +INIT_YMM avx2 +cglobal cdef_dir, 3, 4, 15, src, stride, var, stride3 + lea stride3q, [strideq*3] + movq xm0, [srcq+strideq*0] + movq xm1, [srcq+strideq*1] + movq xm2, [srcq+strideq*2] + movq xm3, [srcq+stride3q] + lea srcq, [srcq+strideq*4] + vpbroadcastq m4, [srcq+strideq*0] + vpbroadcastq m5, [srcq+strideq*1] + vpbroadcastq m6, [srcq+strideq*2] + vpbroadcastq m7, [srcq+stride3q] + vpbroadcastd m8, [pw_128] + pxor m9, m9 + + vpblendd m0, m0, m7, 0xf0 + vpblendd m1, m1, m6, 0xf0 + vpblendd m2, m2, m5, 0xf0 + vpblendd m3, m3, m4, 0xf0 + + punpcklbw m0, m9 + punpcklbw m1, m9 + punpcklbw m2, m9 + punpcklbw m3, m9 + + psubw m0, m8 + psubw m1, m8 + psubw m2, m8 + psubw m3, m8 + + ; shuffle registers to generate partial_sum_diag[0-1] together + vpermq m7, m0, q1032 + vpermq m6, m1, q1032 + vpermq m5, m2, q1032 + vpermq m4, m3, q1032 + + ; start with partial_sum_hv[0-1] + paddw m8, m0, m1 + paddw m9, m2, m3 + phaddw m10, m0, m1 + phaddw m11, m2, m3 + paddw m8, m9 + phaddw m10, m11 + vextracti128 xm9, m8, 1 + vextracti128 xm11, m10, 1 + paddw xm8, xm9 ; partial_sum_hv[1] + phaddw xm10, xm11 ; partial_sum_hv[0] + vinserti128 m8, xm10, 1 + vpbroadcastd m9, [div_table+44] + pmaddwd m8, m8 + pmulld m8, m9 ; cost6[2a-d] | cost2[a-d] + + ; create aggregates [lower half]: + ; m9 = m0:01234567+m1:x0123456+m2:xx012345+m3:xxx01234+ + ; m4:xxxx0123+m5:xxxxx012+m6:xxxxxx01+m7:xxxxxxx0 + ; m10= m1:7xxxxxxx+m2:67xxxxxx+m3:567xxxxx+ + ; m4:4567xxxx+m5:34567xxx+m6:234567xx+m7:1234567x + ; and [upper half]: + ; m9 = m0:xxxxxxx0+m1:xxxxxx01+m2:xxxxx012+m3:xxxx0123+ + ; m4:xxx01234+m5:xx012345+m6:x0123456+m7:01234567 + ; m10= m0:1234567x+m1:234567xx+m2:34567xxx+m3:4567xxxx+ + ; m4:567xxxxx+m5:67xxxxxx+m6:7xxxxxxx + ; and then shuffle m11 [shufw_6543210x], unpcklwd, pmaddwd, pmulld, paddd + + pslldq m9, m1, 2 + psrldq m10, m1, 14 + pslldq m11, m2, 4 + psrldq m12, m2, 12 + pslldq m13, m3, 6 + psrldq m14, m3, 10 + paddw m9, m11 + paddw m10, m12 + paddw m9, m13 + paddw m10, m14 + pslldq m11, m4, 8 + psrldq m12, m4, 8 + pslldq m13, m5, 10 + psrldq m14, m5, 6 + paddw m9, m11 + paddw m10, m12 + paddw m9, m13 + paddw m10, m14 + pslldq m11, m6, 12 + psrldq m12, m6, 4 + pslldq m13, m7, 14 + psrldq m14, m7, 2 + paddw m9, m11 + paddw m10, m12 + paddw m9, m13 + paddw m10, m14 ; partial_sum_diag[0/1][8-14,zero] + vbroadcasti128 m14, [shufw_6543210x] + vbroadcasti128 m13, [div_table+16] + vbroadcasti128 m12, [div_table+0] + paddw m9, m0 ; partial_sum_diag[0/1][0-7] + pshufb m10, m14 + punpckhwd m11, m9, m10 + punpcklwd m9, m10 + pmaddwd m11, m11 + pmaddwd m9, m9 + pmulld m11, m13 + pmulld m9, m12 + paddd m9, m11 ; cost0[a-d] | cost4[a-d] + + ; merge horizontally and vertically for partial_sum_alt[0-3] + paddw m10, m0, m1 + paddw m11, m2, m3 + paddw m12, m4, m5 + paddw m13, m6, m7 + phaddw m0, m4 + phaddw m1, m5 + phaddw m2, m6 + phaddw m3, m7 + + ; create aggregates [lower half]: + ; m4 = m10:01234567+m11:x0123456+m12:xx012345+m13:xxx01234 + ; m11= m11:7xxxxxxx+m12:67xxxxxx+m13:567xxxxx + ; and [upper half]: + ; m4 = m10:xxx01234+m11:xx012345+m12:x0123456+m13:01234567 + ; m11= m10:567xxxxx+m11:67xxxxxx+m12:7xxxxxxx + ; and then pshuflw m11 3012, unpcklwd, pmaddwd, pmulld, paddd + + pslldq m4, m11, 2 + psrldq m11, 14 + pslldq m5, m12, 4 + psrldq m12, 12 + pslldq m6, m13, 6 + psrldq m13, 10 + paddw m4, m10 + paddw m11, m12 + vpbroadcastd m12, [div_table+44] + paddw m5, m6 + paddw m11, m13 ; partial_sum_alt[3/2] right + vbroadcasti128 m13, [div_table+32] + paddw m4, m5 ; partial_sum_alt[3/2] left + pshuflw m5, m11, q3012 + punpckhwd m6, m11, m4 + punpcklwd m4, m5 + pmaddwd m6, m6 + pmaddwd m4, m4 + pmulld m6, m12 + pmulld m4, m13 + paddd m4, m6 ; cost7[a-d] | cost5[a-d] + + ; create aggregates [lower half]: + ; m5 = m0:01234567+m1:x0123456+m2:xx012345+m3:xxx01234 + ; m1 = m1:7xxxxxxx+m2:67xxxxxx+m3:567xxxxx + ; and [upper half]: + ; m5 = m0:xxx01234+m1:xx012345+m2:x0123456+m3:01234567 + ; m1 = m0:567xxxxx+m1:67xxxxxx+m2:7xxxxxxx + ; and then pshuflw m1 3012, unpcklwd, pmaddwd, pmulld, paddd + + pslldq m5, m1, 2 + psrldq m1, 14 + pslldq m6, m2, 4 + psrldq m2, 12 + pslldq m7, m3, 6 + psrldq m3, 10 + paddw m5, m0 + paddw m1, m2 + paddw m6, m7 + paddw m1, m3 ; partial_sum_alt[0/1] right + paddw m5, m6 ; partial_sum_alt[0/1] left + pshuflw m0, m1, q3012 + punpckhwd m1, m5 + punpcklwd m5, m0 + pmaddwd m1, m1 + pmaddwd m5, m5 + pmulld m1, m12 + pmulld m5, m13 + paddd m5, m1 ; cost1[a-d] | cost3[a-d] + + mova xm0, [pd_47130256+ 16] + mova m1, [pd_47130256] + phaddd m9, m8 + phaddd m5, m4 + phaddd m9, m5 + vpermd m0, m9 ; cost[0-3] + vpermd m1, m9 ; cost[4-7] | cost[0-3] + + ; now find the best cost + pmaxsd xm2, xm0, xm1 + pshufd xm3, xm2, q1032 + pmaxsd xm2, xm3 + pshufd xm3, xm2, q2301 + pmaxsd xm2, xm3 ; best cost + + ; find the idx using minpos + ; make everything other than the best cost negative via subtraction + ; find the min of unsigned 16-bit ints to sort out the negative values + psubd xm4, xm1, xm2 + psubd xm3, xm0, xm2 + packssdw xm3, xm4 + phminposuw xm3, xm3 + + ; convert idx to 32-bits + psrld xm3, 16 + movd eax, xm3 + + ; get idx^4 complement + vpermd m3, m1 + psubd xm2, xm3 + psrld xm2, 10 + movd [varq], xm2 + RET + +%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/cdef_avx512.asm b/third_party/dav1d/src/x86/cdef_avx512.asm new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/x86/cdef_avx512.asm @@ -0,0 +1,867 @@ +; Copyright © 2020, VideoLAN and dav1d authors +; Copyright © 2020, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "ext/x86/x86inc.asm" + +%if HAVE_AVX512ICL && ARCH_X86_64 + +%macro DUP4 1-* + %rep %0 + times 4 db %1 + %rotate 1 + %endrep +%endmacro + +%macro DIRS 16 ; cdef_directions[] + %rep 4 + 16 + 4 ; 6 7 0 1 2 3 4 5 6 7 0 1 + ; masking away unused bits allows us to use a single vpaddd {1to16} + ; instruction instead of having to do vpbroadcastd + paddb + db %13 & 0x3f, -%13 & 0x3f + %rotate 1 + %endrep +%endmacro + +SECTION_RODATA 64 + +lut_perm_4x4: db 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79 + db 16, 17, 0, 1, 2, 3, 4, 5, 18, 19, 8, 9, 10, 11, 12, 13 + db 20, 21, 80, 81, 82, 83, 84, 85, 22, 23, 32, 33, 34, 35, 36, 37 + db 98, 99,100,101,102,103,104,105, 50, 51, 52, 53, 54, 55, 56, 57 +lut_perm_4x8a: db 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79 + db 96, 97, 0, 1, 2, 3, 4, 5, 98, 99, 8, 9, 10, 11, 12, 13 +lut_perm_4x8b:db 100,101, 16, 17, 18, 19, 20, 21,102,103, 24, 25, 26, 27, 28, 29 + db 104,105, 32, 33, 34, 35, 36, 37,106,107, 40, 41, 42, 43, 44, 45 + db 108,109, 48, 49, 50, 51, 52, 53,110,111, 56, 57, 58, 59, 60, 61 + db 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95 +pd_01234567: dd 0, 1, 2, 3, 4, 5, 6, 7 +lut_perm_8x8a: db 0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23 + db -1, -1, 34, 35, 36, 37, 38, 39, -1, -1, 50, 51, 52, 53, 54, 55 + db -1, -1, 66, 67, 68, 69, 70, 71, -1, -1, 82, 83, 84, 85, 86, 87 + db 96, 97, 98, 99,100,101,102,103,112,113,114,115,116,117,118,119 +lut_perm_8x8b: db 4, 5, 6, 7, 8, 9, 10, 11, 20, 21, 22, 23, 24, 25, 26, 27 + db 36, 37, 38, 39, 40, 41, 42, 43, 52, 53, 54, 55, 56, 57, 58, 59 + db 68, 69, 70, 71, 72, 73, 74, 75, 84, 85, 86, 87, 88, 89, 90, 91 + db 100,101,102,103,104,105,106,107,116,117,118,119,120,121,122,123 +edge_mask: dq 0x00003c3c3c3c0000, 0x00003f3f3f3f0000 ; 0000, 0001 + dq 0x0000fcfcfcfc0000, 0x0000ffffffff0000 ; 0010, 0011 + dq 0x00003c3c3c3c3c3c, 0x00003f3f3f3f3f3f ; 0100, 0101 + dq 0x0000fcfcfcfcfcfc, 0x0000ffffffffffff ; 0110, 0111 + dq 0x3c3c3c3c3c3c0000, 0x3f3f3f3f3f3f0000 ; 1000, 1001 + dq 0xfcfcfcfcfcfc0000, 0xffffffffffff0000 ; 1010, 1011 + dq 0x3c3c3c3c3c3c3c3c, 0x3f3f3f3f3f3f3f3f ; 1100, 1101 + dq 0xfcfcfcfcfcfcfcfc, 0xffffffffffffffff ; 1110, 1111 +px_idx: DUP4 18, 19, 20, 21, 26, 27, 28, 29, 34, 35, 36, 37, 42, 43, 44, 45 +cdef_dirs: DIRS -7,-14, 1, -6, 1, 2, 1, 10, 9, 18, 8, 17, 8, 16, 8, 15 +gf_shr: dq 0x0102040810204080, 0x0102040810204080 ; >> 0, >> 0 + dq 0x0204081020408000, 0x0408102040800000 ; >> 1, >> 2 + dq 0x0810204080000000, 0x1020408000000000 ; >> 3, >> 4 + dq 0x2040800000000000, 0x4080000000000000 ; >> 5, >> 6 + times 16 db 0 ; realign (introduced by cdef_dirs) +end_perm_w8clip:db 0, 4, 8, 12, 2, 6, 10, 14, 16, 20, 24, 28, 18, 22, 26, 30 + db 32, 36, 40, 44, 34, 38, 42, 46, 48, 52, 56, 60, 50, 54, 58, 62 + db 1, 5, 9, 13, 3, 7, 11, 15, 17, 21, 25, 29, 19, 23, 27, 31 + db 33, 37, 41, 45, 35, 39, 43, 47, 49, 53, 57, 61, 51, 55, 59, 63 +end_perm: db 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61 + db 3, 7, 11, 15, 19, 23, 27, 31, 35, 39, 43, 47, 51, 55, 59, 63 +pri_tap: db 64, 64, 32, 32, 48, 48, 48, 48 ; left-shifted by 4 +sec_tap: db 32, 32, 16, 16 +pd_268435568: dd 268435568 + +SECTION .text + +%if WIN64 +DECLARE_REG_TMP 5, 6 +%else +DECLARE_REG_TMP 8, 5 +%endif + +; lut: +; t0 t1 t2 t3 t4 t5 t6 t7 +; T0 T1 T2 T3 T4 T5 T6 T7 +; L0 L1 00 01 02 03 04 05 +; L2 L3 10 11 12 13 14 15 +; L4 L5 20 21 22 23 24 25 +; L6 L7 30 31 32 33 34 35 +; 4e 4f 40 41 42 43 44 45 +; 5e 5f 50 51 52 53 54 55 + +INIT_ZMM avx512icl +cglobal cdef_filter_4x4, 4, 8, 13, dst, stride, left, top, pri, sec, dir, damping, edge +%define base r7-edge_mask + movq xmm0, [dstq+strideq*0] + movhps xmm0, [dstq+strideq*1] + lea r7, [edge_mask] + movq xmm1, [topq+strideq*0-2] + movhps xmm1, [topq+strideq*1-2] + mov r6d, edgem + vinserti32x4 ym0, ymm0, [leftq], 1 + lea r2, [strideq*3] + vinserti32x4 ym1, ymm1, [dstq+strideq*2], 1 + mova m5, [base+lut_perm_4x4] + vinserti32x4 m0, [dstq+r2], 2 + test r6b, 0x08 ; avoid buffer overread + jz .main + lea r3, [dstq+strideq*4-4] + vinserti32x4 m1, [r3+strideq*0], 2 + vinserti32x4 m0, [r3+strideq*1], 3 +.main: + movifnidn prid, prim + mov t0d, dirm + mova m3, [base+px_idx] + mov r3d, dampingm + vpermi2b m5, m0, m1 ; lut + vpbroadcastd m0, [base+pd_268435568] ; (1 << 28) + (7 << 4) + pxor m7, m7 + lea r3, [r7+r3*8] ; gf_shr + (damping - 30) * 8 + vpermb m6, m3, m5 ; px + cmp r6d, 0x0f + jne .mask_edges ; mask edges only if required + test prid, prid + jz .sec_only + vpaddd m1, m3, [base+cdef_dirs+(t0+2)*4] {1to16} ; dir + vpermb m1, m1, m5 ; k0p0 k0p1 k1p0 k1p1 +%macro CDEF_FILTER_4x4_PRI 0 + vpcmpub k1, m6, m1, 6 ; px > pN + psubb m2, m1, m6 + lzcnt r6d, prid + vpsubb m2{k1}, m6, m1 ; abs(diff) + vpbroadcastb m4, prid + and prid, 1 + vgf2p8affineqb m9, m2, [r3+r6*8] {1to8}, 0 ; abs(diff) >> shift + movifnidn t1d, secm + vpbroadcastd m10, [base+pri_tap+priq*4] + vpsubb m10{k1}, m7, m10 ; apply_sign(pri_tap) + psubusb m4, m9 ; imax(0, pri_strength - (abs(diff) >> shift))) + pminub m2, m4 + vpdpbusd m0, m2, m10 ; sum +%endmacro + CDEF_FILTER_4x4_PRI + test t1d, t1d ; sec + jz .end_no_clip + call .sec +.end_clip: + pminub m4, m6, m1 + pmaxub m1, m6 + pminub m5, m2, m3 + pmaxub m2, m3 + pminub m4, m5 + pmaxub m2, m1 + psrldq m1, m4, 2 + psrldq m3, m2, 2 + pminub m1, m4 + vpcmpw k1, m0, m7, 1 + vpshldd m6, m0, 8 + pmaxub m2, m3 + pslldq m3, m1, 1 + psubw m7, m0 + paddusw m0, m6 ; clip >0xff + vpsubusw m0{k1}, m6, m7 ; clip <0x00 + pslldq m4, m2, 1 + pminub m1, m3 + pmaxub m2, m4 + pmaxub m0, m1 + pminub m0, m2 + jmp .end +.sec_only: + movifnidn t1d, secm + call .sec +.end_no_clip: + vpshldd m6, m0, 8 ; (px << 8) + ((sum > -8) << 4) + paddw m0, m6 ; (px << 8) + ((sum + (sum > -8) + 7) << 4) +.end: + mova xm1, [base+end_perm] + vpermb m0, m1, m0 ; output in bits 8-15 of each dword + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + pextrd [dstq+strideq*2], xm0, 2 + pextrd [dstq+r2 ], xm0, 3 + RET +.mask_edges_sec_only: + movifnidn t1d, secm + call .mask_edges_sec + jmp .end_no_clip +ALIGN function_align +.mask_edges: + vpbroadcastq m8, [base+edge_mask+r6*8] + test prid, prid + jz .mask_edges_sec_only + vpaddd m2, m3, [base+cdef_dirs+(t0+2)*4] {1to16} + vpshufbitqmb k1, m8, m2 ; index in-range + mova m1, m6 + vpermb m1{k1}, m2, m5 + CDEF_FILTER_4x4_PRI + test t1d, t1d + jz .end_no_clip + call .mask_edges_sec + jmp .end_clip +.mask_edges_sec: + vpaddd m4, m3, [base+cdef_dirs+(t0+4)*4] {1to16} + vpaddd m9, m3, [base+cdef_dirs+(t0+0)*4] {1to16} + vpshufbitqmb k1, m8, m4 + mova m2, m6 + vpermb m2{k1}, m4, m5 + vpshufbitqmb k1, m8, m9 + mova m3, m6 + vpermb m3{k1}, m9, m5 + jmp .sec_main +ALIGN function_align +.sec: + vpaddd m2, m3, [base+cdef_dirs+(t0+4)*4] {1to16} ; dir + 2 + vpaddd m3, [base+cdef_dirs+(t0+0)*4] {1to16} ; dir - 2 + vpermb m2, m2, m5 ; k0s0 k0s1 k1s0 k1s1 + vpermb m3, m3, m5 ; k0s2 k0s3 k1s2 k1s3 +.sec_main: + vpbroadcastd m8, [base+sec_tap] + vpcmpub k1, m6, m2, 6 + psubb m4, m2, m6 + vpbroadcastb m12, t1d + lzcnt t1d, t1d + vpsubb m4{k1}, m6, m2 + vpcmpub k2, m6, m3, 6 + vpbroadcastq m11, [r3+t1*8] + gf2p8affineqb m10, m4, m11, 0 + psubb m5, m3, m6 + mova m9, m8 + vpsubb m8{k1}, m7, m8 + psubusb m10, m12, m10 + vpsubb m5{k2}, m6, m3 + pminub m4, m10 + vpdpbusd m0, m4, m8 + gf2p8affineqb m11, m5, m11, 0 + vpsubb m9{k2}, m7, m9 + psubusb m12, m11 + pminub m5, m12 + vpdpbusd m0, m5, m9 + ret + +DECLARE_REG_TMP 2, 7 + +; lut top lut bottom +; t0 t1 t2 t3 t4 t5 t6 t7 L4 L5 20 21 22 23 24 25 +; T0 T1 T2 T3 T4 T5 T6 T7 L6 L7 30 31 32 33 34 35 +; L0 L1 00 01 02 03 04 05 L8 L9 40 41 42 43 44 45 +; L2 L3 10 11 12 13 14 15 La Lb 50 51 52 53 54 55 +; L4 L5 20 21 22 23 24 25 Lc Ld 60 61 62 63 64 65 +; L6 L7 30 31 32 33 34 35 Le Lf 70 71 72 73 74 75 +; L8 L9 40 41 42 43 44 45 8e 8f 80 81 82 83 84 85 +; La Lb 50 51 52 53 54 55 9e 9f 90 91 92 93 94 95 + +cglobal cdef_filter_4x8, 4, 9, 22, dst, stride, left, top, \ + pri, sec, dir, damping, edge +%define base r8-edge_mask + vpbroadcastd ym21, strided + mov r6d, edgem + lea r8, [edge_mask] + movq xm1, [topq+strideq*0-2] + pmulld ym21, [base+pd_01234567] + kxnorb k1, k1, k1 + movq xm2, [topq+strideq*1-2] + vpgatherdq m0{k1}, [dstq+ym21] ; +0+1 +2+3 +4+5 +6+7 + mova m14, [base+lut_perm_4x8a] + movu m15, [base+lut_perm_4x8b] + test r6b, 0x08 ; avoid buffer overread + jz .main + lea r7, [dstq+strideq*8-2] + vinserti32x4 ym1, [r7+strideq*0], 1 + vinserti32x4 ym2, [r7+strideq*1], 1 +.main: + punpcklqdq ym1, ym2 + vinserti32x4 m1, [leftq], 2 ; -2-1 +8+9 left ____ + movifnidn prid, prim + mov t0d, dirm + mova m16, [base+px_idx] + mov r3d, dampingm + vpermi2b m14, m0, m1 ; lut top + vpermi2b m15, m0, m1 ; lut bottom + vpbroadcastd m0, [base+pd_268435568] ; (1 << 28) + (7 << 4) + pxor m20, m20 + lea r3, [r8+r3*8] ; gf_shr + (damping - 30) * 8 + vpermb m2, m16, m14 ; pxt + vpermb m3, m16, m15 ; pxb + mova m1, m0 + cmp r6b, 0x0f + jne .mask_edges ; mask edges only if required + test prid, prid + jz .sec_only + vpaddd m6, m16, [base+cdef_dirs+(t0+2)*4] {1to16} ; dir + vpermb m4, m6, m14 ; pNt k0p0 k0p1 k1p0 k1p1 + vpermb m5, m6, m15 ; pNb +%macro CDEF_FILTER_4x8_PRI 0 + vpcmpub k1, m2, m4, 6 ; pxt > pNt + vpcmpub k2, m3, m5, 6 ; pxb > pNb + psubb m6, m4, m2 + psubb m7, m5, m3 + lzcnt r6d, prid + vpsubb m6{k1}, m2, m4 ; abs(diff_top) + vpsubb m7{k2}, m3, m5 ; abs(diff_bottom) + vpbroadcastb m13, prid + vpbroadcastq m9, [r3+r6*8] + and prid, 1 + vpbroadcastd m11, [base+pri_tap+priq*4] + vgf2p8affineqb m8, m6, m9, 0 ; abs(dt) >> shift + vgf2p8affineqb m9, m7, m9, 0 ; abs(db) >> shift + mova m10, m11 + movifnidn t1d, secm + vpsubb m10{k1}, m20, m11 ; apply_sign(pri_tap_top) + vpsubb m11{k2}, m20, m11 ; apply_sign(pri_tap_bottom) + psubusb m12, m13, m8 ; imax(0, pri_strength - (abs(dt) >> shift))) + psubusb m13, m13, m9 ; imax(0, pri_strength - (abs(db) >> shift))) + pminub m6, m12 + pminub m7, m13 + vpdpbusd m0, m6, m10 ; sum top + vpdpbusd m1, m7, m11 ; sum bottom +%endmacro + CDEF_FILTER_4x8_PRI + test t1d, t1d ; sec + jz .end_no_clip + call .sec +.end_clip: + pminub m10, m4, m2 + pminub m12, m6, m8 + pminub m11, m5, m3 + pminub m13, m7, m9 + pmaxub m4, m2 + pmaxub m6, m8 + pmaxub m5, m3 + pmaxub m7, m9 + pminub m10, m12 + pminub m11, m13 + pmaxub m4, m6 + pmaxub m5, m7 + mov r2d, 0xAAAAAAAA + kmovd k1, r2d + kxnorb k2, k2, k2 ; hw lw + vpshrdd m12, m0, m1, 16 ; m1lw m0hw + vpshrdd m6, m10, m11, 16 ; m11lw m10hw + vpshrdd m8, m4, m5, 16 ; m5lw m4hw + vpblendmw m7{k1}, m10, m11 ; m11hw m10lw + vpblendmw m9{k1}, m4, m5 ; m5hw m4lw + vpblendmw m4{k1}, m0, m12 ; m1lw m0lw + vpblendmw m5{k1}, m12, m1 ; m1hw m0hw + vpshrdd m2, m3, 16 + pminub m6, m7 + pmaxub m8, m9 + mova ym14, [base+end_perm] + vpcmpw k1, m4, m20, 1 + vpshldw m2, m5, 8 + pslldq m7, m6, 1 + pslldq m9, m8, 1 + psubw m5, m20, m4 + paddusw m0, m4, m2 ; clip >0xff + pminub m6, m7 + pmaxub m8, m9 + psubusw m0{k1}, m2, m5 ; clip <0x00 + pmaxub m0, m6 + pminub m0, m8 + vpermb m0, m14, m0 + vpscatterdd [dstq+ym21]{k2}, ym0 + RET +.sec_only: + movifnidn t1d, secm + call .sec +.end_no_clip: + mova ym4, [base+end_perm] + kxnorb k1, k1, k1 + vpshldd m2, m0, 8 ; (px << 8) + ((sum > -8) << 4) + vpshldd m3, m1, 8 + paddw m0, m2 ; (px << 8) + ((sum + (sum > -8) + 7) << 4) + paddw m1, m3 + pslld m0, 16 + vpshrdd m0, m1, 16 + vpermb m0, m4, m0 ; output in bits 8-15 of each word + vpscatterdd [dstq+ym21]{k1}, ym0 + RET +.mask_edges_sec_only: + movifnidn t1d, secm + call .mask_edges_sec + jmp .end_no_clip +ALIGN function_align +.mask_edges: + mov t1d, r6d + or r6d, 8 ; top 4x4 has bottom + or t1d, 4 ; bottom 4x4 has top + vpbroadcastq m17, [base+edge_mask+r6*8] + vpbroadcastq m18, [base+edge_mask+t1*8] + test prid, prid + jz .mask_edges_sec_only + vpaddd m6, m16, [base+cdef_dirs+(t0+2)*4] {1to16} + vpshufbitqmb k1, m17, m6 ; index in-range + vpshufbitqmb k2, m18, m6 + mova m4, m2 + mova m5, m3 + vpermb m4{k1}, m6, m14 + vpermb m5{k2}, m6, m15 + CDEF_FILTER_4x8_PRI + test t1d, t1d + jz .end_no_clip + call .mask_edges_sec + jmp .end_clip +.mask_edges_sec: + vpaddd m10, m16, [base+cdef_dirs+(t0+4)*4] {1to16} + vpaddd m11, m16, [base+cdef_dirs+(t0+0)*4] {1to16} + vpshufbitqmb k1, m17, m10 + vpshufbitqmb k2, m18, m10 + vpshufbitqmb k3, m17, m11 + vpshufbitqmb k4, m18, m11 + mova m6, m2 + mova m7, m3 + mova m8, m2 + mova m9, m3 + vpermb m6{k1}, m10, m14 + vpermb m7{k2}, m10, m15 + vpermb m8{k3}, m11, m14 + vpermb m9{k4}, m11, m15 + jmp .sec_main +ALIGN function_align +.sec: + vpaddd m8, m16, [base+cdef_dirs+(t0+4)*4] {1to16} ; dir + 2 + vpaddd m9, m16, [base+cdef_dirs+(t0+0)*4] {1to16} ; dir - 2 + vpermb m6, m8, m14 ; pNt k0s0 k0s1 k1s0 k1s1 + vpermb m7, m8, m15 ; pNb + vpermb m8, m9, m14 ; pNt k0s2 k0s3 k1s2 k1s3 + vpermb m9, m9, m15 ; pNb +.sec_main: + vpbroadcastb m18, t1d + lzcnt t1d, t1d + vpcmpub k1, m2, m6, 6 + vpcmpub k2, m3, m7, 6 + vpcmpub k3, m2, m8, 6 + vpcmpub k4, m3, m9, 6 + vpbroadcastq m17, [r3+t1*8] + psubb m10, m6, m2 + psubb m11, m7, m3 + psubb m12, m8, m2 + psubb m13, m9, m3 + vpsubb m10{k1}, m2, m6 ; abs(dt0) + vpsubb m11{k2}, m3, m7 ; abs(db0) + vpsubb m12{k3}, m2, m8 ; abs(dt1) + vpsubb m13{k4}, m3, m9 ; abs(db1) + vpbroadcastd m19, [base+sec_tap] + gf2p8affineqb m14, m10, m17, 0 ; abs(dt0) >> shift + gf2p8affineqb m15, m11, m17, 0 ; abs(db0) >> shift + gf2p8affineqb m16, m12, m17, 0 ; abs(dt1) >> shift + gf2p8affineqb m17, m13, m17, 0 ; abs(db1) >> shift + psubusb m14, m18, m14 ; imax(0, sec_strength - (abs(dt0) >> shift))) + psubusb m15, m18, m15 ; imax(0, sec_strength - (abs(db0) >> shift))) + psubusb m16, m18, m16 ; imax(0, sec_strength - (abs(dt1) >> shift))) + psubusb m17, m18, m17 ; imax(0, sec_strength - (abs(db1) >> shift))) + pminub m10, m14 + pminub m11, m15 + pminub m12, m16 + pminub m13, m17 + mova m14, m19 + mova m15, m19 + mova m16, m19 + vpsubb m14{k1}, m20, m19 ; apply_sign(sec_tap_top_0) + vpsubb m15{k2}, m20, m19 ; apply_sign(sec_tap_bottom_0) + vpsubb m16{k3}, m20, m19 ; apply_sign(sec_tap_top_1) + vpsubb m19{k4}, m20, m19 ; apply_sign(sec_tap_bottom_1) + vpdpbusd m0, m10, m14 + vpdpbusd m1, m11, m15 + vpdpbusd m0, m12, m16 + vpdpbusd m1, m13, m19 + ret + +; lut tl lut tr +; t0 t1 t2 t3 t4 t5 t6 t7 t6 t7 t8 t9 ta tb tc td +; T0 T1 T2 T3 T4 T5 T6 T7 T6 T7 T8 T9 TA TB TC TD +; L0 L1 00 01 02 03 04 05 04 05 06 07 08 09 0a 0b +; L2 L3 10 11 12 13 14 15 14 15 16 17 18 19 1a 1b +; L4 L5 20 21 22 23 24 25 24 25 26 27 28 29 2a 2b +; L6 L7 30 31 32 33 34 35 34 35 36 37 38 39 3a 3b +; L8 L9 40 41 42 43 44 45 44 45 46 47 48 49 4a 4b +; La Lb 50 51 52 53 54 55 54 55 56 57 58 59 5a 5b +; lut bl lut br +; L4 L5 20 21 22 23 24 25 24 25 26 27 28 29 2a 2b +; L6 L7 30 31 32 33 34 35 34 35 36 37 38 39 3a 3b +; L8 L9 40 41 42 43 44 45 44 45 46 47 48 49 4a 4b +; La Lb 50 51 52 53 54 55 54 55 56 57 58 59 5a 5b +; Lc Ld 60 61 62 63 64 65 64 65 66 67 68 69 6a 6b +; Le Lf 70 71 72 73 74 75 74 75 76 77 78 79 7a 7b +; 8e 8f 80 81 82 83 84 85 84 85 86 87 88 89 8a 8b +; 9e 9f 90 91 92 93 94 95 94 95 96 97 98 99 9a 9b + +cglobal cdef_filter_8x8, 4, 11, 32, 4*64, dst, stride, left, top, \ + pri, sec, dir, damping, edge +%define base r8-edge_mask + mov r6d, edgem + lea r10, [dstq+strideq*4-2] + movu xmm0, [topq+strideq*0-2] + movu xmm1, [dstq+strideq*2-2] + movu xmm2, [r10 +strideq*2 ] + lea r8, [edge_mask] + lea r9, [strideq*3] + pmovzxwq m10, [leftq-4] + vinserti32x4 ym0, ymm0, [topq+strideq*1-2], 1 + vinserti32x4 ym1, ymm1, [dstq+r9 -2], 1 + vinserti32x4 ym2, ymm2, [r10 +r9 ], 1 + lea r7, [r10 +strideq*4 ] + pmovzxwq m11, [leftq+4] + vinserti32x4 m0, [dstq+strideq*0-2], 2 + vinserti32x4 m1, [r10 +strideq*0 ], 2 + mova m12, [base+lut_perm_8x8a] + movu m13, [base+lut_perm_8x8b] + vinserti32x4 m0, [dstq+strideq*1-2], 3 + vinserti32x4 m1, [r10 +strideq*1 ], 3 + test r6b, 0x08 ; avoid buffer overread + jz .main + vinserti32x4 m2, [r7 +strideq*0], 2 + vinserti32x4 m2, [r7 +strideq*1], 3 +.main: + mov t1d, 0x11111100 + mova m14, m12 + mova m15, m13 + kmovd k1, t1d + kshiftrd k2, k1, 8 + movifnidn prid, prim + mov t0d, dirm + mova m30, [base+px_idx] + mov r3d, dampingm + vpermi2b m12, m0, m1 ; lut tl + vpermi2b m14, m1, m2 ; lut bl + vpermi2b m13, m0, m1 ; lut tr + vpermi2b m15, m1, m2 ; lut br + vpblendmw m12{k1}, m12, m10 + vpblendmw m14{k2}, m14, m11 + vpbroadcastd m0, [base+pd_268435568] ; (1 << 28) + (7 << 4) + pxor m31, m31 + lea r3, [r8+r3*8] ; gf_shr + (damping - 30) * 8 + vpermb m4, m30, m12 ; pxtl + vpermb m5, m30, m13 ; pxtr + vpermb m6, m30, m14 ; pxbl + vpermb m7, m30, m15 ; pxbr + mova m1, m0 + mova m2, m0 + mova m3, m0 + cmp r6b, 0x0f + jne .mask_edges ; mask edges only if required + test prid, prid + jz .sec_only + vpaddd m11, m30, [base+cdef_dirs+(t0+2)*4] {1to16} ; dir + vpermb m8, m11, m12 ; pNtl k0p0 k0p1 k1p0 k1p1 + vpermb m9, m11, m13 ; pNtr + vpermb m10, m11, m14 ; pNbl + vpermb m11, m11, m15 ; pNbr +%macro CDEF_FILTER_8x8_PRI 0 + vpcmpub k1, m4, m8, 6 ; pxtl > pNtl + vpcmpub k2, m5, m9, 6 ; pxtr > pNtr + vpcmpub k3, m6, m10, 6 ; pxbl > pNbl + vpcmpub k4, m7, m11, 6 ; pxbr > pNbr + psubb m16, m8, m4 + psubb m17, m9, m5 + psubb m18, m10, m6 + psubb m19, m11, m7 + lzcnt r6d, prid + vpsubb m16{k1}, m4, m8 ; abs(diff_tl) + vpsubb m17{k2}, m5, m9 ; abs(diff_tr) + vpsubb m18{k3}, m6, m10 ; abs(diff_bl) + vpsubb m19{k4}, m7, m11 ; abs(diff_br) + vpbroadcastq m28, [r3+r6*8] + vpbroadcastb m29, prid + and prid, 1 + vpbroadcastd m27, [base+pri_tap+priq*4] + vgf2p8affineqb m20, m16, m28, 0 ; abs(dtl) >> shift + vgf2p8affineqb m21, m17, m28, 0 ; abs(dtr) >> shift + vgf2p8affineqb m22, m18, m28, 0 ; abs(dbl) >> shift + vgf2p8affineqb m23, m19, m28, 0 ; abs(dbl) >> shift + mova m24, m27 + mova m25, m27 + mova m26, m27 + movifnidn t1d, secm + vpsubb m24{k1}, m31, m27 ; apply_sign(pri_tap_tl) + vpsubb m25{k2}, m31, m27 ; apply_sign(pri_tap_tr) + vpsubb m26{k3}, m31, m27 ; apply_sign(pri_tap_tl) + vpsubb m27{k4}, m31, m27 ; apply_sign(pri_tap_tr) + psubusb m20, m29, m20 ; imax(0, pri_strength - (abs(dtl) >> shift))) + psubusb m21, m29, m21 ; imax(0, pri_strength - (abs(dtr) >> shift))) + psubusb m22, m29, m22 ; imax(0, pri_strength - (abs(dbl) >> shift))) + psubusb m23, m29, m23 ; imax(0, pri_strength - (abs(dbr) >> shift))) + pminub m16, m20 + pminub m17, m21 + pminub m18, m22 + pminub m19, m23 + vpdpbusd m0, m16, m24 ; sum tl + vpdpbusd m1, m17, m25 ; sum tr + vpdpbusd m2, m18, m26 ; sum bl + vpdpbusd m3, m19, m27 ; sum br +%endmacro + CDEF_FILTER_8x8_PRI + test t1d, t1d ; sec + jz .end_no_clip + call .sec +.end_clip: + pminub m20, m8, m4 + pminub m24, m12, m16 + pminub m21, m9, m5 + pminub m25, m13, m17 + pminub m22, m10, m6 + pminub m26, m14, m18 + pminub m23, m11, m7 + pminub m27, m15, m19 + pmaxub m8, m4 + pmaxub m12, m16 + pmaxub m9, m5 + pmaxub m13, m17 + pmaxub m10, m6 + pmaxub m14, m18 + pmaxub m11, m7 + pmaxub m15, m19 + pminub m20, m24 + pminub m21, m25 + pminub m22, m26 + pminub m23, m27 + pmaxub m8, m12 + pmaxub m9, m13 + pmaxub m10, m14 + pmaxub m11, m15 + mov r2d, 0xAAAAAAAA + kmovd k1, r2d + vpshrdd m24, m0, m1, 16 + vpshrdd m25, m2, m3, 16 + vpshrdd m12, m20, m21, 16 + vpshrdd m14, m22, m23, 16 + vpshrdd m16, m8, m9, 16 + vpshrdd m18, m10, m11, 16 + vpblendmw m13{k1}, m20, m21 + vpblendmw m15{k1}, m22, m23 + vpblendmw m17{k1}, m8, m9 + vpblendmw m19{k1}, m10, m11 + vpblendmw m20{k1}, m0, m24 + vpblendmw m21{k1}, m24, m1 + vpblendmw m22{k1}, m2, m25 + vpblendmw m23{k1}, m25, m3 + vpshrdd m4, m5, 16 + vpshrdd m6, m7, 16 + pminub m12, m13 + pminub m14, m15 + pmaxub m16, m17 + pmaxub m18, m19 + mova m8, [base+end_perm_w8clip] + vpcmpw k2, m20, m31, 1 + vpcmpw k3, m22, m31, 1 + vpshldw m4, m21, 8 + vpshldw m6, m23, 8 + kunpckdq k1, k1, k1 + kxnorb k4, k4, k4 + vpshrdw m11, m12, m14, 8 + vpshrdw m15, m16, m18, 8 + vpblendmb m13{k1}, m12, m14 + vpblendmb m17{k1}, m16, m18 + psubw m21, m31, m20 + psubw m23, m31, m22 + paddusw m0, m20, m4 ; clip >0xff + paddusw m1, m22, m6 + pminub m11, m13 + pmaxub m15, m17 + psubusw m0{k2}, m4, m21 ; clip <0x00 + psubusw m1{k3}, m6, m23 + psrlw m0, 8 + vmovdqu8 m0{k1}, m1 + pmaxub m0, m11 + pminub m0, m15 + vpermb m0, m8, m0 + add r10, 2 + vextracti32x4 xm1, m0, 1 + vextracti32x4 xm2, m0, 2 + vextracti32x4 xm3, m0, 3 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*2], xm1 + movq [r10 +strideq*0], xm2 + movq [r10 +strideq*2], xm3 + movhps [dstq+strideq*1], xm0 + movhps [dstq+r9 ], xm1 + movhps [r10 +strideq*1], xm2 + movhps [r10 +r9 ], xm3 + RET +.sec_only: + movifnidn t1d, secm + call .sec +.end_no_clip: + mova xm8, [base+end_perm] + kxnorb k1, k1, k1 + vpshldd m4, m0, 8 ; (px << 8) + ((sum > -8) << 4) + vpshldd m5, m1, 8 + vpshldd m6, m2, 8 + vpshldd m7, m3, 8 + paddw m0, m4 ; (px << 8) + ((sum + (sum > -8) + 7) << 4) + paddw m1, m5 + paddw m2, m6 + paddw m3, m7 + vpermb m0, m8, m0 + vpermb m1, m8, m1 + vpermb m2, m8, m2 + vpermb m3, m8, m3 + add r10, 2 + punpckldq m4, m0, m1 + punpckhdq m0, m1 + punpckldq m5, m2, m3 + punpckhdq m2, m3 + movq [dstq+strideq*0], xm4 + movq [dstq+strideq*2], xm0 + movq [r10 +strideq*0], xm5 + movq [r10 +strideq*2], xm2 + movhps [dstq+strideq*1], xm4 + movhps [dstq+r9 ], xm0 + movhps [r10 +strideq*1], xm5 + movhps [r10 +r9 ], xm2 + RET +.mask_edges_sec_only: + movifnidn t1d, secm + call .mask_edges_sec + jmp .end_no_clip +ALIGN function_align +.mask_edges: + mov t0d, r6d + mov t1d, r6d + or t0d, 0xA ; top-left 4x4 has bottom and right + or t1d, 0x9 ; top-right 4x4 has bottom and left + vpbroadcastq m26, [base+edge_mask+t0*8] + vpbroadcastq m27, [base+edge_mask+t1*8] + mov t1d, r6d + or r6d, 0x6 ; bottom-left 4x4 has top and right + or t1d, 0x5 ; bottom-right 4x4 has top and left + vpbroadcastq m28, [base+edge_mask+r6*8] + vpbroadcastq m29, [base+edge_mask+t1*8] + mov t0d, dirm + test prid, prid + jz .mask_edges_sec_only + vpaddd m20, m30, [base+cdef_dirs+(t0+2)*4] {1to16} + vpshufbitqmb k1, m26, m20 ; index in-range + vpshufbitqmb k2, m27, m20 + vpshufbitqmb k3, m28, m20 + vpshufbitqmb k4, m29, m20 + mova m8, m4 + mova m9, m5 + mova m10, m6 + mova m11, m7 + vpermb m8{k1}, m20, m12 + vpermb m9{k2}, m20, m13 + vpermb m10{k3}, m20, m14 + vpermb m11{k4}, m20, m15 + mova [rsp+0x00], m26 + mova [rsp+0x40], m27 + mova [rsp+0x80], m28 + mova [rsp+0xC0], m29 + CDEF_FILTER_8x8_PRI + test t1d, t1d + jz .end_no_clip + mova m26, [rsp+0x00] + mova m27, [rsp+0x40] + mova m28, [rsp+0x80] + mova m29, [rsp+0xC0] + call .mask_edges_sec + jmp .end_clip +.mask_edges_sec: + vpaddd m20, m30, [base+cdef_dirs+(t0+4)*4] {1to16} + vpaddd m21, m30, [base+cdef_dirs+(t0+0)*4] {1to16} + vpshufbitqmb k1, m26, m20 + vpshufbitqmb k2, m27, m20 + vpshufbitqmb k3, m28, m20 + vpshufbitqmb k4, m29, m20 + mova m16, m4 + mova m17, m5 + mova m18, m6 + mova m19, m7 + vpermb m16{k1}, m20, m12 + vpermb m17{k2}, m20, m13 + vpermb m18{k3}, m20, m14 + vpermb m19{k4}, m20, m15 + vpshufbitqmb k1, m26, m21 + vpshufbitqmb k2, m27, m21 + vpshufbitqmb k3, m28, m21 + vpshufbitqmb k4, m29, m21 + vpermb m12, m21, m12 + vpermb m13, m21, m13 + vpermb m14, m21, m14 + vpermb m15, m21, m15 + vpblendmb m12{k1}, m4, m12 + vpblendmb m13{k2}, m5, m13 + vpblendmb m14{k3}, m6, m14 + vpblendmb m15{k4}, m7, m15 + jmp .sec_main +ALIGN function_align +.sec: + vpaddd m20, m30, [base+cdef_dirs+(t0+4)*4] {1to16} ; dir + 2 + vpaddd m21, m30, [base+cdef_dirs+(t0+0)*4] {1to16} ; dir - 2 + vpermb m16, m20, m12 ; pNtl k0s0 k0s1 k1s0 k1s1 + vpermb m17, m20, m13 ; pNtr + vpermb m18, m20, m14 ; pNbl + vpermb m19, m20, m15 ; pNbr + vpermb m12, m21, m12 ; pNtl k0s2 k0s3 k1s2 k1s3 + vpermb m13, m21, m13 ; pNtr + vpermb m14, m21, m14 ; pNbl + vpermb m15, m21, m15 ; pNbr +.sec_main: +%macro CDEF_FILTER_8x8_SEC 4-5 0 ; load constants + vpcmpub k1, m4, %1, 6 + vpcmpub k2, m5, %2, 6 + vpcmpub k3, m6, %3, 6 + vpcmpub k4, m7, %4, 6 + psubb m20, %1, m4 + psubb m21, %2, m5 + psubb m22, %3, m6 + psubb m23, %4, m7 +%if %5 + vpbroadcastb m28, t1d + lzcnt t1d, t1d + vpbroadcastq m29, [r3+t1*8] +%endif + vpsubb m20{k1}, m4, %1 + vpsubb m21{k2}, m5, %2 + vpsubb m22{k3}, m6, %3 + vpsubb m23{k4}, m7, %4 + gf2p8affineqb m24, m20, m29, 0 + gf2p8affineqb m25, m21, m29, 0 + gf2p8affineqb m26, m22, m29, 0 + gf2p8affineqb m27, m23, m29, 0 +%if %5 + vpbroadcastd m30, [base+sec_tap] +%endif + psubusb m24, m28, m24 + psubusb m25, m28, m25 + psubusb m26, m28, m26 + psubusb m27, m28, m27 + pminub m20, m24 + pminub m21, m25 + pminub m22, m26 + pminub m23, m27 + mova m24, m30 + mova m25, m30 + mova m26, m30 + mova m27, m30 + vpsubb m24{k1}, m31, m30 + vpsubb m25{k2}, m31, m30 + vpsubb m26{k3}, m31, m30 + vpsubb m27{k4}, m31, m30 + vpdpbusd m0, m20, m24 + vpdpbusd m1, m21, m25 + vpdpbusd m2, m22, m26 + vpdpbusd m3, m23, m27 +%endmacro + CDEF_FILTER_8x8_SEC m16, m17, m18, m19, 1 + CDEF_FILTER_8x8_SEC m12, m13, m14, m15 + ret + +%endif ; HAVE_AVX512ICL && ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/cdef_init_tmpl.c b/third_party/dav1d/src/x86/cdef_init_tmpl.c --- a/third_party/dav1d/src/x86/cdef_init_tmpl.c +++ b/third_party/dav1d/src/x86/cdef_init_tmpl.c @@ -82,11 +82,13 @@ c->fb[2] = dav1d_cdef_filter_4x4_avx2; #endif if (!(flags & DAV1D_X86_CPU_FLAG_AVX512ICL)) return; -#if BITDEPTH == 8 +#if HAVE_AVX512ICL && BITDEPTH == 8 + c->fb[0] = dav1d_cdef_filter_8x8_avx512icl; + c->fb[1] = dav1d_cdef_filter_4x8_avx512icl; c->fb[2] = dav1d_cdef_filter_4x4_avx512icl; #endif #endif } diff --git a/third_party/dav1d/src/x86/film_grain.asm b/third_party/dav1d/src/x86/film_grain.asm --- a/third_party/dav1d/src/x86/film_grain.asm +++ b/third_party/dav1d/src/x86/film_grain.asm @@ -26,10 +26,12 @@ %include "ext/x86/x86inc.asm" %if ARCH_X86_64 SECTION_RODATA 32 +pb_8x_27_17_8x_17_27: times 8 db 27, 17 + times 8 db 17, 27 pw_1024: times 16 dw 1024 pb_mask: db 0, 0x80, 0x80, 0, 0x80, 0, 0, 0x80, 0x80, 0, 0, 0x80, 0, 0x80, 0x80, 0 rnd_next_upperbit_mask: dw 0x100B, 0x2016, 0x402C, 0x8058 byte_blend: db 0, 0, 0, 0xff, 0, 0, 0, 0 pw_seed_xor: times 2 dw 0xb524 @@ -58,10 +60,12 @@ %endmacro ALIGN 4 JMP_TABLE generate_grain_y_avx2, 0, 1, 2, 3 JMP_TABLE generate_grain_uv_420_avx2, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_422_avx2, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_444_avx2, 0, 1, 2, 3 struc FGData .seed: resd 1 .num_y_points: resd 1 .y_points: resb 14 * 2 @@ -411,12 +415,13 @@ add bufq, 82 dec hd jg .y_loop_ar3 RET +%macro generate_grain_uv_fn 3 ; ss_name, ss_x, ss_y INIT_XMM avx2 -cglobal generate_grain_uv_420, 4, 10, 16, buf, bufy, fg_data, uv +cglobal generate_grain_uv_%1, 4, 10, 16, buf, bufy, fg_data, uv lea r4, [pb_mask] %define base r4-pb_mask movq xm1, [base+rnd_next_upperbit_mask] movq xm4, [base+mul_bits] movq xm7, [base+hmul_bits] @@ -426,15 +431,21 @@ vpbroadcastw xm0, [fg_dataq+FGData.seed] vpbroadcastw xm9, [base+pw_seed_xor+uvq*4] pxor xm0, xm9 vpbroadcastd xm9, [base+pd_m65536] lea r6, [gaussian_sequence] - mov r7d, 38 +%if %2 + mov r7d, 73-35*%3 add bufq, 44 .loop_y: mov r5, -44 .loop_x: +%else + mov r5, -73*82 + sub bufq, r5 +.loop: +%endif pand xm2, xm0, xm1 psrlw xm3, xm2, 10 por xm2, xm3 ; bits 0xf, 0x1e, 0x3c and 0x78 are set pmullw xm2, xm4 ; bits 0x0f00 are set pshufb xm2, xm5, xm2 ; set 15th bit for next 4 seeds @@ -453,19 +464,23 @@ packusdw xm2, xm2 pmulhrsw xm2, xm8 packsswb xm2, xm2 movd [bufq+r5], xm2 add r5, 4 +%if %2 jl .loop_x add bufq, 82 dec r7d jg .loop_y +%else + jl .loop +%endif ; auto-regression code movsxd r5, [fg_dataq+FGData.ar_coeff_lag] - movsxd r5, [base+generate_grain_uv_420_avx2_table+r5*4] - lea r5, [r5+base+generate_grain_uv_420_avx2_table] + movsxd r5, [base+generate_grain_uv_%1_avx2_table+r5*4] + lea r5, [r5+base+generate_grain_uv_%1_avx2_table] jmp r5 .ar0: INIT_YMM avx2 DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift @@ -473,67 +488,130 @@ mov shiftd, [fg_dataq+FGData.ar_coeff_shift] movd xm4, [fg_dataq+FGData.ar_coeffs_uv+uvq] movd xm3, [base+hmul_bits+shiftq*2] DEFINE_ARGS buf, bufy, h pmovsxbw xm4, xm4 +%if %2 vpbroadcastd m7, [pb_1] - vpbroadcastw m6, [hmul_bits+4] + vpbroadcastw m6, [hmul_bits+2+%3*2] +%endif vpbroadcastw m4, xm4 vpbroadcastw m3, xm3 - sub bufq, 82*38+82-(82*3+41) + pxor m12, m12 +%if %2 + sub bufq, 82*(73-35*%3)+82-(82*3+41) +%else + sub bufq, 82*70-3 +%endif add bufyq, 3+82*3 - mov hd, 35 + mov hd, 70-35*%3 .y_loop_ar0: +%if %2 ; first 32 pixels movu xm8, [bufyq] +%if %3 movu xm9, [bufyq+82] +%endif movu xm10, [bufyq+16] +%if %3 movu xm11, [bufyq+82+16] +%endif vinserti128 m8, [bufyq+32], 1 +%if %3 vinserti128 m9, [bufyq+82+32], 1 +%endif vinserti128 m10, [bufyq+48], 1 +%if %3 vinserti128 m11, [bufyq+82+48], 1 +%endif pmaddubsw m8, m7, m8 +%if %3 pmaddubsw m9, m7, m9 +%endif pmaddubsw m10, m7, m10 +%if %3 pmaddubsw m11, m7, m11 paddw m8, m9 paddw m10, m11 +%endif pmulhrsw m8, m6 pmulhrsw m10, m6 +%else + xor r3d, r3d + ; first 32x2 pixels +.x_loop_ar0: + movu m8, [bufyq+r3] + pcmpgtb m9, m12, m8 + punpckhbw m10, m8, m9 + punpcklbw m8, m9 +%endif pmullw m8, m4 pmullw m10, m4 pmulhrsw m8, m3 pmulhrsw m10, m3 - packsswb m8, m10 +%if %2 movu m0, [bufq] - punpckhbw m1, m0, m8 - punpcklbw m0, m8 - pmaddubsw m1, m7, m1 - pmaddubsw m0, m7, m0 - packsswb m0, m1 +%else + movu m0, [bufq+r3] +%endif + pcmpgtb m1, m12, m0 + punpckhbw m9, m0, m1 + punpcklbw m0, m1 + paddw m0, m8 + paddw m9, m10 + packsswb m0, m9 +%if %2 movu [bufq], m0 +%else + movu [bufq+r3], m0 + add r3d, 32 + cmp r3d, 64 + jl .x_loop_ar0 +%endif - ; last 6 pixels + ; last 6/12 pixels movu xm8, [bufyq+32*2] +%if %2 +%if %3 movu xm9, [bufyq+32*2+82] +%endif pmaddubsw xm8, xm7, xm8 +%if %3 pmaddubsw xm9, xm7, xm9 paddw xm8, xm9 +%endif pmulhrsw xm8, xm6 pmullw xm8, xm4 pmulhrsw xm8, xm3 - packsswb xm8, xm8 movq xm0, [bufq+32] - punpcklbw xm8, xm0 - pmaddubsw xm8, xm7, xm8 + pcmpgtb xm9, xm12, xm0 + punpcklbw xm9, xm0, xm9 + paddw xm8, xm9 packsswb xm8, xm8 vpblendw xm0, xm8, xm0, 1000b movq [bufq+32], xm0 +%else + pcmpgtb xm9, xm12, xm8 + punpckhbw xm10, xm8, xm9 + punpcklbw xm8, xm9 + pmullw xm10, xm4 + pmullw xm8, xm4 + pmulhrsw xm10, xm3 + pmulhrsw xm8, xm3 + movu xm0, [bufq+64] + pcmpgtb xm9, xm12, xm0 + punpcklbw xm1, xm0, xm9 + punpckhbw xm9, xm0, xm9 + paddw xm1, xm8 + paddw xm9, xm10 + packsswb xm1, xm9 + vpblendw xm0, xm1, xm0, 11000000b + movu [bufq+64], xm0 +%endif add bufq, 82 - add bufyq, 82*2 + add bufyq, 82<<%3 dec hd jg .y_loop_ar0 RET .ar1: @@ -547,31 +625,47 @@ DEFINE_ARGS buf, bufy, h, val0, val3, cf3, min, max, x, shift pmovsxbw xm4, xm4 pshufd xm5, xm4, q1111 pshufd xm4, xm4, q0000 pmovsxwd xm3, [base+round_vals+shiftq*2-12] ; rnd +%if %2 vpbroadcastd xm7, [pb_1] - vpbroadcastw xm6, [hmul_bits+4] + vpbroadcastw xm6, [hmul_bits+2+%3*2] +%endif vpbroadcastd xm3, xm3 - sub bufq, 82*38+44-(82*3+41) +%if %2 + sub bufq, 82*(73-35*%3)+44-(82*3+41) +%else + sub bufq, 82*70-(82-3) +%endif add bufyq, 79+82*3 - mov hd, 35 + mov hd, 70-35*%3 mov mind, -128 mov maxd, 127 .y_loop_ar1: - mov xq, -38 + mov xq, -(76>>%2) movsx val3d, byte [bufq+xq-1] .x_loop_ar1: pmovsxbw xm0, [bufq+xq-82-1] ; top/left +%if %2 movq xm8, [bufyq+xq*2] +%if %3 movq xm9, [bufyq+xq*2+82] +%endif +%endif psrldq xm2, xm0, 2 ; top psrldq xm1, xm0, 4 ; top/right +%if %2 pmaddubsw xm8, xm7, xm8 +%if %3 pmaddubsw xm9, xm7, xm9 paddw xm8, xm9 +%endif pmulhrsw xm8, xm6 +%else + pmovsxbw xm8, [bufyq+xq] +%endif punpcklwd xm0, xm2 punpcklwd xm1, xm8 pmaddwd xm0, xm4 pmaddwd xm1, xm5 paddd xm0, xm1 @@ -596,11 +690,11 @@ jnz .x_loop_ar1_inner jmp .x_loop_ar1 .x_loop_ar1_end: add bufq, 82 - add bufyq, 82*2 + add bufyq, 82<<%3 dec hd jg .y_loop_ar1 RET .ar2: @@ -609,25 +703,31 @@ imul uvd, 28 vpbroadcastw xm15, [base+round_vals-12+shiftq*2] pmovsxbw xm8, [fg_dataq+FGData.ar_coeffs_uv+uvq+0] ; cf0-7 pmovsxbw xm9, [fg_dataq+FGData.ar_coeffs_uv+uvq+8] ; cf8-12 pinsrw xm9, [base+pw_1], 5 - vpbroadcastw xm7, [base+hmul_bits+4] +%if %2 + vpbroadcastw xm7, [base+hmul_bits+2+%3*2] vpbroadcastd xm6, [base+pb_1] +%endif DEFINE_ARGS buf, bufy, fg_data, h, unused, x pshufd xm12, xm9, q0000 pshufd xm13, xm9, q1111 pshufd xm14, xm9, q2222 pshufd xm11, xm8, q3333 pshufd xm10, xm8, q2222 pshufd xm9, xm8, q1111 pshufd xm8, xm8, q0000 - sub bufq, 82*38+44-(82*3+41) +%if %2 + sub bufq, 82*(73-35*%3)+44-(82*3+41) +%else + sub bufq, 82*70-(82-3) +%endif add bufyq, 79+82*3 - mov hd, 35 + mov hd, 70-35*%3 .y_loop_ar2: - mov xq, -38 + mov xq, -(76>>%2) .x_loop_ar2: pmovsxbw xm0, [bufq+xq-82*2-2] ; y=-2,x=[-2,+5] pmovsxbw xm1, [bufq+xq-82*1-2] ; y=-1,x=[-2,+5] psrldq xm2, xm0, 2 ; y=-2,x=[-1,+5] @@ -652,16 +752,24 @@ pmaddwd xm3, xm12 paddd xm4, xm0 paddd xm2, xm3 paddd xm2, xm4 +%if %2 movq xm0, [bufyq+xq*2] +%if %3 movq xm3, [bufyq+xq*2+82] +%endif pmaddubsw xm0, xm6, xm0 +%if %3 pmaddubsw xm3, xm6, xm3 paddw xm0, xm3 +%endif pmulhrsw xm0, xm7 +%else + pmovsxbw xm0, [bufyq+xq] +%endif punpcklwd xm0, xm15 pmaddwd xm0, xm14 paddd xm2, xm0 movq xm0, [bufq+xq-2] ; y=0,x=[-2,+5] @@ -683,11 +791,11 @@ jnz .x_loop_ar2_inner jmp .x_loop_ar2 .x_loop_ar2_end: add bufq, 82 - add bufyq, 82*2 + add bufyq, 82<<%3 dec hd jg .y_loop_ar2 RET .ar3: @@ -728,18 +836,24 @@ mova [rsp+ 7*16], xm8 mova [rsp+ 8*16], xm2 mova [rsp+ 9*16], xm3 mova [rsp+10*16], xm4 mova [rsp+11*16], xm5 +%if %2 vpbroadcastd xm13, [base+pb_1] - vpbroadcastw xm15, [base+hmul_bits+4] + vpbroadcastw xm15, [base+hmul_bits+2+%3*2] +%endif DEFINE_ARGS buf, bufy, fg_data, h, unused, x - sub bufq, 82*38+44-(82*3+41) +%if %2 + sub bufq, 82*(73-35*%3)+44-(82*3+41) +%else + sub bufq, 82*70-(82-3) +%endif add bufyq, 79+82*3 - mov hd, 35 + mov hd, 70-35*%3 .y_loop_ar3: - mov xq, -38 + mov xq, -(76>>%2) .x_loop_ar3: movu xm0, [bufq+xq-82*3-3] ; y=-3,x=[-3,+12] movu xm1, [bufq+xq-82*2-3] ; y=-2,x=[-3,+12] movu xm2, [bufq+xq-82*1-3] ; y=-1,x=[-3,+12] @@ -798,16 +912,24 @@ psrldq xm7, xm2, 6 psrldq xm8, xm2, 8 palignr xm9, xm5, xm2, 10 palignr xm5, xm5, xm2, 12 +%if %2 movq xm1, [bufyq+xq*2] +%if %3 movq xm2, [bufyq+xq*2+82] +%endif pmaddubsw xm1, xm13, xm1 +%if %3 pmaddubsw xm2, xm13, xm2 paddw xm1, xm2 +%endif pmulhrsw xm1, xm15 +%else + pmovsxbw xm1, [bufyq+xq] +%endif punpcklwd xm6, xm7 punpcklwd xm8, xm9 punpcklwd xm5, xm1 pmaddwd xm6, [rsp+ 8*16] @@ -839,14 +961,19 @@ jnz .x_loop_ar3_inner jmp .x_loop_ar3 .x_loop_ar3_end: add bufq, 82 - add bufyq, 82*2 + add bufyq, 82<<%3 dec hd jg .y_loop_ar3 RET +%endmacro + +generate_grain_uv_fn 420, 1, 1 +generate_grain_uv_fn 422, 1, 0 +generate_grain_uv_fn 444, 0, 0 INIT_YMM avx2 cglobal fgy_32x32xn, 6, 13, 16, dst, src, stride, fg_data, w, scaling, grain_lut pcmpeqw m10, m10 psrld m10, 24 @@ -1186,13 +1313,12 @@ add grain_lutq, 82 dec hw jz .end_y_v_overlap ; 2 lines get vertical overlap, then fall back to non-overlap code for ; remaining (up to) 30 lines - xor hd, 0x10000 - test hd, 0x10000 - jnz .loop_y_v_overlap + btc hd, 16 + jnc .loop_y_v_overlap jmp .loop_y .end_y_v_overlap: add wq, 32 jge .end_hv @@ -1319,25 +1445,25 @@ add grain_lutq, 82 dec hw jz .end_y_hv_overlap ; 2 lines get vertical overlap, then fall back to non-overlap code for ; remaining (up to) 30 lines - xor hd, 0x10000 - test hd, 0x10000 - jnz .loop_y_hv_overlap + btc hd, 16 + jnc .loop_y_hv_overlap jmp .loop_y_h_overlap .end_y_hv_overlap: add wq, 32 lea srcq, [src_bakq+wq] jl .loop_x_hv_overlap .end_hv: RET -cglobal fguv_32x32xn_i420, 6, 15, 16, dst, src, stride, fg_data, w, scaling, \ - grain_lut, h, sby, luma, lstride, uv_pl, is_id +%macro FGUV_FN 3 ; name, ss_hor, ss_ver +cglobal fguv_32x32xn_i%1, 6, 15, 16, dst, src, stride, fg_data, w, scaling, \ + grain_lut, h, sby, luma, lstride, uv_pl, is_id pcmpeqw m10, m10 psrld m10, 24 mov r7d, [fg_dataq+FGData.scaling_shift] lea r8, [pb_mask] %define base r8-pb_mask @@ -1349,22 +1475,26 @@ vpbroadcastw m12, [base+max+r7*2] cmp byte [fg_dataq+FGData.chroma_scaling_from_luma], 0 jne .csfl -%macro FGUV_32x32xN_LOOP 1 ; not-csfl +%macro %%FGUV_32x32xN_LOOP 3 ; not-csfl, ss_hor, ss_ver DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused, sby, see, overlap %if %1 mov r7d, dword r11m vpbroadcastb m0, [fg_dataq+FGData.uv_mult+r7*4] vpbroadcastb m1, [fg_dataq+FGData.uv_luma_mult+r7*4] punpcklbw m14, m1, m0 vpbroadcastw m15, [fg_dataq+FGData.uv_offset+r7*4] %else vpbroadcastd m14, [pw_1024] +%if %2 vpbroadcastd m15, [pb_23_22] +%else + vpbroadcastd xm15, [pb_27_17_17_27] +%endif %endif mov overlapd, [fg_dataq+FGData.overlap_flag] movifnidn sbyd, sbym test sbyd, sbyd @@ -1382,11 +1512,11 @@ unused2, unused3, see, overlap, unused4, unused5, lstride mov lumaq, r9mp lea r12, [srcq+wq] lea r13, [dstq+wq] - lea r14, [lumaq+wq*2] + lea r14, [lumaq+wq*(1+%2)] mov r11mp, r12 mov r12mp, r13 mov lstrideq, r10mp neg wq @@ -1403,35 +1533,43 @@ mov offxd, seed rorx offyd, seed, 8 shr offxd, 12 and offyd, 0xf - imul offyd, 82 - lea offyq, [offyq+offxq+498] ; offy*stride+offx + imul offyd, 164>>%3 + lea offyq, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ h, offxy, see, overlap, unused1, unused2, lstride mov hd, hm mov grain_lutq, grain_lutmp %%loop_y: ; src +%if %2 mova xm4, [lumaq+lstrideq*0+ 0] mova xm6, [lumaq+lstrideq*0+16] mova xm0, [srcq] vpbroadcastd m7, [pb_1] - vinserti128 m4, [lumaq+lstrideq*2 +0], 1 - vinserti128 m6, [lumaq+lstrideq*2+16], 1 + vinserti128 m4, [lumaq+lstrideq*(1+%3) +0], 1 + vinserti128 m6, [lumaq+lstrideq*(1+%3)+16], 1 vinserti128 m0, [srcq+strideq], 1 pxor m2, m2 pmaddubsw m4, m7 pmaddubsw m6, m7 pavgw m4, m2 pavgw m6, m2 +%else + pxor m2, m2 + mova m4, [lumaq] + mova m0, [srcq] +%endif %if %1 +%if %2 packuswb m4, m6 ; luma +%endif punpckhbw m6, m4, m0 punpcklbw m4, m0 ; { luma, chroma } pmaddubsw m6, m14 pmaddubsw m4, m14 psraw m6, 6 @@ -1439,10 +1577,13 @@ paddw m6, m15 paddw m4, m15 packuswb m4, m6 ; pack+unpack = clip punpckhbw m6, m4, m2 punpcklbw m4, m2 +%elif %2 == 0 + punpckhbw m6, m4, m2 + punpcklbw m4, m2 %endif punpckhwd m5, m4, m2 punpcklwd m4, m2 punpckhwd m7, m6, m2 @@ -1467,12 +1608,16 @@ ; unpack chroma_source punpckhbw m1, m0, m2 punpcklbw m0, m2 ; m0-1: src as word ; grain = grain_lut[offy+y][offx+x] +%if %2 movu xm3, [grain_lutq+offxyq+ 0] vinserti128 m3, [grain_lutq+offxyq+82], 1 +%else + movu m3, [grain_lutq+offxyq] +%endif pcmpgtb m7, m2, m3 punpcklbw m2, m3, m7 punpckhbw m3, m7 ; noise = round2(scaling[luma_src] * grain, scaling_shift) @@ -1487,25 +1632,35 @@ pmaxsw m0, m13 pmaxsw m1, m13 pminsw m0, m12 pminsw m1, m12 packuswb m0, m1 +%if %2 mova [dstq], xm0 vextracti128 [dstq+strideq], m0, 1 +%else + mova [dstq], m0 +%endif +%if %2 lea srcq, [srcq+strideq*2] lea dstq, [dstq+strideq*2] - lea lumaq, [lumaq+lstrideq*4] - add grain_lutq, 82*2 - sub hb, 2 + lea lumaq, [lumaq+lstrideq*(2<<%3)] +%else + add srcq, strideq + add dstq, strideq + add lumaq, lstrideq +%endif + add grain_lutq, 82<<%2 + sub hb, 1+%2 jg %%loop_y - add wq, 16 + add wq, 32>>%2 jge %%end mov srcq, r11mp mov dstq, r12mp - lea lumaq, [r14+wq*2] + lea lumaq, [r14+wq*(1+%2)] add srcq, wq add dstq, wq test overlapd, overlapd jz %%loop_x @@ -1523,40 +1678,48 @@ cmovp seed, r6d ; updated seed DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ offx, offy, see, left_offxy, unused1, unused2, lstride - lea left_offxyd, [offyd+16] ; previous column's offy*stride+offx + lea left_offxyd, [offyd+(32>>%2)] ; previous column's offy*stride+offx mov offxd, seed rorx offyd, seed, 8 shr offxd, 12 and offyd, 0xf - imul offyd, 82 - lea offyq, [offyq+offxq+498] ; offy*stride+offx + imul offyd, 164>>%3 + lea offyq, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ h, offxy, see, left_offxy, unused1, unused2, lstride mov hd, hm mov grain_lutq, grain_lutmp %%loop_y_h_overlap: ; src +%if %2 mova xm4, [lumaq+lstrideq*0+ 0] mova xm6, [lumaq+lstrideq*0+16] mova xm0, [srcq] vpbroadcastd m7, [pb_1] - vinserti128 m4, [lumaq+lstrideq*2 +0], 1 - vinserti128 m6, [lumaq+lstrideq*2+16], 1 + vinserti128 m4, [lumaq+lstrideq*(1+%3) +0], 1 + vinserti128 m6, [lumaq+lstrideq*(1+%3)+16], 1 vinserti128 m0, [srcq+strideq], 1 pxor m2, m2 pmaddubsw m4, m7 pmaddubsw m6, m7 pavgw m4, m2 pavgw m6, m2 +%else + mova m4, [lumaq] + mova m0, [srcq] + pxor m2, m2 +%endif %if %1 +%if %2 packuswb m4, m6 ; luma +%endif punpckhbw m6, m4, m0 punpcklbw m4, m0 ; { luma, chroma } pmaddubsw m6, m14 pmaddubsw m4, m14 psraw m6, 6 @@ -1564,10 +1727,13 @@ paddw m6, m15 paddw m4, m15 packuswb m4, m6 ; pack+unpack = clip punpckhbw m6, m4, m2 punpcklbw m4, m2 +%elif %2 == 0 + punpckhbw m6, m4, m2 + punpcklbw m4, m2 %endif punpckhwd m5, m4, m2 punpcklwd m4, m2 punpckhwd m7, m6, m2 @@ -1592,10 +1758,11 @@ ; unpack chroma_source punpckhbw m1, m0, m2 punpcklbw m0, m2 ; m0-1: src as word ; grain = grain_lut[offy+y][offx+x] +%if %2 %if %1 vpbroadcastd m6, [pb_23_22] ; FIXME %endif movu xm3, [grain_lutq+offxyq+ 0] movd xm4, [grain_lutq+left_offxyq+ 0] @@ -1611,10 +1778,29 @@ %endif packsswb m4, m4 pcmpeqw m6, m6 ; FIXME psrldq m6, 15 ; FIXME vpblendvb m3, m3, m4, m6 +%else +%if %1 + vpbroadcastd xm6, [pb_27_17_17_27] +%endif + movu m3, [grain_lutq+offxyq] + movd xm4, [grain_lutq+left_offxyq] + punpcklbw xm4, xm3 +%if %1 + pmaddubsw xm4, xm6, xm4 + pmulhrsw xm4, [pw_1024] +%else + pmaddubsw xm4, xm15, xm4 + pmulhrsw xm4, xm14 +%endif + packsswb xm4, xm4 + pcmpeqw xm6, xm6 + psrldq xm6, 14 + vpblendvb m3, m3, m4, m6 +%endif pcmpgtb m7, m2, m3 punpcklbw m2, m3, m7 punpckhbw m3, m7 ; noise = round2(scaling[luma_src] * grain, scaling_shift) @@ -1629,25 +1815,35 @@ pmaxsw m0, m13 pmaxsw m1, m13 pminsw m0, m12 pminsw m1, m12 packuswb m0, m1 +%if %2 mova [dstq], xm0 vextracti128 [dstq+strideq], m0, 1 +%else + mova [dstq], m0 +%endif +%if %2 lea srcq, [srcq+strideq*2] lea dstq, [dstq+strideq*2] - lea lumaq, [lumaq+lstrideq*4] - add grain_lutq, 82*2 - sub hb, 2 + lea lumaq, [lumaq+lstrideq*(2<<%3)] +%else + add srcq, strideq + add dstq, strideq + add lumaq, lstrideq +%endif + add grain_lutq, 82*(1+%2) + sub hb, 1+%2 jg %%loop_y_h_overlap - add wq, 16 + add wq, 32>>%2 jge %%end mov srcq, r11mp mov dstq, r12mp - lea lumaq, [r14+wq*2] + lea lumaq, [r14+wq*(1+%2)] add srcq, wq add dstq, wq ; r8m = sbym cmp dword r8m, 0 @@ -1676,11 +1872,11 @@ unused1, unused2, see, overlap, unused3, unused4, lstride mov lumaq, r9mp lea r12, [srcq+wq] lea r13, [dstq+wq] - lea r14, [lumaq+wq*2] + lea r14, [lumaq+wq*(1+%2)] mov r11mp, r12 mov r12mp, r13 mov lstrideq, r10mp neg wq @@ -1703,39 +1899,50 @@ rorx offyd, seed, 8 rorx offxd, seed, 12 and offyd, 0xf000f and offxd, 0xf000f - imul offyd, 82 + imul offyd, 164>>%3 ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy - lea offyq, [offyq+offxq+0x10001*498+16*82] + lea offyq, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ h, offxy, see, overlap, top_offxy, unused, lstride movzx top_offxyd, offxyw shr offxyd, 16 mov hd, hm mov grain_lutq, grain_lutmp +%if %2 == 0 + vbroadcasti128 m1, [pb_8x_27_17_8x_17_27] +%endif %%loop_y_v_overlap: ; src +%if %2 mova xm4, [lumaq+lstrideq*0+ 0] mova xm6, [lumaq+lstrideq*0+16] mova xm0, [srcq] vpbroadcastd m7, [pb_1] - vinserti128 m4, [lumaq+lstrideq*2 +0], 1 - vinserti128 m6, [lumaq+lstrideq*2+16], 1 + vinserti128 m4, [lumaq+lstrideq*(1+%3) +0], 1 + vinserti128 m6, [lumaq+lstrideq*(1+%3)+16], 1 vinserti128 m0, [srcq+strideq], 1 pxor m2, m2 pmaddubsw m4, m7 pmaddubsw m6, m7 pavgw m4, m2 pavgw m6, m2 +%else + mova m4, [lumaq] + mova m0, [srcq] + pxor m2, m2 +%endif %if %1 +%if %2 packuswb m4, m6 ; luma +%endif punpckhbw m6, m4, m0 punpcklbw m4, m0 ; { luma, chroma } pmaddubsw m6, m14 pmaddubsw m4, m14 psraw m6, 6 @@ -1743,10 +1950,13 @@ paddw m6, m15 paddw m4, m15 packuswb m4, m6 ; pack+unpack = clip punpckhbw m6, m4, m2 punpcklbw m4, m2 +%elif %2 == 0 + punpckhbw m6, m4, m2 + punpcklbw m4, m2 %endif punpckhwd m5, m4, m2 punpcklwd m4, m2 punpckhwd m7, m6, m2 @@ -1766,15 +1976,46 @@ pand m5, m10 pand m6, m10 packusdw m8, m4 packusdw m5, m6 +%if %2 ; unpack chroma_source punpckhbw m1, m0, m2 punpcklbw m0, m2 ; m0-1: src as word +%endif ; grain = grain_lut[offy+y][offx+x] +%if %3 == 0 +%if %2 + mova m6, [pb_8x_27_17_8x_17_27] + movu xm3, [grain_lutq+offxyq] + movu xm4, [grain_lutq+top_offxyq] + vinserti128 m3, [grain_lutq+offxyq+82], 1 + vinserti128 m4, [grain_lutq+top_offxyq+82], 1 +%else + movu m3, [grain_lutq+offxyq] + movu m4, [grain_lutq+top_offxyq] +%endif + punpckhbw m9, m4, m3 + punpcklbw m4, m3 +%if %2 + pmaddubsw m9, m6, m9 + pmaddubsw m4, m6, m4 +%else + pmaddubsw m9, m1, m9 + pmaddubsw m4, m1, m4 +%endif +%if %1 + pmulhrsw m9, [pw_1024] + pmulhrsw m4, [pw_1024] +%else + pmulhrsw m9, m14 + pmulhrsw m4, m14 +%endif + packsswb m3, m4, m9 +%else %if %1 vpbroadcastd m6, [pb_23_22] %endif movq xm3, [grain_lutq+offxyq] movq xm4, [grain_lutq+top_offxyq] @@ -1790,10 +2031,11 @@ %endif packsswb m4, m4 vpermq m4, m4, q3120 ; only interpolate first line, insert second line unmodified vinserti128 m3, m4, [grain_lutq+offxyq+82], 1 +%endif pcmpgtb m7, m2, m3 punpcklbw m2, m3, m7 punpckhbw m3, m7 ; noise = round2(scaling[luma_src] * grain, scaling_shift) @@ -1801,34 +2043,60 @@ pmullw m3, m5 pmulhrsw m2, m11 pmulhrsw m3, m11 ; dst = clip_pixel(src, noise) +%if %2 paddw m0, m2 paddw m1, m3 pmaxsw m0, m13 pmaxsw m1, m13 pminsw m0, m12 pminsw m1, m12 packuswb m0, m1 mova [dstq], xm0 vextracti128 [dstq+strideq], m0, 1 +%else + pxor m6, m6 + punpckhbw m9, m0, m6 + punpcklbw m0, m6 ; m0-1: src as word - sub hb, 2 + paddw m0, m2 + paddw m9, m3 + pmaxsw m0, m13 + pmaxsw m9, m13 + pminsw m0, m12 + pminsw m9, m12 + packuswb m0, m9 + mova [dstq], m0 +%endif + + sub hb, 1+%2 jl %%end_y_v_overlap +%if %2 lea srcq, [srcq+strideq*2] lea dstq, [dstq+strideq*2] - lea lumaq, [lumaq+lstrideq*4] - add grain_lutq, 82*2 + lea lumaq, [lumaq+lstrideq*(2<<%3)] +%else + add srcq, strideq + add dstq, strideq + add lumaq, lstrideq +%endif + add grain_lutq, 82<<%2 +%if %2 == 0 + vbroadcasti128 m1, [pb_8x_27_17_8x_17_27+16] + btc hd, 16 + jnc %%loop_y_v_overlap +%endif jmp %%loop_y %%end_y_v_overlap: - add wq, 16 + add wq, 32>>%2 jge %%end_hv mov srcq, r11mp mov dstq, r12mp - lea lumaq, [r14+wq*2] + lea lumaq, [r14+wq*(1+%2)] add srcq, wq add dstq, wq ; since fg_dataq.overlap is guaranteed to be set, we never jump ; back to .loop_x_v_overlap, and instead always fall-through to @@ -1849,45 +2117,56 @@ rorx seed, r7d, 1 ; updated (cur_seed << 16) | top_seed DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ offx, offy, see, left_offxy, top_offxy, topleft_offxy, lstride - lea topleft_offxyq, [top_offxyq+16] - lea left_offxyq, [offyq+16] + lea topleft_offxyq, [top_offxyq+(32>>%2)] + lea left_offxyq, [offyq+(32>>%2)] rorx offyd, seed, 8 rorx offxd, seed, 12 and offyd, 0xf000f and offxd, 0xf000f - imul offyd, 82 + imul offyd, 164>>%3 ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy - lea offyq, [offyq+offxq+0x10001*498+16*82] + lea offyq, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ h, offxy, see, left_offxy, top_offxy, topleft_offxy, lstride movzx top_offxyd, offxyw shr offxyd, 16 mov hd, hm mov grain_lutq, grain_lutmp +%if %2 == 0 + vbroadcasti128 m1, [pb_8x_27_17_8x_17_27] +%endif %%loop_y_hv_overlap: ; src +%if %2 mova xm4, [lumaq+lstrideq*0+ 0] mova xm6, [lumaq+lstrideq*0+16] mova xm0, [srcq] vpbroadcastd m7, [pb_1] - vinserti128 m4, [lumaq+lstrideq*2 +0], 1 - vinserti128 m6, [lumaq+lstrideq*2+16], 1 + vinserti128 m4, [lumaq+lstrideq*(1+%3) +0], 1 + vinserti128 m6, [lumaq+lstrideq*(1+%3)+16], 1 vinserti128 m0, [srcq+strideq], 1 pxor m2, m2 pmaddubsw m4, m7 pmaddubsw m6, m7 pavgw m4, m2 pavgw m6, m2 +%else + mova m4, [lumaq] + mova m0, [srcq] + pxor m2, m2 +%endif %if %1 +%if %2 packuswb m4, m6 ; luma +%endif punpckhbw m6, m4, m0 punpcklbw m4, m0 ; { luma, chroma } pmaddubsw m6, m14 pmaddubsw m4, m14 psraw m6, 6 @@ -1895,10 +2174,13 @@ paddw m6, m15 paddw m4, m15 packuswb m4, m6 ; pack+unpack = clip punpckhbw m6, m4, m2 punpcklbw m4, m2 +%elif %2 == 0 + punpckhbw m6, m4, m2 + punpcklbw m4, m2 %endif punpckhwd m5, m4, m2 punpcklwd m4, m2 punpckhwd m7, m6, m2 @@ -1918,48 +2200,98 @@ pand m5, m10 pand m6, m10 packusdw m8, m4 packusdw m5, m6 +%if %2 ; unpack chroma source punpckhbw m1, m0, m2 punpcklbw m0, m2 ; m0-1: src as word +%endif ; grain = grain_lut[offy+y][offx+x] %if %1 +%if %2 vpbroadcastd m9, [pb_23_22] +%else + vpbroadcastd xm9, [pb_27_17_17_27] %endif +%endif + +%if %2 movu xm3, [grain_lutq+offxyq] +%if %3 movq xm6, [grain_lutq+top_offxyq] +%else + movu xm6, [grain_lutq+top_offxyq] +%endif vinserti128 m3, [grain_lutq+offxyq+82], 1 +%if %3 vinserti128 m6, [grain_lutq+top_offxyq+8], 1 +%else + vinserti128 m6, [grain_lutq+top_offxyq+82], 1 +%endif +%else + movu m3, [grain_lutq+offxyq] + movu m6, [grain_lutq+top_offxyq] +%endif movd xm4, [grain_lutq+left_offxyq] movd xm7, [grain_lutq+topleft_offxyq] +%if %2 vinserti128 m4, [grain_lutq+left_offxyq+82], 1 +%if %3 == 0 + vinserti128 m7, [grain_lutq+topleft_offxyq+82], 1 +%endif +%endif + ; do h interpolation first (so top | top/left -> top, left | cur -> cur) +%if %2 punpcklbw m4, m3 +%if %3 punpcklbw xm7, xm6 +%else + punpcklbw m7, m6 +%endif + punpcklwd m4, m7 %if %1 pmaddubsw m4, m9, m4 - pmaddubsw xm7, xm9, xm7 pmulhrsw m4, [pw_1024] - pmulhrsw xm7, [pw_1024] %else pmaddubsw m4, m15, m4 - pmaddubsw xm7, xm15, xm7 pmulhrsw m4, m14 - pmulhrsw xm7, xm14 %endif packsswb m4, m4 - packsswb xm7, xm7 pcmpeqw m9, m9 ; this is kind of ugly psrldq m9, 15 vpblendvb m3, m3, m4, m9 - shufpd m9, m9, m9, 1110b - vpblendvb m6, m6, m7, m9 + psrldq m4, 1 +%if %3 + shufpd m9, m9, m9, 1110b ; clear upper lane +%endif + vpblendvb m6, m6, m4, m9 +%else + punpcklbw xm4, xm3 + punpcklbw xm7, xm6 + punpckldq xm4, xm7 +%if %1 + pmaddubsw xm4, xm9, xm4 + pmulhrsw xm4, [pw_1024] +%else + pmaddubsw xm4, xm15, xm4 + pmulhrsw xm4, xm14 +%endif + packsswb xm4, xm4 + pcmpeqw xm9, xm9 ; this is kind of ugly + psrldq xm9, 14 + vpblendvb m3, m3, m4, m9 + psrldq xm4, 2 + vpblendvb m6, m6, m4, m9 +%endif + + ; followed by v interpolation (top | cur -> cur) +%if %3 vpermq m9, m3, q3120 - ; followed by v interpolation (top | cur -> cur) punpcklbw m6, m9 %if %1 vpbroadcastd m9, [pb_23_22] pmaddubsw m6, m9, m6 pmulhrsw m6, [pw_1024] @@ -1968,10 +2300,30 @@ pmulhrsw m6, m14 %endif packsswb m6, m6 vpermq m6, m6, q3120 vpblendd m3, m3, m6, 00001111b +%else + punpckhbw m9, m6, m3 + punpcklbw m6, m3 +%if %2 + mova m3, [pb_8x_27_17_8x_17_27] + pmaddubsw m9, m3, m9 + pmaddubsw m6, m3, m6 +%else + pmaddubsw m9, m1, m9 + pmaddubsw m6, m1, m6 +%endif +%if %1 + pmulhrsw m9, [pw_1024] + pmulhrsw m6, [pw_1024] +%else + pmulhrsw m9, m14 + pmulhrsw m6, m14 +%endif + packsswb m3, m6, m9 +%endif pcmpgtb m7, m2, m3 punpcklbw m2, m3, m7 punpckhbw m3, m7 ; noise = round2(scaling[src] * grain, scaling_shift) @@ -1979,41 +2331,74 @@ pmullw m3, m5 pmulhrsw m2, m11 pmulhrsw m3, m11 ; dst = clip_pixel(src, noise) +%if %2 paddw m0, m2 paddw m1, m3 pmaxsw m0, m13 pmaxsw m1, m13 pminsw m0, m12 pminsw m1, m12 packuswb m0, m1 mova [dstq], xm0 vextracti128 [dstq+strideq], m0, 1 +%else + pxor m6, m6 + punpckhbw m9, m0, m6 + punpcklbw m0, m6 ; m0-1: src as word + paddw m0, m2 + paddw m9, m3 + pmaxsw m0, m13 + pmaxsw m9, m13 + pminsw m0, m12 + pminsw m9, m12 + packuswb m0, m9 + mova [dstq], m0 +%endif +%if %2 lea srcq, [srcq+strideq*2] lea dstq, [dstq+strideq*2] - lea lumaq, [lumaq+lstrideq*4] - add grain_lutq, 82*2 - sub hb, 2 + lea lumaq, [lumaq+lstrideq*(2<<%3)] +%else + add srcq, strideq + add dstq, strideq + add lumaq, lstrideq +%endif + add grain_lutq, 82<<%2 + sub hb, 1+%2 +%if %2 jg %%loop_y_h_overlap +%else + je %%end_y_hv_overlap + vbroadcasti128 m1, [pb_8x_27_17_8x_17_27+16] + btc hd, 16 + jnc %%loop_y_hv_overlap + jmp %%loop_y_h_overlap +%endif %%end_y_hv_overlap: - add wq, 16 + add wq, 32>>%2 jge %%end_hv mov srcq, r11mp mov dstq, r12mp - lea lumaq, [r14+wq*2] + lea lumaq, [r14+wq*(1+%2)] add srcq, wq add dstq, wq jmp %%loop_x_hv_overlap %%end_hv: RET %endmacro - FGUV_32x32xN_LOOP 1 + %%FGUV_32x32xN_LOOP 1, %2, %3 .csfl: - FGUV_32x32xN_LOOP 0 + %%FGUV_32x32xN_LOOP 0, %2, %3 +%endmacro + +FGUV_FN 420, 1, 1 +FGUV_FN 422, 1, 0 +FGUV_FN 444, 0, 0 %endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/film_grain_init_tmpl.c b/third_party/dav1d/src/x86/film_grain_init_tmpl.c --- a/third_party/dav1d/src/x86/film_grain_init_tmpl.c +++ b/third_party/dav1d/src/x86/film_grain_init_tmpl.c @@ -28,34 +28,50 @@ #include "src/cpu.h" #include "src/film_grain.h" decl_generate_grain_y_fn(dav1d_generate_grain_y_ssse3); decl_generate_grain_uv_fn(dav1d_generate_grain_uv_420_ssse3); +decl_generate_grain_uv_fn(dav1d_generate_grain_uv_422_ssse3); +decl_generate_grain_uv_fn(dav1d_generate_grain_uv_444_ssse3); decl_fgy_32x32xn_fn(dav1d_fgy_32x32xn_ssse3); decl_fguv_32x32xn_fn(dav1d_fguv_32x32xn_i420_ssse3); +decl_fguv_32x32xn_fn(dav1d_fguv_32x32xn_i422_ssse3); +decl_fguv_32x32xn_fn(dav1d_fguv_32x32xn_i444_ssse3); decl_generate_grain_y_fn(dav1d_generate_grain_y_avx2); decl_generate_grain_uv_fn(dav1d_generate_grain_uv_420_avx2); +decl_generate_grain_uv_fn(dav1d_generate_grain_uv_422_avx2); +decl_generate_grain_uv_fn(dav1d_generate_grain_uv_444_avx2); decl_fgy_32x32xn_fn(dav1d_fgy_32x32xn_avx2); decl_fguv_32x32xn_fn(dav1d_fguv_32x32xn_i420_avx2); +decl_fguv_32x32xn_fn(dav1d_fguv_32x32xn_i422_avx2); +decl_fguv_32x32xn_fn(dav1d_fguv_32x32xn_i444_avx2); COLD void bitfn(dav1d_film_grain_dsp_init_x86)(Dav1dFilmGrainDSPContext *const c) { const unsigned flags = dav1d_get_cpu_flags(); if (!(flags & DAV1D_X86_CPU_FLAG_SSSE3)) return; #if BITDEPTH == 8 c->generate_grain_y = dav1d_generate_grain_y_ssse3; c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I420 - 1] = dav1d_generate_grain_uv_420_ssse3; + c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I422 - 1] = dav1d_generate_grain_uv_422_ssse3; + c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I444 - 1] = dav1d_generate_grain_uv_444_ssse3; c->fgy_32x32xn = dav1d_fgy_32x32xn_ssse3; c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I420 - 1] = dav1d_fguv_32x32xn_i420_ssse3; + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I422 - 1] = dav1d_fguv_32x32xn_i422_ssse3; + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I444 - 1] = dav1d_fguv_32x32xn_i444_ssse3; #endif if (!(flags & DAV1D_X86_CPU_FLAG_AVX2)) return; #if BITDEPTH == 8 && ARCH_X86_64 c->generate_grain_y = dav1d_generate_grain_y_avx2; c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I420 - 1] = dav1d_generate_grain_uv_420_avx2; + c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I422 - 1] = dav1d_generate_grain_uv_422_avx2; + c->generate_grain_uv[DAV1D_PIXEL_LAYOUT_I444 - 1] = dav1d_generate_grain_uv_444_avx2; c->fgy_32x32xn = dav1d_fgy_32x32xn_avx2; c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I420 - 1] = dav1d_fguv_32x32xn_i420_avx2; + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I422 - 1] = dav1d_fguv_32x32xn_i422_avx2; + c->fguv_32x32xn[DAV1D_PIXEL_LAYOUT_I444 - 1] = dav1d_fguv_32x32xn_i444_avx2; #endif } diff --git a/third_party/dav1d/src/x86/film_grain_ssse3.asm b/third_party/dav1d/src/x86/film_grain_ssse3.asm --- a/third_party/dav1d/src/x86/film_grain_ssse3.asm +++ b/third_party/dav1d/src/x86/film_grain_ssse3.asm @@ -58,10 +58,12 @@ %endrep %endmacro JMP_TABLE generate_grain_y_ssse3, 0, 1, 2, 3 JMP_TABLE generate_grain_uv_420_ssse3, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_422_ssse3, 0, 1, 2, 3 +JMP_TABLE generate_grain_uv_444_ssse3, 0, 1, 2, 3 struc FGData .seed: resd 1 .num_y_points: resd 1 .y_points: resb 14 * 2 @@ -500,12 +502,13 @@ add bufq, 82 dec hd jg .y_loop_ar3 RET +%macro generate_grain_uv_fn 3 ; ss_name, ss_x, ss_y INIT_XMM ssse3 -cglobal generate_grain_uv_420, 1, 7 + 3 * ARCH_X86_64, 16, buf, bufy, fg_data, uv +cglobal generate_grain_uv_%1, 1, 7 + 3 * ARCH_X86_64, 16, buf, bufy, fg_data, uv movifnidn r2, r2mp movifnidn r3, r3mp LEA r4, $$ %define base r4-$$ movq m1, [base+rnd_next_upperbit_mask] @@ -518,19 +521,25 @@ movd m2, [base+pw_seed_xor+uvq*4] pxor m0, m2 pshuflw m6, m6, q0000 pshuflw m0, m0, q0000 lea r6, [base+gaussian_sequence] +%if %2 %if ARCH_X86_64 - mov r7d, 38 + mov r7d, 73-35*%3 %else - mov r3mp, 38 + mov r3mp, 73-35*%3 %endif add bufq, 44 .loop_y: mov r5, -44 .loop_x: +%else + mov r5, -82*73 + sub bufq, r5 +.loop: +%endif pand m2, m0, m1 psrlw m3, m2, 10 por m2, m3 ; bits 0xf, 0x1e, 0x3c and 0x78 are set pmullw m2, m4 ; bits 0x0f00 are set pshufb m3, m5, m2 ; set 15th bit for next 4 seeds @@ -575,27 +584,31 @@ %endif pmulhrsw m3, m6 packsswb m3, m3 movd [bufq+r5], m3 add r5, 4 +%if %2 jl .loop_x add bufq, 82 %if ARCH_X86_64 dec r7d %else dec r3mp %endif jg .loop_y +%else + jl .loop +%endif %if ARCH_X86_32 mov r2, r2mp %endif ; auto-regression code movsxd r5, [fg_dataq+FGData.ar_coeff_lag] - movsxd r5, [base+generate_grain_uv_420_ssse3_table+r5*4] - lea r5, [r5+base+generate_grain_uv_420_ssse3_table] + movsxd r5, [base+generate_grain_uv_%1_ssse3_table+r5*4] + lea r5, [r5+base+generate_grain_uv_%1_ssse3_table] jmp r5 .ar0: DEFINE_ARGS buf, bufy, fg_data, uv, unused, shift movifnidn bufyq, bufymp @@ -605,83 +618,134 @@ %endif imul uvd, 28 mov shiftd, [fg_dataq+FGData.ar_coeff_shift] movd m5, [fg_dataq+FGData.ar_coeffs_uv+uvq] movd m4, [base+hmul_bits+shiftq*2] - movd m1, [base+byte_blend] - DEFINE_ARGS buf, bufy, h + DEFINE_ARGS buf, bufy, h, x pxor m0, m0 pcmpgtb m0, m5 punpcklbw m5, m0 movd m7, [base+pb_1] - movd m6, [base+hmul_bits+4] +%if %2 + movd m6, [base+hmul_bits+2+%3*2] +%endif pshuflw m5, m5, q0000 pshuflw m4, m4, q0000 pshufd m7, m7, q0000 +%if %2 pshuflw m6, m6, q0000 +%endif punpcklqdq m5, m5 punpcklqdq m4, m4 +%if %2 punpcklqdq m6, m6 - punpcklbw m1, m1 +%endif + pcmpeqw m1, m1 + pslldq m1, 12>>%2 SCRATCH 1, 8, 0 SCRATCH 4, 9, 1 - sub bufq, 82*38+82-(82*3+41) +%if %2 + sub bufq, 82*(73-35*%3)+82-(82*3+41) +%else + sub bufq, 82*70-3 +%endif add bufyq, 3+82*3 - mov hd, 35 + mov hd, 70-35*%3 .y_loop_ar0: + xor xd, xd +.x_loop_ar0: ; first 32 pixels - movu m1, [bufyq] - movu m2, [bufyq+82] - movu m3, [bufyq+16] - movu m4, [bufyq+82+16] +%if %2 + movu m1, [bufyq+xq*2] +%if %3 + movu m2, [bufyq+xq*2+82] +%endif + movu m3, [bufyq+xq*2+16] +%if %3 + movu m4, [bufyq+xq*2+82+16] +%endif pmaddubsw m0, m7, m1 +%if %3 pmaddubsw m1, m7, m2 +%endif pmaddubsw m2, m7, m3 +%if %3 pmaddubsw m3, m7, m4 paddw m0, m1 paddw m2, m3 +%endif pmulhrsw m0, m6 pmulhrsw m2, m6 +%else + movu m0, [bufyq+xq] + pxor m6, m6 + pcmpgtb m6, m0 + punpckhbw m2, m0, m6 + punpcklbw m0, m6 +%endif pmullw m0, m5 pmullw m2, m5 pmulhrsw m0, m9 pmulhrsw m2, m9 + movu m1, [bufq+xq] + pxor m4, m4 + pcmpgtb m4, m1 + punpckhbw m3, m1, m4 +%if %2 + punpcklbw m1, m4 + paddw m2, m3 + paddw m0, m1 +%else + punpcklbw m6, m1, m4 + paddw m2, m3 + paddw m0, m6 +%endif packsswb m0, m2 - movu m1, [bufq] - punpckhbw m2, m0, m1 - punpcklbw m0, m1 - pmaddubsw m1, m7, m2 - pmaddubsw m2, m7, m0 - packsswb m2, m1 - movu [bufq], m2 - add bufyq, 32 - add bufq, 16 - xor hd, 0x10000 - test hd, 0x10000 - jnz .y_loop_ar0 +%if %2 + movu [bufq+xq], m0 + add xd, 16 + cmp xd, 32 + jl .x_loop_ar0 - ; last 6 pixels - movu m1, [bufyq] - movu m2, [bufyq+82] + ; last 6/12 pixels + movu m1, [bufyq+xq*(1+%2)] +%if %3 + movu m2, [bufyq+xq*2+82] +%endif pmaddubsw m0, m7, m1 +%if %3 pmaddubsw m1, m7, m2 paddw m0, m1 +%endif pmulhrsw m0, m6 pmullw m0, m5 pmulhrsw m0, m9 + movq m1, [bufq+xq] + pxor m4, m4 + pcmpgtb m4, m1 + punpcklbw m2, m1, m4 + paddw m0, m2 packsswb m0, m0 - movq m1, [bufq] - punpcklbw m0, m1 - pmaddubsw m2, m7, m0 - packsswb m2, m2 - pandn m0, m8, m2 + pandn m2, m8, m0 pand m1, m8 - por m0, m1 - movq [bufq], m0 + por m2, m1 + movq [bufq+xq], m2 +%else + add xd, 16 + cmp xd, 80 + je .y_loop_final_ar0 + movu [bufq+xq-16], m0 + jmp .x_loop_ar0 +.y_loop_final_ar0: + pandn m2, m8, m0 + pand m1, m8 + por m2, m1 + movu [bufq+xq-16], m2 +%endif - add bufq, 82-32 - add bufyq, 82*2-64 + add bufq, 82 + add bufyq, 82<<%3 dec hd jg .y_loop_ar0 RET .ar1: @@ -704,56 +768,82 @@ %else DEFINE_ARGS buf, bufy, fg_data, shift, val3, cf3, min, max, x %endif mov shiftd, [fg_dataq+FGData.ar_coeff_shift] movd m3, [base+round_vals+shiftq*2-12] ; rnd +%if %2 movd m7, [base+pb_1] - movd m6, [base+hmul_bits+4] + movd m6, [base+hmul_bits+2+%3*2] +%endif psrldq m4, 1 %if ARCH_X86_32 DEFINE_ARGS buf, shift, val0, val3, min, max, x %elif WIN64 DEFINE_ARGS shift, bufy, h, buf, val3, cf3, min, max, x, val0 %else DEFINE_ARGS buf, bufy, h, shift, val3, cf3, min, max, x, val0 %endif pxor m5, m5 punpcklwd m3, m5 +%if %2 punpcklwd m6, m6 +%endif pcmpgtb m5, m4 punpcklbw m4, m5 pshufd m5, m4, q1111 pshufd m4, m4, q0000 pshufd m3, m3, q0000 +%if %2 pshufd m7, m7, q0000 pshufd m6, m6, q0000 - sub bufq, 82*38+44-(82*3+41) + sub bufq, 82*(73-35*%3)+44-(82*3+41) +%else + sub bufq, 82*69+3 +%endif %if ARCH_X86_32 add r1mp, 79+82*3 - mov r0mp, 35 + mov r0mp, 70-35*%3 %else add bufyq, 79+82*3 - mov hd, 35 + mov hd, 70-35*%3 %endif mov mind, -128 mov maxd, 127 .y_loop_ar1: - mov xq, -38 + mov xq, -(76>>%2) movsx val3d, byte [bufq+xq-1] .x_loop_ar1: +%if %2 %if ARCH_X86_32 mov r2, r1mp movq m0, [r2+xq*2] +%if %3 movq m1, [r2+xq*2+82] +%endif %else movq m0, [bufyq+xq*2] +%if %3 movq m1, [bufyq+xq*2+82] %endif +%endif pmaddubsw m2, m7, m0 +%if %3 pmaddubsw m0, m7, m1 paddw m2, m0 +%endif pmulhrsw m2, m6 +%else +%if ARCH_X86_32 + mov r2, r1mp + movd m2, [r2+xq] +%else + movd m2, [bufyq+xq] +%endif + pxor m0, m0 + pcmpgtb m0, m2 + punpcklbw m2, m0 +%endif movq m0, [bufq+xq-82-1] ; top/left pxor m1, m1 pcmpgtb m1, m0 punpcklbw m0, m1 @@ -790,14 +880,14 @@ jmp .x_loop_ar1 .x_loop_ar1_end: add bufq, 82 %if ARCH_X86_32 - add r1mp, 82*2 + add r1mp, 82<<%3 dec r0mp %else - add bufyq, 82*2 + add bufyq, 82<<%3 dec hd %endif jg .y_loop_ar1 RET @@ -835,20 +925,24 @@ SCRATCH 3, 11, 3 SCRATCH 4, 12, 4 SCRATCH 5, 13, 5 SCRATCH 6, 14, 6 SCRATCH 7, 15, 7 - movd m7, [base+hmul_bits+4] +%if %2 + movd m7, [base+hmul_bits+2+%3*2] movd m6, [base+pb_1] punpcklwd m7, m7 pshufd m6, m6, q0000 pshufd m7, m7, q0000 - sub bufq, 82*38+44-(82*3+41) + sub bufq, 82*(73-35*%3)+44-(82*3+41) +%else + sub bufq, 82*69+3 +%endif add bufyq, 79+82*3 - mov hd, 35 + mov hd, 70-35*%3 .y_loop_ar2: - mov xq, -38 + mov xq, -(76>>%2) .x_loop_ar2: pxor m2, m2 movq m0, [bufq+xq-82*2-2] ; y=-2,x=[-2,+5] movhps m0, [bufq+xq-82*1-2] ; y=-1,x=[-2,+5] @@ -877,16 +971,27 @@ pmaddwd m3, m12 paddd m4, m0 paddd m2, m3 paddd m2, m4 - movq m0, [bufyq+xq*2] +%if %2 + movq m1, [bufyq+xq*2] +%if %3 movq m3, [bufyq+xq*2+82] - pmaddubsw m1, m6, m0 - pmaddubsw m0, m6, m3 +%endif + pmaddubsw m0, m6, m1 +%if %3 + pmaddubsw m1, m6, m3 paddw m0, m1 +%endif pmulhrsw m0, m7 +%else + movd m0, [bufyq+xq] + pxor m1, m1 + pcmpgtb m1, m0 + punpcklbw m0, m1 +%endif punpcklwd m0, m15 pmaddwd m0, m14 paddd m2, m0 movq m0, [bufq+xq-2] ; y=0,x=[-2,+5] @@ -912,11 +1017,11 @@ jnz .x_loop_ar2_inner jmp .x_loop_ar2 .x_loop_ar2_end: add bufq, 82 - add bufyq, 82*2 + add bufyq, 82<<%3 dec hd jg .y_loop_ar2 RET .ar3: @@ -975,28 +1080,40 @@ SCRATCH 3, 10, 9 SCRATCH 4, 11, 10 SCRATCH 5, 12, 11 movd m2, [base+round_vals-12+shiftq*2] +%if %2 movd m1, [base+pb_1] - movd m3, [base+hmul_bits+4] + movd m3, [base+hmul_bits+2+%3*2] +%endif pxor m0, m0 punpcklwd m2, m0 +%if %2 punpcklwd m3, m3 +%endif pshufd m2, m2, q0000 +%if %2 pshufd m1, m1, q0000 pshufd m3, m3, q0000 SCRATCH 1, 13, 12 +%endif SCRATCH 2, 14, 13 +%if %2 SCRATCH 3, 15, 14 +%endif DEFINE_ARGS buf, bufy, fg_data, h, unused, x - sub bufq, 82*38+44-(82*3+41) +%if %2 + sub bufq, 82*(73-35*%3)+44-(82*3+41) +%else + sub bufq, 82*69+3 +%endif add bufyq, 79+82*3 - mov hd, 35 + mov hd, 70-35*%3 .y_loop_ar3: - mov xq, -38 + mov xq, -(76>>%2) .x_loop_ar3: movu m0, [bufq+xq-82*3-3] ; y=-3,x=[-3,+12] pxor m4, m4 pcmpgtb m4, m0 @@ -1056,16 +1173,27 @@ pmaddwd m3, [rsp+ 4*16] pmaddwd m5, [rsp+ 5*16] paddd m3, m5 paddd m0, m3 +%if %2 movq m1, [bufyq+xq*2] +%if %3 movq m3, [bufyq+xq*2+82] - pmaddubsw m5, m13, m1 - pmaddubsw m7, m13, m3 +%endif + pmaddubsw m7, m13, m1 +%if %3 + pmaddubsw m5, m13, m3 paddw m7, m5 +%endif pmulhrsw m7, m15 +%else + movd m7, [bufyq+xq] + pxor m1, m1 + pcmpgtb m1, m7 + punpcklbw m7, m1 +%endif psrldq m1, m2, 4 psrldq m3, m2, 6 palignr m4, m6, m2, 10 palignr m6, m2, 12 @@ -1108,14 +1236,19 @@ jnz .x_loop_ar3_inner jmp .x_loop_ar3 .x_loop_ar3_end: add bufq, 82 - add bufyq, 82*2 + add bufyq, 82<<%3 dec hd jg .y_loop_ar3 RET +%endmacro + +generate_grain_uv_fn 420, 1, 1 +generate_grain_uv_fn 422, 1, 0 +generate_grain_uv_fn 444, 0, 0 %macro vpgatherdw 5-6 ; dst, src, base, tmp_gpr[x2], tmp_xmm_reg %assign %%idx 0 %define %%tmp %2 %if %0 == 6 @@ -1357,17 +1490,15 @@ %endif jge .end %if ARCH_X86_32 mov srcq, r1mp add srcq, r4mp - xor r8mp, 4 - test r8mp, 4 %else lea srcq, [src_bakq+wq] - test srcq, 16 ; this relies on buffer alignment... %endif - jz .next_blk + btc dword r8m, 2 + jc .next_blk add offxyd, 16 test dword r8m, 2 ; r8m & 2 = have_top_overlap jz .loop_x_odd @@ -1505,15 +1636,14 @@ %endif jge .end %if ARCH_X86_32 mov srcq, r1m add srcq, r4m - xor r8mp, 4 %else lea srcq, [src_bakq+wq] %endif - ; assert(srcq & 16) != 0 + xor dword r8m, 4 add offxyd, 16 ; since this half-block had left-overlap, the next does not test dword r8m, 2 ; have_top_overlap jz .loop_x_odd @@ -1710,13 +1840,12 @@ add grain_lutq, 82 dec hw jz .end_y_v_overlap ; 2 lines get vertical overlap, then fall back to non-overlap code for ; remaining (up to) 30 lines - xor hd, 0x10000 - test hd, 0x10000 - jnz .loop_y_v_overlap + btc hd, 16 + jnc .loop_y_v_overlap jmp .loop_y .end_y_v_overlap: %if ARCH_X86_32 add r4mp, 16 @@ -1725,17 +1854,15 @@ %endif jge .end_hv %if ARCH_X86_32 mov srcq, r1mp add srcq, r4mp - xor r8mp, 4 - test r8mp, 4 %else lea srcq, [src_bakq+wq] - test srcq, 16 %endif - jz .loop_x_hv_overlap + btc dword r8m, 2 + jc .loop_x_hv_overlap add offxyd, 16 %if ARCH_X86_32 add dword [rsp+6*mmsize+1*gprsize], 16 %else add top_offxyd, 16 @@ -1913,13 +2040,12 @@ add grain_lutq, 82 dec hw jz .end_y_hv_overlap ; 2 lines get vertical overlap, then fall back to non-overlap code for ; remaining (up to) 30 lines - xor hd, 0x10000 - test hd, 0x10000 - jnz .loop_y_hv_overlap + btc hd, 16 + jnc .loop_y_hv_overlap jmp .loop_y_h_overlap .end_y_hv_overlap: %if ARCH_X86_32 add r4mp, 16 @@ -1928,15 +2054,14 @@ %endif jge .end_hv %if ARCH_X86_32 mov srcq, r1m add srcq, r4m - xor r8mp, 4 %else lea srcq, [src_bakq+wq] %endif - ; assert(srcq & 16) != 0 + xor dword r8m, 4 add offxyd, 16 %if ARCH_X86_32 add dword [rsp+6*mmsize+1*gprsize], 16 %else add top_offxyd, 16 @@ -1944,17 +2069,18 @@ jmp .loop_x_odd_v_overlap .end_hv: RET +%macro FGUV_FN 3 ; name, ss_hor, ss_ver INIT_XMM ssse3 %if ARCH_X86_32 ; fguv_32x32xn_i420_ssse3(dst, src, stride, fg_data, w, scaling, grain_lut, h, ; sby, luma, lstride, uv_pl, is_id) %if STACK_ALIGNMENT < mmsize DECLARE_ARG 0, 1, 2, 3, 4, 5, 6, 7, 8 -cglobal fguv_32x32xn_i420, 0, 7, 8, 0 - (8 * mmsize + (13 + 3) * gprsize), \ +cglobal fguv_32x32xn_i%1, 0, 7, 8, 0 - (8 * mmsize + (13 + 3) * gprsize), \ tmp, src, scaling, h, fg_data, picptr, unused mov r0, r0m mov r1, r2m mov r2, r4m mov r3, r6m @@ -1973,11 +2099,11 @@ mov [rsp+8*mmsize+11*gprsize], r0 mov [rsp+8*mmsize+12*gprsize], r1 mov [rsp+8*mmsize+13*gprsize], r2 mov [rsp+8*mmsize+14*gprsize], r4 %else -cglobal fguv_32x32xn_i420, 0, 7, 8, 8 * mmsize + (4) * gprsize, \ +cglobal fguv_32x32xn_i%1, 0, 7, 8, 8 * mmsize + (4) * gprsize, \ tmp, src, scaling, h, fg_data, picptr, unused %endif mov srcq, srcm mov fg_dataq, r3m mov scalingq, r5m @@ -1998,17 +2124,17 @@ %endif LEA r5, pb_mask %define base r5-pb_mask mov r5m, r5 %else -cglobal fguv_32x32xn_i420, 6, 15, 16, dst, src, stride, fg_data, w, scaling, \ - grain_lut, tmp, sby, luma, lstride, uv_pl, is_id +cglobal fguv_32x32xn_i%1, 6, 15, 16, dst, src, stride, fg_data, w, scaling, \ + grain_lut, tmp, sby, luma, lstride, uv_pl, is_id lea r8, [pb_mask] %define base r8-pb_mask %endif mov r6d, [fg_dataq+FGData.scaling_shift] - movd m2, [base+byte_blend+3] + pcmpeqw m2, m2 movd m3, [base+mul_bits+r6*2-14] mov r6d, [fg_dataq+FGData.clip_to_restricted_range] lea tmpd, [r6d*2] %if ARCH_X86_32 && STACK_ALIGNMENT < mmsize test r3, r3 @@ -2016,10 +2142,11 @@ cmp dword r12m, 0 ; is_idm %endif movd m5, [base+min+r6*2] cmovne r6d, tmpd movd m4, [base+max+r6*2] + psrldq m2, 14+%2 punpcklwd m3, m3 punpcklwd m5, m5 punpcklwd m4, m4 pshufd m3, m3, q0000 pshufd m5, m5, q0000 @@ -2030,11 +2157,11 @@ SCRATCH 5, 13, 3 cmp byte [fg_dataq+FGData.chroma_scaling_from_luma], 0 jne .csfl -%macro FGUV_32x32xN_LOOP 1 ; not-csfl +%macro %%FGUV_32x32xN_LOOP 3 ; not-csfl, ss_hor, ss_ver %if ARCH_X86_32 DEFINE_ARGS dst, src, scaling, sby, fg_data, picptr, overlap %else DEFINE_ARGS dst, src, stride, fg_data, w, scaling, grain_lut, unused, sby, see, overlap %endif @@ -2056,14 +2183,22 @@ mov sbyd, r8m mov overlapd, [fg_dataq+FGData.overlap_flag] ; left_overlap: overlap & 1 test overlapd, overlapd jz %%no_vertical_overlap %if ARCH_X86_32 +%if %2 movd m1, [base+pb_23_22] +%else + movd m1, [base+pb_27_17_17_27] +%endif mova m0, [base+pw_1024] %else +%if %2 movd m1, [pb_23_22] +%else + movd m1, [pb_27_17_17_27] +%endif mova m0, [pw_1024] %endif pshufd m1, m1, q0000 SCRATCH 0, 8, 6 SCRATCH 1, 9, 7 @@ -2089,32 +2224,34 @@ DEFINE_ARGS luma, src, scaling, see, w, picptr, src_bak %define luma_bakq lumaq mov wq, r4m +%if %3 shl r10mp, 1 +%endif %else DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ unused2, unused3, see, overlap, unused4, src_bak, lstride, luma_bak mov lstrideq, r10mp %endif mov lumaq, r9mp lea src_bakq, [srcq+wq] - lea luma_bakq, [lumaq+wq*2] + lea luma_bakq, [lumaq+wq*(1+%2)] neg wq sub r0mp, srcq %if ARCH_X86_32 mov r1m, src_bakq mov r11m, luma_bakq mov r4m, wq DEFINE_ARGS tmp, src, scaling, see, unused1, picptr, unused2 %else mov r11mp, src_bakq - mov r10mp, strideq + mov r12mp, strideq %endif %%loop_x: %if ARCH_X86_32 mov seed, r3m @@ -2139,27 +2276,29 @@ mov offxd, seed %endif ror offyd, 8 shr offxd, 12 and offyd, 0xf - imul offyd, 82 - lea offyq, [offyq+offxq+498] ; offy*stride+offx + imul offyd, 164>>%3 + lea offyq, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+(3+(6>>%2))] ; offy*stride+offx %if ARCH_X86_32 DEFINE_ARGS luma, src, scaling, offxy, h, picptr, grain_lut %else DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ h, offxy, see, overlap, unused1, unused2, lstride, luma_bak %endif +%%loop_x_odd: mov hd, r7m mov grain_lutq, grain_lutmp %%loop_y: ; src %if ARCH_X86_32 mov lumaq, r9mp %endif +%if %2 mova m4, [lumaq+ 0] mova m6, [lumaq+16] mova m0, [srcq] %if ARCH_X86_32 add lumaq, r10mp @@ -2173,13 +2312,24 @@ pxor m2, m2 pmaddubsw m4, m7 pmaddubsw m6, m7 pavgw m4, m2 pavgw m6, m2 +%else + mova m4, [lumaq] + mova m0, [srcq] +%if ARCH_X86_32 + add lumaq, r10mp + mov r9mp, lumaq +%endif + pxor m2, m2 +%endif %if %1 +%if %2 packuswb m4, m6 ; luma +%endif punpckhbw m6, m4, m0 punpcklbw m4, m0 ; { luma, chroma } pmaddubsw m6, m14 pmaddubsw m4, m14 psraw m6, 6 @@ -2187,10 +2337,13 @@ paddw m6, m15 paddw m4, m15 packuswb m4, m6 ; pack+unpack = clip punpckhbw m6, m4, m2 punpcklbw m4, m2 +%elif %2 == 0 + punpckhbw m6, m4, m2 + punpcklbw m4, m2 %endif ; scaling[luma_src] %if ARCH_X86_32 vpgatherdw m7, m4, scalingq, r0, r5 @@ -2237,12 +2390,16 @@ %if ARCH_X86_32 add srcq, r2mp ; we already incremented lumaq above %else - add srcq, r10mp + add srcq, r12mp +%if %3 lea lumaq, [lumaq+lstrideq*2] +%else + add lumaq, lstrideq +%endif %endif add grain_lutq, 82 dec hw jg %%loop_y @@ -2257,37 +2414,60 @@ mov srcq, r1mp mov lumaq, r11mp %else mov srcq, r11mp %endif - lea lumaq, [luma_bakq+wq*2] + lea lumaq, [luma_bakq+wq*(1+%2)] add srcq, wq %if ARCH_X86_32 mov r4m, wq mov r9m, lumaq %endif +%if %2 == 0 + ; adjust top_offxy +%if ARCH_X86_32 + add dword [rsp+8*mmsize+1*gprsize], 16 +%else + add r11d, 16 +%endif + add offxyd, 16 + btc dword r8m, 2 + jc %%loop_x_even + test dword r8m, 2 + jz %%loop_x_odd + jmp %%loop_x_odd_v_overlap +%%loop_x_even: +%endif test dword r8m, 1 jz %%loop_x ; r8m = sbym test dword r8m, 2 jne %%loop_x_hv_overlap ; horizontal overlap (without vertical overlap) %%loop_x_h_overlap: %if ARCH_X86_32 +%if %2 lea r6, [offxyd+16] mov [rsp+8*mmsize+0*gprsize], r6 +%else + mov [rsp+8*mmsize+0*gprsize], offxyd +%endif DEFINE_ARGS luma, src, scaling, see, w, picptr, grain_lut mov seed, r3m %else DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ offx, offy, see, left_offxy, unused1, unused2, lstride +%if %2 lea left_offxyd, [offyd+16] ; previous column's offy*stride+offx +%else + mov left_offxyd, offyd +%endif %endif mov r6d, seed or seed, 0xEFF4 shr r6d, 1 test seeb, seeh @@ -2308,12 +2488,12 @@ mov offxd, seed %endif ror offyd, 8 shr offxd, 12 and offyd, 0xf - imul offyd, 82 - lea offyq, [offyq+offxq+498] ; offy*stride+offx + imul offyd, 164>>%3 + lea offyq, [offyq+offxq*(2-%2)+(3+(6>>%3))*82+3+(6>>%2)] ; offy*stride+offx %if ARCH_X86_32 DEFINE_ARGS luma, src, scaling, offxy, h, picptr, grain_lut %else DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ @@ -2325,10 +2505,11 @@ %%loop_y_h_overlap: ; src %if ARCH_X86_32 mov lumaq, r9mp %endif +%if %2 mova m4, [lumaq+ 0] mova m6, [lumaq+16] mova m0, [srcq] %if ARCH_X86_32 add lumaq, r10mp @@ -2342,13 +2523,24 @@ pxor m2, m2 pmaddubsw m4, m7 pmaddubsw m6, m7 pavgw m4, m2 pavgw m6, m2 +%else + mova m4, [lumaq] + mova m0, [srcq] +%if ARCH_X86_32 + add lumaq, r10mp + mov r9mp, lumaq +%endif + pxor m2, m2 +%endif %if %1 +%if %2 packuswb m4, m6 ; luma +%endif punpckhbw m6, m4, m0 punpcklbw m4, m0 ; { luma, chroma } pmaddubsw m6, m14 pmaddubsw m4, m14 psraw m6, 6 @@ -2356,10 +2548,13 @@ paddw m6, m15 paddw m4, m15 packuswb m4, m6 ; pack+unpack = clip punpckhbw m6, m4, m2 punpcklbw m4, m2 +%elif %2 == 0 + punpckhbw m6, m4, m2 + punpcklbw m4, m2 %endif ; scaling[luma_src] %if ARCH_X86_32 vpgatherdw m7, m4, scalingq, r0, r5 @@ -2420,12 +2615,16 @@ %if ARCH_X86_32 add srcq, r2mp ; lumaq has already been incremented above %else - add srcq, r10mp + add srcq, r12mp +%if %3 lea lumaq, [lumaq+lstrideq*2] +%else + add lumaq, lstrideq +%endif %endif add grain_lutq, 82 dec hw jg %%loop_y_h_overlap @@ -2440,21 +2639,36 @@ mov srcq, r1mp mov lumaq, r11mp %else mov srcq, r11mp %endif - lea lumaq, [luma_bakq+wq*2] + lea lumaq, [luma_bakq+wq*(1+%2)] add srcq, wq %if ARCH_X86_32 mov r4m, wq mov r9m, lumaq %endif +%if %2 == 0 + xor dword r8m, 4 + ; adjust top_offxyd +%if ARCH_X86_32 + add dword [rsp+8*mmsize+1*gprsize], 16 +%else + add r11d, 16 +%endif + add offxyd, 16 +%endif ; r8m = sbym test dword r8m, 2 +%if %2 jne %%loop_x_hv_overlap jmp %%loop_x_h_overlap +%else + jne %%loop_x_odd_v_overlap + jmp %%loop_x_odd +%endif %%end: RET %%vertical_overlap: @@ -2485,11 +2699,13 @@ DEFINE_ARGS luma, src, scaling, see, w, picptr, src_bak mov r3m, seed mov wq, r4m +%if %3 shl r10mp, 1 +%endif %else xor seed, sbyd ; (cur_seed << 16) | top_seed DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ tmp, unused2, see, overlap, unused3, src_bak, lstride, luma_bak @@ -2497,22 +2713,22 @@ mov lstrideq, r10mp %endif mov lumaq, r9mp lea src_bakq, [srcq+wq] - lea luma_bakq, [lumaq+wq*2] + lea luma_bakq, [lumaq+wq*(1+%2)] neg wq sub r0mp, srcq %if ARCH_X86_32 mov r1m, src_bakq mov r11m, luma_bakq mov r4m, wq DEFINE_ARGS tmp, src, scaling, see, unused1, picptr, unused2 %else mov r11mp, src_bakq - mov r10mp, strideq + mov r12mp, strideq %endif %%loop_x_v_overlap: %if ARCH_X86_32 mov seed, r3m @@ -2547,13 +2763,13 @@ %endif ror offyd, 8 ror offxd, 12 and offyd, 0xf000f and offxd, 0xf000f - imul offyd, 82 + imul offyd, 164>>%3 ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy - lea offyq, [offyq+offxq+0x10001*498+16*82] + lea offyq, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] %if ARCH_X86_32 DEFINE_ARGS tmp, src, scaling, offxy, h, picptr, top_offxy %else DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ @@ -2566,16 +2782,24 @@ mov [rsp+8*mmsize+1*gprsize], top_offxyd DEFINE_ARGS luma, src, scaling, offxy, h, picptr, grain_lut %endif +%%loop_x_odd_v_overlap: mov hd, r7m mov grain_lutq, grain_lutmp +%if ARCH_X86_32 + mov r5, r5m + mova m1, [base+pb_27_17] +%else + mova m1, [pb_27_17] +%endif %%loop_y_v_overlap: %if ARCH_X86_32 mov lumaq, r9mp %endif +%if %2 mova m4, [lumaq+ 0] mova m6, [lumaq+16] mova m0, [srcq] %if ARCH_X86_32 add lumaq, r10mp @@ -2589,13 +2813,24 @@ pxor m2, m2 pmaddubsw m4, m7 pmaddubsw m6, m7 pavgw m4, m2 pavgw m6, m2 +%else + mova m4, [lumaq] + mova m0, [srcq] +%if ARCH_X86_32 + add lumaq, r10mp + mov r9mp, lumaq +%endif + pxor m2, m2 +%endif %if %1 +%if %2 packuswb m4, m6 ; luma +%endif punpckhbw m6, m4, m0 punpcklbw m4, m0 ; { luma, chroma } pmaddubsw m6, m14 pmaddubsw m4, m14 psraw m6, 6 @@ -2603,81 +2838,103 @@ paddw m6, m15 paddw m4, m15 packuswb m4, m6 ; pack+unpack = clip punpckhbw m6, m4, m2 punpcklbw m4, m2 +%elif %2 == 0 + punpckhbw m6, m4, m2 + punpcklbw m4, m2 %endif ; scaling[luma_src] %if ARCH_X86_32 vpgatherdw m7, m4, scalingq, r0, r5 vpgatherdw m5, m6, scalingq, r0, r5 %else vpgatherdw m7, m4, scalingq, r12, r2 vpgatherdw m5, m6, scalingq, r12, r2 %endif - pcmpeqw m1, m1 - psrlw m1, 8 - pand m7, m1 - pand m5, m1 + pcmpeqw m4, m4 + psrlw m4, 8 + pand m7, m4 + pand m5, m4 ; grain = grain_lut[offy+y][offx+x] movu m3, [grain_lutq+offxyq] %if ARCH_X86_32 mov r0, [rsp+8*mmsize+1*gprsize] movu m4, [grain_lutq+r0] %else movu m4, [grain_lutq+top_offxyq] %endif - punpckhbw m1, m4, m3 + punpckhbw m6, m4, m3 punpcklbw m4, m3 - pmaddubsw m2, m9, m1 +%if %3 + pmaddubsw m2, m9, m6 pmaddubsw m3, m9, m4 +%else + pmaddubsw m2, m1, m6 + pmaddubsw m3, m1, m4 +%endif pmulhrsw m2, m8 pmulhrsw m3, m8 packsswb m3, m2 - pxor m1, m1 - pcmpgtb m1, m3 - punpcklbw m2, m3, m1 - punpckhbw m3, m1 + pxor m6, m6 + pcmpgtb m6, m3 + punpcklbw m2, m3, m6 + punpckhbw m3, m6 ; noise = round2(scaling[luma_src] * grain, scaling_shift) pmullw m2, m7 pmullw m3, m5 pmulhrsw m2, m11 pmulhrsw m3, m11 ; unpack chroma_source pxor m4, m4 - punpckhbw m1, m0, m4 + punpckhbw m6, m0, m4 punpcklbw m0, m4 ; m0-1: src as word %if ARCH_X86_32 DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut %endif ; dst = clip_pixel(src, noise) paddw m0, m2 - paddw m1, m3 + paddw m6, m3 pmaxsw m0, m13 - pmaxsw m1, m13 + pmaxsw m6, m13 pminsw m0, m12 - pminsw m1, m12 - packuswb m0, m1 + pminsw m6, m12 + packuswb m0, m6 movifnidn dstq, dstmp mova [dstq+srcq], m0 dec hw je %%end_y_v_overlap %if ARCH_X86_32 add srcq, r2mp ; lumaq has already been incremented above %else - add srcq, r10mp + add srcq, r12mp +%if %3 lea lumaq, [lumaq+lstrideq*2] +%else + add lumaq, lstrideq +%endif %endif add grain_lutq, 82 +%if %3 == 0 + btc hd, 16 +%if ARCH_X86_32 + mov r5, r5m + mova m1, [base+pb_17_27] +%else + mova m1, [pb_17_27] +%endif + jnc %%loop_y_v_overlap +%endif jmp %%loop_y %%end_y_v_overlap: %if ARCH_X86_32 DEFINE_ARGS luma, src, scaling, offxy, w, picptr, grain_lut @@ -2690,41 +2947,61 @@ mov srcq, r1mp mov lumaq, r11mp %else mov srcq, r11mp %endif - lea lumaq, [luma_bakq+wq*2] + lea lumaq, [luma_bakq+wq*(1+%2)] add srcq, wq %if ARCH_X86_32 mov r4m, wq mov r9m, lumaq %endif +%if %2 ; since fg_dataq.overlap is guaranteed to be set, we never jump ; back to .loop_x_v_overlap, and instead always fall-through to ; h+v overlap +%else +%if ARCH_X86_32 + add dword [rsp+8*mmsize+1*gprsize], 16 +%else + add top_offxyd, 16 +%endif + add offxyd, 16 + btc dword r8m, 2 + jnc %%loop_x_odd_v_overlap +%endif %%loop_x_hv_overlap: %if ARCH_X86_32 DEFINE_ARGS tmp, src, scaling, offxy, w, picptr, unused mov r6, [rsp+8*mmsize+1*gprsize] +%if %2 lea r0, [r3d+16] add r6, 16 mov [rsp+8*mmsize+0*gprsize], r0 ; left_offxy +%else + mov [rsp+8*mmsize+0*gprsize], r3 ; left_offxy +%endif mov [rsp+8*mmsize+2*gprsize], r6 ; topleft_offxy DEFINE_ARGS tmp, src, scaling, see, w, picptr, unused mov seed, r3m xor tmpd, tmpd %else DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ tmp, offxy, see, left_offxy, top_offxy, topleft_offxy, lstride +%if %2 lea topleft_offxyq, [top_offxyq+16] lea left_offxyq, [offxyq+16] +%else + mov topleft_offxyq, top_offxyq + mov left_offxyq, offxyq +%endif ; we assume from the block above that bits 8-15 of tmpd are zero'ed %endif mov r6d, seed or seed, 0xeff4eff4 @@ -2754,13 +3031,13 @@ %endif ror offyd, 8 ror offxd, 12 and offyd, 0xf000f and offxd, 0xf000f - imul offyd, 82 + imul offyd, 164>>%3 ; offxy=offy*stride+offx, (cur_offxy << 16) | top_offxy - lea offyq, [offyq+offxq+0x10001*498+16*82] + lea offyq, [offyq+offxq*(2-%2)+0x10001*((3+(6>>%3))*82+3+(6>>%2))+(32>>%3)*82] %if ARCH_X86_32 DEFINE_ARGS top_offxy, src, scaling, offxy, h, picptr, grain_lut %else DEFINE_ARGS dst, src, stride, luma, w, scaling, grain_lut, \ @@ -2773,17 +3050,24 @@ mov [rsp+8*mmsize+1*gprsize], top_offxyd %endif mov hd, r7m mov grain_lutq, grain_lutmp +%if ARCH_X86_32 + mov r5, r5m + mova m3, [base+pb_27_17] +%else + mova m3, [pb_27_17] +%endif %%loop_y_hv_overlap: ; src %if ARCH_X86_32 DEFINE_ARGS luma, src, scaling, offxy, w, picptr, grain_lut mov lumaq, r9mp %endif +%if %2 mova m4, [lumaq+ 0] mova m6, [lumaq+16] mova m0, [srcq] %if ARCH_X86_32 add lumaq, r10mp @@ -2797,13 +3081,24 @@ pxor m2, m2 pmaddubsw m4, m7 pmaddubsw m6, m7 pavgw m4, m2 pavgw m6, m2 +%else + mova m4, [lumaq] + mova m0, [srcq] +%if ARCH_X86_32 + add lumaq, r10mp + mov r9mp, lumaq +%endif + pxor m2, m2 +%endif %if %1 +%if %2 packuswb m4, m6 ; luma +%endif punpckhbw m6, m4, m0 punpcklbw m4, m0 ; { luma, chroma } pmaddubsw m6, m14 pmaddubsw m4, m14 psraw m6, 6 @@ -2811,20 +3106,28 @@ paddw m6, m15 paddw m4, m15 packuswb m4, m6 ; pack+unpack = clip punpckhbw m6, m4, m2 punpcklbw m4, m2 +%elif %2 == 0 + punpckhbw m6, m4, m2 + punpcklbw m4, m2 %endif ; scaling[src] %if ARCH_X86_32 vpgatherdw m7, m4, scalingq, r0, r5 vpgatherdw m5, m6, scalingq, r0, r5 %else movd m1, [grain_lutq+topleft_offxyq] +%if %3 vpgatherdw m7, m4, scalingq, r2, r12 vpgatherdw m5, m6, scalingq, r2, r12 +%else + vpgatherdw m7, m4, scalingq, r2, r13 + vpgatherdw m5, m6, scalingq, r2, r13 +%endif %endif pcmpeqw m2, m2 psrlw m2, 8 pand m7, m2 pand m5, m2 @@ -2834,37 +3137,46 @@ mov r0, [rsp+8*mmsize+2*gprsize] ; topleft_offxy mov r5, [rsp+8*mmsize+1*gprsize] ; top_offxy movd m1, [grain_lutq+r0] mov r0, [rsp+8*mmsize+0*gprsize] ; left_offxy %endif - movu m3, [grain_lutq+offxyq] + movu m2, [grain_lutq+offxyq] %if ARCH_X86_32 movu m6, [grain_lutq+r5] movd m4, [grain_lutq+r0] %else movu m6, [grain_lutq+top_offxyq] movd m4, [grain_lutq+left_offxyq] %endif ; do h interpolation first (so top | top/left -> top, left | cur -> cur) punpcklbw m1, m6 - punpcklbw m4, m3 + punpcklbw m4, m2 +%if %2 punpcklwd m4, m1 +%else + punpckldq m4, m1 +%endif pmaddubsw m1, m9, m4 pmulhrsw m1, m8 packsswb m1, m1 - pandn m4, m10, m3 - pandn m3, m10, m6 - psrldq m6, m1, 1 + pandn m4, m10, m2 + pandn m2, m10, m6 + psrldq m6, m1, 2-%2 pand m1, m10 pand m6, m10 por m4, m1 - por m3, m6 + por m2, m6 ; followed by v interpolation (top | cur -> cur) - punpckhbw m1, m3, m4 - punpcklbw m3, m4 + punpckhbw m1, m2, m4 + punpcklbw m2, m4 +%if %3 pmaddubsw m4, m9, m1 - pmaddubsw m1, m9, m3 + pmaddubsw m1, m9, m2 +%else + pmaddubsw m4, m3, m1 + pmaddubsw m1, m3, m2 +%endif pmulhrsw m4, m8 pmulhrsw m1, m8 packsswb m1, m4 pxor m4, m4 pcmpgtb m4, m1 @@ -2881,34 +3193,58 @@ DEFINE_ARGS dst, src, scaling, offxy, h, picptr, grain_lut %endif ; unpack chroma source pxor m4, m4 - punpckhbw m3, m0, m4 + punpckhbw m5, m0, m4 punpcklbw m0, m4 ; m0-1: src as word ; dst = clip_pixel(src, noise) paddw m0, m2 - paddw m3, m1 + paddw m5, m1 pmaxsw m0, m13 - pmaxsw m3, m13 + pmaxsw m5, m13 pminsw m0, m12 - pminsw m3, m12 - packuswb m0, m3 + pminsw m5, m12 + packuswb m0, m5 movifnidn dstq, dstmp mova [dstq+srcq], m0 %if ARCH_X86_32 add srcq, r2mp ; lumaq has been adjusted above already %else - add srcq, r10mp - lea lumaq, [lumaq+lstrideq*2] + add srcq, r12mp +%if %3 + lea lumaq, [lumaq+lstrideq*(1+%2)] +%else + add lumaq, r10mp +%endif %endif add grain_lutq, 82 dec hw +%if %3 jg %%loop_y_h_overlap +%else + jle %%end_y_hv_overlap +%if ARCH_X86_32 + mov r5, r5m + mova m3, [base+pb_17_27] +%else + mova m3, [pb_17_27] +%endif + btc hd, 16 + jnc %%loop_y_hv_overlap +%if ARCH_X86_64 + mov lstrideq, r10mp +%endif + jmp %%loop_y_h_overlap +%%end_y_hv_overlap: +%if ARCH_X86_64 + mov lstrideq, r10mp +%endif +%endif %if ARCH_X86_32 DEFINE_ARGS luma, src, scaling, offxy, w, picptr, grain_lut mov wq, r4m @@ -2919,20 +3255,46 @@ mov srcq, r1mp mov lumaq, r11mp %else mov srcq, r11mp %endif - lea lumaq, [luma_bakq+wq*2] + lea lumaq, [luma_bakq+wq*(1+%2)] add srcq, wq %if ARCH_X86_32 mov r4m, wq mov r9m, lumaq %endif +%if %2 jmp %%loop_x_hv_overlap +%else +%if ARCH_X86_32 + add dword [rsp+8*mmsize+1*gprsize], 16 +%else + add top_offxyd, 16 +%endif + add offxyd, 16 + xor dword r8m, 4 + jmp %%loop_x_odd_v_overlap +%endif %%end_hv: RET %endmacro - FGUV_32x32xN_LOOP 1 + %%FGUV_32x32xN_LOOP 1, %2, %3 .csfl: - FGUV_32x32xN_LOOP 0 + %%FGUV_32x32xN_LOOP 0, %2, %3 +%endmacro + +FGUV_FN 420, 1, 1 + +%if STACK_ALIGNMENT < mmsize +DECLARE_ARG 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 +%endif + +FGUV_FN 422, 1, 0 + +%if STACK_ALIGNMENT < mmsize +DECLARE_ARG 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 +%endif + +FGUV_FN 444, 0, 0 diff --git a/third_party/dav1d/src/x86/ipred.asm b/third_party/dav1d/src/x86/ipred.asm --- a/third_party/dav1d/src/x86/ipred.asm +++ b/third_party/dav1d/src/x86/ipred.asm @@ -98,10 +98,12 @@ db 2, 3, 2, 3, 6, 7, 6, 7, 10, 11, 10, 11, 14, 15, 14, 15 ipred_h_shuf: db 7, 7, 7, 7, 3, 3, 3, 3, 5, 5, 5, 5, 1, 1, 1, 1 db 6, 6, 6, 6, 2, 2, 2, 2, 4, 4, 4, 4; 0, 0, 0, 0 pw_64: times 2 dw 64 +cfl_ac_444_w16_pad1_shuffle: db 0, -1, 1, -1, 2, -1, 3, -1, 4, -1, 5, -1, 6, -1 + times 9 db 7, -1 cfl_ac_w16_pad_shuffle: ; w=16, w_pad=1 db 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 ; w=8, w_pad=1 as well as second half of previous one cfl_ac_w8_pad1_shuffle: db 0, 1, 2, 3, 4, 5 times 5 db 6, 7 @@ -164,10 +166,11 @@ JMP_TABLE ipred_cfl, avx2, h4, h8, h16, h32, w4, w8, w16, w32, \ s4-8*4, s8-8*4, s16-8*4, s32-8*4 JMP_TABLE ipred_cfl_left, avx2, h4, h8, h16, h32 JMP_TABLE ipred_cfl_ac_420, avx2, w16_pad1, w16_pad2, w16_pad3 JMP_TABLE ipred_cfl_ac_422, avx2, w16_pad1, w16_pad2, w16_pad3 +JMP_TABLE ipred_cfl_ac_444, avx2, w32_pad1, w32_pad2, w32_pad3, w4, w8, w16, w32 JMP_TABLE pal_pred, avx2, w4, w8, w16, w32, w64 cextern dr_intra_derivative cextern filter_intra_taps @@ -1407,22 +1410,23 @@ pcmpeqb m1, m0, [base+z_filter_wh] pand m1, m2 mova xm2, [r3+angleq*8] ; upper ymm half zero in both cases pcmpgtb m1, m2 pmovmskb r5d, m1 - popcnt r5d, r5d ; sets ZF which can be used by caller ret .w4_no_upsample: %assign stack_offset org_stack_offset ALLOC_STACK -16, 11 mov maxbased, 7 test angled, 0x400 ; !enable_intra_edge_filter jnz .w4_main lea maxbased, [hq+3] call .filter_strength mov maxbased, 7 + test r5d, r5d jz .w4_main ; filter_strength == 0 + popcnt r5d, r5d vpbroadcastd m7, [base+pb_8] vbroadcasti128 m2, [tlq-1] pminub m1, m7, [base+z_filter_s] vpbroadcastd m8, [base+z_filter_k-4+r5*4+12*0] pminub m7, [base+z_filter_s+8] @@ -1591,11 +1595,13 @@ ALLOC_STACK -32, 10 lea maxbased, [hq+7] test angled, 0x400 jnz .w8_no_intra_edge_filter call .filter_strength + test r5d, r5d jz .w8_main ; filter_strength == 0 + popcnt r5d, r5d movu xm2, [tlq] pminub xm1, xm0, [base+z_filter_s+14] vinserti128 m2, [tlq-1], 1 vinserti128 m1, [base+z_filter_s+ 0], 1 vpbroadcastd m7, [base+z_filter_k-4+r5*4+12*0] @@ -1693,11 +1699,13 @@ ALLOC_STACK -64, 12 lea maxbased, [hq+15] test angled, 0x400 jnz .w16_no_intra_edge_filter call .filter_strength + test r5d, r5d jz .w16_main ; filter_strength == 0 + popcnt r5d, r5d vpbroadcastd m1, [base+pb_12] vbroadcasti128 m6, [base+z_filter_s+8] vinserti128 m2, m6, [base+z_filter_s], 0 vinserti128 m6, [base+z_filter_s+16], 1 mova xm10, [tlq-1] @@ -2200,11 +2208,10 @@ pcmpeqb m8, [base+z_filter_wh] mova xm9, [r9+r3*8] pand m0, m8, m7 pcmpgtb m0, m9 pmovmskb r3d, m0 - popcnt r3d, r3d ret ALIGN function_align .upsample_above: ; w4/w8 pshufb xm2, xm1, [base+z_upsample1-2] pminub xm3, [base+z_filter_s+4] @@ -2250,11 +2257,13 @@ ret .w4_no_upsample_above: lea r3d, [hq+3] sub angled, 1112 ; angle - 90 call .filter_strength + test r3d, r3d jz .w4_no_filter_above + popcnt r3d, r3d vpbroadcastd xm2, [base+pb_4] pminub xm2, [base+z_filter_s] vpbroadcastd xm0, [base+z_filter_k-4+r3*4+12*0] vpbroadcastd xm4, [base+z_filter_k-4+r3*4+12*1] pshufb xm3, xm1, xm2 ; 00 01 12 23 @@ -2285,13 +2294,14 @@ vpbroadcastd xm0, [base+pb_90] psubb xm0, xm7 ; 180 - angle pand xm0, xm8 ; reuse from previous filter_strength call pcmpgtb xm0, xm9 pmovmskb r3d, xm0 +.w4_filter_left: + test r3d, r3d + jz .w4_main popcnt r3d, r3d -.w4_filter_left: - jz .w4_main mov r5d, 10 cmp hd, 16 movu xm2, [rsp+49] vinserti128 m2, [rsp+43], 1 cmovs r5d, hd @@ -2438,11 +2448,13 @@ jmp .w8_filter_left .w8_no_upsample_above: lea r3d, [hq+7] sub angled, 90 ; angle - 90 call .filter_strength + test r3d, r3d jz .w8_no_filter_above + popcnt r3d, r3d vpbroadcastd xm3, [base+pb_8] pminub xm3, [base+z_filter_s+8] vpbroadcastd xm0, [base+z_filter_k-4+r3*4+12*0] vpbroadcastd xm4, [base+z_filter_k-4+r3*4+12*1] pshufb xm2, xm1, [base+z_filter_s] ; 00 01 12 23 34 45 56 67 @@ -2471,13 +2483,14 @@ vpbroadcastd m0, [base+pb_90] psubb m0, m7 pand m0, m8 pcmpgtb m0, m9 pmovmskb r3d, m0 +.w8_filter_left: + test r3d, r3d + jz .w8_main popcnt r3d, r3d -.w8_filter_left: - jz .w8_main vpbroadcastd m7, [base+z_filter_k-4+r3*4+12*0] vpbroadcastd m8, [base+z_filter_k-4+r3*4+12*1] vpbroadcastd m9, [base+z_filter_k-4+r3*4+12*2] cmp hd, 32 jne .w8_filter_left_h16 @@ -2645,11 +2658,13 @@ test angled, 0x400 jnz .w16_main lea r3d, [hq+15] sub angled, 90 call .filter_strength + test r3d, r3d jz .w16_no_filter_above + popcnt r3d, r3d vbroadcasti128 m6, [tlq+1] mova xm2, [base+z_filter_s] vinserti128 m2, [base+z_filter_s+14], 1 ; 00 01 12 23 34 45 56 67 67 78 89 9a ab bc cd de movu xm3, [base+z_filter_s+8] vinserti128 m3, [base+z_filter_s+22], 1 ; 34 45 56 67 78 89 9a ab ab bc cd de ef ff ff ff @@ -2678,12 +2693,13 @@ vpbroadcastd m0, [base+pb_90] psubb m0, m7 pand m0, m8 pcmpgtb m0, m9 pmovmskb r3d, m0 + test r3d, r3d + jz .w16_main popcnt r3d, r3d - jz .w16_main vpbroadcastd m7, [base+z_filter_k-4+r3*4+12*0] vpbroadcastd m8, [base+z_filter_k-4+r3*4+12*1] vpbroadcastd m9, [base+z_filter_k-4+r3*4+12*2] .w16_filter_left: movd xm6, r7m ; max_height @@ -3081,22 +3097,23 @@ pcmpeqb m1, m0, [base+z_filter_wh] pand m1, m2 mova xm2, [r4+angleq*8] pcmpgtb m1, m2 pmovmskb r5d, m1 - popcnt r5d, r5d ret .h4_no_upsample: %assign stack_offset org_stack_offset ALLOC_STACK -16, 12 mov maxbased, 7 test angled, 0x400 ; !enable_intra_edge_filter jnz .h4_main lea maxbased, [wq+3] call .filter_strength mov maxbased, 7 + test r5d, r5d jz .h4_main ; filter_strength == 0 + popcnt r5d, r5d vpbroadcastd m7, [base+pb_7] vbroadcasti128 m2, [tlq-14] pmaxub m1, m7, [base+z_filter_s-4] vpbroadcastd m8, [base+z_filter_k-4+r5*4+12*0] pmaxub m7, [base+z_filter_s+4] @@ -3283,11 +3300,13 @@ ALLOC_STACK -32, 10 lea maxbased, [wq+7] test angled, 0x400 jnz .h8_no_intra_edge_filter call .filter_strength + test r5d, r5d jz .h8_main ; filter_strength == 0 + popcnt r5d, r5d vpbroadcastd xm6, [base+pb_15] pcmpeqb xm1, xm1 psubusb xm6, xm0 psubb xm6, xm1 ; w == 4 ? 5 : 1 movu xm2, [tlq-16] @@ -3439,11 +3458,13 @@ ALLOC_STACK -64, 12 lea maxbased, [wq+15] test angled, 0x400 jnz .h16_no_intra_edge_filter call .filter_strength + test r5d, r5d jz .h16_main ; filter_strength == 0 + popcnt r5d, r5d vpbroadcastd m11, [base+pb_27] vpbroadcastd m1, [base+pb_1] vbroadcasti128 m6, [base+z_filter_s+12] vinserti128 m2, m6, [base+z_filter_s+4], 0 vinserti128 m6, [base+z_filter_s+20], 1 @@ -5052,10 +5073,240 @@ add ac_bakq, 32 sub szd, 16 jg .sub_loop RET +cglobal ipred_cfl_ac_444, 4, 9, 6, ac, y, stride, wpad, hpad, w, h, sz, ac_bak + movifnidn hpadd, hpadm + movifnidn wd, wm + mov hd, hm + mov szd, wd + imul szd, hd + shl hpadd, 2 + sub hd, hpadd + pxor m4, m4 + vpbroadcastd m5, [pw_1] + tzcnt r8d, wd + lea r5, [ipred_cfl_ac_444_avx2_table] + movsxd r8, [r5+r8*4+12] + add r5, r8 + + DEFINE_ARGS ac, y, stride, wpad, hpad, stride3, h, sz, ac_bak + mov ac_bakq, acq + jmp r5 + +.w4: + lea stride3q, [strideq*3] + pxor xm2, xm2 +.w4_loop: + movd xm1, [yq] + movd xm0, [yq+strideq*2] + pinsrd xm1, [yq+strideq], 1 + pinsrd xm0, [yq+stride3q], 1 + punpcklbw xm1, xm2 + punpcklbw xm0, xm2 + psllw xm1, 3 + psllw xm0, 3 + mova [acq], xm1 + mova [acq+16], xm0 + paddw xm1, xm0 + paddw xm4, xm1 + lea yq, [yq+strideq*4] + add acq, 32 + sub hd, 4 + jg .w4_loop + test hpadd, hpadd + jz .calc_avg_mul + pshufd xm0, xm0, q3232 + paddw xm1, xm0, xm0 +.w4_hpad_loop: + mova [acq], xm0 + mova [acq+16], xm0 + paddw xm4, xm1 + add acq, 32 + sub hpadd, 4 + jg .w4_hpad_loop + jmp .calc_avg_mul + +.w8: + lea stride3q, [strideq*3] + pxor m2, m2 +.w8_loop: + movq xm1, [yq] + movq xm0, [yq+strideq*2] + vinserti128 m1, [yq+strideq], 1 + vinserti128 m0, [yq+stride3q], 1 + punpcklbw m1, m2 + punpcklbw m0, m2 + psllw m1, 3 + psllw m0, 3 + mova [acq], m1 + mova [acq+32], m0 + paddw m1, m0 + paddw m4, m1 + lea yq, [yq+strideq*4] + add acq, 64 + sub hd, 4 + jg .w8_loop + test hpadd, hpadd + jz .calc_avg_mul + vpermq m0, m0, q3232 + paddw m1, m0, m0 +.w8_hpad_loop: + mova [acq], m0 + mova [acq+32], m0 + paddw m4, m1 + add acq, 64 + sub hpadd, 4 + jg .w8_hpad_loop + jmp .calc_avg_mul + +.w16: + test wpadd, wpadd + jnz .w16_wpad +.w16_loop: + pmovzxbw m1, [yq] + pmovzxbw m0, [yq+strideq] + psllw m1, 3 + psllw m0, 3 + mova [acq], m1 + mova [acq+32], m0 + paddw m1, m0 + pmaddwd m1, m5 + paddd m4, m1 + lea yq, [yq+strideq*2] + add acq, 64 + sub hd, 2 + jg .w16_loop + test hpadd, hpadd + jz .calc_avg + jmp .w16_hpad +.w16_wpad: + mova m3, [cfl_ac_444_w16_pad1_shuffle] +.w16_wpad_loop: + vpbroadcastq m1, [yq] + vpbroadcastq m0, [yq+strideq] + pshufb m1, m3 + pshufb m0, m3 + psllw m1, 3 + psllw m0, 3 + mova [acq], m1 + mova [acq+32], m0 + paddw m1, m0 + pmaddwd m1, m5 + paddd m4, m1 + lea yq, [yq+strideq*2] + add acq, 64 + sub hd, 2 + jg .w16_wpad_loop + test hpadd, hpadd + jz .calc_avg +.w16_hpad: + paddw m1, m0, m0 + pmaddwd m1, m5 +.w16_hpad_loop: + mova [acq], m0 + mova [acq+32], m0 + paddd m4, m1 + add acq, 64 + sub hpadd, 2 + jg .w16_hpad_loop + jmp .calc_avg + +.w32: + test wpadd, wpadd + jnz .w32_wpad +.w32_loop: + pmovzxbw m1, [yq] + pmovzxbw m0, [yq+16] + psllw m1, 3 + psllw m0, 3 + mova [acq], m1 + mova [acq+32], m0 + paddw m2, m1, m0 + pmaddwd m2, m5 + paddd m4, m2 + add yq, strideq + add acq, 64 + dec hd + jg .w32_loop + test hpadd, hpadd + jz .calc_avg + jmp .w32_hpad_loop +.w32_wpad: + DEFINE_ARGS ac, y, stride, wpad, hpad, iptr, h, sz, ac_bak + lea iptrq, [ipred_cfl_ac_444_avx2_table] + add wpadd, wpadd + mova m3, [iptrq+cfl_ac_444_w16_pad1_shuffle-ipred_cfl_ac_444_avx2_table] + movsxd wpadq, [iptrq+wpadq+4] + add iptrq, wpadq + jmp iptrq +.w32_pad3: + vpbroadcastq m1, [yq] + pshufb m1, m3 + vpermq m0, m1, q3232 + jmp .w32_wpad_end +.w32_pad2: + pmovzxbw m1, [yq] + pshufhw m0, m1, q3333 + vpermq m0, m0, q3333 + jmp .w32_wpad_end +.w32_pad1: + pmovzxbw m1, [yq] + vpbroadcastq m0, [yq+16] + pshufb m0, m3 + ; fall-through +.w32_wpad_end: + psllw m1, 3 + psllw m0, 3 + mova [acq], m1 + mova [acq+32], m0 + paddw m2, m1, m0 + pmaddwd m2, m5 + paddd m4, m2 + add yq, strideq + add acq, 64 + dec hd + jz .w32_wpad_done + jmp iptrq +.w32_wpad_done: + test hpadd, hpadd + jz .calc_avg +.w32_hpad_loop: + mova [acq], m1 + mova [acq+32], m0 + paddd m4, m2 + add acq, 64 + dec hpadd + jg .w32_hpad_loop + jmp .calc_avg + +.calc_avg_mul: + pmaddwd m4, m5 +.calc_avg: + vextracti128 xm1, m4, 1 + tzcnt r1d, szd + paddd xm0, xm4, xm1 + movd xm2, r1d + movd xm3, szd + punpckhqdq xm1, xm0, xm0 + paddd xm0, xm1 + psrad xm3, 1 + psrlq xm1, xm0, 32 + paddd xm0, xm3 + paddd xm0, xm1 + psrad xm0, xm2 + vpbroadcastw m0, xm0 +.sub_loop: + mova m1, [ac_bakq] + psubw m1, m0 + mova [ac_bakq], m1 + add ac_bakq, 32 + sub szd, 16 + jg .sub_loop + RET + cglobal pal_pred, 4, 6, 5, dst, stride, pal, idx, w, h vbroadcasti128 m4, [palq] lea r2, [pal_pred_avx2_table] tzcnt wd, wm movifnidn hd, hm diff --git a/third_party/dav1d/src/x86/ipred_init_tmpl.c b/third_party/dav1d/src/x86/ipred_init_tmpl.c --- a/third_party/dav1d/src/x86/ipred_init_tmpl.c +++ b/third_party/dav1d/src/x86/ipred_init_tmpl.c @@ -48,10 +48,11 @@ decl_cfl_pred_fn(dav1d_ipred_cfl_top_avx2); decl_cfl_pred_fn(dav1d_ipred_cfl_left_avx2); decl_cfl_ac_fn(dav1d_ipred_cfl_ac_420_avx2); decl_cfl_ac_fn(dav1d_ipred_cfl_ac_422_avx2); +decl_cfl_ac_fn(dav1d_ipred_cfl_ac_444_avx2); decl_pal_pred_fn(dav1d_pal_pred_avx2); decl_angular_ipred_fn(dav1d_ipred_dc_ssse3); decl_angular_ipred_fn(dav1d_ipred_dc_128_ssse3); @@ -129,9 +130,10 @@ c->cfl_pred[TOP_DC_PRED] = dav1d_ipred_cfl_top_avx2; c->cfl_pred[LEFT_DC_PRED] = dav1d_ipred_cfl_left_avx2; c->cfl_ac[DAV1D_PIXEL_LAYOUT_I420 - 1] = dav1d_ipred_cfl_ac_420_avx2; c->cfl_ac[DAV1D_PIXEL_LAYOUT_I422 - 1] = dav1d_ipred_cfl_ac_422_avx2; + c->cfl_ac[DAV1D_PIXEL_LAYOUT_I444 - 1] = dav1d_ipred_cfl_ac_444_avx2; c->pal_pred = dav1d_pal_pred_avx2; #endif } diff --git a/third_party/dav1d/src/x86/itx.asm b/third_party/dav1d/src/x86/itx.asm --- a/third_party/dav1d/src/x86/itx.asm +++ b/third_party/dav1d/src/x86/itx.asm @@ -25,19 +25,14 @@ %include "ext/x86/x86inc.asm" %if ARCH_X86_64 -SECTION_RODATA 32 +SECTION_RODATA 16 ; Note: The order of (at least some of) those constants matter! -iadst4_dconly2a: dw 10568, 10568, 10568, 10568, 19856, 19856, 19856, 19856 -iadst4_dconly2b: dw 26752, 26752, 26752, 26752, 30424, 30424, 30424, 30424 -iadst4_dconly1a: dw 10568, 19856, 26752, 30424 -iadst4_dconly1b: dw 30424, 26752, 19856, 10568 - deint_shuf: db 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15 %macro COEF_PAIR 2 pw_%1_%2: dw %1, %2 pw_m%2_%1: dw -%2, %1 @@ -130,11 +125,11 @@ ; Code size reduction trickery: Intead of using rip-relative loads with ; mandatory 4-byte offsets everywhere, we can set up a base pointer with a ; single rip-relative lea and then address things relative from that with ; 1-byte offsets as long as data is within +-128 bytes of the base pointer. -%define o_base iadst4_dconly2a + 128 +%define o_base deint_shuf + 128 %define o(x) (rax - (o_base) + (x)) %macro REPX 2-* %xdefine %%f(x) %1 %rep %0 - 1 @@ -178,20 +173,20 @@ %macro ITX_MUL4X_PACK 9-10 0 ; dst/src, tmp[1-3], rnd, coef[1-4], flags %if %10 & 1 vpbroadcastd m%3, [o(pw_%8_%9)] vpbroadcastd m%4, [o(pw_m%9_%8)] vpbroadcastd xm%2, [o(pw_%6_%7)] - vpblendd m%2, m%2, m%3, 0xf0 + vpblendd m%2, m%3, 0xf0 vpbroadcastd xm%3, [o(pw_m%7_%6)] %else vpbroadcastd m%3, [o(pw_m%9_%8)] vpbroadcastd m%4, [o(pw_%8_%9)] vpbroadcastd xm%2, [o(pw_m%7_%6)] - vpblendd m%2, m%2, m%3, 0xf0 + vpblendd m%2, m%3, 0xf0 vpbroadcastd xm%3, [o(pw_%6_%7)] %endif - vpblendd m%3, m%3, m%4, 0xf0 + vpblendd m%3, m%4, 0xf0 ITX_MUL2X_PACK %1, %4, _, %5, %2, %3, (4|%10) %endmacro ; dst1 = (src1 * coef1 - src2 * coef2 + rnd) >> 12 ; dst2 = (src1 * coef2 + src2 * coef1 + rnd) >> 12 @@ -358,83 +353,41 @@ punpckhwd m0, m1 punpcklwd m3, m1, m2 punpckhdq m1, m0, m3 punpckldq m0, m3 IWHT4_1D_PACKED - vpblendd m0, m0, m2, 0x03 + vpblendd m0, m2, 0x03 ITX4_END 3, 0, 2, 1, 0 -%macro INV_TXFM_FN 4 ; type1, type2, fast_thresh, size -cglobal inv_txfm_add_%1_%2_%4, 4, 5, 0, dst, stride, c, eob, tx2 - %undef cmp - %define %%p1 m(i%1_%4_internal) +%macro INV_TXFM_FN 3 ; type1, type2, size +cglobal inv_txfm_add_%1_%2_%3, 4, 5, 0, dst, stride, c, eob, tx2 + %define %%p1 m(i%1_%3_internal) lea rax, [o_base] ; Jump to the 1st txfm function if we're not taking the fast path, which ; in turn performs an indirect jump to the 2nd txfm function. - lea tx2q, [m(i%2_%4_internal).pass2] -%if %3 > 0 - cmp eobd, %3 - jg %%p1 -%elif %3 == 0 + lea tx2q, [m(i%2_%3_internal).pass2] +%ifidn %1_%2, dct_dct test eobd, eobd jnz %%p1 %else ; jump to the 1st txfm function unless it's located directly after this times ((%%end - %%p1) >> 31) & 1 jmp %%p1 ALIGN function_align %%end: %endif %endmacro -%macro INV_TXFM_4X4_FN 2-3 -1 ; type1, type2, fast_thresh - INV_TXFM_FN %1, %2, %3, 4x4 -%ifidn %1_%2, dct_identity - vpbroadcastd m0, [o(pw_2896x8)] - pmulhrsw m0, [cq] - vpbroadcastd m1, [o(pw_1697x8)] - pmulhrsw m1, m0 - paddsw m0, m1 - punpcklwd m0, m0 - punpckhdq m1, m0, m0 - punpckldq m0, m0 - jmp m(iadst_4x4_internal).end -%elifidn %1_%2, identity_dct - mova m0, [cq+16*0] - packusdw m0, [cq+16*1] - vpbroadcastd m1, [o(pw_1697x8)] - vpbroadcastd m2, [o(pw_2896x8)] - packusdw m0, m0 - pmulhrsw m1, m0 - paddsw m0, m1 - pmulhrsw m0, m2 - mova m1, m0 - jmp m(iadst_4x4_internal).end -%elif %3 >= 0 +%macro INV_TXFM_4X4_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 4x4 +%ifidn %1_%2, dct_dct vpbroadcastw m0, [cq] -%ifidn %1, dct vpbroadcastd m1, [o(pw_2896x8)] pmulhrsw m0, m1 -%elifidn %1, adst - movddup m1, [o(iadst4_dconly1a)] - pmulhrsw m0, m1 -%elifidn %1, flipadst - movddup m1, [o(iadst4_dconly1b)] - pmulhrsw m0, m1 -%endif mov [cq], eobd ; 0 -%ifidn %2, dct -%ifnidn %1, dct - vpbroadcastd m1, [o(pw_2896x8)] -%endif pmulhrsw m0, m1 mova m1, m0 jmp m(iadst_4x4_internal).end2 -%else ; adst / flipadst - pmulhrsw m1, m0, [o(iadst4_dconly2b)] - pmulhrsw m0, [o(iadst4_dconly2a)] - jmp m(i%2_4x4_internal).end2 -%endif %endif %endmacro %macro IDCT4_1D_PACKED 0 vpbroadcastd m4, [o(pd_2048)] @@ -475,35 +428,35 @@ REPX {psrad x, 12}, m1, m2, m0, m5 packssdw m0, m5 ; out0 out1 packssdw m1, m2 ; out2 out3 %endmacro -INV_TXFM_4X4_FN dct, dct, 0 -INV_TXFM_4X4_FN dct, adst, 0 -INV_TXFM_4X4_FN dct, flipadst, 0 -INV_TXFM_4X4_FN dct, identity, 3 +INV_TXFM_4X4_FN dct, dct +INV_TXFM_4X4_FN dct, adst +INV_TXFM_4X4_FN dct, flipadst +INV_TXFM_4X4_FN dct, identity cglobal idct_4x4_internal, 0, 5, 6, dst, stride, c, eob, tx2 mova m0, [cq+16*0] mova m1, [cq+16*1] IDCT4_1D_PACKED mova m2, [o(deint_shuf)] shufps m3, m0, m1, q1331 - shufps m0, m0, m1, q0220 + shufps m0, m1, q0220 pshufb m0, m2 pshufb m1, m3, m2 jmp tx2q .pass2: IDCT4_1D_PACKED pxor m2, m2 mova [cq+16*0], m2 mova [cq+16*1], m2 ITX4_END 0, 1, 3, 2 -INV_TXFM_4X4_FN adst, dct, 0 -INV_TXFM_4X4_FN adst, adst, 0 -INV_TXFM_4X4_FN adst, flipadst, 0 +INV_TXFM_4X4_FN adst, dct +INV_TXFM_4X4_FN adst, adst +INV_TXFM_4X4_FN adst, flipadst INV_TXFM_4X4_FN adst, identity cglobal iadst_4x4_internal, 0, 5, 6, dst, stride, c, eob, tx2 mova m0, [cq+16*0] mova m1, [cq+16*1] @@ -524,13 +477,13 @@ ALIGN function_align .main: IADST4_1D_PACKED ret -INV_TXFM_4X4_FN flipadst, dct, 0 -INV_TXFM_4X4_FN flipadst, adst, 0 -INV_TXFM_4X4_FN flipadst, flipadst, 0 +INV_TXFM_4X4_FN flipadst, dct +INV_TXFM_4X4_FN flipadst, adst +INV_TXFM_4X4_FN flipadst, flipadst INV_TXFM_4X4_FN flipadst, identity cglobal iflipadst_4x4_internal, 0, 5, 6, dst, stride, c, eob, tx2 mova m0, [cq+16*0] mova m1, [cq+16*1] @@ -547,11 +500,11 @@ mova [cq+16*0], m2 mova [cq+16*1], m2 .end2: ITX4_END 3, 2, 1, 0 -INV_TXFM_4X4_FN identity, dct, 3 +INV_TXFM_4X4_FN identity, dct INV_TXFM_4X4_FN identity, adst INV_TXFM_4X4_FN identity, flipadst INV_TXFM_4X4_FN identity, identity cglobal iidentity_4x4_internal, 0, 5, 6, dst, stride, c, eob, tx2 @@ -598,69 +551,23 @@ pextrd [r2 +strideq*1], xm5, 1 pextrd [r2 +strideq*2], xm5, 2 pextrd [r2 +r3 ], xm5, 3 %endmacro -%macro INV_TXFM_4X8_FN 2-3 -1 ; type1, type2, fast_thresh - INV_TXFM_FN %1, %2, %3, 4x8 -%if %3 >= 0 -%ifidn %1_%2, dct_identity - vpbroadcastd xm0, [o(pw_2896x8)] - pmulhrsw xm1, xm0, [cq] - vpbroadcastd xm2, [o(pw_4096)] - pmulhrsw xm1, xm0 - pmulhrsw xm1, xm2 - vpermq m1, m1, q1100 - punpcklwd m1, m1 - punpckldq m0, m1, m1 - punpckhdq m1, m1 - jmp m(iadst_4x8_internal).end3 -%elifidn %1_%2, identity_dct - movd xm0, [cq+16*0] - punpcklwd xm0, [cq+16*1] - movd xm1, [cq+16*2] - punpcklwd xm1, [cq+16*3] - vpbroadcastd xm2, [o(pw_2896x8)] - vpbroadcastd xm3, [o(pw_1697x8)] - vpbroadcastd xm4, [o(pw_2048)] - punpckldq xm0, xm1 - pmulhrsw xm0, xm2 - pmulhrsw xm3, xm0 - paddsw xm0, xm3 - pmulhrsw xm0, xm2 - pmulhrsw xm0, xm4 - vpbroadcastq m0, xm0 - mova m1, m0 - jmp m(iadst_4x8_internal).end3 -%elifidn %1_%2, dct_dct +%macro INV_TXFM_4X8_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 4x8 +%ifidn %1_%2, dct_dct movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] movd xm2, [o(pw_2048)] mov [cq], eobd pmulhrsw xm0, xm1 pmulhrsw xm0, xm1 pmulhrsw xm0, xm2 vpbroadcastw m0, xm0 mova m1, m0 - jmp m(iadst_4x8_internal).end4 -%else ; adst_dct / flipadst_dct - vpbroadcastw xm0, [cq] - vpbroadcastd xm1, [o(pw_2896x8)] - pmulhrsw xm0, xm1 - pmulhrsw xm0, [o(iadst4_dconly1a)] - vpbroadcastd xm2, [o(pw_2048)] - mov [cq], eobd - pmulhrsw xm0, xm1 - pmulhrsw xm0, xm2 -%ifidn %1, adst - vpbroadcastq m0, xm0 -%else ; flipadst - vpermq m0, m0, q1111 -%endif - mova m1, m0 - jmp m(iadst_4x8_internal).end4 -%endif + jmp m(iadst_4x8_internal).end3 %endif %endmacro %macro IDCT8_1D_PACKED 0 vpbroadcastd m6, [o(pd_2048)] @@ -758,55 +665,55 @@ paddsw m0, m1 ; out0 -out7 psubsw m1, m4, m5 ; t7 t6 paddsw m4, m5 ; out6 -out1 vpbroadcastd m5, [o(pw_2896x8)] vpblendd m3, m0, m4, 0x33 ; out6 -out7 - vpblendd m0, m0, m4, 0xcc ; out0 -out1 + vpblendd m0, m4, 0xcc ; out0 -out1 shufps m4, m2, m1, q1032 ; t3 t7 - vpblendd m1, m2, m1, 0xcc ; t2 t6 + vpblendd m1, m2, 0x33 ; t2 t6 psubsw m2, m1, m4 ; t2-t3 t6-t7 paddsw m1, m4 ; t2+t3 t6+t7 pmulhrsw m2, m5 ; out4 -out5 pshufd m1, m1, q1032 pmulhrsw m1, m5 ; out2 -out3 %endif %endmacro INIT_YMM avx2 -INV_TXFM_4X8_FN dct, dct, 0 -INV_TXFM_4X8_FN dct, identity, 7 +INV_TXFM_4X8_FN dct, dct INV_TXFM_4X8_FN dct, adst INV_TXFM_4X8_FN dct, flipadst +INV_TXFM_4X8_FN dct, identity cglobal idct_4x8_internal, 0, 5, 7, dst, stride, c, eob, tx2 vpermq m0, [cq+32*0], q3120 vpermq m1, [cq+32*1], q3120 vpbroadcastd m2, [o(pw_2896x8)] pmulhrsw m0, m2 pmulhrsw m1, m2 IDCT4_1D_PACKED vbroadcasti128 m2, [o(deint_shuf)] shufps m3, m0, m1, q1331 - shufps m0, m0, m1, q0220 + shufps m0, m1, q0220 pshufb m0, m2 pshufb m1, m3, m2 jmp tx2q .pass2: vextracti128 xm2, m0, 1 vextracti128 xm3, m1, 1 call .main vpbroadcastd m4, [o(pw_2048)] - vinserti128 m0, m0, xm2, 1 - vinserti128 m1, m1, xm3, 1 + vinserti128 m0, xm2, 1 + vinserti128 m1, xm3, 1 pshufd m1, m1, q1032 jmp m(iadst_4x8_internal).end2 ALIGN function_align .main: WRAP_XMM IDCT8_1D_PACKED ret -INV_TXFM_4X8_FN adst, dct, 0 +INV_TXFM_4X8_FN adst, dct INV_TXFM_4X8_FN adst, adst INV_TXFM_4X8_FN adst, flipadst INV_TXFM_4X8_FN adst, identity cglobal iadst_4x8_internal, 0, 5, 7, dst, stride, c, eob, tx2 @@ -826,25 +733,24 @@ vextracti128 xm3, m1, 1 pshufd xm4, xm0, q1032 pshufd xm5, xm1, q1032 call .main_pass2 vpbroadcastd m4, [o(pw_2048)] - vinserti128 m0, m0, xm2, 1 - vinserti128 m1, m1, xm3, 1 + vinserti128 m0, xm2, 1 + vinserti128 m1, xm3, 1 pxor m5, m5 psubw m5, m4 .end: - vpblendd m4, m4, m5, 0xcc + vpblendd m4, m5, 0xcc .end2: pmulhrsw m0, m4 pmulhrsw m1, m4 WIN64_RESTORE_XMM -.end3: pxor m2, m2 mova [cq+32*0], m2 mova [cq+32*1], m2 -.end4: +.end3: lea r2, [dstq+strideq*4] lea r3, [strideq*3] WRITE_4X8 0, 1 RET ALIGN function_align @@ -854,11 +760,11 @@ ALIGN function_align .main_pass2: WRAP_XMM IADST8_1D_PACKED 2 ret -INV_TXFM_4X8_FN flipadst, dct, 0 +INV_TXFM_4X8_FN flipadst, dct INV_TXFM_4X8_FN flipadst, adst INV_TXFM_4X8_FN flipadst, flipadst INV_TXFM_4X8_FN flipadst, identity cglobal iflipadst_4x8_internal, 0, 5, 7, dst, stride, c, eob, tx2 @@ -878,19 +784,19 @@ vextracti128 xm3, m1, 1 pshufd xm4, xm0, q1032 pshufd xm5, xm1, q1032 call m(iadst_4x8_internal).main_pass2 vpbroadcastd m5, [o(pw_2048)] - vinserti128 m3, m3, xm1, 1 - vinserti128 m2, m2, xm0, 1 + vinserti128 m3, xm1, 1 + vinserti128 m2, xm0, 1 pxor m4, m4 psubw m4, m5 pshufd m0, m3, q1032 pshufd m1, m2, q1032 jmp m(iadst_4x8_internal).end -INV_TXFM_4X8_FN identity, dct, 3 +INV_TXFM_4X8_FN identity, dct INV_TXFM_4X8_FN identity, adst INV_TXFM_4X8_FN identity, flipadst INV_TXFM_4X8_FN identity, identity cglobal iidentity_4x8_internal, 0, 5, 7, dst, stride, c, eob, tx2 @@ -911,53 +817,13 @@ jmp tx2q .pass2: vpbroadcastd m4, [o(pw_4096)] jmp m(iadst_4x8_internal).end2 -%macro INV_TXFM_4X16_FN 2-3 -1 ; type1, type2, fast_thresh - INV_TXFM_FN %1, %2, %3, 4x16 -%if %3 >= 0 -%ifidn %1_%2, dct_identity - vpbroadcastd m0, [o(pw_2896x8)] - pmulhrsw m0, [cq] - vpbroadcastd m1, [o(pw_16384)] - vpbroadcastd m2, [o(pw_1697x16)] - vpbroadcastd m3, [o(pw_2048)] - pmulhrsw m0, m1 - pmulhrsw m2, m0 - paddsw m0, m0 - paddsw m0, m2 - pmulhrsw m3, m0 - punpcklwd m1, m3, m3 - punpckhwd m3, m3 - punpckldq m0, m1, m1 - punpckhdq m1, m1 - punpckldq m2, m3, m3 - punpckhdq m3, m3 - jmp m(iadst_4x16_internal).end3 -%elifidn %1_%2, identity_dct - movd xm0, [cq+32*0] - punpcklwd xm0, [cq+32*1] - movd xm1, [cq+32*2] - punpcklwd xm1, [cq+32*3] - vpbroadcastd xm2, [o(pw_1697x8)] - vpbroadcastd xm3, [o(pw_2896x8)] - vpbroadcastd xm4, [o(pw_2048)] - punpckldq xm0, xm1 - pcmpeqw xm1, xm1 - pmulhrsw xm2, xm0 - pcmpeqw xm1, xm0 - pxor xm0, xm1 - pavgw xm0, xm2 - pmulhrsw xm0, xm3 - pmulhrsw xm0, xm4 - vpbroadcastq m0, xm0 - mova m1, m0 - mova m2, m0 - mova m3, m0 - jmp m(iadst_4x16_internal).end3 -%elifidn %1_%2, dct_dct +%macro INV_TXFM_4X16_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 4x16 +%ifidn %1_%2, dct_dct movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] movd xm2, [o(pw_16384)] movd xm3, [o(pw_2048)] mov [cq], eobd @@ -966,31 +832,11 @@ pmulhrsw xm0, xm3 vpbroadcastw m0, xm0 mova m1, m0 mova m2, m0 mova m3, m0 - jmp m(iadst_4x16_internal).end4 -%else ; adst_dct / flipadst_dct - vpbroadcastw xm0, [cq] - pmulhrsw xm0, [o(iadst4_dconly1a)] - vpbroadcastd xm1, [o(pw_16384)] - vpbroadcastd xm2, [o(pw_2896x8)] - mov [cq], eobd - pmulhrsw xm0, xm1 - psrlw xm1, 3 ; pw_2048 - pmulhrsw xm0, xm2 - pmulhrsw xm0, xm1 -%ifidn %1, adst - vpbroadcastq m0, xm0 -%else ; flipadst - vpermq m0, m0, q1111 -%endif - mova m1, m0 - mova m2, m0 - mova m3, m0 - jmp m(iadst_4x16_internal).end4 -%endif + jmp m(iadst_4x16_internal).end3 %endif %endmacro %macro IDCT16_1D_PACKED 0 vpbroadcastd m10, [o(pd_2048)] @@ -1036,11 +882,11 @@ vpbroadcastd m0, [o(pw_m2896_2896)] ITX_MUL2X_PACK 4, 5, _, 10, 5, 0, 4 ; t11 t12 vpbroadcastd m5, [o(pw_2896_2896)] ITX_MUL2X_PACK 1, 0, _, 10, 0, 5, 4 ; t6 t5 vpbroadcastd m0, [o(pw_m2896_2896)] - ITX_MUL2X_PACK 2, 0, _, 10, 0, 5, 4, ; t13a t10a + ITX_MUL2X_PACK 2, 0, _, 10, 0, 5, 4 ; t13a t10a punpckhqdq m0, m8, m3 ; t15a t14 punpcklqdq m8, m3 ; t8a t9 shufps m5, m4, m2, q1032 ; t12 t13a vpblendd m4, m2, 0xcc ; t11 t10a shufps m2, m7, m1, q1032 ; t7 t6 @@ -1059,14 +905,14 @@ paddsw m2, m4 ; out4 out5 psubsw m4, m3, m8 ; out8 out9 paddsw m3, m8 ; out7 out6 %endmacro -INV_TXFM_4X16_FN dct, dct, 0 -INV_TXFM_4X16_FN dct, identity, 15 +INV_TXFM_4X16_FN dct, dct INV_TXFM_4X16_FN dct, adst INV_TXFM_4X16_FN dct, flipadst +INV_TXFM_4X16_FN dct, identity cglobal idct_4x16_internal, 0, 5, 11, dst, stride, c, eob, tx2 mova m0, [cq+32*0] mova m1, [cq+32*1] mova m2, [cq+32*2] @@ -1087,24 +933,24 @@ vextracti128 xm4, m0, 1 vextracti128 xm5, m1, 1 vextracti128 xm6, m2, 1 vextracti128 xm7, m3, 1 call .main - vinserti128 m0, m0, xm4, 1 - vinserti128 m1, m1, xm5, 1 + vinserti128 m0, xm4, 1 + vinserti128 m1, xm5, 1 vpbroadcastd m5, [o(pw_2048)] - vinserti128 m2, m2, xm6, 1 - vinserti128 m3, m3, xm7, 1 + vinserti128 m2, xm6, 1 + vinserti128 m3, xm7, 1 pshufd m1, m1, q1032 pshufd m3, m3, q1032 jmp m(iadst_4x16_internal).end2 ALIGN function_align .main: WRAP_XMM IDCT16_1D_PACKED ret -INV_TXFM_4X16_FN adst, dct, 0 +INV_TXFM_4X16_FN adst, dct INV_TXFM_4X16_FN adst, adst INV_TXFM_4X16_FN adst, flipadst INV_TXFM_4X16_FN adst, identity cglobal iadst_4x16_internal, 0, 5, 11, dst, stride, c, eob, tx2 @@ -1132,43 +978,42 @@ pmulhrsw m1, m5 ; -out7 out4 out6 -out5 pmulhrsw m2, m5 ; out8 -out11 -out9 out10 vpbroadcastd m5, [o(pw_2048)] pshufd m1, m1, q1032 vpblendd m4, m1, m0, 0x33 - vpblendd m0, m0, m2, 0x33 - vpblendd m2, m2, m3, 0x33 - vpblendd m3, m3, m1, 0x33 + vpblendd m0, m2, 0x33 + vpblendd m2, m3, 0x33 + vpblendd m3, m1, 0x33 vpermq m0, m0, q2031 vpermq m1, m2, q1302 vpermq m2, m3, q3120 vpermq m3, m4, q0213 psubw m6, m7, m5 .end: - vpblendd m5, m5, m6, 0xcc + vpblendd m5, m6, 0xcc .end2: REPX {pmulhrsw x, m5}, m0, m1, m2, m3 WIN64_RESTORE_XMM -.end3: pxor m4, m4 mova [cq+32*0], m4 mova [cq+32*1], m4 mova [cq+32*2], m4 mova [cq+32*3], m4 -.end4: +.end3: lea r2, [dstq+strideq*8] lea r3, [strideq*3] WRITE_4X8 0, 1 lea dstq, [dstq+strideq*4] lea r2, [r2 +strideq*4] WRITE_4X8 2, 3 RET ALIGN function_align .main: vpblendd m4, m1, m0, 0xcc - vpblendd m1, m1, m0, 0x33 + vpblendd m1, m0, 0x33 vpblendd m5, m2, m3, 0xcc - vpblendd m2, m2, m3, 0x33 + vpblendd m2, m3, 0x33 vperm2i128 m3, m5, m2, 0x31 vinserti128 m0, m1, xm4, 1 ; in0 in3 in2 in1 vperm2i128 m4, m1, m4, 0x31 vinserti128 m1, m5, xm2, 1 ; in4 in7 in6 in5 pshufd m3, m3, q1032 ; in12 in15 in13 in14 @@ -1196,27 +1041,27 @@ psubsw m4, m0, m1 ; t5 t4 t7 t6 paddsw m0, m1 ; t1 t0 t3 t2 psubsw m1, m2, m3 ; t13a t12a t15a t14a paddsw m2, m3 ; t9a t8a t11a t10a psubw m3, m7, m6 ; pw_3784_m1567 - vpblendd m6, m6, m3, 0xf0 + vpblendd m6, m3, 0xf0 ITX_MUL2X_PACK 4, 3, _, 8, 6, 5, 4 ; t4a t5a t7a t6a ITX_MUL2X_PACK 1, 3, _, 8, 6, 5, 4 ; t12 t13 t15 t14 vbroadcasti128 m5, [o(deint_shuf)] pshufb m0, m5 pshufb m2, m5 vperm2i128 m3, m0, m2, 0x31 ; t3 t2 t11a t10a - vinserti128 m0, m0, xm2, 1 ; t1 t0 t9a t8a + vinserti128 m0, xm2, 1 ; t1 t0 t9a t8a vperm2i128 m2, m4, m1, 0x31 ; t7a t6a t15 t14 - vinserti128 m4, m4, xm1, 1 ; t4a t5a t12 t13 + vinserti128 m4, xm1, 1 ; t4a t5a t12 t13 pshufd m2, m2, q1032 ; t6a t7a t14 t15 psubsw m1, m0, m3 ; t3a t2a t11 t10 paddsw m0, m3 ; -out15 out0 out14 -out1 paddsw m3, m4, m2 ; -out3 out12 out2 -out13 psubsw m4, m2 ; t6 t7 t14a t15a shufps m2, m1, m4, q1032 ; t2a t6 t10 t14a - vpblendd m4, m4, m1, 0x33 ; t3a t7 t11 t15a + vpblendd m4, m1, 0x33 ; t3a t7 t11 t15a ret ALIGN function_align .main_pass1_end: vpbroadcastd m5, [o(pw_m2896_2896)] vpbroadcastd m6, [o(pw_2896_2896)] @@ -1230,11 +1075,11 @@ REPX {psrad x, 12}, m5, m2, m1, m4 packssdw m2, m5 ; -out11 out8 out10 -out9 packssdw m1, m4 ; -out7 out4 out6 -out5 ret -INV_TXFM_4X16_FN flipadst, dct, 0 +INV_TXFM_4X16_FN flipadst, dct INV_TXFM_4X16_FN flipadst, adst INV_TXFM_4X16_FN flipadst, flipadst INV_TXFM_4X16_FN flipadst, identity cglobal iflipadst_4x16_internal, 0, 5, 11, dst, stride, c, eob, tx2 @@ -1262,21 +1107,21 @@ pmulhrsw m1, m5 ; -out7 out4 out6 -out5 pmulhrsw m2, m5 ; out8 -out11 -out9 out10 vpbroadcastd m6, [o(pw_2048)] pshufd m1, m1, q1032 vpblendd m4, m0, m2, 0x33 - vpblendd m0, m0, m1, 0xcc - vpblendd m1, m1, m3, 0xcc - vpblendd m2, m2, m3, 0x33 + vpblendd m0, m1, 0xcc + vpblendd m1, m3, 0xcc + vpblendd m2, m3, 0x33 vpermq m0, m0, q3120 vpermq m1, m1, q0213 vpermq m2, m2, q2031 vpermq m3, m4, q1302 psubw m5, m7, m6 jmp m(iadst_4x16_internal).end -INV_TXFM_4X16_FN identity, dct, 3 +INV_TXFM_4X16_FN identity, dct INV_TXFM_4X16_FN identity, adst INV_TXFM_4X16_FN identity, flipadst INV_TXFM_4X16_FN identity, identity cglobal iidentity_4x16_internal, 0, 5, 11, dst, stride, c, eob, tx2 @@ -1323,11 +1168,11 @@ paddsw m1, m6 paddsw m2, m7 paddsw m3, m8 jmp m(iadst_4x16_internal).end2 -%macro WRITE_8X4 4-7 strideq*1, strideq*2, r3, ; coefs[1-2], tmp[1-2], off[1-3] +%macro WRITE_8X4 4-7 strideq*1, strideq*2, r3 ; coefs[1-2], tmp[1-2], off[1-3] movq xm%3, [dstq ] movhps xm%3, [dstq+%5] movq xm%4, [dstq+%6] movhps xm%4, [dstq+%7] pmovzxbw m%3, xm%3 @@ -1348,73 +1193,29 @@ movhps [dstq+%6], xm%3 movq [dstq+%5], xm%4 movhps [dstq+%7], xm%4 %endmacro -%macro INV_TXFM_8X4_FN 2-3 -1 ; type1, type2, fast_thresh - INV_TXFM_FN %1, %2, %3, 8x4 -%if %3 >= 0 -%ifidn %1_%2, dct_identity - vpbroadcastd xm0, [o(pw_2896x8)] - pmulhrsw xm1, xm0, [cq] - vpbroadcastd xm2, [o(pw_1697x8)] - vpbroadcastd xm3, [o(pw_2048)] - pmulhrsw xm1, xm0 - pmulhrsw xm2, xm1 - paddsw xm1, xm2 - pmulhrsw xm1, xm3 - punpcklwd xm1, xm1 - punpckldq xm0, xm1, xm1 - punpckhdq xm1, xm1 - vpermq m0, m0, q1100 - vpermq m1, m1, q1100 -%elifidn %1_%2, identity_dct - mova xm0, [cq+16*0] - packusdw xm0, [cq+16*1] - mova xm1, [cq+16*2] - packusdw xm1, [cq+16*3] - vpbroadcastd xm2, [o(pw_2896x8)] - vpbroadcastd xm3, [o(pw_2048)] - packusdw xm0, xm1 - pmulhrsw xm0, xm2 - paddsw xm0, xm0 - pmulhrsw xm0, xm2 - pmulhrsw xm0, xm3 - vinserti128 m0, m0, xm0, 1 - mova m1, m0 -%else +%macro INV_TXFM_8X4_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 8x4 +%ifidn %1_%2, dct_dct movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] pmulhrsw xm0, xm1 -%ifidn %2, dct movd xm2, [o(pw_2048)] pmulhrsw xm0, xm1 pmulhrsw xm0, xm2 vpbroadcastw m0, xm0 mova m1, m0 -%else ; adst / flipadst - vpbroadcastw m0, xm0 - pmulhrsw m0, [o(iadst4_dconly2a)] - vpbroadcastd m1, [o(pw_2048)] - pmulhrsw m1, m0 -%ifidn %2, adst - vpermq m0, m1, q1100 - vpermq m1, m1, q3322 -%else ; flipadst - vpermq m0, m1, q2233 - vpermq m1, m1, q0011 -%endif -%endif -%endif jmp m(iadst_8x4_internal).end3 %endif %endmacro -INV_TXFM_8X4_FN dct, dct, 0 -INV_TXFM_8X4_FN dct, adst, 0 -INV_TXFM_8X4_FN dct, flipadst, 0 -INV_TXFM_8X4_FN dct, identity, 3 +INV_TXFM_8X4_FN dct, dct +INV_TXFM_8X4_FN dct, adst +INV_TXFM_8X4_FN dct, flipadst +INV_TXFM_8X4_FN dct, identity cglobal idct_8x4_internal, 0, 5, 7, dst, stride, c, eob, tx2 vpbroadcastd xm3, [o(pw_2896x8)] pmulhrsw xm0, xm3, [cq+16*0] pmulhrsw xm1, xm3, [cq+16*1] @@ -1423,11 +1224,11 @@ call m(idct_4x8_internal).main vbroadcasti128 m4, [o(deint_shuf)] vinserti128 m3, m1, xm3, 1 vinserti128 m1, m0, xm2, 1 shufps m0, m1, m3, q0220 - shufps m1, m1, m3, q1331 + shufps m1, m3, q1331 pshufb m0, m4 pshufb m1, m4 jmp tx2q .pass2: IDCT4_1D_PACKED @@ -1447,12 +1248,12 @@ pshufd xm5, [cq+16*1], q1032 pmulhrsw xm2, xm0, [cq+16*2] pmulhrsw xm4, xm0 pmulhrsw xm5, xm0 call m(iadst_4x8_internal).main_pass1 - vinserti128 m0, m0, xm2, 1 - vinserti128 m1, m1, xm3, 1 + vinserti128 m0, xm2, 1 + vinserti128 m1, xm3, 1 punpckhwd m2, m0, m1 punpcklwd m0, m1 pxor m3, m3 psubsw m3, m2 punpckhwd m1, m0, m3 @@ -1492,12 +1293,12 @@ pshufd xm5, [cq+16*1], q1032 pmulhrsw xm2, xm0, [cq+16*2] pmulhrsw xm4, xm0 pmulhrsw xm5, xm0 call m(iadst_4x8_internal).main_pass1 - vinserti128 m3, m3, xm1, 1 - vinserti128 m2, m2, xm0, 1 + vinserti128 m3, xm1, 1 + vinserti128 m2, xm0, 1 punpckhwd m1, m3, m2 punpcklwd m3, m2 pxor m0, m0 psubsw m0, m1 punpckhwd m1, m0, m3 @@ -1508,20 +1309,20 @@ mova m2, m1 vpermq m1, m0, q2031 vpermq m0, m2, q2031 jmp m(iadst_8x4_internal).end2 -INV_TXFM_8X4_FN identity, dct, 7 +INV_TXFM_8X4_FN identity, dct INV_TXFM_8X4_FN identity, adst INV_TXFM_8X4_FN identity, flipadst INV_TXFM_8X4_FN identity, identity cglobal iidentity_8x4_internal, 0, 5, 7, dst, stride, c, eob, tx2 - mova xm2, [cq+16*0] - mova xm0, [cq+16*1] - vinserti128 m2, m2, [cq+16*2], 1 - vinserti128 m0, m0, [cq+16*3], 1 + mova xm2, [cq+16*0] + mova xm0, [cq+16*1] + vinserti128 m2, [cq+16*2], 1 + vinserti128 m0, [cq+16*3], 1 vpbroadcastd m3, [o(pw_2896x8)] punpcklwd m1, m2, m0 punpckhwd m2, m0 pmulhrsw m1, m3 pmulhrsw m2, m3 @@ -1536,29 +1337,13 @@ pmulhrsw m3, m1 paddsw m0, m2 paddsw m1, m3 jmp m(iadst_8x4_internal).end -%macro INV_TXFM_8X8_FN 2-3 -1 ; type1, type2, fast_thresh - INV_TXFM_FN %1, %2, %3, 8x8 -%ifidn %1_%2, dct_identity - vpbroadcastd xm0, [o(pw_2896x8)] - pmulhrsw xm0, [cq] - vpbroadcastd xm1, [o(pw_16384)] - pmulhrsw xm0, xm1 - psrlw xm1, 2 ; pw_4096 - pmulhrsw xm0, xm1 - pshufb xm0, [o(deint_shuf)] - vpermq m3, m0, q1100 - punpcklwd m3, m3 - pshufd m0, m3, q0000 - pshufd m1, m3, q1111 - pshufd m2, m3, q2222 - pshufd m3, m3, q3333 - jmp m(iadst_8x8_internal).end4 -%elif %3 >= 0 -%ifidn %1, dct +%macro INV_TXFM_8X8_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 8x8 +%ifidn %1_%2, dct_dct movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] movd xm2, [o(pw_16384)] mov [cq], eobd pmulhrsw xm0, xm2 @@ -1574,37 +1359,17 @@ WRITE_8X4 0, 0, 1, 2 lea dstq, [dstq+strideq*4] dec r2d jg .loop RET -%else ; identity - mova m0, [cq+32*0] - punpcklwd m0, [cq+32*1] - mova m1, [cq+32*2] - punpcklwd m1, [cq+32*3] - vpbroadcastd m2, [o(pw_2896x8)] - vpbroadcastd m3, [o(pw_2048)] - pxor m4, m4 - mova [cq+32*0], m4 - mova [cq+32*1], m4 - mova [cq+32*2], m4 - mova [cq+32*3], m4 - punpckldq m0, m1 - vpermq m1, m0, q3232 - vpermq m0, m0, q1010 - punpcklwd m0, m1 - pmulhrsw m0, m2 - pmulhrsw m0, m3 - jmp m(inv_txfm_add_dct_dct_8x8).end -%endif %endif %endmacro -INV_TXFM_8X8_FN dct, dct, 0 -INV_TXFM_8X8_FN dct, identity, 7 +INV_TXFM_8X8_FN dct, dct INV_TXFM_8X8_FN dct, adst INV_TXFM_8X8_FN dct, flipadst +INV_TXFM_8X8_FN dct, identity cglobal idct_8x8_internal, 0, 5, 7, dst, stride, c, eob, tx2 vpermq m0, [cq+32*0], q3120 ; 0 1 vpermq m3, [cq+32*3], q3120 ; 6 7 vpermq m2, [cq+32*2], q3120 ; 4 5 @@ -1747,24 +1512,24 @@ vpermq m2, m1, q2031 pmulhrsw m1, m0, m4 pmulhrsw m0, m5, m4 jmp m(iadst_8x8_internal).end3 -INV_TXFM_8X8_FN identity, dct, 7 +INV_TXFM_8X8_FN identity, dct INV_TXFM_8X8_FN identity, adst INV_TXFM_8X8_FN identity, flipadst INV_TXFM_8X8_FN identity, identity cglobal iidentity_8x8_internal, 0, 5, 7, dst, stride, c, eob, tx2 - mova xm3, [cq+16*0] - mova xm2, [cq+16*1] - vinserti128 m3, m3, [cq+16*4], 1 - vinserti128 m2, m2, [cq+16*5], 1 - mova xm4, [cq+16*2] - mova xm0, [cq+16*3] - vinserti128 m4, m4, [cq+16*6], 1 - vinserti128 m0, m0, [cq+16*7], 1 + mova xm3, [cq+16*0] + mova xm2, [cq+16*1] + vinserti128 m3, [cq+16*4], 1 + vinserti128 m2, [cq+16*5], 1 + mova xm4, [cq+16*2] + mova xm0, [cq+16*3] + vinserti128 m4, [cq+16*6], 1 + vinserti128 m0, [cq+16*7], 1 punpcklwd m1, m3, m2 punpckhwd m3, m2 punpcklwd m2, m4, m0 punpckhwd m4, m0 punpckldq m0, m1, m2 @@ -1774,12 +1539,12 @@ jmp tx2q .pass2: vpbroadcastd m4, [o(pw_4096)] jmp m(iadst_8x8_internal).end -%macro INV_TXFM_8X16_FN 2-3 -1 ; type1, type2, fast_thresh - INV_TXFM_FN %1, %2, %3, 8x16 +%macro INV_TXFM_8X16_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 8x16 %ifidn %1_%2, dct_dct movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] movd xm2, [o(pw_16384)] mov [cq], eobd @@ -1789,70 +1554,10 @@ pmulhrsw xm0, xm1 pmulhrsw xm0, xm2 vpbroadcastw m0, xm0 mov r2d, 4 jmp m(inv_txfm_add_dct_dct_8x8).end2 -%elifidn %1_%2, dct_identity - WIN64_SPILL_XMM 13 - vpbroadcastd m0, [o(pw_2896x8)] - pmulhrsw m7, m0, [cq] - vpbroadcastd m1, [o(pw_16384)] - vpbroadcastd m2, [o(pw_1697x16)] - pxor m3, m3 - mova [cq], m3 - pmulhrsw m7, m0 - pmulhrsw m7, m1 - psrlw m1, 3 ; pw_2048 - pmulhrsw m2, m7 - paddsw m7, m7 - paddsw m7, m2 - pmulhrsw m7, m1 - punpcklwd m5, m7, m7 - punpckhwd m7, m7 - punpcklwd m4, m5, m5 - punpckhwd m5, m5 - punpcklwd m6, m7, m7 - punpckhwd m7, m7 - vpermq m0, m4, q1100 - vpermq m1, m5, q1100 - vpermq m2, m6, q1100 - vpermq m3, m7, q1100 - vpermq m4, m4, q3322 - vpermq m5, m5, q3322 - vpermq m6, m6, q3322 - vpermq m7, m7, q3322 - jmp m(idct_8x16_internal).end4 -%elifidn %1_%2, identity_dct - movd xm0, [cq+32*0] - punpcklwd xm0, [cq+32*1] - movd xm2, [cq+32*2] - punpcklwd xm2, [cq+32*3] - add cq, 32*4 - movd xm1, [cq+32*0] - punpcklwd xm1, [cq+32*1] - movd xm3, [cq+32*2] - punpcklwd xm3, [cq+32*3] - vpbroadcastd xm4, [o(pw_2896x8)] - vpbroadcastd xm5, [o(pw_2048)] - xor eax, eax - mov [cq-32*4], eax - mov [cq-32*3], eax - mov [cq-32*2], eax - mov [cq-32*1], eax - punpckldq xm0, xm2 - punpckldq xm1, xm3 - punpcklqdq xm0, xm1 - pmulhrsw xm0, xm4 - pmulhrsw xm0, xm4 - pmulhrsw xm0, xm5 - mov [cq+32*0], eax - mov [cq+32*1], eax - mov [cq+32*2], eax - mov [cq+32*3], eax - vinserti128 m0, m0, xm0, 1 - mov r2d, 4 - jmp m(inv_txfm_add_dct_dct_8x8).end2 %endif %endmacro %macro ITX_8X16_LOAD_COEFS 0 vpbroadcastd m4, [o(pw_2896x8)] @@ -1865,28 +1570,28 @@ pmulhrsw m5, m4, [cq+32*1] pmulhrsw m3, m4, [cq-32*1] pmulhrsw m4, [cq+32*0] %endmacro -INV_TXFM_8X16_FN dct, dct, 0 -INV_TXFM_8X16_FN dct, identity, 15 +INV_TXFM_8X16_FN dct, dct INV_TXFM_8X16_FN dct, adst INV_TXFM_8X16_FN dct, flipadst +INV_TXFM_8X16_FN dct, identity cglobal idct_8x16_internal, 0, 5, 13, dst, stride, c, eob, tx2 ITX_8X16_LOAD_COEFS call m(idct_16x8_internal).main vpbroadcastd m10, [o(pw_16384)] .pass1_end: vperm2i128 m9, m3, m7, 0x31 - vinserti128 m3, m3, xm7, 1 + vinserti128 m3, xm7, 1 vperm2i128 m8, m2, m6, 0x31 - vinserti128 m2, m2, xm6, 1 + vinserti128 m2, xm6, 1 vperm2i128 m6, m1, m5, 0x31 - vinserti128 m1, m1, xm5, 1 + vinserti128 m1, xm5, 1 vperm2i128 m5, m0, m4, 0x31 - vinserti128 m0, m0, xm4, 1 + vinserti128 m0, xm4, 1 punpckhwd m4, m2, m3 punpcklwd m2, m3 punpckhwd m3, m0, m1 punpcklwd m0, m1 .pass1_end2: @@ -1913,11 +1618,10 @@ .end2: REPX {pmulhrsw x, m8}, m0, m1, m2, m3, m4, m5, m6, m7 .end3: pxor m8, m8 REPX {mova [cq+32*x], m8}, -4, -3, -2, -1, 0, 1, 2, 3 -.end4: lea r3, [strideq*3] WRITE_8X4 0, 1, 8, 9 lea dstq, [dstq+strideq*4] WRITE_8X4 2, 3, 0, 1 lea dstq, [dstq+strideq*4] @@ -2118,11 +1822,11 @@ pmulhrsw m5, m6, m8 pmulhrsw m6, m7, m8 pmulhrsw m7, m9, m8 jmp m(idct_8x16_internal).end3 -INV_TXFM_8X16_FN identity, dct, 7 +INV_TXFM_8X16_FN identity, dct INV_TXFM_8X16_FN identity, adst INV_TXFM_8X16_FN identity, flipadst INV_TXFM_8X16_FN identity, identity %macro IDTX16 3-4 ; src/dst, tmp, pw_1697x16, [pw_16394] @@ -2134,28 +1838,28 @@ %endif paddsw m%1, m%2 %endmacro cglobal iidentity_8x16_internal, 0, 5, 13, dst, stride, c, eob, tx2 - mova xm3, [cq+16*0] - mova xm2, [cq+16*2] + mova xm3, [cq+16*0] + mova xm2, [cq+16*2] add cq, 16*8 - vinserti128 m3, m3, [cq+16*0], 1 - vinserti128 m2, m2, [cq+16*2], 1 + vinserti128 m3, [cq+16*0], 1 + vinserti128 m2, [cq+16*2], 1 vpbroadcastd m9, [o(pw_2896x8)] - mova xm4, [cq-16*4] - mova xm5, [cq-16*2] - vinserti128 m4, m4, [cq+16*4], 1 - vinserti128 m5, m5, [cq+16*6], 1 - mova xm7, [cq-16*7] - mova xm6, [cq-16*5] - vinserti128 m7, m7, [cq+16*1], 1 - vinserti128 m6, m6, [cq+16*3], 1 - mova xm8, [cq-16*3] - mova xm0, [cq-16*1] - vinserti128 m8, m8, [cq+16*5], 1 - vinserti128 m0, m0, [cq+16*7], 1 + mova xm4, [cq-16*4] + mova xm5, [cq-16*2] + vinserti128 m4, [cq+16*4], 1 + vinserti128 m5, [cq+16*6], 1 + mova xm7, [cq-16*7] + mova xm6, [cq-16*5] + vinserti128 m7, [cq+16*1], 1 + vinserti128 m6, [cq+16*3], 1 + mova xm8, [cq-16*3] + mova xm0, [cq-16*1] + vinserti128 m8, [cq+16*5], 1 + vinserti128 m0, [cq+16*7], 1 punpcklwd m1, m3, m2 punpckhwd m3, m2 punpcklwd m2, m4, m5 punpckhwd m4, m5 punpcklwd m5, m7, m6 @@ -2195,68 +1899,15 @@ vpermq m%3, m%3, q3120 mova [dstq+%5], xm%3 vextracti128 [dstq+%6], m%3, 1 %endmacro -%macro INV_TXFM_16X4_FN 2-3 -1 ; type1, type2, fast_thresh - INV_TXFM_FN %1, %2, %3, 16x4 -%if %3 >= 0 -%ifidn %1_%2, dct_identity - vpbroadcastd xm3, [o(pw_2896x8)] - pmulhrsw xm3, [cq] - vpbroadcastd xm0, [o(pw_16384)] - vpbroadcastd xm1, [o(pw_1697x8)] - pmulhrsw xm3, xm0 - psrlw xm0, 3 ; pw_2048 - pmulhrsw xm1, xm3 - paddsw xm3, xm1 - pmulhrsw xm3, xm0 - punpcklwd xm3, xm3 - punpckldq xm1, xm3, xm3 - punpckhdq xm3, xm3 - vpbroadcastq m0, xm1 - vpermq m1, m1, q1111 - vpbroadcastq m2, xm3 - vpermq m3, m3, q1111 - jmp m(iadst_16x4_internal).end2 -%elifidn %1_%2, identity_dct - mova xm0, [cq+16*0] - mova xm2, [cq+16*1] - vinserti128 m0, m0, [cq+16*4], 1 - vinserti128 m2, m2, [cq+16*5], 1 - mova xm1, [cq+16*2] - mova xm3, [cq+16*3] - vinserti128 m1, m1, [cq+16*6], 1 - vinserti128 m3, m3, [cq+16*7], 1 - vpbroadcastd m4, [o(pw_1697x16)] - vpbroadcastd m5, [o(pw_16384)] - packusdw m0, m2 - packusdw m1, m3 - packusdw m0, m1 - vpbroadcastd m1, [o(pw_2896x8)] - pmulhrsw m4, m0 - pmulhrsw m4, m5 - paddsw m0, m4 - psrlw m5, 3 ; pw_2048 - pmulhrsw m0, m1 - pmulhrsw m0, m5 - mov r3d, 2 -.end: - pxor m3, m3 -.end_loop: - mova [cq+32*0], m3 - mova [cq+32*1], m3 - add cq, 32*2 - WRITE_16X2 0, 0, 1, 2, strideq*0, strideq*1 - lea dstq, [dstq+strideq*2] - dec r3d - jg .end_loop - RET -%else +%macro INV_TXFM_16X4_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 16x4 +%ifidn %1_%2, dct_dct movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] -%ifidn %2, dct movd xm2, [o(pw_16384)] mov [cq], eobd mov r2d, 2 .dconly: pmulhrsw xm0, xm2 @@ -2265,11 +1916,11 @@ pmulhrsw xm0, xm2 vpbroadcastw m0, xm0 pxor m3, m3 .dconly_loop: mova xm1, [dstq] - vinserti128 m1, m1, [dstq+strideq], 1 + vinserti128 m1, [dstq+strideq], 1 punpckhbw m2, m1, m3 punpcklbw m1, m3 paddw m2, m0 paddw m1, m0 packuswb m1, m2 @@ -2277,39 +1928,17 @@ vextracti128 [dstq+strideq], m1, 1 lea dstq, [dstq+strideq*2] dec r2d jg .dconly_loop RET -%else ; adst / flipadst - movd xm2, [o(pw_16384)] - pmulhrsw xm0, xm2 - vpbroadcastw m0, xm0 - pmulhrsw m0, [o(iadst4_dconly2a)] - vpbroadcastd m3, [o(pw_2048)] - mov [cq], eobd - pmulhrsw m3, m0 -%ifidn %2, adst - vpbroadcastq m0, xm3 - vpermq m1, m3, q1111 - vpermq m2, m3, q2222 - vpermq m3, m3, q3333 -%else ; flipadst - vpermq m0, m3, q3333 - vpermq m1, m3, q2222 - vpermq m2, m3, q1111 - vpbroadcastq m3, xm3 -%endif - jmp m(iadst_16x4_internal).end3 -%endif -%endif %endif %endmacro -INV_TXFM_16X4_FN dct, dct, 0 -INV_TXFM_16X4_FN dct, adst, 0 -INV_TXFM_16X4_FN dct, flipadst, 0 -INV_TXFM_16X4_FN dct, identity, 3 +INV_TXFM_16X4_FN dct, dct +INV_TXFM_16X4_FN dct, adst +INV_TXFM_16X4_FN dct, flipadst +INV_TXFM_16X4_FN dct, identity cglobal idct_16x4_internal, 0, 5, 11, dst, stride, c, eob, tx2 mova xm0, [cq+16*0] mova xm1, [cq+16*1] mova xm2, [cq+16*2] @@ -2479,24 +2108,24 @@ WRITE_16X2 3, 2, 4, 5, strideq*0, strideq*1 lea dstq, [dstq+strideq*2] WRITE_16X2 1, 0, 4, 5, strideq*0, strideq*1 RET -INV_TXFM_16X4_FN identity, dct, 15 +INV_TXFM_16X4_FN identity, dct INV_TXFM_16X4_FN identity, adst INV_TXFM_16X4_FN identity, flipadst INV_TXFM_16X4_FN identity, identity cglobal iidentity_16x4_internal, 0, 5, 11, dst, stride, c, eob, tx2 - mova xm2, [cq+16*0] - mova xm4, [cq+16*1] - vinserti128 m2, m2, [cq+16*4], 1 - vinserti128 m4, m4, [cq+16*5], 1 - mova xm0, [cq+16*2] - mova xm1, [cq+16*3] - vinserti128 m0, m0, [cq+16*6], 1 - vinserti128 m1, m1, [cq+16*7], 1 + mova xm2, [cq+16*0] + mova xm4, [cq+16*1] + vinserti128 m2, [cq+16*4], 1 + vinserti128 m4, [cq+16*5], 1 + mova xm0, [cq+16*2] + mova xm1, [cq+16*3] + vinserti128 m0, [cq+16*6], 1 + vinserti128 m1, [cq+16*7], 1 vpbroadcastd m7, [o(pw_1697x16)] vpbroadcastd m8, [o(pw_16384)] punpcklwd m3, m2, m4 punpckhwd m2, m4 punpcklwd m4, m0, m1 @@ -2529,73 +2158,20 @@ paddsw m1, m5 paddsw m2, m6 paddsw m3, m7 jmp m(iadst_16x4_internal).end -%macro INV_TXFM_16X8_FN 2-3 -1 ; type1, type2, fast_thresh - INV_TXFM_FN %1, %2, %3, 16x8 +%macro INV_TXFM_16X8_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 16x8 %ifidn %1_%2, dct_dct movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] movd xm2, [o(pw_16384)] mov [cq], eobd pmulhrsw xm0, xm1 mov r2d, 4 jmp m(inv_txfm_add_dct_dct_16x4).dconly -%elifidn %1_%2, dct_identity - WIN64_SPILL_XMM 13 - vbroadcasti128 m7, [cq] - vpbroadcastd m0, [o(pw_2896x8)] - vpbroadcastd m1, [o(pw_16384)] - pxor xm2, xm2 - mova [cq], xm2 - pmulhrsw m7, m0 - pmulhrsw m7, m0 - pmulhrsw m7, m1 - psrlw m1, 2 ; pw_4096 - pmulhrsw m7, m1 - punpcklwd m3, m7, m7 - punpckhwd m7, m7 - pshufd m0, m3, q0000 - pshufd m1, m3, q1111 - pshufd m2, m3, q2222 - pshufd m3, m3, q3333 - pshufd m4, m7, q0000 - pshufd m5, m7, q1111 - pshufd m6, m7, q2222 - pshufd m7, m7, q3333 - lea r3, [strideq*3] - WRITE_16X2 0, 1, 8, 0, strideq*0, strideq*1 - WRITE_16X2 2, 3, 0, 1, strideq*2, r3 - jmp m(idct_16x8_internal).end4 -%elifidn %1_%2, identity_dct - mova m0, [cq+32*0] - packusdw m0, [cq+32*1] - mova m2, [cq+32*2] - packusdw m2, [cq+32*3] - mova m1, [cq+32*4] - packusdw m1, [cq+32*5] - mova m3, [cq+32*6] - packusdw m3, [cq+32*7] - vpbroadcastd m4, [o(pw_2896x8)] - vpbroadcastd m5, [o(pw_1697x16)] - packusdw m0, m2 - packusdw m1, m3 - vpbroadcastd m2, [o(pw_16384)] - packusdw m0, m1 - vpermq m1, m0, q3322 - vpermq m0, m0, q1100 - punpcklwd m0, m1 - pmulhrsw m0, m4 - pmulhrsw m5, m0 - pmulhrsw m5, m2 - paddsw m0, m5 - psrlw m2, 3 ; pw_2048 - pmulhrsw m0, m4 - pmulhrsw m0, m2 - mov r3d, 4 - jmp m(inv_txfm_add_identity_dct_16x4).end %endif %endmacro %macro ITX_16X8_LOAD_COEFS 1 ; shuf_odd vpbroadcastd m8, [o(pw_2896x8)] @@ -2609,14 +2185,14 @@ vpermq m3, [cq-32*1], q%1 vpermq m4, [cq+32*0], q3120 REPX {pmulhrsw x, m8}, m0, m7, m1, m6, m2, m5, m3, m4 %endmacro -INV_TXFM_16X8_FN dct, dct, 0 -INV_TXFM_16X8_FN dct, identity, 7 +INV_TXFM_16X8_FN dct, dct INV_TXFM_16X8_FN dct, adst INV_TXFM_16X8_FN dct, flipadst +INV_TXFM_16X8_FN dct, identity cglobal idct_16x8_internal, 0, 5, 13, dst, stride, c, eob, tx2 ITX_16X8_LOAD_COEFS 3120 call m(idct_8x16_internal).main vpbroadcastd m10, [o(pw_16384)] @@ -2646,17 +2222,17 @@ punpckldq m6, m7, m4 punpckhdq m7, m4 punpckldq m8, m9, m5 punpckhdq m9, m5 vperm2i128 m4, m0, m6, 0x31 - vinserti128 m0, m0, xm6, 1 + vinserti128 m0, xm6, 1 vperm2i128 m5, m1, m7, 0x31 - vinserti128 m1, m1, xm7, 1 + vinserti128 m1, xm7, 1 vperm2i128 m6, m2, m8, 0x31 - vinserti128 m2, m2, xm8, 1 + vinserti128 m2, xm8, 1 vperm2i128 m7, m3, m9, 0x31 - vinserti128 m3, m3, xm9, 1 + vinserti128 m3, xm9, 1 jmp tx2q .pass2: call .main vpbroadcastd m8, [o(pw_2048)] .end: @@ -2809,17 +2385,17 @@ punpckhdq m4, m3, m5 punpckldq m3, m5 punpckldq m5, m8, m2 punpckhdq m8, m2 vinserti128 m2, m6, xm5, 1 - vperm2i128 m6, m6, m5, 0x31 + vperm2i128 m6, m5, 0x31 vperm2i128 m5, m1, m4, 0x31 - vinserti128 m1, m1, xm4, 1 + vinserti128 m1, xm4, 1 vperm2i128 m4, m0, m3, 0x31 - vinserti128 m0, m0, xm3, 1 + vinserti128 m0, xm3, 1 vinserti128 m3, m7, xm8, 1 - vperm2i128 m7, m7, m8, 0x31 + vperm2i128 m7, m8, 0x31 jmp tx2q .pass2: call m(iadst_16x8_internal).main call m(iadst_16x8_internal).main_pass2_end pxor m8, m8 @@ -2835,34 +2411,34 @@ lea r3, [strideq*3] WRITE_16X2 10, 0, 8, 9, strideq*0, strideq*1 WRITE_16X2 1, 2, 0, 1, strideq*2, r3 jmp m(idct_16x8_internal).end3 -INV_TXFM_16X8_FN identity, dct, 15 +INV_TXFM_16X8_FN identity, dct INV_TXFM_16X8_FN identity, adst INV_TXFM_16X8_FN identity, flipadst INV_TXFM_16X8_FN identity, identity cglobal iidentity_16x8_internal, 0, 5, 13, dst, stride, c, eob, tx2 - mova xm7, [cq+16*0] - mova xm2, [cq+16*1] + mova xm7, [cq+16*0] + mova xm2, [cq+16*1] add cq, 16*8 vpbroadcastd m3, [o(pw_2896x8)] - vinserti128 m7, m7, [cq+16*0], 1 - vinserti128 m2, m2, [cq+16*1], 1 - mova xm6, [cq-16*6] - mova xm4, [cq-16*5] - vinserti128 m6, m6, [cq+16*2], 1 - vinserti128 m4, m4, [cq+16*3], 1 - mova xm8, [cq-16*4] - mova xm5, [cq-16*3] - vinserti128 m8, m8, [cq+16*4], 1 - vinserti128 m5, m5, [cq+16*5], 1 - mova xm0, [cq-16*2] - mova xm1, [cq-16*1] - vinserti128 m0, m0, [cq+16*6], 1 - vinserti128 m1, m1, [cq+16*7], 1 + vinserti128 m7, [cq+16*0], 1 + vinserti128 m2, [cq+16*1], 1 + mova xm6, [cq-16*6] + mova xm4, [cq-16*5] + vinserti128 m6, [cq+16*2], 1 + vinserti128 m4, [cq+16*3], 1 + mova xm8, [cq-16*4] + mova xm5, [cq-16*3] + vinserti128 m8, [cq+16*4], 1 + vinserti128 m5, [cq+16*5], 1 + mova xm0, [cq-16*2] + mova xm1, [cq-16*1] + vinserti128 m0, [cq+16*6], 1 + vinserti128 m1, [cq+16*7], 1 vpbroadcastd m10, [o(pw_1697x16)] vpbroadcastd m11, [o(pw_16384)] REPX {pmulhrsw x, m3}, m7, m2, m6, m4, m8, m5, m0, m1 punpcklwd m3, m7, m2 punpckhwd m7, m2 @@ -2894,85 +2470,19 @@ vpbroadcastd m8, [o(pw_4096)] jmp m(idct_16x8_internal).end %define o_base pw_5 + 128 -%macro INV_TXFM_16X16_FN 2-3 -1 ; type1, type2, fast_thresh - INV_TXFM_FN %1, %2, %3, 16x16 +%macro INV_TXFM_16X16_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 16x16 %ifidn %1_%2, dct_dct movd xm1, [o(pw_2896x8)] pmulhrsw xm0, xm1, [cq] movd xm2, [o(pw_8192)] mov [cq], eobd mov r2d, 8 jmp m(inv_txfm_add_dct_dct_16x4).dconly -%elifidn %1_%2, dct_identity - WIN64_SPILL_XMM 7 - vpbroadcastd m3, [o(pw_2896x8)] - pmulhrsw m3, [cq] - vpbroadcastd m0, [o(pw_8192)] - vpbroadcastd m1, [o(pw_1697x16)] - vpbroadcastw m4, [o(deint_shuf)] ; pb_0_1 - pcmpeqb m5, m5 - pxor m6, m6 - mova [cq], m6 - paddb m5, m5 ; pb_m2 - pmulhrsw m3, m0 - psrlw m0, 2 ; pw_2048 - IDTX16 3, 1, 1 - pmulhrsw m3, m0 - mov r3d, 8 -.loop: - mova xm1, [dstq] - vinserti128 m1, m1, [dstq+strideq*8], 1 - pshufb m0, m3, m4 - psubb m4, m5 ; += 2 - punpckhbw m2, m1, m6 - punpcklbw m1, m6 - paddw m2, m0 - paddw m1, m0 - packuswb m1, m2 - mova [dstq], xm1 - vextracti128 [dstq+strideq*8], m1, 1 - add dstq, strideq - dec r3d - jg .loop - RET -%elifidn %1_%2, identity_dct - movd xm0, [cq+32*0 ] - movd xm2, [cq+32*1 ] - movd xm1, [cq+32*2 ] - movd xm3, [cq+32*3 ] - vinserti128 m0, m0, [cq+32*8 ], 1 - vinserti128 m2, m2, [cq+32*9 ], 1 - vinserti128 m1, m1, [cq+32*10], 1 - vinserti128 m3, m3, [cq+32*11], 1 - punpcklwd m0, m2 - punpcklwd m1, m3 - punpckldq m0, m1 - movd xm1, [cq+32*4 ] - movd xm3, [cq+32*5 ] - movd xm2, [cq+32*6 ] - movd xm4, [cq+32*7 ] - vinserti128 m1, m1, [cq+32*12], 1 - vinserti128 m3, m3, [cq+32*13], 1 - vinserti128 m2, m2, [cq+32*14], 1 - vinserti128 m4, m4, [cq+32*15], 1 - punpcklwd m1, m3 - vpbroadcastd m3, [o(pw_1697x16)] - punpcklwd m2, m4 - vpbroadcastd m4, [o(pw_2896x8)] - punpckldq m1, m2 - vpbroadcastd m2, [o(pw_2048)] - punpcklqdq m0, m1 - pmulhrsw m3, m0 - psraw m3, 1 - pavgw m0, m3 - pmulhrsw m0, m4 - pmulhrsw m0, m2 - mov r3d, 8 - jmp m(inv_txfm_add_identity_dct_16x4).end %endif %endmacro %macro ITX_16X16_LOAD_COEFS 0 mova m0, [cq+32*0] @@ -2993,14 +2503,14 @@ mova m14, [cq+32*6] mova m15, [cq+32*7] mova [rsp], m15 %endmacro -INV_TXFM_16X16_FN dct, dct, 0 -INV_TXFM_16X16_FN dct, identity, 15 +INV_TXFM_16X16_FN dct, dct INV_TXFM_16X16_FN dct, adst INV_TXFM_16X16_FN dct, flipadst +INV_TXFM_16X16_FN dct, identity cglobal idct_16x16_internal, 0, 5, 16, 32*3, dst, stride, c, eob, tx2 ITX_16X16_LOAD_COEFS call .main .pass1_end: @@ -3012,23 +2522,23 @@ vextracti128 [rsp+16*4], m0, 1 mova [rsp+16*0], xm0 REPX {pmulhrsw x, m1}, m3, m5, m7, m9, m11, m13, m15 pmulhrsw m1, [rsp+32*1] vperm2i128 m8, m1, m9, 0x31 - vinserti128 m1, m1, xm9, 1 + vinserti128 m1, xm9, 1 vperm2i128 m9, m2, m10, 0x31 - vinserti128 m2, m2, xm10, 1 + vinserti128 m2, xm10, 1 vperm2i128 m10, m3, m11, 0x31 - vinserti128 m3, m3, xm11, 1 + vinserti128 m3, xm11, 1 vperm2i128 m11, m4, m12, 0x31 - vinserti128 m4, m4, xm12, 1 + vinserti128 m4, xm12, 1 vperm2i128 m12, m5, m13, 0x31 - vinserti128 m5, m5, xm13, 1 + vinserti128 m5, xm13, 1 vperm2i128 m13, m6, m14, 0x31 - vinserti128 m6, m6, xm14, 1 + vinserti128 m6, xm14, 1 vperm2i128 m14, m7, m15, 0x31 - vinserti128 m7, m7, xm15, 1 + vinserti128 m7, xm15, 1 mova m15, [rsp+32*2] .pass1_end3: punpcklwd m0, m9, m10 punpckhwd m9, m10 punpcklwd m10, m15, m8 @@ -3393,11 +2903,11 @@ pmulhrsw m%2, m%3, m%1 psraw m%2, 1 pavgw m%1, m%2 ; signs are guaranteed to be equal %endmacro -INV_TXFM_16X16_FN identity, dct, 15 +INV_TXFM_16X16_FN identity, dct INV_TXFM_16X16_FN identity, identity cglobal iidentity_16x16_internal, 0, 5, 16, 32*3, dst, stride, c, eob, tx2 vpbroadcastd m7, [o(pw_1697x16)] mova xm0, [cq+16* 0] @@ -3454,11 +2964,11 @@ pmulhrsw m15, m1 paddsw m1, m1 paddsw m15, m1 jmp m(idct_16x16_internal).end -%define o_base iadst4_dconly2a + 128 +%define o_base deint_shuf + 128 %macro LOAD_8ROWS 2-3 0 ; src, stride, is_rect2 %if %3 vpbroadcastd m15, [o(pw_2896x8)] pmulhrsw m0, m15, [%1+%2*0] @@ -3524,17 +3034,17 @@ cmp eobd, 106 jle .fast LOAD_8ROWS cq+32*1, 32*2 call m(idct_16x8_internal).main vperm2i128 m11, m0, m4, 0x31 - vinserti128 m0, m0, xm4, 1 + vinserti128 m0, xm4, 1 vperm2i128 m4, m1, m5, 0x31 - vinserti128 m1, m1, xm5, 1 + vinserti128 m1, xm5, 1 vperm2i128 m5, m2, m6, 0x31 - vinserti128 m2, m2, xm6, 1 + vinserti128 m2, xm6, 1 vperm2i128 m6, m3, m7, 0x31 - vinserti128 m3, m3, xm7, 1 + vinserti128 m3, xm7, 1 pxor m7, m7 REPX {mova [cq+32*x], m7}, 1, 3, 5, 7, 9, 11, 13, 15 punpckhwd m7, m0, m1 punpcklwd m0, m1 punpckhwd m1, m2, m3 @@ -3564,17 +3074,17 @@ mova [rsp+32*2], m2 .fast: LOAD_8ROWS cq+32*0, 32*2 call m(idct_16x8_internal).main vperm2i128 m8, m0, m4, 0x31 - vinserti128 m0, m0, xm4, 1 + vinserti128 m0, xm4, 1 vperm2i128 m4, m1, m5, 0x31 - vinserti128 m1, m1, xm5, 1 + vinserti128 m1, xm5, 1 vperm2i128 m5, m2, m6, 0x31 - vinserti128 m2, m2, xm6, 1 + vinserti128 m2, xm6, 1 vperm2i128 m6, m3, m7, 0x31 - vinserti128 m3, m3, xm7, 1 + vinserti128 m3, xm7, 1 vpbroadcastd m9, [o(pw_8192)] pxor m7, m7 REPX {mova [cq+32*x], m7}, 0, 2, 4, 6, 8, 10, 12, 14 punpckhwd m7, m0, m1 punpcklwd m0, m1 @@ -3773,11 +3283,11 @@ ret %macro LOAD_PACKED_16X2 4 ; dst, tmp, row[1-2] vbroadcasti128 m%1, [cq+16*%3] vbroadcasti128 m%2, [cq+16*%4] - shufpd m%1, m%1, m%2, 0x0c + shufpd m%1, m%2, 0x0c %endmacro cglobal inv_txfm_add_dct_dct_32x8, 4, 4, 0, dst, stride, c, eob lea rax, [o_base] test eobd, eobd @@ -3875,17 +3385,17 @@ punpckhdq m10, m5 REPX {pmulhrsw x, m12}, m0, m1, m2, m3, m6, m7, m9, m10 pmulhrsw m12, [rsp+32*0] mova [rsp+32*0], m8 vperm2i128 m4, m0, m6, 0x31 - vinserti128 m0, m0, xm6, 1 + vinserti128 m0, xm6, 1 vperm2i128 m5, m1, m7, 0x31 - vinserti128 m1, m1, xm7, 1 + vinserti128 m1, xm7, 1 vperm2i128 m6, m2, m9, 0x31 - vinserti128 m2, m2, xm9, 1 + vinserti128 m2, xm9, 1 vperm2i128 m7, m3, m10, 0x31 - vinserti128 m3, m3, xm10, 1 + vinserti128 m3, xm10, 1 call m(idct_16x8_internal).main vpbroadcastd m8, [o(pw_2048)] REPX {pmulhrsw x, m8}, m0, m1, m2, m3, m4, m5, m6, m7 lea r2, [strideq*3] WRITE_16X2 0, 1, 8, 0, strideq*0, strideq*1 @@ -3920,17 +3430,17 @@ punpckldq m6, m7, m4 punpckhdq m7, m4 punpckldq m9, m12, m5 punpckhdq m12, m5 vperm2i128 m4, m0, m6, 0x31 - vinserti128 m0, m0, xm6, 1 + vinserti128 m0, xm6, 1 vperm2i128 m5, m1, m7, 0x31 - vinserti128 m1, m1, xm7, 1 + vinserti128 m1, xm7, 1 vperm2i128 m6, m2, m9, 0x31 - vinserti128 m2, m2, xm9, 1 + vinserti128 m2, xm9, 1 vperm2i128 m7, m3, m12, 0x31 - vinserti128 m3, m3, xm12, 1 + vinserti128 m3, xm12, 1 call m(idct_16x8_internal).main2 vpbroadcastd m8, [o(pw_2048)] REPX {pmulhrsw x, m8}, m0, m1, m2, m3, m4, m5, m6, m7 add r0, 16 add r3, 16 @@ -3945,30 +3455,30 @@ cglobal inv_txfm_add_identity_identity_8x32, 4, 5, 11, dst, stride, c, eob vpbroadcastd m9, [pw_5] lea r4, [strideq*3] sub eobd, 107 ; loop_iterations = 1 + (eobd >= 107) .loop: - mova xm0, [cq+16* 0] - mova xm1, [cq+16* 4] - vinserti128 m0, m0, [cq+16* 1], 1 - vinserti128 m1, m1, [cq+16* 5], 1 + mova xm0,[cq+16* 0] + mova xm1, [cq+16* 4] + vinserti128 m0, [cq+16* 1], 1 + vinserti128 m1, [cq+16* 5], 1 pxor m8, m8 mova [cq+32*0], m8 mova [cq+32*2], m8 add cq, 16*16 - mova xm2, [cq-16* 8] - mova xm3, [cq-16* 4] - vinserti128 m2, m2, [cq-16* 7], 1 - vinserti128 m3, m3, [cq-16* 3], 1 - mova xm4, [cq+16* 0] - mova xm5, [cq+16* 4] - vinserti128 m4, m4, [cq+16* 1], 1 - vinserti128 m5, m5, [cq+16* 5], 1 - mova xm6, [cq+16* 8] - mova xm7, [cq+16*12] - vinserti128 m6, m6, [cq+16* 9], 1 - vinserti128 m7, m7, [cq+16*13], 1 + mova xm2, [cq-16* 8] + mova xm3, [cq-16* 4] + vinserti128 m2, [cq-16* 7], 1 + vinserti128 m3, [cq-16* 3], 1 + mova xm4, [cq+16* 0] + mova xm5, [cq+16* 4] + vinserti128 m4, [cq+16* 1], 1 + vinserti128 m5, [cq+16* 5], 1 + mova xm6, [cq+16* 8] + mova xm7, [cq+16*12] + vinserti128 m6, [cq+16* 9], 1 + vinserti128 m7, [cq+16*13], 1 REPX {mova [cq+32*x], m8}, -4, -2, 0, 2, 4, 6 REPX {paddsw x, m9}, m0, m1, m2, m3, m4, m5, m6, m7 call .transpose8x8 REPX {psraw x, 3 }, m0, m1, m2, m3, m4, m5, m6, m7 WRITE_8X4 0, 4, 8, 10, strideq*8, strideq*4, r4*4 @@ -4017,26 +3527,26 @@ vpbroadcastd m9, [pw_4096] lea r4, [strideq*3] lea r5, [dstq+strideq*4] sub eobd, 107 .loop: - mova xm0, [cq-16*8] - mova xm1, [cq-16*7] - vinserti128 m0, m0, [cq+16*0], 1 - vinserti128 m1, m1, [cq+16*1], 1 - mova xm2, [cq-16*6] - mova xm3, [cq-16*5] - vinserti128 m2, m2, [cq+16*2], 1 - vinserti128 m3, m3, [cq+16*3], 1 - mova xm4, [cq-16*4] - mova xm5, [cq-16*3] - vinserti128 m4, m4, [cq+16*4], 1 - vinserti128 m5, m5, [cq+16*5], 1 - mova xm6, [cq-16*2] - mova xm7, [cq-16*1] - vinserti128 m6, m6, [cq+16*6], 1 - vinserti128 m7, m7, [cq+16*7], 1 + mova xm0, [cq-16*8] + mova xm1, [cq-16*7] + vinserti128 m0, [cq+16*0], 1 + vinserti128 m1, [cq+16*1], 1 + mova xm2, [cq-16*6] + mova xm3, [cq-16*5] + vinserti128 m2, [cq+16*2], 1 + vinserti128 m3, [cq+16*3], 1 + mova xm4, [cq-16*4] + mova xm5, [cq-16*3] + vinserti128 m4, [cq+16*4], 1 + vinserti128 m5, [cq+16*5], 1 + mova xm6, [cq-16*2] + mova xm7, [cq-16*1] + vinserti128 m6, [cq+16*6], 1 + vinserti128 m7, [cq+16*7], 1 pxor m8, m8 REPX {mova [cq+32*x], m8}, -4, -3, -2, -1, 0, 1, 2, 3 call m(inv_txfm_add_identity_identity_8x32).transpose8x8 REPX {pmulhrsw x, m9}, m0, m1, m2, m3, m4, m5, m6, m7 WRITE_16X2 0, 1, 8, 0, strideq*0, strideq*1 @@ -4204,32 +3714,32 @@ vextracti128 [r2+32*2+16], m12, 1 mova [r2-32*1+16], xm14 vextracti128 [r2+32*3+16], m14, 1 vinserti128 m8, m1, xm9, 1 vperm2i128 m12, m1, m9, 0x31 - mova xm0, [tmp1q-32*4] - mova xm1, [tmp1q-32*3] - vinserti128 m0, m0, [tmp1q+32*0], 1 - vinserti128 m1, m1, [tmp1q+32*1], 1 + mova xm0, [tmp1q-32*4] + mova xm1, [tmp1q-32*3] + vinserti128 m0, [tmp1q+32*0], 1 + vinserti128 m1, [tmp1q+32*1], 1 vinserti128 m10, m5, xm13, 1 vperm2i128 m14, m5, m13, 0x31 - mova xm4, [tmp1q-32*4+16] - mova xm5, [tmp1q-32*3+16] - vinserti128 m4, m4, [tmp1q+32*0+16], 1 - vinserti128 m5, m5, [tmp1q+32*1+16], 1 + mova xm4, [tmp1q-32*4+16] + mova xm5, [tmp1q-32*3+16] + vinserti128 m4, [tmp1q+32*0+16], 1 + vinserti128 m5, [tmp1q+32*1+16], 1 vinserti128 m9, m3, xm11, 1 vperm2i128 m13, m3, m11, 0x31 - mova xm2, [tmp1q-32*2] - mova xm3, [tmp1q-32*1] - vinserti128 m2, m2, [tmp1q+32*2], 1 - vinserti128 m3, m3, [tmp1q+32*3], 1 + mova xm2, [tmp1q-32*2] + mova xm3, [tmp1q-32*1] + vinserti128 m2, [tmp1q+32*2], 1 + vinserti128 m3, [tmp1q+32*3], 1 vinserti128 m11, m7, xm15, 1 vperm2i128 m15, m7, m15, 0x31 - mova xm6, [tmp1q-32*2+16] - mova xm7, [tmp1q-32*1+16] - vinserti128 m6, m6, [tmp1q+32*2+16], 1 - vinserti128 m7, m7, [tmp1q+32*3+16], 1 + mova xm6, [tmp1q-32*2+16] + mova xm7, [tmp1q-32*1+16] + vinserti128 m6, [tmp1q+32*2+16], 1 + vinserti128 m7, [tmp1q+32*3+16], 1 call .main_oddhalf LOAD_8ROWS_H r2-32*4, 32 .idct16: LOAD_8ROWS tmp3q-32*4, 32 mova [rsp], m15 @@ -4473,11 +3983,11 @@ paddsw m%4, m%2, m%3 psubsw m%2, m%3 mova [tmp1q+32*(11-%2)], xm%2 vextracti128 [tmp2q+32*( 3-%1)], m%2, 1 vperm2i128 m%2, m%1, m%4, 0x31 - vinserti128 m%1, m%1, xm%4, 1 + vinserti128 m%1, xm%4, 1 %endmacro cglobal inv_txfm_add_dct_dct_32x16, 4, 4, 0, dst, stride, c, eob lea rax, [o_base] test eobd, eobd @@ -4591,26 +4101,26 @@ adc r4b, al ; iteration_count++ lea r3, [strideq*3] mov rax, cq paddw m11, m12, m12 ; pw_16384 .loop: - mova xm0, [cq+64* 0] - mova xm1, [cq+64* 1] - vinserti128 m0, m0, [cq+64* 8], 1 - vinserti128 m1, m1, [cq+64* 9], 1 - mova xm2, [cq+64* 2] - mova xm3, [cq+64* 3] - vinserti128 m2, m2, [cq+64*10], 1 - vinserti128 m3, m3, [cq+64*11], 1 - mova xm4, [cq+64* 4] - mova xm5, [cq+64* 5] - vinserti128 m4, m4, [cq+64*12], 1 - vinserti128 m5, m5, [cq+64*13], 1 - mova xm6, [cq+64* 6] - mova xm7, [cq+64* 7] - vinserti128 m6, m6, [cq+64*14], 1 - vinserti128 m7, m7, [cq+64*15], 1 + mova xm0, [cq+64* 0] + mova xm1, [cq+64* 1] + vinserti128 m0, [cq+64* 8], 1 + vinserti128 m1, [cq+64* 9], 1 + mova xm2, [cq+64* 2] + mova xm3, [cq+64* 3] + vinserti128 m2, [cq+64*10], 1 + vinserti128 m3, [cq+64*11], 1 + mova xm4, [cq+64* 4] + mova xm5, [cq+64* 5] + vinserti128 m4, [cq+64*12], 1 + vinserti128 m5, [cq+64*13], 1 + mova xm6, [cq+64* 6] + mova xm7, [cq+64* 7] + vinserti128 m6, [cq+64*14], 1 + vinserti128 m7, [cq+64*15], 1 REPX {pmulhrsw x, m9 }, m0, m1, m2, m3, m4, m5, m6, m7 REPX {IDTX16 x, 8, 10, 11}, 0, 1, 2, 3, 4, 5, 6, 7 call m(inv_txfm_add_identity_identity_8x32).transpose8x8 REPX {pmulhrsw x, m12}, m0, m1, m2, m3, m4, m5, m6, m7 WRITE_16X2 0, 1, 8, 0, strideq*0, strideq*1 @@ -4659,26 +4169,26 @@ lea r4d, [r4+r3*2] lea r3, [strideq*3] mov r5, dstq mov rax, cq .loop: - mova xm0, [cq+32* 0] - mova xm1, [cq+32* 1] - vinserti128 m0, m0, [cq+32* 8], 1 - vinserti128 m1, m1, [cq+32* 9], 1 - mova xm2, [cq+32* 2] - mova xm3, [cq+32* 3] - vinserti128 m2, m2, [cq+32*10], 1 - vinserti128 m3, m3, [cq+32*11], 1 - mova xm4, [cq+32* 4] - mova xm5, [cq+32* 5] - vinserti128 m4, m4, [cq+32*12], 1 - vinserti128 m5, m5, [cq+32*13], 1 - mova xm6, [cq+32* 6] - mova xm7, [cq+32* 7] - vinserti128 m6, m6, [cq+32*14], 1 - vinserti128 m7, m7, [cq+32*15], 1 + mova xm0, [cq+32* 0] + mova xm1, [cq+32* 1] + vinserti128 m0, [cq+32* 8], 1 + vinserti128 m1, [cq+32* 9], 1 + mova xm2, [cq+32* 2] + mova xm3, [cq+32* 3] + vinserti128 m2, [cq+32*10], 1 + vinserti128 m3, [cq+32*11], 1 + mova xm4, [cq+32* 4] + mova xm5, [cq+32* 5] + vinserti128 m4, [cq+32*12], 1 + vinserti128 m5, [cq+32*13], 1 + mova xm6, [cq+32* 6] + mova xm7, [cq+32* 7] + vinserti128 m6, [cq+32*14], 1 + vinserti128 m7, [cq+32*15], 1 REPX {pmulhrsw x, m9 }, m0, m1, m2, m3, m4, m5, m6, m7 REPX {paddsw x, x }, m0, m1, m2, m3, m4, m5, m6, m7 call m(inv_txfm_add_identity_identity_8x32).transpose8x8 REPX {IDTX16 x, 8, 10}, 0, 1, 2, 3, 4, 5, 6, 7 REPX {pmulhrsw x, m11}, m0, m1, m2, m3, m4, m5, m6, m7 @@ -4862,26 +4372,26 @@ lea eobd, [eobq*2-8] lea r4, [strideq*3] mov r5, dstq lea rax, [cq+32] .loop: - mova xm0, [cq+64* 0] - mova xm1, [cq+64* 1] - vinserti128 m0, m0, [cq+64* 8], 1 - vinserti128 m1, m1, [cq+64* 9], 1 - mova xm2, [cq+64* 2] - mova xm3, [cq+64* 3] - vinserti128 m2, m2, [cq+64*10], 1 - vinserti128 m3, m3, [cq+64*11], 1 - mova xm4, [cq+64* 4] - mova xm5, [cq+64* 5] - vinserti128 m4, m4, [cq+64*12], 1 - vinserti128 m5, m5, [cq+64*13], 1 - mova xm6, [cq+64* 6] - mova xm7, [cq+64* 7] - vinserti128 m6, m6, [cq+64*14], 1 - vinserti128 m7, m7, [cq+64*15], 1 + mova xm0, [cq+64* 0] + mova xm1, [cq+64* 1] + vinserti128 m0, [cq+64* 8], 1 + vinserti128 m1, [cq+64* 9], 1 + mova xm2, [cq+64* 2] + mova xm3, [cq+64* 3] + vinserti128 m2, [cq+64*10], 1 + vinserti128 m3, [cq+64*11], 1 + mova xm4, [cq+64* 4] + mova xm5, [cq+64* 5] + vinserti128 m4, [cq+64*12], 1 + vinserti128 m5, [cq+64*13], 1 + mova xm6, [cq+64* 6] + mova xm7, [cq+64* 7] + vinserti128 m6, [cq+64*14], 1 + vinserti128 m7, [cq+64*15], 1 call m(inv_txfm_add_identity_identity_8x32).transpose8x8 REPX {pmulhrsw x, m9}, m0, m1, m2, m3, m4, m5, m6, m7 WRITE_16X2 0, 1, 8, 0, strideq*0, strideq*1 WRITE_16X2 2, 3, 0, 1, strideq*2, r4 lea dstq, [dstq+strideq*4] @@ -5020,31 +4530,31 @@ add tmp1q, 32*8 add tmp2q, 32*8 add eobd, 0x80000000 jnc .pass1_loop lea r2, [rsp+32*23] - mova xm0, [r2-32*4+ 0] - mova xm1, [r2-32*2+ 0] - vinserti128 m0, m0, [r2+32*0+ 0], 1 - vinserti128 m1, m1, [r2+32*2+ 0], 1 - mova xm2, [r2-32*4+16] - mova xm3, [r2-32*2+16] - vinserti128 m2, m2, [r2+32*0+16], 1 - vinserti128 m3, m3, [r2+32*2+16], 1 + mova xm0, [r2-32*4+ 0] + mova xm1, [r2-32*2+ 0] + vinserti128 m0, [r2+32*0+ 0], 1 + vinserti128 m1, [r2+32*2+ 0], 1 + mova xm2, [r2-32*4+16] + mova xm3, [r2-32*2+16] + vinserti128 m2, [r2+32*0+16], 1 + vinserti128 m3, [r2+32*2+16], 1 pxor m4, m4 REPX {mova x, m4}, m5, m6, m7, m8, m9, m10, m11, m12, m13, m14 test r7d, r7d jl .fast lea r3, [r2+32*8] - mova xm4, [r3-32*4+ 0] - mova xm5, [r3-32*2+ 0] - vinserti128 m4, m4, [r3+32*0+ 0], 1 - vinserti128 m5, m5, [r3+32*2+ 0], 1 - mova xm6, [r3-32*4+16] - mova xm7, [r3-32*2+16] - vinserti128 m6, m6, [r3+32*0+16], 1 - vinserti128 m7, m7, [r3+32*2+16], 1 + mova xm4, [r3-32*4+ 0] + mova xm5, [r3-32*2+ 0] + vinserti128 m4, [r3+32*0+ 0], 1 + vinserti128 m5, [r3+32*2+ 0], 1 + mova xm6, [r3-32*4+16] + mova xm7, [r3-32*2+16] + vinserti128 m6, [r3+32*0+16], 1 + vinserti128 m7, [r3+32*2+16], 1 .fast: mova [rsp], m8 lea tmp1q, [rsp+32*7] call m(idct_16x16_internal).main mova m1, [rsp+32*1] @@ -5063,85 +4573,85 @@ mova [tmp1q-32*1], m11 mova [tmp1q+32*0], m12 mova [tmp1q+32*1], m13 mova [tmp1q+32*2], m14 mova [tmp1q+32*3], m15 - mova xm0, [r2-32*3+ 0] - mova xm1, [r2-32*1+ 0] - vinserti128 m0, m0, [r2+32*1+ 0], 1 - vinserti128 m1, m1, [r2+32*3+ 0], 1 - mova xm2, [r2-32*3+16] - mova xm3, [r2-32*1+16] - vinserti128 m2, m2, [r2+32*1+16], 1 - vinserti128 m3, m3, [r2+32*3+16], 1 + mova xm0, [r2-32*3+ 0] + mova xm1, [r2-32*1+ 0] + vinserti128 m0, [r2+32*1+ 0], 1 + vinserti128 m1, [r2+32*3+ 0], 1 + mova xm2, [r2-32*3+16] + mova xm3, [r2-32*1+16] + vinserti128 m2, [r2+32*1+16], 1 + vinserti128 m3, [r2+32*3+16], 1 pxor m4, m4 REPX {mova x, m4}, m5, m6, m7 test r7d, r7d jl .fast2 - mova xm4, [r3-32*3+ 0] - mova xm5, [r3-32*1+ 0] - vinserti128 m4, m4, [r3+32*1+ 0], 1 - vinserti128 m5, m5, [r3+32*3+ 0], 1 - mova xm6, [r3-32*3+16] - mova xm7, [r3-32*1+16] - vinserti128 m6, m6, [r3+32*1+16], 1 - vinserti128 m7, m7, [r3+32*3+16], 1 + mova xm4, [r3-32*3+ 0] + mova xm5, [r3-32*1+ 0] + vinserti128 m4, [r3+32*1+ 0], 1 + vinserti128 m5, [r3+32*3+ 0], 1 + mova xm6, [r3-32*3+16] + mova xm7, [r3-32*1+16] + vinserti128 m6, [r3+32*1+16], 1 + vinserti128 m7, [r3+32*3+16], 1 .fast2: add tmp1q, 32*8 lea tmp2q, [tmp1q+32*8] call m(inv_txfm_add_dct_dct_16x32).main_oddhalf_fast add r2, 32*24 vpbroadcastd m15, [o(pd_2048)] add tmp1q, 32*16 add tmp2q, 32*32 - mova xm0, [r2-32*4+ 0] - mova xm3, [r2-32*1+16] - vinserti128 m0, m0, [r2+32*0+ 0], 1 - vinserti128 m3, m3, [r2+32*3+16], 1 - mova xm4, [r2-32*4+16] - mova xm7, [r2-32*1+ 0] - vinserti128 m4, m4, [r2+32*0+16], 1 - vinserti128 m7, m7, [r2+32*3+ 0], 1 + mova xm0, [r2-32*4+ 0] + mova xm3, [r2-32*1+16] + vinserti128 m0, [r2+32*0+ 0], 1 + vinserti128 m3, [r2+32*3+16], 1 + mova xm4, [r2-32*4+16] + mova xm7, [r2-32*1+ 0] + vinserti128 m4, [r2+32*0+16], 1 + vinserti128 m7, [r2+32*3+ 0], 1 pxor m1, m1 REPX {mova x, m1}, m2, m5, m6 test r7d, r7d jl .fast3 add r3, 32*24 - mova xm1, [r3-32*1+16] - mova xm2, [r3-32*4+ 0] - vinserti128 m1, m1, [r3+32*3+16], 1 - vinserti128 m2, m2, [r3+32*0+ 0], 1 - mova xm5, [r3-32*1+ 0] - mova xm6, [r3-32*4+16] - vinserti128 m5, m5, [r3+32*3+ 0], 1 - vinserti128 m6, m6, [r3+32*0+16], 1 + mova xm1, [r3-32*1+16] + mova xm2, [r3-32*4+ 0] + vinserti128 m1, [r3+32*3+16], 1 + vinserti128 m2, [r3+32*0+ 0], 1 + mova xm5, [r3-32*1+ 0] + mova xm6, [r3-32*4+16] + vinserti128 m5, [r3+32*3+ 0], 1 + vinserti128 m6, [r3+32*0+16], 1 .fast3: add rax, o_idct64_offset call m(inv_txfm_add_dct_dct_16x64).main_part1 add rax, 8 add tmp1q, 32*8 sub tmp2q, 32*8 - mova xm0, [r2-32*2+ 0] - mova xm3, [r2-32*3+16] - vinserti128 m0, m0, [r2+32*2+ 0], 1 - vinserti128 m3, m3, [r2+32*1+16], 1 - mova xm4, [r2-32*2+16] - mova xm7, [r2-32*3+ 0] - vinserti128 m4, m4, [r2+32*2+16], 1 - vinserti128 m7, m7, [r2+32*1+ 0], 1 + mova xm0, [r2-32*2+ 0] + mova xm3, [r2-32*3+16] + vinserti128 m0, [r2+32*2+ 0], 1 + vinserti128 m3, [r2+32*1+16], 1 + mova xm4, [r2-32*2+16] + mova xm7, [r2-32*3+ 0] + vinserti128 m4, [r2+32*2+16], 1 + vinserti128 m7, [r2+32*1+ 0], 1 pxor m1, m1 REPX {mova x, m1}, m2, m5, m6 test r7d, r7d jl .fast4 - mova xm1, [r3-32*3+16] - mova xm2, [r3-32*2+ 0] - vinserti128 m1, m1, [r3+32*1+16], 1 - vinserti128 m2, m2, [r3+32*2+ 0], 1 - mova xm5, [r3-32*3+ 0] - mova xm6, [r3-32*2+16] - vinserti128 m5, m5, [r3+32*1+ 0], 1 - vinserti128 m6, m6, [r3+32*2+16], 1 + mova xm1, [r3-32*3+16] + mova xm2, [r3-32*2+ 0] + vinserti128 m1, [r3+32*1+16], 1 + vinserti128 m2, [r3+32*2+ 0], 1 + mova xm5, [r3-32*3+ 0] + mova xm6, [r3-32*2+16] + vinserti128 m5, [r3+32*1+ 0], 1 + vinserti128 m6, [r3+32*2+16], 1 .fast4: call m(inv_txfm_add_dct_dct_16x64).main_part1 call m(inv_txfm_add_dct_dct_16x64).main_part2_pass2 RET ALIGN function_align @@ -5421,42 +4931,42 @@ sub tmp1q, 32*36 lea r2, [strideq*3] mov tmp2d, 4 .pass2_loop: lea r3, [tmp1q-32*8] - mova xm0, [r3 -32*4] - mova xm1, [r3 -32*3] - vinserti128 m0, m0, [tmp1q-32*4], 1 - vinserti128 m1, m1, [tmp1q-32*3], 1 - mova xm2, [r3 -32*2] - mova xm3, [r3 -32*1] - vinserti128 m2, m2, [tmp1q-32*2], 1 - vinserti128 m3, m3, [tmp1q-32*1], 1 - mova xm4, [r3 +32*0] - mova xm5, [r3 +32*1] - vinserti128 m4, m4, [tmp1q+32*0], 1 - vinserti128 m5, m5, [tmp1q+32*1], 1 - mova xm6, [r3 +32*2] - mova xm7, [r3 +32*3] - vinserti128 m6, m6, [tmp1q+32*2], 1 - vinserti128 m7, m7, [tmp1q+32*3], 1 - mova xm8, [r3 -32*4+16] - mova xm9, [r3 -32*3+16] - vinserti128 m8, m8, [tmp1q-32*4+16], 1 - vinserti128 m9, m9, [tmp1q-32*3+16], 1 - mova xm10, [r3 -32*2+16] - mova xm11, [r3 -32*1+16] - vinserti128 m10, m10, [tmp1q-32*2+16], 1 - vinserti128 m11, m11, [tmp1q-32*1+16], 1 - mova xm12, [r3 +32*0+16] - mova xm13, [r3 +32*1+16] - vinserti128 m12, m12, [tmp1q+32*0+16], 1 - vinserti128 m13, m13, [tmp1q+32*1+16], 1 - mova xm14, [r3 +32*2+16] - mova xm15, [r3 +32*3+16] - vinserti128 m14, m14, [tmp1q+32*2+16], 1 - vinserti128 m15, m15, [tmp1q+32*3+16], 1 + mova xm0, [r3 -32*4] + mova xm1, [r3 -32*3] + vinserti128 m0, [tmp1q-32*4], 1 + vinserti128 m1, [tmp1q-32*3], 1 + mova xm2, [r3 -32*2] + mova xm3, [r3 -32*1] + vinserti128 m2, [tmp1q-32*2], 1 + vinserti128 m3, [tmp1q-32*1], 1 + mova xm4, [r3 +32*0] + mova xm5, [r3 +32*1] + vinserti128 m4, [tmp1q+32*0], 1 + vinserti128 m5, [tmp1q+32*1], 1 + mova xm6, [r3 +32*2] + mova xm7, [r3 +32*3] + vinserti128 m6, [tmp1q+32*2], 1 + vinserti128 m7, [tmp1q+32*3], 1 + mova xm8, [r3 -32*4+16] + mova xm9, [r3 -32*3+16] + vinserti128 m8, [tmp1q-32*4+16], 1 + vinserti128 m9, [tmp1q-32*3+16], 1 + mova xm10, [r3 -32*2+16] + mova xm11, [r3 -32*1+16] + vinserti128 m10, [tmp1q-32*2+16], 1 + vinserti128 m11, [tmp1q-32*1+16], 1 + mova xm12, [r3 +32*0+16] + mova xm13, [r3 +32*1+16] + vinserti128 m12, [tmp1q+32*0+16], 1 + vinserti128 m13, [tmp1q+32*1+16], 1 + mova xm14, [r3 +32*2+16] + mova xm15, [r3 +32*3+16] + vinserti128 m14, [tmp1q+32*2+16], 1 + vinserti128 m15, [tmp1q+32*3+16], 1 mova [rsp+32*0], m6 mova [rsp+32*1], m7 vpbroadcastd m7, [o(pw_8192)] call m(inv_txfm_add_dct_dct_16x32).transpose_2x8x8_round call m(idct_16x16_internal).main @@ -5808,52 +5318,52 @@ ALIGN function_align .transpose_round_interleave: mov tmp3d, 4 .loop: lea tmp2q, [tmp1q+32*8] - mova xm0, [tmp1q-32*4] - mova xm1, [tmp1q-32*3] - vinserti128 m0, m0, [tmp2q-32*4], 1 - vinserti128 m1, m1, [tmp2q-32*3], 1 - mova xm2, [tmp1q-32*2] - mova xm3, [tmp1q-32*1] - vinserti128 m2, m2, [tmp2q-32*2], 1 - vinserti128 m3, m3, [tmp2q-32*1], 1 - mova xm4, [tmp1q+32*0] - mova xm5, [tmp1q+32*1] - vinserti128 m4, m4, [tmp2q+32*0], 1 - vinserti128 m5, m5, [tmp2q+32*1], 1 - mova xm6, [tmp1q+32*2] - mova xm7, [tmp1q+32*3] - vinserti128 m6, m6, [tmp2q+32*2], 1 - vinserti128 m7, m7, [tmp2q+32*3], 1 + mova xm0, [tmp1q-32*4] + mova xm1, [tmp1q-32*3] + vinserti128 m0, [tmp2q-32*4], 1 + vinserti128 m1, [tmp2q-32*3], 1 + mova xm2, [tmp1q-32*2] + mova xm3, [tmp1q-32*1] + vinserti128 m2, [tmp2q-32*2], 1 + vinserti128 m3, [tmp2q-32*1], 1 + mova xm4, [tmp1q+32*0] + mova xm5, [tmp1q+32*1] + vinserti128 m4, [tmp2q+32*0], 1 + vinserti128 m5, [tmp2q+32*1], 1 + mova xm6, [tmp1q+32*2] + mova xm7, [tmp1q+32*3] + vinserti128 m6, [tmp2q+32*2], 1 + vinserti128 m7, [tmp2q+32*3], 1 REPX {pmulhrsw x, m10}, m0, m1, m2, m3, m4, m5, m6, m7 call m(inv_txfm_add_identity_identity_8x32).transpose8x8 - mova xm8, [tmp1q-32*4+16] - mova xm9, [tmp1q-32*3+16] - vinserti128 m8, m8, [tmp2q-32*4+16], 1 - vinserti128 m9, m9, [tmp2q-32*3+16], 1 + mova xm8, [tmp1q-32*4+16] + mova xm9, [tmp1q-32*3+16] + vinserti128 m8, [tmp2q-32*4+16], 1 + vinserti128 m9, [tmp2q-32*3+16], 1 mova [tmp1q-32*4], m0 mova [tmp2q-32*4], m1 mova [tmp1q-32*3], m2 mova [tmp2q-32*3], m3 - mova xm2, [tmp1q-32*2+16] - mova xm3, [tmp1q-32*1+16] - vinserti128 m2, m2, [tmp2q-32*2+16], 1 - vinserti128 m3, m3, [tmp2q-32*1+16], 1 + mova xm2, [tmp1q-32*2+16] + mova xm3, [tmp1q-32*1+16] + vinserti128 m2, [tmp2q-32*2+16], 1 + vinserti128 m3, [tmp2q-32*1+16], 1 mova [tmp1q-32*2], m4 mova [tmp2q-32*2], m5 mova [tmp1q-32*1], m6 mova [tmp2q-32*1], m7 - mova xm4, [tmp1q+32*0+16] - mova xm5, [tmp1q+32*1+16] - vinserti128 m4, m4, [tmp2q+32*0+16], 1 - vinserti128 m5, m5, [tmp2q+32*1+16], 1 - mova xm6, [tmp1q+32*2+16] - mova xm7, [tmp1q+32*3+16] - vinserti128 m6, m6, [tmp2q+32*2+16], 1 - vinserti128 m7, m7, [tmp2q+32*3+16], 1 + mova xm4, [tmp1q+32*0+16] + mova xm5, [tmp1q+32*1+16] + vinserti128 m4, [tmp2q+32*0+16], 1 + vinserti128 m5, [tmp2q+32*1+16], 1 + mova xm6, [tmp1q+32*2+16] + mova xm7, [tmp1q+32*3+16] + vinserti128 m6, [tmp2q+32*2+16], 1 + vinserti128 m7, [tmp2q+32*3+16], 1 pmulhrsw m0, m8, m10 pmulhrsw m1, m9, m10 REPX {pmulhrsw x, m10}, m2, m3, m4, m5, m6, m7 call m(inv_txfm_add_identity_identity_8x32).transpose8x8 mova [tmp1q+32*0], m0 diff --git a/third_party/dav1d/src/x86/itx_ssse3.asm b/third_party/dav1d/src/x86/itx_ssse3.asm --- a/third_party/dav1d/src/x86/itx_ssse3.asm +++ b/third_party/dav1d/src/x86/itx_ssse3.asm @@ -137,15 +137,10 @@ pw_3102x8: times 8 dw 3102*8 pw_2675x8: times 8 dw 2675*8 pw_4085x8: times 8 dw 4085*8 pw_m301x8: times 8 dw -301*8 -iadst4_dconly1a: times 2 dw 10568, 19856, 26752, 30424 -iadst4_dconly1b: times 2 dw 30424, 26752, 19856, 10568 -iadst4_dconly2a: dw 10568, 10568, 10568, 10568, 19856, 19856, 19856, 19856 -iadst4_dconly2b: dw 26752, 26752, 26752, 26752, 30424, 30424, 30424, 30424 - SECTION .text %macro REPX 2-* %xdefine %%f(x) %1 %rep %0 - 1 @@ -241,102 +236,58 @@ ITX_MUL2X_PACK 0, 1, 3, 2896, 2896 psubsw m1, m0, m2 ;high: out2 ;low: out3 paddsw m0, m2 ;high: out1 ;low: out0 %endmacro -%macro INV_TXFM_FN 5+ ; type1, type2, fast_thresh, size, xmm/stack -cglobal inv_txfm_add_%1_%2_%4, 4, 6, %5, dst, stride, coeff, eob, tx2 - %undef cmp - %define %%p1 m(i%1_%4_internal) +%macro INV_TXFM_FN 4+ ; type1, type2, size, xmm/stack +cglobal inv_txfm_add_%1_%2_%3, 4, 6, %4, dst, stride, coeff, eob, tx2 + %define %%p1 m(i%1_%3_internal) %if ARCH_X86_32 LEA r5, $$ %endif %if has_epilogue -%if %3 > 0 - cmp eobd, %3 - jle %%end -%elif %3 == 0 +%ifidn %1_%2, dct_dct test eobd, eobd jz %%end %endif - lea tx2q, [o(m(i%2_%4_internal).pass2)] + lea tx2q, [o(m(i%2_%3_internal).pass2)] call %%p1 RET %%end: %else - lea tx2q, [o(m(i%2_%4_internal).pass2)] -%if %3 > 0 - cmp eobd, %3 - jg %%p1 -%elif %3 == 0 + lea tx2q, [o(m(i%2_%3_internal).pass2)] +%ifidn %1_%2, dct_dct test eobd, eobd jnz %%p1 %else times ((%%end - %%p1) >> 31) & 1 jmp %%p1 ALIGN function_align %%end: %endif %endif %endmacro -%macro INV_TXFM_4X4_FN 2-3 -1 ; type1, type2, fast_thresh - INV_TXFM_FN %1, %2, %3, 4x4, 6 -%ifidn %1_%2, dct_identity - mova m0, [o(pw_2896x8)] - pmulhrsw m0, [coeffq] - pmulhrsw m1, m0, [o(pw_1697x8)] - paddsw m0, m1 - punpcklwd m0, m0 - punpckhdq m1, m0, m0 - punpckldq m0, m0 - TAIL_CALL m(iadst_4x4_internal).end -%elifidn %1_%2, identity_dct - mova m1, [coeffq+16*0] - mova m2, [coeffq+16*1] - punpcklwd m0, m1, m2 - punpckhwd m1, m2 - punpcklwd m0, m1 - punpcklqdq m0, m0 - pmulhrsw m1, m0, [o(pw_1697x8)] - paddsw m0, m1 - pmulhrsw m0, [o(pw_2896x8)] - mova m1, m0 - TAIL_CALL m(iadst_4x4_internal).end -%elif %3 >= 0 +%macro INV_TXFM_4X4_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 4x4, 6 +%ifidn %1_%2, dct_dct pshuflw m0, [coeffq], q0000 punpcklqdq m0, m0 -%ifidn %1, dct mova m1, [o(pw_2896x8)] pmulhrsw m0, m1 -%elifidn %1, adst - pmulhrsw m0, [o(iadst4_dconly1a)] -%elifidn %1, flipadst - pmulhrsw m0, [o(iadst4_dconly1b)] -%endif mov [coeffq], eobd ;0 -%ifidn %2, dct -%ifnidn %1, dct - pmulhrsw m0, [o(pw_2896x8)] -%else pmulhrsw m0, m1 -%endif mova m1, m0 TAIL_CALL m(iadst_4x4_internal).end2 -%else ; adst / flipadst - pmulhrsw m1, m0, [o(iadst4_dconly2b)] - pmulhrsw m0, [o(iadst4_dconly2a)] - TAIL_CALL m(i%2_4x4_internal).end2 -%endif %endif %endmacro INIT_XMM ssse3 -INV_TXFM_4X4_FN dct, dct, 0 -INV_TXFM_4X4_FN dct, adst, 0 -INV_TXFM_4X4_FN dct, flipadst, 0 -INV_TXFM_4X4_FN dct, identity, 3 +INV_TXFM_4X4_FN dct, dct +INV_TXFM_4X4_FN dct, adst +INV_TXFM_4X4_FN dct, flipadst +INV_TXFM_4X4_FN dct, identity cglobal idct_4x4_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m0, [coeffq+16*0] ;high: in1 ;low: in0 mova m1, [coeffq+16*1] ;high: in3 ;low in2 @@ -356,13 +307,13 @@ mova [coeffq+16*0], m2 mova [coeffq+16*1], m2 ;memset(coeff, 0, sizeof(*coeff) * sh * sw); ITX4_END 0, 1, 3, 2 -INV_TXFM_4X4_FN adst, dct, 0 -INV_TXFM_4X4_FN adst, adst, 0 -INV_TXFM_4X4_FN adst, flipadst, 0 +INV_TXFM_4X4_FN adst, dct +INV_TXFM_4X4_FN adst, adst +INV_TXFM_4X4_FN adst, flipadst INV_TXFM_4X4_FN adst, identity cglobal iadst_4x4_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m0, [coeffq+16*0] mova m1, [coeffq+16*1] @@ -408,13 +359,13 @@ REPX {psrad x, 12}, m1, m0, m5, m2 packssdw m0, m5 ;high: out1 ;low: out0 packssdw m1, m2 ;high: out3 ;low: out3 ret -INV_TXFM_4X4_FN flipadst, dct, 0 -INV_TXFM_4X4_FN flipadst, adst, 0 -INV_TXFM_4X4_FN flipadst, flipadst, 0 +INV_TXFM_4X4_FN flipadst, dct +INV_TXFM_4X4_FN flipadst, adst +INV_TXFM_4X4_FN flipadst, flipadst INV_TXFM_4X4_FN flipadst, identity cglobal iflipadst_4x4_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m0, [coeffq+16*0] mova m1, [coeffq+16*1] @@ -434,11 +385,11 @@ mova [coeffq+16*1], m2 .end2: ITX4_END 3, 2, 1, 0 -INV_TXFM_4X4_FN identity, dct, 3 +INV_TXFM_4X4_FN identity, dct INV_TXFM_4X4_FN identity, adst INV_TXFM_4X4_FN identity, flipadst INV_TXFM_4X4_FN identity, identity cglobal iidentity_4x4_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 @@ -593,43 +544,13 @@ punpckldq m0, m2 ;low: in0 high: in1 punpckldq m2, m3, m4 ;low: in4 high: in5 punpckhdq m3, m4 ;low: in6 high: in7 %endmacro -%macro INV_TXFM_4X8_FN 2-3 -1 ; type1, type2, fast_thresh - INV_TXFM_FN %1, %2, %3, 4x8, 8 -%if %3 >= 0 -%ifidn %1_%2, dct_identity - mova m1, [o(pw_2896x8)] - pmulhrsw m0, m1, [coeffq] - pmulhrsw m0, m1 - pmulhrsw m0, [o(pw_4096)] - punpckhwd m2, m0, m0 - punpcklwd m0, m0 - punpckhdq m1, m0, m0 - punpckldq m0, m0 - punpckhdq m3, m2, m2 - punpckldq m2, m2 - TAIL_CALL m(iadst_4x8_internal).end3 -%elifidn %1_%2, identity_dct - movd m0, [coeffq+16*0] - punpcklwd m0, [coeffq+16*1] - movd m1, [coeffq+16*2] - punpcklwd m1, [coeffq+16*3] - mova m2, [o(pw_2896x8)] - punpckldq m0, m1 - pmulhrsw m0, m2 - pmulhrsw m1, m0, [o(pw_1697x8)] - paddsw m0, m1 - pmulhrsw m0, m2 - pmulhrsw m0, [o(pw_2048)] - punpcklqdq m0, m0 - mova m1, m0 - mova m2, m0 - mova m3, m0 - TAIL_CALL m(iadst_4x8_internal).end3 -%elifidn %1_%2, dct_dct +%macro INV_TXFM_4X8_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 4x8, 8 +%ifidn %1_%2, dct_dct pshuflw m0, [coeffq], q0000 punpcklqdq m0, m0 mova m1, [o(pw_2896x8)] pmulhrsw m0, m1 mov [coeffq], eobd @@ -637,36 +558,18 @@ pmulhrsw m0, m1 pmulhrsw m0, [o(pw_2048)] mova m1, m0 mova m2, m0 mova m3, m0 - TAIL_CALL m(iadst_4x8_internal).end4 -%else ; adst_dct / flipadst_dct - pshuflw m0, [coeffq], q0000 - punpcklqdq m0, m0 - mova m1, [o(pw_2896x8)] - pmulhrsw m0, m1 -%ifidn %1, adst - pmulhrsw m0, [o(iadst4_dconly1a)] -%else ; flipadst - pmulhrsw m0, [o(iadst4_dconly1b)] -%endif - mov [coeffq], eobd - pmulhrsw m0, m1 - pmulhrsw m0, [o(pw_2048)] - mova m1, m0 - mova m2, m0 - mova m3, m0 - TAIL_CALL m(iadst_4x8_internal).end4 -%endif + TAIL_CALL m(iadst_4x8_internal).end3 %endif %endmacro -INV_TXFM_4X8_FN dct, dct, 0 -INV_TXFM_4X8_FN dct, identity, 7 +INV_TXFM_4X8_FN dct, dct INV_TXFM_4X8_FN dct, adst INV_TXFM_4X8_FN dct, flipadst +INV_TXFM_4X8_FN dct, identity cglobal idct_4x8_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m3, [o(pw_2896x8)] pmulhrsw m0, m3, [coeffq+16*0] pmulhrsw m1, m3, [coeffq+16*1] @@ -688,11 +591,11 @@ .main: IDCT8_1D_PACKED ret -INV_TXFM_4X8_FN adst, dct, 0 +INV_TXFM_4X8_FN adst, dct INV_TXFM_4X8_FN adst, adst INV_TXFM_4X8_FN adst, flipadst INV_TXFM_4X8_FN adst, identity cglobal iadst_4x8_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 @@ -723,19 +626,17 @@ .end2: pmulhrsw m0, m4 pmulhrsw m1, m4 pmulhrsw m2, m4 pmulhrsw m3, m4 - -.end3: pxor m5, m5 mova [coeffq+16*0], m5 mova [coeffq+16*1], m5 mova [coeffq+16*2], m5 mova [coeffq+16*3], m5 -.end4: +.end3: WRITE_4X8 0, 1, 2, 3 RET ALIGN function_align .main: @@ -781,11 +682,11 @@ REPX {psrad x, 12}, m4, m2, m1, m5 packssdw m1, m5 ;low: out2 high: -out3 packssdw m2, m4 ;low: out4 high: -out5 ret -INV_TXFM_4X8_FN flipadst, dct, 0 +INV_TXFM_4X8_FN flipadst, dct INV_TXFM_4X8_FN flipadst, adst INV_TXFM_4X8_FN flipadst, flipadst INV_TXFM_4X8_FN flipadst, identity cglobal iflipadst_4x8_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 @@ -822,11 +723,11 @@ mova m5, [o(pw_2048)] pxor m4, m4 psubw m4, m5 jmp m(iadst_4x8_internal).end -INV_TXFM_4X8_FN identity, dct, 3 +INV_TXFM_4X8_FN identity, dct INV_TXFM_4X8_FN identity, adst INV_TXFM_4X8_FN identity, flipadst INV_TXFM_4X8_FN identity, identity cglobal iidentity_4x8_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 @@ -879,88 +780,32 @@ WRITE_8X2 %1, %2, %5, %6, %7 lea dstq, [dstq+strideq*2] WRITE_8X2 %3, %4, %5, %6, %7 %endmacro -%macro INV_TXFM_8X4_FN 2-3 -1 ; type1, type2, fast_thresh - INV_TXFM_FN %1, %2, %3, 8x4, 8 -%if %3 >= 0 -%ifidn %1_%2, dct_identity - mova m0, [o(pw_2896x8)] - pmulhrsw m1, m0, [coeffq] - pmulhrsw m1, m0 - pmulhrsw m0, m1, [o(pw_1697x8)] - paddsw m1, m0 - pmulhrsw m1, [o(pw_2048)] - punpcklwd m1, m1 - punpckhdq m2, m1, m1 - punpckldq m1, m1 - punpckhdq m3, m2, m2 - punpckldq m2, m2 - punpckldq m0, m1, m1 - punpckhdq m1, m1 -%elifidn %1_%2, identity_dct - mova m0, [coeffq+16*0] - mova m1, [coeffq+16*1] - mova m2, [coeffq+16*2] - mova m3, [coeffq+16*3] - punpckhwd m4, m0, m1 - punpcklwd m0, m1 - punpckhwd m5, m2, m3 - punpcklwd m2, m3 - punpcklwd m0, m4 - punpcklwd m2, m5 - punpcklqdq m0, m2 - mova m4, [o(pw_2896x8)] - pmulhrsw m0, m4 - paddsw m0, m0 - pmulhrsw m0, m4 - pmulhrsw m0, [o(pw_2048)] - mova m1, m0 - mova m2, m0 - mova m3, m0 -%else +%macro INV_TXFM_8X4_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 8x4, 8 +%ifidn %1_%2, dct_dct pshuflw m0, [coeffq], q0000 punpcklqdq m0, m0 mova m1, [o(pw_2896x8)] pmulhrsw m0, m1 pmulhrsw m0, m1 -%ifidn %2, dct mova m2, [o(pw_2048)] pmulhrsw m0, m1 pmulhrsw m0, m2 mova m1, m0 mova m2, m0 mova m3, m0 -%else ; adst / flipadst - pmulhrsw m2, m0, [o(iadst4_dconly2b)] - pmulhrsw m0, [o(iadst4_dconly2a)] - mova m1, [o(pw_2048)] - pmulhrsw m0, m1 - pmulhrsw m2, m1 -%ifidn %2, adst - punpckhqdq m1, m0, m0 - punpcklqdq m0, m0 - punpckhqdq m3, m2, m2 - punpcklqdq m2, m2 -%else ; flipadst - mova m3, m0 - punpckhqdq m0, m2, m2 - punpcklqdq m1, m2, m2 - punpckhqdq m2, m3, m3 - punpcklqdq m3, m3 -%endif -%endif -%endif TAIL_CALL m(iadst_8x4_internal).end2 %endif %endmacro -INV_TXFM_8X4_FN dct, dct, 0 -INV_TXFM_8X4_FN dct, adst, 0 -INV_TXFM_8X4_FN dct, flipadst, 0 -INV_TXFM_8X4_FN dct, identity, 3 +INV_TXFM_8X4_FN dct, dct +INV_TXFM_8X4_FN dct, adst +INV_TXFM_8X4_FN dct, flipadst +INV_TXFM_8X4_FN dct, identity cglobal idct_8x4_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 mova m3, [o(pw_2896x8)] pmulhrsw m0, m3, [coeffq+16*0] pmulhrsw m1, m3, [coeffq+16*1] @@ -1155,11 +1000,11 @@ mova m1, m2 mova m2, m5 mova m3, m4 jmp m(iadst_8x4_internal).end -INV_TXFM_8X4_FN identity, dct, 7 +INV_TXFM_8X4_FN identity, dct INV_TXFM_8X4_FN identity, adst INV_TXFM_8X4_FN identity, flipadst INV_TXFM_8X4_FN identity, identity cglobal iidentity_8x4_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 @@ -1197,34 +1042,13 @@ paddsw m1, m5 paddsw m2, m6 paddsw m3, m7 jmp m(iadst_8x4_internal).end -%macro INV_TXFM_8X8_FN 2-3 -1 ; type1, type2, fast_thresh - INV_TXFM_FN %1, %2, %3, 8x8, 8, 16*4 -%ifidn %1_%2, dct_identity - mova m0, [o(pw_2896x8)] - pmulhrsw m0, [coeffq] - mova m1, [o(pw_16384)] - pmulhrsw m0, m1 - psrlw m1, 2 - pmulhrsw m0, m1 - punpckhwd m7, m0, m0 - punpcklwd m0, m0 - pshufd m3, m0, q3333 - pshufd m2, m0, q2222 - pshufd m1, m0, q1111 - pshufd m0, m0, q0000 - call m(iadst_8x4_internal).end2 - pshufd m3, m7, q3333 - pshufd m2, m7, q2222 - pshufd m1, m7, q1111 - pshufd m0, m7, q0000 - lea dstq, [dstq+strideq*2] - TAIL_CALL m(iadst_8x4_internal).end3 -%elif %3 >= 0 -%ifidn %1, dct +%macro INV_TXFM_8X8_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 8x8, 8, 16*4 +%ifidn %1_%2, dct_dct pshuflw m0, [coeffq], q0000 punpcklwd m0, m0 mova m1, [o(pw_2896x8)] pmulhrsw m0, m1 mova m2, [o(pw_16384)] @@ -1242,28 +1066,10 @@ dec r3d jg .loop jmp tx2q .end3: RET -%else ; identity - mova m0, [coeffq+16*0] - mova m1, [coeffq+16*1] - mova m2, [coeffq+16*2] - mova m3, [coeffq+16*3] - punpcklwd m0, [coeffq+16*4] - punpcklwd m1, [coeffq+16*5] - punpcklwd m2, [coeffq+16*6] - punpcklwd m3, [coeffq+16*7] - punpcklwd m0, m2 - punpcklwd m1, m3 - punpcklwd m0, m1 - pmulhrsw m0, [o(pw_2896x8)] - pmulhrsw m0, [o(pw_2048)] - pxor m4, m4 - REPX {mova [coeffq+16*x], m4}, 0, 1, 2, 3, 4, 5, 6, 7 - jmp m(inv_txfm_add_dct_dct_8x8).end -%endif %endif %endmacro %macro LOAD_8ROWS 2-3 0 ; src, stride, is_rect2 %if %3 @@ -1296,14 +1102,14 @@ psubsw m%5, m%1, m%3 ;t5a paddsw m%1, m%3 ;t4 ITX_MULSUB_2W %2, %5, %3, %6, %7, 2896, 2896, 1 ;t5, t6 %endmacro -INV_TXFM_8X8_FN dct, dct, 0 -INV_TXFM_8X8_FN dct, identity, 7 +INV_TXFM_8X8_FN dct, dct INV_TXFM_8X8_FN dct, adst INV_TXFM_8X8_FN dct, flipadst +INV_TXFM_8X8_FN dct, identity cglobal idct_8x8_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 LOAD_8ROWS coeffq, 16 .pass1: @@ -1608,11 +1414,11 @@ pmulhrsw m0, [rsp+gprsize+16*0] mova m3, m5 mova [rsp+gprsize+16*0], m7 jmp m(idct_8x8_internal).end3 -INV_TXFM_8X8_FN identity, dct, 7 +INV_TXFM_8X8_FN identity, dct INV_TXFM_8X8_FN identity, adst INV_TXFM_8X8_FN identity, flipadst INV_TXFM_8X8_FN identity, identity cglobal iidentity_8x8_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 @@ -1632,85 +1438,21 @@ mova [rsp+gprsize+16*2], m5 mova [rsp+gprsize+16*1], m6 jmp m(idct_8x8_internal).end3 -%macro INV_TXFM_4X16_FN 2-3 -1 ; type1, type2, fast_thresh - INV_TXFM_FN %1, %2, %3, 4x16, 8 -%if %3 >= 0 -%ifidn %1_%2, dct_identity - mova m0, [o(pw_2896x8)] - mova m1, m0 - pmulhrsw m0, [coeffq+16*0] - pmulhrsw m1, [coeffq+16*1] - mova m2, [o(pw_16384)] - mova m3, [o(pw_1697x16)] - mova m4, [o(pw_2048)] - pmulhrsw m0, m2 - pmulhrsw m1, m2 - pmulhrsw m2, m3, m0 - pmulhrsw m3, m1 - paddsw m0, m0 - paddsw m1, m1 - paddsw m0, m2 - paddsw m1, m3 - pmulhrsw m0, m4 - pmulhrsw m4, m1 - punpckhwd m2, m0, m0 - punpcklwd m0, m0 - punpckhwd m6, m4, m4 - punpcklwd m4, m4 - punpckhdq m1, m0, m0 - punpckldq m0, m0 - punpckhdq m3, m2, m2 - punpckldq m2, m2 - punpckhdq m5, m4, m4 - punpckldq m4, m4 - punpckhdq m7, m6, m6 - punpckldq m6, m6 - mova [coeffq+16*4], m4 - TAIL_CALL m(iadst_4x16_internal).end2 -%elifidn %1_%2, identity_dct - movd m0, [coeffq+32*0] - punpcklwd m0, [coeffq+32*1] - movd m1, [coeffq+32*2] - punpcklwd m1, [coeffq+32*3] - punpckldq m0, m1 - pmulhrsw m1, m0, [o(pw_1697x8)] - pcmpeqw m2, m2 - pcmpeqw m2, m0 - pxor m0, m2 - pavgw m0, m1 - pmulhrsw m0, [o(pw_2896x8)] - pmulhrsw m0, [o(pw_2048)] - punpcklqdq m0, m0 - pxor m1, m1 - REPX {mova [coeffq+32*x], m1}, 0, 1, 2, 3 -%elifidn %1_%2, dct_dct +%macro INV_TXFM_4X16_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 4x16, 8 +%ifidn %1_%2, dct_dct pshuflw m0, [coeffq], q0000 punpcklwd m0, m0 mova m1, [o(pw_2896x8)] pmulhrsw m0, m1 mov [coeffq], eobd pmulhrsw m0, [o(pw_16384)] pmulhrsw m0, m1 pmulhrsw m0, [o(pw_2048)] -%else ; adst_dct / flipadst_dct - pshuflw m0, [coeffq], q0000 - punpcklwd m0, m0 -%ifidn %1, adst - pmulhrsw m0, [o(iadst4_dconly1a)] -%else ; flipadst - pmulhrsw m0, [o(iadst4_dconly1b)] -%endif - mova m1, [o(pw_16384)] - mov [coeffq], eobd - pmulhrsw m0, m1 - psrlw m1, 3 ; pw_2048 - pmulhrsw m0, [o(pw_2896x8)] - pmulhrsw m0, m1 -%endif .end: WRITE_4X4 0, 0, 1, 2, 3, 0, 1, 2, 3 lea dstq, [dstq+strideq*4] WRITE_4X4 0, 0, 1, 2, 3, 0, 1, 2, 3 lea dstq, [dstq+strideq*4] @@ -1719,14 +1461,14 @@ WRITE_4X4 0, 0, 1, 2, 3, 0, 1, 2, 3 RET %endif %endmacro -INV_TXFM_4X16_FN dct, dct, 0 -INV_TXFM_4X16_FN dct, identity, 15 +INV_TXFM_4X16_FN dct, dct INV_TXFM_4X16_FN dct, adst INV_TXFM_4X16_FN dct, flipadst +INV_TXFM_4X16_FN dct, identity cglobal idct_4x16_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 lea r3, [o(m(idct_4x8_internal).pass1)] .pass1: @@ -1788,11 +1530,11 @@ .end2: pxor m7, m7 REPX {mova [r3+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7 ret -INV_TXFM_4X16_FN adst, dct, 0 +INV_TXFM_4X16_FN adst, dct INV_TXFM_4X16_FN adst, adst INV_TXFM_4X16_FN adst, flipadst INV_TXFM_4X16_FN adst, identity cglobal iadst_4x16_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 @@ -1856,11 +1598,11 @@ pxor m7, m7 REPX {mova [r3+16*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7 ret -INV_TXFM_4X16_FN flipadst, dct, 0 +INV_TXFM_4X16_FN flipadst, dct INV_TXFM_4X16_FN flipadst, adst INV_TXFM_4X16_FN flipadst, flipadst INV_TXFM_4X16_FN flipadst, identity cglobal iflipadst_4x16_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 @@ -1886,11 +1628,11 @@ mova m7, [o(pw_m2048)] jmp m(iadst_4x16_internal).end1 -INV_TXFM_4X16_FN identity, dct, 3 +INV_TXFM_4X16_FN identity, dct INV_TXFM_4X16_FN identity, adst INV_TXFM_4X16_FN identity, flipadst INV_TXFM_4X16_FN identity, identity %macro IDTX16 3-4 ; src/dst, tmp, pw_1697x16, [pw_16394] @@ -1962,72 +1704,15 @@ pmulhrsw m7, [coeffq+16*7] mova [coeffq+16*4], m4 jmp m(iadst_4x16_internal).end2 -%macro INV_TXFM_16X4_FN 2-3 -1 ; type1, type2, fast_thresh - INV_TXFM_FN %1, %2, %3, 16x4, 8 -%if %3 >= 0 -%ifidn %1_%2, dct_identity - mova m3, [o(pw_2896x8)] - pmulhrsw m3, [coeffq] - mova m0, [o(pw_16384)] - pmulhrsw m3, m0 - psrlw m0, 3 ; pw_2048 - pmulhrsw m1, m3, [o(pw_1697x8)] - paddsw m3, m1 - pmulhrsw m3, m0 - punpcklwd m3, m3 - pshufd m0, m3, q0000 - pshufd m1, m3, q1111 - pshufd m2, m3, q2222 - pshufd m3, m3, q3333 - lea tx2q, [dstq+8] - call m(iadst_8x4_internal).end2 - add coeffq, 16*4 - mov dstq, tx2q - TAIL_CALL m(iadst_8x4_internal).end2 -%elifidn %1_%2, identity_dct - mova m4, [o(pw_1697x16)] - mova m5, [o(pw_16384)] - mova m6, [o(pw_2896x8)] - mov r3d, 2 - psrlw m7, m5, 3 ; pw_2048 -.main_loop: - mova m0, [coeffq+16*0] - mova m1, [coeffq+16*1] - punpckhwd m2, m0, m1 - punpcklwd m0, m1 - punpcklwd m0, m2 - mova m1, [coeffq+16*2] - mova m2, [coeffq+16*3] - punpckhwd m3, m1, m2 - punpcklwd m1, m2 - punpcklwd m1, m3 - punpcklqdq m0, m1 - pmulhrsw m1, m4, m0 - pmulhrsw m1, m5 - paddsw m0, m1 - pmulhrsw m0, m6 - pmulhrsw m0, m7 -.end: - pxor m3, m3 - mova [coeffq+16*0], m3 - mova [coeffq+16*1], m3 - mova [coeffq+16*2], m3 - mova [coeffq+16*3], m3 - add coeffq, 16*4 - lea tx2q, [dstq+8] - WRITE_8X4 0, 0, 0, 0, 1, 2, 3 - mov dstq, tx2q - dec r3d - jg .main_loop - RET -%else +%macro INV_TXFM_16X4_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 16x4, 8 +%ifidn %1_%2, dct_dct movd m1, [o(pw_2896x8)] pmulhrsw m0, m1, [coeffq] -%ifidn %2, dct movd m2, [o(pw_16384)] mov [coeffq], eobd mov r2d, 2 lea tx2q, [o(m(inv_txfm_add_dct_dct_16x4).end)] .dconly: @@ -2057,39 +1742,10 @@ dec r2d jg .dconly_loop jmp tx2q .end: RET -%else ; adst / flipadst - movd m2, [o(pw_16384)] - pmulhrsw m0, m2 - pshuflw m0, m0, q0000 - punpcklwd m0, m0 - mov [coeffq], eobd - pmulhrsw m2, m0, [o(iadst4_dconly2b)] - pmulhrsw m0, [o(iadst4_dconly2a)] - mova m1, [o(pw_2048)] - pmulhrsw m0, m1 - pmulhrsw m2, m1 -%ifidn %2, adst - punpckhqdq m1, m0, m0 - punpcklqdq m0, m0 - punpckhqdq m3, m2, m2 - punpcklqdq m2, m2 -%else ; flipadst - mova m3, m0 - punpckhqdq m0, m2, m2 - punpcklqdq m1, m2, m2 - punpckhqdq m2, m3, m3 - punpcklqdq m3, m3 -%endif - lea tx2q, [dstq+8] - call m(iadst_8x4_internal).end3 - mov dstq, tx2q - TAIL_CALL m(iadst_8x4_internal).end3 -%endif -%endif %endif %endmacro %macro LOAD_7ROWS 2 ;src, stride mova m0, [%1+%2*0] @@ -2142,14 +1798,14 @@ packssdw m%3, m%5 ;low: t12 high: t13a punpckhqdq m%4, m%1, m%6 ;low: t15a high: t14 punpcklqdq m%1, m%6 ;low: t8a high: t9 %endmacro -INV_TXFM_16X4_FN dct, dct, 0 -INV_TXFM_16X4_FN dct, adst, 0 -INV_TXFM_16X4_FN dct, flipadst, 0 -INV_TXFM_16X4_FN dct, identity, 3 +INV_TXFM_16X4_FN dct, dct +INV_TXFM_16X4_FN dct, adst +INV_TXFM_16X4_FN dct, flipadst +INV_TXFM_16X4_FN dct, identity cglobal idct_16x4_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 LOAD_7ROWS coeffq, 16 call .main @@ -2462,11 +2118,11 @@ .pass2: lea tx2q, [o(m(iflipadst_8x4_internal).pass2)] jmp m(idct_16x4_internal).pass2_end -INV_TXFM_16X4_FN identity, dct, 15 +INV_TXFM_16X4_FN identity, dct INV_TXFM_16X4_FN identity, adst INV_TXFM_16X4_FN identity, flipadst INV_TXFM_16X4_FN identity, identity cglobal iidentity_16x4_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 @@ -2535,12 +2191,12 @@ mova [%1+%2*5], m5 mova [%1+%2*6], m6 mova [%1+%2*7], m7 %endmacro -%macro INV_TXFM_8X16_FN 2-3 -1 ; type1, type2, fast_thresh - INV_TXFM_FN %1, %2, %3, 8x16, 8, 16*16 +%macro INV_TXFM_8X16_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 8x16, 8, 16*16 %ifidn %1_%2, dct_dct pshuflw m0, [coeffq], q0000 punpcklwd m0, m0 mova m1, [o(pw_2896x8)] pmulhrsw m0, m1 @@ -2554,82 +2210,17 @@ mov r3d, 4 lea tx2q, [o(m(inv_txfm_add_dct_dct_8x16).end)] jmp m(inv_txfm_add_dct_dct_8x8).loop .end: RET -%elifidn %1_%2, dct_identity - mov r3d, 2 -.loop: - mova m0, [o(pw_2896x8)] - pmulhrsw m7, m0, [coeffq] - mova m1, [o(pw_16384)] - pxor m2, m2 - mova [coeffq], m2 - pmulhrsw m7, m0 - pmulhrsw m7, m1 - psrlw m1, 3 ; pw_2048 - pmulhrsw m0, m7, [o(pw_1697x16)] - paddsw m7, m7 - paddsw m7, m0 - pmulhrsw m7, m1 - punpcklwd m0, m7, m7 - punpckhwd m7, m7 - pshufd m3, m0, q3333 - pshufd m2, m0, q2222 - pshufd m1, m0, q1111 - pshufd m0, m0, q0000 - call m(iadst_8x4_internal).end3 - pshufd m3, m7, q3333 - pshufd m2, m7, q2222 - pshufd m1, m7, q1111 - pshufd m0, m7, q0000 - lea dstq, [dstq+strideq*2] - call m(iadst_8x4_internal).end3 - - add coeffq, 16 - lea dstq, [dstq+strideq*2] - dec r3d - jg .loop - RET -%elifidn %1_%2, identity_dct - movd m0, [coeffq+32*0] - punpcklwd m0, [coeffq+32*1] - movd m2, [coeffq+32*2] - punpcklwd m2, [coeffq+32*3] - add coeffq, 32*4 - movd m1, [coeffq+32*0] - punpcklwd m1, [coeffq+32*1] - movd m3, [coeffq+32*2] - punpcklwd m3, [coeffq+32*3] - mova m4, [o(pw_2896x8)] - xor eobd, eobd - mov [coeffq-32*4], eobd - mov [coeffq-32*3], eobd - mov [coeffq-32*2], eobd - mov [coeffq-32*1], eobd - punpckldq m0, m2 - punpckldq m1, m3 - punpcklqdq m0, m1 - pmulhrsw m0, m4 - pmulhrsw m0, m4 - pmulhrsw m0, [o(pw_2048)] - mov [coeffq+32*0], eobd - mov [coeffq+32*1], eobd - mov [coeffq+32*2], eobd - mov [coeffq+32*3], eobd - mov r3d, 4 - lea tx2q, [o(m(inv_txfm_add_identity_dct_8x16).end)] - jmp m(inv_txfm_add_dct_dct_8x8).loop -.end: - RET %endif %endmacro -INV_TXFM_8X16_FN dct, dct, 0 -INV_TXFM_8X16_FN dct, identity, 15 +INV_TXFM_8X16_FN dct, dct INV_TXFM_8X16_FN dct, adst INV_TXFM_8X16_FN dct, flipadst +INV_TXFM_8X16_FN dct, identity cglobal idct_8x16_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 lea r3, [o(m(idct_8x8_internal).pass1)] .pass1: @@ -2788,11 +2379,11 @@ lea tx2q, [o(m(idct_8x16_internal).end1)] mov dstq, r3 jmp m(iflipadst_8x8_internal).end -INV_TXFM_8X16_FN identity, dct, 7 +INV_TXFM_8X16_FN identity, dct INV_TXFM_8X16_FN identity, adst INV_TXFM_8X16_FN identity, flipadst INV_TXFM_8X16_FN identity, identity cglobal iidentity_8x16_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 @@ -2835,12 +2426,12 @@ lea tx2q, [o(m(idct_8x16_internal).end1)] lea dstq, [dstq+strideq*2] jmp .end -%macro INV_TXFM_16X8_FN 2-3 -1 ; type1, type2, fast_thresh - INV_TXFM_FN %1, %2, %3, 16x8, 8, 16*16 +%macro INV_TXFM_16X8_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 16x8, 8, 16*16 %ifidn %1_%2, dct_dct movd m1, [o(pw_2896x8)] pmulhrsw m0, m1, [coeffq] movd m2, [o(pw_16384)] mov [coeffq], eobd @@ -2848,87 +2439,17 @@ mov r2d, 4 lea tx2q, [o(m(inv_txfm_add_dct_dct_16x8).end)] jmp m(inv_txfm_add_dct_dct_16x4).dconly .end: RET -%elifidn %1_%2, dct_identity - mova m7, [coeffq] - mova m0, [o(pw_2896x8)] - mova m1, [o(pw_16384)] - pxor m2, m2 - mova [coeffq], m2 - pmulhrsw m7, m0 - pmulhrsw m7, m0 - pmulhrsw m7, m1 - psrlw m1, 2 ; pw_4096 - pmulhrsw m7, m1 - punpcklwd m3, m7, m7 - punpckhwd m7, m7 - pshufd m0, m3, q0000 - pshufd m1, m3, q1111 - pshufd m2, m3, q2222 - pshufd m3, m3, q3333 - lea r3, [dstq+strideq*4] - lea tx2q, [dstq+8] - call m(iadst_8x4_internal).end2 - add coeffq, 16*4 - mov dstq, tx2q - call m(iadst_8x4_internal).end2 - mov dstq, r3 - add coeffq, 16*4 - pshufd m0, m7, q0000 - pshufd m1, m7, q1111 - pshufd m2, m7, q2222 - pshufd m3, m7, q3333 - lea tx2q, [dstq+8] - call m(iadst_8x4_internal).end2 - add coeffq, 16*4 - mov dstq, tx2q - TAIL_CALL m(iadst_8x4_internal).end2 -%elifidn %1_%2, identity_dct - mova m4, [o(pw_2896x8)] - mova m5, [o(pw_1697x16)] - mova m6, [o(pw_16384)] - psrlw m7, m6, 3 ; pw_2048 - mov r3d, 2 -.main_loop: - mova m0, [coeffq+16*0] - punpcklwd m0, [coeffq+16*1] - mova m1, [coeffq+16*2] - punpcklwd m1, [coeffq+16*3] - punpckldq m0, m1 - mova m1, [coeffq+16*4] - punpcklwd m1, [coeffq+16*5] - mova m2, [coeffq+16*6] - punpcklwd m2, [coeffq+16*7] - punpckldq m1, m2 - punpcklqdq m0, m1 - pmulhrsw m0, m4 - pmulhrsw m1, m5, m0 - pmulhrsw m1, m6 - paddsw m0, m1 - pmulhrsw m0, m4 - pmulhrsw m0, m7 -.end: - pxor m1, m1 - REPX {mova [coeffq+16*x], m1}, 0, 1, 2, 3, 4, 5, 6, 7 - add coeffq, 16*8 - lea tx2q, [dstq+8] - WRITE_8X4 0, 0, 0, 0, 1, 2, 3 - lea dstq, [dstq+strideq*2] - WRITE_8X4 0, 0, 0, 0, 1, 2, 3 - mov dstq, tx2q - dec r3d - jg .main_loop - RET %endif %endmacro -INV_TXFM_16X8_FN dct, dct, 0 -INV_TXFM_16X8_FN dct, identity, 7 +INV_TXFM_16X8_FN dct, dct INV_TXFM_16X8_FN dct, adst INV_TXFM_16X8_FN dct, flipadst +INV_TXFM_16X8_FN dct, identity cglobal idct_16x8_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 LOAD_8ROWS coeffq+16*0, 32, 1 call m(idct_8x8_internal).main SAVE_7ROWS rsp+gprsize+16*3, 16 @@ -3380,11 +2901,11 @@ lea tx2q, [o(m(idct_8x16_internal).end1)] mov dstq, r3 jmp m(iflipadst_8x8_internal).pass2_main -INV_TXFM_16X8_FN identity, dct, 15 +INV_TXFM_16X8_FN identity, dct INV_TXFM_16X8_FN identity, adst INV_TXFM_16X8_FN identity, flipadst INV_TXFM_16X8_FN identity, identity cglobal iidentity_16x8_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 @@ -3461,120 +2982,29 @@ lea tx2q, [o(m(idct_8x16_internal).end1)] mov dstq, r3 jmp m(iidentity_8x8_internal).end -%macro INV_TXFM_16X16_FN 2-3 -1 ; type1, type2, fast_thresh - INV_TXFM_FN %1, %2, %3, 16x16, 8, 16*16 +%macro INV_TXFM_16X16_FN 2 ; type1, type2 + INV_TXFM_FN %1, %2, 16x16, 8, 16*16 %ifidn %1_%2, dct_dct movd m1, [o(pw_2896x8)] pmulhrsw m0, m1, [coeffq] movd m2, [o(pw_8192)] mov [coeffq], eobd mov r2d, 8 lea tx2q, [o(m(inv_txfm_add_dct_dct_16x16).end)] jmp m(inv_txfm_add_dct_dct_16x4).dconly .end: RET -%elifidn %1_%2, dct_identity - mova m3, [o(pw_2896x8)] - pmulhrsw m2, m3, [coeffq+16*0] - pmulhrsw m3, [coeffq+16*1] - mova m0, [o(pw_8192)] - mova m1, [o(pw_1697x16)] - pshuflw m4, [o(deint_shuf)], q0000 ;pb_0_1 - punpcklwd m4, m4 - pcmpeqb m5, m5 - pxor m6, m6 - mova [coeffq+16*0], m6 - mova [coeffq+16*1], m6 - paddb m5, m5 ;pb_m2 - pmulhrsw m2, m0 - pmulhrsw m3, m0 - psrlw m0, 2 ;pw_2048 - pmulhrsw m7, m1, m2 - pmulhrsw m1, m3 - paddsw m2, m2 - paddsw m3, m3 - paddsw m2, m7 - paddsw m3, m1 - pmulhrsw m2, m0 - pmulhrsw m3, m0 - mov r3d, 8 -.loop: - mova m1, [dstq] - pshufb m0, m2, m4 - punpckhbw m7, m1, m6 - punpcklbw m1, m6 - paddw m7, m0 - paddw m1, m0 - packuswb m1, m7 - mova [dstq], m1 - mova m1, [dstq+strideq*8] - pshufb m0, m3, m4 - psubb m4, m5 ; += 2 - punpckhbw m7, m1, m6 - punpcklbw m1, m6 - paddw m7, m0 - paddw m1, m0 - packuswb m1, m7 - mova [dstq+strideq*8], m1 - add dstq, strideq - dec r3d - jg .loop - RET -%elifidn %1_%2, identity_dct - mova m4, [o(pw_1697x16)] - mova m5, [o(pw_2896x8)] - mova m6, [o(pw_2048)] - xor eobd, eobd - lea tx2q, [o(m(inv_txfm_add_identity_dct_16x16).end)] - lea r3, [dstq+8] - mov [rsp+16*0], r3 -.main: - movd m0, [coeffq+32*0] - punpcklwd m0, [coeffq+32*1] - movd m1, [coeffq+32*2] - punpcklwd m1, [coeffq+32*3] - add coeffq, 32*4 - punpckldq m0, m1 - movd m1, [coeffq+32*0] - punpcklwd m1, [coeffq+32*1] - movd m2, [coeffq+32*2] - punpcklwd m2, [coeffq+32*3] - xor eobd, eobd - mov [coeffq-32*4], eobd - mov [coeffq-32*3], eobd - mov [coeffq-32*2], eobd - mov [coeffq-32*1], eobd - punpckldq m1, m2 - punpcklqdq m0, m1 - pmulhrsw m1, m4, m0 - psraw m1, 1 - pavgw m0, m1 - pmulhrsw m0, m5 - pmulhrsw m0, m6 - mov [coeffq+32*0], eobd - mov [coeffq+32*1], eobd - mov [coeffq+32*2], eobd - mov [coeffq+32*3], eobd - mov r3d, 4 - jmp m(inv_txfm_add_dct_dct_8x8).loop -.end: - lea tx2q, [o(m(inv_txfm_add_identity_dct_16x16).end1)] - add coeffq, 32*4 - mov dstq, [rsp+16*0] - jmp .main -.end1: - RET %endif %endmacro -INV_TXFM_16X16_FN dct, dct, 0 -INV_TXFM_16X16_FN dct, identity, 15 +INV_TXFM_16X16_FN dct, dct INV_TXFM_16X16_FN dct, adst INV_TXFM_16X16_FN dct, flipadst +INV_TXFM_16X16_FN dct, identity cglobal idct_16x16_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 LOAD_8ROWS coeffq+16*1, 64 call m(idct_8x8_internal).main SAVE_7ROWS rsp+gprsize+16*3, 16 @@ -3863,11 +3293,11 @@ pmulhrsw m%2, m%3, m%1 psraw m%2, 1 pavgw m%1, m%2 %endmacro -INV_TXFM_16X16_FN identity, dct, 15 +INV_TXFM_16X16_FN identity, dct INV_TXFM_16X16_FN identity, identity cglobal iidentity_16x16_internal, 0, 0, 0, dst, stride, coeff, eob, tx2 add coeffq, 16*17 mov r3, tx2q diff --git a/third_party/dav1d/src/x86/looprestoration.asm b/third_party/dav1d/src/x86/looprestoration.asm --- a/third_party/dav1d/src/x86/looprestoration.asm +++ b/third_party/dav1d/src/x86/looprestoration.asm @@ -49,24 +49,27 @@ cextern sgr_x_by_x SECTION .text INIT_YMM avx2 -cglobal wiener_filter_h, 8, 12, 16, dst, left, src, stride, fh, w, h, edge +cglobal wiener_filter_h, 5, 12, 16, dst, left, src, stride, fh, w, h, edge + mov edged, edgem vpbroadcastb m15, [fhq+0] + movifnidn wd, wm vpbroadcastb m14, [fhq+2] + mov hd, hm vpbroadcastb m13, [fhq+4] vpbroadcastw m12, [fhq+6] vpbroadcastd m11, [pw_2048] vpbroadcastd m10, [pw_16380] lea r11, [pb_right_ext_mask] DEFINE_ARGS dst, left, src, stride, x, w, h, edge, srcptr, dstptr, xlim ; if (edge & has_right) align_w_to_32 ; else w -= 32, and use that as limit in x loop - test edged, 2 ; has_right + test edgeb, 2 ; has_right jnz .align mov xlimq, -3 jmp .loop .align: add wd, 31 @@ -78,11 +81,11 @@ mov srcptrq, srcq mov dstptrq, dstq lea xq, [wq+xlimq] ; load left edge pixels - test edged, 1 ; have_left + test edgeb, 1 ; have_left jz .emu_left test leftq, leftq ; left == NULL for the edge-extended bottom/top jz .load_left_combined movd xm0, [leftq] add leftq, 4 @@ -167,18 +170,25 @@ pmullw m5, m12 paddw m0, m7 paddw m2, m4 paddw m0, m6 paddw m2, m5 - paddsw m0, m8 + ; for a signed overflow to happen we need filter and pixels as follow: + ; filter => -5,-23,-17,90,-17,-23,-5 + ; pixels => 255,255,255,0,255,255,255 or 0,0,0,255,0,0,0 + ; m0 would fall in the range [-59A6;+59A6] = [A65A;59A6] + ; m8 would fall in the range [-3FFC;+3F84] = [C004;3F84] + ; 32-bit arithmetic m0+m8 = [-99A2;+992A] = [FFFF665E;992A] + ; => signed 16-bit overflow occurs + paddsw m0, m8 ; paddsw clips this range to [-8000;+7FFF] paddsw m2, m3 - psraw m0, 3 + psraw m0, 3 ; shift changes the range to [-1000;+FFF] psraw m2, 3 - paddw m0, m11 - paddw m2, m11 - mova [dstptrq], xm0 - mova [dstptrq+16], xm2 + paddw m0, m11 ; adding back 800 (removed in m8) changes the + paddw m2, m11 ; range to [-800;+17FF] as defined in the spec + mova [dstptrq], xm0 ; (note that adding another 800 would give us + mova [dstptrq+16], xm2; the same range as in the C code => [0;1FFF]) vextracti128 [dstptrq+32], m0, 1 vextracti128 [dstptrq+48], m2, 1 vextracti128 xm0, m1, 1 add srcptrq, 32 add dstptrq, 64 @@ -194,29 +204,31 @@ add dstq, 384*2 dec hd jg .loop RET -cglobal wiener_filter_v, 7, 10, 16, dst, stride, mid, w, h, fv, edge - vpbroadcastd m14, [fvq+4] - vpbroadcastd m15, [fvq] - vpbroadcastd m13, [pw_0_128] - paddw m14, m13 +cglobal wiener_filter_v, 4, 10, 13, dst, stride, mid, w, h, fv, edge + movifnidn fvq, fvmp + mov edged, edgem + movifnidn hd, hm + vpbroadcastd m10, [fvq] + vpbroadcastd m11, [fvq+4] + vpbroadcastd m0, [pw_0_128] vpbroadcastd m12, [pd_1024] DEFINE_ARGS dst, stride, mid, w, h, ylim, edge, y, mptr, dstptr - mov ylimd, edged - and ylimd, 8 ; have_bottom - shr ylimd, 2 + rorx ylimd, edged, 2 + paddw m11, m0 + and ylimd, 2 ; have_bottom sub ylimd, 3 ; main x loop for vertical filter, does one column of 16 pixels .loop_x: mova m3, [midq] ; middle line ; load top pixels - test edged, 4 ; have_top + test edgeb, 4 ; have_top jz .emu_top mova m0, [midq-384*4] mova m2, [midq-384*2] mova m1, m0 jmp .load_bottom_pixels @@ -267,95 +279,96 @@ ; load one line into m6. if that pixel is no longer available, do ; nothing, since m6 still has the data from the previous line in it. We ; try to structure the loop so that the common case is evaluated fastest mova m6, [mptrq+384*6] .loop: - paddw m7, m0, m6 - paddw m8, m1, m5 - paddw m9, m2, m4 - punpcklwd m10, m7, m8 - punpckhwd m7, m8 - punpcklwd m11, m9, m3 - punpckhwd m9, m3 - pmaddwd m10, m15 - pmaddwd m7, m15 - pmaddwd m11, m14 - pmaddwd m9, m14 - paddd m10, m11 + paddw m0, m6 + paddw m7, m1, m5 + paddw m8, m2, m4 + punpcklwd m9, m0, m7 + punpckhwd m0, m7 + punpcklwd m7, m8, m3 + punpckhwd m8, m3 + pmaddwd m9, m10 + pmaddwd m0, m10 + pmaddwd m7, m11 + pmaddwd m8, m11 + add mptrq, 384*2 paddd m7, m9 - paddd m10, m12 + paddd m0, m8 paddd m7, m12 - psrad m10, 11 + paddd m0, m12 psrad m7, 11 - packssdw m10, m7 - packuswb m10, m10 - vpermq m10, m10, q3120 - mova [dstptrq], xm10 + psrad m0, 11 + packssdw m7, m0 + vextracti128 xm0, m7, 1 + packuswb xm7, xm0 + mova [dstptrq], xm7 ; shift pixels one position mova m0, m1 mova m1, m2 mova m2, m3 mova m3, m4 mova m4, m5 mova m5, m6 add dstptrq, strideq - add mptrq, 384*2 dec yd jg .loop_load ; for the bottom pixels, continue using m6 (as extended edge) cmp yd, ylimd jg .loop - + add midq, 32 add dstq, 16 - add midq, 32 sub wd, 16 jg .loop_x RET INIT_YMM avx2 -cglobal sgr_box3_h, 8, 11, 8, sumsq, sum, left, src, stride, w, h, edge, x, xlim - mov xlimd, edged +cglobal sgr_box3_h, 5, 11, 7, sumsq, sum, left, src, stride, w, h, edge, x, xlim + mov xlimd, edgem + movifnidn wd, wm + mov hd, hm + mov edged, xlimd and xlimd, 2 ; have_right - add wd, xlimd - xor xlimd, 2 ; 2*!have_right - jnz .no_right - add wd, 15 + jz .no_right + add wd, 2+15 and wd, ~15 .no_right: + lea r10, [pb_right_ext_mask+32] + xor xlimd, 2 ; 2*!have_right pxor m1, m1 - lea srcq, [srcq+wq] + add srcq, wq lea sumq, [sumq+wq*2-2] lea sumsqq, [sumsqq+wq*4-4] neg wq - lea r10, [pb_right_ext_mask+32] .loop_y: mov xq, wq ; load left - test edged, 1 ; have_left + test edgeb, 1 ; have_left jz .no_left test leftq, leftq jz .load_left_from_main - pinsrw xm0, [leftq+2], 7 + vpbroadcastw xm0, [leftq+2] add leftq, 4 jmp .expand_x .no_left: vpbroadcastb xm0, [srcq+xq] jmp .expand_x .load_left_from_main: - pinsrw xm0, [srcq+xq-2], 7 + vpbroadcastw xm0, [srcq+xq-2] .expand_x: punpckhbw xm0, xm1 ; when we reach this, xm0 contains left two px in highest words cmp xd, -16 jle .loop_x .partial_load_and_extend: vpbroadcastb m3, [srcq-1] pmovzxbw m2, [srcq+xq] + movu m4, [r10+xq*2] punpcklbw m3, m1 - movu m4, [r10+xq*2] pand m2, m4 pandn m4, m3 por m2, m4 jmp .loop_x_noload .right_extend: @@ -371,26 +384,25 @@ palignr m4, m2, m0, 14 punpcklwd m5, m3, m2 punpckhwd m6, m3, m2 paddw m3, m4 - punpcklwd m7, m4, m1 + punpcklwd m0, m4, m1 punpckhwd m4, m1 pmaddwd m5, m5 pmaddwd m6, m6 - pmaddwd m7, m7 + pmaddwd m0, m0 pmaddwd m4, m4 - paddd m5, m7 + paddw m3, m2 + paddd m5, m0 + vextracti128 xm0, m2, 1 paddd m6, m4 - paddw m3, m2 movu [sumq+xq*2], m3 - movu [sumsqq+xq*4+ 0], xm5 - movu [sumsqq+xq*4+16], xm6 + movu [sumsqq+xq*4+ 0], xm5 + movu [sumsqq+xq*4+16], xm6 vextracti128 [sumsqq+xq*4+32], m5, 1 vextracti128 [sumsqq+xq*4+48], m6, 1 - - vextracti128 xm0, m2, 1 add xq, 16 ; if x <= -16 we can reload more pixels ; else if x < 0 we reload and extend (this implies have_right=0) ; else if x < xlimd we extend from previous load (this implies have_right=0) @@ -409,29 +421,29 @@ dec hd jg .loop_y RET INIT_YMM avx2 -cglobal sgr_box3_v, 5, 10, 9, sumsq, sum, w, h, edge, x, y, sumsq_ptr, sum_ptr, ylim +cglobal sgr_box3_v, 4, 10, 9, sumsq, sum, w, h, edge, x, y, sumsq_ptr, sum_ptr, ylim + movifnidn edged, edgem mov xq, -2 - mov ylimd, edged - and ylimd, 8 ; have_bottom - shr ylimd, 2 + rorx ylimd, edged, 2 + and ylimd, 2 ; have_bottom sub ylimd, 2 ; -2 if have_bottom=0, else 0 .loop_x: lea yd, [hq+ylimq+2] lea sumsq_ptrq, [sumsqq+xq*4+4-(384+16)*4] lea sum_ptrq, [sumq+xq*2+2-(384+16)*2] - test edged, 4 ; have_top + test edgeb, 4 ; have_top jnz .load_top movu m0, [sumsq_ptrq+(384+16)*4*1] movu m1, [sumsq_ptrq+(384+16)*4*1+32] + movu m6, [sum_ptrq+(384+16)*2*1] mova m2, m0 mova m3, m1 mova m4, m0 mova m5, m1 - movu m6, [sum_ptrq+(384+16)*2*1] mova m7, m6 mova m8, m6 jmp .loop_y_noload .load_top: movu m0, [sumsq_ptrq-(384+16)*4*1] ; l2sq [left] @@ -541,12 +553,14 @@ sub hd, 2 jg .loop_y RET INIT_YMM avx2 -cglobal sgr_finish_filter1, 7, 13, 16, t, src, stride, a, b, w, h, \ +cglobal sgr_finish_filter1, 5, 13, 16, t, src, stride, a, b, w, h, \ tmp_ptr, src_ptr, a_ptr, b_ptr, x, y + movifnidn wd, wm + mov hd, hm vpbroadcastd m15, [pw_16] xor xd, xd .loop_x: lea tmp_ptrq, [tq+xq*2] lea src_ptrq, [srcq+xq*1] @@ -645,79 +659,87 @@ cmp xd, wd jl .loop_x RET INIT_YMM avx2 -cglobal sgr_weighted1, 6, 6, 7, dst, stride, t, w, h, wt - movd xm0, wtd - vpbroadcastw m0, xm0 - psllw m0, 4 +cglobal sgr_weighted1, 4, 6, 6, dst, stride, t, w, h, wt +%ifidn wtd, wtm + shl wtd, 4 + movd xm5, wtd + vpbroadcastw m5, xm5 +%else + vpbroadcastw m5, wtm + mov hd, hm + psllw m5, 4 +%endif DEFINE_ARGS dst, stride, t, w, h, idx .loop_y: xor idxd, idxd .loop_x: - mova m1, [tq+idxq*2+ 0] - mova m4, [tq+idxq*2+32] + mova m0, [tq+idxq*2+ 0] + mova m1, [tq+idxq*2+32] pmovzxbw m2, [dstq+idxq+ 0] - pmovzxbw m5, [dstq+idxq+16] - psllw m3, m2, 4 - psllw m6, m5, 4 - psubw m1, m3 - psubw m4, m6 - pmulhrsw m1, m0 - pmulhrsw m4, m0 - paddw m1, m2 - paddw m4, m5 - packuswb m1, m4 - vpermq m1, m1, q3120 - mova [dstq+idxq], m1 + pmovzxbw m3, [dstq+idxq+16] + psllw m4, m2, 4 + psubw m0, m4 + psllw m4, m3, 4 + psubw m1, m4 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + paddw m0, m2 + paddw m1, m3 + packuswb m0, m1 + vpermq m0, m0, q3120 + mova [dstq+idxq], m0 add idxd, 32 cmp idxd, wd jl .loop_x + add tq, 384*2 add dstq, strideq - add tq, 384 * 2 dec hd jg .loop_y RET INIT_YMM avx2 -cglobal sgr_box5_h, 8, 11, 10, sumsq, sum, left, src, stride, w, h, edge, x, xlim - test edged, 2 ; have_right +cglobal sgr_box5_h, 5, 11, 10, sumsq, sum, left, src, stride, w, h, edge, x, xlim + mov edged, edgem + movifnidn wd, wm + mov hd, hm + test edgeb, 2 ; have_right jz .no_right xor xlimd, xlimd - add wd, 2 - add wd, 15 + add wd, 2+15 and wd, ~15 jmp .right_done .no_right: mov xlimd, 3 sub wd, 1 .right_done: + lea r10, [pb_right_ext_mask+32] pxor m1, m1 lea srcq, [srcq+wq+1] lea sumq, [sumq+wq*2-2] lea sumsqq, [sumsqq+wq*4-4] neg wq - lea r10, [pb_right_ext_mask+32] .loop_y: mov xq, wq ; load left - test edged, 1 ; have_left + test edgeb, 1 ; have_left jz .no_left test leftq, leftq jz .load_left_from_main - movd xm0, [leftq] - pinsrd xm0, [srcq+xq-1], 1 - pslldq xm0, 11 + vpbroadcastd xm2, [leftq] + movd xm0, [srcq+xq-1] add leftq, 4 + palignr xm0, xm2, 1 jmp .expand_x .no_left: vpbroadcastb xm0, [srcq+xq-1] jmp .expand_x .load_left_from_main: - pinsrd xm0, [srcq+xq-4], 3 + vpbroadcastd xm0, [srcq+xq-4] .expand_x: punpckhbw xm0, xm1 ; when we reach this, xm0 contains left two px in highest words cmp xd, -16 @@ -725,12 +747,12 @@ test xd, xd jge .right_extend .partial_load_and_extend: vpbroadcastb m3, [srcq-1] pmovzxbw m2, [srcq+xq] + movu m4, [r10+xq*2] punpcklbw m3, m1 - movu m4, [r10+xq*2] pand m2, m4 pandn m4, m3 por m2, m4 jmp .loop_x_noload .right_extend: @@ -766,12 +788,12 @@ paddd m7, m8 paddd m3, m4 paddd m7, m9 paddd m3, m5 movu [sumq+xq*2], m0 - movu [sumsqq+xq*4+ 0], xm7 - movu [sumsqq+xq*4+16], xm3 + movu [sumsqq+xq*4+ 0], xm7 + movu [sumsqq+xq*4+16], xm3 vextracti128 [sumsqq+xq*4+32], m7, 1 vextracti128 [sumsqq+xq*4+48], m3, 1 vextracti128 xm0, m2, 1 add xq, 16 @@ -786,52 +808,52 @@ test xd, xd jl .partial_load_and_extend cmp xd, xlimd jl .right_extend + add srcq, strideq add sumsqq, (384+16)*4 add sumq, (384+16)*2 - add srcq, strideq dec hd jg .loop_y RET INIT_YMM avx2 -cglobal sgr_box5_v, 5, 10, 15, sumsq, sum, w, h, edge, x, y, sumsq_ptr, sum_ptr, ylim +cglobal sgr_box5_v, 4, 10, 15, sumsq, sum, w, h, edge, x, y, sumsq_ptr, sum_ptr, ylim + movifnidn edged, edgem mov xq, -2 - mov ylimd, edged - and ylimd, 8 ; have_bottom - shr ylimd, 2 + rorx ylimd, edged, 2 + and ylimd, 2 ; have_bottom sub ylimd, 3 ; -3 if have_bottom=0, else -1 .loop_x: lea yd, [hq+ylimq+2] lea sumsq_ptrq, [sumsqq+xq*4+4-(384+16)*4] lea sum_ptrq, [sumq+xq*2+2-(384+16)*2] - test edged, 4 ; have_top + test edgeb, 4 ; have_top jnz .load_top movu m0, [sumsq_ptrq+(384+16)*4*1] movu m1, [sumsq_ptrq+(384+16)*4*1+32] + movu m10, [sum_ptrq+(384+16)*2*1] mova m2, m0 mova m3, m1 mova m4, m0 mova m5, m1 mova m6, m0 mova m7, m1 - movu m10, [sum_ptrq+(384+16)*2*1] mova m11, m10 mova m12, m10 mova m13, m10 jmp .loop_y_second_load .load_top: movu m0, [sumsq_ptrq-(384+16)*4*1] ; l3/4sq [left] movu m1, [sumsq_ptrq-(384+16)*4*1+32] ; l3/4sq [right] movu m4, [sumsq_ptrq-(384+16)*4*0] ; l2sq [left] movu m5, [sumsq_ptrq-(384+16)*4*0+32] ; l2sq [right] + movu m10, [sum_ptrq-(384+16)*2*1] ; l3/4 + movu m12, [sum_ptrq-(384+16)*2*0] ; l2 mova m2, m0 mova m3, m1 - movu m10, [sum_ptrq-(384+16)*2*1] ; l3/4 - movu m12, [sum_ptrq-(384+16)*2*0] ; l2 mova m11, m10 .loop_y: movu m6, [sumsq_ptrq+(384+16)*4*1] ; l1sq [left] movu m7, [sumsq_ptrq+(384+16)*4*1+32] ; l1sq [right] movu m13, [sum_ptrq+(384+16)*2*1] ; l1 @@ -958,12 +980,14 @@ sub hd, 2 jg .loop_y RET INIT_YMM avx2 -cglobal sgr_finish_filter2, 7, 13, 13, t, src, stride, a, b, w, h, \ +cglobal sgr_finish_filter2, 5, 13, 13, t, src, stride, a, b, w, h, \ tmp_ptr, src_ptr, a_ptr, b_ptr, x, y + movifnidn wd, wm + mov hd, hm vpbroadcastd m9, [pw_5_6] vpbroadcastd m12, [pw_256] psrlw m11, m12, 1 ; pw_128 psrlw m10, m12, 8 ; pw_1 xor xd, xd @@ -1075,12 +1099,14 @@ cmp xd, wd jl .loop_x RET INIT_YMM avx2 -cglobal sgr_weighted2, 7, 7, 11, dst, stride, t1, t2, w, h, wt - vpbroadcastd m0, [wtq] +cglobal sgr_weighted2, 4, 7, 11, dst, stride, t1, t2, w, h, wt + movifnidn wd, wm + movifnidn hd, hm + vpbroadcastd m0, wtm vpbroadcastd m10, [pd_1024] DEFINE_ARGS dst, stride, t1, t2, w, h, idx .loop_y: xor idxd, idxd .loop_x: diff --git a/third_party/dav1d/src/x86/looprestoration_init_tmpl.c b/third_party/dav1d/src/x86/looprestoration_init_tmpl.c --- a/third_party/dav1d/src/x86/looprestoration_init_tmpl.c +++ b/third_party/dav1d/src/x86/looprestoration_init_tmpl.c @@ -167,11 +167,11 @@ const coef *t1, const int w, const int h, \ const int wt); \ void dav1d_sgr_weighted2_##ext(pixel *dst, const ptrdiff_t stride, \ const coef *t1, const coef *t2, \ const int w, const int h, \ - const int16_t wt[2]); \ + const uint32_t wt); \ \ static void sgr_filter_##ext(pixel *const dst, const ptrdiff_t dst_stride, \ const pixel (*const left)[4], \ const pixel *lpf, const ptrdiff_t lpf_stride, \ const int w, const int h, const int sgr_idx, \ @@ -192,11 +192,11 @@ ALIGN_STK_32(coef, tmp2, 64 * 384,); \ dav1d_sgr_filter2_##ext(tmp1, dst, dst_stride, left, lpf, lpf_stride, \ w, h, dav1d_sgr_params[sgr_idx][2], edges); \ dav1d_sgr_filter1_##ext(tmp2, dst, dst_stride, left, lpf, lpf_stride, \ w, h, dav1d_sgr_params[sgr_idx][3], edges); \ - const int16_t wt[2] = { sgr_wt[0], 128 - sgr_wt[0] - sgr_wt[1] }; \ + const uint32_t wt = ((128 - sgr_wt[0] - sgr_wt[1]) << 16) | (uint16_t) sgr_wt[0]; \ dav1d_sgr_weighted2_##ext(dst, dst_stride, tmp1, tmp2, w, h, wt); \ } \ } #define DEF_LR_FILTERS(ext) \ diff --git a/third_party/dav1d/src/x86/looprestoration_ssse3.asm b/third_party/dav1d/src/x86/looprestoration_ssse3.asm --- a/third_party/dav1d/src/x86/looprestoration_ssse3.asm +++ b/third_party/dav1d/src/x86/looprestoration_ssse3.asm @@ -186,17 +186,17 @@ DEFINE_ARGS dst, left, src, stride, x, w, h, edge %define srcptrq srcq %define dstptrq dstq %define hd dword [esp+ 0] - %define edged dword [esp+12] + %define edgeb byte [esp+12] %define xlimd dword [esp+16] %endif ; if (edge & has_right) align_w_to_16 ; else w -= 3, and use that as limit in x loop - test edged, 2 ; has_right + test edgeb, 2 ; has_right jnz .align mov xlimd, -3 jmp .loop .align: add wd, 15 @@ -219,11 +219,11 @@ mov xd, xlimd add xd, wd %endif ; load left edge pixels - test edged, 1 ; have_left + test edgeb, 1 ; have_left jz .emu_left test leftq, leftq ; left == NULL for the edge-extended bottom/top jz .load_left_combined movd m0, [leftq] movd m1, [srcq] @@ -357,12 +357,12 @@ pmullw m5, m12 paddw m0, m7 paddw m2, m4 paddw m0, m3 paddw m2, m5 - paddsw m0, m8 - paddsw m2, m6 + paddsw m0, m8 ; see the avx2 for an explanation + paddsw m2, m6 ; of how the clipping works here psraw m0, 3 psraw m2, 3 paddw m0, m11 paddw m2, m11 mova [dstptrq+ 0], m0 @@ -475,19 +475,19 @@ mova [esp+0x40], m1 DEFINE_ARGS dst, stride, mid, w, h, y, edge %define mptrq midq %define dstptrq dstq - %define edged dword [esp] + %define edgeb byte [esp] %endif ; main x loop for vertical filter, does one column of 16 pixels .loop_x: mova m3, [midq] ; middle line ; load top pixels - test edged, 4 ; have_top + test edgeb, 4 ; have_top jz .emu_top mova m0, [midq-384*4] mova m2, [midq-384*2] mova m1, m0 jmp .load_bottom_pixels @@ -602,24 +602,24 @@ mova m1, m2 mova m2, m3 mova m3, m4 mova m4, m5 mova m5, m6 + add mptrq, 384*2 add dstptrq, strideq - add mptrq, 384*2 dec yd jg .loop_load ; for the bottom pixels, continue using m6 (as extended edge) cmp yd, ylimd jg .loop %if ARCH_X86_32 mov midq, [esp+8] mov dstq, [esp+4] %endif + add midq, 16 add dstq, 8 - add midq, 16 sub wd, 8 jg .loop_x RET %endmacro @@ -677,11 +677,11 @@ %else cglobal sgr_box3_h, 6, 7, 8, sumsq, sum, left, src, stride, x, h, edge, w, xlim %define wq r0m %define xlimd r1m %define hd hmp - %define edged edgemp + %define edgeb byte edgem mov r6, edgem and r6, 2 ; have_right add xd, r6 xor r6, 2 ; 2*!have_right @@ -704,11 +704,11 @@ %endif .loop_y: mov xq, wq ; load left - test edged, 1 ; have_left + test edgeb, 1 ; have_left jz .no_left test leftq, leftq jz .load_left_from_main movd m0, [leftq] pslldq m0, 12 @@ -793,15 +793,17 @@ %if ARCH_X86_64 cglobal sgr_box3_v, 4, 10, 9, sumsq, sum, w, h, edge, x, y, sumsq_base, sum_base, ylim movifnidn edged, edgem %else -cglobal sgr_box3_v, 5, 7, 8, -28, sumsq, sum, w, h, edge, x, y +cglobal sgr_box3_v, 3, 7, 8, -28, sumsq, sum, w, edge, h, x, y %define sumsq_baseq dword [esp+0] %define sum_baseq dword [esp+4] %define ylimd dword [esp+8] %define m8 [esp+12] + mov edged, r4m + mov hd, r3m %endif mov xq, -2 %if ARCH_X86_64 mov ylimd, edged and ylimd, 8 ; have_bottom @@ -810,11 +812,11 @@ mov sumsq_baseq, sumsqq mov sum_baseq, sumq .loop_x: mov sumsqq, sumsq_baseq mov sumq, sum_baseq - lea yd, [hd+ylimd+2] + lea yd, [hq+ylimq+2] %else mov yd, edged and yd, 8 ; have_bottom shr yd, 2 sub yd, 2 ; -2 if have_bottom=0, else 0 @@ -822,16 +824,16 @@ mov sum_baseq, sumq mov ylimd, yd .loop_x: mov sumsqd, sumsq_baseq mov sumd, sum_baseq - lea yd, [hd+2] + lea yd, [hq+2] add yd, ylimd %endif lea sumsqq, [sumsqq+xq*4+4-(384+16)*4] lea sumq, [sumq+xq*2+2-(384+16)*2] - test edged, 4 ; have_top + test edgeb, 4 ; have_top jnz .load_top movu m0, [sumsqq+(384+16)*4*1] movu m1, [sumsqq+(384+16)*4*1+16] mova m2, m0 mova m3, m1 @@ -1178,14 +1180,14 @@ mova [esp+0x60], m7 psubd m1, [aq-(384+16)*4*2] ; a:ctr+bottom [first half] psubd m3, [aq-(384+16)*4*2+16] ; a:ctr+bottom [second half] %endif + add srcq, strideq add aq, (384+16)*4 add bq, (384+16)*2 add tq, 384*2 - add srcq, strideq dec yd jg .loop_y add xd, 8 cmp xd, wd jl .loop_x @@ -1235,11 +1237,11 @@ mov hd, hm mova m10, [pb_0] mova m11, [pb_0_1] %else cglobal sgr_box5_h, 7, 7, 8, sumsq, sum, left, src, xlim, x, h, edge - %define edged edgemp + %define edgeb byte edgem %define wd xd %define wq wd %define wm r5m %define strideq r4m SUB esp, 8 @@ -1247,11 +1249,11 @@ %define m10 [PIC_sym(pb_0)] %define m11 [PIC_sym(pb_0_1)] %endif - test edged, 2 ; have_right + test edgeb, 2 ; have_right jz .no_right xor xlimd, xlimd add wd, 2 add wd, 15 and wd, ~15 @@ -1273,11 +1275,11 @@ %endif .loop_y: mov xq, wq ; load left - test edged, 1 ; have_left + test edgeb, 1 ; have_left jz .no_left test leftq, leftq jz .load_left_from_main movd m0, [leftq] movd m2, [srcq+xq-1] @@ -1399,13 +1401,13 @@ test xd, xd jl .partial_load_and_extend cmp xd, xlimd jl .right_extend + add srcq, strideq add sumsqq, (384+16)*4 add sumq, (384+16)*2 - add srcq, strideq dec hd jg .loop_y %if ARCH_X86_32 ADD esp, 8 %endif @@ -1432,11 +1434,11 @@ %if ARCH_X86_64 .loop_x: lea yd, [hd+ylimd+2] lea sumsq_ptrq, [sumsqq+xq*4+4-(384+16)*4] lea sum_ptrq, [ sumq+xq*2+2-(384+16)*2] - test edged, 4 ; have_top + test edgeb, 4 ; have_top jnz .load_top movu m0, [sumsq_ptrq+(384+16)*4*1] movu m1, [sumsq_ptrq+(384+16)*4*1+16] mova m2, m0 mova m3, m1 @@ -1518,11 +1520,11 @@ %else .sumsq_loop_x: lea yd, [ylimd+2] add yd, hm lea sumsq_ptrq, [sumsqq+xq*4+4-(384+16)*4] - test dword edgem, 4 ; have_top + test byte edgem, 4 ; have_top jnz .sumsq_load_top movu m0, [sumsq_ptrq+(384+16)*4*1] movu m1, [sumsq_ptrq+(384+16)*4*1+16] mova m4, m0 mova m5, m1 @@ -1580,11 +1582,11 @@ mov xd, -2 .sum_loop_x: lea yd, [ylimd+2] add yd, hm lea sum_ptrq, [sumq+xq*2+2-(384+16)*2] - test dword edgem, 4 ; have_top + test byte edgem, 4 ; have_top jnz .sum_load_top movu m0, [sum_ptrq+(384+16)*2*1] mova m1, m0 mova m2, m0 mova m3, m0 @@ -1880,21 +1882,20 @@ %endif RET cglobal sgr_weighted2, 4, 7, 12, dst, stride, t1, t2, w, h, wt movifnidn wd, wm - mov wtq, wtmp + movd m0, wtm %if ARCH_X86_64 movifnidn hd, hm mova m10, [pd_1024] pxor m11, m11 %else SETUP_PIC hd, 0 %define m10 [PIC_sym(pd_1024)] %define m11 m7 %endif - movd m0, [wtq] pshufd m0, m0, 0 DEFINE_ARGS dst, stride, t1, t2, w, h, idx %if ARCH_X86_32 %define hd hmp %endif diff --git a/third_party/dav1d/src/x86/mc.asm b/third_party/dav1d/src/x86/mc.asm deleted file mode 100644 --- a/third_party/dav1d/src/x86/mc.asm +++ /dev/null @@ -1,6026 +0,0 @@ -; Copyright © 2018, VideoLAN and dav1d authors -; Copyright © 2018, Two Orioles, LLC -; All rights reserved. -; -; Redistribution and use in source and binary forms, with or without -; modification, are permitted provided that the following conditions are met: -; -; 1. Redistributions of source code must retain the above copyright notice, this -; list of conditions and the following disclaimer. -; -; 2. Redistributions in binary form must reproduce the above copyright notice, -; this list of conditions and the following disclaimer in the documentation -; and/or other materials provided with the distribution. -; -; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -%include "ext/x86/x86inc.asm" - -%if ARCH_X86_64 - -SECTION_RODATA 64 - -; dav1d_obmc_masks[] with 64-x interleaved -obmc_masks: db 0, 0, 0, 0 - ; 2 - db 45, 19, 64, 0 - ; 4 - db 39, 25, 50, 14, 59, 5, 64, 0 - ; 8 - db 36, 28, 42, 22, 48, 16, 53, 11, 57, 7, 61, 3, 64, 0, 64, 0 - ; 16 - db 34, 30, 37, 27, 40, 24, 43, 21, 46, 18, 49, 15, 52, 12, 54, 10 - db 56, 8, 58, 6, 60, 4, 61, 3, 64, 0, 64, 0, 64, 0, 64, 0 - ; 32 - db 33, 31, 35, 29, 36, 28, 38, 26, 40, 24, 41, 23, 43, 21, 44, 20 - db 45, 19, 47, 17, 48, 16, 50, 14, 51, 13, 52, 12, 53, 11, 55, 9 - db 56, 8, 57, 7, 58, 6, 59, 5, 60, 4, 60, 4, 61, 3, 62, 2 - db 64, 0, 64, 0, 64, 0, 64, 0, 64, 0, 64, 0, 64, 0, 64, 0 - -bidir_sctr_w4: dd 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15 -wm_420_perm4: db 1, 3, 9, 11, 5, 7, 13, 15, 17, 19, 25, 27, 21, 23, 29, 31 - db 33, 35, 41, 43, 37, 39, 45, 47, 49, 51, 57, 59, 53, 55, 61, 63 - db 0, 2, 8, 10, 4, 6, 12, 14, 16, 18, 24, 26, 20, 22, 28, 30 - db 32, 34, 40, 42, 36, 38, 44, 46, 48, 50, 56, 58, 52, 54, 60, 62 -wm_420_perm8: db 1, 3, 17, 19, 5, 7, 21, 23, 9, 11, 25, 27, 13, 15, 29, 31 - db 33, 35, 49, 51, 37, 39, 53, 55, 41, 43, 57, 59, 45, 47, 61, 63 - db 0, 2, 16, 18, 4, 6, 20, 22, 8, 10, 24, 26, 12, 14, 28, 30 - db 32, 34, 48, 50, 36, 38, 52, 54, 40, 42, 56, 58, 44, 46, 60, 62 -wm_420_perm16: db 1, 3, 33, 35, 5, 7, 37, 39, 9, 11, 41, 43, 13, 15, 45, 47 - db 17, 19, 49, 51, 21, 23, 53, 55, 25, 27, 57, 59, 29, 31, 61, 63 - db 0, 2, 32, 34, 4, 6, 36, 38, 8, 10, 40, 42, 12, 14, 44, 46 - db 16, 18, 48, 50, 20, 22, 52, 54, 24, 26, 56, 58, 28, 30, 60, 62 -wm_420_mask: db 3, 7, 11, 15, 19, 23, 27, 31, 35, 39, 43, 47, 51, 55, 59, 63 - db 67, 71, 75, 79, 83, 87, 91, 95, 99,103,107,111,115,119,123,127 - db 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61 - db 65, 69, 73, 77, 81, 85, 89, 93, 97,101,105,109,113,117,121,125 -wm_422_mask: db 2, 6, 10, 14, 18, 22, 26, 30, 34, 38, 42, 46, 50, 54, 58, 62 - db 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61 - db 66, 70, 74, 78, 82, 86, 90, 94, 98,102,106,110,114,118,122,126 - db 65, 69, 73, 77, 81, 85, 89, 93, 97,101,105,109,113,117,121,125 -wm_444_mask: db 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 - db 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63 - db 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 - db 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62 -bilin_h_perm16: db 1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7 - db 9, 8, 10, 9, 11, 10, 12, 11, 13, 12, 14, 13, 15, 14, 16, 15 - db 33, 32, 34, 33, 35, 34, 36, 35, 37, 36, 38, 37, 39, 38, 40, 39 - db 41, 40, 42, 41, 43, 42, 44, 43, 45, 44, 46, 45, 47, 46, 48, 47 -bilin_h_perm32: db 1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7 - db 9, 8, 10, 9, 11, 10, 12, 11, 13, 12, 14, 13, 15, 14, 16, 15 - db 17, 16, 18, 17, 19, 18, 20, 19, 21, 20, 22, 21, 23, 22, 24, 23 - db 25, 24, 26, 25, 27, 26, 28, 27, 29, 28, 30, 29, 31, 30, 32, 31 -bilin_v_perm8: db 16, 0, 17, 1, 18, 2, 19, 3, 20, 4, 21, 5, 22, 6, 23, 7 - db 80, 16, 81, 17, 82, 18, 83, 19, 84, 20, 85, 21, 86, 22, 87, 23 - db 32, 80, 33, 81, 34, 82, 35, 83, 36, 84, 37, 85, 38, 86, 39, 87 - db 64, 32, 65, 33, 66, 34, 67, 35, 68, 36, 69, 37, 70, 38, 71, 39 -bilin_v_perm16: db 16, 0, 17, 1, 18, 2, 19, 3, 20, 4, 21, 5, 22, 6, 23, 7 - db 24, 8, 25, 9, 26, 10, 27, 11, 28, 12, 29, 13, 30, 14, 31, 15 - db 64, 16, 65, 17, 66, 18, 67, 19, 68, 20, 69, 21, 70, 22, 71, 23 - db 72, 24, 73, 25, 74, 26, 75, 27, 76, 28, 77, 29, 78, 30, 79, 31 -bilin_v_perm32: db 64, 0, 65, 1, 66, 2, 67, 3, 68, 4, 69, 5, 70, 6, 71, 7 - db 72, 8, 73, 9, 74, 10, 75, 11, 76, 12, 77, 13, 78, 14, 79, 15 - db 80, 16, 81, 17, 82, 18, 83, 19, 84, 20, 85, 21, 86, 22, 87, 23 - db 88, 24, 89, 25, 90, 26, 91, 27, 92, 28, 93, 29, 94, 30, 95, 31 -bilin_v_perm64: dq 0, 4, 1, 5, 2, 6, 3, 7 -spel_h_perm16a: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 - db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 - db 32, 33, 34, 35, 33, 34, 35, 36, 34, 35, 36, 37, 35, 36, 37, 38 - db 40, 41, 42, 43, 41, 42, 43, 44, 42, 43, 44, 45, 43, 44, 45, 46 -spel_h_perm16b: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 - db 12, 13, 14, 15, 13, 14, 15, 16, 14, 15, 16, 17, 15, 16, 17, 18 - db 36, 37, 38, 39, 37, 38, 39, 40, 38, 39, 40, 41, 39, 40, 41, 42 - db 44, 45, 46, 47, 45, 46, 47, 48, 46, 47, 48, 49, 47, 48, 49, 50 -spel_h_perm16c: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 - db 16, 17, 18, 19, 17, 18, 19, 20, 18, 19, 20, 21, 19, 20, 21, 22 - db 40, 41, 42, 43, 41, 42, 43, 44, 42, 43, 44, 45, 43, 44, 45, 46 - db 48, 49, 50, 51, 49, 50, 51, 52, 50, 51, 52, 53, 51, 52, 53, 54 -spel_h_perm32a: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 - db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 - db 16, 17, 18, 19, 17, 18, 19, 20, 18, 19, 20, 21, 19, 20, 21, 22 - db 24, 25, 26, 27, 25, 26, 27, 28, 26, 27, 28, 29, 27, 28, 29, 30 -spel_h_perm32b: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 - db 12, 13, 14, 15, 13, 14, 15, 16, 14, 15, 16, 17, 15, 16, 17, 18 - db 20, 21, 22, 23, 21, 22, 23, 24, 22, 23, 24, 25, 23, 24, 25, 26 - db 28, 29, 30, 31, 29, 30, 31, 32, 30, 31, 32, 33, 31, 32, 33, 34 -spel_h_perm32c: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 - db 16, 17, 18, 19, 17, 18, 19, 20, 18, 19, 20, 21, 19, 20, 21, 22 - db 24, 25, 26, 27, 25, 26, 27, 28, 26, 27, 28, 29, 27, 28, 29, 30 - db 32, 33, 34, 35, 33, 34, 35, 36, 34, 35, 36, 37, 35, 36, 37, 38 -spel_hv_perm4a: db 8, 9, 16, 17, 10, 11, 18, 19, 12, 13, 20, 21, 14, 15, 22, 23 - db 16, 17, 24, 25, 18, 19, 26, 27, 20, 21, 28, 29, 22, 23, 30, 31 -spel_hv_perm4b: db 24, 25, 32, 33, 26, 27, 34, 35, 28, 29, 36, 37, 30, 31, 38, 39 - db 32, 33, 40, 41, 34, 35, 42, 43, 36, 37, 44, 45, 38, 39, 46, 47 - db 40, 41, 48, 49, 42, 43, 50, 51, 44, 45, 52, 53, 46, 47, 54, 55 - db 48, 49, 56, 57, 50, 51, 58, 59, 52, 53, 60, 61, 54, 55, 62, 63 - -warp_8x8_shufA: db 0, 2, 4, 6, 1, 3, 5, 7, 1, 3, 5, 7, 2, 4, 6, 8 - db 4, 6, 8, 10, 5, 7, 9, 11, 5, 7, 9, 11, 6, 8, 10, 12 -warp_8x8_shufB: db 2, 4, 6, 8, 3, 5, 7, 9, 3, 5, 7, 9, 4, 6, 8, 10 - db 6, 8, 10, 12, 7, 9, 11, 13, 7, 9, 11, 13, 8, 10, 12, 14 -subpel_h_shuf4: db 0, 1, 2, 3, 1, 2, 3, 4, 8, 9, 10, 11, 9, 10, 11, 12 - db 2, 3, 4, 5, 3, 4, 5, 6, 10, 11, 12, 13, 11, 12, 13, 14 -subpel_h_shufA: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 -subpel_h_shufB: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 -subpel_h_shufC: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 -subpel_v_shuf4: db 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15 -bilin_h_shuf4: db 1, 0, 2, 1, 3, 2, 4, 3, 9, 8, 10, 9, 11, 10, 12, 11 -bilin_h_shuf8: db 1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7 -bilin_v_shuf4: db 4, 0, 5, 1, 6, 2, 7, 3, 8, 4, 9, 5, 10, 6, 11, 7 -deint_shuf4: db 0, 4, 1, 5, 2, 6, 3, 7, 4, 8, 5, 9, 6, 10, 7, 11 -blend_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3 - -wm_420_perm64: dq 0xfedcba9876543210 -wm_420_sign: dd 0x01020102, 0x01010101 -wm_422_sign: dd 0x80808080, 0x7f7f7f7f -wm_sign_avx512: dd 0x40804080, 0xc0c0c0c0, 0x40404040 - -pw_m128 times 2 dw -128 -pw_34: times 2 dw 34 -pw_258: times 2 dw 258 -pw_512: times 2 dw 512 -pw_1024: times 2 dw 1024 -pw_2048: times 2 dw 2048 -pw_6903: times 2 dw 6903 -pw_8192: times 2 dw 8192 -pd_2: dd 2 -pd_32: dd 32 -pd_512: dd 512 -pd_32768: dd 32768 - -%define pb_m64 (wm_sign_avx512+4) -%define pb_64 (wm_sign_avx512+8) -%define pb_127 (wm_422_sign +4) - -cextern mc_subpel_filters -cextern mc_warp_filter -%define subpel_filters (mangle(private_prefix %+ _mc_subpel_filters)-8) - -%macro BASE_JMP_TABLE 3-* - %xdefine %1_%2_table (%%table - %3) - %xdefine %%base %1_%2 - %%table: - %rep %0 - 2 - dw %%base %+ _w%3 - %%base - %rotate 1 - %endrep -%endmacro - -%macro HV_JMP_TABLE 5-* - %xdefine %%prefix mangle(private_prefix %+ _%1_%2_%3) - %xdefine %%base %1_%3 - %assign %%types %4 - %if %%types & 1 - %xdefine %1_%2_h_%3_table (%%h - %5) - %%h: - %rep %0 - 4 - dw %%prefix %+ .h_w%5 - %%base - %rotate 1 - %endrep - %rotate 4 - %endif - %if %%types & 2 - %xdefine %1_%2_v_%3_table (%%v - %5) - %%v: - %rep %0 - 4 - dw %%prefix %+ .v_w%5 - %%base - %rotate 1 - %endrep - %rotate 4 - %endif - %if %%types & 4 - %xdefine %1_%2_hv_%3_table (%%hv - %5) - %%hv: - %rep %0 - 4 - dw %%prefix %+ .hv_w%5 - %%base - %rotate 1 - %endrep - %endif -%endmacro - -%macro BIDIR_JMP_TABLE 1-* - %xdefine %1_table (%%table - 2*%2) - %xdefine %%base %1_table - %xdefine %%prefix mangle(private_prefix %+ _%1) - %%table: - %rep %0 - 1 - dd %%prefix %+ .w%2 - %%base - %rotate 1 - %endrep -%endmacro - -%xdefine put_avx2 mangle(private_prefix %+ _put_bilin_avx2.put) -%xdefine prep_avx2 mangle(private_prefix %+ _prep_bilin_avx2.prep) -%xdefine prep_avx512icl mangle(private_prefix %+ _prep_bilin_avx512icl.prep) - -%define table_offset(type, fn) type %+ fn %+ SUFFIX %+ _table - type %+ SUFFIX - -BASE_JMP_TABLE put, avx2, 2, 4, 8, 16, 32, 64, 128 -BASE_JMP_TABLE prep, avx2, 4, 8, 16, 32, 64, 128 -HV_JMP_TABLE put, bilin, avx2, 7, 2, 4, 8, 16, 32, 64, 128 -HV_JMP_TABLE prep, bilin, avx2, 7, 4, 8, 16, 32, 64, 128 -HV_JMP_TABLE put, 8tap, avx2, 3, 2, 4, 8, 16, 32, 64, 128 -HV_JMP_TABLE prep, 8tap, avx2, 1, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE avg_avx2, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE w_avg_avx2, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE mask_avx2, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE w_mask_420_avx2, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE w_mask_422_avx2, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE w_mask_444_avx2, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE blend_avx2, 4, 8, 16, 32 -BIDIR_JMP_TABLE blend_v_avx2, 2, 4, 8, 16, 32 -BIDIR_JMP_TABLE blend_h_avx2, 2, 4, 8, 16, 32, 32, 32 - -BASE_JMP_TABLE prep, avx512icl, 4, 8, 16, 32, 64, 128 -HV_JMP_TABLE prep, bilin, avx512icl, 7, 4, 8, 16, 32, 64, 128 -HV_JMP_TABLE prep, 8tap, avx512icl, 7, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE avg_avx512icl, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE w_avg_avx512icl, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE mask_avx512icl, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE w_mask_420_avx512icl, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE w_mask_422_avx512icl, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE w_mask_444_avx512icl, 4, 8, 16, 32, 64, 128 - -SECTION .text - -INIT_XMM avx2 -DECLARE_REG_TMP 4, 6, 7 -cglobal put_bilin, 4, 8, 0, dst, ds, src, ss, w, h, mxy - movifnidn mxyd, r6m ; mx - lea t2, [put_avx2] - tzcnt wd, wm - movifnidn hd, hm - test mxyd, mxyd - jnz .h - mov mxyd, r7m ; my - test mxyd, mxyd - jnz .v -.put: - movzx wd, word [t2+wq*2+table_offset(put,)] - add wq, t2 - jmp wq -.put_w2: - movzx t0d, word [srcq+ssq*0] - movzx t1d, word [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - mov [dstq+dsq*0], t0w - mov [dstq+dsq*1], t1w - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .put_w2 - RET -.put_w4: - mov t0d, [srcq+ssq*0] - mov t1d, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - mov [dstq+dsq*0], t0d - mov [dstq+dsq*1], t1d - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .put_w4 - RET -.put_w8: - mov t0, [srcq+ssq*0] - mov t1, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - mov [dstq+dsq*0], t0 - mov [dstq+dsq*1], t1 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .put_w8 - RET -.put_w16: - movu m0, [srcq+ssq*0] - movu m1, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - mova [dstq+dsq*0], m0 - mova [dstq+dsq*1], m1 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .put_w16 - RET -INIT_YMM avx2 -.put_w32: - movu m0, [srcq+ssq*0] - movu m1, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - mova [dstq+dsq*0], m0 - mova [dstq+dsq*1], m1 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .put_w32 - RET -.put_w64: - movu m0, [srcq+ssq*0+32*0] - movu m1, [srcq+ssq*0+32*1] - movu m2, [srcq+ssq*1+32*0] - movu m3, [srcq+ssq*1+32*1] - lea srcq, [srcq+ssq*2] - mova [dstq+dsq*0+32*0], m0 - mova [dstq+dsq*0+32*1], m1 - mova [dstq+dsq*1+32*0], m2 - mova [dstq+dsq*1+32*1], m3 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .put_w64 - RET -.put_w128: - movu m0, [srcq+32*0] - movu m1, [srcq+32*1] - movu m2, [srcq+32*2] - movu m3, [srcq+32*3] - add srcq, ssq - mova [dstq+32*0], m0 - mova [dstq+32*1], m1 - mova [dstq+32*2], m2 - mova [dstq+32*3], m3 - add dstq, dsq - dec hd - jg .put_w128 - RET -.h: - ; (16 * src[x] + (mx * (src[x + 1] - src[x])) + 8) >> 4 - ; = ((16 - mx) * src[x] + mx * src[x + 1] + 8) >> 4 - imul mxyd, 0xff01 - vbroadcasti128 m4, [bilin_h_shuf8] - add mxyd, 16 << 8 - movd xm5, mxyd - mov mxyd, r7m ; my - vpbroadcastw m5, xm5 - test mxyd, mxyd - jnz .hv - movzx wd, word [t2+wq*2+table_offset(put, _bilin_h)] - vpbroadcastd m3, [pw_2048] - add wq, t2 - jmp wq -.h_w2: - movd xm0, [srcq+ssq*0] - pinsrd xm0, [srcq+ssq*1], 1 - lea srcq, [srcq+ssq*2] - pshufb xm0, xm4 - pmaddubsw xm0, xm5 - pmulhrsw xm0, xm3 - packuswb xm0, xm0 - pextrw [dstq+dsq*0], xm0, 0 - pextrw [dstq+dsq*1], xm0, 2 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .h_w2 - RET -.h_w4: - mova xm4, [bilin_h_shuf4] -.h_w4_loop: - movq xm0, [srcq+ssq*0] - movhps xm0, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - pshufb xm0, xm4 - pmaddubsw xm0, xm5 - pmulhrsw xm0, xm3 - packuswb xm0, xm0 - movd [dstq+dsq*0], xm0 - pextrd [dstq+dsq*1], xm0, 1 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .h_w4_loop - RET -.h_w8: - movu xm0, [srcq+ssq*0] - movu xm1, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - pshufb xm0, xm4 - pshufb xm1, xm4 - pmaddubsw xm0, xm5 - pmaddubsw xm1, xm5 - pmulhrsw xm0, xm3 - pmulhrsw xm1, xm3 - packuswb xm0, xm1 - movq [dstq+dsq*0], xm0 - movhps [dstq+dsq*1], xm0 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .h_w8 - RET -.h_w16: - movu xm0, [srcq+ssq*0+8*0] - vinserti128 m0, m0, [srcq+ssq*1+8*0], 1 - movu xm1, [srcq+ssq*0+8*1] - vinserti128 m1, m1, [srcq+ssq*1+8*1], 1 - lea srcq, [srcq+ssq*2] - pshufb m0, m4 - pshufb m1, m4 - pmaddubsw m0, m5 - pmaddubsw m1, m5 - pmulhrsw m0, m3 - pmulhrsw m1, m3 - packuswb m0, m1 - mova [dstq+dsq*0], xm0 - vextracti128 [dstq+dsq*1], m0, 1 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .h_w16 - RET -.h_w32: - movu m0, [srcq+8*0] - movu m1, [srcq+8*1] - add srcq, ssq - pshufb m0, m4 - pshufb m1, m4 - pmaddubsw m0, m5 - pmaddubsw m1, m5 - pmulhrsw m0, m3 - pmulhrsw m1, m3 - packuswb m0, m1 - mova [dstq], m0 - add dstq, dsq - dec hd - jg .h_w32 - RET -.h_w64: - movu m0, [srcq+8*0] - movu m1, [srcq+8*1] - pshufb m0, m4 - pshufb m1, m4 - pmaddubsw m0, m5 - pmaddubsw m1, m5 - pmulhrsw m0, m3 - pmulhrsw m1, m3 - packuswb m0, m1 - movu m1, [srcq+8*4] - movu m2, [srcq+8*5] - add srcq, ssq - pshufb m1, m4 - pshufb m2, m4 - pmaddubsw m1, m5 - pmaddubsw m2, m5 - pmulhrsw m1, m3 - pmulhrsw m2, m3 - packuswb m1, m2 - mova [dstq+32*0], m0 - mova [dstq+32*1], m1 - add dstq, dsq - dec hd - jg .h_w64 - RET -.h_w128: - mov t1, -32*3 -.h_w128_loop: - movu m0, [srcq+t1+32*3+8*0] - movu m1, [srcq+t1+32*3+8*1] - pshufb m0, m4 - pshufb m1, m4 - pmaddubsw m0, m5 - pmaddubsw m1, m5 - pmulhrsw m0, m3 - pmulhrsw m1, m3 - packuswb m0, m1 - mova [dstq+t1+32*3], m0 - add t1, 32 - jle .h_w128_loop - add srcq, ssq - add dstq, dsq - dec hd - jg .h_w128 - RET -.v: - movzx wd, word [t2+wq*2+table_offset(put, _bilin_v)] - imul mxyd, 0xff01 - vpbroadcastd m5, [pw_2048] - add mxyd, 16 << 8 - add wq, t2 - movd xm4, mxyd - vpbroadcastw m4, xm4 - jmp wq -.v_w2: - movd xm0, [srcq+ssq*0] -.v_w2_loop: - pinsrw xm1, xm0, [srcq+ssq*1], 1 ; 0 1 - lea srcq, [srcq+ssq*2] - pinsrw xm0, xm1, [srcq+ssq*0], 0 ; 2 1 - pshuflw xm1, xm1, q2301 ; 1 0 - punpcklbw xm1, xm0, xm1 - pmaddubsw xm1, xm4 - pmulhrsw xm1, xm5 - packuswb xm1, xm1 - pextrw [dstq+dsq*0], xm1, 1 - pextrw [dstq+dsq*1], xm1, 0 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .v_w2_loop - RET -.v_w4: - movd xm0, [srcq+ssq*0] -.v_w4_loop: - vpbroadcastd xm1, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - vpblendd xm2, xm1, xm0, 0x01 ; 0 1 - vpbroadcastd xm0, [srcq+ssq*0] - vpblendd xm1, xm1, xm0, 0x02 ; 1 2 - punpcklbw xm1, xm2 - pmaddubsw xm1, xm4 - pmulhrsw xm1, xm5 - packuswb xm1, xm1 - movd [dstq+dsq*0], xm1 - pextrd [dstq+dsq*1], xm1, 1 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .v_w4_loop - RET -.v_w8: - movq xm0, [srcq+ssq*0] -.v_w8_loop: - movq xm3, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - punpcklbw xm1, xm3, xm0 - movq xm0, [srcq+ssq*0] - punpcklbw xm2, xm0, xm3 - pmaddubsw xm1, xm4 - pmaddubsw xm2, xm4 - pmulhrsw xm1, xm5 - pmulhrsw xm2, xm5 - packuswb xm1, xm2 - movq [dstq+dsq*0], xm1 - movhps [dstq+dsq*1], xm1 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .v_w8_loop - RET -.v_w16: - movu xm0, [srcq+ssq*0] -.v_w16_loop: - vbroadcasti128 m2, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - vpblendd m3, m2, m0, 0x0f ; 0 1 - vbroadcasti128 m0, [srcq+ssq*0] - vpblendd m2, m2, m0, 0xf0 ; 1 2 - punpcklbw m1, m2, m3 - punpckhbw m2, m3 - pmaddubsw m1, m4 - pmaddubsw m2, m4 - pmulhrsw m1, m5 - pmulhrsw m2, m5 - packuswb m1, m2 - mova [dstq+dsq*0], xm1 - vextracti128 [dstq+dsq*1], m1, 1 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .v_w16_loop - RET -.v_w32: -%macro PUT_BILIN_V_W32 0 - movu m0, [srcq+ssq*0] -%%loop: - movu m3, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - punpcklbw m1, m3, m0 - punpckhbw m2, m3, m0 - movu m0, [srcq+ssq*0] - pmaddubsw m1, m4 - pmaddubsw m2, m4 - pmulhrsw m1, m5 - pmulhrsw m2, m5 - packuswb m1, m2 - mova [dstq+dsq*0], m1 - punpcklbw m1, m0, m3 - punpckhbw m2, m0, m3 - pmaddubsw m1, m4 - pmaddubsw m2, m4 - pmulhrsw m1, m5 - pmulhrsw m2, m5 - packuswb m1, m2 - mova [dstq+dsq*1], m1 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg %%loop -%endmacro - PUT_BILIN_V_W32 - RET -.v_w64: - movu m0, [srcq+32*0] - movu m1, [srcq+32*1] -.v_w64_loop: - add srcq, ssq - movu m3, [srcq+32*0] - punpcklbw m2, m3, m0 - punpckhbw m0, m3, m0 - pmaddubsw m2, m4 - pmaddubsw m0, m4 - pmulhrsw m2, m5 - pmulhrsw m0, m5 - packuswb m2, m0 - mova m0, m3 - movu m3, [srcq+32*1] - mova [dstq+32*0], m2 - punpcklbw m2, m3, m1 - punpckhbw m1, m3, m1 - pmaddubsw m2, m4 - pmaddubsw m1, m4 - pmulhrsw m2, m5 - pmulhrsw m1, m5 - packuswb m2, m1 - mova m1, m3 - mova [dstq+32*1], m2 - add dstq, dsq - dec hd - jg .v_w64_loop - RET -.v_w128: - mov t0, dstq - mov t1, srcq - lea t2d, [hq+(3<<8)] -.v_w128_loop: - PUT_BILIN_V_W32 - movzx hd, t2b - add t0, 32 - add t1, 32 - mov dstq, t0 - mov srcq, t1 - sub t2d, 1<<8 - jg .v_w128_loop - RET -.hv: - ; (16 * src[x] + (my * (src[x + src_stride] - src[x])) + 128) >> 8 - ; = (src[x] + ((my * (src[x + src_stride] - src[x])) >> 4) + 8) >> 4 - movzx wd, word [t2+wq*2+table_offset(put, _bilin_hv)] - WIN64_SPILL_XMM 8 - shl mxyd, 11 ; can't shift by 12 due to signed overflow - vpbroadcastd m7, [pw_2048] - movd xm6, mxyd - add wq, t2 - vpbroadcastw m6, xm6 - jmp wq -.hv_w2: - vpbroadcastd xm0, [srcq+ssq*0] - pshufb xm0, xm4 - pmaddubsw xm0, xm5 -.hv_w2_loop: - movd xm1, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - pinsrd xm1, [srcq+ssq*0], 1 - pshufb xm1, xm4 - pmaddubsw xm1, xm5 ; 1 _ 2 _ - shufps xm2, xm0, xm1, q1032 ; 0 _ 1 _ - mova xm0, xm1 - psubw xm1, xm2 - paddw xm1, xm1 - pmulhw xm1, xm6 - paddw xm1, xm2 - pmulhrsw xm1, xm7 - packuswb xm1, xm1 - pextrw [dstq+dsq*0], xm1, 0 - pextrw [dstq+dsq*1], xm1, 2 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .hv_w2_loop - RET -.hv_w4: - mova xm4, [bilin_h_shuf4] - movddup xm0, [srcq+ssq*0] - pshufb xm0, xm4 - pmaddubsw xm0, xm5 -.hv_w4_loop: - movq xm1, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - movhps xm1, [srcq+ssq*0] - pshufb xm1, xm4 - pmaddubsw xm1, xm5 ; 1 2 - shufps xm2, xm0, xm1, q1032 ; 0 1 - mova xm0, xm1 - psubw xm1, xm2 - paddw xm1, xm1 - pmulhw xm1, xm6 - paddw xm1, xm2 - pmulhrsw xm1, xm7 - packuswb xm1, xm1 - movd [dstq+dsq*0], xm1 - pextrd [dstq+dsq*1], xm1, 1 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .hv_w4_loop - RET -.hv_w8: - vbroadcasti128 m0, [srcq+ssq*0] - pshufb m0, m4 - pmaddubsw m0, m5 -.hv_w8_loop: - movu xm1, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - vinserti128 m1, m1, [srcq+ssq*0], 1 - pshufb m1, m4 - pmaddubsw m1, m5 ; 1 2 - vperm2i128 m2, m0, m1, 0x21 ; 0 1 - mova m0, m1 - psubw m1, m2 - paddw m1, m1 - pmulhw m1, m6 - paddw m1, m2 - pmulhrsw m1, m7 - vextracti128 xm2, m1, 1 - packuswb xm1, xm2 - movq [dstq+dsq*0], xm1 - movhps [dstq+dsq*1], xm1 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .hv_w8_loop - RET -.hv_w16: - movu m0, [srcq+ssq*0+8*0] - vinserti128 m0, m0, [srcq+ssq*0+8*1], 1 - pshufb m0, m4 - pmaddubsw m0, m5 -.hv_w16_loop: - movu xm2, [srcq+ssq*1+8*0] - vinserti128 m2, m2, [srcq+ssq*1+8*1], 1 - lea srcq, [srcq+ssq*2] - movu xm3, [srcq+ssq*0+8*0] - vinserti128 m3, m3, [srcq+ssq*0+8*1], 1 - pshufb m2, m4 - pshufb m3, m4 - pmaddubsw m2, m5 - psubw m1, m2, m0 - paddw m1, m1 - pmulhw m1, m6 - paddw m1, m0 - pmaddubsw m0, m3, m5 - psubw m3, m0, m2 - paddw m3, m3 - pmulhw m3, m6 - paddw m3, m2 - pmulhrsw m1, m7 - pmulhrsw m3, m7 - packuswb m1, m3 - vpermq m1, m1, q3120 - mova [dstq+dsq*0], xm1 - vextracti128 [dstq+dsq*1], m1, 1 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .hv_w16_loop - RET -.hv_w32: - xor t2d, t2d -.hv_w32gt: - mov t0, dstq - mov t1, srcq -%if WIN64 - movaps r4m, xmm8 -%endif -.hv_w32_loop0: - movu m0, [srcq+8*0] - vinserti128 m0, m0, [srcq+8*2], 1 - movu m1, [srcq+8*1] - vinserti128 m1, m1, [srcq+8*3], 1 - pshufb m0, m4 - pshufb m1, m4 - pmaddubsw m0, m5 - pmaddubsw m1, m5 -.hv_w32_loop: - add srcq, ssq - movu xm2, [srcq+8*1] - vinserti128 m2, m2, [srcq+8*3], 1 - pshufb m2, m4 - pmaddubsw m2, m5 - psubw m3, m2, m1 - paddw m3, m3 - pmulhw m3, m6 - paddw m3, m1 - mova m1, m2 - pmulhrsw m8, m3, m7 - movu xm2, [srcq+8*0] - vinserti128 m2, m2, [srcq+8*2], 1 - pshufb m2, m4 - pmaddubsw m2, m5 - psubw m3, m2, m0 - paddw m3, m3 - pmulhw m3, m6 - paddw m3, m0 - mova m0, m2 - pmulhrsw m3, m7 - packuswb m3, m8 - mova [dstq], m3 - add dstq, dsq - dec hd - jg .hv_w32_loop - movzx hd, t2b - add t0, 32 - add t1, 32 - mov dstq, t0 - mov srcq, t1 - sub t2d, 1<<8 - jg .hv_w32_loop0 -%if WIN64 - movaps xmm8, r4m -%endif - RET -.hv_w64: - lea t2d, [hq+(1<<8)] - jmp .hv_w32gt -.hv_w128: - lea t2d, [hq+(3<<8)] - jmp .hv_w32gt - -%macro PREP_BILIN 0 -DECLARE_REG_TMP 3, 5, 6 -cglobal prep_bilin, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3 - movifnidn mxyd, r5m ; mx - lea t2, [prep%+SUFFIX] - tzcnt wd, wm - movifnidn hd, hm - test mxyd, mxyd - jnz .h - mov mxyd, r6m ; my - test mxyd, mxyd - jnz .v -.prep: - movzx wd, word [t2+wq*2+table_offset(prep,)] - add wq, t2 - lea stride3q, [strideq*3] - jmp wq -.prep_w4: - movd xm0, [srcq+strideq*0] - pinsrd xm0, [srcq+strideq*1], 1 - pinsrd xm0, [srcq+strideq*2], 2 - pinsrd xm0, [srcq+stride3q ], 3 - lea srcq, [srcq+strideq*4] - pmovzxbw ym0, xm0 - psllw ym0, 4 - mova [tmpq], ym0 - add tmpq, 32 - sub hd, 4 - jg .prep_w4 - RET -.prep_w8: - movq xm0, [srcq+strideq*0] -%if cpuflag(avx512) - movq xm1, [srcq+strideq*1] - vinserti128 ym0, [srcq+strideq*2], 1 - vinserti128 ym1, [srcq+stride3q ], 1 - lea srcq, [srcq+strideq*4] - punpcklqdq ym0, ym1 - pmovzxbw m0, ym0 - psllw m0, 4 - mova [tmpq], m0 -%else - movhps xm0, [srcq+strideq*1] - movq xm1, [srcq+strideq*2] - movhps xm1, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - pmovzxbw m0, xm0 - pmovzxbw m1, xm1 - psllw m0, 4 - psllw m1, 4 - mova [tmpq+32*0], m0 - mova [tmpq+32*1], m1 -%endif - add tmpq, 32*2 - sub hd, 4 - jg .prep_w8 - RET -.prep_w16: -%if cpuflag(avx512) - movu xm0, [srcq+strideq*0] - vinserti128 ym0, [srcq+strideq*1], 1 - movu xm1, [srcq+strideq*2] - vinserti128 ym1, [srcq+stride3q ], 1 - pmovzxbw m0, ym0 - pmovzxbw m1, ym1 -%else - pmovzxbw m0, [srcq+strideq*0] - pmovzxbw m1, [srcq+strideq*1] - pmovzxbw m2, [srcq+strideq*2] - pmovzxbw m3, [srcq+stride3q ] -%endif - lea srcq, [srcq+strideq*4] - psllw m0, 4 - psllw m1, 4 -%if notcpuflag(avx512) - psllw m2, 4 - psllw m3, 4 -%endif - mova [tmpq+mmsize*0], m0 - mova [tmpq+mmsize*1], m1 -%if notcpuflag(avx512) - mova [tmpq+32*2], m2 - mova [tmpq+32*3], m3 -%endif - add tmpq, 32*4 - sub hd, 4 - jg .prep_w16 - RET -.prep_w32: -%if cpuflag(avx512) - pmovzxbw m0, [srcq+strideq*0] - pmovzxbw m1, [srcq+strideq*1] - pmovzxbw m2, [srcq+strideq*2] - pmovzxbw m3, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] -%else - pmovzxbw m0, [srcq+strideq*0+16*0] - pmovzxbw m1, [srcq+strideq*0+16*1] - pmovzxbw m2, [srcq+strideq*1+16*0] - pmovzxbw m3, [srcq+strideq*1+16*1] - lea srcq, [srcq+strideq*2] -%endif - psllw m0, 4 - psllw m1, 4 - psllw m2, 4 - psllw m3, 4 - mova [tmpq+mmsize*0], m0 - mova [tmpq+mmsize*1], m1 - mova [tmpq+mmsize*2], m2 - mova [tmpq+mmsize*3], m3 - add tmpq, mmsize*4 - sub hd, mmsize*4/(32*2) - jg .prep_w32 - RET -.prep_w64: -%if cpuflag(avx512) - pmovzxbw m0, [srcq+strideq*0+32*0] - pmovzxbw m1, [srcq+strideq*0+32*1] - pmovzxbw m2, [srcq+strideq*1+32*0] - pmovzxbw m3, [srcq+strideq*1+32*1] - lea srcq, [srcq+strideq*2] -%else - pmovzxbw m0, [srcq+16*0] - pmovzxbw m1, [srcq+16*1] - pmovzxbw m2, [srcq+16*2] - pmovzxbw m3, [srcq+16*3] - add srcq, strideq -%endif - psllw m0, 4 - psllw m1, 4 - psllw m2, 4 - psllw m3, 4 - mova [tmpq+mmsize*0], m0 - mova [tmpq+mmsize*1], m1 - mova [tmpq+mmsize*2], m2 - mova [tmpq+mmsize*3], m3 - add tmpq, mmsize*4 -%if cpuflag(avx512) - sub hd, 2 -%else - dec hd -%endif - jg .prep_w64 - RET -.prep_w128: - pmovzxbw m0, [srcq+(mmsize/2)*0] - pmovzxbw m1, [srcq+(mmsize/2)*1] - pmovzxbw m2, [srcq+(mmsize/2)*2] - pmovzxbw m3, [srcq+(mmsize/2)*3] - psllw m0, 4 - psllw m1, 4 - psllw m2, 4 - psllw m3, 4 - mova [tmpq+mmsize*0], m0 - mova [tmpq+mmsize*1], m1 - mova [tmpq+mmsize*2], m2 - mova [tmpq+mmsize*3], m3 -%if notcpuflag(avx512) - pmovzxbw m0, [srcq+16*4] - pmovzxbw m1, [srcq+16*5] - pmovzxbw m2, [srcq+16*6] - pmovzxbw m3, [srcq+16*7] -%endif - add tmpq, 32*8 - add srcq, strideq -%if notcpuflag(avx512) - psllw m0, 4 - psllw m1, 4 - psllw m2, 4 - psllw m3, 4 - mova [tmpq-32*4], m0 - mova [tmpq-32*3], m1 - mova [tmpq-32*2], m2 - mova [tmpq-32*1], m3 -%endif - dec hd - jg .prep_w128 - RET -.h: - ; 16 * src[x] + (mx * (src[x + 1] - src[x])) - ; = (16 - mx) * src[x] + mx * src[x + 1] - imul mxyd, 0xff01 - add mxyd, 16 << 8 -%if cpuflag(avx512) - vpbroadcastw m5, mxyd -%else - movd xm5, mxyd - vbroadcasti128 m4, [bilin_h_shuf8] - vpbroadcastw m5, xm5 -%endif - mov mxyd, r6m ; my - test mxyd, mxyd - jnz .hv - movzx wd, word [t2+wq*2+table_offset(prep, _bilin_h)] - add wq, t2 - lea stride3q, [strideq*3] - jmp wq -.h_w4: - vbroadcasti128 ym4, [bilin_h_shuf4] -.h_w4_loop: - movq xm0, [srcq+strideq*0] - movhps xm0, [srcq+strideq*1] - movq xm1, [srcq+strideq*2] - movhps xm1, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - vinserti128 ym0, xm1, 1 - pshufb ym0, ym4 - pmaddubsw ym0, ym5 - mova [tmpq], ym0 - add tmpq, 32 - sub hd, 4 - jg .h_w4_loop - RET -.h_w8: -%if cpuflag(avx512) - vbroadcasti128 m4, [bilin_h_shuf8] -.h_w8_loop: - movu xm0, [srcq+strideq*0] - vinserti128 ym0, [srcq+strideq*1], 1 - vinserti128 m0, [srcq+strideq*2], 2 - vinserti128 m0, [srcq+stride3q ], 3 - lea srcq, [srcq+strideq*4] - pshufb m0, m4 - pmaddubsw m0, m5 - mova [tmpq+64*0], m0 -%else -.h_w8_loop: - movu xm0, [srcq+strideq*0] - vinserti128 m0, [srcq+strideq*1], 1 - movu xm1, [srcq+strideq*2] - vinserti128 m1, [srcq+stride3q ], 1 - lea srcq, [srcq+strideq*4] - pshufb m0, m4 - pshufb m1, m4 - pmaddubsw m0, m5 - pmaddubsw m1, m5 - mova [tmpq+32*0], m0 - mova [tmpq+32*1], m1 -%endif - add tmpq, 32*2 - sub hd, 4 - jg .h_w8_loop - RET -.h_w16: -%if cpuflag(avx512icl) - mova m4, [bilin_h_perm16] -.h_w16_loop: - movu ym0, [srcq+strideq*0] - vinserti32x8 m0, [srcq+strideq*1], 1 - movu ym1, [srcq+strideq*2] - vinserti32x8 m1, [srcq+stride3q ], 1 - lea srcq, [srcq+strideq*4] - vpermb m0, m4, m0 - vpermb m1, m4, m1 - pmaddubsw m0, m5 - pmaddubsw m1, m5 - mova [tmpq+64*0], m0 - mova [tmpq+64*1], m1 -%else -.h_w16_loop: - movu xm0, [srcq+strideq*0+8*0] - vinserti128 m0, [srcq+strideq*0+8*1], 1 - movu xm1, [srcq+strideq*1+8*0] - vinserti128 m1, [srcq+strideq*1+8*1], 1 - movu xm2, [srcq+strideq*2+8*0] - vinserti128 m2, [srcq+strideq*2+8*1], 1 - movu xm3, [srcq+stride3q +8*0] - vinserti128 m3, [srcq+stride3q +8*1], 1 - lea srcq, [srcq+strideq*4] - pshufb m0, m4 - pshufb m1, m4 - pshufb m2, m4 - pshufb m3, m4 - pmaddubsw m0, m5 - pmaddubsw m1, m5 - pmaddubsw m2, m5 - pmaddubsw m3, m5 - mova [tmpq+32*0], m0 - mova [tmpq+32*1], m1 - mova [tmpq+32*2], m2 - mova [tmpq+32*3], m3 -%endif - add tmpq, 32*4 - sub hd, 4 - jg .h_w16_loop - RET -.h_w32: -%if cpuflag(avx512icl) - mova m4, [bilin_h_perm32] -.h_w32_loop: - vpermb m0, m4, [srcq+strideq*0] - vpermb m1, m4, [srcq+strideq*1] - vpermb m2, m4, [srcq+strideq*2] - vpermb m3, m4, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] -%else -.h_w32_loop: - movu xm0, [srcq+strideq*0+8*0] - vinserti128 m0, [srcq+strideq*0+8*1], 1 - movu xm1, [srcq+strideq*0+8*2] - vinserti128 m1, [srcq+strideq*0+8*3], 1 - movu xm2, [srcq+strideq*1+8*0] - vinserti128 m2, [srcq+strideq*1+8*1], 1 - movu xm3, [srcq+strideq*1+8*2] - vinserti128 m3, [srcq+strideq*1+8*3], 1 - lea srcq, [srcq+strideq*2] - pshufb m0, m4 - pshufb m1, m4 - pshufb m2, m4 - pshufb m3, m4 -%endif - pmaddubsw m0, m5 - pmaddubsw m1, m5 - pmaddubsw m2, m5 - pmaddubsw m3, m5 - mova [tmpq+mmsize*0], m0 - mova [tmpq+mmsize*1], m1 - mova [tmpq+mmsize*2], m2 - mova [tmpq+mmsize*3], m3 - add tmpq, mmsize*4 - sub hd, mmsize*4/(32*2) - jg .h_w32_loop - RET -.h_w64: -%if cpuflag(avx512icl) - mova m4, [bilin_h_perm32] -.h_w64_loop: - vpermb m0, m4, [srcq+strideq*0+32*0] - vpermb m1, m4, [srcq+strideq*0+32*1] - vpermb m2, m4, [srcq+strideq*1+32*0] - vpermb m3, m4, [srcq+strideq*1+32*1] - lea srcq, [srcq+strideq*2] -%else -.h_w64_loop: - movu xm0, [srcq+8*0] - vinserti128 m0, [srcq+8*1], 1 - movu xm1, [srcq+8*2] - vinserti128 m1, [srcq+8*3], 1 - movu xm2, [srcq+8*4] - vinserti128 m2, [srcq+8*5], 1 - movu xm3, [srcq+8*6] - vinserti128 m3, [srcq+8*7], 1 - add srcq, strideq - pshufb m0, m4 - pshufb m1, m4 - pshufb m2, m4 - pshufb m3, m4 -%endif - pmaddubsw m0, m5 - pmaddubsw m1, m5 - pmaddubsw m2, m5 - pmaddubsw m3, m5 - mova [tmpq+mmsize*0], m0 - mova [tmpq+mmsize*1], m1 - mova [tmpq+mmsize*2], m2 - mova [tmpq+mmsize*3], m3 - add tmpq, mmsize*4 -%if cpuflag(avx512) - sub hd, 2 -%else - dec hd -%endif - jg .h_w64_loop - RET -.h_w128: -%if cpuflag(avx512icl) - mova m4, [bilin_h_perm32] -.h_w128_loop: - vpermb m0, m4, [srcq+32*0] - vpermb m1, m4, [srcq+32*1] - vpermb m2, m4, [srcq+32*2] - vpermb m3, m4, [srcq+32*3] -%else -.h_w128_loop: - movu xm0, [srcq+8*0] - vinserti128 m0, [srcq+8*1], 1 - movu xm1, [srcq+8*2] - vinserti128 m1, [srcq+8*3], 1 - movu xm2, [srcq+8*4] - vinserti128 m2, [srcq+8*5], 1 - movu xm3, [srcq+8*6] - vinserti128 m3, [srcq+8*7], 1 - pshufb m0, m4 - pshufb m1, m4 - pshufb m2, m4 - pshufb m3, m4 -%endif - pmaddubsw m0, m5 - pmaddubsw m1, m5 - pmaddubsw m2, m5 - pmaddubsw m3, m5 - mova [tmpq+mmsize*0], m0 - mova [tmpq+mmsize*1], m1 - mova [tmpq+mmsize*2], m2 - mova [tmpq+mmsize*3], m3 -%if notcpuflag(avx512) - movu xm0, [srcq+8* 8] - vinserti128 m0, [srcq+8* 9], 1 - movu xm1, [srcq+8*10] - vinserti128 m1, [srcq+8*11], 1 - movu xm2, [srcq+8*12] - vinserti128 m2, [srcq+8*13], 1 - movu xm3, [srcq+8*14] - vinserti128 m3, [srcq+8*15], 1 -%endif - add tmpq, 32*8 - add srcq, strideq -%if notcpuflag(avx512) - pshufb m0, m4 - pshufb m1, m4 - pshufb m2, m4 - pshufb m3, m4 - pmaddubsw m0, m5 - pmaddubsw m1, m5 - pmaddubsw m2, m5 - pmaddubsw m3, m5 - mova [tmpq-32*4], m0 - mova [tmpq-32*3], m1 - mova [tmpq-32*2], m2 - mova [tmpq-32*1], m3 -%endif - dec hd - jg .h_w128_loop - RET -.v: - WIN64_SPILL_XMM 7 - movzx wd, word [t2+wq*2+table_offset(prep, _bilin_v)] - imul mxyd, 0xff01 - add mxyd, 16 << 8 - add wq, t2 - lea stride3q, [strideq*3] -%if cpuflag(avx512) - vpbroadcastw m6, mxyd -%else - movd xm6, mxyd - vpbroadcastw m6, xm6 -%endif - jmp wq -.v_w4: -%if cpuflag(avx512) - vpbroadcastd xm0, [srcq+strideq*0] - mov r3d, 0x29 - vbroadcasti128 ym3, [bilin_v_shuf4] - kmovb k1, r3d -.v_w4_loop: - vpblendmd xm1{k1}, xm0, [srcq+strideq*1] {1to4} ; __01 ____ - vpbroadcastd ym2, [srcq+strideq*2] - vpbroadcastd ym2{k1}, [srcq+stride3q ] ; __2_ 23__ - lea srcq, [srcq+strideq*4] - vpbroadcastd ym0, [srcq+strideq*0] - punpckhqdq ym2{k1}, ym1, ym0 ; 012_ 234_ - pshufb ym2, ym3 -%else - movd xm0, [srcq+strideq*0] -.v_w4_loop: - vpbroadcastd m1, [srcq+strideq*2] - vpbroadcastd xm2, [srcq+strideq*1] - vpbroadcastd m3, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - vpblendd m1, m1, m0, 0x05 ; 0 2 2 2 - vpbroadcastd m0, [srcq+strideq*0] - vpblendd m3, m3, m2, 0x0f ; 1 1 3 3 - vpblendd m2, m1, m0, 0xa0 ; 0 2 2 4 - vpblendd m1, m1, m3, 0xaa ; 0 1 2 3 - vpblendd m2, m2, m3, 0x55 ; 1 2 3 4 - punpcklbw m2, m1 -%endif - pmaddubsw ym2, ym6 - mova [tmpq], ym2 - add tmpq, 32 - sub hd, 4 - jg .v_w4_loop - RET -.v_w8: -%if cpuflag(avx512icl) - mova m5, [bilin_v_perm8] - vbroadcasti128 ym0, [srcq+strideq*0] -%else - movq xm0, [srcq+strideq*0] -%endif -.v_w8_loop: -%if cpuflag(avx512icl) - vinserti128 ym1, ym0, [srcq+strideq*1], 1 - vpbroadcastq ym0, [srcq+strideq*2] - vinserti128 m1, [srcq+stride3q ], 2 - lea srcq, [srcq+strideq*4] - vinserti128 ym0, [srcq+strideq*0], 0 - vpermt2b m1, m5, m0 - pmaddubsw m1, m6 - mova [tmpq], m1 -%else - vpbroadcastq m1, [srcq+strideq*2] - vpbroadcastq m2, [srcq+strideq*1] - vpbroadcastq m3, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - vpblendd m1, m1, m0, 0x03 ; 0 2 2 2 - vpbroadcastq m0, [srcq+strideq*0] - vpblendd m3, m3, m2, 0x33 ; 1 3 1 3 - vpblendd m2, m1, m3, 0x0f ; 1 3 2 2 - vpblendd m1, m1, m3, 0xf0 ; 0 2 1 3 - vpblendd m2, m2, m0, 0xc0 ; 1 3 2 4 - punpcklbw m3, m2, m1 - punpckhbw m2, m1 - pmaddubsw m3, m6 - pmaddubsw m2, m6 - mova [tmpq+32*0], m3 - mova [tmpq+32*1], m2 -%endif - add tmpq, 32*2 - sub hd, 4 - jg .v_w8_loop - RET -.v_w16: -%if cpuflag(avx512icl) - mova m5, [bilin_v_perm16] - movu xm0, [srcq+strideq*0] -.v_w16_loop: - movu xm2, [srcq+strideq*2] - vinserti128 ym1, ym0, [srcq+strideq*1], 1 - vpermt2b m1, m5, m2 - vinserti128 ym2, [srcq+stride3q ], 1 - lea srcq, [srcq+strideq*4] - movu xm0, [srcq+strideq*0] - vpermt2b m2, m5, m0 - pmaddubsw m1, m6 - pmaddubsw m2, m6 - mova [tmpq+64*0], m1 - mova [tmpq+64*1], m2 -%else - vbroadcasti128 m0, [srcq+strideq*0] -.v_w16_loop: - vbroadcasti128 m1, [srcq+strideq*2] - vbroadcasti128 m2, [srcq+strideq*1] - vbroadcasti128 m3, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - shufpd m4, m0, m1, 0x0c ; 0 2 ; 0l2l 0h2h - vbroadcasti128 m0, [srcq+strideq*0] - shufpd m2, m2, m3, 0x0c ; 1 3 ; 1l3l 1h3h - shufpd m1, m1, m0, 0x0c ; 2 4 ; 2l4l 2h4h - punpcklbw m3, m2, m4 - punpcklbw m5, m1, m2 - punpckhbw m1, m2 - punpckhbw m2, m4 - pmaddubsw m3, m6 - pmaddubsw m5, m6 - pmaddubsw m2, m6 - pmaddubsw m1, m6 - mova [tmpq+32*0], m3 - mova [tmpq+32*1], m5 - mova [tmpq+32*2], m2 - mova [tmpq+32*3], m1 -%endif - add tmpq, 32*4 - sub hd, 4 - jg .v_w16_loop - RET -.v_w32: -%if cpuflag(avx512icl) - mova m5, [bilin_v_perm32] - movu ym0, [srcq+strideq*0] -.v_w32_loop: - movu ym2, [srcq+strideq*1] - movu ym3, [srcq+strideq*2] - movu ym4, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - vpermt2b m0, m5, m2 - vpermt2b m2, m5, m3 - vpermt2b m3, m5, m4 - pmaddubsw m1, m0, m6 - movu ym0, [srcq+strideq*0] - vpermt2b m4, m5, m0 - pmaddubsw m2, m6 - pmaddubsw m3, m6 - pmaddubsw m4, m6 - mova [tmpq+64*0], m1 - mova [tmpq+64*1], m2 - mova [tmpq+64*2], m3 - mova [tmpq+64*3], m4 - add tmpq, 64*4 -%else - vpermq ym0, [srcq+strideq*0], q3120 -.v_w32_loop: - vpermq ym1, [srcq+strideq*1], q3120 - vpermq ym2, [srcq+strideq*2], q3120 - vpermq ym3, [srcq+stride3q ], q3120 - lea srcq, [srcq+strideq*4] - punpcklbw m4, m1, m0 - punpckhbw m5, m1, m0 - vpermq ym0, [srcq+strideq*0], q3120 - pmaddubsw m4, m6 - pmaddubsw m5, m6 - mova [tmpq+32*0], ym4 - mova [tmpq+32*1], ym5 - punpcklbw m4, m2, m1 - punpckhbw m5, m2, m1 - pmaddubsw m4, m6 - pmaddubsw m5, m6 - mova [tmpq+32*2], ym4 - mova [tmpq+32*3], ym5 - add tmpq, 32*8 - punpcklbw m4, m3, m2 - punpckhbw m5, m3, m2 - punpcklbw m1, m0, m3 - punpckhbw m2, m0, m3 - pmaddubsw m4, m6 - pmaddubsw m5, m6 - pmaddubsw m1, m6 - pmaddubsw m2, m6 - mova [tmpq-32*4], m4 - mova [tmpq-32*3], m5 - mova [tmpq-32*2], m1 - mova [tmpq-32*1], m2 -%endif - sub hd, 4 - jg .v_w32_loop - RET -.v_w64: -%if cpuflag(avx512) - mova m5, [bilin_v_perm64] - vpermq m0, m5, [srcq+strideq*0] -.v_w64_loop: - vpermq m1, m5, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - punpcklbw m4, m1, m0 - punpckhbw m2, m1, m0 - vpermq m0, m5, [srcq+strideq*0] - punpcklbw m3, m0, m1 - punpckhbw m1, m0, m1 - pmaddubsw m4, m6 - pmaddubsw m2, m6 - pmaddubsw m3, m6 - pmaddubsw m1, m6 - mova [tmpq+64*0], m4 - mova [tmpq+64*1], m2 - mova [tmpq+64*2], m3 - mova [tmpq+64*3], m1 - add tmpq, 64*4 -%else - vpermq m0, [srcq+strideq*0+32*0], q3120 - vpermq m1, [srcq+strideq*0+32*1], q3120 -.v_w64_loop: - vpermq m2, [srcq+strideq*1+32*0], q3120 - vpermq m3, [srcq+strideq*1+32*1], q3120 - lea srcq, [srcq+strideq*2] - punpcklbw m4, m2, m0 - punpckhbw m5, m2, m0 - pmaddubsw m4, m6 - pmaddubsw m5, m6 - mova [tmpq+32*0], m4 - mova [tmpq+32*1], m5 - punpcklbw m4, m3, m1 - punpckhbw m5, m3, m1 - vpermq m0, [srcq+strideq*0+32*0], q3120 - vpermq m1, [srcq+strideq*0+32*1], q3120 - pmaddubsw m4, m6 - pmaddubsw m5, m6 - mova [tmpq+32*2], m4 - mova [tmpq+32*3], m5 - add tmpq, 32*8 - punpcklbw m4, m0, m2 - punpckhbw m5, m0, m2 - punpcklbw m2, m1, m3 - punpckhbw m3, m1, m3 - pmaddubsw m4, m6 - pmaddubsw m5, m6 - pmaddubsw m2, m6 - pmaddubsw m3, m6 - mova [tmpq-32*4], m4 - mova [tmpq-32*3], m5 - mova [tmpq-32*2], m2 - mova [tmpq-32*1], m3 -%endif - sub hd, 2 - jg .v_w64_loop - RET -.v_w128: -%if cpuflag(avx512) - mova m5, [bilin_v_perm64] - vpermq m0, m5, [srcq+strideq*0+ 0] - vpermq m1, m5, [srcq+strideq*0+64] -.v_w128_loop: - vpermq m2, m5, [srcq+strideq*1+ 0] - vpermq m3, m5, [srcq+strideq*1+64] - lea srcq, [srcq+strideq*2] - punpcklbw m4, m2, m0 - punpckhbw m0, m2, m0 - pmaddubsw m4, m6 - pmaddubsw m0, m6 - mova [tmpq+64*0], m4 - mova [tmpq+64*1], m0 - punpcklbw m4, m3, m1 - punpckhbw m1, m3, m1 - pmaddubsw m4, m6 - pmaddubsw m1, m6 - mova [tmpq+64*2], m4 - mova [tmpq+64*3], m1 - vpermq m0, m5, [srcq+strideq*0+ 0] - vpermq m1, m5, [srcq+strideq*0+64] - punpcklbw m4, m0, m2 - punpckhbw m2, m0, m2 - pmaddubsw m4, m6 - pmaddubsw m2, m6 - mova [tmpq+64*4], m4 - mova [tmpq+64*5], m2 - punpcklbw m4, m1, m3 - punpckhbw m3, m1, m3 - pmaddubsw m4, m6 - pmaddubsw m3, m6 - mova [tmpq+64*6], m4 - mova [tmpq+64*7], m3 - add tmpq, 64*8 - sub hd, 2 - jg .v_w128_loop -%else - mov t0, tmpq - mov t1, srcq - lea t2d, [hq+(3<<8)] -.v_w128_loop0: - vpermq m0, [srcq+strideq*0], q3120 -.v_w128_loop: - vpermq m1, [srcq+strideq*1], q3120 - lea srcq, [srcq+strideq*2] - punpcklbw m2, m1, m0 - punpckhbw m3, m1, m0 - vpermq m0, [srcq+strideq*0], q3120 - punpcklbw m4, m0, m1 - punpckhbw m5, m0, m1 - pmaddubsw m2, m6 - pmaddubsw m3, m6 - pmaddubsw m4, m6 - pmaddubsw m5, m6 - mova [tmpq+32*0], m2 - mova [tmpq+32*1], m3 - mova [tmpq+32*8], m4 - mova [tmpq+32*9], m5 - add tmpq, 32*16 - sub hd, 2 - jg .v_w128_loop - movzx hd, t2b - add t0, 64 - add t1, 32 - mov tmpq, t0 - mov srcq, t1 - sub t2d, 1<<8 - jg .v_w128_loop0 -%endif - RET -.hv: - ; (16 * src[x] + (my * (src[x + src_stride] - src[x])) + 8) >> 4 - ; = src[x] + (((my * (src[x + src_stride] - src[x])) + 8) >> 4) - %assign stack_offset stack_offset - stack_size_padded - WIN64_SPILL_XMM 7 - movzx wd, word [t2+wq*2+table_offset(prep, _bilin_hv)] - shl mxyd, 11 -%if cpuflag(avx512) - vpbroadcastw m6, mxyd -%else - movd xm6, mxyd - vpbroadcastw m6, xm6 -%endif - add wq, t2 - lea stride3q, [strideq*3] - jmp wq -.hv_w4: - vbroadcasti128 ym4, [bilin_h_shuf4] - vpbroadcastq ym0, [srcq+strideq*0] - pshufb ym0, ym4 - pmaddubsw ym0, ym5 -.hv_w4_loop: - movq xm1, [srcq+strideq*1] - movhps xm1, [srcq+strideq*2] - movq xm2, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - movhps xm2, [srcq+strideq*0] - vinserti128 ym1, xm2, 1 - pshufb ym1, ym4 - pmaddubsw ym1, ym5 ; 1 2 3 4 -%if cpuflag(avx512) - valignq ym2, ym1, ym0, 3 ; 0 1 2 3 -%else - vpblendd ym2, ym1, ym0, 0xc0 - vpermq ym2, ym2, q2103 ; 0 1 2 3 -%endif - mova ym0, ym1 - psubw ym1, ym2 - pmulhrsw ym1, ym6 - paddw ym1, ym2 - mova [tmpq], ym1 - add tmpq, 32 - sub hd, 4 - jg .hv_w4_loop - RET -.hv_w8: -%if cpuflag(avx512) - vbroadcasti128 m4, [bilin_h_shuf8] -%endif - vbroadcasti128 m0, [srcq+strideq*0] - pshufb m0, m4 - pmaddubsw m0, m5 -.hv_w8_loop: - movu xm1, [srcq+strideq*1] -%if cpuflag(avx512) - vinserti128 ym1, [srcq+strideq*2], 1 - vinserti128 m1, [srcq+stride3q ], 2 - lea srcq, [srcq+strideq*4] - vinserti128 m1, [srcq+strideq*0], 3 - pshufb m1, m4 - pmaddubsw m1, m5 ; 1 2 3 4 - valignq m2, m1, m0, 6 ; 0 1 2 3 - mova m0, m1 - psubw m1, m2 - pmulhrsw m1, m6 - paddw m1, m2 - mova [tmpq], m1 -%else - vinserti128 m1, m1, [srcq+strideq*2], 1 - movu xm2, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - vinserti128 m2, m2, [srcq+strideq*0], 1 - pshufb m1, m4 - pshufb m2, m4 - pmaddubsw m1, m5 ; 1 2 - vperm2i128 m3, m0, m1, 0x21 ; 0 1 - pmaddubsw m0, m2, m5 ; 3 4 - vperm2i128 m2, m1, m0, 0x21 ; 2 3 - psubw m1, m3 - pmulhrsw m1, m6 - paddw m1, m3 - psubw m3, m0, m2 - pmulhrsw m3, m6 - paddw m3, m2 - mova [tmpq+32*0], m1 - mova [tmpq+32*1], m3 -%endif - add tmpq, 32*2 - sub hd, 4 - jg .hv_w8_loop - RET -.hv_w16: -%if cpuflag(avx512icl) - mova m4, [bilin_h_perm16] - vbroadcasti32x8 m0, [srcq+strideq*0] - vpermb m0, m4, m0 -%else - movu xm0, [srcq+strideq*0+8*0] - vinserti128 m0, [srcq+strideq*0+8*1], 1 - pshufb m0, m4 -%endif - pmaddubsw m0, m5 -.hv_w16_loop: -%if cpuflag(avx512icl) - movu ym1, [srcq+strideq*1] - vinserti32x8 m1, [srcq+strideq*2], 1 - movu ym2, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - vinserti32x8 m2, [srcq+strideq*0], 1 - vpermb m1, m4, m1 - vpermb m2, m4, m2 - pmaddubsw m1, m5 ; 1 2 - vshufi32x4 m3, m0, m1, q1032 ; 0 1 - pmaddubsw m0, m2, m5 ; 3 4 - vshufi32x4 m2, m1, m0, q1032 ; 2 3 - psubw m1, m3 - pmulhrsw m1, m6 - paddw m1, m3 - psubw m3, m0, m2 - pmulhrsw m3, m6 - paddw m3, m2 - mova [tmpq+64*0], m1 - mova [tmpq+64*1], m3 -%else - movu xm1, [srcq+strideq*1+8*0] - vinserti128 m1, [srcq+strideq*1+8*1], 1 - lea srcq, [srcq+strideq*2] - movu xm2, [srcq+strideq*0+8*0] - vinserti128 m2, [srcq+strideq*0+8*1], 1 - pshufb m1, m4 - pshufb m2, m4 - pmaddubsw m1, m5 - psubw m3, m1, m0 - pmulhrsw m3, m6 - paddw m3, m0 - pmaddubsw m0, m2, m5 - psubw m2, m0, m1 - pmulhrsw m2, m6 - paddw m2, m1 - mova [tmpq+32*0], m3 - mova [tmpq+32*1], m2 -%endif - add tmpq, mmsize*2 - sub hd, mmsize*2/(16*2) - jg .hv_w16_loop - RET -.hv_w32: -%if cpuflag(avx512icl) - mova m4, [bilin_h_perm32] - vpermb m0, m4, [srcq+strideq*0] - pmaddubsw m0, m5 -.hv_w32_loop: - vpermb m1, m4, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - vpermb m2, m4, [srcq+strideq*0] - pmaddubsw m1, m5 - psubw m3, m1, m0 - pmulhrsw m3, m6 - paddw m3, m0 - pmaddubsw m0, m2, m5 - psubw m2, m0, m1 - pmulhrsw m2, m6 - paddw m2, m1 - mova [tmpq+64*0], m3 - mova [tmpq+64*1], m2 - add tmpq, 64*2 - sub hd, 2 -%else - movu xm0, [srcq+8*0] - vinserti128 m0, [srcq+8*1], 1 - movu xm1, [srcq+8*2] - vinserti128 m1, [srcq+8*3], 1 - pshufb m0, m4 - pshufb m1, m4 - pmaddubsw m0, m5 - pmaddubsw m1, m5 -.hv_w32_loop: - add srcq, strideq - movu xm2, [srcq+8*0] - vinserti128 m2, m2, [srcq+8*1], 1 - pshufb m2, m4 - pmaddubsw m2, m5 - psubw m3, m2, m0 - pmulhrsw m3, m6 - paddw m3, m0 - mova m0, m2 - mova [tmpq+ 0], m3 - movu xm2, [srcq+8*2] - vinserti128 m2, m2, [srcq+8*3], 1 - pshufb m2, m4 - pmaddubsw m2, m5 - psubw m3, m2, m1 - pmulhrsw m3, m6 - paddw m3, m1 - mova m1, m2 - mova [tmpq+32], m3 - add tmpq, 32*2 - dec hd -%endif - jg .hv_w32_loop - RET -.hv_w64: -%if cpuflag(avx512icl) - mova m4, [bilin_h_perm32] - vpermb m0, m4, [srcq+32*0] - vpermb m1, m4, [srcq+32*1] - pmaddubsw m0, m5 - pmaddubsw m1, m5 -.hv_w64_loop: - add srcq, strideq - vpermb m2, m4, [srcq+32*0] - vpermb m3, m4, [srcq+32*1] - pmaddubsw m2, m5 - pmaddubsw m3, m5 - psubw m7, m2, m0 - psubw m8, m3, m1 - pmulhrsw m7, m6 - pmulhrsw m8, m6 - paddw m7, m0 - paddw m8, m1 - mova [tmpq+ 0], m7 - mova [tmpq+64], m8 - mova m0, m2 - mova m1, m3 - add tmpq, 64*2 - dec hd - jg .hv_w64_loop -%else - mov t0, tmpq - mov t1, srcq - lea t2d, [hq+(3<<8)] -.hv_w64_loop0: - movu xm0, [srcq+strideq*0+8*0] - vinserti128 m0, m0, [srcq+strideq*0+8*1], 1 - pshufb m0, m4 - pmaddubsw m0, m5 -.hv_w64_loop: - movu xm1, [srcq+strideq*1+8*0] - vinserti128 m1, m1, [srcq+strideq*1+8*1], 1 - lea srcq, [srcq+strideq*2] - movu xm2, [srcq+strideq*0+8*0] - vinserti128 m2, m2, [srcq+strideq*0+8*1], 1 - pshufb m1, m4 - pshufb m2, m4 - pmaddubsw m1, m5 - psubw m3, m1, m0 - pmulhrsw m3, m6 - paddw m3, m0 - pmaddubsw m0, m2, m5 - psubw m2, m0, m1 - pmulhrsw m2, m6 - paddw m2, m1 - mova [tmpq+32*0], m3 - add tmpq, 32*8 - mova [tmpq-32*4], m2 - sub hd, 2 - jg .hv_w64_loop - movzx hd, t2b - add t0, 32 - add t1, 16 - mov tmpq, t0 - mov srcq, t1 - sub t2d, 1<<8 - jg .hv_w64_loop0 -%endif - RET -.hv_w128: -%if cpuflag(avx512icl) - mova m4, [bilin_h_perm32] - vpermb m0, m4, [srcq+32*0] - vpermb m1, m4, [srcq+32*1] - vpermb m2, m4, [srcq+32*2] - vpermb m3, m4, [srcq+32*3] - pmaddubsw m0, m5 - pmaddubsw m1, m5 - pmaddubsw m2, m5 - pmaddubsw m3, m5 -.hv_w128_loop: - add srcq, strideq - vpermb m7, m4, [srcq+32*0] - vpermb m8, m4, [srcq+32*1] - vpermb m9, m4, [srcq+32*2] - vpermb m10, m4, [srcq+32*3] - pmaddubsw m7, m5 - pmaddubsw m8, m5 - pmaddubsw m9, m5 - pmaddubsw m10, m5 - psubw m11, m7, m0 - psubw m12, m8, m1 - psubw m13, m9, m2 - psubw m14, m10, m3 - pmulhrsw m11, m6 - pmulhrsw m12, m6 - pmulhrsw m13, m6 - pmulhrsw m14, m6 - paddw m11, m0 - paddw m12, m1 - paddw m13, m2 - paddw m14, m3 - mova [tmpq+64*0], m11 - mova [tmpq+64*1], m12 - mova [tmpq+64*2], m13 - mova [tmpq+64*3], m14 - mova m0, m7 - mova m1, m8 - mova m2, m9 - mova m3, m10 - add tmpq, 64*4 - dec hd - jg .hv_w128_loop -%else - mov t0, tmpq - mov t1, srcq - lea t2d, [hq+(7<<8)] -.hv_w128_loop0: - movu xm0, [srcq+strideq*0+8*0] - vinserti128 m0, m0, [srcq+strideq*0+8*1], 1 - pshufb m0, m4 - pmaddubsw m0, m5 -.hv_w128_loop: - movu xm1, [srcq+strideq*1+8*0] - vinserti128 m1, m1, [srcq+strideq*1+8*1], 1 - lea srcq, [srcq+strideq*2] - movu xm2, [srcq+strideq*0+8*0] - vinserti128 m2, m2, [srcq+strideq*0+8*1], 1 - pshufb m1, m4 - pshufb m2, m4 - pmaddubsw m1, m5 - psubw m3, m1, m0 - pmulhrsw m3, m6 - paddw m3, m0 - pmaddubsw m0, m2, m5 - psubw m2, m0, m1 - pmulhrsw m2, m6 - paddw m2, m1 - mova [tmpq+32*0], m3 - mova [tmpq+32*8], m2 - add tmpq, 32*16 - sub hd, 2 - jg .hv_w128_loop - movzx hd, t2b - add t0, mmsize - add t1, mmsize/2 - mov tmpq, t0 - mov srcq, t1 - sub t2d, 1<<8 - jg .hv_w128_loop0 -%endif - RET -%endmacro - -; int8_t subpel_filters[5][15][8] -%assign FILTER_REGULAR (0*15 << 16) | 3*15 -%assign FILTER_SMOOTH (1*15 << 16) | 4*15 -%assign FILTER_SHARP (2*15 << 16) | 3*15 - -%if WIN64 -DECLARE_REG_TMP 4, 5 -%else -DECLARE_REG_TMP 7, 8 -%endif -%macro PUT_8TAP_FN 3 ; type, type_h, type_v -cglobal put_8tap_%1 - mov t0d, FILTER_%2 - mov t1d, FILTER_%3 -%ifnidn %1, sharp_smooth ; skip the jump in the last filter - jmp mangle(private_prefix %+ _put_8tap %+ SUFFIX) -%endif -%endmacro - -PUT_8TAP_FN regular, REGULAR, REGULAR -PUT_8TAP_FN regular_sharp, REGULAR, SHARP -PUT_8TAP_FN regular_smooth, REGULAR, SMOOTH -PUT_8TAP_FN smooth_regular, SMOOTH, REGULAR -PUT_8TAP_FN smooth, SMOOTH, SMOOTH -PUT_8TAP_FN smooth_sharp, SMOOTH, SHARP -PUT_8TAP_FN sharp_regular, SHARP, REGULAR -PUT_8TAP_FN sharp, SHARP, SHARP -PUT_8TAP_FN sharp_smooth, SHARP, SMOOTH - -cglobal put_8tap, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 - imul mxd, mxm, 0x010101 - add mxd, t0d ; 8tap_h, mx, 4tap_h - imul myd, mym, 0x010101 - add myd, t1d ; 8tap_v, my, 4tap_v - lea r8, [put_avx2] - movsxd wq, wm - movifnidn hd, hm - test mxd, 0xf00 - jnz .h - test myd, 0xf00 - jnz .v - tzcnt wd, wd - movzx wd, word [r8+wq*2+table_offset(put,)] - add wq, r8 - lea r6, [ssq*3] - lea r7, [dsq*3] -%if WIN64 - pop r8 -%endif - jmp wq -.h: - test myd, 0xf00 - jnz .hv - vpbroadcastd m5, [pw_34] ; 2 + (8 << 2) - WIN64_SPILL_XMM 11 - cmp wd, 4 - jl .h_w2 - vbroadcasti128 m6, [subpel_h_shufA] - je .h_w4 - tzcnt wd, wd - vbroadcasti128 m7, [subpel_h_shufB] - vbroadcasti128 m8, [subpel_h_shufC] - shr mxd, 16 - sub srcq, 3 - movzx wd, word [r8+wq*2+table_offset(put, _8tap_h)] - vpbroadcastd m9, [r8+mxq*8+subpel_filters-put_avx2+0] - vpbroadcastd m10, [r8+mxq*8+subpel_filters-put_avx2+4] - add wq, r8 - jmp wq -.h_w2: - movzx mxd, mxb - dec srcq - mova xm4, [subpel_h_shuf4] - vpbroadcastd xm3, [r8+mxq*8+subpel_filters-put_avx2+2] -.h_w2_loop: - movq xm0, [srcq+ssq*0] - movhps xm0, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - pshufb xm0, xm4 - pmaddubsw xm0, xm3 - phaddw xm0, xm0 - paddw xm0, xm5 - psraw xm0, 6 - packuswb xm0, xm0 - pextrw [dstq+dsq*0], xm0, 0 - pextrw [dstq+dsq*1], xm0, 1 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .h_w2_loop - RET -.h_w4: - movzx mxd, mxb - dec srcq - vpbroadcastd xm3, [r8+mxq*8+subpel_filters-put_avx2+2] -.h_w4_loop: - movq xm0, [srcq+ssq*0] - movq xm1, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - pshufb xm0, xm6 - pshufb xm1, xm6 - pmaddubsw xm0, xm3 - pmaddubsw xm1, xm3 - phaddw xm0, xm1 - paddw xm0, xm5 - psraw xm0, 6 - packuswb xm0, xm0 - movd [dstq+dsq*0], xm0 - pextrd [dstq+dsq*1], xm0, 1 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .h_w4_loop - RET -.h_w8: -%macro PUT_8TAP_H 4 ; dst/src, tmp[1-3] - pshufb m%2, m%1, m7 - pshufb m%3, m%1, m8 - pshufb m%1, m6 - pmaddubsw m%4, m%2, m9 - pmaddubsw m%2, m10 - pmaddubsw m%3, m10 - pmaddubsw m%1, m9 - paddw m%3, m%4 - paddw m%1, m%2 - phaddw m%1, m%3 - paddw m%1, m5 - psraw m%1, 6 -%endmacro - movu xm0, [srcq+ssq*0] - vinserti128 m0, m0, [srcq+ssq*1], 1 - lea srcq, [srcq+ssq*2] - PUT_8TAP_H 0, 1, 2, 3 - vextracti128 xm1, m0, 1 - packuswb xm0, xm1 - movq [dstq+dsq*0], xm0 - movhps [dstq+dsq*1], xm0 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .h_w8 - RET -.h_w16: - movu xm0, [srcq+ssq*0+8*0] - vinserti128 m0, m0, [srcq+ssq*1+8*0], 1 - movu xm1, [srcq+ssq*0+8*1] - vinserti128 m1, m1, [srcq+ssq*1+8*1], 1 - PUT_8TAP_H 0, 2, 3, 4 - lea srcq, [srcq+ssq*2] - PUT_8TAP_H 1, 2, 3, 4 - packuswb m0, m1 - mova [dstq+dsq*0], xm0 - vextracti128 [dstq+dsq*1], m0, 1 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .h_w16 - RET -.h_w32: - xor r6d, r6d - jmp .h_start -.h_w64: - mov r6, -32*1 - jmp .h_start -.h_w128: - mov r6, -32*3 -.h_start: - sub srcq, r6 - sub dstq, r6 - mov r4, r6 -.h_loop: - movu m0, [srcq+r6+8*0] - movu m1, [srcq+r6+8*1] - PUT_8TAP_H 0, 2, 3, 4 - PUT_8TAP_H 1, 2, 3, 4 - packuswb m0, m1 - mova [dstq+r6], m0 - add r6, 32 - jle .h_loop - add srcq, ssq - add dstq, dsq - mov r6, r4 - dec hd - jg .h_loop - RET -.v: - %assign stack_offset stack_offset - stack_size_padded - WIN64_SPILL_XMM 16 - movzx mxd, myb - shr myd, 16 - cmp hd, 6 - cmovs myd, mxd - tzcnt r6d, wd - movzx r6d, word [r8+r6*2+table_offset(put, _8tap_v)] - vpbroadcastd m7, [pw_512] - lea myq, [r8+myq*8+subpel_filters-put_avx2] - vpbroadcastw m8, [myq+0] - vpbroadcastw m9, [myq+2] - vpbroadcastw m10, [myq+4] - vpbroadcastw m11, [myq+6] - add r6, r8 - lea ss3q, [ssq*3] - sub srcq, ss3q - jmp r6 -.v_w2: - movd xm2, [srcq+ssq*0] - pinsrw xm2, [srcq+ssq*1], 2 - pinsrw xm2, [srcq+ssq*2], 4 - pinsrw xm2, [srcq+ss3q ], 6 ; 0 1 2 3 - lea srcq, [srcq+ssq*4] - movd xm3, [srcq+ssq*0] - vpbroadcastd xm1, [srcq+ssq*1] - vpbroadcastd xm0, [srcq+ssq*2] - add srcq, ss3q - vpblendd xm3, xm3, xm1, 0x02 ; 4 5 - vpblendd xm1, xm1, xm0, 0x02 ; 5 6 - palignr xm4, xm3, xm2, 4 ; 1 2 3 4 - punpcklbw xm3, xm1 ; 45 56 - punpcklbw xm1, xm2, xm4 ; 01 12 - punpckhbw xm2, xm4 ; 23 34 -.v_w2_loop: - pmaddubsw xm5, xm1, xm8 ; a0 b0 - mova xm1, xm2 - pmaddubsw xm2, xm9 ; a1 b1 - paddw xm5, xm2 - mova xm2, xm3 - pmaddubsw xm3, xm10 ; a2 b2 - paddw xm5, xm3 - vpbroadcastd xm4, [srcq+ssq*0] - vpblendd xm3, xm0, xm4, 0x02 ; 6 7 - vpbroadcastd xm0, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - vpblendd xm4, xm4, xm0, 0x02 ; 7 8 - punpcklbw xm3, xm4 ; 67 78 - pmaddubsw xm4, xm3, xm11 ; a3 b3 - paddw xm5, xm4 - pmulhrsw xm5, xm7 - packuswb xm5, xm5 - pextrw [dstq+dsq*0], xm5, 0 - pextrw [dstq+dsq*1], xm5, 2 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .v_w2_loop - RET -.v_w4: - movd xm2, [srcq+ssq*0] - pinsrd xm2, [srcq+ssq*1], 1 - pinsrd xm2, [srcq+ssq*2], 2 - pinsrd xm2, [srcq+ss3q ], 3 ; 0 1 2 3 - lea srcq, [srcq+ssq*4] - movd xm3, [srcq+ssq*0] - vpbroadcastd xm1, [srcq+ssq*1] - vpbroadcastd xm0, [srcq+ssq*2] - add srcq, ss3q - vpblendd xm3, xm3, xm1, 0x02 ; 4 5 - vpblendd xm1, xm1, xm0, 0x02 ; 5 6 - palignr xm4, xm3, xm2, 4 ; 1 2 3 4 - punpcklbw xm3, xm1 ; 45 56 - punpcklbw xm1, xm2, xm4 ; 01 12 - punpckhbw xm2, xm4 ; 23 34 -.v_w4_loop: - pmaddubsw xm5, xm1, xm8 ; a0 b0 - mova xm1, xm2 - pmaddubsw xm2, xm9 ; a1 b1 - paddw xm5, xm2 - mova xm2, xm3 - pmaddubsw xm3, xm10 ; a2 b2 - paddw xm5, xm3 - vpbroadcastd xm4, [srcq+ssq*0] - vpblendd xm3, xm0, xm4, 0x02 ; 6 7 - vpbroadcastd xm0, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - vpblendd xm4, xm4, xm0, 0x02 ; 7 8 - punpcklbw xm3, xm4 ; 67 78 - pmaddubsw xm4, xm3, xm11 ; a3 b3 - paddw xm5, xm4 - pmulhrsw xm5, xm7 - packuswb xm5, xm5 - movd [dstq+dsq*0], xm5 - pextrd [dstq+dsq*1], xm5, 1 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .v_w4_loop - RET -.v_w8: - movq xm1, [srcq+ssq*0] - vpbroadcastq m4, [srcq+ssq*1] - vpbroadcastq m2, [srcq+ssq*2] - vpbroadcastq m5, [srcq+ss3q ] - lea srcq, [srcq+ssq*4] - vpbroadcastq m3, [srcq+ssq*0] - vpbroadcastq m6, [srcq+ssq*1] - vpbroadcastq m0, [srcq+ssq*2] - add srcq, ss3q - vpblendd m1, m1, m4, 0x30 - vpblendd m4, m4, m2, 0x30 - punpcklbw m1, m4 ; 01 12 - vpblendd m2, m2, m5, 0x30 - vpblendd m5, m5, m3, 0x30 - punpcklbw m2, m5 ; 23 34 - vpblendd m3, m3, m6, 0x30 - vpblendd m6, m6, m0, 0x30 - punpcklbw m3, m6 ; 45 56 -.v_w8_loop: - pmaddubsw m5, m1, m8 ; a0 b0 - mova m1, m2 - pmaddubsw m2, m9 ; a1 b1 - paddw m5, m2 - mova m2, m3 - pmaddubsw m3, m10 ; a2 b2 - paddw m5, m3 - vpbroadcastq m4, [srcq+ssq*0] - vpblendd m3, m0, m4, 0x30 - vpbroadcastq m0, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - vpblendd m4, m4, m0, 0x30 - punpcklbw m3, m4 ; 67 78 - pmaddubsw m4, m3, m11 ; a3 b3 - paddw m5, m4 - pmulhrsw m5, m7 - vextracti128 xm4, m5, 1 - packuswb xm5, xm4 - movq [dstq+dsq*0], xm5 - movhps [dstq+dsq*1], xm5 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .v_w8_loop - RET -.v_w16: -.v_w32: -.v_w64: -.v_w128: - lea r6d, [wq-16] - mov r4, dstq - mov r7, srcq - shl r6d, 4 - mov r6b, hb -.v_w16_loop0: - vbroadcasti128 m4, [srcq+ssq*0] - vbroadcasti128 m5, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - vbroadcasti128 m0, [srcq+ssq*1] - vbroadcasti128 m6, [srcq+ssq*0] - lea srcq, [srcq+ssq*2] - vbroadcasti128 m1, [srcq+ssq*0] - vbroadcasti128 m2, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - vbroadcasti128 m3, [srcq+ssq*0] - shufpd m4, m4, m0, 0x0c - shufpd m5, m5, m1, 0x0c - punpcklbw m1, m4, m5 ; 01 - punpckhbw m4, m5 ; 34 - shufpd m6, m6, m2, 0x0c - punpcklbw m2, m5, m6 ; 12 - punpckhbw m5, m6 ; 45 - shufpd m0, m0, m3, 0x0c - punpcklbw m3, m6, m0 ; 23 - punpckhbw m6, m0 ; 56 -.v_w16_loop: - vbroadcasti128 m12, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - vbroadcasti128 m13, [srcq+ssq*0] - pmaddubsw m14, m1, m8 ; a0 - pmaddubsw m15, m2, m8 ; b0 - mova m1, m3 - mova m2, m4 - pmaddubsw m3, m9 ; a1 - pmaddubsw m4, m9 ; b1 - paddw m14, m3 - paddw m15, m4 - mova m3, m5 - mova m4, m6 - pmaddubsw m5, m10 ; a2 - pmaddubsw m6, m10 ; b2 - paddw m14, m5 - paddw m15, m6 - shufpd m6, m0, m12, 0x0d - shufpd m0, m12, m13, 0x0c - punpcklbw m5, m6, m0 ; 67 - punpckhbw m6, m0 ; 78 - pmaddubsw m12, m5, m11 ; a3 - pmaddubsw m13, m6, m11 ; b3 - paddw m14, m12 - paddw m15, m13 - pmulhrsw m14, m7 - pmulhrsw m15, m7 - packuswb m14, m15 - vpermq m14, m14, q3120 - mova [dstq+dsq*0], xm14 - vextracti128 [dstq+dsq*1], m14, 1 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .v_w16_loop - movzx hd, r6b - add r4, 16 - add r7, 16 - mov dstq, r4 - mov srcq, r7 - sub r6d, 1<<8 - jg .v_w16_loop0 - RET -.hv: - %assign stack_offset stack_offset - stack_size_padded - WIN64_SPILL_XMM 16 - cmp wd, 4 - jg .hv_w8 - movzx mxd, mxb - dec srcq - vpbroadcastd m7, [r8+mxq*8+subpel_filters-put_avx2+2] - movzx mxd, myb - shr myd, 16 - cmp hd, 6 - cmovs myd, mxd - vpbroadcastq m0, [r8+myq*8+subpel_filters-put_avx2] - lea ss3q, [ssq*3] - sub srcq, ss3q - punpcklbw m0, m0 - psraw m0, 8 ; sign-extend - vpbroadcastd m8, [pw_8192] - vpbroadcastd m9, [pd_512] - pshufd m10, m0, q0000 - pshufd m11, m0, q1111 - pshufd m12, m0, q2222 - pshufd m13, m0, q3333 - cmp wd, 4 - je .hv_w4 - vbroadcasti128 m6, [subpel_h_shuf4] - movq xm2, [srcq+ssq*0] - movhps xm2, [srcq+ssq*1] - movq xm0, [srcq+ssq*2] - movhps xm0, [srcq+ss3q ] - lea srcq, [srcq+ssq*4] - vpbroadcastq m3, [srcq+ssq*0] - vpbroadcastq m4, [srcq+ssq*1] - vpbroadcastq m1, [srcq+ssq*2] - add srcq, ss3q - vpblendd m2, m2, m3, 0x30 - vpblendd m0, m0, m1, 0x30 - vpblendd m2, m2, m4, 0xc0 - pshufb m2, m6 - pshufb m0, m6 - pmaddubsw m2, m7 - pmaddubsw m0, m7 - phaddw m2, m0 - pmulhrsw m2, m8 - vextracti128 xm3, m2, 1 - palignr xm4, xm3, xm2, 4 - punpcklwd xm1, xm2, xm4 ; 01 12 - punpckhwd xm2, xm4 ; 23 34 - pshufd xm0, xm3, q2121 - punpcklwd xm3, xm0 ; 45 56 -.hv_w2_loop: - pmaddwd xm5, xm1, xm10 ; a0 b0 - mova xm1, xm2 - pmaddwd xm2, xm11 ; a1 b1 - paddd xm5, xm2 - mova xm2, xm3 - pmaddwd xm3, xm12 ; a2 b2 - paddd xm5, xm3 - movq xm4, [srcq+ssq*0] - movhps xm4, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - pshufb xm4, xm6 - pmaddubsw xm4, xm7 - phaddw xm4, xm4 - pmulhrsw xm4, xm8 - palignr xm3, xm4, xm0, 12 - mova xm0, xm4 - punpcklwd xm3, xm0 ; 67 78 - pmaddwd xm4, xm3, xm13 ; a3 b3 - paddd xm5, xm9 - paddd xm5, xm4 - psrad xm5, 10 - packssdw xm5, xm5 - packuswb xm5, xm5 - pextrw [dstq+dsq*0], xm5, 0 - pextrw [dstq+dsq*1], xm5, 1 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .hv_w2_loop - RET -.hv_w4: - mova m6, [subpel_h_shuf4] - vpbroadcastq m2, [srcq+ssq*0] - vpbroadcastq m4, [srcq+ssq*1] - vpbroadcastq m0, [srcq+ssq*2] - vpbroadcastq m5, [srcq+ss3q ] - lea srcq, [srcq+ssq*4] - vpbroadcastq m3, [srcq+ssq*0] - vpblendd m2, m2, m4, 0xcc ; 0 1 - vpbroadcastq m4, [srcq+ssq*1] - vpbroadcastq m1, [srcq+ssq*2] - add srcq, ss3q - vpblendd m0, m0, m5, 0xcc ; 2 3 - vpblendd m3, m3, m4, 0xcc ; 4 5 - pshufb m2, m6 - pshufb m0, m6 - pshufb m3, m6 - pshufb m1, m6 - pmaddubsw m2, m7 - pmaddubsw m0, m7 - pmaddubsw m3, m7 - pmaddubsw m1, m7 - phaddw m2, m0 - phaddw m3, m1 - pmulhrsw m2, m8 - pmulhrsw m3, m8 - palignr m4, m3, m2, 4 - punpcklwd m1, m2, m4 ; 01 12 - punpckhwd m2, m4 ; 23 34 - pshufd m0, m3, q2121 - punpcklwd m3, m0 ; 45 56 -.hv_w4_loop: - pmaddwd m5, m1, m10 ; a0 b0 - mova m1, m2 - pmaddwd m2, m11 ; a1 b1 - paddd m5, m2 - mova m2, m3 - pmaddwd m3, m12 ; a2 b2 - paddd m5, m3 - vpbroadcastq m4, [srcq+ssq*0] - vpbroadcastq m3, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - vpblendd m4, m4, m3, 0xcc ; 7 8 - pshufb m4, m6 - pmaddubsw m4, m7 - phaddw m4, m4 - pmulhrsw m4, m8 - palignr m3, m4, m0, 12 - mova m0, m4 - punpcklwd m3, m0 ; 67 78 - pmaddwd m4, m3, m13 ; a3 b3 - paddd m5, m9 - paddd m5, m4 - psrad m5, 10 - vextracti128 xm4, m5, 1 - packssdw xm5, xm4 - packuswb xm5, xm5 - pshuflw xm5, xm5, q3120 - movd [dstq+dsq*0], xm5 - pextrd [dstq+dsq*1], xm5, 1 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .hv_w4_loop - RET -.hv_w8: - shr mxd, 16 - sub srcq, 3 - vpbroadcastd m10, [r8+mxq*8+subpel_filters-put_avx2+0] - vpbroadcastd m11, [r8+mxq*8+subpel_filters-put_avx2+4] - movzx mxd, myb - shr myd, 16 - cmp hd, 6 - cmovs myd, mxd - vpbroadcastq m0, [r8+myq*8+subpel_filters-put_avx2] - lea ss3q, [ssq*3] - sub srcq, ss3q - punpcklbw m0, m0 - psraw m0, 8 ; sign-extend - pshufd m12, m0, q0000 - pshufd m13, m0, q1111 - pshufd m14, m0, q2222 - pshufd m15, m0, q3333 - lea r6d, [wq-8] - mov r4, dstq - mov r7, srcq - shl r6d, 5 - mov r6b, hb -.hv_w8_loop0: - vbroadcasti128 m7, [subpel_h_shufA] - vbroadcasti128 m8, [subpel_h_shufB] - vbroadcasti128 m9, [subpel_h_shufC] - movu xm4, [srcq+ssq*0] - movu xm5, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - movu xm6, [srcq+ssq*0] - vbroadcasti128 m0, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - vpblendd m4, m4, m0, 0xf0 ; 0 3 - vinserti128 m5, m5, [srcq+ssq*0], 1 ; 1 4 - vinserti128 m6, m6, [srcq+ssq*1], 1 ; 2 5 - lea srcq, [srcq+ssq*2] - vinserti128 m0, m0, [srcq+ssq*0], 1 ; 3 6 -%macro HV_H_W8 4-7 ; src/dst, tmp[1-3], shuf[1-3] - pshufb %3, %1, %6 - pshufb %4, %1, %7 - pshufb %1, %5 - pmaddubsw %2, %3, m10 - pmaddubsw %4, m11 - pmaddubsw %3, m11 - pmaddubsw %1, m10 - paddw %2, %4 - paddw %1, %3 - phaddw %1, %2 -%endmacro - HV_H_W8 m4, m1, m2, m3, m7, m8, m9 - HV_H_W8 m5, m1, m2, m3, m7, m8, m9 - HV_H_W8 m6, m1, m2, m3, m7, m8, m9 - HV_H_W8 m0, m1, m2, m3, m7, m8, m9 - vpbroadcastd m7, [pw_8192] - vpermq m4, m4, q3120 - vpermq m5, m5, q3120 - vpermq m6, m6, q3120 - pmulhrsw m0, m7 - pmulhrsw m4, m7 - pmulhrsw m5, m7 - pmulhrsw m6, m7 - vpermq m7, m0, q3120 - punpcklwd m1, m4, m5 ; 01 - punpckhwd m4, m5 ; 34 - punpcklwd m2, m5, m6 ; 12 - punpckhwd m5, m6 ; 45 - punpcklwd m3, m6, m7 ; 23 - punpckhwd m6, m7 ; 56 -.hv_w8_loop: - vextracti128 r6m, m0, 1 ; not enough registers - movu xm0, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - vinserti128 m0, m0, [srcq+ssq*0], 1 ; 7 8 - pmaddwd m8, m1, m12 ; a0 - pmaddwd m9, m2, m12 ; b0 - mova m1, m3 - mova m2, m4 - pmaddwd m3, m13 ; a1 - pmaddwd m4, m13 ; b1 - paddd m8, m3 - paddd m9, m4 - mova m3, m5 - mova m4, m6 - pmaddwd m5, m14 ; a2 - pmaddwd m6, m14 ; b2 - paddd m8, m5 - paddd m9, m6 - vbroadcasti128 m6, [subpel_h_shufB] - vbroadcasti128 m7, [subpel_h_shufC] - vbroadcasti128 m5, [subpel_h_shufA] - HV_H_W8 m0, m5, m6, m7, m5, m6, m7 - vpbroadcastd m5, [pw_8192] - vpbroadcastd m7, [pd_512] - vbroadcasti128 m6, r6m - pmulhrsw m0, m5 - paddd m8, m7 - paddd m9, m7 - vpermq m7, m0, q3120 ; 7 8 - shufpd m6, m6, m7, 0x04 ; 6 7 - punpcklwd m5, m6, m7 ; 67 - punpckhwd m6, m7 ; 78 - pmaddwd m7, m5, m15 ; a3 - paddd m8, m7 - pmaddwd m7, m6, m15 ; b3 - paddd m7, m9 - psrad m8, 10 - psrad m7, 10 - packssdw m8, m7 - vextracti128 xm7, m8, 1 - packuswb xm8, xm7 - pshufd xm7, xm8, q3120 - movq [dstq+dsq*0], xm7 - movhps [dstq+dsq*1], xm7 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .hv_w8_loop - movzx hd, r6b - add r4, 8 - add r7, 8 - mov dstq, r4 - mov srcq, r7 - sub r6d, 1<<8 - jg .hv_w8_loop0 - RET - -%macro PREP_8TAP_H 0 - %if cpuflag(avx512) - vpermb m10, m5, m0 - vpermb m11, m5, m1 - vpermb m12, m6, m0 - vpermb m13, m6, m1 - vpermb m14, m7, m0 - vpermb m15, m7, m1 - mova m0, m4 - mova m2, m4 - mova m1, m4 - mova m3, m4 - vpdpbusd m0, m10, m8 - vpdpbusd m2, m12, m8 - vpdpbusd m1, m11, m8 - vpdpbusd m3, m13, m8 - vpdpbusd m0, m12, m9 - vpdpbusd m2, m14, m9 - vpdpbusd m1, m13, m9 - vpdpbusd m3, m15, m9 - packssdw m0, m2 - packssdw m1, m3 - psraw m0, 2 - psraw m1, 2 - mova [tmpq+ 0], m0 - mova [tmpq+64], m1 - %else - pshufb m1, m0, m5 - pshufb m2, m0, m6 - pshufb m3, m0, m7 - pmaddubsw m1, m8 - pmaddubsw m0, m2, m8 - pmaddubsw m2, m9 - pmaddubsw m3, m9 - paddw m1, m2 - paddw m0, m3 - phaddw m0, m1, m0 - pmulhrsw m0, m4 - %endif -%endmacro - -%macro PREP_8TAP_V_W4 5 ; round, weights - movd xm0, [srcq+strideq*0] - vpbroadcastd ym1, [srcq+strideq*2] - vpbroadcastd xm2, [srcq+strideq*1] - vpbroadcastd ym3, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - vpblendd ym1, ym1, ym0, 0x01 ; 0 2 2 _ 2 _ _ _ - vpblendd ym3, ym3, ym2, 0x03 ; 1 1 3 3 3 3 _ _ - vpbroadcastd ym0, [srcq+strideq*0] - vpbroadcastd ym2, [srcq+strideq*1] - vpblendd ym1, ym1, ym0, 0x68 ; 0 2 2 4 2 4 4 _ - vpbroadcastd ym0, [srcq+strideq*2] - vbroadcasti128 ym5, [deint_shuf4] - vpblendd ym3, ym3, ym2, 0xc0 ; 1 1 3 3 3 3 5 5 - vpblendd ym2, ym3, ym1, 0x55 ; 0 1 2 3 2 3 4 5 - vpblendd ym3, ym3, ym1, 0xaa ; 1 2 3 4 3 4 5 _ - punpcklbw ym1, ym2, ym3 ; 01 12 23 34 - vpblendd ym3, ym3, ym0, 0x80 ; 1 2 3 4 3 4 5 6 - punpckhbw ym2, ym3 ; 23 34 45 56 -.v_w4_loop: - pinsrd xm0, [srcq+stride3q ], 1 - lea srcq, [srcq+strideq*4] - vpbroadcastd ym3, [srcq+strideq*0] - vpbroadcastd ym4, [srcq+strideq*1] - vpblendd ym3, ym3, ym4, 0x20 ; _ _ 8 _ 8 9 _ _ - vpblendd ym3, ym3, ym0, 0x03 ; 6 7 8 _ 8 9 _ _ - vpbroadcastd ym0, [srcq+strideq*2] - vpblendd ym3, ym3, ym0, 0x40 ; 6 7 8 _ 8 9 a _ - pshufb ym3, ym5 ; 67 78 89 9a - pmaddubsw ym4, ym1, ym%2 - vperm2i128 ym1, ym2, ym3, 0x21 ; 45 56 67 78 - pmaddubsw ym2, ym%3 - paddw ym4, ym2 - mova ym2, ym3 - pmaddubsw ym3, ym%5 - paddw ym3, ym4 - pmaddubsw ym4, ym1, ym%4 - paddw ym3, ym4 - pmulhrsw ym3, ym%1 - mova [tmpq], ym3 -%endmacro - -%macro PREP_8TAP_FN 3 ; type, type_h, type_v -cglobal prep_8tap_%1 - mov t0d, FILTER_%2 - mov t1d, FILTER_%3 -%ifnidn %1, sharp_smooth ; skip the jump in the last filter - jmp mangle(private_prefix %+ _prep_8tap %+ SUFFIX) -%endif -%endmacro - -%macro PREP_8TAP 0 - %if WIN64 - DECLARE_REG_TMP 6, 4 - %else - DECLARE_REG_TMP 6, 7 - %endif -PREP_8TAP_FN regular, REGULAR, REGULAR -PREP_8TAP_FN regular_sharp, REGULAR, SHARP -PREP_8TAP_FN regular_smooth, REGULAR, SMOOTH -PREP_8TAP_FN smooth_regular, SMOOTH, REGULAR -PREP_8TAP_FN smooth, SMOOTH, SMOOTH -PREP_8TAP_FN smooth_sharp, SMOOTH, SHARP -PREP_8TAP_FN sharp_regular, SHARP, REGULAR -PREP_8TAP_FN sharp, SHARP, SHARP -PREP_8TAP_FN sharp_smooth, SHARP, SMOOTH - -cglobal prep_8tap, 3, 8, 0, tmp, src, stride, w, h, mx, my, stride3 - imul mxd, mxm, 0x010101 - add mxd, t0d ; 8tap_h, mx, 4tap_h - imul myd, mym, 0x010101 - add myd, t1d ; 8tap_v, my, 4tap_v - lea r7, [prep%+SUFFIX] - movsxd wq, wm - movifnidn hd, hm - test mxd, 0xf00 - jnz .h - test myd, 0xf00 - jnz .v - tzcnt wd, wd - movzx wd, word [r7+wq*2+table_offset(prep,)] - add wq, r7 - lea r6, [strideq*3] -%if WIN64 - pop r7 -%endif - jmp wq -.h: - test myd, 0xf00 - jnz .hv -%if cpuflag(avx512) - vpbroadcastd m4, [pd_2] -%else - vpbroadcastd m4, [pw_8192] - vbroadcasti128 m5, [subpel_h_shufA] -%endif - WIN64_SPILL_XMM 10 - cmp wd, 4 - je .h_w4 - tzcnt wd, wd -%if notcpuflag(avx512) - vbroadcasti128 m6, [subpel_h_shufB] - vbroadcasti128 m7, [subpel_h_shufC] -%endif - shr mxd, 16 - sub srcq, 3 - movzx wd, word [r7+wq*2+table_offset(prep, _8tap_h)] - vpbroadcastd m8, [r7+mxq*8+subpel_filters-prep%+SUFFIX+0] - vpbroadcastd m9, [r7+mxq*8+subpel_filters-prep%+SUFFIX+4] - add wq, r7 - jmp wq -.h_w4: -%if cpuflag(avx512) - mov r3d, 0x4 - kmovb k1, r3d - vbroadcasti128 ym5, [subpel_h_shufA] -%endif - movzx mxd, mxb - dec srcq - vpbroadcastd ym6, [r7+mxq*8+subpel_filters-prep%+SUFFIX+2] - lea stride3q, [strideq*3] -.h_w4_loop: -%if cpuflag(avx512icl) - mova ym0, ym4 - mova ym1, ym4 - movq xm2, [srcq+strideq*0] - movq xm3, [srcq+strideq*1] - vpbroadcastq ym2{k1}, [srcq+strideq*2] - vpbroadcastq ym3{k1}, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - pshufb ym2, ym5 - pshufb ym3, ym5 - vpdpbusd ym0, ym2, ym6 - vpdpbusd ym1, ym3, ym6 - packssdw ym0, ym1 - psraw ym0, 2 -%else - movq xm0, [srcq+strideq*0] - vpbroadcastq m2, [srcq+strideq*2] - movq xm1, [srcq+strideq*1] - vpblendd m0, m0, m2, 0xf0 - vpbroadcastq m2, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - vpblendd m1, m1, m2, 0xf0 - pshufb m0, m5 - pshufb m1, m5 - pmaddubsw m0, m6 - pmaddubsw m1, m6 - phaddw m0, m1 - pmulhrsw m0, m4 -%endif - mova [tmpq], ym0 - add tmpq, 32 - sub hd, 4 - jg .h_w4_loop - RET -.h_w8: -%if cpuflag(avx512) - vbroadcasti128 m5, [subpel_h_shufA] - vbroadcasti128 m6, [subpel_h_shufB] - vbroadcasti128 m7, [subpel_h_shufC] - lea stride3q, [strideq*3] -%endif -.h_w8_loop: - movu xm0, [srcq+strideq*0] - vinserti128 ym0, [srcq+strideq*1], 1 -%if cpuflag(avx512) - vinserti128 m0, [srcq+strideq*2], 2 - vinserti128 m0, [srcq+stride3q ], 3 -%endif - lea srcq, [srcq+strideq*(mmsize/(8*2))] -%if cpuflag(avx512icl) - mova m10, m4 - mova m11, m4 - pshufb m1, m0, m5 - pshufb m2, m0, m6 - pshufb m3, m0, m7 - vpdpbusd m10, m1, m8 - vpdpbusd m11, m2, m8 - vpdpbusd m10, m2, m9 - vpdpbusd m11, m3, m9 - packssdw m10, m11 - psraw m0, m10, 2 -%else - PREP_8TAP_H -%endif - mova [tmpq], m0 - add tmpq, mmsize - sub hd, mmsize/(8*2) - jg .h_w8_loop - RET -.h_w16: -%if cpuflag(avx512icl) - mova m5, [spel_h_perm16a] - mova m6, [spel_h_perm16b] - mova m7, [spel_h_perm16c] - lea stride3q, [strideq*3] -.h_w16_loop: - movu ym0, [srcq+strideq*0] - movu ym1, [srcq+strideq*2] - vinserti32x8 m0, [srcq+strideq*1], 1 - vinserti32x8 m1, [srcq+stride3q ], 1 - lea srcq, [srcq+strideq*4] - PREP_8TAP_H -%else -.h_w16_loop: - movu xm0, [srcq+strideq*0+8*0] - vinserti128 m0, [srcq+strideq*0+8*1], 1 - PREP_8TAP_H - mova [tmpq+32*0], m0 - movu xm0, [srcq+strideq*1+8*0] - vinserti128 m0, m0, [srcq+strideq*1+8*1], 1 - lea srcq, [srcq+strideq*2] - PREP_8TAP_H - mova [tmpq+32*1], m0 -%endif - add tmpq, mmsize*2 - sub hd, mmsize*2/(16*2) - jg .h_w16_loop - RET -.h_w32: -%if cpuflag(avx512icl) - mova m5, [spel_h_perm32a] - mova m6, [spel_h_perm32b] - mova m7, [spel_h_perm32c] -.h_w32_loop: - movu m0, [srcq+strideq*0] - movu m1, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - PREP_8TAP_H - add tmpq, 64*2 - sub hd, 2 - jg .h_w32_loop - RET -%else - xor r6d, r6d - jmp .h_start -%endif -.h_w64: -%if cpuflag(avx512) - xor r6d, r6d -%else - mov r6, -32*1 -%endif - jmp .h_start -.h_w128: -%if cpuflag(avx512) - mov r6, -64*1 -%else - mov r6, -32*3 -%endif -.h_start: -%if cpuflag(avx512) - mova m5, [spel_h_perm32a] - mova m6, [spel_h_perm32b] - mova m7, [spel_h_perm32c] -%endif - sub srcq, r6 - mov r5, r6 -.h_loop: -%if cpuflag(avx512icl) - movu m0, [srcq+r6+32*0] - movu m1, [srcq+r6+32*1] - PREP_8TAP_H -%else - movu xm0, [srcq+r6+8*0] - vinserti128 ym0, [srcq+r6+8*1], 1 - PREP_8TAP_H - mova [tmpq+32*0], m0 - movu xm0, [srcq+r6+8*2] - vinserti128 ym0, [srcq+r6+8*3], 1 - PREP_8TAP_H - mova [tmpq+32*1], m0 -%endif - add tmpq, mmsize*2 - add r6, mmsize - jle .h_loop - add srcq, strideq - mov r6, r5 - dec hd - jg .h_loop - RET -.v: - %assign stack_offset stack_offset - stack_size_padded - WIN64_SPILL_XMM 16 - movzx mxd, myb ; Select 4-tap/8-tap filter multipliers. - shr myd, 16 ; Note that the code is 8-tap only, having - cmp hd, 4 ; a separate 4-tap code path for (4|8|16)x4 - cmove myd, mxd ; had a negligible effect on performance. - ; TODO: Would a 6-tap code path be worth it? -%if cpuflag(avx512) - tzcnt wd, wd - movzx wd, word [r7+wq*2+table_offset(prep, _8tap_v)] - add wq, r7 -%endif - lea myq, [r7+myq*8+subpel_filters-prep%+SUFFIX] - lea stride3q, [strideq*3] - sub srcq, stride3q - vpbroadcastd m7, [pw_8192] - vpbroadcastw m8, [myq+0] - vpbroadcastw m9, [myq+2] - vpbroadcastw m10, [myq+4] - vpbroadcastw m11, [myq+6] -%if cpuflag(avx512) - jmp wq -%else - cmp wd, 8 - jg .v_w16 - je .v_w8 -%endif -.v_w4: -%if cpuflag(avx512) - AVX512_MM_PERMUTATION - PREP_8TAP_V_W4 23, 24, 25, 26, 27 - AVX512_MM_PERMUTATION -%else - PREP_8TAP_V_W4 7, 8, 9, 10, 11 -%endif - add tmpq, 32 - sub hd, 4 - jg .v_w4_loop -%if cpuflag(avx512) - vzeroupper -%endif - RET -.v_w8: -%if cpuflag(avx512) - mov r3d, 0xf044 - kmovw k1, r3d - kshiftrw k2, k1, 8 - movq xm0, [srcq+strideq*0] - vpbroadcastq ym1, [srcq+strideq*1] - vpbroadcastq m2, [srcq+strideq*2] - vpbroadcastq m3, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - vpbroadcastq m4, [srcq+strideq*0] - vpbroadcastq m5, [srcq+strideq*1] - vpbroadcastq m6, [srcq+strideq*2] - vmovdqa64 ym0{k1}, ym1 - vmovdqa64 ym1{k1}, ym2 - vmovdqa64 m2{k1}, m3 - vmovdqa64 m3{k1}, m4 - vmovdqa64 m4{k1}, m5 - vmovdqa64 m5{k1}, m6 - punpcklbw ym0, ym1 ; 01 12 __ __ - punpcklbw m2, m3 ; 23 34 23 34 - punpcklbw m4, m5 ; 45 56 45 56 - vmovdqa64 m0{k2}, m2 ; 01 12 23 34 - vmovdqa64 m2{k2}, m4 ; 23 34 45 56 -.v_w8_loop: - vpbroadcastq m1, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - vpbroadcastq m3, [srcq+strideq*0] - vpbroadcastq m5, [srcq+strideq*1] - pmaddubsw m14, m0, m8 - pmaddubsw m15, m2, m9 - vpblendmq m0{k1}, m6, m1 - vpblendmq m2{k1}, m1, m3 - vpbroadcastq m6, [srcq+strideq*2] - paddw m14, m15 - punpcklbw m2, m0, m2 ; 67 78 67 78 - vpblendmq m12{k1}, m3, m5 - vpblendmq m13{k1}, m5, m6 - vpblendmq m0{k2}, m4, m2 ; 45 56 67 78 - punpcklbw m4, m12, m13 ; 89 9a 89 9a - vmovdqa64 m2{k2}, m4 ; 67 78 89 9a - pmaddubsw m12, m0, m10 - pmaddubsw m13, m2, m11 - paddw m14, m12 - paddw m14, m13 - pmulhrsw m14, m7 - mova [tmpq], m14 -%else - movq xm1, [srcq+strideq*0] - vpbroadcastq m4, [srcq+strideq*1] - vpbroadcastq m2, [srcq+strideq*2] - vpbroadcastq m5, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - vpbroadcastq m3, [srcq+strideq*0] - vpbroadcastq m6, [srcq+strideq*1] - vpbroadcastq m0, [srcq+strideq*2] - vpblendd m1, m1, m4, 0x30 - vpblendd m4, m4, m2, 0x30 - punpcklbw m1, m4 ; 01 12 - vpblendd m2, m2, m5, 0x30 - vpblendd m5, m5, m3, 0x30 - punpcklbw m2, m5 ; 23 34 - vpblendd m3, m3, m6, 0x30 - vpblendd m6, m6, m0, 0x30 - punpcklbw m3, m6 ; 45 56 -.v_w8_loop: - vpbroadcastq m4, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - pmaddubsw m5, m2, m9 ; a1 - pmaddubsw m6, m2, m8 ; b0 - vpblendd m2, m0, m4, 0x30 - vpbroadcastq m0, [srcq+strideq*0] - vpblendd m4, m4, m0, 0x30 - punpcklbw m2, m4 ; 67 78 - pmaddubsw m1, m8 ; a0 - pmaddubsw m4, m3, m9 ; b1 - paddw m5, m1 - mova m1, m3 - pmaddubsw m3, m10 ; a2 - paddw m6, m4 - paddw m5, m3 - vpbroadcastq m4, [srcq+strideq*1] - vpblendd m3, m0, m4, 0x30 - vpbroadcastq m0, [srcq+strideq*2] - vpblendd m4, m4, m0, 0x30 - punpcklbw m3, m4 ; 89 9a - pmaddubsw m4, m2, m11 ; a3 - paddw m5, m4 - pmaddubsw m4, m2, m10 ; b2 - paddw m6, m4 - pmaddubsw m4, m3, m11 ; b3 - paddw m6, m4 - pmulhrsw m5, m7 - pmulhrsw m6, m7 - mova [tmpq+32*0], m5 - mova [tmpq+32*1], m6 -%endif - add tmpq, 32*2 - sub hd, 4 - jg .v_w8_loop - RET -.v_w16: -%if cpuflag(avx512) - mov r3d, 0xf0 - kmovb k1, r3d - vbroadcasti128 m0, [srcq+strideq*0] - vbroadcasti128 m1, [srcq+strideq*1] - vbroadcasti128 m2, [srcq+strideq*2] - vbroadcasti128 m3, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - vbroadcasti128 m4, [srcq+strideq*0] - vbroadcasti128 m5, [srcq+strideq*1] - vbroadcasti128 m6, [srcq+strideq*2] - vmovdqa64 m0{k1}, m1 - vmovdqa64 m1{k1}, m2 - vmovdqa64 m2{k1}, m3 - vmovdqa64 m3{k1}, m4 - vmovdqa64 m4{k1}, m5 - vmovdqa64 m5{k1}, m6 - shufpd m0, m2, 0xcc ; 0a_2a 0b_2b 1a_3a 1b_3b - shufpd m1, m3, 0xcc ; 1a_3a 1b_3b 2a_4a 2b_4b - shufpd m4, m4, 0x44 ; 4a_-- 4b_-- 5a_-- 5b_-- - shufpd m5, m5, 0x44 ; 5a_-- 5b_-- 6a_-- 6b_-- - punpckhbw m2, m0, m1 ; 23a 23b 34a 34b - punpcklbw m0, m1 ; 01a 01b 12a 12b - punpcklbw m4, m5 ; 45a 45b 56a 56b -.v_w16_loop: - vbroadcasti128 m3, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - vbroadcasti128 m5, [srcq+strideq*0] - vpblendmq m1{k1}, m6, m3 - vmovdqa64 m3{k1}, m5 - pmaddubsw m12, m0, m8 - pmaddubsw m13, m2, m8 - pmaddubsw m14, m2, m9 - pmaddubsw m15, m4, m9 - pmaddubsw m0, m4, m10 - vbroadcasti128 m2, [srcq+strideq*1] - vbroadcasti128 m6, [srcq+strideq*2] - paddw m12, m14 - paddw m13, m15 - paddw m12, m0 - vmovdqa64 m5{k1}, m2 - vmovdqa64 m2{k1}, m6 - mova m0, m4 - shufpd m1, m5, 0xcc ; 6a_8a 6b_8b 7a_9a 7b_9b - shufpd m3, m2, 0xcc ; 7a_9a 7b_9b 8a_Aa 8b_Ab - punpcklbw m2, m1, m3 ; 67a 67b 78a 78b - punpckhbw m4, m1, m3 ; 89a 89b 9Aa 9Ab - pmaddubsw m14, m2, m10 - pmaddubsw m15, m2, m11 - paddw m13, m14 - paddw m12, m15 - pmaddubsw m14, m4, m11 - paddw m13, m14 - pmulhrsw m12, m7 - pmulhrsw m13, m7 - mova [tmpq+ 0], m12 - mova [tmpq+64], m13 - add tmpq, 64*2 - sub hd, 4 - jg .v_w16_loop -%else - lea r6d, [wq-16] - mov r5, tmpq - mov r7, srcq - shl r6d, 4 - mov r6b, hb -.v_w16_loop0: - vbroadcasti128 m4, [srcq+strideq*0] - vbroadcasti128 m5, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - vbroadcasti128 m0, [srcq+strideq*1] - vbroadcasti128 m6, [srcq+strideq*0] - lea srcq, [srcq+strideq*2] - vbroadcasti128 m1, [srcq+strideq*0] - vbroadcasti128 m2, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - vbroadcasti128 m3, [srcq+strideq*0] - shufpd m4, m4, m0, 0x0c - shufpd m5, m5, m1, 0x0c - punpcklbw m1, m4, m5 ; 01 - punpckhbw m4, m5 ; 34 - shufpd m6, m6, m2, 0x0c - punpcklbw m2, m5, m6 ; 12 - punpckhbw m5, m6 ; 45 - shufpd m0, m0, m3, 0x0c - punpcklbw m3, m6, m0 ; 23 - punpckhbw m6, m0 ; 56 -.v_w16_loop: - vbroadcasti128 m12, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - vbroadcasti128 m13, [srcq+strideq*0] - pmaddubsw m14, m1, m8 ; a0 - pmaddubsw m15, m2, m8 ; b0 - mova m1, m3 - mova m2, m4 - pmaddubsw m3, m9 ; a1 - pmaddubsw m4, m9 ; b1 - paddw m14, m3 - paddw m15, m4 - mova m3, m5 - mova m4, m6 - pmaddubsw m5, m10 ; a2 - pmaddubsw m6, m10 ; b2 - paddw m14, m5 - paddw m15, m6 - shufpd m6, m0, m12, 0x0d - shufpd m0, m12, m13, 0x0c - punpcklbw m5, m6, m0 ; 67 - punpckhbw m6, m0 ; 78 - pmaddubsw m12, m5, m11 ; a3 - pmaddubsw m13, m6, m11 ; b3 - paddw m14, m12 - paddw m15, m13 - pmulhrsw m14, m7 - pmulhrsw m15, m7 - mova [tmpq+wq*0], m14 - mova [tmpq+wq*2], m15 - lea tmpq, [tmpq+wq*4] - sub hd, 2 - jg .v_w16_loop - movzx hd, r6b - add r5, 32 - add r7, 16 - mov tmpq, r5 - mov srcq, r7 - sub r6d, 1<<8 - jg .v_w16_loop0 -%endif - RET -%if cpuflag(avx512) -.v_w32: - mova m18, [bilin_v_perm64] - movu ym0, [srcq+strideq*0] - movu ym1, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - movu ym2, [srcq+strideq*0] - movu ym3, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - movu ym4, [srcq+strideq*0] - movu ym5, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - movu ym6, [srcq+strideq*0] - vpermq m0, m18, m0 - vpermq m1, m18, m1 - vpermq m2, m18, m2 - vpermq m3, m18, m3 - vpermq m4, m18, m4 - vpermq m5, m18, m5 - vpermq m6, m18, m6 - punpcklbw m0, m1 - punpcklbw m1, m2 - punpcklbw m2, m3 - punpcklbw m3, m4 - punpcklbw m4, m5 - punpcklbw m5, m6 -.v_w32_loop: - movu ym12, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - movu ym13, [srcq+strideq*0] - pmaddubsw m14, m0, m8 - pmaddubsw m16, m2, m9 - pmaddubsw m15, m1, m8 - pmaddubsw m17, m3, m9 - mova m0, m2 - mova m1, m3 - vpermq m12, m18, m12 - vpermq m13, m18, m13 - paddw m14, m16 - paddw m15, m17 - pmaddubsw m16, m4, m10 - pmaddubsw m17, m5, m10 - punpcklbw m6, m12 - punpcklbw m12, m13 - mova m2, m4 - mova m3, m5 - paddw m14, m16 - paddw m15, m17 - pmaddubsw m16, m6, m11 - pmaddubsw m17, m12, m11 - mova m4, m6 - mova m5, m12 - paddw m14, m16 - paddw m15, m17 - pmulhrsw m14, m7 - pmulhrsw m15, m7 - mova m6, m13 - mova [tmpq+ 0], m14 - mova [tmpq+64], m15 - add tmpq, 64*2 - sub hd, 2 - jg .v_w32_loop - vzeroupper - RET -.v_w64: - mov r6d, hd - mov wd, 64 - jmp .v_start -.v_w128: - lea r6d, [(1<<8)+hq] - mov wd, 128 -.v_start: - WIN64_SPILL_XMM 27 - mova m26, [bilin_v_perm64] - mov r5, tmpq - mov r7, srcq -.v_loop0: - vpermq m0, m26, [srcq+strideq*0] - vpermq m1, m26, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - vpermq m2, m26, [srcq+strideq*0] - vpermq m3, m26, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - vpermq m4, m26, [srcq+strideq*0] - vpermq m5, m26, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - vpermq m6, m26, [srcq+strideq*0] - punpckhbw m12, m0, m1 - punpcklbw m0, m1 - punpckhbw m13, m1, m2 - punpcklbw m1, m2 - punpckhbw m14, m2, m3 - punpcklbw m2, m3 - punpckhbw m15, m3, m4 - punpcklbw m3, m4 - punpckhbw m16, m4, m5 - punpcklbw m4, m5 - punpckhbw m17, m5, m6 - punpcklbw m5, m6 -.v_loop: - vpermq m18, m26, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - vpermq m19, m26, [srcq+strideq*0] - pmaddubsw m20, m0, m8 - pmaddubsw m21, m12, m8 - pmaddubsw m22, m1, m8 - pmaddubsw m23, m13, m8 - mova m0, m2 - mova m12, m14 - mova m1, m3 - mova m13, m15 - pmaddubsw m2, m9 - pmaddubsw m14, m9 - pmaddubsw m3, m9 - pmaddubsw m15, m9 - punpckhbw m24, m6, m18 - punpcklbw m6, m18 - paddw m20, m2 - paddw m21, m14 - paddw m22, m3 - paddw m23, m15 - mova m2, m4 - mova m14, m16 - mova m3, m5 - mova m15, m17 - pmaddubsw m4, m10 - pmaddubsw m16, m10 - pmaddubsw m5, m10 - pmaddubsw m17, m10 - punpckhbw m25, m18, m19 - punpcklbw m18, m19 - paddw m20, m4 - paddw m21, m16 - paddw m22, m5 - paddw m23, m17 - mova m4, m6 - mova m16, m24 - mova m5, m18 - mova m17, m25 - pmaddubsw m6, m11 - pmaddubsw m24, m11 - pmaddubsw m18, m11 - pmaddubsw m25, m11 - paddw m20, m6 - paddw m21, m24 - paddw m22, m18 - paddw m23, m25 - pmulhrsw m20, m7 - pmulhrsw m21, m7 - pmulhrsw m22, m7 - pmulhrsw m23, m7 - mova m6, m19 - mova [tmpq+wq*0+ 0], m20 - mova [tmpq+wq*0+64], m21 - mova [tmpq+wq*2+ 0], m22 - mova [tmpq+wq*2+64], m23 - lea tmpq, [tmpq+wq*4] - sub hd, 2 - jg .v_loop - movzx hd, r6b - add r5, 64*2 - add r7, 64 - mov tmpq, r5 - mov srcq, r7 - sub r6d, 1<<8 - jg .v_loop0 -%endif - RET -.hv: - %assign stack_offset stack_offset - stack_size_padded - %assign stack_size_padded 0 - WIN64_SPILL_XMM 16 - cmp wd, 4 - je .hv_w4 - shr mxd, 16 - sub srcq, 3 - vpbroadcastd m10, [r7+mxq*8+subpel_filters-prep%+SUFFIX+0] - vpbroadcastd m11, [r7+mxq*8+subpel_filters-prep%+SUFFIX+4] - movzx mxd, myb - shr myd, 16 - cmp hd, 4 - cmove myd, mxd -%if cpuflag(avx512) - tzcnt wd, wd - vpbroadcastd m8, [pd_2] - movzx wd, word [r7+wq*2+table_offset(prep, _8tap_hv)] - vpbroadcastd m9, [pd_32] - add wq, r7 -%endif - vpbroadcastq m0, [r7+myq*8+subpel_filters-prep%+SUFFIX] - lea stride3q, [strideq*3] - sub srcq, stride3q - punpcklbw m0, m0 - psraw m0, 8 ; sign-extend - pshufd m12, m0, q0000 - pshufd m13, m0, q1111 - pshufd m14, m0, q2222 - pshufd m15, m0, q3333 -%if cpuflag(avx512) - jmp wq -%else - jmp .hv_w8 -%endif -.hv_w4: - movzx mxd, mxb - dec srcq - vpbroadcastd m8, [r7+mxq*8+subpel_filters-prep%+SUFFIX+2] - movzx mxd, myb - shr myd, 16 - cmp hd, 4 - cmove myd, mxd - vpbroadcastq m0, [r7+myq*8+subpel_filters-prep%+SUFFIX] - lea stride3q, [strideq*3] - sub srcq, stride3q -%if cpuflag(avx512) - mov r3d, 0x04 - kmovb k1, r3d - kshiftlb k2, k1, 2 - kshiftlb k3, k1, 4 - vpbroadcastd m10, [pd_2] - vbroadcasti128 m16, [subpel_h_shufA] -%else - mova m7, [subpel_h_shuf4] - pmovzxbd m9, [deint_shuf4] - vpbroadcastd m10, [pw_8192] -%endif - punpcklbw m0, m0 - psraw m0, 8 ; sign-extend - vpbroadcastd m11, [pd_32] - pshufd m12, m0, q0000 - pshufd m13, m0, q1111 - pshufd m14, m0, q2222 - pshufd m15, m0, q3333 -%if cpuflag(avx512icl) - movq xm3, [srcq+strideq*0] - vpbroadcastq ym2, [srcq+strideq*1] - vpbroadcastq ym3{k1}, [srcq+strideq*2] - vpbroadcastq m2{k2}, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - vpbroadcastq m3{k2}, [srcq+strideq*0] - vpbroadcastq m2{k3}, [srcq+strideq*1] - vpbroadcastq m3{k3}, [srcq+strideq*2] - mova m17, [spel_hv_perm4a] - movu m18, [spel_hv_perm4b] - mova m0, m10 - mova m1, m10 - pshufb m2, m16 - pshufb m3, m16 - vpdpbusd m0, m2, m8 - vpdpbusd m1, m3, m8 - packssdw m0, m1 ; _ 0 1 2 3 4 5 6 - psraw m0, 2 - vpermb m1, m17, m0 ; 01 12 23 34 - vpermb m2, m18, m0 ; 23 34 45 56 -.hv_w4_loop: - movq xm3, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - movq xm4, [srcq+strideq*0] - vpbroadcastq ym3{k1}, [srcq+strideq*1] - vpbroadcastq ym4{k1}, [srcq+strideq*2] - mova ym5, ym10 - mova ym6, ym10 - pshufb ym3, ym16 - pshufb ym4, ym16 - vpdpbusd ym5, ym3, ym8 - vpdpbusd ym6, ym4, ym8 - mova m7, m11 - packssdw ym5, ym6 ; 7 8 9 a _ _ _ _ - psraw ym5, 2 - valignq m0, m5, m0, 4 ; _ 4 5 6 7 8 9 a - vpdpwssd m7, m1, m12 - vpdpwssd m7, m2, m13 - vpermb m1, m17, m0 ; 45 56 67 78 - vpermb m2, m18, m0 ; 67 78 89 9a - vpdpwssd m7, m1, m14 - vpdpwssd m7, m2, m15 - psrad m7, 6 - vpmovdw [tmpq], m7 -%else - vpbroadcastq m2, [srcq+strideq*0] - vpbroadcastq m4, [srcq+strideq*1] - vpbroadcastq m0, [srcq+strideq*2] - vpbroadcastq m5, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - vpbroadcastq m3, [srcq+strideq*0] - vpbroadcastq m6, [srcq+strideq*1] - vpbroadcastq m1, [srcq+strideq*2] - vpblendd m2, m2, m4, 0xcc ; 0 1 - vpblendd m0, m0, m5, 0xcc ; 2 3 - vpblendd m3, m3, m6, 0xcc ; 4 5 - pshufb m2, m7 ; 00 01 10 11 02 03 12 13 - pshufb m0, m7 ; 20 21 30 31 22 23 32 33 - pshufb m3, m7 ; 40 41 50 51 42 43 52 53 - pshufb m1, m7 ; 60 61 60 61 62 63 62 63 - pmaddubsw m2, m8 - pmaddubsw m0, m8 - pmaddubsw m3, m8 - pmaddubsw m1, m8 - phaddw m2, m0 ; 0a 1a 2a 3a 0b 1b 2b 3b - phaddw m3, m1 ; 4a 5a 6a __ 4b 5b 6b __ - pmulhrsw m2, m10 - pmulhrsw m3, m10 - palignr m4, m3, m2, 4 ; 1a 2a 3a 4a 1b 2b 3b 4b - punpcklwd m1, m2, m4 ; 01 12 - punpckhwd m2, m4 ; 23 34 - pshufd m0, m3, q2121 - punpcklwd m3, m0 ; 45 56 -.hv_w4_loop: - pmaddwd m5, m1, m12 ; a0 b0 - pmaddwd m6, m2, m12 ; c0 d0 - pmaddwd m2, m13 ; a1 b1 - pmaddwd m4, m3, m13 ; c1 d1 - mova m1, m3 - pmaddwd m3, m14 ; a2 b2 - paddd m5, m2 - vpbroadcastq m2, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - paddd m6, m4 - paddd m5, m3 - vpbroadcastq m4, [srcq+strideq*0] - vpbroadcastq m3, [srcq+strideq*1] - vpblendd m2, m2, m4, 0xcc - vpbroadcastq m4, [srcq+strideq*2] - vpblendd m3, m3, m4, 0xcc - pshufb m2, m7 - pshufb m3, m7 - pmaddubsw m2, m8 - pmaddubsw m3, m8 - phaddw m2, m3 - pmulhrsw m2, m10 - palignr m3, m2, m0, 12 - mova m0, m2 - punpcklwd m2, m3, m0 ; 67 78 - punpckhwd m3, m0 ; 89 9a - pmaddwd m4, m2, m14 ; c2 d2 - paddd m6, m11 - paddd m5, m11 - paddd m6, m4 - pmaddwd m4, m2, m15 ; a3 b3 - paddd m5, m4 - pmaddwd m4, m3, m15 ; c3 d3 - paddd m6, m4 - psrad m5, 6 - psrad m6, 6 - packssdw m5, m6 - vpermd m5, m9, m5 - mova [tmpq], m5 -%endif - add tmpq, 32 - sub hd, 4 - jg .hv_w4_loop -%if cpuflag(avx512) - vzeroupper -%endif - RET -.hv_w8: -%if cpuflag(avx512icl) - WIN64_SPILL_XMM 24 - vbroadcasti128 m16, [subpel_h_shufA] - vbroadcasti128 m17, [subpel_h_shufB] - vbroadcasti128 m18, [subpel_h_shufC] - vinserti128 ym0, [srcq+strideq*0], 1 - vinserti128 m0, [srcq+strideq*1], 2 - vinserti128 m0, [srcq+strideq*2], 3 - movu xm1, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - vinserti128 ym1, [srcq+strideq*0], 1 - vinserti128 m1, [srcq+strideq*1], 2 - vinserti128 m1, [srcq+strideq*2], 3 - mova m2, m8 - mova m4, m8 - mova m3, m8 - mova m5, m8 - pshufb m20, m0, m16 - pshufb m21, m0, m17 - pshufb m22, m0, m18 - pshufb m23, m1, m16 - pshufb m6, m1, m17 - pshufb m7, m1, m18 - vpdpbusd m2, m20, m10 - vpdpbusd m4, m21, m10 - vpdpbusd m2, m21, m11 - vpdpbusd m4, m22, m11 - vpdpbusd m3, m23, m10 - vpdpbusd m5, m6, m10 - vpdpbusd m3, m6, m11 - vpdpbusd m5, m7, m11 - packssdw m2, m4 - packssdw m3, m5 - psraw m2, 2 ; _ 0 1 2 - psraw m3, 2 ; 3 4 5 6 - valignq m0, m3, m2, 2 ; 0 1 2 3 - valignq m1, m3, m2, 4 ; 1 2 3 4 - valignq m2, m3, m2, 6 ; 2 3 4 5 - punpcklwd m4, m0, m1 ; 01a 12a 23a 34a - punpckhwd m5, m0, m1 ; 01b 12b 23b 34b - punpcklwd m6, m2, m3 ; 23a 34a 45a 56a - punpckhwd m7, m2, m3 ; 23b 34b 45b 56b -.hv_w8_loop: - movu xm19, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - vinserti128 ym19, [srcq+strideq*0], 1 - vinserti128 m19, [srcq+strideq*1], 2 - vinserti128 m19, [srcq+strideq*2], 3 - mova m20, m9 - mova m21, m9 - mova m22, m8 - mova m23, m8 - vpdpwssd m20, m4, m12 - vpdpwssd m21, m5, m12 - vpdpwssd m20, m6, m13 - vpdpwssd m21, m7, m13 - pshufb m0, m19, m16 - pshufb m1, m19, m17 - pshufb m2, m19, m18 - vpdpbusd m22, m0, m10 - vpdpbusd m23, m1, m10 - vpdpbusd m22, m1, m11 - vpdpbusd m23, m2, m11 - packssdw m22, m23 - psraw m22, 2 ; 7 8 9 A - valignq m0, m22, m3, 2 ; 4 5 6 7 - valignq m1, m22, m3, 4 ; 5 6 7 8 - valignq m2, m22, m3, 6 ; 6 7 8 9 - mova m3, m22 - punpcklwd m4, m0, m1 ; 45a 56a 67a 78a - punpckhwd m5, m0, m1 ; 45b 56b 67b 78b - punpcklwd m6, m2, m3 ; 67a 78a 89a 9Aa - punpckhwd m7, m2, m3 ; 67b 78b 89b 9Ab - vpdpwssd m20, m4, m14 - vpdpwssd m21, m5, m14 - vpdpwssd m20, m6, m15 - vpdpwssd m21, m7, m15 - psrad m20, 6 - psrad m21, 6 - packssdw m20, m21 - mova [tmpq], m20 - add tmpq, 64 - sub hd, 4 - jg .hv_w8_loop -%else - lea r6d, [wq-8] - mov r5, tmpq - mov r7, srcq - shl r6d, 5 - mov r6b, hb -.hv_w8_loop0: - vbroadcasti128 m7, [subpel_h_shufA] - vbroadcasti128 m8, [subpel_h_shufB] - vbroadcasti128 m9, [subpel_h_shufC] - movu xm4, [srcq+strideq*0] - movu xm5, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - movu xm6, [srcq+strideq*0] - vbroadcasti128 m0, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - vpblendd m4, m4, m0, 0xf0 ; 0 3 - vinserti128 m5, m5, [srcq+strideq*0], 1 ; 1 4 - vinserti128 m6, m6, [srcq+strideq*1], 1 ; 2 5 - lea srcq, [srcq+strideq*2] - vinserti128 m0, m0, [srcq+strideq*0], 1 ; 3 6 - HV_H_W8 m4, m1, m2, m3, m7, m8, m9 - HV_H_W8 m5, m1, m2, m3, m7, m8, m9 - HV_H_W8 m6, m1, m2, m3, m7, m8, m9 - HV_H_W8 m0, m1, m2, m3, m7, m8, m9 - vpbroadcastd m7, [pw_8192] - vpermq m4, m4, q3120 - vpermq m5, m5, q3120 - vpermq m6, m6, q3120 - pmulhrsw m0, m7 - pmulhrsw m4, m7 - pmulhrsw m5, m7 - pmulhrsw m6, m7 - vpermq m7, m0, q3120 - punpcklwd m1, m4, m5 ; 01 - punpckhwd m4, m5 ; 34 - punpcklwd m2, m5, m6 ; 12 - punpckhwd m5, m6 ; 45 - punpcklwd m3, m6, m7 ; 23 - punpckhwd m6, m7 ; 56 -.hv_w8_loop: - vextracti128 [tmpq], m0, 1 ; not enough registers - movu xm0, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - vinserti128 m0, m0, [srcq+strideq*0], 1 ; 7 8 - pmaddwd m8, m1, m12 ; a0 - pmaddwd m9, m2, m12 ; b0 - mova m1, m3 - mova m2, m4 - pmaddwd m3, m13 ; a1 - pmaddwd m4, m13 ; b1 - paddd m8, m3 - paddd m9, m4 - mova m3, m5 - mova m4, m6 - pmaddwd m5, m14 ; a2 - pmaddwd m6, m14 ; b2 - paddd m8, m5 - paddd m9, m6 - vbroadcasti128 m6, [subpel_h_shufB] - vbroadcasti128 m7, [subpel_h_shufC] - vbroadcasti128 m5, [subpel_h_shufA] - HV_H_W8 m0, m5, m6, m7, m5, m6, m7 - vpbroadcastd m5, [pw_8192] - vpbroadcastd m7, [pd_32] - vbroadcasti128 m6, [tmpq] - pmulhrsw m0, m5 - paddd m8, m7 - paddd m9, m7 - vpermq m7, m0, q3120 ; 7 8 - shufpd m6, m6, m7, 0x04 ; 6 7 - punpcklwd m5, m6, m7 ; 67 - punpckhwd m6, m7 ; 78 - pmaddwd m7, m5, m15 ; a3 - paddd m8, m7 - pmaddwd m7, m6, m15 ; b3 - paddd m7, m9 - psrad m8, 6 - psrad m7, 6 - packssdw m8, m7 - vpermq m7, m8, q3120 - mova [tmpq+wq*0], xm7 - vextracti128 [tmpq+wq*2], m7, 1 - lea tmpq, [tmpq+wq*4] - sub hd, 2 - jg .hv_w8_loop - movzx hd, r6b - add r5, 16 - add r7, 8 - mov tmpq, r5 - mov srcq, r7 - sub r6d, 1<<8 - jg .hv_w8_loop0 -%endif - RET -%if cpuflag(avx512icl) -.hv_w16: - mov wd, 16*2 - jmp .hv_start -.hv_w32: - mov wd, 32*2 - jmp .hv_start -.hv_w64: - mov wd, 64*2 - jmp .hv_start -.hv_w128: - mov wd, 128*2 -.hv_start: - WIN64_SPILL_XMM 31 - mova m16, [spel_h_perm16a] - mova m17, [spel_h_perm16b] - mova m18, [spel_h_perm16c] - lea r6d, [wq*8-16*2*8+hq] - mov r5, tmpq - mov r7, srcq -.hv_loop0: - movu ym0, [srcq+strideq*0] - vinserti32x8 m0, [srcq+strideq*1], 1 - lea srcq, [srcq+strideq*2] - movu ym1, [srcq+strideq*0] - vinserti32x8 m1, [srcq+strideq*1], 1 - lea srcq, [srcq+strideq*2] - movu ym2, [srcq+strideq*0] - vinserti32x8 m2, [srcq+strideq*1], 1 - lea srcq, [srcq+strideq*2] - movu ym3, [srcq+strideq*0] - mova m4, m8 - mova m5, m8 - mova m6, m8 - mova m7, m8 - vpermb m19, m16, m0 - vpermb m20, m17, m0 - vpermb m21, m18, m0 - vpermb m22, m16, m1 - vpermb m23, m17, m1 - vpermb m24, m18, m1 - vpermb m25, m16, m2 - vpermb m26, m17, m2 - vpermb m27, m18, m2 - vpermb ym28, ym16, ym3 - vpermb ym29, ym17, ym3 - vpermb ym30, ym18, ym3 - mova m0, m8 - mova m1, m8 - mova ym2, ym8 - mova ym3, ym8 - vpdpbusd m4, m19, m10 - vpdpbusd m5, m20, m10 - vpdpbusd m6, m22, m10 - vpdpbusd m7, m23, m10 - vpdpbusd m0, m25, m10 - vpdpbusd m1, m26, m10 - vpdpbusd ym2, ym28, ym10 - vpdpbusd ym3, ym29, ym10 - vpdpbusd m4, m20, m11 - vpdpbusd m5, m21, m11 - vpdpbusd m6, m23, m11 - vpdpbusd m7, m24, m11 - vpdpbusd m0, m26, m11 - vpdpbusd m1, m27, m11 - vpdpbusd ym2, ym29, ym11 - vpdpbusd ym3, ym30, ym11 - packssdw m4, m5 - packssdw m6, m7 - packssdw m0, m1 - packssdw ym2, ym3 - psraw m4, 2 ; 0a 0b 1a 1b - psraw m6, 2 ; 2a 2b 3a 3b - psraw m0, 2 ; 4a 4b 5a 5b - psraw ym2, 2 ; 6a 6b __ __ - vshufi32x4 m5, m4, m6, q1032 ; 1a 1b 2a 2b - vshufi32x4 m7, m6, m0, q1032 ; 3a 3b 4a 4b - vshufi32x4 m1, m0, m2, q1032 ; 5a 5b 6a 6b - punpcklwd m2, m4, m5 ; 01a 01c 12a 12c - punpckhwd m3, m4, m5 ; 01b 01d 12b 12d - punpcklwd m4, m6, m7 ; 23a 23c 34a 34c - punpckhwd m5, m6, m7 ; 23b 23d 34b 34d - punpcklwd m6, m0, m1 ; 45a 45c 56a 56c - punpckhwd m7, m0, m1 ; 45b 45d 56b 56d -.hv_loop: - movu ym19, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - vinserti32x8 m19, [srcq+strideq*0], 1 - mova m20, m9 - mova m21, m9 - mova m22, m8 - mova m23, m8 - vpdpwssd m20, m2, m12 - vpdpwssd m21, m3, m12 - vpdpwssd m20, m4, m13 - vpdpwssd m21, m5, m13 - vpermb m24, m16, m19 - vpermb m25, m17, m19 - vpermb m26, m18, m19 - vpdpbusd m22, m24, m10 - vpdpbusd m23, m25, m10 - vpdpbusd m22, m25, m11 - vpdpbusd m23, m26, m11 - packssdw m22, m23 - psraw m22, 2 ; 7a 7b 8a 8b - vshufi32x4 m0, m1, m22, q1032 ; 6a 6b 7a 7b - mova m2, m4 - mova m3, m5 - mova m1, m22 - mova m4, m6 - mova m5, m7 - punpcklwd m6, m0, m1 ; 67a 67c 78a 78c - punpckhwd m7, m0, m1 ; 67b 67d 78b 78d - vpdpwssd m20, m4, m14 - vpdpwssd m21, m5, m14 - vpdpwssd m20, m6, m15 - vpdpwssd m21, m7, m15 - psrad m20, 6 - psrad m21, 6 - packssdw m20, m21 - mova [tmpq+wq*0], ym20 - vextracti32x8 [tmpq+wq*1], m20, 1 - lea tmpq, [tmpq+wq*2] - sub hd, 2 - jg .hv_loop - movzx hd, r6b - add r5, 32 - add r7, 16 - mov tmpq, r5 - mov srcq, r7 - sub r6d, 1<<8 - jg .hv_loop0 -%endif - RET -%endmacro - -%macro WARP_V 5 ; dst, 02, 46, 13, 57 - ; Can be done using gathers, but that's terribly slow on many CPU:s - lea tmp1d, [myq+deltaq*4] - lea tmp2d, [myq+deltaq*1] - shr myd, 10 - shr tmp1d, 10 - movq xm8, [filterq+myq *8] - vinserti128 m8, [filterq+tmp1q*8], 1 ; a e - lea tmp1d, [tmp2q+deltaq*4] - lea myd, [tmp2q+deltaq*1] - shr tmp2d, 10 - shr tmp1d, 10 - movq xm0, [filterq+tmp2q*8] - vinserti128 m0, [filterq+tmp1q*8], 1 ; b f - lea tmp1d, [myq+deltaq*4] - lea tmp2d, [myq+deltaq*1] - shr myd, 10 - shr tmp1d, 10 - movq xm9, [filterq+myq *8] - vinserti128 m9, [filterq+tmp1q*8], 1 ; c g - lea tmp1d, [tmp2q+deltaq*4] - lea myd, [tmp2q+gammaq] ; my += gamma - shr tmp2d, 10 - shr tmp1d, 10 - punpcklwd m8, m0 - movq xm0, [filterq+tmp2q*8] - vinserti128 m0, [filterq+tmp1q*8], 1 ; d h - punpcklwd m0, m9, m0 - punpckldq m9, m8, m0 - punpckhdq m0, m8, m0 - punpcklbw m8, m11, m9 ; a0 a2 b0 b2 c0 c2 d0 d2 << 8 - punpckhbw m9, m11, m9 ; a4 a6 b4 b6 c4 c6 d4 d6 << 8 - pmaddwd m%2, m8 - pmaddwd m9, m%3 - punpcklbw m8, m11, m0 ; a1 a3 b1 b3 c1 c3 d1 d3 << 8 - punpckhbw m0, m11, m0 ; a5 a7 b5 b7 c5 c7 d5 d7 << 8 - pmaddwd m8, m%4 - pmaddwd m0, m%5 - paddd m%2, m9 - paddd m0, m8 - paddd m%1, m0, m%2 -%endmacro - -cglobal warp_affine_8x8t, 0, 14, 0, tmp, ts -%if WIN64 - sub rsp, 0xa0 -%endif - call mangle(private_prefix %+ _warp_affine_8x8_avx2).main -.loop: - psrad m7, 13 - psrad m0, 13 - packssdw m7, m0 - pmulhrsw m7, m14 ; (x + (1 << 6)) >> 7 - vpermq m7, m7, q3120 - mova [tmpq+tsq*0], xm7 - vextracti128 [tmpq+tsq*2], m7, 1 - dec r4d - jz mangle(private_prefix %+ _warp_affine_8x8_avx2).end - call mangle(private_prefix %+ _warp_affine_8x8_avx2).main2 - lea tmpq, [tmpq+tsq*4] - jmp .loop - -cglobal warp_affine_8x8, 0, 14, 0, dst, ds, src, ss, abcd, mx, tmp2, alpha, \ - beta, filter, tmp1, delta, my, gamma -%if WIN64 - sub rsp, 0xa0 - %assign xmm_regs_used 16 - %assign stack_size_padded 0xa0 - %assign stack_offset stack_offset+stack_size_padded -%endif - call .main - jmp .start -.loop: - call .main2 - lea dstq, [dstq+dsq*2] -.start: - psrad m7, 18 - psrad m0, 18 - packusdw m7, m0 - pavgw m7, m11 ; (x + (1 << 10)) >> 11 - vextracti128 xm0, m7, 1 - packuswb xm7, xm0 - pshufd xm7, xm7, q3120 - movq [dstq+dsq*0], xm7 - movhps [dstq+dsq*1], xm7 - dec r4d - jg .loop -.end: - RET -ALIGN function_align -.main: - ; Stack args offset by one (r4m -> r5m etc.) due to call -%if WIN64 - mov abcdq, r5m - mov mxd, r6m - movaps [rsp+stack_offset+0x10], xmm6 - movaps [rsp+stack_offset+0x20], xmm7 - movaps [rsp+0x28], xmm8 - movaps [rsp+0x38], xmm9 - movaps [rsp+0x48], xmm10 - movaps [rsp+0x58], xmm11 - movaps [rsp+0x68], xmm12 - movaps [rsp+0x78], xmm13 - movaps [rsp+0x88], xmm14 - movaps [rsp+0x98], xmm15 -%endif - movsx alphad, word [abcdq+2*0] - movsx betad, word [abcdq+2*1] - mova m12, [warp_8x8_shufA] - mova m13, [warp_8x8_shufB] - vpbroadcastd m14, [pw_8192] - vpbroadcastd m15, [pd_32768] - pxor m11, m11 - lea filterq, [mc_warp_filter] - lea tmp1q, [ssq*3+3] - add mxd, 512+(64<<10) - lea tmp2d, [alphaq*3] - sub srcq, tmp1q ; src -= src_stride*3 + 3 - sub betad, tmp2d ; beta -= alpha*3 - mov myd, r7m - call .h - psrld m1, m0, 16 - call .h - psrld m4, m0, 16 - call .h - pblendw m1, m0, 0xaa ; 02 - call .h - pblendw m4, m0, 0xaa ; 13 - call .h - psrld m2, m1, 16 - pblendw m2, m0, 0xaa ; 24 - call .h - psrld m5, m4, 16 - pblendw m5, m0, 0xaa ; 35 - call .h - psrld m3, m2, 16 - pblendw m3, m0, 0xaa ; 46 - movsx deltad, word [abcdq+2*2] - movsx gammad, word [abcdq+2*3] - add myd, 512+(64<<10) - mov r4d, 4 - lea tmp1d, [deltaq*3] - sub gammad, tmp1d ; gamma -= delta*3 -.main2: - call .h - psrld m6, m5, 16 - pblendw m6, m0, 0xaa ; 57 - WARP_V 7, 1, 3, 4, 6 - call .h - mova m1, m2 - mova m2, m3 - psrld m3, 16 - pblendw m3, m0, 0xaa ; 68 - WARP_V 0, 4, 6, 1, 3 - mova m4, m5 - mova m5, m6 - ret -ALIGN function_align -.h: - lea tmp1d, [mxq+alphaq*4] - lea tmp2d, [mxq+alphaq*1] - vbroadcasti128 m10, [srcq] - shr mxd, 10 - shr tmp1d, 10 - movq xm8, [filterq+mxq *8] - vinserti128 m8, [filterq+tmp1q*8], 1 - lea tmp1d, [tmp2q+alphaq*4] - lea mxd, [tmp2q+alphaq*1] - shr tmp2d, 10 - shr tmp1d, 10 - movq xm0, [filterq+tmp2q*8] - vinserti128 m0, [filterq+tmp1q*8], 1 - lea tmp1d, [mxq+alphaq*4] - lea tmp2d, [mxq+alphaq*1] - shr mxd, 10 - shr tmp1d, 10 - movq xm9, [filterq+mxq *8] - vinserti128 m9, [filterq+tmp1q*8], 1 - lea tmp1d, [tmp2q+alphaq*4] - lea mxd, [tmp2q+betaq] ; mx += beta - shr tmp2d, 10 - shr tmp1d, 10 - punpcklqdq m8, m0 ; 0 1 4 5 - movq xm0, [filterq+tmp2q*8] - vinserti128 m0, [filterq+tmp1q*8], 1 - punpcklqdq m9, m0 ; 2 3 6 7 - pshufb m0, m10, m12 - pmaddubsw m0, m8 - pshufb m10, m13 - pmaddubsw m10, m9 - add srcq, ssq - phaddw m0, m10 - pmaddwd m0, m14 ; 17-bit intermediate, upshifted by 13 - paddd m0, m15 ; rounded 14-bit result in upper 16 bits of dword - ret - -%macro WRAP_YMM 1+ - INIT_YMM cpuname - %1 - INIT_ZMM cpuname -%endmacro - -%macro BIDIR_FN 1 ; op -%if mmsize == 64 - lea stride3q, [strideq*3] - jmp wq -.w4: - cmp hd, 8 - jg .w4_h16 - WRAP_YMM %1 0 - vextracti32x4 xmm1, ym0, 1 - movd [dstq ], xm0 - pextrd [dstq+strideq*1], xm0, 1 - movd [dstq+strideq*2], xmm1 - pextrd [dstq+stride3q ], xmm1, 1 - jl .w4_ret - lea dstq, [dstq+strideq*4] - pextrd [dstq ], xm0, 2 - pextrd [dstq+strideq*1], xm0, 3 - pextrd [dstq+strideq*2], xmm1, 2 - pextrd [dstq+stride3q ], xmm1, 3 -.w4_ret: - RET -.w4_h16: - vpbroadcastd m7, strided - pmulld m7, [bidir_sctr_w4] - %1 0 - kxnorw k1, k1, k1 - vpscatterdd [dstq+m7]{k1}, m0 - RET -.w8: - cmp hd, 4 - jne .w8_h8 - WRAP_YMM %1 0 - vextracti128 xmm1, ym0, 1 - movq [dstq ], xm0 - movq [dstq+strideq*1], xmm1 - movhps [dstq+strideq*2], xm0 - movhps [dstq+stride3q ], xmm1 - RET -.w8_loop: - %1_INC_PTR 2 - lea dstq, [dstq+strideq*4] -.w8_h8: - %1 0 - vextracti32x4 xmm1, ym0, 1 - vextracti32x4 xmm2, m0, 2 - vextracti32x4 xmm3, m0, 3 - movq [dstq ], xm0 - movq [dstq+strideq*1], xmm1 - movq [dstq+strideq*2], xmm2 - movq [dstq+stride3q ], xmm3 - lea dstq, [dstq+strideq*4] - movhps [dstq ], xm0 - movhps [dstq+strideq*1], xmm1 - movhps [dstq+strideq*2], xmm2 - movhps [dstq+stride3q ], xmm3 - sub hd, 8 - jg .w8_loop - RET -.w16_loop: - %1_INC_PTR 2 - lea dstq, [dstq+strideq*4] -.w16: - %1 0 - vpermq m0, m0, q3120 - mova [dstq ], xm0 - vextracti32x4 [dstq+strideq*1], m0, 2 - vextracti32x4 [dstq+strideq*2], ym0, 1 - vextracti32x4 [dstq+stride3q ], m0, 3 - sub hd, 4 - jg .w16_loop - RET -.w32: - pmovzxbq m7, [warp_8x8_shufA] -.w32_loop: - %1 0 - %1_INC_PTR 2 - vpermq m0, m7, m0 - mova [dstq+strideq*0], ym0 - vextracti32x8 [dstq+strideq*1], m0, 1 - lea dstq, [dstq+strideq*2] - sub hd, 2 - jg .w32_loop - RET -.w64: - pmovzxbq m7, [warp_8x8_shufA] -.w64_loop: - %1 0 - %1_INC_PTR 2 - vpermq m0, m7, m0 - mova [dstq], m0 - add dstq, strideq - dec hd - jg .w64_loop - RET -.w128: - pmovzxbq m7, [warp_8x8_shufA] -.w128_loop: - %1 0 - vpermq m6, m7, m0 - %1 2 - mova [dstq+64*0], m6 - %1_INC_PTR 4 - vpermq m6, m7, m0 - mova [dstq+64*1], m6 - add dstq, strideq - dec hd - jg .w128_loop - RET -%else - %1 0 - lea stride3q, [strideq*3] - jmp wq -.w4: - vextracti128 xm1, m0, 1 - movd [dstq ], xm0 - pextrd [dstq+strideq*1], xm0, 1 - movd [dstq+strideq*2], xm1 - pextrd [dstq+stride3q ], xm1, 1 - cmp hd, 4 - je .ret - lea dstq, [dstq+strideq*4] - pextrd [dstq ], xm0, 2 - pextrd [dstq+strideq*1], xm0, 3 - pextrd [dstq+strideq*2], xm1, 2 - pextrd [dstq+stride3q ], xm1, 3 - cmp hd, 8 - je .ret - %1 2 - lea dstq, [dstq+strideq*4] - vextracti128 xm1, m0, 1 - movd [dstq ], xm0 - pextrd [dstq+strideq*1], xm0, 1 - movd [dstq+strideq*2], xm1 - pextrd [dstq+stride3q ], xm1, 1 - lea dstq, [dstq+strideq*4] - pextrd [dstq ], xm0, 2 - pextrd [dstq+strideq*1], xm0, 3 - pextrd [dstq+strideq*2], xm1, 2 - pextrd [dstq+stride3q ], xm1, 3 -.ret: - RET -.w8_loop: - %1_INC_PTR 2 - %1 0 - lea dstq, [dstq+strideq*4] -.w8: - vextracti128 xm1, m0, 1 - movq [dstq ], xm0 - movq [dstq+strideq*1], xm1 - movhps [dstq+strideq*2], xm0 - movhps [dstq+stride3q ], xm1 - sub hd, 4 - jg .w8_loop - RET -.w16_loop: - %1_INC_PTR 4 - %1 0 - lea dstq, [dstq+strideq*4] -.w16: - vpermq m0, m0, q3120 - mova [dstq ], xm0 - vextracti128 [dstq+strideq*1], m0, 1 - %1 2 - vpermq m0, m0, q3120 - mova [dstq+strideq*2], xm0 - vextracti128 [dstq+stride3q ], m0, 1 - sub hd, 4 - jg .w16_loop - RET -.w32_loop: - %1_INC_PTR 4 - %1 0 - lea dstq, [dstq+strideq*2] -.w32: - vpermq m0, m0, q3120 - mova [dstq+strideq*0], m0 - %1 2 - vpermq m0, m0, q3120 - mova [dstq+strideq*1], m0 - sub hd, 2 - jg .w32_loop - RET -.w64_loop: - %1_INC_PTR 4 - %1 0 - add dstq, strideq -.w64: - vpermq m0, m0, q3120 - mova [dstq], m0 - %1 2 - vpermq m0, m0, q3120 - mova [dstq+32], m0 - dec hd - jg .w64_loop - RET -.w128_loop: - %1 0 - add dstq, strideq -.w128: - vpermq m0, m0, q3120 - mova [dstq+0*32], m0 - %1 2 - vpermq m0, m0, q3120 - mova [dstq+1*32], m0 - %1_INC_PTR 8 - %1 -4 - vpermq m0, m0, q3120 - mova [dstq+2*32], m0 - %1 -2 - vpermq m0, m0, q3120 - mova [dstq+3*32], m0 - dec hd - jg .w128_loop - RET -%endif -%endmacro - -%macro AVG 1 ; src_offset - mova m0, [tmp1q+(%1+0)*mmsize] - paddw m0, [tmp2q+(%1+0)*mmsize] - mova m1, [tmp1q+(%1+1)*mmsize] - paddw m1, [tmp2q+(%1+1)*mmsize] - pmulhrsw m0, m2 - pmulhrsw m1, m2 - packuswb m0, m1 -%endmacro - -%macro AVG_INC_PTR 1 - add tmp1q, %1*mmsize - add tmp2q, %1*mmsize -%endmacro - -%macro AVG_FN 0 -cglobal avg, 4, 7, 3, dst, stride, tmp1, tmp2, w, h, stride3 -%define base r6-avg %+ SUFFIX %+ _table - lea r6, [avg %+ SUFFIX %+ _table] - tzcnt wd, wm - movifnidn hd, hm - movsxd wq, dword [r6+wq*4] - vpbroadcastd m2, [base+pw_1024] - add wq, r6 - BIDIR_FN AVG -%endmacro - -%macro W_AVG 1 ; src_offset - ; (a * weight + b * (16 - weight) + 128) >> 8 - ; = ((a - b) * weight + (b << 4) + 128) >> 8 - ; = ((((a - b) * ((weight-16) << 12)) >> 16) + a + 8) >> 4 - ; = ((((b - a) * (-weight << 12)) >> 16) + b + 8) >> 4 - mova m0, [tmp1q+(%1+0)*mmsize] - psubw m2, m0, [tmp2q+(%1+0)*mmsize] - mova m1, [tmp1q+(%1+1)*mmsize] - psubw m3, m1, [tmp2q+(%1+1)*mmsize] - pmulhw m2, m4 - pmulhw m3, m4 - paddw m0, m2 - paddw m1, m3 - pmulhrsw m0, m5 - pmulhrsw m1, m5 - packuswb m0, m1 -%endmacro - -%define W_AVG_INC_PTR AVG_INC_PTR - -%macro W_AVG_FN 0 -cglobal w_avg, 4, 7, 6, dst, stride, tmp1, tmp2, w, h, stride3 -%define base r6-w_avg %+ SUFFIX %+ _table - lea r6, [w_avg %+ SUFFIX %+ _table] - tzcnt wd, wm - movifnidn hd, hm - vpbroadcastw m4, r6m ; weight - movsxd wq, dword [r6+wq*4] - vpbroadcastd m5, [base+pw_2048] - psllw m4, 12 ; (weight-16) << 12 when interpreted as signed - add wq, r6 - cmp dword r6m, 7 - jg .weight_gt7 - mov r6, tmp1q - pxor m0, m0 - mov tmp1q, tmp2q - psubw m4, m0, m4 ; -weight - mov tmp2q, r6 -.weight_gt7: - BIDIR_FN W_AVG -%endmacro - -%macro MASK 1 ; src_offset - ; (a * m + b * (64 - m) + 512) >> 10 - ; = ((a - b) * m + (b << 6) + 512) >> 10 - ; = ((((b - a) * (-m << 10)) >> 16) + b + 8) >> 4 -%if mmsize == 64 - vpermq m3, m8, [maskq+%1*32] -%else - vpermq m3, [maskq+%1*16], q3120 -%endif - mova m0, [tmp2q+(%1+0)*mmsize] - psubw m1, m0, [tmp1q+(%1+0)*mmsize] - psubb m3, m4, m3 - paddw m1, m1 ; (b - a) << 1 - paddb m3, m3 - punpcklbw m2, m4, m3 ; -m << 9 - pmulhw m1, m2 - paddw m0, m1 - mova m1, [tmp2q+(%1+1)*mmsize] - psubw m2, m1, [tmp1q+(%1+1)*mmsize] - paddw m2, m2 - punpckhbw m3, m4, m3 - pmulhw m2, m3 - paddw m1, m2 - pmulhrsw m0, m5 - pmulhrsw m1, m5 - packuswb m0, m1 -%endmacro - -%macro MASK_INC_PTR 1 - add maskq, %1*mmsize/2 - add tmp2q, %1*mmsize - add tmp1q, %1*mmsize -%endmacro - -%macro MASK_FN 0 -cglobal mask, 4, 8, 6, dst, stride, tmp1, tmp2, w, h, mask, stride3 -%define base r7-mask %+ SUFFIX %+ _table - lea r7, [mask %+ SUFFIX %+ _table] - tzcnt wd, wm - movifnidn hd, hm - mov maskq, maskmp - movsxd wq, dword [r7+wq*4] - pxor m4, m4 -%if mmsize == 64 - mova m8, [base+bilin_v_perm64] -%endif - vpbroadcastd m5, [base+pw_2048] - add wq, r7 - BIDIR_FN MASK -%endmacro MASK_FN - -%macro W_MASK 4-5 0 ; dst, mask, tmp_offset[1-2], 4:4:4 - mova m%1, [tmp1q+mmsize*%3] - mova m1, [tmp2q+mmsize*%3] - psubw m1, m%1 - pabsw m%2, m1 - psubusw m%2, m6, m%2 - psrlw m%2, 8 ; 64 - m - psllw m2, m%2, 10 - pmulhw m1, m2 - paddw m%1, m1 - mova m1, [tmp1q+mmsize*%4] - mova m2, [tmp2q+mmsize*%4] - psubw m2, m1 - pabsw m3, m2 - psubusw m3, m6, m3 -%if cpuflag(avx512icl) - vpshldw m%2, m3, 8 - psllw m3, m%2, 10 -%if %5 - psubb m%2, m5, m%2 -%endif -%else - psrlw m3, 8 -%if %5 - packuswb m%2, m3 - psubb m%2, m5, m%2 - vpermq m%2, m%2, q3120 -%else - phaddw m%2, m3 -%endif - psllw m3, 10 -%endif - pmulhw m2, m3 - paddw m1, m2 - pmulhrsw m%1, m7 - pmulhrsw m1, m7 - packuswb m%1, m1 -%endmacro - -cglobal blend, 3, 7, 7, dst, ds, tmp, w, h, mask -%define base r6-blend_avx2_table - lea r6, [blend_avx2_table] - tzcnt wd, wm - movifnidn hd, hm - movifnidn maskq, maskmp - movsxd wq, dword [r6+wq*4] - vpbroadcastd m4, [base+pb_64] - vpbroadcastd m5, [base+pw_512] - add wq, r6 - lea r6, [dsq*3] - jmp wq -.w4: - movd xm0, [dstq+dsq*0] - pinsrd xm0, [dstq+dsq*1], 1 - vpbroadcastd xm1, [dstq+dsq*2] - pinsrd xm1, [dstq+r6 ], 3 - mova xm6, [maskq] - psubb xm3, xm4, xm6 - punpcklbw xm2, xm3, xm6 - punpckhbw xm3, xm6 - mova xm6, [tmpq] - add maskq, 4*4 - add tmpq, 4*4 - punpcklbw xm0, xm6 - punpckhbw xm1, xm6 - pmaddubsw xm0, xm2 - pmaddubsw xm1, xm3 - pmulhrsw xm0, xm5 - pmulhrsw xm1, xm5 - packuswb xm0, xm1 - movd [dstq+dsq*0], xm0 - pextrd [dstq+dsq*1], xm0, 1 - pextrd [dstq+dsq*2], xm0, 2 - pextrd [dstq+r6 ], xm0, 3 - lea dstq, [dstq+dsq*4] - sub hd, 4 - jg .w4 - RET -ALIGN function_align -.w8: - movq xm1, [dstq+dsq*0] - movhps xm1, [dstq+dsq*1] - vpbroadcastq m2, [dstq+dsq*2] - vpbroadcastq m3, [dstq+r6 ] - mova m0, [maskq] - mova m6, [tmpq] - add maskq, 8*4 - add tmpq, 8*4 - vpblendd m1, m2, 0x30 - vpblendd m1, m3, 0xc0 - psubb m3, m4, m0 - punpcklbw m2, m3, m0 - punpckhbw m3, m0 - punpcklbw m0, m1, m6 - punpckhbw m1, m6 - pmaddubsw m0, m2 - pmaddubsw m1, m3 - pmulhrsw m0, m5 - pmulhrsw m1, m5 - packuswb m0, m1 - vextracti128 xm1, m0, 1 - movq [dstq+dsq*0], xm0 - movhps [dstq+dsq*1], xm0 - movq [dstq+dsq*2], xm1 - movhps [dstq+r6 ], xm1 - lea dstq, [dstq+dsq*4] - sub hd, 4 - jg .w8 - RET -ALIGN function_align -.w16: - mova m0, [maskq] - mova xm1, [dstq+dsq*0] - vinserti128 m1, [dstq+dsq*1], 1 - psubb m3, m4, m0 - punpcklbw m2, m3, m0 - punpckhbw m3, m0 - mova m6, [tmpq] - add maskq, 16*2 - add tmpq, 16*2 - punpcklbw m0, m1, m6 - punpckhbw m1, m6 - pmaddubsw m0, m2 - pmaddubsw m1, m3 - pmulhrsw m0, m5 - pmulhrsw m1, m5 - packuswb m0, m1 - mova [dstq+dsq*0], xm0 - vextracti128 [dstq+dsq*1], m0, 1 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .w16 - RET -ALIGN function_align -.w32: - mova m0, [maskq] - mova m1, [dstq] - mova m6, [tmpq] - add maskq, 32 - add tmpq, 32 - psubb m3, m4, m0 - punpcklbw m2, m3, m0 - punpckhbw m3, m0 - punpcklbw m0, m1, m6 - punpckhbw m1, m6 - pmaddubsw m0, m2 - pmaddubsw m1, m3 - pmulhrsw m0, m5 - pmulhrsw m1, m5 - packuswb m0, m1 - mova [dstq], m0 - add dstq, dsq - dec hd - jg .w32 - RET - -cglobal blend_v, 3, 6, 6, dst, ds, tmp, w, h, mask -%define base r5-blend_v_avx2_table - lea r5, [blend_v_avx2_table] - tzcnt wd, wm - movifnidn hd, hm - movsxd wq, dword [r5+wq*4] - vpbroadcastd m5, [base+pw_512] - add wq, r5 - add maskq, obmc_masks-blend_v_avx2_table - jmp wq -.w2: - vpbroadcastd xm2, [maskq+2*2] -.w2_s0_loop: - movd xm0, [dstq+dsq*0] - pinsrw xm0, [dstq+dsq*1], 1 - movd xm1, [tmpq] - add tmpq, 2*2 - punpcklbw xm0, xm1 - pmaddubsw xm0, xm2 - pmulhrsw xm0, xm5 - packuswb xm0, xm0 - pextrw [dstq+dsq*0], xm0, 0 - pextrw [dstq+dsq*1], xm0, 1 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .w2_s0_loop - RET -ALIGN function_align -.w4: - vpbroadcastq xm2, [maskq+4*2] -.w4_loop: - movd xm0, [dstq+dsq*0] - pinsrd xm0, [dstq+dsq*1], 1 - movq xm1, [tmpq] - add tmpq, 4*2 - punpcklbw xm0, xm1 - pmaddubsw xm0, xm2 - pmulhrsw xm0, xm5 - packuswb xm0, xm0 - movd [dstq+dsq*0], xm0 - pextrd [dstq+dsq*1], xm0, 1 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .w4_loop - RET -ALIGN function_align -.w8: - vbroadcasti128 m4, [maskq+8*2] -.w8_loop: - vpbroadcastq m2, [dstq+dsq*0] - movq xm0, [dstq+dsq*1] - vpblendd m0, m2, 0x30 - movq xm1, [tmpq+8*1] - vinserti128 m1, [tmpq+8*0], 1 - add tmpq, 8*2 - punpcklbw m0, m1 - pmaddubsw m0, m4 - pmulhrsw m0, m5 - vextracti128 xm1, m0, 1 - packuswb xm0, xm1 - movhps [dstq+dsq*0], xm0 - movq [dstq+dsq*1], xm0 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .w8_loop - RET -ALIGN function_align -.w16: - vbroadcasti128 m3, [maskq+16*2] - vbroadcasti128 m4, [maskq+16*3] -.w16_loop: - mova xm1, [dstq+dsq*0] - vinserti128 m1, [dstq+dsq*1], 1 - mova m2, [tmpq] - add tmpq, 16*2 - punpcklbw m0, m1, m2 - punpckhbw m1, m2 - pmaddubsw m0, m3 - pmaddubsw m1, m4 - pmulhrsw m0, m5 - pmulhrsw m1, m5 - packuswb m0, m1 - mova [dstq+dsq*0], xm0 - vextracti128 [dstq+dsq*1], m0, 1 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .w16_loop - RET -ALIGN function_align -.w32: - mova xm3, [maskq+16*4] - vinserti128 m3, [maskq+16*6], 1 - mova xm4, [maskq+16*5] - vinserti128 m4, [maskq+16*7], 1 -.w32_loop: - mova m1, [dstq] - mova m2, [tmpq] - add tmpq, 32 - punpcklbw m0, m1, m2 - punpckhbw m1, m2 - pmaddubsw m0, m3 - pmaddubsw m1, m4 - pmulhrsw m0, m5 - pmulhrsw m1, m5 - packuswb m0, m1 - mova [dstq], m0 - add dstq, dsq - dec hd - jg .w32_loop - RET - -cglobal blend_h, 4, 7, 6, dst, ds, tmp, w, h, mask -%define base r5-blend_h_avx2_table - lea r5, [blend_h_avx2_table] - mov r6d, wd - tzcnt wd, wd - mov hd, hm - movsxd wq, dword [r5+wq*4] - vpbroadcastd m5, [base+pw_512] - add wq, r5 - lea maskq, [base+obmc_masks+hq*2] - lea hd, [hq*3] - shr hd, 2 ; h * 3/4 - lea maskq, [maskq+hq*2] - neg hq - jmp wq -.w2: - movd xm0, [dstq+dsq*0] - pinsrw xm0, [dstq+dsq*1], 1 - movd xm2, [maskq+hq*2] - movd xm1, [tmpq] - add tmpq, 2*2 - punpcklwd xm2, xm2 - punpcklbw xm0, xm1 - pmaddubsw xm0, xm2 - pmulhrsw xm0, xm5 - packuswb xm0, xm0 - pextrw [dstq+dsq*0], xm0, 0 - pextrw [dstq+dsq*1], xm0, 1 - lea dstq, [dstq+dsq*2] - add hq, 2 - jl .w2 - RET -ALIGN function_align -.w4: - mova xm3, [blend_shuf] -.w4_loop: - movd xm0, [dstq+dsq*0] - pinsrd xm0, [dstq+dsq*1], 1 - movd xm2, [maskq+hq*2] - movq xm1, [tmpq] - add tmpq, 4*2 - pshufb xm2, xm3 - punpcklbw xm0, xm1 - pmaddubsw xm0, xm2 - pmulhrsw xm0, xm5 - packuswb xm0, xm0 - movd [dstq+dsq*0], xm0 - pextrd [dstq+dsq*1], xm0, 1 - lea dstq, [dstq+dsq*2] - add hq, 2 - jl .w4_loop - RET -ALIGN function_align -.w8: - vbroadcasti128 m4, [blend_shuf] - shufpd m4, m4, 0x03 -.w8_loop: - vpbroadcastq m1, [dstq+dsq*0] - movq xm0, [dstq+dsq*1] - vpblendd m0, m1, 0x30 - vpbroadcastd m3, [maskq+hq*2] - movq xm1, [tmpq+8*1] - vinserti128 m1, [tmpq+8*0], 1 - add tmpq, 8*2 - pshufb m3, m4 - punpcklbw m0, m1 - pmaddubsw m0, m3 - pmulhrsw m0, m5 - vextracti128 xm1, m0, 1 - packuswb xm0, xm1 - movhps [dstq+dsq*0], xm0 - movq [dstq+dsq*1], xm0 - lea dstq, [dstq+dsq*2] - add hq, 2 - jl .w8_loop - RET -ALIGN function_align -.w16: - vbroadcasti128 m4, [blend_shuf] - shufpd m4, m4, 0x0c -.w16_loop: - mova xm1, [dstq+dsq*0] - vinserti128 m1, [dstq+dsq*1], 1 - vpbroadcastd m3, [maskq+hq*2] - mova m2, [tmpq] - add tmpq, 16*2 - pshufb m3, m4 - punpcklbw m0, m1, m2 - punpckhbw m1, m2 - pmaddubsw m0, m3 - pmaddubsw m1, m3 - pmulhrsw m0, m5 - pmulhrsw m1, m5 - packuswb m0, m1 - mova [dstq+dsq*0], xm0 - vextracti128 [dstq+dsq*1], m0, 1 - lea dstq, [dstq+dsq*2] - add hq, 2 - jl .w16_loop - RET -ALIGN function_align -.w32: ; w32/w64/w128 - sub dsq, r6 -.w32_loop0: - vpbroadcastw m3, [maskq+hq*2] - mov wd, r6d -.w32_loop: - mova m1, [dstq] - mova m2, [tmpq] - add tmpq, 32 - punpcklbw m0, m1, m2 - punpckhbw m1, m2 - pmaddubsw m0, m3 - pmaddubsw m1, m3 - pmulhrsw m0, m5 - pmulhrsw m1, m5 - packuswb m0, m1 - mova [dstq], m0 - add dstq, 32 - sub wd, 32 - jg .w32_loop - add dstq, dsq - inc hq - jl .w32_loop0 - RET - -cglobal emu_edge, 10, 13, 1, bw, bh, iw, ih, x, y, dst, dstride, src, sstride, \ - bottomext, rightext - ; we assume that the buffer (stride) is larger than width, so we can - ; safely overwrite by a few bytes - - ; ref += iclip(y, 0, ih - 1) * PXSTRIDE(ref_stride) - xor r12d, r12d - lea r10, [ihq-1] - cmp yq, ihq - cmovs r10, yq - test yq, yq - cmovs r10, r12 - imul r10, sstrideq - add srcq, r10 - - ; ref += iclip(x, 0, iw - 1) - lea r10, [iwq-1] - cmp xq, iwq - cmovs r10, xq - test xq, xq - cmovs r10, r12 - add srcq, r10 - - ; bottom_ext = iclip(y + bh - ih, 0, bh - 1) - lea bottomextq, [yq+bhq] - sub bottomextq, ihq - lea r3, [bhq-1] - cmovs bottomextq, r12 - - DEFINE_ARGS bw, bh, iw, ih, x, topext, dst, dstride, src, sstride, \ - bottomext, rightext - - ; top_ext = iclip(-y, 0, bh - 1) - neg topextq - cmovs topextq, r12 - cmp bottomextq, bhq - cmovns bottomextq, r3 - cmp topextq, bhq - cmovg topextq, r3 - - ; right_ext = iclip(x + bw - iw, 0, bw - 1) - lea rightextq, [xq+bwq] - sub rightextq, iwq - lea r2, [bwq-1] - cmovs rightextq, r12 - - DEFINE_ARGS bw, bh, iw, ih, leftext, topext, dst, dstride, src, sstride, \ - bottomext, rightext - - ; left_ext = iclip(-x, 0, bw - 1) - neg leftextq - cmovs leftextq, r12 - cmp rightextq, bwq - cmovns rightextq, r2 - cmp leftextq, bwq - cmovns leftextq, r2 - - DEFINE_ARGS bw, centerh, centerw, dummy, leftext, topext, \ - dst, dstride, src, sstride, bottomext, rightext - - ; center_h = bh - top_ext - bottom_ext - lea r3, [bottomextq+topextq] - sub centerhq, r3 - - ; blk += top_ext * PXSTRIDE(dst_stride) - mov r2, topextq - imul r2, dstrideq - add dstq, r2 - mov r9m, dstq - - ; center_w = bw - left_ext - right_ext - mov centerwq, bwq - lea r3, [rightextq+leftextq] - sub centerwq, r3 - -%macro v_loop 3 ; need_left_ext, need_right_ext, suffix -.v_loop_%3: -%if %1 - test leftextq, leftextq - jz .body_%3 - - ; left extension - xor r3, r3 - vpbroadcastb m0, [srcq] -.left_loop_%3: - mova [dstq+r3], m0 - add r3, 32 - cmp r3, leftextq - jl .left_loop_%3 - - ; body -.body_%3: - lea r12, [dstq+leftextq] -%endif - xor r3, r3 -.body_loop_%3: - movu m0, [srcq+r3] -%if %1 - movu [r12+r3], m0 -%else - movu [dstq+r3], m0 -%endif - add r3, 32 - cmp r3, centerwq - jl .body_loop_%3 - -%if %2 - ; right extension - test rightextq, rightextq - jz .body_loop_end_%3 -%if %1 - add r12, centerwq -%else - lea r12, [dstq+centerwq] -%endif - xor r3, r3 - vpbroadcastb m0, [srcq+centerwq-1] -.right_loop_%3: - movu [r12+r3], m0 - add r3, 32 - cmp r3, rightextq - jl .right_loop_%3 - -.body_loop_end_%3: -%endif - add dstq, dstrideq - add srcq, sstrideq - dec centerhq - jg .v_loop_%3 -%endmacro - - test leftextq, leftextq - jnz .need_left_ext - test rightextq, rightextq - jnz .need_right_ext - v_loop 0, 0, 0 - jmp .body_done - -.need_left_ext: - test rightextq, rightextq - jnz .need_left_right_ext - v_loop 1, 0, 1 - jmp .body_done - -.need_left_right_ext: - v_loop 1, 1, 2 - jmp .body_done - -.need_right_ext: - v_loop 0, 1, 3 - -.body_done: - ; bottom edge extension - test bottomextq, bottomextq - jz .top - mov srcq, dstq - sub srcq, dstrideq - xor r1, r1 -.bottom_x_loop: - mova m0, [srcq+r1] - lea r3, [dstq+r1] - mov r4, bottomextq -.bottom_y_loop: - mova [r3], m0 - add r3, dstrideq - dec r4 - jg .bottom_y_loop - add r1, 32 - cmp r1, bwq - jl .bottom_x_loop - -.top: - ; top edge extension - test topextq, topextq - jz .end - mov srcq, r9m - mov dstq, dstm - xor r1, r1 -.top_x_loop: - mova m0, [srcq+r1] - lea r3, [dstq+r1] - mov r4, topextq -.top_y_loop: - mova [r3], m0 - add r3, dstrideq - dec r4 - jg .top_y_loop - add r1, 32 - cmp r1, bwq - jl .top_x_loop - -.end: - RET - -INIT_YMM avx2 -PREP_BILIN -PREP_8TAP -AVG_FN -W_AVG_FN -MASK_FN - -cglobal w_mask_420, 4, 8, 14, dst, stride, tmp1, tmp2, w, h, mask, stride3 -%define base r7-w_mask_420_avx2_table - lea r7, [w_mask_420_avx2_table] - tzcnt wd, wm - mov r6d, r7m ; sign - movifnidn hd, hm - movsxd wq, [r7+wq*4] - vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 - vpbroadcastd m7, [base+pw_2048] - pmovzxbd m9, [base+deint_shuf4] - vpbroadcastd m8, [base+wm_420_sign+r6*4] ; 258 - sign - add wq, r7 - W_MASK 0, 4, 0, 1 - mov maskq, maskmp - lea stride3q, [strideq*3] - jmp wq -.w4: - vextracti128 xm1, m0, 1 - movd [dstq+strideq*0], xm0 - pextrd [dstq+strideq*1], xm0, 1 - movd [dstq+strideq*2], xm1 - pextrd [dstq+stride3q ], xm1, 1 - cmp hd, 8 - jl .w4_end - lea dstq, [dstq+strideq*4] - pextrd [dstq+strideq*0], xm0, 2 - pextrd [dstq+strideq*1], xm0, 3 - pextrd [dstq+strideq*2], xm1, 2 - pextrd [dstq+stride3q ], xm1, 3 - jg .w4_h16 -.w4_end: - vextracti128 xm0, m4, 1 - vpblendd xm1, xm4, xm0, 0x05 - vpblendd xm4, xm4, xm0, 0x0a - pshufd xm1, xm1, q2301 - psubw xm4, xm8, xm4 - psubw xm4, xm1 - psrlw xm4, 2 - packuswb xm4, xm4 - movq [maskq], xm4 - RET -.w4_h16: - W_MASK 0, 5, 2, 3 - lea dstq, [dstq+strideq*4] - phaddd m4, m5 - vextracti128 xm1, m0, 1 - psubw m4, m8, m4 - psrlw m4, 2 - vpermd m4, m9, m4 - vextracti128 xm5, m4, 1 - packuswb xm4, xm5 - movd [dstq+strideq*0], xm0 - pextrd [dstq+strideq*1], xm0, 1 - movd [dstq+strideq*2], xm1 - pextrd [dstq+stride3q], xm1, 1 - lea dstq, [dstq+strideq*4] - pextrd [dstq+strideq*0], xm0, 2 - pextrd [dstq+strideq*1], xm0, 3 - pextrd [dstq+strideq*2], xm1, 2 - pextrd [dstq+stride3q ], xm1, 3 - mova [maskq], xm4 - RET -.w8_loop: - add tmp1q, 2*32 - add tmp2q, 2*32 - W_MASK 0, 4, 0, 1 - lea dstq, [dstq+strideq*4] - add maskq, 8 -.w8: - vextracti128 xm2, m4, 1 - vextracti128 xm1, m0, 1 - psubw xm4, xm8, xm4 - psubw xm4, xm2 - psrlw xm4, 2 - packuswb xm4, xm4 - movq [dstq+strideq*0], xm0 - movq [dstq+strideq*1], xm1 - movhps [dstq+strideq*2], xm0 - movhps [dstq+stride3q ], xm1 - movq [maskq], xm4 - sub hd, 4 - jg .w8_loop - RET -.w16_loop: - add tmp1q, 4*32 - add tmp2q, 4*32 - W_MASK 0, 4, 0, 1 - lea dstq, [dstq+strideq*4] - add maskq, 16 -.w16: - vpermq m0, m0, q3120 - mova [dstq+strideq*0], xm0 - vextracti128 [dstq+strideq*1], m0, 1 - W_MASK 0, 5, 2, 3 - punpckhqdq m1, m4, m5 - punpcklqdq m4, m5 - psubw m1, m8, m1 - psubw m1, m4 - psrlw m1, 2 - vpermq m0, m0, q3120 - packuswb m1, m1 - vpermd m1, m9, m1 - mova [dstq+strideq*2], xm0 - vextracti128 [dstq+stride3q ], m0, 1 - mova [maskq], xm1 - sub hd, 4 - jg .w16_loop - RET -.w32_loop: - add tmp1q, 4*32 - add tmp2q, 4*32 - W_MASK 0, 4, 0, 1 - lea dstq, [dstq+strideq*2] - add maskq, 16 -.w32: - vpermq m0, m0, q3120 - mova [dstq+strideq*0], m0 - W_MASK 0, 5, 2, 3 - psubw m4, m8, m4 - psubw m4, m5 - psrlw m4, 2 - vpermq m0, m0, q3120 - packuswb m4, m4 - vpermd m4, m9, m4 - mova [dstq+strideq*1], m0 - mova [maskq], xm4 - sub hd, 2 - jg .w32_loop - RET -.w64_loop_even: - psubw m10, m8, m4 - psubw m11, m8, m5 - dec hd -.w64_loop: - add tmp1q, 4*32 - add tmp2q, 4*32 - W_MASK 0, 4, 0, 1 - add dstq, strideq -.w64: - vpermq m0, m0, q3120 - mova [dstq+32*0], m0 - W_MASK 0, 5, 2, 3 - vpermq m0, m0, q3120 - mova [dstq+32*1], m0 - test hd, 1 - jz .w64_loop_even - psubw m4, m10, m4 - psubw m5, m11, m5 - psrlw m4, 2 - psrlw m5, 2 - packuswb m4, m5 - vpermd m4, m9, m4 - mova [maskq], m4 - add maskq, 32 - dec hd - jg .w64_loop - RET -.w128_loop_even: - psubw m12, m8, m4 - psubw m13, m8, m5 - dec hd -.w128_loop: - W_MASK 0, 4, 0, 1 - add dstq, strideq -.w128: - vpermq m0, m0, q3120 - mova [dstq+32*0], m0 - W_MASK 0, 5, 2, 3 - vpermq m0, m0, q3120 - mova [dstq+32*1], m0 - add tmp1q, 8*32 - add tmp2q, 8*32 - test hd, 1 - jz .w128_even - psubw m4, m10, m4 - psubw m5, m11, m5 - psrlw m4, 2 - psrlw m5, 2 - packuswb m4, m5 - vpermd m4, m9, m4 - mova [maskq+32*0], m4 - jmp .w128_odd -.w128_even: - psubw m10, m8, m4 - psubw m11, m8, m5 -.w128_odd: - W_MASK 0, 4, -4, -3 - vpermq m0, m0, q3120 - mova [dstq+32*2], m0 - W_MASK 0, 5, -2, -1 - vpermq m0, m0, q3120 - mova [dstq+32*3], m0 - test hd, 1 - jz .w128_loop_even - psubw m4, m12, m4 - psubw m5, m13, m5 - psrlw m4, 2 - psrlw m5, 2 - packuswb m4, m5 - vpermd m4, m9, m4 - mova [maskq+32*1], m4 - add maskq, 64 - dec hd - jg .w128_loop - RET - -cglobal w_mask_422, 4, 8, 11, dst, stride, tmp1, tmp2, w, h, mask, stride3 -%define base r7-w_mask_422_avx2_table - lea r7, [w_mask_422_avx2_table] - tzcnt wd, wm - mov r6d, r7m ; sign - movifnidn hd, hm - pxor m9, m9 - movsxd wq, dword [r7+wq*4] - vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 - vpbroadcastd m7, [base+pw_2048] - pmovzxbd m10, [base+deint_shuf4] - vpbroadcastd m8, [base+wm_422_sign+r6*4] ; 128 - sign - add wq, r7 - mov maskq, maskmp - W_MASK 0, 4, 0, 1 - lea stride3q, [strideq*3] - jmp wq -.w4: - vextracti128 xm1, m0, 1 - movd [dstq+strideq*0], xm0 - pextrd [dstq+strideq*1], xm0, 1 - movd [dstq+strideq*2], xm1 - pextrd [dstq+stride3q ], xm1, 1 - cmp hd, 8 - jl .w4_end - lea dstq, [dstq+strideq*4] - pextrd [dstq+strideq*0], xm0, 2 - pextrd [dstq+strideq*1], xm0, 3 - pextrd [dstq+strideq*2], xm1, 2 - pextrd [dstq+stride3q ], xm1, 3 - jg .w4_h16 -.w4_end: - vextracti128 xm5, m4, 1 - packuswb xm4, xm5 - psubb xm5, xm8, xm4 - pavgb xm5, xm9 - pshufd xm5, xm5, q3120 - mova [maskq], xm5 - RET -.w4_h16: - W_MASK 0, 5, 2, 3 - lea dstq, [dstq+strideq*4] - packuswb m4, m5 - psubb m5, m8, m4 - pavgb m5, m9 - vpermd m5, m10, m5 - vextracti128 xm1, m0, 1 - movd [dstq+strideq*0], xm0 - pextrd [dstq+strideq*1], xm0, 1 - movd [dstq+strideq*2], xm1 - pextrd [dstq+stride3q ], xm1, 1 - lea dstq, [dstq+strideq*4] - pextrd [dstq+strideq*0], xm0, 2 - pextrd [dstq+strideq*1], xm0, 3 - pextrd [dstq+strideq*2], xm1, 2 - pextrd [dstq+stride3q ], xm1, 3 - mova [maskq], m5 - RET -.w8_loop: - add tmp1q, 32*2 - add tmp2q, 32*2 - W_MASK 0, 4, 0, 1 - lea dstq, [dstq+strideq*4] - add maskq, 16 -.w8: - vextracti128 xm5, m4, 1 - vextracti128 xm1, m0, 1 - packuswb xm4, xm5 - psubb xm5, xm8, xm4 - pavgb xm5, xm9 - pshufd xm5, xm5, q3120 - movq [dstq+strideq*0], xm0 - movq [dstq+strideq*1], xm1 - movhps [dstq+strideq*2], xm0 - movhps [dstq+stride3q ], xm1 - mova [maskq], xm5 - sub hd, 4 - jg .w8_loop - RET -.w16_loop: - add tmp1q, 32*4 - add tmp2q, 32*4 - W_MASK 0, 4, 0, 1 - lea dstq, [dstq+strideq*4] - add maskq, 32 -.w16: - vpermq m0, m0, q3120 - mova [dstq+strideq*0], xm0 - vextracti128 [dstq+strideq*1], m0, 1 - W_MASK 0, 5, 2, 3 - packuswb m4, m5 - psubb m5, m8, m4 - pavgb m5, m9 - vpermq m0, m0, q3120 - vpermd m5, m10, m5 - mova [dstq+strideq*2], xm0 - vextracti128 [dstq+stride3q ], m0, 1 - mova [maskq], m5 - sub hd, 4 - jg .w16_loop - RET -.w32_loop: - add tmp1q, 32*4 - add tmp2q, 32*4 - W_MASK 0, 4, 0, 1 - lea dstq, [dstq+strideq*2] - add maskq, 32 -.w32: - vpermq m0, m0, q3120 - mova [dstq+strideq*0], m0 - W_MASK 0, 5, 2, 3 - packuswb m4, m5 - psubb m5, m8, m4 - pavgb m5, m9 - vpermq m0, m0, q3120 - vpermd m5, m10, m5 - mova [dstq+strideq*1], m0 - mova [maskq], m5 - sub hd, 2 - jg .w32_loop - RET -.w64_loop: - add tmp1q, 32*4 - add tmp2q, 32*4 - W_MASK 0, 4, 0, 1 - add dstq, strideq - add maskq, 32 -.w64: - vpermq m0, m0, q3120 - mova [dstq+32*0], m0 - W_MASK 0, 5, 2, 3 - packuswb m4, m5 - psubb m5, m8, m4 - pavgb m5, m9 - vpermq m0, m0, q3120 - vpermd m5, m10, m5 - mova [dstq+32*1], m0 - mova [maskq], m5 - dec hd - jg .w64_loop - RET -.w128_loop: - add tmp1q, 32*8 - add tmp2q, 32*8 - W_MASK 0, 4, 0, 1 - add dstq, strideq - add maskq, 32*2 -.w128: - vpermq m0, m0, q3120 - mova [dstq+32*0], m0 - W_MASK 0, 5, 2, 3 - packuswb m4, m5 - psubb m5, m8, m4 - pavgb m5, m9 - vpermq m0, m0, q3120 - vpermd m5, m10, m5 - mova [dstq+32*1], m0 - mova [maskq+32*0], m5 - W_MASK 0, 4, 4, 5 - vpermq m0, m0, q3120 - mova [dstq+32*2], m0 - W_MASK 0, 5, 6, 7 - packuswb m4, m5 - psubb m5, m8, m4 - pavgb m5, m9 - vpermq m0, m0, q3120 - vpermd m5, m10, m5 - mova [dstq+32*3], m0 - mova [maskq+32*1], m5 - dec hd - jg .w128_loop - RET - -cglobal w_mask_444, 4, 8, 8, dst, stride, tmp1, tmp2, w, h, mask, stride3 -%define base r7-w_mask_444_avx2_table - lea r7, [w_mask_444_avx2_table] - tzcnt wd, wm - movifnidn hd, hm - mov maskq, maskmp - movsxd wq, dword [r7+wq*4] - vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 - vpbroadcastd m5, [base+pb_64] - vpbroadcastd m7, [base+pw_2048] - add wq, r7 - W_MASK 0, 4, 0, 1, 1 - lea stride3q, [strideq*3] - jmp wq -.w4: - vextracti128 xm1, m0, 1 - movd [dstq+strideq*0], xm0 - pextrd [dstq+strideq*1], xm0, 1 - movd [dstq+strideq*2], xm1 - pextrd [dstq+stride3q ], xm1, 1 - mova [maskq+32*0], m4 - cmp hd, 8 - jl .w4_end - lea dstq, [dstq+strideq*4] - pextrd [dstq+strideq*0], xm0, 2 - pextrd [dstq+strideq*1], xm0, 3 - pextrd [dstq+strideq*2], xm1, 2 - pextrd [dstq+stride3q ], xm1, 3 - je .w4_end - W_MASK 0, 4, 2, 3, 1 - lea dstq, [dstq+strideq*4] - vextracti128 xm1, m0, 1 - movd [dstq+strideq*0], xm0 - pextrd [dstq+strideq*1], xm0, 1 - movd [dstq+strideq*2], xm1 - pextrd [dstq+stride3q ], xm1, 1 - lea dstq, [dstq+strideq*4] - pextrd [dstq+strideq*0], xm0, 2 - pextrd [dstq+strideq*1], xm0, 3 - pextrd [dstq+strideq*2], xm1, 2 - pextrd [dstq+stride3q ], xm1, 3 - mova [maskq+32*1], m4 -.w4_end: - RET -.w8_loop: - add tmp1q, 32*2 - add tmp2q, 32*2 - W_MASK 0, 4, 0, 1, 1 - lea dstq, [dstq+strideq*4] - add maskq, 32 -.w8: - vextracti128 xm1, m0, 1 - movq [dstq+strideq*0], xm0 - movq [dstq+strideq*1], xm1 - movhps [dstq+strideq*2], xm0 - movhps [dstq+stride3q ], xm1 - mova [maskq], m4 - sub hd, 4 - jg .w8_loop - RET -.w16_loop: - add tmp1q, 32*2 - add tmp2q, 32*2 - W_MASK 0, 4, 0, 1, 1 - lea dstq, [dstq+strideq*2] - add maskq, 32 -.w16: - vpermq m0, m0, q3120 - mova [dstq+strideq*0], xm0 - vextracti128 [dstq+strideq*1], m0, 1 - mova [maskq], m4 - sub hd, 2 - jg .w16_loop - RET -.w32_loop: - add tmp1q, 32*2 - add tmp2q, 32*2 - W_MASK 0, 4, 0, 1, 1 - add dstq, strideq - add maskq, 32 -.w32: - vpermq m0, m0, q3120 - mova [dstq], m0 - mova [maskq], m4 - dec hd - jg .w32_loop - RET -.w64_loop: - add tmp1q, 32*4 - add tmp2q, 32*4 - W_MASK 0, 4, 0, 1, 1 - add dstq, strideq - add maskq, 32*2 -.w64: - vpermq m0, m0, q3120 - mova [dstq+32*0], m0 - mova [maskq+32*0], m4 - W_MASK 0, 4, 2, 3, 1 - vpermq m0, m0, q3120 - mova [dstq+32*1], m0 - mova [maskq+32*1], m4 - dec hd - jg .w64_loop - RET -.w128_loop: - add tmp1q, 32*8 - add tmp2q, 32*8 - W_MASK 0, 4, 0, 1, 1 - add dstq, strideq - add maskq, 32*4 -.w128: - vpermq m0, m0, q3120 - mova [dstq+32*0], m0 - mova [maskq+32*0], m4 - W_MASK 0, 4, 2, 3, 1 - vpermq m0, m0, q3120 - mova [dstq+32*1], m0 - mova [maskq+32*1], m4 - W_MASK 0, 4, 4, 5, 1 - vpermq m0, m0, q3120 - mova [dstq+32*2], m0 - mova [maskq+32*2], m4 - W_MASK 0, 4, 6, 7, 1 - vpermq m0, m0, q3120 - mova [dstq+32*3], m0 - mova [maskq+32*3], m4 - dec hd - jg .w128_loop - RET - -INIT_ZMM avx512icl -PREP_BILIN -PREP_8TAP -AVG_FN -W_AVG_FN -MASK_FN - -cglobal w_mask_420, 4, 8, 16, dst, stride, tmp1, tmp2, w, h, mask, stride3 -%define base r7-w_mask_420_avx512icl_table - lea r7, [w_mask_420_avx512icl_table] - tzcnt wd, wm - mov r6d, r7m ; sign - movifnidn hd, hm - movsxd wq, [r7+wq*4] - vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 - vpbroadcastd m7, [base+pw_2048] - vpbroadcastd m9, [base+pb_m64] ; -1 << 6 - mova ym10, [base+wm_420_mask+32] - vpbroadcastd m8, [base+wm_sign_avx512+r6*8] ; (258 - sign) << 6 - add wq, r7 - mov maskq, maskmp - lea stride3q, [strideq*3] - jmp wq -.w4: - mova m5, [wm_420_perm4] - cmp hd, 8 - jg .w4_h16 - WRAP_YMM W_MASK 0, 4, 0, 1 - vinserti128 ym5, [wm_420_perm4+32], 1 - vpermb ym4, ym5, ym4 - vpdpbusd ym8, ym4, ym9 - vextracti128 xmm1, m0, 1 - movd [dstq+strideq*0], xm0 - pextrd [dstq+strideq*1], xm0, 1 - movd [dstq+strideq*2], xmm1 - pextrd [dstq+stride3q ], xmm1, 1 - jl .w4_end - lea dstq, [dstq+strideq*4] - pextrd [dstq+strideq*0], xm0, 2 - pextrd [dstq+strideq*1], xm0, 3 - pextrd [dstq+strideq*2], xmm1, 2 - pextrd [dstq+stride3q ], xmm1, 3 -.w4_end: - vpermb ym8, ym10, ym8 - movq [maskq], xm8 - RET -.w4_h16: - vpbroadcastd m11, strided - pmulld m11, [bidir_sctr_w4] - W_MASK 0, 4, 0, 1 - vpermb m4, m5, m4 - vpdpbusd m8, m4, m9 - kxnorw k1, k1, k1 - vpermb m8, m10, m8 - mova [maskq], xm8 - vpscatterdd [dstq+m11]{k1}, m0 - RET -.w8: - mova m5, [wm_420_perm8] - cmp hd, 4 - jne .w8_h8 - WRAP_YMM W_MASK 0, 4, 0, 1 - vinserti128 ym5, [wm_420_perm8+32], 1 - vpermb ym4, ym5, ym4 - vpdpbusd ym8, ym4, ym9 - vpermb m8, m10, m8 - mova [maskq], xm8 - vextracti128 xmm1, ym0, 1 - movq [dstq+strideq*0], xm0 - movq [dstq+strideq*1], xmm1 - movhps [dstq+strideq*2], xm0 - movhps [dstq+stride3q ], xmm1 - RET -.w8_loop: - add tmp1q, 128 - add tmp2q, 128 - add maskq, 16 - lea dstq, [dstq+strideq*4] -.w8_h8: - W_MASK 0, 4, 0, 1 - vpermb m4, m5, m4 - mova m1, m8 - vpdpbusd m1, m4, m9 - vpermb m1, m10, m1 - mova [maskq], xm1 - vextracti32x4 xmm1, ym0, 1 - vextracti32x4 xmm2, m0, 2 - vextracti32x4 xmm3, m0, 3 - movq [dstq+strideq*0], xm0 - movq [dstq+strideq*1], xmm1 - movq [dstq+strideq*2], xmm2 - movq [dstq+stride3q ], xmm3 - lea dstq, [dstq+strideq*4] - movhps [dstq+strideq*0], xm0 - movhps [dstq+strideq*1], xmm1 - movhps [dstq+strideq*2], xmm2 - movhps [dstq+stride3q ], xmm3 - sub hd, 8 - jg .w8_loop - RET -.w16: - mova m5, [wm_420_perm16] -.w16_loop: - W_MASK 0, 4, 0, 1 - vpermb m4, m5, m4 - mova m1, m8 - vpdpbusd m1, m4, m9 - add tmp1q, 128 - add tmp2q, 128 - vpermb m1, m10, m1 - vpermq m0, m0, q3120 - mova [maskq], xm1 - add maskq, 16 - mova [dstq+strideq*0], xm0 - vextracti32x4 [dstq+strideq*1], m0, 2 - vextracti32x4 [dstq+strideq*2], ym0, 1 - vextracti32x4 [dstq+stride3q ], m0, 3 - lea dstq, [dstq+strideq*4] - sub hd, 4 - jg .w16_loop - RET -.w32: - pmovzxbq m5, [warp_8x8_shufA] -.w32_loop: - W_MASK 0, 4, 0, 1 - mova m1, m8 - vpdpbusd m1, m4, m9 - add tmp1q, 128 - add tmp2q, 128 - vpermb m1, m10, m1 - vpermq m0, m5, m0 - mova [maskq], xm1 - add maskq, 16 - mova [dstq+strideq*0], ym0 - vextracti32x8 [dstq+strideq*1], m0, 1 - lea dstq, [dstq+strideq*2] - sub hd, 2 - jg .w32_loop - RET -.w64: - pmovzxbq m12, [wm_420_perm64] ; 0, 2, 4, 6, 8, 10, 12, 14 - psrlq m13, m12, 4 ; 1, 3, 5, 7, 9, 11, 13, 15 -.w64_loop: - W_MASK 0, 4, 0, 2 - W_MASK 11, 5, 1, 3 - mova m2, m8 - vpdpbusd m2, m4, m9 - mova m3, m8 - vpdpbusd m3, m5, m9 - add tmp1q, 256 - add tmp2q, 256 - vpermt2b m2, m10, m3 - mova m1, m0 - vpermt2q m0, m12, m11 - vpermt2q m1, m13, m11 - mova [maskq], ym2 - add maskq, 32 - mova [dstq+strideq*0], m0 - mova [dstq+strideq*1], m1 - lea dstq, [dstq+strideq*2] - sub hd, 2 - jg .w64_loop - RET -.w128: - pmovzxbq m14, [wm_420_perm64] - mova m10, [wm_420_mask] - psrlq m15, m14, 4 -.w128_loop: - W_MASK 0, 12, 0, 4 - W_MASK 11, 13, 1, 5 - mova m4, m8 - vpdpbusd m4, m12, m9 - mova m5, m8 - vpdpbusd m5, m13, m9 - mova m1, m0 - vpermt2q m0, m14, m11 - vpermt2q m1, m15, m11 - mova [dstq+strideq*0+64*0], m0 - mova [dstq+strideq*1+64*0], m1 - W_MASK 0, 12, 2, 6 - W_MASK 11, 13, 3, 7 - vprold m4, 16 - vprold m5, 16 - vpdpbusd m4, m12, m9 - vpdpbusd m5, m13, m9 - add tmp1q, 512 - add tmp2q, 512 - vpermt2b m4, m10, m5 - mova m1, m0 - vpermt2q m0, m14, m11 - vpermt2q m1, m15, m11 - mova [maskq], m4 - add maskq, 64 - mova [dstq+strideq*0+64*1], m0 - mova [dstq+strideq*1+64*1], m1 - lea dstq, [dstq+strideq*2] - sub hd, 2 - jg .w128_loop - RET - -cglobal w_mask_422, 4, 8, 14, dst, stride, tmp1, tmp2, w, h, mask, stride3 -%define base r7-w_mask_422_avx512icl_table - lea r7, [w_mask_422_avx512icl_table] - tzcnt wd, wm - mov r6d, r7m ; sign - movifnidn hd, hm - movsxd wq, dword [r7+wq*4] - vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 - vpbroadcastd m7, [base+pw_2048] - vpbroadcastd m9, [base+pw_m128] - mova m10, [base+wm_422_mask] - vpbroadcastd m11, [base+pb_127] - add wq, r7 - vpbroadcastd m8, [base+wm_sign_avx512+4+r6*4] - mov maskq, maskmp - lea stride3q, [strideq*3] - jmp wq -.w4: - cmp hd, 8 - jg .w4_h16 - WRAP_YMM W_MASK 0, 4, 0, 1 - movhps xm10, [wm_422_mask+16] - vpdpwssd ym8, ym4, ym9 - vpermb ym8, ym10, ym8 - vextracti128 xmm1, m0, 1 - movd [dstq+strideq*0], xm0 - pextrd [dstq+strideq*1], xm0, 1 - movd [dstq+strideq*2], xmm1 - pextrd [dstq+stride3q ], xmm1, 1 - jl .w4_end - lea dstq, [dstq+strideq*4] - pextrd [dstq+strideq*0], xm0, 2 - pextrd [dstq+strideq*1], xm0, 3 - pextrd [dstq+strideq*2], xmm1, 2 - pextrd [dstq+stride3q ], xmm1, 3 -.w4_end: - pand xm8, xm11 - mova [maskq], xm8 - RET -.w4_h16: - vpbroadcastd m5, strided - pmulld m5, [bidir_sctr_w4] - W_MASK 0, 4, 0, 1 - vpdpwssd m8, m4, m9 - kxnorw k1, k1, k1 - vpermb m8, m10, m8 - pand ym8, ym11 - mova [maskq], ym8 - vpscatterdd [dstq+m5]{k1}, m0 - RET -.w8: - cmp hd, 4 - jne .w8_h8 - WRAP_YMM W_MASK 0, 4, 0, 1 - movhps xm10, [wm_422_mask+16] - vpdpwssd ym8, ym4, ym9 - vpermb ym8, ym10, ym8 - pand xm8, xm11 - mova [maskq], xm8 - vextracti128 xmm1, ym0, 1 - movq [dstq+strideq*0], xm0 - movq [dstq+strideq*1], xmm1 - movhps [dstq+strideq*2], xm0 - movhps [dstq+stride3q ], xmm1 - RET -.w8_loop: - add tmp1q, 128 - add tmp2q, 128 - add maskq, 32 - lea dstq, [dstq+strideq*4] -.w8_h8: - W_MASK 0, 4, 0, 1 - mova m1, m8 - vpdpwssd m1, m4, m9 - vpermb m1, m10, m1 - pand ym1, ym11 - mova [maskq], ym1 - vextracti32x4 xmm1, ym0, 1 - vextracti32x4 xmm2, m0, 2 - vextracti32x4 xmm3, m0, 3 - movq [dstq+strideq*0], xm0 - movq [dstq+strideq*1], xmm1 - movq [dstq+strideq*2], xmm2 - movq [dstq+stride3q ], xmm3 - lea dstq, [dstq+strideq*4] - movhps [dstq+strideq*0], xm0 - movhps [dstq+strideq*1], xmm1 - movhps [dstq+strideq*2], xmm2 - movhps [dstq+stride3q ], xmm3 - sub hd, 8 - jg .w8_loop - RET -.w16_loop: - add tmp1q, 128 - add tmp2q, 128 - add maskq, 32 - lea dstq, [dstq+strideq*4] -.w16: - W_MASK 0, 4, 0, 1 - mova m1, m8 - vpdpwssd m1, m4, m9 - vpermb m1, m10, m1 - vpermq m0, m0, q3120 - pand ym1, ym11 - mova [maskq], ym1 - mova [dstq+strideq*0], xm0 - vextracti32x4 [dstq+strideq*1], m0, 2 - vextracti32x4 [dstq+strideq*2], ym0, 1 - vextracti32x4 [dstq+stride3q ], m0, 3 - sub hd, 4 - jg .w16_loop - RET -.w32: - pmovzxbq m5, [warp_8x8_shufA] -.w32_loop: - W_MASK 0, 4, 0, 1 - mova m1, m8 - vpdpwssd m1, m4, m9 - add tmp1q, 128 - add tmp2q, 128 - vpermb m1, m10, m1 - vpermq m0, m5, m0 - pand ym1, ym11 - mova [maskq], ym1 - add maskq, 32 - mova [dstq+strideq*0], ym0 - vextracti32x8 [dstq+strideq*1], m0, 1 - lea dstq, [dstq+strideq*2] - sub hd, 2 - jg .w32_loop - RET -.w64: - pmovzxbq m5, [warp_8x8_shufA] -.w64_loop: - W_MASK 0, 4, 0, 1 - mova m1, m8 - vpdpwssd m1, m4, m9 - add tmp1q, 128 - add tmp2q, 128 - vpermb m1, m10, m1 - vpermq m0, m5, m0 - pand ym1, ym11 - mova [maskq], ym1 - add maskq, 32 - mova [dstq], m0 - add dstq, strideq - dec hd - jg .w64_loop - RET -.w128: - pmovzxbq m13, [warp_8x8_shufA] -.w128_loop: - W_MASK 0, 4, 0, 1 - W_MASK 12, 5, 2, 3 - mova m2, m8 - vpdpwssd m2, m4, m9 - mova m3, m8 - vpdpwssd m3, m5, m9 - add tmp1q, 256 - add tmp2q, 256 - vpermt2b m2, m10, m3 - vpermq m0, m13, m0 - vpermq m1, m13, m12 - pand m2, m11 - mova [maskq], m2 - add maskq, 64 - mova [dstq+64*0], m0 - mova [dstq+64*1], m1 - add dstq, strideq - dec hd - jg .w128_loop - RET - -cglobal w_mask_444, 4, 8, 12, dst, stride, tmp1, tmp2, w, h, mask, stride3 -%define base r7-w_mask_444_avx512icl_table - lea r7, [w_mask_444_avx512icl_table] - tzcnt wd, wm - movifnidn hd, hm - movsxd wq, dword [r7+wq*4] - vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 - vpbroadcastd m5, [base+pb_64] - vpbroadcastd m7, [base+pw_2048] - mova m8, [base+wm_444_mask] - add wq, r7 - mov maskq, maskmp - lea stride3q, [strideq*3] - jmp wq -.w4: - cmp hd, 8 - jg .w4_h16 - WRAP_YMM W_MASK 0, 4, 0, 1, 1 - vinserti128 ym8, [wm_444_mask+32], 1 - vpermb ym4, ym8, ym4 - mova [maskq], ym4 - vextracti128 xmm1, m0, 1 - movd [dstq+strideq*0], xm0 - pextrd [dstq+strideq*1], xm0, 1 - movd [dstq+strideq*2], xmm1 - pextrd [dstq+stride3q ], xmm1, 1 - jl .w4_end - lea dstq, [dstq+strideq*4] - pextrd [dstq+strideq*0], xm0, 2 - pextrd [dstq+strideq*1], xm0, 3 - pextrd [dstq+strideq*2], xmm1, 2 - pextrd [dstq+stride3q ], xmm1, 3 -.w4_end: - RET -.w4_h16: - vpbroadcastd m9, strided - pmulld m9, [bidir_sctr_w4] - W_MASK 0, 4, 0, 1, 1 - vpermb m4, m8, m4 - kxnorw k1, k1, k1 - mova [maskq], m4 - vpscatterdd [dstq+m9]{k1}, m0 - RET -.w8: - cmp hd, 4 - jne .w8_h8 - WRAP_YMM W_MASK 0, 4, 0, 1, 1 - vinserti128 ym8, [wm_444_mask+32], 1 - vpermb ym4, ym8, ym4 - mova [maskq], ym4 - vextracti128 xmm1, ym0, 1 - movq [dstq+strideq*0], xm0 - movq [dstq+strideq*1], xmm1 - movhps [dstq+strideq*2], xm0 - movhps [dstq+stride3q ], xmm1 - RET -.w8_loop: - add tmp1q, 128 - add tmp2q, 128 - add maskq, 64 - lea dstq, [dstq+strideq*4] -.w8_h8: - W_MASK 0, 4, 0, 1, 1 - vpermb m4, m8, m4 - mova [maskq], m4 - vextracti32x4 xmm1, ym0, 1 - vextracti32x4 xmm2, m0, 2 - vextracti32x4 xmm3, m0, 3 - movq [dstq+strideq*0], xm0 - movq [dstq+strideq*1], xmm1 - movq [dstq+strideq*2], xmm2 - movq [dstq+stride3q ], xmm3 - lea dstq, [dstq+strideq*4] - movhps [dstq+strideq*0], xm0 - movhps [dstq+strideq*1], xmm1 - movhps [dstq+strideq*2], xmm2 - movhps [dstq+stride3q ], xmm3 - sub hd, 8 - jg .w8_loop - RET -.w16_loop: - add tmp1q, 128 - add tmp2q, 128 - add maskq, 64 - lea dstq, [dstq+strideq*4] -.w16: - W_MASK 0, 4, 0, 1, 1 - vpermb m4, m8, m4 - vpermq m0, m0, q3120 - mova [maskq], m4 - mova [dstq+strideq*0], xm0 - vextracti32x4 [dstq+strideq*1], m0, 2 - vextracti32x4 [dstq+strideq*2], ym0, 1 - vextracti32x4 [dstq+stride3q ], m0, 3 - sub hd, 4 - jg .w16_loop - RET -.w32: - pmovzxbq m9, [warp_8x8_shufA] -.w32_loop: - W_MASK 0, 4, 0, 1, 1 - vpermb m4, m8, m4 - add tmp1q, 128 - add tmp2q, 128 - vpermq m0, m9, m0 - mova [maskq], m4 - add maskq, 64 - mova [dstq+strideq*0], ym0 - vextracti32x8 [dstq+strideq*1], m0, 1 - lea dstq, [dstq+strideq*2] - sub hd, 2 - jg .w32_loop - RET -.w64: - pmovzxbq m9, [warp_8x8_shufA] -.w64_loop: - W_MASK 0, 4, 0, 1, 1 - vpermb m4, m8, m4 - add tmp1q, 128 - add tmp2q, 128 - vpermq m0, m9, m0 - mova [maskq], m4 - add maskq, 64 - mova [dstq], m0 - add dstq, strideq - dec hd - jg .w64_loop - RET -.w128: - pmovzxbq m11, [warp_8x8_shufA] -.w128_loop: - W_MASK 0, 4, 0, 1, 1 - W_MASK 10, 9, 2, 3, 1 - vpermb m4, m8, m4 - vpermb m9, m8, m9 - add tmp1q, 256 - add tmp2q, 256 - vpermq m0, m11, m0 - vpermq m10, m11, m10 - mova [maskq+64*0], m4 - mova [maskq+64*1], m9 - add maskq, 128 - mova [dstq+64*0], m0 - mova [dstq+64*1], m10 - add dstq, strideq - dec hd - jg .w128_loop - RET - -%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/mc_avx2.asm b/third_party/dav1d/src/x86/mc_avx2.asm new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/x86/mc_avx2.asm @@ -0,0 +1,6167 @@ +; Copyright © 2018-2020, VideoLAN and dav1d authors +; Copyright © 2018-2020, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "ext/x86/x86inc.asm" + +%if ARCH_X86_64 + +SECTION_RODATA 32 + +; dav1d_obmc_masks[] with 64-x interleaved +obmc_masks: db 0, 0, 0, 0 + ; 2 + db 45, 19, 64, 0 + ; 4 + db 39, 25, 50, 14, 59, 5, 64, 0 + ; 8 + db 36, 28, 42, 22, 48, 16, 53, 11, 57, 7, 61, 3, 64, 0, 64, 0 + ; 16 + db 34, 30, 37, 27, 40, 24, 43, 21, 46, 18, 49, 15, 52, 12, 54, 10 + db 56, 8, 58, 6, 60, 4, 61, 3, 64, 0, 64, 0, 64, 0, 64, 0 + ; 32 + db 33, 31, 35, 29, 36, 28, 38, 26, 40, 24, 41, 23, 43, 21, 44, 20 + db 45, 19, 47, 17, 48, 16, 50, 14, 51, 13, 52, 12, 53, 11, 55, 9 + db 56, 8, 57, 7, 58, 6, 59, 5, 60, 4, 60, 4, 61, 3, 62, 2 + db 64, 0, 64, 0, 64, 0, 64, 0, 64, 0, 64, 0, 64, 0, 64, 0 + +warp_8x8_shufA: db 0, 2, 4, 6, 1, 3, 5, 7, 1, 3, 5, 7, 2, 4, 6, 8 + db 4, 6, 8, 10, 5, 7, 9, 11, 5, 7, 9, 11, 6, 8, 10, 12 +warp_8x8_shufB: db 2, 4, 6, 8, 3, 5, 7, 9, 3, 5, 7, 9, 4, 6, 8, 10 + db 6, 8, 10, 12, 7, 9, 11, 13, 7, 9, 11, 13, 8, 10, 12, 14 +subpel_h_shuf4: db 0, 1, 2, 3, 1, 2, 3, 4, 8, 9, 10, 11, 9, 10, 11, 12 + db 2, 3, 4, 5, 3, 4, 5, 6, 10, 11, 12, 13, 11, 12, 13, 14 +subpel_h_shufA: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 +subpel_h_shufB: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 +subpel_h_shufC: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 +subpel_v_shuf4: db 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15 +subpel_s_shuf2: db 0, 1, 2, 3, 0, 1, 2, 3, 8, 9, 10, 11, 8, 9, 10, 11 +subpel_s_shuf8: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15 +bilin_h_shuf4: db 1, 0, 2, 1, 3, 2, 4, 3, 9, 8, 10, 9, 11, 10, 12, 11 +bilin_h_shuf8: db 1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7 +bilin_v_shuf4: db 4, 0, 5, 1, 6, 2, 7, 3, 8, 4, 9, 5, 10, 6, 11, 7 +deint_shuf4: db 0, 4, 1, 5, 2, 6, 3, 7, 4, 8, 5, 9, 6, 10, 7, 11 +blend_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3 +pb_8x0_8x8: db 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 8, 8, 8, 8 +bdct_lb_dw: db 0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 8, 12, 12, 12, 12 +wswap: db 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13 +rescale_mul: dd 0, 1, 2, 3, 4, 5, 6, 7 +resize_shuf: db 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 7, 7, 7, 7 + db 7, 7, 7, 7, 7, 7, 7, 7 + +wm_420_sign: dd 0x01020102, 0x01010101 +wm_422_sign: dd 0x80808080, 0x7f7f7f7f + +pb_64: times 4 db 64 +pw_m256: times 2 dw -256 +pw_32: times 2 dw 32 +pw_34: times 2 dw 34 +pw_258: times 2 dw 258 +pw_512: times 2 dw 512 +pw_1024: times 2 dw 1024 +pw_2048: times 2 dw 2048 +pw_6903: times 2 dw 6903 +pw_8192: times 2 dw 8192 +pd_32: dd 32 +pd_63: dd 63 +pd_512: dd 512 +pd_32768: dd 32768 +pd_0x3ff: dd 0x3ff +pd_0x4000: dd 0x4000 +pq_0x40000000: dq 0x40000000 + +cextern mc_subpel_filters +cextern mc_warp_filter +cextern resize_filter + +%define subpel_filters (mangle(private_prefix %+ _mc_subpel_filters)-8) + +%macro BASE_JMP_TABLE 3-* + %xdefine %1_%2_table (%%table - %3) + %xdefine %%base %1_%2 + %%table: + %rep %0 - 2 + dw %%base %+ _w%3 - %%base + %rotate 1 + %endrep +%endmacro + +%macro HV_JMP_TABLE 5-* + %xdefine %%prefix mangle(private_prefix %+ _%1_%2_%3) + %xdefine %%base %1_%3 + %assign %%types %4 + %if %%types & 1 + %xdefine %1_%2_h_%3_table (%%h - %5) + %%h: + %rep %0 - 4 + dw %%prefix %+ .h_w%5 - %%base + %rotate 1 + %endrep + %rotate 4 + %endif + %if %%types & 2 + %xdefine %1_%2_v_%3_table (%%v - %5) + %%v: + %rep %0 - 4 + dw %%prefix %+ .v_w%5 - %%base + %rotate 1 + %endrep + %rotate 4 + %endif + %if %%types & 4 + %xdefine %1_%2_hv_%3_table (%%hv - %5) + %%hv: + %rep %0 - 4 + dw %%prefix %+ .hv_w%5 - %%base + %rotate 1 + %endrep + %endif +%endmacro + +%macro BIDIR_JMP_TABLE 1-* + %xdefine %1_table (%%table - 2*%2) + %xdefine %%base %1_table + %xdefine %%prefix mangle(private_prefix %+ _%1) + %%table: + %rep %0 - 1 + dd %%prefix %+ .w%2 - %%base + %rotate 1 + %endrep +%endmacro + +%macro SCALED_JMP_TABLE 1-* + %xdefine %1_table (%%table - %2) + %xdefine %%base mangle(private_prefix %+ _%1) +%%table: + %rep %0 - 1 + dw %%base %+ .w%2 - %%base + %rotate 1 + %endrep + %rotate 1 +%%dy_1024: + %xdefine %1_dy1_table (%%dy_1024 - %2) + %rep %0 - 1 + dw %%base %+ .dy1_w%2 - %%base + %rotate 1 + %endrep + %rotate 1 +%%dy_2048: + %xdefine %1_dy2_table (%%dy_2048 - %2) + %rep %0 - 1 + dw %%base %+ .dy2_w%2 - %%base + %rotate 1 + %endrep +%endmacro + +%xdefine put_avx2 mangle(private_prefix %+ _put_bilin_avx2.put) +%xdefine prep_avx2 mangle(private_prefix %+ _prep_bilin_avx2.prep) + +%define table_offset(type, fn) type %+ fn %+ SUFFIX %+ _table - type %+ SUFFIX + +BASE_JMP_TABLE put, avx2, 2, 4, 8, 16, 32, 64, 128 +BASE_JMP_TABLE prep, avx2, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE put, bilin, avx2, 7, 2, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE prep, bilin, avx2, 7, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE put, 8tap, avx2, 3, 2, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE prep, 8tap, avx2, 1, 4, 8, 16, 32, 64, 128 +SCALED_JMP_TABLE put_8tap_scaled_avx2, 2, 4, 8, 16, 32, 64, 128 +SCALED_JMP_TABLE prep_8tap_scaled_avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE avg_avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_avg_avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE mask_avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_420_avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_422_avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_444_avx2, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE blend_avx2, 4, 8, 16, 32 +BIDIR_JMP_TABLE blend_v_avx2, 2, 4, 8, 16, 32 +BIDIR_JMP_TABLE blend_h_avx2, 2, 4, 8, 16, 32, 32, 32 + +SECTION .text + +INIT_XMM avx2 +DECLARE_REG_TMP 4, 6, 7 +cglobal put_bilin, 4, 8, 0, dst, ds, src, ss, w, h, mxy + movifnidn mxyd, r6m ; mx + lea t2, [put_avx2] + tzcnt wd, wm + movifnidn hd, hm + test mxyd, mxyd + jnz .h + mov mxyd, r7m ; my + test mxyd, mxyd + jnz .v +.put: + movzx wd, word [t2+wq*2+table_offset(put,)] + add wq, t2 + jmp wq +.put_w2: + movzx t0d, word [srcq+ssq*0] + movzx t1d, word [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mov [dstq+dsq*0], t0w + mov [dstq+dsq*1], t1w + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w2 + RET +.put_w4: + mov t0d, [srcq+ssq*0] + mov t1d, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mov [dstq+dsq*0], t0d + mov [dstq+dsq*1], t1d + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w4 + RET +.put_w8: + mov t0, [srcq+ssq*0] + mov t1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mov [dstq+dsq*0], t0 + mov [dstq+dsq*1], t1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w8 + RET +.put_w16: + movu m0, [srcq+ssq*0] + movu m1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mova [dstq+dsq*0], m0 + mova [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w16 + RET +INIT_YMM avx2 +.put_w32: + movu m0, [srcq+ssq*0] + movu m1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mova [dstq+dsq*0], m0 + mova [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w32 + RET +.put_w64: + movu m0, [srcq+ssq*0+32*0] + movu m1, [srcq+ssq*0+32*1] + movu m2, [srcq+ssq*1+32*0] + movu m3, [srcq+ssq*1+32*1] + lea srcq, [srcq+ssq*2] + mova [dstq+dsq*0+32*0], m0 + mova [dstq+dsq*0+32*1], m1 + mova [dstq+dsq*1+32*0], m2 + mova [dstq+dsq*1+32*1], m3 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w64 + RET +.put_w128: + movu m0, [srcq+32*0] + movu m1, [srcq+32*1] + movu m2, [srcq+32*2] + movu m3, [srcq+32*3] + add srcq, ssq + mova [dstq+32*0], m0 + mova [dstq+32*1], m1 + mova [dstq+32*2], m2 + mova [dstq+32*3], m3 + add dstq, dsq + dec hd + jg .put_w128 + RET +.h: + ; (16 * src[x] + (mx * (src[x + 1] - src[x])) + 8) >> 4 + ; = ((16 - mx) * src[x] + mx * src[x + 1] + 8) >> 4 + imul mxyd, 0xff01 + vbroadcasti128 m4, [bilin_h_shuf8] + add mxyd, 16 << 8 + movd xm5, mxyd + mov mxyd, r7m ; my + vpbroadcastw m5, xm5 + test mxyd, mxyd + jnz .hv + movzx wd, word [t2+wq*2+table_offset(put, _bilin_h)] + vpbroadcastd m3, [pw_2048] + add wq, t2 + jmp wq +.h_w2: + movd xm0, [srcq+ssq*0] + pinsrd xm0, [srcq+ssq*1], 1 + lea srcq, [srcq+ssq*2] + pshufb xm0, xm4 + pmaddubsw xm0, xm5 + pmulhrsw xm0, xm3 + packuswb xm0, xm0 + pextrw [dstq+dsq*0], xm0, 0 + pextrw [dstq+dsq*1], xm0, 2 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w2 + RET +.h_w4: + mova xm4, [bilin_h_shuf4] +.h_w4_loop: + movq xm0, [srcq+ssq*0] + movhps xm0, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pshufb xm0, xm4 + pmaddubsw xm0, xm5 + pmulhrsw xm0, xm3 + packuswb xm0, xm0 + movd [dstq+dsq*0], xm0 + pextrd [dstq+dsq*1], xm0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w4_loop + RET +.h_w8: + movu xm0, [srcq+ssq*0] + movu xm1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pshufb xm0, xm4 + pshufb xm1, xm4 + pmaddubsw xm0, xm5 + pmaddubsw xm1, xm5 + pmulhrsw xm0, xm3 + pmulhrsw xm1, xm3 + packuswb xm0, xm1 + movq [dstq+dsq*0], xm0 + movhps [dstq+dsq*1], xm0 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w8 + RET +.h_w16: + movu xm0, [srcq+ssq*0+8*0] + vinserti128 m0, [srcq+ssq*1+8*0], 1 + movu xm1, [srcq+ssq*0+8*1] + vinserti128 m1, [srcq+ssq*1+8*1], 1 + lea srcq, [srcq+ssq*2] + pshufb m0, m4 + pshufb m1, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmulhrsw m0, m3 + pmulhrsw m1, m3 + packuswb m0, m1 + mova [dstq+dsq*0], xm0 + vextracti128 [dstq+dsq*1], m0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w16 + RET +.h_w32: + movu m0, [srcq+8*0] + movu m1, [srcq+8*1] + add srcq, ssq + pshufb m0, m4 + pshufb m1, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmulhrsw m0, m3 + pmulhrsw m1, m3 + packuswb m0, m1 + mova [dstq], m0 + add dstq, dsq + dec hd + jg .h_w32 + RET +.h_w64: + movu m0, [srcq+8*0] + movu m1, [srcq+8*1] + pshufb m0, m4 + pshufb m1, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmulhrsw m0, m3 + pmulhrsw m1, m3 + packuswb m0, m1 + movu m1, [srcq+8*4] + movu m2, [srcq+8*5] + add srcq, ssq + pshufb m1, m4 + pshufb m2, m4 + pmaddubsw m1, m5 + pmaddubsw m2, m5 + pmulhrsw m1, m3 + pmulhrsw m2, m3 + packuswb m1, m2 + mova [dstq+32*0], m0 + mova [dstq+32*1], m1 + add dstq, dsq + dec hd + jg .h_w64 + RET +.h_w128: + mov t1, -32*3 +.h_w128_loop: + movu m0, [srcq+t1+32*3+8*0] + movu m1, [srcq+t1+32*3+8*1] + pshufb m0, m4 + pshufb m1, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmulhrsw m0, m3 + pmulhrsw m1, m3 + packuswb m0, m1 + mova [dstq+t1+32*3], m0 + add t1, 32 + jle .h_w128_loop + add srcq, ssq + add dstq, dsq + dec hd + jg .h_w128 + RET +.v: + movzx wd, word [t2+wq*2+table_offset(put, _bilin_v)] + imul mxyd, 0xff01 + vpbroadcastd m5, [pw_2048] + add mxyd, 16 << 8 + add wq, t2 + movd xm4, mxyd + vpbroadcastw m4, xm4 + jmp wq +.v_w2: + movd xm0, [srcq+ssq*0] +.v_w2_loop: + pinsrw xm1, xm0, [srcq+ssq*1], 1 ; 0 1 + lea srcq, [srcq+ssq*2] + pinsrw xm0, xm1, [srcq+ssq*0], 0 ; 2 1 + pshuflw xm1, xm1, q2301 ; 1 0 + punpcklbw xm1, xm0, xm1 + pmaddubsw xm1, xm4 + pmulhrsw xm1, xm5 + packuswb xm1, xm1 + pextrw [dstq+dsq*0], xm1, 1 + pextrw [dstq+dsq*1], xm1, 0 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w2_loop + RET +.v_w4: + movd xm0, [srcq+ssq*0] +.v_w4_loop: + vpbroadcastd xm1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vpblendd xm2, xm1, xm0, 0x01 ; 0 1 + vpbroadcastd xm0, [srcq+ssq*0] + vpblendd xm1, xm0, 0x02 ; 1 2 + punpcklbw xm1, xm2 + pmaddubsw xm1, xm4 + pmulhrsw xm1, xm5 + packuswb xm1, xm1 + movd [dstq+dsq*0], xm1 + pextrd [dstq+dsq*1], xm1, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w4_loop + RET +.v_w8: + movq xm0, [srcq+ssq*0] +.v_w8_loop: + movq xm3, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + punpcklbw xm1, xm3, xm0 + movq xm0, [srcq+ssq*0] + punpcklbw xm2, xm0, xm3 + pmaddubsw xm1, xm4 + pmaddubsw xm2, xm4 + pmulhrsw xm1, xm5 + pmulhrsw xm2, xm5 + packuswb xm1, xm2 + movq [dstq+dsq*0], xm1 + movhps [dstq+dsq*1], xm1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w8_loop + RET +.v_w16: + movu xm0, [srcq+ssq*0] +.v_w16_loop: + vbroadcasti128 m2, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vpblendd m3, m2, m0, 0x0f ; 0 1 + vbroadcasti128 m0, [srcq+ssq*0] + vpblendd m2, m0, 0xf0 ; 1 2 + punpcklbw m1, m2, m3 + punpckhbw m2, m3 + pmaddubsw m1, m4 + pmaddubsw m2, m4 + pmulhrsw m1, m5 + pmulhrsw m2, m5 + packuswb m1, m2 + mova [dstq+dsq*0], xm1 + vextracti128 [dstq+dsq*1], m1, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w16_loop + RET +.v_w32: +%macro PUT_BILIN_V_W32 0 + movu m0, [srcq+ssq*0] +%%loop: + movu m3, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + punpcklbw m1, m3, m0 + punpckhbw m2, m3, m0 + movu m0, [srcq+ssq*0] + pmaddubsw m1, m4 + pmaddubsw m2, m4 + pmulhrsw m1, m5 + pmulhrsw m2, m5 + packuswb m1, m2 + mova [dstq+dsq*0], m1 + punpcklbw m1, m0, m3 + punpckhbw m2, m0, m3 + pmaddubsw m1, m4 + pmaddubsw m2, m4 + pmulhrsw m1, m5 + pmulhrsw m2, m5 + packuswb m1, m2 + mova [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg %%loop +%endmacro + PUT_BILIN_V_W32 + RET +.v_w64: + movu m0, [srcq+32*0] + movu m1, [srcq+32*1] +.v_w64_loop: + add srcq, ssq + movu m3, [srcq+32*0] + punpcklbw m2, m3, m0 + punpckhbw m0, m3, m0 + pmaddubsw m2, m4 + pmaddubsw m0, m4 + pmulhrsw m2, m5 + pmulhrsw m0, m5 + packuswb m2, m0 + mova m0, m3 + movu m3, [srcq+32*1] + mova [dstq+32*0], m2 + punpcklbw m2, m3, m1 + punpckhbw m1, m3, m1 + pmaddubsw m2, m4 + pmaddubsw m1, m4 + pmulhrsw m2, m5 + pmulhrsw m1, m5 + packuswb m2, m1 + mova m1, m3 + mova [dstq+32*1], m2 + add dstq, dsq + dec hd + jg .v_w64_loop + RET +.v_w128: + mov t0, dstq + mov t1, srcq + lea t2d, [hq+(3<<8)] +.v_w128_loop: + PUT_BILIN_V_W32 + movzx hd, t2b + add t0, 32 + add t1, 32 + mov dstq, t0 + mov srcq, t1 + sub t2d, 1<<8 + jg .v_w128_loop + RET +.hv: + ; (16 * src[x] + (my * (src[x + src_stride] - src[x])) + 128) >> 8 + ; = (src[x] + ((my * (src[x + src_stride] - src[x])) >> 4) + 8) >> 4 + movzx wd, word [t2+wq*2+table_offset(put, _bilin_hv)] + WIN64_SPILL_XMM 8 + shl mxyd, 11 ; can't shift by 12 due to signed overflow + vpbroadcastd m7, [pw_2048] + movd xm6, mxyd + add wq, t2 + vpbroadcastw m6, xm6 + jmp wq +.hv_w2: + vpbroadcastd xm0, [srcq+ssq*0] + pshufb xm0, xm4 + pmaddubsw xm0, xm5 +.hv_w2_loop: + movd xm1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pinsrd xm1, [srcq+ssq*0], 1 + pshufb xm1, xm4 + pmaddubsw xm1, xm5 ; 1 _ 2 _ + shufps xm2, xm0, xm1, q1032 ; 0 _ 1 _ + mova xm0, xm1 + psubw xm1, xm2 + paddw xm1, xm1 + pmulhw xm1, xm6 + paddw xm1, xm2 + pmulhrsw xm1, xm7 + packuswb xm1, xm1 + pextrw [dstq+dsq*0], xm1, 0 + pextrw [dstq+dsq*1], xm1, 2 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w2_loop + RET +.hv_w4: + mova xm4, [bilin_h_shuf4] + movddup xm0, [srcq+ssq*0] + pshufb xm0, xm4 + pmaddubsw xm0, xm5 +.hv_w4_loop: + movq xm1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + movhps xm1, [srcq+ssq*0] + pshufb xm1, xm4 + pmaddubsw xm1, xm5 ; 1 2 + shufps xm2, xm0, xm1, q1032 ; 0 1 + mova xm0, xm1 + psubw xm1, xm2 + paddw xm1, xm1 + pmulhw xm1, xm6 + paddw xm1, xm2 + pmulhrsw xm1, xm7 + packuswb xm1, xm1 + movd [dstq+dsq*0], xm1 + pextrd [dstq+dsq*1], xm1, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w4_loop + RET +.hv_w8: + vbroadcasti128 m0, [srcq+ssq*0] + pshufb m0, m4 + pmaddubsw m0, m5 +.hv_w8_loop: + movu xm1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vinserti128 m1, [srcq+ssq*0], 1 + pshufb m1, m4 + pmaddubsw m1, m5 ; 1 2 + vperm2i128 m2, m0, m1, 0x21 ; 0 1 + mova m0, m1 + psubw m1, m2 + paddw m1, m1 + pmulhw m1, m6 + paddw m1, m2 + pmulhrsw m1, m7 + vextracti128 xm2, m1, 1 + packuswb xm1, xm2 + movq [dstq+dsq*0], xm1 + movhps [dstq+dsq*1], xm1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w8_loop + RET +.hv_w16: + movu m0, [srcq+ssq*0+8*0] + vinserti128 m0, [srcq+ssq*0+8*1], 1 + pshufb m0, m4 + pmaddubsw m0, m5 +.hv_w16_loop: + movu xm2, [srcq+ssq*1+8*0] + vinserti128 m2, [srcq+ssq*1+8*1], 1 + lea srcq, [srcq+ssq*2] + movu xm3, [srcq+ssq*0+8*0] + vinserti128 m3, [srcq+ssq*0+8*1], 1 + pshufb m2, m4 + pshufb m3, m4 + pmaddubsw m2, m5 + psubw m1, m2, m0 + paddw m1, m1 + pmulhw m1, m6 + paddw m1, m0 + pmaddubsw m0, m3, m5 + psubw m3, m0, m2 + paddw m3, m3 + pmulhw m3, m6 + paddw m3, m2 + pmulhrsw m1, m7 + pmulhrsw m3, m7 + packuswb m1, m3 + vpermq m1, m1, q3120 + mova [dstq+dsq*0], xm1 + vextracti128 [dstq+dsq*1], m1, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w16_loop + RET +.hv_w32: + xor t2d, t2d +.hv_w32gt: + mov t0, dstq + mov t1, srcq +%if WIN64 + movaps r4m, xmm8 +%endif +.hv_w32_loop0: + movu m0, [srcq+8*0] + vinserti128 m0, [srcq+8*2], 1 + movu m1, [srcq+8*1] + vinserti128 m1, [srcq+8*3], 1 + pshufb m0, m4 + pshufb m1, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 +.hv_w32_loop: + add srcq, ssq + movu xm2, [srcq+8*1] + vinserti128 m2, [srcq+8*3], 1 + pshufb m2, m4 + pmaddubsw m2, m5 + psubw m3, m2, m1 + paddw m3, m3 + pmulhw m3, m6 + paddw m3, m1 + mova m1, m2 + pmulhrsw m8, m3, m7 + movu xm2, [srcq+8*0] + vinserti128 m2, [srcq+8*2], 1 + pshufb m2, m4 + pmaddubsw m2, m5 + psubw m3, m2, m0 + paddw m3, m3 + pmulhw m3, m6 + paddw m3, m0 + mova m0, m2 + pmulhrsw m3, m7 + packuswb m3, m8 + mova [dstq], m3 + add dstq, dsq + dec hd + jg .hv_w32_loop + movzx hd, t2b + add t0, 32 + add t1, 32 + mov dstq, t0 + mov srcq, t1 + sub t2d, 1<<8 + jg .hv_w32_loop0 +%if WIN64 + movaps xmm8, r4m +%endif + RET +.hv_w64: + lea t2d, [hq+(1<<8)] + jmp .hv_w32gt +.hv_w128: + lea t2d, [hq+(3<<8)] + jmp .hv_w32gt + +DECLARE_REG_TMP 3, 5, 6 +cglobal prep_bilin, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3 + movifnidn mxyd, r5m ; mx + lea t2, [prep%+SUFFIX] + tzcnt wd, wm + movifnidn hd, hm + test mxyd, mxyd + jnz .h + mov mxyd, r6m ; my + test mxyd, mxyd + jnz .v +.prep: + movzx wd, word [t2+wq*2+table_offset(prep,)] + add wq, t2 + lea stride3q, [strideq*3] + jmp wq +.prep_w4: + movd xm0, [srcq+strideq*0] + pinsrd xm0, [srcq+strideq*1], 1 + pinsrd xm0, [srcq+strideq*2], 2 + pinsrd xm0, [srcq+stride3q ], 3 + lea srcq, [srcq+strideq*4] + pmovzxbw m0, xm0 + psllw m0, 4 + mova [tmpq], m0 + add tmpq, 32 + sub hd, 4 + jg .prep_w4 + RET +.prep_w8: + movq xm0, [srcq+strideq*0] + movhps xm0, [srcq+strideq*1] + movq xm1, [srcq+strideq*2] + movhps xm1, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + pmovzxbw m0, xm0 + pmovzxbw m1, xm1 + psllw m0, 4 + psllw m1, 4 + mova [tmpq+32*0], m0 + mova [tmpq+32*1], m1 + add tmpq, 32*2 + sub hd, 4 + jg .prep_w8 + RET +.prep_w16: + pmovzxbw m0, [srcq+strideq*0] + pmovzxbw m1, [srcq+strideq*1] + pmovzxbw m2, [srcq+strideq*2] + pmovzxbw m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + psllw m0, 4 + psllw m1, 4 + psllw m2, 4 + psllw m3, 4 + mova [tmpq+32*0], m0 + mova [tmpq+32*1], m1 + mova [tmpq+32*2], m2 + mova [tmpq+32*3], m3 + add tmpq, 32*4 + sub hd, 4 + jg .prep_w16 + RET +.prep_w32: + pmovzxbw m0, [srcq+strideq*0+16*0] + pmovzxbw m1, [srcq+strideq*0+16*1] + pmovzxbw m2, [srcq+strideq*1+16*0] + pmovzxbw m3, [srcq+strideq*1+16*1] + lea srcq, [srcq+strideq*2] + psllw m0, 4 + psllw m1, 4 + psllw m2, 4 + psllw m3, 4 + mova [tmpq+32*0], m0 + mova [tmpq+32*1], m1 + mova [tmpq+32*2], m2 + mova [tmpq+32*3], m3 + add tmpq, 32*4 + sub hd, 2 + jg .prep_w32 + RET +.prep_w64: + pmovzxbw m0, [srcq+16*0] + pmovzxbw m1, [srcq+16*1] + pmovzxbw m2, [srcq+16*2] + pmovzxbw m3, [srcq+16*3] + add srcq, strideq + psllw m0, 4 + psllw m1, 4 + psllw m2, 4 + psllw m3, 4 + mova [tmpq+32*0], m0 + mova [tmpq+32*1], m1 + mova [tmpq+32*2], m2 + mova [tmpq+32*3], m3 + add tmpq, 32*4 + dec hd + jg .prep_w64 + RET +.prep_w128: + pmovzxbw m0, [srcq+16*0] + pmovzxbw m1, [srcq+16*1] + pmovzxbw m2, [srcq+16*2] + pmovzxbw m3, [srcq+16*3] + psllw m0, 4 + psllw m1, 4 + psllw m2, 4 + psllw m3, 4 + mova [tmpq+32*0], m0 + mova [tmpq+32*1], m1 + mova [tmpq+32*2], m2 + mova [tmpq+32*3], m3 + pmovzxbw m0, [srcq+16*4] + pmovzxbw m1, [srcq+16*5] + pmovzxbw m2, [srcq+16*6] + pmovzxbw m3, [srcq+16*7] + add tmpq, 32*8 + add srcq, strideq + psllw m0, 4 + psllw m1, 4 + psllw m2, 4 + psllw m3, 4 + mova [tmpq-32*4], m0 + mova [tmpq-32*3], m1 + mova [tmpq-32*2], m2 + mova [tmpq-32*1], m3 + dec hd + jg .prep_w128 + RET +.h: + ; 16 * src[x] + (mx * (src[x + 1] - src[x])) + ; = (16 - mx) * src[x] + mx * src[x + 1] + imul mxyd, 0xff01 + vbroadcasti128 m4, [bilin_h_shuf8] + add mxyd, 16 << 8 + movd xm5, mxyd + mov mxyd, r6m ; my + vpbroadcastw m5, xm5 + test mxyd, mxyd + jnz .hv + movzx wd, word [t2+wq*2+table_offset(prep, _bilin_h)] + add wq, t2 + lea stride3q, [strideq*3] + jmp wq +.h_w4: + vbroadcasti128 m4, [bilin_h_shuf4] +.h_w4_loop: + movq xm0, [srcq+strideq*0] + movhps xm0, [srcq+strideq*1] + movq xm1, [srcq+strideq*2] + movhps xm1, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vinserti128 m0, xm1, 1 + pshufb m0, m4 + pmaddubsw m0, m5 + mova [tmpq], m0 + add tmpq, 32 + sub hd, 4 + jg .h_w4_loop + RET +.h_w8: +.h_w8_loop: + movu xm0, [srcq+strideq*0] + vinserti128 m0, [srcq+strideq*1], 1 + movu xm1, [srcq+strideq*2] + vinserti128 m1, [srcq+stride3q ], 1 + lea srcq, [srcq+strideq*4] + pshufb m0, m4 + pshufb m1, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + mova [tmpq+32*0], m0 + mova [tmpq+32*1], m1 + add tmpq, 32*2 + sub hd, 4 + jg .h_w8_loop + RET +.h_w16: +.h_w16_loop: + movu xm0, [srcq+strideq*0+8*0] + vinserti128 m0, [srcq+strideq*0+8*1], 1 + movu xm1, [srcq+strideq*1+8*0] + vinserti128 m1, [srcq+strideq*1+8*1], 1 + movu xm2, [srcq+strideq*2+8*0] + vinserti128 m2, [srcq+strideq*2+8*1], 1 + movu xm3, [srcq+stride3q +8*0] + vinserti128 m3, [srcq+stride3q +8*1], 1 + lea srcq, [srcq+strideq*4] + pshufb m0, m4 + pshufb m1, m4 + pshufb m2, m4 + pshufb m3, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmaddubsw m2, m5 + pmaddubsw m3, m5 + mova [tmpq+32*0], m0 + mova [tmpq+32*1], m1 + mova [tmpq+32*2], m2 + mova [tmpq+32*3], m3 + add tmpq, 32*4 + sub hd, 4 + jg .h_w16_loop + RET +.h_w32: +.h_w32_loop: + movu xm0, [srcq+strideq*0+8*0] + vinserti128 m0, [srcq+strideq*0+8*1], 1 + movu xm1, [srcq+strideq*0+8*2] + vinserti128 m1, [srcq+strideq*0+8*3], 1 + movu xm2, [srcq+strideq*1+8*0] + vinserti128 m2, [srcq+strideq*1+8*1], 1 + movu xm3, [srcq+strideq*1+8*2] + vinserti128 m3, [srcq+strideq*1+8*3], 1 + lea srcq, [srcq+strideq*2] + pshufb m0, m4 + pshufb m1, m4 + pshufb m2, m4 + pshufb m3, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmaddubsw m2, m5 + pmaddubsw m3, m5 + mova [tmpq+32*0], m0 + mova [tmpq+32*1], m1 + mova [tmpq+32*2], m2 + mova [tmpq+32*3], m3 + add tmpq, 32*4 + sub hd, 2 + jg .h_w32_loop + RET +.h_w64: + movu xm0, [srcq+8*0] + vinserti128 m0, [srcq+8*1], 1 + movu xm1, [srcq+8*2] + vinserti128 m1, [srcq+8*3], 1 + movu xm2, [srcq+8*4] + vinserti128 m2, [srcq+8*5], 1 + movu xm3, [srcq+8*6] + vinserti128 m3, [srcq+8*7], 1 + add srcq, strideq + pshufb m0, m4 + pshufb m1, m4 + pshufb m2, m4 + pshufb m3, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmaddubsw m2, m5 + pmaddubsw m3, m5 + mova [tmpq+32*0], m0 + mova [tmpq+32*1], m1 + mova [tmpq+32*2], m2 + mova [tmpq+32*3], m3 + add tmpq, 32*4 + dec hd + jg .h_w64 + RET +.h_w128: + movu xm0, [srcq+8*0] + vinserti128 m0, [srcq+8*1], 1 + movu xm1, [srcq+8*2] + vinserti128 m1, [srcq+8*3], 1 + movu xm2, [srcq+8*4] + vinserti128 m2, [srcq+8*5], 1 + movu xm3, [srcq+8*6] + vinserti128 m3, [srcq+8*7], 1 + pshufb m0, m4 + pshufb m1, m4 + pshufb m2, m4 + pshufb m3, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmaddubsw m2, m5 + pmaddubsw m3, m5 + mova [tmpq+32*0], m0 + mova [tmpq+32*1], m1 + mova [tmpq+32*2], m2 + mova [tmpq+32*3], m3 + movu xm0, [srcq+8* 8] + vinserti128 m0, [srcq+8* 9], 1 + movu xm1, [srcq+8*10] + vinserti128 m1, [srcq+8*11], 1 + movu xm2, [srcq+8*12] + vinserti128 m2, [srcq+8*13], 1 + movu xm3, [srcq+8*14] + vinserti128 m3, [srcq+8*15], 1 + add tmpq, 32*8 + add srcq, strideq + pshufb m0, m4 + pshufb m1, m4 + pshufb m2, m4 + pshufb m3, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmaddubsw m2, m5 + pmaddubsw m3, m5 + mova [tmpq-32*4], m0 + mova [tmpq-32*3], m1 + mova [tmpq-32*2], m2 + mova [tmpq-32*1], m3 + dec hd + jg .h_w128 + RET +.v: + WIN64_SPILL_XMM 7 + movzx wd, word [t2+wq*2+table_offset(prep, _bilin_v)] + imul mxyd, 0xff01 + add mxyd, 16 << 8 + add wq, t2 + lea stride3q, [strideq*3] + movd xm6, mxyd + vpbroadcastw m6, xm6 + jmp wq +.v_w4: + movd xm0, [srcq+strideq*0] +.v_w4_loop: + vpbroadcastd m1, [srcq+strideq*2] + vpbroadcastd xm2, [srcq+strideq*1] + vpbroadcastd m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpblendd m1, m0, 0x05 ; 0 2 2 2 + vpbroadcastd m0, [srcq+strideq*0] + vpblendd m3, m2, 0x0f ; 1 1 3 3 + vpblendd m2, m1, m0, 0xa0 ; 0 2 2 4 + vpblendd m1, m3, 0xaa ; 0 1 2 3 + vpblendd m2, m3, 0x55 ; 1 2 3 4 + punpcklbw m2, m1 + pmaddubsw m2, m6 + mova [tmpq], m2 + add tmpq, 32 + sub hd, 4 + jg .v_w4_loop + RET +.v_w8: + movq xm0, [srcq+strideq*0] +.v_w8_loop: + vpbroadcastq m1, [srcq+strideq*2] + vpbroadcastq m2, [srcq+strideq*1] + vpbroadcastq m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpblendd m1, m0, 0x03 ; 0 2 2 2 + vpbroadcastq m0, [srcq+strideq*0] + vpblendd m3, m2, 0x33 ; 1 3 1 3 + vpblendd m2, m1, m3, 0x0f ; 1 3 2 2 + vpblendd m1, m3, 0xf0 ; 0 2 1 3 + vpblendd m2, m0, 0xc0 ; 1 3 2 4 + punpcklbw m3, m2, m1 + punpckhbw m2, m1 + pmaddubsw m3, m6 + pmaddubsw m2, m6 + mova [tmpq+32*0], m3 + mova [tmpq+32*1], m2 + add tmpq, 32*2 + sub hd, 4 + jg .v_w8_loop + RET +.v_w16: + vbroadcasti128 m0, [srcq+strideq*0] +.v_w16_loop: + vbroadcasti128 m1, [srcq+strideq*2] + vbroadcasti128 m2, [srcq+strideq*1] + vbroadcasti128 m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + shufpd m4, m0, m1, 0x0c ; 0 2 ; 0l2l 0h2h + vbroadcasti128 m0, [srcq+strideq*0] + shufpd m2, m2, m3, 0x0c ; 1 3 ; 1l3l 1h3h + shufpd m1, m1, m0, 0x0c ; 2 4 ; 2l4l 2h4h + punpcklbw m3, m2, m4 + punpcklbw m5, m1, m2 + punpckhbw m1, m2 + punpckhbw m2, m4 + pmaddubsw m3, m6 + pmaddubsw m5, m6 + pmaddubsw m2, m6 + pmaddubsw m1, m6 + mova [tmpq+32*0], m3 + mova [tmpq+32*1], m5 + mova [tmpq+32*2], m2 + mova [tmpq+32*3], m1 + add tmpq, 32*4 + sub hd, 4 + jg .v_w16_loop + RET +.v_w32: + vpermq m0, [srcq+strideq*0], q3120 +.v_w32_loop: + vpermq m1, [srcq+strideq*1], q3120 + vpermq m2, [srcq+strideq*2], q3120 + vpermq m3, [srcq+stride3q ], q3120 + lea srcq, [srcq+strideq*4] + punpcklbw m4, m1, m0 + punpckhbw m5, m1, m0 + vpermq m0, [srcq+strideq*0], q3120 + pmaddubsw m4, m6 + pmaddubsw m5, m6 + mova [tmpq+32*0], m4 + mova [tmpq+32*1], m5 + punpcklbw m4, m2, m1 + punpckhbw m5, m2, m1 + pmaddubsw m4, m6 + pmaddubsw m5, m6 + mova [tmpq+32*2], m4 + mova [tmpq+32*3], m5 + add tmpq, 32*8 + punpcklbw m4, m3, m2 + punpckhbw m5, m3, m2 + punpcklbw m1, m0, m3 + punpckhbw m2, m0, m3 + pmaddubsw m4, m6 + pmaddubsw m5, m6 + pmaddubsw m1, m6 + pmaddubsw m2, m6 + mova [tmpq-32*4], m4 + mova [tmpq-32*3], m5 + mova [tmpq-32*2], m1 + mova [tmpq-32*1], m2 + sub hd, 4 + jg .v_w32_loop + RET +.v_w64: + vpermq m0, [srcq+strideq*0+32*0], q3120 + vpermq m1, [srcq+strideq*0+32*1], q3120 +.v_w64_loop: + vpermq m2, [srcq+strideq*1+32*0], q3120 + vpermq m3, [srcq+strideq*1+32*1], q3120 + lea srcq, [srcq+strideq*2] + punpcklbw m4, m2, m0 + punpckhbw m5, m2, m0 + pmaddubsw m4, m6 + pmaddubsw m5, m6 + mova [tmpq+32*0], m4 + mova [tmpq+32*1], m5 + punpcklbw m4, m3, m1 + punpckhbw m5, m3, m1 + vpermq m0, [srcq+strideq*0+32*0], q3120 + vpermq m1, [srcq+strideq*0+32*1], q3120 + pmaddubsw m4, m6 + pmaddubsw m5, m6 + mova [tmpq+32*2], m4 + mova [tmpq+32*3], m5 + add tmpq, 32*8 + punpcklbw m4, m0, m2 + punpckhbw m5, m0, m2 + punpcklbw m2, m1, m3 + punpckhbw m3, m1, m3 + pmaddubsw m4, m6 + pmaddubsw m5, m6 + pmaddubsw m2, m6 + pmaddubsw m3, m6 + mova [tmpq-32*4], m4 + mova [tmpq-32*3], m5 + mova [tmpq-32*2], m2 + mova [tmpq-32*1], m3 + sub hd, 2 + jg .v_w64_loop + RET +.v_w128: + mov t0, tmpq + mov t1, srcq + lea t2d, [hq+(3<<8)] +.v_w128_loop0: + vpermq m0, [srcq+strideq*0], q3120 +.v_w128_loop: + vpermq m1, [srcq+strideq*1], q3120 + lea srcq, [srcq+strideq*2] + punpcklbw m2, m1, m0 + punpckhbw m3, m1, m0 + vpermq m0, [srcq+strideq*0], q3120 + punpcklbw m4, m0, m1 + punpckhbw m5, m0, m1 + pmaddubsw m2, m6 + pmaddubsw m3, m6 + pmaddubsw m4, m6 + pmaddubsw m5, m6 + mova [tmpq+32*0], m2 + mova [tmpq+32*1], m3 + mova [tmpq+32*8], m4 + mova [tmpq+32*9], m5 + add tmpq, 32*16 + sub hd, 2 + jg .v_w128_loop + movzx hd, t2b + add t0, 64 + add t1, 32 + mov tmpq, t0 + mov srcq, t1 + sub t2d, 1<<8 + jg .v_w128_loop0 + RET +.hv: + ; (16 * src[x] + (my * (src[x + src_stride] - src[x])) + 8) >> 4 + ; = src[x] + (((my * (src[x + src_stride] - src[x])) + 8) >> 4) + %assign stack_offset stack_offset - stack_size_padded + WIN64_SPILL_XMM 7 + movzx wd, word [t2+wq*2+table_offset(prep, _bilin_hv)] + shl mxyd, 11 + movd xm6, mxyd + vpbroadcastw m6, xm6 + add wq, t2 + lea stride3q, [strideq*3] + jmp wq +.hv_w4: + vbroadcasti128 m4, [bilin_h_shuf4] + vpbroadcastq m0, [srcq+strideq*0] + pshufb m0, m4 + pmaddubsw m0, m5 +.hv_w4_loop: + movq xm1, [srcq+strideq*1] + movhps xm1, [srcq+strideq*2] + movq xm2, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + movhps xm2, [srcq+strideq*0] + vinserti128 m1, xm2, 1 + pshufb m1, m4 + pmaddubsw m1, m5 ; 1 2 3 4 + vpblendd m2, m1, m0, 0xc0 + vpermq m2, m2, q2103 ; 0 1 2 3 + mova m0, m1 + psubw m1, m2 + pmulhrsw m1, m6 + paddw m1, m2 + mova [tmpq], m1 + add tmpq, 32 + sub hd, 4 + jg .hv_w4_loop + RET +.hv_w8: + vbroadcasti128 m0, [srcq+strideq*0] + pshufb m0, m4 + pmaddubsw m0, m5 +.hv_w8_loop: + movu xm1, [srcq+strideq*1] + vinserti128 m1, [srcq+strideq*2], 1 + movu xm2, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vinserti128 m2, [srcq+strideq*0], 1 + pshufb m1, m4 + pshufb m2, m4 + pmaddubsw m1, m5 ; 1 2 + vperm2i128 m3, m0, m1, 0x21 ; 0 1 + pmaddubsw m0, m2, m5 ; 3 4 + vperm2i128 m2, m1, m0, 0x21 ; 2 3 + psubw m1, m3 + pmulhrsw m1, m6 + paddw m1, m3 + psubw m3, m0, m2 + pmulhrsw m3, m6 + paddw m3, m2 + mova [tmpq+32*0], m1 + mova [tmpq+32*1], m3 + add tmpq, 32*2 + sub hd, 4 + jg .hv_w8_loop + RET +.hv_w16: + movu xm0, [srcq+strideq*0+8*0] + vinserti128 m0, [srcq+strideq*0+8*1], 1 + pshufb m0, m4 + pmaddubsw m0, m5 +.hv_w16_loop: + movu xm1, [srcq+strideq*1+8*0] + vinserti128 m1, [srcq+strideq*1+8*1], 1 + lea srcq, [srcq+strideq*2] + movu xm2, [srcq+strideq*0+8*0] + vinserti128 m2, [srcq+strideq*0+8*1], 1 + pshufb m1, m4 + pshufb m2, m4 + pmaddubsw m1, m5 + psubw m3, m1, m0 + pmulhrsw m3, m6 + paddw m3, m0 + pmaddubsw m0, m2, m5 + psubw m2, m0, m1 + pmulhrsw m2, m6 + paddw m2, m1 + mova [tmpq+32*0], m3 + mova [tmpq+32*1], m2 + add tmpq, 32*2 + sub hd, 2 + jg .hv_w16_loop + RET +.hv_w32: + movu xm0, [srcq+8*0] + vinserti128 m0, [srcq+8*1], 1 + movu xm1, [srcq+8*2] + vinserti128 m1, [srcq+8*3], 1 + pshufb m0, m4 + pshufb m1, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 +.hv_w32_loop: + add srcq, strideq + movu xm2, [srcq+8*0] + vinserti128 m2, [srcq+8*1], 1 + pshufb m2, m4 + pmaddubsw m2, m5 + psubw m3, m2, m0 + pmulhrsw m3, m6 + paddw m3, m0 + mova m0, m2 + movu xm2, [srcq+8*2] + vinserti128 m2, [srcq+8*3], 1 + pshufb m2, m4 + pmaddubsw m2, m5 + mova [tmpq+32*0], m3 + psubw m3, m2, m1 + pmulhrsw m3, m6 + paddw m3, m1 + mova m1, m2 + mova [tmpq+32*1], m3 + add tmpq, 32*2 + dec hd + jg .hv_w32_loop + RET +.hv_w64: + mov t0, tmpq + mov t1, srcq + lea t2d, [hq+(3<<8)] +.hv_w64_loop0: + movu xm0, [srcq+strideq*0+8*0] + vinserti128 m0, [srcq+strideq*0+8*1], 1 + pshufb m0, m4 + pmaddubsw m0, m5 +.hv_w64_loop: + movu xm1, [srcq+strideq*1+8*0] + vinserti128 m1, [srcq+strideq*1+8*1], 1 + lea srcq, [srcq+strideq*2] + movu xm2, [srcq+strideq*0+8*0] + vinserti128 m2, [srcq+strideq*0+8*1], 1 + pshufb m1, m4 + pshufb m2, m4 + pmaddubsw m1, m5 + psubw m3, m1, m0 + pmulhrsw m3, m6 + paddw m3, m0 + pmaddubsw m0, m2, m5 + psubw m2, m0, m1 + pmulhrsw m2, m6 + paddw m2, m1 + mova [tmpq+32*0], m3 + add tmpq, 32*8 + mova [tmpq-32*4], m2 + sub hd, 2 + jg .hv_w64_loop + movzx hd, t2b + add t0, 32 + add t1, 16 + mov tmpq, t0 + mov srcq, t1 + sub t2d, 1<<8 + jg .hv_w64_loop0 + RET +.hv_w128: + mov t0, tmpq + mov t1, srcq + lea t2d, [hq+(7<<8)] +.hv_w128_loop0: + movu xm0, [srcq+strideq*0+8*0] + vinserti128 m0, [srcq+strideq*0+8*1], 1 + pshufb m0, m4 + pmaddubsw m0, m5 +.hv_w128_loop: + movu xm1, [srcq+strideq*1+8*0] + vinserti128 m1, [srcq+strideq*1+8*1], 1 + lea srcq, [srcq+strideq*2] + movu xm2, [srcq+strideq*0+8*0] + vinserti128 m2, [srcq+strideq*0+8*1], 1 + pshufb m1, m4 + pshufb m2, m4 + pmaddubsw m1, m5 + psubw m3, m1, m0 + pmulhrsw m3, m6 + paddw m3, m0 + pmaddubsw m0, m2, m5 + psubw m2, m0, m1 + pmulhrsw m2, m6 + paddw m2, m1 + mova [tmpq+32*0], m3 + mova [tmpq+32*8], m2 + add tmpq, 32*16 + sub hd, 2 + jg .hv_w128_loop + movzx hd, t2b + add t0, 32 + add t1, 16 + mov tmpq, t0 + mov srcq, t1 + sub t2d, 1<<8 + jg .hv_w128_loop0 + RET + +; int8_t subpel_filters[5][15][8] +%assign FILTER_REGULAR (0*15 << 16) | 3*15 +%assign FILTER_SMOOTH (1*15 << 16) | 4*15 +%assign FILTER_SHARP (2*15 << 16) | 3*15 + +%macro FN 4 ; fn, type, type_h, type_v +cglobal %1_%2 + mov t0d, FILTER_%3 +%ifidn %3, %4 + mov t1d, t0d +%else + mov t1d, FILTER_%4 +%endif +%ifnidn %2, regular ; skip the jump in the last filter + jmp mangle(private_prefix %+ _%1 %+ SUFFIX) +%endif +%endmacro + +%if WIN64 +DECLARE_REG_TMP 4, 5 +%else +DECLARE_REG_TMP 7, 8 +%endif + +%define PUT_8TAP_FN FN put_8tap, + +PUT_8TAP_FN sharp, SHARP, SHARP +PUT_8TAP_FN sharp_smooth, SHARP, SMOOTH +PUT_8TAP_FN smooth_sharp, SMOOTH, SHARP +PUT_8TAP_FN smooth, SMOOTH, SMOOTH +PUT_8TAP_FN sharp_regular, SHARP, REGULAR +PUT_8TAP_FN regular_sharp, REGULAR, SHARP +PUT_8TAP_FN smooth_regular, SMOOTH, REGULAR +PUT_8TAP_FN regular_smooth, REGULAR, SMOOTH +PUT_8TAP_FN regular, REGULAR, REGULAR + +cglobal put_8tap, 4, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 + imul mxd, mxm, 0x010101 + add mxd, t0d ; 8tap_h, mx, 4tap_h + imul myd, mym, 0x010101 + add myd, t1d ; 8tap_v, my, 4tap_v + lea r8, [put_avx2] + movsxd wq, wm + movifnidn hd, hm + test mxd, 0xf00 + jnz .h + test myd, 0xf00 + jnz .v + tzcnt wd, wd + movzx wd, word [r8+wq*2+table_offset(put,)] + add wq, r8 + lea r6, [ssq*3] + lea r7, [dsq*3] +%if WIN64 + pop r8 +%endif + jmp wq +.h: + test myd, 0xf00 + jnz .hv + vpbroadcastd m5, [pw_34] ; 2 + (8 << 2) + WIN64_SPILL_XMM 11 + cmp wd, 4 + jl .h_w2 + vbroadcasti128 m6, [subpel_h_shufA] + je .h_w4 + tzcnt wd, wd + vbroadcasti128 m7, [subpel_h_shufB] + vbroadcasti128 m8, [subpel_h_shufC] + shr mxd, 16 + sub srcq, 3 + movzx wd, word [r8+wq*2+table_offset(put, _8tap_h)] + vpbroadcastd m9, [r8+mxq*8+subpel_filters-put_avx2+0] + vpbroadcastd m10, [r8+mxq*8+subpel_filters-put_avx2+4] + add wq, r8 + jmp wq +.h_w2: + movzx mxd, mxb + dec srcq + mova xm4, [subpel_h_shuf4] + vpbroadcastd xm3, [r8+mxq*8+subpel_filters-put_avx2+2] +.h_w2_loop: + movq xm0, [srcq+ssq*0] + movhps xm0, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pshufb xm0, xm4 + pmaddubsw xm0, xm3 + phaddw xm0, xm0 + paddw xm0, xm5 + psraw xm0, 6 + packuswb xm0, xm0 + pextrw [dstq+dsq*0], xm0, 0 + pextrw [dstq+dsq*1], xm0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w2_loop + RET +.h_w4: + movzx mxd, mxb + dec srcq + vpbroadcastd xm3, [r8+mxq*8+subpel_filters-put_avx2+2] +.h_w4_loop: + movq xm0, [srcq+ssq*0] + movq xm1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pshufb xm0, xm6 + pshufb xm1, xm6 + pmaddubsw xm0, xm3 + pmaddubsw xm1, xm3 + phaddw xm0, xm1 + paddw xm0, xm5 + psraw xm0, 6 + packuswb xm0, xm0 + movd [dstq+dsq*0], xm0 + pextrd [dstq+dsq*1], xm0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w4_loop + RET +.h_w8: +%macro PUT_8TAP_H 4 ; dst/src, tmp[1-3] + pshufb m%2, m%1, m7 + pshufb m%3, m%1, m8 + pshufb m%1, m6 + pmaddubsw m%4, m%2, m9 + pmaddubsw m%2, m10 + pmaddubsw m%3, m10 + pmaddubsw m%1, m9 + paddw m%3, m%4 + paddw m%1, m%2 + phaddw m%1, m%3 + paddw m%1, m5 + psraw m%1, 6 +%endmacro + movu xm0, [srcq+ssq*0] + vinserti128 m0, [srcq+ssq*1], 1 + lea srcq, [srcq+ssq*2] + PUT_8TAP_H 0, 1, 2, 3 + vextracti128 xm1, m0, 1 + packuswb xm0, xm1 + movq [dstq+dsq*0], xm0 + movhps [dstq+dsq*1], xm0 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w8 + RET +.h_w16: + movu xm0, [srcq+ssq*0+8*0] + vinserti128 m0, [srcq+ssq*1+8*0], 1 + movu xm1, [srcq+ssq*0+8*1] + vinserti128 m1, [srcq+ssq*1+8*1], 1 + PUT_8TAP_H 0, 2, 3, 4 + lea srcq, [srcq+ssq*2] + PUT_8TAP_H 1, 2, 3, 4 + packuswb m0, m1 + mova [dstq+dsq*0], xm0 + vextracti128 [dstq+dsq*1], m0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w16 + RET +.h_w32: + xor r6d, r6d + jmp .h_start +.h_w64: + mov r6, -32*1 + jmp .h_start +.h_w128: + mov r6, -32*3 +.h_start: + sub srcq, r6 + sub dstq, r6 + mov r4, r6 +.h_loop: + movu m0, [srcq+r6+8*0] + movu m1, [srcq+r6+8*1] + PUT_8TAP_H 0, 2, 3, 4 + PUT_8TAP_H 1, 2, 3, 4 + packuswb m0, m1 + mova [dstq+r6], m0 + add r6, 32 + jle .h_loop + add srcq, ssq + add dstq, dsq + mov r6, r4 + dec hd + jg .h_loop + RET +.v: + %assign stack_offset stack_offset - stack_size_padded + WIN64_SPILL_XMM 16 + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + tzcnt r6d, wd + movzx r6d, word [r8+r6*2+table_offset(put, _8tap_v)] + vpbroadcastd m7, [pw_512] + lea myq, [r8+myq*8+subpel_filters-put_avx2] + vpbroadcastw m8, [myq+0] + vpbroadcastw m9, [myq+2] + vpbroadcastw m10, [myq+4] + vpbroadcastw m11, [myq+6] + add r6, r8 + lea ss3q, [ssq*3] + sub srcq, ss3q + jmp r6 +.v_w2: + movd xm2, [srcq+ssq*0] + pinsrw xm2, [srcq+ssq*1], 2 + pinsrw xm2, [srcq+ssq*2], 4 + pinsrw xm2, [srcq+ss3q ], 6 ; 0 1 2 3 + lea srcq, [srcq+ssq*4] + movd xm3, [srcq+ssq*0] + vpbroadcastd xm1, [srcq+ssq*1] + vpbroadcastd xm0, [srcq+ssq*2] + add srcq, ss3q + vpblendd xm3, xm1, 0x02 ; 4 5 + vpblendd xm1, xm0, 0x02 ; 5 6 + palignr xm4, xm3, xm2, 4 ; 1 2 3 4 + punpcklbw xm3, xm1 ; 45 56 + punpcklbw xm1, xm2, xm4 ; 01 12 + punpckhbw xm2, xm4 ; 23 34 +.v_w2_loop: + pmaddubsw xm5, xm1, xm8 ; a0 b0 + mova xm1, xm2 + pmaddubsw xm2, xm9 ; a1 b1 + paddw xm5, xm2 + mova xm2, xm3 + pmaddubsw xm3, xm10 ; a2 b2 + paddw xm5, xm3 + vpbroadcastd xm4, [srcq+ssq*0] + vpblendd xm3, xm0, xm4, 0x02 ; 6 7 + vpbroadcastd xm0, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vpblendd xm4, xm0, 0x02 ; 7 8 + punpcklbw xm3, xm4 ; 67 78 + pmaddubsw xm4, xm3, xm11 ; a3 b3 + paddw xm5, xm4 + pmulhrsw xm5, xm7 + packuswb xm5, xm5 + pextrw [dstq+dsq*0], xm5, 0 + pextrw [dstq+dsq*1], xm5, 2 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w2_loop + RET +.v_w4: + movd xm2, [srcq+ssq*0] + pinsrd xm2, [srcq+ssq*1], 1 + pinsrd xm2, [srcq+ssq*2], 2 + pinsrd xm2, [srcq+ss3q ], 3 ; 0 1 2 3 + lea srcq, [srcq+ssq*4] + movd xm3, [srcq+ssq*0] + vpbroadcastd xm1, [srcq+ssq*1] + vpbroadcastd xm0, [srcq+ssq*2] + add srcq, ss3q + vpblendd xm3, xm1, 0x02 ; 4 5 + vpblendd xm1, xm0, 0x02 ; 5 6 + palignr xm4, xm3, xm2, 4 ; 1 2 3 4 + punpcklbw xm3, xm1 ; 45 56 + punpcklbw xm1, xm2, xm4 ; 01 12 + punpckhbw xm2, xm4 ; 23 34 +.v_w4_loop: + pmaddubsw xm5, xm1, xm8 ; a0 b0 + mova xm1, xm2 + pmaddubsw xm2, xm9 ; a1 b1 + paddw xm5, xm2 + mova xm2, xm3 + pmaddubsw xm3, xm10 ; a2 b2 + paddw xm5, xm3 + vpbroadcastd xm4, [srcq+ssq*0] + vpblendd xm3, xm0, xm4, 0x02 ; 6 7 + vpbroadcastd xm0, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vpblendd xm4, xm0, 0x02 ; 7 8 + punpcklbw xm3, xm4 ; 67 78 + pmaddubsw xm4, xm3, xm11 ; a3 b3 + paddw xm5, xm4 + pmulhrsw xm5, xm7 + packuswb xm5, xm5 + movd [dstq+dsq*0], xm5 + pextrd [dstq+dsq*1], xm5, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w4_loop + RET +.v_w8: + movq xm1, [srcq+ssq*0] + vpbroadcastq m4, [srcq+ssq*1] + vpbroadcastq m2, [srcq+ssq*2] + vpbroadcastq m5, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + vpbroadcastq m3, [srcq+ssq*0] + vpbroadcastq m6, [srcq+ssq*1] + vpbroadcastq m0, [srcq+ssq*2] + add srcq, ss3q + vpblendd m1, m4, 0x30 + vpblendd m4, m2, 0x30 + punpcklbw m1, m4 ; 01 12 + vpblendd m2, m5, 0x30 + vpblendd m5, m3, 0x30 + punpcklbw m2, m5 ; 23 34 + vpblendd m3, m6, 0x30 + vpblendd m6, m0, 0x30 + punpcklbw m3, m6 ; 45 56 +.v_w8_loop: + pmaddubsw m5, m1, m8 ; a0 b0 + mova m1, m2 + pmaddubsw m2, m9 ; a1 b1 + paddw m5, m2 + mova m2, m3 + pmaddubsw m3, m10 ; a2 b2 + paddw m5, m3 + vpbroadcastq m4, [srcq+ssq*0] + vpblendd m3, m0, m4, 0x30 + vpbroadcastq m0, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vpblendd m4, m0, 0x30 + punpcklbw m3, m4 ; 67 78 + pmaddubsw m4, m3, m11 ; a3 b3 + paddw m5, m4 + pmulhrsw m5, m7 + vextracti128 xm4, m5, 1 + packuswb xm5, xm4 + movq [dstq+dsq*0], xm5 + movhps [dstq+dsq*1], xm5 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w8_loop + RET +.v_w16: +.v_w32: +.v_w64: +.v_w128: + lea r6d, [wq-16] + mov r4, dstq + mov r7, srcq + shl r6d, 4 + mov r6b, hb +.v_w16_loop0: + vbroadcasti128 m4, [srcq+ssq*0] + vbroadcasti128 m5, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vbroadcasti128 m0, [srcq+ssq*1] + vbroadcasti128 m6, [srcq+ssq*0] + lea srcq, [srcq+ssq*2] + vbroadcasti128 m1, [srcq+ssq*0] + vbroadcasti128 m2, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vbroadcasti128 m3, [srcq+ssq*0] + shufpd m4, m4, m0, 0x0c + shufpd m5, m5, m1, 0x0c + punpcklbw m1, m4, m5 ; 01 + punpckhbw m4, m5 ; 34 + shufpd m6, m6, m2, 0x0c + punpcklbw m2, m5, m6 ; 12 + punpckhbw m5, m6 ; 45 + shufpd m0, m0, m3, 0x0c + punpcklbw m3, m6, m0 ; 23 + punpckhbw m6, m0 ; 56 +.v_w16_loop: + vbroadcasti128 m12, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vbroadcasti128 m13, [srcq+ssq*0] + pmaddubsw m14, m1, m8 ; a0 + pmaddubsw m15, m2, m8 ; b0 + mova m1, m3 + mova m2, m4 + pmaddubsw m3, m9 ; a1 + pmaddubsw m4, m9 ; b1 + paddw m14, m3 + paddw m15, m4 + mova m3, m5 + mova m4, m6 + pmaddubsw m5, m10 ; a2 + pmaddubsw m6, m10 ; b2 + paddw m14, m5 + paddw m15, m6 + shufpd m6, m0, m12, 0x0d + shufpd m0, m12, m13, 0x0c + punpcklbw m5, m6, m0 ; 67 + punpckhbw m6, m0 ; 78 + pmaddubsw m12, m5, m11 ; a3 + pmaddubsw m13, m6, m11 ; b3 + paddw m14, m12 + paddw m15, m13 + pmulhrsw m14, m7 + pmulhrsw m15, m7 + packuswb m14, m15 + vpermq m14, m14, q3120 + mova [dstq+dsq*0], xm14 + vextracti128 [dstq+dsq*1], m14, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w16_loop + movzx hd, r6b + add r4, 16 + add r7, 16 + mov dstq, r4 + mov srcq, r7 + sub r6d, 1<<8 + jg .v_w16_loop0 + RET +.hv: + %assign stack_offset stack_offset - stack_size_padded + WIN64_SPILL_XMM 16 + cmp wd, 4 + jg .hv_w8 + movzx mxd, mxb + dec srcq + vpbroadcastd m7, [r8+mxq*8+subpel_filters-put_avx2+2] + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + vpbroadcastq m0, [r8+myq*8+subpel_filters-put_avx2] + lea ss3q, [ssq*3] + sub srcq, ss3q + punpcklbw m0, m0 + psraw m0, 8 ; sign-extend + vpbroadcastd m8, [pw_8192] + vpbroadcastd m9, [pd_512] + pshufd m10, m0, q0000 + pshufd m11, m0, q1111 + pshufd m12, m0, q2222 + pshufd m13, m0, q3333 + cmp wd, 4 + je .hv_w4 + vbroadcasti128 m6, [subpel_h_shuf4] + movq xm2, [srcq+ssq*0] + movhps xm2, [srcq+ssq*1] + movq xm0, [srcq+ssq*2] + movhps xm0, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + vpbroadcastq m3, [srcq+ssq*0] + vpbroadcastq m4, [srcq+ssq*1] + vpbroadcastq m1, [srcq+ssq*2] + add srcq, ss3q + vpblendd m2, m3, 0x30 + vpblendd m0, m1, 0x30 + vpblendd m2, m4, 0xc0 + pshufb m2, m6 + pshufb m0, m6 + pmaddubsw m2, m7 + pmaddubsw m0, m7 + phaddw m2, m0 + pmulhrsw m2, m8 + vextracti128 xm3, m2, 1 + palignr xm4, xm3, xm2, 4 + punpcklwd xm1, xm2, xm4 ; 01 12 + punpckhwd xm2, xm4 ; 23 34 + pshufd xm0, xm3, q2121 + punpcklwd xm3, xm0 ; 45 56 +.hv_w2_loop: + pmaddwd xm5, xm1, xm10 ; a0 b0 + mova xm1, xm2 + pmaddwd xm2, xm11 ; a1 b1 + paddd xm5, xm2 + mova xm2, xm3 + pmaddwd xm3, xm12 ; a2 b2 + paddd xm5, xm3 + movq xm4, [srcq+ssq*0] + movhps xm4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pshufb xm4, xm6 + pmaddubsw xm4, xm7 + phaddw xm4, xm4 + pmulhrsw xm4, xm8 + palignr xm3, xm4, xm0, 12 + mova xm0, xm4 + punpcklwd xm3, xm0 ; 67 78 + pmaddwd xm4, xm3, xm13 ; a3 b3 + paddd xm5, xm9 + paddd xm5, xm4 + psrad xm5, 10 + packssdw xm5, xm5 + packuswb xm5, xm5 + pextrw [dstq+dsq*0], xm5, 0 + pextrw [dstq+dsq*1], xm5, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w2_loop + RET +.hv_w4: + mova m6, [subpel_h_shuf4] + vpbroadcastq m2, [srcq+ssq*0] + vpbroadcastq m4, [srcq+ssq*1] + vpbroadcastq m0, [srcq+ssq*2] + vpbroadcastq m5, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + vpbroadcastq m3, [srcq+ssq*0] + vpblendd m2, m4, 0xcc ; 0 1 + vpbroadcastq m4, [srcq+ssq*1] + vpbroadcastq m1, [srcq+ssq*2] + add srcq, ss3q + vpblendd m0, m5, 0xcc ; 2 3 + vpblendd m3, m4, 0xcc ; 4 5 + pshufb m2, m6 + pshufb m0, m6 + pshufb m3, m6 + pshufb m1, m6 + pmaddubsw m2, m7 + pmaddubsw m0, m7 + pmaddubsw m3, m7 + pmaddubsw m1, m7 + phaddw m2, m0 + phaddw m3, m1 + pmulhrsw m2, m8 + pmulhrsw m3, m8 + palignr m4, m3, m2, 4 + punpcklwd m1, m2, m4 ; 01 12 + punpckhwd m2, m4 ; 23 34 + pshufd m0, m3, q2121 + punpcklwd m3, m0 ; 45 56 +.hv_w4_loop: + pmaddwd m5, m1, m10 ; a0 b0 + mova m1, m2 + pmaddwd m2, m11 ; a1 b1 + paddd m5, m2 + mova m2, m3 + pmaddwd m3, m12 ; a2 b2 + paddd m5, m3 + vpbroadcastq m4, [srcq+ssq*0] + vpbroadcastq m3, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vpblendd m4, m3, 0xcc ; 7 8 + pshufb m4, m6 + pmaddubsw m4, m7 + phaddw m4, m4 + pmulhrsw m4, m8 + palignr m3, m4, m0, 12 + mova m0, m4 + punpcklwd m3, m0 ; 67 78 + pmaddwd m4, m3, m13 ; a3 b3 + paddd m5, m9 + paddd m5, m4 + psrad m5, 10 + vextracti128 xm4, m5, 1 + packssdw xm5, xm4 + packuswb xm5, xm5 + pshuflw xm5, xm5, q3120 + movd [dstq+dsq*0], xm5 + pextrd [dstq+dsq*1], xm5, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w4_loop + RET +.hv_w8: + shr mxd, 16 + sub srcq, 3 + vpbroadcastd m10, [r8+mxq*8+subpel_filters-put_avx2+0] + vpbroadcastd m11, [r8+mxq*8+subpel_filters-put_avx2+4] + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + vpbroadcastq m0, [r8+myq*8+subpel_filters-put_avx2] + lea ss3q, [ssq*3] + sub srcq, ss3q + punpcklbw m0, m0 + psraw m0, 8 ; sign-extend + pshufd m12, m0, q0000 + pshufd m13, m0, q1111 + pshufd m14, m0, q2222 + pshufd m15, m0, q3333 + lea r6d, [wq-8] + mov r4, dstq + mov r7, srcq + shl r6d, 5 + mov r6b, hb +.hv_w8_loop0: + vbroadcasti128 m7, [subpel_h_shufA] + vbroadcasti128 m8, [subpel_h_shufB] + vbroadcasti128 m9, [subpel_h_shufC] + movu xm4, [srcq+ssq*0] + movu xm5, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + movu xm6, [srcq+ssq*0] + vbroadcasti128 m0, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vpblendd m4, m0, 0xf0 ; 0 3 + vinserti128 m5, [srcq+ssq*0], 1 ; 1 4 + vinserti128 m6, [srcq+ssq*1], 1 ; 2 5 + lea srcq, [srcq+ssq*2] + vinserti128 m0, [srcq+ssq*0], 1 ; 3 6 +%macro HV_H_W8 4-7 ; src/dst, tmp[1-3], shuf[1-3] + pshufb %3, %1, %6 + pshufb %4, %1, %7 + pshufb %1, %5 + pmaddubsw %2, %3, m10 + pmaddubsw %4, m11 + pmaddubsw %3, m11 + pmaddubsw %1, m10 + paddw %2, %4 + paddw %1, %3 + phaddw %1, %2 +%endmacro + HV_H_W8 m4, m1, m2, m3, m7, m8, m9 + HV_H_W8 m5, m1, m2, m3, m7, m8, m9 + HV_H_W8 m6, m1, m2, m3, m7, m8, m9 + HV_H_W8 m0, m1, m2, m3, m7, m8, m9 + vpbroadcastd m7, [pw_8192] + vpermq m4, m4, q3120 + vpermq m5, m5, q3120 + vpermq m6, m6, q3120 + pmulhrsw m0, m7 + pmulhrsw m4, m7 + pmulhrsw m5, m7 + pmulhrsw m6, m7 + vpermq m7, m0, q3120 + punpcklwd m1, m4, m5 ; 01 + punpckhwd m4, m5 ; 34 + punpcklwd m2, m5, m6 ; 12 + punpckhwd m5, m6 ; 45 + punpcklwd m3, m6, m7 ; 23 + punpckhwd m6, m7 ; 56 +.hv_w8_loop: + vextracti128 r6m, m0, 1 ; not enough registers + movu xm0, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + vinserti128 m0, [srcq+ssq*0], 1 ; 7 8 + pmaddwd m8, m1, m12 ; a0 + pmaddwd m9, m2, m12 ; b0 + mova m1, m3 + mova m2, m4 + pmaddwd m3, m13 ; a1 + pmaddwd m4, m13 ; b1 + paddd m8, m3 + paddd m9, m4 + mova m3, m5 + mova m4, m6 + pmaddwd m5, m14 ; a2 + pmaddwd m6, m14 ; b2 + paddd m8, m5 + paddd m9, m6 + vbroadcasti128 m6, [subpel_h_shufB] + vbroadcasti128 m7, [subpel_h_shufC] + vbroadcasti128 m5, [subpel_h_shufA] + HV_H_W8 m0, m5, m6, m7, m5, m6, m7 + vpbroadcastd m5, [pw_8192] + vpbroadcastd m7, [pd_512] + vbroadcasti128 m6, r6m + pmulhrsw m0, m5 + paddd m8, m7 + paddd m9, m7 + vpermq m7, m0, q3120 ; 7 8 + shufpd m6, m6, m7, 0x04 ; 6 7 + punpcklwd m5, m6, m7 ; 67 + punpckhwd m6, m7 ; 78 + pmaddwd m7, m5, m15 ; a3 + paddd m8, m7 + pmaddwd m7, m6, m15 ; b3 + paddd m7, m9 + psrad m8, 10 + psrad m7, 10 + packssdw m8, m7 + vextracti128 xm7, m8, 1 + packuswb xm8, xm7 + pshufd xm7, xm8, q3120 + movq [dstq+dsq*0], xm7 + movhps [dstq+dsq*1], xm7 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w8_loop + movzx hd, r6b + add r4, 8 + add r7, 8 + mov dstq, r4 + mov srcq, r7 + sub r6d, 1<<8 + jg .hv_w8_loop0 + RET + +%macro PREP_8TAP_H 0 + pshufb m1, m0, m5 + pshufb m2, m0, m6 + pshufb m3, m0, m7 + pmaddubsw m1, m8 + pmaddubsw m0, m2, m8 + pmaddubsw m2, m9 + pmaddubsw m3, m9 + paddw m1, m2 + paddw m0, m3 + phaddw m0, m1, m0 + pmulhrsw m0, m4 +%endmacro + +%macro PREP_8TAP_V_W4 5 ; round, weights + movd xm0, [srcq+strideq*0] + vpbroadcastd m1, [srcq+strideq*2] + vpbroadcastd xm2, [srcq+strideq*1] + vpbroadcastd m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpblendd m1, m0, 0x01 ; 0 2 2 _ 2 _ _ _ + vpblendd m3, m2, 0x03 ; 1 1 3 3 3 3 _ _ + vpbroadcastd m0, [srcq+strideq*0] + vpbroadcastd m2, [srcq+strideq*1] + vpblendd m1, m0, 0x68 ; 0 2 2 4 2 4 4 _ + vpbroadcastd m0, [srcq+strideq*2] + vbroadcasti128 m5, [deint_shuf4] + vpblendd m3, m2, 0xc0 ; 1 1 3 3 3 3 5 5 + vpblendd m2, m3, m1, 0x55 ; 0 1 2 3 2 3 4 5 + vpblendd m3, m1, 0xaa ; 1 2 3 4 3 4 5 _ + punpcklbw m1, m2, m3 ; 01 12 23 34 + vpblendd m3, m0, 0x80 ; 1 2 3 4 3 4 5 6 + punpckhbw m2, m3 ; 23 34 45 56 +.v_w4_loop: + pinsrd xm0, [srcq+stride3q ], 1 + lea srcq, [srcq+strideq*4] + vpbroadcastd m3, [srcq+strideq*0] + vpbroadcastd m4, [srcq+strideq*1] + vpblendd m3, m4, 0x20 ; _ _ 8 _ 8 9 _ _ + vpblendd m3, m0, 0x03 ; 6 7 8 _ 8 9 _ _ + vpbroadcastd m0, [srcq+strideq*2] + vpblendd m3, m0, 0x40 ; 6 7 8 _ 8 9 a _ + pshufb m3, m5 ; 67 78 89 9a + pmaddubsw m4, m1, m%2 + vperm2i128 m1, m2, m3, 0x21 ; 45 56 67 78 + pmaddubsw m2, m%3 + paddw m4, m2 + mova m2, m3 + pmaddubsw m3, m%5 + paddw m3, m4 + pmaddubsw m4, m1, m%4 + paddw m3, m4 + pmulhrsw m3, m%1 + mova [tmpq], m3 +%endmacro + +%if WIN64 +DECLARE_REG_TMP 6, 4 +%else +DECLARE_REG_TMP 6, 7 +%endif + +%define PREP_8TAP_FN FN prep_8tap, + +PREP_8TAP_FN sharp, SHARP, SHARP +PREP_8TAP_FN sharp_smooth, SHARP, SMOOTH +PREP_8TAP_FN smooth_sharp, SMOOTH, SHARP +PREP_8TAP_FN smooth, SMOOTH, SMOOTH +PREP_8TAP_FN sharp_regular, SHARP, REGULAR +PREP_8TAP_FN regular_sharp, REGULAR, SHARP +PREP_8TAP_FN smooth_regular, SMOOTH, REGULAR +PREP_8TAP_FN regular_smooth, REGULAR, SMOOTH +PREP_8TAP_FN regular, REGULAR, REGULAR + +cglobal prep_8tap, 3, 8, 0, tmp, src, stride, w, h, mx, my, stride3 + imul mxd, mxm, 0x010101 + add mxd, t0d ; 8tap_h, mx, 4tap_h + imul myd, mym, 0x010101 + add myd, t1d ; 8tap_v, my, 4tap_v + lea r7, [prep%+SUFFIX] + movsxd wq, wm + movifnidn hd, hm + test mxd, 0xf00 + jnz .h + test myd, 0xf00 + jnz .v + tzcnt wd, wd + movzx wd, word [r7+wq*2+table_offset(prep,)] + add wq, r7 + lea r6, [strideq*3] +%if WIN64 + pop r7 +%endif + jmp wq +.h: + test myd, 0xf00 + jnz .hv + vpbroadcastd m4, [pw_8192] + vbroadcasti128 m5, [subpel_h_shufA] + WIN64_SPILL_XMM 10 + cmp wd, 4 + je .h_w4 + tzcnt wd, wd + vbroadcasti128 m6, [subpel_h_shufB] + vbroadcasti128 m7, [subpel_h_shufC] + shr mxd, 16 + sub srcq, 3 + movzx wd, word [r7+wq*2+table_offset(prep, _8tap_h)] + vpbroadcastd m8, [r7+mxq*8+subpel_filters-prep%+SUFFIX+0] + vpbroadcastd m9, [r7+mxq*8+subpel_filters-prep%+SUFFIX+4] + add wq, r7 + jmp wq +.h_w4: + movzx mxd, mxb + dec srcq + vpbroadcastd m6, [r7+mxq*8+subpel_filters-prep%+SUFFIX+2] + lea stride3q, [strideq*3] +.h_w4_loop: + movq xm0, [srcq+strideq*0] + vpbroadcastq m2, [srcq+strideq*2] + movq xm1, [srcq+strideq*1] + vpblendd m0, m2, 0xf0 + vpbroadcastq m2, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpblendd m1, m2, 0xf0 + pshufb m0, m5 + pshufb m1, m5 + pmaddubsw m0, m6 + pmaddubsw m1, m6 + phaddw m0, m1 + pmulhrsw m0, m4 + mova [tmpq], m0 + add tmpq, 32 + sub hd, 4 + jg .h_w4_loop + RET +.h_w8: + movu xm0, [srcq+strideq*0] + vinserti128 m0, [srcq+strideq*1], 1 + lea srcq, [srcq+strideq*2] + PREP_8TAP_H + mova [tmpq], m0 + add tmpq, 32 + sub hd, 2 + jg .h_w8 + RET +.h_w16: + movu xm0, [srcq+strideq*0+8*0] + vinserti128 m0, [srcq+strideq*0+8*1], 1 + PREP_8TAP_H + mova [tmpq+32*0], m0 + movu xm0, [srcq+strideq*1+8*0] + vinserti128 m0, [srcq+strideq*1+8*1], 1 + lea srcq, [srcq+strideq*2] + PREP_8TAP_H + mova [tmpq+32*1], m0 + add tmpq, 32*2 + sub hd, 2 + jg .h_w16 + RET +.h_w32: + xor r6d, r6d + jmp .h_start +.h_w64: + mov r6, -32*1 + jmp .h_start +.h_w128: + mov r6, -32*3 +.h_start: + sub srcq, r6 + mov r5, r6 +.h_loop: + movu xm0, [srcq+r6+8*0] + vinserti128 m0, [srcq+r6+8*1], 1 + PREP_8TAP_H + mova [tmpq+32*0], m0 + movu xm0, [srcq+r6+8*2] + vinserti128 m0, [srcq+r6+8*3], 1 + PREP_8TAP_H + mova [tmpq+32*1], m0 + add tmpq, 32*2 + add r6, 32 + jle .h_loop + add srcq, strideq + mov r6, r5 + dec hd + jg .h_loop + RET +.v: + %assign stack_offset stack_offset - stack_size_padded + WIN64_SPILL_XMM 16 + movzx mxd, myb ; Select 4-tap/8-tap filter multipliers. + shr myd, 16 ; Note that the code is 8-tap only, having + cmp hd, 4 ; a separate 4-tap code path for (4|8|16)x4 + cmove myd, mxd ; had a negligible effect on performance. + ; TODO: Would a 6-tap code path be worth it? + lea myq, [r7+myq*8+subpel_filters-prep%+SUFFIX] + lea stride3q, [strideq*3] + sub srcq, stride3q + vpbroadcastd m7, [pw_8192] + vpbroadcastw m8, [myq+0] + vpbroadcastw m9, [myq+2] + vpbroadcastw m10, [myq+4] + vpbroadcastw m11, [myq+6] + cmp wd, 8 + jg .v_w16 + je .v_w8 +.v_w4: + PREP_8TAP_V_W4 7, 8, 9, 10, 11 + add tmpq, 32 + sub hd, 4 + jg .v_w4_loop + RET +.v_w8: + movq xm1, [srcq+strideq*0] + vpbroadcastq m4, [srcq+strideq*1] + vpbroadcastq m2, [srcq+strideq*2] + vpbroadcastq m5, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpbroadcastq m3, [srcq+strideq*0] + vpbroadcastq m6, [srcq+strideq*1] + vpbroadcastq m0, [srcq+strideq*2] + vpblendd m1, m4, 0x30 + vpblendd m4, m2, 0x30 + punpcklbw m1, m4 ; 01 12 + vpblendd m2, m5, 0x30 + vpblendd m5, m3, 0x30 + punpcklbw m2, m5 ; 23 34 + vpblendd m3, m6, 0x30 + vpblendd m6, m0, 0x30 + punpcklbw m3, m6 ; 45 56 +.v_w8_loop: + vpbroadcastq m4, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + pmaddubsw m5, m2, m9 ; a1 + pmaddubsw m6, m2, m8 ; b0 + vpblendd m2, m0, m4, 0x30 + vpbroadcastq m0, [srcq+strideq*0] + vpblendd m4, m0, 0x30 + punpcklbw m2, m4 ; 67 78 + pmaddubsw m1, m8 ; a0 + pmaddubsw m4, m3, m9 ; b1 + paddw m5, m1 + mova m1, m3 + pmaddubsw m3, m10 ; a2 + paddw m6, m4 + paddw m5, m3 + vpbroadcastq m4, [srcq+strideq*1] + vpblendd m3, m0, m4, 0x30 + vpbroadcastq m0, [srcq+strideq*2] + vpblendd m4, m0, 0x30 + punpcklbw m3, m4 ; 89 9a + pmaddubsw m4, m2, m11 ; a3 + paddw m5, m4 + pmaddubsw m4, m2, m10 ; b2 + paddw m6, m4 + pmaddubsw m4, m3, m11 ; b3 + paddw m6, m4 + pmulhrsw m5, m7 + pmulhrsw m6, m7 + mova [tmpq+32*0], m5 + mova [tmpq+32*1], m6 + add tmpq, 32*2 + sub hd, 4 + jg .v_w8_loop + RET +.v_w16: + lea r6d, [wq-16] + mov r5, tmpq + mov r7, srcq + shl r6d, 4 + mov r6b, hb +.v_w16_loop0: + vbroadcasti128 m4, [srcq+strideq*0] + vbroadcasti128 m5, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vbroadcasti128 m0, [srcq+strideq*1] + vbroadcasti128 m6, [srcq+strideq*0] + lea srcq, [srcq+strideq*2] + vbroadcasti128 m1, [srcq+strideq*0] + vbroadcasti128 m2, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vbroadcasti128 m3, [srcq+strideq*0] + shufpd m4, m4, m0, 0x0c + shufpd m5, m5, m1, 0x0c + punpcklbw m1, m4, m5 ; 01 + punpckhbw m4, m5 ; 34 + shufpd m6, m6, m2, 0x0c + punpcklbw m2, m5, m6 ; 12 + punpckhbw m5, m6 ; 45 + shufpd m0, m0, m3, 0x0c + punpcklbw m3, m6, m0 ; 23 + punpckhbw m6, m0 ; 56 +.v_w16_loop: + vbroadcasti128 m12, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vbroadcasti128 m13, [srcq+strideq*0] + pmaddubsw m14, m1, m8 ; a0 + pmaddubsw m15, m2, m8 ; b0 + mova m1, m3 + mova m2, m4 + pmaddubsw m3, m9 ; a1 + pmaddubsw m4, m9 ; b1 + paddw m14, m3 + paddw m15, m4 + mova m3, m5 + mova m4, m6 + pmaddubsw m5, m10 ; a2 + pmaddubsw m6, m10 ; b2 + paddw m14, m5 + paddw m15, m6 + shufpd m6, m0, m12, 0x0d + shufpd m0, m12, m13, 0x0c + punpcklbw m5, m6, m0 ; 67 + punpckhbw m6, m0 ; 78 + pmaddubsw m12, m5, m11 ; a3 + pmaddubsw m13, m6, m11 ; b3 + paddw m14, m12 + paddw m15, m13 + pmulhrsw m14, m7 + pmulhrsw m15, m7 + mova [tmpq+wq*0], m14 + mova [tmpq+wq*2], m15 + lea tmpq, [tmpq+wq*4] + sub hd, 2 + jg .v_w16_loop + movzx hd, r6b + add r5, 32 + add r7, 16 + mov tmpq, r5 + mov srcq, r7 + sub r6d, 1<<8 + jg .v_w16_loop0 + RET +.hv: + %assign stack_offset stack_offset - stack_size_padded + %assign stack_size_padded 0 + WIN64_SPILL_XMM 16 + cmp wd, 4 + je .hv_w4 + shr mxd, 16 + sub srcq, 3 + vpbroadcastd m10, [r7+mxq*8+subpel_filters-prep%+SUFFIX+0] + vpbroadcastd m11, [r7+mxq*8+subpel_filters-prep%+SUFFIX+4] + movzx mxd, myb + shr myd, 16 + cmp hd, 4 + cmove myd, mxd + vpbroadcastq m0, [r7+myq*8+subpel_filters-prep%+SUFFIX] + lea stride3q, [strideq*3] + sub srcq, stride3q + punpcklbw m0, m0 + psraw m0, 8 ; sign-extend + pshufd m12, m0, q0000 + pshufd m13, m0, q1111 + pshufd m14, m0, q2222 + pshufd m15, m0, q3333 + jmp .hv_w8 +.hv_w4: + movzx mxd, mxb + dec srcq + vpbroadcastd m8, [r7+mxq*8+subpel_filters-prep%+SUFFIX+2] + movzx mxd, myb + shr myd, 16 + cmp hd, 4 + cmove myd, mxd + vpbroadcastq m0, [r7+myq*8+subpel_filters-prep%+SUFFIX] + lea stride3q, [strideq*3] + sub srcq, stride3q + mova m7, [subpel_h_shuf4] + pmovzxbd m9, [deint_shuf4] + vpbroadcastd m10, [pw_8192] + punpcklbw m0, m0 + psraw m0, 8 ; sign-extend + vpbroadcastd m11, [pd_32] + pshufd m12, m0, q0000 + pshufd m13, m0, q1111 + pshufd m14, m0, q2222 + pshufd m15, m0, q3333 + vpbroadcastq m2, [srcq+strideq*0] + vpbroadcastq m4, [srcq+strideq*1] + vpbroadcastq m0, [srcq+strideq*2] + vpbroadcastq m5, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpbroadcastq m3, [srcq+strideq*0] + vpbroadcastq m6, [srcq+strideq*1] + vpbroadcastq m1, [srcq+strideq*2] + vpblendd m2, m4, 0xcc ; 0 1 + vpblendd m0, m5, 0xcc ; 2 3 + vpblendd m3, m6, 0xcc ; 4 5 + pshufb m2, m7 ; 00 01 10 11 02 03 12 13 + pshufb m0, m7 ; 20 21 30 31 22 23 32 33 + pshufb m3, m7 ; 40 41 50 51 42 43 52 53 + pshufb m1, m7 ; 60 61 60 61 62 63 62 63 + pmaddubsw m2, m8 + pmaddubsw m0, m8 + pmaddubsw m3, m8 + pmaddubsw m1, m8 + phaddw m2, m0 ; 0a 1a 2a 3a 0b 1b 2b 3b + phaddw m3, m1 ; 4a 5a 6a __ 4b 5b 6b __ + pmulhrsw m2, m10 + pmulhrsw m3, m10 + palignr m4, m3, m2, 4 ; 1a 2a 3a 4a 1b 2b 3b 4b + punpcklwd m1, m2, m4 ; 01 12 + punpckhwd m2, m4 ; 23 34 + pshufd m0, m3, q2121 + punpcklwd m3, m0 ; 45 56 +.hv_w4_loop: + pmaddwd m5, m1, m12 ; a0 b0 + pmaddwd m6, m2, m12 ; c0 d0 + pmaddwd m2, m13 ; a1 b1 + pmaddwd m4, m3, m13 ; c1 d1 + mova m1, m3 + pmaddwd m3, m14 ; a2 b2 + paddd m5, m2 + vpbroadcastq m2, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + paddd m6, m4 + paddd m5, m3 + vpbroadcastq m4, [srcq+strideq*0] + vpbroadcastq m3, [srcq+strideq*1] + vpblendd m2, m4, 0xcc + vpbroadcastq m4, [srcq+strideq*2] + vpblendd m3, m4, 0xcc + pshufb m2, m7 + pshufb m3, m7 + pmaddubsw m2, m8 + pmaddubsw m3, m8 + phaddw m2, m3 + pmulhrsw m2, m10 + palignr m3, m2, m0, 12 + mova m0, m2 + punpcklwd m2, m3, m0 ; 67 78 + punpckhwd m3, m0 ; 89 9a + pmaddwd m4, m2, m14 ; c2 d2 + paddd m6, m11 + paddd m5, m11 + paddd m6, m4 + pmaddwd m4, m2, m15 ; a3 b3 + paddd m5, m4 + pmaddwd m4, m3, m15 ; c3 d3 + paddd m6, m4 + psrad m5, 6 + psrad m6, 6 + packssdw m5, m6 + vpermd m5, m9, m5 + mova [tmpq], m5 + add tmpq, 32 + sub hd, 4 + jg .hv_w4_loop + RET +.hv_w8: + lea r6d, [wq-8] + mov r5, tmpq + mov r7, srcq + shl r6d, 5 + mov r6b, hb +.hv_w8_loop0: + vbroadcasti128 m7, [subpel_h_shufA] + vbroadcasti128 m8, [subpel_h_shufB] + vbroadcasti128 m9, [subpel_h_shufC] + movu xm4, [srcq+strideq*0] + movu xm5, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + movu xm6, [srcq+strideq*0] + vbroadcasti128 m0, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vpblendd m4, m0, 0xf0 ; 0 3 + vinserti128 m5, [srcq+strideq*0], 1 ; 1 4 + vinserti128 m6, [srcq+strideq*1], 1 ; 2 5 + lea srcq, [srcq+strideq*2] + vinserti128 m0, [srcq+strideq*0], 1 ; 3 6 + HV_H_W8 m4, m1, m2, m3, m7, m8, m9 + HV_H_W8 m5, m1, m2, m3, m7, m8, m9 + HV_H_W8 m6, m1, m2, m3, m7, m8, m9 + HV_H_W8 m0, m1, m2, m3, m7, m8, m9 + vpbroadcastd m7, [pw_8192] + vpermq m4, m4, q3120 + vpermq m5, m5, q3120 + vpermq m6, m6, q3120 + pmulhrsw m0, m7 + pmulhrsw m4, m7 + pmulhrsw m5, m7 + pmulhrsw m6, m7 + vpermq m7, m0, q3120 + punpcklwd m1, m4, m5 ; 01 + punpckhwd m4, m5 ; 34 + punpcklwd m2, m5, m6 ; 12 + punpckhwd m5, m6 ; 45 + punpcklwd m3, m6, m7 ; 23 + punpckhwd m6, m7 ; 56 +.hv_w8_loop: + vextracti128 [tmpq], m0, 1 ; not enough registers + movu xm0, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vinserti128 m0, [srcq+strideq*0], 1 ; 7 8 + pmaddwd m8, m1, m12 ; a0 + pmaddwd m9, m2, m12 ; b0 + mova m1, m3 + mova m2, m4 + pmaddwd m3, m13 ; a1 + pmaddwd m4, m13 ; b1 + paddd m8, m3 + paddd m9, m4 + mova m3, m5 + mova m4, m6 + pmaddwd m5, m14 ; a2 + pmaddwd m6, m14 ; b2 + paddd m8, m5 + paddd m9, m6 + vbroadcasti128 m6, [subpel_h_shufB] + vbroadcasti128 m7, [subpel_h_shufC] + vbroadcasti128 m5, [subpel_h_shufA] + HV_H_W8 m0, m5, m6, m7, m5, m6, m7 + vpbroadcastd m5, [pw_8192] + vpbroadcastd m7, [pd_32] + vbroadcasti128 m6, [tmpq] + pmulhrsw m0, m5 + paddd m8, m7 + paddd m9, m7 + vpermq m7, m0, q3120 ; 7 8 + shufpd m6, m6, m7, 0x04 ; 6 7 + punpcklwd m5, m6, m7 ; 67 + punpckhwd m6, m7 ; 78 + pmaddwd m7, m5, m15 ; a3 + paddd m8, m7 + pmaddwd m7, m6, m15 ; b3 + paddd m7, m9 + psrad m8, 6 + psrad m7, 6 + packssdw m8, m7 + vpermq m7, m8, q3120 + mova [tmpq+wq*0], xm7 + vextracti128 [tmpq+wq*2], m7, 1 + lea tmpq, [tmpq+wq*4] + sub hd, 2 + jg .hv_w8_loop + movzx hd, r6b + add r5, 16 + add r7, 8 + mov tmpq, r5 + mov srcq, r7 + sub r6d, 1<<8 + jg .hv_w8_loop0 + RET + +%macro movifprep 2 + %if isprep + mov %1, %2 + %endif +%endmacro + +%macro REMAP_REG 2 + %xdefine r%1 r%2 + %xdefine r%1q r%2q + %xdefine r%1d r%2d +%endmacro + +%macro MCT_8TAP_SCALED_REMAP_REGS_TO_PREV 0 + %if isprep + %xdefine r14_save r14 + %assign %%i 14 + %rep 14 + %assign %%j %%i-1 + REMAP_REG %%i, %%j + %assign %%i %%i-1 + %endrep + %endif +%endmacro + +%macro MCT_8TAP_SCALED_REMAP_REGS_TO_DEFAULT 0 + %if isprep + %assign %%i 1 + %rep 13 + %assign %%j %%i+1 + REMAP_REG %%i, %%j + %assign %%i %%i+1 + %endrep + %xdefine r14 r14_save + %undef r14_save + %endif +%endmacro + +%macro MC_8TAP_SCALED_RET 0-1 1 ; leave_mapping_unchanged + MCT_8TAP_SCALED_REMAP_REGS_TO_DEFAULT + RET + %if %1 + MCT_8TAP_SCALED_REMAP_REGS_TO_PREV + %endif +%endmacro + +%macro MC_8TAP_SCALED_H 8 ; dst, tmp[0-6] + movq xm%1, [srcq+ r4] + movq xm%2, [srcq+ r6] + movhps xm%1, [srcq+ r7] + movhps xm%2, [srcq+ r9] + vinserti128 m%1, [srcq+r10], 1 + vinserti128 m%2, [srcq+r11], 1 + vpbroadcastq m%5, [srcq+r13] + vpbroadcastq m%6, [srcq+ rX] + add srcq, ssq + movq xm%3, [srcq+ r4] + movq xm%4, [srcq+ r6] + movhps xm%3, [srcq+ r7] + movhps xm%4, [srcq+ r9] + vinserti128 m%3, [srcq+r10], 1 + vinserti128 m%4, [srcq+r11], 1 + vpbroadcastq m%7, [srcq+r13] + vpbroadcastq m%8, [srcq+ rX] + add srcq, ssq + vpblendd m%1, m%5, 0xc0 + vpblendd m%2, m%6, 0xc0 + vpblendd m%3, m%7, 0xc0 + vpblendd m%4, m%8, 0xc0 + pmaddubsw m%1, m15 + pmaddubsw m%2, m10 + pmaddubsw m%3, m15 + pmaddubsw m%4, m10 + phaddw m%1, m%2 + phaddw m%3, m%4 + phaddw m%1, m%3 + pmulhrsw m%1, m12 +%endmacro + +%macro MC_8TAP_SCALED 1 +%ifidn %1, put + %assign isprep 0 + %if required_stack_alignment <= STACK_ALIGNMENT +cglobal put_8tap_scaled, 4, 15, 16, 96, dst, ds, src, ss, w, h, mx, my, dx, dy + %else +cglobal put_8tap_scaled, 4, 14, 16, 112, dst, ds, src, ss, w, h, mx, my, dx, dy + %endif + %xdefine base_reg r12 + %define rndshift 10 +%else + %assign isprep 1 + %if required_stack_alignment <= STACK_ALIGNMENT +cglobal prep_8tap_scaled, 4, 15, 16, 112, tmp, src, ss, w, h, mx, my, dx, dy + %xdefine tmp_stridem r14q + %else +cglobal prep_8tap_scaled, 4, 14, 16, 112, tmp, src, ss, w, h, mx, my, dx, dy + %define tmp_stridem qword [rsp+104] + %endif + %xdefine base_reg r11 + %define rndshift 6 +%endif + lea base_reg, [%1_8tap_scaled_avx2] +%define base base_reg-%1_8tap_scaled_avx2 + tzcnt wd, wm + vpbroadcastd m8, dxm +%if isprep && UNIX64 + movd xm14, mxd + vpbroadcastd m14, xm14 + mov r5d, t0d + DECLARE_REG_TMP 5, 7 +%else + vpbroadcastd m14, mxm +%endif + mov dyd, dym +%ifidn %1, put + %if WIN64 + mov r8d, hm + DEFINE_ARGS dst, ds, src, ss, w, _, _, my, h, dy, ss3 + %define hm r5m + %define dxm r8m + %else + DEFINE_ARGS dst, ds, src, ss, w, h, _, my, dx, dy, ss3 + %define hm r6m + %endif + %if required_stack_alignment > STACK_ALIGNMENT + %define dsm [rsp+96] + %define rX r1 + %define rXd r1d + %else + %define dsm dsq + %define rX r14 + %define rXd r14d + %endif +%else ; prep + %if WIN64 + mov r7d, hm + DEFINE_ARGS tmp, src, ss, w, _, _, my, h, dy, ss3 + %define hm r4m + %define dxm r7m + %else + DEFINE_ARGS tmp, src, ss, w, h, _, my, dx, dy, ss3 + %define hm [rsp+96] + %endif + MCT_8TAP_SCALED_REMAP_REGS_TO_PREV + %define rX r14 + %define rXd r14d +%endif + vpbroadcastd m10, [base+pd_0x3ff] + vpbroadcastd m12, [base+pw_8192] +%ifidn %1, put + vpbroadcastd m13, [base+pd_512] +%else + vpbroadcastd m13, [base+pd_32] +%endif + pxor m9, m9 + lea ss3q, [ssq*3] + movzx r7d, t1b + shr t1d, 16 + cmp hd, 6 + cmovs t1d, r7d + sub srcq, ss3q + cmp dyd, 1024 + je .dy1 + cmp dyd, 2048 + je .dy2 + movzx wd, word [base+%1_8tap_scaled_avx2_table+wq*2] + add wq, base_reg + jmp wq +%ifidn %1, put +.w2: + mov myd, mym + movzx t0d, t0b + dec srcq + movd xm15, t0d + punpckldq m8, m9, m8 + paddd m14, m8 ; mx+dx*[0-1] + vpbroadcastd m11, [base+pd_0x4000] + vpbroadcastd xm15, xm15 + pand m8, m14, m10 + psrld m8, 6 + paddd xm15, xm8 + movd r4d, xm15 + pextrd r6d, xm15, 1 + vbroadcasti128 m5, [base+bdct_lb_dw] + vbroadcasti128 m6, [base+subpel_s_shuf2] + vpbroadcastd m15, [base+subpel_filters+r4*8+2] + vpbroadcastd m7, [base+subpel_filters+r6*8+2] + pcmpeqd m8, m9 + psrld m14, 10 + movq xm0, [srcq+ssq*0] + movq xm1, [srcq+ssq*2] + movhps xm0, [srcq+ssq*1] + movhps xm1, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + pshufb m14, m5 + paddb m14, m6 + vinserti128 m0, [srcq+ssq*0], 1 + vinserti128 m1, [srcq+ssq*2], 1 + vpbroadcastq m2, [srcq+ssq*1] + vpbroadcastq m3, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + vpblendd m15, m7, 0xaa + vpblendd m0, m2, 0xc0 ; 0 1 4 5 + vpblendd m1, m3, 0xc0 ; 2 3 6 7 + pblendvb m15, m11, m8 + pshufb m0, m14 + pshufb m1, m14 + pmaddubsw m0, m15 + pmaddubsw m1, m15 + phaddw m0, m1 + pmulhrsw m0, m12 ; 0 1 2 3 4 5 6 7 + vextracti128 xm1, m0, 1 ; 4 5 6 7 + palignr xm2, xm1, xm0, 4 ; 1 2 3 4 + punpcklwd xm3, xm0, xm2 ; 01 12 + punpckhwd xm0, xm2 ; 23 34 + pshufd xm4, xm1, q0321 ; 5 6 7 _ + punpcklwd xm2, xm1, xm4 ; 45 56 + punpckhwd xm4, xm1, xm4 ; 67 __ +.w2_loop: + and myd, 0x3ff + mov r6d, 64 << 24 + mov r4d, myd + shr r4d, 6 + lea r4d, [t1+r4] + cmovnz r6q, [base+subpel_filters+r4*8] + movq xm11, r6q + punpcklbw xm11, xm11 + psraw xm11, 8 + pshufd xm8, xm11, q0000 + pshufd xm9, xm11, q1111 + pshufd xm10, xm11, q2222 + pshufd xm11, xm11, q3333 + pmaddwd xm5, xm3, xm8 + pmaddwd xm6, xm0, xm9 + pmaddwd xm7, xm2, xm10 + pmaddwd xm8, xm4, xm11 + paddd xm5, xm6 + paddd xm7, xm8 + paddd xm5, xm13 + paddd xm5, xm7 + psrad xm5, 10 + packssdw xm5, xm5 + packuswb xm5, xm5 + pextrw [dstq], xm5, 0 + add dstq, dsq + dec hd + jz .ret + add myd, dyd + test myd, ~0x3ff + jz .w2_loop + movq xm5, [srcq] + test myd, 0x400 + jz .w2_skip_line + add srcq, ssq + shufps xm3, xm0, q1032 ; 01 12 + shufps xm0, xm2, q1032 ; 23 34 + shufps xm2, xm4, q1032 ; 45 56 + pshufb xm5, xm14 + pmaddubsw xm5, xm15 + phaddw xm5, xm5 + pmulhrsw xm5, xm12 + palignr xm1, xm5, xm1, 12 + punpcklqdq xm1, xm1 ; 6 7 6 7 + punpcklwd xm4, xm1, xm5 ; 67 __ + jmp .w2_loop +.w2_skip_line: + movhps xm5, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mova xm3, xm0 ; 01 12 + mova xm0, xm2 ; 23 34 + pshufb xm5, xm14 + pmaddubsw xm5, xm15 + phaddw xm5, xm5 + pmulhrsw xm5, xm12 ; 6 7 6 7 + palignr xm1, xm5, xm1, 8 ; 4 5 6 7 + pshufd xm5, xm1, q0321 ; 5 6 7 _ + punpcklwd xm2, xm1, xm5 ; 45 56 + punpckhwd xm4, xm1, xm5 ; 67 __ + jmp .w2_loop +%endif +.w4: + mov myd, mym + vbroadcasti128 m7, [base+rescale_mul] + movzx t0d, t0b + dec srcq + movd xm15, t0d + pmaddwd m8, m7 + vpbroadcastd m11, [base+pd_0x4000] + vpbroadcastd xm15, xm15 + paddd m14, m8 ; mx+dx*[0-3] + pand m0, m14, m10 + psrld m0, 6 + paddd xm15, xm0 + movd r4d, xm15 + pextrd r6d, xm15, 1 + pextrd r11d, xm15, 2 + pextrd r13d, xm15, 3 + movd xm15, [base+subpel_filters+r4*8+2] + vbroadcasti128 m5, [base+bdct_lb_dw] + vpbroadcastq m6, [base+subpel_s_shuf2] + pinsrd xm15, [base+subpel_filters+r6*8+2], 1 + pcmpeqd m0, m9 + psrld m14, 10 + movu xm7, [srcq+ssq*0] + movu xm9, [srcq+ssq*1] + pinsrd xm15, [base+subpel_filters+r11*8+2], 2 + movu xm8, [srcq+ssq*2] + movu xm10, [srcq+ss3q ] + pinsrd xm15, [base+subpel_filters+r13*8+2], 3 + lea srcq, [srcq+ssq*4] + pshufb m14, m5 + paddb m14, m6 + vinserti128 m7, [srcq+ssq*0], 1 + vinserti128 m9, [srcq+ssq*1], 1 + vinserti128 m15, xm15, 1 + vinserti128 m8, [srcq+ssq*2], 1 + vinserti128 m10, [srcq+ss3q ], 1 + lea srcq, [srcq+ssq*4] + pblendvb m15, m11, m0 + pshufb m7, m14 + pshufb m9, m14 + pshufb m8, m14 + pshufb m10, m14 + pmaddubsw m7, m15 + pmaddubsw m9, m15 + pmaddubsw m8, m15 + pmaddubsw m10, m15 + phaddw m7, m9 + phaddw m8, m10 + pmulhrsw m7, m12 ; 0 1 4 5 + pmulhrsw m8, m12 ; 2 3 6 7 + vextracti128 xm9, m7, 1 ; 4 5 + vextracti128 xm3, m8, 1 ; 6 7 + shufps xm4, xm7, xm8, q1032 ; 1 2 + shufps xm5, xm8, xm9, q1032 ; 3 4 + shufps xm6, xm9, xm3, q1032 ; 5 6 + psrldq xm11, xm3, 8 ; 7 _ + punpcklwd xm0, xm7, xm4 ; 01 + punpckhwd xm7, xm4 ; 12 + punpcklwd xm1, xm8, xm5 ; 23 + punpckhwd xm8, xm5 ; 34 + punpcklwd xm2, xm9, xm6 ; 45 + punpckhwd xm9, xm6 ; 56 + punpcklwd xm3, xm11 ; 67 + mova [rsp+0x00], xm7 + mova [rsp+0x10], xm8 + mova [rsp+0x20], xm9 +.w4_loop: + and myd, 0x3ff + mov r6d, 64 << 24 + mov r4d, myd + shr r4d, 6 + lea r4d, [t1+r4] + cmovnz r6q, [base+subpel_filters+r4*8] + movq xm10, r6q + punpcklbw xm10, xm10 + psraw xm10, 8 + pshufd xm7, xm10, q0000 + pshufd xm8, xm10, q1111 + pshufd xm9, xm10, q2222 + pshufd xm10, xm10, q3333 + pmaddwd xm4, xm0, xm7 + pmaddwd xm5, xm1, xm8 + pmaddwd xm6, xm2, xm9 + pmaddwd xm7, xm3, xm10 + paddd xm4, xm5 + paddd xm6, xm7 + paddd xm4, xm13 + paddd xm4, xm6 + psrad xm4, rndshift + packssdw xm4, xm4 +%ifidn %1, put + packuswb xm4, xm4 + movd [dstq], xm4 + add dstq, dsq +%else + movq [tmpq], xm4 + add tmpq, 8 +%endif + dec hd + jz .ret + add myd, dyd + test myd, ~0x3ff + jz .w4_loop + movu xm4, [srcq] + test myd, 0x400 + jz .w4_skip_line + mova xm0, [rsp+0x00] + mova [rsp+0x00], xm1 + mova xm1, [rsp+0x10] + mova [rsp+0x10], xm2 + mova xm2, [rsp+0x20] + mova [rsp+0x20], xm3 + pshufb xm4, xm14 + pmaddubsw xm4, xm15 + phaddw xm4, xm4 + pmulhrsw xm4, xm12 + punpcklwd xm3, xm11, xm4 + mova xm11, xm4 + add srcq, ssq + jmp .w4_loop +.w4_skip_line: + movu xm5, [srcq+ssq*1] + movu m6, [rsp+0x10] + pshufb xm4, xm14 + pshufb xm5, xm14 + pmaddubsw xm4, xm15 + pmaddubsw xm5, xm15 + movu [rsp+0x00], m6 + phaddw xm4, xm5 + pmulhrsw xm4, xm12 + punpcklwd xm9, xm11, xm4 + mova [rsp+0x20], xm9 + psrldq xm11, xm4, 8 + mova xm0, xm1 + mova xm1, xm2 + mova xm2, xm3 + punpcklwd xm3, xm4, xm11 + lea srcq, [srcq+ssq*2] + jmp .w4_loop +.w8: +%ifidn %1, put + movifnidn dsm, dsq +%endif + shr t0d, 16 + sub srcq, 3 + movd xm15, t0d + pmaddwd m8, [base+rescale_mul] + vpbroadcastq m11, [base+pq_0x40000000] + vpbroadcastd m15, xm15 + paddd m14, m8 ; mx+dx*[0-7] + pand m6, m14, m10 + psrld m6, 6 + paddd m15, m6 + pcmpeqd m6, m9 + vextracti128 xm7, m15, 1 + movd r4d, xm15 + pextrd r6d, xm15, 2 + pextrd r7d, xm15, 1 + pextrd r9d, xm15, 3 + movd r10d, xm7 + pextrd r11d, xm7, 2 + pextrd r13d, xm7, 1 + pextrd rXd, xm7, 3 + movq xm15, [base+subpel_filters+r4*8] + movq xm10, [base+subpel_filters+r6*8] + movhps xm15, [base+subpel_filters+r7*8] + movhps xm10, [base+subpel_filters+r9*8] + vinserti128 m15, [base+subpel_filters+r10*8], 1 + vinserti128 m10, [base+subpel_filters+r11*8], 1 + vpbroadcastq m9, [base+subpel_filters+r13*8] + vpbroadcastq m8, [base+subpel_filters+rX*8] + psrld m14, 10 + mova [rsp], xm14 + vextracti128 xm7, m14, 1 + movd r4d, xm14 + pextrd r6d, xm14, 2 + pextrd r7d, xm14, 1 + pextrd r9d, xm14, 3 + movd r10d, xm7 + pextrd r11d, xm7, 2 + pextrd r13d, xm7, 1 + pextrd rXd, xm7, 3 + pshufd m5, m6, q1100 + pshufd m6, m6, q3322 + vpblendd m15, m9, 0xc0 + vpblendd m10, m8, 0xc0 + pblendvb m15, m11, m5 + pblendvb m10, m11, m6 + vbroadcasti128 m14, [base+subpel_s_shuf8] + MC_8TAP_SCALED_H 0, 1, 2, 3, 4, 5, 6, 7 ; 0a 1a 0b 1b + MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 7, 8 ; 2a 3a 2b 3b + MC_8TAP_SCALED_H 2, 3, 4, 5, 6, 7, 8, 9 ; 4a 5a 4b 5b + MC_8TAP_SCALED_H 3, 4, 5, 6, 7, 8, 9, 11 ; 6a 7a 6b 7b + mov myd, mym + mov dyd, dym + pshufb m0, m14 ; 01a 01b + pshufb m1, m14 ; 23a 23b + pshufb m2, m14 ; 45a 45b + pshufb m3, m14 ; 67a 67b + vbroadcasti128 m14, [base+wswap] +.w8_loop: + and myd, 0x3ff + mov r6d, 64 << 24 + mov r4d, myd + shr r4d, 6 + lea r4d, [t1+r4] + cmovnz r6q, [base+subpel_filters+r4*8] + movq xm11, r6q + punpcklbw xm11, xm11 + psraw xm11, 8 + vinserti128 m11, xm11, 1 + pshufd m8, m11, q0000 + pshufd m9, m11, q1111 + pmaddwd m4, m0, m8 + pmaddwd m5, m1, m9 + pshufd m8, m11, q2222 + pshufd m11, m11, q3333 + pmaddwd m6, m2, m8 + pmaddwd m7, m3, m11 + paddd m4, m5 + paddd m6, m7 + paddd m4, m13 + paddd m4, m6 + psrad m4, rndshift + vextracti128 xm5, m4, 1 + packssdw xm4, xm5 +%ifidn %1, put + packuswb xm4, xm4 + movq [dstq], xm4 + add dstq, dsm +%else + mova [tmpq], xm4 + add tmpq, 16 +%endif + dec hd + jz .ret + add myd, dyd + test myd, ~0x3ff + jz .w8_loop + test myd, 0x400 + mov [rsp+16], myd + mov r4d, [rsp+ 0] + mov r6d, [rsp+ 8] + mov r7d, [rsp+ 4] + mov r9d, [rsp+12] + jz .w8_skip_line + vpbroadcastq m6, [srcq+r13] + vpbroadcastq m7, [srcq+ rX] + movq xm4, [srcq+ r4] + movq xm5, [srcq+ r6] + movhps xm4, [srcq+ r7] + movhps xm5, [srcq+ r9] + vinserti128 m4, [srcq+r10], 1 + vinserti128 m5, [srcq+r11], 1 + add srcq, ssq + mov myd, [rsp+16] + mov dyd, dym + pshufb m0, m14 + pshufb m1, m14 + pshufb m2, m14 + pshufb m3, m14 + vpblendd m4, m6, 0xc0 + vpblendd m5, m7, 0xc0 + pmaddubsw m4, m15 + pmaddubsw m5, m10 + phaddw m4, m5 + pslld m5, m4, 16 + paddw m4, m5 + pmulhrsw m4, m12 + pblendw m0, m1, 0xaa + pblendw m1, m2, 0xaa + pblendw m2, m3, 0xaa + pblendw m3, m4, 0xaa + jmp .w8_loop +.w8_skip_line: + mova m0, m1 + mova m1, m2 + mova m2, m3 + vpbroadcastq m7, [srcq+r13] + vpbroadcastq m8, [srcq+ rX] + movq xm3, [srcq+ r4] + movq xm4, [srcq+ r6] + movhps xm3, [srcq+ r7] + movhps xm4, [srcq+ r9] + vinserti128 m3, [srcq+r10], 1 + vinserti128 m4, [srcq+r11], 1 + add srcq, ssq + movq xm5, [srcq+ r4] + movq xm6, [srcq+ r6] + movhps xm5, [srcq+ r7] + movhps xm6, [srcq+ r9] + vinserti128 m5, [srcq+r10], 1 + vinserti128 m6, [srcq+r11], 1 + vpbroadcastq m9, [srcq+r13] + vpbroadcastq m11, [srcq+ rX] + add srcq, ssq + mov myd, [rsp+16] + mov dyd, dym + vpblendd m3, m7, 0xc0 + vpblendd m4, m8, 0xc0 + vpblendd m5, m9, 0xc0 + vpblendd m6, m11, 0xc0 + pmaddubsw m3, m15 + pmaddubsw m4, m10 + pmaddubsw m5, m15 + pmaddubsw m6, m10 + phaddw m3, m4 + phaddw m5, m6 + psrld m4, m3, 16 + pslld m6, m5, 16 + paddw m3, m4 + paddw m5, m6 + pblendw m3, m5, 0xaa + pmulhrsw m3, m12 + jmp .w8_loop +.w16: + mov dword [rsp+48], 2 + movifprep tmp_stridem, 32 + jmp .w_start +.w32: + mov dword [rsp+48], 4 + movifprep tmp_stridem, 64 + jmp .w_start +.w64: + mov dword [rsp+48], 8 + movifprep tmp_stridem, 128 + jmp .w_start +.w128: + mov dword [rsp+48], 16 + movifprep tmp_stridem, 256 +.w_start: +%ifidn %1, put + movifnidn dsm, dsq +%endif + shr t0d, 16 + sub srcq, 3 + pmaddwd m8, [base+rescale_mul] + movd xm15, t0d + mov [rsp+72], t0d + mov [rsp+56], srcq + mov [rsp+64], r0q ; dstq / tmpq +%if UNIX64 + mov hm, hd +%endif + shl dword dxm, 3 ; dx*8 + vpbroadcastd m15, xm15 + paddd m14, m8 ; mx+dx*[0-7] + jmp .hloop +.hloop_prep: + dec dword [rsp+48] + jz .ret + add qword [rsp+64], 8*(isprep+1) + mov hd, hm + vpbroadcastd m8, dxm + vpbroadcastd m10, [base+pd_0x3ff] + paddd m14, m8, [rsp+16] + vpbroadcastd m15, [rsp+72] + pxor m9, m9 + mov srcq, [rsp+56] + mov r0q, [rsp+64] ; dstq / tmpq +.hloop: + vpbroadcastq m11, [base+pq_0x40000000] + pand m6, m14, m10 + psrld m6, 6 + paddd m15, m6 + pcmpeqd m6, m9 + vextracti128 xm7, m15, 1 + movd r4d, xm15 + pextrd r6d, xm15, 2 + pextrd r7d, xm15, 1 + pextrd r9d, xm15, 3 + movd r10d, xm7 + pextrd r11d, xm7, 2 + pextrd r13d, xm7, 1 + pextrd rXd, xm7, 3 + movu [rsp+16], m14 + movq xm15, [base+subpel_filters+ r4*8] + movq xm10, [base+subpel_filters+ r6*8] + movhps xm15, [base+subpel_filters+ r7*8] + movhps xm10, [base+subpel_filters+ r9*8] + vinserti128 m15, [base+subpel_filters+r10*8], 1 + vinserti128 m10, [base+subpel_filters+r11*8], 1 + vpbroadcastq m9, [base+subpel_filters+r13*8] + vpbroadcastq m8, [base+subpel_filters+ rX*8] + psrld m14, 10 + vextracti128 xm7, m14, 1 + mova [rsp], xm14 + movd r4d, xm14 + pextrd r6d, xm14, 2 + pextrd r7d, xm14, 1 + pextrd r9d, xm14, 3 + movd r10d, xm7 + pextrd r11d, xm7, 2 + pextrd r13d, xm7, 1 + pextrd rXd, xm7, 3 + pshufd m5, m6, q1100 + pshufd m6, m6, q3322 + vpblendd m15, m9, 0xc0 + vpblendd m10, m8, 0xc0 + pblendvb m15, m11, m5 + pblendvb m10, m11, m6 + vbroadcasti128 m14, [base+subpel_s_shuf8] + MC_8TAP_SCALED_H 0, 1, 2, 3, 4, 5, 6, 7 ; 0a 1a 0b 1b + MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 7, 8 ; 2a 3a 2b 3b + MC_8TAP_SCALED_H 2, 3, 4, 5, 6, 7, 8, 9 ; 4a 5a 4b 5b + MC_8TAP_SCALED_H 3, 4, 5, 6, 7, 8, 9, 11 ; 6a 7a 6b 7b + mov myd, mym + mov dyd, dym + pshufb m0, m14 ; 01a 01b + pshufb m1, m14 ; 23a 23b + pshufb m2, m14 ; 45a 45b + pshufb m3, m14 ; 67a 67b + vbroadcasti128 m14, [base+wswap] +.vloop: + and myd, 0x3ff + mov r6d, 64 << 24 + mov r4d, myd + shr r4d, 6 + lea r4d, [t1+r4] + cmovnz r6q, [base+subpel_filters+r4*8] + movq xm11, r6q + punpcklbw xm11, xm11 + psraw xm11, 8 + vinserti128 m11, xm11, 1 + pshufd m8, m11, q0000 + pshufd m9, m11, q1111 + pmaddwd m4, m0, m8 + pmaddwd m5, m1, m9 + pshufd m8, m11, q2222 + pshufd m11, m11, q3333 + pmaddwd m6, m2, m8 + pmaddwd m7, m3, m11 + paddd m4, m5 + paddd m6, m7 + paddd m4, m13 + paddd m4, m6 + psrad m4, rndshift + vextracti128 xm5, m4, 1 + packssdw xm4, xm5 +%ifidn %1, put + packuswb xm4, xm4 + movq [dstq], xm4 + add dstq, dsm +%else + mova [tmpq], xm4 + add tmpq, tmp_stridem +%endif + dec hd + jz .hloop_prep + add myd, dyd + test myd, ~0x3ff + jz .vloop + test myd, 0x400 + mov [rsp+52], myd + mov r4d, [rsp+ 0] + mov r6d, [rsp+ 8] + mov r7d, [rsp+ 4] + mov r9d, [rsp+12] + jz .skip_line + vpbroadcastq m6, [srcq+r13] + vpbroadcastq m7, [srcq+ rX] + movq xm4, [srcq+ r4] + movq xm5, [srcq+ r6] + movhps xm4, [srcq+ r7] + movhps xm5, [srcq+ r9] + vinserti128 m4, [srcq+r10], 1 + vinserti128 m5, [srcq+r11], 1 + add srcq, ssq + mov myd, [rsp+52] + mov dyd, dym + pshufb m0, m14 + pshufb m1, m14 + pshufb m2, m14 + pshufb m3, m14 + vpblendd m4, m6, 0xc0 + vpblendd m5, m7, 0xc0 + pmaddubsw m4, m15 + pmaddubsw m5, m10 + phaddw m4, m5 + pslld m5, m4, 16 + paddw m4, m5 + pmulhrsw m4, m12 + pblendw m0, m1, 0xaa + pblendw m1, m2, 0xaa + pblendw m2, m3, 0xaa + pblendw m3, m4, 0xaa + jmp .vloop +.skip_line: + mova m0, m1 + mova m1, m2 + mova m2, m3 + vpbroadcastq m7, [srcq+r13] + vpbroadcastq m8, [srcq+ rX] + movq xm3, [srcq+ r4] + movq xm4, [srcq+ r6] + movhps xm3, [srcq+ r7] + movhps xm4, [srcq+ r9] + vinserti128 m3, [srcq+r10], 1 + vinserti128 m4, [srcq+r11], 1 + add srcq, ssq + movq xm5, [srcq+ r4] + movq xm6, [srcq+ r6] + movhps xm5, [srcq+ r7] + movhps xm6, [srcq+ r9] + vinserti128 m5, [srcq+r10], 1 + vinserti128 m6, [srcq+r11], 1 + vpbroadcastq m9, [srcq+r13] + vpbroadcastq m11, [srcq+ rX] + add srcq, ssq + mov myd, [rsp+52] + mov dyd, dym + vpblendd m3, m7, 0xc0 + vpblendd m4, m8, 0xc0 + vpblendd m5, m9, 0xc0 + vpblendd m6, m11, 0xc0 + pmaddubsw m3, m15 + pmaddubsw m4, m10 + pmaddubsw m5, m15 + pmaddubsw m6, m10 + phaddw m3, m4 + phaddw m5, m6 + psrld m4, m3, 16 + pslld m6, m5, 16 + paddw m3, m4 + paddw m5, m6 + pblendw m3, m5, 0xaa + pmulhrsw m3, m12 + jmp .vloop +.dy1: + movzx wd, word [base+%1_8tap_scaled_avx2_dy1_table+wq*2] + add wq, base_reg + jmp wq +%ifidn %1, put +.dy1_w2: + mov myd, mym + movzx t0d, t0b + dec srcq + movd xm15, t0d + punpckldq m8, m9, m8 + paddd m14, m8 ; mx+dx*[0-1] + vpbroadcastd m11, [base+pd_0x4000] + vpbroadcastd xm15, xm15 + pand m8, m14, m10 + psrld m8, 6 + paddd xm15, xm8 + movd r4d, xm15 + pextrd r6d, xm15, 1 + vbroadcasti128 m5, [base+bdct_lb_dw] + vbroadcasti128 m6, [base+subpel_s_shuf2] + vpbroadcastd m15, [base+subpel_filters+r4*8+2] + vpbroadcastd m7, [base+subpel_filters+r6*8+2] + pcmpeqd m8, m9 + psrld m14, 10 + movq xm0, [srcq+ssq*0] + movq xm1, [srcq+ssq*2] + movhps xm0, [srcq+ssq*1] + movhps xm1, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + shr myd, 6 + mov r4d, 64 << 24 + lea myd, [t1+myq] + cmovnz r4q, [base+subpel_filters+myq*8] + pshufb m14, m5 + paddb m14, m6 + vinserti128 m0, [srcq+ssq*0], 1 + vinserti128 m1, [srcq+ssq*2], 1 + vpbroadcastq m2, [srcq+ssq*1] + add srcq, ss3q + movq xm10, r4q + punpcklbw xm10, xm10 + psraw xm10, 8 + vpblendd m15, m7, 0xaa + pblendvb m15, m11, m8 + pshufd xm8, xm10, q0000 + pshufd xm9, xm10, q1111 + pshufd xm11, xm10, q3333 + pshufd xm10, xm10, q2222 + vpblendd m0, m2, 0xc0 + pshufb m1, m14 + pshufb m0, m14 + pmaddubsw m1, m15 + pmaddubsw m0, m15 + phaddw m0, m1 + pmulhrsw m0, m12 + vextracti128 xm1, m0, 1 + palignr xm2, xm1, xm0, 4 + pshufd xm4, xm1, q2121 + punpcklwd xm3, xm0, xm2 ; 01 12 + punpckhwd xm0, xm2 ; 23 34 + punpcklwd xm2, xm1, xm4 ; 45 56 +.dy1_w2_loop: + movq xm1, [srcq+ssq*0] + movhps xm1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pmaddwd xm5, xm3, xm8 + pmaddwd xm6, xm0, xm9 + pmaddwd xm7, xm2, xm10 + mova xm3, xm0 + mova xm0, xm2 + paddd xm5, xm13 + paddd xm6, xm7 + pshufb xm1, xm14 + pmaddubsw xm1, xm15 + phaddw xm1, xm1 + pmulhrsw xm1, xm12 + palignr xm7, xm1, xm4, 12 + punpcklwd xm2, xm7, xm1 ; 67 78 + pmaddwd xm7, xm2, xm11 + mova xm4, xm1 + paddd xm5, xm6 + paddd xm5, xm7 + psrad xm5, rndshift + packssdw xm5, xm5 + packuswb xm5, xm5 + pextrw [dstq+dsq*0], xm5, 0 + pextrw [dstq+dsq*1], xm5, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .dy1_w2_loop + RET +%endif +.dy1_w4: + mov myd, mym + vbroadcasti128 m7, [base+rescale_mul] + movzx t0d, t0b + dec srcq + movd xm15, t0d + pmaddwd m8, m7 + vpbroadcastd m11, [base+pd_0x4000] + vpbroadcastd xm15, xm15 + paddd m14, m8 ; mx+dx*[0-3] + pand m8, m14, m10 + psrld m8, 6 + paddd xm15, xm8 + vpermq m8, m8, q3120 + movd r4d, xm15 + pextrd r6d, xm15, 2 + pextrd r11d, xm15, 1 + pextrd r13d, xm15, 3 + movd xm15, [base+subpel_filters+r4*8+2] + vpbroadcastd m7, [base+subpel_filters+r6*8+2] + movu xm2, [srcq+ssq*0] + movu xm3, [srcq+ssq*2] + vbroadcasti128 m5, [base+bdct_lb_dw] + vpbroadcastq m6, [base+subpel_s_shuf2] + pcmpeqd m8, m9 + psrld m14, 10 + pinsrd xm15, [base+subpel_filters+r11*8+2], 1 + vpblendd m7, [base+subpel_filters+r13*8+2-20], 0x20 + vinserti128 m2, [srcq+ssq*1], 1 + vinserti128 m3, [srcq+ss3q ], 1 + lea srcq, [srcq+ssq*4] + shr myd, 6 + mov r4d, 64 << 24 + lea myd, [t1+myq] + cmovnz r4q, [base+subpel_filters+myq*8] + pshufb m14, m5 + paddb m14, m6 + movu xm4, [srcq+ssq*0] + movu xm5, [srcq+ssq*2] + vinserti128 m4, [srcq+ssq*1], 1 + add srcq, ss3q + vpblendd m15, m7, 0x30 + punpcklqdq m15, m15 + pblendvb m15, m11, m8 + movq xm10, r4q + punpcklbw xm10, xm10 + psraw xm10, 8 + vinserti128 m10, xm10, 1 + pshufb m2, m14 + pshufb m3, m14 + pshufb m4, m14 + pshufb xm5, xm14 + vpermq m2, m2, q3120 + vpermq m3, m3, q3120 + vpermq m4, m4, q3120 + vpermq m5, m5, q3120 + pshufd m7, m10, q0000 + pshufd m8, m10, q1111 + pshufd m9, m10, q2222 + pshufd m10, m10, q3333 + pmaddubsw m2, m15 + pmaddubsw m3, m15 + pmaddubsw m4, m15 + pmaddubsw m5, m15 + phaddw m2, m3 + phaddw m4, m5 + pmulhrsw m2, m12 + pmulhrsw m4, m12 + palignr m5, m4, m2, 4 + pshufd m3, m4, q2121 + punpcklwd m0, m2, m5 ; 01 12 + punpckhwd m1, m2, m5 ; 23 34 + punpcklwd m2, m4, m3 ; 45 56 +.dy1_w4_loop: + movu xm11, [srcq+ssq*0] + vinserti128 m11, [srcq+ssq*1], 1 + lea srcq, [srcq+ssq*2] + pmaddwd m4, m0, m7 + pmaddwd m5, m1, m8 + pmaddwd m6, m2, m9 + mova m0, m1 + mova m1, m2 + paddd m4, m13 + paddd m5, m6 + pshufb m11, m14 + vpermq m11, m11, q3120 + pmaddubsw m11, m15 + phaddw m11, m11 + pmulhrsw m11, m12 + palignr m6, m11, m3, 12 + punpcklwd m2, m6, m11 ; 67 78 + mova m3, m11 + pmaddwd m6, m2, m10 + paddd m4, m5 + paddd m4, m6 + psrad m4, rndshift + vextracti128 xm5, m4, 1 + packssdw xm4, xm5 +%ifidn %1, put + packuswb xm4, xm4 + pshuflw xm4, xm4, q3120 + movd [dstq+dsq*0], xm4 + pextrd [dstq+dsq*1], xm4, 1 + lea dstq, [dstq+dsq*2] +%else + pshufd xm4, xm4, q3120 + mova [tmpq], xm4 + add tmpq, 16 +%endif + sub hd, 2 + jg .dy1_w4_loop + MC_8TAP_SCALED_RET +.dy1_w8: +%ifidn %1, put + movifnidn dsm, dsq +%endif + shr t0d, 16 + sub srcq, 3 + movd xm15, t0d + pmaddwd m8, [base+rescale_mul] + vpbroadcastq m11, [base+pq_0x40000000] + vpbroadcastd m15, xm15 + paddd m14, m8 ; mx+dx*[0-7] + pand m6, m14, m10 + psrld m6, 6 + paddd m15, m6 + pcmpeqd m6, m9 + vextracti128 xm7, m15, 1 + movd r4d, xm15 + pextrd r6d, xm15, 2 + pextrd r7d, xm15, 1 + pextrd r9d, xm15, 3 + movd r10d, xm7 + pextrd r11d, xm7, 2 + pextrd r13d, xm7, 1 + pextrd rXd, xm7, 3 + movq xm15, [base+subpel_filters+ r4*8] + movq xm10, [base+subpel_filters+ r6*8] + movhps xm15, [base+subpel_filters+ r7*8] + movhps xm10, [base+subpel_filters+ r9*8] + vinserti128 m15, [base+subpel_filters+r10*8], 1 + vinserti128 m10, [base+subpel_filters+r11*8], 1 + vpbroadcastq m9, [base+subpel_filters+r13*8] + vpbroadcastq m8, [base+subpel_filters+ rX*8] + psrld m14, 10 + vextracti128 xm7, m14, 1 + movd r4d, xm14 + pextrd r6d, xm14, 2 + pextrd r7d, xm14, 1 + pextrd r9d, xm14, 3 + movd r10d, xm7 + pextrd r11d, xm7, 2 + pextrd r13d, xm7, 1 + pextrd rXd, xm7, 3 + mov [rsp+32], r7d + pshufd m5, m6, q1100 + pshufd m6, m6, q3322 + vpblendd m15, m9, 0xc0 + vpblendd m10, m8, 0xc0 + pblendvb m15, m11, m5 + pblendvb m10, m11, m6 + vbroadcasti128 m14, [base+subpel_s_shuf8] + MC_8TAP_SCALED_H 0, 1, 2, 3, 4, 5, 6, 7 ; 0a 1a 0b 1b + MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 7, 8 ; 2a 3a 2b 3b + MC_8TAP_SCALED_H 2, 3, 4, 5, 6, 7, 8, 9 ; 4a 5a 4b 5b + MC_8TAP_SCALED_H 3, 4, 5, 6, 7, 8, 9, 11 ; 6a 7a 6b 7b + mov myd, mym + movu [rsp], m10 + pshufb m0, m14 ; 01a 01b + pshufb m1, m14 ; 23a 23b + pshufb m2, m14 ; 45a 45b + pshufb m3, m14 ; 67a 67b + shr myd, 6 + lea myd, [t1+myq] + mov t1d, 64 << 24 + cmovnz t1q, [base+subpel_filters+myq*8] + vbroadcasti128 m14, [base+wswap] + movq xm11, t1q + punpcklbw xm11, xm11 + psraw xm11, 8 + vinserti128 m11, xm11, 1 + mov r7d, [rsp+32] + pshufd m8, m11, q0000 + pshufd m9, m11, q1111 + pshufd m10, m11, q2222 + pshufd m11, m11, q3333 +.dy1_w8_loop: + pmaddwd m4, m0, m8 + pmaddwd m5, m1, m9 + pmaddwd m6, m2, m10 + pmaddwd m7, m3, m11 + paddd m4, m5 + paddd m6, m7 + paddd m4, m13 + paddd m4, m6 + psrad m4, rndshift + vextracti128 xm5, m4, 1 + packssdw xm4, xm5 +%ifidn %1, put + packuswb xm4, xm4 + movq [dstq], xm4 + add dstq, dsm +%else + mova [tmpq], xm4 + add tmpq, 16 +%endif + dec hd + jz .ret + movq xm4, [srcq+ r4] + movq xm5, [srcq+ r6] + movhps xm4, [srcq+ r7] + movhps xm5, [srcq+ r9] + vinserti128 m4, [srcq+r10], 1 + vinserti128 m5, [srcq+r11], 1 + vpbroadcastq m6, [srcq+r13] + vpbroadcastq m7, [srcq+ rX] + add srcq, ssq + pshufb m0, m14 + pshufb m1, m14 + pshufb m2, m14 + pshufb m3, m14 + vpblendd m4, m6, 0xc0 + vpblendd m5, m7, 0xc0 + pmaddubsw m4, m15 + pmaddubsw m5, [rsp] + phaddw m4, m5 + pslld m5, m4, 16 + paddw m4, m5 + pmulhrsw m4, m12 + pblendw m0, m1, 0xaa + pblendw m1, m2, 0xaa + pblendw m2, m3, 0xaa + pblendw m3, m4, 0xaa + jmp .dy1_w8_loop +.dy1_w16: + mov dword [rsp+72], 2 + movifprep tmp_stridem, 32 + jmp .dy1_w_start +.dy1_w32: + mov dword [rsp+72], 4 + movifprep tmp_stridem, 64 + jmp .dy1_w_start +.dy1_w64: + mov dword [rsp+72], 8 + movifprep tmp_stridem, 128 + jmp .dy1_w_start +.dy1_w128: + mov dword [rsp+72], 16 + movifprep tmp_stridem, 256 +.dy1_w_start: +%ifidn %1, put + movifnidn dsm, dsq +%endif + shr t0d, 16 + sub srcq, 3 + pmaddwd m8, [base+rescale_mul] + movd xm15, t0d + mov [rsp+76], t0d + mov [rsp+80], srcq + mov [rsp+88], r0q ; dstq / tmpq +%if UNIX64 + mov hm, hd +%endif + shl dword dxm, 3 ; dx*8 + vpbroadcastd m15, xm15 + paddd m14, m8 ; mx+dx*[0-7] + jmp .dy1_hloop +.dy1_hloop_prep: + dec dword [rsp+72] + jz .ret + add qword [rsp+88], 8*(isprep+1) + mov hd, hm + vpbroadcastd m8, dxm + vpbroadcastd m10, [base+pd_0x3ff] + paddd m14, m8, [rsp+32] + vpbroadcastd m15, [rsp+76] + pxor m9, m9 + mov srcq, [rsp+80] + mov r0q, [rsp+88] ; dstq / tmpq +.dy1_hloop: + vpbroadcastq m11, [base+pq_0x40000000] + pand m6, m14, m10 + psrld m6, 6 + paddd m15, m6 + pcmpeqd m6, m9 + vextracti128 xm7, m15, 1 + movd r4d, xm15 + pextrd r6d, xm15, 2 + pextrd r7d, xm15, 1 + pextrd r9d, xm15, 3 + movd r10d, xm7 + pextrd r11d, xm7, 2 + pextrd r13d, xm7, 1 + pextrd rXd, xm7, 3 + movu [rsp+32], m14 + movq xm15, [base+subpel_filters+ r4*8] + movq xm10, [base+subpel_filters+ r6*8] + movhps xm15, [base+subpel_filters+ r7*8] + movhps xm10, [base+subpel_filters+ r9*8] + vinserti128 m15, [base+subpel_filters+r10*8], 1 + vinserti128 m10, [base+subpel_filters+r11*8], 1 + vpbroadcastq m9, [base+subpel_filters+r13*8] + vpbroadcastq m8, [base+subpel_filters+ rX*8] + psrld m14, 10 + vextracti128 xm7, m14, 1 + movq [rsp+64], xm14 + movd r4d, xm14 + pextrd r6d, xm14, 2 + pextrd r7d, xm14, 1 + pextrd r9d, xm14, 3 + movd r10d, xm7 + pextrd r11d, xm7, 2 + pextrd r13d, xm7, 1 + pextrd rXd, xm7, 3 + pshufd m5, m6, q1100 + pshufd m6, m6, q3322 + vpblendd m15, m9, 0xc0 + vpblendd m10, m8, 0xc0 + pblendvb m15, m11, m5 + pblendvb m10, m11, m6 + vbroadcasti128 m14, [base+subpel_s_shuf8] + MC_8TAP_SCALED_H 0, 1, 2, 3, 4, 5, 6, 7 ; 0a 1a 0b 1b + MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 7, 8 ; 2a 3a 2b 3b + MC_8TAP_SCALED_H 2, 3, 4, 5, 6, 7, 8, 9 ; 4a 5a 4b 5b + MC_8TAP_SCALED_H 3, 4, 5, 6, 7, 8, 9, 11 ; 6a 7a 6b 7b + mov myd, mym + movu [rsp], m10 + pshufb m0, m14 ; 01a 01b + pshufb m1, m14 ; 23a 23b + pshufb m2, m14 ; 45a 45b + pshufb m3, m14 ; 67a 67b + shr myd, 6 + mov r4d, 64 << 24 + lea myd, [t1+myq] + cmovnz r4q, [base+subpel_filters+myq*8] + vbroadcasti128 m14, [base+wswap] + movq xm11, r4q + punpcklbw xm11, xm11 + psraw xm11, 8 + vinserti128 m11, xm11, 1 + mov r4d, [rsp+64] + mov r7d, [rsp+68] + pshufd m8, m11, q0000 + pshufd m9, m11, q1111 + pshufd m10, m11, q2222 + pshufd m11, m11, q3333 +.dy1_vloop: + pmaddwd m4, m0, m8 + pmaddwd m5, m1, m9 + pmaddwd m6, m2, m10 + pmaddwd m7, m3, m11 + paddd m4, m5 + paddd m6, m7 + paddd m4, m13 + paddd m4, m6 + psrad m4, rndshift + vextracti128 xm5, m4, 1 + packssdw xm4, xm5 +%ifidn %1, put + packuswb xm4, xm4 + movq [dstq], xm4 + add dstq, dsm +%else + mova [tmpq], xm4 + add tmpq, tmp_stridem +%endif + dec hd + jz .dy1_hloop_prep + movq xm4, [srcq+ r4] + movq xm5, [srcq+ r6] + movhps xm4, [srcq+ r7] + movhps xm5, [srcq+ r9] + vinserti128 m4, [srcq+r10], 1 + vinserti128 m5, [srcq+r11], 1 + vpbroadcastq m6, [srcq+r13] + vpbroadcastq m7, [srcq+ rX] + add srcq, ssq + pshufb m0, m14 + pshufb m1, m14 + pshufb m2, m14 + pshufb m3, m14 + vpblendd m4, m6, 0xc0 + vpblendd m5, m7, 0xc0 + pmaddubsw m4, m15 + pmaddubsw m5, [rsp] + phaddw m4, m5 + pslld m5, m4, 16 + paddw m4, m5 + pmulhrsw m4, m12 + pblendw m0, m1, 0xaa + pblendw m1, m2, 0xaa + pblendw m2, m3, 0xaa + pblendw m3, m4, 0xaa + jmp .dy1_vloop +.dy2: + movzx wd, word [base+%1_8tap_scaled_avx2_dy2_table+wq*2] + add wq, base_reg + jmp wq +%ifidn %1, put +.dy2_w2: + mov myd, mym + movzx t0d, t0b + dec srcq + movd xm15, t0d + punpckldq m8, m9, m8 + paddd m14, m8 ; mx+dx*[0-1] + vpbroadcastd m11, [base+pd_0x4000] + vpbroadcastd xm15, xm15 + pand m8, m14, m10 + psrld m8, 6 + paddd xm15, xm8 + movd r4d, xm15 + pextrd r6d, xm15, 1 + vbroadcasti128 m5, [base+bdct_lb_dw] + vbroadcasti128 m6, [base+subpel_s_shuf2] + vpbroadcastd m15, [base+subpel_filters+r4*8+2] + vpbroadcastd m7, [base+subpel_filters+r6*8+2] + pcmpeqd m8, m9 + psrld m14, 10 + movq xm0, [srcq+ssq*0] + vpbroadcastq m2, [srcq+ssq*1] + movhps xm0, [srcq+ssq*2] + vpbroadcastq m3, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + pshufb m14, m5 + paddb m14, m6 + vpblendd m15, m7, 0xaa + pblendvb m15, m11, m8 + movhps xm1, [srcq+ssq*0] + vpbroadcastq m4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + shr myd, 6 + mov r4d, 64 << 24 + lea myd, [t1+myq] + cmovnz r4q, [base+subpel_filters+myq*8] + vpblendd m0, m2, 0x30 + vpblendd m1, m4, 0xc0 + vpblendd m0, m3, 0xc0 + pshufb m0, m14 + pshufb m1, m14 + pmaddubsw m0, m15 + pmaddubsw m1, m15 + movq xm11, r4q + punpcklbw xm11, xm11 + psraw xm11, 8 + phaddw m0, m1 + pmulhrsw m0, m12 ; 0 2 _ 4 1 3 _ 5 + pshufd xm8, xm11, q0000 + pshufd xm9, xm11, q1111 + pshufd xm10, xm11, q2222 + pshufd xm11, xm11, q3333 + pshufd m2, m0, q3110 ; 0 2 2 4 1 3 3 5 + vextracti128 xm1, m2, 1 + punpcklwd xm3, xm2, xm1 ; 01 23 + punpckhwd xm2, xm1 ; 23 45 +.dy2_w2_loop: + movq xm6, [srcq+ssq*0] + vpbroadcastq m7, [srcq+ssq*1] + movhps xm6, [srcq+ssq*2] + vpbroadcastq m1, [srcq+ss3q ] + lea srcq, [srcq+ssq*4] + pmaddwd xm4, xm3, xm8 + pmaddwd xm5, xm2, xm9 + vpblendd m6, m7, 0x30 + vpblendd m6, m1, 0xc0 + pshufb m6, m14 + pmaddubsw m6, m15 + phaddw m6, m6 + pmulhrsw m6, m12 + palignr m0, m6, m0, 8 + pshufd m2, m0, q3221 + vextracti128 xm1, m2, 1 + punpcklwd xm3, xm2, xm1 ; 45 67 + punpckhwd xm2, xm1 ; 67 89 + pmaddwd xm6, xm3, xm10 + pmaddwd xm7, xm2, xm11 + paddd xm4, xm5 + paddd xm4, xm13 + paddd xm6, xm7 + paddd xm4, xm6 + psrad xm4, rndshift + packssdw xm4, xm4 + packuswb xm4, xm4 + pextrw [dstq+dsq*0], xm4, 0 + pextrw [dstq+dsq*1], xm4, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .dy2_w2_loop + RET +%endif +.dy2_w4: + mov myd, mym + vbroadcasti128 m7, [base+rescale_mul] + movzx t0d, t0b + dec srcq + movd xm15, t0d + pmaddwd m8, m7 + vpbroadcastd m11, [base+pd_0x4000] + vpbroadcastd xm15, xm15 + paddd m14, m8 ; mx+dx*[0-3] + pand m8, m14, m10 + psrld m8, 6 + paddd xm15, xm8 + movd r4d, xm15 + pextrd r6d, xm15, 1 + pextrd r11d, xm15, 2 + pextrd r13d, xm15, 3 + movd xm15, [base+subpel_filters+r4*8+2] + vbroadcasti128 m5, [base+bdct_lb_dw] + vpbroadcastq m6, [base+subpel_s_shuf2] + pinsrd xm15, [base+subpel_filters+r6*8+2], 1 + pcmpeqd m8, m9 + psrld m14, 10 + movu xm0, [srcq+ssq*0] + movu xm2, [srcq+ssq*2] + pinsrd xm15, [base+subpel_filters+r11*8+2], 2 + movu xm1, [srcq+ssq*1] + movu xm3, [srcq+ss3q ] + pinsrd xm15, [base+subpel_filters+r13*8+2], 3 + lea srcq, [srcq+ssq*4] + shr myd, 6 + mov r4d, 64 << 24 + lea myd, [t1+myq] + cmovnz r4q, [base+subpel_filters+myq*8] + vinserti128 m15, xm15, 1 + pshufb m14, m5 + paddb m14, m6 + vinserti128 m2, [srcq+ssq*0], 1 + vinserti128 m3, [srcq+ssq*1], 1 + lea srcq, [srcq+ssq*2] + pblendvb m15, m11, m8 + pshufb xm0, xm14 + pshufb m2, m14 + pshufb xm1, xm14 + pshufb m3, m14 + pmaddubsw xm0, xm15 + pmaddubsw m2, m15 + pmaddubsw xm1, xm15 + pmaddubsw m3, m15 + movq xm11, r4q + punpcklbw xm11, xm11 + psraw xm11, 8 + vinserti128 m11, xm11, 1 + phaddw m0, m2 + phaddw m1, m3 + pmulhrsw m0, m12 ; 0 2 _ 4 + pmulhrsw m1, m12 ; 1 3 _ 5 + pshufd m8, m11, q0000 + pshufd m9, m11, q1111 + pshufd m10, m11, q2222 + pshufd m11, m11, q3333 + punpcklwd xm2, xm0, xm1 + punpckhwd m1, m0, m1 ; 23 45 + vinserti128 m0, m2, xm1, 1 ; 01 23 +.dy2_w4_loop: + movu xm6, [srcq+ssq*0] + movu xm7, [srcq+ssq*1] + vinserti128 m6, [srcq+ssq*2], 1 + vinserti128 m7, [srcq+ss3q ], 1 + lea srcq, [srcq+ssq*4] + pmaddwd m4, m0, m8 + pmaddwd m5, m1, m9 + pshufb m6, m14 + pshufb m7, m14 + pmaddubsw m6, m15 + pmaddubsw m7, m15 + psrld m2, m6, 16 + pslld m3, m7, 16 + paddw m6, m2 + paddw m7, m3 + pblendw m6, m7, 0xaa ; 67 89 + pmulhrsw m6, m12 + paddd m4, m5 + vpblendd m0, m1, m6, 0x0f + mova m1, m6 + vpermq m0, m0, q1032 ; 45 67 + pmaddwd m6, m0, m10 + pmaddwd m7, m1, m11 + paddd m4, m13 + paddd m6, m7 + paddd m4, m6 + psrad m4, rndshift + vextracti128 xm5, m4, 1 + packssdw xm4, xm5 +%ifidn %1, put + packuswb xm4, xm4 + movd [dstq+dsq*0], xm4 + pextrd [dstq+dsq*1], xm4, 1 + lea dstq, [dstq+dsq*2] +%else + mova [tmpq], xm4 + add tmpq, 16 +%endif + sub hd, 2 + jg .dy2_w4_loop + MC_8TAP_SCALED_RET +.dy2_w8: +%ifidn %1, put + movifnidn dsm, dsq +%endif + shr t0d, 16 + sub srcq, 3 + movd xm15, t0d + pmaddwd m8, [base+rescale_mul] + vpbroadcastq m11, [base+pq_0x40000000] + vpbroadcastd m15, xm15 + paddd m14, m8 ; mx+dx*[0-7] + pand m6, m14, m10 + psrld m6, 6 + paddd m15, m6 + pcmpeqd m6, m9 + vextracti128 xm7, m15, 1 + movd r4d, xm15 + pextrd r6d, xm15, 2 + pextrd r7d, xm15, 1 + pextrd r9d, xm15, 3 + movd r10d, xm7 + pextrd r11d, xm7, 2 + pextrd r13d, xm7, 1 + pextrd rXd, xm7, 3 + movq xm15, [base+subpel_filters+ r4*8] + movq xm10, [base+subpel_filters+ r6*8] + movhps xm15, [base+subpel_filters+ r7*8] + movhps xm10, [base+subpel_filters+ r9*8] + vinserti128 m15, [base+subpel_filters+r10*8], 1 + vinserti128 m10, [base+subpel_filters+r11*8], 1 + vpbroadcastq m9, [base+subpel_filters+r13*8] + vpbroadcastq m8, [base+subpel_filters+ rX*8] + psrld m14, 10 + vextracti128 xm7, m14, 1 + movd r4d, xm14 + pextrd r6d, xm14, 2 + pextrd r7d, xm14, 1 + pextrd r9d, xm14, 3 + movd r10d, xm7 + pextrd r11d, xm7, 2 + pextrd r13d, xm7, 1 + pextrd rXd, xm7, 3 + mov [rsp], r7d + pshufd m5, m6, q1100 + pshufd m6, m6, q3322 + vpblendd m15, m9, 0xc0 + vpblendd m10, m8, 0xc0 + pblendvb m15, m11, m5 + pblendvb m10, m11, m6 + vbroadcasti128 m14, [base+subpel_s_shuf8] + MC_8TAP_SCALED_H 0, 1, 2, 3, 4, 5, 6, 7 ; 0a 1a 0b 1b + MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 7, 8 ; 2a 3a 2b 3b + MC_8TAP_SCALED_H 2, 3, 4, 5, 6, 7, 8, 9 ; 4a 5a 4b 5b + MC_8TAP_SCALED_H 3, 4, 5, 6, 7, 8, 9, 11 ; 6a 7a 6b 7b + mov myd, mym + pshufb m0, m14 ; 01a 01b + pshufb m1, m14 ; 23a 23b + pshufb m2, m14 ; 45a 45b + pshufb m3, m14 ; 67a 67b + shr myd, 6 + lea myd, [t1+myq] + mov t1d, 64 << 24 + cmovnz t1q, [base+subpel_filters+myq*8] + movq xm11, t1q + punpcklbw xm11, xm11 + psraw xm11, 8 + vinserti128 m11, xm11, 1 + mov r7d, [rsp] + pshufd m8, m11, q0000 + pshufd m9, m11, q1111 + pshufd m14, m11, q2222 + pshufd m11, m11, q3333 +.dy2_w8_loop: + pmaddwd m4, m0, m8 + pmaddwd m5, m1, m9 + pmaddwd m6, m2, m14 + pmaddwd m7, m3, m11 + paddd m4, m5 + paddd m6, m7 + paddd m4, m13 + paddd m4, m6 + psrad m4, rndshift + vextracti128 xm5, m4, 1 + packssdw xm4, xm5 +%ifidn %1, put + packuswb xm4, xm4 + movq [dstq], xm4 + add dstq, dsm +%else + mova [tmpq], xm4 + add tmpq, 16 +%endif + dec hd + jz .ret + mova m0, m1 + mova m1, m2 + mova m2, m3 + movq xm3, [srcq+ r4] + movq xm4, [srcq+ r6] + movhps xm3, [srcq+ r7] + movhps xm4, [srcq+ r9] + vinserti128 m3, [srcq+r10], 1 + vinserti128 m4, [srcq+r11], 1 + vpbroadcastq m5, [srcq+r13] + vpbroadcastq m6, [srcq+ rX] + add srcq, ssq + vpblendd m3, m5, 0xc0 + vpblendd m4, m6, 0xc0 + pmaddubsw m3, m15 + pmaddubsw m4, m10 + phaddw m3, m4 + movq xm4, [srcq+ r4] + movq xm5, [srcq+ r6] + movhps xm4, [srcq+ r7] + movhps xm5, [srcq+ r9] + vinserti128 m4, [srcq+r10], 1 + vinserti128 m5, [srcq+r11], 1 + vpbroadcastq m6, [srcq+r13] + vpbroadcastq m7, [srcq+ rX] + add srcq, ssq + vpblendd m4, m6, 0xc0 + vpblendd m5, m7, 0xc0 + pmaddubsw m4, m15 + pmaddubsw m5, m10 + phaddw m4, m5 + psrld m5, m3, 16 + pslld m6, m4, 16 + paddw m3, m5 + paddw m4, m6 + pblendw m3, m4, 0xaa + pmulhrsw m3, m12 + jmp .dy2_w8_loop +.dy2_w16: + mov dword [rsp+40], 2 + movifprep tmp_stridem, 32 + jmp .dy2_w_start +.dy2_w32: + mov dword [rsp+40], 4 + movifprep tmp_stridem, 64 + jmp .dy2_w_start +.dy2_w64: + mov dword [rsp+40], 8 + movifprep tmp_stridem, 128 + jmp .dy2_w_start +.dy2_w128: + mov dword [rsp+40], 16 + movifprep tmp_stridem, 256 +.dy2_w_start: +%ifidn %1, put + movifnidn dsm, dsq +%endif + shr t0d, 16 + sub srcq, 3 + pmaddwd m8, [base+rescale_mul] + movd xm15, t0d + mov [rsp+64], t0d + mov [rsp+48], srcq + mov [rsp+56], r0q ; dstq / tmpq +%if UNIX64 + mov hm, hd +%endif + shl dword dxm, 3 ; dx*8 + vpbroadcastd m15, xm15 + paddd m14, m8 ; mx+dx*[0-7] + jmp .dy2_hloop +.dy2_hloop_prep: + dec dword [rsp+40] + jz .ret + add qword [rsp+56], 8*(isprep+1) + mov hd, hm + vpbroadcastd m8, dxm + vpbroadcastd m10, [base+pd_0x3ff] + paddd m14, m8, [rsp] + vpbroadcastd m15, [rsp+64] + pxor m9, m9 + mov srcq, [rsp+48] + mov r0q, [rsp+56] ; dstq / tmpq +.dy2_hloop: + vpbroadcastq m11, [base+pq_0x40000000] + pand m6, m14, m10 + psrld m6, 6 + paddd m15, m6 + pcmpeqd m6, m9 + vextracti128 xm7, m15, 1 + movd r4d, xm15 + pextrd r6d, xm15, 2 + pextrd r7d, xm15, 1 + pextrd r9d, xm15, 3 + movd r10d, xm7 + pextrd r11d, xm7, 2 + pextrd r13d, xm7, 1 + pextrd rXd, xm7, 3 + movu [rsp], m14 + movq xm15, [base+subpel_filters+ r4*8] + movq xm10, [base+subpel_filters+ r6*8] + movhps xm15, [base+subpel_filters+ r7*8] + movhps xm10, [base+subpel_filters+ r9*8] + vinserti128 m15, [base+subpel_filters+r10*8], 1 + vinserti128 m10, [base+subpel_filters+r11*8], 1 + vpbroadcastq m9, [base+subpel_filters+r13*8] + vpbroadcastq m8, [base+subpel_filters+ rX*8] + psrld m14, 10 + vextracti128 xm7, m14, 1 + movq [rsp+32], xm14 + movd r4d, xm14 + pextrd r6d, xm14, 2 + pextrd r7d, xm14, 1 + pextrd r9d, xm14, 3 + movd r10d, xm7 + pextrd r11d, xm7, 2 + pextrd r13d, xm7, 1 + pextrd rXd, xm7, 3 + pshufd m5, m6, q1100 + pshufd m6, m6, q3322 + vpblendd m15, m9, 0xc0 + vpblendd m10, m8, 0xc0 + pblendvb m15, m11, m5 + pblendvb m10, m11, m6 + vbroadcasti128 m14, [base+subpel_s_shuf8] + MC_8TAP_SCALED_H 0, 1, 2, 3, 4, 5, 6, 7 ; 0a 1a 0b 1b + MC_8TAP_SCALED_H 1, 2, 3, 4, 5, 6, 7, 8 ; 2a 3a 2b 3b + MC_8TAP_SCALED_H 2, 3, 4, 5, 6, 7, 8, 9 ; 4a 5a 4b 5b + MC_8TAP_SCALED_H 3, 4, 5, 6, 7, 8, 9, 11 ; 6a 7a 6b 7b + mov myd, mym + pshufb m0, m14 ; 01a 01b + pshufb m1, m14 ; 23a 23b + pshufb m2, m14 ; 45a 45b + pshufb m3, m14 ; 67a 67b + shr myd, 6 + mov r4d, 64 << 24 + lea myd, [t1+myq] + cmovnz r4q, [base+subpel_filters+myq*8] + movq xm14, r4q + punpcklbw xm14, xm14 + psraw xm14, 8 + vinserti128 m14, xm14, 1 + mov r4d, [rsp+32] + mov r7d, [rsp+36] + pshufd m8, m14, q0000 + pshufd m9, m14, q1111 + pshufd m11, m14, q2222 + pshufd m14, m14, q3333 +.dy2_vloop: + pmaddwd m4, m0, m8 + pmaddwd m5, m1, m9 + pmaddwd m6, m2, m11 + pmaddwd m7, m3, m14 + paddd m4, m5 + paddd m6, m7 + paddd m4, m13 + paddd m4, m6 + psrad m4, rndshift + vextracti128 xm5, m4, 1 + packssdw xm4, xm5 +%ifidn %1, put + packuswb xm4, xm4 + movq [dstq], xm4 + add dstq, dsm +%else + mova [tmpq], xm4 + add tmpq, tmp_stridem +%endif + dec hd + jz .dy2_hloop_prep + mova m0, m1 + mova m1, m2 + mova m2, m3 + movq xm3, [srcq+ r4] + movq xm4, [srcq+ r6] + movhps xm3, [srcq+ r7] + movhps xm4, [srcq+ r9] + vinserti128 m3, [srcq+r10], 1 + vinserti128 m4, [srcq+r11], 1 + vpbroadcastq m5, [srcq+r13] + vpbroadcastq m6, [srcq+ rX] + add srcq, ssq + vpblendd m3, m5, 0xc0 + vpblendd m4, m6, 0xc0 + pmaddubsw m3, m15 + pmaddubsw m4, m10 + phaddw m3, m4 + movq xm4, [srcq+ r4] + movq xm5, [srcq+ r6] + movhps xm4, [srcq+ r7] + movhps xm5, [srcq+ r9] + vinserti128 m4, [srcq+r10], 1 + vinserti128 m5, [srcq+r11], 1 + vpbroadcastq m6, [srcq+r13] + vpbroadcastq m7, [srcq+ rX] + add srcq, ssq + vpblendd m4, m6, 0xc0 + vpblendd m5, m7, 0xc0 + pmaddubsw m4, m15 + pmaddubsw m5, m10 + phaddw m4, m5 + psrld m5, m3, 16 + pslld m6, m4, 16 + paddw m3, m5 + paddw m4, m6 + pblendw m3, m4, 0xaa + pmulhrsw m3, m12 + jmp .dy2_vloop +.ret: + MC_8TAP_SCALED_RET 0 +%undef isprep +%endmacro + +%macro BILIN_SCALED_FN 1 +cglobal %1_bilin_scaled + mov t0d, (5*15 << 16) | 5*15 + mov t1d, t0d + jmp mangle(private_prefix %+ _%1_8tap_scaled %+ SUFFIX) +%endmacro + +%if WIN64 +DECLARE_REG_TMP 6, 5 +%else +DECLARE_REG_TMP 6, 8 +%endif + +%define PUT_8TAP_SCALED_FN FN put_8tap_scaled, +%define PREP_8TAP_SCALED_FN FN prep_8tap_scaled, + +BILIN_SCALED_FN put +PUT_8TAP_SCALED_FN sharp, SHARP, SHARP +PUT_8TAP_SCALED_FN sharp_smooth, SHARP, SMOOTH +PUT_8TAP_SCALED_FN smooth_sharp, SMOOTH, SHARP +PUT_8TAP_SCALED_FN smooth, SMOOTH, SMOOTH +PUT_8TAP_SCALED_FN sharp_regular, SHARP, REGULAR +PUT_8TAP_SCALED_FN regular_sharp, REGULAR, SHARP +PUT_8TAP_SCALED_FN smooth_regular, SMOOTH, REGULAR +PUT_8TAP_SCALED_FN regular_smooth, REGULAR, SMOOTH +PUT_8TAP_SCALED_FN regular, REGULAR, REGULAR +MC_8TAP_SCALED put + +%if WIN64 +DECLARE_REG_TMP 5, 4 +%else +DECLARE_REG_TMP 6, 7 +%endif + +BILIN_SCALED_FN prep +PREP_8TAP_SCALED_FN sharp, SHARP, SHARP +PREP_8TAP_SCALED_FN sharp_smooth, SHARP, SMOOTH +PREP_8TAP_SCALED_FN smooth_sharp, SMOOTH, SHARP +PREP_8TAP_SCALED_FN smooth, SMOOTH, SMOOTH +PREP_8TAP_SCALED_FN sharp_regular, SHARP, REGULAR +PREP_8TAP_SCALED_FN regular_sharp, REGULAR, SHARP +PREP_8TAP_SCALED_FN smooth_regular, SMOOTH, REGULAR +PREP_8TAP_SCALED_FN regular_smooth, REGULAR, SMOOTH +PREP_8TAP_SCALED_FN regular, REGULAR, REGULAR +MC_8TAP_SCALED prep + +%macro WARP_V 5 ; dst, 02, 46, 13, 57 + ; Can be done using gathers, but that's terribly slow on many CPU:s + lea tmp1d, [myq+deltaq*4] + lea tmp2d, [myq+deltaq*1] + shr myd, 10 + shr tmp1d, 10 + movq xm8, [filterq+myq *8] + vinserti128 m8, [filterq+tmp1q*8], 1 ; a e + lea tmp1d, [tmp2q+deltaq*4] + lea myd, [tmp2q+deltaq*1] + shr tmp2d, 10 + shr tmp1d, 10 + movq xm0, [filterq+tmp2q*8] + vinserti128 m0, [filterq+tmp1q*8], 1 ; b f + lea tmp1d, [myq+deltaq*4] + lea tmp2d, [myq+deltaq*1] + shr myd, 10 + shr tmp1d, 10 + movq xm9, [filterq+myq *8] + vinserti128 m9, [filterq+tmp1q*8], 1 ; c g + lea tmp1d, [tmp2q+deltaq*4] + lea myd, [tmp2q+gammaq] ; my += gamma + shr tmp2d, 10 + shr tmp1d, 10 + punpcklwd m8, m0 + movq xm0, [filterq+tmp2q*8] + vinserti128 m0, [filterq+tmp1q*8], 1 ; d h + punpcklwd m0, m9, m0 + punpckldq m9, m8, m0 + punpckhdq m0, m8, m0 + punpcklbw m8, m11, m9 ; a0 a2 b0 b2 c0 c2 d0 d2 << 8 + punpckhbw m9, m11, m9 ; a4 a6 b4 b6 c4 c6 d4 d6 << 8 + pmaddwd m%2, m8 + pmaddwd m9, m%3 + punpcklbw m8, m11, m0 ; a1 a3 b1 b3 c1 c3 d1 d3 << 8 + punpckhbw m0, m11, m0 ; a5 a7 b5 b7 c5 c7 d5 d7 << 8 + pmaddwd m8, m%4 + pmaddwd m0, m%5 + paddd m%2, m9 + paddd m0, m8 + paddd m%1, m0, m%2 +%endmacro + +cglobal warp_affine_8x8t, 0, 14, 0, tmp, ts +%if WIN64 + sub rsp, 0xa0 +%endif + call mangle(private_prefix %+ _warp_affine_8x8_avx2).main +.loop: + psrad m7, 13 + psrad m0, 13 + packssdw m7, m0 + pmulhrsw m7, m14 ; (x + (1 << 6)) >> 7 + vpermq m7, m7, q3120 + mova [tmpq+tsq*0], xm7 + vextracti128 [tmpq+tsq*2], m7, 1 + dec r4d + jz mangle(private_prefix %+ _warp_affine_8x8_avx2).end + call mangle(private_prefix %+ _warp_affine_8x8_avx2).main2 + lea tmpq, [tmpq+tsq*4] + jmp .loop + +cglobal warp_affine_8x8, 0, 14, 0, dst, ds, src, ss, abcd, mx, tmp2, alpha, \ + beta, filter, tmp1, delta, my, gamma +%if WIN64 + sub rsp, 0xa0 + %assign xmm_regs_used 16 + %assign stack_size_padded 0xa0 + %assign stack_offset stack_offset+stack_size_padded +%endif + call .main + jmp .start +.loop: + call .main2 + lea dstq, [dstq+dsq*2] +.start: + psrad m7, 18 + psrad m0, 18 + packusdw m7, m0 + pavgw m7, m11 ; (x + (1 << 10)) >> 11 + vextracti128 xm0, m7, 1 + packuswb xm7, xm0 + pshufd xm7, xm7, q3120 + movq [dstq+dsq*0], xm7 + movhps [dstq+dsq*1], xm7 + dec r4d + jg .loop +.end: + RET +ALIGN function_align +.main: + ; Stack args offset by one (r4m -> r5m etc.) due to call +%if WIN64 + mov abcdq, r5m + mov mxd, r6m + movaps [rsp+stack_offset+0x10], xmm6 + movaps [rsp+stack_offset+0x20], xmm7 + movaps [rsp+0x28], xmm8 + movaps [rsp+0x38], xmm9 + movaps [rsp+0x48], xmm10 + movaps [rsp+0x58], xmm11 + movaps [rsp+0x68], xmm12 + movaps [rsp+0x78], xmm13 + movaps [rsp+0x88], xmm14 + movaps [rsp+0x98], xmm15 +%endif + movsx alphad, word [abcdq+2*0] + movsx betad, word [abcdq+2*1] + mova m12, [warp_8x8_shufA] + mova m13, [warp_8x8_shufB] + vpbroadcastd m14, [pw_8192] + vpbroadcastd m15, [pd_32768] + pxor m11, m11 + lea filterq, [mc_warp_filter] + lea tmp1q, [ssq*3+3] + add mxd, 512+(64<<10) + lea tmp2d, [alphaq*3] + sub srcq, tmp1q ; src -= src_stride*3 + 3 + sub betad, tmp2d ; beta -= alpha*3 + mov myd, r7m + call .h + psrld m1, m0, 16 + call .h + psrld m4, m0, 16 + call .h + pblendw m1, m0, 0xaa ; 02 + call .h + pblendw m4, m0, 0xaa ; 13 + call .h + psrld m2, m1, 16 + pblendw m2, m0, 0xaa ; 24 + call .h + psrld m5, m4, 16 + pblendw m5, m0, 0xaa ; 35 + call .h + psrld m3, m2, 16 + pblendw m3, m0, 0xaa ; 46 + movsx deltad, word [abcdq+2*2] + movsx gammad, word [abcdq+2*3] + add myd, 512+(64<<10) + mov r4d, 4 + lea tmp1d, [deltaq*3] + sub gammad, tmp1d ; gamma -= delta*3 +.main2: + call .h + psrld m6, m5, 16 + pblendw m6, m0, 0xaa ; 57 + WARP_V 7, 1, 3, 4, 6 + call .h + mova m1, m2 + mova m2, m3 + psrld m3, 16 + pblendw m3, m0, 0xaa ; 68 + WARP_V 0, 4, 6, 1, 3 + mova m4, m5 + mova m5, m6 + ret +ALIGN function_align +.h: + lea tmp1d, [mxq+alphaq*4] + lea tmp2d, [mxq+alphaq*1] + vbroadcasti128 m10, [srcq] + shr mxd, 10 + shr tmp1d, 10 + movq xm8, [filterq+mxq *8] + vinserti128 m8, [filterq+tmp1q*8], 1 + lea tmp1d, [tmp2q+alphaq*4] + lea mxd, [tmp2q+alphaq*1] + shr tmp2d, 10 + shr tmp1d, 10 + movq xm0, [filterq+tmp2q*8] + vinserti128 m0, [filterq+tmp1q*8], 1 + lea tmp1d, [mxq+alphaq*4] + lea tmp2d, [mxq+alphaq*1] + shr mxd, 10 + shr tmp1d, 10 + movq xm9, [filterq+mxq *8] + vinserti128 m9, [filterq+tmp1q*8], 1 + lea tmp1d, [tmp2q+alphaq*4] + lea mxd, [tmp2q+betaq] ; mx += beta + shr tmp2d, 10 + shr tmp1d, 10 + punpcklqdq m8, m0 ; 0 1 4 5 + movq xm0, [filterq+tmp2q*8] + vinserti128 m0, [filterq+tmp1q*8], 1 + punpcklqdq m9, m0 ; 2 3 6 7 + pshufb m0, m10, m12 + pmaddubsw m0, m8 + pshufb m10, m13 + pmaddubsw m10, m9 + add srcq, ssq + phaddw m0, m10 + pmaddwd m0, m14 ; 17-bit intermediate, upshifted by 13 + paddd m0, m15 ; rounded 14-bit result in upper 16 bits of dword + ret + +%macro BIDIR_FN 1 ; op + %1 0 + lea stride3q, [strideq*3] + jmp wq +.w4: + vextracti128 xm1, m0, 1 + movd [dstq ], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xm1 + pextrd [dstq+stride3q ], xm1, 1 + cmp hd, 4 + je .ret + lea dstq, [dstq+strideq*4] + pextrd [dstq ], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xm1, 2 + pextrd [dstq+stride3q ], xm1, 3 + cmp hd, 8 + je .ret + %1 2 + lea dstq, [dstq+strideq*4] + vextracti128 xm1, m0, 1 + movd [dstq ], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xm1 + pextrd [dstq+stride3q ], xm1, 1 + lea dstq, [dstq+strideq*4] + pextrd [dstq ], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xm1, 2 + pextrd [dstq+stride3q ], xm1, 3 +.ret: + RET +.w8_loop: + %1_INC_PTR 2 + %1 0 + lea dstq, [dstq+strideq*4] +.w8: + vextracti128 xm1, m0, 1 + movq [dstq ], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xm1 + sub hd, 4 + jg .w8_loop + RET +.w16_loop: + %1_INC_PTR 4 + %1 0 + lea dstq, [dstq+strideq*4] +.w16: + vpermq m0, m0, q3120 + mova [dstq ], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + %1 2 + vpermq m0, m0, q3120 + mova [dstq+strideq*2], xm0 + vextracti128 [dstq+stride3q ], m0, 1 + sub hd, 4 + jg .w16_loop + RET +.w32_loop: + %1_INC_PTR 4 + %1 0 + lea dstq, [dstq+strideq*2] +.w32: + vpermq m0, m0, q3120 + mova [dstq+strideq*0], m0 + %1 2 + vpermq m0, m0, q3120 + mova [dstq+strideq*1], m0 + sub hd, 2 + jg .w32_loop + RET +.w64_loop: + %1_INC_PTR 4 + %1 0 + add dstq, strideq +.w64: + vpermq m0, m0, q3120 + mova [dstq], m0 + %1 2 + vpermq m0, m0, q3120 + mova [dstq+32], m0 + dec hd + jg .w64_loop + RET +.w128_loop: + %1 0 + add dstq, strideq +.w128: + vpermq m0, m0, q3120 + mova [dstq+0*32], m0 + %1 2 + vpermq m0, m0, q3120 + mova [dstq+1*32], m0 + %1_INC_PTR 8 + %1 -4 + vpermq m0, m0, q3120 + mova [dstq+2*32], m0 + %1 -2 + vpermq m0, m0, q3120 + mova [dstq+3*32], m0 + dec hd + jg .w128_loop + RET +%endmacro + +%macro AVG 1 ; src_offset + mova m0, [tmp1q+(%1+0)*32] + paddw m0, [tmp2q+(%1+0)*32] + mova m1, [tmp1q+(%1+1)*32] + paddw m1, [tmp2q+(%1+1)*32] + pmulhrsw m0, m2 + pmulhrsw m1, m2 + packuswb m0, m1 +%endmacro + +%macro AVG_INC_PTR 1 + add tmp1q, %1*32 + add tmp2q, %1*32 +%endmacro + +cglobal avg, 4, 7, 3, dst, stride, tmp1, tmp2, w, h, stride3 +%define base r6-avg %+ SUFFIX %+ _table + lea r6, [avg %+ SUFFIX %+ _table] + tzcnt wd, wm + movifnidn hd, hm + movsxd wq, dword [r6+wq*4] + vpbroadcastd m2, [base+pw_1024] + add wq, r6 + BIDIR_FN AVG + +%macro W_AVG 1 ; src_offset + ; (a * weight + b * (16 - weight) + 128) >> 8 + ; = ((a - b) * weight + (b << 4) + 128) >> 8 + ; = ((((a - b) * ((weight-16) << 12)) >> 16) + a + 8) >> 4 + ; = ((((b - a) * (-weight << 12)) >> 16) + b + 8) >> 4 + mova m0, [tmp1q+(%1+0)*32] + psubw m2, m0, [tmp2q+(%1+0)*32] + mova m1, [tmp1q+(%1+1)*32] + psubw m3, m1, [tmp2q+(%1+1)*32] + pmulhw m2, m4 + pmulhw m3, m4 + paddw m0, m2 + paddw m1, m3 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + packuswb m0, m1 +%endmacro + +%define W_AVG_INC_PTR AVG_INC_PTR + +cglobal w_avg, 4, 7, 6, dst, stride, tmp1, tmp2, w, h, stride3 +%define base r6-w_avg %+ SUFFIX %+ _table + lea r6, [w_avg %+ SUFFIX %+ _table] + tzcnt wd, wm + movifnidn hd, hm + vpbroadcastw m4, r6m ; weight + movsxd wq, dword [r6+wq*4] + vpbroadcastd m5, [base+pw_2048] + psllw m4, 12 ; (weight-16) << 12 when interpreted as signed + add wq, r6 + cmp dword r6m, 7 + jg .weight_gt7 + mov r6, tmp1q + pxor m0, m0 + mov tmp1q, tmp2q + psubw m4, m0, m4 ; -weight + mov tmp2q, r6 +.weight_gt7: + BIDIR_FN W_AVG + +%macro MASK 1 ; src_offset + ; (a * m + b * (64 - m) + 512) >> 10 + ; = ((a - b) * m + (b << 6) + 512) >> 10 + ; = ((((b - a) * (-m << 10)) >> 16) + b + 8) >> 4 + vpermq m3, [maskq+%1*16], q3120 + mova m0, [tmp2q+(%1+0)*32] + psubw m1, m0, [tmp1q+(%1+0)*32] + psubb m3, m4, m3 + paddw m1, m1 ; (b - a) << 1 + paddb m3, m3 + punpcklbw m2, m4, m3 ; -m << 9 + pmulhw m1, m2 + paddw m0, m1 + mova m1, [tmp2q+(%1+1)*32] + psubw m2, m1, [tmp1q+(%1+1)*32] + paddw m2, m2 + punpckhbw m3, m4, m3 + pmulhw m2, m3 + paddw m1, m2 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + packuswb m0, m1 +%endmacro + +%macro MASK_INC_PTR 1 + add maskq, %1*16 + add tmp2q, %1*32 + add tmp1q, %1*32 +%endmacro + +cglobal mask, 4, 8, 6, dst, stride, tmp1, tmp2, w, h, mask, stride3 +%define base r7-mask %+ SUFFIX %+ _table + lea r7, [mask %+ SUFFIX %+ _table] + tzcnt wd, wm + movifnidn hd, hm + mov maskq, maskmp + movsxd wq, dword [r7+wq*4] + vpbroadcastd m5, [base+pw_2048] + pxor m4, m4 + add wq, r7 + BIDIR_FN MASK + +%macro W_MASK 4-5 0 ; dst, mask, tmp_offset[1-2], 4:4:4 + mova m%1, [tmp1q+32*%3] + mova m1, [tmp2q+32*%3] + psubw m1, m%1 + pabsw m%2, m1 + psubusw m%2, m6, m%2 + psrlw m%2, 8 ; 64 - m + psllw m2, m%2, 10 + pmulhw m1, m2 + paddw m%1, m1 + mova m1, [tmp1q+32*%4] + mova m2, [tmp2q+32*%4] + psubw m2, m1 + pabsw m3, m2 + psubusw m3, m6, m3 + psrlw m3, 8 +%if %5 + packuswb m%2, m3 + psubb m%2, m5, m%2 + vpermq m%2, m%2, q3120 +%else + phaddw m%2, m3 +%endif + psllw m3, 10 + pmulhw m2, m3 + paddw m1, m2 + pmulhrsw m%1, m7 + pmulhrsw m1, m7 + packuswb m%1, m1 +%endmacro + +cglobal blend, 3, 7, 7, dst, ds, tmp, w, h, mask +%define base r6-blend_avx2_table + lea r6, [blend_avx2_table] + tzcnt wd, wm + movifnidn hd, hm + movifnidn maskq, maskmp + movsxd wq, dword [r6+wq*4] + vpbroadcastd m4, [base+pb_64] + vpbroadcastd m5, [base+pw_512] + add wq, r6 + lea r6, [dsq*3] + jmp wq +.w4: + movd xm0, [dstq+dsq*0] + pinsrd xm0, [dstq+dsq*1], 1 + vpbroadcastd xm1, [dstq+dsq*2] + pinsrd xm1, [dstq+r6 ], 3 + mova xm6, [maskq] + psubb xm3, xm4, xm6 + punpcklbw xm2, xm3, xm6 + punpckhbw xm3, xm6 + mova xm6, [tmpq] + add maskq, 4*4 + add tmpq, 4*4 + punpcklbw xm0, xm6 + punpckhbw xm1, xm6 + pmaddubsw xm0, xm2 + pmaddubsw xm1, xm3 + pmulhrsw xm0, xm5 + pmulhrsw xm1, xm5 + packuswb xm0, xm1 + movd [dstq+dsq*0], xm0 + pextrd [dstq+dsq*1], xm0, 1 + pextrd [dstq+dsq*2], xm0, 2 + pextrd [dstq+r6 ], xm0, 3 + lea dstq, [dstq+dsq*4] + sub hd, 4 + jg .w4 + RET +ALIGN function_align +.w8: + movq xm1, [dstq+dsq*0] + movhps xm1, [dstq+dsq*1] + vpbroadcastq m2, [dstq+dsq*2] + vpbroadcastq m3, [dstq+r6 ] + mova m0, [maskq] + mova m6, [tmpq] + add maskq, 8*4 + add tmpq, 8*4 + vpblendd m1, m2, 0x30 + vpblendd m1, m3, 0xc0 + psubb m3, m4, m0 + punpcklbw m2, m3, m0 + punpckhbw m3, m0 + punpcklbw m0, m1, m6 + punpckhbw m1, m6 + pmaddubsw m0, m2 + pmaddubsw m1, m3 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + packuswb m0, m1 + vextracti128 xm1, m0, 1 + movq [dstq+dsq*0], xm0 + movhps [dstq+dsq*1], xm0 + movq [dstq+dsq*2], xm1 + movhps [dstq+r6 ], xm1 + lea dstq, [dstq+dsq*4] + sub hd, 4 + jg .w8 + RET +ALIGN function_align +.w16: + mova m0, [maskq] + mova xm1, [dstq+dsq*0] + vinserti128 m1, [dstq+dsq*1], 1 + psubb m3, m4, m0 + punpcklbw m2, m3, m0 + punpckhbw m3, m0 + mova m6, [tmpq] + add maskq, 16*2 + add tmpq, 16*2 + punpcklbw m0, m1, m6 + punpckhbw m1, m6 + pmaddubsw m0, m2 + pmaddubsw m1, m3 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + packuswb m0, m1 + mova [dstq+dsq*0], xm0 + vextracti128 [dstq+dsq*1], m0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w16 + RET +ALIGN function_align +.w32: + mova m0, [maskq] + mova m1, [dstq] + mova m6, [tmpq] + add maskq, 32 + add tmpq, 32 + psubb m3, m4, m0 + punpcklbw m2, m3, m0 + punpckhbw m3, m0 + punpcklbw m0, m1, m6 + punpckhbw m1, m6 + pmaddubsw m0, m2 + pmaddubsw m1, m3 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + packuswb m0, m1 + mova [dstq], m0 + add dstq, dsq + dec hd + jg .w32 + RET + +cglobal blend_v, 3, 6, 6, dst, ds, tmp, w, h, mask +%define base r5-blend_v_avx2_table + lea r5, [blend_v_avx2_table] + tzcnt wd, wm + movifnidn hd, hm + movsxd wq, dword [r5+wq*4] + vpbroadcastd m5, [base+pw_512] + add wq, r5 + add maskq, obmc_masks-blend_v_avx2_table + jmp wq +.w2: + vpbroadcastd xm2, [maskq+2*2] +.w2_s0_loop: + movd xm0, [dstq+dsq*0] + pinsrw xm0, [dstq+dsq*1], 1 + movd xm1, [tmpq] + add tmpq, 2*2 + punpcklbw xm0, xm1 + pmaddubsw xm0, xm2 + pmulhrsw xm0, xm5 + packuswb xm0, xm0 + pextrw [dstq+dsq*0], xm0, 0 + pextrw [dstq+dsq*1], xm0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w2_s0_loop + RET +ALIGN function_align +.w4: + vpbroadcastq xm2, [maskq+4*2] +.w4_loop: + movd xm0, [dstq+dsq*0] + pinsrd xm0, [dstq+dsq*1], 1 + movq xm1, [tmpq] + add tmpq, 4*2 + punpcklbw xm0, xm1 + pmaddubsw xm0, xm2 + pmulhrsw xm0, xm5 + packuswb xm0, xm0 + movd [dstq+dsq*0], xm0 + pextrd [dstq+dsq*1], xm0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w4_loop + RET +ALIGN function_align +.w8: + vbroadcasti128 m4, [maskq+8*2] +.w8_loop: + vpbroadcastq m2, [dstq+dsq*0] + movq xm0, [dstq+dsq*1] + vpblendd m0, m2, 0x30 + movq xm1, [tmpq+8*1] + vinserti128 m1, [tmpq+8*0], 1 + add tmpq, 8*2 + punpcklbw m0, m1 + pmaddubsw m0, m4 + pmulhrsw m0, m5 + vextracti128 xm1, m0, 1 + packuswb xm0, xm1 + movhps [dstq+dsq*0], xm0 + movq [dstq+dsq*1], xm0 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w8_loop + RET +ALIGN function_align +.w16: + vbroadcasti128 m3, [maskq+16*2] + vbroadcasti128 m4, [maskq+16*3] +.w16_loop: + mova xm1, [dstq+dsq*0] + vinserti128 m1, [dstq+dsq*1], 1 + mova m2, [tmpq] + add tmpq, 16*2 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + pmaddubsw m0, m3 + pmaddubsw m1, m4 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + packuswb m0, m1 + mova [dstq+dsq*0], xm0 + vextracti128 [dstq+dsq*1], m0, 1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w16_loop + RET +ALIGN function_align +.w32: + mova xm3, [maskq+16*4] + vinserti128 m3, [maskq+16*6], 1 + mova xm4, [maskq+16*5] + vinserti128 m4, [maskq+16*7], 1 +.w32_loop: + mova m1, [dstq] + mova m2, [tmpq] + add tmpq, 32 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + pmaddubsw m0, m3 + pmaddubsw m1, m4 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + packuswb m0, m1 + mova [dstq], m0 + add dstq, dsq + dec hd + jg .w32_loop + RET + +cglobal blend_h, 4, 7, 6, dst, ds, tmp, w, h, mask +%define base r5-blend_h_avx2_table + lea r5, [blend_h_avx2_table] + mov r6d, wd + tzcnt wd, wd + mov hd, hm + movsxd wq, dword [r5+wq*4] + vpbroadcastd m5, [base+pw_512] + add wq, r5 + lea maskq, [base+obmc_masks+hq*2] + lea hd, [hq*3] + shr hd, 2 ; h * 3/4 + lea maskq, [maskq+hq*2] + neg hq + jmp wq +.w2: + movd xm0, [dstq+dsq*0] + pinsrw xm0, [dstq+dsq*1], 1 + movd xm2, [maskq+hq*2] + movd xm1, [tmpq] + add tmpq, 2*2 + punpcklwd xm2, xm2 + punpcklbw xm0, xm1 + pmaddubsw xm0, xm2 + pmulhrsw xm0, xm5 + packuswb xm0, xm0 + pextrw [dstq+dsq*0], xm0, 0 + pextrw [dstq+dsq*1], xm0, 1 + lea dstq, [dstq+dsq*2] + add hq, 2 + jl .w2 + RET +ALIGN function_align +.w4: + mova xm3, [blend_shuf] +.w4_loop: + movd xm0, [dstq+dsq*0] + pinsrd xm0, [dstq+dsq*1], 1 + movd xm2, [maskq+hq*2] + movq xm1, [tmpq] + add tmpq, 4*2 + pshufb xm2, xm3 + punpcklbw xm0, xm1 + pmaddubsw xm0, xm2 + pmulhrsw xm0, xm5 + packuswb xm0, xm0 + movd [dstq+dsq*0], xm0 + pextrd [dstq+dsq*1], xm0, 1 + lea dstq, [dstq+dsq*2] + add hq, 2 + jl .w4_loop + RET +ALIGN function_align +.w8: + vbroadcasti128 m4, [blend_shuf] + shufpd m4, m4, 0x03 +.w8_loop: + vpbroadcastq m1, [dstq+dsq*0] + movq xm0, [dstq+dsq*1] + vpblendd m0, m1, 0x30 + vpbroadcastd m3, [maskq+hq*2] + movq xm1, [tmpq+8*1] + vinserti128 m1, [tmpq+8*0], 1 + add tmpq, 8*2 + pshufb m3, m4 + punpcklbw m0, m1 + pmaddubsw m0, m3 + pmulhrsw m0, m5 + vextracti128 xm1, m0, 1 + packuswb xm0, xm1 + movhps [dstq+dsq*0], xm0 + movq [dstq+dsq*1], xm0 + lea dstq, [dstq+dsq*2] + add hq, 2 + jl .w8_loop + RET +ALIGN function_align +.w16: + vbroadcasti128 m4, [blend_shuf] + shufpd m4, m4, 0x0c +.w16_loop: + mova xm1, [dstq+dsq*0] + vinserti128 m1, [dstq+dsq*1], 1 + vpbroadcastd m3, [maskq+hq*2] + mova m2, [tmpq] + add tmpq, 16*2 + pshufb m3, m4 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + pmaddubsw m0, m3 + pmaddubsw m1, m3 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + packuswb m0, m1 + mova [dstq+dsq*0], xm0 + vextracti128 [dstq+dsq*1], m0, 1 + lea dstq, [dstq+dsq*2] + add hq, 2 + jl .w16_loop + RET +ALIGN function_align +.w32: ; w32/w64/w128 + sub dsq, r6 +.w32_loop0: + vpbroadcastw m3, [maskq+hq*2] + mov wd, r6d +.w32_loop: + mova m1, [dstq] + mova m2, [tmpq] + add tmpq, 32 + punpcklbw m0, m1, m2 + punpckhbw m1, m2 + pmaddubsw m0, m3 + pmaddubsw m1, m3 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + packuswb m0, m1 + mova [dstq], m0 + add dstq, 32 + sub wd, 32 + jg .w32_loop + add dstq, dsq + inc hq + jl .w32_loop0 + RET + +cglobal emu_edge, 10, 13, 1, bw, bh, iw, ih, x, y, dst, dstride, src, sstride, \ + bottomext, rightext + ; we assume that the buffer (stride) is larger than width, so we can + ; safely overwrite by a few bytes + + ; ref += iclip(y, 0, ih - 1) * PXSTRIDE(ref_stride) + xor r12d, r12d + lea r10, [ihq-1] + cmp yq, ihq + cmovs r10, yq + test yq, yq + cmovs r10, r12 + imul r10, sstrideq + add srcq, r10 + + ; ref += iclip(x, 0, iw - 1) + lea r10, [iwq-1] + cmp xq, iwq + cmovs r10, xq + test xq, xq + cmovs r10, r12 + add srcq, r10 + + ; bottom_ext = iclip(y + bh - ih, 0, bh - 1) + lea bottomextq, [yq+bhq] + sub bottomextq, ihq + lea r3, [bhq-1] + cmovs bottomextq, r12 + + DEFINE_ARGS bw, bh, iw, ih, x, topext, dst, dstride, src, sstride, \ + bottomext, rightext + + ; top_ext = iclip(-y, 0, bh - 1) + neg topextq + cmovs topextq, r12 + cmp bottomextq, bhq + cmovns bottomextq, r3 + cmp topextq, bhq + cmovg topextq, r3 + + ; right_ext = iclip(x + bw - iw, 0, bw - 1) + lea rightextq, [xq+bwq] + sub rightextq, iwq + lea r2, [bwq-1] + cmovs rightextq, r12 + + DEFINE_ARGS bw, bh, iw, ih, leftext, topext, dst, dstride, src, sstride, \ + bottomext, rightext + + ; left_ext = iclip(-x, 0, bw - 1) + neg leftextq + cmovs leftextq, r12 + cmp rightextq, bwq + cmovns rightextq, r2 + cmp leftextq, bwq + cmovns leftextq, r2 + + DEFINE_ARGS bw, centerh, centerw, dummy, leftext, topext, \ + dst, dstride, src, sstride, bottomext, rightext + + ; center_h = bh - top_ext - bottom_ext + lea r3, [bottomextq+topextq] + sub centerhq, r3 + + ; blk += top_ext * PXSTRIDE(dst_stride) + mov r2, topextq + imul r2, dstrideq + add dstq, r2 + mov r9m, dstq + + ; center_w = bw - left_ext - right_ext + mov centerwq, bwq + lea r3, [rightextq+leftextq] + sub centerwq, r3 + +%macro v_loop 3 ; need_left_ext, need_right_ext, suffix +.v_loop_%3: +%if %1 + ; left extension + xor r3, r3 + vpbroadcastb m0, [srcq] +.left_loop_%3: + mova [dstq+r3], m0 + add r3, 32 + cmp r3, leftextq + jl .left_loop_%3 + + ; body + lea r12, [dstq+leftextq] +%endif + xor r3, r3 +.body_loop_%3: + movu m0, [srcq+r3] +%if %1 + movu [r12+r3], m0 +%else + movu [dstq+r3], m0 +%endif + add r3, 32 + cmp r3, centerwq + jl .body_loop_%3 + +%if %2 + ; right extension +%if %1 + add r12, centerwq +%else + lea r12, [dstq+centerwq] +%endif + xor r3, r3 + vpbroadcastb m0, [srcq+centerwq-1] +.right_loop_%3: + movu [r12+r3], m0 + add r3, 32 + cmp r3, rightextq + jl .right_loop_%3 + +%endif + add dstq, dstrideq + add srcq, sstrideq + dec centerhq + jg .v_loop_%3 +%endmacro + + test leftextq, leftextq + jnz .need_left_ext + test rightextq, rightextq + jnz .need_right_ext + v_loop 0, 0, 0 + jmp .body_done + +.need_left_ext: + test rightextq, rightextq + jnz .need_left_right_ext + v_loop 1, 0, 1 + jmp .body_done + +.need_left_right_ext: + v_loop 1, 1, 2 + jmp .body_done + +.need_right_ext: + v_loop 0, 1, 3 + +.body_done: + ; bottom edge extension + test bottomextq, bottomextq + jz .top + mov srcq, dstq + sub srcq, dstrideq + xor r1, r1 +.bottom_x_loop: + mova m0, [srcq+r1] + lea r3, [dstq+r1] + mov r4, bottomextq +.bottom_y_loop: + mova [r3], m0 + add r3, dstrideq + dec r4 + jg .bottom_y_loop + add r1, 32 + cmp r1, bwq + jl .bottom_x_loop + +.top: + ; top edge extension + test topextq, topextq + jz .end + mov srcq, r9m + mov dstq, dstm + xor r1, r1 +.top_x_loop: + mova m0, [srcq+r1] + lea r3, [dstq+r1] + mov r4, topextq +.top_y_loop: + mova [r3], m0 + add r3, dstrideq + dec r4 + jg .top_y_loop + add r1, 32 + cmp r1, bwq + jl .top_x_loop + +.end: + RET + +cglobal resize, 6, 14, 16, dst, dst_stride, src, src_stride, \ + dst_w, h, src_w, dx, mx0 + sub dword mx0m, 4<<14 + sub dword src_wm, 8 + vpbroadcastd m5, dxm + vpbroadcastd m8, mx0m + vpbroadcastd m6, src_wm + + DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, h, x, picptr + LEA r7, $$ +%define base r7-$$ + + vpbroadcastd m3, [base+pw_m256] + vpbroadcastd m7, [base+pd_63] + vbroadcasti128 m15, [base+pb_8x0_8x8] + pmaddwd m2, m5, [base+rescale_mul] ; dx*[0,1,2,3,4,5,6,7] + pslld m5, 3 ; dx*8 + pslld m6, 14 + paddd m8, m2 ; mx+[0..7]*dx + pxor m2, m2 + + ; m2 = 0, m3 = pmulhrsw constant for x=(x+64)>>7 + ; m8 = mx+[0..7]*dx, m5 = dx*8, m6 = src_w, m7 = 0x3f, m15=0,8 + +.loop_y: + xor xd, xd + mova m4, m8 ; per-line working version of mx + +.loop_x: + pmaxsd m0, m4, m2 + psrad m9, m4, 8 ; filter offset (unmasked) + pminsd m0, m6 ; iclip(mx, 0, src_w-8) + psubd m1, m4, m0 ; pshufb offset + psrad m0, 14 ; clipped src_x offset + psrad m1, 14 ; pshufb edge_emu offset + pand m9, m7 ; filter offset (masked) + + ; load source pixels - this ugly code is vpgatherdq emulation since + ; directly using vpgatherdq on Haswell is quite a bit slower :( + movd r8d, xm0 + pextrd r9d, xm0, 1 + pextrd r10d, xm0, 2 + pextrd r11d, xm0, 3 + vextracti128 xm0, m0, 1 + movq xm12, [srcq+r8] + movq xm13, [srcq+r10] + movhps xm12, [srcq+r9] + movhps xm13, [srcq+r11] + movd r8d, xm0 + pextrd r9d, xm0, 1 + pextrd r10d, xm0, 2 + pextrd r11d, xm0, 3 + vinserti128 m12, [srcq+r8], 1 + vinserti128 m13, [srcq+r10], 1 + vpbroadcastq m10, [srcq+r9] + vpbroadcastq m11, [srcq+r11] + vpblendd m12, m10, 11000000b + vpblendd m13, m11, 11000000b + + ; if no emulation is required, we don't need to shuffle or emulate edges + ; this also saves 2 quasi-vpgatherdqs + vptest m1, m1 + jz .filter + + movd r8d, xm1 + pextrd r9d, xm1, 1 + pextrd r10d, xm1, 2 + pextrd r11d, xm1, 3 + movsxd r8, r8d + movsxd r9, r9d + movsxd r10, r10d + movsxd r11, r11d + vextracti128 xm1, m1, 1 + movq xm14, [base+resize_shuf+4+r8] + movq xm0, [base+resize_shuf+4+r10] + movhps xm14, [base+resize_shuf+4+r9] + movhps xm0, [base+resize_shuf+4+r11] + movd r8d, xm1 + pextrd r9d, xm1, 1 + pextrd r10d, xm1, 2 + pextrd r11d, xm1, 3 + movsxd r8, r8d + movsxd r9, r9d + movsxd r10, r10d + movsxd r11, r11d + vinserti128 m14, [base+resize_shuf+4+r8], 1 + vinserti128 m0, [base+resize_shuf+4+r10], 1 + vpbroadcastq m10, [base+resize_shuf+4+r9] + vpbroadcastq m11, [base+resize_shuf+4+r11] + vpblendd m14, m10, 11000000b + vpblendd m0, m11, 11000000b + + paddb m14, m15 + paddb m0, m15 + pshufb m12, m14 + pshufb m13, m0 + +.filter: + movd r8d, xm9 + pextrd r9d, xm9, 1 + pextrd r10d, xm9, 2 + pextrd r11d, xm9, 3 + vextracti128 xm9, m9, 1 + movq xm10, [base+resize_filter+r8*8] + movq xm11, [base+resize_filter+r10*8] + movhps xm10, [base+resize_filter+r9*8] + movhps xm11, [base+resize_filter+r11*8] + movd r8d, xm9 + pextrd r9d, xm9, 1 + pextrd r10d, xm9, 2 + pextrd r11d, xm9, 3 + vinserti128 m10, [base+resize_filter+r8*8], 1 + vinserti128 m11, [base+resize_filter+r10*8], 1 + vpbroadcastq m14, [base+resize_filter+r9*8] + vpbroadcastq m1, [base+resize_filter+r11*8] + vpblendd m10, m14, 11000000b + vpblendd m11, m1, 11000000b + + pmaddubsw m12, m10 + pmaddubsw m13, m11 + phaddw m12, m13 + vextracti128 xm13, m12, 1 + phaddsw xm12, xm13 + pmulhrsw xm12, xm3 ; x=(x+64)>>7 + packuswb xm12, xm12 + movq [dstq+xq], xm12 + + paddd m4, m5 + add xd, 8 + cmp xd, dst_wd + jl .loop_x + + add dstq, dst_strideq + add srcq, src_strideq + dec hd + jg .loop_y + RET + +cglobal w_mask_420, 4, 8, 14, dst, stride, tmp1, tmp2, w, h, mask, stride3 +%define base r7-w_mask_420_avx2_table + lea r7, [w_mask_420_avx2_table] + tzcnt wd, wm + mov r6d, r7m ; sign + movifnidn hd, hm + movsxd wq, [r7+wq*4] + vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 + vpbroadcastd m7, [base+pw_2048] + pmovzxbd m9, [base+deint_shuf4] + vpbroadcastd m8, [base+wm_420_sign+r6*4] ; 258 - sign + add wq, r7 + W_MASK 0, 4, 0, 1 + mov maskq, maskmp + lea stride3q, [strideq*3] + jmp wq +.w4: + vextracti128 xm1, m0, 1 + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xm1 + pextrd [dstq+stride3q ], xm1, 1 + cmp hd, 8 + jl .w4_end + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xm1, 2 + pextrd [dstq+stride3q ], xm1, 3 + jg .w4_h16 +.w4_end: + vextracti128 xm0, m4, 1 + vpblendd xm1, xm4, xm0, 0x05 + vpblendd xm4, xm0, 0x0a + pshufd xm1, xm1, q2301 + psubw xm4, xm8, xm4 + psubw xm4, xm1 + psrlw xm4, 2 + packuswb xm4, xm4 + movq [maskq], xm4 + RET +.w4_h16: + W_MASK 0, 5, 2, 3 + lea dstq, [dstq+strideq*4] + phaddd m4, m5 + vextracti128 xm1, m0, 1 + psubw m4, m8, m4 + psrlw m4, 2 + vpermd m4, m9, m4 + vextracti128 xm5, m4, 1 + packuswb xm4, xm5 + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xm1 + pextrd [dstq+stride3q], xm1, 1 + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xm1, 2 + pextrd [dstq+stride3q ], xm1, 3 + mova [maskq], xm4 + RET +.w8_loop: + add tmp1q, 2*32 + add tmp2q, 2*32 + W_MASK 0, 4, 0, 1 + lea dstq, [dstq+strideq*4] + add maskq, 8 +.w8: + vextracti128 xm2, m4, 1 + vextracti128 xm1, m0, 1 + psubw xm4, xm8, xm4 + psubw xm4, xm2 + psrlw xm4, 2 + packuswb xm4, xm4 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xm1 + movq [maskq], xm4 + sub hd, 4 + jg .w8_loop + RET +.w16_loop: + add tmp1q, 4*32 + add tmp2q, 4*32 + W_MASK 0, 4, 0, 1 + lea dstq, [dstq+strideq*4] + add maskq, 16 +.w16: + vpermq m0, m0, q3120 + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + W_MASK 0, 5, 2, 3 + punpckhqdq m1, m4, m5 + punpcklqdq m4, m5 + psubw m1, m8, m1 + psubw m1, m4 + psrlw m1, 2 + vpermq m0, m0, q3120 + packuswb m1, m1 + vpermd m1, m9, m1 + mova [dstq+strideq*2], xm0 + vextracti128 [dstq+stride3q ], m0, 1 + mova [maskq], xm1 + sub hd, 4 + jg .w16_loop + RET +.w32_loop: + add tmp1q, 4*32 + add tmp2q, 4*32 + W_MASK 0, 4, 0, 1 + lea dstq, [dstq+strideq*2] + add maskq, 16 +.w32: + vpermq m0, m0, q3120 + mova [dstq+strideq*0], m0 + W_MASK 0, 5, 2, 3 + psubw m4, m8, m4 + psubw m4, m5 + psrlw m4, 2 + vpermq m0, m0, q3120 + packuswb m4, m4 + vpermd m4, m9, m4 + mova [dstq+strideq*1], m0 + mova [maskq], xm4 + sub hd, 2 + jg .w32_loop + RET +.w64_loop_even: + psubw m10, m8, m4 + psubw m11, m8, m5 + dec hd +.w64_loop: + add tmp1q, 4*32 + add tmp2q, 4*32 + W_MASK 0, 4, 0, 1 + add dstq, strideq +.w64: + vpermq m0, m0, q3120 + mova [dstq+32*0], m0 + W_MASK 0, 5, 2, 3 + vpermq m0, m0, q3120 + mova [dstq+32*1], m0 + test hd, 1 + jz .w64_loop_even + psubw m4, m10, m4 + psubw m5, m11, m5 + psrlw m4, 2 + psrlw m5, 2 + packuswb m4, m5 + vpermd m4, m9, m4 + mova [maskq], m4 + add maskq, 32 + dec hd + jg .w64_loop + RET +.w128_loop_even: + psubw m12, m8, m4 + psubw m13, m8, m5 + dec hd +.w128_loop: + W_MASK 0, 4, 0, 1 + add dstq, strideq +.w128: + vpermq m0, m0, q3120 + mova [dstq+32*0], m0 + W_MASK 0, 5, 2, 3 + vpermq m0, m0, q3120 + mova [dstq+32*1], m0 + add tmp1q, 8*32 + add tmp2q, 8*32 + test hd, 1 + jz .w128_even + psubw m4, m10, m4 + psubw m5, m11, m5 + psrlw m4, 2 + psrlw m5, 2 + packuswb m4, m5 + vpermd m4, m9, m4 + mova [maskq+32*0], m4 + jmp .w128_odd +.w128_even: + psubw m10, m8, m4 + psubw m11, m8, m5 +.w128_odd: + W_MASK 0, 4, -4, -3 + vpermq m0, m0, q3120 + mova [dstq+32*2], m0 + W_MASK 0, 5, -2, -1 + vpermq m0, m0, q3120 + mova [dstq+32*3], m0 + test hd, 1 + jz .w128_loop_even + psubw m4, m12, m4 + psubw m5, m13, m5 + psrlw m4, 2 + psrlw m5, 2 + packuswb m4, m5 + vpermd m4, m9, m4 + mova [maskq+32*1], m4 + add maskq, 64 + dec hd + jg .w128_loop + RET + +cglobal w_mask_422, 4, 8, 11, dst, stride, tmp1, tmp2, w, h, mask, stride3 +%define base r7-w_mask_422_avx2_table + lea r7, [w_mask_422_avx2_table] + tzcnt wd, wm + mov r6d, r7m ; sign + movifnidn hd, hm + pxor m9, m9 + movsxd wq, dword [r7+wq*4] + vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 + vpbroadcastd m7, [base+pw_2048] + pmovzxbd m10, [base+deint_shuf4] + vpbroadcastd m8, [base+wm_422_sign+r6*4] ; 128 - sign + add wq, r7 + mov maskq, maskmp + W_MASK 0, 4, 0, 1 + lea stride3q, [strideq*3] + jmp wq +.w4: + vextracti128 xm1, m0, 1 + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xm1 + pextrd [dstq+stride3q ], xm1, 1 + cmp hd, 8 + jl .w4_end + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xm1, 2 + pextrd [dstq+stride3q ], xm1, 3 + jg .w4_h16 +.w4_end: + vextracti128 xm5, m4, 1 + packuswb xm4, xm5 + psubb xm5, xm8, xm4 + pavgb xm5, xm9 + pshufd xm5, xm5, q3120 + mova [maskq], xm5 + RET +.w4_h16: + W_MASK 0, 5, 2, 3 + lea dstq, [dstq+strideq*4] + packuswb m4, m5 + psubb m5, m8, m4 + pavgb m5, m9 + vpermd m5, m10, m5 + vextracti128 xm1, m0, 1 + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xm1 + pextrd [dstq+stride3q ], xm1, 1 + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xm1, 2 + pextrd [dstq+stride3q ], xm1, 3 + mova [maskq], m5 + RET +.w8_loop: + add tmp1q, 32*2 + add tmp2q, 32*2 + W_MASK 0, 4, 0, 1 + lea dstq, [dstq+strideq*4] + add maskq, 16 +.w8: + vextracti128 xm5, m4, 1 + vextracti128 xm1, m0, 1 + packuswb xm4, xm5 + psubb xm5, xm8, xm4 + pavgb xm5, xm9 + pshufd xm5, xm5, q3120 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xm1 + mova [maskq], xm5 + sub hd, 4 + jg .w8_loop + RET +.w16_loop: + add tmp1q, 32*4 + add tmp2q, 32*4 + W_MASK 0, 4, 0, 1 + lea dstq, [dstq+strideq*4] + add maskq, 32 +.w16: + vpermq m0, m0, q3120 + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + W_MASK 0, 5, 2, 3 + packuswb m4, m5 + psubb m5, m8, m4 + pavgb m5, m9 + vpermq m0, m0, q3120 + vpermd m5, m10, m5 + mova [dstq+strideq*2], xm0 + vextracti128 [dstq+stride3q ], m0, 1 + mova [maskq], m5 + sub hd, 4 + jg .w16_loop + RET +.w32_loop: + add tmp1q, 32*4 + add tmp2q, 32*4 + W_MASK 0, 4, 0, 1 + lea dstq, [dstq+strideq*2] + add maskq, 32 +.w32: + vpermq m0, m0, q3120 + mova [dstq+strideq*0], m0 + W_MASK 0, 5, 2, 3 + packuswb m4, m5 + psubb m5, m8, m4 + pavgb m5, m9 + vpermq m0, m0, q3120 + vpermd m5, m10, m5 + mova [dstq+strideq*1], m0 + mova [maskq], m5 + sub hd, 2 + jg .w32_loop + RET +.w64_loop: + add tmp1q, 32*4 + add tmp2q, 32*4 + W_MASK 0, 4, 0, 1 + add dstq, strideq + add maskq, 32 +.w64: + vpermq m0, m0, q3120 + mova [dstq+32*0], m0 + W_MASK 0, 5, 2, 3 + packuswb m4, m5 + psubb m5, m8, m4 + pavgb m5, m9 + vpermq m0, m0, q3120 + vpermd m5, m10, m5 + mova [dstq+32*1], m0 + mova [maskq], m5 + dec hd + jg .w64_loop + RET +.w128_loop: + add tmp1q, 32*8 + add tmp2q, 32*8 + W_MASK 0, 4, 0, 1 + add dstq, strideq + add maskq, 32*2 +.w128: + vpermq m0, m0, q3120 + mova [dstq+32*0], m0 + W_MASK 0, 5, 2, 3 + packuswb m4, m5 + psubb m5, m8, m4 + pavgb m5, m9 + vpermq m0, m0, q3120 + vpermd m5, m10, m5 + mova [dstq+32*1], m0 + mova [maskq+32*0], m5 + W_MASK 0, 4, 4, 5 + vpermq m0, m0, q3120 + mova [dstq+32*2], m0 + W_MASK 0, 5, 6, 7 + packuswb m4, m5 + psubb m5, m8, m4 + pavgb m5, m9 + vpermq m0, m0, q3120 + vpermd m5, m10, m5 + mova [dstq+32*3], m0 + mova [maskq+32*1], m5 + dec hd + jg .w128_loop + RET + +cglobal w_mask_444, 4, 8, 8, dst, stride, tmp1, tmp2, w, h, mask, stride3 +%define base r7-w_mask_444_avx2_table + lea r7, [w_mask_444_avx2_table] + tzcnt wd, wm + movifnidn hd, hm + mov maskq, maskmp + movsxd wq, dword [r7+wq*4] + vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 + vpbroadcastd m5, [base+pb_64] + vpbroadcastd m7, [base+pw_2048] + add wq, r7 + W_MASK 0, 4, 0, 1, 1 + lea stride3q, [strideq*3] + jmp wq +.w4: + vextracti128 xm1, m0, 1 + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xm1 + pextrd [dstq+stride3q ], xm1, 1 + mova [maskq+32*0], m4 + cmp hd, 8 + jl .w4_end + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xm1, 2 + pextrd [dstq+stride3q ], xm1, 3 + je .w4_end + W_MASK 0, 4, 2, 3, 1 + lea dstq, [dstq+strideq*4] + vextracti128 xm1, m0, 1 + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xm1 + pextrd [dstq+stride3q ], xm1, 1 + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xm1, 2 + pextrd [dstq+stride3q ], xm1, 3 + mova [maskq+32*1], m4 +.w4_end: + RET +.w8_loop: + add tmp1q, 32*2 + add tmp2q, 32*2 + W_MASK 0, 4, 0, 1, 1 + lea dstq, [dstq+strideq*4] + add maskq, 32 +.w8: + vextracti128 xm1, m0, 1 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xm1 + mova [maskq], m4 + sub hd, 4 + jg .w8_loop + RET +.w16_loop: + add tmp1q, 32*2 + add tmp2q, 32*2 + W_MASK 0, 4, 0, 1, 1 + lea dstq, [dstq+strideq*2] + add maskq, 32 +.w16: + vpermq m0, m0, q3120 + mova [dstq+strideq*0], xm0 + vextracti128 [dstq+strideq*1], m0, 1 + mova [maskq], m4 + sub hd, 2 + jg .w16_loop + RET +.w32_loop: + add tmp1q, 32*2 + add tmp2q, 32*2 + W_MASK 0, 4, 0, 1, 1 + add dstq, strideq + add maskq, 32 +.w32: + vpermq m0, m0, q3120 + mova [dstq], m0 + mova [maskq], m4 + dec hd + jg .w32_loop + RET +.w64_loop: + add tmp1q, 32*4 + add tmp2q, 32*4 + W_MASK 0, 4, 0, 1, 1 + add dstq, strideq + add maskq, 32*2 +.w64: + vpermq m0, m0, q3120 + mova [dstq+32*0], m0 + mova [maskq+32*0], m4 + W_MASK 0, 4, 2, 3, 1 + vpermq m0, m0, q3120 + mova [dstq+32*1], m0 + mova [maskq+32*1], m4 + dec hd + jg .w64_loop + RET +.w128_loop: + add tmp1q, 32*8 + add tmp2q, 32*8 + W_MASK 0, 4, 0, 1, 1 + add dstq, strideq + add maskq, 32*4 +.w128: + vpermq m0, m0, q3120 + mova [dstq+32*0], m0 + mova [maskq+32*0], m4 + W_MASK 0, 4, 2, 3, 1 + vpermq m0, m0, q3120 + mova [dstq+32*1], m0 + mova [maskq+32*1], m4 + W_MASK 0, 4, 4, 5, 1 + vpermq m0, m0, q3120 + mova [dstq+32*2], m0 + mova [maskq+32*2], m4 + W_MASK 0, 4, 6, 7, 1 + vpermq m0, m0, q3120 + mova [dstq+32*3], m0 + mova [maskq+32*3], m4 + dec hd + jg .w128_loop + RET + +%endif ; ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/mc_avx512.asm b/third_party/dav1d/src/x86/mc_avx512.asm new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/x86/mc_avx512.asm @@ -0,0 +1,2394 @@ +; Copyright © 2020, VideoLAN and dav1d authors +; Copyright © 2020, Two Orioles, LLC +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "ext/x86/x86inc.asm" + +%if HAVE_AVX512ICL && ARCH_X86_64 + +SECTION_RODATA 64 + +bidir_sctr_w4: dd 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15 +wm_420_perm4: db 1, 3, 9, 11, 5, 7, 13, 15, 17, 19, 25, 27, 21, 23, 29, 31 + db 33, 35, 41, 43, 37, 39, 45, 47, 49, 51, 57, 59, 53, 55, 61, 63 + db 0, 2, 8, 10, 4, 6, 12, 14, 16, 18, 24, 26, 20, 22, 28, 30 + db 32, 34, 40, 42, 36, 38, 44, 46, 48, 50, 56, 58, 52, 54, 60, 62 +wm_420_perm8: db 1, 3, 17, 19, 5, 7, 21, 23, 9, 11, 25, 27, 13, 15, 29, 31 + db 33, 35, 49, 51, 37, 39, 53, 55, 41, 43, 57, 59, 45, 47, 61, 63 + db 0, 2, 16, 18, 4, 6, 20, 22, 8, 10, 24, 26, 12, 14, 28, 30 + db 32, 34, 48, 50, 36, 38, 52, 54, 40, 42, 56, 58, 44, 46, 60, 62 +wm_420_perm16: db 1, 3, 33, 35, 5, 7, 37, 39, 9, 11, 41, 43, 13, 15, 45, 47 + db 17, 19, 49, 51, 21, 23, 53, 55, 25, 27, 57, 59, 29, 31, 61, 63 + db 0, 2, 32, 34, 4, 6, 36, 38, 8, 10, 40, 42, 12, 14, 44, 46 + db 16, 18, 48, 50, 20, 22, 52, 54, 24, 26, 56, 58, 28, 30, 60, 62 +wm_420_mask: db 3, 7, 11, 15, 19, 23, 27, 31, 35, 39, 43, 47, 51, 55, 59, 63 + db 67, 71, 75, 79, 83, 87, 91, 95, 99,103,107,111,115,119,123,127 + db 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61 + db 65, 69, 73, 77, 81, 85, 89, 93, 97,101,105,109,113,117,121,125 +wm_422_mask: db 2, 6, 10, 14, 18, 22, 26, 30, 34, 38, 42, 46, 50, 54, 58, 62 + db 1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53, 57, 61 + db 66, 70, 74, 78, 82, 86, 90, 94, 98,102,106,110,114,118,122,126 + db 65, 69, 73, 77, 81, 85, 89, 93, 97,101,105,109,113,117,121,125 +wm_444_mask: db 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31 + db 33, 35, 37, 39, 41, 43, 45, 47, 49, 51, 53, 55, 57, 59, 61, 63 + db 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 + db 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62 +bilin_h_perm16: db 1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7 + db 9, 8, 10, 9, 11, 10, 12, 11, 13, 12, 14, 13, 15, 14, 16, 15 + db 33, 32, 34, 33, 35, 34, 36, 35, 37, 36, 38, 37, 39, 38, 40, 39 + db 41, 40, 42, 41, 43, 42, 44, 43, 45, 44, 46, 45, 47, 46, 48, 47 +bilin_h_perm32: db 1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7 + db 9, 8, 10, 9, 11, 10, 12, 11, 13, 12, 14, 13, 15, 14, 16, 15 + db 17, 16, 18, 17, 19, 18, 20, 19, 21, 20, 22, 21, 23, 22, 24, 23 + db 25, 24, 26, 25, 27, 26, 28, 27, 29, 28, 30, 29, 31, 30, 32, 31 +bilin_v_perm8: db 16, 0, 17, 1, 18, 2, 19, 3, 20, 4, 21, 5, 22, 6, 23, 7 + db 80, 16, 81, 17, 82, 18, 83, 19, 84, 20, 85, 21, 86, 22, 87, 23 + db 32, 80, 33, 81, 34, 82, 35, 83, 36, 84, 37, 85, 38, 86, 39, 87 + db 64, 32, 65, 33, 66, 34, 67, 35, 68, 36, 69, 37, 70, 38, 71, 39 +bilin_v_perm16: db 16, 0, 17, 1, 18, 2, 19, 3, 20, 4, 21, 5, 22, 6, 23, 7 + db 24, 8, 25, 9, 26, 10, 27, 11, 28, 12, 29, 13, 30, 14, 31, 15 + db 64, 16, 65, 17, 66, 18, 67, 19, 68, 20, 69, 21, 70, 22, 71, 23 + db 72, 24, 73, 25, 74, 26, 75, 27, 76, 28, 77, 29, 78, 30, 79, 31 +bilin_v_perm32: db 64, 0, 65, 1, 66, 2, 67, 3, 68, 4, 69, 5, 70, 6, 71, 7 + db 72, 8, 73, 9, 74, 10, 75, 11, 76, 12, 77, 13, 78, 14, 79, 15 + db 80, 16, 81, 17, 82, 18, 83, 19, 84, 20, 85, 21, 86, 22, 87, 23 + db 88, 24, 89, 25, 90, 26, 91, 27, 92, 28, 93, 29, 94, 30, 95, 31 +bilin_v_perm64: dq 0, 4, 1, 5, 2, 6, 3, 7 +spel_h_perm16a: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 + db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 + db 32, 33, 34, 35, 33, 34, 35, 36, 34, 35, 36, 37, 35, 36, 37, 38 + db 40, 41, 42, 43, 41, 42, 43, 44, 42, 43, 44, 45, 43, 44, 45, 46 +spel_h_perm16b: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 + db 12, 13, 14, 15, 13, 14, 15, 16, 14, 15, 16, 17, 15, 16, 17, 18 + db 36, 37, 38, 39, 37, 38, 39, 40, 38, 39, 40, 41, 39, 40, 41, 42 + db 44, 45, 46, 47, 45, 46, 47, 48, 46, 47, 48, 49, 47, 48, 49, 50 +spel_h_perm16c: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 + db 16, 17, 18, 19, 17, 18, 19, 20, 18, 19, 20, 21, 19, 20, 21, 22 + db 40, 41, 42, 43, 41, 42, 43, 44, 42, 43, 44, 45, 43, 44, 45, 46 + db 48, 49, 50, 51, 49, 50, 51, 52, 50, 51, 52, 53, 51, 52, 53, 54 +spel_h_perm32a: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 + db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 + db 16, 17, 18, 19, 17, 18, 19, 20, 18, 19, 20, 21, 19, 20, 21, 22 + db 24, 25, 26, 27, 25, 26, 27, 28, 26, 27, 28, 29, 27, 28, 29, 30 +spel_h_perm32b: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 + db 12, 13, 14, 15, 13, 14, 15, 16, 14, 15, 16, 17, 15, 16, 17, 18 + db 20, 21, 22, 23, 21, 22, 23, 24, 22, 23, 24, 25, 23, 24, 25, 26 + db 28, 29, 30, 31, 29, 30, 31, 32, 30, 31, 32, 33, 31, 32, 33, 34 +spel_h_perm32c: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 + db 16, 17, 18, 19, 17, 18, 19, 20, 18, 19, 20, 21, 19, 20, 21, 22 + db 24, 25, 26, 27, 25, 26, 27, 28, 26, 27, 28, 29, 27, 28, 29, 30 + db 32, 33, 34, 35, 33, 34, 35, 36, 34, 35, 36, 37, 35, 36, 37, 38 +spel_hv_perm4a: db 8, 9, 16, 17, 10, 11, 18, 19, 12, 13, 20, 21, 14, 15, 22, 23 + db 16, 17, 24, 25, 18, 19, 26, 27, 20, 21, 28, 29, 22, 23, 30, 31 +spel_hv_perm4b: db 24, 25, 32, 33, 26, 27, 34, 35, 28, 29, 36, 37, 30, 31, 38, 39 + db 32, 33, 40, 41, 34, 35, 42, 43, 36, 37, 44, 45, 38, 39, 46, 47 +spel_hv_perm4c: db 40, 41, 48, 49, 42, 43, 50, 51, 44, 45, 52, 53, 46, 47, 54, 55 + db 48, 49, 56, 57, 50, 51, 58, 59, 52, 53, 60, 61, 54, 55, 62, 63 +deint_shuf4: db 0, 4, 1, 5, 2, 6, 3, 7, 4, 8, 5, 9, 6, 10, 7, 11 +subpel_h_shufA: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 +subpel_h_shufB: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 +subpel_h_shufC: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 +bilin_h_shuf4: db 1, 0, 2, 1, 3, 2, 4, 3, 9, 8, 10, 9, 11, 10, 12, 11 +bilin_h_shuf8: db 1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7 +bilin_v_shuf4: db 4, 0, 5, 1, 6, 2, 7, 3, 8, 4, 9, 5, 10, 6, 11, 7 +pb_02461357: db 0, 2, 4, 6, 1, 3, 5, 7 + +wm_420_perm64: dq 0xfedcba9876543210 +wm_sign: dd 0x40804080, 0xc0c0c0c0, 0x40404040 + +pb_127: times 4 db 127 +pw_m128 times 2 dw -128 +pw_512: times 2 dw 512 +pw_1024: times 2 dw 1024 +pw_2048: times 2 dw 2048 +pw_6903: times 2 dw 6903 +pw_8192: times 2 dw 8192 +pd_2: dd 2 +pd_32: dd 32 +pd_32768: dd 32768 + +%define pb_m64 (wm_sign+4) +%define pb_64 (wm_sign+8) + +cextern mc_subpel_filters +%define subpel_filters (mangle(private_prefix %+ _mc_subpel_filters)-8) + +%macro BASE_JMP_TABLE 3-* + %xdefine %1_%2_table (%%table - %3) + %xdefine %%base %1_%2 + %%table: + %rep %0 - 2 + dw %%base %+ _w%3 - %%base + %rotate 1 + %endrep +%endmacro + +%macro HV_JMP_TABLE 5-* + %xdefine %%prefix mangle(private_prefix %+ _%1_%2_%3) + %xdefine %%base %1_%3 + %assign %%types %4 + %if %%types & 1 + %xdefine %1_%2_h_%3_table (%%h - %5) + %%h: + %rep %0 - 4 + dw %%prefix %+ .h_w%5 - %%base + %rotate 1 + %endrep + %rotate 4 + %endif + %if %%types & 2 + %xdefine %1_%2_v_%3_table (%%v - %5) + %%v: + %rep %0 - 4 + dw %%prefix %+ .v_w%5 - %%base + %rotate 1 + %endrep + %rotate 4 + %endif + %if %%types & 4 + %xdefine %1_%2_hv_%3_table (%%hv - %5) + %%hv: + %rep %0 - 4 + dw %%prefix %+ .hv_w%5 - %%base + %rotate 1 + %endrep + %endif +%endmacro + +%macro BIDIR_JMP_TABLE 1-* + %xdefine %1_table (%%table - 2*%2) + %xdefine %%base %1_table + %xdefine %%prefix mangle(private_prefix %+ _%1) + %%table: + %rep %0 - 1 + dd %%prefix %+ .w%2 - %%base + %rotate 1 + %endrep +%endmacro + +%xdefine prep_avx512icl mangle(private_prefix %+ _prep_bilin_avx512icl.prep) + +%define table_offset(type, fn) type %+ fn %+ SUFFIX %+ _table - type %+ SUFFIX + +BASE_JMP_TABLE prep, avx512icl, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE prep, bilin, avx512icl, 7, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE prep, 8tap, avx512icl, 7, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE avg_avx512icl, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_avg_avx512icl, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE mask_avx512icl, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_420_avx512icl, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_422_avx512icl, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_444_avx512icl, 4, 8, 16, 32, 64, 128 + +SECTION .text + +%macro REPX 2-* + %xdefine %%f(x) %1 +%rep %0 - 1 + %rotate 1 + %%f(%1) +%endrep +%endmacro + +%macro WRAP_YMM 1+ +INIT_YMM cpuname + %1 +INIT_ZMM cpuname +%endmacro + +DECLARE_REG_TMP 3, 5, 6 + +INIT_ZMM avx512icl +cglobal prep_bilin, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3 + movifnidn mxyd, r5m ; mx + lea t2, [prep_avx512icl] + tzcnt wd, wm + movifnidn hd, hm + test mxyd, mxyd + jnz .h + mov mxyd, r6m ; my + test mxyd, mxyd + jnz .v +.prep: + movzx wd, word [t2+wq*2+table_offset(prep,)] + add wq, t2 + lea stride3q, [strideq*3] + jmp wq +.prep_w4: + movd xmm0, [srcq+strideq*0] + pinsrd xmm0, [srcq+strideq*1], 1 + pinsrd xmm0, [srcq+strideq*2], 2 + pinsrd xmm0, [srcq+stride3q ], 3 + lea srcq, [srcq+strideq*4] + pmovzxbw ym0, xmm0 + psllw ym0, 4 + mova [tmpq], ym0 + add tmpq, 32 + sub hd, 4 + jg .prep_w4 + RET +.prep_w8: + movq xmm0, [srcq+strideq*0] + movq xmm1, [srcq+strideq*1] + vinserti128 ym0, ymm0, [srcq+strideq*2], 1 + vinserti128 ym1, ymm1, [srcq+stride3q ], 1 + lea srcq, [srcq+strideq*4] + punpcklqdq ym0, ym1 + pmovzxbw m0, ym0 + psllw m0, 4 + mova [tmpq], m0 + add tmpq, 32*2 + sub hd, 4 + jg .prep_w8 + RET +.prep_w16: + movu xmm0, [srcq+strideq*0] + vinserti128 ym0, ymm0, [srcq+strideq*1], 1 + movu xmm1, [srcq+strideq*2] + vinserti128 ym1, ymm1, [srcq+stride3q ], 1 + lea srcq, [srcq+strideq*4] + pmovzxbw m0, ym0 + pmovzxbw m1, ym1 + psllw m0, 4 + psllw m1, 4 + mova [tmpq+64*0], m0 + mova [tmpq+64*1], m1 + add tmpq, 32*4 + sub hd, 4 + jg .prep_w16 + RET +.prep_w32: + pmovzxbw m0, [srcq+strideq*0] + pmovzxbw m1, [srcq+strideq*1] + pmovzxbw m2, [srcq+strideq*2] + pmovzxbw m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + REPX {psllw x, 4}, m0, m1, m2, m3 + mova [tmpq+64*0], m0 + mova [tmpq+64*1], m1 + mova [tmpq+64*2], m2 + mova [tmpq+64*3], m3 + add tmpq, 64*4 + sub hd, 4 + jg .prep_w32 + RET +.prep_w64: + pmovzxbw m0, [srcq+strideq*0+32*0] + pmovzxbw m1, [srcq+strideq*0+32*1] + pmovzxbw m2, [srcq+strideq*1+32*0] + pmovzxbw m3, [srcq+strideq*1+32*1] + lea srcq, [srcq+strideq*2] + REPX {psllw x, 4}, m0, m1, m2, m3 + mova [tmpq+64*0], m0 + mova [tmpq+64*1], m1 + mova [tmpq+64*2], m2 + mova [tmpq+64*3], m3 + add tmpq, 64*4 + sub hd, 2 + jg .prep_w64 + RET +.prep_w128: + pmovzxbw m0, [srcq+32*0] + pmovzxbw m1, [srcq+32*1] + pmovzxbw m2, [srcq+32*2] + pmovzxbw m3, [srcq+32*3] + REPX {psllw x, 4}, m0, m1, m2, m3 + mova [tmpq+64*0], m0 + mova [tmpq+64*1], m1 + mova [tmpq+64*2], m2 + mova [tmpq+64*3], m3 + add tmpq, 64*4 + add srcq, strideq + dec hd + jg .prep_w128 + RET +.h: + ; 16 * src[x] + (mx * (src[x + 1] - src[x])) + ; = (16 - mx) * src[x] + mx * src[x + 1] + imul mxyd, 0xff01 + add mxyd, 16 << 8 + vpbroadcastw m5, mxyd + mov mxyd, r6m ; my + test mxyd, mxyd + jnz .hv + movzx wd, word [t2+wq*2+table_offset(prep, _bilin_h)] + add wq, t2 + lea stride3q, [strideq*3] + jmp wq +.h_w4: + vbroadcasti32x4 ym4, [bilin_h_shuf4] +.h_w4_loop: + movq xmm0, [srcq+strideq*0] + movq xmm1, [srcq+strideq*1] + vinserti32x4 ym0, ymm0, [srcq+strideq*2], 1 + vinserti32x4 ym1, ymm1, [srcq+stride3q ], 1 + lea srcq, [srcq+strideq*4] + punpcklqdq ym0, ym1 + pshufb ym0, ym4 + pmaddubsw ym0, ym5 + mova [tmpq], ym0 + add tmpq, 32 + sub hd, 4 + jg .h_w4_loop + RET +.h_w8: + vbroadcasti32x4 m4, [bilin_h_shuf8] +.h_w8_loop: + movu xmm0, [srcq+strideq*0] + vinserti32x4 ym0, ymm0, [srcq+strideq*1], 1 + vinserti32x4 m0, [srcq+strideq*2], 2 + vinserti32x4 m0, [srcq+stride3q ], 3 + lea srcq, [srcq+strideq*4] + pshufb m0, m4 + pmaddubsw m0, m5 + mova [tmpq], m0 + add tmpq, 64 + sub hd, 4 + jg .h_w8_loop + RET +.h_w16: + mova m4, [bilin_h_perm16] +.h_w16_loop: + movu ym0, [srcq+strideq*0] + vinserti32x8 m0, [srcq+strideq*1], 1 + movu ym1, [srcq+strideq*2] + vinserti32x8 m1, [srcq+stride3q ], 1 + lea srcq, [srcq+strideq*4] + vpermb m0, m4, m0 + vpermb m1, m4, m1 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + mova [tmpq+64*0], m0 + mova [tmpq+64*1], m1 + add tmpq, 64*2 + sub hd, 4 + jg .h_w16_loop + RET +.h_w32: + mova m4, [bilin_h_perm32] +.h_w32_loop: + vpermb m0, m4, [srcq+strideq*0] + vpermb m1, m4, [srcq+strideq*1] + vpermb m2, m4, [srcq+strideq*2] + vpermb m3, m4, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmaddubsw m2, m5 + pmaddubsw m3, m5 + mova [tmpq+64*0], m0 + mova [tmpq+64*1], m1 + mova [tmpq+64*2], m2 + mova [tmpq+64*3], m3 + add tmpq, 64*4 + sub hd, 4 + jg .h_w32_loop + RET +.h_w64: + mova m4, [bilin_h_perm32] +.h_w64_loop: + vpermb m0, m4, [srcq+strideq*0+32*0] + vpermb m1, m4, [srcq+strideq*0+32*1] + vpermb m2, m4, [srcq+strideq*1+32*0] + vpermb m3, m4, [srcq+strideq*1+32*1] + lea srcq, [srcq+strideq*2] + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmaddubsw m2, m5 + pmaddubsw m3, m5 + mova [tmpq+64*0], m0 + mova [tmpq+64*1], m1 + mova [tmpq+64*2], m2 + mova [tmpq+64*3], m3 + add tmpq, 64*4 + sub hd, 2 + jg .h_w64_loop + RET +.h_w128: + mova m4, [bilin_h_perm32] +.h_w128_loop: + vpermb m0, m4, [srcq+32*0] + vpermb m1, m4, [srcq+32*1] + vpermb m2, m4, [srcq+32*2] + vpermb m3, m4, [srcq+32*3] + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmaddubsw m2, m5 + pmaddubsw m3, m5 + mova [tmpq+64*0], m0 + mova [tmpq+64*1], m1 + mova [tmpq+64*2], m2 + mova [tmpq+64*3], m3 + add tmpq, 64*4 + add srcq, strideq + dec hd + jg .h_w128_loop + RET +.v: + WIN64_SPILL_XMM 7 + movzx wd, word [t2+wq*2+table_offset(prep, _bilin_v)] + imul mxyd, 0xff01 + add mxyd, 16 << 8 + add wq, t2 + lea stride3q, [strideq*3] + vpbroadcastw m6, mxyd + jmp wq +.v_w4: + vpbroadcastd xm0, [srcq+strideq*0] + mov r3d, 0x29 + vbroadcasti32x4 ym3, [bilin_v_shuf4] + kmovb k1, r3d +.v_w4_loop: + vpblendmd xm1{k1}, xm0, [srcq+strideq*1] {1to4} ; __01 ____ + vpbroadcastd ym2, [srcq+strideq*2] + vpbroadcastd ym2{k1}, [srcq+stride3q ] ; __2_ 23__ + lea srcq, [srcq+strideq*4] + vpbroadcastd ym0, [srcq+strideq*0] + punpckhqdq ym2{k1}, ym1, ym0 ; 012_ 234_ + pshufb ym2, ym3 + pmaddubsw ym2, ym6 + mova [tmpq], ym2 + add tmpq, 32 + sub hd, 4 + jg .v_w4_loop + RET +.v_w8: + mova m5, [bilin_v_perm8] + vbroadcasti32x4 ym0, [srcq+strideq*0] +.v_w8_loop: + vinserti32x4 ym1, ym0, [srcq+strideq*1], 1 + vpbroadcastq ym0, [srcq+strideq*2] + vinserti32x4 m1, [srcq+stride3q ], 2 + lea srcq, [srcq+strideq*4] + vinserti32x4 ym0, [srcq+strideq*0], 0 + vpermt2b m1, m5, m0 + pmaddubsw m1, m6 + mova [tmpq], m1 + add tmpq, 64 + sub hd, 4 + jg .v_w8_loop + RET +.v_w16: + mova m5, [bilin_v_perm16] + movu xm0, [srcq+strideq*0] +.v_w16_loop: + movu xm2, [srcq+strideq*2] + vinserti32x4 ym1, ym0, [srcq+strideq*1], 1 + vpermt2b m1, m5, m2 + vinserti32x4 ym2, [srcq+stride3q ], 1 + lea srcq, [srcq+strideq*4] + movu xm0, [srcq+strideq*0] + vpermt2b m2, m5, m0 + pmaddubsw m1, m6 + pmaddubsw m2, m6 + mova [tmpq+64*0], m1 + mova [tmpq+64*1], m2 + add tmpq, 64*2 + sub hd, 4 + jg .v_w16_loop + RET +.v_w32: + mova m5, [bilin_v_perm32] + movu ym0, [srcq+strideq*0] +.v_w32_loop: + movu ym2, [srcq+strideq*1] + movu ym3, [srcq+strideq*2] + movu ym4, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpermt2b m0, m5, m2 + vpermt2b m2, m5, m3 + vpermt2b m3, m5, m4 + pmaddubsw m1, m0, m6 + movu ym0, [srcq+strideq*0] + vpermt2b m4, m5, m0 + pmaddubsw m2, m6 + pmaddubsw m3, m6 + pmaddubsw m4, m6 + mova [tmpq+64*0], m1 + mova [tmpq+64*1], m2 + mova [tmpq+64*2], m3 + mova [tmpq+64*3], m4 + add tmpq, 64*4 + sub hd, 4 + jg .v_w32_loop + RET +.v_w64: + mova m5, [bilin_v_perm64] + vpermq m0, m5, [srcq+strideq*0] +.v_w64_loop: + vpermq m1, m5, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + punpcklbw m4, m1, m0 + punpckhbw m2, m1, m0 + vpermq m0, m5, [srcq+strideq*0] + punpcklbw m3, m0, m1 + punpckhbw m1, m0, m1 + pmaddubsw m4, m6 + pmaddubsw m2, m6 + pmaddubsw m3, m6 + pmaddubsw m1, m6 + mova [tmpq+64*0], m4 + mova [tmpq+64*1], m2 + mova [tmpq+64*2], m3 + mova [tmpq+64*3], m1 + add tmpq, 64*4 + sub hd, 2 + jg .v_w64_loop + RET +.v_w128: + mova m5, [bilin_v_perm64] + vpermq m0, m5, [srcq+strideq*0+ 0] + vpermq m1, m5, [srcq+strideq*0+64] +.v_w128_loop: + vpermq m2, m5, [srcq+strideq*1+ 0] + vpermq m3, m5, [srcq+strideq*1+64] + lea srcq, [srcq+strideq*2] + punpcklbw m4, m2, m0 + punpckhbw m0, m2, m0 + pmaddubsw m4, m6 + pmaddubsw m0, m6 + mova [tmpq+64*0], m4 + mova [tmpq+64*1], m0 + punpcklbw m4, m3, m1 + punpckhbw m1, m3, m1 + pmaddubsw m4, m6 + pmaddubsw m1, m6 + mova [tmpq+64*2], m4 + mova [tmpq+64*3], m1 + vpermq m0, m5, [srcq+strideq*0+ 0] + vpermq m1, m5, [srcq+strideq*0+64] + punpcklbw m4, m0, m2 + punpckhbw m2, m0, m2 + pmaddubsw m4, m6 + pmaddubsw m2, m6 + mova [tmpq+64*4], m4 + mova [tmpq+64*5], m2 + punpcklbw m4, m1, m3 + punpckhbw m3, m1, m3 + pmaddubsw m4, m6 + pmaddubsw m3, m6 + mova [tmpq+64*6], m4 + mova [tmpq+64*7], m3 + add tmpq, 64*8 + sub hd, 2 + jg .v_w128_loop + RET +.hv: + ; (16 * src[x] + (my * (src[x + src_stride] - src[x])) + 8) >> 4 + ; = src[x] + (((my * (src[x + src_stride] - src[x])) + 8) >> 4) + %assign stack_offset stack_offset - stack_size_padded + WIN64_SPILL_XMM 7 + movzx wd, word [t2+wq*2+table_offset(prep, _bilin_hv)] + shl mxyd, 11 + vpbroadcastw m6, mxyd + add wq, t2 + lea stride3q, [strideq*3] + jmp wq +.hv_w4: + vbroadcasti32x4 ym4, [bilin_h_shuf4] + vpbroadcastq ym0, [srcq+strideq*0] + pshufb ym0, ym4 + pmaddubsw ym0, ym5 +.hv_w4_loop: + movq xmm1, [srcq+strideq*1] + movq xmm2, [srcq+strideq*2] + vinserti32x4 ym1, ymm1, [srcq+stride3q ], 1 + lea srcq, [srcq+strideq*4] + vinserti32x4 ym2, ymm2, [srcq+strideq*0], 1 + punpcklqdq ym1, ym2 + pshufb ym1, ym4 + pmaddubsw ym1, ym5 ; 1 2 3 4 + valignq ym2, ym1, ym0, 3 ; 0 1 2 3 + mova ym0, ym1 + psubw ym1, ym2 + pmulhrsw ym1, ym6 + paddw ym1, ym2 + mova [tmpq], ym1 + add tmpq, 32 + sub hd, 4 + jg .hv_w4_loop + RET +.hv_w8: + vbroadcasti32x4 m4, [bilin_h_shuf8] + vbroadcasti32x4 m0, [srcq+strideq*0] + pshufb m0, m4 + pmaddubsw m0, m5 +.hv_w8_loop: + movu xmm1, [srcq+strideq*1] + vinserti128 ym1, ymm1, [srcq+strideq*2], 1 + vinserti128 m1, [srcq+stride3q ], 2 + lea srcq, [srcq+strideq*4] + vinserti128 m1, [srcq+strideq*0], 3 + pshufb m1, m4 + pmaddubsw m1, m5 ; 1 2 3 4 + valignq m2, m1, m0, 6 ; 0 1 2 3 + mova m0, m1 + psubw m1, m2 + pmulhrsw m1, m6 + paddw m1, m2 + mova [tmpq], m1 + add tmpq, 64 + sub hd, 4 + jg .hv_w8_loop + RET +.hv_w16: + mova m4, [bilin_h_perm16] + vbroadcasti32x8 m0, [srcq+strideq*0] + vpermb m0, m4, m0 + pmaddubsw m0, m5 +.hv_w16_loop: + movu ym1, [srcq+strideq*1] + vinserti32x8 m1, [srcq+strideq*2], 1 + movu ym2, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vinserti32x8 m2, [srcq+strideq*0], 1 + vpermb m1, m4, m1 + vpermb m2, m4, m2 + pmaddubsw m1, m5 ; 1 2 + vshufi32x4 m3, m0, m1, q1032 ; 0 1 + pmaddubsw m0, m2, m5 ; 3 4 + vshufi32x4 m2, m1, m0, q1032 ; 2 3 + psubw m1, m3 + pmulhrsw m1, m6 + paddw m1, m3 + psubw m3, m0, m2 + pmulhrsw m3, m6 + paddw m3, m2 + mova [tmpq+64*0], m1 + mova [tmpq+64*1], m3 + add tmpq, 64*2 + sub hd, 4 + jg .hv_w16_loop + RET +.hv_w32: + mova m4, [bilin_h_perm32] + vpermb m0, m4, [srcq+strideq*0] + pmaddubsw m0, m5 +.hv_w32_loop: + vpermb m1, m4, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vpermb m2, m4, [srcq+strideq*0] + pmaddubsw m1, m5 + psubw m3, m1, m0 + pmulhrsw m3, m6 + paddw m3, m0 + pmaddubsw m0, m2, m5 + psubw m2, m0, m1 + pmulhrsw m2, m6 + paddw m2, m1 + mova [tmpq+64*0], m3 + mova [tmpq+64*1], m2 + add tmpq, 64*2 + sub hd, 2 + jg .hv_w32_loop + RET +.hv_w64: + mova m4, [bilin_h_perm32] + vpermb m0, m4, [srcq+32*0] + vpermb m1, m4, [srcq+32*1] + pmaddubsw m0, m5 + pmaddubsw m1, m5 +.hv_w64_loop: + add srcq, strideq + vpermb m2, m4, [srcq+32*0] + vpermb m3, m4, [srcq+32*1] + pmaddubsw m2, m5 + pmaddubsw m3, m5 + psubw m7, m2, m0 + psubw m8, m3, m1 + pmulhrsw m7, m6 + pmulhrsw m8, m6 + paddw m7, m0 + mova m0, m2 + paddw m8, m1 + mova m1, m3 + mova [tmpq+64*0], m7 + mova [tmpq+64*1], m8 + add tmpq, 64*2 + dec hd + jg .hv_w64_loop + RET +.hv_w128: + mova m4, [bilin_h_perm32] + vpermb m0, m4, [srcq+32*0] + vpermb m1, m4, [srcq+32*1] + vpermb m2, m4, [srcq+32*2] + vpermb m3, m4, [srcq+32*3] + REPX {pmaddubsw x, m5}, m0, m1, m2, m3 +.hv_w128_loop: + add srcq, strideq + vpermb m7, m4, [srcq+32*0] + vpermb m8, m4, [srcq+32*1] + vpermb m9, m4, [srcq+32*2] + vpermb m10, m4, [srcq+32*3] + REPX {pmaddubsw x, m5}, m7, m8, m9, m10 + psubw m11, m7, m0 + psubw m12, m8, m1 + psubw m13, m9, m2 + psubw m14, m10, m3 + REPX {pmulhrsw x, m6}, m11, m12, m13, m14 + paddw m11, m0 + mova m0, m7 + paddw m12, m1 + mova m1, m8 + paddw m13, m2 + mova m2, m9 + paddw m14, m3 + mova m3, m10 + mova [tmpq+64*0], m11 + mova [tmpq+64*1], m12 + mova [tmpq+64*2], m13 + mova [tmpq+64*3], m14 + add tmpq, 64*4 + dec hd + jg .hv_w128_loop + RET + +; int8_t subpel_filters[5][15][8] +%assign FILTER_REGULAR (0*15 << 16) | 3*15 +%assign FILTER_SMOOTH (1*15 << 16) | 4*15 +%assign FILTER_SHARP (2*15 << 16) | 3*15 + +%macro FN 4 ; fn, type, type_h, type_v +cglobal %1_%2 + mov t0d, FILTER_%3 +%ifidn %3, %4 + mov t1d, t0d +%else + mov t1d, FILTER_%4 +%endif +%ifnidn %2, regular ; skip the jump in the last filter + jmp mangle(private_prefix %+ _%1 %+ SUFFIX) +%endif +%endmacro + +%macro PREP_8TAP_H 0 + vpermb m10, m5, m0 + vpermb m11, m5, m1 + vpermb m12, m6, m0 + vpermb m13, m6, m1 + vpermb m14, m7, m0 + vpermb m15, m7, m1 + mova m0, m4 + vpdpbusd m0, m10, m8 + mova m2, m4 + vpdpbusd m2, m12, m8 + mova m1, m4 + vpdpbusd m1, m11, m8 + mova m3, m4 + vpdpbusd m3, m13, m8 + vpdpbusd m0, m12, m9 + vpdpbusd m2, m14, m9 + vpdpbusd m1, m13, m9 + vpdpbusd m3, m15, m9 + packssdw m0, m2 + packssdw m1, m3 + psraw m0, 2 + psraw m1, 2 + mova [tmpq+64*0], m0 + mova [tmpq+64*1], m1 +%endmacro + +%if WIN64 +DECLARE_REG_TMP 6, 4 +%else +DECLARE_REG_TMP 6, 7 +%endif + +%define PREP_8TAP_FN FN prep_8tap, + +PREP_8TAP_FN sharp, SHARP, SHARP +PREP_8TAP_FN sharp_smooth, SHARP, SMOOTH +PREP_8TAP_FN smooth_sharp, SMOOTH, SHARP +PREP_8TAP_FN smooth, SMOOTH, SMOOTH +PREP_8TAP_FN sharp_regular, SHARP, REGULAR +PREP_8TAP_FN regular_sharp, REGULAR, SHARP +PREP_8TAP_FN smooth_regular, SMOOTH, REGULAR +PREP_8TAP_FN regular_smooth, REGULAR, SMOOTH +PREP_8TAP_FN regular, REGULAR, REGULAR + +cglobal prep_8tap, 3, 8, 0, tmp, src, stride, w, h, mx, my, stride3 + imul mxd, mxm, 0x010101 + add mxd, t0d ; 8tap_h, mx, 4tap_h + imul myd, mym, 0x010101 + add myd, t1d ; 8tap_v, my, 4tap_v + lea r7, [prep_avx512icl] + movsxd wq, wm + movifnidn hd, hm + test mxd, 0xf00 + jnz .h + test myd, 0xf00 + jnz .v + tzcnt wd, wd + movzx wd, word [r7+wq*2+table_offset(prep,)] + add wq, r7 + lea r6, [strideq*3] +%if WIN64 + pop r7 +%endif + jmp wq +.h: + test myd, 0xf00 + jnz .hv + vpbroadcastd m4, [pd_2] + WIN64_SPILL_XMM 10 + cmp wd, 4 + je .h_w4 + tzcnt wd, wd + shr mxd, 16 + sub srcq, 3 + movzx wd, word [r7+wq*2+table_offset(prep, _8tap_h)] + vpbroadcastd m8, [r7+mxq*8+subpel_filters-prep_avx512icl+0] + vpbroadcastd m9, [r7+mxq*8+subpel_filters-prep_avx512icl+4] + add wq, r7 + jmp wq +.h_w4: + movzx mxd, mxb + vbroadcasti128 ym5, [subpel_h_shufA] + mov r3d, 0x4 + dec srcq + vpbroadcastd ym6, [r7+mxq*8+subpel_filters-prep_avx512icl+2] + kmovb k1, r3d + lea stride3q, [strideq*3] +.h_w4_loop: + movq xm2, [srcq+strideq*0] + movq xm3, [srcq+strideq*1] + vpbroadcastq ym2{k1}, [srcq+strideq*2] + vpbroadcastq ym3{k1}, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + pshufb ym2, ym5 + pshufb ym3, ym5 + mova ym0, ym4 + vpdpbusd ym0, ym2, ym6 + mova ym1, ym4 + vpdpbusd ym1, ym3, ym6 + packssdw ym0, ym1 + psraw ym0, 2 + mova [tmpq], ym0 + add tmpq, 32 + sub hd, 4 + jg .h_w4_loop + RET +.h_w8: + vbroadcasti128 m5, [subpel_h_shufA] + vbroadcasti128 m6, [subpel_h_shufB] + vbroadcasti128 m7, [subpel_h_shufC] + lea stride3q, [strideq*3] +.h_w8_loop: + movu xmm3, [srcq+strideq*0] + vinserti128 ym3, ymm3, [srcq+strideq*1], 1 + vinserti128 m3, [srcq+strideq*2], 2 + vinserti128 m3, [srcq+stride3q ], 3 + lea srcq, [srcq+strideq*4] + pshufb m1, m3, m5 + pshufb m2, m3, m6 + mova m0, m4 + vpdpbusd m0, m1, m8 + mova m1, m4 + vpdpbusd m1, m2, m8 + pshufb m3, m7 + vpdpbusd m0, m2, m9 + vpdpbusd m1, m3, m9 + packssdw m0, m1 + psraw m0, 2 + mova [tmpq], m0 + add tmpq, 64 + sub hd, 4 + jg .h_w8_loop + RET +.h_w16: + mova m5, [spel_h_perm16a] + mova m6, [spel_h_perm16b] + mova m7, [spel_h_perm16c] + lea stride3q, [strideq*3] +.h_w16_loop: + movu ym0, [srcq+strideq*0] + movu ym1, [srcq+strideq*2] + vinserti32x8 m0, [srcq+strideq*1], 1 + vinserti32x8 m1, [srcq+stride3q ], 1 + lea srcq, [srcq+strideq*4] + PREP_8TAP_H + add tmpq, 64*2 + sub hd, 4 + jg .h_w16_loop + RET +.h_w32: + mova m5, [spel_h_perm32a] + mova m6, [spel_h_perm32b] + mova m7, [spel_h_perm32c] +.h_w32_loop: + movu m0, [srcq+strideq*0] + movu m1, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + PREP_8TAP_H + add tmpq, 64*2 + sub hd, 2 + jg .h_w32_loop + RET +.h_w64: + xor r6d, r6d + jmp .h_start +.h_w128: + mov r6, -64*1 +.h_start: + mova m5, [spel_h_perm32a] + mova m6, [spel_h_perm32b] + mova m7, [spel_h_perm32c] + sub srcq, r6 + mov r5, r6 +.h_loop: + movu m0, [srcq+r6+32*0] + movu m1, [srcq+r6+32*1] + PREP_8TAP_H + add tmpq, 64*2 + add r6, 64 + jle .h_loop + add srcq, strideq + mov r6, r5 + dec hd + jg .h_loop + RET +.v: + movzx mxd, myb ; Select 4-tap/8-tap filter multipliers. + shr myd, 16 ; Note that the code is 8-tap only, having + tzcnt wd, wd + cmp hd, 4 ; a separate 4-tap code path for (4|8|16)x4 + cmove myd, mxd ; had a negligible effect on performance. + ; TODO: Would a 6-tap code path be worth it? + lea myq, [r7+myq*8+subpel_filters-prep_avx512icl] + movzx wd, word [r7+wq*2+table_offset(prep, _8tap_v)] + add wq, r7 + lea stride3q, [strideq*3] + sub srcq, stride3q + vpbroadcastd m7, [pw_8192] + vpbroadcastw m8, [myq+0] + vpbroadcastw m9, [myq+2] + vpbroadcastw m10, [myq+4] + vpbroadcastw m11, [myq+6] + jmp wq +.v_w4: + movd xmm0, [srcq+strideq*0] + vpbroadcastd ymm1, [srcq+strideq*2] + vpbroadcastd xmm2, [srcq+strideq*1] + vpbroadcastd ymm3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpblendd ymm1, ymm0, 0x01 ; 0 2 2 _ 2 _ _ _ + vpblendd ymm3, ymm2, 0x03 ; 1 1 3 3 3 3 _ _ + vpbroadcastd ymm0, [srcq+strideq*0] + vpbroadcastd ymm2, [srcq+strideq*1] + vpblendd ymm1, ymm0, 0x68 ; 0 2 2 4 2 4 4 _ + vpbroadcastd ymm0, [srcq+strideq*2] + vbroadcasti128 ymm5, [deint_shuf4] + vpblendd ymm3, ymm2, 0xc0 ; 1 1 3 3 3 3 5 5 + vpblendd ymm2, ymm3, ymm1, 0x55 ; 0 1 2 3 2 3 4 5 + vpblendd ymm3, ymm1, 0xaa ; 1 2 3 4 3 4 5 _ + punpcklbw ymm1, ymm2, ymm3 ; 01 12 23 34 + vpblendd ymm3, ymm0, 0x80 ; 1 2 3 4 3 4 5 6 + punpckhbw ymm2, ymm3 ; 23 34 45 56 +.v_w4_loop: + pinsrd xmm0, [srcq+stride3q ], 1 + lea srcq, [srcq+strideq*4] + vpbroadcastd ymm3, [srcq+strideq*0] + vpbroadcastd ymm4, [srcq+strideq*1] + vpblendd ymm3, ymm4, 0x20 ; _ _ 8 _ 8 9 _ _ + vpblendd ymm3, ymm0, 0x03 ; 6 7 8 _ 8 9 _ _ + vpbroadcastd ymm0, [srcq+strideq*2] + vpblendd ymm3, ymm0, 0x40 ; 6 7 8 _ 8 9 a _ + pshufb ymm3, ymm5 ; 67 78 89 9a + pmaddubsw ymm4, ymm1, ym8 + vperm2i128 ymm1, ymm2, ymm3, 0x21 ; 45 56 67 78 + pmaddubsw ymm2, ym9 + paddw ymm4, ymm2 + mova ymm2, ymm3 + pmaddubsw ymm3, ym11 + paddw ymm3, ymm4 + pmaddubsw ymm4, ymm1, ym10 + paddw ymm3, ymm4 + pmulhrsw ymm3, ym7 + mova [tmpq], ymm3 + add tmpq, 32 + sub hd, 4 + jg .v_w4_loop + vzeroupper + RET +.v_w8: + mov r3d, 0xf044 + kmovw k1, r3d + kshiftrw k2, k1, 8 + movq xm0, [srcq+strideq*0] + vpbroadcastq ym1, [srcq+strideq*1] + vpbroadcastq m2, [srcq+strideq*2] + vpbroadcastq m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpbroadcastq m4, [srcq+strideq*0] + vpbroadcastq m5, [srcq+strideq*1] + vpbroadcastq m6, [srcq+strideq*2] + vmovdqa64 ym0{k1}, ym1 + vmovdqa64 ym1{k1}, ym2 + vmovdqa64 m2{k1}, m3 + vmovdqa64 m3{k1}, m4 + vmovdqa64 m4{k1}, m5 + vmovdqa64 m5{k1}, m6 + punpcklbw ym0, ym1 ; 01 12 __ __ + punpcklbw m2, m3 ; 23 34 23 34 + punpcklbw m4, m5 ; 45 56 45 56 + vmovdqa64 m0{k2}, m2 ; 01 12 23 34 + vmovdqa64 m2{k2}, m4 ; 23 34 45 56 +.v_w8_loop: + vpbroadcastq m1, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpbroadcastq m3, [srcq+strideq*0] + vpbroadcastq m5, [srcq+strideq*1] + pmaddubsw m14, m0, m8 + pmaddubsw m15, m2, m9 + vpblendmq m0{k1}, m6, m1 + vpblendmq m2{k1}, m1, m3 + vpbroadcastq m6, [srcq+strideq*2] + paddw m14, m15 + punpcklbw m2, m0, m2 ; 67 78 67 78 + vpblendmq m12{k1}, m3, m5 + vpblendmq m13{k1}, m5, m6 + vpblendmq m0{k2}, m4, m2 ; 45 56 67 78 + punpcklbw m4, m12, m13 ; 89 9a 89 9a + vmovdqa64 m2{k2}, m4 ; 67 78 89 9a + pmaddubsw m12, m0, m10 + pmaddubsw m13, m2, m11 + paddw m14, m12 + paddw m14, m13 + pmulhrsw m14, m7 + mova [tmpq], m14 + add tmpq, 64 + sub hd, 4 + jg .v_w8_loop + RET +.v_w16: + mov r3d, 0xf0 + kmovb k1, r3d + vbroadcasti128 m0, [srcq+strideq*0] + vbroadcasti128 m1, [srcq+strideq*1] + vbroadcasti128 m2, [srcq+strideq*2] + vbroadcasti128 m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vbroadcasti128 m4, [srcq+strideq*0] + vbroadcasti128 m5, [srcq+strideq*1] + vbroadcasti128 m6, [srcq+strideq*2] + vmovdqa64 m0{k1}, m1 + vmovdqa64 m1{k1}, m2 + vmovdqa64 m2{k1}, m3 + vmovdqa64 m3{k1}, m4 + vmovdqa64 m4{k1}, m5 + vmovdqa64 m5{k1}, m6 + shufpd m0, m2, 0xcc ; 0a_2a 0b_2b 1a_3a 1b_3b + shufpd m1, m3, 0xcc ; 1a_3a 1b_3b 2a_4a 2b_4b + shufpd m4, m4, 0x44 ; 4a_-- 4b_-- 5a_-- 5b_-- + shufpd m5, m5, 0x44 ; 5a_-- 5b_-- 6a_-- 6b_-- + punpckhbw m2, m0, m1 ; 23a 23b 34a 34b + punpcklbw m0, m1 ; 01a 01b 12a 12b + punpcklbw m4, m5 ; 45a 45b 56a 56b +.v_w16_loop: + vbroadcasti128 m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vbroadcasti128 m5, [srcq+strideq*0] + vpblendmq m1{k1}, m6, m3 + vmovdqa64 m3{k1}, m5 + pmaddubsw m12, m0, m8 + pmaddubsw m13, m2, m8 + pmaddubsw m14, m2, m9 + pmaddubsw m15, m4, m9 + pmaddubsw m0, m4, m10 + vbroadcasti128 m2, [srcq+strideq*1] + vbroadcasti128 m6, [srcq+strideq*2] + paddw m12, m14 + paddw m13, m15 + paddw m12, m0 + vmovdqa64 m5{k1}, m2 + vmovdqa64 m2{k1}, m6 + mova m0, m4 + shufpd m1, m5, 0xcc ; 6a_8a 6b_8b 7a_9a 7b_9b + shufpd m3, m2, 0xcc ; 7a_9a 7b_9b 8a_Aa 8b_Ab + punpcklbw m2, m1, m3 ; 67a 67b 78a 78b + punpckhbw m4, m1, m3 ; 89a 89b 9Aa 9Ab + pmaddubsw m14, m2, m10 + pmaddubsw m15, m2, m11 + paddw m13, m14 + paddw m12, m15 + pmaddubsw m14, m4, m11 + paddw m13, m14 + pmulhrsw m12, m7 + pmulhrsw m13, m7 + mova [tmpq+ 0], m12 + mova [tmpq+64], m13 + add tmpq, 64*2 + sub hd, 4 + jg .v_w16_loop + RET +.v_w32: + mova m18, [bilin_v_perm64] + movu ym0, [srcq+strideq*0] + movu ym1, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + movu ym2, [srcq+strideq*0] + movu ym3, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + movu ym4, [srcq+strideq*0] + movu ym5, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + movu ym6, [srcq+strideq*0] + vpermq m0, m18, m0 + vpermq m1, m18, m1 + vpermq m2, m18, m2 + vpermq m3, m18, m3 + vpermq m4, m18, m4 + vpermq m5, m18, m5 + vpermq m6, m18, m6 + punpcklbw m0, m1 + punpcklbw m1, m2 + punpcklbw m2, m3 + punpcklbw m3, m4 + punpcklbw m4, m5 + punpcklbw m5, m6 +.v_w32_loop: + movu ym12, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + movu ym13, [srcq+strideq*0] + pmaddubsw m14, m0, m8 + pmaddubsw m16, m2, m9 + pmaddubsw m15, m1, m8 + pmaddubsw m17, m3, m9 + mova m0, m2 + mova m1, m3 + vpermq m12, m18, m12 + vpermq m13, m18, m13 + paddw m14, m16 + paddw m15, m17 + pmaddubsw m16, m4, m10 + pmaddubsw m17, m5, m10 + punpcklbw m6, m12 + punpcklbw m12, m13 + mova m2, m4 + mova m3, m5 + paddw m14, m16 + paddw m15, m17 + pmaddubsw m16, m6, m11 + pmaddubsw m17, m12, m11 + mova m4, m6 + mova m5, m12 + paddw m14, m16 + paddw m15, m17 + pmulhrsw m14, m7 + pmulhrsw m15, m7 + mova m6, m13 + mova [tmpq+ 0], m14 + mova [tmpq+64], m15 + add tmpq, 64*2 + sub hd, 2 + jg .v_w32_loop + vzeroupper + RET +.v_w64: + mov wd, 64 + jmp .v_start +.v_w128: + mov wd, 128 +.v_start: + WIN64_SPILL_XMM 27 + mova m26, [bilin_v_perm64] + lea r6d, [hq+wq*2] + mov r5, srcq + mov r7, tmpq +.v_loop0: + vpermq m0, m26, [srcq+strideq*0] + vpermq m1, m26, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vpermq m2, m26, [srcq+strideq*0] + vpermq m3, m26, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vpermq m4, m26, [srcq+strideq*0] + vpermq m5, m26, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vpermq m6, m26, [srcq+strideq*0] + punpckhbw m12, m0, m1 + punpcklbw m0, m1 + punpckhbw m13, m1, m2 + punpcklbw m1, m2 + punpckhbw m14, m2, m3 + punpcklbw m2, m3 + punpckhbw m15, m3, m4 + punpcklbw m3, m4 + punpckhbw m16, m4, m5 + punpcklbw m4, m5 + punpckhbw m17, m5, m6 + punpcklbw m5, m6 +.v_loop: + vpermq m18, m26, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vpermq m19, m26, [srcq+strideq*0] + pmaddubsw m20, m0, m8 + pmaddubsw m21, m12, m8 + pmaddubsw m22, m1, m8 + pmaddubsw m23, m13, m8 + mova m0, m2 + mova m12, m14 + mova m1, m3 + mova m13, m15 + pmaddubsw m2, m9 + pmaddubsw m14, m9 + pmaddubsw m3, m9 + pmaddubsw m15, m9 + punpckhbw m24, m6, m18 + punpcklbw m6, m18 + paddw m20, m2 + paddw m21, m14 + paddw m22, m3 + paddw m23, m15 + mova m2, m4 + mova m14, m16 + mova m3, m5 + mova m15, m17 + pmaddubsw m4, m10 + pmaddubsw m16, m10 + pmaddubsw m5, m10 + pmaddubsw m17, m10 + punpckhbw m25, m18, m19 + punpcklbw m18, m19 + paddw m20, m4 + paddw m21, m16 + paddw m22, m5 + paddw m23, m17 + mova m4, m6 + mova m16, m24 + mova m5, m18 + mova m17, m25 + pmaddubsw m6, m11 + pmaddubsw m24, m11 + pmaddubsw m18, m11 + pmaddubsw m25, m11 + paddw m20, m6 + paddw m21, m24 + paddw m22, m18 + paddw m23, m25 + pmulhrsw m20, m7 + pmulhrsw m21, m7 + pmulhrsw m22, m7 + pmulhrsw m23, m7 + mova m6, m19 + mova [tmpq+wq*0+ 0], m20 + mova [tmpq+wq*0+64], m21 + mova [tmpq+wq*2+ 0], m22 + mova [tmpq+wq*2+64], m23 + lea tmpq, [tmpq+wq*4] + sub hd, 2 + jg .v_loop + add r5, 64 + add r7, 128 + movzx hd, r6b + mov srcq, r5 + mov tmpq, r7 + sub r6d, 1<<8 + jg .v_loop0 + RET +.hv: + %assign stack_offset stack_offset - stack_size_padded + %assign stack_size_padded 0 + WIN64_SPILL_XMM 16 + cmp wd, 4 + je .hv_w4 + shr mxd, 16 + sub srcq, 3 + vpbroadcastd m10, [r7+mxq*8+subpel_filters-prep_avx512icl+0] + vpbroadcastd m11, [r7+mxq*8+subpel_filters-prep_avx512icl+4] + movzx mxd, myb + shr myd, 16 + cmp hd, 4 + cmove myd, mxd + tzcnt wd, wd + vpbroadcastd m8, [pd_2] + movzx wd, word [r7+wq*2+table_offset(prep, _8tap_hv)] + vpbroadcastd m9, [pd_32] + add wq, r7 + vpbroadcastq m0, [r7+myq*8+subpel_filters-prep_avx512icl] + lea stride3q, [strideq*3] + sub srcq, stride3q + punpcklbw m0, m0 + psraw m0, 8 ; sign-extend + pshufd m12, m0, q0000 + pshufd m13, m0, q1111 + pshufd m14, m0, q2222 + pshufd m15, m0, q3333 + jmp wq +.hv_w4: + movzx mxd, mxb + dec srcq + vpbroadcastd m8, [r7+mxq*8+subpel_filters-prep_avx512icl+2] + movzx mxd, myb + shr myd, 16 + cmp hd, 4 + cmove myd, mxd + vpbroadcastq m0, [r7+myq*8+subpel_filters-prep_avx512icl] + lea stride3q, [strideq*3] + sub srcq, stride3q + mov r3d, 0x04 + kmovb k1, r3d + kshiftlb k2, k1, 2 + kshiftlb k3, k1, 4 + vpbroadcastd m10, [pd_2] + vbroadcasti128 m16, [subpel_h_shufA] + punpcklbw m0, m0 + psraw m0, 8 ; sign-extend + vpbroadcastd m11, [pd_32] + pshufd m12, m0, q0000 + pshufd m13, m0, q1111 + pshufd m14, m0, q2222 + pshufd m15, m0, q3333 + movq xm3, [srcq+strideq*0] + vpbroadcastq ym2, [srcq+strideq*1] + vpbroadcastq ym3{k1}, [srcq+strideq*2] + vpbroadcastq m2{k2}, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vpbroadcastq m3{k2}, [srcq+strideq*0] + vpbroadcastq m2{k3}, [srcq+strideq*1] + vpbroadcastq m3{k3}, [srcq+strideq*2] + mova m17, [spel_hv_perm4a] + movu m18, [spel_hv_perm4b] + mova m0, m10 + mova m1, m10 + pshufb m2, m16 + pshufb m3, m16 + vpdpbusd m0, m2, m8 + vpdpbusd m1, m3, m8 + packssdw m0, m1 ; _ 0 1 2 3 4 5 6 + psraw m0, 2 + vpermb m1, m17, m0 ; 01 12 23 34 + vpermb m2, m18, m0 ; 23 34 45 56 +.hv_w4_loop: + movq xm3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + movq xm4, [srcq+strideq*0] + vpbroadcastq ym3{k1}, [srcq+strideq*1] + vpbroadcastq ym4{k1}, [srcq+strideq*2] + mova ym5, ym10 + mova ym6, ym10 + pshufb ym3, ym16 + pshufb ym4, ym16 + vpdpbusd ym5, ym3, ym8 + vpdpbusd ym6, ym4, ym8 + mova m7, m11 + packssdw ym5, ym6 ; 7 8 9 a _ _ _ _ + psraw ym5, 2 + valignq m0, m5, m0, 4 ; _ 4 5 6 7 8 9 a + vpdpwssd m7, m1, m12 + vpdpwssd m7, m2, m13 + vpermb m1, m17, m0 ; 45 56 67 78 + vpermb m2, m18, m0 ; 67 78 89 9a + vpdpwssd m7, m1, m14 + vpdpwssd m7, m2, m15 + psrad m7, 6 + vpmovdw [tmpq], m7 + add tmpq, 32 + sub hd, 4 + jg .hv_w4_loop + vzeroupper + RET +.hv_w8: + WIN64_SPILL_XMM 24 + vbroadcasti128 m16, [subpel_h_shufA] + vbroadcasti128 m17, [subpel_h_shufB] + vbroadcasti128 m18, [subpel_h_shufC] + vinserti128 ym0, [srcq+strideq*0], 1 + vinserti128 m0, [srcq+strideq*1], 2 + vinserti128 m0, [srcq+strideq*2], 3 + movu xm1, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vinserti128 ym1, [srcq+strideq*0], 1 + vinserti128 m1, [srcq+strideq*1], 2 + vinserti128 m1, [srcq+strideq*2], 3 + mova m2, m8 + mova m4, m8 + mova m3, m8 + mova m5, m8 + pshufb m20, m0, m16 + pshufb m21, m0, m17 + pshufb m22, m0, m18 + pshufb m23, m1, m16 + pshufb m6, m1, m17 + pshufb m7, m1, m18 + vpdpbusd m2, m20, m10 + vpdpbusd m4, m21, m10 + vpdpbusd m2, m21, m11 + vpdpbusd m4, m22, m11 + vpdpbusd m3, m23, m10 + vpdpbusd m5, m6, m10 + vpdpbusd m3, m6, m11 + vpdpbusd m5, m7, m11 + packssdw m2, m4 + packssdw m3, m5 + psraw m2, 2 ; _ 0 1 2 + psraw m3, 2 ; 3 4 5 6 + valignq m0, m3, m2, 2 ; 0 1 2 3 + valignq m1, m3, m2, 4 ; 1 2 3 4 + valignq m2, m3, m2, 6 ; 2 3 4 5 + punpcklwd m4, m0, m1 ; 01a 12a 23a 34a + punpckhwd m5, m0, m1 ; 01b 12b 23b 34b + punpcklwd m6, m2, m3 ; 23a 34a 45a 56a + punpckhwd m7, m2, m3 ; 23b 34b 45b 56b +.hv_w8_loop: + movu xm19, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + vinserti128 ym19, [srcq+strideq*0], 1 + vinserti128 m19, [srcq+strideq*1], 2 + vinserti128 m19, [srcq+strideq*2], 3 + mova m20, m9 + mova m21, m9 + mova m22, m8 + mova m23, m8 + vpdpwssd m20, m4, m12 + vpdpwssd m21, m5, m12 + vpdpwssd m20, m6, m13 + vpdpwssd m21, m7, m13 + pshufb m0, m19, m16 + pshufb m1, m19, m17 + pshufb m2, m19, m18 + vpdpbusd m22, m0, m10 + vpdpbusd m23, m1, m10 + vpdpbusd m22, m1, m11 + vpdpbusd m23, m2, m11 + packssdw m22, m23 + psraw m22, 2 ; 7 8 9 A + valignq m0, m22, m3, 2 ; 4 5 6 7 + valignq m1, m22, m3, 4 ; 5 6 7 8 + valignq m2, m22, m3, 6 ; 6 7 8 9 + mova m3, m22 + punpcklwd m4, m0, m1 ; 45a 56a 67a 78a + punpckhwd m5, m0, m1 ; 45b 56b 67b 78b + punpcklwd m6, m2, m3 ; 67a 78a 89a 9Aa + punpckhwd m7, m2, m3 ; 67b 78b 89b 9Ab + vpdpwssd m20, m4, m14 + vpdpwssd m21, m5, m14 + vpdpwssd m20, m6, m15 + vpdpwssd m21, m7, m15 + psrad m20, 6 + psrad m21, 6 + packssdw m20, m21 + mova [tmpq], m20 + add tmpq, 64 + sub hd, 4 + jg .hv_w8_loop + RET +.hv_w16: + mov wd, 16*2 + jmp .hv_start +.hv_w32: + mov wd, 32*2 + jmp .hv_start +.hv_w64: + mov wd, 64*2 + jmp .hv_start +.hv_w128: + mov wd, 128*2 +.hv_start: + WIN64_SPILL_XMM 31 + mova m16, [spel_h_perm16a] + mova m17, [spel_h_perm16b] + mova m18, [spel_h_perm16c] + lea r6d, [hq+wq*8-256] + mov r5, srcq + mov r7, tmpq +.hv_loop0: + movu ym0, [srcq+strideq*0] + vinserti32x8 m0, [srcq+strideq*1], 1 + lea srcq, [srcq+strideq*2] + movu ym1, [srcq+strideq*0] + vinserti32x8 m1, [srcq+strideq*1], 1 + lea srcq, [srcq+strideq*2] + movu ym2, [srcq+strideq*0] + vinserti32x8 m2, [srcq+strideq*1], 1 + lea srcq, [srcq+strideq*2] + movu ym3, [srcq+strideq*0] + mova m4, m8 + mova m5, m8 + mova m6, m8 + mova m7, m8 + vpermb m19, m16, m0 + vpermb m20, m17, m0 + vpermb m21, m18, m0 + vpermb m22, m16, m1 + vpermb m23, m17, m1 + vpermb m24, m18, m1 + vpermb m25, m16, m2 + vpermb m26, m17, m2 + vpermb m27, m18, m2 + vpermb ym28, ym16, ym3 + vpermb ym29, ym17, ym3 + vpermb ym30, ym18, ym3 + mova m0, m8 + mova m1, m8 + mova ym2, ym8 + mova ym3, ym8 + vpdpbusd m4, m19, m10 + vpdpbusd m5, m20, m10 + vpdpbusd m6, m22, m10 + vpdpbusd m7, m23, m10 + vpdpbusd m0, m25, m10 + vpdpbusd m1, m26, m10 + vpdpbusd ym2, ym28, ym10 + vpdpbusd ym3, ym29, ym10 + vpdpbusd m4, m20, m11 + vpdpbusd m5, m21, m11 + vpdpbusd m6, m23, m11 + vpdpbusd m7, m24, m11 + vpdpbusd m0, m26, m11 + vpdpbusd m1, m27, m11 + vpdpbusd ym2, ym29, ym11 + vpdpbusd ym3, ym30, ym11 + packssdw m4, m5 + packssdw m6, m7 + packssdw m0, m1 + packssdw ym2, ym3 + psraw m4, 2 ; 0a 0b 1a 1b + psraw m6, 2 ; 2a 2b 3a 3b + psraw m0, 2 ; 4a 4b 5a 5b + psraw ym2, 2 ; 6a 6b __ __ + vshufi32x4 m5, m4, m6, q1032 ; 1a 1b 2a 2b + vshufi32x4 m7, m6, m0, q1032 ; 3a 3b 4a 4b + vshufi32x4 m1, m0, m2, q1032 ; 5a 5b 6a 6b + punpcklwd m2, m4, m5 ; 01a 01c 12a 12c + punpckhwd m3, m4, m5 ; 01b 01d 12b 12d + punpcklwd m4, m6, m7 ; 23a 23c 34a 34c + punpckhwd m5, m6, m7 ; 23b 23d 34b 34d + punpcklwd m6, m0, m1 ; 45a 45c 56a 56c + punpckhwd m7, m0, m1 ; 45b 45d 56b 56d +.hv_loop: + movu ym19, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + vinserti32x8 m19, [srcq+strideq*0], 1 + mova m20, m9 + mova m21, m9 + mova m22, m8 + mova m23, m8 + vpdpwssd m20, m2, m12 + vpdpwssd m21, m3, m12 + vpdpwssd m20, m4, m13 + vpdpwssd m21, m5, m13 + vpermb m24, m16, m19 + vpermb m25, m17, m19 + vpermb m26, m18, m19 + vpdpbusd m22, m24, m10 + vpdpbusd m23, m25, m10 + vpdpbusd m22, m25, m11 + vpdpbusd m23, m26, m11 + packssdw m22, m23 + psraw m22, 2 ; 7a 7b 8a 8b + vshufi32x4 m0, m1, m22, q1032 ; 6a 6b 7a 7b + mova m2, m4 + mova m3, m5 + mova m1, m22 + mova m4, m6 + mova m5, m7 + punpcklwd m6, m0, m1 ; 67a 67c 78a 78c + punpckhwd m7, m0, m1 ; 67b 67d 78b 78d + vpdpwssd m20, m4, m14 + vpdpwssd m21, m5, m14 + vpdpwssd m20, m6, m15 + vpdpwssd m21, m7, m15 + psrad m20, 6 + psrad m21, 6 + packssdw m20, m21 + mova [tmpq+wq*0], ym20 + vextracti32x8 [tmpq+wq*1], m20, 1 + lea tmpq, [tmpq+wq*2] + sub hd, 2 + jg .hv_loop + add r5, 16 + add r7, 32 + movzx hd, r6b + mov srcq, r5 + mov tmpq, r7 + sub r6d, 1<<8 + jg .hv_loop0 + RET + +%macro BIDIR_FN 1 ; op + lea stride3q, [strideq*3] + jmp wq +.w4: + cmp hd, 8 + jg .w4_h16 + WRAP_YMM %1 0 + vextracti32x4 xmm1, ym0, 1 + movd [dstq ], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xmm1 + pextrd [dstq+stride3q ], xmm1, 1 + jl .w4_ret + lea dstq, [dstq+strideq*4] + pextrd [dstq ], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xmm1, 2 + pextrd [dstq+stride3q ], xmm1, 3 +.w4_ret: + RET +.w4_h16: + vpbroadcastd m7, strided + pmulld m7, [bidir_sctr_w4] + %1 0 + kxnorw k1, k1, k1 + vpscatterdd [dstq+m7]{k1}, m0 + RET +.w8: + cmp hd, 4 + jne .w8_h8 + WRAP_YMM %1 0 + vextracti128 xmm1, ym0, 1 + movq [dstq ], xm0 + movq [dstq+strideq*1], xmm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xmm1 + RET +.w8_loop: + %1_INC_PTR 2 + lea dstq, [dstq+strideq*4] +.w8_h8: + %1 0 + vextracti32x4 xmm1, ym0, 1 + vextracti32x4 xmm2, m0, 2 + vextracti32x4 xmm3, m0, 3 + movq [dstq ], xm0 + movq [dstq+strideq*1], xmm1 + movq [dstq+strideq*2], xmm2 + movq [dstq+stride3q ], xmm3 + lea dstq, [dstq+strideq*4] + movhps [dstq ], xm0 + movhps [dstq+strideq*1], xmm1 + movhps [dstq+strideq*2], xmm2 + movhps [dstq+stride3q ], xmm3 + sub hd, 8 + jg .w8_loop + RET +.w16_loop: + %1_INC_PTR 2 + lea dstq, [dstq+strideq*4] +.w16: + %1 0 + vpermq m0, m0, q3120 + mova [dstq ], xm0 + vextracti32x4 [dstq+strideq*1], m0, 2 + vextracti32x4 [dstq+strideq*2], ym0, 1 + vextracti32x4 [dstq+stride3q ], m0, 3 + sub hd, 4 + jg .w16_loop + RET +.w32: + pmovzxbq m7, [pb_02461357] +.w32_loop: + %1 0 + %1_INC_PTR 2 + vpermq m0, m7, m0 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w32_loop + RET +.w64: + pmovzxbq m7, [pb_02461357] +.w64_loop: + %1 0 + %1_INC_PTR 2 + vpermq m0, m7, m0 + mova [dstq], m0 + add dstq, strideq + dec hd + jg .w64_loop + RET +.w128: + pmovzxbq m7, [pb_02461357] +.w128_loop: + %1 0 + vpermq m6, m7, m0 + %1 2 + mova [dstq+64*0], m6 + %1_INC_PTR 4 + vpermq m6, m7, m0 + mova [dstq+64*1], m6 + add dstq, strideq + dec hd + jg .w128_loop + RET +%endmacro + +%macro AVG 1 ; src_offset + mova m0, [tmp1q+(%1+0)*mmsize] + paddw m0, [tmp2q+(%1+0)*mmsize] + mova m1, [tmp1q+(%1+1)*mmsize] + paddw m1, [tmp2q+(%1+1)*mmsize] + pmulhrsw m0, m2 + pmulhrsw m1, m2 + packuswb m0, m1 +%endmacro + +%macro AVG_INC_PTR 1 + add tmp1q, %1*mmsize + add tmp2q, %1*mmsize +%endmacro + +cglobal avg, 4, 7, 3, dst, stride, tmp1, tmp2, w, h, stride3 +%define base r6-avg_avx512icl_table + lea r6, [avg_avx512icl_table] + tzcnt wd, wm + movifnidn hd, hm + movsxd wq, dword [r6+wq*4] + vpbroadcastd m2, [base+pw_1024] + add wq, r6 + BIDIR_FN AVG + +%macro W_AVG 1 ; src_offset + ; (a * weight + b * (16 - weight) + 128) >> 8 + ; = ((a - b) * weight + (b << 4) + 128) >> 8 + ; = ((((a - b) * ((weight-16) << 12)) >> 16) + a + 8) >> 4 + ; = ((((b - a) * (-weight << 12)) >> 16) + b + 8) >> 4 + mova m0, [tmp1q+(%1+0)*mmsize] + psubw m2, m0, [tmp2q+(%1+0)*mmsize] + mova m1, [tmp1q+(%1+1)*mmsize] + psubw m3, m1, [tmp2q+(%1+1)*mmsize] + pmulhw m2, m4 + pmulhw m3, m4 + paddw m0, m2 + paddw m1, m3 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + packuswb m0, m1 +%endmacro + +%define W_AVG_INC_PTR AVG_INC_PTR + +cglobal w_avg, 4, 7, 6, dst, stride, tmp1, tmp2, w, h, stride3 +%define base r6-w_avg_avx512icl_table + lea r6, [w_avg_avx512icl_table] + tzcnt wd, wm + movifnidn hd, hm + vpbroadcastw m4, r6m ; weight + movsxd wq, dword [r6+wq*4] + vpbroadcastd m5, [base+pw_2048] + psllw m4, 12 ; (weight-16) << 12 when interpreted as signed + add wq, r6 + cmp dword r6m, 7 + jg .weight_gt7 + mov r6, tmp1q + pxor m0, m0 + mov tmp1q, tmp2q + psubw m4, m0, m4 ; -weight + mov tmp2q, r6 +.weight_gt7: + BIDIR_FN W_AVG + +%macro MASK 1 ; src_offset + ; (a * m + b * (64 - m) + 512) >> 10 + ; = ((a - b) * m + (b << 6) + 512) >> 10 + ; = ((((b - a) * (-m << 10)) >> 16) + b + 8) >> 4 +%if mmsize == 64 + vpermq m3, m8, [maskq+%1*32] +%else + vpermq m3, [maskq+%1*16], q3120 +%endif + mova m0, [tmp2q+(%1+0)*mmsize] + psubw m1, m0, [tmp1q+(%1+0)*mmsize] + psubb m3, m4, m3 + paddw m1, m1 ; (b - a) << 1 + paddb m3, m3 + punpcklbw m2, m4, m3 ; -m << 9 + pmulhw m1, m2 + paddw m0, m1 + mova m1, [tmp2q+(%1+1)*mmsize] + psubw m2, m1, [tmp1q+(%1+1)*mmsize] + paddw m2, m2 + punpckhbw m3, m4, m3 + pmulhw m2, m3 + paddw m1, m2 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + packuswb m0, m1 +%endmacro + +%macro MASK_INC_PTR 1 + add maskq, %1*32 + add tmp2q, %1*64 + add tmp1q, %1*64 +%endmacro + +cglobal mask, 4, 8, 6, dst, stride, tmp1, tmp2, w, h, mask, stride3 +%define base r7-mask_avx512icl_table + lea r7, [mask_avx512icl_table] + tzcnt wd, wm + movifnidn hd, hm + mov maskq, maskmp + movsxd wq, dword [r7+wq*4] + pxor m4, m4 + mova m8, [base+bilin_v_perm64] + vpbroadcastd m5, [base+pw_2048] + add wq, r7 + BIDIR_FN MASK + +%macro W_MASK 4-5 0 ; dst, mask, tmp_offset[1-2], 4:4:4 + mova m%1, [tmp1q+mmsize*%3] + mova m1, [tmp2q+mmsize*%3] + psubw m1, m%1 + pabsw m%2, m1 + psubusw m%2, m6, m%2 + psrlw m%2, 8 ; 64 - m + psllw m2, m%2, 10 + pmulhw m1, m2 + paddw m%1, m1 + mova m1, [tmp1q+mmsize*%4] + mova m2, [tmp2q+mmsize*%4] + psubw m2, m1 + pabsw m3, m2 + psubusw m3, m6, m3 + vpshldw m%2, m3, 8 + psllw m3, m%2, 10 +%if %5 + psubb m%2, m5, m%2 +%endif + pmulhw m2, m3 + paddw m1, m2 + pmulhrsw m%1, m7 + pmulhrsw m1, m7 + packuswb m%1, m1 +%endmacro + +cglobal w_mask_420, 4, 8, 16, dst, stride, tmp1, tmp2, w, h, mask, stride3 +%define base r7-w_mask_420_avx512icl_table + lea r7, [w_mask_420_avx512icl_table] + tzcnt wd, wm + mov r6d, r7m ; sign + movifnidn hd, hm + movsxd wq, [r7+wq*4] + vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 + vpbroadcastd m7, [base+pw_2048] + vpbroadcastd m9, [base+pb_m64] ; -1 << 6 + mova ym10, [base+wm_420_mask+32] + vpbroadcastd m8, [base+wm_sign+r6*8] ; (258 - sign) << 6 + add wq, r7 + mov maskq, maskmp + lea stride3q, [strideq*3] + jmp wq +.w4: + mova m5, [wm_420_perm4] + cmp hd, 8 + jg .w4_h16 + WRAP_YMM W_MASK 0, 4, 0, 1 + vinserti128 ym5, [wm_420_perm4+32], 1 + vpermb ym4, ym5, ym4 + vpdpbusd ym8, ym4, ym9 + vextracti128 xmm1, m0, 1 + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xmm1 + pextrd [dstq+stride3q ], xmm1, 1 + jl .w4_end + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xmm1, 2 + pextrd [dstq+stride3q ], xmm1, 3 +.w4_end: + vpermb ym8, ym10, ym8 + movq [maskq], xm8 + RET +.w4_h16: + vpbroadcastd m11, strided + pmulld m11, [bidir_sctr_w4] + W_MASK 0, 4, 0, 1 + vpermb m4, m5, m4 + vpdpbusd m8, m4, m9 + kxnorw k1, k1, k1 + vpermb m8, m10, m8 + mova [maskq], xm8 + vpscatterdd [dstq+m11]{k1}, m0 + RET +.w8: + mova m5, [wm_420_perm8] + cmp hd, 4 + jne .w8_h8 + WRAP_YMM W_MASK 0, 4, 0, 1 + vinserti128 ym5, [wm_420_perm8+32], 1 + vpermb ym4, ym5, ym4 + vpdpbusd ym8, ym4, ym9 + vpermb m8, m10, m8 + mova [maskq], xm8 + vextracti128 xmm1, ym0, 1 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xmm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xmm1 + RET +.w8_loop: + add tmp1q, 128 + add tmp2q, 128 + add maskq, 16 + lea dstq, [dstq+strideq*4] +.w8_h8: + W_MASK 0, 4, 0, 1 + vpermb m4, m5, m4 + mova m1, m8 + vpdpbusd m1, m4, m9 + vpermb m1, m10, m1 + mova [maskq], xm1 + vextracti32x4 xmm1, ym0, 1 + vextracti32x4 xmm2, m0, 2 + vextracti32x4 xmm3, m0, 3 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xmm1 + movq [dstq+strideq*2], xmm2 + movq [dstq+stride3q ], xmm3 + lea dstq, [dstq+strideq*4] + movhps [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xmm1 + movhps [dstq+strideq*2], xmm2 + movhps [dstq+stride3q ], xmm3 + sub hd, 8 + jg .w8_loop + RET +.w16: + mova m5, [wm_420_perm16] +.w16_loop: + W_MASK 0, 4, 0, 1 + vpermb m4, m5, m4 + mova m1, m8 + vpdpbusd m1, m4, m9 + add tmp1q, 128 + add tmp2q, 128 + vpermb m1, m10, m1 + vpermq m0, m0, q3120 + mova [maskq], xm1 + add maskq, 16 + mova [dstq+strideq*0], xm0 + vextracti32x4 [dstq+strideq*1], m0, 2 + vextracti32x4 [dstq+strideq*2], ym0, 1 + vextracti32x4 [dstq+stride3q ], m0, 3 + lea dstq, [dstq+strideq*4] + sub hd, 4 + jg .w16_loop + RET +.w32: + pmovzxbq m5, [pb_02461357] +.w32_loop: + W_MASK 0, 4, 0, 1 + mova m1, m8 + vpdpbusd m1, m4, m9 + add tmp1q, 128 + add tmp2q, 128 + vpermb m1, m10, m1 + vpermq m0, m5, m0 + mova [maskq], xm1 + add maskq, 16 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w32_loop + RET +.w64: + pmovzxbq m12, [wm_420_perm64] ; 0, 2, 4, 6, 8, 10, 12, 14 + psrlq m13, m12, 4 ; 1, 3, 5, 7, 9, 11, 13, 15 +.w64_loop: + W_MASK 0, 4, 0, 2 + W_MASK 11, 5, 1, 3 + mova m2, m8 + vpdpbusd m2, m4, m9 + mova m3, m8 + vpdpbusd m3, m5, m9 + add tmp1q, 256 + add tmp2q, 256 + vpermt2b m2, m10, m3 + mova m1, m0 + vpermt2q m0, m12, m11 + vpermt2q m1, m13, m11 + mova [maskq], ym2 + add maskq, 32 + mova [dstq+strideq*0], m0 + mova [dstq+strideq*1], m1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w64_loop + RET +.w128: + pmovzxbq m14, [wm_420_perm64] + mova m10, [wm_420_mask] + psrlq m15, m14, 4 +.w128_loop: + W_MASK 0, 12, 0, 4 + W_MASK 11, 13, 1, 5 + mova m4, m8 + vpdpbusd m4, m12, m9 + mova m5, m8 + vpdpbusd m5, m13, m9 + mova m1, m0 + vpermt2q m0, m14, m11 + vpermt2q m1, m15, m11 + mova [dstq+strideq*0+64*0], m0 + mova [dstq+strideq*1+64*0], m1 + W_MASK 0, 12, 2, 6 + W_MASK 11, 13, 3, 7 + vprold m4, 16 + vprold m5, 16 + vpdpbusd m4, m12, m9 + vpdpbusd m5, m13, m9 + add tmp1q, 512 + add tmp2q, 512 + vpermt2b m4, m10, m5 + mova m1, m0 + vpermt2q m0, m14, m11 + vpermt2q m1, m15, m11 + mova [maskq], m4 + add maskq, 64 + mova [dstq+strideq*0+64*1], m0 + mova [dstq+strideq*1+64*1], m1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w128_loop + RET + +cglobal w_mask_422, 4, 8, 14, dst, stride, tmp1, tmp2, w, h, mask, stride3 +%define base r7-w_mask_422_avx512icl_table + lea r7, [w_mask_422_avx512icl_table] + tzcnt wd, wm + mov r6d, r7m ; sign + movifnidn hd, hm + movsxd wq, dword [r7+wq*4] + vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 + vpbroadcastd m7, [base+pw_2048] + vpbroadcastd m9, [base+pw_m128] + mova m10, [base+wm_422_mask] + vpbroadcastd m11, [base+pb_127] + add wq, r7 + vpbroadcastd m8, [base+wm_sign+4+r6*4] + mov maskq, maskmp + lea stride3q, [strideq*3] + jmp wq +.w4: + cmp hd, 8 + jg .w4_h16 + WRAP_YMM W_MASK 0, 4, 0, 1 + movhps xm10, [wm_422_mask+16] + vpdpwssd ym8, ym4, ym9 + vpermb ym8, ym10, ym8 + vextracti128 xmm1, m0, 1 + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xmm1 + pextrd [dstq+stride3q ], xmm1, 1 + jl .w4_end + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xmm1, 2 + pextrd [dstq+stride3q ], xmm1, 3 +.w4_end: + pand xm8, xm11 + mova [maskq], xm8 + RET +.w4_h16: + vpbroadcastd m5, strided + pmulld m5, [bidir_sctr_w4] + W_MASK 0, 4, 0, 1 + vpdpwssd m8, m4, m9 + kxnorw k1, k1, k1 + vpermb m8, m10, m8 + pand ym8, ym11 + mova [maskq], ym8 + vpscatterdd [dstq+m5]{k1}, m0 + RET +.w8: + cmp hd, 4 + jne .w8_h8 + WRAP_YMM W_MASK 0, 4, 0, 1 + movhps xm10, [wm_422_mask+16] + vpdpwssd ym8, ym4, ym9 + vpermb ym8, ym10, ym8 + pand xm8, xm11 + mova [maskq], xm8 + vextracti128 xmm1, ym0, 1 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xmm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xmm1 + RET +.w8_loop: + add tmp1q, 128 + add tmp2q, 128 + add maskq, 32 + lea dstq, [dstq+strideq*4] +.w8_h8: + W_MASK 0, 4, 0, 1 + mova m1, m8 + vpdpwssd m1, m4, m9 + vpermb m1, m10, m1 + pand ym1, ym11 + mova [maskq], ym1 + vextracti32x4 xmm1, ym0, 1 + vextracti32x4 xmm2, m0, 2 + vextracti32x4 xmm3, m0, 3 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xmm1 + movq [dstq+strideq*2], xmm2 + movq [dstq+stride3q ], xmm3 + lea dstq, [dstq+strideq*4] + movhps [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xmm1 + movhps [dstq+strideq*2], xmm2 + movhps [dstq+stride3q ], xmm3 + sub hd, 8 + jg .w8_loop + RET +.w16_loop: + add tmp1q, 128 + add tmp2q, 128 + add maskq, 32 + lea dstq, [dstq+strideq*4] +.w16: + W_MASK 0, 4, 0, 1 + mova m1, m8 + vpdpwssd m1, m4, m9 + vpermb m1, m10, m1 + vpermq m0, m0, q3120 + pand ym1, ym11 + mova [maskq], ym1 + mova [dstq+strideq*0], xm0 + vextracti32x4 [dstq+strideq*1], m0, 2 + vextracti32x4 [dstq+strideq*2], ym0, 1 + vextracti32x4 [dstq+stride3q ], m0, 3 + sub hd, 4 + jg .w16_loop + RET +.w32: + pmovzxbq m5, [pb_02461357] +.w32_loop: + W_MASK 0, 4, 0, 1 + mova m1, m8 + vpdpwssd m1, m4, m9 + add tmp1q, 128 + add tmp2q, 128 + vpermb m1, m10, m1 + vpermq m0, m5, m0 + pand ym1, ym11 + mova [maskq], ym1 + add maskq, 32 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w32_loop + RET +.w64: + pmovzxbq m5, [pb_02461357] +.w64_loop: + W_MASK 0, 4, 0, 1 + mova m1, m8 + vpdpwssd m1, m4, m9 + add tmp1q, 128 + add tmp2q, 128 + vpermb m1, m10, m1 + vpermq m0, m5, m0 + pand ym1, ym11 + mova [maskq], ym1 + add maskq, 32 + mova [dstq], m0 + add dstq, strideq + dec hd + jg .w64_loop + RET +.w128: + pmovzxbq m13, [pb_02461357] +.w128_loop: + W_MASK 0, 4, 0, 1 + W_MASK 12, 5, 2, 3 + mova m2, m8 + vpdpwssd m2, m4, m9 + mova m3, m8 + vpdpwssd m3, m5, m9 + add tmp1q, 256 + add tmp2q, 256 + vpermt2b m2, m10, m3 + vpermq m0, m13, m0 + vpermq m1, m13, m12 + pand m2, m11 + mova [maskq], m2 + add maskq, 64 + mova [dstq+64*0], m0 + mova [dstq+64*1], m1 + add dstq, strideq + dec hd + jg .w128_loop + RET + +cglobal w_mask_444, 4, 8, 12, dst, stride, tmp1, tmp2, w, h, mask, stride3 +%define base r7-w_mask_444_avx512icl_table + lea r7, [w_mask_444_avx512icl_table] + tzcnt wd, wm + movifnidn hd, hm + movsxd wq, dword [r7+wq*4] + vpbroadcastd m6, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 + vpbroadcastd m5, [base+pb_64] + vpbroadcastd m7, [base+pw_2048] + mova m8, [base+wm_444_mask] + add wq, r7 + mov maskq, maskmp + lea stride3q, [strideq*3] + jmp wq +.w4: + cmp hd, 8 + jg .w4_h16 + WRAP_YMM W_MASK 0, 4, 0, 1, 1 + vinserti128 ym8, [wm_444_mask+32], 1 + vpermb ym4, ym8, ym4 + mova [maskq], ym4 + vextracti128 xmm1, m0, 1 + movd [dstq+strideq*0], xm0 + pextrd [dstq+strideq*1], xm0, 1 + movd [dstq+strideq*2], xmm1 + pextrd [dstq+stride3q ], xmm1, 1 + jl .w4_end + lea dstq, [dstq+strideq*4] + pextrd [dstq+strideq*0], xm0, 2 + pextrd [dstq+strideq*1], xm0, 3 + pextrd [dstq+strideq*2], xmm1, 2 + pextrd [dstq+stride3q ], xmm1, 3 +.w4_end: + RET +.w4_h16: + vpbroadcastd m9, strided + pmulld m9, [bidir_sctr_w4] + W_MASK 0, 4, 0, 1, 1 + vpermb m4, m8, m4 + kxnorw k1, k1, k1 + mova [maskq], m4 + vpscatterdd [dstq+m9]{k1}, m0 + RET +.w8: + cmp hd, 4 + jne .w8_h8 + WRAP_YMM W_MASK 0, 4, 0, 1, 1 + vinserti128 ym8, [wm_444_mask+32], 1 + vpermb ym4, ym8, ym4 + mova [maskq], ym4 + vextracti128 xmm1, ym0, 1 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xmm1 + movhps [dstq+strideq*2], xm0 + movhps [dstq+stride3q ], xmm1 + RET +.w8_loop: + add tmp1q, 128 + add tmp2q, 128 + add maskq, 64 + lea dstq, [dstq+strideq*4] +.w8_h8: + W_MASK 0, 4, 0, 1, 1 + vpermb m4, m8, m4 + mova [maskq], m4 + vextracti32x4 xmm1, ym0, 1 + vextracti32x4 xmm2, m0, 2 + vextracti32x4 xmm3, m0, 3 + movq [dstq+strideq*0], xm0 + movq [dstq+strideq*1], xmm1 + movq [dstq+strideq*2], xmm2 + movq [dstq+stride3q ], xmm3 + lea dstq, [dstq+strideq*4] + movhps [dstq+strideq*0], xm0 + movhps [dstq+strideq*1], xmm1 + movhps [dstq+strideq*2], xmm2 + movhps [dstq+stride3q ], xmm3 + sub hd, 8 + jg .w8_loop + RET +.w16_loop: + add tmp1q, 128 + add tmp2q, 128 + add maskq, 64 + lea dstq, [dstq+strideq*4] +.w16: + W_MASK 0, 4, 0, 1, 1 + vpermb m4, m8, m4 + vpermq m0, m0, q3120 + mova [maskq], m4 + mova [dstq+strideq*0], xm0 + vextracti32x4 [dstq+strideq*1], m0, 2 + vextracti32x4 [dstq+strideq*2], ym0, 1 + vextracti32x4 [dstq+stride3q ], m0, 3 + sub hd, 4 + jg .w16_loop + RET +.w32: + pmovzxbq m9, [pb_02461357] +.w32_loop: + W_MASK 0, 4, 0, 1, 1 + vpermb m4, m8, m4 + add tmp1q, 128 + add tmp2q, 128 + vpermq m0, m9, m0 + mova [maskq], m4 + add maskq, 64 + mova [dstq+strideq*0], ym0 + vextracti32x8 [dstq+strideq*1], m0, 1 + lea dstq, [dstq+strideq*2] + sub hd, 2 + jg .w32_loop + RET +.w64: + pmovzxbq m9, [pb_02461357] +.w64_loop: + W_MASK 0, 4, 0, 1, 1 + vpermb m4, m8, m4 + add tmp1q, 128 + add tmp2q, 128 + vpermq m0, m9, m0 + mova [maskq], m4 + add maskq, 64 + mova [dstq], m0 + add dstq, strideq + dec hd + jg .w64_loop + RET +.w128: + pmovzxbq m11, [pb_02461357] +.w128_loop: + W_MASK 0, 4, 0, 1, 1 + W_MASK 10, 9, 2, 3, 1 + vpermb m4, m8, m4 + vpermb m9, m8, m9 + add tmp1q, 256 + add tmp2q, 256 + vpermq m0, m11, m0 + vpermq m10, m11, m10 + mova [maskq+64*0], m4 + mova [maskq+64*1], m9 + add maskq, 128 + mova [dstq+64*0], m0 + mova [dstq+64*1], m10 + add dstq, strideq + dec hd + jg .w128_loop + RET + +%endif ; HAVE_AVX512ICL && ARCH_X86_64 diff --git a/third_party/dav1d/src/x86/mc_init_tmpl.c b/third_party/dav1d/src/x86/mc_init_tmpl.c --- a/third_party/dav1d/src/x86/mc_init_tmpl.c +++ b/third_party/dav1d/src/x86/mc_init_tmpl.c @@ -50,37 +50,69 @@ decl_mc_fn(dav1d_put_bilin_ssse3); decl_mct_fn(dav1d_prep_8tap_regular_avx512icl); decl_mct_fn(dav1d_prep_8tap_regular_avx2); decl_mct_fn(dav1d_prep_8tap_regular_ssse3); +decl_mct_fn(dav1d_prep_8tap_regular_sse2); decl_mct_fn(dav1d_prep_8tap_regular_smooth_avx512icl); decl_mct_fn(dav1d_prep_8tap_regular_smooth_avx2); decl_mct_fn(dav1d_prep_8tap_regular_smooth_ssse3); +decl_mct_fn(dav1d_prep_8tap_regular_smooth_sse2); decl_mct_fn(dav1d_prep_8tap_regular_sharp_avx512icl); decl_mct_fn(dav1d_prep_8tap_regular_sharp_avx2); decl_mct_fn(dav1d_prep_8tap_regular_sharp_ssse3); +decl_mct_fn(dav1d_prep_8tap_regular_sharp_sse2); decl_mct_fn(dav1d_prep_8tap_smooth_avx512icl); decl_mct_fn(dav1d_prep_8tap_smooth_avx2); decl_mct_fn(dav1d_prep_8tap_smooth_ssse3); +decl_mct_fn(dav1d_prep_8tap_smooth_sse2); decl_mct_fn(dav1d_prep_8tap_smooth_regular_avx512icl); decl_mct_fn(dav1d_prep_8tap_smooth_regular_avx2); decl_mct_fn(dav1d_prep_8tap_smooth_regular_ssse3); +decl_mct_fn(dav1d_prep_8tap_smooth_regular_sse2); decl_mct_fn(dav1d_prep_8tap_smooth_sharp_avx512icl); decl_mct_fn(dav1d_prep_8tap_smooth_sharp_avx2); decl_mct_fn(dav1d_prep_8tap_smooth_sharp_ssse3); +decl_mct_fn(dav1d_prep_8tap_smooth_sharp_sse2); decl_mct_fn(dav1d_prep_8tap_sharp_avx512icl); decl_mct_fn(dav1d_prep_8tap_sharp_avx2); decl_mct_fn(dav1d_prep_8tap_sharp_ssse3); +decl_mct_fn(dav1d_prep_8tap_sharp_sse2); decl_mct_fn(dav1d_prep_8tap_sharp_regular_avx512icl); decl_mct_fn(dav1d_prep_8tap_sharp_regular_avx2); decl_mct_fn(dav1d_prep_8tap_sharp_regular_ssse3); +decl_mct_fn(dav1d_prep_8tap_sharp_regular_sse2); decl_mct_fn(dav1d_prep_8tap_sharp_smooth_avx512icl); decl_mct_fn(dav1d_prep_8tap_sharp_smooth_avx2); decl_mct_fn(dav1d_prep_8tap_sharp_smooth_ssse3); +decl_mct_fn(dav1d_prep_8tap_sharp_smooth_sse2); decl_mct_fn(dav1d_prep_bilin_avx512icl); decl_mct_fn(dav1d_prep_bilin_avx2); decl_mct_fn(dav1d_prep_bilin_ssse3); +decl_mct_fn(dav1d_prep_bilin_sse2); + +decl_mc_scaled_fn(dav1d_put_8tap_scaled_regular_avx2); +decl_mc_scaled_fn(dav1d_put_8tap_scaled_regular_smooth_avx2); +decl_mc_scaled_fn(dav1d_put_8tap_scaled_regular_sharp_avx2); +decl_mc_scaled_fn(dav1d_put_8tap_scaled_smooth_avx2); +decl_mc_scaled_fn(dav1d_put_8tap_scaled_smooth_regular_avx2); +decl_mc_scaled_fn(dav1d_put_8tap_scaled_smooth_sharp_avx2); +decl_mc_scaled_fn(dav1d_put_8tap_scaled_sharp_avx2); +decl_mc_scaled_fn(dav1d_put_8tap_scaled_sharp_regular_avx2); +decl_mc_scaled_fn(dav1d_put_8tap_scaled_sharp_smooth_avx2); +decl_mc_scaled_fn(dav1d_put_bilin_scaled_avx2); + +decl_mct_scaled_fn(dav1d_prep_8tap_scaled_regular_avx2); +decl_mct_scaled_fn(dav1d_prep_8tap_scaled_regular_smooth_avx2); +decl_mct_scaled_fn(dav1d_prep_8tap_scaled_regular_sharp_avx2); +decl_mct_scaled_fn(dav1d_prep_8tap_scaled_smooth_avx2); +decl_mct_scaled_fn(dav1d_prep_8tap_scaled_smooth_regular_avx2); +decl_mct_scaled_fn(dav1d_prep_8tap_scaled_smooth_sharp_avx2); +decl_mct_scaled_fn(dav1d_prep_8tap_scaled_sharp_avx2); +decl_mct_scaled_fn(dav1d_prep_8tap_scaled_sharp_regular_avx2); +decl_mct_scaled_fn(dav1d_prep_8tap_scaled_sharp_smooth_avx2); +decl_mct_scaled_fn(dav1d_prep_bilin_scaled_avx2); decl_avg_fn(dav1d_avg_avx512icl); decl_avg_fn(dav1d_avg_avx2); decl_avg_fn(dav1d_avg_ssse3); decl_w_avg_fn(dav1d_w_avg_avx512icl); @@ -113,39 +145,58 @@ decl_warp8x8t_fn(dav1d_warp_affine_8x8t_sse2); decl_emu_edge_fn(dav1d_emu_edge_avx2); decl_emu_edge_fn(dav1d_emu_edge_ssse3); +decl_resize_fn(dav1d_resize_avx2); +decl_resize_fn(dav1d_resize_ssse3); + COLD void bitfn(dav1d_mc_dsp_init_x86)(Dav1dMCDSPContext *const c) { #define init_mc_fn(type, name, suffix) \ c->mc[type] = dav1d_put_##name##_##suffix #define init_mct_fn(type, name, suffix) \ c->mct[type] = dav1d_prep_##name##_##suffix +#define init_mc_scaled_fn(type, name, suffix) \ + c->mc_scaled[type] = dav1d_put_##name##_##suffix +#define init_mct_scaled_fn(type, name, suffix) \ + c->mct_scaled[type] = dav1d_prep_##name##_##suffix + const unsigned flags = dav1d_get_cpu_flags(); if(!(flags & DAV1D_X86_CPU_FLAG_SSE2)) return; #if BITDEPTH == 8 + init_mct_fn(FILTER_2D_BILINEAR, bilin, sse2); + init_mct_fn(FILTER_2D_8TAP_REGULAR, 8tap_regular, sse2); + init_mct_fn(FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_regular_smooth, sse2); + init_mct_fn(FILTER_2D_8TAP_REGULAR_SHARP, 8tap_regular_sharp, sse2); + init_mct_fn(FILTER_2D_8TAP_SMOOTH_REGULAR, 8tap_smooth_regular, sse2); + init_mct_fn(FILTER_2D_8TAP_SMOOTH, 8tap_smooth, sse2); + init_mct_fn(FILTER_2D_8TAP_SMOOTH_SHARP, 8tap_smooth_sharp, sse2); + init_mct_fn(FILTER_2D_8TAP_SHARP_REGULAR, 8tap_sharp_regular, sse2); + init_mct_fn(FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_sharp_smooth, sse2); + init_mct_fn(FILTER_2D_8TAP_SHARP, 8tap_sharp, sse2); + c->warp8x8 = dav1d_warp_affine_8x8_sse2; c->warp8x8t = dav1d_warp_affine_8x8t_sse2; #endif if(!(flags & DAV1D_X86_CPU_FLAG_SSSE3)) return; #if BITDEPTH == 8 - init_mc_fn (FILTER_2D_BILINEAR, bilin, ssse3); - init_mc_fn (FILTER_2D_8TAP_REGULAR, 8tap_regular, ssse3); - init_mc_fn (FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_regular_smooth, ssse3); - init_mc_fn (FILTER_2D_8TAP_REGULAR_SHARP, 8tap_regular_sharp, ssse3); - init_mc_fn (FILTER_2D_8TAP_SMOOTH_REGULAR, 8tap_smooth_regular, ssse3); - init_mc_fn (FILTER_2D_8TAP_SMOOTH, 8tap_smooth, ssse3); - init_mc_fn (FILTER_2D_8TAP_SMOOTH_SHARP, 8tap_smooth_sharp, ssse3); - init_mc_fn (FILTER_2D_8TAP_SHARP_REGULAR, 8tap_sharp_regular, ssse3); - init_mc_fn (FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_sharp_smooth, ssse3); - init_mc_fn (FILTER_2D_8TAP_SHARP, 8tap_sharp, ssse3); + init_mc_fn(FILTER_2D_BILINEAR, bilin, ssse3); + init_mc_fn(FILTER_2D_8TAP_REGULAR, 8tap_regular, ssse3); + init_mc_fn(FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_regular_smooth, ssse3); + init_mc_fn(FILTER_2D_8TAP_REGULAR_SHARP, 8tap_regular_sharp, ssse3); + init_mc_fn(FILTER_2D_8TAP_SMOOTH_REGULAR, 8tap_smooth_regular, ssse3); + init_mc_fn(FILTER_2D_8TAP_SMOOTH, 8tap_smooth, ssse3); + init_mc_fn(FILTER_2D_8TAP_SMOOTH_SHARP, 8tap_smooth_sharp, ssse3); + init_mc_fn(FILTER_2D_8TAP_SHARP_REGULAR, 8tap_sharp_regular, ssse3); + init_mc_fn(FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_sharp_smooth, ssse3); + init_mc_fn(FILTER_2D_8TAP_SHARP, 8tap_sharp, ssse3); init_mct_fn(FILTER_2D_BILINEAR, bilin, ssse3); init_mct_fn(FILTER_2D_8TAP_REGULAR, 8tap_regular, ssse3); init_mct_fn(FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_regular_smooth, ssse3); init_mct_fn(FILTER_2D_8TAP_REGULAR_SHARP, 8tap_regular_sharp, ssse3); @@ -166,10 +217,11 @@ c->warp8x8 = dav1d_warp_affine_8x8_ssse3; c->warp8x8t = dav1d_warp_affine_8x8t_ssse3; c->emu_edge = dav1d_emu_edge_ssse3; + c->resize = dav1d_resize_ssse3; #endif if(!(flags & DAV1D_X86_CPU_FLAG_SSE41)) return; @@ -181,20 +233,20 @@ #if ARCH_X86_64 if (!(flags & DAV1D_X86_CPU_FLAG_AVX2)) return; #if BITDEPTH == 8 - init_mc_fn (FILTER_2D_8TAP_REGULAR, 8tap_regular, avx2); - init_mc_fn (FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_regular_smooth, avx2); - init_mc_fn (FILTER_2D_8TAP_REGULAR_SHARP, 8tap_regular_sharp, avx2); - init_mc_fn (FILTER_2D_8TAP_SMOOTH_REGULAR, 8tap_smooth_regular, avx2); - init_mc_fn (FILTER_2D_8TAP_SMOOTH, 8tap_smooth, avx2); - init_mc_fn (FILTER_2D_8TAP_SMOOTH_SHARP, 8tap_smooth_sharp, avx2); - init_mc_fn (FILTER_2D_8TAP_SHARP_REGULAR, 8tap_sharp_regular, avx2); - init_mc_fn (FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_sharp_smooth, avx2); - init_mc_fn (FILTER_2D_8TAP_SHARP, 8tap_sharp, avx2); - init_mc_fn (FILTER_2D_BILINEAR, bilin, avx2); + init_mc_fn(FILTER_2D_8TAP_REGULAR, 8tap_regular, avx2); + init_mc_fn(FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_regular_smooth, avx2); + init_mc_fn(FILTER_2D_8TAP_REGULAR_SHARP, 8tap_regular_sharp, avx2); + init_mc_fn(FILTER_2D_8TAP_SMOOTH_REGULAR, 8tap_smooth_regular, avx2); + init_mc_fn(FILTER_2D_8TAP_SMOOTH, 8tap_smooth, avx2); + init_mc_fn(FILTER_2D_8TAP_SMOOTH_SHARP, 8tap_smooth_sharp, avx2); + init_mc_fn(FILTER_2D_8TAP_SHARP_REGULAR, 8tap_sharp_regular, avx2); + init_mc_fn(FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_sharp_smooth, avx2); + init_mc_fn(FILTER_2D_8TAP_SHARP, 8tap_sharp, avx2); + init_mc_fn(FILTER_2D_BILINEAR, bilin, avx2); init_mct_fn(FILTER_2D_8TAP_REGULAR, 8tap_regular, avx2); init_mct_fn(FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_regular_smooth, avx2); init_mct_fn(FILTER_2D_8TAP_REGULAR_SHARP, 8tap_regular_sharp, avx2); init_mct_fn(FILTER_2D_8TAP_SMOOTH_REGULAR, 8tap_smooth_regular, avx2); @@ -203,10 +255,32 @@ init_mct_fn(FILTER_2D_8TAP_SHARP_REGULAR, 8tap_sharp_regular, avx2); init_mct_fn(FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_sharp_smooth, avx2); init_mct_fn(FILTER_2D_8TAP_SHARP, 8tap_sharp, avx2); init_mct_fn(FILTER_2D_BILINEAR, bilin, avx2); + init_mc_scaled_fn(FILTER_2D_8TAP_REGULAR, 8tap_scaled_regular, avx2); + init_mc_scaled_fn(FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_scaled_regular_smooth, avx2); + init_mc_scaled_fn(FILTER_2D_8TAP_REGULAR_SHARP, 8tap_scaled_regular_sharp, avx2); + init_mc_scaled_fn(FILTER_2D_8TAP_SMOOTH_REGULAR, 8tap_scaled_smooth_regular, avx2); + init_mc_scaled_fn(FILTER_2D_8TAP_SMOOTH, 8tap_scaled_smooth, avx2); + init_mc_scaled_fn(FILTER_2D_8TAP_SMOOTH_SHARP, 8tap_scaled_smooth_sharp, avx2); + init_mc_scaled_fn(FILTER_2D_8TAP_SHARP_REGULAR, 8tap_scaled_sharp_regular, avx2); + init_mc_scaled_fn(FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_scaled_sharp_smooth, avx2); + init_mc_scaled_fn(FILTER_2D_8TAP_SHARP, 8tap_scaled_sharp, avx2); + init_mc_scaled_fn(FILTER_2D_BILINEAR, bilin_scaled, avx2); + + init_mct_scaled_fn(FILTER_2D_8TAP_REGULAR, 8tap_scaled_regular, avx2); + init_mct_scaled_fn(FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_scaled_regular_smooth, avx2); + init_mct_scaled_fn(FILTER_2D_8TAP_REGULAR_SHARP, 8tap_scaled_regular_sharp, avx2); + init_mct_scaled_fn(FILTER_2D_8TAP_SMOOTH_REGULAR, 8tap_scaled_smooth_regular, avx2); + init_mct_scaled_fn(FILTER_2D_8TAP_SMOOTH, 8tap_scaled_smooth, avx2); + init_mct_scaled_fn(FILTER_2D_8TAP_SMOOTH_SHARP, 8tap_scaled_smooth_sharp, avx2); + init_mct_scaled_fn(FILTER_2D_8TAP_SHARP_REGULAR, 8tap_scaled_sharp_regular, avx2); + init_mct_scaled_fn(FILTER_2D_8TAP_SHARP_SMOOTH, 8tap_scaled_sharp_smooth, avx2); + init_mct_scaled_fn(FILTER_2D_8TAP_SHARP, 8tap_scaled_sharp, avx2); + init_mct_scaled_fn(FILTER_2D_BILINEAR, bilin_scaled, avx2); + c->avg = dav1d_avg_avx2; c->w_avg = dav1d_w_avg_avx2; c->mask = dav1d_mask_avx2; c->w_mask[0] = dav1d_w_mask_444_avx2; c->w_mask[1] = dav1d_w_mask_422_avx2; @@ -217,16 +291,17 @@ c->warp8x8 = dav1d_warp_affine_8x8_avx2; c->warp8x8t = dav1d_warp_affine_8x8t_avx2; c->emu_edge = dav1d_emu_edge_avx2; + c->resize = dav1d_resize_avx2; #endif if (!(flags & DAV1D_X86_CPU_FLAG_AVX512ICL)) return; -#if BITDEPTH == 8 +#if HAVE_AVX512ICL && BITDEPTH == 8 init_mct_fn(FILTER_2D_8TAP_REGULAR, 8tap_regular, avx512icl); init_mct_fn(FILTER_2D_8TAP_REGULAR_SMOOTH, 8tap_regular_smooth, avx512icl); init_mct_fn(FILTER_2D_8TAP_REGULAR_SHARP, 8tap_regular_sharp, avx512icl); init_mct_fn(FILTER_2D_8TAP_SMOOTH_REGULAR, 8tap_smooth_regular, avx512icl); init_mct_fn(FILTER_2D_8TAP_SMOOTH, 8tap_smooth, avx512icl); diff --git a/third_party/dav1d/src/x86/mc_sse.asm b/third_party/dav1d/src/x86/mc_sse.asm new file mode 100644 --- /dev/null +++ b/third_party/dav1d/src/x86/mc_sse.asm @@ -0,0 +1,5892 @@ +; Copyright © 2018, VideoLAN and dav1d authors +; Copyright © 2018, Two Orioles, LLC +; Copyright © 2018, VideoLabs +; All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions are met: +; +; 1. Redistributions of source code must retain the above copyright notice, this +; list of conditions and the following disclaimer. +; +; 2. Redistributions in binary form must reproduce the above copyright notice, +; this list of conditions and the following disclaimer in the documentation +; and/or other materials provided with the distribution. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +%include "ext/x86/x86inc.asm" + +SECTION_RODATA 16 + +; dav1d_obmc_masks[] with 64-x interleaved +obmc_masks: db 0, 0, 0, 0 + ; 2 @4 + db 45, 19, 64, 0 + ; 4 @8 + db 39, 25, 50, 14, 59, 5, 64, 0 + ; 8 @16 + db 36, 28, 42, 22, 48, 16, 53, 11, 57, 7, 61, 3, 64, 0, 64, 0 + ; 16 @32 + db 34, 30, 37, 27, 40, 24, 43, 21, 46, 18, 49, 15, 52, 12, 54, 10 + db 56, 8, 58, 6, 60, 4, 61, 3, 64, 0, 64, 0, 64, 0, 64, 0 + ; 32 @64 + db 33, 31, 35, 29, 36, 28, 38, 26, 40, 24, 41, 23, 43, 21, 44, 20 + db 45, 19, 47, 17, 48, 16, 50, 14, 51, 13, 52, 12, 53, 11, 55, 9 + db 56, 8, 57, 7, 58, 6, 59, 5, 60, 4, 60, 4, 61, 3, 62, 2 + +warp_8x8_shufA: db 0, 2, 4, 6, 1, 3, 5, 7, 1, 3, 5, 7, 2, 4, 6, 8 +warp_8x8_shufB: db 4, 6, 8, 10, 5, 7, 9, 11, 5, 7, 9, 11, 6, 8, 10, 12 +warp_8x8_shufC: db 2, 4, 6, 8, 3, 5, 7, 9, 3, 5, 7, 9, 4, 6, 8, 10 +warp_8x8_shufD: db 6, 8, 10, 12, 7, 9, 11, 13, 7, 9, 11, 13, 8, 10, 12, 14 +blend_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3 +subpel_h_shuf4: db 0, 1, 2, 3, 1, 2, 3, 4, 8, 9, 10, 11, 9, 10, 11, 12 + db 2, 3, 4, 5, 3, 4, 5, 6, 10, 11, 12, 13, 11, 12, 13, 14 +subpel_h_shufA: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 +subpel_h_shufB: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 +subpel_h_shufC: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 +bilin_h_shuf4: db 1, 0, 2, 1, 3, 2, 4, 3, 9, 8, 10, 9, 11, 10, 12, 11 +bilin_h_shuf8: db 1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7 + +pb_8x0_8x8: times 8 db 0 + times 8 db 8 +resize_mul: dd 0, 1, 2, 3 +resize_shuf: times 5 db 0 + db 1, 2, 3, 4, 5, 6 + times 5+16 db 7 + +pb_64: times 16 db 64 +pw_m256: times 8 dw -256 +pw_1: times 8 dw 1 +pw_2: times 8 dw 2 +pw_8: times 8 dw 8 +pw_26: times 8 dw 26 +pw_34: times 8 dw 34 +pw_512: times 8 dw 512 +pw_1024: times 8 dw 1024 +pw_2048: times 8 dw 2048 +pw_6903: times 8 dw 6903 +pw_8192: times 8 dw 8192 +pd_32: times 4 dd 32 +pd_63: times 4 dd 63 +pd_512: times 4 dd 512 +pd_16384: times 4 dd 16484 +pd_32768: times 4 dd 32768 +pd_262144:times 4 dd 262144 + +pw_258: times 2 dw 258 + +cextern mc_subpel_filters +%define subpel_filters (mangle(private_prefix %+ _mc_subpel_filters)-8) + +%macro BIDIR_JMP_TABLE 1-* + ;evaluated at definition time (in loop below) + %xdefine %1_table (%%table - 2*%2) + %xdefine %%base %1_table + %xdefine %%prefix mangle(private_prefix %+ _%1) + ; dynamically generated label + %%table: + %rep %0 - 1 ; repeat for num args + dd %%prefix %+ .w%2 - %%base + %rotate 1 + %endrep +%endmacro + +BIDIR_JMP_TABLE avg_ssse3, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_avg_ssse3, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE mask_ssse3, 4, 8, 16, 32, 64, 128 +BIDIR_JMP_TABLE w_mask_420_ssse3, 4, 8, 16, 16, 16, 16 +BIDIR_JMP_TABLE blend_ssse3, 4, 8, 16, 32 +BIDIR_JMP_TABLE blend_v_ssse3, 2, 4, 8, 16, 32 +BIDIR_JMP_TABLE blend_h_ssse3, 2, 4, 8, 16, 16, 16, 16 + +%macro BASE_JMP_TABLE 3-* + %xdefine %1_%2_table (%%table - %3) + %xdefine %%base %1_%2 + %%table: + %rep %0 - 2 + dw %%base %+ _w%3 - %%base + %rotate 1 + %endrep +%endmacro + +%xdefine prep_sse2 mangle(private_prefix %+ _prep_bilin_sse2.prep) +%xdefine put_ssse3 mangle(private_prefix %+ _put_bilin_ssse3.put) +%xdefine prep_ssse3 mangle(private_prefix %+ _prep_bilin_ssse3.prep) + +BASE_JMP_TABLE put, ssse3, 2, 4, 8, 16, 32, 64, 128 +BASE_JMP_TABLE prep, ssse3, 4, 8, 16, 32, 64, 128 + +%macro HV_JMP_TABLE 5-* + %xdefine %%prefix mangle(private_prefix %+ _%1_%2_%3) + %xdefine %%base %1_%3 + %assign %%types %4 + %if %%types & 1 + %xdefine %1_%2_h_%3_table (%%h - %5) + %%h: + %rep %0 - 4 + dw %%prefix %+ .h_w%5 - %%base + %rotate 1 + %endrep + %rotate 4 + %endif + %if %%types & 2 + %xdefine %1_%2_v_%3_table (%%v - %5) + %%v: + %rep %0 - 4 + dw %%prefix %+ .v_w%5 - %%base + %rotate 1 + %endrep + %rotate 4 + %endif + %if %%types & 4 + %xdefine %1_%2_hv_%3_table (%%hv - %5) + %%hv: + %rep %0 - 4 + dw %%prefix %+ .hv_w%5 - %%base + %rotate 1 + %endrep + %endif +%endmacro + +HV_JMP_TABLE prep, 8tap, sse2, 1, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE prep, bilin, sse2, 7, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE put, 8tap, ssse3, 3, 2, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE prep, 8tap, ssse3, 1, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE put, bilin, ssse3, 7, 2, 4, 8, 16, 32, 64, 128 +HV_JMP_TABLE prep, bilin, ssse3, 7, 4, 8, 16, 32, 64, 128 + +%define table_offset(type, fn) type %+ fn %+ SUFFIX %+ _table - type %+ SUFFIX + +cextern mc_warp_filter + +SECTION .text + +INIT_XMM ssse3 + +%if ARCH_X86_32 + DECLARE_REG_TMP 1 + %define base t0-put_ssse3 +%else + DECLARE_REG_TMP 7 + %define base 0 +%endif +; +%macro RESTORE_DSQ_32 1 + %if ARCH_X86_32 + mov %1, dsm ; restore dsq + %endif +%endmacro +; +cglobal put_bilin, 4, 8, 0, dst, ds, src, ss, w, h, mxy, bak + movifnidn mxyd, r6m ; mx + LEA t0, put_ssse3 + tzcnt wd, wm + mov hd, hm + test mxyd, mxyd + jnz .h + mov mxyd, r7m ; my + test mxyd, mxyd + jnz .v +.put: + movzx wd, word [t0+wq*2+table_offset(put,)] + add wq, t0 + RESTORE_DSQ_32 t0 + jmp wq +.put_w2: + movzx r4d, word [srcq+ssq*0] + movzx r6d, word [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mov [dstq+dsq*0], r4w + mov [dstq+dsq*1], r6w + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w2 + RET +.put_w4: + mov r4d, [srcq+ssq*0] + mov r6d, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mov [dstq+dsq*0], r4d + mov [dstq+dsq*1], r6d + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w4 + RET +.put_w8: + movq m0, [srcq+ssq*0] + movq m1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + movq [dstq+dsq*0], m0 + movq [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w8 + RET +.put_w16: + movu m0, [srcq+ssq*0] + movu m1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + mova [dstq+dsq*0], m0 + mova [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w16 + RET +.put_w32: + movu m0, [srcq+ssq*0+16*0] + movu m1, [srcq+ssq*0+16*1] + movu m2, [srcq+ssq*1+16*0] + movu m3, [srcq+ssq*1+16*1] + lea srcq, [srcq+ssq*2] + mova [dstq+dsq*0+16*0], m0 + mova [dstq+dsq*0+16*1], m1 + mova [dstq+dsq*1+16*0], m2 + mova [dstq+dsq*1+16*1], m3 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .put_w32 + RET +.put_w64: + movu m0, [srcq+16*0] + movu m1, [srcq+16*1] + movu m2, [srcq+16*2] + movu m3, [srcq+16*3] + add srcq, ssq + mova [dstq+16*0], m0 + mova [dstq+16*1], m1 + mova [dstq+16*2], m2 + mova [dstq+16*3], m3 + add dstq, dsq + dec hd + jg .put_w64 + RET +.put_w128: + movu m0, [srcq+16*0] + movu m1, [srcq+16*1] + movu m2, [srcq+16*2] + movu m3, [srcq+16*3] + mova [dstq+16*0], m0 + mova [dstq+16*1], m1 + mova [dstq+16*2], m2 + mova [dstq+16*3], m3 + movu m0, [srcq+16*4] + movu m1, [srcq+16*5] + movu m2, [srcq+16*6] + movu m3, [srcq+16*7] + mova [dstq+16*4], m0 + mova [dstq+16*5], m1 + mova [dstq+16*6], m2 + mova [dstq+16*7], m3 + add srcq, ssq + add dstq, dsq + dec hd + jg .put_w128 + RET +.h: + ; (16 * src[x] + (mx * (src[x + 1] - src[x])) + 8) >> 4 + ; = ((16 - mx) * src[x] + mx * src[x + 1] + 8) >> 4 + imul mxyd, 0xff01 + mova m4, [base+bilin_h_shuf8] + mova m0, [base+bilin_h_shuf4] + add mxyd, 16 << 8 + movd m5, mxyd + mov mxyd, r7m ; my + pshuflw m5, m5, q0000 + punpcklqdq m5, m5 + test mxyd, mxyd + jnz .hv + movzx wd, word [t0+wq*2+table_offset(put, _bilin_h)] + mova m3, [base+pw_2048] + add wq, t0 + RESTORE_DSQ_32 t0 + jmp wq +.h_w2: + pshufd m4, m4, q3120 ; m4 = {1, 0, 2, 1, 5, 4, 6, 5} +.h_w2_loop: + movd m0, [srcq+ssq*0] + movd m1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + punpckldq m0, m1 + pshufb m0, m4 + pmaddubsw m0, m5 + pmulhrsw m0, m3 + packuswb m0, m0 + movd r6d, m0 + mov [dstq+dsq*0], r6w + shr r6d, 16 + mov [dstq+dsq*1], r6w + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w2_loop + RET +.h_w4: + movq m4, [srcq+ssq*0] + movhps m4, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pshufb m4, m0 + pmaddubsw m4, m5 + pmulhrsw m4, m3 + packuswb m4, m4 + movd [dstq+dsq*0], m4 + psrlq m4, 32 + movd [dstq+dsq*1], m4 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w4 + RET +.h_w8: + movu m0, [srcq+ssq*0] + movu m1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pshufb m0, m4 + pshufb m1, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmulhrsw m0, m3 + pmulhrsw m1, m3 + packuswb m0, m1 + movq [dstq+dsq*0], m0 + movhps [dstq+dsq*1], m0 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w8 + RET +.h_w16: + movu m0, [srcq+8*0] + movu m1, [srcq+8*1] + add srcq, ssq + pshufb m0, m4 + pshufb m1, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmulhrsw m0, m3 + pmulhrsw m1, m3 + packuswb m0, m1 + mova [dstq], m0 + add dstq, dsq + dec hd + jg .h_w16 + RET +.h_w32: + movu m0, [srcq+mmsize*0+8*0] + movu m1, [srcq+mmsize*0+8*1] + pshufb m0, m4 + pshufb m1, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmulhrsw m0, m3 + pmulhrsw m1, m3 + packuswb m0, m1 + movu m1, [srcq+mmsize*1+8*0] + movu m2, [srcq+mmsize*1+8*1] + add srcq, ssq + pshufb m1, m4 + pshufb m2, m4 + pmaddubsw m1, m5 + pmaddubsw m2, m5 + pmulhrsw m1, m3 + pmulhrsw m2, m3 + packuswb m1, m2 + mova [dstq+16*0], m0 + mova [dstq+16*1], m1 + add dstq, dsq + dec hd + jg .h_w32 + RET +.h_w64: + mov r6, -16*3 +.h_w64_loop: + movu m0, [srcq+r6+16*3+8*0] + movu m1, [srcq+r6+16*3+8*1] + pshufb m0, m4 + pshufb m1, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmulhrsw m0, m3 + pmulhrsw m1, m3 + packuswb m0, m1 + mova [dstq+r6+16*3], m0 + add r6, 16 + jle .h_w64_loop + add srcq, ssq + add dstq, dsq + dec hd + jg .h_w64 + RET +.h_w128: + mov r6, -16*7 +.h_w128_loop: + movu m0, [srcq+r6+16*7+8*0] + movu m1, [srcq+r6+16*7+8*1] + pshufb m0, m4 + pshufb m1, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 + pmulhrsw m0, m3 + pmulhrsw m1, m3 + packuswb m0, m1 + mova [dstq+r6+16*7], m0 + add r6, 16 + jle .h_w128_loop + add srcq, ssq + add dstq, dsq + dec hd + jg .h_w128 + RET +.v: + movzx wd, word [t0+wq*2+table_offset(put, _bilin_v)] + imul mxyd, 0xff01 + mova m5, [base+pw_2048] + add mxyd, 16 << 8 + add wq, t0 + movd m4, mxyd + pshuflw m4, m4, q0000 + punpcklqdq m4, m4 + RESTORE_DSQ_32 t0 + jmp wq +.v_w2: + movd m0, [srcq+ssq*0] +.v_w2_loop: + pinsrw m0, [srcq+ssq*1], 1 ; 0 1 + lea srcq, [srcq+ssq*2] + pshuflw m2, m0, q2301 + pinsrw m0, [srcq+ssq*0], 0 ; 2 1 + punpcklbw m1, m0, m2 + pmaddubsw m1, m4 + pmulhrsw m1, m5 + packuswb m1, m1 + movd r6d, m1 + mov [dstq+dsq*1], r6w + shr r6d, 16 + mov [dstq+dsq*0], r6w + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w2_loop + RET +.v_w4: + movd m0, [srcq+ssq*0] +.v_w4_loop: + movd m1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + punpckldq m2, m0, m1 ; 0 1 + movd m0, [srcq+ssq*0] + punpckldq m1, m0 ; 1 2 + punpcklbw m1, m2 + pmaddubsw m1, m4 + pmulhrsw m1, m5 + packuswb m1, m1 + movd [dstq+dsq*0], m1 + psrlq m1, 32 + movd [dstq+dsq*1], m1 + ; + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w4_loop + RET +.v_w8: + movq m0, [srcq+ssq*0] +.v_w8_loop: + movq m3, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + punpcklbw m1, m3, m0 + movq m0, [srcq+ssq*0] + punpcklbw m2, m0, m3 + pmaddubsw m1, m4 + pmaddubsw m2, m4 + pmulhrsw m1, m5 + pmulhrsw m2, m5 + packuswb m1, m2 + movq [dstq+dsq*0], m1 + movhps [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w8_loop + RET + ; +%macro PUT_BILIN_V_W16 0 + movu m0, [srcq+ssq*0] +%%loop: + movu m3, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + punpcklbw m1, m3, m0 + punpckhbw m2, m3, m0 + movu m0, [srcq+ssq*0] + pmaddubsw m1, m4 + pmaddubsw m2, m4 + pmulhrsw m1, m5 + pmulhrsw m2, m5 + packuswb m1, m2 + mova [dstq+dsq*0], m1 + punpcklbw m1, m0, m3 + punpckhbw m2, m0, m3 + pmaddubsw m1, m4 + pmaddubsw m2, m4 + pmulhrsw m1, m5 + pmulhrsw m2, m5 + packuswb m1, m2 + mova [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg %%loop +%endmacro + ; +.v_w16: + PUT_BILIN_V_W16 + RET +.v_w16gt: + mov r4, dstq + mov r6, srcq +.v_w16gt_loop: +%if ARCH_X86_32 + mov bakm, t0q + RESTORE_DSQ_32 t0 + PUT_BILIN_V_W16 + mov t0q, bakm +%else + PUT_BILIN_V_W16 +%endif + mov hw, t0w + add r4, mmsize + add r6, mmsize + mov dstq, r4 + mov srcq, r6 + sub t0d, 1<<16 + jg .v_w16gt + RET +.v_w32: + lea t0d, [hq+(1<<16)] + jmp .v_w16gt +.v_w64: + lea t0d, [hq+(3<<16)] + jmp .v_w16gt +.v_w128: + lea t0d, [hq+(7<<16)] + jmp .v_w16gt +.hv: + ; (16 * src[x] + (my * (src[x + src_stride] - src[x])) + 128) >> 8 + ; = (src[x] + ((my * (src[x + src_stride] - src[x])) >> 4) + 8) >> 4 + movzx wd, word [t0+wq*2+table_offset(put, _bilin_hv)] + WIN64_SPILL_XMM 8 + shl mxyd, 11 ; can't shift by 12 due to signed overflow + mova m7, [base+pw_2048] + movd m6, mxyd + add wq, t0 + pshuflw m6, m6, q0000 + punpcklqdq m6, m6 + jmp wq +.hv_w2: + RESTORE_DSQ_32 t0 + movd m0, [srcq+ssq*0] + pshufd m0, m0, q0000 ; src[x - src_stride] + pshufb m0, m4 + pmaddubsw m0, m5 +.hv_w2_loop: + movd m1, [srcq+ssq*1] ; src[x] + lea srcq, [srcq+ssq*2] + movhps m1, [srcq+ssq*0] ; src[x + src_stride] + pshufd m1, m1, q3120 + pshufb m1, m4 + pmaddubsw m1, m5 ; 1 _ 2 _ + shufps m2, m0, m1, q1032 ; 0 _ 1 _ + mova m0, m1 + psubw m1, m2 ; src[x + src_stride] - src[x] + paddw m1, m1 + pmulhw m1, m6 ; (my * (src[x + src_stride] - src[x]) + paddw m1, m2 ; src[x] + (my * (src[x + src_stride] - src[x]) + pmulhrsw m1, m7 + packuswb m1, m1 +%if ARCH_X86_64 + movq r6, m1 +%else + pshuflw m1, m1, q2020 + movd r6d, m1 +%endif + mov [dstq+dsq*0], r6w + shr r6, gprsize*4 + mov [dstq+dsq*1], r6w + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w2_loop + RET +.hv_w4: + mova m4, [base+bilin_h_shuf4] + RESTORE_DSQ_32 t0 + movddup xm0, [srcq+ssq*0] + pshufb m0, m4 + pmaddubsw m0, m5 +.hv_w4_loop: + movq m1, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + movhps m1, [srcq+ssq*0] + pshufb m1, m4 + pmaddubsw m1, m5 ; 1 2 + shufps m2, m0, m1, q1032 ; 0 1 + mova m0, m1 + psubw m1, m2 + paddw m1, m1 + pmulhw m1, m6 + paddw m1, m2 + pmulhrsw m1, m7 + packuswb m1, m1 + movd [dstq+dsq*0], m1 + psrlq m1, 32 + movd [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w4_loop + RET +.hv_w8: + RESTORE_DSQ_32 t0 + movu m0, [srcq+ssq*0+8*0] + pshufb m0, m4 + pmaddubsw m0, m5 +.hv_w8_loop: + movu m2, [srcq+ssq*1+8*0] + lea srcq, [srcq+ssq*2] + pshufb m2, m4 + pmaddubsw m2, m5 + psubw m1, m2, m0 + paddw m1, m1 + pmulhw m1, m6 + paddw m1, m0 + movu m0, [srcq+ssq*0+8*0] + pshufb m0, m4 + pmaddubsw m0, m5 + psubw m3, m0, m2 + paddw m3, m3 + pmulhw m3, m6 + paddw m3, m2 + pmulhrsw m1, m7 + pmulhrsw m3, m7 + packuswb m1, m3 + movq [dstq+dsq*0], m1 + movhps [dstq+dsq*1], m1 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w8_loop + RET +.hv_w16: + xor t0d, t0d +.hv_w16gt: + mov r4, dstq + mov r6, srcq + %if WIN64 + movaps r4m, xmm8 + %endif +.hv_w16_loop0: + movu m0, [srcq+8*0] + movu m1, [srcq+8*1] + pshufb m0, m4 + pshufb m1, m4 + pmaddubsw m0, m5 + pmaddubsw m1, m5 +.hv_w16_loop: +%if ARCH_X86_32 + %define m0tmp [dstq] +%else + %define m0tmp m8 +%endif + add srcq, ssq + movu m2, [srcq+8*0] + movu m3, [srcq+8*1] + pshufb m2, m4 + pshufb m3, m4 + pmaddubsw m2, m5 + pmaddubsw m3, m5 + mova m0tmp, m2 + psubw m2, m0 + paddw m2, m2 + pmulhw m2, m6 + paddw m2, m0 + mova m0, m3 + psubw m3, m1 + paddw m3, m3 + pmulhw m3, m6 + paddw m3, m1 + mova m1, m0 + mova m0, m0tmp + pmulhrsw m2, m7 + pmulhrsw m3, m7 + packuswb m2, m3 + mova [dstq], m2 + add dstq, dsmp + dec hd + jg .hv_w16_loop + movzx hd, t0w + add r4, mmsize + add r6, mmsize + mov dstq, r4 + mov srcq, r6 + sub t0d, 1<<16 + jg .hv_w16_loop0 + %if WIN64 + movaps xmm8, r4m + %endif + RET +.hv_w32: + lea t0d, [hq+(1<<16)] + jmp .hv_w16gt +.hv_w64: + lea t0d, [hq+(3<<16)] + jmp .hv_w16gt +.hv_w128: + lea t0d, [hq+(7<<16)] + jmp .hv_w16gt + +%macro PSHUFB_0X1X 1-2 ; dst[, src] + %if cpuflag(ssse3) + pshufb %1, %2 + %else + punpcklbw %1, %1 + psraw %1, 8 + pshufd %1, %1, q0000 + %endif +%endmacro + +%macro PSHUFB_BILIN_H8 2 ; dst, src + %if cpuflag(ssse3) + pshufb %1, %2 + %else + mova %2, %1 + psrldq %1, 1 + punpcklbw %1, %2 + %endif +%endmacro + +%macro PSHUFB_BILIN_H4 3 ; dst, src, tmp + %if cpuflag(ssse3) + pshufb %1, %2 + %else + mova %2, %1 + psrldq %1, 1 + punpckhbw %3, %1, %2 + punpcklbw %1, %2 + punpcklqdq %1, %3 + %endif +%endmacro + +%macro PMADDUBSW 5 ; dst/src1, src2, zero, tmp, reset_zero + %if cpuflag(ssse3) + pmaddubsw %1, %2 + %else + %if %5 == 1 + pxor %3, %3 + %endif + punpckhbw %4, %1, %3 + punpcklbw %1, %1, %3 + pmaddwd %4, %2 + pmaddwd %1, %2 + packssdw %1, %4 + %endif +%endmacro + +%macro PMULHRSW 5 ; dst, src, tmp, rndval, shift + %if cpuflag(ssse3) + pmulhrsw %1, %2 + %else + punpckhwd %3, %1, %4 + punpcklwd %1, %4 + pmaddwd %3, %2 + pmaddwd %1, %2 + psrad %3, %5 + psrad %1, %5 + packssdw %1, %3 + %endif +%endmacro + +%macro PREP_BILIN 0 + +DECLARE_REG_TMP 3, 5, 6 +%if ARCH_X86_32 + %define base t2-prep%+SUFFIX +%else + %define base 0 +%endif + +cglobal prep_bilin, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3 + movifnidn mxyd, r5m ; mx + LEA t2, prep%+SUFFIX + tzcnt wd, wm + movifnidn hd, hm + test mxyd, mxyd + jnz .h + mov mxyd, r6m ; my + test mxyd, mxyd + jnz .v +.prep: +%if notcpuflag(ssse3) + add t2, prep_ssse3 - prep_sse2 + jmp prep_ssse3 +%else + movzx wd, word [t2+wq*2+table_offset(prep,)] + add wq, t2 + lea stride3q, [strideq*3] + jmp wq +.prep_w4: + movd m0, [srcq+strideq*0] + movd m1, [srcq+strideq*1] + movd m2, [srcq+strideq*2] + movd m3, [srcq+stride3q ] + punpckldq m0, m1 + punpckldq m2, m3 + lea srcq, [srcq+strideq*4] + pxor m1, m1 + punpcklbw m0, m1 + punpcklbw m2, m1 + psllw m0, 4 + psllw m2, 4 + mova [tmpq+mmsize*0], m0 + mova [tmpq+mmsize*1], m2 + add tmpq, 32 + sub hd, 4 + jg .prep_w4 + RET +.prep_w8: + movq m0, [srcq+strideq*0] + movq m1, [srcq+strideq*1] + movq m2, [srcq+strideq*2] + movq m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + pxor m4, m4 + punpcklbw m0, m4 + punpcklbw m1, m4 + punpcklbw m2, m4 + punpcklbw m3, m4 + psllw m0, 4 + psllw m1, 4 + psllw m2, 4 + psllw m3, 4 + mova [tmpq+16*0], m0 + mova [tmpq+16*1], m1 + mova [tmpq+16*2], m2 + mova [tmpq+16*3], m3 + add tmpq, 16*4 + sub hd, 4 + jg .prep_w8 + RET +.prep_w16: + movq m0, [srcq+strideq*0+8*0] + movq m1, [srcq+strideq*0+8*1] + movq m2, [srcq+strideq*1+8*0] + movq m3, [srcq+strideq*1+8*1] + lea srcq, [srcq+strideq*2] + pxor m4, m4 + punpcklbw m0, m4 + punpcklbw m1, m4 + punpcklbw m2, m4 + punpcklbw m3, m4 + psllw m0, 4 + psllw m1, 4 + psllw m2, 4 + psllw m3, 4 + mova [tmpq+16*0], m0 + mova [tmpq+16*1], m1 + mova [tmpq+16*2], m2 + mova [tmpq+16*3], m3 + add tmpq, 16*4 + sub hd, 2 + jg .prep_w16 + RET +.prep_w32: + mov t2d, 1 + jmp .prep_w32_vloop +.prep_w64: + mov t2d, 2 + jmp .prep_w32_vloop +.prep_w128: + mov t2d, 4 +.prep_w32_vloop: + mov t1q, srcq + mov r3d, t2d +.prep_w32_hloop: + movq m0, [t1q+8*0] + movq m1, [t1q+8*1] + movq m2, [t1q+8*2] + movq m3, [t1q+8*3] + pxor m4, m4 + punpcklbw m0, m4 + punpcklbw m1, m4 + punpcklbw m2, m4 + punpcklbw m3, m4 + psllw m0, 4 + psllw m1, 4 + psllw m2, 4 + psllw m3, 4 + mova [tmpq+16*0], m0 + mova [tmpq+16*1], m1 + mova [tmpq+16*2], m2 + mova [tmpq+16*3], m3 + add tmpq, 16*4 + add t1q, 32 + dec r3d + jg .prep_w32_hloop + lea srcq, [srcq+strideq] + dec hd + jg .prep_w32_vloop + RET +%endif +.h: + ; 16 * src[x] + (mx * (src[x + 1] - src[x])) + ; = (16 - mx) * src[x] + mx * src[x + 1] + imul mxyd, 0xff01 +%if cpuflag(ssse3) + mova m4, [base+bilin_h_shuf8] +%endif + add mxyd, 16 << 8 + movd m5, mxyd + mov mxyd, r6m ; my +%if cpuflag(ssse3) + pshuflw m5, m5, q0000 + punpcklqdq m5, m5 +%else + PSHUFB_0X1X m5 +%endif + test mxyd, mxyd + jnz .hv +%if ARCH_X86_32 + mov t1, t2 ; save base reg for w4 +%endif + movzx wd, word [t2+wq*2+table_offset(prep, _bilin_h)] +%if notcpuflag(ssse3) + WIN64_SPILL_XMM 8 + pxor m6, m6 +%endif + add wq, t2 + lea stride3q, [strideq*3] + jmp wq +.h_w4: +%if cpuflag(ssse3) + %if ARCH_X86_32 + mova m4, [t1-prep_ssse3+bilin_h_shuf4] + %else + mova m4, [bilin_h_shuf4] + %endif +%endif +.h_w4_loop: + movq m0, [srcq+strideq*0] + movhps m0, [srcq+strideq*1] + movq m1, [srcq+strideq*2] + movhps m1, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + PSHUFB_BILIN_H4 m0, m4, m2 + PMADDUBSW m0, m5, m6, m2, 0 + PSHUFB_BILIN_H4 m1, m4, m2 + PMADDUBSW m1, m5, m6, m2, 0 + mova [tmpq+0 ], m0 + mova [tmpq+16], m1 + add tmpq, 32 + sub hd, 4 + jg .h_w4_loop + RET +.h_w8: + movu m0, [srcq+strideq*0] + movu m1, [srcq+strideq*1] + movu m2, [srcq+strideq*2] + movu m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + PSHUFB_BILIN_H8 m0, m4 + PSHUFB_BILIN_H8 m1, m4 + PSHUFB_BILIN_H8 m2, m4 + PSHUFB_BILIN_H8 m3, m4 + PMADDUBSW m0, m5, m6, m7, 0 + PMADDUBSW m1, m5, m6, m7, 0 + PMADDUBSW m2, m5, m6, m7, 0 + PMADDUBSW m3, m5, m6, m7, 0 + mova [tmpq+16*0], m0 + mova [tmpq+16*1], m1 + mova [tmpq+16*2], m2 + mova [tmpq+16*3], m3 + add tmpq, 16*4 + sub hd, 4 + jg .h_w8 + RET +.h_w16: + movu m0, [srcq+strideq*0+8*0] + movu m1, [srcq+strideq*0+8*1] + movu m2, [srcq+strideq*1+8*0] + movu m3, [srcq+strideq*1+8*1] + lea srcq, [srcq+strideq*2] + PSHUFB_BILIN_H8 m0, m4 + PSHUFB_BILIN_H8 m1, m4 + PSHUFB_BILIN_H8 m2, m4 + PSHUFB_BILIN_H8 m3, m4 + PMADDUBSW m0, m5, m6, m7, 0 + PMADDUBSW m1, m5, m6, m7, 0 + PMADDUBSW m2, m5, m6, m7, 0 + PMADDUBSW m3, m5, m6, m7, 0 + mova [tmpq+16*0], m0 + mova [tmpq+16*1], m1 + mova [tmpq+16*2], m2 + mova [tmpq+16*3], m3 + add tmpq, 16*4 + sub hd, 2 + jg .h_w16 + RET +.h_w32: + mov t2d, 1 << 0 + jmp .h_w32_vloop +.h_w64: + mov t2d, 1 << 1 + jmp .h_w32_vloop +.h_w128: + mov t2d, 1 << 3 +.h_w32_vloop: + mov t1q, srcq + mov r3d, t2d +.h_w32_hloop: + movu m0, [t1q+8*0] + movu m1, [t1q+8*1] + movu m2, [t1q+8*2] + movu m3, [t1q+8*3] + PSHUFB_BILIN_H8 m0, m4 + PSHUFB_BILIN_H8 m1, m4 + PSHUFB_BILIN_H8 m2, m4 + PSHUFB_BILIN_H8 m3, m4 + PMADDUBSW m0, m5, m6, m7, 0 + PMADDUBSW m1, m5, m6, m7, 0 + PMADDUBSW m2, m5, m6, m7, 0 + PMADDUBSW m3, m5, m6, m7, 0 + mova [tmpq+16*0], m0 + mova [tmpq+16*1], m1 + mova [tmpq+16*2], m2 + mova [tmpq+16*3], m3 + add tmpq, 16*4 + add t1q, 32 + shr r3d, 1 + jnz .h_w32_hloop + lea srcq, [srcq+strideq] + sub hd, 1 + jg .h_w32_vloop + RET +.v: +%if notcpuflag(ssse3) + %assign stack_offset stack_offset - stack_size_padded + WIN64_SPILL_XMM 8 +%endif + movzx wd, word [t2+wq*2+table_offset(prep, _bilin_v)] + imul mxyd, 0xff01 + add mxyd, 16 << 8 + add wq, t2 + lea stride3q, [strideq*3] + movd m5, mxyd +%if cpuflag(ssse3) + pshuflw m5, m5, q0000 + punpcklqdq m5, m5 +%else + PSHUFB_0X1X m5 + pxor m6, m6 +%endif + jmp wq +.v_w4: + movd m0, [srcq+strideq*0] +.v_w4_loop: + movd m1, [srcq+strideq*1] + movd m2, [srcq+strideq*2] + movd m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + punpcklwd m0, m1 ; 0 1 _ _ + punpcklwd m1, m2 ; 1 2 _ _ + punpcklbw m1, m0 + PMADDUBSW m1, m5, m6, m7, 0 + pshufd m1, m1, q3120 + mova [tmpq+16*0], m1 + movd m0, [srcq+strideq*0] + punpcklwd m2, m3 ; 2 3 _ _ + punpcklwd m3, m0 ; 3 4 _ _ + punpcklbw m3, m2 + PMADDUBSW m3, m5, m6, m7, 0 + pshufd m3, m3, q3120 + mova [tmpq+16*1], m3 + add tmpq, 32 + sub hd, 4 + jg .v_w4_loop + RET +.v_w8: + movq m0, [srcq+strideq*0] +.v_w8_loop: + movq m1, [srcq+strideq*2] + movq m2, [srcq+strideq*1] + movq m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + shufpd m4, m0, m1, 0x0c ; 0 2 + movq m0, [srcq+strideq*0] + shufpd m2, m3, 0x0c ; 1 3 + shufpd m1, m0, 0x0c ; 2 4 + punpcklbw m3, m2, m4 + PMADDUBSW m3, m5, m6, m7, 0 + mova [tmpq+16*0], m3 + punpckhbw m3, m2, m4 + PMADDUBSW m3, m5, m6, m7, 0 + mova [tmpq+16*2], m3 + punpcklbw m3, m1, m2 + punpckhbw m1, m2 + PMADDUBSW m3, m5, m6, m7, 0 + PMADDUBSW m1, m5, m6, m7, 0 + mova [tmpq+16*1], m3 + mova [tmpq+16*3], m1 + add tmpq, 16*4 + sub hd, 4 + jg .v_w8_loop + RET +.v_w16: + movu m0, [srcq+strideq*0] +.v_w16_loop: + movu m1, [srcq+strideq*1] + movu m2, [srcq+strideq*2] + punpcklbw m3, m1, m0 + punpckhbw m4, m1, m0 + PMADDUBSW m3, m5, m6, m7, 0 + PMADDUBSW m4, m5, m6, m7, 0 + mova [tmpq+16*0], m3 + mova [tmpq+16*1], m4 + punpcklbw m3, m2, m1 + punpckhbw m4, m2, m1 + PMADDUBSW m3, m5, m6, m7, 0 + PMADDUBSW m4, m5, m6, m7, 0 + mova [tmpq+16*2], m3 + mova [tmpq+16*3], m4 + movu m3, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + movu m0, [srcq+strideq*0] + add tmpq, 16*8 + punpcklbw m1, m3, m2 + punpckhbw m4, m3, m2 + PMADDUBSW m1, m5, m6, m7, 0 + PMADDUBSW m4, m5, m6, m7, 0 + mova [tmpq-16*4], m1 + mova [tmpq-16*3], m4 + punpcklbw m1, m0, m3 + punpckhbw m2, m0, m3 + PMADDUBSW m1, m5, m6, m7, 0 + PMADDUBSW m2, m5, m6, m7, 0 + mova [tmpq-16*2], m1 + mova [tmpq-16*1], m2 + sub hd, 4 + jg .v_w16_loop + RET +.v_w32: + lea t2d, [hq+(0<<16)] + mov t0d, 64 + jmp .v_w32_start +.v_w64: + lea t2d, [hq+(1<<16)] + mov t0d, 128 + jmp .v_w32_start +.v_w128: + lea t2d, [hq+(3<<16)] + mov t0d, 256 +.v_w32_start: +%if ARCH_X86_64 + %if WIN64 + PUSH r7 + %endif + mov r7, tmpq +%endif + mov t1, srcq +.v_w32_hloop: + movu m0, [srcq+strideq*0+16*0] + movu m1, [srcq+strideq*0+16*1] +.v_w32_vloop: + movu m2, [srcq+strideq*1+16*0] + movu m3, [srcq+strideq*1+16*1] + lea srcq, [srcq+strideq*2] + punpcklbw m4, m2, m0 + PMADDUBSW m4, m5, m6, m7, 0 + mova [tmpq+16*0], m4 + punpckhbw m4, m2, m0 + PMADDUBSW m4, m5, m6, m7, 0 + mova [tmpq+16*1], m4 + punpcklbw m4, m3, m1 + PMADDUBSW m4, m5, m6, m7, 0 + mova [tmpq+16*2], m4 + punpckhbw m4, m3, m1 + PMADDUBSW m4, m5, m6, m7, 0 + mova [tmpq+16*3], m4 + add tmpq, t0q + movu m0, [srcq+strideq*0+16*0] + movu m1, [srcq+strideq*0+16*1] + punpcklbw m4, m0, m2 + PMADDUBSW m4, m5, m6, m7, 0 + mova [tmpq+16*0], m4 + punpckhbw m4, m0, m2 + PMADDUBSW m4, m5, m6, m7, 0 + mova [tmpq+16*1], m4 + punpcklbw m4, m1, m3 + PMADDUBSW m4, m5, m6, m7, 0 + mova [tmpq+16*2], m4 + punpckhbw m4, m1, m3 + PMADDUBSW m4, m5, m6, m7, 0 + mova [tmpq+16*3], m4 + add tmpq, t0q + sub hd, 2 + jg .v_w32_vloop + movzx hd, t2w + add t1, 32 + mov srcq, t1 +%if ARCH_X86_64 + add r7, 2*16*2 + mov tmpq, r7 +%else + mov tmpq, tmpmp + add tmpq, 2*16*2 + mov tmpmp, tmpq +%endif + sub t2d, 1<<16 + jg .v_w32_hloop +%if WIN64 + POP r7 +%endif + RET +.hv: + ; (16 * src[x] + (my * (src[x + src_stride] - src[x])) + 8) >> 4 + ; = src[x] + (((my * (src[x + src_stride] - src[x])) + 8) >> 4) +%assign stack_offset stack_offset - stack_size_padded +%if cpuflag(ssse3) + WIN64_SPILL_XMM 8 +%else + WIN64_SPILL_XMM 10 +%endif + movzx wd, word [t2+wq*2+table_offset(prep, _bilin_hv)] +%if cpuflag(ssse3) + shl mxyd, 11 +%else + %if ARCH_X86_64 + mova m8, [pw_8] + %else + %define m8 [t1-prep_sse2+pw_8] + %endif + pxor m7, m7 +%endif + movd m6, mxyd + add wq, t2 + pshuflw m6, m6, q0000 +%if cpuflag(ssse3) + punpcklqdq m6, m6 +%elif ARCH_X86_64 + psrlw m0, m8, 3 + punpcklwd m6, m0 +%else + punpcklwd m6, [base+pw_1] +%endif +%if ARCH_X86_32 + mov t1, t2 ; save base reg for w4 +%endif + lea stride3q, [strideq*3] + jmp wq +.hv_w4: +%if cpuflag(ssse3) + %if ARCH_X86_32 + mova m4, [t1-prep_ssse3+bilin_h_shuf4] + %else + mova m4, [bilin_h_shuf4] + %endif +%endif + movhps m0, [srcq+strideq*0] + PSHUFB_BILIN_H4 m0, m4, m3 + PMADDUBSW m0, m5, m7, m4, 0 ; _ 0 +.hv_w4_loop: + movq m1, [srcq+strideq*1] + movhps m1, [srcq+strideq*2] + movq m2, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + movhps m2, [srcq+strideq*0] + PSHUFB_BILIN_H4 m1, m4, m3 + PSHUFB_BILIN_H4 m2, m4, m3 + PMADDUBSW m1, m5, m7, m4, 0 ; 1 2 + shufpd m3, m0, m1, 0x01 ; 0 1 + mova m0, m2 + PMADDUBSW m0, m5, m7, m4, 0 ; 3 4 + shufpd m2, m1, m0, 0x01 ; 2 3 + psubw m1, m3 + PMULHRSW m1, m6, m4, m8, 4 + paddw m1, m3 + psubw m3, m0, m2 + PMULHRSW m3, m6, m4, m8, 4 + paddw m3, m2 + mova [tmpq+16*0], m1 + mova [tmpq+16*1], m3 + add tmpq, 32 + sub hd, 4 + jg .hv_w4_loop + RET +.hv_w8: + movu m0, [srcq+strideq*0] + PSHUFB_BILIN_H8 m0, m4 + PMADDUBSW m0, m5, m7, m4, 0 ; 0 +.hv_w8_loop: + movu m1, [srcq+strideq*1] + movu m2, [srcq+strideq*2] + PSHUFB_BILIN_H8 m1, m4 + PSHUFB_BILIN_H8 m2, m4 + PMADDUBSW m1, m5, m7, m4, 0 ; 1 + PMADDUBSW m2, m5, m7, m4, 0 ; 2 + psubw m3, m1, m0 + PMULHRSW m3, m6, m4, m8, 4 + paddw m3, m0 +%if notcpuflag(ssse3) && ARCH_X86_64 + SWAP m9, m7 +%endif + psubw m7, m2, m1 + PMULHRSW m7, m6, m4, m8, 4 + paddw m7, m1 + mova [tmpq+16*0], m3 + mova [tmpq+16*1], m7 +%if notcpuflag(ssse3) && ARCH_X86_64 + SWAP m7, m9 +%endif + movu m1, [srcq+stride3q ] + lea srcq, [srcq+strideq*4] + movu m0, [srcq+strideq*0] + PSHUFB_BILIN_H8 m1, m4 + PSHUFB_BILIN_H8 m0, m4 + PMADDUBSW m1, m5, m7, m4, ARCH_X86_32 ; 3 + PMADDUBSW m0, m5, m7, m4, 0 ; 4 + psubw m3, m1, m2 + PMULHRSW m3, m6, m4, m8, 4 + paddw m3, m2 +%if notcpuflag(ssse3) && ARCH_X86_64 + SWAP m9, m7 +%endif + psubw m7, m0, m1 + PMULHRSW m7, m6, m4, m8, 4 + paddw m7, m1 + mova [tmpq+16*2], m3 + mova [tmpq+16*3], m7 +%if notcpuflag(ssse3) + %if ARCH_X86_64 + SWAP m7, m9 + %else + pxor m7, m7 + %endif +%endif + add tmpq, 16*4 + sub hd, 4 + jg .hv_w8_loop + RET +.hv_w16: + mov t2d, hd + mov t0d, 32 + jmp .hv_w16_start +.hv_w32: + lea t2d, [hq+(1<<16)] + mov t0d, 64 + jmp .hv_w16_start +.hv_w64: + lea t2d, [hq+(3<<16)] + mov t0d, 128 + jmp .hv_w16_start +.hv_w128: + lea t2d, [hq+(7<<16)] + mov t0d, 256 +.hv_w16_start: +%if ARCH_X86_64 + %if WIN64 + PUSH r7 + %endif + mov r7, tmpq + mov r5, srcq +%endif +.hv_w16_hloop: + movu m0, [srcq+strideq*0+8*0] + movu m1, [srcq+strideq*0+8*1] + PSHUFB_BILIN_H8 m0, m4 + PSHUFB_BILIN_H8 m1, m4 + PMADDUBSW m0, m5, m7, m4, 0 ; 0a + PMADDUBSW m1, m5, m7, m4, 0 ; 0b +.hv_w16_vloop: + movu m2, [srcq+strideq*1+8*0] + PSHUFB_BILIN_H8 m2, m4 + PMADDUBSW m2, m5, m7, m4, 0 ; 1a + psubw m3, m2, m0 + PMULHRSW m3, m6, m4, m8, 4 + paddw m3, m0 + mova [tmpq+16*0], m3 + movu m3, [srcq+strideq*1+8*1] + lea srcq, [srcq+strideq*2] + PSHUFB_BILIN_H8 m3, m4 + PMADDUBSW m3, m5, m7, m4, 0 ; 1b + psubw m0, m3, m1 + PMULHRSW m0, m6, m4, m8, 4 + paddw m0, m1 + mova [tmpq+16*1], m0 + add tmpq, t0q + movu m0, [srcq+strideq*0+8*0] + PSHUFB_BILIN_H8 m0, m4 + PMADDUBSW m0, m5, m7, m4, 0 ; 2a + psubw m1, m0, m2 + PMULHRSW m1, m6, m4, m8, 4 + paddw m1, m2 + mova [tmpq+16*0], m1 + movu m1, [srcq+strideq*0+8*1] + PSHUFB_BILIN_H8 m1, m4 + PMADDUBSW m1, m5, m7, m4, 0 ; 2b + psubw m2, m1, m3 + PMULHRSW m2, m6, m4, m8, 4 + paddw m2, m3 + mova [tmpq+16*1], m2 + add tmpq, t0q + sub hd, 2 + jg .hv_w16_vloop + movzx hd, t2w +%if ARCH_X86_64 + add r5, 16 + add r7, 2*16 + mov srcq, r5 + mov tmpq, r7 +%else + mov srcq, srcmp + mov tmpq, tmpmp + add srcq, 16 + add tmpq, 2*16 + mov srcmp, srcq + mov tmpmp, tmpq +%endif + sub t2d, 1<<16 + jg .hv_w16_hloop +%if WIN64 + POP r7 +%endif + RET +%endmacro + +; int8_t subpel_filters[5][15][8] +%assign FILTER_REGULAR (0*15 << 16) | 3*15 +%assign FILTER_SMOOTH (1*15 << 16) | 4*15 +%assign FILTER_SHARP (2*15 << 16) | 3*15 + +%macro MC_8TAP_FN 4 ; prefix, type, type_h, type_v +cglobal %1_8tap_%2 + mov t0d, FILTER_%3 +%ifidn %3, %4 + mov t1d, t0d +%else + mov t1d, FILTER_%4 +%endif +%ifnidn %2, regular ; skip the jump in the last filter + jmp mangle(private_prefix %+ _%1_8tap %+ SUFFIX) +%endif +%endmacro + +%if ARCH_X86_32 +DECLARE_REG_TMP 1, 2 +%elif WIN64 +DECLARE_REG_TMP 4, 5 +%else +DECLARE_REG_TMP 7, 8 +%endif + +MC_8TAP_FN put, sharp, SHARP, SHARP +MC_8TAP_FN put, sharp_smooth, SHARP, SMOOTH +MC_8TAP_FN put, smooth_sharp, SMOOTH, SHARP +MC_8TAP_FN put, smooth, SMOOTH, SMOOTH +MC_8TAP_FN put, sharp_regular, SHARP, REGULAR +MC_8TAP_FN put, regular_sharp, REGULAR, SHARP +MC_8TAP_FN put, smooth_regular, SMOOTH, REGULAR +MC_8TAP_FN put, regular_smooth, REGULAR, SMOOTH +MC_8TAP_FN put, regular, REGULAR, REGULAR + +%if ARCH_X86_32 + %define base_reg r1 + %define base base_reg-put_ssse3 + %define W32_RESTORE_DSQ mov dsq, dsm + %define W32_RESTORE_SSQ mov ssq, ssm +%else + %define base_reg r8 + %define base 0 + %define W32_RESTORE_DSQ + %define W32_RESTORE_SSQ +%endif + +cglobal put_8tap, 1, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 +%assign org_stack_offset stack_offset + imul mxd, mxm, 0x010101 + add mxd, t0d ; 8tap_h, mx, 4tap_h +%if ARCH_X86_64 + imul myd, mym, 0x010101 + add myd, t1d ; 8tap_v, my, 4tap_v +%else + imul ssd, mym, 0x010101 + add ssd, t1d ; 8tap_v, my, 4tap_v + mov srcq, srcm +%endif + mov wd, wm + movifnidn hd, hm + LEA base_reg, put_ssse3 + test mxd, 0xf00 + jnz .h +%if ARCH_X86_32 + test ssd, 0xf00 +%else + test myd, 0xf00 +%endif + jnz .v + tzcnt wd, wd + movzx wd, word [base_reg+wq*2+table_offset(put,)] + add wq, base_reg +; put_bilin mangling jump +%assign stack_offset org_stack_offset +%if ARCH_X86_32 + mov dsq, dsm + mov ssq, ssm +%elif WIN64 + pop r8 +%endif + lea r6, [ssq*3] + jmp wq +.h: +%if ARCH_X86_32 + test ssd, 0xf00 +%else + test myd, 0xf00 +%endif + jnz .hv + W32_RESTORE_SSQ + WIN64_SPILL_XMM 12 + cmp wd, 4 + jl .h_w2 + je .h_w4 + tzcnt wd, wd +%if ARCH_X86_64 + mova m10, [base+subpel_h_shufA] + mova m11, [base+subpel_h_shufB] + mova m9, [base+subpel_h_shufC] +%endif + shr mxd, 16 + sub srcq, 3 + movzx wd, word [base_reg+wq*2+table_offset(put, _8tap_h)] + movd m5, [base_reg+mxq*8+subpel_filters-put_ssse3+0] + pshufd m5, m5, q0000 + movd m6, [base_reg+mxq*8+subpel_filters-put_ssse3+4] + pshufd m6, m6, q0000 + mova m7, [base+pw_34] ; 2 + (8 << 2) + add wq, base_reg + jmp wq +.h_w2: +%if ARCH_X86_32 + and mxd, 0x7f +%else + movzx mxd, mxb +%endif + dec srcq + mova m4, [base+subpel_h_shuf4] + movd m3, [base_reg+mxq*8+subpel_filters-put_ssse3+2] + pshufd m3, m3, q0000 + mova m5, [base+pw_34] ; 2 + (8 << 2) + W32_RESTORE_DSQ +.h_w2_loop: + movq m0, [srcq+ssq*0] + movhps m0, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + pshufb m0, m4 + pmaddubsw m0, m3 + phaddw m0, m0 + paddw m0, m5 ; pw34 + psraw m0, 6 + packuswb m0, m0 + movd r4d, m0 + mov [dstq+dsq*0], r4w + shr r4d, 16 + mov [dstq+dsq*1], r4w + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w2_loop + RET +.h_w4: +%if ARCH_X86_32 + and mxd, 0x7f +%else + movzx mxd, mxb +%endif + dec srcq + movd m3, [base_reg+mxq*8+subpel_filters-put_ssse3+2] + pshufd m3, m3, q0000 + mova m5, [base+pw_34] ; 2 + (8 << 2) + mova m6, [base+subpel_h_shufA] + W32_RESTORE_DSQ +.h_w4_loop: + movq m0, [srcq+ssq*0] ; 1 + movq m1, [srcq+ssq*1] ; 2 + lea srcq, [srcq+ssq*2] + pshufb m0, m6 ; subpel_h_shufA + pshufb m1, m6 ; subpel_h_shufA + pmaddubsw m0, m3 ; subpel_filters + pmaddubsw m1, m3 ; subpel_filters + phaddw m0, m1 + paddw m0, m5 ; pw34 + psraw m0, 6 + packuswb m0, m0 + movd [dstq+dsq*0], m0 + psrlq m0, 32 + movd [dstq+dsq*1], m0 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .h_w4_loop + RET + ; +%macro PUT_8TAP_H 4 ; dst/src, tmp[1-3] + %if ARCH_X86_32 + pshufb %2, %1, [base+subpel_h_shufB] + pshufb %3, %1, [base+subpel_h_shufC] + pshufb %1, [base+subpel_h_shufA] + %else + pshufb %2, %1, m11; subpel_h_shufB + pshufb %3, %1, m9 ; subpel_h_shufC + pshufb %1, m10 ; subpel_h_shufA + %endif + pmaddubsw %4, %2, m5 ; subpel +0 B0 + pmaddubsw %2, m6 ; subpel +4 B4 + pmaddubsw %3, m6 ; C4 + pmaddubsw %1, m5 ; A0 + paddw %3, %4 ; C4+B0 + paddw %1, %2 ; A0+B4 + phaddw %1, %3 + paddw %1, m7 ; pw34 + psraw %1, 6 +%endmacro + ; +.h_w8: + movu m0, [srcq+ssq*0] + movu m1, [srcq+ssq*1] + PUT_8TAP_H m0, m2, m3, m4 + lea srcq, [srcq+ssq*2] + PUT_8TAP_H m1, m2, m3, m4 + packuswb m0, m1 +%if ARCH_X86_32 + movq [dstq ], m0 + add dstq, dsm + movhps [dstq ], m0 + add dstq, dsm +%else + movq [dstq+dsq*0], m0 + movhps [dstq+dsq*1], m0 + lea dstq, [dstq+dsq*2] +%endif + sub hd, 2 + jg .h_w8 + RET +.h_w16: + xor r6d, r6d + jmp .h_start +.h_w32: + mov r6, -16*1 + jmp .h_start +.h_w64: + mov r6, -16*3 + jmp .h_start +.h_w128: + mov r6, -16*7 +.h_start: + sub srcq, r6 + sub dstq, r6 + mov r4, r6 +.h_loop: + movu m0, [srcq+r6+8*0] + movu m1, [srcq+r6+8*1] + PUT_8TAP_H m0, m2, m3, m4 + PUT_8TAP_H m1, m2, m3, m4 + packuswb m0, m1 + mova [dstq+r6], m0 + add r6, mmsize + jle .h_loop + add srcq, ssq +%if ARCH_X86_32 + add dstq, dsm +%else + add dstq, dsq +%endif + mov r6, r4 + dec hd + jg .h_loop + RET +.v: +%if ARCH_X86_32 + movzx mxd, ssb + shr ssd, 16 + cmp hd, 6 + cmovs ssd, mxd + lea ssq, [base_reg+ssq*8+subpel_filters-put_ssse3] +%else + %assign stack_offset org_stack_offset + WIN64_SPILL_XMM 16 + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + lea myq, [base_reg+myq*8+subpel_filters-put_ssse3] +%endif + tzcnt r6d, wd + movzx r6d, word [base_reg+r6*2+table_offset(put, _8tap_v)] + mova m7, [base+pw_512] + psrlw m2, m7, 1 ; 0x0100 + add r6, base_reg +%if ARCH_X86_32 + %define subpel0 [rsp+mmsize*0] + %define subpel1 [rsp+mmsize*1] + %define subpel2 [rsp+mmsize*2] + %define subpel3 [rsp+mmsize*3] +%assign regs_used 2 ; use r1 (ds) as tmp for stack alignment if needed + ALLOC_STACK -mmsize*4 +%assign regs_used 7 + movd m0, [ssq+0] + pshufb m0, m2 + mova subpel0, m0 + movd m0, [ssq+2] + pshufb m0, m2 + mova subpel1, m0 + movd m0, [ssq+4] + pshufb m0, m2 + mova subpel2, m0 + movd m0, [ssq+6] + pshufb m0, m2 + mova subpel3, m0 + mov ssq, [rstk+stack_offset+gprsize*4] + lea ssq, [ssq*3] + sub srcq, ssq + mov ssq, [rstk+stack_offset+gprsize*4] + mov dsq, [rstk+stack_offset+gprsize*2] +%else + %define subpel0 m8 + %define subpel1 m9 + %define subpel2 m10 + %define subpel3 m11 + movd subpel0, [myq+0] + pshufb subpel0, m2 + movd subpel1, [myq+2] + pshufb subpel1, m2 + movd subpel2, [myq+4] + pshufb subpel2, m2 + movd subpel3, [myq+6] + pshufb subpel3, m2 + lea ss3q, [ssq*3] + sub srcq, ss3q +%endif + jmp r6 +.v_w2: + movd m2, [srcq+ssq*0] ; 0 + pinsrw m2, [srcq+ssq*1], 2 ; 0 1 + pinsrw m2, [srcq+ssq*2], 4 ; 0 1 2 +%if ARCH_X86_32 + lea srcq, [srcq+ssq*2] + add srcq, ssq + pinsrw m2, [srcq+ssq*0], 6 ; 0 1 2 3 + add srcq, ssq +%else + pinsrw m2, [srcq+ss3q ], 6 ; 0 1 2 3 + lea srcq, [srcq+ssq*4] +%endif + movd m3, [srcq+ssq*0] ; 4 + movd m1, [srcq+ssq*1] ; 5 + movd m0, [srcq+ssq*2] ; 6 +%if ARCH_X86_32 + lea srcq, [srcq+ssq*2] + add srcq, ssq +%else + add srcq, ss3q +%endif + punpckldq m3, m1 ; 4 5 _ _ + punpckldq m1, m0 ; 5 6 _ _ + palignr m4, m3, m2, 4 ; 1 2 3 4 + punpcklbw m3, m1 ; 45 56 + punpcklbw m1, m2, m4 ; 01 12 + punpckhbw m2, m4 ; 23 34 +.v_w2_loop: + pmaddubsw m5, m1, subpel0 ; a0 b0 + mova m1, m2 + pmaddubsw m2, subpel1 ; a1 b1 + paddw m5, m2 + mova m2, m3 + pmaddubsw m3, subpel2 ; a2 b2 + paddw m5, m3 + movd m4, [srcq+ssq*0] ; 7 + punpckldq m3, m0, m4 ; 6 7 _ _ + movd m0, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + punpckldq m4, m0 ; 7 8 _ _ + punpcklbw m3, m4 ; 67 78 + pmaddubsw m4, m3, subpel3 ; a3 b3 + paddw m5, m4 + pmulhrsw m5, m7 + packuswb m5, m5 + pshuflw m5, m5, q2020 + movd r6d, m5 + mov [dstq+dsq*0], r6w + shr r6d, 16 + mov [dstq+dsq*1], r6w + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w2_loop + RET +.v_w4: +%if ARCH_X86_32 +.v_w8: +.v_w16: +.v_w32: +.v_w64: +.v_w128: +%endif ; ARCH_X86_32 + lea r6d, [wq - 4] ; horizontal loop + mov r4, dstq +%if ARCH_X86_32 +%if STACK_ALIGNMENT < mmsize + %define srcm [rsp+mmsize*4+gprsize] +%endif + mov srcm, srcq +%else + mov r7, srcq +%endif + shl r6d, (16 - 2) ; (wq / 4) << 16 + mov r6w, hw +.v_w4_loop0: + movd m2, [srcq+ssq*0] ; 0 + movhps m2, [srcq+ssq*2] ; 0 _ 2 + movd m3, [srcq+ssq*1] ; 1 +%if ARCH_X86_32 + lea srcq, [srcq+ssq*2] + add srcq, ssq + movhps m3, [srcq+ssq*0] ; 1 _ 3 + lea srcq, [srcq+ssq*1] +%else + movhps m3, [srcq+ss3q ] ; 1 _ 3 + lea srcq, [srcq+ssq*4] +%endif + pshufd m2, m2, q2020 ; 0 2 0 2 + pshufd m3, m3, q2020 ; 1 3 1 3 + punpckldq m2, m3 ; 0 1 2 3 + movd m3, [srcq+ssq*0] ; 4 + movd m1, [srcq+ssq*1] ; 5 + movd m0, [srcq+ssq*2] ; 6 +%if ARCH_X86_32 + lea srcq, [srcq+ssq*2] + add srcq, ssq +%else + add srcq, ss3q +%endif + punpckldq m3, m1 ; 4 5 _ _ + punpckldq m1, m0 ; 5 6 _ _ + palignr m4, m3, m2, 4 ; 1 2 3 4 + punpcklbw m3, m1 ; 45 56 + punpcklbw m1, m2, m4 ; 01 12 + punpckhbw m2, m4 ; 23 34 +.v_w4_loop: + pmaddubsw m5, m1, subpel0 ; a0 b0 + mova m1, m2 + pmaddubsw m2, subpel1 ; a1 b1 + paddw m5, m2 + mova m2, m3 + pmaddubsw m3, subpel2 ; a2 b2 + paddw m5, m3 + movd m4, [srcq+ssq*0] + punpckldq m3, m0, m4 ; 6 7 _ _ + movd m0, [srcq+ssq*1] + lea srcq, [srcq+ssq*2] + punpckldq m4, m0 ; 7 8 _ _ + punpcklbw m3, m4 ; 67 78 + pmaddubsw m4, m3, subpel3 ; a3 b3 + paddw m5, m4 + pmulhrsw m5, m7 + packuswb m5, m5 + movd [dstq+dsq*0], m5 + pshufd m5, m5, q0101 + movd [dstq+dsq*1], m5 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w4_loop + mov hw, r6w ; reset vertical loop + add r4, 4 + mov dstq, r4 +%if ARCH_X86_32 + mov srcq, srcm + add srcq, 4 + mov srcm, srcq +%else + add r7, 4 + mov srcq, r7 +%endif + sub r6d, 1<<16 ; horizontal-- + jg .v_w4_loop0 + RET +%if ARCH_X86_64 +.v_w8: +.v_w16: +.v_w32: +.v_w64: +.v_w128: + lea r6d, [wq - 8] ; horizontal loop + mov r4, dstq + mov r7, srcq + shl r6d, 8 - 3; (wq / 8) << 8 + mov r6b, hb +.v_w8_loop0: + movq m4, [srcq+ssq*0] ; 0 + movq m5, [srcq+ssq*1] ; 1 + lea srcq, [srcq+ssq*2] + movq m6, [srcq+ssq*0] ; 2 + movq m0, [srcq+ssq*1] ; 3 + lea srcq, [srcq+ssq*2] + movq m1, [srcq+ssq*0] ; 4 + movq m2, [srcq+ssq*1] ; 5 + lea srcq, [srcq+ssq*2] ; + movq m3, [srcq+ssq*0] ; 6 + shufpd m4, m0, 0x0c + shufpd m5, m1, 0x0c + punpcklbw m1, m4, m5 ; 01 + punpckhbw m4, m5 ; 34 + shufpd m6, m2, 0x0c + punpcklbw m2, m5, m6 ; 12 + punpckhbw m5, m6 ; 45 + shufpd m0, m3, 0x0c + punpcklbw m3, m6, m0 ; 23 + punpckhbw m6, m0 ; 56 +.v_w8_loop: + movq m12, [srcq+ssq*1] ; 8 + lea srcq, [srcq+ssq*2] + movq m13, [srcq+ssq*0] ; 9 + pmaddubsw m14, m1, subpel0 ; a0 + pmaddubsw m15, m2, subpel0 ; b0 + mova m1, m3 + mova m2, m4 + pmaddubsw m3, subpel1 ; a1 + pmaddubsw m4, subpel1 ; b1 + paddw m14, m3 + paddw m15, m4 + mova m3, m5 + mova m4, m6 + pmaddubsw m5, subpel2 ; a2 + pmaddubsw m6, subpel2 ; b2 + paddw m14, m5 + paddw m15, m6 + shufpd m6, m0, m12, 0x0d + shufpd m0, m12, m13, 0x0c + punpcklbw m5, m6, m0 ; 67 + punpckhbw m6, m0 ; 78 + pmaddubsw m12, m5, subpel3 ; a3 + pmaddubsw m13, m6, subpel3 ; b3 + paddw m14, m12 + paddw m15, m13 + pmulhrsw m14, m7 + pmulhrsw m15, m7 + packuswb m14, m15 + movq [dstq+dsq*0], xm14 + movhps [dstq+dsq*1], xm14 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .v_w8_loop + movzx hd, r6b ; reset vertical loop + add r4, 8 + add r7, 8 + mov dstq, r4 + mov srcq, r7 + sub r6d, 1<<8 ; horizontal-- + jg .v_w8_loop0 + RET +%endif ;ARCH_X86_64 +%undef subpel0 +%undef subpel1 +%undef subpel2 +%undef subpel3 +.hv: + %assign stack_offset org_stack_offset + cmp wd, 4 + jg .hv_w8 +%if ARCH_X86_32 + and mxd, 0x7f +%else + movzx mxd, mxb +%endif + dec srcq + movd m1, [base_reg+mxq*8+subpel_filters-put_ssse3+2] +%if ARCH_X86_32 + movzx mxd, ssb + shr ssd, 16 + cmp hd, 6 + cmovs ssd, mxd + movq m0, [base_reg+ssq*8+subpel_filters-put_ssse3] + W32_RESTORE_SSQ + lea r6, [ssq*3] + sub srcq, r6 + %define base_reg r6 + mov r6, r1; use as new base + %assign regs_used 2 + ALLOC_STACK -mmsize*14 + %assign regs_used 7 + mov dsq, [rstk+stack_offset+gprsize*2] + %define subpelv0 [rsp+mmsize*0] + %define subpelv1 [rsp+mmsize*1] + %define subpelv2 [rsp+mmsize*2] + %define subpelv3 [rsp+mmsize*3] + punpcklqdq m0, m0 + punpcklbw m0, m0 + psraw m0, 8 ; sign-extend + pshufd m6, m0, q0000 + mova subpelv0, m6 + pshufd m6, m0, q1111 + mova subpelv1, m6 + pshufd m6, m0, q2222 + mova subpelv2, m6 + pshufd m6, m0, q3333 + mova subpelv3, m6 +%else + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + movq m0, [base_reg+myq*8+subpel_filters-put_ssse3] + ALLOC_STACK mmsize*14, 14 + lea ss3q, [ssq*3] + sub srcq, ss3q + %define subpelv0 m10 + %define subpelv1 m11 + %define subpelv2 m12 + %define subpelv3 m13 + punpcklqdq m0, m0 + punpcklbw m0, m0 + psraw m0, 8 ; sign-extend + mova m8, [base+pw_8192] + mova m9, [base+pd_512] + pshufd m10, m0, q0000 + pshufd m11, m0, q1111 + pshufd m12, m0, q2222 + pshufd m13, m0, q3333 +%endif + pshufd m7, m1, q0000 + cmp wd, 4 + je .hv_w4 +.hv_w2: + mova m6, [base+subpel_h_shuf4] + ; + movq m2, [srcq+ssq*0] ; 0 + movhps m2, [srcq+ssq*1] ; 0 _ 1 + movq m0, [srcq+ssq*2] ; 2 +%if ARCH_X86_32 + %define w8192reg [base+pw_8192] + %define d512reg [base+pd_512] + lea srcq, [srcq+ssq*2] + add srcq, ssq + movhps m0, [srcq+ssq*0] ; 2 _ 3 + lea srcq, [srcq+ssq*1] +%else + %define w8192reg m8 + %define d512reg m9 + movhps m0, [srcq+ss3q ] ; 2 _ 3 + lea srcq, [srcq+ssq*4] +%endif + pshufb m2, m6 ; 0 ~ 1 ~ + pshufb m0, m6 ; 2 ~ 3 ~ + pmaddubsw m2, m7 ; subpel_filters + pmaddubsw m0, m7 ; subpel_filters + phaddw m2, m0 ; 0 1 2 3 + pmulhrsw m2, w8192reg + ; + movq m3, [srcq+ssq*0] ; 4 + movhps m3, [srcq+ssq*1] ; 4 _ 5 + movq m0, [srcq+ssq*2] ; 6 +%if ARCH_X86_32 + lea srcq, [srcq+ssq*2] + add srcq, ssq +%else + add srcq, ss3q +%endif + pshufb m3, m6 ; 4 ~ 5 ~ + pshufb m0, m6 ; 6 ~ + pmaddubsw m3, m7 ; subpel_filters + pmaddubsw m0, m7 ; subpel_filters + phaddw m3, m0 ; 4 5 6 _ + pmulhrsw m3, w8192reg + ; + palignr m4, m3, m2, 4; V 1 2 3 4 + punpcklwd m1, m2, m4 ; V 01 12 0 1 1 2 + punpckhwd m2, m4 ; V 23 34 2 3 3 4 + pshufd m0, m3, q2121; V 5 6 5 6 + punpcklwd m3, m0 ; V 45 56 4 5 5 6 +.hv_w2_loop: + pmaddwd m5, m1, subpelv0; V a0 b0 + mova m1, m2 ; V + pmaddwd m2, subpelv1 ; V a1 b1 + paddd m5, m2 ; V + mova m2, m3 ; V + pmaddwd m3, subpelv2 ; a2 b2 + paddd m5, m3 ; V + movq m4, [srcq+ssq*0] ; V 7 + movhps m4, [srcq+ssq*1] ; V 7 8 + lea srcq, [srcq+ssq*2] ; V + pshufb m4, m6 + pmaddubsw m4, m7 + phaddw m4, m4 + pmulhrsw m4, w8192reg + palignr m3, m4, m0, 12 + mova m0, m4 + punpcklwd m3, m0 ; V 67 78 + pmaddwd m4, m3, subpelv3 ; V a3 b3 + paddd m5, d512reg + paddd m5, m4 + psrad m5, 10 + packssdw m5, m5 + packuswb m5, m5 + movd r4d, m5 + mov [dstq+dsq*0], r4w + shr r4d, 16 + mov [dstq+dsq*1], r4w + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .hv_w2_loop + RET +%undef w8192reg +%undef d512reg + ; +.hv_w4: +%define hv4_line_0_0 4 +%define hv4_line_0_1 5 +%define hv4_line_0_2 6 +%define hv4_line_0_3 7 +%define hv4_line_0_4 8 +%define hv4_line_0_5 9 +%define hv4_line_1_0 10 +%define hv4_line_1_1 11 +%define hv4_line_1_2 12 +%define hv4_line_1_3 13 + ; +%macro SAVELINE_W4 3 + mova [rsp+mmsize*hv4_line_%3_%2], %1 +%endmacro +%macro RESTORELINE_W4 3 + mova %1, [rsp+mmsize*hv4_line_%3_%2] +%endmacro + ; +%if ARCH_X86_32 + %define w8192reg [base+pw_8192] + %define d512reg [base+pd_512] +%else + %define w8192reg m8 + %define d512reg m9 +%endif + ; lower shuffle 0 1 2 3 4 + mova m6, [base+subpel_h_shuf4] + movq m5, [srcq+ssq*0] ; 0 _ _ _ + movhps m5, [srcq+ssq*1] ; 0 _ 1 _ + movq m4, [srcq+ssq*2] ; 2 _ _ _ +%if ARCH_X86_32 + lea srcq, [srcq+ssq*2] + add srcq, ssq + movhps m4, [srcq+ssq*0] ; 2 _ 3 _ + add srcq, ssq +%else + movhps m4, [srcq+ss3q ] ; 2 _ 3 _ + lea srcq, [srcq+ssq*4] +%endif + pshufb m2, m5, m6 ;H subpel_h_shuf4 0 ~ 1 ~ + pshufb m0, m4, m6 ;H subpel_h_shuf4 2 ~ 3 ~ + pmaddubsw m2, m7 ;H subpel_filters + pmaddubsw m0, m7 ;H subpel_filters + phaddw m2, m0 ;H 0 1 2 3 + pmulhrsw m2, w8192reg ;H pw_8192 + SAVELINE_W4 m2, 2, 0 + ; upper shuffle 2 3 4 5 6 + mova m6, [base+subpel_h_shuf4+16] + pshufb m2, m5, m6 ;H subpel_h_shuf4 0 ~ 1 ~ + pshufb m0, m4, m6 ;H subpel_h_shuf4 2 ~ 3 ~ + pmaddubsw m2, m7 ;H subpel_filters + pmaddubsw m0, m7 ;H subpel_filters + phaddw m2, m0 ;H 0 1 2 3 + pmulhrsw m2, w8192reg ;H pw_8192 + ; + ; lower shuffle + mova m6, [base+subpel_h_shuf4] + movq m5, [srcq+ssq*0] ; 4 _ _ _ + movhps m5, [srcq+ssq*1] ; 4 _ 5 _ + movq m4, [srcq+ssq*2] ; 6 _ _ _ + pshufb m3, m5, m6 ;H subpel_h_shuf4 4 ~ 5 ~ + pshufb m0, m4, m6 ;H subpel_h_shuf4 6 ~ 6 ~ + pmaddubsw m3, m7 ;H subpel_filters + pmaddubsw m0, m7 ;H subpel_filters + phaddw m3, m0 ;H 4 5 6 7 + pmulhrsw m3, w8192reg ;H pw_8192 + SAVELINE_W4 m3, 3, 0 + ; upper shuffle + mova m6, [base+subpel_h_shuf4+16] + pshufb m3, m5, m6 ;H subpel_h_shuf4 4 ~ 5 ~ + pshufb m0, m4, m6 ;H subpel_h_shuf4 6 ~ 6 ~ + pmaddubsw m3, m7 ;H subpel_filters + pmaddubsw m0, m7 ;H subpel_filters + phaddw m3, m0 ;H 4 5 6 7 + pmulhrsw m3, w8192reg ;H pw_8192 + ; +%if ARCH_X86_32 + lea srcq, [srcq+ssq*2] + add srcq, ssq +%else + add srcq, ss3q +%endif + ;process high + palignr m4, m3, m2, 4;V 1 2 3 4 + punpcklwd m1, m2, m4 ; V 01 12 + punpckhwd m2, m4 ; V 23 34 + pshufd m0, m3, q2121;V 5 6 5 6 + punpcklwd m3, m0 ; V 45 56 + SAVELINE_W4 m0, 0, 1 + SAVELINE_W4 m1, 1, 1 + SAVELINE_W4 m2, 2, 1 + SAVELINE_W4 m3, 3, 1 + ;process low + RESTORELINE_W4 m2, 2, 0 + RESTORELINE_W4 m3, 3, 0 + palignr m4, m3, m2, 4;V 1 2 3 4 + punpcklwd m1, m2, m4 ; V 01 12 + punpckhwd m2, m4 ; V 23 34 + pshufd m0, m3, q2121;V 5 6 5 6 + punpcklwd m3, m0 ; V 45 56 +.hv_w4_loop: + ;process low + pmaddwd m5, m1, subpelv0 ; V a0 b0 + mova m1, m2 + pmaddwd m2, subpelv1; V a1 b1 + paddd m5, m2 + mova m2, m3 + pmaddwd m3, subpelv2; V a2 b2 + paddd m5, m3 + ; + mova m6, [base+subpel_h_shuf4] + movq m4, [srcq+ssq*0] ; 7 + movhps m4, [srcq+ssq*1] ; 7 _ 8 _ + pshufb m4, m6 ;H subpel_h_shuf4 7 ~ 8 ~ + pmaddubsw m4, m7 ;H subpel_filters + phaddw m4, m4 ;H 7 8 7 8 + pmulhrsw m4, w8192reg ;H pw_8192 + palignr m3, m4, m0, 12 ; 6 7 8 7 + mova m0, m4 + punpcklwd m3, m4 ; 67 78 + pmaddwd m4, m3, subpelv3; a3 b3 + paddd m5, d512reg ; pd_512 + paddd m5, m4 + psrad m5, 10 + SAVELINE_W4 m0, 0, 0 + SAVELINE_W4 m1, 1, 0 + SAVELINE_W4 m2, 2, 0 + SAVELINE_W4 m3, 3, 0 + SAVELINE_W4 m5, 5, 0 + ;process high + RESTORELINE_W4 m0, 0, 1 + RESTORELINE_W4 m1, 1, 1 + RESTORELINE_W4 m2, 2, 1 + RESTORELINE_W4 m3, 3, 1 + pmaddwd m5, m1, subpelv0; V a0 b0 + mova m1, m2 + pmaddwd m2, subpelv1; V a1 b1 + paddd m5, m2 + mova m2, m3 + pmaddwd m3, subpelv2; V a2 b2 + paddd m5, m3 + ; + mova m6, [base+subpel_h_shuf4+16] + movq m4, [srcq+ssq*0] ; 7 + movhps m4, [srcq+ssq*1] ; 7 _ 8 _ + pshufb m4, m6 ;H subpel_h_shuf4 7 ~ 8 ~ + pmaddubsw m4, m7 ;H subpel_filters + phaddw m4, m4 ;H 7 8 7 8 + pmulhrsw m4, w8192reg ;H pw_8192 + palignr m3, m4, m0, 12 ; 6 7 8 7 + mova m0, m4 + punpcklwd m3, m4 ; 67 78 + pmaddwd m4, m3, subpelv3; a3 b3 + paddd m5, d512reg ; pd_512 + paddd m5, m4 + psrad m4, m5, 10 + ; + RESTORELINE_W4 m5, 5, 0 + packssdw m5, m4 ; d -> w + packuswb m5, m5 ; w -> b + pshuflw m5, m5, q3120 + lea srcq, [srcq+ssq*2] + movd [dstq+dsq*0], m5 + psrlq m5, 32 + movd [dstq+dsq*1], m5 + lea dstq, [dstq+dsq*2] + sub hd, 2 + SAVELINE_W4 m0, 0, 1 + SAVELINE_W4 m1, 1, 1 + SAVELINE_W4 m2, 2, 1 + SAVELINE_W4 m3, 3, 1 + RESTORELINE_W4 m0, 0, 0 + RESTORELINE_W4 m1, 1, 0 + RESTORELINE_W4 m2, 2, 0 + RESTORELINE_W4 m3, 3, 0 + jg .hv_w4_loop + RET +%undef subpelv0 +%undef subpelv1 +%undef subpelv2 +%undef subpelv3 + ; +.hv_w8: + %assign stack_offset org_stack_offset +%define hv8_line_1 0 +%define hv8_line_2 1 +%define hv8_line_3 2 +%define hv8_line_4 3 +%define hv8_line_6 4 +%macro SAVELINE_W8 2 + mova [rsp+hv8_line_%1*mmsize], %2 +%endmacro +%macro RESTORELINE_W8 2 + mova %2, [rsp+hv8_line_%1*mmsize] +%endmacro + shr mxd, 16 + sub srcq, 3 +%if ARCH_X86_32 + %define base_reg r1 + %define subpelh0 [rsp+mmsize*5] + %define subpelh1 [rsp+mmsize*6] + %define subpelv0 [rsp+mmsize*7] + %define subpelv1 [rsp+mmsize*8] + %define subpelv2 [rsp+mmsize*9] + %define subpelv3 [rsp+mmsize*10] + %define accuv0 [rsp+mmsize*11] + %define accuv1 [rsp+mmsize*12] + movq m1, [base_reg+mxq*8+subpel_filters-put_ssse3] + movzx mxd, ssb + shr ssd, 16 + cmp hd, 6 + cmovs ssd, mxd + movq m5, [base_reg+ssq*8+subpel_filters-put_ssse3] + mov ssq, ssmp + ALLOC_STACK -mmsize*13 +%if STACK_ALIGNMENT < 16 + %define srcm [rsp+mmsize*13+gprsize*1] + %define dsm [rsp+mmsize*13+gprsize*2] + mov r6, [rstk+stack_offset+gprsize*2] + mov dsm, r6 +%endif + pshufd m0, m1, q0000 + pshufd m1, m1, q1111 + punpcklbw m5, m5 + psraw m5, 8 ; sign-extend + pshufd m2, m5, q0000 + pshufd m3, m5, q1111 + pshufd m4, m5, q2222 + pshufd m5, m5, q3333 + mova subpelh0, m0 + mova subpelh1, m1 + mova subpelv0, m2 + mova subpelv1, m3 + mova subpelv2, m4 + mova subpelv3, m5 + lea r6, [ssq*3] + sub srcq, r6 + mov srcm, srcq +%else + ALLOC_STACK mmsize*5, 16 + %define subpelh0 m10 + %define subpelh1 m11 + %define subpelv0 m12 + %define subpelv1 m13 + %define subpelv2 m14 + %define subpelv3 m15 + %define accuv0 m8 + %define accuv1 m9 + movq m0, [base_reg+mxq*8+subpel_filters-put_ssse3] + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + movq m1, [base_reg+myq*8+subpel_filters-put_ssse3] + pshufd subpelh0, m0, q0000 + pshufd subpelh1, m0, q1111 + punpcklqdq m1, m1 + punpcklbw m1, m1 + psraw m1, 8 ; sign-extend + pshufd subpelv0, m1, q0000 + pshufd subpelv1, m1, q1111 + pshufd subpelv2, m1, q2222 + pshufd subpelv3, m1, q3333 + lea ss3q, [ssq*3] + sub srcq, ss3q + mov r7, srcq +%endif + lea r6d, [wq-4] + mov r4, dstq + shl r6d, (16 - 2) + mov r6w, hw +.hv_w8_loop0: + movu m4, [srcq+ssq*0] ; 0 = _ _ + movu m5, [srcq+ssq*1] ; 1 = _ _ + lea srcq, [srcq+ssq*2] + ; +%macro HV_H_W8 4-7 ; src/dst, tmp[1-3], shuf[1-3] + %if ARCH_X86_32 + pshufb %3, %1, [base+subpel_h_shufB] + pshufb %4, %1, [base+subpel_h_shufC] + pshufb %1, [base+subpel_h_shufA] + %else + pshufb %3, %1, %6 ; subpel_h_shufB + pshufb %4, %1, %7 ; subpel_h_shufC + pshufb %1, %5 ; subpel_h_shufA + %endif + pmaddubsw %2, %3, subpelh0 ; subpel +0 C0 + pmaddubsw %4, subpelh1; subpel +4 B4 + pmaddubsw %3, subpelh1; C4 + pmaddubsw %1, subpelh0; A0 + paddw %2, %4 ; C0+B4 + paddw %1, %3 ; A0+C4 + phaddw %1, %2 +%endmacro + ; +%if ARCH_X86_64 + mova m7, [base+subpel_h_shufA] + mova m8, [base+subpel_h_shufB] + mova m9, [base+subpel_h_shufC] +%endif + HV_H_W8 m4, m1, m2, m3, m7, m8, m9 ; 0 ~ ~ ~ + HV_H_W8 m5, m1, m2, m3, m7, m8, m9 ; 1 ~ ~ ~ + movu m6, [srcq+ssq*0] ; 2 = _ _ + movu m0, [srcq+ssq*1] ; 3 = _ _ + lea srcq, [srcq+ssq*2] + HV_H_W8 m6, m1, m2, m3, m7, m8, m9 ; 2 ~ ~ ~ + HV_H_W8 m0, m1, m2, m3, m7, m8, m9 ; 3 ~ ~ ~ + ; + mova m7, [base+pw_8192] + pmulhrsw m4, m7 ; H pw_8192 + pmulhrsw m5, m7 ; H pw_8192 + pmulhrsw m6, m7 ; H pw_8192 + pmulhrsw m0, m7 ; H pw_8192 + punpcklwd m1, m4, m5 ; 0 1 ~ + punpcklwd m2, m5, m6 ; 1 2 ~ + punpcklwd m3, m6, m0 ; 2 3 ~ + SAVELINE_W8 1, m1 + SAVELINE_W8 2, m2 + SAVELINE_W8 3, m3 + ; + mova m7, [base+subpel_h_shufA] + movu m4, [srcq+ssq*0] ; 4 = _ _ + movu m5, [srcq+ssq*1] ; 5 = _ _ + lea srcq, [srcq+ssq*2] + movu m6, [srcq+ssq*0] ; 6 = _ _ + HV_H_W8 m4, m1, m2, m3, m7, m8, m9 ; 4 ~ ~ ~ + HV_H_W8 m5, m1, m2, m3, m7, m8, m9 ; 5 ~ ~ ~ + HV_H_W8 m6, m1, m2, m3, m7, m8, m9 ; 6 ~ ~ ~ + mova m7, [base+pw_8192] + pmulhrsw m1, m4, m7 ; H pw_8192 4 ~ + pmulhrsw m2, m5, m7 ; H pw_8192 5 ~ + pmulhrsw m3, m6, m7 ; H pw_8192 6 ~ + punpcklwd m4, m0, m1 ; 3 4 ~ + punpcklwd m5, m1, m2 ; 4 5 ~ + punpcklwd m6, m2, m3 ; 5 6 ~ + ; + SAVELINE_W8 6, m3 + RESTORELINE_W8 1, m1 + RESTORELINE_W8 2, m2 + RESTORELINE_W8 3, m3 +.hv_w8_loop: + ; m8 accu for V a + ; m9 accu for V b + SAVELINE_W8 1, m3 + SAVELINE_W8 2, m4 + SAVELINE_W8 3, m5 + SAVELINE_W8 4, m6 +%if ARCH_X86_32 + pmaddwd m0, m1, subpelv0 ; a0 + pmaddwd m7, m2, subpelv0 ; b0 + pmaddwd m3, subpelv1 ; a1 + pmaddwd m4, subpelv1 ; b1 + paddd m0, m3 + paddd m7, m4 + pmaddwd m5, subpelv2 ; a2 + pmaddwd m6, subpelv2 ; b2 + paddd m0, m5 + paddd m7, m6 + mova m5, [base+pd_512] + paddd m0, m5 ; pd_512 + paddd m7, m5 ; pd_512 + mova accuv0, m0 + mova accuv1, m7 +%else + pmaddwd m8, m1, subpelv0 ; a0 + pmaddwd m9, m2, subpelv0 ; b0 + pmaddwd m3, subpelv1 ; a1 + pmaddwd m4, subpelv1 ; b1 + paddd m8, m3 + paddd m9, m4 + pmaddwd m5, subpelv2 ; a2 + pmaddwd m6, subpelv2 ; b2 + paddd m8, m5 + paddd m9, m6 + mova m7, [base+pd_512] + paddd m8, m7 ; pd_512 + paddd m9, m7 ; pd_512 + mova m7, [base+subpel_h_shufB] + mova m6, [base+subpel_h_shufC] + mova m5, [base+subpel_h_shufA] +%endif + movu m0, [srcq+ssq*1] ; 7 + movu m4, [srcq+ssq*2] ; 8 + lea srcq, [srcq+ssq*2] + HV_H_W8 m0, m1, m2, m3, m5, m7, m6 + HV_H_W8 m4, m1, m2, m3, m5, m7, m6 + mova m5, [base+pw_8192] + pmulhrsw m0, m5 ; H pw_8192 + pmulhrsw m4, m5 ; H pw_8192 + RESTORELINE_W8 6, m6 + punpcklwd m5, m6, m0 ; 6 7 ~ + punpcklwd m6, m0, m4 ; 7 8 ~ + pmaddwd m1, m5, subpelv3 ; a3 + paddd m2, m1, accuv0 + pmaddwd m1, m6, subpelv3 ; b3 + paddd m1, m1, accuv1 ; H + V + psrad m2, 10 + psrad m1, 10 + packssdw m2, m1 ; d -> w + packuswb m2, m1 ; w -> b + movd [dstq+dsq*0], m2 + psrlq m2, 32 +%if ARCH_X86_32 + add dstq, dsm + movd [dstq+dsq*0], m2 + add dstq, dsm +%else + movd [dstq+dsq*1], m2 + lea dstq, [dstq+dsq*2] +%endif + sub hd, 2 + jle .hv_w8_outer + SAVELINE_W8 6, m4 + RESTORELINE_W8 1, m1 + RESTORELINE_W8 2, m2 + RESTORELINE_W8 3, m3 + RESTORELINE_W8 4, m4 + jmp .hv_w8_loop +.hv_w8_outer: + movzx hd, r6w + add r4, 4 + mov dstq, r4 +%if ARCH_X86_32 + mov srcq, srcm + add srcq, 4 + mov srcm, srcq +%else + add r7, 4 + mov srcq, r7 +%endif + sub r6d, 1<<16 + jg .hv_w8_loop0 + RET + +%macro PSHUFB_SUBPEL_H_4 5 ; dst/src1, src2/mask, tmp1, tmp2, reset_mask + %if cpuflag(ssse3) + pshufb %1, %2 + %else + %if %5 == 1 + pcmpeqd %2, %2 + psrlq %2, 32 + %endif + psrldq %3, %1, 1 + pshufd %3, %3, q2301 + pand %1, %2 + pandn %4, %2, %3 + por %1, %4 + %endif +%endmacro + +%macro PSHUFB_SUBPEL_H_4a 6 ; dst, src1, src2/mask, tmp1, tmp2, reset_mask + %ifnidn %1, %2 + mova %1, %2 + %endif + PSHUFB_SUBPEL_H_4 %1, %3, %4, %5, %6 +%endmacro + +%macro PSHUFB_SUBPEL_H_4b 6 ; dst, src1, src2/mask, tmp1, tmp2, reset_mask + %if notcpuflag(ssse3) + psrlq %1, %2, 16 + %elifnidn %1, %2 + mova %1, %2 + %endif + PSHUFB_SUBPEL_H_4 %1, %3, %4, %5, %6 +%endmacro + +%macro PALIGNR 4-5 ; dst, src1, src2, shift[, tmp] + %if cpuflag(ssse3) + palignr %1, %2, %3, %4 + %else + %if %0 == 4 + %assign %%i regnumof%+%1 + 1 + %define %%tmp m %+ %%i + %else + %define %%tmp %5 + %endif + psrldq %1, %3, %4 + pslldq %%tmp, %2, 16-%4 + por %1, %%tmp + %endif +%endmacro + +%macro PHADDW 4 ; dst, src, pw_1/tmp, load_pw_1 + %if cpuflag(ssse3) + phaddw %1, %2 + %elifnidn %1, %2 + %if %4 == 1 + mova %3, [base+pw_1] + %endif + pmaddwd %1, %3 + pmaddwd %2, %3 + packssdw %1, %2 + %else + %if %4 == 1 + pmaddwd %1, [base+pw_1] + %else + pmaddwd %1, %3 + %endif + packssdw %1, %1 + %endif +%endmacro + +%macro PMULHRSW_POW2 4 ; dst, src1, src2, shift + %if cpuflag(ssse3) + pmulhrsw %1, %2, %3 + %else + paddw %1, %2, %3 + psraw %1, %4 + %endif +%endmacro + +%macro PMULHRSW_8192 3 ; dst, src1, src2 + PMULHRSW_POW2 %1, %2, %3, 2 +%endmacro + +%macro PREP_8TAP_H_LOAD4 5 ; dst, src_memloc, tmp[1-2] + movd %1, [%2+0] + movd %3, [%2+1] + movd %4, [%2+2] + movd %5, [%2+3] + punpckldq %1, %3 + punpckldq %4, %5 + punpcklqdq %1, %4 +%endmacro + +%macro PREP_8TAP_H_LOAD 2 ; dst0, src_memloc + %if cpuflag(ssse3) + movu m%1, [%2] + pshufb m2, m%1, m11 ; subpel_h_shufB + pshufb m3, m%1, m9 ; subpel_h_shufC + pshufb m%1, m10 ; subpel_h_shufA + %else + %if ARCH_X86_64 + SWAP m12, m5 + SWAP m13, m6 + SWAP m14, m7 + %define %%mx0 m%+%%i + %define %%mx1 m%+%%j + %assign %%i 0 + %rep 12 + movd %%mx0, [%2+%%i] + %assign %%i %%i+1 + %endrep + %assign %%i 0 + %rep 6 + %assign %%j %%i+1 + punpckldq %%mx0, %%mx1 + %assign %%i %%i+2 + %endrep + %assign %%i 0 + %rep 3 + %assign %%j %%i+2 + punpcklqdq %%mx0, %%mx1 + %assign %%i %%i+4 + %endrep + SWAP m%1, m0 + SWAP m2, m4 + SWAP m3, m8 + SWAP m5, m12 + SWAP m6, m13 + SWAP m7, m14 + %else + PREP_8TAP_H_LOAD4 m0, %2+0, m1, m4, m7 + PREP_8TAP_H_LOAD4 m2, %2+4, m1, m4, m7 + PREP_8TAP_H_LOAD4 m3, %2+8, m1, m4, m7 + SWAP m%1, m0 + %endif + %endif +%endmacro + +%macro PREP_8TAP_H 2 ; dst, src_memloc + PREP_8TAP_H_LOAD %1, %2 + %if ARCH_X86_64 && notcpuflag(ssse3) + SWAP m8, m1 + SWAP m9, m7 + %endif + %xdefine mX m%+%1 + %assign %%i regnumof%+mX + %define mX m%+%%i + mova m4, m2 + PMADDUBSW m4, m5, m1, m7, 1 ; subpel +0 B0 + PMADDUBSW m2, m6, m1, m7, 0 ; subpel +4 B4 + PMADDUBSW m3, m6, m1, m7, 0 ; subpel +4 C4 + PMADDUBSW mX, m5, m1, m7, 0 ; subpel +0 A0 + %undef mX + %if ARCH_X86_64 && notcpuflag(ssse3) + SWAP m1, m8 + SWAP m7, m9 + %endif + paddw m3, m4 + paddw m%1, m2 + PHADDW m%1, m3, m15, ARCH_X86_32 + %if ARCH_X86_64 || cpuflag(ssse3) + PMULHRSW_8192 m%1, m%1, m7 + %else + PMULHRSW_8192 m%1, m%1, [base+pw_2] + %endif +%endmacro + +%macro PREP_8TAP_HV 4 ; dst, src_memloc, tmp[1-2] + %if cpuflag(ssse3) + movu %1, [%2] + pshufb m2, %1, shufB + pshufb m3, %1, shufC + pshufb %1, shufA + %else + PREP_8TAP_H_LOAD4 %1, %2+0, m1, %3, %4 + PREP_8TAP_H_LOAD4 m2, %2+4, m1, %3, %4 + PREP_8TAP_H_LOAD4 m3, %2+8, m1, %3, %4 + %endif + mova m1, m2 + PMADDUBSW m1, subpelh0, %3, %4, 1 ; subpel +0 C0 + PMADDUBSW m3, subpelh1, %3, %4, 0 ; subpel +4 B4 + PMADDUBSW m2, subpelh1, %3, %4, 0 ; C4 + PMADDUBSW %1, subpelh0, %3, %4, 0 ; A0 + paddw m1, m3 ; C0+B4 + paddw %1, m2 ; A0+C4 + PHADDW %1, m1, %3, 1 +%endmacro + +%macro PREP_8TAP 0 +%if ARCH_X86_32 + DECLARE_REG_TMP 1, 2 +%elif WIN64 + DECLARE_REG_TMP 6, 4 +%else + DECLARE_REG_TMP 6, 7 +%endif + +MC_8TAP_FN prep, sharp, SHARP, SHARP +MC_8TAP_FN prep, sharp_smooth, SHARP, SMOOTH +MC_8TAP_FN prep, smooth_sharp, SMOOTH, SHARP +MC_8TAP_FN prep, smooth, SMOOTH, SMOOTH +MC_8TAP_FN prep, sharp_regular, SHARP, REGULAR +MC_8TAP_FN prep, regular_sharp, REGULAR, SHARP +MC_8TAP_FN prep, smooth_regular, SMOOTH, REGULAR +MC_8TAP_FN prep, regular_smooth, REGULAR, SMOOTH +MC_8TAP_FN prep, regular, REGULAR, REGULAR + +%if ARCH_X86_32 + %define base_reg r2 + %define base base_reg-prep%+SUFFIX +%else + %define base_reg r7 + %define base 0 +%endif +cglobal prep_8tap, 1, 9, 0, tmp, src, stride, w, h, mx, my, stride3 +%assign org_stack_offset stack_offset + imul mxd, mxm, 0x010101 + add mxd, t0d ; 8tap_h, mx, 4tap_h + imul myd, mym, 0x010101 + add myd, t1d ; 8tap_v, my, 4tap_v + movsxd wq, wm + movifnidn srcd, srcm + movifnidn hd, hm + test mxd, 0xf00 + jnz .h + test myd, 0xf00 + jnz .v + LEA base_reg, prep_ssse3 + tzcnt wd, wd + movzx wd, word [base_reg-prep_ssse3+prep_ssse3_table+wq*2] + add wq, base_reg + movifnidn strided, stridem + lea r6, [strideq*3] + %assign stack_offset org_stack_offset +%if WIN64 + pop r8 + pop r7 +%endif + jmp wq +.h: + LEA base_reg, prep%+SUFFIX + test myd, 0xf00 + jnz .hv +%if cpuflag(ssse3) + WIN64_SPILL_XMM 12 +%else + WIN64_SPILL_XMM 16 +%endif +%if ARCH_X86_32 + %define strideq r6 + mov strideq, stridem +%endif + cmp wd, 4 + je .h_w4 + tzcnt wd, wd +%if cpuflag(ssse3) + %if ARCH_X86_64 + mova m10, [base+subpel_h_shufA] + mova m11, [base+subpel_h_shufB] + mova m9, [base+subpel_h_shufC] + %else + %define m10 [base+subpel_h_shufA] + %define m11 [base+subpel_h_shufB] + %define m9 [base+subpel_h_shufC] + %endif +%endif + shr mxd, 16 + sub srcq, 3 + movzx wd, word [base_reg+wq*2+table_offset(prep, _8tap_h)] + movd m5, [base_reg+mxq*8+subpel_filters-prep%+SUFFIX+0] + pshufd m5, m5, q0000 + movd m6, [base_reg+mxq*8+subpel_filters-prep%+SUFFIX+4] + pshufd m6, m6, q0000 +%if cpuflag(ssse3) + mova m7, [base+pw_8192] +%else + punpcklbw m5, m5 + punpcklbw m6, m6 + psraw m5, 8 + psraw m6, 8 + %if ARCH_X86_64 + mova m7, [pw_2] + mova m15, [pw_1] + %else + %define m15 m4 + %endif +%endif + add wq, base_reg + jmp wq +.h_w4: +%if ARCH_X86_32 + and mxd, 0x7f +%else + movzx mxd, mxb +%endif + dec srcq + movd m4, [base_reg+mxq*8+subpel_filters-prep%+SUFFIX+2] + pshufd m4, m4, q0000 +%if cpuflag(ssse3) + mova m6, [base+pw_8192] + mova m5, [base+subpel_h_shufA] +%else + mova m6, [base+pw_2] + %if ARCH_X86_64 + mova m14, [pw_1] + %else + %define m14 m7 + %endif + punpcklbw m4, m4 + psraw m4, 8 +%endif +%if ARCH_X86_64 + lea stride3q, [strideq*3] +%endif +.h_w4_loop: +%if cpuflag(ssse3) + movq m0, [srcq+strideq*0] ; 0 + movq m1, [srcq+strideq*1] ; 1 + %if ARCH_X86_32 + lea srcq, [srcq+strideq*2] + movq m2, [srcq+strideq*0] ; 2 + movq m3, [srcq+strideq*1] ; 3 + lea srcq, [srcq+strideq*2] + %else + movq m2, [srcq+strideq*2] ; 2 + movq m3, [srcq+stride3q ] ; 3 + lea srcq, [srcq+strideq*4] + %endif + pshufb m0, m5 + pshufb m1, m5 + pshufb m2, m5 + pshufb m3, m5 +%elif ARCH_X86_64 + movd m0, [srcq+strideq*0+0] + movd m12, [srcq+strideq*0+1] + movd m1, [srcq+strideq*1+0] + movd m5, [srcq+strideq*1+1] + movd m2, [srcq+strideq*2+0] + movd m13, [srcq+strideq*2+1] + movd m3, [srcq+stride3q +0] + movd m7, [srcq+stride3q +1] + punpckldq m0, m12 + punpckldq m1, m5 + punpckldq m2, m13 + punpckldq m3, m7 + movd m12, [srcq+strideq*0+2] + movd m8, [srcq+strideq*0+3] + movd m5, [srcq+strideq*1+2] + movd m9, [srcq+strideq*1+3] + movd m13, [srcq+strideq*2+2] + movd m10, [srcq+strideq*2+3] + movd m7, [srcq+stride3q +2] + movd m11, [srcq+stride3q +3] + lea srcq, [srcq+strideq*4] + punpckldq m12, m8 + punpckldq m5, m9 + punpckldq m13, m10 + punpckldq m7, m11 + punpcklqdq m0, m12 ; 0 + punpcklqdq m1, m5 ; 1 + punpcklqdq m2, m13 ; 2 + punpcklqdq m3, m7 ; 3 +%else + movd m0, [srcq+strideq*0+0] + movd m1, [srcq+strideq*0+1] + movd m2, [srcq+strideq*0+2] + movd m3, [srcq+strideq*0+3] + punpckldq m0, m1 + punpckldq m2, m3 + punpcklqdq m0, m2 ; 0 + movd m1, [srcq+strideq*1+0] + movd m2, [srcq+strideq*1+1] + movd m3, [srcq+strideq*1+2] + movd m7, [srcq+strideq*1+3] + lea srcq, [srcq+strideq*2] + punpckldq m1, m2 + punpckldq m3, m7 + punpcklqdq m1, m3 ; 1 + movd m2, [srcq+strideq*0+0] + movd m3, [srcq+strideq*0+1] + movd m7, [srcq+strideq*0+2] + movd m5, [srcq+strideq*0+3] + punpckldq m2, m3 + punpckldq m7, m5 + punpcklqdq m2, m7 ; 2 + movd m3, [srcq+strideq*1+0] + movd m7, [srcq+strideq*1+1] + punpckldq m3, m7 + movd m7, [srcq+strideq*1+2] + movd m5, [srcq+strideq*1+3] + lea srcq, [srcq+strideq*2] + punpckldq m7, m5 + punpcklqdq m3, m7 ; 3 +%endif + PMADDUBSW m0, m4, m5, m7, 1 ; subpel_filters + 2 + PMADDUBSW m1, m4, m5, m7, 0 + PMADDUBSW m2, m4, m5, m7, 0 + PMADDUBSW m3, m4, m5, m7, 0 + PHADDW m0, m1, m14, ARCH_X86_32 + PHADDW m2, m3, m14, 0 + PMULHRSW_8192 m0, m0, m6 + PMULHRSW_8192 m2, m2, m6 + mova [tmpq+16*0], m0 + mova [tmpq+16*1], m2 + add tmpq, 32 + sub hd, 4 + jg .h_w4_loop + RET +.h_w8: +%if cpuflag(ssse3) + PREP_8TAP_H 0, srcq+strideq*0 + PREP_8TAP_H 1, srcq+strideq*1 + mova [tmpq+16*0], m0 + mova [tmpq+16*1], m1 + lea srcq, [srcq+strideq*2] + add tmpq, 32 + sub hd, 2 +%else + PREP_8TAP_H 0, srcq + mova [tmpq], m0 + add srcq, strideq + add tmpq, 16 + dec hd +%endif + jg .h_w8 + RET +.h_w16: + mov r3, -16*1 + jmp .h_start +.h_w32: + mov r3, -16*2 + jmp .h_start +.h_w64: + mov r3, -16*4 + jmp .h_start +.h_w128: + mov r3, -16*8 +.h_start: + sub srcq, r3 + mov r5, r3 +.h_loop: +%if cpuflag(ssse3) + PREP_8TAP_H 0, srcq+r3+8*0 + PREP_8TAP_H 1, srcq+r3+8*1 + mova [tmpq+16*0], m0 + mova [tmpq+16*1], m1 + add tmpq, 32 + add r3, 16 +%else + PREP_8TAP_H 0, srcq+r3 + mova [tmpq], m0 + add tmpq, 16 + add r3, 8 +%endif + jl .h_loop + add srcq, strideq + mov r3, r5 + dec hd + jg .h_loop + RET +.v: + LEA base_reg, prep%+SUFFIX +%if ARCH_X86_32 + mov mxd, myd + and mxd, 0x7f +%else + %assign stack_offset org_stack_offset + WIN64_SPILL_XMM 16 + movzx mxd, myb +%endif + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + lea myq, [base_reg+myq*8+subpel_filters-prep%+SUFFIX] +%if cpuflag(ssse3) + mova m2, [base+pw_512] + psrlw m2, m2, 1 ; 0x0100 + mova m7, [base+pw_8192] +%endif +%if ARCH_X86_32 + %define subpel0 [rsp+mmsize*0] + %define subpel1 [rsp+mmsize*1] + %define subpel2 [rsp+mmsize*2] + %define subpel3 [rsp+mmsize*3] +%assign regs_used 6 ; use r5 (mx) as tmp for stack alignment if needed + %if cpuflag(ssse3) + ALLOC_STACK -mmsize*4 + %else + ALLOC_STACK -mmsize*5 + %endif +%assign regs_used 7 + movd m0, [myq+0] + PSHUFB_0X1X m0, m2 + mova subpel0, m0 + movd m0, [myq+2] + PSHUFB_0X1X m0, m2 + mova subpel1, m0 + movd m0, [myq+4] + PSHUFB_0X1X m0, m2 + mova subpel2, m0 + movd m0, [myq+6] + PSHUFB_0X1X m0, m2 + mova subpel3, m0 + mov strideq, [rstk+stack_offset+gprsize*3] + lea r5, [strideq*3] + sub srcq, r5 +%else + %define subpel0 m8 + %define subpel1 m9 + %define subpel2 m10 + %define subpel3 m11 + movd subpel0, [myq+0] + PSHUFB_0X1X subpel0, m2 + movd subpel1, [myq+2] + PSHUFB_0X1X subpel1, m2 + movd subpel2, [myq+4] + PSHUFB_0X1X subpel2, m2 + movd subpel3, [myq+6] + PSHUFB_0X1X subpel3, m2 + lea stride3q, [strideq*3] + sub srcq, stride3q + cmp wd, 8 + jns .v_w8 +%endif +.v_w4: +%if notcpuflag(ssse3) + pxor m6, m6 + %if ARCH_X86_64 + mova m7, [base+pw_2] + %endif +%endif +%if ARCH_X86_32 + %if STACK_ALIGNMENT < mmsize + %define srcm [esp+stack_size+gprsize*1] + %define tmpm [esp+stack_size+gprsize*2] + %endif + mov tmpm, tmpq + mov srcm, srcq + lea r5d, [wq - 4] ; horizontal loop + shl r5d, (16 - 2) ; (wq / 4) << 16 + mov r5w, hw +.v_w4_loop0: +%endif + movd m2, [srcq+strideq*0] ; 0 + movhps m2, [srcq+strideq*2] ; 0 _ 2 + movd m3, [srcq+strideq*1] ; 1 +%if ARCH_X86_32 + lea srcq, [srcq+strideq*2] + movhps m3, [srcq+strideq*1] ; 1 _ 3 + lea srcq, [srcq+strideq*2] +%else + movhps m3, [srcq+stride3q ] ; 1 _ 3 + lea srcq, [srcq+strideq*4] +%endif + pshufd m2, m2, q2020 ; 0 2 0 2 + pshufd m3, m3, q2020 ; 1 3 1 3 + punpckldq m2, m3 ; 0 1 2 3 + movd m3, [srcq+strideq*0] ; 4 + movd m1, [srcq+strideq*1] ; 5 + movd m0, [srcq+strideq*2] ; 6 +%if ARCH_X86_32 + lea srcq, [srcq+strideq*2] + add srcq, strideq +%else + add srcq, stride3q +%endif + punpckldq m3, m1 ; 4 5 _ _ + punpckldq m1, m0 ; 5 6 _ _ + PALIGNR m4, m3, m2, 4 ; 1 2 3 4 + punpcklbw m3, m1 ; 45 56 + punpcklbw m1, m2, m4 ; 01 12 + punpckhbw m2, m4 ; 23 34 +.v_w4_loop: +%if ARCH_X86_32 && notcpuflag(ssse3) + mova m7, subpel0 + %define subpel0 m7 +%endif + mova m5, m1 + PMADDUBSW m5, subpel0, m6, m4, 0 ; a0 b0 +%if ARCH_X86_32 && notcpuflag(ssse3) + mova m7, subpel1 + %define subpel1 m7 +%endif + mova m1, m2 + PMADDUBSW m2, subpel1, m6, m4, 0 ; a1 b1 + paddw m5, m2 +%if ARCH_X86_32 && notcpuflag(ssse3) + mova m7, subpel2 + %define subpel2 m7 +%endif + mova m2, m3 + PMADDUBSW m3, subpel2, m6, m4, 0 ; a2 b2 + paddw m5, m3 + movd m4, [srcq+strideq*0] + punpckldq m3, m0, m4 ; 6 7 _ _ + movd m0, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + punpckldq m4, m0 ; 7 8 _ _ + punpcklbw m3, m4 ; 67 78 +%if notcpuflag(ssse3) + %if ARCH_X86_64 + SWAP m12, m0 + %else + mova [esp+mmsize*4], m0 + mova m7, subpel3 + %define subpel3 m7 + %endif +%endif + mova m4, m3 + PMADDUBSW m4, subpel3, m6, m0, 0 ; a3 b3 + paddw m5, m4 +%if ARCH_X86_64 || cpuflag(ssse3) + %if notcpuflag(ssse3) + SWAP m0, m12 + %endif + PMULHRSW_8192 m5, m5, m7 +%else + mova m0, [esp+mmsize*4] + PMULHRSW_8192 m5, m5, [base+pw_2] +%endif + movq [tmpq+wq*0], m5 + movhps [tmpq+wq*2], m5 + lea tmpq, [tmpq+wq*4] + sub hd, 2 + jg .v_w4_loop +%if ARCH_X86_32 + mov hw, r5w ; reset vertical loop + mov tmpq, tmpm + mov srcq, srcm + add tmpq, 8 + add srcq, 4 + mov tmpm, tmpq + mov srcm, srcq + sub r5d, 1<<16 ; horizontal-- + jg .v_w4_loop0 +%endif + RET +%if ARCH_X86_64 +.v_w8: + lea r5d, [wq - 8] ; horizontal loop + mov r8, tmpq + mov r6, srcq + shl r5d, 8 - 3; (wq / 8) << 8 + mov r5b, hb +.v_w8_loop0: + movq m4, [srcq+strideq*0] + movq m5, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + movq m6, [srcq+strideq*0] + movq m0, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + movq m1, [srcq+strideq*0] + movq m2, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + movq m3, [srcq+strideq*0] + shufpd m4, m0, 0x0c + shufpd m5, m1, 0x0c + punpcklbw m1, m4, m5 ; 01 + punpckhbw m4, m5 ; 34 + shufpd m6, m2, 0x0c + punpcklbw m2, m5, m6 ; 12 + punpckhbw m5, m6 ; 45 + shufpd m0, m3, 0x0c + punpcklbw m3, m6, m0 ; 23 + punpckhbw m6, m0 ; 56 +.v_w8_loop: +%if cpuflag(ssse3) + movq m12, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + movq m13, [srcq+strideq*0] + pmaddubsw m14, m1, subpel0 ; a0 + pmaddubsw m15, m2, subpel0 ; b0 + mova m1, m3 + mova m2, m4 + pmaddubsw m3, subpel1 ; a1 + pmaddubsw m4, subpel1 ; b1 + paddw m14, m3 + paddw m15, m4 + mova m3, m5 + mova m4, m6 + pmaddubsw m5, subpel2 ; a2 + pmaddubsw m6, subpel2 ; b2 + paddw m14, m5 + paddw m15, m6 + shufpd m6, m0, m12, 0x0d + shufpd m0, m12, m13, 0x0c + punpcklbw m5, m6, m0 ; 67 + punpckhbw m6, m0 ; 78 + pmaddubsw m12, m5, subpel3 ; a3 + pmaddubsw m13, m6, subpel3 ; b3 + paddw m14, m12 + paddw m15, m13 + pmulhrsw m14, m7 + pmulhrsw m15, m7 + movu [tmpq+wq*0], m14 + movu [tmpq+wq*2], m15 +%else + mova m14, m1 + PMADDUBSW m14, subpel0, m7, m12, 1 ; a0 + mova m1, m3 + PMADDUBSW m3, subpel1, m7, m12, 0 ; a1 + paddw m14, m3 + mova m3, m5 + PMADDUBSW m5, subpel2, m7, m12, 0 ; a2 + paddw m14, m5 + movq m12, [srcq+strideq*1] + lea srcq, [srcq+strideq*2] + movq m13, [srcq+strideq*0] + shufpd m15, m0, m12, 0x0d + shufpd m0, m12, m13, 0x0c + punpcklbw m5, m15, m0 ; 67 + punpckhbw m15, m0 ; 78 + mova m13, m5 + PMADDUBSW m13, subpel3, m7, m12, 0 ; a3 + paddw m14, m13 + PMULHRSW_8192 m14, m14, [base+pw_2] + movu [tmpq+wq*0], m14 + mova m14, m2 + PMADDUBSW m14, subpel0, m7, m12, 0 ; b0 + mova m2, m4 + PMADDUBSW m4, subpel1, m7, m12, 0 ; b1 + paddw m14, m4 + mova m4, m6 + PMADDUBSW m6, subpel2, m7, m12, 0 ; b2 + paddw m14, m6 + mova m6, m15 + PMADDUBSW m15, subpel3, m7, m12, 0 ; b3 + paddw m14, m15 + PMULHRSW_8192 m14, m14, [base+pw_2] + movu [tmpq+wq*2], m14 +%endif + lea tmpq, [tmpq+wq*4] + sub hd, 2 + jg .v_w8_loop + movzx hd, r5b ; reset vertical loop + add r8, 16 + add r6, 8 + mov tmpq, r8 + mov srcq, r6 + sub r5d, 1<<8 ; horizontal-- + jg .v_w8_loop0 + RET +%endif ;ARCH_X86_64 +%undef subpel0 +%undef subpel1 +%undef subpel2 +%undef subpel3 + ; +.hv: + %assign stack_offset org_stack_offset + cmp wd, 4 + jg .hv_w8 + and mxd, 0x7f + movd m1, [base_reg+mxq*8+subpel_filters-prep%+SUFFIX+2] +%if ARCH_X86_32 + mov mxd, myd + shr myd, 16 + and mxd, 0x7f + cmp hd, 6 + cmovs myd, mxd + movq m0, [base_reg+myq*8+subpel_filters-prep%+SUFFIX] + mov strideq, stridem + %assign regs_used 6 + ALLOC_STACK -mmsize*14 + %assign regs_used 7 + lea r5, [strideq*3+1] + sub srcq, r5 + %define subpelv0 [rsp+mmsize*0] + %define subpelv1 [rsp+mmsize*1] + %define subpelv2 [rsp+mmsize*2] + %define subpelv3 [rsp+mmsize*3] + punpcklbw m0, m0 + psraw m0, 8 + pshufd m6, m0, q0000 + mova subpelv0, m6 + pshufd m6, m0, q1111 + mova subpelv1, m6 + pshufd m6, m0, q2222 + mova subpelv2, m6 + pshufd m6, m0, q3333 + mova subpelv3, m6 +%else + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + movq m0, [base_reg+myq*8+subpel_filters-prep%+SUFFIX] + %if cpuflag(ssse3) + ALLOC_STACK mmsize*14, 14 + %else + ALLOC_STACK mmsize*14, 16 + %endif + lea stride3q, [strideq*3] + sub srcq, stride3q + dec srcq + %define subpelv0 m10 + %define subpelv1 m11 + %define subpelv2 m12 + %define subpelv3 m13 + punpcklbw m0, m0 + psraw m0, 8 + %if cpuflag(ssse3) + mova m8, [base+pw_8192] + %else + mova m8, [base+pw_2] + %endif + mova m9, [base+pd_32] + pshufd m10, m0, q0000 + pshufd m11, m0, q1111 + pshufd m12, m0, q2222 + pshufd m13, m0, q3333 +%endif + pshufd m7, m1, q0000 +%if notcpuflag(ssse3) + punpcklbw m7, m7 + psraw m7, 8 +%endif +%define hv4_line_0_0 4 +%define hv4_line_0_1 5 +%define hv4_line_0_2 6 +%define hv4_line_0_3 7 +%define hv4_line_0_4 8 +%define hv4_line_0_5 9 +%define hv4_line_1_0 10 +%define hv4_line_1_1 11 +%define hv4_line_1_2 12 +%define hv4_line_1_3 13 +%if ARCH_X86_32 + %if cpuflag(ssse3) + %define w8192reg [base+pw_8192] + %else + %define w8192reg [base+pw_2] + %endif + %define d32reg [base+pd_32] +%else + %define w8192reg m8 + %define d32reg m9 +%endif + ; lower shuffle 0 1 2 3 4 +%if cpuflag(ssse3) + mova m6, [base+subpel_h_shuf4] +%else + %if ARCH_X86_64 + mova m15, [pw_1] + %else + %define m15 m1 + %endif +%endif + movq m5, [srcq+strideq*0] ; 0 _ _ _ + movhps m5, [srcq+strideq*1] ; 0 _ 1 _ + movq m4, [srcq+strideq*2] ; 2 _ _ _ +%if ARCH_X86_32 + lea srcq, [srcq+strideq*2] + add srcq, strideq + movhps m4, [srcq+strideq*0] ; 2 _ 3 _ + add srcq, strideq +%else + movhps m4, [srcq+stride3q ] ; 2 _ 3 _ + lea srcq, [srcq+strideq*4] +%endif + PSHUFB_SUBPEL_H_4a m2, m5, m6, m1, m3, 1 ;H subpel_h_shuf4 0~1~ + PSHUFB_SUBPEL_H_4a m0, m4, m6, m1, m3, 0 ;H subpel_h_shuf4 2~3~ + PMADDUBSW m2, m7, m1, m3, 1 ;H subpel_filters + PMADDUBSW m0, m7, m1, m3, 0 ;H subpel_filters + PHADDW m2, m0, m15, ARCH_X86_32 ;H 0 1 2 3 + PMULHRSW_8192 m2, m2, w8192reg + SAVELINE_W4 m2, 2, 0 + ; upper shuffle 2 3 4 5 6 +%if cpuflag(ssse3) + mova m6, [base+subpel_h_shuf4+16] +%endif + PSHUFB_SUBPEL_H_4b m2, m5, m6, m1, m3, 0 ;H subpel_h_shuf4 0~1~ + PSHUFB_SUBPEL_H_4b m0, m4, m6, m1, m3, 0 ;H subpel_h_shuf4 2~3~ + PMADDUBSW m2, m7, m1, m3, 1 ;H subpel_filters + PMADDUBSW m0, m7, m1, m3, 0 ;H subpel_filters + PHADDW m2, m0, m15, ARCH_X86_32 ;H 0 1 2 3 + PMULHRSW_8192 m2, m2, w8192reg +%if notcpuflag(ssse3) + %if ARCH_X86_64 + SWAP m14, m2 + %else + mova [esp+mmsize*4], m2 + %endif +%endif + ; lower shuffle +%if cpuflag(ssse3) + mova m6, [base+subpel_h_shuf4] +%endif + movq m5, [srcq+strideq*0] ; 4 _ _ _ + movhps m5, [srcq+strideq*1] ; 4 _ 5 _ + movq m4, [srcq+strideq*2] ; 6 _ _ _ + PSHUFB_SUBPEL_H_4a m3, m5, m6, m1, m2, 0 ;H subpel_h_shuf4 4~5~ + PSHUFB_SUBPEL_H_4a m0, m4, m6, m1, m2, 0 ;H subpel_h_shuf4 6~6~ + PMADDUBSW m3, m7, m1, m2, 1 ;H subpel_filters + PMADDUBSW m0, m7, m1, m2, 0 ;H subpel_filters + PHADDW m3, m0, m15, ARCH_X86_32 ;H 4 5 6 7 + PMULHRSW_8192 m3, m3, w8192reg + SAVELINE_W4 m3, 3, 0 + ; upper shuffle +%if cpuflag(ssse3) + mova m6, [base+subpel_h_shuf4+16] +%endif + PSHUFB_SUBPEL_H_4b m3, m5, m6, m1, m2, 0 ;H subpel_h_shuf4 4~5~ + PSHUFB_SUBPEL_H_4b m0, m4, m6, m1, m2, 0 ;H subpel_h_shuf4 6~6~ + PMADDUBSW m3, m7, m1, m2, 1 ;H subpel_filters + PMADDUBSW m0, m7, m1, m2, 0 ;H subpel_filters + PHADDW m3, m0, m15, ARCH_X86_32 ;H 4 5 6 7 + PMULHRSW_8192 m3, m3, w8192reg +%if notcpuflag(ssse3) + %if ARCH_X86_64 + SWAP m2, m14 + %else + mova m2, [esp+mmsize*4] + %endif +%endif +%if ARCH_X86_32 + lea srcq, [srcq+strideq*2] + add srcq, strideq +%else + add srcq, stride3q +%endif + ;process high + PALIGNR m4, m3, m2, 4;V 1 2 3 4 + punpcklwd m1, m2, m4 ; V 01 12 + punpckhwd m2, m4 ; V 23 34 + pshufd m0, m3, q2121;V 5 6 5 6 + punpcklwd m3, m0 ; V 45 56 + SAVELINE_W4 m0, 0, 1 + SAVELINE_W4 m1, 1, 1 + SAVELINE_W4 m2, 2, 1 + SAVELINE_W4 m3, 3, 1 + ;process low + RESTORELINE_W4 m2, 2, 0 + RESTORELINE_W4 m3, 3, 0 + PALIGNR m4, m3, m2, 4;V 1 2 3 4 + punpcklwd m1, m2, m4 ; V 01 12 + punpckhwd m2, m4 ; V 23 34 + pshufd m0, m3, q2121;V 5 6 5 6 + punpcklwd m3, m0 ; V 45 56 +.hv_w4_loop: + ;process low + pmaddwd m5, m1, subpelv0 ; V a0 b0 + mova m1, m2 + pmaddwd m2, subpelv1; V a1 b1 + paddd m5, m2 + mova m2, m3 + pmaddwd m3, subpelv2; V a2 b2 + paddd m5, m3 +%if notcpuflag(ssse3) + %if ARCH_X86_64 + SWAP m14, m5 + %else + mova [esp+mmsize*4], m5 + %define m15 m3 + %endif +%endif + ; +%if cpuflag(ssse3) + mova m6, [base+subpel_h_shuf4] +%endif + movq m4, [srcq+strideq*0] ; 7 + movhps m4, [srcq+strideq*1] ; 7 _ 8 _ + PSHUFB_SUBPEL_H_4a m4, m4, m6, m3, m5, 0 ; H subpel_h_shuf4 7~8~ + PMADDUBSW m4, m7, m3, m5, 1 ; H subpel_filters + PHADDW m4, m4, m15, ARCH_X86_32 ; H 7878 + PMULHRSW_8192 m4, m4, w8192reg + PALIGNR m3, m4, m0, 12, m5 ; 6787 + mova m0, m4 + punpcklwd m3, m4 ; 67 78 + pmaddwd m4, m3, subpelv3; a3 b3 +%if notcpuflag(ssse3) + %if ARCH_X86_64 + SWAP m5, m14 + %else + mova m5, [esp+mmsize*4] + %endif +%endif + paddd m5, d32reg ; pd_32 + paddd m5, m4 + psrad m5, 6 + SAVELINE_W4 m0, 0, 0 + SAVELINE_W4 m1, 1, 0 + SAVELINE_W4 m2, 2, 0 + SAVELINE_W4 m3, 3, 0 + SAVELINE_W4 m5, 5, 0 + ;process high + RESTORELINE_W4 m0, 0, 1 + RESTORELINE_W4 m1, 1, 1 + RESTORELINE_W4 m2, 2, 1 + RESTORELINE_W4 m3, 3, 1 + pmaddwd m5, m1, subpelv0; V a0 b0 + mova m1, m2 + pmaddwd m2, subpelv1; V a1 b1 + paddd m5, m2 + mova m2, m3 + pmaddwd m3, subpelv2; V a2 b2 + paddd m5, m3 +%if notcpuflag(ssse3) + %if ARCH_X86_64 + SWAP m14, m5 + %else + mova [esp+0xA0], m5 + %endif +%endif + ; +%if cpuflag(ssse3) + mova m6, [base+subpel_h_shuf4+16] +%endif + movq m4, [srcq+strideq*0] ; 7 + movhps m4, [srcq+strideq*1] ; 7 _ 8 _ + PSHUFB_SUBPEL_H_4b m4, m4, m6, m3, m5, 0 ; H subpel_h_shuf4 7~8~ + PMADDUBSW m4, m7, m3, m5, 1 ; H subpel_filters + PHADDW m4, m4, m15, ARCH_X86_32 ; H 7878 + PMULHRSW_8192 m4, m4, w8192reg + PALIGNR m3, m4, m0, 12, m5 ; 6787 + mova m0, m4 + punpcklwd m3, m4 ; 67 78 + pmaddwd m4, m3, subpelv3; a3 b3 +%if notcpuflag(ssse3) + %if ARCH_X86_64 + SWAP m5, m14 + %else + mova m5, [esp+0xA0] + %endif +%endif + paddd m5, d32reg ; pd_32 + paddd m5, m4 + psrad m4, m5, 6 + ; + RESTORELINE_W4 m5, 5, 0 + packssdw m5, m4 + pshufd m5, m5, q3120 + movu [tmpq], m5 + lea srcq, [srcq+strideq*2] + add tmpq, 16 + sub hd, 2 + SAVELINE_W4 m0, 0, 1 + SAVELINE_W4 m1, 1, 1 + SAVELINE_W4 m2, 2, 1 + SAVELINE_W4 m3, 3, 1 + RESTORELINE_W4 m0, 0, 0 + RESTORELINE_W4 m1, 1, 0 + RESTORELINE_W4 m2, 2, 0 + RESTORELINE_W4 m3, 3, 0 + jg .hv_w4_loop + RET +%undef subpelv0 +%undef subpelv1 +%undef subpelv2 +%undef subpelv3 + ; +.hv_w8: + %assign stack_offset org_stack_offset +%define hv8_line_1 0 +%define hv8_line_2 1 +%define hv8_line_3 2 +%define hv8_line_4 3 +%define hv8_line_6 4 + shr mxd, 16 +%if ARCH_X86_32 + %define subpelh0 [rsp+mmsize*5] + %define subpelh1 [rsp+mmsize*6] + %define subpelv0 [rsp+mmsize*7] + %define subpelv1 [rsp+mmsize*8] + %define subpelv2 [rsp+mmsize*9] + %define subpelv3 [rsp+mmsize*10] + %define accuv0 [rsp+mmsize*11] + %define accuv1 [rsp+mmsize*12] + movq m1, [base_reg+mxq*8+subpel_filters-prep%+SUFFIX] + mov mxd, myd + shr myd, 16 + and mxd, 0x7f + cmp hd, 6 + cmovs myd, mxd + movq m5, [base_reg+myq*8+subpel_filters-prep%+SUFFIX] + mov strideq, stridem + %assign regs_used 6 + ALLOC_STACK -mmsize*14 + %assign regs_used 7 + %if STACK_ALIGNMENT < mmsize + %define tmpm [rsp+mmsize*13+gprsize*1] + %define srcm [rsp+mmsize*13+gprsize*2] + %define stridem [rsp+mmsize*13+gprsize*3] + mov stridem, strideq + %endif + pshufd m0, m1, q0000 + pshufd m1, m1, q1111 + punpcklbw m5, m5 + %if notcpuflag(ssse3) + punpcklbw m0, m0 + punpcklbw m1, m1 + %endif + psraw m5, 8 + %if notcpuflag(ssse3) + psraw m0, 8 + psraw m1, 8 + %endif + pshufd m2, m5, q0000 + pshufd m3, m5, q1111 + pshufd m4, m5, q2222 + pshufd m5, m5, q3333 + mova subpelh0, m0 + mova subpelh1, m1 + mova subpelv0, m2 + mova subpelv1, m3 + mova subpelv2, m4 + mova subpelv3, m5 + lea r5, [strideq*3+3] + sub srcq, r5 + mov srcm, srcq +%else + ALLOC_STACK mmsize*5, 16 + %define subpelh0 m10 + %define subpelh1 m11 + %define subpelv0 m12 + %define subpelv1 m13 + %define subpelv2 m14 + %define subpelv3 m15 + %define accuv0 m8 + %define accuv1 m9 + movq m0, [base_reg+mxq*8+subpel_filters-prep%+SUFFIX] + movzx mxd, myb + shr myd, 16 + cmp hd, 6 + cmovs myd, mxd + movq m1, [base_reg+myq*8+subpel_filters-prep%+SUFFIX] + pshufd subpelh0, m0, q0000 + pshufd subpelh1, m0, q1111 + punpcklbw m1, m1 + %if notcpuflag(ssse3) + punpcklbw subpelh0, subpelh0 + punpcklbw subpelh1, subpelh1 + %endif + psraw m1, 8 + %if notcpuflag(ssse3) + psraw subpelh0, 8 + psraw subpelh1, 8 + %endif + pshufd subpelv0, m1, q0000 + pshufd subpelv1, m1, q1111 + pshufd subpelv2, m1, q2222 + pshufd subpelv3, m1, q3333 + %if notcpuflag(ssse3) + mova m7, [base+pw_2] + %endif + lea stride3q, [strideq*3] + sub srcq, 3 + sub srcq, stride3q + mov r6, srcq +%endif + lea r5d, [wq-4] +%if ARCH_X86_64 + mov r8, tmpq +%else + mov tmpm, tmpq +%endif + shl r5d, (16 - 2) + mov r5w, hw +.hv_w8_loop0: +%if cpuflag(ssse3) + %if ARCH_X86_64 + mova m7, [base+subpel_h_shufA] + mova m8, [base+subpel_h_shufB] + mova m9, [base+subpel_h_shufC] + %define shufA m7 + %define shufB m8 + %define shufC m9 + %else + %define shufA [base+subpel_h_shufA] + %define shufB [base+subpel_h_shufB] + %define shufC [base+subpel_h_shufC] + %endif +%endif + PREP_8TAP_HV m4, srcq+strideq*0, m7, m0 + PREP_8TAP_HV m5, srcq+strideq*1, m7, m0 + lea srcq, [srcq+strideq*2] +%if notcpuflag(ssse3) + %if ARCH_X86_64 + SWAP m9, m4 + %else + mova [esp], m4 + %endif +%endif + PREP_8TAP_HV m6, srcq+strideq*0, m7, m4 + PREP_8TAP_HV m0, srcq+strideq*1, m7, m4 + lea srcq, [srcq+strideq*2] +%if cpuflag(ssse3) + mova m7, [base+pw_8192] +%else + mova m7, [base+pw_2] + %if ARCH_X86_64 + SWAP m4, m9 + %else + mova m4, [esp] + %endif +%endif + PMULHRSW_8192 m4, m4, m7 + PMULHRSW_8192 m5, m5, m7 + PMULHRSW_8192 m6, m6, m7 + PMULHRSW_8192 m0, m0, m7 + punpcklwd m1, m4, m5 ; 01 + punpcklwd m2, m5, m6 ; 12 + punpcklwd m3, m6, m0 ; 23 + SAVELINE_W8 1, m1 + SAVELINE_W8 2, m2 + SAVELINE_W8 3, m3 +%if cpuflag(ssse3) + mova m7, [base+subpel_h_shufA] +%else + %if ARCH_X86_64 + SWAP m8, m7 + SWAP m9, m0 + %else + mova [esp+0x30], m0 + %endif +%endif + PREP_8TAP_HV m4, srcq+strideq*0, m7, m0 + PREP_8TAP_HV m5, srcq+strideq*1, m7, m0 + PREP_8TAP_HV m6, srcq+strideq*2, m7, m0 + lea srcq, [srcq+strideq*2] +%if cpuflag(ssse3) + mova m7, [base+pw_8192] +%else + %if ARCH_X86_64 + SWAP m0, m9 + SWAP m7, m8 + %else + mova m0, [esp+0x30] + mova m7, [base+pw_2] + %endif +%endif + PMULHRSW_8192 m1, m4, m7 + PMULHRSW_8192 m2, m5, m7 + PMULHRSW_8192 m3, m6, m7 + punpcklwd m4, m0, m1 ; 34 + punpcklwd m5, m1, m2 ; 45 + punpcklwd m6, m2, m3 ; 56 + SAVELINE_W8 6, m3 + RESTORELINE_W8 1, m1 + RESTORELINE_W8 2, m2 + RESTORELINE_W8 3, m3 +.hv_w8_loop: + SAVELINE_W8 1, m3 + SAVELINE_W8 2, m4 + SAVELINE_W8 3, m5 + SAVELINE_W8 4, m6 +%if ARCH_X86_32 + pmaddwd m0, m1, subpelv0 ; a0 + pmaddwd m7, m2, subpelv0 ; b0 + pmaddwd m3, subpelv1 ; a1 + pmaddwd m4, subpelv1 ; b1 + paddd m0, m3 + paddd m7, m4 + pmaddwd m5, subpelv2 ; a2 + pmaddwd m6, subpelv2 ; b2 + paddd m0, m5 + paddd m7, m6 + mova m5, [base+pd_32] + paddd m0, m5 + paddd m7, m5 + mova accuv0, m0 + mova accuv1, m7 +%else + pmaddwd accuv0, m1, subpelv0 ; a0 + pmaddwd accuv1, m2, subpelv0 ; b0 + pmaddwd m3, subpelv1 ; a1 + pmaddwd m4, subpelv1 ; b1 + paddd accuv0, m3 + paddd accuv1, m4 + pmaddwd m5, subpelv2 ; a2 + pmaddwd m6, subpelv2 ; b2 + paddd accuv0, m5 + paddd accuv1, m6 + mova m7, [base+pd_32] + paddd accuv0, m7 + paddd accuv1, m7 + %if cpuflag(ssse3) + mova m7, [base+subpel_h_shufB] + mova m6, [base+subpel_h_shufC] + mova m5, [base+subpel_h_shufA] + %define shufA m5 + %define shufB m7 + %define shufC m6 + %endif +%endif + PREP_8TAP_HV m0, srcq+strideq*1, m5, m6 + PREP_8TAP_HV m4, srcq+strideq*2, m5, m6 + lea srcq, [srcq+strideq*2] +%if cpuflag(ssse3) + mova m5, [base+pw_8192] +%else + mova m5, [base+pw_2] +%endif + PMULHRSW_8192 m0, m0, m5 + PMULHRSW_8192 m4, m4, m5 + RESTORELINE_W8 6, m6 + punpcklwd m5, m6, m0 ; 67 + punpcklwd m6, m0, m4 ; 78 + pmaddwd m1, m5, subpelv3 ; a3 + paddd m2, m1, accuv0 + pmaddwd m1, m6, subpelv3 ; b3 + paddd m1, m1, accuv1 + psrad m2, 6 + psrad m1, 6 + packssdw m2, m1 + movq [tmpq+wq*0], m2 + movhps [tmpq+wq*2], m2 + lea tmpq, [tmpq+wq*4] + sub hd, 2 + jle .hv_w8_outer + SAVELINE_W8 6, m4 + RESTORELINE_W8 1, m1 + RESTORELINE_W8 2, m2 + RESTORELINE_W8 3, m3 + RESTORELINE_W8 4, m4 + jmp .hv_w8_loop +.hv_w8_outer: + movzx hd, r5w +%if ARCH_X86_32 + mov srcq, srcm + mov tmpq, tmpm + add srcq, 4 + add tmpq, 8 + mov srcm, srcq + mov tmpm, tmpq +%else + add r8, 8 + mov tmpq, r8 + add r6, 4 + mov srcq, r6 +%endif + sub r5d, 1<<16 + jg .hv_w8_loop0 + RET +%endmacro + +%if ARCH_X86_32 + %macro SAVE_ALPHA_BETA 0 + mov alpham, alphad + mov betam, betad + %endmacro + + %macro SAVE_DELTA_GAMMA 0 + mov deltam, deltad + mov gammam, gammad + %endmacro + + %macro LOAD_ALPHA_BETA_MX 0 + mov mym, myd + mov alphad, alpham + mov betad, betam + mov mxd, mxm + %endmacro + + %macro LOAD_DELTA_GAMMA_MY 0 + mov mxm, mxd + mov deltad, deltam + mov gammad, gammam + mov myd, mym + %endmacro + + %define PIC_reg r2 + %define PIC_base_offset $$ + %define PIC_sym(sym) (PIC_reg+(sym)-PIC_base_offset) +%else + %define SAVE_ALPHA_BETA + %define SAVE_DELTA_GAMMA + %define PIC_sym(sym) sym +%endif + +%if ARCH_X86_32 + %if STACK_ALIGNMENT < required_stack_alignment + %assign copy_args 8*4 + %else + %assign copy_args 0 + %endif +%endif + +%macro RELOC_ARGS 0 + %if copy_args + mov r0, r0m + mov r1, r1m + mov r2, r2m + mov r3, r3m + mov r5, r5m + mov dstm, r0 + mov dsm, r1 + mov srcm, r2 + mov ssm, r3 + mov mxm, r5 + mov r0, r6m + mov mym, r0 + %endif +%endmacro + +%macro BLENDHWDW 2 ; blend high words from dwords, src1, src2 + %if cpuflag(sse4) + pblendw %1, %2, 0xAA + %else + pand %2, m10 + por %1, %2 + %endif +%endmacro + +%macro WARP_V 10 ; dst0, dst1, 0, 2, 4, 6, 1, 3, 5, 7 + %if ARCH_X86_32 + %define m8 m4 + %define m9 m5 + %define m14 m6 + %define m15 m7 + %define m11 m7 + %endif + %if notcpuflag(ssse3) || ARCH_X86_32 + pxor m11, m11 + %endif + lea tmp1d, [myq+deltaq*4] + lea tmp2d, [myq+deltaq*1] + shr myd, 10 + shr tmp1d, 10 + movq m2, [filterq+myq *8] ; a + movq m8, [filterq+tmp1q*8] ; e + lea tmp1d, [tmp2q+deltaq*4] + lea myd, [tmp2q+deltaq*1] + shr tmp2d, 10 + shr tmp1d, 10 + movq m3, [filterq+tmp2q*8] ; b + movq m0, [filterq+tmp1q*8] ; f + punpcklwd m2, m3 + punpcklwd m8, m0 + lea tmp1d, [myq+deltaq*4] + lea tmp2d, [myq+deltaq*1] + shr myd, 10 + shr tmp1d, 10 + movq m0, [filterq+myq *8] ; c + movq m9, [filterq+tmp1q*8] ; g + lea tmp1d, [tmp2q+deltaq*4] + lea myd, [tmp2q+gammaq] ; my += gamma + shr tmp2d, 10 + shr tmp1d, 10 + movq m3, [filterq+tmp2q*8] ; d + movq m1, [filterq+tmp1q*8] ; h + punpcklwd m0, m3 + punpcklwd m9, m1 + punpckldq m1, m2, m0 + punpckhdq m2, m0 + punpcklbw m0, m11, m1 ; a0 a2 b0 b2 c0 c2 d0 d2 << 8 + punpckhbw m3, m11, m1 ; a4 a6 b4 b6 c4 c6 d4 d6 << 8 + punpcklbw m1, m11, m2 ; a1 a3 b1 b3 c1 c3 d1 d3 << 8 + punpckhbw m14, m11, m2 ; a5 a7 b5 b7 c5 c7 d5 d7 << 8 + pmaddwd m0, %3 + pmaddwd m3, %5 + pmaddwd m1, %7 + pmaddwd m14, %9 + paddd m0, m3 + paddd m1, m14 + paddd m0, m1 + mova %1, m0 + %if ARCH_X86_64 + SWAP m3, m14 + %endif + punpckldq m0, m8, m9 + punpckhdq m8, m9 + punpcklbw m1, m11, m0 ; e0 e2 f0 f2 g0 g2 h0 h2 << 8 + punpckhbw m14, m11, m0 ; e4 e6 f4 f6 g4 g6 h4 h6 << 8 + punpcklbw m2, m11, m8 ; e1 e3 f1 f3 g1 g3 h1 h3 << 8 + punpckhbw m15, m11, m8 ; e5 e7 f5 f7 g5 g7 h5 h7 << 8 + pmaddwd m1, %4 + pmaddwd m14, %6 + pmaddwd m2, %8 + pmaddwd m15, %10 + paddd m1, m14 + paddd m2, m15 + paddd m1, m2 + mova %2, m1 + %if ARCH_X86_64 + SWAP m14, m3 + %endif +%endmacro + +%if ARCH_X86_64 + %define counterd r4d +%else + %if copy_args == 0 + %define counterd dword r4m + %else + %define counterd dword [esp+stack_size-4*7] + %endif +%endif + +%macro WARP_AFFINE_8X8T 0 +%if ARCH_X86_64 +cglobal warp_affine_8x8t, 6, 14, 16, 0x90, tmp, ts +%else +cglobal warp_affine_8x8t, 0, 7, 16, -0x130-copy_args, tmp, ts + %if copy_args + %define tmpm [esp+stack_size-4*1] + %define tsm [esp+stack_size-4*2] + %endif +%endif + call mangle(private_prefix %+ _warp_affine_8x8_%+cpuname).main +.loop: +%if ARCH_X86_32 + %define m12 m4 + %define m13 m5 + %define m14 m6 + %define m15 m7 + mova m12, [esp+0xC0] + mova m13, [esp+0xD0] + mova m14, [esp+0xE0] + mova m15, [esp+0xF0] +%endif +%if cpuflag(ssse3) + psrad m12, 13 + psrad m13, 13 + psrad m14, 13 + psrad m15, 13 + packssdw m12, m13 + packssdw m14, m15 + mova m13, [PIC_sym(pw_8192)] + pmulhrsw m12, m13 ; (x + (1 << 6)) >> 7 + pmulhrsw m14, m13 +%else + %if ARCH_X86_32 + %define m10 m0 + %endif + mova m10, [PIC_sym(pd_16384)] + paddd m12, m10 + paddd m13, m10 + paddd m14, m10 + paddd m15, m10 + psrad m12, 15 + psrad m13, 15 + psrad m14, 15 + psrad m15, 15 + packssdw m12, m13 + packssdw m14, m15 +%endif + mova [tmpq+tsq*0], m12 + mova [tmpq+tsq*2], m14 + dec counterd + jz mangle(private_prefix %+ _warp_affine_8x8_%+cpuname).end +%if ARCH_X86_32 + mov tmpm, tmpd + mov r0, [esp+0x100] + mov r1, [esp+0x104] +%endif + call mangle(private_prefix %+ _warp_affine_8x8_%+cpuname).main2 + lea tmpq, [tmpq+tsq*4] + jmp .loop +%endmacro + +%macro WARP_AFFINE_8X8 0 +%if ARCH_X86_64 +cglobal warp_affine_8x8, 6, 14, 16, 0x90, \ + dst, ds, src, ss, abcd, mx, tmp2, alpha, beta, \ + filter, tmp1, delta, my, gamma +%else +cglobal warp_affine_8x8, 0, 7, 16, -0x130-copy_args, \ + dst, ds, src, ss, abcd, mx, tmp2, alpha, beta, \ + filter, tmp1, delta, my, gamma + %define alphaq r0 + %define alphad r0 + %define alpham [esp+gprsize+0x100] + %define betaq r1 + %define betad r1 + %define betam [esp+gprsize+0x104] + %define deltaq r0 + %define deltad r0 + %define deltam [esp+gprsize+0x108] + %define gammaq r1 + %define gammad r1 + %define gammam [esp+gprsize+0x10C] + %define filterq r3 + %define tmp1q r4 + %define tmp1d r4 + %define tmp1m [esp+gprsize+0x110] + %define myq r5 + %define myd r5 + %define mym r6m + %if copy_args + %define dstm [esp+stack_size-4*1] + %define dsm [esp+stack_size-4*2] + %define srcm [esp+stack_size-4*3] + %define ssm [esp+stack_size-4*4] + %define mxm [esp+stack_size-4*5] + %define mym [esp+stack_size-4*6] + %endif +%endif + call .main + jmp .start +.loop: +%if ARCH_X86_32 + mov dstm, dstd + mov alphad, [esp+0x100] + mov betad, [esp+0x104] +%endif + call .main2 + lea dstq, [dstq+dsq*2] +.start: +%if notcpuflag(sse4) + %if cpuflag(ssse3) + %define roundval pw_8192 + %else + %define roundval pd_262144 + %endif + %if ARCH_X86_64 + mova m10, [PIC_sym(roundval)] + %else + %define m10 [PIC_sym(roundval)] + %endif +%endif +%if ARCH_X86_32 + %define m12 m5 + %define m13 m6 + mova m12, [esp+0xC0] + mova m13, [esp+0xD0] +%endif +%if cpuflag(sse4) + %if ARCH_X86_32 + %define m11 m4 + pxor m11, m11 + %endif + psrad m12, 18 + psrad m13, 18 + packusdw m12, m13 + pavgw m12, m11 ; (x + (1 << 10)) >> 11 +%else + %if cpuflag(ssse3) + psrad m12, 17 + psrad m13, 17 + packssdw m12, m13 + pmulhrsw m12, m10 + %else + paddd m12, m10 + paddd m13, m10 + psrad m12, 19 + psrad m13, 19 + packssdw m12, m13 + %endif +%endif +%if ARCH_X86_32 + %define m14 m6 + %define m15 m7 + mova m14, [esp+0xE0] + mova m15, [esp+0xF0] +%endif +%if cpuflag(sse4) + psrad m14, 18 + psrad m15, 18 + packusdw m14, m15 + pavgw m14, m11 ; (x + (1 << 10)) >> 11 +%else + %if cpuflag(ssse3) + psrad m14, 17 + psrad m15, 17 + packssdw m14, m15 + pmulhrsw m14, m10 + %else + paddd m14, m10 + paddd m15, m10 + psrad m14, 19 + psrad m15, 19 + packssdw m14, m15 + %endif +%endif + packuswb m12, m14 + movq [dstq+dsq*0], m12 + movhps [dstq+dsq*1], m12 + dec counterd + jg .loop +.end: + RET +ALIGN function_align +.main: +%assign stack_offset stack_offset+gprsize +%if ARCH_X86_32 + %assign stack_size stack_size+4 + %if copy_args + %assign stack_offset stack_offset-4 + %endif + RELOC_ARGS + LEA PIC_reg, $$ + %define PIC_mem [esp+gprsize+0x114] + mov abcdd, abcdm + %if copy_args == 0 + mov ssd, ssm + mov mxd, mxm + %endif + mov PIC_mem, PIC_reg + mov srcd, srcm +%endif + movsx deltad, word [abcdq+2*2] + movsx gammad, word [abcdq+2*3] + lea tmp1d, [deltaq*3] + sub gammad, tmp1d ; gamma -= delta*3 + SAVE_DELTA_GAMMA +%if ARCH_X86_32 + mov abcdd, abcdm +%endif + movsx alphad, word [abcdq+2*0] + movsx betad, word [abcdq+2*1] + lea tmp1q, [ssq*3+3] + add mxd, 512+(64<<10) + lea tmp2d, [alphaq*3] + sub srcq, tmp1q ; src -= src_stride*3 + 3 +%if ARCH_X86_32 + mov srcm, srcd + mov PIC_reg, PIC_mem +%endif + sub betad, tmp2d ; beta -= alpha*3 + lea filterq, [PIC_sym(mc_warp_filter)] +%if ARCH_X86_64 + mov myd, r6m + %if cpuflag(ssse3) + pxor m11, m11 + %endif +%endif + call .h + psrld m2, m0, 16 + psrld m3, m1, 16 +%if ARCH_X86_32 + %if notcpuflag(ssse3) + mova [esp+gprsize+0x00], m2 + %endif + mova [esp+gprsize+0x10], m3 +%endif + call .h + psrld m4, m0, 16 + psrld m5, m1, 16 +%if ARCH_X86_32 + mova [esp+gprsize+0x20], m4 + mova [esp+gprsize+0x30], m5 +%endif + call .h +%if ARCH_X86_64 + %define blendmask [rsp+gprsize+0x80] +%else + %if notcpuflag(ssse3) + mova m2, [esp+gprsize+0x00] + %endif + mova m3, [esp+gprsize+0x10] + %define blendmask [esp+gprsize+0x120] + %define m10 m7 +%endif + pcmpeqd m10, m10 + pslld m10, 16 + mova blendmask, m10 + BLENDHWDW m2, m0 ; 0 + BLENDHWDW m3, m1 ; 2 + mova [rsp+gprsize+0x00], m2 + mova [rsp+gprsize+0x10], m3 + call .h +%if ARCH_X86_32 + mova m4, [esp+gprsize+0x20] + mova m5, [esp+gprsize+0x30] +%endif + mova m10, blendmask + BLENDHWDW m4, m0 ; 1 + BLENDHWDW m5, m1 ; 3 + mova [rsp+gprsize+0x20], m4 + mova [rsp+gprsize+0x30], m5 + call .h +%if ARCH_X86_32 + %if notcpuflag(ssse3) + mova m2, [esp+gprsize+0x00] + %endif + mova m3, [esp+gprsize+0x10] + %define m10 m5 +%endif + psrld m6, m2, 16 + psrld m7, m3, 16 + mova m10, blendmask + BLENDHWDW m6, m0 ; 2 + BLENDHWDW m7, m1 ; 4 + mova [rsp+gprsize+0x40], m6 + mova [rsp+gprsize+0x50], m7 + call .h +%if ARCH_X86_32 + mova m4, [esp+gprsize+0x20] + mova m5, [esp+gprsize+0x30] +%endif + psrld m2, m4, 16 + psrld m3, m5, 16 + mova m10, blendmask + BLENDHWDW m2, m0 ; 3 + BLENDHWDW m3, m1 ; 5 + mova [rsp+gprsize+0x60], m2 + mova [rsp+gprsize+0x70], m3 + call .h +%if ARCH_X86_32 + mova m6, [esp+gprsize+0x40] + mova m7, [esp+gprsize+0x50] + %define m10 m7 +%endif + psrld m4, m6, 16 + psrld m5, m7, 16 + mova m10, blendmask + BLENDHWDW m4, m0 ; 4 + BLENDHWDW m5, m1 ; 6 +%if ARCH_X86_64 + add myd, 512+(64<<10) + mova m6, m2 + mova m7, m3 +%else + mova [esp+gprsize+0x80], m4 + mova [esp+gprsize+0x90], m5 + add dword mym, 512+(64<<10) +%endif + mov counterd, 4 + SAVE_ALPHA_BETA +.main2: + call .h +%if ARCH_X86_32 + mova m6, [esp+gprsize+0x60] + mova m7, [esp+gprsize+0x70] + %define m10 m5 +%endif + psrld m6, 16 + psrld m7, 16 + mova m10, blendmask + BLENDHWDW m6, m0 ; 5 + BLENDHWDW m7, m1 ; 7 +%if ARCH_X86_64 + WARP_V m12, m13, [rsp+gprsize+0x00], [rsp+gprsize+0x10], \ + m4, m5, \ + [rsp+gprsize+0x20], [rsp+gprsize+0x30], \ + m6, m7 +%else + mova [esp+gprsize+0xA0], m6 + mova [esp+gprsize+0xB0], m7 + LOAD_DELTA_GAMMA_MY + WARP_V [esp+gprsize+0xC0], [esp+gprsize+0xD0], \ + [esp+gprsize+0x00], [esp+gprsize+0x10], \ + [esp+gprsize+0x80], [esp+gprsize+0x90], \ + [esp+gprsize+0x20], [esp+gprsize+0x30], \ + [esp+gprsize+0xA0], [esp+gprsize+0xB0] + LOAD_ALPHA_BETA_MX +%endif + call .h + mova m2, [rsp+gprsize+0x40] + mova m3, [rsp+gprsize+0x50] +%if ARCH_X86_32 + mova m4, [rsp+gprsize+0x80] + mova m5, [rsp+gprsize+0x90] + %define m10 m7 +%endif + mova [rsp+gprsize+0x00], m2 + mova [rsp+gprsize+0x10], m3 + mova [rsp+gprsize+0x40], m4 + mova [rsp+gprsize+0x50], m5 + psrld m4, 16 + psrld m5, 16 + mova m10, blendmask + BLENDHWDW m4, m0 ; 6 + BLENDHWDW m5, m1 ; 8 +%if ARCH_X86_64 + WARP_V m14, m15, [rsp+gprsize+0x20], [rsp+gprsize+0x30], \ + m6, m7, \ + [rsp+gprsize+0x00], [rsp+gprsize+0x10], \ + m4, m5 +%else + mova [esp+gprsize+0x80], m4 + mova [esp+gprsize+0x90], m5 + LOAD_DELTA_GAMMA_MY + WARP_V [esp+gprsize+0xE0], [esp+gprsize+0xF0], \ + [esp+gprsize+0x20], [esp+gprsize+0x30], \ + [esp+gprsize+0xA0], [esp+gprsize+0xB0], \ + [esp+gprsize+0x00], [esp+gprsize+0x10], \ + [esp+gprsize+0x80], [esp+gprsize+0x90] + mov mym, myd + mov dstd, dstm + mov dsd, dsm + mov mxd, mxm +%endif + mova m2, [rsp+gprsize+0x60] + mova m3, [rsp+gprsize+0x70] +%if ARCH_X86_32 + mova m6, [esp+gprsize+0xA0] + mova m7, [esp+gprsize+0xB0] +%endif + mova [rsp+gprsize+0x20], m2 + mova [rsp+gprsize+0x30], m3 + mova [rsp+gprsize+0x60], m6 + mova [rsp+gprsize+0x70], m7 + ret +ALIGN function_align +.h: +%if ARCH_X86_32 + %define m8 m3 + %define m9 m4 + %define m10 m5 + %define m14 m6 + %define m15 m7 +%endif + lea tmp1d, [mxq+alphaq*4] + lea tmp2d, [mxq+alphaq*1] +%if ARCH_X86_32 + %assign stack_offset stack_offset+4 + %assign stack_size stack_size+4 + %define PIC_mem [esp+gprsize*2+0x114] + mov PIC_mem, PIC_reg + mov srcd, srcm +%endif + movu m10, [srcq] +%if ARCH_X86_32 + add srcd, ssm + mov srcm, srcd + mov PIC_reg, PIC_mem +%else + add srcq, ssq +%endif + shr mxd, 10 + shr tmp1d, 10 + movq m1, [filterq+mxq *8] ; 0 X + movq m8, [filterq+tmp1q*8] ; 4 X + lea tmp1d, [tmp2q+alphaq*4] + lea mxd, [tmp2q+alphaq*1] + shr tmp2d, 10 + shr tmp1d, 10 + movhps m1, [filterq+tmp2q*8] ; 0 1 + movhps m8, [filterq+tmp1q*8] ; 4 5 + lea tmp1d, [mxq+alphaq*4] + lea tmp2d, [mxq+alphaq*1] + shr mxd, 10 + shr tmp1d, 10 +%if cpuflag(ssse3) + movq m14, [filterq+mxq *8] ; 2 X + movq m9, [filterq+tmp1q*8] ; 6 X + lea tmp1d, [tmp2q+alphaq*4] + lea mxd, [tmp2q+betaq] ; mx += beta + shr tmp2d, 10 + shr tmp1d, 10 + movhps m14, [filterq+tmp2q*8] ; 2 3 + movhps m9, [filterq+tmp1q*8] ; 6 7 + pshufb m0, m10, [PIC_sym(warp_8x8_shufA)] + pmaddubsw m0, m1 + pshufb m1, m10, [PIC_sym(warp_8x8_shufB)] + pmaddubsw m1, m8 + pshufb m15, m10, [PIC_sym(warp_8x8_shufC)] + pmaddubsw m15, m14 + pshufb m10, m10, [PIC_sym(warp_8x8_shufD)] + pmaddubsw m10, m9 + phaddw m0, m15 + phaddw m1, m10 +%else + %if ARCH_X86_32 + %define m11 m2 + %endif + pcmpeqw m0, m0 + psrlw m14, m0, 8 + psrlw m15, m10, 8 ; 01 03 05 07 09 11 13 15 + pand m14, m10 ; 00 02 04 06 08 10 12 14 + packuswb m14, m15 ; 00 02 04 06 08 10 12 14 01 03 05 07 09 11 13 15 + psrldq m9, m0, 4 + pshufd m0, m14, q0220 + pand m0, m9 + psrldq m14, 1 ; 02 04 06 08 10 12 14 01 03 05 07 09 11 13 15 __ + pslldq m15, m14, 12 + por m0, m15 ; shufA + psrlw m15, m0, 8 + psraw m11, m1, 8 + psllw m0, 8 + psllw m1, 8 + psrlw m0, 8 + psraw m1, 8 + pmullw m15, m11 + pmullw m0, m1 + paddw m0, m15 ; pmaddubsw m0, m1 + pshufd m15, m14, q0220 + pand m15, m9 + psrldq m14, 1 ; 04 06 08 10 12 14 01 03 05 07 09 11 13 15 __ __ + pslldq m1, m14, 12 + por m15, m1 ; shufC + pshufd m1, m14, q0220 + pand m1, m9 + psrldq m14, 1 ; 06 08 10 12 14 01 03 05 07 09 11 13 15 __ __ __ + pslldq m11, m14, 12 + por m1, m11 ; shufB + pshufd m10, m14, q0220 + pand m10, m9 + psrldq m14, 1 ; 08 10 12 14 01 03 05 07 09 11 13 15 __ __ __ __ + pslldq m14, m14, 12 + por m10, m14 ; shufD + psrlw m9, m1, 8 + psraw m11, m8, 8 + psllw m1, 8 + psllw m8, 8 + psrlw m1, 8 + psraw m8, 8 + pmullw m9, m11 + pmullw m1, m8 + paddw m1, m9 ; pmaddubsw m1, m8 + movq m14, [filterq+mxq *8] ; 2 X + movq m9, [filterq+tmp1q*8] ; 6 X + lea tmp1d, [tmp2q+alphaq*4] + lea mxd, [tmp2q+betaq] ; mx += beta + shr tmp2d, 10 + shr tmp1d, 10 + movhps m14, [filterq+tmp2q*8] ; 2 3 + movhps m9, [filterq+tmp1q*8] ; 6 7 + psrlw m8, m15, 8 + psraw m11, m14, 8 + psllw m15, 8 + psllw m14, 8 + psrlw m15, 8 + psraw m14, 8 + pmullw m8, m11 + pmullw m15, m14 + paddw m15, m8 ; pmaddubsw m15, m14 + psrlw m8, m10, 8 + psraw m11, m9, 8 + psllw m10, 8 + psllw m9, 8 + psrlw m10, 8 + psraw m9, 8 + pmullw m8, m11 + pmullw m10, m9 + paddw m10, m8 ; pmaddubsw m10, m9 + pslld m8, m0, 16 + pslld m9, m1, 16 + pslld m14, m15, 16 + pslld m11, m10, 16 + paddw m0, m8 + paddw m1, m9 + paddw m15, m14 + paddw m10, m11 + psrad m0, 16 + psrad m1, 16 + psrad m15, 16 + psrad m10, 16 + packssdw m0, m15 ; phaddw m0, m15 + packssdw m1, m10 ; phaddw m1, m10 +%endif + mova m14, [PIC_sym(pw_8192)] + mova m9, [PIC_sym(pd_32768)] + pmaddwd m0, m14 ; 17-bit intermediate, upshifted by 13 + pmaddwd m1, m14 + paddd m0, m9 ; rounded 14-bit result in upper 16 bits of dword + paddd m1, m9 + ret +%endmacro + +%if WIN64 +DECLARE_REG_TMP 6, 4 +%else +DECLARE_REG_TMP 6, 7 +%endif + +%macro BIDIR_FN 1 ; op + %1 0 + lea stride3q, [strideq*3] + jmp wq +.w4_loop: + %1_INC_PTR 2 + %1 0 + lea dstq, [dstq+strideq*4] +.w4: ; tile 4x + movd [dstq ], m0 ; copy dw[0] + pshuflw m1, m0, q1032 ; swap dw[1] and dw[0] + movd [dstq+strideq*1], m1 ; copy dw[1] + punpckhqdq m0, m0 ; swap dw[3,2] with dw[1,0] + movd [dstq+strideq*2], m0 ; dw[2] + psrlq m0, 32 ; shift right in dw[3] + movd [dstq+stride3q ], m0 ; copy + sub hd, 4 + jg .w4_loop + RET +.w8_loop: + %1_INC_PTR 2 + %1 0 + lea dstq, [dstq+strideq*2] +.w8: + movq [dstq ], m0 + movhps [dstq+strideq*1], m0 + sub hd, 2 + jg .w8_loop + RET +.w16_loop: + %1_INC_PTR 2 + %1 0 + lea dstq, [dstq+strideq] +.w16: + mova [dstq ], m0 + dec hd + jg .w16_loop + RET +.w32_loop: + %1_INC_PTR 4 + %1 0 + lea dstq, [dstq+strideq] +.w32: + mova [dstq ], m0 + %1 2 + mova [dstq + 16 ], m0 + dec hd + jg .w32_loop + RET +.w64_loop: + %1_INC_PTR 8 + %1 0 + add dstq, strideq +.w64: + %assign i 0 + %rep 4 + mova [dstq + i*16 ], m0 + %assign i i+1 + %if i < 4 + %1 2*i + %endif + %endrep + dec hd + jg .w64_loop + RET +.w128_loop: + %1_INC_PTR 16 + %1 0 + add dstq, strideq +.w128: + %assign i 0 + %rep 8 + mova [dstq + i*16 ], m0 + %assign i i+1 + %if i < 8 + %1 2*i + %endif + %endrep + dec hd + jg .w128_loop + RET +%endmacro + +%macro AVG 1 ; src_offset + ; writes AVG of tmp1 tmp2 uint16 coeffs into uint8 pixel + mova m0, [tmp1q+(%1+0)*mmsize] ; load 8 coef(2bytes) from tmp1 + paddw m0, [tmp2q+(%1+0)*mmsize] ; load/add 8 coef(2bytes) tmp2 + mova m1, [tmp1q+(%1+1)*mmsize] + paddw m1, [tmp2q+(%1+1)*mmsize] + pmulhrsw m0, m2 + pmulhrsw m1, m2 + packuswb m0, m1 ; pack/trunc 16 bits from m0 & m1 to 8 bit +%endmacro + +%macro AVG_INC_PTR 1 + add tmp1q, %1*mmsize + add tmp2q, %1*mmsize +%endmacro + +cglobal avg, 4, 7, 3, dst, stride, tmp1, tmp2, w, h, stride3 + LEA r6, avg_ssse3_table + tzcnt wd, wm ; leading zeros + movifnidn hd, hm ; move h(stack) to h(register) if not already that register + movsxd wq, dword [r6+wq*4] ; push table entry matching the tile width (tzcnt) in widen reg + mova m2, [pw_1024+r6-avg_ssse3_table] ; fill m2 with shift/align + add wq, r6 + BIDIR_FN AVG + +%macro W_AVG 1 ; src_offset + ; (a * weight + b * (16 - weight) + 128) >> 8 + ; = ((a - b) * weight + (b << 4) + 128) >> 8 + ; = ((((a - b) * ((weight-16) << 12)) >> 16) + a + 8) >> 4 + ; = ((((b - a) * (-weight << 12)) >> 16) + b + 8) >> 4 + mova m2, [tmp1q+(%1+0)*mmsize] + mova m0, m2 + psubw m2, [tmp2q+(%1+0)*mmsize] + mova m3, [tmp1q+(%1+1)*mmsize] + mova m1, m3 + psubw m3, [tmp2q+(%1+1)*mmsize] + pmulhw m2, m4 + pmulhw m3, m4 + paddw m0, m2 + paddw m1, m3 + pmulhrsw m0, m5 + pmulhrsw m1, m5 + packuswb m0, m1 +%endmacro + +%define W_AVG_INC_PTR AVG_INC_PTR + +cglobal w_avg, 4, 7, 6, dst, stride, tmp1, tmp2, w, h, stride3 + LEA r6, w_avg_ssse3_table + tzcnt wd, wm + movd m4, r6m + movifnidn hd, hm + pxor m0, m0 + movsxd wq, dword [r6+wq*4] + mova m5, [pw_2048+r6-w_avg_ssse3_table] + pshufb m4, m0 + psllw m4, 12 ; (weight-16) << 12 when interpreted as signed + add wq, r6 + cmp dword r6m, 7 + jg .weight_gt7 + mov r6, tmp1q + psubw m0, m4 + mov tmp1q, tmp2q + mova m4, m0 ; -weight + mov tmp2q, r6 +.weight_gt7: + BIDIR_FN W_AVG + +%macro MASK 1 ; src_offset + ; (a * m + b * (64 - m) + 512) >> 10 + ; = ((a - b) * m + (b << 6) + 512) >> 10 + ; = ((((b - a) * (-m << 10)) >> 16) + b + 8) >> 4 + mova m3, [maskq+(%1+0)*(mmsize/2)] + mova m0, [tmp2q+(%1+0)*mmsize] ; b + psubw m1, m0, [tmp1q+(%1+0)*mmsize] ; b - a + mova m6, m3 ; m + psubb m3, m4, m6 ; -m + paddw m1, m1 ; (b - a) << 1 + paddb m3, m3 ; -m << 1 + punpcklbw m2, m4, m3 ; -m << 9 (<< 8 when ext as uint16) + pmulhw m1, m2 ; (-m * (b - a)) << 10 + paddw m0, m1 ; + b + mova m1, [tmp2q+(%1+1)*mmsize] ; b + psubw m2, m1, [tmp1q+(%1+1)*mmsize] ; b - a + paddw m2, m2 ; (b - a) << 1 + mova m6, m3 ; (-m << 1) + punpckhbw m3, m4, m6 ; (-m << 9) + pmulhw m2, m3 ; (-m << 9) + paddw m1, m2 ; (-m * (b - a)) << 10 + pmulhrsw m0, m5 ; round + pmulhrsw m1, m5 ; round + packuswb m0, m1 ; interleave 16 -> 8 +%endmacro + +%macro MASK_INC_PTR 1 + add maskq, %1*mmsize/2 + add tmp1q, %1*mmsize + add tmp2q, %1*mmsize +%endmacro + +%if ARCH_X86_64 +cglobal mask, 4, 8, 7, dst, stride, tmp1, tmp2, w, h, mask, stride3 + movifnidn hd, hm +%else +cglobal mask, 4, 7, 7, dst, stride, tmp1, tmp2, w, mask, stride3 +%define hd dword r5m +%endif +%define base r6-mask_ssse3_table + LEA r6, mask_ssse3_table + tzcnt wd, wm + movsxd wq, dword [r6+wq*4] + pxor m4, m4 + mova m5, [base+pw_2048] + add wq, r6 + mov maskq, r6m + BIDIR_FN MASK +%undef hd + +%macro W_MASK_420_B 2 ; src_offset in bytes, mask_out + ;**** do m0 = u16.dst[7..0], m%2 = u16.m[7..0] **** + mova m0, [tmp1q+(%1)] + mova m1, [tmp2q+(%1)] + mova m2, reg_pw_6903 + psubw m1, m0 + pabsw m%2, m1 ; abs(tmp1 - tmp2) + mova m3, m2 + psubusw m2, m%2 + psrlw m2, 8 ; 64 - m + mova m%2, m2 + psllw m2, 10 + pmulhw m1, m2 ; tmp2 * () + paddw m0, m1 ; tmp1 + () + ;**** do m1 = u16.dst[7..0], m%2 = u16.m[7..0] **** + mova m1, [tmp1q+(%1)+mmsize] + mova m2, [tmp2q+(%1)+mmsize] + psubw m2, m1 + pabsw m7, m2 ; abs(tmp1 - tmp2) + psubusw m3, m7 + psrlw m3, 8 ; 64 - m + phaddw m%2, m3 ; pack both u16.m[8..0]runs as u8.m [15..0] + psllw m3, 10 + pmulhw m2, m3 +%if ARCH_X86_32 + mova reg_pw_2048, [base+pw_2048] +%endif + paddw m1, m2 + pmulhrsw m0, reg_pw_2048 ; round/scale 2048 + pmulhrsw m1, reg_pw_2048 ; round/scale 2048 + packuswb m0, m1 ; concat m0 = u8.dst[15..0] +%endmacro + +%macro W_MASK_420 2 + W_MASK_420_B (%1*16), %2 +%endmacro + +%define base r6-w_mask_420_ssse3_table +%if ARCH_X86_64 +%define reg_pw_6903 m8 +%define reg_pw_2048 m9 +; args: dst, stride, tmp1, tmp2, w, h, mask, sign +cglobal w_mask_420, 4, 8, 10, dst, stride, tmp1, tmp2, w, h, mask + lea r6, [w_mask_420_ssse3_table] + mov wd, wm + tzcnt r7d, wd + movd m0, r7m ; sign + movifnidn hd, hm + movsxd r7, [r6+r7*4] + mova reg_pw_6903, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 + mova reg_pw_2048, [base+pw_2048] + movd m6, [base+pw_258] ; 64 * 4 + 2 + add r7, r6 + mov maskq, maskmp + psubw m6, m0 + pshuflw m6, m6, q0000 + punpcklqdq m6, m6 + W_MASK_420 0, 4 + jmp r7 + %define loop_w r7d +%else +%define reg_pw_6903 [base+pw_6903] +%define reg_pw_2048 m3 +cglobal w_mask_420, 4, 7, 8, dst, stride, tmp1, tmp2, w, mask + tzcnt wd, wm + LEA r6, w_mask_420_ssse3_table + movd m0, r7m ; sign + mov maskq, r6mp + mov wd, [r6+wq*4] + movd m6, [base+pw_258] + add wq, r6 + psubw m6, m0 + pshuflw m6, m6, q0000 + punpcklqdq m6, m6 + W_MASK_420 0, 4 + jmp wd + %define loop_w dword r0m + %define hd dword r5m +%endif +.w4_loop: + add tmp1q, 2*16 + add tmp2q, 2*16 + W_MASK_420 0, 4 + lea dstq, [dstq+strideq*2] + add maskq, 4 +.w4: + movd [dstq ], m0 ; copy m0[0] + pshuflw m1, m0, q1032 + movd [dstq+strideq*1], m1 ; copy m0[1] + lea dstq, [dstq+strideq*2] + punpckhqdq m0, m0 + movd [dstq+strideq*0], m0 ; copy m0[2] + psrlq m0, 32 + movd [dstq+strideq*1], m0 ; copy m0[3] + psubw m1, m6, m4 ; a _ c _ + psrlq m4, 32 ; b _ d _ + psubw m1, m4 + psrlw m1, 2 + packuswb m1, m1 + pshuflw m1, m1, q2020 + movd [maskq], m1 + sub hd, 4 + jg .w4_loop + RET +.w8_loop: + add tmp1q, 2*16 + add tmp2q, 2*16 + W_MASK_420 0, 4 + lea dstq, [dstq+strideq*2] + add maskq, 4 +.w8: + movq [dstq ], m0 + movhps [dstq+strideq*1], m0 + psubw m0, m6, m4 + punpckhqdq m4, m4 + psubw m0, m4 + psrlw m0, 2 + packuswb m0, m0 + movd [maskq], m0 + sub hd, 2 + jg .w8_loop + RET +.w16: ; w32/64/128 +%if ARCH_X86_32 + mov wd, wm ; because we altered it in 32bit setup +%endif + mov loop_w, wd ; use width as counter + jmp .w16ge_inner_loop_first +.w16ge_loop: + lea tmp1q, [tmp1q+wq*2] ; skip even line pixels + lea tmp2q, [tmp2q+wq*2] ; skip even line pixels + sub dstq, wq + mov loop_w, wd + lea dstq, [dstq+strideq*2] +.w16ge_inner_loop: + W_MASK_420_B 0, 4 +.w16ge_inner_loop_first: + mova [dstq ], m0 + W_MASK_420_B wq*2, 5 ; load matching even line (offset = widthpx * (16+16)) + mova [dstq+strideq*1], m0 + psubw m1, m6, m4 ; m9 == 64 * 4 + 2 + psubw m1, m5 ; - odd line mask + psrlw m1, 2 ; >> 2 + packuswb m1, m1 + movq [maskq], m1 + add tmp1q, 2*16 + add tmp2q, 2*16 + add maskq, 8 + add dstq, 16 + sub loop_w, 16 + jg .w16ge_inner_loop + sub hd, 2 + jg .w16ge_loop + RET + +%undef reg_pw_6903 +%undef reg_pw_2048 +%undef dst_bak +%undef loop_w +%undef orig_w +%undef hd + +%macro BLEND_64M 4; a, b, mask1, mask2 + punpcklbw m0, %1, %2; {b;a}[7..0] + punpckhbw %1, %2 ; {b;a}[15..8] + pmaddubsw m0, %3 ; {b*m[0] + (64-m[0])*a}[7..0] u16 + pmaddubsw %1, %4 ; {b*m[1] + (64-m[1])*a}[15..8] u16 + pmulhrsw m0, m5 ; {((b*m[0] + (64-m[0])*a) + 1) / 32}[7..0] u16 + pmulhrsw %1, m5 ; {((b*m[1] + (64-m[0])*a) + 1) / 32}[15..8] u16 + packuswb m0, %1 ; {blendpx}[15..0] u8 +%endmacro + +%macro BLEND 2; a, b + psubb m3, m4, m0 ; m3 = (64 - m) + punpcklbw m2, m3, m0 ; {m;(64-m)}[7..0] + punpckhbw m3, m0 ; {m;(64-m)}[15..8] + BLEND_64M %1, %2, m2, m3 +%endmacro + +cglobal blend, 3, 7, 7, dst, ds, tmp, w, h, mask +%define base r6-blend_ssse3_table + LEA r6, blend_ssse3_table + tzcnt wd, wm + movifnidn hd, hm + movifnidn maskq, maskmp + movsxd wq, dword [r6+wq*4] + mova m4, [base+pb_64] + mova m5, [base+pw_512] + add wq, r6 + lea r6, [dsq*3] + jmp wq +.w4: + movq m0, [maskq]; m + movd m1, [dstq+dsq*0] ; a + movd m6, [dstq+dsq*1] + punpckldq m1, m6 + movq m6, [tmpq] ; b + psubb m3, m4, m0 ; m3 = (64 - m) + punpcklbw m2, m3, m0 ; {m;(64-m)}[7..0] + punpcklbw m1, m6 ; {b;a}[7..0] + pmaddubsw m1, m2 ; {b*m[0] + (64-m[0])*a}[7..0] u16 + pmulhrsw m1, m5 ; {((b*m[0] + (64-m[0])*a) + 1) / 32}[7..0] u16 + packuswb m1, m0 ; {blendpx}[15..0] u8 + movd [dstq+dsq*0], m1 + psrlq m1, 32 + movd [dstq+dsq*1], m1 + add maskq, 8 + add tmpq, 8 + lea dstq, [dstq+dsq*2] ; dst_stride * 2 + sub hd, 2 + jg .w4 + RET +.w8: + mova m0, [maskq]; m + movq m1, [dstq+dsq*0] ; a + movhps m1, [dstq+dsq*1] + mova m6, [tmpq] ; b + BLEND m1, m6 + movq [dstq+dsq*0], m0 + movhps [dstq+dsq*1], m0 + add maskq, 16 + add tmpq, 16 + lea dstq, [dstq+dsq*2] ; dst_stride * 2 + sub hd, 2 + jg .w8 + RET +.w16: + mova m0, [maskq]; m + mova m1, [dstq] ; a + mova m6, [tmpq] ; b + BLEND m1, m6 + mova [dstq], m0 + add maskq, 16 + add tmpq, 16 + add dstq, dsq ; dst_stride + dec hd + jg .w16 + RET +.w32: + %assign i 0 + %rep 2 + mova m0, [maskq+16*i]; m + mova m1, [dstq+16*i] ; a + mova m6, [tmpq+16*i] ; b + BLEND m1, m6 + mova [dstq+i*16], m0 + %assign i i+1 + %endrep + add maskq, 32 + add tmpq, 32 + add dstq, dsq ; dst_stride + dec hd + jg .w32 + RET + +cglobal blend_v, 3, 6, 6, dst, ds, tmp, w, h, mask +%define base r5-blend_v_ssse3_table + LEA r5, blend_v_ssse3_table + tzcnt wd, wm + movifnidn hd, hm + movsxd wq, dword [r5+wq*4] + mova m5, [base+pw_512] + add wq, r5 + add maskq, obmc_masks-blend_v_ssse3_table + jmp wq +.w2: + movd m3, [maskq+4] + punpckldq m3, m3 + ; 2 mask blend is provided for 4 pixels / 2 lines +.w2_loop: + movd m1, [dstq+dsq*0] ; a {..;a;a} + pinsrw m1, [dstq+dsq*1], 1 + movd m2, [tmpq] ; b + punpcklbw m0, m1, m2; {b;a}[7..0] + pmaddubsw m0, m3 ; {b*m + (64-m)*a}[7..0] u16 + pmulhrsw m0, m5 ; {((b*m + (64-m)*a) + 1) / 32}[7..0] u16 + packuswb m0, m1 ; {blendpx}[8..0] u8 + movd r3d, m0 + mov [dstq+dsq*0], r3w + shr r3d, 16 + mov [dstq+dsq*1], r3w + add tmpq, 2*2 + lea dstq, [dstq + dsq * 2] + sub hd, 2 + jg .w2_loop + RET +.w4: + movddup m3, [maskq+8] + ; 4 mask blend is provided for 8 pixels / 2 lines +.w4_loop: + movd m1, [dstq+dsq*0] ; a + movd m2, [dstq+dsq*1] ; + punpckldq m1, m2 + movq m2, [tmpq] ; b + punpcklbw m1, m2 ; {b;a}[7..0] + pmaddubsw m1, m3 ; {b*m + (64-m)*a}[7..0] u16 + pmulhrsw m1, m5 ; {((b*m + (64-m)*a) + 1) / 32}[7..0] u16 + packuswb m1, m1 ; {blendpx}[8..0] u8 + movd [dstq], m1 + psrlq m1, 32 + movd [dstq+dsq*1], m1 + add tmpq, 2*4 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w4_loop + RET +.w8: + mova m3, [maskq+16] + ; 8 mask blend is provided for 16 pixels +.w8_loop: + movq m1, [dstq+dsq*0] ; a + movhps m1, [dstq+dsq*1] + mova m2, [tmpq]; b + BLEND_64M m1, m2, m3, m3 + movq [dstq+dsq*0], m0 + movhps [dstq+dsq*1], m0 + add tmpq, 16 + lea dstq, [dstq+dsq*2] + sub hd, 2 + jg .w8_loop + RET +.w16: + ; 16 mask blend is provided for 32 pixels + mova m3, [maskq+32] ; obmc_masks_16[0] (64-m[0]) + mova m4, [maskq+48] ; obmc_masks_16[1] (64-m[1]) +.w16_loop: + mova m1, [dstq] ; a + mova m2, [tmpq] ; b + BLEND_64M m1, m2, m3, m4 + mova [dstq], m0 + add tmpq, 16 + add dstq, dsq + dec hd + jg .w16_loop + RET +.w32: +%if WIN64 + mova [rsp+8], xmm6 +%endif + mova m3, [maskq+64] ; obmc_masks_32[0] (64-m[0]) + mova m4, [maskq+80] ; obmc_masks_32[1] (64-m[1]) + mova m6, [maskq+96] ; obmc_masks_32[2] (64-m[2]) + ; 16 mask blend is provided for 64 pixels +.w32_loop: + mova m1, [dstq+16*0] ; a + mova m2, [tmpq+16*0] ; b + BLEND_64M m1, m2, m3, m4 + movq m1, [dstq+16*1] ; a + punpcklbw m1, [tmpq+16*1] ; b + pmaddubsw m1, m6 + pmulhrsw m1, m5 + packuswb m1, m1 + mova [dstq+16*0], m0 + movq [dstq+16*1], m1 + add tmpq, 32 + add dstq, dsq + dec hd + jg .w32_loop +%if WIN64 + mova xmm6, [rsp+8] +%endif + RET + +cglobal blend_h, 3, 7, 6, dst, ds, tmp, w, h, mask +%define base t0-blend_h_ssse3_table +%if ARCH_X86_32 + ; We need to keep the PIC pointer for w4, reload wd from stack instead + DECLARE_REG_TMP 6 +%else + DECLARE_REG_TMP 5 + mov r6d, wd +%endif + LEA t0, blend_h_ssse3_table + tzcnt wd, wm + mov hd, hm + movsxd wq, dword [t0+wq*4] + mova m5, [base+pw_512] + add wq, t0 + lea maskq, [base+obmc_masks+hq*2] + lea hd, [hq*3] + shr hd, 2 ; h * 3/4 + lea maskq, [maskq+hq*2] + neg hq + jmp wq +.w2: + movd m0, [dstq+dsq*0] + pinsrw m0, [dstq+dsq*1], 1 + movd m2, [maskq+hq*2] + movd m1, [tmpq] + punpcklwd m2, m2 + punpcklbw m0, m1 + pmaddubsw m0, m2 + pmulhrsw m0, m5 + packuswb m0, m0 + movd r3d, m0 + mov [dstq+dsq*0], r3w + shr r3d, 16 + mov [dstq+dsq*1], r3w + lea dstq, [dstq+dsq*2] + add tmpq, 2*2 + add hq, 2 + jl .w2 + RET +.w4: +%if ARCH_X86_32 + mova m3, [base+blend_shuf] +%else + mova m3, [blend_shuf] +%endif +.w4_loop: + movd m0, [dstq+dsq*0] + movd m2, [dstq+dsq*1] + punpckldq m0, m2 ; a + movq m1, [tmpq] ; b + movq m2, [maskq+hq*2] ; m + pshufb m2, m3 + punpcklbw m0, m1 + pmaddubsw m0, m2 + pmulhrsw m0, m5 + packuswb m0, m0 + movd [dstq+dsq*0], m0 + psrlq m0, 32 + movd [dstq+dsq*1], m0 + lea dstq, [dstq+dsq*2] + add tmpq, 4*2 + add hq, 2 + jl .w4_loop + RET +.w8: + movd m4, [maskq+hq*2] + punpcklwd m4, m4 + pshufd m3, m4, q0000 + pshufd m4, m4, q1111 + movq m1, [dstq+dsq*0] ; a + movhps m1, [dstq+dsq*1] + mova m2, [tmpq] + BLEND_64M m1, m2, m3, m4 + movq [dstq+dsq*0], m0 + movhps [dstq+dsq*1], m0 + lea dstq, [dstq+dsq*2] + add tmpq, 8*2 + add hq, 2 + jl .w8 + RET +; w16/w32/w64/w128 +.w16: +%if ARCH_X86_32 + mov r6d, wm +%endif + sub dsq, r6 +.w16_loop0: + movd m3, [maskq+hq*2] + pshuflw m3, m3, q0000 + punpcklqdq m3, m3 + mov wd, r6d +.w16_loop: + mova m1, [dstq] ; a + mova m2, [tmpq] ; b + BLEND_64M m1, m2, m3, m3 + mova [dstq], m0 + add dstq, 16 + add tmpq, 16 + sub wd, 16 + jg .w16_loop + add dstq, dsq + inc hq + jl .w16_loop0 + RET + +; emu_edge args: +; const intptr_t bw, const intptr_t bh, const intptr_t iw, const intptr_t ih, +; const intptr_t x, const intptr_t y, pixel *dst, const ptrdiff_t dst_stride, +; const pixel *ref, const ptrdiff_t ref_stride +; +; bw, bh total filled size +; iw, ih, copied block -> fill bottom, right +; x, y, offset in bw/bh -> fill top, left +cglobal emu_edge, 10, 13, 2, bw, bh, iw, ih, x, \ + y, dst, dstride, src, sstride, \ + bottomext, rightext, blk + ; we assume that the buffer (stride) is larger than width, so we can + ; safely overwrite by a few bytes + pxor m1, m1 + +%if ARCH_X86_64 + %define reg_zero r12q + %define reg_tmp r10 + %define reg_src srcq + %define reg_bottomext bottomextq + %define reg_rightext rightextq + %define reg_blkm r9m +%else + %define reg_zero r6 + %define reg_tmp r0 + %define reg_src r1 + %define reg_bottomext r0 + %define reg_rightext r1 + %define reg_blkm r2m +%endif + ; + ; ref += iclip(y, 0, ih - 1) * PXSTRIDE(ref_stride) + xor reg_zero, reg_zero + lea reg_tmp, [ihq-1] + cmp yq, ihq + cmovs reg_tmp, yq + test yq, yq + cmovs reg_tmp, reg_zero +%if ARCH_X86_64 + imul reg_tmp, sstrideq + add srcq, reg_tmp +%else + imul reg_tmp, sstridem + mov reg_src, srcm + add reg_src, reg_tmp +%endif + ; + ; ref += iclip(x, 0, iw - 1) + lea reg_tmp, [iwq-1] + cmp xq, iwq + cmovs reg_tmp, xq + test xq, xq + cmovs reg_tmp, reg_zero + add reg_src, reg_tmp +%if ARCH_X86_32 + mov srcm, reg_src +%endif + ; + ; bottom_ext = iclip(y + bh - ih, 0, bh - 1) +%if ARCH_X86_32 + mov r1, r1m ; restore bh +%endif + lea reg_bottomext, [yq+bhq] + sub reg_bottomext, ihq + lea r3, [bhq-1] + cmovs reg_bottomext, reg_zero + ; + + DEFINE_ARGS bw, bh, iw, ih, x, \ + topext, dst, dstride, src, sstride, \ + bottomext, rightext, blk + + ; top_ext = iclip(-y, 0, bh - 1) + neg topextq + cmovs topextq, reg_zero + cmp reg_bottomext, bhq + cmovns reg_bottomext, r3 + cmp topextq, bhq + cmovg topextq, r3 + %if ARCH_X86_32 + mov r4m, reg_bottomext + ; + ; right_ext = iclip(x + bw - iw, 0, bw - 1) + mov r0, r0m ; restore bw + %endif + lea reg_rightext, [xq+bwq] + sub reg_rightext, iwq + lea r2, [bwq-1] + cmovs reg_rightext, reg_zero + + DEFINE_ARGS bw, bh, iw, ih, leftext, \ + topext, dst, dstride, src, sstride, \ + bottomext, rightext, blk + + ; left_ext = iclip(-x, 0, bw - 1) + neg leftextq + cmovs leftextq, reg_zero + cmp reg_rightext, bwq + cmovns reg_rightext, r2 + %if ARCH_X86_32 + mov r3m, r1 + %endif + cmp leftextq, bwq + cmovns leftextq, r2 + +%undef reg_zero +%undef reg_tmp +%undef reg_src +%undef reg_bottomext +%undef reg_rightext + + DEFINE_ARGS bw, centerh, centerw, dummy, leftext, \ + topext, dst, dstride, src, sstride, \ + bottomext, rightext, blk + + ; center_h = bh - top_ext - bottom_ext +%if ARCH_X86_64 + lea r3, [bottomextq+topextq] + sub centerhq, r3 +%else + mov r1, centerhm ; restore r1 + sub centerhq, topextq + sub centerhq, r4m + mov r1m, centerhq +%endif + ; + ; blk += top_ext * PXSTRIDE(dst_stride) + mov r2, topextq +%if ARCH_X86_64 + imul r2, dstrideq +%else + mov r6, r6m ; restore dstq + imul r2, dstridem +%endif + add dstq, r2 + mov reg_blkm, dstq ; save pointer for ext + ; + ; center_w = bw - left_ext - right_ext + mov centerwq, bwq +%if ARCH_X86_64 + lea r3, [rightextq+leftextq] + sub centerwq, r3 +%else + sub centerwq, r3m + sub centerwq, leftextq +%endif + +; vloop Macro +%macro v_loop 3 ; need_left_ext, need_right_ext, suffix + %if ARCH_X86_64 + %define reg_tmp r12 + %else + %define reg_tmp r0 + %endif +.v_loop_%3: + %if ARCH_X86_32 + mov r0, r0m + mov r1, r1m + %endif +%if %1 + ; left extension + %if ARCH_X86_64 + movd m0, [srcq] + %else + mov r3, srcm + movd m0, [r3] + %endif + pshufb m0, m1 + xor r3, r3 +.left_loop_%3: + mova [dstq+r3], m0 + add r3, mmsize + cmp r3, leftextq + jl .left_loop_%3 + ; body + lea reg_tmp, [dstq+leftextq] +%endif + xor r3, r3 +.body_loop_%3: + %if ARCH_X86_64 + movu m0, [srcq+r3] + %else + mov r1, srcm + movu m0, [r1+r3] + %endif +%if %1 + movu [reg_tmp+r3], m0 +%else + movu [dstq+r3], m0 +%endif + add r3, mmsize + cmp r3, centerwq + jl .body_loop_%3 +%if %2 + ; right extension +%if %1 + add reg_tmp, centerwq +%else + lea reg_tmp, [dstq+centerwq] +%endif + %if ARCH_X86_64 + movd m0, [srcq+centerwq-1] + %else + mov r3, srcm + movd m0, [r3+centerwq-1] + %endif + pshufb m0, m1 + xor r3, r3 +.right_loop_%3: + movu [reg_tmp+r3], m0 + add r3, mmsize + %if ARCH_X86_64 + cmp r3, rightextq + %else + cmp r3, r3m + %endif + jl .right_loop_%3 +%endif + %if ARCH_X86_64 + add dstq, dstrideq + add srcq, sstrideq + dec centerhq + jg .v_loop_%3 + %else + add dstq, dstridem + mov r0, sstridem + add srcm, r0 + sub dword centerhm, 1 + jg .v_loop_%3 + mov r0, r0m ; restore r0 + %endif +%endmacro ; vloop MACRO + + test leftextq, leftextq + jnz .need_left_ext + %if ARCH_X86_64 + test rightextq, rightextq + jnz .need_right_ext + %else + cmp leftextq, r3m ; leftextq == 0 + jne .need_right_ext + %endif + v_loop 0, 0, 0 + jmp .body_done + + ;left right extensions +.need_left_ext: + %if ARCH_X86_64 + test rightextq, rightextq + %else + mov r3, r3m + test r3, r3 + %endif + jnz .need_left_right_ext + v_loop 1, 0, 1 + jmp .body_done + +.need_left_right_ext: + v_loop 1, 1, 2 + jmp .body_done + +.need_right_ext: + v_loop 0, 1, 3 + +.body_done: +; r0 ; bw +; r1 ;; x loop +; r4 ;; y loop +; r5 ; topextq +; r6 ;dstq +; r7 ;dstrideq +; r8 ; srcq +%if ARCH_X86_64 + %define reg_dstride dstrideq +%else + %define reg_dstride r2 +%endif + ; + ; bottom edge extension + %if ARCH_X86_64 + test bottomextq, bottomextq + jz .top + %else + xor r1, r1 + cmp r1, r4m + je .top + %endif + ; + %if ARCH_X86_64 + mov srcq, dstq + sub srcq, dstrideq + xor r1, r1 + %else + mov r3, dstq + mov reg_dstride, dstridem + sub r3, reg_dstride + mov srcm, r3 + %endif + ; +.bottom_x_loop: + %if ARCH_X86_64 + mova m0, [srcq+r1] + lea r3, [dstq+r1] + mov r4, bottomextq + %else + mov r3, srcm + mova m0, [r3+r1] + lea r3, [dstq+r1] + mov r4, r4m + %endif + ; +.bottom_y_loop: + mova [r3], m0 + add r3, reg_dstride + dec r4 + jg .bottom_y_loop + add r1, mmsize + cmp r1, bwq + jl .bottom_x_loop + +.top: + ; top edge extension + test topextq, topextq + jz .end +%if ARCH_X86_64 + mov srcq, reg_blkm +%else + mov r3, reg_blkm + mov reg_dstride, dstridem +%endif + mov dstq, dstm + xor r1, r1 + ; +.top_x_loop: +%if ARCH_X86_64 + mova m0, [srcq+r1] +%else + mov r3, reg_blkm + mova m0, [r3+r1] +%endif + lea r3, [dstq+r1] + mov r4, topextq + ; +.top_y_loop: + mova [r3], m0 + add r3, reg_dstride + dec r4 + jg .top_y_loop + add r1, mmsize + cmp r1, bwq + jl .top_x_loop + +.end: + RET + +%undef reg_dstride +%undef reg_blkm +%undef reg_tmp + +cextern resize_filter + +%macro SCRATCH 3 +%if ARCH_X86_32 + mova [rsp+%3*mmsize], m%1 +%define m%2 [rsp+%3*mmsize] +%else + SWAP %1, %2 +%endif +%endmacro + +%if ARCH_X86_64 +cglobal resize, 0, 14, 16, dst, dst_stride, src, src_stride, \ + dst_w, h, src_w, dx, mx0 +%elif STACK_ALIGNMENT >= 16 +cglobal resize, 0, 7, 8, 3 * 16, dst, dst_stride, src, src_stride, \ + dst_w, h, src_w, dx, mx0 +%else +cglobal resize, 0, 6, 8, 3 * 16, dst, dst_stride, src, src_stride, \ + dst_w, h, src_w, dx, mx0 +%endif + movifnidn dstq, dstmp + movifnidn srcq, srcmp +%if STACK_ALIGNMENT >= 16 + movifnidn dst_wd, dst_wm +%endif +%if ARCH_X86_64 + movifnidn hd, hm +%endif + sub dword mx0m, 4<<14 + sub dword src_wm, 8 + movd m7, dxm + movd m6, mx0m + movd m5, src_wm + pshufd m7, m7, q0000 + pshufd m6, m6, q0000 + pshufd m5, m5, q0000 + +%if ARCH_X86_64 + DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, h, x, picptr + LEA r7, $$ +%define base r7-$$ +%else + DEFINE_ARGS dst, dst_stride, src, src_stride, dst_w, x +%if STACK_ALIGNMENT >= 16 + LEA r6, $$ +%define base r6-$$ +%else + LEA r4, $$ +%define base r4-$$ +%endif +%endif + +%if ARCH_X86_64 + mova m12, [base+pw_m256] + mova m11, [base+pd_63] + mova m10, [base+pb_8x0_8x8] +%else +%define m12 [base+pw_m256] +%define m11 [base+pd_63] +%define m10 [base+pb_8x0_8x8] +%endif + pmaddwd m4, m7, [base+resize_mul] ; dx*[0,1,2,3] + pslld m7, 2 ; dx*4 + pslld m5, 14 + paddd m6, m4 ; mx+[0..3]*dx + SCRATCH 7, 15, 0 + SCRATCH 6, 14, 1 + SCRATCH 5, 13, 2 + + ; m2 = 0, m3 = pmulhrsw constant for x=(x+64)>>7 + ; m8 = mx+[0..3]*dx, m5 = dx*4, m6 = src_w, m7 = 0x3f, m15=0,8 + +.loop_y: + xor xd, xd + mova m0, m14 ; per-line working version of mx + +.loop_x: + pxor m1, m1 + pcmpgtd m1, m0 + pandn m1, m0 + psrad m2, m0, 8 ; filter offset (unmasked) + pcmpgtd m3, m13, m1 + pand m1, m3 + pandn m3, m13 + por m1, m3 + psubd m3, m0, m1 ; pshufb offset + psrad m1, 14 ; clipped src_x offset + psrad m3, 14 ; pshufb edge_emu offset + pand m2, m11 ; filter offset (masked) + + ; load source pixels +%if ARCH_X86_64 + movd r8d, xm1 + pshuflw xm1, xm1, q3232 + movd r9d, xm1 + punpckhqdq xm1, xm1 + movd r10d, xm1 + psrlq xm1, 32 + movd r11d, xm1 + movq xm4, [srcq+r8] + movq xm5, [srcq+r10] + movhps xm4, [srcq+r9] + movhps xm5, [srcq+r11] +%else + movd r3d, xm1 + pshufd xm1, xm1, q3312 + movd r1d, xm1 + pshuflw xm1, xm1, q3232 + movq xm4, [srcq+r3] + movq xm5, [srcq+r1] + movd r3d, xm1 + punpckhqdq xm1, xm1 + movd r1d, xm1 + movhps xm4, [srcq+r3] + movhps xm5, [srcq+r1] +%endif + + ; if no emulation is required, we don't need to shuffle or emulate edges + ; this also saves 2 quasi-vpgatherdqs + pxor m6, m6 + pcmpeqb m6, m3 +%if ARCH_X86_64 + pmovmskb r8d, m6 + cmp r8d, 0xffff +%else + pmovmskb r3d, m6 + cmp r3d, 0xffff +%endif + je .filter + +%if ARCH_X86_64 + movd r8d, xm3 + pshuflw xm3, xm3, q3232 + movd r9d, xm3 + punpckhqdq xm3, xm3 + movd r10d, xm3 + psrlq xm3, 32 + movd r11d, xm3 + movsxd r8, r8d + movsxd r9, r9d + movsxd r10, r10d + movsxd r11, r11d + movq xm6, [base+resize_shuf+4+r8] + movq xm7, [base+resize_shuf+4+r10] + movhps xm6, [base+resize_shuf+4+r9] + movhps xm7, [base+resize_shuf+4+r11] +%else + movd r3d, xm3 + pshufd xm3, xm3, q3312 + movd r1d, xm3 + pshuflw xm3, xm3, q3232 + movq xm6, [base+resize_shuf+4+r3] + movq xm7, [base+resize_shuf+4+r1] + movd r3d, xm3 + punpckhqdq xm3, xm3 + movd r1d, xm3 + movhps xm6, [base+resize_shuf+4+r3] + movhps xm7, [base+resize_shuf+4+r1] +%endif + + paddb m6, m10 + paddb m7, m10 + pshufb m4, m6 + pshufb m5, m7 + +.filter: +%if ARCH_X86_64 + movd r8d, xm2 + pshuflw xm2, xm2, q3232 + movd r9d, xm2 + punpckhqdq xm2, xm2 + movd r10d, xm2 + psrlq xm2, 32 + movd r11d, xm2 + movq xm6, [base+resize_filter+r8*8] + movq xm7, [base+resize_filter+r10*8] + movhps xm6, [base+resize_filter+r9*8] + movhps xm7, [base+resize_filter+r11*8] +%else + movd r3d, xm2 + pshufd xm2, xm2, q3312 + movd r1d, xm2 + pshuflw xm2, xm2, q3232 + movq xm6, [base+resize_filter+r3*8] + movq xm7, [base+resize_filter+r1*8] + movd r3d, xm2 + punpckhqdq xm2, xm2 + movd r1d, xm2 + movhps xm6, [base+resize_filter+r3*8] + movhps xm7, [base+resize_filter+r1*8] +%endif + + pmaddubsw m4, m6 + pmaddubsw m5, m7 + phaddw m4, m5 + phaddsw m4, m4 + pmulhrsw m4, m12 ; x=(x+64)>>7 + packuswb m4, m4 + movd [dstq+xq], m4 + + paddd m0, m15 + add xd, 4 +%if STACK_ALIGNMENT >= 16 + cmp xd, dst_wd +%else + cmp xd, dst_wm +%endif + jl .loop_x + +%if ARCH_X86_64 + add dstq, dst_strideq + add srcq, src_strideq + dec hd +%else + add dstq, dst_stridem + add srcq, src_stridem + dec dword r5m +%endif + jg .loop_y + RET + +INIT_XMM ssse3 +PREP_BILIN +PREP_8TAP +WARP_AFFINE_8X8 +WARP_AFFINE_8X8T + +INIT_XMM sse4 +WARP_AFFINE_8X8 +WARP_AFFINE_8X8T + +INIT_XMM sse2 +PREP_BILIN +PREP_8TAP +WARP_AFFINE_8X8 +WARP_AFFINE_8X8T diff --git a/third_party/dav1d/src/x86/mc_ssse3.asm b/third_party/dav1d/src/x86/mc_ssse3.asm deleted file mode 100644 --- a/third_party/dav1d/src/x86/mc_ssse3.asm +++ /dev/null @@ -1,5083 +0,0 @@ -; Copyright © 2018, VideoLAN and dav1d authors -; Copyright © 2018, Two Orioles, LLC -; Copyright © 2018, VideoLabs -; All rights reserved. -; -; Redistribution and use in source and binary forms, with or without -; modification, are permitted provided that the following conditions are met: -; -; 1. Redistributions of source code must retain the above copyright notice, this -; list of conditions and the following disclaimer. -; -; 2. Redistributions in binary form must reproduce the above copyright notice, -; this list of conditions and the following disclaimer in the documentation -; and/or other materials provided with the distribution. -; -; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -%include "ext/x86/x86inc.asm" - -SECTION_RODATA 16 - -; dav1d_obmc_masks[] with 64-x interleaved -obmc_masks: db 0, 0, 0, 0 - ; 2 @4 - db 45, 19, 64, 0 - ; 4 @8 - db 39, 25, 50, 14, 59, 5, 64, 0 - ; 8 @16 - db 36, 28, 42, 22, 48, 16, 53, 11, 57, 7, 61, 3, 64, 0, 64, 0 - ; 16 @32 - db 34, 30, 37, 27, 40, 24, 43, 21, 46, 18, 49, 15, 52, 12, 54, 10 - db 56, 8, 58, 6, 60, 4, 61, 3, 64, 0, 64, 0, 64, 0, 64, 0 - ; 32 @64 - db 33, 31, 35, 29, 36, 28, 38, 26, 40, 24, 41, 23, 43, 21, 44, 20 - db 45, 19, 47, 17, 48, 16, 50, 14, 51, 13, 52, 12, 53, 11, 55, 9 - db 56, 8, 57, 7, 58, 6, 59, 5, 60, 4, 60, 4, 61, 3, 62, 2 - -warp_8x8_shufA: db 0, 2, 4, 6, 1, 3, 5, 7, 1, 3, 5, 7, 2, 4, 6, 8 -warp_8x8_shufB: db 4, 6, 8, 10, 5, 7, 9, 11, 5, 7, 9, 11, 6, 8, 10, 12 -warp_8x8_shufC: db 2, 4, 6, 8, 3, 5, 7, 9, 3, 5, 7, 9, 4, 6, 8, 10 -warp_8x8_shufD: db 6, 8, 10, 12, 7, 9, 11, 13, 7, 9, 11, 13, 8, 10, 12, 14 -blend_shuf: db 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3 -subpel_h_shuf4: db 0, 1, 2, 3, 1, 2, 3, 4, 8, 9, 10, 11, 9, 10, 11, 12 - db 2, 3, 4, 5, 3, 4, 5, 6, 10, 11, 12, 13, 11, 12, 13, 14 -subpel_h_shufA: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 -subpel_h_shufB: db 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 -subpel_h_shufC: db 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 -bilin_h_shuf4: db 1, 0, 2, 1, 3, 2, 4, 3, 9, 8, 10, 9, 11, 10, 12, 11 -bilin_h_shuf8: db 1, 0, 2, 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7 - -pb_64: times 16 db 64 -pw_8: times 8 dw 8 -pw_26: times 8 dw 26 -pw_34: times 8 dw 34 -pw_512: times 8 dw 512 -pw_1024: times 8 dw 1024 -pw_2048: times 8 dw 2048 -pw_6903: times 8 dw 6903 -pw_8192: times 8 dw 8192 -pd_32: times 4 dd 32 -pd_512: times 4 dd 512 -pd_16384: times 4 dd 16484 -pd_32768: times 4 dd 32768 -pd_262144:times 4 dd 262144 - -pw_258: times 2 dw 258 - -cextern mc_subpel_filters -%define subpel_filters (mangle(private_prefix %+ _mc_subpel_filters)-8) - -%macro BIDIR_JMP_TABLE 1-* - ;evaluated at definition time (in loop below) - %xdefine %1_table (%%table - 2*%2) - %xdefine %%base %1_table - %xdefine %%prefix mangle(private_prefix %+ _%1) - ; dynamically generated label - %%table: - %rep %0 - 1 ; repeat for num args - dd %%prefix %+ .w%2 - %%base - %rotate 1 - %endrep -%endmacro - -BIDIR_JMP_TABLE avg_ssse3, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE w_avg_ssse3, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE mask_ssse3, 4, 8, 16, 32, 64, 128 -BIDIR_JMP_TABLE w_mask_420_ssse3, 4, 8, 16, 16, 16, 16 -BIDIR_JMP_TABLE blend_ssse3, 4, 8, 16, 32 -BIDIR_JMP_TABLE blend_v_ssse3, 2, 4, 8, 16, 32 -BIDIR_JMP_TABLE blend_h_ssse3, 2, 4, 8, 16, 16, 16, 16 - -%macro BASE_JMP_TABLE 3-* - %xdefine %1_%2_table (%%table - %3) - %xdefine %%base %1_%2 - %%table: - %rep %0 - 2 - dw %%base %+ _w%3 - %%base - %rotate 1 - %endrep -%endmacro - -%xdefine put_ssse3 mangle(private_prefix %+ _put_bilin_ssse3.put) -%xdefine prep_ssse3 mangle(private_prefix %+ _prep_bilin_ssse3.prep) - -BASE_JMP_TABLE put, ssse3, 2, 4, 8, 16, 32, 64, 128 -BASE_JMP_TABLE prep, ssse3, 4, 8, 16, 32, 64, 128 - -%macro HV_JMP_TABLE 5-* - %xdefine %%prefix mangle(private_prefix %+ _%1_%2_%3) - %xdefine %%base %1_%3 - %assign %%types %4 - %if %%types & 1 - %xdefine %1_%2_h_%3_table (%%h - %5) - %%h: - %rep %0 - 4 - dw %%prefix %+ .h_w%5 - %%base - %rotate 1 - %endrep - %rotate 4 - %endif - %if %%types & 2 - %xdefine %1_%2_v_%3_table (%%v - %5) - %%v: - %rep %0 - 4 - dw %%prefix %+ .v_w%5 - %%base - %rotate 1 - %endrep - %rotate 4 - %endif - %if %%types & 4 - %xdefine %1_%2_hv_%3_table (%%hv - %5) - %%hv: - %rep %0 - 4 - dw %%prefix %+ .hv_w%5 - %%base - %rotate 1 - %endrep - %endif -%endmacro - -HV_JMP_TABLE put, 8tap, ssse3, 3, 2, 4, 8, 16, 32, 64, 128 -HV_JMP_TABLE prep, 8tap, ssse3, 1, 4, 8, 16, 32, 64, 128 -HV_JMP_TABLE put, bilin, ssse3, 7, 2, 4, 8, 16, 32, 64, 128 -HV_JMP_TABLE prep, bilin, ssse3, 7, 4, 8, 16, 32, 64, 128 - -%define table_offset(type, fn) type %+ fn %+ SUFFIX %+ _table - type %+ SUFFIX - -cextern mc_warp_filter - -SECTION .text - -INIT_XMM ssse3 - -%if ARCH_X86_32 - DECLARE_REG_TMP 1 - %define base t0-put_ssse3 -%else - DECLARE_REG_TMP 7 - %define base 0 -%endif -; -%macro RESTORE_DSQ_32 1 - %if ARCH_X86_32 - mov %1, dsm ; restore dsq - %endif -%endmacro -; -cglobal put_bilin, 4, 8, 0, dst, ds, src, ss, w, h, mxy, bak - movifnidn mxyd, r6m ; mx - LEA t0, put_ssse3 - tzcnt wd, wm - mov hd, hm - test mxyd, mxyd - jnz .h - mov mxyd, r7m ; my - test mxyd, mxyd - jnz .v -.put: - movzx wd, word [t0+wq*2+table_offset(put,)] - add wq, t0 - RESTORE_DSQ_32 t0 - jmp wq -.put_w2: - movzx r4d, word [srcq+ssq*0] - movzx r6d, word [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - mov [dstq+dsq*0], r4w - mov [dstq+dsq*1], r6w - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .put_w2 - RET -.put_w4: - mov r4d, [srcq+ssq*0] - mov r6d, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - mov [dstq+dsq*0], r4d - mov [dstq+dsq*1], r6d - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .put_w4 - RET -.put_w8: - movq m0, [srcq+ssq*0] - movq m1, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - movq [dstq+dsq*0], m0 - movq [dstq+dsq*1], m1 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .put_w8 - RET -.put_w16: - movu m0, [srcq+ssq*0] - movu m1, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - mova [dstq+dsq*0], m0 - mova [dstq+dsq*1], m1 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .put_w16 - RET -.put_w32: - movu m0, [srcq+ssq*0+16*0] - movu m1, [srcq+ssq*0+16*1] - movu m2, [srcq+ssq*1+16*0] - movu m3, [srcq+ssq*1+16*1] - lea srcq, [srcq+ssq*2] - mova [dstq+dsq*0+16*0], m0 - mova [dstq+dsq*0+16*1], m1 - mova [dstq+dsq*1+16*0], m2 - mova [dstq+dsq*1+16*1], m3 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .put_w32 - RET -.put_w64: - movu m0, [srcq+16*0] - movu m1, [srcq+16*1] - movu m2, [srcq+16*2] - movu m3, [srcq+16*3] - add srcq, ssq - mova [dstq+16*0], m0 - mova [dstq+16*1], m1 - mova [dstq+16*2], m2 - mova [dstq+16*3], m3 - add dstq, dsq - dec hd - jg .put_w64 - RET -.put_w128: - movu m0, [srcq+16*0] - movu m1, [srcq+16*1] - movu m2, [srcq+16*2] - movu m3, [srcq+16*3] - mova [dstq+16*0], m0 - mova [dstq+16*1], m1 - mova [dstq+16*2], m2 - mova [dstq+16*3], m3 - movu m0, [srcq+16*4] - movu m1, [srcq+16*5] - movu m2, [srcq+16*6] - movu m3, [srcq+16*7] - mova [dstq+16*4], m0 - mova [dstq+16*5], m1 - mova [dstq+16*6], m2 - mova [dstq+16*7], m3 - add srcq, ssq - add dstq, dsq - dec hd - jg .put_w128 - RET -.h: - ; (16 * src[x] + (mx * (src[x + 1] - src[x])) + 8) >> 4 - ; = ((16 - mx) * src[x] + mx * src[x + 1] + 8) >> 4 - imul mxyd, 0xff01 - mova m4, [base+bilin_h_shuf8] - mova m0, [base+bilin_h_shuf4] - add mxyd, 16 << 8 - movd m5, mxyd - mov mxyd, r7m ; my - pshuflw m5, m5, q0000 - punpcklqdq m5, m5 - test mxyd, mxyd - jnz .hv - movzx wd, word [t0+wq*2+table_offset(put, _bilin_h)] - mova m3, [base+pw_2048] - add wq, t0 - RESTORE_DSQ_32 t0 - jmp wq -.h_w2: - pshufd m4, m4, q3120 ; m4 = {1, 0, 2, 1, 5, 4, 6, 5} -.h_w2_loop: - movd m0, [srcq+ssq*0] - movd m1, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - punpckldq m0, m1 - pshufb m0, m4 - pmaddubsw m0, m5 - pmulhrsw m0, m3 - packuswb m0, m0 - movd r6d, m0 - mov [dstq+dsq*0], r6w - shr r6d, 16 - mov [dstq+dsq*1], r6w - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .h_w2_loop - RET -.h_w4: - movq m4, [srcq+ssq*0] - movhps m4, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - pshufb m4, m0 - pmaddubsw m4, m5 - pmulhrsw m4, m3 - packuswb m4, m4 - movd [dstq+dsq*0], m4 - psrlq m4, 32 - movd [dstq+dsq*1], m4 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .h_w4 - RET -.h_w8: - movu m0, [srcq+ssq*0] - movu m1, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - pshufb m0, m4 - pshufb m1, m4 - pmaddubsw m0, m5 - pmaddubsw m1, m5 - pmulhrsw m0, m3 - pmulhrsw m1, m3 - packuswb m0, m1 - movq [dstq+dsq*0], m0 - movhps [dstq+dsq*1], m0 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .h_w8 - RET -.h_w16: - movu m0, [srcq+8*0] - movu m1, [srcq+8*1] - add srcq, ssq - pshufb m0, m4 - pshufb m1, m4 - pmaddubsw m0, m5 - pmaddubsw m1, m5 - pmulhrsw m0, m3 - pmulhrsw m1, m3 - packuswb m0, m1 - mova [dstq], m0 - add dstq, dsq - dec hd - jg .h_w16 - RET -.h_w32: - movu m0, [srcq+mmsize*0+8*0] - movu m1, [srcq+mmsize*0+8*1] - pshufb m0, m4 - pshufb m1, m4 - pmaddubsw m0, m5 - pmaddubsw m1, m5 - pmulhrsw m0, m3 - pmulhrsw m1, m3 - packuswb m0, m1 - movu m1, [srcq+mmsize*1+8*0] - movu m2, [srcq+mmsize*1+8*1] - add srcq, ssq - pshufb m1, m4 - pshufb m2, m4 - pmaddubsw m1, m5 - pmaddubsw m2, m5 - pmulhrsw m1, m3 - pmulhrsw m2, m3 - packuswb m1, m2 - mova [dstq+16*0], m0 - mova [dstq+16*1], m1 - add dstq, dsq - dec hd - jg .h_w32 - RET -.h_w64: - mov r6, -16*3 -.h_w64_loop: - movu m0, [srcq+r6+16*3+8*0] - movu m1, [srcq+r6+16*3+8*1] - pshufb m0, m4 - pshufb m1, m4 - pmaddubsw m0, m5 - pmaddubsw m1, m5 - pmulhrsw m0, m3 - pmulhrsw m1, m3 - packuswb m0, m1 - mova [dstq+r6+16*3], m0 - add r6, 16 - jle .h_w64_loop - add srcq, ssq - add dstq, dsq - dec hd - jg .h_w64 - RET -.h_w128: - mov r6, -16*7 -.h_w128_loop: - movu m0, [srcq+r6+16*7+8*0] - movu m1, [srcq+r6+16*7+8*1] - pshufb m0, m4 - pshufb m1, m4 - pmaddubsw m0, m5 - pmaddubsw m1, m5 - pmulhrsw m0, m3 - pmulhrsw m1, m3 - packuswb m0, m1 - mova [dstq+r6+16*7], m0 - add r6, 16 - jle .h_w128_loop - add srcq, ssq - add dstq, dsq - dec hd - jg .h_w128 - RET -.v: - movzx wd, word [t0+wq*2+table_offset(put, _bilin_v)] - imul mxyd, 0xff01 - mova m5, [base+pw_2048] - add mxyd, 16 << 8 - add wq, t0 - movd m4, mxyd - pshuflw m4, m4, q0000 - punpcklqdq m4, m4 - RESTORE_DSQ_32 t0 - jmp wq -.v_w2: - movd m0, [srcq+ssq*0] -.v_w2_loop: - pinsrw m0, [srcq+ssq*1], 1 ; 0 1 - lea srcq, [srcq+ssq*2] - pshuflw m2, m0, q2301 - pinsrw m0, [srcq+ssq*0], 0 ; 2 1 - punpcklbw m1, m0, m2 - pmaddubsw m1, m4 - pmulhrsw m1, m5 - packuswb m1, m1 - movd r6d, m1 - mov [dstq+dsq*1], r6w - shr r6d, 16 - mov [dstq+dsq*0], r6w - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .v_w2_loop - RET -.v_w4: - movd m0, [srcq+ssq*0] -.v_w4_loop: - movd m1, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - punpckldq m2, m0, m1 ; 0 1 - movd m0, [srcq+ssq*0] - punpckldq m1, m0 ; 1 2 - punpcklbw m1, m2 - pmaddubsw m1, m4 - pmulhrsw m1, m5 - packuswb m1, m1 - movd [dstq+dsq*0], m1 - psrlq m1, 32 - movd [dstq+dsq*1], m1 - ; - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .v_w4_loop - RET -.v_w8: - movq m0, [srcq+ssq*0] -.v_w8_loop: - movq m3, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - punpcklbw m1, m3, m0 - movq m0, [srcq+ssq*0] - punpcklbw m2, m0, m3 - pmaddubsw m1, m4 - pmaddubsw m2, m4 - pmulhrsw m1, m5 - pmulhrsw m2, m5 - packuswb m1, m2 - movq [dstq+dsq*0], m1 - movhps [dstq+dsq*1], m1 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .v_w8_loop - RET - ; -%macro PUT_BILIN_V_W16 0 - movu m0, [srcq+ssq*0] -%%loop: - movu m3, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - punpcklbw m1, m3, m0 - punpckhbw m2, m3, m0 - movu m0, [srcq+ssq*0] - pmaddubsw m1, m4 - pmaddubsw m2, m4 - pmulhrsw m1, m5 - pmulhrsw m2, m5 - packuswb m1, m2 - mova [dstq+dsq*0], m1 - punpcklbw m1, m0, m3 - punpckhbw m2, m0, m3 - pmaddubsw m1, m4 - pmaddubsw m2, m4 - pmulhrsw m1, m5 - pmulhrsw m2, m5 - packuswb m1, m2 - mova [dstq+dsq*1], m1 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg %%loop -%endmacro - ; -.v_w16: - PUT_BILIN_V_W16 - RET -.v_w16gt: - mov r4, dstq - mov r6, srcq -.v_w16gt_loop: -%if ARCH_X86_32 - mov bakm, t0q - RESTORE_DSQ_32 t0 - PUT_BILIN_V_W16 - mov t0q, bakm -%else - PUT_BILIN_V_W16 -%endif - mov hw, t0w - add r4, mmsize - add r6, mmsize - mov dstq, r4 - mov srcq, r6 - sub t0d, 1<<16 - jg .v_w16gt - RET -.v_w32: - lea t0d, [hq+(1<<16)] - jmp .v_w16gt -.v_w64: - lea t0d, [hq+(3<<16)] - jmp .v_w16gt -.v_w128: - lea t0d, [hq+(7<<16)] - jmp .v_w16gt -.hv: - ; (16 * src[x] + (my * (src[x + src_stride] - src[x])) + 128) >> 8 - ; = (src[x] + ((my * (src[x + src_stride] - src[x])) >> 4) + 8) >> 4 - movzx wd, word [t0+wq*2+table_offset(put, _bilin_hv)] - WIN64_SPILL_XMM 8 - shl mxyd, 11 ; can't shift by 12 due to signed overflow - mova m7, [base+pw_2048] - movd m6, mxyd - add wq, t0 - pshuflw m6, m6, q0000 - punpcklqdq m6, m6 - jmp wq -.hv_w2: - RESTORE_DSQ_32 t0 - movd m0, [srcq+ssq*0] - pshufd m0, m0, q0000 ; src[x - src_stride] - pshufb m0, m4 - pmaddubsw m0, m5 -.hv_w2_loop: - movd m1, [srcq+ssq*1] ; src[x] - lea srcq, [srcq+ssq*2] - movhps m1, [srcq+ssq*0] ; src[x + src_stride] - pshufd m1, m1, q3120 - pshufb m1, m4 - pmaddubsw m1, m5 ; 1 _ 2 _ - shufps m2, m0, m1, q1032 ; 0 _ 1 _ - mova m0, m1 - psubw m1, m2 ; src[x + src_stride] - src[x] - paddw m1, m1 - pmulhw m1, m6 ; (my * (src[x + src_stride] - src[x]) - paddw m1, m2 ; src[x] + (my * (src[x + src_stride] - src[x]) - pmulhrsw m1, m7 - packuswb m1, m1 -%if ARCH_X86_64 - movq r6, m1 -%else - pshuflw m1, m1, q2020 - movd r6d, m1 -%endif - mov [dstq+dsq*0], r6w - shr r6, gprsize*4 - mov [dstq+dsq*1], r6w - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .hv_w2_loop - RET -.hv_w4: - mova m4, [base+bilin_h_shuf4] - RESTORE_DSQ_32 t0 - movddup xm0, [srcq+ssq*0] - pshufb m0, m4 - pmaddubsw m0, m5 -.hv_w4_loop: - movq m1, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - movhps m1, [srcq+ssq*0] - pshufb m1, m4 - pmaddubsw m1, m5 ; 1 2 - shufps m2, m0, m1, q1032 ; 0 1 - mova m0, m1 - psubw m1, m2 - paddw m1, m1 - pmulhw m1, m6 - paddw m1, m2 - pmulhrsw m1, m7 - packuswb m1, m1 - movd [dstq+dsq*0], m1 - psrlq m1, 32 - movd [dstq+dsq*1], m1 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .hv_w4_loop - RET -.hv_w8: - RESTORE_DSQ_32 t0 - movu m0, [srcq+ssq*0+8*0] - pshufb m0, m4 - pmaddubsw m0, m5 -.hv_w8_loop: - movu m2, [srcq+ssq*1+8*0] - lea srcq, [srcq+ssq*2] - pshufb m2, m4 - pmaddubsw m2, m5 - psubw m1, m2, m0 - paddw m1, m1 - pmulhw m1, m6 - paddw m1, m0 - movu m0, [srcq+ssq*0+8*0] - pshufb m0, m4 - pmaddubsw m0, m5 - psubw m3, m0, m2 - paddw m3, m3 - pmulhw m3, m6 - paddw m3, m2 - pmulhrsw m1, m7 - pmulhrsw m3, m7 - packuswb m1, m3 - movq [dstq+dsq*0], m1 - movhps [dstq+dsq*1], m1 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .hv_w8_loop - RET -.hv_w16: - xor t0d, t0d -.hv_w16gt: - mov r4, dstq - mov r6, srcq - %if WIN64 - movaps r4m, xmm8 - %endif -.hv_w16_loop0: - movu m0, [srcq+8*0] - movu m1, [srcq+8*1] - pshufb m0, m4 - pshufb m1, m4 - pmaddubsw m0, m5 - pmaddubsw m1, m5 -.hv_w16_loop: -%if ARCH_X86_32 - %define m0tmp [dstq] -%else - %define m0tmp m8 -%endif - add srcq, ssq - movu m2, [srcq+8*0] - movu m3, [srcq+8*1] - pshufb m2, m4 - pshufb m3, m4 - pmaddubsw m2, m5 - pmaddubsw m3, m5 - mova m0tmp, m2 - psubw m2, m0 - paddw m2, m2 - pmulhw m2, m6 - paddw m2, m0 - mova m0, m3 - psubw m3, m1 - paddw m3, m3 - pmulhw m3, m6 - paddw m3, m1 - mova m1, m0 - mova m0, m0tmp - pmulhrsw m2, m7 - pmulhrsw m3, m7 - packuswb m2, m3 - mova [dstq], m2 - add dstq, dsmp - dec hd - jg .hv_w16_loop - movzx hd, t0w - add r4, mmsize - add r6, mmsize - mov dstq, r4 - mov srcq, r6 - sub t0d, 1<<16 - jg .hv_w16_loop0 - %if WIN64 - movaps xmm8, r4m - %endif - RET -.hv_w32: - lea t0d, [hq+(1<<16)] - jmp .hv_w16gt -.hv_w64: - lea t0d, [hq+(3<<16)] - jmp .hv_w16gt -.hv_w128: - lea t0d, [hq+(7<<16)] - jmp .hv_w16gt - -DECLARE_REG_TMP 3, 5, 6 -%if ARCH_X86_32 - %define base t2-prep_ssse3 -%else - %define base 0 -%endif -cglobal prep_bilin, 3, 7, 0, tmp, src, stride, w, h, mxy, stride3 - movifnidn mxyd, r5m ; mx - LEA t2, prep_ssse3 - tzcnt wd, wm - movifnidn hd, hm - test mxyd, mxyd - jnz .h - mov mxyd, r6m ; my - test mxyd, mxyd - jnz .v -.prep: - movzx wd, word [t2+wq*2+table_offset(prep,)] - add wq, t2 - lea stride3q, [strideq*3] - jmp wq -.prep_w4: - movd m0, [srcq+strideq*0] - movd m1, [srcq+strideq*1] - movd m2, [srcq+strideq*2] - movd m3, [srcq+stride3q ] - punpckldq m0, m1 - punpckldq m2, m3 - lea srcq, [srcq+strideq*4] - pxor m1, m1 - punpcklbw m0, m1 - punpcklbw m2, m1 - psllw m0, 4 - psllw m2, 4 - mova [tmpq+mmsize*0], m0 - mova [tmpq+mmsize*1], m2 - add tmpq, 32 - sub hd, 4 - jg .prep_w4 - RET -.prep_w8: - movq m0, [srcq+strideq*0] - movq m1, [srcq+strideq*1] - movq m2, [srcq+strideq*2] - movq m3, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - pxor m4, m4 - punpcklbw m0, m4 - punpcklbw m1, m4 - punpcklbw m2, m4 - punpcklbw m3, m4 - psllw m0, 4 - psllw m1, 4 - psllw m2, 4 - psllw m3, 4 - mova [tmpq+16*0], m0 - mova [tmpq+16*1], m1 - mova [tmpq+16*2], m2 - mova [tmpq+16*3], m3 - add tmpq, 16*4 - sub hd, 4 - jg .prep_w8 - RET -.prep_w16: - movq m0, [srcq+strideq*0+8*0] - movq m1, [srcq+strideq*0+8*1] - movq m2, [srcq+strideq*1+8*0] - movq m3, [srcq+strideq*1+8*1] - lea srcq, [srcq+strideq*2] - pxor m4, m4 - punpcklbw m0, m4 - punpcklbw m1, m4 - punpcklbw m2, m4 - punpcklbw m3, m4 - psllw m0, 4 - psllw m1, 4 - psllw m2, 4 - psllw m3, 4 - mova [tmpq+16*0], m0 - mova [tmpq+16*1], m1 - mova [tmpq+16*2], m2 - mova [tmpq+16*3], m3 - add tmpq, 16*4 - sub hd, 2 - jg .prep_w16 - RET -.prep_w16gt: - mov t1q, srcq - mov r3q, t2q -.prep_w16gt_hloop: - movq m0, [t1q+8*0] - movq m1, [t1q+8*1] - movq m2, [t1q+8*2] - movq m3, [t1q+8*3] - pxor m4, m4 - punpcklbw m0, m4 - punpcklbw m1, m4 - punpcklbw m2, m4 - punpcklbw m3, m4 - psllw m0, 4 - psllw m1, 4 - psllw m2, 4 - psllw m3, 4 - mova [tmpq+16*0], m0 - mova [tmpq+16*1], m1 - mova [tmpq+16*2], m2 - mova [tmpq+16*3], m3 - add tmpq, 16*4 - add t1q, 32 - sub r3q, 1 - jg .prep_w16gt_hloop - lea srcq, [srcq+strideq] - sub hd, 1 - jg .prep_w16gt - RET -.prep_w32: - mov t2q, 1 - jmp .prep_w16gt -.prep_w64: - mov t2q, 2 - jmp .prep_w16gt -.prep_w128: - mov t2q, 4 - jmp .prep_w16gt -.h: - ; 16 * src[x] + (mx * (src[x + 1] - src[x])) - ; = (16 - mx) * src[x] + mx * src[x + 1] - imul mxyd, 0xff01 - mova m4, [base+bilin_h_shuf8] - add mxyd, 16 << 8 - movd xm5, mxyd - mov mxyd, r6m ; my - pshuflw m5, m5, q0000 - punpcklqdq m5, m5 - test mxyd, mxyd - jnz .hv -%if ARCH_X86_32 - mov t1, t2 ; save base reg for w4 -%endif - movzx wd, word [t2+wq*2+table_offset(prep, _bilin_h)] - add wq, t2 - lea stride3q, [strideq*3] - jmp wq -.h_w4: -%if ARCH_X86_32 - mova m4, [t1-prep_ssse3+bilin_h_shuf4] -%else - mova m4, [bilin_h_shuf4] -%endif -.h_w4_loop: - movq m0, [srcq+strideq*0] - movhps m0, [srcq+strideq*1] - movq m1, [srcq+strideq*2] - movhps m1, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - pshufb m0, m4 - pmaddubsw m0, m5 - pshufb m1, m4 - pmaddubsw m1, m5 - mova [tmpq+0 ], m0 - mova [tmpq+16], m1 - add tmpq, 32 - sub hd, 4 - jg .h_w4_loop - RET -.h_w8: - movu m0, [srcq+strideq*0] - movu m1, [srcq+strideq*1] - movu m2, [srcq+strideq*2] - movu m3, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - pshufb m0, m4 - pshufb m1, m4 - pshufb m2, m4 - pshufb m3, m4 - pmaddubsw m0, m5 - pmaddubsw m1, m5 - pmaddubsw m2, m5 - pmaddubsw m3, m5 - mova [tmpq+16*0], m0 - mova [tmpq+16*1], m1 - mova [tmpq+16*2], m2 - mova [tmpq+16*3], m3 - add tmpq, 16*4 - sub hd, 4 - jg .h_w8 - RET -.h_w16: - movu m0, [srcq+strideq*0+8*0] - movu m1, [srcq+strideq*0+8*1] - movu m2, [srcq+strideq*1+8*0] - movu m3, [srcq+strideq*1+8*1] - lea srcq, [srcq+strideq*2] - pshufb m0, m4 - pshufb m1, m4 - pshufb m2, m4 - pshufb m3, m4 - pmaddubsw m0, m5 - pmaddubsw m1, m5 - pmaddubsw m2, m5 - pmaddubsw m3, m5 - mova [tmpq+16*0], m0 - mova [tmpq+16*1], m1 - mova [tmpq+16*2], m2 - mova [tmpq+16*3], m3 - add tmpq, 16*4 - sub hd, 2 - jg .h_w16 - RET -.h_w16gt: - mov t1q, srcq - mov r3q, t2q -.h_w16gt_hloop: - movu m0, [t1q+8*0] - movu m1, [t1q+8*1] - movu m2, [t1q+8*2] - movu m3, [t1q+8*3] - pshufb m0, m4 - pshufb m1, m4 - pshufb m2, m4 - pshufb m3, m4 - pmaddubsw m0, m5 - pmaddubsw m1, m5 - pmaddubsw m2, m5 - pmaddubsw m3, m5 - mova [tmpq+16*0], m0 - mova [tmpq+16*1], m1 - mova [tmpq+16*2], m2 - mova [tmpq+16*3], m3 - add tmpq, 16*4 - add t1q, 32 - sub r3q, 1 - jg .h_w16gt_hloop - lea srcq, [srcq+strideq] - sub hd, 1 - jg .h_w16gt - RET -.h_w32: - mov t2q, 1 - jmp .h_w16gt -.h_w64: - mov t2q, 2 - jmp .h_w16gt -.h_w128: - mov t2q, 4 - jmp .h_w16gt -.v: - movzx wd, word [t2+wq*2+table_offset(prep, _bilin_v)] - imul mxyd, 0xff01 - add mxyd, 16 << 8 - add wq, t2 - lea stride3q, [strideq*3] - movd m5, mxyd - pshuflw m5, m5, q0000 - punpcklqdq m5, m5 - jmp wq -.v_w4: - movd m0, [srcq+strideq*0] -.v_w4_loop: - movd m1, [srcq+strideq*1] - movd m2, [srcq+strideq*2] - movd m3, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - punpcklwd m0, m1 ; 0 1 _ _ - punpcklwd m1, m2 ; 1 2 _ _ - punpcklbw m1, m0 - pmaddubsw m1, m5 - pshufd m1, m1, q3120 - mova [tmpq+16*0], m1 - movd m0, [srcq+strideq*0] - punpcklwd m2, m3 ; 2 3 _ _ - punpcklwd m3, m0 ; 3 4 _ _ - punpcklbw m3, m2 - pmaddubsw m3, m5 - pshufd m3, m3, q3120 - mova [tmpq+16*1], m3 - add tmpq, 32 - sub hd, 4 - jg .v_w4_loop - RET -.v_w8: - movq m0, [srcq+strideq*0] -.v_w8_loop: - movq m1, [srcq+strideq*2] - movq m2, [srcq+strideq*1] - movq m3, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - shufpd m4, m0, m1, 0x0c ; 0 2 - movq m0, [srcq+strideq*0] - shufpd m2, m3, 0x0c ; 1 3 - shufpd m1, m0, 0x0c ; 2 4 - punpcklbw m3, m2, m4 - pmaddubsw m3, m5 - mova [tmpq+16*0], m3 - punpckhbw m3, m2, m4 - pmaddubsw m3, m5 - mova [tmpq+16*2], m3 - punpcklbw m3, m1, m2 - punpckhbw m1, m2 - pmaddubsw m3, m5 - pmaddubsw m1, m5 - mova [tmpq+16*1], m3 - mova [tmpq+16*3], m1 - add tmpq, 16*4 - sub hd, 4 - jg .v_w8_loop - RET -.v_w16: - movu m0, [srcq+strideq*0] -.v_w16_loop: - movu m1, [srcq+strideq*1] - movu m2, [srcq+strideq*2] - punpcklbw m3, m1, m0 - punpckhbw m4, m1, m0 - pmaddubsw m3, m5 - pmaddubsw m4, m5 - mova [tmpq+16*0], m3 - mova [tmpq+16*1], m4 - punpcklbw m3, m2, m1 - punpckhbw m4, m2, m1 - pmaddubsw m3, m5 - pmaddubsw m4, m5 - mova [tmpq+16*2], m3 - mova [tmpq+16*3], m4 - movu m3, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - movu m0, [srcq+strideq*0] - add tmpq, 16*8 - punpcklbw m1, m3, m2 - punpckhbw m4, m3, m2 - pmaddubsw m1, m5 - pmaddubsw m4, m5 - mova [tmpq-16*4], m1 - mova [tmpq-16*3], m4 - punpcklbw m1, m0, m3 - punpckhbw m2, m0, m3 - pmaddubsw m1, m5 - pmaddubsw m2, m5 - mova [tmpq-16*2], m1 - mova [tmpq-16*1], m2 - sub hd, 4 - jg .v_w16_loop - RET -.v_w32: - lea t2d, [hq+(0<<16)] - mov t0d, 64 -.v_w32_start: -%if ARCH_X86_64 - %if WIN64 - PUSH r7 - %endif - mov r7, tmpq -%endif - mov t1, srcq -.v_w32_loop_h: - movu m0, [srcq+strideq*0+16*0] ; 0L - movu m1, [srcq+strideq*0+16*1] ; 0U -.v_w32_loop_v: - movu m2, [srcq+strideq*1+16*0] ; 1L - movu m3, [srcq+strideq*1+16*1] ; 1U - lea srcq, [srcq+strideq*2] - punpcklbw m4, m2, m0 - pmaddubsw m4, m5 - mova [tmpq+16*0], m4 - punpckhbw m4, m2, m0 - pmaddubsw m4, m5 - mova [tmpq+16*1], m4 - punpcklbw m4, m3, m1 - pmaddubsw m4, m5 - mova [tmpq+16*2], m4 - punpckhbw m4, m3, m1 - pmaddubsw m4, m5 - mova [tmpq+16*3], m4 - add tmpq, t0q - movu m0, [srcq+strideq*0+16*0] ; 2L - movu m1, [srcq+strideq*0+16*1] ; 2U - punpcklbw m4, m0, m2 - pmaddubsw m4, m5 - mova [tmpq+16*0], m4 - punpckhbw m4, m0, m2 - pmaddubsw m4, m5 - mova [tmpq+16*1], m4 - punpcklbw m4, m1, m3 - pmaddubsw m4, m5 - mova [tmpq+16*2], m4 - punpckhbw m4, m1, m3 - pmaddubsw m4, m5 - mova [tmpq+16*3], m4 - add tmpq, t0q - sub hd, 2 - jg .v_w32_loop_v - movzx hd, t2w - add t1, 32 - mov srcq, t1 -%if ARCH_X86_64 - add r7, 2*16*2 - mov tmpq, r7 -%else - mov tmpq, tmpmp - add tmpq, 2*16*2 - mov tmpmp, tmpq -%endif - sub t2d, 1<<16 - jg .v_w32_loop_h -%if WIN64 - POP r7 -%endif - RET -.v_w64: - lea t2d, [hq+(1<<16)] - mov t0d, 128 - jmp .v_w32_start -.v_w128: - lea t2d, [hq+(3<<16)] - mov t0d, 256 - jmp .v_w32_start -.hv: - ; (16 * src[x] + (my * (src[x + src_stride] - src[x])) + 8) >> 4 - ; = src[x] + (((my * (src[x + src_stride] - src[x])) + 8) >> 4) - %assign stack_offset stack_offset - stack_size_padded - WIN64_SPILL_XMM 8 - movzx wd, word [t2+wq*2+table_offset(prep, _bilin_hv)] - shl mxyd, 11 - movd xm6, mxyd - add wq, t2 - pshuflw m6, m6, q0000 - punpcklqdq m6, m6 -%if ARCH_X86_32 - mov t1, t2 ; save base reg for w4 -%endif - lea stride3q, [strideq*3] - jmp wq -.hv_w4: -%if ARCH_X86_32 - mova m4, [t1-prep_ssse3+bilin_h_shuf4] -%else - mova m4, [bilin_h_shuf4] -%endif - movq m0, [srcq+strideq*0] ; 0 _ - punpcklqdq m0, m0 - pshufb m0, m4 - pmaddubsw m0, m5 -.hv_w4_loop: - movq m1, [srcq+strideq*1] - movhps m1, [srcq+strideq*2] ; 1 _ 2 _ - movq m2, [srcq+stride3q ] - lea srcq, [srcq+strideq*4] - movhps m2, [srcq+strideq*0] ; 3 _ 4 _ - pshufb m1, m4 - pshufb m2, m4 - pmaddubsw m1, m5 ; 1 + 2 + - shufpd m3, m0, m1, 0x01 ; 0 + 1 + - pmaddubsw m0, m2, m5 ; 3 + 4 + - shufpd m2, m1, m0, 0x01 ; 2 + 3 + - psubw m1, m3 - pmulhrsw m1, m6 - paddw m1, m3 - psubw m3, m0, m2 - pmulhrsw m3, m6 - paddw m3, m2 - mova [tmpq+16*0], m1 - mova [tmpq+16*1], m3 - add tmpq, 32 - sub hd, 4 - jg .hv_w4_loop - RET -.hv_w8: - movu m0, [srcq+strideq*0] - pshufb m0, m4 - pmaddubsw m0, m5 ; 0 + -.hv_w8_loop: - movu m1, [srcq+strideq*1] ; 1 - movu m2, [srcq+strideq*2] ; 2 - pshufb m1, m4 - pshufb m2, m4 - pmaddubsw m1, m5 ; 1 + - pmaddubsw m2, m5 ; 2 + - psubw m3, m1, m0 ; 1-0 - pmulhrsw m3, m6 - paddw m3, m0 - psubw m7, m2, m1 ; 2-1 - pmulhrsw m7, m6 - paddw m7, m1 - mova [tmpq+16*0], m3 - mova [tmpq+16*1], m7 - movu m1, [srcq+stride3q ] ; 3 - lea srcq, [srcq+strideq*4] - movu m0, [srcq+strideq*0] ; 4 - pshufb m1, m4 - pshufb m0, m4 - pmaddubsw m1, m5 ; 3 + - pmaddubsw m0, m5 ; 4 + - psubw m3, m1, m2 ; 3-2 - pmulhrsw m3, m6 - paddw m3, m2 - psubw m7, m0, m1 ; 4-3 - pmulhrsw m7, m6 - paddw m7, m1 - mova [tmpq+16*2], m3 - mova [tmpq+16*3], m7 - add tmpq, 16*4 - sub hd, 4 - jg .hv_w8_loop - RET -.hv_w16: - lea t2d, [hq+(0<<16)] - mov t0d, 32 -.hv_w16_start: -%if ARCH_X86_64 - %if WIN64 - PUSH r7 - %endif - mov r7, tmpq -%endif - mov t1, srcq -.hv_w16_loop_h: - movu m0, [srcq+strideq*0+8*0] ; 0L - movu m1, [srcq+strideq*0+8*1] ; 0U - pshufb m0, m4 - pshufb m1, m4 - pmaddubsw m0, m5 ; 0L + - pmaddubsw m1, m5 ; 0U + -.hv_w16_loop_v: - movu m2, [srcq+strideq*1+8*0] ; 1L - pshufb m2, m4 - pmaddubsw m2, m5 ; 1L + - psubw m3, m2, m0 ; 1L-0L - pmulhrsw m3, m6 - paddw m3, m0 - mova [tmpq+16*0], m3 - movu m3, [srcq+strideq*1+8*1] ; 1U - lea srcq, [srcq+strideq*2] - pshufb m3, m4 - pmaddubsw m3, m5 ; 1U + - psubw m0, m3, m1 ; 1U-0U - pmulhrsw m0, m6 - paddw m0, m1 - mova [tmpq+16*1], m0 - add tmpq, t0q - movu m0, [srcq+strideq*0+8*0] ; 2L - pshufb m0, m4 - pmaddubsw m0, m5 ; 2L + - psubw m1, m0, m2 ; 2L-1L - pmulhrsw m1, m6 - paddw m1, m2 - mova [tmpq+16*0], m1 - movu m1, [srcq+strideq*0+8*1] ; 2U - pshufb m1, m4 - pmaddubsw m1, m5 ; 2U + - psubw m2, m1, m3 ; 2U-1U - pmulhrsw m2, m6 - paddw m2, m3 - mova [tmpq+16*1], m2 - add tmpq, t0q - sub hd, 2 - jg .hv_w16_loop_v - movzx hd, t2w - add t1, 16 - mov srcq, t1 -%if ARCH_X86_64 - add r7, 2*16 - mov tmpq, r7 -%else - mov tmpq, tmpmp - add tmpq, 2*16 - mov tmpmp, tmpq -%endif - sub t2d, 1<<16 - jg .hv_w16_loop_h -%if WIN64 - POP r7 -%endif - RET -.hv_w32: - lea t2d, [hq+(1<<16)] - mov t0d, 64 - jmp .hv_w16_start -.hv_w64: - lea t2d, [hq+(3<<16)] - mov t0d, 128 - jmp .hv_w16_start -.hv_w128: - lea t2d, [hq+(7<<16)] - mov t0d, 256 - jmp .hv_w16_start - -; int8_t subpel_filters[5][15][8] -%assign FILTER_REGULAR (0*15 << 16) | 3*15 -%assign FILTER_SMOOTH (1*15 << 16) | 4*15 -%assign FILTER_SHARP (2*15 << 16) | 3*15 - -%if ARCH_X86_32 -DECLARE_REG_TMP 1, 2 -%elif WIN64 -DECLARE_REG_TMP 4, 5 -%else -DECLARE_REG_TMP 7, 8 -%endif - -%macro PUT_8TAP_FN 3 ; type, type_h, type_v -cglobal put_8tap_%1 - mov t0d, FILTER_%2 - mov t1d, FILTER_%3 -%ifnidn %1, sharp_smooth ; skip the jump in the last filter - jmp mangle(private_prefix %+ _put_8tap %+ SUFFIX) -%endif -%endmacro - -PUT_8TAP_FN regular, REGULAR, REGULAR -PUT_8TAP_FN regular_sharp, REGULAR, SHARP -PUT_8TAP_FN regular_smooth, REGULAR, SMOOTH -PUT_8TAP_FN smooth_regular, SMOOTH, REGULAR -PUT_8TAP_FN smooth, SMOOTH, SMOOTH -PUT_8TAP_FN smooth_sharp, SMOOTH, SHARP -PUT_8TAP_FN sharp_regular, SHARP, REGULAR -PUT_8TAP_FN sharp, SHARP, SHARP -PUT_8TAP_FN sharp_smooth, SHARP, SMOOTH - -%if ARCH_X86_32 - %define base_reg r1 - %define base base_reg-put_ssse3 - %define W32_RESTORE_DSQ mov dsq, dsm - %define W32_RESTORE_SSQ mov ssq, ssm -%else - %define base_reg r8 - %define base 0 - %define W32_RESTORE_DSQ - %define W32_RESTORE_SSQ -%endif - -cglobal put_8tap, 1, 9, 0, dst, ds, src, ss, w, h, mx, my, ss3 -%assign org_stack_offset stack_offset - imul mxd, mxm, 0x010101 - add mxd, t0d ; 8tap_h, mx, 4tap_h -%if ARCH_X86_64 - imul myd, mym, 0x010101 - add myd, t1d ; 8tap_v, my, 4tap_v -%else - imul ssd, mym, 0x010101 - add ssd, t1d ; 8tap_v, my, 4tap_v - mov srcq, srcm -%endif - mov wd, wm - movifnidn hd, hm - LEA base_reg, put_ssse3 - test mxd, 0xf00 - jnz .h -%if ARCH_X86_32 - test ssd, 0xf00 -%else - test myd, 0xf00 -%endif - jnz .v - tzcnt wd, wd - movzx wd, word [base_reg+wq*2+table_offset(put,)] - add wq, base_reg -; put_bilin mangling jump -%assign stack_offset org_stack_offset -%if ARCH_X86_32 - mov dsq, dsm - mov ssq, ssm -%elif WIN64 - pop r8 -%endif - lea r6, [ssq*3] - jmp wq -.h: -%if ARCH_X86_32 - test ssd, 0xf00 -%else - test myd, 0xf00 -%endif - jnz .hv - W32_RESTORE_SSQ - WIN64_SPILL_XMM 12 - cmp wd, 4 - jl .h_w2 - je .h_w4 - tzcnt wd, wd -%if ARCH_X86_64 - mova m10, [base+subpel_h_shufA] - mova m11, [base+subpel_h_shufB] - mova m9, [base+subpel_h_shufC] -%endif - shr mxd, 16 - sub srcq, 3 - movzx wd, word [base_reg+wq*2+table_offset(put, _8tap_h)] - movd m5, [base_reg+mxq*8+subpel_filters-put_ssse3+0] - pshufd m5, m5, q0000 - movd m6, [base_reg+mxq*8+subpel_filters-put_ssse3+4] - pshufd m6, m6, q0000 - mova m7, [base+pw_34] ; 2 + (8 << 2) - add wq, base_reg - jmp wq -.h_w2: -%if ARCH_X86_32 - and mxd, 0x7f -%else - movzx mxd, mxb -%endif - dec srcq - mova m4, [base+subpel_h_shuf4] - movd m3, [base_reg+mxq*8+subpel_filters-put_ssse3+2] - pshufd m3, m3, q0000 - mova m5, [base+pw_34] ; 2 + (8 << 2) - W32_RESTORE_DSQ -.h_w2_loop: - movq m0, [srcq+ssq*0] - movhps m0, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - pshufb m0, m4 - pmaddubsw m0, m3 - phaddw m0, m0 - paddw m0, m5 ; pw34 - psraw m0, 6 - packuswb m0, m0 - movd r4d, m0 - mov [dstq+dsq*0], r4w - shr r4d, 16 - mov [dstq+dsq*1], r4w - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .h_w2_loop - RET -.h_w4: -%if ARCH_X86_32 - and mxd, 0x7f -%else - movzx mxd, mxb -%endif - dec srcq - movd m3, [base_reg+mxq*8+subpel_filters-put_ssse3+2] - pshufd m3, m3, q0000 - mova m5, [base+pw_34] ; 2 + (8 << 2) - mova m6, [base+subpel_h_shufA] - W32_RESTORE_DSQ -.h_w4_loop: - movq m0, [srcq+ssq*0] ; 1 - movq m1, [srcq+ssq*1] ; 2 - lea srcq, [srcq+ssq*2] - pshufb m0, m6 ; subpel_h_shufA - pshufb m1, m6 ; subpel_h_shufA - pmaddubsw m0, m3 ; subpel_filters - pmaddubsw m1, m3 ; subpel_filters - phaddw m0, m1 - paddw m0, m5 ; pw34 - psraw m0, 6 - packuswb m0, m0 - movd [dstq+dsq*0], m0 - psrlq m0, 32 - movd [dstq+dsq*1], m0 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .h_w4_loop - RET - ; -%macro PUT_8TAP_H 4 ; dst/src, tmp[1-3] - %if ARCH_X86_32 - pshufb %2, %1, [base+subpel_h_shufB] - pshufb %3, %1, [base+subpel_h_shufC] - pshufb %1, [base+subpel_h_shufA] - %else - pshufb %2, %1, m11; subpel_h_shufB - pshufb %3, %1, m9 ; subpel_h_shufC - pshufb %1, m10 ; subpel_h_shufA - %endif - pmaddubsw %4, %2, m5 ; subpel +0 B0 - pmaddubsw %2, m6 ; subpel +4 B4 - pmaddubsw %3, m6 ; C4 - pmaddubsw %1, m5 ; A0 - paddw %3, %4 ; C4+B0 - paddw %1, %2 ; A0+B4 - phaddw %1, %3 - paddw %1, m7 ; pw34 - psraw %1, 6 -%endmacro - ; -.h_w8: - movu m0, [srcq+ssq*0] - movu m1, [srcq+ssq*1] - PUT_8TAP_H m0, m2, m3, m4 - lea srcq, [srcq+ssq*2] - PUT_8TAP_H m1, m2, m3, m4 - packuswb m0, m1 -%if ARCH_X86_32 - movq [dstq ], m0 - add dstq, dsm - movhps [dstq ], m0 - add dstq, dsm -%else - movq [dstq+dsq*0], m0 - movhps [dstq+dsq*1], m0 - lea dstq, [dstq+dsq*2] -%endif - sub hd, 2 - jg .h_w8 - RET -.h_w16: - xor r6d, r6d - jmp .h_start -.h_w32: - mov r6, -16*1 - jmp .h_start -.h_w64: - mov r6, -16*3 - jmp .h_start -.h_w128: - mov r6, -16*7 -.h_start: - sub srcq, r6 - sub dstq, r6 - mov r4, r6 -.h_loop: - movu m0, [srcq+r6+8*0] - movu m1, [srcq+r6+8*1] - PUT_8TAP_H m0, m2, m3, m4 - PUT_8TAP_H m1, m2, m3, m4 - packuswb m0, m1 - mova [dstq+r6], m0 - add r6, mmsize - jle .h_loop - add srcq, ssq -%if ARCH_X86_32 - add dstq, dsm -%else - add dstq, dsq -%endif - mov r6, r4 - dec hd - jg .h_loop - RET -.v: -%if ARCH_X86_32 - movzx mxd, ssb - shr ssd, 16 - cmp hd, 6 - cmovs ssd, mxd - lea ssq, [base_reg+ssq*8+subpel_filters-put_ssse3] -%else - %assign stack_offset org_stack_offset - WIN64_SPILL_XMM 16 - movzx mxd, myb - shr myd, 16 - cmp hd, 6 - cmovs myd, mxd - lea myq, [base_reg+myq*8+subpel_filters-put_ssse3] -%endif - tzcnt r6d, wd - movzx r6d, word [base_reg+r6*2+table_offset(put, _8tap_v)] - mova m7, [base+pw_512] - psrlw m2, m7, 1 ; 0x0100 - add r6, base_reg -%if ARCH_X86_32 - %define subpel0 [rsp+mmsize*0] - %define subpel1 [rsp+mmsize*1] - %define subpel2 [rsp+mmsize*2] - %define subpel3 [rsp+mmsize*3] -%assign regs_used 2 ; use r1 (ds) as tmp for stack alignment if needed - ALLOC_STACK -mmsize*4 -%assign regs_used 7 - movd m0, [ssq+0] - pshufb m0, m2 - mova subpel0, m0 - movd m0, [ssq+2] - pshufb m0, m2 - mova subpel1, m0 - movd m0, [ssq+4] - pshufb m0, m2 - mova subpel2, m0 - movd m0, [ssq+6] - pshufb m0, m2 - mova subpel3, m0 - mov ssq, [rstk+stack_offset+gprsize*4] - lea ssq, [ssq*3] - sub srcq, ssq - mov ssq, [rstk+stack_offset+gprsize*4] - mov dsq, [rstk+stack_offset+gprsize*2] -%else - %define subpel0 m8 - %define subpel1 m9 - %define subpel2 m10 - %define subpel3 m11 - movd subpel0, [myq+0] - pshufb subpel0, m2 - movd subpel1, [myq+2] - pshufb subpel1, m2 - movd subpel2, [myq+4] - pshufb subpel2, m2 - movd subpel3, [myq+6] - pshufb subpel3, m2 - lea ss3q, [ssq*3] - sub srcq, ss3q -%endif - jmp r6 -.v_w2: - movd m2, [srcq+ssq*0] ; 0 - pinsrw m2, [srcq+ssq*1], 2 ; 0 1 - pinsrw m2, [srcq+ssq*2], 4 ; 0 1 2 -%if ARCH_X86_32 - lea srcq, [srcq+ssq*2] - add srcq, ssq - pinsrw m2, [srcq+ssq*0], 6 ; 0 1 2 3 - add srcq, ssq -%else - pinsrw m2, [srcq+ss3q ], 6 ; 0 1 2 3 - lea srcq, [srcq+ssq*4] -%endif - movd m3, [srcq+ssq*0] ; 4 - movd m1, [srcq+ssq*1] ; 5 - movd m0, [srcq+ssq*2] ; 6 -%if ARCH_X86_32 - lea srcq, [srcq+ssq*2] - add srcq, ssq -%else - add srcq, ss3q -%endif - punpckldq m3, m1 ; 4 5 _ _ - punpckldq m1, m0 ; 5 6 _ _ - palignr m4, m3, m2, 4 ; 1 2 3 4 - punpcklbw m3, m1 ; 45 56 - punpcklbw m1, m2, m4 ; 01 12 - punpckhbw m2, m4 ; 23 34 -.v_w2_loop: - pmaddubsw m5, m1, subpel0 ; a0 b0 - mova m1, m2 - pmaddubsw m2, subpel1 ; a1 b1 - paddw m5, m2 - mova m2, m3 - pmaddubsw m3, subpel2 ; a2 b2 - paddw m5, m3 - movd m4, [srcq+ssq*0] ; 7 - punpckldq m3, m0, m4 ; 6 7 _ _ - movd m0, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - punpckldq m4, m0 ; 7 8 _ _ - punpcklbw m3, m4 ; 67 78 - pmaddubsw m4, m3, subpel3 ; a3 b3 - paddw m5, m4 - pmulhrsw m5, m7 - packuswb m5, m5 - pshuflw m5, m5, q2020 - movd r6d, m5 - mov [dstq+dsq*0], r6w - shr r6d, 16 - mov [dstq+dsq*1], r6w - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .v_w2_loop - RET -.v_w4: -%if ARCH_X86_32 -.v_w8: -.v_w16: -.v_w32: -.v_w64: -.v_w128: -%endif ; ARCH_X86_32 - lea r6d, [wq - 4] ; horizontal loop - mov r4, dstq -%if ARCH_X86_32 -%if STACK_ALIGNMENT < mmsize - %define srcm [rsp+mmsize*4+gprsize] -%endif - mov srcm, srcq -%else - mov r7, srcq -%endif - shl r6d, (16 - 2) ; (wq / 4) << 16 - mov r6w, hw -.v_w4_loop0: - movd m2, [srcq+ssq*0] ; 0 - movhps m2, [srcq+ssq*2] ; 0 _ 2 - movd m3, [srcq+ssq*1] ; 1 -%if ARCH_X86_32 - lea srcq, [srcq+ssq*2] - add srcq, ssq - movhps m3, [srcq+ssq*0] ; 1 _ 3 - lea srcq, [srcq+ssq*1] -%else - movhps m3, [srcq+ss3q ] ; 1 _ 3 - lea srcq, [srcq+ssq*4] -%endif - pshufd m2, m2, q2020 ; 0 2 0 2 - pshufd m3, m3, q2020 ; 1 3 1 3 - punpckldq m2, m3 ; 0 1 2 3 - movd m3, [srcq+ssq*0] ; 4 - movd m1, [srcq+ssq*1] ; 5 - movd m0, [srcq+ssq*2] ; 6 -%if ARCH_X86_32 - lea srcq, [srcq+ssq*2] - add srcq, ssq -%else - add srcq, ss3q -%endif - punpckldq m3, m1 ; 4 5 _ _ - punpckldq m1, m0 ; 5 6 _ _ - palignr m4, m3, m2, 4 ; 1 2 3 4 - punpcklbw m3, m1 ; 45 56 - punpcklbw m1, m2, m4 ; 01 12 - punpckhbw m2, m4 ; 23 34 -.v_w4_loop: - pmaddubsw m5, m1, subpel0 ; a0 b0 - mova m1, m2 - pmaddubsw m2, subpel1 ; a1 b1 - paddw m5, m2 - mova m2, m3 - pmaddubsw m3, subpel2 ; a2 b2 - paddw m5, m3 - movd m4, [srcq+ssq*0] - punpckldq m3, m0, m4 ; 6 7 _ _ - movd m0, [srcq+ssq*1] - lea srcq, [srcq+ssq*2] - punpckldq m4, m0 ; 7 8 _ _ - punpcklbw m3, m4 ; 67 78 - pmaddubsw m4, m3, subpel3 ; a3 b3 - paddw m5, m4 - pmulhrsw m5, m7 - packuswb m5, m5 - movd [dstq+dsq*0], m5 - pshufd m5, m5, q0101 - movd [dstq+dsq*1], m5 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .v_w4_loop - mov hw, r6w ; reset vertical loop - add r4, 4 - mov dstq, r4 -%if ARCH_X86_32 - mov srcq, srcm - add srcq, 4 - mov srcm, srcq -%else - add r7, 4 - mov srcq, r7 -%endif - sub r6d, 1<<16 ; horizontal-- - jg .v_w4_loop0 - RET -%if ARCH_X86_64 -.v_w8: -.v_w16: -.v_w32: -.v_w64: -.v_w128: - lea r6d, [wq - 8] ; horizontal loop - mov r4, dstq - mov r7, srcq - shl r6d, 8 - 3; (wq / 8) << 8 - mov r6b, hb -.v_w8_loop0: - movq m4, [srcq+ssq*0] ; 0 - movq m5, [srcq+ssq*1] ; 1 - lea srcq, [srcq+ssq*2] - movq m6, [srcq+ssq*0] ; 2 - movq m0, [srcq+ssq*1] ; 3 - lea srcq, [srcq+ssq*2] - movq m1, [srcq+ssq*0] ; 4 - movq m2, [srcq+ssq*1] ; 5 - lea srcq, [srcq+ssq*2] ; - movq m3, [srcq+ssq*0] ; 6 - shufpd m4, m0, 0x0c - shufpd m5, m1, 0x0c - punpcklbw m1, m4, m5 ; 01 - punpckhbw m4, m5 ; 34 - shufpd m6, m2, 0x0c - punpcklbw m2, m5, m6 ; 12 - punpckhbw m5, m6 ; 45 - shufpd m0, m3, 0x0c - punpcklbw m3, m6, m0 ; 23 - punpckhbw m6, m0 ; 56 -.v_w8_loop: - movq m12, [srcq+ssq*1] ; 8 - lea srcq, [srcq+ssq*2] - movq m13, [srcq+ssq*0] ; 9 - pmaddubsw m14, m1, subpel0 ; a0 - pmaddubsw m15, m2, subpel0 ; b0 - mova m1, m3 - mova m2, m4 - pmaddubsw m3, subpel1 ; a1 - pmaddubsw m4, subpel1 ; b1 - paddw m14, m3 - paddw m15, m4 - mova m3, m5 - mova m4, m6 - pmaddubsw m5, subpel2 ; a2 - pmaddubsw m6, subpel2 ; b2 - paddw m14, m5 - paddw m15, m6 - shufpd m6, m0, m12, 0x0d - shufpd m0, m12, m13, 0x0c - punpcklbw m5, m6, m0 ; 67 - punpckhbw m6, m0 ; 78 - pmaddubsw m12, m5, subpel3 ; a3 - pmaddubsw m13, m6, subpel3 ; b3 - paddw m14, m12 - paddw m15, m13 - pmulhrsw m14, m7 - pmulhrsw m15, m7 - packuswb m14, m15 - movq [dstq+dsq*0], xm14 - movhps [dstq+dsq*1], xm14 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .v_w8_loop - movzx hd, r6b ; reset vertical loop - add r4, 8 - add r7, 8 - mov dstq, r4 - mov srcq, r7 - sub r6d, 1<<8 ; horizontal-- - jg .v_w8_loop0 - RET -%endif ;ARCH_X86_64 -%undef subpel0 -%undef subpel1 -%undef subpel2 -%undef subpel3 -.hv: - %assign stack_offset org_stack_offset - cmp wd, 4 - jg .hv_w8 -%if ARCH_X86_32 - and mxd, 0x7f -%else - movzx mxd, mxb -%endif - dec srcq - movd m1, [base_reg+mxq*8+subpel_filters-put_ssse3+2] -%if ARCH_X86_32 - movzx mxd, ssb - shr ssd, 16 - cmp hd, 6 - cmovs ssd, mxd - movq m0, [base_reg+ssq*8+subpel_filters-put_ssse3] - W32_RESTORE_SSQ - lea r6, [ssq*3] - sub srcq, r6 - %define base_reg r6 - mov r6, r1; use as new base - %assign regs_used 2 - ALLOC_STACK -mmsize*14 - %assign regs_used 7 - mov dsq, [rstk+stack_offset+gprsize*2] - %define subpelv0 [rsp+mmsize*0] - %define subpelv1 [rsp+mmsize*1] - %define subpelv2 [rsp+mmsize*2] - %define subpelv3 [rsp+mmsize*3] - punpcklqdq m0, m0 - punpcklbw m0, m0 - psraw m0, 8 ; sign-extend - pshufd m6, m0, q0000 - mova subpelv0, m6 - pshufd m6, m0, q1111 - mova subpelv1, m6 - pshufd m6, m0, q2222 - mova subpelv2, m6 - pshufd m6, m0, q3333 - mova subpelv3, m6 -%else - movzx mxd, myb - shr myd, 16 - cmp hd, 6 - cmovs myd, mxd - movq m0, [base_reg+myq*8+subpel_filters-put_ssse3] - ALLOC_STACK mmsize*14, 14 - lea ss3q, [ssq*3] - sub srcq, ss3q - %define subpelv0 m10 - %define subpelv1 m11 - %define subpelv2 m12 - %define subpelv3 m13 - punpcklqdq m0, m0 - punpcklbw m0, m0 - psraw m0, 8 ; sign-extend - mova m8, [base+pw_8192] - mova m9, [base+pd_512] - pshufd m10, m0, q0000 - pshufd m11, m0, q1111 - pshufd m12, m0, q2222 - pshufd m13, m0, q3333 -%endif - pshufd m7, m1, q0000 - cmp wd, 4 - je .hv_w4 -.hv_w2: - mova m6, [base+subpel_h_shuf4] - ; - movq m2, [srcq+ssq*0] ; 0 - movhps m2, [srcq+ssq*1] ; 0 _ 1 - movq m0, [srcq+ssq*2] ; 2 -%if ARCH_X86_32 - %define w8192reg [base+pw_8192] - %define d512reg [base+pd_512] - lea srcq, [srcq+ssq*2] - add srcq, ssq - movhps m0, [srcq+ssq*0] ; 2 _ 3 - lea srcq, [srcq+ssq*1] -%else - %define w8192reg m8 - %define d512reg m9 - movhps m0, [srcq+ss3q ] ; 2 _ 3 - lea srcq, [srcq+ssq*4] -%endif - pshufb m2, m6 ; 0 ~ 1 ~ - pshufb m0, m6 ; 2 ~ 3 ~ - pmaddubsw m2, m7 ; subpel_filters - pmaddubsw m0, m7 ; subpel_filters - phaddw m2, m0 ; 0 1 2 3 - pmulhrsw m2, w8192reg - ; - movq m3, [srcq+ssq*0] ; 4 - movhps m3, [srcq+ssq*1] ; 4 _ 5 - movq m0, [srcq+ssq*2] ; 6 -%if ARCH_X86_32 - lea srcq, [srcq+ssq*2] - add srcq, ssq -%else - add srcq, ss3q -%endif - pshufb m3, m6 ; 4 ~ 5 ~ - pshufb m0, m6 ; 6 ~ - pmaddubsw m3, m7 ; subpel_filters - pmaddubsw m0, m7 ; subpel_filters - phaddw m3, m0 ; 4 5 6 _ - pmulhrsw m3, w8192reg - ; - palignr m4, m3, m2, 4; V 1 2 3 4 - punpcklwd m1, m2, m4 ; V 01 12 0 1 1 2 - punpckhwd m2, m4 ; V 23 34 2 3 3 4 - pshufd m0, m3, q2121; V 5 6 5 6 - punpcklwd m3, m0 ; V 45 56 4 5 5 6 -.hv_w2_loop: - pmaddwd m5, m1, subpelv0; V a0 b0 - mova m1, m2 ; V - pmaddwd m2, subpelv1 ; V a1 b1 - paddd m5, m2 ; V - mova m2, m3 ; V - pmaddwd m3, subpelv2 ; a2 b2 - paddd m5, m3 ; V - movq m4, [srcq+ssq*0] ; V 7 - movhps m4, [srcq+ssq*1] ; V 7 8 - lea srcq, [srcq+ssq*2] ; V - pshufb m4, m6 - pmaddubsw m4, m7 - phaddw m4, m4 - pmulhrsw m4, w8192reg - palignr m3, m4, m0, 12 - mova m0, m4 - punpcklwd m3, m0 ; V 67 78 - pmaddwd m4, m3, subpelv3 ; V a3 b3 - paddd m5, d512reg - paddd m5, m4 - psrad m5, 10 - packssdw m5, m5 - packuswb m5, m5 - movd r4d, m5 - mov [dstq+dsq*0], r4w - shr r4d, 16 - mov [dstq+dsq*1], r4w - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .hv_w2_loop - RET -%undef w8192reg -%undef d512reg - ; -.hv_w4: -%define hv4_line_0_0 4 -%define hv4_line_0_1 5 -%define hv4_line_0_2 6 -%define hv4_line_0_3 7 -%define hv4_line_0_4 8 -%define hv4_line_0_5 9 -%define hv4_line_1_0 10 -%define hv4_line_1_1 11 -%define hv4_line_1_2 12 -%define hv4_line_1_3 13 - ; -%macro SAVELINE_W4 3 - mova [rsp+mmsize*hv4_line_%3_%2], %1 -%endmacro -%macro RESTORELINE_W4 3 - mova %1, [rsp+mmsize*hv4_line_%3_%2] -%endmacro - ; -%if ARCH_X86_32 - %define w8192reg [base+pw_8192] - %define d512reg [base+pd_512] -%else - %define w8192reg m8 - %define d512reg m9 -%endif - ; lower shuffle 0 1 2 3 4 - mova m6, [base+subpel_h_shuf4] - movq m5, [srcq+ssq*0] ; 0 _ _ _ - movhps m5, [srcq+ssq*1] ; 0 _ 1 _ - movq m4, [srcq+ssq*2] ; 2 _ _ _ -%if ARCH_X86_32 - lea srcq, [srcq+ssq*2] - add srcq, ssq - movhps m4, [srcq+ssq*0] ; 2 _ 3 _ - add srcq, ssq -%else - movhps m4, [srcq+ss3q ] ; 2 _ 3 _ - lea srcq, [srcq+ssq*4] -%endif - pshufb m2, m5, m6 ;H subpel_h_shuf4 0 ~ 1 ~ - pshufb m0, m4, m6 ;H subpel_h_shuf4 2 ~ 3 ~ - pmaddubsw m2, m7 ;H subpel_filters - pmaddubsw m0, m7 ;H subpel_filters - phaddw m2, m0 ;H 0 1 2 3 - pmulhrsw m2, w8192reg ;H pw_8192 - SAVELINE_W4 m2, 2, 0 - ; upper shuffle 2 3 4 5 6 - mova m6, [base+subpel_h_shuf4+16] - pshufb m2, m5, m6 ;H subpel_h_shuf4 0 ~ 1 ~ - pshufb m0, m4, m6 ;H subpel_h_shuf4 2 ~ 3 ~ - pmaddubsw m2, m7 ;H subpel_filters - pmaddubsw m0, m7 ;H subpel_filters - phaddw m2, m0 ;H 0 1 2 3 - pmulhrsw m2, w8192reg ;H pw_8192 - ; - ; lower shuffle - mova m6, [base+subpel_h_shuf4] - movq m5, [srcq+ssq*0] ; 4 _ _ _ - movhps m5, [srcq+ssq*1] ; 4 _ 5 _ - movq m4, [srcq+ssq*2] ; 6 _ _ _ - pshufb m3, m5, m6 ;H subpel_h_shuf4 4 ~ 5 ~ - pshufb m0, m4, m6 ;H subpel_h_shuf4 6 ~ 6 ~ - pmaddubsw m3, m7 ;H subpel_filters - pmaddubsw m0, m7 ;H subpel_filters - phaddw m3, m0 ;H 4 5 6 7 - pmulhrsw m3, w8192reg ;H pw_8192 - SAVELINE_W4 m3, 3, 0 - ; upper shuffle - mova m6, [base+subpel_h_shuf4+16] - pshufb m3, m5, m6 ;H subpel_h_shuf4 4 ~ 5 ~ - pshufb m0, m4, m6 ;H subpel_h_shuf4 6 ~ 6 ~ - pmaddubsw m3, m7 ;H subpel_filters - pmaddubsw m0, m7 ;H subpel_filters - phaddw m3, m0 ;H 4 5 6 7 - pmulhrsw m3, w8192reg ;H pw_8192 - ; -%if ARCH_X86_32 - lea srcq, [srcq+ssq*2] - add srcq, ssq -%else - add srcq, ss3q -%endif - ;process high - palignr m4, m3, m2, 4;V 1 2 3 4 - punpcklwd m1, m2, m4 ; V 01 12 - punpckhwd m2, m4 ; V 23 34 - pshufd m0, m3, q2121;V 5 6 5 6 - punpcklwd m3, m0 ; V 45 56 - SAVELINE_W4 m0, 0, 1 - SAVELINE_W4 m1, 1, 1 - SAVELINE_W4 m2, 2, 1 - SAVELINE_W4 m3, 3, 1 - ;process low - RESTORELINE_W4 m2, 2, 0 - RESTORELINE_W4 m3, 3, 0 - palignr m4, m3, m2, 4;V 1 2 3 4 - punpcklwd m1, m2, m4 ; V 01 12 - punpckhwd m2, m4 ; V 23 34 - pshufd m0, m3, q2121;V 5 6 5 6 - punpcklwd m3, m0 ; V 45 56 -.hv_w4_loop: - ;process low - pmaddwd m5, m1, subpelv0 ; V a0 b0 - mova m1, m2 - pmaddwd m2, subpelv1; V a1 b1 - paddd m5, m2 - mova m2, m3 - pmaddwd m3, subpelv2; V a2 b2 - paddd m5, m3 - ; - mova m6, [base+subpel_h_shuf4] - movq m4, [srcq+ssq*0] ; 7 - movhps m4, [srcq+ssq*1] ; 7 _ 8 _ - pshufb m4, m6 ;H subpel_h_shuf4 7 ~ 8 ~ - pmaddubsw m4, m7 ;H subpel_filters - phaddw m4, m4 ;H 7 8 7 8 - pmulhrsw m4, w8192reg ;H pw_8192 - palignr m3, m4, m0, 12 ; 6 7 8 7 - mova m0, m4 - punpcklwd m3, m4 ; 67 78 - pmaddwd m4, m3, subpelv3; a3 b3 - paddd m5, d512reg ; pd_512 - paddd m5, m4 - psrad m5, 10 - SAVELINE_W4 m0, 0, 0 - SAVELINE_W4 m1, 1, 0 - SAVELINE_W4 m2, 2, 0 - SAVELINE_W4 m3, 3, 0 - SAVELINE_W4 m5, 5, 0 - ;process high - RESTORELINE_W4 m0, 0, 1 - RESTORELINE_W4 m1, 1, 1 - RESTORELINE_W4 m2, 2, 1 - RESTORELINE_W4 m3, 3, 1 - pmaddwd m5, m1, subpelv0; V a0 b0 - mova m1, m2 - pmaddwd m2, subpelv1; V a1 b1 - paddd m5, m2 - mova m2, m3 - pmaddwd m3, subpelv2; V a2 b2 - paddd m5, m3 - ; - mova m6, [base+subpel_h_shuf4+16] - movq m4, [srcq+ssq*0] ; 7 - movhps m4, [srcq+ssq*1] ; 7 _ 8 _ - pshufb m4, m6 ;H subpel_h_shuf4 7 ~ 8 ~ - pmaddubsw m4, m7 ;H subpel_filters - phaddw m4, m4 ;H 7 8 7 8 - pmulhrsw m4, w8192reg ;H pw_8192 - palignr m3, m4, m0, 12 ; 6 7 8 7 - mova m0, m4 - punpcklwd m3, m4 ; 67 78 - pmaddwd m4, m3, subpelv3; a3 b3 - paddd m5, d512reg ; pd_512 - paddd m5, m4 - psrad m4, m5, 10 - ; - RESTORELINE_W4 m5, 5, 0 - packssdw m5, m4 ; d -> w - packuswb m5, m5 ; w -> b - pshuflw m5, m5, q3120 - lea srcq, [srcq+ssq*2] - movd [dstq+dsq*0], m5 - psrlq m5, 32 - movd [dstq+dsq*1], m5 - lea dstq, [dstq+dsq*2] - sub hd, 2 - SAVELINE_W4 m0, 0, 1 - SAVELINE_W4 m1, 1, 1 - SAVELINE_W4 m2, 2, 1 - SAVELINE_W4 m3, 3, 1 - RESTORELINE_W4 m0, 0, 0 - RESTORELINE_W4 m1, 1, 0 - RESTORELINE_W4 m2, 2, 0 - RESTORELINE_W4 m3, 3, 0 - jg .hv_w4_loop - RET -%undef subpelv0 -%undef subpelv1 -%undef subpelv2 -%undef subpelv3 - ; -.hv_w8: - %assign stack_offset org_stack_offset -%define hv8_line_1 0 -%define hv8_line_2 1 -%define hv8_line_3 2 -%define hv8_line_4 3 -%define hv8_line_6 4 -%macro SAVELINE_W8 2 - mova [rsp+hv8_line_%1*mmsize], %2 -%endmacro -%macro RESTORELINE_W8 2 - mova %2, [rsp+hv8_line_%1*mmsize] -%endmacro - shr mxd, 16 - sub srcq, 3 -%if ARCH_X86_32 - %define base_reg r1 - %define subpelh0 [rsp+mmsize*5] - %define subpelh1 [rsp+mmsize*6] - %define subpelv0 [rsp+mmsize*7] - %define subpelv1 [rsp+mmsize*8] - %define subpelv2 [rsp+mmsize*9] - %define subpelv3 [rsp+mmsize*10] - %define accuv0 [rsp+mmsize*11] - %define accuv1 [rsp+mmsize*12] - movq m1, [base_reg+mxq*8+subpel_filters-put_ssse3] - movzx mxd, ssb - shr ssd, 16 - cmp hd, 6 - cmovs ssd, mxd - movq m5, [base_reg+ssq*8+subpel_filters-put_ssse3] - mov ssq, ssmp - ALLOC_STACK -mmsize*13 -%if STACK_ALIGNMENT < 16 - %define srcm [rsp+mmsize*13+gprsize*1] - %define dsm [rsp+mmsize*13+gprsize*2] - mov r6, [rstk+stack_offset+gprsize*2] - mov dsm, r6 -%endif - pshufd m0, m1, q0000 - pshufd m1, m1, q1111 - punpcklbw m5, m5 - psraw m5, 8 ; sign-extend - pshufd m2, m5, q0000 - pshufd m3, m5, q1111 - pshufd m4, m5, q2222 - pshufd m5, m5, q3333 - mova subpelh0, m0 - mova subpelh1, m1 - mova subpelv0, m2 - mova subpelv1, m3 - mova subpelv2, m4 - mova subpelv3, m5 - lea r6, [ssq*3] - sub srcq, r6 - mov srcm, srcq -%else - ALLOC_STACK mmsize*5, 16 - %define subpelh0 m10 - %define subpelh1 m11 - %define subpelv0 m12 - %define subpelv1 m13 - %define subpelv2 m14 - %define subpelv3 m15 - %define accuv0 m8 - %define accuv1 m9 - movq m0, [base_reg+mxq*8+subpel_filters-put_ssse3] - movzx mxd, myb - shr myd, 16 - cmp hd, 6 - cmovs myd, mxd - movq m1, [base_reg+myq*8+subpel_filters-put_ssse3] - pshufd subpelh0, m0, q0000 - pshufd subpelh1, m0, q1111 - punpcklqdq m1, m1 - punpcklbw m1, m1 - psraw m1, 8 ; sign-extend - pshufd subpelv0, m1, q0000 - pshufd subpelv1, m1, q1111 - pshufd subpelv2, m1, q2222 - pshufd subpelv3, m1, q3333 - lea ss3q, [ssq*3] - sub srcq, ss3q - mov r7, srcq -%endif - lea r6d, [wq-4] - mov r4, dstq - shl r6d, (16 - 2) - mov r6w, hw -.hv_w8_loop0: - movu m4, [srcq+ssq*0] ; 0 = _ _ - movu m5, [srcq+ssq*1] ; 1 = _ _ - lea srcq, [srcq+ssq*2] - ; -%macro HV_H_W8 4-7 ; src/dst, tmp[1-3], shuf[1-3] - %if ARCH_X86_32 - pshufb %3, %1, [base+subpel_h_shufB] - pshufb %4, %1, [base+subpel_h_shufC] - pshufb %1, [base+subpel_h_shufA] - %else - pshufb %3, %1, %6 ; subpel_h_shufB - pshufb %4, %1, %7 ; subpel_h_shufC - pshufb %1, %5 ; subpel_h_shufA - %endif - pmaddubsw %2, %3, subpelh0 ; subpel +0 C0 - pmaddubsw %4, subpelh1; subpel +4 B4 - pmaddubsw %3, subpelh1; C4 - pmaddubsw %1, subpelh0; A0 - paddw %2, %4 ; C0+B4 - paddw %1, %3 ; A0+C4 - phaddw %1, %2 -%endmacro - ; -%if ARCH_X86_64 - mova m7, [base+subpel_h_shufA] - mova m8, [base+subpel_h_shufB] - mova m9, [base+subpel_h_shufC] -%endif - HV_H_W8 m4, m1, m2, m3, m7, m8, m9 ; 0 ~ ~ ~ - HV_H_W8 m5, m1, m2, m3, m7, m8, m9 ; 1 ~ ~ ~ - movu m6, [srcq+ssq*0] ; 2 = _ _ - movu m0, [srcq+ssq*1] ; 3 = _ _ - lea srcq, [srcq+ssq*2] - HV_H_W8 m6, m1, m2, m3, m7, m8, m9 ; 2 ~ ~ ~ - HV_H_W8 m0, m1, m2, m3, m7, m8, m9 ; 3 ~ ~ ~ - ; - mova m7, [base+pw_8192] - pmulhrsw m4, m7 ; H pw_8192 - pmulhrsw m5, m7 ; H pw_8192 - pmulhrsw m6, m7 ; H pw_8192 - pmulhrsw m0, m7 ; H pw_8192 - punpcklwd m1, m4, m5 ; 0 1 ~ - punpcklwd m2, m5, m6 ; 1 2 ~ - punpcklwd m3, m6, m0 ; 2 3 ~ - SAVELINE_W8 1, m1 - SAVELINE_W8 2, m2 - SAVELINE_W8 3, m3 - ; - mova m7, [base+subpel_h_shufA] - movu m4, [srcq+ssq*0] ; 4 = _ _ - movu m5, [srcq+ssq*1] ; 5 = _ _ - lea srcq, [srcq+ssq*2] - movu m6, [srcq+ssq*0] ; 6 = _ _ - HV_H_W8 m4, m1, m2, m3, m7, m8, m9 ; 4 ~ ~ ~ - HV_H_W8 m5, m1, m2, m3, m7, m8, m9 ; 5 ~ ~ ~ - HV_H_W8 m6, m1, m2, m3, m7, m8, m9 ; 6 ~ ~ ~ - mova m7, [base+pw_8192] - pmulhrsw m1, m4, m7 ; H pw_8192 4 ~ - pmulhrsw m2, m5, m7 ; H pw_8192 5 ~ - pmulhrsw m3, m6, m7 ; H pw_8192 6 ~ - punpcklwd m4, m0, m1 ; 3 4 ~ - punpcklwd m5, m1, m2 ; 4 5 ~ - punpcklwd m6, m2, m3 ; 5 6 ~ - ; - SAVELINE_W8 6, m3 - RESTORELINE_W8 1, m1 - RESTORELINE_W8 2, m2 - RESTORELINE_W8 3, m3 -.hv_w8_loop: - ; m8 accu for V a - ; m9 accu for V b - SAVELINE_W8 1, m3 - SAVELINE_W8 2, m4 - SAVELINE_W8 3, m5 - SAVELINE_W8 4, m6 -%if ARCH_X86_32 - pmaddwd m0, m1, subpelv0 ; a0 - pmaddwd m7, m2, subpelv0 ; b0 - pmaddwd m3, subpelv1 ; a1 - pmaddwd m4, subpelv1 ; b1 - paddd m0, m3 - paddd m7, m4 - pmaddwd m5, subpelv2 ; a2 - pmaddwd m6, subpelv2 ; b2 - paddd m0, m5 - paddd m7, m6 - mova m5, [base+pd_512] - paddd m0, m5 ; pd_512 - paddd m7, m5 ; pd_512 - mova accuv0, m0 - mova accuv1, m7 -%else - pmaddwd m8, m1, subpelv0 ; a0 - pmaddwd m9, m2, subpelv0 ; b0 - pmaddwd m3, subpelv1 ; a1 - pmaddwd m4, subpelv1 ; b1 - paddd m8, m3 - paddd m9, m4 - pmaddwd m5, subpelv2 ; a2 - pmaddwd m6, subpelv2 ; b2 - paddd m8, m5 - paddd m9, m6 - mova m7, [base+pd_512] - paddd m8, m7 ; pd_512 - paddd m9, m7 ; pd_512 - mova m7, [base+subpel_h_shufB] - mova m6, [base+subpel_h_shufC] - mova m5, [base+subpel_h_shufA] -%endif - movu m0, [srcq+ssq*1] ; 7 - movu m4, [srcq+ssq*2] ; 8 - lea srcq, [srcq+ssq*2] - HV_H_W8 m0, m1, m2, m3, m5, m7, m6 - HV_H_W8 m4, m1, m2, m3, m5, m7, m6 - mova m5, [base+pw_8192] - pmulhrsw m0, m5 ; H pw_8192 - pmulhrsw m4, m5 ; H pw_8192 - RESTORELINE_W8 6, m6 - punpcklwd m5, m6, m0 ; 6 7 ~ - punpcklwd m6, m0, m4 ; 7 8 ~ - pmaddwd m1, m5, subpelv3 ; a3 - paddd m2, m1, accuv0 - pmaddwd m1, m6, subpelv3 ; b3 - paddd m1, m1, accuv1 ; H + V - psrad m2, 10 - psrad m1, 10 - packssdw m2, m1 ; d -> w - packuswb m2, m1 ; w -> b - movd [dstq+dsq*0], m2 - psrlq m2, 32 -%if ARCH_X86_32 - add dstq, dsm - movd [dstq+dsq*0], m2 - add dstq, dsm -%else - movd [dstq+dsq*1], m2 - lea dstq, [dstq+dsq*2] -%endif - sub hd, 2 - jle .hv_w8_outer - SAVELINE_W8 6, m4 - RESTORELINE_W8 1, m1 - RESTORELINE_W8 2, m2 - RESTORELINE_W8 3, m3 - RESTORELINE_W8 4, m4 - jmp .hv_w8_loop -.hv_w8_outer: - movzx hd, r6w - add r4, 4 - mov dstq, r4 -%if ARCH_X86_32 - mov srcq, srcm - add srcq, 4 - mov srcm, srcq -%else - add r7, 4 - mov srcq, r7 -%endif - sub r6d, 1<<16 - jg .hv_w8_loop0 - RET - -%if ARCH_X86_32 -DECLARE_REG_TMP 1, 2 -%elif WIN64 -DECLARE_REG_TMP 6, 4 -%else -DECLARE_REG_TMP 6, 7 -%endif -%macro PREP_8TAP_FN 3 ; type, type_h, type_v -cglobal prep_8tap_%1 - mov t0d, FILTER_%2 - mov t1d, FILTER_%3 -%ifnidn %1, sharp_smooth ; skip the jump in the last filter - jmp mangle(private_prefix %+ _prep_8tap %+ SUFFIX) -%endif -%endmacro - -PREP_8TAP_FN regular, REGULAR, REGULAR -PREP_8TAP_FN regular_sharp, REGULAR, SHARP -PREP_8TAP_FN regular_smooth, REGULAR, SMOOTH -PREP_8TAP_FN smooth_regular, SMOOTH, REGULAR -PREP_8TAP_FN smooth, SMOOTH, SMOOTH -PREP_8TAP_FN smooth_sharp, SMOOTH, SHARP -PREP_8TAP_FN sharp_regular, SHARP, REGULAR -PREP_8TAP_FN sharp, SHARP, SHARP -PREP_8TAP_FN sharp_smooth, SHARP, SMOOTH - -%if ARCH_X86_32 - %define base_reg r2 - %define base base_reg-prep_ssse3 - %define W32_RESTORE_SSQ mov strideq, stridem -%else - %define base_reg r7 - %define base 0 - %define W32_RESTORE_SSQ -%endif - -cglobal prep_8tap, 1, 9, 0, tmp, src, stride, w, h, mx, my, stride3 -%assign org_stack_offset stack_offset - imul mxd, mxm, 0x010101 - add mxd, t0d ; 8tap_h, mx, 4tap_h - imul myd, mym, 0x010101 - add myd, t1d ; 8tap_v, my, 4tap_v - movsxd wq, wm - movifnidn srcd, srcm - movifnidn hd, hm - LEA base_reg, prep_ssse3 - test mxd, 0xf00 - jnz .h - test myd, 0xf00 - jnz .v - tzcnt wd, wd - movzx wd, word [base_reg+wq*2+table_offset(prep,)] - add wq, base_reg - movifnidn strided, stridem - lea r6, [strideq*3] - %assign stack_offset org_stack_offset -%if WIN64 - pop r8 - pop r7 -%endif - jmp wq -.h: - test myd, 0xf00 - jnz .hv - WIN64_SPILL_XMM 12 - cmp wd, 4 - je .h_w4 - tzcnt wd, wd -%if ARCH_X86_64 - mova m10, [base+subpel_h_shufA] - mova m11, [base+subpel_h_shufB] - mova m9, [base+subpel_h_shufC] -%endif - shr mxd, 16 - sub srcq, 3 - movzx wd, word [base_reg+wq*2+table_offset(prep, _8tap_h)] - movd m5, [base_reg+mxq*8+subpel_filters-prep_ssse3+0] - pshufd m5, m5, q0000 - movd m6, [base_reg+mxq*8+subpel_filters-prep_ssse3+4] - pshufd m6, m6, q0000 - mova m7, [base+pw_8192] - add wq, base_reg - jmp wq -.h_w4: -%if ARCH_X86_32 - and mxd, 0x7f -%else - movzx mxd, mxb -%endif - dec srcq - movd m4, [base_reg+mxq*8+subpel_filters-prep_ssse3+2] - pshufd m4, m4, q0000 - mova m6, [base+pw_8192] - mova m5, [base+subpel_h_shufA] - W32_RESTORE_SSQ -%if ARCH_X86_64 - lea stride3q, [strideq*3] -%endif -.h_w4_loop: - movq m0, [srcq+strideq*0] ; 0 - movq m1, [srcq+strideq*1] ; 1 -%if ARCH_X86_32 - lea srcq, [srcq+strideq*2] - movq m2, [srcq+strideq*0] ; 2 - movq m3, [srcq+strideq*1] ; 3 - lea srcq, [srcq+strideq*2] -%else - movq m2, [srcq+strideq*2] ; 2 - movq m3, [srcq+stride3q ] ; 3 - lea srcq, [srcq+strideq*4] -%endif - pshufb m0, m5 ; subpel_h_shufA - pshufb m1, m5 - pshufb m2, m5 - pshufb m3, m5 - pmaddubsw m0, m4 ; subpel_filters + 2 - pmaddubsw m1, m4 - pmaddubsw m2, m4 - pmaddubsw m3, m4 - phaddw m0, m1 - phaddw m2, m3 - pmulhrsw m0, m6 ; pw_8192 - pmulhrsw m2, m6 ; pw_8192 - mova [tmpq+16*0], m0 - mova [tmpq+16*1], m2 - add tmpq, 32 - sub hd, 4 - jg .h_w4_loop - RET - ; -%macro PREP_8TAP_H 4 ; dst/src, tmp[1-3] -%if ARCH_X86_32 - pshufb %2, %1, [base+subpel_h_shufB] - pshufb %3, %1, [base+subpel_h_shufC] - pshufb %1, [base+subpel_h_shufA] -%else - pshufb %2, %1, m11; subpel_h_shufB - pshufb %3, %1, m9 ; subpel_h_shufC - pshufb %1, m10 ; subpel_h_shufA -%endif - pmaddubsw %4, %2, m5 ; subpel +0 B0 - pmaddubsw %2, m6 ; subpel +4 B4 - pmaddubsw %3, m6 ; subpel +4 C4 - pmaddubsw %1, m5 ; subpel +0 A0 - paddw %3, %4 - paddw %1, %2 - phaddw %1, %3 - pmulhrsw %1, m7 ; 8192 -%endmacro - ; -.h_w8: -%if ARCH_X86_32 - mov r3, r2 - %define base_reg r3 - W32_RESTORE_SSQ -%endif -.h_w8_loop: - movu m0, [srcq+strideq*0] - movu m1, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - PREP_8TAP_H m0, m2, m3, m4 - PREP_8TAP_H m1, m2, m3, m4 - mova [tmpq+16*0], m0 - mova [tmpq+16*1], m1 - add tmpq, 32 - sub hd, 2 - jg .h_w8_loop - RET -.h_w16: - xor r6d, r6d - jmp .h_start -.h_w32: - mov r6, -16*1 - jmp .h_start -.h_w64: - mov r6, -16*3 - jmp .h_start -.h_w128: - mov r6, -16*7 -.h_start: -%if ARCH_X86_32 - mov r3, r2 - %define base_reg r3 -%endif - sub srcq, r6 - mov r5, r6 - W32_RESTORE_SSQ -.h_loop: - movu m0, [srcq+r6+8*0] - movu m1, [srcq+r6+8*1] - PREP_8TAP_H m0, m2, m3, m4 - PREP_8TAP_H m1, m2, m3, m4 - mova [tmpq+16*0], m0 - mova [tmpq+16*1], m1 - add tmpq, 32 - add r6, 16 - jle .h_loop - add srcq, strideq - mov r6, r5 - dec hd - jg .h_loop - RET -%if ARCH_X86_32 - %define base_reg r2 -%endif - -.v: -%if ARCH_X86_32 - mov mxd, myd - and mxd, 0x7f -%else - %assign stack_offset org_stack_offset - WIN64_SPILL_XMM 16 - movzx mxd, myb -%endif - shr myd, 16 - cmp hd, 6 - cmovs myd, mxd - lea myq, [base_reg+myq*8+subpel_filters-prep_ssse3] - mova m2, [base+pw_512] - psrlw m2, m2, 1 ; 0x0100 - mova m7, [base+pw_8192] -%if ARCH_X86_32 - %define subpel0 [rsp+mmsize*0] - %define subpel1 [rsp+mmsize*1] - %define subpel2 [rsp+mmsize*2] - %define subpel3 [rsp+mmsize*3] -%assign regs_used 2 ; use r1 (src) as tmp for stack alignment if needed - ALLOC_STACK -mmsize*4 -%assign regs_used 7 - movd m0, [myq+0] - pshufb m0, m2 - mova subpel0, m0 - movd m0, [myq+2] - pshufb m0, m2 - mova subpel1, m0 - movd m0, [myq+4] - pshufb m0, m2 - mova subpel2, m0 - movd m0, [myq+6] - pshufb m0, m2 - mova subpel3, m0 - mov strideq, [rstk+stack_offset+gprsize*3] - lea strideq, [strideq*3] - sub [rstk+stack_offset+gprsize*2], strideq - mov strideq, [rstk+stack_offset+gprsize*3] - mov srcq, [rstk+stack_offset+gprsize*2] -%else - %define subpel0 m8 - %define subpel1 m9 - %define subpel2 m10 - %define subpel3 m11 - movd subpel0, [myq+0] - pshufb subpel0, m2 - movd subpel1, [myq+2] - pshufb subpel1, m2 - movd subpel2, [myq+4] - pshufb subpel2, m2 - movd subpel3, [myq+6] - pshufb subpel3, m2 - lea stride3q, [strideq*3] - sub srcq, stride3q - cmp wd, 8 - jg .v_w16 - je .v_w8 -%endif -.v_w4: -%if ARCH_X86_32 -%if STACK_ALIGNMENT < mmsize - %define srcm [rsp+mmsize*4+gprsize*1] - %define tmpm [rsp+mmsize*4+gprsize*2] -%endif - mov tmpm, tmpq - mov srcm, srcq - lea r5d, [wq - 4] ; horizontal loop - shl r5d, (16 - 2) ; (wq / 4) << 16 - mov r5w, hw -.v_w4_loop0: -%endif - movd m2, [srcq+strideq*0] ; 0 - movhps m2, [srcq+strideq*2] ; 0 _ 2 - movd m3, [srcq+strideq*1] ; 1 -%if ARCH_X86_32 - lea srcq, [srcq+strideq*2] - movhps m3, [srcq+strideq*1] ; 1 _ 3 - lea srcq, [srcq+strideq*2] -%else - movhps m3, [srcq+stride3q ] ; 1 _ 3 - lea srcq, [srcq+strideq*4] -%endif - pshufd m2, m2, q2020 ; 0 2 0 2 - pshufd m3, m3, q2020 ; 1 3 1 3 - punpckldq m2, m3 ; 0 1 2 3 - movd m3, [srcq+strideq*0] ; 4 - movd m1, [srcq+strideq*1] ; 5 - movd m0, [srcq+strideq*2] ; 6 -%if ARCH_X86_32 - lea srcq, [srcq+strideq*2] - add srcq, strideq -%else - add srcq, stride3q -%endif - punpckldq m3, m1 ; 4 5 _ _ - punpckldq m1, m0 ; 5 6 _ _ - palignr m4, m3, m2, 4 ; 1 2 3 4 - punpcklbw m3, m1 ; 45 56 - punpcklbw m1, m2, m4 ; 01 12 - punpckhbw m2, m4 ; 23 34 -.v_w4_loop: - pmaddubsw m5, m1, subpel0 ; a0 b0 - mova m1, m2 - pmaddubsw m2, subpel1 ; a1 b1 - paddw m5, m2 - mova m2, m3 - pmaddubsw m3, subpel2 ; a2 b2 - paddw m5, m3 - movd m4, [srcq+strideq*0] - punpckldq m3, m0, m4 ; 6 7 _ _ - movd m0, [srcq+strideq*1] - lea srcq, [srcq+strideq*2] - punpckldq m4, m0 ; 7 8 _ _ - punpcklbw m3, m4 ; 67 78 - pmaddubsw m4, m3, subpel3 ; a3 b3 - paddw m5, m4 - pmulhrsw m5, m7 - movq [tmpq+wq*0], m5 - movhps [tmpq+wq*2], m5 - lea tmpq, [tmpq+wq*4] - sub hd, 2 - jg .v_w4_loop -%if ARCH_X86_32 - mov hw, r5w ; reset vertical loop - mov tmpq, tmpm - mov srcq, srcm - add tmpq, 8 - add srcq, 4 - mov tmpm, tmpq - mov srcm, srcq - sub r5d, 1<<16 ; horizontal-- - jg .v_w4_loop0 -%endif - RET - -%if ARCH_X86_64 -.v_w8: -.v_w16: - lea r5d, [wq - 8] ; horizontal loop - mov r8, tmpq - mov r6, srcq - shl r5d, 8 - 3; (wq / 8) << 8 - mov r5b, hb -.v_w8_loop0: - movq m4, [srcq+strideq*0] ; 0 - movq m5, [srcq+strideq*1] ; 1 - lea srcq, [srcq+strideq*2] - movq m6, [srcq+strideq*0] ; 2 - movq m0, [srcq+strideq*1] ; 3 - lea srcq, [srcq+strideq*2] - movq m1, [srcq+strideq*0] ; 4 - movq m2, [srcq+strideq*1] ; 5 - lea srcq, [srcq+strideq*2] ; - movq m3, [srcq+strideq*0] ; 6 - shufpd m4, m0, 0x0c - shufpd m5, m1, 0x0c - punpcklbw m1, m4, m5 ; 01 - punpckhbw m4, m5 ; 34 - shufpd m6, m2, 0x0c - punpcklbw m2, m5, m6 ; 12 - punpckhbw m5, m6 ; 45 - shufpd m0, m3, 0x0c - punpcklbw m3, m6, m0 ; 23 - punpckhbw m6, m0 ; 56 -.v_w8_loop: - movq m12, [srcq+strideq*1] ; 8 - lea srcq, [srcq+strideq*2] - movq m13, [srcq+strideq*0] ; 9 - pmaddubsw m14, m1, subpel0 ; a0 - pmaddubsw m15, m2, subpel0 ; b0 - mova m1, m3 - mova m2, m4 - pmaddubsw m3, subpel1 ; a1 - pmaddubsw m4, subpel1 ; b1 - paddw m14, m3 - paddw m15, m4 - mova m3, m5 - mova m4, m6 - pmaddubsw m5, subpel2 ; a2 - pmaddubsw m6, subpel2 ; b2 - paddw m14, m5 - paddw m15, m6 - shufpd m6, m0, m12, 0x0d - shufpd m0, m12, m13, 0x0c - punpcklbw m5, m6, m0 ; 67 - punpckhbw m6, m0 ; 78 - pmaddubsw m12, m5, subpel3 ; a3 - pmaddubsw m13, m6, subpel3 ; b3 - paddw m14, m12 - paddw m15, m13 - pmulhrsw m14, m7 - pmulhrsw m15, m7 - movu [tmpq+wq*0], xm14 - movu [tmpq+wq*2], xm15 - lea tmpq, [tmpq+wq*4] - sub hd, 2 - jg .v_w8_loop - movzx hd, r5b ; reset vertical loop - add r8, 16 - add r6, 8 - mov tmpq, r8 - mov srcq, r6 - sub r5d, 1<<8 ; horizontal-- - jg .v_w8_loop0 - RET -%endif ;ARCH_X86_64 -%undef subpel0 -%undef subpel1 -%undef subpel2 -%undef subpel3 - -.hv: - %assign stack_offset org_stack_offset - cmp wd, 4 - jg .hv_w8 - and mxd, 0x7f - movd m1, [base_reg+mxq*8+subpel_filters-prep_ssse3+2] -%if ARCH_X86_32 - mov mxd, myd - shr myd, 16 - and mxd, 0x7f - cmp hd, 6 - cmovs myd, mxd - movq m0, [base_reg+myq*8+subpel_filters-prep_ssse3] - mov r5, r2; use as new base - %define base_reg r5 - %assign regs_used 2 - ALLOC_STACK -mmsize*14 - %assign regs_used 7 - mov strideq, [rstk+stack_offset+gprsize*3] - lea strideq, [strideq*3 + 1] - sub [rstk+stack_offset+gprsize*2], strideq - mov strideq, [rstk+stack_offset+gprsize*3] - mov srcq, [rstk+stack_offset+gprsize*2] - %define subpelv0 [rsp+mmsize*0] - %define subpelv1 [rsp+mmsize*1] - %define subpelv2 [rsp+mmsize*2] - %define subpelv3 [rsp+mmsize*3] - punpcklbw m0, m0 - psraw m0, 8 ; sign-extend - pshufd m6, m0, q0000 - mova subpelv0, m6 - pshufd m6, m0, q1111 - mova subpelv1, m6 - pshufd m6, m0, q2222 - mova subpelv2, m6 - pshufd m6, m0, q3333 - mova subpelv3, m6 -%else - movzx mxd, myb - shr myd, 16 - cmp hd, 6 - cmovs myd, mxd - movq m0, [base_reg+myq*8+subpel_filters-prep_ssse3] - ALLOC_STACK mmsize*14, 14 - lea stride3q, [strideq*3] - sub srcq, stride3q - dec srcq - %define subpelv0 m10 - %define subpelv1 m11 - %define subpelv2 m12 - %define subpelv3 m13 - punpcklbw m0, m0 - psraw m0, 8 ; sign-extend - mova m8, [base+pw_8192] - mova m9, [base+pd_32] - pshufd m10, m0, q0000 - pshufd m11, m0, q1111 - pshufd m12, m0, q2222 - pshufd m13, m0, q3333 -%endif - pshufd m7, m1, q0000 -.hv_w4: -%define hv4_line_0_0 4 -%define hv4_line_0_1 5 -%define hv4_line_0_2 6 -%define hv4_line_0_3 7 -%define hv4_line_0_4 8 -%define hv4_line_0_5 9 -%define hv4_line_1_0 10 -%define hv4_line_1_1 11 -%define hv4_line_1_2 12 -%define hv4_line_1_3 13 - ; - ; -%if ARCH_X86_32 - %define w8192reg [base+pw_8192] - %define d32reg [base+pd_32] -%else - %define w8192reg m8 - %define d32reg m9 -%endif - ; lower shuffle 0 1 2 3 4 - mova m6, [base+subpel_h_shuf4] - movq m5, [srcq+strideq*0] ; 0 _ _ _ - movhps m5, [srcq+strideq*1] ; 0 _ 1 _ - movq m4, [srcq+strideq*2] ; 2 _ _ _ -%if ARCH_X86_32 - lea srcq, [srcq+strideq*2] - add srcq, strideq - movhps m4, [srcq+strideq*0] ; 2 _ 3 _ - add srcq, strideq -%else - movhps m4, [srcq+stride3q ] ; 2 _ 3 _ - lea srcq, [srcq+strideq*4] -%endif - pshufb m2, m5, m6 ;H subpel_h_shuf4 0 ~ 1 ~ - pshufb m0, m4, m6 ;H subpel_h_shuf4 2 ~ 3 ~ - pmaddubsw m2, m7 ;H subpel_filters - pmaddubsw m0, m7 ;H subpel_filters - phaddw m2, m0 ;H 0 1 2 3 - pmulhrsw m2, w8192reg ;H pw_8192 - SAVELINE_W4 m2, 2, 0 - ; upper shuffle 2 3 4 5 6 - mova m6, [base+subpel_h_shuf4+16] - pshufb m2, m5, m6 ;H subpel_h_shuf4 0 ~ 1 ~ - pshufb m0, m4, m6 ;H subpel_h_shuf4 2 ~ 3 ~ - pmaddubsw m2, m7 ;H subpel_filters - pmaddubsw m0, m7 ;H subpel_filters - phaddw m2, m0 ;H 0 1 2 3 - pmulhrsw m2, w8192reg ;H pw_8192 - ; - ; lower shuffle - mova m6, [base+subpel_h_shuf4] - movq m5, [srcq+strideq*0] ; 4 _ _ _ - movhps m5, [srcq+strideq*1] ; 4 _ 5 _ - movq m4, [srcq+strideq*2] ; 6 _ _ _ - pshufb m3, m5, m6 ;H subpel_h_shuf4 4 ~ 5 ~ - pshufb m0, m4, m6 ;H subpel_h_shuf4 6 ~ 6 ~ - pmaddubsw m3, m7 ;H subpel_filters - pmaddubsw m0, m7 ;H subpel_filters - phaddw m3, m0 ;H 4 5 6 7 - pmulhrsw m3, w8192reg ;H pw_8192 - SAVELINE_W4 m3, 3, 0 - ; upper shuffle - mova m6, [base+subpel_h_shuf4+16] - pshufb m3, m5, m6 ;H subpel_h_shuf4 4 ~ 5 ~ - pshufb m0, m4, m6 ;H subpel_h_shuf4 6 ~ 6 ~ - pmaddubsw m3, m7 ;H subpel_filters - pmaddubsw m0, m7 ;H subpel_filters - phaddw m3, m0 ;H 4 5 6 7 - pmulhrsw m3, w8192reg ;H pw_8192 - ; -%if ARCH_X86_32 - lea srcq, [srcq+strideq*2] - add srcq, strideq -%else - add srcq, stride3q -%endif - ;process high - palignr m4, m3, m2, 4;V 1 2 3 4 - punpcklwd m1, m2, m4 ; V 01 12 - punpckhwd m2, m4 ; V 23 34 - pshufd m0, m3, q2121;V 5 6 5 6 - punpcklwd m3, m0 ; V 45 56 - SAVELINE_W4 m0, 0, 1 - SAVELINE_W4 m1, 1, 1 - SAVELINE_W4 m2, 2, 1 - SAVELINE_W4 m3, 3, 1 - ;process low - RESTORELINE_W4 m2, 2, 0 - RESTORELINE_W4 m3, 3, 0 - palignr m4, m3, m2, 4;V 1 2 3 4 - punpcklwd m1, m2, m4 ; V 01 12 - punpckhwd m2, m4 ; V 23 34 - pshufd m0, m3, q2121;V 5 6 5 6 - punpcklwd m3, m0 ; V 45 56 -.hv_w4_loop: - ;process low - pmaddwd m5, m1, subpelv0 ; V a0 b0 - mova m1, m2 - pmaddwd m2, subpelv1; V a1 b1 - paddd m5, m2 - mova m2, m3 - pmaddwd m3, subpelv2; V a2 b2 - paddd m5, m3 - ; - mova m6, [base+subpel_h_shuf4] - movq m4, [srcq+strideq*0] ; 7 - movhps m4, [srcq+strideq*1] ; 7 _ 8 _ - pshufb m4, m6 ;H subpel_h_shuf4 7 ~ 8 ~ - pmaddubsw m4, m7 ;H subpel_filters - phaddw m4, m4 ;H 7 8 7 8 - pmulhrsw m4, w8192reg ;H pw_8192 - palignr m3, m4, m0, 12 ; 6 7 8 7 - mova m0, m4 - punpcklwd m3, m4 ; 67 78 - pmaddwd m4, m3, subpelv3; a3 b3 - paddd m5, d32reg ; pd_32 - paddd m5, m4 - psrad m5, 6 - SAVELINE_W4 m0, 0, 0 - SAVELINE_W4 m1, 1, 0 - SAVELINE_W4 m2, 2, 0 - SAVELINE_W4 m3, 3, 0 - SAVELINE_W4 m5, 5, 0 - ;process high - RESTORELINE_W4 m0, 0, 1 - RESTORELINE_W4 m1, 1, 1 - RESTORELINE_W4 m2, 2, 1 - RESTORELINE_W4 m3, 3, 1 - pmaddwd m5, m1, subpelv0; V a0 b0 - mova m1, m2 - pmaddwd m2, subpelv1; V a1 b1 - paddd m5, m2 - mova m2, m3 - pmaddwd m3, subpelv2; V a2 b2 - paddd m5, m3 - ; - mova m6, [base+subpel_h_shuf4+16] - movq m4, [srcq+strideq*0] ; 7 - movhps m4, [srcq+strideq*1] ; 7 _ 8 _ - pshufb m4, m6 ;H subpel_h_shuf4 7 ~ 8 ~ - pmaddubsw m4, m7 ;H subpel_filters - phaddw m4, m4 ;H 7 8 7 8 - pmulhrsw m4, w8192reg ;H pw_8192 - palignr m3, m4, m0, 12 ; 6 7 8 7 - mova m0, m4 - punpcklwd m3, m4 ; 67 78 - pmaddwd m4, m3, subpelv3; a3 b3 - paddd m5, d32reg ; pd_32 - paddd m5, m4 - psrad m4, m5, 6 - ; - RESTORELINE_W4 m5, 5, 0 - packssdw m5, m4 - pshufd m5, m5, q3120 - movu [tmpq], m5 - lea srcq, [srcq+strideq*2] - add tmpq, 16 - sub hd, 2 - SAVELINE_W4 m0, 0, 1 - SAVELINE_W4 m1, 1, 1 - SAVELINE_W4 m2, 2, 1 - SAVELINE_W4 m3, 3, 1 - RESTORELINE_W4 m0, 0, 0 - RESTORELINE_W4 m1, 1, 0 - RESTORELINE_W4 m2, 2, 0 - RESTORELINE_W4 m3, 3, 0 - jg .hv_w4_loop - RET -%undef subpelv0 -%undef subpelv1 -%undef subpelv2 -%undef subpelv3 - ; - - -.hv_w8: - %assign stack_offset org_stack_offset -%define hv8_line_1 0 -%define hv8_line_2 1 -%define hv8_line_3 2 -%define hv8_line_4 3 -%define hv8_line_6 4 - shr mxd, 16 -%if ARCH_X86_32 - %define base_reg r2 - %define subpelh0 [rsp+mmsize*5] - %define subpelh1 [rsp+mmsize*6] - %define subpelv0 [rsp+mmsize*7] - %define subpelv1 [rsp+mmsize*8] - %define subpelv2 [rsp+mmsize*9] - %define subpelv3 [rsp+mmsize*10] - %define accuv0 [rsp+mmsize*11] - %define accuv1 [rsp+mmsize*12] - movq m1, [base_reg+mxq*8+subpel_filters-prep_ssse3] - mov mxd, myd - shr myd, 16 - and mxd, 0x7f - cmp hd, 6 - cmovs myd, mxd - movq m5, [base_reg+myq*8+subpel_filters-prep_ssse3] - ALLOC_STACK -mmsize*13 -%if STACK_ALIGNMENT < mmsize - mov rstk, r2m - %define tmpm [rsp+mmsize*13+gprsize*1] - %define srcm [rsp+mmsize*13+gprsize*2] - %define stridem [rsp+mmsize*13+gprsize*3] - mov stridem, rstk -%endif - mov r6, r2 -%define base_reg r6 - pshufd m0, m1, q0000 - pshufd m1, m1, q1111 - punpcklbw m5, m5 - psraw m5, 8 ; sign-extend - pshufd m2, m5, q0000 - pshufd m3, m5, q1111 - pshufd m4, m5, q2222 - pshufd m5, m5, q3333 - mova subpelh0, m0 - mova subpelh1, m1 - mova subpelv0, m2 - mova subpelv1, m3 - mova subpelv2, m4 - mova subpelv3, m5 - W32_RESTORE_SSQ - lea strided, [strided*3] - sub srcd, strided - sub srcd, 3 - mov srcm, srcd - W32_RESTORE_SSQ -%else - ALLOC_STACK mmsize*5, 16 - %define subpelh0 m10 - %define subpelh1 m11 - %define subpelv0 m12 - %define subpelv1 m13 - %define subpelv2 m14 - %define subpelv3 m15 - %define accuv0 m8 - %define accuv1 m9 - movq m0, [base_reg+mxq*8+subpel_filters-prep_ssse3] - movzx mxd, myb - shr myd, 16 - cmp hd, 6 - cmovs myd, mxd - movq m1, [base_reg+myq*8+subpel_filters-prep_ssse3] - pshufd subpelh0, m0, q0000 - pshufd subpelh1, m0, q1111 - punpcklbw m1, m1 - psraw m1, 8 ; sign-extend - pshufd subpelv0, m1, q0000 - pshufd subpelv1, m1, q1111 - pshufd subpelv2, m1, q2222 - pshufd subpelv3, m1, q3333 - lea stride3q, [strideq*3] - sub srcq, 3 - sub srcq, stride3q - mov r6, srcq -%endif - lea r5d, [wq-4] -%if ARCH_X86_64 - mov r8, tmpq -%else - mov tmpm, tmpq -%endif - shl r5d, (16 - 2) - mov r5w, hw -.hv_w8_loop0: - movu m4, [srcq+strideq*0] ; 0 = _ _ - movu m5, [srcq+strideq*1] ; 1 = _ _ - lea srcq, [srcq+strideq*2] -%if ARCH_X86_64 - mova m7, [base+subpel_h_shufA] - mova m8, [base+subpel_h_shufB] - mova m9, [base+subpel_h_shufC] -%endif - HV_H_W8 m4, m1, m2, m3, m7, m8, m9 ; 0 ~ ~ ~ - HV_H_W8 m5, m1, m2, m3, m7, m8, m9 ; 1 ~ ~ ~ - movu m6, [srcq+strideq*0] ; 2 = _ _ - movu m0, [srcq+strideq*1] ; 3 = _ _ - lea srcq, [srcq+strideq*2] - HV_H_W8 m6, m1, m2, m3, m7, m8, m9 ; 2 ~ ~ ~ - HV_H_W8 m0, m1, m2, m3, m7, m8, m9 ; 3 ~ ~ ~ - ; - mova m7, [base+pw_8192] - pmulhrsw m4, m7 ; H pw_8192 - pmulhrsw m5, m7 ; H pw_8192 - pmulhrsw m6, m7 ; H pw_8192 - pmulhrsw m0, m7 ; H pw_8192 - punpcklwd m1, m4, m5 ; 0 1 ~ - punpcklwd m2, m5, m6 ; 1 2 ~ - punpcklwd m3, m6, m0 ; 2 3 ~ - SAVELINE_W8 1, m1 - SAVELINE_W8 2, m2 - SAVELINE_W8 3, m3 - ; - mova m7, [base+subpel_h_shufA] - movu m4, [srcq+strideq*0] ; 4 = _ _ - movu m5, [srcq+strideq*1] ; 5 = _ _ - lea srcq, [srcq+strideq*2] - movu m6, [srcq+strideq*0] ; 6 = _ _ - HV_H_W8 m4, m1, m2, m3, m7, m8, m9 ; 4 ~ ~ ~ - HV_H_W8 m5, m1, m2, m3, m7, m8, m9 ; 5 ~ ~ ~ - HV_H_W8 m6, m1, m2, m3, m7, m8, m9 ; 6 ~ ~ ~ - mova m7, [base+pw_8192] - pmulhrsw m1, m4, m7 ; H pw_8192 4 ~ - pmulhrsw m2, m5, m7 ; H pw_8192 5 ~ - pmulhrsw m3, m6, m7 ; H pw_8192 6 ~ - punpcklwd m4, m0, m1 ; 3 4 ~ - punpcklwd m5, m1, m2 ; 4 5 ~ - punpcklwd m6, m2, m3 ; 5 6 ~ - ; - SAVELINE_W8 6, m3 - RESTORELINE_W8 1, m1 - RESTORELINE_W8 2, m2 - RESTORELINE_W8 3, m3 -.hv_w8_loop: - ; m8 accu for V a - ; m9 accu for V b - SAVELINE_W8 1, m3 - SAVELINE_W8 2, m4 - SAVELINE_W8 3, m5 - SAVELINE_W8 4, m6 -%if ARCH_X86_32 - pmaddwd m0, m1, subpelv0 ; a0 - pmaddwd m7, m2, subpelv0 ; b0 - pmaddwd m3, subpelv1 ; a1 - pmaddwd m4, subpelv1 ; b1 - paddd m0, m3 - paddd m7, m4 - pmaddwd m5, subpelv2 ; a2 - pmaddwd m6, subpelv2 ; b2 - paddd m0, m5 - paddd m7, m6 - mova m5, [base+pd_32] - paddd m0, m5 ; pd_512 - paddd m7, m5 ; pd_512 - mova accuv0, m0 - mova accuv1, m7 -%else - pmaddwd m8, m1, subpelv0 ; a0 - pmaddwd m9, m2, subpelv0 ; b0 - pmaddwd m3, subpelv1 ; a1 - pmaddwd m4, subpelv1 ; b1 - paddd m8, m3 - paddd m9, m4 - pmaddwd m5, subpelv2 ; a2 - pmaddwd m6, subpelv2 ; b2 - paddd m8, m5 - paddd m9, m6 - mova m7, [base+pd_32] - paddd m8, m7 ; pd_512 - paddd m9, m7 ; pd_512 - mova m7, [base+subpel_h_shufB] - mova m6, [base+subpel_h_shufC] - mova m5, [base+subpel_h_shufA] -%endif - movu m0, [srcq+strideq*1] ; 7 - movu m4, [srcq+strideq*2] ; 8 - lea srcq, [srcq+strideq*2] - HV_H_W8 m0, m1, m2, m3, m5, m7, m6 - HV_H_W8 m4, m1, m2, m3, m5, m7, m6 - mova m5, [base+pw_8192] - pmulhrsw m0, m5 ; H pw_8192 - pmulhrsw m4, m5 ; H pw_8192 - RESTORELINE_W8 6, m6 - punpcklwd m5, m6, m0 ; 6 7 ~ - punpcklwd m6, m0, m4 ; 7 8 ~ - pmaddwd m1, m5, subpelv3 ; a3 - paddd m2, m1, accuv0 - pmaddwd m1, m6, subpelv3 ; b3 - paddd m1, m1, accuv1 ; H + V - psrad m2, 6 - psrad m1, 6 - packssdw m2, m1 ; d -> w - movq [tmpq+wq*0], m2 - movhps [tmpq+wq*2], m2 - lea tmpq, [tmpq+wq*4] - sub hd, 2 - jle .hv_w8_outer - SAVELINE_W8 6, m4 - RESTORELINE_W8 1, m1 - RESTORELINE_W8 2, m2 - RESTORELINE_W8 3, m3 - RESTORELINE_W8 4, m4 - jmp .hv_w8_loop -.hv_w8_outer: - movzx hd, r5w -%if ARCH_X86_32 - add dword tmpm, 8 - mov tmpq, tmpm - mov srcq, srcm - add srcq, 4 - mov srcm, srcq -%else - add r8, 8 - mov tmpq, r8 - add r6, 4 - mov srcq, r6 -%endif - sub r5d, 1<<16 - jg .hv_w8_loop0 - RET - -%if ARCH_X86_32 - %macro SAVE_ALPHA_BETA 0 - mov alpham, alphad - mov betam, betad - %endmacro - - %macro SAVE_DELTA_GAMMA 0 - mov deltam, deltad - mov gammam, gammad - %endmacro - - %macro LOAD_ALPHA_BETA_MX 0 - mov mym, myd - mov alphad, alpham - mov betad, betam - mov mxd, mxm - %endmacro - - %macro LOAD_DELTA_GAMMA_MY 0 - mov mxm, mxd - mov deltad, deltam - mov gammad, gammam - mov myd, mym - %endmacro - - %define PIC_reg r2 - %define PIC_base_offset $$ - %define PIC_sym(sym) (PIC_reg+(sym)-PIC_base_offset) -%else - %define SAVE_ALPHA_BETA - %define SAVE_DELTA_GAMMA - %define PIC_sym(sym) sym -%endif - -%if ARCH_X86_32 - %if STACK_ALIGNMENT < required_stack_alignment - %assign copy_args 8*4 - %else - %assign copy_args 0 - %endif -%endif - -%macro RELOC_ARGS 0 - %if copy_args - mov r0, r0m - mov r1, r1m - mov r2, r2m - mov r3, r3m - mov r5, r5m - mov dstm, r0 - mov dsm, r1 - mov srcm, r2 - mov ssm, r3 - mov mxm, r5 - mov r0, r6m - mov mym, r0 - %endif -%endmacro - -%macro BLENDHWDW 2 ; blend high words from dwords, src1, src2 - %if cpuflag(sse4) - pblendw %1, %2, 0xAA - %else - pand %2, m10 - por %1, %2 - %endif -%endmacro - -%macro WARP_V 10 ; dst0, dst1, 0, 2, 4, 6, 1, 3, 5, 7 - ; Can be done using gathers, but that's terribly slow on many CPU:s - %if ARCH_X86_32 - %define m8 m4 - %define m9 m5 - %define m14 m6 - %define m15 m7 - %define m11 m7 - %endif - %if notcpuflag(ssse3) || ARCH_X86_32 - pxor m11, m11 - %endif - lea tmp1d, [myq+deltaq*4] - lea tmp2d, [myq+deltaq*1] - shr myd, 10 - shr tmp1d, 10 - movq m2, [filterq+myq *8] ; a - movq m8, [filterq+tmp1q*8] ; e - lea tmp1d, [tmp2q+deltaq*4] - lea myd, [tmp2q+deltaq*1] - shr tmp2d, 10 - shr tmp1d, 10 - movq m3, [filterq+tmp2q*8] ; b - movq m0, [filterq+tmp1q*8] ; f - punpcklwd m2, m3 - punpcklwd m8, m0 - lea tmp1d, [myq+deltaq*4] - lea tmp2d, [myq+deltaq*1] - shr myd, 10 - shr tmp1d, 10 - movq m0, [filterq+myq *8] ; c - movq m9, [filterq+tmp1q*8] ; g - lea tmp1d, [tmp2q+deltaq*4] - lea myd, [tmp2q+gammaq] ; my += gamma - shr tmp2d, 10 - shr tmp1d, 10 - movq m3, [filterq+tmp2q*8] ; d - movq m1, [filterq+tmp1q*8] ; h - punpcklwd m0, m3 - punpcklwd m9, m1 - punpckldq m1, m2, m0 - punpckhdq m2, m0 - punpcklbw m0, m11, m1 ; a0 a2 b0 b2 c0 c2 d0 d2 << 8 - punpckhbw m3, m11, m1 ; a4 a6 b4 b6 c4 c6 d4 d6 << 8 - punpcklbw m1, m11, m2 ; a1 a3 b1 b3 c1 c3 d1 d3 << 8 - punpckhbw m14, m11, m2 ; a5 a7 b5 b7 c5 c7 d5 d7 << 8 - pmaddwd m0, %3 - pmaddwd m3, %5 - pmaddwd m1, %7 - pmaddwd m14, %9 - paddd m0, m3 - paddd m1, m14 - paddd m0, m1 - mova %1, m0 - %if ARCH_X86_64 - SWAP m3, m14 - %endif - punpckldq m0, m8, m9 - punpckhdq m8, m9 - punpcklbw m1, m11, m0 ; e0 e2 f0 f2 g0 g2 h0 h2 << 8 - punpckhbw m14, m11, m0 ; e4 e6 f4 f6 g4 g6 h4 h6 << 8 - punpcklbw m2, m11, m8 ; e1 e3 f1 f3 g1 g3 h1 h3 << 8 - punpckhbw m15, m11, m8 ; e5 e7 f5 f7 g5 g7 h5 h7 << 8 - pmaddwd m1, %4 - pmaddwd m14, %6 - pmaddwd m2, %8 - pmaddwd m15, %10 - paddd m1, m14 - paddd m2, m15 - paddd m1, m2 - mova %2, m1 - %if ARCH_X86_64 - SWAP m14, m3 - %endif -%endmacro - -%if ARCH_X86_64 - %define counterd r4d -%else - %if copy_args == 0 - %define counterd dword r4m - %else - %define counterd dword [esp+stack_size-4*7] - %endif -%endif - -%macro WARP_AFFINE_8X8T 0 -%if ARCH_X86_64 -cglobal warp_affine_8x8t, 6, 14, 16, 0x90, tmp, ts -%else -cglobal warp_affine_8x8t, 0, 7, 16, -0x130-copy_args, tmp, ts - %if copy_args - %define tmpm [esp+stack_size-4*1] - %define tsm [esp+stack_size-4*2] - %endif -%endif - call mangle(private_prefix %+ _warp_affine_8x8_%+cpuname).main -.loop: -%if ARCH_X86_32 - %define m12 m4 - %define m13 m5 - %define m14 m6 - %define m15 m7 - mova m12, [esp+0xC0] - mova m13, [esp+0xD0] - mova m14, [esp+0xE0] - mova m15, [esp+0xF0] -%endif -%if cpuflag(ssse3) - psrad m12, 13 - psrad m13, 13 - psrad m14, 13 - psrad m15, 13 - packssdw m12, m13 - packssdw m14, m15 - mova m13, [PIC_sym(pw_8192)] - pmulhrsw m12, m13 ; (x + (1 << 6)) >> 7 - pmulhrsw m14, m13 -%else - %if ARCH_X86_32 - %define m10 m0 - %endif - mova m10, [PIC_sym(pd_16384)] - paddd m12, m10 - paddd m13, m10 - paddd m14, m10 - paddd m15, m10 - psrad m12, 15 - psrad m13, 15 - psrad m14, 15 - psrad m15, 15 - packssdw m12, m13 - packssdw m14, m15 -%endif - mova [tmpq+tsq*0], m12 - mova [tmpq+tsq*2], m14 - dec counterd - jz mangle(private_prefix %+ _warp_affine_8x8_%+cpuname).end -%if ARCH_X86_32 - mov tmpm, tmpd - mov r0, [esp+0x100] - mov r1, [esp+0x104] -%endif - call mangle(private_prefix %+ _warp_affine_8x8_%+cpuname).main2 - lea tmpq, [tmpq+tsq*4] - jmp .loop -%endmacro - -%macro WARP_AFFINE_8X8 0 -%if ARCH_X86_64 -cglobal warp_affine_8x8, 6, 14, 16, 0x90, \ - dst, ds, src, ss, abcd, mx, tmp2, alpha, beta, \ - filter, tmp1, delta, my, gamma -%else -cglobal warp_affine_8x8, 0, 7, 16, -0x130-copy_args, \ - dst, ds, src, ss, abcd, mx, tmp2, alpha, beta, \ - filter, tmp1, delta, my, gamma - %define alphaq r0 - %define alphad r0 - %define alpham [esp+gprsize+0x100] - %define betaq r1 - %define betad r1 - %define betam [esp+gprsize+0x104] - %define deltaq r0 - %define deltad r0 - %define deltam [esp+gprsize+0x108] - %define gammaq r1 - %define gammad r1 - %define gammam [esp+gprsize+0x10C] - %define filterq r3 - %define tmp1q r4 - %define tmp1d r4 - %define tmp1m [esp+gprsize+0x110] - %define myq r5 - %define myd r5 - %define mym r6m - %if copy_args - %define dstm [esp+stack_size-4*1] - %define dsm [esp+stack_size-4*2] - %define srcm [esp+stack_size-4*3] - %define ssm [esp+stack_size-4*4] - %define mxm [esp+stack_size-4*5] - %define mym [esp+stack_size-4*6] - %endif -%endif - call .main - jmp .start -.loop: -%if ARCH_X86_32 - mov dstm, dstd - mov alphad, [esp+0x100] - mov betad, [esp+0x104] -%endif - call .main2 - lea dstq, [dstq+dsq*2] -.start: -%if notcpuflag(sse4) - %if cpuflag(ssse3) - %define roundval pw_8192 - %else - %define roundval pd_262144 - %endif - %if ARCH_X86_64 - mova m10, [PIC_sym(roundval)] - %else - %define m10 [PIC_sym(roundval)] - %endif -%endif -%if ARCH_X86_32 - %define m12 m5 - %define m13 m6 - mova m12, [esp+0xC0] - mova m13, [esp+0xD0] -%endif -%if cpuflag(sse4) - %if ARCH_X86_32 - %define m11 m4 - pxor m11, m11 - %endif - psrad m12, 18 - psrad m13, 18 - packusdw m12, m13 - pavgw m12, m11 ; (x + (1 << 10)) >> 11 -%else - %if cpuflag(ssse3) - psrad m12, 17 - psrad m13, 17 - packssdw m12, m13 - pmulhrsw m12, m10 - %else - paddd m12, m10 - paddd m13, m10 - psrad m12, 19 - psrad m13, 19 - packssdw m12, m13 - %endif -%endif -%if ARCH_X86_32 - %define m14 m6 - %define m15 m7 - mova m14, [esp+0xE0] - mova m15, [esp+0xF0] -%endif -%if cpuflag(sse4) - psrad m14, 18 - psrad m15, 18 - packusdw m14, m15 - pavgw m14, m11 ; (x + (1 << 10)) >> 11 -%else - %if cpuflag(ssse3) - psrad m14, 17 - psrad m15, 17 - packssdw m14, m15 - pmulhrsw m14, m10 - %else - paddd m14, m10 - paddd m15, m10 - psrad m14, 19 - psrad m15, 19 - packssdw m14, m15 - %endif -%endif - packuswb m12, m14 - movq [dstq+dsq*0], m12 - movhps [dstq+dsq*1], m12 - dec counterd - jg .loop -.end: - RET -ALIGN function_align -.main: -%assign stack_offset stack_offset+gprsize -%if ARCH_X86_32 - %assign stack_size stack_size+4 - %if copy_args - %assign stack_offset stack_offset-4 - %endif - RELOC_ARGS - LEA PIC_reg, $$ - %define PIC_mem [esp+gprsize+0x114] - mov abcdd, abcdm - %if copy_args == 0 - mov ssd, ssm - mov mxd, mxm - %endif - mov PIC_mem, PIC_reg - mov srcd, srcm -%endif - movsx deltad, word [abcdq+2*2] - movsx gammad, word [abcdq+2*3] - lea tmp1d, [deltaq*3] - sub gammad, tmp1d ; gamma -= delta*3 - SAVE_DELTA_GAMMA -%if ARCH_X86_32 - mov abcdd, abcdm -%endif - movsx alphad, word [abcdq+2*0] - movsx betad, word [abcdq+2*1] - lea tmp1q, [ssq*3+3] - add mxd, 512+(64<<10) - lea tmp2d, [alphaq*3] - sub srcq, tmp1q ; src -= src_stride*3 + 3 -%if ARCH_X86_32 - mov srcm, srcd - mov PIC_reg, PIC_mem -%endif - sub betad, tmp2d ; beta -= alpha*3 - lea filterq, [PIC_sym(mc_warp_filter)] -%if ARCH_X86_64 - mov myd, r6m - %if cpuflag(ssse3) - pxor m11, m11 - %endif -%endif - call .h - psrld m2, m0, 16 - psrld m3, m1, 16 -%if ARCH_X86_32 - %if notcpuflag(ssse3) - mova [esp+gprsize+0x00], m2 - %endif - mova [esp+gprsize+0x10], m3 -%endif - call .h - psrld m4, m0, 16 - psrld m5, m1, 16 -%if ARCH_X86_32 - mova [esp+gprsize+0x20], m4 - mova [esp+gprsize+0x30], m5 -%endif - call .h -%if ARCH_X86_64 - %define blendmask [rsp+gprsize+0x80] -%else - %if notcpuflag(ssse3) - mova m2, [esp+gprsize+0x00] - %endif - mova m3, [esp+gprsize+0x10] - %define blendmask [esp+gprsize+0x120] - %define m10 m7 -%endif - pcmpeqd m10, m10 - pslld m10, 16 - mova blendmask, m10 - BLENDHWDW m2, m0 ; 0 - BLENDHWDW m3, m1 ; 2 - mova [rsp+gprsize+0x00], m2 - mova [rsp+gprsize+0x10], m3 - call .h -%if ARCH_X86_32 - mova m4, [esp+gprsize+0x20] - mova m5, [esp+gprsize+0x30] -%endif - mova m10, blendmask - BLENDHWDW m4, m0 ; 1 - BLENDHWDW m5, m1 ; 3 - mova [rsp+gprsize+0x20], m4 - mova [rsp+gprsize+0x30], m5 - call .h -%if ARCH_X86_32 - %if notcpuflag(ssse3) - mova m2, [esp+gprsize+0x00] - %endif - mova m3, [esp+gprsize+0x10] - %define m10 m5 -%endif - psrld m6, m2, 16 - psrld m7, m3, 16 - mova m10, blendmask - BLENDHWDW m6, m0 ; 2 - BLENDHWDW m7, m1 ; 4 - mova [rsp+gprsize+0x40], m6 - mova [rsp+gprsize+0x50], m7 - call .h -%if ARCH_X86_32 - mova m4, [esp+gprsize+0x20] - mova m5, [esp+gprsize+0x30] -%endif - psrld m2, m4, 16 - psrld m3, m5, 16 - mova m10, blendmask - BLENDHWDW m2, m0 ; 3 - BLENDHWDW m3, m1 ; 5 - mova [rsp+gprsize+0x60], m2 - mova [rsp+gprsize+0x70], m3 - call .h -%if ARCH_X86_32 - mova m6, [esp+gprsize+0x40] - mova m7, [esp+gprsize+0x50] - %define m10 m7 -%endif - psrld m4, m6, 16 - psrld m5, m7, 16 - mova m10, blendmask - BLENDHWDW m4, m0 ; 4 - BLENDHWDW m5, m1 ; 6 -%if ARCH_X86_64 - add myd, 512+(64<<10) - mova m6, m2 - mova m7, m3 -%else - mova [esp+gprsize+0x80], m4 - mova [esp+gprsize+0x90], m5 - add dword mym, 512+(64<<10) -%endif - mov counterd, 4 - SAVE_ALPHA_BETA -.main2: - call .h -%if ARCH_X86_32 - mova m6, [esp+gprsize+0x60] - mova m7, [esp+gprsize+0x70] - %define m10 m5 -%endif - psrld m6, 16 - psrld m7, 16 - mova m10, blendmask - BLENDHWDW m6, m0 ; 5 - BLENDHWDW m7, m1 ; 7 -%if ARCH_X86_64 - WARP_V m12, m13, [rsp+gprsize+0x00], [rsp+gprsize+0x10], \ - m4, m5, \ - [rsp+gprsize+0x20], [rsp+gprsize+0x30], \ - m6, m7 -%else - mova [esp+gprsize+0xA0], m6 - mova [esp+gprsize+0xB0], m7 - LOAD_DELTA_GAMMA_MY - WARP_V [esp+gprsize+0xC0], [esp+gprsize+0xD0], \ - [esp+gprsize+0x00], [esp+gprsize+0x10], \ - [esp+gprsize+0x80], [esp+gprsize+0x90], \ - [esp+gprsize+0x20], [esp+gprsize+0x30], \ - [esp+gprsize+0xA0], [esp+gprsize+0xB0] - LOAD_ALPHA_BETA_MX -%endif - call .h - mova m2, [rsp+gprsize+0x40] - mova m3, [rsp+gprsize+0x50] -%if ARCH_X86_32 - mova m4, [rsp+gprsize+0x80] - mova m5, [rsp+gprsize+0x90] - %define m10 m7 -%endif - mova [rsp+gprsize+0x00], m2 - mova [rsp+gprsize+0x10], m3 - mova [rsp+gprsize+0x40], m4 - mova [rsp+gprsize+0x50], m5 - psrld m4, 16 - psrld m5, 16 - mova m10, blendmask - BLENDHWDW m4, m0 ; 6 - BLENDHWDW m5, m1 ; 8 -%if ARCH_X86_64 - WARP_V m14, m15, [rsp+gprsize+0x20], [rsp+gprsize+0x30], \ - m6, m7, \ - [rsp+gprsize+0x00], [rsp+gprsize+0x10], \ - m4, m5 -%else - mova [esp+gprsize+0x80], m4 - mova [esp+gprsize+0x90], m5 - LOAD_DELTA_GAMMA_MY - WARP_V [esp+gprsize+0xE0], [esp+gprsize+0xF0], \ - [esp+gprsize+0x20], [esp+gprsize+0x30], \ - [esp+gprsize+0xA0], [esp+gprsize+0xB0], \ - [esp+gprsize+0x00], [esp+gprsize+0x10], \ - [esp+gprsize+0x80], [esp+gprsize+0x90] - mov mym, myd - mov dstd, dstm - mov dsd, dsm - mov mxd, mxm -%endif - mova m2, [rsp+gprsize+0x60] - mova m3, [rsp+gprsize+0x70] -%if ARCH_X86_32 - mova m6, [esp+gprsize+0xA0] - mova m7, [esp+gprsize+0xB0] -%endif - mova [rsp+gprsize+0x20], m2 - mova [rsp+gprsize+0x30], m3 - mova [rsp+gprsize+0x60], m6 - mova [rsp+gprsize+0x70], m7 - ret -ALIGN function_align -.h: -%if ARCH_X86_32 - %define m8 m3 - %define m9 m4 - %define m10 m5 - %define m14 m6 - %define m15 m7 -%endif - lea tmp1d, [mxq+alphaq*4] - lea tmp2d, [mxq+alphaq*1] -%if ARCH_X86_32 - %assign stack_offset stack_offset+4 - %assign stack_size stack_size+4 - %define PIC_mem [esp+gprsize*2+0x114] - mov PIC_mem, PIC_reg - mov srcd, srcm -%endif - movu m10, [srcq] -%if ARCH_X86_32 - add srcd, ssm - mov srcm, srcd - mov PIC_reg, PIC_mem -%else - add srcq, ssq -%endif - shr mxd, 10 - shr tmp1d, 10 - movq m1, [filterq+mxq *8] ; 0 X - movq m8, [filterq+tmp1q*8] ; 4 X - lea tmp1d, [tmp2q+alphaq*4] - lea mxd, [tmp2q+alphaq*1] - shr tmp2d, 10 - shr tmp1d, 10 - movhps m1, [filterq+tmp2q*8] ; 0 1 - movhps m8, [filterq+tmp1q*8] ; 4 5 - lea tmp1d, [mxq+alphaq*4] - lea tmp2d, [mxq+alphaq*1] - shr mxd, 10 - shr tmp1d, 10 -%if cpuflag(ssse3) - movq m14, [filterq+mxq *8] ; 2 X - movq m9, [filterq+tmp1q*8] ; 6 X - lea tmp1d, [tmp2q+alphaq*4] - lea mxd, [tmp2q+betaq] ; mx += beta - shr tmp2d, 10 - shr tmp1d, 10 - movhps m14, [filterq+tmp2q*8] ; 2 3 - movhps m9, [filterq+tmp1q*8] ; 6 7 - pshufb m0, m10, [PIC_sym(warp_8x8_shufA)] - pmaddubsw m0, m1 - pshufb m1, m10, [PIC_sym(warp_8x8_shufB)] - pmaddubsw m1, m8 - pshufb m15, m10, [PIC_sym(warp_8x8_shufC)] - pmaddubsw m15, m14 - pshufb m10, m10, [PIC_sym(warp_8x8_shufD)] - pmaddubsw m10, m9 - phaddw m0, m15 - phaddw m1, m10 -%else - %if ARCH_X86_32 - %define m11 m2 - %endif - pcmpeqw m0, m0 - psrlw m14, m0, 8 - psrlw m15, m10, 8 ; 01 03 05 07 09 11 13 15 - pand m14, m10 ; 00 02 04 06 08 10 12 14 - packuswb m14, m15 ; 00 02 04 06 08 10 12 14 01 03 05 07 09 11 13 15 - psrldq m9, m0, 4 - pshufd m0, m14, q0220 - pand m0, m9 - psrldq m14, 1 ; 02 04 06 08 10 12 14 01 03 05 07 09 11 13 15 __ - pslldq m15, m14, 12 - por m0, m15 ; shufA - psrlw m15, m0, 8 - psraw m11, m1, 8 - psllw m0, 8 - psllw m1, 8 - psrlw m0, 8 - psraw m1, 8 - pmullw m15, m11 - pmullw m0, m1 - paddw m0, m15 ; pmaddubsw m0, m1 - pshufd m15, m14, q0220 - pand m15, m9 - psrldq m14, 1 ; 04 06 08 10 12 14 01 03 05 07 09 11 13 15 __ __ - pslldq m1, m14, 12 - por m15, m1 ; shufC - pshufd m1, m14, q0220 - pand m1, m9 - psrldq m14, 1 ; 06 08 10 12 14 01 03 05 07 09 11 13 15 __ __ __ - pslldq m11, m14, 12 - por m1, m11 ; shufB - pshufd m10, m14, q0220 - pand m10, m9 - psrldq m14, 1 ; 08 10 12 14 01 03 05 07 09 11 13 15 __ __ __ __ - pslldq m14, m14, 12 - por m10, m14 ; shufD - psrlw m9, m1, 8 - psraw m11, m8, 8 - psllw m1, 8 - psllw m8, 8 - psrlw m1, 8 - psraw m8, 8 - pmullw m9, m11 - pmullw m1, m8 - paddw m1, m9 ; pmaddubsw m1, m8 - movq m14, [filterq+mxq *8] ; 2 X - movq m9, [filterq+tmp1q*8] ; 6 X - lea tmp1d, [tmp2q+alphaq*4] - lea mxd, [tmp2q+betaq] ; mx += beta - shr tmp2d, 10 - shr tmp1d, 10 - movhps m14, [filterq+tmp2q*8] ; 2 3 - movhps m9, [filterq+tmp1q*8] ; 6 7 - psrlw m8, m15, 8 - psraw m11, m14, 8 - psllw m15, 8 - psllw m14, 8 - psrlw m15, 8 - psraw m14, 8 - pmullw m8, m11 - pmullw m15, m14 - paddw m15, m8 ; pmaddubsw m15, m14 - psrlw m8, m10, 8 - psraw m11, m9, 8 - psllw m10, 8 - psllw m9, 8 - psrlw m10, 8 - psraw m9, 8 - pmullw m8, m11 - pmullw m10, m9 - paddw m10, m8 ; pmaddubsw m10, m9 - pslld m8, m0, 16 - pslld m9, m1, 16 - pslld m14, m15, 16 - pslld m11, m10, 16 - paddw m0, m8 - paddw m1, m9 - paddw m15, m14 - paddw m10, m11 - psrad m0, 16 - psrad m1, 16 - psrad m15, 16 - psrad m10, 16 - packssdw m0, m15 ; phaddw m0, m15 - packssdw m1, m10 ; phaddw m1, m10 -%endif - mova m14, [PIC_sym(pw_8192)] - mova m9, [PIC_sym(pd_32768)] - pmaddwd m0, m14 ; 17-bit intermediate, upshifted by 13 - pmaddwd m1, m14 - paddd m0, m9 ; rounded 14-bit result in upper 16 bits of dword - paddd m1, m9 - ret -%endmacro - -INIT_XMM sse4 -WARP_AFFINE_8X8 -WARP_AFFINE_8X8T - -INIT_XMM ssse3 -WARP_AFFINE_8X8 -WARP_AFFINE_8X8T - -INIT_XMM sse2 -WARP_AFFINE_8X8 -WARP_AFFINE_8X8T - -INIT_XMM ssse3 - -%if WIN64 -DECLARE_REG_TMP 6, 4 -%else -DECLARE_REG_TMP 6, 7 -%endif - -%macro BIDIR_FN 1 ; op - %1 0 - lea stride3q, [strideq*3] - jmp wq -.w4_loop: - %1_INC_PTR 2 - %1 0 - lea dstq, [dstq+strideq*4] -.w4: ; tile 4x - movd [dstq ], m0 ; copy dw[0] - pshuflw m1, m0, q1032 ; swap dw[1] and dw[0] - movd [dstq+strideq*1], m1 ; copy dw[1] - punpckhqdq m0, m0 ; swap dw[3,2] with dw[1,0] - movd [dstq+strideq*2], m0 ; dw[2] - psrlq m0, 32 ; shift right in dw[3] - movd [dstq+stride3q ], m0 ; copy - sub hd, 4 - jg .w4_loop - RET -.w8_loop: - %1_INC_PTR 2 - %1 0 - lea dstq, [dstq+strideq*2] -.w8: - movq [dstq ], m0 - movhps [dstq+strideq*1], m0 - sub hd, 2 - jg .w8_loop - RET -.w16_loop: - %1_INC_PTR 2 - %1 0 - lea dstq, [dstq+strideq] -.w16: - mova [dstq ], m0 - dec hd - jg .w16_loop - RET -.w32_loop: - %1_INC_PTR 4 - %1 0 - lea dstq, [dstq+strideq] -.w32: - mova [dstq ], m0 - %1 2 - mova [dstq + 16 ], m0 - dec hd - jg .w32_loop - RET -.w64_loop: - %1_INC_PTR 8 - %1 0 - add dstq, strideq -.w64: - %assign i 0 - %rep 4 - mova [dstq + i*16 ], m0 - %assign i i+1 - %if i < 4 - %1 2*i - %endif - %endrep - dec hd - jg .w64_loop - RET -.w128_loop: - %1_INC_PTR 16 - %1 0 - add dstq, strideq -.w128: - %assign i 0 - %rep 8 - mova [dstq + i*16 ], m0 - %assign i i+1 - %if i < 8 - %1 2*i - %endif - %endrep - dec hd - jg .w128_loop - RET -%endmacro - -%macro AVG 1 ; src_offset - ; writes AVG of tmp1 tmp2 uint16 coeffs into uint8 pixel - mova m0, [tmp1q+(%1+0)*mmsize] ; load 8 coef(2bytes) from tmp1 - paddw m0, [tmp2q+(%1+0)*mmsize] ; load/add 8 coef(2bytes) tmp2 - mova m1, [tmp1q+(%1+1)*mmsize] - paddw m1, [tmp2q+(%1+1)*mmsize] - pmulhrsw m0, m2 - pmulhrsw m1, m2 - packuswb m0, m1 ; pack/trunc 16 bits from m0 & m1 to 8 bit -%endmacro - -%macro AVG_INC_PTR 1 - add tmp1q, %1*mmsize - add tmp2q, %1*mmsize -%endmacro - -cglobal avg, 4, 7, 3, dst, stride, tmp1, tmp2, w, h, stride3 - LEA r6, avg_ssse3_table - tzcnt wd, wm ; leading zeros - movifnidn hd, hm ; move h(stack) to h(register) if not already that register - movsxd wq, dword [r6+wq*4] ; push table entry matching the tile width (tzcnt) in widen reg - mova m2, [pw_1024+r6-avg_ssse3_table] ; fill m2 with shift/align - add wq, r6 - BIDIR_FN AVG - -%macro W_AVG 1 ; src_offset - ; (a * weight + b * (16 - weight) + 128) >> 8 - ; = ((a - b) * weight + (b << 4) + 128) >> 8 - ; = ((((a - b) * ((weight-16) << 12)) >> 16) + a + 8) >> 4 - ; = ((((b - a) * (-weight << 12)) >> 16) + b + 8) >> 4 - mova m2, [tmp1q+(%1+0)*mmsize] - mova m0, m2 - psubw m2, [tmp2q+(%1+0)*mmsize] - mova m3, [tmp1q+(%1+1)*mmsize] - mova m1, m3 - psubw m3, [tmp2q+(%1+1)*mmsize] - pmulhw m2, m4 - pmulhw m3, m4 - paddw m0, m2 - paddw m1, m3 - pmulhrsw m0, m5 - pmulhrsw m1, m5 - packuswb m0, m1 -%endmacro - -%define W_AVG_INC_PTR AVG_INC_PTR - -cglobal w_avg, 4, 7, 6, dst, stride, tmp1, tmp2, w, h, stride3 - LEA r6, w_avg_ssse3_table - tzcnt wd, wm - movd m4, r6m - movifnidn hd, hm - pxor m0, m0 - movsxd wq, dword [r6+wq*4] - mova m5, [pw_2048+r6-w_avg_ssse3_table] - pshufb m4, m0 - psllw m4, 12 ; (weight-16) << 12 when interpreted as signed - add wq, r6 - cmp dword r6m, 7 - jg .weight_gt7 - mov r6, tmp1q - psubw m0, m4 - mov tmp1q, tmp2q - mova m4, m0 ; -weight - mov tmp2q, r6 -.weight_gt7: - BIDIR_FN W_AVG - -%macro MASK 1 ; src_offset - ; (a * m + b * (64 - m) + 512) >> 10 - ; = ((a - b) * m + (b << 6) + 512) >> 10 - ; = ((((b - a) * (-m << 10)) >> 16) + b + 8) >> 4 - mova m3, [maskq+(%1+0)*(mmsize/2)] - mova m0, [tmp2q+(%1+0)*mmsize] ; b - psubw m1, m0, [tmp1q+(%1+0)*mmsize] ; b - a - mova m6, m3 ; m - psubb m3, m4, m6 ; -m - paddw m1, m1 ; (b - a) << 1 - paddb m3, m3 ; -m << 1 - punpcklbw m2, m4, m3 ; -m << 9 (<< 8 when ext as uint16) - pmulhw m1, m2 ; (-m * (b - a)) << 10 - paddw m0, m1 ; + b - mova m1, [tmp2q+(%1+1)*mmsize] ; b - psubw m2, m1, [tmp1q+(%1+1)*mmsize] ; b - a - paddw m2, m2 ; (b - a) << 1 - mova m6, m3 ; (-m << 1) - punpckhbw m3, m4, m6 ; (-m << 9) - pmulhw m2, m3 ; (-m << 9) - paddw m1, m2 ; (-m * (b - a)) << 10 - pmulhrsw m0, m5 ; round - pmulhrsw m1, m5 ; round - packuswb m0, m1 ; interleave 16 -> 8 -%endmacro - -%macro MASK_INC_PTR 1 - add maskq, %1*mmsize/2 - add tmp1q, %1*mmsize - add tmp2q, %1*mmsize -%endmacro - -%if ARCH_X86_64 -cglobal mask, 4, 8, 7, dst, stride, tmp1, tmp2, w, h, mask, stride3 - movifnidn hd, hm -%else -cglobal mask, 4, 7, 7, dst, stride, tmp1, tmp2, w, mask, stride3 -%define hd dword r5m -%endif -%define base r6-mask_ssse3_table - LEA r6, mask_ssse3_table - tzcnt wd, wm - movsxd wq, dword [r6+wq*4] - pxor m4, m4 - mova m5, [base+pw_2048] - add wq, r6 - mov maskq, r6m - BIDIR_FN MASK -%undef hd - -%macro W_MASK_420_B 2 ; src_offset in bytes, mask_out - ;**** do m0 = u16.dst[7..0], m%2 = u16.m[7..0] **** - mova m0, [tmp1q+(%1)] - mova m1, [tmp2q+(%1)] - mova m2, reg_pw_6903 - psubw m1, m0 - pabsw m%2, m1 ; abs(tmp1 - tmp2) - mova m3, m2 - psubusw m2, m%2 - psrlw m2, 8 ; 64 - m - mova m%2, m2 - psllw m2, 10 - pmulhw m1, m2 ; tmp2 * () - paddw m0, m1 ; tmp1 + () - ;**** do m1 = u16.dst[7..0], m%2 = u16.m[7..0] **** - mova m1, [tmp1q+(%1)+mmsize] - mova m2, [tmp2q+(%1)+mmsize] - psubw m2, m1 - pabsw m7, m2 ; abs(tmp1 - tmp2) - psubusw m3, m7 - psrlw m3, 8 ; 64 - m - phaddw m%2, m3 ; pack both u16.m[8..0]runs as u8.m [15..0] - psllw m3, 10 - pmulhw m2, m3 -%if ARCH_X86_32 - mova reg_pw_2048, [base+pw_2048] -%endif - paddw m1, m2 - pmulhrsw m0, reg_pw_2048 ; round/scale 2048 - pmulhrsw m1, reg_pw_2048 ; round/scale 2048 - packuswb m0, m1 ; concat m0 = u8.dst[15..0] -%endmacro - -%macro W_MASK_420 2 - W_MASK_420_B (%1*16), %2 -%endmacro - -%define base r6-w_mask_420_ssse3_table -%if ARCH_X86_64 -%define reg_pw_6903 m8 -%define reg_pw_2048 m9 -; args: dst, stride, tmp1, tmp2, w, h, mask, sign -cglobal w_mask_420, 4, 8, 10, dst, stride, tmp1, tmp2, w, h, mask - lea r6, [w_mask_420_ssse3_table] - mov wd, wm - tzcnt r7d, wd - movd m0, r7m ; sign - movifnidn hd, hm - movsxd r7, [r6+r7*4] - mova reg_pw_6903, [base+pw_6903] ; ((64 - 38) << 8) + 255 - 8 - mova reg_pw_2048, [base+pw_2048] - movd m6, [base+pw_258] ; 64 * 4 + 2 - add r7, r6 - mov maskq, maskmp - psubw m6, m0 - pshuflw m6, m6, q0000 - punpcklqdq m6, m6 - W_MASK_420 0, 4 - jmp r7 - %define loop_w r7d -%else -%define reg_pw_6903 [base+pw_6903] -%define reg_pw_2048 m3 -cglobal w_mask_420, 4, 7, 8, dst, stride, tmp1, tmp2, w, mask - tzcnt wd, wm - LEA r6, w_mask_420_ssse3_table - movd m0, r7m ; sign - mov maskq, r6mp - mov wd, [r6+wq*4] - movd m6, [base+pw_258] - add wq, r6 - psubw m6, m0 - pshuflw m6, m6, q0000 - punpcklqdq m6, m6 - W_MASK_420 0, 4 - jmp wd - %define loop_w dword r0m - %define hd dword r5m -%endif -.w4_loop: - add tmp1q, 2*16 - add tmp2q, 2*16 - W_MASK_420 0, 4 - lea dstq, [dstq+strideq*2] - add maskq, 4 -.w4: - movd [dstq ], m0 ; copy m0[0] - pshuflw m1, m0, q1032 - movd [dstq+strideq*1], m1 ; copy m0[1] - lea dstq, [dstq+strideq*2] - punpckhqdq m0, m0 - movd [dstq+strideq*0], m0 ; copy m0[2] - psrlq m0, 32 - movd [dstq+strideq*1], m0 ; copy m0[3] - psubw m1, m6, m4 ; a _ c _ - psrlq m4, 32 ; b _ d _ - psubw m1, m4 - psrlw m1, 2 - packuswb m1, m1 - pshuflw m1, m1, q2020 - movd [maskq], m1 - sub hd, 4 - jg .w4_loop - RET -.w8_loop: - add tmp1q, 2*16 - add tmp2q, 2*16 - W_MASK_420 0, 4 - lea dstq, [dstq+strideq*2] - add maskq, 4 -.w8: - movq [dstq ], m0 - movhps [dstq+strideq*1], m0 - psubw m0, m6, m4 - punpckhqdq m4, m4 - psubw m0, m4 - psrlw m0, 2 - packuswb m0, m0 - movd [maskq], m0 - sub hd, 2 - jg .w8_loop - RET -.w16: ; w32/64/128 -%if ARCH_X86_32 - mov wd, wm ; because we altered it in 32bit setup -%endif - mov loop_w, wd ; use width as counter - jmp .w16ge_inner_loop_first -.w16ge_loop: - lea tmp1q, [tmp1q+wq*2] ; skip even line pixels - lea tmp2q, [tmp2q+wq*2] ; skip even line pixels - sub dstq, wq - mov loop_w, wd - lea dstq, [dstq+strideq*2] -.w16ge_inner_loop: - W_MASK_420_B 0, 4 -.w16ge_inner_loop_first: - mova [dstq ], m0 - W_MASK_420_B wq*2, 5 ; load matching even line (offset = widthpx * (16+16)) - mova [dstq+strideq*1], m0 - psubw m1, m6, m4 ; m9 == 64 * 4 + 2 - psubw m1, m5 ; - odd line mask - psrlw m1, 2 ; >> 2 - packuswb m1, m1 - movq [maskq], m1 - add tmp1q, 2*16 - add tmp2q, 2*16 - add maskq, 8 - add dstq, 16 - sub loop_w, 16 - jg .w16ge_inner_loop - sub hd, 2 - jg .w16ge_loop - RET - -%undef reg_pw_6903 -%undef reg_pw_2048 -%undef dst_bak -%undef loop_w -%undef orig_w -%undef hd - -%macro BLEND_64M 4; a, b, mask1, mask2 - punpcklbw m0, %1, %2; {b;a}[7..0] - punpckhbw %1, %2 ; {b;a}[15..8] - pmaddubsw m0, %3 ; {b*m[0] + (64-m[0])*a}[7..0] u16 - pmaddubsw %1, %4 ; {b*m[1] + (64-m[1])*a}[15..8] u16 - pmulhrsw m0, m5 ; {((b*m[0] + (64-m[0])*a) + 1) / 32}[7..0] u16 - pmulhrsw %1, m5 ; {((b*m[1] + (64-m[0])*a) + 1) / 32}[15..8] u16 - packuswb m0, %1 ; {blendpx}[15..0] u8 -%endmacro - -%macro BLEND 2; a, b - psubb m3, m4, m0 ; m3 = (64 - m) - punpcklbw m2, m3, m0 ; {m;(64-m)}[7..0] - punpckhbw m3, m0 ; {m;(64-m)}[15..8] - BLEND_64M %1, %2, m2, m3 -%endmacro - -cglobal blend, 3, 7, 7, dst, ds, tmp, w, h, mask -%define base r6-blend_ssse3_table - LEA r6, blend_ssse3_table - tzcnt wd, wm - movifnidn hd, hm - movifnidn maskq, maskmp - movsxd wq, dword [r6+wq*4] - mova m4, [base+pb_64] - mova m5, [base+pw_512] - add wq, r6 - lea r6, [dsq*3] - jmp wq -.w4: - movq m0, [maskq]; m - movd m1, [dstq+dsq*0] ; a - movd m6, [dstq+dsq*1] - punpckldq m1, m6 - movq m6, [tmpq] ; b - psubb m3, m4, m0 ; m3 = (64 - m) - punpcklbw m2, m3, m0 ; {m;(64-m)}[7..0] - punpcklbw m1, m6 ; {b;a}[7..0] - pmaddubsw m1, m2 ; {b*m[0] + (64-m[0])*a}[7..0] u16 - pmulhrsw m1, m5 ; {((b*m[0] + (64-m[0])*a) + 1) / 32}[7..0] u16 - packuswb m1, m0 ; {blendpx}[15..0] u8 - movd [dstq+dsq*0], m1 - psrlq m1, 32 - movd [dstq+dsq*1], m1 - add maskq, 8 - add tmpq, 8 - lea dstq, [dstq+dsq*2] ; dst_stride * 2 - sub hd, 2 - jg .w4 - RET -.w8: - mova m0, [maskq]; m - movq m1, [dstq+dsq*0] ; a - movhps m1, [dstq+dsq*1] - mova m6, [tmpq] ; b - BLEND m1, m6 - movq [dstq+dsq*0], m0 - movhps [dstq+dsq*1], m0 - add maskq, 16 - add tmpq, 16 - lea dstq, [dstq+dsq*2] ; dst_stride * 2 - sub hd, 2 - jg .w8 - RET -.w16: - mova m0, [maskq]; m - mova m1, [dstq] ; a - mova m6, [tmpq] ; b - BLEND m1, m6 - mova [dstq], m0 - add maskq, 16 - add tmpq, 16 - add dstq, dsq ; dst_stride - dec hd - jg .w16 - RET -.w32: - %assign i 0 - %rep 2 - mova m0, [maskq+16*i]; m - mova m1, [dstq+16*i] ; a - mova m6, [tmpq+16*i] ; b - BLEND m1, m6 - mova [dstq+i*16], m0 - %assign i i+1 - %endrep - add maskq, 32 - add tmpq, 32 - add dstq, dsq ; dst_stride - dec hd - jg .w32 - RET - -cglobal blend_v, 3, 6, 6, dst, ds, tmp, w, h, mask -%define base r5-blend_v_ssse3_table - LEA r5, blend_v_ssse3_table - tzcnt wd, wm - movifnidn hd, hm - movsxd wq, dword [r5+wq*4] - mova m5, [base+pw_512] - add wq, r5 - add maskq, obmc_masks-blend_v_ssse3_table - jmp wq -.w2: - movd m3, [maskq+4] - punpckldq m3, m3 - ; 2 mask blend is provided for 4 pixels / 2 lines -.w2_loop: - movd m1, [dstq+dsq*0] ; a {..;a;a} - pinsrw m1, [dstq+dsq*1], 1 - movd m2, [tmpq] ; b - punpcklbw m0, m1, m2; {b;a}[7..0] - pmaddubsw m0, m3 ; {b*m + (64-m)*a}[7..0] u16 - pmulhrsw m0, m5 ; {((b*m + (64-m)*a) + 1) / 32}[7..0] u16 - packuswb m0, m1 ; {blendpx}[8..0] u8 - movd r3d, m0 - mov [dstq+dsq*0], r3w - shr r3d, 16 - mov [dstq+dsq*1], r3w - add tmpq, 2*2 - lea dstq, [dstq + dsq * 2] - sub hd, 2 - jg .w2_loop - RET -.w4: - movddup m3, [maskq+8] - ; 4 mask blend is provided for 8 pixels / 2 lines -.w4_loop: - movd m1, [dstq+dsq*0] ; a - movd m2, [dstq+dsq*1] ; - punpckldq m1, m2 - movq m2, [tmpq] ; b - punpcklbw m1, m2 ; {b;a}[7..0] - pmaddubsw m1, m3 ; {b*m + (64-m)*a}[7..0] u16 - pmulhrsw m1, m5 ; {((b*m + (64-m)*a) + 1) / 32}[7..0] u16 - packuswb m1, m1 ; {blendpx}[8..0] u8 - movd [dstq], m1 - psrlq m1, 32 - movd [dstq+dsq*1], m1 - add tmpq, 2*4 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .w4_loop - RET -.w8: - mova m3, [maskq+16] - ; 8 mask blend is provided for 16 pixels -.w8_loop: - movq m1, [dstq+dsq*0] ; a - movhps m1, [dstq+dsq*1] - mova m2, [tmpq]; b - BLEND_64M m1, m2, m3, m3 - movq [dstq+dsq*0], m0 - movhps [dstq+dsq*1], m0 - add tmpq, 16 - lea dstq, [dstq+dsq*2] - sub hd, 2 - jg .w8_loop - RET -.w16: - ; 16 mask blend is provided for 32 pixels - mova m3, [maskq+32] ; obmc_masks_16[0] (64-m[0]) - mova m4, [maskq+48] ; obmc_masks_16[1] (64-m[1]) -.w16_loop: - mova m1, [dstq] ; a - mova m2, [tmpq] ; b - BLEND_64M m1, m2, m3, m4 - mova [dstq], m0 - add tmpq, 16 - add dstq, dsq - dec hd - jg .w16_loop - RET -.w32: -%if WIN64 - mova [rsp+8], xmm6 -%endif - mova m3, [maskq+64] ; obmc_masks_32[0] (64-m[0]) - mova m4, [maskq+80] ; obmc_masks_32[1] (64-m[1]) - mova m6, [maskq+96] ; obmc_masks_32[2] (64-m[2]) - ; 16 mask blend is provided for 64 pixels -.w32_loop: - mova m1, [dstq+16*0] ; a - mova m2, [tmpq+16*0] ; b - BLEND_64M m1, m2, m3, m4 - movq m1, [dstq+16*1] ; a - punpcklbw m1, [tmpq+16*1] ; b - pmaddubsw m1, m6 - pmulhrsw m1, m5 - packuswb m1, m1 - mova [dstq+16*0], m0 - movq [dstq+16*1], m1 - add tmpq, 32 - add dstq, dsq - dec hd - jg .w32_loop -%if WIN64 - mova xmm6, [rsp+8] -%endif - RET - -cglobal blend_h, 3, 7, 6, dst, ds, tmp, w, h, mask -%define base t0-blend_h_ssse3_table -%if ARCH_X86_32 - ; We need to keep the PIC pointer for w4, reload wd from stack instead - DECLARE_REG_TMP 6 -%else - DECLARE_REG_TMP 5 - mov r6d, wd -%endif - LEA t0, blend_h_ssse3_table - tzcnt wd, wm - mov hd, hm - movsxd wq, dword [t0+wq*4] - mova m5, [base+pw_512] - add wq, t0 - lea maskq, [base+obmc_masks+hq*2] - lea hd, [hq*3] - shr hd, 2 ; h * 3/4 - lea maskq, [maskq+hq*2] - neg hq - jmp wq -.w2: - movd m0, [dstq+dsq*0] - pinsrw m0, [dstq+dsq*1], 1 - movd m2, [maskq+hq*2] - movd m1, [tmpq] - punpcklwd m2, m2 - punpcklbw m0, m1 - pmaddubsw m0, m2 - pmulhrsw m0, m5 - packuswb m0, m0 - movd r3d, m0 - mov [dstq+dsq*0], r3w - shr r3d, 16 - mov [dstq+dsq*1], r3w - lea dstq, [dstq+dsq*2] - add tmpq, 2*2 - add hq, 2 - jl .w2 - RET -.w4: -%if ARCH_X86_32 - mova m3, [base+blend_shuf] -%else - mova m3, [blend_shuf] -%endif -.w4_loop: - movd m0, [dstq+dsq*0] - movd m2, [dstq+dsq*1] - punpckldq m0, m2 ; a - movq m1, [tmpq] ; b - movq m2, [maskq+hq*2] ; m - pshufb m2, m3 - punpcklbw m0, m1 - pmaddubsw m0, m2 - pmulhrsw m0, m5 - packuswb m0, m0 - movd [dstq+dsq*0], m0 - psrlq m0, 32 - movd [dstq+dsq*1], m0 - lea dstq, [dstq+dsq*2] - add tmpq, 4*2 - add hq, 2 - jl .w4_loop - RET -.w8: - movd m4, [maskq+hq*2] - punpcklwd m4, m4 - pshufd m3, m4, q0000 - pshufd m4, m4, q1111 - movq m1, [dstq+dsq*0] ; a - movhps m1, [dstq+dsq*1] - mova m2, [tmpq] - BLEND_64M m1, m2, m3, m4 - movq [dstq+dsq*0], m0 - movhps [dstq+dsq*1], m0 - lea dstq, [dstq+dsq*2] - add tmpq, 8*2 - add hq, 2 - jl .w8 - RET -; w16/w32/w64/w128 -.w16: -%if ARCH_X86_32 - mov r6d, wm -%endif - sub dsq, r6 -.w16_loop0: - movd m3, [maskq+hq*2] - pshuflw m3, m3, q0000 - punpcklqdq m3, m3 - mov wd, r6d -.w16_loop: - mova m1, [dstq] ; a - mova m2, [tmpq] ; b - BLEND_64M m1, m2, m3, m3 - mova [dstq], m0 - add dstq, 16 - add tmpq, 16 - sub wd, 16 - jg .w16_loop - add dstq, dsq - inc hq - jl .w16_loop0 - RET - -; emu_edge args: -; const intptr_t bw, const intptr_t bh, const intptr_t iw, const intptr_t ih, -; const intptr_t x, const intptr_t y, pixel *dst, const ptrdiff_t dst_stride, -; const pixel *ref, const ptrdiff_t ref_stride -; -; bw, bh total filled size -; iw, ih, copied block -> fill bottom, right -; x, y, offset in bw/bh -> fill top, left -cglobal emu_edge, 10, 13, 2, bw, bh, iw, ih, x, \ - y, dst, dstride, src, sstride, \ - bottomext, rightext, blk - ; we assume that the buffer (stride) is larger than width, so we can - ; safely overwrite by a few bytes - pxor m1, m1 - -%if ARCH_X86_64 - %define reg_zero r12q - %define reg_tmp r10 - %define reg_src srcq - %define reg_bottomext bottomextq - %define reg_rightext rightextq - %define reg_blkm r9m -%else - %define reg_zero r6 - %define reg_tmp r0 - %define reg_src r1 - %define reg_bottomext r0 - %define reg_rightext r1 - %define reg_blkm r2m -%endif - ; - ; ref += iclip(y, 0, ih - 1) * PXSTRIDE(ref_stride) - xor reg_zero, reg_zero - lea reg_tmp, [ihq-1] - cmp yq, ihq - cmovs reg_tmp, yq - test yq, yq - cmovs reg_tmp, reg_zero -%if ARCH_X86_64 - imul reg_tmp, sstrideq - add srcq, reg_tmp -%else - imul reg_tmp, sstridem - mov reg_src, srcm - add reg_src, reg_tmp -%endif - ; - ; ref += iclip(x, 0, iw - 1) - lea reg_tmp, [iwq-1] - cmp xq, iwq - cmovs reg_tmp, xq - test xq, xq - cmovs reg_tmp, reg_zero - add reg_src, reg_tmp -%if ARCH_X86_32 - mov srcm, reg_src -%endif - ; - ; bottom_ext = iclip(y + bh - ih, 0, bh - 1) -%if ARCH_X86_32 - mov r1, r1m ; restore bh -%endif - lea reg_bottomext, [yq+bhq] - sub reg_bottomext, ihq - lea r3, [bhq-1] - cmovs reg_bottomext, reg_zero - ; - - DEFINE_ARGS bw, bh, iw, ih, x, \ - topext, dst, dstride, src, sstride, \ - bottomext, rightext, blk - - ; top_ext = iclip(-y, 0, bh - 1) - neg topextq - cmovs topextq, reg_zero - cmp reg_bottomext, bhq - cmovns reg_bottomext, r3 - cmp topextq, bhq - cmovg topextq, r3 - %if ARCH_X86_32 - mov r4m, reg_bottomext - ; - ; right_ext = iclip(x + bw - iw, 0, bw - 1) - mov r0, r0m ; restore bw - %endif - lea reg_rightext, [xq+bwq] - sub reg_rightext, iwq - lea r2, [bwq-1] - cmovs reg_rightext, reg_zero - - DEFINE_ARGS bw, bh, iw, ih, leftext, \ - topext, dst, dstride, src, sstride, \ - bottomext, rightext, blk - - ; left_ext = iclip(-x, 0, bw - 1) - neg leftextq - cmovs leftextq, reg_zero - cmp reg_rightext, bwq - cmovns reg_rightext, r2 - %if ARCH_X86_32 - mov r3m, r1 - %endif - cmp leftextq, bwq - cmovns leftextq, r2 - -%undef reg_zero -%undef reg_tmp -%undef reg_src -%undef reg_bottomext -%undef reg_rightext - - DEFINE_ARGS bw, centerh, centerw, dummy, leftext, \ - topext, dst, dstride, src, sstride, \ - bottomext, rightext, blk - - ; center_h = bh - top_ext - bottom_ext -%if ARCH_X86_64 - lea r3, [bottomextq+topextq] - sub centerhq, r3 -%else - mov r1, centerhm ; restore r1 - sub centerhq, topextq - sub centerhq, r4m - mov r1m, centerhq -%endif - ; - ; blk += top_ext * PXSTRIDE(dst_stride) - mov r2, topextq -%if ARCH_X86_64 - imul r2, dstrideq -%else - mov r6, r6m ; restore dstq - imul r2, dstridem -%endif - add dstq, r2 - mov reg_blkm, dstq ; save pointer for ext - ; - ; center_w = bw - left_ext - right_ext - mov centerwq, bwq -%if ARCH_X86_64 - lea r3, [rightextq+leftextq] - sub centerwq, r3 -%else - sub centerwq, r3m - sub centerwq, leftextq -%endif - -; vloop Macro -%macro v_loop 3 ; need_left_ext, need_right_ext, suffix - %if ARCH_X86_64 - %define reg_tmp r12 - %else - %define reg_tmp r0 - %endif -.v_loop_%3: - %if ARCH_X86_32 - mov r0, r0m - mov r1, r1m - %endif -%if %1 - test leftextq, leftextq - jz .body_%3 - ; left extension - %if ARCH_X86_64 - movd m0, [srcq] - %else - mov r3, srcm - movd m0, [r3] - %endif - pshufb m0, m1 - xor r3, r3 -.left_loop_%3: - mova [dstq+r3], m0 - add r3, mmsize - cmp r3, leftextq - jl .left_loop_%3 - ; body -.body_%3: - lea reg_tmp, [dstq+leftextq] -%endif - xor r3, r3 -.body_loop_%3: - %if ARCH_X86_64 - movu m0, [srcq+r3] - %else - mov r1, srcm - movu m0, [r1+r3] - %endif -%if %1 - movu [reg_tmp+r3], m0 -%else - movu [dstq+r3], m0 -%endif - add r3, mmsize - cmp r3, centerwq - jl .body_loop_%3 -%if %2 - ; right extension - %if ARCH_X86_64 - test rightextq, rightextq - %else - mov r1, r3m - test r1, r1 - %endif - jz .body_loop_end_%3 -%if %1 - add reg_tmp, centerwq -%else - lea reg_tmp, [dstq+centerwq] -%endif - %if ARCH_X86_64 - movd m0, [srcq+centerwq-1] - %else - mov r3, srcm - movd m0, [r3+centerwq-1] - %endif - pshufb m0, m1 - xor r3, r3 -.right_loop_%3: - movu [reg_tmp+r3], m0 - add r3, mmsize - %if ARCH_X86_64 - cmp r3, rightextq - %else - cmp r3, r3m - %endif - jl .right_loop_%3 -.body_loop_end_%3: -%endif - %if ARCH_X86_64 - add dstq, dstrideq - add srcq, sstrideq - dec centerhq - jg .v_loop_%3 - %else - add dstq, dstridem - mov r0, sstridem - add srcm, r0 - sub dword centerhm, 1 - jg .v_loop_%3 - mov r0, r0m ; restore r0 - %endif -%endmacro ; vloop MACRO - - test leftextq, leftextq - jnz .need_left_ext - %if ARCH_X86_64 - test rightextq, rightextq - jnz .need_right_ext - %else - cmp leftextq, r3m ; leftextq == 0 - jne .need_right_ext - %endif - v_loop 0, 0, 0 - jmp .body_done - - ;left right extensions -.need_left_ext: - %if ARCH_X86_64 - test rightextq, rightextq - %else - mov r3, r3m - test r3, r3 - %endif - jnz .need_left_right_ext - v_loop 1, 0, 1 - jmp .body_done - -.need_left_right_ext: - v_loop 1, 1, 2 - jmp .body_done - -.need_right_ext: - v_loop 0, 1, 3 - -.body_done: -; r0 ; bw -; r1 ;; x loop -; r4 ;; y loop -; r5 ; topextq -; r6 ;dstq -; r7 ;dstrideq -; r8 ; srcq -%if ARCH_X86_64 - %define reg_dstride dstrideq -%else - %define reg_dstride r2 -%endif - ; - ; bottom edge extension - %if ARCH_X86_64 - test bottomextq, bottomextq - jz .top - %else - xor r1, r1 - cmp r1, r4m - je .top - %endif - ; - %if ARCH_X86_64 - mov srcq, dstq - sub srcq, dstrideq - xor r1, r1 - %else - mov r3, dstq - mov reg_dstride, dstridem - sub r3, reg_dstride - mov srcm, r3 - %endif - ; -.bottom_x_loop: - %if ARCH_X86_64 - mova m0, [srcq+r1] - lea r3, [dstq+r1] - mov r4, bottomextq - %else - mov r3, srcm - mova m0, [r3+r1] - lea r3, [dstq+r1] - mov r4, r4m - %endif - ; -.bottom_y_loop: - mova [r3], m0 - add r3, reg_dstride - dec r4 - jg .bottom_y_loop - add r1, mmsize - cmp r1, bwq - jl .bottom_x_loop - -.top: - ; top edge extension - test topextq, topextq - jz .end -%if ARCH_X86_64 - mov srcq, reg_blkm -%else - mov r3, reg_blkm - mov reg_dstride, dstridem -%endif - mov dstq, dstm - xor r1, r1 - ; -.top_x_loop: -%if ARCH_X86_64 - mova m0, [srcq+r1] -%else - mov r3, reg_blkm - mova m0, [r3+r1] -%endif - lea r3, [dstq+r1] - mov r4, topextq - ; -.top_y_loop: - mova [r3], m0 - add r3, reg_dstride - dec r4 - jg .top_y_loop - add r1, mmsize - cmp r1, bwq - jl .top_x_loop - -.end: - RET - -%undef reg_dstride -%undef reg_blkm -%undef reg_tmp diff --git a/third_party/dav1d/src/x86/msac.asm b/third_party/dav1d/src/x86/msac.asm --- a/third_party/dav1d/src/x86/msac.asm +++ b/third_party/dav1d/src/x86/msac.asm @@ -155,11 +155,11 @@ shl t2d, cl shl t4, cl mov [t7+msac.rng], t2d not t4 sub t1d, ecx - jge .end ; no refill required + jae .end ; no refill required ; refill: mov t2, [t7+msac.buf] mov rcx, [t7+msac.end] %if ARCH_X86_64 == 0 @@ -502,11 +502,11 @@ shl t4, cl movd m2, t2d mov [t7+msac.rng], t2d not t4 sub t5d, ecx - jge %%end + jae %%end mov t2, [t7+msac.buf] mov rcx, [t7+msac.end] %if UNIX64 == 0 push t8 %endif diff --git a/third_party/dav1d/src/x86/msac_init.c b/third_party/dav1d/src/x86/msac_init.c --- a/third_party/dav1d/src/x86/msac_init.c +++ b/third_party/dav1d/src/x86/msac_init.c @@ -26,10 +26,11 @@ #include "src/cpu.h" #include "src/msac.h" #include "src/x86/msac.h" +#if ARCH_X86_64 void dav1d_msac_init_x86(MsacContext *const s) { const unsigned flags = dav1d_get_cpu_flags(); if (flags & DAV1D_X86_CPU_FLAG_SSE2) { s->symbol_adapt16 = dav1d_msac_decode_symbol_adapt16_sse2; @@ -37,6 +38,6 @@ if (flags & DAV1D_X86_CPU_FLAG_AVX2) { s->symbol_adapt16 = dav1d_msac_decode_symbol_adapt16_avx2; } } - +#endif diff --git a/third_party/dav1d/tests/checkasm/arm/checkasm_32.S b/third_party/dav1d/tests/checkasm/arm/checkasm_32.S --- a/third_party/dav1d/tests/checkasm/arm/checkasm_32.S +++ b/third_party/dav1d/tests/checkasm/arm/checkasm_32.S @@ -46,19 +46,22 @@ .asciz "failed to preserve register FPSCR, changed bits: %x" error_message_gpr: .asciz "failed to preserve register r%d" error_message_vfp: .asciz "failed to preserve register d%d" +error_message_stack: + .asciz "failed to preserve stack" endconst @ max number of args used by any asm function. #define MAX_ARGS 15 #define ARG_STACK 4*(MAX_ARGS - 4) -@ align the used stack space to 8 to preserve the stack alignment -#define ARG_STACK_A (((ARG_STACK + pushed + 7) & ~7) - pushed) +@ Align the used stack space to 8 to preserve the stack alignment. +@ +8 for stack canary reference. +#define ARG_STACK_A (((ARG_STACK + pushed + 7) & ~7) - pushed + 8) .macro clobbercheck variant .equ pushed, 4*9 function checked_call_\variant, export=1 push {r4-r11, lr} @@ -81,18 +84,41 @@ ldr r12, [sp, #ARG_STACK_A + pushed + 8 + pos] str r12, [sp, #pos] .equ pos, pos + 4 .endr + @ For stack overflows, the callee is free to overwrite the parameters + @ that were passed on the stack (if any), so we can only check after + @ that point. First figure out how many parameters the function + @ really took on the stack: + ldr r12, [sp, #ARG_STACK_A + pushed + 8 + 4*(MAX_ARGS-4)] + @ Load the first non-parameter value from the stack, that should be + @ left untouched by the function. Store a copy of it inverted, so that + @ e.g. overwriting everything with zero would be noticed. + ldr r12, [sp, r12, lsl #2] + mvn r12, r12 + str r12, [sp, #ARG_STACK_A - 4] + mov r12, r0 mov r0, r2 mov r1, r3 ldrd r2, r3, [sp, #ARG_STACK_A + pushed] + @ Call the target function blx r12 + + @ Load the number of stack parameters, stack canary and its reference + ldr r12, [sp, #ARG_STACK_A + pushed + 8 + 4*(MAX_ARGS-4)] + ldr r2, [sp, r12, lsl #2] + ldr r3, [sp, #ARG_STACK_A - 4] + add sp, sp, #ARG_STACK_A + push {r0, r1} - push {r0, r1} + mvn r3, r3 + cmp r2, r3 + bne 5f + movrel r12, register_init .ifc \variant, vfp .macro check_reg_vfp, dreg, offset ldrd r2, r3, [r12, #8 * (\offset)] vmov r0, lr, \dreg @@ -142,23 +168,26 @@ #endif check_reg r10, r11 .purgem check_reg b 0f +5: + movrel r0, error_message_stack + b 1f 4: movrel r0, error_message_vfp b 1f 3: movrel r0, error_message_fpscr b 1f 2: movrel r0, error_message_gpr 1: #ifdef PREFIX - blx _checkasm_fail_func + bl _checkasm_fail_func #else - blx checkasm_fail_func + bl checkasm_fail_func #endif 0: pop {r0, r1} .ifc \variant, vfp pop {r2} diff --git a/third_party/dav1d/tests/checkasm/arm/checkasm_64.S b/third_party/dav1d/tests/checkasm/arm/checkasm_64.S --- a/third_party/dav1d/tests/checkasm/arm/checkasm_64.S +++ b/third_party/dav1d/tests/checkasm/arm/checkasm_64.S @@ -51,12 +51,14 @@ .quad 0x71aeaff20b095fd9 .quad 0xab63e2e11fa38ed9 endconst -const error_message +const error_message_register .asciz "failed to preserve register" +error_message_stack: + .asciz "stack clobbered" endconst // max number of args used by any asm function. #define MAX_ARGS 15 @@ -72,11 +74,12 @@ b.gt 1b mov sp, x3 ret endfunc -#define ARG_STACK ((8*(MAX_ARGS - 8) + 15) & ~15) +// + 16 for stack canary reference +#define ARG_STACK ((8*(MAX_ARGS - 8) + 15) & ~15 + 16) function checked_call, export=1 stp x29, x30, [sp, #-16]! mov x29, sp stp x19, x20, [sp, #-16]! @@ -107,26 +110,60 @@ ldr x9, [x29, #16 + 8*8 + pos] str x9, [sp, #pos] .equ pos, pos + 8 .endr + // Fill x8-x17 with garbage. This doesn't have to be preserved, + // but avoids relying on them having any particular value. + movrel x9, register_init + ldp x10, x11, [x9], #32 + ldp x12, x13, [x9], #32 + ldp x14, x15, [x9], #32 + ldp x16, x17, [x9], #32 + ldp x8, x9, [x9] + + // For stack overflows, the callee is free to overwrite the parameters + // that were passed on the stack (if any), so we can only check after + // that point. First figure out how many parameters the function + // really took on the stack: + ldr w2, [x29, #16 + 8*8 + (MAX_ARGS-8)*8] + // Load the first non-parameter value from the stack, that should be + // left untouched by the function. Store a copy of it inverted, so that + // e.g. overwriting everything with zero would be noticed. + ldr x2, [sp, x2, lsl #3] + mvn x2, x2 + str x2, [sp, #ARG_STACK-8] + + // Load the in-register arguments mov x12, x0 ldp x0, x1, [x29, #16] ldp x2, x3, [x29, #32] ldp x4, x5, [x29, #48] ldp x6, x7, [x29, #64] + // Call the target function blr x12 + + // Load the number of stack parameters, stack canary and its reference + ldr w2, [x29, #16 + 8*8 + (MAX_ARGS-8)*8] + ldr x2, [sp, x2, lsl #3] + ldr x3, [sp, #ARG_STACK-8] + add sp, sp, #ARG_STACK stp x0, x1, [sp, #-16]! + + mvn x3, x3 + cmp x2, x3 + b.ne 2f + movrel x9, register_init movi v3.8h, #0 .macro check_reg_neon reg1, reg2 - ldr q0, [x9], #16 - uzp1 v1.2d, v\reg1\().2d, v\reg2\().2d - eor v0.16b, v0.16b, v1.16b - orr v3.16b, v3.16b, v0.16b + ldr q1, [x9], #16 + uzp1 v2.2d, v\reg1\().2d, v\reg2\().2d + eor v1.16b, v1.16b, v2.16b + orr v3.16b, v3.16b, v1.16b .endm check_reg_neon 8, 9 check_reg_neon 10, 11 check_reg_neon 12, 13 check_reg_neon 14, 15 @@ -146,11 +183,15 @@ check_reg x25, x26 check_reg x27, x28 cbz x3, 0f - movrel x0, error_message + movrel x0, error_message_register + b 1f +2: + movrel x0, error_message_stack +1: #ifdef PREFIX bl _checkasm_fail_func #else bl checkasm_fail_func #endif diff --git a/third_party/dav1d/tests/checkasm/checkasm.h b/third_party/dav1d/tests/checkasm/checkasm.h --- a/third_party/dav1d/tests/checkasm/checkasm.h +++ b/third_party/dav1d/tests/checkasm/checkasm.h @@ -199,54 +199,68 @@ * which is problematic when trying to benchmark individual functions. We can * work around this by periodically issuing "dummy" instructions that uses * those registers to keep them powered on. */ void checkasm_simd_warmup(void); #define declare_new(ret, ...)\ - ret (*checked_call)(void *, int, int, int, int, int, __VA_ARGS__) =\ + ret (*checked_call)(void *, int, int, int, int, int, __VA_ARGS__,\ + int, int, int, int, int, int, int, int,\ + int, int, int, int, int, int, int) =\ (void *)checkasm_checked_call; #define CLOB (UINT64_C(0xdeadbeefdeadbeef)) +#ifdef _WIN32 +#define STACKARGS 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 0, 0, 0 +#else +#define STACKARGS 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 0, 0, 0, 0, 0 +#endif #define call_new(...)\ (checkasm_set_signal_handler_state(1),\ checkasm_simd_warmup(),\ checkasm_stack_clobber(CLOB, CLOB, CLOB, CLOB, CLOB, CLOB, CLOB,\ CLOB, CLOB, CLOB, CLOB, CLOB, CLOB, CLOB,\ CLOB, CLOB, CLOB, CLOB, CLOB, CLOB, CLOB),\ - checked_call(func_new, 0, 0, 0, 0, 0, __VA_ARGS__));\ + checked_call(func_new, 0, 0, 0, 0, 0, __VA_ARGS__, STACKARGS));\ checkasm_set_signal_handler_state(0) #elif ARCH_X86_32 #define declare_new(ret, ...)\ - ret (*checked_call)(void *, __VA_ARGS__) = (void *)checkasm_checked_call; + ret (*checked_call)(void *, __VA_ARGS__, int, int, int, int, int, int,\ + int, int, int, int, int, int, int, int, int) =\ + (void *)checkasm_checked_call; #define call_new(...)\ (checkasm_set_signal_handler_state(1),\ - checked_call(func_new, __VA_ARGS__));\ + checked_call(func_new, __VA_ARGS__, 15, 14, 13, 12,\ + 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1));\ checkasm_set_signal_handler_state(0) #elif ARCH_ARM /* Use a dummy argument, to offset the real parameters by 2, not only 1. * This makes sure that potential 8-byte-alignment of parameters is kept * the same even when the extra parameters have been removed. */ void checkasm_checked_call_vfp(void *func, int dummy, ...); #define declare_new(ret, ...)\ - ret (*checked_call)(void *, int dummy, __VA_ARGS__) =\ + ret (*checked_call)(void *, int dummy, __VA_ARGS__,\ + int, int, int, int, int, int, int, int,\ + int, int, int, int, int, int, int) =\ (void *)checkasm_checked_call_vfp; #define call_new(...)\ (checkasm_set_signal_handler_state(1),\ - checked_call(func_new, 0, __VA_ARGS__));\ + checked_call(func_new, 0, __VA_ARGS__, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 0, 0, 0));\ checkasm_set_signal_handler_state(0) #elif ARCH_AARCH64 && !defined(__APPLE__) void checkasm_stack_clobber(uint64_t clobber, ...); #define declare_new(ret, ...)\ ret (*checked_call)(void *, int, int, int, int, int, int, int,\ - __VA_ARGS__) =\ + __VA_ARGS__, int, int, int, int, int, int, int, int,\ + int, int, int, int, int, int, int) =\ (void *)checkasm_checked_call; #define CLOB (UINT64_C(0xdeadbeefdeadbeef)) #define call_new(...)\ (checkasm_set_signal_handler_state(1),\ checkasm_stack_clobber(CLOB, CLOB, CLOB, CLOB, CLOB, CLOB,\ CLOB, CLOB, CLOB, CLOB, CLOB, CLOB,\ CLOB, CLOB, CLOB, CLOB, CLOB, CLOB,\ CLOB, CLOB, CLOB, CLOB, CLOB),\ - checked_call(func_new, 0, 0, 0, 0, 0, 0, 0, __VA_ARGS__));\ + checked_call(func_new, 0, 0, 0, 0, 0, 0, 0, __VA_ARGS__,\ + 7, 6, 5, 4, 3, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0));\ checkasm_set_signal_handler_state(0) #else #define declare_new(ret, ...) #define call_new(...)\ (checkasm_set_signal_handler_state(1),\ @@ -268,12 +282,12 @@ do {\ if (checkasm_bench_func()) {\ checkasm_set_signal_handler_state(1);\ func_type *tfunc = func_new;\ uint64_t tsum = 0;\ - int ti, tcount = 0;\ - for (ti = 0; ti < BENCH_RUNS; ti++) {\ + int tcount = 0;\ + for (int ti = 0; ti < BENCH_RUNS; ti++) {\ uint64_t t = readtime();\ tfunc(__VA_ARGS__);\ tfunc(__VA_ARGS__);\ tfunc(__VA_ARGS__);\ tfunc(__VA_ARGS__);\ diff --git a/third_party/dav1d/tests/checkasm/checkasm.c b/third_party/dav1d/tests/checkasm/checkasm.c --- a/third_party/dav1d/tests/checkasm/checkasm.c +++ b/third_party/dav1d/tests/checkasm/checkasm.c @@ -123,11 +123,11 @@ /* Binary search tree node */ typedef struct CheckasmFunc { struct CheckasmFunc *child[2]; CheckasmFuncVersion versions; uint8_t color; /* 0 = red, 1 = black */ - char name[1]; + char name[]; } CheckasmFunc; /* Internal state */ static struct { CheckasmFunc *funcs; @@ -140,11 +140,11 @@ int num_failed; int nop_time; unsigned cpu_flag; const char *cpu_flag_name; const char *test_name; - unsigned int seed; + unsigned seed; int bench_c; int verbose; int function_listing; #if ARCH_X86_64 void (*simd_warmup)(void); @@ -157,11 +157,11 @@ uint32_t i; } intfloat; static uint32_t xs_state[4]; -static void xor128_srand(unsigned int seed) { +static void xor128_srand(unsigned seed) { xs_state[0] = seed; xs_state[1] = ( seed & 0xffff0000) | (~seed & 0x0000ffff); xs_state[2] = (~seed & 0xffff0000) | ( seed & 0x0000ffff); xs_state[3] = ~seed; } @@ -333,19 +333,19 @@ } /* Measure the overhead of the timing code (in decicycles) */ static int measure_nop_time(void) { uint16_t nops[10000]; - int i, nop_sum = 0; + int nop_sum = 0; - for (i = 0; i < 10000; i++) { + for (int i = 0; i < 10000; i++) { uint64_t t = readtime(); nops[i] = (uint16_t) (readtime() - t); } qsort(nops, 10000, sizeof(uint16_t), cmp_nop); - for (i = 2500; i < 7500; i++) + for (int i = 2500; i < 7500; i++) nop_sum += nops[i]; return nop_sum / 500; } @@ -357,12 +357,12 @@ /* Only print functions with at least one assembly version */ if (state.bench_c || f->versions.cpu || f->versions.next) { const CheckasmFuncVersion *v = &f->versions; do { if (v->iterations) { - int decicycles = (int) (10*v->cycles/v->iterations - - state.nop_time) / 4; + const int decicycles = (int) (10*v->cycles/v->iterations - + state.nop_time) / 4; printf("%s_%s: %d.%d\n", f->name, cpu_suffix(v->cpu), decicycles/10, decicycles%10); } } while ((v = v->next)); } @@ -411,11 +411,11 @@ } #define is_red(f) ((f) && !(f)->color) /* Balance a left-leaning red-black tree at the specified node */ -static void balance_tree(CheckasmFunc **root) { +static void balance_tree(CheckasmFunc **const root) { CheckasmFunc *const f = *root; if (is_red(f->child[0]) && is_red(f->child[1])) { f->color ^= 1; f->child[0]->color = f->child[1]->color = 1; @@ -425,28 +425,28 @@ else if (is_red(f->child[0]) && is_red(f->child[0]->child[0])) *root = rotate_tree(f, 1); /* Rotate right */ } /* Get a node with the specified name, creating it if it doesn't exist */ -static CheckasmFunc *get_func(CheckasmFunc **root, const char *const name) { +static CheckasmFunc *get_func(CheckasmFunc **const root, const char *const name) { CheckasmFunc *f = *root; if (f) { /* Search the tree for a matching node */ - int cmp = cmp_func_names(name, f->name); + const int cmp = cmp_func_names(name, f->name); if (cmp) { f = get_func(&f->child[cmp > 0], name); /* Rebalance the tree on the way up if a new node was inserted */ if (!f->versions.func) balance_tree(root); } } else { /* Allocate and insert a new node into the tree */ - const size_t name_length = strlen(name); - f = *root = checkasm_malloc(sizeof(CheckasmFunc) + name_length); - memcpy(f->name, name, name_length + 1); + const size_t name_length = strlen(name) + 1; + f = *root = checkasm_malloc(offsetof(CheckasmFunc, name) + name_length); + memcpy(f->name, name, name_length); } return f; } @@ -557,32 +557,33 @@ printf("%s\n", tests[i].name); return 0; } else if (!strcmp(argv[1], "--verbose") || !strcmp(argv[1], "-v")) { state.verbose = 1; } else { - state.seed = (unsigned int) strtoul(argv[1], NULL, 10); + state.seed = (unsigned) strtoul(argv[1], NULL, 10); } argc--; argv++; } - fprintf(stderr, "checkasm: using random seed %u\n", state.seed); + dav1d_init_cpu(); - dav1d_init_cpu(); + if (!state.function_listing) { + fprintf(stderr, "checkasm: using random seed %u\n", state.seed); #if ARCH_X86_64 - void checkasm_warmup_avx2(void); - void checkasm_warmup_avx512(void); - unsigned cpu_flags = dav1d_get_cpu_flags(); - if (cpu_flags & DAV1D_X86_CPU_FLAG_AVX512ICL) - state.simd_warmup = checkasm_warmup_avx512; - else if (cpu_flags & DAV1D_X86_CPU_FLAG_AVX2) - state.simd_warmup = checkasm_warmup_avx2; - else - state.simd_warmup = NULL; - checkasm_simd_warmup(); + void checkasm_warmup_avx2(void); + void checkasm_warmup_avx512(void); + const unsigned cpu_flags = dav1d_get_cpu_flags(); + if (cpu_flags & DAV1D_X86_CPU_FLAG_AVX512ICL) + state.simd_warmup = checkasm_warmup_avx512; + else if (cpu_flags & DAV1D_X86_CPU_FLAG_AVX2) + state.simd_warmup = checkasm_warmup_avx2; + checkasm_simd_warmup(); #endif + } + check_cpu_flag(NULL, 0); if (state.function_listing) { print_functions(state.funcs); } else { diff --git a/third_party/dav1d/tests/checkasm/filmgrain.c b/third_party/dav1d/tests/checkasm/filmgrain.c --- a/third_party/dav1d/tests/checkasm/filmgrain.c +++ b/third_party/dav1d/tests/checkasm/filmgrain.c @@ -179,12 +179,12 @@ fg_data[0].num_y_points, scaling); const int w = 1 + (rnd() & 127); const int h = 1 + (rnd() & 31); - for (int y = 0; y < h; y++) - for (int x = 0; x < w; x++) + for (int y = 0; y < 32; y++) + for (int x = 0; x < 128; x++) src[y * PXSTRIDE(stride) + x] = rnd() & bitdepth_max; const int row_num = rnd() & 1 ? rnd() & 0x7ff : 0; fg_data[0].clip_to_restricted_range = rnd() & 1; fg_data[0].scaling_shift = (rnd() & 3) + 8; @@ -258,17 +258,16 @@ dsp->generate_grain_uv[layout_idx](grain_lut[1], grain_lut[0], fg_data, uv_pl HIGHBD_TAIL_SUFFIX); const int w = 1 + (rnd() & (127 >> ss_x)); const int h = 1 + (rnd() & (31 >> ss_y)); - const int lw = w << ss_x, lh = h << ss_y; - for (int y = 0; y < h; y++) - for (int x = 0; x < w; x++) + for (int y = 0; y < 32; y++) + for (int x = 0; x < 128; x++) src[y * PXSTRIDE(stride) + x] = rnd() & bitdepth_max; - for (int y = 0; y < lh; y++) - for (int x = 0; x < lw; x++) + for (int y = 0; y < 32; y++) + for (int x = 0; x < 128; x++) luma_src[y * PXSTRIDE(lstride) + x] = rnd() & bitdepth_max; const int row_num = rnd() & 1 ? rnd() & 0x7ff : 0; if (csfl) { fg_data[0].num_y_points = 2 + (rnd() % 13); diff --git a/third_party/dav1d/tests/checkasm/ipred.c b/third_party/dav1d/tests/checkasm/ipred.c --- a/third_party/dav1d/tests/checkasm/ipred.c +++ b/third_party/dav1d/tests/checkasm/ipred.c @@ -73,63 +73,70 @@ declare_func(void, pixel *dst, ptrdiff_t stride, const pixel *topleft, int width, int height, int angle, int max_width, int max_height HIGHBD_DECL_SUFFIX); - for (int mode = 0; mode < N_IMPL_INTRA_PRED_MODES; mode++) - for (int w = 4; w <= (mode == FILTER_PRED ? 32 : 64); w <<= 1) - if (check_func(c->intra_pred[mode], "intra_pred_%s_w%d_%dbpc", - intra_pred_mode_names[mode], w, BITDEPTH)) - { - for (int h = imax(w / 4, 4); h <= imin(w * 4, - (mode == FILTER_PRED ? 32 : 64)); h <<= 1) + for (int mode = 0; mode < N_IMPL_INTRA_PRED_MODES; mode++) { + int bpc_min = BITDEPTH, bpc_max = BITDEPTH; + if (mode == FILTER_PRED && BITDEPTH == 16) { + bpc_min = 10; + bpc_max = 12; + } + for (int bpc = bpc_min; bpc <= bpc_max; bpc += 2) + for (int w = 4; w <= (mode == FILTER_PRED ? 32 : 64); w <<= 1) + if (check_func(c->intra_pred[mode], "intra_pred_%s_w%d_%dbpc", + intra_pred_mode_names[mode], w, bpc)) { - const ptrdiff_t stride = w * sizeof(pixel); + for (int h = imax(w / 4, 4); h <= imin(w * 4, + (mode == FILTER_PRED ? 32 : 64)); h <<= 1) + { + const ptrdiff_t stride = w * sizeof(pixel); - int a = 0, maxw = 0, maxh = 0; - if (mode >= Z1_PRED && mode <= Z3_PRED) { /* angle */ - a = (90 * (mode - Z1_PRED) + z_angles[rnd() % 27]) | - (rnd() & 0x600); - if (mode == Z2_PRED) { - maxw = rnd(), maxh = rnd(); - maxw = 1 + (maxw & (maxw & 4096 ? 4095 : w - 1)); - maxh = 1 + (maxh & (maxh & 4096 ? 4095 : h - 1)); - } - } else if (mode == FILTER_PRED) /* filter_idx */ - a = (rnd() % 5) | (rnd() & ~511); + int a = 0, maxw = 0, maxh = 0; + if (mode >= Z1_PRED && mode <= Z3_PRED) { /* angle */ + a = (90 * (mode - Z1_PRED) + z_angles[rnd() % 27]) | + (rnd() & 0x600); + if (mode == Z2_PRED) { + maxw = rnd(), maxh = rnd(); + maxw = 1 + (maxw & (maxw & 4096 ? 4095 : w - 1)); + maxh = 1 + (maxh & (maxh & 4096 ? 4095 : h - 1)); + } + } else if (mode == FILTER_PRED) /* filter_idx */ + a = (rnd() % 5) | (rnd() & ~511); -#if BITDEPTH == 16 - const int bitdepth_max = rnd() & 1 ? 0x3ff : 0xfff; -#else - const int bitdepth_max = 0xff; -#endif + int bitdepth_max; + if (bpc == 16) + bitdepth_max = rnd() & 1 ? 0x3ff : 0xfff; + else + bitdepth_max = (1 << bpc) - 1; - for (int i = -h * 2; i <= w * 2; i++) - topleft[i] = rnd() & bitdepth_max; + for (int i = -h * 2; i <= w * 2; i++) + topleft[i] = rnd() & bitdepth_max; - call_ref(c_dst, stride, topleft, w, h, a, maxw, maxh - HIGHBD_TAIL_SUFFIX); - call_new(a_dst, stride, topleft, w, h, a, maxw, maxh - HIGHBD_TAIL_SUFFIX); - if (checkasm_check_pixel(c_dst, stride, a_dst, stride, - w, h, "dst")) - { - if (mode == Z1_PRED || mode == Z3_PRED) - fprintf(stderr, "angle = %d (0x%03x)\n", - a & 0x1ff, a & 0x600); - else if (mode == Z2_PRED) - fprintf(stderr, "angle = %d (0x%03x), " - "max_width = %d, max_height = %d\n", - a & 0x1ff, a & 0x600, maxw, maxh); - else if (mode == FILTER_PRED) - fprintf(stderr, "filter_idx = %d\n", a & 0x1ff); + call_ref(c_dst, stride, topleft, w, h, a, maxw, maxh + HIGHBD_TAIL_SUFFIX); + call_new(a_dst, stride, topleft, w, h, a, maxw, maxh + HIGHBD_TAIL_SUFFIX); + if (checkasm_check_pixel(c_dst, stride, a_dst, stride, + w, h, "dst")) + { + if (mode == Z1_PRED || mode == Z3_PRED) + fprintf(stderr, "angle = %d (0x%03x)\n", + a & 0x1ff, a & 0x600); + else if (mode == Z2_PRED) + fprintf(stderr, "angle = %d (0x%03x), " + "max_width = %d, max_height = %d\n", + a & 0x1ff, a & 0x600, maxw, maxh); + else if (mode == FILTER_PRED) + fprintf(stderr, "filter_idx = %d\n", a & 0x1ff); + } + + bench_new(a_dst, stride, topleft, w, h, a, 128, 128 + HIGHBD_TAIL_SUFFIX); } - - bench_new(a_dst, stride, topleft, w, h, a, 128, 128 - HIGHBD_TAIL_SUFFIX); } - } + } report("intra_pred"); } static void check_cfl_ac(Dav1dIntraPredDSPContext *const c) { ALIGN_STK_64(int16_t, c_dst, 32 * 32,); @@ -140,18 +147,25 @@ int w_pad, int h_pad, int cw, int ch); for (int layout = 1; layout <= DAV1D_PIXEL_LAYOUT_I444; layout++) { const int ss_ver = layout == DAV1D_PIXEL_LAYOUT_I420; const int ss_hor = layout != DAV1D_PIXEL_LAYOUT_I444; + const int h_step = 2 >> ss_hor, v_step = 2 >> ss_ver; for (int w = 4; w <= (32 >> ss_hor); w <<= 1) if (check_func(c->cfl_ac[layout - 1], "cfl_ac_%s_w%d_%dbpc", cfl_ac_names[layout - 1], w, BITDEPTH)) { - for (int h = imax(w / 4, 4); h <= imin(w * 4, (32 >> ss_ver)); h <<= 1) { + for (int h = imax(w / 4, 4); + h <= imin(w * 4, (32 >> ss_ver)); h <<= 1) + { const ptrdiff_t stride = 32 * sizeof(pixel); - for (int w_pad = (w >> 2) - 1; w_pad >= 0; w_pad--) { - for (int h_pad = (h >> 2) - 1; h_pad >= 0; h_pad--) { + for (int w_pad = imax((w >> 2) - h_step, 0); + w_pad >= 0; w_pad -= h_step) + { + for (int h_pad = imax((h >> 2) - v_step, 0); + h_pad >= 0; h_pad -= v_step) + { #if BITDEPTH == 16 const int bitdepth_max = rnd() & 1 ? 0x3ff : 0xfff; #else const int bitdepth_max = 0xff; #endif diff --git a/third_party/dav1d/tests/checkasm/itx.c b/third_party/dav1d/tests/checkasm/itx.c --- a/third_party/dav1d/tests/checkasm/itx.c +++ b/third_party/dav1d/tests/checkasm/itx.c @@ -221,16 +221,20 @@ return copy_subcoefs(buf, tx, txtp, sw, sh, subsh); } void bitfn(checkasm_check_itx)(void) { - Dav1dInvTxfmDSPContext c; - bitfn(dav1d_itx_dsp_init)(&c); +#if BITDEPTH == 16 + const int bpc_min = 10, bpc_max = 12; +#else + const int bpc_min = 8, bpc_max = 8; +#endif ALIGN_STK_64(coef, coeff, 2, [32 * 32]); ALIGN_STK_64(pixel, c_dst, 64 * 64,); ALIGN_STK_64(pixel, a_dst, 64 * 64,); + Dav1dInvTxfmDSPContext c = { { { 0 } } }; /* Zero unused function pointer elements. */ static const uint8_t txfm_size_order[N_RECT_TX_SIZES] = { TX_4X4, RTX_4X8, RTX_4X16, RTX_8X4, TX_8X8, RTX_8X16, RTX_8X32, RTX_16X4, RTX_16X8, TX_16X16, RTX_16X32, RTX_16X64, @@ -248,41 +252,40 @@ const int w = dav1d_txfm_dimensions[tx].w * 4; const int h = dav1d_txfm_dimensions[tx].h * 4; const int subsh_max = subsh_iters[imax(dav1d_txfm_dimensions[tx].lw, dav1d_txfm_dimensions[tx].lh)]; - for (enum TxfmType txtp = 0; txtp < N_TX_TYPES_PLUS_LL; txtp++) - for (int subsh = 0; subsh < subsh_max; subsh++) - if (check_func(c.itxfm_add[tx][txtp], - "inv_txfm_add_%dx%d_%s_%s_%d_%dbpc", - w, h, itx_1d_names[itx_1d_types[txtp][0]], - itx_1d_names[itx_1d_types[txtp][1]], subsh, - BITDEPTH)) - { -#if BITDEPTH == 16 - const int bitdepth_max = rnd() & 1 ? 0x3ff : 0xfff; -#else - const int bitdepth_max = 0xff; -#endif - const int eob = ftx(coeff[0], tx, txtp, w, h, subsh, bitdepth_max); - memcpy(coeff[1], coeff[0], sizeof(*coeff)); + for (int bpc = bpc_min; bpc <= bpc_max; bpc += 2) { + bitfn(dav1d_itx_dsp_init)(&c, bpc); + for (enum TxfmType txtp = 0; txtp < N_TX_TYPES_PLUS_LL; txtp++) + for (int subsh = 0; subsh < subsh_max; subsh++) + if (check_func(c.itxfm_add[tx][txtp], + "inv_txfm_add_%dx%d_%s_%s_%d_%dbpc", + w, h, itx_1d_names[itx_1d_types[txtp][0]], + itx_1d_names[itx_1d_types[txtp][1]], subsh, + bpc)) + { + const int bitdepth_max = (1 << bpc) - 1; + const int eob = ftx(coeff[0], tx, txtp, w, h, subsh, bitdepth_max); + memcpy(coeff[1], coeff[0], sizeof(*coeff)); - for (int j = 0; j < w * h; j++) - c_dst[j] = a_dst[j] = rnd() & bitdepth_max; + for (int j = 0; j < w * h; j++) + c_dst[j] = a_dst[j] = rnd() & bitdepth_max; - call_ref(c_dst, w * sizeof(*c_dst), coeff[0], eob - HIGHBD_TAIL_SUFFIX); - call_new(a_dst, w * sizeof(*c_dst), coeff[1], eob - HIGHBD_TAIL_SUFFIX); + call_ref(c_dst, w * sizeof(*c_dst), coeff[0], eob + HIGHBD_TAIL_SUFFIX); + call_new(a_dst, w * sizeof(*c_dst), coeff[1], eob + HIGHBD_TAIL_SUFFIX); - checkasm_check_pixel(c_dst, w * sizeof(*c_dst), - a_dst, w * sizeof(*a_dst), - w, h, "dst"); - if (memcmp(coeff[0], coeff[1], sizeof(*coeff))) - fail(); + checkasm_check_pixel(c_dst, w * sizeof(*c_dst), + a_dst, w * sizeof(*a_dst), + w, h, "dst"); + if (memcmp(coeff[0], coeff[1], sizeof(*coeff))) + fail(); - bench_new(a_dst, w * sizeof(*c_dst), coeff[0], eob - HIGHBD_TAIL_SUFFIX); - } + bench_new(a_dst, w * sizeof(*c_dst), coeff[0], eob + HIGHBD_TAIL_SUFFIX); + } + } report("add_%dx%d", w, h); } } diff --git a/third_party/dav1d/tests/checkasm/mc.c b/third_party/dav1d/tests/checkasm/mc.c --- a/third_party/dav1d/tests/checkasm/mc.c +++ b/third_party/dav1d/tests/checkasm/mc.c @@ -36,10 +36,11 @@ "8tap_smooth_regular", "8tap_smooth", "8tap_smooth_sharp", "bilinear" }; static const char *const mxy_names[] = { "0", "h", "v", "hv" }; +static const char *const scaled_paths[] = { "", "_dy1", "_dy2" }; static int mc_h_next(const int h) { switch (h) { case 4: case 8: @@ -159,10 +160,116 @@ } } report("mct"); } +static void check_mc_scaled(Dav1dMCDSPContext *const c) { + ALIGN_STK_64(pixel, src_buf, 263 * 263,); + ALIGN_STK_64(pixel, c_dst, 128 * 128,); + ALIGN_STK_64(pixel, a_dst, 128 * 128,); + const pixel *src = src_buf + 263 * 3 + 3; + const ptrdiff_t src_stride = 263 * sizeof(pixel); +#if BITDEPTH == 16 + const int bitdepth_max = rnd() & 1 ? 0x3ff : 0xfff; +#else + const int bitdepth_max = 0xff; +#endif + + declare_func(void, pixel *dst, ptrdiff_t dst_stride, const pixel *src, + ptrdiff_t src_stride, int w, int h, + int mx, int my, int dx, int dy HIGHBD_DECL_SUFFIX); + + for (int filter = 0; filter < N_2D_FILTERS; filter++) + for (int w = 2; w <= 128; w <<= 1) { + const ptrdiff_t dst_stride = w * sizeof(pixel); + for (int p = 0; p < 3; ++p) { + if (check_func(c->mc_scaled[filter], "mc_scaled_%s_w%d%s_%dbpc", + filter_names[filter], w, scaled_paths[p], BITDEPTH)) + { + const int h_min = w <= 32 ? 2 : w / 4; + const int h_max = imax(imin(w * 4, 128), 32); + for (int h = h_min; h <= h_max; h = mc_h_next(h)) { + const int mx = rnd() % 1024; + const int my = rnd() % 1024; + const int dx = rnd() % 2048 + 1; + const int dy = !p + ? rnd() % 2048 + 1 + : p << 10; // ystep=1.0 and ystep=2.0 paths + + for (int k = 0; k < 263 * 263; k++) + src_buf[k] = rnd() & bitdepth_max; + + call_ref(c_dst, dst_stride, src, src_stride, + w, h, mx, my, dx, dy HIGHBD_TAIL_SUFFIX); + call_new(a_dst, dst_stride, src, src_stride, + w, h, mx, my, dx, dy HIGHBD_TAIL_SUFFIX); + checkasm_check_pixel(c_dst, dst_stride, + a_dst, dst_stride, w, h, "dst"); + + if (filter == FILTER_2D_8TAP_REGULAR || + filter == FILTER_2D_BILINEAR) + bench_new(a_dst, dst_stride, src, src_stride, + w, h, mx, my, dx, dy HIGHBD_TAIL_SUFFIX); + } + } + } + } + report("mc_scaled"); +} + +static void check_mct_scaled(Dav1dMCDSPContext *const c) { + ALIGN_STK_64(pixel, src_buf, 263 * 263,); + ALIGN_STK_64(int16_t, c_tmp, 128 * 128,); + ALIGN_STK_64(int16_t, a_tmp, 128 * 128,); + const pixel *src = src_buf + 263 * 3 + 3; + const ptrdiff_t src_stride = 263 * sizeof(pixel); +#if BITDEPTH == 16 + const int bitdepth_max = rnd() & 1 ? 0x3ff : 0xfff; +#else + const int bitdepth_max = 0xff; +#endif + + declare_func(void, int16_t *tmp, const pixel *src, ptrdiff_t src_stride, + int w, int h, int mx, int my, int dx, int dy HIGHBD_DECL_SUFFIX); + + for (int filter = 0; filter < N_2D_FILTERS; filter++) + for (int w = 4; w <= 128; w <<= 1) + for (int p = 0; p < 3; ++p) { + if (check_func(c->mct_scaled[filter], "mct_scaled_%s_w%d%s_%dbpc", + filter_names[filter], w, scaled_paths[p], BITDEPTH)) + { + const int h_min = imax(w / 4, 4); + const int h_max = imin(w * 4, 128); + for (int h = h_min; h <= h_max; h = mc_h_next(h)) { + const int mx = rnd() % 1024; + const int my = rnd() % 1024; + const int dx = rnd() % 2048 + 1; + const int dy = !p + ? rnd() % 2048 + 1 + : p << 10; // ystep=1.0 and ystep=2.0 paths + + for (int k = 0; k < 263 * 263; k++) + src_buf[k] = rnd() & bitdepth_max; + + call_ref(c_tmp, src, src_stride, + w, h, mx, my, dx, dy HIGHBD_TAIL_SUFFIX); + call_new(a_tmp, src, src_stride, + w, h, mx, my, dx, dy HIGHBD_TAIL_SUFFIX); + checkasm_check(int16_t, c_tmp, w * sizeof(*c_tmp), + a_tmp, w * sizeof(*a_tmp), + w, h, "tmp"); + + if (filter == FILTER_2D_8TAP_REGULAR || + filter == FILTER_2D_BILINEAR) + bench_new(a_tmp, src, src_stride, + w, h, mx, my, dx, dy HIGHBD_TAIL_SUFFIX); + } + } + } + report("mct_scaled"); +} + static void init_tmp(Dav1dMCDSPContext *const c, pixel *const buf, int16_t (*const tmp)[128 * 128], const int bitdepth_max) { for (int i = 0; i < 2; i++) { generate_mct_input(buf, bitdepth_max); @@ -571,22 +678,79 @@ } } report("emu_edge"); } +static int get_upscale_x0(const int in_w, const int out_w, const int step) { + const int err = out_w * step - (in_w << 14); + const int x0 = (-((out_w - in_w) << 13) + (out_w >> 1)) / out_w + 128 - (err >> 1); + return x0 & 0x3fff; +} + +static void check_resize(Dav1dMCDSPContext *const c) { + ALIGN_STK_64(pixel, c_dst, 1024 * 64,); + ALIGN_STK_64(pixel, a_dst, 1024 * 64,); + ALIGN_STK_64(pixel, src, 512 * 64,); + + const int height = 64; + const int max_src_width = 512; + const ptrdiff_t dst_stride = 1024 * sizeof(pixel); + const ptrdiff_t src_stride = 512 * sizeof(pixel); + + declare_func(void, pixel *dst, ptrdiff_t dst_stride, + const pixel *src, ptrdiff_t src_stride, + int dst_w, int src_w, int h, int dx, int mx0 + HIGHBD_DECL_SUFFIX); + + if (check_func(c->resize, "resize_%dbpc", BITDEPTH)) { +#if BITDEPTH == 16 + const int bitdepth_max = rnd() & 1 ? 0x3ff : 0xfff; +#else + const int bitdepth_max = 0xff; +#endif + + for (int i = 0; i < max_src_width * height; i++) + src[i] = rnd() & bitdepth_max; + + const int w_den = 9 + (rnd() & 7); + const int src_w = 16 + (rnd() % (max_src_width - 16 + 1)); + const int dst_w = w_den * src_w >> 3; +#define scale_fac(ref_sz, this_sz) \ + ((((ref_sz) << 14) + ((this_sz) >> 1)) / (this_sz)) + const int dx = scale_fac(src_w, dst_w); +#undef scale_fac + const int mx0 = get_upscale_x0(src_w, dst_w, dx); + + call_ref(c_dst, dst_stride, src, src_stride, + dst_w, height, src_w, dx, mx0 HIGHBD_TAIL_SUFFIX); + call_new(a_dst, dst_stride, src, src_stride, + dst_w, height, src_w, dx, mx0 HIGHBD_TAIL_SUFFIX); + checkasm_check_pixel(c_dst, dst_stride, a_dst, dst_stride, + dst_w, height, "dst"); + + bench_new(a_dst, dst_stride, src, src_stride, + 512, height, 512 * 8 / w_den, dx, mx0 HIGHBD_TAIL_SUFFIX); + } + + report("resize"); +} + void bitfn(checkasm_check_mc)(void) { Dav1dMCDSPContext c; bitfn(dav1d_mc_dsp_init)(&c); check_mc(&c); check_mct(&c); + check_mc_scaled(&c); + check_mct_scaled(&c); check_avg(&c); check_w_avg(&c); check_mask(&c); check_w_mask(&c); check_blend(&c); check_blend_v(&c); check_blend_h(&c); check_warp8x8(&c); check_warp8x8t(&c); check_emuedge(&c); + check_resize(&c); } diff --git a/third_party/dav1d/tests/checkasm/msac.c b/third_party/dav1d/tests/checkasm/msac.c --- a/third_party/dav1d/tests/checkasm/msac.c +++ b/third_party/dav1d/tests/checkasm/msac.c @@ -237,18 +237,19 @@ c.bool_adapt = dav1d_msac_decode_bool_adapt_c; c.bool_equi = dav1d_msac_decode_bool_equi_c; c.bool = dav1d_msac_decode_bool_c; c.hi_tok = dav1d_msac_decode_hi_tok_c; -#if ARCH_AARCH64 && HAVE_ASM +#if (ARCH_AARCH64 || ARCH_ARM) && HAVE_ASM if (dav1d_get_cpu_flags() & DAV1D_ARM_CPU_FLAG_NEON) { c.symbol_adapt4 = dav1d_msac_decode_symbol_adapt4_neon; c.symbol_adapt8 = dav1d_msac_decode_symbol_adapt8_neon; c.symbol_adapt16 = dav1d_msac_decode_symbol_adapt16_neon; c.bool_adapt = dav1d_msac_decode_bool_adapt_neon; c.bool_equi = dav1d_msac_decode_bool_equi_neon; c.bool = dav1d_msac_decode_bool_neon; + c.hi_tok = dav1d_msac_decode_hi_tok_neon; } #elif ARCH_X86 && HAVE_ASM if (dav1d_get_cpu_flags() & DAV1D_X86_CPU_FLAG_SSE2) { c.symbol_adapt4 = dav1d_msac_decode_symbol_adapt4_sse2; c.symbol_adapt8 = dav1d_msac_decode_symbol_adapt8_sse2; diff --git a/third_party/dav1d/tests/checkasm/x86/checkasm.asm b/third_party/dav1d/tests/checkasm/x86/checkasm.asm --- a/third_party/dav1d/tests/checkasm/x86/checkasm.asm +++ b/third_party/dav1d/tests/checkasm/x86/checkasm.asm @@ -25,17 +25,15 @@ %define private_prefix checkasm %include "config.asm" %include "ext/x86/x86inc.asm" -SECTION_RODATA - -error_message: db "failed to preserve register", 0 +SECTION_RODATA 16 %if ARCH_X86_64 ; just random numbers to reduce the chance of incidental match -ALIGN 16 +%if WIN64 x6: dq 0x1a1b2550a612b48c,0x79445c159ce79064 x7: dq 0x2eed899d5a28ddcd,0x86b2536fcd8cf636 x8: dq 0xb0856806085e7943,0x3f2bf84fc0fcca4e x9: dq 0xacbd382dcf5b8de2,0xd229e1f5b281303f x10: dq 0x71aeaff20b095fd9,0xab63e2e11fa38ed9 @@ -44,18 +42,22 @@ x13: dq 0x2e8ec680de14b47c,0xdd7b8919edd42786 x14: dq 0x135ce6888fa02cbf,0x11e53e2b2ac655ef x15: dq 0x011ff554472a7a10,0x6de8f4c914c334d5 n7: dq 0x21f86d66c8ca00ce n8: dq 0x75b6ba21077c48ad +%endif n9: dq 0xed56bb2dcb3c7736 n10: dq 0x8bda43d3fd1a7e06 n11: dq 0xb64a9c9e5d318408 n12: dq 0xdf9a54b303f1d3a3 n13: dq 0x4a75479abd64e097 n14: dq 0x249214109d5d1c88 %endif +errmsg_reg: db "failed to preserve register", 0 +errmsg_stack: db "stack corruption", 0 + SECTION .text cextern fail_func ; max number of args used by any asm function. @@ -65,11 +67,11 @@ %if ARCH_X86_64 ;----------------------------------------------------------------------------- ; int checkasm_stack_clobber(uint64_t clobber, ...) ;----------------------------------------------------------------------------- -cglobal stack_clobber, 1,2 +cglobal stack_clobber, 1, 2 ; Clobber the stack with junk below the stack pointer %define argsize (max_args+6)*8 SUB rsp, argsize mov r1, argsize-8 .loop: @@ -79,21 +81,25 @@ ADD rsp, argsize RET %if WIN64 %assign free_regs 7 + %define stack_param rsp+32 ; shadow space + %define num_stack_params rsp+stack_offset+22*8 DECLARE_REG_TMP 4 %else %assign free_regs 9 + %define stack_param rsp + %define num_stack_params rsp+stack_offset+16*8 DECLARE_REG_TMP 7 %endif ;----------------------------------------------------------------------------- ; void checkasm_checked_call(void *func, ...) ;----------------------------------------------------------------------------- INIT_XMM -cglobal checked_call, 2,15,16,max_args*8+8 +cglobal checked_call, 2, 15, 16, max_args*8+64+8 mov t0, r0 ; All arguments have been pushed on the stack instead of registers in ; order to test for incorrect assumptions that 32-bit ints are ; zero-extended to 64-bit. @@ -102,24 +108,11 @@ mov r2, r8mp mov r3, r9mp %if UNIX64 mov r4, r10mp mov r5, r11mp - %assign i 6 - %rep max_args-6 - mov r9, [rsp+stack_offset+(i+1)*8] - mov [rsp+(i-6)*8], r9 - %assign i i+1 - %endrep %else ; WIN64 - %assign i 4 - %rep max_args-4 - mov r9, [rsp+stack_offset+(i+7)*8] - mov [rsp+i*8], r9 - %assign i i+1 - %endrep - ; Move possible floating-point arguments to the correct registers movq m0, r0 movq m1, r1 movq m2, r2 movq m3, r3 @@ -129,42 +122,80 @@ mova m %+ i, [x %+ i] %assign i i+1 %endrep %endif + ; write stack canaries to the area above parameters passed on the stack + mov r9d, [num_stack_params] + mov r8, [rsp+stack_offset] ; return address + not r8 +%assign i 0 +%rep 8 ; 64 bytes + mov [stack_param+(r9+i)*8], r8 + %assign i i+1 +%endrep + dec r9d + jl .stack_setup_done ; no stack parameters +.copy_stack_parameter: + mov r8, [stack_param+stack_offset+7*8+r9*8] + mov [stack_param+r9*8], r8 + dec r9d + jge .copy_stack_parameter +.stack_setup_done: + %assign i 14 %rep 15-free_regs mov r %+ i, [n %+ i] %assign i i-1 %endrep call t0 -%assign i 14 -%rep 15-free_regs + + ; check for failure to preserve registers + xor r14, [n14] + lea r0, [errmsg_reg] +%assign i 13 +%rep 14-free_regs xor r %+ i, [n %+ i] or r14, r %+ i %assign i i-1 %endrep - %if WIN64 - %assign i 6 - %rep 16-6 + pxor m6, [x6] + %assign i 7 + %rep 16-7 pxor m %+ i, [x %+ i] por m6, m %+ i %assign i i+1 %endrep packsswb m6, m6 movq r5, m6 or r14, r5 %endif + jnz .fail - ; Call fail_func() with a descriptive message to mark it as a failure - ; if the called function didn't preserve all callee-saved registers. + ; check for stack corruption + mov r9d, [num_stack_params] + mov r8, [rsp+stack_offset] + mov r4, [stack_param+r9*8] + not r8 + xor r4, r8 +%assign i 1 +%rep 6 + mov r5, [stack_param+(r9+i)*8] + xor r5, r8 + or r4, r5 + %assign i i+1 +%endrep + xor r8, [stack_param+(r9+7)*8] + or r4, r8 + jz .ok + add r0, errmsg_stack-errmsg_reg +.fail: + ; Call fail_func() with a descriptive message to mark it as a failure. ; Save the return value located in rdx:rax first to prevent clobbering. - jz .ok mov r9, rax mov r10, rdx - lea r0, [error_message] xor eax, eax call fail_func mov rdx, r10 mov rax, r9 .ok: @@ -184,42 +215,72 @@ WARMUP %else ; just random numbers to reduce the chance of incidental match -%define n3 dword 0x6549315c -%define n4 dword 0xe02f3e23 -%define n5 dword 0xb78d0d1d -%define n6 dword 0x33627ba7 +%assign n3 0x6549315c +%assign n4 0xe02f3e23 +%assign n5 0xb78d0d1d +%assign n6 0x33627ba7 ;----------------------------------------------------------------------------- ; void checkasm_checked_call(void *func, ...) ;----------------------------------------------------------------------------- -cglobal checked_call, 1,7 +cglobal checked_call, 1, 7 + mov r3, [esp+stack_offset] ; return address + mov r1, [esp+stack_offset+17*4] ; num_stack_params + mov r2, 27 + not r3 + sub r2, r1 +.push_canary: + push r3 + dec r2 + jg .push_canary +.push_parameter: + push dword [esp+32*4] + dec r1 + jg .push_parameter mov r3, n3 mov r4, n4 mov r5, n5 mov r6, n6 -%rep max_args - PUSH dword [esp+20+max_args*4] -%endrep call r0 + + ; check for failure to preserve registers xor r3, n3 xor r4, n4 xor r5, n5 xor r6, n6 or r3, r4 or r5, r6 + LEA r1, errmsg_reg or r3, r5 + jnz .fail + + ; check for stack corruption + mov r3, [esp+48*4] ; num_stack_params + mov r6, [esp+31*4] ; return address + mov r4, [esp+r3*4] + sub r3, 26 + not r6 + xor r4, r6 +.check_canary: + mov r5, [esp+(r3+27)*4] + xor r5, r6 + or r4, r5 + inc r3 + jl .check_canary + test r4, r4 jz .ok + add r1, errmsg_stack-errmsg_reg +.fail: mov r3, eax mov r4, edx - LEA r0, error_message - mov [esp], r0 + mov [esp], r1 call fail_func - mov edx, r4 - mov eax, r3 + mov edx, r4 + mov eax, r3 .ok: - add esp, max_args*4 + add esp, 27*4 RET %endif ; ARCH_X86_64 diff --git a/third_party/dav1d/tests/libfuzzer/dav1d_fuzzer.c b/third_party/dav1d/tests/libfuzzer/dav1d_fuzzer.c --- a/third_party/dav1d/tests/libfuzzer/dav1d_fuzzer.c +++ b/third_party/dav1d/tests/libfuzzer/dav1d_fuzzer.c @@ -67,17 +67,10 @@ int have_seq_hdr = 0; int err; dav1d_version(); - // memory sanitizer is inherently incompatible with asm -#if defined(__has_feature) - #if __has_feature(memory_sanitizer) - dav1d_set_cpu_flags_mask(0); - #endif -#endif - if (size < 32) goto end; #ifdef DAV1D_ALLOC_FAIL unsigned h = djb_xor(ptr, 32); unsigned seed = h; unsigned probability = h > (RAND_MAX >> 5) ? RAND_MAX >> 5 : h; diff --git a/third_party/dav1d/tests/libfuzzer/meson.build b/third_party/dav1d/tests/libfuzzer/meson.build --- a/third_party/dav1d/tests/libfuzzer/meson.build +++ b/third_party/dav1d/tests/libfuzzer/meson.build @@ -24,10 +24,14 @@ # # Build definition for the dav1d fuzzing binaries # +if fuzzing_engine == 'none' and not have_fseeko + subdir_done() +endif + dav1d_fuzzer_sources = files('dav1d_fuzzer.c') fuzzer_ldflags = [] fuzzer_link_lang = {} if get_option('fuzzer_ldflags') != '' diff --git a/third_party/dav1d/tests/meson.build b/third_party/dav1d/tests/meson.build --- a/third_party/dav1d/tests/meson.build +++ b/third_party/dav1d/tests/meson.build @@ -61,11 +61,11 @@ ) checkasm_bitdepth_objs += checkasm_bitdepth_lib.extract_all_objects() endforeach checkasm_nasm_objs = [] - if host_machine.cpu_family() == 'aarch64' + if host_machine.cpu_family() == 'aarch64' or host_machine.cpu() == 'arm64' checkasm_sources += files('checkasm/arm/checkasm_64.S') elif host_machine.cpu_family().startswith('arm') checkasm_sources += files('checkasm/arm/checkasm_32.S') elif host_machine.cpu_family().startswith('x86') checkasm_nasm_objs = nasm_gen.process(files('checkasm/x86/checkasm.asm')) diff --git a/third_party/dav1d/tools/dav1d.c b/third_party/dav1d/tools/dav1d.c --- a/third_party/dav1d/tools/dav1d.c +++ b/third_party/dav1d/tools/dav1d.c @@ -61,11 +61,13 @@ #ifdef _WIN32 LARGE_INTEGER frequency; QueryPerformanceFrequency(&frequency); LARGE_INTEGER t; QueryPerformanceCounter(&t); - return 1000000000 * t.QuadPart / frequency.QuadPart; + uint64_t seconds = t.QuadPart / frequency.QuadPart; + uint64_t fractions = t.QuadPart % frequency.QuadPart; + return 1000000000 * seconds + 1000000000 * fractions / frequency.QuadPart; #elif defined(HAVE_CLOCK_GETTIME) struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return 1000000000ULL * ts.tv_sec + ts.tv_nsec; #elif defined(__APPLE__) @@ -243,11 +245,11 @@ } } if ((res = output_write(out, &p)) < 0) break; n_out++; - if (nspf) { + if (nspf || !cli_settings.quiet) { synchronize(cli_settings.realtime, cli_settings.realtime_cache, n_out, nspf, tfirst, &elapsed, frametimes); } if (!cli_settings.quiet) print_stats(istty, n_out, total, elapsed, i_fps); @@ -280,11 +282,11 @@ } } if ((res = output_write(out, &p)) < 0) break; n_out++; - if (nspf) { + if (nspf || !cli_settings.quiet) { synchronize(cli_settings.realtime, cli_settings.realtime_cache, n_out, nspf, tfirst, &elapsed, frametimes); } if (!cli_settings.quiet) print_stats(istty, n_out, total, elapsed, i_fps); diff --git a/third_party/dav1d/tools/dav1d_cli_parse.c b/third_party/dav1d/tools/dav1d_cli_parse.c --- a/third_party/dav1d/tools/dav1d_cli_parse.c +++ b/third_party/dav1d/tools/dav1d_cli_parse.c @@ -82,13 +82,15 @@ { NULL, 0, NULL, 0 }, }; #if ARCH_AARCH64 || ARCH_ARM #define ALLOWED_CPU_MASKS " or 'neon'" +#elif ARCH_PPC64LE +#define ALLOWED_CPU_MASKS " or 'vsx'" #elif ARCH_X86 #define ALLOWED_CPU_MASKS \ - ", 'sse2', 'ssse3', 'sse41', 'avx2', 'avx512' or 'avx512icl'" + ", 'sse2', 'ssse3', 'sse41', 'avx2' or 'avx512icl'" #else #define ALLOWED_CPU_MASKS "not yet implemented for this architecture" #endif static void usage(const char *const app, const char *const reason, ...) { @@ -100,13 +102,13 @@ va_end(args); fprintf(stderr, "\n\n"); } fprintf(stderr, "Usage: %s [options]\n\n", app); fprintf(stderr, "Supported options:\n" - " --input/-i $file: input file\n" + " --input/-i $file: input file\n" " --output/-o $file: output file\n" - " --demuxer $name: force demuxer type ('ivf', 'section5' or 'annexb'; default: detect from extension)\n" + " --demuxer $name: force demuxer type ('ivf', 'section5' or 'annexb'; default: detect from content)\n" " --muxer $name: force muxer type ('md5', 'yuv', 'yuv4mpeg2' or 'null'; default: detect from extension)\n" " --quiet/-q: disable status messages\n" " --frametimes $file: dump frame times to file\n" " --limit/-l $num: stop decoding after $num frames\n" " --skip/-s $num: skip decoding of the first $num frames\n" @@ -114,11 +116,11 @@ " --realtimecache $num: set the size of the cache in realtime mode (default: 0)\n" " --version/-v: print version and exit\n" " --framethreads $num: number of frame threads (default: 1)\n" " --tilethreads $num: number of tile threads (default: 1)\n" " --filmgrain $num: enable film grain application (default: 1, except if muxer is md5)\n" - " --oppoint $num: select an operating point of a scalable AV1 bitstream (0 - 32)\n" + " --oppoint $num: select an operating point of a scalable AV1 bitstream (0 - 31)\n" " --alllayers $num: output all spatial layers of a scalable AV1 bitstream (default: 1)\n" " --sizelimit $num: stop decoding if the frame size exceeds the specified limit\n" " --verify $md5: verify decoded md5. implies --muxer md5, no output\n" " --cpumask $mask: restrict permitted CPU instruction sets (0" ALLOWED_CPU_MASKS "; default: -1)\n"); exit(1); @@ -185,10 +187,12 @@ #endif static const EnumParseTable cpu_mask_tbl[] = { #if ARCH_AARCH64 || ARCH_ARM { "neon", DAV1D_ARM_CPU_FLAG_NEON }, +#elif ARCH_PPC64LE + { "vsx", DAV1D_PPC_CPU_FLAG_VSX }, #elif ARCH_X86 { "sse2", X86_CPU_MASK_SSE2 }, { "ssse3", X86_CPU_MASK_SSSE3 }, { "sse41", X86_CPU_MASK_SSE41 }, { "avx2", X86_CPU_MASK_AVX2 }, diff --git a/third_party/dav1d/tools/input/input.c b/third_party/dav1d/tools/input/input.c --- a/third_party/dav1d/tools/input/input.c +++ b/third_party/dav1d/tools/input/input.c @@ -80,10 +80,14 @@ if (!probe_data) { fprintf(stderr, "Failed to allocate memory\n"); return DAV1D_ERR(ENOMEM); } FILE *f = fopen(filename, "rb"); + if (!f) { + fprintf(stderr, "Failed to open input file %s: %s\n", filename, strerror(errno)); + return errno ? DAV1D_ERR(errno) : DAV1D_ERR(EIO); + } res = !!fread(probe_data, 1, probe_sz, f); fclose(f); if (!res) { free(probe_data); fprintf(stderr, "Failed to read probe data\n"); diff --git a/third_party/dav1d/tools/input/ivf.c b/third_party/dav1d/tools/input/ivf.c --- a/third_party/dav1d/tools/input/ivf.c +++ b/third_party/dav1d/tools/input/ivf.c @@ -26,10 +26,11 @@ */ #include "config.h" #include +#include #include #include #include #include @@ -90,12 +91,31 @@ for (*num_frames = 0;; (*num_frames)++) { if ((res = fread(data, 4, 1, c->f)) != 1) break; // EOF fseeko(c->f, rl32(data) + 8, SEEK_CUR); } - fps[0] = timebase[0] * *num_frames; - fps[1] = timebase[1] * duration; + + uint64_t fps_num = (uint64_t) timebase[0] * *num_frames; + uint64_t fps_den = (uint64_t) timebase[1] * duration; + if (fps_num && fps_den) { /* Reduce fraction */ + uint64_t gcd = fps_num; + for (uint64_t a = fps_den, b; (b = a % gcd); a = gcd, gcd = b); + fps_num /= gcd; + fps_den /= gcd; + + while ((fps_num | fps_den) > UINT_MAX) { + fps_num >>= 1; + fps_den >>= 1; + } + } + if (fps_num && fps_den) { + fps[0] = (unsigned) fps_num; + fps[1] = (unsigned) fps_den; + } else { + fps[0] = fps[1] = 0; + } + fseeko(c->f, 32, SEEK_SET); return 0; } diff --git a/third_party/dav1d/tools/input/parse.h b/third_party/dav1d/tools/input/parse.h --- a/third_party/dav1d/tools/input/parse.h +++ b/third_party/dav1d/tools/input/parse.h @@ -27,45 +27,47 @@ */ #ifndef DAV1D_INPUT_PARSE_H #define DAV1D_INPUT_PARSE_H +#include + #include "dav1d/headers.h" static int leb128(FILE *const f, size_t *const len) { + uint64_t val = 0; unsigned i = 0, more; - *len = 0; do { - uint8_t byte; - if (fread(&byte, 1, 1, f) < 1) + uint8_t v; + if (fread(&v, 1, 1, f) < 1) return -1; - more = byte & 0x80; - const unsigned bits = byte & 0x7f; - if (i <= 3 || (i == 4 && bits < (1 << 4))) - *len |= bits << (i * 7); - else if (bits) return -1; - if (++i == 8 && more) return -1; - } while (more); + more = v & 0x80; + val |= ((uint64_t) (v & 0x7F)) << (i * 7); + i++; + } while (more && i < 8); + if (val > UINT_MAX || more) + return -1; + *len = (size_t) val; return i; } // these functions are based on an implementation from FFmpeg, and relicensed // with author's permission static int leb(const uint8_t *ptr, int sz, size_t *const len) { + uint64_t val = 0; unsigned i = 0, more; - *len = 0; do { if (!sz--) return -1; - const int byte = *ptr++; - more = byte & 0x80; - const unsigned bits = byte & 0x7f; - if (i <= 3 || (i == 4 && bits < (1 << 4))) - *len |= bits << (i * 7); - else if (bits) return -1; - if (++i == 8 && more) return -1; - } while (more); + const int v = *ptr++; + more = v & 0x80; + val |= ((uint64_t) (v & 0x7F)) << (i * 7); + i++; + } while (more && i < 8); + if (val > UINT_MAX || more) + return -1; + *len = (size_t) val; return i; } static inline int parse_obu_header(const uint8_t *buf, int buf_size, size_t *const obu_size,